From efab707fbc3fb72dfe2caae54323ea6288054b08 Mon Sep 17 00:00:00 2001 From: Kozlov Ivan Date: Tue, 5 Feb 2019 17:55:30 +0300 Subject: [PATCH 001/738] query for reproduce bug --- .../bugs/leak_when_memory_limit_exceeded.sql | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 dbms/tests/queries/bugs/leak_when_memory_limit_exceeded.sql diff --git a/dbms/tests/queries/bugs/leak_when_memory_limit_exceeded.sql b/dbms/tests/queries/bugs/leak_when_memory_limit_exceeded.sql new file mode 100644 index 00000000000..68811ad061e --- /dev/null +++ b/dbms/tests/queries/bugs/leak_when_memory_limit_exceeded.sql @@ -0,0 +1,22 @@ +-- max_memory_usage = 10000000000 (10 GB default) +-- Intel® Xeon® E5-1650 v3 Hexadcore 128 GB DDR4 ECC +-- Estimated time: ~ 250 seconds +-- Read rows: ~ 272 000 000 +SELECT + key, + uniqState(uuid_1) uuid_1_st, + uniqState(uuid_2) uuid_2_st, + uniqState(uuid_3) uuid_3_st +FROM ( + SELECT + rand64() value, + toString(value) value_str, + UUIDNumToString(toFixedString(substring(value_str, 1, 16), 16)) uuid_1, -- Any UUID + UUIDNumToString(toFixedString(substring(value_str, 2, 16), 16)) uuid_2, -- More memory + UUIDNumToString(toFixedString(substring(value_str, 3, 16), 16)) uuid_3, -- And more memory + modulo(value, 5000000) key -- Cardinality in my case + FROM numbers(550000000) +) +GROUP BY + key +LIMIT 100; From c22f91db016889940923c708a330a926440d0a78 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 15 Apr 2020 04:12:32 +0300 Subject: [PATCH 002/738] Allow to pass quota_key in clickhouse-client --- programs/client/Client.cpp | 10 +++++++--- programs/client/ConnectionParameters.cpp | 5 +++++ programs/client/ConnectionParameters.h | 2 +- programs/server/TCPHandler.cpp | 12 ++++++++++-- src/Client/Connection.cpp | 14 +++++++++++--- src/Client/Connection.h | 6 ++++++ src/Core/Defines.h | 3 +++ 7 files changed, 43 insertions(+), 9 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index e01eef98006..e8a473b0be9 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -589,6 +589,9 @@ private: connection_parameters.compression, connection_parameters.security); + if (!connection_parameters.quota_key.empty()) + connection->setQuotaKey(connection_parameters.quota_key); + String server_name; UInt64 server_version_major = 0; UInt64 server_version_minor = 0; @@ -605,9 +608,7 @@ private: server_version = toString(server_version_major) + "." + toString(server_version_minor) + "." + toString(server_version_patch); - if ( - server_display_name = connection->getServerDisplayName(connection_parameters.timeouts); - server_display_name.length() == 0) + if (server_display_name = connection->getServerDisplayName(connection_parameters.timeouts); server_display_name.empty()) { server_display_name = config().getString("host", "localhost"); } @@ -1705,6 +1706,7 @@ public: */ ("password", po::value()->implicit_value("\n", ""), "password") ("ask-password", "ask-password") + ("quota_key", po::value(), "A string to differentiate quotas when the user have keyed quotas configured on server") ("query_id", po::value(), "query_id") ("query,q", po::value(), "query") ("database,d", po::value(), "database") @@ -1840,6 +1842,8 @@ public: config().setString("password", options["password"].as()); if (options.count("ask-password")) config().setBool("ask-password", true); + if (options.count("quota_key")) + config().setString("quota_key", options["quota_key"].as()); if (options.count("multiline")) config().setBool("multiline", true); if (options.count("multiquery")) diff --git a/programs/client/ConnectionParameters.cpp b/programs/client/ConnectionParameters.cpp index e1611af249d..e4c30306e75 100644 --- a/programs/client/ConnectionParameters.cpp +++ b/programs/client/ConnectionParameters.cpp @@ -29,8 +29,10 @@ ConnectionParameters::ConnectionParameters(const Poco::Util::AbstractConfigurati "port", config.getInt(is_secure ? "tcp_port_secure" : "tcp_port", is_secure ? DBMS_DEFAULT_SECURE_PORT : DBMS_DEFAULT_PORT)); default_database = config.getString("database", ""); + /// changed the default value to "default" to fix the issue when the user in the prompt is blank user = config.getString("user", "default"); + bool password_prompt = false; if (config.getBool("ask-password", false)) { @@ -52,6 +54,9 @@ ConnectionParameters::ConnectionParameters(const Poco::Util::AbstractConfigurati if (auto result = readpassphrase(prompt.c_str(), buf, sizeof(buf), 0)) password = result; } + + quota_key = config.getString("quota_key", ""); + compression = config.getBool("compression", true) ? Protocol::Compression::Enable : Protocol::Compression::Disable; timeouts = ConnectionTimeouts( diff --git a/programs/client/ConnectionParameters.h b/programs/client/ConnectionParameters.h index 834f08df9cb..1d3bf41e6f6 100644 --- a/programs/client/ConnectionParameters.h +++ b/programs/client/ConnectionParameters.h @@ -18,12 +18,12 @@ struct ConnectionParameters std::string default_database; std::string user; std::string password; + std::string quota_key; Protocol::Secure security = Protocol::Secure::Disable; Protocol::Compression compression = Protocol::Compression::Enable; ConnectionTimeouts timeouts; ConnectionParameters() {} - ConnectionParameters(const Poco::Util::AbstractConfiguration & config); }; diff --git a/programs/server/TCPHandler.cpp b/programs/server/TCPHandler.cpp index d82c6e31528..b0233c32d6e 100644 --- a/programs/server/TCPHandler.cpp +++ b/programs/server/TCPHandler.cpp @@ -743,6 +743,7 @@ void TCPHandler::receiveHello() UInt64 packet_type = 0; String user = "default"; String password; + String quota_key; readVarUInt(packet_type, *in); if (packet_type != Protocol::Client::Hello) @@ -772,6 +773,9 @@ void TCPHandler::receiveHello() readStringBinary(user, *in); readStringBinary(password, *in); + if (client_revision >= DBMS_MIN_REVISION_WITH_CLIENT_PROVIDED_QUOTA_KEY) + readStringBinary(quota_key, *in); + LOG_DEBUG(log, "Connected " << client_name << " version " << client_version_major << "." << client_version_minor @@ -779,24 +783,28 @@ void TCPHandler::receiveHello() << ", revision: " << client_revision << (!default_database.empty() ? ", database: " + default_database : "") << (!user.empty() ? ", user: " + user : "") + << (!quota_key.empty() ? ", quota_key: " + quota_key : "") /// quota key is not secret info. << "."); - connection_context.setUser(user, password, socket().peerAddress(), ""); + connection_context.setUser(user, password, socket().peerAddress(), quota_key); } void TCPHandler::receiveUnexpectedHello() { UInt64 skip_uint_64; + UInt64 client_revision; String skip_string; readStringBinary(skip_string, *in); readVarUInt(skip_uint_64, *in); readVarUInt(skip_uint_64, *in); - readVarUInt(skip_uint_64, *in); + readVarUInt(client_revision, *in); readStringBinary(skip_string, *in); readStringBinary(skip_string, *in); readStringBinary(skip_string, *in); + if (client_revision >= DBMS_MIN_REVISION_WITH_CLIENT_PROVIDED_QUOTA_KEY) + readStringBinary(skip_string, *in); throw NetException("Unexpected packet Hello received from client", ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT); } diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index f530652caae..5c7b306432d 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -58,8 +58,11 @@ void Connection::connect(const ConnectionTimeouts & timeouts) if (connected) disconnect(); - LOG_TRACE(log_wrapper.get(), "Connecting. Database: " << (default_database.empty() ? "(not specified)" : default_database) << ". User: " << user - << (static_cast(secure) ? ". Secure" : "") << (static_cast(compression) ? "" : ". Uncompressed")); + LOG_TRACE(log_wrapper.get(), "Connecting. Database: " + << (default_database.empty() ? "(not specified)" : default_database) + << ". User: " << user + << (static_cast(secure) ? ". Secure" : "") + << (static_cast(compression) ? "" : ". Uncompressed")); if (static_cast(secure)) { @@ -162,16 +165,21 @@ void Connection::sendHello() || has_control_character(password)) throw Exception("Parameters 'default_database', 'user' and 'password' must not contain ASCII control characters", ErrorCodes::BAD_ARGUMENTS); + auto client_revision = ClickHouseRevision::get(); + writeVarUInt(Protocol::Client::Hello, *out); writeStringBinary((DBMS_NAME " ") + client_name, *out); writeVarUInt(DBMS_VERSION_MAJOR, *out); writeVarUInt(DBMS_VERSION_MINOR, *out); // NOTE For backward compatibility of the protocol, client cannot send its version_patch. - writeVarUInt(ClickHouseRevision::get(), *out); + writeVarUInt(client_revision, *out); writeStringBinary(default_database, *out); writeStringBinary(user, *out); writeStringBinary(password, *out); + if (client_revision >= DBMS_MIN_REVISION_WITH_CLIENT_PROVIDED_QUOTA_KEY) + writeStringBinary(quota_key, *out); + out->next(); } diff --git a/src/Client/Connection.h b/src/Client/Connection.h index de04e3f0ef4..88790e753bd 100644 --- a/src/Client/Connection.h +++ b/src/Client/Connection.h @@ -104,6 +104,11 @@ public: virtual ~Connection() {} + void setQuotaKey(String quota_key_) + { + quota_key = std::move(quota_key_); + } + /// Set throttler of network traffic. One throttler could be used for multiple connections to limit total traffic. void setThrottler(const ThrottlerPtr & throttler_) { @@ -186,6 +191,7 @@ private: String default_database; String user; String password; + String quota_key; /// Address is resolved during the first connection (or the following reconnects) /// Use it only for logging purposes diff --git a/src/Core/Defines.h b/src/Core/Defines.h index c797f527be9..8a3bf73513c 100644 --- a/src/Core/Defines.h +++ b/src/Core/Defines.h @@ -67,6 +67,9 @@ /// Mininum revision supporting SettingsBinaryFormat::STRINGS. #define DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS 54429 +/// Minimum revision when client sends quota key. +#define DBMS_MIN_REVISION_WITH_CLIENT_PROVIDED_QUOTA_KEY 54435 + /// Version of ClickHouse TCP protocol. Set to git tag with latest protocol change. #define DBMS_TCP_PROTOCOL_VERSION 54226 From fe595244816c9b6f5fd522b81e3dfa6a40821e32 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 15 Apr 2020 04:58:10 +0300 Subject: [PATCH 003/738] Implemented in a different way --- programs/client/Client.cpp | 19 ++++++++++++++----- programs/client/ConnectionParameters.cpp | 2 -- programs/client/ConnectionParameters.h | 1 - programs/local/LocalServer.cpp | 2 +- programs/server/HTTPHandler.cpp | 4 +++- programs/server/TCPHandler.cpp | 12 ++---------- src/Client/Connection.cpp | 7 +------ src/Client/Connection.h | 6 ------ src/Core/Defines.h | 3 --- src/Core/MySQLProtocol.h | 6 +++--- .../ClickHouseDictionarySource.cpp | 2 +- src/Interpreters/ClientInfo.cpp | 8 ++++++++ src/Interpreters/ClientInfo.h | 4 ++++ src/Interpreters/Context.cpp | 10 +++++++--- src/Interpreters/Context.h | 3 ++- src/Interpreters/executeQuery.cpp | 2 ++ 16 files changed, 48 insertions(+), 43 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index e8a473b0be9..215e0bf15e9 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -397,6 +397,10 @@ private: ignore_error = config().getBool("ignore-error", false); } + ClientInfo & client_info = context.getClientInfo(); + client_info.setInitialQuery(); + client_info.quota_key = config().getString("quota_key", ""); + connect(); /// Initialize DateLUT here to avoid counting time spent here as query execution time. @@ -589,9 +593,6 @@ private: connection_parameters.compression, connection_parameters.security); - if (!connection_parameters.quota_key.empty()) - connection->setQuotaKey(connection_parameters.quota_key); - String server_name; UInt64 server_version_major = 0; UInt64 server_version_minor = 0; @@ -906,7 +907,7 @@ private: query_id, QueryProcessingStage::Complete, &context.getSettingsRef(), - nullptr, + &context.getClientInfo(), true); sendExternalTables(); @@ -938,7 +939,15 @@ private: if (!parsed_insert_query.data && (is_interactive || (!stdin_is_a_tty && std_in.eof()))) throw Exception("No data to insert", ErrorCodes::NO_DATA_TO_INSERT); - connection->sendQuery(connection_parameters.timeouts, query_without_data, query_id, QueryProcessingStage::Complete, &context.getSettingsRef(), nullptr, true); + connection->sendQuery( + connection_parameters.timeouts, + query_without_data, + query_id, + QueryProcessingStage::Complete, + &context.getSettingsRef(), + &context.getClientInfo(), + true); + sendExternalTables(); /// Receive description of table structure. diff --git a/programs/client/ConnectionParameters.cpp b/programs/client/ConnectionParameters.cpp index e4c30306e75..50cac3b7800 100644 --- a/programs/client/ConnectionParameters.cpp +++ b/programs/client/ConnectionParameters.cpp @@ -55,8 +55,6 @@ ConnectionParameters::ConnectionParameters(const Poco::Util::AbstractConfigurati password = result; } - quota_key = config.getString("quota_key", ""); - compression = config.getBool("compression", true) ? Protocol::Compression::Enable : Protocol::Compression::Disable; timeouts = ConnectionTimeouts( diff --git a/programs/client/ConnectionParameters.h b/programs/client/ConnectionParameters.h index 1d3bf41e6f6..a169df8390a 100644 --- a/programs/client/ConnectionParameters.h +++ b/programs/client/ConnectionParameters.h @@ -18,7 +18,6 @@ struct ConnectionParameters std::string default_database; std::string user; std::string password; - std::string quota_key; Protocol::Secure security = Protocol::Secure::Disable; Protocol::Compression compression = Protocol::Compression::Enable; ConnectionTimeouts timeouts; diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 26752da5d87..3c67bd01f4c 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -276,7 +276,7 @@ void LocalServer::processQueries() context->makeSessionContext(); context->makeQueryContext(); - context->setUser("default", "", Poco::Net::SocketAddress{}, ""); + context->setUser("default", "", Poco::Net::SocketAddress{}); context->setCurrentQueryId(""); applyCmdSettings(); diff --git a/programs/server/HTTPHandler.cpp b/programs/server/HTTPHandler.cpp index ec890c0a96d..c535cf5679a 100644 --- a/programs/server/HTTPHandler.cpp +++ b/programs/server/HTTPHandler.cpp @@ -267,8 +267,10 @@ void HTTPHandler::processQuery( } std::string query_id = params.get("query_id", ""); - context.setUser(user, password, request.clientAddress(), quota_key); + context.setUser(user, password, request.clientAddress()); context.setCurrentQueryId(query_id); + if (!quota_key.empty()) + context.setQuotaKey(quota_key); /// The user could specify session identifier and session timeout. /// It allows to modify settings, create temporary tables and reuse them in subsequent requests. diff --git a/programs/server/TCPHandler.cpp b/programs/server/TCPHandler.cpp index b0233c32d6e..c69319917d1 100644 --- a/programs/server/TCPHandler.cpp +++ b/programs/server/TCPHandler.cpp @@ -743,7 +743,6 @@ void TCPHandler::receiveHello() UInt64 packet_type = 0; String user = "default"; String password; - String quota_key; readVarUInt(packet_type, *in); if (packet_type != Protocol::Client::Hello) @@ -773,9 +772,6 @@ void TCPHandler::receiveHello() readStringBinary(user, *in); readStringBinary(password, *in); - if (client_revision >= DBMS_MIN_REVISION_WITH_CLIENT_PROVIDED_QUOTA_KEY) - readStringBinary(quota_key, *in); - LOG_DEBUG(log, "Connected " << client_name << " version " << client_version_major << "." << client_version_minor @@ -783,28 +779,24 @@ void TCPHandler::receiveHello() << ", revision: " << client_revision << (!default_database.empty() ? ", database: " + default_database : "") << (!user.empty() ? ", user: " + user : "") - << (!quota_key.empty() ? ", quota_key: " + quota_key : "") /// quota key is not secret info. << "."); - connection_context.setUser(user, password, socket().peerAddress(), quota_key); + connection_context.setUser(user, password, socket().peerAddress()); } void TCPHandler::receiveUnexpectedHello() { UInt64 skip_uint_64; - UInt64 client_revision; String skip_string; readStringBinary(skip_string, *in); readVarUInt(skip_uint_64, *in); readVarUInt(skip_uint_64, *in); - readVarUInt(client_revision, *in); + readVarUInt(skip_uint_64, *in); readStringBinary(skip_string, *in); readStringBinary(skip_string, *in); readStringBinary(skip_string, *in); - if (client_revision >= DBMS_MIN_REVISION_WITH_CLIENT_PROVIDED_QUOTA_KEY) - readStringBinary(skip_string, *in); throw NetException("Unexpected packet Hello received from client", ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT); } diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index 5c7b306432d..c90918d040a 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -177,9 +177,6 @@ void Connection::sendHello() writeStringBinary(user, *out); writeStringBinary(password, *out); - if (client_revision >= DBMS_MIN_REVISION_WITH_CLIENT_PROVIDED_QUOTA_KEY) - writeStringBinary(quota_key, *out); - out->next(); } @@ -404,9 +401,7 @@ void Connection::sendQuery( if (!client_info || client_info->empty()) { /// No client info passed - means this query initiated by me. - client_info_to_send.query_kind = ClientInfo::QueryKind::INITIAL_QUERY; - client_info_to_send.fillOSUserHostNameAndVersionInfo(); - client_info_to_send.client_name = (DBMS_NAME " ") + client_name; + client_info_to_send.setInitialQuery(); } else { diff --git a/src/Client/Connection.h b/src/Client/Connection.h index 88790e753bd..de04e3f0ef4 100644 --- a/src/Client/Connection.h +++ b/src/Client/Connection.h @@ -104,11 +104,6 @@ public: virtual ~Connection() {} - void setQuotaKey(String quota_key_) - { - quota_key = std::move(quota_key_); - } - /// Set throttler of network traffic. One throttler could be used for multiple connections to limit total traffic. void setThrottler(const ThrottlerPtr & throttler_) { @@ -191,7 +186,6 @@ private: String default_database; String user; String password; - String quota_key; /// Address is resolved during the first connection (or the following reconnects) /// Use it only for logging purposes diff --git a/src/Core/Defines.h b/src/Core/Defines.h index 8a3bf73513c..c797f527be9 100644 --- a/src/Core/Defines.h +++ b/src/Core/Defines.h @@ -67,9 +67,6 @@ /// Mininum revision supporting SettingsBinaryFormat::STRINGS. #define DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS 54429 -/// Minimum revision when client sends quota key. -#define DBMS_MIN_REVISION_WITH_CLIENT_PROVIDED_QUOTA_KEY 54435 - /// Version of ClickHouse TCP protocol. Set to git tag with latest protocol change. #define DBMS_TCP_PROTOCOL_VERSION 54226 diff --git a/src/Core/MySQLProtocol.h b/src/Core/MySQLProtocol.h index 5255c6f263e..a54bf371999 100644 --- a/src/Core/MySQLProtocol.h +++ b/src/Core/MySQLProtocol.h @@ -955,7 +955,7 @@ public: if (auth_response->empty()) { - context.setUser(user_name, "", address, ""); + context.setUser(user_name, "", address); return; } @@ -978,7 +978,7 @@ public: { password_sha1[i] = digest[i] ^ static_cast((*auth_response)[i]); } - context.setUser(user_name, password_sha1, address, ""); + context.setUser(user_name, password_sha1, address); } private: String scramble; @@ -1120,7 +1120,7 @@ public: password.pop_back(); } - context.setUser(user_name, password, address, ""); + context.setUser(user_name, password, address); } private: diff --git a/src/Dictionaries/ClickHouseDictionarySource.cpp b/src/Dictionaries/ClickHouseDictionarySource.cpp index ad08754e4e7..5e994e57002 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.cpp +++ b/src/Dictionaries/ClickHouseDictionarySource.cpp @@ -73,7 +73,7 @@ ClickHouseDictionarySource::ClickHouseDictionarySource( , load_all_query{query_builder.composeLoadAllQuery()} { /// We should set user info even for the case when the dictionary is loaded in-process (without TCP communication). - context.setUser(user, password, Poco::Net::SocketAddress("127.0.0.1", 0), {}); + context.setUser(user, password, Poco::Net::SocketAddress("127.0.0.1", 0)); /// Processors are not supported here yet. context.setSetting("experimental_use_processors", false); /// Query context is needed because some code in executeQuery function may assume it exists. diff --git a/src/Interpreters/ClientInfo.cpp b/src/Interpreters/ClientInfo.cpp index ed806e5ad57..14e6540a1f1 100644 --- a/src/Interpreters/ClientInfo.cpp +++ b/src/Interpreters/ClientInfo.cpp @@ -113,6 +113,14 @@ void ClientInfo::read(ReadBuffer & in, const UInt64 client_protocol_revision) } +void ClientInfo::setInitialQuery() +{ + query_kind = QueryKind::INITIAL_QUERY; + fillOSUserHostNameAndVersionInfo(); + client_name = (DBMS_NAME " ") + client_name; +} + + void ClientInfo::fillOSUserHostNameAndVersionInfo() { os_user.resize(256, '\0'); diff --git a/src/Interpreters/ClientInfo.h b/src/Interpreters/ClientInfo.h index 492cc299a06..7a4df63c17a 100644 --- a/src/Interpreters/ClientInfo.h +++ b/src/Interpreters/ClientInfo.h @@ -84,6 +84,10 @@ public: void write(WriteBuffer & out, const UInt64 server_protocol_revision) const; void read(ReadBuffer & in, const UInt64 client_protocol_revision); + /// Initialize parameters on client initiating query. + void setInitialQuery(); + +private: void fillOSUserHostNameAndVersionInfo(); }; diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 6e30792277f..b6fc202a008 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -620,15 +620,13 @@ ConfigurationPtr Context::getUsersConfig() } -void Context::setUser(const String & name, const String & password, const Poco::Net::SocketAddress & address, const String & quota_key) +void Context::setUser(const String & name, const String & password, const Poco::Net::SocketAddress & address) { auto lock = getLock(); client_info.current_user = name; client_info.current_password = password; client_info.current_address = address; - if (!quota_key.empty()) - client_info.quota_key = quota_key; auto new_user_id = getAccessControlManager().find(name); std::shared_ptr new_access; @@ -659,6 +657,12 @@ std::shared_ptr Context::getUser() const return access->getUser(); } +void Context::setQuotaKey(String quota_key_) +{ + auto lock = getLock(); + client_info.quota_key = std::move(quota_key_); +} + String Context::getUserName() const { auto lock = getLock(); diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index f68b08bf9f0..83edca86c1e 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -228,7 +228,8 @@ public: /// Sets the current user, checks the password and that the specified host is allowed. /// Must be called before getClientInfo. - void setUser(const String & name, const String & password, const Poco::Net::SocketAddress & address, const String & quota_key); + void setUser(const String & name, const String & password, const Poco::Net::SocketAddress & address); + void setQuotaKey(String quota_key_); UserPtr getUser() const; String getUserName() const; diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 68bebb83619..c97a011f382 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -409,6 +409,8 @@ static std::tuple executeQueryImpl( elem.client_info = context.getClientInfo(); + std::cerr << "Quota key: " << elem.client_info.quota_key << "\n"; + bool log_queries = settings.log_queries && !internal; /// Log into system table start of query execution, if need. From 2491d59938cc264972082a389caea87452e4179c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 15 Apr 2020 05:03:54 +0300 Subject: [PATCH 004/738] Added a test --- tests/queries/0_stateless/01198_client_quota_key.reference | 2 ++ tests/queries/0_stateless/01198_client_quota_key.sh | 6 ++++++ 2 files changed, 8 insertions(+) create mode 100644 tests/queries/0_stateless/01198_client_quota_key.reference create mode 100755 tests/queries/0_stateless/01198_client_quota_key.sh diff --git a/tests/queries/0_stateless/01198_client_quota_key.reference b/tests/queries/0_stateless/01198_client_quota_key.reference new file mode 100644 index 00000000000..c25611c15c6 --- /dev/null +++ b/tests/queries/0_stateless/01198_client_quota_key.reference @@ -0,0 +1,2 @@ +1 +Hello diff --git a/tests/queries/0_stateless/01198_client_quota_key.sh b/tests/queries/0_stateless/01198_client_quota_key.sh new file mode 100755 index 00000000000..f4b66aea6ac --- /dev/null +++ b/tests/queries/0_stateless/01198_client_quota_key.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +$CLICKHOUSE_CLIENT --quota_key Hello --query_id test_quota_key --log_queries 1 --multiquery --query "SELECT 1; SYSTEM FLUSH LOGS; SELECT DISTINCT quota_key FROM system.query_log WHERE event_date >= yesterday() AND event_time >= now() - 300 AND query_id = 'test_quota_key'" From ff817ae7355dcd9d854a17f605c0473e03892d04 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 15 Apr 2020 05:04:26 +0300 Subject: [PATCH 005/738] Remove debug output --- src/Interpreters/executeQuery.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index c97a011f382..68bebb83619 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -409,8 +409,6 @@ static std::tuple executeQueryImpl( elem.client_info = context.getClientInfo(); - std::cerr << "Quota key: " << elem.client_info.quota_key << "\n"; - bool log_queries = settings.log_queries && !internal; /// Log into system table start of query execution, if need. From 79af5cd0ae4dd22e58be8397374abe0661618779 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 26 Apr 2020 18:48:30 +0300 Subject: [PATCH 006/738] Fix UBSan report in Decimal parse --- src/DataTypes/DataTypesDecimal.cpp | 6 +++--- .../queries/0_stateless/01260_ubsan_decimal_parse.reference | 1 + tests/queries/0_stateless/01260_ubsan_decimal_parse.sql | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/01260_ubsan_decimal_parse.reference create mode 100644 tests/queries/0_stateless/01260_ubsan_decimal_parse.sql diff --git a/src/DataTypes/DataTypesDecimal.cpp b/src/DataTypes/DataTypesDecimal.cpp index cbc38429183..b45e26bd5bb 100644 --- a/src/DataTypes/DataTypesDecimal.cpp +++ b/src/DataTypes/DataTypesDecimal.cpp @@ -58,7 +58,7 @@ void DataTypeDecimal::serializeText(const IColumn & column, size_t row_num, W } template -bool DataTypeDecimal::tryReadText(T & x, ReadBuffer & istr, UInt32 precision, UInt32 scale) +bool NO_SANITIZE_UNDEFINED DataTypeDecimal::tryReadText(T & x, ReadBuffer & istr, UInt32 precision, UInt32 scale) { UInt32 unread_scale = scale; bool done = tryReadDecimalText(istr, x, precision, unread_scale); @@ -68,7 +68,7 @@ bool DataTypeDecimal::tryReadText(T & x, ReadBuffer & istr, UInt32 precision, } template -void DataTypeDecimal::readText(T & x, ReadBuffer & istr, UInt32 precision, UInt32 scale, bool csv) +void NO_SANITIZE_UNDEFINED DataTypeDecimal::readText(T & x, ReadBuffer & istr, UInt32 precision, UInt32 scale, bool csv) { UInt32 unread_scale = scale; if (csv) @@ -95,7 +95,7 @@ void DataTypeDecimal::deserializeTextCSV(IColumn & column, ReadBuffer & istr, } template -T DataTypeDecimal::parseFromString(const String & str) const +T NO_SANITIZE_UNDEFINED DataTypeDecimal::parseFromString(const String & str) const { ReadBufferFromMemory buf(str.data(), str.size()); T x; diff --git a/tests/queries/0_stateless/01260_ubsan_decimal_parse.reference b/tests/queries/0_stateless/01260_ubsan_decimal_parse.reference new file mode 100644 index 00000000000..573541ac970 --- /dev/null +++ b/tests/queries/0_stateless/01260_ubsan_decimal_parse.reference @@ -0,0 +1 @@ +0 diff --git a/tests/queries/0_stateless/01260_ubsan_decimal_parse.sql b/tests/queries/0_stateless/01260_ubsan_decimal_parse.sql new file mode 100644 index 00000000000..faf98952a17 --- /dev/null +++ b/tests/queries/0_stateless/01260_ubsan_decimal_parse.sql @@ -0,0 +1 @@ +SELECT ignore(toDecimal32OrZero(CAST(-7174046, 'String'), 6)); From 20b4eed9a1eed9db7bdc2ba6a3e9c655ada23b8c Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 27 Apr 2020 00:56:54 +0300 Subject: [PATCH 007/738] Disable GROUP BY sharding_key optimization for WITH ROLLUP/CUBE/TOTALS --- src/Storages/StorageDistributed.cpp | 14 ++++++++----- ..._GROUP_BY_injective_sharding_key.reference | 21 +++++++++++++++++++ ..._merge_GROUP_BY_injective_sharding_key.sql | 11 ++++++++++ 3 files changed, 41 insertions(+), 5 deletions(-) diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 9c557ad5c8a..72c364905db 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -399,8 +399,17 @@ bool StorageDistributed::canForceGroupByNoMerge(const Context &context, QueryPro const auto & select = query_ptr->as(); + if (select.group_by_with_totals || select.group_by_with_rollup || select.group_by_with_cube) + return false; + + // TODO: The following can be optimized too (but with some caveats, will be addressed later): + // - ORDER BY + // - LIMIT BY + // - LIMIT if (select.orderBy()) return false; + if (select.limitBy() || select.limitLength()) + return false; if (select.distinct) { @@ -416,11 +425,6 @@ bool StorageDistributed::canForceGroupByNoMerge(const Context &context, QueryPro reason = "DISTINCT " + backQuote(serializeAST(*select.select(), true)); } - // This can use distributed_group_by_no_merge but in this case limit stage - // should be done later (which is not the case right now). - if (select.limitBy() || select.limitLength()) - return false; - const ASTPtr group_by = select.groupBy(); if (!group_by) { diff --git a/tests/queries/0_stateless/01247_distributed_group_by_no_merge_GROUP_BY_injective_sharding_key.reference b/tests/queries/0_stateless/01247_distributed_group_by_no_merge_GROUP_BY_injective_sharding_key.reference index b8cf0042ed3..c299e3be7a0 100644 --- a/tests/queries/0_stateless/01247_distributed_group_by_no_merge_GROUP_BY_injective_sharding_key.reference +++ b/tests/queries/0_stateless/01247_distributed_group_by_no_merge_GROUP_BY_injective_sharding_key.reference @@ -67,3 +67,24 @@ GROUP BY (Distributed-over-Distributed) distributed_group_by_no_merge 1 1 1 0 1 1 +extremes +1 0 +1 1 +1 0 +1 1 + +1 0 +1 1 +WITH TOTALS +2 0 +2 1 + +4 0 +WITH ROLLUP +2 0 +2 1 +4 0 +WITH CUBE +2 0 +2 1 +4 0 diff --git a/tests/queries/0_stateless/01247_distributed_group_by_no_merge_GROUP_BY_injective_sharding_key.sql b/tests/queries/0_stateless/01247_distributed_group_by_no_merge_GROUP_BY_injective_sharding_key.sql index 56345e23094..2a4f0e0601d 100644 --- a/tests/queries/0_stateless/01247_distributed_group_by_no_merge_GROUP_BY_injective_sharding_key.sql +++ b/tests/queries/0_stateless/01247_distributed_group_by_no_merge_GROUP_BY_injective_sharding_key.sql @@ -1,3 +1,5 @@ +-- TODO: correct testing with real unique shards + drop table if exists dist_01247; drop table if exists data_01247; @@ -60,3 +62,12 @@ select 'GROUP BY (Distributed-over-Distributed)'; select count(), * from cluster(test_cluster_two_shards, currentDatabase(), dist_01247) group by number; select 'GROUP BY (Distributed-over-Distributed) distributed_group_by_no_merge'; select count(), * from cluster(test_cluster_two_shards, currentDatabase(), dist_01247) group by number settings distributed_group_by_no_merge=1; + +select 'extremes'; +select count(), * from dist_01247 group by number settings extremes=1; +select 'WITH TOTALS'; +select count(), * from dist_01247 group by number with totals; +select 'WITH ROLLUP'; +select count(), * from dist_01247 group by number with rollup; +select 'WITH CUBE'; +select count(), * from dist_01247 group by number with cube; From 4e09d812eff4e7d25385073c9affe659e23a684d Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 27 Apr 2020 00:59:56 +0300 Subject: [PATCH 008/738] Add a test for GROUP BY sharding_key optimization over Distributed-on-Distributed --- ...oup_by_sharding_key_optimization.reference | 17 ++++++++++ ...ist_group_by_sharding_key_optimization.sql | 34 +++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 tests/queries/0_stateless/01247_dist_on_dist_group_by_sharding_key_optimization.reference create mode 100644 tests/queries/0_stateless/01247_dist_on_dist_group_by_sharding_key_optimization.sql diff --git a/tests/queries/0_stateless/01247_dist_on_dist_group_by_sharding_key_optimization.reference b/tests/queries/0_stateless/01247_dist_on_dist_group_by_sharding_key_optimization.reference new file mode 100644 index 00000000000..9fc7f7bfcd7 --- /dev/null +++ b/tests/queries/0_stateless/01247_dist_on_dist_group_by_sharding_key_optimization.reference @@ -0,0 +1,17 @@ +Distributed(number)-over-Distributed(number) +1 0 +1 1 +1 0 +1 1 +1 0 +1 1 +1 0 +1 1 +Distributed(rand)-over-Distributed(number) +4 0 +4 1 +Distributed(rand)-over-Distributed(rand) +2 0 +2 1 +2 0 +2 1 diff --git a/tests/queries/0_stateless/01247_dist_on_dist_group_by_sharding_key_optimization.sql b/tests/queries/0_stateless/01247_dist_on_dist_group_by_sharding_key_optimization.sql new file mode 100644 index 00000000000..dff47118c9c --- /dev/null +++ b/tests/queries/0_stateless/01247_dist_on_dist_group_by_sharding_key_optimization.sql @@ -0,0 +1,34 @@ +-- TODO: correct testing with real unique shards + +drop table if exists dist_01247; +drop table if exists dist_layer_01247; +drop table if exists data_01247; + +create table data_01247 as system.numbers engine=Memory(); +-- since data is not inserted via distributed it will have duplicates +-- (and this is how we ensure that this optimization will work) +insert into data_01247 select * from system.numbers limit 2; + +set max_distributed_connections=1; +set optimize_skip_unused_shards=1; + +select 'Distributed(number)-over-Distributed(number)'; +create table dist_layer_01247 as data_01247 engine=Distributed(test_cluster_two_shards, currentDatabase(), data_01247, number); +create table dist_01247 as data_01247 engine=Distributed(test_cluster_two_shards, currentDatabase(), dist_layer_01247, number); +select count(), * from dist_01247 group by number; +drop table if exists dist_01247; +drop table if exists dist_layer_01247; + +select 'Distributed(rand)-over-Distributed(number)'; +create table dist_layer_01247 as data_01247 engine=Distributed(test_cluster_two_shards, currentDatabase(), data_01247, number); +create table dist_01247 as data_01247 engine=Distributed(test_cluster_two_shards, currentDatabase(), dist_layer_01247, rand()); +select count(), * from dist_01247 group by number; +drop table if exists dist_01247; +drop table if exists dist_layer_01247; + +select 'Distributed(rand)-over-Distributed(rand)'; +create table dist_layer_01247 as data_01247 engine=Distributed(test_cluster_two_shards, currentDatabase(), data_01247, rand()); +create table dist_01247 as data_01247 engine=Distributed(test_cluster_two_shards, currentDatabase(), dist_layer_01247, number); +select count(), * from dist_01247 group by number; +drop table if exists dist_01247; +drop table if exists dist_layer_01247; From 1bcd077fbf1e049b0af4212b7fda7cd2c9247f0f Mon Sep 17 00:00:00 2001 From: bobrovskij artemij Date: Tue, 28 Apr 2020 03:56:44 +0300 Subject: [PATCH 009/738] Support write for XDBCStorage --- programs/odbc-bridge/CMakeLists.txt | 1 + programs/odbc-bridge/HandlerFactory.cpp | 4 +- programs/odbc-bridge/MainHandler.cpp | 133 ++++++++++++------ programs/odbc-bridge/MainHandler.h | 6 +- .../odbc-bridge/ODBCBlockOutputStream.cpp | 114 +++++++++++++++ programs/odbc-bridge/ODBCBlockOutputStream.h | 27 ++++ src/Storages/StorageURL.cpp | 58 ++------ src/Storages/StorageURL.h | 43 +++++- src/Storages/StorageXDBC.cpp | 33 ++++- src/Storages/StorageXDBC.h | 2 + 10 files changed, 324 insertions(+), 97 deletions(-) create mode 100644 programs/odbc-bridge/ODBCBlockOutputStream.cpp create mode 100644 programs/odbc-bridge/ODBCBlockOutputStream.h diff --git a/programs/odbc-bridge/CMakeLists.txt b/programs/odbc-bridge/CMakeLists.txt index 7bb128902e0..5405411641f 100644 --- a/programs/odbc-bridge/CMakeLists.txt +++ b/programs/odbc-bridge/CMakeLists.txt @@ -5,6 +5,7 @@ set(CLICKHOUSE_ODBC_BRIDGE_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/IdentifierQuoteHandler.cpp ${CMAKE_CURRENT_SOURCE_DIR}/MainHandler.cpp ${CMAKE_CURRENT_SOURCE_DIR}/ODBCBlockInputStream.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/ODBCBlockOutputStream.cpp ${CMAKE_CURRENT_SOURCE_DIR}/ODBCBridge.cpp ${CMAKE_CURRENT_SOURCE_DIR}/PingHandler.cpp ${CMAKE_CURRENT_SOURCE_DIR}/validateODBCConnectionString.cpp diff --git a/programs/odbc-bridge/HandlerFactory.cpp b/programs/odbc-bridge/HandlerFactory.cpp index 55c2c8d7637..730d92101f2 100644 --- a/programs/odbc-bridge/HandlerFactory.cpp +++ b/programs/odbc-bridge/HandlerFactory.cpp @@ -30,8 +30,10 @@ Poco::Net::HTTPRequestHandler * HandlerFactory::createRequestHandler(const Poco: #else return nullptr; #endif + else if (uri.getPath() == "/write") + return new ODBCHandler(pool_map, keep_alive_timeout, context, "write"); else - return new ODBCHandler(pool_map, keep_alive_timeout, context); + return new ODBCHandler(pool_map, keep_alive_timeout, context, "read"); } return nullptr; } diff --git a/programs/odbc-bridge/MainHandler.cpp b/programs/odbc-bridge/MainHandler.cpp index 3ae5f49f24b..3fc698619ec 100644 --- a/programs/odbc-bridge/MainHandler.cpp +++ b/programs/odbc-bridge/MainHandler.cpp @@ -5,17 +5,20 @@ #include #include #include "ODBCBlockInputStream.h" +#include "ODBCBlockOutputStream.h" #include #include #include #include -#include #include #include #include #include #include #include +#include +#include +#include namespace DB { @@ -63,49 +66,33 @@ ODBCHandler::PoolPtr ODBCHandler::getPool(const std::string & connection_str) return pool_map->at(connection_str); } +void ODBCHandler::processError(Poco::Net::HTTPServerResponse & response, const std::string & message) +{ + response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); + if (!response.sent()) + response.send() << message << std::endl; + LOG_WARNING(log, message); +} + void ODBCHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) { - Poco::Net::HTMLForm params(request, request.stream()); + Poco::Net::HTMLForm params(request); + if (mode == "read") + params.read(request.stream()); LOG_TRACE(log, "Request URI: " + request.getURI()); - auto process_error = [&response, this](const std::string & message) - { - response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); - if (!response.sent()) - response.send() << message << std::endl; - LOG_WARNING(log, message); - }; - - if (!params.has("query")) - { - process_error("No 'query' in request body"); - return; - } - if (!params.has("columns")) { - process_error("No 'columns' in request URL"); + processError(response, "No 'columns' in request URL"); return; } if (!params.has("connection_string")) { - process_error("No 'connection_string' in request URL"); + processError(response, "No 'connection_string' in request URL"); return; } - UInt64 max_block_size = DEFAULT_BLOCK_SIZE; - if (params.has("max_block_size")) - { - std::string max_block_size_str = params.get("max_block_size", ""); - if (max_block_size_str.empty()) - { - process_error("Empty max_block_size specified"); - return; - } - max_block_size = parse(max_block_size_str); - } - std::string columns = params.get("columns"); std::unique_ptr sample_block; try @@ -114,33 +101,91 @@ void ODBCHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne } catch (const Exception & ex) { - process_error("Invalid 'columns' parameter in request body '" + ex.message() + "'"); + processError(response, "Invalid 'columns' parameter in request body '" + ex.message() + "'"); LOG_WARNING(log, ex.getStackTraceString()); return; } std::string format = params.get("format", "RowBinary"); - std::string query = params.get("query"); - LOG_TRACE(log, "Query: " << query); std::string connection_string = params.get("connection_string"); LOG_TRACE(log, "Connection string: '" << connection_string << "'"); + UInt64 max_block_size = DEFAULT_BLOCK_SIZE; + if (params.has("max_block_size")) + { + std::string max_block_size_str = params.get("max_block_size", ""); + if (max_block_size_str.empty()) + { + processError(response, "Empty max_block_size specified"); + return; + } + max_block_size = parse(max_block_size_str); + } + WriteBufferFromHTTPServerResponse out(request, response, keep_alive_timeout); - try + if (mode == "write") { - BlockOutputStreamPtr writer = FormatFactory::instance().getOutput(format, out, *sample_block, *context); - auto pool = getPool(connection_string); - ODBCBlockInputStream inp(pool->get(), query, *sample_block, max_block_size); - copyData(inp, *writer); + try + { + if (!params.has("db_name")) + { + processError(response, "No 'db_name' in request URL"); + return; + } + if (!params.has("table_name")) + { + processError(response, "No 'table_name' in request URL"); + return; + } + std::string db_name = params.get("db_name"); + std::string table_name = params.get("table_name"); + LOG_TRACE(log, "DB name: '" << db_name << "', table name: '" << table_name << "'"); + + auto pool = getPool(connection_string); + ReadBufferFromIStream read_buf(request.stream()); + BlockInputStreamPtr input_stream = FormatFactory::instance().getInput(format, read_buf, *sample_block, *context, max_block_size); + ODBCBlockOutputStream output_stream(pool->get(), db_name, table_name, *sample_block); + copyData(*input_stream, output_stream); + writeStringBinary("Ok.", out); + } + catch (...) + { + auto message = getCurrentExceptionMessage(true); + response.setStatusAndReason( + Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); + writeStringBinary(message, out); + tryLogCurrentException(log); + } } - catch (...) + else { - auto message = getCurrentExceptionMessage(true); - response.setStatusAndReason( - Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); // can't call process_error, because of too soon response sending - writeStringBinary(message, out); - tryLogCurrentException(log); + if (!params.has("query")) + { + processError(response, "No 'query' in request body"); + return; + } + + std::string query = params.get("query"); + LOG_TRACE(log, "Query: " << query); + + try + { + BlockOutputStreamPtr writer = FormatFactory::instance().getOutput(format, out, *sample_block, *context); + auto pool = getPool(connection_string); + ODBCBlockInputStream inp(pool->get(), query, *sample_block, max_block_size); + copyData(inp, *writer); + } + catch (...) + { + auto message = getCurrentExceptionMessage(true); + response.setStatusAndReason( + Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); // can't call process_error, because of too soon response sending + writeStringBinary(message, out); + tryLogCurrentException(log); + + } } } + } diff --git a/programs/odbc-bridge/MainHandler.h b/programs/odbc-bridge/MainHandler.h index ae139f393f8..334392d6533 100644 --- a/programs/odbc-bridge/MainHandler.h +++ b/programs/odbc-bridge/MainHandler.h @@ -24,11 +24,13 @@ public: ODBCHandler(std::shared_ptr pool_map_, size_t keep_alive_timeout_, - std::shared_ptr context_) + std::shared_ptr context_, + const String & mode_) : log(&Poco::Logger::get("ODBCHandler")) , pool_map(pool_map_) , keep_alive_timeout(keep_alive_timeout_) , context(context_) + , mode(mode_) { } @@ -40,10 +42,12 @@ private: std::shared_ptr pool_map; size_t keep_alive_timeout; std::shared_ptr context; + String mode; static inline std::mutex mutex; PoolPtr getPool(const std::string & connection_str); + void processError(Poco::Net::HTTPServerResponse & response, const std::string & message); }; } diff --git a/programs/odbc-bridge/ODBCBlockOutputStream.cpp b/programs/odbc-bridge/ODBCBlockOutputStream.cpp new file mode 100644 index 00000000000..72cf2e509ce --- /dev/null +++ b/programs/odbc-bridge/ODBCBlockOutputStream.cpp @@ -0,0 +1,114 @@ +#include "ODBCBlockOutputStream.h" + +#include +#include +#include + +namespace DB +{ + +namespace { + using ValueType = ExternalResultDescription::ValueType; + + std::string commaSeparateColumnNames(const ColumnsWithTypeAndName & columns) + { + std::string result = "("; + for (size_t i = 0; i < columns.size(); ++i) { + if (i > 0) + result += ","; + result += columns[i].name; + } + return result + ")"; + } + + std::string getQuestionMarks(size_t n) + { + std::string result = "("; + for (size_t i = 0; i < n; ++i) { + if (i > 0) + result += ","; + result += "?"; + } + return result + ")"; + } + + Poco::Dynamic::Var getVarFromField(const Field & field, const ValueType type) + { + switch (type) { + case ValueType::vtUInt8: + return Poco::Dynamic::Var(static_cast(field.get())).convert(); + case ValueType::vtUInt16: + return Poco::Dynamic::Var(static_cast(field.get())).convert(); + case ValueType::vtUInt32: + return Poco::Dynamic::Var(static_cast(field.get())).convert(); + case ValueType::vtUInt64: + return Poco::Dynamic::Var(field.get()).convert(); + case ValueType::vtInt8: + return Poco::Dynamic::Var(static_cast(field.get())).convert(); + case ValueType::vtInt16: + return Poco::Dynamic::Var(static_cast(field.get())).convert(); + case ValueType::vtInt32: + return Poco::Dynamic::Var(static_cast(field.get())).convert(); + case ValueType::vtInt64: + return Poco::Dynamic::Var(field.get()).convert(); + case ValueType::vtFloat32: + return Poco::Dynamic::Var(field.get()).convert(); + case ValueType::vtFloat64: + return Poco::Dynamic::Var(field.get()).convert(); + case ValueType::vtString: + return Poco::Dynamic::Var(field.get()).convert(); + case ValueType::vtDate: + return Poco::Dynamic::Var(LocalDate(DayNum(field.get())).toString()).convert(); + case ValueType::vtDateTime: + return Poco::Dynamic::Var(LocalDate(time_t(field.get())).toString()).convert(); + case ValueType::vtUUID: + return Poco::Dynamic::Var(UUID(field.get()).toUnderType().toHexString()).convert(); + } + return Poco::Dynamic::Var(); // Throw smth here? + } +} + +ODBCBlockOutputStream::ODBCBlockOutputStream(Poco::Data::Session && session_, + const std::string & remote_database_name_, + const std::string & remote_table_name_, + const Block & sample_block_) + : session(session_) + , db_name(remote_database_name_) + , table_name(remote_table_name_) + , sample_block(sample_block_) + , log(&Logger::get("ODBCBlockOutputStream")) +{ + description.init(sample_block); +} + +Block ODBCBlockOutputStream::getHeader() const +{ + return sample_block; +} + +void ODBCBlockOutputStream::write(const Block & block) +{ + ColumnsWithTypeAndName columns; + for (size_t i = 0; i < block.columns(); ++i) + columns.push_back({block.getColumns()[i], sample_block.getDataTypes()[i], sample_block.getNames()[i]}); + + std::vector row_to_insert(block.columns()); + Poco::Data::Statement statement(session << "INSERT INTO " + db_name + "." + table_name + " " + + commaSeparateColumnNames(columns) + + " VALUES " + getQuestionMarks(block.columns())); + for (size_t i = 0; i < block.columns(); ++i) + statement, Poco::Data::Keywords::use(row_to_insert[i]); + + for (size_t i = 0; i < block.rows(); ++i) + { + for (size_t col_idx = 0; col_idx < block.columns(); ++col_idx) + { + Field val; + columns[col_idx].column->get(i, val); + row_to_insert[col_idx] = getVarFromField(val, description.types[col_idx].first); + } + statement.execute(); + } +} + +} \ No newline at end of file diff --git a/programs/odbc-bridge/ODBCBlockOutputStream.h b/programs/odbc-bridge/ODBCBlockOutputStream.h new file mode 100644 index 00000000000..5de283f1c8b --- /dev/null +++ b/programs/odbc-bridge/ODBCBlockOutputStream.h @@ -0,0 +1,27 @@ +#pragma once + +#include +#include +#include +#include + +namespace DB{ +class ODBCBlockOutputStream : public IBlockOutputStream { +public: + ODBCBlockOutputStream(Poco::Data::Session && session_, const std::string & remote_database_name_, + const std::string & remote_table_name_, const Block & sample_block_); + + Block getHeader() const override; + void write(const Block & block) override; + +private: + Poco::Data::Session session; + std::string db_name; + std::string table_name; + Block sample_block; + + ExternalResultDescription description; + Poco::Logger * log; +}; + +} \ No newline at end of file diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 6c6f79b50e7..a69e140fe5a 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -112,53 +112,21 @@ namespace BlockInputStreamPtr reader; bool initialized = false; }; - - class StorageURLBlockOutputStream : public IBlockOutputStream - { - public: - StorageURLBlockOutputStream(const Poco::URI & uri, - const String & format, - const Block & sample_block_, - const Context & context, - const ConnectionTimeouts & timeouts, - const CompressionMethod compression_method) - : sample_block(sample_block_) - { - write_buf = wrapWriteBufferWithCompressionMethod( - std::make_unique(uri, Poco::Net::HTTPRequest::HTTP_POST, timeouts), - compression_method, 3); - writer = FormatFactory::instance().getOutput(format, *write_buf, sample_block, context); - } - - Block getHeader() const override - { - return sample_block; - } - - void write(const Block & block) override - { - writer->write(block); - } - - void writePrefix() override - { - writer->writePrefix(); - } - - void writeSuffix() override - { - writer->writeSuffix(); - writer->flush(); - write_buf->finalize(); - } - - private: - Block sample_block; - std::unique_ptr write_buf; - BlockOutputStreamPtr writer; - }; } +StorageURLBlockOutputStream::StorageURLBlockOutputStream(const Poco::URI & uri, + const String & format, + const Block & sample_block_, + const Context & context, + const ConnectionTimeouts & timeouts, + const CompressionMethod compression_method) + : sample_block(sample_block_) +{ + write_buf = wrapWriteBufferWithCompressionMethod( + std::make_unique(uri, Poco::Net::HTTPRequest::HTTP_POST, timeouts), + compression_method, 3); + writer = FormatFactory::instance().getOutput(format, *write_buf, sample_block, context); +} std::string IStorageURLBase::getReadMethod() const { diff --git a/src/Storages/StorageURL.h b/src/Storages/StorageURL.h index 0ff84bc4d37..5a6584f0301 100644 --- a/src/Storages/StorageURL.h +++ b/src/Storages/StorageURL.h @@ -3,6 +3,9 @@ #include #include #include +#include +#include +#include namespace DB @@ -39,10 +42,9 @@ protected: Poco::URI uri; const Context & context_global; String compression_method; - -private: String format_name; +private: virtual std::string getReadMethod() const; virtual std::vector> getReadURIParams( @@ -62,6 +64,43 @@ private: virtual Block getHeaderBlock(const Names & column_names) const = 0; }; +class StorageURLBlockOutputStream : public IBlockOutputStream +{ +public: + StorageURLBlockOutputStream(const Poco::URI & uri, + const String & format, + const Block & sample_block_, + const Context & context, + const ConnectionTimeouts & timeouts, + const CompressionMethod compression_method); + + Block getHeader() const override + { + return sample_block; + } + + void write(const Block & block) override + { + writer->write(block); + } + + void writePrefix() override + { + writer->writePrefix(); + } + + void writeSuffix() override + { + writer->writeSuffix(); + writer->flush(); + write_buf->finalize(); + } + +private: + Block sample_block; + std::unique_ptr write_buf; + BlockOutputStreamPtr writer; +}; class StorageURL final : public ext::shared_ptr_helper, public IStorageURLBase { diff --git a/src/Storages/StorageXDBC.cpp b/src/Storages/StorageXDBC.cpp index dd449e490aa..76563124b63 100644 --- a/src/Storages/StorageXDBC.cpp +++ b/src/Storages/StorageXDBC.cpp @@ -3,17 +3,15 @@ #include #include #include +#include #include -#include #include -#include #include #include -#include #include #include #include -#include +#include #include @@ -97,6 +95,33 @@ Pipes StorageXDBC::read(const Names & column_names, return IStorageURLBase::read(column_names, query_info, context, processed_stage, max_block_size, num_streams); } +BlockOutputStreamPtr StorageXDBC::write(const ASTPtr & /*query*/, const Context & context) { + bridge_helper->startBridgeSync(); + LOG_INFO(log, "TAAAK"); + LOG_INFO(log, remote_database_name); + LOG_INFO(log, remote_table_name); +// LOG_INFO(log, bridge_helper->getConnectionString()); + + // some copypaste + NamesAndTypesList cols; + Poco::URI request_uri = uri; + request_uri.setPath("/write"); + for (const String & name : getSampleBlock().getNames()) + { + auto column_data = getColumn(name); + cols.emplace_back(column_data.name, column_data.type); + } + auto url_params = bridge_helper->getURLParams(cols.toString(), 65536); + for (const auto & [param, value] : url_params) + request_uri.addQueryParameter(param, value); + request_uri.addQueryParameter("db_name", remote_database_name); + request_uri.addQueryParameter("table_name", remote_table_name); + + return std::make_shared( + request_uri, format_name, getSampleBlock(), context, + ConnectionTimeouts::getHTTPTimeouts(context), + chooseCompressionMethod(uri.toString(), compression_method)); +} Block StorageXDBC::getHeaderBlock(const Names & column_names) const { diff --git a/src/Storages/StorageXDBC.h b/src/Storages/StorageXDBC.h index c602415d8a5..afc61dac5cd 100644 --- a/src/Storages/StorageXDBC.h +++ b/src/Storages/StorageXDBC.h @@ -29,6 +29,8 @@ public: const ColumnsDescription & columns_, const Context & context_, BridgeHelperPtr bridge_helper_); + BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override; + private: BridgeHelperPtr bridge_helper; From 0cc3d780b3683da5ca9a2745b559c82f0e5aa2ab Mon Sep 17 00:00:00 2001 From: bobrovskij artemij Date: Tue, 28 Apr 2020 14:03:24 +0300 Subject: [PATCH 010/738] Support write for XDBCStorage --- src/Storages/StorageXDBC.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/Storages/StorageXDBC.cpp b/src/Storages/StorageXDBC.cpp index 76563124b63..e4db7aa1fd9 100644 --- a/src/Storages/StorageXDBC.cpp +++ b/src/Storages/StorageXDBC.cpp @@ -97,10 +97,6 @@ Pipes StorageXDBC::read(const Names & column_names, BlockOutputStreamPtr StorageXDBC::write(const ASTPtr & /*query*/, const Context & context) { bridge_helper->startBridgeSync(); - LOG_INFO(log, "TAAAK"); - LOG_INFO(log, remote_database_name); - LOG_INFO(log, remote_table_name); -// LOG_INFO(log, bridge_helper->getConnectionString()); // some copypaste NamesAndTypesList cols; From 007170481a966674e17f6e47e5c5abe8a87d3747 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 28 Apr 2020 22:41:23 +0300 Subject: [PATCH 011/738] Add a test for ALIAS with type different to the type of column --- .../01269_alias_type_differs.reference | 4 ++++ .../0_stateless/01269_alias_type_differs.sql | 20 +++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 tests/queries/0_stateless/01269_alias_type_differs.reference create mode 100644 tests/queries/0_stateless/01269_alias_type_differs.sql diff --git a/tests/queries/0_stateless/01269_alias_type_differs.reference b/tests/queries/0_stateless/01269_alias_type_differs.reference new file mode 100644 index 00000000000..f837a2bd382 --- /dev/null +++ b/tests/queries/0_stateless/01269_alias_type_differs.reference @@ -0,0 +1,4 @@ +UInt8 +0 +UInt8 +0 diff --git a/tests/queries/0_stateless/01269_alias_type_differs.sql b/tests/queries/0_stateless/01269_alias_type_differs.sql new file mode 100644 index 00000000000..3b99e5e7eec --- /dev/null +++ b/tests/queries/0_stateless/01269_alias_type_differs.sql @@ -0,0 +1,20 @@ +DROP TABLE IF EXISTS data_01269; +CREATE TABLE data_01269 +( + key Int32, + value Nullable(Int32), + alias UInt8 ALIAS value>0 +) +ENGINE = MergeTree() +ORDER BY key; +INSERT INTO data_01269 VALUES (1, 0); + +-- after PR#10441 +SELECT toTypeName(alias) FROM data_01269; +SELECT any(alias) FROM data_01269; + +-- even without PR#10441 +ALTER TABLE data_01269 DROP COLUMN alias; +ALTER TABLE data_01269 ADD COLUMN alias UInt8 ALIAS value>0; +SELECT toTypeName(alias) FROM data_01269; +SELECT any(alias) FROM data_01269; From 53aea23de0033b1a111bdd8a31241fddb7bd4c5a Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 28 Apr 2020 22:41:23 +0300 Subject: [PATCH 012/738] Fix SELECT of column ALIAS which default expression type different from column type --- src/Interpreters/InterpreterSelectQuery.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 2f98addd975..b74ce0da5aa 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -44,6 +44,7 @@ #include #include #include +#include #include #include #include @@ -1210,7 +1211,12 @@ void InterpreterSelectQuery::executeFetchColumns( const auto column_default = storage_columns.getDefault(column); bool is_alias = column_default && column_default->kind == ColumnDefaultKind::Alias; if (is_alias) - column_expr = setAlias(column_default->expression->clone(), column); + { + auto column_decl = storage_columns.get(column); + /// TODO: can make CAST only if the type is different (but requires SyntaxAnalyzer). + auto cast_column_default = addTypeConversionToAST(column_default->expression->clone(), column_decl.type->getName()); + column_expr = setAlias(cast_column_default->clone(), column); + } else column_expr = std::make_shared(column); From 038235684ddcc3abf2ba0ab1b39a3ec12e949cd2 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 29 Apr 2020 00:07:30 +0300 Subject: [PATCH 013/738] Add optimize_distributed_group_by_sharding_key and disable it by default I know at least one way to fool that optimization, by using as sharding key something like `if(col1>0, col1, col2)` (although this is not common sharding key I would say, but can be useful if this will work correctly), so let's disable it by default. --- src/Core/Settings.h | 1 + src/Storages/StorageDistributed.cpp | 3 +++ .../01247_dist_on_dist_group_by_sharding_key_optimization.sql | 2 ++ ...buted_group_by_no_merge_GROUP_BY_injective_sharding_key.sql | 2 ++ 4 files changed, 8 insertions(+) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 7b5b130ee72..651b519dba7 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -116,6 +116,7 @@ struct Settings : public SettingsCollection M(SettingBool, distributed_group_by_no_merge, false, "Do not merge aggregation states from different servers for distributed query processing - in case it is for certain that there are different keys on different shards.", 0) \ M(SettingBool, parallel_distributed_insert_select, false, "If true, distributed insert select query in the same cluster will be processed on local tables on every shard", 0) \ M(SettingBool, optimize_skip_unused_shards, false, "Assumes that data is distributed by sharding_key. Optimization to skip unused shards if SELECT query filters by sharding_key.", 0) \ + M(SettingBool, optimize_distributed_group_by_sharding_key, false, "Optimize GROUP BY sharding_key queries (by avodiing costly aggregation on the initiator server).", 0) \ M(SettingUInt64, force_optimize_skip_unused_shards, 0, "Throw an exception if unused shards cannot be skipped (1 - throw only if the table has the sharding key, 2 - always throw.", 0) \ M(SettingBool, force_optimize_skip_unused_shards_no_nested, false, "Do not apply force_optimize_skip_unused_shards for nested Distributed tables.", 0) \ \ diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 72c364905db..221b0df467c 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -389,6 +389,9 @@ bool StorageDistributed::canForceGroupByNoMerge(const Context &context, QueryPro if (settings.distributed_group_by_no_merge) return true; + if (!settings.optimize_distributed_group_by_sharding_key) + return false; + /// Distributed-over-Distributed (see getQueryProcessingStageImpl()) if (to_stage == QueryProcessingStage::WithMergeableState) return false; diff --git a/tests/queries/0_stateless/01247_dist_on_dist_group_by_sharding_key_optimization.sql b/tests/queries/0_stateless/01247_dist_on_dist_group_by_sharding_key_optimization.sql index dff47118c9c..dc2941e63fd 100644 --- a/tests/queries/0_stateless/01247_dist_on_dist_group_by_sharding_key_optimization.sql +++ b/tests/queries/0_stateless/01247_dist_on_dist_group_by_sharding_key_optimization.sql @@ -1,5 +1,7 @@ -- TODO: correct testing with real unique shards +set optimize_distributed_group_by_sharding_key=1; + drop table if exists dist_01247; drop table if exists dist_layer_01247; drop table if exists data_01247; diff --git a/tests/queries/0_stateless/01247_distributed_group_by_no_merge_GROUP_BY_injective_sharding_key.sql b/tests/queries/0_stateless/01247_distributed_group_by_no_merge_GROUP_BY_injective_sharding_key.sql index 2a4f0e0601d..bca3e4830d2 100644 --- a/tests/queries/0_stateless/01247_distributed_group_by_no_merge_GROUP_BY_injective_sharding_key.sql +++ b/tests/queries/0_stateless/01247_distributed_group_by_no_merge_GROUP_BY_injective_sharding_key.sql @@ -1,5 +1,7 @@ -- TODO: correct testing with real unique shards +set optimize_distributed_group_by_sharding_key=1; + drop table if exists dist_01247; drop table if exists data_01247; From 351d62aa837276579c42fdf2616d5857203b3efa Mon Sep 17 00:00:00 2001 From: Potya Date: Wed, 29 Apr 2020 15:17:40 +0300 Subject: [PATCH 014/738] Nothing interesting --- src/Core/Settings.h | 11 +++----- src/Core/SettingsCollection.cpp | 12 ++++---- src/Interpreters/InterpreterSelectQuery.cpp | 2 ++ .../InterpreterSelectWithUnionQuery.cpp | 1 + src/Parsers/ParserSelectWithUnionQuery.cpp | 28 +++++++++++-------- 5 files changed, 29 insertions(+), 25 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 61fcf658ba8..306e2e67f36 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -178,7 +178,6 @@ struct Settings : public SettingsCollection \ M(SettingString, count_distinct_implementation, "uniqExact", "What aggregate function to use for implementation of count(DISTINCT ...)", 0) \ \ - M(SettingBool, output_format_enable_streaming, false, "Enable streaming in output formats that support it.", 0) \ M(SettingBool, output_format_write_statistics, true, "Write statistics about read rows, bytes, time elapsed in suitable output formats.", 0) \ \ M(SettingBool, add_http_cors_header, false, "Write add http CORS header.", 0) \ @@ -225,6 +224,7 @@ struct Settings : public SettingsCollection M(SettingBool, join_use_nulls, 0, "Use NULLs for non-joined rows of outer JOINs for types that can be inside Nullable. If false, use default value of corresponding columns data type.", IMPORTANT) \ \ M(SettingJoinStrictness, join_default_strictness, JoinStrictness::ALL, "Set default strictness in JOIN query. Possible values: empty string, 'ANY', 'ALL'. If empty, query without strictness will throw exception.", 0) \ + M(SettingUnionMode, union_default_mode, UnionMode::ALL, "Set default mode in UNION query. Possible values: empty string, 'DISTINCT', 'ALL'. If empty, query without mode will throw exception.", 0) \ M(SettingBool, any_join_distinct_right_table_keys, false, "Enable old ANY JOIN logic with many-to-one left-to-right table keys mapping for all ANY JOINs. It leads to confusing not equal results for 't1 ANY LEFT JOIN t2' and 't2 ANY RIGHT JOIN t1'. ANY RIGHT JOIN needs one-to-many keys mapping to be consistent with LEFT one.", IMPORTANT) \ \ M(SettingUInt64, preferred_block_size_bytes, 1000000, "", 0) \ @@ -326,11 +326,9 @@ struct Settings : public SettingsCollection M(SettingOverflowMode, join_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \ M(SettingBool, join_any_take_last_row, false, "When disabled (default) ANY JOIN will take the first found row for a key. When enabled, it will take the last row seen if there are multiple rows for the same key.", IMPORTANT) \ M(SettingJoinAlgorithm, join_algorithm, JoinAlgorithm::HASH, "Specify join algorithm: 'auto', 'hash', 'partial_merge', 'prefer_partial_merge'. 'auto' tries to change HashJoin to MergeJoin on the fly to avoid out of memory.", 0) \ - M(SettingBool, partial_merge_join_optimizations, true, "Enable optimizations in partial merge join", 0) \ + M(SettingBool, partial_merge_join_optimizations, false, "Enable optimizations in partial merge join", 0) \ M(SettingUInt64, default_max_bytes_in_join, 1000000000, "Maximum size of right-side table if limit is required but max_bytes_in_join is not set.", 0) \ - M(SettingUInt64, partial_merge_join_rows_in_right_blocks, 65536, "Split right-hand joining data in blocks of specified size. It's a portion of data indexed by min-max values and possibly unloaded on disk.", 0) \ - M(SettingUInt64, join_on_disk_max_files_to_merge, 64, "For MergeJoin on disk set how much files it's allowed to sort simultaneously. Then this value bigger then more memory used and then less disk I/O needed. Minimum is 2.", 0) \ - M(SettingString, temporary_files_codec, "LZ4", "Set compression codec for temporary files (sort and join on disk). I.e. LZ4, NONE.", 0) \ + M(SettingUInt64, partial_merge_join_rows_in_right_blocks, 10000, "Split right-hand joining data in blocks of specified size. It's a portion of data indexed by min-max values and possibly unloaded on disk.", 0) \ \ M(SettingUInt64, max_rows_to_transfer, 0, "Maximum size (in rows) of the transmitted external table obtained when the GLOBAL IN/JOIN section is executed.", 0) \ M(SettingUInt64, max_bytes_to_transfer, 0, "Maximum size (in uncompressed bytes) of the transmitted external table obtained when the GLOBAL IN/JOIN section is executed.", 0) \ @@ -342,6 +340,7 @@ struct Settings : public SettingsCollection \ M(SettingUInt64, max_memory_usage, 0, "Maximum memory usage for processing of single query. Zero means unlimited.", 0) \ M(SettingUInt64, max_memory_usage_for_user, 0, "Maximum memory usage for processing all concurrently running queries for the user. Zero means unlimited.", 0) \ + M(SettingUInt64, max_server_memory_usage, 0, "Maximum memory usage for server. Only has meaning at server startup. It can be specified only for default profile.", 0) \ M(SettingUInt64, memory_profiler_step, 0, "Every number of bytes the memory profiler will collect the allocating stack trace. The minimal effective step is 4 MiB (less values will work as clamped to 4 MiB). Zero means disabled memory profiler.", 0) \ \ M(SettingUInt64, max_network_bandwidth, 0, "The maximum speed of data exchange over the network in bytes per second for a query. Zero means unlimited.", 0) \ @@ -397,8 +396,6 @@ struct Settings : public SettingsCollection M(SettingUInt64, max_live_view_insert_blocks_before_refresh, 64, "Limit maximum number of inserted blocks after which mergeable blocks are dropped and query is re-executed.", 0) \ M(SettingUInt64, min_free_disk_space_for_temporary_data, 0, "The minimum disk space to keep while writing temporary data used in external sorting and aggregation.", 0) \ \ - M(SettingDefaultDatabaseEngine, default_database_engine, DefaultDatabaseEngine::Ordinary, "Default database engine.", 0) \ - M(SettingBool, allow_experimental_database_atomic, false, "Allow to create database with Engine=Atomic.", 0) \ M(SettingBool, enable_scalar_subquery_optimization, true, "If it is set to true, prevent scalar subqueries from (de)serializing large scalar values and possibly avoid running the same subquery more than once.", 0) \ M(SettingBool, optimize_trivial_count_query, true, "Process trivial 'SELECT count() FROM table' query from metadata.", 0) \ M(SettingUInt64, mutations_sync, 0, "Wait for synchronous execution of ALTER TABLE UPDATE/DELETE queries (mutations). 0 - execute asynchronously. 1 - wait current server. 2 - wait all replicas if they exist.", 0) \ diff --git a/src/Core/SettingsCollection.cpp b/src/Core/SettingsCollection.cpp index 4223ab6ea42..4f13666ccda 100644 --- a/src/Core/SettingsCollection.cpp +++ b/src/Core/SettingsCollection.cpp @@ -491,6 +491,12 @@ IMPLEMENT_SETTING_ENUM(LoadBalancing, LOAD_BALANCING_LIST_OF_NAMES, ErrorCodes:: M(ANY, "ANY") IMPLEMENT_SETTING_ENUM(JoinStrictness, JOIN_STRICTNESS_LIST_OF_NAMES, ErrorCodes::UNKNOWN_JOIN) // NOLINT +#define UNION_MODE_LIST_OF_NAMES(M) \ + M(Unspecified, "") \ + M(ALL, "ALL") \ + M(DISTINCT, "DISTINCT") +IMPLEMENT_SETTING_ENUM(UnionMode, UNION_MODE_LIST_OF_NAMES, ErrorCodes::UNKNOWN_JOIN) // NOLINT + #define JOIN_ALGORITHM_NAMES(M) \ M(AUTO, "auto") \ M(HASH, "hash") \ @@ -550,12 +556,6 @@ IMPLEMENT_SETTING_ENUM(LogsLevel, LOGS_LEVEL_LIST_OF_NAMES, ErrorCodes::BAD_ARGU IMPLEMENT_SETTING_ENUM(QueryLogElementType, LOG_QUERIES_TYPE_LIST_OF_NAMES, ErrorCodes::BAD_ARGUMENTS) -#define DEFAULT_DATABASE_ENGINE_LIST_OF_NAMES(M) \ - M(Ordinary, "Ordinary") \ - M(Atomic, "Atomic") -IMPLEMENT_SETTING_ENUM(DefaultDatabaseEngine , DEFAULT_DATABASE_ENGINE_LIST_OF_NAMES, ErrorCodes::BAD_ARGUMENTS) - - namespace details { void SettingsCollectionUtils::serializeName(const StringRef & name, WriteBuffer & buf) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 290bc26856a..f79e2822035 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -2280,6 +2280,8 @@ void InterpreterSelectQuery::executeUnion(Pipeline & pipeline, Block header) pipeline.streams.push_back(pipeline.stream_with_non_joined_data); pipeline.stream_with_non_joined_data = nullptr; } + + } diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index 9cdb19b1934..dc47ce93dab 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -121,6 +121,7 @@ InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery( } options.ignore_limits |= all_nested_ignore_limits; options.ignore_quota |= all_nested_ignore_quota; + } diff --git a/src/Parsers/ParserSelectWithUnionQuery.cpp b/src/Parsers/ParserSelectWithUnionQuery.cpp index cebe8ba876d..3ef8c4d3eaf 100644 --- a/src/Parsers/ParserSelectWithUnionQuery.cpp +++ b/src/Parsers/ParserSelectWithUnionQuery.cpp @@ -27,20 +27,24 @@ bool ParserSelectWithUnionQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & { ASTPtr list_node; - ParserList parser(std::make_unique(), std::make_unique("UNION ALL"), false); - if (!parser.parse(pos, list_node, expected)) + ParserList parser_all(std::make_unique(), std::make_unique("UNION ALL"), false); + ParserList parser_distinct(std::make_unique(), std::make_unique("UNION DISTINCT"), false); + if (parser_all.parse(pos, list_node, expected)) { + auto select_with_union_query = std::make_shared(); + + node = select_with_union_query; + select_with_union_query->list_of_selects = std::make_shared(); + select_with_union_query->children.push_back(select_with_union_query->list_of_selects); + + // flatten inner union query + for (auto & child : list_node->children) + getSelectsFromUnionListNode(child, select_with_union_query->list_of_selects->children); + } else if (parser_all.parse(pos, list_node, expected)) { + // distinct parser + } + else return false; - auto select_with_union_query = std::make_shared(); - - node = select_with_union_query; - select_with_union_query->list_of_selects = std::make_shared(); - select_with_union_query->children.push_back(select_with_union_query->list_of_selects); - - // flatten inner union query - for (auto & child : list_node->children) - getSelectsFromUnionListNode(child, select_with_union_query->list_of_selects->children); - return true; } From faea8afa22b862f8ef0b6d7458bd135d4dae5415 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 30 Apr 2020 10:13:19 +0300 Subject: [PATCH 015/738] Revert behaviour with duplicates in 01213_optimize_skip_unused_shards_DISTINCT Since now optimization is done only under optimize_distributed_group_by_sharding_key. --- .../01213_optimize_skip_unused_shards_DISTINCT.reference | 2 -- .../0_stateless/01213_optimize_skip_unused_shards_DISTINCT.sql | 1 - 2 files changed, 3 deletions(-) diff --git a/tests/queries/0_stateless/01213_optimize_skip_unused_shards_DISTINCT.reference b/tests/queries/0_stateless/01213_optimize_skip_unused_shards_DISTINCT.reference index eac8957dd47..4ade9cd9c5d 100644 --- a/tests/queries/0_stateless/01213_optimize_skip_unused_shards_DISTINCT.reference +++ b/tests/queries/0_stateless/01213_optimize_skip_unused_shards_DISTINCT.reference @@ -6,5 +6,3 @@ optimize_skip_unused_shards optimize_skip_unused_shards lack of WHERE 0 1 -0 -1 diff --git a/tests/queries/0_stateless/01213_optimize_skip_unused_shards_DISTINCT.sql b/tests/queries/0_stateless/01213_optimize_skip_unused_shards_DISTINCT.sql index 67ba1cda870..5b45bea9046 100644 --- a/tests/queries/0_stateless/01213_optimize_skip_unused_shards_DISTINCT.sql +++ b/tests/queries/0_stateless/01213_optimize_skip_unused_shards_DISTINCT.sql @@ -12,7 +12,6 @@ SELECT DISTINCT id FROM dist_01213 WHERE id = 1 SETTINGS distributed_group_by_no SELECT 'optimize_skip_unused_shards'; SELECT DISTINCT id FROM dist_01213 WHERE id = 1 SETTINGS optimize_skip_unused_shards=1; -- check that querying all shards is ok --- (there will be duplicates, since the INSERT was done via local table) SELECT 'optimize_skip_unused_shards lack of WHERE'; SELECT DISTINCT id FROM dist_01213 SETTINGS optimize_skip_unused_shards=1; From 19dadb8c2d7afd357b228a836edb646fa84fc4f9 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 22 Apr 2020 16:52:07 +0300 Subject: [PATCH 016/738] Add parallel final. --- src/Common/WeakHash.h | 2 + src/Core/Settings.h | 2 +- src/Processors/Chunk.cpp | 2 +- src/Processors/ISimpleTransform.cpp | 50 +++-- src/Processors/ISimpleTransform.h | 24 ++- .../Algorithms/AggregatingSortedAlgorithm.cpp | 4 +- .../Algorithms/AggregatingSortedAlgorithm.h | 2 +- .../Algorithms/CollapsingSortedAlgorithm.cpp | 4 +- .../Algorithms/CollapsingSortedAlgorithm.h | 2 + .../Merges/Algorithms/IMergingAlgorithm.h | 2 +- .../IMergingAlgorithmWithDelayedChunk.cpp | 6 +- .../IMergingAlgorithmWithDelayedChunk.h | 2 +- .../IMergingAlgorithmWithSharedChunks.cpp | 6 +- .../IMergingAlgorithmWithSharedChunks.h | 2 +- .../Algorithms/MergingSortedAlgorithm.cpp | 4 +- .../Algorithms/MergingSortedAlgorithm.h | 2 +- src/Processors/Merges/Algorithms/RowRef.h | 10 +- .../Algorithms/SummingSortedAlgorithm.cpp | 4 +- .../Algorithms/SummingSortedAlgorithm.h | 2 +- .../Merges/CollapsingSortedTransform.h | 2 + src/Processors/Merges/IMergingTransform.cpp | 87 +++++++- src/Processors/Merges/IMergingTransform.h | 24 ++- src/Processors/Pipe.cpp | 9 + src/Processors/Pipe.h | 5 + .../Transforms/AddingSelectorTransform.cpp | 76 +++++++ .../Transforms/AddingSelectorTransform.h | 26 +++ src/Processors/Transforms/CopyTransform.cpp | 108 ++++++++++ src/Processors/Transforms/CopyTransform.h | 28 +++ src/Processors/Transforms/SelectorInfo.h | 14 ++ src/Processors/ya.make | 2 + .../MergeTree/MergeTreeDataMergerMutator.cpp | 2 +- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 190 ++++++++++++------ .../MergeTree/MergeTreeDataSelectExecutor.h | 1 + tests/performance/parallel_final.xml | 51 +++++ ...00191_aggregating_merge_tree_and_final.sql | 4 +- .../00564_versioned_collapsing_merge_tree.sql | 48 ++--- .../00915_simple_aggregate_function.sql | 6 +- ...030_incorrect_count_summing_merge_tree.sql | 8 +- 38 files changed, 659 insertions(+), 164 deletions(-) create mode 100644 src/Processors/Transforms/AddingSelectorTransform.cpp create mode 100644 src/Processors/Transforms/AddingSelectorTransform.h create mode 100644 src/Processors/Transforms/CopyTransform.cpp create mode 100644 src/Processors/Transforms/CopyTransform.h create mode 100644 src/Processors/Transforms/SelectorInfo.h create mode 100644 tests/performance/parallel_final.xml diff --git a/src/Common/WeakHash.h b/src/Common/WeakHash.h index 87ddec4deff..bfea75eddf1 100644 --- a/src/Common/WeakHash.h +++ b/src/Common/WeakHash.h @@ -17,6 +17,8 @@ public: explicit WeakHash32(size_t size) : data(size, ~UInt32(0)) {} WeakHash32(const WeakHash32 & other) { data.assign(other.data); } + void reset(size_t size) { data.assign(size, ~UInt32(0)); } + const Container & getData() const { return data; } Container & getData() { return data; } diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 61fcf658ba8..17239152055 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -54,6 +54,7 @@ struct Settings : public SettingsCollection M(SettingUInt64, min_insert_block_size_bytes, (DEFAULT_INSERT_BLOCK_SIZE * 256), "Squash blocks passed to INSERT query to specified size in bytes, if blocks are not big enough.", 0) \ M(SettingUInt64, max_joined_block_size_rows, DEFAULT_BLOCK_SIZE, "Maximum block size for JOIN result (if join algorithm supports it). 0 means unlimited.", 0) \ M(SettingUInt64, max_insert_threads, 0, "The maximum number of threads to execute the INSERT SELECT query. Values 0 or 1 means that INSERT SELECT is not run in parallel. Higher values will lead to higher memory usage. Parallel INSERT SELECT has effect only if the SELECT part is run on parallel, see 'max_threads' setting.", 0) \ + M(SettingUInt64, max_final_threads, 16, "The maximum number of threads to read from table with FINAL.", 0) \ M(SettingMaxThreads, max_threads, 0, "The maximum number of threads to execute the request. By default, it is determined automatically.", 0) \ M(SettingMaxThreads, max_alter_threads, 0, "The maximum number of threads to execute the ALTER requests. By default, it is determined automatically.", 0) \ M(SettingUInt64, max_read_buffer_size, DBMS_DEFAULT_BUFFER_SIZE, "The maximum size of the buffer to read from the filesystem.", 0) \ @@ -429,7 +430,6 @@ struct Settings : public SettingsCollection M(SettingBool, partial_merge_join, false, "Obsolete. Use join_algorithm='prefer_partial_merge' instead.", 0) \ M(SettingUInt64, max_memory_usage_for_all_queries, 0, "Obsolete. Will be removed after 2020-10-20", 0) \ - DECLARE_SETTINGS_COLLECTION(LIST_OF_SETTINGS) /** Set multiple settings from "profile" (in server configuration file (users.xml), profiles contain groups of multiple settings). diff --git a/src/Processors/Chunk.cpp b/src/Processors/Chunk.cpp index d68c2bea5ae..2c3d506fa04 100644 --- a/src/Processors/Chunk.cpp +++ b/src/Processors/Chunk.cpp @@ -46,7 +46,7 @@ Chunk::Chunk(MutableColumns columns_, UInt64 num_rows_, ChunkInfoPtr chunk_info_ Chunk Chunk::clone() const { - return Chunk(getColumns(), getNumRows()); + return Chunk(getColumns(), getNumRows(), chunk_info); } void Chunk::setColumns(Columns columns_, UInt64 num_rows_) diff --git a/src/Processors/ISimpleTransform.cpp b/src/Processors/ISimpleTransform.cpp index fc8f1ba30f4..ac8f2f8b7ae 100644 --- a/src/Processors/ISimpleTransform.cpp +++ b/src/Processors/ISimpleTransform.cpp @@ -29,10 +29,10 @@ ISimpleTransform::Status ISimpleTransform::prepare() } /// Output if has data. - if (transformed) + if (has_output) { - output.pushData(std::move(current_data)); - transformed = false; + output.pushData(std::move(output_data)); + has_output = false; if (!no_more_data_needed) return Status::PortFull; @@ -56,27 +56,17 @@ ISimpleTransform::Status ISimpleTransform::prepare() return Status::Finished; } - if (!input.hasData()) - { - input.setNeeded(); - return Status::NeedData; - } + input.setNeeded(); - current_data = input.pullData(true); + if (!input.hasData()) + return Status::NeedData; + + input_data = input.pullData(set_input_not_needed_after_read); has_input = true; - if (current_data.exception) - { - /// Skip transform in case of exception. - has_input = false; - transformed = true; - + if (input_data.exception) /// No more data needed. Exception will be thrown (or swallowed) later. input.setNotNeeded(); - } - - if (set_input_not_needed_after_read) - input.setNotNeeded(); } /// Now transform. @@ -85,29 +75,35 @@ ISimpleTransform::Status ISimpleTransform::prepare() void ISimpleTransform::work() { - if (current_data.exception) + if (input_data.exception) + { + /// Skip transform in case of exception. + output_data = std::move(input_data); + has_input = false; + has_output = true; return; + } try { - transform(current_data.chunk); + transform(input_data.chunk, output_data.chunk); } catch (DB::Exception &) { - current_data.exception = std::current_exception(); - transformed = true; + output_data.exception = std::current_exception(); + has_output = true; has_input = false; return; } has_input = !needInputData(); - if (!skip_empty_chunks || current_data.chunk) - transformed = true; + if (!skip_empty_chunks || output_data.chunk) + has_output = true; - if (transformed && !current_data.chunk) + if (has_output && !output_data.chunk && getOutputPort().getHeader()) /// Support invariant that chunks must have the same number of columns as header. - current_data.chunk = Chunk(getOutputPort().getHeader().cloneEmpty().getColumns(), 0); + output_data.chunk = Chunk(getOutputPort().getHeader().cloneEmpty().getColumns(), 0); } } diff --git a/src/Processors/ISimpleTransform.h b/src/Processors/ISimpleTransform.h index 1f0c5a5b040..ee92b574d7c 100644 --- a/src/Processors/ISimpleTransform.h +++ b/src/Processors/ISimpleTransform.h @@ -6,6 +6,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int NOT_IMPLEMENTED; +} + /** Has one input and one output. * Simply pull a block from input, transform it, and push it to output. */ @@ -15,18 +20,29 @@ protected: InputPort & input; OutputPort & output; - Port::Data current_data; + Port::Data input_data; + Port::Data output_data; bool has_input = false; - bool transformed = false; + bool has_output = false; bool no_more_data_needed = false; const bool skip_empty_chunks; /// Set input port NotNeeded after chunk was pulled. /// Input port will become needed again only after data was transformed. /// This allows to escape caching chunks in input port, which can lead to uneven data distribution. - bool set_input_not_needed_after_read = false; + bool set_input_not_needed_after_read = true; + + virtual void transform(Chunk &) + { + throw Exception("Method transform is not implemented for " + getName(), ErrorCodes::NOT_IMPLEMENTED); + } + + virtual void transform(Chunk & input_chunk, Chunk & output_chunk) + { + transform(input_chunk); + output_chunk.swap(input_chunk); + } - virtual void transform(Chunk & chunk) = 0; virtual bool needInputData() const { return true; } void stopReading() { no_more_data_needed = true; } diff --git a/src/Processors/Merges/Algorithms/AggregatingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/AggregatingSortedAlgorithm.cpp index 269c69f2747..be9bf3e354c 100644 --- a/src/Processors/Merges/Algorithms/AggregatingSortedAlgorithm.cpp +++ b/src/Processors/Merges/Algorithms/AggregatingSortedAlgorithm.cpp @@ -289,10 +289,10 @@ void AggregatingSortedAlgorithm::initialize(Chunks chunks) initializeQueue(std::move(chunks)); } -void AggregatingSortedAlgorithm::consume(Chunk chunk, size_t source_num) +void AggregatingSortedAlgorithm::consume(Chunk & chunk, size_t source_num) { preprocessChunk(chunk, columns_definition); - updateCursor(std::move(chunk), source_num); + updateCursor(chunk, source_num); } IMergingAlgorithm::Status AggregatingSortedAlgorithm::merge() diff --git a/src/Processors/Merges/Algorithms/AggregatingSortedAlgorithm.h b/src/Processors/Merges/Algorithms/AggregatingSortedAlgorithm.h index b4819ad030c..fe1710adc8b 100644 --- a/src/Processors/Merges/Algorithms/AggregatingSortedAlgorithm.h +++ b/src/Processors/Merges/Algorithms/AggregatingSortedAlgorithm.h @@ -20,7 +20,7 @@ public: SortDescription description_, size_t max_block_size); void initialize(Chunks chunks) override; - void consume(Chunk chunk, size_t source_num) override; + void consume(Chunk & chunk, size_t source_num) override; Status merge() override; struct SimpleAggregateDescription; diff --git a/src/Processors/Merges/Algorithms/CollapsingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/CollapsingSortedAlgorithm.cpp index cd3a193105d..8e799664fae 100644 --- a/src/Processors/Merges/Algorithms/CollapsingSortedAlgorithm.cpp +++ b/src/Processors/Merges/Algorithms/CollapsingSortedAlgorithm.cpp @@ -23,6 +23,7 @@ CollapsingSortedAlgorithm::CollapsingSortedAlgorithm( size_t num_inputs, SortDescription description_, const String & sign_column, + bool only_positive_sign_, size_t max_block_size, WriteBuffer * out_row_sources_buf_, bool use_average_block_sizes, @@ -30,6 +31,7 @@ CollapsingSortedAlgorithm::CollapsingSortedAlgorithm( : IMergingAlgorithmWithSharedChunks(num_inputs, std::move(description_), out_row_sources_buf_, max_row_refs) , merged_data(header.cloneEmptyColumns(), use_average_block_sizes, max_block_size) , sign_column_number(header.getPositionByName(sign_column)) + , only_positive_sign(only_positive_sign_) , log(log_) { } @@ -76,7 +78,7 @@ void CollapsingSortedAlgorithm::insertRows() if (last_is_positive || count_positive != count_negative) { - if (count_positive <= count_negative) + if (count_positive <= count_negative && !only_positive_sign) { insertRow(first_negative_row); diff --git a/src/Processors/Merges/Algorithms/CollapsingSortedAlgorithm.h b/src/Processors/Merges/Algorithms/CollapsingSortedAlgorithm.h index 4158f55a7cd..3cbe95d96e1 100644 --- a/src/Processors/Merges/Algorithms/CollapsingSortedAlgorithm.h +++ b/src/Processors/Merges/Algorithms/CollapsingSortedAlgorithm.h @@ -31,6 +31,7 @@ public: size_t num_inputs, SortDescription description_, const String & sign_column, + bool only_positive_sign_, /// For select final. Skip rows with sum(sign) < 0. size_t max_block_size, WriteBuffer * out_row_sources_buf_, bool use_average_block_sizes, @@ -42,6 +43,7 @@ private: MergedData merged_data; const size_t sign_column_number; + const bool only_positive_sign; static constexpr size_t max_row_refs = 4; /// first_negative, last_positive, last, current. RowRef first_negative_row; diff --git a/src/Processors/Merges/Algorithms/IMergingAlgorithm.h b/src/Processors/Merges/Algorithms/IMergingAlgorithm.h index 263acee4c2d..b49209e462e 100644 --- a/src/Processors/Merges/Algorithms/IMergingAlgorithm.h +++ b/src/Processors/Merges/Algorithms/IMergingAlgorithm.h @@ -21,7 +21,7 @@ public: }; virtual void initialize(Chunks chunks) = 0; - virtual void consume(Chunk chunk, size_t source_num) = 0; + virtual void consume(Chunk & chunk, size_t source_num) = 0; virtual Status merge() = 0; IMergingAlgorithm() = default; diff --git a/src/Processors/Merges/Algorithms/IMergingAlgorithmWithDelayedChunk.cpp b/src/Processors/Merges/Algorithms/IMergingAlgorithmWithDelayedChunk.cpp index 6777109982e..751a08ce69f 100644 --- a/src/Processors/Merges/Algorithms/IMergingAlgorithmWithDelayedChunk.cpp +++ b/src/Processors/Merges/Algorithms/IMergingAlgorithmWithDelayedChunk.cpp @@ -28,15 +28,15 @@ void IMergingAlgorithmWithDelayedChunk::initializeQueue(Chunks chunks) queue = SortingHeap(cursors); } -void IMergingAlgorithmWithDelayedChunk::updateCursor(Chunk chunk, size_t source_num) +void IMergingAlgorithmWithDelayedChunk::updateCursor(Chunk & chunk, size_t source_num) { auto & source_chunk = source_chunks[source_num]; /// Extend lifetime of last chunk. - last_chunk = std::move(source_chunk); + last_chunk.swap(source_chunk); last_chunk_sort_columns = std::move(cursors[source_num].sort_columns); - source_chunk = std::move(chunk); + source_chunk.swap(chunk); cursors[source_num].reset(source_chunk.getColumns(), {}); queue.push(cursors[source_num]); diff --git a/src/Processors/Merges/Algorithms/IMergingAlgorithmWithDelayedChunk.h b/src/Processors/Merges/Algorithms/IMergingAlgorithmWithDelayedChunk.h index d02b9dfcb7a..f7d5f630238 100644 --- a/src/Processors/Merges/Algorithms/IMergingAlgorithmWithDelayedChunk.h +++ b/src/Processors/Merges/Algorithms/IMergingAlgorithmWithDelayedChunk.h @@ -24,7 +24,7 @@ protected: ColumnRawPtrs last_chunk_sort_columns; /// Point to last_chunk if valid. void initializeQueue(Chunks chunks); - void updateCursor(Chunk chunk, size_t source_num); + void updateCursor(Chunk & chunk, size_t source_num); private: /// Chunks currently being merged. diff --git a/src/Processors/Merges/Algorithms/IMergingAlgorithmWithSharedChunks.cpp b/src/Processors/Merges/Algorithms/IMergingAlgorithmWithSharedChunks.cpp index bcea74b5f38..1fe61653ecc 100644 --- a/src/Processors/Merges/Algorithms/IMergingAlgorithmWithSharedChunks.cpp +++ b/src/Processors/Merges/Algorithms/IMergingAlgorithmWithSharedChunks.cpp @@ -39,7 +39,7 @@ void IMergingAlgorithmWithSharedChunks::initialize(Chunks chunks) auto & source_chunk = source_chunks[source_num]; - source_chunk = chunk_allocator.alloc(std::move(chunks[source_num])); + source_chunk = chunk_allocator.alloc(chunks[source_num]); cursors[source_num] = SortCursorImpl(source_chunk->getColumns(), description, source_num); source_chunk->all_columns = cursors[source_num].all_columns; @@ -49,12 +49,12 @@ void IMergingAlgorithmWithSharedChunks::initialize(Chunks chunks) queue = SortingHeap(cursors); } -void IMergingAlgorithmWithSharedChunks::consume(Chunk chunk, size_t source_num) +void IMergingAlgorithmWithSharedChunks::consume(Chunk & chunk, size_t source_num) { prepareChunk(chunk); auto & source_chunk = source_chunks[source_num]; - source_chunk = chunk_allocator.alloc(std::move(chunk)); + source_chunk = chunk_allocator.alloc(chunk); cursors[source_num].reset(source_chunk->getColumns(), {}); source_chunk->all_columns = cursors[source_num].all_columns; diff --git a/src/Processors/Merges/Algorithms/IMergingAlgorithmWithSharedChunks.h b/src/Processors/Merges/Algorithms/IMergingAlgorithmWithSharedChunks.h index 1ef7f540f96..a3dbadc458d 100644 --- a/src/Processors/Merges/Algorithms/IMergingAlgorithmWithSharedChunks.h +++ b/src/Processors/Merges/Algorithms/IMergingAlgorithmWithSharedChunks.h @@ -16,7 +16,7 @@ public: size_t max_row_refs); void initialize(Chunks chunks) override; - void consume(Chunk chunk, size_t source_num) override; + void consume(Chunk & chunk, size_t source_num) override; private: SortDescription description; diff --git a/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp index d06e3c1179a..78221b48255 100644 --- a/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp +++ b/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp @@ -74,10 +74,10 @@ void MergingSortedAlgorithm::initialize(Chunks chunks) queue_without_collation = SortingHeap(cursors); } -void MergingSortedAlgorithm::consume(Chunk chunk, size_t source_num) +void MergingSortedAlgorithm::consume(Chunk & chunk, size_t source_num) { prepareChunk(chunk); - source_chunks[source_num] = std::move(chunk); + source_chunks[source_num].swap(chunk); cursors[source_num].reset(source_chunks[source_num].getColumns(), {}); if (has_collation) diff --git a/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.h b/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.h index 6ff48b520bd..5b361c1000e 100644 --- a/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.h +++ b/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.h @@ -23,7 +23,7 @@ public: void addInput(); void initialize(Chunks chunks) override; - void consume(Chunk chunk, size_t source_num) override; + void consume(Chunk & chunk, size_t source_num) override; Status merge() override; const MergedData & getMergedData() const { return merged_data; } diff --git a/src/Processors/Merges/Algorithms/RowRef.h b/src/Processors/Merges/Algorithms/RowRef.h index 5aeae952067..658442e34cc 100644 --- a/src/Processors/Merges/Algorithms/RowRef.h +++ b/src/Processors/Merges/Algorithms/RowRef.h @@ -63,7 +63,7 @@ public: free_chunks.push_back(i); } - SharedChunkPtr alloc(Chunk && chunk) + SharedChunkPtr alloc(Chunk & chunk) { if (free_chunks.empty()) throw Exception("Not enough space in SharedChunkAllocator. " @@ -72,7 +72,7 @@ public: auto pos = free_chunks.back(); free_chunks.pop_back(); - chunks[pos] = std::move(chunk); + chunks[pos].swap(chunk); chunks[pos].position = pos; chunks[pos].allocator = this; @@ -110,9 +110,9 @@ private: } /// Release memory. It is not obligatory. - ptr->clear(); - ptr->all_columns.clear(); - ptr->sort_columns.clear(); +// ptr->clear(); +// ptr->all_columns.clear(); +// ptr->sort_columns.clear(); free_chunks.push_back(ptr->position); } diff --git a/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp index 5bcbf778d64..89154044ae5 100644 --- a/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp +++ b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp @@ -632,10 +632,10 @@ void SummingSortedAlgorithm::initialize(Chunks chunks) initializeQueue(std::move(chunks)); } -void SummingSortedAlgorithm::consume(Chunk chunk, size_t source_num) +void SummingSortedAlgorithm::consume(Chunk & chunk, size_t source_num) { preprocessChunk(chunk); - updateCursor(std::move(chunk), source_num); + updateCursor(chunk, source_num); } IMergingAlgorithm::Status SummingSortedAlgorithm::merge() diff --git a/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.h b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.h index a38df215ccc..fc5431f1a08 100644 --- a/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.h +++ b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.h @@ -23,7 +23,7 @@ public: size_t max_block_size); void initialize(Chunks chunks) override; - void consume(Chunk chunk, size_t source_num) override; + void consume(Chunk & chunk, size_t source_num) override; Status merge() override; struct AggregateDescription; diff --git a/src/Processors/Merges/CollapsingSortedTransform.h b/src/Processors/Merges/CollapsingSortedTransform.h index d4f40c60938..cdf7c4a1607 100644 --- a/src/Processors/Merges/CollapsingSortedTransform.h +++ b/src/Processors/Merges/CollapsingSortedTransform.h @@ -15,6 +15,7 @@ public: size_t num_inputs, SortDescription description_, const String & sign_column, + bool only_positive_sign, size_t max_block_size, WriteBuffer * out_row_sources_buf_ = nullptr, bool use_average_block_sizes = false) @@ -24,6 +25,7 @@ public: num_inputs, std::move(description_), sign_column, + only_positive_sign, max_block_size, out_row_sources_buf_, use_average_block_sizes, diff --git a/src/Processors/Merges/IMergingTransform.cpp b/src/Processors/Merges/IMergingTransform.cpp index 0dc4cd41991..f42bd44f6ca 100644 --- a/src/Processors/Merges/IMergingTransform.cpp +++ b/src/Processors/Merges/IMergingTransform.cpp @@ -1,4 +1,5 @@ #include +#include namespace DB { @@ -135,7 +136,6 @@ IProcessor::Status IMergingTransformBase::prepare() if (state.is_finished) { - if (is_port_full) return Status::PortFull; @@ -158,11 +158,11 @@ IProcessor::Status IMergingTransformBase::prepare() if (!input.hasData()) return Status::NeedData; - auto chunk = input.pull(); - if (!chunk.hasRows() && !input.isFinished()) + state.input_chunk = input.pull(); + if (!state.input_chunk.hasRows() && !input.isFinished()) return Status::NeedData; - state.input_chunk = std::move(chunk); + state.has_input = true; } state.need_data = false; @@ -174,4 +174,83 @@ IProcessor::Status IMergingTransformBase::prepare() return Status::Ready; } +static void filterChunk(Chunk & chunk, size_t selector_position) +{ + if (!chunk.getChunkInfo()) + throw Exception("IMergingTransformBase expected ChunkInfo for input chunk", ErrorCodes::LOGICAL_ERROR); + + const auto * chunk_info = typeid_cast(chunk.getChunkInfo().get()); + if (!chunk_info) + throw Exception("IMergingTransformBase expected SelectorInfo for input chunk", ErrorCodes::LOGICAL_ERROR); + + auto & selector = chunk_info->selector; + + IColumn::Filter filter; + filter.resize_fill(selector.size()); + + size_t num_rows = chunk.getNumRows(); + auto columns = chunk.detachColumns(); + + size_t num_result_rows = 0; + + for (size_t row = 0; row < num_rows; ++row) + { + if (selector[row] == selector_position) + { + ++num_result_rows; + filter[row] = 1; + } + } + + for (auto & column : columns) + column = column->filter(filter, num_result_rows); + + chunk.clear(); + chunk.setColumns(std::move(columns), num_result_rows); +} + +bool IMergingTransformBase::filterChunks() +{ + if (state.selector_position < 0) + return true; + + bool has_empty_chunk = false; + + if (!state.init_chunks.empty()) + { + for (size_t i = 0; i < input_states.size(); ++i) + { + auto & chunk = state.init_chunks[i]; + if (!chunk || input_states[i].is_filtered) + continue; + + filterChunk(chunk, state.selector_position); + + if (!chunk.hasRows()) + { + chunk.clear(); + has_empty_chunk = true; + input_states[i].is_initialized = false; + is_initialized = false; + } + else + input_states[i].is_filtered = true; + } + } + + if (state.has_input) + { + filterChunk(state.input_chunk, state.selector_position); + if (!state.input_chunk.hasRows()) + { + state.has_input = false; + state.need_data = true; + has_empty_chunk = true; + } + } + + return !has_empty_chunk; +} + + } diff --git a/src/Processors/Merges/IMergingTransform.h b/src/Processors/Merges/IMergingTransform.h index f9c2dba8271..939faa48d26 100644 --- a/src/Processors/Merges/IMergingTransform.h +++ b/src/Processors/Merges/IMergingTransform.h @@ -18,6 +18,10 @@ public: const Block & output_header, bool have_all_inputs_); + virtual ~IMergingTransformBase() = default; + + OutputPort & getOutputPort() { return outputs.front(); } + /// Methods to add additional input port. It is possible to do only before the first call of `prepare`. void addInput(); /// Need to be called after all inputs are added. (only if have_all_inputs was not specified). @@ -25,20 +29,29 @@ public: Status prepare() override; + /// Set position which will be used in selector if input chunk has attached SelectorInfo (see SelectorInfo.h). + /// Columns will be filtered, keep only rows labeled with this position. + /// It is used in parallel final. + void setSelectorPosition(size_t position) { state.selector_position = position; } + protected: virtual void onNewInput(); /// Is called when new input is added. Only if have_all_inputs = false. virtual void onFinish() {} /// Is called when all data is processed. + bool filterChunks(); /// Filter chunks if selector position was set. For parallel final. + /// Processor state. struct State { Chunk output_chunk; Chunk input_chunk; + bool has_input = false; bool is_finished = false; bool need_data = false; size_t next_input_to_read = 0; Chunks init_chunks; + ssize_t selector_position = -1; }; State state; @@ -50,6 +63,7 @@ private: InputPort & port; bool is_initialized = false; + bool is_filtered = false; }; std::vector input_states; @@ -78,14 +92,18 @@ public: void work() override { + if (!filterChunks()) + return; + if (!state.init_chunks.empty()) algorithm.initialize(std::move(state.init_chunks)); - if (state.input_chunk) + if (state.has_input) { // std::cerr << "Consume chunk with " << state.input_chunk.getNumRows() // << " for input " << state.next_input_to_read << std::endl; - algorithm.consume(std::move(state.input_chunk), state.next_input_to_read); + algorithm.consume(state.input_chunk, state.next_input_to_read); + state.has_input = false; } IMergingAlgorithm::Status status = algorithm.merge(); @@ -120,4 +138,6 @@ private: using IMergingTransformBase::state; }; +using MergingTransformPtr = std::shared_ptr; + } diff --git a/src/Processors/Pipe.cpp b/src/Processors/Pipe.cpp index d9b21dbc854..5d92e909a52 100644 --- a/src/Processors/Pipe.cpp +++ b/src/Processors/Pipe.cpp @@ -96,6 +96,15 @@ Pipe::Pipe(Pipes && pipes, ProcessorPtr transform) processors.emplace_back(std::move(transform)); } +Pipe::Pipe(OutputPort * port) : output_port(port) +{ +} + +void Pipe::addProcessors(const Processors & processors_) +{ + processors.insert(processors.end(), processors_.begin(), processors_.end()); +} + void Pipe::addSimpleTransform(ProcessorPtr transform) { checkSimpleTransform(*transform); diff --git a/src/Processors/Pipe.h b/src/Processors/Pipe.h index 42bbd4e06d0..984fa7605c6 100644 --- a/src/Processors/Pipe.h +++ b/src/Processors/Pipe.h @@ -22,6 +22,8 @@ public: /// Transform must have the number of inputs equals to the number of pipes. And single output. /// Will connect pipes outputs with transform inputs automatically. Pipe(Pipes && pipes, ProcessorPtr transform); + /// Create pipe from output port. If pipe was created that way, it possibly will not have tree shape. + Pipe(OutputPort * port); Pipe(const Pipe & other) = delete; Pipe(Pipe && other) = default; @@ -29,6 +31,9 @@ public: Pipe & operator=(const Pipe & other) = delete; Pipe & operator=(Pipe && other) = default; + /// Append processors to pipe. After this, it possibly will not have tree shape. + void addProcessors(const Processors & processors_); + OutputPort & getPort() const { return *output_port; } const Block & getHeader() const { return output_port->getHeader(); } diff --git a/src/Processors/Transforms/AddingSelectorTransform.cpp b/src/Processors/Transforms/AddingSelectorTransform.cpp new file mode 100644 index 00000000000..f75a5920072 --- /dev/null +++ b/src/Processors/Transforms/AddingSelectorTransform.cpp @@ -0,0 +1,76 @@ +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +AddingSelectorTransform::AddingSelectorTransform( + const Block & header, size_t num_outputs_, ColumnNumbers key_columns_) + : ISimpleTransform(header, header, false) + , num_outputs(num_outputs_) + , key_columns(std::move(key_columns_)) + , hash(0) +{ + setInputNotNeededAfterRead(false); + + if (num_outputs <= 1) + throw Exception("SplittingByHashTransform expects more than 1 outputs, got " + std::to_string(num_outputs), + ErrorCodes::LOGICAL_ERROR); + + if (key_columns.empty()) + throw Exception("SplittingByHashTransform cannot split by empty set of key columns", + ErrorCodes::LOGICAL_ERROR); + + for (auto & column : key_columns) + if (column >= header.columns()) + throw Exception("Invalid column number: " + std::to_string(column) + + ". There is only " + std::to_string(header.columns()) + " columns in header", + ErrorCodes::LOGICAL_ERROR); +} + +static void calculateWeakHash32(const Chunk & chunk, const ColumnNumbers & key_columns, WeakHash32 & hash) +{ + auto num_rows = chunk.getNumRows(); + const auto & columns = chunk.getColumns(); + + hash.reset(num_rows); + + for (const auto & column_number : key_columns) + columns[column_number]->updateWeakHash32(hash); +} + +static IColumn::Selector fillSelector(const WeakHash32 & hash, size_t num_outputs) +{ + /// Row from interval [(2^32 / num_outputs) * i, (2^32 / num_outputs) * (i + 1)) goes to bucket with number i. + + const auto & hash_data = hash.getData(); + size_t num_rows = hash_data.size(); + IColumn::Selector selector(num_rows); + + for (size_t row = 0; row < num_rows; ++row) + { + selector[row] = hash_data[row]; /// [0, 2^32) + selector[row] *= num_outputs; /// [0, num_outputs * 2^32), selector stores 64 bit values. + selector[row] >>= 32u; /// [0, num_outputs) + } + + return selector; +} + +void AddingSelectorTransform::transform(Chunk & input_chunk, Chunk & output_chunk) +{ + auto chunk_info = std::make_shared(); + + calculateWeakHash32(input_chunk, key_columns, hash); + chunk_info->selector = fillSelector(hash, num_outputs); + + input_chunk.swap(output_chunk); + output_chunk.setChunkInfo(std::move(chunk_info)); +} + +} diff --git a/src/Processors/Transforms/AddingSelectorTransform.h b/src/Processors/Transforms/AddingSelectorTransform.h new file mode 100644 index 00000000000..e82dcc964d0 --- /dev/null +++ b/src/Processors/Transforms/AddingSelectorTransform.h @@ -0,0 +1,26 @@ +#pragma once +#include +#include +#include +#include + +namespace DB +{ + +/// Add IColumn::Selector to chunk (see SelectorInfo.h). +/// Selector is filled by formula (WeakHash(key_columns) * num_outputs / MAX_INT). +class AddingSelectorTransform : public ISimpleTransform +{ +public: + AddingSelectorTransform(const Block & header, size_t num_outputs_, ColumnNumbers key_columns_); + String getName() const override { return "SplittingByHash"; } + void transform(Chunk & input_chunk, Chunk & output_chunk) override; + +private: + size_t num_outputs; + ColumnNumbers key_columns; + + WeakHash32 hash; +}; + +} diff --git a/src/Processors/Transforms/CopyTransform.cpp b/src/Processors/Transforms/CopyTransform.cpp new file mode 100644 index 00000000000..c9047c942d6 --- /dev/null +++ b/src/Processors/Transforms/CopyTransform.cpp @@ -0,0 +1,108 @@ +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +CopyTransform::CopyTransform(const Block & header, size_t num_outputs) + : IProcessor(InputPorts(1, header), OutputPorts(num_outputs, header)) +{ + if (num_outputs <= 1) + throw Exception("CopyTransform expects more than 1 outputs, got " + std::to_string(num_outputs), ErrorCodes::LOGICAL_ERROR); +} + +IProcessor::Status CopyTransform::prepare() +{ + Status status = Status::Ready; + + while (status == Status::Ready) + { + status = !has_data ? prepareConsume() + : prepareGenerate(); + } + + return status; +} + +IProcessor::Status CopyTransform::prepareConsume() +{ + auto & input = getInputPort(); + + /// Check all outputs are finished or ready to get data. + + bool all_finished = true; + for (auto & output : outputs) + { + if (output.isFinished()) + continue; + + all_finished = false; + } + + if (all_finished) + { + input.close(); + return Status::Finished; + } + + /// Try get chunk from input. + + if (input.isFinished()) + { + for (auto & output : outputs) + output.finish(); + + return Status::Finished; + } + + input.setNeeded(); + if (!input.hasData()) + return Status::NeedData; + + chunk = input.pull(); + has_data = true; + was_output_processed.assign(outputs.size(), false); + + return Status::Ready; +} + +IProcessor::Status CopyTransform::prepareGenerate() +{ + bool all_outputs_processed = true; + + size_t chunk_number = 0; + for (auto & output : outputs) + { + auto & was_processed = was_output_processed[chunk_number]; + ++chunk_number; + + if (was_processed) + continue; + + if (output.isFinished()) + continue; + + if (!output.canPush()) + { + all_outputs_processed = false; + continue; + } + + output.push(chunk.clone()); + was_processed = true; + } + + if (all_outputs_processed) + { + has_data = false; + return Status::Ready; + } + + return Status::PortFull; +} + +} diff --git a/src/Processors/Transforms/CopyTransform.h b/src/Processors/Transforms/CopyTransform.h new file mode 100644 index 00000000000..cf56fdf10d9 --- /dev/null +++ b/src/Processors/Transforms/CopyTransform.h @@ -0,0 +1,28 @@ +#pragma once +#include + +namespace DB +{ + +/// Transform which has single input and num_outputs outputs. +/// Read chunk from input and copy it to all outputs. +class CopyTransform : public IProcessor +{ +public: + CopyTransform(const Block & header, size_t num_outputs); + + String getName() const override { return "Copy"; } + Status prepare() override; + + InputPort & getInputPort() { return inputs.front(); } + +private: + Chunk chunk; + bool has_data = false; + std::vector was_output_processed; + + Status prepareGenerate(); + Status prepareConsume(); +}; + +} diff --git a/src/Processors/Transforms/SelectorInfo.h b/src/Processors/Transforms/SelectorInfo.h new file mode 100644 index 00000000000..2876d64ed28 --- /dev/null +++ b/src/Processors/Transforms/SelectorInfo.h @@ -0,0 +1,14 @@ +#pragma once +#include +#include + +namespace DB +{ + +/// ChunkInfo with IColumn::Selector. It is added by AddingSelectorTransform. +struct SelectorInfo : public ChunkInfo +{ + IColumn::Selector selector; +}; + +} diff --git a/src/Processors/ya.make b/src/Processors/ya.make index 6469532a8e8..7818f2fb183 100644 --- a/src/Processors/ya.make +++ b/src/Processors/ya.make @@ -106,8 +106,10 @@ SRCS( Sources/SourceFromInputStream.cpp Sources/SourceWithProgress.cpp Transforms/AddingMissedTransform.cpp + Transforms/AddingSelectorTransform.cpp Transforms/AggregatingTransform.cpp Transforms/ConvertingTransform.cpp + Transforms/CopyTransform.cpp Transforms/CreatingSetsTransform.cpp Transforms/CubeTransform.cpp Transforms/DistinctTransform.cpp diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 2ab43f8f56c..f1c1904256e 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -752,7 +752,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor case MergeTreeData::MergingParams::Collapsing: merged_transform = std::make_unique( - header, pipes.size(), sort_description, data.merging_params.sign_column, + header, pipes.size(), sort_description, data.merging_params.sign_column, false, merge_block_size, rows_sources_write_buf.get(), blocks_are_granules_size); break; diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 580c95b34dd..4eeb954bd7d 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -41,22 +41,25 @@ namespace std #include #include #include -#include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include namespace ProfileEvents { @@ -617,6 +620,7 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( res = spreadMarkRangesAmongStreamsFinal( std::move(parts_with_ranges), + num_streams, column_names_to_read, max_block_size, settings.use_uncompressed_cache, @@ -1017,6 +1021,7 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder( Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal( RangesInDataParts && parts, + size_t num_streams, const Names & column_names, UInt64 max_block_size, bool use_uncompressed_cache, @@ -1074,71 +1079,122 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal( for (size_t i = 0; i < sort_columns_size; ++i) sort_description.emplace_back(header.getPositionByName(sort_columns[i]), 1, 1); - /// Converts pipes to BlockInputsStreams. - /// It is temporary, till not all merging streams are implemented as processors. - auto streams_to_merge = [&pipes]() + auto get_merging_processor = [&]() -> MergingTransformPtr { - size_t num_streams = pipes.size(); - - BlockInputStreams streams; - streams.reserve(num_streams); - - for (size_t i = 0; i < num_streams; ++i) - streams.emplace_back(std::make_shared(std::move(pipes[i]))); - - pipes.clear(); - return streams; - }; - - BlockInputStreamPtr merged; - ProcessorPtr merged_processor; - switch (data.merging_params.mode) - { - case MergeTreeData::MergingParams::Ordinary: + switch (data.merging_params.mode) { - merged_processor = std::make_shared(header, pipes.size(), - sort_description, max_block_size); - break; + case MergeTreeData::MergingParams::Ordinary: + { + return std::make_shared(header, pipes.size(), + sort_description, max_block_size); + } + + case MergeTreeData::MergingParams::Collapsing: + return std::make_shared(header, pipes.size(), + sort_description, data.merging_params.sign_column, true, max_block_size); + + case MergeTreeData::MergingParams::Summing: + return std::make_shared(header, pipes.size(), + sort_description, data.merging_params.columns_to_sum, max_block_size); + + case MergeTreeData::MergingParams::Aggregating: + return std::make_shared(header, pipes.size(), + sort_description, max_block_size); + + case MergeTreeData::MergingParams::Replacing: + return std::make_shared(header, pipes.size(), + sort_description, data.merging_params.version_column, max_block_size); + + case MergeTreeData::MergingParams::VersionedCollapsing: + return std::make_shared(header, pipes.size(), + sort_description, data.merging_params.sign_column, max_block_size); + + case MergeTreeData::MergingParams::Graphite: + throw Exception("GraphiteMergeTree doesn't support FINAL", ErrorCodes::LOGICAL_ERROR); } - case MergeTreeData::MergingParams::Collapsing: - merged = std::make_shared( - streams_to_merge(), sort_description, data.merging_params.sign_column); - break; + __builtin_unreachable(); + }; - case MergeTreeData::MergingParams::Summing: - merged_processor = std::make_shared(header, pipes.size(), - sort_description, data.merging_params.columns_to_sum, max_block_size); - break; + if (num_streams > settings.max_final_threads) + num_streams = settings.max_final_threads; - case MergeTreeData::MergingParams::Aggregating: - merged_processor = std::make_shared(header, pipes.size(), - sort_description, max_block_size); - break; - - case MergeTreeData::MergingParams::Replacing: /// TODO Make ReplacingFinalBlockInputStream - merged_processor = std::make_shared(header, pipes.size(), - sort_description, data.merging_params.version_column, max_block_size); - break; - - case MergeTreeData::MergingParams::VersionedCollapsing: /// TODO Make VersionedCollapsingFinalBlockInputStream - merged_processor = std::make_shared(header, pipes.size(), - sort_description, data.merging_params.sign_column, max_block_size); - break; - - case MergeTreeData::MergingParams::Graphite: - throw Exception("GraphiteMergeTree doesn't support FINAL", ErrorCodes::LOGICAL_ERROR); - } - - if (merged_processor) + if (num_streams <= 1 || sort_description.empty() || query_info.force_tree_shaped_pipeline) { - Pipe pipe(std::move(pipes), std::move(merged_processor)); + + Pipe pipe(std::move(pipes), get_merging_processor()); pipes = Pipes(); pipes.emplace_back(std::move(pipe)); + + return pipes; } - if (merged) - pipes.emplace_back(std::make_shared(merged)); + ColumnNumbers key_columns; + key_columns.reserve(sort_description.size()); + + for (auto & desc : sort_description) + { + if (!desc.column_name.empty()) + key_columns.push_back(header.getPositionByName(desc.column_name)); + else + key_columns.emplace_back(desc.column_number); + } + + Processors selectors; + Processors copiers; + selectors.reserve(pipes.size()); + + for (auto & pipe : pipes) + { + auto selector = std::make_shared(pipe.getHeader(), num_streams, key_columns); + auto copier = std::make_shared(pipe.getHeader(), num_streams); + connect(pipe.getPort(), selector->getInputPort()); + connect(selector->getOutputPort(), copier->getInputPort()); + selectors.emplace_back(std::move(selector)); + copiers.emplace_back(std::move(copier)); + } + + Processors merges; + std::vector input_ports; + merges.reserve(num_streams); + input_ports.reserve(num_streams); + + for (size_t i = 0; i < num_streams; ++i) + { + auto merge = get_merging_processor(); + merge->setSelectorPosition(i); + input_ports.emplace_back(merge->getInputs().begin()); + merges.emplace_back(std::move(merge)); + } + + /// Connect outputs of i-th splitter with i-th input port of every merge. + for (auto & resize : copiers) + { + size_t input_num = 0; + for (auto & output : resize->getOutputs()) + { + connect(output, *input_ports[input_num]); + ++input_ports[input_num]; + ++input_num; + } + } + + Processors processors; + for (auto & pipe : pipes) + { + auto pipe_processors = std::move(pipe).detachProcessors(); + processors.insert(processors.end(), pipe_processors.begin(), pipe_processors.end()); + } + + pipes.clear(); + pipes.reserve(num_streams); + for (auto & merge : merges) + pipes.emplace_back(&merge->getOutputs().front()); + + pipes.front().addProcessors(processors); + pipes.front().addProcessors(selectors); + pipes.front().addProcessors(copiers); + pipes.front().addProcessors(merges); return pipes; } diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h index e6eb26da7e3..7ded8fcfad5 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h @@ -71,6 +71,7 @@ private: Pipes spreadMarkRangesAmongStreamsFinal( RangesInDataParts && parts, + size_t num_streams, const Names & column_names, UInt64 max_block_size, bool use_uncompressed_cache, diff --git a/tests/performance/parallel_final.xml b/tests/performance/parallel_final.xml new file mode 100644 index 00000000000..f5e7ff51f6e --- /dev/null +++ b/tests/performance/parallel_final.xml @@ -0,0 +1,51 @@ + + + + 1024 + + + + + collapsing + + collapsing_final_16p_ord + collapsing_final_16p_rnd + collapsing_final_16p_int_keys_ord + collapsing_final_16p_int_keys_rnd + collapsing_final_16p_str_keys_ord + collapsing_final_16p_str_keys_rnd + collapsing_final_1024p_ord + collapsing_final_1024p_rnd + + + + + create table collapsing_final_16p_ord (key1 UInt32, key2 String, sign Int8, s UInt64) engine = CollapsingMergeTree(sign) order by (key1, key2) partition by intDiv(key1, 8192 * 64) + create table collapsing_final_16p_rnd (key1 UInt32, key2 String, sign Int8, s UInt64) engine = CollapsingMergeTree(sign) order by (key1, key2) partition by key1 % 16 + create table collapsing_final_16p_int_keys_ord (key1 UInt32, key2 UInt32, key3 UInt32, key4 UInt32, key5 UInt32, key6 UInt32, key7 UInt32, key8 UInt32, sign Int8, s UInt64) engine = CollapsingMergeTree(sign) order by (key1, key2, key3, key4, key5, key6, key7, key8) partition by intDiv(key1, 8192 * 64) + create table collapsing_final_16p_int_keys_rnd (key1 UInt32, key2 UInt32, key3 UInt32, key4 UInt32, key5 UInt32, key6 UInt32, key7 UInt32, key8 UInt32, sign Int8, s UInt64) engine = CollapsingMergeTree(sign) order by (key1, key2, key3, key4, key5, key6, key7, key8) partition by key1 % 16 + create table collapsing_final_16p_str_keys_ord (key1 UInt32, key2 String, key3 String, key4 String, key5 String, key6 String, key7 String, key8 String, sign Int8, s UInt64) engine = CollapsingMergeTree(sign) order by (key1, key2, key3, key4, key5, key6, key7, key8) partition by intDiv(key1, 8192 * 64) + create table collapsing_final_16p_str_keys_rnd (key1 UInt32, key2 String, key3 String, key4 String, key5 String, key6 String, key7 String, key8 String, sign Int8, s UInt64) engine = CollapsingMergeTree(sign) order by (key1, key2, key3, key4, key5, key6, key7, key8) partition by key1 % 16 + create table collapsing_final_1024p_ord (key1 UInt32, sign Int8, s UInt64) engine = CollapsingMergeTree(sign) order by (key1) partition by intDiv(key1, 8192 * 2) + create table collapsing_final_1024p_rnd (key1 UInt32, sign Int8, s UInt64) engine = CollapsingMergeTree(sign) order by (key1) partition by key1 % 1024 + + + insert into collapsing_final_16p_ord select number, number, 1, number from numbers(8388608) + insert into collapsing_final_16p_rnd select sipHash64(number), number, 1, number from numbers(8388608) + insert into collapsing_final_16p_int_keys_ord select number, number, number, number, number, number, number, number, 1, number from numbers(8388608) + insert into collapsing_final_16p_int_keys_rnd select sipHash64(number), number, number, number, number, number, number, number, 1, number from numbers(8388608) + insert into collapsing_final_16p_str_keys_ord select number, number, number, number, number, number, number, number, 1, number from numbers(8388608) + insert into collapsing_final_16p_str_keys_rnd select sipHash64(number), number, number, number, number, number, number, number, 1, number from numbers(8388608) + + + insert into collapsing_final_1024p_ord select number, 1, number from numbers(16777216) + insert into collapsing_final_1024p_rnd select number, 1, number from numbers(16777216) + + optimize table {collapsing} final + + SELECT count() FROM {collapsing} final + SELECT sum(s) FROM {collapsing} final group by key1 limit 10 + SELECT sum(s) FROM {collapsing} final group by key1 % 8192 limit 10 + + DROP TABLE IF EXISTS {collapsing} + diff --git a/tests/queries/0_stateless/00191_aggregating_merge_tree_and_final.sql b/tests/queries/0_stateless/00191_aggregating_merge_tree_and_final.sql index 47b96006113..776edeeb43c 100644 --- a/tests/queries/0_stateless/00191_aggregating_merge_tree_and_final.sql +++ b/tests/queries/0_stateless/00191_aggregating_merge_tree_and_final.sql @@ -4,11 +4,11 @@ CREATE TABLE aggregating_00191 (d Date DEFAULT '2000-01-01', k UInt64, u Aggrega INSERT INTO aggregating_00191 (k, u) SELECT intDiv(number, 100) AS k, uniqState(toUInt64(number % 100)) AS u FROM (SELECT * FROM system.numbers LIMIT 1000) GROUP BY k; INSERT INTO aggregating_00191 (k, u) SELECT intDiv(number, 100) AS k, uniqState(toUInt64(number % 100) + 50) AS u FROM (SELECT * FROM system.numbers LIMIT 500, 1000) GROUP BY k; -SELECT k, finalizeAggregation(u) FROM aggregating_00191 FINAL; +SELECT k, finalizeAggregation(u) FROM aggregating_00191 FINAL order by k; OPTIMIZE TABLE aggregating_00191; SELECT k, finalizeAggregation(u) FROM aggregating_00191; -SELECT k, finalizeAggregation(u) FROM aggregating_00191 FINAL; +SELECT k, finalizeAggregation(u) FROM aggregating_00191 FINAL order by k; DROP TABLE aggregating_00191; diff --git a/tests/queries/0_stateless/00564_versioned_collapsing_merge_tree.sql b/tests/queries/0_stateless/00564_versioned_collapsing_merge_tree.sql index 5b7f59f8b65..b7824e7efdc 100644 --- a/tests/queries/0_stateless/00564_versioned_collapsing_merge_tree.sql +++ b/tests/queries/0_stateless/00564_versioned_collapsing_merge_tree.sql @@ -3,7 +3,7 @@ create table mult_tab (date Date, value String, version UInt64, sign Int8) engin insert into mult_tab select '2018-01-31', 'str_' || toString(number), 0, if(number % 2, 1, -1) from system.numbers limit 10; insert into mult_tab select '2018-01-31', 'str_' || toString(number), 0, if(number % 2, 1, -1) from system.numbers limit 10; select 'table with 2 blocks final'; -select * from mult_tab final; +select * from mult_tab final order by date, value, sign; optimize table mult_tab; select 'table with 2 blocks optimized'; select * from mult_tab; @@ -15,7 +15,7 @@ create table mult_tab (date Date, value String, version UInt64, sign Int8) engin insert into mult_tab select '2018-01-31', 'str_' || toString(number), 0, if(number % 2, 1, -1) from system.numbers limit 10; insert into mult_tab select '2018-01-31', 'str_' || toString(number), 0, if(number % 2, 1, -1) from system.numbers limit 10; select 'table with 2 blocks final'; -select * from mult_tab final; +select * from mult_tab final order by date, value, sign; optimize table mult_tab; select 'table with 2 blocks optimized'; select * from mult_tab; @@ -27,7 +27,7 @@ create table mult_tab (date Date, value String, version UInt64, sign Int8) engin insert into mult_tab select '2018-01-31', 'str_' || toString(number), 0, if(number % 2, 1, -1) from system.numbers limit 10; insert into mult_tab select '2018-01-31', 'str_' || toString(number), 0, if(number % 2, -1, 1) from system.numbers limit 10; select 'table with 2 blocks final'; -select * from mult_tab final; +select * from mult_tab final order by date, value, sign; optimize table mult_tab; select 'table with 2 blocks optimized'; select * from mult_tab; @@ -39,7 +39,7 @@ create table mult_tab (date Date, value String, version UInt64, sign Int8) engin insert into mult_tab select '2018-01-31', 'str_' || toString(number), 0, if(number % 2, 1, -1) from system.numbers limit 10; insert into mult_tab select '2018-01-31', 'str_' || toString(number), 1, if(number % 2, -1, 1) from system.numbers limit 10; select 'table with 2 blocks final'; -select * from mult_tab final; +select * from mult_tab final order by date, value, version, sign; optimize table mult_tab; select 'table with 2 blocks optimized'; select * from mult_tab; @@ -53,7 +53,7 @@ insert into mult_tab select '2018-01-31', 'str_' || toString(number), 0, if(numb insert into mult_tab select '2018-01-31', 'str_' || toString(number), 0, if(number % 2, 1, -1) from system.numbers limit 10; insert into mult_tab select '2018-01-31', 'str_' || toString(number), 0, if(number % 2, -1, 1) from system.numbers limit 10; select 'table with 4 blocks final'; -select * from mult_tab final; +select * from mult_tab final order by date, value, sign; optimize table mult_tab; select 'table with 4 blocks optimized'; select * from mult_tab; @@ -68,7 +68,7 @@ insert into mult_tab select '2018-01-31', 'str_' || toString(number), 1, if(numb insert into mult_tab select '2018-01-31', 'str_' || toString(number), 1, if(number % 3 = 1, 1, -1) from system.numbers limit 10; insert into mult_tab select '2018-01-31', 'str_' || toString(number), 1, if(number % 3 = 2, 1, -1) from system.numbers limit 10; select 'table with 5 blocks final'; -select * from mult_tab final; +select * from mult_tab final order by date, value, sign; optimize table mult_tab; select 'table with 5 blocks optimized'; select * from mult_tab; @@ -80,7 +80,7 @@ create table mult_tab (date Date, value String, version UInt64, sign Int8) engin insert into mult_tab select '2018-01-31', 'str_' || toString(number), 0, if(number % 2, 1, -1) from system.numbers limit 1000000; insert into mult_tab select '2018-01-31', 'str_' || toString(number), 0, if(number % 2, -1, 1) from system.numbers limit 1000000; select 'table with 2 blocks final'; -select * from mult_tab final; +select * from mult_tab final order by date, value, sign; optimize table mult_tab; select 'table with 2 blocks optimized'; select * from mult_tab; @@ -88,14 +88,14 @@ select * from mult_tab; select '-------------------------'; drop table if exists mult_tab; -create table mult_tab (date Date, value UInt64, version UInt64, sign Int8) engine = VersionedCollapsingMergeTree(date, (date), 8192, sign, version); -insert into mult_tab select '2018-01-31', number, 0, if(number < 64, 1, -1) from system.numbers limit 128; -insert into mult_tab select '2018-01-31', number, 0, if(number < 64, -1, 1) from system.numbers limit 128; +create table mult_tab (date Date, value UInt64, key UInt64, version UInt64, sign Int8) engine = VersionedCollapsingMergeTree(date, (date), 8192, sign, version); +insert into mult_tab select '2018-01-31', number, number, 0, if(number < 64, 1, -1) from system.numbers limit 128; +insert into mult_tab select '2018-01-31', number, number + 128, 0, if(number < 64, -1, 1) from system.numbers limit 128; select 'table with 2 blocks final'; -select * from mult_tab final settings max_block_size=33; +select date, value, version, sign from mult_tab final order by date, key, sign settings max_block_size=33; optimize table mult_tab; select 'table with 2 blocks optimized'; -select * from mult_tab; +select date, value, version, sign from mult_tab; select '-------------------------'; select 'Vertival merge'; @@ -106,7 +106,7 @@ create table mult_tab (date Date, value String, version UInt64, sign Int8) engin insert into mult_tab select '2018-01-31', 'str_' || toString(number), 0, if(number % 2, 1, -1) from system.numbers limit 10; insert into mult_tab select '2018-01-31', 'str_' || toString(number), 0, if(number % 2, 1, -1) from system.numbers limit 10; select 'table with 2 blocks final'; -select * from mult_tab final; +select * from mult_tab final order by date, value, sign; optimize table mult_tab; select 'table with 2 blocks optimized'; select * from mult_tab; @@ -118,7 +118,7 @@ create table mult_tab (date Date, value String, version UInt64, sign Int8) engin insert into mult_tab select '2018-01-31', 'str_' || toString(number), 0, if(number % 2, 1, -1) from system.numbers limit 10; insert into mult_tab select '2018-01-31', 'str_' || toString(number), 0, if(number % 2, 1, -1) from system.numbers limit 10; select 'table with 2 blocks final'; -select * from mult_tab final; +select * from mult_tab final order by date, value, sign; optimize table mult_tab; select 'table with 2 blocks optimized'; select * from mult_tab; @@ -130,7 +130,7 @@ create table mult_tab (date Date, value String, version UInt64, sign Int8) engin insert into mult_tab select '2018-01-31', 'str_' || toString(number), 0, if(number % 2, 1, -1) from system.numbers limit 10; insert into mult_tab select '2018-01-31', 'str_' || toString(number), 0, if(number % 2, -1, 1) from system.numbers limit 10; select 'table with 2 blocks final'; -select * from mult_tab final; +select * from mult_tab final order by date, value, sign; optimize table mult_tab; select 'table with 2 blocks optimized'; select * from mult_tab; @@ -142,7 +142,7 @@ create table mult_tab (date Date, value String, version UInt64, sign Int8) engin insert into mult_tab select '2018-01-31', 'str_' || toString(number), 0, if(number % 2, 1, -1) from system.numbers limit 10; insert into mult_tab select '2018-01-31', 'str_' || toString(number), 1, if(number % 2, -1, 1) from system.numbers limit 10; select 'table with 2 blocks final'; -select * from mult_tab final; +select * from mult_tab final order by date, value, version, sign; optimize table mult_tab; select 'table with 2 blocks optimized'; select * from mult_tab; @@ -156,7 +156,7 @@ insert into mult_tab select '2018-01-31', 'str_' || toString(number), 0, if(numb insert into mult_tab select '2018-01-31', 'str_' || toString(number), 0, if(number % 2, 1, -1) from system.numbers limit 10; insert into mult_tab select '2018-01-31', 'str_' || toString(number), 0, if(number % 2, -1, 1) from system.numbers limit 10; select 'table with 4 blocks final'; -select * from mult_tab final; +select * from mult_tab final order by date, value, sign; optimize table mult_tab; select 'table with 4 blocks optimized'; select * from mult_tab; @@ -171,7 +171,7 @@ insert into mult_tab select '2018-01-31', 'str_' || toString(number), 1, if(numb insert into mult_tab select '2018-01-31', 'str_' || toString(number), 1, if(number % 3 = 1, 1, -1) from system.numbers limit 10; insert into mult_tab select '2018-01-31', 'str_' || toString(number), 1, if(number % 3 = 2, 1, -1) from system.numbers limit 10; select 'table with 5 blocks final'; -select * from mult_tab final; +select * from mult_tab final order by date, value, sign; optimize table mult_tab; select 'table with 5 blocks optimized'; select * from mult_tab; @@ -183,7 +183,7 @@ create table mult_tab (date Date, value String, version UInt64, sign Int8) engin insert into mult_tab select '2018-01-31', 'str_' || toString(number), 0, if(number % 2, 1, -1) from system.numbers limit 1000000; insert into mult_tab select '2018-01-31', 'str_' || toString(number), 0, if(number % 2, -1, 1) from system.numbers limit 1000000; select 'table with 2 blocks final'; -select * from mult_tab final; +select * from mult_tab final order by date, value, sign; optimize table mult_tab; select 'table with 2 blocks optimized'; select * from mult_tab; @@ -191,13 +191,13 @@ select * from mult_tab; select '-------------------------'; drop table if exists mult_tab; -create table mult_tab (date Date, value UInt64, version UInt64, sign Int8) engine = VersionedCollapsingMergeTree(sign, version) order by (date) settings enable_vertical_merge_algorithm = 1, vertical_merge_algorithm_min_rows_to_activate = 1, vertical_merge_algorithm_min_columns_to_activate = 0; -insert into mult_tab select '2018-01-31', number, 0, if(number < 64, 1, -1) from system.numbers limit 128; -insert into mult_tab select '2018-01-31', number, 0, if(number < 64, -1, 1) from system.numbers limit 128; +create table mult_tab (date Date, value UInt64, key UInt64, version UInt64, sign Int8) engine = VersionedCollapsingMergeTree(sign, version) order by (date) settings enable_vertical_merge_algorithm = 1, vertical_merge_algorithm_min_rows_to_activate = 1, vertical_merge_algorithm_min_columns_to_activate = 0; +insert into mult_tab select '2018-01-31', number, number, 0, if(number < 64, 1, -1) from system.numbers limit 128; +insert into mult_tab select '2018-01-31', number, number + 128, 0, if(number < 64, -1, 1) from system.numbers limit 128; select 'table with 2 blocks final'; -select * from mult_tab final settings max_block_size=33; +select date, value, version, sign from mult_tab final order by date, key, sign settings max_block_size=33; optimize table mult_tab; select 'table with 2 blocks optimized'; -select * from mult_tab; +select date, value, version, sign from mult_tab; DROP TABLE mult_tab; diff --git a/tests/queries/0_stateless/00915_simple_aggregate_function.sql b/tests/queries/0_stateless/00915_simple_aggregate_function.sql index 1866e2bc8c5..ba4935a6518 100644 --- a/tests/queries/0_stateless/00915_simple_aggregate_function.sql +++ b/tests/queries/0_stateless/00915_simple_aggregate_function.sql @@ -5,13 +5,13 @@ create table simple (id UInt64,val SimpleAggregateFunction(sum,Double)) engine=A insert into simple select number,number from system.numbers limit 10; select * from simple; -select * from simple final; +select * from simple final order by id; select toTypeName(val) from simple limit 1; -- merge insert into simple select number,number from system.numbers limit 10; -select * from simple final; +select * from simple final order by id; optimize table simple final; select * from simple; @@ -33,7 +33,7 @@ insert into simple values(1,null,'2','2.2.2.2', 2, ([1,3], [1,1])); insert into simple values(10,'10','10','10.10.10.10', 4, ([2,3], [1,1])); insert into simple values(10,'2222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222','20','20.20.20.20', 1, ([2, 4], [1,1])); -select * from simple final; +select * from simple final order by id; select toTypeName(nullable_str),toTypeName(low_str),toTypeName(ip),toTypeName(status), toTypeName(tup) from simple limit 1; optimize table simple final; diff --git a/tests/queries/0_stateless/01030_incorrect_count_summing_merge_tree.sql b/tests/queries/0_stateless/01030_incorrect_count_summing_merge_tree.sql index a9f7bf7ecd7..0b5845d3b04 100644 --- a/tests/queries/0_stateless/01030_incorrect_count_summing_merge_tree.sql +++ b/tests/queries/0_stateless/01030_incorrect_count_summing_merge_tree.sql @@ -4,7 +4,7 @@ drop table if exists tst; create table tst (timestamp DateTime, val Nullable(Int8)) engine SummingMergeTree partition by toYYYYMM(timestamp) ORDER by (timestamp); insert into tst values ('2018-02-01 00:00:00', 1), ('2018-02-02 00:00:00', 2); -select * from tst final; +select * from tst final order by timestamp; select '-- 2 2'; select count() from tst; @@ -34,7 +34,7 @@ drop table if exists tst; create table tst (timestamp DateTime, val Nullable(Int8)) engine SummingMergeTree partition by toYYYYMM(timestamp) ORDER by (timestamp); insert into tst values ('2018-02-01 00:00:00', 1), ('2018-02-02 00:00:00', 2), ('2018-02-01 00:00:00', 3), ('2018-02-02 00:00:00', 4); -select * from tst final; +select * from tst final order by timestamp; select '-- 4 2'; select count() from tst; @@ -64,7 +64,7 @@ drop table if exists tst; create table tst (timestamp DateTime, val Int8) engine SummingMergeTree partition by toYYYYMM(timestamp) ORDER by (timestamp); insert into tst values ('2018-02-01 00:00:00', 1), ('2018-02-02 00:00:00', 2); -select * from tst final; +select * from tst final order by timestamp; select '-- 2 2'; select count() from tst; @@ -96,7 +96,7 @@ drop table if exists tst; create table tst (timestamp DateTime, val Int8) engine SummingMergeTree partition by toYYYYMM(timestamp) ORDER by (timestamp); insert into tst values ('2018-02-01 00:00:00', 1), ('2018-02-02 00:00:00', 2), ('2018-02-01 00:00:00', 3), ('2018-02-02 00:00:00', 4); -select * from tst final; +select * from tst final order by timestamp; select '-- 4 2'; select count() from tst; From 62e9be7ba8b180f79175e8304ca8db4f7b4b2f5f Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 30 Apr 2020 14:33:27 +0300 Subject: [PATCH 017/738] Fix build. --- src/Processors/Merges/IMergingTransform.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Processors/Merges/IMergingTransform.h b/src/Processors/Merges/IMergingTransform.h index 939faa48d26..12a366bf21b 100644 --- a/src/Processors/Merges/IMergingTransform.h +++ b/src/Processors/Merges/IMergingTransform.h @@ -18,8 +18,6 @@ public: const Block & output_header, bool have_all_inputs_); - virtual ~IMergingTransformBase() = default; - OutputPort & getOutputPort() { return outputs.front(); } /// Methods to add additional input port. It is possible to do only before the first call of `prepare`. From 10afbc22b5db3f97ab2deacc70bc0ad5ebd99b22 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 30 Apr 2020 16:36:20 +0300 Subject: [PATCH 018/738] Fix build. --- src/Processors/Merges/IMergingTransform.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/Merges/IMergingTransform.cpp b/src/Processors/Merges/IMergingTransform.cpp index f42bd44f6ca..2037a88733a 100644 --- a/src/Processors/Merges/IMergingTransform.cpp +++ b/src/Processors/Merges/IMergingTransform.cpp @@ -183,7 +183,7 @@ static void filterChunk(Chunk & chunk, size_t selector_position) if (!chunk_info) throw Exception("IMergingTransformBase expected SelectorInfo for input chunk", ErrorCodes::LOGICAL_ERROR); - auto & selector = chunk_info->selector; + const auto & selector = chunk_info->selector; IColumn::Filter filter; filter.resize_fill(selector.size()); From 047a78046e573fbb533e76901b4771bf9f47d74f Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 30 Apr 2020 20:58:09 +0300 Subject: [PATCH 019/738] Try make tests faster. --- tests/performance/parallel_final.xml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/performance/parallel_final.xml b/tests/performance/parallel_final.xml index f5e7ff51f6e..bd6a921fc68 100644 --- a/tests/performance/parallel_final.xml +++ b/tests/performance/parallel_final.xml @@ -2,6 +2,7 @@ 1024 + 16 @@ -30,16 +31,16 @@ create table collapsing_final_1024p_rnd (key1 UInt32, sign Int8, s UInt64) engine = CollapsingMergeTree(sign) order by (key1) partition by key1 % 1024 - insert into collapsing_final_16p_ord select number, number, 1, number from numbers(8388608) - insert into collapsing_final_16p_rnd select sipHash64(number), number, 1, number from numbers(8388608) - insert into collapsing_final_16p_int_keys_ord select number, number, number, number, number, number, number, number, 1, number from numbers(8388608) - insert into collapsing_final_16p_int_keys_rnd select sipHash64(number), number, number, number, number, number, number, number, 1, number from numbers(8388608) - insert into collapsing_final_16p_str_keys_ord select number, number, number, number, number, number, number, number, 1, number from numbers(8388608) - insert into collapsing_final_16p_str_keys_rnd select sipHash64(number), number, number, number, number, number, number, number, 1, number from numbers(8388608) + insert into collapsing_final_16p_ord select number, number, 1, number from numbers_mt(8388608) + insert into collapsing_final_16p_rnd select sipHash64(number), number, 1, number from numbers_mt(8388608) + insert into collapsing_final_16p_int_keys_ord select number, number, number, number, number, number, number, number, 1, number from numbers_mt(8388608) + insert into collapsing_final_16p_int_keys_rnd select sipHash64(number), number, number, number, number, number, number, number, 1, number from numbers_mt(8388608) + insert into collapsing_final_16p_str_keys_ord select number, number, number, number, number, number, number, number, 1, number from numbers_mt(8388608) + insert into collapsing_final_16p_str_keys_rnd select sipHash64(number), number, number, number, number, number, number, number, 1, number from numbers_mt(8388608) - insert into collapsing_final_1024p_ord select number, 1, number from numbers(16777216) - insert into collapsing_final_1024p_rnd select number, 1, number from numbers(16777216) + insert into collapsing_final_1024p_ord select number, 1, number from numbers_mt(16777216) + insert into collapsing_final_1024p_rnd select number, 1, number from numbers_mt(16777216) optimize table {collapsing} final From c12cd35099213cb44fa78911b12a4405bbb28e01 Mon Sep 17 00:00:00 2001 From: Alexey Ilyukhov Date: Sat, 2 May 2020 09:51:10 +0300 Subject: [PATCH 020/738] Add point in polygon for non-const polygons --- src/Functions/pointInPolygon.cpp | 177 ++++++++++++++---- ..._point_in_polygon_non_const_poly.reference | 68 +++++++ .../00500_point_in_polygon_non_const_poly.sql | 86 +++++++++ 3 files changed, 297 insertions(+), 34 deletions(-) create mode 100644 tests/queries/0_stateless/00500_point_in_polygon_non_const_poly.reference create mode 100644 tests/queries/0_stateless/00500_point_in_polygon_non_const_poly.sql diff --git a/src/Functions/pointInPolygon.cpp b/src/Functions/pointInPolygon.cpp index 460c60d6e4c..42e6e0ffeeb 100644 --- a/src/Functions/pointInPolygon.cpp +++ b/src/Functions/pointInPolygon.cpp @@ -44,7 +44,7 @@ namespace { template -ColumnPtr callPointInPolygonImplWithPool(const IColumn & x, const IColumn & y, Polygon & polygon) +UInt8 callPointInPolygonImplWithPool(Float64 x, Float64 y, Polygon & polygon) { using Pool = ObjectPoolMap; /// C++11 has thread-safe function-local statics on most modern compilers. @@ -63,14 +63,14 @@ ColumnPtr callPointInPolygonImplWithPool(const IColumn & x, const IColumn & y, P std::string serialized_polygon = serialize(polygon); auto impl = known_polygons.get(serialized_polygon, factory); - return pointInPolygon(x, y, *impl); + return impl->contains(x, y); } template -ColumnPtr callPointInPolygonImpl(const IColumn & x, const IColumn & y, Polygon & polygon) +UInt8 callPointInPolygonImpl(Float64 x, Float64 y, Polygon & polygon) { PointInPolygonImpl impl(polygon); - return pointInPolygon(x, y, impl); + return impl.contains(x, y); } } @@ -116,74 +116,176 @@ public: throw Exception("Too few arguments", ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION); } - auto get_message_prefix = [this](size_t i) { return "Argument " + toString(i + 1) + " for function " + getName(); }; - - for (size_t i = 1; i < arguments.size(); ++i) - { - const auto * array = checkAndGetDataType(arguments[i].get()); - if (array == nullptr && i != 1) - throw Exception(get_message_prefix(i) + " must be array of tuples.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - const auto * tuple = checkAndGetDataType(array ? array->getNestedType().get() : arguments[i].get()); + auto validateTuple = [this](size_t i, const DataTypeTuple * tuple) { if (tuple == nullptr) - throw Exception(get_message_prefix(i) + " must contains tuple.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception(getMessagePrefix(i) + " must contain a tuple", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); const DataTypes & elements = tuple->getElements(); if (elements.size() != 2) - throw Exception(get_message_prefix(i) + " must have exactly two elements.", ErrorCodes::BAD_ARGUMENTS); + throw Exception(getMessagePrefix(i) + " must have exactly two elements", ErrorCodes::BAD_ARGUMENTS); for (auto j : ext::range(0, elements.size())) { if (!isNativeNumber(elements[j])) { - throw Exception(get_message_prefix(i) + " must contains numeric tuple at position " + toString(j + 1), + throw Exception(getMessagePrefix(i) + " must contain numeric tuple at position " + toString(j + 1), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } } + }; + + validateTuple(0, checkAndGetDataType(arguments[0].get())); + + if (arguments.size() == 2) { + auto * array = checkAndGetDataType(arguments[1].get()); + if (array == nullptr) + throw Exception(getMessagePrefix(1) + " must contain an array of tuples or an array of arrays of tuples.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + auto * nested_array = checkAndGetDataType(array->getNestedType().get()); + if (nested_array != nullptr) { + array = nested_array; + } + + validateTuple(1, checkAndGetDataType(array->getNestedType().get())); + } else { + for (size_t i = 1; i < arguments.size(); i++) { + auto * array = checkAndGetDataType(arguments[1].get()); + if (array == nullptr) + throw Exception(getMessagePrefix(i) + " must contain an array of tuples", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + validateTuple(i, checkAndGetDataType(array->getNestedType().get())); + } } return std::make_shared(); } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) override + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { const IColumn * point_col = block.getByPosition(arguments[0]).column.get(); const auto * const_tuple_col = checkAndGetColumn(point_col); if (const_tuple_col) point_col = &const_tuple_col->getDataColumn(); - const auto * tuple_col = checkAndGetColumn(point_col); + const auto * tuple_col = checkAndGetColumn(point_col); if (!tuple_col) throw Exception("First argument for function " + getName() + " must be constant array of tuples.", ErrorCodes::ILLEGAL_COLUMN); - auto & result_column = block.safeGetByPosition(result).column; - const auto & tuple_columns = tuple_col->getColumns(); - result_column = executeForType(*tuple_columns[0], *tuple_columns[1], block, arguments); - if (const_tuple_col) + const IColumn * poly_col = block.getByPosition(arguments[1]).column.get(); + const auto * const_poly_col = checkAndGetColumn(poly_col); + + bool point_is_const = const_tuple_col != nullptr; + bool poly_is_const = const_poly_col != nullptr; + + auto call_impl = use_object_pool + ? callPointInPolygonImplWithPool + : callPointInPolygonImpl; + + size_t size = point_is_const && poly_is_const ? 1 : input_rows_count; + auto execution_result = ColumnVector::create(size); + auto & data = execution_result->getData(); + + Polygon polygon; + for (auto i : ext::range(0, size)) { + if (!poly_is_const || i == 0) { + polygon = parsePolygon(block, arguments, i); + } + + size_t point_index = point_is_const ? 0 : i; + data[i] = call_impl(tuple_columns[0]->getFloat64(point_index), tuple_columns[1]->getFloat64(point_index), polygon); + } + + auto & result_column = block.safeGetByPosition(result).column; + result_column = std::move(execution_result); + if (point_is_const && poly_is_const) result_column = ColumnConst::create(result_column, const_tuple_col->size()); } + private: bool validate; - ColumnPtr executeForType(const IColumn & x, const IColumn & y, Block & block, const ColumnNumbers & arguments) + std::string getMessagePrefix(size_t i) const + { + return "Argument " + toString(i + 1) + " for function " + getName(); + } + + Polygon parsePolygonFromSingleColumn(Block & block, const ColumnNumbers & arguments, size_t i) const + { + const auto & poly = block.getByPosition(arguments[1]).column.get(); + const auto * column_const = checkAndGetColumn(poly); + const auto * array_col = + column_const ? checkAndGetColumn(column_const->getDataColumn()) : checkAndGetColumn(poly); + + if (!array_col) + throw Exception(getMessagePrefix(1) + " must contain an array of tuples or an array of arrays of tuples", + ErrorCodes::ILLEGAL_COLUMN); + + const auto * nested_array_col = checkAndGetColumn(array_col->getData()); + const auto & tuple_data = nested_array_col ? nested_array_col->getData() : array_col->getData(); + const auto & tuple_col = checkAndGetColumn(tuple_data); + if (!tuple_col) + throw Exception(getMessagePrefix(1) + " must contain an array of tuples or an array of arrays of tuples", + ErrorCodes::ILLEGAL_COLUMN); + + const auto & tuple_columns = tuple_col->getColumns(); + const auto & x_column = tuple_columns[0]; + const auto & y_column = tuple_columns[1]; + + auto parse_polygon_part = [&x_column, &y_column](auto & container, size_t l, size_t r) { + for (auto j : ext::range(l, r)) + { + CoordinateType x_coord = x_column->getFloat64(j); + CoordinateType y_coord = y_column->getFloat64(j); + + container.push_back(Point(x_coord, y_coord)); + } + }; + + Polygon polygon; + if (nested_array_col) { + for (auto j : ext::range(array_col->getOffsets()[i - 1], array_col->getOffsets()[i])) { + size_t l = nested_array_col->getOffsets()[j - 1]; + size_t r = nested_array_col->getOffsets()[j]; + if (polygon.outer().empty()) { + parse_polygon_part(polygon.outer(), l, r); + } else { + polygon.inners().emplace_back(); + parse_polygon_part(polygon.inners().back(), l, r); + } + } + } else { + size_t l = array_col->getOffsets()[i - 1]; + size_t r = array_col->getOffsets()[i]; + + parse_polygon_part(polygon.outer(), l, r); + } + + return polygon; + } + + Polygon parsePolygonFromMultipleColumns(Block & block, const ColumnNumbers & arguments, size_t) const { Polygon polygon; - auto get_message_prefix = [this](size_t i) { return "Argument " + toString(i + 1) + " for function " + getName(); }; - for (size_t i = 1; i < arguments.size(); ++i) { const auto * const_col = checkAndGetColumn(block.getByPosition(arguments[i]).column.get()); - const auto * array_col = const_col ? checkAndGetColumn(&const_col->getDataColumn()) : nullptr; + if (!const_col) + throw Exception("Multi-argument version of function " + getName() + " works only with const polygon", + ErrorCodes::BAD_ARGUMENTS); + + const auto * array_col = checkAndGetColumn(&const_col->getDataColumn()); const auto * tuple_col = array_col ? checkAndGetColumn(&array_col->getData()) : nullptr; if (!tuple_col) - throw Exception(get_message_prefix(i) + " must be constant array of tuples.", ErrorCodes::ILLEGAL_COLUMN); + throw Exception(getMessagePrefix(i) + " must be constant array of tuples", ErrorCodes::ILLEGAL_COLUMN); const auto & tuple_columns = tuple_col->getColumns(); const auto & column_x = tuple_columns[0]; @@ -197,7 +299,7 @@ private: auto size = column_x->size(); if (size == 0) - throw Exception(get_message_prefix(i) + " shouldn't be empty.", ErrorCodes::ILLEGAL_COLUMN); + throw Exception(getMessagePrefix(i) + " shouldn't be empty.", ErrorCodes::ILLEGAL_COLUMN); for (auto j : ext::range(0, size)) { @@ -207,6 +309,18 @@ private: } } + return polygon; + } + + Polygon parsePolygon(Block & block, const ColumnNumbers & arguments, size_t i) const + { + Polygon polygon; + if (arguments.size() == 2) { + polygon = parsePolygonFromSingleColumn(block, arguments, i); + } else { + polygon = parsePolygonFromMultipleColumns(block, arguments, i); + } + boost::geometry::correct(polygon); #if !defined(__clang_analyzer__) /// It does not like boost. @@ -218,12 +332,7 @@ private: throw Exception("Polygon is not valid: " + failure_message, ErrorCodes::BAD_ARGUMENTS); } #endif - - auto call_impl = use_object_pool - ? callPointInPolygonImplWithPool - : callPointInPolygonImpl; - - return call_impl(x, y, polygon); + return polygon; } }; diff --git a/tests/queries/0_stateless/00500_point_in_polygon_non_const_poly.reference b/tests/queries/0_stateless/00500_point_in_polygon_non_const_poly.reference new file mode 100644 index 00000000000..083a7ac7236 --- /dev/null +++ b/tests/queries/0_stateless/00500_point_in_polygon_non_const_poly.reference @@ -0,0 +1,68 @@ +Const point; No holes +0 +0 +0 +0 +0 +1 +1 +1 +1 +1 +1 +0 +1 +0 +1 +0 +Non-const point; No holes +0 +0 +0 +0 +0 +1 +1 +1 +1 +1 +1 +0 +1 +0 +1 +0 +Const point; With holes +0 +0 +0 +0 +0 +1 +1 +0 +0 +1 +1 +0 +1 +0 +1 +0 +Non-const point; With holes +0 +0 +0 +0 +0 +1 +1 +0 +0 +1 +1 +0 +1 +0 +1 +0 diff --git a/tests/queries/0_stateless/00500_point_in_polygon_non_const_poly.sql b/tests/queries/0_stateless/00500_point_in_polygon_non_const_poly.sql new file mode 100644 index 00000000000..f38066debbf --- /dev/null +++ b/tests/queries/0_stateless/00500_point_in_polygon_non_const_poly.sql @@ -0,0 +1,86 @@ +DROP TABLE IF EXISTS polygons; + +SELECT 'Const point; No holes'; +create table polygons ( id Int32, poly Array(Tuple(Int32, Int32))) engine = Log(); + +INSERT INTO polygons VALUES (1, [(0, 0), (10, 0), (10, 10), (0, 10)]); +INSERT INTO polygons VALUES (2, [(-5, -5), (5, -5), (5, 5), (-5, 5)]); + +SELECT pointInPolygon((-10, 0), poly) FROM polygons ORDER BY id; +SELECT pointInPolygon((0, -10), poly) FROM polygons ORDER BY id; +SELECT pointInPolygon((-5, -5), poly) FROM polygons ORDER BY id; +SELECT pointInPolygon((0, 0), poly) FROM polygons ORDER BY id; +SELECT pointInPolygon((5, 5), poly) FROM polygons ORDER BY id; +SELECT pointInPolygon((10, 10), poly) FROM polygons ORDER BY id; +SELECT pointInPolygon((10, 5), poly) FROM polygons ORDER BY id; +SELECT pointInPolygon((5, 10), poly) FROM polygons ORDER BY id; + +DROP TABLE polygons; + +SELECT 'Non-const point; No holes'; + +create table polygons ( id Int32, pt Tuple(Int32, Int32), poly Array(Tuple(Int32, Int32))) engine = Log(); + +INSERT INTO polygons VALUES (1, (-10, 0), [(0, 0), (10, 0), (10, 10), (0, 10)]); +INSERT INTO polygons VALUES (2, (-10, 0), [(-5, -5), (5, -5), (5, 5), (-5, 5)]); +INSERT INTO polygons VALUES (3, (0, -10), [(0, 0), (10, 0), (10, 10), (0, 10)]); +INSERT INTO polygons VALUES (4, (0, -10), [(-5, -5), (5, -5), (5, 5), (-5, 5)]); +INSERT INTO polygons VALUES (5, (-5, -5), [(0, 0), (10, 0), (10, 10), (0, 10)]); +INSERT INTO polygons VALUES (6, (-5, -5), [(-5, -5), (5, -5), (5, 5), (-5, 5)]); +INSERT INTO polygons VALUES (7, (0, 0), [(0, 0), (10, 0), (10, 10), (0, 10)]); +INSERT INTO polygons VALUES (8, (0, 0), [(-5, -5), (5, -5), (5, 5), (-5, 5)]); +INSERT INTO polygons VALUES (9, (5, 5), [(0, 0), (10, 0), (10, 10), (0, 10)]); +INSERT INTO polygons VALUES (10, (5, 5), [(-5, -5), (5, -5), (5, 5), (-5, 5)]); +INSERT INTO polygons VALUES (11, (10, 10), [(0, 0), (10, 0), (10, 10), (0, 10)]); +INSERT INTO polygons VALUES (12, (10, 10), [(-5, -5), (5, -5), (5, 5), (-5, 5)]); +INSERT INTO polygons VALUES (13, (10, 5), [(0, 0), (10, 0), (10, 10), (0, 10)]); +INSERT INTO polygons VALUES (14, (10, 5), [(-5, -5), (5, -5), (5, 5), (-5, 5)]); +INSERT INTO polygons VALUES (15, (5, 10), [(0, 0), (10, 0), (10, 10), (0, 10)]); +INSERT INTO polygons VALUES (16, (5, 10), [(-5, -5), (5, -5), (5, 5), (-5, 5)]); + +SELECT pointInPolygon(pt, poly) FROM polygons ORDER BY id; + +DROP TABLE polygons; + +SELECT 'Const point; With holes'; + +create table polygons ( id Int32, poly Array(Array(Tuple(Int32, Int32)))) engine = Log(); + +INSERT INTO polygons VALUES (1, [[(0, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (6, 4), (6, 6), (4, 6)]]); +INSERT INTO polygons VALUES (2, [[(-5, -5), (5, -5), (5, 5), (-5, 5)], [(-1, -1), (1, -1), (1, 1), (-1, 1)]]); + +SELECT pointInPolygon((-10, 0), poly) FROM polygons ORDER BY id; +SELECT pointInPolygon((0, -10), poly) FROM polygons ORDER BY id; +SELECT pointInPolygon((-5, -5), poly) FROM polygons ORDER BY id; +SELECT pointInPolygon((0, 0), poly) FROM polygons ORDER BY id; +SELECT pointInPolygon((5, 5), poly) FROM polygons ORDER BY id; +SELECT pointInPolygon((10, 10), poly) FROM polygons ORDER BY id; +SELECT pointInPolygon((10, 5), poly) FROM polygons ORDER BY id; +SELECT pointInPolygon((5, 10), poly) FROM polygons ORDER BY id; + +DROP TABLE polygons; + +SELECT 'Non-const point; With holes'; + +create table polygons ( id Int32, pt Tuple(Int32, Int32), poly Array(Array(Tuple(Int32, Int32)))) engine = Log(); + +INSERT INTO polygons VALUES (1, (-10, 0), [[(0, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (6, 4), (6, 6), (4, 6)]]); +INSERT INTO polygons VALUES (2, (-10, 0), [[(-5, -5), (5, -5), (5, 5), (-5, 5)], [(-1, -1), (1, -1), (1, 1), (-1, 1)]]); +INSERT INTO polygons VALUES (3, (0, -10), [[(0, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (6, 4), (6, 6), (4, 6)]]); +INSERT INTO polygons VALUES (4, (0, -10), [[(-5, -5), (5, -5), (5, 5), (-5, 5)], [(-1, -1), (1, -1), (1, 1), (-1, 1)]]); +INSERT INTO polygons VALUES (5, (-5, -5), [[(0, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (6, 4), (6, 6), (4, 6)]]); +INSERT INTO polygons VALUES (6, (-5, -5), [[(-5, -5), (5, -5), (5, 5), (-5, 5)], [(-1, -1), (1, -1), (1, 1), (-1, 1)]]); +INSERT INTO polygons VALUES (7, (0, 0), [[(0, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (6, 4), (6, 6), (4, 6)]]); +INSERT INTO polygons VALUES (8, (0, 0), [[(-5, -5), (5, -5), (5, 5), (-5, 5)], [(-1, -1), (1, -1), (1, 1), (-1, 1)]]); +INSERT INTO polygons VALUES (9, (5, 5), [[(0, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (6, 4), (6, 6), (4, 6)]]); +INSERT INTO polygons VALUES (10, (5, 5), [[(-5, -5), (5, -5), (5, 5), (-5, 5)], [(-1, -1), (1, -1), (1, 1), (-1, 1)]]); +INSERT INTO polygons VALUES (11, (10, 10), [[(0, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (6, 4), (6, 6), (4, 6)]]); +INSERT INTO polygons VALUES (12, (10, 10), [[(-5, -5), (5, -5), (5, 5), (-5, 5)], [(-1, -1), (1, -1), (1, 1), (-1, 1)]]); +INSERT INTO polygons VALUES (13, (10, 5), [[(0, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (6, 4), (6, 6), (4, 6)]]); +INSERT INTO polygons VALUES (14, (10, 5), [[(-5, -5), (5, -5), (5, 5), (-5, 5)], [(-1, -1), (1, -1), (1, 1), (-1, 1)]]); +INSERT INTO polygons VALUES (15, (5, 10), [[(0, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (6, 4), (6, 6), (4, 6)]]); +INSERT INTO polygons VALUES (16, (5, 10), [[(-5, -5), (5, -5), (5, 5), (-5, 5)], [(-1, -1), (1, -1), (1, 1), (-1, 1)]]); + +SELECT pointInPolygon(pt, poly) FROM polygons ORDER BY id; + +DROP TABLE polygons; \ No newline at end of file From a03da4eb34fb089aaece2cfe0f10d928ab6a59dd Mon Sep 17 00:00:00 2001 From: Alexey Ilyukhov Date: Sat, 2 May 2020 19:48:36 +0300 Subject: [PATCH 021/738] Fix style and array index --- src/Functions/pointInPolygon.cpp | 57 +++++++++++++++++++++----------- 1 file changed, 38 insertions(+), 19 deletions(-) diff --git a/src/Functions/pointInPolygon.cpp b/src/Functions/pointInPolygon.cpp index 42e6e0ffeeb..f1761c49485 100644 --- a/src/Functions/pointInPolygon.cpp +++ b/src/Functions/pointInPolygon.cpp @@ -116,7 +116,8 @@ public: throw Exception("Too few arguments", ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION); } - auto validateTuple = [this](size_t i, const DataTypeTuple * tuple) { + auto validate_tuple = [this](size_t i, const DataTypeTuple * tuple) + { if (tuple == nullptr) throw Exception(getMessagePrefix(i) + " must contain a tuple", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); @@ -135,28 +136,33 @@ public: } }; - validateTuple(0, checkAndGetDataType(arguments[0].get())); + validate_tuple(0, checkAndGetDataType(arguments[0].get())); - if (arguments.size() == 2) { + if (arguments.size() == 2) + { auto * array = checkAndGetDataType(arguments[1].get()); if (array == nullptr) throw Exception(getMessagePrefix(1) + " must contain an array of tuples or an array of arrays of tuples.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); auto * nested_array = checkAndGetDataType(array->getNestedType().get()); - if (nested_array != nullptr) { + if (nested_array != nullptr) + { array = nested_array; } - validateTuple(1, checkAndGetDataType(array->getNestedType().get())); - } else { - for (size_t i = 1; i < arguments.size(); i++) { - auto * array = checkAndGetDataType(arguments[1].get()); + validate_tuple(1, checkAndGetDataType(array->getNestedType().get())); + } + else + { + for (size_t i = 1; i < arguments.size(); i++) + { + auto * array = checkAndGetDataType(arguments[i].get()); if (array == nullptr) throw Exception(getMessagePrefix(i) + " must contain an array of tuples", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - validateTuple(i, checkAndGetDataType(array->getNestedType().get())); + validate_tuple(i, checkAndGetDataType(array->getNestedType().get())); } } @@ -192,8 +198,10 @@ public: auto & data = execution_result->getData(); Polygon polygon; - for (auto i : ext::range(0, size)) { - if (!poly_is_const || i == 0) { + for (auto i : ext::range(0, size)) + { + if (!poly_is_const || i == 0) + { polygon = parsePolygon(block, arguments, i); } @@ -238,7 +246,8 @@ private: const auto & x_column = tuple_columns[0]; const auto & y_column = tuple_columns[1]; - auto parse_polygon_part = [&x_column, &y_column](auto & container, size_t l, size_t r) { + auto parse_polygon_part = [&x_column, &y_column](auto & container, size_t l, size_t r) + { for (auto j : ext::range(l, r)) { CoordinateType x_coord = x_column->getFloat64(j); @@ -249,18 +258,25 @@ private: }; Polygon polygon; - if (nested_array_col) { - for (auto j : ext::range(array_col->getOffsets()[i - 1], array_col->getOffsets()[i])) { + if (nested_array_col) + { + for (auto j : ext::range(array_col->getOffsets()[i - 1], array_col->getOffsets()[i])) + { size_t l = nested_array_col->getOffsets()[j - 1]; size_t r = nested_array_col->getOffsets()[j]; - if (polygon.outer().empty()) { + if (polygon.outer().empty()) + { parse_polygon_part(polygon.outer(), l, r); - } else { + } + else + { polygon.inners().emplace_back(); parse_polygon_part(polygon.inners().back(), l, r); } } - } else { + } + else + { size_t l = array_col->getOffsets()[i - 1]; size_t r = array_col->getOffsets()[i]; @@ -315,9 +331,12 @@ private: Polygon parsePolygon(Block & block, const ColumnNumbers & arguments, size_t i) const { Polygon polygon; - if (arguments.size() == 2) { + if (arguments.size() == 2) + { polygon = parsePolygonFromSingleColumn(block, arguments, i); - } else { + } + else + { polygon = parsePolygonFromMultipleColumns(block, arguments, i); } From 0ab6936645e6b7c5e83a49379b1fdad70ab6a809 Mon Sep 17 00:00:00 2001 From: Guillaume Tassery Date: Mon, 4 May 2020 19:52:38 +0700 Subject: [PATCH 022/738] Add parser for handle offset without a limit clause --- src/Parsers/ASTSelectQuery.cpp | 5 +++++ src/Parsers/ExpressionElementParsers.cpp | 1 + src/Parsers/ParserSelectQuery.cpp | 7 +++++++ 3 files changed, 13 insertions(+) diff --git a/src/Parsers/ASTSelectQuery.cpp b/src/Parsers/ASTSelectQuery.cpp index 6a88700e3b8..3e9692518b9 100644 --- a/src/Parsers/ASTSelectQuery.cpp +++ b/src/Parsers/ASTSelectQuery.cpp @@ -154,6 +154,11 @@ void ASTSelectQuery::formatImpl(const FormatSettings & s, FormatState & state, F if (limit_with_ties) s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << " WITH TIES" << (s.hilite ? hilite_none : ""); } + else if(limitOffset()) + { + s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "OFFSET " << (s.hilite ? hilite_none : ""); + limitOffset()->formatImpl(s, state, frame); + } if (settings()) { diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index 47c28f8db77..fef4ba3f309 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -1105,6 +1105,7 @@ const char * ParserAlias::restricted_keywords[] = "HAVING", "ORDER", "LIMIT", + "OFFSET", "SETTINGS", "FORMAT", "UNION", diff --git a/src/Parsers/ParserSelectQuery.cpp b/src/Parsers/ParserSelectQuery.cpp index 927ac45001e..02909314199 100644 --- a/src/Parsers/ParserSelectQuery.cpp +++ b/src/Parsers/ParserSelectQuery.cpp @@ -242,6 +242,13 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) throw Exception("Can not use TOP and LIMIT together", ErrorCodes::TOP_AND_LIMIT_TOGETHER); } + + if (s_offset.ignore(pos, expected)) + { + if (!exp_elem.parse(pos, limit_offset, expected)) + return false; + } + /// Because TOP n in totally equals LIMIT n if (top_length) limit_length = top_length; From 6a9355c95dddecf5f996e61e76f624f21095e88b Mon Sep 17 00:00:00 2001 From: Alexey Ilyukhov Date: Tue, 5 May 2020 13:34:58 +0300 Subject: [PATCH 023/738] Do not use grid for non const --- src/Functions/pointInPolygon.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/Functions/pointInPolygon.cpp b/src/Functions/pointInPolygon.cpp index f1761c49485..b46b385f738 100644 --- a/src/Functions/pointInPolygon.cpp +++ b/src/Functions/pointInPolygon.cpp @@ -75,7 +75,7 @@ UInt8 callPointInPolygonImpl(Float64 x, Float64 y, Polygon & polygon) } -template +template class FunctionPointInPolygon : public IFunction { public: @@ -91,7 +91,8 @@ public: static FunctionPtr create(const Context & context) { - return std::make_shared>(context.getSettingsRef().validate_polygons); + return std::make_shared>( + context.getSettingsRef().validate_polygons); } String getName() const override @@ -189,9 +190,9 @@ public: bool point_is_const = const_tuple_col != nullptr; bool poly_is_const = const_poly_col != nullptr; - auto call_impl = use_object_pool - ? callPointInPolygonImplWithPool - : callPointInPolygonImpl; + auto call_impl = poly_is_const + ? callPointInPolygonImplWithPool + : callPointInPolygonImpl; size_t size = point_is_const && poly_is_const ? 1 : input_rows_count; auto execution_result = ColumnVector::create(size); @@ -358,7 +359,7 @@ private: void registerFunctionPointInPolygon(FunctionFactory & factory) { - factory.registerFunction, true>>(); + factory.registerFunction, PointInPolygonTrivial>>(); } } From 26424ec3884685b244654ae83327828c0f8504af Mon Sep 17 00:00:00 2001 From: DoomzD Date: Tue, 5 May 2020 17:46:23 +0300 Subject: [PATCH 024/738] Add perftest for pointInPolygon --- tests/performance/point_in_polygon.xml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 tests/performance/point_in_polygon.xml diff --git a/tests/performance/point_in_polygon.xml b/tests/performance/point_in_polygon.xml new file mode 100644 index 00000000000..d854fb6952b --- /dev/null +++ b/tests/performance/point_in_polygon.xml @@ -0,0 +1,6 @@ + + CREATE TABLE point_in_polygon(`polygon` Array(Array(Float64, Float64))) ENGINE = Log() + insert into point_in_polygon SELECT arrayJoin(arrayMap(y -> [arrayMap(x -> (cos(x / 90. * pi()) * y, sin(x / 90. * pi()) * y), range(180))], arraySlice(range(35000), 2, 35000))) + SELECT pointInPolygon((100, 100), `polygon`) from point_in_polygon + DROP TABLE IF EXISTS point_in_polygon + From a2916c23a55aee1d210898b0ded23e4c61617320 Mon Sep 17 00:00:00 2001 From: bobrovskij artemij Date: Wed, 6 May 2020 02:42:44 +0300 Subject: [PATCH 025/738] fixes/style --- .../odbc-bridge/ODBCBlockOutputStream.cpp | 29 +++++++++++-------- programs/odbc-bridge/ODBCBlockOutputStream.h | 6 ++-- src/Storages/StorageXDBC.cpp | 5 ++-- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/programs/odbc-bridge/ODBCBlockOutputStream.cpp b/programs/odbc-bridge/ODBCBlockOutputStream.cpp index 72cf2e509ce..148f38087cd 100644 --- a/programs/odbc-bridge/ODBCBlockOutputStream.cpp +++ b/programs/odbc-bridge/ODBCBlockOutputStream.cpp @@ -3,17 +3,20 @@ #include #include #include +#include namespace DB { -namespace { +namespace +{ using ValueType = ExternalResultDescription::ValueType; std::string commaSeparateColumnNames(const ColumnsWithTypeAndName & columns) { std::string result = "("; - for (size_t i = 0; i < columns.size(); ++i) { + for (size_t i = 0; i < columns.size(); ++i) + { if (i > 0) result += ","; result += columns[i].name; @@ -24,7 +27,8 @@ namespace { std::string getQuestionMarks(size_t n) { std::string result = "("; - for (size_t i = 0; i < n; ++i) { + for (size_t i = 0; i < n; ++i) + { if (i > 0) result += ","; result += "?"; @@ -34,21 +38,22 @@ namespace { Poco::Dynamic::Var getVarFromField(const Field & field, const ValueType type) { - switch (type) { + switch (type) + { case ValueType::vtUInt8: - return Poco::Dynamic::Var(static_cast(field.get())).convert(); + return Poco::Dynamic::Var(static_cast(field.get())).convert(); case ValueType::vtUInt16: - return Poco::Dynamic::Var(static_cast(field.get())).convert(); + return Poco::Dynamic::Var(static_cast(field.get())).convert(); case ValueType::vtUInt32: - return Poco::Dynamic::Var(static_cast(field.get())).convert(); + return Poco::Dynamic::Var(static_cast(field.get())).convert(); case ValueType::vtUInt64: return Poco::Dynamic::Var(field.get()).convert(); case ValueType::vtInt8: - return Poco::Dynamic::Var(static_cast(field.get())).convert(); + return Poco::Dynamic::Var(static_cast(field.get())).convert(); case ValueType::vtInt16: - return Poco::Dynamic::Var(static_cast(field.get())).convert(); + return Poco::Dynamic::Var(static_cast(field.get())).convert(); case ValueType::vtInt32: - return Poco::Dynamic::Var(static_cast(field.get())).convert(); + return Poco::Dynamic::Var(static_cast(field.get())).convert(); case ValueType::vtInt64: return Poco::Dynamic::Var(field.get()).convert(); case ValueType::vtFloat32: @@ -58,9 +63,9 @@ namespace { case ValueType::vtString: return Poco::Dynamic::Var(field.get()).convert(); case ValueType::vtDate: - return Poco::Dynamic::Var(LocalDate(DayNum(field.get())).toString()).convert(); + return Poco::Dynamic::Var(LocalDate(DayNum(field.get())).toString()).convert(); case ValueType::vtDateTime: - return Poco::Dynamic::Var(LocalDate(time_t(field.get())).toString()).convert(); + return Poco::Dynamic::Var(std::to_string(LocalDateTime(time_t(field.get())))).convert(); case ValueType::vtUUID: return Poco::Dynamic::Var(UUID(field.get()).toUnderType().toHexString()).convert(); } diff --git a/programs/odbc-bridge/ODBCBlockOutputStream.h b/programs/odbc-bridge/ODBCBlockOutputStream.h index 5de283f1c8b..544f19682f4 100644 --- a/programs/odbc-bridge/ODBCBlockOutputStream.h +++ b/programs/odbc-bridge/ODBCBlockOutputStream.h @@ -5,8 +5,10 @@ #include #include -namespace DB{ -class ODBCBlockOutputStream : public IBlockOutputStream { +namespace DB +{ +class ODBCBlockOutputStream : public IBlockOutputStream +{ public: ODBCBlockOutputStream(Poco::Data::Session && session_, const std::string & remote_database_name_, const std::string & remote_table_name_, const Block & sample_block_); diff --git a/src/Storages/StorageXDBC.cpp b/src/Storages/StorageXDBC.cpp index e4db7aa1fd9..a6e26b7482d 100644 --- a/src/Storages/StorageXDBC.cpp +++ b/src/Storages/StorageXDBC.cpp @@ -95,10 +95,10 @@ Pipes StorageXDBC::read(const Names & column_names, return IStorageURLBase::read(column_names, query_info, context, processed_stage, max_block_size, num_streams); } -BlockOutputStreamPtr StorageXDBC::write(const ASTPtr & /*query*/, const Context & context) { +BlockOutputStreamPtr StorageXDBC::write(const ASTPtr & /*query*/, const Context & context) +{ bridge_helper->startBridgeSync(); - // some copypaste NamesAndTypesList cols; Poco::URI request_uri = uri; request_uri.setPath("/write"); @@ -112,6 +112,7 @@ BlockOutputStreamPtr StorageXDBC::write(const ASTPtr & /*query*/, const Context request_uri.addQueryParameter(param, value); request_uri.addQueryParameter("db_name", remote_database_name); request_uri.addQueryParameter("table_name", remote_table_name); + request_uri.addQueryParameter("format_name", format_name); return std::make_shared( request_uri, format_name, getSampleBlock(), context, From 8ce606571ee53b269b71771995add901457888ac Mon Sep 17 00:00:00 2001 From: Guillaume Tassery Date: Wed, 6 May 2020 13:50:55 +0700 Subject: [PATCH 026/738] write first draft of offset clause --- src/DataStreams/OffsetBlockInputStream.cpp | 50 ++++ src/DataStreams/OffsetBlockInputStream.h | 47 +++ src/Interpreters/InterpreterSelectQuery.cpp | 103 ++++++- src/Interpreters/InterpreterSelectQuery.h | 2 + src/Parsers/ParserSelectQuery.cpp | 5 + src/Processors/OffsetTransform.cpp | 308 ++++++++++++++++++++ src/Processors/OffsetTransform.h | 72 +++++ 7 files changed, 582 insertions(+), 5 deletions(-) create mode 100644 src/DataStreams/OffsetBlockInputStream.cpp create mode 100644 src/DataStreams/OffsetBlockInputStream.h create mode 100644 src/Processors/OffsetTransform.cpp create mode 100644 src/Processors/OffsetTransform.h diff --git a/src/DataStreams/OffsetBlockInputStream.cpp b/src/DataStreams/OffsetBlockInputStream.cpp new file mode 100644 index 00000000000..275ad6a079a --- /dev/null +++ b/src/DataStreams/OffsetBlockInputStream.cpp @@ -0,0 +1,50 @@ +#include + +#include + + +namespace DB +{ + +OffsetBlockInputStream::OffsetBlockInputStream( + const BlockInputStreamPtr & input, UInt64 offset_, bool always_read_till_end_, + bool use_limit_as_total_rows_approx, bool with_ties_, const SortDescription & description_) + : offset(offset_), always_read_till_end(always_read_till_end_), with_ties(with_ties_) + , description(description_) +{ + if (use_limit_as_total_rows_approx) + { + addTotalRowsApprox(static_cast(limit)); + } + + children.push_back(input); +} + +Block OffsetBlockInputStream::readImpl() +{ + Block res; + UInt64 rows = 0; + + do + { + res = children.back()->read(); + if (!res) + return res; + rows = res.rows(); + pos += rows; + } while (pos <= offset); + + SharedBlockPtr ptr = new detail::SharedBlock(std::move(res)); + + /// give away a piece of the block + UInt64 start = std::max( + static_cast(0), + static_cast(offset) - static_cast(pos) + static_cast(rows)); + + for (size_t i = 0; i < ptr->columns(); ++i) + ptr->safeGetByPosition(i).column = ptr->safeGetByPosition(i).column->cut(start, rows); + + return *ptr; +} + +} diff --git a/src/DataStreams/OffsetBlockInputStream.h b/src/DataStreams/OffsetBlockInputStream.h new file mode 100644 index 00000000000..e88d161f80a --- /dev/null +++ b/src/DataStreams/OffsetBlockInputStream.h @@ -0,0 +1,47 @@ +#pragma once + +#include +#include +#include + + +namespace DB +{ + + +/** Implements the LIMIT relational operation. + */ +class OffsetBlockInputStream : public IBlockInputStream +{ +public: + /** If always_read_till_end = false (by default), then after reading enough data, + * returns an empty block, and this causes the query to be canceled. + * If always_read_till_end = true - reads all the data to the end, but ignores them. This is necessary in rare cases: + * when otherwise, due to the cancellation of the request, we would not have received the data for GROUP BY WITH TOTALS from the remote server. + * If use_limit_as_total_rows_approx = true, then addTotalRowsApprox is called to use the limit in progress & stats + * with_ties = true, when query has WITH TIES modifier. If so, description should be provided + * description lets us know which row we should check for equality + */ + OffsetBlockInputStream( + const BlockInputStreamPtr & input, UInt64 offset_, + bool always_read_till_end_ = false, bool use_limit_as_total_rows_approx = false, + bool with_ties_ = false, const SortDescription & description_ = {}); + + String getName() const override { return "Offset"; } + + Block getHeader() const override { return children.at(0)->getHeader(); } + +protected: + Block readImpl() override; + +private: + UInt64 limit; + UInt64 offset; + UInt64 pos = 0; + bool always_read_till_end; + bool with_ties; + const SortDescription description; + SharedBlockRowRef ties_row_ref; +}; + +} diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 290bc26856a..16f7b75c036 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -92,6 +93,7 @@ #include #include #include +#include #include #include #include @@ -675,12 +677,9 @@ static std::pair getLimitLengthAndOffset(const ASTSelectQuery & UInt64 offset = 0; if (query.limitLength()) - { length = getLimitUIntValue(query.limitLength(), context); - if (query.limitOffset() && length) - offset = getLimitUIntValue(query.limitOffset(), context); - } - + if (query.limitOffset()) + offset = getLimitUIntValue(query.limitOffset(), context); return {length, offset}; } @@ -1065,6 +1064,8 @@ void InterpreterSelectQuery::executeImpl(TPipeline & pipeline, const BlockInputS if (!(pipeline_with_processors && has_prelimit)) /// Limit is no longer needed if there is prelimit. executeLimit(pipeline); + + executeOffset(pipeline); } } @@ -2435,6 +2436,50 @@ void InterpreterSelectQuery::executeLimit(Pipeline & pipeline) } +void InterpreterSelectQuery::executeOffset(Pipeline & pipeline) +{ + auto & query = getSelectQuery(); + /// If there is LIMIT + if (!query.limitLength() && query.limitOffset()) + { + /** Rare case: + * if there is no WITH TOTALS and there is a subquery in FROM, and there is WITH TOTALS on one of the levels, + * then when using LIMIT, you should read the data to the end, rather than cancel the query earlier, + * because if you cancel the query, we will not get `totals` data from the remote server. + * + * Another case: + * if there is WITH TOTALS and there is no ORDER BY, then read the data to the end, + * otherwise TOTALS is counted according to incomplete data. + */ + bool always_read_till_end = false; + + if (query.group_by_with_totals && !query.orderBy()) + always_read_till_end = true; + + if (!query.group_by_with_totals && hasWithTotalsInAnySubqueryInFromClause(query)) + always_read_till_end = true; + + SortDescription order_descr; + if (query.limit_with_ties) + { + if (!query.orderBy()) + throw Exception("LIMIT WITH TIES without ORDER BY", ErrorCodes::LOGICAL_ERROR); + order_descr = getSortDescription(query, *context); + } + + UInt64 limit_length; + UInt64 limit_offset; + std::tie(limit_length, limit_offset) = getLimitLengthAndOffset(query, *context); + + pipeline.transform([&](auto & stream) + { + std::cout << "BLOCK" << std::endl; + stream = std::make_shared(stream, limit_offset, always_read_till_end, false, query.limit_with_ties, order_descr); + }); + } +} + + void InterpreterSelectQuery::executeWithFill(Pipeline & pipeline) { auto & query = getSelectQuery(); @@ -2529,6 +2574,54 @@ void InterpreterSelectQuery::executeLimit(QueryPipeline & pipeline) } +void InterpreterSelectQuery::executeOffset(QueryPipeline & pipeline) +{ + auto & query = getSelectQuery(); + /// If there is LIMIT + if (!query.limitLength() && query.limitOffset()) + { + /** Rare case: + * if there is no WITH TOTALS and there is a subquery in FROM, and there is WITH TOTALS on one of the levels, + * then when using LIMIT, you should read the data to the end, rather than cancel the query earlier, + * because if you cancel the query, we will not get `totals` data from the remote server. + * + * Another case: + * if there is WITH TOTALS and there is no ORDER BY, then read the data to the end, + * otherwise TOTALS is counted according to incomplete data. + */ + bool always_read_till_end = false; + + if (query.group_by_with_totals && !query.orderBy()) + always_read_till_end = true; + + if (!query.group_by_with_totals && hasWithTotalsInAnySubqueryInFromClause(query)) + always_read_till_end = true; + + UInt64 limit_length; + UInt64 limit_offset; + std::tie(limit_length, limit_offset) = getLimitLengthAndOffset(query, *context); + + SortDescription order_descr; + if (query.limit_with_ties) + { + if (!query.orderBy()) + throw Exception("LIMIT WITH TIES without ORDER BY", ErrorCodes::LOGICAL_ERROR); + order_descr = getSortDescription(query, *context); + } + + pipeline.addSimpleTransform([&](const Block & header, QueryPipeline::StreamType stream_type) -> ProcessorPtr + { + if (stream_type != QueryPipeline::StreamType::Main) + return nullptr; + std::cout << "TRANSFORM" << std::endl; + return std::make_shared( + header, limit_length, limit_offset, 1, always_read_till_end, query.limit_with_ties, order_descr); + }); + } +} + + + void InterpreterSelectQuery::executeExtremes(Pipeline & pipeline) { if (!context->getSettingsRef().extremes) diff --git a/src/Interpreters/InterpreterSelectQuery.h b/src/Interpreters/InterpreterSelectQuery.h index c50f4a2f7b7..1415143dd63 100644 --- a/src/Interpreters/InterpreterSelectQuery.h +++ b/src/Interpreters/InterpreterSelectQuery.h @@ -185,6 +185,7 @@ private: void executeUnion(Pipeline & pipeline, Block header); void executeLimitBy(Pipeline & pipeline); void executeLimit(Pipeline & pipeline); + void executeOffset(Pipeline & pipeline); static void executeProjection(Pipeline & pipeline, const ExpressionActionsPtr & expression); void executeDistinct(Pipeline & pipeline, bool before_order, Names columns); void executeExtremes(Pipeline & pipeline); @@ -203,6 +204,7 @@ private: void executePreLimit(QueryPipeline & pipeline, bool do_not_skip_offset); void executeLimitBy(QueryPipeline & pipeline); void executeLimit(QueryPipeline & pipeline); + void executeOffset(QueryPipeline & pipeline); static void executeProjection(QueryPipeline & pipeline, const ExpressionActionsPtr & expression); void executeDistinct(QueryPipeline & pipeline, bool before_order, Names columns); void executeExtremes(QueryPipeline & pipeline); diff --git a/src/Parsers/ParserSelectQuery.cpp b/src/Parsers/ParserSelectQuery.cpp index 02909314199..313f890d826 100644 --- a/src/Parsers/ParserSelectQuery.cpp +++ b/src/Parsers/ParserSelectQuery.cpp @@ -247,6 +247,11 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { if (!exp_elem.parse(pos, limit_offset, expected)) return false; + if (s_with_ties.ignore(pos, expected)) + { + limit_with_ties_occured = true; + select_query->limit_with_ties = true; + } } /// Because TOP n in totally equals LIMIT n diff --git a/src/Processors/OffsetTransform.cpp b/src/Processors/OffsetTransform.cpp new file mode 100644 index 00000000000..87574c160a6 --- /dev/null +++ b/src/Processors/OffsetTransform.cpp @@ -0,0 +1,308 @@ +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +OffsetTransform::OffsetTransform( + const Block & header_, size_t limit_, size_t offset_, size_t num_streams, + bool always_read_till_end_, bool with_ties_, + SortDescription description_) + : IProcessor(InputPorts(num_streams, header_), OutputPorts(num_streams, header_)) + , limit(limit_), offset(offset_) + , always_read_till_end(always_read_till_end_) + , with_ties(with_ties_), description(std::move(description_)) +{ + if (num_streams != 1 && with_ties) + throw Exception("Cannot use OffsetTransform with multiple ports and ties.", ErrorCodes::LOGICAL_ERROR); + + ports_data.resize(num_streams); + + size_t cur_stream = 0; + for (auto & input : inputs) + { + ports_data[cur_stream].input_port = &input; + ++cur_stream; + } + + cur_stream = 0; + for (auto & output : outputs) + { + ports_data[cur_stream].output_port = &output; + ++cur_stream; + } + + for (const auto & desc : description) + { + if (!desc.column_name.empty()) + sort_column_positions.push_back(header_.getPositionByName(desc.column_name)); + else + sort_column_positions.push_back(desc.column_number); + } +} + +Chunk OffsetTransform::makeChunkWithPreviousRow(const Chunk & chunk, size_t row) const +{ + assert(row < chunk.getNumRows()); + ColumnRawPtrs current_columns = extractSortColumns(chunk.getColumns()); + MutableColumns last_row_sort_columns; + for (size_t i = 0; i < current_columns.size(); ++i) + { + last_row_sort_columns.emplace_back(current_columns[i]->cloneEmpty()); + last_row_sort_columns[i]->insertFrom(*current_columns[i], row); + } + return Chunk(std::move(last_row_sort_columns), 1); +} + + +IProcessor::Status OffsetTransform::prepare( + const PortNumbers & updated_input_ports, + const PortNumbers & updated_output_ports) +{ + bool has_full_port = false; + + auto process_pair = [&](size_t pos) + { + auto status = preparePair(ports_data[pos]); + + switch (status) + { + case IProcessor::Status::Finished: + { + if (!ports_data[pos].is_finished) + { + ports_data[pos].is_finished = true; + ++num_finished_port_pairs; + } + + return; + } + case IProcessor::Status::PortFull: + { + has_full_port = true; + return; + } + case IProcessor::Status::NeedData: + return; + default: + throw Exception( + "Unexpected status for OffsetTransform::preparePair : " + IProcessor::statusToName(status), + ErrorCodes::LOGICAL_ERROR); + + } + }; + + for (auto pos : updated_input_ports) + process_pair(pos); + + for (auto pos : updated_output_ports) + process_pair(pos); + + /// All ports are finished. It may happen even before we reached the limit (has less data then limit). + if (num_finished_port_pairs == ports_data.size()) + return Status::Finished; + + /// If we reached limit for some port, then close others. Otherwise some sources may infinitely read data. + /// Example: SELECT * FROM system.numbers_mt WHERE number = 1000000 LIMIT 1 + // if ((rows_read >= offset) && !previous_row_chunk && !always_read_till_end) + // { + // for (auto & input : inputs) + // input.close(); + + // for (auto & output : outputs) + // output.finish(); + + //return Status::Finished; + //} + + if (has_full_port) + return Status::PortFull; + + return Status::NeedData; +} + +OffsetTransform::Status OffsetTransform::prepare() +{ + if (ports_data.size() != 1) + throw Exception("prepare without arguments is not supported for multi-port OffsetTransform.", + ErrorCodes::LOGICAL_ERROR); + + return prepare({0}, {0}); +} + +OffsetTransform::Status OffsetTransform::preparePair(PortsData & data) +{ + auto & output = *data.output_port; + auto & input = *data.input_port; + + /// Check can output. + bool output_finished = false; + if (output.isFinished()) + { + output_finished = true; + if (!always_read_till_end) + { + input.close(); + return Status::Finished; + } + } + + if (!output_finished && !output.canPush()) + { + input.setNotNeeded(); + return Status::PortFull; + } + + /// Check can input. + + if (input.isFinished()) + { + output.finish(); + return Status::Finished; + } + + input.setNeeded(); + if (!input.hasData()) + return Status::NeedData; + + data.current_chunk = input.pull(true); + + auto rows = data.current_chunk.getNumRows(); + + if (rows_before_limit_at_least) + rows_before_limit_at_least->add(rows); + + /// Skip block (for 'always_read_till_end' case). + if (output_finished) + { + data.current_chunk.clear(); + if (input.isFinished()) + { + output.finish(); + return Status::Finished; + } + + /// Now, we pulled from input, and it must be empty. + input.setNeeded(); + return Status::NeedData; + } + + /// Process block. + + rows_read += rows; + + //if (rows_read <= offset) + //{ + // data.current_chunk.clear(); + // + // if (input.isFinished()) + // { + // output.finish(); + // return Status::Finished; + // } + + /// Now, we pulled from input, and it must be empty. + // input.setNeeded(); + // return Status::NeedData; + //} + + if (rows_read >= offset + rows && rows_read <= offset) + { + /// Return the whole chunk. + + /// Save the last row of current chunk to check if next block begins with the same row (for WITH TIES). + if (with_ties && rows_read == offset + limit) + previous_row_chunk = makeChunkWithPreviousRow(data.current_chunk, data.current_chunk.getNumRows() - 1); + } + else + /// This function may be heavy to execute in prepare. But it happens no more then twice, and make code simpler. + splitChunk(data); + + //bool may_need_more_data_for_ties = previous_row_chunk || rows_read - rows <= offset; + /// No more data is needed. + //if (!always_read_till_end && (rows_read >= offset) && !may_need_more_data_for_ties) + // input.close(); + + output.push(std::move(data.current_chunk)); + + return Status::PortFull; +} + + +void OffsetTransform::splitChunk(PortsData & data) +{ + auto current_chunk_sort_columns = extractSortColumns(data.current_chunk.getColumns()); + size_t num_rows = data.current_chunk.getNumRows(); + size_t num_columns = data.current_chunk.getNumColumns(); + + /// return a piece of the block + size_t start = std::max( + static_cast(0), + static_cast(offset) - static_cast(rows_read) + static_cast(num_rows)); + + //size_t length = std::min( + // static_cast(rows_read) - static_cast(offset), + // static_cast(offset) - static_cast(rows_read) + static_cast(num_rows)); + + size_t length = static_cast(num_rows); + std::cout << "===========================" << std::endl + << start << " " << length << std::endl + << static_cast(rows_read) << " " << static_cast(num_rows) << std::endl + << "===========================" << std::endl; + /// check if other rows in current block equals to last one in limit + if (with_ties && length) + { + size_t current_row_num = start + length; + previous_row_chunk = makeChunkWithPreviousRow(data.current_chunk, current_row_num - 1); + + for (; current_row_num < num_rows; ++current_row_num) + { + if (!sortColumnsEqualAt(current_chunk_sort_columns, current_row_num)) + { + previous_row_chunk = {}; + break; + } + } + + length = current_row_num - start; + } + + if (length == num_rows) + return; + + auto columns = data.current_chunk.detachColumns(); + + for (size_t i = 0; i < num_columns; ++i) + columns[i] = columns[i]->cut(start, length); + + data.current_chunk.setColumns(std::move(columns), length); +} + +ColumnRawPtrs OffsetTransform::extractSortColumns(const Columns & columns) const +{ + ColumnRawPtrs res; + res.reserve(description.size()); + for (size_t pos : sort_column_positions) + res.push_back(columns[pos].get()); + + return res; +} + +bool OffsetTransform::sortColumnsEqualAt(const ColumnRawPtrs & current_chunk_sort_columns, size_t current_chunk_row_num) const +{ + assert(current_chunk_sort_columns.size() == previous_row_chunk.getNumColumns()); + size_t size = current_chunk_sort_columns.size(); + const auto & previous_row_sort_columns = previous_row_chunk.getColumns(); + for (size_t i = 0; i < size; ++i) + if (0 != current_chunk_sort_columns[i]->compareAt(current_chunk_row_num, 0, *previous_row_sort_columns[i], 1)) + return false; + return true; +} + +} + diff --git a/src/Processors/OffsetTransform.h b/src/Processors/OffsetTransform.h new file mode 100644 index 00000000000..9194960cd20 --- /dev/null +++ b/src/Processors/OffsetTransform.h @@ -0,0 +1,72 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ + +/// Implementation for LIMIT N OFFSET M +/// This processor support multiple inputs and outputs (the same number). +/// Each pair of input and output port works independently. +/// The reason to have multiple ports is to be able to stop all sources when limit is reached, in a query like: +/// SELECT * FROM system.numbers_mt WHERE number = 1000000 LIMIT 1 +/// +/// always_read_till_end - read all data from input ports even if limit was reached. +/// with_ties, description - implementation of LIMIT WITH TIES. It works only for single port. +class OffsetTransform : public IProcessor +{ +private: + + size_t limit; + size_t offset; + bool always_read_till_end; + + bool with_ties; + const SortDescription description; + + Chunk previous_row_chunk; /// for WITH TIES, contains only sort columns + std::vector sort_column_positions; + + size_t rows_read = 0; /// including the last read block + RowsBeforeLimitCounterPtr rows_before_limit_at_least; + + /// State of port's pair. + /// Chunks from different port pairs are not mixed for berret cache locality. + struct PortsData + { + Chunk current_chunk; + + InputPort * input_port = nullptr; + OutputPort * output_port = nullptr; + bool is_finished = false; + }; + + std::vector ports_data; + size_t num_finished_port_pairs = 0; + + Chunk makeChunkWithPreviousRow(const Chunk & current_chunk, size_t row_num) const; + ColumnRawPtrs extractSortColumns(const Columns & columns) const; + bool sortColumnsEqualAt(const ColumnRawPtrs & current_chunk_sort_columns, size_t current_chunk_row_num) const; + +public: + OffsetTransform( + const Block & header_, size_t limit_, size_t offset_, size_t num_streams = 1, + bool always_read_till_end_ = false, bool with_ties_ = false, + SortDescription description_ = {}); + + String getName() const override { return "Limit"; } + + Status prepare(const PortNumbers & /*updated_input_ports*/, const PortNumbers & /*updated_output_ports*/) override; + Status prepare() override; /// Compatibility for TreeExecutor. + Status preparePair(PortsData & data); + void splitChunk(PortsData & data); + + InputPort & getInputPort() { return inputs.front(); } + OutputPort & getOutputPort() { return outputs.front(); } + + void setRowsBeforeLimitCounter(RowsBeforeLimitCounterPtr counter) { rows_before_limit_at_least.swap(counter); } +}; + +} From 90e52e7feac2564493d7028de349a5a052fe2a92 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Thu, 7 May 2020 14:54:35 +0200 Subject: [PATCH 027/738] Adding support for ALTER RENAME COLUMN query to Distributed table engine. --- src/Storages/StorageDistributed.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 44707a43fa1..881639a9e05 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -543,7 +543,8 @@ void StorageDistributed::checkAlterIsPossible(const AlterCommands & commands, co if (command.type != AlterCommand::Type::ADD_COLUMN && command.type != AlterCommand::Type::MODIFY_COLUMN && command.type != AlterCommand::Type::DROP_COLUMN - && command.type != AlterCommand::Type::COMMENT_COLUMN) + && command.type != AlterCommand::Type::COMMENT_COLUMN + && command.type != AlterCommand::Type::RENAME_COLUMN) throw Exception("Alter of type '" + alterTypeToString(command.type) + "' is not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED); From 2e719314a31bf4d635b0396ccd1af33e888a80a0 Mon Sep 17 00:00:00 2001 From: Guillaume Tassery Date: Thu, 7 May 2020 20:40:50 +0700 Subject: [PATCH 028/738] push functional draft --- src/Interpreters/InterpreterSelectQuery.cpp | 37 +++----- src/Parsers/ParserSelectQuery.cpp | 5 -- src/Processors/OffsetTransform.cpp | 96 +++------------------ src/Processors/OffsetTransform.h | 5 +- 4 files changed, 25 insertions(+), 118 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 16f7b75c036..1325d013632 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -656,16 +656,16 @@ static SortDescription getSortDescription(const ASTSelectQuery & query, const Co return order_descr; } -static UInt64 getLimitUIntValue(const ASTPtr & node, const Context & context) +static UInt64 getLimitUIntValue(const ASTPtr & node, const Context & context, const std::string & expr) { const auto & [field, type] = evaluateConstantExpression(node, context); if (!isNativeNumber(type)) - throw Exception("Illegal type " + type->getName() + " of LIMIT expression, must be numeric type", ErrorCodes::INVALID_LIMIT_EXPRESSION); + throw Exception("Illegal type " + type->getName() + " of " + expr + " expression, must be numeric type", ErrorCodes::INVALID_LIMIT_EXPRESSION); Field converted = convertFieldToType(field, DataTypeUInt64()); if (converted.isNull()) - throw Exception("The value " + applyVisitor(FieldVisitorToString(), field) + " of LIMIT expression is not representable as UInt64", ErrorCodes::INVALID_LIMIT_EXPRESSION); + throw Exception("The value " + applyVisitor(FieldVisitorToString(), field) + " of " + expr + " expression is not representable as UInt64", ErrorCodes::INVALID_LIMIT_EXPRESSION); return converted.safeGet(); } @@ -677,9 +677,9 @@ static std::pair getLimitLengthAndOffset(const ASTSelectQuery & UInt64 offset = 0; if (query.limitLength()) - length = getLimitUIntValue(query.limitLength(), context); + length = getLimitUIntValue(query.limitLength(), context, "LIMIT"); if (query.limitOffset()) - offset = getLimitUIntValue(query.limitOffset(), context); + offset = getLimitUIntValue(query.limitOffset(), context, "OFFSET"); return {length, offset}; } @@ -2336,8 +2336,8 @@ void InterpreterSelectQuery::executeLimitBy(Pipeline & pipeline) Names columns; for (const auto & elem : query.limitBy()->children) columns.emplace_back(elem->getColumnName()); - UInt64 length = getLimitUIntValue(query.limitByLength(), *context); - UInt64 offset = (query.limitByOffset() ? getLimitUIntValue(query.limitByOffset(), *context) : 0); + UInt64 length = getLimitUIntValue(query.limitByLength(), *context, "LIMIT"); + UInt64 offset = (query.limitByOffset() ? getLimitUIntValue(query.limitByOffset(), *context, "OFFSET") : 0); pipeline.transform([&](auto & stream) { @@ -2355,8 +2355,8 @@ void InterpreterSelectQuery::executeLimitBy(QueryPipeline & pipeline) for (const auto & elem : query.limitBy()->children) columns.emplace_back(elem->getColumnName()); - UInt64 length = getLimitUIntValue(query.limitByLength(), *context); - UInt64 offset = (query.limitByOffset() ? getLimitUIntValue(query.limitByOffset(), *context) : 0); + UInt64 length = getLimitUIntValue(query.limitByLength(), *context, "LIMIT"); + UInt64 offset = (query.limitByOffset() ? getLimitUIntValue(query.limitByOffset(), *context, "OFFSET") : 0); pipeline.addSimpleTransform([&](const Block & header, QueryPipeline::StreamType stream_type) -> ProcessorPtr { @@ -2580,23 +2580,6 @@ void InterpreterSelectQuery::executeOffset(QueryPipeline & pipeline) /// If there is LIMIT if (!query.limitLength() && query.limitOffset()) { - /** Rare case: - * if there is no WITH TOTALS and there is a subquery in FROM, and there is WITH TOTALS on one of the levels, - * then when using LIMIT, you should read the data to the end, rather than cancel the query earlier, - * because if you cancel the query, we will not get `totals` data from the remote server. - * - * Another case: - * if there is WITH TOTALS and there is no ORDER BY, then read the data to the end, - * otherwise TOTALS is counted according to incomplete data. - */ - bool always_read_till_end = false; - - if (query.group_by_with_totals && !query.orderBy()) - always_read_till_end = true; - - if (!query.group_by_with_totals && hasWithTotalsInAnySubqueryInFromClause(query)) - always_read_till_end = true; - UInt64 limit_length; UInt64 limit_offset; std::tie(limit_length, limit_offset) = getLimitLengthAndOffset(query, *context); @@ -2615,7 +2598,7 @@ void InterpreterSelectQuery::executeOffset(QueryPipeline & pipeline) return nullptr; std::cout << "TRANSFORM" << std::endl; return std::make_shared( - header, limit_length, limit_offset, 1, always_read_till_end, query.limit_with_ties, order_descr); + header, limit_offset, 1, true, query.limit_with_ties, order_descr); }); } } diff --git a/src/Parsers/ParserSelectQuery.cpp b/src/Parsers/ParserSelectQuery.cpp index 313f890d826..02909314199 100644 --- a/src/Parsers/ParserSelectQuery.cpp +++ b/src/Parsers/ParserSelectQuery.cpp @@ -247,11 +247,6 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { if (!exp_elem.parse(pos, limit_offset, expected)) return false; - if (s_with_ties.ignore(pos, expected)) - { - limit_with_ties_occured = true; - select_query->limit_with_ties = true; - } } /// Because TOP n in totally equals LIMIT n diff --git a/src/Processors/OffsetTransform.cpp b/src/Processors/OffsetTransform.cpp index 87574c160a6..dda36aa8323 100644 --- a/src/Processors/OffsetTransform.cpp +++ b/src/Processors/OffsetTransform.cpp @@ -10,11 +10,11 @@ namespace ErrorCodes } OffsetTransform::OffsetTransform( - const Block & header_, size_t limit_, size_t offset_, size_t num_streams, + const Block & header_, size_t offset_, size_t num_streams, bool always_read_till_end_, bool with_ties_, SortDescription description_) : IProcessor(InputPorts(num_streams, header_), OutputPorts(num_streams, header_)) - , limit(limit_), offset(offset_) + , offset(offset_) , always_read_till_end(always_read_till_end_) , with_ties(with_ties_), description(std::move(description_)) { @@ -107,19 +107,6 @@ IProcessor::Status OffsetTransform::prepare( if (num_finished_port_pairs == ports_data.size()) return Status::Finished; - /// If we reached limit for some port, then close others. Otherwise some sources may infinitely read data. - /// Example: SELECT * FROM system.numbers_mt WHERE number = 1000000 LIMIT 1 - // if ((rows_read >= offset) && !previous_row_chunk && !always_read_till_end) - // { - // for (auto & input : inputs) - // input.close(); - - // for (auto & output : outputs) - // output.finish(); - - //return Status::Finished; - //} - if (has_full_port) return Status::PortFull; @@ -145,11 +132,6 @@ OffsetTransform::Status OffsetTransform::preparePair(PortsData & data) if (output.isFinished()) { output_finished = true; - if (!always_read_till_end) - { - input.close(); - return Status::Finished; - } } if (!output_finished && !output.canPush()) @@ -167,8 +149,9 @@ OffsetTransform::Status OffsetTransform::preparePair(PortsData & data) } input.setNeeded(); - if (!input.hasData()) + if (!input.hasData()) { return Status::NeedData; + } data.current_chunk = input.pull(true); @@ -177,10 +160,14 @@ OffsetTransform::Status OffsetTransform::preparePair(PortsData & data) if (rows_before_limit_at_least) rows_before_limit_at_least->add(rows); - /// Skip block (for 'always_read_till_end' case). - if (output_finished) + /// Process block. + + rows_read += rows; + + if (rows_read < offset) { data.current_chunk.clear(); + if (input.isFinished()) { output.finish(); @@ -192,42 +179,9 @@ OffsetTransform::Status OffsetTransform::preparePair(PortsData & data) return Status::NeedData; } - /// Process block. - - rows_read += rows; - - //if (rows_read <= offset) - //{ - // data.current_chunk.clear(); - // - // if (input.isFinished()) - // { - // output.finish(); - // return Status::Finished; - // } - - /// Now, we pulled from input, and it must be empty. - // input.setNeeded(); - // return Status::NeedData; - //} - - if (rows_read >= offset + rows && rows_read <= offset) - { - /// Return the whole chunk. - - /// Save the last row of current chunk to check if next block begins with the same row (for WITH TIES). - if (with_ties && rows_read == offset + limit) - previous_row_chunk = makeChunkWithPreviousRow(data.current_chunk, data.current_chunk.getNumRows() - 1); - } - else - /// This function may be heavy to execute in prepare. But it happens no more then twice, and make code simpler. + if (!(rows_read >= offset + rows)) splitChunk(data); - //bool may_need_more_data_for_ties = previous_row_chunk || rows_read - rows <= offset; - /// No more data is needed. - //if (!always_read_till_end && (rows_read >= offset) && !may_need_more_data_for_ties) - // input.close(); - output.push(std::move(data.current_chunk)); return Status::PortFull; @@ -245,32 +199,7 @@ void OffsetTransform::splitChunk(PortsData & data) static_cast(0), static_cast(offset) - static_cast(rows_read) + static_cast(num_rows)); - //size_t length = std::min( - // static_cast(rows_read) - static_cast(offset), - // static_cast(offset) - static_cast(rows_read) + static_cast(num_rows)); - - size_t length = static_cast(num_rows); - std::cout << "===========================" << std::endl - << start << " " << length << std::endl - << static_cast(rows_read) << " " << static_cast(num_rows) << std::endl - << "===========================" << std::endl; - /// check if other rows in current block equals to last one in limit - if (with_ties && length) - { - size_t current_row_num = start + length; - previous_row_chunk = makeChunkWithPreviousRow(data.current_chunk, current_row_num - 1); - - for (; current_row_num < num_rows; ++current_row_num) - { - if (!sortColumnsEqualAt(current_chunk_sort_columns, current_row_num)) - { - previous_row_chunk = {}; - break; - } - } - - length = current_row_num - start; - } + size_t length = static_cast(rows_read) - static_cast(offset); if (length == num_rows) return; @@ -283,6 +212,7 @@ void OffsetTransform::splitChunk(PortsData & data) data.current_chunk.setColumns(std::move(columns), length); } + ColumnRawPtrs OffsetTransform::extractSortColumns(const Columns & columns) const { ColumnRawPtrs res; diff --git a/src/Processors/OffsetTransform.h b/src/Processors/OffsetTransform.h index 9194960cd20..66361a53684 100644 --- a/src/Processors/OffsetTransform.h +++ b/src/Processors/OffsetTransform.h @@ -19,7 +19,6 @@ class OffsetTransform : public IProcessor { private: - size_t limit; size_t offset; bool always_read_till_end; @@ -52,11 +51,11 @@ private: public: OffsetTransform( - const Block & header_, size_t limit_, size_t offset_, size_t num_streams = 1, + const Block & header_, size_t offset_, size_t num_streams = 1, bool always_read_till_end_ = false, bool with_ties_ = false, SortDescription description_ = {}); - String getName() const override { return "Limit"; } + String getName() const override { return "Offset"; } Status prepare(const PortNumbers & /*updated_input_ports*/, const PortNumbers & /*updated_output_ports*/) override; Status prepare() override; /// Compatibility for TreeExecutor. From abe28968f438ac3e9e56b44239450cfd7aff843e Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Thu, 7 May 2020 15:50:42 +0200 Subject: [PATCH 029/738] Adding tests. --- .../test_rename_column/__init__.py | 0 .../configs/remote_servers.xml | 28 ++ tests/integration/test_rename_column/test.py | 381 ++++++++++++++++++ 3 files changed, 409 insertions(+) create mode 100644 tests/integration/test_rename_column/__init__.py create mode 100644 tests/integration/test_rename_column/configs/remote_servers.xml create mode 100644 tests/integration/test_rename_column/test.py diff --git a/tests/integration/test_rename_column/__init__.py b/tests/integration/test_rename_column/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_rename_column/configs/remote_servers.xml b/tests/integration/test_rename_column/configs/remote_servers.xml new file mode 100644 index 00000000000..4c3de4b3905 --- /dev/null +++ b/tests/integration/test_rename_column/configs/remote_servers.xml @@ -0,0 +1,28 @@ + + + + + true + + node1 + 9000 + + + node2 + 9000 + + + + true + + node3 + 9000 + + + node4 + 9000 + + + + + diff --git a/tests/integration/test_rename_column/test.py b/tests/integration/test_rename_column/test.py new file mode 100644 index 00000000000..2e80c88e8f2 --- /dev/null +++ b/tests/integration/test_rename_column/test.py @@ -0,0 +1,381 @@ +from __future__ import print_function + +import time +import pytest + +from multiprocessing.dummy import Pool +from helpers.cluster import ClickHouseCluster +from helpers.client import QueryRuntimeException +from helpers.network import PartitionManager +from helpers.test_tools import TSV + +main_configs=['configs/remote_servers.xml'] + +cluster = ClickHouseCluster(__file__) +node1 = cluster.add_instance('node1', with_zookeeper=True, macros={"shard": 0, "replica": 1}, main_configs=main_configs ) +node2 = cluster.add_instance('node2', with_zookeeper=True, macros={"shard": 0, "replica": 2}, main_configs=main_configs ) +node3 = cluster.add_instance('node3', with_zookeeper=True, macros={"shard": 1, "replica": 1}, main_configs=main_configs ) +node4 = cluster.add_instance('node4', with_zookeeper=True, macros={"shard": 1, "replica": 2}, main_configs=main_configs ) +nodes = [node1, node2, node3, node4] + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + except Exception as ex: + print(ex) + finally: + cluster.shutdown() + +def drop_table(nodes, table_name): + for node in nodes: + node.query("DROP TABLE IF EXISTS {} NO DELAY".format(table_name)) + time.sleep(1) + +def create_table(nodes, table_name): + for node in nodes: + sql = """ + CREATE TABLE {table_name} + ( + num UInt32, + num2 UInt32 DEFAULT num + 1 + ) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{table_name}', '{replica}') + ORDER BY num PARTITION BY num % 100; + """.format(table_name=table_name, replica=node.name) + node.query(sql) + +def create_distributed_table(node, table_name): + sql = """ + CREATE TABLE %(table_name)s_replicated ON CLUSTER test_cluster + ( + num UInt32, + num2 UInt32 DEFAULT num + 1 + ) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{shard}/%(table_name)s_replicated', '{replica}') + ORDER BY num PARTITION BY num %% 100; + """ % dict(table_name=table_name) + node.query(sql) + sql = """ + CREATE TABLE %(table_name)s ON CLUSTER test_cluster AS %(table_name)s_replicated + ENGINE = Distributed(test_cluster, default, %(table_name)s_replicated, rand()) + """ % dict(table_name=table_name) + node.query(sql) + +def drop_distributed_table(node, table_name): + node.query("DROP TABLE IF EXISTS {} ON CLUSTER test_cluster".format(table_name)) + node.query("DROP TABLE IF EXISTS {}_replicated ON CLUSTER test_cluster".format(table_name)) + time.sleep(1) + +def insert(node, table_name, chunk=1000, col_names=None, iterations=1, ignore_exception=False, slow=False, with_many_parts=False, offset=0): + if col_names is None: + col_names = ['num', 'num2'] + for i in range(iterations): + try: + query = ["SET max_partitions_per_insert_block = 10000000"] + if with_many_parts: + query.append("SET max_insert_block_size = 64") + if slow: + query.append( + "INSERT INTO {table_name} ({col0}, {col1}) SELECT number + sleepEachRow(0.001) AS {col0}, number + 1 AS {col1} FROM numbers_mt({chunk})" + .format(table_name=table_name, chunk=chunk, col0=col_names[0], col1=col_names[1])) + else: + query.append( + "INSERT INTO {table_name} ({col0},{col1}) SELECT number + {offset} AS {col0}, number + 1 + {offset} AS {col1} FROM numbers_mt({chunk})" + .format(table_name=table_name, chunk=chunk, col0=col_names[0], col1=col_names[1], offset=str(offset))) + node.query(";\n".join(query)) + except QueryRuntimeException as ex: + if not ignore_exception: + raise + +def select(node, table_name, col_name="num", expected_result=None, iterations=1, ignore_exception=False, slow=False, poll=None): + for i in range(iterations): + start_time = time.time() + while True: + try: + if slow: + r = node.query("SELECT count() FROM (SELECT num2, sleepEachRow(0.5) FROM {} WHERE {} % 1000 > 0)".format(table_name, col_name)) + else: + r = node.query("SELECT count() FROM {} WHERE {} % 1000 > 0".format(table_name, col_name)) + if expected_result: + if r != expected_result and poll and time.time() - start_time < poll: + continue + assert r == expected_result + except QueryRuntimeException as ex: + if not ignore_exception: + raise + break + +def rename_column(node, table_name, name, new_name, iterations=1, ignore_exception=False): + for i in range(iterations): + try: + node.query("ALTER TABLE {table_name} RENAME COLUMN {name} to {new_name}".format( + table_name=table_name, name=name, new_name=new_name + )) + except QueryRuntimeException as ex: + if not ignore_exception: + raise + +def rename_column_on_cluster(node, table_name, name, new_name, iterations=1, ignore_exception=False): + for i in range(iterations): + try: + node.query("ALTER TABLE {table_name} ON CLUSTER test_cluster RENAME COLUMN {name} to {new_name}".format( + table_name=table_name, name=name, new_name=new_name + )) + except QueryRuntimeException as ex: + if not ignore_exception: + raise + +def test_rename_parallel_same_node(started_cluster): + table_name = "test_rename_parallel_same_node" + drop_table(nodes, table_name) + try: + create_table(nodes, table_name) + insert(node1, table_name, 1000) + + p = Pool(15) + tasks = [] + for i in range(1): + tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2", 5, True))) + tasks.append(p.apply_async(rename_column, (node1, table_name, "foo2", "foo3", 5, True))) + tasks.append(p.apply_async(rename_column, (node1, table_name, "foo3", "num2", 5, True))) + for task in tasks: + task.get(timeout=240) + + # rename column back to original + rename_column(node1, table_name, "foo3", "num2", 1, True) + rename_column(node1, table_name, "foo2", "num2", 1, True) + + # check that select still works + select(node1, table_name, "num2", "999\n") + finally: + drop_table(nodes, table_name) + +def test_rename_parallel(started_cluster): + table_name = "test_rename_parallel" + drop_table(nodes, table_name) + try: + create_table(nodes, table_name) + insert(node1, table_name, 1000) + + p = Pool(15) + tasks = [] + for i in range(1): + tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2", 5, True))) + tasks.append(p.apply_async(rename_column, (node2, table_name, "foo2", "foo3", 5, True))) + tasks.append(p.apply_async(rename_column, (node3, table_name, "foo3", "num2", 5, True))) + for task in tasks: + task.get(timeout=240) + + # rename column back to original + rename_column(node1, table_name, "foo3", "num2", 1, True) + rename_column(node1, table_name, "foo2", "num2", 1, True) + + # check that select still works + select(node1, table_name, "num2", "999\n") + finally: + drop_table(nodes, table_name) + +def test_rename_with_parallel_select(started_cluster): + table_name = "test_rename_with_parallel_select" + drop_table(nodes, table_name) + try: + create_table(nodes, table_name) + insert(node1, table_name, 1000) + + p = Pool(15) + tasks = [] + for i in range(1): + tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2", 5, True))) + tasks.append(p.apply_async(rename_column, (node2, table_name, "foo2", "foo3", 5, True))) + tasks.append(p.apply_async(rename_column, (node3, table_name, "foo3", "num2", 5, True))) + tasks.append(p.apply_async(select, (node1, table_name, "foo3", "999\n", 5, True))) + tasks.append(p.apply_async(select, (node2, table_name, "num2", "999\n", 5, True))) + tasks.append(p.apply_async(select, (node3, table_name, "foo2", "999\n", 5, True))) + for task in tasks: + task.get(timeout=240) + + # rename column back to original name + rename_column(node1, table_name, "foo3", "num2", 1, True) + rename_column(node1, table_name, "foo2", "num2", 1, True) + + # check that select still works + select(node1, table_name, "num2", "999\n") + finally: + drop_table(nodes, table_name) + +def test_rename_with_parallel_insert(started_cluster): + table_name = "test_rename_with_parallel_insert" + drop_table(nodes, table_name) + try: + create_table(nodes, table_name) + insert(node1, table_name, 1000) + + p = Pool(15) + tasks = [] + for i in range(1): + tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2", 5, True))) + tasks.append(p.apply_async(rename_column, (node2, table_name, "foo2", "foo3", 5, True))) + tasks.append(p.apply_async(rename_column, (node3, table_name, "foo3", "num2", 5, True))) + tasks.append(p.apply_async(insert, (node1, table_name, 100, ["num", "foo3"], 5, True))) + tasks.append(p.apply_async(insert, (node2, table_name, 100, ["num", "num2"], 5, True))) + tasks.append(p.apply_async(insert, (node3, table_name, 100, ["num", "foo2"], 5, True))) + for task in tasks: + task.get(timeout=240) + + # rename column back to original + rename_column(node1, table_name, "foo3", "num2", 1, True) + rename_column(node1, table_name, "foo2", "num2", 1, True) + + # check that select still works + select(node1, table_name, "num2") + finally: + drop_table(nodes, table_name) + +def test_rename_with_parallel_merges(started_cluster): + table_name = "test_rename_with_parallel_merges" + drop_table(nodes, table_name) + try: + create_table(nodes, table_name) + for i in range(20): + insert(node1, table_name, 100, ["num","num2"], 1, False, False, True, offset=i*100) + + def merge_parts(node, table_name, iterations=1): + for i in range(iterations): + node.query("OPTIMIZE TABLE %s FINAL" % table_name) + + p = Pool(15) + tasks = [] + for i in range(1): + tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2", 25, True))) + tasks.append(p.apply_async(rename_column, (node2, table_name, "foo2", "foo3", 25, True))) + tasks.append(p.apply_async(rename_column, (node3, table_name, "foo3", "num2", 25, True))) + tasks.append(p.apply_async(merge_parts, (node1, table_name, 15))) + tasks.append(p.apply_async(merge_parts, (node2, table_name, 15))) + tasks.append(p.apply_async(merge_parts, (node3, table_name, 15))) + + for task in tasks: + task.get(timeout=240) + + # rename column back to the original name + rename_column(node1, table_name, "foo3", "num2", 1, True) + rename_column(node1, table_name, "foo2", "num2", 1, True) + + # check that select still works + select(node1, table_name, "num2", "1998\n") + select(node2, table_name, "num2", "1998\n") + select(node3, table_name, "num2", "1998\n") + finally: + drop_table(nodes, table_name) + +def test_rename_with_parallel_slow_insert(started_cluster): + table_name = "test_rename_with_parallel_slow_insert" + drop_table(nodes, table_name) + try: + create_table(nodes, table_name) + insert(node1, table_name, 1000) + + p = Pool(15) + tasks = [] + tasks.append(p.apply_async(insert, (node1, table_name, 10000, ["num", "num2"], 1, False, True))) + tasks.append(p.apply_async(insert, (node1, table_name, 10000, ["num", "num2"], 1, True, True))) # deduplicated + time.sleep(0.5) + tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2"))) + + for task in tasks: + task.get(timeout=240) + + insert(node1, table_name, 100, ["num", "foo2"]) + + # rename column back to original + rename_column(node1, table_name, "foo2", "num2") + + # check that select still works + select(node1, table_name, "num2", "11089\n") + select(node2, table_name, "num2", "11089\n", poll=30) + select(node3, table_name, "num2", "11089\n", poll=30) + finally: + drop_table(nodes, table_name) + +def test_rename_with_parallel_slow_select(started_cluster): + table_name = "test_rename_with_parallel_slow_select" + drop_table(nodes, table_name) + try: + create_table(nodes, table_name) + insert(node1, table_name, 1000) + + p = Pool(15) + tasks = [] + + tasks.append(p.apply_async(select, (node1, table_name, "num2", "999\n", 1, True, True))) + time.sleep(0.25) + tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2"))) + + for task in tasks: + task.get(timeout=240) + + insert(node1, table_name, 100, ["num", "foo2"]) + + # rename column back to original + rename_column(node1, table_name, "foo2", "num2") + + # check that select still works + select(node1, table_name, "num2", "1099\n") + select(node2, table_name, "num2", "1099\n", poll=30) + select(node3, table_name, "num2", "1099\n", poll=30) + finally: + drop_table(nodes, table_name) + +def test_rename_distributed(started_cluster): + table_name = 'test_rename_distributed' + try: + create_distributed_table(node1, table_name) + insert(node1, table_name, 1000) + + rename_column_on_cluster(node1, table_name, 'num2', 'foo2') + rename_column_on_cluster(node1, '%s_replicated' % table_name, 'num2', 'foo2') + + insert(node1, table_name, 1000, col_names=['num','foo2']) + + select(node1, table_name, "foo2", '1998\n', poll=30) + finally: + drop_distributed_table(node1, table_name) + +def test_rename_distributed_parallel_insert_and_select(started_cluster): + table_name = 'test_rename_distributed_parallel_insert_and_select' + try: + create_distributed_table(node1, table_name) + insert(node1, table_name, 1000) + + p = Pool(15) + tasks = [] + for i in range(1): + tasks.append(p.apply_async(rename_column_on_cluster, (node1, table_name, 'num2', 'foo2', 5, True))) + tasks.append(p.apply_async(rename_column_on_cluster, (node1, '%s_replicated' % table_name, 'num2', 'foo2', 5, True))) + tasks.append(p.apply_async(rename_column_on_cluster, (node1, table_name, 'foo2', 'foo3', 5, True))) + tasks.append(p.apply_async(rename_column_on_cluster, (node1, '%s_replicated' % table_name, 'foo2', 'foo3', 5, True))) + tasks.append(p.apply_async(rename_column_on_cluster, (node1, table_name, 'foo3', 'num2', 5, True))) + tasks.append(p.apply_async(rename_column_on_cluster, (node1, '%s_replicated' % table_name, 'foo3', 'num2', 5, True))) + tasks.append(p.apply_async(insert, (node1, table_name, 100, ["num", "foo3"], 5, True))) + tasks.append(p.apply_async(insert, (node2, table_name, 100, ["num", "num2"], 5, True))) + tasks.append(p.apply_async(insert, (node3, table_name, 100, ["num", "foo2"], 5, True))) + tasks.append(p.apply_async(select, (node1, table_name, "foo2", None, 5, True))) + tasks.append(p.apply_async(select, (node2, table_name, "foo3", None, 5, True))) + tasks.append(p.apply_async(select, (node3, table_name, "num2", None, 5, True))) + for task in tasks: + task.get(timeout=240) + + rename_column_on_cluster(node1, table_name, 'foo2', 'num2', 1, True) + rename_column_on_cluster(node1, '%s_replicated' % table_name, 'foo2', 'num2', 1, True) + rename_column_on_cluster(node1, table_name, 'foo3', 'num2', 1, True) + rename_column_on_cluster(node1, '%s_replicated' % table_name, 'foo3', 'num2', 1, True) + + insert(node1, table_name, 1000, col_names=['num','num2']) + select(node1, table_name, "num2") + select(node2, table_name, "num2") + select(node3, table_name, "num2") + select(node4, table_name, "num2") + finally: + drop_distributed_table(node1, table_name) + From 215c9c617d3c7fb035b7654bebb4deb2c0faaa7b Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Thu, 7 May 2020 17:01:58 +0300 Subject: [PATCH 030/738] Update AWS module. --- contrib/aws | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/aws b/contrib/aws index 04d54dfa034..fb5c604525f 160000 --- a/contrib/aws +++ b/contrib/aws @@ -1 +1 @@ -Subproject commit 04d54dfa0342d9465fb2eb3bfd4b77a3f7682e99 +Subproject commit fb5c604525f5151d75a856462653e7e38b559b79 From 6d071c430ebe8160d99450e0137ad5f89806b7fb Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Thu, 7 May 2020 19:39:33 +0200 Subject: [PATCH 031/738] Adding stateless test for alter rename column on a distributed table. --- ...74_alter_rename_column_distributed.reference | 4 ++++ .../01274_alter_rename_column_distributed.sql | 17 +++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 tests/queries/0_stateless/01274_alter_rename_column_distributed.reference create mode 100644 tests/queries/0_stateless/01274_alter_rename_column_distributed.sql diff --git a/tests/queries/0_stateless/01274_alter_rename_column_distributed.reference b/tests/queries/0_stateless/01274_alter_rename_column_distributed.reference new file mode 100644 index 00000000000..7ed221867c1 --- /dev/null +++ b/tests/queries/0_stateless/01274_alter_rename_column_distributed.reference @@ -0,0 +1,4 @@ +2020-01-01 hello +2020-01-01 hello +2020-01-02 hello2 +2020-01-02 hello2 diff --git a/tests/queries/0_stateless/01274_alter_rename_column_distributed.sql b/tests/queries/0_stateless/01274_alter_rename_column_distributed.sql new file mode 100644 index 00000000000..a35dc7cca56 --- /dev/null +++ b/tests/queries/0_stateless/01274_alter_rename_column_distributed.sql @@ -0,0 +1,17 @@ +DROP TABLE IF EXISTS visits; +DROP TABLE IF EXISTS visits_dist; + +CREATE TABLE visits(StartDate Date, Name String) ENGINE MergeTree ORDER BY(StartDate); +CREATE TABLE visits_dist AS visits ENGINE Distributed(test_cluster_two_shards_localhost, currentDatabase(), 'visits', rand()); + +INSERT INTO visits_dist (StartDate, Name) VALUES ('2020-01-01', 'hello'); +INSERT INTO visits_dist (StartDate, Name) VALUES ('2020-01-02', 'hello2'); + +ALTER TABLE visits RENAME COLUMN Name TO Name2; +ALTER TABLE visits_dist RENAME COLUMN Name TO Name2; + +SELECT * FROM visits_dist ORDER BY StartDate, Name2; + +DROP TABLE visits; +DROP TABLE visits_dist; + From 65f139f374fe60676f4a3c62e11d234efd6d0787 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Thu, 7 May 2020 21:00:17 +0200 Subject: [PATCH 032/738] Adding more integration tests for alter rename column. --- tests/integration/test_rename_column/test.py | 172 +++++++++++++++++-- 1 file changed, 156 insertions(+), 16 deletions(-) diff --git a/tests/integration/test_rename_column/test.py b/tests/integration/test_rename_column/test.py index 2e80c88e8f2..b09a937211c 100644 --- a/tests/integration/test_rename_column/test.py +++ b/tests/integration/test_rename_column/test.py @@ -1,6 +1,7 @@ from __future__ import print_function import time +import random import pytest from multiprocessing.dummy import Pool @@ -9,13 +10,17 @@ from helpers.client import QueryRuntimeException from helpers.network import PartitionManager from helpers.test_tools import TSV -main_configs=['configs/remote_servers.xml'] +node_options = dict( + with_zookeeper=True, + main_configs=['configs/remote_servers.xml'], + config_dir='configs', + tmpfs=['/external:size=200M']) cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_zookeeper=True, macros={"shard": 0, "replica": 1}, main_configs=main_configs ) -node2 = cluster.add_instance('node2', with_zookeeper=True, macros={"shard": 0, "replica": 2}, main_configs=main_configs ) -node3 = cluster.add_instance('node3', with_zookeeper=True, macros={"shard": 1, "replica": 1}, main_configs=main_configs ) -node4 = cluster.add_instance('node4', with_zookeeper=True, macros={"shard": 1, "replica": 2}, main_configs=main_configs ) +node1 = cluster.add_instance('node1', macros={"shard": 0, "replica": 1}, **node_options) +node2 = cluster.add_instance('node2', macros={"shard": 0, "replica": 2}, **node_options) +node3 = cluster.add_instance('node3', macros={"shard": 1, "replica": 1}, **node_options) +node4 = cluster.add_instance('node4', macros={"shard": 1, "replica": 2}, **node_options) nodes = [node1, node2, node3, node4] @pytest.fixture(scope="module") @@ -33,18 +38,44 @@ def drop_table(nodes, table_name): node.query("DROP TABLE IF EXISTS {} NO DELAY".format(table_name)) time.sleep(1) -def create_table(nodes, table_name): +def create_table(nodes, table_name, with_storage_policy=False, with_time_column=False, + with_ttl_move=False, with_ttl_delete=False): + extra_columns = "" + settings = [] + for node in nodes: sql = """ CREATE TABLE {table_name} - ( - num UInt32, - num2 UInt32 DEFAULT num + 1 - ) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{table_name}', '{replica}') - ORDER BY num PARTITION BY num % 100; - """.format(table_name=table_name, replica=node.name) - node.query(sql) + ( + num UInt32, + num2 UInt32 DEFAULT num + 1{extra_columns} + ) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{table_name}', '{replica}') + ORDER BY num PARTITION BY num % 100 + """ + if with_ttl_move: + sql += """ + TTL time TO DISK 'external' + """ + if with_ttl_delete: + sql += """ + TTL time DELETE + """ + settings.append("merge_with_ttl_timeout = 1") + + if with_storage_policy: + settings.append("storage_policy='default_with_external'") + + if settings: + sql += """ + SETTINGS {} + """.format(", ".join(settings)) + + if with_time_column: + extra_columns = """, + time DateTime + """ + node.query(sql.format(table_name=table_name, replica=node.name, extra_columns=extra_columns)) def create_distributed_table(node, table_name): sql = """ @@ -68,7 +99,8 @@ def drop_distributed_table(node, table_name): node.query("DROP TABLE IF EXISTS {}_replicated ON CLUSTER test_cluster".format(table_name)) time.sleep(1) -def insert(node, table_name, chunk=1000, col_names=None, iterations=1, ignore_exception=False, slow=False, with_many_parts=False, offset=0): +def insert(node, table_name, chunk=1000, col_names=None, iterations=1, ignore_exception=False, + slow=False, with_many_parts=False, offset=0, with_time_column=False): if col_names is None: col_names = ['num', 'num2'] for i in range(iterations): @@ -76,7 +108,11 @@ def insert(node, table_name, chunk=1000, col_names=None, iterations=1, ignore_ex query = ["SET max_partitions_per_insert_block = 10000000"] if with_many_parts: query.append("SET max_insert_block_size = 64") - if slow: + if with_time_column: + query.append( + "INSERT INTO {table_name} ({col0}, {col1}, time) SELECT number AS {col0}, number + 1 AS {col1}, now() + 10 AS time FROM numbers_mt({chunk})" + .format(table_name=table_name, chunk=chunk, col0=col_names[0], col1=col_names[1])) + elif slow: query.append( "INSERT INTO {table_name} ({col0}, {col1}) SELECT number + sleepEachRow(0.001) AS {col0}, number + 1 AS {col1} FROM numbers_mt({chunk})" .format(table_name=table_name, chunk=chunk, col0=col_names[0], col1=col_names[1])) @@ -127,6 +163,17 @@ def rename_column_on_cluster(node, table_name, name, new_name, iterations=1, ign if not ignore_exception: raise +def alter_move(node, table_name, iterations=1, ignore_exception=False): + for i in range(iterations): + move_part = random.randint(0, 99) + move_volume = 'external'# if random.randint(0, 1) > 0 else 'default' + try: + node.query("ALTER TABLE {table_name} MOVE PARTITION '{move_part}' TO VOLUME '{move_volume}'" + .format(table_name=table_name, move_part=move_part, move_volume=move_volume)) + except QueryRuntimeException as ex: + if not ignore_exception: + raise + def test_rename_parallel_same_node(started_cluster): table_name = "test_rename_parallel_same_node" drop_table(nodes, table_name) @@ -327,6 +374,99 @@ def test_rename_with_parallel_slow_select(started_cluster): finally: drop_table(nodes, table_name) +def test_rename_with_parallel_moves(started_cluster): + table_name = "test_rename_with_parallel_moves" + drop_table(nodes, table_name) + try: + create_table(nodes, table_name, with_storage_policy=True) + insert(node1, table_name, 1000) + + p = Pool(15) + tasks = [] + + tasks.append(p.apply_async(alter_move, (node1, table_name, 100, True))) + tasks.append(p.apply_async(alter_move, (node2, table_name, 100, True))) + tasks.append(p.apply_async(alter_move, (node3, table_name, 100, True))) + tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2", 100, True))) + tasks.append(p.apply_async(rename_column, (node2, table_name, "foo2", "foo3", 100, True))) + tasks.append(p.apply_async(rename_column, (node3, table_name, "num3", "num2", 100, True))) + + for task in tasks: + task.get(timeout=240) + + # rename column back to original + rename_column(node1, table_name, "foo2", "num2", 1, True) + rename_column(node1, table_name, "foo3", "num2", 1, True) + + # check that select still works + select(node1, table_name, "num2", "999\n") + select(node2, table_name, "num2", "999\n", poll=30) + select(node3, table_name, "num2", "999\n", poll=30) + finally: + drop_table(nodes, table_name) + +def test_rename_with_parallel_ttl_move(started_cluster): + table_name = 'test_rename_with_parallel_ttl_move' + try: + create_table(nodes, table_name, with_storage_policy=True, with_time_column=True, with_ttl_move=True) + rename_column(node1, table_name, "time", "time2", 1, False) + rename_column(node1, table_name, "time2", "time", 1, False) + + p = Pool(15) + tasks = [] + + tasks.append(p.apply_async(insert, (node1, table_name, 10000, ["num", "num2"], 1, False, False, True, 0, True))) + time.sleep(5) + tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2", 500, True))) + tasks.append(p.apply_async(rename_column, (node2, table_name, "foo2", "foo3", 500, True))) + tasks.append(p.apply_async(rename_column, (node3, table_name, "num3", "num2", 500, True))) + + for task in tasks: + task.get(timeout=240) + + # rename column back to original + rename_column(node1, table_name, "foo2", "num2", 1, True) + rename_column(node1, table_name, "foo3", "num2", 1, True) + + # check that select still works + select(node1, table_name, "num2", "9990\n") + finally: + drop_table(nodes, table_name) + +def test_rename_with_parallel_ttl_delete(started_cluster): + table_name = 'test_rename_with_parallel_ttl_delete' + try: + create_table(nodes, table_name, with_time_column=True, with_ttl_delete=True) + rename_column(node1, table_name, "time", "time2", 1, False) + rename_column(node1, table_name, "time2", "time", 1, False) + + def merge_parts(node, table_name, iterations=1): + for i in range(iterations): + node.query("OPTIMIZE TABLE {}".format(table_name)) + + p = Pool(15) + tasks = [] + + tasks.append(p.apply_async(insert, (node1, table_name, 10000, ["num", "num2"], 1, False, False, True, 0, True))) + time.sleep(15) + tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2", 500, True))) + tasks.append(p.apply_async(rename_column, (node2, table_name, "foo2", "foo3", 500, True))) + tasks.append(p.apply_async(rename_column, (node3, table_name, "num3", "num2", 500, True))) + tasks.append(p.apply_async(merge_parts, (node1, table_name, 500))) + tasks.append(p.apply_async(merge_parts, (node2, table_name, 500))) + tasks.append(p.apply_async(merge_parts, (node3, table_name, 500))) + + for task in tasks: + task.get(timeout=240) + + # rename column back to original + rename_column(node1, table_name, "foo2", "num2", 1, True) + rename_column(node1, table_name, "foo3", "num2", 1, True) + + assert int(node1.query("SELECT count() FROM {}".format(table_name)).strip()) < 9000 + finally: + drop_table(nodes, table_name) + def test_rename_distributed(started_cluster): table_name = 'test_rename_distributed' try: From eb3c6ca0d88067aee855db4a1b811230a6c409e7 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Fri, 8 May 2020 12:36:37 +0200 Subject: [PATCH 033/738] Adding configuration files. --- .../configs/config.d/instant_moves.xml | 4 +++ .../configs/config.d/part_log.xml | 7 ++++++ .../config.d/storage_configuration.xml | 25 +++++++++++++++++++ 3 files changed, 36 insertions(+) create mode 100644 tests/integration/test_rename_column/configs/config.d/instant_moves.xml create mode 100644 tests/integration/test_rename_column/configs/config.d/part_log.xml create mode 100644 tests/integration/test_rename_column/configs/config.d/storage_configuration.xml diff --git a/tests/integration/test_rename_column/configs/config.d/instant_moves.xml b/tests/integration/test_rename_column/configs/config.d/instant_moves.xml new file mode 100644 index 00000000000..7b68c6946ca --- /dev/null +++ b/tests/integration/test_rename_column/configs/config.d/instant_moves.xml @@ -0,0 +1,4 @@ + + 0.5 + 0.5 + diff --git a/tests/integration/test_rename_column/configs/config.d/part_log.xml b/tests/integration/test_rename_column/configs/config.d/part_log.xml new file mode 100644 index 00000000000..53ea0a8fc13 --- /dev/null +++ b/tests/integration/test_rename_column/configs/config.d/part_log.xml @@ -0,0 +1,7 @@ + + + system + part_log
+ 500 + + diff --git a/tests/integration/test_rename_column/configs/config.d/storage_configuration.xml b/tests/integration/test_rename_column/configs/config.d/storage_configuration.xml new file mode 100644 index 00000000000..044009512bc --- /dev/null +++ b/tests/integration/test_rename_column/configs/config.d/storage_configuration.xml @@ -0,0 +1,25 @@ + + + + + + + + /external/ + + + + + + + default + + + external + + + + + + + From 6165c2aa991639ef8b4581b77ff5cc76d595c6a0 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Fri, 8 May 2020 13:53:12 +0300 Subject: [PATCH 034/738] Add possibility to use proxy resolver in DiskS3. --- .../compose/docker_compose_minio.yml | 7 ++ src/Disks/S3/DiskS3.cpp | 2 +- src/Disks/S3/DiskS3.h | 6 +- src/Disks/S3/DynamicProxyConfiguration.h | 24 ------ src/Disks/S3/ProxyConfiguration.h | 18 +++++ ...uration.cpp => ProxyListConfiguration.cpp} | 7 +- src/Disks/S3/ProxyListConfiguration.h | 23 ++++++ src/Disks/S3/ProxyResolverConfiguration.cpp | 59 ++++++++++++++ src/Disks/S3/ProxyResolverConfiguration.h | 28 +++++++ src/Disks/S3/registerDiskS3.cpp | 76 ++++++++++++++----- src/Disks/S3/ya.make | 3 +- tests/integration/helpers/cluster.py | 53 +++++++------ .../configs/config.d/storage_conf.xml | 20 +++++ .../proxy-resolver/__init__.py | 0 .../proxy-resolver/entrypoint.sh | 4 + .../proxy-resolver/resolver.py | 13 ++++ tests/integration/test_s3_with_proxy/test.py | 23 +++++- 17 files changed, 288 insertions(+), 78 deletions(-) delete mode 100644 src/Disks/S3/DynamicProxyConfiguration.h create mode 100644 src/Disks/S3/ProxyConfiguration.h rename src/Disks/S3/{DynamicProxyConfiguration.cpp => ProxyListConfiguration.cpp} (55%) create mode 100644 src/Disks/S3/ProxyListConfiguration.h create mode 100644 src/Disks/S3/ProxyResolverConfiguration.cpp create mode 100644 src/Disks/S3/ProxyResolverConfiguration.h create mode 100644 tests/integration/test_s3_with_proxy/proxy-resolver/__init__.py create mode 100644 tests/integration/test_s3_with_proxy/proxy-resolver/entrypoint.sh create mode 100644 tests/integration/test_s3_with_proxy/proxy-resolver/resolver.py diff --git a/docker/test/integration/compose/docker_compose_minio.yml b/docker/test/integration/compose/docker_compose_minio.yml index 5b93369e3e2..ec35c87fa00 100644 --- a/docker/test/integration/compose/docker_compose_minio.yml +++ b/docker/test/integration/compose/docker_compose_minio.yml @@ -38,5 +38,12 @@ services: ports: - "4082:8888" +# Empty container to run proxy resolver. + resolver: + image: python:3 + ports: + - "4083:8080" + tty: true + volumes: data1-1: diff --git a/src/Disks/S3/DiskS3.cpp b/src/Disks/S3/DiskS3.cpp index 47ca8231001..7643516e197 100644 --- a/src/Disks/S3/DiskS3.cpp +++ b/src/Disks/S3/DiskS3.cpp @@ -391,7 +391,7 @@ private: DiskS3::DiskS3( String name_, std::shared_ptr client_, - std::shared_ptr proxy_configuration_, + std::shared_ptr proxy_configuration_, String bucket_, String s3_root_path_, String metadata_path_, diff --git a/src/Disks/S3/DiskS3.h b/src/Disks/S3/DiskS3.h index 889a6aa97a3..9930a83fd2b 100644 --- a/src/Disks/S3/DiskS3.h +++ b/src/Disks/S3/DiskS3.h @@ -1,7 +1,7 @@ #pragma once #include "Disks/DiskFactory.h" -#include "DynamicProxyConfiguration.h" +#include "ProxyListConfiguration.h" #include #include @@ -22,7 +22,7 @@ public: DiskS3( String name_, std::shared_ptr client_, - std::shared_ptr proxy_configuration_, + std::shared_ptr proxy_configuration_, String bucket_, String s3_root_path_, String metadata_path_, @@ -102,7 +102,7 @@ private: private: const String name; std::shared_ptr client; - std::shared_ptr proxy_configuration; + std::shared_ptr proxy_configuration; const String bucket; const String s3_root_path; const String metadata_path; diff --git a/src/Disks/S3/DynamicProxyConfiguration.h b/src/Disks/S3/DynamicProxyConfiguration.h deleted file mode 100644 index 17eb2f0bb9e..00000000000 --- a/src/Disks/S3/DynamicProxyConfiguration.h +++ /dev/null @@ -1,24 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -namespace DB::S3 -{ -class DynamicProxyConfiguration -{ -public: - explicit DynamicProxyConfiguration(std::vector _proxies); - /// Returns proxy configuration on each HTTP request. - Aws::Client::ClientConfigurationPerRequest getConfiguration(const Aws::Http::HttpRequest & request); - -private: - /// List of configured proxies. - const std::vector proxies; - /// Access counter to get proxy using round-robin strategy. - std::atomic access_counter; -}; - -} diff --git a/src/Disks/S3/ProxyConfiguration.h b/src/Disks/S3/ProxyConfiguration.h new file mode 100644 index 00000000000..62aec0e005e --- /dev/null +++ b/src/Disks/S3/ProxyConfiguration.h @@ -0,0 +1,18 @@ +#pragma once + +#include +#include +#include +#include + +namespace DB::S3 +{ +class ProxyConfiguration +{ +public: + virtual ~ProxyConfiguration() = default; + /// Returns proxy configuration on each HTTP request. + virtual Aws::Client::ClientConfigurationPerRequest getConfiguration(const Aws::Http::HttpRequest & request) = 0; +}; + +} diff --git a/src/Disks/S3/DynamicProxyConfiguration.cpp b/src/Disks/S3/ProxyListConfiguration.cpp similarity index 55% rename from src/Disks/S3/DynamicProxyConfiguration.cpp rename to src/Disks/S3/ProxyListConfiguration.cpp index 92a38762e39..bc68635fa7b 100644 --- a/src/Disks/S3/DynamicProxyConfiguration.cpp +++ b/src/Disks/S3/ProxyListConfiguration.cpp @@ -1,21 +1,22 @@ -#include "DynamicProxyConfiguration.h" +#include "ProxyListConfiguration.h" #include #include namespace DB::S3 { -DynamicProxyConfiguration::DynamicProxyConfiguration(std::vector _proxies) : proxies(std::move(_proxies)), access_counter(0) +ProxyListConfiguration::ProxyListConfiguration(std::vector proxies_) : proxies(std::move(proxies_)), access_counter(0) { } -Aws::Client::ClientConfigurationPerRequest DynamicProxyConfiguration::getConfiguration(const Aws::Http::HttpRequest &) +Aws::Client::ClientConfigurationPerRequest ProxyListConfiguration::getConfiguration(const Aws::Http::HttpRequest &) { /// Avoid atomic increment if number of proxies is 1. size_t index = proxies.size() > 1 ? (access_counter++) % proxies.size() : 0; Aws::Client::ClientConfigurationPerRequest cfg; + cfg.proxyScheme = Aws::Http::SchemeMapper::FromString(proxies[index].getScheme().c_str()); cfg.proxyHost = proxies[index].getHost(); cfg.proxyPort = proxies[index].getPort(); diff --git a/src/Disks/S3/ProxyListConfiguration.h b/src/Disks/S3/ProxyListConfiguration.h new file mode 100644 index 00000000000..a3fe83bfc49 --- /dev/null +++ b/src/Disks/S3/ProxyListConfiguration.h @@ -0,0 +1,23 @@ +#pragma once + +#include "ProxyConfiguration.h" + +namespace DB::S3 +{ +/** + * For each request to S3 it chooses a proxy from the specified list using round-robin strategy. + */ +class ProxyListConfiguration : public ProxyConfiguration +{ +public: + explicit ProxyListConfiguration(std::vector proxies_); + Aws::Client::ClientConfigurationPerRequest getConfiguration(const Aws::Http::HttpRequest & request) override; + +private: + /// List of configured proxies. + const std::vector proxies; + /// Access counter to get proxy using round-robin strategy. + std::atomic access_counter; +}; + +} diff --git a/src/Disks/S3/ProxyResolverConfiguration.cpp b/src/Disks/S3/ProxyResolverConfiguration.cpp new file mode 100644 index 00000000000..dd5af0990b8 --- /dev/null +++ b/src/Disks/S3/ProxyResolverConfiguration.cpp @@ -0,0 +1,59 @@ +#include "ProxyResolverConfiguration.h" + +#include +#include "Poco/StreamCopier.h" +#include +#include +#include + +namespace DB::S3 +{ +ProxyResolverConfiguration::ProxyResolverConfiguration(const Poco::URI & endpoint_, String proxy_scheme_, unsigned proxy_port_) + : endpoint(endpoint_), proxy_scheme(std::move(proxy_scheme_)), proxy_port(proxy_port_) +{ +} + +Aws::Client::ClientConfigurationPerRequest ProxyResolverConfiguration::getConfiguration(const Aws::Http::HttpRequest &) +{ + LOG_DEBUG(&Logger::get("AWSClient"), "Obtain proxy using resolver: " << endpoint.toString()); + + /// 1 second is enough for now. + /// TODO: Make timeouts configurable. + ConnectionTimeouts timeouts( + Poco::Timespan(1000000), /// Connection timeout. + Poco::Timespan(1000000), /// Send timeout. + Poco::Timespan(1000000) /// Receive timeout. + ); + auto session = makeHTTPSession(endpoint, timeouts); + + Aws::Client::ClientConfigurationPerRequest cfg; + try + { + /// It should be just empty GET / request. + Poco::Net::HTTPRequest request(Poco::Net::HTTPRequest::HTTP_1_1); + session->sendRequest(request); + + Poco::Net::HTTPResponse response; + auto & response_body_stream = session->receiveResponse(response); + + String proxy_host; + /// Read proxy host as string from response body. + Poco::StreamCopier::copyToString(response_body_stream, proxy_host); + + LOG_DEBUG(&Logger::get("AWSClient"), "Use proxy: " << proxy_scheme << "://" << proxy_host << ":" << proxy_port); + + cfg.proxyScheme = Aws::Http::SchemeMapper::FromString(proxy_scheme.c_str()); + cfg.proxyHost = proxy_host; + cfg.proxyPort = proxy_port; + + return cfg; + } + catch (Exception & e) + { + LOG_ERROR(&Logger::get("AWSClient"), "Failed to obtain proxy: " << e.message()); + /// Don't use proxy if it can't be obtained. + return cfg; + } +} + +} diff --git a/src/Disks/S3/ProxyResolverConfiguration.h b/src/Disks/S3/ProxyResolverConfiguration.h new file mode 100644 index 00000000000..2944121a078 --- /dev/null +++ b/src/Disks/S3/ProxyResolverConfiguration.h @@ -0,0 +1,28 @@ +#pragma once + +#include "ProxyConfiguration.h" +#include + +namespace DB::S3 +{ +/** + * Proxy configuration where proxy host is obtained each time from specified endpoint. + * For each request to S3 it makes GET request to specified endpoint and reads proxy host from a response body. + * Specified scheme and port added to obtained proxy host to form completed proxy URL. + */ +class ProxyResolverConfiguration : public ProxyConfiguration +{ +public: + ProxyResolverConfiguration(const Poco::URI & endpoint_, String proxy_scheme_, unsigned proxy_port_); + Aws::Client::ClientConfigurationPerRequest getConfiguration(const Aws::Http::HttpRequest & request) override; + +private: + /// Endpoint to obtain a proxy host. + const Poco::URI endpoint; + /// Scheme for obtained proxy. + const String proxy_scheme; + /// Port for obtained proxy. + const unsigned proxy_port; +}; + +} diff --git a/src/Disks/S3/registerDiskS3.cpp b/src/Disks/S3/registerDiskS3.cpp index ae753a2e9c3..c4eb877f30e 100644 --- a/src/Disks/S3/registerDiskS3.cpp +++ b/src/Disks/S3/registerDiskS3.cpp @@ -6,7 +6,9 @@ #include #include "DiskS3.h" #include "Disks/DiskFactory.h" -#include "DynamicProxyConfiguration.h" +#include "ProxyConfiguration.h" +#include "ProxyListConfiguration.h" +#include "ProxyResolverConfiguration.h" namespace DB { @@ -35,35 +37,67 @@ namespace void checkRemoveAccess(IDisk & disk) { disk.remove("test_acl"); } - std::shared_ptr getProxyConfiguration(const Poco::Util::AbstractConfiguration * config) + std::shared_ptr getProxyResolverConfiguration(const Poco::Util::AbstractConfiguration * proxy_resolver_config) { - if (config->has("proxy")) - { - std::vector keys; - config->keys("proxy", keys); + auto endpoint = Poco::URI(proxy_resolver_config->getString("endpoint")); + auto scheme = proxy_resolver_config->getString("scheme"); + if (scheme != "http" && scheme != "https") + throw Exception("Only HTTP/HTTPS schemas allowed in proxy resolver config: " + scheme, ErrorCodes::BAD_ARGUMENTS); + auto port = proxy_resolver_config->getUInt("port"); - std::vector proxies; - for (const auto & key : keys) - if (startsWith(key, "uri")) - { - Poco::URI proxy_uri(config->getString("proxy." + key)); + LOG_DEBUG( + &Logger::get("DiskS3"), "Configured proxy resolver: " << endpoint.toString() << ", Scheme: " << scheme << ", Port: " << port); - if (proxy_uri.getScheme() != "http") - throw Exception("Only HTTP scheme is allowed in proxy configuration at the moment, proxy uri: " + proxy_uri.toString(), ErrorCodes::BAD_ARGUMENTS); - if (proxy_uri.getHost().empty()) - throw Exception("Empty host in proxy configuration, proxy uri: " + proxy_uri.toString(), ErrorCodes::BAD_ARGUMENTS); + return std::make_shared(endpoint, scheme, port); + } - proxies.push_back(proxy_uri); + std::shared_ptr getProxyListConfiguration(const Poco::Util::AbstractConfiguration * proxy_config) + { + std::vector keys; + proxy_config->keys(keys); - LOG_DEBUG(&Logger::get("DiskS3"), "Configured proxy: " << proxy_uri.toString()); - } + std::vector proxies; + for (const auto & key : keys) + if (startsWith(key, "uri")) + { + Poco::URI proxy_uri(proxy_config->getString(key)); + + if (proxy_uri.getScheme() != "http" && proxy_uri.getScheme() != "https") + throw Exception("Only HTTP/HTTPS schemas allowed in proxy uri: " + proxy_uri.toString(), ErrorCodes::BAD_ARGUMENTS); + if (proxy_uri.getHost().empty()) + throw Exception("Empty host in proxy uri: " + proxy_uri.toString(), ErrorCodes::BAD_ARGUMENTS); + + proxies.push_back(proxy_uri); + + LOG_DEBUG(&Logger::get("DiskS3"), "Configured proxy: " << proxy_uri.toString() << " " << key); + } + + if (!proxies.empty()) + return std::make_shared(proxies); - if (!proxies.empty()) - return std::make_shared(proxies); - } return nullptr; } + std::shared_ptr getProxyConfiguration(const Poco::Util::AbstractConfiguration * config) + { + if (!config->has("proxy")) + return nullptr; + + const auto * proxy_config = config->createView("proxy"); + + std::vector configKeys; + proxy_config->keys(configKeys); + + if (auto resolverConfigs = std::count(configKeys.begin(), configKeys.end(), "resolver")) + { + if (resolverConfigs > 1) + throw Exception("Multiple proxy resolver configurations aren't allowed", ErrorCodes::BAD_ARGUMENTS); + + return getProxyResolverConfiguration(proxy_config->createView("resolver")); + } + + return getProxyListConfiguration(proxy_config); + } } diff --git a/src/Disks/S3/ya.make b/src/Disks/S3/ya.make index 446e7bd1cb2..66a32e6f0df 100644 --- a/src/Disks/S3/ya.make +++ b/src/Disks/S3/ya.make @@ -7,7 +7,8 @@ PEERDIR( SRCS( DiskS3.cpp registerDiskS3.cpp - DynamicProxyConfiguration.cpp + ProxyListConfiguration.cpp + ProxyResolverConfiguration.cpp ) END() diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 17f3ed76ed5..53c36ff8924 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -309,6 +309,32 @@ class ClickHouseCluster: container_id = self.get_container_id(instance_name) return self.docker_client.api.logs(container_id) + def exec_in_container(self, container_id, cmd, detach=False, **kwargs): + exec_id = self.docker_client.api.exec_create(container_id, cmd, **kwargs) + output = self.docker_client.api.exec_start(exec_id, detach=detach) + + output = output.decode('utf8') + exit_code = self.docker_client.api.exec_inspect(exec_id)['ExitCode'] + if exit_code: + container_info = self.docker_client.api.inspect_container(container_id) + image_id = container_info.get('Image') + image_info = self.docker_client.api.inspect_image(image_id) + print("Command failed in container {}: ".format(container_id)) + pprint.pprint(container_info) + print("") + print("Container {} uses image {}: ".format(container_id, image_id)) + pprint.pprint(image_info) + print("") + raise Exception('Cmd "{}" failed in container {}. Return code {}. Output: {}'.format(' '.join(cmd), container_id, exit_code, output)) + return output + + def copy_file_to_container(self, container_id, local_path, dest_path): + with open(local_path, 'r') as fdata: + data = fdata.read() + encoded_data = base64.b64encode(data) + self.exec_in_container(container_id, ["bash", "-c", "echo {} | base64 --decode > {}".format(encoded_data, dest_path)], + user='root') + def wait_mysql_to_start(self, timeout=60): start = time.time() while time.time() - start < timeout: @@ -746,24 +772,8 @@ class ClickHouseInstance: assert_eq_with_retry(self, "select 1", "1", retry_count=int(stop_start_wait_sec / 0.5), sleep_time=0.5) def exec_in_container(self, cmd, detach=False, **kwargs): - container = self.get_docker_handle() - exec_id = self.docker_client.api.exec_create(container.id, cmd, **kwargs) - output = self.docker_client.api.exec_start(exec_id, detach=detach) - - output = output.decode('utf8') - exit_code = self.docker_client.api.exec_inspect(exec_id)['ExitCode'] - if exit_code: - container_info = self.docker_client.api.inspect_container(container.id) - image_id = container_info.get('Image') - image_info = self.docker_client.api.inspect_image(image_id) - print("Command failed in container {}: ".format(container.id)) - pprint.pprint(container_info) - print("") - print("Container {} uses image {}: ".format(container.id, image_id)) - pprint.pprint(image_info) - print("") - raise Exception('Cmd "{}" failed in container {}. Return code {}. Output: {}'.format(' '.join(cmd), container.id, exit_code, output)) - return output + container_id = self.get_docker_handle().id + return self.cluster.exec_in_container(container_id, cmd, detach, **kwargs) def contains_in_log(self, substring): result = self.exec_in_container( @@ -771,11 +781,8 @@ class ClickHouseInstance: return len(result) > 0 def copy_file_to_container(self, local_path, dest_path): - with open(local_path, 'r') as fdata: - data = fdata.read() - encoded_data = base64.b64encode(data) - self.exec_in_container(["bash", "-c", "echo {} | base64 --decode > {}".format(encoded_data, dest_path)], - user='root') + container_id = self.get_docker_handle().id + return self.cluster.copy_file_to_container(container_id, local_path, dest_path) def get_process_pid(self, process_name): output = self.exec_in_container(["bash", "-c", diff --git a/tests/integration/test_s3_with_proxy/configs/config.d/storage_conf.xml b/tests/integration/test_s3_with_proxy/configs/config.d/storage_conf.xml index 7827eec4498..2e4d4e6eaf1 100644 --- a/tests/integration/test_s3_with_proxy/configs/config.d/storage_conf.xml +++ b/tests/integration/test_s3_with_proxy/configs/config.d/storage_conf.xml @@ -11,6 +11,19 @@ http://proxy2:8888 + + s3 + http://minio1:9001/root/data/ + minio + minio123 + + + http://resolver:8080 + http + 8888 + + + @@ -20,6 +33,13 @@ + + +
+ s3_with_resolver +
+ + diff --git a/tests/integration/test_s3_with_proxy/proxy-resolver/__init__.py b/tests/integration/test_s3_with_proxy/proxy-resolver/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_s3_with_proxy/proxy-resolver/entrypoint.sh b/tests/integration/test_s3_with_proxy/proxy-resolver/entrypoint.sh new file mode 100644 index 00000000000..e456be666a9 --- /dev/null +++ b/tests/integration/test_s3_with_proxy/proxy-resolver/entrypoint.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +pip install bottle +python resolver.py diff --git a/tests/integration/test_s3_with_proxy/proxy-resolver/resolver.py b/tests/integration/test_s3_with_proxy/proxy-resolver/resolver.py new file mode 100644 index 00000000000..ecafe92cb83 --- /dev/null +++ b/tests/integration/test_s3_with_proxy/proxy-resolver/resolver.py @@ -0,0 +1,13 @@ +import bottle +import random + + +@bottle.route('/') +def index(): + if random.randrange(2) == 0: + return 'proxy1' + else: + return 'proxy2' + + +bottle.run(host='0.0.0.0', port=8080) diff --git a/tests/integration/test_s3_with_proxy/test.py b/tests/integration/test_s3_with_proxy/test.py index 8881d0e4b74..2b616ea2d02 100644 --- a/tests/integration/test_s3_with_proxy/test.py +++ b/tests/integration/test_s3_with_proxy/test.py @@ -1,4 +1,5 @@ import logging +import os import pytest from helpers.cluster import ClickHouseCluster @@ -17,6 +18,17 @@ def prepare_s3_bucket(cluster): minio_client.make_bucket(cluster.minio_bucket) +# Runs simple proxy resolver in python env container. +def run_resolver(cluster): + container_id = cluster.get_container_id('resolver') + current_dir = os.path.dirname(__file__) + cluster.copy_file_to_container(container_id, os.path.join(current_dir, "proxy-resolver", "resolver.py"), + "resolver.py") + cluster.copy_file_to_container(container_id, os.path.join(current_dir, "proxy-resolver", "entrypoint.sh"), + "entrypoint.sh") + cluster.exec_in_container(container_id, ["/bin/bash", "entrypoint.sh"], detach=True) + + @pytest.fixture(scope="module") def cluster(): try: @@ -29,6 +41,9 @@ def cluster(): prepare_s3_bucket(cluster) logging.info("S3 bucket created") + run_resolver(cluster) + logging.info("Proxy resolver started") + yield cluster finally: cluster.shutdown() @@ -41,7 +56,10 @@ def check_proxy_logs(cluster, proxy_instance): assert logs.find(http_method + " http://minio1") >= 0 -def test_s3_with_proxy_list(cluster): +@pytest.mark.parametrize( + "policy", ["s3", "s3_with_resolver"] +) +def test_s3_with_proxy_list(cluster, policy): node = cluster.instances["node"] node.query( @@ -51,8 +69,9 @@ def test_s3_with_proxy_list(cluster): data String ) ENGINE=MergeTree() ORDER BY id - SETTINGS storage_policy='s3' + SETTINGS storage_policy='{}' """ + .format(policy) ) node.query("INSERT INTO s3_test VALUES (0,'data'),(1,'data')") From 8602f7b47ca766de7fea0ee63ff7ab9ae120b832 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Fri, 8 May 2020 13:58:56 +0300 Subject: [PATCH 035/738] Minor updates in proxy resolver parameters naming. --- src/Disks/S3/DiskS3.h | 2 +- src/Disks/S3/registerDiskS3.cpp | 12 ++++++------ .../configs/config.d/storage_conf.xml | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Disks/S3/DiskS3.h b/src/Disks/S3/DiskS3.h index 9930a83fd2b..5fa8e8358a6 100644 --- a/src/Disks/S3/DiskS3.h +++ b/src/Disks/S3/DiskS3.h @@ -1,7 +1,7 @@ #pragma once #include "Disks/DiskFactory.h" -#include "ProxyListConfiguration.h" +#include "ProxyConfiguration.h" #include #include diff --git a/src/Disks/S3/registerDiskS3.cpp b/src/Disks/S3/registerDiskS3.cpp index c4eb877f30e..a1dda6c704f 100644 --- a/src/Disks/S3/registerDiskS3.cpp +++ b/src/Disks/S3/registerDiskS3.cpp @@ -40,15 +40,15 @@ namespace std::shared_ptr getProxyResolverConfiguration(const Poco::Util::AbstractConfiguration * proxy_resolver_config) { auto endpoint = Poco::URI(proxy_resolver_config->getString("endpoint")); - auto scheme = proxy_resolver_config->getString("scheme"); - if (scheme != "http" && scheme != "https") - throw Exception("Only HTTP/HTTPS schemas allowed in proxy resolver config: " + scheme, ErrorCodes::BAD_ARGUMENTS); - auto port = proxy_resolver_config->getUInt("port"); + auto proxy_scheme = proxy_resolver_config->getString("proxy_scheme"); + if (proxy_scheme != "http" && proxy_scheme != "https") + throw Exception("Only HTTP/HTTPS schemas allowed in proxy resolver config: " + proxy_scheme, ErrorCodes::BAD_ARGUMENTS); + auto proxy_port = proxy_resolver_config->getUInt("proxy_port"); LOG_DEBUG( - &Logger::get("DiskS3"), "Configured proxy resolver: " << endpoint.toString() << ", Scheme: " << scheme << ", Port: " << port); + &Logger::get("DiskS3"), "Configured proxy resolver: " << endpoint.toString() << ", Scheme: " << proxy_scheme << ", Port: " << proxy_port); - return std::make_shared(endpoint, scheme, port); + return std::make_shared(endpoint, proxy_scheme, proxy_port); } std::shared_ptr getProxyListConfiguration(const Poco::Util::AbstractConfiguration * proxy_config) diff --git a/tests/integration/test_s3_with_proxy/configs/config.d/storage_conf.xml b/tests/integration/test_s3_with_proxy/configs/config.d/storage_conf.xml index 2e4d4e6eaf1..c791997b6f9 100644 --- a/tests/integration/test_s3_with_proxy/configs/config.d/storage_conf.xml +++ b/tests/integration/test_s3_with_proxy/configs/config.d/storage_conf.xml @@ -19,8 +19,8 @@ http://resolver:8080 - http - 8888 + http + 8888 From 5cc0927874052ec2800542ab5ccb32b5c70908be Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Fri, 8 May 2020 14:07:26 +0300 Subject: [PATCH 036/738] Add comment explaining protocol for proxy resolver configuration. --- .../test_s3_with_proxy/configs/config.d/storage_conf.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/integration/test_s3_with_proxy/configs/config.d/storage_conf.xml b/tests/integration/test_s3_with_proxy/configs/config.d/storage_conf.xml index c791997b6f9..a83c875b134 100644 --- a/tests/integration/test_s3_with_proxy/configs/config.d/storage_conf.xml +++ b/tests/integration/test_s3_with_proxy/configs/config.d/storage_conf.xml @@ -17,6 +17,11 @@ minio minio123 + http://resolver:8080 http From dd5dd5de324419c0f6f80177815e87bdcb75b314 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Fri, 8 May 2020 13:35:50 +0200 Subject: [PATCH 037/738] Updating test_rename_with_parallel_ttl_move to check if parts got moves. Adding renaming of the time column used inside the TTL expression after insert. --- tests/integration/test_rename_column/test.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_rename_column/test.py b/tests/integration/test_rename_column/test.py index b09a937211c..7badbc8a176 100644 --- a/tests/integration/test_rename_column/test.py +++ b/tests/integration/test_rename_column/test.py @@ -55,11 +55,11 @@ def create_table(nodes, table_name, with_storage_policy=False, with_time_column= """ if with_ttl_move: sql += """ - TTL time TO DISK 'external' + TTL time + INTERVAL (num2 % 1) SECOND TO DISK 'external' """ if with_ttl_delete: sql += """ - TTL time DELETE + TTL time + INTERVAL (num2 % 1) SECOND DELETE """ settings.append("merge_with_ttl_timeout = 1") @@ -417,6 +417,9 @@ def test_rename_with_parallel_ttl_move(started_cluster): tasks.append(p.apply_async(insert, (node1, table_name, 10000, ["num", "num2"], 1, False, False, True, 0, True))) time.sleep(5) + rename_column(node1, table_name, "time", "time2", 1, False) + #rename_column(node1, table_name, "time2", "time", 1, False) + time.sleep(4) tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2", 500, True))) tasks.append(p.apply_async(rename_column, (node2, table_name, "foo2", "foo3", 500, True))) tasks.append(p.apply_async(rename_column, (node3, table_name, "num3", "num2", 500, True))) @@ -424,6 +427,9 @@ def test_rename_with_parallel_ttl_move(started_cluster): for task in tasks: task.get(timeout=240) + # check some parts got moved + assert "external" in set(node1.query("SELECT disk_name FROM system.parts WHERE table == '{}' AND active=1 ORDER BY modification_time".format(table_name)).strip().splitlines()) + # rename column back to original rename_column(node1, table_name, "foo2", "num2", 1, True) rename_column(node1, table_name, "foo3", "num2", 1, True) From 089371ddfb596a7b052ac709a5c2778e76f349c0 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Fri, 8 May 2020 14:48:24 +0200 Subject: [PATCH 038/738] Reverting test so that it can pass. --- tests/integration/test_rename_column/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_rename_column/test.py b/tests/integration/test_rename_column/test.py index 7badbc8a176..0086050cbed 100644 --- a/tests/integration/test_rename_column/test.py +++ b/tests/integration/test_rename_column/test.py @@ -418,7 +418,7 @@ def test_rename_with_parallel_ttl_move(started_cluster): tasks.append(p.apply_async(insert, (node1, table_name, 10000, ["num", "num2"], 1, False, False, True, 0, True))) time.sleep(5) rename_column(node1, table_name, "time", "time2", 1, False) - #rename_column(node1, table_name, "time2", "time", 1, False) + rename_column(node1, table_name, "time2", "time", 1, False) time.sleep(4) tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2", 500, True))) tasks.append(p.apply_async(rename_column, (node2, table_name, "foo2", "foo3", 500, True))) From 8d8dc3035cca5588f92ee44e7a0c1d13face0fb0 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Fri, 8 May 2020 16:10:21 +0300 Subject: [PATCH 039/738] Fix naming. --- src/Disks/S3/registerDiskS3.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Disks/S3/registerDiskS3.cpp b/src/Disks/S3/registerDiskS3.cpp index a1dda6c704f..36966d1f34b 100644 --- a/src/Disks/S3/registerDiskS3.cpp +++ b/src/Disks/S3/registerDiskS3.cpp @@ -85,12 +85,12 @@ namespace const auto * proxy_config = config->createView("proxy"); - std::vector configKeys; - proxy_config->keys(configKeys); + std::vector config_keys; + proxy_config->keys(config_keys); - if (auto resolverConfigs = std::count(configKeys.begin(), configKeys.end(), "resolver")) + if (auto resolver_configs = std::count(config_keys.begin(), config_keys.end(), "resolver")) { - if (resolverConfigs > 1) + if (resolver_configs > 1) throw Exception("Multiple proxy resolver configurations aren't allowed", ErrorCodes::BAD_ARGUMENTS); return getProxyResolverConfiguration(proxy_config->createView("resolver")); From e365bf422e17751d02e42eb9fad5f763e6fa488c Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 8 May 2020 17:48:10 +0300 Subject: [PATCH 040/738] Delay exception for PushingToViewsBlockOutputStream mvs. --- .../PushingToViewsBlockOutputStream.cpp | 24 ++++++++++++++++++- .../PushingToViewsBlockOutputStream.h | 1 + 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/DataStreams/PushingToViewsBlockOutputStream.cpp b/src/DataStreams/PushingToViewsBlockOutputStream.cpp index a79cc61bd2d..c5a552eed90 100644 --- a/src/DataStreams/PushingToViewsBlockOutputStream.cpp +++ b/src/DataStreams/PushingToViewsBlockOutputStream.cpp @@ -162,7 +162,12 @@ void PushingToViewsBlockOutputStream::write(const Block & block) { // Process sequentially for (size_t view_num = 0; view_num < views.size(); ++view_num) + { process(block, view_num); + + if (views[view_num].exception) + std::rethrow_exception(views[view_num].exception); + } } } @@ -190,8 +195,18 @@ void PushingToViewsBlockOutputStream::writeSuffix() if (output) output->writeSuffix(); + std::exception_ptr first_exception; + for (auto & view : views) { + if (view.exception) + { + if (!first_exception) + first_exception = view.exception; + + continue; + } + try { view.out->writeSuffix(); @@ -202,6 +217,9 @@ void PushingToViewsBlockOutputStream::writeSuffix() throw; } } + + if (first_exception) + std::rethrow_exception(first_exception); } void PushingToViewsBlockOutputStream::flush() @@ -270,7 +288,11 @@ void PushingToViewsBlockOutputStream::process(const Block & block, size_t view_n catch (Exception & ex) { ex.addMessage("while pushing to view " + view.table_id.getNameForLogs()); - throw; + view.exception = std::current_exception(); + } + catch (...) + { + view.exception = std::current_exception(); } } diff --git a/src/DataStreams/PushingToViewsBlockOutputStream.h b/src/DataStreams/PushingToViewsBlockOutputStream.h index 162c2e1b447..a2a1ca5caf5 100644 --- a/src/DataStreams/PushingToViewsBlockOutputStream.h +++ b/src/DataStreams/PushingToViewsBlockOutputStream.h @@ -40,6 +40,7 @@ private: ASTPtr query; StorageID table_id; BlockOutputStreamPtr out; + std::exception_ptr exception; }; std::vector views; From 869c933972dda6c2bea1e5e539b7e5d06f45b7e1 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 8 May 2020 19:15:49 +0300 Subject: [PATCH 041/738] Added test. --- .../0_stateless/01275_parallel_mv.reference | 4 ++++ .../queries/0_stateless/01275_parallel_mv.sql | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 tests/queries/0_stateless/01275_parallel_mv.reference create mode 100644 tests/queries/0_stateless/01275_parallel_mv.sql diff --git a/tests/queries/0_stateless/01275_parallel_mv.reference b/tests/queries/0_stateless/01275_parallel_mv.reference new file mode 100644 index 00000000000..898d3f7266e --- /dev/null +++ b/tests/queries/0_stateless/01275_parallel_mv.reference @@ -0,0 +1,4 @@ +10 +10 +0 +10 diff --git a/tests/queries/0_stateless/01275_parallel_mv.sql b/tests/queries/0_stateless/01275_parallel_mv.sql new file mode 100644 index 00000000000..b67fbf02f8d --- /dev/null +++ b/tests/queries/0_stateless/01275_parallel_mv.sql @@ -0,0 +1,18 @@ +drop table if exists testX; +drop table if exists testXA; +drop table if exists testXB; +drop table if exists testXC; + +create table testX (A Int64) engine=MergeTree order by tuple(); + +create materialized view testXA engine=MergeTree order by tuple() as select sleep(1) from testX; +create materialized view testXB engine=MergeTree order by tuple() as select sleep(2), throwIf(A=1) from testX; +create materialized view testXC engine=MergeTree order by tuple() as select sleep(1) from testX; + +set parallel_view_processing=1; +insert into testX select number from numbers(10); -- {serverError 395} + +select count() from testX; +select count() from testXA; +select count() from testXB; +select count() from testXC; From b54209e43efe7be30bb2a6c41a46057ecdd39d4b Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sat, 9 May 2020 19:03:22 +0300 Subject: [PATCH 042/738] Modified reservation to work with multiple disks --- src/Disks/DiskLocal.cpp | 14 +++++++++++++- src/Disks/IDisk.h | 9 ++++++--- src/Disks/S3/DiskS3.cpp | 12 +++++++++++- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/src/Disks/DiskLocal.cpp b/src/Disks/DiskLocal.cpp index a1c2641e2f3..9c451c87b8f 100644 --- a/src/Disks/DiskLocal.cpp +++ b/src/Disks/DiskLocal.cpp @@ -17,6 +17,7 @@ namespace ErrorCodes extern const int UNKNOWN_ELEMENT_IN_CONFIG; extern const int EXCESSIVE_ELEMENT_IN_CONFIG; extern const int PATH_ACCESS_DENIED; + extern const int INCORRECT_DISK_INDEX; } std::mutex DiskLocal::reservation_mutex; @@ -34,7 +35,9 @@ public: UInt64 getSize() const override { return size; } - DiskPtr getDisk() const override { return disk; } + DiskPtr getDisk(size_t i = 0) const override; + + Disks getDisks() const override { return {disk}; } void update(UInt64 new_size) override; @@ -282,6 +285,15 @@ void DiskLocal::copy(const String & from_path, const std::shared_ptr & to IDisk::copy(from_path, to_disk, to_path); /// Copy files through buffers. } +DiskPtr DiskLocalReservation::getDisk(size_t i) const +{ + if (i != 0) + { + throw Exception("Can't use i != 0 with single disk reservation", ErrorCodes::INCORRECT_DISK_INDEX); + } + return disk; +} + void DiskLocalReservation::update(UInt64 new_size) { std::lock_guard lock(DiskLocal::reservation_mutex); diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index 7d0b429720e..b89128e7ded 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -24,7 +24,7 @@ class IDiskDirectoryIterator; using DiskDirectoryIteratorPtr = std::unique_ptr; class IReservation; -using ReservationPtr = std::unique_ptr; +using ReservationPtr = std::shared_ptr; class ReadBufferFromFileBase; class WriteBufferFromFileBase; @@ -206,8 +206,11 @@ public: /// Get reservation size. virtual UInt64 getSize() const = 0; - /// Get disk where reservation take place. - virtual DiskPtr getDisk() const = 0; + /// Get i-th disk where reservation take place. + virtual DiskPtr getDisk(size_t i = 0) const = 0; + + /// Get all disks, used in reservation + virtual Disks getDisks() const = 0; /// Changes amount of reserved space. virtual void update(UInt64 new_size) = 0; diff --git a/src/Disks/S3/DiskS3.cpp b/src/Disks/S3/DiskS3.cpp index 47ca8231001..d53c9f172e2 100644 --- a/src/Disks/S3/DiskS3.cpp +++ b/src/Disks/S3/DiskS3.cpp @@ -28,6 +28,7 @@ namespace ErrorCodes extern const int FILE_ALREADY_EXISTS; extern const int CANNOT_SEEK_THROUGH_FILE; extern const int UNKNOWN_FORMAT; + extern const int INCORRECT_DISK_INDEX; } namespace @@ -369,7 +370,16 @@ public: UInt64 getSize() const override { return size; } - DiskPtr getDisk() const override { return disk; } + DiskPtr getDisk(size_t i = 0) const override + { + if (i != 0) + { + throw Exception("Can't use i != 0 with single disk reservation", ErrorCodes::INCORRECT_DISK_INDEX); + } + return disk; + } + + Disks getDisks() const override { return {disk}; } void update(UInt64 new_size) override { From 390b39b2726a97b443408f5aed69a15678ec4dc4 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 10 May 2020 00:24:15 +0300 Subject: [PATCH 043/738] VolumePtr instead of DiskPtr in MergeTreeData* --- programs/server/Server.cpp | 2 +- src/Common/ErrorCodes.cpp | 1 + src/Disks/DiskLocal.cpp | 1 + src/Disks/DiskSelector.cpp | 6 +- src/Disks/DiskSelector.h | 4 + src/Disks/IVolume.h | 22 +++- src/Disks/SingleDiskVolume.cpp | 6 + src/Disks/SingleDiskVolume.h | 26 ++++ src/Disks/StoragePolicy.cpp | 20 +-- src/Disks/StoragePolicy.h | 1 + src/Disks/VolumeJBOD.h | 2 + src/Disks/createVolume.cpp | 17 +++ src/Disks/createVolume.h | 12 ++ src/Interpreters/Context.cpp | 2 +- src/Storages/MergeTree/DataPartsExchange.cpp | 6 +- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 114 +++++++++--------- src/Storages/MergeTree/IMergeTreeDataPart.h | 11 +- .../MergeTree/IMergeTreeDataPartWriter.cpp | 12 +- .../MergeTree/IMergeTreeDataPartWriter.h | 4 +- .../MergeTree/IMergedBlockOutputStream.cpp | 4 +- src/Storages/MergeTree/MergeTreeData.cpp | 57 ++++----- src/Storages/MergeTree/MergeTreeData.h | 8 +- .../MergeTree/MergeTreeDataMergerMutator.cpp | 12 +- .../MergeTree/MergeTreeDataPartCompact.cpp | 34 +++--- .../MergeTree/MergeTreeDataPartCompact.h | 4 +- .../MergeTree/MergeTreeDataPartWide.cpp | 32 ++--- .../MergeTree/MergeTreeDataPartWide.h | 4 +- .../MergeTreeDataPartWriterCompact.cpp | 6 +- .../MergeTreeDataPartWriterCompact.h | 2 +- .../MergeTree/MergeTreeDataPartWriterWide.cpp | 6 +- .../MergeTree/MergeTreeDataPartWriterWide.h | 2 +- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 2 +- .../MergeTree/MergeTreeDataWriter.cpp | 12 +- .../MergeTree/MergeTreeIndexReader.cpp | 2 +- .../MergeTree/MergeTreePartsMover.cpp | 7 +- .../MergeTree/MergeTreeReaderCompact.cpp | 8 +- .../MergeTree/MergeTreeReaderWide.cpp | 2 +- src/Storages/MergeTree/checkDataPart.cpp | 2 +- src/Storages/StorageDistributed.cpp | 6 +- src/Storages/StorageMergeTree.cpp | 6 +- src/Storages/StorageReplicatedMergeTree.cpp | 4 +- src/Storages/System/StorageSystemParts.cpp | 2 +- .../System/StorageSystemPartsColumns.cpp | 2 +- .../System/StorageSystemStoragePolicies.cpp | 4 +- 44 files changed, 297 insertions(+), 202 deletions(-) create mode 100644 src/Disks/SingleDiskVolume.cpp create mode 100644 src/Disks/SingleDiskVolume.h create mode 100644 src/Disks/createVolume.cpp create mode 100644 src/Disks/createVolume.h diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 161916a6103..f306b6ad889 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -372,7 +372,7 @@ int Server::main(const std::vector & /*args*/) std::string tmp_path = config().getString("tmp_path", path + "tmp/"); std::string tmp_policy = config().getString("tmp_policy", ""); const VolumePtr & volume = global_context->setTemporaryStorage(tmp_path, tmp_policy); - for (const DiskPtr & disk : volume->disks) + for (const DiskPtr & disk : volume->getDisks()) setupTmpPath(log, disk->getPath()); } diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 120c9b93a78..dbb539ef2b3 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -493,6 +493,7 @@ namespace ErrorCodes extern const int NO_REMOTE_SHARD_AVAILABLE = 519; extern const int CANNOT_DETACH_DICTIONARY_AS_TABLE = 520; extern const int ATOMIC_RENAME_FAIL = 521; + extern const int INCORRECT_DISK_INDEX = 522; extern const int KEEPER_EXCEPTION = 999; extern const int POCO_EXCEPTION = 1000; diff --git a/src/Disks/DiskLocal.cpp b/src/Disks/DiskLocal.cpp index 9c451c87b8f..a5eee3eada3 100644 --- a/src/Disks/DiskLocal.cpp +++ b/src/Disks/DiskLocal.cpp @@ -12,6 +12,7 @@ namespace DB { + namespace ErrorCodes { extern const int UNKNOWN_ELEMENT_IN_CONFIG; diff --git a/src/Disks/DiskSelector.cpp b/src/Disks/DiskSelector.cpp index b464d8652d6..0ae8763eef3 100644 --- a/src/Disks/DiskSelector.cpp +++ b/src/Disks/DiskSelector.cpp @@ -55,7 +55,7 @@ DiskSelectorPtr DiskSelector::updateFromConfig( constexpr auto default_disk_name = "default"; std::set old_disks_minus_new_disks; - for (const auto & [disk_name, _] : result->disks) + for (const auto & [disk_name, _] : result->getDisksMap()) { old_disks_minus_new_disks.insert(disk_name); } @@ -65,10 +65,10 @@ DiskSelectorPtr DiskSelector::updateFromConfig( if (!std::all_of(disk_name.begin(), disk_name.end(), isWordCharASCII)) throw Exception("Disk name can contain only alphanumeric and '_' (" + disk_name + ")", ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG); - if (result->disks.count(disk_name) == 0) + if (result->getDisksMap().count(disk_name) == 0) { auto disk_config_prefix = config_prefix + "." + disk_name; - result->disks.emplace(disk_name, factory.create(disk_name, config, disk_config_prefix, context)); + result->addToDiskMap(disk_name, factory.create(disk_name, config, disk_config_prefix, context)); } else { diff --git a/src/Disks/DiskSelector.h b/src/Disks/DiskSelector.h index 430ba97c003..85b67f55d0a 100644 --- a/src/Disks/DiskSelector.h +++ b/src/Disks/DiskSelector.h @@ -29,6 +29,10 @@ public: /// Get all disks with names const auto & getDisksMap() const { return disks; } + void addToDiskMap(String name, DiskPtr disk) + { + disks.emplace(name, disk); + } private: std::map disks; diff --git a/src/Disks/IVolume.h b/src/Disks/IVolume.h index 504dded7696..8365102196d 100644 --- a/src/Disks/IVolume.h +++ b/src/Disks/IVolume.h @@ -8,6 +8,17 @@ namespace DB { +enum VolumeType +{ + JBOD, + SINGLE_DISK, + UNKNOWN +}; + +class IVolume; +using VolumePtr = std::shared_ptr; +using Volumes = std::vector; + /** * Disks group by some (user) criteria. For example, * - VolumeJBOD("slow_disks", [d1, d2], 100) @@ -22,7 +33,7 @@ namespace DB class IVolume : public Space { public: - IVolume(String name_, Disks disks_): disks(std::move(disks_)), name(std::move(name_)) + IVolume(const String & name_, Disks disks_): disks(std::move(disks_)), name(name_) { } @@ -37,16 +48,17 @@ public: /// Volume name from config const String & getName() const override { return name; } + virtual VolumeType getType() const = 0; /// Return biggest unreserved space across all disks UInt64 getMaxUnreservedFreeSpace() const; - Disks disks; + DiskPtr getDisk(size_t i = 0) const { return disks[i]; } + const Disks & getDisks() const { return disks; } + protected: + Disks disks; const String name; }; -using VolumePtr = std::shared_ptr; -using Volumes = std::vector; - } diff --git a/src/Disks/SingleDiskVolume.cpp b/src/Disks/SingleDiskVolume.cpp new file mode 100644 index 00000000000..47140407026 --- /dev/null +++ b/src/Disks/SingleDiskVolume.cpp @@ -0,0 +1,6 @@ +#include + +namespace DB +{ + +} diff --git a/src/Disks/SingleDiskVolume.h b/src/Disks/SingleDiskVolume.h new file mode 100644 index 00000000000..c7a6776a7d2 --- /dev/null +++ b/src/Disks/SingleDiskVolume.h @@ -0,0 +1,26 @@ +#pragma once + +#include + +namespace DB +{ + +class SingleDiskVolume : public IVolume +{ +public: + SingleDiskVolume(const String & name_, DiskPtr disk): IVolume(name_, {disk}) + { + } + + ReservationPtr reserve(UInt64 bytes) override + { + return disks[0]->reserve(bytes); + } + + VolumeType getType() const override { return VolumeType::SINGLE_DISK; } + +}; + +using VolumeSingleDiskPtr = std::shared_ptr; + +} diff --git a/src/Disks/StoragePolicy.cpp b/src/Disks/StoragePolicy.cpp index 8518e1516db..e1ae718d9b9 100644 --- a/src/Disks/StoragePolicy.cpp +++ b/src/Disks/StoragePolicy.cpp @@ -55,7 +55,7 @@ StoragePolicy::StoragePolicy( std::set disk_names; for (const auto & volume : volumes) { - for (const auto & disk : volume->disks) + for (const auto & disk : volume->getDisks()) { if (disk_names.find(disk->getName()) != disk_names.end()) throw Exception( @@ -102,7 +102,7 @@ bool StoragePolicy::isDefaultPolicy() const if (volumes[0]->getName() != "default") return false; - const auto & disks = volumes[0]->disks; + const auto & disks = volumes[0]->getDisks(); if (disks.size() != 1) return false; @@ -117,7 +117,7 @@ Disks StoragePolicy::getDisks() const { Disks res; for (const auto & volume : volumes) - for (const auto & disk : volume->disks) + for (const auto & disk : volume->getDisks()) res.push_back(disk); return res; } @@ -130,17 +130,17 @@ DiskPtr StoragePolicy::getAnyDisk() const if (volumes.empty()) throw Exception("StoragePolicy has no volumes. It's a bug.", ErrorCodes::LOGICAL_ERROR); - if (volumes[0]->disks.empty()) + if (volumes[0]->getDisks().empty()) throw Exception("Volume '" + volumes[0]->getName() + "' has no disks. It's a bug.", ErrorCodes::LOGICAL_ERROR); - return volumes[0]->disks[0]; + return volumes[0]->getDisks()[0]; } DiskPtr StoragePolicy::getDiskByName(const String & disk_name) const { for (auto && volume : volumes) - for (auto && disk : volume->disks) + for (auto && disk : volume->getDisks()) if (disk->getName() == disk_name) return disk; return {}; @@ -181,7 +181,7 @@ ReservationPtr StoragePolicy::makeEmptyReservationOnLargestDisk() const DiskPtr max_disk; for (const auto & volume : volumes) { - for (const auto & disk : volume->disks) + for (const auto & disk : volume->getDisks()) { auto avail_space = disk->getAvailableSpace(); if (avail_space > max_space) @@ -207,10 +207,10 @@ void StoragePolicy::checkCompatibleWith(const StoragePolicyPtr & new_storage_pol throw Exception("New storage policy shall contain volumes of old one", ErrorCodes::LOGICAL_ERROR); std::unordered_set new_disk_names; - for (const auto & disk : new_storage_policy->getVolumeByName(volume->getName())->disks) + for (const auto & disk : new_storage_policy->getVolumeByName(volume->getName())->getDisks()) new_disk_names.insert(disk->getName()); - for (const auto & disk : volume->disks) + for (const auto & disk : volume->getDisks()) if (new_disk_names.count(disk->getName()) == 0) throw Exception("New storage policy shall contain disks of old one", ErrorCodes::LOGICAL_ERROR); } @@ -222,7 +222,7 @@ size_t StoragePolicy::getVolumeIndexByDisk(const DiskPtr & disk_ptr) const for (size_t i = 0; i < volumes.size(); ++i) { const auto & volume = volumes[i]; - for (const auto & disk : volume->disks) + for (const auto & disk : volume->getDisks()) if (disk->getName() == disk_ptr->getName()) return i; } diff --git a/src/Disks/StoragePolicy.h b/src/Disks/StoragePolicy.h index a41a62f1223..00ac2c2c9bb 100644 --- a/src/Disks/StoragePolicy.h +++ b/src/Disks/StoragePolicy.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Disks/VolumeJBOD.h b/src/Disks/VolumeJBOD.h index 7399d3cf065..16a96ec9fb9 100644 --- a/src/Disks/VolumeJBOD.h +++ b/src/Disks/VolumeJBOD.h @@ -25,6 +25,8 @@ public: DiskSelectorPtr disk_selector ); + VolumeType getType() const override { return VolumeType::JBOD; } + /// Next disk (round-robin) /// /// - Used with policy for temporary data diff --git a/src/Disks/createVolume.cpp b/src/Disks/createVolume.cpp new file mode 100644 index 00000000000..3faca365bd4 --- /dev/null +++ b/src/Disks/createVolume.cpp @@ -0,0 +1,17 @@ +#include "createVolume.h" + +namespace DB +{ + +VolumePtr createVolumeFromReservation(ReservationPtr reservation, VolumePtr other_volume) +{ + if (other_volume->getType() == VolumeType::JBOD || other_volume->getType() == VolumeType::SINGLE_DISK) + { + // Since reservation on JBOD chosies one of disks and makes reservation there, volume + // for such type of reservation will be with one disk. + return std::make_shared(other_volume->getName(), reservation->getDisk()); + } + return nullptr; +} + +} diff --git a/src/Disks/createVolume.h b/src/Disks/createVolume.h new file mode 100644 index 00000000000..9180d23ea73 --- /dev/null +++ b/src/Disks/createVolume.h @@ -0,0 +1,12 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ + +VolumePtr createVolumeFromReservation(ReservationPtr reservation, VolumePtr other_volume); + +} diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index f5651a1ab77..c3cce73b8fc 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -584,7 +584,7 @@ VolumeJBODPtr Context::setTemporaryStorage(const String & path, const String & p shared->tmp_volume = tmp_policy->getVolume(0); } - if (shared->tmp_volume->disks.empty()) + if (shared->tmp_volume->getDisks().empty()) throw Exception("No disks volume for temporary files", ErrorCodes::NO_ELEMENTS_IN_CONFIG); return shared->tmp_volume; diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index 70c06efff5c..2b491f9ede6 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -115,7 +116,7 @@ void Service::processQuery(const Poco::Net::HTMLForm & params, ReadBuffer & /*bo { String file_name = it.first; - auto disk = part->disk; + auto disk = part->volume->getDisk(); String path = part->getFullRelativePath() + file_name; UInt64 size = disk->getFileSize(path); @@ -316,7 +317,8 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPart( assertEOF(in); - MergeTreeData::MutableDataPartPtr new_data_part = data.createPart(part_name, reservation->getDisk(), part_relative_path); + auto volume = std::make_shared("volume_" + part_name, disk); + MergeTreeData::MutableDataPartPtr new_data_part = data.createPart(part_name, volume, part_relative_path); new_data_part->is_temp = true; new_data_part->modification_time = time(nullptr); new_data_part->loadColumnsChecksumsIndexes(true, false); diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index a5a7f1f4494..a05db49967b 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -137,11 +137,11 @@ void IMergeTreeDataPart::MinMaxIndex::merge(const MinMaxIndex & other) IMergeTreeDataPart::IMergeTreeDataPart( - MergeTreeData & storage_, const String & name_, const DiskPtr & disk_, const std::optional & relative_path_, Type part_type_) + MergeTreeData & storage_, const String & name_, const VolumePtr & volume_, const std::optional & relative_path_, Type part_type_) : storage(storage_) , name(name_) , info(MergeTreePartInfo::fromPartName(name_, storage.format_version)) - , disk(disk_) + , volume(volume_) , relative_path(relative_path_.value_or(name_)) , index_granularity_info(storage_, part_type_) , part_type(part_type_) @@ -152,13 +152,13 @@ IMergeTreeDataPart::IMergeTreeDataPart( const MergeTreeData & storage_, const String & name_, const MergeTreePartInfo & info_, - const DiskPtr & disk_, + const VolumePtr & volume_, const std::optional & relative_path_, Type part_type_) : storage(storage_) , name(name_) , info(info_) - , disk(disk_) + , volume(volume_) , relative_path(relative_path_.value_or(name_)) , index_granularity_info(storage_, part_type_) , part_type(part_type_) @@ -245,7 +245,7 @@ void IMergeTreeDataPart::removeIfNeeded() { auto path = getFullRelativePath(); - if (!disk->exists(path)) + if (!volume->getDisk()->exists(path)) return; if (is_temp) @@ -385,7 +385,7 @@ String IMergeTreeDataPart::getFullPath() const if (relative_path.empty()) throw Exception("Part relative_path cannot be empty. It's bug.", ErrorCodes::LOGICAL_ERROR); - return storage.getFullPathOnDisk(disk) + relative_path + "/"; + return storage.getFullPathOnDisk(volume->getDisk()) + relative_path + "/"; } String IMergeTreeDataPart::getFullRelativePath() const @@ -445,7 +445,7 @@ void IMergeTreeDataPart::loadIndex() } String index_path = getFullRelativePath() + "primary.idx"; - auto index_file = openForReading(disk, index_path); + auto index_file = openForReading(volume->getDisk(), index_path); for (size_t i = 0; i < index_granularity.getMarksCount(); ++i) //-V756 for (size_t j = 0; j < key_size; ++j) @@ -461,7 +461,7 @@ void IMergeTreeDataPart::loadIndex() } if (!index_file->eof()) - throw Exception("Index file " + fullPath(disk, index_path) + " is unexpectedly long", ErrorCodes::EXPECTED_END_OF_FILE); + throw Exception("Index file " + fullPath(volume->getDisk(), index_path) + " is unexpectedly long", ErrorCodes::EXPECTED_END_OF_FILE); index.assign(std::make_move_iterator(loaded_index.begin()), std::make_move_iterator(loaded_index.end())); } @@ -482,9 +482,9 @@ void IMergeTreeDataPart::loadPartitionAndMinMaxIndex() else { String path = getFullRelativePath(); - partition.load(storage, disk, path); + partition.load(storage, volume->getDisk(), path); if (!isEmpty()) - minmax_idx.load(storage, disk, path); + minmax_idx.load(storage, volume->getDisk(), path); } String calculated_partition_id = partition.getID(storage.partition_key_sample); @@ -498,23 +498,23 @@ void IMergeTreeDataPart::loadPartitionAndMinMaxIndex() void IMergeTreeDataPart::loadChecksums(bool require) { String path = getFullRelativePath() + "checksums.txt"; - if (disk->exists(path)) + if (volume->getDisk()->exists(path)) { - auto buf = openForReading(disk, path); + auto buf = openForReading(volume->getDisk(), path); if (checksums.read(*buf)) { assertEOF(*buf); bytes_on_disk = checksums.getTotalSizeOnDisk(); } else - bytes_on_disk = calculateTotalSizeOnDisk(disk, getFullRelativePath()); + bytes_on_disk = calculateTotalSizeOnDisk(volume->getDisk(), getFullRelativePath()); } else { if (require) throw Exception("No checksums.txt in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); - bytes_on_disk = calculateTotalSizeOnDisk(disk, getFullRelativePath()); + bytes_on_disk = calculateTotalSizeOnDisk(volume->getDisk(), getFullRelativePath()); } } @@ -527,10 +527,10 @@ void IMergeTreeDataPart::loadRowsCount() } else if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING || part_type == Type::COMPACT) { - if (!disk->exists(path)) + if (!volume->getDisk()->exists(path)) throw Exception("No count.txt in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); - auto buf = openForReading(disk, path); + auto buf = openForReading(volume->getDisk(), path); readIntText(rows_count, *buf); assertEOF(*buf); } @@ -575,9 +575,9 @@ void IMergeTreeDataPart::loadRowsCount() void IMergeTreeDataPart::loadTTLInfos() { String path = getFullRelativePath() + "ttl.txt"; - if (disk->exists(path)) + if (volume->getDisk()->exists(path)) { - auto in = openForReading(disk, path); + auto in = openForReading(volume->getDisk(), path); assertString("ttl format version: ", *in); size_t format_version; readText(format_version, *in); @@ -602,7 +602,7 @@ void IMergeTreeDataPart::loadTTLInfos() void IMergeTreeDataPart::loadColumns(bool require) { String path = getFullRelativePath() + "columns.txt"; - if (!disk->exists(path)) + if (!volume->getDisk()->exists(path)) { /// We can get list of columns only from columns.txt in compact parts. if (require || part_type == Type::COMPACT) @@ -610,21 +610,21 @@ void IMergeTreeDataPart::loadColumns(bool require) /// If there is no file with a list of columns, write it down. for (const NameAndTypePair & column : storage.getColumns().getAllPhysical()) - if (disk->exists(getFullRelativePath() + getFileNameForColumn(column) + ".bin")) + if (volume->getDisk()->exists(getFullRelativePath() + getFileNameForColumn(column) + ".bin")) columns.push_back(column); if (columns.empty()) throw Exception("No columns in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); { - auto buf = disk->writeFile(path + ".tmp", 4096); + auto buf = volume->getDisk()->writeFile(path + ".tmp", 4096); columns.writeText(*buf); } - disk->moveFile(path + ".tmp", path); + volume->getDisk()->moveFile(path + ".tmp", path); } else { - columns.readText(*disk->readFile(path)); + columns.readText(*volume->getDisk()->readFile(path)); } size_t pos = 0; @@ -652,29 +652,29 @@ void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_ String from = getFullRelativePath(); String to = storage.relative_data_path + new_relative_path + "/"; - if (!disk->exists(from)) - throw Exception("Part directory " + fullPath(disk, from) + " doesn't exist. Most likely it is logical error.", ErrorCodes::FILE_DOESNT_EXIST); + if (!volume->getDisk()->exists(from)) + throw Exception("Part directory " + fullPath(volume->getDisk(), from) + " doesn't exist. Most likely it is logical error.", ErrorCodes::FILE_DOESNT_EXIST); - if (disk->exists(to)) + if (volume->getDisk()->exists(to)) { if (remove_new_dir_if_exists) { Names files; - disk->listFiles(to, files); + volume->getDisk()->listFiles(to, files); - LOG_WARNING(storage.log, "Part directory " << fullPath(disk, to) << " already exists" + LOG_WARNING(storage.log, "Part directory " << fullPath(volume->getDisk(), to) << " already exists" << " and contains " << files.size() << " files. Removing it."); - disk->removeRecursive(to); + volume->getDisk()->removeRecursive(to); } else { - throw Exception("Part directory " + fullPath(disk, to) + " already exists", ErrorCodes::DIRECTORY_ALREADY_EXISTS); + throw Exception("Part directory " + fullPath(volume->getDisk(), to) + " already exists", ErrorCodes::DIRECTORY_ALREADY_EXISTS); } } - disk->setLastModified(from, Poco::Timestamp::fromEpochTime(time(nullptr))); - disk->moveFile(from, to); + volume->getDisk()->setLastModified(from, Poco::Timestamp::fromEpochTime(time(nullptr))); + volume->getDisk()->moveFile(from, to); relative_path = new_relative_path; } @@ -703,29 +703,29 @@ void IMergeTreeDataPart::remove() const String to = storage.relative_data_path + "delete_tmp_" + name; // TODO directory delete_tmp_ is never removed if server crashes before returning from this function - if (disk->exists(to)) + if (volume->getDisk()->exists(to)) { - LOG_WARNING(storage.log, "Directory " << fullPath(disk, to) << " (to which part must be renamed before removing) already exists." + LOG_WARNING(storage.log, "Directory " << fullPath(volume->getDisk(), to) << " (to which part must be renamed before removing) already exists." " Most likely this is due to unclean restart. Removing it."); try { - disk->removeRecursive(to + "/"); + volume->getDisk()->removeRecursive(to + "/"); } catch (...) { - LOG_ERROR(storage.log, "Cannot recursively remove directory " << fullPath(disk, to) << ". Exception: " << getCurrentExceptionMessage(false)); + LOG_ERROR(storage.log, "Cannot recursively remove directory " << fullPath(volume->getDisk(), to) << ". Exception: " << getCurrentExceptionMessage(false)); throw; } } try { - disk->moveFile(from, to); + volume->getDisk()->moveFile(from, to); } catch (const Poco::FileNotFoundException &) { - LOG_ERROR(storage.log, "Directory " << fullPath(disk, to) << " (part to remove) doesn't exist or one of nested files has gone." + LOG_ERROR(storage.log, "Directory " << fullPath(volume->getDisk(), to) << " (part to remove) doesn't exist or one of nested files has gone." " Most likely this is due to manual removing. This should be discouraged. Ignoring."); return; @@ -734,7 +734,7 @@ void IMergeTreeDataPart::remove() const if (checksums.empty()) { /// If the part is not completely written, we cannot use fast path by listing files. - disk->removeRecursive(to + "/"); + volume->getDisk()->removeRecursive(to + "/"); } else { @@ -747,25 +747,25 @@ void IMergeTreeDataPart::remove() const # pragma GCC diagnostic ignored "-Wunused-variable" #endif for (const auto & [file, _] : checksums.files) - disk->remove(to + "/" + file); + volume->getDisk()->remove(to + "/" + file); #if !__clang__ # pragma GCC diagnostic pop #endif for (const auto & file : {"checksums.txt", "columns.txt"}) - disk->remove(to + "/" + file); - disk->removeIfExists(to + "/" + DELETE_ON_DESTROY_MARKER_PATH); + volume->getDisk()->remove(to + "/" + file); + volume->getDisk()->removeIfExists(to + "/" + DELETE_ON_DESTROY_MARKER_PATH); - disk->remove(to); + volume->getDisk()->remove(to); } catch (...) { /// Recursive directory removal does many excessive "stat" syscalls under the hood. - LOG_ERROR(storage.log, "Cannot quickly remove directory " << fullPath(disk, to) << " by removing files; fallback to recursive removal. Reason: " + LOG_ERROR(storage.log, "Cannot quickly remove directory " << fullPath(volume->getDisk(), to) << " by removing files; fallback to recursive removal. Reason: " << getCurrentExceptionMessage(false)); - disk->removeRecursive(to + "/"); + volume->getDisk()->removeRecursive(to + "/"); } } } @@ -786,7 +786,7 @@ String IMergeTreeDataPart::getRelativePathForDetachedPart(const String & prefix) { res = "detached/" + (prefix.empty() ? "" : prefix + "_") + name + (try_no ? "_try" + DB::toString(try_no) : ""); - if (!disk->exists(getFullRelativePath() + res)) + if (!volume->getDisk()->exists(getFullRelativePath() + res)) return res; LOG_WARNING(storage.log, "Directory " << res << " (to detach to) already exists." @@ -810,16 +810,16 @@ void IMergeTreeDataPart::makeCloneInDetached(const String & prefix) const String destination_path = storage.relative_data_path + getRelativePathForDetachedPart(prefix); /// Backup is not recursive (max_level is 0), so do not copy inner directories - localBackup(disk, getFullRelativePath(), destination_path, 0); - disk->removeIfExists(destination_path + "/" + DELETE_ON_DESTROY_MARKER_PATH); + localBackup(volume->getDisk(), getFullRelativePath(), destination_path, 0); + volume->getDisk()->removeIfExists(destination_path + "/" + DELETE_ON_DESTROY_MARKER_PATH); } void IMergeTreeDataPart::makeCloneOnDiskDetached(const ReservationPtr & reservation) const { assertOnDisk(); auto reserved_disk = reservation->getDisk(); - if (reserved_disk->getName() == disk->getName()) - throw Exception("Can not clone data part " + name + " to same disk " + disk->getName(), ErrorCodes::LOGICAL_ERROR); + if (reserved_disk->getName() == volume->getDisk()->getName()) + throw Exception("Can not clone data part " + name + " to same disk " + volume->getDisk()->getName(), ErrorCodes::LOGICAL_ERROR); String path_to_clone = storage.relative_data_path + "detached/"; @@ -827,8 +827,8 @@ void IMergeTreeDataPart::makeCloneOnDiskDetached(const ReservationPtr & reservat throw Exception("Path " + fullPath(reserved_disk, path_to_clone + relative_path) + " already exists. Can not clone ", ErrorCodes::DIRECTORY_ALREADY_EXISTS); reserved_disk->createDirectory(path_to_clone); - disk->copy(getFullRelativePath(), reserved_disk, path_to_clone); - disk->removeIfExists(path_to_clone + "/" + DELETE_ON_DESTROY_MARKER_PATH); + volume->getDisk()->copy(getFullRelativePath(), reserved_disk, path_to_clone); + volume->getDisk()->removeIfExists(path_to_clone + "/" + DELETE_ON_DESTROY_MARKER_PATH); } void IMergeTreeDataPart::checkConsistencyBase() const @@ -858,7 +858,7 @@ void IMergeTreeDataPart::checkConsistencyBase() const } } - checksums.checkSizes(disk, path); + checksums.checkSizes(volume->getDisk(), path); } else { @@ -872,17 +872,17 @@ void IMergeTreeDataPart::checkConsistencyBase() const /// Check that the primary key index is not empty. if (!storage.primary_key_columns.empty()) - check_file_not_empty(disk, path + "primary.idx"); + check_file_not_empty(volume->getDisk(), path + "primary.idx"); if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) { - check_file_not_empty(disk, path + "count.txt"); + check_file_not_empty(volume->getDisk(), path + "count.txt"); if (storage.partition_key_expr) - check_file_not_empty(disk, path + "partition.dat"); + check_file_not_empty(volume->getDisk(), path + "partition.dat"); for (const String & col_name : storage.minmax_idx_columns) - check_file_not_empty(disk, path + "minmax_" + escapeForFileName(col_name) + ".idx"); + check_file_not_empty(volume->getDisk(), path + "minmax_" + escapeForFileName(col_name) + ".idx"); } } } diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index 784a3ff047b..a2fe9661832 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -29,7 +29,10 @@ struct ColumnSize; class MergeTreeData; struct FutureMergedMutatedPart; class IReservation; -using ReservationPtr = std::unique_ptr; +using ReservationPtr = std::shared_ptr; + +class IVolume; +using VolumePtr = std::shared_ptr; class IMergeTreeReader; class IMergeTreeDataPartWriter; @@ -60,14 +63,14 @@ public: const MergeTreeData & storage_, const String & name_, const MergeTreePartInfo & info_, - const DiskPtr & disk, + const VolumePtr & volume, const std::optional & relative_path, Type part_type_); IMergeTreeDataPart( MergeTreeData & storage_, const String & name_, - const DiskPtr & disk, + const VolumePtr & volume, const std::optional & relative_path, Type part_type_); @@ -151,7 +154,7 @@ public: String name; MergeTreePartInfo info; - DiskPtr disk; + VolumePtr volume; mutable String relative_path; MergeTreeIndexGranularityInfo index_granularity_info; diff --git a/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp b/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp index 9928583df80..bb19a80a28a 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp @@ -63,7 +63,7 @@ void IMergeTreeDataPartWriter::Stream::addToChecksums(MergeTreeData::DataPart::C IMergeTreeDataPartWriter::IMergeTreeDataPartWriter( - DiskPtr disk_, + const VolumePtr & volume_, const String & part_path_, const MergeTreeData & storage_, const NamesAndTypesList & columns_list_, @@ -72,7 +72,7 @@ IMergeTreeDataPartWriter::IMergeTreeDataPartWriter( const CompressionCodecPtr & default_codec_, const MergeTreeWriterSettings & settings_, const MergeTreeIndexGranularity & index_granularity_) - : disk(std::move(disk_)) + : volume(volume_) , part_path(part_path_) , storage(storage_) , columns_list(columns_list_) @@ -87,8 +87,8 @@ IMergeTreeDataPartWriter::IMergeTreeDataPartWriter( if (settings.blocks_are_granules_size && !index_granularity.empty()) throw Exception("Can't take information about index granularity from blocks, when non empty index_granularity array specified", ErrorCodes::LOGICAL_ERROR); - if (!disk->exists(part_path)) - disk->createDirectories(part_path); + if (!volume->getDisk()->exists(part_path)) + volume->getDisk()->createDirectories(part_path); } IMergeTreeDataPartWriter::~IMergeTreeDataPartWriter() = default; @@ -165,7 +165,7 @@ void IMergeTreeDataPartWriter::initPrimaryIndex() { if (storage.hasPrimaryKey()) { - index_file_stream = disk->writeFile(part_path + "primary.idx", DBMS_DEFAULT_BUFFER_SIZE, WriteMode::Rewrite); + index_file_stream = volume->getDisk()->writeFile(part_path + "primary.idx", DBMS_DEFAULT_BUFFER_SIZE, WriteMode::Rewrite); index_stream = std::make_unique(*index_file_stream); } @@ -180,7 +180,7 @@ void IMergeTreeDataPartWriter::initSkipIndices() skip_indices_streams.emplace_back( std::make_unique( stream_name, - disk, + volume->getDisk(), part_path + stream_name, INDEX_FILE_EXTENSION, part_path + stream_name, marks_file_extension, default_codec, settings.max_compress_block_size, diff --git a/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index 142ad6ca14e..a4460ab861f 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -61,7 +61,7 @@ public: using StreamPtr = std::unique_ptr; IMergeTreeDataPartWriter( - DiskPtr disk, + const VolumePtr & volume, const String & part_path, const MergeTreeData & storage, const NamesAndTypesList & columns_list, @@ -118,7 +118,7 @@ protected: using SerializationState = IDataType::SerializeBinaryBulkStatePtr; using SerializationStates = std::unordered_map; - DiskPtr disk; + VolumePtr volume; String part_path; const MergeTreeData & storage; NamesAndTypesList columns_list; diff --git a/src/Storages/MergeTree/IMergedBlockOutputStream.cpp b/src/Storages/MergeTree/IMergedBlockOutputStream.cpp index c016ec325da..d146f705b0f 100644 --- a/src/Storages/MergeTree/IMergedBlockOutputStream.cpp +++ b/src/Storages/MergeTree/IMergedBlockOutputStream.cpp @@ -9,7 +9,7 @@ namespace DB IMergedBlockOutputStream::IMergedBlockOutputStream( const MergeTreeDataPartPtr & data_part) : storage(data_part->storage) - , disk(data_part->disk) + , disk(data_part->volume->getDisk()) , part_path(data_part->getFullRelativePath()) { } @@ -82,7 +82,7 @@ NameSet IMergedBlockOutputStream::removeEmptyColumnsFromPart( { if (checksums.files.count(removed_file)) { - data_part->disk->remove(data_part->getFullRelativePath() + removed_file); + data_part->volume->getDisk()->remove(data_part->getFullRelativePath() + removed_file); checksums.files.erase(removed_file); } } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index edc492efdbe..76e4bb62bc8 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -928,7 +928,8 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) if (!MergeTreePartInfo::tryParsePartName(part_name, &part_info, format_version)) return; - auto part = createPart(part_name, part_info, part_disk_ptr, part_name); + auto single_disk_volume = std::make_shared("volume_" + part_name, part_disk_ptr); + auto part = createPart(part_name, part_info, single_disk_volume, part_name); bool broken = false; String part_path = relative_data_path + "/" + part_name; @@ -1550,12 +1551,12 @@ MergeTreeDataPartType MergeTreeData::choosePartType(size_t bytes_uncompressed, s MergeTreeData::MutableDataPartPtr MergeTreeData::createPart(const String & name, MergeTreeDataPartType type, const MergeTreePartInfo & part_info, - const DiskPtr & disk, const String & relative_path) const + const VolumePtr & volume, const String & relative_path) const { if (type == MergeTreeDataPartType::COMPACT) - return std::make_shared(*this, name, part_info, disk, relative_path); + return std::make_shared(*this, name, part_info, volume, relative_path); else if (type == MergeTreeDataPartType::WIDE) - return std::make_shared(*this, name, part_info, disk, relative_path); + return std::make_shared(*this, name, part_info, volume, relative_path); else throw Exception("Unknown type in part " + relative_path, ErrorCodes::UNKNOWN_PART_TYPE); } @@ -1573,18 +1574,18 @@ static MergeTreeDataPartType getPartTypeFromMarkExtension(const String & mrk_ext } MergeTreeData::MutableDataPartPtr MergeTreeData::createPart( - const String & name, const DiskPtr & disk, const String & relative_path) const + const String & name, const VolumePtr & volume, const String & relative_path) const { - return createPart(name, MergeTreePartInfo::fromPartName(name, format_version), disk, relative_path); + return createPart(name, MergeTreePartInfo::fromPartName(name, format_version), volume, relative_path); } MergeTreeData::MutableDataPartPtr MergeTreeData::createPart( const String & name, const MergeTreePartInfo & part_info, - const DiskPtr & disk, const String & relative_path) const + const VolumePtr & volume, const String & relative_path) const { MergeTreeDataPartType type; auto full_path = relative_data_path + relative_path + "/"; - auto mrk_ext = MergeTreeIndexGranularityInfo::getMarksExtensionFromFilesystem(disk, full_path); + auto mrk_ext = MergeTreeIndexGranularityInfo::getMarksExtensionFromFilesystem(volume->getDisk(), full_path); if (mrk_ext) type = getPartTypeFromMarkExtension(*mrk_ext); @@ -1594,7 +1595,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::createPart( type = choosePartType(0, 0); } - return createPart(name, type, part_info, disk, relative_path); + return createPart(name, type, part_info, volume, relative_path); } void MergeTreeData::changeSettings( @@ -2312,7 +2313,7 @@ void MergeTreeData::swapActivePart(MergeTreeData::DataPartPtr part_copy) auto part_it = data_parts_indexes.insert(part_copy).first; modifyPartState(part_it, DataPartState::Committed); - auto disk = original_active_part->disk; + auto disk = original_active_part->volume->getDisk(); String marker_path = original_active_part->getFullRelativePath() + DELETE_ON_DESTROY_MARKER_PATH; try { @@ -2377,7 +2378,7 @@ MergeTreeData::DataPartPtr MergeTreeData::getPartIfExists(const String & part_na static void loadPartAndFixMetadataImpl(MergeTreeData::MutableDataPartPtr part) { - auto disk = part->disk; + auto disk = part->volume->getDisk(); String full_part_path = part->getFullRelativePath(); /// Earlier the list of columns was written incorrectly. Delete it and re-create. @@ -2402,9 +2403,9 @@ static void loadPartAndFixMetadataImpl(MergeTreeData::MutableDataPartPtr part) } } -MergeTreeData::MutableDataPartPtr MergeTreeData::loadPartAndFixMetadata(const DiskPtr & disk, const String & relative_path) const +MergeTreeData::MutableDataPartPtr MergeTreeData::loadPartAndFixMetadata(const VolumePtr & volume, const String & relative_path) const { - MutableDataPartPtr part = createPart(Poco::Path(relative_path).getFileName(), disk, relative_path); + MutableDataPartPtr part = createPart(Poco::Path(relative_path).getFileName(), volume, relative_path); loadPartAndFixMetadataImpl(part); return part; } @@ -2517,7 +2518,7 @@ void MergeTreeData::movePartitionToDisk(const ASTPtr & partition, const String & parts.erase(std::remove_if(parts.begin(), parts.end(), [&](auto part_ptr) { - return part_ptr->disk->getName() == disk->getName(); + return part_ptr->volume->getDisk()->getName() == disk->getName(); }), parts.end()); if (parts.empty()) @@ -2568,9 +2569,9 @@ void MergeTreeData::movePartitionToVolume(const ASTPtr & partition, const String parts.erase(std::remove_if(parts.begin(), parts.end(), [&](auto part_ptr) { - for (const auto & disk : volume->disks) + for (const auto & disk : volume->getDisks()) { - if (part_ptr->disk->getName() == disk->getName()) + if (part_ptr->volume->getDisk()->getName() == disk->getName()) { return true; } @@ -2846,7 +2847,8 @@ MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const for (const auto & part_names : renamed_parts.old_and_new_names) { LOG_DEBUG(log, "Checking part " << part_names.second); - MutableDataPartPtr part = createPart(part_names.first, name_to_disk[part_names.first], source_dir + part_names.second); + auto single_disk_volume = std::make_shared("volume_" + part_names.first, name_to_disk[part_names.first]); + MutableDataPartPtr part = createPart(part_names.first, single_disk_volume, source_dir + part_names.second); loadPartAndFixMetadataImpl(part); loaded_parts.push_back(part); } @@ -2960,12 +2962,12 @@ bool MergeTreeData::TTLEntry::isPartInDestination(StoragePolicyPtr policy, const { if (destination_type == PartDestinationType::VOLUME) { - for (const auto & disk : policy->getVolumeByName(destination_name)->disks) - if (disk->getName() == part.disk->getName()) + for (const auto & disk : policy->getVolumeByName(destination_name)->getDisks()) + if (disk->getName() == part.volume->getDisk()->getName()) return true; } else if (destination_type == PartDestinationType::DISK) - return policy->getDiskByName(destination_name)->getName() == part.disk->getName(); + return policy->getDiskByName(destination_name)->getName() == part.volume->getDisk()->getName(); return false; } @@ -3179,7 +3181,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::cloneAndLoadDataPartOnSameDisk( bool does_storage_policy_allow_same_disk = false; for (const DiskPtr & disk : getStoragePolicy()->getDisks()) { - if (disk->getName() == src_part->disk->getName()) + if (disk->getName() == src_part->volume->getDisk()->getName()) { does_storage_policy_allow_same_disk = true; break; @@ -3192,7 +3194,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::cloneAndLoadDataPartOnSameDisk( String dst_part_name = src_part->getNewName(dst_part_info); String tmp_dst_part_name = tmp_part_prefix + dst_part_name; - auto reservation = reserveSpace(src_part->getBytesOnDisk(), src_part->disk); + auto reservation = reserveSpace(src_part->getBytesOnDisk(), src_part->volume->getDisk()); auto disk = reservation->getDisk(); String src_part_path = src_part->getFullRelativePath(); String dst_part_path = relative_data_path + tmp_dst_part_name; @@ -3204,7 +3206,8 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::cloneAndLoadDataPartOnSameDisk( localBackup(disk, src_part_path, dst_part_path); disk->removeIfExists(dst_part_path + "/" + DELETE_ON_DESTROY_MARKER_PATH); - auto dst_data_part = createPart(dst_part_name, dst_part_info, reservation->getDisk(), tmp_dst_part_name); + auto single_disk_volume = std::make_shared(disk->getName(), disk); + auto dst_data_part = createPart(dst_part_name, dst_part_info, single_disk_volume, tmp_dst_part_name); dst_data_part->is_temp = true; @@ -3276,7 +3279,7 @@ void MergeTreeData::freezePartitionsByMatcher(MatcherFn matcher, const String & if (!matcher(part)) continue; - part->disk->createDirectories(shadow_path); + part->volume->getDisk()->createDirectories(shadow_path); String backup_path = shadow_path + (!with_name.empty() @@ -3287,8 +3290,8 @@ void MergeTreeData::freezePartitionsByMatcher(MatcherFn matcher, const String & LOG_DEBUG(log, "Freezing part " << part->name << " snapshot will be placed at " + backup_path); String backup_part_path = backup_path + relative_data_path + part->relative_path; - localBackup(part->disk, part->getFullRelativePath(), backup_part_path); - part->disk->removeIfExists(backup_part_path + "/" + DELETE_ON_DESTROY_MARKER_PATH); + localBackup(part->volume->getDisk(), part->getFullRelativePath(), backup_part_path); + part->volume->getDisk()->removeIfExists(backup_part_path + "/" + DELETE_ON_DESTROY_MARKER_PATH); part->is_frozen.store(true, std::memory_order_relaxed); ++parts_processed; @@ -3409,7 +3412,7 @@ bool MergeTreeData::areBackgroundMovesNeeded() const if (policy->getVolumes().size() > 1) return true; - return policy->getVolumes().size() == 1 && policy->getVolumes()[0]->disks.size() > 1 && !move_ttl_entries.empty(); + return policy->getVolumes().size() == 1 && policy->getVolumes()[0]->getDisks().size() > 1 && !move_ttl_entries.empty(); } bool MergeTreeData::movePartsToSpace(const DataPartsVector & parts, SpacePtr space) diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index b9aa7ba6d4a..dc6ed5f213f 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -194,14 +194,14 @@ public: /// After this method setColumns must be called MutableDataPartPtr createPart(const String & name, MergeTreeDataPartType type, const MergeTreePartInfo & part_info, - const DiskPtr & disk, const String & relative_path) const; + const VolumePtr & volume, const String & relative_path) const; /// After this methods 'loadColumnsChecksumsIndexes' must be called MutableDataPartPtr createPart(const String & name, - const DiskPtr & disk, const String & relative_path) const; + const VolumePtr & volume, const String & relative_path) const; MutableDataPartPtr createPart(const String & name, const MergeTreePartInfo & part_info, - const DiskPtr & disk, const String & relative_path) const; + const VolumePtr & volume, const String & relative_path) const; /// Auxiliary object to add a set of parts into the working set in two steps: /// * First, as PreCommitted parts (the parts are ready, but not yet in the active set). @@ -537,7 +537,7 @@ public: bool hasAnyTTL() const override { return hasRowsTTL() || hasAnyMoveTTL() || hasAnyColumnTTL(); } /// Check that the part is not broken and calculate the checksums for it if they are not present. - MutableDataPartPtr loadPartAndFixMetadata(const DiskPtr & disk, const String & relative_path) const; + MutableDataPartPtr loadPartAndFixMetadata(const VolumePtr & volume, const String & relative_path) const; /** Create local backup (snapshot) for parts with specified prefix. * Backup is created in directory clickhouse_dir/shadow/i/, where i - incremental number, diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index acdb7616175..00f03317649 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -610,11 +610,12 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor all_columns, data.sorting_key_expr, data.skip_indices, data.merging_params, gathering_columns, gathering_column_names, merging_columns, merging_column_names); + auto single_disk_volume = std::make_shared("volume_" + future_part.name, disk); MergeTreeData::MutableDataPartPtr new_data_part = data.createPart( future_part.name, future_part.type, future_part.part_info, - disk, + single_disk_volume, TMP_PREFIX + future_part.name); new_data_part->setColumns(all_columns); @@ -1028,8 +1029,9 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor in->setProgressCallback(MergeProgressCallback(merge_entry, watch_prev_elapsed, stage_progress)); } + auto single_disk_volume = std::make_shared("volume_" + future_part.name, space_reservation->getDisk()); auto new_data_part = data.createPart( - future_part.name, future_part.type, future_part.part_info, space_reservation->getDisk(), "tmp_mut_" + future_part.name); + future_part.name, future_part.type, future_part.part_info, single_disk_volume, "tmp_mut_" + future_part.name); new_data_part->is_temp = true; new_data_part->ttl_infos = source_part->ttl_infos; @@ -1039,7 +1041,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor new_data_part->setColumns(getColumnsForNewDataPart(source_part, updated_header, all_columns, for_file_renames)); new_data_part->partition.assign(source_part->partition); - auto disk = new_data_part->disk; + auto disk = new_data_part->volume->getDisk(); String new_part_tmp_path = new_data_part->getFullRelativePath(); /// Note: this is done before creating input streams, because otherwise data.data_parts_mutex @@ -1643,7 +1645,7 @@ void MergeTreeDataMergerMutator::finalizeMutatedPart( MergeTreeData::MutableDataPartPtr new_data_part, bool need_remove_expired_values) { - auto disk = new_data_part->disk; + auto disk = new_data_part->volume->getDisk(); if (need_remove_expired_values) { /// Write a file with ttl infos in json format. @@ -1673,7 +1675,7 @@ void MergeTreeDataMergerMutator::finalizeMutatedPart( new_data_part->minmax_idx = source_part->minmax_idx; new_data_part->modification_time = time(nullptr); new_data_part->setBytesOnDisk( - MergeTreeData::DataPart::calculateTotalSizeOnDisk(new_data_part->disk, new_data_part->getFullRelativePath())); + MergeTreeData::DataPart::calculateTotalSizeOnDisk(new_data_part->volume->getDisk(), new_data_part->getFullRelativePath())); new_data_part->calculateColumnsSizesOnDisk(); } diff --git a/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index 134b2fc1ef0..e417adf4eed 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -20,9 +20,9 @@ namespace ErrorCodes MergeTreeDataPartCompact::MergeTreeDataPartCompact( MergeTreeData & storage_, const String & name_, - const DiskPtr & disk_, + const VolumePtr & volume_, const std::optional & relative_path_) - : IMergeTreeDataPart(storage_, name_, disk_, relative_path_, Type::COMPACT) + : IMergeTreeDataPart(storage_, name_, volume_, relative_path_, Type::COMPACT) { } @@ -30,9 +30,9 @@ MergeTreeDataPartCompact::MergeTreeDataPartCompact( const MergeTreeData & storage_, const String & name_, const MergeTreePartInfo & info_, - const DiskPtr & disk_, + const VolumePtr & volume_, const std::optional & relative_path_) - : IMergeTreeDataPart(storage_, name_, info_, disk_, relative_path_, Type::COMPACT) + : IMergeTreeDataPart(storage_, name_, info_, volume_, relative_path_, Type::COMPACT) { } @@ -68,7 +68,7 @@ IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartCompact::getWriter( { return *getColumnPosition(lhs.name) < *getColumnPosition(rhs.name); }); return std::make_unique( - disk, getFullRelativePath(), storage, ordered_columns_list, indices_to_recalc, + volume, getFullRelativePath(), storage, ordered_columns_list, indices_to_recalc, index_granularity_info.marks_file_extension, default_codec, writer_settings, computed_index_granularity); } @@ -99,12 +99,12 @@ void MergeTreeDataPartCompact::loadIndexGranularity() throw Exception("MergeTreeDataPartCompact cannot be created with non-adaptive granulary.", ErrorCodes::NOT_IMPLEMENTED); auto marks_file_path = index_granularity_info.getMarksFilePath(full_path + "data"); - if (!disk->exists(marks_file_path)) - throw Exception("Marks file '" + fullPath(disk, marks_file_path) + "' doesn't exist", ErrorCodes::NO_FILE_IN_DATA_PART); + if (!volume->getDisk()->exists(marks_file_path)) + throw Exception("Marks file '" + fullPath(volume->getDisk(), marks_file_path) + "' doesn't exist", ErrorCodes::NO_FILE_IN_DATA_PART); - size_t marks_file_size = disk->getFileSize(marks_file_path); + size_t marks_file_size = volume->getDisk()->getFileSize(marks_file_path); - auto buffer = disk->readFile(marks_file_path, marks_file_size); + auto buffer = volume->getDisk()->readFile(marks_file_path, marks_file_size); while (!buffer->eof()) { /// Skip offsets for columns @@ -146,9 +146,9 @@ void MergeTreeDataPartCompact::checkConsistency(bool require_part_metadata) cons if (require_part_metadata) { if (!checksums.files.count(mrk_file_name)) - throw Exception("No marks file checksum for column in part " + fullPath(disk, path), ErrorCodes::NO_FILE_IN_DATA_PART); + throw Exception("No marks file checksum for column in part " + fullPath(volume->getDisk(), path), ErrorCodes::NO_FILE_IN_DATA_PART); if (!checksums.files.count(DATA_FILE_NAME_WITH_EXTENSION)) - throw Exception("No data file checksum for in part " + fullPath(disk, path), ErrorCodes::NO_FILE_IN_DATA_PART); + throw Exception("No data file checksum for in part " + fullPath(volume->getDisk(), path), ErrorCodes::NO_FILE_IN_DATA_PART); } } else @@ -156,24 +156,24 @@ void MergeTreeDataPartCompact::checkConsistency(bool require_part_metadata) cons { /// count.txt should be present even in non custom-partitioned parts auto file_path = path + "count.txt"; - if (!disk->exists(file_path) || disk->getFileSize(file_path) == 0) - throw Exception("Part " + path + " is broken: " + fullPath(disk, file_path) + " is empty", ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); + if (!volume->getDisk()->exists(file_path) || volume->getDisk()->getFileSize(file_path) == 0) + throw Exception("Part " + path + " is broken: " + fullPath(volume->getDisk(), file_path) + " is empty", ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); } /// Check that marks are nonempty and have the consistent size with columns number. auto mrk_file_path = path + mrk_file_name; - if (disk->exists(mrk_file_name)) + if (volume->getDisk()->exists(mrk_file_name)) { - UInt64 file_size = disk->getFileSize(mrk_file_name); + UInt64 file_size = volume->getDisk()->getFileSize(mrk_file_name); if (!file_size) - throw Exception("Part " + path + " is broken: " + fullPath(disk, mrk_file_name) + " is empty.", + throw Exception("Part " + path + " is broken: " + fullPath(volume->getDisk(), mrk_file_name) + " is empty.", ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); UInt64 expected_file_size = index_granularity_info.getMarkSizeInBytes(columns.size()) * index_granularity.getMarksCount(); if (expected_file_size != file_size) throw Exception( - "Part " + path + " is broken: bad size of marks file '" + fullPath(disk, mrk_file_name) + "': " + std::to_string(file_size) + ", must be: " + std::to_string(expected_file_size), + "Part " + path + " is broken: bad size of marks file '" + fullPath(volume->getDisk(), mrk_file_name) + "': " + std::to_string(file_size) + ", must be: " + std::to_string(expected_file_size), ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); } } diff --git a/src/Storages/MergeTree/MergeTreeDataPartCompact.h b/src/Storages/MergeTree/MergeTreeDataPartCompact.h index fa98a2c863f..3ce1fd830d5 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartCompact.h +++ b/src/Storages/MergeTree/MergeTreeDataPartCompact.h @@ -26,13 +26,13 @@ public: const MergeTreeData & storage_, const String & name_, const MergeTreePartInfo & info_, - const DiskPtr & disk_, + const VolumePtr & volume_, const std::optional & relative_path_ = {}); MergeTreeDataPartCompact( MergeTreeData & storage_, const String & name_, - const DiskPtr & disk_, + const VolumePtr & volume_, const std::optional & relative_path_ = {}); MergeTreeReaderPtr getReader( diff --git a/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index dfc57d5d5d0..85903d0161e 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -19,9 +19,9 @@ namespace ErrorCodes MergeTreeDataPartWide::MergeTreeDataPartWide( MergeTreeData & storage_, const String & name_, - const DiskPtr & disk_, + const VolumePtr & volume_, const std::optional & relative_path_) - : IMergeTreeDataPart(storage_, name_, disk_, relative_path_, Type::WIDE) + : IMergeTreeDataPart(storage_, name_, volume_, relative_path_, Type::WIDE) { } @@ -29,9 +29,9 @@ MergeTreeDataPartWide::MergeTreeDataPartWide( const MergeTreeData & storage_, const String & name_, const MergeTreePartInfo & info_, - const DiskPtr & disk_, + const VolumePtr & volume_, const std::optional & relative_path_) - : IMergeTreeDataPart(storage_, name_, info_, disk_, relative_path_, Type::WIDE) + : IMergeTreeDataPart(storage_, name_, info_, volume_, relative_path_, Type::WIDE) { } @@ -59,7 +59,7 @@ IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartWide::getWriter( const MergeTreeIndexGranularity & computed_index_granularity) const { return std::make_unique( - disk, getFullRelativePath(), storage, columns_list, indices_to_recalc, + volume, getFullRelativePath(), storage, columns_list, indices_to_recalc, index_granularity_info.marks_file_extension, default_codec, writer_settings, computed_index_granularity); } @@ -99,7 +99,7 @@ ColumnSize MergeTreeDataPartWide::getColumnSizeImpl( void MergeTreeDataPartWide::loadIndexGranularity() { String full_path = getFullRelativePath(); - index_granularity_info.changeGranularityIfRequired(disk, full_path); + index_granularity_info.changeGranularityIfRequired(volume->getDisk(), full_path); if (columns.empty()) @@ -107,10 +107,10 @@ void MergeTreeDataPartWide::loadIndexGranularity() /// We can use any column, it doesn't matter std::string marks_file_path = index_granularity_info.getMarksFilePath(full_path + getFileNameForColumn(columns.front())); - if (!disk->exists(marks_file_path)) - throw Exception("Marks file '" + fullPath(disk, marks_file_path) + "' doesn't exist", ErrorCodes::NO_FILE_IN_DATA_PART); + if (!volume->getDisk()->exists(marks_file_path)) + throw Exception("Marks file '" + fullPath(volume->getDisk(), marks_file_path) + "' doesn't exist", ErrorCodes::NO_FILE_IN_DATA_PART); - size_t marks_file_size = disk->getFileSize(marks_file_path); + size_t marks_file_size = volume->getDisk()->getFileSize(marks_file_path); if (!index_granularity_info.is_adaptive) { @@ -119,7 +119,7 @@ void MergeTreeDataPartWide::loadIndexGranularity() } else { - auto buffer = disk->readFile(marks_file_path, marks_file_size); + auto buffer = volume->getDisk()->readFile(marks_file_path, marks_file_size); while (!buffer->eof()) { buffer->seek(sizeof(size_t) * 2, SEEK_CUR); /// skip offset_in_compressed file and offset_in_decompressed_block @@ -129,7 +129,7 @@ void MergeTreeDataPartWide::loadIndexGranularity() } if (index_granularity.getMarksCount() * index_granularity_info.getMarkSizeInBytes() != marks_file_size) - throw Exception("Cannot read all marks from file " + fullPath(disk, marks_file_path), ErrorCodes::CANNOT_READ_ALL_DATA); + throw Exception("Cannot read all marks from file " + fullPath(volume->getDisk(), marks_file_path), ErrorCodes::CANNOT_READ_ALL_DATA); } index_granularity.setInitialized(); @@ -158,10 +158,10 @@ void MergeTreeDataPartWide::checkConsistency(bool require_part_metadata) const String mrk_file_name = file_name + index_granularity_info.marks_file_extension; String bin_file_name = file_name + ".bin"; if (!checksums.files.count(mrk_file_name)) - throw Exception("No " + mrk_file_name + " file checksum for column " + name_type.name + " in part " + fullPath(disk, path), + throw Exception("No " + mrk_file_name + " file checksum for column " + name_type.name + " in part " + fullPath(volume->getDisk(), path), ErrorCodes::NO_FILE_IN_DATA_PART); if (!checksums.files.count(bin_file_name)) - throw Exception("No " + bin_file_name + " file checksum for column " + name_type.name + " in part " + fullPath(disk, path), + throw Exception("No " + bin_file_name + " file checksum for column " + name_type.name + " in part " + fullPath(volume->getDisk(), path), ErrorCodes::NO_FILE_IN_DATA_PART); }, stream_path); } @@ -179,12 +179,12 @@ void MergeTreeDataPartWide::checkConsistency(bool require_part_metadata) const auto file_path = path + IDataType::getFileNameForStream(name_type.name, substream_path) + index_granularity_info.marks_file_extension; /// Missing file is Ok for case when new column was added. - if (disk->exists(file_path)) + if (volume->getDisk()->exists(file_path)) { - UInt64 file_size = disk->getFileSize(file_path); + UInt64 file_size = volume->getDisk()->getFileSize(file_path); if (!file_size) - throw Exception("Part " + path + " is broken: " + fullPath(disk, file_path) + " is empty.", + throw Exception("Part " + path + " is broken: " + fullPath(volume->getDisk(), file_path) + " is empty.", ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); if (!marks_size) diff --git a/src/Storages/MergeTree/MergeTreeDataPartWide.h b/src/Storages/MergeTree/MergeTreeDataPartWide.h index d66e2108fc1..ba9e0765510 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWide.h +++ b/src/Storages/MergeTree/MergeTreeDataPartWide.h @@ -19,13 +19,13 @@ public: const MergeTreeData & storage_, const String & name_, const MergeTreePartInfo & info_, - const DiskPtr & disk, + const VolumePtr & volume, const std::optional & relative_path = {}); MergeTreeDataPartWide( MergeTreeData & storage_, const String & name_, - const DiskPtr & disk, + const VolumePtr & volume, const std::optional & relative_path = {}); MergeTreeReaderPtr getReader( diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index 2f708ac6954..f107f18792e 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -6,7 +6,7 @@ namespace DB MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact( - DiskPtr disk_, + const VolumePtr & volume_, const String & part_path_, const MergeTreeData & storage_, const NamesAndTypesList & columns_list_, @@ -15,7 +15,7 @@ MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact( const CompressionCodecPtr & default_codec_, const MergeTreeWriterSettings & settings_, const MergeTreeIndexGranularity & index_granularity_) -: IMergeTreeDataPartWriter(disk_, part_path_, +: IMergeTreeDataPartWriter(volume_, part_path_, storage_, columns_list_, indices_to_recalc_, marks_file_extension_, default_codec_, settings_, index_granularity_) @@ -25,7 +25,7 @@ MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact( stream = std::make_unique( data_file_name, - disk_, + volume->getDisk(), part_path + data_file_name, DataPart::DATA_FILE_EXTENSION, part_path + data_file_name, marks_file_extension, default_codec, diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h index 45d72d90b1e..f786a09d93e 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h @@ -8,7 +8,7 @@ class MergeTreeDataPartWriterCompact : public IMergeTreeDataPartWriter { public: MergeTreeDataPartWriterCompact( - DiskPtr disk, + const VolumePtr & volume, const String & part_path, const MergeTreeData & storage, const NamesAndTypesList & columns_list, diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index e5eececacfb..1b0e4441b9f 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -13,7 +13,7 @@ namespace } MergeTreeDataPartWriterWide::MergeTreeDataPartWriterWide( - DiskPtr disk_, + const VolumePtr & volume_, const String & part_path_, const MergeTreeData & storage_, const NamesAndTypesList & columns_list_, @@ -22,7 +22,7 @@ MergeTreeDataPartWriterWide::MergeTreeDataPartWriterWide( const CompressionCodecPtr & default_codec_, const MergeTreeWriterSettings & settings_, const MergeTreeIndexGranularity & index_granularity_) - : IMergeTreeDataPartWriter(disk_, part_path_, + : IMergeTreeDataPartWriter(volume_, part_path_, storage_, columns_list_, indices_to_recalc_, marks_file_extension_, default_codec_, settings_, index_granularity_) { @@ -46,7 +46,7 @@ void MergeTreeDataPartWriterWide::addStreams( column_streams[stream_name] = std::make_unique( stream_name, - disk, + volume->getDisk(), part_path + stream_name, DATA_FILE_EXTENSION, part_path + stream_name, marks_file_extension, effective_codec, diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h index 4e4f4806d53..42357af15a8 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h @@ -11,7 +11,7 @@ public: using ColumnToSize = std::map; MergeTreeDataPartWriterWide( - DiskPtr disk, + const VolumePtr & volume, const String & part_path, const MergeTreeData & storage, const NamesAndTypesList & columns_list, diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 580c95b34dd..4b3c1769551 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -1294,7 +1294,7 @@ MarkRanges MergeTreeDataSelectExecutor::filterMarksUsingIndex( const MarkRanges & ranges, const Settings & settings) const { - if (!part->disk->exists(part->getFullRelativePath() + index->getFileName() + ".idx")) + if (!part->volume->getDisk()->exists(part->getFullRelativePath() + index->getFileName() + ".idx")) { LOG_DEBUG(log, "File for index " << backQuote(index->name) << " does not exist. Skipping it."); return ranges; diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 1814e40e10e..a78e2e5ae32 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -231,12 +232,13 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa NamesAndTypesList columns = data.getColumns().getAllPhysical().filter(block.getNames()); ReservationPtr reservation = data.reserveSpacePreferringTTLRules(expected_size, move_ttl_infos, time(nullptr)); + VolumePtr volume = data.getStoragePolicy()->getVolume(0); auto new_data_part = data.createPart( part_name, data.choosePartType(expected_size, block.rows()), new_part_info, - reservation->getDisk(), + createVolumeFromReservation(reservation, volume), TMP_PREFIX + part_name); new_data_part->setColumns(columns); @@ -247,13 +249,13 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa /// The name could be non-unique in case of stale files from previous runs. String full_path = new_data_part->getFullRelativePath(); - if (new_data_part->disk->exists(full_path)) + if (new_data_part->volume->getDisk()->exists(full_path)) { - LOG_WARNING(log, "Removing old temporary directory " + fullPath(new_data_part->disk, full_path)); - new_data_part->disk->removeRecursive(full_path); + LOG_WARNING(log, "Removing old temporary directory " + fullPath(new_data_part->volume->getDisk(), full_path)); + new_data_part->volume->getDisk()->removeRecursive(full_path); } - new_data_part->disk->createDirectories(full_path); + new_data_part->volume->getDisk()->createDirectories(full_path); /// If we need to calculate some columns to sort. if (data.hasSortingKey() || data.hasSkipIndices()) diff --git a/src/Storages/MergeTree/MergeTreeIndexReader.cpp b/src/Storages/MergeTree/MergeTreeIndexReader.cpp index 55eeb41ceb5..d8f13e49b31 100644 --- a/src/Storages/MergeTree/MergeTreeIndexReader.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexReader.cpp @@ -7,7 +7,7 @@ namespace DB MergeTreeIndexReader::MergeTreeIndexReader( MergeTreeIndexPtr index_, MergeTreeData::DataPartPtr part_, size_t marks_count_, const MarkRanges & all_mark_ranges_) : index(index_), stream( - part_->disk, + part_->volume->getDisk(), part_->getFullRelativePath() + index->getFileName(), ".idx", marks_count_, all_mark_ranges_, MergeTreeReaderSettings{}, nullptr, nullptr, diff --git a/src/Storages/MergeTree/MergeTreePartsMover.cpp b/src/Storages/MergeTree/MergeTreePartsMover.cpp index c6a00b46345..1609e1e517e 100644 --- a/src/Storages/MergeTree/MergeTreePartsMover.cpp +++ b/src/Storages/MergeTree/MergeTreePartsMover.cpp @@ -108,7 +108,7 @@ bool MergeTreePartsMover::selectPartsForMove( /// Do not check last volume for (size_t i = 0; i != volumes.size() - 1; ++i) { - for (const auto & disk : volumes[i]->disks) + for (const auto & disk : volumes[i]->getDisks()) { UInt64 required_maximum_available_space = disk->getTotalSpace() * policy->getMoveFactor(); UInt64 unreserved_space = disk->getUnreservedSpace(); @@ -129,7 +129,7 @@ bool MergeTreePartsMover::selectPartsForMove( continue; auto ttl_entry = part->storage.selectTTLEntryForTTLInfos(part->ttl_infos, time_of_move); - auto to_insert = need_to_move.find(part->disk); + auto to_insert = need_to_move.find(part->volume->getDisk()); ReservationPtr reservation; if (ttl_entry) { @@ -196,8 +196,9 @@ MergeTreeData::DataPartPtr MergeTreePartsMover::clonePart(const MergeTreeMoveEnt LOG_TRACE(log, "Cloning part " << moving_part.part->name); moving_part.part->makeCloneOnDiskDetached(moving_part.reserved_space); + auto single_disk_volume = std::make_shared("volume_" + moving_part.part->name, moving_part.reserved_space->getDisk()); MergeTreeData::MutableDataPartPtr cloned_part = - data->createPart(moving_part.part->name, moving_part.reserved_space->getDisk(), "detached/" + moving_part.part->name); + data->createPart(moving_part.part->name, single_disk_volume, "detached/" + moving_part.part->name); LOG_TRACE(log, "Part " << moving_part.part->name << " was cloned to " << cloned_part->getFullPath()); cloned_part->loadColumnsChecksumsIndexes(true, true); diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index a895149e12e..c4a05a8bfac 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -28,7 +28,7 @@ MergeTreeReaderCompact::MergeTreeReaderCompact( uncompressed_cache_, mark_cache_, std::move(mark_ranges_), std::move(settings_), std::move(avg_value_size_hints_)) , marks_loader( - data_part->disk, + data_part->volume->getDisk(), mark_cache, data_part->index_granularity_info.getMarksFilePath(data_part->getFullRelativePath() + MergeTreeDataPartCompact::DATA_FILE_NAME), data_part->getMarksCount(), data_part->index_granularity_info, @@ -40,10 +40,10 @@ MergeTreeReaderCompact::MergeTreeReaderCompact( if (uncompressed_cache) { auto buffer = std::make_unique( - fullPath(data_part->disk, full_data_path), + fullPath(data_part->volume->getDisk(), full_data_path), [this, full_data_path, buffer_size]() { - return data_part->disk->readFile( + return data_part->volume->getDisk()->readFile( full_data_path, buffer_size, 0, @@ -62,7 +62,7 @@ MergeTreeReaderCompact::MergeTreeReaderCompact( { auto buffer = std::make_unique( - data_part->disk->readFile(full_data_path, buffer_size, 0, settings.min_bytes_to_use_direct_io, 0)); + data_part->volume->getDisk()->readFile(full_data_path, buffer_size, 0, settings.min_bytes_to_use_direct_io, 0)); if (profile_callback_) buffer->setProfileCallback(profile_callback_, clock_type_); diff --git a/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/src/Storages/MergeTree/MergeTreeReaderWide.cpp index 1a03acb5758..34bf095e57e 100644 --- a/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -176,7 +176,7 @@ void MergeTreeReaderWide::addStreams(const String & name, const IDataType & type return; streams.emplace(stream_name, std::make_unique( - data_part->disk, data_part->getFullRelativePath() + stream_name, DATA_FILE_EXTENSION, + data_part->volume->getDisk(), data_part->getFullRelativePath() + stream_name, DATA_FILE_EXTENSION, data_part->getMarksCount(), all_mark_ranges, settings, mark_cache, uncompressed_cache, data_part->getFileSizeOrZero(stream_name + DATA_FILE_EXTENSION), &data_part->index_granularity_info, diff --git a/src/Storages/MergeTree/checkDataPart.cpp b/src/Storages/MergeTree/checkDataPart.cpp index c12ed12c2e0..58ff2af9466 100644 --- a/src/Storages/MergeTree/checkDataPart.cpp +++ b/src/Storages/MergeTree/checkDataPart.cpp @@ -167,7 +167,7 @@ IMergeTreeDataPart::Checksums checkDataPart( std::function is_cancelled) { return checkDataPart( - data_part->disk, + data_part->volume->getDisk(), data_part->getFullRelativePath(), data_part->getColumns(), data_part->getType(), diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 44707a43fa1..2150b288ac0 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -571,7 +571,7 @@ void StorageDistributed::startup() if (!volume) return; - for (const DiskPtr & disk : volume->disks) + for (const DiskPtr & disk : volume->getDisks()) createDirectoryMonitors(disk->getPath()); for (const String & path : getDataPaths()) @@ -599,7 +599,7 @@ Strings StorageDistributed::getDataPaths() const if (relative_data_path.empty()) return paths; - for (const DiskPtr & disk : volume->disks) + for (const DiskPtr & disk : volume->getDisks()) paths.push_back(disk->getPath() + relative_data_path); return paths; @@ -803,7 +803,7 @@ void StorageDistributed::rename(const String & new_path_to_table_data, const Sto } void StorageDistributed::renameOnDisk(const String & new_path_to_table_data) { - for (const DiskPtr & disk : volume->disks) + for (const DiskPtr & disk : volume->getDisks()) { const String path(disk->getPath()); auto new_path = path + new_path_to_table_data; diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 2c7685c3a5f..0ac0fcef458 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -277,7 +277,7 @@ public: /// if we mutate part, than we should reserve space on the same disk, because mutations possible can create hardlinks if (is_mutation) - reserved_space = storage.tryReserveSpace(total_size, future_part_.parts[0]->disk); + reserved_space = storage.tryReserveSpace(total_size, future_part_.parts[0]->volume->getDisk()); else { IMergeTreeDataPart::TTLInfos ttl_infos; @@ -285,7 +285,7 @@ public: for (auto & part_ptr : future_part_.parts) { ttl_infos.update(part_ptr->ttl_infos); - max_volume_index = std::max(max_volume_index, storage.getStoragePolicy()->getVolumeIndexByDisk(part_ptr->disk)); + max_volume_index = std::max(max_volume_index, storage.getStoragePolicy()->getVolumeIndexByDisk(part_ptr->volume->getDisk())); } reserved_space = storage.tryReserveSpacePreferringTTLRules(total_size, ttl_infos, time(nullptr), max_volume_index); @@ -1232,7 +1232,7 @@ CheckResults StorageMergeTree::checkData(const ASTPtr & query, const Context & c for (auto & part : data_parts) { - auto disk = part->disk; + auto disk = part->volume->getDisk(); String part_path = part->getFullRelativePath(); /// If the checksums file is not present, calculate the checksums and write them to disk. String checksums_path = part_path + "checksums.txt"; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 7a5d4dbf2bf..9a00e3ef1dd 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -1049,7 +1049,7 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) for (auto & part_ptr : parts) { ttl_infos.update(part_ptr->ttl_infos); - max_volume_index = std::max(max_volume_index, getStoragePolicy()->getVolumeIndexByDisk(part_ptr->disk)); + max_volume_index = std::max(max_volume_index, getStoragePolicy()->getVolumeIndexByDisk(part_ptr->volume->getDisk())); } ReservationPtr reserved_space = reserveSpacePreferringTTLRules(estimated_space_for_merge, ttl_infos, time(nullptr), max_volume_index); @@ -1188,7 +1188,7 @@ bool StorageReplicatedMergeTree::tryExecutePartMutation(const StorageReplicatedM /// Once we mutate part, we must reserve space on the same disk, because mutations can possibly create hardlinks. /// Can throw an exception. - ReservationPtr reserved_space = reserveSpace(estimated_space_for_result, source_part->disk); + ReservationPtr reserved_space = reserveSpace(estimated_space_for_result, source_part->volume->getDisk()); auto table_lock = lockStructureForShare( false, RWLockImpl::NO_QUERY, storage_settings_ptr->lock_acquire_timeout_for_background_operations); diff --git a/src/Storages/System/StorageSystemParts.cpp b/src/Storages/System/StorageSystemParts.cpp index 2dfbf415100..2418594899e 100644 --- a/src/Storages/System/StorageSystemParts.cpp +++ b/src/Storages/System/StorageSystemParts.cpp @@ -111,7 +111,7 @@ void StorageSystemParts::processNextStorage(MutableColumns & columns_, const Sto columns_[i++]->insert(info.database); columns_[i++]->insert(info.table); columns_[i++]->insert(info.engine); - columns_[i++]->insert(part->disk->getName()); + columns_[i++]->insert(part->volume->getDisk()->getName()); columns_[i++]->insert(part->getFullPath()); if (has_state_column) diff --git a/src/Storages/System/StorageSystemPartsColumns.cpp b/src/Storages/System/StorageSystemPartsColumns.cpp index baa608727aa..b8acdc5f995 100644 --- a/src/Storages/System/StorageSystemPartsColumns.cpp +++ b/src/Storages/System/StorageSystemPartsColumns.cpp @@ -138,7 +138,7 @@ void StorageSystemPartsColumns::processNextStorage(MutableColumns & columns_, co columns_[j++]->insert(info.database); columns_[j++]->insert(info.table); columns_[j++]->insert(info.engine); - columns_[j++]->insert(part->disk->getName()); + columns_[j++]->insert(part->volume->getDisk()->getName()); columns_[j++]->insert(part->getFullPath()); columns_[j++]->insert(column.name); diff --git a/src/Storages/System/StorageSystemStoragePolicies.cpp b/src/Storages/System/StorageSystemStoragePolicies.cpp index 69962d42d18..81b6ddd465a 100644 --- a/src/Storages/System/StorageSystemStoragePolicies.cpp +++ b/src/Storages/System/StorageSystemStoragePolicies.cpp @@ -55,8 +55,8 @@ Pipes StorageSystemStoragePolicies::read( col_volume_name->insert(volumes[i]->getName()); col_priority->insert(i + 1); Array disks; - disks.reserve(volumes[i]->disks.size()); - for (const auto & disk_ptr : volumes[i]->disks) + disks.reserve(volumes[i]->getDisks().size()); + for (const auto & disk_ptr : volumes[i]->getDisks()) disks.push_back(disk_ptr->getName()); col_disks->insert(disks); col_max_part_size->insert(volumes[i]->max_data_part_size); From de8bf3f78c10f5cf5fbce2290a7bd7b565001fc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=82=D0=B5=D0=BC=20=D0=A1=D1=82=D1=80=D0=B5?= =?UTF-8?q?=D0=BB=D1=8C=D1=86=D0=BE=D0=B2?= Date: Sun, 10 May 2020 02:53:01 +0300 Subject: [PATCH 044/738] Draft of ComplexKeyDirectDictionary layout --- .../ComplexKeyDirectDictionary.cpp | 595 ++++++++++++++++++ src/Dictionaries/ComplexKeyDirectDictionary.h | 224 +++++++ .../getDictionaryConfigurationFromAST.cpp | 3 +- src/Dictionaries/registerDictionaries.cpp | 1 + src/Dictionaries/registerDictionaries.h | 1 + src/Functions/FunctionsExternalDictionaries.h | 6 + 6 files changed, 829 insertions(+), 1 deletion(-) create mode 100644 src/Dictionaries/ComplexKeyDirectDictionary.cpp create mode 100644 src/Dictionaries/ComplexKeyDirectDictionary.h diff --git a/src/Dictionaries/ComplexKeyDirectDictionary.cpp b/src/Dictionaries/ComplexKeyDirectDictionary.cpp new file mode 100644 index 00000000000..8c21a6189d4 --- /dev/null +++ b/src/Dictionaries/ComplexKeyDirectDictionary.cpp @@ -0,0 +1,595 @@ +#include "ComplexKeyDirectDictionary.h" +#include +#include "DictionaryBlockInputStream.h" +#include "DictionaryFactory.h" +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int TYPE_MISMATCH; + extern const int BAD_ARGUMENTS; + extern const int UNSUPPORTED_METHOD; +} + + +ComplexKeyDirectDictionary::ComplexKeyDirectDictionary( + const std::string & database_, + const std::string & name_, + const DictionaryStructure & dict_struct_, + DictionarySourcePtr source_ptr_, + BlockPtr saved_block_) + : database(database_) + , name(name_) + , full_name{database_.empty() ? name_ : (database_ + "." + name_)} + , dict_struct(dict_struct_) + , source_ptr{std::move(source_ptr_)} + , saved_block{std::move(saved_block_)} +{ + if (!this->source_ptr->supportsSelectiveLoad()) + throw Exception{full_name + ": source cannot be used with ComplexKeyDirectDictionary", ErrorCodes::UNSUPPORTED_METHOD}; + + + createAttributes(); +} + +#define DECLARE(TYPE) \ + void ComplexKeyDirectDictionary::get##TYPE(const std::string & attribute_name, const Columns & key_columns, const DataTypes & key_types, ResultArrayType & out) const \ + { \ + dict_struct.validateKeyTypes(key_types); \ + const auto & attribute = getAttribute(attribute_name); \ + checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \ +\ + const auto null_value = std::get(attribute.null_values); \ +\ + getItemsImpl( \ + attribute, key_columns, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t) { return null_value; }); \ + } +DECLARE(UInt8) +DECLARE(UInt16) +DECLARE(UInt32) +DECLARE(UInt64) +DECLARE(UInt128) +DECLARE(Int8) +DECLARE(Int16) +DECLARE(Int32) +DECLARE(Int64) +DECLARE(Float32) +DECLARE(Float64) +DECLARE(Decimal32) +DECLARE(Decimal64) +DECLARE(Decimal128) +#undef DECLARE + +void ComplexKeyDirectDictionary::getString( + const std::string & attribute_name, const Columns & key_columns, const DataTypes & key_types, ColumnString * out) const +{ + dict_struct.validateKeyTypes(key_types); + const auto & attribute = getAttribute(attribute_name); + checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString); + + const auto & null_value = std::get(attribute.null_values); + getItemsStringImpl( + attribute, + key_columns, + [&](const size_t, const String value) { const auto ref = StringRef{value}; out->insertData(ref.data, ref.size); }, + [&](const size_t) { return String(null_value.data, null_value.size); }); +} + +#define DECLARE(TYPE) \ + void ComplexKeyDirectDictionary::get##TYPE( \ + const std::string & attribute_name, \ + const Columns & key_columns, \ + const DataTypes & key_types, \ + const PaddedPODArray & def, \ + ResultArrayType & out) const \ + { \ + dict_struct.validateKeyTypes(key_types); \ + const auto & attribute = getAttribute(attribute_name); \ + checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \ +\ + getItemsImpl( \ + attribute, key_columns, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t row) { return def[row]; }); \ + } +DECLARE(UInt8) +DECLARE(UInt16) +DECLARE(UInt32) +DECLARE(UInt64) +DECLARE(UInt128) +DECLARE(Int8) +DECLARE(Int16) +DECLARE(Int32) +DECLARE(Int64) +DECLARE(Float32) +DECLARE(Float64) +DECLARE(Decimal32) +DECLARE(Decimal64) +DECLARE(Decimal128) +#undef DECLARE + +void ComplexKeyDirectDictionary::getString( + const std::string & attribute_name, const Columns & key_columns, const DataTypes & key_types, const ColumnString * const def, ColumnString * const out) const +{ + dict_struct.validateKeyTypes(key_types); + + const auto & attribute = getAttribute(attribute_name); + checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString); + + getItemsStringImpl( + attribute, + key_columns, + [&](const size_t, const String value) { const auto ref = StringRef{value}; out->insertData(ref.data, ref.size); }, + [&](const size_t row) { const auto ref = def->getDataAt(row); return String(ref.data, ref.size); }); +} + +#define DECLARE(TYPE) \ + void ComplexKeyDirectDictionary::get##TYPE( \ + const std::string & attribute_name, const Columns & key_columns, const DataTypes & key_types, const TYPE def, ResultArrayType & out) const \ + { \ + dict_struct.validateKeyTypes(key_types); \ + const auto & attribute = getAttribute(attribute_name); \ + checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \ +\ + getItemsImpl( \ + attribute, key_columns, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t) { return def; }); \ + } +DECLARE(UInt8) +DECLARE(UInt16) +DECLARE(UInt32) +DECLARE(UInt64) +DECLARE(UInt128) +DECLARE(Int8) +DECLARE(Int16) +DECLARE(Int32) +DECLARE(Int64) +DECLARE(Float32) +DECLARE(Float64) +DECLARE(Decimal32) +DECLARE(Decimal64) +DECLARE(Decimal128) +#undef DECLARE + +void ComplexKeyDirectDictionary::getString( + const std::string & attribute_name, const Columns & key_columns, const DataTypes & key_types, const String & def, ColumnString * const out) const +{ + dict_struct.validateKeyTypes(key_types); + + const auto & attribute = getAttribute(attribute_name); + checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString); + + ComplexKeyDirectDictionary::getItemsStringImpl( + attribute, + key_columns, + [&](const size_t, const String value) { const auto ref = StringRef{value}; out->insertData(ref.data, ref.size); }, + [&](const size_t) { return def; }); +} + + +void ComplexKeyDirectDictionary::has(const Columns & key_columns, const DataTypes & key_types, PaddedPODArray & out) const +{ + dict_struct.validateKeyTypes(key_types); + const auto & attribute = attributes.front(); + + switch (attribute.type) + { + case AttributeUnderlyingType::utUInt8: + has(attribute, key_columns, out); + break; + case AttributeUnderlyingType::utUInt16: + has(attribute, key_columns, out); + break; + case AttributeUnderlyingType::utUInt32: + has(attribute, key_columns, out); + break; + case AttributeUnderlyingType::utUInt64: + has(attribute, key_columns, out); + break; + case AttributeUnderlyingType::utUInt128: + has(attribute, key_columns, out); + break; + case AttributeUnderlyingType::utInt8: + has(attribute, key_columns, out); + break; + case AttributeUnderlyingType::utInt16: + has(attribute, key_columns, out); + break; + case AttributeUnderlyingType::utInt32: + has(attribute, key_columns, out); + break; + case AttributeUnderlyingType::utInt64: + has(attribute, key_columns, out); + break; + case AttributeUnderlyingType::utFloat32: + has(attribute, key_columns, out); + break; + case AttributeUnderlyingType::utFloat64: + has(attribute, key_columns, out); + break; + case AttributeUnderlyingType::utString: + has(attribute, key_columns, out); + break; + + case AttributeUnderlyingType::utDecimal32: + has(attribute, key_columns, out); + break; + case AttributeUnderlyingType::utDecimal64: + has(attribute, key_columns, out); + break; + case AttributeUnderlyingType::utDecimal128: + has(attribute, key_columns, out); + break; + } +} + + +void ComplexKeyDirectDictionary::createAttributes() +{ + const auto size = dict_struct.attributes.size(); + attributes.reserve(size); + + for (const auto & attribute : dict_struct.attributes) + { + attribute_index_by_name.emplace(attribute.name, attributes.size()); + attribute_name_by_index.emplace(attributes.size(), attribute.name); + attributes.push_back(createAttributeWithType(attribute.underlying_type, attribute.null_value, attribute.name)); + + if (attribute.hierarchical) + throw Exception{full_name + ": hierarchical attributes not supported for dictionary of type " + getTypeName(), + ErrorCodes::TYPE_MISMATCH}; + } +} + + +template +void ComplexKeyDirectDictionary::createAttributeImpl(Attribute & attribute, const Field & null_value) +{ + attribute.null_values = T(null_value.get>()); +} + +template <> +void ComplexKeyDirectDictionary::createAttributeImpl(Attribute & attribute, const Field & null_value) +{ + attribute.string_arena = std::make_unique(); + const String & string = null_value.get(); + const char * string_in_arena = attribute.string_arena->insert(string.data(), string.size()); + attribute.null_values.emplace(string_in_arena, string.size()); +} + + +ComplexKeyDirectDictionary::Attribute ComplexKeyDirectDictionary::createAttributeWithType(const AttributeUnderlyingType type, const Field & null_value, const std::string & attr_name) +{ + Attribute attr{type, {}, {}, attr_name}; + + switch (type) + { + case AttributeUnderlyingType::utUInt8: + createAttributeImpl(attr, null_value); + break; + case AttributeUnderlyingType::utUInt16: + createAttributeImpl(attr, null_value); + break; + case AttributeUnderlyingType::utUInt32: + createAttributeImpl(attr, null_value); + break; + case AttributeUnderlyingType::utUInt64: + createAttributeImpl(attr, null_value); + break; + case AttributeUnderlyingType::utUInt128: + createAttributeImpl(attr, null_value); + break; + case AttributeUnderlyingType::utInt8: + createAttributeImpl(attr, null_value); + break; + case AttributeUnderlyingType::utInt16: + createAttributeImpl(attr, null_value); + break; + case AttributeUnderlyingType::utInt32: + createAttributeImpl(attr, null_value); + break; + case AttributeUnderlyingType::utInt64: + createAttributeImpl(attr, null_value); + break; + case AttributeUnderlyingType::utFloat32: + createAttributeImpl(attr, null_value); + break; + case AttributeUnderlyingType::utFloat64: + createAttributeImpl(attr, null_value); + break; + case AttributeUnderlyingType::utString: + createAttributeImpl(attr, null_value); + break; + + case AttributeUnderlyingType::utDecimal32: + createAttributeImpl(attr, null_value); + break; + case AttributeUnderlyingType::utDecimal64: + createAttributeImpl(attr, null_value); + break; + case AttributeUnderlyingType::utDecimal128: + createAttributeImpl(attr, null_value); + break; + } + + return attr; +} + +template +StringRef ComplexKeyDirectDictionary::placeKeysInPool( + const size_t row, const Columns & key_columns, StringRefs & keys, const std::vector & key_attributes, Pool & pool) const +{ + const auto keys_size = key_columns.size(); + size_t sum_keys_size{}; + + for (size_t j = 0; j < keys_size; ++j) + { + keys[j] = key_columns[j]->getDataAt(row); + sum_keys_size += keys[j].size; + if (key_attributes[j].underlying_type == AttributeUnderlyingType::utString) + sum_keys_size += sizeof(size_t) + 1; + } + + auto place = pool.alloc(sum_keys_size); + + auto key_start = place; + for (size_t j = 0; j < keys_size; ++j) + { + if (key_attributes[j].underlying_type == AttributeUnderlyingType::utString) + { + auto start = key_start; + auto key_size = keys[j].size + 1; + memcpy(key_start, &key_size, sizeof(size_t)); + key_start += sizeof(size_t); + memcpy(key_start, keys[j].data, keys[j].size); + key_start += keys[j].size; + *key_start = '\0'; + ++key_start; + keys[j].data = start; + keys[j].size += sizeof(size_t) + 1; + } + else + { + memcpy(key_start, keys[j].data, keys[j].size); + keys[j].data = key_start; + key_start += keys[j].size; + } + } + + return {place, sum_keys_size}; +} + + +template +void ComplexKeyDirectDictionary::getItemsImpl( + const Attribute & attribute, const Columns & key_columns, ValueSetter && set_value, DefaultGetter && get_default) const +{ + const auto rows = key_columns.front()->size(); + const auto keys_size = dict_struct.key->size(); + StringRefs keys_array(rows); + std::unordered_map value_by_key; + Arena temporary_keys_pool; + std::vector to_load(rows); + std::vector keys(rows); + + for (const auto row : ext::range(0, rows)) { + const StringRef keyref = placeKeysInPool(row, key_columns, keys_array, *dict_struct.key, temporary_keys_pool); + std::string key(keyref.data, keyref.size); + keys[row] = key; + value_by_key[key] = get_default(row); + to_load[row] = row; + } + + auto stream = source_ptr->loadKeys(key_columns, to_load); + const auto attributes_size = attributes.size(); + + stream->readPrefix(); + + while (const auto block = stream->read()) + { + const auto key_columns = ext::map( + ext::range(0, keys_size), [&](const size_t attribute_idx) { return block.safeGetByPosition(attribute_idx).column; }); + + for (const size_t attribute_idx : ext::range(0, attributes.size())) + { + const IColumn & attribute_column = *block.safeGetByPosition(keys_size + attribute_idx).column; + Arena pool; + + const auto keys_size = dict_struct.key->size(); + StringRefs keys(keys_size); + + for (const auto row_idx : ext::range(0, attribute_column.size())) + { + const StringRef keyref = placeKeysInPool(row_idx, key_columns, keys, *dict_struct.key, pool); + std::string key(keyref.data, keyref.size); + + if (value_by_key.find(key) != value_by_key.end() && attribute.name == attribute_name_by_index.at(attribute_idx)) + { + if (attribute.type == AttributeUnderlyingType::utFloat32) + { + value_by_key[key] = static_cast(attribute_column[row_idx].get()); + } + else + { + value_by_key[key] = static_cast(attribute_column[row_idx].get()); + } + + } + } + } + } + + stream->readSuffix(); + + for (const auto row : ext::range(0, rows)) { + set_value(row, value_by_key[keys[row]]); + } + + query_count.fetch_add(rows, std::memory_order_relaxed); +} + +template +void ComplexKeyDirectDictionary::getItemsStringImpl( + const Attribute & attribute, const Columns & key_columns, ValueSetter && set_value, DefaultGetter && get_default) const +{ + const auto rows = key_columns.front()->size(); + const auto keys_size = dict_struct.key->size(); + StringRefs keys_array(rows); + std::unordered_map value_by_key; + Arena temporary_keys_pool; + std::vector to_load(rows); + std::vector keys(rows); + + for (const auto row : ext::range(0, rows)) { + const StringRef keyref = placeKeysInPool(row, key_columns, keys_array, *dict_struct.key, temporary_keys_pool); + std::string key(keyref.data, keyref.size); + keys[row] = key; + value_by_key[key] = get_default(row); + to_load[row] = row; + } + + auto stream = source_ptr->loadKeys(key_columns, to_load); + const auto attributes_size = attributes.size(); + + stream->readPrefix(); + + while (const auto block = stream->read()) + { + const auto key_columns = ext::map( + ext::range(0, keys_size), [&](const size_t attribute_idx) { return block.safeGetByPosition(attribute_idx).column; }); + + for (const size_t attribute_idx : ext::range(0, attributes.size())) + { + + const IColumn & attribute_column = *block.safeGetByPosition(keys_size + attribute_idx).column; + Arena pool; + + const auto keys_size = dict_struct.key->size(); + StringRefs keys(keys_size); + + for (const auto row_idx : ext::range(0, attribute_column.size())) + { + const StringRef keyref = placeKeysInPool(row_idx, key_columns, keys, *dict_struct.key, pool); + std::string key(keyref.data, keyref.size); + if (value_by_key.find(key) != value_by_key.end() && attribute.name == attribute_name_by_index.at(attribute_idx)) + { + const String from_source = attribute_column[row_idx].get(); + value_by_key[key] = from_source; + } + } + } + } + stream->readSuffix(); + + for (const auto row : ext::range(0, rows)) + set_value(row, value_by_key[keys[row]]); + + query_count.fetch_add(rows, std::memory_order_relaxed); +} + + +const ComplexKeyDirectDictionary::Attribute & ComplexKeyDirectDictionary::getAttribute(const std::string & attribute_name) const +{ + const auto it = attribute_index_by_name.find(attribute_name); + if (it == std::end(attribute_index_by_name)) + throw Exception{full_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS}; + + return attributes[it->second]; +} + + +template +void ComplexKeyDirectDictionary::has(const Attribute & attribute, const Columns & key_columns, PaddedPODArray & out) const +{ + const auto rows = key_columns.front()->size(); + const auto keys_size = dict_struct.key->size(); + StringRefs keys_array(rows); + std::unordered_map has_key; + Arena temporary_keys_pool; + std::vector to_load(rows); + std::vector keys(rows); + + for (const auto row : ext::range(0, rows)) { + const StringRef keyref = placeKeysInPool(row, key_columns, keys_array, *dict_struct.key, temporary_keys_pool); + std::string key(keyref.data, keyref.size); + keys[row] = key; + has_key[key] = 0; + to_load[row] = row; + } + + auto stream = source_ptr->loadKeys(key_columns, to_load); + const auto attributes_size = attributes.size(); + + stream->readPrefix(); + + while (const auto block = stream->read()) + { + const auto key_columns = ext::map( + ext::range(0, keys_size), [&](const size_t attribute_idx) { return block.safeGetByPosition(attribute_idx).column; }); + + for (const size_t attribute_idx : ext::range(0, attributes.size())) + { + + const IColumn & attribute_column = *block.safeGetByPosition(keys_size + attribute_idx).column; + Arena pool; + + const auto keys_size = dict_struct.key->size(); + StringRefs keys(keys_size); + + for (const auto row_idx : ext::range(0, attribute_column.size())) + { + const StringRef keyref = placeKeysInPool(row_idx, key_columns, keys, *dict_struct.key, pool); + std::string key(keyref.data, keyref.size); + if (has_key.find(key) != has_key.end() && attribute.name == attribute_name_by_index.at(attribute_idx)) + { + has_key[key] = 1; + } + } + } + } + stream->readSuffix(); + + for (const auto row : ext::range(0, rows)) + out[row] = has_key[keys[row]]; + + query_count.fetch_add(rows, std::memory_order_relaxed); +} + + +BlockInputStreamPtr ComplexKeyDirectDictionary::getBlockInputStream(const Names & /* column_names */, size_t /* max_block_size */) const +{ + return source_ptr->loadAll(); +} + + +void registerDictionaryComplexKeyDirect(DictionaryFactory & factory) +{ + auto create_layout = [=](const std::string & full_name, + const DictionaryStructure & dict_struct, + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + DictionarySourcePtr source_ptr) -> DictionaryPtr + { + if (!dict_struct.key) + throw Exception{"'key' is required for dictionary of layout 'complex_key_direct'", ErrorCodes::BAD_ARGUMENTS}; + + if (dict_struct.range_min || dict_struct.range_max) + throw Exception{full_name + + ": elements .structure.range_min and .structure.range_max should be defined only " + "for a dictionary of layout 'range_hashed'", + ErrorCodes::BAD_ARGUMENTS}; + + const String database = config.getString(config_prefix + ".database", ""); + const String name = config.getString(config_prefix + ".name"); + + if (config.has(config_prefix + ".lifetime.min") || config.has(config_prefix + ".lifetime.max")) + throw Exception{"'lifetime' parameter is redundant for the dictionary' of layout 'direct'", ErrorCodes::BAD_ARGUMENTS}; + + + return std::make_unique(database, name, dict_struct, std::move(source_ptr)); + }; + factory.registerLayout("complex_key_direct", create_layout, false); +} + + +} diff --git a/src/Dictionaries/ComplexKeyDirectDictionary.h b/src/Dictionaries/ComplexKeyDirectDictionary.h new file mode 100644 index 00000000000..ececd755473 --- /dev/null +++ b/src/Dictionaries/ComplexKeyDirectDictionary.h @@ -0,0 +1,224 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "DictionaryStructure.h" +#include "IDictionary.h" +#include "IDictionarySource.h" + + +namespace DB +{ +using BlockPtr = std::shared_ptr; + +class ComplexKeyDirectDictionary final : public IDictionaryBase +{ +public: + ComplexKeyDirectDictionary( + const std::string & database_, + const std::string & name_, + const DictionaryStructure & dict_struct_, + DictionarySourcePtr source_ptr_, + BlockPtr saved_block_ = nullptr); + + const std::string & getDatabase() const override { return database; } + const std::string & getName() const override { return name; } + const std::string & getFullName() const override { return full_name; } + + std::string getTypeName() const override { return "ComplexKeyDirect"; } + + size_t getBytesAllocated() const override { return 0; } + + size_t getQueryCount() const override { return query_count.load(std::memory_order_relaxed); } + + double getHitRate() const override { return 1.0; } + + size_t getElementCount() const override { return 0; } + + double getLoadFactor() const override { return 0; } + + std::string getKeyDescription() const { return key_description; } + + std::shared_ptr clone() const override + { + return std::make_shared(database, name, dict_struct, source_ptr->clone(), saved_block); + } + + const IDictionarySource * getSource() const override { return source_ptr.get(); } + + const DictionaryLifetime & getLifetime() const override { return dict_lifetime; } + + const DictionaryStructure & getStructure() const override { return dict_struct; } + + bool isInjective(const std::string & attribute_name) const override + { + return dict_struct.attributes[&getAttribute(attribute_name) - attributes.data()].injective; + } + + template + using ResultArrayType = std::conditional_t, DecimalPaddedPODArray, PaddedPODArray>; + +#define DECLARE(TYPE) \ + void get##TYPE(const std::string & attribute_name, const Columns & key_columns, const DataTypes & key_types, ResultArrayType & out) const; + DECLARE(UInt8) + DECLARE(UInt16) + DECLARE(UInt32) + DECLARE(UInt64) + DECLARE(UInt128) + DECLARE(Int8) + DECLARE(Int16) + DECLARE(Int32) + DECLARE(Int64) + DECLARE(Float32) + DECLARE(Float64) + DECLARE(Decimal32) + DECLARE(Decimal64) + DECLARE(Decimal128) +#undef DECLARE + + void getString( + const std::string & attribute_name, const Columns & key_columns, const DataTypes & key_types, ColumnString * out) const; + +#define DECLARE(TYPE) \ + void get##TYPE( \ + const std::string & attribute_name, \ + const Columns & key_columns, \ + const DataTypes & key_types, \ + const PaddedPODArray & def, \ + ResultArrayType & out) const; + DECLARE(UInt8) + DECLARE(UInt16) + DECLARE(UInt32) + DECLARE(UInt64) + DECLARE(UInt128) + DECLARE(Int8) + DECLARE(Int16) + DECLARE(Int32) + DECLARE(Int64) + DECLARE(Float32) + DECLARE(Float64) + DECLARE(Decimal32) + DECLARE(Decimal64) + DECLARE(Decimal128) +#undef DECLARE + + void getString( + const std::string & attribute_name, const Columns & key_columns, const DataTypes & key_types, const ColumnString * const def, ColumnString * const out) const; + +#define DECLARE(TYPE) \ + void get##TYPE(const std::string & attribute_name, const Columns & key_columns, const DataTypes & key_types, const TYPE def, ResultArrayType & out) const; + DECLARE(UInt8) + DECLARE(UInt16) + DECLARE(UInt32) + DECLARE(UInt64) + DECLARE(UInt128) + DECLARE(Int8) + DECLARE(Int16) + DECLARE(Int32) + DECLARE(Int64) + DECLARE(Float32) + DECLARE(Float64) + DECLARE(Decimal32) + DECLARE(Decimal64) + DECLARE(Decimal128) +#undef DECLARE + + void getString( + const std::string & attribute_name, const Columns & key_columns, const DataTypes & key_types, const String & def, ColumnString * const out) const; + + void has(const Columns & key_columns, const DataTypes & key_types, PaddedPODArray & out) const; + + BlockInputStreamPtr getBlockInputStream(const Names & column_names, size_t max_block_size) const override; + +private: + template + using MapType = HashMapWithSavedHash; + + struct Attribute final + { + AttributeUnderlyingType type; + std::variant< + UInt8, + UInt16, + UInt32, + UInt64, + UInt128, + Int8, + Int16, + Int32, + Int64, + Decimal32, + Decimal64, + Decimal128, + Float32, + Float64, + StringRef> + null_values; + std::unique_ptr string_arena; + std::string name; + }; + + void createAttributes(); + + template + void addAttributeSize(const Attribute & attribute); + + void calculateBytesAllocated(); + + template + void createAttributeImpl(Attribute & attribute, const Field & null_value); + + Attribute createAttributeWithType(const AttributeUnderlyingType type, const Field & null_value, const std::string & name); + + template + StringRef placeKeysInPool( + const size_t row, const Columns & key_columns, StringRefs & keys, const std::vector & key_attributes, Pool & pool) const; + + template + void getItemsStringImpl( + const Attribute & attribute, const Columns & key_columns, ValueSetter && set_value, DefaultGetter && get_default) const; + + template + void getItemsImpl( + const Attribute & attribute, const Columns & key_columns, ValueSetter && set_value, DefaultGetter && get_default) const; + + template + void resize(Attribute & attribute, const Key id); + + template + void setAttributeValueImpl(Attribute & attribute, const Key id, const T & value); + + void setAttributeValue(Attribute & attribute, const Key id, const Field & value); + + const Attribute & getAttribute(const std::string & attribute_name) const; + + template + void has(const Attribute & attribute, const Columns & key_columns, PaddedPODArray & out) const; + + const std::string database; + const std::string name; + const std::string full_name; + const DictionaryStructure dict_struct; + const DictionarySourcePtr source_ptr; + const DictionaryLifetime dict_lifetime; + + std::map attribute_index_by_name; + std::map attribute_name_by_index; + std::vector attributes; + + mutable std::atomic query_count{0}; + + BlockPtr saved_block; + const std::string key_description{dict_struct.getKeyDescription()}; +}; + +} \ No newline at end of file diff --git a/src/Dictionaries/getDictionaryConfigurationFromAST.cpp b/src/Dictionaries/getDictionaryConfigurationFromAST.cpp index eabf630ac1e..88832a5cc40 100644 --- a/src/Dictionaries/getDictionaryConfigurationFromAST.cpp +++ b/src/Dictionaries/getDictionaryConfigurationFromAST.cpp @@ -414,7 +414,8 @@ void checkAST(const ASTCreateQuery & query) if (query.dictionary->layout == nullptr) throw Exception("Cannot create dictionary with empty layout", ErrorCodes::INCORRECT_DICTIONARY_DEFINITION); - const auto is_direct_layout = !strcasecmp(query.dictionary->layout->layout_type.data(), "direct"); + const auto is_direct_layout = !strcasecmp(query.dictionary->layout->layout_type.data(), "direct") || + !strcasecmp(query.dictionary->layout->layout_type.data(), "complex_key_direct"); if (query.dictionary->lifetime == nullptr && !is_direct_layout) throw Exception("Cannot create dictionary with empty lifetime", ErrorCodes::INCORRECT_DICTIONARY_DEFINITION); diff --git a/src/Dictionaries/registerDictionaries.cpp b/src/Dictionaries/registerDictionaries.cpp index 299da521519..ad6adbc86fb 100644 --- a/src/Dictionaries/registerDictionaries.cpp +++ b/src/Dictionaries/registerDictionaries.cpp @@ -25,6 +25,7 @@ void registerDictionaries() registerDictionaryRangeHashed(factory); registerDictionaryComplexKeyHashed(factory); registerDictionaryComplexKeyCache(factory); + registerDictionaryComplexKeyDirect(factory); #if !defined(ARCADIA_BUILD) registerDictionaryTrie(factory); #endif diff --git a/src/Dictionaries/registerDictionaries.h b/src/Dictionaries/registerDictionaries.h index eda5ca074f4..a3a4a175d41 100644 --- a/src/Dictionaries/registerDictionaries.h +++ b/src/Dictionaries/registerDictionaries.h @@ -20,6 +20,7 @@ class DictionaryFactory; void registerDictionaryRangeHashed(DictionaryFactory & factory); void registerDictionaryComplexKeyHashed(DictionaryFactory & factory); void registerDictionaryComplexKeyCache(DictionaryFactory & factory); +void registerDictionaryComplexKeyDirect(DictionaryFactory & factory); void registerDictionaryTrie(DictionaryFactory & factory); void registerDictionaryFlat(DictionaryFactory & factory); void registerDictionaryHashed(DictionaryFactory & factory); diff --git a/src/Functions/FunctionsExternalDictionaries.h b/src/Functions/FunctionsExternalDictionaries.h index f9b326f1def..4f808c4766d 100644 --- a/src/Functions/FunctionsExternalDictionaries.h +++ b/src/Functions/FunctionsExternalDictionaries.h @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -136,6 +137,7 @@ private: !executeDispatchSimple(block, arguments, result, dict_ptr) && !executeDispatchComplex(block, arguments, result, dict_ptr) && !executeDispatchComplex(block, arguments, result, dict_ptr) && + !executeDispatchComplex(block, arguments, result, dict_ptr) && #if !defined(ARCADIA_BUILD) !executeDispatchComplex(block, arguments, result, dict_ptr) && #endif @@ -311,6 +313,7 @@ private: !executeDispatch(block, arguments, result, dict_ptr) && !executeDispatchComplex(block, arguments, result, dict_ptr) && !executeDispatchComplex(block, arguments, result, dict_ptr) && + !executeDispatchComplex(block, arguments, result, dict_ptr) && #if !defined(ARCADIA_BUILD) !executeDispatchComplex(block, arguments, result, dict_ptr) && #endif @@ -496,6 +499,7 @@ private: !executeDispatch(block, arguments, result, dict_ptr) && !executeDispatchComplex(block, arguments, result, dict_ptr) && !executeDispatchComplex(block, arguments, result, dict_ptr) && + !executeDispatchComplex(block, arguments, result, dict_ptr) && #if !defined(ARCADIA_BUILD) !executeDispatchComplex(block, arguments, result, dict_ptr) && #endif @@ -837,6 +841,7 @@ private: !executeDispatch(block, arguments, result, dict_ptr) && !executeDispatchComplex(block, arguments, result, dict_ptr) && !executeDispatchComplex(block, arguments, result, dict_ptr) && + !executeDispatchComplex(block, arguments, result, dict_ptr) && #if !defined(ARCADIA_BUILD) !executeDispatchComplex(block, arguments, result, dict_ptr) && #endif @@ -1100,6 +1105,7 @@ private: !executeDispatch(block, arguments, result, dict_ptr) && !executeDispatchComplex(block, arguments, result, dict_ptr) && !executeDispatchComplex(block, arguments, result, dict_ptr) && + !executeDispatchComplex(block, arguments, result, dict_ptr) && #if !defined(ARCADIA_BUILD) !executeDispatchComplex(block, arguments, result, dict_ptr) && #endif From ad2630613180db23f3b361491f7d45d6200ba78a Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sun, 10 May 2020 03:09:51 +0300 Subject: [PATCH 045/738] Fix clang-tidy. --- src/Functions/pointInPolygon.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Functions/pointInPolygon.cpp b/src/Functions/pointInPolygon.cpp index b46b385f738..dcd94af03ee 100644 --- a/src/Functions/pointInPolygon.cpp +++ b/src/Functions/pointInPolygon.cpp @@ -141,12 +141,12 @@ public: if (arguments.size() == 2) { - auto * array = checkAndGetDataType(arguments[1].get()); + const auto * array = checkAndGetDataType(arguments[1].get()); if (array == nullptr) throw Exception(getMessagePrefix(1) + " must contain an array of tuples or an array of arrays of tuples.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - auto * nested_array = checkAndGetDataType(array->getNestedType().get()); + const auto * nested_array = checkAndGetDataType(array->getNestedType().get()); if (nested_array != nullptr) { array = nested_array; @@ -158,7 +158,7 @@ public: { for (size_t i = 1; i < arguments.size(); i++) { - auto * array = checkAndGetDataType(arguments[i].get()); + const auto * array = checkAndGetDataType(arguments[i].get()); if (array == nullptr) throw Exception(getMessagePrefix(i) + " must contain an array of tuples", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); From a961e049a2a5700268e600408ab7df47ebe2ace8 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 10 May 2020 09:26:33 +0300 Subject: [PATCH 046/738] VolumeType as class with enum member class --- src/Common/ErrorCodes.cpp | 1 + src/Disks/IVolume.cpp | 24 ++++++++++++++++++++++++ src/Disks/IVolume.h | 30 ++++++++++++++++++++++++++---- 3 files changed, 51 insertions(+), 4 deletions(-) diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index dbb539ef2b3..9efacdb7b90 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -494,6 +494,7 @@ namespace ErrorCodes extern const int CANNOT_DETACH_DICTIONARY_AS_TABLE = 520; extern const int ATOMIC_RENAME_FAIL = 521; extern const int INCORRECT_DISK_INDEX = 522; + extern const int UNKNOWN_VOLUME_TYPE = 523; extern const int KEEPER_EXCEPTION = 999; extern const int POCO_EXCEPTION = 1000; diff --git a/src/Disks/IVolume.cpp b/src/Disks/IVolume.cpp index 6a122a3e3b2..4da9aec1ab9 100644 --- a/src/Disks/IVolume.cpp +++ b/src/Disks/IVolume.cpp @@ -10,6 +10,30 @@ namespace DB namespace ErrorCodes { extern const int EXCESSIVE_ELEMENT_IN_CONFIG; + extern const int UNKNOWN_VOLUME_TYPE; +} + +void VolumeType::fromString(const String & str) +{ + if (str == "JBOD") + value = JBOD; + else if (str == "SINGLE_DISK") + value = SINGLE_DISK; + else + throw DB::Exception("Unexpected string for volume type: " + str, ErrorCodes::UNKNOWN_VOLUME_TYPE); +} + +String VolumeType::toString() const +{ + switch (value) + { + case JBOD: + return "JBOD"; + case SINGLE_DISK: + return "SINGLE_DISK"; + default: + return "Unknown"; + } } IVolume::IVolume( diff --git a/src/Disks/IVolume.h b/src/Disks/IVolume.h index 8365102196d..0b1c0a20ad9 100644 --- a/src/Disks/IVolume.h +++ b/src/Disks/IVolume.h @@ -8,11 +8,33 @@ namespace DB { -enum VolumeType +class VolumeType { - JBOD, - SINGLE_DISK, - UNKNOWN +public: + enum Value + { + JBOD, + SINGLE_DISK, + UNKNOWN + }; + VolumeType() : value(UNKNOWN) {} + VolumeType(Value value_) : value(value_) {} + + bool operator==(const VolumeType & other) const + { + return value == other.value; + } + + bool operator!=(const VolumeType & other) const + { + return !(*this == other); + } + + void fromString(const String & str); + String toString() const; + +private: + Value value; }; class IVolume; From fbd00d6af33090166b6932809aa2b6e61f0a50ea Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 10 May 2020 12:33:52 +0300 Subject: [PATCH 047/738] removed default value from overriden functions in IReservation children --- src/Disks/DiskLocal.cpp | 2 +- src/Disks/S3/DiskS3.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Disks/DiskLocal.cpp b/src/Disks/DiskLocal.cpp index a5eee3eada3..7e089ba8b51 100644 --- a/src/Disks/DiskLocal.cpp +++ b/src/Disks/DiskLocal.cpp @@ -36,7 +36,7 @@ public: UInt64 getSize() const override { return size; } - DiskPtr getDisk(size_t i = 0) const override; + DiskPtr getDisk(size_t i) const override; Disks getDisks() const override { return {disk}; } diff --git a/src/Disks/S3/DiskS3.cpp b/src/Disks/S3/DiskS3.cpp index d53c9f172e2..8531a58d2ed 100644 --- a/src/Disks/S3/DiskS3.cpp +++ b/src/Disks/S3/DiskS3.cpp @@ -370,7 +370,7 @@ public: UInt64 getSize() const override { return size; } - DiskPtr getDisk(size_t i = 0) const override + DiskPtr getDisk(size_t i) const override { if (i != 0) { From 1235c36f3e9120107658cda60e465d648e9fdeb5 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 10 May 2020 16:33:27 +0300 Subject: [PATCH 048/738] data_part instead of volume in IMergeTreeDataPartWriter (as it done in IMergeTreeReader) --- .../MergeTree/IMergeTreeDataPartWriter.cpp | 18 ++++++++---------- .../MergeTree/IMergeTreeDataPartWriter.h | 6 ++---- .../MergeTree/MergeTreeDataPartCompact.cpp | 2 +- .../MergeTree/MergeTreeDataPartWide.cpp | 2 +- .../MergeTreeDataPartWriterCompact.cpp | 13 +++++-------- .../MergeTree/MergeTreeDataPartWriterCompact.h | 4 +--- .../MergeTree/MergeTreeDataPartWriterWide.cpp | 12 +++++------- .../MergeTree/MergeTreeDataPartWriterWide.h | 4 +--- 8 files changed, 24 insertions(+), 37 deletions(-) diff --git a/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp b/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp index bb19a80a28a..239ccba1d89 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp @@ -63,18 +63,16 @@ void IMergeTreeDataPartWriter::Stream::addToChecksums(MergeTreeData::DataPart::C IMergeTreeDataPartWriter::IMergeTreeDataPartWriter( - const VolumePtr & volume_, - const String & part_path_, - const MergeTreeData & storage_, + const MergeTreeData::DataPartPtr & data_part_, const NamesAndTypesList & columns_list_, const std::vector & indices_to_recalc_, const String & marks_file_extension_, const CompressionCodecPtr & default_codec_, const MergeTreeWriterSettings & settings_, const MergeTreeIndexGranularity & index_granularity_) - : volume(volume_) - , part_path(part_path_) - , storage(storage_) + : data_part(data_part_) + , part_path(data_part_->getFullRelativePath()) + , storage(data_part_->storage) , columns_list(columns_list_) , marks_file_extension(marks_file_extension_) , index_granularity(index_granularity_) @@ -87,8 +85,8 @@ IMergeTreeDataPartWriter::IMergeTreeDataPartWriter( if (settings.blocks_are_granules_size && !index_granularity.empty()) throw Exception("Can't take information about index granularity from blocks, when non empty index_granularity array specified", ErrorCodes::LOGICAL_ERROR); - if (!volume->getDisk()->exists(part_path)) - volume->getDisk()->createDirectories(part_path); + if (!data_part->volume->getDisk()->exists(part_path)) + data_part->volume->getDisk()->createDirectories(part_path); } IMergeTreeDataPartWriter::~IMergeTreeDataPartWriter() = default; @@ -165,7 +163,7 @@ void IMergeTreeDataPartWriter::initPrimaryIndex() { if (storage.hasPrimaryKey()) { - index_file_stream = volume->getDisk()->writeFile(part_path + "primary.idx", DBMS_DEFAULT_BUFFER_SIZE, WriteMode::Rewrite); + index_file_stream = data_part->volume->getDisk()->writeFile(part_path + "primary.idx", DBMS_DEFAULT_BUFFER_SIZE, WriteMode::Rewrite); index_stream = std::make_unique(*index_file_stream); } @@ -180,7 +178,7 @@ void IMergeTreeDataPartWriter::initSkipIndices() skip_indices_streams.emplace_back( std::make_unique( stream_name, - volume->getDisk(), + data_part->volume->getDisk(), part_path + stream_name, INDEX_FILE_EXTENSION, part_path + stream_name, marks_file_extension, default_codec, settings.max_compress_block_size, diff --git a/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index a4460ab861f..ffdba570544 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -61,9 +61,7 @@ public: using StreamPtr = std::unique_ptr; IMergeTreeDataPartWriter( - const VolumePtr & volume, - const String & part_path, - const MergeTreeData & storage, + const MergeTreeData::DataPartPtr & data_part, const NamesAndTypesList & columns_list, const std::vector & indices_to_recalc, const String & marks_file_extension, @@ -118,7 +116,7 @@ protected: using SerializationState = IDataType::SerializeBinaryBulkStatePtr; using SerializationStates = std::unordered_map; - VolumePtr volume; + MergeTreeData::DataPartPtr data_part; String part_path; const MergeTreeData & storage; NamesAndTypesList columns_list; diff --git a/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index e417adf4eed..32acc266e42 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -68,7 +68,7 @@ IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartCompact::getWriter( { return *getColumnPosition(lhs.name) < *getColumnPosition(rhs.name); }); return std::make_unique( - volume, getFullRelativePath(), storage, ordered_columns_list, indices_to_recalc, + shared_from_this(), ordered_columns_list, indices_to_recalc, index_granularity_info.marks_file_extension, default_codec, writer_settings, computed_index_granularity); } diff --git a/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index 85903d0161e..d96b61b4bea 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -59,7 +59,7 @@ IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartWide::getWriter( const MergeTreeIndexGranularity & computed_index_granularity) const { return std::make_unique( - volume, getFullRelativePath(), storage, columns_list, indices_to_recalc, + shared_from_this(), columns_list, indices_to_recalc, index_granularity_info.marks_file_extension, default_codec, writer_settings, computed_index_granularity); } diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index f107f18792e..1a7a757c149 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -6,26 +6,23 @@ namespace DB MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact( - const VolumePtr & volume_, - const String & part_path_, - const MergeTreeData & storage_, + const MergeTreeData::DataPartPtr & data_part_, const NamesAndTypesList & columns_list_, const std::vector & indices_to_recalc_, const String & marks_file_extension_, const CompressionCodecPtr & default_codec_, const MergeTreeWriterSettings & settings_, const MergeTreeIndexGranularity & index_granularity_) -: IMergeTreeDataPartWriter(volume_, part_path_, - storage_, columns_list_, - indices_to_recalc_, marks_file_extension_, - default_codec_, settings_, index_granularity_) + : IMergeTreeDataPartWriter(data_part_, columns_list_, + indices_to_recalc_, marks_file_extension_, + default_codec_, settings_, index_granularity_) { using DataPart = MergeTreeDataPartCompact; String data_file_name = DataPart::DATA_FILE_NAME; stream = std::make_unique( data_file_name, - volume->getDisk(), + data_part->volume->getDisk(), part_path + data_file_name, DataPart::DATA_FILE_EXTENSION, part_path + data_file_name, marks_file_extension, default_codec, diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h index f786a09d93e..07caba94712 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h @@ -8,9 +8,7 @@ class MergeTreeDataPartWriterCompact : public IMergeTreeDataPartWriter { public: MergeTreeDataPartWriterCompact( - const VolumePtr & volume, - const String & part_path, - const MergeTreeData & storage, + const MergeTreeData::DataPartPtr & data_part, const NamesAndTypesList & columns_list, const std::vector & indices_to_recalc, const String & marks_file_extension, diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index 1b0e4441b9f..4cdf57a4700 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -13,18 +13,16 @@ namespace } MergeTreeDataPartWriterWide::MergeTreeDataPartWriterWide( - const VolumePtr & volume_, - const String & part_path_, - const MergeTreeData & storage_, + const MergeTreeData::DataPartPtr & data_part_, const NamesAndTypesList & columns_list_, const std::vector & indices_to_recalc_, const String & marks_file_extension_, const CompressionCodecPtr & default_codec_, const MergeTreeWriterSettings & settings_, const MergeTreeIndexGranularity & index_granularity_) - : IMergeTreeDataPartWriter(volume_, part_path_, - storage_, columns_list_, indices_to_recalc_, - marks_file_extension_, default_codec_, settings_, index_granularity_) + : IMergeTreeDataPartWriter(data_part_, columns_list_, + indices_to_recalc_, marks_file_extension_, + default_codec_, settings_, index_granularity_) { const auto & columns = storage.getColumns(); for (const auto & it : columns_list) @@ -46,7 +44,7 @@ void MergeTreeDataPartWriterWide::addStreams( column_streams[stream_name] = std::make_unique( stream_name, - volume->getDisk(), + data_part->volume->getDisk(), part_path + stream_name, DATA_FILE_EXTENSION, part_path + stream_name, marks_file_extension, effective_codec, diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h index 42357af15a8..acd7f749d00 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h @@ -11,9 +11,7 @@ public: using ColumnToSize = std::map; MergeTreeDataPartWriterWide( - const VolumePtr & volume, - const String & part_path, - const MergeTreeData & storage, + const MergeTreeData::DataPartPtr & data_part, const NamesAndTypesList & columns_list, const std::vector & indices_to_recalc, const String & marks_file_extension, From 330f06328f729a7fc346305bb713cc8fbd2fe55d Mon Sep 17 00:00:00 2001 From: Andrew Onyshchuk Date: Sun, 10 May 2020 18:17:54 -0500 Subject: [PATCH 049/738] Use src_type for convertion in KeyCondition --- src/Storages/MergeTree/KeyCondition.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index 24f460cab64..6ae22885dfd 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -710,8 +710,7 @@ static void castValueToType(const DataTypePtr & desired_type, Field & src_value, try { - /// NOTE: We don't need accurate info about src_type at this moment - src_value = convertFieldToType(src_value, *desired_type); + src_value = convertFieldToType(src_value, *desired_type, src_type.get()); } catch (...) { From da004fba02bee150e87ff03c097c89825e1f14bd Mon Sep 17 00:00:00 2001 From: mnkonkova Date: Mon, 11 May 2020 01:38:46 +0000 Subject: [PATCH 050/738] grpc protobuf --- contrib/grpc-cmake/CMakeLists.txt | 43 ++++++++++++++----------------- 1 file changed, 19 insertions(+), 24 deletions(-) diff --git a/contrib/grpc-cmake/CMakeLists.txt b/contrib/grpc-cmake/CMakeLists.txt index a4b5c80a75a..1d0cf235b48 100644 --- a/contrib/grpc-cmake/CMakeLists.txt +++ b/contrib/grpc-cmake/CMakeLists.txt @@ -3,7 +3,6 @@ cmake_minimum_required(VERSION 3.5.1) set(GRPC_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/grpc) set(GRPC_INCLUDE_DIR ${GRPC_SOURCE_DIR}/include/) set(GRPC_BINARY_DIR ${ClickHouse_BINARY_DIR}/contrib/grpc) - if(UNIX) if(${CMAKE_SYSTEM_NAME} MATCHES "Linux") set(_gRPC_PLATFORM_LINUX ON) @@ -56,29 +55,19 @@ endif() # protobuf.cmake set(PROTOBUF_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../protobuf) -# if(NOT protobuf_BUILD_TESTS) - # set(protobuf_BUILD_TESTS OFF CACHE BOOL "Build protobuf tests") -# endif() + set(protobuf_BUILD_TESTS OFF CACHE BOOL "Build protobuf tests") if(NOT protobuf_WITH_ZLIB) set(protobuf_WITH_ZLIB OFF CACHE BOOL "Build protobuf with zlib.") endif() set(protobuf_MSVC_STATIC_RUNTIME OFF CACHE BOOL "Link static runtime libraries") -if(NOT USE_INTERNAL_PROTOBUF_LIBRARY) - add_subdirectory(${PROTOBUF_ROOT_DIR}/cmake ${GRPC_BINARY_DIR}/third_party/protobuf) -endif() -if(TARGET ${_gRPC_PROTOBUF_LIBRARY_NAME}) - set(_gRPC_PROTOBUF_LIBRARIES ${_gRPC_PROTOBUF_LIBRARY_NAME}) -endif() -if(TARGET libprotoc) - set(_gRPC_PROTOBUF_PROTOC_LIBRARIES libprotoc) -endif() -if(TARGET protoc) - set(_gRPC_PROTOBUF_PROTOC protoc) - set(_gRPC_PROTOBUF_PROTOC_EXECUTABLE $) -endif() -# For well-known .proto files distributed with protobuf -set(_gRPC_PROTOBUF_WELLKNOWN_INCLUDE_DIR "${PROTOBUF_ROOT_DIR}/src") + +set(_gRPC_PROTOBUF_LIBRARIES libprotobuf) +set(_gRPC_PROTOBUF_PROTOC_LIBRARIES libprotoc) +set(_gRPC_PROTOBUF_PROTOC protoc) +set(_gRPC_PROTOBUF_PROTOC_EXECUTABLE $) +set(_gRPC_PROTOBUF_INCLUDE_DIR "${PROTOBUF_ROOT_DIR}/src") + if(gRPC_INSTALL) message(WARNING "gRPC_INSTALL will be forced to FALSE because gRPC_PROTOBUF_PROVIDER is \"module\"") set(gRPC_INSTALL FALSE) @@ -98,12 +87,13 @@ set(_gRPC_UPB_GRPC_GENERATED_DIR "${GRPC_SOURCE_DIR}/src/core/ext/upb-generated" set(_gRPC_UPB_LIBRARIES upb) # zlib.cmake -set(ZLIB_ROOT_DIR ${GRPC_SOURCE_DIR}/third_party/zlib-ng) +set(ZLIB_ROOT_DIR ${GRPC_SOURCE_DIR}/../zlib-ng) include_directories("${ZLIB_ROOT_DIR}") -# add_subdirectory(${ZLIB_ROOT_DIR} ${ZLIB_ROOT_DIR}) -set(_gRPC_ZLIB_LIBRARIES zlibstatic) -set(_gRPC_ZLIB_INCLUDE_DIR "${ZLIB_ROOT_DIR}") - +## add_subdirectory(${ZLIB_ROOT_DIR} ${ZLIB_ROOT_DIR}) +if(TARGET zlibstatic) + set(_gRPC_ZLIB_LIBRARIES zlibstatic) + set(_gRPC_ZLIB_INCLUDE_DIR "${ZLIB_ROOT_DIR}" "${GRPC_SOURCE_DIR}/third_party/zlib") +endif() set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") @@ -134,6 +124,7 @@ target_include_directories(address_sorting ) target_link_libraries(address_sorting ${_gRPC_BASELIB_LIBRARIES} + ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} ) @@ -194,6 +185,7 @@ target_include_directories(gpr ) target_link_libraries(gpr ${_gRPC_ALLTARGETS_LIBRARIES} + ${_gRPC_PROTOBUF_LIBRARIES} ) add_library(grpc @@ -597,6 +589,7 @@ target_link_libraries(grpc ${_gRPC_CARES_LIBRARIES} ${_gRPC_ADDRESS_SORTING_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} + ${_gRPC_PROTOBUF_LIBRARIES} gpr ) if (_gRPC_PLATFORM_MAC) @@ -943,6 +936,7 @@ target_link_libraries(grpc_cronet ${_gRPC_CARES_LIBRARIES} ${_gRPC_ADDRESS_SORTING_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} + ${_gRPC_PROTOBUF_LIBRARIES} gpr ) if (_gRPC_PLATFORM_MAC) @@ -1273,6 +1267,7 @@ target_link_libraries(grpc_unsecure ${_gRPC_CARES_LIBRARIES} ${_gRPC_ADDRESS_SORTING_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} + ${_gRPC_PROTOBUF_LIBRARIES} gpr ) if (_gRPC_PLATFORM_MAC) From f6fe196440c56a0867d0e492f343df154fa9eb9f Mon Sep 17 00:00:00 2001 From: Guillaume Tassery Date: Mon, 11 May 2020 09:44:43 +0700 Subject: [PATCH 051/738] first draft fixed --- src/Interpreters/InterpreterSelectQuery.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 1325d013632..66361ad6a26 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -687,7 +687,7 @@ static std::pair getLimitLengthAndOffset(const ASTSelectQuery & static UInt64 getLimitForSorting(const ASTSelectQuery & query, const Context & context) { /// Partial sort can be done if there is LIMIT but no DISTINCT or LIMIT BY, neither ARRAY JOIN. - if (!query.distinct && !query.limitBy() && !query.limit_with_ties && !query.arrayJoinExpressionList()) + if (!query.distinct && !query.limitBy() && !query.limit_with_ties && !query.arrayJoinExpressionList() && query.limitLength()) { auto [limit_length, limit_offset] = getLimitLengthAndOffset(query, context); return limit_length + limit_offset; @@ -2436,7 +2436,7 @@ void InterpreterSelectQuery::executeLimit(Pipeline & pipeline) } -void InterpreterSelectQuery::executeOffset(Pipeline & pipeline) +void InterpreterSelectQuery::executeOffset(Pipeline & /*pipeline*/) { auto & query = getSelectQuery(); /// If there is LIMIT @@ -2451,6 +2451,7 @@ void InterpreterSelectQuery::executeOffset(Pipeline & pipeline) * if there is WITH TOTALS and there is no ORDER BY, then read the data to the end, * otherwise TOTALS is counted according to incomplete data. */ + /* bool always_read_till_end = false; if (query.group_by_with_totals && !query.orderBy()) @@ -2476,6 +2477,7 @@ void InterpreterSelectQuery::executeOffset(Pipeline & pipeline) std::cout << "BLOCK" << std::endl; stream = std::make_shared(stream, limit_offset, always_read_till_end, false, query.limit_with_ties, order_descr); }); + */ } } From d2c813fa7a7e5a8e3254e0c6bd17a2995e410775 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 11 May 2020 06:23:58 +0300 Subject: [PATCH 052/738] Update 01269_alias_type_differs.sql --- tests/queries/0_stateless/01269_alias_type_differs.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/01269_alias_type_differs.sql b/tests/queries/0_stateless/01269_alias_type_differs.sql index 3b99e5e7eec..b78e46f62c8 100644 --- a/tests/queries/0_stateless/01269_alias_type_differs.sql +++ b/tests/queries/0_stateless/01269_alias_type_differs.sql @@ -18,3 +18,5 @@ ALTER TABLE data_01269 DROP COLUMN alias; ALTER TABLE data_01269 ADD COLUMN alias UInt8 ALIAS value>0; SELECT toTypeName(alias) FROM data_01269; SELECT any(alias) FROM data_01269; + +DROP TABLE data_01269; From 802ae8948a450a8481cf6894c80c404435dadd76 Mon Sep 17 00:00:00 2001 From: Guillaume Tassery Date: Mon, 11 May 2020 10:36:52 +0700 Subject: [PATCH 053/738] cosmetic --- src/DataStreams/OffsetBlockInputStream.cpp | 50 --------------------- src/DataStreams/OffsetBlockInputStream.h | 47 ------------------- src/Interpreters/InterpreterSelectQuery.cpp | 47 +------------------ src/Processors/OffsetTransform.cpp | 36 --------------- src/Processors/OffsetTransform.h | 4 -- 5 files changed, 1 insertion(+), 183 deletions(-) delete mode 100644 src/DataStreams/OffsetBlockInputStream.cpp delete mode 100644 src/DataStreams/OffsetBlockInputStream.h diff --git a/src/DataStreams/OffsetBlockInputStream.cpp b/src/DataStreams/OffsetBlockInputStream.cpp deleted file mode 100644 index 275ad6a079a..00000000000 --- a/src/DataStreams/OffsetBlockInputStream.cpp +++ /dev/null @@ -1,50 +0,0 @@ -#include - -#include - - -namespace DB -{ - -OffsetBlockInputStream::OffsetBlockInputStream( - const BlockInputStreamPtr & input, UInt64 offset_, bool always_read_till_end_, - bool use_limit_as_total_rows_approx, bool with_ties_, const SortDescription & description_) - : offset(offset_), always_read_till_end(always_read_till_end_), with_ties(with_ties_) - , description(description_) -{ - if (use_limit_as_total_rows_approx) - { - addTotalRowsApprox(static_cast(limit)); - } - - children.push_back(input); -} - -Block OffsetBlockInputStream::readImpl() -{ - Block res; - UInt64 rows = 0; - - do - { - res = children.back()->read(); - if (!res) - return res; - rows = res.rows(); - pos += rows; - } while (pos <= offset); - - SharedBlockPtr ptr = new detail::SharedBlock(std::move(res)); - - /// give away a piece of the block - UInt64 start = std::max( - static_cast(0), - static_cast(offset) - static_cast(pos) + static_cast(rows)); - - for (size_t i = 0; i < ptr->columns(); ++i) - ptr->safeGetByPosition(i).column = ptr->safeGetByPosition(i).column->cut(start, rows); - - return *ptr; -} - -} diff --git a/src/DataStreams/OffsetBlockInputStream.h b/src/DataStreams/OffsetBlockInputStream.h deleted file mode 100644 index e88d161f80a..00000000000 --- a/src/DataStreams/OffsetBlockInputStream.h +++ /dev/null @@ -1,47 +0,0 @@ -#pragma once - -#include -#include -#include - - -namespace DB -{ - - -/** Implements the LIMIT relational operation. - */ -class OffsetBlockInputStream : public IBlockInputStream -{ -public: - /** If always_read_till_end = false (by default), then after reading enough data, - * returns an empty block, and this causes the query to be canceled. - * If always_read_till_end = true - reads all the data to the end, but ignores them. This is necessary in rare cases: - * when otherwise, due to the cancellation of the request, we would not have received the data for GROUP BY WITH TOTALS from the remote server. - * If use_limit_as_total_rows_approx = true, then addTotalRowsApprox is called to use the limit in progress & stats - * with_ties = true, when query has WITH TIES modifier. If so, description should be provided - * description lets us know which row we should check for equality - */ - OffsetBlockInputStream( - const BlockInputStreamPtr & input, UInt64 offset_, - bool always_read_till_end_ = false, bool use_limit_as_total_rows_approx = false, - bool with_ties_ = false, const SortDescription & description_ = {}); - - String getName() const override { return "Offset"; } - - Block getHeader() const override { return children.at(0)->getHeader(); } - -protected: - Block readImpl() override; - -private: - UInt64 limit; - UInt64 offset; - UInt64 pos = 0; - bool always_read_till_end; - bool with_ties; - const SortDescription description; - SharedBlockRowRef ties_row_ref; -}; - -} diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 66361ad6a26..d90556783f4 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -2,7 +2,6 @@ #include #include #include -#include #include #include #include @@ -2435,51 +2434,7 @@ void InterpreterSelectQuery::executeLimit(Pipeline & pipeline) } } - -void InterpreterSelectQuery::executeOffset(Pipeline & /*pipeline*/) -{ - auto & query = getSelectQuery(); - /// If there is LIMIT - if (!query.limitLength() && query.limitOffset()) - { - /** Rare case: - * if there is no WITH TOTALS and there is a subquery in FROM, and there is WITH TOTALS on one of the levels, - * then when using LIMIT, you should read the data to the end, rather than cancel the query earlier, - * because if you cancel the query, we will not get `totals` data from the remote server. - * - * Another case: - * if there is WITH TOTALS and there is no ORDER BY, then read the data to the end, - * otherwise TOTALS is counted according to incomplete data. - */ - /* - bool always_read_till_end = false; - - if (query.group_by_with_totals && !query.orderBy()) - always_read_till_end = true; - - if (!query.group_by_with_totals && hasWithTotalsInAnySubqueryInFromClause(query)) - always_read_till_end = true; - - SortDescription order_descr; - if (query.limit_with_ties) - { - if (!query.orderBy()) - throw Exception("LIMIT WITH TIES without ORDER BY", ErrorCodes::LOGICAL_ERROR); - order_descr = getSortDescription(query, *context); - } - - UInt64 limit_length; - UInt64 limit_offset; - std::tie(limit_length, limit_offset) = getLimitLengthAndOffset(query, *context); - - pipeline.transform([&](auto & stream) - { - std::cout << "BLOCK" << std::endl; - stream = std::make_shared(stream, limit_offset, always_read_till_end, false, query.limit_with_ties, order_descr); - }); - */ - } -} +void InterpreterSelectQuery::executeOffset(Pipeline & /* pipeline */) {} void InterpreterSelectQuery::executeWithFill(Pipeline & pipeline) diff --git a/src/Processors/OffsetTransform.cpp b/src/Processors/OffsetTransform.cpp index dda36aa8323..8c7cbe39e84 100644 --- a/src/Processors/OffsetTransform.cpp +++ b/src/Processors/OffsetTransform.cpp @@ -46,19 +46,6 @@ OffsetTransform::OffsetTransform( } } -Chunk OffsetTransform::makeChunkWithPreviousRow(const Chunk & chunk, size_t row) const -{ - assert(row < chunk.getNumRows()); - ColumnRawPtrs current_columns = extractSortColumns(chunk.getColumns()); - MutableColumns last_row_sort_columns; - for (size_t i = 0; i < current_columns.size(); ++i) - { - last_row_sort_columns.emplace_back(current_columns[i]->cloneEmpty()); - last_row_sort_columns[i]->insertFrom(*current_columns[i], row); - } - return Chunk(std::move(last_row_sort_columns), 1); -} - IProcessor::Status OffsetTransform::prepare( const PortNumbers & updated_input_ports, @@ -190,7 +177,6 @@ OffsetTransform::Status OffsetTransform::preparePair(PortsData & data) void OffsetTransform::splitChunk(PortsData & data) { - auto current_chunk_sort_columns = extractSortColumns(data.current_chunk.getColumns()); size_t num_rows = data.current_chunk.getNumRows(); size_t num_columns = data.current_chunk.getNumColumns(); @@ -212,27 +198,5 @@ void OffsetTransform::splitChunk(PortsData & data) data.current_chunk.setColumns(std::move(columns), length); } - -ColumnRawPtrs OffsetTransform::extractSortColumns(const Columns & columns) const -{ - ColumnRawPtrs res; - res.reserve(description.size()); - for (size_t pos : sort_column_positions) - res.push_back(columns[pos].get()); - - return res; -} - -bool OffsetTransform::sortColumnsEqualAt(const ColumnRawPtrs & current_chunk_sort_columns, size_t current_chunk_row_num) const -{ - assert(current_chunk_sort_columns.size() == previous_row_chunk.getNumColumns()); - size_t size = current_chunk_sort_columns.size(); - const auto & previous_row_sort_columns = previous_row_chunk.getColumns(); - for (size_t i = 0; i < size; ++i) - if (0 != current_chunk_sort_columns[i]->compareAt(current_chunk_row_num, 0, *previous_row_sort_columns[i], 1)) - return false; - return true; -} - } diff --git a/src/Processors/OffsetTransform.h b/src/Processors/OffsetTransform.h index 66361a53684..6c62080bec7 100644 --- a/src/Processors/OffsetTransform.h +++ b/src/Processors/OffsetTransform.h @@ -45,10 +45,6 @@ private: std::vector ports_data; size_t num_finished_port_pairs = 0; - Chunk makeChunkWithPreviousRow(const Chunk & current_chunk, size_t row_num) const; - ColumnRawPtrs extractSortColumns(const Columns & columns) const; - bool sortColumnsEqualAt(const ColumnRawPtrs & current_chunk_sort_columns, size_t current_chunk_row_num) const; - public: OffsetTransform( const Block & header_, size_t offset_, size_t num_streams = 1, From 170a341c1f7000ebe09e74c5803ba1e973f282df Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 11 May 2020 06:40:07 +0300 Subject: [PATCH 054/738] Trigger CI --- .../01247_dist_on_dist_group_by_sharding_key_optimization.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01247_dist_on_dist_group_by_sharding_key_optimization.sql b/tests/queries/0_stateless/01247_dist_on_dist_group_by_sharding_key_optimization.sql index dc2941e63fd..7cf171c8c73 100644 --- a/tests/queries/0_stateless/01247_dist_on_dist_group_by_sharding_key_optimization.sql +++ b/tests/queries/0_stateless/01247_dist_on_dist_group_by_sharding_key_optimization.sql @@ -32,5 +32,5 @@ select 'Distributed(rand)-over-Distributed(rand)'; create table dist_layer_01247 as data_01247 engine=Distributed(test_cluster_two_shards, currentDatabase(), data_01247, rand()); create table dist_01247 as data_01247 engine=Distributed(test_cluster_two_shards, currentDatabase(), dist_layer_01247, number); select count(), * from dist_01247 group by number; -drop table if exists dist_01247; -drop table if exists dist_layer_01247; +drop table dist_01247; +drop table dist_layer_01247; From 1ee875a5fcc25d0cd1635106d96bf40b6c265433 Mon Sep 17 00:00:00 2001 From: Andrew Onyshchuk Date: Sun, 10 May 2020 23:15:59 -0500 Subject: [PATCH 055/738] Add test --- ...0155_date_datetime_key_condition.reference | 10 +++++++++ .../00155_date_datetime_key_condition.sql | 22 +++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 tests/queries/1_stateful/00155_date_datetime_key_condition.reference create mode 100644 tests/queries/1_stateful/00155_date_datetime_key_condition.sql diff --git a/tests/queries/1_stateful/00155_date_datetime_key_condition.reference b/tests/queries/1_stateful/00155_date_datetime_key_condition.reference new file mode 100644 index 00000000000..01fa724e3e9 --- /dev/null +++ b/tests/queries/1_stateful/00155_date_datetime_key_condition.reference @@ -0,0 +1,10 @@ +['2020-01-01 10:00:00'] +['2020-01-02 00:00:00'] +['2020-01-01 00:00:00','2020-01-01 10:00:00'] +['2020-01-01 00:00:00','2020-01-01 10:00:00','2020-01-02 00:00:00'] +['2020-01-01 00:00:00','2020-01-01 10:00:00','2020-01-02 00:00:00'] +['2020-01-01 00:00:00','2020-01-01 10:00:00','2020-01-02 00:00:00'] +['2020-01-01 00:00:00','2020-01-01 10:00:00','2020-01-02 00:00:00'] +[] +[] +[] diff --git a/tests/queries/1_stateful/00155_date_datetime_key_condition.sql b/tests/queries/1_stateful/00155_date_datetime_key_condition.sql new file mode 100644 index 00000000000..cee1c7cfb46 --- /dev/null +++ b/tests/queries/1_stateful/00155_date_datetime_key_condition.sql @@ -0,0 +1,22 @@ +DROP TABLE IF EXISTS test.date_datetime_key_condition; + +CREATE TABLE test.date_datetime_key_condition (dt DateTime) ENGINE = MergeTree() ORDER BY dt; +INSERT INTO test.date_datetime_key_condition VALUES ('2020-01-01 00:00:00'), ('2020-01-01 10:00:00'), ('2020-01-02 00:00:00'); + +-- partial +SELECT groupArray(dt) from test.date_datetime_key_condition WHERE dt > toDate('2020-01-01') AND dt < toDate('2020-01-02'); +SELECT groupArray(dt) from test.date_datetime_key_condition WHERE dt >= toDate('2020-01-02'); +SELECT groupArray(dt) from test.date_datetime_key_condition WHERE dt < toDate('2020-01-02'); + +-- inside +SELECT groupArray(dt) from test.date_datetime_key_condition WHERE dt > toDate('2019-01-02'); +SELECT groupArray(dt) from test.date_datetime_key_condition WHERE dt < toDate('2021-01-02'); +SELECT groupArray(dt) from test.date_datetime_key_condition WHERE dt >= toDate('2019-01-02') AND dt < toDate('2021-01-02'); +SELECT groupArray(dt) from test.date_datetime_key_condition WHERE dt > toDate('2019-01-02') OR dt <= toDate('2021-01-02'); + +-- outside +SELECT groupArray(dt) from test.date_datetime_key_condition WHERE dt < toDate('2019-01-02'); +SELECT groupArray(dt) from test.date_datetime_key_condition WHERE dt > toDate('2021-01-02'); +SELECT groupArray(dt) from test.date_datetime_key_condition WHERE dt < toDate('2019-01-02') OR dt > toDate('2021-01-02'); + +DROP TABLE test.date_datetime_key_condition; \ No newline at end of file From 654d6fcd26de5ab1260bbfd4609016316a4639ce Mon Sep 17 00:00:00 2001 From: Guillaume Tassery Date: Mon, 11 May 2020 11:35:21 +0700 Subject: [PATCH 056/738] cosmetic --- src/Interpreters/InterpreterSelectQuery.cpp | 10 +--------- src/Processors/OffsetTransform.cpp | 17 +---------------- src/Processors/OffsetTransform.h | 12 +----------- 3 files changed, 3 insertions(+), 36 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index d90556783f4..53a464f0535 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -2541,21 +2541,13 @@ void InterpreterSelectQuery::executeOffset(QueryPipeline & pipeline) UInt64 limit_offset; std::tie(limit_length, limit_offset) = getLimitLengthAndOffset(query, *context); - SortDescription order_descr; - if (query.limit_with_ties) - { - if (!query.orderBy()) - throw Exception("LIMIT WITH TIES without ORDER BY", ErrorCodes::LOGICAL_ERROR); - order_descr = getSortDescription(query, *context); - } - pipeline.addSimpleTransform([&](const Block & header, QueryPipeline::StreamType stream_type) -> ProcessorPtr { if (stream_type != QueryPipeline::StreamType::Main) return nullptr; std::cout << "TRANSFORM" << std::endl; return std::make_shared( - header, limit_offset, 1, true, query.limit_with_ties, order_descr); + header, limit_offset, 1); }); } } diff --git a/src/Processors/OffsetTransform.cpp b/src/Processors/OffsetTransform.cpp index 8c7cbe39e84..2427846a42b 100644 --- a/src/Processors/OffsetTransform.cpp +++ b/src/Processors/OffsetTransform.cpp @@ -10,17 +10,10 @@ namespace ErrorCodes } OffsetTransform::OffsetTransform( - const Block & header_, size_t offset_, size_t num_streams, - bool always_read_till_end_, bool with_ties_, - SortDescription description_) + const Block & header_, size_t offset_, size_t num_streams) : IProcessor(InputPorts(num_streams, header_), OutputPorts(num_streams, header_)) , offset(offset_) - , always_read_till_end(always_read_till_end_) - , with_ties(with_ties_), description(std::move(description_)) { - if (num_streams != 1 && with_ties) - throw Exception("Cannot use OffsetTransform with multiple ports and ties.", ErrorCodes::LOGICAL_ERROR); - ports_data.resize(num_streams); size_t cur_stream = 0; @@ -36,14 +29,6 @@ OffsetTransform::OffsetTransform( ports_data[cur_stream].output_port = &output; ++cur_stream; } - - for (const auto & desc : description) - { - if (!desc.column_name.empty()) - sort_column_positions.push_back(header_.getPositionByName(desc.column_name)); - else - sort_column_positions.push_back(desc.column_number); - } } diff --git a/src/Processors/OffsetTransform.h b/src/Processors/OffsetTransform.h index 6c62080bec7..5107f5421f9 100644 --- a/src/Processors/OffsetTransform.h +++ b/src/Processors/OffsetTransform.h @@ -20,13 +20,6 @@ class OffsetTransform : public IProcessor private: size_t offset; - bool always_read_till_end; - - bool with_ties; - const SortDescription description; - - Chunk previous_row_chunk; /// for WITH TIES, contains only sort columns - std::vector sort_column_positions; size_t rows_read = 0; /// including the last read block RowsBeforeLimitCounterPtr rows_before_limit_at_least; @@ -46,10 +39,7 @@ private: size_t num_finished_port_pairs = 0; public: - OffsetTransform( - const Block & header_, size_t offset_, size_t num_streams = 1, - bool always_read_till_end_ = false, bool with_ties_ = false, - SortDescription description_ = {}); + OffsetTransform(const Block & header_, size_t offset_, size_t num_streams = 1); String getName() const override { return "Offset"; } From c7dfe9e93dac141334623aa664755e5c593316ca Mon Sep 17 00:00:00 2001 From: Guillaume Tassery Date: Mon, 11 May 2020 11:47:31 +0700 Subject: [PATCH 057/738] cosmetic --- src/Interpreters/InterpreterSelectQuery.cpp | 6 ++---- src/Processors/OffsetTransform.h | 7 +------ 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 53a464f0535..93161249e09 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -2534,7 +2534,7 @@ void InterpreterSelectQuery::executeLimit(QueryPipeline & pipeline) void InterpreterSelectQuery::executeOffset(QueryPipeline & pipeline) { auto & query = getSelectQuery(); - /// If there is LIMIT + /// If there is not a LIMIT but an offset if (!query.limitLength() && query.limitOffset()) { UInt64 limit_length; @@ -2545,9 +2545,7 @@ void InterpreterSelectQuery::executeOffset(QueryPipeline & pipeline) { if (stream_type != QueryPipeline::StreamType::Main) return nullptr; - std::cout << "TRANSFORM" << std::endl; - return std::make_shared( - header, limit_offset, 1); + return std::make_shared(header, limit_offset, 1); }); } } diff --git a/src/Processors/OffsetTransform.h b/src/Processors/OffsetTransform.h index 5107f5421f9..7d29402f634 100644 --- a/src/Processors/OffsetTransform.h +++ b/src/Processors/OffsetTransform.h @@ -7,14 +7,9 @@ namespace DB { -/// Implementation for LIMIT N OFFSET M +/// Implementation for OFFSET N (without limit) /// This processor support multiple inputs and outputs (the same number). /// Each pair of input and output port works independently. -/// The reason to have multiple ports is to be able to stop all sources when limit is reached, in a query like: -/// SELECT * FROM system.numbers_mt WHERE number = 1000000 LIMIT 1 -/// -/// always_read_till_end - read all data from input ports even if limit was reached. -/// with_ties, description - implementation of LIMIT WITH TIES. It works only for single port. class OffsetTransform : public IProcessor { private: From 9ed397bc2c1215ba56ea9b4f05c0a86e423aada3 Mon Sep 17 00:00:00 2001 From: Guillaume Tassery Date: Mon, 11 May 2020 12:48:19 +0700 Subject: [PATCH 058/738] add tests --- .../01272_offset_without_limit.reference | 45 +++++++++++++++++++ .../01272_offset_without_limit.sql | 13 ++++++ 2 files changed, 58 insertions(+) create mode 100644 tests/queries/0_stateless/01272_offset_without_limit.reference create mode 100644 tests/queries/0_stateless/01272_offset_without_limit.sql diff --git a/tests/queries/0_stateless/01272_offset_without_limit.reference b/tests/queries/0_stateless/01272_offset_without_limit.reference new file mode 100644 index 00000000000..780c35c9d70 --- /dev/null +++ b/tests/queries/0_stateless/01272_offset_without_limit.reference @@ -0,0 +1,45 @@ +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 diff --git a/tests/queries/0_stateless/01272_offset_without_limit.sql b/tests/queries/0_stateless/01272_offset_without_limit.sql new file mode 100644 index 00000000000..769808b2edd --- /dev/null +++ b/tests/queries/0_stateless/01272_offset_without_limit.sql @@ -0,0 +1,13 @@ +DROP TABLE IF EXISTS offset_without_limit; + +CREATE TABLE offset_without_limit ( + value UInt32 +) Engine = MergeTree() + PRIMARY KEY value + ORDER BY value; + +INSERT INTO offset_without_limit SELECT * FROM system.numbers LIMIT 50; + +SELECT value FROM offset_without_limit ORDER BY value OFFSET 5; + +DROP TABLE offset_without_limit; From 3cfb3056644e29bcba9f3a613509135693bed0a0 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 11 May 2020 08:52:40 +0300 Subject: [PATCH 059/738] Slightly better test --- .../0_stateless/01070_exception_code_in_query_log_table.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01070_exception_code_in_query_log_table.sql b/tests/queries/0_stateless/01070_exception_code_in_query_log_table.sql index 9cca089ce08..2c99ba54112 100644 --- a/tests/queries/0_stateless/01070_exception_code_in_query_log_table.sql +++ b/tests/queries/0_stateless/01070_exception_code_in_query_log_table.sql @@ -3,5 +3,5 @@ SELECT * FROM test_table_for_01070_exception_code_in_query_log_table; -- { serve CREATE TABLE test_table_for_01070_exception_code_in_query_log_table (value UInt64) ENGINE=Memory(); SELECT * FROM test_table_for_01070_exception_code_in_query_log_table; SYSTEM FLUSH LOGS; -SELECT exception_code FROM system.query_log WHERE query='SELECT * FROM test_table_for_01070_exception_code_in_query_log_table' ORDER BY exception_code; +SELECT exception_code FROM system.query_log WHERE query = 'SELECT * FROM test_table_for_01070_exception_code_in_query_log_table' AND event_date >= yesterday() AND event_time > now() - INTERVAL 5 MINUTE ORDER BY exception_code; DROP TABLE IF EXISTS test_table_for_01070_exception_code_in_query_log_table; From 89c8581d4ec1cf7b3ea972193fd58b79eea558a1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 11 May 2020 09:15:47 +0300 Subject: [PATCH 060/738] Fix infinite loop in test --- .../queries/0_stateless/01019_alter_materialized_view_atomic.sh | 2 ++ .../0_stateless/01019_alter_materialized_view_consistent.sh | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh b/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh index e36c91ff513..626f990ef8d 100755 --- a/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh +++ b/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh @@ -5,6 +5,8 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh +trap 'kill -9 $(jobs -p)' EXIT + $CLICKHOUSE_CLIENT --multiquery < Date: Mon, 11 May 2020 09:21:26 +0300 Subject: [PATCH 061/738] Update CPUID after changes were integrated to upstream --- contrib/libcpuid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/libcpuid b/contrib/libcpuid index f670a1fa508..8db3b8d2d32 160000 --- a/contrib/libcpuid +++ b/contrib/libcpuid @@ -1 +1 @@ -Subproject commit f670a1fa508427b6e94f1c801cb12545a97b3cc1 +Subproject commit 8db3b8d2d32d22437f063ce692a1b9bb15e42d18 From d6eae04eb00a1d566c90cb749a74e2447dc19925 Mon Sep 17 00:00:00 2001 From: Guillaume Tassery Date: Mon, 11 May 2020 13:26:01 +0700 Subject: [PATCH 062/738] cosmetic --- src/Parsers/ASTSelectQuery.cpp | 2 +- src/Processors/OffsetTransform.cpp | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Parsers/ASTSelectQuery.cpp b/src/Parsers/ASTSelectQuery.cpp index 3e9692518b9..9e65543babe 100644 --- a/src/Parsers/ASTSelectQuery.cpp +++ b/src/Parsers/ASTSelectQuery.cpp @@ -154,7 +154,7 @@ void ASTSelectQuery::formatImpl(const FormatSettings & s, FormatState & state, F if (limit_with_ties) s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << " WITH TIES" << (s.hilite ? hilite_none : ""); } - else if(limitOffset()) + else if (limitOffset()) { s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "OFFSET " << (s.hilite ? hilite_none : ""); limitOffset()->formatImpl(s, state, frame); diff --git a/src/Processors/OffsetTransform.cpp b/src/Processors/OffsetTransform.cpp index 2427846a42b..6a3f3c01dc8 100644 --- a/src/Processors/OffsetTransform.cpp +++ b/src/Processors/OffsetTransform.cpp @@ -121,9 +121,8 @@ OffsetTransform::Status OffsetTransform::preparePair(PortsData & data) } input.setNeeded(); - if (!input.hasData()) { + if (!input.hasData()) return Status::NeedData; - } data.current_chunk = input.pull(true); From f67cd0d3f2d71f04f2ea430728215e5aa8114652 Mon Sep 17 00:00:00 2001 From: Guillaume Tassery Date: Mon, 11 May 2020 13:46:16 +0700 Subject: [PATCH 063/738] remove new line --- src/Interpreters/InterpreterSelectQuery.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 93161249e09..3e8b62c0fd6 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -2436,7 +2436,6 @@ void InterpreterSelectQuery::executeLimit(Pipeline & pipeline) void InterpreterSelectQuery::executeOffset(Pipeline & /* pipeline */) {} - void InterpreterSelectQuery::executeWithFill(Pipeline & pipeline) { auto & query = getSelectQuery(); From e2b3cf2b5266d739257ce63a05ab1e2328bcb43a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 11 May 2020 09:50:38 +0300 Subject: [PATCH 064/738] Remove obsolete suppression --- base/common/demangle.cpp | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/base/common/demangle.cpp b/base/common/demangle.cpp index 0ebeea6e070..d7fac7c6990 100644 --- a/base/common/demangle.cpp +++ b/base/common/demangle.cpp @@ -1,18 +1,6 @@ #include -#if defined(__has_feature) - #if __has_feature(memory_sanitizer) - #define MEMORY_SANITIZER 1 - #else - #define MEMORY_SANITIZER 0 - #endif -#elif defined(__MEMORY_SANITIZER__) - #define MEMORY_SANITIZER 1 -#else - #define MEMORY_SANITIZER 0 -#endif - -#if defined(_MSC_VER) || MEMORY_SANITIZER +#if defined(_MSC_VER) DemangleResult tryDemangle(const char *) { From 875a8ab2c31cf6e4bf3cd23d2db60e4d5ff99f5e Mon Sep 17 00:00:00 2001 From: Guillaume Tassery Date: Mon, 11 May 2020 13:53:10 +0700 Subject: [PATCH 065/738] fix parser --- src/Parsers/ParserSelectQuery.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Parsers/ParserSelectQuery.cpp b/src/Parsers/ParserSelectQuery.cpp index 02909314199..d2d7bbf9f21 100644 --- a/src/Parsers/ParserSelectQuery.cpp +++ b/src/Parsers/ParserSelectQuery.cpp @@ -241,9 +241,7 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) if (top_length && limit_length) throw Exception("Can not use TOP and LIMIT together", ErrorCodes::TOP_AND_LIMIT_TOGETHER); } - - - if (s_offset.ignore(pos, expected)) + else if (s_offset.ignore(pos, expected)) { if (!exp_elem.parse(pos, limit_offset, expected)) return false; From 99e64729bb42755f29af31fbbe631ad5d1df420e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 11 May 2020 10:05:46 +0300 Subject: [PATCH 066/738] Remove obsolete TSan suppressions --- cmake/sanitize.cmake | 6 ++++-- tests/tsan_suppressions.txt | 6 +----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/cmake/sanitize.cmake b/cmake/sanitize.cmake index e9fa0a01d4a..32443ed78c3 100644 --- a/cmake/sanitize.cmake +++ b/cmake/sanitize.cmake @@ -36,8 +36,10 @@ if (SANITIZE) endif () elseif (SANITIZE STREQUAL "thread") - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=thread") - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=thread") + set (TSAN_FLAGS "-fsanitize=thread -fsanitize-blacklist=${CMAKE_SOURCE_DIR}/tests/tsan_suppressions.txt") + + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} ${TSAN_FLAGS}") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} ${TSAN_FLAGS}") if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=thread") endif() diff --git a/tests/tsan_suppressions.txt b/tests/tsan_suppressions.txt index 3dc306ee133..912e0361bff 100644 --- a/tests/tsan_suppressions.txt +++ b/tests/tsan_suppressions.txt @@ -1,5 +1 @@ -# libc++ -race:locale - -# Too many mutexes: https://github.com/google/sanitizers/issues/950 -deadlock:DB::MergeTreeReadPool::fillPerPartInfo +# Fortunately, we have no suppressions! From a64d44799d512c10b68850672754c6fca0b43e0f Mon Sep 17 00:00:00 2001 From: Guillaume Tassery Date: Mon, 11 May 2020 14:13:33 +0700 Subject: [PATCH 067/738] newline --- src/Interpreters/InterpreterSelectQuery.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 3e8b62c0fd6..b7bc688bcbd 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -2550,7 +2550,6 @@ void InterpreterSelectQuery::executeOffset(QueryPipeline & pipeline) } - void InterpreterSelectQuery::executeExtremes(Pipeline & pipeline) { if (!context->getSettingsRef().extremes) From 776ce037779fbecfee54d39d659df06ecb22e466 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Mon, 11 May 2020 10:42:47 +0300 Subject: [PATCH 068/738] Fixed DateLUTImpl constructors to avoid accidental copying Fixed one case of copying DateLUTImpl --- base/common/DateLUTImpl.h | 7 ++++++- src/DataStreams/TTLBlockInputStream.h | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 6841598b7ca..18078299cd9 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -37,7 +37,12 @@ using YearWeek = std::pair; class DateLUTImpl { public: - DateLUTImpl(const std::string & time_zone); + explicit DateLUTImpl(const std::string & time_zone); + + DateLUTImpl(const DateLUTImpl &) = delete; + DateLUTImpl & operator=(const DateLUTImpl &) = delete; + DateLUTImpl(const DateLUTImpl &&) = delete; + DateLUTImpl & operator=(const DateLUTImpl &&) = delete; public: /// The order of fields matters for alignment and sizeof. diff --git a/src/DataStreams/TTLBlockInputStream.h b/src/DataStreams/TTLBlockInputStream.h index a3c10ec94bf..3896e5232f8 100644 --- a/src/DataStreams/TTLBlockInputStream.h +++ b/src/DataStreams/TTLBlockInputStream.h @@ -45,7 +45,7 @@ private: size_t rows_removed = 0; Logger * log; - DateLUTImpl date_lut; + const DateLUTImpl & date_lut; /// TODO rewrite defaults logic to evaluteMissingDefaults std::unordered_map defaults_result_column; From 2c2005817f0ece6da6381ace79de663edaf9d65e Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Wed, 6 May 2020 15:56:03 +0300 Subject: [PATCH 069/738] New function toStartOfSecond(DateTime64) -> DateTime64 This funtion nullifies sub-second part of DateTime64 value. --- src/Core/DecimalFunctions.h | 53 +++++++++------ src/DataTypes/DataTypeDateTime64.h | 66 ++++++++++++++----- src/Functions/DateTimeTransforms.h | 35 ++++++++++ src/Functions/FunctionCustomWeekToSomething.h | 2 +- .../FunctionDateOrDateTimeToSomething.h | 34 +++------- src/Functions/registerFunctionsDateTime.cpp | 2 + src/Functions/toStartOfInterval.cpp | 2 +- src/Functions/toStartOfSecond.cpp | 18 +++++ .../01269_toStartOfSecond.reference | 6 ++ .../0_stateless/01269_toStartOfSecond.sql | 13 ++++ 10 files changed, 169 insertions(+), 62 deletions(-) create mode 100644 src/Functions/toStartOfSecond.cpp create mode 100644 tests/queries/0_stateless/01269_toStartOfSecond.reference create mode 100644 tests/queries/0_stateless/01269_toStartOfSecond.sql diff --git a/src/Core/DecimalFunctions.h b/src/Core/DecimalFunctions.h index d8fdccf8628..b1f75c860dc 100644 --- a/src/Core/DecimalFunctions.h +++ b/src/Core/DecimalFunctions.h @@ -50,9 +50,10 @@ struct DecimalComponents * If `scale` is to big (scale > maxPrecision), result is undefined. */ template -DecimalType decimalFromComponentsWithMultiplier(const typename DecimalType::NativeType & whole, - const typename DecimalType::NativeType & fractional, - typename DecimalType::NativeType scale_multiplier) +inline DecimalType decimalFromComponentsWithMultiplier( + const typename DecimalType::NativeType & whole, + const typename DecimalType::NativeType & fractional, + typename DecimalType::NativeType scale_multiplier) { using T = typename DecimalType::NativeType; const auto fractional_sign = whole < 0 ? -1 : 1; @@ -70,8 +71,10 @@ DecimalType decimalFromComponentsWithMultiplier(const typename DecimalType::Nati * @see `decimalFromComponentsWithMultiplier` for details. */ template -DecimalType decimalFromComponents( - const typename DecimalType::NativeType & whole, const typename DecimalType::NativeType & fractional, UInt32 scale) +inline DecimalType decimalFromComponents( + const typename DecimalType::NativeType & whole, + const typename DecimalType::NativeType & fractional, + UInt32 scale) { using T = typename DecimalType::NativeType; @@ -82,8 +85,9 @@ DecimalType decimalFromComponents( * @see `decimalFromComponentsWithMultiplier` for details. */ template -DecimalType decimalFromComponents( - const DecimalComponents & components, UInt32 scale) +inline DecimalType decimalFromComponents( + const DecimalComponents & components, + UInt32 scale) { return decimalFromComponents(components.whole, components.fractional, scale); } @@ -92,8 +96,9 @@ DecimalType decimalFromComponents( * This is an optimization to reduce number of calls to scaleMultiplier on known scale. */ template -DecimalComponents splitWithScaleMultiplier( - const DecimalType & decimal, typename DecimalType::NativeType scale_multiplier) +inline DecimalComponents splitWithScaleMultiplier( + const DecimalType & decimal, + typename DecimalType::NativeType scale_multiplier) { using T = typename DecimalType::NativeType; const auto whole = decimal.value / scale_multiplier; @@ -106,7 +111,7 @@ DecimalComponents splitWithScaleMultiplier( /// Split decimal into components: whole and fractional part, @see `DecimalComponents` for details. template -DecimalComponents split(const DecimalType & decimal, UInt32 scale) +inline DecimalComponents split(const DecimalType & decimal, UInt32 scale) { if (scale == 0) { @@ -121,7 +126,7 @@ DecimalComponents split(const DecimalType & de * If scale is to big, result is undefined. */ template -typename DecimalType::NativeType getWholePart(const DecimalType & decimal, size_t scale) +inline typename DecimalType::NativeType getWholePart(const DecimalType & decimal, size_t scale) { if (scale == 0) return decimal.value; @@ -129,24 +134,34 @@ typename DecimalType::NativeType getWholePart(const DecimalType & decimal, size_ return decimal.value / scaleMultiplier(scale); } + +template +inline typename DecimalType::NativeType getFractionalPartWithScaleMultiplier( + const DecimalType & decimal, + typename DecimalType::NativeType scale_multiplier) +{ + using T = typename DecimalType::NativeType; + + T result = decimal.value; + if constexpr (!keep_sign) + if (result < T(0)) + result = -result; + + return result % scale_multiplier; +} + /** Get fractional part from decimal * * Result is always positive. * If scale is to big, result is undefined. */ template -typename DecimalType::NativeType getFractionalPart(const DecimalType & decimal, size_t scale) +inline typename DecimalType::NativeType getFractionalPart(const DecimalType & decimal, size_t scale) { - using T = typename DecimalType::NativeType; - if (scale == 0) return 0; - T result = decimal.value; - if (result < T(0)) - result *= T(-1); - - return result % scaleMultiplier(scale); + return getFractionalPartWithScaleMultiplier(decimal, scaleMultiplier(scale)); } } diff --git a/src/DataTypes/DataTypeDateTime64.h b/src/DataTypes/DataTypeDateTime64.h index 44efe4f41b6..2ec87b37e7a 100644 --- a/src/DataTypes/DataTypeDateTime64.h +++ b/src/DataTypes/DataTypeDateTime64.h @@ -45,40 +45,74 @@ public: bool equals(const IDataType & rhs) const override; }; - -/** Basic wrapper for Tansform-types for DateTime64. +/** Tansform-type wrapper for DateTime64, applies given Transform to DateTime64 value or only to a whole part of it. * - * Allows reusing existing Transform (that takes DateTime-values as UInt32) with DateTime64-values, - * by discarding fractional part and producing SAME return type as original Transform. + * Depending on what overloads of Transform::execute() are available, when called with DateTime64 value, + * invokes Transform::execute() with: + * * whole part of DateTime64 value, discarding fractional part. + * * DateTime64 value and scale factor. * - * Such Transfotm-types are commonly used in Date/DateTime manipulation functions, - * and implement static execute fucntion with following signature: - * R execute(UInt32, T, const DateLUTImpl &) + * Suitable Transfotm-types are commonly used in Date/DateTime manipulation functions, + * and should implement static (or const) fucntion with following signatures: + * R execute(UInt32 whole_value, ... , const TimeZoneImpl &) + * OR + * R execute(DateTime64 value, Int64 scale_factor, ... , const TimeZoneImpl &) * * Wehere R and T could be arbitrary types. */ template -class DateTime64BasicTransformWrapper : public Transform +class TransformDateTime64 : public Transform { +private: + // ExtractName::name is Transform::name or default. + struct ExtractName + { + template static constexpr auto getName(...) { return ""; } + template> static constexpr auto getName(void*) { return U::name; } + static constexpr auto name = getName(nullptr); + }; + + // Detect if Transform::execute is const or static method + // with signature defined by template args (ignoring result type). + template + struct TransformHasExecuteOverload : std::false_type {}; + + template + struct TransformHasExecuteOverload().execute(std::declval()...))>, Args...> + : std::true_type {}; + + template + static constexpr bool TransformHasExecuteOverload_v = TransformHasExecuteOverload::value; + public: + static constexpr auto name = ExtractName::name; + using Transform::execute; - explicit DateTime64BasicTransformWrapper(UInt32 scale_) + // non-explicit constructor to allow creating from scale value (or with no scale at all), indispensable in some contexts. + TransformDateTime64(UInt32 scale_ = 0) : scale_multiplier(DecimalUtils::scaleMultiplier(scale_)) {} - template - auto execute(DateTime64 t, T v, const DateLUTImpl & time_zone) const + template + inline auto execute(const DateTime64 & t, Args && ... args) const { - const auto components = DecimalUtils::splitWithScaleMultiplier(t, scale_multiplier); - return static_cast(this)->execute( - static_cast(components.whole), v, time_zone); + const auto transform = static_cast(this); + + if constexpr (TransformHasExecuteOverload_v) + { + return transform->execute(t, scale_multiplier, std::forward(args)...); + } + else + { + const auto components = DecimalUtils::splitWithScaleMultiplier(t, scale_multiplier); + return transform->execute(static_cast(components.whole), std::forward(args)...); + } } private: - UInt32 scale_multiplier = 1; + DateTime64::NativeType scale_multiplier = 1; }; - } diff --git a/src/Functions/DateTimeTransforms.h b/src/Functions/DateTimeTransforms.h index c0db838412e..6e2c3ea9ea6 100644 --- a/src/Functions/DateTimeTransforms.h +++ b/src/Functions/DateTimeTransforms.h @@ -175,6 +175,41 @@ struct ToStartOfMinuteImpl using FactorTransform = ZeroTransform; }; +// Rounding towards negative infinity. +// 1.01 => 1.00 +// -1.01 => -2 +struct ToStartOfSecondImpl +{ + static constexpr auto name = "toStartOfSecond"; + + static inline DateTime64 execute(const DateTime64 & datetime64, Int64 scale_multiplier, const DateLUTImpl &) + { + auto fractional_with_sign = DecimalUtils::getFractionalPartWithScaleMultiplier(datetime64, scale_multiplier); + + // given that scale is 3, scale_multiplier is 1000 + // for DateTime64 value of 123.456: + // 123456 - 456 = 123000 + // for DateTime64 value of -123.456: + // -123456 - (1000 + (-456)) = -124000 + + if (fractional_with_sign < 0) + fractional_with_sign += scale_multiplier; + + return datetime64 - fractional_with_sign; + } + + static inline UInt32 execute(UInt32, const DateLUTImpl &) + { + throw Exception("Illegal type DateTime of argument for function " + std::string(name), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + static inline UInt32 execute(UInt16, const DateLUTImpl &) + { + return dateIsNotSupported(name); + } + + using FactorTransform = ZeroTransform; +}; + struct ToStartOfFiveMinuteImpl { static constexpr auto name = "toStartOfFiveMinute"; diff --git a/src/Functions/FunctionCustomWeekToSomething.h b/src/Functions/FunctionCustomWeekToSomething.h index 045e3ffd669..5dda225032d 100644 --- a/src/Functions/FunctionCustomWeekToSomething.h +++ b/src/Functions/FunctionCustomWeekToSomething.h @@ -110,7 +110,7 @@ public: { CustomWeekTransformImpl::execute( block, arguments, result, input_rows_count, - DateTime64BasicTransformWrapper{assert_cast(from_type)->getScale()}); + TransformDateTime64{assert_cast(from_type)->getScale()}); } else throw Exception( diff --git a/src/Functions/FunctionDateOrDateTimeToSomething.h b/src/Functions/FunctionDateOrDateTimeToSomething.h index c935fb4563a..85fb89ea4c1 100644 --- a/src/Functions/FunctionDateOrDateTimeToSomething.h +++ b/src/Functions/FunctionDateOrDateTimeToSomething.h @@ -16,26 +16,6 @@ namespace ErrorCodes extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } -template -struct WithDateTime64Converter : public Transform -{ - UInt8 scale; - Transform transform; - - explicit WithDateTime64Converter(UInt8 scale_, Transform transform_ = {}) - : scale(scale_), - transform(std::move(transform_)) - {} - - inline auto execute(DataTypeDateTime64::FieldType t, const DateLUTImpl & time_zone) const - { - auto x = DateTime64(t); - auto res = transform.execute(static_cast(DecimalUtils::getWholePart(x, scale)), time_zone); - return res; - } -}; - - /// See DateTimeTransforms.h template class FunctionDateOrDateTimeToSomething : public IFunction @@ -90,7 +70,13 @@ public: if constexpr (std::is_same_v) return std::make_shared(extractTimeZoneNameFromFunctionArguments(arguments, 1, 0)); if constexpr (std::is_same_v) - return std::make_shared(extractTimeZoneNameFromFunctionArguments(arguments, 1, 0)); + { + Int64 scale = DataTypeDateTime64::default_scale; + if (const auto * dt64 = checkAndGetDataType(arguments[0].type.get())) + scale = dt64->getScale(); + + return std::make_shared(scale, extractTimeZoneNameFromFunctionArguments(arguments, 1, 0)); + } else return std::make_shared(); } @@ -110,16 +96,14 @@ public: else if (which.isDateTime64()) { const auto scale = static_cast(from_type)->getScale(); - WithDateTime64Converter transformer(scale); - - DateTimeTransformImpl>::execute(block, arguments, result, input_rows_count, transformer); + const TransformDateTime64 transformer(scale); + DateTimeTransformImpl::execute(block, arguments, result, input_rows_count, transformer); } else throw Exception("Illegal type " + block.getByPosition(arguments[0]).type->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } - bool hasInformationAboutMonotonicity() const override { return true; diff --git a/src/Functions/registerFunctionsDateTime.cpp b/src/Functions/registerFunctionsDateTime.cpp index 1dd1b991337..07c45241535 100644 --- a/src/Functions/registerFunctionsDateTime.cpp +++ b/src/Functions/registerFunctionsDateTime.cpp @@ -11,6 +11,7 @@ void registerFunctionToDayOfWeek(FunctionFactory &); void registerFunctionToDayOfYear(FunctionFactory &); void registerFunctionToHour(FunctionFactory &); void registerFunctionToMinute(FunctionFactory &); +void registerFunctionToStartOfSecond(FunctionFactory &); void registerFunctionToSecond(FunctionFactory &); void registerFunctionToStartOfDay(FunctionFactory &); void registerFunctionToMonday(FunctionFactory &); @@ -84,6 +85,7 @@ void registerFunctionsDateTime(FunctionFactory & factory) registerFunctionToStartOfMonth(factory); registerFunctionToStartOfQuarter(factory); registerFunctionToStartOfYear(factory); + registerFunctionToStartOfSecond(factory); registerFunctionToStartOfMinute(factory); registerFunctionToStartOfFiveMinute(factory); registerFunctionToStartOfTenMinutes(factory); diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index f470504a3a9..f644b410907 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -311,7 +311,7 @@ private: if constexpr (std::is_same_v) { - const auto transform = DateTime64BasicTransformWrapper>{from_datatype.getScale()}; + const auto transform = TransformDateTime64>{from_datatype.getScale()}; for (size_t i = 0; i != size; ++i) result_data[i] = transform.execute(time_data[i], num_units, time_zone); } diff --git a/src/Functions/toStartOfSecond.cpp b/src/Functions/toStartOfSecond.cpp new file mode 100644 index 00000000000..f796dce410a --- /dev/null +++ b/src/Functions/toStartOfSecond.cpp @@ -0,0 +1,18 @@ +#include +#include +#include +#include +#include + + +namespace DB +{ + +using FunctionToStartOfSecond = FunctionDateOrDateTimeToSomething; + +void registerFunctionToStartOfSecond(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} diff --git a/tests/queries/0_stateless/01269_toStartOfSecond.reference b/tests/queries/0_stateless/01269_toStartOfSecond.reference new file mode 100644 index 00000000000..d2d1b4b3881 --- /dev/null +++ b/tests/queries/0_stateless/01269_toStartOfSecond.reference @@ -0,0 +1,6 @@ +2019-09-16 16:20:11.000 DateTime64(3, \'UTC\') +2019-09-16 19:20:11 DateTime64(0, \'UTC\') +2019-09-16 19:20:11.000 DateTime64(3, \'UTC\') +2019-09-16 19:20:11.000000000 DateTime64(9, \'UTC\') +non-const column +2019-09-16 19:20:11.000 DateTime64(3, \'UTC\') diff --git a/tests/queries/0_stateless/01269_toStartOfSecond.sql b/tests/queries/0_stateless/01269_toStartOfSecond.sql new file mode 100644 index 00000000000..5fe6aa9602f --- /dev/null +++ b/tests/queries/0_stateless/01269_toStartOfSecond.sql @@ -0,0 +1,13 @@ +-- Error cases +SELECT toStartOfSecond('123'); -- {serverError 43} +SELECT toStartOfSecond(now()); -- {serverError 43} +SELECT toStartOfSecond(); -- {serverError 42} +SELECT toStartOfSecond(now64(), 123); -- {serverError 43} + +WITH toDateTime64('2019-09-16 19:20:11', 3) AS dt64 SELECT toStartOfSecond(dt64, 'UTC') AS res, toTypeName(res); +WITH toDateTime64('2019-09-16 19:20:11', 0, 'UTC') AS dt64 SELECT toStartOfSecond(dt64) AS res, toTypeName(res); +WITH toDateTime64('2019-09-16 19:20:11.123', 3, 'UTC') AS dt64 SELECT toStartOfSecond(dt64) AS res, toTypeName(res); +WITH toDateTime64('2019-09-16 19:20:11.123', 9, 'UTC') AS dt64 SELECT toStartOfSecond(dt64) AS res, toTypeName(res); + +SELECT 'non-const column'; +WITH toDateTime64('2019-09-16 19:20:11.123', 3, 'UTC') AS dt64 SELECT toStartOfSecond(materialize(dt64)) AS res, toTypeName(res); \ No newline at end of file From 7d480ffa3d3be1a927c8f4d66f413e4c2e594139 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 15 Apr 2020 02:46:29 +0300 Subject: [PATCH 070/738] Added failing tests --- .../01131_max_rows_to_sort.reference | 1 + .../0_stateless/01131_max_rows_to_sort.sql | 7 + .../01132_max_rows_to_read.reference | 6 + .../0_stateless/01132_max_rows_to_read.sql | 17 ++ .../01133_max_result_rows.reference | 158 ++++++++++++++++++ .../0_stateless/01133_max_result_rows.sql | 15 ++ .../01134_set_overflow_mode.reference | 13 ++ .../0_stateless/01134_set_overflow_mode.sql | 17 ++ 8 files changed, 234 insertions(+) create mode 100644 tests/queries/0_stateless/01131_max_rows_to_sort.reference create mode 100644 tests/queries/0_stateless/01131_max_rows_to_sort.sql create mode 100644 tests/queries/0_stateless/01132_max_rows_to_read.reference create mode 100644 tests/queries/0_stateless/01132_max_rows_to_read.sql create mode 100644 tests/queries/0_stateless/01133_max_result_rows.reference create mode 100644 tests/queries/0_stateless/01133_max_result_rows.sql create mode 100644 tests/queries/0_stateless/01134_set_overflow_mode.reference create mode 100644 tests/queries/0_stateless/01134_set_overflow_mode.sql diff --git a/tests/queries/0_stateless/01131_max_rows_to_sort.reference b/tests/queries/0_stateless/01131_max_rows_to_sort.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/01131_max_rows_to_sort.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/01131_max_rows_to_sort.sql b/tests/queries/0_stateless/01131_max_rows_to_sort.sql new file mode 100644 index 00000000000..a6109700045 --- /dev/null +++ b/tests/queries/0_stateless/01131_max_rows_to_sort.sql @@ -0,0 +1,7 @@ +SET max_rows_to_sort = 100; +SELECT * FROM system.numbers ORDER BY number; -- { serverError 396 } + +SET sort_overflow_mode = 'break'; +SET max_block_size = 1000; + +SELECT count() >= 100 AND count() <= 1000 FROM (SELECT * FROM system.numbers ORDER BY number); diff --git a/tests/queries/0_stateless/01132_max_rows_to_read.reference b/tests/queries/0_stateless/01132_max_rows_to_read.reference new file mode 100644 index 00000000000..91653a56ef7 --- /dev/null +++ b/tests/queries/0_stateless/01132_max_rows_to_read.reference @@ -0,0 +1,6 @@ +19 +20 +30 +19 +20 +21 diff --git a/tests/queries/0_stateless/01132_max_rows_to_read.sql b/tests/queries/0_stateless/01132_max_rows_to_read.sql new file mode 100644 index 00000000000..7a00365ed4d --- /dev/null +++ b/tests/queries/0_stateless/01132_max_rows_to_read.sql @@ -0,0 +1,17 @@ +SET max_block_size = 10; +SET max_rows_to_read = 20; +SET read_overflow_mode = 'throw'; + +SELECT count() FROM numbers(30); -- { serverError 158 } +SELECT count() FROM numbers(19); +SELECT count() FROM numbers(20); +SELECT count() FROM numbers(21); -- { serverError 158 } + +SET read_overflow_mode = 'break'; + +SELECT count() FROM numbers(19); +SELECT count() FROM numbers(20); +SELECT count() FROM numbers(21); +SELECT count() FROM numbers(29); -- one extra block is read and it is Ok. +SELECT count() FROM numbers(30); +SELECT count() FROM numbers(31); \ No newline at end of file diff --git a/tests/queries/0_stateless/01133_max_result_rows.reference b/tests/queries/0_stateless/01133_max_result_rows.reference new file mode 100644 index 00000000000..61ecaca3948 --- /dev/null +++ b/tests/queries/0_stateless/01133_max_result_rows.reference @@ -0,0 +1,158 @@ +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 diff --git a/tests/queries/0_stateless/01133_max_result_rows.sql b/tests/queries/0_stateless/01133_max_result_rows.sql new file mode 100644 index 00000000000..fe86a5902cf --- /dev/null +++ b/tests/queries/0_stateless/01133_max_result_rows.sql @@ -0,0 +1,15 @@ +SET max_block_size = 10; +SET max_result_rows = 20; +SET result_overflow_mode = 'throw'; + +SELECT DISTINCT intDiv(number, 10) FROM numbers(300); -- { serverError 396 } +SELECT DISTINCT intDiv(number, 10) FROM numbers(190); +SELECT DISTINCT intDiv(number, 10) FROM numbers(200); +SELECT DISTINCT intDiv(number, 10) FROM numbers(210); -- { serverError 396 } + +SET result_overflow_mode = 'break'; + +SELECT DISTINCT intDiv(number, 10) FROM numbers(300); +SELECT DISTINCT intDiv(number, 10) FROM numbers(190); +SELECT DISTINCT intDiv(number, 10) FROM numbers(200); +SELECT DISTINCT intDiv(number, 10) FROM numbers(210); diff --git a/tests/queries/0_stateless/01134_set_overflow_mode.reference b/tests/queries/0_stateless/01134_set_overflow_mode.reference new file mode 100644 index 00000000000..2e981dc7c97 --- /dev/null +++ b/tests/queries/0_stateless/01134_set_overflow_mode.reference @@ -0,0 +1,13 @@ +1 +0 +1 +0 +--- +1 +0 +1 +0 +1 +0 +1 +0 diff --git a/tests/queries/0_stateless/01134_set_overflow_mode.sql b/tests/queries/0_stateless/01134_set_overflow_mode.sql new file mode 100644 index 00000000000..791bc6d7f9e --- /dev/null +++ b/tests/queries/0_stateless/01134_set_overflow_mode.sql @@ -0,0 +1,17 @@ +SET max_block_size = 10; +SET max_rows_in_set = 20; +SET set_overflow_mode = 'throw'; + +SELECT arrayJoin([5, 25]) IN (SELECT DISTINCT toUInt8(intDiv(number, 10)) FROM numbers(300)); -- { serverError 191 } +SELECT arrayJoin([5, 25]) IN (SELECT DISTINCT toUInt8(intDiv(number, 10)) FROM numbers(190)); +SELECT arrayJoin([5, 25]) IN (SELECT DISTINCT toUInt8(intDiv(number, 10)) FROM numbers(200)); +SELECT arrayJoin([5, 25]) IN (SELECT DISTINCT toUInt8(intDiv(number, 10)) FROM numbers(210)); -- { serverError 191 } + +SET set_overflow_mode = 'break'; + +SELECT '---'; + +SELECT arrayJoin([5, 25]) IN (SELECT DISTINCT toUInt8(intDiv(number, 10)) FROM numbers(300)); +SELECT arrayJoin([5, 25]) IN (SELECT DISTINCT toUInt8(intDiv(number, 10)) FROM numbers(190)); +SELECT arrayJoin([5, 25]) IN (SELECT DISTINCT toUInt8(intDiv(number, 10)) FROM numbers(200)); +SELECT arrayJoin([5, 25]) IN (SELECT DISTINCT toUInt8(intDiv(number, 10)) FROM numbers(210)); From 0fe9ed93a63f28332b9d292b0ff1861b28b3bf87 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 11 May 2020 12:07:11 +0300 Subject: [PATCH 071/738] Update 01132_max_rows_to_read.sql --- tests/queries/0_stateless/01132_max_rows_to_read.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01132_max_rows_to_read.sql b/tests/queries/0_stateless/01132_max_rows_to_read.sql index 7a00365ed4d..f8e5e32c0b5 100644 --- a/tests/queries/0_stateless/01132_max_rows_to_read.sql +++ b/tests/queries/0_stateless/01132_max_rows_to_read.sql @@ -14,4 +14,4 @@ SELECT count() FROM numbers(20); SELECT count() FROM numbers(21); SELECT count() FROM numbers(29); -- one extra block is read and it is Ok. SELECT count() FROM numbers(30); -SELECT count() FROM numbers(31); \ No newline at end of file +SELECT count() FROM numbers(31); From e24954eebf82895905069da912c9c015098b8b21 Mon Sep 17 00:00:00 2001 From: alex-zaitsev Date: Mon, 11 May 2020 13:56:47 +0300 Subject: [PATCH 072/738] Add a description of Altinity services (#10812) * Add a description of Altinity services * Sales pitch removed --- docs/en/commercial/support.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/commercial/support.md b/docs/en/commercial/support.md index 3324bfd2123..37bc54e3e8b 100644 --- a/docs/en/commercial/support.md +++ b/docs/en/commercial/support.md @@ -10,7 +10,7 @@ toc_title: Support ## Altinity {#altinity} -[Service description](https://www.altinity.com/24x7-support) +Altinity has offered enterprise ClickHouse support and services since 2017. Altinity customers range from Fortune 100 enterprises to startups. Visit [www.altinity.com](https://www.altinity.com/) for more information. ## Mafiree {#mafiree} From b99ff5aa2d7e4e16cbb66c19eb74d7fa203156a4 Mon Sep 17 00:00:00 2001 From: Guillaume Tassery Date: Mon, 11 May 2020 18:01:23 +0700 Subject: [PATCH 073/738] const keyword --- src/Processors/OffsetTransform.cpp | 2 +- src/Processors/OffsetTransform.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Processors/OffsetTransform.cpp b/src/Processors/OffsetTransform.cpp index 6a3f3c01dc8..f380a5a5159 100644 --- a/src/Processors/OffsetTransform.cpp +++ b/src/Processors/OffsetTransform.cpp @@ -159,7 +159,7 @@ OffsetTransform::Status OffsetTransform::preparePair(PortsData & data) } -void OffsetTransform::splitChunk(PortsData & data) +void OffsetTransform::splitChunk(PortsData & data) const { size_t num_rows = data.current_chunk.getNumRows(); size_t num_columns = data.current_chunk.getNumColumns(); diff --git a/src/Processors/OffsetTransform.h b/src/Processors/OffsetTransform.h index 7d29402f634..3fee4e791a5 100644 --- a/src/Processors/OffsetTransform.h +++ b/src/Processors/OffsetTransform.h @@ -41,7 +41,7 @@ public: Status prepare(const PortNumbers & /*updated_input_ports*/, const PortNumbers & /*updated_output_ports*/) override; Status prepare() override; /// Compatibility for TreeExecutor. Status preparePair(PortsData & data); - void splitChunk(PortsData & data); + void splitChunk(PortsData & data) const; InputPort & getInputPort() { return inputs.front(); } OutputPort & getOutputPort() { return outputs.front(); } From c25ffb59300df0663fcde3b3fe3a821e4e717a86 Mon Sep 17 00:00:00 2001 From: Guillaume Tassery Date: Mon, 11 May 2020 19:09:02 +0700 Subject: [PATCH 074/738] Fix with a limit of 0 for system.numbers --- src/Interpreters/InterpreterSelectQuery.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 95f9c0cab28..3b617cf6d51 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -689,7 +689,7 @@ static UInt64 getLimitForSorting(const ASTSelectQuery & query, const Context & c if (!query.distinct && !query.limitBy() && !query.limit_with_ties && !query.arrayJoinExpressionList() && query.limitLength()) { auto [limit_length, limit_offset] = getLimitLengthAndOffset(query, context); - return limit_length + limit_offset; + return limit_length != 0 ? limit_length + limit_offset : 0; } return 0; } @@ -2314,6 +2314,9 @@ void InterpreterSelectQuery::executePreLimit(QueryPipeline & pipeline, bool do_n { auto [limit_length, limit_offset] = getLimitLengthAndOffset(query, *context); + if (limit_length == 0) + limit_offset = 0; + if (do_not_skip_offset) { limit_length += limit_offset; @@ -2427,6 +2430,9 @@ void InterpreterSelectQuery::executeLimit(Pipeline & pipeline) UInt64 limit_offset; std::tie(limit_length, limit_offset) = getLimitLengthAndOffset(query, *context); + if (limit_length == 0) + limit_offset = 0; + pipeline.transform([&](auto & stream) { stream = std::make_shared(stream, limit_length, limit_offset, always_read_till_end, false, query.limit_with_ties, order_descr); @@ -2510,6 +2516,9 @@ void InterpreterSelectQuery::executeLimit(QueryPipeline & pipeline) UInt64 limit_offset; std::tie(limit_length, limit_offset) = getLimitLengthAndOffset(query, *context); + if (limit_length == 0) + limit_offset = 0; + SortDescription order_descr; if (query.limit_with_ties) { From 41d2e9d52da699af1c33148408807a331305e2e9 Mon Sep 17 00:00:00 2001 From: mnkonkova Date: Mon, 11 May 2020 12:12:54 +0000 Subject: [PATCH 075/738] test --- contrib/grpc-cmake/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/grpc-cmake/CMakeLists.txt b/contrib/grpc-cmake/CMakeLists.txt index 1d0cf235b48..0180c0c1d31 100644 --- a/contrib/grpc-cmake/CMakeLists.txt +++ b/contrib/grpc-cmake/CMakeLists.txt @@ -1,5 +1,6 @@ cmake_minimum_required(VERSION 3.5.1) + set(GRPC_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/grpc) set(GRPC_INCLUDE_DIR ${GRPC_SOURCE_DIR}/include/) set(GRPC_BINARY_DIR ${ClickHouse_BINARY_DIR}/contrib/grpc) From b0df037f25a3a37b95d70a13e08994b7842b3885 Mon Sep 17 00:00:00 2001 From: Guillaume Tassery Date: Mon, 11 May 2020 19:18:07 +0700 Subject: [PATCH 076/738] Fix when we set a LIMIT to 0 --- src/Interpreters/InterpreterSelectQuery.cpp | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 3b617cf6d51..269914e0a9e 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -676,8 +676,12 @@ static std::pair getLimitLengthAndOffset(const ASTSelectQuery & UInt64 offset = 0; if (query.limitLength()) + { length = getLimitUIntValue(query.limitLength(), context, "LIMIT"); - if (query.limitOffset()) + if (query.limitOffset() && length) + offset = getLimitUIntValue(query.limitOffset(), context, "OFFSET"); + } + else if (query.limitOffset()) offset = getLimitUIntValue(query.limitOffset(), context, "OFFSET"); return {length, offset}; } @@ -689,7 +693,7 @@ static UInt64 getLimitForSorting(const ASTSelectQuery & query, const Context & c if (!query.distinct && !query.limitBy() && !query.limit_with_ties && !query.arrayJoinExpressionList() && query.limitLength()) { auto [limit_length, limit_offset] = getLimitLengthAndOffset(query, context); - return limit_length != 0 ? limit_length + limit_offset : 0; + return limit_length + limit_offset; } return 0; } @@ -2314,9 +2318,6 @@ void InterpreterSelectQuery::executePreLimit(QueryPipeline & pipeline, bool do_n { auto [limit_length, limit_offset] = getLimitLengthAndOffset(query, *context); - if (limit_length == 0) - limit_offset = 0; - if (do_not_skip_offset) { limit_length += limit_offset; @@ -2430,9 +2431,6 @@ void InterpreterSelectQuery::executeLimit(Pipeline & pipeline) UInt64 limit_offset; std::tie(limit_length, limit_offset) = getLimitLengthAndOffset(query, *context); - if (limit_length == 0) - limit_offset = 0; - pipeline.transform([&](auto & stream) { stream = std::make_shared(stream, limit_length, limit_offset, always_read_till_end, false, query.limit_with_ties, order_descr); @@ -2516,9 +2514,6 @@ void InterpreterSelectQuery::executeLimit(QueryPipeline & pipeline) UInt64 limit_offset; std::tie(limit_length, limit_offset) = getLimitLengthAndOffset(query, *context); - if (limit_length == 0) - limit_offset = 0; - SortDescription order_descr; if (query.limit_with_ties) { From 9780da3e747726086cf705281775af98f38a3e0c Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Mon, 11 May 2020 19:01:06 +0200 Subject: [PATCH 077/738] Adding more tests to check renaming of columns. --- ...alter_rename_column_default_expr.reference | 87 ++++++++++++++++++ ...01275_alter_rename_column_default_expr.sql | 34 +++++++ ..._rename_column_materialized_expr.reference | 90 +++++++++++++++++++ ..._alter_rename_column_materialized_expr.sql | 37 ++++++++ ...er_rename_column_constraint_expr.reference | 0 ...77_alter_rename_column_constraint_expr.sql | 43 +++++++++ 6 files changed, 291 insertions(+) create mode 100644 tests/queries/0_stateless/01275_alter_rename_column_default_expr.reference create mode 100644 tests/queries/0_stateless/01275_alter_rename_column_default_expr.sql create mode 100644 tests/queries/0_stateless/01276_alter_rename_column_materialized_expr.reference create mode 100644 tests/queries/0_stateless/01276_alter_rename_column_materialized_expr.sql create mode 100644 tests/queries/0_stateless/01277_alter_rename_column_constraint_expr.reference create mode 100644 tests/queries/0_stateless/01277_alter_rename_column_constraint_expr.sql diff --git a/tests/queries/0_stateless/01275_alter_rename_column_default_expr.reference b/tests/queries/0_stateless/01275_alter_rename_column_default_expr.reference new file mode 100644 index 00000000000..d81601b92c5 --- /dev/null +++ b/tests/queries/0_stateless/01275_alter_rename_column_default_expr.reference @@ -0,0 +1,87 @@ +2019-10-01 0 0 1 0 + 1 +2019-10-02 1 1 2 1 + 2 +2019-10-03 2 2 3 2 + 3 +2019-10-01 3 3 4 3 + 4 +2019-10-02 4 4 5 4 + 5 +2019-10-03 5 5 6 5 + 6 +2019-10-01 6 6 7 6 + 7 +2019-10-02 7 7 8 7 + 8 +2019-10-03 8 8 9 8 + 9 +CREATE TABLE default.table_for_rename\n(\n `date` Date, \n `key` UInt64, \n `value4` String, \n `value5` String, \n `value3` String DEFAULT concat(value4, \' + \', value5)\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 +2019-10-01 0 0 1 0 + 1 +2019-10-02 1 1 2 1 + 2 +2019-10-03 2 2 3 2 + 3 +2019-10-01 3 3 4 3 + 4 +2019-10-02 4 4 5 4 + 5 +2019-10-03 5 5 6 5 + 6 +2019-10-01 6 6 7 6 + 7 +2019-10-02 7 7 8 7 + 8 +2019-10-03 8 8 9 8 + 9 +2019-10-01 0 0 1 0 + 1 +2019-10-02 1 1 2 1 + 2 +2019-10-03 2 2 3 2 + 3 +2019-10-01 3 3 4 3 + 4 +2019-10-02 4 4 5 4 + 5 +2019-10-03 5 5 6 5 + 6 +2019-10-01 6 6 7 6 + 7 +2019-10-02 7 7 8 7 + 8 +2019-10-03 8 8 9 8 + 9 +2019-10-02 10 10 11 10 + 11 +2019-10-03 11 11 12 11 + 12 +2019-10-01 12 12 13 12 + 13 +2019-10-02 13 13 14 13 + 14 +2019-10-03 14 14 15 14 + 15 +2019-10-01 15 15 16 15 + 16 +2019-10-02 16 16 17 16 + 17 +2019-10-03 17 17 18 17 + 18 +2019-10-01 18 18 19 18 + 19 +2019-10-02 19 19 20 19 + 20 +CREATE TABLE default.table_for_rename\n(\n `date` Date, \n `key` UInt64, \n `value1` String, \n `value2` String, \n `value3` String DEFAULT concat(value1, \' + \', value2)\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 +2019-10-01 0 0 1 0 + 1 +2019-10-02 1 1 2 1 + 2 +2019-10-03 2 2 3 2 + 3 +2019-10-01 3 3 4 3 + 4 +2019-10-02 4 4 5 4 + 5 +2019-10-03 5 5 6 5 + 6 +2019-10-01 6 6 7 6 + 7 +2019-10-02 7 7 8 7 + 8 +2019-10-03 8 8 9 8 + 9 +2019-10-02 10 10 11 10 + 11 +2019-10-03 11 11 12 11 + 12 +2019-10-01 12 12 13 12 + 13 +2019-10-02 13 13 14 13 + 14 +2019-10-03 14 14 15 14 + 15 +2019-10-01 15 15 16 15 + 16 +2019-10-02 16 16 17 16 + 17 +2019-10-03 17 17 18 17 + 18 +2019-10-01 18 18 19 18 + 19 +2019-10-02 19 19 20 19 + 20 +2019-10-01 0 0 1 0 + 1 +2019-10-02 1 1 2 1 + 2 +2019-10-03 2 2 3 2 + 3 +2019-10-01 3 3 4 3 + 4 +2019-10-02 4 4 5 4 + 5 +2019-10-03 5 5 6 5 + 6 +2019-10-01 6 6 7 6 + 7 +2019-10-02 7 7 8 7 + 8 +2019-10-03 8 8 9 8 + 9 +2019-10-02 10 10 11 10 + 11 +2019-10-03 11 11 12 11 + 12 +2019-10-01 12 12 13 12 + 13 +2019-10-02 13 13 14 13 + 14 +2019-10-03 14 14 15 14 + 15 +2019-10-01 15 15 16 15 + 16 +2019-10-02 16 16 17 16 + 17 +2019-10-03 17 17 18 17 + 18 +2019-10-01 18 18 19 18 + 19 +2019-10-02 19 19 20 19 + 20 +2019-10-03 20 20 21 20 + 21 +2019-10-01 21 21 22 21 + 22 +2019-10-02 22 22 23 22 + 23 +2019-10-03 23 23 24 23 + 24 +2019-10-01 24 24 25 24 + 25 +2019-10-02 25 25 26 25 + 26 +2019-10-03 26 26 27 26 + 27 +2019-10-01 27 27 28 27 + 28 +2019-10-02 28 28 29 28 + 29 +2019-10-03 29 29 30 29 + 30 diff --git a/tests/queries/0_stateless/01275_alter_rename_column_default_expr.sql b/tests/queries/0_stateless/01275_alter_rename_column_default_expr.sql new file mode 100644 index 00000000000..21106d200af --- /dev/null +++ b/tests/queries/0_stateless/01275_alter_rename_column_default_expr.sql @@ -0,0 +1,34 @@ +DROP TABLE IF EXISTS table_for_rename; + +CREATE TABLE table_for_rename +( + date Date, + key UInt64, + value1 String, + value2 String, + value3 String DEFAULT concat(value1, ' + ', value2) +) +ENGINE = MergeTree() +PARTITION BY date +ORDER BY key; + +INSERT INTO table_for_rename (date, key, value1, value2) SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1) from numbers(9); +SELECT * FROM table_for_rename ORDER BY key; + +ALTER TABLE table_for_rename RENAME COLUMN value1 TO value4; +ALTER TABLE table_for_rename RENAME COLUMN value2 TO value5; +SHOW CREATE TABLE table_for_rename; +SELECT * FROM table_for_rename ORDER BY key; + +INSERT INTO table_for_rename (date, key, value4, value5) SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1) from numbers(10, 10); +SELECT * FROM table_for_rename ORDER BY key; + +ALTER TABLE table_for_rename RENAME COLUMN value4 TO value1; +ALTER TABLE table_for_rename RENAME COLUMN value5 TO value2; +SHOW CREATE TABLE table_for_rename; +SELECT * FROM table_for_rename ORDER BY key; + +INSERT INTO table_for_rename (date, key, value1, value2) SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1) from numbers(20,10); +SELECT * FROM table_for_rename ORDER BY key; + +DROP TABLE IF EXISTS table_for_rename; diff --git a/tests/queries/0_stateless/01276_alter_rename_column_materialized_expr.reference b/tests/queries/0_stateless/01276_alter_rename_column_materialized_expr.reference new file mode 100644 index 00000000000..5d721230db3 --- /dev/null +++ b/tests/queries/0_stateless/01276_alter_rename_column_materialized_expr.reference @@ -0,0 +1,90 @@ +2019-10-01 0 0 1 +2019-10-02 1 1 2 +2019-10-03 2 2 3 +2019-10-01 3 3 4 +2019-10-02 4 4 5 +2019-10-03 5 5 6 +2019-10-01 6 6 7 +2019-10-02 7 7 8 +2019-10-03 8 8 9 +CREATE TABLE default.table_for_rename\n(\n `date` Date, \n `key` UInt64, \n `value4` String, \n `value5` String, \n `value3` String MATERIALIZED concat(value4, \' + \', value5)\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 +2019-10-01 0 0 1 +2019-10-02 1 1 2 +2019-10-03 2 2 3 +2019-10-01 3 3 4 +2019-10-02 4 4 5 +2019-10-03 5 5 6 +2019-10-01 6 6 7 +2019-10-02 7 7 8 +2019-10-03 8 8 9 +-- insert after rename -- +2019-10-01 0 0 1 +2019-10-02 1 1 2 +2019-10-03 2 2 3 +2019-10-01 3 3 4 +2019-10-02 4 4 5 +2019-10-03 5 5 6 +2019-10-01 6 6 7 +2019-10-02 7 7 8 +2019-10-03 8 8 9 +2019-10-02 10 10 11 +2019-10-03 11 11 12 +2019-10-01 12 12 13 +2019-10-02 13 13 14 +2019-10-03 14 14 15 +2019-10-01 15 15 16 +2019-10-02 16 16 17 +2019-10-03 17 17 18 +2019-10-01 18 18 19 +2019-10-02 19 19 20 +-- rename columns back -- +CREATE TABLE default.table_for_rename\n(\n `date` Date, \n `key` UInt64, \n `value1` String, \n `value2` String, \n `value3` String MATERIALIZED concat(value1, \' + \', value2)\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 +2019-10-01 0 0 1 +2019-10-02 1 1 2 +2019-10-03 2 2 3 +2019-10-01 3 3 4 +2019-10-02 4 4 5 +2019-10-03 5 5 6 +2019-10-01 6 6 7 +2019-10-02 7 7 8 +2019-10-03 8 8 9 +2019-10-02 10 10 11 +2019-10-03 11 11 12 +2019-10-01 12 12 13 +2019-10-02 13 13 14 +2019-10-03 14 14 15 +2019-10-01 15 15 16 +2019-10-02 16 16 17 +2019-10-03 17 17 18 +2019-10-01 18 18 19 +2019-10-02 19 19 20 +-- insert after rename column -- +2019-10-01 0 0 1 +2019-10-02 1 1 2 +2019-10-03 2 2 3 +2019-10-01 3 3 4 +2019-10-02 4 4 5 +2019-10-03 5 5 6 +2019-10-01 6 6 7 +2019-10-02 7 7 8 +2019-10-03 8 8 9 +2019-10-02 10 10 11 +2019-10-03 11 11 12 +2019-10-01 12 12 13 +2019-10-02 13 13 14 +2019-10-03 14 14 15 +2019-10-01 15 15 16 +2019-10-02 16 16 17 +2019-10-03 17 17 18 +2019-10-01 18 18 19 +2019-10-02 19 19 20 +2019-10-03 20 20 21 +2019-10-01 21 21 22 +2019-10-02 22 22 23 +2019-10-03 23 23 24 +2019-10-01 24 24 25 +2019-10-02 25 25 26 +2019-10-03 26 26 27 +2019-10-01 27 27 28 +2019-10-02 28 28 29 +2019-10-03 29 29 30 diff --git a/tests/queries/0_stateless/01276_alter_rename_column_materialized_expr.sql b/tests/queries/0_stateless/01276_alter_rename_column_materialized_expr.sql new file mode 100644 index 00000000000..9089c52edf6 --- /dev/null +++ b/tests/queries/0_stateless/01276_alter_rename_column_materialized_expr.sql @@ -0,0 +1,37 @@ +DROP TABLE IF EXISTS table_for_rename; + +CREATE TABLE table_for_rename +( + date Date, + key UInt64, + value1 String, + value2 String, + value3 String MATERIALIZED concat(value1, ' + ', value2) +) +ENGINE = MergeTree() +PARTITION BY date +ORDER BY key; + +INSERT INTO table_for_rename (date, key, value1, value2) SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1) from numbers(9); +SELECT * FROM table_for_rename ORDER BY key; + +ALTER TABLE table_for_rename RENAME COLUMN value1 TO value4; +ALTER TABLE table_for_rename RENAME COLUMN value2 TO value5; +SHOW CREATE TABLE table_for_rename; +SELECT * FROM table_for_rename ORDER BY key; + +SELECT '-- insert after rename --'; +INSERT INTO table_for_rename (date, key, value4, value5) SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1) from numbers(10, 10); +SELECT * FROM table_for_rename ORDER BY key; + +SELECT '-- rename columns back --'; +ALTER TABLE table_for_rename RENAME COLUMN value4 TO value1; +ALTER TABLE table_for_rename RENAME COLUMN value5 TO value2; +SHOW CREATE TABLE table_for_rename; +SELECT * FROM table_for_rename ORDER BY key; + +SELECT '-- insert after rename column --'; +INSERT INTO table_for_rename (date, key, value1, value2) SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1) from numbers(20,10); +SELECT * FROM table_for_rename ORDER BY key; + +DROP TABLE IF EXISTS table_for_rename; diff --git a/tests/queries/0_stateless/01277_alter_rename_column_constraint_expr.reference b/tests/queries/0_stateless/01277_alter_rename_column_constraint_expr.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01277_alter_rename_column_constraint_expr.sql b/tests/queries/0_stateless/01277_alter_rename_column_constraint_expr.sql new file mode 100644 index 00000000000..72fbb045601 --- /dev/null +++ b/tests/queries/0_stateless/01277_alter_rename_column_constraint_expr.sql @@ -0,0 +1,43 @@ +DROP TABLE IF EXISTS table_for_rename; + +CREATE TABLE table_for_rename +( + date Date, + key UInt64, + value1 String, + value2 String, + value3 String, + CONSTRAINT cs_value1 CHECK toInt64(value1) < toInt64(value2), + CONSTRAINT cs_value2 CHECK toInt64(value2) < toInt64(value3) +) +ENGINE = MergeTree() +PARTITION BY date +ORDER BY key; + +INSERT INTO table_for_rename SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1), toString(number + 2) from numbers(9); +INSERT INTO table_for_rename SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1), toString(number) from numbers(9); ; --{serverError 469} + +SELECT * FROM table_for_rename ORDER BY key; + +ALTER TABLE table_for_rename RENAME COLUMN value1 TO value4; +ALTER TABLE table_for_rename RENAME COLUMN value2 TO value5; +SHOW CREATE TABLE table_for_rename; +SELECT * FROM table_for_rename ORDER BY key; + +SELECT '-- insert after rename --'; +INSERT INTO table_for_rename SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1), toString(number + 2) from numbers(10, 10); +INSERT INTO table_for_rename SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1), toString(number) from numbers(10, 10); ; --{serverError 469} +SELECT * FROM table_for_rename ORDER BY key; + +SELECT '-- rename columns back --'; +ALTER TABLE table_for_rename RENAME COLUMN value4 TO value1; +ALTER TABLE table_for_rename RENAME COLUMN value5 TO value2; +SHOW CREATE TABLE table_for_rename; +SELECT * FROM table_for_rename ORDER BY key; + +SELECT '-- insert after rename column --'; +INSERT INTO table_for_rename SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1), toString(number + 2) from numbers(20,10); +INSERT INTO table_for_rename SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number), toString(number + 2) from numbers(20, 10); ; --{serverError 469} +SELECT * FROM table_for_rename ORDER BY key; + +DROP TABLE IF EXISTS table_for_rename; From d255bb998a56751e7e159a72ddeb9618b2452b58 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Mon, 11 May 2020 20:20:26 +0300 Subject: [PATCH 078/738] VolumePtr instead of DiskPtr in IMergedBlockOutputStream and inherited classes --- .../MergeTree/IMergedBlockOutputStream.cpp | 2 +- src/Storages/MergeTree/IMergedBlockOutputStream.h | 2 +- src/Storages/MergeTree/MergedBlockOutputStream.cpp | 14 +++++++------- src/Storages/StorageMergeTree.cpp | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/Storages/MergeTree/IMergedBlockOutputStream.cpp b/src/Storages/MergeTree/IMergedBlockOutputStream.cpp index d146f705b0f..329a8ee4508 100644 --- a/src/Storages/MergeTree/IMergedBlockOutputStream.cpp +++ b/src/Storages/MergeTree/IMergedBlockOutputStream.cpp @@ -9,7 +9,7 @@ namespace DB IMergedBlockOutputStream::IMergedBlockOutputStream( const MergeTreeDataPartPtr & data_part) : storage(data_part->storage) - , disk(data_part->volume->getDisk()) + , volume(data_part->volume) , part_path(data_part->getFullRelativePath()) { } diff --git a/src/Storages/MergeTree/IMergedBlockOutputStream.h b/src/Storages/MergeTree/IMergedBlockOutputStream.h index 7b808ef6784..6a06d4b0c75 100644 --- a/src/Storages/MergeTree/IMergedBlockOutputStream.h +++ b/src/Storages/MergeTree/IMergedBlockOutputStream.h @@ -37,7 +37,7 @@ protected: protected: const MergeTreeData & storage; - DiskPtr disk; + VolumePtr volume; String part_path; static Block getBlockAndPermute(const Block & block, const Names & names, const IColumn::Permutation * permutation); diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 2b482ac7c29..e0760e87d00 100644 --- a/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -49,7 +49,7 @@ MergedBlockOutputStream::MergedBlockOutputStream( } } - disk->createDirectories(part_path); + volume->getDisk()->createDirectories(part_path); writer = data_part->getWriter(columns_list, skip_indices, default_codec, writer_settings); writer->initPrimaryIndex(); @@ -99,14 +99,14 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING || isCompactPart(new_part)) { - new_part->partition.store(storage, disk, part_path, checksums); + new_part->partition.store(storage, volume->getDisk(), part_path, checksums); if (new_part->minmax_idx.initialized) - new_part->minmax_idx.store(storage, disk, part_path, checksums); + new_part->minmax_idx.store(storage, volume->getDisk(), part_path, checksums); else if (rows_count) throw Exception("MinMax index was not initialized for new non-empty part " + new_part->name + ". It is a bug.", ErrorCodes::LOGICAL_ERROR); - auto count_out = disk->writeFile(part_path + "count.txt", 4096); + auto count_out = volume->getDisk()->writeFile(part_path + "count.txt", 4096); HashingWriteBuffer count_out_hashing(*count_out); writeIntText(rows_count, count_out_hashing); count_out_hashing.next(); @@ -117,7 +117,7 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( if (!new_part->ttl_infos.empty()) { /// Write a file with ttl infos in json format. - auto out = disk->writeFile(part_path + "ttl.txt", 4096); + auto out = volume->getDisk()->writeFile(part_path + "ttl.txt", 4096); HashingWriteBuffer out_hashing(*out); new_part->ttl_infos.write(out_hashing); checksums.files["ttl.txt"].file_size = out_hashing.count(); @@ -128,13 +128,13 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( { /// Write a file with a description of columns. - auto out = disk->writeFile(part_path + "columns.txt", 4096); + auto out = volume->getDisk()->writeFile(part_path + "columns.txt", 4096); part_columns.writeText(*out); } { /// Write file with checksums. - auto out = disk->writeFile(part_path + "checksums.txt", 4096); + auto out = volume->getDisk()->writeFile(part_path + "checksums.txt", 4096); checksums.write(*out); } diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index d2ee3eba053..1a9d088bb20 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -280,7 +280,7 @@ public: /// if we mutate part, than we should reserve space on the same disk, because mutations possible can create hardlinks if (is_mutation) - reserved_space = storage.tryReserveSpace(total_size, future_part_.parts[0]->volume->getDisk()); + reserved_space = storage.tryReserveSpace(total_size, future_part_.parts[0]->volume); else { IMergeTreeDataPart::TTLInfos ttl_infos; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index f2d98322060..38cbb4c28ba 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -1188,7 +1188,7 @@ bool StorageReplicatedMergeTree::tryExecutePartMutation(const StorageReplicatedM /// Once we mutate part, we must reserve space on the same disk, because mutations can possibly create hardlinks. /// Can throw an exception. - ReservationPtr reserved_space = reserveSpace(estimated_space_for_result, source_part->volume->getDisk()); + ReservationPtr reserved_space = reserveSpace(estimated_space_for_result, source_part->volume); auto table_lock = lockStructureForShare( false, RWLockImpl::NO_QUERY, storage_settings_ptr->lock_acquire_timeout_for_background_operations); From cb72b332e610cd5ae025132543aa76f9645d96ba Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 11 May 2020 20:21:57 +0300 Subject: [PATCH 079/738] utils/generate-ya-make$ ./generate-ya-make.sh --- src/Functions/ya.make | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Functions/ya.make b/src/Functions/ya.make index 43836039e3b..afce90f0dee 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -380,6 +380,7 @@ SRCS( toStartOfMinute.cpp toStartOfMonth.cpp toStartOfQuarter.cpp + toStartOfSecond.cpp toStartOfTenMinutes.cpp toStartOfYear.cpp toTime.cpp From d9b1d49ed04df5a86a465da4c27acf05c80f1eb9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 11 May 2020 20:32:26 +0300 Subject: [PATCH 080/738] Fix test with obvious race condition --- docker/test/split_build_smoke_test/run.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docker/test/split_build_smoke_test/run.sh b/docker/test/split_build_smoke_test/run.sh index 63cd5fada5b..eac9848030e 100755 --- a/docker/test/split_build_smoke_test/run.sh +++ b/docker/test/split_build_smoke_test/run.sh @@ -6,11 +6,14 @@ install_and_run_server() { mkdir /unpacked tar -xzf /package_folder/shared_build.tgz -C /unpacked --strip 1 LD_LIBRARY_PATH=/unpacked /unpacked/clickhouse-server --config /unpacked/config/config.xml >/var/log/clickhouse-server/stderr.log 2>&1 & - sleep 5 } run_client() { - LD_LIBRARY_PATH=/unpacked /unpacked/clickhouse-client --query "select 'OK'" 2>/var/log/clickhouse-server/clientstderr.log || echo "FAIL" + for i in {1..100}; do + sleep 1 + LD_LIBRARY_PATH=/unpacked /unpacked/clickhouse-client --query "select 'OK'" 2>/var/log/clickhouse-server/clientstderr.log && break + [[ $i == 100 ]] && echo 'FAIL' + done } install_and_run_server From 1efc372c0cbfb50d97e8928510ec42e0f0f82c85 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Mon, 11 May 2020 20:09:45 +0200 Subject: [PATCH 081/738] Fixing test_rename_with_parallel_select. --- tests/integration/test_rename_column/test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/integration/test_rename_column/test.py b/tests/integration/test_rename_column/test.py index 0086050cbed..ab7d5c8c74b 100644 --- a/tests/integration/test_rename_column/test.py +++ b/tests/integration/test_rename_column/test.py @@ -231,6 +231,10 @@ def test_rename_with_parallel_select(started_cluster): create_table(nodes, table_name) insert(node1, table_name, 1000) + select(node1, table_name, "num2", "999\n", poll=30) + select(node2, table_name, "num2", "999\n", poll=30) + select(node3, table_name, "num2", "999\n", poll=30) + p = Pool(15) tasks = [] for i in range(1): From 3505ccc2fcfbc2043e9799e7840a455efd4be5f9 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Mon, 11 May 2020 20:27:19 +0200 Subject: [PATCH 082/738] Increasing zookeeper_session_timeout setting to 15 sec. --- .../configs/config.d/zookeeper_session_timeout.xml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 tests/integration/test_rename_column/configs/config.d/zookeeper_session_timeout.xml diff --git a/tests/integration/test_rename_column/configs/config.d/zookeeper_session_timeout.xml b/tests/integration/test_rename_column/configs/config.d/zookeeper_session_timeout.xml new file mode 100644 index 00000000000..caa0ff11137 --- /dev/null +++ b/tests/integration/test_rename_column/configs/config.d/zookeeper_session_timeout.xml @@ -0,0 +1,6 @@ + + + + 15000 + + From 25cf8424912c111d8edcf98726cd0d8f48bc989a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 11 May 2020 23:05:08 +0300 Subject: [PATCH 083/738] Fix the issue with ODBC bridge and identifier_quoting_style = None #7984 --- src/Storages/transformQueryForExternalDatabase.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/transformQueryForExternalDatabase.cpp b/src/Storages/transformQueryForExternalDatabase.cpp index 839175c9b72..11b98b782e0 100644 --- a/src/Storages/transformQueryForExternalDatabase.cpp +++ b/src/Storages/transformQueryForExternalDatabase.cpp @@ -194,8 +194,8 @@ String transformQueryForExternalDatabase( std::stringstream out; IAST::FormatSettings settings(out, true); - settings.always_quote_identifiers = true; settings.identifier_quoting_style = identifier_quoting_style; + settings.always_quote_identifiers = identifier_quoting_style != IdentifierQuotingStyle::None; select->format(settings); From 4a142a44a0e114427e4ca3c308f67da531dc890b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 12 May 2020 00:18:50 +0300 Subject: [PATCH 084/738] Added a patch from Vitaly Stoyan --- src/Functions/FunctionFactory.cpp | 9 +++++++++ src/Functions/FunctionFactory.h | 2 ++ 2 files changed, 11 insertions(+) diff --git a/src/Functions/FunctionFactory.cpp b/src/Functions/FunctionFactory.cpp index 6ebfbc208cb..a19fc20bb40 100644 --- a/src/Functions/FunctionFactory.cpp +++ b/src/Functions/FunctionFactory.cpp @@ -56,6 +56,15 @@ FunctionOverloadResolverImplPtr FunctionFactory::getImpl( return res; } +std::vector FunctionFactory::getAllNames() const { + std::vector res; + res.reserve(functions.size()); + for (const auto& func : functions) { + res.emplace_back(func.first); + } + return res; +} + FunctionOverloadResolverPtr FunctionFactory::get( const std::string & name, const Context & context) const diff --git a/src/Functions/FunctionFactory.h b/src/Functions/FunctionFactory.h index 18847006d08..8544a8cec6b 100644 --- a/src/Functions/FunctionFactory.h +++ b/src/Functions/FunctionFactory.h @@ -39,6 +39,8 @@ public: registerFunction(name, &Function::create, case_sensitiveness); } + std::vector getAllNames() const; + /// Throws an exception if not found. FunctionOverloadResolverPtr get(const std::string & name, const Context & context) const; From 4a4961fb10425f39a43c5139a7b5f24506b644e6 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 12 May 2020 00:21:40 +0300 Subject: [PATCH 085/738] Update FunctionFactory.cpp --- src/Functions/FunctionFactory.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Functions/FunctionFactory.cpp b/src/Functions/FunctionFactory.cpp index a19fc20bb40..63f12188771 100644 --- a/src/Functions/FunctionFactory.cpp +++ b/src/Functions/FunctionFactory.cpp @@ -56,12 +56,12 @@ FunctionOverloadResolverImplPtr FunctionFactory::getImpl( return res; } -std::vector FunctionFactory::getAllNames() const { +std::vector FunctionFactory::getAllNames() const +{ std::vector res; res.reserve(functions.size()); - for (const auto& func : functions) { + for (const auto & func : functions) res.emplace_back(func.first); - } return res; } From 46e41f8d866910e33c728350e26893aeae8b6f70 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 12 May 2020 00:22:34 +0300 Subject: [PATCH 086/738] Update FunctionFactory.h --- src/Functions/FunctionFactory.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Functions/FunctionFactory.h b/src/Functions/FunctionFactory.h index 8544a8cec6b..ccaf2044693 100644 --- a/src/Functions/FunctionFactory.h +++ b/src/Functions/FunctionFactory.h @@ -39,6 +39,7 @@ public: registerFunction(name, &Function::create, case_sensitiveness); } + /// This function is used by YQL - internal Yandex product that depends on ClickHouse by source code. std::vector getAllNames() const; /// Throws an exception if not found. From eb97afa9be2f2a507049db86be643f97e61de239 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 12 May 2020 02:41:51 +0300 Subject: [PATCH 087/738] Add a test for INSERT into Buffer() with different order of columns (via MV) --- .../01277_buffer_column_order.reference | 0 .../0_stateless/01277_buffer_column_order.sql | 59 +++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 tests/queries/0_stateless/01277_buffer_column_order.reference create mode 100644 tests/queries/0_stateless/01277_buffer_column_order.sql diff --git a/tests/queries/0_stateless/01277_buffer_column_order.reference b/tests/queries/0_stateless/01277_buffer_column_order.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01277_buffer_column_order.sql b/tests/queries/0_stateless/01277_buffer_column_order.sql new file mode 100644 index 00000000000..8ddbc757392 --- /dev/null +++ b/tests/queries/0_stateless/01277_buffer_column_order.sql @@ -0,0 +1,59 @@ +-- Check for Block::sortColumns(), can be done using Buffer. + +drop table if exists out_01277; +drop table if exists in_01277; +drop table if exists buffer_01277; +drop table if exists mv_01277_1; +drop table if exists mv_01277_2; + +create table out_01277 +( + k1 Int, + k2 Int, + a1 Int, + a2 Int, + b1 Int, + b2 Int, + c Int +) Engine=Null(); +create table buffer_01277 as out_01277 Engine=Buffer(currentDatabase(), out_01277, 1, + 86400, 86400, + 1e5, 1e6, + 10e6, 100e6); +create table in_01277 as out_01277 Engine=Null(); + +-- differs in order of fields in SELECT clause +create materialized view mv_01277_1 to buffer_01277 as select k1, k2, a1, a2, b1, b2, c from in_01277; +create materialized view mv_01277_2 to buffer_01277 as select a1, a2, k1, k2, b1, b2, c from in_01277; + +-- column order is ignored, just for humans +insert into mv_01277_1 select + number k1, + number k2, + number a1, + number a2, + number b1, + number b2, + number c +from numbers(1); + +-- with wrong order in Block::sortColumns() triggers: +-- +-- Code: 171. DB::Exception: Received from localhost:9000. DB::Exception: Block structure mismatch in Buffer stream: different names of columns: +-- c Int32 Int32(size = 1), b2 Int32 Int32(size = 1), a2 Int32 Int32(size = 1), a1 Int32 Int32(size = 1), k2 Int32 Int32(size = 1), b1 Int32 Int32(size = 1), k1 Int32 Int32(size = 1) +-- c Int32 Int32(size = 1), b2 Int32 Int32(size = 1), k2 Int32 Int32(size = 1), a1 Int32 Int32(size = 1), b1 Int32 Int32(size = 1), k1 Int32 Int32(size = 1), a2 Int32 Int32(size = 1). +insert into mv_01277_2 select + number a1, + number a2, + number k1, + number k2, + number b1, + number b2, + number c +from numbers(1); + +drop table mv_01277_1; +drop table mv_01277_2; +drop table buffer_01277; +drop table out_01277; +drop table in_01277; From ce26c09d02fac215381ac20c19be62e299dd7bf6 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 12 May 2020 02:44:23 +0300 Subject: [PATCH 088/738] Make order of columns strict in Block::sortColumns() Before it depends on the buckets, since it uses plain unordered_set. --- src/Core/Block.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Core/Block.cpp b/src/Core/Block.cpp index d70ae0ccdb3..0b82ab001ee 100644 --- a/src/Core/Block.cpp +++ b/src/Core/Block.cpp @@ -13,6 +13,7 @@ #include #include +#include namespace DB @@ -403,7 +404,12 @@ Block Block::sortColumns() const { Block sorted_block; + /// std::unordered_map cannot be used to guarantee the sort order + std::map sorted_index_by_name; for (const auto & name : index_by_name) + sorted_index_by_name[name.first] = name.second; + + for (const auto & name : sorted_index_by_name) sorted_block.insert(data[name.second]); return sorted_block; From f61576fdfade56af35e292985a43a6aa19dcfe1f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 12 May 2020 02:53:03 +0300 Subject: [PATCH 089/738] Fix logical error in convertFieldToType --- src/Interpreters/convertFieldToType.cpp | 4 ++-- .../01277_convert_field_to_type_logical_error.reference | 0 .../0_stateless/01277_convert_field_to_type_logical_error.sql | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/01277_convert_field_to_type_logical_error.reference create mode 100644 tests/queries/0_stateless/01277_convert_field_to_type_logical_error.sql diff --git a/src/Interpreters/convertFieldToType.cpp b/src/Interpreters/convertFieldToType.cpp index 9fc4c3f9d04..3cb774596c0 100644 --- a/src/Interpreters/convertFieldToType.cpp +++ b/src/Interpreters/convertFieldToType.cpp @@ -32,9 +32,9 @@ namespace DB namespace ErrorCodes { extern const int ARGUMENT_OUT_OF_BOUND; - extern const int LOGICAL_ERROR; extern const int TYPE_MISMATCH; extern const int TOO_LARGE_STRING_SIZE; + extern const int CANNOT_CONVERT_TYPE; } @@ -201,7 +201,7 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID if (const auto * ptype = typeid_cast *>(&type)) return convertDecimalType(src, *ptype); if (!which_type.isDateOrDateTime() && !which_type.isUUID() && !which_type.isEnum()) - throw Exception{"Logical error: unknown numeric type " + type.getName(), ErrorCodes::LOGICAL_ERROR}; + throw Exception{"Cannot convert field to type " + type.getName(), ErrorCodes::CANNOT_CONVERT_TYPE}; if (which_type.isEnum() && (src.getType() == Field::Types::UInt64 || src.getType() == Field::Types::Int64)) { diff --git a/tests/queries/0_stateless/01277_convert_field_to_type_logical_error.reference b/tests/queries/0_stateless/01277_convert_field_to_type_logical_error.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01277_convert_field_to_type_logical_error.sql b/tests/queries/0_stateless/01277_convert_field_to_type_logical_error.sql new file mode 100644 index 00000000000..05295575cf1 --- /dev/null +++ b/tests/queries/0_stateless/01277_convert_field_to_type_logical_error.sql @@ -0,0 +1 @@ +SELECT -2487, globalNullIn(toIntervalMinute(-88074), 'qEkek..'), [-27.537293]; -- { serverError 70 } From b56e62ddf69d083e318ed71332aeefcf27106957 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 12 May 2020 03:11:58 +0300 Subject: [PATCH 090/738] Better DNS exception message --- contrib/poco | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/poco b/contrib/poco index 9b347d4ab71..be2ab90ba5d 160000 --- a/contrib/poco +++ b/contrib/poco @@ -1 +1 @@ -Subproject commit 9b347d4ab71e4436d8215aace978024e93462731 +Subproject commit be2ab90ba5dccd46919a116e3fe4fa77bb85063b From 0e117abd2b795a6ed13d90cff76eb7660ef1ac71 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 12 May 2020 03:33:53 +0300 Subject: [PATCH 091/738] Optimize Block::sortColumns() Suggested-by: @alexey-milovidov --- src/Core/Block.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Core/Block.cpp b/src/Core/Block.cpp index 0b82ab001ee..1cf67db679a 100644 --- a/src/Core/Block.cpp +++ b/src/Core/Block.cpp @@ -13,7 +13,6 @@ #include #include -#include namespace DB @@ -404,10 +403,11 @@ Block Block::sortColumns() const { Block sorted_block; - /// std::unordered_map cannot be used to guarantee the sort order - std::map sorted_index_by_name; - for (const auto & name : index_by_name) - sorted_index_by_name[name.first] = name.second; + /// std::unordered_map (index_by_name) cannot be used to guarantee the sort order + std::vector> sorted_index_by_name(index_by_name.begin(), index_by_name.end()); + std::sort(sorted_index_by_name.begin(), sorted_index_by_name.end(), [](const auto & lhs, const auto & rhs) { + return lhs.first < rhs.first; + }); for (const auto & name : sorted_index_by_name) sorted_block.insert(data[name.second]); From 4de7d7a84ffa543531ef0ba40d64e53afa7bd4d5 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 12 May 2020 03:51:58 +0300 Subject: [PATCH 092/738] Sort iterators to avoid extra std::string creation in Block::sortColumns() Since std::string is pretty heavy. Suggested-by: @alexey-milovidov --- src/Core/Block.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/Core/Block.cpp b/src/Core/Block.cpp index 1cf67db679a..95e7a91f0e1 100644 --- a/src/Core/Block.cpp +++ b/src/Core/Block.cpp @@ -404,13 +404,18 @@ Block Block::sortColumns() const Block sorted_block; /// std::unordered_map (index_by_name) cannot be used to guarantee the sort order - std::vector> sorted_index_by_name(index_by_name.begin(), index_by_name.end()); + std::vector sorted_index_by_name(index_by_name.size()); + { + size_t i = 0; + for (auto it = index_by_name.begin(); it != index_by_name.end(); ++it) + sorted_index_by_name[i++] = it; + } std::sort(sorted_index_by_name.begin(), sorted_index_by_name.end(), [](const auto & lhs, const auto & rhs) { - return lhs.first < rhs.first; + return lhs->first < rhs->first; }); - for (const auto & name : sorted_index_by_name) - sorted_block.insert(data[name.second]); + for (const auto & it : sorted_index_by_name) + sorted_block.insert(data[it->second]); return sorted_block; } From b9630859209a17ca1545de7e7c5093234fc7d06d Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 12 May 2020 04:04:25 +0300 Subject: [PATCH 093/738] Fix style check in Block::sortColumns() (sigh) --- src/Core/Block.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Core/Block.cpp b/src/Core/Block.cpp index 95e7a91f0e1..75b50377a64 100644 --- a/src/Core/Block.cpp +++ b/src/Core/Block.cpp @@ -410,7 +410,8 @@ Block Block::sortColumns() const for (auto it = index_by_name.begin(); it != index_by_name.end(); ++it) sorted_index_by_name[i++] = it; } - std::sort(sorted_index_by_name.begin(), sorted_index_by_name.end(), [](const auto & lhs, const auto & rhs) { + std::sort(sorted_index_by_name.begin(), sorted_index_by_name.end(), [](const auto & lhs, const auto & rhs) + { return lhs->first < rhs->first; }); From f26f8b99dc2a851458cd492df45f311f841ee7c4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 12 May 2020 04:28:53 +0300 Subject: [PATCH 094/738] Fix "Arcadia" build --- src/DataTypes/DataTypeDateTime64.h | 10 +--------- src/Functions/toStartOfInterval.cpp | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/DataTypes/DataTypeDateTime64.h b/src/DataTypes/DataTypeDateTime64.h index 2ec87b37e7a..85b77d88ef4 100644 --- a/src/DataTypes/DataTypeDateTime64.h +++ b/src/DataTypes/DataTypeDateTime64.h @@ -64,14 +64,6 @@ template class TransformDateTime64 : public Transform { private: - // ExtractName::name is Transform::name or default. - struct ExtractName - { - template static constexpr auto getName(...) { return ""; } - template> static constexpr auto getName(void*) { return U::name; } - static constexpr auto name = getName(nullptr); - }; - // Detect if Transform::execute is const or static method // with signature defined by template args (ignoring result type). template @@ -85,7 +77,7 @@ private: static constexpr bool TransformHasExecuteOverload_v = TransformHasExecuteOverload::value; public: - static constexpr auto name = ExtractName::name; + static constexpr auto name = Transform::name; using Transform::execute; diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index f644b410907..010a9d438eb 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -31,6 +31,8 @@ namespace template <> struct Transform { + static constexpr auto name = function_name; + static UInt16 execute(UInt16 d, UInt64 years, const DateLUTImpl & time_zone) { return time_zone.toStartOfYearInterval(DayNum(d), years); @@ -45,6 +47,8 @@ namespace template <> struct Transform { + static constexpr auto name = function_name; + static UInt16 execute(UInt16 d, UInt64 quarters, const DateLUTImpl & time_zone) { return time_zone.toStartOfQuarterInterval(DayNum(d), quarters); @@ -59,6 +63,8 @@ namespace template <> struct Transform { + static constexpr auto name = function_name; + static UInt16 execute(UInt16 d, UInt64 months, const DateLUTImpl & time_zone) { return time_zone.toStartOfMonthInterval(DayNum(d), months); @@ -73,6 +79,8 @@ namespace template <> struct Transform { + static constexpr auto name = function_name; + static UInt16 execute(UInt16 d, UInt64 weeks, const DateLUTImpl & time_zone) { return time_zone.toStartOfWeekInterval(DayNum(d), weeks); @@ -87,6 +95,8 @@ namespace template <> struct Transform { + static constexpr auto name = function_name; + static UInt32 execute(UInt16 d, UInt64 days, const DateLUTImpl & time_zone) { return time_zone.toStartOfDayInterval(DayNum(d), days); @@ -101,6 +111,8 @@ namespace template <> struct Transform { + static constexpr auto name = function_name; + static UInt32 execute(UInt16, UInt64, const DateLUTImpl &) { return dateIsNotSupported(function_name); } static UInt32 execute(UInt32 t, UInt64 hours, const DateLUTImpl & time_zone) { return time_zone.toStartOfHourInterval(t, hours); } @@ -109,6 +121,8 @@ namespace template <> struct Transform { + static constexpr auto name = function_name; + static UInt32 execute(UInt16, UInt64, const DateLUTImpl &) { return dateIsNotSupported(function_name); } static UInt32 execute(UInt32 t, UInt64 minutes, const DateLUTImpl & time_zone) @@ -120,6 +134,8 @@ namespace template <> struct Transform { + static constexpr auto name = function_name; + static UInt32 execute(UInt16, UInt64, const DateLUTImpl &) { return dateIsNotSupported(function_name); } static UInt32 execute(UInt32 t, UInt64 seconds, const DateLUTImpl & time_zone) From 2dd1a1d7b42fdf2a02b702e7c7bd0280878ac0a2 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 12 May 2020 05:12:08 +0300 Subject: [PATCH 095/738] Fix paths in compiler error messages #10434 --- CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ef9050b4fe2..b2453860d28 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -211,7 +211,8 @@ if (COMPILER_CLANG) endif() # Display absolute paths in error messages. Otherwise KDevelop fails to navigate to correct file and opens a new file instead. - set(COMPILER_FLAGS "${COMPILER_FLAGS} -fdiagnostics-absolute-paths") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdiagnostics-absolute-paths") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fdiagnostics-absolute-paths") option(ENABLE_THINLTO "Enable Thin LTO. Only applicable for clang. It's also suppressed when building with tests or sanitizers." ON) From 365acc65ea01e2905f78ca14bfe735353acd2ea0 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 12 May 2020 06:31:53 +0300 Subject: [PATCH 096/738] Better support for sanitizers --- base/daemon/BaseDaemon.cpp | 50 ++++++++++++++++++++++++++++++++++--- src/Columns/ColumnConst.cpp | 14 +++++++++++ 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index 2a8f569c6c7..9f7ace336d5 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -266,9 +266,6 @@ private: if (stack_trace.getSize()) { - /// Write bare stack trace (addresses) just in case if we will fail to print symbolized stack trace. - /// NOTE This still require memory allocations and mutex lock inside logger. BTW we can also print it to stderr using write syscalls. - std::stringstream bare_stacktrace; bare_stacktrace << "Stack trace:"; for (size_t i = stack_trace.getOffset(); i < stack_trace.getSize(); ++i) @@ -283,6 +280,49 @@ private: }; +#if defined(SANITIZER) +extern "C" void __sanitizer_set_death_callback(void (*)()); + +static void sanitizerDeathCallback() +{ + Logger * log = &Logger::get("BaseDaemon"); + + StringRef query_id = CurrentThread::getQueryId(); /// This is signal safe. + + { + std::stringstream message; + message << "(version " << VERSION_STRING << VERSION_OFFICIAL << ")"; + message << " (from thread " << getThreadId() << ")"; + if (query_id.size == 0) + message << " (no query)"; + else + message << " (query_id: " << query_id << ")"; + message << " Sanitizer trap."; + + LOG_FATAL(log, message.rdbuf()); + } + + /// Just in case print our own stack trace. In case when llvm-symbolizer does not work. + StackTrace stack_trace; + if (stack_trace.getSize()) + { + /// Write bare stack trace (addresses) just in case if we will fail to print symbolized stack trace. + /// NOTE This still require memory allocations and mutex lock inside logger. BTW we can also print it to stderr using write syscalls. + + std::stringstream bare_stacktrace; + bare_stacktrace << "Stack trace:"; + for (size_t i = stack_trace.getOffset(); i < stack_trace.getSize(); ++i) + bare_stacktrace << ' ' << stack_trace.getFrames()[i]; + + LOG_FATAL(log, bare_stacktrace.rdbuf()); + } + + /// Write symbolized stack trace line by line for better grep-ability. + stack_trace.toStringEveryLine([&](const std::string & s) { LOG_FATAL(log, s); }); +} +#endif + + /** To use with std::set_terminate. * Collects slightly more info than __gnu_cxx::__verbose_terminate_handler, * and send it to pipe. Other thread will read this info from pipe and asynchronously write it to log. @@ -659,6 +699,10 @@ void BaseDaemon::initializeTerminationAndSignalProcessing() add_signal_handler({SIGHUP, SIGUSR1}, closeLogsSignalHandler); add_signal_handler({SIGINT, SIGQUIT, SIGTERM}, terminateRequestedSignalHandler); +#if defined(SANITIZER) + __sanitizer_set_death_callback(sanitizerDeathCallback); +#endif + /// Set up Poco ErrorHandler for Poco Threads. static KillingErrorHandler killing_error_handler; Poco::ErrorHandler::set(&killing_error_handler); diff --git a/src/Columns/ColumnConst.cpp b/src/Columns/ColumnConst.cpp index 29aa09cb554..63f520a4c05 100644 --- a/src/Columns/ColumnConst.cpp +++ b/src/Columns/ColumnConst.cpp @@ -6,6 +6,11 @@ #include #include #include +#include + +#if defined(MEMORY_SANITIZER) + #include +#endif namespace DB @@ -27,6 +32,15 @@ ColumnConst::ColumnConst(const ColumnPtr & data_, size_t s_) if (data->size() != 1) throw Exception("Incorrect size of nested column in constructor of ColumnConst: " + toString(data->size()) + ", must be 1.", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH); + + /// Check that the value is initialized. We do it earlier, before it will be used, to ease debugging. +#if defined(MEMORY_SANITIZER) + if (data->isFixedAndContiguous()) + { + StringRef value = data->getDataAt(0); + __msan_check_mem_is_initialized(value.data, value.size); + } +#endif } ColumnPtr ColumnConst::convertToFullColumn() const From 2864012d1bf22f7b8bf2e7163a2c77b7f826e343 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 12 May 2020 06:36:09 +0300 Subject: [PATCH 097/738] Better support for sanitizers --- base/daemon/BaseDaemon.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index 9f7ace336d5..6625ba80b7b 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -266,6 +266,9 @@ private: if (stack_trace.getSize()) { + /// Write bare stack trace (addresses) just in case if we will fail to print symbolized stack trace. + /// NOTE This still require memory allocations and mutex lock inside logger. BTW we can also print it to stderr using write syscalls. + std::stringstream bare_stacktrace; bare_stacktrace << "Stack trace:"; for (size_t i = stack_trace.getOffset(); i < stack_trace.getSize(); ++i) @@ -306,9 +309,6 @@ static void sanitizerDeathCallback() StackTrace stack_trace; if (stack_trace.getSize()) { - /// Write bare stack trace (addresses) just in case if we will fail to print symbolized stack trace. - /// NOTE This still require memory allocations and mutex lock inside logger. BTW we can also print it to stderr using write syscalls. - std::stringstream bare_stacktrace; bare_stacktrace << "Stack trace:"; for (size_t i = stack_trace.getOffset(); i < stack_trace.getSize(); ++i) From 1d7b9b2cf2852aebbf00894fdd65704790dcda1b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 12 May 2020 06:37:21 +0300 Subject: [PATCH 098/738] Additional assert in ColumnVector --- src/Columns/ColumnVector.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Columns/ColumnVector.h b/src/Columns/ColumnVector.h index 258e7a87252..3551efe890c 100644 --- a/src/Columns/ColumnVector.h +++ b/src/Columns/ColumnVector.h @@ -201,6 +201,7 @@ public: Field operator[](size_t n) const override { + assert(n < data.size()); /// This assert is more strict than the corresponding assert inside PODArray. return data[n]; } From c811e1f0d0c439b74843770082569aa7fd5f9743 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 12 May 2020 06:41:35 +0300 Subject: [PATCH 099/738] Fix double whitespace --- tests/queries/0_stateless/01079_bad_alters_zookeeper.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/01079_bad_alters_zookeeper.sh b/tests/queries/0_stateless/01079_bad_alters_zookeeper.sh index b00215271fc..c53ff043eeb 100755 --- a/tests/queries/0_stateless/01079_bad_alters_zookeeper.sh +++ b/tests/queries/0_stateless/01079_bad_alters_zookeeper.sh @@ -5,10 +5,10 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS table_for_bad_alters"; -$CLICKHOUSE_CLIENT -n --query "CREATE TABLE table_for_bad_alters ( - key UInt64, - value1 UInt8, - value2 String +$CLICKHOUSE_CLIENT -n --query "CREATE TABLE table_for_bad_alters ( + key UInt64, + value1 UInt8, + value2 String ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/table_for_bad_alters', '1') ORDER BY key;" From d38e6b6840a2760d47d54fe8ed8202b2598acf89 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 12 May 2020 08:43:42 +0300 Subject: [PATCH 100/738] Fix MSan failure in cache dictionary --- src/Dictionaries/CacheDictionary.cpp | 8 ++++---- src/Dictionaries/CacheDictionary.inc.h | 18 ++++++++---------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/Dictionaries/CacheDictionary.cpp b/src/Dictionaries/CacheDictionary.cpp index 241610db77b..479f6ba8922 100644 --- a/src/Dictionaries/CacheDictionary.cpp +++ b/src/Dictionaries/CacheDictionary.cpp @@ -777,7 +777,7 @@ void CacheDictionary::updateThreadFunction() UpdateUnitPtr current_unit_ptr; - while (!update_request.empty() && update_queue.tryPop(current_unit_ptr)) + while (update_request.size() < current_queue_size + 1 && update_queue.tryPop(current_unit_ptr)) update_request.emplace_back(std::move(current_unit_ptr)); BunchUpdateUnit bunch_update_unit(update_request); @@ -817,7 +817,7 @@ void CacheDictionary::waitForCurrentUpdateFinish(UpdateUnitPtr & update_unit_ptr bool result = is_update_finished.wait_for( update_lock, std::chrono::milliseconds(timeout_for_wait), - [&] {return update_unit_ptr->is_done || update_unit_ptr->current_exception; }); + [&] { return update_unit_ptr->is_done || update_unit_ptr->current_exception; }); if (!result) { @@ -936,7 +936,6 @@ void CacheDictionary::update(BunchUpdateUnit & bunch_update_unit) const else cell.setExpiresAt(std::chrono::time_point::max()); - bunch_update_unit.informCallersAboutPresentId(id, cell_idx); /// mark corresponding id as found remaining_ids[id] = 1; @@ -962,7 +961,8 @@ void CacheDictionary::update(BunchUpdateUnit & bunch_update_unit) const } } - size_t not_found_num = 0, found_num = 0; + size_t not_found_num = 0; + size_t found_num = 0; /// Check which ids have not been found and require setting null_value for (const auto & id_found_pair : remaining_ids) diff --git a/src/Dictionaries/CacheDictionary.inc.h b/src/Dictionaries/CacheDictionary.inc.h index 746b2609a36..71f3f1857ce 100644 --- a/src/Dictionaries/CacheDictionary.inc.h +++ b/src/Dictionaries/CacheDictionary.inc.h @@ -38,12 +38,16 @@ template void CacheDictionary::getItemsNumberImpl( Attribute & attribute, const PaddedPODArray & ids, ResultArrayType & out, DefaultGetter && get_default) const { + /// First fill everything with default values + const auto rows = ext::size(ids); + for (const auto row : ext::range(0, rows)) + out[row] = get_default(row); + /// Mapping: -> { all indices `i` of `ids` such that `ids[i]` = } std::unordered_map> cache_expired_ids; std::unordered_map> cache_not_found_ids; auto & attribute_array = std::get>(attribute.arrays); - const auto rows = ext::size(ids); size_t cache_hit = 0; @@ -67,7 +71,8 @@ void CacheDictionary::getItemsNumberImpl( { const auto & cell_idx = find_result.cell_idx; const auto & cell = cells[cell_idx]; - out[row] = cell.isDefault() ? get_default(row) : static_cast(attribute_array[cell_idx]); + if (!cell.isDefault()) + out[row] = static_cast(attribute_array[cell_idx]); }; if (!find_result.valid) @@ -154,14 +159,7 @@ void CacheDictionary::getItemsNumberImpl( out[row] = static_cast(attribute_value); }; - auto on_id_not_found = [&] (const auto id, const auto) - { - for (const size_t row : cache_not_found_ids[id]) - out[row] = get_default(row); - - for (const size_t row : cache_expired_ids[id]) - out[row] = get_default(row); - }; + auto on_id_not_found = [&] (auto, auto) {}; /// Request new values auto update_unit_ptr = std::make_shared(required_ids, on_cell_updated, on_id_not_found); From 46137881b8af68738c9c4c98818c97b092e9f702 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 12 May 2020 11:34:13 +0300 Subject: [PATCH 101/738] Fix build. --- src/DataStreams/PushingToViewsBlockOutputStream.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DataStreams/PushingToViewsBlockOutputStream.cpp b/src/DataStreams/PushingToViewsBlockOutputStream.cpp index c5a552eed90..ce0922bf282 100644 --- a/src/DataStreams/PushingToViewsBlockOutputStream.cpp +++ b/src/DataStreams/PushingToViewsBlockOutputStream.cpp @@ -90,7 +90,7 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream( else out = std::make_shared(dependent_table, *views_context, ASTPtr()); - views.emplace_back(ViewInfo{std::move(query), database_table, std::move(out)}); + views.emplace_back(ViewInfo{std::move(query), database_table, std::move(out), nullptr}); } /// Do not push to destination table if the flag is set From 5d13327ad83861945bbd3c94d0c2d7301e8bf058 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 12 May 2020 12:22:01 +0300 Subject: [PATCH 102/738] Try fix merge with master. --- src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index fdd7612d1c2..4c0eb1a479c 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -1203,7 +1203,6 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal( Processors processors; for (auto & pipe : pipes) { - pipe.addSimpleTransform(std::make_shared(pipe.getHeader(), projection)); auto pipe_processors = std::move(pipe).detachProcessors(); processors.insert(processors.end(), pipe_processors.begin(), pipe_processors.end()); } @@ -1218,6 +1217,9 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal( pipes.front().addProcessors(copiers); pipes.front().addProcessors(merges); + for (auto & pipe : pipes) + pipe.addSimpleTransform(std::make_shared(pipe.getHeader(), projection)); + return pipes; } From 8bea42e2505c2517a721a4b14ad104e028b860cf Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Tue, 12 May 2020 12:23:18 +0300 Subject: [PATCH 103/738] Fix includes in ProxyResolverConfiguration.h --- src/Disks/S3/ProxyResolverConfiguration.cpp | 1 + src/Disks/S3/ProxyResolverConfiguration.h | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Disks/S3/ProxyResolverConfiguration.cpp b/src/Disks/S3/ProxyResolverConfiguration.cpp index dd5af0990b8..bf06804cd23 100644 --- a/src/Disks/S3/ProxyResolverConfiguration.cpp +++ b/src/Disks/S3/ProxyResolverConfiguration.cpp @@ -1,6 +1,7 @@ #include "ProxyResolverConfiguration.h" #include +#include #include "Poco/StreamCopier.h" #include #include diff --git a/src/Disks/S3/ProxyResolverConfiguration.h b/src/Disks/S3/ProxyResolverConfiguration.h index 2944121a078..0b23ae77c4a 100644 --- a/src/Disks/S3/ProxyResolverConfiguration.h +++ b/src/Disks/S3/ProxyResolverConfiguration.h @@ -1,7 +1,6 @@ #pragma once #include "ProxyConfiguration.h" -#include namespace DB::S3 { From 2521a37c9464e48381900ceba5b86fff70d5faf3 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 12 May 2020 12:30:08 +0300 Subject: [PATCH 104/738] Some fixes at ru merge-tree-settings.md (#10837) --- docs/ru/operations/settings/merge-tree-settings.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/ru/operations/settings/merge-tree-settings.md b/docs/ru/operations/settings/merge-tree-settings.md index 0f5e2fe5d47..c0123cfb5f8 100644 --- a/docs/ru/operations/settings/merge-tree-settings.md +++ b/docs/ru/operations/settings/merge-tree-settings.md @@ -1,6 +1,6 @@ # Настройки MergeTree таблиц {#merge-tree-settings} -Значения настроек merge-tree (для всех MergeTree таблиц) можно посмотреть в таблице `system.merge_tree_settings`, их можно переопределить в `config.xml` в секции `merge_tree`, или задать в секции `SETTINGS` у каждой таблицы. +Значения настроек для всех MergeTree таблиц можно посмотреть в таблице `system.merge_tree_settings`, их можно переопределить в `config.xml` в секции `merge_tree`, или задать в секции `SETTINGS` у каждой таблицы. Пример переопределения в `config.xml`: @@ -97,7 +97,7 @@ Eсли суммарное число активных кусков во все Значение по умолчанию: 100. -Команда `Insert` создает один или несколько блоков (кусков). При вставке в Replicated таблицы ClickHouse для [дедупликации вставок](../../engines/table-engines/mergetree-family/replication/) записывает в Zookeeper хеш-суммы созданных кусков. Но хранятся хеш-суммы не всех кусков, а только последние `replicated_deduplication_window`. Наиболее старые хеш-суммы удаляются из Zookeeper. +Команда `Insert` создает один или несколько блоков (кусков). При вставке в Replicated таблицы ClickHouse для [дедупликации вставок](../../engines/table-engines/mergetree-family/replication.md) записывает в Zookeeper хеш-суммы созданных кусков. Но хранятся хеш-суммы не всех кусков, а только последние `replicated_deduplication_window`. Наиболее старые хеш-суммы удаляются из Zookeeper. Большое число `replicated_deduplication_window` замедляет `Insert`-ы. Хеш-сумма рассчитывается от композиции имен и типов полей, а также данных вставленного куска (потока байт). ## replicated\_deduplication\_window\_seconds {#replicated-deduplication-window-seconds} @@ -110,7 +110,7 @@ Eсли суммарное число активных кусков во все Значение по умолчанию: 604800 (1 неделя). -Аналогично [replicated_deduplication_window](#replicated_deduplication_window), задает, сколько времени хранить хеш-суммы блоков для дедупликции `Insert`-в. Хеш-суммы старше `replicated_deduplication_window_seconds` удаляются из Zookeeper, даже если их меньше чем `replicated_deduplication_window`. +Аналогично [replicated_deduplication_window](#replicated-deduplication-window), задает, сколько времени хранить хеш-суммы блоков для дедупликции `Insert`-в. Хеш-суммы старше `replicated_deduplication_window_seconds` удаляются из Zookeeper, даже если их меньше чем `replicated_deduplication_window`. ## old\_parts\_lifetime {#old-parts-lifetime} @@ -181,4 +181,4 @@ Eсли суммарное число активных кусков во все При старте ClickHouse читает все куски всех таблиц (читает файлы с метаданными кусков), чтобы построить в ОЗУ список всех кусков. В некоторых системах с большим количеством кусков этот процесс может занимать длительное время, и это время можно сократить, увеличив `max_part_loading_threads` (если при этом процессе есть недозагруженность CPU и диска). -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/merge_tree_settings/) +{## [Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/merge-tree-settings/) ##} From 723bebafb1d04328d4329a46f6c2f60322f534c4 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 12 May 2020 12:32:28 +0300 Subject: [PATCH 105/738] CLICKHOUSE-4862: fix link (#10836) --- docs/ru/sql-reference/statements/create.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/statements/create.md b/docs/ru/sql-reference/statements/create.md index 67d19ad991b..2a57ac0a143 100644 --- a/docs/ru/sql-reference/statements/create.md +++ b/docs/ru/sql-reference/statements/create.md @@ -128,7 +128,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] ### Выражение для TTL {#vyrazhenie-dlia-ttl} -Определяет время хранения значений. Может быть указано только для таблиц семейства MergeTree. Подробнее смотрите в [TTL для столбцов и таблиц](../../sql-reference/statements/create.md#table_engine-mergetree-ttl). +Определяет время хранения значений. Может быть указано только для таблиц семейства MergeTree. Подробнее смотрите в [TTL для столбцов и таблиц](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-ttl). ### Кодеки сжатия столбцов {#codecs} From ff0ae624ef199fa515b7d60e482c88c8fab9adff Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 12 May 2020 12:49:36 +0300 Subject: [PATCH 106/738] CLICKHOUSE-4862: some markdown fixes @ ru mergetree.md (#10835) --- .../mergetree-family/mergetree.md | 64 ++++++------------- 1 file changed, 21 insertions(+), 43 deletions(-) diff --git a/docs/ru/engines/table-engines/mergetree-family/mergetree.md b/docs/ru/engines/table-engines/mergetree-family/mergetree.md index d674a41d2b3..d1334f4db25 100644 --- a/docs/ru/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/mergetree.md @@ -6,21 +6,13 @@ Основные возможности: -- Хранит данные, отсортированные по первичному ключу. +- **Хранит данные, отсортированные по первичному ключу.** Это позволяет создавать разреженный индекс небольшого объёма, который позволяет быстрее находить данные. - Это позволяет создавать разреженный индекс небольшого объёма, который позволяет быстрее находить данные. +- **Позволяет оперировать партициями, если задан [ключ партиционирования](custom-partitioning-key.md).** ClickHouse поддерживает отдельные операции с партициями, которые работают эффективнее, чем общие операции с этим же результатом над этими же данными. Также, ClickHouse автоматически отсекает данные по партициям там, где ключ партиционирования указан в запросе. Это также увеличивает эффективность выполнения запросов. -- Позволяет оперировать партициями, если задан [ключ партиционирования](custom-partitioning-key.md). +- **Поддерживает репликацию данных.** Для этого используется семейство таблиц `ReplicatedMergeTree`. Подробнее читайте в разделе [Репликация данных](replication.md). - ClickHouse поддерживает отдельные операции с партициями, которые работают эффективнее, чем общие операции с этим же результатом над этими же данными. Также, ClickHouse автоматически отсекает данные по партициям там, где ключ партиционирования указан в запросе. Это также увеличивает эффективность выполнения запросов. - -- Поддерживает репликацию данных. - - Для этого используется семейство таблиц `ReplicatedMergeTree`. Подробнее читайте в разделе [Репликация данных](replication.md). - -- Поддерживает сэмплирование данных. - - При необходимости можно задать способ сэмплирования данных в таблице. +- **Поддерживает сэмплирование данных.** При необходимости можно задать способ сэмплирования данных в таблице. !!! info "Info" Движок [Merge](../special/merge.md#merge) не относится к семейству `*MergeTree`. @@ -53,44 +45,30 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] - `ENGINE` — имя и параметры движка. `ENGINE = MergeTree()`. `MergeTree` не имеет параметров. -- `PARTITION BY` — [ключ партиционирования](custom-partitioning-key.md). +- `PARTITION BY` — [ключ партиционирования](custom-partitioning-key.md). Для партиционирования по месяцам используйте выражение `toYYYYMM(date_column)`, где `date_column` — столбец с датой типа [Date](../../../engines/table_engines/mergetree_family/mergetree.md). В этом случае имена партиций имеют формат `"YYYYMM"`. - Для партиционирования по месяцам используйте выражение `toYYYYMM(date_column)`, где `date_column` — столбец с датой типа [Date](../../../engines/table_engines/mergetree_family/mergetree.md). В этом случае имена партиций имеют формат `"YYYYMM"`. +- `ORDER BY` — ключ сортировки. Кортеж столбцов или произвольных выражений. Пример: `ORDER BY (CounterID, EventDate)`. -- `ORDER BY` — ключ сортировки. +- `PRIMARY KEY` — первичный ключ, если он [отличается от ключа сортировки](#pervichnyi-kliuch-otlichnyi-ot-kliucha-sortirovki). По умолчанию первичный ключ совпадает с ключом сортировки (который задаётся секцией `ORDER BY`.) Поэтому в большинстве случаев секцию `PRIMARY KEY` отдельно указывать не нужно. - Кортеж столбцов или произвольных выражений. Пример: `ORDER BY (CounterID, EventDate)`. +- `SAMPLE BY` — выражение для сэмплирования. Если используется выражение для сэмплирования, то первичный ключ должен содержать его. Пример: `SAMPLE BY intHash32(UserID) ORDER BY (CounterID, EventDate, intHash32(UserID))`. -- `PRIMARY KEY` — первичный ключ, если он [отличается от ключа сортировки](#pervichnyi-kliuch-otlichnyi-ot-kliucha-sortirovki). - - По умолчанию первичный ключ совпадает с ключом сортировки (который задаётся секцией `ORDER BY`.) Поэтому в большинстве случаев секцию `PRIMARY KEY` отдельно указывать не нужно. - -- `SAMPLE BY` — выражение для сэмплирования. - - Если используется выражение для сэмплирования, то первичный ключ должен содержать его. Пример: - - `SAMPLE BY intHash32(UserID) ORDER BY (CounterID, EventDate, intHash32(UserID))`. - -- `TTL` — список правил, определяющих длительности хранения строк, а также задающих правила перемещения частей на определённые тома или диски. - - Выражение должно возвращать столбец `Date` или `DateTime`. Пример: `TTL date + INTERVAL 1 DAY`. - - Тип правила `DELETE|TO DISK 'xxx'|TO VOLUME 'xxx'` указывает действие, которое будет выполнено с частью, удаление строк (прореживание), перемещение (при выполнении условия для всех строк части) на определённый диск (`TO DISK 'xxx'`) или том (`TO VOLUME 'xxx'`). Поведение по умолчанию соответствует удалению строк (`DELETE`). В списке правил может быть указано только одно выражение с поведением `DELETE`. - - Дополнительные сведения смотрите в разделе [TTL для столбцов и таблиц](#table_engine-mergetree-ttl) +- `TTL` — список правил, определяющих длительности хранения строк, а также задающих правила перемещения частей на определённые тома или диски. Выражение должно возвращать столбец `Date` или `DateTime`. Пример: `TTL date + INTERVAL 1 DAY`. + - Тип правила `DELETE|TO DISK 'xxx'|TO VOLUME 'xxx'` указывает действие, которое будет выполнено с частью, удаление строк (прореживание), перемещение (при выполнении условия для всех строк части) на определённый диск (`TO DISK 'xxx'`) или том (`TO VOLUME 'xxx'`). + - Поведение по умолчанию соответствует удалению строк (`DELETE`). В списке правил может быть указано только одно выражение с поведением `DELETE`. + - Дополнительные сведения смотрите в разделе [TTL для столбцов и таблиц](#table_engine-mergetree-ttl) - `SETTINGS` — дополнительные параметры, регулирующие поведение `MergeTree`: - - `index_granularity` — максимальное количество строк данных между засечками индекса. По умолчанию — 8192. Смотрите [Хранение данных](#mergetree-data-storage). - - `index_granularity_bytes` — максимальный размер гранул данных в байтах. По умолчанию — 10Mb. Чтобы ограничить размер гранул только количеством строк, установите значение 0 (не рекомендовано). Смотрите [Хранение данных](#mergetree-data-storage). - - `enable_mixed_granularity_parts` — включает или выключает переход к ограничению размера гранул с помощью настройки `index_granularity_bytes`. До версии 19.11, размер гранул ограничивался только настройкой `index_granularity`. Настройка `index_granularity_bytes` улучшает производительность ClickHouse при выборке данных из таблиц с большими (десятки и сотни мегабайтов) строками. Если у вас есть таблицы с большими строками, можно включить эту настройку, чтобы повысить эффективность запросов `SELECT`. - - `use_minimalistic_part_header_in_zookeeper` — Способ хранения заголовков кусков данных в ZooKeeper. Если `use_minimalistic_part_header_in_zookeeper = 1`, то ZooKeeper хранит меньше данных. Подробнее читайте в [описании настройки](../../../operations/server_configuration_parameters/settings.md#server-settings-use_minimalistic_part_header_in_zookeeper) в разделе "Конфигурационные параметры сервера". - - `min_merge_bytes_to_use_direct_io` — минимальный объём данных при слиянии, необходимый для прямого (небуферизованного) чтения/записи (direct I/O) на диск. При слиянии частей данных ClickHouse вычисляет общий объём хранения всех данных, подлежащих слиянию. Если общий объём хранения всех данных для чтения превышает `min_bytes_to_use_direct_io` байт, тогда ClickHouse использует флаг `O_DIRECT` при чтении данных с диска. Если `min_merge_bytes_to_use_direct_io = 0`, тогда прямой ввод-вывод отключен. Значение по умолчанию: `10 * 1024 * 1024 * 1024` байтов. - - - `merge_with_ttl_timeout` — минимальное время в секундах перед повторным слиянием с TTL. По умолчанию — 86400 (1 день). - - `write_final_mark` — включает или отключает запись последней засечки индекса в конце куска данных, указывающей за последний байт. По умолчанию — 1. Не отключайте её. - - `merge_max_block_size` — Максимальное количество строк в блоке для операций слияния. Значение по умолчанию: 8192. - - `storage_policy` — политика хранения данных. Смотрите [Хранение данных таблицы на нескольких блочных устройствах](#table_engine-mergetree-multiple-volumes). + - `index_granularity` — максимальное количество строк данных между засечками индекса. По умолчанию — 8192. Смотрите [Хранение данных](#mergetree-data-storage). + - `index_granularity_bytes` — максимальный размер гранул данных в байтах. По умолчанию — 10Mb. Чтобы ограничить размер гранул только количеством строк, установите значение 0 (не рекомендовано). Смотрите [Хранение данных](#mergetree-data-storage). + - `enable_mixed_granularity_parts` — включает или выключает переход к ограничению размера гранул с помощью настройки `index_granularity_bytes`. Настройка `index_granularity_bytes` улучшает производительность ClickHouse при выборке данных из таблиц с большими (десятки и сотни мегабайтов) строками. Если у вас есть таблицы с большими строками, можно включить эту настройку, чтобы повысить эффективность запросов `SELECT`. + - `use_minimalistic_part_header_in_zookeeper` — Способ хранения заголовков кусков данных в ZooKeeper. Если `use_minimalistic_part_header_in_zookeeper = 1`, то ZooKeeper хранит меньше данных. Подробнее читайте в [описании настройки](../../../operations/server_configuration_parameters/settings.md#server-settings-use_minimalistic_part_header_in_zookeeper) в разделе "Конфигурационные параметры сервера". + - `min_merge_bytes_to_use_direct_io` — минимальный объём данных при слиянии, необходимый для прямого (небуферизованного) чтения/записи (direct I/O) на диск. При слиянии частей данных ClickHouse вычисляет общий объём хранения всех данных, подлежащих слиянию. Если общий объём хранения всех данных для чтения превышает `min_bytes_to_use_direct_io` байт, тогда ClickHouse использует флаг `O_DIRECT` при чтении данных с диска. Если `min_merge_bytes_to_use_direct_io = 0`, тогда прямой ввод-вывод отключен. Значение по умолчанию: `10 * 1024 * 1024 * 1024` байтов. + - `merge_with_ttl_timeout` — минимальное время в секундах перед повторным слиянием с TTL. По умолчанию — 86400 (1 день). + - `write_final_mark` — включает или отключает запись последней засечки индекса в конце куска данных, указывающей за последний байт. По умолчанию — 1. Не отключайте её. + - `merge_max_block_size` — Максимальное количество строк в блоке для операций слияния. Значение по умолчанию: 8192. + - `storage_policy` — политика хранения данных. Смотрите [Хранение данных таблицы на нескольких блочных устройствах](#table_engine-mergetree-multiple-volumes). **Пример задания секций** From 979779e06ff298ad328b24bef527d77e4cb76c81 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 12 May 2020 14:26:44 +0300 Subject: [PATCH 107/738] Fix constraints after column rename --- .../CheckConstraintsBlockOutputStream.cpp | 5 +- src/Storages/AlterCommands.cpp | 3 + src/Storages/ConstraintsDescription.cpp | 7 +- src/Storages/ConstraintsDescription.h | 2 +- ...7_alter_rename_column_constraint.reference | 90 +++++++++++++++++++ .../01277_alter_rename_column_constraint.sql | 43 +++++++++ ...name_column_constraint_zookeeper.reference | 90 +++++++++++++++++++ ...ter_rename_column_constraint_zookeeper.sql | 43 +++++++++ 8 files changed, 277 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/01277_alter_rename_column_constraint.reference create mode 100644 tests/queries/0_stateless/01277_alter_rename_column_constraint.sql create mode 100644 tests/queries/0_stateless/01277_alter_rename_column_constraint_zookeeper.reference create mode 100644 tests/queries/0_stateless/01277_alter_rename_column_constraint_zookeeper.sql diff --git a/src/DataStreams/CheckConstraintsBlockOutputStream.cpp b/src/DataStreams/CheckConstraintsBlockOutputStream.cpp index 25e4340f655..1b87cf29dd4 100644 --- a/src/DataStreams/CheckConstraintsBlockOutputStream.cpp +++ b/src/DataStreams/CheckConstraintsBlockOutputStream.cpp @@ -61,10 +61,11 @@ void CheckConstraintsBlockOutputStream::write(const Block & block) std::stringstream exception_message; - exception_message << "Constraint " << backQuote(constraints.constraints[i]->name) + auto constraint_ptr = constraints.constraints[i]->as(); + exception_message << "Constraint " << backQuote(constraint_ptr->name) << " for table " << table_id.getNameForLogs() << " is violated at row " << (rows_written + row_idx + 1) - << ". Expression: (" << serializeAST(*(constraints.constraints[i]->expr), true) << ")" + << ". Expression: (" << serializeAST(*(constraint_ptr->expr), true) << ")" << ". Column values"; bool first = true; diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 121974c8b82..67bd88d10a8 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -467,6 +467,9 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata) const } if (metadata.ttl_for_table_ast) rename_visitor.visit(metadata.ttl_for_table_ast); + + for (auto & constraint : metadata.constraints.constraints) + rename_visitor.visit(constraint); } else throw Exception("Wrong parameter type in ALTER query", ErrorCodes::LOGICAL_ERROR); diff --git a/src/Storages/ConstraintsDescription.cpp b/src/Storages/ConstraintsDescription.cpp index 8c38af0cd5e..023a94a5628 100644 --- a/src/Storages/ConstraintsDescription.cpp +++ b/src/Storages/ConstraintsDescription.cpp @@ -33,7 +33,7 @@ ConstraintsDescription ConstraintsDescription::parse(const String & str) ASTPtr list = parseQuery(parser, str, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); for (const auto & constraint : list->children) - res.constraints.push_back(std::dynamic_pointer_cast(constraint)); + res.constraints.push_back(constraint); return res; } @@ -46,9 +46,10 @@ ConstraintsExpressions ConstraintsDescription::getExpressions(const DB::Context for (const auto & constraint : constraints) { // SyntaxAnalyzer::analyze has query as non-const argument so to avoid accidental query changes we clone it - ASTPtr expr = constraint->expr->clone(); + auto constraint_ptr = constraint->as(); + ASTPtr expr = constraint_ptr->expr->clone(); auto syntax_result = SyntaxAnalyzer(context).analyze(expr, source_columns_); - res.push_back(ExpressionAnalyzer(constraint->expr->clone(), syntax_result, context).getActions(false)); + res.push_back(ExpressionAnalyzer(constraint_ptr->expr->clone(), syntax_result, context).getActions(false)); } return res; } diff --git a/src/Storages/ConstraintsDescription.h b/src/Storages/ConstraintsDescription.h index 3ced0e8ddc9..b7bfc1a1acf 100644 --- a/src/Storages/ConstraintsDescription.h +++ b/src/Storages/ConstraintsDescription.h @@ -6,7 +6,7 @@ namespace DB { -using ConstraintsASTs = std::vector>; +using ConstraintsASTs = std::vector; using ConstraintsExpressions = std::vector; struct ConstraintsDescription diff --git a/tests/queries/0_stateless/01277_alter_rename_column_constraint.reference b/tests/queries/0_stateless/01277_alter_rename_column_constraint.reference new file mode 100644 index 00000000000..cb1842f95da --- /dev/null +++ b/tests/queries/0_stateless/01277_alter_rename_column_constraint.reference @@ -0,0 +1,90 @@ +2019-10-01 0 0 1 2 +2019-10-02 1 1 2 3 +2019-10-03 2 2 3 4 +2019-10-01 3 3 4 5 +2019-10-02 4 4 5 6 +2019-10-03 5 5 6 7 +2019-10-01 6 6 7 8 +2019-10-02 7 7 8 9 +2019-10-03 8 8 9 10 +CREATE TABLE default.table_for_rename\n(\n `date` Date, \n `key` UInt64, \n `value4` String, \n `value5` String, \n `value3` String, \n CONSTRAINT cs_value1 CHECK toInt64(value4) < toInt64(value5), \n CONSTRAINT cs_value2 CHECK toInt64(value5) < toInt64(value3)\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 +2019-10-01 0 0 1 2 +2019-10-02 1 1 2 3 +2019-10-03 2 2 3 4 +2019-10-01 3 3 4 5 +2019-10-02 4 4 5 6 +2019-10-03 5 5 6 7 +2019-10-01 6 6 7 8 +2019-10-02 7 7 8 9 +2019-10-03 8 8 9 10 +-- insert after rename -- +2019-10-01 0 0 1 2 +2019-10-02 1 1 2 3 +2019-10-03 2 2 3 4 +2019-10-01 3 3 4 5 +2019-10-02 4 4 5 6 +2019-10-03 5 5 6 7 +2019-10-01 6 6 7 8 +2019-10-02 7 7 8 9 +2019-10-03 8 8 9 10 +2019-10-02 10 10 11 12 +2019-10-03 11 11 12 13 +2019-10-01 12 12 13 14 +2019-10-02 13 13 14 15 +2019-10-03 14 14 15 16 +2019-10-01 15 15 16 17 +2019-10-02 16 16 17 18 +2019-10-03 17 17 18 19 +2019-10-01 18 18 19 20 +2019-10-02 19 19 20 21 +-- rename columns back -- +CREATE TABLE default.table_for_rename\n(\n `date` Date, \n `key` UInt64, \n `value1` String, \n `value2` String, \n `value3` String, \n CONSTRAINT cs_value1 CHECK toInt64(value1) < toInt64(value2), \n CONSTRAINT cs_value2 CHECK toInt64(value2) < toInt64(value3)\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 +2019-10-01 0 0 1 2 +2019-10-02 1 1 2 3 +2019-10-03 2 2 3 4 +2019-10-01 3 3 4 5 +2019-10-02 4 4 5 6 +2019-10-03 5 5 6 7 +2019-10-01 6 6 7 8 +2019-10-02 7 7 8 9 +2019-10-03 8 8 9 10 +2019-10-02 10 10 11 12 +2019-10-03 11 11 12 13 +2019-10-01 12 12 13 14 +2019-10-02 13 13 14 15 +2019-10-03 14 14 15 16 +2019-10-01 15 15 16 17 +2019-10-02 16 16 17 18 +2019-10-03 17 17 18 19 +2019-10-01 18 18 19 20 +2019-10-02 19 19 20 21 +-- insert after rename column -- +2019-10-01 0 0 1 2 +2019-10-02 1 1 2 3 +2019-10-03 2 2 3 4 +2019-10-01 3 3 4 5 +2019-10-02 4 4 5 6 +2019-10-03 5 5 6 7 +2019-10-01 6 6 7 8 +2019-10-02 7 7 8 9 +2019-10-03 8 8 9 10 +2019-10-02 10 10 11 12 +2019-10-03 11 11 12 13 +2019-10-01 12 12 13 14 +2019-10-02 13 13 14 15 +2019-10-03 14 14 15 16 +2019-10-01 15 15 16 17 +2019-10-02 16 16 17 18 +2019-10-03 17 17 18 19 +2019-10-01 18 18 19 20 +2019-10-02 19 19 20 21 +2019-10-03 20 20 21 22 +2019-10-01 21 21 22 23 +2019-10-02 22 22 23 24 +2019-10-03 23 23 24 25 +2019-10-01 24 24 25 26 +2019-10-02 25 25 26 27 +2019-10-03 26 26 27 28 +2019-10-01 27 27 28 29 +2019-10-02 28 28 29 30 +2019-10-03 29 29 30 31 diff --git a/tests/queries/0_stateless/01277_alter_rename_column_constraint.sql b/tests/queries/0_stateless/01277_alter_rename_column_constraint.sql new file mode 100644 index 00000000000..72fbb045601 --- /dev/null +++ b/tests/queries/0_stateless/01277_alter_rename_column_constraint.sql @@ -0,0 +1,43 @@ +DROP TABLE IF EXISTS table_for_rename; + +CREATE TABLE table_for_rename +( + date Date, + key UInt64, + value1 String, + value2 String, + value3 String, + CONSTRAINT cs_value1 CHECK toInt64(value1) < toInt64(value2), + CONSTRAINT cs_value2 CHECK toInt64(value2) < toInt64(value3) +) +ENGINE = MergeTree() +PARTITION BY date +ORDER BY key; + +INSERT INTO table_for_rename SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1), toString(number + 2) from numbers(9); +INSERT INTO table_for_rename SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1), toString(number) from numbers(9); ; --{serverError 469} + +SELECT * FROM table_for_rename ORDER BY key; + +ALTER TABLE table_for_rename RENAME COLUMN value1 TO value4; +ALTER TABLE table_for_rename RENAME COLUMN value2 TO value5; +SHOW CREATE TABLE table_for_rename; +SELECT * FROM table_for_rename ORDER BY key; + +SELECT '-- insert after rename --'; +INSERT INTO table_for_rename SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1), toString(number + 2) from numbers(10, 10); +INSERT INTO table_for_rename SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1), toString(number) from numbers(10, 10); ; --{serverError 469} +SELECT * FROM table_for_rename ORDER BY key; + +SELECT '-- rename columns back --'; +ALTER TABLE table_for_rename RENAME COLUMN value4 TO value1; +ALTER TABLE table_for_rename RENAME COLUMN value5 TO value2; +SHOW CREATE TABLE table_for_rename; +SELECT * FROM table_for_rename ORDER BY key; + +SELECT '-- insert after rename column --'; +INSERT INTO table_for_rename SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1), toString(number + 2) from numbers(20,10); +INSERT INTO table_for_rename SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number), toString(number + 2) from numbers(20, 10); ; --{serverError 469} +SELECT * FROM table_for_rename ORDER BY key; + +DROP TABLE IF EXISTS table_for_rename; diff --git a/tests/queries/0_stateless/01277_alter_rename_column_constraint_zookeeper.reference b/tests/queries/0_stateless/01277_alter_rename_column_constraint_zookeeper.reference new file mode 100644 index 00000000000..9ca17dbbc0a --- /dev/null +++ b/tests/queries/0_stateless/01277_alter_rename_column_constraint_zookeeper.reference @@ -0,0 +1,90 @@ +2019-10-01 0 0 1 2 +2019-10-02 1 1 2 3 +2019-10-03 2 2 3 4 +2019-10-01 3 3 4 5 +2019-10-02 4 4 5 6 +2019-10-03 5 5 6 7 +2019-10-01 6 6 7 8 +2019-10-02 7 7 8 9 +2019-10-03 8 8 9 10 +CREATE TABLE default.table_for_rename1\n(\n `date` Date, \n `key` UInt64, \n `value4` String, \n `value5` String, \n `value3` String, \n CONSTRAINT cs_value1 CHECK toInt64(value4) < toInt64(value5), \n CONSTRAINT cs_value2 CHECK toInt64(value5) < toInt64(value3)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_for_rename\', \'1\')\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 +2019-10-01 0 0 1 2 +2019-10-02 1 1 2 3 +2019-10-03 2 2 3 4 +2019-10-01 3 3 4 5 +2019-10-02 4 4 5 6 +2019-10-03 5 5 6 7 +2019-10-01 6 6 7 8 +2019-10-02 7 7 8 9 +2019-10-03 8 8 9 10 +-- insert after rename -- +2019-10-01 0 0 1 2 +2019-10-02 1 1 2 3 +2019-10-03 2 2 3 4 +2019-10-01 3 3 4 5 +2019-10-02 4 4 5 6 +2019-10-03 5 5 6 7 +2019-10-01 6 6 7 8 +2019-10-02 7 7 8 9 +2019-10-03 8 8 9 10 +2019-10-02 10 10 11 12 +2019-10-03 11 11 12 13 +2019-10-01 12 12 13 14 +2019-10-02 13 13 14 15 +2019-10-03 14 14 15 16 +2019-10-01 15 15 16 17 +2019-10-02 16 16 17 18 +2019-10-03 17 17 18 19 +2019-10-01 18 18 19 20 +2019-10-02 19 19 20 21 +-- rename columns back -- +CREATE TABLE default.table_for_rename1\n(\n `date` Date, \n `key` UInt64, \n `value1` String, \n `value2` String, \n `value3` String, \n CONSTRAINT cs_value1 CHECK toInt64(value1) < toInt64(value2), \n CONSTRAINT cs_value2 CHECK toInt64(value2) < toInt64(value3)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_for_rename\', \'1\')\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 +2019-10-01 0 0 1 2 +2019-10-02 1 1 2 3 +2019-10-03 2 2 3 4 +2019-10-01 3 3 4 5 +2019-10-02 4 4 5 6 +2019-10-03 5 5 6 7 +2019-10-01 6 6 7 8 +2019-10-02 7 7 8 9 +2019-10-03 8 8 9 10 +2019-10-02 10 10 11 12 +2019-10-03 11 11 12 13 +2019-10-01 12 12 13 14 +2019-10-02 13 13 14 15 +2019-10-03 14 14 15 16 +2019-10-01 15 15 16 17 +2019-10-02 16 16 17 18 +2019-10-03 17 17 18 19 +2019-10-01 18 18 19 20 +2019-10-02 19 19 20 21 +-- insert after rename column -- +2019-10-01 0 0 1 2 +2019-10-02 1 1 2 3 +2019-10-03 2 2 3 4 +2019-10-01 3 3 4 5 +2019-10-02 4 4 5 6 +2019-10-03 5 5 6 7 +2019-10-01 6 6 7 8 +2019-10-02 7 7 8 9 +2019-10-03 8 8 9 10 +2019-10-02 10 10 11 12 +2019-10-03 11 11 12 13 +2019-10-01 12 12 13 14 +2019-10-02 13 13 14 15 +2019-10-03 14 14 15 16 +2019-10-01 15 15 16 17 +2019-10-02 16 16 17 18 +2019-10-03 17 17 18 19 +2019-10-01 18 18 19 20 +2019-10-02 19 19 20 21 +2019-10-03 20 20 21 22 +2019-10-01 21 21 22 23 +2019-10-02 22 22 23 24 +2019-10-03 23 23 24 25 +2019-10-01 24 24 25 26 +2019-10-02 25 25 26 27 +2019-10-03 26 26 27 28 +2019-10-01 27 27 28 29 +2019-10-02 28 28 29 30 +2019-10-03 29 29 30 31 diff --git a/tests/queries/0_stateless/01277_alter_rename_column_constraint_zookeeper.sql b/tests/queries/0_stateless/01277_alter_rename_column_constraint_zookeeper.sql new file mode 100644 index 00000000000..2d3a7e0198a --- /dev/null +++ b/tests/queries/0_stateless/01277_alter_rename_column_constraint_zookeeper.sql @@ -0,0 +1,43 @@ +DROP TABLE IF EXISTS table_for_rename1; + +CREATE TABLE table_for_rename1 +( + date Date, + key UInt64, + value1 String, + value2 String, + value3 String, + CONSTRAINT cs_value1 CHECK toInt64(value1) < toInt64(value2), + CONSTRAINT cs_value2 CHECK toInt64(value2) < toInt64(value3) +) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_for_rename', '1') +PARTITION BY date +ORDER BY key; + +INSERT INTO table_for_rename1 SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1), toString(number + 2) from numbers(9); +INSERT INTO table_for_rename1 SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1), toString(number) from numbers(9); ; --{serverError 469} + +SELECT * FROM table_for_rename1 ORDER BY key; + +ALTER TABLE table_for_rename1 RENAME COLUMN value1 TO value4; +ALTER TABLE table_for_rename1 RENAME COLUMN value2 TO value5; +SHOW CREATE TABLE table_for_rename1; +SELECT * FROM table_for_rename1 ORDER BY key; + +SELECT '-- insert after rename --'; +INSERT INTO table_for_rename1 SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1), toString(number + 2) from numbers(10, 10); +INSERT INTO table_for_rename1 SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1), toString(number) from numbers(10, 10); ; --{serverError 469} +SELECT * FROM table_for_rename1 ORDER BY key; + +SELECT '-- rename columns back --'; +ALTER TABLE table_for_rename1 RENAME COLUMN value4 TO value1; +ALTER TABLE table_for_rename1 RENAME COLUMN value5 TO value2; +SHOW CREATE TABLE table_for_rename1; +SELECT * FROM table_for_rename1 ORDER BY key; + +SELECT '-- insert after rename column --'; +INSERT INTO table_for_rename1 SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1), toString(number + 2) from numbers(20,10); +INSERT INTO table_for_rename1 SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number), toString(number + 2) from numbers(20, 10); ; --{serverError 469} +SELECT * FROM table_for_rename1 ORDER BY key; + +DROP TABLE IF EXISTS table_for_rename1; From 2b51b7aa2a8ae39557e898305b1c8a45fcf1377e Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 12 May 2020 14:47:59 +0300 Subject: [PATCH 108/738] Remove redundant using --- src/Storages/ConstraintsDescription.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Storages/ConstraintsDescription.h b/src/Storages/ConstraintsDescription.h index b7bfc1a1acf..f4da4376041 100644 --- a/src/Storages/ConstraintsDescription.h +++ b/src/Storages/ConstraintsDescription.h @@ -6,12 +6,11 @@ namespace DB { -using ConstraintsASTs = std::vector; using ConstraintsExpressions = std::vector; struct ConstraintsDescription { - ConstraintsASTs constraints; + std::vector constraints; ConstraintsDescription() = default; From 433cd6b99387369e5d7a3ab41b3862ec25a4ffe7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=82=D0=B5=D0=BC=20=D0=A1=D1=82=D1=80=D0=B5?= =?UTF-8?q?=D0=BB=D1=8C=D1=86=D0=BE=D0=B2?= Date: Tue, 12 May 2020 15:39:44 +0300 Subject: [PATCH 109/738] bug fixes --- .../ComplexKeyDirectDictionary.cpp | 105 +++++++++--------- src/Dictionaries/ComplexKeyDirectDictionary.h | 5 +- 2 files changed, 57 insertions(+), 53 deletions(-) diff --git a/src/Dictionaries/ComplexKeyDirectDictionary.cpp b/src/Dictionaries/ComplexKeyDirectDictionary.cpp index 8c21a6189d4..333739f3ca8 100644 --- a/src/Dictionaries/ComplexKeyDirectDictionary.cpp +++ b/src/Dictionaries/ComplexKeyDirectDictionary.cpp @@ -4,7 +4,6 @@ #include "DictionaryFactory.h" #include - namespace DB { namespace ErrorCodes @@ -366,15 +365,14 @@ void ComplexKeyDirectDictionary::getItemsImpl( { const auto rows = key_columns.front()->size(); const auto keys_size = dict_struct.key->size(); - StringRefs keys_array(rows); - std::unordered_map value_by_key; + StringRefs keys_array(keys_size); + MapType value_by_key; Arena temporary_keys_pool; std::vector to_load(rows); - std::vector keys(rows); + PODArray keys(rows); for (const auto row : ext::range(0, rows)) { - const StringRef keyref = placeKeysInPool(row, key_columns, keys_array, *dict_struct.key, temporary_keys_pool); - std::string key(keyref.data, keyref.size); + const StringRef key = placeKeysInPool(row, key_columns, keys_array, *dict_struct.key, temporary_keys_pool); keys[row] = key; value_by_key[key] = get_default(row); to_load[row] = row; @@ -387,31 +385,34 @@ void ComplexKeyDirectDictionary::getItemsImpl( while (const auto block = stream->read()) { - const auto key_columns = ext::map( + const auto columns = ext::map( ext::range(0, keys_size), [&](const size_t attribute_idx) { return block.safeGetByPosition(attribute_idx).column; }); + const auto attribute_columns = ext::map(ext::range(0, attributes_size), [&](const size_t attribute_idx) + { + return block.safeGetByPosition(keys_size + attribute_idx).column; + }); for (const size_t attribute_idx : ext::range(0, attributes.size())) { - const IColumn & attribute_column = *block.safeGetByPosition(keys_size + attribute_idx).column; + const IColumn & attribute_column = *attribute_columns[attribute_idx]; Arena pool; - const auto keys_size = dict_struct.key->size(); - StringRefs keys(keys_size); + StringRefs keys_temp(keys_size); - for (const auto row_idx : ext::range(0, attribute_column.size())) + const auto columns_size = columns.front()->size(); + + for (const auto row_idx : ext::range(0, columns_size)) { - const StringRef keyref = placeKeysInPool(row_idx, key_columns, keys, *dict_struct.key, pool); - std::string key(keyref.data, keyref.size); - - if (value_by_key.find(key) != value_by_key.end() && attribute.name == attribute_name_by_index.at(attribute_idx)) + const StringRef key = placeKeysInPool(row_idx, columns, keys_temp, *dict_struct.key, pool); + if (value_by_key.has(key) && attribute.name == attribute_name_by_index.at(attribute_idx)) { if (attribute.type == AttributeUnderlyingType::utFloat32) { - value_by_key[key] = static_cast(attribute_column[row_idx].get()); + value_by_key[key] = static_cast(attribute_column[row_idx].template get()); } else { - value_by_key[key] = static_cast(attribute_column[row_idx].get()); + value_by_key[key] = static_cast(attribute_column[row_idx].template get()); } } @@ -434,15 +435,14 @@ void ComplexKeyDirectDictionary::getItemsStringImpl( { const auto rows = key_columns.front()->size(); const auto keys_size = dict_struct.key->size(); - StringRefs keys_array(rows); - std::unordered_map value_by_key; + StringRefs keys_array(keys_size); + MapType value_by_key; Arena temporary_keys_pool; std::vector to_load(rows); - std::vector keys(rows); + PODArray keys(rows); for (const auto row : ext::range(0, rows)) { - const StringRef keyref = placeKeysInPool(row, key_columns, keys_array, *dict_struct.key, temporary_keys_pool); - std::string key(keyref.data, keyref.size); + const StringRef key = placeKeysInPool(row, key_columns, keys_array, *dict_struct.key, temporary_keys_pool); keys[row] = key; value_by_key[key] = get_default(row); to_load[row] = row; @@ -455,34 +455,39 @@ void ComplexKeyDirectDictionary::getItemsStringImpl( while (const auto block = stream->read()) { - const auto key_columns = ext::map( - ext::range(0, keys_size), [&](const size_t attribute_idx) { return block.safeGetByPosition(attribute_idx).column; }); + const auto columns = ext::map( + ext::range(0, keys_size), [&](const size_t attribute_idx) { return block.safeGetByPosition(attribute_idx).column; }); + const auto attribute_columns = ext::map(ext::range(0, attributes_size), [&](const size_t attribute_idx) + { + return block.safeGetByPosition(keys_size + attribute_idx).column; + }); for (const size_t attribute_idx : ext::range(0, attributes.size())) { - - const IColumn & attribute_column = *block.safeGetByPosition(keys_size + attribute_idx).column; + const IColumn & attribute_column = *attribute_columns[attribute_idx]; Arena pool; - const auto keys_size = dict_struct.key->size(); - StringRefs keys(keys_size); + StringRefs keys_temp(keys_size); - for (const auto row_idx : ext::range(0, attribute_column.size())) + const auto columns_size = columns.front()->size(); + + for (const auto row_idx : ext::range(0, columns_size)) { - const StringRef keyref = placeKeysInPool(row_idx, key_columns, keys, *dict_struct.key, pool); - std::string key(keyref.data, keyref.size); - if (value_by_key.find(key) != value_by_key.end() && attribute.name == attribute_name_by_index.at(attribute_idx)) + const StringRef key = placeKeysInPool(row_idx, columns, keys_temp, *dict_struct.key, pool); + if (value_by_key.has(key) && attribute.name == attribute_name_by_index.at(attribute_idx)) { - const String from_source = attribute_column[row_idx].get(); + const String from_source = attribute_column[row_idx].template get(); value_by_key[key] = from_source; } } } } + stream->readSuffix(); - for (const auto row : ext::range(0, rows)) + for (const auto row : ext::range(0, rows)) { set_value(row, value_by_key[keys[row]]); + } query_count.fetch_add(rows, std::memory_order_relaxed); } @@ -503,54 +508,52 @@ void ComplexKeyDirectDictionary::has(const Attribute & attribute, const Columns { const auto rows = key_columns.front()->size(); const auto keys_size = dict_struct.key->size(); - StringRefs keys_array(rows); - std::unordered_map has_key; + StringRefs keys_array(keys_size); + MapType has_key; Arena temporary_keys_pool; std::vector to_load(rows); - std::vector keys(rows); + PODArray keys(rows); for (const auto row : ext::range(0, rows)) { - const StringRef keyref = placeKeysInPool(row, key_columns, keys_array, *dict_struct.key, temporary_keys_pool); - std::string key(keyref.data, keyref.size); + const StringRef key = placeKeysInPool(row, key_columns, keys_array, *dict_struct.key, temporary_keys_pool); keys[row] = key; has_key[key] = 0; to_load[row] = row; } auto stream = source_ptr->loadKeys(key_columns, to_load); - const auto attributes_size = attributes.size(); stream->readPrefix(); while (const auto block = stream->read()) { - const auto key_columns = ext::map( - ext::range(0, keys_size), [&](const size_t attribute_idx) { return block.safeGetByPosition(attribute_idx).column; }); + const auto columns = ext::map( + ext::range(0, keys_size), [&](const size_t attribute_idx) { return block.safeGetByPosition(attribute_idx).column; }); for (const size_t attribute_idx : ext::range(0, attributes.size())) { - - const IColumn & attribute_column = *block.safeGetByPosition(keys_size + attribute_idx).column; Arena pool; - const auto keys_size = dict_struct.key->size(); - StringRefs keys(keys_size); + StringRefs keys_temp(keys_size); - for (const auto row_idx : ext::range(0, attribute_column.size())) + const auto columns_size = columns.front()->size(); + + for (const auto row_idx : ext::range(0, columns_size)) { - const StringRef keyref = placeKeysInPool(row_idx, key_columns, keys, *dict_struct.key, pool); - std::string key(keyref.data, keyref.size); - if (has_key.find(key) != has_key.end() && attribute.name == attribute_name_by_index.at(attribute_idx)) + const StringRef key = placeKeysInPool(row_idx, columns, keys_temp, *dict_struct.key, pool); + if (has_key.has(key) && attribute.name == attribute_name_by_index.at(attribute_idx)) { has_key[key] = 1; } } } } + stream->readSuffix(); - for (const auto row : ext::range(0, rows)) + for (const auto row : ext::range(0, rows)) { out[row] = has_key[keys[row]]; + } query_count.fetch_add(rows, std::memory_order_relaxed); } diff --git a/src/Dictionaries/ComplexKeyDirectDictionary.h b/src/Dictionaries/ComplexKeyDirectDictionary.h index ececd755473..e814c5dde82 100644 --- a/src/Dictionaries/ComplexKeyDirectDictionary.h +++ b/src/Dictionaries/ComplexKeyDirectDictionary.h @@ -7,10 +7,11 @@ #include #include #include + +#include #include #include #include -#include #include "DictionaryStructure.h" #include "IDictionary.h" #include "IDictionarySource.h" @@ -221,4 +222,4 @@ private: const std::string key_description{dict_struct.getKeyDescription()}; }; -} \ No newline at end of file +} From c860f9f2db76f59bb5a6c4924c489745f1d5b270 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=82=D0=B5=D0=BC=20=D0=A1=D1=82=D1=80=D0=B5?= =?UTF-8?q?=D0=BB=D1=8C=D1=86=D0=BE=D0=B2?= Date: Tue, 12 May 2020 15:41:47 +0300 Subject: [PATCH 110/738] added exception for not supporting selective load to Direct Dictionary --- src/Dictionaries/DirectDictionary.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Dictionaries/DirectDictionary.cpp b/src/Dictionaries/DirectDictionary.cpp index dbdcab1ed8d..61d8e21341b 100644 --- a/src/Dictionaries/DirectDictionary.cpp +++ b/src/Dictionaries/DirectDictionary.cpp @@ -28,6 +28,9 @@ DirectDictionary::DirectDictionary( , source_ptr{std::move(source_ptr_)} , saved_block{std::move(saved_block_)} { + if (!this->source_ptr->supportsSelectiveLoad()) + throw Exception{full_name + ": source cannot be used with DirectDictionary", ErrorCodes::UNSUPPORTED_METHOD}; + createAttributes(); } From 723e61c410eb962586262c715d418907af9e6fbc Mon Sep 17 00:00:00 2001 From: potya Date: Tue, 12 May 2020 16:19:38 +0300 Subject: [PATCH 111/738] go back --- src/Core/Settings.h | 11 +++++++---- src/Core/SettingsCollection.cpp | 12 ++++++------ 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 306e2e67f36..61fcf658ba8 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -178,6 +178,7 @@ struct Settings : public SettingsCollection \ M(SettingString, count_distinct_implementation, "uniqExact", "What aggregate function to use for implementation of count(DISTINCT ...)", 0) \ \ + M(SettingBool, output_format_enable_streaming, false, "Enable streaming in output formats that support it.", 0) \ M(SettingBool, output_format_write_statistics, true, "Write statistics about read rows, bytes, time elapsed in suitable output formats.", 0) \ \ M(SettingBool, add_http_cors_header, false, "Write add http CORS header.", 0) \ @@ -224,7 +225,6 @@ struct Settings : public SettingsCollection M(SettingBool, join_use_nulls, 0, "Use NULLs for non-joined rows of outer JOINs for types that can be inside Nullable. If false, use default value of corresponding columns data type.", IMPORTANT) \ \ M(SettingJoinStrictness, join_default_strictness, JoinStrictness::ALL, "Set default strictness in JOIN query. Possible values: empty string, 'ANY', 'ALL'. If empty, query without strictness will throw exception.", 0) \ - M(SettingUnionMode, union_default_mode, UnionMode::ALL, "Set default mode in UNION query. Possible values: empty string, 'DISTINCT', 'ALL'. If empty, query without mode will throw exception.", 0) \ M(SettingBool, any_join_distinct_right_table_keys, false, "Enable old ANY JOIN logic with many-to-one left-to-right table keys mapping for all ANY JOINs. It leads to confusing not equal results for 't1 ANY LEFT JOIN t2' and 't2 ANY RIGHT JOIN t1'. ANY RIGHT JOIN needs one-to-many keys mapping to be consistent with LEFT one.", IMPORTANT) \ \ M(SettingUInt64, preferred_block_size_bytes, 1000000, "", 0) \ @@ -326,9 +326,11 @@ struct Settings : public SettingsCollection M(SettingOverflowMode, join_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \ M(SettingBool, join_any_take_last_row, false, "When disabled (default) ANY JOIN will take the first found row for a key. When enabled, it will take the last row seen if there are multiple rows for the same key.", IMPORTANT) \ M(SettingJoinAlgorithm, join_algorithm, JoinAlgorithm::HASH, "Specify join algorithm: 'auto', 'hash', 'partial_merge', 'prefer_partial_merge'. 'auto' tries to change HashJoin to MergeJoin on the fly to avoid out of memory.", 0) \ - M(SettingBool, partial_merge_join_optimizations, false, "Enable optimizations in partial merge join", 0) \ + M(SettingBool, partial_merge_join_optimizations, true, "Enable optimizations in partial merge join", 0) \ M(SettingUInt64, default_max_bytes_in_join, 1000000000, "Maximum size of right-side table if limit is required but max_bytes_in_join is not set.", 0) \ - M(SettingUInt64, partial_merge_join_rows_in_right_blocks, 10000, "Split right-hand joining data in blocks of specified size. It's a portion of data indexed by min-max values and possibly unloaded on disk.", 0) \ + M(SettingUInt64, partial_merge_join_rows_in_right_blocks, 65536, "Split right-hand joining data in blocks of specified size. It's a portion of data indexed by min-max values and possibly unloaded on disk.", 0) \ + M(SettingUInt64, join_on_disk_max_files_to_merge, 64, "For MergeJoin on disk set how much files it's allowed to sort simultaneously. Then this value bigger then more memory used and then less disk I/O needed. Minimum is 2.", 0) \ + M(SettingString, temporary_files_codec, "LZ4", "Set compression codec for temporary files (sort and join on disk). I.e. LZ4, NONE.", 0) \ \ M(SettingUInt64, max_rows_to_transfer, 0, "Maximum size (in rows) of the transmitted external table obtained when the GLOBAL IN/JOIN section is executed.", 0) \ M(SettingUInt64, max_bytes_to_transfer, 0, "Maximum size (in uncompressed bytes) of the transmitted external table obtained when the GLOBAL IN/JOIN section is executed.", 0) \ @@ -340,7 +342,6 @@ struct Settings : public SettingsCollection \ M(SettingUInt64, max_memory_usage, 0, "Maximum memory usage for processing of single query. Zero means unlimited.", 0) \ M(SettingUInt64, max_memory_usage_for_user, 0, "Maximum memory usage for processing all concurrently running queries for the user. Zero means unlimited.", 0) \ - M(SettingUInt64, max_server_memory_usage, 0, "Maximum memory usage for server. Only has meaning at server startup. It can be specified only for default profile.", 0) \ M(SettingUInt64, memory_profiler_step, 0, "Every number of bytes the memory profiler will collect the allocating stack trace. The minimal effective step is 4 MiB (less values will work as clamped to 4 MiB). Zero means disabled memory profiler.", 0) \ \ M(SettingUInt64, max_network_bandwidth, 0, "The maximum speed of data exchange over the network in bytes per second for a query. Zero means unlimited.", 0) \ @@ -396,6 +397,8 @@ struct Settings : public SettingsCollection M(SettingUInt64, max_live_view_insert_blocks_before_refresh, 64, "Limit maximum number of inserted blocks after which mergeable blocks are dropped and query is re-executed.", 0) \ M(SettingUInt64, min_free_disk_space_for_temporary_data, 0, "The minimum disk space to keep while writing temporary data used in external sorting and aggregation.", 0) \ \ + M(SettingDefaultDatabaseEngine, default_database_engine, DefaultDatabaseEngine::Ordinary, "Default database engine.", 0) \ + M(SettingBool, allow_experimental_database_atomic, false, "Allow to create database with Engine=Atomic.", 0) \ M(SettingBool, enable_scalar_subquery_optimization, true, "If it is set to true, prevent scalar subqueries from (de)serializing large scalar values and possibly avoid running the same subquery more than once.", 0) \ M(SettingBool, optimize_trivial_count_query, true, "Process trivial 'SELECT count() FROM table' query from metadata.", 0) \ M(SettingUInt64, mutations_sync, 0, "Wait for synchronous execution of ALTER TABLE UPDATE/DELETE queries (mutations). 0 - execute asynchronously. 1 - wait current server. 2 - wait all replicas if they exist.", 0) \ diff --git a/src/Core/SettingsCollection.cpp b/src/Core/SettingsCollection.cpp index 4f13666ccda..4223ab6ea42 100644 --- a/src/Core/SettingsCollection.cpp +++ b/src/Core/SettingsCollection.cpp @@ -491,12 +491,6 @@ IMPLEMENT_SETTING_ENUM(LoadBalancing, LOAD_BALANCING_LIST_OF_NAMES, ErrorCodes:: M(ANY, "ANY") IMPLEMENT_SETTING_ENUM(JoinStrictness, JOIN_STRICTNESS_LIST_OF_NAMES, ErrorCodes::UNKNOWN_JOIN) // NOLINT -#define UNION_MODE_LIST_OF_NAMES(M) \ - M(Unspecified, "") \ - M(ALL, "ALL") \ - M(DISTINCT, "DISTINCT") -IMPLEMENT_SETTING_ENUM(UnionMode, UNION_MODE_LIST_OF_NAMES, ErrorCodes::UNKNOWN_JOIN) // NOLINT - #define JOIN_ALGORITHM_NAMES(M) \ M(AUTO, "auto") \ M(HASH, "hash") \ @@ -556,6 +550,12 @@ IMPLEMENT_SETTING_ENUM(LogsLevel, LOGS_LEVEL_LIST_OF_NAMES, ErrorCodes::BAD_ARGU IMPLEMENT_SETTING_ENUM(QueryLogElementType, LOG_QUERIES_TYPE_LIST_OF_NAMES, ErrorCodes::BAD_ARGUMENTS) +#define DEFAULT_DATABASE_ENGINE_LIST_OF_NAMES(M) \ + M(Ordinary, "Ordinary") \ + M(Atomic, "Atomic") +IMPLEMENT_SETTING_ENUM(DefaultDatabaseEngine , DEFAULT_DATABASE_ENGINE_LIST_OF_NAMES, ErrorCodes::BAD_ARGUMENTS) + + namespace details { void SettingsCollectionUtils::serializeName(const StringRef & name, WriteBuffer & buf) From 82b3bc168387b601c6322f8569685ccd553fbc1e Mon Sep 17 00:00:00 2001 From: potya Date: Tue, 12 May 2020 16:29:20 +0300 Subject: [PATCH 112/738] Provide data type symonims --- src/DataTypes/DataTypeString.cpp | 1 + src/DataTypes/DataTypesDecimal.cpp | 2 ++ src/DataTypes/DataTypesNumber.cpp | 6 ++++++ 3 files changed, 9 insertions(+) diff --git a/src/DataTypes/DataTypeString.cpp b/src/DataTypes/DataTypeString.cpp index efaf844a845..c1afa8b90ea 100644 --- a/src/DataTypes/DataTypeString.cpp +++ b/src/DataTypes/DataTypeString.cpp @@ -380,6 +380,7 @@ void registerDataTypeString(DataTypeFactory & factory) factory.registerAlias("TEXT", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("TINYTEXT", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("MEDIUMTEXT", "String", DataTypeFactory::CaseInsensitive); + factory.registerAlias("LONG", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("LONGTEXT", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("BLOB", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("TINYBLOB", "String", DataTypeFactory::CaseInsensitive); diff --git a/src/DataTypes/DataTypesDecimal.cpp b/src/DataTypes/DataTypesDecimal.cpp index cbc38429183..2586f54dc87 100644 --- a/src/DataTypes/DataTypesDecimal.cpp +++ b/src/DataTypes/DataTypesDecimal.cpp @@ -179,6 +179,8 @@ void registerDataTypeDecimal(DataTypeFactory & factory) factory.registerDataType("Decimal", create, DataTypeFactory::CaseInsensitive); factory.registerAlias("DEC", "Decimal", DataTypeFactory::CaseInsensitive); + factory.registerAlias("NUMERIC", "Decimal", DataTypeFactory::CaseInsensitive); + factory.registerAlias("FIXED", "Decimal", DataTypeFactory::CaseInsensitive); } /// Explicit template instantiations. diff --git a/src/DataTypes/DataTypesNumber.cpp b/src/DataTypes/DataTypesNumber.cpp index 5739a64d815..c98852bd2bc 100644 --- a/src/DataTypes/DataTypesNumber.cpp +++ b/src/DataTypes/DataTypesNumber.cpp @@ -23,10 +23,16 @@ void registerDataTypeNumbers(DataTypeFactory & factory) /// These synonyms are added for compatibility. factory.registerAlias("TINYINT", "Int8", DataTypeFactory::CaseInsensitive); + factory.registerAlias("BOOL", "Int8", DataTypeFactory::CaseInsensitive); + factory.registerAlias("BOOLEAN", "Int8", DataTypeFactory::CaseInsensitive); + factory.registerAlias("INT1", "Int8", DataTypeFactory::CaseInsensitive); factory.registerAlias("SMALLINT", "Int16", DataTypeFactory::CaseInsensitive); + factory.registerAlias("INT2", "Int16", DataTypeFactory::CaseInsensitive); factory.registerAlias("INT", "Int32", DataTypeFactory::CaseInsensitive); + factory.registerAlias("INT4", "Int32", DataTypeFactory::CaseInsensitive); factory.registerAlias("INTEGER", "Int32", DataTypeFactory::CaseInsensitive); factory.registerAlias("BIGINT", "Int64", DataTypeFactory::CaseInsensitive); + factory.registerAlias("INT8", "Int64", DataTypeFactory::CaseInsensitive); factory.registerAlias("FLOAT", "Float32", DataTypeFactory::CaseInsensitive); factory.registerAlias("DOUBLE", "Float64", DataTypeFactory::CaseInsensitive); } From a6ea36162cb0f38f54d69fed9011a7d9b9e68a6c Mon Sep 17 00:00:00 2001 From: potya Date: Tue, 12 May 2020 16:48:24 +0300 Subject: [PATCH 113/738] Remove useless code --- src/Parsers/ParserSelectWithUnionQuery.cpp | 28 ++++++++++------------ 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/src/Parsers/ParserSelectWithUnionQuery.cpp b/src/Parsers/ParserSelectWithUnionQuery.cpp index 3ef8c4d3eaf..cebe8ba876d 100644 --- a/src/Parsers/ParserSelectWithUnionQuery.cpp +++ b/src/Parsers/ParserSelectWithUnionQuery.cpp @@ -27,24 +27,20 @@ bool ParserSelectWithUnionQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & { ASTPtr list_node; - ParserList parser_all(std::make_unique(), std::make_unique("UNION ALL"), false); - ParserList parser_distinct(std::make_unique(), std::make_unique("UNION DISTINCT"), false); - if (parser_all.parse(pos, list_node, expected)) { - auto select_with_union_query = std::make_shared(); - - node = select_with_union_query; - select_with_union_query->list_of_selects = std::make_shared(); - select_with_union_query->children.push_back(select_with_union_query->list_of_selects); - - // flatten inner union query - for (auto & child : list_node->children) - getSelectsFromUnionListNode(child, select_with_union_query->list_of_selects->children); - } else if (parser_all.parse(pos, list_node, expected)) { - // distinct parser - } - else + ParserList parser(std::make_unique(), std::make_unique("UNION ALL"), false); + if (!parser.parse(pos, list_node, expected)) return false; + auto select_with_union_query = std::make_shared(); + + node = select_with_union_query; + select_with_union_query->list_of_selects = std::make_shared(); + select_with_union_query->children.push_back(select_with_union_query->list_of_selects); + + // flatten inner union query + for (auto & child : list_node->children) + getSelectsFromUnionListNode(child, select_with_union_query->list_of_selects->children); + return true; } From 8771402cb5bfb36381e8a020a9ba1b9198dd1541 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 12 May 2020 17:09:36 +0300 Subject: [PATCH 114/738] [docs] faster local/debug build (#10840) --- docs/tools/build.py | 4 ++++ docs/tools/mdx_clickhouse.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/docs/tools/build.py b/docs/tools/build.py index 05f611a9a30..406f5689bc4 100755 --- a/docs/tools/build.py +++ b/docs/tools/build.py @@ -241,6 +241,7 @@ if __name__ == '__main__': arg_parser.add_argument('--skip-amp', action='store_true') arg_parser.add_argument('--skip-pdf', action='store_true') arg_parser.add_argument('--skip-website', action='store_true') + arg_parser.add_argument('--skip-git-log', action='store_true') arg_parser.add_argument('--test-only', action='store_true') arg_parser.add_argument('--minify', action='store_true') arg_parser.add_argument('--htmlproofer', action='store_true') @@ -273,6 +274,9 @@ if __name__ == '__main__': args.skip_pdf = True args.skip_amp = True + if args.skip_git_log or args.skip_amp: + mdx_clickhouse.PatchedMacrosPlugin.skip_git_log = True + from build import build build(args) diff --git a/docs/tools/mdx_clickhouse.py b/docs/tools/mdx_clickhouse.py index 3afd1bb76ac..6e41c7d07bb 100755 --- a/docs/tools/mdx_clickhouse.py +++ b/docs/tools/mdx_clickhouse.py @@ -86,6 +86,7 @@ def get_translations(dirname, lang): class PatchedMacrosPlugin(macros.plugin.MacrosPlugin): disabled = False + skip_git_log = False def on_config(self, config): super(PatchedMacrosPlugin, self).on_config(config) @@ -120,6 +121,8 @@ class PatchedMacrosPlugin(macros.plugin.MacrosPlugin): markdown = super(PatchedMacrosPlugin, self).on_page_markdown(markdown, page, config, files) if config.data['extra'].get('version_prefix') or config.data['extra'].get('single_page'): return markdown + if self.skip_git_log: + return markdown src_path = page.file.abs_src_path try: git_log = subprocess.check_output(f'git log --follow --date=iso8601 "{src_path}"', shell=True) From 84a8b21df902003b77ff1614f4f5f48735685a52 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 12 May 2020 17:17:26 +0300 Subject: [PATCH 115/738] Adjust ld+json images, remove feather icons (#10843) * [docs] adjust ld+json images * rm feathericons --- .gitmodules | 3 --- website/images/clickhouse-209x60.png | Bin 0 -> 975 bytes website/images/feathericons | 1 - website/images/yandex.png | Bin 2084 -> 4457 bytes website/templates/docs/ld_json.html | 8 ++++---- 5 files changed, 4 insertions(+), 8 deletions(-) create mode 100644 website/images/clickhouse-209x60.png delete mode 160000 website/images/feathericons diff --git a/.gitmodules b/.gitmodules index 9a1ac00af0d..fb8f6796ebc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -148,9 +148,6 @@ path = contrib/avro url = https://github.com/ClickHouse-Extras/avro.git ignore = untracked -[submodule "website/images/feathericons"] - path = website/images/feathericons - url = https://github.com/feathericons/feather [submodule "contrib/msgpack-c"] path = contrib/msgpack-c url = https://github.com/msgpack/msgpack-c diff --git a/website/images/clickhouse-209x60.png b/website/images/clickhouse-209x60.png new file mode 100644 index 0000000000000000000000000000000000000000..0ba3ae8893bec71faa9508b8d52d4968e7c17ada GIT binary patch literal 975 zcmV;=12FuFP)Y6005JfnAFwQxx2l5eSczOWkp3tR8?2u;^Xk}@t>ijZ*X#~uCa%S zj3FT+|Mb-V>aG6clm5<6@@GlyASC?O#rw&$|NHQ^(6Yt=000SaNLh0L01FcU01FcV z0GgZ_0009XNkl2BO05P+?Q;V=daR-#to{hzbL<%6}iQtPBiJM&|A@o;_2 zg?IQ?Bo0~LO6#SxUQ6q>v|dW#Xy`rzeW9MY<5uFdkx{#?gsqbarVOMZ2*XC0Zzg)o z(gz}5_cn-?Xs+8lAu{cFc(FrAq?^~K%}n4{gWpQzpAwlv4$nlP>?9FQu8AtMCw*S z`d%Vj+FcX1EU?;$ZB_P^C_W{^X`ODBKsE#0n=9 zCA;F&WnVPzy#+A@9-pzQVF{O&Nq=E38?{$)6A*txCO3 zrIuxcggMO_gQ=GVnG3bf2PtNI`}Ro#EPY^ISA7`nF1?s*2If=ii`&I9*b*iL)w}^;{h=_=Y xh=_>j|0Mce3$z3gLBucy<%*zyD54_6FfW)n_m88) zK6{^Y=8OnU-uW$-Yv!A8_UyBN`|<5>f8Um(X&Rm^5))y>h=;EY1{g8~{`D`g;?IW7 z6E3f~82=9<)@D6wl=tMfh9>!?wiT2czT$=a|ceJ zW=Uqv0w*VHD_h1oefKUrcmNdDCzDFS!viE{b^^T~%FCg%vrppX1pxt1Uz2II+NP!^ z$)on6Y1+lbW#q__mR@hVriKP!{kgc%4Gpxp&}!+qb6Mp?M$#%3{nX*orAt^F?%TJ| z@~bU@2@_z>99G5e-GjP1v%5@73;g&aOEP~x*xNsa3m!4s+1lFv71rsYLt*7gj?I!r zV_r%5`F*dMwICtEx;k1C(-Z{(0T39-5@%+DN@ZS28#f|L0u5ndFlLOkeJ#xP_V~3_k^AYWD3+L_puQd&8pPUS z6%{PWKmTdnMq4V=BS*qpZ?P6y0NB4@R2i+UtTAn412bm8(4p4GwMeD`AT_nmY-Vuw zEZo0uqJZ9BC@p14WHRvfwzjMVGd*Su%%2bFA?DSqrYG>;UO04!CGqtIUtep>S}@ag zcK8msu@TOnhu&VH0_y7E{CU<)kd$O?R?B8uB7xvweB%A_N6>183OIbYZ(U~)8*6P= z%Vrt?0sx3g!1hljOIEfYi>nTrdgrNQpqEa8zO@b|yt znk_AWo!`aBgQFwo&*=4ftyU|S%L@t$PMtbcifqhMQc}Xg!e-5y<>KOEV2%G_&3Sown;$u1y}Z1ZFJHcB(IPJ|FDcr0O~GI=sMYGq z%F6uw{Leo7j1kV8H!nImdfvQwJv}{-*Vf(L-Ip$1I(YElkt0Xi+S&lX&(Ckkk|oQQ zEgLgtjGdhw=eKk^T~$>T`lEJsc9SPh=F?Xw6g4$9`bQHjP1EDXjq~#I0*;eR^29nZ zkwfj=IhKyAD_vU3c`{m4QlRJZ zm6ergX=zybMr3DacXoD)JjYe3RQdV&K|w)iY3NDin{U2})#2N3zcpEGMMcFzB%}_j z;p)|^8+c*@)N1vnO`EV^tgf!+`=Q^6P`=8_N}8sD^I7S?{f)(9a4;>GvvhiT=#M|{ zQ=lk%=1ltjeXOL;&dzPywgm?Vi|qB|$B(bAtrcK;>(;HPzK4fLdwY9+e*SaMJtti9 z+O=!N(A?YGo0ynrvYr@ECYe5Y^5nD6J}X@2=+UFEU%$@z^`fF8NA_fymzSs4>$$=~ zv>$x%0kWNv#Ky*|)oLE5^?Lg7VSHF0J<8HiDrq}AmI7B(R#sM;HNQ!v((&WR^Nl}} z>9J$S9yoBov@s%)066N2*RNj>NJxqCMSOgG zT3T8He_(|G)A#P(Lp6sC8DgAc7A;yt7PVo+2F_=H|NZwd&HDO!QtkNhFMccuQkc^nl)>%spXSTK0zO!nMI87_xHbY40(%)R^Wy8w`slyv#>Wln{bmKJPk0)QJgZir`k z+O%nHZEd9f9Cm!1jg1Xsn{X~~WU;GPukP;dCYAT?+b42N%jI%UPfwH{wCNcX78Z^g zH3}_l-@bjE!#WB%S!OynHi4Iwm0`W1si{c>)2*#Iof9TJ{AmAvyc4vv@D+3L;K2h2 z4k#1~ftmA*FTTLq?)&e*7s+&7TpTCOG);fW_W4Mq(&FM`PR-2BOprV6etv$K zgd8`Vu(h?t=)|bS-f||ER#a3d6bekpX)c)srBeCTS6`vO&gbrU;e{74$!V@2zG1_L zEnT`4m6Vs4GwU}1)Ya9UJ$u&pfZ^fc0qnF2K@Maxcw``PFfqA!J)aMqoby# z1{1#Y(o0nT`YtozFbUT5*u$cIi`h7km^yW8KtKTM*)1t4(dl$Lo$fvMDdOC@b4{Ia zuzL{%1%a0r>zJ~!5nZw%Apy*BDO(!4ySvYyKaWaYdF2&>6+t|X965qX1l(lN(a|Ek zE2~zmLJftjTetS~^!)VGPnc$Sc(^%C1Aw*^L^{qEle_e_;3OJ`Sa(Ol$40E z@SU8TGBY!`Y}sOzY}~jJU4W!gX($#>B8q7sLcvj(bKikRqiJkx+_h^LYLQYD_2e&F zF$d(n%gM>9|BE!{1O(?ms_}SA<8snXJln%4dn{K7QwWe8zdzG zd63}j4AZBJeDbogvaGDEqeqVlZ-fRPO-)Uh3^}!&CQr3H8pkDu3enZOHmZ5 zfBW`r%(aPn5mv5T`6X8@kgKa}L_`E2zX7n7JUwZf9p_Cwh&52m^xeBYS)Mn3xz&rXwOE0(fpG5Xba{3E<;{hYp5?iAmJR z%F5chb*s^NVw|pngM;1O-JPAC9UL5_QfY2(E?6NN-rnAmCQTx9_FY_D?0It5#W5|B zKzux)o4nv)ARK$-%zWm|na!IwQxs)*l-@_v^pYh@l9G}A^r>Cc*NA>;m_Pq1XJA9uO6cj{8Mv@M8$-p%t zb7sfT%yhBz^mL1&832qSF;5oLrcD!XVVMi;_!4uy<>chtzI~gc^9@`h?%A^kTXo3O z$v++BB-#JXH0N$uB6(s>jrp{H_~8ev{O`W|Zt$2ko~9X*V38NxkOXiJ2y;Ph)yO!4 z$HOnb{DQt+v0_C?NXXi?YmF9wR4U!HX_Jb$re>g+zH{deCc#e8Ed*|EZkR;Pm#|_E z$cTe_f%GFZ87D(Y~wyWc&E|U<<*C z6DRs#Te)10roT*{JlWURm)TxphgYXhpT>gYB=G-@v17-QgO3bBF&9owPUN1rLFBzL z{5$OG>S{C^Cp0wF-`^hq92^{$u>*uEDk`wJoxx-JvD29b6!!M^SY+zaqeoTd-o?fR zG<%4_@4x@vp8{1X6=rG~BPJvgiA*N*^z=mg>&1%~(Rp;Bm?m93OpL0f!O_ui(xge4 zun~>6cw8(X5{yeqO8V2?8yXsp9Xp0ve5jEtlgR=D15pDrGcyzQ>JJpt%tD23UGnqu zMS6$^6T^oOC)36bA3iKlRgq6Yx+Dt=3&|~scw}d1qejoBO`A{`x4pf6LP7#sx2spL zUc7j5z?sI@KW2!3{q@(J%4GVMbPyw!o`3#%U}t;e=H?0{Q(^lYJ~%r&V<|Rrx%``N zz7e)ewOVayX(_4G($cbb?_Siz!{S5%VBWlWm>I}QO%;2ib6OfJlz++;`oI4b{!%J) z?G=IIMeod3n-THd~Wo38@chJ>Q1yu3V4O@?d~!NI|ejg4dlm?Q*| zxt5icm1LBlK-OnTNl9#MEW-iFH1@f|% zFJBHsdYDWmyKvzGQ*u{VmqMW^E-q$*2APHS*I$3dGAL)xoQZZXW^Ulj55nrUef#zS zXS%nycip;moJ}K>$&9vxR4O&PBvFH6Fq!5|W;6oLnwLlAQFD#>B+%l`NCV zMvop%%2%ygg=GgiA-8k-`ufVTboNK+?(RlCR1%3KGBT3MI~>4WmQ*TDNl6I_2_c)N zwzd|v9eR3tKKkgR?c29w2E|~SX35FPu{`%{p##O%*4AJ!-8Ug_+e^lDrVDi<|YjsMvoqyot-T({uqoHF`}fTWX_y9fui#n+}+)eA3rYU zss#p1mMpn`{kn0kRwxu`)ILqq)2C18J8YFor6D09zU(_5Po6xf)9J*`-WmgjXC4Ijpuc_$h=s0lT z0GTJoV8ezDOtNHGR~I^T_U_$lvJpBvJC7YZ79|j93$StH##5(GVY#woBC5cG#iaa` zXQr*Kt>U6yXc|;1z)XJ!2M`^nt<&k++uMKs`RAKAZ{E0Z!^XzO-`_tXBErYV$HBpY z>AyWaJ*dOk*4CB`iR$j|#&#f%j*fg=GL1%q1q2QmGQ?!)fx%$VYPI$C^=HnUsjI6~ zsZ>LU4)yc%i;9Z!@bF+#$e0E-)`YXFoKUOPZEbDk<>gndTw!AOM~xa45D?(!=NA+d vWN&XzMi3~KN_4iPY1(8sI{6hxN5}sGw1xzT^6*D300000NkvXXu0mjfVq3+> literal 2084 zcmZ`)do&aL8~@r^x&Qj*QrPSA)+mj+l)EyMTPwHrZY^vva<7o|t_FI3x=lp)3bDrmOKF{-f&U4Q5&y(eL+)-XeO$GoU@9c!} z5VJrWvD=~Iym2;1Sxl0Fc33+A8ggVe$x>pi<>%yq1t1;)KyoSo>*7-K0szq{06vib zuq*&T^<2?ocWZIO+wGW_gGh{!5D}Fs>h2Z+AfnSn2!sfY78MnVrlv$Zo`}N{t^NXs z!{Lb~izT+!)YORZcu`Hw)YMc+NC+N}N2Ae2MMd4+-7FT1N~IzY2s)jRTnI&7Phl7(vScX7Pz1putr8Dr-a`-We%7+QKbTjE_}l5MoWH)p`6KcHj|K__fPmQ&Q$hxKG|4Je6dsymSNngPRQnR&&JBj z1M6Sw39Gh*)ts@Jun}PhvT)8!xmdnkGxk5zofBnWRuJnv&qTH4-!@gK14*(10#?`+ z!ujY6O-%fz!*72o+(~2h1{QO|{W(O4e!0WcC{CqgFQX7+DhQ=;?m#+ity)xiFD0pN znb6k4?<5sVj~?C`E7xsRWpA}*KvKRUYDG`Lny+DLy>6SU6qOF7;B0G~A#;IF8$Wi+ zU;Wf-58F>K$+35aw54{-iy7mmO(hc#cJOdner4-to$jW!x(0r&=&#UML>ph_bg5bW5X7w?ww2fb=^3YIPhJsRzfBGAR?ni6 zQXs@>LOz^ZHvmW@F1k2anN4tpqxLwz~Mx zTYj+}rQiF{P1b)~VRY{xVZ6oNw?C%M3W_9g?=WezcZK!Pl2(?hi-lzjjM3h-sLRyK zjBu~h*sA;Uv>JbJVlm?fhBY{9FLZx!x#+XgYeoIaT{jYx@CtSq(xJOKL$Je#TchBt ziDLTk@WL|zm+auGgPKiwyyVb>^@e^U@Akl~EmW7Z_YqMamEyI6kuWKIym}s z<@aFD%M)fcF4DFn$%nE6Ic*Ld`a0GtpEj>0cr?$7HZpL1(C-tro~mK)7Vv1f%hY{O z?A%Dmed&i;iUl;?<_=8O*apJgz*Vt3aIB1`CU8)SqbTJ2D9EcpRoF*na(!BY)IKym zj92%pjO?S?OUUKMYbb5=C2Ztobd~9wKS72|olIOWMGI$?yq-h!$n1YlNW*_&cp0!fOt-#w7p3lp)8AsA9U^KTul|4~% zGo|LW)@^dE6eDRUPG~hqdYt?rMSfRyH6(h3_SBdwGjtU&FrPtx&e$?O!#q9HO(TlS zklXclF+knClg+j;{gOlz-@2i`pu3u@T-cckOib?pwj8u)y=wR^a=((*ATzCcN}9M` zz~T#3g(GUJ6PL(|uIjSZ>1W!l2U%wJ@)Ed>=>f&s!tv9dbTZGGnLQ6*K3t-L4Rit8 zY$Y>W^;e#*jLHtqmewpyG5ji`ee)VSo*&eY9m;)C25aOU;d>Xj7;KFrsRB2xsN;x-PnvQnm8apB`YUAY1maj`u#gH`2&|Qaxc>2 z+%@XFH``b)c6j;G6{daU&2BdG5I2Tj5qL8Lf%-Y~iO%S)To)$x432jI*n z_8N|U#W{`+XMa#*cfhfDDU&=DX|BiK;0DRzMo3udKvq?wZj~<9kk1v~6r|Lg z4~5#hyP>U83Uh`=Ahiy-_dk$_mPAgbrLp@arJBE^X+xG{Zs+FudTQ3PQ5WkQ@ox3e zkd&4+8AcW?{TZJDE&mb-SAOv_t#xQ&cCk5}w8)Q2Eful}K9eu6?|U3Q%x79T4^cQ} z(ARa4+Tx0-r8$B=0`iI#x^-S?6*DJOmt;z8e%5Gqv(({J^HxIkWtri)6Myh>IJVWy zvWm@evVG&d{{_t|5wtn@sHCFtiuil&jdH+8`IDn4mVOr~Vgkk}V-yl~5NTxOg)*@; zF|sr<*GHi&QK&GFKbQYu2#fF!3W)pv1`S1RPqAV5uMM6N0Z}pJ3lw1IA904F From 0e2d423aac2606fedf817429d6dad85db5cc9bbc Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 12 May 2020 17:20:47 +0300 Subject: [PATCH 116/738] fix docs build --- docs/ru/engines/table-engines/mergetree-family/mergetree.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ru/engines/table-engines/mergetree-family/mergetree.md b/docs/ru/engines/table-engines/mergetree-family/mergetree.md index d1334f4db25..ecbc44f71ee 100644 --- a/docs/ru/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/mergetree.md @@ -45,7 +45,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] - `ENGINE` — имя и параметры движка. `ENGINE = MergeTree()`. `MergeTree` не имеет параметров. -- `PARTITION BY` — [ключ партиционирования](custom-partitioning-key.md). Для партиционирования по месяцам используйте выражение `toYYYYMM(date_column)`, где `date_column` — столбец с датой типа [Date](../../../engines/table_engines/mergetree_family/mergetree.md). В этом случае имена партиций имеют формат `"YYYYMM"`. +- `PARTITION BY` — [ключ партиционирования](custom-partitioning-key.md). Для партиционирования по месяцам используйте выражение `toYYYYMM(date_column)`, где `date_column` — столбец с датой типа [Date](../../../engines/table-engines/mergetree-family/mergetree.md). В этом случае имена партиций имеют формат `"YYYYMM"`. - `ORDER BY` — ключ сортировки. Кортеж столбцов или произвольных выражений. Пример: `ORDER BY (CounterID, EventDate)`. @@ -63,7 +63,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] - `index_granularity` — максимальное количество строк данных между засечками индекса. По умолчанию — 8192. Смотрите [Хранение данных](#mergetree-data-storage). - `index_granularity_bytes` — максимальный размер гранул данных в байтах. По умолчанию — 10Mb. Чтобы ограничить размер гранул только количеством строк, установите значение 0 (не рекомендовано). Смотрите [Хранение данных](#mergetree-data-storage). - `enable_mixed_granularity_parts` — включает или выключает переход к ограничению размера гранул с помощью настройки `index_granularity_bytes`. Настройка `index_granularity_bytes` улучшает производительность ClickHouse при выборке данных из таблиц с большими (десятки и сотни мегабайтов) строками. Если у вас есть таблицы с большими строками, можно включить эту настройку, чтобы повысить эффективность запросов `SELECT`. - - `use_minimalistic_part_header_in_zookeeper` — Способ хранения заголовков кусков данных в ZooKeeper. Если `use_minimalistic_part_header_in_zookeeper = 1`, то ZooKeeper хранит меньше данных. Подробнее читайте в [описании настройки](../../../operations/server_configuration_parameters/settings.md#server-settings-use_minimalistic_part_header_in_zookeeper) в разделе "Конфигурационные параметры сервера". + - `use_minimalistic_part_header_in_zookeeper` — Способ хранения заголовков кусков данных в ZooKeeper. Если `use_minimalistic_part_header_in_zookeeper = 1`, то ZooKeeper хранит меньше данных. Подробнее читайте в [описании настройки](../../../operations/server-configuration-parameters/settings.md#server-settings-use_minimalistic_part_header_in_zookeeper) в разделе "Конфигурационные параметры сервера". - `min_merge_bytes_to_use_direct_io` — минимальный объём данных при слиянии, необходимый для прямого (небуферизованного) чтения/записи (direct I/O) на диск. При слиянии частей данных ClickHouse вычисляет общий объём хранения всех данных, подлежащих слиянию. Если общий объём хранения всех данных для чтения превышает `min_bytes_to_use_direct_io` байт, тогда ClickHouse использует флаг `O_DIRECT` при чтении данных с диска. Если `min_merge_bytes_to_use_direct_io = 0`, тогда прямой ввод-вывод отключен. Значение по умолчанию: `10 * 1024 * 1024 * 1024` байтов. - `merge_with_ttl_timeout` — минимальное время в секундах перед повторным слиянием с TTL. По умолчанию — 86400 (1 день). - `write_final_mark` — включает или отключает запись последней засечки индекса в конце куска данных, указывающей за последний байт. По умолчанию — 1. Не отключайте её. From 506634786e849d691fd2774a1d93470e42722f04 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Tue, 12 May 2020 17:34:37 +0300 Subject: [PATCH 117/738] Handle exceptions in ProxyResolverConfiguration properly. --- docker/test/integration/compose/docker_compose_minio.yml | 6 ++++++ src/Disks/S3/ProxyResolverConfiguration.cpp | 4 ++-- src/Disks/S3/registerDiskS3.cpp | 4 ++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/docker/test/integration/compose/docker_compose_minio.yml b/docker/test/integration/compose/docker_compose_minio.yml index ec35c87fa00..79023e24244 100644 --- a/docker/test/integration/compose/docker_compose_minio.yml +++ b/docker/test/integration/compose/docker_compose_minio.yml @@ -16,6 +16,9 @@ services: interval: 30s timeout: 20s retries: 3 + depends_on: + - redirect + - resolver # Redirects all requests to origin Minio. redirect: @@ -44,6 +47,9 @@ services: ports: - "4083:8080" tty: true + depends_on: + - proxy1 + - proxy2 volumes: data1-1: diff --git a/src/Disks/S3/ProxyResolverConfiguration.cpp b/src/Disks/S3/ProxyResolverConfiguration.cpp index bf06804cd23..a574809596f 100644 --- a/src/Disks/S3/ProxyResolverConfiguration.cpp +++ b/src/Disks/S3/ProxyResolverConfiguration.cpp @@ -49,9 +49,9 @@ Aws::Client::ClientConfigurationPerRequest ProxyResolverConfiguration::getConfig return cfg; } - catch (Exception & e) + catch (...) { - LOG_ERROR(&Logger::get("AWSClient"), "Failed to obtain proxy: " << e.message()); + tryLogCurrentException("AWSClient", "Failed to obtain proxy"); /// Don't use proxy if it can't be obtained. return cfg; } diff --git a/src/Disks/S3/registerDiskS3.cpp b/src/Disks/S3/registerDiskS3.cpp index 36966d1f34b..2b72f872dd2 100644 --- a/src/Disks/S3/registerDiskS3.cpp +++ b/src/Disks/S3/registerDiskS3.cpp @@ -69,7 +69,7 @@ namespace proxies.push_back(proxy_uri); - LOG_DEBUG(&Logger::get("DiskS3"), "Configured proxy: " << proxy_uri.toString() << " " << key); + LOG_DEBUG(&Logger::get("DiskS3"), "Configured proxy: " << proxy_uri.toString()); } if (!proxies.empty()) @@ -134,7 +134,7 @@ void registerDiskS3(DiskFactory & factory) auto s3disk = std::make_shared( name, client, - std::move(proxy_config), + proxy_config, uri.bucket, uri.key, metadata_path, From 0ef9b20823092305b46acfc13529ea1dc3e9008c Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 12 May 2020 17:37:19 +0300 Subject: [PATCH 118/738] Try fix sample final. --- src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 4c0eb1a479c..0ab0d43e15d 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -1144,6 +1144,7 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal( { Pipe pipe(std::move(pipes), get_merging_processor()); + pipe.addSimpleTransform(std::make_shared(pipe.getHeader(), projection)); pipes = Pipes(); pipes.emplace_back(std::move(pipe)); From f093c8f2069b94d0fb8185c60574544883a5e809 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 12 May 2020 17:38:45 +0300 Subject: [PATCH 119/738] Fix test. --- tests/queries/0_stateless/01137_order_by_func_final.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01137_order_by_func_final.sql b/tests/queries/0_stateless/01137_order_by_func_final.sql index 32a9085e9ee..afbced30131 100644 --- a/tests/queries/0_stateless/01137_order_by_func_final.sql +++ b/tests/queries/0_stateless/01137_order_by_func_final.sql @@ -5,6 +5,6 @@ INSERT INTO pk_func SELECT '2020-05-05 01:00:00', number FROM numbers(100000); INSERT INTO pk_func SELECT '2020-05-06 01:00:00', number FROM numbers(100000); INSERT INTO pk_func SELECT '2020-05-07 01:00:00', number FROM numbers(100000); -SELECT toDate(d), ui FROM pk_func FINAL; +SELECT toDate(d), ui FROM pk_func FINAL order by d; DROP TABLE pk_func; From 6425e5453b459916f2712c80312bf688b0000807 Mon Sep 17 00:00:00 2001 From: dgrr Date: Tue, 12 May 2020 16:56:52 +0200 Subject: [PATCH 120/738] Added multiple query formatting on clickhouse-format --- programs/format/Format.cpp | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/programs/format/Format.cpp b/programs/format/Format.cpp index b5a4e2d1603..9993ab449b8 100644 --- a/programs/format/Format.cpp +++ b/programs/format/Format.cpp @@ -21,6 +21,7 @@ int mainEntryClickHouseFormat(int argc, char ** argv) ("hilite", "add syntax highlight with ANSI terminal escape sequences") ("oneline", "format in single line") ("quiet,q", "just check syntax, no output on success") + ("multiquery,n", "allow multiple queries in the same file") ; boost::program_options::variables_map options; @@ -38,6 +39,7 @@ int mainEntryClickHouseFormat(int argc, char ** argv) bool hilite = options.count("hilite"); bool oneline = options.count("oneline"); bool quiet = options.count("quiet"); + bool multiple = options.count("multiquery"); if (quiet && (hilite || oneline)) { @@ -53,13 +55,17 @@ int mainEntryClickHouseFormat(int argc, char ** argv) const char * end = pos + query.size(); ParserQuery parser(end); - ASTPtr res = parseQuery(parser, pos, end, "query", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); - - if (!quiet) - { - formatAST(*res, std::cout, hilite, oneline); - std::cout << std::endl; - } + do { + ASTPtr res = parseQueryAndMovePosition(parser, pos, end, "query", multiple, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); + if (!quiet) + { + formatAST(*res, std::cout, hilite, oneline); + if (multiple) { + std::cout << "\n;\n"; + } + std::cout << std::endl; + } + } while(multiple && pos != end); } catch (...) { From 10f6817fddae56e9fc52f2cb72f7f2e5c557b0ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=82=D0=B5=D0=BC=20=D0=A1=D1=82=D1=80=D0=B5?= =?UTF-8?q?=D0=BB=D1=8C=D1=86=D0=BE=D0=B2?= Date: Tue, 12 May 2020 18:00:08 +0300 Subject: [PATCH 121/738] added new layout to integration test --- .../test_dictionaries_all_layouts_and_sources/dictionary.py | 3 ++- .../test_dictionaries_all_layouts_and_sources/test.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_dictionaries_all_layouts_and_sources/dictionary.py b/tests/integration/test_dictionaries_all_layouts_and_sources/dictionary.py index 58ccbc32c87..178cdcf44ef 100644 --- a/tests/integration/test_dictionaries_all_layouts_and_sources/dictionary.py +++ b/tests/integration/test_dictionaries_all_layouts_and_sources/dictionary.py @@ -13,6 +13,7 @@ class Layout(object): 'complex_key_cache': '128', 'range_hashed': '', 'direct': '', + 'complex_key_direct': '' } def __init__(self, name): @@ -307,7 +308,7 @@ class Dictionary(object): def generate_config(self): with open(self.config_path, 'w') as result: - if self.structure.layout.get_str() != '': + if 'direct' not in self.structure.layout.get_str(): result.write(''' diff --git a/tests/integration/test_dictionaries_all_layouts_and_sources/test.py b/tests/integration/test_dictionaries_all_layouts_and_sources/test.py index cc899ffd0cc..2debb8f11dd 100644 --- a/tests/integration/test_dictionaries_all_layouts_and_sources/test.py +++ b/tests/integration/test_dictionaries_all_layouts_and_sources/test.py @@ -112,7 +112,8 @@ LAYOUTS = [ Layout("complex_key_hashed"), Layout("complex_key_cache"), Layout("range_hashed"), - Layout("direct") + Layout("direct"), + Layout("complex_key_direct") ] SOURCES = [ From 35a9a203f1c2dcb59dbef5f17124ca4e827c1611 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 12 May 2020 18:01:43 +0300 Subject: [PATCH 122/738] Update test.py --- tests/integration/test_rename_column/test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_rename_column/test.py b/tests/integration/test_rename_column/test.py index ab7d5c8c74b..6ef14c3c4e8 100644 --- a/tests/integration/test_rename_column/test.py +++ b/tests/integration/test_rename_column/test.py @@ -491,7 +491,8 @@ def test_rename_distributed(started_cluster): select(node1, table_name, "foo2", '1998\n', poll=30) finally: drop_distributed_table(node1, table_name) - + +@pytest.mark.skip(reason="temporary disabled") def test_rename_distributed_parallel_insert_and_select(started_cluster): table_name = 'test_rename_distributed_parallel_insert_and_select' try: From 9edb0ba2936a93f955e70cb2e0760a0073e5555a Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Tue, 12 May 2020 18:45:48 +0300 Subject: [PATCH 123/738] Update performance_comparison.md --- docker/test/performance-comparison/performance_comparison.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/test/performance-comparison/performance_comparison.md b/docker/test/performance-comparison/performance_comparison.md index 66fce3426fb..430cdbe1422 100644 --- a/docker/test/performance-comparison/performance_comparison.md +++ b/docker/test/performance-comparison/performance_comparison.md @@ -29,7 +29,7 @@ pull requests (0 for master) manually. ``` docker run --network=host --volume=$(pwd)/workspace:/workspace --volume=$(pwd)/output:/output - [-e REF_PR={} -e REF_SHA={} -e ] + [-e REF_PR={} -e REF_SHA={}] -e PR_TO_TEST={} -e SHA_TO_TEST={} yandex/clickhouse-performance-comparison ``` @@ -40,6 +40,7 @@ There are some environment variables that influence what the test does: * `-e CHCP_RUNS` -- the number of runs; * `-e CHPC_TEST_GREP` -- the names of the tests (xml files) to run, interpreted as a grep pattern. + * `-e CHPC_LOCAL_SCRIPT` -- use the comparison scripts from the docker container and not from the tested commit. #### Re-genarate report with your tweaks From the workspace directory (extracted test output archive): From 340ca06979ad7d8552e6cf8193d0a50e39e91b77 Mon Sep 17 00:00:00 2001 From: dgrr Date: Tue, 12 May 2020 18:13:34 +0200 Subject: [PATCH 124/738] Changed formatting of Format.cpp (to fit ClickHouse's) --- programs/format/Format.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/programs/format/Format.cpp b/programs/format/Format.cpp index 9993ab449b8..59338dfc685 100644 --- a/programs/format/Format.cpp +++ b/programs/format/Format.cpp @@ -55,14 +55,14 @@ int mainEntryClickHouseFormat(int argc, char ** argv) const char * end = pos + query.size(); ParserQuery parser(end); - do { + do + { ASTPtr res = parseQueryAndMovePosition(parser, pos, end, "query", multiple, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); if (!quiet) { formatAST(*res, std::cout, hilite, oneline); - if (multiple) { + if (multiple) std::cout << "\n;\n"; - } std::cout << std::endl; } } while(multiple && pos != end); From 580c2186805be6e324cbf6c93830fd6ce2ae2491 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 12 May 2020 19:21:37 +0300 Subject: [PATCH 125/738] Update Format.cpp --- programs/format/Format.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/format/Format.cpp b/programs/format/Format.cpp index 59338dfc685..daf2d671568 100644 --- a/programs/format/Format.cpp +++ b/programs/format/Format.cpp @@ -65,7 +65,7 @@ int mainEntryClickHouseFormat(int argc, char ** argv) std::cout << "\n;\n"; std::cout << std::endl; } - } while(multiple && pos != end); + } while (multiple && pos != end); } catch (...) { From f2e2e993e54210a01516169298ee6343db58bace Mon Sep 17 00:00:00 2001 From: BayoNet Date: Tue, 12 May 2020 19:37:21 +0300 Subject: [PATCH 126/738] DOCS-289: randConstant (#10838) * DOCSUP-778 (#111) * docs(randConstant): add description * docs(randConstant): edit link * docs(randConstant): corrections after review * docs(randConstant): add description in Russian Co-authored-by: egilmanova * CLICKHOUSEDOCS-289: Updated descriptions. * CLICKHOUSEDOCS-289: Fixed links. * Update docs/ru/sql-reference/functions/random-functions.md Co-authored-by: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> * Update docs/en/sql-reference/functions/random-functions.md Co-authored-by: Ivan Blinkov Co-authored-by: egilmanova <63861789+egilmanova@users.noreply.github.com> Co-authored-by: egilmanova Co-authored-by: Sergei Shtykov Co-authored-by: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Co-authored-by: Ivan Blinkov --- .../aggregate-functions/reference.md | 4 +- .../functions/random-functions.md | 37 ++++++++++++++++- docs/ru/sql-reference/functions/index.md | 2 +- .../functions/random-functions.md | 41 ++++++++++++++++++- 4 files changed, 79 insertions(+), 5 deletions(-) diff --git a/docs/en/sql-reference/aggregate-functions/reference.md b/docs/en/sql-reference/aggregate-functions/reference.md index 6eb9ca5c3c7..61a27f7f70e 100644 --- a/docs/en/sql-reference/aggregate-functions/reference.md +++ b/docs/en/sql-reference/aggregate-functions/reference.md @@ -516,10 +516,10 @@ And the result will be: ## timeSeriesGroupRateSum(uid, ts, val) {#agg-function-timeseriesgroupratesum} -Similarly timeSeriesGroupRateSum, timeSeriesGroupRateSum will Calculate the rate of time-series and then sum rates together. +Similarly to `timeSeriesGroupSum`, `timeSeriesGroupRateSum` calculates the rate of time-series and then sum rates together. Also, timestamp should be in ascend order before use this function. -Use this function, the result above case will be: +Applying this function to the data from the `timeSeriesGroupSum` example, you get the following result: ``` text [(2,0),(3,0.1),(7,0.3),(8,0.3),(12,0.3),(17,0.3),(18,0.3),(24,0.3),(25,0.1)] diff --git a/docs/en/sql-reference/functions/random-functions.md b/docs/en/sql-reference/functions/random-functions.md index 65dd1b50673..5d480664ca1 100644 --- a/docs/en/sql-reference/functions/random-functions.md +++ b/docs/en/sql-reference/functions/random-functions.md @@ -23,6 +23,41 @@ Uses a linear congruential generator. ## randConstant {#randconstant} -Returns a pseudo-random UInt32 number, The value is one for different blocks. +Produces a constant column with a random value. + +**Syntax** + +``` sql +randConstant([x]) +``` + +**Parameters** + +- `x` — [Expression](../syntax.md#syntax-expressions) resulting in any of the [supported data types](../data-types/index.md#data_types). The resulting value is discarded, but the expression itself if used for bypassing [common subexpression elimination](index.md#common-subexpression-elimination) if the function is called multiple times in one query. Optional parameter. + +**Returned value** + +- Pseudo-random number. + +Type: [UInt32](../data-types/int-uint.md). + +**Example** + +Query: + +``` sql +SELECT rand(), rand(1), rand(number), randConstant(), randConstant(1), randConstant(number) +FROM numbers(3) +``` + +Result: + +``` text +┌─────rand()─┬────rand(1)─┬─rand(number)─┬─randConstant()─┬─randConstant(1)─┬─randConstant(number)─┐ +│ 3047369878 │ 4132449925 │ 4044508545 │ 2740811946 │ 4229401477 │ 1924032898 │ +│ 2938880146 │ 1267722397 │ 4154983056 │ 2740811946 │ 4229401477 │ 1924032898 │ +│ 956619638 │ 4238287282 │ 1104342490 │ 2740811946 │ 4229401477 │ 1924032898 │ +└────────────┴────────────┴──────────────┴────────────────┴─────────────────┴──────────────────────┘ +``` [Original article](https://clickhouse.tech/docs/en/query_language/functions/random_functions/) diff --git a/docs/ru/sql-reference/functions/index.md b/docs/ru/sql-reference/functions/index.md index 1f7d736f691..06d3d892cf9 100644 --- a/docs/ru/sql-reference/functions/index.md +++ b/docs/ru/sql-reference/functions/index.md @@ -10,7 +10,7 @@ В ClickHouse, в отличие от стандартного SQL, типизация является строгой. То есть, не производится неявных преобразований между типами. Все функции работают для определённого набора типов. Это значит, что иногда вам придётся использовать функции преобразования типов. -## Склейка одинаковых выражений {#skleika-odinakovykh-vyrazhenii} +## Склейка одинаковых выражений {#common-subexpression-elimination} Все выражения в запросе, имеющие одинаковые AST (одинаковую запись или одинаковый результат синтаксического разбора), считаются имеющими одинаковые значения. Такие выражения склеиваются и исполняются один раз. Одинаковые подзапросы тоже склеиваются. diff --git a/docs/ru/sql-reference/functions/random-functions.md b/docs/ru/sql-reference/functions/random-functions.md index b6ce2b178ee..b425505b69d 100644 --- a/docs/ru/sql-reference/functions/random-functions.md +++ b/docs/ru/sql-reference/functions/random-functions.md @@ -1,4 +1,4 @@ -# Функции генерации псевдослучайных чисел {#funktsii-generatsii-psevdosluchainykh-chisel} +# Функции генерации псевдослучайных чисел {#functions-for-generating-pseudo-random-numbers} Используются не криптографические генераторы псевдослучайных чисел. @@ -16,4 +16,43 @@ Возвращает псевдослучайное число типа UInt64, равномерно распределённое среди всех чисел типа UInt64. Используется linear congruential generator. +## randConstant {#randconstant} + +Создает константный столбец с псевдослучайным значением. + +**Синтаксис** + +``` sql +randConstant([x]) +``` + +**Параметры** + +- `x` — [Выражение](../syntax.md#syntax-expressions), возвращающее значение одного из [поддерживаемых типов данных](../data-types/index.md#data_types). Значение используется, чтобы избежать [склейки одинаковых выражений](index.md#common-subexpression-elimination), если функция вызывается несколько раз в одном запросе. Необязательный параметр. + +**Возвращаемое значение** + +- Псевдослучайное число. + +Тип: [UInt32](../data-types/int-uint.md). + +**Пример** + +Запрос: + +``` sql +SELECT rand(), rand(1), rand(number), randConstant(), randConstant(1), randConstant(number) +FROM numbers(3) +``` + +Результат: + +``` text +┌─────rand()─┬────rand(1)─┬─rand(number)─┬─randConstant()─┬─randConstant(1)─┬─randConstant(number)─┐ +│ 3047369878 │ 4132449925 │ 4044508545 │ 2740811946 │ 4229401477 │ 1924032898 │ +│ 2938880146 │ 1267722397 │ 4154983056 │ 2740811946 │ 4229401477 │ 1924032898 │ +│ 956619638 │ 4238287282 │ 1104342490 │ 2740811946 │ 4229401477 │ 1924032898 │ +└────────────┴────────────┴──────────────┴────────────────┴─────────────────┴──────────────────────┘ +``` + [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/random_functions/) From 262b6287ab8223cee350e767d7bb20787c4b42e0 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 12 May 2020 19:38:11 +0300 Subject: [PATCH 127/738] Fix clang-tidy warnings --- src/DataStreams/CheckConstraintsBlockOutputStream.cpp | 2 +- src/Storages/ConstraintsDescription.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/DataStreams/CheckConstraintsBlockOutputStream.cpp b/src/DataStreams/CheckConstraintsBlockOutputStream.cpp index 1b87cf29dd4..878ab0c4e37 100644 --- a/src/DataStreams/CheckConstraintsBlockOutputStream.cpp +++ b/src/DataStreams/CheckConstraintsBlockOutputStream.cpp @@ -61,7 +61,7 @@ void CheckConstraintsBlockOutputStream::write(const Block & block) std::stringstream exception_message; - auto constraint_ptr = constraints.constraints[i]->as(); + auto * constraint_ptr = constraints.constraints[i]->as(); exception_message << "Constraint " << backQuote(constraint_ptr->name) << " for table " << table_id.getNameForLogs() << " is violated at row " << (rows_written + row_idx + 1) diff --git a/src/Storages/ConstraintsDescription.cpp b/src/Storages/ConstraintsDescription.cpp index 023a94a5628..d86796908a7 100644 --- a/src/Storages/ConstraintsDescription.cpp +++ b/src/Storages/ConstraintsDescription.cpp @@ -46,7 +46,7 @@ ConstraintsExpressions ConstraintsDescription::getExpressions(const DB::Context for (const auto & constraint : constraints) { // SyntaxAnalyzer::analyze has query as non-const argument so to avoid accidental query changes we clone it - auto constraint_ptr = constraint->as(); + auto * constraint_ptr = constraint->as(); ASTPtr expr = constraint_ptr->expr->clone(); auto syntax_result = SyntaxAnalyzer(context).analyze(expr, source_columns_); res.push_back(ExpressionAnalyzer(constraint_ptr->expr->clone(), syntax_result, context).getActions(false)); From 5ffb7372aff6efbc66d28e10b3b593577b19ca02 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 12 May 2020 21:10:10 +0300 Subject: [PATCH 128/738] fix livelock with watch queries and database atomic --- src/Storages/LiveView/LiveViewBlockInputStream.h | 6 +++--- src/Storages/LiveView/LiveViewEventsBlockInputStream.h | 6 +++--- src/Storages/LiveView/StorageLiveView.cpp | 4 ++++ 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/Storages/LiveView/LiveViewBlockInputStream.h b/src/Storages/LiveView/LiveViewBlockInputStream.h index cea0bc53106..9115c6330df 100644 --- a/src/Storages/LiveView/LiveViewBlockInputStream.h +++ b/src/Storages/LiveView/LiveViewBlockInputStream.h @@ -46,7 +46,7 @@ public: void cancel(bool kill) override { - if (isCancelled() || storage->is_dropped) + if (isCancelled() || storage->shutdown_called) return; IBlockInputStream::cancel(kill); std::lock_guard lock(storage->mutex); @@ -115,7 +115,7 @@ protected: end = blocks->end(); } - if (isCancelled() || storage->is_dropped) + if (isCancelled() || storage->shutdown_called) { return { Block(), true }; } @@ -155,7 +155,7 @@ protected: bool signaled = std::cv_status::no_timeout == storage->condition.wait_for(lock, std::chrono::microseconds(std::max(UInt64(0), heartbeat_interval_usec - (timestamp_usec - last_event_timestamp_usec)))); - if (isCancelled() || storage->is_dropped) + if (isCancelled() || storage->shutdown_called) { return { Block(), true }; } diff --git a/src/Storages/LiveView/LiveViewEventsBlockInputStream.h b/src/Storages/LiveView/LiveViewEventsBlockInputStream.h index bf971d75d01..1f86c41a078 100644 --- a/src/Storages/LiveView/LiveViewEventsBlockInputStream.h +++ b/src/Storages/LiveView/LiveViewEventsBlockInputStream.h @@ -65,7 +65,7 @@ public: void cancel(bool kill) override { - if (isCancelled() || storage->is_dropped) + if (isCancelled() || storage->shutdown_called) return; IBlockInputStream::cancel(kill); std::lock_guard lock(storage->mutex); @@ -149,7 +149,7 @@ protected: end = blocks->end(); } - if (isCancelled() || storage->is_dropped) + if (isCancelled() || storage->shutdown_called) { return { Block(), true }; } @@ -190,7 +190,7 @@ protected: bool signaled = std::cv_status::no_timeout == storage->condition.wait_for(lock, std::chrono::microseconds(std::max(UInt64(0), heartbeat_interval_usec - (timestamp_usec - last_event_timestamp_usec)))); - if (isCancelled() || storage->is_dropped) + if (isCancelled() || storage->shutdown_called) { return { Block(), true }; } diff --git a/src/Storages/LiveView/StorageLiveView.cpp b/src/Storages/LiveView/StorageLiveView.cpp index 66a8690b43e..cd660407c89 100644 --- a/src/Storages/LiveView/StorageLiveView.cpp +++ b/src/Storages/LiveView/StorageLiveView.cpp @@ -468,6 +468,10 @@ void StorageLiveView::shutdown() if (!shutdown_called.compare_exchange_strong(expected, true)) return; + /// WATCH queries should be stopped after setting shutdown_called to true. + /// Otherwise livelock is possible for LiveView table in Atomic database: + /// WATCH query will wait for table to be dropped and DatabaseCatalog will wait for queries to finish + { std::lock_guard no_users_thread_lock(no_users_thread_mutex); if (no_users_thread.joinable()) From 67213b8ad4cf4bb7e87536f252cfae99192de04d Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Tue, 12 May 2020 21:22:58 +0300 Subject: [PATCH 129/738] fix sample with final --- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 42 +++++++++++-------- .../MergeTree/MergeTreeDataSelectExecutor.h | 6 ++- .../0_stateless/01137_sample_final.reference | 10 +++++ .../0_stateless/01137_sample_final.sql | 13 ++++++ 4 files changed, 51 insertions(+), 20 deletions(-) create mode 100644 tests/queries/0_stateless/01137_sample_final.reference create mode 100644 tests/queries/0_stateless/01137_sample_final.sql diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index c0785899aab..ddd1e14381d 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -601,6 +601,11 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( .save_marks_in_cache = true }; + /// Projection, that needed to drop columns, which have appeared by execution + /// of some extra expressions, and to allow execute the same expressions later. + /// NOTE: It may lead to double computation of expressions. + ExpressionActionsPtr result_projection; + if (select.final()) { /// Add columns needed to calculate the sorting expression and the sign. @@ -623,7 +628,8 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( query_info, virt_column_names, settings, - reader_settings); + reader_settings, + result_projection); } else if (settings.optimize_read_in_order && query_info.input_sorting_info) { @@ -644,7 +650,8 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( sorting_key_prefix_expr, virt_column_names, settings, - reader_settings); + reader_settings, + result_projection); } else { @@ -667,6 +674,13 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( pipe.getHeader(), filter_expression, filter_function->getColumnName(), false)); } + if (result_projection) + { + for (auto & pipe : res) + pipe.addSimpleTransform(std::make_shared( + pipe.getHeader(), result_projection)); + } + /// By the way, if a distributed query or query to a Merge table is made, then the `_sample_factor` column can have different values. if (sample_factor_column_queried) { @@ -831,7 +845,8 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder( const ExpressionActionsPtr & sorting_key_prefix_expr, const Names & virt_columns, const Settings & settings, - const MergeTreeReaderSettings & reader_settings) const + const MergeTreeReaderSettings & reader_settings, + ExpressionActionsPtr & out_projection) const { size_t sum_marks = 0; const InputSortingInfoPtr & input_sorting_info = query_info.input_sorting_info; @@ -1007,19 +1022,14 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder( sort_description.emplace_back(data.sorting_key_columns[j], input_sorting_info->direction, 1); - /// Project input columns to drop columns from sorting_key_prefix_expr - /// to allow execute the same expression later. - /// NOTE: It may lead to double computation of expression. - auto projection = createProjection(pipes.back(), data); + out_projection = createProjection(pipes.back(), data); for (auto & pipe : pipes) pipe.addSimpleTransform(std::make_shared(pipe.getHeader(), sorting_key_prefix_expr)); auto merging_sorted = std::make_shared( pipes.back().getHeader(), pipes.size(), sort_description, max_block_size); - Pipe merged(std::move(pipes), std::move(merging_sorted)); - merged.addSimpleTransform(std::make_shared(merged.getHeader(), projection)); - res.emplace_back(std::move(merged)); + res.emplace_back(std::move(pipes), std::move(merging_sorted)); } else res.emplace_back(std::move(pipes.front())); @@ -1037,7 +1047,8 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal( const SelectQueryInfo & query_info, const Names & virt_columns, const Settings & settings, - const MergeTreeReaderSettings & reader_settings) const + const MergeTreeReaderSettings & reader_settings, + ExpressionActionsPtr & out_projection) const { const auto data_settings = data.getSettings(); size_t sum_marks = 0; @@ -1065,10 +1076,6 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal( use_uncompressed_cache = false; Pipes pipes; - /// Project input columns to drop columns from sorting_key_expr - /// to allow execute the same expression later. - /// NOTE: It may lead to double computation of expression. - ExpressionActionsPtr projection; for (const auto & part : parts) { @@ -1079,8 +1086,8 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal( virt_columns, part.part_index_in_query); Pipe pipe(std::move(source_processor)); - if (!projection) - projection = createProjection(pipe, data); + if (!out_projection) + out_projection = createProjection(pipe, data); pipe.addSimpleTransform(std::make_shared(pipe.getHeader(), data.sorting_key_expr)); pipes.emplace_back(std::move(pipe)); @@ -1154,7 +1161,6 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal( if (merged_processor) { Pipe pipe(std::move(pipes), std::move(merged_processor)); - pipe.addSimpleTransform(std::make_shared(pipe.getHeader(), projection)); pipes = Pipes(); pipes.emplace_back(std::move(pipe)); } diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h index e6eb26da7e3..05f330bf643 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h @@ -67,7 +67,8 @@ private: const ExpressionActionsPtr & sorting_key_prefix_expr, const Names & virt_columns, const Settings & settings, - const MergeTreeReaderSettings & reader_settings) const; + const MergeTreeReaderSettings & reader_settings, + ExpressionActionsPtr & out_projection) const; Pipes spreadMarkRangesAmongStreamsFinal( RangesInDataParts && parts, @@ -77,7 +78,8 @@ private: const SelectQueryInfo & query_info, const Names & virt_columns, const Settings & settings, - const MergeTreeReaderSettings & reader_settings) const; + const MergeTreeReaderSettings & reader_settings, + ExpressionActionsPtr & out_projection) const; /// Get the approximate value (bottom estimate - only by full marks) of the number of rows falling under the index. size_t getApproximateTotalRowsToRead( diff --git a/tests/queries/0_stateless/01137_sample_final.reference b/tests/queries/0_stateless/01137_sample_final.reference new file mode 100644 index 00000000000..91dfe010161 --- /dev/null +++ b/tests/queries/0_stateless/01137_sample_final.reference @@ -0,0 +1,10 @@ +8 8 +9 9 +10 10 +14 14 +15 15 +9140302661501632497 +9199082625845137542 +8769270213041934496 +4926958392161000708 +89922286675368115 diff --git a/tests/queries/0_stateless/01137_sample_final.sql b/tests/queries/0_stateless/01137_sample_final.sql new file mode 100644 index 00000000000..99fac514767 --- /dev/null +++ b/tests/queries/0_stateless/01137_sample_final.sql @@ -0,0 +1,13 @@ +drop table if exists tab; + +create table tab (x UInt64, v UInt64) engine = ReplacingMergeTree(v) order by (x, sipHash64(x)) sample by sipHash64(x); +insert into tab select number, number from numbers(1000); +select * from tab final sample 1/2 order by x limit 5; + +drop table tab; + +create table tab (x UInt64, v UInt64) engine = ReplacingMergeTree(v) order by (x, sipHash64(x)) sample by sipHash64(x); +insert into tab select number, number from numbers(1000); +select sipHash64(x) from tab sample 1/2 order by x, sipHash64(x) limit 5; + +drop table tab; From acd8cfc5d27ed893c711837efe113307a4939fb4 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Tue, 12 May 2020 23:19:15 +0400 Subject: [PATCH 130/738] Added contrib/openldap submodule Added OpenLDAP find/detection cmake scripts Added integration for OpenLDAP Linux and Darwin x86_64 platforms (following OpenSSL integration approach) --- .gitmodules | 3 + CMakeLists.txt | 1 + cmake/Modules/FindOpenLDAP.cmake | 55 + cmake/find/ldap.cmake | 60 + contrib/CMakeLists.txt | 4 + contrib/openldap | 1 + contrib/openldap-cmake/CMakeLists.txt | 186 +++ .../Darwin_x86_64/include/lber_types.h | 63 + .../Darwin_x86_64/include/ldap_config.h | 74 ++ .../Darwin_x86_64/include/ldap_features.h | 61 + .../Darwin_x86_64/include/portable.h | 1169 +++++++++++++++++ .../Linux_x86_64/include/lber_types.h | 63 + .../Linux_x86_64/include/ldap_config.h | 74 ++ .../Linux_x86_64/include/ldap_features.h | 61 + .../Linux_x86_64/include/portable.h | 1169 +++++++++++++++++ src/CMakeLists.txt | 5 + 16 files changed, 3049 insertions(+) create mode 100644 cmake/Modules/FindOpenLDAP.cmake create mode 100644 cmake/find/ldap.cmake create mode 160000 contrib/openldap create mode 100644 contrib/openldap-cmake/CMakeLists.txt create mode 100644 contrib/openldap-cmake/Darwin_x86_64/include/lber_types.h create mode 100644 contrib/openldap-cmake/Darwin_x86_64/include/ldap_config.h create mode 100644 contrib/openldap-cmake/Darwin_x86_64/include/ldap_features.h create mode 100644 contrib/openldap-cmake/Darwin_x86_64/include/portable.h create mode 100644 contrib/openldap-cmake/Linux_x86_64/include/lber_types.h create mode 100644 contrib/openldap-cmake/Linux_x86_64/include/ldap_config.h create mode 100644 contrib/openldap-cmake/Linux_x86_64/include/ldap_features.h create mode 100644 contrib/openldap-cmake/Linux_x86_64/include/portable.h diff --git a/.gitmodules b/.gitmodules index cebde5699d8..75ea99e9d80 100644 --- a/.gitmodules +++ b/.gitmodules @@ -154,3 +154,6 @@ [submodule "contrib/msgpack-c"] path = contrib/msgpack-c url = https://github.com/msgpack/msgpack-c +[submodule "contrib/openldap"] + path = contrib/openldap + url = https://github.com/openldap/openldap.git diff --git a/CMakeLists.txt b/CMakeLists.txt index c1451990bdb..1a4d526e1b4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -344,6 +344,7 @@ include (cmake/lib_name.cmake) find_contrib_lib(double-conversion) # Must be before parquet include (cmake/find/ssl.cmake) +include (cmake/find/ldap.cmake) # after ssl include (cmake/find/icu.cmake) include (cmake/find/boost.cmake) include (cmake/find/zlib.cmake) diff --git a/cmake/Modules/FindOpenLDAP.cmake b/cmake/Modules/FindOpenLDAP.cmake new file mode 100644 index 00000000000..c33eafdcb2e --- /dev/null +++ b/cmake/Modules/FindOpenLDAP.cmake @@ -0,0 +1,55 @@ +# Find OpenLDAP libraries. +# +# Can be configured with: +# OPENLDAP_ROOT_DIR - path to the OpenLDAP installation prefix +# OPENLDAP_USE_STATIC_LIBS - look for static version of the libraries +# OPENLDAP_USE_REENTRANT_LIBS - look for thread-safe version of the libraries +# +# Sets values of: +# OPENLDAP_FOUND - TRUE if found +# OPENLDAP_INCLUDE_DIR - path to the include directory +# OPENLDAP_LIBRARIES - paths to the libldap and liblber libraries +# OPENLDAP_LDAP_LIBRARY - paths to the libldap library +# OPENLDAP_LBER_LIBRARY - paths to the liblber library +# + +if(OPENLDAP_USE_STATIC_LIBS) + set(_orig_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) + if(WIN32) + set(CMAKE_FIND_LIBRARY_SUFFIXES ".lib" ".a" ${CMAKE_FIND_LIBRARY_SUFFIXES}) + else() + set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") + endif() +endif() + +set(_r_suffix) +if(OPENLDAP_USE_REENTRANT_LIBS) + set(_r_suffix "_r") +endif() + +if(OPENLDAP_ROOT_DIR) + find_path(OPENLDAP_INCLUDE_DIR NAMES "ldap.h" "lber.h" PATHS "${OPENLDAP_ROOT_DIR}" PATH_SUFFIXES "include" NO_DEFAULT_PATH) + find_library(OPENLDAP_LDAP_LIBRARY NAMES "ldap${_r_suffix}" PATHS "${OPENLDAP_ROOT_DIR}" PATH_SUFFIXES "lib" NO_DEFAULT_PATH) + find_library(OPENLDAP_LBER_LIBRARY NAMES "lber" PATHS "${OPENLDAP_ROOT_DIR}" PATH_SUFFIXES "lib" NO_DEFAULT_PATH) +else() + find_path(OPENLDAP_INCLUDE_DIR NAMES "ldap.h" "lber.h") + find_library(OPENLDAP_LDAP_LIBRARY NAMES "ldap${_r_suffix}") + find_library(OPENLDAP_LBER_LIBRARY NAMES "lber") +endif() + +unset(_r_suffix) + +set(OPENLDAP_LIBRARIES ${OPENLDAP_LDAP_LIBRARY} ${OPENLDAP_LBER_LIBRARY}) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args( + OpenLDAP DEFAULT_MSG + OPENLDAP_INCLUDE_DIR OPENLDAP_LDAP_LIBRARY OPENLDAP_LBER_LIBRARY +) + +mark_as_advanced(OPENLDAP_INCLUDE_DIR OPENLDAP_LIBRARIES OPENLDAP_LDAP_LIBRARY OPENLDAP_LBER_LIBRARY) + +if(OPENLDAP_USE_STATIC_LIBS) + set(CMAKE_FIND_LIBRARY_SUFFIXES ${_orig_CMAKE_FIND_LIBRARY_SUFFIXES}) + unset(_orig_CMAKE_FIND_LIBRARY_SUFFIXES) +endif() diff --git a/cmake/find/ldap.cmake b/cmake/find/ldap.cmake new file mode 100644 index 00000000000..f531afc446e --- /dev/null +++ b/cmake/find/ldap.cmake @@ -0,0 +1,60 @@ +option (ENABLE_LDAP "Enable LDAP" ${ENABLE_LIBRARIES}) + +if (ENABLE_LDAP) + option (USE_INTERNAL_LDAP_LIBRARY "Set to FALSE to use system *LDAP library instead of bundled" ${NOT_UNBUNDLED}) + + if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/openldap/README") + if (USE_INTERNAL_LDAP_LIBRARY) + message (WARNING "Submodule contrib/openldap is missing. To fix try running:\n git submodule update --init --recursive") + endif () + + set (USE_INTERNAL_LDAP_LIBRARY 0) + set (MISSING_INTERNAL_LDAP_LIBRARY 1) + endif () + + set (OPENLDAP_USE_STATIC_LIBS ${USE_STATIC_LIBRARIES}) + set (OPENLDAP_USE_REENTRANT_LIBS 1) + + if (NOT USE_INTERNAL_LDAP_LIBRARY) + if (APPLE AND NOT OPENLDAP_ROOT_DIR) + set (OPENLDAP_ROOT_DIR "/usr/local/opt/openldap") + endif () + + find_package (OpenLDAP) + endif () + + if (NOT OPENLDAP_FOUND AND NOT MISSING_INTERNAL_LDAP_LIBRARY) + if ( + ("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux" AND "${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "x86_64") OR + ("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin" AND "${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "x86_64") + ) + set (_ldap_supported_platform TRUE) + endif () + + if (NOT _ldap_supported_platform) + message (WARNING "LDAP support using the bundled library is not implemented for ${CMAKE_SYSTEM_NAME} ${CMAKE_SYSTEM_PROCESSOR} platform.") + elseif (NOT USE_SSL) + message (WARNING "LDAP support using the bundled library is not possible if SSL is not used.") + else () + set (USE_INTERNAL_LDAP_LIBRARY 1) + set (OPENLDAP_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/openldap") + set (OPENLDAP_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/openldap/include") + # Below, 'ldap'/'ldap_r' and 'lber' will be resolved to + # the targets defined in contrib/openldap-cmake/CMakeLists.txt + if (OPENLDAP_USE_REENTRANT_LIBS) + set (OPENLDAP_LDAP_LIBRARY "ldap_r") + else () + set (OPENLDAP_LDAP_LIBRARY "ldap") + endif() + set (OPENLDAP_LBER_LIBRARY "lber") + set (OPENLDAP_LIBRARIES ${OPENLDAP_LDAP_LIBRARY} ${OPENLDAP_LBER_LIBRARY}) + set (OPENLDAP_FOUND 1) + endif () + endif () + + if (OPENLDAP_FOUND) + set (USE_LDAP 1) + endif () +endif () + +message (STATUS "Using ldap=${USE_LDAP}: ${OPENLDAP_INCLUDE_DIR} : ${OPENLDAP_LIBRARIES}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 01e9f14d241..c9f5154ffe4 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -107,6 +107,10 @@ if (USE_INTERNAL_SSL_LIBRARY) add_library(OpenSSL::SSL ALIAS ${OPENSSL_SSL_LIBRARY}) endif () +if (ENABLE_LDAP AND USE_INTERNAL_LDAP_LIBRARY) + add_subdirectory (openldap-cmake) +endif () + function(mysql_support) set(CLIENT_PLUGIN_CACHING_SHA2_PASSWORD STATIC) set(CLIENT_PLUGIN_SHA256_PASSWORD STATIC) diff --git a/contrib/openldap b/contrib/openldap new file mode 160000 index 00000000000..34b9ba94b30 --- /dev/null +++ b/contrib/openldap @@ -0,0 +1 @@ +Subproject commit 34b9ba94b30319ed6389a4e001d057f7983fe363 diff --git a/contrib/openldap-cmake/CMakeLists.txt b/contrib/openldap-cmake/CMakeLists.txt new file mode 100644 index 00000000000..b7bcd2468bd --- /dev/null +++ b/contrib/openldap-cmake/CMakeLists.txt @@ -0,0 +1,186 @@ +set(OPENLDAP_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/openldap) + +# How these lists were generated? +# I compiled the original OpenLDAP with it's original build system and copied the list of source files from build commands. + +set(_libs_type SHARED) +if(OPENLDAP_USE_STATIC_LIBS) + set(_libs_type STATIC) +endif() + +set(OPENLDAP_VERSION_STRING "2.5.X") + +macro(mkversion _lib_name) + add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${_lib_name}-version.c + COMMAND ${CMAKE_COMMAND} -E env bash -c "${OPENLDAP_SOURCE_DIR}/build/mkversion -v '${OPENLDAP_VERSION_STRING}' liblber.la > ${CMAKE_CURRENT_BINARY_DIR}/${_lib_name}-version.c" + MAIN_DEPENDENCY ${OPENLDAP_SOURCE_DIR}/build/mkversion + WORKING_DIRECTORY ${OPENLDAP_SOURCE_DIR} + VERBATIM + ) +endmacro() + +set(_extra_build_dir "${CMAKE_CURRENT_SOURCE_DIR}/${CMAKE_SYSTEM_NAME}_${CMAKE_SYSTEM_PROCESSOR}") + +set(_lber_srcs + ${OPENLDAP_SOURCE_DIR}/libraries/liblber/assert.c + ${OPENLDAP_SOURCE_DIR}/libraries/liblber/decode.c + ${OPENLDAP_SOURCE_DIR}/libraries/liblber/encode.c + ${OPENLDAP_SOURCE_DIR}/libraries/liblber/io.c + ${OPENLDAP_SOURCE_DIR}/libraries/liblber/bprint.c + ${OPENLDAP_SOURCE_DIR}/libraries/liblber/debug.c + ${OPENLDAP_SOURCE_DIR}/libraries/liblber/memory.c + ${OPENLDAP_SOURCE_DIR}/libraries/liblber/options.c + ${OPENLDAP_SOURCE_DIR}/libraries/liblber/sockbuf.c + ${OPENLDAP_SOURCE_DIR}/libraries/liblber/stdio.c +) + +mkversion(lber) + +add_library(lber ${_libs_type} + ${_lber_srcs} + ${CMAKE_CURRENT_BINARY_DIR}/lber-version.c +) + +target_link_libraries(lber + PRIVATE ${OPENSSL_LIBRARIES} +) + +target_include_directories(lber + PRIVATE ${_extra_build_dir}/include + PRIVATE ${OPENLDAP_SOURCE_DIR}/include + PRIVATE ${OPENLDAP_SOURCE_DIR}/libraries/liblber + PRIVATE ${OPENSSL_INCLUDE_DIR} +) + +target_compile_definitions(lber + PRIVATE LBER_LIBRARY +) + +set(_ldap_srcs + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/bind.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/open.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/result.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/error.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/compare.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/search.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/controls.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/messages.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/references.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/extended.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/cyrus.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/modify.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/add.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/modrdn.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/delete.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/abandon.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/sasl.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/sbind.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/unbind.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/cancel.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/filter.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/free.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/sort.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/passwd.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/whoami.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/vc.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/getdn.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/getentry.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/getattr.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/getvalues.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/addentry.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/request.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/os-ip.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/url.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/pagectrl.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/sortctrl.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/vlvctrl.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/init.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/options.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/print.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/string.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/util-int.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/schema.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/charray.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/os-local.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/dnssrv.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/utf-8.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/utf-8-conv.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/tls2.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/tls_o.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/tls_g.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/turn.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/ppolicy.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/dds.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/txn.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/ldap_sync.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/stctrl.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/assertion.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/deref.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/ldifutil.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/ldif.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/fetch.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/lbase64.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/msctrl.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/psearchctrl.c +) + +mkversion(ldap) + +add_library(ldap ${_libs_type} + ${_ldap_srcs} + ${CMAKE_CURRENT_BINARY_DIR}/ldap-version.c +) + +target_link_libraries(ldap + PRIVATE ${OPENSSL_LIBRARIES} +) + +target_include_directories(ldap + PRIVATE ${_extra_build_dir}/include + PRIVATE ${OPENLDAP_SOURCE_DIR}/include + PRIVATE ${OPENLDAP_SOURCE_DIR}/libraries/libldap + PRIVATE ${OPENSSL_INCLUDE_DIR} +) + +target_compile_definitions(ldap + PRIVATE LDAP_LIBRARY +) + +set(_ldap_r_specific_srcs + ${OPENLDAP_SOURCE_DIR}/libraries/libldap_r/threads.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap_r/rdwr.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap_r/tpool.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap_r/rq.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap_r/thr_posix.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap_r/thr_thr.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap_r/thr_nt.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap_r/thr_pth.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap_r/thr_stub.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap_r/thr_debug.c +) + +mkversion(ldap_r) + +add_library(ldap_r ${_libs_type} + ${_ldap_r_specific_srcs} + ${_ldap_srcs} + ${CMAKE_CURRENT_BINARY_DIR}/ldap_r-version.c +) + +target_link_libraries(ldap_r + PRIVATE ${OPENSSL_LIBRARIES} +) + +target_include_directories(ldap_r + PRIVATE ${_extra_build_dir}/include + PRIVATE ${OPENLDAP_SOURCE_DIR}/include + PRIVATE ${OPENLDAP_SOURCE_DIR}/libraries/libldap_r + PRIVATE ${OPENLDAP_SOURCE_DIR}/libraries/libldap + PRIVATE ${OPENSSL_INCLUDE_DIR} +) + +target_compile_definitions(ldap_r + PRIVATE LDAP_R_COMPILE + PRIVATE LDAP_LIBRARY +) diff --git a/contrib/openldap-cmake/Darwin_x86_64/include/lber_types.h b/contrib/openldap-cmake/Darwin_x86_64/include/lber_types.h new file mode 100644 index 00000000000..0e04ad96034 --- /dev/null +++ b/contrib/openldap-cmake/Darwin_x86_64/include/lber_types.h @@ -0,0 +1,63 @@ +/* include/lber_types.h. Generated from lber_types.hin by configure. */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software . + * + * Copyright 1998-2020 The OpenLDAP Foundation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +/* + * LBER types + */ + +#ifndef _LBER_TYPES_H +#define _LBER_TYPES_H + +#include + +LDAP_BEGIN_DECL + +/* LBER boolean, enum, integers (32 bits or larger) */ +#define LBER_INT_T int + +/* LBER tags (32 bits or larger) */ +#define LBER_TAG_T long + +/* LBER socket descriptor */ +#define LBER_SOCKET_T int + +/* LBER lengths (32 bits or larger) */ +#define LBER_LEN_T long + +/* ------------------------------------------------------------ */ + +/* booleans, enumerations, and integers */ +typedef LBER_INT_T ber_int_t; + +/* signed and unsigned versions */ +typedef signed LBER_INT_T ber_sint_t; +typedef unsigned LBER_INT_T ber_uint_t; + +/* tags */ +typedef unsigned LBER_TAG_T ber_tag_t; + +/* "socket" descriptors */ +typedef LBER_SOCKET_T ber_socket_t; + +/* lengths */ +typedef unsigned LBER_LEN_T ber_len_t; + +/* signed lengths */ +typedef signed LBER_LEN_T ber_slen_t; + +LDAP_END_DECL + +#endif /* _LBER_TYPES_H */ diff --git a/contrib/openldap-cmake/Darwin_x86_64/include/ldap_config.h b/contrib/openldap-cmake/Darwin_x86_64/include/ldap_config.h new file mode 100644 index 00000000000..ce1c6de9714 --- /dev/null +++ b/contrib/openldap-cmake/Darwin_x86_64/include/ldap_config.h @@ -0,0 +1,74 @@ +/* Generated from /Users/denis/dev/altinity/ClickHouse/contrib/openldap/include/ldap_config.hin on Sat May 9 00:43:25 +04 2020 */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software . + * + * Copyright 1998-2020 The OpenLDAP Foundation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +/* + * This file works in conjunction with OpenLDAP configure system. + * If you do no like the values below, adjust your configure options. + */ + +#ifndef _LDAP_CONFIG_H +#define _LDAP_CONFIG_H + +/* directory separator */ +#ifndef LDAP_DIRSEP +#ifndef _WIN32 +#define LDAP_DIRSEP "/" +#else +#define LDAP_DIRSEP "\\" +#endif +#endif + +/* directory for temporary files */ +#if defined(_WIN32) +# define LDAP_TMPDIR "C:\\." /* we don't have much of a choice */ +#elif defined( _P_tmpdir ) +# define LDAP_TMPDIR _P_tmpdir +#elif defined( P_tmpdir ) +# define LDAP_TMPDIR P_tmpdir +#elif defined( _PATH_TMPDIR ) +# define LDAP_TMPDIR _PATH_TMPDIR +#else +# define LDAP_TMPDIR LDAP_DIRSEP "tmp" +#endif + +/* directories */ +#ifndef LDAP_BINDIR +#define LDAP_BINDIR "/tmp/ldap-prefix/bin" +#endif +#ifndef LDAP_SBINDIR +#define LDAP_SBINDIR "/tmp/ldap-prefix/sbin" +#endif +#ifndef LDAP_DATADIR +#define LDAP_DATADIR "/tmp/ldap-prefix/share/openldap" +#endif +#ifndef LDAP_SYSCONFDIR +#define LDAP_SYSCONFDIR "/tmp/ldap-prefix/etc/openldap" +#endif +#ifndef LDAP_LIBEXECDIR +#define LDAP_LIBEXECDIR "/tmp/ldap-prefix/libexec" +#endif +#ifndef LDAP_MODULEDIR +#define LDAP_MODULEDIR "/tmp/ldap-prefix/libexec/openldap" +#endif +#ifndef LDAP_RUNDIR +#define LDAP_RUNDIR "/tmp/ldap-prefix/var" +#endif +#ifndef LDAP_LOCALEDIR +#define LDAP_LOCALEDIR "" +#endif + + +#endif /* _LDAP_CONFIG_H */ diff --git a/contrib/openldap-cmake/Darwin_x86_64/include/ldap_features.h b/contrib/openldap-cmake/Darwin_x86_64/include/ldap_features.h new file mode 100644 index 00000000000..7ec8de4e9b2 --- /dev/null +++ b/contrib/openldap-cmake/Darwin_x86_64/include/ldap_features.h @@ -0,0 +1,61 @@ +/* include/ldap_features.h. Generated from ldap_features.hin by configure. */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software . + * + * Copyright 1998-2020 The OpenLDAP Foundation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +/* + * LDAP Features + */ + +#ifndef _LDAP_FEATURES_H +#define _LDAP_FEATURES_H 1 + +/* OpenLDAP API version macros */ +#define LDAP_VENDOR_VERSION 20501 +#define LDAP_VENDOR_VERSION_MAJOR 2 +#define LDAP_VENDOR_VERSION_MINOR 5 +#define LDAP_VENDOR_VERSION_PATCH X + +/* +** WORK IN PROGRESS! +** +** OpenLDAP reentrancy/thread-safeness should be dynamically +** checked using ldap_get_option(). +** +** The -lldap implementation is not thread-safe. +** +** The -lldap_r implementation is: +** LDAP_API_FEATURE_THREAD_SAFE (basic thread safety) +** but also be: +** LDAP_API_FEATURE_SESSION_THREAD_SAFE +** LDAP_API_FEATURE_OPERATION_THREAD_SAFE +** +** The preprocessor flag LDAP_API_FEATURE_X_OPENLDAP_THREAD_SAFE +** can be used to determine if -lldap_r is available at compile +** time. You must define LDAP_THREAD_SAFE if and only if you +** link with -lldap_r. +** +** If you fail to define LDAP_THREAD_SAFE when linking with +** -lldap_r or define LDAP_THREAD_SAFE when linking with -lldap, +** provided header definitions and declarations may be incorrect. +** +*/ + +/* is -lldap_r available or not */ +#define LDAP_API_FEATURE_X_OPENLDAP_THREAD_SAFE 1 + +/* LDAP v2 Referrals */ +/* #undef LDAP_API_FEATURE_X_OPENLDAP_V2_REFERRALS */ + +#endif /* LDAP_FEATURES */ diff --git a/contrib/openldap-cmake/Darwin_x86_64/include/portable.h b/contrib/openldap-cmake/Darwin_x86_64/include/portable.h new file mode 100644 index 00000000000..e66e842acee --- /dev/null +++ b/contrib/openldap-cmake/Darwin_x86_64/include/portable.h @@ -0,0 +1,1169 @@ +/* include/portable.h. Generated from portable.hin by configure. */ +/* include/portable.hin. Generated from configure.in by autoheader. */ + + +/* begin of portable.h.pre */ +/* This work is part of OpenLDAP Software . + * + * Copyright 1998-2020 The OpenLDAP Foundation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#ifndef _LDAP_PORTABLE_H +#define _LDAP_PORTABLE_H + +/* define this if needed to get reentrant functions */ +#ifndef REENTRANT +#define REENTRANT 1 +#endif +#ifndef _REENTRANT +#define _REENTRANT 1 +#endif + +/* define this if needed to get threadsafe functions */ +#ifndef THREADSAFE +#define THREADSAFE 1 +#endif +#ifndef _THREADSAFE +#define _THREADSAFE 1 +#endif +#ifndef THREAD_SAFE +#define THREAD_SAFE 1 +#endif +#ifndef _THREAD_SAFE +#define _THREAD_SAFE 1 +#endif + +#ifndef _SGI_MP_SOURCE +#define _SGI_MP_SOURCE 1 +#endif + +/* end of portable.h.pre */ + + +/* Define if building universal (internal helper macro) */ +/* #undef AC_APPLE_UNIVERSAL_BUILD */ + +/* define to use both and */ +/* #undef BOTH_STRINGS_H */ + +/* define if cross compiling */ +/* #undef CROSS_COMPILING */ + +/* set to the number of arguments ctime_r() expects */ +#define CTIME_R_NARGS 2 + +/* define if toupper() requires islower() */ +/* #undef C_UPPER_LOWER */ + +/* define if sys_errlist is not declared in stdio.h or errno.h */ +/* #undef DECL_SYS_ERRLIST */ + +/* define to enable slapi library */ +/* #undef ENABLE_SLAPI */ + +/* defined to be the EXE extension */ +#define EXEEXT "" + +/* set to the number of arguments gethostbyaddr_r() expects */ +/* #undef GETHOSTBYADDR_R_NARGS */ + +/* set to the number of arguments gethostbyname_r() expects */ +/* #undef GETHOSTBYNAME_R_NARGS */ + +/* Define to 1 if `TIOCGWINSZ' requires . */ +/* #undef GWINSZ_IN_SYS_IOCTL */ + +/* define if you have AIX security lib */ +/* #undef HAVE_AIX_SECURITY */ + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_INET_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_NAMESER_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ASSERT_H 1 + +/* Define to 1 if you have the `bcopy' function. */ +#define HAVE_BCOPY 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_BITS_TYPES_H */ + +/* Define to 1 if you have the `chroot' function. */ +#define HAVE_CHROOT 1 + +/* Define to 1 if you have the `closesocket' function. */ +/* #undef HAVE_CLOSESOCKET */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_CONIO_H */ + +/* define if crypt(3) is available */ +/* #undef HAVE_CRYPT */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_CRYPT_H */ + +/* define if crypt_r() is also available */ +/* #undef HAVE_CRYPT_R */ + +/* Define to 1 if you have the `ctime_r' function. */ +#define HAVE_CTIME_R 1 + +/* define if you have Cyrus SASL */ +/* #undef HAVE_CYRUS_SASL */ + +/* define if your system supports /dev/poll */ +/* #undef HAVE_DEVPOLL */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_DIRECT_H */ + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +#define HAVE_DIRENT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_DLFCN_H 1 + +/* Define to 1 if you don't have `vprintf' but do have `_doprnt.' */ +/* #undef HAVE_DOPRNT */ + +/* define if system uses EBCDIC instead of ASCII */ +/* #undef HAVE_EBCDIC */ + +/* Define to 1 if you have the `endgrent' function. */ +#define HAVE_ENDGRENT 1 + +/* Define to 1 if you have the `endpwent' function. */ +#define HAVE_ENDPWENT 1 + +/* define if your system supports epoll */ +/* #undef HAVE_EPOLL */ + +/* Define to 1 if you have the header file. */ +#define HAVE_ERRNO_H 1 + +/* Define to 1 if you have the `fcntl' function. */ +#define HAVE_FCNTL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_FCNTL_H 1 + +/* define if you actually have FreeBSD fetch(3) */ +/* #undef HAVE_FETCH */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_FILIO_H */ + +/* Define to 1 if you have the `flock' function. */ +#define HAVE_FLOCK 1 + +/* Define to 1 if you have the `fstat' function. */ +#define HAVE_FSTAT 1 + +/* Define to 1 if you have the `gai_strerror' function. */ +#define HAVE_GAI_STRERROR 1 + +/* Define to 1 if you have the `getaddrinfo' function. */ +#define HAVE_GETADDRINFO 1 + +/* Define to 1 if you have the `getdtablesize' function. */ +#define HAVE_GETDTABLESIZE 1 + +/* Define to 1 if you have the `geteuid' function. */ +#define HAVE_GETEUID 1 + +/* Define to 1 if you have the `getgrgid' function. */ +#define HAVE_GETGRGID 1 + +/* Define to 1 if you have the `gethostbyaddr_r' function. */ +/* #undef HAVE_GETHOSTBYADDR_R */ + +/* Define to 1 if you have the `gethostbyname_r' function. */ +/* #undef HAVE_GETHOSTBYNAME_R */ + +/* Define to 1 if you have the `gethostname' function. */ +#define HAVE_GETHOSTNAME 1 + +/* Define to 1 if you have the `getnameinfo' function. */ +#define HAVE_GETNAMEINFO 1 + +/* Define to 1 if you have the `getopt' function. */ +#define HAVE_GETOPT 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_GETOPT_H 1 + +/* Define to 1 if you have the `getpassphrase' function. */ +/* #undef HAVE_GETPASSPHRASE */ + +/* Define to 1 if you have the `getpeereid' function. */ +#define HAVE_GETPEEREID 1 + +/* Define to 1 if you have the `getpeerucred' function. */ +/* #undef HAVE_GETPEERUCRED */ + +/* Define to 1 if you have the `getpwnam' function. */ +#define HAVE_GETPWNAM 1 + +/* Define to 1 if you have the `getpwuid' function. */ +#define HAVE_GETPWUID 1 + +/* Define to 1 if you have the `getspnam' function. */ +/* #undef HAVE_GETSPNAM */ + +/* Define to 1 if you have the `gettimeofday' function. */ +#define HAVE_GETTIMEOFDAY 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_GMP_H */ + +/* Define to 1 if you have the `gmtime_r' function. */ +#define HAVE_GMTIME_R 1 + +/* define if you have GNUtls */ +/* #undef HAVE_GNUTLS */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_GNUTLS_GNUTLS_H */ + +/* if you have GNU Pth */ +/* #undef HAVE_GNU_PTH */ + +/* Define to 1 if you have the header file. */ +#define HAVE_GRP_H 1 + +/* Define to 1 if you have the `hstrerror' function. */ +#define HAVE_HSTRERROR 1 + +/* define to you inet_aton(3) is available */ +#define HAVE_INET_ATON 1 + +/* Define to 1 if you have the `inet_ntoa_b' function. */ +/* #undef HAVE_INET_NTOA_B */ + +/* Define to 1 if you have the `inet_ntop' function. */ +#define HAVE_INET_NTOP 1 + +/* Define to 1 if you have the `initgroups' function. */ +#define HAVE_INITGROUPS 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_INTTYPES_H 1 + +/* Define to 1 if you have the `ioctl' function. */ +#define HAVE_IOCTL 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_IO_H */ + +/* define if your system supports kqueue */ +#define HAVE_KQUEUE 1 + +/* Define to 1 if you have the `gen' library (-lgen). */ +/* #undef HAVE_LIBGEN */ + +/* Define to 1 if you have the `gmp' library (-lgmp). */ +/* #undef HAVE_LIBGMP */ + +/* Define to 1 if you have the `inet' library (-linet). */ +/* #undef HAVE_LIBINET */ + +/* define if you have libtool -ltdl */ +/* #undef HAVE_LIBLTDL */ + +/* Define to 1 if you have the `net' library (-lnet). */ +/* #undef HAVE_LIBNET */ + +/* Define to 1 if you have the `nsl' library (-lnsl). */ +/* #undef HAVE_LIBNSL */ + +/* Define to 1 if you have the `nsl_s' library (-lnsl_s). */ +/* #undef HAVE_LIBNSL_S */ + +/* Define to 1 if you have the `socket' library (-lsocket). */ +/* #undef HAVE_LIBSOCKET */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_LIBUTIL_H */ + +/* Define to 1 if you have the `V3' library (-lV3). */ +/* #undef HAVE_LIBV3 */ + +/* Define to 1 if you have the header file. */ +#define HAVE_LIMITS_H 1 + +/* if you have LinuxThreads */ +/* #undef HAVE_LINUX_THREADS */ + +/* Define to 1 if you have the header file. */ +#define HAVE_LOCALE_H 1 + +/* Define to 1 if you have the `localtime_r' function. */ +#define HAVE_LOCALTIME_R 1 + +/* Define to 1 if you have the `lockf' function. */ +#define HAVE_LOCKF 1 + +/* Define to 1 if the system has the type `long long'. */ +#define HAVE_LONG_LONG 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_LTDL_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_MALLOC_H */ + +/* Define to 1 if you have the `memcpy' function. */ +#define HAVE_MEMCPY 1 + +/* Define to 1 if you have the `memmove' function. */ +#define HAVE_MEMMOVE 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_MEMORY_H 1 + +/* Define to 1 if you have the `memrchr' function. */ +/* #undef HAVE_MEMRCHR */ + +/* Define to 1 if you have the `mkstemp' function. */ +#define HAVE_MKSTEMP 1 + +/* Define to 1 if you have the `mktemp' function. */ +#define HAVE_MKTEMP 1 + +/* define this if you have mkversion */ +#define HAVE_MKVERSION 1 + +/* Define to 1 if you have the header file, and it defines `DIR'. */ +/* #undef HAVE_NDIR_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_NETINET_TCP_H 1 + +/* define if strerror_r returns char* instead of int */ +/* #undef HAVE_NONPOSIX_STRERROR_R */ + +/* if you have NT Event Log */ +/* #undef HAVE_NT_EVENT_LOG */ + +/* if you have NT Service Manager */ +/* #undef HAVE_NT_SERVICE_MANAGER */ + +/* if you have NT Threads */ +/* #undef HAVE_NT_THREADS */ + +/* define if you have OpenSSL */ +#define HAVE_OPENSSL 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_OPENSSL_BN_H */ + +/* define if you have OpenSSL with CRL checking capability */ +#define HAVE_OPENSSL_CRL 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_OPENSSL_CRYPTO_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_OPENSSL_SSL_H 1 + +/* Define to 1 if you have the `pipe' function. */ +#define HAVE_PIPE 1 + +/* Define to 1 if you have the `poll' function. */ +#define HAVE_POLL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_POLL_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_PROCESS_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_PSAP_H */ + +/* define to pthreads API spec revision */ +#define HAVE_PTHREADS 10 + +/* define if you have pthread_detach function */ +#define HAVE_PTHREAD_DETACH 1 + +/* Define to 1 if you have the `pthread_getconcurrency' function. */ +#define HAVE_PTHREAD_GETCONCURRENCY 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_PTHREAD_H 1 + +/* Define to 1 if you have the `pthread_kill' function. */ +#define HAVE_PTHREAD_KILL 1 + +/* Define to 1 if you have the `pthread_kill_other_threads_np' function. */ +/* #undef HAVE_PTHREAD_KILL_OTHER_THREADS_NP */ + +/* define if you have pthread_rwlock_destroy function */ +#define HAVE_PTHREAD_RWLOCK_DESTROY 1 + +/* Define to 1 if you have the `pthread_setconcurrency' function. */ +#define HAVE_PTHREAD_SETCONCURRENCY 1 + +/* Define to 1 if you have the `pthread_yield' function. */ +/* #undef HAVE_PTHREAD_YIELD */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_PTH_H */ + +/* Define to 1 if the system has the type `ptrdiff_t'. */ +#define HAVE_PTRDIFF_T 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_PWD_H 1 + +/* Define to 1 if you have the `read' function. */ +#define HAVE_READ 1 + +/* Define to 1 if you have the `recv' function. */ +#define HAVE_RECV 1 + +/* Define to 1 if you have the `recvfrom' function. */ +#define HAVE_RECVFROM 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_REGEX_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_RESOLV_H 1 + +/* define if you have res_query() */ +#define HAVE_RES_QUERY 1 + +/* define if OpenSSL needs RSAref */ +/* #undef HAVE_RSAREF */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SASL_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SASL_SASL_H */ + +/* define if your SASL library has sasl_version() */ +/* #undef HAVE_SASL_VERSION */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SCHED_H 1 + +/* Define to 1 if you have the `sched_yield' function. */ +#define HAVE_SCHED_YIELD 1 + +/* Define to 1 if you have the `send' function. */ +#define HAVE_SEND 1 + +/* Define to 1 if you have the `sendmsg' function. */ +#define HAVE_SENDMSG 1 + +/* Define to 1 if you have the `sendto' function. */ +#define HAVE_SENDTO 1 + +/* Define to 1 if you have the `setegid' function. */ +#define HAVE_SETEGID 1 + +/* Define to 1 if you have the `seteuid' function. */ +#define HAVE_SETEUID 1 + +/* Define to 1 if you have the `setgid' function. */ +#define HAVE_SETGID 1 + +/* Define to 1 if you have the `setpwfile' function. */ +/* #undef HAVE_SETPWFILE */ + +/* Define to 1 if you have the `setsid' function. */ +#define HAVE_SETSID 1 + +/* Define to 1 if you have the `setuid' function. */ +#define HAVE_SETUID 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SGTTY_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SHADOW_H */ + +/* Define to 1 if you have the `sigaction' function. */ +#define HAVE_SIGACTION 1 + +/* Define to 1 if you have the `signal' function. */ +#define HAVE_SIGNAL 1 + +/* Define to 1 if you have the `sigset' function. */ +#define HAVE_SIGSET 1 + +/* define if you have -lslp */ +/* #undef HAVE_SLP */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SLP_H */ + +/* Define to 1 if you have the `snprintf' function. */ +#define HAVE_SNPRINTF 1 + +/* if you have spawnlp() */ +/* #undef HAVE_SPAWNLP */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SQLEXT_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SQL_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STDDEF_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDINT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDLIB_H 1 + +/* Define to 1 if you have the `strdup' function. */ +#define HAVE_STRDUP 1 + +/* Define to 1 if you have the `strerror' function. */ +#define HAVE_STRERROR 1 + +/* Define to 1 if you have the `strerror_r' function. */ +#define HAVE_STRERROR_R 1 + +/* Define to 1 if you have the `strftime' function. */ +#define HAVE_STRFTIME 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRINGS_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRING_H 1 + +/* Define to 1 if you have the `strpbrk' function. */ +#define HAVE_STRPBRK 1 + +/* Define to 1 if you have the `strrchr' function. */ +#define HAVE_STRRCHR 1 + +/* Define to 1 if you have the `strsep' function. */ +#define HAVE_STRSEP 1 + +/* Define to 1 if you have the `strspn' function. */ +#define HAVE_STRSPN 1 + +/* Define to 1 if you have the `strstr' function. */ +#define HAVE_STRSTR 1 + +/* Define to 1 if you have the `strtol' function. */ +#define HAVE_STRTOL 1 + +/* Define to 1 if you have the `strtoll' function. */ +#define HAVE_STRTOLL 1 + +/* Define to 1 if you have the `strtoq' function. */ +#define HAVE_STRTOQ 1 + +/* Define to 1 if you have the `strtoul' function. */ +#define HAVE_STRTOUL 1 + +/* Define to 1 if you have the `strtoull' function. */ +#define HAVE_STRTOULL 1 + +/* Define to 1 if you have the `strtouq' function. */ +#define HAVE_STRTOUQ 1 + +/* Define to 1 if `msg_accrightslen' is a member of `struct msghdr'. */ +/* #undef HAVE_STRUCT_MSGHDR_MSG_ACCRIGHTSLEN */ + +/* Define to 1 if `msg_control' is a member of `struct msghdr'. */ +/* #undef HAVE_STRUCT_MSGHDR_MSG_CONTROL */ + +/* Define to 1 if `pw_gecos' is a member of `struct passwd'. */ +#define HAVE_STRUCT_PASSWD_PW_GECOS 1 + +/* Define to 1 if `pw_passwd' is a member of `struct passwd'. */ +#define HAVE_STRUCT_PASSWD_PW_PASSWD 1 + +/* Define to 1 if `st_blksize' is a member of `struct stat'. */ +#define HAVE_STRUCT_STAT_ST_BLKSIZE 1 + +/* Define to 1 if `st_fstype' is a member of `struct stat'. */ +/* #undef HAVE_STRUCT_STAT_ST_FSTYPE */ + +/* define to 1 if st_fstype is char * */ +/* #undef HAVE_STRUCT_STAT_ST_FSTYPE_CHAR */ + +/* define to 1 if st_fstype is int */ +/* #undef HAVE_STRUCT_STAT_ST_FSTYPE_INT */ + +/* Define to 1 if `st_vfstype' is a member of `struct stat'. */ +/* #undef HAVE_STRUCT_STAT_ST_VFSTYPE */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYNCH_H */ + +/* Define to 1 if you have the `sysconf' function. */ +#define HAVE_SYSCONF 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYSEXITS_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYSLOG_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_DEVPOLL_H */ + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +/* #undef HAVE_SYS_DIR_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_EPOLL_H */ + +/* define if you actually have sys_errlist in your libs */ +#define HAVE_SYS_ERRLIST 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_ERRNO_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_EVENT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_FILE_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_FILIO_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_FSTYP_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_IOCTL_H 1 + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +/* #undef HAVE_SYS_NDIR_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_PARAM_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_POLL_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_PRIVGRP_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_RESOURCE_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SELECT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SOCKET_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_STAT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SYSLOG_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TIME_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_UCRED_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_UIO_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_UN_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_UUID_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_VMOUNT_H */ + +/* Define to 1 if you have that is POSIX.1 compatible. */ +#define HAVE_SYS_WAIT_H 1 + +/* define if you have -lwrap */ +/* #undef HAVE_TCPD */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_TCPD_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_TERMIOS_H 1 + +/* if you have Solaris LWP (thr) package */ +/* #undef HAVE_THR */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_THREAD_H */ + +/* Define to 1 if you have the `thr_getconcurrency' function. */ +/* #undef HAVE_THR_GETCONCURRENCY */ + +/* Define to 1 if you have the `thr_setconcurrency' function. */ +/* #undef HAVE_THR_SETCONCURRENCY */ + +/* Define to 1 if you have the `thr_yield' function. */ +/* #undef HAVE_THR_YIELD */ + +/* define if you have TLS */ +#define HAVE_TLS 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_UNISTD_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_UTIME_H 1 + +/* define if you have uuid_generate() */ +/* #undef HAVE_UUID_GENERATE */ + +/* define if you have uuid_to_str() */ +/* #undef HAVE_UUID_TO_STR */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_UUID_UUID_H */ + +/* Define to 1 if you have the `vprintf' function. */ +#define HAVE_VPRINTF 1 + +/* Define to 1 if you have the `vsnprintf' function. */ +#define HAVE_VSNPRINTF 1 + +/* Define to 1 if you have the `wait4' function. */ +#define HAVE_WAIT4 1 + +/* Define to 1 if you have the `waitpid' function. */ +#define HAVE_WAITPID 1 + +/* define if you have winsock */ +/* #undef HAVE_WINSOCK */ + +/* define if you have winsock2 */ +/* #undef HAVE_WINSOCK2 */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_WINSOCK2_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_WINSOCK_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_WIREDTIGER_H */ + +/* Define to 1 if you have the `write' function. */ +#define HAVE_WRITE 1 + +/* define if select implicitly yields */ +#define HAVE_YIELDING_SELECT 1 + +/* Define to 1 if you have the `_vsnprintf' function. */ +/* #undef HAVE__VSNPRINTF */ + +/* define to 32-bit or greater integer type */ +#define LBER_INT_T int + +/* define to large integer type */ +#define LBER_LEN_T long + +/* define to socket descriptor type */ +#define LBER_SOCKET_T int + +/* define to large integer type */ +#define LBER_TAG_T long + +/* define to 1 if library is thread safe */ +#define LDAP_API_FEATURE_X_OPENLDAP_THREAD_SAFE 1 + +/* define to LDAP VENDOR VERSION */ +/* #undef LDAP_API_FEATURE_X_OPENLDAP_V2_REFERRALS */ + +/* define this to add debugging code */ +/* #undef LDAP_DEBUG */ + +/* define if LDAP libs are dynamic */ +/* #undef LDAP_LIBS_DYNAMIC */ + +/* define to support PF_INET6 */ +#define LDAP_PF_INET6 1 + +/* define to support PF_LOCAL */ +#define LDAP_PF_LOCAL 1 + +/* define this to add SLAPI code */ +/* #undef LDAP_SLAPI */ + +/* define this to add syslog code */ +/* #undef LDAP_SYSLOG */ + +/* Version */ +#define LDAP_VENDOR_VERSION 20501 + +/* Major */ +#define LDAP_VENDOR_VERSION_MAJOR 2 + +/* Minor */ +#define LDAP_VENDOR_VERSION_MINOR 5 + +/* Patch */ +#define LDAP_VENDOR_VERSION_PATCH X + +/* Define to the sub-directory where libtool stores uninstalled libraries. */ +#define LT_OBJDIR ".libs/" + +/* define if memcmp is not 8-bit clean or is otherwise broken */ +/* #undef NEED_MEMCMP_REPLACEMENT */ + +/* define if you have (or want) no threads */ +/* #undef NO_THREADS */ + +/* define to use the original debug style */ +/* #undef OLD_DEBUG */ + +/* Package */ +#define OPENLDAP_PACKAGE "OpenLDAP" + +/* Version */ +#define OPENLDAP_VERSION "2.5.X" + +/* Define to the address where bug reports for this package should be sent. */ +#define PACKAGE_BUGREPORT "" + +/* Define to the full name of this package. */ +#define PACKAGE_NAME "" + +/* Define to the full name and version of this package. */ +#define PACKAGE_STRING "" + +/* Define to the one symbol short name of this package. */ +#define PACKAGE_TARNAME "" + +/* Define to the home page for this package. */ +#define PACKAGE_URL "" + +/* Define to the version of this package. */ +#define PACKAGE_VERSION "" + +/* define if sched_yield yields the entire process */ +/* #undef REPLACE_BROKEN_YIELD */ + +/* Define as the return type of signal handlers (`int' or `void'). */ +#define RETSIGTYPE void + +/* Define to the type of arg 1 for `select'. */ +#define SELECT_TYPE_ARG1 int + +/* Define to the type of args 2, 3 and 4 for `select'. */ +#define SELECT_TYPE_ARG234 (fd_set *) + +/* Define to the type of arg 5 for `select'. */ +#define SELECT_TYPE_ARG5 (struct timeval *) + +/* The size of `int', as computed by sizeof. */ +#define SIZEOF_INT 4 + +/* The size of `long', as computed by sizeof. */ +#define SIZEOF_LONG 8 + +/* The size of `long long', as computed by sizeof. */ +#define SIZEOF_LONG_LONG 8 + +/* The size of `short', as computed by sizeof. */ +#define SIZEOF_SHORT 2 + +/* The size of `wchar_t', as computed by sizeof. */ +#define SIZEOF_WCHAR_T 4 + +/* define to support per-object ACIs */ +/* #undef SLAPD_ACI_ENABLED */ + +/* define to support LDAP Async Metadirectory backend */ +/* #undef SLAPD_ASYNCMETA */ + +/* define to support cleartext passwords */ +/* #undef SLAPD_CLEARTEXT */ + +/* define to support crypt(3) passwords */ +/* #undef SLAPD_CRYPT */ + +/* define to support DNS SRV backend */ +/* #undef SLAPD_DNSSRV */ + +/* define to support LDAP backend */ +/* #undef SLAPD_LDAP */ + +/* define to support MDB backend */ +/* #undef SLAPD_MDB */ + +/* define to support LDAP Metadirectory backend */ +/* #undef SLAPD_META */ + +/* define to support modules */ +/* #undef SLAPD_MODULES */ + +/* dynamically linked module */ +#define SLAPD_MOD_DYNAMIC 2 + +/* statically linked module */ +#define SLAPD_MOD_STATIC 1 + +/* define to support cn=Monitor backend */ +/* #undef SLAPD_MONITOR */ + +/* define to support NDB backend */ +/* #undef SLAPD_NDB */ + +/* define to support NULL backend */ +/* #undef SLAPD_NULL */ + +/* define for In-Directory Access Logging overlay */ +/* #undef SLAPD_OVER_ACCESSLOG */ + +/* define for Audit Logging overlay */ +/* #undef SLAPD_OVER_AUDITLOG */ + +/* define for Automatic Certificate Authority overlay */ +/* #undef SLAPD_OVER_AUTOCA */ + +/* define for Collect overlay */ +/* #undef SLAPD_OVER_COLLECT */ + +/* define for Attribute Constraint overlay */ +/* #undef SLAPD_OVER_CONSTRAINT */ + +/* define for Dynamic Directory Services overlay */ +/* #undef SLAPD_OVER_DDS */ + +/* define for Dynamic Directory Services overlay */ +/* #undef SLAPD_OVER_DEREF */ + +/* define for Dynamic Group overlay */ +/* #undef SLAPD_OVER_DYNGROUP */ + +/* define for Dynamic List overlay */ +/* #undef SLAPD_OVER_DYNLIST */ + +/* define for Reverse Group Membership overlay */ +/* #undef SLAPD_OVER_MEMBEROF */ + +/* define for Password Policy overlay */ +/* #undef SLAPD_OVER_PPOLICY */ + +/* define for Proxy Cache overlay */ +/* #undef SLAPD_OVER_PROXYCACHE */ + +/* define for Referential Integrity overlay */ +/* #undef SLAPD_OVER_REFINT */ + +/* define for Return Code overlay */ +/* #undef SLAPD_OVER_RETCODE */ + +/* define for Rewrite/Remap overlay */ +/* #undef SLAPD_OVER_RWM */ + +/* define for Sequential Modify overlay */ +/* #undef SLAPD_OVER_SEQMOD */ + +/* define for ServerSideSort/VLV overlay */ +/* #undef SLAPD_OVER_SSSVLV */ + +/* define for Syncrepl Provider overlay */ +/* #undef SLAPD_OVER_SYNCPROV */ + +/* define for Translucent Proxy overlay */ +/* #undef SLAPD_OVER_TRANSLUCENT */ + +/* define for Attribute Uniqueness overlay */ +/* #undef SLAPD_OVER_UNIQUE */ + +/* define for Value Sorting overlay */ +/* #undef SLAPD_OVER_VALSORT */ + +/* define to support PASSWD backend */ +/* #undef SLAPD_PASSWD */ + +/* define to support PERL backend */ +/* #undef SLAPD_PERL */ + +/* define to support relay backend */ +/* #undef SLAPD_RELAY */ + +/* define to support reverse lookups */ +/* #undef SLAPD_RLOOKUPS */ + +/* define to support SHELL backend */ +/* #undef SLAPD_SHELL */ + +/* define to support SOCK backend */ +/* #undef SLAPD_SOCK */ + +/* define to support SASL passwords */ +/* #undef SLAPD_SPASSWD */ + +/* define to support SQL backend */ +/* #undef SLAPD_SQL */ + +/* define to support WiredTiger backend */ +/* #undef SLAPD_WT */ + +/* define to support run-time loadable ACL */ +/* #undef SLAP_DYNACL */ + +/* Define to 1 if you have the ANSI C header files. */ +#define STDC_HEADERS 1 + +/* Define to 1 if you can safely include both and . */ +#define TIME_WITH_SYS_TIME 1 + +/* Define to 1 if your declares `struct tm'. */ +/* #undef TM_IN_SYS_TIME */ + +/* set to urandom device */ +#define URANDOM_DEVICE "/dev/urandom" + +/* define to use OpenSSL BIGNUM for MP */ +/* #undef USE_MP_BIGNUM */ + +/* define to use GMP for MP */ +/* #undef USE_MP_GMP */ + +/* define to use 'long' for MP */ +/* #undef USE_MP_LONG */ + +/* define to use 'long long' for MP */ +/* #undef USE_MP_LONG_LONG */ + +/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most + significant byte first (like Motorola and SPARC, unlike Intel). */ +#if defined AC_APPLE_UNIVERSAL_BUILD +# if defined __BIG_ENDIAN__ +# define WORDS_BIGENDIAN 1 +# endif +#else +# ifndef WORDS_BIGENDIAN +/* # undef WORDS_BIGENDIAN */ +# endif +#endif + +/* Define to the type of arg 3 for `accept'. */ +#define ber_socklen_t socklen_t + +/* Define to `char *' if does not define. */ +/* #undef caddr_t */ + +/* Define to empty if `const' does not conform to ANSI C. */ +/* #undef const */ + +/* Define to `int' if doesn't define. */ +/* #undef gid_t */ + +/* Define to `int' if does not define. */ +/* #undef mode_t */ + +/* Define to `long' if does not define. */ +/* #undef off_t */ + +/* Define to `int' if does not define. */ +/* #undef pid_t */ + +/* Define to `int' if does not define. */ +/* #undef sig_atomic_t */ + +/* Define to `unsigned' if does not define. */ +/* #undef size_t */ + +/* define to snprintf routine */ +/* #undef snprintf */ + +/* Define like ber_socklen_t if does not define. */ +/* #undef socklen_t */ + +/* Define to `signed int' if does not define. */ +/* #undef ssize_t */ + +/* Define to `int' if doesn't define. */ +/* #undef uid_t */ + +/* define as empty if volatile is not supported */ +/* #undef volatile */ + +/* define to snprintf routine */ +/* #undef vsnprintf */ + + +/* begin of portable.h.post */ + +#ifdef _WIN32 + /* don't suck in all of the win32 api */ +# define WIN32_LEAN_AND_MEAN 1 +#endif + +#ifndef LDAP_NEEDS_PROTOTYPES +/* force LDAP_P to always include prototypes */ +#define LDAP_NEEDS_PROTOTYPES 1 +#endif + +#ifndef LDAP_REL_ENG +#if (LDAP_VENDOR_VERSION == 000000) && !defined(LDAP_DEVEL) +#define LDAP_DEVEL +#endif +#if defined(LDAP_DEVEL) && !defined(LDAP_TEST) +#define LDAP_TEST +#endif +#endif + +#ifdef HAVE_STDDEF_H +# include +#endif + +#ifdef HAVE_EBCDIC +/* ASCII/EBCDIC converting replacements for stdio funcs + * vsnprintf and snprintf are used too, but they are already + * checked by the configure script + */ +#define fputs ber_pvt_fputs +#define fgets ber_pvt_fgets +#define printf ber_pvt_printf +#define fprintf ber_pvt_fprintf +#define vfprintf ber_pvt_vfprintf +#define vsprintf ber_pvt_vsprintf +#endif + +#include "ac/fdset.h" + +#include "ldap_cdefs.h" +#include "ldap_features.h" + +#include "ac/assert.h" +#include "ac/localize.h" + +#endif /* _LDAP_PORTABLE_H */ +/* end of portable.h.post */ + diff --git a/contrib/openldap-cmake/Linux_x86_64/include/lber_types.h b/contrib/openldap-cmake/Linux_x86_64/include/lber_types.h new file mode 100644 index 00000000000..0e04ad96034 --- /dev/null +++ b/contrib/openldap-cmake/Linux_x86_64/include/lber_types.h @@ -0,0 +1,63 @@ +/* include/lber_types.h. Generated from lber_types.hin by configure. */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software . + * + * Copyright 1998-2020 The OpenLDAP Foundation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +/* + * LBER types + */ + +#ifndef _LBER_TYPES_H +#define _LBER_TYPES_H + +#include + +LDAP_BEGIN_DECL + +/* LBER boolean, enum, integers (32 bits or larger) */ +#define LBER_INT_T int + +/* LBER tags (32 bits or larger) */ +#define LBER_TAG_T long + +/* LBER socket descriptor */ +#define LBER_SOCKET_T int + +/* LBER lengths (32 bits or larger) */ +#define LBER_LEN_T long + +/* ------------------------------------------------------------ */ + +/* booleans, enumerations, and integers */ +typedef LBER_INT_T ber_int_t; + +/* signed and unsigned versions */ +typedef signed LBER_INT_T ber_sint_t; +typedef unsigned LBER_INT_T ber_uint_t; + +/* tags */ +typedef unsigned LBER_TAG_T ber_tag_t; + +/* "socket" descriptors */ +typedef LBER_SOCKET_T ber_socket_t; + +/* lengths */ +typedef unsigned LBER_LEN_T ber_len_t; + +/* signed lengths */ +typedef signed LBER_LEN_T ber_slen_t; + +LDAP_END_DECL + +#endif /* _LBER_TYPES_H */ diff --git a/contrib/openldap-cmake/Linux_x86_64/include/ldap_config.h b/contrib/openldap-cmake/Linux_x86_64/include/ldap_config.h new file mode 100644 index 00000000000..f63e813c052 --- /dev/null +++ b/contrib/openldap-cmake/Linux_x86_64/include/ldap_config.h @@ -0,0 +1,74 @@ +/* Generated from /home/denis/dev/altinity/ClickHouse/contrib/openldap/include/ldap_config.hin on Sat May 9 03:29:40 +04 2020 */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software . + * + * Copyright 1998-2020 The OpenLDAP Foundation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +/* + * This file works in conjunction with OpenLDAP configure system. + * If you do no like the values below, adjust your configure options. + */ + +#ifndef _LDAP_CONFIG_H +#define _LDAP_CONFIG_H + +/* directory separator */ +#ifndef LDAP_DIRSEP +#ifndef _WIN32 +#define LDAP_DIRSEP "/" +#else +#define LDAP_DIRSEP "\\" +#endif +#endif + +/* directory for temporary files */ +#if defined(_WIN32) +# define LDAP_TMPDIR "C:\\." /* we don't have much of a choice */ +#elif defined( _P_tmpdir ) +# define LDAP_TMPDIR _P_tmpdir +#elif defined( P_tmpdir ) +# define LDAP_TMPDIR P_tmpdir +#elif defined( _PATH_TMPDIR ) +# define LDAP_TMPDIR _PATH_TMPDIR +#else +# define LDAP_TMPDIR LDAP_DIRSEP "tmp" +#endif + +/* directories */ +#ifndef LDAP_BINDIR +#define LDAP_BINDIR "/tmp/ldap-prefix/bin" +#endif +#ifndef LDAP_SBINDIR +#define LDAP_SBINDIR "/tmp/ldap-prefix/sbin" +#endif +#ifndef LDAP_DATADIR +#define LDAP_DATADIR "/tmp/ldap-prefix/share/openldap" +#endif +#ifndef LDAP_SYSCONFDIR +#define LDAP_SYSCONFDIR "/tmp/ldap-prefix/etc/openldap" +#endif +#ifndef LDAP_LIBEXECDIR +#define LDAP_LIBEXECDIR "/tmp/ldap-prefix/libexec" +#endif +#ifndef LDAP_MODULEDIR +#define LDAP_MODULEDIR "/tmp/ldap-prefix/libexec/openldap" +#endif +#ifndef LDAP_RUNDIR +#define LDAP_RUNDIR "/tmp/ldap-prefix/var" +#endif +#ifndef LDAP_LOCALEDIR +#define LDAP_LOCALEDIR "" +#endif + + +#endif /* _LDAP_CONFIG_H */ diff --git a/contrib/openldap-cmake/Linux_x86_64/include/ldap_features.h b/contrib/openldap-cmake/Linux_x86_64/include/ldap_features.h new file mode 100644 index 00000000000..7ec8de4e9b2 --- /dev/null +++ b/contrib/openldap-cmake/Linux_x86_64/include/ldap_features.h @@ -0,0 +1,61 @@ +/* include/ldap_features.h. Generated from ldap_features.hin by configure. */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software . + * + * Copyright 1998-2020 The OpenLDAP Foundation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +/* + * LDAP Features + */ + +#ifndef _LDAP_FEATURES_H +#define _LDAP_FEATURES_H 1 + +/* OpenLDAP API version macros */ +#define LDAP_VENDOR_VERSION 20501 +#define LDAP_VENDOR_VERSION_MAJOR 2 +#define LDAP_VENDOR_VERSION_MINOR 5 +#define LDAP_VENDOR_VERSION_PATCH X + +/* +** WORK IN PROGRESS! +** +** OpenLDAP reentrancy/thread-safeness should be dynamically +** checked using ldap_get_option(). +** +** The -lldap implementation is not thread-safe. +** +** The -lldap_r implementation is: +** LDAP_API_FEATURE_THREAD_SAFE (basic thread safety) +** but also be: +** LDAP_API_FEATURE_SESSION_THREAD_SAFE +** LDAP_API_FEATURE_OPERATION_THREAD_SAFE +** +** The preprocessor flag LDAP_API_FEATURE_X_OPENLDAP_THREAD_SAFE +** can be used to determine if -lldap_r is available at compile +** time. You must define LDAP_THREAD_SAFE if and only if you +** link with -lldap_r. +** +** If you fail to define LDAP_THREAD_SAFE when linking with +** -lldap_r or define LDAP_THREAD_SAFE when linking with -lldap, +** provided header definitions and declarations may be incorrect. +** +*/ + +/* is -lldap_r available or not */ +#define LDAP_API_FEATURE_X_OPENLDAP_THREAD_SAFE 1 + +/* LDAP v2 Referrals */ +/* #undef LDAP_API_FEATURE_X_OPENLDAP_V2_REFERRALS */ + +#endif /* LDAP_FEATURES */ diff --git a/contrib/openldap-cmake/Linux_x86_64/include/portable.h b/contrib/openldap-cmake/Linux_x86_64/include/portable.h new file mode 100644 index 00000000000..ce6cb8900e8 --- /dev/null +++ b/contrib/openldap-cmake/Linux_x86_64/include/portable.h @@ -0,0 +1,1169 @@ +/* include/portable.h. Generated from portable.hin by configure. */ +/* include/portable.hin. Generated from configure.in by autoheader. */ + + +/* begin of portable.h.pre */ +/* This work is part of OpenLDAP Software . + * + * Copyright 1998-2020 The OpenLDAP Foundation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#ifndef _LDAP_PORTABLE_H +#define _LDAP_PORTABLE_H + +/* define this if needed to get reentrant functions */ +#ifndef REENTRANT +#define REENTRANT 1 +#endif +#ifndef _REENTRANT +#define _REENTRANT 1 +#endif + +/* define this if needed to get threadsafe functions */ +#ifndef THREADSAFE +#define THREADSAFE 1 +#endif +#ifndef _THREADSAFE +#define _THREADSAFE 1 +#endif +#ifndef THREAD_SAFE +#define THREAD_SAFE 1 +#endif +#ifndef _THREAD_SAFE +#define _THREAD_SAFE 1 +#endif + +#ifndef _SGI_MP_SOURCE +#define _SGI_MP_SOURCE 1 +#endif + +/* end of portable.h.pre */ + + +/* Define if building universal (internal helper macro) */ +/* #undef AC_APPLE_UNIVERSAL_BUILD */ + +/* define to use both and */ +/* #undef BOTH_STRINGS_H */ + +/* define if cross compiling */ +/* #undef CROSS_COMPILING */ + +/* set to the number of arguments ctime_r() expects */ +#define CTIME_R_NARGS 2 + +/* define if toupper() requires islower() */ +/* #undef C_UPPER_LOWER */ + +/* define if sys_errlist is not declared in stdio.h or errno.h */ +/* #undef DECL_SYS_ERRLIST */ + +/* define to enable slapi library */ +/* #undef ENABLE_SLAPI */ + +/* defined to be the EXE extension */ +#define EXEEXT "" + +/* set to the number of arguments gethostbyaddr_r() expects */ +#define GETHOSTBYADDR_R_NARGS 8 + +/* set to the number of arguments gethostbyname_r() expects */ +#define GETHOSTBYNAME_R_NARGS 6 + +/* Define to 1 if `TIOCGWINSZ' requires . */ +#define GWINSZ_IN_SYS_IOCTL 1 + +/* define if you have AIX security lib */ +/* #undef HAVE_AIX_SECURITY */ + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_INET_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_NAMESER_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ASSERT_H 1 + +/* Define to 1 if you have the `bcopy' function. */ +#define HAVE_BCOPY 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_BITS_TYPES_H 1 + +/* Define to 1 if you have the `chroot' function. */ +#define HAVE_CHROOT 1 + +/* Define to 1 if you have the `closesocket' function. */ +/* #undef HAVE_CLOSESOCKET */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_CONIO_H */ + +/* define if crypt(3) is available */ +/* #undef HAVE_CRYPT */ + +/* Define to 1 if you have the header file. */ +#define HAVE_CRYPT_H 1 + +/* define if crypt_r() is also available */ +/* #undef HAVE_CRYPT_R */ + +/* Define to 1 if you have the `ctime_r' function. */ +#define HAVE_CTIME_R 1 + +/* define if you have Cyrus SASL */ +/* #undef HAVE_CYRUS_SASL */ + +/* define if your system supports /dev/poll */ +/* #undef HAVE_DEVPOLL */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_DIRECT_H */ + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +#define HAVE_DIRENT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_DLFCN_H 1 + +/* Define to 1 if you don't have `vprintf' but do have `_doprnt.' */ +/* #undef HAVE_DOPRNT */ + +/* define if system uses EBCDIC instead of ASCII */ +/* #undef HAVE_EBCDIC */ + +/* Define to 1 if you have the `endgrent' function. */ +#define HAVE_ENDGRENT 1 + +/* Define to 1 if you have the `endpwent' function. */ +#define HAVE_ENDPWENT 1 + +/* define if your system supports epoll */ +#define HAVE_EPOLL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ERRNO_H 1 + +/* Define to 1 if you have the `fcntl' function. */ +#define HAVE_FCNTL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_FCNTL_H 1 + +/* define if you actually have FreeBSD fetch(3) */ +/* #undef HAVE_FETCH */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_FILIO_H */ + +/* Define to 1 if you have the `flock' function. */ +#define HAVE_FLOCK 1 + +/* Define to 1 if you have the `fstat' function. */ +#define HAVE_FSTAT 1 + +/* Define to 1 if you have the `gai_strerror' function. */ +#define HAVE_GAI_STRERROR 1 + +/* Define to 1 if you have the `getaddrinfo' function. */ +#define HAVE_GETADDRINFO 1 + +/* Define to 1 if you have the `getdtablesize' function. */ +#define HAVE_GETDTABLESIZE 1 + +/* Define to 1 if you have the `geteuid' function. */ +#define HAVE_GETEUID 1 + +/* Define to 1 if you have the `getgrgid' function. */ +#define HAVE_GETGRGID 1 + +/* Define to 1 if you have the `gethostbyaddr_r' function. */ +#define HAVE_GETHOSTBYADDR_R 1 + +/* Define to 1 if you have the `gethostbyname_r' function. */ +#define HAVE_GETHOSTBYNAME_R 1 + +/* Define to 1 if you have the `gethostname' function. */ +#define HAVE_GETHOSTNAME 1 + +/* Define to 1 if you have the `getnameinfo' function. */ +#define HAVE_GETNAMEINFO 1 + +/* Define to 1 if you have the `getopt' function. */ +#define HAVE_GETOPT 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_GETOPT_H 1 + +/* Define to 1 if you have the `getpassphrase' function. */ +/* #undef HAVE_GETPASSPHRASE */ + +/* Define to 1 if you have the `getpeereid' function. */ +/* #undef HAVE_GETPEEREID */ + +/* Define to 1 if you have the `getpeerucred' function. */ +/* #undef HAVE_GETPEERUCRED */ + +/* Define to 1 if you have the `getpwnam' function. */ +#define HAVE_GETPWNAM 1 + +/* Define to 1 if you have the `getpwuid' function. */ +#define HAVE_GETPWUID 1 + +/* Define to 1 if you have the `getspnam' function. */ +#define HAVE_GETSPNAM 1 + +/* Define to 1 if you have the `gettimeofday' function. */ +#define HAVE_GETTIMEOFDAY 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_GMP_H */ + +/* Define to 1 if you have the `gmtime_r' function. */ +#define HAVE_GMTIME_R 1 + +/* define if you have GNUtls */ +/* #undef HAVE_GNUTLS */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_GNUTLS_GNUTLS_H */ + +/* if you have GNU Pth */ +/* #undef HAVE_GNU_PTH */ + +/* Define to 1 if you have the header file. */ +#define HAVE_GRP_H 1 + +/* Define to 1 if you have the `hstrerror' function. */ +#define HAVE_HSTRERROR 1 + +/* define to you inet_aton(3) is available */ +#define HAVE_INET_ATON 1 + +/* Define to 1 if you have the `inet_ntoa_b' function. */ +/* #undef HAVE_INET_NTOA_B */ + +/* Define to 1 if you have the `inet_ntop' function. */ +#define HAVE_INET_NTOP 1 + +/* Define to 1 if you have the `initgroups' function. */ +#define HAVE_INITGROUPS 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_INTTYPES_H 1 + +/* Define to 1 if you have the `ioctl' function. */ +#define HAVE_IOCTL 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_IO_H */ + +/* define if your system supports kqueue */ +/* #undef HAVE_KQUEUE */ + +/* Define to 1 if you have the `gen' library (-lgen). */ +/* #undef HAVE_LIBGEN */ + +/* Define to 1 if you have the `gmp' library (-lgmp). */ +/* #undef HAVE_LIBGMP */ + +/* Define to 1 if you have the `inet' library (-linet). */ +/* #undef HAVE_LIBINET */ + +/* define if you have libtool -ltdl */ +/* #undef HAVE_LIBLTDL */ + +/* Define to 1 if you have the `net' library (-lnet). */ +/* #undef HAVE_LIBNET */ + +/* Define to 1 if you have the `nsl' library (-lnsl). */ +/* #undef HAVE_LIBNSL */ + +/* Define to 1 if you have the `nsl_s' library (-lnsl_s). */ +/* #undef HAVE_LIBNSL_S */ + +/* Define to 1 if you have the `socket' library (-lsocket). */ +/* #undef HAVE_LIBSOCKET */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_LIBUTIL_H */ + +/* Define to 1 if you have the `V3' library (-lV3). */ +/* #undef HAVE_LIBV3 */ + +/* Define to 1 if you have the header file. */ +#define HAVE_LIMITS_H 1 + +/* if you have LinuxThreads */ +/* #undef HAVE_LINUX_THREADS */ + +/* Define to 1 if you have the header file. */ +#define HAVE_LOCALE_H 1 + +/* Define to 1 if you have the `localtime_r' function. */ +#define HAVE_LOCALTIME_R 1 + +/* Define to 1 if you have the `lockf' function. */ +#define HAVE_LOCKF 1 + +/* Define to 1 if the system has the type `long long'. */ +#define HAVE_LONG_LONG 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_LTDL_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_MALLOC_H 1 + +/* Define to 1 if you have the `memcpy' function. */ +#define HAVE_MEMCPY 1 + +/* Define to 1 if you have the `memmove' function. */ +#define HAVE_MEMMOVE 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_MEMORY_H 1 + +/* Define to 1 if you have the `memrchr' function. */ +#define HAVE_MEMRCHR 1 + +/* Define to 1 if you have the `mkstemp' function. */ +#define HAVE_MKSTEMP 1 + +/* Define to 1 if you have the `mktemp' function. */ +#define HAVE_MKTEMP 1 + +/* define this if you have mkversion */ +#define HAVE_MKVERSION 1 + +/* Define to 1 if you have the header file, and it defines `DIR'. */ +/* #undef HAVE_NDIR_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_NETINET_TCP_H 1 + +/* define if strerror_r returns char* instead of int */ +/* #undef HAVE_NONPOSIX_STRERROR_R */ + +/* if you have NT Event Log */ +/* #undef HAVE_NT_EVENT_LOG */ + +/* if you have NT Service Manager */ +/* #undef HAVE_NT_SERVICE_MANAGER */ + +/* if you have NT Threads */ +/* #undef HAVE_NT_THREADS */ + +/* define if you have OpenSSL */ +#define HAVE_OPENSSL 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_OPENSSL_BN_H */ + +/* define if you have OpenSSL with CRL checking capability */ +#define HAVE_OPENSSL_CRL 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_OPENSSL_CRYPTO_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_OPENSSL_SSL_H 1 + +/* Define to 1 if you have the `pipe' function. */ +#define HAVE_PIPE 1 + +/* Define to 1 if you have the `poll' function. */ +#define HAVE_POLL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_POLL_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_PROCESS_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_PSAP_H */ + +/* define to pthreads API spec revision */ +#define HAVE_PTHREADS 10 + +/* define if you have pthread_detach function */ +#define HAVE_PTHREAD_DETACH 1 + +/* Define to 1 if you have the `pthread_getconcurrency' function. */ +#define HAVE_PTHREAD_GETCONCURRENCY 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_PTHREAD_H 1 + +/* Define to 1 if you have the `pthread_kill' function. */ +#define HAVE_PTHREAD_KILL 1 + +/* Define to 1 if you have the `pthread_kill_other_threads_np' function. */ +/* #undef HAVE_PTHREAD_KILL_OTHER_THREADS_NP */ + +/* define if you have pthread_rwlock_destroy function */ +#define HAVE_PTHREAD_RWLOCK_DESTROY 1 + +/* Define to 1 if you have the `pthread_setconcurrency' function. */ +#define HAVE_PTHREAD_SETCONCURRENCY 1 + +/* Define to 1 if you have the `pthread_yield' function. */ +#define HAVE_PTHREAD_YIELD 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_PTH_H */ + +/* Define to 1 if the system has the type `ptrdiff_t'. */ +#define HAVE_PTRDIFF_T 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_PWD_H 1 + +/* Define to 1 if you have the `read' function. */ +#define HAVE_READ 1 + +/* Define to 1 if you have the `recv' function. */ +#define HAVE_RECV 1 + +/* Define to 1 if you have the `recvfrom' function. */ +#define HAVE_RECVFROM 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_REGEX_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_RESOLV_H 1 + +/* define if you have res_query() */ +#define HAVE_RES_QUERY 1 + +/* define if OpenSSL needs RSAref */ +/* #undef HAVE_RSAREF */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SASL_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SASL_SASL_H */ + +/* define if your SASL library has sasl_version() */ +/* #undef HAVE_SASL_VERSION */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SCHED_H 1 + +/* Define to 1 if you have the `sched_yield' function. */ +#define HAVE_SCHED_YIELD 1 + +/* Define to 1 if you have the `send' function. */ +#define HAVE_SEND 1 + +/* Define to 1 if you have the `sendmsg' function. */ +#define HAVE_SENDMSG 1 + +/* Define to 1 if you have the `sendto' function. */ +#define HAVE_SENDTO 1 + +/* Define to 1 if you have the `setegid' function. */ +#define HAVE_SETEGID 1 + +/* Define to 1 if you have the `seteuid' function. */ +#define HAVE_SETEUID 1 + +/* Define to 1 if you have the `setgid' function. */ +#define HAVE_SETGID 1 + +/* Define to 1 if you have the `setpwfile' function. */ +/* #undef HAVE_SETPWFILE */ + +/* Define to 1 if you have the `setsid' function. */ +#define HAVE_SETSID 1 + +/* Define to 1 if you have the `setuid' function. */ +#define HAVE_SETUID 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SGTTY_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SHADOW_H 1 + +/* Define to 1 if you have the `sigaction' function. */ +#define HAVE_SIGACTION 1 + +/* Define to 1 if you have the `signal' function. */ +#define HAVE_SIGNAL 1 + +/* Define to 1 if you have the `sigset' function. */ +#define HAVE_SIGSET 1 + +/* define if you have -lslp */ +/* #undef HAVE_SLP */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SLP_H */ + +/* Define to 1 if you have the `snprintf' function. */ +#define HAVE_SNPRINTF 1 + +/* if you have spawnlp() */ +/* #undef HAVE_SPAWNLP */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SQLEXT_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SQL_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STDDEF_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDINT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDLIB_H 1 + +/* Define to 1 if you have the `strdup' function. */ +#define HAVE_STRDUP 1 + +/* Define to 1 if you have the `strerror' function. */ +#define HAVE_STRERROR 1 + +/* Define to 1 if you have the `strerror_r' function. */ +#define HAVE_STRERROR_R 1 + +/* Define to 1 if you have the `strftime' function. */ +#define HAVE_STRFTIME 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRINGS_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRING_H 1 + +/* Define to 1 if you have the `strpbrk' function. */ +#define HAVE_STRPBRK 1 + +/* Define to 1 if you have the `strrchr' function. */ +#define HAVE_STRRCHR 1 + +/* Define to 1 if you have the `strsep' function. */ +#define HAVE_STRSEP 1 + +/* Define to 1 if you have the `strspn' function. */ +#define HAVE_STRSPN 1 + +/* Define to 1 if you have the `strstr' function. */ +#define HAVE_STRSTR 1 + +/* Define to 1 if you have the `strtol' function. */ +#define HAVE_STRTOL 1 + +/* Define to 1 if you have the `strtoll' function. */ +#define HAVE_STRTOLL 1 + +/* Define to 1 if you have the `strtoq' function. */ +#define HAVE_STRTOQ 1 + +/* Define to 1 if you have the `strtoul' function. */ +#define HAVE_STRTOUL 1 + +/* Define to 1 if you have the `strtoull' function. */ +#define HAVE_STRTOULL 1 + +/* Define to 1 if you have the `strtouq' function. */ +#define HAVE_STRTOUQ 1 + +/* Define to 1 if `msg_accrightslen' is a member of `struct msghdr'. */ +/* #undef HAVE_STRUCT_MSGHDR_MSG_ACCRIGHTSLEN */ + +/* Define to 1 if `msg_control' is a member of `struct msghdr'. */ +#define HAVE_STRUCT_MSGHDR_MSG_CONTROL 1 + +/* Define to 1 if `pw_gecos' is a member of `struct passwd'. */ +#define HAVE_STRUCT_PASSWD_PW_GECOS 1 + +/* Define to 1 if `pw_passwd' is a member of `struct passwd'. */ +#define HAVE_STRUCT_PASSWD_PW_PASSWD 1 + +/* Define to 1 if `st_blksize' is a member of `struct stat'. */ +#define HAVE_STRUCT_STAT_ST_BLKSIZE 1 + +/* Define to 1 if `st_fstype' is a member of `struct stat'. */ +/* #undef HAVE_STRUCT_STAT_ST_FSTYPE */ + +/* define to 1 if st_fstype is char * */ +/* #undef HAVE_STRUCT_STAT_ST_FSTYPE_CHAR */ + +/* define to 1 if st_fstype is int */ +/* #undef HAVE_STRUCT_STAT_ST_FSTYPE_INT */ + +/* Define to 1 if `st_vfstype' is a member of `struct stat'. */ +/* #undef HAVE_STRUCT_STAT_ST_VFSTYPE */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYNCH_H */ + +/* Define to 1 if you have the `sysconf' function. */ +#define HAVE_SYSCONF 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYSEXITS_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYSLOG_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_DEVPOLL_H */ + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +/* #undef HAVE_SYS_DIR_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_EPOLL_H 1 + +/* define if you actually have sys_errlist in your libs */ +#define HAVE_SYS_ERRLIST 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_ERRNO_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_EVENT_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_FILE_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_FILIO_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_FSTYP_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_IOCTL_H 1 + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +/* #undef HAVE_SYS_NDIR_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_PARAM_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_POLL_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_PRIVGRP_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_RESOURCE_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SELECT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SOCKET_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_STAT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SYSLOG_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TIME_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_UCRED_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_UIO_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_UN_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_UUID_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_VMOUNT_H */ + +/* Define to 1 if you have that is POSIX.1 compatible. */ +#define HAVE_SYS_WAIT_H 1 + +/* define if you have -lwrap */ +/* #undef HAVE_TCPD */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_TCPD_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_TERMIOS_H 1 + +/* if you have Solaris LWP (thr) package */ +/* #undef HAVE_THR */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_THREAD_H */ + +/* Define to 1 if you have the `thr_getconcurrency' function. */ +/* #undef HAVE_THR_GETCONCURRENCY */ + +/* Define to 1 if you have the `thr_setconcurrency' function. */ +/* #undef HAVE_THR_SETCONCURRENCY */ + +/* Define to 1 if you have the `thr_yield' function. */ +/* #undef HAVE_THR_YIELD */ + +/* define if you have TLS */ +#define HAVE_TLS 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_UNISTD_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_UTIME_H 1 + +/* define if you have uuid_generate() */ +/* #undef HAVE_UUID_GENERATE */ + +/* define if you have uuid_to_str() */ +/* #undef HAVE_UUID_TO_STR */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_UUID_UUID_H */ + +/* Define to 1 if you have the `vprintf' function. */ +#define HAVE_VPRINTF 1 + +/* Define to 1 if you have the `vsnprintf' function. */ +#define HAVE_VSNPRINTF 1 + +/* Define to 1 if you have the `wait4' function. */ +#define HAVE_WAIT4 1 + +/* Define to 1 if you have the `waitpid' function. */ +#define HAVE_WAITPID 1 + +/* define if you have winsock */ +/* #undef HAVE_WINSOCK */ + +/* define if you have winsock2 */ +/* #undef HAVE_WINSOCK2 */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_WINSOCK2_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_WINSOCK_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_WIREDTIGER_H */ + +/* Define to 1 if you have the `write' function. */ +#define HAVE_WRITE 1 + +/* define if select implicitly yields */ +#define HAVE_YIELDING_SELECT 1 + +/* Define to 1 if you have the `_vsnprintf' function. */ +/* #undef HAVE__VSNPRINTF */ + +/* define to 32-bit or greater integer type */ +#define LBER_INT_T int + +/* define to large integer type */ +#define LBER_LEN_T long + +/* define to socket descriptor type */ +#define LBER_SOCKET_T int + +/* define to large integer type */ +#define LBER_TAG_T long + +/* define to 1 if library is thread safe */ +#define LDAP_API_FEATURE_X_OPENLDAP_THREAD_SAFE 1 + +/* define to LDAP VENDOR VERSION */ +/* #undef LDAP_API_FEATURE_X_OPENLDAP_V2_REFERRALS */ + +/* define this to add debugging code */ +/* #undef LDAP_DEBUG */ + +/* define if LDAP libs are dynamic */ +/* #undef LDAP_LIBS_DYNAMIC */ + +/* define to support PF_INET6 */ +#define LDAP_PF_INET6 1 + +/* define to support PF_LOCAL */ +#define LDAP_PF_LOCAL 1 + +/* define this to add SLAPI code */ +/* #undef LDAP_SLAPI */ + +/* define this to add syslog code */ +/* #undef LDAP_SYSLOG */ + +/* Version */ +#define LDAP_VENDOR_VERSION 20501 + +/* Major */ +#define LDAP_VENDOR_VERSION_MAJOR 2 + +/* Minor */ +#define LDAP_VENDOR_VERSION_MINOR 5 + +/* Patch */ +#define LDAP_VENDOR_VERSION_PATCH X + +/* Define to the sub-directory where libtool stores uninstalled libraries. */ +#define LT_OBJDIR ".libs/" + +/* define if memcmp is not 8-bit clean or is otherwise broken */ +/* #undef NEED_MEMCMP_REPLACEMENT */ + +/* define if you have (or want) no threads */ +/* #undef NO_THREADS */ + +/* define to use the original debug style */ +/* #undef OLD_DEBUG */ + +/* Package */ +#define OPENLDAP_PACKAGE "OpenLDAP" + +/* Version */ +#define OPENLDAP_VERSION "2.5.X" + +/* Define to the address where bug reports for this package should be sent. */ +#define PACKAGE_BUGREPORT "" + +/* Define to the full name of this package. */ +#define PACKAGE_NAME "" + +/* Define to the full name and version of this package. */ +#define PACKAGE_STRING "" + +/* Define to the one symbol short name of this package. */ +#define PACKAGE_TARNAME "" + +/* Define to the home page for this package. */ +#define PACKAGE_URL "" + +/* Define to the version of this package. */ +#define PACKAGE_VERSION "" + +/* define if sched_yield yields the entire process */ +/* #undef REPLACE_BROKEN_YIELD */ + +/* Define as the return type of signal handlers (`int' or `void'). */ +#define RETSIGTYPE void + +/* Define to the type of arg 1 for `select'. */ +#define SELECT_TYPE_ARG1 int + +/* Define to the type of args 2, 3 and 4 for `select'. */ +#define SELECT_TYPE_ARG234 (fd_set *) + +/* Define to the type of arg 5 for `select'. */ +#define SELECT_TYPE_ARG5 (struct timeval *) + +/* The size of `int', as computed by sizeof. */ +#define SIZEOF_INT 4 + +/* The size of `long', as computed by sizeof. */ +#define SIZEOF_LONG 8 + +/* The size of `long long', as computed by sizeof. */ +#define SIZEOF_LONG_LONG 8 + +/* The size of `short', as computed by sizeof. */ +#define SIZEOF_SHORT 2 + +/* The size of `wchar_t', as computed by sizeof. */ +#define SIZEOF_WCHAR_T 4 + +/* define to support per-object ACIs */ +/* #undef SLAPD_ACI_ENABLED */ + +/* define to support LDAP Async Metadirectory backend */ +/* #undef SLAPD_ASYNCMETA */ + +/* define to support cleartext passwords */ +/* #undef SLAPD_CLEARTEXT */ + +/* define to support crypt(3) passwords */ +/* #undef SLAPD_CRYPT */ + +/* define to support DNS SRV backend */ +/* #undef SLAPD_DNSSRV */ + +/* define to support LDAP backend */ +/* #undef SLAPD_LDAP */ + +/* define to support MDB backend */ +/* #undef SLAPD_MDB */ + +/* define to support LDAP Metadirectory backend */ +/* #undef SLAPD_META */ + +/* define to support modules */ +/* #undef SLAPD_MODULES */ + +/* dynamically linked module */ +#define SLAPD_MOD_DYNAMIC 2 + +/* statically linked module */ +#define SLAPD_MOD_STATIC 1 + +/* define to support cn=Monitor backend */ +/* #undef SLAPD_MONITOR */ + +/* define to support NDB backend */ +/* #undef SLAPD_NDB */ + +/* define to support NULL backend */ +/* #undef SLAPD_NULL */ + +/* define for In-Directory Access Logging overlay */ +/* #undef SLAPD_OVER_ACCESSLOG */ + +/* define for Audit Logging overlay */ +/* #undef SLAPD_OVER_AUDITLOG */ + +/* define for Automatic Certificate Authority overlay */ +/* #undef SLAPD_OVER_AUTOCA */ + +/* define for Collect overlay */ +/* #undef SLAPD_OVER_COLLECT */ + +/* define for Attribute Constraint overlay */ +/* #undef SLAPD_OVER_CONSTRAINT */ + +/* define for Dynamic Directory Services overlay */ +/* #undef SLAPD_OVER_DDS */ + +/* define for Dynamic Directory Services overlay */ +/* #undef SLAPD_OVER_DEREF */ + +/* define for Dynamic Group overlay */ +/* #undef SLAPD_OVER_DYNGROUP */ + +/* define for Dynamic List overlay */ +/* #undef SLAPD_OVER_DYNLIST */ + +/* define for Reverse Group Membership overlay */ +/* #undef SLAPD_OVER_MEMBEROF */ + +/* define for Password Policy overlay */ +/* #undef SLAPD_OVER_PPOLICY */ + +/* define for Proxy Cache overlay */ +/* #undef SLAPD_OVER_PROXYCACHE */ + +/* define for Referential Integrity overlay */ +/* #undef SLAPD_OVER_REFINT */ + +/* define for Return Code overlay */ +/* #undef SLAPD_OVER_RETCODE */ + +/* define for Rewrite/Remap overlay */ +/* #undef SLAPD_OVER_RWM */ + +/* define for Sequential Modify overlay */ +/* #undef SLAPD_OVER_SEQMOD */ + +/* define for ServerSideSort/VLV overlay */ +/* #undef SLAPD_OVER_SSSVLV */ + +/* define for Syncrepl Provider overlay */ +/* #undef SLAPD_OVER_SYNCPROV */ + +/* define for Translucent Proxy overlay */ +/* #undef SLAPD_OVER_TRANSLUCENT */ + +/* define for Attribute Uniqueness overlay */ +/* #undef SLAPD_OVER_UNIQUE */ + +/* define for Value Sorting overlay */ +/* #undef SLAPD_OVER_VALSORT */ + +/* define to support PASSWD backend */ +/* #undef SLAPD_PASSWD */ + +/* define to support PERL backend */ +/* #undef SLAPD_PERL */ + +/* define to support relay backend */ +/* #undef SLAPD_RELAY */ + +/* define to support reverse lookups */ +/* #undef SLAPD_RLOOKUPS */ + +/* define to support SHELL backend */ +/* #undef SLAPD_SHELL */ + +/* define to support SOCK backend */ +/* #undef SLAPD_SOCK */ + +/* define to support SASL passwords */ +/* #undef SLAPD_SPASSWD */ + +/* define to support SQL backend */ +/* #undef SLAPD_SQL */ + +/* define to support WiredTiger backend */ +/* #undef SLAPD_WT */ + +/* define to support run-time loadable ACL */ +/* #undef SLAP_DYNACL */ + +/* Define to 1 if you have the ANSI C header files. */ +#define STDC_HEADERS 1 + +/* Define to 1 if you can safely include both and . */ +#define TIME_WITH_SYS_TIME 1 + +/* Define to 1 if your declares `struct tm'. */ +/* #undef TM_IN_SYS_TIME */ + +/* set to urandom device */ +#define URANDOM_DEVICE "/dev/urandom" + +/* define to use OpenSSL BIGNUM for MP */ +/* #undef USE_MP_BIGNUM */ + +/* define to use GMP for MP */ +/* #undef USE_MP_GMP */ + +/* define to use 'long' for MP */ +/* #undef USE_MP_LONG */ + +/* define to use 'long long' for MP */ +/* #undef USE_MP_LONG_LONG */ + +/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most + significant byte first (like Motorola and SPARC, unlike Intel). */ +#if defined AC_APPLE_UNIVERSAL_BUILD +# if defined __BIG_ENDIAN__ +# define WORDS_BIGENDIAN 1 +# endif +#else +# ifndef WORDS_BIGENDIAN +/* # undef WORDS_BIGENDIAN */ +# endif +#endif + +/* Define to the type of arg 3 for `accept'. */ +#define ber_socklen_t socklen_t + +/* Define to `char *' if does not define. */ +/* #undef caddr_t */ + +/* Define to empty if `const' does not conform to ANSI C. */ +/* #undef const */ + +/* Define to `int' if doesn't define. */ +/* #undef gid_t */ + +/* Define to `int' if does not define. */ +/* #undef mode_t */ + +/* Define to `long' if does not define. */ +/* #undef off_t */ + +/* Define to `int' if does not define. */ +/* #undef pid_t */ + +/* Define to `int' if does not define. */ +/* #undef sig_atomic_t */ + +/* Define to `unsigned' if does not define. */ +/* #undef size_t */ + +/* define to snprintf routine */ +/* #undef snprintf */ + +/* Define like ber_socklen_t if does not define. */ +/* #undef socklen_t */ + +/* Define to `signed int' if does not define. */ +/* #undef ssize_t */ + +/* Define to `int' if doesn't define. */ +/* #undef uid_t */ + +/* define as empty if volatile is not supported */ +/* #undef volatile */ + +/* define to snprintf routine */ +/* #undef vsnprintf */ + + +/* begin of portable.h.post */ + +#ifdef _WIN32 + /* don't suck in all of the win32 api */ +# define WIN32_LEAN_AND_MEAN 1 +#endif + +#ifndef LDAP_NEEDS_PROTOTYPES +/* force LDAP_P to always include prototypes */ +#define LDAP_NEEDS_PROTOTYPES 1 +#endif + +#ifndef LDAP_REL_ENG +#if (LDAP_VENDOR_VERSION == 000000) && !defined(LDAP_DEVEL) +#define LDAP_DEVEL +#endif +#if defined(LDAP_DEVEL) && !defined(LDAP_TEST) +#define LDAP_TEST +#endif +#endif + +#ifdef HAVE_STDDEF_H +# include +#endif + +#ifdef HAVE_EBCDIC +/* ASCII/EBCDIC converting replacements for stdio funcs + * vsnprintf and snprintf are used too, but they are already + * checked by the configure script + */ +#define fputs ber_pvt_fputs +#define fgets ber_pvt_fgets +#define printf ber_pvt_printf +#define fprintf ber_pvt_fprintf +#define vfprintf ber_pvt_vfprintf +#define vsprintf ber_pvt_vsprintf +#endif + +#include "ac/fdset.h" + +#include "ldap_cdefs.h" +#include "ldap_features.h" + +#include "ac/assert.h" +#include "ac/localize.h" + +#endif /* _LDAP_PORTABLE_H */ +/* end of portable.h.post */ + diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e0ba1eeb2aa..3282446ddf3 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -548,6 +548,11 @@ if (OPENSSL_CRYPTO_LIBRARY) target_link_libraries (clickhouse_common_io PRIVATE ${OPENSSL_CRYPTO_LIBRARY}) endif () +if (USE_LDAP) + dbms_target_include_directories (SYSTEM BEFORE PRIVATE ${OPENLDAP_INCLUDE_DIR}) + dbms_target_link_libraries (PRIVATE ${OPENLDAP_LIBRARIES}) +endif () + dbms_target_include_directories (SYSTEM BEFORE PRIVATE ${DIVIDE_INCLUDE_DIR}) dbms_target_include_directories (SYSTEM BEFORE PRIVATE ${SPARSEHASH_INCLUDE_DIR}) From b9a6d9bf5f44422b436755c4e3c4efc32941f510 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Tue, 12 May 2020 23:34:54 +0400 Subject: [PATCH 131/738] Remove contrib/openldap temporarily --- .gitmodules | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 75ea99e9d80..cebde5699d8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -154,6 +154,3 @@ [submodule "contrib/msgpack-c"] path = contrib/msgpack-c url = https://github.com/msgpack/msgpack-c -[submodule "contrib/openldap"] - path = contrib/openldap - url = https://github.com/openldap/openldap.git From 1b9d73762b643fce150c72c4e7ae726ca736893d Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Wed, 13 May 2020 00:37:48 +0400 Subject: [PATCH 132/738] Re-add contrib/openldap --- .gitmodules | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitmodules b/.gitmodules index fb8f6796ebc..f7a16b84d37 100644 --- a/.gitmodules +++ b/.gitmodules @@ -154,3 +154,6 @@ [submodule "contrib/libcpuid"] path = contrib/libcpuid url = https://github.com/ClickHouse-Extras/libcpuid.git +[submodule "contrib/openldap"] + path = contrib/openldap + url = https://github.com/openldap/openldap.git From f9439acde1f8a02b855ea0c8aeb38cf103398f19 Mon Sep 17 00:00:00 2001 From: Denis Zhuravlev Date: Tue, 12 May 2020 17:57:11 -0300 Subject: [PATCH 133/738] typo --- docs/ru/operations/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/index.md b/docs/ru/operations/index.md index 6e3157d02e7..f3f90aec7c3 100644 --- a/docs/ru/operations/index.md +++ b/docs/ru/operations/index.md @@ -19,7 +19,7 @@ toc_title: intro - [Квоты](quotas.md) - [Системные таблицы](system-tables.md) - [Конфигурационные параметры сервера](server-configuration-parameters/index.md) -- [Тестирование севреров с помощью ClickHouse](performance-test.md) +- [Тестирование серверов с помощью ClickHouse](performance-test.md) - [Настройки](settings/index.md#settings) - [Утилиты](utilities/index.md) From e5330b4ba7e440ee11f3cf71b63c1f546c8df27b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 13 May 2020 01:12:11 +0300 Subject: [PATCH 134/738] Documentation: fix broken link --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b1c482baad..885b4fc656f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -786,4 +786,4 @@ #### Security Fix * Fixed the possibility of reading directories structure in tables with `File` table engine. This fixes [#8536](https://github.com/ClickHouse/ClickHouse/issues/8536). [#8537](https://github.com/ClickHouse/ClickHouse/pull/8537) ([alexey-milovidov](https://github.com/alexey-milovidov)) -## [Changelog for 2019](https://github.com/ClickHouse/ClickHouse/blob/master/docs/en/whats_new/changelog/2019.md) +## [Changelog for 2019](https://github.com/ClickHouse/ClickHouse/blob/master/docs/en/whats-new/changelog/2019.md) From 88039e83710833e34814c5958347ad86530dc4c1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 13 May 2020 01:17:32 +0300 Subject: [PATCH 135/738] Fix data rot @blinkov --- docs/en/whats-new/changelog/2019.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/en/whats-new/changelog/2019.md b/docs/en/whats-new/changelog/2019.md index 0dbf81ebde7..d5a984905b7 100644 --- a/docs/en/whats-new/changelog/2019.md +++ b/docs/en/whats-new/changelog/2019.md @@ -3,9 +3,9 @@ toc_priority: 77 toc_title: '2019' --- -## ClickHouse Release V19.17 {#clickhouse-release-v19-17} +## ClickHouse Release 19.17 {#clickhouse-release-v19-17} -### ClickHouse Release V19.17.6.36, 2019-12-27 {#clickhouse-release-v19-17-6-36-2019-12-27} +### ClickHouse Release 19.17.6.36, 2019-12-27 {#clickhouse-release-v19-17-6-36-2019-12-27} #### Bug Fix {#bug-fix} @@ -40,7 +40,7 @@ toc_title: '2019' - Now an exception will be thrown in case of using WITH TIES alongside LIMIT BY. And now it’s possible to use TOP with LIMIT BY. [\#7637](https://github.com/ClickHouse/ClickHouse/pull/7637) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) - Fix dictionary reload if it has `invalidate_query`, which stopped updates and some exception on previous update tries. [\#8029](https://github.com/ClickHouse/ClickHouse/pull/8029) ([alesapin](https://github.com/alesapin)) -### ClickHouse Release V19.17.4.11, 2019-11-22 {#clickhouse-release-v19-17-4-11-2019-11-22} +### ClickHouse Release 19.17.4.11, 2019-11-22 {#clickhouse-release-v19-17-4-11-2019-11-22} #### Backward Incompatible Change {#backward-incompatible-change} @@ -126,13 +126,13 @@ toc_title: '2019' - Added ANTLR4 grammar for ClickHouse SQL dialect [\#7595](https://github.com/ClickHouse/ClickHouse/issues/7595) [\#7596](https://github.com/ClickHouse/ClickHouse/pull/7596) ([alexey-milovidov](https://github.com/alexey-milovidov)) -## ClickHouse Release V19.16 {#clickhouse-release-v19-16} +## ClickHouse Release 19.16 {#clickhouse-release-v19-16} -#### ClickHouse Release V19.16.14.65, 2020-03-25 {#clickhouse-release-v19-16-14-65-2020-03-25} +#### ClickHouse Release 19.16.14.65, 2020-03-25 {#clickhouse-release-v19-16-14-65-2020-03-25} - Fixed up a bug in batched calculations of ternary logical OPs on multiple arguments (more than 10). [\#8718](https://github.com/ClickHouse/ClickHouse/pull/8718) ([Alexander Kazakov](https://github.com/Akazz)) This bugfix was backported to version 19.16 by a special request from Altinity. -#### ClickHouse Release V19.16.14.65, 2020-03-05 {#clickhouse-release-v19-16-14-65-2020-03-05} +#### ClickHouse Release 19.16.14.65, 2020-03-05 {#clickhouse-release-v19-16-14-65-2020-03-05} - Fix distributed subqueries incompatibility with older CH versions. Fixes [\#7851](https://github.com/ClickHouse/ClickHouse/issues/7851) [(tabplubix)](https://github.com/tavplubix) @@ -152,7 +152,7 @@ toc_title: '2019' - Add `deduplicate_blocks_in_dependent_materialized_views` option to control the behaviour of idempotent inserts into tables with materialized views. This new feature was added to the bugfix release by a special request from Altinity. [\#9070](https://github.com/ClickHouse/ClickHouse/pull/9070) [(urykhy)](https://github.com/urykhy) -### ClickHouse Release V19.16.2.2, 2019-10-30 {#clickhouse-release-v19-16-2-2-2019-10-30} +### ClickHouse Release 19.16.2.2, 2019-10-30 {#clickhouse-release-v19-16-2-2-2019-10-30} #### Backward Incompatible Change {#backward-incompatible-change-1} From 889f54b5494503ece17e7e8c79c75671a18a8f74 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 13 May 2020 01:21:40 +0300 Subject: [PATCH 136/738] Fix ENOENT exception on current_batch.txt in DirectoryMonitor current_batch.txt will not exist if there was no send, this is the case when all batches that was pending has been marked as pending. --- src/Storages/Distributed/DirectoryMonitor.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp index d83fbd84b4b..2a5e47fdf04 100644 --- a/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/src/Storages/Distributed/DirectoryMonitor.cpp @@ -666,7 +666,10 @@ void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map batch.send(); } - Poco::File{current_batch_file_path}.remove(); + /// current_batch.txt will not exist if there was no send + /// (this is the case when all batches that was pending has been marked as pending) + if (Poco::File{current_batch_file_path}.exists()) + Poco::File{current_batch_file_path}.remove(); } bool StorageDistributedDirectoryMonitor::isFileBrokenErrorCode(int code) From 085bafad0523ccc1ba18a99235dae3de007543b6 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 8 May 2020 01:31:31 +0300 Subject: [PATCH 137/738] Handle prefer_localhost_replica on INSERT into Distributed Right now it will issue remote send even if finally the local replica will be selected - not good I guess. This should also fix load_balancing. --- .../Distributed/DistributedBlockOutputStream.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/src/Storages/Distributed/DistributedBlockOutputStream.cpp index ae0af5f9cf4..e8c45826a35 100644 --- a/src/Storages/Distributed/DistributedBlockOutputStream.cpp +++ b/src/Storages/Distributed/DistributedBlockOutputStream.cpp @@ -275,12 +275,12 @@ ThreadPool::Job DistributedBlockOutputStream::runWritingJob(DistributedBlockOutp } const Block & shard_block = (num_shards > 1) ? job.current_shard_block : current_block; + const Settings & settings = context.getSettingsRef(); - if (!job.is_local_job) + if (!job.is_local_job || !settings.prefer_localhost_replica) { if (!job.stream) { - const Settings & settings = context.getSettingsRef(); auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(settings); if (shard_info.hasInternalReplication()) { @@ -318,7 +318,7 @@ ThreadPool::Job DistributedBlockOutputStream::runWritingJob(DistributedBlockOutp CurrentMetrics::Increment metric_increment{CurrentMetrics::DistributedSend}; job.stream->write(shard_block); } - else + else // local { if (!job.stream) { @@ -507,10 +507,11 @@ void DistributedBlockOutputStream::writeSplitAsync(const Block & block) void DistributedBlockOutputStream::writeAsyncImpl(const Block & block, const size_t shard_id) { const auto & shard_info = cluster->getShardsInfo()[shard_id]; + const auto & settings = context.getSettingsRef(); if (shard_info.hasInternalReplication()) { - if (shard_info.getLocalNodeCount() > 0) + if (shard_info.getLocalNodeCount() > 0 && settings.prefer_localhost_replica) { /// Prefer insert into current instance directly writeToLocal(block, shard_info.getLocalNodeCount()); @@ -531,7 +532,7 @@ void DistributedBlockOutputStream::writeAsyncImpl(const Block & block, const siz std::vector dir_names; for (const auto & address : cluster->getShardsAddresses()[shard_id]) if (!address.is_local) - dir_names.push_back(address.toFullString(context.getSettingsRef().use_compact_format_in_distributed_parts_names)); + dir_names.push_back(address.toFullString(settings.use_compact_format_in_distributed_parts_names)); if (!dir_names.empty()) writeToShard(block, dir_names); From 1d52c1b9d0ac0e210761471e970ba497cc78b5f2 Mon Sep 17 00:00:00 2001 From: Andrei Nekrashevich Date: Wed, 13 May 2020 01:48:55 +0300 Subject: [PATCH 138/738] Added function randomFixedString --- src/Functions/randomFixedString.cpp | 90 +++++++++++++++++++++++ src/Functions/registerFunctionsRandom.cpp | 2 + tests/performance/random_fixed_string.xml | 9 +++ 3 files changed, 101 insertions(+) create mode 100644 src/Functions/randomFixedString.cpp create mode 100644 tests/performance/random_fixed_string.xml diff --git a/src/Functions/randomFixedString.cpp b/src/Functions/randomFixedString.cpp new file mode 100644 index 00000000000..cd19dee0bbc --- /dev/null +++ b/src/Functions/randomFixedString.cpp @@ -0,0 +1,90 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int ILLEGAL_COLUMN; + extern const int DECIMAL_OVERFLOW; +} + + +/* Generate random fixed string with fully random bytes (including zero). */ +class FunctionRandomFixedString : public IFunction +{ +public: + static constexpr auto name = "randomFixedString"; + + static FunctionPtr create(const Context &) { return std::make_shared(); } + + String getName() const override { return name; } + + bool isVariadic() const override { return false; } + + size_t getNumberOfArguments() const override { return 1; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + if (!isUnsignedInteger(arguments[0].type)) + throw Exception("First argument for function " + getName() + " must be unsigned integer", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + if (!arguments[0].column) + throw Exception("First argument for function " + getName() + " must be constant", ErrorCodes::ILLEGAL_COLUMN); + + const size_t n = arguments[0].column->getUInt(0); + return std::make_shared(n); + } + + bool isDeterministic() const override { return false; } + bool isDeterministicInScopeOfQuery() const override { return false; } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + { + const auto n = block.getByPosition(arguments[0]).column->getUInt(0); + + auto col_to = ColumnFixedString::create(n); + ColumnFixedString::Chars & data_to = col_to->getChars(); + + if (input_rows_count == 0) + { + block.getByPosition(result).column = std::move(col_to); + return; + } + + size_t total_size; + if (common::mulOverflow(input_rows_count, n, total_size)) + throw Exception("Decimal math overflow", ErrorCodes::DECIMAL_OVERFLOW); + + /// Fill random bytes. + data_to.resize(total_size); + pcg64_fast rng(randomSeed()); /// TODO It is inefficient. We should use SIMD PRNG instead. + + auto * pos = data_to.data(); + auto * end = pos + data_to.size(); + while (pos < end) + { + unalignedStore(pos, rng()); + pos += sizeof(UInt64); // We have padding in column buffers that we can overwrite. + } + + block.getByPosition(result).column = std::move(col_to); + } +}; + +void registerFunctionRandomFixedString(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} diff --git a/src/Functions/registerFunctionsRandom.cpp b/src/Functions/registerFunctionsRandom.cpp index 5826fe78419..896fee24ae5 100644 --- a/src/Functions/registerFunctionsRandom.cpp +++ b/src/Functions/registerFunctionsRandom.cpp @@ -8,6 +8,7 @@ void registerFunctionRandConstant(FunctionFactory & factory); void registerFunctionGenerateUUIDv4(FunctionFactory & factory); void registerFunctionRandomPrintableASCII(FunctionFactory & factory); void registerFunctionRandomString(FunctionFactory & factory); +void registerFunctionRandomFixedString(FunctionFactory & factory); void registerFunctionsRandom(FunctionFactory & factory) { @@ -17,6 +18,7 @@ void registerFunctionsRandom(FunctionFactory & factory) registerFunctionGenerateUUIDv4(factory); registerFunctionRandomPrintableASCII(factory); registerFunctionRandomString(factory); + registerFunctionRandomFixedString(factory); } } diff --git a/tests/performance/random_fixed_string.xml b/tests/performance/random_fixed_string.xml new file mode 100644 index 00000000000..3883acf5f62 --- /dev/null +++ b/tests/performance/random_fixed_string.xml @@ -0,0 +1,9 @@ + + + + + SELECT count() FROM zeros(1000000) WHERE NOT ignore(randomFixedString(10)) + SELECT count() FROM zeros(1000000) WHERE NOT ignore(randomFixedString(100)) + SELECT count() FROM zeros(100000) WHERE NOT ignore(randomFixedString(1000)) + SELECT count() FROM zeros(10000) WHERE NOT ignore(randomFixedString(10000)) + From fd16838150dea0c3b8adbdebad8735fdbd46b7ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=82=D0=B5=D0=BC=20=D0=A1=D1=82=D1=80=D0=B5?= =?UTF-8?q?=D0=BB=D1=8C=D1=86=D0=BE=D0=B2?= Date: Wed, 13 May 2020 01:52:56 +0300 Subject: [PATCH 139/738] fixed style and build fails --- .../ComplexKeyDirectDictionary.cpp | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/Dictionaries/ComplexKeyDirectDictionary.cpp b/src/Dictionaries/ComplexKeyDirectDictionary.cpp index 333739f3ca8..81058f28ff4 100644 --- a/src/Dictionaries/ComplexKeyDirectDictionary.cpp +++ b/src/Dictionaries/ComplexKeyDirectDictionary.cpp @@ -371,7 +371,8 @@ void ComplexKeyDirectDictionary::getItemsImpl( std::vector to_load(rows); PODArray keys(rows); - for (const auto row : ext::range(0, rows)) { + for (const auto row : ext::range(0, rows)) + { const StringRef key = placeKeysInPool(row, key_columns, keys_array, *dict_struct.key, temporary_keys_pool); keys[row] = key; value_by_key[key] = get_default(row); @@ -422,7 +423,8 @@ void ComplexKeyDirectDictionary::getItemsImpl( stream->readSuffix(); - for (const auto row : ext::range(0, rows)) { + for (const auto row : ext::range(0, rows)) + { set_value(row, value_by_key[keys[row]]); } @@ -441,7 +443,8 @@ void ComplexKeyDirectDictionary::getItemsStringImpl( std::vector to_load(rows); PODArray keys(rows); - for (const auto row : ext::range(0, rows)) { + for (const auto row : ext::range(0, rows)) + { const StringRef key = placeKeysInPool(row, key_columns, keys_array, *dict_struct.key, temporary_keys_pool); keys[row] = key; value_by_key[key] = get_default(row); @@ -485,7 +488,8 @@ void ComplexKeyDirectDictionary::getItemsStringImpl( stream->readSuffix(); - for (const auto row : ext::range(0, rows)) { + for (const auto row : ext::range(0, rows)) + { set_value(row, value_by_key[keys[row]]); } @@ -509,12 +513,13 @@ void ComplexKeyDirectDictionary::has(const Attribute & attribute, const Columns const auto rows = key_columns.front()->size(); const auto keys_size = dict_struct.key->size(); StringRefs keys_array(keys_size); - MapType has_key; + MapType has_key; Arena temporary_keys_pool; std::vector to_load(rows); PODArray keys(rows); - for (const auto row : ext::range(0, rows)) { + for (const auto row : ext::range(0, rows)) + { const StringRef key = placeKeysInPool(row, key_columns, keys_array, *dict_struct.key, temporary_keys_pool); keys[row] = key; has_key[key] = 0; @@ -551,7 +556,8 @@ void ComplexKeyDirectDictionary::has(const Attribute & attribute, const Columns stream->readSuffix(); - for (const auto row : ext::range(0, rows)) { + for (const auto row : ext::range(0, rows)) + { out[row] = has_key[keys[row]]; } From 0ea341113746981576388af64efc39220a60d87b Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 13 May 2020 02:24:39 +0300 Subject: [PATCH 140/738] Add integration test for load balancing for Distributed INSERT --- .../__init__.py | 0 .../configs/remote_servers.xml | 18 ++++++ .../test.py | 62 +++++++++++++++++++ 3 files changed, 80 insertions(+) create mode 100644 tests/integration/test_insert_distributed_load_balancing/__init__.py create mode 100644 tests/integration/test_insert_distributed_load_balancing/configs/remote_servers.xml create mode 100644 tests/integration/test_insert_distributed_load_balancing/test.py diff --git a/tests/integration/test_insert_distributed_load_balancing/__init__.py b/tests/integration/test_insert_distributed_load_balancing/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_insert_distributed_load_balancing/configs/remote_servers.xml b/tests/integration/test_insert_distributed_load_balancing/configs/remote_servers.xml new file mode 100644 index 00000000000..61bc5af1f7d --- /dev/null +++ b/tests/integration/test_insert_distributed_load_balancing/configs/remote_servers.xml @@ -0,0 +1,18 @@ + + + + + true + + n2 + 9000 + + + n1 + 9000 + + + + + + diff --git a/tests/integration/test_insert_distributed_load_balancing/test.py b/tests/integration/test_insert_distributed_load_balancing/test.py new file mode 100644 index 00000000000..35fe1cdabfd --- /dev/null +++ b/tests/integration/test_insert_distributed_load_balancing/test.py @@ -0,0 +1,62 @@ +# pylint: disable=unused-argument +# pylint: disable=redefined-outer-name +# pylint: disable=line-too-long + +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +n1 = cluster.add_instance('n1', main_configs=['configs/remote_servers.xml']) +n2 = cluster.add_instance('n2', main_configs=['configs/remote_servers.xml']) + +@pytest.fixture(scope='module', autouse=True) +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + +@pytest.fixture(scope='function', autouse=True) +def create_tables(): + n1.query('DROP TABLE IF EXISTS data') + n2.query('DROP TABLE IF EXISTS data') + n1.query('DROP TABLE IF EXISTS dist') + + n1.query('CREATE TABLE data (key Int) Engine=Memory()') + n2.query('CREATE TABLE data (key Int) Engine=Memory()') + n1.query(""" + CREATE TABLE dist AS data + Engine=Distributed( + integration_test_cluster, + currentDatabase(), + data, + rand() + ) + """) + +def insert_data(**settings): + n1.query('INSERT INTO dist SELECT * FROM numbers(10)', settings=settings) + n1.query('SYSTEM FLUSH DISTRIBUTED dist') + +def test_prefer_localhost_replica_1(): + insert_data() + assert int(n1.query('SELECT count() FROM data')) == 10 + assert int(n2.query('SELECT count() FROM data')) == 0 + +def test_prefer_localhost_replica_1_load_balancing_in_order(): + insert_data(load_balancing='in_order') + assert int(n1.query('SELECT count() FROM data')) == 10 + assert int(n2.query('SELECT count() FROM data')) == 0 + +def test_prefer_localhost_replica_0_load_balancing_in_order(): + insert_data(load_balancing='in_order', prefer_localhost_replica=0) + assert int(n1.query('SELECT count() FROM data')) == 0 + assert int(n2.query('SELECT count() FROM data')) == 10 + +def test_prefer_localhost_replica_0_load_balancing_in_order_sync(): + insert_data(load_balancing='in_order', prefer_localhost_replica=0, insert_distributed_sync=1) + assert int(n1.query('SELECT count() FROM data')) == 0 + assert int(n2.query('SELECT count() FROM data')) == 10 From 1b42a6ccdc3b4791377e409dde87fe258265fae4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 13 May 2020 02:55:16 +0300 Subject: [PATCH 141/738] Fix bad tests --- .../01019_alter_materialized_view_atomic.sh | 37 ++++------ ...1019_alter_materialized_view_consistent.sh | 73 +++++++++---------- 2 files changed, 50 insertions(+), 60 deletions(-) diff --git a/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh b/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh index 626f990ef8d..cb2435289d4 100755 --- a/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh +++ b/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh @@ -5,14 +5,12 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh -trap 'kill -9 $(jobs -p)' EXIT - $CLICKHOUSE_CLIENT --multiquery </dev/null && break done - ) done -# Enough alters. -kill -INT $alter_pid - -wait - -# This was a fun ride. $CLICKHOUSE_CLIENT -q "SELECT count() FROM mv;" +wait diff --git a/tests/queries/0_stateless/01019_alter_materialized_view_consistent.sh b/tests/queries/0_stateless/01019_alter_materialized_view_consistent.sh index 32d106884b2..a4256499469 100755 --- a/tests/queries/0_stateless/01019_alter_materialized_view_consistent.sh +++ b/tests/queries/0_stateless/01019_alter_materialized_view_consistent.sh @@ -5,18 +5,16 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh -trap 'kill -9 $(jobs -p)' EXIT - $CLICKHOUSE_CLIENT --multiquery </dev/null & + while true; do + # trigger 100 concurrent inserts at a time + for i in {0..100}; do + # ignore `Possible deadlock avoided. Client should retry` + $CLICKHOUSE_CLIENT -q "${INSERT[$RANDOM % 2]}" 2>/dev/null & + done + wait done - - wait - done } function alter_thread() { - trap 'exit' INT + trap 'exit' INT - ALTER[0]="ALTER TABLE mv MODIFY QUERY SELECT v == 1 as test, v as case FROM src_a;" - ALTER[1]="ALTER TABLE mv MODIFY QUERY SELECT v == 2 as test, v as case FROM src_b;" + ALTER[0]="ALTER TABLE mv MODIFY QUERY SELECT v == 1 as test, v as case FROM src_a;" + ALTER[1]="ALTER TABLE mv MODIFY QUERY SELECT v == 2 as test, v as case FROM src_b;" - while true; do - $CLICKHOUSE_CLIENT --allow_experimental_alter_materialized_view_structure=1 \ - -q "${ALTER[$RANDOM % 2]}" - sleep "0.0$RANDOM" - done + while true; do + $CLICKHOUSE_CLIENT --allow_experimental_alter_materialized_view_structure=1 \ + -q "${ALTER[$RANDOM % 2]}" + sleep "0.0$RANDOM" + done } -insert_thread & -insert_thread_pid=$! +export -f insert_thread; +export -f alter_thread; -alter_thread & -alter_thread_pid=$! +timeout 30 bash -c insert_thread & +timeout 30 bash -c alter_thread & -while true; do - is_done=$($CLICKHOUSE_CLIENT -q "SELECT countIf(case = 1) > 0 AND countIf(case = 2) > 0 FROM mv;") +function check_thread() { + while true; do + is_done=$($CLICKHOUSE_CLIENT -q "SELECT countIf(case = 1) > 0 AND countIf(case = 2) > 0 FROM mv;") - if [ "$is_done" -eq "1" ]; then - break - fi -done + if [ "$is_done" -eq "1" ]; then + break + fi + done +} -kill -INT $insert_thread_pid -kill -INT $alter_thread_pid +export -f check_thread +timeout 30 bash -c check_thread wait From 2800df445088a4cc2d2e55de61097b4fb0c7582f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 13 May 2020 03:02:53 +0300 Subject: [PATCH 142/738] Improve tests for ALTER MOVE --- ...034_move_partition_from_table_zookeeper.sh | 88 +++++++++---------- ...ent_move_partition_from_table_zookeeper.sh | 20 ++--- 2 files changed, 54 insertions(+), 54 deletions(-) diff --git a/tests/queries/0_stateless/01034_move_partition_from_table_zookeeper.sh b/tests/queries/0_stateless/01034_move_partition_from_table_zookeeper.sh index aee0da15a67..1bff3056767 100755 --- a/tests/queries/0_stateless/01034_move_partition_from_table_zookeeper.sh +++ b/tests/queries/0_stateless/01034_move_partition_from_table_zookeeper.sh @@ -22,77 +22,77 @@ function query_with_retry echo "Query '$1' failed with '$result'" } -$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS $CLICKHOUSE_DATABASE.src NO DELAY;" -$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS $CLICKHOUSE_DATABASE.dst NO DELAY;" +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS src NO DELAY;" +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS dst NO DELAY;" sleep 1 -$CLICKHOUSE_CLIENT --query="CREATE TABLE $CLICKHOUSE_DATABASE.src (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/src', '1') PARTITION BY p ORDER BY k;" -$CLICKHOUSE_CLIENT --query="CREATE TABLE $CLICKHOUSE_DATABASE.dst (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/dst', '1') PARTITION BY p ORDER BY k SETTINGS old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=0;" +$CLICKHOUSE_CLIENT --query="CREATE TABLE src (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/src', '1') PARTITION BY p ORDER BY k;" +$CLICKHOUSE_CLIENT --query="CREATE TABLE dst (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/dst', '1') PARTITION BY p ORDER BY k SETTINGS old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=0;" -$CLICKHOUSE_CLIENT --query="INSERT INTO $CLICKHOUSE_DATABASE.src VALUES (0, '0', 1);" -$CLICKHOUSE_CLIENT --query="INSERT INTO $CLICKHOUSE_DATABASE.src VALUES (1, '0', 1);" -$CLICKHOUSE_CLIENT --query="INSERT INTO $CLICKHOUSE_DATABASE.src VALUES (1, '1', 1);" -$CLICKHOUSE_CLIENT --query="INSERT INTO $CLICKHOUSE_DATABASE.src VALUES (2, '0', 1);" +$CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (0, '0', 1);" +$CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (1, '0', 1);" +$CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (1, '1', 1);" +$CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (2, '0', 1);" $CLICKHOUSE_CLIENT --query="SELECT 'Initial';" -$CLICKHOUSE_CLIENT --query="INSERT INTO $CLICKHOUSE_DATABASE.dst VALUES (0, '1', 2);" -$CLICKHOUSE_CLIENT --query="INSERT INTO $CLICKHOUSE_DATABASE.dst VALUES (1, '1', 2), (1, '2', 2);" -$CLICKHOUSE_CLIENT --query="INSERT INTO $CLICKHOUSE_DATABASE.dst VALUES (2, '1', 2);" +$CLICKHOUSE_CLIENT --query="INSERT INTO dst VALUES (0, '1', 2);" +$CLICKHOUSE_CLIENT --query="INSERT INTO dst VALUES (1, '1', 2), (1, '2', 2);" +$CLICKHOUSE_CLIENT --query="INSERT INTO dst VALUES (2, '1', 2);" -$CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA $CLICKHOUSE_DATABASE.dst;" -$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM $CLICKHOUSE_DATABASE.src;" -$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM $CLICKHOUSE_DATABASE.dst;" +$CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA dst;" +$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM src;" +$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM dst;" $CLICKHOUSE_CLIENT --query="SELECT 'MOVE simple';" -query_with_retry "ALTER TABLE $CLICKHOUSE_DATABASE.src MOVE PARTITION 1 TO TABLE $CLICKHOUSE_DATABASE.dst;" +query_with_retry "ALTER TABLE src MOVE PARTITION 1 TO TABLE dst;" -$CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA $CLICKHOUSE_DATABASE.dst;" -$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM $CLICKHOUSE_DATABASE.src;" -$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM $CLICKHOUSE_DATABASE.dst;" +$CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA dst;" +$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM src;" +$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM dst;" -$CLICKHOUSE_CLIENT --query="DROP TABLE $CLICKHOUSE_DATABASE.src NO DELAY;" -$CLICKHOUSE_CLIENT --query="DROP TABLE $CLICKHOUSE_DATABASE.dst NO DELAY;" +$CLICKHOUSE_CLIENT --query="DROP TABLE src NO DELAY;" +$CLICKHOUSE_CLIENT --query="DROP TABLE dst NO DELAY;" sleep 1 $CLICKHOUSE_CLIENT --query="SELECT 'MOVE incompatible schema missing column';" -$CLICKHOUSE_CLIENT --query="CREATE TABLE $CLICKHOUSE_DATABASE.src (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/src', '1') PARTITION BY p ORDER BY (d, p);" -$CLICKHOUSE_CLIENT --query="CREATE TABLE $CLICKHOUSE_DATABASE.dst (p UInt64, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/dst', '1') PARTITION BY p ORDER BY (d, p) SETTINGS old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=0;" +$CLICKHOUSE_CLIENT --query="CREATE TABLE src (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/src', '1') PARTITION BY p ORDER BY (d, p);" +$CLICKHOUSE_CLIENT --query="CREATE TABLE dst (p UInt64, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/dst', '1') PARTITION BY p ORDER BY (d, p) SETTINGS old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=0;" -$CLICKHOUSE_CLIENT --query="INSERT INTO $CLICKHOUSE_DATABASE.src VALUES (0, '0', 1);" -$CLICKHOUSE_CLIENT --query="INSERT INTO $CLICKHOUSE_DATABASE.src VALUES (1, '0', 1);" -$CLICKHOUSE_CLIENT --query="INSERT INTO $CLICKHOUSE_DATABASE.src VALUES (1, '1', 1);" -$CLICKHOUSE_CLIENT --query="INSERT INTO $CLICKHOUSE_DATABASE.src VALUES (2, '0', 1);" +$CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (0, '0', 1);" +$CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (1, '0', 1);" +$CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (1, '1', 1);" +$CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (2, '0', 1);" -query_with_retry "ALTER TABLE $CLICKHOUSE_DATABASE.src MOVE PARTITION 1 TO TABLE $CLICKHOUSE_DATABASE.dst;" &>- -$CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA $CLICKHOUSE_DATABASE.dst;" +query_with_retry "ALTER TABLE src MOVE PARTITION 1 TO TABLE dst;" &>- +$CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA dst;" -$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM $CLICKHOUSE_DATABASE.src;" -$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM $CLICKHOUSE_DATABASE.dst;" +$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM src;" +$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM dst;" -$CLICKHOUSE_CLIENT --query="DROP TABLE $CLICKHOUSE_DATABASE.src NO DELAY;" -$CLICKHOUSE_CLIENT --query="DROP TABLE $CLICKHOUSE_DATABASE.dst NO DELAY;" +$CLICKHOUSE_CLIENT --query="DROP TABLE src NO DELAY;" +$CLICKHOUSE_CLIENT --query="DROP TABLE dst NO DELAY;" sleep 1 $CLICKHOUSE_CLIENT --query="SELECT 'MOVE incompatible schema different order by';" -$CLICKHOUSE_CLIENT --query="CREATE TABLE $CLICKHOUSE_DATABASE.src (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/src', '1') PARTITION BY p ORDER BY (p, k, d);" -$CLICKHOUSE_CLIENT --query="CREATE TABLE $CLICKHOUSE_DATABASE.dst (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/dst', '1') PARTITION BY p ORDER BY (d, k, p);" +$CLICKHOUSE_CLIENT --query="CREATE TABLE src (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/src', '1') PARTITION BY p ORDER BY (p, k, d);" +$CLICKHOUSE_CLIENT --query="CREATE TABLE dst (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/dst', '1') PARTITION BY p ORDER BY (d, k, p);" -$CLICKHOUSE_CLIENT --query="INSERT INTO $CLICKHOUSE_DATABASE.src VALUES (0, '0', 1);" -$CLICKHOUSE_CLIENT --query="INSERT INTO $CLICKHOUSE_DATABASE.src VALUES (1, '0', 1);" -$CLICKHOUSE_CLIENT --query="INSERT INTO $CLICKHOUSE_DATABASE.src VALUES (1, '1', 1);" -$CLICKHOUSE_CLIENT --query="INSERT INTO $CLICKHOUSE_DATABASE.src VALUES (2, '0', 1);" +$CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (0, '0', 1);" +$CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (1, '0', 1);" +$CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (1, '1', 1);" +$CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (2, '0', 1);" -query_with_retry "ALTER TABLE $CLICKHOUSE_DATABASE.src MOVE PARTITION 1 TO TABLE $CLICKHOUSE_DATABASE.dst;" &>- -$CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA $CLICKHOUSE_DATABASE.dst;" +query_with_retry "ALTER TABLE src MOVE PARTITION 1 TO TABLE dst;" &>- +$CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA dst;" -$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM $CLICKHOUSE_DATABASE.src;" -$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM $CLICKHOUSE_DATABASE.dst;" +$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM src;" +$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM dst;" -$CLICKHOUSE_CLIENT --query="DROP TABLE $CLICKHOUSE_DATABASE.src NO DELAY;" -$CLICKHOUSE_CLIENT --query="DROP TABLE $CLICKHOUSE_DATABASE.dst NO DELAY;" +$CLICKHOUSE_CLIENT --query="DROP TABLE src NO DELAY;" +$CLICKHOUSE_CLIENT --query="DROP TABLE dst NO DELAY;" sleep 1 diff --git a/tests/queries/0_stateless/01035_concurrent_move_partition_from_table_zookeeper.sh b/tests/queries/0_stateless/01035_concurrent_move_partition_from_table_zookeeper.sh index 5495feb89db..99c0798f66e 100755 --- a/tests/queries/0_stateless/01035_concurrent_move_partition_from_table_zookeeper.sh +++ b/tests/queries/0_stateless/01035_concurrent_move_partition_from_table_zookeeper.sh @@ -5,17 +5,17 @@ set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh -$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS $CLICKHOUSE_DATABASE.src;" -$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS $CLICKHOUSE_DATABASE.dst;" +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS src;" +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS dst;" -$CLICKHOUSE_CLIENT --query="CREATE TABLE $CLICKHOUSE_DATABASE.src (p UInt64, k String) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/src', '1') PARTITION BY p ORDER BY k;" -$CLICKHOUSE_CLIENT --query="CREATE TABLE $CLICKHOUSE_DATABASE.dst (p UInt64, k String) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/dst', '1') PARTITION BY p ORDER BY k SETTINGS old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=0;" +$CLICKHOUSE_CLIENT --query="CREATE TABLE src (p UInt64, k String) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/src', '1') PARTITION BY p ORDER BY k;" +$CLICKHOUSE_CLIENT --query="CREATE TABLE dst (p UInt64, k String) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/dst', '1') PARTITION BY p ORDER BY k SETTINGS old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=0;" function thread1() { while true; do - $CLICKHOUSE_CLIENT --query="ALTER TABLE $CLICKHOUSE_DATABASE.src MOVE PARTITION 1 TO TABLE $CLICKHOUSE_DATABASE.dst;" --query_id=query1 + $CLICKHOUSE_CLIENT --query="ALTER TABLE src MOVE PARTITION 1 TO TABLE dst;" --query_id=query1 done } @@ -23,7 +23,7 @@ function thread2() { while true; do - $CLICKHOUSE_CLIENT --query="INSERT INTO $CLICKHOUSE_DATABASE.src SELECT number % 2, toString(number) FROM system.numbers LIMIT 100000" --query_id=query2 + $CLICKHOUSE_CLIENT --query="INSERT INTO src SELECT number % 2, toString(number) FROM system.numbers LIMIT 100000" --query_id=query2 done } @@ -31,7 +31,7 @@ function thread3() { while true; do - $CLICKHOUSE_CLIENT --query="SELECT * FROM $CLICKHOUSE_DATABASE.src" --query_id=query3 1> /dev/null + $CLICKHOUSE_CLIENT --query="SELECT * FROM src" --query_id=query3 1> /dev/null done } @@ -39,7 +39,7 @@ function thread4() { while true; do - $CLICKHOUSE_CLIENT --query="SELECT * FROM $CLICKHOUSE_DATABASE.dst" --query_id=query4 1> /dev/null + $CLICKHOUSE_CLIENT --query="SELECT * FROM dst" --query_id=query4 1> /dev/null done } @@ -58,8 +58,8 @@ timeout $TIMEOUT bash -c thread4 2> /dev/null & wait -echo "DROP TABLE $CLICKHOUSE_DATABASE.src NO DELAY" | ${CLICKHOUSE_CLIENT} -echo "DROP TABLE $CLICKHOUSE_DATABASE.dst NO DELAY" | ${CLICKHOUSE_CLIENT} +echo "DROP TABLE src NO DELAY" | ${CLICKHOUSE_CLIENT} +echo "DROP TABLE dst NO DELAY" | ${CLICKHOUSE_CLIENT} sleep 5 # Check for deadlocks From 767a790fcea63f4634f4af71f66c25478a9d87c4 Mon Sep 17 00:00:00 2001 From: Andrei Nekrashevich Date: Wed, 13 May 2020 03:20:41 +0300 Subject: [PATCH 143/738] fix issues --- src/Functions/randomFixedString.cpp | 2 +- tests/queries/0_stateless/01277_random_fixed_string.reference | 1 + tests/queries/0_stateless/01277_random_fixed_string.sql | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/01277_random_fixed_string.reference create mode 100644 tests/queries/0_stateless/01277_random_fixed_string.sql diff --git a/src/Functions/randomFixedString.cpp b/src/Functions/randomFixedString.cpp index cd19dee0bbc..70c6b013dce 100644 --- a/src/Functions/randomFixedString.cpp +++ b/src/Functions/randomFixedString.cpp @@ -39,7 +39,7 @@ public: if (!isUnsignedInteger(arguments[0].type)) throw Exception("First argument for function " + getName() + " must be unsigned integer", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - if (!arguments[0].column) + if (!arguments[0].column || !isColumnConst(*arguments[0].column)) throw Exception("First argument for function " + getName() + " must be constant", ErrorCodes::ILLEGAL_COLUMN); const size_t n = arguments[0].column->getUInt(0); diff --git a/tests/queries/0_stateless/01277_random_fixed_string.reference b/tests/queries/0_stateless/01277_random_fixed_string.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/01277_random_fixed_string.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/01277_random_fixed_string.sql b/tests/queries/0_stateless/01277_random_fixed_string.sql new file mode 100644 index 00000000000..3fb07114065 --- /dev/null +++ b/tests/queries/0_stateless/01277_random_fixed_string.sql @@ -0,0 +1 @@ +SELECT DISTINCT c > 30000 FROM (SELECT arrayJoin(arrayMap(x -> reinterpretAsUInt8(substring(randomFixedString(100), x + 1, 1)), range(100))) AS byte, count() AS c FROM numbers(100000) GROUP BY byte ORDER BY byte); From e102ad373afb6d634a23eb69353ed50c32eafabe Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 13 May 2020 03:32:09 +0300 Subject: [PATCH 144/738] Fix SIGSEGV in StringHashTable if such key is not exists --- src/Common/HashTable/StringHashTable.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Common/HashTable/StringHashTable.h b/src/Common/HashTable/StringHashTable.h index 101327ed809..e316f9b6520 100644 --- a/src/Common/HashTable/StringHashTable.h +++ b/src/Common/HashTable/StringHashTable.h @@ -333,7 +333,11 @@ public: template auto ALWAYS_INLINE operator()(Submap & map, const SubmapKey & key, size_t hash) { - return &map.find(key, hash)->getMapped(); + auto it = map.find(key, hash); + if (!it) + return decltype(&it->getMapped()){}; + else + return &it->getMapped(); } }; From 6e11c8a1a2bc4809a4bd31418389570c190cf7d9 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 13 May 2020 03:16:17 +0300 Subject: [PATCH 145/738] Add test for missing items in string hashtables via GROUP BY --- tests/queries/0_stateless/01279_dist_group_by.reference | 0 tests/queries/0_stateless/01279_dist_group_by.sql | 9 +++++++++ 2 files changed, 9 insertions(+) create mode 100644 tests/queries/0_stateless/01279_dist_group_by.reference create mode 100644 tests/queries/0_stateless/01279_dist_group_by.sql diff --git a/tests/queries/0_stateless/01279_dist_group_by.reference b/tests/queries/0_stateless/01279_dist_group_by.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01279_dist_group_by.sql b/tests/queries/0_stateless/01279_dist_group_by.sql new file mode 100644 index 00000000000..207b203f5d0 --- /dev/null +++ b/tests/queries/0_stateless/01279_dist_group_by.sql @@ -0,0 +1,9 @@ +drop table if exists data_01279; + +create table data_01279 (key String) Engine=TinyLog(); +insert into data_01279 select reinterpretAsString(number) from numbers(100000); + +set max_rows_to_group_by=10; +set group_by_overflow_mode='any'; +set group_by_two_level_threshold=100; +select * from data_01279 group by key format Null; From aaba17665b6fc23608545c088c715165c4dc19f7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 13 May 2020 03:50:21 +0300 Subject: [PATCH 146/738] Preparation for MSan UBSan stress tests --- docker/test/stress/Dockerfile | 2 +- docker/test/stress/stress | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docker/test/stress/Dockerfile b/docker/test/stress/Dockerfile index f022411a031..a5aa3bbf004 100644 --- a/docker/test/stress/Dockerfile +++ b/docker/test/stress/Dockerfile @@ -48,4 +48,4 @@ CMD dpkg -i package_folder/clickhouse-common-static_*.deb; \ && clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits" \ && clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits" \ && clickhouse-client --query "SHOW TABLES FROM test" \ - && ./stress --output-folder test_output + && ./stress --output-folder test_output --skip-func-tests "$SKIP_TESTS_OPTION" diff --git a/docker/test/stress/stress b/docker/test/stress/stress index 8fb351c1395..fc19ed80fa6 100755 --- a/docker/test/stress/stress +++ b/docker/test/stress/stress @@ -14,14 +14,14 @@ def run_perf_test(cmd, xmls_path, output_folder): p = Popen("{} --skip-tags=long --recursive --input-files {}".format(cmd, xmls_path), shell=True, stdout=f, stderr=f) return p -def run_func_test(cmd, output_prefix, num_processes): +def run_func_test(cmd, output_prefix, num_processes, skip_tests_option): output_paths = [os.path.join(output_prefix, "stress_test_run_{}.txt".format(i)) for i in range(num_processes)] f = open(output_paths[0], 'w') pipes = [Popen("{}".format(cmd), shell=True, stdout=f, stderr=f)] for output_path in output_paths[1:]: time.sleep(0.5) f = open(output_path, 'w') - p = Popen("{} --order=random".format(cmd), shell=True, stdout=f, stderr=f) + p = Popen("{} --order=random {}".format(cmd, skip_tests_option), shell=True, stdout=f, stderr=f) pipes.append(p) return pipes @@ -37,6 +37,7 @@ if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') parser = argparse.ArgumentParser(description="ClickHouse script for running stresstest") parser.add_argument("--test-cmd", default='clickhouse-test') + parser.add_argument("--skip-func-tests", default='') parser.add_argument("--client-cmd", default='clickhouse-client') parser.add_argument("--perf-test-cmd", default='clickhouse-performance-test') parser.add_argument("--perf-test-xml-path", default='/usr/share/clickhouse-test/performance/') @@ -49,7 +50,7 @@ if __name__ == "__main__": perf_process = None try: perf_process = run_perf_test(args.perf_test_cmd, args.perf_test_xml_path, args.output_folder) - func_pipes = run_func_test(args.test_cmd, args.output_folder, args.num_parallel) + func_pipes = run_func_test(args.test_cmd, args.output_folder, args.num_parallel, args.skip_func_tests) logging.info("Will wait functests to finish") while True: From 322e5bc4f5b5014926a75a18bd84fb0c6efa5166 Mon Sep 17 00:00:00 2001 From: Andrei Nekrashevich Date: Wed, 13 May 2020 04:22:57 +0300 Subject: [PATCH 147/738] fix issues --- tests/queries/0_stateless/01277_random_fixed_string.reference | 1 + tests/queries/0_stateless/01277_random_fixed_string.sql | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/tests/queries/0_stateless/01277_random_fixed_string.reference b/tests/queries/0_stateless/01277_random_fixed_string.reference index d00491fd7e5..68189063594 100644 --- a/tests/queries/0_stateless/01277_random_fixed_string.reference +++ b/tests/queries/0_stateless/01277_random_fixed_string.reference @@ -1 +1,2 @@ +FixedString(10) 1 diff --git a/tests/queries/0_stateless/01277_random_fixed_string.sql b/tests/queries/0_stateless/01277_random_fixed_string.sql index 3fb07114065..99782c1ac34 100644 --- a/tests/queries/0_stateless/01277_random_fixed_string.sql +++ b/tests/queries/0_stateless/01277_random_fixed_string.sql @@ -1 +1,5 @@ +SELECT randomFixedString('string'); -- { serverError 43 } +SELECT randomFixedString(0); -- { serverError 69 } +SELECT randomFixedString(rand() % 10); -- { serverError 44 } +SELECT toTypeName(randomFixedString(10)); SELECT DISTINCT c > 30000 FROM (SELECT arrayJoin(arrayMap(x -> reinterpretAsUInt8(substring(randomFixedString(100), x + 1, 1)), range(100))) AS byte, count() AS c FROM numbers(100000) GROUP BY byte ORDER BY byte); From 13c8ffc0c9817555dbacefc51cfb710f24f42774 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 13 May 2020 06:35:43 +0300 Subject: [PATCH 148/738] Fix exception message after server shutdown (#10841) * After shutdown cancel background logs pulling after task were deleted from pool * Fix comment * Cancel with lock * Remove whitspace --- src/Storages/StorageReplicatedMergeTree.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 2bd0a111f0d..6c9d6c88332 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -2980,7 +2980,6 @@ void StorageReplicatedMergeTree::shutdown() fetcher.blocker.cancelForever(); merger_mutator.merges_blocker.cancelForever(); parts_mover.moves_blocker.cancelForever(); - queue.pull_log_blocker.cancelForever(); restarting_thread.shutdown(); @@ -2992,6 +2991,11 @@ void StorageReplicatedMergeTree::shutdown() /// queue processes finished and after that reset queue_task_handle. auto lock = queue.lockQueue(); queue_task_handle.reset(); + + /// Cancel logs pulling after background task were cancelled. It's still + /// required because we can trigger pullLogsToQueue during manual OPTIMIZE, + /// MUTATE, etc. query. + queue.pull_log_blocker.cancelForever(); } if (move_parts_task_handle) From d52f9d9785ba8cff9ac89f013a7e9414f1f7cb7d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 13 May 2020 03:08:07 +0300 Subject: [PATCH 149/738] Fix bad test --- tests/queries/0_stateless/01056_create_table_as.sql | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/queries/0_stateless/01056_create_table_as.sql b/tests/queries/0_stateless/01056_create_table_as.sql index f95df9b7906..e8a7a67a956 100644 --- a/tests/queries/0_stateless/01056_create_table_as.sql +++ b/tests/queries/0_stateless/01056_create_table_as.sql @@ -1,4 +1,9 @@ DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +DROP TABLE IF EXISTS t3; +DROP TABLE IF EXISTS v; +DROP TABLE IF EXISTS lv; + CREATE TABLE t1 (key Int) Engine=Memory(); CREATE TABLE t2 AS t1; DROP TABLE t2; @@ -31,3 +36,6 @@ SOURCE(CLICKHOUSE( LIFETIME(MIN 0 MAX 0) LAYOUT(SPARSE_HASHED()); CREATE TABLE t3 AS dict; -- { serverError 80; } + +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t3; From 32ad64ae01bfbf45fcc1c6b5686c60a059ca5b39 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 13 May 2020 03:13:45 +0300 Subject: [PATCH 150/738] Fix LIVE VIEW tests --- .../0_stateless/00980_create_temporary_live_view.sql | 1 + .../0_stateless/01235_live_view_over_distributed.sql | 1 + .../01236_distributed_over_live_view_over_distributed.sql | 7 ++++--- ...w_over_distributed_with_subquery_select_table_alias.sql | 1 + 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/00980_create_temporary_live_view.sql b/tests/queries/0_stateless/00980_create_temporary_live_view.sql index 2dda5231e73..c19ebd1673d 100644 --- a/tests/queries/0_stateless/00980_create_temporary_live_view.sql +++ b/tests/queries/0_stateless/00980_create_temporary_live_view.sql @@ -14,3 +14,4 @@ SELECT sleep(2); SHOW TABLES WHERE database=currentDatabase() and name LIKE 'lv'; DROP TABLE mt; +DROP TABLE lv; diff --git a/tests/queries/0_stateless/01235_live_view_over_distributed.sql b/tests/queries/0_stateless/01235_live_view_over_distributed.sql index f3950c16002..dd9ff80f30e 100644 --- a/tests/queries/0_stateless/01235_live_view_over_distributed.sql +++ b/tests/queries/0_stateless/01235_live_view_over_distributed.sql @@ -17,3 +17,4 @@ SELECT * FROM lv; DROP TABLE visits; DROP TABLE visits_layer; +DROP TABLE lv; diff --git a/tests/queries/0_stateless/01236_distributed_over_live_view_over_distributed.sql b/tests/queries/0_stateless/01236_distributed_over_live_view_over_distributed.sql index 4408880ec5f..618bdc1c5d2 100644 --- a/tests/queries/0_stateless/01236_distributed_over_live_view_over_distributed.sql +++ b/tests/queries/0_stateless/01236_distributed_over_live_view_over_distributed.sql @@ -4,12 +4,12 @@ DROP TABLE IF EXISTS lv; DROP TABLE IF EXISTS visits; DROP TABLE IF EXISTS visits_layer; -CREATE TABLE visits(StartDate Date) ENGINE MergeTree ORDER BY(StartDate); -CREATE TABLE visits_layer(StartDate Date) ENGINE Distributed(test_cluster_two_shards_localhost, currentDatabase(), 'visits', rand()); +CREATE TABLE visits (StartDate Date) ENGINE MergeTree ORDER BY(StartDate); +CREATE TABLE visits_layer (StartDate Date) ENGINE Distributed(test_cluster_two_shards_localhost, currentDatabase(), 'visits', rand()); CREATE LIVE VIEW lv AS SELECT * FROM visits_layer ORDER BY StartDate; -CREATE TABLE visits_layer_lv(StartDate Date) ENGINE Distributed(test_cluster_two_shards_localhost, currentDatabase(), 'lv', rand()); +CREATE TABLE visits_layer_lv (StartDate Date) ENGINE Distributed(test_cluster_two_shards_localhost, currentDatabase(), 'lv', rand()); INSERT INTO visits_layer (StartDate) VALUES ('2020-01-01'); INSERT INTO visits_layer (StartDate) VALUES ('2020-01-02'); @@ -19,3 +19,4 @@ SELECT * FROM visits_layer_lv; DROP TABLE visits; DROP TABLE visits_layer; +DROP TABLE lv; diff --git a/tests/queries/0_stateless/01237_live_view_over_distributed_with_subquery_select_table_alias.sql b/tests/queries/0_stateless/01237_live_view_over_distributed_with_subquery_select_table_alias.sql index dc57e001122..a572074de3c 100644 --- a/tests/queries/0_stateless/01237_live_view_over_distributed_with_subquery_select_table_alias.sql +++ b/tests/queries/0_stateless/01237_live_view_over_distributed_with_subquery_select_table_alias.sql @@ -17,3 +17,4 @@ SELECT * FROM lv; DROP TABLE visits; DROP TABLE visits_layer; +DROP TABLE lv; From 3927647fbb0fde1e31ad7332d968694e8e72afa4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 13 May 2020 06:50:25 +0300 Subject: [PATCH 151/738] Fix test --- tests/queries/0_stateless/00980_create_temporary_live_view.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/queries/0_stateless/00980_create_temporary_live_view.sql b/tests/queries/0_stateless/00980_create_temporary_live_view.sql index c19ebd1673d..2dda5231e73 100644 --- a/tests/queries/0_stateless/00980_create_temporary_live_view.sql +++ b/tests/queries/0_stateless/00980_create_temporary_live_view.sql @@ -14,4 +14,3 @@ SELECT sleep(2); SHOW TABLES WHERE database=currentDatabase() and name LIKE 'lv'; DROP TABLE mt; -DROP TABLE lv; From 4e891091dfa6b460b71eec457ad81ad4c52aac0b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 13 May 2020 06:55:19 +0300 Subject: [PATCH 152/738] Allow expected number of failures in performance test --- docker/test/performance-comparison/report.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/test/performance-comparison/report.py b/docker/test/performance-comparison/report.py index 0c78f3642e1..a6203ec40e6 100755 --- a/docker/test/performance-comparison/report.py +++ b/docker/test/performance-comparison/report.py @@ -340,7 +340,8 @@ if args.report == 'main': message_array.append(str(faster_queries) + ' faster') if slower_queries: - status = 'failure' + if slower_queries > 3: + status = 'failure' message_array.append(str(slower_queries) + ' slower') if unstable_queries: From 681dc69ae7de827fd4c66a82689522a93b7a2620 Mon Sep 17 00:00:00 2001 From: AsiaKorushkina <43650329+AsiaKorushkina@users.noreply.github.com> Date: Wed, 13 May 2020 08:46:20 +0300 Subject: [PATCH 153/738] asiana21-DOCSUP-789 (#110) * docs(groupArrayInsertAt): added the function description * docs(groupArrayInsertAt): some changes * docs(groupArrayInsertAt): some changes 2 * docs(groupArrayInsertAt): some changes 3 * docs(groupArrayInsertAt): parameter section changed * Update docs/en/sql-reference/aggregate-functions/reference.md Co-authored-by: BayoNet * Update docs/en/sql-reference/aggregate-functions/reference.md Co-authored-by: BayoNet * Update docs/en/sql-reference/aggregate-functions/reference.md Co-authored-by: BayoNet * Update docs/en/sql-reference/aggregate-functions/reference.md Co-authored-by: BayoNet * Update docs/en/sql-reference/aggregate-functions/reference.md Co-authored-by: BayoNet * Update docs/en/sql-reference/aggregate-functions/reference.md Co-authored-by: BayoNet * Update docs/en/sql-reference/aggregate-functions/reference.md Co-authored-by: BayoNet * Update docs/en/sql-reference/aggregate-functions/reference.md Co-authored-by: BayoNet * Update docs/en/sql-reference/aggregate-functions/reference.md Co-authored-by: BayoNet * docs(groupArrayInsertAt): some examples are added * docs(groupArrayInsertAt): russian translation added * docs(groupArrayInsertAt): some change in russian translation Co-authored-by: asiana21 Co-authored-by: BayoNet --- .../aggregate-functions/reference.md | 119 ++++++++++++++++-- .../aggregate-functions/reference.md | 116 +++++++++++++++-- 2 files changed, 220 insertions(+), 15 deletions(-) diff --git a/docs/en/sql-reference/aggregate-functions/reference.md b/docs/en/sql-reference/aggregate-functions/reference.md index 9c8b5917e51..d083c87caca 100644 --- a/docs/en/sql-reference/aggregate-functions/reference.md +++ b/docs/en/sql-reference/aggregate-functions/reference.md @@ -729,19 +729,122 @@ For example, `groupArray (1) (x)` is equivalent to `[any (x)]`. In some cases, you can still rely on the order of execution. This applies to cases when `SELECT` comes from a subquery that uses `ORDER BY`. -## groupArrayInsertAt(value, position) {#grouparrayinsertatvalue-position} +## groupArrayInsertAt {#grouparrayinsertat} -Inserts a value into the array in the specified position. +Inserts a value into the array at the specified position. -!!! note "Note" - This function uses zero-based positions, contrary to the conventional one-based positions for SQL arrays. +**Syntax** -Accepts the value and position as input. If several values ​​are inserted into the same position, any of them might end up in the resulting array (the first one will be used in the case of single-threaded execution). If no value is inserted into a position, the position is assigned the default value. +```sql +groupArrayInsertAt(default_x, size)(x, pos); +``` -Optional parameters: +If in one query several values are inserted into the same position, the function behaves in the following ways: + +- If a query is executed in a single thread, the first one of the inserted values is used. +- If a query is executed in multiple threads, the resulting value is an undetermined one of the inserted values. + +**Parameters** + +- `x` — Value to be inserted. [Expression](../syntax.md#syntax-expressions) resulting in one of the [supported data types](../../sql-reference/data-types/index.md). +- `pos` — Position at which the specified element `x` is to be inserted. Index numbering in the array starts from zero. [UInt32](../../sql-reference/data-types/int-uint.md#uint-ranges). +- `default_x`— Default value for substituting in empty positions. Optional parameter. [Expression](../syntax.md#syntax-expressions) resulting in the data type configured for the `x` parameter. If `default_x` is not defined, the [default values](../../sql-reference/statements/create.md#create-default-values) are used. +- `size`— Length of the resulting array. Optional parameter. When using this parameter, the default value must be specified. [UInt32](../../sql-reference/data-types/int-uint.md#uint-ranges). + +**Returned value** + +- Array with inserted values. + +Type: [Array](../../sql-reference/data-types/array.md#data-type-array). + +**Example** + +Query: + +```sql +SELECT groupArrayInsertAt(toString(number), number * 2) FROM numbers(5); +``` + +Result: + +```text +┌─groupArrayInsertAt(toString(number), multiply(number, 2))─┐ +│ ['0','','1','','2','','3','','4'] │ +└───────────────────────────────────────────────────────────┘ +``` + +Query: + +```sql +SELECT groupArrayInsertAt('-')(toString(number), number * 2) FROM numbers(5); +``` + +Result: + +```text +┌─groupArrayInsertAt('-')(toString(number), multiply(number, 2))─┐ +│ ['0','-','1','-','2','-','3','-','4'] │ +└────────────────────────────────────────────────────────────────┘ +``` + +Query: + +```sql +SELECT groupArrayInsertAt('-', 5)(toString(number), number * 2) FROM numbers(5); +``` + +Result: + +```text +┌─groupArrayInsertAt('-', 5)(toString(number), multiply(number, 2))─┐ +│ ['0','-','1','-','2'] │ +└───────────────────────────────────────────────────────────────────┘ +``` + +Multi-threaded insertion of elements into one position. + +Query: + +```sql +SELECT groupArrayInsertAt(number, 0) FROM numbers_mt(10) SETTINGS max_block_size = 1; +``` + +Result: + +```text +┌─groupArrayInsertAt(number, 0)─┐ +│ [0] │ +└───────────────────────────────┘ +``` + +Query: + +```sql +SELECT groupArrayInsertAt(number, 0) FROM numbers_mt(10) SETTINGS max_block_size = 1; +``` + +Result: + +```text +┌─groupArrayInsertAt(number, 0)─┐ +│ [2] │ +└───────────────────────────────┘ +``` + +Query: + +```sql +SELECT groupArrayInsertAt(number, 0) FROM numbers_mt(10) SETTINGS max_block_size = 1; +``` + +Result: + +```text +┌─groupArrayInsertAt(number, 0)─┐ +│ [7] │ +└───────────────────────────────┘ +``` -- The default value for substituting in empty positions. -- The length of the resulting array. This allows you to receive arrays of the same size for all the aggregate keys. When using this parameter, the default value must be specified. ## groupArrayMovingSum {#agg_function-grouparraymovingsum} diff --git a/docs/ru/sql-reference/aggregate-functions/reference.md b/docs/ru/sql-reference/aggregate-functions/reference.md index 30ce6c87c6c..b4c89f54682 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference.md +++ b/docs/ru/sql-reference/aggregate-functions/reference.md @@ -721,19 +721,121 @@ uniqExact(x[, ...]) В некоторых случаях, вы всё же можете рассчитывать на порядок выполнения запроса. Это — случаи, когда `SELECT` идёт из подзапроса, в котором используется `ORDER BY`. -## groupArrayInsertAt(value, position) {#grouparrayinsertatvalue-position} +## groupArrayInsertAt(value, position) {#grouparrayinsertat} Вставляет в массив значение в заданную позицию. -!!! note "Примечание" - Эта функция использует нумерацию массивов с нуля, в отличие от принятой в SQL нумерации с единицы. +**Синтаксис** -Принимает на вход значение и позицию. Если на одну и ту же позицию вставляется несколько значений, в результирующем массиве может оказаться любое (первое в случае однопоточного выполнения). Если в позицию не вставляется ни одного значения, то позиции присваивается значение по умолчанию. +```sql +groupArrayInsertAt(default_x, size)(x, pos); +``` -Опциональные параметры: +Если в запросе на одну и ту же позицию вставляется несколько значений, то функция ведет себя следующим образом: -- Значение по умолчанию для подстановки на пустые позиции. -- Длина результирующего массива. Например, если вы хотите получать массивы одинакового размера для всех агрегатных ключей. При использовании этого параметра значение по умолчанию задавать обязательно. +- Если запрос выполняется в одном потоке, то используется первое из вставляемых значений. +- Если запрос выполняется в нескольких потоках, то в результирующем массиве может оказаться любое из вставляемых значений. + +**Параметры** + +- `x` — Значение, которое будет вставлено. [Выражение](../syntax.md#syntax-expressions), возвращающее значение одного из [поддерживаемых типов данных](../../sql-reference/data-types/index.md#data_types). +- `pos` — Позиция, в которую вставляется заданный элемент `x`. Нумерация индексов в массиве начинается с нуля. [UInt32](../../sql-reference/data-types/int-uint.md#uint8-uint16-uint32-uint64-int8-int16-int32-int64). +- `default_x`— Значение по умолчанию для подстановки на пустые позиции. Опциональный параметр. [Выражение](../syntax.md#syntax-expressions), возвращающее значение типа параметра `x`. Если `default_x` не определен, используются [значения по умолчанию](../../sql-reference/statements/create.md#create-default-values). +- `size`— Длина результирующего массива. Опциональный параметр. При использовании этого параметра должно быть указано значение по умолчанию. [UInt32](../../sql-reference/data-types/int-uint.md#uint-ranges). + +**Возвращаемое значение** + +- Массив со вставленными значениями. + +Тип: [Array](../../sql-reference/data-types/array.md#data-type-array). + +**Примеры** + +Запрос: + +```sql +SELECT groupArrayInsertAt(toString(number), number * 2) FROM numbers(5); +``` + +Результат: + +```text +┌─groupArrayInsertAt(toString(number), multiply(number, 2))─┐ +│ ['0','','1','','2','','3','','4'] │ +└───────────────────────────────────────────────────────────┘ +``` + +Запрос: + +```sql +SELECT groupArrayInsertAt('-')(toString(number), number * 2) FROM numbers(5); +``` + +Результат: + +```text +┌─groupArrayInsertAt('-')(toString(number), multiply(number, 2))─┐ +│ ['0','-','1','-','2','-','3','-','4'] │ +└────────────────────────────────────────────────────────────────┘ +``` + +Запрос: + +```sql +SELECT groupArrayInsertAt('-', 5)(toString(number), number * 2) FROM numbers(5); +``` + +Результат: + +```text +┌─groupArrayInsertAt('-', 5)(toString(number), multiply(number, 2))─┐ +│ ['0','-','1','-','2'] │ +└───────────────────────────────────────────────────────────────────┘ +``` + +Многопоточная вставка элементов в одну позицию. + +Запрос: + +```sql +SELECT groupArrayInsertAt(number, 0) FROM numbers_mt(10) SETTINGS max_block_size = 1; +``` + +Результат: + +```text +┌─groupArrayInsertAt(number, 0)─┐ +│ [0] │ +└───────────────────────────────┘ +``` + +Запрос: + +```sql +SELECT groupArrayInsertAt(number, 0) FROM numbers_mt(10) SETTINGS max_block_size = 1; +``` + +Результат: + +```text +┌─groupArrayInsertAt(number, 0)─┐ +│ [2] │ +└───────────────────────────────┘ +``` + +Запрос: + +```sql +SELECT groupArrayInsertAt(number, 0) FROM numbers_mt(10) SETTINGS max_block_size = 1; +``` + +Результат: + +```text +┌─groupArrayInsertAt(number, 0)─┐ +│ [7] │ +└───────────────────────────────┘ +``` ## groupArrayMovingSum {#agg_function-grouparraymovingsum} From 5d2bf3f6ac44ed8d5a024772dae6a5243d0d0bc2 Mon Sep 17 00:00:00 2001 From: Sergei Shtykov Date: Wed, 13 May 2020 08:58:51 +0300 Subject: [PATCH 154/738] CLICKHOUSEDOCS-504: groupArrayInsertAt --- .../aggregate-functions/reference.md | 33 +-------------- .../aggregate-functions/reference.md | 41 ++++--------------- 2 files changed, 9 insertions(+), 65 deletions(-) diff --git a/docs/en/sql-reference/aggregate-functions/reference.md b/docs/en/sql-reference/aggregate-functions/reference.md index d083c87caca..318ebda59c1 100644 --- a/docs/en/sql-reference/aggregate-functions/reference.md +++ b/docs/en/sql-reference/aggregate-functions/reference.md @@ -749,7 +749,7 @@ If in one query several values are inserted into the same position, the function - `x` — Value to be inserted. [Expression](../syntax.md#syntax-expressions) resulting in one of the [supported data types](../../sql-reference/data-types/index.md). - `pos` — Position at which the specified element `x` is to be inserted. Index numbering in the array starts from zero. [UInt32](../../sql-reference/data-types/int-uint.md#uint-ranges). - `default_x`— Default value for substituting in empty positions. Optional parameter. [Expression](../syntax.md#syntax-expressions) resulting in the data type configured for the `x` parameter. If `default_x` is not defined, the [default values](../../sql-reference/statements/create.md#create-default-values) are used. -- `size`— Length of the resulting array. Optional parameter. When using this parameter, the default value must be specified. [UInt32](../../sql-reference/data-types/int-uint.md#uint-ranges). +- `size`— Length of the resulting array. Optional parameter. When using this parameter, the default value `default_x` must be specified. [UInt32](../../sql-reference/data-types/int-uint.md#uint-ranges). **Returned value** @@ -809,35 +809,7 @@ Query: SELECT groupArrayInsertAt(number, 0) FROM numbers_mt(10) SETTINGS max_block_size = 1; ``` -Result: - -```text -┌─groupArrayInsertAt(number, 0)─┐ -│ [0] │ -└───────────────────────────────┘ -``` - -Query: - -```sql -SELECT groupArrayInsertAt(number, 0) FROM numbers_mt(10) SETTINGS max_block_size = 1; -``` - -Result: - -```text -┌─groupArrayInsertAt(number, 0)─┐ -│ [2] │ -└───────────────────────────────┘ -``` - -Query: - -```sql -SELECT groupArrayInsertAt(number, 0) FROM numbers_mt(10) SETTINGS max_block_size = 1; -``` - -Result: +As a result of this query you get random integer in the `[0,9]` range. For example: ```text ┌─groupArrayInsertAt(number, 0)─┐ @@ -845,7 +817,6 @@ Result: └───────────────────────────────┘ ``` - ## groupArrayMovingSum {#agg_function-grouparraymovingsum} Calculates the moving sum of input values. diff --git a/docs/ru/sql-reference/aggregate-functions/reference.md b/docs/ru/sql-reference/aggregate-functions/reference.md index b4c89f54682..faababd09ef 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference.md +++ b/docs/ru/sql-reference/aggregate-functions/reference.md @@ -721,9 +721,9 @@ uniqExact(x[, ...]) В некоторых случаях, вы всё же можете рассчитывать на порядок выполнения запроса. Это — случаи, когда `SELECT` идёт из подзапроса, в котором используется `ORDER BY`. -## groupArrayInsertAt(value, position) {#grouparrayinsertat} +## groupArrayInsertAt {#grouparrayinsertat} -Вставляет в массив значение в заданную позицию. +Вставляет значение в заданную позицию массива. **Синтаксис** @@ -731,7 +731,7 @@ uniqExact(x[, ...]) groupArrayInsertAt(default_x, size)(x, pos); ``` -Если в запросе на одну и ту же позицию вставляется несколько значений, то функция ведет себя следующим образом: +Если запрос вставляет вставляется несколько значений в одну и ту же позицию, то функция ведет себя следующим образом: - Если запрос выполняется в одном потоке, то используется первое из вставляемых значений. - Если запрос выполняется в нескольких потоках, то в результирующем массиве может оказаться любое из вставляемых значений. @@ -740,8 +740,8 @@ groupArrayInsertAt(default_x, size)(x, pos); - `x` — Значение, которое будет вставлено. [Выражение](../syntax.md#syntax-expressions), возвращающее значение одного из [поддерживаемых типов данных](../../sql-reference/data-types/index.md#data_types). - `pos` — Позиция, в которую вставляется заданный элемент `x`. Нумерация индексов в массиве начинается с нуля. [UInt32](../../sql-reference/data-types/int-uint.md#uint8-uint16-uint32-uint64-int8-int16-int32-int64). -- `default_x`— Значение по умолчанию для подстановки на пустые позиции. Опциональный параметр. [Выражение](../syntax.md#syntax-expressions), возвращающее значение типа параметра `x`. Если `default_x` не определен, используются [значения по умолчанию](../../sql-reference/statements/create.md#create-default-values). -- `size`— Длина результирующего массива. Опциональный параметр. При использовании этого параметра должно быть указано значение по умолчанию. [UInt32](../../sql-reference/data-types/int-uint.md#uint-ranges). +- `default_x` — Значение по умолчанию для подстановки на пустые позиции. Опциональный параметр. [Выражение](../syntax.md#syntax-expressions), возвращающее значение с типом параметра `x`. Если `default_x` не определен, используются [значения по умолчанию](../../sql-reference/statements/create.md#create-default-values). +- `size`— Длина результирующего массива. Опциональный параметр. При использовании этого параметра должно быть указано значение по умолчанию `default_x`. [UInt32](../../sql-reference/data-types/int-uint.md#uint-ranges). **Возвращаемое значение** @@ -801,35 +801,7 @@ SELECT groupArrayInsertAt('-', 5)(toString(number), number * 2) FROM numbers(5); SELECT groupArrayInsertAt(number, 0) FROM numbers_mt(10) SETTINGS max_block_size = 1; ``` -Результат: - -```text -┌─groupArrayInsertAt(number, 0)─┐ -│ [0] │ -└───────────────────────────────┘ -``` - -Запрос: - -```sql -SELECT groupArrayInsertAt(number, 0) FROM numbers_mt(10) SETTINGS max_block_size = 1; -``` - -Результат: - -```text -┌─groupArrayInsertAt(number, 0)─┐ -│ [2] │ -└───────────────────────────────┘ -``` - -Запрос: - -```sql -SELECT groupArrayInsertAt(number, 0) FROM numbers_mt(10) SETTINGS max_block_size = 1; -``` - -Результат: +В результат этого запроса мы получите случайное целое число в диапазоне `[0,9]`. Например: ```text ┌─groupArrayInsertAt(number, 0)─┐ @@ -837,6 +809,7 @@ SELECT groupArrayInsertAt(number, 0) FROM numbers_mt(10) SETTINGS max_block_size └───────────────────────────────┘ ``` + ## groupArrayMovingSum {#agg_function-grouparraymovingsum} Вычисляет скользящую сумму входных значений. From 1a4ce6b3c806f77e548dacf4207e06542231fa19 Mon Sep 17 00:00:00 2001 From: BayoNet Date: Wed, 13 May 2020 09:08:35 +0300 Subject: [PATCH 155/738] DOCS-628: Stable versions of statistical functions (#10848) * CLICKHOUSEDOCS-628: Mentioned Stable versions of staticsical functions. * CLICKHOUSEDOCS-628: Sync from EN to RU. * Update docs/ru/sql-reference/aggregate-functions/reference.md Co-authored-by: Ivan Blinkov * Update docs/en/sql-reference/aggregate-functions/reference.md Co-authored-by: Ivan Blinkov * CLICKHOUSEDOCS-628: Updated formatting. Co-authored-by: Sergei Shtykov Co-authored-by: Ivan Blinkov --- contrib/libcpuid | 1 - .../aggregate-functions/reference.md | 21 +++++++++++++++++ .../third-party/client-libraries.md | 1 + .../aggregate-functions/reference.md | 23 +++++++++++++++++++ 4 files changed, 45 insertions(+), 1 deletion(-) delete mode 160000 contrib/libcpuid diff --git a/contrib/libcpuid b/contrib/libcpuid deleted file mode 160000 index 8db3b8d2d32..00000000000 --- a/contrib/libcpuid +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8db3b8d2d32d22437f063ce692a1b9bb15e42d18 diff --git a/docs/en/sql-reference/aggregate-functions/reference.md b/docs/en/sql-reference/aggregate-functions/reference.md index 61a27f7f70e..6f5a58b765c 100644 --- a/docs/en/sql-reference/aggregate-functions/reference.md +++ b/docs/en/sql-reference/aggregate-functions/reference.md @@ -1469,20 +1469,32 @@ It represents an unbiased estimate of the variance of a random variable if passe Returns `Float64`. When `n <= 1`, returns `+∞`. +!!! note "Note" + This function uses numerically unstable algorithm. If you need [numerical stability](https://en.wikipedia.org/wiki/Numerical_stability) in calculations, use the `varSampStable` function. It works slower, but provides lower computational error. + ## varPop(x) {#varpopx} Calculates the amount `Σ((x - x̅)^2) / n`, where `n` is the sample size and `x̅`is the average value of `x`. In other words, dispersion for a set of values. Returns `Float64`. +!!! note "Note" + This function uses numerically unstable algorithm. If you need [numerical stability](https://en.wikipedia.org/wiki/Numerical_stability) in calculations, use the `varPopStable` function. It works slower, but provides lower computational error. + ## stddevSamp(x) {#stddevsampx} The result is equal to the square root of `varSamp(x)`. +!!! note "Note" + This function uses numerically unstable algorithm. If you need [numerical stability](https://en.wikipedia.org/wiki/Numerical_stability) in calculations, use the `stddevSampStable` function. It works slower, but provides lower computational error. + ## stddevPop(x) {#stddevpopx} The result is equal to the square root of `varPop(x)`. +!!! note "Note" + This function uses numerically unstable algorithm. If you need [numerical stability](https://en.wikipedia.org/wiki/Numerical_stability) in calculations, use the `stddevPopStable` function. It works slower, but provides lower computational error. + ## topK(N)(x) {#topknx} Returns an array of the approximately most frequent values in the specified column. The resulting array is sorted in descending order of approximate frequency of values (not by the values themselves). @@ -1567,14 +1579,23 @@ Calculates the value of `Σ((x - x̅)(y - y̅)) / (n - 1)`. Returns Float64. When `n <= 1`, returns +∞. +!!! note "Note" + This function uses numerically unstable algorithm. If you need [numerical stability](https://en.wikipedia.org/wiki/Numerical_stability) in calculations, use the `covarSampStable` function. It works slower, but provides lower computational error. + ## covarPop(x, y) {#covarpopx-y} Calculates the value of `Σ((x - x̅)(y - y̅)) / n`. +!!! note "Note" + This function uses numerically unstable algorithm. If you need [numerical stability](https://en.wikipedia.org/wiki/Numerical_stability) in calculations, use the `covarPopStable` function. It works slower but provides a lower computational error. + ## corr(x, y) {#corrx-y} Calculates the Pearson correlation coefficient: `Σ((x - x̅)(y - y̅)) / sqrt(Σ((x - x̅)^2) * Σ((y - y̅)^2))`. +!!! note "Note" + This function uses numerically unstable algorithm. If you need [numerical stability](https://en.wikipedia.org/wiki/Numerical_stability) in calculations, use the `corrStable` function. It works slower, but provides lower computational error. + ## categoricalInformationValue {#categoricalinformationvalue} Calculates the value of `(P(tag = 1) - P(tag = 0))(log(P(tag = 1)) - log(P(tag = 0)))` for each category. diff --git a/docs/ru/interfaces/third-party/client-libraries.md b/docs/ru/interfaces/third-party/client-libraries.md index 27fb3f67512..928c1393c26 100644 --- a/docs/ru/interfaces/third-party/client-libraries.md +++ b/docs/ru/interfaces/third-party/client-libraries.md @@ -45,6 +45,7 @@ - [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net) - Elixir - [clickhousex](https://github.com/appodeal/clickhousex/) + - [pillar](https://github.com/sofakingworld/pillar) - Nim - [nim-clickhouse](https://github.com/leonardoce/nim-clickhouse) diff --git a/docs/ru/sql-reference/aggregate-functions/reference.md b/docs/ru/sql-reference/aggregate-functions/reference.md index 76b44686a51..3494f5e09f4 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference.md +++ b/docs/ru/sql-reference/aggregate-functions/reference.md @@ -1458,20 +1458,33 @@ SELECT medianDeterministic(val, 1) FROM t Возвращает `Float64`. В случае, когда `n <= 1`, возвращается `+∞`. +!!! note "Примечание" + Функция использует вычислительно неустойчивый алгоритм. Если для ваших расчётов необходима [вычислительная устойчивость](https://ru.wikipedia.org/wiki/Вычислительная_устойчивость), используйте функцию `varSampStable`. Она работает медленнее, но обеспечиват меньшую вычислительную ошибку. + ## varPop(x) {#varpopx} Вычисляет величину `Σ((x - x̅)^2) / n`, где `n` - размер выборки, `x̅`- среднее значение `x`. То есть, дисперсию для множества значений. Возвращает `Float64`. +!!! note "Примечание" + Функция использует вычислительно неустойчивый алгоритм. Если для ваших расчётов необходима [вычислительная устойчивость](https://ru.wikipedia.org/wiki/Вычислительная_устойчивость), используйте функцию `varPopStable`. Она работает медленнее, но обеспечивает меньшую вычислительную ошибку. + ## stddevSamp(x) {#stddevsampx} Результат равен квадратному корню от `varSamp(x)`. +!!! note "Примечание" + Функция использует вычислительно неустойчивый алгоритм. Если для ваших расчётов необходима [вычислительная устойчивость](https://ru.wikipedia.org/wiki/Вычислительная_устойчивость), используйте функцию `stddevSampStable`. Она работает медленнее, но обеспечивает меньшую вычислительную ошибку. + ## stddevPop(x) {#stddevpopx} Результат равен квадратному корню от `varPop(x)`. +!!! note "Примечание" + Функция использует вычислительно неустойчивый алгоритм. Если для ваших расчётов необходима [вычислительная устойчивость](https://ru.wikipedia.org/wiki/Вычислительная_устойчивость), используйте функцию `stddevPopStable`. Она работает медленнее, но обеспечивает меньшую вычислительную ошибку. + + ## topK(N)(column) {#topkncolumn} Возвращает массив наиболее часто встречающихся значений в указанном столбце. Результирующий массив упорядочен по убыванию частоты значения (не по самим значениям). @@ -1551,14 +1564,24 @@ SELECT topKWeighted(10)(number, number) FROM numbers(1000) Возвращает Float64. В случае, когда `n <= 1`, возвращается +∞. +!!! note "Примечание" + Функция использует вычислительно неустойчивый алгоритм. Если для ваших расчётов необходима [вычислительная устойчивость](https://ru.wikipedia.org/wiki/Вычислительная_устойчивость), используйте функцию `covarSampStable`. Она работает медленнее, но обеспечивает меньшую вычислительную ошибку. + ## covarPop(x, y) {#covarpopx-y} Вычисляет величину `Σ((x - x̅)(y - y̅)) / n`. +!!! note "Примечание" + Функция использует вычислительно неустойчивый алгоритм. Если для ваших расчётов необходима [вычислительная устойчивость](https://ru.wikipedia.org/wiki/Вычислительная_устойчивость), используйте функцию `covarPopStable`. Она работает медленнее, но обеспечивает меньшую вычислительную ошибку. + + ## corr(x, y) {#corrx-y} Вычисляет коэффициент корреляции Пирсона: `Σ((x - x̅)(y - y̅)) / sqrt(Σ((x - x̅)^2) * Σ((y - y̅)^2))`. +!!! note "Примечание" + Функция использует вычислительно неустойчивый алгоритм. Если для ваших расчётов необходима [вычислительная устойчивость](https://ru.wikipedia.org/wiki/Вычислительная_устойчивость), используйте функцию `corrStable`. Она работает медленнее, но обеспечивает меньшую вычислительную ошибку. + ## simpleLinearRegression {#simplelinearregression} Выполняет простую (одномерную) линейную регрессию. From 032197b015d1b306ed535fbdd9ba5ea1ea527d10 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 12 May 2020 17:11:09 +0300 Subject: [PATCH 156/738] fix alter and optimize hangs when replica becomes inactive --- src/Common/ZooKeeper/ZooKeeper.cpp | 48 ++++++++++--------- src/Common/ZooKeeper/ZooKeeper.h | 3 +- src/Storages/StorageReplicatedMergeTree.cpp | 40 +++++++++++----- src/Storages/StorageReplicatedMergeTree.h | 2 +- ...9_parallel_alter_modify_zookeeper.referece | 12 ----- 5 files changed, 58 insertions(+), 47 deletions(-) delete mode 100644 tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.referece diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index 032d1e90ff5..3bd102cb10d 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -629,51 +629,55 @@ namespace { struct WaitForDisappearState { - int32_t code = 0; - int32_t event_type = 0; + std::atomic_int32_t code = 0; + std::atomic_int32_t event_type = 0; Poco::Event event; }; using WaitForDisappearStatePtr = std::shared_ptr; } -void ZooKeeper::waitForDisappear(const std::string & path) +bool ZooKeeper::waitForDisappear(const std::string & path, const WaitCondition & condition) { WaitForDisappearStatePtr state = std::make_shared(); - while (true) + auto callback = [state](const Coordination::ExistsResponse & response) { - auto callback = [state](const Coordination::ExistsResponse & response) + state->code = response.error; + if (state->code) + state->event.set(); + }; + + auto watch = [state](const Coordination::WatchResponse & response) + { + if (!state->code) { state->code = response.error; - if (state->code) - state->event.set(); - }; - - auto watch = [state](const Coordination::WatchResponse & response) - { if (!state->code) - { - state->code = response.error; - if (!state->code) - state->event_type = response.type; - state->event.set(); - } - }; + state->event_type = response.type; + state->event.set(); + } + }; + while (!condition || !condition()) + { /// NOTE: if the node doesn't exist, the watch will leak. - impl->exists(path, callback, watch); - state->event.wait(); + if (!condition) + state->event.wait(); + else if (!state->event.tryWait(1000)) + continue; if (state->code == Coordination::ZNONODE) - return; + return true; if (state->code) throw KeeperException(state->code, path); if (state->event_type == Coordination::DELETED) - return; + return true; } + + return false; } ZooKeeperPtr ZooKeeper::startNewSession() const diff --git a/src/Common/ZooKeeper/ZooKeeper.h b/src/Common/ZooKeeper/ZooKeeper.h index db166314a07..74209ce2f4f 100644 --- a/src/Common/ZooKeeper/ZooKeeper.h +++ b/src/Common/ZooKeeper/ZooKeeper.h @@ -185,8 +185,9 @@ public: /// Remove all children nodes (non recursive). void removeChildren(const std::string & path); + using WaitCondition = std::function; /// Wait for the node to disappear or return immediately if it doesn't exist. - void waitForDisappear(const std::string & path); + bool waitForDisappear(const std::string & path, const WaitCondition & condition = {}); /// Async interface (a small subset of operations is implemented). /// diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 2bd0a111f0d..76f8a560a18 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -361,8 +361,9 @@ void StorageReplicatedMergeTree::waitMutationToFinishOnReplicas( else if (mutation_pointer_value >= mutation_id) /// Maybe we already processed more fresh mutation break; /// (numbers like 0000000000 and 0000000001) - /// We wait without timeout. - wait_event->wait(); + /// Replica can become inactive, so wait with timeout and recheck it + if (wait_event->tryWait(1000)) + break; } if (partial_shutdown_called) @@ -3837,7 +3838,8 @@ Strings StorageReplicatedMergeTree::waitForAllReplicasToProcessLogEntry(const Re { if (wait_for_non_active || zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/is_active")) { - waitForReplicaToProcessLogEntry(replica, entry); + if (!waitForReplicaToProcessLogEntry(replica, entry, wait_for_non_active)) + unwaited.push_back(replica); } else { @@ -3850,7 +3852,7 @@ Strings StorageReplicatedMergeTree::waitForAllReplicasToProcessLogEntry(const Re } -void StorageReplicatedMergeTree::waitForReplicaToProcessLogEntry(const String & replica, const ReplicatedMergeTreeLogEntryData & entry) +bool StorageReplicatedMergeTree::waitForReplicaToProcessLogEntry(const String & replica, const ReplicatedMergeTreeLogEntryData & entry, bool wait_for_non_active) { String entry_str = entry.toString(); String log_node_name; @@ -3871,6 +3873,12 @@ void StorageReplicatedMergeTree::waitForReplicaToProcessLogEntry(const String & * To do this, check its node `log_pointer` - the maximum number of the element taken from `log` + 1. */ + const auto & check_replica_become_inactive = [this, &replica]() + { + return !getZooKeeper()->exists(zookeeper_path + "/replicas/" + replica + "/is_active"); + }; + constexpr auto event_wait_timeout_ms = 1000; + if (startsWith(entry.znode_name, "log-")) { /** In this case, just take the number from the node name `log-xxxxxxxxxx`. @@ -3882,7 +3890,7 @@ void StorageReplicatedMergeTree::waitForReplicaToProcessLogEntry(const String & LOG_DEBUG(log, "Waiting for " << replica << " to pull " << log_node_name << " to queue"); /// Let's wait until entry gets into the replica queue. - while (true) + while (wait_for_non_active || !check_replica_become_inactive()) { zkutil::EventPtr event = std::make_shared(); @@ -3890,7 +3898,10 @@ void StorageReplicatedMergeTree::waitForReplicaToProcessLogEntry(const String & if (!log_pointer.empty() && parse(log_pointer) > log_index) break; - event->wait(); + if (wait_for_non_active) + event->wait(); + else + event->tryWait(event_wait_timeout_ms); } } else if (startsWith(entry.znode_name, "queue-")) @@ -3927,7 +3938,7 @@ void StorageReplicatedMergeTree::waitForReplicaToProcessLogEntry(const String & LOG_DEBUG(log, "Waiting for " << replica << " to pull " << log_node_name << " to queue"); /// Let's wait until the entry gets into the replica queue. - while (true) + while (wait_for_non_active || !check_replica_become_inactive()) { zkutil::EventPtr event = std::make_shared(); @@ -3935,7 +3946,10 @@ void StorageReplicatedMergeTree::waitForReplicaToProcessLogEntry(const String & if (!log_pointer_new.empty() && parse(log_pointer_new) > log_index) break; - event->wait(); + if (wait_for_non_active) + event->wait(); + else + event->tryWait(event_wait_timeout_ms); } } } @@ -3970,13 +3984,17 @@ void StorageReplicatedMergeTree::waitForReplicaToProcessLogEntry(const String & if (queue_entry_to_wait_for.empty()) { LOG_DEBUG(log, "No corresponding node found. Assuming it has been already processed." " Found " << queue_entries.size() << " nodes."); - return; + return true; } LOG_DEBUG(log, "Waiting for " << queue_entry_to_wait_for << " to disappear from " << replica << " queue"); - /// Third - wait until the entry disappears from the replica queue. - getZooKeeper()->waitForDisappear(zookeeper_path + "/replicas/" + replica + "/queue/" + queue_entry_to_wait_for); + /// Third - wait until the entry disappears from the replica queue or replica become inactive. + String path_to_wait_on = zookeeper_path + "/replicas/" + replica + "/queue/" + queue_entry_to_wait_for; + if (wait_for_non_active) + return getZooKeeper()->waitForDisappear(path_to_wait_on); + + return getZooKeeper()->waitForDisappear(path_to_wait_on, check_replica_become_inactive); } diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 70fb48e9b35..f01e51bd769 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -486,7 +486,7 @@ private: /** Wait until the specified replica executes the specified action from the log. * NOTE: See comment about locks above. */ - void waitForReplicaToProcessLogEntry(const String & replica_name, const ReplicatedMergeTreeLogEntryData & entry); + bool waitForReplicaToProcessLogEntry(const String & replica_name, const ReplicatedMergeTreeLogEntryData & entry, bool wait_for_non_active = true); /// Choose leader replica, send requst to it and wait. void sendRequestToLeaderReplica(const ASTPtr & query, const Context & query_context); diff --git a/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.referece b/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.referece deleted file mode 100644 index be716831212..00000000000 --- a/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.referece +++ /dev/null @@ -1,12 +0,0 @@ -1725 -1725 -1725 -1725 -1725 -Starting alters -Finishing alters -1 -1 -1 -1 -1 From 0797b33be37605cdc54de75578ceb188beec2b38 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 13 May 2020 07:41:39 +0000 Subject: [PATCH 157/738] Bump mkdocs from 1.1 to 1.1.1 in /docs/tools Bumps [mkdocs](https://github.com/mkdocs/mkdocs) from 1.1 to 1.1.1. - [Release notes](https://github.com/mkdocs/mkdocs/releases) - [Commits](https://github.com/mkdocs/mkdocs/compare/1.1...1.1.1) Signed-off-by: dependabot-preview[bot] --- docs/tools/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tools/requirements.txt b/docs/tools/requirements.txt index 29d7aafe69b..9b79ff96456 100644 --- a/docs/tools/requirements.txt +++ b/docs/tools/requirements.txt @@ -16,7 +16,7 @@ jsmin==2.2.2 livereload==2.6.1 Markdown==3.2.1 MarkupSafe==1.1.1 -mkdocs==1.1 +mkdocs==1.1.1 mkdocs-htmlproofer-plugin==0.0.3 mkdocs-macros-plugin==0.4.6 nltk==3.5 From 91de241d7f061ff6a6eb81a7dea52b44bc9a76f2 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 13 May 2020 08:06:09 +0000 Subject: [PATCH 158/738] Bump mkdocs-macros-plugin from 0.4.6 to 0.4.7 in /docs/tools Bumps [mkdocs-macros-plugin](https://github.com/fralau/mkdocs_macros_plugin) from 0.4.6 to 0.4.7. - [Release notes](https://github.com/fralau/mkdocs_macros_plugin/releases) - [Commits](https://github.com/fralau/mkdocs_macros_plugin/commits) Signed-off-by: dependabot-preview[bot] --- docs/tools/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tools/requirements.txt b/docs/tools/requirements.txt index 9b79ff96456..b675765a2da 100644 --- a/docs/tools/requirements.txt +++ b/docs/tools/requirements.txt @@ -18,7 +18,7 @@ Markdown==3.2.1 MarkupSafe==1.1.1 mkdocs==1.1.1 mkdocs-htmlproofer-plugin==0.0.3 -mkdocs-macros-plugin==0.4.6 +mkdocs-macros-plugin==0.4.7 nltk==3.5 nose==1.3.7 protobuf==3.11.3 From cd8659a6d4d42476bd11dcee106ce8261ea47e86 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 13 May 2020 13:51:02 +0300 Subject: [PATCH 159/738] trigger CI From f2090525192165c0a35acb035ebd58cdd48ef434 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 13 May 2020 14:04:27 +0300 Subject: [PATCH 160/738] Better comments --- src/Storages/IStorage.h | 13 +++++++------ src/Storages/MergeTree/MergeTreeData.h | 5 ++++- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index c0aa57081d2..8c36fb64f72 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -452,22 +452,23 @@ public: /// Returns sampling expression AST for storage or nullptr if there is none. virtual ASTPtr getSamplingKeyAST() const { return nullptr; } - /// Returns additional columns that need to be read to calculate partition key. + /// Returns column names that need to be read to calculate partition key. virtual Names getColumnsRequiredForPartitionKey() const { return {}; } - /// Returns additional columns that need to be read to calculate sorting key. + /// Returns column names that need to be read to calculate sorting key. virtual Names getColumnsRequiredForSortingKey() const { return {}; } - /// Returns additional columns that need to be read to calculate primary key. + /// Returns column names that need to be read to calculate primary key. virtual Names getColumnsRequiredForPrimaryKey() const { return {}; } - /// Returns additional columns that need to be read to calculate sampling key. + /// Returns column names that need to be read to calculate sampling key. virtual Names getColumnsRequiredForSampling() const { return {}; } - /// Returns additional columns that need to be read for FINAL to work. + /// Returns column names that need to be read for FINAL to work. virtual Names getColumnsRequiredForFinal() const { return {}; } - /// Returns names of primary key + secondary sorting columns + /// Returns columns names in sorting key specified by user in ORDER BY + /// expression. For example: 'a', 'x * y', 'toStartOfMonth(date)', etc. virtual Names getSortingKeyColumns() const { return {}; } /// Returns columns, which will be needed to calculate dependencies diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index b9aa7ba6d4a..bd70a25169f 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -331,7 +331,9 @@ public: BrokenPartCallback broken_part_callback_ = [](const String &){}); + /// See comments about methods below in IStorage interface StorageInMemoryMetadata getInMemoryMetadata() const override; + ASTPtr getPartitionKeyAST() const override { return partition_by_ast; } ASTPtr getSortingKeyAST() const override { return sorting_key_expr_ast; } ASTPtr getPrimaryKeyAST() const override { return primary_key_expr_ast; } @@ -661,7 +663,8 @@ public: ExpressionActionsPtr primary_key_and_skip_indices_expr; ExpressionActionsPtr sorting_key_and_skip_indices_expr; - /// Names of columns for primary key + secondary sorting columns. + /// Names of sorting key columns in ORDER BY expression. For example: 'a', + /// 'x * y', 'toStartOfMonth(date)', etc. Names sorting_key_columns; ASTPtr sorting_key_expr_ast; ExpressionActionsPtr sorting_key_expr; From bb1dfe271e5f5ef555a4827f1d871bb13b8359e3 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Wed, 13 May 2020 14:15:59 +0300 Subject: [PATCH 161/738] Make temorary changes in docker to move files easier --- docker/test/stateful/Dockerfile | 4 ++++ docker/test/stateful_with_coverage/run.sh | 7 +++++++ docker/test/stateless/Dockerfile | 4 ++++ docker/test/stateless_with_coverage/run.sh | 7 +++++++ 4 files changed, 22 insertions(+) diff --git a/docker/test/stateful/Dockerfile b/docker/test/stateful/Dockerfile index 5fa2bb33af4..e51efadf653 100644 --- a/docker/test/stateful/Dockerfile +++ b/docker/test/stateful/Dockerfile @@ -16,6 +16,10 @@ CMD dpkg -i package_folder/clickhouse-common-static_*.deb; \ dpkg -i package_folder/clickhouse-server_*.deb; \ dpkg -i package_folder/clickhouse-client_*.deb; \ dpkg -i package_folder/clickhouse-test_*.deb; \ + mkdir -p /etc/clickhouse-server/dict_examples; \ + ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/dict_examples/; \ + ln -s /usr/share/clickhouse-test/config/strings_dictionary.xml /etc/clickhouse-server/dict_examples/; \ + ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/dict_examples/; \ ln -s /usr/share/clickhouse-test/config/zookeeper.xml /etc/clickhouse-server/config.d/; \ ln -s /usr/share/clickhouse-test/config/listen.xml /etc/clickhouse-server/config.d/; \ ln -s /usr/share/clickhouse-test/config/part_log.xml /etc/clickhouse-server/config.d/; \ diff --git a/docker/test/stateful_with_coverage/run.sh b/docker/test/stateful_with_coverage/run.sh index 2ada9753fd4..b946f5b187d 100755 --- a/docker/test/stateful_with_coverage/run.sh +++ b/docker/test/stateful_with_coverage/run.sh @@ -48,6 +48,13 @@ mkdir -p /var/lib/clickhouse mkdir -p /var/log/clickhouse-server chmod 777 -R /var/log/clickhouse-server/ +# Temorary way to keep CI green while moving dictionaries to separate directory +mkdir -p /etc/clickhouse-server/dict_examples +chmod 777 -R /etc/clickhouse-server/dict_examples +ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/dict_examples/; \ + ln -s /usr/share/clickhouse-test/config/strings_dictionary.xml /etc/clickhouse-server/dict_examples/; \ + ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/dict_examples/; + ln -s /usr/share/clickhouse-test/config/zookeeper.xml /etc/clickhouse-server/config.d/; \ ln -s /usr/share/clickhouse-test/config/listen.xml /etc/clickhouse-server/config.d/; \ ln -s /usr/share/clickhouse-test/config/part_log.xml /etc/clickhouse-server/config.d/; \ diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile index afede33f3aa..35a8a5a9d3d 100644 --- a/docker/test/stateless/Dockerfile +++ b/docker/test/stateless/Dockerfile @@ -54,6 +54,10 @@ CMD dpkg -i package_folder/clickhouse-common-static_*.deb; \ dpkg -i package_folder/clickhouse-server_*.deb; \ dpkg -i package_folder/clickhouse-client_*.deb; \ dpkg -i package_folder/clickhouse-test_*.deb; \ + mkdir -p /etc/clickhouse-server/dict_examples; \ + ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/dict_examples/; \ + ln -s /usr/share/clickhouse-test/config/strings_dictionary.xml /etc/clickhouse-server/dict_examples/; \ + ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/dict_examples/; \ ln -s /usr/share/clickhouse-test/config/zookeeper.xml /etc/clickhouse-server/config.d/; \ ln -s /usr/share/clickhouse-test/config/listen.xml /etc/clickhouse-server/config.d/; \ ln -s /usr/share/clickhouse-test/config/part_log.xml /etc/clickhouse-server/config.d/; \ diff --git a/docker/test/stateless_with_coverage/run.sh b/docker/test/stateless_with_coverage/run.sh index 238c7567694..185dc95c783 100755 --- a/docker/test/stateless_with_coverage/run.sh +++ b/docker/test/stateless_with_coverage/run.sh @@ -39,6 +39,13 @@ mkdir -p /var/log/clickhouse-server chmod 777 -R /var/lib/clickhouse chmod 777 -R /var/log/clickhouse-server/ +# Temorary way to keep CI green while moving dictionaries to separate directory +mkdir -p /etc/clickhouse-server/dict_examples +chmod 777 -R /etc/clickhouse-server/dict_examples +ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/dict_examples/; \ + ln -s /usr/share/clickhouse-test/config/strings_dictionary.xml /etc/clickhouse-server/dict_examples/; \ + ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/dict_examples/; + ln -s /usr/share/clickhouse-test/config/zookeeper.xml /etc/clickhouse-server/config.d/; \ ln -s /usr/share/clickhouse-test/config/listen.xml /etc/clickhouse-server/config.d/; \ ln -s /usr/share/clickhouse-test/config/part_log.xml /etc/clickhouse-server/config.d/; \ From 1db8f3367a7bf7b69ec3eb2897cba100530db317 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Wed, 13 May 2020 15:16:59 +0400 Subject: [PATCH 162/738] Link ldap[_r] with lber --- contrib/openldap-cmake/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/openldap-cmake/CMakeLists.txt b/contrib/openldap-cmake/CMakeLists.txt index b7bcd2468bd..4b9926ad439 100644 --- a/contrib/openldap-cmake/CMakeLists.txt +++ b/contrib/openldap-cmake/CMakeLists.txt @@ -133,6 +133,7 @@ add_library(ldap ${_libs_type} ) target_link_libraries(ldap + PRIVATE lber PRIVATE ${OPENSSL_LIBRARIES} ) @@ -169,6 +170,7 @@ add_library(ldap_r ${_libs_type} ) target_link_libraries(ldap_r + PRIVATE lber PRIVATE ${OPENSSL_LIBRARIES} ) From a89653effec4378b8ff7ca3e059b52f99c6bd95a Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 13 May 2020 14:53:35 +0300 Subject: [PATCH 163/738] Add libldap2-dev to build image --- docker/packager/deb/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/packager/deb/Dockerfile b/docker/packager/deb/Dockerfile index 172dcad4e3c..aac4d68a6df 100644 --- a/docker/packager/deb/Dockerfile +++ b/docker/packager/deb/Dockerfile @@ -67,7 +67,9 @@ RUN apt-get --allow-unauthenticated update -y \ gdb \ pigz \ moreutils \ - libcctz-dev + libcctz-dev \ + libldap2-dev + # Special dpkg-deb (https://github.com/ClickHouse-Extras/dpkg) version which is able # to compress files using pigz (https://zlib.net/pigz/) instead of gzip. From ee9aaa499a17a8b39f3349d94d5ef8b566a7f597 Mon Sep 17 00:00:00 2001 From: Vladimir Bunchuk Date: Wed, 13 May 2020 14:59:02 +0300 Subject: [PATCH 164/738] Update access-rights.md typo fix --- docs/en/operations/access-rights.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/access-rights.md b/docs/en/operations/access-rights.md index 56f0715d654..aac2befab4c 100644 --- a/docs/en/operations/access-rights.md +++ b/docs/en/operations/access-rights.md @@ -141,7 +141,7 @@ Management queries: - Enable SQL-driven access control and account management for at least one user account. - By default SQL-driven access control and account management is turned of for all users. You need to configure at least one user in the `users.xml` configuration file and assign 1 to the [access_management](settings/settings-users.md#access_management-user-setting) setting. + By default, SQL-driven access control and account management is turned off for all users. You need to configure at least one user in the `users.xml` configuration file and assign 1 to the [access_management](settings/settings-users.md#access_management-user-setting) setting. [Original article](https://clickhouse.tech/docs/en/operations/access_rights/) From 1c4564d9ed68b15d766459a913aebaa84f495952 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 13 May 2020 15:56:32 +0300 Subject: [PATCH 165/738] Simplify rename tests --- tests/integration/test_rename_column/test.py | 93 ++++++++++++-------- 1 file changed, 56 insertions(+), 37 deletions(-) diff --git a/tests/integration/test_rename_column/test.py b/tests/integration/test_rename_column/test.py index 6ef14c3c4e8..db6a20c5592 100644 --- a/tests/integration/test_rename_column/test.py +++ b/tests/integration/test_rename_column/test.py @@ -23,6 +23,7 @@ node3 = cluster.add_instance('node3', macros={"shard": 1, "replica": 1}, **node_ node4 = cluster.add_instance('node4', macros={"shard": 1, "replica": 2}, **node_options) nodes = [node1, node2, node3, node4] + @pytest.fixture(scope="module") def started_cluster(): try: @@ -33,13 +34,15 @@ def started_cluster(): finally: cluster.shutdown() + def drop_table(nodes, table_name): for node in nodes: node.query("DROP TABLE IF EXISTS {} NO DELAY".format(table_name)) time.sleep(1) + def create_table(nodes, table_name, with_storage_policy=False, with_time_column=False, - with_ttl_move=False, with_ttl_delete=False): + with_ttl_move=False, with_ttl_delete=False): extra_columns = "" settings = [] @@ -77,6 +80,7 @@ def create_table(nodes, table_name, with_storage_policy=False, with_time_column= """ node.query(sql.format(table_name=table_name, replica=node.name, extra_columns=extra_columns)) + def create_distributed_table(node, table_name): sql = """ CREATE TABLE %(table_name)s_replicated ON CLUSTER test_cluster @@ -94,11 +98,13 @@ def create_distributed_table(node, table_name): """ % dict(table_name=table_name) node.query(sql) + def drop_distributed_table(node, table_name): node.query("DROP TABLE IF EXISTS {} ON CLUSTER test_cluster".format(table_name)) node.query("DROP TABLE IF EXISTS {}_replicated ON CLUSTER test_cluster".format(table_name)) time.sleep(1) + def insert(node, table_name, chunk=1000, col_names=None, iterations=1, ignore_exception=False, slow=False, with_many_parts=False, offset=0, with_time_column=False): if col_names is None: @@ -125,6 +131,7 @@ def insert(node, table_name, chunk=1000, col_names=None, iterations=1, ignore_ex if not ignore_exception: raise + def select(node, table_name, col_name="num", expected_result=None, iterations=1, ignore_exception=False, slow=False, poll=None): for i in range(iterations): start_time = time.time() @@ -143,6 +150,7 @@ def select(node, table_name, col_name="num", expected_result=None, iterations=1, raise break + def rename_column(node, table_name, name, new_name, iterations=1, ignore_exception=False): for i in range(iterations): try: @@ -153,6 +161,7 @@ def rename_column(node, table_name, name, new_name, iterations=1, ignore_excepti if not ignore_exception: raise + def rename_column_on_cluster(node, table_name, name, new_name, iterations=1, ignore_exception=False): for i in range(iterations): try: @@ -163,6 +172,7 @@ def rename_column_on_cluster(node, table_name, name, new_name, iterations=1, ign if not ignore_exception: raise + def alter_move(node, table_name, iterations=1, ignore_exception=False): for i in range(iterations): move_part = random.randint(0, 99) @@ -174,6 +184,7 @@ def alter_move(node, table_name, iterations=1, ignore_exception=False): if not ignore_exception: raise + def test_rename_parallel_same_node(started_cluster): table_name = "test_rename_parallel_same_node" drop_table(nodes, table_name) @@ -199,6 +210,7 @@ def test_rename_parallel_same_node(started_cluster): finally: drop_table(nodes, table_name) + def test_rename_parallel(started_cluster): table_name = "test_rename_parallel" drop_table(nodes, table_name) @@ -224,6 +236,7 @@ def test_rename_parallel(started_cluster): finally: drop_table(nodes, table_name) + def test_rename_with_parallel_select(started_cluster): table_name = "test_rename_with_parallel_select" drop_table(nodes, table_name) @@ -256,6 +269,7 @@ def test_rename_with_parallel_select(started_cluster): finally: drop_table(nodes, table_name) + def test_rename_with_parallel_insert(started_cluster): table_name = "test_rename_with_parallel_insert" drop_table(nodes, table_name) @@ -284,6 +298,7 @@ def test_rename_with_parallel_insert(started_cluster): finally: drop_table(nodes, table_name) + def test_rename_with_parallel_merges(started_cluster): table_name = "test_rename_with_parallel_merges" drop_table(nodes, table_name) @@ -299,12 +314,12 @@ def test_rename_with_parallel_merges(started_cluster): p = Pool(15) tasks = [] for i in range(1): - tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2", 25, True))) - tasks.append(p.apply_async(rename_column, (node2, table_name, "foo2", "foo3", 25, True))) - tasks.append(p.apply_async(rename_column, (node3, table_name, "foo3", "num2", 25, True))) - tasks.append(p.apply_async(merge_parts, (node1, table_name, 15))) - tasks.append(p.apply_async(merge_parts, (node2, table_name, 15))) - tasks.append(p.apply_async(merge_parts, (node3, table_name, 15))) + tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2", 5, True))) + tasks.append(p.apply_async(rename_column, (node2, table_name, "foo2", "foo3", 5, True))) + tasks.append(p.apply_async(rename_column, (node3, table_name, "foo3", "num2", 5, True))) + tasks.append(p.apply_async(merge_parts, (node1, table_name, 5))) + tasks.append(p.apply_async(merge_parts, (node2, table_name, 5))) + tasks.append(p.apply_async(merge_parts, (node3, table_name, 5))) for task in tasks: task.get(timeout=240) @@ -320,6 +335,7 @@ def test_rename_with_parallel_merges(started_cluster): finally: drop_table(nodes, table_name) + def test_rename_with_parallel_slow_insert(started_cluster): table_name = "test_rename_with_parallel_slow_insert" drop_table(nodes, table_name) @@ -349,6 +365,7 @@ def test_rename_with_parallel_slow_insert(started_cluster): finally: drop_table(nodes, table_name) + def test_rename_with_parallel_slow_select(started_cluster): table_name = "test_rename_with_parallel_slow_select" drop_table(nodes, table_name) @@ -360,7 +377,7 @@ def test_rename_with_parallel_slow_select(started_cluster): tasks = [] tasks.append(p.apply_async(select, (node1, table_name, "num2", "999\n", 1, True, True))) - time.sleep(0.25) + time.sleep(0.5) tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2"))) for task in tasks: @@ -378,6 +395,7 @@ def test_rename_with_parallel_slow_select(started_cluster): finally: drop_table(nodes, table_name) + def test_rename_with_parallel_moves(started_cluster): table_name = "test_rename_with_parallel_moves" drop_table(nodes, table_name) @@ -388,12 +406,12 @@ def test_rename_with_parallel_moves(started_cluster): p = Pool(15) tasks = [] - tasks.append(p.apply_async(alter_move, (node1, table_name, 100, True))) - tasks.append(p.apply_async(alter_move, (node2, table_name, 100, True))) - tasks.append(p.apply_async(alter_move, (node3, table_name, 100, True))) - tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2", 100, True))) - tasks.append(p.apply_async(rename_column, (node2, table_name, "foo2", "foo3", 100, True))) - tasks.append(p.apply_async(rename_column, (node3, table_name, "num3", "num2", 100, True))) + tasks.append(p.apply_async(alter_move, (node1, table_name, 20, True))) + tasks.append(p.apply_async(alter_move, (node2, table_name, 20, True))) + tasks.append(p.apply_async(alter_move, (node3, table_name, 20, True))) + tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2", 20, True))) + tasks.append(p.apply_async(rename_column, (node2, table_name, "foo2", "foo3", 20, True))) + tasks.append(p.apply_async(rename_column, (node3, table_name, "num3", "num2", 20, True))) for task in tasks: task.get(timeout=240) @@ -409,6 +427,7 @@ def test_rename_with_parallel_moves(started_cluster): finally: drop_table(nodes, table_name) + def test_rename_with_parallel_ttl_move(started_cluster): table_name = 'test_rename_with_parallel_ttl_move' try: @@ -422,11 +441,10 @@ def test_rename_with_parallel_ttl_move(started_cluster): tasks.append(p.apply_async(insert, (node1, table_name, 10000, ["num", "num2"], 1, False, False, True, 0, True))) time.sleep(5) rename_column(node1, table_name, "time", "time2", 1, False) - rename_column(node1, table_name, "time2", "time", 1, False) time.sleep(4) - tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2", 500, True))) - tasks.append(p.apply_async(rename_column, (node2, table_name, "foo2", "foo3", 500, True))) - tasks.append(p.apply_async(rename_column, (node3, table_name, "num3", "num2", 500, True))) + tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2", 20, True))) + tasks.append(p.apply_async(rename_column, (node2, table_name, "foo2", "foo3", 20, True))) + tasks.append(p.apply_async(rename_column, (node3, table_name, "num3", "num2", 20, True))) for task in tasks: task.get(timeout=240) @@ -443,6 +461,7 @@ def test_rename_with_parallel_ttl_move(started_cluster): finally: drop_table(nodes, table_name) + def test_rename_with_parallel_ttl_delete(started_cluster): table_name = 'test_rename_with_parallel_ttl_delete' try: @@ -459,12 +478,12 @@ def test_rename_with_parallel_ttl_delete(started_cluster): tasks.append(p.apply_async(insert, (node1, table_name, 10000, ["num", "num2"], 1, False, False, True, 0, True))) time.sleep(15) - tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2", 500, True))) - tasks.append(p.apply_async(rename_column, (node2, table_name, "foo2", "foo3", 500, True))) - tasks.append(p.apply_async(rename_column, (node3, table_name, "num3", "num2", 500, True))) - tasks.append(p.apply_async(merge_parts, (node1, table_name, 500))) - tasks.append(p.apply_async(merge_parts, (node2, table_name, 500))) - tasks.append(p.apply_async(merge_parts, (node3, table_name, 500))) + tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2", 20, True))) + tasks.append(p.apply_async(rename_column, (node2, table_name, "foo2", "foo3", 20, True))) + tasks.append(p.apply_async(rename_column, (node3, table_name, "num3", "num2", 20, True))) + tasks.append(p.apply_async(merge_parts, (node1, table_name, 20))) + tasks.append(p.apply_async(merge_parts, (node2, table_name, 20))) + tasks.append(p.apply_async(merge_parts, (node3, table_name, 20))) for task in tasks: task.get(timeout=240) @@ -473,10 +492,11 @@ def test_rename_with_parallel_ttl_delete(started_cluster): rename_column(node1, table_name, "foo2", "num2", 1, True) rename_column(node1, table_name, "foo3", "num2", 1, True) - assert int(node1.query("SELECT count() FROM {}".format(table_name)).strip()) < 9000 + assert int(node1.query("SELECT count() FROM {}".format(table_name)).strip()) < 10000 finally: drop_table(nodes, table_name) + def test_rename_distributed(started_cluster): table_name = 'test_rename_distributed' try: @@ -491,8 +511,8 @@ def test_rename_distributed(started_cluster): select(node1, table_name, "foo2", '1998\n', poll=30) finally: drop_distributed_table(node1, table_name) - -@pytest.mark.skip(reason="temporary disabled") + + def test_rename_distributed_parallel_insert_and_select(started_cluster): table_name = 'test_rename_distributed_parallel_insert_and_select' try: @@ -502,15 +522,15 @@ def test_rename_distributed_parallel_insert_and_select(started_cluster): p = Pool(15) tasks = [] for i in range(1): - tasks.append(p.apply_async(rename_column_on_cluster, (node1, table_name, 'num2', 'foo2', 5, True))) - tasks.append(p.apply_async(rename_column_on_cluster, (node1, '%s_replicated' % table_name, 'num2', 'foo2', 5, True))) - tasks.append(p.apply_async(rename_column_on_cluster, (node1, table_name, 'foo2', 'foo3', 5, True))) - tasks.append(p.apply_async(rename_column_on_cluster, (node1, '%s_replicated' % table_name, 'foo2', 'foo3', 5, True))) - tasks.append(p.apply_async(rename_column_on_cluster, (node1, table_name, 'foo3', 'num2', 5, True))) - tasks.append(p.apply_async(rename_column_on_cluster, (node1, '%s_replicated' % table_name, 'foo3', 'num2', 5, True))) - tasks.append(p.apply_async(insert, (node1, table_name, 100, ["num", "foo3"], 5, True))) - tasks.append(p.apply_async(insert, (node2, table_name, 100, ["num", "num2"], 5, True))) - tasks.append(p.apply_async(insert, (node3, table_name, 100, ["num", "foo2"], 5, True))) + tasks.append(p.apply_async(rename_column_on_cluster, (node1, table_name, 'num2', 'foo2', 3, True))) + tasks.append(p.apply_async(rename_column_on_cluster, (node1, '%s_replicated' % table_name, 'num2', 'foo2', 3, True))) + tasks.append(p.apply_async(rename_column_on_cluster, (node1, table_name, 'foo2', 'foo3', 3, True))) + tasks.append(p.apply_async(rename_column_on_cluster, (node1, '%s_replicated' % table_name, 'foo2', 'foo3', 3, True))) + tasks.append(p.apply_async(rename_column_on_cluster, (node1, table_name, 'foo3', 'num2', 3, True))) + tasks.append(p.apply_async(rename_column_on_cluster, (node1, '%s_replicated' % table_name, 'foo3', 'num2', 3, True))) + tasks.append(p.apply_async(insert, (node1, table_name, 10, ["num", "foo3"], 5, True))) + tasks.append(p.apply_async(insert, (node2, table_name, 10, ["num", "num2"], 5, True))) + tasks.append(p.apply_async(insert, (node3, table_name, 10, ["num", "foo2"], 5, True))) tasks.append(p.apply_async(select, (node1, table_name, "foo2", None, 5, True))) tasks.append(p.apply_async(select, (node2, table_name, "foo3", None, 5, True))) tasks.append(p.apply_async(select, (node3, table_name, "num2", None, 5, True))) @@ -529,4 +549,3 @@ def test_rename_distributed_parallel_insert_and_select(started_cluster): select(node4, table_name, "num2") finally: drop_distributed_table(node1, table_name) - From 22c7a843a620143a144405024ce0f3e9436685e5 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 13 May 2020 16:03:33 +0300 Subject: [PATCH 166/738] Remove already added tests --- ...er_rename_column_constraint_expr.reference | 0 ...77_alter_rename_column_constraint_expr.sql | 43 ------------------- 2 files changed, 43 deletions(-) delete mode 100644 tests/queries/0_stateless/01277_alter_rename_column_constraint_expr.reference delete mode 100644 tests/queries/0_stateless/01277_alter_rename_column_constraint_expr.sql diff --git a/tests/queries/0_stateless/01277_alter_rename_column_constraint_expr.reference b/tests/queries/0_stateless/01277_alter_rename_column_constraint_expr.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/queries/0_stateless/01277_alter_rename_column_constraint_expr.sql b/tests/queries/0_stateless/01277_alter_rename_column_constraint_expr.sql deleted file mode 100644 index 72fbb045601..00000000000 --- a/tests/queries/0_stateless/01277_alter_rename_column_constraint_expr.sql +++ /dev/null @@ -1,43 +0,0 @@ -DROP TABLE IF EXISTS table_for_rename; - -CREATE TABLE table_for_rename -( - date Date, - key UInt64, - value1 String, - value2 String, - value3 String, - CONSTRAINT cs_value1 CHECK toInt64(value1) < toInt64(value2), - CONSTRAINT cs_value2 CHECK toInt64(value2) < toInt64(value3) -) -ENGINE = MergeTree() -PARTITION BY date -ORDER BY key; - -INSERT INTO table_for_rename SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1), toString(number + 2) from numbers(9); -INSERT INTO table_for_rename SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1), toString(number) from numbers(9); ; --{serverError 469} - -SELECT * FROM table_for_rename ORDER BY key; - -ALTER TABLE table_for_rename RENAME COLUMN value1 TO value4; -ALTER TABLE table_for_rename RENAME COLUMN value2 TO value5; -SHOW CREATE TABLE table_for_rename; -SELECT * FROM table_for_rename ORDER BY key; - -SELECT '-- insert after rename --'; -INSERT INTO table_for_rename SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1), toString(number + 2) from numbers(10, 10); -INSERT INTO table_for_rename SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1), toString(number) from numbers(10, 10); ; --{serverError 469} -SELECT * FROM table_for_rename ORDER BY key; - -SELECT '-- rename columns back --'; -ALTER TABLE table_for_rename RENAME COLUMN value4 TO value1; -ALTER TABLE table_for_rename RENAME COLUMN value5 TO value2; -SHOW CREATE TABLE table_for_rename; -SELECT * FROM table_for_rename ORDER BY key; - -SELECT '-- insert after rename column --'; -INSERT INTO table_for_rename SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1), toString(number + 2) from numbers(20,10); -INSERT INTO table_for_rename SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number), toString(number + 2) from numbers(20, 10); ; --{serverError 469} -SELECT * FROM table_for_rename ORDER BY key; - -DROP TABLE IF EXISTS table_for_rename; From 612385d703a48f4b499535e396df7a2e8435b7ce Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 13 May 2020 16:32:36 +0300 Subject: [PATCH 167/738] Revert "DOCS-628: Stable versions of statistical functions (#10848)" This reverts commit 1a4ce6b3c806f77e548dacf4207e06542231fa19. --- contrib/libcpuid | 1 + .../aggregate-functions/reference.md | 21 ----------------- .../third-party/client-libraries.md | 1 - .../aggregate-functions/reference.md | 23 ------------------- 4 files changed, 1 insertion(+), 45 deletions(-) create mode 160000 contrib/libcpuid diff --git a/contrib/libcpuid b/contrib/libcpuid new file mode 160000 index 00000000000..8db3b8d2d32 --- /dev/null +++ b/contrib/libcpuid @@ -0,0 +1 @@ +Subproject commit 8db3b8d2d32d22437f063ce692a1b9bb15e42d18 diff --git a/docs/en/sql-reference/aggregate-functions/reference.md b/docs/en/sql-reference/aggregate-functions/reference.md index 57cca001603..3bfaae8e696 100644 --- a/docs/en/sql-reference/aggregate-functions/reference.md +++ b/docs/en/sql-reference/aggregate-functions/reference.md @@ -1543,32 +1543,20 @@ It represents an unbiased estimate of the variance of a random variable if passe Returns `Float64`. When `n <= 1`, returns `+∞`. -!!! note "Note" - This function uses numerically unstable algorithm. If you need [numerical stability](https://en.wikipedia.org/wiki/Numerical_stability) in calculations, use the `varSampStable` function. It works slower, but provides lower computational error. - ## varPop(x) {#varpopx} Calculates the amount `Σ((x - x̅)^2) / n`, where `n` is the sample size and `x̅`is the average value of `x`. In other words, dispersion for a set of values. Returns `Float64`. -!!! note "Note" - This function uses numerically unstable algorithm. If you need [numerical stability](https://en.wikipedia.org/wiki/Numerical_stability) in calculations, use the `varPopStable` function. It works slower, but provides lower computational error. - ## stddevSamp(x) {#stddevsampx} The result is equal to the square root of `varSamp(x)`. -!!! note "Note" - This function uses numerically unstable algorithm. If you need [numerical stability](https://en.wikipedia.org/wiki/Numerical_stability) in calculations, use the `stddevSampStable` function. It works slower, but provides lower computational error. - ## stddevPop(x) {#stddevpopx} The result is equal to the square root of `varPop(x)`. -!!! note "Note" - This function uses numerically unstable algorithm. If you need [numerical stability](https://en.wikipedia.org/wiki/Numerical_stability) in calculations, use the `stddevPopStable` function. It works slower, but provides lower computational error. - ## topK(N)(x) {#topknx} Returns an array of the approximately most frequent values in the specified column. The resulting array is sorted in descending order of approximate frequency of values (not by the values themselves). @@ -1653,23 +1641,14 @@ Calculates the value of `Σ((x - x̅)(y - y̅)) / (n - 1)`. Returns Float64. When `n <= 1`, returns +∞. -!!! note "Note" - This function uses numerically unstable algorithm. If you need [numerical stability](https://en.wikipedia.org/wiki/Numerical_stability) in calculations, use the `covarSampStable` function. It works slower, but provides lower computational error. - ## covarPop(x, y) {#covarpopx-y} Calculates the value of `Σ((x - x̅)(y - y̅)) / n`. -!!! note "Note" - This function uses numerically unstable algorithm. If you need [numerical stability](https://en.wikipedia.org/wiki/Numerical_stability) in calculations, use the `covarPopStable` function. It works slower but provides a lower computational error. - ## corr(x, y) {#corrx-y} Calculates the Pearson correlation coefficient: `Σ((x - x̅)(y - y̅)) / sqrt(Σ((x - x̅)^2) * Σ((y - y̅)^2))`. -!!! note "Note" - This function uses numerically unstable algorithm. If you need [numerical stability](https://en.wikipedia.org/wiki/Numerical_stability) in calculations, use the `corrStable` function. It works slower, but provides lower computational error. - ## categoricalInformationValue {#categoricalinformationvalue} Calculates the value of `(P(tag = 1) - P(tag = 0))(log(P(tag = 1)) - log(P(tag = 0)))` for each category. diff --git a/docs/ru/interfaces/third-party/client-libraries.md b/docs/ru/interfaces/third-party/client-libraries.md index 928c1393c26..27fb3f67512 100644 --- a/docs/ru/interfaces/third-party/client-libraries.md +++ b/docs/ru/interfaces/third-party/client-libraries.md @@ -45,7 +45,6 @@ - [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net) - Elixir - [clickhousex](https://github.com/appodeal/clickhousex/) - - [pillar](https://github.com/sofakingworld/pillar) - Nim - [nim-clickhouse](https://github.com/leonardoce/nim-clickhouse) diff --git a/docs/ru/sql-reference/aggregate-functions/reference.md b/docs/ru/sql-reference/aggregate-functions/reference.md index dd23b0db77e..8de1018026c 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference.md +++ b/docs/ru/sql-reference/aggregate-functions/reference.md @@ -1533,33 +1533,20 @@ SELECT medianDeterministic(val, 1) FROM t Возвращает `Float64`. В случае, когда `n <= 1`, возвращается `+∞`. -!!! note "Примечание" - Функция использует вычислительно неустойчивый алгоритм. Если для ваших расчётов необходима [вычислительная устойчивость](https://ru.wikipedia.org/wiki/Вычислительная_устойчивость), используйте функцию `varSampStable`. Она работает медленнее, но обеспечиват меньшую вычислительную ошибку. - ## varPop(x) {#varpopx} Вычисляет величину `Σ((x - x̅)^2) / n`, где `n` - размер выборки, `x̅`- среднее значение `x`. То есть, дисперсию для множества значений. Возвращает `Float64`. -!!! note "Примечание" - Функция использует вычислительно неустойчивый алгоритм. Если для ваших расчётов необходима [вычислительная устойчивость](https://ru.wikipedia.org/wiki/Вычислительная_устойчивость), используйте функцию `varPopStable`. Она работает медленнее, но обеспечивает меньшую вычислительную ошибку. - ## stddevSamp(x) {#stddevsampx} Результат равен квадратному корню от `varSamp(x)`. -!!! note "Примечание" - Функция использует вычислительно неустойчивый алгоритм. Если для ваших расчётов необходима [вычислительная устойчивость](https://ru.wikipedia.org/wiki/Вычислительная_устойчивость), используйте функцию `stddevSampStable`. Она работает медленнее, но обеспечивает меньшую вычислительную ошибку. - ## stddevPop(x) {#stddevpopx} Результат равен квадратному корню от `varPop(x)`. -!!! note "Примечание" - Функция использует вычислительно неустойчивый алгоритм. Если для ваших расчётов необходима [вычислительная устойчивость](https://ru.wikipedia.org/wiki/Вычислительная_устойчивость), используйте функцию `stddevPopStable`. Она работает медленнее, но обеспечивает меньшую вычислительную ошибку. - - ## topK(N)(column) {#topkncolumn} Возвращает массив наиболее часто встречающихся значений в указанном столбце. Результирующий массив упорядочен по убыванию частоты значения (не по самим значениям). @@ -1639,24 +1626,14 @@ SELECT topKWeighted(10)(number, number) FROM numbers(1000) Возвращает Float64. В случае, когда `n <= 1`, возвращается +∞. -!!! note "Примечание" - Функция использует вычислительно неустойчивый алгоритм. Если для ваших расчётов необходима [вычислительная устойчивость](https://ru.wikipedia.org/wiki/Вычислительная_устойчивость), используйте функцию `covarSampStable`. Она работает медленнее, но обеспечивает меньшую вычислительную ошибку. - ## covarPop(x, y) {#covarpopx-y} Вычисляет величину `Σ((x - x̅)(y - y̅)) / n`. -!!! note "Примечание" - Функция использует вычислительно неустойчивый алгоритм. Если для ваших расчётов необходима [вычислительная устойчивость](https://ru.wikipedia.org/wiki/Вычислительная_устойчивость), используйте функцию `covarPopStable`. Она работает медленнее, но обеспечивает меньшую вычислительную ошибку. - - ## corr(x, y) {#corrx-y} Вычисляет коэффициент корреляции Пирсона: `Σ((x - x̅)(y - y̅)) / sqrt(Σ((x - x̅)^2) * Σ((y - y̅)^2))`. -!!! note "Примечание" - Функция использует вычислительно неустойчивый алгоритм. Если для ваших расчётов необходима [вычислительная устойчивость](https://ru.wikipedia.org/wiki/Вычислительная_устойчивость), используйте функцию `corrStable`. Она работает медленнее, но обеспечивает меньшую вычислительную ошибку. - ## simpleLinearRegression {#simplelinearregression} Выполняет простую (одномерную) линейную регрессию. From 39518fe725da8a571fc48b4058f327c688308241 Mon Sep 17 00:00:00 2001 From: tavplubix Date: Wed, 13 May 2020 16:45:39 +0300 Subject: [PATCH 168/738] trigger CI --- src/Common/ZooKeeper/ZooKeeper.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index 3bd102cb10d..2e94cfe9992 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -676,7 +676,6 @@ bool ZooKeeper::waitForDisappear(const std::string & path, const WaitCondition & if (state->event_type == Coordination::DELETED) return true; } - return false; } From d13a11b42101e152aaf6129812e40f3b16ca1a46 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Wed, 13 May 2020 17:14:05 +0300 Subject: [PATCH 169/738] fix backward compatibility with tuples and distributed --- src/Parsers/ASTLiteral.cpp | 41 ++++++++++++++-- .../__init__.py | 0 .../configs/remote_servers.xml | 18 +++++++ .../test.py | 47 +++++++++++++++++++ 4 files changed, 101 insertions(+), 5 deletions(-) create mode 100644 tests/integration/test_distributed_backward_compatability/__init__.py create mode 100644 tests/integration/test_distributed_backward_compatability/configs/remote_servers.xml create mode 100644 tests/integration/test_distributed_backward_compatability/test.py diff --git a/src/Parsers/ASTLiteral.cpp b/src/Parsers/ASTLiteral.cpp index 92d57687426..184d361ae2e 100644 --- a/src/Parsers/ASTLiteral.cpp +++ b/src/Parsers/ASTLiteral.cpp @@ -2,6 +2,8 @@ #include #include #include +#include +#include namespace DB @@ -14,30 +16,59 @@ void ASTLiteral::updateTreeHashImpl(SipHash & hash_state) const applyVisitor(FieldVisitorHash(hash_state), value); } +/// Writes 'tuple' word before tuple literals for backward compatibility reasons. +/// TODO: remove, when versions lower than 20.3 will be rearely used. +class FieldVisitorToColumnName : public StaticVisitor +{ +public: + template + String operator() (const T & x) const { return visitor(x); } + +private: + FieldVisitorToString visitor; +}; + +template<> +String FieldVisitorToColumnName::operator() (const Tuple & x) const +{ + WriteBufferFromOwnString wb; + + wb << "tuple("; + for (auto it = x.begin(); it != x.end(); ++it) + { + if (it != x.begin()) + wb << ", "; + wb << applyVisitor(*this, *it); + } + wb << ')'; + + return wb.str(); +} + void ASTLiteral::appendColumnNameImpl(WriteBuffer & ostr) const { /// 100 - just arbitrary value. constexpr auto min_elements_for_hashing = 100; - /// Special case for very large arrays and tuples. Instead of listing all elements, will use hash of them. + /// Special case for very large arrays. Instead of listing all elements, will use hash of them. /// (Otherwise column name will be too long, that will lead to significant slowdown of expression analysis.) + /// TODO: Also do hashing for large tuples, when versions lower than 20.3 will be rarely used, because it breaks backward compatibility. auto type = value.getType(); - if ((type == Field::Types::Array && value.get().size() > min_elements_for_hashing) - || (type == Field::Types::Tuple && value.get().size() > min_elements_for_hashing)) + if ((type == Field::Types::Array && value.get().size() > min_elements_for_hashing)) { SipHash hash; applyVisitor(FieldVisitorHash(hash), value); UInt64 low, high; hash.get128(low, high); - writeCString(type == Field::Types::Array ? "__array_" : "__tuple_", ostr); + writeCString("__array_", ostr); writeText(low, ostr); ostr.write('_'); writeText(high, ostr); } else { - String column_name = applyVisitor(FieldVisitorToString(), value); + String column_name = applyVisitor(FieldVisitorToColumnName(), value); writeString(column_name, ostr); } } diff --git a/tests/integration/test_distributed_backward_compatability/__init__.py b/tests/integration/test_distributed_backward_compatability/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_distributed_backward_compatability/configs/remote_servers.xml b/tests/integration/test_distributed_backward_compatability/configs/remote_servers.xml new file mode 100644 index 00000000000..998a969e87e --- /dev/null +++ b/tests/integration/test_distributed_backward_compatability/configs/remote_servers.xml @@ -0,0 +1,18 @@ + + + + + + node1 + 9000 + + + + + node2 + 9000 + + + + + \ No newline at end of file diff --git a/tests/integration/test_distributed_backward_compatability/test.py b/tests/integration/test_distributed_backward_compatability/test.py new file mode 100644 index 00000000000..2b050f379a2 --- /dev/null +++ b/tests/integration/test_distributed_backward_compatability/test.py @@ -0,0 +1,47 @@ +import pytest +import time + +from helpers.cluster import ClickHouseCluster +from helpers.network import PartitionManager +from helpers.test_tools import TSV + + +cluster = ClickHouseCluster(__file__) + +node_old = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], image='yandex/clickhouse-server:19.17.8.54', stay_alive=True, with_installed_binary=True) +node_new = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml']) + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + for node in (node_old, node_new): + node.query("CREATE TABLE local_table(id UInt32, val String) ENGINE = MergeTree ORDER BY id") + + node_old.query("INSERT INTO local_table VALUES (1, 'node1')") + node_new.query("INSERT INTO local_table VALUES (2, 'node2')") + + node_old.query("CREATE TABLE distributed(id UInt32, val String) ENGINE = Distributed(test_cluster, default, local_table)") + node_new.query("CREATE TABLE distributed(id UInt32, val String) ENGINE = Distributed(test_cluster, default, local_table)") + + yield cluster + + finally: + cluster.shutdown() + +def test_distributed_in_tuple(started_cluster): + query1 = "SELECT count() FROM distributed WHERE (id, val) IN ((1, 'node1'), (2, 'a'), (3, 'b'))" + query2 = "SELECT sum((id, val) IN ((1, 'node1'), (2, 'a'), (3, 'b'))) FROM distributed" + assert node_old.query(query1) == "1\n" + assert node_old.query(query2) == "1\n" + assert node_new.query(query1) == "1\n" + assert node_new.query(query2) == "1\n" + + large_set = '(' + ','.join([str(i) for i in range(1000)]) + ')' + query3 = "SELECT count() FROM distributed WHERE id IN " + large_set + query4 = "SELECT sum(id IN {}) FROM distributed".format(large_set) + assert node_old.query(query3) == "2\n" + assert node_old.query(query4) == "2\n" + assert node_new.query(query3) == "2\n" + assert node_new.query(query4) == "2\n" \ No newline at end of file From 774f9fccc5bbc28be0e952f5580297bb6b746bd6 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 13 May 2020 18:25:55 +0300 Subject: [PATCH 170/738] Update test.py --- tests/integration/test_rename_column/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_rename_column/test.py b/tests/integration/test_rename_column/test.py index db6a20c5592..85d054fefeb 100644 --- a/tests/integration/test_rename_column/test.py +++ b/tests/integration/test_rename_column/test.py @@ -176,7 +176,7 @@ def rename_column_on_cluster(node, table_name, name, new_name, iterations=1, ign def alter_move(node, table_name, iterations=1, ignore_exception=False): for i in range(iterations): move_part = random.randint(0, 99) - move_volume = 'external'# if random.randint(0, 1) > 0 else 'default' + move_volume = 'external' try: node.query("ALTER TABLE {table_name} MOVE PARTITION '{move_part}' TO VOLUME '{move_volume}'" .format(table_name=table_name, move_part=move_part, move_volume=move_volume)) From efa1e9562bbaf422ddf1777f662d8bc1f28c3e6c Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 13 May 2020 18:39:33 +0300 Subject: [PATCH 171/738] Update aggregatefunction.md --- docs/en/sql-reference/data-types/aggregatefunction.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/sql-reference/data-types/aggregatefunction.md b/docs/en/sql-reference/data-types/aggregatefunction.md index 5c62f5088c4..f745bd7cf5a 100644 --- a/docs/en/sql-reference/data-types/aggregatefunction.md +++ b/docs/en/sql-reference/data-types/aggregatefunction.md @@ -1,13 +1,13 @@ --- toc_priority: 52 -toc_title: AggregateFunction(name, types_of_arguments...) +toc_title: AggregateFunction --- -# AggregateFunction(name, types\_of\_arguments…) {#data-type-aggregatefunction} +# AggregateFunction {#data-type-aggregatefunction} Aggregate functions can have an implementation-defined intermediate state that can be serialized to an AggregateFunction(…) data type and stored in a table, usually, by means of [a materialized view](../../sql-reference/statements/select.md#create-view). The common way to produce an aggregate function state is by calling the aggregate function with the `-State` suffix. To get the final result of aggregation in the future, you must use the same aggregate function with the `-Merge`suffix. -`AggregateFunction` — parametric data type. +`AggregateFunction(name, types_of_arguments…)` — parametric data type. **Parameters** From 1ceabf1e10dbd354ad28ef2c858601c74233f253 Mon Sep 17 00:00:00 2001 From: potya Date: Wed, 13 May 2020 18:49:17 +0300 Subject: [PATCH 172/738] at start --- src/DataTypes/DataTypesDecimal.cpp | 1 + src/DataTypes/DataTypesNumber.cpp | 1 - src/Interpreters/InterpreterSelectQuery.cpp | 2 -- .../InterpreterSelectWithUnionQuery.cpp | 1 - .../queries/0_stateless/00700_decimal_arithm.sql | 16 ++++++++++++---- 5 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/DataTypes/DataTypesDecimal.cpp b/src/DataTypes/DataTypesDecimal.cpp index 2586f54dc87..400cd30d969 100644 --- a/src/DataTypes/DataTypesDecimal.cpp +++ b/src/DataTypes/DataTypesDecimal.cpp @@ -181,6 +181,7 @@ void registerDataTypeDecimal(DataTypeFactory & factory) factory.registerAlias("DEC", "Decimal", DataTypeFactory::CaseInsensitive); factory.registerAlias("NUMERIC", "Decimal", DataTypeFactory::CaseInsensitive); factory.registerAlias("FIXED", "Decimal", DataTypeFactory::CaseInsensitive); + factory.registerAlias("REAL", "Decimal32", DataTypeFactory::CaseInsensitive); } /// Explicit template instantiations. diff --git a/src/DataTypes/DataTypesNumber.cpp b/src/DataTypes/DataTypesNumber.cpp index c98852bd2bc..4da767ae359 100644 --- a/src/DataTypes/DataTypesNumber.cpp +++ b/src/DataTypes/DataTypesNumber.cpp @@ -32,7 +32,6 @@ void registerDataTypeNumbers(DataTypeFactory & factory) factory.registerAlias("INT4", "Int32", DataTypeFactory::CaseInsensitive); factory.registerAlias("INTEGER", "Int32", DataTypeFactory::CaseInsensitive); factory.registerAlias("BIGINT", "Int64", DataTypeFactory::CaseInsensitive); - factory.registerAlias("INT8", "Int64", DataTypeFactory::CaseInsensitive); factory.registerAlias("FLOAT", "Float32", DataTypeFactory::CaseInsensitive); factory.registerAlias("DOUBLE", "Float64", DataTypeFactory::CaseInsensitive); } diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index f79e2822035..290bc26856a 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -2280,8 +2280,6 @@ void InterpreterSelectQuery::executeUnion(Pipeline & pipeline, Block header) pipeline.streams.push_back(pipeline.stream_with_non_joined_data); pipeline.stream_with_non_joined_data = nullptr; } - - } diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index dc47ce93dab..9cdb19b1934 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -121,7 +121,6 @@ InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery( } options.ignore_limits |= all_nested_ignore_limits; options.ignore_quota |= all_nested_ignore_quota; - } diff --git a/tests/queries/0_stateless/00700_decimal_arithm.sql b/tests/queries/0_stateless/00700_decimal_arithm.sql index 8cdc81e21b4..f456fcdf807 100644 --- a/tests/queries/0_stateless/00700_decimal_arithm.sql +++ b/tests/queries/0_stateless/00700_decimal_arithm.sql @@ -11,12 +11,20 @@ CREATE TABLE IF NOT EXISTS decimal g Decimal(9, 3), h decimal(18, 9), i deciMAL(38, 18), - j dec(4,2) + j dec(4,2), + k NumEriC(4, 23), + l numeric(3,9), + m NUMEric(18, 9), + n FixED(12, 6), + o fixed(8, 6), + p real(3, 1), + q REal(28, 12), + r real(9, 9) ) ENGINE = Memory; -INSERT INTO decimal (a, b, c, d, e, f, g, h, i, j) VALUES (0, 0, 0, 0, 0, 0, 0, 0, 0, 0); -INSERT INTO decimal (a, b, c, d, e, f, g, h, i, j) VALUES (42, 42, 42, 0.42, 0.42, 0.42, 42.42, 42.42, 42.42, 42.42); -INSERT INTO decimal (a, b, c, d, e, f, g, h, i, j) VALUES (-42, -42, -42, -0.42, -0.42, -0.42, -42.42, -42.42, -42.42, -42.42); +INSERT INTO decimal (a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r) VALUES (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,); +INSERT INTO decimal (a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r) VALUES (42, 42, 42, 0.42, 0.42, 0.42, 42.42, 42.42, 42.42, 42.42, 42.42, 42.42, 42.42, 42.42, 42.42, 42.42, 42.42, 42.42); +INSERT INTO decimal (a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r) VALUES (-42, -42, -42, -0.42, -0.42, -0.42, -42.42, -42.42, -42.42, -42.42, -42.42, -42.42, -42.42, -42.42, -42.42, -42.42, -42.42, -42.42); SELECT a + a, a - a, a * a, a / a, intDiv(a, a), intDivOrZero(a, a) FROM decimal WHERE a = 42; SELECT b + b, b - b, b * b, b / b, intDiv(b, b), intDivOrZero(b, b) FROM decimal WHERE b = 42; From 84d4ad4315eb5be47f9bfafb68f5da3ef1b59d82 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Wed, 13 May 2020 18:53:47 +0300 Subject: [PATCH 173/738] add some comments --- src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp | 2 ++ src/Storages/MergeTree/MergeTreeDataSelectExecutor.h | 1 + 2 files changed, 3 insertions(+) diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index ddd1e14381d..d52bc0e45bf 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -1022,6 +1022,7 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder( sort_description.emplace_back(data.sorting_key_columns[j], input_sorting_info->direction, 1); + /// Drop temporary columns, added by 'sorting_key_prefix_expr' out_projection = createProjection(pipes.back(), data); for (auto & pipe : pipes) pipe.addSimpleTransform(std::make_shared(pipe.getHeader(), sorting_key_prefix_expr)); @@ -1086,6 +1087,7 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal( virt_columns, part.part_index_in_query); Pipe pipe(std::move(source_processor)); + /// Drop temporary columns, added by 'sorting_key_expr' if (!out_projection) out_projection = createProjection(pipe, data); diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h index 05f330bf643..09b7958c59f 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h @@ -57,6 +57,7 @@ private: const Settings & settings, const MergeTreeReaderSettings & reader_settings) const; + /// out_projection - save projection only with columns, requested to read Pipes spreadMarkRangesAmongStreamsWithOrder( RangesInDataParts && parts, size_t num_streams, From 6b12cfb8e4812cfc351cad7d09cfcc1f0b6a9423 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=82=D0=B5=D0=BC=20=D0=A1=D1=82=D1=80=D0=B5?= =?UTF-8?q?=D0=BB=D1=8C=D1=86=D0=BE=D0=B2?= Date: Wed, 13 May 2020 19:05:05 +0300 Subject: [PATCH 174/738] documentation added --- .../external-dictionaries/external-dicts-dict-layout.md | 5 +++++ .../external-dictionaries/external-dicts-dict-layout.md | 6 ++++++ .../external-dictionaries/external-dicts-dict-layout.md | 5 +++++ 3 files changed, 16 insertions(+) diff --git a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md index 2d838c964fc..bdadf97cd11 100644 --- a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md +++ b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md @@ -58,6 +58,7 @@ LAYOUT(LAYOUT_TYPE(param value)) -- layout settings - [range\_hashed](#range-hashed) - [complex\_key\_hashed](#complex-key-hashed) - [complex\_key\_cache](#complex-key-cache) +- [complex\_key\_direct](#complex-key-direct) - [ip\_trie](#ip-trie) ### flat {#flat} @@ -317,6 +318,10 @@ or LAYOUT(DIRECT()) ``` +### complex\_key\_direct {#complex-key-direct} + +This type of storage is for use with composite [keys](external-dicts-dict-structure.md). Similar to `direct`. + ### ip\_trie {#ip-trie} This type of storage is for mapping network prefixes (IP addresses) to metadata such as ASN. diff --git a/docs/fr/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md b/docs/fr/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md index b62dadebc1e..9c2583e3394 100644 --- a/docs/fr/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md +++ b/docs/fr/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md @@ -60,6 +60,7 @@ LAYOUT(LAYOUT_TYPE(param value)) -- layout settings - [range\_hashed](#range-hashed) - [complex\_key\_hashed](#complex-key-hashed) - [complex\_key\_cache](#complex-key-cache) +- [complex\_key\_direct](#complex-key-direct) - [ip\_trie](#ip-trie) ### plat {#flat} @@ -319,6 +320,11 @@ ou LAYOUT(DIRECT()) ``` +### complex\_key\_cache {#complex-key-cache} + +Ce type de stockage est pour une utilisation avec composite [touches](external-dicts-dict-structure.md). Semblable à `direct`. + + ### ip\_trie {#ip-trie} Ce type de stockage permet de mapper des préfixes de réseau (adresses IP) à des métadonnées telles que ASN. diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md index aedc701482d..9256fab5e0c 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md @@ -53,6 +53,7 @@ LAYOUT(LAYOUT_TYPE(param value)) -- layout settings - [range\_hashed](#range-hashed) - [complex\_key\_hashed](#complex-key-hashed) - [complex\_key\_cache](#complex-key-cache) +- [complex\_key\_direct](#complex-key-direct) - [ip\_trie](#ip-trie) ### flat {#flat} @@ -315,6 +316,10 @@ LAYOUT(CACHE(SIZE_IN_CELLS 1000000000)) LAYOUT(DIRECT()) ``` +### complex\_key\_direct {#complex-key-direct} + +Тип размещения предназначен для использования с составными [ключами](external-dicts-dict-structure.md). Аналогичен `direct`. + ### ip\_trie {#ip-trie} Тип размещения предназначен для сопоставления префиксов сети (IP адресов) с метаданными, такими как ASN. From bd77908c9319ca66581f64e4aa62725836679621 Mon Sep 17 00:00:00 2001 From: dgrr Date: Wed, 13 May 2020 18:31:23 +0200 Subject: [PATCH 175/738] Added multiline test --- .../01278_format_multiple_queries.reference | 16 ++++++++++++++++ .../01278_format_multiple_queries.sh | 18 ++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 tests/queries/0_stateless/01278_format_multiple_queries.reference create mode 100755 tests/queries/0_stateless/01278_format_multiple_queries.sh diff --git a/tests/queries/0_stateless/01278_format_multiple_queries.reference b/tests/queries/0_stateless/01278_format_multiple_queries.reference new file mode 100644 index 00000000000..cba2cc7b320 --- /dev/null +++ b/tests/queries/0_stateless/01278_format_multiple_queries.reference @@ -0,0 +1,16 @@ +SELECT + a, + b AS x +FROM table AS t +INNER JOIN table2 AS t2 ON t.id = t2.t_id +WHERE 1 = 1 +; + +SELECT + a, + b AS x, + if(x = 0, a, b) +FROM table2 AS t +WHERE t.id != 0 +; + diff --git a/tests/queries/0_stateless/01278_format_multiple_queries.sh b/tests/queries/0_stateless/01278_format_multiple_queries.sh new file mode 100755 index 00000000000..090e1f9d664 --- /dev/null +++ b/tests/queries/0_stateless/01278_format_multiple_queries.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +set -e + +format="$CLICKHOUSE_FORMAT -n" + +$format < Date: Tue, 21 Apr 2020 01:07:00 +0300 Subject: [PATCH 176/738] Refactoring of getting information about access rights. --- src/Access/AccessFlags.h | 4 + src/Access/AccessRights.cpp | 91 +++++--- src/Access/AccessRights.h | 10 +- src/Access/AccessRightsElement.cpp | 197 +++++++++++++++--- src/Access/AccessRightsElement.h | 15 +- src/Access/AccessType.h | 30 ++- src/Access/ContextAccess.cpp | 12 +- src/Access/ExtendedRoleSet.cpp | 16 +- src/Access/ExtendedRoleSet.h | 2 - src/Access/GrantedAccess.cpp | 22 ++ src/Access/GrantedAccess.h | 55 +++++ src/Access/GrantedRoles.cpp | 64 ++++++ src/Access/GrantedRoles.h | 39 ++++ src/Access/Role.cpp | 5 +- src/Access/Role.h | 11 +- src/Access/RoleCache.cpp | 8 +- src/Access/User.cpp | 6 +- src/Access/User.h | 11 +- src/Access/UsersConfigAccessStorage.cpp | 14 +- src/Access/ya.make | 2 + .../InterpreterCreateUserQuery.cpp | 2 +- src/Interpreters/InterpreterGrantQuery.cpp | 27 ++- src/Interpreters/InterpreterSetRoleQuery.cpp | 6 +- .../InterpreterShowGrantsQuery.cpp | 87 ++++---- src/Interpreters/tests/users.cpp | 2 +- src/Parsers/ASTGrantQuery.cpp | 102 ++++----- .../01073_grant_and_revoke.reference | 4 +- 27 files changed, 584 insertions(+), 260 deletions(-) create mode 100644 src/Access/GrantedAccess.cpp create mode 100644 src/Access/GrantedAccess.h create mode 100644 src/Access/GrantedRoles.cpp create mode 100644 src/Access/GrantedRoles.h diff --git a/src/Access/AccessFlags.h b/src/Access/AccessFlags.h index cbba295be1a..54a8b9cde76 100644 --- a/src/Access/AccessFlags.h +++ b/src/Access/AccessFlags.h @@ -61,6 +61,10 @@ public: friend bool operator ==(const AccessFlags & left, const AccessFlags & right) { return left.flags == right.flags; } friend bool operator !=(const AccessFlags & left, const AccessFlags & right) { return !(left == right); } + friend bool operator <(const AccessFlags & left, const AccessFlags & right) { return memcmp(&left.flags, &right.flags, sizeof(Flags)) < 0; } + friend bool operator >(const AccessFlags & left, const AccessFlags & right) { return right < left; } + friend bool operator <=(const AccessFlags & left, const AccessFlags & right) { return !(right < left); } + friend bool operator >=(const AccessFlags & left, const AccessFlags & right) { return !(left < right); } void clear() { flags.reset(); } diff --git a/src/Access/AccessRights.cpp b/src/Access/AccessRights.cpp index 79d77e3f352..086d740ede1 100644 --- a/src/Access/AccessRights.cpp +++ b/src/Access/AccessRights.cpp @@ -606,69 +606,102 @@ void AccessRights::revoke(const AccessRightsElements & elements, std::string_vie } -AccessRights::Elements AccessRights::getElements() const +AccessRightsElements AccessRights::getGrants() const +{ + AccessRightsElements grants; + getGrantsAndPartialRevokesImpl(&grants, nullptr); + return grants; +} + +AccessRightsElements AccessRights::getPartialRevokes() const +{ + AccessRightsElements partial_revokes; + getGrantsAndPartialRevokesImpl(nullptr, &partial_revokes); + return partial_revokes; +} + +AccessRights::GrantsAndPartialRevokes AccessRights::getGrantsAndPartialRevokes() const +{ + GrantsAndPartialRevokes res; + getGrantsAndPartialRevokesImpl(&res.grants, &res.revokes); + return res; +} + + +void AccessRights::getGrantsAndPartialRevokesImpl(AccessRightsElements * out_grants, AccessRightsElements * out_partial_revokes) const { if (!root) - return {}; - Elements res; + return; auto global_access = root->access; - if (global_access) - res.grants.push_back({global_access}); + if (out_grants && global_access) + out_grants->push_back({global_access}); if (root->children) { for (const auto & [db_name, db_node] : *root->children) { - auto db_grants = db_node.access - global_access; - auto db_partial_revokes = global_access - db_node.access; - if (db_partial_revokes) - res.partial_revokes.push_back({db_partial_revokes, db_name}); - if (db_grants) - res.grants.push_back({db_grants, db_name}); + if (out_grants) + { + if (auto db_grants = db_node.access - global_access) + out_grants->push_back({db_grants, db_name}); + } + if (out_partial_revokes) + { + if (auto db_partial_revokes = global_access - db_node.access) + out_partial_revokes->push_back({db_partial_revokes, db_name}); + } if (db_node.children) { for (const auto & [table_name, table_node] : *db_node.children) { - auto table_grants = table_node.access - db_node.access; - auto table_partial_revokes = db_node.access - table_node.access; - if (table_partial_revokes) - res.partial_revokes.push_back({table_partial_revokes, db_name, table_name}); - if (table_grants) - res.grants.push_back({table_grants, db_name, table_name}); + if (out_grants) + { + if (auto table_grants = table_node.access - db_node.access) + out_grants->push_back({table_grants, db_name, table_name}); + } + if (out_partial_revokes) + { + if (auto table_partial_revokes = db_node.access - table_node.access) + out_partial_revokes->push_back({table_partial_revokes, db_name, table_name}); + } if (table_node.children) { for (const auto & [column_name, column_node] : *table_node.children) { - auto column_grants = column_node.access - table_node.access; - auto column_partial_revokes = table_node.access - column_node.access; - if (column_partial_revokes) - res.partial_revokes.push_back({column_partial_revokes, db_name, table_name, column_name}); - if (column_grants) - res.grants.push_back({column_grants, db_name, table_name, column_name}); + if (out_grants) + { + if (auto column_grants = column_node.access - table_node.access) + out_grants->push_back({column_grants, db_name, table_name, column_name}); + } + if (out_partial_revokes) + { + if (auto column_partial_revokes = table_node.access - column_node.access) + out_partial_revokes->push_back({column_partial_revokes, db_name, table_name, column_name}); + } } + } } } } } - return res; } String AccessRights::toString() const { - auto elements = getElements(); String res; - if (!elements.grants.empty()) + auto gr = getGrantsAndPartialRevokes(); + if (!gr.grants.empty()) { res += "GRANT "; - res += elements.grants.toString(); + res += gr.grants.toString(); } - if (!elements.partial_revokes.empty()) + if (!gr.revokes.empty()) { if (!res.empty()) res += ", "; res += "REVOKE "; - res += elements.partial_revokes.toString(); + res += gr.revokes.toString(); } if (res.empty()) res = "GRANT USAGE ON *.*"; diff --git a/src/Access/AccessRights.h b/src/Access/AccessRights.h index 133038f2d44..c32514e8feb 100644 --- a/src/Access/AccessRights.h +++ b/src/Access/AccessRights.h @@ -49,12 +49,14 @@ public: void revoke(const AccessRightsElements & elements, std::string_view current_database = {}); /// Returns the information about all the access granted. - struct Elements + struct GrantsAndPartialRevokes { AccessRightsElements grants; - AccessRightsElements partial_revokes; + AccessRightsElements revokes; }; - Elements getElements() const; + AccessRightsElements getGrants() const; + AccessRightsElements getPartialRevokes() const; + GrantsAndPartialRevokes getGrantsAndPartialRevokes() const; /// Returns the information about all the access granted as a string. String toString() const; @@ -92,6 +94,8 @@ private: template AccessFlags getAccessImpl(const Args &... args) const; + void getGrantsAndPartialRevokesImpl(AccessRightsElements * grants, AccessRightsElements * partial_revokes) const; + void logTree() const; struct Node; diff --git a/src/Access/AccessRightsElement.cpp b/src/Access/AccessRightsElement.cpp index 81237a334e2..db1ea5d3d5c 100644 --- a/src/Access/AccessRightsElement.cpp +++ b/src/Access/AccessRightsElement.cpp @@ -1,10 +1,124 @@ #include #include #include +#include +#include +#include +#include +#include namespace DB { +namespace +{ + size_t groupElements(AccessRightsElements & elements, size_t start) + { + auto & start_element = elements[start]; + auto it = std::find_if(elements.begin() + start + 1, elements.end(), + [&](const AccessRightsElement & element) + { + return (element.database != start_element.database) || + (element.any_database != start_element.any_database) || + (element.table != start_element.table) || + (element.any_table != start_element.any_table) || + (element.any_column != start_element.any_column); + }); + size_t end = it - elements.begin(); + + /// All the elements at indices from start to end here specify + /// the same database and table. + + if (start_element.any_column) + { + /// Easy case: the elements don't specify columns. + /// All we need is to combine the access flags. + for (size_t i = start + 1; i != end; ++i) + { + start_element.access_flags |= elements[i].access_flags; + elements[i].access_flags = {}; + } + return end; + } + + /// Difficult case: the elements specify columns. + /// We have to find groups of columns with common access flags. + for (size_t i = start; i != end; ++i) + { + if (!elements[i].access_flags) + continue; + + AccessFlags common_flags = elements[i].access_flags; + size_t num_elements_with_common_flags = 1; + for (size_t j = i + 1; j != end; ++j) + { + auto new_common_flags = common_flags & elements[j].access_flags; + if (new_common_flags) + { + common_flags = new_common_flags; + ++num_elements_with_common_flags; + } + } + + if (num_elements_with_common_flags == 1) + continue; + + if (elements[i].access_flags != common_flags) + { + elements.insert(elements.begin() + i + 1, elements[i]); + elements[i].access_flags = common_flags; + elements[i].columns.clear(); + ++end; + } + + for (size_t j = i + 1; j != end; ++j) + { + if ((elements[j].access_flags & common_flags) == common_flags) + { + boost::range::push_back(elements[i].columns, elements[j].columns); + elements[j].access_flags -= common_flags; + } + } + } + + return end; + } + + /// Tries to combine elements to decrease their number. + void groupElements(AccessRightsElements & elements) + { + if (!boost::range::is_sorted(elements)) + boost::range::sort(elements); /// Algorithm in groupElement() requires elements to be sorted. + for (size_t start = 0; start != elements.size();) + start = groupElements(elements, start); + } + + /// Removes unnecessary elements, sorts elements and makes them unique. + void sortElementsAndMakeUnique(AccessRightsElements & elements) + { + /// Remove empty elements. + boost::range::remove_erase_if(elements, [](const AccessRightsElement & element) + { + return !element.access_flags || (!element.any_column && element.columns.empty()); + }); + + /// Sort columns and make them unique. + for (auto & element : elements) + { + if (element.any_column) + continue; + + if (!boost::range::is_sorted(element.columns)) + boost::range::sort(element.columns); + element.columns.erase(std::unique(element.columns.begin(), element.columns.end()), element.columns.end()); + } + + /// Sort elements themselves. + boost::range::sort(elements); + elements.erase(std::unique(elements.begin(), elements.end()), elements.end()); + } +} + void AccessRightsElement::setDatabase(const String & new_database) { database = new_database; @@ -26,10 +140,29 @@ bool AccessRightsElement::isEmptyDatabase() const String AccessRightsElement::toString() const +{ + String msg = toStringWithoutON(); + msg += " ON "; + + if (any_database) + msg += "*."; + else if (!database.empty()) + msg += backQuoteIfNeed(database) + "."; + + if (any_table) + msg += "*"; + else + msg += backQuoteIfNeed(table); + return msg; +} + +String AccessRightsElement::toStringWithoutON() const { String columns_in_parentheses; if (!any_column) { + if (columns.empty()) + return "USAGE"; for (const auto & column : columns) { columns_in_parentheses += columns_in_parentheses.empty() ? "(" : ", "; @@ -38,25 +171,17 @@ String AccessRightsElement::toString() const columns_in_parentheses += ")"; } + auto keywords = access_flags.toKeywords(); + if (keywords.empty()) + return "USAGE"; + String msg; - for (const std::string_view & keyword : access_flags.toKeywords()) + for (const std::string_view & keyword : keywords) { if (!msg.empty()) msg += ", "; msg += String{keyword} + columns_in_parentheses; } - - msg += " ON "; - - if (any_database) - msg += "*."; - else if (!database.empty() && (database != IDictionary::NO_DATABASE_TAG)) - msg += backQuoteIfNeed(database) + "."; - - if (any_table) - msg += "*"; - else - msg += backQuoteIfNeed(table); return msg; } @@ -68,19 +193,41 @@ void AccessRightsElements::replaceEmptyDatabase(const String & new_database) } -String AccessRightsElements::toString() const +String AccessRightsElements::toString() { - String res; - bool need_comma = false; - for (const auto & element : *this) - { - if (std::exchange(need_comma, true)) - res += ", "; - res += element.toString(); - } + normalize(); - if (res.empty()) - res = "USAGE ON *.*"; - return res; + if (empty()) + return "USAGE ON *.*"; + + String msg; + bool need_comma = false; + for (size_t i = 0; i != size(); ++i) + { + const auto & element = (*this)[i]; + if (std::exchange(need_comma, true)) + msg += ", "; + bool next_element_on_same_db_and_table = false; + if (i != size() - 1) + { + const auto & next_element = (*this)[i + 1]; + if ((element.database == next_element.database) && (element.any_database == next_element.any_database) + && (element.table == next_element.table) && (element.any_table == next_element.any_table)) + next_element_on_same_db_and_table = true; + } + if (next_element_on_same_db_and_table) + msg += element.toStringWithoutON(); + else + msg += element.toString(); + } + return msg; } + + +void AccessRightsElements::normalize() +{ + groupElements(*this); + sortElementsAndMakeUnique(*this); +} + } diff --git a/src/Access/AccessRightsElement.h b/src/Access/AccessRightsElement.h index 3894b6f5157..70eb95c2d17 100644 --- a/src/Access/AccessRightsElement.h +++ b/src/Access/AccessRightsElement.h @@ -71,6 +71,14 @@ struct AccessRightsElement { } + auto toTuple() const { return std::tie(access_flags, database, any_database, table, any_table, columns, any_column); } + friend bool operator==(const AccessRightsElement & left, const AccessRightsElement & right) { return left.toTuple() == right.toTuple(); } + friend bool operator!=(const AccessRightsElement & left, const AccessRightsElement & right) { return left.toTuple() != right.toTuple(); } + friend bool operator<(const AccessRightsElement & left, const AccessRightsElement & right) { return left.toTuple() < right.toTuple(); } + friend bool operator>(const AccessRightsElement & left, const AccessRightsElement & right) { return left.toTuple() > right.toTuple(); } + friend bool operator<=(const AccessRightsElement & left, const AccessRightsElement & right) { return left.toTuple() <= right.toTuple(); } + friend bool operator>=(const AccessRightsElement & left, const AccessRightsElement & right) { return left.toTuple() >= right.toTuple(); } + /// Sets the database. void setDatabase(const String & new_database); @@ -82,6 +90,7 @@ struct AccessRightsElement /// Returns a human-readable representation like "SELECT, UPDATE(x, y) ON db.table". /// The returned string isn't prefixed with the "GRANT" keyword. String toString() const; + String toStringWithoutON() const; }; @@ -94,7 +103,11 @@ public: /// Returns a human-readable representation like "SELECT, UPDATE(x, y) ON db.table". /// The returned string isn't prefixed with the "GRANT" keyword. - String toString() const; + String toString() const { return AccessRightsElements(*this).toString(); } + String toString(); + + /// Reorder and group elements to show them in more readable form. + void normalize(); }; } diff --git a/src/Access/AccessType.h b/src/Access/AccessType.h index d0665a6e55f..c4fdbc46b71 100644 --- a/src/Access/AccessType.h +++ b/src/Access/AccessType.h @@ -167,50 +167,48 @@ enum class AccessType #undef DECLARE_ACCESS_TYPE_ENUM_CONST }; -std::string_view toString(AccessType type); - namespace impl { template - class AccessTypeToKeywordConverter + class AccessTypeToStringConverter { public: - static const AccessTypeToKeywordConverter & instance() + static const AccessTypeToStringConverter & instance() { - static const AccessTypeToKeywordConverter res; + static const AccessTypeToStringConverter res; return res; } std::string_view convert(AccessType type) const { - return access_type_to_keyword_mapping[static_cast(type)]; + return access_type_to_string_mapping[static_cast(type)]; } private: - AccessTypeToKeywordConverter() + AccessTypeToStringConverter() { -#define INSERT_ACCESS_TYPE_KEYWORD_PAIR_TO_MAPPING(name, aliases, node_type, parent_group_name) \ - insertToMapping(AccessType::name, #name); +#define ACCESS_TYPE_TO_STRING_CONVERTER_ADD_TO_MAPPING(name, aliases, node_type, parent_group_name) \ + addToMapping(AccessType::name, #name); - APPLY_FOR_ACCESS_TYPES(INSERT_ACCESS_TYPE_KEYWORD_PAIR_TO_MAPPING) + APPLY_FOR_ACCESS_TYPES(ACCESS_TYPE_TO_STRING_CONVERTER_ADD_TO_MAPPING) -#undef INSERT_ACCESS_TYPE_KEYWORD_PAIR_TO_MAPPING +#undef ACCESS_TYPE_TO_STRING_CONVERTER_ADD_TO_MAPPING } - void insertToMapping(AccessType type, const std::string_view & str) + void addToMapping(AccessType type, const std::string_view & str) { String str2{str}; boost::replace_all(str2, "_", " "); size_t index = static_cast(type); - access_type_to_keyword_mapping.resize(std::max(index + 1, access_type_to_keyword_mapping.size())); - access_type_to_keyword_mapping[index] = str2; + access_type_to_string_mapping.resize(std::max(index + 1, access_type_to_string_mapping.size())); + access_type_to_string_mapping[index] = str2; } - Strings access_type_to_keyword_mapping; + Strings access_type_to_string_mapping; }; } -inline std::string_view toKeyword(AccessType type) { return impl::AccessTypeToKeywordConverter<>::instance().convert(type); } +inline std::string_view toString(AccessType type) { return impl::AccessTypeToStringConverter<>::instance().convert(type); } } diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index ab504e32579..3a06a9aeead 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -134,12 +134,12 @@ void ContextAccess::setUser(const UserPtr & user_) const std::vector current_roles, current_roles_with_admin_option; if (params.use_default_roles) { - for (const UUID & id : user->granted_roles) + for (const UUID & id : user->granted_roles.roles) { if (user->default_roles.match(id)) current_roles.push_back(id); } - boost::range::set_intersection(current_roles, user->granted_roles_with_admin_option, + boost::range::set_intersection(current_roles, user->granted_roles.roles_with_admin_option, std::back_inserter(current_roles_with_admin_option)); } else @@ -147,9 +147,9 @@ void ContextAccess::setUser(const UserPtr & user_) const current_roles.reserve(params.current_roles.size()); for (const auto & id : params.current_roles) { - if (user->granted_roles.count(id)) + if (user->granted_roles.roles.contains(id)) current_roles.push_back(id); - if (user->granted_roles_with_admin_option.count(id)) + if (user->granted_roles.roles_with_admin_option.contains(id)) current_roles_with_admin_option.push_back(id); } } @@ -397,13 +397,13 @@ boost::shared_ptr ContextAccess::calculateResultAccess(bool if (grant_option) { - *merged_access = user->access_with_grant_option; + *merged_access = user->access.access_with_grant_option; if (roles_info) merged_access->merge(roles_info->access_with_grant_option); } else { - *merged_access = user->access; + *merged_access = user->access.access; if (roles_info) merged_access->merge(roles_info->access); } diff --git a/src/Access/ExtendedRoleSet.cpp b/src/Access/ExtendedRoleSet.cpp index 145bd0fe7e5..adee5fea669 100644 --- a/src/Access/ExtendedRoleSet.cpp +++ b/src/Access/ExtendedRoleSet.cpp @@ -1,3 +1,4 @@ + #include #include #include @@ -43,12 +44,6 @@ ExtendedRoleSet::ExtendedRoleSet(const std::vector & ids_) } -ExtendedRoleSet::ExtendedRoleSet(const boost::container::flat_set & ids_) -{ - add(ids_); -} - - ExtendedRoleSet::ExtendedRoleSet(const ASTExtendedRoleSet & ast) { init(ast, nullptr); @@ -126,6 +121,7 @@ std::shared_ptr ExtendedRoleSet::toAST() const ast->names.reserve(ids.size()); for (const UUID & id : ids) ast->names.emplace_back(::DB::toString(id)); + boost::range::sort(ast->names); } if (!except_ids.empty()) @@ -133,6 +129,7 @@ std::shared_ptr ExtendedRoleSet::toAST() const ast->except_names.reserve(except_ids.size()); for (const UUID & except_id : except_ids) ast->except_names.emplace_back(::DB::toString(except_id)); + boost::range::sort(ast->except_names); } return ast; @@ -244,13 +241,6 @@ void ExtendedRoleSet::add(const std::vector & ids_) } -void ExtendedRoleSet::add(const boost::container::flat_set & ids_) -{ - for (const auto & id : ids_) - add(id); -} - - bool ExtendedRoleSet::match(const UUID & id) const { return (all || ids.count(id)) && !except_ids.count(id); diff --git a/src/Access/ExtendedRoleSet.h b/src/Access/ExtendedRoleSet.h index 486b4277337..ba84ade49f8 100644 --- a/src/Access/ExtendedRoleSet.h +++ b/src/Access/ExtendedRoleSet.h @@ -28,7 +28,6 @@ struct ExtendedRoleSet ExtendedRoleSet(const UUID & id); ExtendedRoleSet(const std::vector & ids_); - ExtendedRoleSet(const boost::container::flat_set & ids_); /// The constructor from AST requires the AccessControlManager if `ast.id_mode == false`. ExtendedRoleSet(const ASTExtendedRoleSet & ast); @@ -48,7 +47,6 @@ struct ExtendedRoleSet void clear(); void add(const UUID & id); void add(const std::vector & ids_); - void add(const boost::container::flat_set & ids_); /// Checks if a specified ID matches this ExtendedRoleSet. bool match(const UUID & id) const; diff --git a/src/Access/GrantedAccess.cpp b/src/Access/GrantedAccess.cpp new file mode 100644 index 00000000000..2af1e0b44ec --- /dev/null +++ b/src/Access/GrantedAccess.cpp @@ -0,0 +1,22 @@ +#include + + +namespace DB +{ + +GrantedAccess::GrantsAndPartialRevokes GrantedAccess::getGrantsAndPartialRevokes() const +{ + GrantsAndPartialRevokes res; + res.grants_with_grant_option = access_with_grant_option.getGrants(); + AccessRights access_without_gg = access; + access_without_gg.revoke(res.grants_with_grant_option); + auto gr = access_without_gg.getGrantsAndPartialRevokes(); + res.grants = std::move(gr.grants); + res.revokes = std::move(gr.revokes); + AccessRights access_with_grant_options_without_r = access_with_grant_option; + access_with_grant_options_without_r.grant(res.revokes); + res.revokes_grant_option = access_with_grant_options_without_r.getPartialRevokes(); + return res; +} + +} diff --git a/src/Access/GrantedAccess.h b/src/Access/GrantedAccess.h new file mode 100644 index 00000000000..b8f6bdfe8fb --- /dev/null +++ b/src/Access/GrantedAccess.h @@ -0,0 +1,55 @@ +#pragma once + +#include + + +namespace DB +{ +/// Access rights as they are granted to a role or user. +/// Stores both the access rights themselves and the access rights with grant option. +struct GrantedAccess +{ + AccessRights access; + AccessRights access_with_grant_option; + + template + void grant(const Args &... args) + { + access.grant(args...); + } + + template + void grantWithGrantOption(const Args &... args) + { + access.grant(args...); + access_with_grant_option.grant(args...); + } + + template + void revoke(const Args &... args) + { + access.revoke(args...); + access_with_grant_option.revoke(args...); + } + + template + void revokeGrantOption(const Args &... args) + { + access_with_grant_option.revoke(args...); + } + + struct GrantsAndPartialRevokes + { + AccessRightsElements grants; + AccessRightsElements revokes; + AccessRightsElements grants_with_grant_option; + AccessRightsElements revokes_grant_option; + }; + + /// Retrieves the information about grants and partial revokes. + GrantsAndPartialRevokes getGrantsAndPartialRevokes() const; + + friend bool operator ==(const GrantedAccess & left, const GrantedAccess & right) { return (left.access == right.access) && (left.access_with_grant_option == right.access_with_grant_option); } + friend bool operator !=(const GrantedAccess & left, const GrantedAccess & right) { return !(left == right); } +}; +} diff --git a/src/Access/GrantedRoles.cpp b/src/Access/GrantedRoles.cpp new file mode 100644 index 00000000000..4d7007c4db6 --- /dev/null +++ b/src/Access/GrantedRoles.cpp @@ -0,0 +1,64 @@ +#include +#include + + +namespace DB +{ +void GrantedRoles::grant(const UUID & role) +{ + roles.insert(role); +} + +void GrantedRoles::grant(const std::vector & roles_) +{ + for (const UUID & role : roles_) + grant(role); +} + +void GrantedRoles::grantWithAdminOption(const UUID & role) +{ + roles.insert(role); + roles_with_admin_option.insert(role); +} + +void GrantedRoles::grantWithAdminOption(const std::vector & roles_) +{ + for (const UUID & role : roles_) + grantWithAdminOption(role); +} + + +void GrantedRoles::revoke(const UUID & role) +{ + roles.erase(role); + roles_with_admin_option.erase(role); +} + +void GrantedRoles::revoke(const std::vector & roles_) +{ + for (const UUID & role : roles_) + revoke(role); +} + +void GrantedRoles::revokeAdminOption(const UUID & role) +{ + roles_with_admin_option.erase(role); +} + +void GrantedRoles::revokeAdminOption(const std::vector & roles_) +{ + for (const UUID & role : roles_) + revokeAdminOption(role); +} + + +GrantedRoles::Grants GrantedRoles::getGrants() const +{ + Grants res; + res.grants_with_admin_option.insert(res.grants_with_admin_option.end(), roles_with_admin_option.begin(), roles_with_admin_option.end()); + res.grants.reserve(roles.size() - roles_with_admin_option.size()); + boost::range::set_difference(roles, roles_with_admin_option, std::back_inserter(res.grants)); + return res; +} + +} diff --git a/src/Access/GrantedRoles.h b/src/Access/GrantedRoles.h new file mode 100644 index 00000000000..fd091755a80 --- /dev/null +++ b/src/Access/GrantedRoles.h @@ -0,0 +1,39 @@ +#pragma once + +#include +#include +#include + + +namespace DB +{ +/// Roles when they are granted to a role or user. +/// Stores both the roles themselves and the roles with admin option. +struct GrantedRoles +{ + boost::container::flat_set roles; + boost::container::flat_set roles_with_admin_option; + + void grant(const UUID & role); + void grant(const std::vector & roles_); + void grantWithAdminOption(const UUID & role); + void grantWithAdminOption(const std::vector & roles_); + + void revoke(const UUID & role); + void revoke(const std::vector & roles_); + void revokeAdminOption(const UUID & role); + void revokeAdminOption(const std::vector & roles_); + + struct Grants + { + std::vector grants; + std::vector grants_with_admin_option; + }; + + /// Retrieves the information about grants. + Grants getGrants() const; + + friend bool operator ==(const GrantedRoles & left, const GrantedRoles & right) { return (left.roles == right.roles) && (left.roles_with_admin_option == right.roles_with_admin_option); } + friend bool operator !=(const GrantedRoles & left, const GrantedRoles & right) { return !(left == right); } +}; +} diff --git a/src/Access/Role.cpp b/src/Access/Role.cpp index f20ef9b9bfa..d7bec28c576 100644 --- a/src/Access/Role.cpp +++ b/src/Access/Role.cpp @@ -9,9 +9,6 @@ bool Role::equal(const IAccessEntity & other) const if (!IAccessEntity::equal(other)) return false; const auto & other_role = typeid_cast(other); - return (access == other_role.access) && (access_with_grant_option == other_role.access_with_grant_option) - && (granted_roles == other_role.granted_roles) && (granted_roles_with_admin_option == other_role.granted_roles_with_admin_option) - && (settings == other_role.settings); + return (access == other_role.access) && (granted_roles == other_role.granted_roles) && (settings == other_role.settings); } - } diff --git a/src/Access/Role.h b/src/Access/Role.h index 04330ba85f5..01a5c6ea2ce 100644 --- a/src/Access/Role.h +++ b/src/Access/Role.h @@ -1,10 +1,9 @@ #pragma once #include -#include +#include +#include #include -#include -#include namespace DB @@ -12,10 +11,8 @@ namespace DB struct Role : public IAccessEntity { - AccessRights access; - AccessRights access_with_grant_option; - boost::container::flat_set granted_roles; - boost::container::flat_set granted_roles_with_admin_option; + GrantedAccess access; + GrantedRoles granted_roles; SettingsProfileElements settings; bool equal(const IAccessEntity & other) const override; diff --git a/src/Access/RoleCache.cpp b/src/Access/RoleCache.cpp index 0263b793017..441e3844a07 100644 --- a/src/Access/RoleCache.cpp +++ b/src/Access/RoleCache.cpp @@ -37,10 +37,10 @@ namespace if (!role) return; - for (const auto & granted_role : role->granted_roles) + for (const auto & granted_role : role->granted_roles.roles) collectRoles(collected_roles, get_role_function, granted_role, false, false); - for (const auto & granted_role : role->granted_roles_with_admin_option) + for (const auto & granted_role : role->granted_roles.roles_with_admin_option) collectRoles(collected_roles, get_role_function, granted_role, false, true); } @@ -59,8 +59,8 @@ namespace if (collect_info.with_admin_option) new_info->enabled_roles_with_admin_option.emplace_back(role_id); new_info->names_of_roles[role_id] = role->getName(); - new_info->access.merge(role->access); - new_info->access_with_grant_option.merge(role->access_with_grant_option); + new_info->access.merge(role->access.access); + new_info->access_with_grant_option.merge(role->access.access_with_grant_option); new_info->settings_from_enabled_roles.merge(role->settings); } return new_info; diff --git a/src/Access/User.cpp b/src/Access/User.cpp index 4a751c31e25..459357731ed 100644 --- a/src/Access/User.cpp +++ b/src/Access/User.cpp @@ -10,9 +10,7 @@ bool User::equal(const IAccessEntity & other) const return false; const auto & other_user = typeid_cast(other); return (authentication == other_user.authentication) && (allowed_client_hosts == other_user.allowed_client_hosts) - && (access == other_user.access) && (access_with_grant_option == other_user.access_with_grant_option) - && (granted_roles == other_user.granted_roles) && (granted_roles_with_admin_option == other_user.granted_roles_with_admin_option) - && (default_roles == other_user.default_roles) && (settings == other_user.settings); + && (access == other_user.access) && (granted_roles == other_user.granted_roles) && (default_roles == other_user.default_roles) + && (settings == other_user.settings); } - } diff --git a/src/Access/User.h b/src/Access/User.h index 6df3b3e4d3c..b20f6538e4d 100644 --- a/src/Access/User.h +++ b/src/Access/User.h @@ -3,11 +3,10 @@ #include #include #include -#include +#include +#include #include #include -#include -#include namespace DB @@ -18,10 +17,8 @@ struct User : public IAccessEntity { Authentication authentication; AllowedClientHosts allowed_client_hosts = AllowedClientHosts::AnyHostTag{}; - AccessRights access; - AccessRights access_with_grant_option; - boost::container::flat_set granted_roles; - boost::container::flat_set granted_roles_with_admin_option; + GrantedAccess access; + GrantedRoles granted_roles; ExtendedRoleSet default_roles = ExtendedRoleSet::AllTag{}; SettingsProfileElements settings; diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index 0842839dec8..c4335640b25 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -151,30 +151,30 @@ namespace } } - user->access.grant(AccessType::ALL); /// By default all databases are accessible. + /// By default all databases are accessible + /// and the user can grant everything he has. + user->access.grantWithGrantOption(AccessType::ALL); if (databases) { user->access.revoke(AccessFlags::allFlags() - AccessFlags::allGlobalFlags()); - user->access.grant(AccessFlags::allDictionaryFlags(), IDictionary::NO_DATABASE_TAG); + user->access.grantWithGrantOption(AccessFlags::allDictionaryFlags(), IDictionary::NO_DATABASE_TAG); for (const String & database : *databases) - user->access.grant(AccessFlags::allFlags(), database); + user->access.grantWithGrantOption(AccessFlags::allFlags(), database); } if (dictionaries) { user->access.revoke(AccessFlags::allDictionaryFlags(), IDictionary::NO_DATABASE_TAG); for (const String & dictionary : *dictionaries) - user->access.grant(AccessFlags::allDictionaryFlags(), IDictionary::NO_DATABASE_TAG, dictionary); + user->access.grantWithGrantOption(AccessFlags::allDictionaryFlags(), IDictionary::NO_DATABASE_TAG, dictionary); } - user->access_with_grant_option = user->access; /// By default the user can grant everything he has. - bool access_management = config.getBool(user_config + ".access_management", false); if (!access_management) { user->access.revoke(AccessType::ACCESS_MANAGEMENT); - user->access_with_grant_option.clear(); + user->access.revokeGrantOption(AccessType::ALL); } return user; diff --git a/src/Access/ya.make b/src/Access/ya.make index fb2e23e0684..5b85330a7f4 100644 --- a/src/Access/ya.make +++ b/src/Access/ya.make @@ -18,6 +18,8 @@ SRCS( EnabledRowPolicies.cpp EnabledSettings.cpp ExtendedRoleSet.cpp + GrantedAccess.cpp + GrantedRoles.cpp IAccessEntity.cpp IAccessStorage.cpp MemoryAccessStorage.cpp diff --git a/src/Interpreters/InterpreterCreateUserQuery.cpp b/src/Interpreters/InterpreterCreateUserQuery.cpp index e4900e5c518..7c488ddf8e9 100644 --- a/src/Interpreters/InterpreterCreateUserQuery.cpp +++ b/src/Interpreters/InterpreterCreateUserQuery.cpp @@ -48,7 +48,7 @@ namespace if (default_roles) { if (!query.alter && !default_roles->all) - boost::range::copy(default_roles->getMatchingIDs(), std::inserter(user.granted_roles, user.granted_roles.end())); + user.granted_roles.grant(default_roles->getMatchingIDs()); InterpreterSetRoleQuery::updateUserSetDefaultRoles(user, *default_roles); } diff --git a/src/Interpreters/InterpreterGrantQuery.cpp b/src/Interpreters/InterpreterGrantQuery.cpp index a5f13dbbbfe..c72e48c2019 100644 --- a/src/Interpreters/InterpreterGrantQuery.cpp +++ b/src/Interpreters/InterpreterGrantQuery.cpp @@ -23,14 +23,16 @@ namespace { if (query.kind == Kind::GRANT) { - grantee.access.grant(query.access_rights_elements, current_database); if (query.grant_option) - grantee.access_with_grant_option.grant(query.access_rights_elements, current_database); + grantee.access.grantWithGrantOption(query.access_rights_elements, current_database); + else + grantee.access.grant(query.access_rights_elements, current_database); } else { - grantee.access_with_grant_option.revoke(query.access_rights_elements, current_database); - if (!query.grant_option) + if (query.grant_option) + grantee.access.revokeGrantOption(query.access_rights_elements, current_database); + else grantee.access.revoke(query.access_rights_elements, current_database); } } @@ -39,18 +41,21 @@ namespace { if (query.kind == Kind::GRANT) { - boost::range::copy(roles_from_query, std::inserter(grantee.granted_roles, grantee.granted_roles.end())); if (query.admin_option) - boost::range::copy(roles_from_query, std::inserter(grantee.granted_roles_with_admin_option, grantee.granted_roles_with_admin_option.end())); + grantee.granted_roles.grantWithAdminOption(roles_from_query); + else + grantee.granted_roles.grant(roles_from_query); } else { - for (const UUID & role_from_query : roles_from_query) + if (query.admin_option) + grantee.granted_roles.revokeAdminOption(roles_from_query); + else + grantee.granted_roles.revoke(roles_from_query); + + if constexpr (std::is_same_v) { - grantee.granted_roles_with_admin_option.erase(role_from_query); - if (!query.admin_option) - grantee.granted_roles.erase(role_from_query); - if constexpr (std::is_same_v) + for (const UUID & role_from_query : roles_from_query) grantee.default_roles.ids.erase(role_from_query); } } diff --git a/src/Interpreters/InterpreterSetRoleQuery.cpp b/src/Interpreters/InterpreterSetRoleQuery.cpp index 8f085d66c4c..f56926332ec 100644 --- a/src/Interpreters/InterpreterSetRoleQuery.cpp +++ b/src/Interpreters/InterpreterSetRoleQuery.cpp @@ -42,7 +42,7 @@ void InterpreterSetRoleQuery::setRole(const ASTSetRoleQuery & query) std::vector new_current_roles; if (roles_from_query.all) { - for (const auto & id : user->granted_roles) + for (const auto & id : user->granted_roles.roles) if (roles_from_query.match(id)) new_current_roles.push_back(id); } @@ -50,7 +50,7 @@ void InterpreterSetRoleQuery::setRole(const ASTSetRoleQuery & query) { for (const auto & id : roles_from_query.getMatchingIDs()) { - if (!user->granted_roles.count(id)) + if (!user->granted_roles.roles.contains(id)) throw Exception("Role should be granted to set current", ErrorCodes::SET_NON_GRANTED_ROLE); new_current_roles.push_back(id); } @@ -85,7 +85,7 @@ void InterpreterSetRoleQuery::updateUserSetDefaultRoles(User & user, const Exten { for (const auto & id : roles_from_query.getMatchingIDs()) { - if (!user.granted_roles.count(id)) + if (!user.granted_roles.roles.contains(id)) throw Exception("Role should be granted to set default", ErrorCodes::SET_NON_GRANTED_ROLE); } } diff --git a/src/Interpreters/InterpreterShowGrantsQuery.cpp b/src/Interpreters/InterpreterShowGrantsQuery.cpp index da1d46f0cab..aa139b8e10e 100644 --- a/src/Interpreters/InterpreterShowGrantsQuery.cpp +++ b/src/Interpreters/InterpreterShowGrantsQuery.cpp @@ -10,8 +10,6 @@ #include #include #include -#include -#include namespace DB @@ -23,37 +21,6 @@ namespace ErrorCodes namespace { - std::vector groupByTable(AccessRightsElements && elements) - { - using Key = std::tuple; - std::map grouping_map; - for (auto & element : elements) - { - Key key(element.database, element.any_database, element.table, element.any_table); - grouping_map[key].emplace_back(std::move(element)); - } - std::vector res; - res.reserve(grouping_map.size()); - boost::range::copy(grouping_map | boost::adaptors::map_values, std::back_inserter(res)); - return res; - } - - - struct GroupedGrantsAndPartialRevokes - { - std::vector grants; - std::vector partial_revokes; - }; - - GroupedGrantsAndPartialRevokes groupByTable(AccessRights::Elements && elements) - { - GroupedGrantsAndPartialRevokes res; - res.grants = groupByTable(std::move(elements.grants)); - res.partial_revokes = groupByTable(std::move(elements.partial_revokes)); - return res; - } - - template ASTs getGrantQueriesImpl( const T & grantee, @@ -65,35 +32,51 @@ namespace std::shared_ptr to_roles = std::make_shared(); to_roles->names.push_back(grantee.getName()); - for (bool grant_option : {true, false}) - { - if (!grant_option && (grantee.access == grantee.access_with_grant_option)) - continue; - const auto & access_rights = grant_option ? grantee.access_with_grant_option : grantee.access; - const auto grouped_elements = groupByTable(access_rights.getElements()); + auto grants_and_partial_revokes = grantee.access.getGrantsAndPartialRevokes(); + for (bool grant_option : {false, true}) + { using Kind = ASTGrantQuery::Kind; for (Kind kind : {Kind::GRANT, Kind::REVOKE}) { - for (const auto & elements : (kind == Kind::GRANT ? grouped_elements.grants : grouped_elements.partial_revokes)) + AccessRightsElements * elements = nullptr; + if (grant_option) + elements = (kind == Kind::GRANT) ? &grants_and_partial_revokes.grants_with_grant_option : &grants_and_partial_revokes.revokes_grant_option; + else + elements = (kind == Kind::GRANT) ? &grants_and_partial_revokes.grants : &grants_and_partial_revokes.revokes; + elements->normalize(); + + std::shared_ptr grant_query = nullptr; + for (size_t i = 0; i != elements->size(); ++i) { - auto grant_query = std::make_shared(); - grant_query->kind = kind; - grant_query->attach = attach_mode; - grant_query->grant_option = grant_option; - grant_query->to_roles = to_roles; - grant_query->access_rights_elements = elements; - res.push_back(std::move(grant_query)); + const auto & element = (*elements)[i]; + bool prev_element_on_same_db_and_table = false; + if (grant_query) + { + const auto & prev_element = grant_query->access_rights_elements.back(); + if ((element.database == prev_element.database) && (element.any_database == prev_element.any_database) + && (element.table == prev_element.table) && (element.any_table == prev_element.any_table)) + prev_element_on_same_db_and_table = true; + } + if (!prev_element_on_same_db_and_table) + { + grant_query = std::make_shared(); + grant_query->kind = kind; + grant_query->attach = attach_mode; + grant_query->grant_option = grant_option; + grant_query->to_roles = to_roles; + res.push_back(grant_query); + } + grant_query->access_rights_elements.emplace_back(std::move(element)); } } } - for (bool admin_option : {true, false}) - { - if (!admin_option && (grantee.granted_roles == grantee.granted_roles_with_admin_option)) - continue; + auto grants_roles = grantee.granted_roles.getGrants(); - const auto & roles = admin_option ? grantee.granted_roles_with_admin_option : grantee.granted_roles; + for (bool admin_option : {false, true}) + { + const auto & roles = admin_option ? grants_roles.grants_with_admin_option : grants_roles.grants; if (roles.empty()) continue; diff --git a/src/Interpreters/tests/users.cpp b/src/Interpreters/tests/users.cpp index acd0cfd0519..5c7d66ed7ed 100644 --- a/src/Interpreters/tests/users.cpp +++ b/src/Interpreters/tests/users.cpp @@ -218,7 +218,7 @@ void runOneTest(const TestDescriptor & test_descriptor) try { - res = acl_manager.read(entry.user_name)->access.isGranted(DB::AccessType::ALL, entry.database_name); + res = acl_manager.read(entry.user_name)->access.access.isGranted(DB::AccessType::ALL, entry.database_name); } catch (const Poco::Exception &) { diff --git a/src/Parsers/ASTGrantQuery.cpp b/src/Parsers/ASTGrantQuery.cpp index e6764fc067a..8114bef0766 100644 --- a/src/Parsers/ASTGrantQuery.cpp +++ b/src/Parsers/ASTGrantQuery.cpp @@ -1,10 +1,6 @@ #include #include #include -#include -#include -#include -#include namespace DB @@ -16,61 +12,13 @@ namespace ErrorCodes namespace { - using KeywordToColumnsMap = std::map /* columns */>; - using TableToAccessMap = std::map; - - TableToAccessMap prepareTableToAccessMap(const AccessRightsElements & elements) + void formatColumnNames(const Strings & columns, const IAST::FormatSettings & settings) { - TableToAccessMap res; - for (const auto & element : elements) - { - String database_and_table_name; - if (element.any_database) - { - if (element.any_table) - database_and_table_name = "*.*"; - else - database_and_table_name = "*." + backQuoteIfNeed(element.table); - } - else if (element.database.empty()) - { - if (element.any_table) - database_and_table_name = "*"; - else - database_and_table_name = backQuoteIfNeed(element.table); - } - else - { - if (element.any_table) - database_and_table_name = backQuoteIfNeed(element.database) + ".*"; - else - database_and_table_name = backQuoteIfNeed(element.database) + "." + backQuoteIfNeed(element.table); - } - - KeywordToColumnsMap & keyword_to_columns = res[database_and_table_name]; - for (const auto & keyword : element.access_flags.toKeywords()) - boost::range::push_back(keyword_to_columns[keyword], element.columns); - } - - for (auto & keyword_to_columns : res | boost::adaptors::map_values) - { - for (auto & columns : keyword_to_columns | boost::adaptors::map_values) - boost::range::sort(columns); - } - return res; - } - - - void formatColumnNames(const std::vector & columns, const IAST::FormatSettings & settings) - { - if (columns.empty()) - return; - settings.ostr << "("; - bool need_comma_after_column_name = false; + bool need_comma = false; for (const auto & column : columns) { - if (std::exchange(need_comma_after_column_name, true)) + if (std::exchange(need_comma, true)) settings.ostr << ", "; settings.ostr << backQuoteIfNeed(column); } @@ -80,20 +28,50 @@ namespace void formatAccessRightsElements(const AccessRightsElements & elements, const IAST::FormatSettings & settings) { - bool need_comma = false; - for (const auto & [database_and_table, keyword_to_columns] : prepareTableToAccessMap(elements)) + bool no_output = true; + for (size_t i = 0; i != elements.size(); ++i) { - for (const auto & [keyword, columns] : keyword_to_columns) + const auto & element = elements[i]; + auto keywords = element.access_flags.toKeywords(); + if (keywords.empty() || (!element.any_column && element.columns.empty())) + continue; + + for (const auto & keyword : keywords) { - if (std::exchange(need_comma, true)) + if (!std::exchange(no_output, false)) settings.ostr << ", "; settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << keyword << (settings.hilite ? IAST::hilite_none : ""); - formatColumnNames(columns, settings); + if (!element.any_column) + formatColumnNames(element.columns, settings); } - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " ON " << (settings.hilite ? IAST::hilite_none : "") << database_and_table; + bool next_element_on_same_db_and_table = false; + if (i != elements.size() - 1) + { + const auto & next_element = elements[i + 1]; + if ((element.database == next_element.database) && (element.any_database == next_element.any_database) + && (element.table == next_element.table) && (element.any_table == next_element.any_table)) + next_element_on_same_db_and_table = true; + } + + if (!next_element_on_same_db_and_table) + { + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " ON " << (settings.hilite ? IAST::hilite_none : ""); + if (element.any_database) + settings.ostr << "*."; + else if (!element.database.empty()) + settings.ostr << backQuoteIfNeed(element.database) + "."; + + if (element.any_table) + settings.ostr << "*"; + else + settings.ostr << backQuoteIfNeed(element.table); + } } + + if (no_output) + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "USAGE ON " << (settings.hilite ? IAST::hilite_none : "") << "*.*"; } @@ -134,7 +112,7 @@ void ASTGrantQuery::formatImpl(const FormatSettings & settings, FormatState &, F settings.ostr << (settings.hilite ? hilite_keyword : "") << " ADMIN OPTION FOR" << (settings.hilite ? hilite_none : ""); } - if ((!!roles + !access_rights_elements.empty()) != 1) + if (roles && !access_rights_elements.empty()) throw Exception("Either roles or access rights elements should be set", ErrorCodes::LOGICAL_ERROR); settings.ostr << " "; diff --git a/tests/queries/0_stateless/01073_grant_and_revoke.reference b/tests/queries/0_stateless/01073_grant_and_revoke.reference index d7d97fa28fe..134256c8113 100644 --- a/tests/queries/0_stateless/01073_grant_and_revoke.reference +++ b/tests/queries/0_stateless/01073_grant_and_revoke.reference @@ -1,11 +1,11 @@ CREATE USER test_user_01073 A B -GRANT ALTER DELETE, INSERT ON *.* TO test_user_01073 GRANT SELECT ON db1.* TO test_user_01073 GRANT SELECT ON db2.table TO test_user_01073 GRANT SELECT(col1) ON db3.table TO test_user_01073 GRANT SELECT(col1, col2) ON db4.table TO test_user_01073 +GRANT INSERT, ALTER DELETE ON *.* TO test_user_01073 C -GRANT ALTER DELETE ON *.* TO test_user_01073 GRANT SELECT(col1) ON db4.table TO test_user_01073 +GRANT ALTER DELETE ON *.* TO test_user_01073 From b93a15ef36beef3c3bcea1fdb892fe79c10bfd9a Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Tue, 21 Apr 2020 13:21:03 +0300 Subject: [PATCH 177/738] Refactoring of settings profiles to store setting_index instead of setting_name. --- src/Access/SettingsConstraints.cpp | 68 ++++++++++++++------ src/Access/SettingsConstraints.h | 29 ++++++--- src/Access/SettingsProfileElement.cpp | 35 +++++----- src/Access/SettingsProfileElement.h | 4 +- src/Access/SettingsProfilesCache.cpp | 1 + src/Access/UsersConfigAccessStorage.cpp | 5 +- src/Parsers/ASTSettingsProfileElement.cpp | 2 +- src/Parsers/ASTSettingsProfileElement.h | 4 +- src/Parsers/ParserSettingsProfileElement.cpp | 6 +- 9 files changed, 98 insertions(+), 56 deletions(-) diff --git a/src/Access/SettingsConstraints.cpp b/src/Access/SettingsConstraints.cpp index 21ca2c8ab26..11dbd016e64 100644 --- a/src/Access/SettingsConstraints.cpp +++ b/src/Access/SettingsConstraints.cpp @@ -30,16 +30,23 @@ void SettingsConstraints::clear() } -void SettingsConstraints::setMinValue(const StringRef & name, const Field & min_value) +void SettingsConstraints::setMinValue(const StringRef & setting_name, const Field & min_value) +{ + setMinValue(Settings::findIndexStrict(setting_name), min_value); +} + +void SettingsConstraints::setMinValue(size_t setting_index, const Field & min_value) { - size_t setting_index = Settings::findIndexStrict(name); getConstraintRef(setting_index).min_value = Settings::valueToCorrespondingType(setting_index, min_value); } - -Field SettingsConstraints::getMinValue(const StringRef & name) const +Field SettingsConstraints::getMinValue(const StringRef & setting_name) const +{ + return getMinValue(Settings::findIndexStrict(setting_name)); +} + +Field SettingsConstraints::getMinValue(size_t setting_index) const { - size_t setting_index = Settings::findIndexStrict(name); const auto * ptr = tryGetConstraint(setting_index); if (ptr) return ptr->min_value; @@ -50,14 +57,21 @@ Field SettingsConstraints::getMinValue(const StringRef & name) const void SettingsConstraints::setMaxValue(const StringRef & name, const Field & max_value) { - size_t setting_index = Settings::findIndexStrict(name); + setMaxValue(Settings::findIndexStrict(name), max_value); +} + +void SettingsConstraints::setMaxValue(size_t setting_index, const Field & max_value) +{ getConstraintRef(setting_index).max_value = Settings::valueToCorrespondingType(setting_index, max_value); } - -Field SettingsConstraints::getMaxValue(const StringRef & name) const +Field SettingsConstraints::getMaxValue(const StringRef & setting_name) const +{ + return getMaxValue(Settings::findIndexStrict(setting_name)); +} + +Field SettingsConstraints::getMaxValue(size_t setting_index) const { - size_t setting_index = Settings::findIndexStrict(name); const auto * ptr = tryGetConstraint(setting_index); if (ptr) return ptr->max_value; @@ -66,16 +80,23 @@ Field SettingsConstraints::getMaxValue(const StringRef & name) const } -void SettingsConstraints::setReadOnly(const StringRef & name, bool read_only) +void SettingsConstraints::setReadOnly(const StringRef & setting_name, bool read_only) +{ + setReadOnly(Settings::findIndexStrict(setting_name), read_only); +} + +void SettingsConstraints::setReadOnly(size_t setting_index, bool read_only) { - size_t setting_index = Settings::findIndexStrict(name); getConstraintRef(setting_index).read_only = read_only; } - -bool SettingsConstraints::isReadOnly(const StringRef & name) const +bool SettingsConstraints::isReadOnly(const StringRef & setting_name) const +{ + return isReadOnly(Settings::findIndexStrict(setting_name)); +} + +bool SettingsConstraints::isReadOnly(size_t setting_index) const { - size_t setting_index = Settings::findIndexStrict(name); const auto * ptr = tryGetConstraint(setting_index); if (ptr) return ptr->read_only; @@ -83,20 +104,26 @@ bool SettingsConstraints::isReadOnly(const StringRef & name) const return false; } - -void SettingsConstraints::set(const StringRef & name, const Field & min_value, const Field & max_value, bool read_only) +void SettingsConstraints::set(const StringRef & setting_name, const Field & min_value, const Field & max_value, bool read_only) +{ + set(Settings::findIndexStrict(setting_name), min_value, max_value, read_only); +} + +void SettingsConstraints::set(size_t setting_index, const Field & min_value, const Field & max_value, bool read_only) { - size_t setting_index = Settings::findIndexStrict(name); auto & ref = getConstraintRef(setting_index); ref.min_value = min_value; ref.max_value = max_value; ref.read_only = read_only; } - -void SettingsConstraints::get(const StringRef & name, Field & min_value, Field & max_value, bool & read_only) const +void SettingsConstraints::get(const StringRef & setting_name, Field & min_value, Field & max_value, bool & read_only) const +{ + get(Settings::findIndexStrict(setting_name), min_value, max_value, read_only); +} + +void SettingsConstraints::get(size_t setting_index, Field & min_value, Field & max_value, bool & read_only) const { - size_t setting_index = Settings::findIndexStrict(name); const auto * ptr = tryGetConstraint(setting_index); if (ptr) { @@ -112,7 +139,6 @@ void SettingsConstraints::get(const StringRef & name, Field & min_value, Field & } } - void SettingsConstraints::merge(const SettingsConstraints & other) { for (const auto & [setting_index, other_constraint] : other.constraints_by_index) diff --git a/src/Access/SettingsConstraints.h b/src/Access/SettingsConstraints.h index 074dc66d123..65537250957 100644 --- a/src/Access/SettingsConstraints.h +++ b/src/Access/SettingsConstraints.h @@ -1,7 +1,10 @@ #pragma once +#include #include -#include +#include +#include + namespace Poco { @@ -60,17 +63,25 @@ public: void clear(); bool empty() const { return constraints_by_index.empty(); } - void setMinValue(const StringRef & name, const Field & min_value); - Field getMinValue(const StringRef & name) const; + void setMinValue(const StringRef & setting_name, const Field & min_value); + void setMinValue(size_t setting_index, const Field & min_value); + Field getMinValue(const StringRef & setting_name) const; + Field getMinValue(size_t setting_index) const; - void setMaxValue(const StringRef & name, const Field & max_value); - Field getMaxValue(const StringRef & name) const; + void setMaxValue(const StringRef & setting_name, const Field & max_value); + void setMaxValue(size_t setting_index, const Field & max_value); + Field getMaxValue(const StringRef & setting_name) const; + Field getMaxValue(size_t setting_index) const; - void setReadOnly(const StringRef & name, bool read_only); - bool isReadOnly(const StringRef & name) const; + void setReadOnly(const StringRef & setting_name, bool read_only); + void setReadOnly(size_t setting_index, bool read_only); + bool isReadOnly(const StringRef & setting_name) const; + bool isReadOnly(size_t setting_index) const; - void set(const StringRef & name, const Field & min_value, const Field & max_value, bool read_only); - void get(const StringRef & name, Field & min_value, Field & max_value, bool & read_only) const; + void set(const StringRef & setting_name, const Field & min_value, const Field & max_value, bool read_only); + void set(size_t setting_index, const Field & min_value, const Field & max_value, bool read_only); + void get(const StringRef & setting_name, Field & min_value, Field & max_value, bool & read_only) const; + void get(size_t setting_index, Field & min_value, Field & max_value, bool & read_only) const; void merge(const SettingsConstraints & other); diff --git a/src/Access/SettingsProfileElement.cpp b/src/Access/SettingsProfileElement.cpp index b052f8b5e75..d4f6ff5d0f2 100644 --- a/src/Access/SettingsProfileElement.cpp +++ b/src/Access/SettingsProfileElement.cpp @@ -33,21 +33,20 @@ void SettingsProfileElement::init(const ASTSettingsProfileElement & ast, const A if (!ast.parent_profile.empty()) parent_profile = name_to_id(ast.parent_profile); - if (!ast.name.empty()) + if (!ast.setting_name.empty()) { - name = ast.name; + setting_index = Settings::findIndexStrict(ast.setting_name); value = ast.value; min_value = ast.min_value; max_value = ast.max_value; readonly = ast.readonly; - size_t index = Settings::findIndexStrict(name); if (!value.isNull()) - value = Settings::valueToCorrespondingType(index, value); + value = Settings::valueToCorrespondingType(setting_index, value); if (!min_value.isNull()) - min_value = Settings::valueToCorrespondingType(index, min_value); + min_value = Settings::valueToCorrespondingType(setting_index, min_value); if (!max_value.isNull()) - max_value = Settings::valueToCorrespondingType(index, max_value); + max_value = Settings::valueToCorrespondingType(setting_index, max_value); } } @@ -60,7 +59,9 @@ std::shared_ptr SettingsProfileElement::toAST() const if (parent_profile) ast->parent_profile = ::DB::toString(*parent_profile); - ast->name = name; + if (setting_index != static_cast(-1)) + ast->setting_name = Settings::getName(setting_index).toString(); + ast->value = value; ast->min_value = min_value; ast->max_value = max_value; @@ -81,7 +82,9 @@ std::shared_ptr SettingsProfileElement::toASTWithName ast->parent_profile = *parent_profile_name; } - ast->name = name; + if (setting_index != static_cast(-1)) + ast->setting_name = Settings::getName(setting_index).toString(); + ast->value = value; ast->min_value = min_value; ast->max_value = max_value; @@ -132,8 +135,8 @@ Settings SettingsProfileElements::toSettings() const Settings res; for (const auto & elem : *this) { - if (!elem.name.empty() && !elem.value.isNull()) - res.set(elem.name, elem.value); + if ((elem.setting_index != static_cast(-1)) && !elem.value.isNull()) + res.set(elem.setting_index, elem.value); } return res; } @@ -143,8 +146,8 @@ SettingsChanges SettingsProfileElements::toSettingsChanges() const SettingsChanges res; for (const auto & elem : *this) { - if (!elem.name.empty() && !elem.value.isNull()) - res.push_back({elem.name, elem.value}); + if ((elem.setting_index != static_cast(-1)) && !elem.value.isNull()) + res.push_back({Settings::getName(elem.setting_index).toString(), elem.value}); } return res; } @@ -154,14 +157,14 @@ SettingsConstraints SettingsProfileElements::toSettingsConstraints() const SettingsConstraints res; for (const auto & elem : *this) { - if (!elem.name.empty()) + if (elem.setting_index != static_cast(-1)) { if (!elem.min_value.isNull()) - res.setMinValue(elem.name, elem.min_value); + res.setMinValue(elem.setting_index, elem.min_value); if (!elem.max_value.isNull()) - res.setMaxValue(elem.name, elem.max_value); + res.setMaxValue(elem.setting_index, elem.max_value); if (elem.readonly) - res.setReadOnly(elem.name, *elem.readonly); + res.setReadOnly(elem.setting_index, *elem.readonly); } } return res; diff --git a/src/Access/SettingsProfileElement.h b/src/Access/SettingsProfileElement.h index abcac2567c8..f64317344b8 100644 --- a/src/Access/SettingsProfileElement.h +++ b/src/Access/SettingsProfileElement.h @@ -20,13 +20,13 @@ class AccessControlManager; struct SettingsProfileElement { std::optional parent_profile; - String name; + size_t setting_index = static_cast(-1); Field value; Field min_value; Field max_value; std::optional readonly; - auto toTuple() const { return std::tie(parent_profile, name, value, min_value, max_value, readonly); } + auto toTuple() const { return std::tie(parent_profile, setting_index, value, min_value, max_value, readonly); } friend bool operator==(const SettingsProfileElement & lhs, const SettingsProfileElement & rhs) { return lhs.toTuple() == rhs.toTuple(); } friend bool operator!=(const SettingsProfileElement & lhs, const SettingsProfileElement & rhs) { return !(lhs == rhs); } friend bool operator <(const SettingsProfileElement & lhs, const SettingsProfileElement & rhs) { return lhs.toTuple() < rhs.toTuple(); } diff --git a/src/Access/SettingsProfilesCache.cpp b/src/Access/SettingsProfilesCache.cpp index f283715e129..516b47213fe 100644 --- a/src/Access/SettingsProfilesCache.cpp +++ b/src/Access/SettingsProfilesCache.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index c4335640b25..0389734e236 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -362,7 +363,7 @@ namespace for (const String & name : names) { SettingsProfileElement profile_element; - profile_element.name = name; + profile_element.setting_index = Settings::findIndexStrict(name); Poco::Util::AbstractConfiguration::Keys constraint_types; String path_to_name = path_to_constraints + "." + name; config.keys(path_to_name, constraint_types); @@ -411,7 +412,7 @@ namespace } SettingsProfileElement profile_element; - profile_element.name = key; + profile_element.setting_index = Settings::findIndexStrict(key); profile_element.value = config.getString(profile_config + "." + key); profile->elements.emplace_back(std::move(profile_element)); } diff --git a/src/Parsers/ASTSettingsProfileElement.cpp b/src/Parsers/ASTSettingsProfileElement.cpp index 24f1aa60813..a7955411347 100644 --- a/src/Parsers/ASTSettingsProfileElement.cpp +++ b/src/Parsers/ASTSettingsProfileElement.cpp @@ -31,7 +31,7 @@ void ASTSettingsProfileElement::formatImpl(const FormatSettings & settings, Form return; } - settings.ostr << name; + settings.ostr << setting_name; if (!value.isNull()) { diff --git a/src/Parsers/ASTSettingsProfileElement.h b/src/Parsers/ASTSettingsProfileElement.h index ee1ee28c383..6a54bca3213 100644 --- a/src/Parsers/ASTSettingsProfileElement.h +++ b/src/Parsers/ASTSettingsProfileElement.h @@ -13,7 +13,7 @@ class ASTSettingsProfileElement : public IAST { public: String parent_profile; - String name; + String setting_name; Field value; Field min_value; Field max_value; @@ -21,7 +21,7 @@ public: bool id_mode = false; /// If true then `parent_profile` keeps UUID, not a name. bool use_inherit_keyword = false; /// If true then this element is a part of ASTCreateSettingsProfileQuery. - bool empty() const { return parent_profile.empty() && name.empty(); } + bool empty() const { return parent_profile.empty() && setting_name.empty(); } String getID(char) const override { return "SettingsProfileElement"; } ASTPtr clone() const override { return std::make_shared(*this); } diff --git a/src/Parsers/ParserSettingsProfileElement.cpp b/src/Parsers/ParserSettingsProfileElement.cpp index 31bc339f544..37044e8ccbe 100644 --- a/src/Parsers/ParserSettingsProfileElement.cpp +++ b/src/Parsers/ParserSettingsProfileElement.cpp @@ -102,7 +102,7 @@ namespace bool ParserSettingsProfileElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { String parent_profile; - String name; + String setting_name; Field value; Field min_value; Field max_value; @@ -119,7 +119,7 @@ bool ParserSettingsProfileElement::parseImpl(Pos & pos, ASTPtr & node, Expected ASTPtr name_ast; if (!ParserIdentifier{}.parse(pos, name_ast, expected)) return false; - name = getIdentifierName(name_ast); + setting_name = getIdentifierName(name_ast); bool has_value_or_constraint = false; while (parseValue(pos, expected, value) || parseMinMaxValue(pos, expected, min_value, max_value) @@ -134,7 +134,7 @@ bool ParserSettingsProfileElement::parseImpl(Pos & pos, ASTPtr & node, Expected auto result = std::make_shared(); result->parent_profile = std::move(parent_profile); - result->name = std::move(name); + result->setting_name = std::move(setting_name); result->value = std::move(value); result->min_value = std::move(min_value); result->max_value = std::move(max_value); From c7213ab607669f49293a7e584c546bd5e73e2aad Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Wed, 29 Apr 2020 22:35:56 +0300 Subject: [PATCH 178/738] Use boost::flat_set instead of vector to store current and enabled roles. --- src/Access/AccessControlManager.cpp | 14 +-- src/Access/AccessControlManager.h | 13 +-- src/Access/ContextAccess.cpp | 56 ++++-------- src/Access/ContextAccess.h | 28 ++++-- src/Access/EnabledQuota.h | 2 +- src/Access/EnabledRoles.h | 5 +- src/Access/EnabledRolesInfo.h | 8 +- src/Access/EnabledRowPolicies.h | 2 +- src/Access/EnabledSettings.h | 3 +- src/Access/ExtendedRoleSet.cpp | 86 ++++++++---------- src/Access/ExtendedRoleSet.h | 6 +- src/Access/QuotaCache.cpp | 7 +- src/Access/QuotaCache.h | 8 +- src/Access/RoleCache.cpp | 93 +++++++++----------- src/Access/RoleCache.h | 8 +- src/Access/RowPolicyCache.cpp | 2 +- src/Access/RowPolicyCache.h | 2 +- src/Access/SettingsProfilesCache.cpp | 2 +- src/Access/SettingsProfilesCache.h | 2 +- src/Interpreters/Context.cpp | 20 ++--- src/Interpreters/Context.h | 12 +-- src/Interpreters/InterpreterSetRoleQuery.cpp | 10 +-- 22 files changed, 172 insertions(+), 217 deletions(-) diff --git a/src/Access/AccessControlManager.cpp b/src/Access/AccessControlManager.cpp index f8f15e425ed..e2589969fd8 100644 --- a/src/Access/AccessControlManager.cpp +++ b/src/Access/AccessControlManager.cpp @@ -42,7 +42,7 @@ public: std::shared_ptr getContextAccess( const UUID & user_id, - const std::vector & current_roles, + const boost::container::flat_set & current_roles, bool use_default_roles, const Settings & settings, const String & current_database, @@ -113,7 +113,7 @@ void AccessControlManager::setDefaultProfileName(const String & default_profile_ std::shared_ptr AccessControlManager::getContextAccess( const UUID & user_id, - const std::vector & current_roles, + const boost::container::flat_set & current_roles, bool use_default_roles, const Settings & settings, const String & current_database, @@ -124,21 +124,21 @@ std::shared_ptr AccessControlManager::getContextAccess( std::shared_ptr AccessControlManager::getEnabledRoles( - const std::vector & current_roles, - const std::vector & current_roles_with_admin_option) const + const boost::container::flat_set & current_roles, + const boost::container::flat_set & current_roles_with_admin_option) const { return role_cache->getEnabledRoles(current_roles, current_roles_with_admin_option); } -std::shared_ptr AccessControlManager::getEnabledRowPolicies(const UUID & user_id, const std::vector & enabled_roles) const +std::shared_ptr AccessControlManager::getEnabledRowPolicies(const UUID & user_id, const boost::container::flat_set & enabled_roles) const { return row_policy_cache->getEnabledRowPolicies(user_id, enabled_roles); } std::shared_ptr AccessControlManager::getEnabledQuota( - const UUID & user_id, const String & user_name, const std::vector & enabled_roles, const Poco::Net::IPAddress & address, const String & custom_quota_key) const + const UUID & user_id, const String & user_name, const boost::container::flat_set & enabled_roles, const Poco::Net::IPAddress & address, const String & custom_quota_key) const { return quota_cache->getEnabledQuota(user_id, user_name, enabled_roles, address, custom_quota_key); } @@ -153,7 +153,7 @@ std::vector AccessControlManager::getQuotaUsageInfo() const std::shared_ptr AccessControlManager::getEnabledSettings( const UUID & user_id, const SettingsProfileElements & settings_from_user, - const std::vector & enabled_roles, + const boost::container::flat_set & enabled_roles, const SettingsProfileElements & settings_from_enabled_roles) const { return settings_profiles_cache->getEnabledSettings(user_id, settings_from_user, enabled_roles, settings_from_enabled_roles); diff --git a/src/Access/AccessControlManager.h b/src/Access/AccessControlManager.h index 810970a8379..94bc22eb3e5 100644 --- a/src/Access/AccessControlManager.h +++ b/src/Access/AccessControlManager.h @@ -2,6 +2,7 @@ #include #include +#include #include @@ -51,24 +52,24 @@ public: std::shared_ptr getContextAccess( const UUID & user_id, - const std::vector & current_roles, + const boost::container::flat_set & current_roles, bool use_default_roles, const Settings & settings, const String & current_database, const ClientInfo & client_info) const; std::shared_ptr getEnabledRoles( - const std::vector & current_roles, - const std::vector & current_roles_with_admin_option) const; + const boost::container::flat_set & current_roles, + const boost::container::flat_set & current_roles_with_admin_option) const; std::shared_ptr getEnabledRowPolicies( const UUID & user_id, - const std::vector & enabled_roles) const; + const boost::container::flat_set & enabled_roles) const; std::shared_ptr getEnabledQuota( const UUID & user_id, const String & user_name, - const std::vector & enabled_roles, + const boost::container::flat_set & enabled_roles, const Poco::Net::IPAddress & address, const String & custom_quota_key) const; @@ -76,7 +77,7 @@ public: std::shared_ptr getEnabledSettings(const UUID & user_id, const SettingsProfileElements & settings_from_user, - const std::vector & enabled_roles, + const boost::container::flat_set & enabled_roles, const SettingsProfileElements & settings_from_enabled_roles) const; std::shared_ptr getProfileSettings(const String & profile_name) const; diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index 3a06a9aeead..475a225ba05 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -121,7 +121,6 @@ void ContextAccess::setUser(const UserPtr & user_) const subscription_for_roles_changes = {}; enabled_roles = nullptr; roles_info = nullptr; - roles_with_admin_option = nullptr; enabled_row_policies = nullptr; enabled_quota = nullptr; enabled_settings = nullptr; @@ -131,29 +130,28 @@ void ContextAccess::setUser(const UserPtr & user_) const user_name = user->getName(); trace_log = &Poco::Logger::get("ContextAccess (" + user_name + ")"); - std::vector current_roles, current_roles_with_admin_option; + boost::container::flat_set current_roles, current_roles_with_admin_option; if (params.use_default_roles) { for (const UUID & id : user->granted_roles.roles) { if (user->default_roles.match(id)) - current_roles.push_back(id); + current_roles.emplace(id); } - boost::range::set_intersection(current_roles, user->granted_roles.roles_with_admin_option, - std::back_inserter(current_roles_with_admin_option)); } else { - current_roles.reserve(params.current_roles.size()); - for (const auto & id : params.current_roles) - { - if (user->granted_roles.roles.contains(id)) - current_roles.push_back(id); - if (user->granted_roles.roles_with_admin_option.contains(id)) - current_roles_with_admin_option.push_back(id); - } + boost::range::set_intersection( + params.current_roles, + user->granted_roles.roles, + std::inserter(current_roles, current_roles.end())); } + boost::range::set_intersection( + current_roles, + user->granted_roles.roles_with_admin_option, + std::inserter(current_roles_with_admin_option, current_roles_with_admin_option.end())); + subscription_for_roles_changes = {}; enabled_roles = manager->getEnabledRoles(current_roles, current_roles_with_admin_option); subscription_for_roles_changes = enabled_roles->subscribeForChanges([this](const std::shared_ptr & roles_info_) @@ -170,7 +168,6 @@ void ContextAccess::setRolesInfo(const std::shared_ptr & { assert(roles_info_); roles_info = roles_info_; - roles_with_admin_option.store(boost::make_shared>(roles_info->enabled_roles_with_admin_option.begin(), roles_info->enabled_roles_with_admin_option.end())); boost::range::fill(result_access, nullptr /* need recalculate */); enabled_row_policies = manager->getEnabledRowPolicies(*params.user_id, roles_info->enabled_roles); enabled_quota = manager->getEnabledQuota(*params.user_id, user_name, roles_info->enabled_roles, params.address, params.quota_key); @@ -357,10 +354,13 @@ void ContextAccess::checkAdminOption(const UUID & role_id) const if (isGranted(AccessType::ROLE_ADMIN)) return; - auto roles_with_admin_option_loaded = roles_with_admin_option.load(); - if (roles_with_admin_option_loaded && roles_with_admin_option_loaded->count(role_id)) + auto info = getRolesInfo(); + if (info && info->enabled_roles_with_admin_option.count(role_id)) return; + if (!user) + throw Exception(user_name + ": User has been dropped", ErrorCodes::UNKNOWN_USER); + std::optional role_name = manager->readName(role_id); if (!role_name) role_name = "ID {" + toString(role_id) + "}"; @@ -485,30 +485,6 @@ std::shared_ptr ContextAccess::getRolesInfo() const return roles_info; } -std::vector ContextAccess::getCurrentRoles() const -{ - std::lock_guard lock{mutex}; - return roles_info ? roles_info->current_roles : std::vector{}; -} - -Strings ContextAccess::getCurrentRolesNames() const -{ - std::lock_guard lock{mutex}; - return roles_info ? roles_info->getCurrentRolesNames() : Strings{}; -} - -std::vector ContextAccess::getEnabledRoles() const -{ - std::lock_guard lock{mutex}; - return roles_info ? roles_info->enabled_roles : std::vector{}; -} - -Strings ContextAccess::getEnabledRolesNames() const -{ - std::lock_guard lock{mutex}; - return roles_info ? roles_info->getEnabledRolesNames() : Strings{}; -} - std::shared_ptr ContextAccess::getRowPolicies() const { std::lock_guard lock{mutex}; diff --git a/src/Access/ContextAccess.h b/src/Access/ContextAccess.h index e0fbf58dbe8..9a9f0e31470 100644 --- a/src/Access/ContextAccess.h +++ b/src/Access/ContextAccess.h @@ -35,7 +35,7 @@ public: struct Params { std::optional user_id; - std::vector current_roles; + boost::container::flat_set current_roles; bool use_default_roles = false; UInt64 readonly = 0; bool allow_ddl = false; @@ -56,22 +56,36 @@ public: }; const Params & getParams() const { return params; } + + /// Returns the current user. The function can return nullptr. UserPtr getUser() const; String getUserName() const; bool isCorrectPassword(const String & password) const; bool isClientHostAllowed() const; + /// Returns information about current and enabled roles. + /// The function can return nullptr. std::shared_ptr getRolesInfo() const; - std::vector getCurrentRoles() const; - Strings getCurrentRolesNames() const; - std::vector getEnabledRoles() const; - Strings getEnabledRolesNames() const; + /// Returns information about enabled row policies. + /// The function can return nullptr. std::shared_ptr getRowPolicies() const; + + /// Returns the row policy filter for a specified table. + /// The function returns nullptr if there is no filter to apply. ASTPtr getRowPolicyCondition(const String & database, const String & table_name, RowPolicy::ConditionType index, const ASTPtr & extra_condition = nullptr) const; + + /// Returns the quota to track resource consumption. + /// The function returns nullptr if no tracking or limitation is needed. std::shared_ptr getQuota() const; + + /// Returns the default settings, i.e. the settings to apply on user's login. + /// The function returns nullptr if it's no need to apply settings. std::shared_ptr getDefaultSettings() const; + + /// Returns the settings' constraints. + /// The function returns nullptr if there are no constraints. std::shared_ptr getSettingsConstraints() const; /// Checks if a specified access is granted, and throws an exception if not. @@ -118,7 +132,8 @@ public: /// Checks if a specified role is granted with admin option, and throws an exception if not. void checkAdminOption(const UUID & role_id) const; - /// Returns an instance of ContextAccess which has full access to everything. + /// Makes an instance of ContextAccess which provides full access to everything + /// without any limitations. This is used for the global context. static std::shared_ptr getFullAccess(); private: @@ -157,7 +172,6 @@ private: mutable std::shared_ptr enabled_roles; mutable ext::scope_guard subscription_for_roles_changes; mutable std::shared_ptr roles_info; - mutable boost::atomic_shared_ptr> roles_with_admin_option; mutable boost::atomic_shared_ptr result_access[7]; mutable std::shared_ptr enabled_row_policies; mutable std::shared_ptr enabled_quota; diff --git a/src/Access/EnabledQuota.h b/src/Access/EnabledQuota.h index 5a624c651af..e0a04b02206 100644 --- a/src/Access/EnabledQuota.h +++ b/src/Access/EnabledQuota.h @@ -23,7 +23,7 @@ public: { UUID user_id; String user_name; - std::vector enabled_roles; + boost::container::flat_set enabled_roles; Poco::Net::IPAddress client_address; String client_key; diff --git a/src/Access/EnabledRoles.h b/src/Access/EnabledRoles.h index 122b1a16fe3..0c5b799a30b 100644 --- a/src/Access/EnabledRoles.h +++ b/src/Access/EnabledRoles.h @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -16,8 +17,8 @@ class EnabledRoles public: struct Params { - std::vector current_roles; - std::vector current_roles_with_admin_option; + boost::container::flat_set current_roles; + boost::container::flat_set current_roles_with_admin_option; auto toTuple() const { return std::tie(current_roles, current_roles_with_admin_option); } friend bool operator ==(const Params & lhs, const Params & rhs) { return lhs.toTuple() == rhs.toTuple(); } diff --git a/src/Access/EnabledRolesInfo.h b/src/Access/EnabledRolesInfo.h index 837d4b74ad5..45e1bfd9057 100644 --- a/src/Access/EnabledRolesInfo.h +++ b/src/Access/EnabledRolesInfo.h @@ -3,8 +3,8 @@ #include #include #include +#include #include -#include namespace DB @@ -13,9 +13,9 @@ namespace DB /// Information about a role. struct EnabledRolesInfo { - std::vector current_roles; - std::vector enabled_roles; - std::vector enabled_roles_with_admin_option; + boost::container::flat_set current_roles; + boost::container::flat_set enabled_roles; + boost::container::flat_set enabled_roles_with_admin_option; std::unordered_map names_of_roles; AccessRights access; AccessRights access_with_grant_option; diff --git a/src/Access/EnabledRowPolicies.h b/src/Access/EnabledRowPolicies.h index 9befb65ff0b..6274850363c 100644 --- a/src/Access/EnabledRowPolicies.h +++ b/src/Access/EnabledRowPolicies.h @@ -21,7 +21,7 @@ public: struct Params { UUID user_id; - std::vector enabled_roles; + boost::container::flat_set enabled_roles; auto toTuple() const { return std::tie(user_id, enabled_roles); } friend bool operator ==(const Params & lhs, const Params & rhs) { return lhs.toTuple() == rhs.toTuple(); } diff --git a/src/Access/EnabledSettings.h b/src/Access/EnabledSettings.h index d8e969d685d..8e92298328c 100644 --- a/src/Access/EnabledSettings.h +++ b/src/Access/EnabledSettings.h @@ -5,6 +5,7 @@ #include #include #include +#include #include @@ -17,7 +18,7 @@ public: struct Params { UUID user_id; - std::vector enabled_roles; + boost::container::flat_set enabled_roles; SettingsProfileElements settings_from_enabled_roles; SettingsProfileElements settings_from_user; diff --git a/src/Access/ExtendedRoleSet.cpp b/src/Access/ExtendedRoleSet.cpp index adee5fea669..a29ee40380c 100644 --- a/src/Access/ExtendedRoleSet.cpp +++ b/src/Access/ExtendedRoleSet.cpp @@ -136,26 +136,6 @@ std::shared_ptr ExtendedRoleSet::toAST() const } -String ExtendedRoleSet::toString() const -{ - auto ast = toAST(); - return serializeAST(*ast); -} - - -Strings ExtendedRoleSet::toStrings() const -{ - if (all || !except_ids.empty()) - return {toString()}; - - Strings names; - names.reserve(ids.size()); - for (const UUID & id : ids) - names.emplace_back(::DB::toString(id)); - return names; -} - - std::shared_ptr ExtendedRoleSet::toASTWithNames(const AccessControlManager & manager) const { auto ast = std::make_shared(); @@ -189,6 +169,13 @@ std::shared_ptr ExtendedRoleSet::toASTWithNames(const Access } +String ExtendedRoleSet::toString() const +{ + auto ast = toAST(); + return serializeAST(*ast); +} + + String ExtendedRoleSet::toStringWithNames(const AccessControlManager & manager) const { auto ast = toASTWithNames(manager); @@ -198,19 +185,39 @@ String ExtendedRoleSet::toStringWithNames(const AccessControlManager & manager) Strings ExtendedRoleSet::toStringsWithNames(const AccessControlManager & manager) const { - if (all || !except_ids.empty()) - return {toStringWithNames(manager)}; + if (!all && ids.empty()) + return {}; - Strings names; - names.reserve(ids.size()); - for (const UUID & id : ids) + Strings res; + res.reserve(ids.size() + except_ids.size()); + + if (all) + res.emplace_back("ALL"); + else { - auto name = manager.tryReadName(id); - if (name) - names.emplace_back(std::move(*name)); + for (const UUID & id : ids) + { + auto name = manager.tryReadName(id); + if (name) + res.emplace_back(std::move(*name)); + } + std::sort(res.begin(), res.end()); } - boost::range::sort(names); - return names; + + if (!except_ids.empty()) + { + res.emplace_back("EXCEPT"); + size_t old_size = res.size(); + for (const UUID & id : except_ids) + { + auto name = manager.tryReadName(id); + if (name) + res.emplace_back(std::move(*name)); + } + std::sort(res.begin() + old_size, res.end()); + } + + return res; } @@ -247,25 +254,6 @@ bool ExtendedRoleSet::match(const UUID & id) const } -bool ExtendedRoleSet::match(const UUID & user_id, const std::vector & enabled_roles) const -{ - if (!all && !ids.count(user_id)) - { - bool found_enabled_role = std::any_of( - enabled_roles.begin(), enabled_roles.end(), [this](const UUID & enabled_role) { return ids.count(enabled_role); }); - if (!found_enabled_role) - return false; - } - - if (except_ids.count(user_id)) - return false; - - bool in_except_list = std::any_of( - enabled_roles.begin(), enabled_roles.end(), [this](const UUID & enabled_role) { return except_ids.count(enabled_role); }); - return !in_except_list; -} - - bool ExtendedRoleSet::match(const UUID & user_id, const boost::container::flat_set & enabled_roles) const { if (!all && !ids.count(user_id)) diff --git a/src/Access/ExtendedRoleSet.h b/src/Access/ExtendedRoleSet.h index ba84ade49f8..eeb4af84f78 100644 --- a/src/Access/ExtendedRoleSet.h +++ b/src/Access/ExtendedRoleSet.h @@ -36,10 +36,9 @@ struct ExtendedRoleSet ExtendedRoleSet(const ASTExtendedRoleSet & ast, const AccessControlManager & manager, const std::optional & current_user_id); std::shared_ptr toAST() const; - String toString() const; - Strings toStrings() const; - std::shared_ptr toASTWithNames(const AccessControlManager & manager) const; + + String toString() const; String toStringWithNames(const AccessControlManager & manager) const; Strings toStringsWithNames(const AccessControlManager & manager) const; @@ -50,7 +49,6 @@ struct ExtendedRoleSet /// Checks if a specified ID matches this ExtendedRoleSet. bool match(const UUID & id) const; - bool match(const UUID & user_id, const std::vector & enabled_roles) const; bool match(const UUID & user_id, const boost::container::flat_set & enabled_roles) const; /// Returns a list of matching IDs. The function must not be called if `all` == `true`. diff --git a/src/Access/QuotaCache.cpp b/src/Access/QuotaCache.cpp index dca844bf746..2939d2d1ef2 100644 --- a/src/Access/QuotaCache.cpp +++ b/src/Access/QuotaCache.cpp @@ -167,12 +167,7 @@ QuotaCache::QuotaCache(const AccessControlManager & access_control_manager_) QuotaCache::~QuotaCache() = default; -std::shared_ptr QuotaCache::getEnabledQuota( - const UUID & user_id, - const String & user_name, - const std::vector & enabled_roles, - const Poco::Net::IPAddress & client_address, - const String & client_key) +std::shared_ptr QuotaCache::getEnabledQuota(const UUID & user_id, const String & user_name, const boost::container::flat_set & enabled_roles, const Poco::Net::IPAddress & client_address, const String & client_key) { std::lock_guard lock{mutex}; ensureAllQuotasRead(); diff --git a/src/Access/QuotaCache.h b/src/Access/QuotaCache.h index 8399c5f73eb..b9996ed8456 100644 --- a/src/Access/QuotaCache.h +++ b/src/Access/QuotaCache.h @@ -20,13 +20,7 @@ public: QuotaCache(const AccessControlManager & access_control_manager_); ~QuotaCache(); - std::shared_ptr getEnabledQuota( - const UUID & user_id, - const String & user_name, - const std::vector & enabled_roles, - const Poco::Net::IPAddress & address, - const String & client_key); - + std::shared_ptr getEnabledQuota(const UUID & user_id, const String & user_name, const boost::container::flat_set & enabled_roles, const Poco::Net::IPAddress & address, const String & client_key); std::vector getUsageInfo() const; private: diff --git a/src/Access/RoleCache.cpp b/src/Access/RoleCache.cpp index 441e3844a07..ca8065145f3 100644 --- a/src/Access/RoleCache.cpp +++ b/src/Access/RoleCache.cpp @@ -2,68 +2,56 @@ #include #include #include -#include +#include namespace DB { namespace { - struct CollectedRoleInfo - { - RolePtr role; - bool is_current_role = false; - bool with_admin_option = false; - }; - - - void collectRoles(boost::container::flat_map & collected_roles, + void collectRoles(EnabledRolesInfo & roles_info, + boost::container::flat_set & skip_ids, const std::function & get_role_function, const UUID & role_id, bool is_current_role, bool with_admin_option) { - auto it = collected_roles.find(role_id); - if (it != collected_roles.end()) + if (roles_info.enabled_roles.count(role_id)) { - it->second.is_current_role |= is_current_role; - it->second.with_admin_option |= with_admin_option; + if (is_current_role) + roles_info.current_roles.emplace(role_id); + if (with_admin_option) + roles_info.enabled_roles_with_admin_option.emplace(role_id); return; } + if (skip_ids.count(role_id)) + return; + auto role = get_role_function(role_id); - collected_roles[role_id] = CollectedRoleInfo{role, is_current_role, with_admin_option}; if (!role) + { + skip_ids.emplace(role_id); return; + } + + roles_info.enabled_roles.emplace(role_id); + if (is_current_role) + roles_info.current_roles.emplace(role_id); + if (with_admin_option) + roles_info.enabled_roles_with_admin_option.emplace(role_id); + + roles_info.names_of_roles[role_id] = role->getName(); + roles_info.access.merge(role->access.access); + roles_info.access_with_grant_option.merge(role->access.access_with_grant_option); + roles_info.settings_from_enabled_roles.merge(role->settings); for (const auto & granted_role : role->granted_roles.roles) - collectRoles(collected_roles, get_role_function, granted_role, false, false); + collectRoles(roles_info, skip_ids, get_role_function, granted_role, false, false); for (const auto & granted_role : role->granted_roles.roles_with_admin_option) - collectRoles(collected_roles, get_role_function, granted_role, false, true); - } - - - std::shared_ptr collectInfoForRoles(const boost::container::flat_map & roles) - { - auto new_info = std::make_shared(); - for (const auto & [role_id, collect_info] : roles) - { - const auto & role = collect_info.role; - if (!role) - continue; - if (collect_info.is_current_role) - new_info->current_roles.emplace_back(role_id); - new_info->enabled_roles.emplace_back(role_id); - if (collect_info.with_admin_option) - new_info->enabled_roles_with_admin_option.emplace_back(role_id); - new_info->names_of_roles[role_id] = role->getName(); - new_info->access.merge(role->access.access); - new_info->access_with_grant_option.merge(role->access.access_with_grant_option); - new_info->settings_from_enabled_roles.merge(role->settings); - } - return new_info; + collectRoles(roles_info, skip_ids, get_role_function, granted_role, false, true); } } @@ -75,8 +63,8 @@ RoleCache::RoleCache(const AccessControlManager & manager_) RoleCache::~RoleCache() = default; -std::shared_ptr RoleCache::getEnabledRoles( - const std::vector & roles, const std::vector & roles_with_admin_option) +std::shared_ptr +RoleCache::getEnabledRoles(const boost::container::flat_set & roles, const boost::container::flat_set & roles_with_admin_option) { std::lock_guard lock{mutex}; @@ -93,13 +81,13 @@ std::shared_ptr RoleCache::getEnabledRoles( } auto res = std::shared_ptr(new EnabledRoles(params)); - collectRolesInfoFor(*res); + collectEnabledRoles(*res); enabled_roles.emplace(std::move(params), res); return res; } -void RoleCache::collectRolesInfo() +void RoleCache::collectEnabledRoles() { /// `mutex` is already locked. @@ -110,28 +98,29 @@ void RoleCache::collectRolesInfo() i = enabled_roles.erase(i); else { - collectRolesInfoFor(*elem); + collectEnabledRoles(*elem); ++i; } } } -void RoleCache::collectRolesInfoFor(EnabledRoles & enabled) +void RoleCache::collectEnabledRoles(EnabledRoles & enabled) { /// `mutex` is already locked. - /// Collect roles in use. That includes the current roles, the roles granted to the current roles, and so on. - boost::container::flat_map collected_roles; + /// Collect enabled roles. That includes the current roles, the roles granted to the current roles, and so on. + auto new_info = std::make_shared(); + boost::container::flat_set skip_ids; auto get_role_function = [this](const UUID & id) { return getRole(id); }; for (const auto & current_role : enabled.params.current_roles) - collectRoles(collected_roles, get_role_function, current_role, true, false); + collectRoles(*new_info, skip_ids, get_role_function, current_role, true, false); for (const auto & current_role : enabled.params.current_roles_with_admin_option) - collectRoles(collected_roles, get_role_function, current_role, true, true); + collectRoles(*new_info, skip_ids, get_role_function, current_role, true, true); /// Collect data from the collected roles. - enabled.setRolesInfo(collectInfoForRoles(collected_roles)); + enabled.setRolesInfo(new_info); } @@ -174,7 +163,7 @@ void RoleCache::roleChanged(const UUID & role_id, const RolePtr & changed_role) return; role_from_cache->first = changed_role; cache.update(role_id, role_from_cache); - collectRolesInfo(); + collectEnabledRoles(); } @@ -182,7 +171,7 @@ void RoleCache::roleRemoved(const UUID & role_id) { std::lock_guard lock{mutex}; cache.remove(role_id); - collectRolesInfo(); + collectEnabledRoles(); } } diff --git a/src/Access/RoleCache.h b/src/Access/RoleCache.h index 69f4cb2ebe8..f1ef783b5c4 100644 --- a/src/Access/RoleCache.h +++ b/src/Access/RoleCache.h @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -18,11 +19,12 @@ public: RoleCache(const AccessControlManager & manager_); ~RoleCache(); - std::shared_ptr getEnabledRoles(const std::vector & current_roles, const std::vector & current_roles_with_admin_option); + std::shared_ptr getEnabledRoles( + const boost::container::flat_set & current_roles, const boost::container::flat_set & current_roles_with_admin_option); private: - void collectRolesInfo(); - void collectRolesInfoFor(EnabledRoles & enabled); + void collectEnabledRoles(); + void collectEnabledRoles(EnabledRoles & enabled); RolePtr getRole(const UUID & role_id); void roleChanged(const UUID & role_id, const RolePtr & changed_role); void roleRemoved(const UUID & role_id); diff --git a/src/Access/RowPolicyCache.cpp b/src/Access/RowPolicyCache.cpp index 0c9fea69dc4..06ccf021dad 100644 --- a/src/Access/RowPolicyCache.cpp +++ b/src/Access/RowPolicyCache.cpp @@ -99,7 +99,7 @@ RowPolicyCache::RowPolicyCache(const AccessControlManager & access_control_manag RowPolicyCache::~RowPolicyCache() = default; -std::shared_ptr RowPolicyCache::getEnabledRowPolicies(const UUID & user_id, const std::vector & enabled_roles) +std::shared_ptr RowPolicyCache::getEnabledRowPolicies(const UUID & user_id, const boost::container::flat_set & enabled_roles) { std::lock_guard lock{mutex}; ensureAllRowPoliciesRead(); diff --git a/src/Access/RowPolicyCache.h b/src/Access/RowPolicyCache.h index d0ec74b9ab8..3b4ecd6e8f5 100644 --- a/src/Access/RowPolicyCache.h +++ b/src/Access/RowPolicyCache.h @@ -18,7 +18,7 @@ public: RowPolicyCache(const AccessControlManager & access_control_manager_); ~RowPolicyCache(); - std::shared_ptr getEnabledRowPolicies(const UUID & user_id, const std::vector & enabled_roles); + std::shared_ptr getEnabledRowPolicies(const UUID & user_id, const boost::container::flat_set & enabled_roles); private: using ParsedConditions = EnabledRowPolicies::ParsedConditions; diff --git a/src/Access/SettingsProfilesCache.cpp b/src/Access/SettingsProfilesCache.cpp index 516b47213fe..3f9399ca484 100644 --- a/src/Access/SettingsProfilesCache.cpp +++ b/src/Access/SettingsProfilesCache.cpp @@ -183,7 +183,7 @@ void SettingsProfilesCache::substituteProfiles(SettingsProfileElements & element std::shared_ptr SettingsProfilesCache::getEnabledSettings( const UUID & user_id, const SettingsProfileElements & settings_from_user, - const std::vector & enabled_roles, + const boost::container::flat_set & enabled_roles, const SettingsProfileElements & settings_from_enabled_roles) { std::lock_guard lock{mutex}; diff --git a/src/Access/SettingsProfilesCache.h b/src/Access/SettingsProfilesCache.h index 656ffc6fce6..42dd05df351 100644 --- a/src/Access/SettingsProfilesCache.h +++ b/src/Access/SettingsProfilesCache.h @@ -29,7 +29,7 @@ public: std::shared_ptr getEnabledSettings( const UUID & user_id, const SettingsProfileElements & settings_from_user_, - const std::vector & enabled_roles, + const boost::container::flat_set & enabled_roles, const SettingsProfileElements & settings_from_enabled_roles_); std::shared_ptr getProfileSettings(const String & profile_name); diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index f5651a1ab77..d6176c50ba4 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -698,7 +699,7 @@ std::optional Context::getUserID() const } -void Context::setCurrentRoles(const std::vector & current_roles_) +void Context::setCurrentRoles(const boost::container::flat_set & current_roles_) { auto lock = getLock(); if (current_roles == current_roles_ && !use_default_roles) @@ -718,24 +719,19 @@ void Context::setCurrentRolesDefault() calculateAccessRights(); } -std::vector Context::getCurrentRoles() const +boost::container::flat_set Context::getCurrentRoles() const { - return getAccess()->getCurrentRoles(); + return getRolesInfo()->current_roles; } -Strings Context::getCurrentRolesNames() const +boost::container::flat_set Context::getEnabledRoles() const { - return getAccess()->getCurrentRolesNames(); + return getRolesInfo()->enabled_roles; } -std::vector Context::getEnabledRoles() const +std::shared_ptr Context::getRolesInfo() const { - return getAccess()->getEnabledRoles(); -} - -Strings Context::getEnabledRolesNames() const -{ - return getAccess()->getEnabledRolesNames(); + return getAccess()->getRolesInfo(); } diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 0d2b3cdb5af..6cb2893af98 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -51,6 +51,7 @@ class Context; class ContextAccess; struct User; using UserPtr = std::shared_ptr; +struct EnabledRolesInfo; class EnabledRowPolicies; class EnabledQuota; class AccessFlags; @@ -166,7 +167,7 @@ private: InputBlocksReader input_blocks_reader; std::optional user_id; - std::vector current_roles; + boost::container::flat_set current_roles; bool use_default_roles = false; std::shared_ptr access; std::shared_ptr initial_row_policy; @@ -254,12 +255,11 @@ public: String getUserName() const; std::optional getUserID() const; - void setCurrentRoles(const std::vector & current_roles_); + void setCurrentRoles(const boost::container::flat_set & current_roles_); void setCurrentRolesDefault(); - std::vector getCurrentRoles() const; - Strings getCurrentRolesNames() const; - std::vector getEnabledRoles() const; - Strings getEnabledRolesNames() const; + boost::container::flat_set getCurrentRoles() const; + boost::container::flat_set getEnabledRoles() const; + std::shared_ptr getRolesInfo() const; /// Checks access rights. /// Empty database means the current database. diff --git a/src/Interpreters/InterpreterSetRoleQuery.cpp b/src/Interpreters/InterpreterSetRoleQuery.cpp index f56926332ec..f8e0167d748 100644 --- a/src/Interpreters/InterpreterSetRoleQuery.cpp +++ b/src/Interpreters/InterpreterSetRoleQuery.cpp @@ -39,20 +39,20 @@ void InterpreterSetRoleQuery::setRole(const ASTSetRoleQuery & query) else { ExtendedRoleSet roles_from_query{*query.roles, access_control}; - std::vector new_current_roles; + boost::container::flat_set new_current_roles; if (roles_from_query.all) { for (const auto & id : user->granted_roles.roles) if (roles_from_query.match(id)) - new_current_roles.push_back(id); + new_current_roles.emplace(id); } else { for (const auto & id : roles_from_query.getMatchingIDs()) { - if (!user->granted_roles.roles.contains(id)) + if (!user->granted_roles.roles.count(id)) throw Exception("Role should be granted to set current", ErrorCodes::SET_NON_GRANTED_ROLE); - new_current_roles.push_back(id); + new_current_roles.emplace(id); } } session_context.setCurrentRoles(new_current_roles); @@ -85,7 +85,7 @@ void InterpreterSetRoleQuery::updateUserSetDefaultRoles(User & user, const Exten { for (const auto & id : roles_from_query.getMatchingIDs()) { - if (!user.granted_roles.roles.contains(id)) + if (!user.granted_roles.roles.count(id)) throw Exception("Role should be granted to set default", ErrorCodes::SET_NON_GRANTED_ROLE); } } From 6f15a0d4435afac12760418ded60fdd1a11c287b Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Sat, 2 May 2020 19:05:01 +0300 Subject: [PATCH 179/738] Improve the function range() to allow iterating through enum values. --- base/ext/range.h | 80 ++++++++++++------- src/Access/EnabledQuota.cpp | 2 +- src/Access/EnabledQuota.h | 2 +- src/Access/EnabledRowPolicies.h | 3 +- src/Access/Quota.cpp | 9 ++- src/Access/Quota.h | 21 +++-- src/Access/QuotaCache.cpp | 6 +- src/Access/QuotaUsageInfo.h | 2 +- src/Access/RowPolicy.cpp | 12 ++- src/Access/RowPolicy.h | 3 +- src/Access/RowPolicyCache.cpp | 4 +- ...InterpreterShowCreateAccessEntityQuery.cpp | 2 +- .../InterpreterShowQuotasQuery.cpp | 2 +- src/Parsers/ASTCreateQuotaQuery.cpp | 2 +- src/Parsers/ASTCreateQuotaQuery.h | 2 +- src/Parsers/ParserCreateQuotaQuery.cpp | 6 +- .../System/StorageSystemQuotaUsage.cpp | 2 +- src/Storages/System/StorageSystemQuotas.cpp | 4 +- .../System/StorageSystemRowPolicies.cpp | 2 +- 19 files changed, 107 insertions(+), 59 deletions(-) diff --git a/base/ext/range.h b/base/ext/range.h index c379d453f7b..266016f5779 100644 --- a/base/ext/range.h +++ b/base/ext/range.h @@ -1,42 +1,62 @@ #pragma once -#include #include #include +#include namespace ext { - /// For loop adaptor which is used to iterate through a half-closed interval [begin, end). - template - inline auto range(BeginType begin, EndType end) +namespace internal +{ + template + auto rangeImpl(BeginType begin, EndType end) { - using CommonType = typename std::common_type::type; - return boost::counting_range(begin, end); - } - - template - inline auto range(Type end) - { - return range(static_cast(0), end); - } - - /// The same as range(), but every value is casted statically to a specified `ValueType`. - /// This is useful to iterate through all constants of a enum. - template - inline auto range_with_static_cast(BeginType begin, EndType end) - { - using CommonType = typename std::common_type::type; - if constexpr (std::is_same_v) - return boost::counting_range(begin, end); + if constexpr (std::is_same_v) + return boost::counting_range(static_cast(begin), static_cast(end)); else - return boost::counting_range(begin, end) - | boost::adaptors::transformed([](CommonType x) -> ValueType { return static_cast(x); }); - } - - template - inline auto range_with_static_cast(EndType end) - { - return range_with_static_cast(static_cast(0), end); + return boost::counting_range(static_cast(begin), static_cast(end)) + | boost::adaptors::transformed([](CountingType x) { return static_cast(x); }); } } + + +/// For loop adaptor which is used to iterate through a half-closed interval [begin, end). +/// The parameters `begin` and `end` can have any integral or enum types. +template || std::is_enum_v) && + (std::is_integral_v || std::is_enum_v) && + (!std::is_enum_v || !std::is_enum_v || std::is_same_v), void>> +inline auto range(BeginType begin, EndType end) +{ + if constexpr (std::is_integral_v && std::is_integral_v) + { + using CommonType = std::common_type_t; + return internal::rangeImpl(begin, end); + } + else if constexpr (std::is_enum_v) + { + return internal::rangeImpl>(begin, end); + } + else + { + return internal::rangeImpl>(begin, end); + } +} + + +/// For loop adaptor which is used to iterate through a half-closed interval [0, end). +/// The parameter `end` can have any integral or enum type. +/// The same as range(0, end). +template || std::is_enum_v, void>> +inline auto range(Type end) +{ + if constexpr (std::is_integral_v) + return internal::rangeImpl(0, end); + else + return internal::rangeImpl>(0, end); +} +} diff --git a/src/Access/EnabledQuota.cpp b/src/Access/EnabledQuota.cpp index 92257ce0002..6b3ec33ce6b 100644 --- a/src/Access/EnabledQuota.cpp +++ b/src/Access/EnabledQuota.cpp @@ -128,7 +128,7 @@ struct EnabledQuota::Impl const Intervals & intervals, std::chrono::system_clock::time_point current_time) { - for (auto resource_type : ext::range_with_static_cast(Quota::MAX_RESOURCE_TYPE)) + for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) checkExceeded(user_name, intervals, resource_type, current_time); } }; diff --git a/src/Access/EnabledQuota.h b/src/Access/EnabledQuota.h index e0a04b02206..965d7ea6da9 100644 --- a/src/Access/EnabledQuota.h +++ b/src/Access/EnabledQuota.h @@ -65,7 +65,7 @@ private: const String & getUserName() const { return params.user_name; } - static constexpr size_t MAX_RESOURCE_TYPE = Quota::MAX_RESOURCE_TYPE; + static constexpr auto MAX_RESOURCE_TYPE = Quota::MAX_RESOURCE_TYPE; struct Interval { diff --git a/src/Access/EnabledRowPolicies.h b/src/Access/EnabledRowPolicies.h index 6274850363c..5255b2a791c 100644 --- a/src/Access/EnabledRowPolicies.h +++ b/src/Access/EnabledRowPolicies.h @@ -58,8 +58,7 @@ private: { size_t operator()(const DatabaseAndTableNameRef & database_and_table_name) const; }; - static constexpr size_t MAX_CONDITION_TYPE = RowPolicy::MAX_CONDITION_TYPE; - using ParsedConditions = std::array; + using ParsedConditions = std::array; struct MixedConditions { std::unique_ptr database_and_table_name_keeper; diff --git a/src/Access/Quota.cpp b/src/Access/Quota.cpp index e3a9e11eb10..a5afe5fda4f 100644 --- a/src/Access/Quota.cpp +++ b/src/Access/Quota.cpp @@ -5,6 +5,12 @@ namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + + Quota::Limits::Limits() { boost::range::fill(max, 0); @@ -38,8 +44,9 @@ const char * Quota::resourceTypeToColumnName(ResourceType resource_type) case Quota::READ_ROWS: return "read_rows"; case Quota::READ_BYTES: return "read_bytes"; case Quota::EXECUTION_TIME: return "execution_time"; + case Quota::MAX_RESOURCE_TYPE: break; } - __builtin_unreachable(); + throw Exception("Unexpected resource type: " + std::to_string(static_cast(resource_type)), ErrorCodes::LOGICAL_ERROR); } } diff --git a/src/Access/Quota.h b/src/Access/Quota.h index 714d582e98f..a3666aa9b52 100644 --- a/src/Access/Quota.h +++ b/src/Access/Quota.h @@ -7,6 +7,12 @@ namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + + /** Quota for resources consumption for specific interval. * Used to limit resource usage by user. * Quota is applied "softly" - could be slightly exceed, because it is checked usually only on each block of processed data. @@ -26,8 +32,9 @@ struct Quota : public IAccessEntity READ_ROWS, /// Number of rows read from tables. READ_BYTES, /// Number of bytes read from tables. EXECUTION_TIME, /// Total amount of query execution time in nanoseconds. + + MAX_RESOURCE_TYPE }; - static constexpr size_t MAX_RESOURCE_TYPE = 7; using ResourceAmount = UInt64; static constexpr ResourceAmount UNLIMITED = 0; /// 0 means unlimited. @@ -58,8 +65,9 @@ struct Quota : public IAccessEntity CLIENT_KEY, /// Client should explicitly supply a key to use. CLIENT_KEY_OR_USER_NAME, /// Same as CLIENT_KEY, but use USER_NAME if the client doesn't supply a key. CLIENT_KEY_OR_IP_ADDRESS, /// Same as CLIENT_KEY, but use IP_ADDRESS if the client doesn't supply a key. + + MAX }; - static constexpr size_t MAX_KEY_TYPE = 6; KeyType key_type = KeyType::NONE; /// Which roles or users should use this quota. @@ -88,8 +96,9 @@ inline const char * Quota::getNameOfResourceType(ResourceType resource_type) case Quota::READ_ROWS: return "read rows"; case Quota::READ_BYTES: return "read bytes"; case Quota::EXECUTION_TIME: return "execution time"; + case Quota::MAX_RESOURCE_TYPE: break; } - __builtin_unreachable(); + throw Exception("Unexpected resource type: " + std::to_string(static_cast(resource_type)), ErrorCodes::LOGICAL_ERROR); } @@ -104,8 +113,9 @@ inline const char * Quota::resourceTypeToKeyword(ResourceType resource_type) case Quota::READ_ROWS: return "READ ROWS"; case Quota::READ_BYTES: return "READ BYTES"; case Quota::EXECUTION_TIME: return "EXECUTION TIME"; + case Quota::MAX_RESOURCE_TYPE: break; } - __builtin_unreachable(); + throw Exception("Unexpected resource type: " + std::to_string(static_cast(resource_type)), ErrorCodes::LOGICAL_ERROR); } @@ -119,8 +129,9 @@ inline const char * Quota::getNameOfKeyType(KeyType key_type) case KeyType::CLIENT_KEY: return "client key"; case KeyType::CLIENT_KEY_OR_USER_NAME: return "client key or user name"; case KeyType::CLIENT_KEY_OR_IP_ADDRESS: return "client key or ip address"; + case KeyType::MAX: break; } - __builtin_unreachable(); + throw Exception("Unexpected quota key type: " + std::to_string(static_cast(key_type)), ErrorCodes::LOGICAL_ERROR); } diff --git a/src/Access/QuotaCache.cpp b/src/Access/QuotaCache.cpp index 2939d2d1ef2..842106b3ba9 100644 --- a/src/Access/QuotaCache.cpp +++ b/src/Access/QuotaCache.cpp @@ -17,6 +17,7 @@ namespace DB namespace ErrorCodes { extern const int QUOTA_REQUIRES_CLIENT_KEY; + extern const int LOGICAL_ERROR; } @@ -72,8 +73,9 @@ String QuotaCache::QuotaInfo::calculateKey(const EnabledQuota & enabled) const return params.client_key; return params.client_address.toString(); } + case KeyType::MAX: break; } - __builtin_unreachable(); + throw Exception("Unexpected quota key type: " + std::to_string(static_cast(quota->key_type)), ErrorCodes::LOGICAL_ERROR); } @@ -101,7 +103,7 @@ boost::shared_ptr QuotaCache::QuotaInfo::rebuildI new_intervals->quota_key = key; auto & intervals = new_intervals->intervals; intervals.reserve(quota->all_limits.size()); - static constexpr size_t MAX_RESOURCE_TYPE = Quota::MAX_RESOURCE_TYPE; + static constexpr auto MAX_RESOURCE_TYPE = Quota::MAX_RESOURCE_TYPE; for (const auto & limits : quota->all_limits) { intervals.emplace_back(); diff --git a/src/Access/QuotaUsageInfo.h b/src/Access/QuotaUsageInfo.h index 94e16fb9f69..45ee7898145 100644 --- a/src/Access/QuotaUsageInfo.h +++ b/src/Access/QuotaUsageInfo.h @@ -11,7 +11,7 @@ struct QuotaUsageInfo { using ResourceType = Quota::ResourceType; using ResourceAmount = Quota::ResourceAmount; - static constexpr size_t MAX_RESOURCE_TYPE = Quota::MAX_RESOURCE_TYPE; + static constexpr auto MAX_RESOURCE_TYPE = Quota::MAX_RESOURCE_TYPE; struct Interval { diff --git a/src/Access/RowPolicy.cpp b/src/Access/RowPolicy.cpp index 65b9451a453..d087705da22 100644 --- a/src/Access/RowPolicy.cpp +++ b/src/Access/RowPolicy.cpp @@ -6,6 +6,12 @@ namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + + namespace { void generateFullNameImpl(const String & database_, const String & table_name_, const String & policy_name_, String & full_name_) @@ -90,8 +96,9 @@ const char * RowPolicy::conditionTypeToString(ConditionType index) case UPDATE_FILTER: return "UPDATE_FILTER"; case UPDATE_CHECK: return "UPDATE_CHECK"; case DELETE_FILTER: return "DELETE_FILTER"; + case MAX_CONDITION_TYPE: break; } - __builtin_unreachable(); + throw Exception("Unexpected condition type: " + std::to_string(static_cast(index)), ErrorCodes::LOGICAL_ERROR); } @@ -104,8 +111,9 @@ const char * RowPolicy::conditionTypeToColumnName(ConditionType index) case UPDATE_FILTER: return "update_filter"; case UPDATE_CHECK: return "update_check"; case DELETE_FILTER: return "delete_filter"; + case MAX_CONDITION_TYPE: break; } - __builtin_unreachable(); + throw Exception("Unexpected condition type: " + std::to_string(static_cast(index)), ErrorCodes::LOGICAL_ERROR); } } diff --git a/src/Access/RowPolicy.h b/src/Access/RowPolicy.h index 08219edb46b..71f8d14cdb4 100644 --- a/src/Access/RowPolicy.h +++ b/src/Access/RowPolicy.h @@ -44,8 +44,9 @@ struct RowPolicy : public IAccessEntity UPDATE_FILTER, UPDATE_CHECK, DELETE_FILTER, + + MAX_CONDITION_TYPE }; - static constexpr size_t MAX_CONDITION_TYPE = 5; static const char * conditionTypeToString(ConditionType index); static const char * conditionTypeToColumnName(ConditionType index); diff --git a/src/Access/RowPolicyCache.cpp b/src/Access/RowPolicyCache.cpp index 06ccf021dad..b342b735879 100644 --- a/src/Access/RowPolicyCache.cpp +++ b/src/Access/RowPolicyCache.cpp @@ -16,7 +16,7 @@ namespace DB namespace { using ConditionType = RowPolicy::ConditionType; - constexpr size_t MAX_CONDITION_TYPE = RowPolicy::MAX_CONDITION_TYPE; + constexpr auto MAX_CONDITION_TYPE = RowPolicy::MAX_CONDITION_TYPE; /// Accumulates conditions from multiple row policies and joins them using the AND logical operation. @@ -58,7 +58,7 @@ void RowPolicyCache::PolicyInfo::setPolicy(const RowPolicyPtr & policy_) policy = policy_; roles = &policy->to_roles; - for (auto type : ext::range_with_static_cast(0, MAX_CONDITION_TYPE)) + for (auto type : ext::range(0, MAX_CONDITION_TYPE)) { parsed_conditions[type] = nullptr; const String & condition = policy->conditions[type]; diff --git a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp index 0d3b88facce..469a2f1196b 100644 --- a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp +++ b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp @@ -166,7 +166,7 @@ namespace if (policy.isRestrictive()) query->is_restrictive = policy.isRestrictive(); - for (auto index : ext::range_with_static_cast(RowPolicy::MAX_CONDITION_TYPE)) + for (auto index : ext::range(RowPolicy::MAX_CONDITION_TYPE)) { const auto & condition = policy.conditions[index]; if (!condition.empty()) diff --git a/src/Interpreters/InterpreterShowQuotasQuery.cpp b/src/Interpreters/InterpreterShowQuotasQuery.cpp index 73653e26781..4500046577c 100644 --- a/src/Interpreters/InterpreterShowQuotasQuery.cpp +++ b/src/Interpreters/InterpreterShowQuotasQuery.cpp @@ -30,7 +30,7 @@ String InterpreterShowQuotasQuery::getRewrittenQuery() expr = "name || ' key=\\'' || key || '\\'' || if(isNull(end_of_interval), '', ' interval=[' || " "toString(end_of_interval - duration) || ' .. ' || " "toString(end_of_interval) || ']'"; - for (auto resource_type : ext::range_with_static_cast(Quota::MAX_RESOURCE_TYPE)) + for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) { String column_name = Quota::resourceTypeToColumnName(resource_type); expr += String{" || ' "} + column_name + "=' || toString(" + column_name + ")"; diff --git a/src/Parsers/ASTCreateQuotaQuery.cpp b/src/Parsers/ASTCreateQuotaQuery.cpp index edb510d1a2e..e990b9939bc 100644 --- a/src/Parsers/ASTCreateQuotaQuery.cpp +++ b/src/Parsers/ASTCreateQuotaQuery.cpp @@ -67,7 +67,7 @@ namespace else { bool limit_found = false; - for (auto resource_type : ext::range_with_static_cast(Quota::MAX_RESOURCE_TYPE)) + for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) { if (limits.max[resource_type]) { diff --git a/src/Parsers/ASTCreateQuotaQuery.h b/src/Parsers/ASTCreateQuotaQuery.h index b001ec44a0c..ad20348a139 100644 --- a/src/Parsers/ASTCreateQuotaQuery.h +++ b/src/Parsers/ASTCreateQuotaQuery.h @@ -42,7 +42,7 @@ public: using ResourceType = Quota::ResourceType; using ResourceAmount = Quota::ResourceAmount; - static constexpr size_t MAX_RESOURCE_TYPE = Quota::MAX_RESOURCE_TYPE; + static constexpr auto MAX_RESOURCE_TYPE = Quota::MAX_RESOURCE_TYPE; struct Limits { diff --git a/src/Parsers/ParserCreateQuotaQuery.cpp b/src/Parsers/ParserCreateQuotaQuery.cpp index 6007d6206ec..4d29a135131 100644 --- a/src/Parsers/ParserCreateQuotaQuery.cpp +++ b/src/Parsers/ParserCreateQuotaQuery.cpp @@ -48,7 +48,7 @@ namespace return false; const String & key_type_str = key_type_ast->as().value.safeGet(); - for (auto kt : ext::range_with_static_cast(Quota::MAX_KEY_TYPE)) + for (auto kt : ext::range(Quota::KeyType::MAX)) if (boost::iequals(Quota::getNameOfKeyType(kt), key_type_str)) { key_type = kt; @@ -56,7 +56,7 @@ namespace } String all_key_types_str; - for (auto kt : ext::range_with_static_cast(Quota::MAX_KEY_TYPE)) + for (auto kt : ext::range(Quota::KeyType::MAX)) all_key_types_str += String(all_key_types_str.empty() ? "" : ", ") + "'" + Quota::getNameOfKeyType(kt) + "'"; String msg = "Quota cannot be keyed by '" + key_type_str + "'. Expected one of these literals: " + all_key_types_str; throw Exception(msg, ErrorCodes::SYNTAX_ERROR); @@ -81,7 +81,7 @@ namespace } bool resource_type_set = false; - for (auto rt : ext::range_with_static_cast(Quota::MAX_RESOURCE_TYPE)) + for (auto rt : ext::range(Quota::MAX_RESOURCE_TYPE)) { if (ParserKeyword{Quota::resourceTypeToKeyword(rt)}.ignore(pos, expected)) { diff --git a/src/Storages/System/StorageSystemQuotaUsage.cpp b/src/Storages/System/StorageSystemQuotaUsage.cpp index 1f943d02446..27400329b27 100644 --- a/src/Storages/System/StorageSystemQuotaUsage.cpp +++ b/src/Storages/System/StorageSystemQuotaUsage.cpp @@ -23,7 +23,7 @@ NamesAndTypesList StorageSystemQuotaUsage::getNamesAndTypes() {"duration", std::make_shared(std::make_shared())}, {"end_of_interval", std::make_shared(std::make_shared())}}; - for (auto resource_type : ext::range_with_static_cast(Quota::MAX_RESOURCE_TYPE)) + for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) { DataTypePtr data_type; if (resource_type == Quota::EXECUTION_TIME) diff --git a/src/Storages/System/StorageSystemQuotas.cpp b/src/Storages/System/StorageSystemQuotas.cpp index a22bb11bbc3..1da7725fe15 100644 --- a/src/Storages/System/StorageSystemQuotas.cpp +++ b/src/Storages/System/StorageSystemQuotas.cpp @@ -20,7 +20,7 @@ namespace DataTypeEnum8::Values getKeyTypeEnumValues() { DataTypeEnum8::Values enum_values; - for (auto key_type : ext::range_with_static_cast(Quota::MAX_KEY_TYPE)) + for (auto key_type : ext::range(Quota::KeyType::MAX)) enum_values.push_back({Quota::getNameOfKeyType(key_type), static_cast(key_type)}); return enum_values; } @@ -38,7 +38,7 @@ NamesAndTypesList StorageSystemQuotas::getNamesAndTypes() {"intervals.duration", std::make_shared(std::make_shared())}, {"intervals.randomize_interval", std::make_shared(std::make_shared())}}; - for (auto resource_type : ext::range_with_static_cast(Quota::MAX_RESOURCE_TYPE)) + for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) { DataTypePtr data_type; if (resource_type == Quota::EXECUTION_TIME) diff --git a/src/Storages/System/StorageSystemRowPolicies.cpp b/src/Storages/System/StorageSystemRowPolicies.cpp index 12221cc52de..39fdf926f92 100644 --- a/src/Storages/System/StorageSystemRowPolicies.cpp +++ b/src/Storages/System/StorageSystemRowPolicies.cpp @@ -25,7 +25,7 @@ NamesAndTypesList StorageSystemRowPolicies::getNamesAndTypes() {"restrictive", std::make_shared()}, }; - for (auto index : ext::range_with_static_cast(RowPolicy::MAX_CONDITION_TYPE)) + for (auto index : ext::range(RowPolicy::MAX_CONDITION_TYPE)) names_and_types.push_back({RowPolicy::conditionTypeToColumnName(index), std::make_shared()}); return names_and_types; From b6fe72677790fae84a38f5a30369bf75b87ff8b1 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Sun, 3 May 2020 01:30:28 +0300 Subject: [PATCH 180/738] Rename row policy's 'name' to 'short_name', 'full_name' to 'name'. This change simplifies the interface of IAccesEntity. --- src/Access/DiskAccessStorage.cpp | 8 +- src/Access/IAccessEntity.cpp | 2 +- src/Access/IAccessEntity.h | 7 +- src/Access/IAccessStorage.h | 2 +- src/Access/MemoryAccessStorage.cpp | 20 ++--- src/Access/RowPolicy.cpp | 75 ++++++++----------- src/Access/RowPolicy.h | 39 +++++----- src/Access/RowPolicyCache.cpp | 2 +- src/Access/UsersConfigAccessStorage.cpp | 10 +-- src/Functions/currentRowPolicies.cpp | 4 +- .../InterpreterCreateRowPolicyQuery.cpp | 37 +++------ .../InterpreterDropAccessEntityQuery.cpp | 17 +++-- ...InterpreterShowCreateAccessEntityQuery.cpp | 10 ++- .../InterpreterShowCreateAccessEntityQuery.h | 2 +- .../InterpreterShowRowPoliciesQuery.cpp | 2 +- src/Parsers/ASTCreateRowPolicyQuery.cpp | 12 +-- src/Parsers/ASTCreateRowPolicyQuery.h | 4 +- src/Parsers/ASTDropAccessEntityQuery.cpp | 10 +-- src/Parsers/ASTDropAccessEntityQuery.h | 2 +- .../ASTShowCreateAccessEntityQuery.cpp | 8 +- src/Parsers/ASTShowCreateAccessEntityQuery.h | 2 +- src/Parsers/ParserCreateRowPolicyQuery.cpp | 16 ++-- src/Parsers/ParserDropAccessEntityQuery.cpp | 20 ++--- .../ParserShowCreateAccessEntityQuery.cpp | 12 +-- .../System/StorageSystemRowPolicies.cpp | 4 +- 25 files changed, 153 insertions(+), 174 deletions(-) diff --git a/src/Access/DiskAccessStorage.cpp b/src/Access/DiskAccessStorage.cpp index bf4316cc195..df9cbab891f 100644 --- a/src/Access/DiskAccessStorage.cpp +++ b/src/Access/DiskAccessStorage.cpp @@ -525,7 +525,7 @@ bool DiskAccessStorage::rebuildLists() auto type = entity->getType(); auto & name_to_id_map = name_to_id_maps.at(type); - auto it_by_name = name_to_id_map.emplace(entity->getFullName(), id).first; + auto it_by_name = name_to_id_map.emplace(entity->getName(), id).first; id_to_entry_map.emplace(id, Entry{it_by_name->first, type}); } @@ -609,7 +609,7 @@ UUID DiskAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool repl void DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, Notifications & notifications) { - const String & name = new_entity->getFullName(); + const String & name = new_entity->getName(); std::type_index type = new_entity->getType(); if (!initialized) throw Exception( @@ -622,7 +622,7 @@ void DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & ne if (it_by_id != id_to_entry_map.end()) { const auto & existing_entry = it_by_id->second; - throwIDCollisionCannotInsert(id, type, name, existing_entry.entity->getType(), existing_entry.entity->getFullName()); + throwIDCollisionCannotInsert(id, type, name, existing_entry.entity->getType(), existing_entry.entity->getName()); } auto & name_to_id_map = name_to_id_maps.at(type); @@ -703,7 +703,7 @@ void DiskAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & update_ if (*new_entity == *old_entity) return; - String new_name = new_entity->getFullName(); + String new_name = new_entity->getName(); auto old_name = entry.name; const std::type_index type = entry.type; bool name_changed = (new_name != old_name); diff --git a/src/Access/IAccessEntity.cpp b/src/Access/IAccessEntity.cpp index 5dbc056b71c..e5e484c4c33 100644 --- a/src/Access/IAccessEntity.cpp +++ b/src/Access/IAccessEntity.cpp @@ -43,6 +43,6 @@ const char * IAccessEntity::getKeyword(std::type_index type) bool IAccessEntity::equal(const IAccessEntity & other) const { - return (full_name == other.full_name) && (getType() == other.getType()); + return (name == other.name) && (getType() == other.getType()); } } diff --git a/src/Access/IAccessEntity.h b/src/Access/IAccessEntity.h index 9214d64aa8c..9a4e80fbde9 100644 --- a/src/Access/IAccessEntity.h +++ b/src/Access/IAccessEntity.h @@ -27,15 +27,14 @@ struct IAccessEntity bool isTypeOf() const { return isTypeOf(typeid(EntityType)); } bool isTypeOf(std::type_index type) const { return type == getType(); } - virtual void setName(const String & name_) { full_name = name_; } - virtual String getName() const { return full_name; } - String getFullName() const { return full_name; } + virtual void setName(const String & name_) { name = name_; } + const String & getName() const { return name; } friend bool operator ==(const IAccessEntity & lhs, const IAccessEntity & rhs) { return lhs.equal(rhs); } friend bool operator !=(const IAccessEntity & lhs, const IAccessEntity & rhs) { return !(lhs == rhs); } protected: - String full_name; + String name; virtual bool equal(const IAccessEntity & other) const; diff --git a/src/Access/IAccessStorage.h b/src/Access/IAccessStorage.h index 30a1a6bdc32..d2413abc4a5 100644 --- a/src/Access/IAccessStorage.h +++ b/src/Access/IAccessStorage.h @@ -179,7 +179,7 @@ std::shared_ptr IAccessStorage::read(const UUID & id) const auto ptr = typeid_cast>(entity); if (ptr) return ptr; - throwBadCast(id, entity->getType(), entity->getFullName(), typeid(EntityType)); + throwBadCast(id, entity->getType(), entity->getName(), typeid(EntityType)); } diff --git a/src/Access/MemoryAccessStorage.cpp b/src/Access/MemoryAccessStorage.cpp index ce668bcd8b3..46efd25191b 100644 --- a/src/Access/MemoryAccessStorage.cpp +++ b/src/Access/MemoryAccessStorage.cpp @@ -55,7 +55,7 @@ AccessEntityPtr MemoryAccessStorage::readImpl(const UUID & id) const String MemoryAccessStorage::readNameImpl(const UUID & id) const { - return readImpl(id)->getFullName(); + return readImpl(id)->getName(); } @@ -73,7 +73,7 @@ UUID MemoryAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool re void MemoryAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, Notifications & notifications) { - const String & name = new_entity->getFullName(); + const String & name = new_entity->getName(); std::type_index type = new_entity->getType(); /// Check that we can insert. @@ -81,7 +81,7 @@ void MemoryAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & if (it != entries.end()) { const auto & existing_entry = it->second; - throwIDCollisionCannotInsert(id, type, name, existing_entry.entity->getType(), existing_entry.entity->getFullName()); + throwIDCollisionCannotInsert(id, type, name, existing_entry.entity->getType(), existing_entry.entity->getName()); } auto it2 = names.find({name, type}); @@ -120,7 +120,7 @@ void MemoryAccessStorage::removeNoLock(const UUID & id, Notifications & notifica throwNotFound(id); Entry & entry = it->second; - const String & name = entry.entity->getFullName(); + const String & name = entry.entity->getName(); std::type_index type = entry.entity->getType(); prepareNotifications(entry, true, notifications); @@ -156,14 +156,14 @@ void MemoryAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & updat entry.entity = new_entity; - if (new_entity->getFullName() != old_entity->getFullName()) + if (new_entity->getName() != old_entity->getName()) { - auto it2 = names.find({new_entity->getFullName(), new_entity->getType()}); + auto it2 = names.find({new_entity->getName(), new_entity->getType()}); if (it2 != names.end()) - throwNameCollisionCannotRename(old_entity->getType(), old_entity->getFullName(), new_entity->getFullName()); + throwNameCollisionCannotRename(old_entity->getType(), old_entity->getName(), new_entity->getName()); - names.erase({old_entity->getFullName(), old_entity->getType()}); - names[std::pair{new_entity->getFullName(), new_entity->getType()}] = &entry; + names.erase({old_entity->getName(), old_entity->getType()}); + names[std::pair{new_entity->getName(), new_entity->getType()}] = &entry; } prepareNotifications(entry, false, notifications); @@ -211,7 +211,7 @@ void MemoryAccessStorage::setAllNoLock(const std::vectorgetFullName(), entity->getType()}); + auto it2 = names.find({entity->getName(), entity->getType()}); if (it2 != names.end()) { Entry & entry = *(it2->second); diff --git a/src/Access/RowPolicy.cpp b/src/Access/RowPolicy.cpp index d087705da22..bb3fc54a590 100644 --- a/src/Access/RowPolicy.cpp +++ b/src/Access/RowPolicy.cpp @@ -1,5 +1,4 @@ #include -#include #include #include @@ -8,71 +7,62 @@ namespace DB { namespace ErrorCodes { + extern const int NOT_IMPLEMENTED; extern const int LOGICAL_ERROR; } -namespace +String RowPolicy::NameParts::getName() const { - void generateFullNameImpl(const String & database_, const String & table_name_, const String & policy_name_, String & full_name_) + String name; + name.reserve(database.length() + table_name.length() + short_name.length() + 6); + name += backQuoteIfNeed(short_name); + name += " ON "; + if (!name.empty()) { - full_name_.clear(); - full_name_.reserve(database_.length() + table_name_.length() + policy_name_.length() + 6); - full_name_ += backQuoteIfNeed(policy_name_); - full_name_ += " ON "; - if (!database_.empty()) - { - full_name_ += backQuoteIfNeed(database_); - full_name_ += '.'; - } - full_name_ += backQuoteIfNeed(table_name_); + name += backQuoteIfNeed(database); + name += '.'; } + name += backQuoteIfNeed(table_name); + return name; } -String RowPolicy::FullNameParts::getFullName() const +void RowPolicy::setDatabase(const String & database) { - String full_name; - generateFullNameImpl(database, table_name, policy_name, full_name); - return full_name; + name_parts.database = database; + IAccessEntity::setName(name_parts.getName()); } - -String RowPolicy::FullNameParts::getFullName(const Context & context) const +void RowPolicy::setTableName(const String & table_name) { - String full_name; - generateFullNameImpl(database.empty() ? context.getCurrentDatabase() : database, table_name, policy_name, full_name); - return full_name; + name_parts.table_name = table_name; + IAccessEntity::setName(name_parts.getName()); } - -void RowPolicy::setDatabase(const String & database_) +void RowPolicy::setShortName(const String & short_name) { - database = database_; - generateFullNameImpl(database, table_name, policy_name, full_name); + name_parts.short_name = short_name; + IAccessEntity::setName(name_parts.getName()); } - -void RowPolicy::setTableName(const String & table_name_) +void RowPolicy::setNameParts(const String & short_name, const String & database, const String & table_name) { - table_name = table_name_; - generateFullNameImpl(database, table_name, policy_name, full_name); + name_parts.short_name = short_name; + name_parts.database = database; + name_parts.table_name = table_name; + IAccessEntity::setName(name_parts.getName()); } - -void RowPolicy::setName(const String & policy_name_) +void RowPolicy::setNameParts(const NameParts & name_parts_) { - policy_name = policy_name_; - generateFullNameImpl(database, table_name, policy_name, full_name); + name_parts = name_parts_; + IAccessEntity::setName(name_parts.getName()); } - -void RowPolicy::setFullName(const String & database_, const String & table_name_, const String & policy_name_) +void RowPolicy::setName(const String &) { - database = database_; - table_name = table_name_; - policy_name = policy_name_; - generateFullNameImpl(database, table_name, policy_name, full_name); + throw Exception("RowPolicy::setName() is not implemented", ErrorCodes::NOT_IMPLEMENTED); } @@ -81,9 +71,8 @@ bool RowPolicy::equal(const IAccessEntity & other) const if (!IAccessEntity::equal(other)) return false; const auto & other_policy = typeid_cast(other); - return (database == other_policy.database) && (table_name == other_policy.table_name) && (policy_name == other_policy.policy_name) - && boost::range::equal(conditions, other_policy.conditions) && restrictive == other_policy.restrictive - && (to_roles == other_policy.to_roles); + return (name_parts == other_policy.name_parts) && boost::range::equal(conditions, other_policy.conditions) + && restrictive == other_policy.restrictive && (to_roles == other_policy.to_roles); } diff --git a/src/Access/RowPolicy.h b/src/Access/RowPolicy.h index 71f8d14cdb4..b3d490e2dbe 100644 --- a/src/Access/RowPolicy.h +++ b/src/Access/RowPolicy.h @@ -6,31 +6,34 @@ namespace DB { -class Context; - /** Represents a row level security policy for a table. */ struct RowPolicy : public IAccessEntity { - void setDatabase(const String & database_); - void setTableName(const String & table_name_); - void setName(const String & policy_name_) override; - void setFullName(const String & database_, const String & table_name_, const String & policy_name_); - - String getDatabase() const { return database; } - String getTableName() const { return table_name; } - String getName() const override { return policy_name; } - - struct FullNameParts + struct NameParts { + String short_name; String database; String table_name; - String policy_name; - String getFullName() const; - String getFullName(const Context & context) const; + + String getName() const; + auto toTuple() const { return std::tie(short_name, database, table_name); } + friend bool operator ==(const NameParts & left, const NameParts & right) { return left.toTuple() == right.toTuple(); } + friend bool operator !=(const NameParts & left, const NameParts & right) { return left.toTuple() != right.toTuple(); } }; + void setShortName(const String & short_name); + void setDatabase(const String & database); + void setTableName(const String & table_name); + void setNameParts(const String & short_name, const String & database, const String & table_name); + void setNameParts(const NameParts & name_parts); + + const String & getDatabase() const { return name_parts.database; } + const String & getTableName() const { return name_parts.table_name; } + const String & getShortName() const { return name_parts.short_name; } + const NameParts & getNameParts() const { return name_parts; } + /// Filter is a SQL conditional expression used to figure out which rows should be visible /// for user or available for modification. If the expression returns NULL or false for some rows /// those rows are silently suppressed. @@ -71,9 +74,9 @@ struct RowPolicy : public IAccessEntity ExtendedRoleSet to_roles; private: - String database; - String table_name; - String policy_name; + void setName(const String & name_) override; + + NameParts name_parts; bool restrictive = false; }; diff --git a/src/Access/RowPolicyCache.cpp b/src/Access/RowPolicyCache.cpp index b342b735879..bc3a2b9a15e 100644 --- a/src/Access/RowPolicyCache.cpp +++ b/src/Access/RowPolicyCache.cpp @@ -85,7 +85,7 @@ void RowPolicyCache::PolicyInfo::setPolicy(const RowPolicyPtr & policy_) tryLogCurrentException( &Poco::Logger::get("RowPolicy"), String("Could not parse the condition ") + RowPolicy::conditionTypeToString(type) + " of row policy " - + backQuote(policy->getFullName())); + + backQuote(policy->getName())); } } } diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index 0389734e236..7f96d838e32 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -54,7 +54,7 @@ namespace } - UUID generateID(const IAccessEntity & entity) { return generateID(entity.getType(), entity.getFullName()); } + UUID generateID(const IAccessEntity & entity) { return generateID(entity.getType(), entity.getName()); } UserPtr parseUser(const Poco::Util::AbstractConfiguration & config, const String & user_name) { @@ -344,7 +344,7 @@ namespace String filter = (it != user_to_filters.end()) ? it->second : "1"; auto policy = std::make_shared(); - policy->setFullName(database, table_name, user_name); + policy->setNameParts(user_name, database, table_name); policy->conditions[RowPolicy::SELECT_FILTER] = filter; policy->to_roles.add(generateID(typeid(User), user_name)); policies.push_back(policy); @@ -494,21 +494,21 @@ String UsersConfigAccessStorage::readNameImpl(const UUID & id) const UUID UsersConfigAccessStorage::insertImpl(const AccessEntityPtr & entity, bool) { - throwReadonlyCannotInsert(entity->getType(), entity->getFullName()); + throwReadonlyCannotInsert(entity->getType(), entity->getName()); } void UsersConfigAccessStorage::removeImpl(const UUID & id) { auto entity = read(id); - throwReadonlyCannotRemove(entity->getType(), entity->getFullName()); + throwReadonlyCannotRemove(entity->getType(), entity->getName()); } void UsersConfigAccessStorage::updateImpl(const UUID & id, const UpdateFunc &) { auto entity = read(id); - throwReadonlyCannotUpdate(entity->getType(), entity->getFullName()); + throwReadonlyCannotUpdate(entity->getType(), entity->getName()); } diff --git a/src/Functions/currentRowPolicies.cpp b/src/Functions/currentRowPolicies.cpp index 0248f77c9b5..6efe57b2956 100644 --- a/src/Functions/currentRowPolicies.cpp +++ b/src/Functions/currentRowPolicies.cpp @@ -74,7 +74,7 @@ public: { const String database = policy->getDatabase(); const String table_name = policy->getTableName(); - const String policy_name = policy->getName(); + const String policy_name = policy->getShortName(); database_column->insertData(database.data(), database.length()); table_name_column->insertData(table_name.data(), table_name.length()); policy_name_column->insertData(policy_name.data(), policy_name.length()); @@ -123,7 +123,7 @@ public: const auto policy = context.getAccessControlManager().tryRead(policy_id); if (policy) { - const String policy_name = policy->getName(); + const String policy_name = policy->getShortName(); policy_name_column->insertData(policy_name.data(), policy_name.length()); } } diff --git a/src/Interpreters/InterpreterCreateRowPolicyQuery.cpp b/src/Interpreters/InterpreterCreateRowPolicyQuery.cpp index c3de3876c46..1d3c3a47bad 100644 --- a/src/Interpreters/InterpreterCreateRowPolicyQuery.cpp +++ b/src/Interpreters/InterpreterCreateRowPolicyQuery.cpp @@ -11,37 +11,20 @@ namespace DB { -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; -} - namespace { - const String & checkCurrentDatabase(const String & current_database) - { - if (current_database.empty()) - throw Exception("No current database", ErrorCodes::LOGICAL_ERROR); - return current_database; - } - void updateRowPolicyFromQueryImpl( RowPolicy & policy, const ASTCreateRowPolicyQuery & query, - const std::optional & roles_from_query = {}, - const String & current_database = {}) + const std::optional & roles_from_query = {}) { if (query.alter) { - if (!query.new_policy_name.empty()) - policy.setName(query.new_policy_name); + if (!query.new_short_name.empty()) + policy.setShortName(query.new_short_name); } else - { - policy.setDatabase(!query.name_parts.database.empty() ? query.name_parts.database : checkCurrentDatabase(current_database)); - policy.setTableName(query.name_parts.table_name); - policy.setName(query.name_parts.policy_name); - } + policy.setNameParts(query.name_parts); if (query.is_restrictive) policy.setRestrictive(*query.is_restrictive); @@ -78,29 +61,29 @@ BlockIO InterpreterCreateRowPolicyQuery::execute() if (query.roles) roles_from_query = ExtendedRoleSet{*query.roles, access_control, context.getUserID()}; - const String current_database = context.getCurrentDatabase(); + if (query.name_parts.database.empty()) + query.name_parts.database = context.getCurrentDatabase(); if (query.alter) { auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr { auto updated_policy = typeid_cast>(entity->clone()); - updateRowPolicyFromQueryImpl(*updated_policy, query, roles_from_query, current_database); + updateRowPolicyFromQueryImpl(*updated_policy, query, roles_from_query); return updated_policy; }; - String full_name = query.name_parts.getFullName(context); if (query.if_exists) { - if (auto id = access_control.find(full_name)) + if (auto id = access_control.find(query.name_parts.getName())) access_control.tryUpdate(*id, update_func); } else - access_control.update(access_control.getID(full_name), update_func); + access_control.update(access_control.getID(query.name_parts.getName()), update_func); } else { auto new_policy = std::make_shared(); - updateRowPolicyFromQueryImpl(*new_policy, query, roles_from_query, current_database); + updateRowPolicyFromQueryImpl(*new_policy, query, roles_from_query); if (query.if_not_exists) access_control.tryInsert(new_policy); diff --git a/src/Interpreters/InterpreterDropAccessEntityQuery.cpp b/src/Interpreters/InterpreterDropAccessEntityQuery.cpp index e67e0659796..2a47639e15f 100644 --- a/src/Interpreters/InterpreterDropAccessEntityQuery.cpp +++ b/src/Interpreters/InterpreterDropAccessEntityQuery.cpp @@ -47,7 +47,7 @@ namespace BlockIO InterpreterDropAccessEntityQuery::execute() { - const auto & query = query_ptr->as(); + auto & query = query_ptr->as(); auto & access_control = context.getAccessControlManager(); std::type_index type = getType(query.kind); @@ -58,14 +58,17 @@ BlockIO InterpreterDropAccessEntityQuery::execute() if (query.kind == Kind::ROW_POLICY) { - Strings full_names; - boost::range::transform( - query.row_policies_names, std::back_inserter(full_names), - [this](const RowPolicy::FullNameParts & row_policy_name) { return row_policy_name.getFullName(context); }); + Strings names; + for (auto & name_parts : query.row_policies_name_parts) + { + if (name_parts.database.empty()) + name_parts.database = context.getCurrentDatabase(); + names.emplace_back(name_parts.getName()); + } if (query.if_exists) - access_control.tryRemove(access_control.find(full_names)); + access_control.tryRemove(access_control.find(names)); else - access_control.remove(access_control.getIDs(full_names)); + access_control.remove(access_control.getIDs(names)); return {}; } diff --git a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp index 469a2f1196b..beea1bf953e 100644 --- a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp +++ b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp @@ -160,7 +160,7 @@ namespace bool attach_mode) { auto query = std::make_shared(); - query->name_parts = RowPolicy::FullNameParts{policy.getDatabase(), policy.getTableName(), policy.getName()}; + query->name_parts = policy.getNameParts(); query->attach = attach_mode; if (policy.isRestrictive()) @@ -233,7 +233,7 @@ BlockIO InterpreterShowCreateAccessEntityQuery::execute() BlockInputStreamPtr InterpreterShowCreateAccessEntityQuery::executeImpl() { - const auto & show_query = query_ptr->as(); + auto & show_query = query_ptr->as(); /// Build a create query. ASTPtr create_query = getCreateQuery(show_query); @@ -257,7 +257,7 @@ BlockInputStreamPtr InterpreterShowCreateAccessEntityQuery::executeImpl() } -ASTPtr InterpreterShowCreateAccessEntityQuery::getCreateQuery(const ASTShowCreateAccessEntityQuery & show_query) const +ASTPtr InterpreterShowCreateAccessEntityQuery::getCreateQuery(ASTShowCreateAccessEntityQuery & show_query) const { const auto & access_control = context.getAccessControlManager(); context.checkAccess(getRequiredAccess()); @@ -277,7 +277,9 @@ ASTPtr InterpreterShowCreateAccessEntityQuery::getCreateQuery(const ASTShowCreat auto type = getType(show_query.kind); if (show_query.kind == Kind::ROW_POLICY) { - RowPolicyPtr policy = access_control.read(show_query.row_policy_name.getFullName(context)); + if (show_query.row_policy_name_parts.database.empty()) + show_query.row_policy_name_parts.database = context.getCurrentDatabase(); + RowPolicyPtr policy = access_control.read(show_query.row_policy_name_parts.getName()); return getCreateQueryImpl(*policy, &access_control, false); } diff --git a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.h b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.h index 0183c59766f..fa610c8cc65 100644 --- a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.h +++ b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.h @@ -30,7 +30,7 @@ public: private: BlockInputStreamPtr executeImpl(); - ASTPtr getCreateQuery(const ASTShowCreateAccessEntityQuery & show_query) const; + ASTPtr getCreateQuery(ASTShowCreateAccessEntityQuery & show_query) const; AccessRightsElements getRequiredAccess() const; ASTPtr query_ptr; diff --git a/src/Interpreters/InterpreterShowRowPoliciesQuery.cpp b/src/Interpreters/InterpreterShowRowPoliciesQuery.cpp index 3fcae82064d..a4405615ceb 100644 --- a/src/Interpreters/InterpreterShowRowPoliciesQuery.cpp +++ b/src/Interpreters/InterpreterShowRowPoliciesQuery.cpp @@ -49,7 +49,7 @@ String InterpreterShowRowPoliciesQuery::getRewrittenQuery() const filter = "database = " + quoteString(database) + " AND table = " + quoteString(table_name); } - String expr = table_name.empty() ? "full_name" : "name"; + String expr = table_name.empty() ? "name" : "short_name"; return "SELECT " + expr + " AS " + backQuote(getResultDescription()) + " from system.row_policies" + (filter.empty() ? "" : " WHERE " + filter) + " ORDER BY " + expr; diff --git a/src/Parsers/ASTCreateRowPolicyQuery.cpp b/src/Parsers/ASTCreateRowPolicyQuery.cpp index fc6af51da6f..ed11ab936ff 100644 --- a/src/Parsers/ASTCreateRowPolicyQuery.cpp +++ b/src/Parsers/ASTCreateRowPolicyQuery.cpp @@ -12,10 +12,10 @@ namespace { using ConditionType = RowPolicy::ConditionType; - void formatRenameTo(const String & new_policy_name, const IAST::FormatSettings & settings) + void formatRenameTo(const String & new_short_name, const IAST::FormatSettings & settings) { settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " RENAME TO " << (settings.hilite ? IAST::hilite_none : "") - << backQuote(new_policy_name); + << backQuote(new_short_name); } @@ -153,14 +153,14 @@ void ASTCreateRowPolicyQuery::formatImpl(const FormatSettings & settings, Format const String & database = name_parts.database; const String & table_name = name_parts.table_name; - const String & policy_name = name_parts.policy_name; - settings.ostr << " " << backQuoteIfNeed(policy_name) << (settings.hilite ? hilite_keyword : "") << " ON " + const String & short_name = name_parts.short_name; + settings.ostr << " " << backQuoteIfNeed(short_name) << (settings.hilite ? hilite_keyword : "") << " ON " << (settings.hilite ? hilite_none : "") << (database.empty() ? String{} : backQuoteIfNeed(database) + ".") << table_name; formatOnCluster(settings); - if (!new_policy_name.empty()) - formatRenameTo(new_policy_name, settings); + if (!new_short_name.empty()) + formatRenameTo(new_short_name, settings); if (is_restrictive) formatAsRestrictiveOrPermissive(*is_restrictive, settings); diff --git a/src/Parsers/ASTCreateRowPolicyQuery.h b/src/Parsers/ASTCreateRowPolicyQuery.h index 2ad64255c04..dc14d40c7ad 100644 --- a/src/Parsers/ASTCreateRowPolicyQuery.h +++ b/src/Parsers/ASTCreateRowPolicyQuery.h @@ -36,8 +36,8 @@ public: bool if_not_exists = false; bool or_replace = false; - RowPolicy::FullNameParts name_parts; - String new_policy_name; + RowPolicy::NameParts name_parts; + String new_short_name; std::optional is_restrictive; using ConditionType = RowPolicy::ConditionType; diff --git a/src/Parsers/ASTDropAccessEntityQuery.cpp b/src/Parsers/ASTDropAccessEntityQuery.cpp index 06a820bfbb5..a0e6753af57 100644 --- a/src/Parsers/ASTDropAccessEntityQuery.cpp +++ b/src/Parsers/ASTDropAccessEntityQuery.cpp @@ -51,15 +51,15 @@ void ASTDropAccessEntityQuery::formatImpl(const FormatSettings & settings, Forma if (kind == Kind::ROW_POLICY) { bool need_comma = false; - for (const auto & row_policy_name : row_policies_names) + for (const auto & name_parts : row_policies_name_parts) { if (need_comma) settings.ostr << ','; need_comma = true; - const String & database = row_policy_name.database; - const String & table_name = row_policy_name.table_name; - const String & policy_name = row_policy_name.policy_name; - settings.ostr << ' ' << backQuoteIfNeed(policy_name) << (settings.hilite ? hilite_keyword : "") << " ON " + const String & database = name_parts.database; + const String & table_name = name_parts.table_name; + const String & short_name = name_parts.short_name; + settings.ostr << ' ' << backQuoteIfNeed(short_name) << (settings.hilite ? hilite_keyword : "") << " ON " << (settings.hilite ? hilite_none : "") << (database.empty() ? String{} : backQuoteIfNeed(database) + ".") << backQuoteIfNeed(table_name); } diff --git a/src/Parsers/ASTDropAccessEntityQuery.h b/src/Parsers/ASTDropAccessEntityQuery.h index a3b358dcfb9..a630869e027 100644 --- a/src/Parsers/ASTDropAccessEntityQuery.h +++ b/src/Parsers/ASTDropAccessEntityQuery.h @@ -29,7 +29,7 @@ public: const Kind kind; bool if_exists = false; Strings names; - std::vector row_policies_names; + std::vector row_policies_name_parts; ASTDropAccessEntityQuery(Kind kind_); String getID(char) const override; diff --git a/src/Parsers/ASTShowCreateAccessEntityQuery.cpp b/src/Parsers/ASTShowCreateAccessEntityQuery.cpp index 9e562043f09..eed5766bcd9 100644 --- a/src/Parsers/ASTShowCreateAccessEntityQuery.cpp +++ b/src/Parsers/ASTShowCreateAccessEntityQuery.cpp @@ -54,10 +54,10 @@ void ASTShowCreateAccessEntityQuery::formatQueryImpl(const FormatSettings & sett settings.ostr << (settings.hilite ? hilite_keyword : "") << " CURRENT" << (settings.hilite ? hilite_none : ""); else if (kind == Kind::ROW_POLICY) { - const String & database = row_policy_name.database; - const String & table_name = row_policy_name.table_name; - const String & policy_name = row_policy_name.policy_name; - settings.ostr << ' ' << backQuoteIfNeed(policy_name) << (settings.hilite ? hilite_keyword : "") << " ON " + const String & database = row_policy_name_parts.database; + const String & table_name = row_policy_name_parts.table_name; + const String & short_name = row_policy_name_parts.short_name; + settings.ostr << ' ' << backQuoteIfNeed(short_name) << (settings.hilite ? hilite_keyword : "") << " ON " << (settings.hilite ? hilite_none : "") << (database.empty() ? String{} : backQuoteIfNeed(database) + ".") << backQuoteIfNeed(table_name); } diff --git a/src/Parsers/ASTShowCreateAccessEntityQuery.h b/src/Parsers/ASTShowCreateAccessEntityQuery.h index e76a9177979..7f82e9f5e34 100644 --- a/src/Parsers/ASTShowCreateAccessEntityQuery.h +++ b/src/Parsers/ASTShowCreateAccessEntityQuery.h @@ -28,7 +28,7 @@ public: String name; bool current_quota = false; bool current_user = false; - RowPolicy::FullNameParts row_policy_name; + RowPolicy::NameParts row_policy_name_parts; ASTShowCreateAccessEntityQuery(Kind kind_); String getID(char) const override; diff --git a/src/Parsers/ParserCreateRowPolicyQuery.cpp b/src/Parsers/ParserCreateRowPolicyQuery.cpp index b6840f0ed6a..22fcf47b379 100644 --- a/src/Parsers/ParserCreateRowPolicyQuery.cpp +++ b/src/Parsers/ParserCreateRowPolicyQuery.cpp @@ -21,14 +21,14 @@ namespace { using ConditionType = RowPolicy::ConditionType; - bool parseRenameTo(IParserBase::Pos & pos, Expected & expected, String & new_policy_name) + bool parseRenameTo(IParserBase::Pos & pos, Expected & expected, String & new_short_name) { return IParserBase::wrapParseImpl(pos, [&] { if (!ParserKeyword{"RENAME TO"}.ignore(pos, expected)) return false; - return parseIdentifierOrStringLiteral(pos, expected, new_policy_name); + return parseIdentifierOrStringLiteral(pos, expected, new_short_name); }); } @@ -246,22 +246,22 @@ bool ParserCreateRowPolicyQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & or_replace = true; } - RowPolicy::FullNameParts name_parts; + RowPolicy::NameParts name_parts; String & database = name_parts.database; String & table_name = name_parts.table_name; - String & policy_name = name_parts.policy_name; - if (!parseIdentifierOrStringLiteral(pos, expected, policy_name) || !ParserKeyword{"ON"}.ignore(pos, expected) + String & short_name = name_parts.short_name; + if (!parseIdentifierOrStringLiteral(pos, expected, short_name) || !ParserKeyword{"ON"}.ignore(pos, expected) || !parseDatabaseAndTableName(pos, expected, database, table_name)) return false; - String new_policy_name; + String new_short_name; std::optional is_restrictive; std::vector> conditions; String cluster; while (true) { - if (alter && new_policy_name.empty() && parseRenameTo(pos, expected, new_policy_name)) + if (alter && new_short_name.empty() && parseRenameTo(pos, expected, new_short_name)) continue; if (!is_restrictive && parseAsRestrictiveOrPermissive(pos, expected, is_restrictive)) @@ -292,7 +292,7 @@ bool ParserCreateRowPolicyQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & query->or_replace = or_replace; query->cluster = std::move(cluster); query->name_parts = std::move(name_parts); - query->new_policy_name = std::move(new_policy_name); + query->new_short_name = std::move(new_short_name); query->is_restrictive = is_restrictive; query->conditions = std::move(conditions); query->roles = std::move(roles); diff --git a/src/Parsers/ParserDropAccessEntityQuery.cpp b/src/Parsers/ParserDropAccessEntityQuery.cpp index ecda1691240..034124a42f0 100644 --- a/src/Parsers/ParserDropAccessEntityQuery.cpp +++ b/src/Parsers/ParserDropAccessEntityQuery.cpp @@ -30,25 +30,25 @@ namespace }); } - bool parseRowPolicyNames(IParserBase::Pos & pos, Expected & expected, std::vector & names) + bool parseRowPolicyNames(IParserBase::Pos & pos, Expected & expected, std::vector & name_parts) { return IParserBase::wrapParseImpl(pos, [&] { - std::vector res_names; + std::vector res_name_parts; do { - Strings policy_names; - if (!parseNames(pos, expected, policy_names)) + Strings short_names; + if (!parseNames(pos, expected, short_names)) return false; String database, table_name; if (!ParserKeyword{"ON"}.ignore(pos, expected) || !parseDatabaseAndTableName(pos, expected, database, table_name)) return false; - for (const String & policy_name : policy_names) - res_names.push_back({database, table_name, policy_name}); + for (String & short_name : short_names) + res_name_parts.push_back({std::move(short_name), database, table_name}); } while (ParserToken{TokenType::Comma}.ignore(pos, expected)); - names = std::move(res_names); + name_parts = std::move(res_name_parts); return true; }); } @@ -99,7 +99,7 @@ bool ParserDropAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & if_exists = true; Strings names; - std::vector row_policies_names; + std::vector row_policies_name_parts; if ((kind == Kind::USER) || (kind == Kind::ROLE)) { @@ -108,7 +108,7 @@ bool ParserDropAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & } else if (kind == Kind::ROW_POLICY) { - if (!parseRowPolicyNames(pos, expected, row_policies_names)) + if (!parseRowPolicyNames(pos, expected, row_policies_name_parts)) return false; } else @@ -130,7 +130,7 @@ bool ParserDropAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & query->if_exists = if_exists; query->cluster = std::move(cluster); query->names = std::move(names); - query->row_policies_names = std::move(row_policies_names); + query->row_policies_name_parts = std::move(row_policies_name_parts); return true; } diff --git a/src/Parsers/ParserShowCreateAccessEntityQuery.cpp b/src/Parsers/ParserShowCreateAccessEntityQuery.cpp index faf9a0a1554..48cd36f68d3 100644 --- a/src/Parsers/ParserShowCreateAccessEntityQuery.cpp +++ b/src/Parsers/ParserShowCreateAccessEntityQuery.cpp @@ -32,7 +32,7 @@ bool ParserShowCreateAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expe String name; bool current_quota = false; bool current_user = false; - RowPolicy::FullNameParts row_policy_name; + RowPolicy::NameParts row_policy_name_parts; if (kind == Kind::USER) { @@ -46,10 +46,10 @@ bool ParserShowCreateAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expe } else if (kind == Kind::ROW_POLICY) { - String & database = row_policy_name.database; - String & table_name = row_policy_name.table_name; - String & policy_name = row_policy_name.policy_name; - if (!parseIdentifierOrStringLiteral(pos, expected, policy_name) || !ParserKeyword{"ON"}.ignore(pos, expected) + String & database = row_policy_name_parts.database; + String & table_name = row_policy_name_parts.table_name; + String & short_name = row_policy_name_parts.short_name; + if (!parseIdentifierOrStringLiteral(pos, expected, short_name) || !ParserKeyword{"ON"}.ignore(pos, expected) || !parseDatabaseAndTableName(pos, expected, database, table_name)) return false; } @@ -82,7 +82,7 @@ bool ParserShowCreateAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expe query->name = std::move(name); query->current_quota = current_quota; query->current_user = current_user; - query->row_policy_name = std::move(row_policy_name); + query->row_policy_name_parts = std::move(row_policy_name_parts); return true; } diff --git a/src/Storages/System/StorageSystemRowPolicies.cpp b/src/Storages/System/StorageSystemRowPolicies.cpp index 39fdf926f92..cacd657b46f 100644 --- a/src/Storages/System/StorageSystemRowPolicies.cpp +++ b/src/Storages/System/StorageSystemRowPolicies.cpp @@ -18,8 +18,8 @@ NamesAndTypesList StorageSystemRowPolicies::getNamesAndTypes() NamesAndTypesList names_and_types{ {"database", std::make_shared()}, {"table", std::make_shared()}, + {"short_name", std::make_shared()}, {"name", std::make_shared()}, - {"full_name", std::make_shared()}, {"id", std::make_shared()}, {"source", std::make_shared()}, {"restrictive", std::make_shared()}, @@ -48,8 +48,8 @@ void StorageSystemRowPolicies::fillData(MutableColumns & res_columns, const Cont size_t i = 0; res_columns[i++]->insert(policy->getDatabase()); res_columns[i++]->insert(policy->getTableName()); + res_columns[i++]->insert(policy->getShortName()); res_columns[i++]->insert(policy->getName()); - res_columns[i++]->insert(policy->getFullName()); res_columns[i++]->insert(id); res_columns[i++]->insert(storage ? storage->getStorageName() : ""); res_columns[i++]->insert(policy->isRestrictive()); From dd8b29b4fbd72f35b5bd62c42972583032432cc1 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Sun, 3 May 2020 06:12:03 +0300 Subject: [PATCH 181/738] Use enum Type instead of std::type_index to represent the type of IAccessEntity. This change simplifies handling of access entities in access storages. --- src/Access/DiskAccessStorage.cpp | 259 +++++++++--------- src/Access/DiskAccessStorage.h | 25 +- src/Access/IAccessEntity.cpp | 38 +-- src/Access/IAccessEntity.h | 120 +++++++- src/Access/IAccessStorage.cpp | 104 ++++--- src/Access/IAccessStorage.h | 122 +++++---- src/Access/MemoryAccessStorage.cpp | 141 +++++----- src/Access/MemoryAccessStorage.h | 23 +- src/Access/MultipleAccessStorage.cpp | 20 +- src/Access/MultipleAccessStorage.h | 10 +- src/Access/Quota.h | 2 + src/Access/Role.cpp | 1 + src/Access/Role.h | 2 + src/Access/RowPolicy.h | 2 + src/Access/SettingsProfile.cpp | 2 + src/Access/SettingsProfile.h | 2 + src/Access/User.cpp | 1 + src/Access/User.h | 2 + src/Access/UsersConfigAccessStorage.cpp | 37 +-- src/Access/UsersConfigAccessStorage.h | 8 +- src/Common/ErrorCodes.cpp | 1 + .../InterpreterDropAccessEntityQuery.cpp | 62 ++--- .../InterpreterDropAccessEntityQuery.h | 4 + ...InterpreterShowCreateAccessEntityQuery.cpp | 57 ++-- .../InterpreterShowGrantsQuery.cpp | 2 +- src/Parsers/ASTDropAccessEntityQuery.cpp | 30 +- src/Parsers/ASTDropAccessEntityQuery.h | 12 +- .../ASTShowCreateAccessEntityQuery.cpp | 30 +- src/Parsers/ASTShowCreateAccessEntityQuery.h | 12 +- src/Parsers/ParserDropAccessEntityQuery.cpp | 35 +-- .../ParserShowCreateAccessEntityQuery.cpp | 42 +-- .../test_access_control_on_cluster/test.py | 6 +- .../test_disk_access_storage/test.py | 6 +- 33 files changed, 626 insertions(+), 594 deletions(-) diff --git a/src/Access/DiskAccessStorage.cpp b/src/Access/DiskAccessStorage.cpp index df9cbab891f..bc3e35a4fc7 100644 --- a/src/Access/DiskAccessStorage.cpp +++ b/src/Access/DiskAccessStorage.cpp @@ -53,6 +53,9 @@ namespace ErrorCodes namespace { + using EntityType = IAccessStorage::EntityType; + using EntityTypeInfo = IAccessStorage::EntityTypeInfo; + /// Special parser for the 'ATTACH access entity' queries. class ParserAttachAccessEntity : public IParserBase { @@ -79,7 +82,7 @@ namespace /// Reads a file containing ATTACH queries and then parses it to build an access entity. - AccessEntityPtr readAccessEntityFile(const std::filesystem::path & file_path) + AccessEntityPtr readEntityFile(const std::filesystem::path & file_path) { /// Read the file. ReadBufferFromFile in{file_path}; @@ -164,11 +167,11 @@ namespace } - AccessEntityPtr tryReadAccessEntityFile(const std::filesystem::path & file_path, Poco::Logger & log) + AccessEntityPtr tryReadEntityFile(const std::filesystem::path & file_path, Poco::Logger & log) { try { - return readAccessEntityFile(file_path); + return readEntityFile(file_path); } catch (...) { @@ -179,12 +182,12 @@ namespace /// Writes ATTACH queries for building a specified access entity to a file. - void writeAccessEntityFile(const std::filesystem::path & file_path, const IAccessEntity & entity) + void writeEntityFile(const std::filesystem::path & file_path, const IAccessEntity & entity) { /// Build list of ATTACH queries. ASTs queries; queries.push_back(InterpreterShowCreateAccessEntityQuery::getAttachQuery(entity)); - if (entity.getType() == typeid(User) || entity.getType() == typeid(Role)) + if ((entity.getType() == EntityType::USER) || (entity.getType() == EntityType::ROLE)) boost::range::push_back(queries, InterpreterShowGrantsQuery::getAttachGrantQueries(entity)); /// Serialize the list of ATTACH queries to a string. @@ -213,21 +216,21 @@ namespace /// Calculates the path to a file named .sql for saving an access entity. - std::filesystem::path getAccessEntityFilePath(const String & directory_path, const UUID & id) + std::filesystem::path getEntityFilePath(const String & directory_path, const UUID & id) { return std::filesystem::path(directory_path).append(toString(id)).replace_extension(".sql"); } /// Reads a map of name of access entity to UUID for access entities of some type from a file. - std::unordered_map readListFile(const std::filesystem::path & file_path) + std::vector> readListFile(const std::filesystem::path & file_path) { ReadBufferFromFile in(file_path); size_t num; readVarUInt(num, in); - std::unordered_map res; - res.reserve(num); + std::vector> id_name_pairs; + id_name_pairs.reserve(num); for (size_t i = 0; i != num; ++i) { @@ -235,19 +238,19 @@ namespace readStringBinary(name, in); UUID id; readUUIDText(id, in); - res[name] = id; + id_name_pairs.emplace_back(id, std::move(name)); } - return res; + return id_name_pairs; } /// Writes a map of name of access entity to UUID for access entities of some type to a file. - void writeListFile(const std::filesystem::path & file_path, const std::unordered_map & map) + void writeListFile(const std::filesystem::path & file_path, const std::vector> & id_name_pairs) { WriteBufferFromFile out(file_path); - writeVarUInt(map.size(), out); - for (const auto & [name, id] : map) + writeVarUInt(id_name_pairs.size(), out); + for (const auto & [id, name] : id_name_pairs) { writeStringBinary(name, out); writeUUIDText(id, out); @@ -256,24 +259,10 @@ namespace /// Calculates the path for storing a map of name of access entity to UUID for access entities of some type. - std::filesystem::path getListFilePath(const String & directory_path, std::type_index type) + std::filesystem::path getListFilePath(const String & directory_path, EntityType type) { - std::string_view file_name; - if (type == typeid(User)) - file_name = "users"; - else if (type == typeid(Role)) - file_name = "roles"; - else if (type == typeid(Quota)) - file_name = "quotas"; - else if (type == typeid(RowPolicy)) - file_name = "row_policies"; - else if (type == typeid(SettingsProfile)) - file_name = "settings_profiles"; - else - throw Exception("Unexpected type of access entity: " + IAccessEntity::getTypeName(type), - ErrorCodes::LOGICAL_ERROR); - - return std::filesystem::path(directory_path).append(file_name).replace_extension(".list"); + std::string_view file_name = EntityTypeInfo::get(type).list_filename; + return std::filesystem::path(directory_path).append(file_name); } @@ -297,21 +286,12 @@ namespace return false; } } - - - const std::vector & getAllAccessEntityTypes() - { - static const std::vector res = {typeid(User), typeid(Role), typeid(RowPolicy), typeid(Quota), typeid(SettingsProfile)}; - return res; - } } DiskAccessStorage::DiskAccessStorage() : IAccessStorage("disk") { - for (auto type : getAllAccessEntityTypes()) - name_to_id_maps[type]; } @@ -363,18 +343,27 @@ void DiskAccessStorage::initialize(const String & directory_path_, Notifications writeLists(); } - for (const auto & [id, entry] : id_to_entry_map) + for (const auto & [id, entry] : entries_by_id) prepareNotifications(id, entry, false, notifications); } +void DiskAccessStorage::clear() +{ + entries_by_id.clear(); + for (auto type : ext::range(EntityType::MAX)) + entries_by_name_and_type[static_cast(type)].clear(); +} + + bool DiskAccessStorage::readLists() { - assert(id_to_entry_map.empty()); + clear(); + bool ok = true; - for (auto type : getAllAccessEntityTypes()) + for (auto type : ext::range(EntityType::MAX)) { - auto & name_to_id_map = name_to_id_maps.at(type); + auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; auto file_path = getListFilePath(directory_path, type); if (!std::filesystem::exists(file_path)) { @@ -385,7 +374,14 @@ bool DiskAccessStorage::readLists() try { - name_to_id_map = readListFile(file_path); + for (const auto & [id, name] : readListFile(file_path)) + { + auto & entry = entries_by_id[id]; + entry.id = id; + entry.type = type; + entry.name = name; + entries_by_name[entry.name] = &entry; + } } catch (...) { @@ -393,17 +389,10 @@ bool DiskAccessStorage::readLists() ok = false; break; } - - for (const auto & [name, id] : name_to_id_map) - id_to_entry_map.emplace(id, Entry{name, type}); } if (!ok) - { - id_to_entry_map.clear(); - for (auto & name_to_id_map : name_to_id_maps | boost::adaptors::map_values) - name_to_id_map.clear(); - } + clear(); return ok; } @@ -419,11 +408,15 @@ bool DiskAccessStorage::writeLists() for (const auto & type : types_of_lists_to_write) { - const auto & name_to_id_map = name_to_id_maps.at(type); + auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; auto file_path = getListFilePath(directory_path, type); try { - writeListFile(file_path, name_to_id_map); + std::vector> id_name_pairs; + id_name_pairs.reserve(entries_by_name.size()); + for (const auto * entry : entries_by_name | boost::adaptors::map_values) + id_name_pairs.emplace_back(entry->id, entry->name); + writeListFile(file_path, id_name_pairs); } catch (...) { @@ -441,7 +434,7 @@ bool DiskAccessStorage::writeLists() } -void DiskAccessStorage::scheduleWriteLists(std::type_index type) +void DiskAccessStorage::scheduleWriteLists(EntityType type) { if (failed_to_write_lists) return; @@ -504,7 +497,7 @@ void DiskAccessStorage::listsWritingThreadFunc() bool DiskAccessStorage::rebuildLists() { LOG_WARNING(getLogger(), "Recovering lists in directory " + directory_path); - assert(id_to_entry_map.empty()); + clear(); for (const auto & directory_entry : std::filesystem::directory_iterator(directory_path)) { @@ -518,58 +511,64 @@ bool DiskAccessStorage::rebuildLists() if (!tryParseUUID(path.stem(), id)) continue; - const auto access_entity_file_path = getAccessEntityFilePath(directory_path, id); - auto entity = tryReadAccessEntityFile(access_entity_file_path, *getLogger()); + const auto access_entity_file_path = getEntityFilePath(directory_path, id); + auto entity = tryReadEntityFile(access_entity_file_path, *getLogger()); if (!entity) continue; + const String & name = entity->getName(); auto type = entity->getType(); - auto & name_to_id_map = name_to_id_maps.at(type); - auto it_by_name = name_to_id_map.emplace(entity->getName(), id).first; - id_to_entry_map.emplace(id, Entry{it_by_name->first, type}); + auto & entry = entries_by_id[id]; + entry.id = id; + entry.type = type; + entry.name = name; + entry.entity = entity; + auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; + entries_by_name[entry.name] = &entry; } - for (auto type : getAllAccessEntityTypes()) + for (auto type : ext::range(EntityType::MAX)) types_of_lists_to_write.insert(type); return true; } -std::optional DiskAccessStorage::findImpl(std::type_index type, const String & name) const +std::optional DiskAccessStorage::findImpl(EntityType type, const String & name) const { std::lock_guard lock{mutex}; - const auto & name_to_id_map = name_to_id_maps.at(type); - auto it = name_to_id_map.find(name); - if (it == name_to_id_map.end()) + const auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; + auto it = entries_by_name.find(name); + if (it == entries_by_name.end()) return {}; - return it->second; + return it->second->id; } -std::vector DiskAccessStorage::findAllImpl(std::type_index type) const +std::vector DiskAccessStorage::findAllImpl(EntityType type) const { std::lock_guard lock{mutex}; - const auto & name_to_id_map = name_to_id_maps.at(type); + const auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; std::vector res; - res.reserve(name_to_id_map.size()); - boost::range::copy(name_to_id_map | boost::adaptors::map_values, std::back_inserter(res)); + res.reserve(entries_by_name.size()); + for (const auto * entry : entries_by_name | boost::adaptors::map_values) + res.emplace_back(entry->id); return res; } bool DiskAccessStorage::existsImpl(const UUID & id) const { std::lock_guard lock{mutex}; - return id_to_entry_map.count(id); + return entries_by_id.count(id); } AccessEntityPtr DiskAccessStorage::readImpl(const UUID & id) const { std::lock_guard lock{mutex}; - auto it = id_to_entry_map.find(id); - if (it == id_to_entry_map.end()) + auto it = entries_by_id.find(id); + if (it == entries_by_id.end()) throwNotFound(id); const auto & entry = it->second; @@ -582,8 +581,8 @@ AccessEntityPtr DiskAccessStorage::readImpl(const UUID & id) const String DiskAccessStorage::readNameImpl(const UUID & id) const { std::lock_guard lock{mutex}; - auto it = id_to_entry_map.find(id); - if (it == id_to_entry_map.end()) + auto it = entries_by_id.find(id); + if (it == entries_by_id.end()) throwNotFound(id); return String{it->second.name}; } @@ -610,24 +609,24 @@ UUID DiskAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool repl void DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, Notifications & notifications) { const String & name = new_entity->getName(); - std::type_index type = new_entity->getType(); + EntityType type = new_entity->getType(); if (!initialized) throw Exception( - "Cannot insert " + new_entity->getTypeName() + " " + backQuote(name) + " to " + getStorageName() - + " because the output directory is not set", + "Cannot insert " + new_entity->outputTypeAndName() + " to storage [" + getStorageName() + + "] because the output directory is not set", ErrorCodes::LOGICAL_ERROR); /// Check that we can insert. - auto it_by_id = id_to_entry_map.find(id); - if (it_by_id != id_to_entry_map.end()) + auto it_by_id = entries_by_id.find(id); + if (it_by_id != entries_by_id.end()) { const auto & existing_entry = it_by_id->second; throwIDCollisionCannotInsert(id, type, name, existing_entry.entity->getType(), existing_entry.entity->getName()); } - auto & name_to_id_map = name_to_id_maps.at(type); - auto it_by_name = name_to_id_map.find(name); - bool name_collision = (it_by_name != name_to_id_map.end()); + auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; + auto it_by_name = entries_by_name.find(name); + bool name_collision = (it_by_name != entries_by_name.end()); if (name_collision && !replace_if_exists) throwNameCollisionCannotInsert(type, name); @@ -636,13 +635,15 @@ void DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & ne writeAccessEntityToDisk(id, *new_entity); if (name_collision && replace_if_exists) - removeNoLock(it_by_name->second, notifications); + removeNoLock(it_by_name->second->id, notifications); /// Do insertion. - it_by_name = name_to_id_map.emplace(name, id).first; - it_by_id = id_to_entry_map.emplace(id, Entry{it_by_name->first, type}).first; - auto & entry = it_by_id->second; + auto & entry = entries_by_id[id]; + entry.id = id; + entry.type = type; + entry.name = name; entry.entity = new_entity; + entries_by_name[entry.name] = &entry; prepareNotifications(id, entry, false, notifications); } @@ -659,22 +660,21 @@ void DiskAccessStorage::removeImpl(const UUID & id) void DiskAccessStorage::removeNoLock(const UUID & id, Notifications & notifications) { - auto it = id_to_entry_map.find(id); - if (it == id_to_entry_map.end()) + auto it = entries_by_id.find(id); + if (it == entries_by_id.end()) throwNotFound(id); Entry & entry = it->second; - String name{it->second.name}; - std::type_index type = it->second.type; + EntityType type = entry.type; scheduleWriteLists(type); deleteAccessEntityOnDisk(id); /// Do removing. prepareNotifications(id, entry, true, notifications); - id_to_entry_map.erase(it); - auto & name_to_id_map = name_to_id_maps.at(type); - name_to_id_map.erase(name); + auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; + entries_by_name.erase(entry.name); + entries_by_id.erase(it); } @@ -690,8 +690,8 @@ void DiskAccessStorage::updateImpl(const UUID & id, const UpdateFunc & update_fu void DiskAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & update_func, Notifications & notifications) { - auto it = id_to_entry_map.find(id); - if (it == id_to_entry_map.end()) + auto it = entries_by_id.find(id); + if (it == entries_by_id.end()) throwNotFound(id); Entry & entry = it->second; @@ -700,18 +700,22 @@ void DiskAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & update_ auto old_entity = entry.entity; auto new_entity = update_func(old_entity); + if (!new_entity->isTypeOf(old_entity->getType())) + throwBadCast(id, new_entity->getType(), new_entity->getName(), old_entity->getType()); + if (*new_entity == *old_entity) return; - String new_name = new_entity->getName(); - auto old_name = entry.name; - const std::type_index type = entry.type; + const String & new_name = new_entity->getName(); + const String & old_name = old_entity->getName(); + const EntityType type = entry.type; + auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; + bool name_changed = (new_name != old_name); if (name_changed) { - const auto & name_to_id_map = name_to_id_maps.at(type); - if (name_to_id_map.count(new_name)) - throwNameCollisionCannotRename(type, String{old_name}, new_name); + if (entries_by_name.count(new_name)) + throwNameCollisionCannotRename(type, old_name, new_name); scheduleWriteLists(type); } @@ -720,10 +724,9 @@ void DiskAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & update_ if (name_changed) { - auto & name_to_id_map = name_to_id_maps.at(type); - name_to_id_map.erase(String{old_name}); - auto it_by_name = name_to_id_map.emplace(new_name, id).first; - entry.name = it_by_name->first; + entries_by_name.erase(entry.name); + entry.name = new_name; + entries_by_name[entry.name] = &entry; } prepareNotifications(id, entry, false, notifications); @@ -732,19 +735,19 @@ void DiskAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & update_ AccessEntityPtr DiskAccessStorage::readAccessEntityFromDisk(const UUID & id) const { - return readAccessEntityFile(getAccessEntityFilePath(directory_path, id)); + return readEntityFile(getEntityFilePath(directory_path, id)); } void DiskAccessStorage::writeAccessEntityToDisk(const UUID & id, const IAccessEntity & entity) const { - writeAccessEntityFile(getAccessEntityFilePath(directory_path, id), entity); + writeEntityFile(getEntityFilePath(directory_path, id), entity); } void DiskAccessStorage::deleteAccessEntityOnDisk(const UUID & id) const { - auto file_path = getAccessEntityFilePath(directory_path, id); + auto file_path = getEntityFilePath(directory_path, id); if (!std::filesystem::remove(file_path)) throw Exception("Couldn't delete " + file_path.string(), ErrorCodes::FILE_DOESNT_EXIST); } @@ -759,17 +762,16 @@ void DiskAccessStorage::prepareNotifications(const UUID & id, const Entry & entr for (const auto & handler : entry.handlers_by_id) notifications.push_back({handler, id, entity}); - auto range = handlers_by_type.equal_range(entry.type); - for (auto it = range.first; it != range.second; ++it) - notifications.push_back({it->second, id, entity}); + for (const auto & handler : handlers_by_type[static_cast(entry.type)]) + notifications.push_back({handler, id, entity}); } ext::scope_guard DiskAccessStorage::subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const { std::lock_guard lock{mutex}; - auto it = id_to_entry_map.find(id); - if (it == id_to_entry_map.end()) + auto it = entries_by_id.find(id); + if (it == entries_by_id.end()) return {}; const Entry & entry = it->second; auto handler_it = entry.handlers_by_id.insert(entry.handlers_by_id.end(), handler); @@ -777,8 +779,8 @@ ext::scope_guard DiskAccessStorage::subscribeForChangesImpl(const UUID & id, con return [this, id, handler_it] { std::lock_guard lock2{mutex}; - auto it2 = id_to_entry_map.find(id); - if (it2 != id_to_entry_map.end()) + auto it2 = entries_by_id.find(id); + if (it2 != entries_by_id.end()) { const Entry & entry2 = it2->second; entry2.handlers_by_id.erase(handler_it); @@ -786,23 +788,26 @@ ext::scope_guard DiskAccessStorage::subscribeForChangesImpl(const UUID & id, con }; } -ext::scope_guard DiskAccessStorage::subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const +ext::scope_guard DiskAccessStorage::subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const { std::lock_guard lock{mutex}; - auto handler_it = handlers_by_type.emplace(type, handler); + auto & handlers = handlers_by_type[static_cast(type)]; + handlers.push_back(handler); + auto handler_it = std::prev(handlers.end()); - return [this, handler_it] + return [this, type, handler_it] { std::lock_guard lock2{mutex}; - handlers_by_type.erase(handler_it); + auto & handlers2 = handlers_by_type[static_cast(type)]; + handlers2.erase(handler_it); }; } bool DiskAccessStorage::hasSubscriptionImpl(const UUID & id) const { std::lock_guard lock{mutex}; - auto it = id_to_entry_map.find(id); - if (it != id_to_entry_map.end()) + auto it = entries_by_id.find(id); + if (it != entries_by_id.end()) { const Entry & entry = it->second; return !entry.handlers_by_id.empty(); @@ -810,11 +815,11 @@ bool DiskAccessStorage::hasSubscriptionImpl(const UUID & id) const return false; } -bool DiskAccessStorage::hasSubscriptionImpl(std::type_index type) const +bool DiskAccessStorage::hasSubscriptionImpl(EntityType type) const { std::lock_guard lock{mutex}; - auto range = handlers_by_type.equal_range(type); - return range.first != range.second; + const auto & handlers = handlers_by_type[static_cast(type)]; + return !handlers.empty(); } } diff --git a/src/Access/DiskAccessStorage.h b/src/Access/DiskAccessStorage.h index 104c0f1fa38..79a11195318 100644 --- a/src/Access/DiskAccessStorage.h +++ b/src/Access/DiskAccessStorage.h @@ -17,8 +17,8 @@ public: void setDirectory(const String & directory_path_); private: - std::optional findImpl(std::type_index type, const String & name) const override; - std::vector findAllImpl(std::type_index type) const override; + std::optional findImpl(EntityType type, const String & name) const override; + std::vector findAllImpl(EntityType type) const override; bool existsImpl(const UUID & id) const override; AccessEntityPtr readImpl(const UUID & id) const override; String readNameImpl(const UUID & id) const override; @@ -27,14 +27,15 @@ private: void removeImpl(const UUID & id) override; void updateImpl(const UUID & id, const UpdateFunc & update_func) override; ext::scope_guard subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override; - ext::scope_guard subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const override; + ext::scope_guard subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const override; bool hasSubscriptionImpl(const UUID & id) const override; - bool hasSubscriptionImpl(std::type_index type) const override; + bool hasSubscriptionImpl(EntityType type) const override; void initialize(const String & directory_path_, Notifications & notifications); + void clear(); bool readLists(); bool writeLists(); - void scheduleWriteLists(std::type_index type); + void scheduleWriteLists(EntityType type); bool rebuildLists(); void startListsWritingThread(); @@ -52,9 +53,9 @@ private: using NameToIDMap = std::unordered_map; struct Entry { - Entry(const std::string_view & name_, std::type_index type_) : name(name_), type(type_) {} - std::string_view name; /// view points to a string in `name_to_id_maps`. - std::type_index type; + UUID id; + String name; + EntityType type; mutable AccessEntityPtr entity; /// may be nullptr, if the entity hasn't been loaded yet. mutable std::list handlers_by_id; }; @@ -63,14 +64,14 @@ private: String directory_path; bool initialized = false; - std::unordered_map name_to_id_maps; - std::unordered_map id_to_entry_map; - boost::container::flat_set types_of_lists_to_write; + std::unordered_map entries_by_id; + std::unordered_map entries_by_name_and_type[static_cast(EntityType::MAX)]; + boost::container::flat_set types_of_lists_to_write; bool failed_to_write_lists = false; /// Whether writing of the list files has been failed since the recent restart of the server. ThreadFromGlobalPool lists_writing_thread; /// List files are written in a separate thread. std::condition_variable lists_writing_thread_should_exit; /// Signals `lists_writing_thread` to exit. std::atomic lists_writing_thread_exited = false; - mutable std::unordered_multimap handlers_by_type; + mutable std::list handlers_by_type[static_cast(EntityType::MAX)]; mutable std::mutex mutex; }; } diff --git a/src/Access/IAccessEntity.cpp b/src/Access/IAccessEntity.cpp index e5e484c4c33..5dc566fe456 100644 --- a/src/Access/IAccessEntity.cpp +++ b/src/Access/IAccessEntity.cpp @@ -1,48 +1,12 @@ #include -#include -#include -#include -#include -#include -#include namespace DB { -String IAccessEntity::getTypeName(std::type_index type) -{ - if (type == typeid(User)) - return "User"; - if (type == typeid(Quota)) - return "Quota"; - if (type == typeid(RowPolicy)) - return "Row policy"; - if (type == typeid(Role)) - return "Role"; - if (type == typeid(SettingsProfile)) - return "Settings profile"; - return demangle(type.name()); -} - - -const char * IAccessEntity::getKeyword(std::type_index type) -{ - if (type == typeid(User)) - return "USER"; - if (type == typeid(Quota)) - return "QUOTA"; - if (type == typeid(RowPolicy)) - return "ROW POLICY"; - if (type == typeid(Role)) - return "ROLE"; - if (type == typeid(SettingsProfile)) - return "SETTINGS PROFILE"; - __builtin_unreachable(); -} - bool IAccessEntity::equal(const IAccessEntity & other) const { return (name == other.name) && (getType() == other.getType()); } + } diff --git a/src/Access/IAccessEntity.h b/src/Access/IAccessEntity.h index 9a4e80fbde9..39a5cefa7d7 100644 --- a/src/Access/IAccessEntity.h +++ b/src/Access/IAccessEntity.h @@ -2,12 +2,24 @@ #include #include +#include +#include #include -#include namespace DB { +namespace ErrorCodes +{ + extern const int UNKNOWN_USER; + extern const int UNKNOWN_ROLE; + extern const int UNKNOWN_ROW_POLICY; + extern const int UNKNOWN_QUOTA; + extern const int THERE_IS_NO_PROFILE; + extern const int LOGICAL_ERROR; +} + + /// Access entity is a set of data which have a name and a type. Access entity control something related to the access control. /// Entities can be stored to a file or another storage, see IAccessStorage. struct IAccessEntity @@ -17,15 +29,39 @@ struct IAccessEntity virtual ~IAccessEntity() = default; virtual std::shared_ptr clone() const = 0; - std::type_index getType() const { return typeid(*this); } - static String getTypeName(std::type_index type); - const String getTypeName() const { return getTypeName(getType()); } - static const char * getKeyword(std::type_index type); - const char * getKeyword() const { return getKeyword(getType()); } + enum class Type + { + USER, + ROLE, + SETTINGS_PROFILE, + ROW_POLICY, + QUOTA, - template - bool isTypeOf() const { return isTypeOf(typeid(EntityType)); } - bool isTypeOf(std::type_index type) const { return type == getType(); } + MAX, + }; + + virtual Type getType() const = 0; + + struct TypeInfo + { + const char * const raw_name; + const String name; /// Uppercased with spaces instead of underscores, e.g. "SETTINGS PROFILE". + const String alias; /// Alias of the keyword or empty string, e.g. "PROFILE". + const String name_for_output_with_entity_name; /// Lowercased with spaces instead of underscores, e.g. "settings profile". + const char unique_char; /// Unique character for this type. E.g. 'P' for SETTINGS_PROFILE. + const String list_filename; /// Name of the file containing list of objects of this type, including the file extension ".list". + const int not_found_error_code; + + static const TypeInfo & get(Type type_); + String outputWithEntityName(const String & entity_name) const; + }; + + const TypeInfo & getTypeInfo() const { return TypeInfo::get(getType()); } + String outputTypeAndName() const { return getTypeInfo().outputWithEntityName(getName()); } + + template + bool isTypeOf() const { return isTypeOf(EntityClassT::TYPE); } + bool isTypeOf(Type type) const { return type == getType(); } virtual void setName(const String & name_) { name = name_; } const String & getName() const { return name; } @@ -39,12 +75,74 @@ protected: virtual bool equal(const IAccessEntity & other) const; /// Helper function to define clone() in the derived classes. - template + template std::shared_ptr cloneImpl() const { - return std::make_shared(typeid_cast(*this)); + return std::make_shared(typeid_cast(*this)); } }; using AccessEntityPtr = std::shared_ptr; + + +inline const IAccessEntity::TypeInfo & IAccessEntity::TypeInfo::get(Type type_) +{ + static constexpr auto make_info = [](const char * raw_name_, char unique_char_, const char * list_filename_, int not_found_error_code_) + { + String init_name = raw_name_; + boost::to_upper(init_name); + boost::replace_all(init_name, "_", " "); + String init_alias; + if (auto underscore_pos = init_name.find_first_of(" "); underscore_pos != String::npos) + init_alias = init_name.substr(underscore_pos + 1); + String init_name_for_output_with_entity_name = init_name; + boost::to_lower(init_name_for_output_with_entity_name); + return TypeInfo{raw_name_, std::move(init_name), std::move(init_alias), std::move(init_name_for_output_with_entity_name), unique_char_, list_filename_, not_found_error_code_}; + }; + + switch (type_) + { + case Type::USER: + { + static const auto info = make_info("USER", 'U', "users.list", ErrorCodes::UNKNOWN_USER); + return info; + } + case Type::ROLE: + { + static const auto info = make_info("ROLE", 'R', "roles.list", ErrorCodes::UNKNOWN_ROLE); + return info; + } + case Type::SETTINGS_PROFILE: + { + static const auto info = make_info("SETTINGS_PROFILE", 'S', "settings_profiles.list", ErrorCodes::THERE_IS_NO_PROFILE); + return info; + } + case Type::ROW_POLICY: + { + static const auto info = make_info("ROW_POLICY", 'P', "row_policies.list", ErrorCodes::UNKNOWN_ROW_POLICY); + return info; + } + case Type::QUOTA: + { + static const auto info = make_info("QUOTA", 'Q', "quotas.list", ErrorCodes::UNKNOWN_QUOTA); + return info; + } + case Type::MAX: break; + } + throw Exception("Unknown type: " + std::to_string(static_cast(type_)), ErrorCodes::LOGICAL_ERROR); +} + +inline String IAccessEntity::TypeInfo::outputWithEntityName(const String & entity_name) const +{ + String msg = name_for_output_with_entity_name; + msg += " "; + msg += backQuote(entity_name); + return msg; +} + +inline String toString(IAccessEntity::Type type) +{ + return IAccessEntity::TypeInfo::get(type).name; +} + } diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index 3dfc3e232ba..8e4314ec7c5 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -13,27 +13,44 @@ namespace DB namespace ErrorCodes { extern const int BAD_CAST; - extern const int ACCESS_ENTITY_NOT_FOUND; extern const int ACCESS_ENTITY_ALREADY_EXISTS; + extern const int ACCESS_ENTITY_NOT_FOUND; extern const int ACCESS_STORAGE_READONLY; - extern const int UNKNOWN_USER; - extern const int UNKNOWN_ROLE; } -std::vector IAccessStorage::findAll(std::type_index type) const +namespace +{ + using EntityType = IAccessStorage::EntityType; + using EntityTypeInfo = IAccessStorage::EntityTypeInfo; + + bool isNotFoundErrorCode(int error_code) + { + if (error_code == ErrorCodes::ACCESS_ENTITY_NOT_FOUND) + return true; + + for (auto type : ext::range(EntityType::MAX)) + if (error_code == EntityTypeInfo::get(type).not_found_error_code) + return true; + + return false; + } +} + + +std::vector IAccessStorage::findAll(EntityType type) const { return findAllImpl(type); } -std::optional IAccessStorage::find(std::type_index type, const String & name) const +std::optional IAccessStorage::find(EntityType type, const String & name) const { return findImpl(type, name); } -std::vector IAccessStorage::find(std::type_index type, const Strings & names) const +std::vector IAccessStorage::find(EntityType type, const Strings & names) const { std::vector ids; ids.reserve(names.size()); @@ -47,7 +64,7 @@ std::vector IAccessStorage::find(std::type_index type, const Strings & nam } -UUID IAccessStorage::getID(std::type_index type, const String & name) const +UUID IAccessStorage::getID(EntityType type, const String & name) const { auto id = findImpl(type, name); if (id) @@ -56,7 +73,7 @@ UUID IAccessStorage::getID(std::type_index type, const String & name) const } -std::vector IAccessStorage::getIDs(std::type_index type, const Strings & names) const +std::vector IAccessStorage::getIDs(EntityType type, const Strings & names) const { std::vector ids; ids.reserve(names.size()); @@ -190,6 +207,7 @@ void IAccessStorage::remove(const UUID & id) void IAccessStorage::remove(const std::vector & ids) { String error_message; + std::optional error_code; for (const auto & id : ids) { try @@ -198,13 +216,17 @@ void IAccessStorage::remove(const std::vector & ids) } catch (Exception & e) { - if (e.code() != ErrorCodes::ACCESS_ENTITY_NOT_FOUND) + if (!isNotFoundErrorCode(e.code())) throw; error_message += (error_message.empty() ? "" : ". ") + e.message(); + if (error_code && (*error_code != e.code())) + error_code = ErrorCodes::ACCESS_ENTITY_NOT_FOUND; + else + error_code = e.code(); } } if (!error_message.empty()) - throw Exception(error_message, ErrorCodes::ACCESS_ENTITY_NOT_FOUND); + throw Exception(error_message, *error_code); } @@ -250,6 +272,7 @@ void IAccessStorage::update(const UUID & id, const UpdateFunc & update_func) void IAccessStorage::update(const std::vector & ids, const UpdateFunc & update_func) { String error_message; + std::optional error_code; for (const auto & id : ids) { try @@ -258,13 +281,17 @@ void IAccessStorage::update(const std::vector & ids, const UpdateFunc & up } catch (Exception & e) { - if (e.code() != ErrorCodes::ACCESS_ENTITY_NOT_FOUND) + if (!isNotFoundErrorCode(e.code())) throw; error_message += (error_message.empty() ? "" : ". ") + e.message(); + if (error_code && (*error_code != e.code())) + error_code = ErrorCodes::ACCESS_ENTITY_NOT_FOUND; + else + error_code = e.code(); } } if (!error_message.empty()) - throw Exception(error_message, ErrorCodes::ACCESS_ENTITY_NOT_FOUND); + throw Exception(error_message, *error_code); } @@ -301,7 +328,7 @@ std::vector IAccessStorage::tryUpdate(const std::vector & ids, const } -ext::scope_guard IAccessStorage::subscribeForChanges(std::type_index type, const OnChangedHandler & handler) const +ext::scope_guard IAccessStorage::subscribeForChanges(EntityType type, const OnChangedHandler & handler) const { return subscribeForChangesImpl(type, handler); } @@ -322,7 +349,7 @@ ext::scope_guard IAccessStorage::subscribeForChanges(const std::vector & i } -bool IAccessStorage::hasSubscription(std::type_index type) const +bool IAccessStorage::hasSubscription(EntityType type) const { return hasSubscriptionImpl(type); } @@ -361,79 +388,72 @@ Poco::Logger * IAccessStorage::getLogger() const void IAccessStorage::throwNotFound(const UUID & id) const { - throw Exception("ID {" + toString(id) + "} not found in " + getStorageName(), ErrorCodes::ACCESS_ENTITY_NOT_FOUND); + throw Exception("ID {" + toString(id) + "} not found in [" + getStorageName() + "]", ErrorCodes::ACCESS_ENTITY_NOT_FOUND); } -void IAccessStorage::throwNotFound(std::type_index type, const String & name) const +void IAccessStorage::throwNotFound(EntityType type, const String & name) const { - int error_code; - if (type == typeid(User)) - error_code = ErrorCodes::UNKNOWN_USER; - else if (type == typeid(Role)) - error_code = ErrorCodes::UNKNOWN_ROLE; - else - error_code = ErrorCodes::ACCESS_ENTITY_NOT_FOUND; - - throw Exception(getTypeName(type) + " " + backQuote(name) + " not found in " + getStorageName(), error_code); + int error_code = EntityTypeInfo::get(type).not_found_error_code; + throw Exception("There is no " + outputEntityTypeAndName(type, name) + " in [" + getStorageName() + "]", error_code); } -void IAccessStorage::throwBadCast(const UUID & id, std::type_index type, const String & name, std::type_index required_type) +void IAccessStorage::throwBadCast(const UUID & id, EntityType type, const String & name, EntityType required_type) { throw Exception( - "ID {" + toString(id) + "}: " + getTypeName(type) + backQuote(name) + " expected to be of type " + getTypeName(required_type), + "ID {" + toString(id) + "}: " + outputEntityTypeAndName(type, name) + " expected to be of type " + toString(required_type), ErrorCodes::BAD_CAST); } -void IAccessStorage::throwIDCollisionCannotInsert(const UUID & id, std::type_index type, const String & name, std::type_index existing_type, const String & existing_name) const +void IAccessStorage::throwIDCollisionCannotInsert(const UUID & id, EntityType type, const String & name, EntityType existing_type, const String & existing_name) const { throw Exception( - getTypeName(type) + " " + backQuote(name) + ": cannot insert because the ID {" + toString(id) + "} is already used by " - + getTypeName(existing_type) + " " + backQuote(existing_name) + " in " + getStorageName(), + outputEntityTypeAndName(type, name) + ": cannot insert because the ID {" + toString(id) + "} is already used by " + + outputEntityTypeAndName(existing_type, existing_name) + " in [" + getStorageName() + "]", ErrorCodes::ACCESS_ENTITY_ALREADY_EXISTS); } -void IAccessStorage::throwNameCollisionCannotInsert(std::type_index type, const String & name) const +void IAccessStorage::throwNameCollisionCannotInsert(EntityType type, const String & name) const { throw Exception( - getTypeName(type) + " " + backQuote(name) + ": cannot insert because " + getTypeName(type) + " " + backQuote(name) - + " already exists in " + getStorageName(), + outputEntityTypeAndName(type, name) + ": cannot insert because " + outputEntityTypeAndName(type, name) + " already exists in [" + + getStorageName() + "]", ErrorCodes::ACCESS_ENTITY_ALREADY_EXISTS); } -void IAccessStorage::throwNameCollisionCannotRename(std::type_index type, const String & old_name, const String & new_name) const +void IAccessStorage::throwNameCollisionCannotRename(EntityType type, const String & old_name, const String & new_name) const { throw Exception( - getTypeName(type) + " " + backQuote(old_name) + ": cannot rename to " + backQuote(new_name) + " because " + getTypeName(type) + " " - + backQuote(new_name) + " already exists in " + getStorageName(), + outputEntityTypeAndName(type, old_name) + ": cannot rename to " + backQuote(new_name) + " because " + + outputEntityTypeAndName(type, new_name) + " already exists in [" + getStorageName() + "]", ErrorCodes::ACCESS_ENTITY_ALREADY_EXISTS); } -void IAccessStorage::throwReadonlyCannotInsert(std::type_index type, const String & name) const +void IAccessStorage::throwReadonlyCannotInsert(EntityType type, const String & name) const { throw Exception( - "Cannot insert " + getTypeName(type) + " " + backQuote(name) + " to " + getStorageName() + " because this storage is readonly", + "Cannot insert " + outputEntityTypeAndName(type, name) + " to [" + getStorageName() + "] because this storage is readonly", ErrorCodes::ACCESS_STORAGE_READONLY); } -void IAccessStorage::throwReadonlyCannotUpdate(std::type_index type, const String & name) const +void IAccessStorage::throwReadonlyCannotUpdate(EntityType type, const String & name) const { throw Exception( - "Cannot update " + getTypeName(type) + " " + backQuote(name) + " in " + getStorageName() + " because this storage is readonly", + "Cannot update " + outputEntityTypeAndName(type, name) + " in [" + getStorageName() + "] because this storage is readonly", ErrorCodes::ACCESS_STORAGE_READONLY); } -void IAccessStorage::throwReadonlyCannotRemove(std::type_index type, const String & name) const +void IAccessStorage::throwReadonlyCannotRemove(EntityType type, const String & name) const { throw Exception( - "Cannot remove " + getTypeName(type) + " " + backQuote(name) + " from " + getStorageName() + " because this storage is readonly", + "Cannot remove " + outputEntityTypeAndName(type, name) + " from [" + getStorageName() + "] because this storage is readonly", ErrorCodes::ACCESS_STORAGE_READONLY); } } diff --git a/src/Access/IAccessStorage.h b/src/Access/IAccessStorage.h index d2413abc4a5..081fed87bd2 100644 --- a/src/Access/IAccessStorage.h +++ b/src/Access/IAccessStorage.h @@ -25,50 +25,53 @@ public: /// Returns the name of this storage. const String & getStorageName() const { return storage_name; } - /// Returns the identifiers of all the entities of a specified type contained in the storage. - std::vector findAll(std::type_index type) const; + using EntityType = IAccessEntity::Type; + using EntityTypeInfo = IAccessEntity::TypeInfo; - template - std::vector findAll() const { return findAll(typeid(EntityType)); } + /// Returns the identifiers of all the entities of a specified type contained in the storage. + std::vector findAll(EntityType type) const; + + template + std::vector findAll() const { return findAll(EntityClassT::TYPE); } /// Searchs for an entity with specified type and name. Returns std::nullopt if not found. - std::optional find(std::type_index type, const String & name) const; + std::optional find(EntityType type, const String & name) const; - template - std::optional find(const String & name) const { return find(typeid(EntityType), name); } + template + std::optional find(const String & name) const { return find(EntityClassT::TYPE, name); } - std::vector find(std::type_index type, const Strings & names) const; + std::vector find(EntityType type, const Strings & names) const; - template - std::vector find(const Strings & names) const { return find(typeid(EntityType), names); } + template + std::vector find(const Strings & names) const { return find(EntityClassT::TYPE, names); } /// Searchs for an entity with specified name and type. Throws an exception if not found. - UUID getID(std::type_index type, const String & name) const; + UUID getID(EntityType type, const String & name) const; - template - UUID getID(const String & name) const { return getID(typeid(EntityType), name); } + template + UUID getID(const String & name) const { return getID(EntityClassT::TYPE, name); } - std::vector getIDs(std::type_index type, const Strings & names) const; + std::vector getIDs(EntityType type, const Strings & names) const; - template - std::vector getIDs(const Strings & names) const { return getIDs(typeid(EntityType), names); } + template + std::vector getIDs(const Strings & names) const { return getIDs(EntityClassT::TYPE, names); } /// Returns whether there is an entity with such identifier in the storage. bool exists(const UUID & id) const; /// Reads an entity. Throws an exception if not found. - template - std::shared_ptr read(const UUID & id) const; + template + std::shared_ptr read(const UUID & id) const; - template - std::shared_ptr read(const String & name) const; + template + std::shared_ptr read(const String & name) const; /// Reads an entity. Returns nullptr if not found. - template - std::shared_ptr tryRead(const UUID & id) const; + template + std::shared_ptr tryRead(const UUID & id) const; - template - std::shared_ptr tryRead(const String & name) const; + template + std::shared_ptr tryRead(const String & name) const; /// Reads only name of an entity. String readName(const UUID & id) const; @@ -118,22 +121,22 @@ public: /// Subscribes for all changes. /// Can return nullptr if cannot subscribe (identifier not found) or if it doesn't make sense (the storage is read-only). - ext::scope_guard subscribeForChanges(std::type_index type, const OnChangedHandler & handler) const; + ext::scope_guard subscribeForChanges(EntityType type, const OnChangedHandler & handler) const; - template - ext::scope_guard subscribeForChanges(OnChangedHandler handler) const { return subscribeForChanges(typeid(EntityType), handler); } + template + ext::scope_guard subscribeForChanges(OnChangedHandler handler) const { return subscribeForChanges(EntityClassT::TYPE, handler); } /// Subscribes for changes of a specific entry. /// Can return nullptr if cannot subscribe (identifier not found) or if it doesn't make sense (the storage is read-only). ext::scope_guard subscribeForChanges(const UUID & id, const OnChangedHandler & handler) const; ext::scope_guard subscribeForChanges(const std::vector & ids, const OnChangedHandler & handler) const; - bool hasSubscription(std::type_index type) const; + bool hasSubscription(EntityType type) const; bool hasSubscription(const UUID & id) const; protected: - virtual std::optional findImpl(std::type_index type, const String & name) const = 0; - virtual std::vector findAllImpl(std::type_index type) const = 0; + virtual std::optional findImpl(EntityType type, const String & name) const = 0; + virtual std::vector findAllImpl(EntityType type) const = 0; virtual bool existsImpl(const UUID & id) const = 0; virtual AccessEntityPtr readImpl(const UUID & id) const = 0; virtual String readNameImpl(const UUID & id) const = 0; @@ -142,23 +145,23 @@ protected: virtual void removeImpl(const UUID & id) = 0; virtual void updateImpl(const UUID & id, const UpdateFunc & update_func) = 0; virtual ext::scope_guard subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const = 0; - virtual ext::scope_guard subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const = 0; + virtual ext::scope_guard subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const = 0; virtual bool hasSubscriptionImpl(const UUID & id) const = 0; - virtual bool hasSubscriptionImpl(std::type_index type) const = 0; + virtual bool hasSubscriptionImpl(EntityType type) const = 0; static UUID generateRandomID(); Poco::Logger * getLogger() const; - static String getTypeName(std::type_index type) { return IAccessEntity::getTypeName(type); } + static String outputEntityTypeAndName(EntityType type, const String & name) { return EntityTypeInfo::get(type).outputWithEntityName(name); } [[noreturn]] void throwNotFound(const UUID & id) const; - [[noreturn]] void throwNotFound(std::type_index type, const String & name) const; - [[noreturn]] static void throwBadCast(const UUID & id, std::type_index type, const String & name, std::type_index required_type); + [[noreturn]] void throwNotFound(EntityType type, const String & name) const; + [[noreturn]] static void throwBadCast(const UUID & id, EntityType type, const String & name, EntityType required_type); [[noreturn]] void throwIDCollisionCannotInsert( - const UUID & id, std::type_index type, const String & name, std::type_index existing_type, const String & existing_name) const; - [[noreturn]] void throwNameCollisionCannotInsert(std::type_index type, const String & name) const; - [[noreturn]] void throwNameCollisionCannotRename(std::type_index type, const String & old_name, const String & new_name) const; - [[noreturn]] void throwReadonlyCannotInsert(std::type_index type, const String & name) const; - [[noreturn]] void throwReadonlyCannotUpdate(std::type_index type, const String & name) const; - [[noreturn]] void throwReadonlyCannotRemove(std::type_index type, const String & name) const; + const UUID & id, EntityType type, const String & name, EntityType existing_type, const String & existing_name) const; + [[noreturn]] void throwNameCollisionCannotInsert(EntityType type, const String & name) const; + [[noreturn]] void throwNameCollisionCannotRename(EntityType type, const String & old_name, const String & new_name) const; + [[noreturn]] void throwReadonlyCannotInsert(EntityType type, const String & name) const; + [[noreturn]] void throwReadonlyCannotUpdate(EntityType type, const String & name) const; + [[noreturn]] void throwReadonlyCannotRemove(EntityType type, const String & name) const; using Notification = std::tuple; using Notifications = std::vector; @@ -172,38 +175,43 @@ private: }; -template -std::shared_ptr IAccessStorage::read(const UUID & id) const +template +std::shared_ptr IAccessStorage::read(const UUID & id) const { auto entity = readImpl(id); - auto ptr = typeid_cast>(entity); - if (ptr) - return ptr; - throwBadCast(id, entity->getType(), entity->getName(), typeid(EntityType)); + if constexpr (std::is_same_v) + return entity; + else + { + auto ptr = typeid_cast>(entity); + if (ptr) + return ptr; + throwBadCast(id, entity->getType(), entity->getName(), EntityClassT::TYPE); + } } -template -std::shared_ptr IAccessStorage::read(const String & name) const +template +std::shared_ptr IAccessStorage::read(const String & name) const { - return read(getID(name)); + return read(getID(name)); } -template -std::shared_ptr IAccessStorage::tryRead(const UUID & id) const +template +std::shared_ptr IAccessStorage::tryRead(const UUID & id) const { auto entity = tryReadBase(id); if (!entity) return nullptr; - return typeid_cast>(entity); + return typeid_cast>(entity); } -template -std::shared_ptr IAccessStorage::tryRead(const String & name) const +template +std::shared_ptr IAccessStorage::tryRead(const String & name) const { - auto id = find(name); - return id ? tryRead(*id) : nullptr; + auto id = find(name); + return id ? tryRead(*id) : nullptr; } } diff --git a/src/Access/MemoryAccessStorage.cpp b/src/Access/MemoryAccessStorage.cpp index 46efd25191b..720b82796b7 100644 --- a/src/Access/MemoryAccessStorage.cpp +++ b/src/Access/MemoryAccessStorage.cpp @@ -1,6 +1,8 @@ #include #include -#include +#include +#include +#include namespace DB @@ -11,11 +13,12 @@ MemoryAccessStorage::MemoryAccessStorage(const String & storage_name_) } -std::optional MemoryAccessStorage::findImpl(std::type_index type, const String & name) const +std::optional MemoryAccessStorage::findImpl(EntityType type, const String & name) const { std::lock_guard lock{mutex}; - auto it = names.find({name, type}); - if (it == names.end()) + const auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; + auto it = entries_by_name.find(name); + if (it == entries_by_name.end()) return {}; Entry & entry = *(it->second); @@ -23,12 +26,12 @@ std::optional MemoryAccessStorage::findImpl(std::type_index type, const St } -std::vector MemoryAccessStorage::findAllImpl(std::type_index type) const +std::vector MemoryAccessStorage::findAllImpl(EntityType type) const { std::lock_guard lock{mutex}; std::vector result; - result.reserve(entries.size()); - for (const auto & [id, entry] : entries) + result.reserve(entries_by_id.size()); + for (const auto & [id, entry] : entries_by_id) if (entry.entity->isTypeOf(type)) result.emplace_back(id); return result; @@ -38,15 +41,15 @@ std::vector MemoryAccessStorage::findAllImpl(std::type_index type) const bool MemoryAccessStorage::existsImpl(const UUID & id) const { std::lock_guard lock{mutex}; - return entries.count(id); + return entries_by_id.count(id); } AccessEntityPtr MemoryAccessStorage::readImpl(const UUID & id) const { std::lock_guard lock{mutex}; - auto it = entries.find(id); - if (it == entries.end()) + auto it = entries_by_id.find(id); + if (it == entries_by_id.end()) throwNotFound(id); const Entry & entry = it->second; return entry.entity; @@ -74,18 +77,19 @@ UUID MemoryAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool re void MemoryAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, Notifications & notifications) { const String & name = new_entity->getName(); - std::type_index type = new_entity->getType(); + EntityType type = new_entity->getType(); /// Check that we can insert. - auto it = entries.find(id); - if (it != entries.end()) + auto it = entries_by_id.find(id); + if (it != entries_by_id.end()) { const auto & existing_entry = it->second; throwIDCollisionCannotInsert(id, type, name, existing_entry.entity->getType(), existing_entry.entity->getName()); } - auto it2 = names.find({name, type}); - if (it2 != names.end()) + auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; + auto it2 = entries_by_name.find(name); + if (it2 != entries_by_name.end()) { const auto & existing_entry = *(it2->second); if (replace_if_exists) @@ -95,10 +99,10 @@ void MemoryAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & } /// Do insertion. - auto & entry = entries[id]; + auto & entry = entries_by_id[id]; entry.id = id; entry.entity = new_entity; - names[std::pair{name, type}] = &entry; + entries_by_name[name] = &entry; prepareNotifications(entry, false, notifications); } @@ -115,19 +119,20 @@ void MemoryAccessStorage::removeImpl(const UUID & id) void MemoryAccessStorage::removeNoLock(const UUID & id, Notifications & notifications) { - auto it = entries.find(id); - if (it == entries.end()) + auto it = entries_by_id.find(id); + if (it == entries_by_id.end()) throwNotFound(id); Entry & entry = it->second; const String & name = entry.entity->getName(); - std::type_index type = entry.entity->getType(); + EntityType type = entry.entity->getType(); prepareNotifications(entry, true, notifications); /// Do removing. - names.erase({name, type}); - entries.erase(it); + auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; + entries_by_name.erase(name); + entries_by_id.erase(it); } @@ -143,14 +148,17 @@ void MemoryAccessStorage::updateImpl(const UUID & id, const UpdateFunc & update_ void MemoryAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & update_func, Notifications & notifications) { - auto it = entries.find(id); - if (it == entries.end()) + auto it = entries_by_id.find(id); + if (it == entries_by_id.end()) throwNotFound(id); Entry & entry = it->second; auto old_entity = entry.entity; auto new_entity = update_func(old_entity); + if (!new_entity->isTypeOf(old_entity->getType())) + throwBadCast(id, new_entity->getType(), new_entity->getName(), old_entity->getType()); + if (*new_entity == *old_entity) return; @@ -158,12 +166,12 @@ void MemoryAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & updat if (new_entity->getName() != old_entity->getName()) { - auto it2 = names.find({new_entity->getName(), new_entity->getType()}); - if (it2 != names.end()) + auto & entries_by_name = entries_by_name_and_type[static_cast(old_entity->getType())]; + auto it2 = entries_by_name.find(new_entity->getName()); + if (it2 != entries_by_name.end()) throwNameCollisionCannotRename(old_entity->getType(), old_entity->getName(), new_entity->getName()); - names.erase({old_entity->getName(), old_entity->getType()}); - names[std::pair{new_entity->getName(), new_entity->getType()}] = &entry; + entries_by_name[new_entity->getName()] = &entry; } prepareNotifications(entry, false, notifications); @@ -192,43 +200,47 @@ void MemoryAccessStorage::setAll(const std::vector> & all_entities, Notifications & notifications) { - /// Get list of the currently used IDs. Later we will remove those of them which are not used anymore. - std::unordered_set not_used_ids; - for (const auto & id_and_entry : entries) - not_used_ids.emplace(id_and_entry.first); + boost::container::flat_set not_used_ids; + std::vector conflicting_ids; - /// Remove conflicting entities. + /// Get the list of currently used IDs. Later we will remove those of them which are not used anymore. + for (const auto & id : entries_by_id | boost::adaptors::map_keys) + not_used_ids.emplace(id); + + /// Get the list of conflicting IDs and update the list of currently used ones. for (const auto & [id, entity] : all_entities) { - auto it = entries.find(id); - if (it != entries.end()) + auto it = entries_by_id.find(id); + if (it != entries_by_id.end()) { not_used_ids.erase(id); /// ID is used. + Entry & entry = it->second; if (entry.entity->getType() != entity->getType()) - { - removeNoLock(id, notifications); - continue; - } + conflicting_ids.emplace_back(id); /// Conflict: same ID, different type. } - auto it2 = names.find({entity->getName(), entity->getType()}); - if (it2 != names.end()) + + const auto & entries_by_name = entries_by_name_and_type[static_cast(entity->getType())]; + auto it2 = entries_by_name.find(entity->getName()); + if (it2 != entries_by_name.end()) { Entry & entry = *(it2->second); if (entry.id != id) - removeNoLock(id, notifications); + conflicting_ids.emplace_back(entry.id); /// Conflict: same name and type, different ID. } } - /// Remove entities which are not used anymore. - for (const auto & id : not_used_ids) + /// Remove entities which are not used anymore and which are in conflict with new entities. + boost::container::flat_set ids_to_remove = std::move(not_used_ids); + boost::range::copy(conflicting_ids, std::inserter(ids_to_remove, ids_to_remove.end())); + for (const auto & id : ids_to_remove) removeNoLock(id, notifications); /// Insert or update entities. for (const auto & [id, entity] : all_entities) { - auto it = entries.find(id); - if (it != entries.end()) + auto it = entries_by_id.find(id); + if (it != entries_by_id.end()) { if (*(it->second.entity) != *entity) { @@ -244,24 +256,27 @@ void MemoryAccessStorage::setAllNoLock(const std::vectorgetType()); - for (auto it = range.first; it != range.second; ++it) - notifications.push_back({it->second, entry.id, remove ? nullptr : entry.entity}); + for (const auto & handler : handlers_by_type[static_cast(entry.entity->getType())]) + notifications.push_back({handler, entry.id, entity}); } -ext::scope_guard MemoryAccessStorage::subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const +ext::scope_guard MemoryAccessStorage::subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const { std::lock_guard lock{mutex}; - auto handler_it = handlers_by_type.emplace(type, handler); + auto & handlers = handlers_by_type[static_cast(type)]; + handlers.push_back(handler); + auto handler_it = std::prev(handlers.end()); - return [this, handler_it] + return [this, type, handler_it] { std::lock_guard lock2{mutex}; - handlers_by_type.erase(handler_it); + auto & handlers2 = handlers_by_type[static_cast(type)]; + handlers2.erase(handler_it); }; } @@ -269,8 +284,8 @@ ext::scope_guard MemoryAccessStorage::subscribeForChangesImpl(std::type_index ty ext::scope_guard MemoryAccessStorage::subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const { std::lock_guard lock{mutex}; - auto it = entries.find(id); - if (it == entries.end()) + auto it = entries_by_id.find(id); + if (it == entries_by_id.end()) return {}; const Entry & entry = it->second; auto handler_it = entry.handlers_by_id.insert(entry.handlers_by_id.end(), handler); @@ -278,8 +293,8 @@ ext::scope_guard MemoryAccessStorage::subscribeForChangesImpl(const UUID & id, c return [this, id, handler_it] { std::lock_guard lock2{mutex}; - auto it2 = entries.find(id); - if (it2 != entries.end()) + auto it2 = entries_by_id.find(id); + if (it2 != entries_by_id.end()) { const Entry & entry2 = it2->second; entry2.handlers_by_id.erase(handler_it); @@ -291,8 +306,8 @@ ext::scope_guard MemoryAccessStorage::subscribeForChangesImpl(const UUID & id, c bool MemoryAccessStorage::hasSubscriptionImpl(const UUID & id) const { std::lock_guard lock{mutex}; - auto it = entries.find(id); - if (it != entries.end()) + auto it = entries_by_id.find(id); + if (it != entries_by_id.end()) { const Entry & entry = it->second; return !entry.handlers_by_id.empty(); @@ -301,10 +316,10 @@ bool MemoryAccessStorage::hasSubscriptionImpl(const UUID & id) const } -bool MemoryAccessStorage::hasSubscriptionImpl(std::type_index type) const +bool MemoryAccessStorage::hasSubscriptionImpl(EntityType type) const { std::lock_guard lock{mutex}; - auto range = handlers_by_type.equal_range(type); - return range.first != range.second; + const auto & handlers = handlers_by_type[static_cast(type)]; + return !handlers.empty(); } } diff --git a/src/Access/MemoryAccessStorage.h b/src/Access/MemoryAccessStorage.h index b93c2868d34..a2fdd0d0044 100644 --- a/src/Access/MemoryAccessStorage.h +++ b/src/Access/MemoryAccessStorage.h @@ -20,8 +20,8 @@ public: void setAll(const std::vector> & all_entities); private: - std::optional findImpl(std::type_index type, const String & name) const override; - std::vector findAllImpl(std::type_index type) const override; + std::optional findImpl(EntityType type, const String & name) const override; + std::vector findAllImpl(EntityType type) const override; bool existsImpl(const UUID & id) const override; AccessEntityPtr readImpl(const UUID & id) const override; String readNameImpl(const UUID & id) const override; @@ -30,9 +30,9 @@ private: void removeImpl(const UUID & id) override; void updateImpl(const UUID & id, const UpdateFunc & update_func) override; ext::scope_guard subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override; - ext::scope_guard subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const override; + ext::scope_guard subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const override; bool hasSubscriptionImpl(const UUID & id) const override; - bool hasSubscriptionImpl(std::type_index type) const override; + bool hasSubscriptionImpl(EntityType type) const override; struct Entry { @@ -47,18 +47,9 @@ private: void setAllNoLock(const std::vector> & all_entities, Notifications & notifications); void prepareNotifications(const Entry & entry, bool remove, Notifications & notifications) const; - using NameTypePair = std::pair; - struct Hash - { - size_t operator()(const NameTypePair & key) const - { - return std::hash{}(key.first) - std::hash{}(key.second); - } - }; - mutable std::mutex mutex; - std::unordered_map entries; /// We want to search entries both by ID and by the pair of name and type. - std::unordered_map names; /// and by the pair of name and type. - mutable std::unordered_multimap handlers_by_type; + std::unordered_map entries_by_id; /// We want to search entries both by ID and by the pair of name and type. + std::unordered_map entries_by_name_and_type[static_cast(EntityType::MAX)]; + mutable std::list handlers_by_type[static_cast(EntityType::MAX)]; }; } diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index 740fe1dac04..0dd1f142f31 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -38,7 +38,7 @@ MultipleAccessStorage::MultipleAccessStorage( } -std::vector MultipleAccessStorage::findMultiple(std::type_index type, const String & name) const +std::vector MultipleAccessStorage::findMultiple(EntityType type, const String & name) const { std::vector ids; for (const auto & nested_storage : nested_storages) @@ -55,7 +55,7 @@ std::vector MultipleAccessStorage::findMultiple(std::type_index type, cons } -std::optional MultipleAccessStorage::findImpl(std::type_index type, const String & name) const +std::optional MultipleAccessStorage::findImpl(EntityType type, const String & name) const { auto ids = findMultiple(type, name); if (ids.empty()) @@ -72,13 +72,13 @@ std::optional MultipleAccessStorage::findImpl(std::type_index type, const } throw Exception( - "Found " + getTypeName(type) + " " + backQuote(name) + " in " + std::to_string(ids.size()) - + " storages: " + joinStorageNames(storages_with_duplicates), + "Found " + outputEntityTypeAndName(type, name) + " in " + std::to_string(ids.size()) + + " storages [" + joinStorageNames(storages_with_duplicates) + "]", ErrorCodes::ACCESS_ENTITY_FOUND_DUPLICATES); } -std::vector MultipleAccessStorage::findAllImpl(std::type_index type) const +std::vector MultipleAccessStorage::findAllImpl(EntityType type) const { std::vector all_ids; for (const auto & nested_storage : nested_storages) @@ -180,11 +180,7 @@ UUID MultipleAccessStorage::insertImpl(const AccessEntityPtr & entity, bool repl } if (!nested_storage_for_insertion) - { - throw Exception( - "Not found a storage to insert " + entity->getTypeName() + backQuote(entity->getName()), - ErrorCodes::ACCESS_STORAGE_FOR_INSERTION_NOT_FOUND); - } + throw Exception("Not found a storage to insert " + entity->outputTypeAndName(), ErrorCodes::ACCESS_STORAGE_FOR_INSERTION_NOT_FOUND); auto id = replace_if_exists ? nested_storage_for_insertion->insertOrReplace(entity) : nested_storage_for_insertion->insert(entity); std::lock_guard lock{ids_cache_mutex}; @@ -214,7 +210,7 @@ ext::scope_guard MultipleAccessStorage::subscribeForChangesImpl(const UUID & id, } -ext::scope_guard MultipleAccessStorage::subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const +ext::scope_guard MultipleAccessStorage::subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const { ext::scope_guard subscriptions; for (const auto & nested_storage : nested_storages) @@ -234,7 +230,7 @@ bool MultipleAccessStorage::hasSubscriptionImpl(const UUID & id) const } -bool MultipleAccessStorage::hasSubscriptionImpl(std::type_index type) const +bool MultipleAccessStorage::hasSubscriptionImpl(EntityType type) const { for (const auto & nested_storage : nested_storages) { diff --git a/src/Access/MultipleAccessStorage.h b/src/Access/MultipleAccessStorage.h index 898d55d30de..ec8c8f2a101 100644 --- a/src/Access/MultipleAccessStorage.h +++ b/src/Access/MultipleAccessStorage.h @@ -15,7 +15,7 @@ public: MultipleAccessStorage(std::vector> nested_storages_); - std::vector findMultiple(std::type_index type, const String & name) const; + std::vector findMultiple(EntityType type, const String & name) const; template std::vector findMultiple(const String & name) const { return findMultiple(EntityType::TYPE, name); } @@ -29,8 +29,8 @@ public: const Storage & getStorageByIndex(size_t i) const { return *(nested_storages[i]); } protected: - std::optional findImpl(std::type_index type, const String & name) const override; - std::vector findAllImpl(std::type_index type) const override; + std::optional findImpl(EntityType type, const String & name) const override; + std::vector findAllImpl(EntityType type) const override; bool existsImpl(const UUID & id) const override; AccessEntityPtr readImpl(const UUID & id) const override; String readNameImpl(const UUID &id) const override; @@ -39,9 +39,9 @@ protected: void removeImpl(const UUID & id) override; void updateImpl(const UUID & id, const UpdateFunc & update_func) override; ext::scope_guard subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override; - ext::scope_guard subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const override; + ext::scope_guard subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const override; bool hasSubscriptionImpl(const UUID & id) const override; - bool hasSubscriptionImpl(std::type_index type) const override; + bool hasSubscriptionImpl(EntityType type) const override; private: std::vector> nested_storages; diff --git a/src/Access/Quota.h b/src/Access/Quota.h index a3666aa9b52..317ed2dbc47 100644 --- a/src/Access/Quota.h +++ b/src/Access/Quota.h @@ -75,6 +75,8 @@ struct Quota : public IAccessEntity bool equal(const IAccessEntity & other) const override; std::shared_ptr clone() const override { return cloneImpl(); } + static constexpr const Type TYPE = Type::QUOTA; + Type getType() const override { return TYPE; } static const char * getNameOfResourceType(ResourceType resource_type); static const char * resourceTypeToKeyword(ResourceType resource_type); diff --git a/src/Access/Role.cpp b/src/Access/Role.cpp index d7bec28c576..3df562ad1f0 100644 --- a/src/Access/Role.cpp +++ b/src/Access/Role.cpp @@ -11,4 +11,5 @@ bool Role::equal(const IAccessEntity & other) const const auto & other_role = typeid_cast(other); return (access == other_role.access) && (granted_roles == other_role.granted_roles) && (settings == other_role.settings); } + } diff --git a/src/Access/Role.h b/src/Access/Role.h index 01a5c6ea2ce..9acb97bdfbd 100644 --- a/src/Access/Role.h +++ b/src/Access/Role.h @@ -17,6 +17,8 @@ struct Role : public IAccessEntity bool equal(const IAccessEntity & other) const override; std::shared_ptr clone() const override { return cloneImpl(); } + static constexpr const Type TYPE = Type::ROLE; + Type getType() const override { return TYPE; } }; using RolePtr = std::shared_ptr; diff --git a/src/Access/RowPolicy.h b/src/Access/RowPolicy.h index b3d490e2dbe..9d582d27045 100644 --- a/src/Access/RowPolicy.h +++ b/src/Access/RowPolicy.h @@ -69,6 +69,8 @@ struct RowPolicy : public IAccessEntity bool equal(const IAccessEntity & other) const override; std::shared_ptr clone() const override { return cloneImpl(); } + static constexpr const Type TYPE = Type::ROW_POLICY; + Type getType() const override { return TYPE; } /// Which roles or users should use this row policy. ExtendedRoleSet to_roles; diff --git a/src/Access/SettingsProfile.cpp b/src/Access/SettingsProfile.cpp index c2f868502c0..64fb91eb66b 100644 --- a/src/Access/SettingsProfile.cpp +++ b/src/Access/SettingsProfile.cpp @@ -3,6 +3,7 @@ namespace DB { + bool SettingsProfile::equal(const IAccessEntity & other) const { if (!IAccessEntity::equal(other)) @@ -10,4 +11,5 @@ bool SettingsProfile::equal(const IAccessEntity & other) const const auto & other_profile = typeid_cast(other); return (elements == other_profile.elements) && (to_roles == other_profile.to_roles); } + } diff --git a/src/Access/SettingsProfile.h b/src/Access/SettingsProfile.h index b73b45d57cf..9589b5b3eb5 100644 --- a/src/Access/SettingsProfile.h +++ b/src/Access/SettingsProfile.h @@ -18,6 +18,8 @@ struct SettingsProfile : public IAccessEntity bool equal(const IAccessEntity & other) const override; std::shared_ptr clone() const override { return cloneImpl(); } + static constexpr const Type TYPE = Type::SETTINGS_PROFILE; + Type getType() const override { return TYPE; } }; using SettingsProfilePtr = std::shared_ptr; diff --git a/src/Access/User.cpp b/src/Access/User.cpp index 459357731ed..f57ec7c1359 100644 --- a/src/Access/User.cpp +++ b/src/Access/User.cpp @@ -13,4 +13,5 @@ bool User::equal(const IAccessEntity & other) const && (access == other_user.access) && (granted_roles == other_user.granted_roles) && (default_roles == other_user.default_roles) && (settings == other_user.settings); } + } diff --git a/src/Access/User.h b/src/Access/User.h index b20f6538e4d..da2fb14e131 100644 --- a/src/Access/User.h +++ b/src/Access/User.h @@ -24,6 +24,8 @@ struct User : public IAccessEntity bool equal(const IAccessEntity & other) const override; std::shared_ptr clone() const override { return cloneImpl(); } + static constexpr const Type TYPE = Type::USER; + Type getType() const override { return TYPE; } }; using UserPtr = std::shared_ptr; diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index 7f96d838e32..43a9e355911 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -27,35 +27,24 @@ namespace ErrorCodes namespace { - char getTypeChar(std::type_index type) - { - if (type == typeid(User)) - return 'U'; - if (type == typeid(Quota)) - return 'Q'; - if (type == typeid(RowPolicy)) - return 'P'; - if (type == typeid(SettingsProfile)) - return 'S'; - return 0; - } + using EntityType = IAccessStorage::EntityType; + using EntityTypeInfo = IAccessStorage::EntityTypeInfo; - - UUID generateID(std::type_index type, const String & name) + UUID generateID(EntityType type, const String & name) { Poco::MD5Engine md5; md5.update(name); char type_storage_chars[] = " USRSXML"; - type_storage_chars[0] = getTypeChar(type); + type_storage_chars[0] = EntityTypeInfo::get(type).unique_char; md5.update(type_storage_chars, strlen(type_storage_chars)); UUID result; memcpy(&result, md5.digest().data(), md5.digestLength()); return result; } - UUID generateID(const IAccessEntity & entity) { return generateID(entity.getType(), entity.getName()); } + UserPtr parseUser(const Poco::Util::AbstractConfiguration & config, const String & user_name) { auto user = std::make_shared(); @@ -95,7 +84,7 @@ namespace { auto profile_name = config.getString(profile_name_config); SettingsProfileElement profile_element; - profile_element.parent_profile = generateID(typeid(SettingsProfile), profile_name); + profile_element.parent_profile = generateID(EntityType::SETTINGS_PROFILE, profile_name); user->settings.push_back(std::move(profile_element)); } @@ -260,7 +249,7 @@ namespace for (const auto & user_name : user_names) { if (config.has("users." + user_name + ".quota")) - quota_to_user_ids[config.getString("users." + user_name + ".quota")].push_back(generateID(typeid(User), user_name)); + quota_to_user_ids[config.getString("users." + user_name + ".quota")].push_back(generateID(EntityType::USER, user_name)); } Poco::Util::AbstractConfiguration::Keys quota_names; @@ -346,7 +335,7 @@ namespace auto policy = std::make_shared(); policy->setNameParts(user_name, database, table_name); policy->conditions[RowPolicy::SELECT_FILTER] = filter; - policy->to_roles.add(generateID(typeid(User), user_name)); + policy->to_roles.add(generateID(EntityType::USER, user_name)); policies.push_back(policy); } } @@ -400,7 +389,7 @@ namespace { String parent_profile_name = config.getString(profile_config + "." + key); SettingsProfileElement profile_element; - profile_element.parent_profile = generateID(typeid(SettingsProfile), parent_profile_name); + profile_element.parent_profile = generateID(EntityType::SETTINGS_PROFILE, parent_profile_name); profile->elements.emplace_back(std::move(profile_element)); continue; } @@ -462,13 +451,13 @@ void UsersConfigAccessStorage::setConfiguration(const Poco::Util::AbstractConfig } -std::optional UsersConfigAccessStorage::findImpl(std::type_index type, const String & name) const +std::optional UsersConfigAccessStorage::findImpl(EntityType type, const String & name) const { return memory_storage.find(type, name); } -std::vector UsersConfigAccessStorage::findAllImpl(std::type_index type) const +std::vector UsersConfigAccessStorage::findAllImpl(EntityType type) const { return memory_storage.findAll(type); } @@ -518,7 +507,7 @@ ext::scope_guard UsersConfigAccessStorage::subscribeForChangesImpl(const UUID & } -ext::scope_guard UsersConfigAccessStorage::subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const +ext::scope_guard UsersConfigAccessStorage::subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const { return memory_storage.subscribeForChanges(type, handler); } @@ -530,7 +519,7 @@ bool UsersConfigAccessStorage::hasSubscriptionImpl(const UUID & id) const } -bool UsersConfigAccessStorage::hasSubscriptionImpl(std::type_index type) const +bool UsersConfigAccessStorage::hasSubscriptionImpl(EntityType type) const { return memory_storage.hasSubscription(type); } diff --git a/src/Access/UsersConfigAccessStorage.h b/src/Access/UsersConfigAccessStorage.h index 773d8caa570..d7012cda4ff 100644 --- a/src/Access/UsersConfigAccessStorage.h +++ b/src/Access/UsersConfigAccessStorage.h @@ -23,8 +23,8 @@ public: void setConfiguration(const Poco::Util::AbstractConfiguration & config); private: - std::optional findImpl(std::type_index type, const String & name) const override; - std::vector findAllImpl(std::type_index type) const override; + std::optional findImpl(EntityType type, const String & name) const override; + std::vector findAllImpl(EntityType type) const override; bool existsImpl(const UUID & id) const override; AccessEntityPtr readImpl(const UUID & id) const override; String readNameImpl(const UUID & id) const override; @@ -33,9 +33,9 @@ private: void removeImpl(const UUID & id) override; void updateImpl(const UUID & id, const UpdateFunc & update_func) override; ext::scope_guard subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override; - ext::scope_guard subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const override; + ext::scope_guard subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const override; bool hasSubscriptionImpl(const UUID & id) const override; - bool hasSubscriptionImpl(std::type_index type) const override; + bool hasSubscriptionImpl(EntityType type) const override; MemoryAccessStorage memory_storage; }; diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 120c9b93a78..cc8250158a3 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -493,6 +493,7 @@ namespace ErrorCodes extern const int NO_REMOTE_SHARD_AVAILABLE = 519; extern const int CANNOT_DETACH_DICTIONARY_AS_TABLE = 520; extern const int ATOMIC_RENAME_FAIL = 521; + extern const int UNKNOWN_ROW_POLICY = 522; extern const int KEEPER_EXCEPTION = 999; extern const int POCO_EXCEPTION = 1000; diff --git a/src/Interpreters/InterpreterDropAccessEntityQuery.cpp b/src/Interpreters/InterpreterDropAccessEntityQuery.cpp index 2a47639e15f..be82147a322 100644 --- a/src/Interpreters/InterpreterDropAccessEntityQuery.cpp +++ b/src/Interpreters/InterpreterDropAccessEntityQuery.cpp @@ -9,54 +9,28 @@ #include #include #include -#include namespace DB { -namespace +namespace ErrorCodes { - using Kind = ASTDropAccessEntityQuery::Kind; - - std::type_index getType(Kind kind) - { - switch (kind) - { - case Kind::USER: return typeid(User); - case Kind::ROLE: return typeid(Role); - case Kind::QUOTA: return typeid(Quota); - case Kind::ROW_POLICY: return typeid(RowPolicy); - case Kind::SETTINGS_PROFILE: return typeid(SettingsProfile); - } - __builtin_unreachable(); - } - - AccessType getRequiredAccessType(Kind kind) - { - switch (kind) - { - case Kind::USER: return AccessType::DROP_USER; - case Kind::ROLE: return AccessType::DROP_ROLE; - case Kind::QUOTA: return AccessType::DROP_QUOTA; - case Kind::ROW_POLICY: return AccessType::DROP_ROW_POLICY; - case Kind::SETTINGS_PROFILE: return AccessType::DROP_SETTINGS_PROFILE; - } - __builtin_unreachable(); - } + extern const int NOT_IMPLEMENTED; } +using EntityType = IAccessEntity::Type; + + BlockIO InterpreterDropAccessEntityQuery::execute() { auto & query = query_ptr->as(); auto & access_control = context.getAccessControlManager(); - - std::type_index type = getType(query.kind); - context.checkAccess(getRequiredAccessType(query.kind)); + context.checkAccess(getRequiredAccess()); if (!query.cluster.empty()) return executeDDLQueryOnCluster(query_ptr, context); - if (query.kind == Kind::ROW_POLICY) + if (query.type == EntityType::ROW_POLICY) { Strings names; for (auto & name_parts : query.row_policies_name_parts) @@ -73,10 +47,28 @@ BlockIO InterpreterDropAccessEntityQuery::execute() } if (query.if_exists) - access_control.tryRemove(access_control.find(type, query.names)); + access_control.tryRemove(access_control.find(query.type, query.names)); else - access_control.remove(access_control.getIDs(type, query.names)); + access_control.remove(access_control.getIDs(query.type, query.names)); return {}; } + +AccessRightsElements InterpreterDropAccessEntityQuery::getRequiredAccess() const +{ + const auto & query = query_ptr->as(); + AccessRightsElements res; + switch (query.type) + { + case EntityType::USER: res.emplace_back(AccessType::DROP_USER); return res; + case EntityType::ROLE: res.emplace_back(AccessType::DROP_ROLE); return res; + case EntityType::SETTINGS_PROFILE: res.emplace_back(AccessType::DROP_SETTINGS_PROFILE); return res; + case EntityType::ROW_POLICY: res.emplace_back(AccessType::DROP_ROW_POLICY); return res; + case EntityType::QUOTA: res.emplace_back(AccessType::DROP_QUOTA); return res; + case EntityType::MAX: break; + } + throw Exception( + toString(query.type) + ": type is not supported by DROP query", ErrorCodes::NOT_IMPLEMENTED); +} + } diff --git a/src/Interpreters/InterpreterDropAccessEntityQuery.h b/src/Interpreters/InterpreterDropAccessEntityQuery.h index 2a0e749b265..0db68a0ad78 100644 --- a/src/Interpreters/InterpreterDropAccessEntityQuery.h +++ b/src/Interpreters/InterpreterDropAccessEntityQuery.h @@ -6,6 +6,8 @@ namespace DB { +class AccessRightsElements; + class InterpreterDropAccessEntityQuery : public IInterpreter { public: @@ -14,6 +16,8 @@ public: BlockIO execute() override; private: + AccessRightsElements getRequiredAccess() const; + ASTPtr query_ptr; Context & context; }; diff --git a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp index beea1bf953e..11354d1f2a5 100644 --- a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp +++ b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp @@ -30,9 +30,10 @@ namespace DB { namespace ErrorCodes { - extern const int LOGICAL_ERROR; + extern const int NOT_IMPLEMENTED; } + namespace { ASTPtr getCreateQueryImpl( @@ -203,23 +204,10 @@ namespace return getCreateQueryImpl(*quota, manager, attach_mode); if (const SettingsProfile * profile = typeid_cast(&entity)) return getCreateQueryImpl(*profile, manager, attach_mode); - throw Exception("Unexpected type of access entity: " + entity.getTypeName(), ErrorCodes::LOGICAL_ERROR); + throw Exception(entity.outputTypeAndName() + ": type is not supported by SHOW CREATE query", ErrorCodes::NOT_IMPLEMENTED); } - using Kind = ASTShowCreateAccessEntityQuery::Kind; - - std::type_index getType(Kind kind) - { - switch (kind) - { - case Kind::USER: return typeid(User); - case Kind::ROLE: return typeid(Role); - case Kind::QUOTA: return typeid(Quota); - case Kind::ROW_POLICY: return typeid(RowPolicy); - case Kind::SETTINGS_PROFILE: return typeid(SettingsProfile); - } - __builtin_unreachable(); - } + using EntityType = IAccessEntity::Type; } @@ -274,8 +262,7 @@ ASTPtr InterpreterShowCreateAccessEntityQuery::getCreateQuery(ASTShowCreateAcces return getCreateQueryImpl(*quota, &access_control, false); } - auto type = getType(show_query.kind); - if (show_query.kind == Kind::ROW_POLICY) + if (show_query.type == Type::ROW_POLICY) { if (show_query.row_policy_name_parts.database.empty()) show_query.row_policy_name_parts.database = context.getCurrentDatabase(); @@ -283,30 +270,30 @@ ASTPtr InterpreterShowCreateAccessEntityQuery::getCreateQuery(ASTShowCreateAcces return getCreateQueryImpl(*policy, &access_control, false); } - auto entity = access_control.read(access_control.getID(type, show_query.name)); + auto entity = access_control.read(access_control.getID(show_query.type, show_query.name)); return getCreateQueryImpl(*entity, &access_control, false); } -AccessRightsElements InterpreterShowCreateAccessEntityQuery::getRequiredAccess() const -{ - const auto & show_query = query_ptr->as(); - AccessRightsElements res; - switch (show_query.kind) - { - case Kind::USER: res.emplace_back(AccessType::SHOW_USERS); break; - case Kind::ROLE: res.emplace_back(AccessType::SHOW_ROLES); break; - case Kind::ROW_POLICY: res.emplace_back(AccessType::SHOW_ROW_POLICIES); break; - case Kind::SETTINGS_PROFILE: res.emplace_back(AccessType::SHOW_SETTINGS_PROFILES); break; - case Kind::QUOTA: res.emplace_back(AccessType::SHOW_QUOTAS); break; - } - return res; -} - - ASTPtr InterpreterShowCreateAccessEntityQuery::getAttachQuery(const IAccessEntity & entity) { return getCreateQueryImpl(entity, nullptr, true); } + +AccessRightsElements InterpreterShowCreateAccessEntityQuery::getRequiredAccess() const +{ + const auto & show_query = query_ptr->as(); + AccessRightsElements res; + switch (show_query.type) + { + case EntityType::USER: res.emplace_back(AccessType::SHOW_USERS); return res; + case EntityType::ROLE: res.emplace_back(AccessType::SHOW_ROLES); return res; + case EntityType::SETTINGS_PROFILE: res.emplace_back(AccessType::SHOW_SETTINGS_PROFILES); return res; + case EntityType::ROW_POLICY: res.emplace_back(AccessType::SHOW_ROW_POLICIES); return res; + case EntityType::QUOTA: res.emplace_back(AccessType::SHOW_QUOTAS); return res; + case EntityType::MAX: break; + } + throw Exception(toString(show_query.type) + ": type is not supported by SHOW CREATE query", ErrorCodes::NOT_IMPLEMENTED); +} } diff --git a/src/Interpreters/InterpreterShowGrantsQuery.cpp b/src/Interpreters/InterpreterShowGrantsQuery.cpp index aa139b8e10e..130749526c7 100644 --- a/src/Interpreters/InterpreterShowGrantsQuery.cpp +++ b/src/Interpreters/InterpreterShowGrantsQuery.cpp @@ -105,7 +105,7 @@ namespace return getGrantQueriesImpl(*user, manager, attach_mode); if (const Role * role = typeid_cast(&entity)) return getGrantQueriesImpl(*role, manager, attach_mode); - throw Exception("Unexpected type of access entity: " + entity.getTypeName(), ErrorCodes::LOGICAL_ERROR); + throw Exception(entity.outputTypeAndName() + " is expected to be user or role", ErrorCodes::LOGICAL_ERROR); } } diff --git a/src/Parsers/ASTDropAccessEntityQuery.cpp b/src/Parsers/ASTDropAccessEntityQuery.cpp index a0e6753af57..9f7a1d86221 100644 --- a/src/Parsers/ASTDropAccessEntityQuery.cpp +++ b/src/Parsers/ASTDropAccessEntityQuery.cpp @@ -4,34 +4,12 @@ namespace DB { -namespace -{ - using Kind = ASTDropAccessEntityQuery::Kind; - - const char * getKeyword(Kind kind) - { - switch (kind) - { - case Kind::USER: return "USER"; - case Kind::ROLE: return "ROLE"; - case Kind::QUOTA: return "QUOTA"; - case Kind::ROW_POLICY: return "ROW POLICY"; - case Kind::SETTINGS_PROFILE: return "SETTINGS PROFILE"; - } - __builtin_unreachable(); - } -} - - -ASTDropAccessEntityQuery::ASTDropAccessEntityQuery(Kind kind_) - : kind(kind_) -{ -} +using EntityTypeInfo = IAccessEntity::TypeInfo; String ASTDropAccessEntityQuery::getID(char) const { - return String("DROP ") + getKeyword(kind) + " query"; + return String("DROP ") + toString(type) + " query"; } @@ -44,11 +22,11 @@ ASTPtr ASTDropAccessEntityQuery::clone() const void ASTDropAccessEntityQuery::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const { settings.ostr << (settings.hilite ? hilite_keyword : "") - << "DROP " << getKeyword(kind) + << "DROP " << EntityTypeInfo::get(type).name << (if_exists ? " IF EXISTS" : "") << (settings.hilite ? hilite_none : ""); - if (kind == Kind::ROW_POLICY) + if (type == EntityType::ROW_POLICY) { bool need_comma = false; for (const auto & name_parts : row_policies_name_parts) diff --git a/src/Parsers/ASTDropAccessEntityQuery.h b/src/Parsers/ASTDropAccessEntityQuery.h index a630869e027..160b0c2e212 100644 --- a/src/Parsers/ASTDropAccessEntityQuery.h +++ b/src/Parsers/ASTDropAccessEntityQuery.h @@ -17,21 +17,13 @@ namespace DB class ASTDropAccessEntityQuery : public IAST, public ASTQueryWithOnCluster { public: - enum class Kind - { - USER, - ROLE, - QUOTA, - ROW_POLICY, - SETTINGS_PROFILE, - }; + using EntityType = IAccessEntity::Type; - const Kind kind; + EntityType type; bool if_exists = false; Strings names; std::vector row_policies_name_parts; - ASTDropAccessEntityQuery(Kind kind_); String getID(char) const override; ASTPtr clone() const override; void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; diff --git a/src/Parsers/ASTShowCreateAccessEntityQuery.cpp b/src/Parsers/ASTShowCreateAccessEntityQuery.cpp index eed5766bcd9..954fab673e7 100644 --- a/src/Parsers/ASTShowCreateAccessEntityQuery.cpp +++ b/src/Parsers/ASTShowCreateAccessEntityQuery.cpp @@ -4,34 +4,12 @@ namespace DB { -namespace -{ - using Kind = ASTShowCreateAccessEntityQuery::Kind; - - const char * getKeyword(Kind kind) - { - switch (kind) - { - case Kind::USER: return "USER"; - case Kind::ROLE: return "ROLE"; - case Kind::QUOTA: return "QUOTA"; - case Kind::ROW_POLICY: return "ROW POLICY"; - case Kind::SETTINGS_PROFILE: return "SETTINGS PROFILE"; - } - __builtin_unreachable(); - } -} - - -ASTShowCreateAccessEntityQuery::ASTShowCreateAccessEntityQuery(Kind kind_) - : kind(kind_) -{ -} +using EntityTypeInfo = IAccessEntity::TypeInfo; String ASTShowCreateAccessEntityQuery::getID(char) const { - return String("SHOW CREATE ") + getKeyword(kind) + " query"; + return String("SHOW CREATE ") + toString(type) + " query"; } @@ -44,7 +22,7 @@ ASTPtr ASTShowCreateAccessEntityQuery::clone() const void ASTShowCreateAccessEntityQuery::formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const { settings.ostr << (settings.hilite ? hilite_keyword : "") - << "SHOW CREATE " << getKeyword(kind) + << "SHOW CREATE " << EntityTypeInfo::get(type).name << (settings.hilite ? hilite_none : ""); if (current_user) @@ -52,7 +30,7 @@ void ASTShowCreateAccessEntityQuery::formatQueryImpl(const FormatSettings & sett } else if (current_quota) settings.ostr << (settings.hilite ? hilite_keyword : "") << " CURRENT" << (settings.hilite ? hilite_none : ""); - else if (kind == Kind::ROW_POLICY) + else if (type == EntityType::ROW_POLICY) { const String & database = row_policy_name_parts.database; const String & table_name = row_policy_name_parts.table_name; diff --git a/src/Parsers/ASTShowCreateAccessEntityQuery.h b/src/Parsers/ASTShowCreateAccessEntityQuery.h index 7f82e9f5e34..df7be2e257c 100644 --- a/src/Parsers/ASTShowCreateAccessEntityQuery.h +++ b/src/Parsers/ASTShowCreateAccessEntityQuery.h @@ -15,22 +15,14 @@ namespace DB class ASTShowCreateAccessEntityQuery : public ASTQueryWithOutput { public: - enum class Kind - { - USER, - ROLE, - QUOTA, - ROW_POLICY, - SETTINGS_PROFILE, - }; + using EntityType = IAccessEntity::Type; - const Kind kind; + EntityType type; String name; bool current_quota = false; bool current_user = false; RowPolicy::NameParts row_policy_name_parts; - ASTShowCreateAccessEntityQuery(Kind kind_); String getID(char) const override; ASTPtr clone() const override; diff --git a/src/Parsers/ParserDropAccessEntityQuery.cpp b/src/Parsers/ParserDropAccessEntityQuery.cpp index 034124a42f0..15f8bbf0a62 100644 --- a/src/Parsers/ParserDropAccessEntityQuery.cpp +++ b/src/Parsers/ParserDropAccessEntityQuery.cpp @@ -4,12 +4,16 @@ #include #include #include +#include namespace DB { namespace { + using EntityType = IAccessEntity::Type; + using EntityTypeInfo = IAccessEntity::TypeInfo; + bool parseNames(IParserBase::Pos & pos, Expected & expected, Strings & names) { return IParserBase::wrapParseImpl(pos, [&] @@ -79,19 +83,17 @@ bool ParserDropAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & if (!ParserKeyword{"DROP"}.ignore(pos, expected)) return false; - using Kind = ASTDropAccessEntityQuery::Kind; - Kind kind; - if (ParserKeyword{"USER"}.ignore(pos, expected)) - kind = Kind::USER; - else if (ParserKeyword{"ROLE"}.ignore(pos, expected)) - kind = Kind::ROLE; - else if (ParserKeyword{"QUOTA"}.ignore(pos, expected)) - kind = Kind::QUOTA; - else if (ParserKeyword{"POLICY"}.ignore(pos, expected) || ParserKeyword{"ROW POLICY"}.ignore(pos, expected)) - kind = Kind::ROW_POLICY; - else if (ParserKeyword{"SETTINGS PROFILE"}.ignore(pos, expected) || ParserKeyword{"PROFILE"}.ignore(pos, expected)) - kind = Kind::SETTINGS_PROFILE; - else + std::optional type; + for (auto type_i : ext::range(EntityType::MAX)) + { + const auto & type_info = EntityTypeInfo::get(type_i); + if (ParserKeyword{type_info.name.c_str()}.ignore(pos, expected) + || (!type_info.alias.empty() && ParserKeyword{type_info.alias.c_str()}.ignore(pos, expected))) + { + type = type_i; + } + } + if (!type) return false; bool if_exists = false; @@ -101,12 +103,12 @@ bool ParserDropAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & Strings names; std::vector row_policies_name_parts; - if ((kind == Kind::USER) || (kind == Kind::ROLE)) + if ((type == EntityType::USER) || (type == EntityType::ROLE)) { if (!parseUserNames(pos, expected, names)) return false; } - else if (kind == Kind::ROW_POLICY) + else if (type == EntityType::ROW_POLICY) { if (!parseRowPolicyNames(pos, expected, row_policies_name_parts)) return false; @@ -124,9 +126,10 @@ bool ParserDropAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & return false; } - auto query = std::make_shared(kind); + auto query = std::make_shared(); node = query; + query->type = *type; query->if_exists = if_exists; query->cluster = std::move(cluster); query->names = std::move(names); diff --git a/src/Parsers/ParserShowCreateAccessEntityQuery.cpp b/src/Parsers/ParserShowCreateAccessEntityQuery.cpp index 48cd36f68d3..308a1bd7795 100644 --- a/src/Parsers/ParserShowCreateAccessEntityQuery.cpp +++ b/src/Parsers/ParserShowCreateAccessEntityQuery.cpp @@ -4,29 +4,32 @@ #include #include #include +#include #include namespace DB { +using EntityType = IAccessEntity::Type; +using EntityTypeInfo = IAccessEntity::TypeInfo; + + bool ParserShowCreateAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { if (!ParserKeyword{"SHOW CREATE"}.ignore(pos, expected)) return false; - using Kind = ASTShowCreateAccessEntityQuery::Kind; - Kind kind; - if (ParserKeyword{"USER"}.ignore(pos, expected)) - kind = Kind::USER; - else if (ParserKeyword{"QUOTA"}.ignore(pos, expected)) - kind = Kind::QUOTA; - else if (ParserKeyword{"POLICY"}.ignore(pos, expected) || ParserKeyword{"ROW POLICY"}.ignore(pos, expected)) - kind = Kind::ROW_POLICY; - else if (ParserKeyword{"ROLE"}.ignore(pos, expected)) - kind = Kind::ROLE; - else if (ParserKeyword{"SETTINGS PROFILE"}.ignore(pos, expected) || ParserKeyword{"PROFILE"}.ignore(pos, expected)) - kind = Kind::SETTINGS_PROFILE; - else + std::optional type; + for (auto type_i : ext::range(EntityType::MAX)) + { + const auto & type_info = EntityTypeInfo::get(type_i); + if (ParserKeyword{type_info.name.c_str()}.ignore(pos, expected) + || (!type_info.alias.empty() && ParserKeyword{type_info.alias.c_str()}.ignore(pos, expected))) + { + type = type_i; + } + } + if (!type) return false; String name; @@ -34,17 +37,17 @@ bool ParserShowCreateAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expe bool current_user = false; RowPolicy::NameParts row_policy_name_parts; - if (kind == Kind::USER) + if (type == EntityType::USER) { if (!parseUserNameOrCurrentUserTag(pos, expected, name, current_user)) current_user = true; } - else if (kind == Kind::ROLE) + else if (type == EntityType::ROLE) { if (!parseRoleName(pos, expected, name)) return false; } - else if (kind == Kind::ROW_POLICY) + else if (type == EntityType::ROW_POLICY) { String & database = row_policy_name_parts.database; String & table_name = row_policy_name_parts.table_name; @@ -53,7 +56,7 @@ bool ParserShowCreateAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expe || !parseDatabaseAndTableName(pos, expected, database, table_name)) return false; } - else if (kind == Kind::QUOTA) + else if (type == EntityType::QUOTA) { if (ParserKeyword{"CURRENT"}.ignore(pos, expected)) { @@ -70,15 +73,16 @@ bool ParserShowCreateAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expe current_quota = true; } } - else if (kind == Kind::SETTINGS_PROFILE) + else if (type == EntityType::SETTINGS_PROFILE) { if (!parseIdentifierOrStringLiteral(pos, expected, name)) return false; } - auto query = std::make_shared(kind); + auto query = std::make_shared(); node = query; + query->type = *type; query->name = std::move(name); query->current_quota = current_quota; query->current_user = current_user; diff --git a/tests/integration/test_access_control_on_cluster/test.py b/tests/integration/test_access_control_on_cluster/test.py index 6ca4ac15398..4dc9baca0a0 100644 --- a/tests/integration/test_access_control_on_cluster/test.py +++ b/tests/integration/test_access_control_on_cluster/test.py @@ -35,7 +35,7 @@ def test_access_control_on_cluster(): assert ch3.query("SHOW GRANTS FOR Alex") == "" ch2.query("DROP USER Alex ON CLUSTER 'cluster'") - assert "User `Alex` not found" in ch1.query_and_get_error("SHOW CREATE USER Alex") - assert "User `Alex` not found" in ch2.query_and_get_error("SHOW CREATE USER Alex") - assert "User `Alex` not found" in ch3.query_and_get_error("SHOW CREATE USER Alex") + assert "There is no user `Alex`" in ch1.query_and_get_error("SHOW CREATE USER Alex") + assert "There is no user `Alex`" in ch2.query_and_get_error("SHOW CREATE USER Alex") + assert "There is no user `Alex`" in ch3.query_and_get_error("SHOW CREATE USER Alex") diff --git a/tests/integration/test_disk_access_storage/test.py b/tests/integration/test_disk_access_storage/test.py index babceee7c76..315440b4358 100644 --- a/tests/integration/test_disk_access_storage/test.py +++ b/tests/integration/test_disk_access_storage/test.py @@ -97,9 +97,9 @@ def test_drop(): def check(): assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1\n" assert instance.query("SHOW CREATE SETTINGS PROFILE s2") == "CREATE SETTINGS PROFILE s2\n" - assert "User `u2` not found" in instance.query_and_get_error("SHOW CREATE USER u2") - assert "Row policy `p ON mydb.mytable` not found" in instance.query_and_get_error("SHOW CREATE ROW POLICY p ON mydb.mytable") - assert "Quota `q` not found" in instance.query_and_get_error("SHOW CREATE QUOTA q") + assert "There is no user `u2`" in instance.query_and_get_error("SHOW CREATE USER u2") + assert "There is no row policy `p ON mydb.mytable`" in instance.query_and_get_error("SHOW CREATE ROW POLICY p ON mydb.mytable") + assert "There is no quota `q`" in instance.query_and_get_error("SHOW CREATE QUOTA q") check() instance.restart_clickhouse() # Check persistency From 9a89a04c1f2ae5f974fbf7ddf79aeb229ca388f1 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 7 May 2020 05:44:11 +0300 Subject: [PATCH 182/738] Improve TSV class in test tools: now it can get a table as a list of lists. --- tests/integration/helpers/test_tools.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/integration/helpers/test_tools.py b/tests/integration/helpers/test_tools.py index e90580de44c..93265d280df 100644 --- a/tests/integration/helpers/test_tools.py +++ b/tests/integration/helpers/test_tools.py @@ -5,16 +5,29 @@ class TSV: """Helper to get pretty diffs between expected and actual tab-separated value files""" def __init__(self, contents): - raw_lines = contents.readlines() if isinstance(contents, file) else contents.splitlines(True) + if isinstance(contents, file): + raw_lines = contents.readlines() + elif isinstance(contents, str) or isinstance(contents, unicode): + raw_lines = contents.splitlines(True) + elif isinstance(contents, list): + raw_lines = ['\t'.join(map(str, l)) if isinstance(l, list) else str(l) for l in contents] + else: + raise TypeError("contents must be either file or string or list, actual type: " + type(contents).__name__) self.lines = [l.strip() for l in raw_lines if l.strip()] def __eq__(self, other): + if not isinstance(other, TSV): + return self == TSV(other) return self.lines == other.lines def __ne__(self, other): + if not isinstance(other, TSV): + return self != TSV(other) return self.lines != other.lines def diff(self, other, n1=None, n2=None): + if not isinstance(other, TSV): + return self.diff(TSV(other), n1=n1, n2=n2) return list(line.rstrip() for line in difflib.unified_diff(self.lines, other.lines, fromfile=n1, tofile=n2))[2:] def __str__(self): From e64e2ebdf6dd92b3fff3f1e127870096c76734b7 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 7 May 2020 05:45:27 +0300 Subject: [PATCH 183/738] Improve system table for row policies. Remove function currentRowPolicies(). --- src/Access/ContextAccess.cpp | 2 +- src/Access/ContextAccess.h | 2 +- src/Access/EnabledRowPolicies.cpp | 42 +-- src/Access/EnabledRowPolicies.h | 34 +- src/Access/RowPolicy.cpp | 31 -- src/Access/RowPolicy.h | 76 +++- src/Access/RowPolicyCache.cpp | 53 +-- src/Access/RowPolicyCache.h | 5 +- src/Functions/currentRowPolicies.cpp | 237 ------------ .../registerFunctionsMiscellaneous.cpp | 2 - src/Functions/ya.make | 1 - src/Interpreters/Context.cpp | 5 - src/Interpreters/Context.h | 1 - .../InterpreterCreateRowPolicyQuery.cpp | 8 +- src/Interpreters/InterpreterFactory.cpp | 14 +- .../InterpreterShowAccessEntitiesQuery.cpp | 67 ++++ ...h => InterpreterShowAccessEntitiesQuery.h} | 5 +- ...InterpreterShowCreateAccessEntityQuery.cpp | 6 +- .../InterpreterShowRowPoliciesQuery.cpp | 69 ---- src/Interpreters/ya.make | 2 +- src/Parsers/ASTCreateRowPolicyQuery.cpp | 149 ++++---- src/Parsers/ASTCreateRowPolicyQuery.h | 7 +- src/Parsers/ASTShowAccessEntitiesQuery.cpp | 37 ++ ...esQuery.h => ASTShowAccessEntitiesQuery.h} | 14 +- src/Parsers/ASTShowRowPoliciesQuery.cpp | 22 -- src/Parsers/ParserCreateRowPolicyQuery.cpp | 115 +++--- src/Parsers/ParserQueryWithOutput.cpp | 10 +- ....cpp => ParserShowAccessEntitiesQuery.cpp} | 18 +- ...uery.h => ParserShowAccessEntitiesQuery.h} | 6 +- src/Parsers/ya.make | 4 +- .../System/StorageSystemRowPolicies.cpp | 113 +++++- .../configs/config.d/remote_servers.xml | 4 +- tests/integration/test_row_policy/test.py | 356 +++++++++--------- 33 files changed, 681 insertions(+), 836 deletions(-) delete mode 100644 src/Functions/currentRowPolicies.cpp create mode 100644 src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp rename src/Interpreters/{InterpreterShowRowPoliciesQuery.h => InterpreterShowAccessEntitiesQuery.h} (58%) delete mode 100644 src/Interpreters/InterpreterShowRowPoliciesQuery.cpp create mode 100644 src/Parsers/ASTShowAccessEntitiesQuery.cpp rename src/Parsers/{ASTShowRowPoliciesQuery.h => ASTShowAccessEntitiesQuery.h} (53%) delete mode 100644 src/Parsers/ASTShowRowPoliciesQuery.cpp rename src/Parsers/{ParserShowRowPoliciesQuery.cpp => ParserShowAccessEntitiesQuery.cpp} (72%) rename src/Parsers/{ParserShowRowPoliciesQuery.h => ParserShowAccessEntitiesQuery.h} (50%) diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index 475a225ba05..ef9791bafa0 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -485,7 +485,7 @@ std::shared_ptr ContextAccess::getRolesInfo() const return roles_info; } -std::shared_ptr ContextAccess::getRowPolicies() const +std::shared_ptr ContextAccess::getEnabledRowPolicies() const { std::lock_guard lock{mutex}; return enabled_row_policies; diff --git a/src/Access/ContextAccess.h b/src/Access/ContextAccess.h index 9a9f0e31470..b10a1fdba69 100644 --- a/src/Access/ContextAccess.h +++ b/src/Access/ContextAccess.h @@ -70,7 +70,7 @@ public: /// Returns information about enabled row policies. /// The function can return nullptr. - std::shared_ptr getRowPolicies() const; + std::shared_ptr getEnabledRowPolicies() const; /// Returns the row policy filter for a specified table. /// The function returns nullptr if there is no filter to apply. diff --git a/src/Access/EnabledRowPolicies.cpp b/src/Access/EnabledRowPolicies.cpp index 56c73aaf40d..efd5ed4ae10 100644 --- a/src/Access/EnabledRowPolicies.cpp +++ b/src/Access/EnabledRowPolicies.cpp @@ -6,9 +6,9 @@ namespace DB { -size_t EnabledRowPolicies::Hash::operator()(const DatabaseAndTableNameRef & database_and_table_name) const +size_t EnabledRowPolicies::Hash::operator()(const MixedConditionKey & key) const { - return std::hash{}(database_and_table_name.first) - std::hash{}(database_and_table_name.second); + return std::hash{}(key.database) - std::hash{}(key.table_name) + static_cast(key.condition_type); } @@ -20,16 +20,22 @@ EnabledRowPolicies::EnabledRowPolicies(const Params & params_) EnabledRowPolicies::~EnabledRowPolicies() = default; -ASTPtr EnabledRowPolicies::getCondition(const String & database, const String & table_name, ConditionType type) const +ASTPtr EnabledRowPolicies::getCondition(const String & database, const String & table_name, ConditionType condition_type) const { /// We don't lock `mutex` here. auto loaded = map_of_mixed_conditions.load(); - auto it = loaded->find({database, table_name}); + auto it = loaded->find({database, table_name, condition_type}); if (it == loaded->end()) return {}; - return it->second.mixed_conditions[type]; -} + auto condition = it->second.ast; + + bool value; + if (tryGetLiteralBool(condition.get(), value) && value) + return nullptr; /// The condition is always true, no need to check it. + + return condition; +} ASTPtr EnabledRowPolicies::getCondition(const String & database, const String & table_name, ConditionType type, const ASTPtr & extra_condition) const { @@ -41,31 +47,9 @@ ASTPtr EnabledRowPolicies::getCondition(const String & database, const String & bool value; if (tryGetLiteralBool(condition.get(), value) && value) - condition = nullptr; /// The condition is always true, no need to check it. + return nullptr; /// The condition is always true, no need to check it. return condition; } - -std::vector EnabledRowPolicies::getCurrentPolicyIDs() const -{ - /// We don't lock `mutex` here. - auto loaded = map_of_mixed_conditions.load(); - std::vector policy_ids; - for (const auto & mixed_conditions : *loaded | boost::adaptors::map_values) - boost::range::copy(mixed_conditions.policy_ids, std::back_inserter(policy_ids)); - return policy_ids; -} - - -std::vector EnabledRowPolicies::getCurrentPolicyIDs(const String & database, const String & table_name) const -{ - /// We don't lock `mutex` here. - auto loaded = map_of_mixed_conditions.load(); - auto it = loaded->find({database, table_name}); - if (it == loaded->end()) - return {}; - return it->second.policy_ids; -} - } diff --git a/src/Access/EnabledRowPolicies.h b/src/Access/EnabledRowPolicies.h index 5255b2a791c..b92939afb03 100644 --- a/src/Access/EnabledRowPolicies.h +++ b/src/Access/EnabledRowPolicies.h @@ -4,8 +4,8 @@ #include #include #include -#include #include +#include namespace DB @@ -42,30 +42,32 @@ public: ASTPtr getCondition(const String & database, const String & table_name, ConditionType type) const; ASTPtr getCondition(const String & database, const String & table_name, ConditionType type, const ASTPtr & extra_condition) const; - /// Returns IDs of all the policies used by the current user. - std::vector getCurrentPolicyIDs() const; - - /// Returns IDs of the policies used by a concrete table. - std::vector getCurrentPolicyIDs(const String & database, const String & table_name) const; - private: friend class RowPolicyCache; EnabledRowPolicies(const Params & params_); - using DatabaseAndTableName = std::pair; - using DatabaseAndTableNameRef = std::pair; + struct MixedConditionKey + { + std::string_view database; + std::string_view table_name; + ConditionType condition_type; + + auto toTuple() const { return std::tie(database, table_name, condition_type); } + friend bool operator==(const MixedConditionKey & left, const MixedConditionKey & right) { return left.toTuple() == right.toTuple(); } + friend bool operator!=(const MixedConditionKey & left, const MixedConditionKey & right) { return left.toTuple() != right.toTuple(); } + }; + struct Hash { - size_t operator()(const DatabaseAndTableNameRef & database_and_table_name) const; + size_t operator()(const MixedConditionKey & key) const; }; - using ParsedConditions = std::array; - struct MixedConditions + + struct MixedCondition { - std::unique_ptr database_and_table_name_keeper; - ParsedConditions mixed_conditions; - std::vector policy_ids; + ASTPtr ast; + std::shared_ptr> database_and_table_name; }; - using MapOfMixedConditions = std::unordered_map; + using MapOfMixedConditions = std::unordered_map; const Params params; mutable boost::atomic_shared_ptr map_of_mixed_conditions; diff --git a/src/Access/RowPolicy.cpp b/src/Access/RowPolicy.cpp index bb3fc54a590..4249f351eae 100644 --- a/src/Access/RowPolicy.cpp +++ b/src/Access/RowPolicy.cpp @@ -8,7 +8,6 @@ namespace DB namespace ErrorCodes { extern const int NOT_IMPLEMENTED; - extern const int LOGICAL_ERROR; } @@ -75,34 +74,4 @@ bool RowPolicy::equal(const IAccessEntity & other) const && restrictive == other_policy.restrictive && (to_roles == other_policy.to_roles); } - -const char * RowPolicy::conditionTypeToString(ConditionType index) -{ - switch (index) - { - case SELECT_FILTER: return "SELECT_FILTER"; - case INSERT_CHECK: return "INSERT_CHECK"; - case UPDATE_FILTER: return "UPDATE_FILTER"; - case UPDATE_CHECK: return "UPDATE_CHECK"; - case DELETE_FILTER: return "DELETE_FILTER"; - case MAX_CONDITION_TYPE: break; - } - throw Exception("Unexpected condition type: " + std::to_string(static_cast(index)), ErrorCodes::LOGICAL_ERROR); -} - - -const char * RowPolicy::conditionTypeToColumnName(ConditionType index) -{ - switch (index) - { - case SELECT_FILTER: return "select_filter"; - case INSERT_CHECK: return "insert_check"; - case UPDATE_FILTER: return "update_filter"; - case UPDATE_CHECK: return "update_check"; - case DELETE_FILTER: return "delete_filter"; - case MAX_CONDITION_TYPE: break; - } - throw Exception("Unexpected condition type: " + std::to_string(static_cast(index)), ErrorCodes::LOGICAL_ERROR); -} - } diff --git a/src/Access/RowPolicy.h b/src/Access/RowPolicy.h index 9d582d27045..7febf5991fb 100644 --- a/src/Access/RowPolicy.h +++ b/src/Access/RowPolicy.h @@ -2,10 +2,16 @@ #include #include +#include namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + /** Represents a row level security policy for a table. */ @@ -43,17 +49,27 @@ struct RowPolicy : public IAccessEntity enum ConditionType { SELECT_FILTER, + +#if 0 /// Row-level security for INSERT, UPDATE, DELETE is not implemented yet. INSERT_CHECK, UPDATE_FILTER, UPDATE_CHECK, DELETE_FILTER, +#endif MAX_CONDITION_TYPE }; - static const char * conditionTypeToString(ConditionType index); - static const char * conditionTypeToColumnName(ConditionType index); - String conditions[MAX_CONDITION_TYPE]; + struct ConditionTypeInfo + { + const char * const raw_name; + const String name; /// Lowercased with underscores, e.g. "select_filter". + const String command; /// Uppercased without last word, e.g. "SELECT". + const bool is_check; /// E.g. false for SELECT_FILTER. + static const ConditionTypeInfo & get(ConditionType type); + }; + + std::array conditions; /// Sets that the policy is permissive. /// A row is only accessible if at least one of the permissive policies passes, @@ -83,4 +99,58 @@ private: }; using RowPolicyPtr = std::shared_ptr; + + +inline const RowPolicy::ConditionTypeInfo & RowPolicy::ConditionTypeInfo::get(ConditionType type_) +{ + static constexpr auto make_info = [](const char * raw_name_) + { + String init_name = raw_name_; + boost::to_lower(init_name); + size_t underscore_pos = init_name.find('_'); + String init_command = init_name.substr(0, underscore_pos); + boost::to_upper(init_command); + bool init_is_check = (std::string_view{init_name}.substr(underscore_pos + 1) == "check"); + return ConditionTypeInfo{raw_name_, std::move(init_name), std::move(init_command), init_is_check}; + }; + + switch (type_) + { + case SELECT_FILTER: + { + static const ConditionTypeInfo info = make_info("SELECT_FILTER"); + return info; + } +#if 0 /// Row-level security for INSERT, UPDATE, DELETE is not implemented yet. + case INSERT_CHECK: + { + static const ConditionTypeInfo info = make_info("INSERT_CHECK"); + return info; + } + case UPDATE_FILTER: + { + static const ConditionTypeInfo info = make_info("UPDATE_FILTER"); + return info; + } + case UPDATE_CHECK: + { + static const ConditionTypeInfo info = make_info("UPDATE_CHECK"); + return info; + } + case DELETE_FILTER: + { + static const ConditionTypeInfo info = make_info("DELETE_FILTER"); + return info; + } +#endif + case MAX_CONDITION_TYPE: break; + } + throw Exception("Unknown type: " + std::to_string(static_cast(type_)), ErrorCodes::LOGICAL_ERROR); +} + +inline String toString(RowPolicy::ConditionType type) +{ + return RowPolicy::ConditionTypeInfo::get(type).raw_name; +} + } diff --git a/src/Access/RowPolicyCache.cpp b/src/Access/RowPolicyCache.cpp index bc3a2b9a15e..243f9c85d70 100644 --- a/src/Access/RowPolicyCache.cpp +++ b/src/Access/RowPolicyCache.cpp @@ -57,6 +57,7 @@ void RowPolicyCache::PolicyInfo::setPolicy(const RowPolicyPtr & policy_) { policy = policy_; roles = &policy->to_roles; + database_and_table_name = std::make_shared>(policy->getDatabase(), policy->getTableName()); for (auto type : ext::range(0, MAX_CONDITION_TYPE)) { @@ -84,7 +85,7 @@ void RowPolicyCache::PolicyInfo::setPolicy(const RowPolicyPtr & policy_) { tryLogCurrentException( &Poco::Logger::get("RowPolicy"), - String("Could not parse the condition ") + RowPolicy::conditionTypeToString(type) + " of row policy " + String("Could not parse the condition ") + toString(type) + " of row policy " + backQuote(policy->getName())); } } @@ -196,43 +197,45 @@ void RowPolicyCache::mixConditions() void RowPolicyCache::mixConditionsFor(EnabledRowPolicies & enabled) { /// `mutex` is already locked. - struct Mixers - { - ConditionsMixer mixers[MAX_CONDITION_TYPE]; - std::vector policy_ids; - }; + using MapOfMixedConditions = EnabledRowPolicies::MapOfMixedConditions; - using DatabaseAndTableName = EnabledRowPolicies::DatabaseAndTableName; - using DatabaseAndTableNameRef = EnabledRowPolicies::DatabaseAndTableNameRef; + using MixedConditionKey = EnabledRowPolicies::MixedConditionKey; using Hash = EnabledRowPolicies::Hash; - std::unordered_map map_of_mixers; + struct MixerWithNames + { + ConditionsMixer mixer; + std::shared_ptr> database_and_table_name; + }; + + std::unordered_map map_of_mixers; for (const auto & [policy_id, info] : all_policies) { const auto & policy = *info.policy; - auto & mixers = map_of_mixers[std::pair{policy.getDatabase(), policy.getTableName()}]; - if (info.roles->match(enabled.params.user_id, enabled.params.enabled_roles)) + bool match = info.roles->match(enabled.params.user_id, enabled.params.enabled_roles); + MixedConditionKey key; + key.database = info.database_and_table_name->first; + key.table_name = info.database_and_table_name->second; + for (auto type : ext::range(0, MAX_CONDITION_TYPE)) { - mixers.policy_ids.push_back(policy_id); - for (auto type : ext::range(0, MAX_CONDITION_TYPE)) - if (info.parsed_conditions[type]) - mixers.mixers[type].add(info.parsed_conditions[type], policy.isRestrictive()); + if (info.parsed_conditions[type]) + { + key.condition_type = type; + auto & mixer = map_of_mixers[key]; + mixer.database_and_table_name = info.database_and_table_name; + if (match) + mixer.mixer.add(info.parsed_conditions[type], policy.isRestrictive()); + } } } auto map_of_mixed_conditions = boost::make_shared(); - for (auto & [database_and_table_name, mixers] : map_of_mixers) + for (auto & [key, mixer] : map_of_mixers) { - auto database_and_table_name_keeper = std::make_unique(); - database_and_table_name_keeper->first = database_and_table_name.first; - database_and_table_name_keeper->second = database_and_table_name.second; - auto & mixed_conditions = (*map_of_mixed_conditions)[DatabaseAndTableNameRef{database_and_table_name_keeper->first, - database_and_table_name_keeper->second}]; - mixed_conditions.database_and_table_name_keeper = std::move(database_and_table_name_keeper); - mixed_conditions.policy_ids = std::move(mixers.policy_ids); - for (auto type : ext::range(0, MAX_CONDITION_TYPE)) - mixed_conditions.mixed_conditions[type] = std::move(mixers.mixers[type]).getResult(); + auto & mixed_condition = (*map_of_mixed_conditions)[key]; + mixed_condition.database_and_table_name = mixer.database_and_table_name; + mixed_condition.ast = std::move(mixer.mixer).getResult(); } enabled.map_of_mixed_conditions.store(map_of_mixed_conditions); diff --git a/src/Access/RowPolicyCache.h b/src/Access/RowPolicyCache.h index 3b4ecd6e8f5..139949ae815 100644 --- a/src/Access/RowPolicyCache.h +++ b/src/Access/RowPolicyCache.h @@ -21,8 +21,6 @@ public: std::shared_ptr getEnabledRowPolicies(const UUID & user_id, const boost::container::flat_set & enabled_roles); private: - using ParsedConditions = EnabledRowPolicies::ParsedConditions; - struct PolicyInfo { PolicyInfo(const RowPolicyPtr & policy_) { setPolicy(policy_); } @@ -30,7 +28,8 @@ private: RowPolicyPtr policy; const ExtendedRoleSet * roles = nullptr; - ParsedConditions parsed_conditions; + std::shared_ptr> database_and_table_name; + ASTPtr parsed_conditions[RowPolicy::MAX_CONDITION_TYPE]; }; void ensureAllRowPoliciesRead(); diff --git a/src/Functions/currentRowPolicies.cpp b/src/Functions/currentRowPolicies.cpp deleted file mode 100644 index 6efe57b2956..00000000000 --- a/src/Functions/currentRowPolicies.cpp +++ /dev/null @@ -1,237 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace DB -{ -namespace ErrorCodes -{ - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; -} - - -/// The currentRowPolicies() function can be called with 0..2 arguments: -/// currentRowPolicies() returns array of tuples (database, table_name, row_policy_name) for all the row policies applied for the current user; -/// currentRowPolicies(table_name) is equivalent to currentRowPolicies(currentDatabase(), table_name); -/// currentRowPolicies(database, table_name) returns array of names of the row policies applied to a specific table and for the current user. -class FunctionCurrentRowPolicies : public IFunction -{ -public: - static constexpr auto name = "currentRowPolicies"; - - static FunctionPtr create(const Context & context_) { return std::make_shared(context_); } - explicit FunctionCurrentRowPolicies(const Context & context_) : context(context_) {} - - String getName() const override { return name; } - size_t getNumberOfArguments() const override { return 0; } - bool isVariadic() const override { return true; } - - void checkNumberOfArgumentsIfVariadic(size_t number_of_arguments) const override - { - if (number_of_arguments > 2) - throw Exception("Number of arguments for function " + String(name) + " doesn't match: passed " - + toString(number_of_arguments) + ", should be 0..2", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - } - - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override - { - if (arguments.empty()) - return std::make_shared(std::make_shared( - DataTypes{std::make_shared(), std::make_shared(), std::make_shared()})); - else - return std::make_shared(std::make_shared()); - } - - bool isDeterministic() const override { return false; } - - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result_pos, size_t input_rows_count) override - { - if (arguments.empty()) - { - auto database_column = ColumnString::create(); - auto table_name_column = ColumnString::create(); - auto policy_name_column = ColumnString::create(); - if (auto policies = context.getRowPolicies()) - { - for (const auto & policy_id : policies->getCurrentPolicyIDs()) - { - const auto policy = context.getAccessControlManager().tryRead(policy_id); - if (policy) - { - const String database = policy->getDatabase(); - const String table_name = policy->getTableName(); - const String policy_name = policy->getShortName(); - database_column->insertData(database.data(), database.length()); - table_name_column->insertData(table_name.data(), table_name.length()); - policy_name_column->insertData(policy_name.data(), policy_name.length()); - } - } - } - auto offset_column = ColumnArray::ColumnOffsets::create(); - offset_column->insertValue(policy_name_column->size()); - block.getByPosition(result_pos).column = ColumnConst::create( - ColumnArray::create( - ColumnTuple::create(Columns{std::move(database_column), std::move(table_name_column), std::move(policy_name_column)}), - std::move(offset_column)), - input_rows_count); - return; - } - - const IColumn * database_column = nullptr; - if (arguments.size() == 2) - { - const auto & database_column_with_type = block.getByPosition(arguments[0]); - if (!isStringOrFixedString(database_column_with_type.type)) - throw Exception{"The first argument of function " + String(name) - + " should be a string containing database name, illegal type: " - + database_column_with_type.type->getName(), - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - database_column = database_column_with_type.column.get(); - } - - const auto & table_name_column_with_type = block.getByPosition(arguments[arguments.size() - 1]); - if (!isStringOrFixedString(table_name_column_with_type.type)) - throw Exception{"The" + String(database_column ? " last" : "") + " argument of function " + String(name) - + " should be a string containing table name, illegal type: " + table_name_column_with_type.type->getName(), - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - const IColumn * table_name_column = table_name_column_with_type.column.get(); - - auto policy_name_column = ColumnString::create(); - auto offset_column = ColumnArray::ColumnOffsets::create(); - for (const auto i : ext::range(0, input_rows_count)) - { - String database = database_column ? database_column->getDataAt(i).toString() : context.getCurrentDatabase(); - String table_name = table_name_column->getDataAt(i).toString(); - if (auto policies = context.getRowPolicies()) - { - for (const auto & policy_id : policies->getCurrentPolicyIDs(database, table_name)) - { - const auto policy = context.getAccessControlManager().tryRead(policy_id); - if (policy) - { - const String policy_name = policy->getShortName(); - policy_name_column->insertData(policy_name.data(), policy_name.length()); - } - } - } - offset_column->insertValue(policy_name_column->size()); - } - - block.getByPosition(result_pos).column = ColumnArray::create(std::move(policy_name_column), std::move(offset_column)); - } - -private: - const Context & context; -}; - - -/// The currentRowPolicyIDs() function can be called with 0..2 arguments: -/// currentRowPolicyIDs() returns array of IDs of all the row policies applied for the current user; -/// currentRowPolicyIDs(table_name) is equivalent to currentRowPolicyIDs(currentDatabase(), table_name); -/// currentRowPolicyIDs(database, table_name) returns array of IDs of the row policies applied to a specific table and for the current user. -class FunctionCurrentRowPolicyIDs : public IFunction -{ -public: - static constexpr auto name = "currentRowPolicyIDs"; - - static FunctionPtr create(const Context & context_) { return std::make_shared(context_); } - explicit FunctionCurrentRowPolicyIDs(const Context & context_) : context(context_) {} - - String getName() const override { return name; } - size_t getNumberOfArguments() const override { return 0; } - bool isVariadic() const override { return true; } - - void checkNumberOfArgumentsIfVariadic(size_t number_of_arguments) const override - { - if (number_of_arguments > 2) - throw Exception("Number of arguments for function " + String(name) + " doesn't match: passed " - + toString(number_of_arguments) + ", should be 0..2", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - } - - DataTypePtr getReturnTypeImpl(const DataTypes & /* arguments */) const override - { - return std::make_shared(std::make_shared()); - } - - bool isDeterministic() const override { return false; } - - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result_pos, size_t input_rows_count) override - { - if (arguments.empty()) - { - auto policy_id_column = ColumnVector::create(); - if (auto policies = context.getRowPolicies()) - { - for (const auto & policy_id : policies->getCurrentPolicyIDs()) - policy_id_column->insertValue(policy_id); - } - auto offset_column = ColumnArray::ColumnOffsets::create(); - offset_column->insertValue(policy_id_column->size()); - block.getByPosition(result_pos).column - = ColumnConst::create(ColumnArray::create(std::move(policy_id_column), std::move(offset_column)), input_rows_count); - return; - } - - const IColumn * database_column = nullptr; - if (arguments.size() == 2) - { - const auto & database_column_with_type = block.getByPosition(arguments[0]); - if (!isStringOrFixedString(database_column_with_type.type)) - throw Exception{"The first argument of function " + String(name) - + " should be a string containing database name, illegal type: " - + database_column_with_type.type->getName(), - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - database_column = database_column_with_type.column.get(); - } - - const auto & table_name_column_with_type = block.getByPosition(arguments[arguments.size() - 1]); - if (!isStringOrFixedString(table_name_column_with_type.type)) - throw Exception{"The" + String(database_column ? " last" : "") + " argument of function " + String(name) - + " should be a string containing table name, illegal type: " + table_name_column_with_type.type->getName(), - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - const IColumn * table_name_column = table_name_column_with_type.column.get(); - - auto policy_id_column = ColumnVector::create(); - auto offset_column = ColumnArray::ColumnOffsets::create(); - for (const auto i : ext::range(0, input_rows_count)) - { - String database = database_column ? database_column->getDataAt(i).toString() : context.getCurrentDatabase(); - String table_name = table_name_column->getDataAt(i).toString(); - if (auto policies = context.getRowPolicies()) - { - for (const auto & policy_id : policies->getCurrentPolicyIDs(database, table_name)) - policy_id_column->insertValue(policy_id); - } - offset_column->insertValue(policy_id_column->size()); - } - - block.getByPosition(result_pos).column = ColumnArray::create(std::move(policy_id_column), std::move(offset_column)); - } - -private: - const Context & context; -}; - - -void registerFunctionCurrentRowPolicies(FunctionFactory & factory) -{ - factory.registerFunction(); - factory.registerFunction(); -} - -} diff --git a/src/Functions/registerFunctionsMiscellaneous.cpp b/src/Functions/registerFunctionsMiscellaneous.cpp index 221e14fcce1..737cde10e74 100644 --- a/src/Functions/registerFunctionsMiscellaneous.cpp +++ b/src/Functions/registerFunctionsMiscellaneous.cpp @@ -10,7 +10,6 @@ class FunctionFactory; void registerFunctionCurrentDatabase(FunctionFactory &); void registerFunctionCurrentUser(FunctionFactory &); void registerFunctionCurrentQuota(FunctionFactory &); -void registerFunctionCurrentRowPolicies(FunctionFactory &); void registerFunctionHostName(FunctionFactory &); void registerFunctionFQDN(FunctionFactory &); void registerFunctionVisibleWidth(FunctionFactory &); @@ -69,7 +68,6 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory) registerFunctionCurrentDatabase(factory); registerFunctionCurrentUser(factory); registerFunctionCurrentQuota(factory); - registerFunctionCurrentRowPolicies(factory); registerFunctionHostName(factory); registerFunctionFQDN(factory); registerFunctionVisibleWidth(factory); diff --git a/src/Functions/ya.make b/src/Functions/ya.make index afce90f0dee..087fe3824c6 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -128,7 +128,6 @@ SRCS( CRC.cpp currentDatabase.cpp currentQuota.cpp - currentRowPolicies.cpp currentUser.cpp dateDiff.cpp defaultValueOfArgumentType.cpp diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index d6176c50ba4..ebeeba54d1d 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -776,11 +776,6 @@ ASTPtr Context::getRowPolicyCondition(const String & database, const String & ta return getAccess()->getRowPolicyCondition(database, table_name, type, initial_condition); } -std::shared_ptr Context::getRowPolicies() const -{ - return getAccess()->getRowPolicies(); -} - void Context::setInitialRowPolicy() { auto lock = getLock(); diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 6cb2893af98..f0ae44f5548 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -278,7 +278,6 @@ public: std::shared_ptr getAccess() const; - std::shared_ptr getRowPolicies() const; ASTPtr getRowPolicyCondition(const String & database, const String & table_name, RowPolicy::ConditionType type) const; /// Sets an extra row policy based on `client_info.initial_user`, if it exists. diff --git a/src/Interpreters/InterpreterCreateRowPolicyQuery.cpp b/src/Interpreters/InterpreterCreateRowPolicyQuery.cpp index 1d3c3a47bad..778a32019d9 100644 --- a/src/Interpreters/InterpreterCreateRowPolicyQuery.cpp +++ b/src/Interpreters/InterpreterCreateRowPolicyQuery.cpp @@ -29,8 +29,12 @@ namespace if (query.is_restrictive) policy.setRestrictive(*query.is_restrictive); - for (const auto & [index, condition] : query.conditions) - policy.conditions[index] = condition ? serializeAST(*condition) : String{}; + for (auto condition_type : ext::range(RowPolicy::MAX_CONDITION_TYPE)) + { + const auto & condition = query.conditions[condition_type]; + if (condition) + policy.conditions[condition_type] = *condition ? serializeAST(**condition) : String{}; + } const ExtendedRoleSet * roles = nullptr; std::optional temp_role_set; diff --git a/src/Interpreters/InterpreterFactory.cpp b/src/Interpreters/InterpreterFactory.cpp index 0c34d6ed79f..dae025a1b30 100644 --- a/src/Interpreters/InterpreterFactory.cpp +++ b/src/Interpreters/InterpreterFactory.cpp @@ -16,11 +16,11 @@ #include #include #include +#include #include -#include #include #include -#include +#include #include #include #include @@ -50,12 +50,12 @@ #include #include #include +#include #include -#include -#include #include #include -#include +#include +#include #include #include #include @@ -230,9 +230,9 @@ std::unique_ptr InterpreterFactory::get(ASTPtr & query, Context & { return std::make_unique(query, context); } - else if (query->as()) + else if (query->as()) { - return std::make_unique(query, context); + return std::make_unique(query, context); } else throw Exception("Unknown type of query: " + query->getID(), ErrorCodes::UNKNOWN_TYPE_OF_QUERY); diff --git a/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp b/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp new file mode 100644 index 00000000000..8a892685ed4 --- /dev/null +++ b/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp @@ -0,0 +1,67 @@ +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int NOT_IMPLEMENTED; +} + +using EntityType = IAccessEntity::Type; + + +InterpreterShowAccessEntitiesQuery::InterpreterShowAccessEntitiesQuery(const ASTPtr & query_ptr_, Context & context_) + : query_ptr(query_ptr_), context(context_) +{ +} + + +BlockIO InterpreterShowAccessEntitiesQuery::execute() +{ + return executeQuery(getRewrittenQuery(), context, true); +} + + +String InterpreterShowAccessEntitiesQuery::getRewrittenQuery() const +{ + const auto & query = query_ptr->as(); + String origin; + String expr = "name"; + String filter; + + if (query.type == EntityType::ROW_POLICY) + { + origin = "row_policies"; + + const String & table_name = query.table_name; + String database; + bool show_short_name = false; + if (!table_name.empty()) + { + database = query.database; + if (database.empty()) + database = context.getCurrentDatabase(); + show_short_name = true; + } + + if (!table_name.empty()) + filter = "database = " + quoteString(database) + " AND table = " + quoteString(table_name); + + if (show_short_name) + expr = "short_name"; + } + else + throw Exception(toString(query.type) + ": type is not supported by SHOW query", ErrorCodes::NOT_IMPLEMENTED); + + return "SELECT " + expr + " from system." + origin + + (filter.empty() ? "" : " WHERE " + filter) + " ORDER BY " + expr; +} + +} diff --git a/src/Interpreters/InterpreterShowRowPoliciesQuery.h b/src/Interpreters/InterpreterShowAccessEntitiesQuery.h similarity index 58% rename from src/Interpreters/InterpreterShowRowPoliciesQuery.h rename to src/Interpreters/InterpreterShowAccessEntitiesQuery.h index 84cf6299b84..f5bf0d0c7bb 100644 --- a/src/Interpreters/InterpreterShowRowPoliciesQuery.h +++ b/src/Interpreters/InterpreterShowAccessEntitiesQuery.h @@ -8,15 +8,14 @@ namespace DB { class Context; -class InterpreterShowRowPoliciesQuery : public IInterpreter +class InterpreterShowAccessEntitiesQuery : public IInterpreter { public: - InterpreterShowRowPoliciesQuery(const ASTPtr & query_ptr_, Context & context_); + InterpreterShowAccessEntitiesQuery(const ASTPtr & query_ptr_, Context & context_); BlockIO execute() override; private: String getRewrittenQuery() const; - String getResultDescription() const; ASTPtr query_ptr; Context & context; diff --git a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp index 11354d1f2a5..51ac3644ff5 100644 --- a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp +++ b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp @@ -167,14 +167,14 @@ namespace if (policy.isRestrictive()) query->is_restrictive = policy.isRestrictive(); - for (auto index : ext::range(RowPolicy::MAX_CONDITION_TYPE)) + for (auto type : ext::range(RowPolicy::MAX_CONDITION_TYPE)) { - const auto & condition = policy.conditions[index]; + const auto & condition = policy.conditions[static_cast(type)]; if (!condition.empty()) { ParserExpression parser; ASTPtr expr = parseQuery(parser, condition, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); - query->conditions.push_back(std::pair{index, expr}); + query->conditions[static_cast(type)] = expr; } } diff --git a/src/Interpreters/InterpreterShowRowPoliciesQuery.cpp b/src/Interpreters/InterpreterShowRowPoliciesQuery.cpp deleted file mode 100644 index a4405615ceb..00000000000 --- a/src/Interpreters/InterpreterShowRowPoliciesQuery.cpp +++ /dev/null @@ -1,69 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace DB -{ -InterpreterShowRowPoliciesQuery::InterpreterShowRowPoliciesQuery(const ASTPtr & query_ptr_, Context & context_) - : query_ptr(query_ptr_), context(context_) -{ -} - - -BlockIO InterpreterShowRowPoliciesQuery::execute() -{ - return executeQuery(getRewrittenQuery(), context, true); -} - - -String InterpreterShowRowPoliciesQuery::getRewrittenQuery() const -{ - const auto & query = query_ptr->as(); - - const String & table_name = query.table_name; - String database; - if (!table_name.empty()) - { - database = query.database; - if (database.empty()) - database = context.getCurrentDatabase(); - } - - String filter; - if (query.current) - { - if (table_name.empty()) - filter = "has(currentRowPolicyIDs(), id)"; - else - filter = "has(currentRowPolicyIDs(" + quoteString(database) + ", " + quoteString(table_name) + "), id)"; - } - else - { - if (!table_name.empty()) - filter = "database = " + quoteString(database) + " AND table = " + quoteString(table_name); - } - - String expr = table_name.empty() ? "name" : "short_name"; - - return "SELECT " + expr + " AS " + backQuote(getResultDescription()) + " from system.row_policies" - + (filter.empty() ? "" : " WHERE " + filter) + " ORDER BY " + expr; -} - - -String InterpreterShowRowPoliciesQuery::getResultDescription() const -{ - std::stringstream ss; - formatAST(*query_ptr, ss, false, true); - String desc = ss.str(); - String prefix = "SHOW "; - if (startsWith(desc, prefix)) - desc = desc.substr(prefix.length()); /// `desc` always starts with "SHOW ", so we can trim this prefix. - return desc; -} -} diff --git a/src/Interpreters/ya.make b/src/Interpreters/ya.make index 12b9f12a2e2..440e1e1e1c0 100644 --- a/src/Interpreters/ya.make +++ b/src/Interpreters/ya.make @@ -83,12 +83,12 @@ SRCS( InterpreterSelectWithUnionQuery.cpp InterpreterSetQuery.cpp InterpreterSetRoleQuery.cpp + InterpreterShowAccessEntitiesQuery.cpp InterpreterShowCreateAccessEntityQuery.cpp InterpreterShowCreateQuery.cpp InterpreterShowGrantsQuery.cpp InterpreterShowProcesslistQuery.cpp InterpreterShowQuotasQuery.cpp - InterpreterShowRowPoliciesQuery.cpp InterpreterShowTablesQuery.cpp InterpreterSystemQuery.cpp InterpreterUseQuery.cpp diff --git a/src/Parsers/ASTCreateRowPolicyQuery.cpp b/src/Parsers/ASTCreateRowPolicyQuery.cpp index ed11ab936ff..973f3c6b930 100644 --- a/src/Parsers/ASTCreateRowPolicyQuery.cpp +++ b/src/Parsers/ASTCreateRowPolicyQuery.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -11,6 +12,9 @@ namespace DB namespace { using ConditionType = RowPolicy::ConditionType; + using ConditionTypeInfo = RowPolicy::ConditionTypeInfo; + constexpr auto MAX_CONDITION_TYPE = RowPolicy::MAX_CONDITION_TYPE; + void formatRenameTo(const String & new_short_name, const IAST::FormatSettings & settings) { @@ -28,90 +32,89 @@ namespace void formatConditionalExpression(const ASTPtr & expr, const IAST::FormatSettings & settings) { - if (!expr) - { + if (expr) + expr->format(settings); + else settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " NONE" << (settings.hilite ? IAST::hilite_none : ""); - return; - } - expr->format(settings); } - std::vector> - conditionalExpressionsToStrings(const std::vector> & exprs, const IAST::FormatSettings & settings) + void formatCondition(const boost::container::flat_set & commands, const String & filter, const String & check, bool alter, const IAST::FormatSettings & settings) { - std::vector> result; - std::stringstream ss; - IAST::FormatSettings temp_settings(ss, settings); - boost::range::transform(exprs, std::back_inserter(result), [&](const std::pair & in) - { - formatConditionalExpression(in.second, temp_settings); - auto out = std::pair{in.first, ss.str()}; - ss.str(""); - return out; - }); - return result; - } - - - void formatConditions(const char * op, const std::optional & filter, const std::optional & check, bool alter, const IAST::FormatSettings & settings) - { - if (op) - { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " FOR" << (settings.hilite ? IAST::hilite_none : ""); - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << ' ' << op << (settings.hilite ? IAST::hilite_none : ""); - } - - if (filter) - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " USING " << (settings.hilite ? IAST::hilite_none : "") << *filter; - - if (check && (alter || (check != filter))) - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " WITH CHECK " << (settings.hilite ? IAST::hilite_none : "") << *check; - } - - - void formatMultipleConditions(const std::vector> & conditions, bool alter, const IAST::FormatSettings & settings) - { - std::optional scond[RowPolicy::MAX_CONDITION_TYPE]; - for (const auto & [index, scondition] : conditionalExpressionsToStrings(conditions, settings)) - scond[index] = scondition; - - if ((scond[RowPolicy::SELECT_FILTER] == scond[RowPolicy::UPDATE_FILTER]) - && (scond[RowPolicy::UPDATE_FILTER] == scond[RowPolicy::DELETE_FILTER]) - && (scond[RowPolicy::INSERT_CHECK] == scond[RowPolicy::UPDATE_CHECK]) - && (scond[RowPolicy::SELECT_FILTER] || scond[RowPolicy::INSERT_CHECK])) - { - formatConditions(nullptr, scond[RowPolicy::SELECT_FILTER], scond[RowPolicy::INSERT_CHECK], alter, settings); - return; - } - + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " FOR " << (settings.hilite ? IAST::hilite_none : ""); bool need_comma = false; - if (scond[RowPolicy::SELECT_FILTER]) + for (const auto & command : commands) { if (std::exchange(need_comma, true)) - settings.ostr << ','; - formatConditions("SELECT", scond[RowPolicy::SELECT_FILTER], {}, alter, settings); - } - if (scond[RowPolicy::INSERT_CHECK]) - { - if (std::exchange(need_comma, true)) - settings.ostr << ','; - formatConditions("INSERT", {}, scond[RowPolicy::INSERT_CHECK], alter, settings); - } - if (scond[RowPolicy::UPDATE_FILTER] || scond[RowPolicy::UPDATE_CHECK]) - { - if (std::exchange(need_comma, true)) - settings.ostr << ','; - formatConditions("UPDATE", scond[RowPolicy::UPDATE_FILTER], scond[RowPolicy::UPDATE_CHECK], alter, settings); - } - if (scond[RowPolicy::DELETE_FILTER]) - { - if (std::exchange(need_comma, true)) - settings.ostr << ','; - formatConditions("DELETE", scond[RowPolicy::DELETE_FILTER], {}, alter, settings); + settings.ostr << ", "; + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << command << (settings.hilite ? IAST::hilite_none : ""); } + + if (!filter.empty()) + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " USING " << (settings.hilite ? IAST::hilite_none : "") << filter; + + if (!check.empty() && (alter || (check != filter))) + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " WITH CHECK " << (settings.hilite ? IAST::hilite_none : "") << check; } + + void formatMultipleConditions(const std::array, MAX_CONDITION_TYPE> & conditions, bool alter, const IAST::FormatSettings & settings) + { + std::array conditions_as_strings; + std::stringstream temp_sstream; + IAST::FormatSettings temp_settings(temp_sstream, settings); + for (auto condition_type : ext::range(MAX_CONDITION_TYPE)) + { + const auto & condition = conditions[condition_type]; + if (condition) + { + formatConditionalExpression(*condition, temp_settings); + conditions_as_strings[condition_type] = temp_sstream.str(); + temp_sstream.str(""); + } + } + + boost::container::flat_set commands; + String filter, check; + + do + { + commands.clear(); + filter.clear(); + check.clear(); + + /// Collect commands using the same filter and check conditions. + for (auto condition_type : ext::range(MAX_CONDITION_TYPE)) + { + const String & condition = conditions_as_strings[condition_type]; + if (condition.empty()) + continue; + const auto & type_info = ConditionTypeInfo::get(condition_type); + if (type_info.is_check) + { + if (check.empty()) + check = condition; + else if (check != condition) + continue; + } + else + { + if (filter.empty()) + filter = condition; + else if (filter != condition) + continue; + } + commands.emplace(type_info.command); + conditions_as_strings[condition_type].clear(); /// Skip this condition on the next iteration. + } + + if (!filter.empty() || !check.empty()) + formatCondition(commands, filter, check, alter, settings); + } + while (!filter.empty() || !check.empty()); + } + + void formatToRoles(const ASTExtendedRoleSet & roles, const IAST::FormatSettings & settings) { settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " TO " << (settings.hilite ? IAST::hilite_none : ""); diff --git a/src/Parsers/ASTCreateRowPolicyQuery.h b/src/Parsers/ASTCreateRowPolicyQuery.h index dc14d40c7ad..8aa44b784aa 100644 --- a/src/Parsers/ASTCreateRowPolicyQuery.h +++ b/src/Parsers/ASTCreateRowPolicyQuery.h @@ -3,8 +3,8 @@ #include #include #include -#include -#include +#include +#include namespace DB @@ -40,8 +40,7 @@ public: String new_short_name; std::optional is_restrictive; - using ConditionType = RowPolicy::ConditionType; - std::vector> conditions; + std::array, RowPolicy::MAX_CONDITION_TYPE> conditions; /// `nullopt` means "not set", `nullptr` means set to NONE. std::shared_ptr roles; diff --git a/src/Parsers/ASTShowAccessEntitiesQuery.cpp b/src/Parsers/ASTShowAccessEntitiesQuery.cpp new file mode 100644 index 00000000000..ec67fc5fb7f --- /dev/null +++ b/src/Parsers/ASTShowAccessEntitiesQuery.cpp @@ -0,0 +1,37 @@ +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int NOT_IMPLEMENTED; +} + + +String ASTShowAccessEntitiesQuery::getID(char) const +{ + if (type == EntityType::ROW_POLICY) + return "SHOW ROW POLICIES query"; + else + throw Exception(toString(type) + ": type is not supported by SHOW query", ErrorCodes::NOT_IMPLEMENTED); +} + +void ASTShowAccessEntitiesQuery::formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +{ + if (type == EntityType::ROW_POLICY) + settings.ostr << (settings.hilite ? hilite_keyword : "") << "SHOW ROW POLICIES" << (settings.hilite ? hilite_none : ""); + else + throw Exception(toString(type) + ": type is not supported by SHOW query", ErrorCodes::NOT_IMPLEMENTED); + + if ((type == EntityType::ROW_POLICY) && !table_name.empty()) + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << " ON " << (settings.hilite ? hilite_none : ""); + if (!database.empty()) + settings.ostr << backQuoteIfNeed(database) << "."; + settings.ostr << backQuoteIfNeed(table_name); + } +} + +} diff --git a/src/Parsers/ASTShowRowPoliciesQuery.h b/src/Parsers/ASTShowAccessEntitiesQuery.h similarity index 53% rename from src/Parsers/ASTShowRowPoliciesQuery.h rename to src/Parsers/ASTShowAccessEntitiesQuery.h index ce82902e96d..82d3474f3ad 100644 --- a/src/Parsers/ASTShowRowPoliciesQuery.h +++ b/src/Parsers/ASTShowAccessEntitiesQuery.h @@ -1,20 +1,24 @@ #pragma once #include +#include namespace DB { -/// SHOW [ROW] POLICIES [CURRENT] [ON [database.]table] -class ASTShowRowPoliciesQuery : public ASTQueryWithOutput + +/// SHOW [ROW] POLICIES [ON [database.]table] +class ASTShowAccessEntitiesQuery : public ASTQueryWithOutput { public: - bool current = false; + using EntityType = IAccessEntity::Type; + + EntityType type; String database; String table_name; - String getID(char) const override { return "SHOW POLICIES query"; } - ASTPtr clone() const override { return std::make_shared(*this); } + String getID(char) const override; + ASTPtr clone() const override { return std::make_shared(*this); } protected: void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; diff --git a/src/Parsers/ASTShowRowPoliciesQuery.cpp b/src/Parsers/ASTShowRowPoliciesQuery.cpp deleted file mode 100644 index 15e0e81f218..00000000000 --- a/src/Parsers/ASTShowRowPoliciesQuery.cpp +++ /dev/null @@ -1,22 +0,0 @@ -#include -#include - - -namespace DB -{ -void ASTShowRowPoliciesQuery::formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const -{ - settings.ostr << (settings.hilite ? hilite_keyword : "") << "SHOW POLICIES" << (settings.hilite ? hilite_none : ""); - - if (current) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " CURRENT" << (settings.hilite ? hilite_none : ""); - - if (!table_name.empty()) - { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " ON " << (settings.hilite ? hilite_none : ""); - if (!database.empty()) - settings.ostr << backQuoteIfNeed(database) << "."; - settings.ostr << backQuoteIfNeed(table_name); - } -} -} diff --git a/src/Parsers/ParserCreateRowPolicyQuery.cpp b/src/Parsers/ParserCreateRowPolicyQuery.cpp index 22fcf47b379..2456cb80368 100644 --- a/src/Parsers/ParserCreateRowPolicyQuery.cpp +++ b/src/Parsers/ParserCreateRowPolicyQuery.cpp @@ -8,18 +8,17 @@ #include #include #include +#include namespace DB { -namespace ErrorCodes -{ -} - - namespace { using ConditionType = RowPolicy::ConditionType; + using ConditionTypeInfo = RowPolicy::ConditionTypeInfo; + constexpr auto MAX_CONDITION_TYPE = RowPolicy::MAX_CONDITION_TYPE; + bool parseRenameTo(IParserBase::Pos & pos, Expected & expected, String & new_short_name) { @@ -73,111 +72,93 @@ namespace }); } - bool parseConditions(IParserBase::Pos & pos, Expected & expected, bool alter, std::vector> & conditions) + bool parseConditions( + IParserBase::Pos & pos, Expected & expected, bool alter, std::array, MAX_CONDITION_TYPE> & conditions) { return IParserBase::wrapParseImpl(pos, [&] { - static constexpr char select_op[] = "SELECT"; - static constexpr char insert_op[] = "INSERT"; - static constexpr char update_op[] = "UPDATE"; - static constexpr char delete_op[] = "DELETE"; - std::vector ops; + boost::container::flat_set commands; + + auto add_all_commands = [&] + { + for (auto condition_type : ext::range(MAX_CONDITION_TYPE)) + { + const std::string_view & command = ConditionTypeInfo::get(condition_type).command; + commands.emplace(command); + } + }; if (ParserKeyword{"FOR"}.ignore(pos, expected)) { do { - if (ParserKeyword{"SELECT"}.ignore(pos, expected)) - ops.push_back(select_op); -#if 0 /// INSERT, UPDATE, DELETE are not supported yet - else if (ParserKeyword{"INSERT"}.ignore(pos, expected)) - ops.push_back(insert_op); - else if (ParserKeyword{"UPDATE"}.ignore(pos, expected)) - ops.push_back(update_op); - else if (ParserKeyword{"DELETE"}.ignore(pos, expected)) - ops.push_back(delete_op); - else if (ParserKeyword{"ALL"}.ignore(pos, expected)) + size_t old_size = commands.size(); + if (ParserKeyword{"ALL"}.ignore(pos, expected)) { + add_all_commands(); } -#endif else + { + for (auto condition_type : ext::range(MAX_CONDITION_TYPE)) + { + const std::string_view & command = ConditionTypeInfo::get(condition_type).command; + if (ParserKeyword{command.data()}.ignore(pos, expected)) + { + commands.emplace(command); + break; + } + } + } + if (commands.size() == old_size) return false; } while (ParserToken{TokenType::Comma}.ignore(pos, expected)); } - if (ops.empty()) - { - ops.push_back(select_op); -#if 0 /// INSERT, UPDATE, DELETE are not supported yet - ops.push_back(insert_op); - ops.push_back(update_op); - ops.push_back(delete_op); -#endif - } - std::optional filter; std::optional check; - bool keyword_using = false, keyword_with_check = false; if (ParserKeyword{"USING"}.ignore(pos, expected)) { - keyword_using = true; if (!parseConditionalExpression(pos, expected, filter)) return false; } -#if 0 /// INSERT, UPDATE, DELETE are not supported yet if (ParserKeyword{"WITH CHECK"}.ignore(pos, expected)) { - keyword_with_check = true; if (!parseConditionalExpression(pos, expected, check)) return false; } -#endif - if (!keyword_using && !keyword_with_check) + + if (!filter && !check) return false; - if (filter && !check && !alter) + if (commands.empty()) + add_all_commands(); + + if (!check && !alter) check = filter; - auto set_condition = [&](ConditionType index, const ASTPtr & condition) + for (auto condition_type : ext::range(MAX_CONDITION_TYPE)) { - auto it = std::find_if(conditions.begin(), conditions.end(), [index](const std::pair & element) + const auto & type_info = ConditionTypeInfo::get(condition_type); + if (commands.count(type_info.command)) { - return element.first == index; - }); - if (it == conditions.end()) - it = conditions.insert(conditions.end(), std::pair{index, nullptr}); - it->second = condition; - }; - - for (const auto & op : ops) - { - if ((op == select_op) && filter) - set_condition(RowPolicy::SELECT_FILTER, *filter); - else if ((op == insert_op) && check) - set_condition(RowPolicy::INSERT_CHECK, *check); - else if (op == update_op) - { - if (filter) - set_condition(RowPolicy::UPDATE_FILTER, *filter); - if (check) - set_condition(RowPolicy::UPDATE_CHECK, *check); + if (type_info.is_check && check) + conditions[condition_type] = check; + else if (filter) + conditions[condition_type] = filter; } - else if ((op == delete_op) && filter) - set_condition(RowPolicy::DELETE_FILTER, *filter); - else - __builtin_unreachable(); } return true; }); } - bool parseMultipleConditions(IParserBase::Pos & pos, Expected & expected, bool alter, std::vector> & conditions) + bool parseMultipleConditions( + IParserBase::Pos & pos, Expected & expected, bool alter, std::array, MAX_CONDITION_TYPE> & conditions) { return IParserBase::wrapParseImpl(pos, [&] { - std::vector> res_conditions; + std::array, MAX_CONDITION_TYPE> res_conditions; do { if (!parseConditions(pos, expected, alter, res_conditions)) @@ -256,7 +237,7 @@ bool ParserCreateRowPolicyQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & String new_short_name; std::optional is_restrictive; - std::vector> conditions; + std::array, MAX_CONDITION_TYPE> conditions; String cluster; while (true) diff --git a/src/Parsers/ParserQueryWithOutput.cpp b/src/Parsers/ParserQueryWithOutput.cpp index f818e72243f..93329723c2b 100644 --- a/src/Parsers/ParserQueryWithOutput.cpp +++ b/src/Parsers/ParserQueryWithOutput.cpp @@ -14,10 +14,10 @@ #include #include #include -#include +#include #include +#include #include -#include namespace DB @@ -38,10 +38,10 @@ bool ParserQueryWithOutput::parseImpl(Pos & pos, ASTPtr & node, Expected & expec ParserOptimizeQuery optimize_p; ParserKillQueryQuery kill_query_p; ParserWatchQuery watch_p; + ParserShowAccessEntitiesQuery show_access_entities_p; ParserShowCreateAccessEntityQuery show_create_access_entity_p; ParserShowGrantsQuery show_grants_p; ParserShowQuotasQuery show_quotas_p; - ParserShowRowPoliciesQuery show_row_policies_p; ASTPtr query; @@ -70,9 +70,9 @@ bool ParserQueryWithOutput::parseImpl(Pos & pos, ASTPtr & node, Expected & expec || kill_query_p.parse(pos, query, expected) || optimize_p.parse(pos, query, expected) || watch_p.parse(pos, query, expected) + || show_access_entities_p.parse(pos, query, expected) || show_grants_p.parse(pos, query, expected) - || show_quotas_p.parse(pos, query, expected) - || show_row_policies_p.parse(pos, query, expected); + || show_quotas_p.parse(pos, query, expected); if (!parsed) return false; diff --git a/src/Parsers/ParserShowRowPoliciesQuery.cpp b/src/Parsers/ParserShowAccessEntitiesQuery.cpp similarity index 72% rename from src/Parsers/ParserShowRowPoliciesQuery.cpp rename to src/Parsers/ParserShowAccessEntitiesQuery.cpp index b07e7a386ba..cd16633c434 100644 --- a/src/Parsers/ParserShowRowPoliciesQuery.cpp +++ b/src/Parsers/ParserShowAccessEntitiesQuery.cpp @@ -1,5 +1,5 @@ -#include -#include +#include +#include #include #include @@ -8,6 +8,8 @@ namespace DB { namespace { + using EntityType = IAccessEntity::Type; + bool parseONDatabaseAndTableName(IParserBase::Pos & pos, Expected & expected, String & database, String & table_name) { return IParserBase::wrapParseImpl(pos, [&] @@ -20,21 +22,21 @@ namespace } -bool ParserShowRowPoliciesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +bool ParserShowAccessEntitiesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { if (!ParserKeyword{"SHOW POLICIES"}.ignore(pos, expected) && !ParserKeyword{"SHOW ROW POLICIES"}.ignore(pos, expected)) return false; - bool current = ParserKeyword{"CURRENT"}.ignore(pos, expected); - String database, table_name; parseONDatabaseAndTableName(pos, expected, database, table_name); - auto query = std::make_shared(); - query->current = current; + auto query = std::make_shared(); + node = query; + + query->type = EntityType::ROW_POLICY; query->database = std::move(database); query->table_name = std::move(table_name); - node = query; + return true; } } diff --git a/src/Parsers/ParserShowRowPoliciesQuery.h b/src/Parsers/ParserShowAccessEntitiesQuery.h similarity index 50% rename from src/Parsers/ParserShowRowPoliciesQuery.h rename to src/Parsers/ParserShowAccessEntitiesQuery.h index df7413fb604..05b9550cdf8 100644 --- a/src/Parsers/ParserShowRowPoliciesQuery.h +++ b/src/Parsers/ParserShowAccessEntitiesQuery.h @@ -6,12 +6,12 @@ namespace DB { /** Parses queries like - * SHOW [ROW] POLICIES [CURRENT] [ON [database.]table] + * SHOW [ROW] POLICIES [ON [database.]table] */ -class ParserShowRowPoliciesQuery : public IParserBase +class ParserShowAccessEntitiesQuery : public IParserBase { protected: - const char * getName() const override { return "SHOW POLICIES query"; } + const char * getName() const override { return "ShowAccessEntitiesQuery"; } bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; } diff --git a/src/Parsers/ya.make b/src/Parsers/ya.make index 942124c2c7a..2490bcf24c4 100644 --- a/src/Parsers/ya.make +++ b/src/Parsers/ya.make @@ -42,10 +42,10 @@ SRCS( ASTSelectWithUnionQuery.cpp ASTSetRoleQuery.cpp ASTSettingsProfileElement.cpp + ASTShowAccessEntitiesQuery.cpp ASTShowCreateAccessEntityQuery.cpp ASTShowGrantsQuery.cpp ASTShowQuotasQuery.cpp - ASTShowRowPoliciesQuery.cpp ASTShowTablesQuery.cpp ASTSubquery.cpp ASTSystemQuery.cpp @@ -94,10 +94,10 @@ SRCS( ParserSetQuery.cpp ParserSetRoleQuery.cpp ParserSettingsProfileElement.cpp + ParserShowAccessEntitiesQuery.cpp ParserShowCreateAccessEntityQuery.cpp ParserShowGrantsQuery.cpp ParserShowQuotasQuery.cpp - ParserShowRowPoliciesQuery.cpp ParserShowTablesQuery.cpp ParserSystemQuery.cpp ParserTablePropertiesQuery.cpp diff --git a/src/Storages/System/StorageSystemRowPolicies.cpp b/src/Storages/System/StorageSystemRowPolicies.cpp index cacd657b46f..ca77a5182a5 100644 --- a/src/Storages/System/StorageSystemRowPolicies.cpp +++ b/src/Storages/System/StorageSystemRowPolicies.cpp @@ -2,32 +2,52 @@ #include #include #include -#include #include +#include +#include +#include +#include +#include #include +#include #include #include #include #include +#include namespace DB { +using ConditionTypeInfo = RowPolicy::ConditionTypeInfo; +constexpr auto MAX_CONDITION_TYPE = RowPolicy::MAX_CONDITION_TYPE; + + NamesAndTypesList StorageSystemRowPolicies::getNamesAndTypes() { NamesAndTypesList names_and_types{ + {"name", std::make_shared()}, + {"short_name", std::make_shared()}, {"database", std::make_shared()}, {"table", std::make_shared()}, - {"short_name", std::make_shared()}, - {"name", std::make_shared()}, {"id", std::make_shared()}, {"source", std::make_shared()}, - {"restrictive", std::make_shared()}, }; - for (auto index : ext::range(RowPolicy::MAX_CONDITION_TYPE)) - names_and_types.push_back({RowPolicy::conditionTypeToColumnName(index), std::make_shared()}); + for (auto type : ext::range(MAX_CONDITION_TYPE)) + { + const String & column_name = ConditionTypeInfo::get(type).name; + names_and_types.push_back({column_name, std::make_shared(std::make_shared())}); + } + NamesAndTypesList extra_names_and_types{ + {"is_restrictive", std::make_shared()}, + {"apply_to_all", std::make_shared()}, + {"apply_to_list", std::make_shared(std::make_shared())}, + {"apply_to_except", std::make_shared(std::make_shared())} + }; + + boost::range::push_back(names_and_types, std::move(extra_names_and_types)); return names_and_types; } @@ -38,24 +58,83 @@ void StorageSystemRowPolicies::fillData(MutableColumns & res_columns, const Cont const auto & access_control = context.getAccessControlManager(); std::vector ids = access_control.findAll(); + size_t column_index = 0; + auto & column_name = assert_cast(*res_columns[column_index++]); + auto & column_short_name = assert_cast(*res_columns[column_index++]); + auto & column_database = assert_cast(*res_columns[column_index++]); + auto & column_table = assert_cast(*res_columns[column_index++]); + auto & column_id = assert_cast(*res_columns[column_index++]).getData(); + auto & column_storage = assert_cast(*res_columns[column_index++]); + + ColumnString * column_condition[MAX_CONDITION_TYPE]; + NullMap * column_condition_null_map[MAX_CONDITION_TYPE]; + for (auto condition_type : ext::range(MAX_CONDITION_TYPE)) + { + column_condition[condition_type] = &assert_cast(assert_cast(*res_columns[column_index]).getNestedColumn()); + column_condition_null_map[condition_type] = &assert_cast(*res_columns[column_index++]).getNullMapData(); + } + + auto & column_is_restrictive = assert_cast(*res_columns[column_index++]).getData(); + auto & column_apply_to_all = assert_cast(*res_columns[column_index++]).getData(); + auto & column_apply_to_list = assert_cast(assert_cast(*res_columns[column_index]).getData()); + auto & column_apply_to_list_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); + auto & column_apply_to_except = assert_cast(assert_cast(*res_columns[column_index]).getData()); + auto & column_apply_to_except_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); + + auto add_row = [&](const String & name, + const RowPolicy::NameParts & name_parts, + const UUID & id, + const String & storage_name, + const std::array & conditions, + bool is_restrictive, + const ExtendedRoleSet & apply_to) + { + column_name.insertData(name.data(), name.length()); + column_short_name.insertData(name_parts.short_name.data(), name_parts.short_name.length()); + column_database.insertData(name_parts.database.data(), name_parts.database.length()); + column_table.insertData(name_parts.table_name.data(), name_parts.table_name.length()); + column_id.push_back(id); + column_storage.insertData(storage_name.data(), storage_name.length()); + + for (auto condition_type : ext::range(MAX_CONDITION_TYPE)) + { + const String & condition = conditions[condition_type]; + if (condition.empty()) + { + column_condition[condition_type]->insertDefault(); + column_condition_null_map[condition_type]->push_back(true); + } + else + { + column_condition[condition_type]->insertData(condition.data(), condition.length()); + column_condition_null_map[condition_type]->push_back(false); + } + } + + column_is_restrictive.push_back(is_restrictive); + + auto apply_to_ast = apply_to.toASTWithNames(access_control); + column_apply_to_all.push_back(apply_to_ast->all); + + for (const auto & role_name : apply_to_ast->names) + column_apply_to_list.insertData(role_name.data(), role_name.length()); + column_apply_to_list_offsets.push_back(column_apply_to_list.size()); + + for (const auto & role_name : apply_to_ast->except_names) + column_apply_to_except.insertData(role_name.data(), role_name.length()); + column_apply_to_except_offsets.push_back(column_apply_to_except.size()); + }; + for (const auto & id : ids) { auto policy = access_control.tryRead(id); if (!policy) continue; const auto * storage = access_control.findStorage(id); + if (!storage) + continue; - size_t i = 0; - res_columns[i++]->insert(policy->getDatabase()); - res_columns[i++]->insert(policy->getTableName()); - res_columns[i++]->insert(policy->getShortName()); - res_columns[i++]->insert(policy->getName()); - res_columns[i++]->insert(id); - res_columns[i++]->insert(storage ? storage->getStorageName() : ""); - res_columns[i++]->insert(policy->isRestrictive()); - - for (auto index : ext::range(RowPolicy::MAX_CONDITION_TYPE)) - res_columns[i++]->insert(policy->conditions[index]); + add_row(policy->getName(), policy->getNameParts(), id, storage->getStorageName(), policy->conditions, policy->isRestrictive(), policy->to_roles); } } } diff --git a/tests/integration/test_row_policy/configs/config.d/remote_servers.xml b/tests/integration/test_row_policy/configs/config.d/remote_servers.xml index 8eea90f523b..395e97b97ac 100644 --- a/tests/integration/test_row_policy/configs/config.d/remote_servers.xml +++ b/tests/integration/test_row_policy/configs/config.d/remote_servers.xml @@ -3,13 +3,13 @@ - instance1 + node 9000 - instance2 + node2 9000 diff --git a/tests/integration/test_row_policy/test.py b/tests/integration/test_row_policy/test.py index ec7304f47d3..a1884d059c7 100644 --- a/tests/integration/test_row_policy/test.py +++ b/tests/integration/test_row_policy/test.py @@ -1,21 +1,22 @@ import pytest from helpers.cluster import ClickHouseCluster -from helpers.test_tools import assert_eq_with_retry +from helpers.test_tools import assert_eq_with_retry, TSV import os import re import time cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance1', config_dir="configs", with_zookeeper=True) -instance2 = cluster.add_instance('instance2', config_dir="configs", with_zookeeper=True) +node = cluster.add_instance('node', config_dir="configs", with_zookeeper=True) +node2 = cluster.add_instance('node2', config_dir="configs", with_zookeeper=True) +nodes = [node, node2] def copy_policy_xml(local_file_name, reload_immediately = True): script_dir = os.path.dirname(os.path.realpath(__file__)) - instance.copy_file_to_container(os.path.join(script_dir, local_file_name), '/etc/clickhouse-server/users.d/row_policy.xml') - instance2.copy_file_to_container(os.path.join(script_dir, local_file_name), '/etc/clickhouse-server/users.d/row_policy.xml') - if reload_immediately: - instance.query("SYSTEM RELOAD CONFIG") + for current_node in nodes: + current_node.copy_file_to_container(os.path.join(script_dir, local_file_name), '/etc/clickhouse-server/users.d/row_policy.xml') + if reload_immediately: + current_node.query("SYSTEM RELOAD CONFIG") @pytest.fixture(scope="module", autouse=True) @@ -23,42 +24,25 @@ def started_cluster(): try: cluster.start() - instance.query(''' - CREATE DATABASE mydb ENGINE=Ordinary; + for current_node in nodes: + current_node.query(''' + CREATE DATABASE mydb ENGINE=Ordinary; - CREATE TABLE mydb.filtered_table1 (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a; - INSERT INTO mydb.filtered_table1 values (0, 0), (0, 1), (1, 0), (1, 1); + CREATE TABLE mydb.filtered_table1 (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a; + INSERT INTO mydb.filtered_table1 values (0, 0), (0, 1), (1, 0), (1, 1); - CREATE TABLE mydb.table (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a; - INSERT INTO mydb.table values (0, 0), (0, 1), (1, 0), (1, 1); + CREATE TABLE mydb.table (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a; + INSERT INTO mydb.table values (0, 0), (0, 1), (1, 0), (1, 1); - CREATE TABLE mydb.filtered_table2 (a UInt8, b UInt8, c UInt8, d UInt8) ENGINE MergeTree ORDER BY a; - INSERT INTO mydb.filtered_table2 values (0, 0, 0, 0), (1, 2, 3, 4), (4, 3, 2, 1), (0, 0, 6, 0); + CREATE TABLE mydb.filtered_table2 (a UInt8, b UInt8, c UInt8, d UInt8) ENGINE MergeTree ORDER BY a; + INSERT INTO mydb.filtered_table2 values (0, 0, 0, 0), (1, 2, 3, 4), (4, 3, 2, 1), (0, 0, 6, 0); - CREATE TABLE mydb.filtered_table3 (a UInt8, b UInt8, c UInt16 ALIAS a + b) ENGINE MergeTree ORDER BY a; - INSERT INTO mydb.filtered_table3 values (0, 0), (0, 1), (1, 0), (1, 1); + CREATE TABLE mydb.filtered_table3 (a UInt8, b UInt8, c UInt16 ALIAS a + b) ENGINE MergeTree ORDER BY a; + INSERT INTO mydb.filtered_table3 values (0, 0), (0, 1), (1, 0), (1, 1); - CREATE TABLE mydb.`.filtered_table4` (a UInt8, b UInt8, c UInt16 ALIAS a + b) ENGINE MergeTree ORDER BY a; - INSERT INTO mydb.`.filtered_table4` values (0, 0), (0, 1), (1, 0), (1, 1); - ''') - instance2.query(''' - CREATE DATABASE mydb ENGINE=Ordinary; - - CREATE TABLE mydb.filtered_table1 (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a; - INSERT INTO mydb.filtered_table1 values (0, 0), (0, 1), (1, 0), (1, 1); - - CREATE TABLE mydb.table (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a; - INSERT INTO mydb.table values (0, 0), (0, 1), (1, 0), (1, 1); - - CREATE TABLE mydb.filtered_table2 (a UInt8, b UInt8, c UInt8, d UInt8) ENGINE MergeTree ORDER BY a; - INSERT INTO mydb.filtered_table2 values (0, 0, 0, 0), (1, 2, 3, 4), (4, 3, 2, 1), (0, 0, 6, 0); - - CREATE TABLE mydb.filtered_table3 (a UInt8, b UInt8, c UInt16 ALIAS a + b) ENGINE MergeTree ORDER BY a; - INSERT INTO mydb.filtered_table3 values (0, 0), (0, 1), (1, 0), (1, 1); - - CREATE TABLE mydb.`.filtered_table4` (a UInt8, b UInt8, c UInt16 ALIAS a + b) ENGINE MergeTree ORDER BY a; - INSERT INTO mydb.`.filtered_table4` values (0, 0), (0, 1), (1, 0), (1, 1); - ''') + CREATE TABLE mydb.`.filtered_table4` (a UInt8, b UInt8, c UInt16 ALIAS a + b) ENGINE MergeTree ORDER BY a; + INSERT INTO mydb.`.filtered_table4` values (0, 0), (0, 1), (1, 0), (1, 1); + ''') yield cluster @@ -72,243 +56,239 @@ def reset_policies(): yield finally: copy_policy_xml('normal_filters.xml') - instance.query("DROP POLICY IF EXISTS pA, pB ON mydb.filtered_table1") + for current_node in nodes: + current_node.query("DROP POLICY IF EXISTS pA, pB ON mydb.filtered_table1") def test_smoke(): - assert instance.query("SELECT * FROM mydb.filtered_table1") == "1\t0\n1\t1\n" - assert instance.query("SELECT * FROM mydb.filtered_table2") == "0\t0\t0\t0\n0\t0\t6\t0\n" - assert instance.query("SELECT * FROM mydb.filtered_table3") == "0\t1\n1\t0\n" + assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[1,0], [1, 1]]) + assert node.query("SELECT * FROM mydb.filtered_table2") == TSV([[0, 0, 0, 0], [0, 0, 6, 0]]) + assert node.query("SELECT * FROM mydb.filtered_table3") == TSV([[0, 1], [1, 0]]) - assert instance.query("SELECT a FROM mydb.filtered_table1") == "1\n1\n" - assert instance.query("SELECT b FROM mydb.filtered_table1") == "0\n1\n" - assert instance.query("SELECT a FROM mydb.filtered_table1 WHERE a = 1") == "1\n1\n" - assert instance.query("SELECT a FROM mydb.filtered_table1 WHERE a IN (1)") == "1\n1\n" - assert instance.query("SELECT a = 1 FROM mydb.filtered_table1") == "1\n1\n" + assert node.query("SELECT a FROM mydb.filtered_table1") == TSV([[1], [1]]) + assert node.query("SELECT b FROM mydb.filtered_table1") == TSV([[0], [1]]) + assert node.query("SELECT a FROM mydb.filtered_table1 WHERE a = 1") == TSV([[1], [1]]) + assert node.query("SELECT a FROM mydb.filtered_table1 WHERE a IN (1)") == TSV([[1], [1]]) + assert node.query("SELECT a = 1 FROM mydb.filtered_table1") == TSV([[1], [1]]) - assert instance.query("SELECT a FROM mydb.filtered_table3") == "0\n1\n" - assert instance.query("SELECT b FROM mydb.filtered_table3") == "1\n0\n" - assert instance.query("SELECT c FROM mydb.filtered_table3") == "1\n1\n" - assert instance.query("SELECT a + b FROM mydb.filtered_table3") == "1\n1\n" - assert instance.query("SELECT a FROM mydb.filtered_table3 WHERE c = 1") == "0\n1\n" - assert instance.query("SELECT c = 1 FROM mydb.filtered_table3") == "1\n1\n" - assert instance.query("SELECT a + b = 1 FROM mydb.filtered_table3") == "1\n1\n" + assert node.query("SELECT a FROM mydb.filtered_table3") == TSV([[0], [1]]) + assert node.query("SELECT b FROM mydb.filtered_table3") == TSV([[1], [0]]) + assert node.query("SELECT c FROM mydb.filtered_table3") == TSV([[1], [1]]) + assert node.query("SELECT a + b FROM mydb.filtered_table3") == TSV([[1], [1]]) + assert node.query("SELECT a FROM mydb.filtered_table3 WHERE c = 1") == TSV([[0], [1]]) + assert node.query("SELECT c = 1 FROM mydb.filtered_table3") == TSV([[1], [1]]) + assert node.query("SELECT a + b = 1 FROM mydb.filtered_table3") == TSV([[1], [1]]) def test_join(): - assert instance.query("SELECT * FROM mydb.filtered_table1 as t1 ANY LEFT JOIN mydb.filtered_table1 as t2 ON t1.a = t2.b") == "1\t0\t1\t1\n1\t1\t1\t1\n" - assert instance.query("SELECT * FROM mydb.filtered_table1 as t2 ANY RIGHT JOIN mydb.filtered_table1 as t1 ON t2.b = t1.a") == "1\t1\t1\t0\n" + assert node.query("SELECT * FROM mydb.filtered_table1 as t1 ANY LEFT JOIN mydb.filtered_table1 as t2 ON t1.a = t2.b") == TSV([[1, 0, 1, 1], [1, 1, 1, 1]]) + assert node.query("SELECT * FROM mydb.filtered_table1 as t2 ANY RIGHT JOIN mydb.filtered_table1 as t1 ON t2.b = t1.a") == TSV([[1, 1, 1, 0]]) def test_cannot_trick_row_policy_with_keyword_with(): - assert instance.query("WITH 0 AS a SELECT * FROM mydb.filtered_table1") == "1\t0\n1\t1\n" - assert instance.query("WITH 0 AS a SELECT a, b FROM mydb.filtered_table1") == "0\t0\n0\t1\n" - assert instance.query("WITH 0 AS a SELECT a FROM mydb.filtered_table1") == "0\n0\n" - assert instance.query("WITH 0 AS a SELECT b FROM mydb.filtered_table1") == "0\n1\n" + assert node.query("WITH 0 AS a SELECT * FROM mydb.filtered_table1") == TSV([[1, 0], [1, 1]]) + assert node.query("WITH 0 AS a SELECT a, b FROM mydb.filtered_table1") == TSV([[0, 0], [0, 1]]) + assert node.query("WITH 0 AS a SELECT a FROM mydb.filtered_table1") == TSV([[0], [0]]) + assert node.query("WITH 0 AS a SELECT b FROM mydb.filtered_table1") == TSV([[0], [1]]) def test_prewhere_not_supported(): expected_error = "PREWHERE is not supported if the table is filtered by row-level security" - assert expected_error in instance.query_and_get_error("SELECT * FROM mydb.filtered_table1 PREWHERE 1") - assert expected_error in instance.query_and_get_error("SELECT * FROM mydb.filtered_table2 PREWHERE 1") - assert expected_error in instance.query_and_get_error("SELECT * FROM mydb.filtered_table3 PREWHERE 1") + assert expected_error in node.query_and_get_error("SELECT * FROM mydb.filtered_table1 PREWHERE 1") + assert expected_error in node.query_and_get_error("SELECT * FROM mydb.filtered_table2 PREWHERE 1") + assert expected_error in node.query_and_get_error("SELECT * FROM mydb.filtered_table3 PREWHERE 1") # However PREWHERE should still work for user without filtering. - assert instance.query("SELECT * FROM mydb.filtered_table1 PREWHERE 1", user="another") == "0\t0\n0\t1\n1\t0\n1\t1\n" + assert node.query("SELECT * FROM mydb.filtered_table1 PREWHERE 1", user="another") == TSV([[0, 0], [0, 1], [1, 0], [1, 1]]) def test_single_table_name(): copy_policy_xml('tag_with_table_name.xml') - assert instance.query("SELECT * FROM mydb.table") == "1\t0\n1\t1\n" - assert instance.query("SELECT * FROM mydb.filtered_table2") == "0\t0\t0\t0\n0\t0\t6\t0\n" - assert instance.query("SELECT * FROM mydb.filtered_table3") == "0\t1\n1\t0\n" + assert node.query("SELECT * FROM mydb.table") == TSV([[1, 0], [1, 1]]) + assert node.query("SELECT * FROM mydb.filtered_table2") == TSV([[0, 0, 0, 0], [0, 0, 6, 0]]) + assert node.query("SELECT * FROM mydb.filtered_table3") == TSV([[0, 1], [1, 0]]) - assert instance.query("SELECT a FROM mydb.table") == "1\n1\n" - assert instance.query("SELECT b FROM mydb.table") == "0\n1\n" - assert instance.query("SELECT a FROM mydb.table WHERE a = 1") == "1\n1\n" - assert instance.query("SELECT a = 1 FROM mydb.table") == "1\n1\n" + assert node.query("SELECT a FROM mydb.table") == TSV([[1], [1]]) + assert node.query("SELECT b FROM mydb.table") == TSV([[0], [1]]) + assert node.query("SELECT a FROM mydb.table WHERE a = 1") == TSV([[1], [1]]) + assert node.query("SELECT a = 1 FROM mydb.table") == TSV([[1], [1]]) - assert instance.query("SELECT a FROM mydb.filtered_table3") == "0\n1\n" - assert instance.query("SELECT b FROM mydb.filtered_table3") == "1\n0\n" - assert instance.query("SELECT c FROM mydb.filtered_table3") == "1\n1\n" - assert instance.query("SELECT a + b FROM mydb.filtered_table3") == "1\n1\n" - assert instance.query("SELECT a FROM mydb.filtered_table3 WHERE c = 1") == "0\n1\n" - assert instance.query("SELECT c = 1 FROM mydb.filtered_table3") == "1\n1\n" - assert instance.query("SELECT a + b = 1 FROM mydb.filtered_table3") == "1\n1\n" + assert node.query("SELECT a FROM mydb.filtered_table3") == TSV([[0], [1]]) + assert node.query("SELECT b FROM mydb.filtered_table3") == TSV([[1], [0]]) + assert node.query("SELECT c FROM mydb.filtered_table3") == TSV([[1], [1]]) + assert node.query("SELECT a + b FROM mydb.filtered_table3") == TSV([[1], [1]]) + assert node.query("SELECT a FROM mydb.filtered_table3 WHERE c = 1") == TSV([[0], [1]]) + assert node.query("SELECT c = 1 FROM mydb.filtered_table3") == TSV([[1], [1]]) + assert node.query("SELECT a + b = 1 FROM mydb.filtered_table3") == TSV([[1], [1]]) def test_custom_table_name(): copy_policy_xml('multiple_tags_with_table_names.xml') - assert instance.query("SELECT * FROM mydb.table") == "1\t0\n1\t1\n" - assert instance.query("SELECT * FROM mydb.filtered_table2") == "0\t0\t0\t0\n0\t0\t6\t0\n" - assert instance.query("SELECT * FROM mydb.`.filtered_table4`") == "0\t1\n1\t0\n" + assert node.query("SELECT * FROM mydb.table") == TSV([[1, 0], [1, 1]]) + assert node.query("SELECT * FROM mydb.filtered_table2") == TSV([[0, 0, 0, 0], [0, 0, 6, 0]]) + assert node.query("SELECT * FROM mydb.`.filtered_table4`") == TSV([[0, 1], [1, 0]]) - assert instance.query("SELECT a FROM mydb.table") == "1\n1\n" - assert instance.query("SELECT b FROM mydb.table") == "0\n1\n" - assert instance.query("SELECT a FROM mydb.table WHERE a = 1") == "1\n1\n" - assert instance.query("SELECT a = 1 FROM mydb.table") == "1\n1\n" + assert node.query("SELECT a FROM mydb.table") == TSV([[1], [1]]) + assert node.query("SELECT b FROM mydb.table") == TSV([[0], [1]]) + assert node.query("SELECT a FROM mydb.table WHERE a = 1") == TSV([[1], [1]]) + assert node.query("SELECT a = 1 FROM mydb.table") == TSV([[1], [1]]) - assert instance.query("SELECT a FROM mydb.`.filtered_table4`") == "0\n1\n" - assert instance.query("SELECT b FROM mydb.`.filtered_table4`") == "1\n0\n" - assert instance.query("SELECT c FROM mydb.`.filtered_table4`") == "1\n1\n" - assert instance.query("SELECT a + b FROM mydb.`.filtered_table4`") == "1\n1\n" - assert instance.query("SELECT a FROM mydb.`.filtered_table4` WHERE c = 1") == "0\n1\n" - assert instance.query("SELECT c = 1 FROM mydb.`.filtered_table4`") == "1\n1\n" - assert instance.query("SELECT a + b = 1 FROM mydb.`.filtered_table4`") == "1\n1\n" + assert node.query("SELECT a FROM mydb.`.filtered_table4`") == TSV([[0], [1]]) + assert node.query("SELECT b FROM mydb.`.filtered_table4`") == TSV([[1], [0]]) + assert node.query("SELECT c FROM mydb.`.filtered_table4`") == TSV([[1], [1]]) + assert node.query("SELECT a + b FROM mydb.`.filtered_table4`") == TSV([[1], [1]]) + assert node.query("SELECT a FROM mydb.`.filtered_table4` WHERE c = 1") == TSV([[0], [1]]) + assert node.query("SELECT c = 1 FROM mydb.`.filtered_table4`") == TSV([[1], [1]]) + assert node.query("SELECT a + b = 1 FROM mydb.`.filtered_table4`") == TSV([[1], [1]]) def test_change_of_users_xml_changes_row_policies(): copy_policy_xml('normal_filters.xml') - assert instance.query("SELECT * FROM mydb.filtered_table1") == "1\t0\n1\t1\n" - assert instance.query("SELECT * FROM mydb.filtered_table2") == "0\t0\t0\t0\n0\t0\t6\t0\n" - assert instance.query("SELECT * FROM mydb.filtered_table3") == "0\t1\n1\t0\n" + assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[1, 0], [1, 1]]) + assert node.query("SELECT * FROM mydb.filtered_table2") == TSV([[0, 0, 0, 0], [0, 0, 6, 0]]) + assert node.query("SELECT * FROM mydb.filtered_table3") == TSV([[0, 1], [1, 0]]) copy_policy_xml('all_rows.xml') - assert instance.query("SELECT * FROM mydb.filtered_table1") == "0\t0\n0\t1\n1\t0\n1\t1\n" - assert instance.query("SELECT * FROM mydb.filtered_table2") == "0\t0\t0\t0\n0\t0\t6\t0\n1\t2\t3\t4\n4\t3\t2\t1\n" - assert instance.query("SELECT * FROM mydb.filtered_table3") == "0\t0\n0\t1\n1\t0\n1\t1\n" + assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[0, 0], [0, 1], [1, 0], [1, 1]]) + assert node.query("SELECT * FROM mydb.filtered_table2") == TSV([[0, 0, 0, 0], [0, 0, 6, 0], [1, 2, 3, 4], [4, 3, 2, 1]]) + assert node.query("SELECT * FROM mydb.filtered_table3") == TSV([[0, 0], [0, 1], [1, 0], [1, 1]]) copy_policy_xml('no_rows.xml') - assert instance.query("SELECT * FROM mydb.filtered_table1") == "" - assert instance.query("SELECT * FROM mydb.filtered_table2") == "" - assert instance.query("SELECT * FROM mydb.filtered_table3") == "" + assert node.query("SELECT * FROM mydb.filtered_table1") == "" + assert node.query("SELECT * FROM mydb.filtered_table2") == "" + assert node.query("SELECT * FROM mydb.filtered_table3") == "" copy_policy_xml('normal_filters.xml') - assert instance.query("SELECT * FROM mydb.filtered_table1") == "1\t0\n1\t1\n" - assert instance.query("SELECT * FROM mydb.filtered_table2") == "0\t0\t0\t0\n0\t0\t6\t0\n" - assert instance.query("SELECT * FROM mydb.filtered_table3") == "0\t1\n1\t0\n" + assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[1, 0], [1, 1]]) + assert node.query("SELECT * FROM mydb.filtered_table2") == TSV([[0, 0, 0, 0], [0, 0, 6, 0]]) + assert node.query("SELECT * FROM mydb.filtered_table3") == TSV([[0, 1], [1, 0]]) copy_policy_xml('no_filters.xml') - assert instance.query("SELECT * FROM mydb.filtered_table1") == "0\t0\n0\t1\n1\t0\n1\t1\n" - assert instance.query("SELECT * FROM mydb.filtered_table2") == "0\t0\t0\t0\n0\t0\t6\t0\n1\t2\t3\t4\n4\t3\t2\t1\n" - assert instance.query("SELECT * FROM mydb.filtered_table3") == "0\t0\n0\t1\n1\t0\n1\t1\n" + assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[0, 0], [0, 1], [1, 0], [1, 1]]) + assert node.query("SELECT * FROM mydb.filtered_table2") == TSV([[0, 0, 0, 0], [0, 0, 6, 0], [1, 2, 3, 4], [4, 3, 2, 1]]) + assert node.query("SELECT * FROM mydb.filtered_table3") == TSV([[0, 0], [0, 1], [1, 0], [1, 1]]) copy_policy_xml('normal_filters.xml') - assert instance.query("SELECT * FROM mydb.filtered_table1") == "1\t0\n1\t1\n" - assert instance.query("SELECT * FROM mydb.filtered_table2") == "0\t0\t0\t0\n0\t0\t6\t0\n" - assert instance.query("SELECT * FROM mydb.filtered_table3") == "0\t1\n1\t0\n" + assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[1, 0], [1, 1]]) + assert node.query("SELECT * FROM mydb.filtered_table2") == TSV([[0, 0, 0, 0], [0, 0, 6, 0]]) + assert node.query("SELECT * FROM mydb.filtered_table3") == TSV([[0, 1], [1, 0]]) def test_reload_users_xml_by_timer(): copy_policy_xml('normal_filters.xml') - assert instance.query("SELECT * FROM mydb.filtered_table1") == "1\t0\n1\t1\n" - assert instance.query("SELECT * FROM mydb.filtered_table2") == "0\t0\t0\t0\n0\t0\t6\t0\n" - assert instance.query("SELECT * FROM mydb.filtered_table3") == "0\t1\n1\t0\n" + assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[1, 0], [1, 1]]) + assert node.query("SELECT * FROM mydb.filtered_table2") == TSV([[0, 0, 0, 0], [0, 0, 6, 0]]) + assert node.query("SELECT * FROM mydb.filtered_table3") == TSV([[0, 1], [1, 0]]) time.sleep(1) # The modification time of the 'row_policy.xml' file should be different. copy_policy_xml('all_rows.xml', False) - assert_eq_with_retry(instance, "SELECT * FROM mydb.filtered_table1", "0\t0\n0\t1\n1\t0\n1\t1") - assert_eq_with_retry(instance, "SELECT * FROM mydb.filtered_table2", "0\t0\t0\t0\n0\t0\t6\t0\n1\t2\t3\t4\n4\t3\t2\t1") - assert_eq_with_retry(instance, "SELECT * FROM mydb.filtered_table3", "0\t0\n0\t1\n1\t0\n1\t1") + assert_eq_with_retry(node, "SELECT * FROM mydb.filtered_table1", [[0, 0], [0, 1], [1, 0], [1, 1]]) + assert_eq_with_retry(node, "SELECT * FROM mydb.filtered_table2", [[0, 0, 0, 0], [0, 0, 6, 0], [1, 2, 3, 4], [4, 3, 2, 1]]) + assert_eq_with_retry(node, "SELECT * FROM mydb.filtered_table3", [[0, 0], [0, 1], [1, 0], [1, 1]]) time.sleep(1) # The modification time of the 'row_policy.xml' file should be different. copy_policy_xml('normal_filters.xml', False) - assert_eq_with_retry(instance, "SELECT * FROM mydb.filtered_table1", "1\t0\n1\t1") - assert_eq_with_retry(instance, "SELECT * FROM mydb.filtered_table2", "0\t0\t0\t0\n0\t0\t6\t0") - assert_eq_with_retry(instance, "SELECT * FROM mydb.filtered_table3", "0\t1\n1\t0") + assert_eq_with_retry(node, "SELECT * FROM mydb.filtered_table1", [[1, 0], [1, 1]]) + assert_eq_with_retry(node, "SELECT * FROM mydb.filtered_table2", [[0, 0, 0, 0], [0, 0, 6, 0]]) + assert_eq_with_retry(node, "SELECT * FROM mydb.filtered_table3", [[0, 1], [1, 0]]) def test_introspection(): - assert instance.query("SELECT currentRowPolicies('mydb', 'filtered_table1')") == "['default']\n" - assert instance.query("SELECT currentRowPolicies('mydb', 'filtered_table2')") == "['default']\n" - assert instance.query("SELECT currentRowPolicies('mydb', 'filtered_table3')") == "['default']\n" - assert instance.query("SELECT arraySort(currentRowPolicies())") == "[('mydb','filtered_table1','default'),('mydb','filtered_table2','default'),('mydb','filtered_table3','default'),('mydb','local','default')]\n" - - policy1 = "mydb\tfiltered_table1\tdefault\tdefault ON mydb.filtered_table1\t9e8a8f62-4965-2b5e-8599-57c7b99b3549\tusers.xml\t0\ta = 1\t\t\t\t\n" - policy2 = "mydb\tfiltered_table2\tdefault\tdefault ON mydb.filtered_table2\tcffae79d-b9bf-a2ef-b798-019c18470b25\tusers.xml\t0\ta + b < 1 or c - d > 5\t\t\t\t\n" - policy3 = "mydb\tfiltered_table3\tdefault\tdefault ON mydb.filtered_table3\t12fc5cef-e3da-3940-ec79-d8be3911f42b\tusers.xml\t0\tc = 1\t\t\t\t\n" - policy4 = "mydb\tlocal\tdefault\tdefault ON mydb.local\tcdacaeb5-1d97-f99d-2bb0-4574f290629c\tusers.xml\t0\t1\t\t\t\t\n" - assert instance.query("SELECT * from system.row_policies WHERE has(currentRowPolicyIDs('mydb', 'filtered_table1'), id) ORDER BY table, name") == policy1 - assert instance.query("SELECT * from system.row_policies WHERE has(currentRowPolicyIDs('mydb', 'filtered_table2'), id) ORDER BY table, name") == policy2 - assert instance.query("SELECT * from system.row_policies WHERE has(currentRowPolicyIDs('mydb', 'filtered_table3'), id) ORDER BY table, name") == policy3 - assert instance.query("SELECT * from system.row_policies WHERE has(currentRowPolicyIDs('mydb', 'local'), id) ORDER BY table, name") == policy4 - assert instance.query("SELECT * from system.row_policies WHERE has(currentRowPolicyIDs(), id) ORDER BY table, name") == policy1 + policy2 + policy3 + policy4 + policies = [ + ["another ON mydb.filtered_table1", "another", "mydb", "filtered_table1", "6068883a-0e9d-f802-7e22-0144f8e66d3c", "users.xml", "1", 0, 0, "['another']", "[]"], + ["another ON mydb.filtered_table2", "another", "mydb", "filtered_table2", "c019e957-c60b-d54e-cc52-7c90dac5fb01", "users.xml", "1", 0, 0, "['another']", "[]"], + ["another ON mydb.filtered_table3", "another", "mydb", "filtered_table3", "4cb080d0-44e8-dbef-6026-346655143628", "users.xml", "1", 0, 0, "['another']", "[]"], + ["another ON mydb.local", "another", "mydb", "local", "5b23c389-7e18-06bf-a6bc-dd1afbbc0a97", "users.xml", "a = 1", 0, 0, "['another']", "[]"], + ["default ON mydb.filtered_table1", "default", "mydb", "filtered_table1", "9e8a8f62-4965-2b5e-8599-57c7b99b3549", "users.xml", "a = 1", 0, 0, "['default']", "[]"], + ["default ON mydb.filtered_table2", "default", "mydb", "filtered_table2", "cffae79d-b9bf-a2ef-b798-019c18470b25", "users.xml", "a + b < 1 or c - d > 5", 0, 0, "['default']", "[]"], + ["default ON mydb.filtered_table3", "default", "mydb", "filtered_table3", "12fc5cef-e3da-3940-ec79-d8be3911f42b", "users.xml", "c = 1", 0, 0, "['default']", "[]"], + ["default ON mydb.local", "default", "mydb", "local", "cdacaeb5-1d97-f99d-2bb0-4574f290629c", "users.xml", "1", 0, 0, "['default']", "[]"] + ] + assert node.query("SELECT * from system.row_policies ORDER BY short_name, database, table") == TSV(policies) def test_dcl_introspection(): - assert instance.query("SHOW POLICIES ON mydb.filtered_table1") == "another\ndefault\n" - assert instance.query("SHOW POLICIES CURRENT ON mydb.filtered_table2") == "default\n" - assert instance.query("SHOW POLICIES") == "another ON mydb.filtered_table1\nanother ON mydb.filtered_table2\nanother ON mydb.filtered_table3\nanother ON mydb.local\ndefault ON mydb.filtered_table1\ndefault ON mydb.filtered_table2\ndefault ON mydb.filtered_table3\ndefault ON mydb.local\n" - assert instance.query("SHOW POLICIES CURRENT") == "default ON mydb.filtered_table1\ndefault ON mydb.filtered_table2\ndefault ON mydb.filtered_table3\ndefault ON mydb.local\n" + assert node.query("SHOW POLICIES") == TSV(["another ON mydb.filtered_table1", "another ON mydb.filtered_table2", "another ON mydb.filtered_table3", "another ON mydb.local", "default ON mydb.filtered_table1", "default ON mydb.filtered_table2", "default ON mydb.filtered_table3", "default ON mydb.local"]) + assert node.query("SHOW POLICIES ON mydb.filtered_table1") == TSV(["another", "default"]) + assert node.query("SHOW POLICIES ON mydb.local") == TSV(["another", "default"]) - assert instance.query("SHOW CREATE POLICY default ON mydb.filtered_table1") == "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING a = 1 TO default\n" - assert instance.query("SHOW CREATE POLICY default ON mydb.filtered_table2") == "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING ((a + b) < 1) OR ((c - d) > 5) TO default\n" - assert instance.query("SHOW CREATE POLICY default ON mydb.filtered_table3") == "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING c = 1 TO default\n" - assert instance.query("SHOW CREATE POLICY default ON mydb.local") == "CREATE ROW POLICY default ON mydb.local FOR SELECT USING 1 TO default\n" + assert node.query("SHOW CREATE POLICY default ON mydb.filtered_table1") == "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING a = 1 TO default\n" + assert node.query("SHOW CREATE POLICY default ON mydb.filtered_table2") == "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING ((a + b) < 1) OR ((c - d) > 5) TO default\n" + assert node.query("SHOW CREATE POLICY default ON mydb.filtered_table3") == "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING c = 1 TO default\n" + assert node.query("SHOW CREATE POLICY default ON mydb.local") == "CREATE ROW POLICY default ON mydb.local FOR SELECT USING 1 TO default\n" copy_policy_xml('all_rows.xml') - assert instance.query("SHOW POLICIES CURRENT") == "default ON mydb.filtered_table1\ndefault ON mydb.filtered_table2\ndefault ON mydb.filtered_table3\n" - assert instance.query("SHOW CREATE POLICY default ON mydb.filtered_table1") == "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING 1 TO default\n" - assert instance.query("SHOW CREATE POLICY default ON mydb.filtered_table2") == "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING 1 TO default\n" - assert instance.query("SHOW CREATE POLICY default ON mydb.filtered_table3") == "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING 1 TO default\n" + assert node.query("SHOW POLICIES") == TSV(["another ON mydb.filtered_table1", "another ON mydb.filtered_table2", "another ON mydb.filtered_table3", "default ON mydb.filtered_table1", "default ON mydb.filtered_table2", "default ON mydb.filtered_table3"]) + assert node.query("SHOW CREATE POLICY default ON mydb.filtered_table1") == "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING 1 TO default\n" + assert node.query("SHOW CREATE POLICY default ON mydb.filtered_table2") == "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING 1 TO default\n" + assert node.query("SHOW CREATE POLICY default ON mydb.filtered_table3") == "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING 1 TO default\n" copy_policy_xml('no_rows.xml') - assert instance.query("SHOW POLICIES CURRENT") == "default ON mydb.filtered_table1\ndefault ON mydb.filtered_table2\ndefault ON mydb.filtered_table3\n" - assert instance.query("SHOW CREATE POLICY default ON mydb.filtered_table1") == "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING NULL TO default\n" - assert instance.query("SHOW CREATE POLICY default ON mydb.filtered_table2") == "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING NULL TO default\n" - assert instance.query("SHOW CREATE POLICY default ON mydb.filtered_table3") == "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING NULL TO default\n" + assert node.query("SHOW POLICIES") == TSV(["another ON mydb.filtered_table1", "another ON mydb.filtered_table2", "another ON mydb.filtered_table3", "default ON mydb.filtered_table1", "default ON mydb.filtered_table2", "default ON mydb.filtered_table3"]) + assert node.query("SHOW CREATE POLICY default ON mydb.filtered_table1") == "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING NULL TO default\n" + assert node.query("SHOW CREATE POLICY default ON mydb.filtered_table2") == "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING NULL TO default\n" + assert node.query("SHOW CREATE POLICY default ON mydb.filtered_table3") == "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING NULL TO default\n" copy_policy_xml('no_filters.xml') - assert instance.query("SHOW POLICIES") == "" + assert node.query("SHOW POLICIES") == "" def test_dcl_management(): copy_policy_xml('no_filters.xml') - assert instance.query("SHOW POLICIES") == "" + assert node.query("SHOW POLICIES") == "" - instance.query("CREATE POLICY pA ON mydb.filtered_table1 FOR SELECT USING ab") - assert instance.query("SELECT * FROM mydb.filtered_table1") == "1\t0\n" + node.query("ALTER POLICY pA ON mydb.filtered_table1 FOR SELECT USING a>b") + assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[1, 0]]) - instance.query("ALTER POLICY pA ON mydb.filtered_table1 RENAME TO pB") - assert instance.query("SELECT * FROM mydb.filtered_table1") == "1\t0\n" - assert instance.query("SHOW POLICIES CURRENT ON mydb.filtered_table1") == "pB\n" - assert instance.query("SHOW CREATE POLICY pB ON mydb.filtered_table1") == "CREATE ROW POLICY pB ON mydb.filtered_table1 FOR SELECT USING a > b TO default\n" + node.query("ALTER POLICY pA ON mydb.filtered_table1 RENAME TO pB") + assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[1, 0]]) + assert node.query("SHOW POLICIES ON mydb.filtered_table1") == "pB\n" + assert node.query("SHOW CREATE POLICY pB ON mydb.filtered_table1") == "CREATE ROW POLICY pB ON mydb.filtered_table1 FOR SELECT USING a > b TO default\n" - instance.query("DROP POLICY pB ON mydb.filtered_table1") - assert instance.query("SELECT * FROM mydb.filtered_table1") == "0\t0\n0\t1\n1\t0\n1\t1\n" - assert instance.query("SHOW POLICIES") == "" + node.query("DROP POLICY pB ON mydb.filtered_table1") + assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[0, 0], [0, 1], [1, 0], [1, 1]]) + assert node.query("SHOW POLICIES") == "" def test_users_xml_is_readonly(): - assert re.search("storage is readonly", instance.query_and_get_error("DROP POLICY default ON mydb.filtered_table1")) + assert re.search("storage is readonly", node.query_and_get_error("DROP POLICY default ON mydb.filtered_table1")) def test_miscellaneous_engines(): copy_policy_xml('normal_filters.xml') # ReplicatedMergeTree - instance.query("DROP TABLE mydb.filtered_table1") - instance.query("CREATE TABLE mydb.filtered_table1 (a UInt8, b UInt8) ENGINE ReplicatedMergeTree('/clickhouse/tables/00-00/filtered_table1', 'replica1') ORDER BY a") - instance.query("INSERT INTO mydb.filtered_table1 values (0, 0), (0, 1), (1, 0), (1, 1)") - assert instance.query("SELECT * FROM mydb.filtered_table1") == "1\t0\n1\t1\n" + node.query("DROP TABLE mydb.filtered_table1") + node.query("CREATE TABLE mydb.filtered_table1 (a UInt8, b UInt8) ENGINE ReplicatedMergeTree('/clickhouse/tables/00-00/filtered_table1', 'replica1') ORDER BY a") + node.query("INSERT INTO mydb.filtered_table1 values (0, 0), (0, 1), (1, 0), (1, 1)") + assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[1, 0], [1, 1]]) # CollapsingMergeTree - instance.query("DROP TABLE mydb.filtered_table1") - instance.query("CREATE TABLE mydb.filtered_table1 (a UInt8, b Int8) ENGINE CollapsingMergeTree(b) ORDER BY a") - instance.query("INSERT INTO mydb.filtered_table1 values (0, 1), (0, 1), (1, 1), (1, 1)") - assert instance.query("SELECT * FROM mydb.filtered_table1") == "1\t1\n1\t1\n" + node.query("DROP TABLE mydb.filtered_table1") + node.query("CREATE TABLE mydb.filtered_table1 (a UInt8, b Int8) ENGINE CollapsingMergeTree(b) ORDER BY a") + node.query("INSERT INTO mydb.filtered_table1 values (0, 1), (0, 1), (1, 1), (1, 1)") + assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[1, 1], [1, 1]]) # ReplicatedCollapsingMergeTree - instance.query("DROP TABLE mydb.filtered_table1") - instance.query("CREATE TABLE mydb.filtered_table1 (a UInt8, b Int8) ENGINE ReplicatedCollapsingMergeTree('/clickhouse/tables/00-00/filtered_table1', 'replica1', b) ORDER BY a") - instance.query("INSERT INTO mydb.filtered_table1 values (0, 1), (0, 1), (1, 1), (1, 1)") - assert instance.query("SELECT * FROM mydb.filtered_table1") == "1\t1\n1\t1\n" + node.query("DROP TABLE mydb.filtered_table1") + node.query("CREATE TABLE mydb.filtered_table1 (a UInt8, b Int8) ENGINE ReplicatedCollapsingMergeTree('/clickhouse/tables/00-00/filtered_table1', 'replica1', b) ORDER BY a") + node.query("INSERT INTO mydb.filtered_table1 values (0, 1), (0, 1), (1, 1), (1, 1)") + assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[1, 1], [1, 1]]) # DistributedMergeTree - instance.query("DROP TABLE IF EXISTS mydb.not_filtered_table") - instance.query("CREATE TABLE mydb.not_filtered_table (a UInt8, b UInt8) ENGINE Distributed('test_local_cluster', mydb, local)") - instance.query("CREATE TABLE mydb.local (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a") - instance2.query("CREATE TABLE mydb.local (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a") - instance.query("INSERT INTO mydb.local values (2, 0), (2, 1), (1, 0), (1, 1)") - instance2.query("INSERT INTO mydb.local values (3, 0), (3, 1), (1, 0), (1, 1)") - assert instance.query("SELECT * FROM mydb.not_filtered_table", user="another") == "1\t0\n1\t1\n1\t0\n1\t1\n" - assert instance.query("SELECT sum(a), b FROM mydb.not_filtered_table GROUP BY b ORDER BY b", user="another") == "2\t0\n2\t1\n" + node.query("DROP TABLE IF EXISTS mydb.not_filtered_table") + node.query("CREATE TABLE mydb.not_filtered_table (a UInt8, b UInt8) ENGINE Distributed('test_local_cluster', mydb, local)") + node.query("CREATE TABLE mydb.local (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a") + node2.query("CREATE TABLE mydb.local (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a") + node.query("INSERT INTO mydb.local values (2, 0), (2, 1), (1, 0), (1, 1)") + node2.query("INSERT INTO mydb.local values (3, 0), (3, 1), (1, 0), (1, 1)") + assert node.query("SELECT * FROM mydb.not_filtered_table", user="another") == TSV([[1, 0], [1, 1], [1, 0], [1, 1]]) + assert node.query("SELECT sum(a), b FROM mydb.not_filtered_table GROUP BY b ORDER BY b", user="another") == TSV([[2, 0], [2, 1]]) From 5b84121d811b31265961520f6e218a5ce014dc44 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Fri, 8 May 2020 15:50:45 +0300 Subject: [PATCH 184/738] Improve system tables for quotas. Remove function currentQuota(). --- src/Access/AccessControlManager.cpp | 6 +- src/Access/AccessControlManager.h | 4 +- src/Access/ContextAccess.cpp | 8 + src/Access/ContextAccess.h | 2 + src/Access/EnabledQuota.cpp | 46 ++-- src/Access/EnabledQuota.h | 14 +- src/Access/Quota.cpp | 29 --- src/Access/Quota.h | 200 +++++++++++----- src/Access/QuotaCache.cpp | 17 +- src/Access/QuotaCache.h | 2 +- src/Access/QuotaUsage.cpp | 16 ++ src/Access/{QuotaUsageInfo.h => QuotaUsage.h} | 6 +- src/Access/QuotaUsageInfo.cpp | 17 -- src/Access/UsersConfigAccessStorage.cpp | 15 +- src/Access/ya.make | 2 +- src/Functions/currentQuota.cpp | 135 ----------- .../registerFunctionsMiscellaneous.cpp | 2 - src/Functions/ya.make | 1 - src/Interpreters/Context.cpp | 16 +- src/Interpreters/Context.h | 2 + .../InterpreterCreateQuotaQuery.cpp | 7 +- src/Interpreters/InterpreterFactory.cpp | 6 - src/Interpreters/InterpreterSelectQuery.cpp | 11 +- .../InterpreterShowAccessEntitiesQuery.cpp | 23 +- .../InterpreterShowAccessEntitiesQuery.h | 5 + ...InterpreterShowCreateAccessEntityQuery.cpp | 31 ++- .../InterpreterShowCreateAccessEntityQuery.h | 8 +- .../InterpreterShowQuotasQuery.cpp | 73 ------ src/Interpreters/InterpreterShowQuotasQuery.h | 28 --- src/Interpreters/ya.make | 1 - src/Parsers/ASTCreateQuotaQuery.cpp | 14 +- src/Parsers/ASTCreateQuotaQuery.h | 10 +- src/Parsers/ASTShowAccessEntitiesQuery.cpp | 4 + src/Parsers/ASTShowAccessEntitiesQuery.h | 3 + .../ASTShowCreateAccessEntityQuery.cpp | 4 +- src/Parsers/ASTShowQuotasQuery.cpp | 35 --- src/Parsers/ASTShowQuotasQuery.h | 24 -- src/Parsers/ParserCreateQuotaQuery.cpp | 27 ++- src/Parsers/ParserQueryWithOutput.cpp | 5 +- src/Parsers/ParserShowAccessEntitiesQuery.cpp | 27 ++- src/Parsers/ParserShowAccessEntitiesQuery.h | 2 + .../ParserShowCreateAccessEntityQuery.cpp | 11 +- .../ParserShowCreateAccessEntityQuery.h | 2 +- src/Parsers/ParserShowQuotasQuery.cpp | 42 ---- src/Parsers/ParserShowQuotasQuery.h | 18 -- src/Parsers/ya.make | 2 - .../System/StorageSystemQuotaLimits.cpp | 122 ++++++++++ .../System/StorageSystemQuotaLimits.h | 24 ++ .../System/StorageSystemQuotaUsage.cpp | 193 ++++++++++++---- src/Storages/System/StorageSystemQuotaUsage.h | 9 +- src/Storages/System/StorageSystemQuotas.cpp | 124 +++++----- src/Storages/System/StorageSystemQuotas.h | 2 - .../System/StorageSystemQuotasUsage.cpp | 22 ++ .../System/StorageSystemQuotasUsage.h | 26 +++ src/Storages/System/attachSystemTables.cpp | 4 + src/Storages/ya.make | 4 +- tests/integration/test_quota/test.py | 213 ++++++++++-------- 57 files changed, 912 insertions(+), 794 deletions(-) create mode 100644 src/Access/QuotaUsage.cpp rename src/Access/{QuotaUsageInfo.h => QuotaUsage.h} (87%) delete mode 100644 src/Access/QuotaUsageInfo.cpp delete mode 100644 src/Functions/currentQuota.cpp delete mode 100644 src/Interpreters/InterpreterShowQuotasQuery.cpp delete mode 100644 src/Interpreters/InterpreterShowQuotasQuery.h delete mode 100644 src/Parsers/ASTShowQuotasQuery.cpp delete mode 100644 src/Parsers/ASTShowQuotasQuery.h delete mode 100644 src/Parsers/ParserShowQuotasQuery.cpp delete mode 100644 src/Parsers/ParserShowQuotasQuery.h create mode 100644 src/Storages/System/StorageSystemQuotaLimits.cpp create mode 100644 src/Storages/System/StorageSystemQuotaLimits.h create mode 100644 src/Storages/System/StorageSystemQuotasUsage.cpp create mode 100644 src/Storages/System/StorageSystemQuotasUsage.h diff --git a/src/Access/AccessControlManager.cpp b/src/Access/AccessControlManager.cpp index e2589969fd8..1c1215a0e28 100644 --- a/src/Access/AccessControlManager.cpp +++ b/src/Access/AccessControlManager.cpp @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include #include #include @@ -144,9 +144,9 @@ std::shared_ptr AccessControlManager::getEnabledQuota( } -std::vector AccessControlManager::getQuotaUsageInfo() const +std::vector AccessControlManager::getAllQuotasUsage() const { - return quota_cache->getUsageInfo(); + return quota_cache->getAllQuotasUsage(); } diff --git a/src/Access/AccessControlManager.h b/src/Access/AccessControlManager.h index 94bc22eb3e5..6bcf8d7c504 100644 --- a/src/Access/AccessControlManager.h +++ b/src/Access/AccessControlManager.h @@ -29,7 +29,7 @@ class EnabledRowPolicies; class RowPolicyCache; class EnabledQuota; class QuotaCache; -struct QuotaUsageInfo; +struct QuotaUsage; struct SettingsProfile; using SettingsProfilePtr = std::shared_ptr; class EnabledSettings; @@ -73,7 +73,7 @@ public: const Poco::Net::IPAddress & address, const String & custom_quota_key) const; - std::vector getQuotaUsageInfo() const; + std::vector getAllQuotasUsage() const; std::shared_ptr getEnabledSettings(const UUID & user_id, const SettingsProfileElements & settings_from_user, diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index ef9791bafa0..f973e93c76b 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -504,6 +505,13 @@ std::shared_ptr ContextAccess::getQuota() const } +std::optional ContextAccess::getQuotaUsage() const +{ + std::lock_guard lock{mutex}; + return enabled_quota ? enabled_quota->getUsage() : std::optional{}; +} + + std::shared_ptr ContextAccess::getFullAccess() { static const std::shared_ptr res = [] diff --git a/src/Access/ContextAccess.h b/src/Access/ContextAccess.h index b10a1fdba69..27bb29a878c 100644 --- a/src/Access/ContextAccess.h +++ b/src/Access/ContextAccess.h @@ -22,6 +22,7 @@ class EnabledRoles; class EnabledRowPolicies; class EnabledQuota; class EnabledSettings; +struct QuotaUsage; struct Settings; class SettingsConstraints; class AccessControlManager; @@ -79,6 +80,7 @@ public: /// Returns the quota to track resource consumption. /// The function returns nullptr if no tracking or limitation is needed. std::shared_ptr getQuota() const; + std::optional getQuotaUsage() const; /// Returns the default settings, i.e. the settings to apply on user's login. /// The function returns nullptr if it's no need to apply settings. diff --git a/src/Access/EnabledQuota.cpp b/src/Access/EnabledQuota.cpp index 6b3ec33ce6b..e9d586a692f 100644 --- a/src/Access/EnabledQuota.cpp +++ b/src/Access/EnabledQuota.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include #include @@ -30,9 +30,10 @@ struct EnabledQuota::Impl if (resource_type == Quota::EXECUTION_TIME) amount_to_string = [&](UInt64 amount) { return ext::to_string(std::chrono::nanoseconds(amount)); }; + const auto & type_info = Quota::ResourceTypeInfo::get(resource_type); throw Exception( "Quota for user " + backQuote(user_name) + " for " + ext::to_string(duration) + " has been exceeded: " - + Quota::getNameOfResourceType(resource_type) + " = " + amount_to_string(used) + "/" + amount_to_string(max) + ". " + + type_info.outputWithAmount(used) + "/" + type_info.amountToString(max) + ". " + "Interval will end at " + ext::to_string(end_of_interval) + ". " + "Name of quota template: " + backQuote(quota_name), ErrorCodes::QUOTA_EXPIRED); } @@ -83,7 +84,7 @@ struct EnabledQuota::Impl { ResourceAmount used = (interval.used[resource_type] += amount); ResourceAmount max = interval.max[resource_type]; - if (max == Quota::UNLIMITED) + if (!max) continue; if (used > max) { @@ -111,7 +112,7 @@ struct EnabledQuota::Impl { ResourceAmount used = interval.used[resource_type]; ResourceAmount max = interval.max[resource_type]; - if (max == Quota::UNLIMITED) + if (!max) continue; if (used > max) { @@ -134,6 +135,16 @@ struct EnabledQuota::Impl }; +EnabledQuota::Interval::Interval() +{ + for (auto resource_type : ext::range(MAX_RESOURCE_TYPE)) + { + used[resource_type].store(0); + max[resource_type] = 0; + } +} + + EnabledQuota::Interval & EnabledQuota::Interval::operator =(const Interval & src) { if (this == &src) @@ -151,27 +162,30 @@ EnabledQuota::Interval & EnabledQuota::Interval::operator =(const Interval & src } -QuotaUsageInfo EnabledQuota::Intervals::getUsageInfo(std::chrono::system_clock::time_point current_time) const +std::optional EnabledQuota::Intervals::getUsage(std::chrono::system_clock::time_point current_time) const { - QuotaUsageInfo info; - info.quota_id = quota_id; - info.quota_name = quota_name; - info.quota_key = quota_key; - info.intervals.reserve(intervals.size()); + if (!quota_id) + return {}; + QuotaUsage usage; + usage.quota_id = *quota_id; + usage.quota_name = quota_name; + usage.quota_key = quota_key; + usage.intervals.reserve(intervals.size()); for (const auto & in : intervals) { - info.intervals.push_back({}); - auto & out = info.intervals.back(); + usage.intervals.push_back({}); + auto & out = usage.intervals.back(); out.duration = in.duration; out.randomize_interval = in.randomize_interval; out.end_of_interval = Impl::getEndOfInterval(in, current_time); for (auto resource_type : ext::range(MAX_RESOURCE_TYPE)) { - out.max[resource_type] = in.max[resource_type]; + if (in.max[resource_type]) + out.max[resource_type] = in.max[resource_type]; out.used[resource_type] = in.used[resource_type]; } } - return info; + return usage; } @@ -238,10 +252,10 @@ void EnabledQuota::checkExceeded(ResourceType resource_type) const } -QuotaUsageInfo EnabledQuota::getUsageInfo() const +std::optional EnabledQuota::getUsage() const { auto loaded = intervals.load(); - return loaded->getUsageInfo(std::chrono::system_clock::now()); + return loaded->getUsage(std::chrono::system_clock::now()); } diff --git a/src/Access/EnabledQuota.h b/src/Access/EnabledQuota.h index 965d7ea6da9..25e804dd050 100644 --- a/src/Access/EnabledQuota.h +++ b/src/Access/EnabledQuota.h @@ -12,7 +12,7 @@ namespace DB { -struct QuotaUsageInfo; +struct QuotaUsage; /// Instances of `EnabledQuota` are used to track resource consumption. @@ -53,7 +53,7 @@ public: void checkExceeded(ResourceType resource_type) const; /// Returns the information about quota consumption. - QuotaUsageInfo getUsageInfo() const; + std::optional getUsage() const; /// Returns an instance of EnabledQuota which is never exceeded. static std::shared_ptr getUnlimitedQuota(); @@ -71,11 +71,11 @@ private: { mutable std::atomic used[MAX_RESOURCE_TYPE]; ResourceAmount max[MAX_RESOURCE_TYPE]; - std::chrono::seconds duration; - bool randomize_interval; + std::chrono::seconds duration = std::chrono::seconds::zero(); + bool randomize_interval = false; mutable std::atomic end_of_interval; - Interval() {} + Interval(); Interval(const Interval & src) { *this = src; } Interval & operator =(const Interval & src); }; @@ -83,11 +83,11 @@ private: struct Intervals { std::vector intervals; - UUID quota_id; + std::optional quota_id; String quota_name; String quota_key; - QuotaUsageInfo getUsageInfo(std::chrono::system_clock::time_point current_time) const; + std::optional getUsage(std::chrono::system_clock::time_point current_time) const; }; struct Impl; diff --git a/src/Access/Quota.cpp b/src/Access/Quota.cpp index a5afe5fda4f..dc855599999 100644 --- a/src/Access/Quota.cpp +++ b/src/Access/Quota.cpp @@ -1,22 +1,9 @@ #include #include -#include namespace DB { -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; -} - - -Quota::Limits::Limits() -{ - boost::range::fill(max, 0); -} - - bool operator ==(const Quota::Limits & lhs, const Quota::Limits & rhs) { return boost::range::equal(lhs.max, rhs.max) && (lhs.duration == rhs.duration) @@ -32,21 +19,5 @@ bool Quota::equal(const IAccessEntity & other) const return (all_limits == other_quota.all_limits) && (key_type == other_quota.key_type) && (to_roles == other_quota.to_roles); } - -const char * Quota::resourceTypeToColumnName(ResourceType resource_type) -{ - switch (resource_type) - { - case Quota::QUERIES: return "queries"; - case Quota::ERRORS: return "errors"; - case Quota::RESULT_ROWS: return "result_rows"; - case Quota::RESULT_BYTES: return "result_bytes"; - case Quota::READ_ROWS: return "read_rows"; - case Quota::READ_BYTES: return "read_bytes"; - case Quota::EXECUTION_TIME: return "execution_time"; - case Quota::MAX_RESOURCE_TYPE: break; - } - throw Exception("Unexpected resource type: " + std::to_string(static_cast(resource_type)), ErrorCodes::LOGICAL_ERROR); -} } diff --git a/src/Access/Quota.h b/src/Access/Quota.h index 317ed2dbc47..25b56756dc1 100644 --- a/src/Access/Quota.h +++ b/src/Access/Quota.h @@ -2,6 +2,7 @@ #include #include +#include #include @@ -23,6 +24,8 @@ namespace ErrorCodes */ struct Quota : public IAccessEntity { + using ResourceAmount = UInt64; + enum ResourceType { QUERIES, /// Number of queries. @@ -36,19 +39,28 @@ struct Quota : public IAccessEntity MAX_RESOURCE_TYPE }; - using ResourceAmount = UInt64; - static constexpr ResourceAmount UNLIMITED = 0; /// 0 means unlimited. + struct ResourceTypeInfo + { + const char * const raw_name; + const String name; /// Lowercased with underscores, e.g. "result_rows". + const String keyword; /// Uppercased with spaces, e.g. "RESULT ROWS". + const bool output_as_float = false; + const UInt64 output_denominator = 1; + String amountToString(ResourceAmount amount) const; + ResourceAmount amountFromString(const String & str) const; + String outputWithAmount(ResourceAmount amount) const; + static const ResourceTypeInfo & get(ResourceType type); + }; /// Amount of resources available to consume for each duration. struct Limits { - ResourceAmount max[MAX_RESOURCE_TYPE]; + std::optional max[MAX_RESOURCE_TYPE]; std::chrono::seconds duration = std::chrono::seconds::zero(); /// Intervals can be randomized (to avoid DoS if intervals for many users end at one time). bool randomize_interval = false; - Limits(); friend bool operator ==(const Limits & lhs, const Limits & rhs); friend bool operator !=(const Limits & lhs, const Limits & rhs) { return !(lhs == rhs); } }; @@ -68,6 +80,14 @@ struct Quota : public IAccessEntity MAX }; + + struct KeyTypeInfo + { + const char * const raw_name; + const String name; /// Lowercased with spaces, e.g. "client key". + static const KeyTypeInfo & get(KeyType type); + }; + KeyType key_type = KeyType::NONE; /// Which roles or users should use this quota. @@ -77,76 +97,144 @@ struct Quota : public IAccessEntity std::shared_ptr clone() const override { return cloneImpl(); } static constexpr const Type TYPE = Type::QUOTA; Type getType() const override { return TYPE; } - - static const char * getNameOfResourceType(ResourceType resource_type); - static const char * resourceTypeToKeyword(ResourceType resource_type); - static const char * resourceTypeToColumnName(ResourceType resource_type); - static const char * getNameOfKeyType(KeyType key_type); - static double executionTimeToSeconds(ResourceAmount ns); - static ResourceAmount secondsToExecutionTime(double s); }; -inline const char * Quota::getNameOfResourceType(ResourceType resource_type) +inline String Quota::ResourceTypeInfo::amountToString(ResourceAmount amount) const { - switch (resource_type) + if (!(amount % output_denominator)) + return std::to_string(amount / output_denominator); + else + return boost::lexical_cast(static_cast(amount) / output_denominator); +} + +inline Quota::ResourceAmount Quota::ResourceTypeInfo::amountFromString(const String & str) const +{ + if (output_denominator == 1) + return static_cast(std::strtoul(str.c_str(), nullptr, 10)); + else + return static_cast(std::strtod(str.c_str(), nullptr) * output_denominator); +} + +inline String Quota::ResourceTypeInfo::outputWithAmount(ResourceAmount amount) const +{ + String res = name; + res += " = "; + res += amountToString(amount); + return res; +} + +inline String toString(Quota::ResourceType type) +{ + return Quota::ResourceTypeInfo::get(type).raw_name; +} + +inline const Quota::ResourceTypeInfo & Quota::ResourceTypeInfo::get(ResourceType type) +{ + static constexpr auto make_info = [](const char * raw_name_, UInt64 output_denominator_) { - case Quota::QUERIES: return "queries"; - case Quota::ERRORS: return "errors"; - case Quota::RESULT_ROWS: return "result rows"; - case Quota::RESULT_BYTES: return "result bytes"; - case Quota::READ_ROWS: return "read rows"; - case Quota::READ_BYTES: return "read bytes"; - case Quota::EXECUTION_TIME: return "execution time"; + String init_name = raw_name_; + boost::to_lower(init_name); + String init_keyword = raw_name_; + boost::replace_all(init_keyword, "_", " "); + bool init_output_as_float = (output_denominator_ != 1); + return ResourceTypeInfo{raw_name_, std::move(init_name), std::move(init_keyword), init_output_as_float, output_denominator_}; + }; + + switch (type) + { + case Quota::QUERIES: + { + static const auto info = make_info("QUERIES", 1); + return info; + } + case Quota::ERRORS: + { + static const auto info = make_info("ERRORS", 1); + return info; + } + case Quota::RESULT_ROWS: + { + static const auto info = make_info("RESULT_ROWS", 1); + return info; + } + case Quota::RESULT_BYTES: + { + static const auto info = make_info("RESULT_BYTES", 1); + return info; + } + case Quota::READ_ROWS: + { + static const auto info = make_info("READ_ROWS", 1); + return info; + } + case Quota::READ_BYTES: + { + static const auto info = make_info("READ_BYTES", 1); + return info; + } + case Quota::EXECUTION_TIME: + { + static const auto info = make_info("EXECUTION_TIME", 1000000000 /* execution_time is stored in nanoseconds */); + return info; + } case Quota::MAX_RESOURCE_TYPE: break; } - throw Exception("Unexpected resource type: " + std::to_string(static_cast(resource_type)), ErrorCodes::LOGICAL_ERROR); + throw Exception("Unexpected resource type: " + std::to_string(static_cast(type)), ErrorCodes::LOGICAL_ERROR); } -inline const char * Quota::resourceTypeToKeyword(ResourceType resource_type) +inline String toString(Quota::KeyType type) { - switch (resource_type) - { - case Quota::QUERIES: return "QUERIES"; - case Quota::ERRORS: return "ERRORS"; - case Quota::RESULT_ROWS: return "RESULT ROWS"; - case Quota::RESULT_BYTES: return "RESULT BYTES"; - case Quota::READ_ROWS: return "READ ROWS"; - case Quota::READ_BYTES: return "READ BYTES"; - case Quota::EXECUTION_TIME: return "EXECUTION TIME"; - case Quota::MAX_RESOURCE_TYPE: break; - } - throw Exception("Unexpected resource type: " + std::to_string(static_cast(resource_type)), ErrorCodes::LOGICAL_ERROR); + return Quota::KeyTypeInfo::get(type).raw_name; } - -inline const char * Quota::getNameOfKeyType(KeyType key_type) +inline const Quota::KeyTypeInfo & Quota::KeyTypeInfo::get(KeyType type) { - switch (key_type) + static constexpr auto make_info = [](const char * raw_name_) { - case KeyType::NONE: return "none"; - case KeyType::USER_NAME: return "user name"; - case KeyType::IP_ADDRESS: return "ip address"; - case KeyType::CLIENT_KEY: return "client key"; - case KeyType::CLIENT_KEY_OR_USER_NAME: return "client key or user name"; - case KeyType::CLIENT_KEY_OR_IP_ADDRESS: return "client key or ip address"; + String init_name = raw_name_; + boost::to_lower(init_name); + boost::replace_all(init_name, "_", " "); + return KeyTypeInfo{raw_name_, std::move(init_name)}; + }; + + switch (type) + { + case KeyType::NONE: + { + static const auto info = make_info("NONE"); + return info; + } + case KeyType::USER_NAME: + { + static const auto info = make_info("USER_NAME"); + return info; + } + case KeyType::IP_ADDRESS: + { + static const auto info = make_info("IP_ADDRESS"); + return info; + } + case KeyType::CLIENT_KEY: + { + static const auto info = make_info("CLIENT_KEY"); + return info; + } + case KeyType::CLIENT_KEY_OR_USER_NAME: + { + static const auto info = make_info("CLIENT_KEY_OR_USER_NAME"); + return info; + } + case KeyType::CLIENT_KEY_OR_IP_ADDRESS: + { + static const auto info = make_info("CLIENT_KEY_OR_IP_ADDRESS"); + return info; + } case KeyType::MAX: break; } - throw Exception("Unexpected quota key type: " + std::to_string(static_cast(key_type)), ErrorCodes::LOGICAL_ERROR); + throw Exception("Unexpected quota key type: " + std::to_string(static_cast(type)), ErrorCodes::LOGICAL_ERROR); } - -inline double Quota::executionTimeToSeconds(ResourceAmount ns) -{ - return std::chrono::duration_cast>(std::chrono::nanoseconds{ns}).count(); -} - -inline Quota::ResourceAmount Quota::secondsToExecutionTime(double s) -{ - return std::chrono::duration_cast(std::chrono::duration(s)).count(); -} - - using QuotaPtr = std::shared_ptr; } diff --git a/src/Access/QuotaCache.cpp b/src/Access/QuotaCache.cpp index 842106b3ba9..d3ebca9529c 100644 --- a/src/Access/QuotaCache.cpp +++ b/src/Access/QuotaCache.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #include #include #include @@ -116,7 +116,8 @@ boost::shared_ptr QuotaCache::QuotaInfo::rebuildI interval.end_of_interval = end_of_interval.time_since_epoch(); for (auto resource_type : ext::range(MAX_RESOURCE_TYPE)) { - interval.max[resource_type] = limits.max[resource_type]; + if (limits.max[resource_type]) + interval.max[resource_type] = *limits.max[resource_type]; interval.used[resource_type] = 0; } } @@ -287,16 +288,20 @@ void QuotaCache::chooseQuotaToConsumeFor(EnabledQuota & enabled) } -std::vector QuotaCache::getUsageInfo() const +std::vector QuotaCache::getAllQuotasUsage() const { std::lock_guard lock{mutex}; - std::vector all_infos; + std::vector all_usage; auto current_time = std::chrono::system_clock::now(); for (const auto & info : all_quotas | boost::adaptors::map_values) { for (const auto & intervals : info.key_to_intervals | boost::adaptors::map_values) - all_infos.push_back(intervals->getUsageInfo(current_time)); + { + auto usage = intervals->getUsage(current_time); + if (usage) + all_usage.push_back(std::move(usage).value()); + } } - return all_infos; + return all_usage; } } diff --git a/src/Access/QuotaCache.h b/src/Access/QuotaCache.h index b9996ed8456..6e794f0bbd2 100644 --- a/src/Access/QuotaCache.h +++ b/src/Access/QuotaCache.h @@ -21,7 +21,7 @@ public: ~QuotaCache(); std::shared_ptr getEnabledQuota(const UUID & user_id, const String & user_name, const boost::container::flat_set & enabled_roles, const Poco::Net::IPAddress & address, const String & client_key); - std::vector getUsageInfo() const; + std::vector getAllQuotasUsage() const; private: using Interval = EnabledQuota::Interval; diff --git a/src/Access/QuotaUsage.cpp b/src/Access/QuotaUsage.cpp new file mode 100644 index 00000000000..cf03c4f478a --- /dev/null +++ b/src/Access/QuotaUsage.cpp @@ -0,0 +1,16 @@ +#include +#include + + +namespace DB +{ +QuotaUsage::QuotaUsage() : quota_id(UUID(UInt128(0))) +{ +} + + +QuotaUsage::Interval::Interval() +{ + boost::range::fill(used, 0); +} +} diff --git a/src/Access/QuotaUsageInfo.h b/src/Access/QuotaUsage.h similarity index 87% rename from src/Access/QuotaUsageInfo.h rename to src/Access/QuotaUsage.h index 45ee7898145..9e53c0cf7d7 100644 --- a/src/Access/QuotaUsageInfo.h +++ b/src/Access/QuotaUsage.h @@ -7,7 +7,7 @@ namespace DB { /// The information about a quota consumption. -struct QuotaUsageInfo +struct QuotaUsage { using ResourceType = Quota::ResourceType; using ResourceAmount = Quota::ResourceAmount; @@ -16,7 +16,7 @@ struct QuotaUsageInfo struct Interval { ResourceAmount used[MAX_RESOURCE_TYPE]; - ResourceAmount max[MAX_RESOURCE_TYPE]; + std::optional max[MAX_RESOURCE_TYPE]; std::chrono::seconds duration = std::chrono::seconds::zero(); bool randomize_interval = false; std::chrono::system_clock::time_point end_of_interval; @@ -27,6 +27,6 @@ struct QuotaUsageInfo UUID quota_id; String quota_name; String quota_key; - QuotaUsageInfo(); + QuotaUsage(); }; } diff --git a/src/Access/QuotaUsageInfo.cpp b/src/Access/QuotaUsageInfo.cpp deleted file mode 100644 index bcdf2b50062..00000000000 --- a/src/Access/QuotaUsageInfo.cpp +++ /dev/null @@ -1,17 +0,0 @@ -#include -#include - - -namespace DB -{ -QuotaUsageInfo::QuotaUsageInfo() : quota_id(UUID(UInt128(0))) -{ -} - - -QuotaUsageInfo::Interval::Interval() -{ - boost::range::fill(used, 0); - boost::range::fill(max, 0); -} -} diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index 43a9e355911..ce33383548f 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -225,14 +225,13 @@ namespace limits.duration = duration; limits.randomize_interval = config.getBool(interval_config + ".randomize", false); - using ResourceType = Quota::ResourceType; - limits.max[ResourceType::QUERIES] = config.getUInt64(interval_config + ".queries", Quota::UNLIMITED); - limits.max[ResourceType::ERRORS] = config.getUInt64(interval_config + ".errors", Quota::UNLIMITED); - limits.max[ResourceType::RESULT_ROWS] = config.getUInt64(interval_config + ".result_rows", Quota::UNLIMITED); - limits.max[ResourceType::RESULT_BYTES] = config.getUInt64(interval_config + ".result_bytes", Quota::UNLIMITED); - limits.max[ResourceType::READ_ROWS] = config.getUInt64(interval_config + ".read_rows", Quota::UNLIMITED); - limits.max[ResourceType::READ_BYTES] = config.getUInt64(interval_config + ".read_bytes", Quota::UNLIMITED); - limits.max[ResourceType::EXECUTION_TIME] = Quota::secondsToExecutionTime(config.getUInt64(interval_config + ".execution_time", Quota::UNLIMITED)); + for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) + { + const auto & type_info = Quota::ResourceTypeInfo::get(resource_type); + auto value = config.getString(interval_config + "." + type_info.name, "0"); + if (value != "0") + limits.max[resource_type] = type_info.amountFromString(value); + } } quota->to_roles.add(user_ids); diff --git a/src/Access/ya.make b/src/Access/ya.make index 5b85330a7f4..970c0714a93 100644 --- a/src/Access/ya.make +++ b/src/Access/ya.make @@ -26,7 +26,7 @@ SRCS( MultipleAccessStorage.cpp Quota.cpp QuotaCache.cpp - QuotaUsageInfo.cpp + QuotaUsage.cpp Role.cpp RoleCache.cpp RowPolicy.cpp diff --git a/src/Functions/currentQuota.cpp b/src/Functions/currentQuota.cpp deleted file mode 100644 index b16a8a7c1ec..00000000000 --- a/src/Functions/currentQuota.cpp +++ /dev/null @@ -1,135 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace DB -{ - -class FunctionCurrentQuota : public IFunction -{ - const String quota_name; - -public: - static constexpr auto name = "currentQuota"; - static FunctionPtr create(const Context & context) - { - return std::make_shared(context.getQuota()->getUsageInfo().quota_name); - } - - explicit FunctionCurrentQuota(const String & quota_name_) : quota_name{quota_name_} - { - } - - String getName() const override - { - return name; - } - size_t getNumberOfArguments() const override - { - return 0; - } - - DataTypePtr getReturnTypeImpl(const DataTypes & /*arguments*/) const override - { - return std::make_shared(); - } - - bool isDeterministic() const override { return false; } - - void executeImpl(Block & block, const ColumnNumbers &, size_t result, size_t input_rows_count) override - { - block.getByPosition(result).column = DataTypeString().createColumnConst(input_rows_count, quota_name); - } -}; - - -class FunctionCurrentQuotaId : public IFunction -{ - const UUID quota_id; - -public: - static constexpr auto name = "currentQuotaID"; - static FunctionPtr create(const Context & context) - { - return std::make_shared(context.getQuota()->getUsageInfo().quota_id); - } - - explicit FunctionCurrentQuotaId(const UUID quota_id_) : quota_id{quota_id_} - { - } - - String getName() const override - { - return name; - } - size_t getNumberOfArguments() const override - { - return 0; - } - - DataTypePtr getReturnTypeImpl(const DataTypes & /*arguments*/) const override - { - return std::make_shared(); - } - - bool isDeterministic() const override { return false; } - - void executeImpl(Block & block, const ColumnNumbers &, size_t result, size_t input_rows_count) override - { - block.getByPosition(result).column = DataTypeUUID().createColumnConst(input_rows_count, quota_id); - } -}; - - -class FunctionCurrentQuotaKey : public IFunction -{ - const String quota_key; - -public: - static constexpr auto name = "currentQuotaKey"; - static FunctionPtr create(const Context & context) - { - return std::make_shared(context.getQuota()->getUsageInfo().quota_key); - } - - explicit FunctionCurrentQuotaKey(const String & quota_key_) : quota_key{quota_key_} - { - } - - String getName() const override - { - return name; - } - size_t getNumberOfArguments() const override - { - return 0; - } - - DataTypePtr getReturnTypeImpl(const DataTypes & /*arguments*/) const override - { - return std::make_shared(); - } - - bool isDeterministic() const override { return false; } - - void executeImpl(Block & block, const ColumnNumbers &, size_t result, size_t input_rows_count) override - { - block.getByPosition(result).column = DataTypeString().createColumnConst(input_rows_count, quota_key); - } -}; - - -void registerFunctionCurrentQuota(FunctionFactory & factory) -{ - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); -} - -} diff --git a/src/Functions/registerFunctionsMiscellaneous.cpp b/src/Functions/registerFunctionsMiscellaneous.cpp index 737cde10e74..c8c6d6d79e2 100644 --- a/src/Functions/registerFunctionsMiscellaneous.cpp +++ b/src/Functions/registerFunctionsMiscellaneous.cpp @@ -9,7 +9,6 @@ class FunctionFactory; void registerFunctionCurrentDatabase(FunctionFactory &); void registerFunctionCurrentUser(FunctionFactory &); -void registerFunctionCurrentQuota(FunctionFactory &); void registerFunctionHostName(FunctionFactory &); void registerFunctionFQDN(FunctionFactory &); void registerFunctionVisibleWidth(FunctionFactory &); @@ -67,7 +66,6 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory) { registerFunctionCurrentDatabase(factory); registerFunctionCurrentUser(factory); - registerFunctionCurrentQuota(factory); registerFunctionHostName(factory); registerFunctionFQDN(factory); registerFunctionVisibleWidth(factory); diff --git a/src/Functions/ya.make b/src/Functions/ya.make index 087fe3824c6..67eeb8cc884 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -127,7 +127,6 @@ SRCS( cos.cpp CRC.cpp currentDatabase.cpp - currentQuota.cpp currentUser.cpp dateDiff.cpp defaultValueOfArgumentType.cpp diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index ebeeba54d1d..2022c66b4e5 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -682,14 +683,12 @@ void Context::setUser(const String & name, const String & password, const Poco:: std::shared_ptr Context::getUser() const { - auto lock = getLock(); - return access->getUser(); + return getAccess()->getUser(); } String Context::getUserName() const { - auto lock = getLock(); - return access->getUserName(); + return getAccess()->getUserName(); } std::optional Context::getUserID() const @@ -792,6 +791,12 @@ std::shared_ptr Context::getQuota() const } +std::optional Context::getQuotaUsage() const +{ + return getAccess()->getQuotaUsage(); +} + + void Context::setProfile(const String & profile_name) { applySettingsChanges(*getAccessControlManager().getProfileSettings(profile_name)); @@ -1002,8 +1007,7 @@ void Context::clampToSettingsConstraints(SettingsChanges & changes) const std::shared_ptr Context::getSettingsConstraints() const { - auto lock = getLock(); - return access->getSettingsConstraints(); + return getAccess()->getSettingsConstraints(); } diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index f0ae44f5548..31d7b1f1e13 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -54,6 +54,7 @@ using UserPtr = std::shared_ptr; struct EnabledRolesInfo; class EnabledRowPolicies; class EnabledQuota; +struct QuotaUsage; class AccessFlags; struct AccessRightsElement; class AccessRightsElements; @@ -286,6 +287,7 @@ public: void setInitialRowPolicy(); std::shared_ptr getQuota() const; + std::optional getQuotaUsage() const; /// We have to copy external tables inside executeQuery() to track limits. Therefore, set callback for it. Must set once. void setExternalTablesInitializer(ExternalTablesInitializer && initializer); diff --git a/src/Interpreters/InterpreterCreateQuotaQuery.cpp b/src/Interpreters/InterpreterCreateQuotaQuery.cpp index 80987993c96..907532c3d89 100644 --- a/src/Interpreters/InterpreterCreateQuotaQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuotaQuery.cpp @@ -56,12 +56,7 @@ void updateQuotaFromQueryImpl(Quota & quota, const ASTCreateQuotaQuery & query, auto & quota_limits = *it; quota_limits.randomize_interval = query_limits.randomize_interval; for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) - { - if (query_limits.max[resource_type]) - quota_limits.max[resource_type] = *query_limits.max[resource_type]; - else - quota_limits.max[resource_type] = Quota::UNLIMITED; - } + quota_limits.max[resource_type] = query_limits.max[resource_type]; } const ExtendedRoleSet * roles = nullptr; diff --git a/src/Interpreters/InterpreterFactory.cpp b/src/Interpreters/InterpreterFactory.cpp index dae025a1b30..c740da0601d 100644 --- a/src/Interpreters/InterpreterFactory.cpp +++ b/src/Interpreters/InterpreterFactory.cpp @@ -19,7 +19,6 @@ #include #include #include -#include #include #include #include @@ -53,7 +52,6 @@ #include #include #include -#include #include #include #include @@ -226,10 +224,6 @@ std::unique_ptr InterpreterFactory::get(ASTPtr & query, Context & { return std::make_unique(query, context); } - else if (query->as()) - { - return std::make_unique(query, context); - } else if (query->as()) { return std::make_unique(query, context); diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 7a15f71e180..848910156f9 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -423,11 +423,14 @@ InterpreterSelectQuery::InterpreterSelectQuery( context->checkAccess(AccessType::SELECT, left_table_id, required_columns); /// Remove limits for some tables in the `system` database. - if (left_table_id.database_name == "system" && - ((left_table_id.table_name == "quotas") || (left_table_id.table_name == "quota_usage") || (left_table_id.table_name == "one"))) + if (left_table_id.database_name == "system") { - options.ignore_quota = true; - options.ignore_limits = true; + static const boost::container::flat_set system_tables_ignoring_quota{"quotas", "quota_limits", "quota_usage", "quotas_usage", "one"}; + if (system_tables_ignoring_quota.count(left_table_id.table_name)) + { + options.ignore_quota = true; + options.ignore_limits = true; + } } /// Blocks used in expression analysis contains size 1 const columns for constant folding and diff --git a/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp b/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp index 8a892685ed4..f6527d0ba8e 100644 --- a/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp +++ b/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp @@ -18,7 +18,7 @@ using EntityType = IAccessEntity::Type; InterpreterShowAccessEntitiesQuery::InterpreterShowAccessEntitiesQuery(const ASTPtr & query_ptr_, Context & context_) - : query_ptr(query_ptr_), context(context_) + : query_ptr(query_ptr_), context(context_), ignore_quota(query_ptr->as().type == EntityType::QUOTA) { } @@ -34,7 +34,7 @@ String InterpreterShowAccessEntitiesQuery::getRewrittenQuery() const const auto & query = query_ptr->as(); String origin; String expr = "name"; - String filter; + String filter, order; if (query.type == EntityType::ROW_POLICY) { @@ -57,11 +57,28 @@ String InterpreterShowAccessEntitiesQuery::getRewrittenQuery() const if (show_short_name) expr = "short_name"; } + else if (query.type == EntityType::QUOTA) + { + if (query.current_quota) + { + origin = "quota_usage"; + expr = "*"; + order = "duration"; + } + else + { + origin = "quotas"; + } + } else throw Exception(toString(query.type) + ": type is not supported by SHOW query", ErrorCodes::NOT_IMPLEMENTED); + if (order.empty() && expr != "*") + order = expr; + return "SELECT " + expr + " from system." + origin + - (filter.empty() ? "" : " WHERE " + filter) + " ORDER BY " + expr; + (filter.empty() ? "" : " WHERE " + filter) + + (order.empty() ? "" : " ORDER BY " + order); } } diff --git a/src/Interpreters/InterpreterShowAccessEntitiesQuery.h b/src/Interpreters/InterpreterShowAccessEntitiesQuery.h index f5bf0d0c7bb..5e20bdfa231 100644 --- a/src/Interpreters/InterpreterShowAccessEntitiesQuery.h +++ b/src/Interpreters/InterpreterShowAccessEntitiesQuery.h @@ -12,13 +12,18 @@ class InterpreterShowAccessEntitiesQuery : public IInterpreter { public: InterpreterShowAccessEntitiesQuery(const ASTPtr & query_ptr_, Context & context_); + BlockIO execute() override; + bool ignoreQuota() const override { return ignore_quota; } + bool ignoreLimits() const override { return ignore_quota; } + private: String getRewrittenQuery() const; ASTPtr query_ptr; Context & context; + bool ignore_quota = false; }; } diff --git a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp index 51ac3644ff5..87e1265f793 100644 --- a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp +++ b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include #include @@ -138,8 +138,7 @@ namespace create_query_limits.duration = limits.duration; create_query_limits.randomize_interval = limits.randomize_interval; for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) - if (limits.max[resource_type] != Quota::UNLIMITED) - create_query_limits.max[resource_type] = limits.max[resource_type]; + create_query_limits.max[resource_type] = limits.max[resource_type]; query->all_limits.push_back(create_query_limits); } @@ -211,6 +210,12 @@ namespace } +InterpreterShowCreateAccessEntityQuery::InterpreterShowCreateAccessEntityQuery(const ASTPtr & query_ptr_, const Context & context_) + : query_ptr(query_ptr_), context(context_), ignore_quota(query_ptr->as().type == EntityType::QUOTA) +{ +} + + BlockIO InterpreterShowCreateAccessEntityQuery::execute() { BlockIO res; @@ -227,11 +232,14 @@ BlockInputStreamPtr InterpreterShowCreateAccessEntityQuery::executeImpl() ASTPtr create_query = getCreateQuery(show_query); /// Build the result column. - std::stringstream create_query_ss; - formatAST(*create_query, create_query_ss, false, true); - String create_query_str = create_query_ss.str(); MutableColumnPtr column = ColumnString::create(); - column->insert(create_query_str); + if (create_query) + { + std::stringstream create_query_ss; + formatAST(*create_query, create_query_ss, false, true); + String create_query_str = create_query_ss.str(); + column->insert(create_query_str); + } /// Prepare description of the result column. std::stringstream desc_ss; @@ -253,16 +261,21 @@ ASTPtr InterpreterShowCreateAccessEntityQuery::getCreateQuery(ASTShowCreateAcces if (show_query.current_user) { auto user = context.getUser(); + if (!user) + return nullptr; return getCreateQueryImpl(*user, &access_control, false); } if (show_query.current_quota) { - auto quota = access_control.read(context.getQuota()->getUsageInfo().quota_id); + auto usage = context.getQuotaUsage(); + if (!usage) + return nullptr; + auto quota = access_control.read(usage->quota_id); return getCreateQueryImpl(*quota, &access_control, false); } - if (show_query.type == Type::ROW_POLICY) + if (show_query.type == EntityType::ROW_POLICY) { if (show_query.row_policy_name_parts.database.empty()) show_query.row_policy_name_parts.database = context.getCurrentDatabase(); diff --git a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.h b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.h index fa610c8cc65..ee28bfbdc4a 100644 --- a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.h +++ b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.h @@ -18,13 +18,12 @@ struct IAccessEntity; class InterpreterShowCreateAccessEntityQuery : public IInterpreter { public: - InterpreterShowCreateAccessEntityQuery(const ASTPtr & query_ptr_, const Context & context_) - : query_ptr(query_ptr_), context(context_) {} + InterpreterShowCreateAccessEntityQuery(const ASTPtr & query_ptr_, const Context & context_); BlockIO execute() override; - bool ignoreQuota() const override { return true; } - bool ignoreLimits() const override { return true; } + bool ignoreQuota() const override { return ignore_quota; } + bool ignoreLimits() const override { return ignore_quota; } static ASTPtr getAttachQuery(const IAccessEntity & entity); @@ -35,6 +34,7 @@ private: ASTPtr query_ptr; const Context & context; + bool ignore_quota = false; }; diff --git a/src/Interpreters/InterpreterShowQuotasQuery.cpp b/src/Interpreters/InterpreterShowQuotasQuery.cpp deleted file mode 100644 index 4500046577c..00000000000 --- a/src/Interpreters/InterpreterShowQuotasQuery.cpp +++ /dev/null @@ -1,73 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace DB -{ -InterpreterShowQuotasQuery::InterpreterShowQuotasQuery(const ASTPtr & query_ptr_, Context & context_) - : query_ptr(query_ptr_), context(context_) -{ -} - - -String InterpreterShowQuotasQuery::getRewrittenQuery() -{ - const auto & query = query_ptr->as(); - - /// Transform the query into some kind of "SELECT from system.quotas" query. - String expr; - String filter; - String table_name; - String order_by; - if (query.usage) - { - expr = "name || ' key=\\'' || key || '\\'' || if(isNull(end_of_interval), '', ' interval=[' || " - "toString(end_of_interval - duration) || ' .. ' || " - "toString(end_of_interval) || ']'"; - for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) - { - String column_name = Quota::resourceTypeToColumnName(resource_type); - expr += String{" || ' "} + column_name + "=' || toString(" + column_name + ")"; - expr += String{" || if(max_"} + column_name + "=0, '', '/' || toString(max_" + column_name + "))"; - } - expr += ")"; - - if (query.current) - filter = "(id = currentQuotaID()) AND (key = currentQuotaKey())"; - - table_name = "system.quota_usage"; - order_by = "name, key, duration"; - } - else - { - expr = "name"; - table_name = "system.quotas"; - order_by = "name"; - } - - /// Prepare description of the result column. - std::stringstream ss; - formatAST(query, ss, false, true); - String desc = ss.str(); - String prefix = "SHOW "; - if (startsWith(desc, prefix)) - desc = desc.substr(prefix.length()); /// `desc` always starts with "SHOW ", so we can trim this prefix. - - /// Build a new query. - return "SELECT " + expr + " AS " + backQuote(desc) + " FROM " + table_name + (filter.empty() ? "" : (" WHERE " + filter)) - + (order_by.empty() ? "" : (" ORDER BY " + order_by)); -} - - -BlockIO InterpreterShowQuotasQuery::execute() -{ - return executeQuery(getRewrittenQuery(), context, true); -} - -} diff --git a/src/Interpreters/InterpreterShowQuotasQuery.h b/src/Interpreters/InterpreterShowQuotasQuery.h deleted file mode 100644 index ae608e81ce5..00000000000 --- a/src/Interpreters/InterpreterShowQuotasQuery.h +++ /dev/null @@ -1,28 +0,0 @@ -#pragma once - -#include -#include - - -namespace DB -{ -class Context; - -class InterpreterShowQuotasQuery : public IInterpreter -{ -public: - InterpreterShowQuotasQuery(const ASTPtr & query_ptr_, Context & context_); - - BlockIO execute() override; - - bool ignoreQuota() const override { return true; } - bool ignoreLimits() const override { return true; } - -private: - ASTPtr query_ptr; - Context & context; - - String getRewrittenQuery(); -}; - -} diff --git a/src/Interpreters/ya.make b/src/Interpreters/ya.make index 440e1e1e1c0..7c5a86aa1cb 100644 --- a/src/Interpreters/ya.make +++ b/src/Interpreters/ya.make @@ -88,7 +88,6 @@ SRCS( InterpreterShowCreateQuery.cpp InterpreterShowGrantsQuery.cpp InterpreterShowProcesslistQuery.cpp - InterpreterShowQuotasQuery.cpp InterpreterShowTablesQuery.cpp InterpreterSystemQuery.cpp InterpreterUseQuery.cpp diff --git a/src/Parsers/ASTCreateQuotaQuery.cpp b/src/Parsers/ASTCreateQuotaQuery.cpp index e990b9939bc..bdfd1b32e96 100644 --- a/src/Parsers/ASTCreateQuotaQuery.cpp +++ b/src/Parsers/ASTCreateQuotaQuery.cpp @@ -10,14 +10,16 @@ namespace DB namespace { using KeyType = Quota::KeyType; + using KeyTypeInfo = Quota::KeyTypeInfo; using ResourceType = Quota::ResourceType; + using ResourceTypeInfo = Quota::ResourceTypeInfo; using ResourceAmount = Quota::ResourceAmount; void formatKeyType(const KeyType & key_type, const IAST::FormatSettings & settings) { settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " KEYED BY " << (settings.hilite ? IAST::hilite_none : "") << "'" - << Quota::getNameOfKeyType(key_type) << "'"; + << KeyTypeInfo::get(key_type).name << "'"; } @@ -35,13 +37,9 @@ namespace else settings.ostr << ","; - settings.ostr << " " << (settings.hilite ? IAST::hilite_keyword : "") << Quota::resourceTypeToKeyword(resource_type) - << (settings.hilite ? IAST::hilite_none : "") << " "; - - if (resource_type == Quota::EXECUTION_TIME) - settings.ostr << Quota::executionTimeToSeconds(max); - else - settings.ostr << max; + const auto & type_info = ResourceTypeInfo::get(resource_type); + settings.ostr << " " << (settings.hilite ? IAST::hilite_keyword : "") << type_info.keyword + << (settings.hilite ? IAST::hilite_none : "") << " " << type_info.amountToString(max); } diff --git a/src/Parsers/ASTCreateQuotaQuery.h b/src/Parsers/ASTCreateQuotaQuery.h index ad20348a139..b9994e8ec3a 100644 --- a/src/Parsers/ASTCreateQuotaQuery.h +++ b/src/Parsers/ASTCreateQuotaQuery.h @@ -35,18 +35,16 @@ public: bool if_not_exists = false; bool or_replace = false; + using KeyType = Quota::KeyType; + using ResourceAmount = Quota::ResourceAmount; + String name; String new_name; - using KeyType = Quota::KeyType; std::optional key_type; - using ResourceType = Quota::ResourceType; - using ResourceAmount = Quota::ResourceAmount; - static constexpr auto MAX_RESOURCE_TYPE = Quota::MAX_RESOURCE_TYPE; - struct Limits { - std::optional max[MAX_RESOURCE_TYPE]; + std::optional max[Quota::MAX_RESOURCE_TYPE]; bool drop = false; std::chrono::seconds duration = std::chrono::seconds::zero(); bool randomize_interval = false; diff --git a/src/Parsers/ASTShowAccessEntitiesQuery.cpp b/src/Parsers/ASTShowAccessEntitiesQuery.cpp index ec67fc5fb7f..a1da1ac04e2 100644 --- a/src/Parsers/ASTShowAccessEntitiesQuery.cpp +++ b/src/Parsers/ASTShowAccessEntitiesQuery.cpp @@ -14,6 +14,8 @@ String ASTShowAccessEntitiesQuery::getID(char) const { if (type == EntityType::ROW_POLICY) return "SHOW ROW POLICIES query"; + else if (type == EntityType::QUOTA) + return current_quota ? "SHOW CURRENT QUOTA query" : "SHOW QUOTAS query"; else throw Exception(toString(type) + ": type is not supported by SHOW query", ErrorCodes::NOT_IMPLEMENTED); } @@ -22,6 +24,8 @@ void ASTShowAccessEntitiesQuery::formatQueryImpl(const FormatSettings & settings { if (type == EntityType::ROW_POLICY) settings.ostr << (settings.hilite ? hilite_keyword : "") << "SHOW ROW POLICIES" << (settings.hilite ? hilite_none : ""); + else if (type == EntityType::QUOTA) + settings.ostr << (settings.hilite ? hilite_keyword : "") << (current_quota ? "SHOW CURRENT QUOTA" : "SHOW QUOTAS") << (settings.hilite ? hilite_none : ""); else throw Exception(toString(type) + ": type is not supported by SHOW query", ErrorCodes::NOT_IMPLEMENTED); diff --git a/src/Parsers/ASTShowAccessEntitiesQuery.h b/src/Parsers/ASTShowAccessEntitiesQuery.h index 82d3474f3ad..09ccbe9cf91 100644 --- a/src/Parsers/ASTShowAccessEntitiesQuery.h +++ b/src/Parsers/ASTShowAccessEntitiesQuery.h @@ -8,6 +8,8 @@ namespace DB { /// SHOW [ROW] POLICIES [ON [database.]table] +/// SHOW QUOTAS +/// SHOW [CURRENT] QUOTA class ASTShowAccessEntitiesQuery : public ASTQueryWithOutput { public: @@ -16,6 +18,7 @@ public: EntityType type; String database; String table_name; + bool current_quota = false; String getID(char) const override; ASTPtr clone() const override { return std::make_shared(*this); } diff --git a/src/Parsers/ASTShowCreateAccessEntityQuery.cpp b/src/Parsers/ASTShowCreateAccessEntityQuery.cpp index 954fab673e7..f81b30428fb 100644 --- a/src/Parsers/ASTShowCreateAccessEntityQuery.cpp +++ b/src/Parsers/ASTShowCreateAccessEntityQuery.cpp @@ -25,11 +25,9 @@ void ASTShowCreateAccessEntityQuery::formatQueryImpl(const FormatSettings & sett << "SHOW CREATE " << EntityTypeInfo::get(type).name << (settings.hilite ? hilite_none : ""); - if (current_user) + if (current_user || current_quota) { } - else if (current_quota) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " CURRENT" << (settings.hilite ? hilite_none : ""); else if (type == EntityType::ROW_POLICY) { const String & database = row_policy_name_parts.database; diff --git a/src/Parsers/ASTShowQuotasQuery.cpp b/src/Parsers/ASTShowQuotasQuery.cpp deleted file mode 100644 index ca7bd5e853f..00000000000 --- a/src/Parsers/ASTShowQuotasQuery.cpp +++ /dev/null @@ -1,35 +0,0 @@ -#include -#include - - -namespace DB -{ -String ASTShowQuotasQuery::getID(char) const -{ - if (usage) - return "SHOW QUOTA USAGE query"; - else - return "SHOW QUOTAS query"; -} - - -ASTPtr ASTShowQuotasQuery::clone() const -{ - return std::make_shared(*this); -} - - -void ASTShowQuotasQuery::formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const -{ - settings.ostr << (settings.hilite ? hilite_keyword : ""); - - if (usage && current) - settings.ostr << "SHOW QUOTA USAGE"; - else if (usage) - settings.ostr << "SHOW QUOTA USAGE ALL"; - else - settings.ostr << "SHOW QUOTAS"; - - settings.ostr << (settings.hilite ? hilite_none : ""); -} -} diff --git a/src/Parsers/ASTShowQuotasQuery.h b/src/Parsers/ASTShowQuotasQuery.h deleted file mode 100644 index 27a08a99a54..00000000000 --- a/src/Parsers/ASTShowQuotasQuery.h +++ /dev/null @@ -1,24 +0,0 @@ -#pragma once - -#include - - -namespace DB -{ -/** SHOW QUOTAS - * SHOW QUOTA USAGE [CURRENT | ALL] - */ -class ASTShowQuotasQuery : public ASTQueryWithOutput -{ -public: - bool usage = false; - bool current = false; - - String getID(char) const override; - ASTPtr clone() const override; - -protected: - void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; -}; - -} diff --git a/src/Parsers/ParserCreateQuotaQuery.cpp b/src/Parsers/ParserCreateQuotaQuery.cpp index 4d29a135131..b505fd25a95 100644 --- a/src/Parsers/ParserCreateQuotaQuery.cpp +++ b/src/Parsers/ParserCreateQuotaQuery.cpp @@ -22,9 +22,12 @@ namespace ErrorCodes namespace { using KeyType = Quota::KeyType; + using KeyTypeInfo = Quota::KeyTypeInfo; using ResourceType = Quota::ResourceType; + using ResourceTypeInfo = Quota::ResourceTypeInfo; using ResourceAmount = Quota::ResourceAmount; + bool parseRenameTo(IParserBase::Pos & pos, Expected & expected, String & new_name) { return IParserBase::wrapParseImpl(pos, [&] @@ -49,7 +52,7 @@ namespace const String & key_type_str = key_type_ast->as().value.safeGet(); for (auto kt : ext::range(Quota::KeyType::MAX)) - if (boost::iequals(Quota::getNameOfKeyType(kt), key_type_str)) + if (boost::iequals(KeyTypeInfo::get(kt).name, key_type_str)) { key_type = kt; return true; @@ -57,7 +60,7 @@ namespace String all_key_types_str; for (auto kt : ext::range(Quota::KeyType::MAX)) - all_key_types_str += String(all_key_types_str.empty() ? "" : ", ") + "'" + Quota::getNameOfKeyType(kt) + "'"; + all_key_types_str += String(all_key_types_str.empty() ? "" : ", ") + "'" + KeyTypeInfo::get(kt).name + "'"; String msg = "Quota cannot be keyed by '" + key_type_str + "'. Expected one of these literals: " + all_key_types_str; throw Exception(msg, ErrorCodes::SYNTAX_ERROR); }); @@ -80,31 +83,35 @@ namespace ParserKeyword{"MAX"}.ignore(pos, expected); } - bool resource_type_set = false; + std::optional res_resource_type; for (auto rt : ext::range(Quota::MAX_RESOURCE_TYPE)) { - if (ParserKeyword{Quota::resourceTypeToKeyword(rt)}.ignore(pos, expected)) + if (ParserKeyword{ResourceTypeInfo::get(rt).keyword.c_str()}.ignore(pos, expected)) { - resource_type = rt; - resource_type_set = true; + res_resource_type = rt; break; } } - if (!resource_type_set) + if (!res_resource_type) return false; + ResourceAmount res_max; ASTPtr max_ast; if (ParserNumber{}.parse(pos, max_ast, expected)) { const Field & max_field = max_ast->as().value; - if (resource_type == Quota::EXECUTION_TIME) - max = Quota::secondsToExecutionTime(applyVisitor(FieldVisitorConvertToNumber(), max_field)); + const auto & type_info = ResourceTypeInfo::get(*res_resource_type); + if (type_info.output_denominator == 1) + res_max = applyVisitor(FieldVisitorConvertToNumber(), max_field); else - max = applyVisitor(FieldVisitorConvertToNumber(), max_field); + res_max = static_cast( + applyVisitor(FieldVisitorConvertToNumber(), max_field) * type_info.output_denominator); } else return false; + resource_type = *res_resource_type; + max = res_max; return true; }); } diff --git a/src/Parsers/ParserQueryWithOutput.cpp b/src/Parsers/ParserQueryWithOutput.cpp index 93329723c2b..83967ae9922 100644 --- a/src/Parsers/ParserQueryWithOutput.cpp +++ b/src/Parsers/ParserQueryWithOutput.cpp @@ -17,7 +17,6 @@ #include #include #include -#include namespace DB @@ -41,7 +40,6 @@ bool ParserQueryWithOutput::parseImpl(Pos & pos, ASTPtr & node, Expected & expec ParserShowAccessEntitiesQuery show_access_entities_p; ParserShowCreateAccessEntityQuery show_create_access_entity_p; ParserShowGrantsQuery show_grants_p; - ParserShowQuotasQuery show_quotas_p; ASTPtr query; @@ -71,8 +69,7 @@ bool ParserQueryWithOutput::parseImpl(Pos & pos, ASTPtr & node, Expected & expec || optimize_p.parse(pos, query, expected) || watch_p.parse(pos, query, expected) || show_access_entities_p.parse(pos, query, expected) - || show_grants_p.parse(pos, query, expected) - || show_quotas_p.parse(pos, query, expected); + || show_grants_p.parse(pos, query, expected); if (!parsed) return false; diff --git a/src/Parsers/ParserShowAccessEntitiesQuery.cpp b/src/Parsers/ParserShowAccessEntitiesQuery.cpp index cd16633c434..7ca9782dd2c 100644 --- a/src/Parsers/ParserShowAccessEntitiesQuery.cpp +++ b/src/Parsers/ParserShowAccessEntitiesQuery.cpp @@ -24,16 +24,37 @@ namespace bool ParserShowAccessEntitiesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { - if (!ParserKeyword{"SHOW POLICIES"}.ignore(pos, expected) && !ParserKeyword{"SHOW ROW POLICIES"}.ignore(pos, expected)) + if (!ParserKeyword{"SHOW"}.ignore(pos, expected)) + return false; + + std::optional type; + bool current_quota = false; + + if (ParserKeyword{"POLICIES"}.ignore(pos, expected) || ParserKeyword{"ROW POLICIES"}.ignore(pos, expected)) + { + type = EntityType::ROW_POLICY; + } + else if (ParserKeyword{"QUOTAS"}.ignore(pos, expected)) + { + type = EntityType::QUOTA; + } + else if (ParserKeyword{"QUOTA"}.ignore(pos, expected) || ParserKeyword{"CURRENT QUOTA"}.ignore(pos, expected)) + { + type = EntityType::QUOTA; + current_quota = true; + } + else return false; String database, table_name; - parseONDatabaseAndTableName(pos, expected, database, table_name); + if (type == EntityType::ROW_POLICY) + parseONDatabaseAndTableName(pos, expected, database, table_name); auto query = std::make_shared(); node = query; - query->type = EntityType::ROW_POLICY; + query->type = *type; + query->current_quota = current_quota; query->database = std::move(database); query->table_name = std::move(table_name); diff --git a/src/Parsers/ParserShowAccessEntitiesQuery.h b/src/Parsers/ParserShowAccessEntitiesQuery.h index 05b9550cdf8..77838e726e7 100644 --- a/src/Parsers/ParserShowAccessEntitiesQuery.h +++ b/src/Parsers/ParserShowAccessEntitiesQuery.h @@ -7,6 +7,8 @@ namespace DB { /** Parses queries like * SHOW [ROW] POLICIES [ON [database.]table] + SHOW QUOTAS + SHOW [CURRENT] QUOTA */ class ParserShowAccessEntitiesQuery : public IParserBase { diff --git a/src/Parsers/ParserShowCreateAccessEntityQuery.cpp b/src/Parsers/ParserShowCreateAccessEntityQuery.cpp index 308a1bd7795..ca520f4df6f 100644 --- a/src/Parsers/ParserShowCreateAccessEntityQuery.cpp +++ b/src/Parsers/ParserShowCreateAccessEntityQuery.cpp @@ -58,16 +58,7 @@ bool ParserShowCreateAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expe } else if (type == EntityType::QUOTA) { - if (ParserKeyword{"CURRENT"}.ignore(pos, expected)) - { - /// SHOW CREATE QUOTA CURRENT - current_quota = true; - } - else if (parseIdentifierOrStringLiteral(pos, expected, name)) - { - /// SHOW CREATE QUOTA name - } - else + if (!parseIdentifierOrStringLiteral(pos, expected, name)) { /// SHOW CREATE QUOTA current_quota = true; diff --git a/src/Parsers/ParserShowCreateAccessEntityQuery.h b/src/Parsers/ParserShowCreateAccessEntityQuery.h index 4572b54de27..025949d7fca 100644 --- a/src/Parsers/ParserShowCreateAccessEntityQuery.h +++ b/src/Parsers/ParserShowCreateAccessEntityQuery.h @@ -6,7 +6,7 @@ namespace DB { /** Parses queries like - * SHOW CREATE QUOTA [name | CURRENT] + * SHOW CREATE QUOTA [name] */ class ParserShowCreateAccessEntityQuery : public IParserBase { diff --git a/src/Parsers/ParserShowQuotasQuery.cpp b/src/Parsers/ParserShowQuotasQuery.cpp deleted file mode 100644 index 69cbd352969..00000000000 --- a/src/Parsers/ParserShowQuotasQuery.cpp +++ /dev/null @@ -1,42 +0,0 @@ -#include -#include -#include -#include -#include -#include - - -namespace DB -{ -bool ParserShowQuotasQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) -{ - bool usage; - bool current; - if (ParserKeyword{"SHOW QUOTAS"}.ignore(pos, expected)) - { - usage = false; - current = false; - } - else if (ParserKeyword{"SHOW QUOTA USAGE"}.ignore(pos, expected)) - { - usage = true; - if (ParserKeyword{"ALL"}.ignore(pos, expected)) - { - current = false; - } - else - { - ParserKeyword{"CURRENT"}.ignore(pos, expected); - current = true; - } - } - else - return false; - - auto query = std::make_shared(); - query->usage = usage; - query->current = current; - node = query; - return true; -} -} diff --git a/src/Parsers/ParserShowQuotasQuery.h b/src/Parsers/ParserShowQuotasQuery.h deleted file mode 100644 index 5b00b525f98..00000000000 --- a/src/Parsers/ParserShowQuotasQuery.h +++ /dev/null @@ -1,18 +0,0 @@ -#pragma once - -#include - - -namespace DB -{ -/** Parses queries like - * SHOW QUOTAS - * SHOW QUOTA USAGE [CURRENT | ALL] - */ -class ParserShowQuotasQuery : public IParserBase -{ -protected: - const char * getName() const override { return "SHOW QUOTA query"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; -}; -} diff --git a/src/Parsers/ya.make b/src/Parsers/ya.make index 2490bcf24c4..2862b02a2c7 100644 --- a/src/Parsers/ya.make +++ b/src/Parsers/ya.make @@ -45,7 +45,6 @@ SRCS( ASTShowAccessEntitiesQuery.cpp ASTShowCreateAccessEntityQuery.cpp ASTShowGrantsQuery.cpp - ASTShowQuotasQuery.cpp ASTShowTablesQuery.cpp ASTSubquery.cpp ASTSystemQuery.cpp @@ -97,7 +96,6 @@ SRCS( ParserShowAccessEntitiesQuery.cpp ParserShowCreateAccessEntityQuery.cpp ParserShowGrantsQuery.cpp - ParserShowQuotasQuery.cpp ParserShowTablesQuery.cpp ParserSystemQuery.cpp ParserTablePropertiesQuery.cpp diff --git a/src/Storages/System/StorageSystemQuotaLimits.cpp b/src/Storages/System/StorageSystemQuotaLimits.cpp new file mode 100644 index 00000000000..fcfe11a0d23 --- /dev/null +++ b/src/Storages/System/StorageSystemQuotaLimits.cpp @@ -0,0 +1,122 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +using ResourceAmount = Quota::ResourceAmount; +using ResourceType = Quota::ResourceType; +using ResourceTypeInfo = Quota::ResourceTypeInfo; +constexpr auto MAX_RESOURCE_TYPE = Quota::MAX_RESOURCE_TYPE; + + +namespace +{ + void addValue(IColumn & out_column, NullMap & out_column_null_map, ResourceAmount amount, const ResourceTypeInfo & type_info) + { + out_column_null_map.push_back(false); + if (type_info.output_as_float) + static_cast(out_column).getData().push_back(double(amount) / type_info.output_denominator); + else + static_cast(out_column).getData().push_back(amount / type_info.output_denominator); + } + + void addValue(IColumn & out_column, NullMap & out_column_null_map, std::optional amount, const ResourceTypeInfo & type_info) + { + if (amount) + addValue(out_column, out_column_null_map, *amount, type_info); + else + { + out_column_null_map.push_back(true); + out_column.insertDefault(); + } + } +} + + +NamesAndTypesList StorageSystemQuotaLimits::getNamesAndTypes() +{ + NamesAndTypesList names_and_types{ + {"quota_name", std::make_shared()}, + {"duration", std::make_shared()}, + {"is_randomized_interval", std::make_shared()}, + }; + + for (auto resource_type : ext::range(MAX_RESOURCE_TYPE)) + { + const auto & type_info = ResourceTypeInfo::get(resource_type); + String column_name = "max_" + type_info.name; + DataTypePtr data_type; + if (type_info.output_as_float) + data_type = std::make_shared(); + else + data_type = std::make_shared(); + names_and_types.push_back({column_name, std::make_shared(data_type)}); + } + + return names_and_types; +} + + +void StorageSystemQuotaLimits::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const +{ + context.checkAccess(AccessType::SHOW_QUOTAS); + const auto & access_control = context.getAccessControlManager(); + std::vector ids = access_control.findAll(); + + size_t column_index = 0; + auto & column_quota_name = assert_cast(*res_columns[column_index++]); + auto & column_duration = assert_cast(*res_columns[column_index++]).getData(); + auto & column_is_randomized_interval = assert_cast(*res_columns[column_index++]).getData(); + + IColumn * column_max[MAX_RESOURCE_TYPE]; + NullMap * column_max_null_map[MAX_RESOURCE_TYPE]; + for (auto resource_type : ext::range(MAX_RESOURCE_TYPE)) + { + column_max[resource_type] = &assert_cast(*res_columns[column_index]).getNestedColumn(); + column_max_null_map[resource_type] = &assert_cast(*res_columns[column_index++]).getNullMapData(); + } + + auto add_row = [&](const String & quota_name, const Quota::Limits & limits) + { + column_quota_name.insertData(quota_name.data(), quota_name.length()); + column_duration.push_back(limits.duration.count()); + column_is_randomized_interval.push_back(limits.randomize_interval); + + for (auto resource_type : ext::range(MAX_RESOURCE_TYPE)) + { + const auto & type_info = ResourceTypeInfo::get(resource_type); + addValue(*column_max[resource_type], *column_max_null_map[resource_type], limits.max[resource_type], type_info); + } + }; + + auto add_rows = [&](const String & quota_name, const std::vector & all_limits) + { + for (const auto & limits : all_limits) + add_row(quota_name, limits); + }; + + for (const auto & id : ids) + { + auto quota = access_control.tryRead(id); + if (!quota) + continue; + const auto * storage = access_control.findStorage(id); + if (!storage) + continue; + + add_rows(quota->getName(), quota->all_limits); + } +} +} diff --git a/src/Storages/System/StorageSystemQuotaLimits.h b/src/Storages/System/StorageSystemQuotaLimits.h new file mode 100644 index 00000000000..e9ae7fc09d0 --- /dev/null +++ b/src/Storages/System/StorageSystemQuotaLimits.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class Context; + +/// Implements `quota_limits` system table, which allows you to get information about the limits set for quotas. +class StorageSystemQuotaLimits final : public ext::shared_ptr_helper, public IStorageSystemOneBlock +{ +public: + std::string getName() const override { return "SystemQuotaLimits"; } + static NamesAndTypesList getNamesAndTypes(); + +protected: + friend struct ext::shared_ptr_helper; + using IStorageSystemOneBlock::IStorageSystemOneBlock; + void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const override; +}; + +} diff --git a/src/Storages/System/StorageSystemQuotaUsage.cpp b/src/Storages/System/StorageSystemQuotaUsage.cpp index 27400329b27..002ab081bcf 100644 --- a/src/Storages/System/StorageSystemQuotaUsage.cpp +++ b/src/Storages/System/StorageSystemQuotaUsage.cpp @@ -1,40 +1,82 @@ #include #include #include -#include #include #include +#include +#include +#include #include #include -#include -#include +#include #include #include namespace DB { +using ResourceAmount = Quota::ResourceAmount; +using ResourceType = Quota::ResourceType; +using ResourceTypeInfo = Quota::ResourceTypeInfo; +constexpr auto MAX_RESOURCE_TYPE = Quota::MAX_RESOURCE_TYPE; + + +namespace +{ + void addValue(IColumn & out_column, NullMap & out_column_null_map, ResourceAmount amount, const ResourceTypeInfo & type_info) + { + out_column_null_map.push_back(false); + if (type_info.output_as_float) + static_cast(out_column).getData().push_back(double(amount) / type_info.output_denominator); + else + static_cast(out_column).getData().push_back(amount / type_info.output_denominator); + } + + void addValue(IColumn & out_column, NullMap & out_column_null_map, std::optional amount, const ResourceTypeInfo & type_info) + { + if (amount) + addValue(out_column, out_column_null_map, *amount, type_info); + else + { + out_column_null_map.push_back(true); + out_column.insertDefault(); + } + } +} + + NamesAndTypesList StorageSystemQuotaUsage::getNamesAndTypes() { - NamesAndTypesList names_and_types{ - {"name", std::make_shared()}, - {"id", std::make_shared()}, - {"key", std::make_shared()}, - {"duration", std::make_shared(std::make_shared())}, - {"end_of_interval", std::make_shared(std::make_shared())}}; + return getNamesAndTypesImpl(/* add_column_is_current = */ false); +} - for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) +NamesAndTypesList StorageSystemQuotaUsage::getNamesAndTypesImpl(bool add_column_is_current) +{ + NamesAndTypesList names_and_types{ + {"quota_name", std::make_shared()}, + {"quota_key", std::make_shared()} + }; + + if (add_column_is_current) + names_and_types.push_back({"is_current", std::make_shared()}); + + names_and_types.push_back({"start_time", std::make_shared(std::make_shared())}); + names_and_types.push_back({"end_time", std::make_shared(std::make_shared())}); + names_and_types.push_back({"duration", std::make_shared(std::make_shared())}); + + for (auto resource_type : ext::range(MAX_RESOURCE_TYPE)) { + const auto & type_info = ResourceTypeInfo::get(resource_type); + String column_name = type_info.name; DataTypePtr data_type; - if (resource_type == Quota::EXECUTION_TIME) + if (type_info.output_as_float) data_type = std::make_shared(); else data_type = std::make_shared(); - - String column_name = Quota::resourceTypeToColumnName(resource_type); names_and_types.push_back({column_name, std::make_shared(data_type)}); names_and_types.push_back({String("max_") + column_name, std::make_shared(data_type)}); } + return names_and_types; } @@ -42,42 +84,111 @@ NamesAndTypesList StorageSystemQuotaUsage::getNamesAndTypes() void StorageSystemQuotaUsage::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const { context.checkAccess(AccessType::SHOW_QUOTAS); - const auto & access_control = context.getAccessControlManager(); + auto usage = context.getQuotaUsage(); + if (!usage) + return; - for (const auto & info : access_control.getQuotaUsageInfo()) + fillDataImpl(res_columns, context, /* add_column_is_current = */ false, {std::move(usage).value()}); +} + + +void StorageSystemQuotaUsage::fillDataImpl( + MutableColumns & res_columns, + const Context & context, + bool add_column_is_current, + const std::vector & quotas_usage) +{ + size_t column_index = 0; + auto & column_quota_name = assert_cast(*res_columns[column_index++]); + auto & column_quota_key = assert_cast(*res_columns[column_index++]); + + ColumnUInt8::Container * column_is_current = nullptr; + if (add_column_is_current) + column_is_current = &assert_cast(*res_columns[column_index++]).getData(); + + auto & column_start_time = assert_cast(assert_cast(*res_columns[column_index]).getNestedColumn()); + auto & column_start_time_null_map = assert_cast(*res_columns[column_index++]).getNullMapData(); + auto & column_end_time = assert_cast(assert_cast(*res_columns[column_index]).getNestedColumn()); + auto & column_end_time_null_map = assert_cast(*res_columns[column_index++]).getNullMapData(); + auto & column_duration = assert_cast(assert_cast(*res_columns[column_index]).getNestedColumn()); + auto & column_duration_null_map = assert_cast(*res_columns[column_index++]).getNullMapData(); + + IColumn * column_usage[MAX_RESOURCE_TYPE]; + NullMap * column_usage_null_map[MAX_RESOURCE_TYPE]; + IColumn * column_max[MAX_RESOURCE_TYPE]; + NullMap * column_max_null_map[MAX_RESOURCE_TYPE]; + for (auto resource_type : ext::range(MAX_RESOURCE_TYPE)) { - for (const auto & interval : info.intervals) + column_usage[resource_type] = &assert_cast(*res_columns[column_index]).getNestedColumn(); + column_usage_null_map[resource_type] = &assert_cast(*res_columns[column_index++]).getNullMapData(); + column_max[resource_type] = &assert_cast(*res_columns[column_index]).getNestedColumn(); + column_max_null_map[resource_type] = &assert_cast(*res_columns[column_index++]).getNullMapData(); + } + + std::optional current_quota_id; + if (add_column_is_current) + { + if (auto current_usage = context.getQuotaUsage()) + current_quota_id = current_usage->quota_id; + } + + auto add_row = [&](const String & quota_name, const UUID & quota_id, const String & quota_key, const QuotaUsage::Interval * interval) + { + column_quota_name.insertData(quota_name.data(), quota_name.length()); + column_quota_key.insertData(quota_key.data(), quota_key.length()); + + if (!interval) { - size_t i = 0; - res_columns[i++]->insert(info.quota_name); - res_columns[i++]->insert(info.quota_id); - res_columns[i++]->insert(info.quota_key); - res_columns[i++]->insert(std::chrono::seconds{interval.duration}.count()); - res_columns[i++]->insert(std::chrono::system_clock::to_time_t(interval.end_of_interval)); - for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) + column_start_time.insertDefault(); + column_start_time_null_map.push_back(true); + column_end_time.insertDefault(); + column_end_time_null_map.push_back(true); + column_duration.insertDefault(); + column_duration_null_map.push_back(true); + for (auto resource_type : ext::range(MAX_RESOURCE_TYPE)) { - if (resource_type == Quota::EXECUTION_TIME) - { - res_columns[i++]->insert(Quota::executionTimeToSeconds(interval.used[resource_type])); - res_columns[i++]->insert(Quota::executionTimeToSeconds(interval.max[resource_type])); - } - else - { - res_columns[i++]->insert(interval.used[resource_type]); - res_columns[i++]->insert(interval.max[resource_type]); - } + column_usage[resource_type]->insertDefault(); + column_usage_null_map[resource_type]->push_back(true); + column_max[resource_type]->insertDefault(); + column_max_null_map[resource_type]->push_back(true); } + return; } - if (info.intervals.empty()) + time_t end_time = std::chrono::system_clock::to_time_t(interval->end_of_interval); + UInt32 duration = static_cast(std::chrono::duration_cast(interval->duration).count()); + time_t start_time = end_time - duration; + column_start_time.getData().push_back(start_time); + column_end_time.getData().push_back(end_time); + column_duration.getData().push_back(duration); + column_start_time_null_map.push_back(false); + column_end_time_null_map.push_back(false); + column_duration_null_map.push_back(false); + + for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) { - size_t i = 0; - res_columns[i++]->insert(info.quota_name); - res_columns[i++]->insert(info.quota_id); - res_columns[i++]->insert(info.quota_key); - for (size_t j = 0; j != Quota::MAX_RESOURCE_TYPE * 2 + 2; ++j) - res_columns[i++]->insertDefault(); + const auto & type_info = ResourceTypeInfo::get(resource_type); + addValue(*column_max[resource_type], *column_max_null_map[resource_type], interval->max[resource_type], type_info); + addValue(*column_usage[resource_type], *column_usage_null_map[resource_type], interval->used[resource_type], type_info); } - } + + if (add_column_is_current) + column_is_current->push_back(quota_id == current_quota_id); + }; + + auto add_rows = [&](const String & quota_name, const UUID & quota_id, const String & quota_key, const std::vector & intervals) + { + if (intervals.empty()) + { + add_row(quota_name, quota_id, quota_key, nullptr); + return; + } + + for (const auto & interval : intervals) + add_row(quota_name, quota_id, quota_key, &interval); + }; + + for (const auto & usage : quotas_usage) + add_rows(usage.quota_name, usage.quota_id, usage.quota_key, usage.intervals); } } diff --git a/src/Storages/System/StorageSystemQuotaUsage.h b/src/Storages/System/StorageSystemQuotaUsage.h index d410decf394..abb9505eb5a 100644 --- a/src/Storages/System/StorageSystemQuotaUsage.h +++ b/src/Storages/System/StorageSystemQuotaUsage.h @@ -6,12 +6,12 @@ namespace DB { - class Context; +struct QuotaUsage; -/** Implements the `quota_usage` system tables, which allows you to get information about - * how the quotas are used by all users. +/** Implements the `quota_usage` system table, which allows you to get information about + * how the current user uses the quota. */ class StorageSystemQuotaUsage final : public ext::shared_ptr_helper, public IStorageSystemOneBlock { @@ -19,6 +19,9 @@ public: std::string getName() const override { return "SystemQuotaUsage"; } static NamesAndTypesList getNamesAndTypes(); + static NamesAndTypesList getNamesAndTypesImpl(bool add_column_is_current); + static void fillDataImpl(MutableColumns & res_columns, const Context & context, bool add_column_is_current, const std::vector & quotas_usage); + protected: friend struct ext::shared_ptr_helper; using IStorageSystemOneBlock::IStorageSystemOneBlock; diff --git a/src/Storages/System/StorageSystemQuotas.cpp b/src/Storages/System/StorageSystemQuotas.cpp index 1da7725fe15..a3b687dc011 100644 --- a/src/Storages/System/StorageSystemQuotas.cpp +++ b/src/Storages/System/StorageSystemQuotas.cpp @@ -3,10 +3,12 @@ #include #include #include -#include #include #include +#include +#include #include +#include #include #include #include @@ -17,11 +19,14 @@ namespace DB { namespace { + using KeyType = Quota::KeyType; + using KeyTypeInfo = Quota::KeyTypeInfo; + DataTypeEnum8::Values getKeyTypeEnumValues() { DataTypeEnum8::Values enum_values; - for (auto key_type : ext::range(Quota::KeyType::MAX)) - enum_values.push_back({Quota::getNameOfKeyType(key_type), static_cast(key_type)}); + for (auto key_type : ext::range(KeyType::MAX)) + enum_values.push_back({KeyTypeInfo::get(key_type).name, static_cast(key_type)}); return enum_values; } } @@ -34,21 +39,11 @@ NamesAndTypesList StorageSystemQuotas::getNamesAndTypes() {"id", std::make_shared()}, {"source", std::make_shared()}, {"key_type", std::make_shared(getKeyTypeEnumValues())}, - {"roles", std::make_shared(std::make_shared())}, - {"intervals.duration", std::make_shared(std::make_shared())}, - {"intervals.randomize_interval", std::make_shared(std::make_shared())}}; - - for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) - { - DataTypePtr data_type; - if (resource_type == Quota::EXECUTION_TIME) - data_type = std::make_shared(); - else - data_type = std::make_shared(); - - String column_name = String("intervals.max_") + Quota::resourceTypeToColumnName(resource_type); - names_and_types.push_back({column_name, std::make_shared(data_type)}); - } + {"durations", std::make_shared(std::make_shared())}, + {"apply_to_all", std::make_shared()}, + {"apply_to_list", std::make_shared(std::make_shared())}, + {"apply_to_except", std::make_shared(std::make_shared())} + }; return names_and_types; } @@ -56,61 +51,60 @@ NamesAndTypesList StorageSystemQuotas::getNamesAndTypes() void StorageSystemQuotas::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const { context.checkAccess(AccessType::SHOW_QUOTAS); - - size_t i = 0; - auto & name_column = *res_columns[i++]; - auto & id_column = *res_columns[i++]; - auto & storage_name_column = *res_columns[i++]; - auto & key_type_column = *res_columns[i++]; - auto & roles_data = assert_cast(*res_columns[i]).getData(); - auto & roles_offsets = assert_cast(*res_columns[i++]).getOffsets(); - auto & durations_data = assert_cast(*res_columns[i]).getData(); - auto & durations_offsets = assert_cast(*res_columns[i++]).getOffsets(); - auto & randomize_intervals_data = assert_cast(*res_columns[i]).getData(); - auto & randomize_intervals_offsets = assert_cast(*res_columns[i++]).getOffsets(); - IColumn * limits_data[Quota::MAX_RESOURCE_TYPE]; - ColumnArray::Offsets * limits_offsets[Quota::MAX_RESOURCE_TYPE]; - for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) - { - limits_data[resource_type] = &assert_cast(*res_columns[i]).getData(); - limits_offsets[resource_type] = &assert_cast(*res_columns[i++]).getOffsets(); - } - const auto & access_control = context.getAccessControlManager(); + std::vector ids = access_control.findAll(); + + size_t column_index = 0; + auto & column_name = assert_cast(*res_columns[column_index++]); + auto & column_id = assert_cast(*res_columns[column_index++]).getData(); + auto & column_storage = assert_cast(*res_columns[column_index++]); + auto & column_key_type = assert_cast(*res_columns[column_index++]).getData(); + auto & column_durations = assert_cast(assert_cast(*res_columns[column_index]).getData()).getData(); + auto & column_durations_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); + auto & column_apply_to_all = assert_cast(*res_columns[column_index++]).getData(); + auto & column_apply_to_list = assert_cast(assert_cast(*res_columns[column_index]).getData()); + auto & column_apply_to_list_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); + auto & column_apply_to_except = assert_cast(assert_cast(*res_columns[column_index]).getData()); + auto & column_apply_to_except_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); + + auto add_row = [&](const String & name, + const UUID & id, + const String & storage_name, + const std::vector & all_limits, + KeyType key_type, + const ExtendedRoleSet & apply_to) + { + column_name.insertData(name.data(), name.length()); + column_id.push_back(id); + column_storage.insertData(storage_name.data(), storage_name.length()); + column_key_type.push_back(static_cast(key_type)); + + for (const auto & limits : all_limits) + column_durations.push_back(std::chrono::duration_cast(limits.duration).count()); + column_durations_offsets.push_back(column_durations.size()); + + auto apply_to_ast = apply_to.toASTWithNames(access_control); + column_apply_to_all.push_back(apply_to_ast->all); + + for (const auto & role_name : apply_to_ast->names) + column_apply_to_list.insertData(role_name.data(), role_name.length()); + column_apply_to_list_offsets.push_back(column_apply_to_list.size()); + + for (const auto & role_name : apply_to_ast->except_names) + column_apply_to_except.insertData(role_name.data(), role_name.length()); + column_apply_to_except_offsets.push_back(column_apply_to_except.size()); + }; + for (const auto & id : access_control.findAll()) { auto quota = access_control.tryRead(id); if (!quota) continue; const auto * storage = access_control.findStorage(id); - String storage_name = storage ? storage->getStorageName() : ""; + if (!storage) + continue; - name_column.insert(quota->getName()); - id_column.insert(id); - storage_name_column.insert(storage_name); - key_type_column.insert(static_cast(quota->key_type)); - - for (const String & role : quota->to_roles.toStringsWithNames(access_control)) - roles_data.insert(role); - roles_offsets.push_back(roles_data.size()); - - for (const auto & limits : quota->all_limits) - { - durations_data.insert(std::chrono::seconds{limits.duration}.count()); - randomize_intervals_data.insert(static_cast(limits.randomize_interval)); - for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) - { - if (resource_type == Quota::EXECUTION_TIME) - limits_data[resource_type]->insert(Quota::executionTimeToSeconds(limits.max[resource_type])); - else - limits_data[resource_type]->insert(limits.max[resource_type]); - } - } - - durations_offsets.push_back(durations_data.size()); - randomize_intervals_offsets.push_back(randomize_intervals_data.size()); - for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) - limits_offsets[resource_type]->push_back(limits_data[resource_type]->size()); + add_row(quota->getName(), id, storage->getStorageName(), quota->all_limits, quota->key_type, quota->to_roles); } } } diff --git a/src/Storages/System/StorageSystemQuotas.h b/src/Storages/System/StorageSystemQuotas.h index dda188c7ab7..8d1da53d641 100644 --- a/src/Storages/System/StorageSystemQuotas.h +++ b/src/Storages/System/StorageSystemQuotas.h @@ -6,10 +6,8 @@ namespace DB { - class Context; - /** Implements the `quotas` system tables, which allows you to get information about quotas. */ class StorageSystemQuotas final : public ext::shared_ptr_helper, public IStorageSystemOneBlock diff --git a/src/Storages/System/StorageSystemQuotasUsage.cpp b/src/Storages/System/StorageSystemQuotasUsage.cpp new file mode 100644 index 00000000000..5c6879cd143 --- /dev/null +++ b/src/Storages/System/StorageSystemQuotasUsage.cpp @@ -0,0 +1,22 @@ +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +NamesAndTypesList StorageSystemQuotasUsage::getNamesAndTypes() +{ + return StorageSystemQuotaUsage::getNamesAndTypesImpl(/* add_column_is_current = */ true); +} + +void StorageSystemQuotasUsage::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const +{ + context.checkAccess(AccessType::SHOW_QUOTAS); + auto all_quotas_usage = context.getAccessControlManager().getAllQuotasUsage(); + StorageSystemQuotaUsage::fillDataImpl(res_columns, context, /* add_column_is_current = */ true, all_quotas_usage); +} +} diff --git a/src/Storages/System/StorageSystemQuotasUsage.h b/src/Storages/System/StorageSystemQuotasUsage.h new file mode 100644 index 00000000000..d4fd93b577d --- /dev/null +++ b/src/Storages/System/StorageSystemQuotasUsage.h @@ -0,0 +1,26 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class Context; + +/** Implements the `quotas_usage` system table, which allows you to get information about + * how all users use the quotas. + */ +class StorageSystemQuotasUsage final : public ext::shared_ptr_helper, public IStorageSystemOneBlock +{ +public: + std::string getName() const override { return "SystemQuotasUsage"; } + static NamesAndTypesList getNamesAndTypes(); + +protected: + friend struct ext::shared_ptr_helper; + using IStorageSystemOneBlock::IStorageSystemOneBlock; + void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const override; +}; + +} diff --git a/src/Storages/System/attachSystemTables.cpp b/src/Storages/System/attachSystemTables.cpp index 4c9f34b82e1..69ad147664f 100644 --- a/src/Storages/System/attachSystemTables.cpp +++ b/src/Storages/System/attachSystemTables.cpp @@ -26,7 +26,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -67,7 +69,9 @@ void attachSystemTablesLocal(IDatabase & system_database) system_database.attachTable("events", StorageSystemEvents::create("events")); system_database.attachTable("settings", StorageSystemSettings::create("settings")); system_database.attachTable("quotas", StorageSystemQuotas::create("quotas")); + system_database.attachTable("quota_limits", StorageSystemQuotaLimits::create("quota_limits")); system_database.attachTable("quota_usage", StorageSystemQuotaUsage::create("quota_usage")); + system_database.attachTable("quotas_usage", StorageSystemQuotasUsage::create("all_quotas_usage")); system_database.attachTable("row_policies", StorageSystemRowPolicies::create("row_policies")); system_database.attachTable("merge_tree_settings", SystemMergeTreeSettings::create("merge_tree_settings")); system_database.attachTable("build_options", StorageSystemBuildOptions::create("build_options")); diff --git a/src/Storages/ya.make b/src/Storages/ya.make index 5ed6fcbf637..2e00dbaec97 100644 --- a/src/Storages/ya.make +++ b/src/Storages/ya.make @@ -111,8 +111,10 @@ SRCS( System/StorageSystemPartsBase.cpp System/StorageSystemPartsColumns.cpp System/StorageSystemProcesses.cpp - System/StorageSystemQuotas.cpp + System/StorageSystemQuotaLimits.cpp System/StorageSystemQuotaUsage.cpp + System/StorageSystemQuotas.cpp + System/StorageSystemQuotasUsage.cpp System/StorageSystemReplicas.cpp System/StorageSystemReplicationQueue.cpp System/StorageSystemRowPolicies.cpp diff --git a/tests/integration/test_quota/test.py b/tests/integration/test_quota/test.py index ae68a34a03e..6e00bf7241b 100644 --- a/tests/integration/test_quota/test.py +++ b/tests/integration/test_quota/test.py @@ -1,6 +1,6 @@ import pytest from helpers.cluster import ClickHouseCluster -from helpers.test_tools import assert_eq_with_retry +from helpers.test_tools import assert_eq_with_retry, TSV import os import re import time @@ -9,17 +9,24 @@ cluster = ClickHouseCluster(__file__) instance = cluster.add_instance('instance', config_dir="configs") -query_from_system_quotas = "SELECT * FROM system.quotas ORDER BY name"; - -query_from_system_quota_usage = "SELECT id, key, duration, "\ - "queries, errors, result_rows, result_bytes, read_rows, read_bytes "\ - "FROM system.quota_usage ORDER BY id, key, duration"; def system_quotas(): - return instance.query(query_from_system_quotas).rstrip('\n') + return TSV(instance.query("SELECT * FROM system.quotas ORDER BY name")) + +def system_quota_limits(): + return TSV(instance.query("SELECT * FROM system.quota_limits ORDER BY quota_name, duration")) def system_quota_usage(): - return instance.query(query_from_system_quota_usage).rstrip('\n') + query = "SELECT quota_name, quota_key, duration, queries, max_queries, errors, max_errors, result_rows, max_result_rows,"\ + "result_bytes, max_result_bytes, read_rows, max_read_rows, read_bytes, max_read_bytes, max_execution_time "\ + "FROM system.quota_usage ORDER BY duration" + return TSV(instance.query(query)) + +def system_quotas_usage(): + query = "SELECT quota_name, quota_key, is_current, duration, queries, max_queries, errors, max_errors, result_rows, max_result_rows, "\ + "result_bytes, max_result_bytes, read_rows, max_read_rows, read_bytes, max_read_bytes, max_execution_time "\ + "FROM system.quotas_usage ORDER BY quota_name, quota_key, duration" + return TSV(instance.query(query)) def copy_quota_xml(local_file_name, reload_immediately = True): @@ -54,197 +61,225 @@ def reset_quotas_and_usage_info(): def test_quota_from_users_xml(): - assert instance.query("SELECT currentQuota()") == "myQuota\n" - assert instance.query("SELECT currentQuotaID()") == "e651da9c-a748-8703-061a-7e5e5096dae7\n" - assert instance.query("SELECT currentQuotaKey()") == "default\n" - assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]" - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t0\t0\t0\t0\t0\t0" + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", [31556952], 0, "['default']", "[]"]] + assert system_quota_limits() == [["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"]] + assert system_quota_usage() == [["myQuota", "default", 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"]] + assert system_quotas_usage() == [["myQuota", "default", 1, 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"]] instance.query("SELECT * from test_table") - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t1\t0\t50\t200\t50\t200" + assert system_quota_usage() == [["myQuota", "default", 31556952, 1, 1000, 0, "\N", 50, "\N", 200, "\N", 50, 1000, 200, "\N", "\N"]] instance.query("SELECT COUNT() from test_table") - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t2\t0\t51\t208\t50\t200" + assert system_quota_usage() == [["myQuota", "default", 31556952, 2, 1000, 0, "\N", 51, "\N", 208, "\N", 50, 1000, 200, "\N", "\N"]] def test_simpliest_quota(): # Simpliest quota doesn't even track usage. copy_quota_xml('simpliest.xml') - assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t[]\t[]\t[]\t[]\t[]\t[]\t[]\t[]\t[]" - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N" + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", "[]", 0, "['default']", "[]"]] + assert system_quota_limits() == "" + assert system_quota_usage() == [["myQuota", "default", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N"]] instance.query("SELECT * from test_table") - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N" + assert system_quota_usage() == [["myQuota", "default", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N"]] def test_tracking_quota(): # Now we're tracking usage. copy_quota_xml('tracking.xml') - assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t[31556952]\t[0]\t[0]\t[0]\t[0]\t[0]\t[0]\t[0]\t[0]" - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t0\t0\t0\t0\t0\t0" + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", "[31556952]", 0, "['default']", "[]"]] + assert system_quota_limits() == [["myQuota", 31556952, 0, "\N", "\N", "\N", "\N", "\N", "\N", "\N"]] + assert system_quota_usage() == [["myQuota", "default", 31556952, 0, "\N", 0, "\N", 0, "\N", 0, "\N", 0, "\N", 0, "\N", "\N"]] instance.query("SELECT * from test_table") - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t1\t0\t50\t200\t50\t200" + assert system_quota_usage() == [["myQuota", "default", 31556952, 1, "\N", 0, "\N", 50, "\N", 200, "\N", 50, "\N", 200, "\N", "\N"]] instance.query("SELECT COUNT() from test_table") - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t2\t0\t51\t208\t50\t200" + assert system_quota_usage() == [["myQuota", "default", 31556952, 2, "\N", 0, "\N", 51, "\N", 208, "\N", 50, "\N", 200, "\N", "\N"]] def test_exceed_quota(): # Change quota, now the limits are tiny so we will exceed the quota. copy_quota_xml('tiny_limits.xml') - assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t[31556952]\t[0]\t[1]\t[1]\t[1]\t[0]\t[1]\t[0]\t[0]" - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t0\t0\t0\t0\t0\t0" + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", "[31556952]", 0, "['default']", "[]"]] + assert system_quota_limits() == [["myQuota", 31556952, 0, 1, 1, 1, "\N", 1, "\N", "\N"]] + assert system_quota_usage() == [["myQuota", "default", 31556952, 0, 1, 0, 1, 0, 1, 0, "\N", 0, 1, 0, "\N", "\N"]] assert re.search("Quota.*has\ been\ exceeded", instance.query_and_get_error("SELECT * from test_table")) - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t1\t1\t0\t0\t50\t0" + assert system_quota_usage() == [["myQuota", "default", 31556952, 1, 1, 1, 1, 0, 1, 0, "\N", 50, 1, 0, "\N", "\N"]] # Change quota, now the limits are enough to execute queries. copy_quota_xml('normal_limits.xml') - assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]" - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t1\t1\t0\t0\t50\t0" + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", "[31556952]", 0, "['default']", "[]"]] + assert system_quota_limits() == [["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"]] + assert system_quota_usage() == [["myQuota", "default", 31556952, 1, 1000, 1, "\N", 0, "\N", 0, "\N", 50, 1000, 0, "\N", "\N"]] instance.query("SELECT * from test_table") - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t2\t1\t50\t200\t100\t200" + assert system_quota_usage() == [["myQuota", "default", 31556952, 2, 1000, 1, "\N", 50, "\N", 200, "\N", 100, 1000, 200, "\N", "\N"]] def test_add_remove_interval(): - assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]" - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t0\t0\t0\t0\t0\t0" + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", [31556952], 0, "['default']", "[]"]] + assert system_quota_limits() == [["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"]] + assert system_quota_usage() == [["myQuota", "default", 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"]] # Add interval. copy_quota_xml('two_intervals.xml') - assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t[31556952,63113904]\t[0,1]\t[1000,0]\t[0,0]\t[0,0]\t[0,30000]\t[1000,0]\t[0,20000]\t[0,120]" - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t0\t0\t0\t0\t0\t0\n"\ - "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t63113904\t0\t0\t0\t0\t0\t0" + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", "[31556952,63113904]", 0, "['default']", "[]"]] + assert system_quota_limits() == [["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"], + ["myQuota", 63113904, 1, "\N", "\N", "\N", 30000, "\N", 20000, 120]] + assert system_quota_usage() == [["myQuota", "default", 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"], + ["myQuota", "default", 63113904, 0, "\N", 0, "\N", 0, "\N", 0, 30000, 0, "\N", 0, 20000, 120]] instance.query("SELECT * from test_table") - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t1\t0\t50\t200\t50\t200\n"\ - "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t63113904\t1\t0\t50\t200\t50\t200" + assert system_quota_usage() == [["myQuota", "default", 31556952, 1, 1000, 0, "\N", 50, "\N", 200, "\N", 50, 1000, 200, "\N", "\N"], + ["myQuota", "default", 63113904, 1, "\N", 0, "\N", 50, "\N", 200, 30000, 50, "\N", 200, 20000, 120]] # Remove interval. copy_quota_xml('normal_limits.xml') - assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]" - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t1\t0\t50\t200\t50\t200" + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", [31556952], 0, "['default']", "[]"]] + assert system_quota_limits() == [["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"]] + assert system_quota_usage() == [["myQuota", "default", 31556952, 1, 1000, 0, "\N", 50, "\N", 200, "\N", 50, 1000, 200, "\N", "\N"]] instance.query("SELECT * from test_table") - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t2\t0\t100\t400\t100\t400" + assert system_quota_usage() == [["myQuota", "default", 31556952, 2, 1000, 0, "\N", 100, "\N", 400, "\N", 100, 1000, 400, "\N", "\N"]] # Remove all intervals. copy_quota_xml('simpliest.xml') - assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t[]\t[]\t[]\t[]\t[]\t[]\t[]\t[]\t[]" - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N" + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", "[]", 0, "['default']", "[]"]] + assert system_quota_limits() == "" + assert system_quota_usage() == [["myQuota", "default", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N"]] instance.query("SELECT * from test_table") - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N" + assert system_quota_usage() == [["myQuota", "default", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N"]] # Add one interval back. copy_quota_xml('normal_limits.xml') - assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]" - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t0\t0\t0\t0\t0\t0" + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", [31556952], 0, "['default']", "[]"]] + assert system_quota_limits() == [["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"]] + assert system_quota_usage() == [["myQuota", "default", 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"]] def test_add_remove_quota(): - assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]" - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t0\t0\t0\t0\t0\t0" + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", [31556952], 0, "['default']", "[]"]] + assert system_quota_limits() == [["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"]] + assert system_quotas_usage() == [["myQuota", "default", 1, 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"]] # Add quota. copy_quota_xml('two_quotas.xml') - assert system_quotas() ==\ - "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]\n"\ - "myQuota2\t4590510c-4d13-bf21-ec8a-c2187b092e73\tusers.xml\tclient key or user name\t[]\t[3600,2629746]\t[1,0]\t[0,0]\t[0,0]\t[4000,0]\t[400000,0]\t[4000,0]\t[400000,0]\t[60,1800]" + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", "[31556952]", 0, "['default']", "[]"], + ["myQuota2", "4590510c-4d13-bf21-ec8a-c2187b092e73", "users.xml", "client key or user name", "[3600,2629746]", 0, "[]", "[]"]] + assert system_quota_limits() == [["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"], + ["myQuota2", 3600, 1, "\N", "\N", 4000, 400000, 4000, 400000, 60], + ["myQuota2", 2629746, 0, "\N", "\N", "\N", "\N", "\N", "\N", 1800]] + assert system_quotas_usage() == [["myQuota", "default", 1, 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"]] # Drop quota. copy_quota_xml('normal_limits.xml') - assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]" + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", "[31556952]", 0, "['default']", "[]"]] + assert system_quota_limits() == [["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"]] + assert system_quotas_usage() == [["myQuota", "default", 1, 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"]] # Drop all quotas. copy_quota_xml('no_quotas.xml') assert system_quotas() == "" - assert system_quota_usage() == "" + assert system_quota_limits() == "" + assert system_quotas_usage() == "" # Add one quota back. copy_quota_xml('normal_limits.xml') - assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]" - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t0\t0\t0\t0\t0\t0" + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", "[31556952]", 0, "['default']", "[]"]] + assert system_quota_limits() == [["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"]] + assert system_quotas_usage() == [["myQuota", "default", 1, 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"]] def test_reload_users_xml_by_timer(): - assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]" + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", "[31556952]", 0, "['default']", "[]"]] + assert system_quota_limits() == [["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"]] time.sleep(1) # The modification time of the 'quota.xml' file should be different, # because config files are reload by timer only when the modification time is changed. copy_quota_xml('tiny_limits.xml', reload_immediately=False) - assert_eq_with_retry(instance, query_from_system_quotas, "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t[31556952]\t[0]\t[1]\t[1]\t[1]\t[0]\t[1]\t[0]\t[0]") + assert_eq_with_retry(instance, "SELECT * FROM system.quotas", [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", "[31556952]", 0, "['default']", "[]"]]) + assert_eq_with_retry(instance, "SELECT * FROM system.quota_limits", [["myQuota", 31556952, 0, 1, 1, 1, "\N", 1, "\N", "\N"]]) def test_dcl_introspection(): assert instance.query("SHOW QUOTAS") == "myQuota\n" - assert instance.query("SHOW CREATE QUOTA myQuota") == "CREATE QUOTA myQuota KEYED BY \\'user name\\' FOR INTERVAL 1 YEAR MAX QUERIES 1000, READ ROWS 1000 TO default\n" - expected_usage = "myQuota key=\\\\'default\\\\' interval=\[.*\] queries=0/1000 errors=0 result_rows=0 result_bytes=0 read_rows=0/1000 read_bytes=0 execution_time=0" - assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) - assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE CURRENT")) - assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE ALL")) + assert instance.query("SHOW CREATE QUOTA") == "CREATE QUOTA myQuota KEYED BY \\'user name\\' FOR INTERVAL 1 YEAR MAX QUERIES 1000, READ ROWS 1000 TO default\n" + assert re.match("myQuota\\tdefault\\t.*\\t31556952\\t0\\t1000\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t1000\\t0\\t\\\\N\\t.*\\t\\\\N\n", + instance.query("SHOW QUOTA")) instance.query("SELECT * from test_table") - expected_usage = "myQuota key=\\\\'default\\\\' interval=\[.*\] queries=1/1000 errors=0 result_rows=50 result_bytes=200 read_rows=50/1000 read_bytes=200 execution_time=.*" - assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) + assert re.match("myQuota\\tdefault\\t.*\\t31556952\\t1\\t1000\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t1000\\t200\\t\\\\N\\t.*\\t\\\\N\n", + instance.query("SHOW QUOTA")) # Add interval. copy_quota_xml('two_intervals.xml') assert instance.query("SHOW QUOTAS") == "myQuota\n" - assert instance.query("SHOW CREATE QUOTA myQuota") == "CREATE QUOTA myQuota KEYED BY \\'user name\\' FOR INTERVAL 1 YEAR MAX QUERIES 1000, READ ROWS 1000, FOR RANDOMIZED INTERVAL 2 YEAR MAX RESULT BYTES 30000, READ BYTES 20000, EXECUTION TIME 120 TO default\n" - expected_usage = "myQuota key=\\\\'default\\\\' interval=\[.*\] queries=1/1000 errors=0 result_rows=50 result_bytes=200 read_rows=50/1000 read_bytes=200 execution_time=.*\n"\ - "myQuota key=\\\\'default\\\\' interval=\[.*\] queries=0 errors=0 result_rows=0 result_bytes=0/30000 read_rows=0 read_bytes=0/20000 execution_time=0/120" - assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) + assert instance.query("SHOW CREATE QUOTA") == "CREATE QUOTA myQuota KEYED BY \\'user name\\' FOR INTERVAL 1 YEAR MAX QUERIES 1000, READ ROWS 1000, FOR RANDOMIZED INTERVAL 2 YEAR MAX RESULT BYTES 30000, READ BYTES 20000, EXECUTION TIME 120 TO default\n" + assert re.match("myQuota\\tdefault\\t.*\\t31556952\\t1\\t1000\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t1000\\t200\\t\\\\N\\t.*\\t\\\\N\n" + "myQuota\\tdefault\\t.*\\t63113904\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t30000\\t0\\t\\\\N\\t0\\t20000\\t.*\\t120", + instance.query("SHOW QUOTA")) # Drop interval, add quota. copy_quota_xml('two_quotas.xml') assert instance.query("SHOW QUOTAS") == "myQuota\nmyQuota2\n" assert instance.query("SHOW CREATE QUOTA myQuota") == "CREATE QUOTA myQuota KEYED BY \\'user name\\' FOR INTERVAL 1 YEAR MAX QUERIES 1000, READ ROWS 1000 TO default\n" assert instance.query("SHOW CREATE QUOTA myQuota2") == "CREATE QUOTA myQuota2 KEYED BY \\'client key or user name\\' FOR RANDOMIZED INTERVAL 1 HOUR MAX RESULT ROWS 4000, RESULT BYTES 400000, READ ROWS 4000, READ BYTES 400000, EXECUTION TIME 60, FOR INTERVAL 1 MONTH MAX EXECUTION TIME 1800\n" - expected_usage = "myQuota key=\\\\'default\\\\' interval=\[.*\] queries=1/1000 errors=0 result_rows=50 result_bytes=200 read_rows=50/1000 read_bytes=200 execution_time=.*" - assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) + assert re.match("myQuota\\tdefault\\t.*\\t31556952\\t1\\t1000\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t1000\\t200\\t\\\\N\\t.*\\t\\\\N\n", + instance.query("SHOW QUOTA")) + + # Drop all quotas. + copy_quota_xml('no_quotas.xml') + assert instance.query("SHOW QUOTAS") == "" + assert instance.query("SHOW CREATE QUOTA") == "" + assert instance.query("SHOW QUOTA") == "" def test_dcl_management(): copy_quota_xml('no_quotas.xml') - assert instance.query("SHOW QUOTAS") == "" - assert instance.query("SHOW QUOTA USAGE") == "" + assert instance.query("SHOW QUOTA") == "" instance.query("CREATE QUOTA qA FOR INTERVAL 15 MONTH MAX QUERIES 123 TO CURRENT_USER") - assert instance.query("SHOW QUOTAS") == "qA\n" assert instance.query("SHOW CREATE QUOTA qA") == "CREATE QUOTA qA KEYED BY \\'none\\' FOR INTERVAL 5 QUARTER MAX QUERIES 123 TO default\n" - expected_usage = "qA key=\\\\'\\\\' interval=\[.*\] queries=0/123 errors=0 result_rows=0 result_bytes=0 read_rows=0 read_bytes=0 execution_time=.*" - assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) - + assert re.match("qA\\t\\t.*\\t39446190\\t0\\t123\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t.*\\t\\\\N\n", + instance.query("SHOW QUOTA")) + instance.query("SELECT * from test_table") - expected_usage = "qA key=\\\\'\\\\' interval=\[.*\] queries=1/123 errors=0 result_rows=50 result_bytes=200 read_rows=50 read_bytes=200 execution_time=.*" - assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) + assert re.match("qA\\t\\t.*\\t39446190\\t1\\t123\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t.*\\t\\\\N\n", + instance.query("SHOW QUOTA")) instance.query("ALTER QUOTA qA FOR INTERVAL 15 MONTH MAX QUERIES 321, MAX ERRORS 10, FOR INTERVAL 0.5 HOUR MAX EXECUTION TIME 0.5") assert instance.query("SHOW CREATE QUOTA qA") == "CREATE QUOTA qA KEYED BY \\'none\\' FOR INTERVAL 30 MINUTE MAX EXECUTION TIME 0.5, FOR INTERVAL 5 QUARTER MAX QUERIES 321, ERRORS 10 TO default\n" - expected_usage = "qA key=\\\\'\\\\' interval=\[.*\] queries=0 errors=0 result_rows=0 result_bytes=0 read_rows=0 read_bytes=0 execution_time=.*/0.5\n"\ - "qA key=\\\\'\\\\' interval=\[.*\] queries=1/321 errors=0/10 result_rows=50 result_bytes=200 read_rows=50 read_bytes=200 execution_time=.*" - assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) - - instance.query("ALTER QUOTA qA FOR INTERVAL 15 MONTH NO LIMITS, FOR RANDOMIZED INTERVAL 16 MONTH TRACKING ONLY, FOR INTERVAL 1800 SECOND NO LIMITS") - assert instance.query("SHOW CREATE QUOTA qA") == "CREATE QUOTA qA KEYED BY \\'none\\' FOR RANDOMIZED INTERVAL 16 MONTH TRACKING ONLY TO default\n" - expected_usage = "qA key=\\\\'\\\\' interval=\[.*\] queries=0 errors=0 result_rows=0 result_bytes=0 read_rows=0 read_bytes=0 execution_time=.*" - assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) + assert re.match("qA\\t\\t.*\\t1800\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t.*\\t0.5\n" + "qA\\t\\t.*\\t39446190\\t1\\t321\\t0\\t10\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t.*\\t\\\\N\n", + instance.query("SHOW QUOTA")) instance.query("SELECT * from test_table") - expected_usage = "qA key=\\\\'\\\\' interval=\[.*\] queries=1 errors=0 result_rows=50 result_bytes=200 read_rows=50 read_bytes=200 execution_time=.*" - assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) + assert re.match("qA\\t\\t.*\\t1800\\t1\\t\\\\N\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t.*\\t0.5\n" + "qA\\t\\t.*\\t39446190\\t2\\t321\\t0\\t10\\t100\\t\\\\N\\t400\\t\\\\N\\t100\\t\\\\N\\t400\\t\\\\N\\t.*\\t\\\\N\n", + instance.query("SHOW QUOTA")) + + instance.query("ALTER QUOTA qA FOR INTERVAL 15 MONTH NO LIMITS, FOR RANDOMIZED INTERVAL 16 MONTH TRACKING ONLY, FOR INTERVAL 1800 SECOND NO LIMITS") + assert re.match("qA\\t\\t.*\\t42075936\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t.*\\t\\\\N\n", + instance.query("SHOW QUOTA")) + + instance.query("SELECT * from test_table") + assert re.match("qA\\t\\t.*\\t42075936\\t1\\t\\\\N\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t.*\\t\\\\N\n", + instance.query("SHOW QUOTA")) instance.query("ALTER QUOTA qA RENAME TO qB") assert instance.query("SHOW CREATE QUOTA qB") == "CREATE QUOTA qB KEYED BY \\'none\\' FOR RANDOMIZED INTERVAL 16 MONTH TRACKING ONLY TO default\n" - expected_usage = "qB key=\\\\'\\\\' interval=\[.*\] queries=1 errors=0 result_rows=50 result_bytes=200 read_rows=50 read_bytes=200 execution_time=.*" - assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) + assert re.match("qB\\t\\t.*\\t42075936\\t1\\t\\\\N\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t.*\\t\\\\N\n", + instance.query("SHOW QUOTA")) + + instance.query("SELECT * from test_table") + assert re.match("qB\\t\\t.*\\t42075936\\t2\\t\\\\N\\t0\\t\\\\N\\t100\\t\\\\N\\t400\\t\\\\N\\t100\\t\\\\N\\t400\\t\\\\N\\t.*\\t\\\\N\n", + instance.query("SHOW QUOTA")) instance.query("DROP QUOTA qB") - assert instance.query("SHOW QUOTAS") == "" - assert instance.query("SHOW QUOTA USAGE") == "" + assert instance.query("SHOW QUOTA") == "" def test_users_xml_is_readonly(): From ab760846c2e494981b664dbc95e02649962519d4 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 13 May 2020 20:43:30 +0300 Subject: [PATCH 185/738] Fix some rename cases --- src/Core/Names.h | 1 + src/Storages/AlterCommands.cpp | 34 ++++++++++++------- .../MergeTree/MergeTreeDataMergerMutator.cpp | 23 +++++++------ .../MergeTree/MergeTreeDataMergerMutator.h | 3 +- .../0_stateless/01213_alter_rename_column.sql | 2 +- .../01278_alter_rename_combination.reference | 3 ++ .../01278_alter_rename_combination.sql | 14 ++++++++ 7 files changed, 53 insertions(+), 27 deletions(-) create mode 100644 tests/queries/0_stateless/01278_alter_rename_combination.reference create mode 100644 tests/queries/0_stateless/01278_alter_rename_combination.sql diff --git a/src/Core/Names.h b/src/Core/Names.h index 5489a233b6e..3281daa560e 100644 --- a/src/Core/Names.h +++ b/src/Core/Names.h @@ -15,6 +15,7 @@ using NameSet = std::unordered_set; using NameOrderedSet = std::set; using NameToNameMap = std::unordered_map; using NameToNameSetMap = std::unordered_map; +using NameToNameVector = std::vector>; using NameWithAlias = std::pair; using NamesWithAliases = std::vector; diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 67bd88d10a8..43f7c15b784 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -733,7 +733,6 @@ void AlterCommands::validate(const StorageInMemoryMetadata & metadata, const Con auto all_columns = metadata.columns; /// Default expression for all added/modified columns ASTPtr default_expr_list = std::make_shared(); - NameToNameMap renames_map; for (size_t i = 0; i < size(); ++i) { const auto & command = (*this)[i]; @@ -809,31 +808,40 @@ void AlterCommands::validate(const StorageInMemoryMetadata & metadata, const Con } else if (command.type == AlterCommand::RENAME_COLUMN) { + for (size_t j = i + 1; j < size(); ++j) + { + auto next_command = (*this)[j]; + if (next_command.type == AlterCommand::RENAME_COLUMN) + { + if (next_command.column_name == command.rename_to) + throw Exception{"Transitive renames in a single ALTER query are not allowed (don't make sence)", + ErrorCodes::NOT_IMPLEMENTED}; + else if (next_command.column_name == command.column_name) + throw Exception{"Cannot rename column '" + backQuote(command.column_name) + + "' to two different names in a single ALTER query", + ErrorCodes::BAD_ARGUMENTS}; + } + } + /// TODO Implement nested rename - if (metadata.columns.hasNested(command.column_name)) + if (all_columns.hasNested(command.column_name)) { throw Exception{"Cannot rename whole Nested struct", ErrorCodes::NOT_IMPLEMENTED}; } - if (!metadata.columns.has(command.column_name)) + if (!all_columns.has(command.column_name)) { if (!command.if_exists) throw Exception{"Wrong column name. Cannot find column " + backQuote(command.column_name) + " to rename", ErrorCodes::NOT_FOUND_COLUMN_IN_BLOCK}; + else + continue; } - if (metadata.columns.has(command.rename_to)) + if (all_columns.has(command.rename_to)) throw Exception{"Cannot rename to " + backQuote(command.rename_to) + ": column with this name already exists", ErrorCodes::DUPLICATE_COLUMN}; - - if (renames_map.count(command.column_name)) - throw Exception{"Cannot rename column '" + backQuote(command.column_name) + "' to two different names in a single ALTER query", ErrorCodes::BAD_ARGUMENTS}; - - if (renames_map.count(command.rename_to)) - throw Exception{"Rename loop detected in ALTER query", - ErrorCodes::BAD_ARGUMENTS}; - String from_nested_table_name = Nested::extractTableName(command.column_name); String to_nested_table_name = Nested::extractTableName(command.rename_to); bool from_nested = from_nested_table_name != command.column_name; @@ -846,7 +854,7 @@ void AlterCommands::validate(const StorageInMemoryMetadata & metadata, const Con } else if (!from_nested && !to_nested) { - renames_map[command.column_name] = command.rename_to; + all_columns.rename(command.column_name, command.rename_to); } else { diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index acdb7616175..61eb038f6b0 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -1085,7 +1085,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor auto indices_to_recalc = getIndicesToRecalculate(in, storage_from_source_part, updated_header.getNamesAndTypesList(), context); NameSet files_to_skip = collectFilesToSkip(updated_header, indices_to_recalc, mrk_extension); - NameToNameMap files_to_rename = collectFilesForRenames(source_part, for_file_renames, mrk_extension); + NameToNameVector files_to_rename = collectFilesForRenames(source_part, for_file_renames, mrk_extension); if (need_remove_expired_values) files_to_skip.insert("ttl.txt"); @@ -1097,7 +1097,8 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor continue; String destination = new_part_tmp_path + "/"; - auto rename_it = files_to_rename.find(it->name()); + String file_name = it->name(); + auto rename_it = std::find_if(files_to_rename.begin(), files_to_rename.end(), [&file_name](const auto & rename_pair) { return rename_pair.first == file_name; }); if (rename_it != files_to_rename.end()) { if (rename_it->second.empty()) @@ -1328,7 +1329,7 @@ void MergeTreeDataMergerMutator::splitMutationCommands( } -NameToNameMap MergeTreeDataMergerMutator::collectFilesForRenames( +NameToNameVector MergeTreeDataMergerMutator::collectFilesForRenames( MergeTreeData::DataPartPtr source_part, const MutationCommands & commands_for_removes, const String & mrk_extension) { /// Collect counts for shared streams of different columns. As an example, Nested columns have shared stream with array sizes. @@ -1343,14 +1344,14 @@ NameToNameMap MergeTreeDataMergerMutator::collectFilesForRenames( {}); } - NameToNameMap rename_map; + NameToNameVector rename_vector; /// Remove old indices for (const auto & command : commands_for_removes) { if (command.type == MutationCommand::Type::DROP_INDEX) { - rename_map.emplace("skp_idx_" + command.column_name + ".idx", ""); - rename_map.emplace("skp_idx_" + command.column_name + mrk_extension, ""); + rename_vector.emplace_back("skp_idx_" + command.column_name + ".idx", ""); + rename_vector.emplace_back("skp_idx_" + command.column_name + mrk_extension, ""); } else if (command.type == MutationCommand::Type::DROP_COLUMN) { @@ -1360,8 +1361,8 @@ NameToNameMap MergeTreeDataMergerMutator::collectFilesForRenames( /// Delete files if they are no longer shared with another column. if (--stream_counts[stream_name] == 0) { - rename_map.emplace(stream_name + ".bin", ""); - rename_map.emplace(stream_name + mrk_extension, ""); + rename_vector.emplace_back(stream_name + ".bin", ""); + rename_vector.emplace_back(stream_name + mrk_extension, ""); } }; @@ -1383,8 +1384,8 @@ NameToNameMap MergeTreeDataMergerMutator::collectFilesForRenames( if (stream_from != stream_to) { - rename_map.emplace(stream_from + ".bin", stream_to + ".bin"); - rename_map.emplace(stream_from + mrk_extension, stream_to + mrk_extension); + rename_vector.emplace_back(stream_from + ".bin", stream_to + ".bin"); + rename_vector.emplace_back(stream_from + mrk_extension, stream_to + mrk_extension); } }; IDataType::SubstreamPath stream_path; @@ -1394,7 +1395,7 @@ NameToNameMap MergeTreeDataMergerMutator::collectFilesForRenames( } } - return rename_map; + return rename_vector; } NameSet MergeTreeDataMergerMutator::collectFilesToSkip( diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.h b/src/Storages/MergeTree/MergeTreeDataMergerMutator.h index f2f4ea95fd0..145798a1d87 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.h +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.h @@ -143,10 +143,9 @@ private: MutationCommands & for_interpreter, MutationCommands & for_file_renames); - /// Apply commands to source_part i.e. remove some columns in source_part /// and return set of files, that have to be removed from filesystem and checksums - static NameToNameMap collectFilesForRenames(MergeTreeData::DataPartPtr source_part, const MutationCommands & commands_for_removes, const String & mrk_extension); + static NameToNameVector collectFilesForRenames(MergeTreeData::DataPartPtr source_part, const MutationCommands & commands_for_removes, const String & mrk_extension); /// Files, that we don't need to remove and don't need to hardlink, for example columns.txt and checksums.txt. /// Because we will generate new versions of them after we perform mutation. diff --git a/tests/queries/0_stateless/01213_alter_rename_column.sql b/tests/queries/0_stateless/01213_alter_rename_column.sql index 59e4191d7d5..1732ea88274 100644 --- a/tests/queries/0_stateless/01213_alter_rename_column.sql +++ b/tests/queries/0_stateless/01213_alter_rename_column.sql @@ -24,7 +24,7 @@ SELECT * FROM table_for_rename WHERE key = 1 FORMAT TSVWithNames; ALTER TABLE table_for_rename RENAME COLUMN value3 to value2; --{serverError 15} ALTER TABLE table_for_rename RENAME COLUMN value3 TO r1, RENAME COLUMN value3 TO r2; --{serverError 36} -ALTER TABLE table_for_rename RENAME COLUMN value3 TO r1, RENAME COLUMN r1 TO value1; --{serverError 10} +ALTER TABLE table_for_rename RENAME COLUMN value3 TO r1, RENAME COLUMN r1 TO value1; --{serverError 48} ALTER TABLE table_for_rename RENAME COLUMN value2 TO renamed_value2, RENAME COLUMN value3 TO renamed_value3; diff --git a/tests/queries/0_stateless/01278_alter_rename_combination.reference b/tests/queries/0_stateless/01278_alter_rename_combination.reference new file mode 100644 index 00000000000..c3b3d5464c4 --- /dev/null +++ b/tests/queries/0_stateless/01278_alter_rename_combination.reference @@ -0,0 +1,3 @@ +CREATE TABLE default.rename_table\n(\n `key` Int32, \n `old_value1` Int32, \n `value1` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +key old_value1 value1 +1 2 3 diff --git a/tests/queries/0_stateless/01278_alter_rename_combination.sql b/tests/queries/0_stateless/01278_alter_rename_combination.sql new file mode 100644 index 00000000000..f00adc369c1 --- /dev/null +++ b/tests/queries/0_stateless/01278_alter_rename_combination.sql @@ -0,0 +1,14 @@ +DROP TABLE IF EXISTS rename_table; + +CREATE TABLE rename_table (key Int32, value1 Int32, value2 Int32) ENGINE = MergeTree ORDER BY tuple(); + +INSERT INTO rename_table VALUES (1, 2, 3); + +-- replace one with other +ALTER TABLE rename_table RENAME COLUMN value1 TO old_value1, RENAME COLUMN value2 TO value1; + +SHOW CREATE TABLE rename_table; + +SELECT * FROM rename_table FORMAT TSVWithNames; + +DROP TABLE IF EXISTS rename_table; From b1ed51df6eda50cddd0c4d44f8bcfa77f233a1bc Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 13 May 2020 22:47:35 +0300 Subject: [PATCH 186/738] add simple test --- tests/integration/test_odbc_interaction/test.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/integration/test_odbc_interaction/test.py b/tests/integration/test_odbc_interaction/test.py index 41f54ddd0e6..2a52cedb636 100644 --- a/tests/integration/test_odbc_interaction/test.py +++ b/tests/integration/test_odbc_interaction/test.py @@ -200,6 +200,13 @@ def test_postgres_odbc_hached_dictionary_no_tty_pipe_overflow(started_cluster): assert node1.query("select dictGetString('postgres_odbc_hashed', 'column2', toUInt64(3))") == "xxx\n" +def test_postgres_insert(started_cluster): + conn = get_postgres_conn() + conn.cursor().execute("truncate table clickhouse.test_table") + node1.query("create table pg_insert (column1 UInt8, column2 String) engine=ODBC('DSN=postgresql_odbc;', 'clickhouse', 'test_table')") + node1.query("insert into pg_insert values (1, 'hello'), (2, 'world')") + assert node1.query("select * from pg_insert") == '1\thello\n2\tworld\n' + def test_bridge_dies_with_parent(started_cluster): node1.query("select dictGetString('postgres_odbc_hashed', 'column2', toUInt64(1))") From 6c671eb6a6f6563a832757382734aa8c4eb19d78 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 13 May 2020 23:03:10 +0300 Subject: [PATCH 187/738] clickhouse-test: recreate database for every test (part 1) --- tests/clickhouse-test | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index dc65ceec14f..ea6c4f328fc 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -68,11 +68,19 @@ def run_single_test(args, ext, server_logs_level, client_options, case_file, std command = pattern.format(**params) #print(command) + if args.is_temporary_database: + clickhouse_proc_create = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE) + clickhouse_proc_create.communicate("CREATE DATABASE IF NOT EXISTS " + args.database) + proc = Popen(command, shell=True, env=os.environ) start_time = datetime.now() while (datetime.now() - start_time).total_seconds() < args.timeout and proc.poll() is None: sleep(0.01) + if args.is_temporary_database: + clickhouse_proc_create = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE) + clickhouse_proc_create.communicate("DROP DATABASE " + args.database) + total_time = (datetime.now() - start_time).total_seconds() # Normalize randomized database names in stdout, stderr files. @@ -108,6 +116,7 @@ def get_stacktraces_from_gdb(server_pid): # collect server stacktraces from system.stack_trace table +# it does not work in Sandbox def get_stacktraces_from_clickhouse(client): try: return subprocess.check_output("{} --allow_introspection_functions=1 --query \"SELECT arrayStringConcat(arrayMap(x, y -> concat(x, ': ', y), arrayMap(x -> addressToLine(x), trace), arrayMap(x -> demangle(addressToSymbol(x)), trace)), '\n') as trace FROM system.stack_trace format Vertical\"".format(client), shell=True) @@ -355,6 +364,7 @@ def main(args): clickhouse_proc_create = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE) clickhouse_proc_create.communicate("CREATE DATABASE IF NOT EXISTS " + args.database) + if args.database != "test": clickhouse_proc_create = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE) clickhouse_proc_create.communicate("CREATE DATABASE IF NOT EXISTS test") @@ -469,8 +479,11 @@ def main(args): server_pid = get_server_pid(clickhouse_tcp_port) if server_pid: print("\nLocated ClickHouse server process {} listening at TCP port {}".format(server_pid, clickhouse_tcp_port)) - print("\nCollecting stacktraces from system.stacktraces table:") - print(get_stacktraces_from_clickhouse(args.client)) + + # It does not work in Sandbox + #print("\nCollecting stacktraces from system.stacktraces table:") + #print(get_stacktraces_from_clickhouse(args.client)) + print("\nCollecting stacktraces from all running threads with gdb:") print(get_stacktraces_from_gdb(server_pid)) else: @@ -609,8 +622,12 @@ if __name__ == '__main__': os.environ['CLICKHOUSE_URL_PARAMS'] += get_additional_client_options_url(args) - args.client_with_database = args.client + + # If --database is not specified, we will create temporary database with unique name + # And we will recreate and drop it for each test + args.is_temporary_database = 0 + if not args.database: def random_str(length=6): import random @@ -618,6 +635,8 @@ if __name__ == '__main__': alphabet = string.ascii_lowercase + string.digits return ''.join(random.choice(alphabet) for _ in range(length)) args.database = 'test_{suffix}'.format(suffix=random_str()) + args.is_temporary_database = 1 + args.client_with_database += ' --database=' + args.database if args.extract_from_config is None: From 0faf7bc39b2d5b6101ba0ea5e74c2ce27ebdb3ab Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 13 May 2020 23:17:12 +0300 Subject: [PATCH 188/738] clickhouse-test: create unique database for every test (part 2) --- tests/clickhouse-test | 65 +++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 34 deletions(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index ea6c4f328fc..1a2735296e8 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -51,8 +51,27 @@ def run_single_test(args, ext, server_logs_level, client_options, case_file, std # print(client_options) + if args.database: + database = args.database + os.environ.setdefault("CLICKHOUSE_DATABASE", database) + + else: + # If --database is not specified, we will create temporary database with unique name + # And we will recreate and drop it for each test + def random_str(length=6): + import random + import string + alphabet = string.ascii_lowercase + string.digits + return ''.join(random.choice(alphabet) for _ in range(length)) + database = 'test_{suffix}'.format(suffix=random_str()) + + clickhouse_proc_create = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE) + clickhouse_proc_create.communicate("CREATE DATABASE " + database) + + os.environ["CLICKHOUSE_DATABASE"] = database + params = { - 'client': args.client_with_database, + 'client': args.client + ' --database=' + database, 'logs_level': server_logs_level, 'options': client_options, 'test': case_file, @@ -68,24 +87,20 @@ def run_single_test(args, ext, server_logs_level, client_options, case_file, std command = pattern.format(**params) #print(command) - if args.is_temporary_database: - clickhouse_proc_create = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE) - clickhouse_proc_create.communicate("CREATE DATABASE IF NOT EXISTS " + args.database) - proc = Popen(command, shell=True, env=os.environ) start_time = datetime.now() while (datetime.now() - start_time).total_seconds() < args.timeout and proc.poll() is None: sleep(0.01) - if args.is_temporary_database: + if not args.database: clickhouse_proc_create = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE) - clickhouse_proc_create.communicate("DROP DATABASE " + args.database) + clickhouse_proc_create.communicate("DROP DATABASE " + database) total_time = (datetime.now() - start_time).total_seconds() # Normalize randomized database names in stdout, stderr files. - os.system("LC_ALL=C sed -i -e 's/{test_db}/default/g' {file}".format(test_db=args.database, file=stdout_file)) - os.system("LC_ALL=C sed -i -e 's/{test_db}/default/g' {file}".format(test_db=args.database, file=stderr_file)) + os.system("LC_ALL=C sed -i -e 's/{test_db}/default/g' {file}".format(test_db=database, file=stdout_file)) + os.system("LC_ALL=C sed -i -e 's/{test_db}/default/g' {file}".format(test_db=database, file=stderr_file)) stdout = open(stdout_file, 'r').read() if os.path.exists(stdout_file) else '' stdout = unicode(stdout, errors='replace', encoding='utf-8') @@ -216,7 +231,7 @@ def run_tests_array(all_tests_with_params): else: if args.testname: - clickhouse_proc = Popen(shlex.split(args.client_with_database), stdin=PIPE, stdout=PIPE, stderr=PIPE) + clickhouse_proc = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE) clickhouse_proc.communicate("SELECT 'Running test {suite}/{case} from pid={pid}';".format(pid = os.getpid(), case = case, suite = suite)) reference_file = os.path.join(suite_dir, name) + '.reference' @@ -339,7 +354,6 @@ def main(args): if args.configclient: os.environ.setdefault("CLICKHOUSE_CONFIG_CLIENT", args.configclient) os.environ.setdefault("CLICKHOUSE_TMP", tmp_dir) - os.environ.setdefault("CLICKHOUSE_DATABASE", args.database) # Force to print server warnings in stderr # Shell scripts could change logging level @@ -362,12 +376,12 @@ def main(args): else: args.shard = False - clickhouse_proc_create = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE) - clickhouse_proc_create.communicate("CREATE DATABASE IF NOT EXISTS " + args.database) - - if args.database != "test": + if args.database and args.database != "test": clickhouse_proc_create = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE) - clickhouse_proc_create.communicate("CREATE DATABASE IF NOT EXISTS test") + clickhouse_proc_create.communicate("CREATE DATABASE IF NOT EXISTS " + args.database) + + clickhouse_proc_create = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE) + clickhouse_proc_create.communicate("CREATE DATABASE IF NOT EXISTS test") def is_test_from_dir(suite_dir, case): case_file = os.path.join(suite_dir, case) @@ -470,7 +484,7 @@ def main(args): total_tests_run += tests_n if args.hung_check: - processlist = get_processlist(args.client_with_database) + processlist = get_processlist(args.client) if processlist: print(colored("\nFound hung queries in processlist:", args, "red", attrs=["bold"])) print(processlist) @@ -622,23 +636,6 @@ if __name__ == '__main__': os.environ['CLICKHOUSE_URL_PARAMS'] += get_additional_client_options_url(args) - args.client_with_database = args.client - - # If --database is not specified, we will create temporary database with unique name - # And we will recreate and drop it for each test - args.is_temporary_database = 0 - - if not args.database: - def random_str(length=6): - import random - import string - alphabet = string.ascii_lowercase + string.digits - return ''.join(random.choice(alphabet) for _ in range(length)) - args.database = 'test_{suffix}'.format(suffix=random_str()) - args.is_temporary_database = 1 - - args.client_with_database += ' --database=' + args.database - if args.extract_from_config is None: if os.access(args.binary + '-extract-from-config', os.X_OK): args.extract_from_config = args.binary + '-extract-from-config' From 63c6eb1a37b4f33f0759fe48f67bee9a84aa3e86 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 13 May 2020 23:40:54 +0300 Subject: [PATCH 189/738] Update ZooKeeper.h --- src/Common/ZooKeeper/ZooKeeper.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Common/ZooKeeper/ZooKeeper.h b/src/Common/ZooKeeper/ZooKeeper.h index 74209ce2f4f..e8ab06c2182 100644 --- a/src/Common/ZooKeeper/ZooKeeper.h +++ b/src/Common/ZooKeeper/ZooKeeper.h @@ -187,6 +187,8 @@ public: using WaitCondition = std::function; /// Wait for the node to disappear or return immediately if it doesn't exist. + /// If condition is speficied, it is used to return early (when condition returns false) + /// The function returns true if waited and false if waiting was interrupted by condition. bool waitForDisappear(const std::string & path, const WaitCondition & condition = {}); /// Async interface (a small subset of operations is implemented). From c3cea85b107416af9f13cbbdd41cf2a4012ab070 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 14 May 2020 00:31:57 +0300 Subject: [PATCH 190/738] Support for unicode whitespaces in Lexer --- src/Common/StringUtils/StringUtils.h | 63 ++++++++++++++++++++++++++++ src/Parsers/Lexer.cpp | 9 +++- 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/src/Common/StringUtils/StringUtils.h b/src/Common/StringUtils/StringUtils.h index b8aa023eb3a..52b54035892 100644 --- a/src/Common/StringUtils/StringUtils.h +++ b/src/Common/StringUtils/StringUtils.h @@ -131,6 +131,69 @@ inline char alternateCaseIfAlphaASCII(char c) return c ^ 0x20; } +inline const char * skipWhitespacesUTF8(const char * pos, const char * end) +{ + /// https://en.wikipedia.org/wiki/Whitespace_character + /// with some adjustments. + + /// Code points: 0085 00A0 180E 2000..200A 2028..2029 200B..200D 202F 205F 2060 3000 FEFF + /// The corresponding UTF-8 is: C285 C2A0 E1A08E E28080..E2808A E280A8..E280A9 E2808B..E2808D E280AF E2819F E281A0 E38080 EFBBBF + + /// We check for these bytes directly in UTF8 for simplicity reasons. + + /** C2 + * 85 + * A0 + * E1 A0 8E + * E2 + * 80 + * 80..8A + * A8..A9 + * 8B..8D + * AF + * 81 + * 9F + * A0 + * E3 80 80 + * EF BB BF + */ + + while (pos < end) + { + if (isWhitespaceASCII(*pos)) + { + ++pos; + } + else + { + const char8_t * upos = reinterpret_cast(pos); + + if (pos + 1 < end && upos[0] == 0xC2 && (upos[1] == 0x85 || upos[1] == 0xA0)) + { + pos += 2; + } + else if (pos + 2 < end + && ((upos[0] == 0xE1 && upos[1] == 0xA0 && upos[2] == 0x8E) + || (upos[0] == 0xE2 + && ((upos[1] == 0x80 + && ((upos[2] >= 0x80 && upos[2] <= 0x8A) + || (upos[2] >= 0xA8 && upos[2] <= 0xA9) + || (upos[2] >= 0x8B && upos[2] <= 0x8D) + || (upos[2] == 0xAF))) + || (upos[1] == 0x81 && (upos[2] == 0x9F || upos[2] == 0xA0)))) + || (upos[0] == 0xE3 && upos[1] == 0x80 && upos[2] == 0x80) + || (upos[0] == 0xEF && upos[1] == 0xBB && upos[2] == 0xBF))) + { + pos += 3; + } + else + break; + } + } + + return pos; +} + inline bool equalsCaseInsensitive(char a, char b) { return a == b || (isAlphaASCII(a) && alternateCaseIfAlphaASCII(a) == b); diff --git a/src/Parsers/Lexer.cpp b/src/Parsers/Lexer.cpp index c5017870d82..7efc8663416 100644 --- a/src/Parsers/Lexer.cpp +++ b/src/Parsers/Lexer.cpp @@ -316,7 +316,14 @@ Token Lexer::nextTokenImpl() return Token(TokenType::BareWord, token_begin, pos); } else - return Token(TokenType::Error, token_begin, ++pos); + { + /// We will also skip unicode whitespaces in UTF-8 to support for queries copy-pasted from MS Word and similar. + pos = skipWhitespacesUTF8(pos, end); + if (pos > token_begin) + return Token(TokenType::Whitespace, token_begin, pos); + else + return Token(TokenType::Error, token_begin, ++pos); + } } } From 09c3bb2613141d201ee71833daecfce410cf662a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 14 May 2020 00:37:20 +0300 Subject: [PATCH 191/738] Added a test --- .../0_stateless/01280_unicode_whitespaces_lexer.reference | 3 +++ tests/queries/0_stateless/01280_unicode_whitespaces_lexer.sql | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 tests/queries/0_stateless/01280_unicode_whitespaces_lexer.reference create mode 100644 tests/queries/0_stateless/01280_unicode_whitespaces_lexer.sql diff --git a/tests/queries/0_stateless/01280_unicode_whitespaces_lexer.reference b/tests/queries/0_stateless/01280_unicode_whitespaces_lexer.reference new file mode 100644 index 00000000000..01e79c32a8c --- /dev/null +++ b/tests/queries/0_stateless/01280_unicode_whitespaces_lexer.reference @@ -0,0 +1,3 @@ +1 +2 +3 diff --git a/tests/queries/0_stateless/01280_unicode_whitespaces_lexer.sql b/tests/queries/0_stateless/01280_unicode_whitespaces_lexer.sql new file mode 100644 index 00000000000..e3292b50933 --- /dev/null +++ b/tests/queries/0_stateless/01280_unicode_whitespaces_lexer.sql @@ -0,0 +1,3 @@ +SELECT1; +SELECT 2; +…   
SELECT
  1　᠎​+‌‍2⁠; From cdf3845e433e762846175cd8fa1f6c96d50da665 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 14 May 2020 00:52:34 +0300 Subject: [PATCH 192/738] Respect load_balancing in DirectoryMonitor, to fix w/o internal_replication --- src/Storages/Distributed/DirectoryMonitor.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp index d83fbd84b4b..e38e485b3c9 100644 --- a/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/src/Storages/Distributed/DirectoryMonitor.cpp @@ -215,8 +215,10 @@ ConnectionPoolPtr StorageDistributedDirectoryMonitor::createPool(const std::stri auto pools = createPoolsForAddresses(name, pool_factory); const auto settings = storage.global_context.getSettings(); - return pools.size() == 1 ? pools.front() : std::make_shared(pools, LoadBalancing::RANDOM, - settings.distributed_replica_error_half_life.totalSeconds(), settings.distributed_replica_error_cap); + return pools.size() == 1 ? pools.front() : std::make_shared(pools, + settings.load_balancing, + settings.distributed_replica_error_half_life.totalSeconds(), + settings.distributed_replica_error_cap); } From de1d529e866c386ee260bfd5d924f57af126f04e Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 14 May 2020 01:50:56 +0300 Subject: [PATCH 193/738] Cover nearest_hostname in test_insert_distributed_load_balancing --- .../test_insert_distributed_load_balancing/test.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/integration/test_insert_distributed_load_balancing/test.py b/tests/integration/test_insert_distributed_load_balancing/test.py index 35fe1cdabfd..f98b6c6629d 100644 --- a/tests/integration/test_insert_distributed_load_balancing/test.py +++ b/tests/integration/test_insert_distributed_load_balancing/test.py @@ -51,6 +51,11 @@ def test_prefer_localhost_replica_1_load_balancing_in_order(): assert int(n1.query('SELECT count() FROM data')) == 10 assert int(n2.query('SELECT count() FROM data')) == 0 +def test_prefer_localhost_replica_0_load_balancing_nearest_hostname(): + insert_data(load_balancing='nearest_hostname', prefer_localhost_replica=0) + assert int(n1.query('SELECT count() FROM data')) == 10 + assert int(n2.query('SELECT count() FROM data')) == 0 + def test_prefer_localhost_replica_0_load_balancing_in_order(): insert_data(load_balancing='in_order', prefer_localhost_replica=0) assert int(n1.query('SELECT count() FROM data')) == 0 From 770cc9e86327d35f9f94aabca2a9a535034e32e5 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 14 May 2020 02:12:16 +0300 Subject: [PATCH 194/738] Fix "Arcadia" build that is using obsolete version of C++ standard --- src/Common/StringUtils/StringUtils.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Common/StringUtils/StringUtils.h b/src/Common/StringUtils/StringUtils.h index 52b54035892..0843d4102a8 100644 --- a/src/Common/StringUtils/StringUtils.h +++ b/src/Common/StringUtils/StringUtils.h @@ -4,6 +4,7 @@ #include #include #include +#include namespace detail From e1d4837753588694222ab3356f35489bd6a652df Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 14 May 2020 01:53:49 +0300 Subject: [PATCH 195/738] Fix list of possible nodes for Distributed INSERT for internal_replication=0 --- .../Distributed/DistributedBlockOutputStream.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/src/Storages/Distributed/DistributedBlockOutputStream.cpp index e8c45826a35..fa32e52ba0e 100644 --- a/src/Storages/Distributed/DistributedBlockOutputStream.cpp +++ b/src/Storages/Distributed/DistributedBlockOutputStream.cpp @@ -167,6 +167,7 @@ std::string DistributedBlockOutputStream::getCurrentStateDescription() void DistributedBlockOutputStream::initWritingJobs(const Block & first_block) { + const Settings & settings = context.getSettingsRef(); const auto & addresses_with_failovers = cluster->getShardsAddresses(); const auto & shards_info = cluster->getShardsInfo(); size_t num_shards = shards_info.size(); @@ -180,14 +181,14 @@ void DistributedBlockOutputStream::initWritingJobs(const Block & first_block) const auto & shard_info = shards_info[shard_index]; auto & shard_jobs = per_shard_jobs[shard_index]; - /// If hasInternalReplication, than prefer local replica - if (!shard_info.hasInternalReplication() || !shard_info.isLocal()) + /// If hasInternalReplication, than prefer local replica (if !prefer_localhost_replica) + if (!shard_info.hasInternalReplication() || !shard_info.isLocal() || !settings.prefer_localhost_replica) { const auto & replicas = addresses_with_failovers[shard_index]; for (size_t replica_index : ext::range(0, replicas.size())) { - if (!replicas[replica_index].is_local) + if (!replicas[replica_index].is_local || !settings.prefer_localhost_replica) { shard_jobs.replicas_jobs.emplace_back(shard_index, replica_index, false, first_block); ++remote_jobs_count; @@ -198,7 +199,7 @@ void DistributedBlockOutputStream::initWritingJobs(const Block & first_block) } } - if (shard_info.isLocal()) + if (shard_info.isLocal() && settings.prefer_localhost_replica) { shard_jobs.replicas_jobs.emplace_back(shard_index, 0, true, first_block); ++local_jobs_count; @@ -531,7 +532,7 @@ void DistributedBlockOutputStream::writeAsyncImpl(const Block & block, const siz std::vector dir_names; for (const auto & address : cluster->getShardsAddresses()[shard_id]) - if (!address.is_local) + if (!address.is_local || !settings.prefer_localhost_replica) dir_names.push_back(address.toFullString(settings.use_compact_format_in_distributed_parts_names)); if (!dir_names.empty()) From 52d73c7f45193ad77aa357e48ea39a6cb688947f Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 14 May 2020 03:02:28 +0300 Subject: [PATCH 196/738] Fix prefer_localhost_replica=0 and load_balancing for Distributed INSERT --- src/Interpreters/Cluster.cpp | 54 +++++++++++++++---- src/Interpreters/Cluster.h | 6 ++- .../DistributedBlockOutputStream.cpp | 13 ++--- 3 files changed, 51 insertions(+), 22 deletions(-) diff --git a/src/Interpreters/Cluster.cpp b/src/Interpreters/Cluster.cpp index 151dfc5c9bb..7fad9af8960 100644 --- a/src/Interpreters/Cluster.cpp +++ b/src/Interpreters/Cluster.cpp @@ -328,8 +328,8 @@ Cluster::Cluster(const Poco::Util::AbstractConfiguration & config, const Setting /// In case of internal_replication we will be appending names to dir_name_for_internal_replication std::string dir_name_for_internal_replication; + std::string dir_name_for_internal_replication_with_local; - auto first = true; for (const auto & replica_key : replica_keys) { if (startsWith(replica_key, "weight") || startsWith(replica_key, "internal_replication")) @@ -340,18 +340,20 @@ Cluster::Cluster(const Poco::Util::AbstractConfiguration & config, const Setting replica_addresses.emplace_back(config, partial_prefix + replica_key, current_shard_num, current_replica_num); ++current_replica_num; - if (!replica_addresses.back().is_local) + if (internal_replication) { - if (internal_replication) + auto dir_name = replica_addresses.back().toFullString(settings.use_compact_format_in_distributed_parts_names); + if (!replica_addresses.back().is_local) { - auto dir_name = replica_addresses.back().toFullString(settings.use_compact_format_in_distributed_parts_names); - if (first) + if (dir_name_for_internal_replication.empty()) dir_name_for_internal_replication = dir_name; else dir_name_for_internal_replication += "," + dir_name; } - - if (first) first = false; + if (dir_name_for_internal_replication_with_local.empty()) + dir_name_for_internal_replication_with_local = dir_name; + else + dir_name_for_internal_replication_with_local += "," + dir_name; } } else @@ -383,8 +385,16 @@ Cluster::Cluster(const Poco::Util::AbstractConfiguration & config, const Setting if (weight) slot_to_shard.insert(std::end(slot_to_shard), weight, shards_info.size()); - shards_info.push_back({std::move(dir_name_for_internal_replication), current_shard_num, weight, - std::move(shard_local_addresses), std::move(shard_pool), std::move(all_replicas_pools), internal_replication}); + shards_info.push_back({ + std::move(dir_name_for_internal_replication), + std::move(dir_name_for_internal_replication_with_local), + current_shard_num, + weight, + std::move(shard_local_addresses), + std::move(shard_pool), + std::move(all_replicas_pools), + internal_replication + }); } else throw Exception("Unknown element in config: " + key, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); @@ -433,8 +443,16 @@ Cluster::Cluster(const Settings & settings, const std::vector 0 && settings.prefer_localhost_replica) - { + if (shard_info.isLocal() && settings.prefer_localhost_replica) /// Prefer insert into current instance directly writeToLocal(block, shard_info.getLocalNodeCount()); - } else - { - if (shard_info.dir_name_for_internal_replication.empty()) - throw Exception("Directory name for async inserts is empty, table " + storage.getStorageID().getNameForLogs(), ErrorCodes::LOGICAL_ERROR); - - writeToShard(block, {shard_info.dir_name_for_internal_replication}); - } + writeToShard(block, {shard_info.pathForInsert(settings.prefer_localhost_replica)}); } else { - if (shard_info.getLocalNodeCount() > 0) + if (shard_info.isLocal()) writeToLocal(block, shard_info.getLocalNodeCount()); std::vector dir_names; From dac0b7d1f57b0c7b82d6f6be8a67f7350cdcf0f3 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 14 May 2020 03:22:21 +0300 Subject: [PATCH 197/738] Create table with the same load_balancing policy in test_insert_distributed_load_balancing --- .../test_insert_distributed_load_balancing/test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_insert_distributed_load_balancing/test.py b/tests/integration/test_insert_distributed_load_balancing/test.py index f98b6c6629d..99d74ddc8df 100644 --- a/tests/integration/test_insert_distributed_load_balancing/test.py +++ b/tests/integration/test_insert_distributed_load_balancing/test.py @@ -19,8 +19,7 @@ def start_cluster(): finally: cluster.shutdown() -@pytest.fixture(scope='function', autouse=True) -def create_tables(): +def create_tables(**dist_settings): n1.query('DROP TABLE IF EXISTS data') n2.query('DROP TABLE IF EXISTS data') n1.query('DROP TABLE IF EXISTS dist') @@ -35,9 +34,10 @@ def create_tables(): data, rand() ) - """) + """, settings=dist_settings) def insert_data(**settings): + create_tables(**settings) n1.query('INSERT INTO dist SELECT * FROM numbers(10)', settings=settings) n1.query('SYSTEM FLUSH DISTRIBUTED dist') From f65305878b59f97296b1cecea0ee72ed6c24aae6 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 14 May 2020 10:59:14 +0300 Subject: [PATCH 198/738] Destructive IAggregateFunction::insertResultInto and ColumnAggregateFunction::convertToValues (#10890) * Destructive IAggregateFunction::insertResultInto and ColumnAggregateFunction::convertToValues * Try fix build. * Try fix build. * Fix build. * Make convertToValues static. * fix build. * Remove const casts. * Added comment. * Fix build. * Fix build. * Add test. * Fix test. --- .../AggregateFunctionAggThrow.cpp | 2 +- .../AggregateFunctionArgMinMax.h | 2 +- .../AggregateFunctionArray.h | 2 +- src/AggregateFunctions/AggregateFunctionAvg.h | 2 +- .../AggregateFunctionBitwise.h | 2 +- .../AggregateFunctionBoundingRatio.h | 2 +- ...egateFunctionCategoricalInformationValue.h | 2 +- .../AggregateFunctionCount.h | 4 ++-- .../AggregateFunctionEntropy.h | 2 +- .../AggregateFunctionForEach.h | 6 ++--- .../AggregateFunctionGroupArray.h | 6 ++--- .../AggregateFunctionGroupArrayInsertAt.h | 2 +- .../AggregateFunctionGroupArrayMoving.h | 2 +- .../AggregateFunctionGroupBitmap.h | 4 ++-- .../AggregateFunctionGroupUniqArray.h | 4 ++-- .../AggregateFunctionHistogram.h | 4 ++-- src/AggregateFunctions/AggregateFunctionIf.h | 2 +- .../AggregateFunctionMLMethod.h | 2 +- .../AggregateFunctionMaxIntersections.h | 4 ++-- .../AggregateFunctionMerge.h | 2 +- .../AggregateFunctionMinMaxAny.h | 2 +- .../AggregateFunctionNothing.h | 2 +- .../AggregateFunctionNull.h | 2 +- .../AggregateFunctionOrFill.h | 2 +- .../AggregateFunctionQuantile.h | 4 ++-- .../AggregateFunctionResample.h | 2 +- .../AggregateFunctionRetention.h | 2 +- .../AggregateFunctionSequenceMatch.h | 6 ++--- .../AggregateFunctionSimpleLinearRegression.h | 2 +- .../AggregateFunctionState.h | 4 ++-- .../AggregateFunctionStatistics.h | 4 ++-- .../AggregateFunctionStatisticsSimple.h | 2 +- src/AggregateFunctions/AggregateFunctionSum.h | 2 +- .../AggregateFunctionSumMap.h | 4 ++-- .../AggregateFunctionTimeSeriesGroupSum.h | 2 +- .../AggregateFunctionTopK.h | 4 ++-- .../AggregateFunctionUniq.h | 4 ++-- .../AggregateFunctionUniqCombined.h | 4 ++-- .../AggregateFunctionUniqUpTo.h | 4 ++-- .../AggregateFunctionWindowFunnel.h | 6 ++--- src/AggregateFunctions/IAggregateFunction.h | 4 +++- src/AggregateFunctions/QuantileTiming.h | 20 ++++++++-------- src/Columns/ColumnAggregateFunction.cpp | 12 +++++++--- src/Columns/ColumnAggregateFunction.h | 8 +++---- src/DataStreams/finalizeBlock.cpp | 5 +++- src/Functions/finalizeAggregation.cpp | 9 ++++---- .../Transforms/TotalsHavingTransform.cpp | 9 ++++++-- .../01246_finalize_aggregation_race.reference | 18 +++++++++++++++ .../01246_finalize_aggregation_race.sql | 23 +++++++++++++++++++ 49 files changed, 144 insertions(+), 86 deletions(-) create mode 100644 tests/queries/0_stateless/01246_finalize_aggregation_race.reference create mode 100644 tests/queries/0_stateless/01246_finalize_aggregation_race.sql diff --git a/src/AggregateFunctions/AggregateFunctionAggThrow.cpp b/src/AggregateFunctions/AggregateFunctionAggThrow.cpp index 2bf00676d77..ea3eb9b1a20 100644 --- a/src/AggregateFunctions/AggregateFunctionAggThrow.cpp +++ b/src/AggregateFunctions/AggregateFunctionAggThrow.cpp @@ -93,7 +93,7 @@ public: buf.read(c); } - void insertResultInto(ConstAggregateDataPtr, IColumn & to) const override + void insertResultInto(AggregateDataPtr, IColumn & to) const override { to.insertDefault(); } diff --git a/src/AggregateFunctions/AggregateFunctionArgMinMax.h b/src/AggregateFunctions/AggregateFunctionArgMinMax.h index 88ad9430bf0..9a0c428d75b 100644 --- a/src/AggregateFunctions/AggregateFunctionArgMinMax.h +++ b/src/AggregateFunctions/AggregateFunctionArgMinMax.h @@ -85,7 +85,7 @@ public: return Data::allocatesMemoryInArena(); } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { this->data(place).result.insertResultInto(to); } diff --git a/src/AggregateFunctions/AggregateFunctionArray.h b/src/AggregateFunctions/AggregateFunctionArray.h index cc4d5ffebb2..4fe5e459ae1 100644 --- a/src/AggregateFunctions/AggregateFunctionArray.h +++ b/src/AggregateFunctions/AggregateFunctionArray.h @@ -119,7 +119,7 @@ public: nested_func->deserialize(place, buf, arena); } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { nested_func->insertResultInto(place, to); } diff --git a/src/AggregateFunctions/AggregateFunctionAvg.h b/src/AggregateFunctions/AggregateFunctionAvg.h index a269dd74ad5..d9ef8647b82 100644 --- a/src/AggregateFunctions/AggregateFunctionAvg.h +++ b/src/AggregateFunctions/AggregateFunctionAvg.h @@ -80,7 +80,7 @@ public: readBinary(this->data(place).denominator, buf); } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { auto & column = static_cast(to); column.getData().push_back(this->data(place).template result()); diff --git a/src/AggregateFunctions/AggregateFunctionBitwise.h b/src/AggregateFunctions/AggregateFunctionBitwise.h index 29afa7db8d5..a4e5f7ddafa 100644 --- a/src/AggregateFunctions/AggregateFunctionBitwise.h +++ b/src/AggregateFunctions/AggregateFunctionBitwise.h @@ -74,7 +74,7 @@ public: readBinary(this->data(place).value, buf); } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { assert_cast &>(to).getData().push_back(this->data(place).value); } diff --git a/src/AggregateFunctions/AggregateFunctionBoundingRatio.h b/src/AggregateFunctions/AggregateFunctionBoundingRatio.h index 8451d6532f6..81846db4bac 100644 --- a/src/AggregateFunctions/AggregateFunctionBoundingRatio.h +++ b/src/AggregateFunctions/AggregateFunctionBoundingRatio.h @@ -150,7 +150,7 @@ public: data(place).deserialize(buf); } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { assert_cast(to).getData().push_back(getBoundingRatio(data(place))); } diff --git a/src/AggregateFunctions/AggregateFunctionCategoricalInformationValue.h b/src/AggregateFunctions/AggregateFunctionCategoricalInformationValue.h index 25e5c2d1f1a..1c397c26631 100644 --- a/src/AggregateFunctions/AggregateFunctionCategoricalInformationValue.h +++ b/src/AggregateFunctions/AggregateFunctionCategoricalInformationValue.h @@ -118,7 +118,7 @@ public: } void insertResultInto( - ConstAggregateDataPtr place, + AggregateDataPtr place, IColumn & to ) const override { diff --git a/src/AggregateFunctions/AggregateFunctionCount.h b/src/AggregateFunctions/AggregateFunctionCount.h index 3e153d89d01..092ffc6b6cf 100644 --- a/src/AggregateFunctions/AggregateFunctionCount.h +++ b/src/AggregateFunctions/AggregateFunctionCount.h @@ -57,7 +57,7 @@ public: readVarUInt(data(place).count, buf); } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { assert_cast(to).getData().push_back(data(place).count); } @@ -108,7 +108,7 @@ public: readVarUInt(data(place).count, buf); } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { assert_cast(to).getData().push_back(data(place).count); } diff --git a/src/AggregateFunctions/AggregateFunctionEntropy.h b/src/AggregateFunctions/AggregateFunctionEntropy.h index 942de8ffe98..7586cebd8ec 100644 --- a/src/AggregateFunctions/AggregateFunctionEntropy.h +++ b/src/AggregateFunctions/AggregateFunctionEntropy.h @@ -140,7 +140,7 @@ public: this->data(place).deserialize(buf); } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { auto & column = assert_cast &>(to); column.getData().push_back(this->data(place).get()); diff --git a/src/AggregateFunctions/AggregateFunctionForEach.h b/src/AggregateFunctions/AggregateFunctionForEach.h index 94e60c75382..23a3487de47 100644 --- a/src/AggregateFunctions/AggregateFunctionForEach.h +++ b/src/AggregateFunctions/AggregateFunctionForEach.h @@ -225,15 +225,15 @@ public: } } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { - const AggregateFunctionForEachData & state = data(place); + AggregateFunctionForEachData & state = data(place); ColumnArray & arr_to = assert_cast(to); ColumnArray::Offsets & offsets_to = arr_to.getOffsets(); IColumn & elems_to = arr_to.getData(); - const char * nested_state = state.array_of_aggregate_datas; + char * nested_state = state.array_of_aggregate_datas; for (size_t i = 0; i < state.dynamic_array_size; ++i) { nested_func->insertResultInto(nested_state, elems_to); diff --git a/src/AggregateFunctions/AggregateFunctionGroupArray.h b/src/AggregateFunctions/AggregateFunctionGroupArray.h index 2d345cff1f7..b76efd9f6c2 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupArray.h +++ b/src/AggregateFunctions/AggregateFunctionGroupArray.h @@ -282,7 +282,7 @@ public: // if constexpr (Trait::sampler == Sampler::DETERMINATOR) } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { const auto & value = this->data(place).value; size_t size = value.size(); @@ -600,7 +600,7 @@ public: // if constexpr (Trait::sampler == Sampler::DETERMINATOR) } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { auto & column_array = assert_cast(to); @@ -815,7 +815,7 @@ public: data(place).last = prev; } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { auto & column_array = assert_cast(to); diff --git a/src/AggregateFunctions/AggregateFunctionGroupArrayInsertAt.h b/src/AggregateFunctions/AggregateFunctionGroupArrayInsertAt.h index 395d13f7d34..0eec38c51a7 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupArrayInsertAt.h +++ b/src/AggregateFunctions/AggregateFunctionGroupArrayInsertAt.h @@ -179,7 +179,7 @@ public: } } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { ColumnArray & to_array = assert_cast(to); IColumn & to_data = to_array.getData(); diff --git a/src/AggregateFunctions/AggregateFunctionGroupArrayMoving.h b/src/AggregateFunctions/AggregateFunctionGroupArrayMoving.h index 2c5c56999b2..8f93a7eb25a 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupArrayMoving.h +++ b/src/AggregateFunctions/AggregateFunctionGroupArrayMoving.h @@ -158,7 +158,7 @@ public: this->data(place).sum = value.back(); } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { const auto & data = this->data(place); size_t size = data.value.size(); diff --git a/src/AggregateFunctions/AggregateFunctionGroupBitmap.h b/src/AggregateFunctions/AggregateFunctionGroupBitmap.h index 56901e28e01..766479cc08d 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupBitmap.h +++ b/src/AggregateFunctions/AggregateFunctionGroupBitmap.h @@ -48,7 +48,7 @@ public: this->data(place).rbs.read(buf); } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { assert_cast &>(to).getData().push_back(this->data(place).rbs.size()); } @@ -113,7 +113,7 @@ public: this->data(place).rbs.read(buf); } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { assert_cast &>(to).getData().push_back(this->data(place).rbs.size()); } diff --git a/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h b/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h index b437bb2d7bf..9dbf2c921c2 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h +++ b/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h @@ -102,7 +102,7 @@ public: this->data(place).value.read(buf); } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { ColumnArray & arr_to = assert_cast(to); ColumnArray::Offsets & offsets_to = arr_to.getOffsets(); @@ -241,7 +241,7 @@ public: } } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { ColumnArray & arr_to = assert_cast(to); ColumnArray::Offsets & offsets_to = arr_to.getOffsets(); diff --git a/src/AggregateFunctions/AggregateFunctionHistogram.h b/src/AggregateFunctions/AggregateFunctionHistogram.h index 96ee01652de..8eaa42fdac4 100644 --- a/src/AggregateFunctions/AggregateFunctionHistogram.h +++ b/src/AggregateFunctions/AggregateFunctionHistogram.h @@ -353,9 +353,9 @@ public: this->data(place).read(buf, max_bins); } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { - auto & data = this->data(const_cast(place)); + auto & data = this->data(place); auto & to_array = assert_cast(to); ColumnArray::Offsets & offsets_to = to_array.getOffsets(); diff --git a/src/AggregateFunctions/AggregateFunctionIf.h b/src/AggregateFunctions/AggregateFunctionIf.h index e33fb1df53d..bf4f0b24de3 100644 --- a/src/AggregateFunctions/AggregateFunctionIf.h +++ b/src/AggregateFunctions/AggregateFunctionIf.h @@ -95,7 +95,7 @@ public: nested_func->deserialize(place, buf, arena); } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { nested_func->insertResultInto(place, to); } diff --git a/src/AggregateFunctions/AggregateFunctionMLMethod.h b/src/AggregateFunctions/AggregateFunctionMLMethod.h index bb241074a1e..ce4ef98e0cf 100644 --- a/src/AggregateFunctions/AggregateFunctionMLMethod.h +++ b/src/AggregateFunctions/AggregateFunctionMLMethod.h @@ -389,7 +389,7 @@ public: /** This function is called if aggregate function without State modifier is selected in a query. * Inserts all weights of the model into the column 'to', so user may use such information if needed */ - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { this->data(place).returnWeights(to); } diff --git a/src/AggregateFunctions/AggregateFunctionMaxIntersections.h b/src/AggregateFunctions/AggregateFunctionMaxIntersections.h index 13ed6ae42fe..050c5fd78ea 100644 --- a/src/AggregateFunctions/AggregateFunctionMaxIntersections.h +++ b/src/AggregateFunctions/AggregateFunctionMaxIntersections.h @@ -129,14 +129,14 @@ public: buf.read(reinterpret_cast(value.data()), size * sizeof(value[0])); } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { Int64 current_intersections = 0; Int64 max_intersections = 0; PointType position_of_max_intersections = 0; /// const_cast because we will sort the array - auto & array = const_cast::Array &>(this->data(place).value); + auto & array = this->data(place).value; /// Sort by position; for equal position, sort by weight to get deterministic result. std::sort(array.begin(), array.end()); diff --git a/src/AggregateFunctions/AggregateFunctionMerge.h b/src/AggregateFunctions/AggregateFunctionMerge.h index 2e803211e8f..51a3c11118f 100644 --- a/src/AggregateFunctions/AggregateFunctionMerge.h +++ b/src/AggregateFunctions/AggregateFunctionMerge.h @@ -93,7 +93,7 @@ public: nested_func->deserialize(place, buf, arena); } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { nested_func->insertResultInto(place, to); } diff --git a/src/AggregateFunctions/AggregateFunctionMinMaxAny.h b/src/AggregateFunctions/AggregateFunctionMinMaxAny.h index e2f204e1c07..69504f7b249 100644 --- a/src/AggregateFunctions/AggregateFunctionMinMaxAny.h +++ b/src/AggregateFunctions/AggregateFunctionMinMaxAny.h @@ -746,7 +746,7 @@ public: return Data::allocatesMemoryInArena(); } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { this->data(place).insertResultInto(to); } diff --git a/src/AggregateFunctions/AggregateFunctionNothing.h b/src/AggregateFunctions/AggregateFunctionNothing.h index d9c8f9cea19..511dbbecd38 100644 --- a/src/AggregateFunctions/AggregateFunctionNothing.h +++ b/src/AggregateFunctions/AggregateFunctionNothing.h @@ -67,7 +67,7 @@ public: { } - void insertResultInto(ConstAggregateDataPtr, IColumn & to) const override + void insertResultInto(AggregateDataPtr, IColumn & to) const override { to.insertDefault(); } diff --git a/src/AggregateFunctions/AggregateFunctionNull.h b/src/AggregateFunctions/AggregateFunctionNull.h index a0fe96b6f62..e5309e1300a 100644 --- a/src/AggregateFunctions/AggregateFunctionNull.h +++ b/src/AggregateFunctions/AggregateFunctionNull.h @@ -146,7 +146,7 @@ public: } } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { if (result_is_nullable) { diff --git a/src/AggregateFunctions/AggregateFunctionOrFill.h b/src/AggregateFunctions/AggregateFunctionOrFill.h index be085da9c44..1bbf2ea3135 100644 --- a/src/AggregateFunctions/AggregateFunctionOrFill.h +++ b/src/AggregateFunctions/AggregateFunctionOrFill.h @@ -147,7 +147,7 @@ public: } void insertResultInto( - ConstAggregateDataPtr place, + AggregateDataPtr place, IColumn & to) const override { if (place[size_of_data]) diff --git a/src/AggregateFunctions/AggregateFunctionQuantile.h b/src/AggregateFunctions/AggregateFunctionQuantile.h index f85eb15b3ab..cc90a22da81 100644 --- a/src/AggregateFunctions/AggregateFunctionQuantile.h +++ b/src/AggregateFunctions/AggregateFunctionQuantile.h @@ -138,10 +138,10 @@ public: this->data(place).deserialize(buf); } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { /// const_cast is required because some data structures apply finalizaton (like sorting) for obtain a result. - auto & data = this->data(const_cast(place)); + auto & data = this->data(place); if constexpr (returns_many) { diff --git a/src/AggregateFunctions/AggregateFunctionResample.h b/src/AggregateFunctions/AggregateFunctionResample.h index 0f348899884..49cc312287e 100644 --- a/src/AggregateFunctions/AggregateFunctionResample.h +++ b/src/AggregateFunctions/AggregateFunctionResample.h @@ -173,7 +173,7 @@ public: } void insertResultInto( - ConstAggregateDataPtr place, + AggregateDataPtr place, IColumn & to) const override { auto & col = assert_cast(to); diff --git a/src/AggregateFunctions/AggregateFunctionRetention.h b/src/AggregateFunctions/AggregateFunctionRetention.h index dd75007726e..3a76ba9f055 100644 --- a/src/AggregateFunctions/AggregateFunctionRetention.h +++ b/src/AggregateFunctions/AggregateFunctionRetention.h @@ -123,7 +123,7 @@ public: this->data(place).deserialize(buf); } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { auto & data_to = assert_cast(assert_cast(to).getData()).getData(); auto & offsets_to = assert_cast(to).getOffsets(); diff --git a/src/AggregateFunctions/AggregateFunctionSequenceMatch.h b/src/AggregateFunctions/AggregateFunctionSequenceMatch.h index 5d4a7098951..416786f8fcb 100644 --- a/src/AggregateFunctions/AggregateFunctionSequenceMatch.h +++ b/src/AggregateFunctions/AggregateFunctionSequenceMatch.h @@ -560,9 +560,9 @@ public: DataTypePtr getReturnType() const override { return std::make_shared(); } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { - const_cast(this->data(place)).sort(); + this->data(place).sort(); const auto & data_ref = this->data(place); @@ -588,7 +588,7 @@ public: DataTypePtr getReturnType() const override { return std::make_shared(); } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { const_cast(this->data(place)).sort(); assert_cast(to).getData().push_back(count(place)); diff --git a/src/AggregateFunctions/AggregateFunctionSimpleLinearRegression.h b/src/AggregateFunctions/AggregateFunctionSimpleLinearRegression.h index db4e57c0c6c..d1405172e27 100644 --- a/src/AggregateFunctions/AggregateFunctionSimpleLinearRegression.h +++ b/src/AggregateFunctions/AggregateFunctionSimpleLinearRegression.h @@ -169,7 +169,7 @@ public: } void insertResultInto( - ConstAggregateDataPtr place, + AggregateDataPtr place, IColumn & to ) const override { diff --git a/src/AggregateFunctions/AggregateFunctionState.h b/src/AggregateFunctions/AggregateFunctionState.h index 8879a324827..126d63573af 100644 --- a/src/AggregateFunctions/AggregateFunctionState.h +++ b/src/AggregateFunctions/AggregateFunctionState.h @@ -80,9 +80,9 @@ public: nested_func->deserialize(place, buf, arena); } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { - assert_cast(to).getData().push_back(const_cast(place)); + assert_cast(to).getData().push_back(place); } /// Aggregate function or aggregate function state. diff --git a/src/AggregateFunctions/AggregateFunctionStatistics.h b/src/AggregateFunctions/AggregateFunctionStatistics.h index c691559b3f2..7f6de43f5e1 100644 --- a/src/AggregateFunctions/AggregateFunctionStatistics.h +++ b/src/AggregateFunctions/AggregateFunctionStatistics.h @@ -143,7 +143,7 @@ public: this->data(place).deserialize(buf); } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { this->data(place).publish(to); } @@ -395,7 +395,7 @@ public: this->data(place).deserialize(buf); } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { this->data(place).publish(to); } diff --git a/src/AggregateFunctions/AggregateFunctionStatisticsSimple.h b/src/AggregateFunctions/AggregateFunctionStatisticsSimple.h index 185ef3594dd..72c2ce014e4 100644 --- a/src/AggregateFunctions/AggregateFunctionStatisticsSimple.h +++ b/src/AggregateFunctions/AggregateFunctionStatisticsSimple.h @@ -445,7 +445,7 @@ public: this->data(place).read(buf); } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { const auto & data = this->data(place); auto & dst = static_cast(to).getData(); diff --git a/src/AggregateFunctions/AggregateFunctionSum.h b/src/AggregateFunctions/AggregateFunctionSum.h index 46b90555c27..e9a6e50d9ef 100644 --- a/src/AggregateFunctions/AggregateFunctionSum.h +++ b/src/AggregateFunctions/AggregateFunctionSum.h @@ -156,7 +156,7 @@ public: this->data(place).read(buf); } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { auto & column = static_cast(to); column.getData().push_back(this->data(place).get()); diff --git a/src/AggregateFunctions/AggregateFunctionSumMap.h b/src/AggregateFunctions/AggregateFunctionSumMap.h index 581f4bba0f9..e2aef611955 100644 --- a/src/AggregateFunctions/AggregateFunctionSumMap.h +++ b/src/AggregateFunctions/AggregateFunctionSumMap.h @@ -242,10 +242,10 @@ public: } } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { // Final step does compaction of keys that have zero values, this mutates the state - auto & merged_maps = this->data(const_cast(place)).merged_maps; + auto & merged_maps = this->data(place).merged_maps; for (auto it = merged_maps.cbegin(); it != merged_maps.cend();) { // Key is not compacted if it has at least one non-zero value diff --git a/src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.h b/src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.h index 7dd453f5e87..ad83324e483 100644 --- a/src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.h +++ b/src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.h @@ -253,7 +253,7 @@ public: void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena *) const override { this->data(place).deserialize(buf); } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { const auto & value = this->data(place).result; size_t size = value.size(); diff --git a/src/AggregateFunctions/AggregateFunctionTopK.h b/src/AggregateFunctions/AggregateFunctionTopK.h index dec6baf6ed3..9c5e62bb6d7 100644 --- a/src/AggregateFunctions/AggregateFunctionTopK.h +++ b/src/AggregateFunctions/AggregateFunctionTopK.h @@ -84,7 +84,7 @@ public: set.read(buf); } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { ColumnArray & arr_to = assert_cast(to); ColumnArray::Offsets & offsets_to = arr_to.getOffsets(); @@ -211,7 +211,7 @@ public: this->data(place).value.merge(this->data(rhs).value); } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { ColumnArray & arr_to = assert_cast(to); ColumnArray::Offsets & offsets_to = arr_to.getOffsets(); diff --git a/src/AggregateFunctions/AggregateFunctionUniq.h b/src/AggregateFunctions/AggregateFunctionUniq.h index 9e869435ce0..334e809ebe7 100644 --- a/src/AggregateFunctions/AggregateFunctionUniq.h +++ b/src/AggregateFunctions/AggregateFunctionUniq.h @@ -240,7 +240,7 @@ public: this->data(place).set.read(buf); } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { assert_cast(to).getData().push_back(this->data(place).set.size()); } @@ -294,7 +294,7 @@ public: this->data(place).set.read(buf); } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { assert_cast(to).getData().push_back(this->data(place).set.size()); } diff --git a/src/AggregateFunctions/AggregateFunctionUniqCombined.h b/src/AggregateFunctions/AggregateFunctionUniqCombined.h index 44d92b72365..a92caa4a551 100644 --- a/src/AggregateFunctions/AggregateFunctionUniqCombined.h +++ b/src/AggregateFunctions/AggregateFunctionUniqCombined.h @@ -167,7 +167,7 @@ public: this->data(place).set.read(buf); } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { assert_cast(to).getData().push_back(this->data(place).set.size()); } @@ -229,7 +229,7 @@ public: this->data(place).set.read(buf); } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { assert_cast(to).getData().push_back(this->data(place).set.size()); } diff --git a/src/AggregateFunctions/AggregateFunctionUniqUpTo.h b/src/AggregateFunctions/AggregateFunctionUniqUpTo.h index f16a7cc475e..4c71215141c 100644 --- a/src/AggregateFunctions/AggregateFunctionUniqUpTo.h +++ b/src/AggregateFunctions/AggregateFunctionUniqUpTo.h @@ -180,7 +180,7 @@ public: this->data(place).read(buf, threshold); } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { assert_cast(to).getData().push_back(this->data(place).size()); } @@ -242,7 +242,7 @@ public: this->data(place).read(buf, threshold); } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { assert_cast(to).getData().push_back(this->data(place).size()); } diff --git a/src/AggregateFunctions/AggregateFunctionWindowFunnel.h b/src/AggregateFunctions/AggregateFunctionWindowFunnel.h index 3a1d2adee4a..726656d1ca8 100644 --- a/src/AggregateFunctions/AggregateFunctionWindowFunnel.h +++ b/src/AggregateFunctions/AggregateFunctionWindowFunnel.h @@ -151,14 +151,14 @@ private: // The level path must be 1---2---3---...---check_events_size, find the max event level that statisfied the path in the sliding window. // If found, returns the max event level, else return 0. // The Algorithm complexity is O(n). - UInt8 getEventLevel(const Data & data) const + UInt8 getEventLevel(Data & data) const { if (data.size() == 0) return 0; if (!strict_order && events_size == 1) return 1; - const_cast(data).sort(); + data.sort(); /// events_timestamp stores the timestamp that latest i-th level event happen withing time window after previous level event. /// timestamp defaults to -1, which unsigned timestamp value never meet @@ -279,7 +279,7 @@ public: this->data(place).deserialize(buf); } - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + void insertResultInto(AggregateDataPtr place, IColumn & to) const override { assert_cast(to).getData().push_back(getEventLevel(this->data(place))); } diff --git a/src/AggregateFunctions/IAggregateFunction.h b/src/AggregateFunctions/IAggregateFunction.h index 48d31793a5a..ad074feffc5 100644 --- a/src/AggregateFunctions/IAggregateFunction.h +++ b/src/AggregateFunctions/IAggregateFunction.h @@ -104,7 +104,9 @@ public: } /// Inserts results into a column. - virtual void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const = 0; + /// This method must be called once, from single thread. + /// After this method was called for state, you can't do anything with state but destroy. + virtual void insertResultInto(AggregateDataPtr place, IColumn & to) const = 0; /// Used for machine learning methods. Predict result from trained model. /// Will insert result into `to` column for rows in range [offset, offset + limit). diff --git a/src/AggregateFunctions/QuantileTiming.h b/src/AggregateFunctions/QuantileTiming.h index d7f425ee2d7..2ab8c866615 100644 --- a/src/AggregateFunctions/QuantileTiming.h +++ b/src/AggregateFunctions/QuantileTiming.h @@ -167,7 +167,7 @@ namespace detail buf.readStrict(reinterpret_cast(elems.data()), size * sizeof(elems[0])); } - UInt16 get(double level) const + UInt16 get(double level) { UInt16 quantile = 0; @@ -178,7 +178,7 @@ namespace detail : (elems.size() - 1); /// Sorting an array will not be considered a violation of constancy. - auto & array = const_cast(elems); + auto & array = elems; std::nth_element(array.begin(), array.begin() + n, array.end()); quantile = array[n]; } @@ -187,10 +187,10 @@ namespace detail } template - void getMany(const double * levels, const size_t * levels_permutation, size_t size, ResultType * result) const + void getMany(const double * levels, const size_t * levels_permutation, size_t size, ResultType * result) { size_t prev_n = 0; - auto & array = const_cast(elems); + auto & array = elems; for (size_t i = 0; i < size; ++i) { auto level_index = levels_permutation[i]; @@ -208,14 +208,14 @@ namespace detail } /// Same, but in the case of an empty state, NaN is returned. - float getFloat(double level) const + float getFloat(double level) { return !elems.empty() ? get(level) : std::numeric_limits::quiet_NaN(); } - void getManyFloat(const double * levels, const size_t * levels_permutation, size_t size, float * result) const + void getManyFloat(const double * levels, const size_t * levels_permutation, size_t size, float * result) { if (!elems.empty()) getMany(levels, levels_permutation, size, result); @@ -707,7 +707,7 @@ public: } /// Get the value of the `level` quantile. The level must be between 0 and 1. - UInt16 get(double level) const + UInt16 get(double level) { Kind kind = which(); @@ -728,7 +728,7 @@ public: /// Get the size values of the quantiles of the `levels` levels. Record `size` results starting with `result` address. template - void getMany(const double * levels, const size_t * levels_permutation, size_t size, ResultType * result) const + void getMany(const double * levels, const size_t * levels_permutation, size_t size, ResultType * result) { Kind kind = which(); @@ -748,14 +748,14 @@ public: } /// The same, but in the case of an empty state, NaN is returned. - float getFloat(double level) const + float getFloat(double level) { return tiny.count ? get(level) : std::numeric_limits::quiet_NaN(); } - void getManyFloat(const double * levels, const size_t * levels_permutation, size_t size, float * result) const + void getManyFloat(const double * levels, const size_t * levels_permutation, size_t size, float * result) { if (tiny.count) getMany(levels, levels_permutation, size, result); diff --git a/src/Columns/ColumnAggregateFunction.cpp b/src/Columns/ColumnAggregateFunction.cpp index e52d3e5303f..2f3a766b8f5 100644 --- a/src/Columns/ColumnAggregateFunction.cpp +++ b/src/Columns/ColumnAggregateFunction.cpp @@ -39,7 +39,7 @@ void ColumnAggregateFunction::addArena(ConstArenaPtr arena_) foreign_arenas.push_back(arena_); } -MutableColumnPtr ColumnAggregateFunction::convertToValues() const +MutableColumnPtr ColumnAggregateFunction::convertToValues(MutableColumnPtr column) { /** If the aggregate function returns an unfinalized/unfinished state, * then you just need to copy pointers to it and also shared ownership of data. @@ -65,20 +65,26 @@ MutableColumnPtr ColumnAggregateFunction::convertToValues() const * `AggregateFunction(quantileTimingState(0.5), UInt64)` * into `AggregateFunction(quantileTiming(0.5), UInt64)` * - in the same states. - * + *column_aggregate_func * Then `finalizeAggregation` function will be calculated, which will call `convertToValues` already on the result. * And this converts a column of type * AggregateFunction(quantileTiming(0.5), UInt64) * into UInt16 - already finished result of `quantileTiming`. */ + auto & column_aggregate_func = assert_cast(*column); + auto & func = column_aggregate_func.func; + auto & data = column_aggregate_func.data; + if (const AggregateFunctionState *function_state = typeid_cast(func.get())) { - auto res = createView(); + auto res = column_aggregate_func.createView(); res->set(function_state->getNestedFunction()); res->data.assign(data.begin(), data.end()); return res; } + column_aggregate_func.ensureOwnership(); + MutableColumnPtr res = func->getReturnType()->createColumn(); res->reserve(data.size()); diff --git a/src/Columns/ColumnAggregateFunction.h b/src/Columns/ColumnAggregateFunction.h index 4fd75674fe7..f257351a4d0 100644 --- a/src/Columns/ColumnAggregateFunction.h +++ b/src/Columns/ColumnAggregateFunction.h @@ -114,14 +114,14 @@ public: /// Take shared ownership of Arena, that holds memory for states of aggregate functions. void addArena(ConstArenaPtr arena_); - /** Transform column with states of aggregate functions to column with final result values. - */ - MutableColumnPtr convertToValues() const; + /// Transform column with states of aggregate functions to column with final result values. + /// It expects ColumnAggregateFunction as an argument, this column will be destroyed. + /// This method is made static and receive MutableColumnPtr object to explicitly destroy it. + static MutableColumnPtr convertToValues(MutableColumnPtr column); std::string getName() const override { return "AggregateFunction(" + func->getName() + ")"; } const char * getFamilyName() const override { return "AggregateFunction"; } - bool tryFinalizeAggregateFunction(MutableColumnPtr* res_) const; MutableColumnPtr predictValues(Block & block, const ColumnNumbers & arguments, const Context & context) const; size_t size() const override diff --git a/src/DataStreams/finalizeBlock.cpp b/src/DataStreams/finalizeBlock.cpp index 50fbaf2bfe1..144f1a28129 100644 --- a/src/DataStreams/finalizeBlock.cpp +++ b/src/DataStreams/finalizeBlock.cpp @@ -17,7 +17,10 @@ namespace DB { current.type = unfinalized_type->getReturnType(); if (current.column) - current.column = typeid_cast(*current.column).convertToValues(); + { + auto mut_column = (*std::move(current.column)).mutate(); + current.column = ColumnAggregateFunction::convertToValues(std::move(mut_column)); + } } } } diff --git a/src/Functions/finalizeAggregation.cpp b/src/Functions/finalizeAggregation.cpp index 70642722a04..66c6268841b 100644 --- a/src/Functions/finalizeAggregation.cpp +++ b/src/Functions/finalizeAggregation.cpp @@ -57,15 +57,16 @@ public: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) override { - const ColumnAggregateFunction * column_with_states - = typeid_cast(&*block.getByPosition(arguments.at(0)).column); - if (!column_with_states) + auto column = block.getByPosition(arguments.at(0)).column; + if (!typeid_cast(column.get())) throw Exception("Illegal column " + block.getByPosition(arguments.at(0)).column->getName() + " of first argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN); - block.getByPosition(result).column = column_with_states->convertToValues(); + /// Column is copied here, because there is no guarantee that we own it. + auto mut_column = (*std::move(column)).mutate(); + block.getByPosition(result).column = ColumnAggregateFunction::convertToValues(std::move(mut_column)); } }; diff --git a/src/Processors/Transforms/TotalsHavingTransform.cpp b/src/Processors/Transforms/TotalsHavingTransform.cpp index 6a21c5aabf0..451415f7133 100644 --- a/src/Processors/Transforms/TotalsHavingTransform.cpp +++ b/src/Processors/Transforms/TotalsHavingTransform.cpp @@ -21,8 +21,13 @@ void finalizeChunk(Chunk & chunk) auto columns = chunk.detachColumns(); for (auto & column : columns) - if (const auto * agg_function = typeid_cast(column.get())) - column = agg_function->convertToValues(); + { + if (typeid_cast(column.get())) + { + auto mut_column = (*std::move(column)).mutate(); + column = ColumnAggregateFunction::convertToValues(std::move(mut_column)); + } + } chunk.setColumns(std::move(columns), num_rows); } diff --git a/tests/queries/0_stateless/01246_finalize_aggregation_race.reference b/tests/queries/0_stateless/01246_finalize_aggregation_race.reference new file mode 100644 index 00000000000..f4d508c87cb --- /dev/null +++ b/tests/queries/0_stateless/01246_finalize_aggregation_race.reference @@ -0,0 +1,18 @@ +200 +200 +200 +200 +200 +200 +200 +200 +200 +200 +200 +200 +200 +200 +200 +200 +200 +200 diff --git a/tests/queries/0_stateless/01246_finalize_aggregation_race.sql b/tests/queries/0_stateless/01246_finalize_aggregation_race.sql new file mode 100644 index 00000000000..336fe6bcfea --- /dev/null +++ b/tests/queries/0_stateless/01246_finalize_aggregation_race.sql @@ -0,0 +1,23 @@ +drop table if exists test_quantile; +create table test_quantile (x AggregateFunction(quantileTiming(0.2), UInt64)) engine = Memory; +insert into test_quantile select medianTimingState(.2)(number) from (select * from numbers(1000) order by number desc); +select y from ( +select finalizeAggregation(x) as y from test_quantile union all +select finalizeAggregation(x) as y from test_quantile union all +select finalizeAggregation(x) as y from test_quantile union all +select finalizeAggregation(x) as y from test_quantile union all +select finalizeAggregation(x) as y from test_quantile union all +select finalizeAggregation(x) as y from test_quantile union all +select finalizeAggregation(x) as y from test_quantile union all +select finalizeAggregation(x) as y from test_quantile union all +select finalizeAggregation(x) as y from test_quantile union all +select finalizeAggregation(x) as y from test_quantile union all +select finalizeAggregation(x) as y from test_quantile union all +select finalizeAggregation(x) as y from test_quantile union all +select finalizeAggregation(x) as y from test_quantile union all +select finalizeAggregation(x) as y from test_quantile union all +select finalizeAggregation(x) as y from test_quantile union all +select finalizeAggregation(x) as y from test_quantile union all +select finalizeAggregation(x) as y from test_quantile union all +select finalizeAggregation(x) as y from test_quantile) +order by y; From 4d0503f0214c2482d8a8c33d044537dda2a434d5 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 14 May 2020 11:30:18 +0300 Subject: [PATCH 199/738] Make IColumn::mutate() static. --- programs/obfuscator/Obfuscator.cpp | 4 ++-- src/Columns/ColumnLowCardinality.cpp | 12 ++++++------ src/Columns/FilterDescription.cpp | 2 +- src/Columns/IColumn.h | 6 +++--- src/Common/COW.h | 6 +++--- src/Common/tests/cow_columns.cpp | 4 ++-- src/Common/tests/cow_compositions.cpp | 8 ++++---- src/Core/Block.cpp | 2 +- src/DataStreams/AddingDefaultsBlockInputStream.cpp | 2 +- src/DataStreams/MergingSortedBlockInputStream.cpp | 2 +- src/DataStreams/SquashingTransform.cpp | 3 +-- src/DataStreams/finalizeBlock.cpp | 2 +- src/DataStreams/tests/finish_sorting_stream.cpp | 2 +- src/DataTypes/DataTypeLowCardinality.cpp | 2 +- src/Dictionaries/PolygonDictionary.cpp | 2 +- src/Functions/FunctionsExternalModels.cpp | 2 +- src/Functions/IFunction.cpp | 2 +- src/Functions/array/FunctionArrayMapped.h | 2 +- src/Functions/finalizeAggregation.cpp | 2 +- src/Functions/if.cpp | 4 ++-- src/Interpreters/DictionaryReader.cpp | 2 +- src/Interpreters/HashJoin.cpp | 4 ++-- src/Interpreters/NullableUtils.cpp | 2 +- src/Interpreters/join_common.cpp | 2 +- src/Processors/Chunk.cpp | 2 +- .../Formats/Impl/ValuesBlockInputFormat.cpp | 4 ++-- src/Processors/Merges/Algorithms/MergedData.h | 2 +- src/Processors/Transforms/TotalsHavingTransform.cpp | 2 +- src/Storages/StorageBuffer.cpp | 4 ++-- 29 files changed, 47 insertions(+), 48 deletions(-) diff --git a/programs/obfuscator/Obfuscator.cpp b/programs/obfuscator/Obfuscator.cpp index 9a80dc8d035..8b5a8c73ca4 100644 --- a/programs/obfuscator/Obfuscator.cpp +++ b/programs/obfuscator/Obfuscator.cpp @@ -858,7 +858,7 @@ public: ColumnPtr new_nested_column = nested_model->generate(nested_column); - return ColumnArray::create((*std::move(new_nested_column)).mutate(), (*std::move(column_array.getOffsetsPtr())).mutate()); + return ColumnArray::create(IColumn::mutate(std::move(new_nested_column)), IColumn::mutate(std::move(column_array.getOffsetsPtr()))); } void updateSeed() override @@ -896,7 +896,7 @@ public: ColumnPtr new_nested_column = nested_model->generate(nested_column); - return ColumnNullable::create((*std::move(new_nested_column)).mutate(), (*std::move(column_nullable.getNullMapColumnPtr())).mutate()); + return ColumnNullable::create(IColumn::mutate(std::move(new_nested_column)), IColumn::mutate(std::move(column_nullable.getNullMapColumnPtr()))); } void updateSeed() override diff --git a/src/Columns/ColumnLowCardinality.cpp b/src/Columns/ColumnLowCardinality.cpp index e87b3b4cbf6..d6f0df1d53a 100644 --- a/src/Columns/ColumnLowCardinality.cpp +++ b/src/Columns/ColumnLowCardinality.cpp @@ -190,7 +190,7 @@ void ColumnLowCardinality::insertRangeFrom(const IColumn & src, size_t start, si /// TODO: Support native insertion from other unique column. It will help to avoid null map creation. - auto sub_idx = (*low_cardinality_src->getIndexes().cut(start, length)).mutate(); + auto sub_idx = IColumn::mutate(low_cardinality_src->getIndexes().cut(start, length)); auto idx_map = mapUniqueIndex(*sub_idx); auto src_nested = low_cardinality_src->getDictionary().getNestedColumn(); @@ -268,7 +268,7 @@ MutableColumnPtr ColumnLowCardinality::cloneResized(size_t size) const if (size == 0) unique_ptr = unique_ptr->cloneEmpty(); - return ColumnLowCardinality::create((*std::move(unique_ptr)).mutate(), getIndexes().cloneResized(size)); + return ColumnLowCardinality::create(IColumn::mutate(std::move(unique_ptr)), getIndexes().cloneResized(size)); } int ColumnLowCardinality::compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const @@ -320,7 +320,7 @@ std::vector ColumnLowCardinality::scatter(ColumnIndex num_colu for (auto & column : columns) { auto unique_ptr = dictionary.getColumnUniquePtr(); - column = ColumnLowCardinality::create((*std::move(unique_ptr)).mutate(), std::move(column)); + column = ColumnLowCardinality::create(IColumn::mutate(std::move(unique_ptr)), std::move(column)); } return columns; @@ -337,7 +337,7 @@ void ColumnLowCardinality::setSharedDictionary(const ColumnPtr & column_unique) ColumnLowCardinality::MutablePtr ColumnLowCardinality::cutAndCompact(size_t start, size_t length) const { - auto sub_positions = (*idx.getPositions()->cut(start, length)).mutate(); + auto sub_positions = IColumn::mutate(idx.getPositions()->cut(start, length)); /// Create column with new indexes and old dictionary. /// Dictionary is shared, but will be recreated after compactInplace call. auto column = ColumnLowCardinality::create(getDictionary().assumeMutable(), std::move(sub_positions)); @@ -364,7 +364,7 @@ void ColumnLowCardinality::compactIfSharedDictionary() ColumnLowCardinality::DictionaryEncodedColumn ColumnLowCardinality::getMinimalDictionaryEncodedColumn(UInt64 offset, UInt64 limit) const { - MutableColumnPtr sub_indexes = (*std::move(idx.getPositions()->cut(offset, limit))).mutate(); + MutableColumnPtr sub_indexes = IColumn::mutate(idx.getPositions()->cut(offset, limit)); auto indexes_map = mapUniqueIndex(*sub_indexes); auto sub_keys = getDictionary().getNestedColumn()->index(*indexes_map, 0); @@ -710,7 +710,7 @@ void ColumnLowCardinality::Dictionary::compact(ColumnPtr & positions) auto sub_keys = unique.getNestedColumn()->index(*indexes, 0); auto new_indexes = new_unique.uniqueInsertRangeFrom(*sub_keys, 0, sub_keys->size()); - positions = (*new_indexes->index(*positions, 0)).mutate(); + positions = IColumn::mutate(new_indexes->index(*positions, 0)); column_unique = std::move(new_column_unique); shared = false; diff --git a/src/Columns/FilterDescription.cpp b/src/Columns/FilterDescription.cpp index 27336d3db58..d216094eaab 100644 --- a/src/Columns/FilterDescription.cpp +++ b/src/Columns/FilterDescription.cpp @@ -64,7 +64,7 @@ FilterDescription::FilterDescription(const IColumn & column_) if (const auto * nullable_column = checkAndGetColumn(column)) { ColumnPtr nested_column = nullable_column->getNestedColumnPtr(); - MutableColumnPtr mutable_holder = (*std::move(nested_column)).mutate(); + MutableColumnPtr mutable_holder = IColumn::mutate(std::move(nested_column)); ColumnUInt8 * concrete_column = typeid_cast(mutable_holder.get()); if (!concrete_column) diff --git a/src/Columns/IColumn.h b/src/Columns/IColumn.h index 4af593bb658..11ade9b3b84 100644 --- a/src/Columns/IColumn.h +++ b/src/Columns/IColumn.h @@ -304,10 +304,10 @@ public: } - MutablePtr mutate() const && + static MutablePtr mutate(Ptr ptr) { - MutablePtr res = shallowMutate(); - res->forEachSubcolumn([](WrappedPtr & subcolumn) { subcolumn = std::move(*subcolumn).mutate(); }); + MutablePtr res = ptr->shallowMutate(); + res->forEachSubcolumn([](WrappedPtr & subcolumn) { subcolumn = IColumn::mutate(std::move(subcolumn)); }); return res; } diff --git a/src/Common/COW.h b/src/Common/COW.h index d8b50c54368..66b2bbde5b0 100644 --- a/src/Common/COW.h +++ b/src/Common/COW.h @@ -50,7 +50,7 @@ /// Change value of x. { /// Creating mutable ptr. It can clone an object under the hood if it was shared. - Column::MutablePtr mutate_x = std::move(*x).mutate(); + Column::MutablePtr mutate_x = IColumn::mutate(std::move(x)); /// Using non-const methods of an object. mutate_x->set(2); /// Assigning pointer 'x' to mutated object. @@ -185,9 +185,9 @@ protected: } public: - MutablePtr mutate() const && + static MutablePtr mutate(Ptr ptr) { - return shallowMutate(); + return ptr.shallowMutate(); } MutablePtr assumeMutable() const diff --git a/src/Common/tests/cow_columns.cpp b/src/Common/tests/cow_columns.cpp index b4c3637be5a..fa84fc9ebc2 100644 --- a/src/Common/tests/cow_columns.cpp +++ b/src/Common/tests/cow_columns.cpp @@ -53,7 +53,7 @@ int main(int, char **) std::cerr << "addresses: " << x.get() << ", " << y.get() << "\n"; { - MutableColumnPtr mut = std::move(*y).mutate(); + MutableColumnPtr mut = IColumn::mutate(std::move(y)); mut->set(2); std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << ", " << mut->use_count() << "\n"; @@ -72,7 +72,7 @@ int main(int, char **) std::cerr << "addresses: " << x.get() << ", " << y.get() << "\n"; { - MutableColumnPtr mut = std::move(*y).mutate(); + MutableColumnPtr mut = IColumn::mutate(std::move(y)); mut->set(3); std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << ", " << mut->use_count() << "\n"; diff --git a/src/Common/tests/cow_compositions.cpp b/src/Common/tests/cow_compositions.cpp index 0335693d1bd..be33f392497 100644 --- a/src/Common/tests/cow_compositions.cpp +++ b/src/Common/tests/cow_compositions.cpp @@ -18,7 +18,7 @@ public: virtual int get() const = 0; virtual void set(int value) = 0; - MutablePtr mutate() const && { return deepMutate(); } + static MutablePtr mutate(Ptr ptr) { return ptr->deepMutate(); } }; using ColumnPtr = IColumn::Ptr; @@ -52,7 +52,7 @@ private: { std::cerr << "Mutating\n"; auto res = shallowMutate(); - res->wrapped = std::move(*wrapped).mutate(); + res->wrapped = IColumn::mutate(std::move(wrapped)); return res; } @@ -72,7 +72,7 @@ int main(int, char **) std::cerr << "addresses: " << x.get() << ", " << y.get() << "\n"; { - MutableColumnPtr mut = std::move(*y).mutate(); + MutableColumnPtr mut = IColumn::mutate(std::move(y)); mut->set(2); std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << ", " << mut->use_count() << "\n"; @@ -91,7 +91,7 @@ int main(int, char **) std::cerr << "addresses: " << x.get() << ", " << y.get() << "\n"; { - MutableColumnPtr mut = std::move(*y).mutate(); + MutableColumnPtr mut = IColumn::mutate(std::move(y)); mut->set(3); std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << ", " << mut->use_count() << "\n"; diff --git a/src/Core/Block.cpp b/src/Core/Block.cpp index 75b50377a64..da8bfa5451b 100644 --- a/src/Core/Block.cpp +++ b/src/Core/Block.cpp @@ -335,7 +335,7 @@ MutableColumns Block::mutateColumns() size_t num_columns = data.size(); MutableColumns columns(num_columns); for (size_t i = 0; i < num_columns; ++i) - columns[i] = data[i].column ? (*std::move(data[i].column)).mutate() : data[i].type->createColumn(); + columns[i] = data[i].column ? IColumn::mutate(std::move(data[i].column)) : data[i].type->createColumn(); return columns; } diff --git a/src/DataStreams/AddingDefaultsBlockInputStream.cpp b/src/DataStreams/AddingDefaultsBlockInputStream.cpp index d2df3dbc496..4caf396eb49 100644 --- a/src/DataStreams/AddingDefaultsBlockInputStream.cpp +++ b/src/DataStreams/AddingDefaultsBlockInputStream.cpp @@ -191,7 +191,7 @@ Block AddingDefaultsBlockInputStream::readImpl() /// TODO: FixedString if (isColumnedAsNumber(column_read.type) || isDecimal(column_read.type)) { - MutableColumnPtr column_mixed = (*std::move(column_read.column)).mutate(); + MutableColumnPtr column_mixed = IColumn::mutate(std::move(column_read.column)); mixNumberColumns(column_read.type->getTypeId(), column_mixed, column_def.column, defaults_mask); column_read.column = std::move(column_mixed); } diff --git a/src/DataStreams/MergingSortedBlockInputStream.cpp b/src/DataStreams/MergingSortedBlockInputStream.cpp index 1ed7c6dff22..5d4328045dd 100644 --- a/src/DataStreams/MergingSortedBlockInputStream.cpp +++ b/src/DataStreams/MergingSortedBlockInputStream.cpp @@ -185,7 +185,7 @@ void MergingSortedBlockInputStream::merge(MutableColumns & merged_columns, TSort throw Exception("Logical error in MergingSortedBlockInputStream", ErrorCodes::LOGICAL_ERROR); for (size_t i = 0; i < num_columns; ++i) - merged_columns[i] = (*std::move(source_blocks[source_num].getByPosition(i).column)).mutate(); + merged_columns[i] = IColumn::mutate(std::move(source_blocks[source_num].getByPosition(i).column)); // std::cerr << "copied columns\n"; diff --git a/src/DataStreams/SquashingTransform.cpp b/src/DataStreams/SquashingTransform.cpp index 8cbbebb75ab..c57e2351230 100644 --- a/src/DataStreams/SquashingTransform.cpp +++ b/src/DataStreams/SquashingTransform.cpp @@ -95,8 +95,7 @@ void SquashingTransform::append(ReferenceType input_block) { const auto source_column = input_block.getByPosition(i).column; - auto mutable_column = (*std::move( - accumulated_block.getByPosition(i).column)).mutate(); + auto mutable_column = IColumn::mutate(std::move(accumulated_block.getByPosition(i).column)); if (reserve_memory) { diff --git a/src/DataStreams/finalizeBlock.cpp b/src/DataStreams/finalizeBlock.cpp index 144f1a28129..56068edcc29 100644 --- a/src/DataStreams/finalizeBlock.cpp +++ b/src/DataStreams/finalizeBlock.cpp @@ -18,7 +18,7 @@ namespace DB current.type = unfinalized_type->getReturnType(); if (current.column) { - auto mut_column = (*std::move(current.column)).mutate(); + auto mut_column = IColumn::mutate(std::move(current.column)); current.column = ColumnAggregateFunction::convertToValues(std::move(mut_column)); } } diff --git a/src/DataStreams/tests/finish_sorting_stream.cpp b/src/DataStreams/tests/finish_sorting_stream.cpp index cfc9ba217b3..d10ae3fbb63 100644 --- a/src/DataStreams/tests/finish_sorting_stream.cpp +++ b/src/DataStreams/tests/finish_sorting_stream.cpp @@ -80,7 +80,7 @@ int main(int argc, char ** argv) { for (size_t i = 0; i < block.columns(); ++i) { - MutableColumnPtr ptr = (*std::move(res_block.getByPosition(i).column)).mutate(); + MutableColumnPtr ptr = IColumn::mutate(std::move(res_block.getByPosition(i).column)); ptr->insertRangeFrom(*block.getByPosition(i).column.get(), 0, block.rows()); } } diff --git a/src/DataTypes/DataTypeLowCardinality.cpp b/src/DataTypes/DataTypeLowCardinality.cpp index 95ef60bed6e..b4f28a8853f 100644 --- a/src/DataTypes/DataTypeLowCardinality.cpp +++ b/src/DataTypes/DataTypeLowCardinality.cpp @@ -672,7 +672,7 @@ void DataTypeLowCardinality::deserializeBinaryBulkWithMultipleStreams( ColumnLowCardinality::Index(indexes_column->getPtr()).check( maps.dictionary_map->size() + maps.additional_keys_map->size()); - auto used_keys = (*std::move(global_dictionary->getNestedColumn()->index(*maps.dictionary_map, 0))).mutate(); + auto used_keys = IColumn::mutate(global_dictionary->getNestedColumn()->index(*maps.dictionary_map, 0)); if (!maps.additional_keys_map->empty()) { diff --git a/src/Dictionaries/PolygonDictionary.cpp b/src/Dictionaries/PolygonDictionary.cpp index dd231b3f2d7..01c9522e952 100644 --- a/src/Dictionaries/PolygonDictionary.cpp +++ b/src/Dictionaries/PolygonDictionary.cpp @@ -200,7 +200,7 @@ void IPolygonDictionary::blockToAttributes(const DB::Block &block) const auto & column = block.safeGetByPosition(i + 1); if (attributes[i]) { - MutableColumnPtr mutated = std::move(*attributes[i]).mutate(); + MutableColumnPtr mutated = IColumn::mutate(std::move(attributes[i])); mutated->insertRangeFrom(*column.column, 0, column.column->size()); attributes[i] = std::move(mutated); } diff --git a/src/Functions/FunctionsExternalModels.cpp b/src/Functions/FunctionsExternalModels.cpp index a7ec5947c4f..f570fb661dd 100644 --- a/src/Functions/FunctionsExternalModels.cpp +++ b/src/Functions/FunctionsExternalModels.cpp @@ -100,7 +100,7 @@ void FunctionModelEvaluate::executeImpl(Block & block, const ColumnNumbers & arg null_map = col_nullable->getNullMapColumnPtr(); else { - auto mut_null_map = (*std::move(null_map)).mutate(); + auto mut_null_map = IColumn::mutate(std::move(null_map)); NullMap & result_null_map = assert_cast(*mut_null_map).getData(); const NullMap & src_null_map = col_nullable->getNullMapColumn().getData(); diff --git a/src/Functions/IFunction.cpp b/src/Functions/IFunction.cpp index 618caab232e..4bb702015df 100644 --- a/src/Functions/IFunction.cpp +++ b/src/Functions/IFunction.cpp @@ -147,7 +147,7 @@ ColumnPtr wrapInNullable(const ColumnPtr & src, const Block & block, const Colum } else { - MutableColumnPtr mutable_result_null_map_column = (*std::move(result_null_map_column)).mutate(); + MutableColumnPtr mutable_result_null_map_column = IColumn::mutate(std::move(result_null_map_column)); NullMap & result_null_map = assert_cast(*mutable_result_null_map_column).getData(); const NullMap & src_null_map = assert_cast(*null_map_column).getData(); diff --git a/src/Functions/array/FunctionArrayMapped.h b/src/Functions/array/FunctionArrayMapped.h index 3313f2a8e6d..ee4c5c083d8 100644 --- a/src/Functions/array/FunctionArrayMapped.h +++ b/src/Functions/array/FunctionArrayMapped.h @@ -222,7 +222,7 @@ public: } /// Put all the necessary columns multiplied by the sizes of arrays into the block. - auto replicated_column_function_ptr = (*column_function->replicate(column_first_array->getOffsets())).mutate(); + auto replicated_column_function_ptr = IColumn::mutate(column_function->replicate(column_first_array->getOffsets())); auto * replicated_column_function = typeid_cast(replicated_column_function_ptr.get()); replicated_column_function->appendArguments(arrays); diff --git a/src/Functions/finalizeAggregation.cpp b/src/Functions/finalizeAggregation.cpp index 66c6268841b..522a645b8e7 100644 --- a/src/Functions/finalizeAggregation.cpp +++ b/src/Functions/finalizeAggregation.cpp @@ -65,7 +65,7 @@ public: ErrorCodes::ILLEGAL_COLUMN); /// Column is copied here, because there is no guarantee that we own it. - auto mut_column = (*std::move(column)).mutate(); + auto mut_column = IColumn::mutate(std::move(column)); block.getByPosition(result).column = ColumnAggregateFunction::convertToValues(std::move(mut_column)); } }; diff --git a/src/Functions/if.cpp b/src/Functions/if.cpp index 58dbf74af5e..02c3d938d2b 100644 --- a/src/Functions/if.cpp +++ b/src/Functions/if.cpp @@ -816,7 +816,7 @@ private: if (isColumnNullable(*arg_else.column)) { auto arg_else_column = arg_else.column; - auto result_column = (*std::move(arg_else_column)).mutate(); + auto result_column = IColumn::mutate(std::move(arg_else_column)); assert_cast(*result_column).applyNullMap(assert_cast(*arg_cond.column)); block.getByPosition(result).column = std::move(result_column); } @@ -858,7 +858,7 @@ private: if (isColumnNullable(*arg_then.column)) { auto arg_then_column = arg_then.column; - auto result_column = (*std::move(arg_then_column)).mutate(); + auto result_column = IColumn::mutate(std::move(arg_then_column)); assert_cast(*result_column).applyNegatedNullMap(assert_cast(*arg_cond.column)); block.getByPosition(result).column = std::move(result_column); } diff --git a/src/Interpreters/DictionaryReader.cpp b/src/Interpreters/DictionaryReader.cpp index e996be65095..61d40748617 100644 --- a/src/Interpreters/DictionaryReader.cpp +++ b/src/Interpreters/DictionaryReader.cpp @@ -119,7 +119,7 @@ void DictionaryReader::readKeys(const IColumn & keys, Block & out_block, ColumnV /// calculate and extract dictHas() function_has->execute(working_block, size); ColumnWithTypeAndName & has_column = working_block.getByPosition(has_position); - auto mutable_has = (*std::move(has_column.column)).mutate(); + auto mutable_has = IColumn::mutate(std::move(has_column.column)); found.swap(typeid_cast &>(*mutable_has).getData()); has_column.column = nullptr; diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index e5b3f9e55f3..0e463a84dc4 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -108,7 +108,7 @@ static ColumnWithTypeAndName correctNullability(ColumnWithTypeAndName && column, JoinCommon::convertColumnToNullable(column); if (column.type->isNullable() && !negative_null_map.empty()) { - MutableColumnPtr mutable_column = (*std::move(column.column)).mutate(); + MutableColumnPtr mutable_column = IColumn::mutate(std::move(column.column)); assert_cast(*mutable_column).applyNegatedNullMap(negative_null_map); column.column = std::move(mutable_column); } @@ -127,7 +127,7 @@ static void changeNullability(MutableColumnPtr & mutable_column) else column = makeNullable(column); - mutable_column = (*std::move(column)).mutate(); + mutable_column = IColumn::mutate(std::move(column)); } static ColumnPtr emptyNotNullableClone(const ColumnPtr & column) diff --git a/src/Interpreters/NullableUtils.cpp b/src/Interpreters/NullableUtils.cpp index aea69f8ea6d..d1586bec06b 100644 --- a/src/Interpreters/NullableUtils.cpp +++ b/src/Interpreters/NullableUtils.cpp @@ -33,7 +33,7 @@ ColumnPtr extractNestedColumnsAndNullMap(ColumnRawPtrs & key_columns, ConstNullM } else { - MutableColumnPtr mutable_null_map_holder = (*std::move(null_map_holder)).mutate(); + MutableColumnPtr mutable_null_map_holder = IColumn::mutate(std::move(null_map_holder)); PaddedPODArray & mutable_null_map = assert_cast(*mutable_null_map_holder).getData(); const PaddedPODArray & other_null_map = column_nullable->getNullMapData(); diff --git a/src/Interpreters/join_common.cpp b/src/Interpreters/join_common.cpp index 701c520c9f8..e3ca9258892 100644 --- a/src/Interpreters/join_common.cpp +++ b/src/Interpreters/join_common.cpp @@ -43,7 +43,7 @@ void removeColumnNullability(ColumnWithTypeAndName & column) { const auto * nullable_column = checkAndGetColumn(*column.column); ColumnPtr nested_column = nullable_column->getNestedColumnPtr(); - MutableColumnPtr mutable_column = (*std::move(nested_column)).mutate(); + MutableColumnPtr mutable_column = IColumn::mutate(std::move(nested_column)); column.column = std::move(mutable_column); } } diff --git a/src/Processors/Chunk.cpp b/src/Processors/Chunk.cpp index d68c2bea5ae..1f73c9f276a 100644 --- a/src/Processors/Chunk.cpp +++ b/src/Processors/Chunk.cpp @@ -76,7 +76,7 @@ MutableColumns Chunk::mutateColumns() size_t num_columns = columns.size(); MutableColumns mut_columns(num_columns); for (size_t i = 0; i < num_columns; ++i) - mut_columns[i] = (*std::move(columns[i])).mutate(); + mut_columns[i] = IColumn::mutate(std::move(columns[i])); columns.clear(); num_rows = 0; diff --git a/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp b/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp index 25e93215ddf..2a4d451d45a 100644 --- a/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp @@ -70,7 +70,7 @@ Chunk ValuesBlockInputFormat::generate() if (!templates[i] || !templates[i]->rowsCount()) continue; if (columns[i]->empty()) - columns[i] = std::move(*templates[i]->evaluateAll(block_missing_values, i)).mutate(); + columns[i] = IColumn::mutate(templates[i]->evaluateAll(block_missing_values, i)); else { ColumnPtr evaluated = templates[i]->evaluateAll(block_missing_values, i, columns[i]->size()); @@ -134,7 +134,7 @@ bool ValuesBlockInputFormat::tryParseExpressionUsingTemplate(MutableColumnPtr & /// Expression in the current row is not match template deduced on the first row. /// Evaluate expressions, which were parsed using this template. if (column->empty()) - column = std::move(*templates[column_idx]->evaluateAll(block_missing_values, column_idx)).mutate(); + column = IColumn::mutate(templates[column_idx]->evaluateAll(block_missing_values, column_idx)); else { ColumnPtr evaluated = templates[column_idx]->evaluateAll(block_missing_values, column_idx, column->size()); diff --git a/src/Processors/Merges/Algorithms/MergedData.h b/src/Processors/Merges/Algorithms/MergedData.h index c96cc9ad6f4..bb530908c1e 100644 --- a/src/Processors/Merges/Algorithms/MergedData.h +++ b/src/Processors/Merges/Algorithms/MergedData.h @@ -44,7 +44,7 @@ public: { num_rows = limit_rows; for (auto & column : columns) - column = (*column->cut(0, num_rows)).mutate(); + column = IColumn::mutate(column->cut(0, num_rows)); } need_flush = true; diff --git a/src/Processors/Transforms/TotalsHavingTransform.cpp b/src/Processors/Transforms/TotalsHavingTransform.cpp index 451415f7133..083066a72d9 100644 --- a/src/Processors/Transforms/TotalsHavingTransform.cpp +++ b/src/Processors/Transforms/TotalsHavingTransform.cpp @@ -24,7 +24,7 @@ void finalizeChunk(Chunk & chunk) { if (typeid_cast(column.get())) { - auto mut_column = (*std::move(column)).mutate(); + auto mut_column = IColumn::mutate(std::move(column)); column = ColumnAggregateFunction::convertToValues(std::move(mut_column)); } } diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index f7563df318a..e71d75816e8 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -286,7 +286,7 @@ static void appendBlock(const Block & from, Block & to) for (size_t column_no = 0, columns = to.columns(); column_no < columns; ++column_no) { const IColumn & col_from = *from.getByPosition(column_no).column.get(); - MutableColumnPtr col_to = (*std::move(to.getByPosition(column_no).column)).mutate(); + MutableColumnPtr col_to = IColumn::mutate(std::move(to.getByPosition(column_no).column)); col_to->insertRangeFrom(col_from, 0, rows); @@ -302,7 +302,7 @@ static void appendBlock(const Block & from, Block & to) { ColumnPtr & col_to = to.getByPosition(column_no).column; if (col_to->size() != old_rows) - col_to = (*std::move(col_to)).mutate()->cut(0, old_rows); + col_to = col_to->cut(0, old_rows); } } catch (...) From a1013f1e715556662379479f0228345645c9ca20 Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Thu, 14 May 2020 10:38:02 +0200 Subject: [PATCH 200/738] Fix for the hang during deletion of engine=Kafka --- .../Kafka/ReadBufferFromKafkaConsumer.cpp | 19 +++++++++++----- tests/integration/test_storage_kafka/test.py | 22 ++++++++++++------- 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp index 62feea8fe34..36832a07221 100644 --- a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp +++ b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp @@ -82,18 +82,27 @@ ReadBufferFromKafkaConsumer::ReadBufferFromKafkaConsumer( ReadBufferFromKafkaConsumer::~ReadBufferFromKafkaConsumer() { - /// NOTE: see https://github.com/edenhill/librdkafka/issues/2077 try { if (!consumer->get_subscription().empty()) consumer->unsubscribe(); - if (!assignment.empty()) - consumer->unassign(); - while (consumer->get_consumer_queue().next_event(100ms)); } catch (const cppkafka::HandleException & e) { - LOG_ERROR(log, "Exception from ReadBufferFromKafkaConsumer destructor: " << e.what()); + LOG_ERROR(log, "Exception from ReadBufferFromKafkaConsumer destructor (unsubscribe): " << e.what()); + } + + try + { + // we need to drain rest of the messages / queued callback calls from the consumer + // after unsubscribe, otherwise consumer will hang on destruction + // see https://github.com/edenhill/librdkafka/issues/2077 + // https://github.com/confluentinc/confluent-kafka-go/issues/189 etc. + while (consumer->poll(100ms)); + } + catch (const cppkafka::HandleException & e) + { + LOG_ERROR(log, "Exception from ReadBufferFromKafkaConsumer destructor (drain): " << e.what()); } } diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py index 92fa06c262e..1c28d774688 100644 --- a/tests/integration/test_storage_kafka/test.py +++ b/tests/integration/test_storage_kafka/test.py @@ -198,7 +198,6 @@ def test_kafka_settings_new_syntax(kafka_cluster): kafka_check_result(result, True) -@pytest.mark.skip(reason="https://github.com/edenhill/librdkafka/issues/2077") @pytest.mark.timeout(180) def test_kafka_consumer_hang(kafka_cluster): @@ -219,7 +218,7 @@ def test_kafka_consumer_hang(kafka_cluster): CREATE MATERIALIZED VIEW test.consumer TO test.view AS SELECT * FROM test.kafka; ''') - time.sleep(12) + time.sleep(10) instance.query('SELECT * FROM test.view') # This should trigger heartbeat fail, @@ -229,18 +228,25 @@ def test_kafka_consumer_hang(kafka_cluster): time.sleep(0.5) kafka_cluster.unpause_container('kafka1') + # print("Attempt to drop") instance.query('DROP TABLE test.kafka') + #kafka_cluster.open_bash_shell('instance') + instance.query(''' DROP TABLE test.consumer; DROP TABLE test.view; ''') - log = '/var/log/clickhouse-server/stderr.log' - instance.exec_in_container(['grep', '-q', 'BROKERFAIL', log]) - instance.exec_in_container(['grep', '-q', '|ASSIGN|', log]) - instance.exec_in_container(['grep', '-q', 'Heartbeat failed: REBALANCE_IN_PROGRESS: group is rebalancing', log]) - instance.exec_in_container(['grep', '-q', 'Group "consumer_hang": waiting for rebalance_cb', log]) + # we expect no hanging drop queries + # 'dr'||'op' to avoid self matching + assert int(instance.query("select count() from system.processes where position(lower(query),'dr'||'op')>0")) == 0 + + # log = '/var/log/clickhouse-server/stderr.log' + # instance.exec_in_container(['grep', '-q', 'BROKERFAIL', log]) + # instance.exec_in_container(['grep', '-q', '|ASSIGN|', log]) + # instance.exec_in_container(['grep', '-q', 'Heartbeat failed: REBALANCE_IN_PROGRESS: group is rebalancing', log]) + # instance.exec_in_container(['grep', '-q', 'Group "consumer_hang": waiting for rebalance_cb', log]) @pytest.mark.timeout(180) @@ -1234,7 +1240,7 @@ def test_exception_from_destructor(kafka_cluster): DROP TABLE test.kafka; ''') - kafka_cluster.open_bash_shell('instance') + #kafka_cluster.open_bash_shell('instance') assert TSV(instance.query('SELECT 1')) == TSV('1') From 471993b03fc9365ca60ccb2b49779b1a2f183406 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 14 May 2020 11:56:27 +0300 Subject: [PATCH 201/738] Fix build. --- src/DataStreams/MergingSortedBlockInputStream.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DataStreams/MergingSortedBlockInputStream.cpp b/src/DataStreams/MergingSortedBlockInputStream.cpp index 5d4328045dd..ae10617d45a 100644 --- a/src/DataStreams/MergingSortedBlockInputStream.cpp +++ b/src/DataStreams/MergingSortedBlockInputStream.cpp @@ -198,7 +198,7 @@ void MergingSortedBlockInputStream::merge(MutableColumns & merged_columns, TSort for (size_t i = 0; i < num_columns; ++i) { auto & column = merged_columns[i]; - column = (*column->cut(0, merged_rows)).mutate(); + column = IColumn::mutate(column->cut(0, merged_rows)); } cancel(false); From d68c385d35f78e322a7aebf074f09619a43757d8 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 14 May 2020 11:59:15 +0300 Subject: [PATCH 202/738] Remove old code. --- src/Common/COW.h | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/src/Common/COW.h b/src/Common/COW.h index 66b2bbde5b0..048bf134295 100644 --- a/src/Common/COW.h +++ b/src/Common/COW.h @@ -79,22 +79,12 @@ private: Derived * derived() { return static_cast(this); } const Derived * derived() const { return static_cast(this); } - template - class IntrusivePtr : public boost::intrusive_ptr - { - public: - using boost::intrusive_ptr::intrusive_ptr; - - T & operator*() const & { return boost::intrusive_ptr::operator*(); } - T && operator*() const && { return const_cast::type &&>(*boost::intrusive_ptr::get()); } - }; - protected: template - class mutable_ptr : public IntrusivePtr + class mutable_ptr : public boost::intrusive_ptr { private: - using Base = IntrusivePtr; + using Base = boost::intrusive_ptr; template friend class COW; template friend class COWHelper; @@ -123,10 +113,10 @@ public: protected: template - class immutable_ptr : public IntrusivePtr + class immutable_ptr : public boost::intrusive_ptr { private: - using Base = IntrusivePtr; + using Base = boost::intrusive_ptr; template friend class COW; template friend class COWHelper; From 757b0ecabb849e07d3669cbcae10b7313971d9ec Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 14 May 2020 12:26:40 +0300 Subject: [PATCH 203/738] Update boost submodule. --- contrib/boost | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/boost b/contrib/boost index 86be2aef20b..a04e72c0464 160000 --- a/contrib/boost +++ b/contrib/boost @@ -1 +1 @@ -Subproject commit 86be2aef20bee2356b744e5569eed6eaded85dbe +Subproject commit a04e72c0464f0c31d3384f18f0c0db36a05538e0 From 9b029eb31076aad865211bd0d7d11c0c69d52e61 Mon Sep 17 00:00:00 2001 From: Artem Zuikov Date: Thu, 14 May 2020 12:36:17 +0300 Subject: [PATCH 204/738] add OpenCL libraries for CI (#10898) --- docker/packager/deb/Dockerfile | 3 +++ docker/packager/deb/build.sh | 1 + 2 files changed, 4 insertions(+) diff --git a/docker/packager/deb/Dockerfile b/docker/packager/deb/Dockerfile index aac4d68a6df..78f396f5c75 100644 --- a/docker/packager/deb/Dockerfile +++ b/docker/packager/deb/Dockerfile @@ -57,6 +57,9 @@ RUN apt-get --allow-unauthenticated update -y \ libre2-dev \ libjemalloc-dev \ libmsgpack-dev \ + opencl-headers \ + ocl-icd-libopencl1 \ + intel-opencl-icd \ unixodbc-dev \ odbcinst \ tzdata \ diff --git a/docker/packager/deb/build.sh b/docker/packager/deb/build.sh index 6d5144266ae..1efed3628a0 100755 --- a/docker/packager/deb/build.sh +++ b/docker/packager/deb/build.sh @@ -11,3 +11,4 @@ mv *.buildinfo /output mv /*.rpm /output ||: # if exists mv /*.tgz /output ||: # if exists ccache --show-stats ||: +ln -s /usr/lib/x86_64-linux-gnu/libOpenCL.so.1.0.0 /usr/lib/libOpenCL.so ||: From 0ba4c373a98b9f592625fc31ce6227ddd374a92d Mon Sep 17 00:00:00 2001 From: Artem Zuikov Date: Thu, 14 May 2020 12:45:22 +0300 Subject: [PATCH 205/738] add OpenCL libraries for CI (part 2) --- docker/packager/binary/Dockerfile | 3 +++ docker/packager/binary/build.sh | 1 + 2 files changed, 4 insertions(+) diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index 784d2130333..34fb8f0ea30 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -41,6 +41,9 @@ RUN apt-get update -y \ ninja-build \ gperf \ git \ + opencl-headers \ + ocl-icd-libopencl1 \ + intel-opencl-icd \ tzdata \ gperf \ cmake \ diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index d1c295d4afa..0d1bdc2a88a 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -15,6 +15,7 @@ mkdir -p build/build_docker cd build/build_docker ccache --show-stats ||: ccache --zero-stats ||: +ln -s /usr/lib/x86_64-linux-gnu/libOpenCL.so.1.0.0 /usr/lib/libOpenCL.so ||: rm -f CMakeCache.txt cmake .. -LA -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DSANITIZE=$SANITIZER $CMAKE_FLAGS ninja From 6499d702f9ba32ff7caec7f4e252fc655e5cd6fb Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Thu, 14 May 2020 12:15:38 +0200 Subject: [PATCH 206/738] After review fixes --- src/Storages/Kafka/StorageKafka.cpp | 7 ++++++- tests/integration/test_storage_kafka/test.py | 16 ++++++++-------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index e3e28563906..2e2797983b6 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -235,14 +235,19 @@ ProducerBufferPtr StorageKafka::createWriteBuffer(const Block & header) ConsumerBufferPtr StorageKafka::createReadBuffer() { cppkafka::Configuration conf; + conf.set("metadata.broker.list", brokers); conf.set("group.id", group); conf.set("client.id", VERSION_FULL); + conf.set("auto.offset.reset", "smallest"); // If no offset stored for this group, read all messages from the start + + updateConfiguration(conf); + + // those settings should not be changed by users. conf.set("enable.auto.commit", "false"); // We manually commit offsets after a stream successfully finished conf.set("enable.auto.offset.store", "false"); // Update offset automatically - to commit them all at once. conf.set("enable.partition.eof", "false"); // Ignore EOF messages - updateConfiguration(conf); // Create a consumer and subscribe to topics auto consumer = std::make_shared(conf); diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py index 1c28d774688..cc384d27b36 100644 --- a/tests/integration/test_storage_kafka/test.py +++ b/tests/integration/test_storage_kafka/test.py @@ -238,17 +238,17 @@ def test_kafka_consumer_hang(kafka_cluster): DROP TABLE test.view; ''') - # we expect no hanging drop queries + # original problem appearance was a sequence of the following messages in kafka logs: + # BROKERFAIL -> |ASSIGN| -> REBALANCE_IN_PROGRESS -> "waiting for rebalance_cb" (repeated forever) + # so it was waiting forever while the application will execute queued rebalance callback + + # now we drain all queued callbacks (visible as 'Rebalance initiated' after 'Waiting for cleanup') + instance.exec_in_container(["bash", "-c", "tail -n 500 /var/log/clickhouse-server/clickhouse-server.log | grep 'Waiting for cleanup' -A 500 | grep -q 'Rebalance initiated. Revoking partitions'"]) + + # from a user perspective: we expect no hanging 'drop' queries # 'dr'||'op' to avoid self matching assert int(instance.query("select count() from system.processes where position(lower(query),'dr'||'op')>0")) == 0 - # log = '/var/log/clickhouse-server/stderr.log' - # instance.exec_in_container(['grep', '-q', 'BROKERFAIL', log]) - # instance.exec_in_container(['grep', '-q', '|ASSIGN|', log]) - # instance.exec_in_container(['grep', '-q', 'Heartbeat failed: REBALANCE_IN_PROGRESS: group is rebalancing', log]) - # instance.exec_in_container(['grep', '-q', 'Group "consumer_hang": waiting for rebalance_cb', log]) - - @pytest.mark.timeout(180) def test_kafka_csv_with_delimiter(kafka_cluster): instance.query(''' From f97c14f7bea31b58727fa89ea6028df8d30b56a8 Mon Sep 17 00:00:00 2001 From: filimonov <1549571+filimonov@users.noreply.github.com> Date: Thu, 14 May 2020 12:18:17 +0200 Subject: [PATCH 207/738] bit better comment --- tests/integration/test_storage_kafka/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py index cc384d27b36..f4bea12fc46 100644 --- a/tests/integration/test_storage_kafka/test.py +++ b/tests/integration/test_storage_kafka/test.py @@ -238,7 +238,7 @@ def test_kafka_consumer_hang(kafka_cluster): DROP TABLE test.view; ''') - # original problem appearance was a sequence of the following messages in kafka logs: + # original problem appearance was a sequence of the following messages in librdkafka logs: # BROKERFAIL -> |ASSIGN| -> REBALANCE_IN_PROGRESS -> "waiting for rebalance_cb" (repeated forever) # so it was waiting forever while the application will execute queued rebalance callback From 316773d853584c9056b6f2e1f423a5232694b3e8 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 14 May 2020 13:30:53 +0300 Subject: [PATCH 208/738] Better comment --- src/Storages/MergeTree/MergeTreeDataMergerMutator.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.h b/src/Storages/MergeTree/MergeTreeDataMergerMutator.h index 145798a1d87..78de7375d70 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.h +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.h @@ -143,8 +143,10 @@ private: MutationCommands & for_interpreter, MutationCommands & for_file_renames); - /// Apply commands to source_part i.e. remove some columns in source_part - /// and return set of files, that have to be removed from filesystem and checksums + /// Apply commands to source_part i.e. remove and rename some columns in + /// source_part and return set of files, that have to be removed or renamed + /// from filesystem and in-memory checksums. Ordered result is important, + /// because we can apply renames that affects each other: x -> z, y -> x. static NameToNameVector collectFilesForRenames(MergeTreeData::DataPartPtr source_part, const MutationCommands & commands_for_removes, const String & mrk_extension); /// Files, that we don't need to remove and don't need to hardlink, for example columns.txt and checksums.txt. From bb726b2d8db44340f71d8b67bd96a43df78488b7 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Thu, 14 May 2020 13:54:07 +0300 Subject: [PATCH 209/738] Update compare.sh --- docker/test/performance-comparison/compare.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 5e6846cb6b6..05c055107ac 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -347,7 +347,7 @@ create table test_time engine Memory as count(*) queries, sum(short) short_queries from query_time full join queries - on query_time.query = queries.query + using test, query group by test; create table test_times_tsv engine File(TSV, 'report/test-times.tsv') as From a14f32272362fa4c48c4e88bbad6e0f55cdeb682 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Tue, 12 May 2020 23:31:30 +0300 Subject: [PATCH 210/738] Add system tables for settings profiles. --- src/Access/SettingsProfilesCache.cpp | 46 ++-- .../InterpreterShowAccessEntitiesQuery.cpp | 4 + src/Parsers/ASTShowAccessEntitiesQuery.cpp | 31 ++- src/Parsers/ASTShowAccessEntitiesQuery.h | 4 + src/Parsers/ParserShowAccessEntitiesQuery.cpp | 4 + src/Parsers/ParserShowAccessEntitiesQuery.h | 1 + .../StorageSystemSettingsProfileElements.cpp | 216 ++++++++++++++++++ .../StorageSystemSettingsProfileElements.h | 24 ++ .../System/StorageSystemSettingsProfiles.cpp | 87 +++++++ .../System/StorageSystemSettingsProfiles.h | 24 ++ src/Storages/System/attachSystemTables.cpp | 26 ++- src/Storages/ya.make | 2 + .../integration/test_settings_profile/test.py | 60 ++++- 13 files changed, 473 insertions(+), 56 deletions(-) create mode 100644 src/Storages/System/StorageSystemSettingsProfileElements.cpp create mode 100644 src/Storages/System/StorageSystemSettingsProfileElements.h create mode 100644 src/Storages/System/StorageSystemSettingsProfiles.cpp create mode 100644 src/Storages/System/StorageSystemSettingsProfiles.h diff --git a/src/Access/SettingsProfilesCache.cpp b/src/Access/SettingsProfilesCache.cpp index 3f9399ca484..95074ff7d0b 100644 --- a/src/Access/SettingsProfilesCache.cpp +++ b/src/Access/SettingsProfilesCache.cpp @@ -150,32 +150,34 @@ void SettingsProfilesCache::mergeSettingsAndConstraintsFor(EnabledSettings & ena void SettingsProfilesCache::substituteProfiles(SettingsProfileElements & elements) const { - bool stop_substituting = false; boost::container::flat_set already_substituted; - while (!stop_substituting) + for (size_t i = 0; i != elements.size();) { - stop_substituting = true; - for (size_t i = 0; i != elements.size(); ++i) + auto & element = elements[i]; + if (!element.parent_profile) { - auto & element = elements[i]; - if (!element.parent_profile) - continue; - - auto parent_profile_id = *element.parent_profile; - element.parent_profile.reset(); - if (already_substituted.count(parent_profile_id)) - continue; - - already_substituted.insert(parent_profile_id); - auto parent_profile = all_profiles.find(parent_profile_id); - if (parent_profile == all_profiles.end()) - continue; - - const auto & parent_profile_elements = parent_profile->second->elements; - elements.insert(elements.begin() + i + 1, parent_profile_elements.begin(), parent_profile_elements.end()); - i += parent_profile_elements.size(); - stop_substituting = false; + ++i; + continue; } + + auto parent_profile_id = *element.parent_profile; + element.parent_profile.reset(); + if (already_substituted.count(parent_profile_id)) + { + ++i; + continue; + } + + already_substituted.insert(parent_profile_id); + auto parent_profile = all_profiles.find(parent_profile_id); + if (parent_profile == all_profiles.end()) + { + ++i; + continue; + } + + const auto & parent_profile_elements = parent_profile->second->elements; + elements.insert(elements.begin() + i, parent_profile_elements.begin(), parent_profile_elements.end()); } } diff --git a/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp b/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp index f6527d0ba8e..a7fb335aacb 100644 --- a/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp +++ b/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp @@ -70,6 +70,10 @@ String InterpreterShowAccessEntitiesQuery::getRewrittenQuery() const origin = "quotas"; } } + else if (query.type == EntityType::SETTINGS_PROFILE) + { + origin = "settings_profiles"; + } else throw Exception(toString(query.type) + ": type is not supported by SHOW query", ErrorCodes::NOT_IMPLEMENTED); diff --git a/src/Parsers/ASTShowAccessEntitiesQuery.cpp b/src/Parsers/ASTShowAccessEntitiesQuery.cpp index a1da1ac04e2..3fe88a2321c 100644 --- a/src/Parsers/ASTShowAccessEntitiesQuery.cpp +++ b/src/Parsers/ASTShowAccessEntitiesQuery.cpp @@ -10,24 +10,31 @@ namespace ErrorCodes } +const char * ASTShowAccessEntitiesQuery::getKeyword() const +{ + switch (type) + { + case EntityType::ROW_POLICY: + return "SHOW ROW POLICIES"; + case EntityType::QUOTA: + return current_quota ? "SHOW CURRENT QUOTA" : "SHOW QUOTAS"; + case EntityType::SETTINGS_PROFILE: + return "SHOW SETTINGS PROFILES"; + default: + throw Exception(toString(type) + ": type is not supported by SHOW query", ErrorCodes::NOT_IMPLEMENTED); + } +} + + String ASTShowAccessEntitiesQuery::getID(char) const { - if (type == EntityType::ROW_POLICY) - return "SHOW ROW POLICIES query"; - else if (type == EntityType::QUOTA) - return current_quota ? "SHOW CURRENT QUOTA query" : "SHOW QUOTAS query"; - else - throw Exception(toString(type) + ": type is not supported by SHOW query", ErrorCodes::NOT_IMPLEMENTED); + return String(getKeyword()) + " query"; } void ASTShowAccessEntitiesQuery::formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const { - if (type == EntityType::ROW_POLICY) - settings.ostr << (settings.hilite ? hilite_keyword : "") << "SHOW ROW POLICIES" << (settings.hilite ? hilite_none : ""); - else if (type == EntityType::QUOTA) - settings.ostr << (settings.hilite ? hilite_keyword : "") << (current_quota ? "SHOW CURRENT QUOTA" : "SHOW QUOTAS") << (settings.hilite ? hilite_none : ""); - else - throw Exception(toString(type) + ": type is not supported by SHOW query", ErrorCodes::NOT_IMPLEMENTED); + const char * keyword = getKeyword(); + settings.ostr << (settings.hilite ? hilite_keyword : "") << keyword << (settings.hilite ? hilite_none : ""); if ((type == EntityType::ROW_POLICY) && !table_name.empty()) { diff --git a/src/Parsers/ASTShowAccessEntitiesQuery.h b/src/Parsers/ASTShowAccessEntitiesQuery.h index 09ccbe9cf91..e41553d2e71 100644 --- a/src/Parsers/ASTShowAccessEntitiesQuery.h +++ b/src/Parsers/ASTShowAccessEntitiesQuery.h @@ -10,6 +10,7 @@ namespace DB /// SHOW [ROW] POLICIES [ON [database.]table] /// SHOW QUOTAS /// SHOW [CURRENT] QUOTA +/// SHOW [SETTINGS] PROFILES class ASTShowAccessEntitiesQuery : public ASTQueryWithOutput { public: @@ -25,6 +26,9 @@ public: protected: void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + +private: + const char * getKeyword() const; }; } diff --git a/src/Parsers/ParserShowAccessEntitiesQuery.cpp b/src/Parsers/ParserShowAccessEntitiesQuery.cpp index 7ca9782dd2c..f70a3ea189c 100644 --- a/src/Parsers/ParserShowAccessEntitiesQuery.cpp +++ b/src/Parsers/ParserShowAccessEntitiesQuery.cpp @@ -43,6 +43,10 @@ bool ParserShowAccessEntitiesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected type = EntityType::QUOTA; current_quota = true; } + else if (ParserKeyword{"PROFILES"}.ignore(pos, expected) || ParserKeyword{"SETTINGS PROFILES"}.ignore(pos, expected)) + { + type = EntityType::SETTINGS_PROFILE; + } else return false; diff --git a/src/Parsers/ParserShowAccessEntitiesQuery.h b/src/Parsers/ParserShowAccessEntitiesQuery.h index 77838e726e7..bb8b37f40e8 100644 --- a/src/Parsers/ParserShowAccessEntitiesQuery.h +++ b/src/Parsers/ParserShowAccessEntitiesQuery.h @@ -9,6 +9,7 @@ namespace DB * SHOW [ROW] POLICIES [ON [database.]table] SHOW QUOTAS SHOW [CURRENT] QUOTA + SHOW [SETTINGS] PROFILES */ class ParserShowAccessEntitiesQuery : public IParserBase { diff --git a/src/Storages/System/StorageSystemSettingsProfileElements.cpp b/src/Storages/System/StorageSystemSettingsProfileElements.cpp new file mode 100644 index 00000000000..2e4d1ad1e05 --- /dev/null +++ b/src/Storages/System/StorageSystemSettingsProfileElements.cpp @@ -0,0 +1,216 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +using EntityType = IAccessEntity::Type; + + +NamesAndTypesList StorageSystemSettingsProfileElements::getNamesAndTypes() +{ + NamesAndTypesList names_and_types{ + {"profile_name", std::make_shared(std::make_shared())}, + {"user_name", std::make_shared(std::make_shared())}, + {"role_name", std::make_shared(std::make_shared())}, + {"index", std::make_shared()}, + {"setting_name", std::make_shared(std::make_shared())}, + {"value", std::make_shared(std::make_shared())}, + {"min", std::make_shared(std::make_shared())}, + {"max", std::make_shared(std::make_shared())}, + {"readonly", std::make_shared(std::make_shared())}, + {"inherit_profile", std::make_shared(std::make_shared())}, + }; + return names_and_types; +} + + +void StorageSystemSettingsProfileElements::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const +{ + context.checkAccess(AccessType::SHOW_SETTINGS_PROFILES); + const auto & access_control = context.getAccessControlManager(); + std::vector ids = access_control.findAll(); + boost::range::push_back(ids, access_control.findAll()); + boost::range::push_back(ids, access_control.findAll()); + + size_t i = 0; + auto & column_profile_name = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()); + auto & column_profile_name_null_map = assert_cast(*res_columns[i++]).getNullMapData(); + auto & column_user_name = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()); + auto & column_user_name_null_map = assert_cast(*res_columns[i++]).getNullMapData(); + auto & column_role_name = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()); + auto & column_role_name_null_map = assert_cast(*res_columns[i++]).getNullMapData(); + auto & column_index = assert_cast(*res_columns[i++]).getData(); + auto & column_setting_name = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()); + auto & column_setting_name_null_map = assert_cast(*res_columns[i++]).getNullMapData(); + auto & column_value = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()); + auto & column_value_null_map = assert_cast(*res_columns[i++]).getNullMapData(); + auto & column_min = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()); + auto & column_min_null_map = assert_cast(*res_columns[i++]).getNullMapData(); + auto & column_max = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()); + auto & column_max_null_map = assert_cast(*res_columns[i++]).getNullMapData(); + auto & column_readonly = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()).getData(); + auto & column_readonly_null_map = assert_cast(*res_columns[i++]).getNullMapData(); + auto & column_inherit_profile = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()); + auto & column_inherit_profile_null_map = assert_cast(*res_columns[i++]).getNullMapData(); + + auto add_rows_for_single_element = [&](const String & owner_name, EntityType owner_type, const SettingsProfileElement & element, size_t & index) + { + switch (owner_type) + { + case EntityType::SETTINGS_PROFILE: + { + column_user_name.insertDefault(); + column_user_name_null_map.push_back(true); + column_role_name.insertDefault(); + column_role_name_null_map.push_back(true); + column_profile_name.insertData(owner_name.data(), owner_name.length()); + column_profile_name_null_map.push_back(false); + break; + } + case EntityType::USER: + { + column_user_name.insertData(owner_name.data(), owner_name.length()); + column_user_name_null_map.push_back(false); + column_profile_name.insertDefault(); + column_profile_name_null_map.push_back(true); + column_role_name.insertDefault(); + column_role_name_null_map.push_back(true); + break; + } + case EntityType::ROLE: + { + column_user_name.insertDefault(); + column_user_name_null_map.push_back(true); + column_role_name.insertData(owner_name.data(), owner_name.length()); + column_role_name_null_map.push_back(false); + column_profile_name.insertDefault(); + column_profile_name_null_map.push_back(true); + break; + } + default: + assert(false); + } + + if (element.parent_profile) + { + auto parent_profile = access_control.tryReadName(*element.parent_profile); + if (parent_profile) + { + column_index.push_back(index++); + column_setting_name.insertDefault(); + column_setting_name_null_map.push_back(true); + column_value.insertDefault(); + column_value_null_map.push_back(true); + column_min.insertDefault(); + column_min_null_map.push_back(true); + column_max.insertDefault(); + column_max_null_map.push_back(true); + column_readonly.push_back(0); + column_readonly_null_map.push_back(true); + const String & parent_profile_str = *parent_profile; + column_inherit_profile.insertData(parent_profile_str.data(), parent_profile_str.length()); + column_inherit_profile_null_map.push_back(false); + } + } + + if ((element.setting_index != static_cast(-1)) + && (!element.value.isNull() || !element.min_value.isNull() || !element.max_value.isNull() || element.readonly)) + { + auto setting_name = Settings::getName(element.setting_index); + column_index.push_back(index++); + column_setting_name.insertData(setting_name.data, setting_name.size); + column_setting_name_null_map.push_back(false); + + if (element.value.isNull()) + { + column_value.insertDefault(); + column_value_null_map.push_back(true); + } + else + { + String str = Settings::valueToString(element.setting_index, element.value); + column_value.insertData(str.data(), str.length()); + column_value_null_map.push_back(false); + } + + if (element.min_value.isNull()) + { + column_min.insertDefault(); + column_min_null_map.push_back(true); + } + else + { + String str = Settings::valueToString(element.setting_index, element.min_value); + column_min.insertData(str.data(), str.length()); + column_min_null_map.push_back(false); + } + + if (element.max_value.isNull()) + { + column_max.insertDefault(); + column_max_null_map.push_back(true); + } + else + { + String str = Settings::valueToString(element.setting_index, element.max_value); + column_max.insertData(str.data(), str.length()); + column_max_null_map.push_back(false); + } + + if (element.readonly) + { + column_readonly.push_back(*element.readonly); + column_readonly_null_map.push_back(false); + } + else + { + column_readonly.push_back(0); + column_readonly_null_map.push_back(true); + } + + column_inherit_profile.insertDefault(); + column_inherit_profile_null_map.push_back(true); + } + }; + + auto add_rows = [&](const String & owner_name, IAccessEntity::Type owner_type, const SettingsProfileElements & elements) + { + size_t index = 0; + for (const auto & element : elements) + add_rows_for_single_element(owner_name, owner_type, element, index); + }; + + for (const auto & id : ids) + { + auto entity = access_control.tryRead(id); + if (!entity) + continue; + + const SettingsProfileElements * settings = nullptr; + if (auto role = typeid_cast(entity)) + settings = &role->settings; + else if (auto user = typeid_cast(entity)) + settings = &user->settings; + else if (auto profile = typeid_cast(entity)) + settings = &profile->elements; + else + continue; + + add_rows(entity->getName(), entity->getType(), *settings); + } +} + +} diff --git a/src/Storages/System/StorageSystemSettingsProfileElements.h b/src/Storages/System/StorageSystemSettingsProfileElements.h new file mode 100644 index 00000000000..2dc79fed0e7 --- /dev/null +++ b/src/Storages/System/StorageSystemSettingsProfileElements.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class Context; + +/// Implements `settings_profile_elements` system table, which allows you to get information about elements of settings profiles. +class StorageSystemSettingsProfileElements final : public ext::shared_ptr_helper, public IStorageSystemOneBlock +{ +public: + std::string getName() const override { return "SystemSettingsProfileElements"; } + static NamesAndTypesList getNamesAndTypes(); + +protected: + friend struct ext::shared_ptr_helper; + using IStorageSystemOneBlock::IStorageSystemOneBlock; + void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const override; +}; + +} diff --git a/src/Storages/System/StorageSystemSettingsProfiles.cpp b/src/Storages/System/StorageSystemSettingsProfiles.cpp new file mode 100644 index 00000000000..d02c5910608 --- /dev/null +++ b/src/Storages/System/StorageSystemSettingsProfiles.cpp @@ -0,0 +1,87 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +NamesAndTypesList StorageSystemSettingsProfiles::getNamesAndTypes() +{ + NamesAndTypesList names_and_types{ + {"name", std::make_shared()}, + {"id", std::make_shared()}, + {"storage", std::make_shared()}, + {"num_elements", std::make_shared()}, + {"apply_to_all", std::make_shared()}, + {"apply_to_list", std::make_shared(std::make_shared())}, + {"apply_to_except", std::make_shared(std::make_shared())}, + }; + return names_and_types; +} + + +void StorageSystemSettingsProfiles::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const +{ + context.checkAccess(AccessType::SHOW_SETTINGS_PROFILES); + const auto & access_control = context.getAccessControlManager(); + std::vector ids = access_control.findAll(); + + size_t column_index = 0; + auto & column_name = assert_cast(*res_columns[column_index++]); + auto & column_id = assert_cast(*res_columns[column_index++]).getData(); + auto & column_storage = assert_cast(*res_columns[column_index++]); + auto & column_num_elements = assert_cast(*res_columns[column_index++]).getData(); + auto & column_apply_to_all = assert_cast(*res_columns[column_index++]).getData(); + auto & column_apply_to_list = assert_cast(assert_cast(*res_columns[column_index]).getData()); + auto & column_apply_to_list_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); + auto & column_apply_to_except = assert_cast(assert_cast(*res_columns[column_index]).getData()); + auto & column_apply_to_except_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); + + auto add_row = [&](const String & name, + const UUID & id, + const String & storage_name, + const SettingsProfileElements & elements, + const ExtendedRoleSet & apply_to) + { + column_name.insertData(name.data(), name.length()); + column_id.push_back(id); + column_storage.insertData(storage_name.data(), storage_name.length()); + column_num_elements.push_back(elements.size()); + + auto apply_to_ast = apply_to.toASTWithNames(access_control); + column_apply_to_all.push_back(apply_to_ast->all); + + for (const auto & role_name : apply_to_ast->names) + column_apply_to_list.insertData(role_name.data(), role_name.length()); + column_apply_to_list_offsets.push_back(column_apply_to_list.size()); + + for (const auto & role_name : apply_to_ast->except_names) + column_apply_to_except.insertData(role_name.data(), role_name.length()); + column_apply_to_except_offsets.push_back(column_apply_to_except.size()); + }; + + for (const auto & id : ids) + { + auto profile = access_control.tryRead(id); + if (!profile) + continue; + + const auto * storage = access_control.findStorage(id); + if (!storage) + continue; + + add_row(profile->getName(), id, storage->getStorageName(), profile->elements, profile->to_roles); + } +} + +} diff --git a/src/Storages/System/StorageSystemSettingsProfiles.h b/src/Storages/System/StorageSystemSettingsProfiles.h new file mode 100644 index 00000000000..c6b887c99df --- /dev/null +++ b/src/Storages/System/StorageSystemSettingsProfiles.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class Context; + +/// Implements `settings_profiles` system table, which allows you to get information about profiles. +class StorageSystemSettingsProfiles final : public ext::shared_ptr_helper, public IStorageSystemOneBlock +{ +public: + std::string getName() const override { return "SystemSettingsProfiles"; } + static NamesAndTypesList getNamesAndTypes(); + +protected: + friend struct ext::shared_ptr_helper; + using IStorageSystemOneBlock::IStorageSystemOneBlock; + void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const override; +}; + +} diff --git a/src/Storages/System/attachSystemTables.cpp b/src/Storages/System/attachSystemTables.cpp index 69ad147664f..d30f4f077c8 100644 --- a/src/Storages/System/attachSystemTables.cpp +++ b/src/Storages/System/attachSystemTables.cpp @@ -15,6 +15,7 @@ #include #include #include + #include #include #include @@ -25,13 +26,8 @@ #include #include #include -#include -#include -#include -#include #include #include -#include #include #include #include @@ -46,6 +42,14 @@ #include #include +#include +#include +#include +#include +#include +#include +#include + #ifdef OS_LINUX #include #endif @@ -68,11 +72,6 @@ void attachSystemTablesLocal(IDatabase & system_database) system_database.attachTable("functions", StorageSystemFunctions::create("functions")); system_database.attachTable("events", StorageSystemEvents::create("events")); system_database.attachTable("settings", StorageSystemSettings::create("settings")); - system_database.attachTable("quotas", StorageSystemQuotas::create("quotas")); - system_database.attachTable("quota_limits", StorageSystemQuotaLimits::create("quota_limits")); - system_database.attachTable("quota_usage", StorageSystemQuotaUsage::create("quota_usage")); - system_database.attachTable("quotas_usage", StorageSystemQuotasUsage::create("all_quotas_usage")); - system_database.attachTable("row_policies", StorageSystemRowPolicies::create("row_policies")); system_database.attachTable("merge_tree_settings", SystemMergeTreeSettings::create("merge_tree_settings")); system_database.attachTable("build_options", StorageSystemBuildOptions::create("build_options")); system_database.attachTable("formats", StorageSystemFormats::create("formats")); @@ -82,6 +81,13 @@ void attachSystemTablesLocal(IDatabase & system_database) system_database.attachTable("collations", StorageSystemCollations::create("collations")); system_database.attachTable("table_engines", StorageSystemTableEngines::create("table_engines")); system_database.attachTable("contributors", StorageSystemContributors::create("contributors")); + system_database.attachTable("settings_profiles", StorageSystemSettingsProfiles::create("settings_profiles")); + system_database.attachTable("settings_profile_elements", StorageSystemSettingsProfileElements::create("settings_profile_elements")); + system_database.attachTable("row_policies", StorageSystemRowPolicies::create("row_policies")); + system_database.attachTable("quotas", StorageSystemQuotas::create("quotas")); + system_database.attachTable("quota_limits", StorageSystemQuotaLimits::create("quota_limits")); + system_database.attachTable("quota_usage", StorageSystemQuotaUsage::create("quota_usage")); + system_database.attachTable("quotas_usage", StorageSystemQuotasUsage::create("all_quotas_usage")); #if !defined(ARCADIA_BUILD) system_database.attachTable("licenses", StorageSystemLicenses::create("licenses")); #endif diff --git a/src/Storages/ya.make b/src/Storages/ya.make index 2e00dbaec97..85b61aca5a0 100644 --- a/src/Storages/ya.make +++ b/src/Storages/ya.make @@ -119,6 +119,8 @@ SRCS( System/StorageSystemReplicationQueue.cpp System/StorageSystemRowPolicies.cpp System/StorageSystemSettings.cpp + System/StorageSystemSettingsProfileElements.cpp + System/StorageSystemSettingsProfiles.cpp System/StorageSystemStackTrace.cpp System/StorageSystemStoragePolicies.cpp System/StorageSystemTableEngines.cpp diff --git a/tests/integration/test_settings_profile/test.py b/tests/integration/test_settings_profile/test.py index 8b9d023d56f..9c8b116e6d2 100644 --- a/tests/integration/test_settings_profile/test.py +++ b/tests/integration/test_settings_profile/test.py @@ -1,10 +1,25 @@ import pytest from helpers.cluster import ClickHouseCluster +from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) instance = cluster.add_instance('instance') +def system_settings_profile(profile_name): + return TSV(instance.query("SELECT name, storage, num_elements, apply_to_all, apply_to_list, apply_to_except FROM system.settings_profiles WHERE name='" + profile_name + "'")) + +def system_settings_profile_elements(profile_name=None, user_name=None, role_name=None): + where = "" + if profile_name: + where = " WHERE profile_name='" + profile_name + "'" + elif user_name: + where = " WHERE user_name='" + user_name + "'" + elif role_name: + where = " WHERE role_name='" + role_name + "'" + return TSV(instance.query("SELECT * FROM system.settings_profile_elements" + where)) + + @pytest.fixture(scope="module", autouse=True) def setup_nodes(): try: @@ -28,19 +43,23 @@ def reset_after_test(): instance.query("DROP SETTINGS PROFILE IF EXISTS xyz, alpha") -def test_settings_profile(): +def test_smoke(): # Set settings and constraints via CREATE SETTINGS PROFILE ... TO user instance.query("CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000 TO robin") assert instance.query("SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000 TO robin\n" assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "100000001\n" assert "Setting max_memory_usage shouldn't be less than 90000000" in instance.query_and_get_error("SET max_memory_usage = 80000000", user="robin") assert "Setting max_memory_usage shouldn't be greater than 110000000" in instance.query_and_get_error("SET max_memory_usage = 120000000", user="robin") + assert system_settings_profile("xyz") == [[ "xyz", "disk", 1, 0, "['robin']", "[]" ]] + assert system_settings_profile_elements(profile_name="xyz") == [[ "xyz", "\N", "\N", 0, "max_memory_usage", 100000001, 90000000, 110000000, "\N", "\N" ]] instance.query("ALTER SETTINGS PROFILE xyz TO NONE") assert instance.query("SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000\n" assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "10000000000\n" instance.query("SET max_memory_usage = 80000000", user="robin") instance.query("SET max_memory_usage = 120000000", user="robin") + assert system_settings_profile("xyz") == [[ "xyz", "disk", 1, 0, "[]", "[]" ]] + assert system_settings_profile_elements(user_name="robin") == [] # Set settings and constraints via CREATE USER ... SETTINGS PROFILE instance.query("ALTER USER robin SETTINGS PROFILE xyz") @@ -48,52 +67,57 @@ def test_settings_profile(): assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "100000001\n" assert "Setting max_memory_usage shouldn't be less than 90000000" in instance.query_and_get_error("SET max_memory_usage = 80000000", user="robin") assert "Setting max_memory_usage shouldn't be greater than 110000000" in instance.query_and_get_error("SET max_memory_usage = 120000000", user="robin") + assert system_settings_profile_elements(user_name="robin") == [[ "\N", "robin", "\N", 0, "\N", "\N", "\N", "\N", "\N", "xyz" ]] instance.query("ALTER USER robin SETTINGS NONE") assert instance.query("SHOW CREATE USER robin") == "CREATE USER robin\n" assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "10000000000\n" instance.query("SET max_memory_usage = 80000000", user="robin") instance.query("SET max_memory_usage = 120000000", user="robin") + assert system_settings_profile_elements(user_name="robin") == [] -def test_settings_profile_from_granted_role(): +def test_settings_from_granted_role(): # Set settings and constraints via granted role - instance.query("CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000") + instance.query("CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MAX 110000000, max_ast_depth = 2000") instance.query("CREATE ROLE worker SETTINGS PROFILE xyz") instance.query("GRANT worker TO robin") - assert instance.query("SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000\n" + assert instance.query("SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MAX 110000000, max_ast_depth = 2000\n" assert instance.query("SHOW CREATE ROLE worker") == "CREATE ROLE worker SETTINGS PROFILE xyz\n" assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "100000001\n" - assert "Setting max_memory_usage shouldn't be less than 90000000" in instance.query_and_get_error("SET max_memory_usage = 80000000", user="robin") + assert instance.query("SELECT value FROM system.settings WHERE name = 'max_ast_depth'", user="robin") == "2000\n" assert "Setting max_memory_usage shouldn't be greater than 110000000" in instance.query_and_get_error("SET max_memory_usage = 120000000", user="robin") + assert system_settings_profile("xyz") == [[ "xyz", "disk", 2, 0, "[]", "[]" ]] + assert system_settings_profile_elements(profile_name="xyz") == [[ "xyz", "\N", "\N", 0, "max_memory_usage", 100000001, "\N", 110000000, "\N", "\N" ], + [ "xyz", "\N", "\N", 1, "max_ast_depth", 2000, "\N", "\N", "\N", "\N" ]] + assert system_settings_profile_elements(role_name="worker") == [[ "\N", "\N", "worker", 0, "\N", "\N", "\N", "\N", "\N", "xyz" ]] instance.query("REVOKE worker FROM robin") assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "10000000000\n" - instance.query("SET max_memory_usage = 80000000", user="robin") instance.query("SET max_memory_usage = 120000000", user="robin") instance.query("ALTER ROLE worker SETTINGS NONE") instance.query("GRANT worker TO robin") assert instance.query("SHOW CREATE ROLE worker") == "CREATE ROLE worker\n" assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "10000000000\n" - instance.query("SET max_memory_usage = 80000000", user="robin") instance.query("SET max_memory_usage = 120000000", user="robin") + assert system_settings_profile_elements(role_name="worker") == [] # Set settings and constraints via CREATE SETTINGS PROFILE ... TO granted role instance.query("ALTER SETTINGS PROFILE xyz TO worker") - assert instance.query("SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000 TO worker\n" + assert instance.query("SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MAX 110000000, max_ast_depth = 2000 TO worker\n" assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "100000001\n" - assert "Setting max_memory_usage shouldn't be less than 90000000" in instance.query_and_get_error("SET max_memory_usage = 80000000", user="robin") assert "Setting max_memory_usage shouldn't be greater than 110000000" in instance.query_and_get_error("SET max_memory_usage = 120000000", user="robin") + assert system_settings_profile("xyz") == [[ "xyz", "disk", 2, 0, "['worker']", "[]" ]] instance.query("ALTER SETTINGS PROFILE xyz TO NONE") - assert instance.query("SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000\n" + assert instance.query("SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MAX 110000000, max_ast_depth = 2000\n" assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "10000000000\n" - instance.query("SET max_memory_usage = 80000000", user="robin") instance.query("SET max_memory_usage = 120000000", user="robin") + assert system_settings_profile("xyz") == [[ "xyz", "disk", 2, 0, "[]", "[]" ]] -def test_inheritance_of_settings_profile(): +def test_inheritance(): instance.query("CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000002 READONLY") instance.query("CREATE SETTINGS PROFILE alpha SETTINGS PROFILE xyz TO robin") assert instance.query("SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000002 READONLY\n" @@ -101,6 +125,12 @@ def test_inheritance_of_settings_profile(): assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "100000002\n" assert "Setting max_memory_usage should not be changed" in instance.query_and_get_error("SET max_memory_usage = 80000000", user="robin") + assert system_settings_profile("xyz") == [[ "xyz", "disk", 1, 0, "[]", "[]" ]] + assert system_settings_profile_elements(profile_name="xyz") == [[ "xyz", "\N", "\N", 0, "max_memory_usage", 100000002, "\N", "\N", 1, "\N" ]] + assert system_settings_profile("alpha") == [[ "alpha", "disk", 1, 0, "['robin']", "[]" ]] + assert system_settings_profile_elements(profile_name="alpha") == [[ "alpha", "\N", "\N", 0, "\N", "\N", "\N", "\N", "\N", "xyz" ]] + assert system_settings_profile_elements(user_name="robin") == [] + def test_alter_and_drop(): instance.query("CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000003 MIN 90000000 MAX 110000000 TO robin") @@ -117,6 +147,12 @@ def test_alter_and_drop(): instance.query("SET max_memory_usage = 120000000", user="robin") +def test_show_profiles(): + instance.query("CREATE SETTINGS PROFILE xyz") + assert instance.query("SHOW SETTINGS PROFILES") == "default\nreadonly\nxyz\n" + assert instance.query("SHOW PROFILES") == "default\nreadonly\nxyz\n" + + def test_allow_introspection(): assert "Not enough privileges" in instance.query_and_get_error("SELECT demangle('a')", user="robin") From b1f362b2dcfc83efa2df3abed38d0af9605c0133 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Wed, 13 May 2020 00:46:14 +0300 Subject: [PATCH 211/738] Add system table for access types. --- src/Interpreters/InterpreterFactory.cpp | 6 + .../InterpreterShowPrivilegesQuery.cpp | 18 +++ .../InterpreterShowPrivilegesQuery.h | 26 ++++ src/Interpreters/ya.make | 1 + src/Parsers/ASTShowPrivilegesQuery.cpp | 0 src/Parsers/ASTShowPrivilegesQuery.h | 17 +++ src/Parsers/ParserQueryWithOutput.cpp | 5 +- src/Parsers/ParserShowPrivilegesQuery.cpp | 21 +++ src/Parsers/ParserShowPrivilegesQuery.h | 18 +++ src/Parsers/ya.make | 2 + .../System/StorageSystemPrivileges.cpp | 137 ++++++++++++++++++ src/Storages/System/StorageSystemPrivileges.h | 25 ++++ src/Storages/System/attachSystemTables.cpp | 2 + src/Storages/ya.make | 1 + .../01271_show_privileges.reference | 113 +++++++++++++++ .../0_stateless/01271_show_privileges.sql | 1 + 16 files changed, 392 insertions(+), 1 deletion(-) create mode 100644 src/Interpreters/InterpreterShowPrivilegesQuery.cpp create mode 100644 src/Interpreters/InterpreterShowPrivilegesQuery.h create mode 100644 src/Parsers/ASTShowPrivilegesQuery.cpp create mode 100644 src/Parsers/ASTShowPrivilegesQuery.h create mode 100644 src/Parsers/ParserShowPrivilegesQuery.cpp create mode 100644 src/Parsers/ParserShowPrivilegesQuery.h create mode 100644 src/Storages/System/StorageSystemPrivileges.cpp create mode 100644 src/Storages/System/StorageSystemPrivileges.h create mode 100644 tests/queries/0_stateless/01271_show_privileges.reference create mode 100644 tests/queries/0_stateless/01271_show_privileges.sql diff --git a/src/Interpreters/InterpreterFactory.cpp b/src/Interpreters/InterpreterFactory.cpp index c740da0601d..60302848367 100644 --- a/src/Interpreters/InterpreterFactory.cpp +++ b/src/Interpreters/InterpreterFactory.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -52,6 +53,7 @@ #include #include #include +#include #include #include #include @@ -228,6 +230,10 @@ std::unique_ptr InterpreterFactory::get(ASTPtr & query, Context & { return std::make_unique(query, context); } + else if (query->as()) + { + return std::make_unique(query, context); + } else throw Exception("Unknown type of query: " + query->getID(), ErrorCodes::UNKNOWN_TYPE_OF_QUERY); } diff --git a/src/Interpreters/InterpreterShowPrivilegesQuery.cpp b/src/Interpreters/InterpreterShowPrivilegesQuery.cpp new file mode 100644 index 00000000000..e88b97d8671 --- /dev/null +++ b/src/Interpreters/InterpreterShowPrivilegesQuery.cpp @@ -0,0 +1,18 @@ +#include +#include + + +namespace DB +{ +InterpreterShowPrivilegesQuery::InterpreterShowPrivilegesQuery(const ASTPtr & query_ptr_, Context & context_) + : query_ptr(query_ptr_), context(context_) +{ +} + + +BlockIO InterpreterShowPrivilegesQuery::execute() +{ + return executeQuery("SELECT * FROM system.privileges", context, true); +} + +} diff --git a/src/Interpreters/InterpreterShowPrivilegesQuery.h b/src/Interpreters/InterpreterShowPrivilegesQuery.h new file mode 100644 index 00000000000..0c1a30107c0 --- /dev/null +++ b/src/Interpreters/InterpreterShowPrivilegesQuery.h @@ -0,0 +1,26 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class Context; + +class InterpreterShowPrivilegesQuery : public IInterpreter +{ +public: + InterpreterShowPrivilegesQuery(const ASTPtr & query_ptr_, Context & context_); + + BlockIO execute() override; + + bool ignoreQuota() const override { return true; } + bool ignoreLimits() const override { return true; } + +private: + ASTPtr query_ptr; + Context & context; +}; + +} diff --git a/src/Interpreters/ya.make b/src/Interpreters/ya.make index 7c5a86aa1cb..b210a1c5b8c 100644 --- a/src/Interpreters/ya.make +++ b/src/Interpreters/ya.make @@ -87,6 +87,7 @@ SRCS( InterpreterShowCreateAccessEntityQuery.cpp InterpreterShowCreateQuery.cpp InterpreterShowGrantsQuery.cpp + InterpreterShowPrivilegesQuery.cpp InterpreterShowProcesslistQuery.cpp InterpreterShowTablesQuery.cpp InterpreterSystemQuery.cpp diff --git a/src/Parsers/ASTShowPrivilegesQuery.cpp b/src/Parsers/ASTShowPrivilegesQuery.cpp new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/Parsers/ASTShowPrivilegesQuery.h b/src/Parsers/ASTShowPrivilegesQuery.h new file mode 100644 index 00000000000..8cc4ed16f96 --- /dev/null +++ b/src/Parsers/ASTShowPrivilegesQuery.h @@ -0,0 +1,17 @@ +#pragma once + +#include + + +namespace DB +{ + +struct ASTShowPrivilegesIDAndQueryName +{ + static constexpr auto ID = "ShowPrivilegesQuery"; + static constexpr auto Query = "SHOW PRIVILEGES"; +}; + +using ASTShowPrivilegesQuery = ASTQueryWithOutputImpl; + +} diff --git a/src/Parsers/ParserQueryWithOutput.cpp b/src/Parsers/ParserQueryWithOutput.cpp index 83967ae9922..a81305c0557 100644 --- a/src/Parsers/ParserQueryWithOutput.cpp +++ b/src/Parsers/ParserQueryWithOutput.cpp @@ -17,6 +17,7 @@ #include #include #include +#include namespace DB @@ -40,6 +41,7 @@ bool ParserQueryWithOutput::parseImpl(Pos & pos, ASTPtr & node, Expected & expec ParserShowAccessEntitiesQuery show_access_entities_p; ParserShowCreateAccessEntityQuery show_create_access_entity_p; ParserShowGrantsQuery show_grants_p; + ParserShowPrivilegesQuery show_privileges_p; ASTPtr query; @@ -69,7 +71,8 @@ bool ParserQueryWithOutput::parseImpl(Pos & pos, ASTPtr & node, Expected & expec || optimize_p.parse(pos, query, expected) || watch_p.parse(pos, query, expected) || show_access_entities_p.parse(pos, query, expected) - || show_grants_p.parse(pos, query, expected); + || show_grants_p.parse(pos, query, expected) + || show_privileges_p.parse(pos, query, expected); if (!parsed) return false; diff --git a/src/Parsers/ParserShowPrivilegesQuery.cpp b/src/Parsers/ParserShowPrivilegesQuery.cpp new file mode 100644 index 00000000000..56b4327dccf --- /dev/null +++ b/src/Parsers/ParserShowPrivilegesQuery.cpp @@ -0,0 +1,21 @@ +#include +#include +#include + + +namespace DB +{ + +bool ParserShowPrivilegesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + auto query = std::make_shared(); + + if (!ParserKeyword("SHOW PRIVILEGES").ignore(pos, expected)) + return false; + + node = query; + + return true; +} + +} diff --git a/src/Parsers/ParserShowPrivilegesQuery.h b/src/Parsers/ParserShowPrivilegesQuery.h new file mode 100644 index 00000000000..2604e7f28c1 --- /dev/null +++ b/src/Parsers/ParserShowPrivilegesQuery.h @@ -0,0 +1,18 @@ +#pragma once + +#include + + +namespace DB +{ + +/** Query SHOW PRIVILEGES + */ +class ParserShowPrivilegesQuery : public IParserBase +{ +protected: + const char * getName() const override { return "SHOW PRIVILEGES query"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; + +} diff --git a/src/Parsers/ya.make b/src/Parsers/ya.make index 2862b02a2c7..8c7e4ff68af 100644 --- a/src/Parsers/ya.make +++ b/src/Parsers/ya.make @@ -45,6 +45,7 @@ SRCS( ASTShowAccessEntitiesQuery.cpp ASTShowCreateAccessEntityQuery.cpp ASTShowGrantsQuery.cpp + ASTShowPrivilegesQuery.cpp ASTShowTablesQuery.cpp ASTSubquery.cpp ASTSystemQuery.cpp @@ -96,6 +97,7 @@ SRCS( ParserShowAccessEntitiesQuery.cpp ParserShowCreateAccessEntityQuery.cpp ParserShowGrantsQuery.cpp + ParserShowPrivilegesQuery.cpp ParserShowTablesQuery.cpp ParserSystemQuery.cpp ParserTablePropertiesQuery.cpp diff --git a/src/Storages/System/StorageSystemPrivileges.cpp b/src/Storages/System/StorageSystemPrivileges.cpp new file mode 100644 index 00000000000..f0a1b21368e --- /dev/null +++ b/src/Storages/System/StorageSystemPrivileges.cpp @@ -0,0 +1,137 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +namespace +{ + enum Level + { + GROUP = -1, + GLOBAL, + DATABASE, + TABLE, + DICTIONARY, + VIEW, + COLUMN, + }; + + DataTypeEnum8::Values getLevelEnumValues() + { + DataTypeEnum8::Values enum_values; + enum_values.emplace_back("GLOBAL", static_cast(GLOBAL)); + enum_values.emplace_back("DATABASE", static_cast(DATABASE)); + enum_values.emplace_back("TABLE", static_cast(TABLE)); + enum_values.emplace_back("DICTIONARY", static_cast(DICTIONARY)); + enum_values.emplace_back("VIEW", static_cast(VIEW)); + enum_values.emplace_back("COLUMN", static_cast(COLUMN)); + return enum_values; + } +} + + +const std::vector> & StorageSystemPrivileges::getAccessTypeEnumValues() +{ + static const std::vector> values = [] + { + std::vector> res; + +#define ADD_ACCESS_TYPE_ENUM_VALUE(name, aliases, node_type, parent_group_name) \ + res.emplace_back(toString(AccessType::name), static_cast(AccessType::name)); + + APPLY_FOR_ACCESS_TYPES(ADD_ACCESS_TYPE_ENUM_VALUE) +#undef ADD_ACCESS_TYPE_ENUM_VALUE + + return res; + }(); + return values; +} + + +NamesAndTypesList StorageSystemPrivileges::getNamesAndTypes() +{ + NamesAndTypesList names_and_types{ + {"privilege", std::make_shared(getAccessTypeEnumValues())}, + {"aliases", std::make_shared(std::make_shared())}, + {"level", std::make_shared(std::make_shared(getLevelEnumValues()))}, + {"parent_group", std::make_shared(std::make_shared(getAccessTypeEnumValues()))}, + }; + return names_and_types; +} + + +void StorageSystemPrivileges::fillData(MutableColumns & res_columns, const Context &, const SelectQueryInfo &) const +{ + size_t column_index = 0; + auto & column_access_type = assert_cast(*res_columns[column_index++]).getData(); + auto & column_aliases = assert_cast(assert_cast(*res_columns[column_index]).getData()); + auto & column_aliases_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); + auto & column_level = assert_cast(assert_cast(*res_columns[column_index]).getNestedColumn()).getData(); + auto & column_level_null_map = assert_cast(*res_columns[column_index++]).getNullMapData(); + auto & column_parent_group = assert_cast(assert_cast(*res_columns[column_index]).getNestedColumn()).getData(); + auto & column_parent_group_null_map = assert_cast(*res_columns[column_index++]).getNullMapData(); + + auto add_row = [&](AccessType access_type, const std::string_view & aliases, Level max_level, AccessType parent_group) + { + column_access_type.push_back(static_cast(access_type)); + + for (size_t pos = 0; pos < aliases.length();) + { + size_t next_pos = aliases.find_first_of(',', pos); + std::string_view alias = aliases.substr(pos, next_pos - pos); + pos = ((next_pos == std::string_view::npos) ? next_pos : next_pos + 1); + + while (alias.starts_with(' ')) + alias.remove_prefix(1); + while (alias.ends_with(' ')) + alias.remove_suffix(1); + column_aliases.insertData(alias.data(), alias.length()); + } + column_aliases_offsets.push_back(column_aliases.size()); + + if (max_level == GROUP) + { + column_level.push_back(0); + column_level_null_map.push_back(true); + } + else + { + column_level.push_back(static_cast(max_level)); + column_level_null_map.push_back(false); + } + + if (parent_group == AccessType::NONE) + { + column_parent_group.push_back(0); + column_parent_group_null_map.push_back(true); + } + else + { + column_parent_group.push_back(static_cast(parent_group)); + column_parent_group_null_map.push_back(false); + } + }; + +#define STORAGE_SYSTEM_PRIVILEGES_ADD_ROW(name, aliases, node_type, parent_group_name) \ + add_row(AccessType::name, aliases, node_type, AccessType::parent_group_name); + + APPLY_FOR_ACCESS_TYPES(STORAGE_SYSTEM_PRIVILEGES_ADD_ROW) + +#undef STORAGE_SYSTEM_PRIVILEGES_ADD_ROW +} + +} diff --git a/src/Storages/System/StorageSystemPrivileges.h b/src/Storages/System/StorageSystemPrivileges.h new file mode 100644 index 00000000000..8540e3d7ec3 --- /dev/null +++ b/src/Storages/System/StorageSystemPrivileges.h @@ -0,0 +1,25 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class Context; + +/// Implements `privileges` system table, which allows you to get information about access types. +class StorageSystemPrivileges final : public ext::shared_ptr_helper, public IStorageSystemOneBlock +{ +public: + std::string getName() const override { return "SystemPrivileges"; } + static NamesAndTypesList getNamesAndTypes(); + static const std::vector> & getAccessTypeEnumValues(); + +protected: + friend struct ext::shared_ptr_helper; + using IStorageSystemOneBlock::IStorageSystemOneBlock; + void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const override; +}; + +} diff --git a/src/Storages/System/attachSystemTables.cpp b/src/Storages/System/attachSystemTables.cpp index d30f4f077c8..514c33f243d 100644 --- a/src/Storages/System/attachSystemTables.cpp +++ b/src/Storages/System/attachSystemTables.cpp @@ -49,6 +49,7 @@ #include #include #include +#include #ifdef OS_LINUX #include @@ -88,6 +89,7 @@ void attachSystemTablesLocal(IDatabase & system_database) system_database.attachTable("quota_limits", StorageSystemQuotaLimits::create("quota_limits")); system_database.attachTable("quota_usage", StorageSystemQuotaUsage::create("quota_usage")); system_database.attachTable("quotas_usage", StorageSystemQuotasUsage::create("all_quotas_usage")); + system_database.attachTable("privileges", StorageSystemPrivileges::create("privileges")); #if !defined(ARCADIA_BUILD) system_database.attachTable("licenses", StorageSystemLicenses::create("licenses")); #endif diff --git a/src/Storages/ya.make b/src/Storages/ya.make index 85b61aca5a0..312f0068e5f 100644 --- a/src/Storages/ya.make +++ b/src/Storages/ya.make @@ -110,6 +110,7 @@ SRCS( System/StorageSystemParts.cpp System/StorageSystemPartsBase.cpp System/StorageSystemPartsColumns.cpp + System/StorageSystemPrivileges.cpp System/StorageSystemProcesses.cpp System/StorageSystemQuotaLimits.cpp System/StorageSystemQuotaUsage.cpp diff --git a/tests/queries/0_stateless/01271_show_privileges.reference b/tests/queries/0_stateless/01271_show_privileges.reference new file mode 100644 index 00000000000..e85dbd89801 --- /dev/null +++ b/tests/queries/0_stateless/01271_show_privileges.reference @@ -0,0 +1,113 @@ +SHOW DATABASES [] DATABASE SHOW +SHOW TABLES [] TABLE SHOW +SHOW COLUMNS [] COLUMN SHOW +SHOW DICTIONARIES [] DICTIONARY SHOW +SHOW [] \N ALL +SELECT [] COLUMN ALL +INSERT [] COLUMN ALL +ALTER UPDATE ['UPDATE'] COLUMN ALTER TABLE +ALTER DELETE ['DELETE'] COLUMN ALTER TABLE +ALTER ADD COLUMN ['ADD COLUMN'] COLUMN ALTER COLUMN +ALTER MODIFY COLUMN ['MODIFY COLUMN'] COLUMN ALTER COLUMN +ALTER DROP COLUMN ['DROP COLUMN'] COLUMN ALTER COLUMN +ALTER COMMENT COLUMN ['COMMENT COLUMN'] COLUMN ALTER COLUMN +ALTER CLEAR COLUMN ['CLEAR COLUMN'] COLUMN ALTER COLUMN +ALTER RENAME COLUMN ['RENAME COLUMN'] COLUMN ALTER COLUMN +ALTER COLUMN [] \N ALTER TABLE +ALTER ORDER BY ['ALTER MODIFY ORDER BY','MODIFY ORDER BY'] TABLE ALTER INDEX +ALTER ADD INDEX ['ADD INDEX'] TABLE ALTER INDEX +ALTER DROP INDEX ['DROP INDEX'] TABLE ALTER INDEX +ALTER MATERIALIZE INDEX ['MATERIALIZE INDEX'] TABLE ALTER INDEX +ALTER CLEAR INDEX ['CLEAR INDEX'] TABLE ALTER INDEX +ALTER INDEX ['INDEX'] \N ALTER TABLE +ALTER ADD CONSTRAINT ['ADD CONSTRAINT'] TABLE ALTER CONSTRAINT +ALTER DROP CONSTRAINT ['DROP CONSTRAINT'] TABLE ALTER CONSTRAINT +ALTER CONSTRAINT ['CONSTRAINT'] \N ALTER TABLE +ALTER TTL ['ALTER MODIFY TTL','MODIFY TTL'] TABLE ALTER TABLE +ALTER MATERIALIZE TTL ['MATERIALIZE TTL'] TABLE ALTER TABLE +ALTER SETTINGS ['ALTER SETTING','ALTER MODIFY SETTING','MODIFY SETTING'] TABLE ALTER TABLE +ALTER MOVE PARTITION ['ALTER MOVE PART','MOVE PARTITION','MOVE PART'] TABLE ALTER TABLE +ALTER FETCH PARTITION ['FETCH PARTITION'] TABLE ALTER TABLE +ALTER FREEZE PARTITION ['FREEZE PARTITION'] TABLE ALTER TABLE +ALTER TABLE [] \N ALTER +ALTER VIEW REFRESH ['ALTER LIVE VIEW REFRESH','REFRESH VIEW'] VIEW ALTER VIEW +ALTER VIEW MODIFY QUERY ['ALTER TABLE MODIFY QUERY'] VIEW ALTER VIEW +ALTER VIEW [] \N ALTER +ALTER [] \N ALL +CREATE DATABASE [] DATABASE CREATE +CREATE TABLE [] TABLE CREATE +CREATE VIEW [] VIEW CREATE +CREATE DICTIONARY [] DICTIONARY CREATE +CREATE TEMPORARY TABLE [] GLOBAL CREATE +CREATE [] \N ALL +DROP DATABASE [] DATABASE DROP +DROP TABLE [] TABLE DROP +DROP VIEW [] VIEW DROP +DROP DICTIONARY [] DICTIONARY DROP +DROP [] \N ALL +TRUNCATE ['TRUNCATE TABLE'] TABLE ALL +OPTIMIZE ['OPTIMIZE TABLE'] TABLE ALL +KILL QUERY [] GLOBAL ALL +CREATE USER [] GLOBAL ACCESS MANAGEMENT +ALTER USER [] GLOBAL ACCESS MANAGEMENT +DROP USER [] GLOBAL ACCESS MANAGEMENT +CREATE ROLE [] GLOBAL ACCESS MANAGEMENT +ALTER ROLE [] GLOBAL ACCESS MANAGEMENT +DROP ROLE [] GLOBAL ACCESS MANAGEMENT +ROLE ADMIN [] GLOBAL ACCESS MANAGEMENT +CREATE ROW POLICY ['CREATE POLICY'] GLOBAL ACCESS MANAGEMENT +ALTER ROW POLICY ['ALTER POLICY'] GLOBAL ACCESS MANAGEMENT +DROP ROW POLICY ['DROP POLICY'] GLOBAL ACCESS MANAGEMENT +CREATE QUOTA [] GLOBAL ACCESS MANAGEMENT +ALTER QUOTA [] GLOBAL ACCESS MANAGEMENT +DROP QUOTA [] GLOBAL ACCESS MANAGEMENT +CREATE SETTINGS PROFILE ['CREATE PROFILE'] GLOBAL ACCESS MANAGEMENT +ALTER SETTINGS PROFILE ['ALTER PROFILE'] GLOBAL ACCESS MANAGEMENT +DROP SETTINGS PROFILE ['DROP PROFILE'] GLOBAL ACCESS MANAGEMENT +SHOW USERS ['SHOW CREATE USER'] GLOBAL SHOW ACCESS +SHOW ROLES ['SHOW CREATE ROLE'] GLOBAL SHOW ACCESS +SHOW ROW POLICIES ['SHOW POLICIES','SHOW CREATE ROW POLICY','SHOW CREATE POLICY'] GLOBAL SHOW ACCESS +SHOW QUOTAS ['SHOW CREATE QUOTA'] GLOBAL SHOW ACCESS +SHOW SETTINGS PROFILES ['SHOW PROFILES','SHOW CREATE SETTINGS PROFILE','SHOW CREATE PROFILE'] GLOBAL SHOW ACCESS +SHOW ACCESS [] \N ACCESS MANAGEMENT +ACCESS MANAGEMENT [] \N ALL +SYSTEM SHUTDOWN ['SYSTEM KILL','SHUTDOWN'] GLOBAL SYSTEM +SYSTEM DROP DNS CACHE ['SYSTEM DROP DNS','DROP DNS CACHE','DROP DNS'] GLOBAL SYSTEM DROP CACHE +SYSTEM DROP MARK CACHE ['SYSTEM DROP MARK','DROP MARK CACHE','DROP MARKS'] GLOBAL SYSTEM DROP CACHE +SYSTEM DROP UNCOMPRESSED CACHE ['SYSTEM DROP UNCOMPRESSED','DROP UNCOMPRESSED CACHE','DROP UNCOMPRESSED'] GLOBAL SYSTEM DROP CACHE +SYSTEM DROP COMPILED EXPRESSION CACHE ['SYSTEM DROP COMPILED EXPRESSION','DROP COMPILED EXPRESSION CACHE','DROP COMPILED EXPRESSIONS'] GLOBAL SYSTEM DROP CACHE +SYSTEM DROP CACHE ['DROP CACHE'] \N SYSTEM +SYSTEM RELOAD CONFIG ['RELOAD CONFIG'] GLOBAL SYSTEM RELOAD +SYSTEM RELOAD DICTIONARY ['SYSTEM RELOAD DICTIONARIES','RELOAD DICTIONARY','RELOAD DICTIONARIES'] GLOBAL SYSTEM RELOAD +SYSTEM RELOAD EMBEDDED DICTIONARIES ['RELOAD EMBEDDED DICTIONARIES'] GLOBAL SYSTEM RELOAD +SYSTEM RELOAD [] \N SYSTEM +SYSTEM MERGES ['SYSTEM STOP MERGES','SYSTEM START MERGES','STOP_MERGES','START MERGES'] TABLE SYSTEM +SYSTEM TTL MERGES ['SYSTEM STOP TTL MERGES','SYSTEM START TTL MERGES','STOP TTL MERGES','START TTL MERGES'] TABLE SYSTEM +SYSTEM FETCHES ['SYSTEM STOP FETCHES','SYSTEM START FETCHES','STOP FETCHES','START FETCHES'] TABLE SYSTEM +SYSTEM MOVES ['SYSTEM STOP MOVES','SYSTEM START MOVES','STOP MOVES','START MOVES'] TABLE SYSTEM +SYSTEM DISTRIBUTED SENDS ['SYSTEM STOP DISTRIBUTED SENDS','SYSTEM START DISTRIBUTED SENDS','STOP DISTRIBUTED SENDS','START DISTRIBUTED SENDS'] TABLE SYSTEM SENDS +SYSTEM REPLICATED SENDS ['SYSTEM STOP REPLICATED SENDS','SYSTEM START REPLICATED SENDS','STOP_REPLICATED_SENDS','START REPLICATED SENDS'] TABLE SYSTEM SENDS +SYSTEM SENDS ['SYSTEM STOP SENDS','SYSTEM START SENDS','STOP SENDS','START SENDS'] \N SYSTEM +SYSTEM REPLICATION QUEUES ['SYSTEM STOP REPLICATION QUEUES','SYSTEM START REPLICATION QUEUES','STOP_REPLICATION_QUEUES','START REPLICATION QUEUES'] TABLE SYSTEM +SYSTEM SYNC REPLICA ['SYNC REPLICA'] TABLE SYSTEM +SYSTEM RESTART REPLICA ['RESTART REPLICA'] TABLE SYSTEM +SYSTEM FLUSH DISTRIBUTED ['FLUSH DISTRIBUTED'] TABLE SYSTEM FLUSH +SYSTEM FLUSH LOGS ['FLUSH LOGS'] GLOBAL SYSTEM FLUSH +SYSTEM FLUSH [] \N SYSTEM +SYSTEM [] \N ALL +dictGet ['dictHas','dictGetHierarchy','dictIsIn'] DICTIONARY ALL +addressToLine [] GLOBAL INTROSPECTION +addressToSymbol [] GLOBAL INTROSPECTION +demangle [] GLOBAL INTROSPECTION +INTROSPECTION ['INTROSPECTION FUNCTIONS'] \N ALL +FILE [] GLOBAL SOURCES +URL [] GLOBAL SOURCES +REMOTE [] GLOBAL SOURCES +MYSQL [] GLOBAL SOURCES +ODBC [] GLOBAL SOURCES +JDBC [] GLOBAL SOURCES +HDFS [] GLOBAL SOURCES +S3 [] GLOBAL SOURCES +SOURCES [] \N ALL +ALL ['ALL PRIVILEGES'] \N \N +NONE ['USAGE','NO PRIVILEGES'] \N \N diff --git a/tests/queries/0_stateless/01271_show_privileges.sql b/tests/queries/0_stateless/01271_show_privileges.sql new file mode 100644 index 00000000000..efd6ddb200c --- /dev/null +++ b/tests/queries/0_stateless/01271_show_privileges.sql @@ -0,0 +1 @@ +SHOW PRIVILEGES; \ No newline at end of file From c30587196abdc608e693dc42a6931ffa761ac941 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Wed, 13 May 2020 02:36:39 +0300 Subject: [PATCH 212/738] Add system tables for users, roles and grants. --- src/Access/AccessFlags.h | 71 ++++--- .../InterpreterShowAccessEntitiesQuery.cpp | 93 ++++++--- src/Parsers/ASTShowAccessEntitiesQuery.cpp | 9 +- src/Parsers/ASTShowAccessEntitiesQuery.h | 4 + src/Parsers/ParserShowAccessEntitiesQuery.cpp | 24 ++- .../System/StorageSystemCurrentRoles.cpp | 53 ++++++ .../System/StorageSystemCurrentRoles.h | 24 +++ .../System/StorageSystemEnabledRoles.cpp | 57 ++++++ .../System/StorageSystemEnabledRoles.h | 24 +++ src/Storages/System/StorageSystemGrants.cpp | 180 ++++++++++++++++++ src/Storages/System/StorageSystemGrants.h | 24 +++ .../System/StorageSystemRoleGrants.cpp | 117 ++++++++++++ src/Storages/System/StorageSystemRoleGrants.h | 24 +++ src/Storages/System/StorageSystemRoles.cpp | 60 ++++++ src/Storages/System/StorageSystemRoles.h | 24 +++ src/Storages/System/StorageSystemUsers.cpp | 135 +++++++++++++ src/Storages/System/StorageSystemUsers.h | 24 +++ src/Storages/System/attachSystemTables.cpp | 12 ++ src/Storages/ya.make | 6 + .../integration/test_grant_and_revoke/test.py | 51 +++++ 20 files changed, 961 insertions(+), 55 deletions(-) create mode 100644 src/Storages/System/StorageSystemCurrentRoles.cpp create mode 100644 src/Storages/System/StorageSystemCurrentRoles.h create mode 100644 src/Storages/System/StorageSystemEnabledRoles.cpp create mode 100644 src/Storages/System/StorageSystemEnabledRoles.h create mode 100644 src/Storages/System/StorageSystemGrants.cpp create mode 100644 src/Storages/System/StorageSystemGrants.h create mode 100644 src/Storages/System/StorageSystemRoleGrants.cpp create mode 100644 src/Storages/System/StorageSystemRoleGrants.h create mode 100644 src/Storages/System/StorageSystemRoles.cpp create mode 100644 src/Storages/System/StorageSystemRoles.h create mode 100644 src/Storages/System/StorageSystemUsers.cpp create mode 100644 src/Storages/System/StorageSystemUsers.h diff --git a/src/Access/AccessFlags.h b/src/Access/AccessFlags.h index 54a8b9cde76..7071661b80b 100644 --- a/src/Access/AccessFlags.h +++ b/src/Access/AccessFlags.h @@ -71,6 +71,9 @@ public: /// Returns a comma-separated list of keywords, like "SELECT, CREATE USER, UPDATE". String toString() const; + /// Returns a list of access types. + std::vector toAccessTypes() const; + /// Returns a list of keywords. std::vector toKeywords() const; @@ -155,21 +158,27 @@ public: return res; } + std::vector flagsToAccessTypes(const Flags & flags_) const + { + std::vector access_types; + flagsToAccessTypesRec(flags_, access_types, *all_node); + return access_types; + } + std::vector flagsToKeywords(const Flags & flags_) const { std::vector keywords; - flagsToKeywordsRec(flags_, keywords, *flags_to_keyword_tree); - - if (keywords.empty()) - keywords.push_back("USAGE"); - + flagsToKeywordsRec(flags_, keywords, *all_node); return keywords; } String flagsToString(const Flags & flags_) const { + auto keywords = flagsToKeywords(flags_); + if (keywords.empty()) + return "USAGE"; String str; - for (const auto & keyword : flagsToKeywords(flags_)) + for (const auto & keyword : keywords) { if (!str.empty()) str += ", "; @@ -205,7 +214,7 @@ private: { const String keyword; NodeType node_type; - AccessType type = AccessType::NONE; + AccessType access_type = AccessType::NONE; Strings aliases; Flags flags; std::vector children; @@ -237,8 +246,8 @@ private: return aliases; } - static void makeFlagsToKeywordTreeNode( - AccessType type, + static void makeNode( + AccessType access_type, const std::string_view & name, const std::string_view & aliases, NodeType node_type, @@ -263,7 +272,7 @@ private: nodes[node->keyword] = node.get(); } - node->type = type; + node->access_type = access_type; node->node_type = node_type; node->aliases = splitAliases(aliases); if (node_type != GROUP) @@ -290,25 +299,25 @@ private: it_parent->second->addChild(std::move(node)); } - void makeFlagsToKeywordTree() + void makeNodes() { std::unordered_map owned_nodes; std::unordered_map nodes; size_t next_flag = 0; -#define MAKE_ACCESS_FLAGS_TO_KEYWORD_TREE_NODE(name, aliases, node_type, parent_group_name) \ - makeFlagsToKeywordTreeNode(AccessType::name, #name, aliases, node_type, #parent_group_name, nodes, owned_nodes, next_flag); +#define MAKE_ACCESS_FLAGS_NODE(name, aliases, node_type, parent_group_name) \ + makeNode(AccessType::name, #name, aliases, node_type, #parent_group_name, nodes, owned_nodes, next_flag); - APPLY_FOR_ACCESS_TYPES(MAKE_ACCESS_FLAGS_TO_KEYWORD_TREE_NODE) + APPLY_FOR_ACCESS_TYPES(MAKE_ACCESS_FLAGS_NODE) -#undef MAKE_ACCESS_FLAGS_TO_KEYWORD_TREE_NODE +#undef MAKE_ACCESS_FLAGS_NODE if (!owned_nodes.count("NONE")) throw Exception("'NONE' not declared", ErrorCodes::LOGICAL_ERROR); if (!owned_nodes.count("ALL")) throw Exception("'ALL' not declared", ErrorCodes::LOGICAL_ERROR); - flags_to_keyword_tree = std::move(owned_nodes["ALL"]); + all_node = std::move(owned_nodes["ALL"]); none_node = std::move(owned_nodes["NONE"]); owned_nodes.erase("ALL"); owned_nodes.erase("NONE"); @@ -328,7 +337,7 @@ private: if (!start_node) { makeKeywordToFlagsMap(none_node.get()); - start_node = flags_to_keyword_tree.get(); + start_node = all_node.get(); } start_node->aliases.emplace_back(start_node->keyword); @@ -347,10 +356,10 @@ private: if (!start_node) { makeAccessTypeToFlagsMapping(none_node.get()); - start_node = flags_to_keyword_tree.get(); + start_node = all_node.get(); } - size_t index = static_cast(start_node->type); + size_t index = static_cast(start_node->access_type); access_type_to_flags_mapping.resize(std::max(index + 1, access_type_to_flags_mapping.size())); access_type_to_flags_mapping[index] = start_node->flags; @@ -362,7 +371,7 @@ private: { if (!start_node) { - start_node = flags_to_keyword_tree.get(); + start_node = all_node.get(); all_flags = start_node->flags; } if (start_node->node_type != GROUP) @@ -376,12 +385,29 @@ private: Impl() { - makeFlagsToKeywordTree(); + makeNodes(); makeKeywordToFlagsMap(); makeAccessTypeToFlagsMapping(); collectAllFlags(); } + static void flagsToAccessTypesRec(const Flags & flags_, std::vector & access_types, const Node & start_node) + { + Flags matching_flags = (flags_ & start_node.flags); + if (matching_flags.any()) + { + if (matching_flags == start_node.flags) + { + access_types.push_back(start_node.access_type); + } + else + { + for (const auto & child : start_node.children) + flagsToAccessTypesRec(flags_, access_types, *child); + } + } + } + static void flagsToKeywordsRec(const Flags & flags_, std::vector & keywords, const Node & start_node) { Flags matching_flags = (flags_ & start_node.flags); @@ -399,7 +425,7 @@ private: } } - NodePtr flags_to_keyword_tree; + NodePtr all_node; NodePtr none_node; std::unordered_map keyword_to_flags_map; std::vector access_type_to_flags_mapping; @@ -413,6 +439,7 @@ inline AccessFlags::AccessFlags(const std::string_view & keyword) : flags(Impl<> inline AccessFlags::AccessFlags(const std::vector & keywords) : flags(Impl<>::instance().keywordsToFlags(keywords)) {} inline AccessFlags::AccessFlags(const Strings & keywords) : flags(Impl<>::instance().keywordsToFlags(keywords)) {} inline String AccessFlags::toString() const { return Impl<>::instance().flagsToString(flags); } +inline std::vector AccessFlags::toAccessTypes() const { return Impl<>::instance().flagsToAccessTypes(flags); } inline std::vector AccessFlags::toKeywords() const { return Impl<>::instance().flagsToKeywords(flags); } inline AccessFlags AccessFlags::allFlags() { return Impl<>::instance().getAllFlags(); } inline AccessFlags AccessFlags::allGlobalFlags() { return Impl<>::instance().getGlobalFlags(); } diff --git a/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp b/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp index a7fb335aacb..8d54da790f8 100644 --- a/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp +++ b/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp @@ -33,48 +33,81 @@ String InterpreterShowAccessEntitiesQuery::getRewrittenQuery() const { const auto & query = query_ptr->as(); String origin; - String expr = "name"; + String expr = "*"; String filter, order; - if (query.type == EntityType::ROW_POLICY) + switch (query.type) { - origin = "row_policies"; - - const String & table_name = query.table_name; - String database; - bool show_short_name = false; - if (!table_name.empty()) + case EntityType::ROW_POLICY: { - database = query.database; - if (database.empty()) - database = context.getCurrentDatabase(); - show_short_name = true; + origin = "row_policies"; + expr = "name"; + const String & table_name = query.table_name; + if (!table_name.empty()) + { + String database = query.database; + if (database.empty()) + database = context.getCurrentDatabase(); + filter = "database = " + quoteString(database) + " AND table = " + quoteString(table_name); + expr = "short_name"; + } + break; } - if (!table_name.empty()) - filter = "database = " + quoteString(database) + " AND table = " + quoteString(table_name); + case EntityType::QUOTA: + { + if (query.current_quota) + { + origin = "quota_usage"; + order = "duration"; + } + else + { + origin = "quotas"; + expr = "name"; + } + break; + } - if (show_short_name) - expr = "short_name"; - } - else if (query.type == EntityType::QUOTA) - { - if (query.current_quota) + case EntityType::SETTINGS_PROFILE: { - origin = "quota_usage"; - expr = "*"; - order = "duration"; + origin = "settings_profiles"; + expr = "name"; + break; } - else + + case EntityType::USER: { - origin = "quotas"; + origin = "users"; + expr = "name"; + break; } + + case EntityType::ROLE: + { + if (query.current_roles) + { + origin = "current_roles"; + order = "role_name"; + } + else if (query.enabled_roles) + { + origin = "enabled_roles"; + order = "role_name"; + } + else + { + origin = "roles"; + expr = "name"; + } + break; + } + + case EntityType::MAX: + break; } - else if (query.type == EntityType::SETTINGS_PROFILE) - { - origin = "settings_profiles"; - } - else + + if (origin.empty()) throw Exception(toString(query.type) + ": type is not supported by SHOW query", ErrorCodes::NOT_IMPLEMENTED); if (order.empty() && expr != "*") diff --git a/src/Parsers/ASTShowAccessEntitiesQuery.cpp b/src/Parsers/ASTShowAccessEntitiesQuery.cpp index 3fe88a2321c..cb1ccff5273 100644 --- a/src/Parsers/ASTShowAccessEntitiesQuery.cpp +++ b/src/Parsers/ASTShowAccessEntitiesQuery.cpp @@ -20,9 +20,14 @@ const char * ASTShowAccessEntitiesQuery::getKeyword() const return current_quota ? "SHOW CURRENT QUOTA" : "SHOW QUOTAS"; case EntityType::SETTINGS_PROFILE: return "SHOW SETTINGS PROFILES"; - default: - throw Exception(toString(type) + ": type is not supported by SHOW query", ErrorCodes::NOT_IMPLEMENTED); + case EntityType::USER: + return "SHOW USERS"; + case EntityType::ROLE: + return current_roles ? "SHOW CURRENT ROLES" : (enabled_roles ? "SHOW ENABLED ROLES" : "SHOW ROLES"); + case EntityType::MAX: + break; } + throw Exception(toString(type) + ": type is not supported by SHOW query", ErrorCodes::NOT_IMPLEMENTED); } diff --git a/src/Parsers/ASTShowAccessEntitiesQuery.h b/src/Parsers/ASTShowAccessEntitiesQuery.h index e41553d2e71..3bf16ad6abd 100644 --- a/src/Parsers/ASTShowAccessEntitiesQuery.h +++ b/src/Parsers/ASTShowAccessEntitiesQuery.h @@ -11,6 +11,8 @@ namespace DB /// SHOW QUOTAS /// SHOW [CURRENT] QUOTA /// SHOW [SETTINGS] PROFILES +/// SHOW USERS +/// SHOW [CURRENT|ENABLED] ROLES class ASTShowAccessEntitiesQuery : public ASTQueryWithOutput { public: @@ -20,6 +22,8 @@ public: String database; String table_name; bool current_quota = false; + bool current_roles = false; + bool enabled_roles = false; String getID(char) const override; ASTPtr clone() const override { return std::make_shared(*this); } diff --git a/src/Parsers/ParserShowAccessEntitiesQuery.cpp b/src/Parsers/ParserShowAccessEntitiesQuery.cpp index f70a3ea189c..c50bd5b402c 100644 --- a/src/Parsers/ParserShowAccessEntitiesQuery.cpp +++ b/src/Parsers/ParserShowAccessEntitiesQuery.cpp @@ -29,8 +29,28 @@ bool ParserShowAccessEntitiesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected std::optional type; bool current_quota = false; + bool current_roles = false; + bool enabled_roles = false; - if (ParserKeyword{"POLICIES"}.ignore(pos, expected) || ParserKeyword{"ROW POLICIES"}.ignore(pos, expected)) + if (ParserKeyword{"USERS"}.ignore(pos, expected)) + { + type = EntityType::USER; + } + else if (ParserKeyword{"ROLES"}.ignore(pos, expected)) + { + type = EntityType::ROLE; + } + else if (ParserKeyword{"CURRENT ROLES"}.ignore(pos, expected)) + { + type = EntityType::ROLE; + current_roles = true; + } + else if (ParserKeyword{"ENABLED ROLES"}.ignore(pos, expected)) + { + type = EntityType::ROLE; + enabled_roles = true; + } + else if (ParserKeyword{"POLICIES"}.ignore(pos, expected) || ParserKeyword{"ROW POLICIES"}.ignore(pos, expected)) { type = EntityType::ROW_POLICY; } @@ -59,6 +79,8 @@ bool ParserShowAccessEntitiesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected query->type = *type; query->current_quota = current_quota; + query->current_roles = current_roles; + query->enabled_roles = enabled_roles; query->database = std::move(database); query->table_name = std::move(table_name); diff --git a/src/Storages/System/StorageSystemCurrentRoles.cpp b/src/Storages/System/StorageSystemCurrentRoles.cpp new file mode 100644 index 00000000000..b0667f2f3ca --- /dev/null +++ b/src/Storages/System/StorageSystemCurrentRoles.cpp @@ -0,0 +1,53 @@ +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +NamesAndTypesList StorageSystemCurrentRoles::getNamesAndTypes() +{ + NamesAndTypesList names_and_types{ + {"role_name", std::make_shared()}, + {"with_admin_option", std::make_shared()}, + {"is_default", std::make_shared()}, + }; + return names_and_types; +} + + +void StorageSystemCurrentRoles::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const +{ + auto roles_info = context.getRolesInfo(); + auto user = context.getUser(); + if (!roles_info || !user) + return; + + size_t column_index = 0; + auto & column_role_name = assert_cast(*res_columns[column_index++]); + auto & column_admin_option = assert_cast(*res_columns[column_index++]).getData(); + auto & column_is_default = assert_cast(*res_columns[column_index++]).getData(); + + auto add_row = [&](const String & role_name, bool admin_option, bool is_default) + { + column_role_name.insertData(role_name.data(), role_name.length()); + column_admin_option.push_back(admin_option); + column_is_default.push_back(is_default); + }; + + for (const auto & role_id : roles_info->current_roles) + { + const String & role_name = roles_info->names_of_roles.at(role_id); + bool admin_option = roles_info->enabled_roles_with_admin_option.count(role_id); + bool is_default = user->default_roles.match(role_id); + add_row(role_name, admin_option, is_default); + } +} + +} diff --git a/src/Storages/System/StorageSystemCurrentRoles.h b/src/Storages/System/StorageSystemCurrentRoles.h new file mode 100644 index 00000000000..807db661371 --- /dev/null +++ b/src/Storages/System/StorageSystemCurrentRoles.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class Context; + +/// Implements `current_roles` system table, which allows you to get information about current roles. +class StorageSystemCurrentRoles final : public ext::shared_ptr_helper, public IStorageSystemOneBlock +{ +public: + std::string getName() const override { return "SystemCurrentRoles"; } + static NamesAndTypesList getNamesAndTypes(); + +protected: + friend struct ext::shared_ptr_helper; + using IStorageSystemOneBlock::IStorageSystemOneBlock; + void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const override; +}; + +} diff --git a/src/Storages/System/StorageSystemEnabledRoles.cpp b/src/Storages/System/StorageSystemEnabledRoles.cpp new file mode 100644 index 00000000000..27a42ca6f8b --- /dev/null +++ b/src/Storages/System/StorageSystemEnabledRoles.cpp @@ -0,0 +1,57 @@ +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +NamesAndTypesList StorageSystemEnabledRoles::getNamesAndTypes() +{ + NamesAndTypesList names_and_types{ + {"role_name", std::make_shared()}, + {"with_admin_option", std::make_shared()}, + {"is_current", std::make_shared()}, + {"is_default", std::make_shared()}, + }; + return names_and_types; +} + + +void StorageSystemEnabledRoles::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const +{ + auto roles_info = context.getRolesInfo(); + auto user = context.getUser(); + if (!roles_info || !user) + return; + + size_t column_index = 0; + auto & column_role_name = assert_cast(*res_columns[column_index++]); + auto & column_admin_option = assert_cast(*res_columns[column_index++]).getData(); + auto & column_is_current = assert_cast(*res_columns[column_index++]).getData(); + auto & column_is_default = assert_cast(*res_columns[column_index++]).getData(); + + auto add_row = [&](const String & role_name, bool admin_option, bool is_current, bool is_default) + { + column_role_name.insertData(role_name.data(), role_name.length()); + column_admin_option.push_back(admin_option); + column_is_current.push_back(is_current); + column_is_default.push_back(is_default); + }; + + for (const auto & role_id : roles_info->enabled_roles) + { + const String & role_name = roles_info->names_of_roles.at(role_id); + bool admin_option = roles_info->enabled_roles_with_admin_option.count(role_id); + bool is_current = roles_info->current_roles.count(role_id); + bool is_default = user->default_roles.match(role_id); + add_row(role_name, admin_option, is_current, is_default); + } +} + +} diff --git a/src/Storages/System/StorageSystemEnabledRoles.h b/src/Storages/System/StorageSystemEnabledRoles.h new file mode 100644 index 00000000000..18df31c646a --- /dev/null +++ b/src/Storages/System/StorageSystemEnabledRoles.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class Context; + +/// Implements `enabled_roles` system table, which allows you to get information about enabled roles. +class StorageSystemEnabledRoles final : public ext::shared_ptr_helper, public IStorageSystemOneBlock +{ +public: + std::string getName() const override { return "SystemEnabledRoles"; } + static NamesAndTypesList getNamesAndTypes(); + +protected: + friend struct ext::shared_ptr_helper; + using IStorageSystemOneBlock::IStorageSystemOneBlock; + void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const override; +}; + +} diff --git a/src/Storages/System/StorageSystemGrants.cpp b/src/Storages/System/StorageSystemGrants.cpp new file mode 100644 index 00000000000..a663e3307fe --- /dev/null +++ b/src/Storages/System/StorageSystemGrants.cpp @@ -0,0 +1,180 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +using EntityType = IAccessEntity::Type; + + +NamesAndTypesList StorageSystemGrants::getNamesAndTypes() +{ + NamesAndTypesList names_and_types{ + {"user_name", std::make_shared(std::make_shared())}, + {"role_name", std::make_shared(std::make_shared())}, + {"access_type", std::make_shared(StorageSystemPrivileges::getAccessTypeEnumValues())}, + {"database", std::make_shared(std::make_shared())}, + {"table", std::make_shared(std::make_shared())}, + {"column", std::make_shared(std::make_shared())}, + {"is_partial_revoke", std::make_shared()}, + {"grant_option", std::make_shared()}, + }; + return names_and_types; +} + + +void StorageSystemGrants::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const +{ + context.checkAccess(AccessType::SHOW_USERS | AccessType::SHOW_ROLES); + const auto & access_control = context.getAccessControlManager(); + std::vector ids = access_control.findAll(); + boost::range::push_back(ids, access_control.findAll()); + + size_t column_index = 0; + auto & column_user_name = assert_cast(assert_cast(*res_columns[column_index]).getNestedColumn()); + auto & column_user_name_null_map = assert_cast(*res_columns[column_index++]).getNullMapData(); + auto & column_role_name = assert_cast(assert_cast(*res_columns[column_index]).getNestedColumn()); + auto & column_role_name_null_map = assert_cast(*res_columns[column_index++]).getNullMapData(); + auto & column_access_type = assert_cast(*res_columns[column_index++]).getData(); + auto & column_database = assert_cast(assert_cast(*res_columns[column_index]).getNestedColumn()); + auto & column_database_null_map = assert_cast(*res_columns[column_index++]).getNullMapData(); + auto & column_table = assert_cast(assert_cast(*res_columns[column_index]).getNestedColumn()); + auto & column_table_null_map = assert_cast(*res_columns[column_index++]).getNullMapData(); + auto & column_column = assert_cast(assert_cast(*res_columns[column_index]).getNestedColumn()); + auto & column_column_null_map = assert_cast(*res_columns[column_index++]).getNullMapData(); + auto & column_is_partial_revoke = assert_cast(*res_columns[column_index++]).getData(); + auto & column_grant_option = assert_cast(*res_columns[column_index++]).getData(); + + auto add_row = [&](const String & grantee_name, + EntityType grantee_type, + AccessType access_type, + const String * database, + const String * table, + const String * column, + bool is_partial_revoke, + bool grant_option) + { + if (grantee_type == EntityType::USER) + { + column_user_name.insertData(grantee_name.data(), grantee_name.length()); + column_user_name_null_map.push_back(false); + column_role_name.insertDefault(); + column_role_name_null_map.push_back(true); + } + else if (grantee_type == EntityType::ROLE) + { + column_user_name.insertDefault(); + column_user_name_null_map.push_back(true); + column_role_name.insertData(grantee_name.data(), grantee_name.length()); + column_role_name_null_map.push_back(false); + } + else + assert(false); + + column_access_type.push_back(static_cast(access_type)); + + if (database) + { + column_database.insertData(database->data(), database->length()); + column_database_null_map.push_back(false); + } + else + { + column_database.insertDefault(); + column_database_null_map.push_back(true); + } + + if (table) + { + column_table.insertData(table->data(), table->length()); + column_table_null_map.push_back(false); + } + else + { + column_table.insertDefault(); + column_table_null_map.push_back(true); + } + + if (column) + { + column_column.insertData(column->data(), column->length()); + column_column_null_map.push_back(false); + } + else + { + column_column.insertDefault(); + column_column_null_map.push_back(true); + } + + column_is_partial_revoke.push_back(is_partial_revoke); + column_grant_option.push_back(grant_option); + }; + + auto add_rows = [&](const String & grantee_name, + IAccessEntity::Type grantee_type, + const AccessRightsElements & elements, + bool is_partial_revoke, + bool grant_option) + { + for (const auto & element : elements) + { + auto access_types = element.access_flags.toAccessTypes(); + if (access_types.empty() || (!element.any_column && element.columns.empty())) + continue; + + const auto * database = element.any_database ? nullptr : &element.database; + const auto * table = element.any_table ? nullptr : &element.table; + + if (element.any_column) + { + for (const auto & access_type : access_types) + add_row(grantee_name, grantee_type, access_type, database, table, nullptr, is_partial_revoke, grant_option); + } + else + { + for (const auto & access_type : access_types) + for (const auto & column : element.columns) + add_row(grantee_name, grantee_type, access_type, database, table, &column, is_partial_revoke, grant_option); + } + } + }; + + for (const auto & id : ids) + { + auto entity = access_control.tryRead(id); + if (!entity) + continue; + + const GrantedAccess * access = nullptr; + if (auto role = typeid_cast(entity)) + access = &role->access; + else if (auto user = typeid_cast(entity)) + access = &user->access; + else + continue; + + const String & grantee_name = entity->getName(); + const auto grantee_type = entity->getType(); + auto grants_and_revokes = access->access.getGrantsAndPartialRevokes(); + auto grants_and_revokes_with_grant_option = access->access_with_grant_option.getGrantsAndPartialRevokes(); + + add_rows(grantee_name, grantee_type, grants_and_revokes.grants, /* is_partial_revoke = */ false, /* grant_option = */ false); + add_rows(grantee_name, grantee_type, grants_and_revokes.revokes, /* is_partial_revoke = */ true, /* grant_option = */ false); + add_rows(grantee_name, grantee_type, grants_and_revokes_with_grant_option.grants, /* is_partial_revoke = */ false, /* grant_option = */ true); + add_rows(grantee_name, grantee_type, grants_and_revokes_with_grant_option.revokes, /* is_partial_revoke = */ true, /* grant_option = */ true); + } +} + +} diff --git a/src/Storages/System/StorageSystemGrants.h b/src/Storages/System/StorageSystemGrants.h new file mode 100644 index 00000000000..39c38deed85 --- /dev/null +++ b/src/Storages/System/StorageSystemGrants.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class Context; + +/// Implements `grants` system table, which allows you to get information about grants. +class StorageSystemGrants final : public ext::shared_ptr_helper, public IStorageSystemOneBlock +{ +public: + std::string getName() const override { return "SystemGrants"; } + static NamesAndTypesList getNamesAndTypes(); + +protected: + friend struct ext::shared_ptr_helper; + using IStorageSystemOneBlock::IStorageSystemOneBlock; + void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const override; +}; + +} diff --git a/src/Storages/System/StorageSystemRoleGrants.cpp b/src/Storages/System/StorageSystemRoleGrants.cpp new file mode 100644 index 00000000000..00147a0dae6 --- /dev/null +++ b/src/Storages/System/StorageSystemRoleGrants.cpp @@ -0,0 +1,117 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +using EntityType = IAccessEntity::Type; + + +NamesAndTypesList StorageSystemRoleGrants::getNamesAndTypes() +{ + NamesAndTypesList names_and_types{ + {"user_name", std::make_shared(std::make_shared())}, + {"role_name", std::make_shared(std::make_shared())}, + {"granted_role_name", std::make_shared()}, + {"granted_role_is_default", std::make_shared()}, + {"with_admin_option", std::make_shared()}, + }; + return names_and_types; +} + + +void StorageSystemRoleGrants::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const +{ + context.checkAccess(AccessType::SHOW_USERS | AccessType::SHOW_ROLES); + const auto & access_control = context.getAccessControlManager(); + std::vector ids = access_control.findAll(); + boost::range::push_back(ids, access_control.findAll()); + + size_t column_index = 0; + auto & column_user_name = assert_cast(assert_cast(*res_columns[column_index]).getNestedColumn()); + auto & column_user_name_null_map = assert_cast(*res_columns[column_index++]).getNullMapData(); + auto & column_role_name = assert_cast(assert_cast(*res_columns[column_index]).getNestedColumn()); + auto & column_role_name_null_map = assert_cast(*res_columns[column_index++]).getNullMapData(); + auto & column_granted_role_name = assert_cast(*res_columns[column_index++]); + auto & column_is_default = assert_cast(*res_columns[column_index++]).getData(); + auto & column_admin_option = assert_cast(*res_columns[column_index++]).getData(); + + auto add_row = [&](const String & grantee_name, + IAccessEntity::Type grantee_type, + const String & granted_role_name, + bool is_default, + bool with_admin_option) + { + if (grantee_type == EntityType::USER) + { + column_user_name.insertData(grantee_name.data(), grantee_name.length()); + column_user_name_null_map.push_back(false); + column_role_name.insertDefault(); + column_role_name_null_map.push_back(true); + } + else if (grantee_type == EntityType::ROLE) + { + column_user_name.insertDefault(); + column_user_name_null_map.push_back(true); + column_role_name.insertData(grantee_name.data(), grantee_name.length()); + column_role_name_null_map.push_back(false); + } + else + assert(false); + + column_granted_role_name.insertData(granted_role_name.data(), granted_role_name.length()); + column_is_default.push_back(is_default); + column_admin_option.push_back(with_admin_option); + }; + + auto add_rows = [&](const String & grantee_name, + IAccessEntity::Type grantee_type, + const GrantedRoles & granted_roles, + const ExtendedRoleSet * default_roles) + { + for (const auto & role_id : granted_roles.roles) + { + auto role_name = access_control.tryReadName(role_id); + if (!role_name) + continue; + + bool is_default = !default_roles || default_roles->match(role_id); + bool with_admin_option = granted_roles.roles_with_admin_option.count(role_id); + add_row(grantee_name, grantee_type, *role_name, is_default, with_admin_option); + } + }; + + for (const auto & id : ids) + { + auto entity = access_control.tryRead(id); + if (!entity) + continue; + + const GrantedRoles * granted_roles = nullptr; + const ExtendedRoleSet * default_roles = nullptr; + if (auto role = typeid_cast(entity)) + granted_roles = &role->granted_roles; + else if (auto user = typeid_cast(entity)) + { + granted_roles = &user->granted_roles; + default_roles = &user->default_roles; + } + else + continue; + + add_rows(entity->getName(), entity->getType(), *granted_roles, default_roles); + } +} + +} diff --git a/src/Storages/System/StorageSystemRoleGrants.h b/src/Storages/System/StorageSystemRoleGrants.h new file mode 100644 index 00000000000..0a02303abc3 --- /dev/null +++ b/src/Storages/System/StorageSystemRoleGrants.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class Context; + +/// Implements `role_grants` system table, which allows you to get information about granted roles. +class StorageSystemRoleGrants final : public ext::shared_ptr_helper, public IStorageSystemOneBlock +{ +public: + std::string getName() const override { return "SystemRoleGrants"; } + static NamesAndTypesList getNamesAndTypes(); + +protected: + friend struct ext::shared_ptr_helper; + using IStorageSystemOneBlock::IStorageSystemOneBlock; + void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const override; +}; + +} diff --git a/src/Storages/System/StorageSystemRoles.cpp b/src/Storages/System/StorageSystemRoles.cpp new file mode 100644 index 00000000000..c04cb6641a6 --- /dev/null +++ b/src/Storages/System/StorageSystemRoles.cpp @@ -0,0 +1,60 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +NamesAndTypesList StorageSystemRoles::getNamesAndTypes() +{ + NamesAndTypesList names_and_types{ + {"name", std::make_shared()}, + {"id", std::make_shared()}, + {"storage", std::make_shared()}, + }; + return names_and_types; +} + + +void StorageSystemRoles::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const +{ + context.checkAccess(AccessType::SHOW_ROLES); + const auto & access_control = context.getAccessControlManager(); + std::vector ids = access_control.findAll(); + + size_t column_index = 0; + auto & column_name = assert_cast(*res_columns[column_index++]); + auto & column_id = assert_cast(*res_columns[column_index++]).getData(); + auto & column_storage = assert_cast(*res_columns[column_index++]); + + auto add_row = [&](const String & name, + const UUID & id, + const String & storage_name) + { + column_name.insertData(name.data(), name.length()); + column_id.push_back(id); + column_storage.insertData(storage_name.data(), storage_name.length()); + }; + + for (const auto & id : ids) + { + auto role = access_control.tryRead(id); + if (!role) + continue; + + const auto * storage = access_control.findStorage(id); + if (!storage) + continue; + + add_row(role->getName(), id, storage->getStorageName()); + } +} + +} diff --git a/src/Storages/System/StorageSystemRoles.h b/src/Storages/System/StorageSystemRoles.h new file mode 100644 index 00000000000..fb44194baff --- /dev/null +++ b/src/Storages/System/StorageSystemRoles.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class Context; + +/// Implements `roles` system table, which allows you to get information about roles. +class StorageSystemRoles final : public ext::shared_ptr_helper, public IStorageSystemOneBlock +{ +public: + std::string getName() const override { return "SystemRoles"; } + static NamesAndTypesList getNamesAndTypes(); + +protected: + friend struct ext::shared_ptr_helper; + using IStorageSystemOneBlock::IStorageSystemOneBlock; + void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const override; +}; + +} diff --git a/src/Storages/System/StorageSystemUsers.cpp b/src/Storages/System/StorageSystemUsers.cpp new file mode 100644 index 00000000000..d0e042d054f --- /dev/null +++ b/src/Storages/System/StorageSystemUsers.cpp @@ -0,0 +1,135 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +NamesAndTypesList StorageSystemUsers::getNamesAndTypes() +{ + NamesAndTypesList names_and_types{ + {"name", std::make_shared()}, + {"id", std::make_shared()}, + {"storage", std::make_shared()}, + {"host_ip", std::make_shared(std::make_shared())}, + {"host_names", std::make_shared(std::make_shared())}, + {"host_names_regexp", std::make_shared(std::make_shared())}, + {"host_names_like", std::make_shared(std::make_shared())}, + {"default_roles_all", std::make_shared()}, + {"default_roles_list", std::make_shared(std::make_shared())}, + {"default_roles_except", std::make_shared(std::make_shared())}, + }; + return names_and_types; +} + + +void StorageSystemUsers::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const +{ + context.checkAccess(AccessType::SHOW_USERS); + const auto & access_control = context.getAccessControlManager(); + std::vector ids = access_control.findAll(); + + size_t column_index = 0; + auto & column_name = assert_cast(*res_columns[column_index++]); + auto & column_id = assert_cast(*res_columns[column_index++]).getData(); + auto & column_storage = assert_cast(*res_columns[column_index++]); + auto & column_host_ip = assert_cast(assert_cast(*res_columns[column_index]).getData()); + auto & column_host_ip_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); + auto & column_host_names = assert_cast(assert_cast(*res_columns[column_index]).getData()); + auto & column_host_names_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); + auto & column_host_names_regexp = assert_cast(assert_cast(*res_columns[column_index]).getData()); + auto & column_host_names_regexp_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); + auto & column_host_names_like = assert_cast(assert_cast(*res_columns[column_index]).getData()); + auto & column_host_names_like_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); + auto & column_default_roles_all = assert_cast(*res_columns[column_index++]).getData(); + auto & column_default_roles_list = assert_cast(assert_cast(*res_columns[column_index]).getData()); + auto & column_default_roles_list_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); + auto & column_default_roles_except = assert_cast(assert_cast(*res_columns[column_index]).getData()); + auto & column_default_roles_except_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); + + auto add_row = [&](const String & name, + const UUID & id, + const String & storage_name, + const AllowedClientHosts & allowed_hosts, + const ExtendedRoleSet & default_roles) + { + column_name.insertData(name.data(), name.length()); + column_id.push_back(id); + column_storage.insertData(storage_name.data(), storage_name.length()); + + if (allowed_hosts.containsAnyHost()) + { + static constexpr std::string_view str{"::/0"}; + column_host_ip.insertData(str.data(), str.length()); + } + else + { + if (allowed_hosts.containsLocalHost()) + { + static constexpr std::string_view str{"localhost"}; + column_host_names.insertData(str.data(), str.length()); + } + + for (const auto & ip : allowed_hosts.getAddresses()) + { + String str = ip.toString(); + column_host_ip.insertData(str.data(), str.length()); + } + for (const auto & subnet : allowed_hosts.getSubnets()) + { + String str = subnet.toString(); + column_host_ip.insertData(str.data(), str.length()); + } + + for (const auto & host_name : allowed_hosts.getNames()) + column_host_names.insertData(host_name.data(), host_name.length()); + + for (const auto & name_regexp : allowed_hosts.getNameRegexps()) + column_host_names_regexp.insertData(name_regexp.data(), name_regexp.length()); + + for (const auto & like_pattern : allowed_hosts.getLikePatterns()) + column_host_names_like.insertData(like_pattern.data(), like_pattern.length()); + } + + column_host_ip_offsets.push_back(column_host_ip.size()); + column_host_names_offsets.push_back(column_host_names.size()); + column_host_names_regexp_offsets.push_back(column_host_names_regexp.size()); + column_host_names_like_offsets.push_back(column_host_names_like.size()); + + auto default_roles_ast = default_roles.toASTWithNames(access_control); + column_default_roles_all.push_back(default_roles_ast->all); + + for (const auto & role_name : default_roles_ast->names) + column_default_roles_list.insertData(role_name.data(), role_name.length()); + column_default_roles_list_offsets.push_back(column_default_roles_list.size()); + + for (const auto & role_name : default_roles_ast->except_names) + column_default_roles_except.insertData(role_name.data(), role_name.length()); + column_default_roles_except_offsets.push_back(column_default_roles_except.size()); + }; + + for (const auto & id : ids) + { + auto user = access_control.tryRead(id); + if (!user) + continue; + + const auto * storage = access_control.findStorage(id); + if (!storage) + continue; + + add_row(user->getName(), id, storage->getStorageName(), user->allowed_client_hosts, user->default_roles); + } +} + +} diff --git a/src/Storages/System/StorageSystemUsers.h b/src/Storages/System/StorageSystemUsers.h new file mode 100644 index 00000000000..707ea94591d --- /dev/null +++ b/src/Storages/System/StorageSystemUsers.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class Context; + +/// Implements `users` system table, which allows you to get information about users. +class StorageSystemUsers final : public ext::shared_ptr_helper, public IStorageSystemOneBlock +{ +public: + std::string getName() const override { return "SystemUsers"; } + static NamesAndTypesList getNamesAndTypes(); + +protected: + friend struct ext::shared_ptr_helper; + using IStorageSystemOneBlock::IStorageSystemOneBlock; + void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const override; +}; + +} diff --git a/src/Storages/System/attachSystemTables.cpp b/src/Storages/System/attachSystemTables.cpp index 514c33f243d..585eab2b4d8 100644 --- a/src/Storages/System/attachSystemTables.cpp +++ b/src/Storages/System/attachSystemTables.cpp @@ -42,6 +42,12 @@ #include #include +#include +#include +#include +#include +#include +#include #include #include #include @@ -82,6 +88,12 @@ void attachSystemTablesLocal(IDatabase & system_database) system_database.attachTable("collations", StorageSystemCollations::create("collations")); system_database.attachTable("table_engines", StorageSystemTableEngines::create("table_engines")); system_database.attachTable("contributors", StorageSystemContributors::create("contributors")); + system_database.attachTable("users", StorageSystemUsers::create("users")); + system_database.attachTable("roles", StorageSystemRoles::create("roles")); + system_database.attachTable("grants", StorageSystemGrants::create("grants")); + system_database.attachTable("role_grants", StorageSystemRoleGrants::create("role_grants")); + system_database.attachTable("current_roles", StorageSystemCurrentRoles::create("current_roles")); + system_database.attachTable("enabled_roles", StorageSystemEnabledRoles::create("enabled_roles")); system_database.attachTable("settings_profiles", StorageSystemSettingsProfiles::create("settings_profiles")); system_database.attachTable("settings_profile_elements", StorageSystemSettingsProfileElements::create("settings_profile_elements")); system_database.attachTable("row_policies", StorageSystemRowPolicies::create("row_policies")); diff --git a/src/Storages/ya.make b/src/Storages/ya.make index 312f0068e5f..ffa3924d11a 100644 --- a/src/Storages/ya.make +++ b/src/Storages/ya.make @@ -86,6 +86,7 @@ SRCS( System/StorageSystemAsynchronousMetrics.cpp System/StorageSystemBuildOptions.cpp System/StorageSystemClusters.cpp + System/StorageSystemCurrentRoles.cpp System/StorageSystemCollations.cpp System/StorageSystemColumns.cpp System/StorageSystemContributors.cpp @@ -95,9 +96,11 @@ SRCS( System/StorageSystemDetachedParts.cpp System/StorageSystemDictionaries.cpp System/StorageSystemDisks.cpp + System/StorageSystemEnabledRoles.cpp System/StorageSystemEvents.cpp System/StorageSystemFormats.cpp System/StorageSystemFunctions.cpp + System/StorageSystemGrants.cpp System/StorageSystemGraphite.cpp System/StorageSystemMacros.cpp System/StorageSystemMerges.cpp @@ -118,6 +121,8 @@ SRCS( System/StorageSystemQuotasUsage.cpp System/StorageSystemReplicas.cpp System/StorageSystemReplicationQueue.cpp + System/StorageSystemRoleGrants.cpp + System/StorageSystemRoles.cpp System/StorageSystemRowPolicies.cpp System/StorageSystemSettings.cpp System/StorageSystemSettingsProfileElements.cpp @@ -127,6 +132,7 @@ SRCS( System/StorageSystemTableEngines.cpp System/StorageSystemTableFunctions.cpp System/StorageSystemTables.cpp + System/StorageSystemUsers.cpp System/StorageSystemZeros.cpp System/StorageSystemZooKeeper.cpp AlterCommands.cpp diff --git a/tests/integration/test_grant_and_revoke/test.py b/tests/integration/test_grant_and_revoke/test.py index 6f4b0be5325..8df68547f38 100644 --- a/tests/integration/test_grant_and_revoke/test.py +++ b/tests/integration/test_grant_and_revoke/test.py @@ -1,5 +1,6 @@ import pytest from helpers.cluster import ClickHouseCluster +from helpers.test_tools import TSV import re cluster = ClickHouseCluster(__file__) @@ -127,3 +128,53 @@ def test_admin_option(): instance.query('GRANT R1 TO A WITH ADMIN OPTION') instance.query("GRANT R1 TO B", user='A') assert instance.query("SELECT * FROM test_table", user='B') == "1\t5\n2\t10\n" + + +def test_introspection(): + instance.query("CREATE USER A") + instance.query("CREATE USER B") + instance.query('CREATE ROLE R1') + instance.query('CREATE ROLE R2') + instance.query('GRANT R1 TO A') + instance.query('GRANT R2 TO B WITH ADMIN OPTION') + instance.query('GRANT SELECT ON test.table TO A, R2') + instance.query('GRANT CREATE ON *.* TO B WITH GRANT OPTION') + instance.query('REVOKE SELECT(x) ON test.table FROM R2') + + assert instance.query("SHOW USERS") == TSV([ "A", "B", "default" ]) + assert instance.query("SHOW ROLES") == TSV([ "R1", "R2" ]) + assert instance.query("SHOW GRANTS FOR A") == TSV([ "GRANT SELECT ON test.table TO A", "GRANT R1 TO A" ]) + assert instance.query("SHOW GRANTS FOR B") == TSV([ "GRANT CREATE ON *.* TO B WITH GRANT OPTION", "GRANT R2 TO B WITH ADMIN OPTION" ]) + assert instance.query("SHOW GRANTS FOR R1") == "" + assert instance.query("SHOW GRANTS FOR R2") == TSV([ "GRANT SELECT ON test.table TO R2", "REVOKE SELECT(x) ON test.table FROM R2" ]) + + assert instance.query("SHOW GRANTS", user='A') == TSV([ "GRANT SELECT ON test.table TO A", "GRANT R1 TO A" ]) + assert instance.query("SHOW GRANTS", user='B') == TSV([ "GRANT CREATE ON *.* TO B WITH GRANT OPTION", "GRANT R2 TO B WITH ADMIN OPTION" ]) + assert instance.query("SHOW CURRENT ROLES", user='A') == TSV([[ "R1", 0, 1 ]]) + assert instance.query("SHOW CURRENT ROLES", user='B') == TSV([[ "R2", 1, 1 ]]) + assert instance.query("SHOW ENABLED ROLES", user='A') == TSV([[ "R1", 0, 1, 1 ]]) + assert instance.query("SHOW ENABLED ROLES", user='B') == TSV([[ "R2", 1, 1, 1 ]]) + + assert instance.query("SELECT name, storage, host_ip, host_names, host_names_regexp, host_names_like, default_roles_all, default_roles_list, default_roles_except from system.users WHERE name IN ('A', 'B') ORDER BY name") ==\ + TSV([[ "A", "disk", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]" ], + [ "B", "disk", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]" ]]) + + assert instance.query("SELECT name, storage from system.roles WHERE name IN ('R1', 'R2') ORDER BY name") ==\ + TSV([[ "R1", "disk" ], + [ "R2", "disk" ]]) + + assert instance.query("SELECT * from system.grants WHERE user_name IN ('A', 'B') OR role_name IN ('R1', 'R2') ORDER BY user_name, role_name, access_type, grant_option") ==\ + TSV([[ "A", "\N", "SELECT", "test", "table", "\N", 0, 0 ], + [ "B", "\N", "CREATE", "\N", "\N", "\N", 0, 0 ], + [ "B", "\N", "CREATE", "\N", "\N", "\N", 0, 1 ], + [ "\N", "R2", "SELECT", "test", "table", "\N", 0, 0 ], + [ "\N", "R2", "SELECT", "test", "table", "x", 1, 0 ]]) + + assert instance.query("SELECT * from system.role_grants WHERE user_name IN ('A', 'B') OR role_name IN ('R1', 'R2') ORDER BY user_name, role_name, granted_role_name") ==\ + TSV([[ "A", "\N", "R1", 1, 0 ], + [ "B", "\N", "R2", 1, 1 ]]) + + assert instance.query("SELECT * from system.current_roles ORDER BY role_name", user='A') == TSV([[ "R1", 0, 1 ]]) + assert instance.query("SELECT * from system.current_roles ORDER BY role_name", user='B') == TSV([[ "R2", 1, 1 ]]) + assert instance.query("SELECT * from system.enabled_roles ORDER BY role_name", user='A') == TSV([[ "R1", 0, 1, 1 ]]) + assert instance.query("SELECT * from system.enabled_roles ORDER BY role_name", user='B') == TSV([[ "R2", 1, 1, 1 ]]) From bf2f38881df2f92e49549641311c7f2f8ca7c16a Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Wed, 13 May 2020 18:55:54 +0300 Subject: [PATCH 213/738] Fix compilation. --- src/Access/AccessFlags.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Access/AccessFlags.h b/src/Access/AccessFlags.h index 7071661b80b..9b801fd88a3 100644 --- a/src/Access/AccessFlags.h +++ b/src/Access/AccessFlags.h @@ -278,7 +278,7 @@ private: if (node_type != GROUP) node->setFlag(next_flag++); - bool has_parent_group = (parent_group_name != "NONE"); + bool has_parent_group = (parent_group_name != std::string_view{"NONE"}); if (!has_parent_group) { std::string_view keyword_as_string_view = node->keyword; From 99d50063f388038b42416d11dec1616f66fc9923 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 14 May 2020 14:36:19 +0300 Subject: [PATCH 214/738] Support MongoDB URI --- src/Dictionaries/MongoDBDictionarySource.cpp | 76 ++++++++++--------- src/Dictionaries/MongoDBDictionarySource.h | 10 +-- .../external_sources.py | 15 ++++ .../test.py | 3 +- 4 files changed, 60 insertions(+), 44 deletions(-) diff --git a/src/Dictionaries/MongoDBDictionarySource.cpp b/src/Dictionaries/MongoDBDictionarySource.cpp index 2e423b7d511..07f0b20e908 100644 --- a/src/Dictionaries/MongoDBDictionarySource.cpp +++ b/src/Dictionaries/MongoDBDictionarySource.cpp @@ -8,15 +8,27 @@ namespace DB void registerDictionarySourceMongoDB(DictionarySourceFactory & factory) { - auto create_table_source = [=](const DictionaryStructure & dict_struct, - const Poco::Util::AbstractConfiguration & config, - const std::string & config_prefix, - Block & sample_block, - const Context & /* context */, - bool /* check_config */) -> DictionarySourcePtr { - return std::make_unique(dict_struct, config, config_prefix + ".mongodb", sample_block); + auto createMongoDBDictionary = []( + const DictionaryStructure & dict_struct, + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + Block & sample_block, + const Context &, + bool /* check_config */) + { + return std::make_unique(dict_struct, + config.getString(config_prefix + ".uri"), + config.getString(config_prefix + ".host"), + config.getUInt(config_prefix + ".port"), + config.getString(config_prefix + ".user", ""), + config.getString(config_prefix + ".password", ""), + config.getString(config_prefix + ".method", ""), + config.getString(config_prefix + ".db", ""), + config.getString(config_prefix + ".collection"), + sample_block); }; - factory.registerSource("mongodb", create_table_source); + + factory.registerSource("mongodb", createMongoDBDictionary); } } @@ -155,6 +167,7 @@ authenticate(Poco::MongoDB::Connection & connection, const std::string & databas MongoDBDictionarySource::MongoDBDictionarySource( const DictionaryStructure & dict_struct_, + const std::string & uri_, const std::string & host_, UInt16 port_, const std::string & user_, @@ -164,6 +177,7 @@ MongoDBDictionarySource::MongoDBDictionarySource( const std::string & collection_, const Block & sample_block_) : dict_struct{dict_struct_} + , uri{uri_} , host{host_} , port{port_} , user{user_} @@ -174,41 +188,31 @@ MongoDBDictionarySource::MongoDBDictionarySource( , sample_block{sample_block_} , connection{std::make_shared(host, port)} { - if (!user.empty()) + if (!uri.empty()) { -#if POCO_VERSION >= 0x01070800 - Poco::MongoDB::Database poco_db(db); - if (!poco_db.authenticate(*connection, user, password, method.empty() ? Poco::MongoDB::Database::AUTH_SCRAM_SHA1 : method)) - throw Exception("Cannot authenticate in MongoDB, incorrect user or password", ErrorCodes::MONGODB_CANNOT_AUTHENTICATE); -#else - authenticate(*connection, db, user, password); -#endif + Poco::MongoDB::Connection::SocketFactory socket_factory; + connection->connect(uri, socket_factory); + } + else + { + connection->connect(host, port); + if (!user.empty()) + { +#if POCO_VERSION >= 0x01070800 + Poco::MongoDB::Database poco_db(db); + if (!poco_db.authenticate(*connection, user, password, method.empty() ? Poco::MongoDB::Database::AUTH_SCRAM_SHA1 : method)) + throw Exception("Cannot authenticate in MongoDB, incorrect user or password", ErrorCodes::MONGODB_CANNOT_AUTHENTICATE); +#else + authenticate(*connection, db, user, password); +#endif + } } -} - - -MongoDBDictionarySource::MongoDBDictionarySource( - const DictionaryStructure & dict_struct_, - const Poco::Util::AbstractConfiguration & config, - const std::string & config_prefix, - Block & sample_block_) - : MongoDBDictionarySource( - dict_struct_, - config.getString(config_prefix + ".host"), - config.getUInt(config_prefix + ".port"), - config.getString(config_prefix + ".user", ""), - config.getString(config_prefix + ".password", ""), - config.getString(config_prefix + ".method", ""), - config.getString(config_prefix + ".db", ""), - config.getString(config_prefix + ".collection"), - sample_block_) -{ } MongoDBDictionarySource::MongoDBDictionarySource(const MongoDBDictionarySource & other) : MongoDBDictionarySource{ - other.dict_struct, other.host, other.port, other.user, other.password, other.method, other.db, other.collection, other.sample_block} + other.dict_struct, other.uri, other.host, other.port, other.user, other.password, other.method, other.db, other.collection, other.sample_block} { } diff --git a/src/Dictionaries/MongoDBDictionarySource.h b/src/Dictionaries/MongoDBDictionarySource.h index 23562c75500..d90f28e1e74 100644 --- a/src/Dictionaries/MongoDBDictionarySource.h +++ b/src/Dictionaries/MongoDBDictionarySource.h @@ -29,8 +29,10 @@ namespace ErrorCodes /// Allows loading dictionaries from a MongoDB collection class MongoDBDictionarySource final : public IDictionarySource { +public: MongoDBDictionarySource( const DictionaryStructure & dict_struct_, + const std::string & uri_, const std::string & host_, UInt16 port_, const std::string & user_, @@ -40,13 +42,6 @@ class MongoDBDictionarySource final : public IDictionarySource const std::string & collection_, const Block & sample_block_); -public: - MongoDBDictionarySource( - const DictionaryStructure & dict_struct, - const Poco::Util::AbstractConfiguration & config, - const std::string & config_prefix, - Block & sample_block); - MongoDBDictionarySource(const MongoDBDictionarySource & other); ~MongoDBDictionarySource() override; @@ -76,6 +71,7 @@ public: private: const DictionaryStructure dict_struct; + const std::string uri; const std::string host; const UInt16 port; const std::string user; diff --git a/tests/integration/test_dictionaries_all_layouts_and_sources/external_sources.py b/tests/integration/test_dictionaries_all_layouts_and_sources/external_sources.py index d4879232172..7d1ded04bdc 100644 --- a/tests/integration/test_dictionaries_all_layouts_and_sources/external_sources.py +++ b/tests/integration/test_dictionaries_all_layouts_and_sources/external_sources.py @@ -178,6 +178,21 @@ class SourceMongo(ExternalSource): result = tbl.insert_many(to_insert) +class SourceMongoURI(SourceMongo): + def get_source_str(self, table_name): + return ''' + + mongodb://{user}:{password}@{host}:{port}/test + {tbl} + + '''.format( + host=self.docker_hostname, + port=self.docker_port, + user=self.user, + password=self.password, + tbl=table_name, + ) + class SourceClickHouse(ExternalSource): def get_source_str(self, table_name): diff --git a/tests/integration/test_dictionaries_all_layouts_and_sources/test.py b/tests/integration/test_dictionaries_all_layouts_and_sources/test.py index cc899ffd0cc..36034bab357 100644 --- a/tests/integration/test_dictionaries_all_layouts_and_sources/test.py +++ b/tests/integration/test_dictionaries_all_layouts_and_sources/test.py @@ -4,7 +4,7 @@ import os from helpers.cluster import ClickHouseCluster from dictionary import Field, Row, Dictionary, DictionaryStructure, Layout from external_sources import SourceMySQL, SourceClickHouse, SourceFile, SourceExecutableCache, SourceExecutableHashed -from external_sources import SourceMongo, SourceHTTP, SourceHTTPS, SourceRedis +from external_sources import SourceMongo, SourceMongoURI, SourceHTTP, SourceHTTPS, SourceRedis import math SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -117,6 +117,7 @@ LAYOUTS = [ SOURCES = [ SourceMongo("MongoDB", "localhost", "27018", "mongo1", "27017", "root", "clickhouse"), + SourceMongoURI("MongoDB", "localhost", "27018", "mongo1", "27017", "root", "clickhouse"), SourceMySQL("MySQL", "localhost", "3308", "mysql1", "3306", "root", "clickhouse"), SourceClickHouse("RemoteClickHouse", "localhost", "9000", "clickhouse1", "9000", "default", ""), SourceClickHouse("LocalClickHouse", "localhost", "9000", "node", "9000", "default", ""), From e5c8c8ca022015025a7cdef1a67322fccc15b446 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 14 May 2020 15:45:42 +0300 Subject: [PATCH 215/738] Fix test --- ...ter_materialized_view_consistent.reference | 1 + ...1019_alter_materialized_view_consistent.sh | 35 ++++++++++--------- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/tests/queries/0_stateless/01019_alter_materialized_view_consistent.reference b/tests/queries/0_stateless/01019_alter_materialized_view_consistent.reference index 726246fba3f..a7bb9e4c153 100755 --- a/tests/queries/0_stateless/01019_alter_materialized_view_consistent.reference +++ b/tests/queries/0_stateless/01019_alter_materialized_view_consistent.reference @@ -1 +1,2 @@ +1 inconsistencies 0 diff --git a/tests/queries/0_stateless/01019_alter_materialized_view_consistent.sh b/tests/queries/0_stateless/01019_alter_materialized_view_consistent.sh index a4256499469..f3a272d091b 100755 --- a/tests/queries/0_stateless/01019_alter_materialized_view_consistent.sh +++ b/tests/queries/0_stateless/01019_alter_materialized_view_consistent.sh @@ -31,12 +31,18 @@ function insert_thread() { INSERT[1]="INSERT INTO TABLE src_b VALUES (2);" while true; do - # trigger 100 concurrent inserts at a time - for i in {0..100}; do + # trigger 50 concurrent inserts at a time + for i in {0..50}; do # ignore `Possible deadlock avoided. Client should retry` $CLICKHOUSE_CLIENT -q "${INSERT[$RANDOM % 2]}" 2>/dev/null & done wait + + is_done=$($CLICKHOUSE_CLIENT -q "SELECT countIf(case = 1) > 0 AND countIf(case = 2) > 0 FROM mv;") + + if [ "$is_done" -eq "1" ]; then + break + fi done } @@ -50,28 +56,23 @@ function alter_thread() { $CLICKHOUSE_CLIENT --allow_experimental_alter_materialized_view_structure=1 \ -q "${ALTER[$RANDOM % 2]}" sleep "0.0$RANDOM" + + is_done=$($CLICKHOUSE_CLIENT -q "SELECT countIf(case = 1) > 0 AND countIf(case = 2) > 0 FROM mv;") + + if [ "$is_done" -eq "1" ]; then + break + fi done } export -f insert_thread; export -f alter_thread; -timeout 30 bash -c insert_thread & -timeout 30 bash -c alter_thread & - -function check_thread() { - while true; do - is_done=$($CLICKHOUSE_CLIENT -q "SELECT countIf(case = 1) > 0 AND countIf(case = 2) > 0 FROM mv;") - - if [ "$is_done" -eq "1" ]; then - break - fi - done -} - -export -f check_thread -timeout 30 bash -c check_thread +# finishes much faster with all builds, except debug with coverage +timeout 120 bash -c insert_thread & +timeout 120 bash -c alter_thread & wait +$CLICKHOUSE_CLIENT -q "SELECT countIf(case = 1) > 0 AND countIf(case = 2) > 0 FROM mv LIMIT 1;" $CLICKHOUSE_CLIENT -q "SELECT 'inconsistencies', count() FROM mv WHERE test == 0;" From dc4af074b2e58194bad6b2d9d6377f16e9e4b7f5 Mon Sep 17 00:00:00 2001 From: potya Date: Thu, 14 May 2020 17:11:59 +0300 Subject: [PATCH 216/738] Remove REAl and INT8, add some tests --- src/DataTypes/DataTypesDecimal.cpp | 1 - tests/queries/0_stateless/00700_decimal_arithm.sql | 11 ++++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/DataTypes/DataTypesDecimal.cpp b/src/DataTypes/DataTypesDecimal.cpp index 400cd30d969..2586f54dc87 100644 --- a/src/DataTypes/DataTypesDecimal.cpp +++ b/src/DataTypes/DataTypesDecimal.cpp @@ -181,7 +181,6 @@ void registerDataTypeDecimal(DataTypeFactory & factory) factory.registerAlias("DEC", "Decimal", DataTypeFactory::CaseInsensitive); factory.registerAlias("NUMERIC", "Decimal", DataTypeFactory::CaseInsensitive); factory.registerAlias("FIXED", "Decimal", DataTypeFactory::CaseInsensitive); - factory.registerAlias("REAL", "Decimal32", DataTypeFactory::CaseInsensitive); } /// Explicit template instantiations. diff --git a/tests/queries/0_stateless/00700_decimal_arithm.sql b/tests/queries/0_stateless/00700_decimal_arithm.sql index f456fcdf807..1331ae2800c 100644 --- a/tests/queries/0_stateless/00700_decimal_arithm.sql +++ b/tests/queries/0_stateless/00700_decimal_arithm.sql @@ -16,15 +16,12 @@ CREATE TABLE IF NOT EXISTS decimal l numeric(3,9), m NUMEric(18, 9), n FixED(12, 6), - o fixed(8, 6), - p real(3, 1), - q REal(28, 12), - r real(9, 9) + o fixed(8, 6) ) ENGINE = Memory; -INSERT INTO decimal (a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r) VALUES (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,); -INSERT INTO decimal (a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r) VALUES (42, 42, 42, 0.42, 0.42, 0.42, 42.42, 42.42, 42.42, 42.42, 42.42, 42.42, 42.42, 42.42, 42.42, 42.42, 42.42, 42.42); -INSERT INTO decimal (a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r) VALUES (-42, -42, -42, -0.42, -0.42, -0.42, -42.42, -42.42, -42.42, -42.42, -42.42, -42.42, -42.42, -42.42, -42.42, -42.42, -42.42, -42.42); +INSERT INTO decimal (a, b, c, d, e, f, g, h, i, j, k, l, m, n, o) VALUES (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); +INSERT INTO decimal (a, b, c, d, e, f, g, h, i, j, k, l, m, n, o) VALUES (42, 42, 42, 0.42, 0.42, 0.42, 42.42, 42.42, 42.42, 42.42, 42.42, 42.42, 42.42, 42.42, 42.42); +INSERT INTO decimal (a, b, c, d, e, f, g, h, i, j, k, l, m, n, o) VALUES (-42, -42, -42, -0.42, -0.42, -0.42, -42.42, -42.42, -42.42, -42.42, -42.42, -42.42, -42.42, -42.42, -42.42); SELECT a + a, a - a, a * a, a / a, intDiv(a, a), intDivOrZero(a, a) FROM decimal WHERE a = 42; SELECT b + b, b - b, b * b, b / b, intDiv(b, b), intDivOrZero(b, b) FROM decimal WHERE b = 42; From b556b7c76e676314a59aa22030537194977102e2 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 14 May 2020 17:21:38 +0300 Subject: [PATCH 217/738] Fix notNullIn with NULL argument. --- src/Functions/ignoreExceptNull.cpp | 56 ---------- src/Functions/in.cpp | 102 ++++++++---------- .../registerFunctionsMiscellaneous.cpp | 2 - src/Interpreters/ActionsVisitor.cpp | 8 +- 4 files changed, 47 insertions(+), 121 deletions(-) delete mode 100644 src/Functions/ignoreExceptNull.cpp diff --git a/src/Functions/ignoreExceptNull.cpp b/src/Functions/ignoreExceptNull.cpp deleted file mode 100644 index ff009533e37..00000000000 --- a/src/Functions/ignoreExceptNull.cpp +++ /dev/null @@ -1,56 +0,0 @@ -#include -#include -#include -#include - - -namespace DB -{ - -/** ignoreExceptNull(...) is a function that takes any arguments, and always returns 0 except Null. - */ - class FunctionIgnoreExceptNull : public IFunction - { - public: - static constexpr auto name = "ignoreExceptNull"; - static FunctionPtr create(const Context &) - { - return std::make_shared(); - } - - bool isVariadic() const override - { - return true; - } - size_t getNumberOfArguments() const override - { - return 0; - } - - String getName() const override - { - return name; - } - - DataTypePtr getReturnTypeImpl(const DataTypes & /*arguments*/) const override - { - return std::make_shared(); - } - - void executeImpl(Block & block, const ColumnNumbers &, size_t result, size_t input_rows_count) override - { - /// This function is mainly used in query analysis instead of "in" functions - /// in the case when only header is needed and set for in is not calculated. - /// Because of that function must return the same column type as "in" function, which is ColumnUInt8. - auto res = ColumnUInt8::create(input_rows_count, 0); - block.getByPosition(result).column = std::move(res); - } - }; - - - void registerFunctionIgnoreExceptNull(FunctionFactory & factory) - { - factory.registerFunction(); - } - -} diff --git a/src/Functions/in.cpp b/src/Functions/in.cpp index a89535c675a..55cae1afc6c 100644 --- a/src/Functions/in.cpp +++ b/src/Functions/in.cpp @@ -21,62 +21,34 @@ namespace ErrorCodes * notIn(x, set) - and NOT IN. */ -template +template struct FunctionInName; -template <> -struct FunctionInName -{ - static constexpr auto name = "in"; -}; +template <> struct FunctionInName { static constexpr auto name = "in"; }; +template <> struct FunctionInName { static constexpr auto name = "globalIn"; }; +template <> struct FunctionInName { static constexpr auto name = "notIn"; }; +template <> struct FunctionInName { static constexpr auto name = "globalNotIn"; }; +template <> struct FunctionInName { static constexpr auto name = "nullIn"; }; +template <> struct FunctionInName { static constexpr auto name = "globalNullIn"; }; +template <> struct FunctionInName { static constexpr auto name = "notNullIn"; }; +template <> struct FunctionInName { static constexpr auto name = "globalNotNullIn"; }; +template <> struct FunctionInName { static constexpr auto name = "inIgnoreSet"; }; +template <> struct FunctionInName { static constexpr auto name = "globalInIgnoreSet"; }; +template <> struct FunctionInName { static constexpr auto name = "notInIgnoreSet"; }; +template <> struct FunctionInName { static constexpr auto name = "globalNotInIgnoreSet"; }; +template <> struct FunctionInName { static constexpr auto name = "nullInIgnoreSet"; }; +template <> struct FunctionInName { static constexpr auto name = "globalNullInIgnoreSet"; }; +template <> struct FunctionInName { static constexpr auto name = "notNullInIgnoreSet"; }; +template <> struct FunctionInName { static constexpr auto name = "globalNotNullInIgnoreSet"; }; -template <> -struct FunctionInName -{ - static constexpr auto name = "globalIn"; -}; - -template <> -struct FunctionInName -{ - static constexpr auto name = "notIn"; -}; - -template <> -struct FunctionInName -{ - static constexpr auto name = "globalNotIn"; -}; - -template <> -struct FunctionInName -{ - static constexpr auto name = "nullIn"; -}; - -template <> -struct FunctionInName -{ - static constexpr auto name = "globalNullIn"; -}; - -template <> -struct FunctionInName -{ - static constexpr auto name = "notNullIn"; -}; - -template <> -struct FunctionInName -{ - static constexpr auto name = "globalNotNullIn"; -}; - -template +template class FunctionIn : public IFunction { public: - static constexpr auto name = FunctionInName::name; + /// ignore_set flag means that we don't use set from the second argument, just return zero column. + /// It is needed to perform type analysis without creation of set. + static constexpr auto name = FunctionInName::name; + static FunctionPtr create(const Context &) { return std::make_shared(); @@ -101,9 +73,13 @@ public: bool useDefaultImplementationForNulls() const override { return null_is_skipped; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) override + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, [[maybe_unused]] size_t input_rows_count) override { - /// NOTE: after updating this code, check that FunctionIgnoreExceptNull returns the same type of column. + if constexpr (ignore_set) + { + block.getByPosition(result).column = DataTypeUInt8().createColumnConst(input_rows_count, 0u); + return; + } /// Second argument must be ColumnSet. ColumnPtr column_set_ptr = block.getByPosition(arguments[1]).column; @@ -146,17 +122,23 @@ public: } }; +template +static void registerFunctionsInImpl(FunctionFactory & factory) +{ + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); +} void registerFunctionsIn(FunctionFactory & factory) { - factory.registerFunction>(); - factory.registerFunction>(); - factory.registerFunction>(); - factory.registerFunction>(); - factory.registerFunction>(); - factory.registerFunction>(); - factory.registerFunction>(); - factory.registerFunction>(); + registerFunctionsInImpl(factory); + registerFunctionsInImpl(factory); } } diff --git a/src/Functions/registerFunctionsMiscellaneous.cpp b/src/Functions/registerFunctionsMiscellaneous.cpp index 221e14fcce1..b932d232834 100644 --- a/src/Functions/registerFunctionsMiscellaneous.cpp +++ b/src/Functions/registerFunctionsMiscellaneous.cpp @@ -29,7 +29,6 @@ void registerFunctionSleep(FunctionFactory &); void registerFunctionSleepEachRow(FunctionFactory &); void registerFunctionMaterialize(FunctionFactory &); void registerFunctionIgnore(FunctionFactory &); -void registerFunctionIgnoreExceptNull(FunctionFactory &); void registerFunctionIdentity(FunctionFactory &); void registerFunctionArrayJoin(FunctionFactory &); void registerFunctionReplicate(FunctionFactory &); @@ -88,7 +87,6 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory) registerFunctionSleepEachRow(factory); registerFunctionMaterialize(factory); registerFunctionIgnore(factory); - registerFunctionIgnoreExceptNull(factory); registerFunctionIdentity(factory); registerFunctionArrayJoin(factory); registerFunctionReplicate(factory); diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 4d032e552ca..81f19d0b3dc 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -381,11 +381,13 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & if (!data.only_consts) { /// We are in the part of the tree that we are not going to compute. You just need to define types. - /// Do not subquery and create sets. We treat "IN" as "ignoreExceptNull" function. + /// Do not subquery and create sets. We replace "in*" function to "in*IgnoreSet". + + auto argument_name = node.arguments->children.at(0)->getColumnName(); data.addAction(ExpressionAction::applyFunction( - FunctionFactory::instance().get("ignoreExceptNull", data.context), - { node.arguments->children.at(0)->getColumnName() }, + FunctionFactory::instance().get(node.name + "IgnoreSet", data.context), + { argument_name, argument_name }, column_name.get(ast))); } return; From b81d96b84e3aa31654a2ede683ab4cac41fa4ed6 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 14 May 2020 17:27:05 +0300 Subject: [PATCH 218/738] Add test. --- tests/queries/0_stateless/01280_null_in.reference | 8 ++++++++ tests/queries/0_stateless/01280_null_in.sql | 9 +++++++++ 2 files changed, 17 insertions(+) create mode 100644 tests/queries/0_stateless/01280_null_in.reference create mode 100644 tests/queries/0_stateless/01280_null_in.sql diff --git a/tests/queries/0_stateless/01280_null_in.reference b/tests/queries/0_stateless/01280_null_in.reference new file mode 100644 index 00000000000..04402256766 --- /dev/null +++ b/tests/queries/0_stateless/01280_null_in.reference @@ -0,0 +1,8 @@ +0 +0 +1 +1 +0 +0 +1 +1 diff --git a/tests/queries/0_stateless/01280_null_in.sql b/tests/queries/0_stateless/01280_null_in.sql new file mode 100644 index 00000000000..76fe4db6786 --- /dev/null +++ b/tests/queries/0_stateless/01280_null_in.sql @@ -0,0 +1,9 @@ +SELECT count(in(NULL, [])); +SELECT count(notIn(NULL, [])); +SELECT count(nullIn(NULL, [])); +SELECT count(notNullIn(NULL, [])); + +SELECT count(in(NULL, tuple(NULL))); +SELECT count(notIn(NULL, tuple(NULL))); +SELECT count(nullIn(NULL, tuple(NULL))); +SELECT count(notNullIn(NULL, tuple(NULL))); From 104860b274a16157e17416c7c98c90ca4c9aad67 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 14 May 2020 18:37:37 +0300 Subject: [PATCH 219/738] Fix stress script --- docker/test/stress/stress | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docker/test/stress/stress b/docker/test/stress/stress index fc19ed80fa6..92dac8bc2d1 100755 --- a/docker/test/stress/stress +++ b/docker/test/stress/stress @@ -17,11 +17,15 @@ def run_perf_test(cmd, xmls_path, output_folder): def run_func_test(cmd, output_prefix, num_processes, skip_tests_option): output_paths = [os.path.join(output_prefix, "stress_test_run_{}.txt".format(i)) for i in range(num_processes)] f = open(output_paths[0], 'w') - pipes = [Popen("{}".format(cmd), shell=True, stdout=f, stderr=f)] + main_command = "{} {}".format(cmd, skip_tests_option) + logging.info("Run func tests main cmd '%s'", main_command) + pipes = [Popen(main_command, shell=True, stdout=f, stderr=f)] for output_path in output_paths[1:]: time.sleep(0.5) f = open(output_path, 'w') - p = Popen("{} --order=random {}".format(cmd, skip_tests_option), shell=True, stdout=f, stderr=f) + full_command = "{} --order=random {}".format(cmd, skip_tests_option) + logging.info("Run func tests '%s'", full_command) + p = Popen(full_command, shell=True, stdout=f, stderr=f) pipes.append(p) return pipes From 948bb815235a44da8257d29394e895f9f1cd594a Mon Sep 17 00:00:00 2001 From: BayoNet Date: Thu, 14 May 2020 19:07:13 +0300 Subject: [PATCH 220/738] DOCS-629: Parameters for Replicated engines (#10854) * Updated parameters description. * CLICKHOUSEDOCS-629: Fixed typo. * CLICKHOUSEDOCS-629: Updated by comment * CLICKHOUSEDOCS-629: Fix. Co-authored-by: Sergei Shtykov --- .../engines/table-engines/mergetree-family/replication.md | 6 ++++-- .../engines/table-engines/mergetree-family/replication.md | 7 +++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/en/engines/table-engines/mergetree-family/replication.md b/docs/en/engines/table-engines/mergetree-family/replication.md index dd83fb61f2b..d813f76d127 100644 --- a/docs/en/engines/table-engines/mergetree-family/replication.md +++ b/docs/en/engines/table-engines/mergetree-family/replication.md @@ -85,6 +85,7 @@ The `Replicated` prefix is added to the table engine name. For example:`Replicat - `zoo_path` — The path to the table in ZooKeeper. - `replica_name` — The replica name in ZooKeeper. +- `other_parameters` — Parameters of an engine which is used for creating the replicated version, for example, version in `ReplacingMergeTree`. Example: @@ -93,8 +94,9 @@ CREATE TABLE table_name ( EventDate DateTime, CounterID UInt32, - UserID UInt32 -) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/table_name', '{replica}') + UserID UInt32, + ver UInt16 +) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{layer}-{shard}/table_name', '{replica}', ver) PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) diff --git a/docs/ru/engines/table-engines/mergetree-family/replication.md b/docs/ru/engines/table-engines/mergetree-family/replication.md index ace4e355b38..51379c5a8cf 100644 --- a/docs/ru/engines/table-engines/mergetree-family/replication.md +++ b/docs/ru/engines/table-engines/mergetree-family/replication.md @@ -80,6 +80,8 @@ ClickHouse хранит метаинформацию о репликах в [Apa - `zoo_path` — путь к таблице в ZooKeeper. - `replica_name` — имя реплики в ZooKeeper. +- `other_parameters` — параметры движка, для которого создаётся реплицированная версия, например, версия для `ReplacingMergeTree`. + Пример: @@ -88,8 +90,9 @@ CREATE TABLE table_name ( EventDate DateTime, CounterID UInt32, - UserID UInt32 -) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/table_name', '{replica}') + UserID UInt32, + ver UInt16 +) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{layer}-{shard}/table_name', '{replica}', ver) PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) From 8eb27f6f1364de94212c7f5b13ae0664e04da2b9 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 14 May 2020 19:56:13 +0300 Subject: [PATCH 221/738] Fix rename with compact parts --- src/Storages/MergeTree/IMergeTreeReader.cpp | 10 +++--- .../01278_alter_rename_combination.reference | 8 +++++ .../01278_alter_rename_combination.sql | 32 +++++++++++++++++++ 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/src/Storages/MergeTree/IMergeTreeReader.cpp b/src/Storages/MergeTree/IMergeTreeReader.cpp index 8243983d837..5d2a5ac3616 100644 --- a/src/Storages/MergeTree/IMergeTreeReader.cpp +++ b/src/Storages/MergeTree/IMergeTreeReader.cpp @@ -187,17 +187,17 @@ void IMergeTreeReader::evaluateMissingDefaults(Block additional_columns, Columns NameAndTypePair IMergeTreeReader::getColumnFromPart(const NameAndTypePair & required_column) const { - auto it = columns_from_part.find(required_column.name); - if (it != columns_from_part.end()) - return {it->first, it->second}; - if (alter_conversions.isColumnRenamed(required_column.name)) { String old_name = alter_conversions.getColumnOldName(required_column.name); - it = columns_from_part.find(old_name); + auto it = columns_from_part.find(old_name); if (it != columns_from_part.end()) return {it->first, it->second}; } + else if (auto it = columns_from_part.find(required_column.name); it != columns_from_part.end()) + { + return {it->first, it->second}; + } return required_column; } diff --git a/tests/queries/0_stateless/01278_alter_rename_combination.reference b/tests/queries/0_stateless/01278_alter_rename_combination.reference index c3b3d5464c4..3d77561b460 100644 --- a/tests/queries/0_stateless/01278_alter_rename_combination.reference +++ b/tests/queries/0_stateless/01278_alter_rename_combination.reference @@ -1,3 +1,11 @@ CREATE TABLE default.rename_table\n(\n `key` Int32, \n `old_value1` Int32, \n `value1` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 key old_value1 value1 1 2 3 +CREATE TABLE default.rename_table\n(\n `k` Int32, \n `v1` Int32, \n `v2` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +k v1 v2 +1 2 3 +4 5 6 +---polymorphic--- +CREATE TABLE default.rename_table_polymorphic\n(\n `key` Int32, \n `old_value1` Int32, \n `value1` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS min_rows_for_wide_part = 10000, index_granularity = 8192 +key old_value1 value1 +1 2 3 diff --git a/tests/queries/0_stateless/01278_alter_rename_combination.sql b/tests/queries/0_stateless/01278_alter_rename_combination.sql index f00adc369c1..06f756cc0fd 100644 --- a/tests/queries/0_stateless/01278_alter_rename_combination.sql +++ b/tests/queries/0_stateless/01278_alter_rename_combination.sql @@ -11,4 +11,36 @@ SHOW CREATE TABLE rename_table; SELECT * FROM rename_table FORMAT TSVWithNames; +INSERT INTO rename_table VALUES (4, 5, 6); + +-- rename all columns simultaneously +ALTER TABLE rename_table RENAME COLUMN old_value1 TO v1, RENAME COLUMN value1 TO v2, RENAME COLUMN key to k; + +SHOW CREATE TABLE rename_table; + +SELECT * FROM rename_table ORDER BY k FORMAT TSVWithNames; + DROP TABLE IF EXISTS rename_table; + +SELECT '---polymorphic---'; + +DROP TABLE IF EXISTS rename_table_polymorphic; + +CREATE TABLE rename_table_polymorphic ( + key Int32, + value1 Int32, + value2 Int32 +) +ENGINE = MergeTree +ORDER BY tuple() +SETTINGS min_rows_for_wide_part = 10000; + +INSERT INTO rename_table_polymorphic VALUES (1, 2, 3); + +ALTER TABLE rename_table_polymorphic RENAME COLUMN value1 TO old_value1, RENAME COLUMN value2 TO value1; + +SHOW CREATE TABLE rename_table_polymorphic; + +SELECT * FROM rename_table_polymorphic FORMAT TSVWithNames; + +DROP TABLE IF EXISTS rename_table_polymorphic; From 81a5d02628f52d3be7ad6bb920d3f034929c558b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 14 May 2020 20:05:55 +0300 Subject: [PATCH 222/738] Fix annoying "Arcadia" build --- src/Common/StringUtils/StringUtils.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Common/StringUtils/StringUtils.h b/src/Common/StringUtils/StringUtils.h index 0843d4102a8..390015f3d2b 100644 --- a/src/Common/StringUtils/StringUtils.h +++ b/src/Common/StringUtils/StringUtils.h @@ -3,8 +3,8 @@ #include #include #include +#include #include -#include namespace detail @@ -167,7 +167,7 @@ inline const char * skipWhitespacesUTF8(const char * pos, const char * end) } else { - const char8_t * upos = reinterpret_cast(pos); + const uint8_t * upos = reinterpret_cast(pos); if (pos + 1 < end && upos[0] == 0xC2 && (upos[1] == 0x85 || upos[1] == 0xA0)) { From 273267ca8b0cdb8d0d38e9deac24ebc2aabeb0f3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 14 May 2020 20:36:59 +0300 Subject: [PATCH 223/738] Added a test for empty external data --- .../01279_empty_external_table.reference | 3 +++ .../0_stateless/01279_empty_external_table.sh | 13 +++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 tests/queries/0_stateless/01279_empty_external_table.reference create mode 100755 tests/queries/0_stateless/01279_empty_external_table.sh diff --git a/tests/queries/0_stateless/01279_empty_external_table.reference b/tests/queries/0_stateless/01279_empty_external_table.reference new file mode 100644 index 00000000000..bb5ee5c21eb --- /dev/null +++ b/tests/queries/0_stateless/01279_empty_external_table.reference @@ -0,0 +1,3 @@ +0 +0 +1 diff --git a/tests/queries/0_stateless/01279_empty_external_table.sh b/tests/queries/0_stateless/01279_empty_external_table.sh new file mode 100755 index 00000000000..62bcf4bb703 --- /dev/null +++ b/tests/queries/0_stateless/01279_empty_external_table.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +set -e + +touch ${CLICKHOUSE_TMP}/empty.tsv +clickhouse-client --query="SELECT count() FROM data" --external --file=${CLICKHOUSE_TMP}/empty.tsv --name=data --types=UInt32 +rm ${CLICKHOUSE_TMP}/empty.tsv + +echo -n | clickhouse-client --query="SELECT count() FROM data" --external --file=- --name=data --types=UInt32 +echo | clickhouse-client --query="SELECT count() FROM data" --external --file=- --name=data --types=String From bab624a628ff0ea3240849eee44baa28a5f3cb93 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 14 May 2020 20:53:38 +0300 Subject: [PATCH 224/738] Fix typo in clickhouse-client dockerfile --- docker/client/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile index 978538e73cc..bdb0582703b 100644 --- a/docker/client/Dockerfile +++ b/docker/client/Dockerfile @@ -16,7 +16,7 @@ RUN apt-get update \ apt-get install --allow-unauthenticated --yes --no-install-recommends \ clickhouse-client=$version \ clickhouse-common-static=$version \ - locales + locales \ && rm -rf /var/lib/apt/lists/* /var/cache/debconf \ && apt-get clean From 8e532a5d1f9504fbf215c72d4da584ef149b45c2 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Thu, 14 May 2020 21:17:15 +0300 Subject: [PATCH 225/738] Update MongoDBDictionarySource.cpp --- src/Dictionaries/MongoDBDictionarySource.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Dictionaries/MongoDBDictionarySource.cpp b/src/Dictionaries/MongoDBDictionarySource.cpp index 07f0b20e908..0828c9ff51b 100644 --- a/src/Dictionaries/MongoDBDictionarySource.cpp +++ b/src/Dictionaries/MongoDBDictionarySource.cpp @@ -17,9 +17,9 @@ void registerDictionarySourceMongoDB(DictionarySourceFactory & factory) bool /* check_config */) { return std::make_unique(dict_struct, - config.getString(config_prefix + ".uri"), - config.getString(config_prefix + ".host"), - config.getUInt(config_prefix + ".port"), + config.getString(config_prefix + ".uri", ""), + config.getString(config_prefix + ".host", ""), + config.getUInt(config_prefix + ".port", 0), config.getString(config_prefix + ".user", ""), config.getString(config_prefix + ".password", ""), config.getString(config_prefix + ".method", ""), From a11c9a3a228a8efb60a2f1c35f1ec3f5d73fdd8d Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Thu, 14 May 2020 21:18:37 +0300 Subject: [PATCH 226/738] Update MongoDBDictionarySource.cpp --- src/Dictionaries/MongoDBDictionarySource.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Dictionaries/MongoDBDictionarySource.cpp b/src/Dictionaries/MongoDBDictionarySource.cpp index 0828c9ff51b..4baa66728c9 100644 --- a/src/Dictionaries/MongoDBDictionarySource.cpp +++ b/src/Dictionaries/MongoDBDictionarySource.cpp @@ -8,7 +8,7 @@ namespace DB void registerDictionarySourceMongoDB(DictionarySourceFactory & factory) { - auto createMongoDBDictionary = []( + auto create_mongo_db_dictionary = []( const DictionaryStructure & dict_struct, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, @@ -28,7 +28,7 @@ void registerDictionarySourceMongoDB(DictionarySourceFactory & factory) sample_block); }; - factory.registerSource("mongodb", createMongoDBDictionary); + factory.registerSource("mongodb", create_mongo_db_dictionary); } } From 83d420e6085a4793bd444af07e45d4a2a92fb996 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 14 May 2020 21:21:35 +0300 Subject: [PATCH 227/738] Try fix tests. --- src/Functions/in.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Functions/in.cpp b/src/Functions/in.cpp index 55cae1afc6c..1b08df861cd 100644 --- a/src/Functions/in.cpp +++ b/src/Functions/in.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -77,7 +78,7 @@ public: { if constexpr (ignore_set) { - block.getByPosition(result).column = DataTypeUInt8().createColumnConst(input_rows_count, 0u); + block.getByPosition(result).column = ColumnUInt8::create(input_rows_count, 0u); return; } From a014a74e8a81cb21bf2f4132498f8cabe972b429 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 14 May 2020 21:37:13 +0300 Subject: [PATCH 228/738] Return tzdata --- debian/control | 2 +- docker/client/Dockerfile | 1 + docker/server/Dockerfile | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/debian/control b/debian/control index 58efd711d27..3ce12b504c2 100644 --- a/debian/control +++ b/debian/control @@ -28,7 +28,7 @@ Description: Client binary for ClickHouse Package: clickhouse-common-static Architecture: any -Depends: ${shlibs:Depends}, ${misc:Depends} +Depends: ${shlibs:Depends}, ${misc:Depends}, tzdata Suggests: clickhouse-common-static-dbg Replaces: clickhouse-common, clickhouse-server-base Provides: clickhouse-common, clickhouse-server-base diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile index bdb0582703b..5ca7e508d56 100644 --- a/docker/client/Dockerfile +++ b/docker/client/Dockerfile @@ -17,6 +17,7 @@ RUN apt-get update \ clickhouse-client=$version \ clickhouse-common-static=$version \ locales \ + tzdata \ && rm -rf /var/lib/apt/lists/* /var/cache/debconf \ && apt-get clean diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index faa5a593eb2..3ad20a9479c 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -21,6 +21,7 @@ RUN apt-get update \ locales \ ca-certificates \ wget \ + tzata \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/debconf \ From 3586fb1dc0c1b30b0c31dbfe61c0ed18ed75b6de Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Thu, 14 May 2020 23:53:11 +0400 Subject: [PATCH 229/738] Clean and adjust the generated files Add FreeBSD amd64 generated files/support --- cmake/find/ldap.cmake | 5 +- .../Darwin_x86_64/include/lber_types.h | 2 +- .../Darwin_x86_64/include/ldap_config.h | 20 +- .../Darwin_x86_64/include/ldap_features.h | 10 +- .../Darwin_x86_64/include/portable.h | 20 +- .../FreeBSD_amd64/include/lber_types.h | 63 + .../FreeBSD_amd64/include/ldap_config.h | 74 ++ .../FreeBSD_amd64/include/ldap_features.h | 61 + .../FreeBSD_amd64/include/portable.h | 1169 +++++++++++++++++ .../Linux_x86_64/include/lber_types.h | 2 +- .../Linux_x86_64/include/ldap_config.h | 20 +- .../Linux_x86_64/include/ldap_features.h | 10 +- .../Linux_x86_64/include/portable.h | 24 +- 13 files changed, 1424 insertions(+), 56 deletions(-) create mode 100644 contrib/openldap-cmake/FreeBSD_amd64/include/lber_types.h create mode 100644 contrib/openldap-cmake/FreeBSD_amd64/include/ldap_config.h create mode 100644 contrib/openldap-cmake/FreeBSD_amd64/include/ldap_features.h create mode 100644 contrib/openldap-cmake/FreeBSD_amd64/include/portable.h diff --git a/cmake/find/ldap.cmake b/cmake/find/ldap.cmake index f531afc446e..cbd8b50d66d 100644 --- a/cmake/find/ldap.cmake +++ b/cmake/find/ldap.cmake @@ -25,8 +25,9 @@ if (ENABLE_LDAP) if (NOT OPENLDAP_FOUND AND NOT MISSING_INTERNAL_LDAP_LIBRARY) if ( - ("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux" AND "${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "x86_64") OR - ("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin" AND "${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "x86_64") + ( "${CMAKE_SYSTEM_NAME}" STREQUAL "Linux" AND "${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "x86_64" ) OR + ( "${CMAKE_SYSTEM_NAME}" STREQUAL "FreeBSD" AND "${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "amd64" ) OR + ( "${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin" AND "${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "x86_64" ) ) set (_ldap_supported_platform TRUE) endif () diff --git a/contrib/openldap-cmake/Darwin_x86_64/include/lber_types.h b/contrib/openldap-cmake/Darwin_x86_64/include/lber_types.h index 0e04ad96034..dbd59430527 100644 --- a/contrib/openldap-cmake/Darwin_x86_64/include/lber_types.h +++ b/contrib/openldap-cmake/Darwin_x86_64/include/lber_types.h @@ -1,4 +1,4 @@ -/* include/lber_types.h. Generated from lber_types.hin by configure. */ +/* include/lber_types.h. Generated from lber_types.hin by configure. */ /* $OpenLDAP$ */ /* This work is part of OpenLDAP Software . * diff --git a/contrib/openldap-cmake/Darwin_x86_64/include/ldap_config.h b/contrib/openldap-cmake/Darwin_x86_64/include/ldap_config.h index ce1c6de9714..89f7b40b884 100644 --- a/contrib/openldap-cmake/Darwin_x86_64/include/ldap_config.h +++ b/contrib/openldap-cmake/Darwin_x86_64/include/ldap_config.h @@ -1,4 +1,4 @@ -/* Generated from /Users/denis/dev/altinity/ClickHouse/contrib/openldap/include/ldap_config.hin on Sat May 9 00:43:25 +04 2020 */ +/* include/ldap_config.h. Generated from ldap_config.hin by configure. */ /* $OpenLDAP$ */ /* This work is part of OpenLDAP Software . * @@ -33,7 +33,7 @@ /* directory for temporary files */ #if defined(_WIN32) -# define LDAP_TMPDIR "C:\\." /* we don't have much of a choice */ +# define LDAP_TMPDIR "C:\\." /* we don't have much of a choice */ #elif defined( _P_tmpdir ) # define LDAP_TMPDIR _P_tmpdir #elif defined( P_tmpdir ) @@ -46,28 +46,28 @@ /* directories */ #ifndef LDAP_BINDIR -#define LDAP_BINDIR "/tmp/ldap-prefix/bin" +#define LDAP_BINDIR "/tmp/ldap-prefix/bin" #endif #ifndef LDAP_SBINDIR -#define LDAP_SBINDIR "/tmp/ldap-prefix/sbin" +#define LDAP_SBINDIR "/tmp/ldap-prefix/sbin" #endif #ifndef LDAP_DATADIR -#define LDAP_DATADIR "/tmp/ldap-prefix/share/openldap" +#define LDAP_DATADIR "/tmp/ldap-prefix/share/openldap" #endif #ifndef LDAP_SYSCONFDIR -#define LDAP_SYSCONFDIR "/tmp/ldap-prefix/etc/openldap" +#define LDAP_SYSCONFDIR "/tmp/ldap-prefix/etc/openldap" #endif #ifndef LDAP_LIBEXECDIR -#define LDAP_LIBEXECDIR "/tmp/ldap-prefix/libexec" +#define LDAP_LIBEXECDIR "/tmp/ldap-prefix/libexec" #endif #ifndef LDAP_MODULEDIR -#define LDAP_MODULEDIR "/tmp/ldap-prefix/libexec/openldap" +#define LDAP_MODULEDIR "/tmp/ldap-prefix/libexec/openldap" #endif #ifndef LDAP_RUNDIR -#define LDAP_RUNDIR "/tmp/ldap-prefix/var" +#define LDAP_RUNDIR "/tmp/ldap-prefix/var" #endif #ifndef LDAP_LOCALEDIR -#define LDAP_LOCALEDIR "" +#define LDAP_LOCALEDIR "" #endif diff --git a/contrib/openldap-cmake/Darwin_x86_64/include/ldap_features.h b/contrib/openldap-cmake/Darwin_x86_64/include/ldap_features.h index 7ec8de4e9b2..f0cc7c3626f 100644 --- a/contrib/openldap-cmake/Darwin_x86_64/include/ldap_features.h +++ b/contrib/openldap-cmake/Darwin_x86_64/include/ldap_features.h @@ -1,4 +1,4 @@ -/* include/ldap_features.h. Generated from ldap_features.hin by configure. */ +/* include/ldap_features.h. Generated from ldap_features.hin by configure. */ /* $OpenLDAP$ */ /* This work is part of OpenLDAP Software . * @@ -14,7 +14,7 @@ * . */ -/* +/* * LDAP Features */ @@ -36,10 +36,10 @@ ** The -lldap implementation is not thread-safe. ** ** The -lldap_r implementation is: -** LDAP_API_FEATURE_THREAD_SAFE (basic thread safety) +** LDAP_API_FEATURE_THREAD_SAFE (basic thread safety) ** but also be: -** LDAP_API_FEATURE_SESSION_THREAD_SAFE -** LDAP_API_FEATURE_OPERATION_THREAD_SAFE +** LDAP_API_FEATURE_SESSION_THREAD_SAFE +** LDAP_API_FEATURE_OPERATION_THREAD_SAFE ** ** The preprocessor flag LDAP_API_FEATURE_X_OPENLDAP_THREAD_SAFE ** can be used to determine if -lldap_r is available at compile diff --git a/contrib/openldap-cmake/Darwin_x86_64/include/portable.h b/contrib/openldap-cmake/Darwin_x86_64/include/portable.h index e66e842acee..fdf4e89017e 100644 --- a/contrib/openldap-cmake/Darwin_x86_64/include/portable.h +++ b/contrib/openldap-cmake/Darwin_x86_64/include/portable.h @@ -1,5 +1,5 @@ -/* include/portable.h. Generated from portable.hin by configure. */ -/* include/portable.hin. Generated from configure.in by autoheader. */ +/* include/portable.h. Generated from portable.hin by configure. */ +/* include/portable.hin. Generated from configure.in by autoheader. */ /* begin of portable.h.pre */ @@ -369,13 +369,13 @@ #define HAVE_OPENSSL 1 /* Define to 1 if you have the header file. */ -/* #undef HAVE_OPENSSL_BN_H */ +#define HAVE_OPENSSL_BN_H 1 /* define if you have OpenSSL with CRL checking capability */ #define HAVE_OPENSSL_CRL 1 /* Define to 1 if you have the header file. */ -/* #undef HAVE_OPENSSL_CRYPTO_H */ +#define HAVE_OPENSSL_CRYPTO_H 1 /* Define to 1 if you have the header file. */ #define HAVE_OPENSSL_SSL_H 1 @@ -444,10 +444,10 @@ #define HAVE_REGEX_H 1 /* Define to 1 if you have the header file. */ -#define HAVE_RESOLV_H 1 +/* #undef HAVE_RESOLV_H */ /* define if you have res_query() */ -#define HAVE_RES_QUERY 1 +/* #undef HAVE_RES_QUERY */ /* define if OpenSSL needs RSAref */ /* #undef HAVE_RSAREF */ @@ -1121,8 +1121,8 @@ /* begin of portable.h.post */ #ifdef _WIN32 - /* don't suck in all of the win32 api */ -# define WIN32_LEAN_AND_MEAN 1 +/* don't suck in all of the win32 api */ +# define WIN32_LEAN_AND_MEAN 1 #endif #ifndef LDAP_NEEDS_PROTOTYPES @@ -1140,10 +1140,10 @@ #endif #ifdef HAVE_STDDEF_H -# include +# include #endif -#ifdef HAVE_EBCDIC +#ifdef HAVE_EBCDIC /* ASCII/EBCDIC converting replacements for stdio funcs * vsnprintf and snprintf are used too, but they are already * checked by the configure script diff --git a/contrib/openldap-cmake/FreeBSD_amd64/include/lber_types.h b/contrib/openldap-cmake/FreeBSD_amd64/include/lber_types.h new file mode 100644 index 00000000000..dbd59430527 --- /dev/null +++ b/contrib/openldap-cmake/FreeBSD_amd64/include/lber_types.h @@ -0,0 +1,63 @@ +/* include/lber_types.h. Generated from lber_types.hin by configure. */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software . + * + * Copyright 1998-2020 The OpenLDAP Foundation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +/* + * LBER types + */ + +#ifndef _LBER_TYPES_H +#define _LBER_TYPES_H + +#include + +LDAP_BEGIN_DECL + +/* LBER boolean, enum, integers (32 bits or larger) */ +#define LBER_INT_T int + +/* LBER tags (32 bits or larger) */ +#define LBER_TAG_T long + +/* LBER socket descriptor */ +#define LBER_SOCKET_T int + +/* LBER lengths (32 bits or larger) */ +#define LBER_LEN_T long + +/* ------------------------------------------------------------ */ + +/* booleans, enumerations, and integers */ +typedef LBER_INT_T ber_int_t; + +/* signed and unsigned versions */ +typedef signed LBER_INT_T ber_sint_t; +typedef unsigned LBER_INT_T ber_uint_t; + +/* tags */ +typedef unsigned LBER_TAG_T ber_tag_t; + +/* "socket" descriptors */ +typedef LBER_SOCKET_T ber_socket_t; + +/* lengths */ +typedef unsigned LBER_LEN_T ber_len_t; + +/* signed lengths */ +typedef signed LBER_LEN_T ber_slen_t; + +LDAP_END_DECL + +#endif /* _LBER_TYPES_H */ diff --git a/contrib/openldap-cmake/FreeBSD_amd64/include/ldap_config.h b/contrib/openldap-cmake/FreeBSD_amd64/include/ldap_config.h new file mode 100644 index 00000000000..89f7b40b884 --- /dev/null +++ b/contrib/openldap-cmake/FreeBSD_amd64/include/ldap_config.h @@ -0,0 +1,74 @@ +/* include/ldap_config.h. Generated from ldap_config.hin by configure. */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software . + * + * Copyright 1998-2020 The OpenLDAP Foundation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +/* + * This file works in conjunction with OpenLDAP configure system. + * If you do no like the values below, adjust your configure options. + */ + +#ifndef _LDAP_CONFIG_H +#define _LDAP_CONFIG_H + +/* directory separator */ +#ifndef LDAP_DIRSEP +#ifndef _WIN32 +#define LDAP_DIRSEP "/" +#else +#define LDAP_DIRSEP "\\" +#endif +#endif + +/* directory for temporary files */ +#if defined(_WIN32) +# define LDAP_TMPDIR "C:\\." /* we don't have much of a choice */ +#elif defined( _P_tmpdir ) +# define LDAP_TMPDIR _P_tmpdir +#elif defined( P_tmpdir ) +# define LDAP_TMPDIR P_tmpdir +#elif defined( _PATH_TMPDIR ) +# define LDAP_TMPDIR _PATH_TMPDIR +#else +# define LDAP_TMPDIR LDAP_DIRSEP "tmp" +#endif + +/* directories */ +#ifndef LDAP_BINDIR +#define LDAP_BINDIR "/tmp/ldap-prefix/bin" +#endif +#ifndef LDAP_SBINDIR +#define LDAP_SBINDIR "/tmp/ldap-prefix/sbin" +#endif +#ifndef LDAP_DATADIR +#define LDAP_DATADIR "/tmp/ldap-prefix/share/openldap" +#endif +#ifndef LDAP_SYSCONFDIR +#define LDAP_SYSCONFDIR "/tmp/ldap-prefix/etc/openldap" +#endif +#ifndef LDAP_LIBEXECDIR +#define LDAP_LIBEXECDIR "/tmp/ldap-prefix/libexec" +#endif +#ifndef LDAP_MODULEDIR +#define LDAP_MODULEDIR "/tmp/ldap-prefix/libexec/openldap" +#endif +#ifndef LDAP_RUNDIR +#define LDAP_RUNDIR "/tmp/ldap-prefix/var" +#endif +#ifndef LDAP_LOCALEDIR +#define LDAP_LOCALEDIR "" +#endif + + +#endif /* _LDAP_CONFIG_H */ diff --git a/contrib/openldap-cmake/FreeBSD_amd64/include/ldap_features.h b/contrib/openldap-cmake/FreeBSD_amd64/include/ldap_features.h new file mode 100644 index 00000000000..f0cc7c3626f --- /dev/null +++ b/contrib/openldap-cmake/FreeBSD_amd64/include/ldap_features.h @@ -0,0 +1,61 @@ +/* include/ldap_features.h. Generated from ldap_features.hin by configure. */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software . + * + * Copyright 1998-2020 The OpenLDAP Foundation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +/* + * LDAP Features + */ + +#ifndef _LDAP_FEATURES_H +#define _LDAP_FEATURES_H 1 + +/* OpenLDAP API version macros */ +#define LDAP_VENDOR_VERSION 20501 +#define LDAP_VENDOR_VERSION_MAJOR 2 +#define LDAP_VENDOR_VERSION_MINOR 5 +#define LDAP_VENDOR_VERSION_PATCH X + +/* +** WORK IN PROGRESS! +** +** OpenLDAP reentrancy/thread-safeness should be dynamically +** checked using ldap_get_option(). +** +** The -lldap implementation is not thread-safe. +** +** The -lldap_r implementation is: +** LDAP_API_FEATURE_THREAD_SAFE (basic thread safety) +** but also be: +** LDAP_API_FEATURE_SESSION_THREAD_SAFE +** LDAP_API_FEATURE_OPERATION_THREAD_SAFE +** +** The preprocessor flag LDAP_API_FEATURE_X_OPENLDAP_THREAD_SAFE +** can be used to determine if -lldap_r is available at compile +** time. You must define LDAP_THREAD_SAFE if and only if you +** link with -lldap_r. +** +** If you fail to define LDAP_THREAD_SAFE when linking with +** -lldap_r or define LDAP_THREAD_SAFE when linking with -lldap, +** provided header definitions and declarations may be incorrect. +** +*/ + +/* is -lldap_r available or not */ +#define LDAP_API_FEATURE_X_OPENLDAP_THREAD_SAFE 1 + +/* LDAP v2 Referrals */ +/* #undef LDAP_API_FEATURE_X_OPENLDAP_V2_REFERRALS */ + +#endif /* LDAP_FEATURES */ diff --git a/contrib/openldap-cmake/FreeBSD_amd64/include/portable.h b/contrib/openldap-cmake/FreeBSD_amd64/include/portable.h new file mode 100644 index 00000000000..10a15fe3ca1 --- /dev/null +++ b/contrib/openldap-cmake/FreeBSD_amd64/include/portable.h @@ -0,0 +1,1169 @@ +/* include/portable.h. Generated from portable.hin by configure. */ +/* include/portable.hin. Generated from configure.in by autoheader. */ + + +/* begin of portable.h.pre */ +/* This work is part of OpenLDAP Software . + * + * Copyright 1998-2020 The OpenLDAP Foundation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#ifndef _LDAP_PORTABLE_H +#define _LDAP_PORTABLE_H + +/* define this if needed to get reentrant functions */ +#ifndef REENTRANT +#define REENTRANT 1 +#endif +#ifndef _REENTRANT +#define _REENTRANT 1 +#endif + +/* define this if needed to get threadsafe functions */ +#ifndef THREADSAFE +#define THREADSAFE 1 +#endif +#ifndef _THREADSAFE +#define _THREADSAFE 1 +#endif +#ifndef THREAD_SAFE +#define THREAD_SAFE 1 +#endif +#ifndef _THREAD_SAFE +#define _THREAD_SAFE 1 +#endif + +#ifndef _SGI_MP_SOURCE +#define _SGI_MP_SOURCE 1 +#endif + +/* end of portable.h.pre */ + + +/* Define if building universal (internal helper macro) */ +/* #undef AC_APPLE_UNIVERSAL_BUILD */ + +/* define to use both and */ +/* #undef BOTH_STRINGS_H */ + +/* define if cross compiling */ +/* #undef CROSS_COMPILING */ + +/* set to the number of arguments ctime_r() expects */ +#define CTIME_R_NARGS 2 + +/* define if toupper() requires islower() */ +/* #undef C_UPPER_LOWER */ + +/* define if sys_errlist is not declared in stdio.h or errno.h */ +/* #undef DECL_SYS_ERRLIST */ + +/* define to enable slapi library */ +/* #undef ENABLE_SLAPI */ + +/* defined to be the EXE extension */ +#define EXEEXT "" + +/* set to the number of arguments gethostbyaddr_r() expects */ +#define GETHOSTBYADDR_R_NARGS 8 + +/* set to the number of arguments gethostbyname_r() expects */ +#define GETHOSTBYNAME_R_NARGS 6 + +/* Define to 1 if `TIOCGWINSZ' requires . */ +/* #undef GWINSZ_IN_SYS_IOCTL */ + +/* define if you have AIX security lib */ +/* #undef HAVE_AIX_SECURITY */ + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_INET_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_NAMESER_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ASSERT_H 1 + +/* Define to 1 if you have the `bcopy' function. */ +#define HAVE_BCOPY 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_BITS_TYPES_H */ + +/* Define to 1 if you have the `chroot' function. */ +#define HAVE_CHROOT 1 + +/* Define to 1 if you have the `closesocket' function. */ +/* #undef HAVE_CLOSESOCKET */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_CONIO_H */ + +/* define if crypt(3) is available */ +/* #undef HAVE_CRYPT */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_CRYPT_H */ + +/* define if crypt_r() is also available */ +/* #undef HAVE_CRYPT_R */ + +/* Define to 1 if you have the `ctime_r' function. */ +#define HAVE_CTIME_R 1 + +/* define if you have Cyrus SASL */ +/* #undef HAVE_CYRUS_SASL */ + +/* define if your system supports /dev/poll */ +/* #undef HAVE_DEVPOLL */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_DIRECT_H */ + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +#define HAVE_DIRENT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_DLFCN_H 1 + +/* Define to 1 if you don't have `vprintf' but do have `_doprnt.' */ +/* #undef HAVE_DOPRNT */ + +/* define if system uses EBCDIC instead of ASCII */ +/* #undef HAVE_EBCDIC */ + +/* Define to 1 if you have the `endgrent' function. */ +#define HAVE_ENDGRENT 1 + +/* Define to 1 if you have the `endpwent' function. */ +#define HAVE_ENDPWENT 1 + +/* define if your system supports epoll */ +/* #undef HAVE_EPOLL */ + +/* Define to 1 if you have the header file. */ +#define HAVE_ERRNO_H 1 + +/* Define to 1 if you have the `fcntl' function. */ +#define HAVE_FCNTL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_FCNTL_H 1 + +/* define if you actually have FreeBSD fetch(3) */ +/* #undef HAVE_FETCH */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_FILIO_H */ + +/* Define to 1 if you have the `flock' function. */ +#define HAVE_FLOCK 1 + +/* Define to 1 if you have the `fstat' function. */ +#define HAVE_FSTAT 1 + +/* Define to 1 if you have the `gai_strerror' function. */ +#define HAVE_GAI_STRERROR 1 + +/* Define to 1 if you have the `getaddrinfo' function. */ +#define HAVE_GETADDRINFO 1 + +/* Define to 1 if you have the `getdtablesize' function. */ +#define HAVE_GETDTABLESIZE 1 + +/* Define to 1 if you have the `geteuid' function. */ +#define HAVE_GETEUID 1 + +/* Define to 1 if you have the `getgrgid' function. */ +#define HAVE_GETGRGID 1 + +/* Define to 1 if you have the `gethostbyaddr_r' function. */ +#define HAVE_GETHOSTBYADDR_R 1 + +/* Define to 1 if you have the `gethostbyname_r' function. */ +#define HAVE_GETHOSTBYNAME_R 1 + +/* Define to 1 if you have the `gethostname' function. */ +#define HAVE_GETHOSTNAME 1 + +/* Define to 1 if you have the `getnameinfo' function. */ +#define HAVE_GETNAMEINFO 1 + +/* Define to 1 if you have the `getopt' function. */ +#define HAVE_GETOPT 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_GETOPT_H 1 + +/* Define to 1 if you have the `getpassphrase' function. */ +/* #undef HAVE_GETPASSPHRASE */ + +/* Define to 1 if you have the `getpeereid' function. */ +#define HAVE_GETPEEREID 1 + +/* Define to 1 if you have the `getpeerucred' function. */ +/* #undef HAVE_GETPEERUCRED */ + +/* Define to 1 if you have the `getpwnam' function. */ +#define HAVE_GETPWNAM 1 + +/* Define to 1 if you have the `getpwuid' function. */ +#define HAVE_GETPWUID 1 + +/* Define to 1 if you have the `getspnam' function. */ +/* #undef HAVE_GETSPNAM */ + +/* Define to 1 if you have the `gettimeofday' function. */ +#define HAVE_GETTIMEOFDAY 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_GMP_H */ + +/* Define to 1 if you have the `gmtime_r' function. */ +#define HAVE_GMTIME_R 1 + +/* define if you have GNUtls */ +/* #undef HAVE_GNUTLS */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_GNUTLS_GNUTLS_H */ + +/* if you have GNU Pth */ +/* #undef HAVE_GNU_PTH */ + +/* Define to 1 if you have the header file. */ +#define HAVE_GRP_H 1 + +/* Define to 1 if you have the `hstrerror' function. */ +#define HAVE_HSTRERROR 1 + +/* define to you inet_aton(3) is available */ +#define HAVE_INET_ATON 1 + +/* Define to 1 if you have the `inet_ntoa_b' function. */ +/* #undef HAVE_INET_NTOA_B */ + +/* Define to 1 if you have the `inet_ntop' function. */ +#define HAVE_INET_NTOP 1 + +/* Define to 1 if you have the `initgroups' function. */ +#define HAVE_INITGROUPS 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_INTTYPES_H 1 + +/* Define to 1 if you have the `ioctl' function. */ +#define HAVE_IOCTL 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_IO_H */ + +/* define if your system supports kqueue */ +#define HAVE_KQUEUE 1 + +/* Define to 1 if you have the `gen' library (-lgen). */ +/* #undef HAVE_LIBGEN */ + +/* Define to 1 if you have the `gmp' library (-lgmp). */ +/* #undef HAVE_LIBGMP */ + +/* Define to 1 if you have the `inet' library (-linet). */ +/* #undef HAVE_LIBINET */ + +/* define if you have libtool -ltdl */ +/* #undef HAVE_LIBLTDL */ + +/* Define to 1 if you have the `net' library (-lnet). */ +/* #undef HAVE_LIBNET */ + +/* Define to 1 if you have the `nsl' library (-lnsl). */ +/* #undef HAVE_LIBNSL */ + +/* Define to 1 if you have the `nsl_s' library (-lnsl_s). */ +/* #undef HAVE_LIBNSL_S */ + +/* Define to 1 if you have the `socket' library (-lsocket). */ +/* #undef HAVE_LIBSOCKET */ + +/* Define to 1 if you have the header file. */ +#define HAVE_LIBUTIL_H 1 + +/* Define to 1 if you have the `V3' library (-lV3). */ +/* #undef HAVE_LIBV3 */ + +/* Define to 1 if you have the header file. */ +#define HAVE_LIMITS_H 1 + +/* if you have LinuxThreads */ +/* #undef HAVE_LINUX_THREADS */ + +/* Define to 1 if you have the header file. */ +#define HAVE_LOCALE_H 1 + +/* Define to 1 if you have the `localtime_r' function. */ +#define HAVE_LOCALTIME_R 1 + +/* Define to 1 if you have the `lockf' function. */ +#define HAVE_LOCKF 1 + +/* Define to 1 if the system has the type `long long'. */ +#define HAVE_LONG_LONG 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_LTDL_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_MALLOC_H */ + +/* Define to 1 if you have the `memcpy' function. */ +#define HAVE_MEMCPY 1 + +/* Define to 1 if you have the `memmove' function. */ +#define HAVE_MEMMOVE 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_MEMORY_H 1 + +/* Define to 1 if you have the `memrchr' function. */ +#define HAVE_MEMRCHR 1 + +/* Define to 1 if you have the `mkstemp' function. */ +#define HAVE_MKSTEMP 1 + +/* Define to 1 if you have the `mktemp' function. */ +#define HAVE_MKTEMP 1 + +/* define this if you have mkversion */ +#define HAVE_MKVERSION 1 + +/* Define to 1 if you have the header file, and it defines `DIR'. */ +/* #undef HAVE_NDIR_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_NETINET_TCP_H 1 + +/* define if strerror_r returns char* instead of int */ +/* #undef HAVE_NONPOSIX_STRERROR_R */ + +/* if you have NT Event Log */ +/* #undef HAVE_NT_EVENT_LOG */ + +/* if you have NT Service Manager */ +/* #undef HAVE_NT_SERVICE_MANAGER */ + +/* if you have NT Threads */ +/* #undef HAVE_NT_THREADS */ + +/* define if you have OpenSSL */ +#define HAVE_OPENSSL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_OPENSSL_BN_H 1 + +/* define if you have OpenSSL with CRL checking capability */ +#define HAVE_OPENSSL_CRL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_OPENSSL_CRYPTO_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_OPENSSL_SSL_H 1 + +/* Define to 1 if you have the `pipe' function. */ +#define HAVE_PIPE 1 + +/* Define to 1 if you have the `poll' function. */ +#define HAVE_POLL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_POLL_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_PROCESS_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_PSAP_H */ + +/* define to pthreads API spec revision */ +#define HAVE_PTHREADS 10 + +/* define if you have pthread_detach function */ +#define HAVE_PTHREAD_DETACH 1 + +/* Define to 1 if you have the `pthread_getconcurrency' function. */ +#define HAVE_PTHREAD_GETCONCURRENCY 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_PTHREAD_H 1 + +/* Define to 1 if you have the `pthread_kill' function. */ +#define HAVE_PTHREAD_KILL 1 + +/* Define to 1 if you have the `pthread_kill_other_threads_np' function. */ +/* #undef HAVE_PTHREAD_KILL_OTHER_THREADS_NP */ + +/* define if you have pthread_rwlock_destroy function */ +#define HAVE_PTHREAD_RWLOCK_DESTROY 1 + +/* Define to 1 if you have the `pthread_setconcurrency' function. */ +#define HAVE_PTHREAD_SETCONCURRENCY 1 + +/* Define to 1 if you have the `pthread_yield' function. */ +#define HAVE_PTHREAD_YIELD 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_PTH_H */ + +/* Define to 1 if the system has the type `ptrdiff_t'. */ +#define HAVE_PTRDIFF_T 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_PWD_H 1 + +/* Define to 1 if you have the `read' function. */ +#define HAVE_READ 1 + +/* Define to 1 if you have the `recv' function. */ +#define HAVE_RECV 1 + +/* Define to 1 if you have the `recvfrom' function. */ +#define HAVE_RECVFROM 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_REGEX_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_RESOLV_H */ + +/* define if you have res_query() */ +/* #undef HAVE_RES_QUERY */ + +/* define if OpenSSL needs RSAref */ +/* #undef HAVE_RSAREF */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SASL_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SASL_SASL_H */ + +/* define if your SASL library has sasl_version() */ +/* #undef HAVE_SASL_VERSION */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SCHED_H 1 + +/* Define to 1 if you have the `sched_yield' function. */ +#define HAVE_SCHED_YIELD 1 + +/* Define to 1 if you have the `send' function. */ +#define HAVE_SEND 1 + +/* Define to 1 if you have the `sendmsg' function. */ +#define HAVE_SENDMSG 1 + +/* Define to 1 if you have the `sendto' function. */ +#define HAVE_SENDTO 1 + +/* Define to 1 if you have the `setegid' function. */ +#define HAVE_SETEGID 1 + +/* Define to 1 if you have the `seteuid' function. */ +#define HAVE_SETEUID 1 + +/* Define to 1 if you have the `setgid' function. */ +#define HAVE_SETGID 1 + +/* Define to 1 if you have the `setpwfile' function. */ +/* #undef HAVE_SETPWFILE */ + +/* Define to 1 if you have the `setsid' function. */ +#define HAVE_SETSID 1 + +/* Define to 1 if you have the `setuid' function. */ +#define HAVE_SETUID 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SGTTY_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SHADOW_H */ + +/* Define to 1 if you have the `sigaction' function. */ +#define HAVE_SIGACTION 1 + +/* Define to 1 if you have the `signal' function. */ +#define HAVE_SIGNAL 1 + +/* Define to 1 if you have the `sigset' function. */ +#define HAVE_SIGSET 1 + +/* define if you have -lslp */ +/* #undef HAVE_SLP */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SLP_H */ + +/* Define to 1 if you have the `snprintf' function. */ +#define HAVE_SNPRINTF 1 + +/* if you have spawnlp() */ +/* #undef HAVE_SPAWNLP */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SQLEXT_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SQL_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STDDEF_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDINT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDLIB_H 1 + +/* Define to 1 if you have the `strdup' function. */ +#define HAVE_STRDUP 1 + +/* Define to 1 if you have the `strerror' function. */ +#define HAVE_STRERROR 1 + +/* Define to 1 if you have the `strerror_r' function. */ +#define HAVE_STRERROR_R 1 + +/* Define to 1 if you have the `strftime' function. */ +#define HAVE_STRFTIME 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRINGS_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRING_H 1 + +/* Define to 1 if you have the `strpbrk' function. */ +#define HAVE_STRPBRK 1 + +/* Define to 1 if you have the `strrchr' function. */ +#define HAVE_STRRCHR 1 + +/* Define to 1 if you have the `strsep' function. */ +#define HAVE_STRSEP 1 + +/* Define to 1 if you have the `strspn' function. */ +#define HAVE_STRSPN 1 + +/* Define to 1 if you have the `strstr' function. */ +#define HAVE_STRSTR 1 + +/* Define to 1 if you have the `strtol' function. */ +#define HAVE_STRTOL 1 + +/* Define to 1 if you have the `strtoll' function. */ +#define HAVE_STRTOLL 1 + +/* Define to 1 if you have the `strtoq' function. */ +#define HAVE_STRTOQ 1 + +/* Define to 1 if you have the `strtoul' function. */ +#define HAVE_STRTOUL 1 + +/* Define to 1 if you have the `strtoull' function. */ +#define HAVE_STRTOULL 1 + +/* Define to 1 if you have the `strtouq' function. */ +#define HAVE_STRTOUQ 1 + +/* Define to 1 if `msg_accrightslen' is a member of `struct msghdr'. */ +/* #undef HAVE_STRUCT_MSGHDR_MSG_ACCRIGHTSLEN */ + +/* Define to 1 if `msg_control' is a member of `struct msghdr'. */ +/* #undef HAVE_STRUCT_MSGHDR_MSG_CONTROL */ + +/* Define to 1 if `pw_gecos' is a member of `struct passwd'. */ +#define HAVE_STRUCT_PASSWD_PW_GECOS 1 + +/* Define to 1 if `pw_passwd' is a member of `struct passwd'. */ +#define HAVE_STRUCT_PASSWD_PW_PASSWD 1 + +/* Define to 1 if `st_blksize' is a member of `struct stat'. */ +#define HAVE_STRUCT_STAT_ST_BLKSIZE 1 + +/* Define to 1 if `st_fstype' is a member of `struct stat'. */ +/* #undef HAVE_STRUCT_STAT_ST_FSTYPE */ + +/* define to 1 if st_fstype is char * */ +/* #undef HAVE_STRUCT_STAT_ST_FSTYPE_CHAR */ + +/* define to 1 if st_fstype is int */ +/* #undef HAVE_STRUCT_STAT_ST_FSTYPE_INT */ + +/* Define to 1 if `st_vfstype' is a member of `struct stat'. */ +/* #undef HAVE_STRUCT_STAT_ST_VFSTYPE */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYNCH_H */ + +/* Define to 1 if you have the `sysconf' function. */ +#define HAVE_SYSCONF 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYSEXITS_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYSLOG_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_DEVPOLL_H */ + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +/* #undef HAVE_SYS_DIR_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_EPOLL_H */ + +/* define if you actually have sys_errlist in your libs */ +#define HAVE_SYS_ERRLIST 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_ERRNO_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_EVENT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_FILE_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_FILIO_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_FSTYP_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_IOCTL_H 1 + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +/* #undef HAVE_SYS_NDIR_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_PARAM_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_POLL_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_PRIVGRP_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_RESOURCE_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SELECT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SOCKET_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_STAT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SYSLOG_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TIME_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_UCRED_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_UIO_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_UN_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_UUID_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_VMOUNT_H */ + +/* Define to 1 if you have that is POSIX.1 compatible. */ +#define HAVE_SYS_WAIT_H 1 + +/* define if you have -lwrap */ +/* #undef HAVE_TCPD */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_TCPD_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_TERMIOS_H 1 + +/* if you have Solaris LWP (thr) package */ +/* #undef HAVE_THR */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_THREAD_H */ + +/* Define to 1 if you have the `thr_getconcurrency' function. */ +/* #undef HAVE_THR_GETCONCURRENCY */ + +/* Define to 1 if you have the `thr_setconcurrency' function. */ +/* #undef HAVE_THR_SETCONCURRENCY */ + +/* Define to 1 if you have the `thr_yield' function. */ +/* #undef HAVE_THR_YIELD */ + +/* define if you have TLS */ +#define HAVE_TLS 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_UNISTD_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_UTIME_H 1 + +/* define if you have uuid_generate() */ +/* #undef HAVE_UUID_GENERATE */ + +/* define if you have uuid_to_str() */ +/* #undef HAVE_UUID_TO_STR */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_UUID_UUID_H */ + +/* Define to 1 if you have the `vprintf' function. */ +#define HAVE_VPRINTF 1 + +/* Define to 1 if you have the `vsnprintf' function. */ +#define HAVE_VSNPRINTF 1 + +/* Define to 1 if you have the `wait4' function. */ +#define HAVE_WAIT4 1 + +/* Define to 1 if you have the `waitpid' function. */ +#define HAVE_WAITPID 1 + +/* define if you have winsock */ +/* #undef HAVE_WINSOCK */ + +/* define if you have winsock2 */ +/* #undef HAVE_WINSOCK2 */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_WINSOCK2_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_WINSOCK_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_WIREDTIGER_H */ + +/* Define to 1 if you have the `write' function. */ +#define HAVE_WRITE 1 + +/* define if select implicitly yields */ +#define HAVE_YIELDING_SELECT 1 + +/* Define to 1 if you have the `_vsnprintf' function. */ +/* #undef HAVE__VSNPRINTF */ + +/* define to 32-bit or greater integer type */ +#define LBER_INT_T int + +/* define to large integer type */ +#define LBER_LEN_T long + +/* define to socket descriptor type */ +#define LBER_SOCKET_T int + +/* define to large integer type */ +#define LBER_TAG_T long + +/* define to 1 if library is thread safe */ +#define LDAP_API_FEATURE_X_OPENLDAP_THREAD_SAFE 1 + +/* define to LDAP VENDOR VERSION */ +/* #undef LDAP_API_FEATURE_X_OPENLDAP_V2_REFERRALS */ + +/* define this to add debugging code */ +/* #undef LDAP_DEBUG */ + +/* define if LDAP libs are dynamic */ +/* #undef LDAP_LIBS_DYNAMIC */ + +/* define to support PF_INET6 */ +#define LDAP_PF_INET6 1 + +/* define to support PF_LOCAL */ +#define LDAP_PF_LOCAL 1 + +/* define this to add SLAPI code */ +/* #undef LDAP_SLAPI */ + +/* define this to add syslog code */ +/* #undef LDAP_SYSLOG */ + +/* Version */ +#define LDAP_VENDOR_VERSION 20501 + +/* Major */ +#define LDAP_VENDOR_VERSION_MAJOR 2 + +/* Minor */ +#define LDAP_VENDOR_VERSION_MINOR 5 + +/* Patch */ +#define LDAP_VENDOR_VERSION_PATCH X + +/* Define to the sub-directory where libtool stores uninstalled libraries. */ +#define LT_OBJDIR ".libs/" + +/* define if memcmp is not 8-bit clean or is otherwise broken */ +/* #undef NEED_MEMCMP_REPLACEMENT */ + +/* define if you have (or want) no threads */ +/* #undef NO_THREADS */ + +/* define to use the original debug style */ +/* #undef OLD_DEBUG */ + +/* Package */ +#define OPENLDAP_PACKAGE "OpenLDAP" + +/* Version */ +#define OPENLDAP_VERSION "2.5.X" + +/* Define to the address where bug reports for this package should be sent. */ +#define PACKAGE_BUGREPORT "" + +/* Define to the full name of this package. */ +#define PACKAGE_NAME "" + +/* Define to the full name and version of this package. */ +#define PACKAGE_STRING "" + +/* Define to the one symbol short name of this package. */ +#define PACKAGE_TARNAME "" + +/* Define to the home page for this package. */ +#define PACKAGE_URL "" + +/* Define to the version of this package. */ +#define PACKAGE_VERSION "" + +/* define if sched_yield yields the entire process */ +/* #undef REPLACE_BROKEN_YIELD */ + +/* Define as the return type of signal handlers (`int' or `void'). */ +#define RETSIGTYPE void + +/* Define to the type of arg 1 for `select'. */ +#define SELECT_TYPE_ARG1 int + +/* Define to the type of args 2, 3 and 4 for `select'. */ +#define SELECT_TYPE_ARG234 (fd_set *) + +/* Define to the type of arg 5 for `select'. */ +#define SELECT_TYPE_ARG5 (struct timeval *) + +/* The size of `int', as computed by sizeof. */ +#define SIZEOF_INT 4 + +/* The size of `long', as computed by sizeof. */ +#define SIZEOF_LONG 8 + +/* The size of `long long', as computed by sizeof. */ +#define SIZEOF_LONG_LONG 8 + +/* The size of `short', as computed by sizeof. */ +#define SIZEOF_SHORT 2 + +/* The size of `wchar_t', as computed by sizeof. */ +#define SIZEOF_WCHAR_T 4 + +/* define to support per-object ACIs */ +/* #undef SLAPD_ACI_ENABLED */ + +/* define to support LDAP Async Metadirectory backend */ +/* #undef SLAPD_ASYNCMETA */ + +/* define to support cleartext passwords */ +/* #undef SLAPD_CLEARTEXT */ + +/* define to support crypt(3) passwords */ +/* #undef SLAPD_CRYPT */ + +/* define to support DNS SRV backend */ +/* #undef SLAPD_DNSSRV */ + +/* define to support LDAP backend */ +/* #undef SLAPD_LDAP */ + +/* define to support MDB backend */ +/* #undef SLAPD_MDB */ + +/* define to support LDAP Metadirectory backend */ +/* #undef SLAPD_META */ + +/* define to support modules */ +/* #undef SLAPD_MODULES */ + +/* dynamically linked module */ +#define SLAPD_MOD_DYNAMIC 2 + +/* statically linked module */ +#define SLAPD_MOD_STATIC 1 + +/* define to support cn=Monitor backend */ +/* #undef SLAPD_MONITOR */ + +/* define to support NDB backend */ +/* #undef SLAPD_NDB */ + +/* define to support NULL backend */ +/* #undef SLAPD_NULL */ + +/* define for In-Directory Access Logging overlay */ +/* #undef SLAPD_OVER_ACCESSLOG */ + +/* define for Audit Logging overlay */ +/* #undef SLAPD_OVER_AUDITLOG */ + +/* define for Automatic Certificate Authority overlay */ +/* #undef SLAPD_OVER_AUTOCA */ + +/* define for Collect overlay */ +/* #undef SLAPD_OVER_COLLECT */ + +/* define for Attribute Constraint overlay */ +/* #undef SLAPD_OVER_CONSTRAINT */ + +/* define for Dynamic Directory Services overlay */ +/* #undef SLAPD_OVER_DDS */ + +/* define for Dynamic Directory Services overlay */ +/* #undef SLAPD_OVER_DEREF */ + +/* define for Dynamic Group overlay */ +/* #undef SLAPD_OVER_DYNGROUP */ + +/* define for Dynamic List overlay */ +/* #undef SLAPD_OVER_DYNLIST */ + +/* define for Reverse Group Membership overlay */ +/* #undef SLAPD_OVER_MEMBEROF */ + +/* define for Password Policy overlay */ +/* #undef SLAPD_OVER_PPOLICY */ + +/* define for Proxy Cache overlay */ +/* #undef SLAPD_OVER_PROXYCACHE */ + +/* define for Referential Integrity overlay */ +/* #undef SLAPD_OVER_REFINT */ + +/* define for Return Code overlay */ +/* #undef SLAPD_OVER_RETCODE */ + +/* define for Rewrite/Remap overlay */ +/* #undef SLAPD_OVER_RWM */ + +/* define for Sequential Modify overlay */ +/* #undef SLAPD_OVER_SEQMOD */ + +/* define for ServerSideSort/VLV overlay */ +/* #undef SLAPD_OVER_SSSVLV */ + +/* define for Syncrepl Provider overlay */ +/* #undef SLAPD_OVER_SYNCPROV */ + +/* define for Translucent Proxy overlay */ +/* #undef SLAPD_OVER_TRANSLUCENT */ + +/* define for Attribute Uniqueness overlay */ +/* #undef SLAPD_OVER_UNIQUE */ + +/* define for Value Sorting overlay */ +/* #undef SLAPD_OVER_VALSORT */ + +/* define to support PASSWD backend */ +/* #undef SLAPD_PASSWD */ + +/* define to support PERL backend */ +/* #undef SLAPD_PERL */ + +/* define to support relay backend */ +/* #undef SLAPD_RELAY */ + +/* define to support reverse lookups */ +/* #undef SLAPD_RLOOKUPS */ + +/* define to support SHELL backend */ +/* #undef SLAPD_SHELL */ + +/* define to support SOCK backend */ +/* #undef SLAPD_SOCK */ + +/* define to support SASL passwords */ +/* #undef SLAPD_SPASSWD */ + +/* define to support SQL backend */ +/* #undef SLAPD_SQL */ + +/* define to support WiredTiger backend */ +/* #undef SLAPD_WT */ + +/* define to support run-time loadable ACL */ +/* #undef SLAP_DYNACL */ + +/* Define to 1 if you have the ANSI C header files. */ +#define STDC_HEADERS 1 + +/* Define to 1 if you can safely include both and . */ +#define TIME_WITH_SYS_TIME 1 + +/* Define to 1 if your declares `struct tm'. */ +/* #undef TM_IN_SYS_TIME */ + +/* set to urandom device */ +#define URANDOM_DEVICE "/dev/urandom" + +/* define to use OpenSSL BIGNUM for MP */ +/* #undef USE_MP_BIGNUM */ + +/* define to use GMP for MP */ +/* #undef USE_MP_GMP */ + +/* define to use 'long' for MP */ +/* #undef USE_MP_LONG */ + +/* define to use 'long long' for MP */ +/* #undef USE_MP_LONG_LONG */ + +/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most + significant byte first (like Motorola and SPARC, unlike Intel). */ +#if defined AC_APPLE_UNIVERSAL_BUILD +# if defined __BIG_ENDIAN__ +# define WORDS_BIGENDIAN 1 +# endif +#else +# ifndef WORDS_BIGENDIAN +/* # undef WORDS_BIGENDIAN */ +# endif +#endif + +/* Define to the type of arg 3 for `accept'. */ +#define ber_socklen_t socklen_t + +/* Define to `char *' if does not define. */ +/* #undef caddr_t */ + +/* Define to empty if `const' does not conform to ANSI C. */ +/* #undef const */ + +/* Define to `int' if doesn't define. */ +/* #undef gid_t */ + +/* Define to `int' if does not define. */ +/* #undef mode_t */ + +/* Define to `long' if does not define. */ +/* #undef off_t */ + +/* Define to `int' if does not define. */ +/* #undef pid_t */ + +/* Define to `int' if does not define. */ +/* #undef sig_atomic_t */ + +/* Define to `unsigned' if does not define. */ +/* #undef size_t */ + +/* define to snprintf routine */ +/* #undef snprintf */ + +/* Define like ber_socklen_t if does not define. */ +/* #undef socklen_t */ + +/* Define to `signed int' if does not define. */ +/* #undef ssize_t */ + +/* Define to `int' if doesn't define. */ +/* #undef uid_t */ + +/* define as empty if volatile is not supported */ +/* #undef volatile */ + +/* define to snprintf routine */ +/* #undef vsnprintf */ + + +/* begin of portable.h.post */ + +#ifdef _WIN32 +/* don't suck in all of the win32 api */ +# define WIN32_LEAN_AND_MEAN 1 +#endif + +#ifndef LDAP_NEEDS_PROTOTYPES +/* force LDAP_P to always include prototypes */ +#define LDAP_NEEDS_PROTOTYPES 1 +#endif + +#ifndef LDAP_REL_ENG +#if (LDAP_VENDOR_VERSION == 000000) && !defined(LDAP_DEVEL) +#define LDAP_DEVEL +#endif +#if defined(LDAP_DEVEL) && !defined(LDAP_TEST) +#define LDAP_TEST +#endif +#endif + +#ifdef HAVE_STDDEF_H +# include +#endif + +#ifdef HAVE_EBCDIC +/* ASCII/EBCDIC converting replacements for stdio funcs + * vsnprintf and snprintf are used too, but they are already + * checked by the configure script + */ +#define fputs ber_pvt_fputs +#define fgets ber_pvt_fgets +#define printf ber_pvt_printf +#define fprintf ber_pvt_fprintf +#define vfprintf ber_pvt_vfprintf +#define vsprintf ber_pvt_vsprintf +#endif + +#include "ac/fdset.h" + +#include "ldap_cdefs.h" +#include "ldap_features.h" + +#include "ac/assert.h" +#include "ac/localize.h" + +#endif /* _LDAP_PORTABLE_H */ +/* end of portable.h.post */ + diff --git a/contrib/openldap-cmake/Linux_x86_64/include/lber_types.h b/contrib/openldap-cmake/Linux_x86_64/include/lber_types.h index 0e04ad96034..dbd59430527 100644 --- a/contrib/openldap-cmake/Linux_x86_64/include/lber_types.h +++ b/contrib/openldap-cmake/Linux_x86_64/include/lber_types.h @@ -1,4 +1,4 @@ -/* include/lber_types.h. Generated from lber_types.hin by configure. */ +/* include/lber_types.h. Generated from lber_types.hin by configure. */ /* $OpenLDAP$ */ /* This work is part of OpenLDAP Software . * diff --git a/contrib/openldap-cmake/Linux_x86_64/include/ldap_config.h b/contrib/openldap-cmake/Linux_x86_64/include/ldap_config.h index f63e813c052..89f7b40b884 100644 --- a/contrib/openldap-cmake/Linux_x86_64/include/ldap_config.h +++ b/contrib/openldap-cmake/Linux_x86_64/include/ldap_config.h @@ -1,4 +1,4 @@ -/* Generated from /home/denis/dev/altinity/ClickHouse/contrib/openldap/include/ldap_config.hin on Sat May 9 03:29:40 +04 2020 */ +/* include/ldap_config.h. Generated from ldap_config.hin by configure. */ /* $OpenLDAP$ */ /* This work is part of OpenLDAP Software . * @@ -33,7 +33,7 @@ /* directory for temporary files */ #if defined(_WIN32) -# define LDAP_TMPDIR "C:\\." /* we don't have much of a choice */ +# define LDAP_TMPDIR "C:\\." /* we don't have much of a choice */ #elif defined( _P_tmpdir ) # define LDAP_TMPDIR _P_tmpdir #elif defined( P_tmpdir ) @@ -46,28 +46,28 @@ /* directories */ #ifndef LDAP_BINDIR -#define LDAP_BINDIR "/tmp/ldap-prefix/bin" +#define LDAP_BINDIR "/tmp/ldap-prefix/bin" #endif #ifndef LDAP_SBINDIR -#define LDAP_SBINDIR "/tmp/ldap-prefix/sbin" +#define LDAP_SBINDIR "/tmp/ldap-prefix/sbin" #endif #ifndef LDAP_DATADIR -#define LDAP_DATADIR "/tmp/ldap-prefix/share/openldap" +#define LDAP_DATADIR "/tmp/ldap-prefix/share/openldap" #endif #ifndef LDAP_SYSCONFDIR -#define LDAP_SYSCONFDIR "/tmp/ldap-prefix/etc/openldap" +#define LDAP_SYSCONFDIR "/tmp/ldap-prefix/etc/openldap" #endif #ifndef LDAP_LIBEXECDIR -#define LDAP_LIBEXECDIR "/tmp/ldap-prefix/libexec" +#define LDAP_LIBEXECDIR "/tmp/ldap-prefix/libexec" #endif #ifndef LDAP_MODULEDIR -#define LDAP_MODULEDIR "/tmp/ldap-prefix/libexec/openldap" +#define LDAP_MODULEDIR "/tmp/ldap-prefix/libexec/openldap" #endif #ifndef LDAP_RUNDIR -#define LDAP_RUNDIR "/tmp/ldap-prefix/var" +#define LDAP_RUNDIR "/tmp/ldap-prefix/var" #endif #ifndef LDAP_LOCALEDIR -#define LDAP_LOCALEDIR "" +#define LDAP_LOCALEDIR "" #endif diff --git a/contrib/openldap-cmake/Linux_x86_64/include/ldap_features.h b/contrib/openldap-cmake/Linux_x86_64/include/ldap_features.h index 7ec8de4e9b2..f0cc7c3626f 100644 --- a/contrib/openldap-cmake/Linux_x86_64/include/ldap_features.h +++ b/contrib/openldap-cmake/Linux_x86_64/include/ldap_features.h @@ -1,4 +1,4 @@ -/* include/ldap_features.h. Generated from ldap_features.hin by configure. */ +/* include/ldap_features.h. Generated from ldap_features.hin by configure. */ /* $OpenLDAP$ */ /* This work is part of OpenLDAP Software . * @@ -14,7 +14,7 @@ * . */ -/* +/* * LDAP Features */ @@ -36,10 +36,10 @@ ** The -lldap implementation is not thread-safe. ** ** The -lldap_r implementation is: -** LDAP_API_FEATURE_THREAD_SAFE (basic thread safety) +** LDAP_API_FEATURE_THREAD_SAFE (basic thread safety) ** but also be: -** LDAP_API_FEATURE_SESSION_THREAD_SAFE -** LDAP_API_FEATURE_OPERATION_THREAD_SAFE +** LDAP_API_FEATURE_SESSION_THREAD_SAFE +** LDAP_API_FEATURE_OPERATION_THREAD_SAFE ** ** The preprocessor flag LDAP_API_FEATURE_X_OPENLDAP_THREAD_SAFE ** can be used to determine if -lldap_r is available at compile diff --git a/contrib/openldap-cmake/Linux_x86_64/include/portable.h b/contrib/openldap-cmake/Linux_x86_64/include/portable.h index ce6cb8900e8..2924b6713a4 100644 --- a/contrib/openldap-cmake/Linux_x86_64/include/portable.h +++ b/contrib/openldap-cmake/Linux_x86_64/include/portable.h @@ -1,5 +1,5 @@ -/* include/portable.h. Generated from portable.hin by configure. */ -/* include/portable.hin. Generated from configure.in by autoheader. */ +/* include/portable.h. Generated from portable.hin by configure. */ +/* include/portable.hin. Generated from configure.in by autoheader. */ /* begin of portable.h.pre */ @@ -113,7 +113,7 @@ /* #undef HAVE_CRYPT */ /* Define to 1 if you have the header file. */ -#define HAVE_CRYPT_H 1 +/* #undef HAVE_CRYPT_H */ /* define if crypt_r() is also available */ /* #undef HAVE_CRYPT_R */ @@ -369,13 +369,13 @@ #define HAVE_OPENSSL 1 /* Define to 1 if you have the header file. */ -/* #undef HAVE_OPENSSL_BN_H */ +#define HAVE_OPENSSL_BN_H 1 /* define if you have OpenSSL with CRL checking capability */ #define HAVE_OPENSSL_CRL 1 /* Define to 1 if you have the header file. */ -/* #undef HAVE_OPENSSL_CRYPTO_H */ +#define HAVE_OPENSSL_CRYPTO_H 1 /* Define to 1 if you have the header file. */ #define HAVE_OPENSSL_SSL_H 1 @@ -444,10 +444,10 @@ #define HAVE_REGEX_H 1 /* Define to 1 if you have the header file. */ -#define HAVE_RESOLV_H 1 +/* #undef HAVE_RESOLV_H */ /* define if you have res_query() */ -#define HAVE_RES_QUERY 1 +/* #undef HAVE_RES_QUERY */ /* define if OpenSSL needs RSAref */ /* #undef HAVE_RSAREF */ @@ -498,7 +498,7 @@ #define HAVE_SGTTY_H 1 /* Define to 1 if you have the header file. */ -#define HAVE_SHADOW_H 1 +/* #undef HAVE_SHADOW_H */ /* Define to 1 if you have the `sigaction' function. */ #define HAVE_SIGACTION 1 @@ -1121,8 +1121,8 @@ /* begin of portable.h.post */ #ifdef _WIN32 - /* don't suck in all of the win32 api */ -# define WIN32_LEAN_AND_MEAN 1 +/* don't suck in all of the win32 api */ +# define WIN32_LEAN_AND_MEAN 1 #endif #ifndef LDAP_NEEDS_PROTOTYPES @@ -1140,10 +1140,10 @@ #endif #ifdef HAVE_STDDEF_H -# include +# include #endif -#ifdef HAVE_EBCDIC +#ifdef HAVE_EBCDIC /* ASCII/EBCDIC converting replacements for stdio funcs * vsnprintf and snprintf are used too, but they are already * checked by the configure script From 21ce8dbbc33bf9afbb6bcd09d6385578d41fdb46 Mon Sep 17 00:00:00 2001 From: Ivan <5627721+abyss7@users.noreply.github.com> Date: Thu, 14 May 2020 22:56:10 +0300 Subject: [PATCH 230/738] Fix CH-over-YDB build (#10921) --- base/common/ya.make | 1 + src/Formats/ya.make | 2 +- src/Processors/ya.make | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/base/common/ya.make b/base/common/ya.make index 0f771cb9103..fbe63baafdf 100644 --- a/base/common/ya.make +++ b/base/common/ya.make @@ -9,6 +9,7 @@ CFLAGS (GLOBAL -DARCADIA_BUILD) CFLAGS (GLOBAL -DUSE_CPUID=1) CFLAGS (GLOBAL -DUSE_JEMALLOC=0) +CFLAGS (GLOBAL -DUSE_RAPIDJSON=1) IF (OS_DARWIN) CFLAGS (GLOBAL -DOS_DARWIN) diff --git a/src/Formats/ya.make b/src/Formats/ya.make index b1b46b46ddb..a4840a609ee 100644 --- a/src/Formats/ya.make +++ b/src/Formats/ya.make @@ -2,7 +2,7 @@ LIBRARY() PEERDIR( clickhouse/src/Common - contrib/libs/protobuf_std + contrib/libs/protobuf ) SRCS( diff --git a/src/Processors/ya.make b/src/Processors/ya.make index b9413bd6bb1..7bcc25edcb8 100644 --- a/src/Processors/ya.make +++ b/src/Processors/ya.make @@ -3,7 +3,7 @@ LIBRARY() PEERDIR( clickhouse/src/Common contrib/libs/msgpack - contrib/libs/protobuf_std + contrib/libs/protobuf ) SRCS( From 38320e6444af83744a499c398f62a2fbc499c343 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Fri, 15 May 2020 00:25:27 +0400 Subject: [PATCH 231/738] Normalize CMAKE_SYSTEM_PROCESSOR and platform dependent folder names --- cmake/find/ldap.cmake | 14 +++++++++++--- contrib/openldap-cmake/CMakeLists.txt | 10 +++++++++- .../include/lber_types.h | 0 .../include/ldap_config.h | 0 .../include/ldap_features.h | 0 .../include/portable.h | 0 6 files changed, 20 insertions(+), 4 deletions(-) rename contrib/openldap-cmake/{FreeBSD_amd64 => FreeBSD_x86_64}/include/lber_types.h (100%) rename contrib/openldap-cmake/{FreeBSD_amd64 => FreeBSD_x86_64}/include/ldap_config.h (100%) rename contrib/openldap-cmake/{FreeBSD_amd64 => FreeBSD_x86_64}/include/ldap_features.h (100%) rename contrib/openldap-cmake/{FreeBSD_amd64 => FreeBSD_x86_64}/include/portable.h (100%) diff --git a/cmake/find/ldap.cmake b/cmake/find/ldap.cmake index cbd8b50d66d..07ceae06113 100644 --- a/cmake/find/ldap.cmake +++ b/cmake/find/ldap.cmake @@ -24,10 +24,18 @@ if (ENABLE_LDAP) endif () if (NOT OPENLDAP_FOUND AND NOT MISSING_INTERNAL_LDAP_LIBRARY) + string (TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" _system_processor) if ( - ( "${CMAKE_SYSTEM_NAME}" STREQUAL "Linux" AND "${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "x86_64" ) OR - ( "${CMAKE_SYSTEM_NAME}" STREQUAL "FreeBSD" AND "${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "amd64" ) OR - ( "${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin" AND "${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "x86_64" ) + "${_system_processor}" STREQUAL "amd64" OR + "${_system_processor}" STREQUAL "x64" + ) + set (_system_processor "x86_64") + endif () + + if ( + ( "${CMAKE_SYSTEM_NAME}" STREQUAL "Linux" AND "${_system_processor}" STREQUAL "x86_64" ) OR + ( "${CMAKE_SYSTEM_NAME}" STREQUAL "FreeBSD" AND "${_system_processor}" STREQUAL "x86_64" ) OR + ( "${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin" AND "${_system_processor}" STREQUAL "x86_64" ) ) set (_ldap_supported_platform TRUE) endif () diff --git a/contrib/openldap-cmake/CMakeLists.txt b/contrib/openldap-cmake/CMakeLists.txt index 4b9926ad439..3f313ef6835 100644 --- a/contrib/openldap-cmake/CMakeLists.txt +++ b/contrib/openldap-cmake/CMakeLists.txt @@ -20,7 +20,15 @@ macro(mkversion _lib_name) ) endmacro() -set(_extra_build_dir "${CMAKE_CURRENT_SOURCE_DIR}/${CMAKE_SYSTEM_NAME}_${CMAKE_SYSTEM_PROCESSOR}") +string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" _system_processor) +if( + "${_system_processor}" STREQUAL "amd64" OR + "${_system_processor}" STREQUAL "x64" +) + set(_system_processor "x86_64") +endif() + +set(_extra_build_dir "${CMAKE_CURRENT_SOURCE_DIR}/${CMAKE_SYSTEM_NAME}_${_system_processor}") set(_lber_srcs ${OPENLDAP_SOURCE_DIR}/libraries/liblber/assert.c diff --git a/contrib/openldap-cmake/FreeBSD_amd64/include/lber_types.h b/contrib/openldap-cmake/FreeBSD_x86_64/include/lber_types.h similarity index 100% rename from contrib/openldap-cmake/FreeBSD_amd64/include/lber_types.h rename to contrib/openldap-cmake/FreeBSD_x86_64/include/lber_types.h diff --git a/contrib/openldap-cmake/FreeBSD_amd64/include/ldap_config.h b/contrib/openldap-cmake/FreeBSD_x86_64/include/ldap_config.h similarity index 100% rename from contrib/openldap-cmake/FreeBSD_amd64/include/ldap_config.h rename to contrib/openldap-cmake/FreeBSD_x86_64/include/ldap_config.h diff --git a/contrib/openldap-cmake/FreeBSD_amd64/include/ldap_features.h b/contrib/openldap-cmake/FreeBSD_x86_64/include/ldap_features.h similarity index 100% rename from contrib/openldap-cmake/FreeBSD_amd64/include/ldap_features.h rename to contrib/openldap-cmake/FreeBSD_x86_64/include/ldap_features.h diff --git a/contrib/openldap-cmake/FreeBSD_amd64/include/portable.h b/contrib/openldap-cmake/FreeBSD_x86_64/include/portable.h similarity index 100% rename from contrib/openldap-cmake/FreeBSD_amd64/include/portable.h rename to contrib/openldap-cmake/FreeBSD_x86_64/include/portable.h From b68664d33255f82bf9f9b48ed8012a0a3d953ecf Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Fri, 15 May 2020 00:40:27 +0400 Subject: [PATCH 232/738] Normalize CMAKE_SYSTEM_NAME and platform dependent folder names --- cmake/find/ldap.cmake | 8 +++++--- contrib/openldap-cmake/CMakeLists.txt | 4 +++- .../{Darwin_x86_64 => darwin_x86_64}/include/lber_types.h | 0 .../include/ldap_config.h | 0 .../include/ldap_features.h | 0 .../{Darwin_x86_64 => darwin_x86_64}/include/portable.h | 0 .../include/lber_types.h | 0 .../include/ldap_config.h | 0 .../include/ldap_features.h | 0 .../{FreeBSD_x86_64 => freebsd_x86_64}/include/portable.h | 0 .../{Linux_x86_64 => linux_x86_64}/include/lber_types.h | 0 .../{Linux_x86_64 => linux_x86_64}/include/ldap_config.h | 0 .../include/ldap_features.h | 0 .../{Linux_x86_64 => linux_x86_64}/include/portable.h | 0 14 files changed, 8 insertions(+), 4 deletions(-) rename contrib/openldap-cmake/{Darwin_x86_64 => darwin_x86_64}/include/lber_types.h (100%) rename contrib/openldap-cmake/{Darwin_x86_64 => darwin_x86_64}/include/ldap_config.h (100%) rename contrib/openldap-cmake/{Darwin_x86_64 => darwin_x86_64}/include/ldap_features.h (100%) rename contrib/openldap-cmake/{Darwin_x86_64 => darwin_x86_64}/include/portable.h (100%) rename contrib/openldap-cmake/{FreeBSD_x86_64 => freebsd_x86_64}/include/lber_types.h (100%) rename contrib/openldap-cmake/{FreeBSD_x86_64 => freebsd_x86_64}/include/ldap_config.h (100%) rename contrib/openldap-cmake/{FreeBSD_x86_64 => freebsd_x86_64}/include/ldap_features.h (100%) rename contrib/openldap-cmake/{FreeBSD_x86_64 => freebsd_x86_64}/include/portable.h (100%) rename contrib/openldap-cmake/{Linux_x86_64 => linux_x86_64}/include/lber_types.h (100%) rename contrib/openldap-cmake/{Linux_x86_64 => linux_x86_64}/include/ldap_config.h (100%) rename contrib/openldap-cmake/{Linux_x86_64 => linux_x86_64}/include/ldap_features.h (100%) rename contrib/openldap-cmake/{Linux_x86_64 => linux_x86_64}/include/portable.h (100%) diff --git a/cmake/find/ldap.cmake b/cmake/find/ldap.cmake index 07ceae06113..204a785b43e 100644 --- a/cmake/find/ldap.cmake +++ b/cmake/find/ldap.cmake @@ -24,7 +24,9 @@ if (ENABLE_LDAP) endif () if (NOT OPENLDAP_FOUND AND NOT MISSING_INTERNAL_LDAP_LIBRARY) + string (TOLOWER "${CMAKE_SYSTEM_NAME}" _system_name) string (TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" _system_processor) + if ( "${_system_processor}" STREQUAL "amd64" OR "${_system_processor}" STREQUAL "x64" @@ -33,9 +35,9 @@ if (ENABLE_LDAP) endif () if ( - ( "${CMAKE_SYSTEM_NAME}" STREQUAL "Linux" AND "${_system_processor}" STREQUAL "x86_64" ) OR - ( "${CMAKE_SYSTEM_NAME}" STREQUAL "FreeBSD" AND "${_system_processor}" STREQUAL "x86_64" ) OR - ( "${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin" AND "${_system_processor}" STREQUAL "x86_64" ) + ( "${_system_name}" STREQUAL "linux" AND "${_system_processor}" STREQUAL "x86_64" ) OR + ( "${_system_name}" STREQUAL "freebsd" AND "${_system_processor}" STREQUAL "x86_64" ) OR + ( "${_system_name}" STREQUAL "darwin" AND "${_system_processor}" STREQUAL "x86_64" ) ) set (_ldap_supported_platform TRUE) endif () diff --git a/contrib/openldap-cmake/CMakeLists.txt b/contrib/openldap-cmake/CMakeLists.txt index 3f313ef6835..c69cb1009ff 100644 --- a/contrib/openldap-cmake/CMakeLists.txt +++ b/contrib/openldap-cmake/CMakeLists.txt @@ -20,7 +20,9 @@ macro(mkversion _lib_name) ) endmacro() +string(TOLOWER "${CMAKE_SYSTEM_NAME}" _system_name) string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" _system_processor) + if( "${_system_processor}" STREQUAL "amd64" OR "${_system_processor}" STREQUAL "x64" @@ -28,7 +30,7 @@ if( set(_system_processor "x86_64") endif() -set(_extra_build_dir "${CMAKE_CURRENT_SOURCE_DIR}/${CMAKE_SYSTEM_NAME}_${_system_processor}") +set(_extra_build_dir "${CMAKE_CURRENT_SOURCE_DIR}/${_system_name}_${_system_processor}") set(_lber_srcs ${OPENLDAP_SOURCE_DIR}/libraries/liblber/assert.c diff --git a/contrib/openldap-cmake/Darwin_x86_64/include/lber_types.h b/contrib/openldap-cmake/darwin_x86_64/include/lber_types.h similarity index 100% rename from contrib/openldap-cmake/Darwin_x86_64/include/lber_types.h rename to contrib/openldap-cmake/darwin_x86_64/include/lber_types.h diff --git a/contrib/openldap-cmake/Darwin_x86_64/include/ldap_config.h b/contrib/openldap-cmake/darwin_x86_64/include/ldap_config.h similarity index 100% rename from contrib/openldap-cmake/Darwin_x86_64/include/ldap_config.h rename to contrib/openldap-cmake/darwin_x86_64/include/ldap_config.h diff --git a/contrib/openldap-cmake/Darwin_x86_64/include/ldap_features.h b/contrib/openldap-cmake/darwin_x86_64/include/ldap_features.h similarity index 100% rename from contrib/openldap-cmake/Darwin_x86_64/include/ldap_features.h rename to contrib/openldap-cmake/darwin_x86_64/include/ldap_features.h diff --git a/contrib/openldap-cmake/Darwin_x86_64/include/portable.h b/contrib/openldap-cmake/darwin_x86_64/include/portable.h similarity index 100% rename from contrib/openldap-cmake/Darwin_x86_64/include/portable.h rename to contrib/openldap-cmake/darwin_x86_64/include/portable.h diff --git a/contrib/openldap-cmake/FreeBSD_x86_64/include/lber_types.h b/contrib/openldap-cmake/freebsd_x86_64/include/lber_types.h similarity index 100% rename from contrib/openldap-cmake/FreeBSD_x86_64/include/lber_types.h rename to contrib/openldap-cmake/freebsd_x86_64/include/lber_types.h diff --git a/contrib/openldap-cmake/FreeBSD_x86_64/include/ldap_config.h b/contrib/openldap-cmake/freebsd_x86_64/include/ldap_config.h similarity index 100% rename from contrib/openldap-cmake/FreeBSD_x86_64/include/ldap_config.h rename to contrib/openldap-cmake/freebsd_x86_64/include/ldap_config.h diff --git a/contrib/openldap-cmake/FreeBSD_x86_64/include/ldap_features.h b/contrib/openldap-cmake/freebsd_x86_64/include/ldap_features.h similarity index 100% rename from contrib/openldap-cmake/FreeBSD_x86_64/include/ldap_features.h rename to contrib/openldap-cmake/freebsd_x86_64/include/ldap_features.h diff --git a/contrib/openldap-cmake/FreeBSD_x86_64/include/portable.h b/contrib/openldap-cmake/freebsd_x86_64/include/portable.h similarity index 100% rename from contrib/openldap-cmake/FreeBSD_x86_64/include/portable.h rename to contrib/openldap-cmake/freebsd_x86_64/include/portable.h diff --git a/contrib/openldap-cmake/Linux_x86_64/include/lber_types.h b/contrib/openldap-cmake/linux_x86_64/include/lber_types.h similarity index 100% rename from contrib/openldap-cmake/Linux_x86_64/include/lber_types.h rename to contrib/openldap-cmake/linux_x86_64/include/lber_types.h diff --git a/contrib/openldap-cmake/Linux_x86_64/include/ldap_config.h b/contrib/openldap-cmake/linux_x86_64/include/ldap_config.h similarity index 100% rename from contrib/openldap-cmake/Linux_x86_64/include/ldap_config.h rename to contrib/openldap-cmake/linux_x86_64/include/ldap_config.h diff --git a/contrib/openldap-cmake/Linux_x86_64/include/ldap_features.h b/contrib/openldap-cmake/linux_x86_64/include/ldap_features.h similarity index 100% rename from contrib/openldap-cmake/Linux_x86_64/include/ldap_features.h rename to contrib/openldap-cmake/linux_x86_64/include/ldap_features.h diff --git a/contrib/openldap-cmake/Linux_x86_64/include/portable.h b/contrib/openldap-cmake/linux_x86_64/include/portable.h similarity index 100% rename from contrib/openldap-cmake/Linux_x86_64/include/portable.h rename to contrib/openldap-cmake/linux_x86_64/include/portable.h From 4c0cbc78a992d85ee4f7dd6041143284bd1f13f8 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Thu, 14 May 2020 23:44:52 +0300 Subject: [PATCH 233/738] trigger CI --- .../configs/remote_servers.xml | 2 +- .../integration/test_distributed_backward_compatability/test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_distributed_backward_compatability/configs/remote_servers.xml b/tests/integration/test_distributed_backward_compatability/configs/remote_servers.xml index 998a969e87e..ebce4697529 100644 --- a/tests/integration/test_distributed_backward_compatability/configs/remote_servers.xml +++ b/tests/integration/test_distributed_backward_compatability/configs/remote_servers.xml @@ -15,4 +15,4 @@ - \ No newline at end of file + diff --git a/tests/integration/test_distributed_backward_compatability/test.py b/tests/integration/test_distributed_backward_compatability/test.py index 2b050f379a2..a0362cea49e 100644 --- a/tests/integration/test_distributed_backward_compatability/test.py +++ b/tests/integration/test_distributed_backward_compatability/test.py @@ -44,4 +44,4 @@ def test_distributed_in_tuple(started_cluster): assert node_old.query(query3) == "2\n" assert node_old.query(query4) == "2\n" assert node_new.query(query3) == "2\n" - assert node_new.query(query4) == "2\n" \ No newline at end of file + assert node_new.query(query4) == "2\n" From 681955661fcef4d9977c6a747915973e1080157b Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 15 May 2020 00:03:38 +0300 Subject: [PATCH 234/738] Add PullingPipelineExecutor. --- programs/server/TCPHandler.cpp | 71 ++----- src/DataStreams/BlockStreamProfileInfo.cpp | 9 +- src/DataStreams/BlockStreamProfileInfo.h | 1 + .../Executors/ParallelPipelineExecutor.cpp | 105 ---------- .../Executors/ParallelPipelineExecutor.h | 41 ---- .../Executors/PullingPipelineExecutor.cpp | 186 ++++++++++++++++++ .../Executors/PullingPipelineExecutor.h | 53 +++++ .../Executors/SequentialPipelineExecutor.cpp | 83 -------- .../Executors/SequentialPipelineExecutor.h | 31 --- src/Processors/Formats/LazyOutputFormat.cpp | 34 +--- src/Processors/Formats/LazyOutputFormat.h | 6 +- src/Processors/ya.make | 3 +- 12 files changed, 270 insertions(+), 353 deletions(-) delete mode 100644 src/Processors/Executors/ParallelPipelineExecutor.cpp delete mode 100644 src/Processors/Executors/ParallelPipelineExecutor.h create mode 100644 src/Processors/Executors/PullingPipelineExecutor.cpp create mode 100644 src/Processors/Executors/PullingPipelineExecutor.h delete mode 100644 src/Processors/Executors/SequentialPipelineExecutor.cpp delete mode 100644 src/Processors/Executors/SequentialPipelineExecutor.h diff --git a/programs/server/TCPHandler.cpp b/programs/server/TCPHandler.cpp index f6903f0820f..93fa78499a1 100644 --- a/programs/server/TCPHandler.cpp +++ b/programs/server/TCPHandler.cpp @@ -28,7 +28,7 @@ #include #include -#include +#include #include "TCPHandler.h" @@ -553,68 +553,23 @@ void TCPHandler::processOrdinaryQueryWithProcessors() /// Send header-block, to allow client to prepare output format for data to send. { - auto & header = pipeline.getHeader(); + const auto & header = pipeline.getHeader(); if (header) sendData(header); } - auto lazy_format = std::make_shared(pipeline.getHeader()); - pipeline.setOutput(lazy_format); - { - auto thread_group = CurrentThread::getGroup(); - ThreadPool pool(1); - auto executor = pipeline.execute(); - std::atomic_bool exception = false; + PullingPipelineExecutor executor(pipeline); + CurrentMetrics::Increment query_thread_metric_increment{CurrentMetrics::QueryThread}; - pool.scheduleOrThrowOnError([&]() - { - /// ThreadStatus thread_status; - - if (thread_group) - CurrentThread::attachTo(thread_group); - - SCOPE_EXIT( - if (thread_group) - CurrentThread::detachQueryIfNotDetached(); - ); - - CurrentMetrics::Increment query_thread_metric_increment{CurrentMetrics::QueryThread}; - setThreadName("QueryPipelineEx"); - - try - { - executor->execute(pipeline.getNumThreads()); - } - catch (...) - { - exception = true; - throw; - } - }); - - /// Wait in case of exception happened outside of pool. - SCOPE_EXIT( - lazy_format->finish(); - - try - { - pool.wait(); - } - catch (...) - { - /// If exception was thrown during pipeline execution, skip it while processing other exception. - tryLogCurrentException(log); - } - ); - - while (!lazy_format->isFinished() && !exception) + Block block; + while (executor.pull(block, query_context->getSettingsRef().interactive_delay / 1000)) { if (isQueryCancelled()) { /// A packet was received requesting to stop execution of the request. - executor->cancel(); + executor.cancel(); break; } @@ -627,17 +582,13 @@ void TCPHandler::processOrdinaryQueryWithProcessors() sendLogs(); - if (auto block = lazy_format->getBlock(query_context->getSettingsRef().interactive_delay / 1000)) + if (block) { if (!state.io.null_format) sendData(block); } } - /// Finish lazy_format before waiting. Otherwise some thread may write into it, and waiting will lock. - lazy_format->finish(); - pool.wait(); - /** If data has run out, we will send the profiling data and total values to * the last zero block to be able to use * this information in the suffix output of stream. @@ -647,9 +598,9 @@ void TCPHandler::processOrdinaryQueryWithProcessors() */ if (!isQueryCancelled()) { - sendTotals(lazy_format->getTotals()); - sendExtremes(lazy_format->getExtremes()); - sendProfileInfo(lazy_format->getProfileInfo()); + sendTotals(executor.getTotalsBlock()); + sendExtremes(executor.getExtremesBlock()); + sendProfileInfo(executor.getProfileInfo()); sendProgress(); sendLogs(); } diff --git a/src/DataStreams/BlockStreamProfileInfo.cpp b/src/DataStreams/BlockStreamProfileInfo.cpp index 81e2be8d4a0..09ad8a8e4ac 100644 --- a/src/DataStreams/BlockStreamProfileInfo.cpp +++ b/src/DataStreams/BlockStreamProfileInfo.cpp @@ -62,10 +62,15 @@ bool BlockStreamProfileInfo::hasAppliedLimit() const void BlockStreamProfileInfo::update(Block & block) +{ + update(block.rows(), block.bytes()); +} + +void BlockStreamProfileInfo::update(size_t num_rows, size_t num_bytes) { ++blocks; - rows += block.rows(); - bytes += block.bytes(); + rows += num_rows; + bytes += num_bytes; } diff --git a/src/DataStreams/BlockStreamProfileInfo.h b/src/DataStreams/BlockStreamProfileInfo.h index 6b60fa95e24..5f75cf9ddea 100644 --- a/src/DataStreams/BlockStreamProfileInfo.h +++ b/src/DataStreams/BlockStreamProfileInfo.h @@ -40,6 +40,7 @@ struct BlockStreamProfileInfo bool hasAppliedLimit() const; void update(Block & block); + void update(size_t num_rows, size_t num_bytes); /// Binary serialization and deserialization of main fields. /// Writes only main fields i.e. fields that required by internal transmission protocol. diff --git a/src/Processors/Executors/ParallelPipelineExecutor.cpp b/src/Processors/Executors/ParallelPipelineExecutor.cpp deleted file mode 100644 index cea020300b6..00000000000 --- a/src/Processors/Executors/ParallelPipelineExecutor.cpp +++ /dev/null @@ -1,105 +0,0 @@ -#include -#include -#include -#include - - -namespace DB -{ -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; -} -// -//ParallelPipelineExecutor::ParallelPipelineExecutor(const std::vector & processors, ThreadPool & pool) -// : processors(processors), pool(pool) -//{ -//} -// -// -//ParallelPipelineExecutor::Status ParallelPipelineExecutor::prepare() -//{ -// current_processor = nullptr; -// -// bool has_someone_to_wait = false; -// -// for (auto & element : processors) -// { -// traverse(*element, -// [&] (IProcessor & processor) -// { -// { -// std::lock_guard lock(mutex); -// if (active_processors.count(&processor)) -// { -// has_someone_to_wait = true; -// return Status::Wait; -// } -// } -// -// Status status = processor.prepare(); -// -// if (status == Status::Wait) -// has_someone_to_wait = true; -// -// if (status == Status::Ready || status == Status::Async) -// { -// current_processor = &processor; -// current_status = status; -// } -// -// return status; -// }); -// -// if (current_processor) -// break; -// } -// -// if (current_processor) -// return Status::Async; -// -// if (has_someone_to_wait) -// return Status::Wait; -// -// for (auto & element : processors) -// { -// if (element->prepare() == Status::NeedData) -// throw Exception("Pipeline stuck: " + element->getName() + " processor needs input data but no one is going to generate it", ErrorCodes::LOGICAL_ERROR); -// if (element->prepare() == Status::PortFull) -// throw Exception("Pipeline stuck: " + element->getName() + " processor has data in output port but no one is going to consume it", ErrorCodes::LOGICAL_ERROR); -// } -// -// return Status::Finished; -//} -// -// -//void ParallelPipelineExecutor::schedule(EventCounter & watch) -//{ -// if (!current_processor) -// throw Exception("Bad pipeline", ErrorCodes::LOGICAL_ERROR); -// -// if (current_status == Status::Async) -// { -// current_processor->schedule(watch); -// } -// else -// { -// { -// std::lock_guard lock(mutex); -// active_processors.insert(current_processor); -// } -// -// pool.scheduleOrThrowOnError([processor = current_processor, &watch, this] -// { -// processor->work(); -// { -// std::lock_guard lock(mutex); -// active_processors.erase(processor); -// } -// watch.notify(); -// }); -// } -//} - -} - diff --git a/src/Processors/Executors/ParallelPipelineExecutor.h b/src/Processors/Executors/ParallelPipelineExecutor.h deleted file mode 100644 index 4997f73f699..00000000000 --- a/src/Processors/Executors/ParallelPipelineExecutor.h +++ /dev/null @@ -1,41 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -template -class ThreadPoolImpl; -class ThreadFromGlobalPool; -using ThreadPool = ThreadPoolImpl; - -namespace DB -{ - -/** Wraps pipeline in a single processor. - * This processor has no inputs and outputs and just executes the pipeline, - * performing all synchronous work within a threadpool. - */ -//class ParallelPipelineExecutor : public IProcessor -//{ -//private: -// Processors processors; -// ThreadPool & pool; -// -// std::set active_processors; -// std::mutex mutex; -// -// IProcessor * current_processor = nullptr; -// Status current_status; -// -//public: -// ParallelPipelineExecutor(const Processors & processors, ThreadPool & pool); -// -// String getName() const override { return "ParallelPipelineExecutor"; } -// -// Status prepare() override; -// void schedule(EventCounter & watch) override; -//}; - -} diff --git a/src/Processors/Executors/PullingPipelineExecutor.cpp b/src/Processors/Executors/PullingPipelineExecutor.cpp new file mode 100644 index 00000000000..f9b957bf66c --- /dev/null +++ b/src/Processors/Executors/PullingPipelineExecutor.cpp @@ -0,0 +1,186 @@ +#include +#include +#include +#include +#include + +#include +#include + +namespace DB +{ + +struct PullingPipelineExecutor::Data +{ + PipelineExecutorPtr executor; + std::exception_ptr exception; + std::atomic_bool is_executed = false; + std::atomic_bool has_exception = false; + ThreadFromGlobalPool thread; + + ~Data() + { + if (thread.joinable()) + thread.join(); + } +}; + +PullingPipelineExecutor::PullingPipelineExecutor(QueryPipeline & pipeline_) : pipeline(pipeline_) +{ + lazy_format = std::make_shared(pipeline.getHeader()); + pipeline.setOutput(lazy_format); +} + +PullingPipelineExecutor::~PullingPipelineExecutor() +{ + try + { + /// Cancel execution if it wasn't finished. + if (data && !data->is_executed) + cancel(); + + /// Finish lazy format. Otherwise thread.join() may hung. + if (!lazy_format->isFinished()) + lazy_format->finish(); + + /// Join thread here to wait for possible exception. + if (data && data->thread.joinable()) + data->thread.join(); + } + catch (...) + { + tryLogCurrentException("PullingPipelineExecutor"); + } + + if (data && data->has_exception) + tryLogException(std::move(data->exception), "PullingPipelineExecutor"); +} + +static void threadFunction(PullingPipelineExecutor::Data & data, ThreadGroupStatusPtr thread_group, size_t num_threads) +{ + if (thread_group) + CurrentThread::attachTo(thread_group); + + SCOPE_EXIT( + if (thread_group) + CurrentThread::detachQueryIfNotDetached(); + ); + + setThreadName("QueryPipelineEx"); + + try + { + data.executor->execute(num_threads); + } + catch (...) + { + data.exception = std::current_exception(); + data.has_exception = true; + } + + data.is_executed = true; +} + + +bool PullingPipelineExecutor::pull(Chunk & chunk, uint64_t milliseconds) +{ + if (!data) + { + data = std::make_unique(); + data->executor = pipeline.execute(); + + auto func = [&, thread_group = CurrentThread::getGroup()]() + { + threadFunction(*data, thread_group, pipeline.getNumThreads()); + }; + + data->thread = ThreadFromGlobalPool(std::move(func)); + } + + if (data->has_exception) + { + /// Finish lazy format in case of exception. Otherwise thread.join() may hung. + lazy_format->finish(); + data->has_exception = false; + std::rethrow_exception(std::move(data->exception)); + } + + if (lazy_format->isFinished()) + return false; + + chunk = lazy_format->getChunk(milliseconds); + return true; +} + +bool PullingPipelineExecutor::pull(Block & block, uint64_t milliseconds) +{ + Chunk chunk; + + if (!pull(chunk, milliseconds)) + return false; + + if (!chunk) + { + /// In case if timeout exceeded. + block.clear(); + return false; + } + + block = lazy_format->getPort(IOutputFormat::PortKind::Main).getHeader().cloneWithColumns(chunk.detachColumns()); + + if (auto chunk_info = chunk.getChunkInfo()) + { + if (const auto * agg_info = typeid_cast(chunk_info.get())) + { + block.info.bucket_num = agg_info->bucket_num; + block.info.is_overflows = agg_info->is_overflows; + } + } + + return true; +} + +void PullingPipelineExecutor::cancel() +{ + if (data && data->executor) + data->executor->cancel(); +} + +Chunk PullingPipelineExecutor::getTotals() +{ + return lazy_format->getTotals(); +} + +Chunk PullingPipelineExecutor::getExtremes() +{ + return lazy_format->getExtremes(); +} + +Block PullingPipelineExecutor::getTotalsBlock() +{ + auto totals = getTotals(); + + if (totals.empty()) + return {}; + + const auto & header = lazy_format->getPort(IOutputFormat::PortKind::Totals).getHeader(); + return header.cloneWithColumns(totals.detachColumns()); +} + +Block PullingPipelineExecutor::getExtremesBlock() +{ + auto extremes = getExtremes(); + + if (extremes.empty()) + return {}; + + const auto & header = lazy_format->getPort(IOutputFormat::PortKind::Extremes).getHeader(); + return header.cloneWithColumns(extremes.detachColumns()); +} + +BlockStreamProfileInfo & PullingPipelineExecutor::getProfileInfo() +{ + return lazy_format->getProfileInfo(); +} + +} diff --git a/src/Processors/Executors/PullingPipelineExecutor.h b/src/Processors/Executors/PullingPipelineExecutor.h new file mode 100644 index 00000000000..d378a4ee8fe --- /dev/null +++ b/src/Processors/Executors/PullingPipelineExecutor.h @@ -0,0 +1,53 @@ +#pragma once +#include + +namespace DB +{ + +class QueryPipeline; +class Block; +class Chunk; +class LazyOutputFormat; +class BlockStreamProfileInfo; + +/// Pulling executor for QueryPipeline. +/// Typical usage is: +/// +/// PullingPipelineExecutor executor(query_pipeline); +/// while (executor.pull(chunk)) +/// ... process chunk ... +class PullingPipelineExecutor +{ +public: + explicit PullingPipelineExecutor(QueryPipeline & pipeline_); + ~PullingPipelineExecutor(); + + /// Methods return false if query is finished. + /// If milliseconds > 0, returns empty object and `true` after timeout exceeded. + /// You can use any pull method. + bool pull(Chunk & chunk, uint64_t milliseconds = 0); + bool pull(Block & block, uint64_t milliseconds = 0); + + /// Stop execution. It is not necessary, but helps to stop execution before executor is destroyed. + void cancel(); + + /// Get totals and extremes. Returns empty chunk if doesn't have any. + Chunk getTotals(); + Chunk getExtremes(); + + /// Get totals and extremes. Returns empty chunk if doesn't have any. + Block getTotalsBlock(); + Block getExtremesBlock(); + + /// Get query profile info. + BlockStreamProfileInfo & getProfileInfo(); + + struct Data; + +private: + QueryPipeline & pipeline; + std::shared_ptr lazy_format; + std::unique_ptr data; +}; + +} diff --git a/src/Processors/Executors/SequentialPipelineExecutor.cpp b/src/Processors/Executors/SequentialPipelineExecutor.cpp deleted file mode 100644 index 9e3b398f372..00000000000 --- a/src/Processors/Executors/SequentialPipelineExecutor.cpp +++ /dev/null @@ -1,83 +0,0 @@ -#include -#include - - -namespace DB -{ -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; -} - -//SequentialPipelineExecutor::SequentialPipelineExecutor(const Processors & processors) -// : processors(processors) -//{ -//} -// -// -//SequentialPipelineExecutor::Status SequentialPipelineExecutor::prepare() -//{ -// current_processor = nullptr; -// -// bool has_someone_to_wait = false; -// Status found_status = Status::Finished; -// -// for (auto & element : processors) -// { -// traverse(*element, -// [&] (IProcessor & processor) -// { -// Status status = processor.prepare(); -// -// if (status == Status::Wait) -// has_someone_to_wait = true; -// -// if (status == Status::Ready || status == Status::Async) -// { -// current_processor = &processor; -// found_status = status; -// } -// -// return status; -// }); -// -// if (current_processor) -// break; -// } -// -// if (current_processor) -// return found_status; -// if (has_someone_to_wait) -// return Status::Wait; -// -// for (auto & element : processors) -// { -// if (element->prepare() == Status::NeedData) -// throw Exception("Pipeline stuck: " + element->getName() + " processor needs input data but no one is going to generate it", ErrorCodes::LOGICAL_ERROR); -// if (element->prepare() == Status::PortFull) -// throw Exception("Pipeline stuck: " + element->getName() + " processor has data in output port but no one is going to consume it", ErrorCodes::LOGICAL_ERROR); -// } -// -// return Status::Finished; -//} -// -// -//void SequentialPipelineExecutor::work() -//{ -// if (!current_processor) -// throw Exception("Bad pipeline", ErrorCodes::LOGICAL_ERROR); -// -// current_processor->work(); -//} -// -// -//void SequentialPipelineExecutor::schedule(EventCounter & watch) -//{ -// if (!current_processor) -// throw Exception("Bad pipeline", ErrorCodes::LOGICAL_ERROR); -// -// current_processor->schedule(watch); -//} - -} - diff --git a/src/Processors/Executors/SequentialPipelineExecutor.h b/src/Processors/Executors/SequentialPipelineExecutor.h deleted file mode 100644 index 5dbd8b73fee..00000000000 --- a/src/Processors/Executors/SequentialPipelineExecutor.h +++ /dev/null @@ -1,31 +0,0 @@ -#pragma once - -#include - -#include - - -namespace DB -{ - -/** Wraps pipeline in a single processor. - * This processor has no inputs and outputs and just executes the pipeline, - * performing all synchronous work from the current thread. - */ -//class SequentialPipelineExecutor : public IProcessor -//{ -//private: -// Processors processors; -// IProcessor * current_processor = nullptr; -// -//public: -// SequentialPipelineExecutor(const Processors & processors); -// -// String getName() const override { return "SequentialPipelineExecutor"; } -// -// Status prepare() override; -// void work() override; -// void schedule(EventCounter & watch) override; -//}; - -} diff --git a/src/Processors/Formats/LazyOutputFormat.cpp b/src/Processors/Formats/LazyOutputFormat.cpp index 920b34b3813..46287d1cce9 100644 --- a/src/Processors/Formats/LazyOutputFormat.cpp +++ b/src/Processors/Formats/LazyOutputFormat.cpp @@ -7,7 +7,7 @@ namespace DB WriteBuffer LazyOutputFormat::out(nullptr, 0); -Block LazyOutputFormat::getBlock(UInt64 milliseconds) +Chunk LazyOutputFormat::getChunk(UInt64 milliseconds) { if (finished_processing) { @@ -19,38 +19,20 @@ Block LazyOutputFormat::getBlock(UInt64 milliseconds) if (!queue.tryPop(chunk, milliseconds)) return {}; - if (!chunk) - return {}; + if (chunk) + info.update(chunk.getNumRows(), chunk.allocatedBytes()); - auto block = getPort(PortKind::Main).getHeader().cloneWithColumns(chunk.detachColumns()); - info.update(block); - - if (auto chunk_info = chunk.getChunkInfo()) - { - if (const auto * agg_info = typeid_cast(chunk_info.get())) - { - block.info.bucket_num = agg_info->bucket_num; - block.info.is_overflows = agg_info->is_overflows; - } - } - - return block; + return chunk; } -Block LazyOutputFormat::getTotals() +Chunk LazyOutputFormat::getTotals() { - if (!totals) - return {}; - - return getPort(PortKind::Totals).getHeader().cloneWithColumns(totals.detachColumns()); + return std::move(totals); } -Block LazyOutputFormat::getExtremes() +Chunk LazyOutputFormat::getExtremes() { - if (!extremes) - return {}; - - return getPort(PortKind::Extremes).getHeader().cloneWithColumns(extremes.detachColumns()); + return std::move(extremes); } void LazyOutputFormat::setRowsBeforeLimit(size_t rows_before_limit) diff --git a/src/Processors/Formats/LazyOutputFormat.h b/src/Processors/Formats/LazyOutputFormat.h index 59cdbc4e81e..24283391355 100644 --- a/src/Processors/Formats/LazyOutputFormat.h +++ b/src/Processors/Formats/LazyOutputFormat.h @@ -19,9 +19,9 @@ public: String getName() const override { return "LazyOutputFormat"; } - Block getBlock(UInt64 milliseconds = 0); - Block getTotals(); - Block getExtremes(); + Chunk getChunk(UInt64 milliseconds = 0); + Chunk getTotals(); + Chunk getExtremes(); bool isFinished() { return finished_processing && queue.size() == 0; } diff --git a/src/Processors/ya.make b/src/Processors/ya.make index b9413bd6bb1..4e6c2379eb7 100644 --- a/src/Processors/ya.make +++ b/src/Processors/ya.make @@ -10,9 +10,8 @@ SRCS( Chunk.cpp ConcatProcessor.cpp DelayedPortsProcessor.cpp - Executors/ParallelPipelineExecutor.cpp Executors/PipelineExecutor.cpp - Executors/SequentialPipelineExecutor.cpp + Executors/PullingPipelineExecutor.cpp Executors/TreeExecutorBlockInputStream.cpp ForkProcessor.cpp Formats/IInputFormat.cpp From 8a40c1a2217e4af8d219ce80db5488644358db7a Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 15 May 2020 00:51:07 +0300 Subject: [PATCH 235/738] minor fixes --- programs/odbc-bridge/ColumnInfoHandler.cpp | 20 +---- programs/odbc-bridge/MainHandler.cpp | 89 +++++++++---------- .../odbc-bridge/ODBCBlockOutputStream.cpp | 43 ++++++--- programs/odbc-bridge/ODBCBlockOutputStream.h | 4 +- programs/odbc-bridge/getIdentifierQuote.cpp | 18 ++++ programs/odbc-bridge/getIdentifierQuote.h | 4 + 6 files changed, 100 insertions(+), 78 deletions(-) diff --git a/programs/odbc-bridge/ColumnInfoHandler.cpp b/programs/odbc-bridge/ColumnInfoHandler.cpp index 0d103a96344..e62aad48b90 100644 --- a/programs/odbc-bridge/ColumnInfoHandler.cpp +++ b/programs/odbc-bridge/ColumnInfoHandler.cpp @@ -16,6 +16,7 @@ # include # include # include +# include # include # include "getIdentifierQuote.h" # include "validateODBCConnectionString.h" @@ -58,11 +59,6 @@ namespace } } -namespace ErrorCodes -{ - extern const int ILLEGAL_TYPE_OF_ARGUMENT; -} - void ODBCColumnsInfoHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) { Poco::Net::HTMLForm params(request, request.stream()); @@ -116,7 +112,7 @@ void ODBCColumnsInfoHandler::handleRequest(Poco::Net::HTTPServerRequest & reques const auto & context_settings = context.getSettingsRef(); /// TODO Why not do SQLColumns instead? - std::string name = schema_name.empty() ? table_name : schema_name + "." + table_name; + std::string name = schema_name.empty() ? backQuoteIfNeed(table_name) : backQuoteIfNeed(schema_name) + "." + backQuoteIfNeed(table_name); std::stringstream ss; std::string input = "SELECT * FROM " + name + " WHERE 1 = 0"; ParserQueryWithOutput parser; @@ -124,17 +120,7 @@ void ODBCColumnsInfoHandler::handleRequest(Poco::Net::HTTPServerRequest & reques IAST::FormatSettings settings(ss, true); settings.always_quote_identifiers = true; - - auto identifier_quote = getIdentifierQuote(hdbc); - if (identifier_quote.length() == 0) - settings.identifier_quoting_style = IdentifierQuotingStyle::None; - else if (identifier_quote[0] == '`') - settings.identifier_quoting_style = IdentifierQuotingStyle::Backticks; - else if (identifier_quote[0] == '"') - settings.identifier_quoting_style = IdentifierQuotingStyle::DoubleQuotes; - else - throw Exception("Can not map quote identifier '" + identifier_quote + "' to IdentifierQuotingStyle value", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - + settings.identifier_quoting_style = getQuotingStyle(hdbc); select->format(settings); std::string query = ss.str(); diff --git a/programs/odbc-bridge/MainHandler.cpp b/programs/odbc-bridge/MainHandler.cpp index 31bf3a2c426..3c16937e7b0 100644 --- a/programs/odbc-bridge/MainHandler.cpp +++ b/programs/odbc-bridge/MainHandler.cpp @@ -18,7 +18,10 @@ #include #include #include -#include +#include "getIdentifierQuote.h" + +#include +#define POCO_SQL_ODBC_CLASS Poco::Data::ODBC namespace DB { @@ -81,6 +84,12 @@ void ODBCHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne params.read(request.stream()); LOG_TRACE(log, "Request URI: " + request.getURI()); + if (mode == "read" && !params.has("query")) + { + processError(response, "No 'query' in request body"); + return; + } + if (!params.has("columns")) { processError(response, "No 'columns' in request URL"); @@ -93,6 +102,18 @@ void ODBCHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne return; } + UInt64 max_block_size = DEFAULT_BLOCK_SIZE; + if (params.has("max_block_size")) + { + std::string max_block_size_str = params.get("max_block_size", ""); + if (max_block_size_str.empty()) + { + processError(response, "Empty max_block_size specified"); + return; + } + max_block_size = parse(max_block_size_str); + } + std::string columns = params.get("columns"); std::unique_ptr sample_block; try @@ -111,22 +132,11 @@ void ODBCHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne std::string connection_string = params.get("connection_string"); LOG_TRACE(log, "Connection string: '" << connection_string << "'"); - UInt64 max_block_size = DEFAULT_BLOCK_SIZE; - if (params.has("max_block_size")) - { - std::string max_block_size_str = params.get("max_block_size", ""); - if (max_block_size_str.empty()) - { - processError(response, "Empty max_block_size specified"); - return; - } - max_block_size = parse(max_block_size_str); - } - WriteBufferFromHTTPServerResponse out(request, response, keep_alive_timeout); - if (mode == "write") + + try { - try + if (mode == "write") { if (!params.has("db_name")) { @@ -140,51 +150,38 @@ void ODBCHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne } std::string db_name = params.get("db_name"); std::string table_name = params.get("table_name"); - LOG_TRACE(log, "DB name: '" << db_name << "', table name: '" << table_name << "'"); + LOG_TRACE(log, "DB name: '" << db_name << "', table name: '" << table_name << "'"); + + POCO_SQL_ODBC_CLASS::SessionImpl session(validateODBCConnectionString(connection_string), DBMS_DEFAULT_CONNECT_TIMEOUT_SEC); + auto quoting_style = getQuotingStyle(session.dbc().handle()); auto pool = getPool(connection_string); ReadBufferFromIStream read_buf(request.stream()); - BlockInputStreamPtr input_stream = FormatFactory::instance().getInput(format, read_buf, *sample_block, context, max_block_size); - ODBCBlockOutputStream output_stream(pool->get(), db_name, table_name, *sample_block); + BlockInputStreamPtr input_stream = FormatFactory::instance().getInput(format, read_buf, *sample_block, + context, max_block_size); + ODBCBlockOutputStream output_stream(pool->get(), db_name, table_name, *sample_block, quoting_style); copyData(*input_stream, output_stream); writeStringBinary("Ok.", out); } - catch (...) + else { - auto message = getCurrentExceptionMessage(true); - response.setStatusAndReason( - Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); - writeStringBinary(message, out); - tryLogCurrentException(log); - } - } - else - { - if (!params.has("query")) - { - processError(response, "No 'query' in request body"); - return; - } + std::string query = params.get("query"); + LOG_TRACE(log, "Query: " << query); - std::string query = params.get("query"); - LOG_TRACE(log, "Query: " << query); - - try - { BlockOutputStreamPtr writer = FormatFactory::instance().getOutput(format, out, *sample_block, context); auto pool = getPool(connection_string); ODBCBlockInputStream inp(pool->get(), query, *sample_block, max_block_size); copyData(inp, *writer); } - catch (...) - { - auto message = getCurrentExceptionMessage(true); - response.setStatusAndReason( - Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); // can't call process_error, because of too soon response sending - writeStringBinary(message, out); - tryLogCurrentException(log); + } + catch (...) + { + auto message = getCurrentExceptionMessage(true); + response.setStatusAndReason( + Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); // can't call process_error, because of too soon response sending + writeStringBinary(message, out); + tryLogCurrentException(log); - } } } diff --git a/programs/odbc-bridge/ODBCBlockOutputStream.cpp b/programs/odbc-bridge/ODBCBlockOutputStream.cpp index 80d6355752b..67990bf1ac0 100644 --- a/programs/odbc-bridge/ODBCBlockOutputStream.cpp +++ b/programs/odbc-bridge/ODBCBlockOutputStream.cpp @@ -4,6 +4,11 @@ #include #include #include +#include +#include +#include +#include "getIdentifierQuote.h" + namespace DB { @@ -12,16 +17,22 @@ namespace { using ValueType = ExternalResultDescription::ValueType; - std::string commaSeparateColumnNames(const ColumnsWithTypeAndName & columns) + std::string getInsertQuery(const std::string & db_name, const std::string & table_name, const ColumnsWithTypeAndName & columns, IdentifierQuotingStyle quoting) { - std::string result = "("; + ASTInsertQuery query; + query.table_id.database_name = db_name; + query.table_id.table_name = table_name; + query.columns = std::make_shared(','); + query.children.push_back(query.columns); for (size_t i = 0; i < columns.size(); ++i) - { - if (i > 0) - result += ","; - result += columns[i].name; - } - return result + ")"; + query.columns->children.emplace_back(std::make_shared(columns[i].name)); + + std::stringstream ss; + IAST::FormatSettings settings(ss, true); + settings.always_quote_identifiers = true; + settings.identifier_quoting_style = quoting; + query.IAST::format(settings); + return ss.str(); } std::string getQuestionMarks(size_t n) @@ -69,18 +80,19 @@ namespace case ValueType::vtUUID: return Poco::Dynamic::Var(UUID(field.get()).toUnderType().toHexString()).convert(); } - return Poco::Dynamic::Var(); // Throw smth here? } } ODBCBlockOutputStream::ODBCBlockOutputStream(Poco::Data::Session && session_, const std::string & remote_database_name_, const std::string & remote_table_name_, - const Block & sample_block_) + const Block & sample_block_, + IdentifierQuotingStyle quoting_) : session(session_) , db_name(remote_database_name_) , table_name(remote_table_name_) , sample_block(sample_block_) + , quoting(quoting_) , log(&Logger::get("ODBCBlockOutputStream")) { description.init(sample_block); @@ -98,11 +110,11 @@ void ODBCBlockOutputStream::write(const Block & block) columns.push_back({block.getColumns()[i], sample_block.getDataTypes()[i], sample_block.getNames()[i]}); std::vector row_to_insert(block.columns()); - Poco::Data::Statement statement(session << "INSERT INTO " + db_name + "." + table_name + " " + - commaSeparateColumnNames(columns) + - " VALUES " + getQuestionMarks(block.columns())); + Poco::Data::Statement statement(session << getInsertQuery(db_name, table_name, columns, quoting) + getQuestionMarks(block.columns())); for (size_t i = 0; i < block.columns(); ++i) + { statement, Poco::Data::Keywords::use(row_to_insert[i]); + } for (size_t i = 0; i < block.rows(); ++i) { @@ -110,7 +122,10 @@ void ODBCBlockOutputStream::write(const Block & block) { Field val; columns[col_idx].column->get(i, val); - row_to_insert[col_idx] = getVarFromField(val, description.types[col_idx].first); + if (val.isNull()) + row_to_insert[col_idx] = Poco::Dynamic::Var(); + else + row_to_insert[col_idx] = getVarFromField(val, description.types[col_idx].first); } statement.execute(); } diff --git a/programs/odbc-bridge/ODBCBlockOutputStream.h b/programs/odbc-bridge/ODBCBlockOutputStream.h index 0153c4fa60c..39e1d6f77ac 100644 --- a/programs/odbc-bridge/ODBCBlockOutputStream.h +++ b/programs/odbc-bridge/ODBCBlockOutputStream.h @@ -4,6 +4,7 @@ #include #include #include +#include namespace DB { @@ -11,7 +12,7 @@ class ODBCBlockOutputStream : public IBlockOutputStream { public: ODBCBlockOutputStream(Poco::Data::Session && session_, const std::string & remote_database_name_, - const std::string & remote_table_name_, const Block & sample_block_); + const std::string & remote_table_name_, const Block & sample_block_, IdentifierQuotingStyle quoting); Block getHeader() const override; void write(const Block & block) override; @@ -21,6 +22,7 @@ private: std::string db_name; std::string table_name; Block sample_block; + IdentifierQuotingStyle quoting; ExternalResultDescription description; Poco::Logger * log; diff --git a/programs/odbc-bridge/getIdentifierQuote.cpp b/programs/odbc-bridge/getIdentifierQuote.cpp index 766d5bf6bde..15b3749d37d 100644 --- a/programs/odbc-bridge/getIdentifierQuote.cpp +++ b/programs/odbc-bridge/getIdentifierQuote.cpp @@ -12,6 +12,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + std::string getIdentifierQuote(SQLHDBC hdbc) { std::string identifier; @@ -36,6 +41,19 @@ std::string getIdentifierQuote(SQLHDBC hdbc) return identifier; } +IdentifierQuotingStyle getQuotingStyle(SQLHDBC hdbc) +{ + auto identifier_quote = getIdentifierQuote(hdbc); + if (identifier_quote.length() == 0) + return IdentifierQuotingStyle::None; + else if (identifier_quote[0] == '`') + return IdentifierQuotingStyle::Backticks; + else if (identifier_quote[0] == '"') + return IdentifierQuotingStyle::DoubleQuotes; + else + throw Exception("Can not map quote identifier '" + identifier_quote + "' to IdentifierQuotingStyle value", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); +} + } #endif diff --git a/programs/odbc-bridge/getIdentifierQuote.h b/programs/odbc-bridge/getIdentifierQuote.h index 8bf119209c2..0fb4c3bddb1 100644 --- a/programs/odbc-bridge/getIdentifierQuote.h +++ b/programs/odbc-bridge/getIdentifierQuote.h @@ -8,11 +8,15 @@ # include +#include + namespace DB { std::string getIdentifierQuote(SQLHDBC hdbc); +IdentifierQuotingStyle getQuotingStyle(SQLHDBC hdbc); + } #endif From 6a496ce9dcc5f4af3001e5dec3d2403757db38e4 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 15 May 2020 00:51:21 +0300 Subject: [PATCH 236/738] better test --- .../integration/test_odbc_interaction/test.py | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_odbc_interaction/test.py b/tests/integration/test_odbc_interaction/test.py index 2a52cedb636..c6b4e0eb5b0 100644 --- a/tests/integration/test_odbc_interaction/test.py +++ b/tests/integration/test_odbc_interaction/test.py @@ -111,6 +111,21 @@ CREATE TABLE {}(id UInt32, name String, age UInt32, money UInt32, column_x Nulla conn.close() +def test_mysql_insert(started_cluster): + mysql_setup = node1.odbc_drivers["MySQL"] + table_name = 'test_insert' + conn = get_mysql_conn() + create_mysql_table(conn, table_name) + odbc_args = "'DSN={}', '{}', '{}'".format(mysql_setup["DSN"], mysql_setup["Database"], table_name) + + node1.query("create table mysql_insert (id Int64, name String, age UInt8, money Float, column_x Nullable(Int16)) Engine=ODBC({})".format(odbc_args)) + node1.query("insert into mysql_insert values (1, 'test', 11, 111, 1111), (2, 'odbc', 22, 222, NULL)") + assert node1.query("select * from mysql_insert") == "1\ttest\t11\t111\t1111\n2\todbc\t22\t222\t\\N\n" + + node1.query("insert into table function odbc({}) values (3, 'insert', 33, 333, 3333)".format(odbc_args)) + node1.query("insert into table function odbc({}) (id, name, age, money) select id*4, upper(name), age*4, money*4 from odbc({}) where id=1".format(odbc_args, odbc_args)) + assert node1.query("select * from mysql_insert where id in (3, 4)") == "3\tinsert\t33\t333\t3333\n4\tTEST\t44\t444\t\\N\n" + def test_sqlite_simple_select_function_works(started_cluster): sqlite_setup = node1.odbc_drivers["SQLite3"] @@ -170,7 +185,11 @@ def test_sqlite_odbc_cached_dictionary(started_cluster): assert node1.query("select dictGetUInt8('sqlite3_odbc_cached', 'Z', toUInt64(1))") == "3\n" - node1.exec_in_container(["bash", "-c", "echo 'INSERT INTO t3 values(200, 2, 7);' | sqlite3 {}".format(sqlite_db)], privileged=True, user='root') + # Allow insert + node1.exec_in_container(["bash", "-c", "chmod a+rw /tmp"], privileged=True, user='root') + node1.exec_in_container(["bash", "-c", "chmod a+rw {}".format(sqlite_db)], privileged=True, user='root') + + node1.query("insert into table function odbc('DSN={};', '', 't3') values (200, 2, 7)".format(node1.odbc_drivers["SQLite3"]["DSN"])) assert node1.query("select dictGetUInt8('sqlite3_odbc_cached', 'Z', toUInt64(200))") == "7\n" # new value @@ -206,6 +225,10 @@ def test_postgres_insert(started_cluster): node1.query("create table pg_insert (column1 UInt8, column2 String) engine=ODBC('DSN=postgresql_odbc;', 'clickhouse', 'test_table')") node1.query("insert into pg_insert values (1, 'hello'), (2, 'world')") assert node1.query("select * from pg_insert") == '1\thello\n2\tworld\n' + node1.query("insert into table function odbc('DSN=postgresql_odbc;', 'clickhouse', 'test_table') format CSV 3,test") + node1.query("insert into table function odbc('DSN=postgresql_odbc;', 'clickhouse', 'test_table') select number, 's' || toString(number) from numbers (4, 7)") + assert node1.query("select sum(column1), count(column1) from pg_insert") == "55\t10\n" + assert node1.query("select sum(n), count(n) from (select (*,).1 as n from (select * from odbc('DSN=postgresql_odbc;', 'clickhouse', 'test_table')))") == "55\t10\n" def test_bridge_dies_with_parent(started_cluster): node1.query("select dictGetString('postgres_odbc_hashed', 'column2', toUInt64(1))") From fc7afaa639933701edad33b24a737ccadb93ab02 Mon Sep 17 00:00:00 2001 From: Ri Date: Fri, 15 May 2020 03:01:14 +0300 Subject: [PATCH 237/738] Bitonic sort on GPU (OpenCL) (#10232) --- CMakeLists.txt | 10 + cmake/find/opencl.cmake | 17 + programs/server/Server.cpp | 8 + src/CMakeLists.txt | 5 + src/Columns/ColumnVector.cpp | 15 + src/Common/BitonicSort.h | 270 ++++ src/Common/ErrorCodes.cpp | 1 + src/Common/bitonicSortKernels.cl | 1250 +++++++++++++++++ src/Common/config.h.in | 1 + src/Common/oclBasics.cpp | 363 +++++ src/Common/tests/CMakeLists.txt | 3 + src/Common/tests/bitonic_sort.cpp | 248 ++++ src/Core/Settings.h | 2 + src/Core/SettingsCollection.cpp | 6 + src/Core/SettingsCollection.h | 9 + src/Core/SortDescription.h | 14 +- src/Interpreters/InterpreterSelectQuery.cpp | 5 +- src/Interpreters/sortBlock.cpp | 18 +- ...80_opencl_bitonic_order_by_uint8.reference | 42 + .../01280_opencl_bitonic_order_by_uint8.sql | 1 + ...281_opencl_bitonic_order_by_int8.reference | 42 + .../01281_opencl_bitonic_order_by_int8.sql | 1 + ...2_opencl_bitonic_order_by_uint16.reference | 42 + .../01282_opencl_bitonic_order_by_uint16.sql | 1 + ...83_opencl_bitonic_order_by_int16.reference | 42 + .../01283_opencl_bitonic_order_by_int16.sql | 1 + ...4_opencl_bitonic_order_by_uint32.reference | 42 + .../01284_opencl_bitonic_order_by_uint32.sql | 1 + ...85_opencl_bitonic_order_by_int32.reference | 42 + .../01285_opencl_bitonic_order_by_int32.sql | 1 + ...6_opencl_bitonic_order_by_uint64.reference | 42 + .../01286_opencl_bitonic_order_by_uint64.sql | 1 + ...87_opencl_bitonic_order_by_int64.reference | 42 + .../01287_opencl_bitonic_order_by_int64.sql | 1 + 34 files changed, 2580 insertions(+), 9 deletions(-) create mode 100644 cmake/find/opencl.cmake create mode 100644 src/Common/BitonicSort.h create mode 100644 src/Common/bitonicSortKernels.cl create mode 100644 src/Common/oclBasics.cpp create mode 100644 src/Common/tests/bitonic_sort.cpp create mode 100644 tests/queries/0_stateless/01280_opencl_bitonic_order_by_uint8.reference create mode 100644 tests/queries/0_stateless/01280_opencl_bitonic_order_by_uint8.sql create mode 100644 tests/queries/0_stateless/01281_opencl_bitonic_order_by_int8.reference create mode 100644 tests/queries/0_stateless/01281_opencl_bitonic_order_by_int8.sql create mode 100644 tests/queries/0_stateless/01282_opencl_bitonic_order_by_uint16.reference create mode 100644 tests/queries/0_stateless/01282_opencl_bitonic_order_by_uint16.sql create mode 100644 tests/queries/0_stateless/01283_opencl_bitonic_order_by_int16.reference create mode 100644 tests/queries/0_stateless/01283_opencl_bitonic_order_by_int16.sql create mode 100644 tests/queries/0_stateless/01284_opencl_bitonic_order_by_uint32.reference create mode 100644 tests/queries/0_stateless/01284_opencl_bitonic_order_by_uint32.sql create mode 100644 tests/queries/0_stateless/01285_opencl_bitonic_order_by_int32.reference create mode 100644 tests/queries/0_stateless/01285_opencl_bitonic_order_by_int32.sql create mode 100644 tests/queries/0_stateless/01286_opencl_bitonic_order_by_uint64.reference create mode 100644 tests/queries/0_stateless/01286_opencl_bitonic_order_by_uint64.sql create mode 100644 tests/queries/0_stateless/01287_opencl_bitonic_order_by_int64.reference create mode 100644 tests/queries/0_stateless/01287_opencl_bitonic_order_by_int64.sql diff --git a/CMakeLists.txt b/CMakeLists.txt index b2453860d28..22fe14e5b55 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -346,6 +346,7 @@ include (cmake/find/libgsasl.cmake) include (cmake/find/rdkafka.cmake) include (cmake/find/capnp.cmake) include (cmake/find/llvm.cmake) +include (cmake/find/opencl.cmake) include (cmake/find/h3.cmake) include (cmake/find/libxml2.cmake) include (cmake/find/brotli.cmake) @@ -382,6 +383,15 @@ if (OS_LINUX AND NOT ENABLE_JEMALLOC) message (WARNING "Non default allocator is disabled. This is not recommended for production Linux builds.") endif () +if (USE_OPENCL) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DBITONIC_SORT_PREFERRED") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DBITONIC_SORT_PREFERRED") + + if (OS_DARWIN) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -framework OpenCL") + endif () +endif () + include (cmake/print_flags.cmake) if (TARGET global-group) diff --git a/cmake/find/opencl.cmake b/cmake/find/opencl.cmake new file mode 100644 index 00000000000..b1bf4630990 --- /dev/null +++ b/cmake/find/opencl.cmake @@ -0,0 +1,17 @@ +if(ENABLE_OPENCL) + +# Intel OpenCl driver: sudo apt install intel-opencl-icd +# TODO It's possible to add it as submodules: https://github.com/intel/compute-runtime/releases + +# OpenCL applications should link wiht ICD loader +# sudo apt install opencl-headers ocl-icd-libopencl1 +# sudo ln -s /usr/lib/x86_64-linux-gnu/libOpenCL.so.1.0.0 /usr/lib/libOpenCL.so + +find_package(OpenCL REQUIRED) +if(OpenCL_FOUND) + set(USE_OPENCL 1) +endif() + +endif() + +message(STATUS "Using opencl=${USE_OPENCL}: ${OpenCL_INCLUDE_DIRS} : ${OpenCL_LIBRARIES}") diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 9373d5dbaab..691ede66eb4 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -61,6 +61,10 @@ #include #include "MySQLHandlerFactory.h" +#ifdef BITONIC_SORT_PREFERRED +#include "Common/BitonicSort.h" +#endif + #if !defined(ARCADIA_BUILD) # include "config_core.h" # include "Common/config_version.h" @@ -221,6 +225,10 @@ int Server::main(const std::vector & /*args*/) registerDictionaries(); registerDisks(); + #if defined (BITONIC_SORT_PREFERRED) + BitonicSort::getInstance().configure(); + #endif + CurrentMetrics::set(CurrentMetrics::Revision, ClickHouseRevision::get()); CurrentMetrics::set(CurrentMetrics::VersionInteger, ClickHouseRevision::getVersionInteger()); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 57d245eac6e..5ea862b7db4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -346,6 +346,11 @@ if (USE_BROTLI) target_include_directories (clickhouse_common_io SYSTEM BEFORE PRIVATE ${BROTLI_INCLUDE_DIR}) endif() +if (USE_OPENCL) + target_link_libraries (clickhouse_common_io PRIVATE ${OpenCL_LIBRARIES}) + target_include_directories (clickhouse_common_io SYSTEM BEFORE PRIVATE ${OpenCL_INCLUDE_DIRS}) +endif () + dbms_target_include_directories (PUBLIC ${DBMS_INCLUDE_DIR}) target_include_directories (clickhouse_common_io PUBLIC ${DBMS_INCLUDE_DIR}) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 814de64a0b6..9b128fcffec 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -18,6 +18,15 @@ #include #include +#if !defined(ARCADIA_BUILD) +# include +# if USE_OPENCL +# include "Common/BitonicSort.h" +# endif +#else +#undef USE_OPENCL +#endif + #ifdef __SSE2__ #include #endif @@ -135,6 +144,12 @@ void ColumnVector::getPermutation(bool reverse, size_t limit, int nan_directi } else { +#if USE_OPENCL + /// If bitonic sort if specified as preferred than `nan_direction_hint` equals specific value 42. + if (nan_direction_hint == 42 && BitonicSort::getInstance().sort(data, res, !reverse)) + return; +#endif + /// A case for radix sort if constexpr (is_arithmetic_v && !std::is_same_v) { diff --git a/src/Common/BitonicSort.h b/src/Common/BitonicSort.h new file mode 100644 index 00000000000..840ab477a45 --- /dev/null +++ b/src/Common/BitonicSort.h @@ -0,0 +1,270 @@ +#pragma once + +#include +#if !defined(__APPLE__) && !defined(__FreeBSD__) +#include +#endif + +#ifdef __APPLE__ +#include +#else +#include +#endif + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "oclBasics.cpp" +#include "bitonicSortKernels.cl" + +class BitonicSort +{ +public: + + static BitonicSort & getInstance() + { + static BitonicSort instance = BitonicSort(); + return instance; + } + + /// Sorts given array in specified order. Returns `true` if given sequence was sorted, `false` otherwise. + template + bool sort(const DB::PaddedPODArray & data, DB::IColumn::Permutation & res, cl_uint sort_ascending) + { + size_t s = data.size(); + + /// Getting the nearest power of 2. + size_t power = 1; + + if (s <= 8) power = 8; + else while (power < s) power <<= 1; + + /// Allocates more space for additional stubs to be added if needed. + std::vector pairs_content(power); + std::vector pairs_indices(power); + for (UInt32 i = 0; i < s; ++i) + { + pairs_content[i] = data[i]; + pairs_indices[i] = i; + } + + bool result = sort(pairs_content.data(), pairs_indices.data(), s, power - s, sort_ascending); + + if (!result) return false; + + for (size_t i = 0, shift = 0; i < power; ++i) + { + if (pairs_indices[i] >= s) + { + ++shift; + continue; + } + res[i - shift] = pairs_indices[i]; + } + + return true; + } + + /// Creating a configuration instance with making all OpenCl required variables + /// such as device, platform, context, queue, program and kernel. + void configure() + { + OCL::Settings settings = OCL::Settings(1, nullptr, 1, nullptr, 1, 0); + + cl_platform_id platform = OCL::getPlatformID(settings); + cl_device_id device = OCL::getDeviceID(platform, settings); + cl_context gpu_context = OCL::makeContext(device, settings); + cl_command_queue command_queue = OCL::makeCommandQueue(device, gpu_context, settings); + + cl_program program = OCL::makeProgram(bitonic_sort_kernels, gpu_context, device, settings); + + /// Creating kernels for each specified data type. + cl_int error = 0; + + kernels["char"] = std::shared_ptr(clCreateKernel(program, "bitonicSort_char", &error), + clReleaseKernel); + kernels["uchar"] = std::shared_ptr(clCreateKernel(program, "bitonicSort_uchar", &error), + clReleaseKernel); + kernels["short"] = std::shared_ptr(clCreateKernel(program, "bitonicSort_short", &error), + clReleaseKernel); + kernels["ushort"] = std::shared_ptr(clCreateKernel(program, "bitonicSort_ushort", &error), + clReleaseKernel); + kernels["int"] = std::shared_ptr(clCreateKernel(program, "bitonicSort_int", &error), + clReleaseKernel); + kernels["uint"] = std::shared_ptr(clCreateKernel(program, "bitonicSort_uint", &error), + clReleaseKernel); + kernels["long"] = std::shared_ptr(clCreateKernel(program, "bitonicSort_long", &error), + clReleaseKernel); + kernels["ulong"] = std::shared_ptr(clCreateKernel(program, "bitonicSort_ulong", &error), + clReleaseKernel); + OCL::checkError(error); + + configuration = std::shared_ptr(new OCL::Configuration(device, gpu_context, command_queue, program)); + } + +private: + /// Dictionary with kernels for each type from list: uchar, char, ushort, short, uint, int, ulong and long. + std::map> kernels; + /// Current configuration with core OpenCL instances. + std::shared_ptr configuration = nullptr; + + /// Returns `true` if given sequence was sorted, `false` otherwise. + template + bool sort(T * p_input, cl_uint * indices, cl_int array_size, cl_int number_of_stubs, cl_uint sort_ascending) + { + if (typeid(T).name() == typeid(cl_char).name()) + sort_char(reinterpret_cast(p_input), indices, array_size, number_of_stubs, sort_ascending); + else if (typeid(T) == typeid(cl_uchar)) + sort_uchar(reinterpret_cast(p_input), indices, array_size, number_of_stubs, sort_ascending); + else if (typeid(T) == typeid(cl_short)) + sort_short(reinterpret_cast(p_input), indices, array_size, number_of_stubs, sort_ascending); + else if (typeid(T) == typeid(cl_ushort)) + sort_ushort(reinterpret_cast(p_input), indices, array_size, number_of_stubs, sort_ascending); + else if (typeid(T) == typeid(cl_int)) + sort_int(reinterpret_cast(p_input), indices, array_size, number_of_stubs, sort_ascending); + else if (typeid(T) == typeid(cl_uint)) + sort_uint(reinterpret_cast(p_input), indices, array_size, number_of_stubs, sort_ascending); + else if (typeid(T) == typeid(cl_long)) + sort_long(reinterpret_cast(p_input), indices, array_size, number_of_stubs, sort_ascending); + else if (typeid(T) == typeid(cl_ulong)) + sort_ulong(reinterpret_cast(p_input), indices, array_size, number_of_stubs, sort_ascending); + else + return false; + + return true; + } + + /// Specific functions for each integer type. + void sort_char(cl_char * p_input, cl_uint * indices, cl_int array_size, cl_int number_of_stubs, cl_uint sort_ascending) + { + cl_char stubs_value = sort_ascending ? CHAR_MAX : CHAR_MIN; + fillWithStubs(number_of_stubs, stubs_value, p_input, indices, array_size); + sort(kernels["char"].get(), p_input, indices, array_size + number_of_stubs, sort_ascending); + } + + void sort_uchar(cl_uchar * p_input, cl_uint * indices, cl_int array_size, cl_int number_of_stubs, cl_uint sort_ascending) + { + cl_uchar stubs_value = sort_ascending ? UCHAR_MAX : 0; + fillWithStubs(number_of_stubs, stubs_value, p_input, indices, array_size); + sort(kernels["uchar"].get(), p_input, indices, array_size + number_of_stubs, sort_ascending); + } + + void sort_short(cl_short * p_input, cl_uint * indices, cl_int array_size, cl_int number_of_stubs, cl_uint sort_ascending) + { + cl_short stubs_value = sort_ascending ? SHRT_MAX : SHRT_MIN; + fillWithStubs(number_of_stubs, stubs_value, p_input, indices, array_size); + sort(kernels["short"].get(), p_input, indices, array_size + number_of_stubs, sort_ascending); + } + + void sort_ushort(cl_ushort * p_input, cl_uint * indices, cl_int array_size, cl_int number_of_stubs, cl_uint sort_ascending) + { + cl_ushort stubs_value = sort_ascending ? USHRT_MAX : 0; + fillWithStubs(number_of_stubs, stubs_value, p_input, indices, array_size); + sort(kernels["ushort"].get(), p_input, indices, array_size + number_of_stubs, sort_ascending); + } + + void sort_int(cl_int * p_input, cl_uint * indices, cl_int array_size, cl_int number_of_stubs, cl_uint sort_ascending) + { + cl_int stubs_value = sort_ascending ? INT_MAX : INT_MIN; + fillWithStubs(number_of_stubs, stubs_value, p_input, indices, array_size); + sort(kernels["int"].get(), p_input, indices, array_size + number_of_stubs, sort_ascending); + } + + void sort_uint(cl_uint * p_input, cl_uint * indices, cl_int array_size, cl_int number_of_stubs, cl_uint sort_ascending) + { + cl_uint stubs_value = sort_ascending ? UINT_MAX : 0; + fillWithStubs(number_of_stubs, stubs_value, p_input, indices, array_size); + sort(kernels["uint"].get(), p_input, indices, array_size + number_of_stubs, sort_ascending); + } + + void sort_long(cl_long * p_input, cl_uint * indices, cl_int array_size, cl_int number_of_stubs, cl_uint sort_ascending) + { + cl_long stubs_value = sort_ascending ? LONG_MAX : LONG_MIN; + fillWithStubs(number_of_stubs, stubs_value, p_input, indices, array_size); + sort(kernels["long"].get(), p_input, indices, array_size + number_of_stubs, sort_ascending); + } + + void sort_ulong(cl_ulong * p_input, cl_uint * indices, cl_int array_size, cl_int number_of_stubs, cl_uint sort_ascending) + { + cl_ulong stubs_value = sort_ascending ? ULONG_MAX : 0; + fillWithStubs(number_of_stubs, stubs_value, p_input, indices, array_size); + sort(kernels["ulong"].get(), p_input, indices, array_size + number_of_stubs, sort_ascending); + } + + /// Sorts p_input inplace with indices. Works only with arrays which size equals to power of two. + template + void sort(cl_kernel kernel, T * p_input, cl_uint * indices, cl_int array_size, cl_uint sort_ascending) + { + cl_int error = CL_SUCCESS; + cl_int num_stages = 0; + + for (cl_int temp = array_size; temp > 2; temp >>= 1) + num_stages++; + + /// Creating OpenCL buffers using input arrays memory. + cl_mem cl_input_buffer = OCL::createBuffer(p_input, array_size, configuration.get()->context()); + cl_mem cl_indices_buffer = OCL::createBuffer(indices, array_size, configuration.get()->context()); + + configureKernel(kernel, 0, static_cast(&cl_input_buffer)); + configureKernel(kernel, 1, static_cast(&cl_indices_buffer)); + configureKernel(kernel, 4, static_cast(&sort_ascending)); + + for (cl_int stage = 0; stage < num_stages; stage++) + { + configureKernel(kernel, 2, static_cast(&stage)); + + for (cl_int pass_of_stage = stage; pass_of_stage >= 0; pass_of_stage--) + { + configureKernel(kernel, 3, static_cast(&pass_of_stage)); + + /// Setting work-item dimensions. + size_t gsize = array_size / (2 * 4); + size_t global_work_size[1] = {pass_of_stage ? gsize : gsize << 1 }; // number of quad items in input array + + /// Executing kernel. + error = clEnqueueNDRangeKernel(configuration.get()->commandQueue(), kernel, 1, nullptr, + global_work_size, nullptr, 0, nullptr, nullptr); + OCL::checkError(error); + } + } + + /// Syncs all threads. + OCL::finishCommandQueue(configuration.get()->commandQueue()); + + OCL::releaseData(p_input, array_size, cl_input_buffer, configuration.get()->commandQueue()); + OCL::releaseData(indices, array_size, cl_indices_buffer, configuration.get()->commandQueue()); + } + + template + void configureKernel(cl_kernel kernel, int number_of_argument, void * source) + { + cl_int error = clSetKernelArg(kernel, number_of_argument, sizeof(T), source); + OCL::checkError(error); + } + + /// Fills given sequences from `arraySize` index with `numberOfStubs` values. + template + void fillWithStubs(cl_int number_of_stubs, T value, T * p_input, + cl_uint * indices, cl_int array_size) + { + for (cl_int index = 0; index < number_of_stubs; ++index) + { + p_input[array_size + index] = value; + indices[array_size + index] = array_size + index; + } + } + + BitonicSort() {} + BitonicSort(BitonicSort const &); + void operator=(BitonicSort const &); +}; diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 120c9b93a78..75f22ddc5eb 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -493,6 +493,7 @@ namespace ErrorCodes extern const int NO_REMOTE_SHARD_AVAILABLE = 519; extern const int CANNOT_DETACH_DICTIONARY_AS_TABLE = 520; extern const int ATOMIC_RENAME_FAIL = 521; + extern const int OPENCL_ERROR = 522; extern const int KEEPER_EXCEPTION = 999; extern const int POCO_EXCEPTION = 1000; diff --git a/src/Common/bitonicSortKernels.cl b/src/Common/bitonicSortKernels.cl new file mode 100644 index 00000000000..c124dfe85bc --- /dev/null +++ b/src/Common/bitonicSortKernels.cl @@ -0,0 +1,1250 @@ +// Copyright (c) 2009-2011 Intel Corporation +// All rights reserved. +// +// WARRANTY DISCLAIMER +// +// THESE MATERIALS ARE PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INTEL OR ITS +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THESE +// MATERIALS, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Intel Corporation is the author of the Materials, and requests that all +// problem reports or change requests be submitted to it directly + +static const char * bitonic_sort_kernels = R"( + +uchar4 makeMask_uchar(uchar4 srcLeft, uchar4 srcRight, uint4 indexLeft, uint4 indexRight) +{ + uchar4 mask = convert_uchar4(srcLeft < srcRight); + + if (srcLeft.x == srcRight.x) + mask.x = (indexLeft.x < indexRight.x) ? UCHAR_MAX : 0; + if (srcLeft.y == srcRight.y) + mask.y = (indexLeft.y < indexRight.y) ? UCHAR_MAX : 0; + if (srcLeft.z == srcRight.z) + mask.z = (indexLeft.z < indexRight.z) ? UCHAR_MAX : 0; + if (srcLeft.w == srcRight.w) + mask.w = (indexLeft.w < indexRight.w) ? UCHAR_MAX : 0; + + return mask; +} + + +char4 makeMask_char(char4 srcLeft, char4 srcRight, uint4 indexLeft, uint4 indexRight) +{ + char4 mask = srcLeft < srcRight; + + if (srcLeft.x == srcRight.x) + mask.x = -(indexLeft.x < indexRight.x); + if (srcLeft.y == srcRight.y) + mask.y = -(indexLeft.y < indexRight.y); + if (srcLeft.z == srcRight.z) + mask.z = -(indexLeft.z < indexRight.z); + if (srcLeft.w == srcRight.w) + mask.w = -(indexLeft.w < indexRight.w); + + return mask; +} + + +ushort4 makeMask_ushort(ushort4 srcLeft, ushort4 srcRight, uint4 indexLeft, uint4 indexRight) +{ + ushort4 mask = convert_ushort4(srcLeft < srcRight); + + if (srcLeft.x == srcRight.x) + mask.x = (indexLeft.x < indexRight.x) ? USHRT_MAX : 0; + if (srcLeft.y == srcRight.y) + mask.y = (indexLeft.y < indexRight.y) ? USHRT_MAX : 0; + if (srcLeft.z == srcRight.z) + mask.z = (indexLeft.z < indexRight.z) ? USHRT_MAX : 0; + if (srcLeft.w == srcRight.w) + mask.w = (indexLeft.w < indexRight.w) ? USHRT_MAX : 0; + + return mask; +} + + +short4 makeMask_short(short4 srcLeft, short4 srcRight, uint4 indexLeft, uint4 indexRight) +{ + short4 mask = srcLeft < srcRight; + + if (srcLeft.x == srcRight.x) + mask.x = -(indexLeft.x < indexRight.x); + if (srcLeft.y == srcRight.y) + mask.y = -(indexLeft.y < indexRight.y); + if (srcLeft.z == srcRight.z) + mask.z = -(indexLeft.z < indexRight.z); + if (srcLeft.w == srcRight.w) + mask.w = -(indexLeft.w < indexRight.w); + + return mask; +} + + +uint4 makeMask_uint(uint4 srcLeft, uint4 srcRight, uint4 indexLeft, uint4 indexRight) +{ + uint4 mask = convert_uint4(srcLeft < srcRight); + + if (srcLeft.x == srcRight.x) + mask.x = (indexLeft.x < indexRight.x) ? UINT_MAX : 0; + if (srcLeft.y == srcRight.y) + mask.y = (indexLeft.y < indexRight.y) ? UINT_MAX : 0; + if (srcLeft.z == srcRight.z) + mask.z = (indexLeft.z < indexRight.z) ? UINT_MAX : 0; + if (srcLeft.w == srcRight.w) + mask.w = (indexLeft.w < indexRight.w) ? UINT_MAX : 0; + + return mask; +} + + +int4 makeMask_int(int4 srcLeft, int4 srcRight, uint4 indexLeft, uint4 indexRight) +{ + int4 mask = srcLeft < srcRight; + + if (srcLeft.x == srcRight.x) + mask.x = -(indexLeft.x < indexRight.x); + if (srcLeft.y == srcRight.y) + mask.y = -(indexLeft.y < indexRight.y); + if (srcLeft.z == srcRight.z) + mask.z = -(indexLeft.z < indexRight.z); + if (srcLeft.w == srcRight.w) + mask.w = -(indexLeft.w < indexRight.w); + + return mask; +} + + +ulong4 makeMask_ulong(ulong4 srcLeft, ulong4 srcRight, uint4 indexLeft, uint4 indexRight) +{ + ulong4 mask = convert_ulong4(srcLeft < srcRight); + + if (srcLeft.x == srcRight.x) + mask.x = (indexLeft.x < indexRight.x) ? ULONG_MAX : 0; + if (srcLeft.y == srcRight.y) + mask.y = (indexLeft.y < indexRight.y) ? ULONG_MAX : 0; + if (srcLeft.z == srcRight.z) + mask.z = (indexLeft.z < indexRight.z) ? ULONG_MAX : 0; + if (srcLeft.w == srcRight.w) + mask.w = (indexLeft.w < indexRight.w) ? ULONG_MAX : 0; + + return mask; +} + + +long4 makeMask_long(long4 srcLeft, long4 srcRight, uint4 indexLeft, uint4 indexRight) +{ + long4 mask = srcLeft < srcRight; + + if (srcLeft.x == srcRight.x) + mask.x = -(indexLeft.x < indexRight.x); + if (srcLeft.y == srcRight.y) + mask.y = -(indexLeft.y < indexRight.y); + if (srcLeft.z == srcRight.z) + mask.z = -(indexLeft.z < indexRight.z); + if (srcLeft.w == srcRight.w) + mask.w = -(indexLeft.w < indexRight.w); + + return mask; +} + + +__kernel void __attribute__((vec_type_hint(char4))) __attribute__((vec_type_hint(uint4))) bitonicSort_char(__global char4 * theArray, + __global uint4 * indices, + const uint stage, + const uint passOfStage, + const uint dir) +{ + size_t i = get_global_id(0); + char4 srcLeft, srcRight, mask; + char4 imask10 = (char4)(0, 0, -1, -1); + char4 imask11 = (char4)(0, -1, 0, -1); + uint4 indexLeft, indexRight; + + if (stage > 0) + { + if (passOfStage > 0) // upper level pass, exchange between two fours + { + size_t r = 1 << (passOfStage - 1); // length of arrow + size_t lmask = r - 1; + size_t left = ((i >> (passOfStage - 1)) << passOfStage) + (i & lmask); + size_t right = left + r; + + srcLeft = theArray[left]; + srcRight = theArray[right]; + indexLeft = indices[left]; + indexRight = indices[right]; + + mask = makeMask_char(srcLeft, srcRight, indexLeft, indexRight); + + char4 imin = (srcLeft & mask) | (srcRight & ~mask); + char4 imax = (srcLeft & ~mask) | (srcRight & mask); + + if (((i >> (stage - 1)) & 1) ^ dir) + { + theArray[left] = imin; + theArray[right] = imax; + + indices[left] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + indices[right] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + else + { + theArray[right] = imin; + theArray[left] = imax; + + indices[right] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + indices[left] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + } + else // last pass, sort inside one four + { + srcLeft = theArray[i]; + srcRight = srcLeft.zwxy; + indexLeft = indices[i]; + indexRight = indexLeft.zwxy; + + mask = makeMask_char(srcLeft, srcRight, indexLeft, indexRight) ^ imask10; + + if (((i >> stage) & 1) ^ dir) + { + srcLeft = (srcLeft & mask) | (srcRight & ~mask); + srcRight = srcLeft.yxwz; + indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + indexRight = indexLeft.yxwz; + + mask = makeMask_char(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; + + theArray[i] = (srcLeft & mask) | (srcRight & ~mask); + indices[i] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + } + else + { + srcLeft = (srcLeft & ~mask) | (srcRight & mask); + srcRight = srcLeft.yxwz; + indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + indexRight = indexLeft.yxwz; + + mask = makeMask_char(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; + + theArray[i] = (srcLeft & ~mask) | (srcRight & mask); + indices[i] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + } + } + else // first stage, sort inside one four + { + char4 imask0 = (char4)(0, -1, -1, 0); + + srcLeft = theArray[i]; + srcRight = srcLeft.yxwz; + indexLeft = indices[i]; + indexRight = indexLeft.yxwz; + + mask = makeMask_char(srcLeft, srcRight, indexLeft, indexRight) ^ imask0; + + if (dir) + { + srcLeft = (srcLeft & mask) | (srcRight & ~mask); + indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + } else + { + srcLeft = (srcLeft & ~mask) | (srcRight & mask); + indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + + srcRight = srcLeft.zwxy; + indexRight = indexLeft.zwxy; + mask = makeMask_char(srcLeft, srcRight, indexLeft, indexRight) ^ imask10; + + if ((i & 1) ^ dir) + { + srcLeft = (srcLeft & mask) | (srcRight & ~mask); + srcRight = srcLeft.yxwz; + indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + indexRight = indexLeft.yxwz; + + mask = makeMask_char(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; + + theArray[i] = (srcLeft & mask) | (srcRight & ~mask); + indices[i] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + } + else + { + srcLeft = (srcLeft & ~mask) | (srcRight & mask); + srcRight = srcLeft.yxwz; + indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + indexRight = indexLeft.yxwz; + + mask = makeMask_char(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; + + theArray[i] = (srcLeft & ~mask) | (srcRight & mask); + indices[i] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + } +} + + +__kernel void /*__attribute__((vec_type_hint(uchar4)))*/ bitonicSort_uchar(__global uchar4 * theArray, + __global uint4 * indices, + const uint stage, + const uint passOfStage, + const uint dir) +{ + size_t i = get_global_id(0); + uchar4 srcLeft, srcRight, mask; + uchar4 imask10 = (uchar4)(0, 0, -1, -1); + uchar4 imask11 = (uchar4)(0, -1, 0, -1); + uint4 indexLeft, indexRight; + + if (stage > 0) + { + if (passOfStage > 0) // upper level pass, exchange between two fours + { + size_t r = 1 << (passOfStage - 1); // length of arrow + size_t lmask = r - 1; + size_t left = ((i >> (passOfStage - 1)) << passOfStage) + (i & lmask); + size_t right = left + r; + + srcLeft = theArray[left]; + srcRight = theArray[right]; + indexLeft = indices[left]; + indexRight = indices[right]; + + mask = makeMask_uchar(srcLeft, srcRight, indexLeft, indexRight); + + uchar4 imin = (srcLeft & mask) | (srcRight & ~mask); + uchar4 imax = (srcLeft & ~mask) | (srcRight & mask); + + if (((i >> (stage - 1)) & 1) ^ dir) + { + theArray[left] = imin; + theArray[right] = imax; + + indices[left] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + indices[right] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + else + { + theArray[right] = imin; + theArray[left] = imax; + + indices[right] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + indices[left] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + } + else // last pass, sort inside one four + { + srcLeft = theArray[i]; + srcRight = srcLeft.zwxy; + indexLeft = indices[i]; + indexRight = indexLeft.zwxy; + + mask = makeMask_uchar(srcLeft, srcRight, indexLeft, indexRight) ^ imask10; + + if (((i >> stage) & 1) ^ dir) + { + srcLeft = (srcLeft & mask) | (srcRight & ~mask); + srcRight = srcLeft.yxwz; + indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + indexRight = indexLeft.yxwz; + + mask = makeMask_uchar(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; + + theArray[i] = (srcLeft & mask) | (srcRight & ~mask); + indices[i] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + } + else + { + srcLeft = (srcLeft & ~mask) | (srcRight & mask); + srcRight = srcLeft.yxwz; + indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + indexRight = indexLeft.yxwz; + + mask = makeMask_uchar(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; + + theArray[i] = (srcLeft & ~mask) | (srcRight & mask); + indices[i] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + } + } + else // first stage, sort inside one four + { + uchar4 imask0 = (uchar4)(0, -1, -1, 0); + + srcLeft = theArray[i]; + srcRight = srcLeft.yxwz; + indexLeft = indices[i]; + indexRight = indexLeft.yxwz; + + mask = makeMask_uchar(srcLeft, srcRight, indexLeft, indexRight) ^ imask0; + + if (dir) { + srcLeft = (srcLeft & mask) | (srcRight & ~mask); + indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + } else { + srcLeft = (srcLeft & ~mask) | (srcRight & mask); + indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + + srcRight = srcLeft.zwxy; + indexRight = indexLeft.zwxy; + mask = makeMask_uchar(srcLeft, srcRight, indexLeft, indexRight) ^ imask10; + + if ((i & 1) ^ dir) + { + srcLeft = (srcLeft & mask) | (srcRight & ~mask); + srcRight = srcLeft.yxwz; + indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + indexRight = indexLeft.yxwz; + + mask = makeMask_uchar(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; + + theArray[i] = (srcLeft & mask) | (srcRight & ~mask); + indices[i] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + } + else + { + srcLeft = (srcLeft & ~mask) | (srcRight & mask); + srcRight = srcLeft.yxwz; + indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + indexRight = indexLeft.yxwz; + + mask = makeMask_uchar(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; + + theArray[i] = (srcLeft & ~mask) | (srcRight & mask); + indices[i] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + } +} + + +__kernel void /*__attribute__((vec_type_hint(short4)))*/ bitonicSort_short(__global short4 * theArray, + __global uint4 * indices, + const uint stage, + const uint passOfStage, + const uint dir) +{ + size_t i = get_global_id(0); + short4 srcLeft, srcRight, mask; + short4 imask10 = (short4)(0, 0, -1, -1); + short4 imask11 = (short4)(0, -1, 0, -1); + uint4 indexLeft, indexRight; + + if (stage > 0) + { + if (passOfStage > 0) // upper level pass, exchange between two fours + { + size_t r = 1 << (passOfStage - 1); // length of arrow + size_t lmask = r - 1; + size_t left = ((i >> (passOfStage - 1)) << passOfStage) + (i & lmask); + size_t right = left + r; + + srcLeft = theArray[left]; + srcRight = theArray[right]; + indexLeft = indices[left]; + indexRight = indices[right]; + + mask = makeMask_short(srcLeft, srcRight, indexLeft, indexRight); + + short4 imin = (srcLeft & mask) | (srcRight & ~mask); + short4 imax = (srcLeft & ~mask) | (srcRight & mask); + + if (((i >> (stage - 1)) & 1) ^ dir) + { + theArray[left] = imin; + theArray[right] = imax; + + indices[left] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + indices[right] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + else + { + theArray[right] = imin; + theArray[left] = imax; + + indices[right] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + indices[left] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + } + else // last pass, sort inside one four + { + srcLeft = theArray[i]; + srcRight = srcLeft.zwxy; + indexLeft = indices[i]; + indexRight = indexLeft.zwxy; + + mask = makeMask_short(srcLeft, srcRight, indexLeft, indexRight) ^ imask10; + + if (((i >> stage) & 1) ^ dir) + { + srcLeft = (srcLeft & mask) | (srcRight & ~mask); + srcRight = srcLeft.yxwz; + indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + indexRight = indexLeft.yxwz; + + mask = makeMask_short(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; + + theArray[i] = (srcLeft & mask) | (srcRight & ~mask); + indices[i] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + } + else + { + srcLeft = (srcLeft & ~mask) | (srcRight & mask); + srcRight = srcLeft.yxwz; + indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + indexRight = indexLeft.yxwz; + + mask = makeMask_short(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; + + theArray[i] = (srcLeft & ~mask) | (srcRight & mask); + indices[i] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + } + } + else // first stage, sort inside one four + { + short4 imask0 = (short4)(0, -1, -1, 0); + + srcLeft = theArray[i]; + srcRight = srcLeft.yxwz; + indexLeft = indices[i]; + indexRight = indexLeft.yxwz; + + mask = makeMask_short(srcLeft, srcRight, indexLeft, indexRight) ^ imask0; + + if (dir) + { + srcLeft = (srcLeft & mask) | (srcRight & ~mask); + indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + } + else + { + srcLeft = (srcLeft & ~mask) | (srcRight & mask); + indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + + srcRight = srcLeft.zwxy; + indexRight = indexLeft.zwxy; + mask = makeMask_short(srcLeft, srcRight, indexLeft, indexRight) ^ imask10; + + if ((i & 1) ^ dir) + { + srcLeft = (srcLeft & mask) | (srcRight & ~mask); + srcRight = srcLeft.yxwz; + indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + indexRight = indexLeft.yxwz; + + mask = makeMask_short(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; + + theArray[i] = (srcLeft & mask) | (srcRight & ~mask); + indices[i] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + } + else + { + srcLeft = (srcLeft & ~mask) | (srcRight & mask); + srcRight = srcLeft.yxwz; + indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + indexRight = indexLeft.yxwz; + + mask = makeMask_short(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; + + theArray[i] = (srcLeft & ~mask) | (srcRight & mask); + indices[i] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + } +} + + +__kernel void /*__attribute__((vec_type_hint(ushort4)))*/ bitonicSort_ushort(__global ushort4 * theArray, + __global uint4 * indices, + const uint stage, + const uint passOfStage, + const uint dir) +{ + size_t i = get_global_id(0); + ushort4 srcLeft, srcRight, mask; + ushort4 imask10 = (ushort4)(0, 0, -1, -1); + ushort4 imask11 = (ushort4)(0, -1, 0, -1); + uint4 indexLeft, indexRight; + + if (stage > 0) + { + if (passOfStage > 0) // upper level pass, exchange between two fours + { + size_t r = 1 << (passOfStage - 1); // length of arrow + size_t lmask = r - 1; + size_t left = ((i >> (passOfStage - 1)) << passOfStage) + (i & lmask); + size_t right = left + r; + + srcLeft = theArray[left]; + srcRight = theArray[right]; + indexLeft = indices[left]; + indexRight = indices[right]; + + mask = makeMask_ushort(srcLeft, srcRight, indexLeft, indexRight); + + ushort4 imin = (srcLeft & mask) | (srcRight & ~mask); + ushort4 imax = (srcLeft & ~mask) | (srcRight & mask); + + if (((i >> (stage - 1)) & 1) ^ dir) + { + theArray[left] = imin; + theArray[right] = imax; + + indices[left] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + indices[right] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + else + { + theArray[right] = imin; + theArray[left] = imax; + + indices[right] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + indices[left] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + } + else // last pass, sort inside one four + { + srcLeft = theArray[i]; + srcRight = srcLeft.zwxy; + indexLeft = indices[i]; + indexRight = indexLeft.zwxy; + + mask = makeMask_ushort(srcLeft, srcRight, indexLeft, indexRight) ^ imask10; + + if (((i >> stage) & 1) ^ dir) + { + srcLeft = (srcLeft & mask) | (srcRight & ~mask); + srcRight = srcLeft.yxwz; + indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + indexRight = indexLeft.yxwz; + + mask = makeMask_ushort(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; + + theArray[i] = (srcLeft & mask) | (srcRight & ~mask); + indices[i] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + } + else + { + srcLeft = (srcLeft & ~mask) | (srcRight & mask); + srcRight = srcLeft.yxwz; + indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + indexRight = indexLeft.yxwz; + + mask = makeMask_ushort(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; + + theArray[i] = (srcLeft & ~mask) | (srcRight & mask); + indices[i] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + } + } + else // first stage, sort inside one four + { + ushort4 imask0 = (ushort4)(0, -1, -1, 0); + + srcLeft = theArray[i]; + srcRight = srcLeft.yxwz; + indexLeft = indices[i]; + indexRight = indexLeft.yxwz; + + mask = makeMask_ushort(srcLeft, srcRight, indexLeft, indexRight) ^ imask0; + + if (dir) { + srcLeft = (srcLeft & mask) | (srcRight & ~mask); + indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + } else { + srcLeft = (srcLeft & ~mask) | (srcRight & mask); + indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + + srcRight = srcLeft.zwxy; + indexRight = indexLeft.zwxy; + mask = makeMask_ushort(srcLeft, srcRight, indexLeft, indexRight) ^ imask10; + + if ((i & 1) ^ dir) + { + srcLeft = (srcLeft & mask) | (srcRight & ~mask); + srcRight = srcLeft.yxwz; + indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + indexRight = indexLeft.yxwz; + + mask = makeMask_ushort(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; + + theArray[i] = (srcLeft & mask) | (srcRight & ~mask); + indices[i] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + } + else + { + srcLeft = (srcLeft & ~mask) | (srcRight & mask); + srcRight = srcLeft.yxwz; + indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + indexRight = indexLeft.yxwz; + + mask = makeMask_ushort(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; + + theArray[i] = (srcLeft & ~mask) | (srcRight & mask); + indices[i] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + } +} + + +__kernel void /*__attribute__((vec_type_hint(int4)))*/ bitonicSort_int(__global int4 * theArray, + __global uint4 * indices, + const uint stage, + const uint passOfStage, + const uint dir) +{ + + size_t i = get_global_id(0); + int4 srcLeft, srcRight, mask; + int4 imask10 = (int4)(0, 0, -1, -1); + int4 imask11 = (int4)(0, -1, 0, -1); + uint4 indexLeft, indexRight; + + int random_id = (i * 5 + stage * 42 + passOfStage * 47 + dir * 88) % 100; + + if (stage > 0) + { + if (passOfStage > 0) // upper level pass, exchange between two fours + { + size_t r = 1 << (passOfStage - 1); // length of arrow + size_t lmask = r - 1; + size_t left = ((i >> (passOfStage - 1)) << passOfStage) + (i & lmask); + size_t right = left + r; + + srcLeft = theArray[left]; + srcRight = theArray[right]; + indexLeft = indices[left]; + indexRight = indices[right]; + + mask = makeMask_int(srcLeft, srcRight, indexLeft, indexRight); + + int4 imin = (srcLeft & mask) | (srcRight & ~mask); + int4 imax = (srcLeft & ~mask) | (srcRight & mask); + + if (((i >> (stage - 1)) & 1) ^ dir) + { + theArray[left] = imin; + theArray[right] = imax; + + indices[left] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + indices[right] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + else + { + theArray[right] = imin; + theArray[left] = imax; + + indices[right] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + indices[left] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + } + else // last pass, sort inside one four + { + srcLeft = theArray[i]; + srcRight = srcLeft.zwxy; + indexLeft = indices[i]; + indexRight = indexLeft.zwxy; + + mask = makeMask_int(srcLeft, srcRight, indexLeft, indexRight) ^ imask10; + + if (((i >> stage) & 1) ^ dir) + { + srcLeft = (srcLeft & mask) | (srcRight & ~mask); + srcRight = srcLeft.yxwz; + indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + indexRight = indexLeft.yxwz; + + mask = makeMask_int(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; + + theArray[i] = (srcLeft & mask) | (srcRight & ~mask); + indices[i] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + } + else + { + srcLeft = (srcLeft & ~mask) | (srcRight & mask); + srcRight = srcLeft.yxwz; + indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + indexRight = indexLeft.yxwz; + + mask = makeMask_int(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; + + theArray[i] = (srcLeft & ~mask) | (srcRight & mask); + indices[i] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + } + } + else // first stage, sort inside one four + { + int4 imask0 = (int4)(0, -1, -1, 0); + + srcLeft = theArray[i]; + srcRight = srcLeft.yxwz; + indexLeft = indices[i]; + indexRight = indexLeft.yxwz; + + mask = makeMask_int(srcLeft, srcRight, indexLeft, indexRight) ^ imask0; + + if (dir) + { + srcLeft = (srcLeft & mask) | (srcRight & ~mask); + indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + } + else + { + srcLeft = (srcLeft & ~mask) | (srcRight & mask); + indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + + srcRight = srcLeft.zwxy; + indexRight = indexLeft.zwxy; + mask = makeMask_int(srcLeft, srcRight, indexLeft, indexRight) ^ imask10; + + if ((i & 1) ^ dir) + { + srcLeft = (srcLeft & mask) | (srcRight & ~mask); + srcRight = srcLeft.yxwz; + indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + indexRight = indexLeft.yxwz; + + mask = makeMask_int(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; + + theArray[i] = (srcLeft & mask) | (srcRight & ~mask); + indices[i] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + } + else + { + srcLeft = (srcLeft & ~mask) | (srcRight & mask); + srcRight = srcLeft.yxwz; + indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + indexRight = indexLeft.yxwz; + + mask = makeMask_int(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; + + theArray[i] = (srcLeft & ~mask) | (srcRight & mask); + indices[i] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + } +} + + +__kernel void /*__attribute__((vec_type_hint(uint4)))*/ bitonicSort_uint(__global uint4 * theArray, + __global uint4 * indices, + const uint stage, + const uint passOfStage, + const uint dir) +{ + size_t i = get_global_id(0); + uint4 srcLeft, srcRight, mask; + uint4 imask10 = (uint4)(0, 0, -1, -1); + uint4 imask11 = (uint4)(0, -1, 0, -1); + uint4 indexLeft, indexRight; + + if (stage > 0) + { + if (passOfStage > 0) // upper level pass, exchange between two fours + { + size_t r = 1 << (passOfStage - 1); // length of arrow + size_t lmask = r - 1; + size_t left = ((i >> (passOfStage - 1)) << passOfStage) + (i & lmask); + size_t right = left + r; + + srcLeft = theArray[left]; + srcRight = theArray[right]; + indexLeft = indices[left]; + indexRight = indices[right]; + + mask = makeMask_uint(srcLeft, srcRight, indexLeft, indexRight); + + uint4 imin = (srcLeft & mask) | (srcRight & ~mask); + uint4 imax = (srcLeft & ~mask) | (srcRight & mask); + + if (((i >> (stage - 1)) & 1) ^ dir) + { + theArray[left] = imin; + theArray[right] = imax; + + indices[left] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + indices[right] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + else + { + theArray[right] = imin; + theArray[left] = imax; + + indices[right] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + indices[left] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + } + else // last pass, sort inside one four + { + srcLeft = theArray[i]; + srcRight = srcLeft.zwxy; + indexLeft = indices[i]; + indexRight = indexLeft.zwxy; + + mask = makeMask_uint(srcLeft, srcRight, indexLeft, indexRight) ^ imask10; + + if (((i >> stage) & 1) ^ dir) + { + srcLeft = (srcLeft & mask) | (srcRight & ~mask); + srcRight = srcLeft.yxwz; + indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + indexRight = indexLeft.yxwz; + + mask = makeMask_uint(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; + + theArray[i] = (srcLeft & mask) | (srcRight & ~mask); + indices[i] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + } + else + { + srcLeft = (srcLeft & ~mask) | (srcRight & mask); + srcRight = srcLeft.yxwz; + indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + indexRight = indexLeft.yxwz; + + mask = makeMask_uint(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; + + theArray[i] = (srcLeft & ~mask) | (srcRight & mask); + indices[i] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + } + } + else // first stage, sort inside one four + { + uint4 imask0 = (uint4)(0, -1, -1, 0); + + srcLeft = theArray[i]; + srcRight = srcLeft.yxwz; + indexLeft = indices[i]; + indexRight = indexLeft.yxwz; + + mask = makeMask_uint(srcLeft, srcRight, indexLeft, indexRight) ^ imask0; + + if (dir) + { + srcLeft = (srcLeft & mask) | (srcRight & ~mask); + indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + } + else + { + srcLeft = (srcLeft & ~mask) | (srcRight & mask); + indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + + srcRight = srcLeft.zwxy; + indexRight = indexLeft.zwxy; + mask = makeMask_uint(srcLeft, srcRight, indexLeft, indexRight) ^ imask10; + + if ((i & 1) ^ dir) + { + srcLeft = (srcLeft & mask) | (srcRight & ~mask); + srcRight = srcLeft.yxwz; + indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + indexRight = indexLeft.yxwz; + + mask = makeMask_uint(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; + + theArray[i] = (srcLeft & mask) | (srcRight & ~mask); + indices[i] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + } + else + { + srcLeft = (srcLeft & ~mask) | (srcRight & mask); + srcRight = srcLeft.yxwz; + indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + indexRight = indexLeft.yxwz; + + mask = makeMask_uint(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; + + theArray[i] = (srcLeft & ~mask) | (srcRight & mask); + indices[i] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + } +} + + +__kernel void /*__attribute__((vec_type_hint(long4)))*/ bitonicSort_long(__global long4 * theArray, + __global uint4 * indices, + const uint stage, + const uint passOfStage, + const uint dir) +{ + size_t i = get_global_id(0); + long4 srcLeft, srcRight, mask; + long4 imask10 = (long4)(0, 0, -1, -1); + long4 imask11 = (long4)(0, -1, 0, -1); + uint4 indexLeft, indexRight; + + if (stage > 0) + { + if (passOfStage > 0) // upper level pass, exchange between two fours + { + size_t r = 1 << (passOfStage - 1); // length of arrow + size_t lmask = r - 1; + size_t left = ((i >> (passOfStage - 1)) << passOfStage) + (i & lmask); + size_t right = left + r; + + srcLeft = theArray[left]; + srcRight = theArray[right]; + indexLeft = indices[left]; + indexRight = indices[right]; + + mask = makeMask_long(srcLeft, srcRight, indexLeft, indexRight); + + long4 imin = (srcLeft & mask) | (srcRight & ~mask); + long4 imax = (srcLeft & ~mask) | (srcRight & mask); + + if (((i >> (stage - 1)) & 1) ^ dir) + { + theArray[left] = imin; + theArray[right] = imax; + + indices[left] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + indices[right] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + else + { + theArray[right] = imin; + theArray[left] = imax; + + indices[right] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + indices[left] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + } + else // last pass, sort inside one four + { + srcLeft = theArray[i]; + srcRight = srcLeft.zwxy; + indexLeft = indices[i]; + indexRight = indexLeft.zwxy; + + mask = makeMask_long(srcLeft, srcRight, indexLeft, indexRight) ^ imask10; + + if (((i >> stage) & 1) ^ dir) + { + srcLeft = (srcLeft & mask) | (srcRight & ~mask); + srcRight = srcLeft.yxwz; + indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + indexRight = indexLeft.yxwz; + + mask = makeMask_long(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; + + theArray[i] = (srcLeft & mask) | (srcRight & ~mask); + indices[i] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + } + else + { + srcLeft = (srcLeft & ~mask) | (srcRight & mask); + srcRight = srcLeft.yxwz; + indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + indexRight = indexLeft.yxwz; + + mask = makeMask_long(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; + + theArray[i] = (srcLeft & ~mask) | (srcRight & mask); + indices[i] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + } + } + else // first stage, sort inside one four + { + long4 imask0 = (long4)(0, -1, -1, 0); + + srcLeft = theArray[i]; + srcRight = srcLeft.yxwz; + indexLeft = indices[i]; + indexRight = indexLeft.yxwz; + + mask = makeMask_long(srcLeft, srcRight, indexLeft, indexRight) ^ imask0; + + if (dir) + { + srcLeft = (srcLeft & mask) | (srcRight & ~mask); + indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + } + else + { + srcLeft = (srcLeft & ~mask) | (srcRight & mask); + indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + + srcRight = srcLeft.zwxy; + indexRight = indexLeft.zwxy; + mask = makeMask_long(srcLeft, srcRight, indexLeft, indexRight) ^ imask10; + + if ((i & 1) ^ dir) + { + srcLeft = (srcLeft & mask) | (srcRight & ~mask); + srcRight = srcLeft.yxwz; + indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + indexRight = indexLeft.yxwz; + + mask = makeMask_long(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; + + theArray[i] = (srcLeft & mask) | (srcRight & ~mask); + indices[i] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + } + else + { + srcLeft = (srcLeft & ~mask) | (srcRight & mask); + srcRight = srcLeft.yxwz; + indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + indexRight = indexLeft.yxwz; + + mask = makeMask_long(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; + + theArray[i] = (srcLeft & ~mask) | (srcRight & mask); + indices[i] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + } +} + + +__kernel void /*__attribute__((vec_type_hint(ulong4)))*/ bitonicSort_ulong(__global ulong4 * theArray, + __global uint4 * indices, + const uint stage, + const uint passOfStage, + const uint dir) +{ + size_t i = get_global_id(0); + ulong4 srcLeft, srcRight, mask; + ulong4 imask10 = (ulong4)(0, 0, -1, -1); + ulong4 imask11 = (ulong4)(0, -1, 0, -1); + uint4 indexLeft, indexRight; + + if (stage > 0) + { + if (passOfStage > 0) // upper level pass, exchange between two fours + { + size_t r = 1 << (passOfStage - 1); // length of arrow + size_t lmask = r - 1; + size_t left = ((i >> (passOfStage - 1)) << passOfStage) + (i & lmask); + size_t right = left + r; + + srcLeft = theArray[left]; + srcRight = theArray[right]; + indexLeft = indices[left]; + indexRight = indices[right]; + + mask = makeMask_ulong(srcLeft, srcRight, indexLeft, indexRight); + + ulong4 imin = (srcLeft & mask) | (srcRight & ~mask); + ulong4 imax = (srcLeft & ~mask) | (srcRight & mask); + + if (((i >> (stage - 1)) & 1) ^ dir) + { + theArray[left] = imin; + theArray[right] = imax; + + indices[left] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + indices[right] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + else + { + theArray[right] = imin; + theArray[left] = imax; + + indices[right] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + indices[left] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + } + else // last pass, sort inside one four + { + srcLeft = theArray[i]; + srcRight = srcLeft.zwxy; + indexLeft = indices[i]; + indexRight = indexLeft.zwxy; + + mask = makeMask_ulong(srcLeft, srcRight, indexLeft, indexRight) ^ imask10; + + if (((i >> stage) & 1) ^ dir) + { + srcLeft = (srcLeft & mask) | (srcRight & ~mask); + srcRight = srcLeft.yxwz; + indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + indexRight = indexLeft.yxwz; + + mask = makeMask_ulong(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; + + theArray[i] = (srcLeft & mask) | (srcRight & ~mask); + indices[i] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + } + else + { + srcLeft = (srcLeft & ~mask) | (srcRight & mask); + srcRight = srcLeft.yxwz; + indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + indexRight = indexLeft.yxwz; + + mask = makeMask_ulong(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; + + theArray[i] = (srcLeft & ~mask) | (srcRight & mask); + indices[i] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + } + } + else // first stage, sort inside one four + { + ulong4 imask0 = (ulong4)(0, -1, -1, 0); + + srcLeft = theArray[i]; + srcRight = srcLeft.yxwz; + indexLeft = indices[i]; + indexRight = indexLeft.yxwz; + + mask = makeMask_ulong(srcLeft, srcRight, indexLeft, indexRight) ^ imask0; + + if (dir) + { + srcLeft = (srcLeft & mask) | (srcRight & ~mask); + indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + } + else + { + srcLeft = (srcLeft & ~mask) | (srcRight & mask); + indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + + srcRight = srcLeft.zwxy; + indexRight = indexLeft.zwxy; + mask = makeMask_ulong(srcLeft, srcRight, indexLeft, indexRight) ^ imask10; + + if ((i & 1) ^ dir) + { + srcLeft = (srcLeft & mask) | (srcRight & ~mask); + srcRight = srcLeft.yxwz; + indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + indexRight = indexLeft.yxwz; + + mask = makeMask_ulong(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; + + theArray[i] = (srcLeft & mask) | (srcRight & ~mask); + indices[i] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); + } + else + { + srcLeft = (srcLeft & ~mask) | (srcRight & mask); + srcRight = srcLeft.yxwz; + indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + indexRight = indexLeft.yxwz; + + mask = makeMask_ulong(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; + + theArray[i] = (srcLeft & ~mask) | (srcRight & mask); + indices[i] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); + } + } +} + +)"; diff --git a/src/Common/config.h.in b/src/Common/config.h.in index a1ffdb2101d..df2359c1c29 100644 --- a/src/Common/config.h.in +++ b/src/Common/config.h.in @@ -8,4 +8,5 @@ #cmakedefine01 USE_AWS_S3 #cmakedefine01 USE_BROTLI #cmakedefine01 USE_UNWIND +#cmakedefine01 USE_OPENCL #cmakedefine01 CLICKHOUSE_SPLIT_BINARY diff --git a/src/Common/oclBasics.cpp b/src/Common/oclBasics.cpp new file mode 100644 index 00000000000..cd90cd945c1 --- /dev/null +++ b/src/Common/oclBasics.cpp @@ -0,0 +1,363 @@ +#include +#if USE_OPENCL + +#if !defined(__APPLE__) && !defined(__FreeBSD__) +#include +#endif + +#ifdef __APPLE__ +#include +#else +#include +#endif + +#include +#include +#include + +#ifndef CL_VERSION_2_0 +#define CL_USE_DEPRECATED_OPENCL_1_2_APIS +#endif + + +using KernelType = std::remove_reference::type; + + +namespace DB +{ + namespace ErrorCodes + { + extern const int OPENCL_ERROR; + } +} + +struct OCL +{ + + /** + * Structure which represents the most essential settings of common OpenCl entities. + */ + struct Settings + { + // Platform info + cl_uint number_of_platform_entries; + cl_uint * number_of_available_platforms; + + // Devices info + cl_uint number_of_devices_entries; + cl_uint * number_of_available_devices; + + // Context settings + cl_context_properties * context_properties; + + void (* context_callback)(const char *, const void *, size_t, void *); + + void * context_callback_data; + + // Command queue settings + cl_command_queue_properties command_queue_properties; + + // Build settings + cl_uint number_of_program_source_pointers; + + void (* build_notification_routine)(cl_program, void *user_data); + + void * build_callback_data; + char * build_options; + + Settings(cl_uint number_of_platform_entries_, + cl_uint * number_of_available_platforms_, + cl_uint number_of_devices_entries_, + cl_uint * number_of_available_devices_, + cl_uint number_of_program_source_pointers_, + cl_command_queue_properties command_queue_properties_, + cl_context_properties * context_properties_ = nullptr, + void * context_data_callback_ = nullptr, + void (* context_callback_)(const char *, const void *, size_t, void *) = nullptr, + void (* build_notification_routine_)(cl_program, void * user_data) = nullptr, + void * build_callback_data_ = nullptr, + char * build_options_ = nullptr) + { + this->number_of_platform_entries = number_of_platform_entries_; + this->number_of_available_platforms = number_of_available_platforms_; + this->number_of_devices_entries = number_of_devices_entries_; + this->number_of_available_devices = number_of_available_devices_; + this->number_of_program_source_pointers = number_of_program_source_pointers_; + this->command_queue_properties = command_queue_properties_; + this->context_properties = context_properties_; + this->context_callback = context_callback_; + this->context_callback_data = context_data_callback_; + this->build_notification_routine = build_notification_routine_; + this->build_callback_data = build_callback_data_; + this->build_options = build_options_; + } + }; + + + /** + * Configuration with already created OpenCl common entities. + */ + class Configuration + { + public: + + Configuration(cl_device_id device, cl_context gpu_context, + cl_command_queue command_queue, cl_program program) + { + this->device_ = device; + this->gpu_context_ = std::shared_ptr(gpu_context, clReleaseContext); + this->command_queue_ = std::shared_ptr(command_queue, clReleaseCommandQueue); + this->program_ = std::shared_ptr(program, clReleaseProgram); + } + + cl_device_id device() { return device_; } + + cl_context context() { return gpu_context_.get(); } + + cl_command_queue commandQueue() { return command_queue_.get(); } + + cl_program program() { return program_.get(); } + + private: + + using ProgramType = std::remove_reference::type; + using CommandQueueType = std::remove_reference::type; + using ContextType = std::remove_reference::type; + + cl_device_id device_; + + std::shared_ptr gpu_context_; + std::shared_ptr command_queue_; + std::shared_ptr program_; + }; + + + static String opencl_error_to_str(cl_int error) + { +#define CASE_CL_CONSTANT(NAME) case NAME: return #NAME; + + // Suppose that no combinations are possible. + switch (error) + { + CASE_CL_CONSTANT(CL_SUCCESS) + CASE_CL_CONSTANT(CL_DEVICE_NOT_FOUND) + CASE_CL_CONSTANT(CL_DEVICE_NOT_AVAILABLE) + CASE_CL_CONSTANT(CL_COMPILER_NOT_AVAILABLE) + CASE_CL_CONSTANT(CL_MEM_OBJECT_ALLOCATION_FAILURE) + CASE_CL_CONSTANT(CL_OUT_OF_RESOURCES) + CASE_CL_CONSTANT(CL_OUT_OF_HOST_MEMORY) + CASE_CL_CONSTANT(CL_PROFILING_INFO_NOT_AVAILABLE) + CASE_CL_CONSTANT(CL_MEM_COPY_OVERLAP) + CASE_CL_CONSTANT(CL_IMAGE_FORMAT_MISMATCH) + CASE_CL_CONSTANT(CL_IMAGE_FORMAT_NOT_SUPPORTED) + CASE_CL_CONSTANT(CL_BUILD_PROGRAM_FAILURE) + CASE_CL_CONSTANT(CL_MAP_FAILURE) + CASE_CL_CONSTANT(CL_MISALIGNED_SUB_BUFFER_OFFSET) + CASE_CL_CONSTANT(CL_EXEC_STATUS_ERROR_FOR_EVENTS_IN_WAIT_LIST) + CASE_CL_CONSTANT(CL_COMPILE_PROGRAM_FAILURE) + CASE_CL_CONSTANT(CL_LINKER_NOT_AVAILABLE) + CASE_CL_CONSTANT(CL_LINK_PROGRAM_FAILURE) + CASE_CL_CONSTANT(CL_DEVICE_PARTITION_FAILED) + CASE_CL_CONSTANT(CL_KERNEL_ARG_INFO_NOT_AVAILABLE) + CASE_CL_CONSTANT(CL_INVALID_VALUE) + CASE_CL_CONSTANT(CL_INVALID_DEVICE_TYPE) + CASE_CL_CONSTANT(CL_INVALID_PLATFORM) + CASE_CL_CONSTANT(CL_INVALID_DEVICE) + CASE_CL_CONSTANT(CL_INVALID_CONTEXT) + CASE_CL_CONSTANT(CL_INVALID_QUEUE_PROPERTIES) + CASE_CL_CONSTANT(CL_INVALID_COMMAND_QUEUE) + CASE_CL_CONSTANT(CL_INVALID_HOST_PTR) + CASE_CL_CONSTANT(CL_INVALID_MEM_OBJECT) + CASE_CL_CONSTANT(CL_INVALID_IMAGE_FORMAT_DESCRIPTOR) + CASE_CL_CONSTANT(CL_INVALID_IMAGE_SIZE) + CASE_CL_CONSTANT(CL_INVALID_SAMPLER) + CASE_CL_CONSTANT(CL_INVALID_BINARY) + CASE_CL_CONSTANT(CL_INVALID_BUILD_OPTIONS) + CASE_CL_CONSTANT(CL_INVALID_PROGRAM) + CASE_CL_CONSTANT(CL_INVALID_PROGRAM_EXECUTABLE) + CASE_CL_CONSTANT(CL_INVALID_KERNEL_NAME) + CASE_CL_CONSTANT(CL_INVALID_KERNEL_DEFINITION) + CASE_CL_CONSTANT(CL_INVALID_KERNEL) + CASE_CL_CONSTANT(CL_INVALID_ARG_INDEX) + CASE_CL_CONSTANT(CL_INVALID_ARG_VALUE) + CASE_CL_CONSTANT(CL_INVALID_ARG_SIZE) + CASE_CL_CONSTANT(CL_INVALID_KERNEL_ARGS) + CASE_CL_CONSTANT(CL_INVALID_WORK_DIMENSION) + CASE_CL_CONSTANT(CL_INVALID_WORK_GROUP_SIZE) + CASE_CL_CONSTANT(CL_INVALID_WORK_ITEM_SIZE) + CASE_CL_CONSTANT(CL_INVALID_GLOBAL_OFFSET) + CASE_CL_CONSTANT(CL_INVALID_EVENT_WAIT_LIST) + CASE_CL_CONSTANT(CL_INVALID_EVENT) + CASE_CL_CONSTANT(CL_INVALID_OPERATION) + CASE_CL_CONSTANT(CL_INVALID_GL_OBJECT) + CASE_CL_CONSTANT(CL_INVALID_BUFFER_SIZE) + CASE_CL_CONSTANT(CL_INVALID_MIP_LEVEL) + CASE_CL_CONSTANT(CL_INVALID_GLOBAL_WORK_SIZE) + CASE_CL_CONSTANT(CL_INVALID_PROPERTY) + CASE_CL_CONSTANT(CL_INVALID_IMAGE_DESCRIPTOR) + CASE_CL_CONSTANT(CL_INVALID_COMPILER_OPTIONS) + CASE_CL_CONSTANT(CL_INVALID_LINKER_OPTIONS) + CASE_CL_CONSTANT(CL_INVALID_DEVICE_PARTITION_COUNT) + default: + return "UNKNOWN ERROR CODE "; + } + +#undef CASE_CL_CONSTANT + } + + + static void checkError(cl_int error) + { + if (error != CL_SUCCESS) + throw DB::Exception("OpenCL error " + opencl_error_to_str(error), DB::ErrorCodes::OPENCL_ERROR); + } + + + /// Getting OpenCl main entities. + + static cl_platform_id getPlatformID(const Settings & settings) + { + cl_platform_id platform; + cl_int error = clGetPlatformIDs(settings.number_of_platform_entries, &platform, + settings.number_of_available_platforms); + checkError(error); + + return platform; + } + + + static cl_device_id getDeviceID(cl_platform_id & platform, const Settings & settings) + { + cl_device_id device; + cl_int error = clGetDeviceIDs(platform, CL_DEVICE_TYPE_GPU, settings.number_of_devices_entries, + &device, settings.number_of_available_devices); + OCL::checkError(error); + + return device; + } + + + static cl_context makeContext(cl_device_id & device, const Settings & settings) + { + cl_int error; + cl_context gpu_context = clCreateContext(settings.context_properties, settings.number_of_devices_entries, + &device, settings.context_callback, settings.context_callback_data, + &error); + OCL::checkError(error); + + return gpu_context; + } + + + static cl_command_queue makeCommandQueue(cl_device_id & device, cl_context & context, const Settings & settings [[maybe_unused]]) + { + cl_int error; +#ifdef CL_USE_DEPRECATED_OPENCL_1_2_APIS + cl_command_queue command_queue = clCreateCommandQueue(context, device, settings.command_queue_properties, &error); +#else + cl_command_queue command_queue = clCreateCommandQueueWithProperties(context, device, nullptr, &error); +#endif + OCL::checkError(error); + + return command_queue; + } + + + static cl_program makeProgram(const char * source_code, cl_context context, + cl_device_id device_id, const Settings & settings) + { + cl_int error = 0; + size_t source_size = strlen(source_code); + + cl_program program = clCreateProgramWithSource(context, settings.number_of_program_source_pointers, &source_code, &source_size, &error); + checkError(error); + + error = clBuildProgram(program, settings.number_of_devices_entries, &device_id, settings.build_options, + settings.build_notification_routine, settings.build_callback_data); + + /// Combining additional logs output when program build failed. + if (error == CL_BUILD_PROGRAM_FAILURE) + { + size_t log_size; + error = clGetProgramBuildInfo(program, device_id, CL_PROGRAM_BUILD_LOG, 0, nullptr, &log_size); + + checkError(error); + + std::vector log(log_size); + clGetProgramBuildInfo(program, device_id, CL_PROGRAM_BUILD_LOG, log_size, log.data(), nullptr); + + checkError(error); + throw DB::Exception(log.data(), DB::ErrorCodes::OPENCL_ERROR); + } + + checkError(error); + + return program; + } + + + /// Configuring buffer for given input data + + template + static cl_mem createBuffer(K * p_input, cl_int array_size, cl_context context, + cl_int elements_size = sizeof(K)) + { + cl_int error = CL_SUCCESS; + cl_mem cl_input_buffer = + clCreateBuffer + ( + context, + CL_MEM_USE_HOST_PTR, + zeroCopySizeAlignment(elements_size * array_size), + p_input, + &error + ); + checkError(error); + + return cl_input_buffer; + } + + + static size_t zeroCopySizeAlignment(size_t required_size) + { + return required_size + (~required_size + 1) % 64; + } + + + /// Manipulating with common OpenCL variables. + + static void finishCommandQueue(cl_command_queue command_queue) + { + // Blocks until all previously queued OpenCL commands in a queue are issued to the associated device. + cl_int error = clFinish(command_queue); + OCL::checkError(error); + } + + + template + static void releaseData(T * origin, cl_int array_size, cl_mem cl_buffer, + cl_command_queue command_queue, size_t offset = 0) + { + cl_int error = CL_SUCCESS; + + void * tmp_ptr = nullptr; + + // No events specified to be completed before enqueueing buffers, + // so `num_events_in_wait_list` passed with `0` value. + + tmp_ptr = clEnqueueMapBuffer(command_queue, cl_buffer, true, CL_MAP_READ, + offset, sizeof(cl_int) * array_size, 0, nullptr, nullptr, &error); + OCL::checkError(error); + if (tmp_ptr != origin) + throw DB::Exception("clEnqueueMapBuffer failed to return original pointer", DB::ErrorCodes::OPENCL_ERROR); + + error = clEnqueueUnmapMemObject(command_queue, cl_buffer, tmp_ptr, 0, nullptr, nullptr); + checkError(error); + + error = clReleaseMemObject(cl_buffer); + checkError(error); + } + +}; + +#endif diff --git a/src/Common/tests/CMakeLists.txt b/src/Common/tests/CMakeLists.txt index 44d43ada300..87ae4858bf1 100644 --- a/src/Common/tests/CMakeLists.txt +++ b/src/Common/tests/CMakeLists.txt @@ -35,6 +35,9 @@ target_link_libraries (compact_array PRIVATE clickhouse_common_io) add_executable (radix_sort radix_sort.cpp) target_link_libraries (radix_sort PRIVATE clickhouse_common_io) +add_executable (bitonic_sort bitonic_sort.cpp) +target_link_libraries (bitonic_sort PRIVATE clickhouse_common_io "-framework OpenCL") + add_executable (arena_with_free_lists arena_with_free_lists.cpp) target_link_libraries (arena_with_free_lists PRIVATE dbms) diff --git a/src/Common/tests/bitonic_sort.cpp b/src/Common/tests/bitonic_sort.cpp new file mode 100644 index 00000000000..adaef94ed4c --- /dev/null +++ b/src/Common/tests/bitonic_sort.cpp @@ -0,0 +1,248 @@ +#include +#include + +#if USE_OPENCL + +#if !defined(__APPLE__) && !defined(__FreeBSD__) +#include +#endif +#include +#include +#include +#include +#include +#include + +#include "Common/BitonicSort.h" + + +using Key = cl_ulong; + + +/// Generates vector of size 8 for testing. +/// Vector contains max possible value, min possible value and duplicate values. +template +static void generateTest(std::vector& data, Type min_value, Type max_value) +{ + int size = 10; + + data.resize(size); + data[0] = 10; + data[1] = max_value; + data[2] = 10; + data[3] = 20; + data[4] = min_value; + data[5] = min_value + 1; + data[6] = max_value - 5; + data[7] = 1; + data[8] = 42; + data[9] = max_value - 1; +} + + +static void check(const std::vector & indices, bool reverse = true) +{ + std::vector reference_indices{4, 5, 7, 0, 2, 3, 8, 6, 9, 1}; + if (reverse) std::reverse(reference_indices.begin(), reference_indices.end()); + + bool success = true; + for (size_t index = 0; index < reference_indices.size(); ++index) + { + if (indices[index] != reference_indices[index]) + { + success = false; + std::cerr << "Test failed. Reason: indices[" << index << "] = " + << indices[index] << ", it must be equal to " << reference_indices[index] << "\n"; + } + } + + std::string order_description = reverse ? "descending" : "ascending"; + std::cerr << "Sorted " << order_description << " sequence. Result: " << (success ? "Ok." : "Fail!") << "\n"; +} + + +template +static void sortBitonicSortWithPodArrays(const std::vector& data, + std::vector & indices, bool ascending = true) +{ + DB::PaddedPODArray pod_array_data = DB::PaddedPODArray(data.size()); + DB::IColumn::Permutation pod_array_indices = DB::IColumn::Permutation(data.size()); + + for (size_t index = 0; index < data.size(); ++index) + { + *(pod_array_data.data() + index) = data[index]; + *(pod_array_indices.data() + index) = index; + } + + BitonicSort::getInstance().configure(); + BitonicSort::getInstance().sort(pod_array_data, pod_array_indices, ascending); + + for (size_t index = 0; index < data.size(); ++index) + indices[index] = pod_array_indices[index]; +} + + +template +static void testBitonicSort(std::string test_name, Type min_value, Type max_value) +{ + std::cerr << test_name << std::endl; + + std::vector data; + generateTest(data, min_value, max_value); + + std::vector indices(data.size()); + + sortBitonicSortWithPodArrays(data, indices, true); + check(indices, false); + + sortBitonicSortWithPodArrays(data, indices, false); + check(indices, true); +} + + +static void straightforwardTests() +{ + testBitonicSort("Test 01: cl_char.", CHAR_MIN, CHAR_MAX); + testBitonicSort("Test 02: cl_uchar.", 0, UCHAR_MAX); + testBitonicSort("Test 03: cl_short.", SHRT_MIN, SHRT_MAX); + testBitonicSort("Test 04: cl_ushort.", 0, USHRT_MAX); + testBitonicSort("Test 05: cl_int.", INT_MIN, INT_MAX); + testBitonicSort("Test 06: cl_uint.", 0, UINT_MAX); + testBitonicSort("Test 07: cl_long.", LONG_MIN, LONG_MAX); + testBitonicSort("Test 08: cl_ulong.", 0, ULONG_MAX); +} + + +static void NO_INLINE sort1(Key * data, size_t size) +{ + std::sort(data, data + size); +} + + +static void NO_INLINE sort2(std::vector & data, std::vector & indices) +{ + BitonicSort::getInstance().configure(); + + sortBitonicSortWithPodArrays(data, indices); + + std::vector result(data.size()); + for (size_t index = 0; index < data.size(); ++index) + result[index] = data[indices[index]]; + + data = std::move(result); +} + + +int main(int argc, char ** argv) +{ + straightforwardTests(); + + if (argc < 3) + { + std::cerr << "Not enough arguments were passed\n"; + return 1; + } + + size_t n = DB::parse(argv[1]); + size_t method = DB::parse(argv[2]); + + std::vector data(n); + std::vector indices(n); + + { + Stopwatch watch; + + for (auto & elem : data) + elem = static_cast(rand()); + + for (size_t i = 0; i < n; ++i) + indices[i] = i; + + watch.stop(); + double elapsed = watch.elapsedSeconds(); + std::cerr + << "Filled in " << elapsed + << " (" << n / elapsed << " elem/sec., " + << n * sizeof(Key) / elapsed / 1048576 << " MB/sec.)" + << std::endl; + } + + if (n <= 100) + { + std::cerr << std::endl; + for (const auto & elem : data) + std::cerr << elem << ' '; + std::cerr << std::endl; + for (const auto & index : indices) + std::cerr << index << ' '; + std::cerr << std::endl; + } + + { + Stopwatch watch; + + if (method == 1) sort1(data.data(), n); + if (method == 2) sort2(data, indices); + + watch.stop(); + double elapsed = watch.elapsedSeconds(); + std::cerr + << "Sorted in " << elapsed + << " (" << n / elapsed << " elem/sec., " + << n * sizeof(Key) / elapsed / 1048576 << " MB/sec.)" + << std::endl; + } + + { + Stopwatch watch; + + size_t i = 1; + while (i < n) + { + if (!(data[i - 1] <= data[i])) + break; + ++i; + } + + watch.stop(); + double elapsed = watch.elapsedSeconds(); + std::cerr + << "Checked in " << elapsed + << " (" << n / elapsed << " elem/sec., " + << n * sizeof(Key) / elapsed / 1048576 << " MB/sec.)" + << std::endl + << "Result: " << (i == n ? "Ok." : "Fail!") << std::endl; + } + + if (n <= 1000) + { + std::cerr << std::endl; + + std::cerr << data[0] << ' '; + for (size_t i = 1; i < n; ++i) + { + if (!(data[i - 1] <= data[i])) + std::cerr << "*** "; + std::cerr << data[i] << ' '; + } + + std::cerr << std::endl; + + for (const auto & index : indices) + std::cerr << index << ' '; + std::cerr << std::endl; + } + + return 0; +} + +#else + +int main() +{ + std::cerr << "Openc CL disabled."; + + return 0; +} + +#endif diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 699a5a7f7d9..ff151e24a99 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -111,6 +111,8 @@ struct Settings : public SettingsCollection M(SettingUInt64, parallel_replicas_count, 0, "", 0) \ M(SettingUInt64, parallel_replica_offset, 0, "", 0) \ \ + M(SettingSpecialSort, special_sort, SpecialSort::NOT_SPECIFIED, "Specifies a sorting algorithm which will be using in ORDER BY query.", 0) \ + \ M(SettingBool, skip_unavailable_shards, false, "If 1, ClickHouse silently skips unavailable shards and nodes unresolvable through DNS. Shard is marked as unavailable when none of the replicas can be reached.", 0) \ \ M(SettingBool, distributed_group_by_no_merge, false, "Do not merge aggregation states from different servers for distributed query processing - in case it is for certain that there are different keys on different shards.", 0) \ diff --git a/src/Core/SettingsCollection.cpp b/src/Core/SettingsCollection.cpp index 4223ab6ea42..364b90e8ce8 100644 --- a/src/Core/SettingsCollection.cpp +++ b/src/Core/SettingsCollection.cpp @@ -485,6 +485,12 @@ void SettingURI::deserialize(ReadBuffer & buf, SettingsBinaryFormat) IMPLEMENT_SETTING_ENUM(LoadBalancing, LOAD_BALANCING_LIST_OF_NAMES, ErrorCodes::UNKNOWN_LOAD_BALANCING) +#define SPECIAL_SORT_ALGORITHM_NAMES(M) \ + M(NOT_SPECIFIED, "not_specified") \ + M(OPENCL_BITONIC, "opencl_bitonic") +IMPLEMENT_SETTING_ENUM(SpecialSort, SPECIAL_SORT_ALGORITHM_NAMES, ErrorCodes::UNKNOWN_JOIN) + + #define JOIN_STRICTNESS_LIST_OF_NAMES(M) \ M(Unspecified, "") \ M(ALL, "ALL") \ diff --git a/src/Core/SettingsCollection.h b/src/Core/SettingsCollection.h index c6744b30aa6..1fe5762de4c 100644 --- a/src/Core/SettingsCollection.h +++ b/src/Core/SettingsCollection.h @@ -251,6 +251,15 @@ enum class JoinAlgorithm }; using SettingJoinAlgorithm = SettingEnum; + +enum class SpecialSort +{ + NOT_SPECIFIED = 0, + OPENCL_BITONIC, +}; +using SettingSpecialSort = SettingEnum; + + /// Which rows should be included in TOTALS. enum class TotalsMode { diff --git a/src/Core/SortDescription.h b/src/Core/SortDescription.h index e1ec142f645..6cc957cac55 100644 --- a/src/Core/SortDescription.h +++ b/src/Core/SortDescription.h @@ -5,6 +5,7 @@ #include #include #include +#include class Collator; @@ -31,21 +32,22 @@ struct SortColumnDescription std::shared_ptr collator; /// Collator for locale-specific comparison of strings bool with_fill; FillColumnDescription fill_description; + SpecialSort special_sort; SortColumnDescription( size_t column_number_, int direction_, int nulls_direction_, - const std::shared_ptr & collator_ = nullptr, bool with_fill_ = false, - const FillColumnDescription & fill_description_ = {}) + const std::shared_ptr & collator_ = nullptr, SpecialSort special_sort_ = SpecialSort::NOT_SPECIFIED, + bool with_fill_ = false, const FillColumnDescription & fill_description_ = {}) : column_number(column_number_), direction(direction_), nulls_direction(nulls_direction_), collator(collator_) - , with_fill(with_fill_), fill_description(fill_description_) {} + , with_fill(with_fill_), fill_description(fill_description_), special_sort(special_sort_) {} SortColumnDescription( const std::string & column_name_, int direction_, int nulls_direction_, - const std::shared_ptr & collator_ = nullptr, bool with_fill_ = false, - const FillColumnDescription & fill_description_ = {}) + const std::shared_ptr & collator_ = nullptr, SpecialSort special_sort_ = SpecialSort::NOT_SPECIFIED, + bool with_fill_ = false, const FillColumnDescription & fill_description_ = {}) : column_name(column_name_), column_number(0), direction(direction_), nulls_direction(nulls_direction_) - , collator(collator_), with_fill(with_fill_), fill_description(fill_description_) {} + , collator(collator_), with_fill(with_fill_), fill_description(fill_description_), special_sort(special_sort_) {} bool operator == (const SortColumnDescription & other) const { diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 7a15f71e180..f09b0fad8b7 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -633,6 +633,7 @@ static SortDescription getSortDescription(const ASTSelectQuery & query, const Co { SortDescription order_descr; order_descr.reserve(query.orderBy()->children.size()); + SpecialSort special_sort = context.getSettings().special_sort.value; for (const auto & elem : query.orderBy()->children) { String name = elem->children.front()->getColumnName(); @@ -646,10 +647,10 @@ static SortDescription getSortDescription(const ASTSelectQuery & query, const Co { FillColumnDescription fill_desc = getWithFillDescription(order_by_elem, context); order_descr.emplace_back(name, order_by_elem.direction, - order_by_elem.nulls_direction, collator, true, fill_desc); + order_by_elem.nulls_direction, collator, special_sort, true, fill_desc); } else - order_descr.emplace_back(name, order_by_elem.direction, order_by_elem.nulls_direction, collator); + order_descr.emplace_back(name, order_by_elem.direction, order_by_elem.nulls_direction, collator, special_sort); } return order_descr; diff --git a/src/Interpreters/sortBlock.cpp b/src/Interpreters/sortBlock.cpp index 7f5af84ab0b..dccea6e8449 100644 --- a/src/Interpreters/sortBlock.cpp +++ b/src/Interpreters/sortBlock.cpp @@ -13,6 +13,7 @@ namespace DB namespace ErrorCodes { extern const int BAD_COLLATION; + extern const int OPENCL_ERROR; } static bool isCollationRequired(const SortColumnDescription & description) @@ -131,7 +132,22 @@ void sortBlock(Block & block, const SortDescription & description, UInt64 limit) } else if (!isColumnConst(*column)) - column->getPermutation(reverse, limit, description[0].nulls_direction, perm); + { + int nan_direction_hint = description[0].nulls_direction; + + /// If in Settings `special_sort` option has been set as `bitonic_sort`, + /// then via `nan_direction_hint` variable a flag which specifies bitonic sort as preferred + /// will be passed to `getPermutation` method with value 42. + if (description[0].special_sort == SpecialSort::OPENCL_BITONIC) + { + nan_direction_hint = 42; +#ifndef BITONIC_SORT_PREFERRED + throw DB::Exception("Bitonic sort specified as preferred, but OpenCL not available", DB::ErrorCodes::OPENCL_ERROR); +#endif + } + + column->getPermutation(reverse, limit, nan_direction_hint, perm); + } else /// we don't need to do anything with const column is_column_const = true; diff --git a/tests/queries/0_stateless/01280_opencl_bitonic_order_by_uint8.reference b/tests/queries/0_stateless/01280_opencl_bitonic_order_by_uint8.reference new file mode 100644 index 00000000000..0d1c70bed38 --- /dev/null +++ b/tests/queries/0_stateless/01280_opencl_bitonic_order_by_uint8.reference @@ -0,0 +1,42 @@ +82 +80 +78 +76 +74 +72 +70 +68 +66 +64 +62 +60 +58 +56 +54 +52 +50 +48 +46 +44 +42 +40 +38 +36 +34 +32 +30 +28 +26 +24 +22 +20 +18 +16 +14 +12 +10 +8 +6 +4 +2 +0 diff --git a/tests/queries/0_stateless/01280_opencl_bitonic_order_by_uint8.sql b/tests/queries/0_stateless/01280_opencl_bitonic_order_by_uint8.sql new file mode 100644 index 00000000000..6c6efdfd23c --- /dev/null +++ b/tests/queries/0_stateless/01280_opencl_bitonic_order_by_uint8.sql @@ -0,0 +1 @@ +select toUInt8(number * 2) as x from numbers(42) order by x desc settings special_sort = 'opencl_bitonic' diff --git a/tests/queries/0_stateless/01281_opencl_bitonic_order_by_int8.reference b/tests/queries/0_stateless/01281_opencl_bitonic_order_by_int8.reference new file mode 100644 index 00000000000..0d1c70bed38 --- /dev/null +++ b/tests/queries/0_stateless/01281_opencl_bitonic_order_by_int8.reference @@ -0,0 +1,42 @@ +82 +80 +78 +76 +74 +72 +70 +68 +66 +64 +62 +60 +58 +56 +54 +52 +50 +48 +46 +44 +42 +40 +38 +36 +34 +32 +30 +28 +26 +24 +22 +20 +18 +16 +14 +12 +10 +8 +6 +4 +2 +0 diff --git a/tests/queries/0_stateless/01281_opencl_bitonic_order_by_int8.sql b/tests/queries/0_stateless/01281_opencl_bitonic_order_by_int8.sql new file mode 100644 index 00000000000..dee3bea1435 --- /dev/null +++ b/tests/queries/0_stateless/01281_opencl_bitonic_order_by_int8.sql @@ -0,0 +1 @@ +select toInt8(number * 2) as x from numbers(42) order by x desc settings special_sort = 'opencl_bitonic' diff --git a/tests/queries/0_stateless/01282_opencl_bitonic_order_by_uint16.reference b/tests/queries/0_stateless/01282_opencl_bitonic_order_by_uint16.reference new file mode 100644 index 00000000000..0d1c70bed38 --- /dev/null +++ b/tests/queries/0_stateless/01282_opencl_bitonic_order_by_uint16.reference @@ -0,0 +1,42 @@ +82 +80 +78 +76 +74 +72 +70 +68 +66 +64 +62 +60 +58 +56 +54 +52 +50 +48 +46 +44 +42 +40 +38 +36 +34 +32 +30 +28 +26 +24 +22 +20 +18 +16 +14 +12 +10 +8 +6 +4 +2 +0 diff --git a/tests/queries/0_stateless/01282_opencl_bitonic_order_by_uint16.sql b/tests/queries/0_stateless/01282_opencl_bitonic_order_by_uint16.sql new file mode 100644 index 00000000000..b83e0bc8073 --- /dev/null +++ b/tests/queries/0_stateless/01282_opencl_bitonic_order_by_uint16.sql @@ -0,0 +1 @@ +select toUInt16(number * 2) as x from numbers(42) order by x desc settings special_sort = 'opencl_bitonic' diff --git a/tests/queries/0_stateless/01283_opencl_bitonic_order_by_int16.reference b/tests/queries/0_stateless/01283_opencl_bitonic_order_by_int16.reference new file mode 100644 index 00000000000..0d1c70bed38 --- /dev/null +++ b/tests/queries/0_stateless/01283_opencl_bitonic_order_by_int16.reference @@ -0,0 +1,42 @@ +82 +80 +78 +76 +74 +72 +70 +68 +66 +64 +62 +60 +58 +56 +54 +52 +50 +48 +46 +44 +42 +40 +38 +36 +34 +32 +30 +28 +26 +24 +22 +20 +18 +16 +14 +12 +10 +8 +6 +4 +2 +0 diff --git a/tests/queries/0_stateless/01283_opencl_bitonic_order_by_int16.sql b/tests/queries/0_stateless/01283_opencl_bitonic_order_by_int16.sql new file mode 100644 index 00000000000..02f6dea47f1 --- /dev/null +++ b/tests/queries/0_stateless/01283_opencl_bitonic_order_by_int16.sql @@ -0,0 +1 @@ +select toInt16(number * 2) as x from numbers(42) order by x desc settings special_sort = 'opencl_bitonic' diff --git a/tests/queries/0_stateless/01284_opencl_bitonic_order_by_uint32.reference b/tests/queries/0_stateless/01284_opencl_bitonic_order_by_uint32.reference new file mode 100644 index 00000000000..0d1c70bed38 --- /dev/null +++ b/tests/queries/0_stateless/01284_opencl_bitonic_order_by_uint32.reference @@ -0,0 +1,42 @@ +82 +80 +78 +76 +74 +72 +70 +68 +66 +64 +62 +60 +58 +56 +54 +52 +50 +48 +46 +44 +42 +40 +38 +36 +34 +32 +30 +28 +26 +24 +22 +20 +18 +16 +14 +12 +10 +8 +6 +4 +2 +0 diff --git a/tests/queries/0_stateless/01284_opencl_bitonic_order_by_uint32.sql b/tests/queries/0_stateless/01284_opencl_bitonic_order_by_uint32.sql new file mode 100644 index 00000000000..ee035f2d0d8 --- /dev/null +++ b/tests/queries/0_stateless/01284_opencl_bitonic_order_by_uint32.sql @@ -0,0 +1 @@ +select toUInt32(number * 2) as x from numbers(42) order by x desc settings special_sort = 'opencl_bitonic' diff --git a/tests/queries/0_stateless/01285_opencl_bitonic_order_by_int32.reference b/tests/queries/0_stateless/01285_opencl_bitonic_order_by_int32.reference new file mode 100644 index 00000000000..0d1c70bed38 --- /dev/null +++ b/tests/queries/0_stateless/01285_opencl_bitonic_order_by_int32.reference @@ -0,0 +1,42 @@ +82 +80 +78 +76 +74 +72 +70 +68 +66 +64 +62 +60 +58 +56 +54 +52 +50 +48 +46 +44 +42 +40 +38 +36 +34 +32 +30 +28 +26 +24 +22 +20 +18 +16 +14 +12 +10 +8 +6 +4 +2 +0 diff --git a/tests/queries/0_stateless/01285_opencl_bitonic_order_by_int32.sql b/tests/queries/0_stateless/01285_opencl_bitonic_order_by_int32.sql new file mode 100644 index 00000000000..bf799dfb45e --- /dev/null +++ b/tests/queries/0_stateless/01285_opencl_bitonic_order_by_int32.sql @@ -0,0 +1 @@ +select toInt32(number * 2) as x from numbers(42) order by x desc settings special_sort = 'opencl_bitonic' diff --git a/tests/queries/0_stateless/01286_opencl_bitonic_order_by_uint64.reference b/tests/queries/0_stateless/01286_opencl_bitonic_order_by_uint64.reference new file mode 100644 index 00000000000..0d1c70bed38 --- /dev/null +++ b/tests/queries/0_stateless/01286_opencl_bitonic_order_by_uint64.reference @@ -0,0 +1,42 @@ +82 +80 +78 +76 +74 +72 +70 +68 +66 +64 +62 +60 +58 +56 +54 +52 +50 +48 +46 +44 +42 +40 +38 +36 +34 +32 +30 +28 +26 +24 +22 +20 +18 +16 +14 +12 +10 +8 +6 +4 +2 +0 diff --git a/tests/queries/0_stateless/01286_opencl_bitonic_order_by_uint64.sql b/tests/queries/0_stateless/01286_opencl_bitonic_order_by_uint64.sql new file mode 100644 index 00000000000..e593ac86561 --- /dev/null +++ b/tests/queries/0_stateless/01286_opencl_bitonic_order_by_uint64.sql @@ -0,0 +1 @@ +select toUInt64(number * 2) as x from numbers(42) order by x desc settings special_sort = 'opencl_bitonic' diff --git a/tests/queries/0_stateless/01287_opencl_bitonic_order_by_int64.reference b/tests/queries/0_stateless/01287_opencl_bitonic_order_by_int64.reference new file mode 100644 index 00000000000..0d1c70bed38 --- /dev/null +++ b/tests/queries/0_stateless/01287_opencl_bitonic_order_by_int64.reference @@ -0,0 +1,42 @@ +82 +80 +78 +76 +74 +72 +70 +68 +66 +64 +62 +60 +58 +56 +54 +52 +50 +48 +46 +44 +42 +40 +38 +36 +34 +32 +30 +28 +26 +24 +22 +20 +18 +16 +14 +12 +10 +8 +6 +4 +2 +0 diff --git a/tests/queries/0_stateless/01287_opencl_bitonic_order_by_int64.sql b/tests/queries/0_stateless/01287_opencl_bitonic_order_by_int64.sql new file mode 100644 index 00000000000..a6c30dc4887 --- /dev/null +++ b/tests/queries/0_stateless/01287_opencl_bitonic_order_by_int64.sql @@ -0,0 +1 @@ +select toInt64(number * 2) as x from numbers(42) order by x desc settings special_sort = 'opencl_bitonic' From 8e8a2a17d67abd901e1ca4c74924d304a7262f96 Mon Sep 17 00:00:00 2001 From: Artem Zuikov Date: Fri, 15 May 2020 02:57:22 +0300 Subject: [PATCH 238/738] build fixes --- CMakeLists.txt | 7 +- programs/server/Server.cpp | 6 +- src/Common/tests/CMakeLists.txt | 6 +- src/Interpreters/sortBlock.cpp | 3 +- .../01280_opencl_bitonic_order_by.reference | 336 ++++++++++++++++++ .../01280_opencl_bitonic_order_by.sql | 10 + ...80_opencl_bitonic_order_by_uint8.reference | 42 --- .../01280_opencl_bitonic_order_by_uint8.sql | 1 - ...281_opencl_bitonic_order_by_int8.reference | 42 --- .../01281_opencl_bitonic_order_by_int8.sql | 1 - ...2_opencl_bitonic_order_by_uint16.reference | 42 --- .../01282_opencl_bitonic_order_by_uint16.sql | 1 - ...83_opencl_bitonic_order_by_int16.reference | 42 --- .../01283_opencl_bitonic_order_by_int16.sql | 1 - ...4_opencl_bitonic_order_by_uint32.reference | 42 --- .../01284_opencl_bitonic_order_by_uint32.sql | 1 - ...85_opencl_bitonic_order_by_int32.reference | 42 --- .../01285_opencl_bitonic_order_by_int32.sql | 1 - ...6_opencl_bitonic_order_by_uint64.reference | 42 --- .../01286_opencl_bitonic_order_by_uint64.sql | 1 - ...87_opencl_bitonic_order_by_int64.reference | 42 --- .../01287_opencl_bitonic_order_by_int64.sql | 1 - 22 files changed, 359 insertions(+), 353 deletions(-) create mode 100644 tests/queries/0_stateless/01280_opencl_bitonic_order_by.reference create mode 100644 tests/queries/0_stateless/01280_opencl_bitonic_order_by.sql delete mode 100644 tests/queries/0_stateless/01280_opencl_bitonic_order_by_uint8.reference delete mode 100644 tests/queries/0_stateless/01280_opencl_bitonic_order_by_uint8.sql delete mode 100644 tests/queries/0_stateless/01281_opencl_bitonic_order_by_int8.reference delete mode 100644 tests/queries/0_stateless/01281_opencl_bitonic_order_by_int8.sql delete mode 100644 tests/queries/0_stateless/01282_opencl_bitonic_order_by_uint16.reference delete mode 100644 tests/queries/0_stateless/01282_opencl_bitonic_order_by_uint16.sql delete mode 100644 tests/queries/0_stateless/01283_opencl_bitonic_order_by_int16.reference delete mode 100644 tests/queries/0_stateless/01283_opencl_bitonic_order_by_int16.sql delete mode 100644 tests/queries/0_stateless/01284_opencl_bitonic_order_by_uint32.reference delete mode 100644 tests/queries/0_stateless/01284_opencl_bitonic_order_by_uint32.sql delete mode 100644 tests/queries/0_stateless/01285_opencl_bitonic_order_by_int32.reference delete mode 100644 tests/queries/0_stateless/01285_opencl_bitonic_order_by_int32.sql delete mode 100644 tests/queries/0_stateless/01286_opencl_bitonic_order_by_uint64.reference delete mode 100644 tests/queries/0_stateless/01286_opencl_bitonic_order_by_uint64.sql delete mode 100644 tests/queries/0_stateless/01287_opencl_bitonic_order_by_int64.reference delete mode 100644 tests/queries/0_stateless/01287_opencl_bitonic_order_by_int64.sql diff --git a/CMakeLists.txt b/CMakeLists.txt index 22fe14e5b55..dd21ff123da 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -384,11 +384,12 @@ if (OS_LINUX AND NOT ENABLE_JEMALLOC) endif () if (USE_OPENCL) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DBITONIC_SORT_PREFERRED") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DBITONIC_SORT_PREFERRED") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DUSE_OPENCL=1") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_OPENCL=1") if (OS_DARWIN) - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -framework OpenCL") + set(OPENCL_LINKER_FLAGS "-framework OpenCL") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OPENCL_LINKER_FLAGS}") endif () endif () diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 691ede66eb4..7c119b40ac8 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -61,7 +61,7 @@ #include #include "MySQLHandlerFactory.h" -#ifdef BITONIC_SORT_PREFERRED +#ifdef USE_OPENCL #include "Common/BitonicSort.h" #endif @@ -225,9 +225,9 @@ int Server::main(const std::vector & /*args*/) registerDictionaries(); registerDisks(); - #if defined (BITONIC_SORT_PREFERRED) +#if defined (USE_OPENCL) BitonicSort::getInstance().configure(); - #endif +#endif CurrentMetrics::set(CurrentMetrics::Revision, ClickHouseRevision::get()); CurrentMetrics::set(CurrentMetrics::VersionInteger, ClickHouseRevision::getVersionInteger()); diff --git a/src/Common/tests/CMakeLists.txt b/src/Common/tests/CMakeLists.txt index 87ae4858bf1..495663bc5d8 100644 --- a/src/Common/tests/CMakeLists.txt +++ b/src/Common/tests/CMakeLists.txt @@ -35,8 +35,10 @@ target_link_libraries (compact_array PRIVATE clickhouse_common_io) add_executable (radix_sort radix_sort.cpp) target_link_libraries (radix_sort PRIVATE clickhouse_common_io) -add_executable (bitonic_sort bitonic_sort.cpp) -target_link_libraries (bitonic_sort PRIVATE clickhouse_common_io "-framework OpenCL") +if (USE_OPENCL) + add_executable (bitonic_sort bitonic_sort.cpp) + target_link_libraries (bitonic_sort PRIVATE clickhouse_common_io ${OPENCL_LINKER_FLAGS}) +endif () add_executable (arena_with_free_lists arena_with_free_lists.cpp) target_link_libraries (arena_with_free_lists PRIVATE dbms) diff --git a/src/Interpreters/sortBlock.cpp b/src/Interpreters/sortBlock.cpp index dccea6e8449..0e98dc0eb4b 100644 --- a/src/Interpreters/sortBlock.cpp +++ b/src/Interpreters/sortBlock.cpp @@ -140,8 +140,9 @@ void sortBlock(Block & block, const SortDescription & description, UInt64 limit) /// will be passed to `getPermutation` method with value 42. if (description[0].special_sort == SpecialSort::OPENCL_BITONIC) { +#ifdef USE_OPENCL nan_direction_hint = 42; -#ifndef BITONIC_SORT_PREFERRED +#else throw DB::Exception("Bitonic sort specified as preferred, but OpenCL not available", DB::ErrorCodes::OPENCL_ERROR); #endif } diff --git a/tests/queries/0_stateless/01280_opencl_bitonic_order_by.reference b/tests/queries/0_stateless/01280_opencl_bitonic_order_by.reference new file mode 100644 index 00000000000..436b874be62 --- /dev/null +++ b/tests/queries/0_stateless/01280_opencl_bitonic_order_by.reference @@ -0,0 +1,336 @@ +82 +80 +78 +76 +74 +72 +70 +68 +66 +64 +62 +60 +58 +56 +54 +52 +50 +48 +46 +44 +42 +40 +38 +36 +34 +32 +30 +28 +26 +24 +22 +20 +18 +16 +14 +12 +10 +8 +6 +4 +2 +0 +82 +80 +78 +76 +74 +72 +70 +68 +66 +64 +62 +60 +58 +56 +54 +52 +50 +48 +46 +44 +42 +40 +38 +36 +34 +32 +30 +28 +26 +24 +22 +20 +18 +16 +14 +12 +10 +8 +6 +4 +2 +0 +82 +80 +78 +76 +74 +72 +70 +68 +66 +64 +62 +60 +58 +56 +54 +52 +50 +48 +46 +44 +42 +40 +38 +36 +34 +32 +30 +28 +26 +24 +22 +20 +18 +16 +14 +12 +10 +8 +6 +4 +2 +0 +82 +80 +78 +76 +74 +72 +70 +68 +66 +64 +62 +60 +58 +56 +54 +52 +50 +48 +46 +44 +42 +40 +38 +36 +34 +32 +30 +28 +26 +24 +22 +20 +18 +16 +14 +12 +10 +8 +6 +4 +2 +0 +82 +80 +78 +76 +74 +72 +70 +68 +66 +64 +62 +60 +58 +56 +54 +52 +50 +48 +46 +44 +42 +40 +38 +36 +34 +32 +30 +28 +26 +24 +22 +20 +18 +16 +14 +12 +10 +8 +6 +4 +2 +0 +82 +80 +78 +76 +74 +72 +70 +68 +66 +64 +62 +60 +58 +56 +54 +52 +50 +48 +46 +44 +42 +40 +38 +36 +34 +32 +30 +28 +26 +24 +22 +20 +18 +16 +14 +12 +10 +8 +6 +4 +2 +0 +82 +80 +78 +76 +74 +72 +70 +68 +66 +64 +62 +60 +58 +56 +54 +52 +50 +48 +46 +44 +42 +40 +38 +36 +34 +32 +30 +28 +26 +24 +22 +20 +18 +16 +14 +12 +10 +8 +6 +4 +2 +0 +82 +80 +78 +76 +74 +72 +70 +68 +66 +64 +62 +60 +58 +56 +54 +52 +50 +48 +46 +44 +42 +40 +38 +36 +34 +32 +30 +28 +26 +24 +22 +20 +18 +16 +14 +12 +10 +8 +6 +4 +2 +0 diff --git a/tests/queries/0_stateless/01280_opencl_bitonic_order_by.sql b/tests/queries/0_stateless/01280_opencl_bitonic_order_by.sql new file mode 100644 index 00000000000..ed32a3d37ca --- /dev/null +++ b/tests/queries/0_stateless/01280_opencl_bitonic_order_by.sql @@ -0,0 +1,10 @@ +-- TODO: set special_sort = 'opencl_bitonic'; + +select toUInt8(number * 2) as x from numbers(42) order by x desc; +select toInt8(number * 2) as x from numbers(42) order by x desc; +select toUInt16(number * 2) as x from numbers(42) order by x desc; +select toInt16(number * 2) as x from numbers(42) order by x desc; +select toUInt32(number * 2) as x from numbers(42) order by x desc; +select toInt32(number * 2) as x from numbers(42) order by x desc; +select toUInt64(number * 2) as x from numbers(42) order by x desc; +select toInt64(number * 2) as x from numbers(42) order by x desc; diff --git a/tests/queries/0_stateless/01280_opencl_bitonic_order_by_uint8.reference b/tests/queries/0_stateless/01280_opencl_bitonic_order_by_uint8.reference deleted file mode 100644 index 0d1c70bed38..00000000000 --- a/tests/queries/0_stateless/01280_opencl_bitonic_order_by_uint8.reference +++ /dev/null @@ -1,42 +0,0 @@ -82 -80 -78 -76 -74 -72 -70 -68 -66 -64 -62 -60 -58 -56 -54 -52 -50 -48 -46 -44 -42 -40 -38 -36 -34 -32 -30 -28 -26 -24 -22 -20 -18 -16 -14 -12 -10 -8 -6 -4 -2 -0 diff --git a/tests/queries/0_stateless/01280_opencl_bitonic_order_by_uint8.sql b/tests/queries/0_stateless/01280_opencl_bitonic_order_by_uint8.sql deleted file mode 100644 index 6c6efdfd23c..00000000000 --- a/tests/queries/0_stateless/01280_opencl_bitonic_order_by_uint8.sql +++ /dev/null @@ -1 +0,0 @@ -select toUInt8(number * 2) as x from numbers(42) order by x desc settings special_sort = 'opencl_bitonic' diff --git a/tests/queries/0_stateless/01281_opencl_bitonic_order_by_int8.reference b/tests/queries/0_stateless/01281_opencl_bitonic_order_by_int8.reference deleted file mode 100644 index 0d1c70bed38..00000000000 --- a/tests/queries/0_stateless/01281_opencl_bitonic_order_by_int8.reference +++ /dev/null @@ -1,42 +0,0 @@ -82 -80 -78 -76 -74 -72 -70 -68 -66 -64 -62 -60 -58 -56 -54 -52 -50 -48 -46 -44 -42 -40 -38 -36 -34 -32 -30 -28 -26 -24 -22 -20 -18 -16 -14 -12 -10 -8 -6 -4 -2 -0 diff --git a/tests/queries/0_stateless/01281_opencl_bitonic_order_by_int8.sql b/tests/queries/0_stateless/01281_opencl_bitonic_order_by_int8.sql deleted file mode 100644 index dee3bea1435..00000000000 --- a/tests/queries/0_stateless/01281_opencl_bitonic_order_by_int8.sql +++ /dev/null @@ -1 +0,0 @@ -select toInt8(number * 2) as x from numbers(42) order by x desc settings special_sort = 'opencl_bitonic' diff --git a/tests/queries/0_stateless/01282_opencl_bitonic_order_by_uint16.reference b/tests/queries/0_stateless/01282_opencl_bitonic_order_by_uint16.reference deleted file mode 100644 index 0d1c70bed38..00000000000 --- a/tests/queries/0_stateless/01282_opencl_bitonic_order_by_uint16.reference +++ /dev/null @@ -1,42 +0,0 @@ -82 -80 -78 -76 -74 -72 -70 -68 -66 -64 -62 -60 -58 -56 -54 -52 -50 -48 -46 -44 -42 -40 -38 -36 -34 -32 -30 -28 -26 -24 -22 -20 -18 -16 -14 -12 -10 -8 -6 -4 -2 -0 diff --git a/tests/queries/0_stateless/01282_opencl_bitonic_order_by_uint16.sql b/tests/queries/0_stateless/01282_opencl_bitonic_order_by_uint16.sql deleted file mode 100644 index b83e0bc8073..00000000000 --- a/tests/queries/0_stateless/01282_opencl_bitonic_order_by_uint16.sql +++ /dev/null @@ -1 +0,0 @@ -select toUInt16(number * 2) as x from numbers(42) order by x desc settings special_sort = 'opencl_bitonic' diff --git a/tests/queries/0_stateless/01283_opencl_bitonic_order_by_int16.reference b/tests/queries/0_stateless/01283_opencl_bitonic_order_by_int16.reference deleted file mode 100644 index 0d1c70bed38..00000000000 --- a/tests/queries/0_stateless/01283_opencl_bitonic_order_by_int16.reference +++ /dev/null @@ -1,42 +0,0 @@ -82 -80 -78 -76 -74 -72 -70 -68 -66 -64 -62 -60 -58 -56 -54 -52 -50 -48 -46 -44 -42 -40 -38 -36 -34 -32 -30 -28 -26 -24 -22 -20 -18 -16 -14 -12 -10 -8 -6 -4 -2 -0 diff --git a/tests/queries/0_stateless/01283_opencl_bitonic_order_by_int16.sql b/tests/queries/0_stateless/01283_opencl_bitonic_order_by_int16.sql deleted file mode 100644 index 02f6dea47f1..00000000000 --- a/tests/queries/0_stateless/01283_opencl_bitonic_order_by_int16.sql +++ /dev/null @@ -1 +0,0 @@ -select toInt16(number * 2) as x from numbers(42) order by x desc settings special_sort = 'opencl_bitonic' diff --git a/tests/queries/0_stateless/01284_opencl_bitonic_order_by_uint32.reference b/tests/queries/0_stateless/01284_opencl_bitonic_order_by_uint32.reference deleted file mode 100644 index 0d1c70bed38..00000000000 --- a/tests/queries/0_stateless/01284_opencl_bitonic_order_by_uint32.reference +++ /dev/null @@ -1,42 +0,0 @@ -82 -80 -78 -76 -74 -72 -70 -68 -66 -64 -62 -60 -58 -56 -54 -52 -50 -48 -46 -44 -42 -40 -38 -36 -34 -32 -30 -28 -26 -24 -22 -20 -18 -16 -14 -12 -10 -8 -6 -4 -2 -0 diff --git a/tests/queries/0_stateless/01284_opencl_bitonic_order_by_uint32.sql b/tests/queries/0_stateless/01284_opencl_bitonic_order_by_uint32.sql deleted file mode 100644 index ee035f2d0d8..00000000000 --- a/tests/queries/0_stateless/01284_opencl_bitonic_order_by_uint32.sql +++ /dev/null @@ -1 +0,0 @@ -select toUInt32(number * 2) as x from numbers(42) order by x desc settings special_sort = 'opencl_bitonic' diff --git a/tests/queries/0_stateless/01285_opencl_bitonic_order_by_int32.reference b/tests/queries/0_stateless/01285_opencl_bitonic_order_by_int32.reference deleted file mode 100644 index 0d1c70bed38..00000000000 --- a/tests/queries/0_stateless/01285_opencl_bitonic_order_by_int32.reference +++ /dev/null @@ -1,42 +0,0 @@ -82 -80 -78 -76 -74 -72 -70 -68 -66 -64 -62 -60 -58 -56 -54 -52 -50 -48 -46 -44 -42 -40 -38 -36 -34 -32 -30 -28 -26 -24 -22 -20 -18 -16 -14 -12 -10 -8 -6 -4 -2 -0 diff --git a/tests/queries/0_stateless/01285_opencl_bitonic_order_by_int32.sql b/tests/queries/0_stateless/01285_opencl_bitonic_order_by_int32.sql deleted file mode 100644 index bf799dfb45e..00000000000 --- a/tests/queries/0_stateless/01285_opencl_bitonic_order_by_int32.sql +++ /dev/null @@ -1 +0,0 @@ -select toInt32(number * 2) as x from numbers(42) order by x desc settings special_sort = 'opencl_bitonic' diff --git a/tests/queries/0_stateless/01286_opencl_bitonic_order_by_uint64.reference b/tests/queries/0_stateless/01286_opencl_bitonic_order_by_uint64.reference deleted file mode 100644 index 0d1c70bed38..00000000000 --- a/tests/queries/0_stateless/01286_opencl_bitonic_order_by_uint64.reference +++ /dev/null @@ -1,42 +0,0 @@ -82 -80 -78 -76 -74 -72 -70 -68 -66 -64 -62 -60 -58 -56 -54 -52 -50 -48 -46 -44 -42 -40 -38 -36 -34 -32 -30 -28 -26 -24 -22 -20 -18 -16 -14 -12 -10 -8 -6 -4 -2 -0 diff --git a/tests/queries/0_stateless/01286_opencl_bitonic_order_by_uint64.sql b/tests/queries/0_stateless/01286_opencl_bitonic_order_by_uint64.sql deleted file mode 100644 index e593ac86561..00000000000 --- a/tests/queries/0_stateless/01286_opencl_bitonic_order_by_uint64.sql +++ /dev/null @@ -1 +0,0 @@ -select toUInt64(number * 2) as x from numbers(42) order by x desc settings special_sort = 'opencl_bitonic' diff --git a/tests/queries/0_stateless/01287_opencl_bitonic_order_by_int64.reference b/tests/queries/0_stateless/01287_opencl_bitonic_order_by_int64.reference deleted file mode 100644 index 0d1c70bed38..00000000000 --- a/tests/queries/0_stateless/01287_opencl_bitonic_order_by_int64.reference +++ /dev/null @@ -1,42 +0,0 @@ -82 -80 -78 -76 -74 -72 -70 -68 -66 -64 -62 -60 -58 -56 -54 -52 -50 -48 -46 -44 -42 -40 -38 -36 -34 -32 -30 -28 -26 -24 -22 -20 -18 -16 -14 -12 -10 -8 -6 -4 -2 -0 diff --git a/tests/queries/0_stateless/01287_opencl_bitonic_order_by_int64.sql b/tests/queries/0_stateless/01287_opencl_bitonic_order_by_int64.sql deleted file mode 100644 index a6c30dc4887..00000000000 --- a/tests/queries/0_stateless/01287_opencl_bitonic_order_by_int64.sql +++ /dev/null @@ -1 +0,0 @@ -select toInt64(number * 2) as x from numbers(42) order by x desc settings special_sort = 'opencl_bitonic' From 201c88f64f209f794fee8ed0a25c524b4f8b3894 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 14 May 2020 17:03:06 +0300 Subject: [PATCH 239/738] Get dictionary and check access rights only once per each call of any function reading external dictionaries. --- src/Functions/FunctionsExternalDictionaries.h | 427 ++++++++---------- .../test_dictionaries_access/__init__.py | 0 .../test_dictionaries_access/test.py | 92 ++++ 3 files changed, 290 insertions(+), 229 deletions(-) create mode 100644 tests/integration/test_dictionaries_access/__init__.py create mode 100644 tests/integration/test_dictionaries_access/test.py diff --git a/src/Functions/FunctionsExternalDictionaries.h b/src/Functions/FunctionsExternalDictionaries.h index f9b326f1def..118c389666f 100644 --- a/src/Functions/FunctionsExternalDictionaries.h +++ b/src/Functions/FunctionsExternalDictionaries.h @@ -54,6 +54,7 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } + /** Functions that use plug-ins (external) dictionaries_loader. * * Get the value of the attribute of the specified type. @@ -69,6 +70,51 @@ namespace ErrorCodes */ +class FunctionDictHelper +{ +public: + FunctionDictHelper(const Context & context_) : context(context_), external_loader(context.getExternalDictionariesLoader()) {} + + std::shared_ptr getDictionary(const String & dictionary_name) + { + auto dict = std::atomic_load(&dictionary); + if (dict) + return dict; + dict = external_loader.getDictionary(dictionary_name); + context.checkAccess(AccessType::dictGet, dict->getDatabaseOrNoDatabaseTag(), dict->getName()); + std::atomic_store(&dictionary, dict); + return dict; + } + + std::shared_ptr getDictionary(const ColumnWithTypeAndName & column) + { + const auto dict_name_col = checkAndGetColumnConst(column.column.get()); + return getDictionary(dict_name_col->getValue()); + } + + bool isDictGetFunctionInjective(const Block & sample_block) + { + if (sample_block.columns() != 3 && sample_block.columns() != 4) + throw Exception{"Function dictGet... takes 3 or 4 arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH}; + + const auto dict_name_col = checkAndGetColumnConst(sample_block.getByPosition(0).column.get()); + if (!dict_name_col) + throw Exception{"First argument of function dictGet... must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; + + const auto attr_name_col = checkAndGetColumnConst(sample_block.getByPosition(1).column.get()); + if (!attr_name_col) + throw Exception{"Second argument of function dictGet... must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; + + return getDictionary(dict_name_col->getValue())->isInjective(attr_name_col->getValue()); + } + +private: + const Context & context; + const ExternalDictionariesLoader & external_loader; + mutable std::shared_ptr dictionary; +}; + + class FunctionDictHas final : public IFunction { public: @@ -76,12 +122,10 @@ public: static FunctionPtr create(const Context & context) { - return std::make_shared(context.getExternalDictionariesLoader(), context); + return std::make_shared(context); } - FunctionDictHas(const ExternalDictionariesLoader & dictionaries_loader_, const Context & context_) - : dictionaries_loader(dictionaries_loader_) - , context(context_) {} + FunctionDictHas(const Context & context_) : helper(context_) {} String getName() const override { return name; } @@ -109,10 +153,6 @@ private: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - const auto dict_name_col = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get()); - if (!dict_name_col) - throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - /** Do not require existence of the dictionary if the function is called for empty block. * This is needed to allow successful query analysis on a server, * that is the initiator of a distributed query, @@ -127,28 +167,26 @@ private: return; } - auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue()); - const auto dict_ptr = dict.get(); - context.checkAccess(AccessType::dictGet, dict_ptr->getDatabaseOrNoDatabaseTag(), dict_ptr->getName()); + auto dict = helper.getDictionary(block.getByPosition(arguments[0])); - if (!executeDispatchSimple(block, arguments, result, dict_ptr) && - !executeDispatchSimple(block, arguments, result, dict_ptr) && - !executeDispatchSimple(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && + if (!executeDispatchSimple(block, arguments, result, dict) && + !executeDispatchSimple(block, arguments, result, dict) && + !executeDispatchSimple(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && #if !defined(ARCADIA_BUILD) - !executeDispatchComplex(block, arguments, result, dict_ptr) && + !executeDispatchComplex(block, arguments, result, dict) && #endif - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchSimple(block, arguments, result, dict_ptr)) - throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchSimple(block, arguments, result, dict)) + throw Exception{"Unsupported dictionary type " + dict->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; } template bool executeDispatchSimple( - Block & block, const ColumnNumbers & arguments, const size_t result, const IDictionaryBase * dictionary) + Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -169,9 +207,9 @@ private: template bool executeDispatchComplex( - Block & block, const ColumnNumbers & arguments, const size_t result, const IDictionaryBase * dictionary) + Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -193,28 +231,11 @@ private: return true; } - const ExternalDictionariesLoader & dictionaries_loader; - const Context & context; +private: + mutable FunctionDictHelper helper; }; -static bool isDictGetFunctionInjective(const ExternalDictionariesLoader & dictionaries_loader, const Block & sample_block) -{ - if (sample_block.columns() != 3 && sample_block.columns() != 4) - throw Exception{"Function dictGet... takes 3 or 4 arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH}; - - const auto dict_name_col = checkAndGetColumnConst(sample_block.getByPosition(0).column.get()); - if (!dict_name_col) - throw Exception{"First argument of function dictGet... must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - - const auto attr_name_col = checkAndGetColumnConst(sample_block.getByPosition(1).column.get()); - if (!attr_name_col) - throw Exception{"Second argument of function dictGet... must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - - return dictionaries_loader.getDictionary(dict_name_col->getValue())->isInjective(attr_name_col->getValue()); -} - - /** For ColumnVector. Either returns a reference to internal data, * or convert it to T type, stores the result in backup_storage and returns a reference to it. */ @@ -229,12 +250,10 @@ public: static FunctionPtr create(const Context & context) { - return std::make_shared(context.getExternalDictionariesLoader(), context); + return std::make_shared(context); } - FunctionDictGetString(const ExternalDictionariesLoader & dictionaries_loader_, const Context & context_) - : dictionaries_loader(dictionaries_loader_) - , context(context_) {} + FunctionDictGetString(const Context & context_) : helper(context_) {} String getName() const override { return name; } @@ -247,7 +266,7 @@ private: bool isInjective(const Block & sample_block) const override { - return isDictGetFunctionInjective(dictionaries_loader, sample_block); + return helper.isDictGetFunctionInjective(sample_block); } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override @@ -290,10 +309,6 @@ private: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - const auto dict_name_col = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get()); - if (!dict_name_col) - throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - if (input_rows_count == 0) { auto & elem = block.getByPosition(result); @@ -301,29 +316,27 @@ private: return; } - auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue()); - const auto dict_ptr = dict.get(); - context.checkAccess(AccessType::dictGet, dict_ptr->getDatabaseOrNoDatabaseTag(), dict_ptr->getName()); + auto dict = helper.getDictionary(block.getByPosition(arguments[0])); - if (!executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && + if (!executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && #if !defined(ARCADIA_BUILD) - !executeDispatchComplex(block, arguments, result, dict_ptr) && + !executeDispatchComplex(block, arguments, result, dict) && #endif - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchRange(block, arguments, result, dict_ptr)) - throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchRange(block, arguments, result, dict)) + throw Exception{"Unsupported dictionary type " + dict->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; } template bool executeDispatch( - Block & block, const ColumnNumbers & arguments, const size_t result, const IDictionaryBase * dictionary) + Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -352,9 +365,9 @@ private: template bool executeDispatchComplex( - Block & block, const ColumnNumbers & arguments, const size_t result, const IDictionaryBase * dictionary) + Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -389,9 +402,9 @@ private: template bool executeDispatchRange( - Block & block, const ColumnNumbers & arguments, const size_t result, const IDictionaryBase * dictionary) + Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -420,8 +433,8 @@ private: return true; } - const ExternalDictionariesLoader & dictionaries_loader; - const Context & context; +private: + mutable FunctionDictHelper helper; }; @@ -432,12 +445,10 @@ public: static FunctionPtr create(const Context & context) { - return std::make_shared(context.getExternalDictionariesLoader(), context); + return std::make_shared(context); } - FunctionDictGetStringOrDefault(const ExternalDictionariesLoader & dictionaries_loader_, const Context & context_) - : dictionaries_loader(dictionaries_loader_) - , context(context_) {} + FunctionDictGetStringOrDefault(const Context & context_) : helper(context_) {} String getName() const override { return name; } @@ -475,10 +486,6 @@ private: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - const auto dict_name_col = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get()); - if (!dict_name_col) - throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - if (input_rows_count == 0) { auto & elem = block.getByPosition(result); @@ -486,28 +493,26 @@ private: return; } - auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue()); - const auto dict_ptr = dict.get(); - context.checkAccess(AccessType::dictGet, dict_ptr->getDatabaseOrNoDatabaseTag(), dict_ptr->getName()); + auto dict = helper.getDictionary(block.getByPosition(arguments[0])); - if (!executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && + if (!executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && #if !defined(ARCADIA_BUILD) - !executeDispatchComplex(block, arguments, result, dict_ptr) && + !executeDispatchComplex(block, arguments, result, dict) && #endif - !executeDispatchComplex(block, arguments, result, dict_ptr)) - throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; + !executeDispatchComplex(block, arguments, result, dict)) + throw Exception{"Unsupported dictionary type " + dict->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; } template bool executeDispatch( - Block & block, const ColumnNumbers & arguments, const size_t result, const IDictionaryBase * dictionary) + Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -530,7 +535,7 @@ private: template void executeDispatch( - Block & block, const ColumnNumbers & arguments, const size_t result, const DictionaryType * dictionary, + Block & block, const ColumnNumbers & arguments, const size_t result, const DictionaryType * dict, const std::string & attr_name, const ColumnUInt64 * id_col) { const auto default_col_untyped = block.getByPosition(arguments[3]).column.get(); @@ -540,7 +545,7 @@ private: /// vector ids, vector defaults auto out = ColumnString::create(); const auto & ids = id_col->getData(); - dictionary->getString(attr_name, ids, default_col, out.get()); + dict->getString(attr_name, ids, default_col, out.get()); block.getByPosition(result).column = std::move(out); } else if (const auto default_col_const = checkAndGetColumnConstStringOrFixedString(default_col_untyped)) @@ -549,7 +554,7 @@ private: auto out = ColumnString::create(); const auto & ids = id_col->getData(); String def = default_col_const->getValue(); - dictionary->getString(attr_name, ids, def, out.get()); + dict->getString(attr_name, ids, def, out.get()); block.getByPosition(result).column = std::move(out); } else @@ -558,7 +563,7 @@ private: template void executeDispatch( - Block & block, const ColumnNumbers & arguments, const size_t result, const DictionaryType * dictionary, + Block & block, const ColumnNumbers & arguments, const size_t result, const DictionaryType * dict, const std::string & attr_name, const ColumnConst * id_col) { const auto default_col_untyped = block.getByPosition(arguments[3]).column.get(); @@ -568,11 +573,11 @@ private: /// const ids, vector defaults const PaddedPODArray ids(1, id_col->getValue()); PaddedPODArray flags(1); - dictionary->has(ids, flags); + dict->has(ids, flags); if (flags.front()) { auto out = ColumnString::create(); - dictionary->getString(attr_name, ids, String(), out.get()); + dict->getString(attr_name, ids, String(), out.get()); block.getByPosition(result).column = DataTypeString().createColumnConst(id_col->size(), out->getDataAt(0).toString()); } else @@ -584,7 +589,7 @@ private: const PaddedPODArray ids(1, id_col->getValue()); auto out = ColumnString::create(); String def = default_col_const->getValue(); - dictionary->getString(attr_name, ids, def, out.get()); + dict->getString(attr_name, ids, def, out.get()); block.getByPosition(result).column = DataTypeString().createColumnConst(id_col->size(), out->getDataAt(0).toString()); } else @@ -593,9 +598,9 @@ private: template bool executeDispatchComplex( - Block & block, const ColumnNumbers & arguments, const size_t result, const IDictionaryBase * dictionary) + Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -631,8 +636,7 @@ private: return true; } - const ExternalDictionariesLoader & dictionaries_loader; - const Context & context; + mutable FunctionDictHelper helper; }; @@ -755,12 +759,11 @@ public: static FunctionPtr create(const Context & context, UInt32 dec_scale = 0) { - return std::make_shared(context.getExternalDictionariesLoader(), context, dec_scale); + return std::make_shared(context, dec_scale); } - FunctionDictGet(const ExternalDictionariesLoader & dictionaries_loader_, const Context & context_, UInt32 dec_scale = 0) - : dictionaries_loader(dictionaries_loader_) - , context(context_) + FunctionDictGet(const Context & context_, UInt32 dec_scale = 0) + : helper(context_) , decimal_scale(dec_scale) {} @@ -775,7 +778,7 @@ private: bool isInjective(const Block & sample_block) const override { - return isDictGetFunctionInjective(dictionaries_loader, sample_block); + return helper.isDictGetFunctionInjective(sample_block); } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override @@ -816,10 +819,6 @@ private: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - const auto dict_name_col = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get()); - if (!dict_name_col) - throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - if (input_rows_count == 0) { auto & elem = block.getByPosition(result); @@ -827,29 +826,26 @@ private: return; } - auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue()); - const auto dict_ptr = dict.get(); - context.checkAccess(AccessType::dictGet, dict_ptr->getDatabaseOrNoDatabaseTag(), dict_ptr->getName()); + auto dict = helper.getDictionary(block.getByPosition(arguments[0])); - if (!executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && + if (!executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && #if !defined(ARCADIA_BUILD) - !executeDispatchComplex(block, arguments, result, dict_ptr) && + !executeDispatchComplex(block, arguments, result, dict) && #endif - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchRange(block, arguments, result, dict_ptr)) - throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchRange(block, arguments, result, dict)) + throw Exception{"Unsupported dictionary type " + dict->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; } template - bool executeDispatch(Block & block, const ColumnNumbers & arguments, const size_t result, - const IDictionaryBase * dictionary) + bool executeDispatch(Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -903,9 +899,9 @@ private: template bool executeDispatchComplex( - Block & block, const ColumnNumbers & arguments, const size_t result, const IDictionaryBase * dictionary) + Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -946,9 +942,9 @@ private: template bool executeDispatchRange( - Block & block, const ColumnNumbers & arguments, const size_t result, const IDictionaryBase * dictionary) + Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -982,8 +978,7 @@ private: return true; } - const ExternalDictionariesLoader & dictionaries_loader; - const Context & context; + mutable FunctionDictHelper helper; UInt32 decimal_scale; }; @@ -1033,12 +1028,11 @@ public: static FunctionPtr create(const Context & context, UInt32 dec_scale = 0) { - return std::make_shared(context.getExternalDictionariesLoader(), context, dec_scale); + return std::make_shared(context, dec_scale); } - FunctionDictGetOrDefault(const ExternalDictionariesLoader & dictionaries_loader_, const Context & context_, UInt32 dec_scale = 0) - : dictionaries_loader(dictionaries_loader_) - , context(context_) + FunctionDictGetOrDefault(const Context & context_, UInt32 dec_scale = 0) + : helper(context_) , decimal_scale(dec_scale) {} @@ -1079,10 +1073,6 @@ private: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - const auto dict_name_col = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get()); - if (!dict_name_col) - throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - if (input_rows_count == 0) { auto & elem = block.getByPosition(result); @@ -1090,28 +1080,25 @@ private: return; } - auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue()); - const auto dict_ptr = dict.get(); - context.checkAccess(AccessType::dictGet, dict_ptr->getDatabaseOrNoDatabaseTag(), dict_ptr->getName()); + auto dict = helper.getDictionary(block.getByPosition(arguments[0])); - if (!executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && + if (!executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && #if !defined(ARCADIA_BUILD) - !executeDispatchComplex(block, arguments, result, dict_ptr) && + !executeDispatchComplex(block, arguments, result, dict) && #endif - !executeDispatchComplex(block, arguments, result, dict_ptr)) - throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; + !executeDispatchComplex(block, arguments, result, dict)) + throw Exception{"Unsupported dictionary type " + dict->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; } template - bool executeDispatch(Block & block, const ColumnNumbers & arguments, const size_t result, - const IDictionaryBase * dictionary) + bool executeDispatch(Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -1134,7 +1121,7 @@ private: template void executeDispatch( - Block & block, const ColumnNumbers & arguments, const size_t result, const DictionaryType * dictionary, + Block & block, const ColumnNumbers & arguments, const size_t result, const DictionaryType * dict, const std::string & attr_name, const ColumnUInt64 * id_col) { const auto default_col_untyped = block.getByPosition(arguments[3]).column.get(); @@ -1150,7 +1137,7 @@ private: const auto & ids = id_col->getData(); auto & data = out->getData(); const auto & defs = default_col->getData(); - DictGetTraits::getOrDefault(dictionary, attr_name, ids, defs, data); + DictGetTraits::getOrDefault(dict, attr_name, ids, defs, data); block.getByPosition(result).column = std::move(out); } else if (const auto default_col_const = checkAndGetColumnConst(default_col_untyped)) @@ -1164,7 +1151,7 @@ private: const auto & ids = id_col->getData(); auto & data = out->getData(); const auto def = default_col_const->template getValue(); - DictGetTraits::getOrDefault(dictionary, attr_name, ids, def, data); + DictGetTraits::getOrDefault(dict, attr_name, ids, def, data); block.getByPosition(result).column = std::move(out); } else @@ -1173,7 +1160,7 @@ private: template void executeDispatch( - Block & block, const ColumnNumbers & arguments, const size_t result, const DictionaryType * dictionary, + Block & block, const ColumnNumbers & arguments, const size_t result, const DictionaryType * dict, const std::string & attr_name, const ColumnConst * id_col) { const auto default_col_untyped = block.getByPosition(arguments[3]).column.get(); @@ -1183,13 +1170,13 @@ private: /// const ids, vector defaults const PaddedPODArray ids(1, id_col->getValue()); PaddedPODArray flags(1); - dictionary->has(ids, flags); + dict->has(ids, flags); if (flags.front()) { if constexpr (IsDataTypeDecimal) { DecimalPaddedPODArray data(1, decimal_scale); - DictGetTraits::getOrDefault(dictionary, attr_name, ids, Type(), data); + DictGetTraits::getOrDefault(dict, attr_name, ids, Type(), data); block.getByPosition(result).column = DataType(DataType::maxPrecision(), decimal_scale).createColumnConst( id_col->size(), toField(data.front(), decimal_scale)); @@ -1197,7 +1184,7 @@ private: else { PaddedPODArray data(1); - DictGetTraits::getOrDefault(dictionary, attr_name, ids, Type(), data); + DictGetTraits::getOrDefault(dict, attr_name, ids, Type(), data); block.getByPosition(result).column = DataType().createColumnConst(id_col->size(), toField(data.front())); } } @@ -1213,7 +1200,7 @@ private: { DecimalPaddedPODArray data(1, decimal_scale); const auto & def = default_col_const->template getValue(); - DictGetTraits::getOrDefault(dictionary, attr_name, ids, def, data); + DictGetTraits::getOrDefault(dict, attr_name, ids, def, data); block.getByPosition(result).column = DataType(DataType::maxPrecision(), decimal_scale).createColumnConst( id_col->size(), toField(data.front(), decimal_scale)); @@ -1222,7 +1209,7 @@ private: { PaddedPODArray data(1); const auto & def = default_col_const->template getValue(); - DictGetTraits::getOrDefault(dictionary, attr_name, ids, def, data); + DictGetTraits::getOrDefault(dict, attr_name, ids, def, data); block.getByPosition(result).column = DataType().createColumnConst(id_col->size(), toField(data.front())); } } @@ -1232,9 +1219,9 @@ private: template bool executeDispatchComplex( - Block & block, const ColumnNumbers & arguments, const size_t result, const IDictionaryBase * dictionary) + Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -1282,8 +1269,7 @@ private: return true; } - const ExternalDictionariesLoader & dictionaries_loader; - const Context & context; + mutable FunctionDictHelper helper; UInt32 decimal_scale; }; @@ -1330,10 +1316,10 @@ public: static FunctionPtr create(const Context & context) { - return std::make_shared(context.getExternalDictionariesLoader(), context); + return std::make_shared(context); } - FunctionDictGetNoType(const ExternalDictionariesLoader & dictionaries_loader_, const Context & context_) : dictionaries_loader(dictionaries_loader_), context(context_) {} + FunctionDictGetNoType(const Context & context_) : context(context_), helper(context_) {} String getName() const override { return name; } @@ -1346,7 +1332,7 @@ private: bool isInjective(const Block & sample_block) const override { - return isDictGetFunctionInjective(dictionaries_loader, sample_block); + return helper.isDictGetFunctionInjective(sample_block); } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override @@ -1386,7 +1372,7 @@ private: + ", must be convertible to " + TypeName::get() + ".", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; } - auto dict = dictionaries_loader.getDictionary(dict_name); + auto dict = helper.getDictionary(dict_name); const DictionaryStructure & structure = dict->getStructure(); for (const auto idx : ext::range(0, structure.attributes.size())) @@ -1466,8 +1452,8 @@ private: } private: - const ExternalDictionariesLoader & dictionaries_loader; const Context & context; + mutable FunctionDictHelper helper; mutable FunctionPtr impl; // underlying function used by dictGet function without explicit type info }; @@ -1479,10 +1465,10 @@ public: static FunctionPtr create(const Context & context) { - return std::make_shared(context.getExternalDictionariesLoader(), context); + return std::make_shared(context); } - FunctionDictGetNoTypeOrDefault(const ExternalDictionariesLoader & dictionaries_loader_, const Context & context_) : dictionaries_loader(dictionaries_loader_), context(context_) {} + FunctionDictGetNoTypeOrDefault(const Context & context_) : context(context_), helper(context_) {} String getName() const override { return name; } @@ -1494,7 +1480,7 @@ private: bool isInjective(const Block & sample_block) const override { - return isDictGetFunctionInjective(dictionaries_loader, sample_block); + return helper.isDictGetFunctionInjective(sample_block); } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override @@ -1522,7 +1508,7 @@ private: throw Exception{"Illegal type " + arguments[2].type->getName() + " of third argument of function " + getName() + ", must be UInt64 or tuple(...).", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - auto dict = dictionaries_loader.getDictionary(dict_name); + auto dict = helper.getDictionary(dict_name); const DictionaryStructure & structure = dict->getStructure(); for (const auto idx : ext::range(0, structure.attributes.size())) @@ -1608,8 +1594,8 @@ private: } private: - const ExternalDictionariesLoader & dictionaries_loader; const Context & context; + mutable FunctionDictHelper helper; mutable FunctionPtr impl; // underlying function used by dictGet function without explicit type info }; @@ -1622,12 +1608,10 @@ public: static FunctionPtr create(const Context & context) { - return std::make_shared(context.getExternalDictionariesLoader(), context); + return std::make_shared(context); } - FunctionDictGetHierarchy(const ExternalDictionariesLoader & dictionaries_loader_, const Context & context_) - : dictionaries_loader(dictionaries_loader_) - , context(context_) {} + FunctionDictGetHierarchy(const Context & context_) : helper(context_) {} String getName() const override { return name; } @@ -1655,10 +1639,6 @@ private: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - const auto dict_name_col = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get()); - if (!dict_name_col) - throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - if (input_rows_count == 0) { auto & elem = block.getByPosition(result); @@ -1666,22 +1646,20 @@ private: return; } - auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue()); - const auto dict_ptr = dict.get(); - context.checkAccess(AccessType::dictGet, dict_ptr->getDatabaseOrNoDatabaseTag(), dict_ptr->getName()); + auto dict = helper.getDictionary(block.getByPosition(arguments[0])); - if (!executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr)) - throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; + if (!executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict)) + throw Exception{"Unsupported dictionary type " + dict->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; } template bool executeDispatch(Block & block, const ColumnNumbers & arguments, const size_t result, - const IDictionaryBase * dictionary) + const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -1772,8 +1750,7 @@ private: return true; } - const ExternalDictionariesLoader & dictionaries_loader; - const Context & context; + mutable FunctionDictHelper helper; }; @@ -1784,12 +1761,11 @@ public: static FunctionPtr create(const Context & context) { - return std::make_shared(context.getExternalDictionariesLoader(), context); + return std::make_shared(context); } - FunctionDictIsIn(const ExternalDictionariesLoader & dictionaries_loader_, const Context & context_) - : dictionaries_loader(dictionaries_loader_) - , context(context_) {} + FunctionDictIsIn(const Context & context_) + : helper(context_) {} String getName() const override { return name; } @@ -1820,10 +1796,6 @@ private: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - const auto dict_name_col = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get()); - if (!dict_name_col) - throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - if (input_rows_count == 0) { auto & elem = block.getByPosition(result); @@ -1831,22 +1803,20 @@ private: return; } - auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue()); - const auto dict_ptr = dict.get(); - context.checkAccess(AccessType::dictGet, dict_ptr->getDatabaseOrNoDatabaseTag(), dict_ptr->getName()); + auto dict = helper.getDictionary(block.getByPosition(arguments[0])); - if (!executeDispatch(block, arguments, result, dict_ptr) - && !executeDispatch(block, arguments, result, dict_ptr) - && !executeDispatch(block, arguments, result, dict_ptr) - && !executeDispatch(block, arguments, result, dict_ptr)) - throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; + if (!executeDispatch(block, arguments, result, dict) + && !executeDispatch(block, arguments, result, dict) + && !executeDispatch(block, arguments, result, dict) + && !executeDispatch(block, arguments, result, dict)) + throw Exception{"Unsupported dictionary type " + dict->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; } template bool executeDispatch(Block & block, const ColumnNumbers & arguments, const size_t result, - const IDictionaryBase * dictionary) + const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -1868,7 +1838,7 @@ private: } template - bool execute(Block & block, const size_t result, const DictionaryType * dictionary, + bool execute(Block & block, const size_t result, const DictionaryType * dict, const ColumnUInt64 * child_id_col, const IColumn * ancestor_id_col_untyped) { if (const auto ancestor_id_col = checkAndGetColumn(ancestor_id_col_untyped)) @@ -1881,7 +1851,7 @@ private: const auto size = child_id_col->size(); data.resize(size); - dictionary->isInVectorVector(child_ids, ancestor_ids, data); + dict->isInVectorVector(child_ids, ancestor_ids, data); block.getByPosition(result).column = std::move(out); } else if (const auto ancestor_id_col_const = checkAndGetColumnConst>(ancestor_id_col_untyped)) @@ -1894,7 +1864,7 @@ private: const auto size = child_id_col->size(); data.resize(size); - dictionary->isInVectorConstant(child_ids, ancestor_id, data); + dict->isInVectorConstant(child_ids, ancestor_id, data); block.getByPosition(result).column = std::move(out); } else @@ -1907,7 +1877,7 @@ private: } template - bool execute(Block & block, const size_t result, const DictionaryType * dictionary, + bool execute(Block & block, const size_t result, const DictionaryType * dict, const ColumnConst * child_id_col, const IColumn * ancestor_id_col_untyped) { if (const auto ancestor_id_col = checkAndGetColumn(ancestor_id_col_untyped)) @@ -1920,7 +1890,7 @@ private: const auto size = child_id_col->size(); data.resize(size); - dictionary->isInConstantVector(child_id, ancestor_ids, data); + dict->isInConstantVector(child_id, ancestor_ids, data); block.getByPosition(result).column = std::move(out); } else if (const auto ancestor_id_col_const = checkAndGetColumnConst>(ancestor_id_col_untyped)) @@ -1929,7 +1899,7 @@ private: const auto ancestor_id = ancestor_id_col_const->getValue(); UInt8 res = 0; - dictionary->isInConstantConstant(child_id, ancestor_id, res); + dict->isInConstantConstant(child_id, ancestor_id, res); block.getByPosition(result).column = DataTypeUInt8().createColumnConst(child_id_col->size(), res); } else @@ -1939,8 +1909,7 @@ private: return true; } - const ExternalDictionariesLoader & dictionaries_loader; - const Context & context; + mutable FunctionDictHelper helper; }; diff --git a/tests/integration/test_dictionaries_access/__init__.py b/tests/integration/test_dictionaries_access/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_dictionaries_access/test.py b/tests/integration/test_dictionaries_access/test.py new file mode 100644 index 00000000000..cbba651a3b7 --- /dev/null +++ b/tests/integration/test_dictionaries_access/test.py @@ -0,0 +1,92 @@ +import pytest +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +instance = cluster.add_instance('instance') + + +@pytest.fixture(scope="module", autouse=True) +def started_cluster(): + try: + cluster.start() + + instance.query("CREATE USER mira") + instance.query("CREATE TABLE test_table(x Int32, y Int32) ENGINE=Log") + instance.query("INSERT INTO test_table VALUES (5,6)") + + yield cluster + + finally: + cluster.shutdown() + + +@pytest.fixture(autouse=True) +def clear_after_test(): + try: + yield + finally: + instance.query("CREATE USER OR REPLACE mira") + instance.query("DROP DICTIONARY IF EXISTS test_dict") + + +create_query = """ + CREATE DICTIONARY test_dict(x Int32, y Int32) PRIMARY KEY x + LAYOUT(FLAT()) + SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'test_table' DB 'default')) + LIFETIME(0) + """ + +drop_query = "DROP DICTIONARY test_dict" + + +def test_create(): + assert instance.query("SHOW GRANTS FOR mira") == "" + assert "Not enough privileges" in instance.query_and_get_error(create_query, user="mira") + + instance.query("GRANT CREATE DICTIONARY ON *.* TO mira") + instance.query(create_query, user="mira") + instance.query(drop_query) + + instance.query("REVOKE CREATE DICTIONARY ON *.* FROM mira") + assert instance.query("SHOW GRANTS FOR mira") == "" + assert "Not enough privileges" in instance.query_and_get_error(create_query, user="mira") + + instance.query("GRANT CREATE DICTIONARY ON default.* TO mira") + instance.query(create_query, user="mira") + instance.query(drop_query) + + instance.query("REVOKE CREATE DICTIONARY ON default.* FROM mira") + assert instance.query("SHOW GRANTS FOR mira") == "" + assert "Not enough privileges" in instance.query_and_get_error(create_query, user="mira") + + instance.query("GRANT CREATE DICTIONARY ON default.test_dict TO mira") + instance.query(create_query, user="mira") + + +def test_drop(): + instance.query(create_query) + + assert instance.query("SHOW GRANTS FOR mira") == "" + assert "Not enough privileges" in instance.query_and_get_error(drop_query, user="mira") + + instance.query("GRANT DROP DICTIONARY ON *.* TO mira") + instance.query(drop_query, user="mira") + instance.query(create_query) + + +def test_dictget(): + instance.query(create_query) + + dictget_query = "SELECT dictGet('default.test_dict', 'y', toUInt64(5))" + instance.query(dictget_query) == "6\n" + assert "Not enough privileges" in instance.query_and_get_error(dictget_query, user='mira') + + instance.query("GRANT dictGet ON default.test_dict TO mira") + instance.query(dictget_query, user='mira') == "6\n" + + dictget_query = "SELECT dictGet('default.test_dict', 'y', toUInt64(1))" + instance.query(dictget_query) == "0\n" + instance.query(dictget_query, user='mira') == "0\n" + + instance.query("REVOKE dictGet ON *.* FROM mira") + assert "Not enough privileges" in instance.query_and_get_error(dictget_query, user='mira') From cd14f9ebcb0e400b90bb6b492af664f8a7b7bad6 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Fri, 15 May 2020 07:34:54 +0300 Subject: [PATCH 240/738] SQL reference refactoring (#10857) * split up select.md * array-join.md basic refactoring * distinct.md basic refactoring * format.md basic refactoring * from.md basic refactoring * group-by.md basic refactoring * having.md basic refactoring * additional index.md refactoring * into-outfile.md basic refactoring * join.md basic refactoring * limit.md basic refactoring * limit-by.md basic refactoring * order-by.md basic refactoring * prewhere.md basic refactoring * adjust operators/index.md links * adjust sample.md links * adjust more links * adjust operatots links * fix some links * adjust aggregate function article titles * basic refactor of remaining select clauses * absolute paths in make_links.sh * run make_links.sh * remove old select.md locations * translate docs/es * translate docs/fr * translate docs/fa * remove old operators.md location * change operators.md links * adjust links in docs/es * adjust links in docs/es * minor texts adjustments * wip * update machine translations to use new links * fix changelog * es build fixes * get rid of some select.md links * temporary adjust ru links * temporary adjust more ru links * improve curly brace handling * adjust ru as well * fa build fix * ru link fixes * zh link fixes * temporary disable part of anchor checks --- .../mergetree-family/mergetree.md | 4 +- docs/en/engines/table-engines/special/join.md | 8 +- docs/en/faq/general.md | 4 +- docs/en/operations/requirements.md | 4 +- .../operations/settings/query-complexity.md | 10 +- docs/en/operations/settings/settings.md | 8 +- .../aggregate-functions/combinators.md | 2 +- .../parametric-functions.md | 4 +- .../aggregate-functions/reference.md | 6 +- .../data-types/aggregatefunction.md | 2 +- docs/en/sql-reference/data-types/datetime.md | 2 +- .../en/sql-reference/data-types/datetime64.md | 2 +- docs/en/sql-reference/data-types/float.md | 2 +- .../data-types/special-data-types/interval.md | 4 +- .../data-types/special-data-types/set.md | 2 +- docs/en/sql-reference/data-types/tuple.md | 2 +- .../functions/conditional-functions.md | 2 +- .../sql-reference/functions/in-functions.md | 2 +- docs/en/sql-reference/index.md | 6 +- docs/en/sql-reference/operators/in.md | 199 ++ .../{operators.md => operators/index.md} | 16 +- docs/en/sql-reference/statements/grant.md | 2 +- docs/en/sql-reference/statements/select.md | 1377 --------- .../statements/select/array-join.md | 277 ++ .../statements/select/distinct.md | 58 + .../sql-reference/statements/select/format.md | 13 + .../sql-reference/statements/select/from.md | 39 + .../statements/select/group-by.md | 127 + .../sql-reference/statements/select/having.md | 9 + .../sql-reference/statements/select/index.md | 160 + .../statements/select/into-outfile.md | 9 + .../sql-reference/statements/select/join.md | 186 ++ .../statements/select/limit-by.md | 66 + .../sql-reference/statements/select/limit.md | 9 + .../statements/select/order-by.md | 67 + .../statements/select/prewhere.md | 18 + .../sql-reference/statements/select/sample.md | 108 + .../statements/select/union-all.md | 30 + .../sql-reference/statements/select/where.md | 10 + .../sql-reference/statements/select/with.md | 75 + docs/en/sql-reference/statements/show.md | 1 - docs/en/sql-reference/syntax.md | 2 +- .../en/sql-reference/table-functions/index.md | 2 +- docs/es/commercial/cloud.md | 8 +- docs/es/commercial/index.md | 5 +- docs/es/commercial/support.md | 24 +- docs/es/development/architecture.md | 20 +- docs/es/development/browse-code.md | 8 +- docs/es/development/build-cross-arm.md | 6 +- docs/es/development/build-cross-osx.md | 6 +- docs/es/development/build-osx.md | 8 +- docs/es/development/build.md | 22 +- docs/es/development/contrib.md | 4 +- docs/es/development/developer-instruction.md | 36 +- docs/es/development/index.md | 6 +- docs/es/development/style.md | 14 +- docs/es/development/tests.md | 57 +- docs/es/engines/database-engines/index.md | 6 +- docs/es/engines/database-engines/lazy.md | 4 +- docs/es/engines/database-engines/mysql.md | 12 +- docs/es/engines/index.md | 4 +- docs/es/engines/table-engines/index.md | 18 +- .../table-engines/integrations/hdfs.md | 4 +- .../table-engines/integrations/index.md | 4 +- .../table-engines/integrations/jdbc.md | 6 +- .../table-engines/integrations/kafka.md | 14 +- .../table-engines/integrations/mysql.md | 6 +- .../table-engines/integrations/odbc.md | 6 +- .../engines/table-engines/log-family/index.md | 4 +- .../table-engines/log-family/log-family.md | 6 +- .../engines/table-engines/log-family/log.md | 2 +- .../table-engines/log-family/stripelog.md | 12 +- .../table-engines/log-family/tinylog.md | 2 +- .../mergetree-family/aggregatingmergetree.md | 11 +- .../mergetree-family/collapsingmergetree.md | 15 +- .../custom-partitioning-key.md | 4 +- .../mergetree-family/graphitemergetree.md | 12 +- .../table-engines/mergetree-family/index.md | 4 +- .../mergetree-family/mergetree.md | 58 +- .../mergetree-family/replacingmergetree.md | 6 +- .../mergetree-family/replication.md | 16 +- .../mergetree-family/summingmergetree.md | 18 +- .../versionedcollapsingmergetree.md | 14 +- .../engines/table-engines/special/buffer.md | 2 +- .../table-engines/special/dictionary.md | 2 +- .../table-engines/special/distributed.md | 10 +- .../table-engines/special/external-data.md | 4 +- docs/es/engines/table-engines/special/file.md | 10 +- .../engines/table-engines/special/generate.md | 6 +- .../es/engines/table-engines/special/index.md | 4 +- docs/es/engines/table-engines/special/join.md | 20 +- .../table-engines/special/materializedview.md | 4 +- .../engines/table-engines/special/memory.md | 2 +- .../es/engines/table-engines/special/merge.md | 2 +- docs/es/engines/table-engines/special/null.md | 2 +- docs/es/engines/table-engines/special/set.md | 2 +- docs/es/engines/table-engines/special/url.md | 6 +- docs/es/engines/table-engines/special/view.md | 2 +- docs/es/faq/general.md | 20 +- docs/es/faq/index.md | 2 +- .../example-datasets/amplab-benchmark.md | 4 +- .../example-datasets/criteo.md | 4 +- .../getting-started/example-datasets/index.md | 4 +- .../example-datasets/metrica.md | 12 +- .../example-datasets/nyc-taxi.md | 10 +- .../example-datasets/ontime.md | 12 +- .../example-datasets/star-schema.md | 2 +- .../example-datasets/wikistat.md | 4 +- docs/es/getting-started/index.md | 6 +- docs/es/getting-started/install.md | 18 +- docs/es/getting-started/playground.md | 4 +- docs/es/getting-started/tutorial.md | 25 +- docs/es/guides/apply-catboost-model.md | 4 +- docs/es/guides/index.md | 6 +- docs/es/index.md | 10 +- docs/es/interfaces/cli.md | 12 +- docs/es/interfaces/cpp.md | 4 +- docs/es/interfaces/formats.md | 54 +- docs/es/interfaces/http.md | 352 ++- docs/es/interfaces/index.md | 4 +- docs/es/interfaces/jdbc.md | 2 +- docs/es/interfaces/mysql.md | 4 +- docs/es/interfaces/odbc.md | 2 +- docs/es/interfaces/tcp.md | 4 +- .../third-party/client-libraries.md | 81 +- docs/es/interfaces/third-party/gui.md | 20 +- docs/es/interfaces/third-party/index.md | 4 +- .../es/interfaces/third-party/integrations.md | 122 +- docs/es/interfaces/third-party/proxy.md | 6 +- docs/es/introduction/adopters.md | 144 +- docs/es/introduction/distinctive-features.md | 26 +- docs/es/introduction/history.md | 8 +- docs/es/introduction/index.md | 4 +- docs/es/introduction/performance.md | 10 +- docs/es/operations/access-rights.md | 190 +- docs/es/operations/backup.md | 12 +- docs/es/operations/configuration-files.md | 10 +- docs/es/operations/index.md | 6 +- docs/es/operations/monitoring.md | 10 +- .../optimizing-performance/index.md | 4 +- .../sampling-query-profiler.md | 4 +- docs/es/operations/performance-test.md | 10 +- docs/es/operations/quotas.md | 18 +- docs/es/operations/requirements.md | 14 +- .../server-configuration-parameters/index.md | 8 +- .../settings.md | 60 +- .../settings/constraints-on-settings.md | 4 +- docs/es/operations/settings/index.md | 9 +- .../settings/permissions-for-queries.md | 6 +- .../operations/settings/query-complexity.md | 48 +- .../operations/settings/settings-profiles.md | 20 +- docs/es/operations/settings/settings-users.md | 20 +- docs/es/operations/settings/settings.md | 163 +- docs/es/operations/system-tables.md | 169 +- docs/es/operations/tips.md | 20 +- docs/es/operations/troubleshooting.md | 16 +- docs/es/operations/update.md | 4 +- .../utilities/clickhouse-benchmark.md | 10 +- .../operations/utilities/clickhouse-copier.md | 10 +- .../operations/utilities/clickhouse-local.md | 4 +- docs/es/operations/utilities/index.md | 6 +- .../aggregate-functions/combinators.md | 101 +- .../aggregate-functions/index.md | 6 +- .../parametric-functions.md | 12 +- .../aggregate-functions/reference.md | 226 +- docs/es/sql-reference/ansi.md | 181 +- .../data-types/aggregatefunction.md | 10 +- docs/es/sql-reference/data-types/array.md | 6 +- docs/es/sql-reference/data-types/boolean.md | 4 +- docs/es/sql-reference/data-types/date.md | 2 +- docs/es/sql-reference/data-types/datetime.md | 6 +- .../es/sql-reference/data-types/datetime64.md | 4 +- docs/es/sql-reference/data-types/decimal.md | 10 +- .../sql-reference/data-types/domains/index.md | 4 +- .../sql-reference/data-types/domains/ipv4.md | 2 +- .../sql-reference/data-types/domains/ipv6.md | 2 +- .../data-types/domains/overview.md | 4 +- docs/es/sql-reference/data-types/enum.md | 6 +- .../sql-reference/data-types/fixedstring.md | 4 +- docs/es/sql-reference/data-types/float.md | 8 +- docs/es/sql-reference/data-types/index.md | 6 +- docs/es/sql-reference/data-types/int-uint.md | 4 +- .../nested-data-structures/index.md | 6 +- .../nested-data-structures/nested.md | 4 +- docs/es/sql-reference/data-types/nullable.md | 8 +- .../data-types/simpleaggregatefunction.md | 39 +- .../special-data-types/expression.md | 2 +- .../data-types/special-data-types/index.md | 6 +- .../data-types/special-data-types/interval.md | 8 +- .../data-types/special-data-types/nothing.md | 2 +- .../data-types/special-data-types/set.md | 4 +- docs/es/sql-reference/data-types/string.md | 2 +- docs/es/sql-reference/data-types/tuple.md | 8 +- docs/es/sql-reference/data-types/uuid.md | 6 +- .../external-dicts-dict-hierarchical.md | 2 +- .../external-dicts-dict-layout.md | 35 +- .../external-dicts-dict-lifetime.md | 9 +- .../external-dicts-dict-sources.md | 36 +- .../external-dicts-dict-structure.md | 6 +- .../external-dicts-dict.md | 4 +- .../external-dictionaries/external-dicts.md | 10 +- .../external-dictionaries/index.md | 4 +- docs/es/sql-reference/dictionaries/index.md | 6 +- .../dictionaries/internal-dicts.md | 6 +- .../functions/arithmetic-functions.md | 10 +- .../functions/array-functions.md | 44 +- docs/es/sql-reference/functions/array-join.md | 2 +- .../sql-reference/functions/bit-functions.md | 2 +- .../functions/bitmap-functions.md | 6 +- .../functions/comparison-functions.md | 10 +- .../functions/conditional-functions.md | 12 +- .../functions/date-time-functions.md | 58 +- .../functions/encoding-functions.md | 4 +- .../functions/ext-dict-functions.md | 10 +- .../functions/functions-for-nulls.md | 6 +- docs/es/sql-reference/functions/geo.md | 8 +- .../sql-reference/functions/hash-functions.md | 50 +- .../functions/higher-order-functions.md | 6 +- .../sql-reference/functions/in-functions.md | 10 +- docs/es/sql-reference/functions/index.md | 20 +- .../sql-reference/functions/introspection.md | 4 +- .../functions/ip-address-functions.md | 26 +- .../sql-reference/functions/json-functions.md | 80 +- .../functions/logical-functions.md | 2 +- .../functions/machine-learning-functions.md | 6 +- .../sql-reference/functions/math-functions.md | 6 +- .../functions/other-functions.md | 146 +- .../functions/random-functions.md | 41 +- .../functions/rounding-functions.md | 6 +- .../functions/splitting-merging-functions.md | 14 +- .../functions/string-functions.md | 14 +- .../functions/string-replace-functions.md | 8 +- .../functions/string-search-functions.md | 4 +- .../functions/type-conversion-functions.md | 16 +- .../sql-reference/functions/url-functions.md | 12 +- .../sql-reference/functions/uuid-functions.md | 4 +- .../functions/ym-dict-functions.md | 14 +- docs/es/sql-reference/index.md | 10 +- docs/es/sql-reference/operators/in.md | 204 ++ .../{operators.md => operators/index.md} | 59 +- docs/es/sql-reference/statements/alter.md | 127 +- docs/es/sql-reference/statements/create.md | 231 +- docs/es/sql-reference/statements/grant.md | 476 +++ docs/es/sql-reference/statements/index.md | 4 +- .../sql-reference/statements/insert-into.md | 10 +- docs/es/sql-reference/statements/misc.md | 118 +- docs/es/sql-reference/statements/revoke.md | 50 + docs/es/sql-reference/statements/select.md | 1379 --------- .../statements/select/array-join.md | 282 ++ .../statements/select/distinct.md | 63 + .../sql-reference/statements/select/format.md | 18 + .../sql-reference/statements/select/from.md | 44 + .../statements/select/group-by.md | 132 + .../sql-reference/statements/select/having.md | 14 + .../sql-reference/statements/select/index.md | 158 + .../statements/select/into-outfile.md | 14 + .../sql-reference/statements/select/join.md | 191 ++ .../statements/select/limit-by.md | 71 + .../sql-reference/statements/select/limit.md | 14 + .../statements/select/order-by.md | 72 + .../statements/select/prewhere.md | 22 + .../sql-reference/statements/select/sample.md | 113 + .../statements/select/union-all.md | 35 + .../sql-reference/statements/select/where.md | 15 + .../sql-reference/statements/select/with.md | 80 + docs/es/sql-reference/statements/show.md | 68 +- docs/es/sql-reference/statements/system.md | 6 +- docs/es/sql-reference/syntax.md | 58 +- docs/es/sql-reference/table-functions/file.md | 2 +- .../sql-reference/table-functions/generate.md | 7 +- docs/es/sql-reference/table-functions/hdfs.md | 2 +- .../es/sql-reference/table-functions/index.md | 8 +- .../es/sql-reference/table-functions/input.md | 2 +- docs/es/sql-reference/table-functions/jdbc.md | 2 +- .../es/sql-reference/table-functions/merge.md | 2 +- .../es/sql-reference/table-functions/mysql.md | 4 +- .../sql-reference/table-functions/numbers.md | 2 +- docs/es/sql-reference/table-functions/odbc.md | 4 +- .../sql-reference/table-functions/remote.md | 2 +- docs/es/sql-reference/table-functions/url.md | 2 +- docs/es/whats-new/changelog/2017.md | 58 +- docs/es/whats-new/changelog/2018.md | 210 +- docs/es/whats-new/changelog/2019.md | 382 +-- docs/es/whats-new/changelog/index.md | 663 +--- docs/es/whats-new/index.md | 4 +- docs/es/whats-new/roadmap.md | 4 +- docs/es/whats-new/security-changelog.md | 16 +- docs/fa/commercial/cloud.md | 8 +- docs/fa/commercial/index.md | 5 +- docs/fa/commercial/support.md | 24 +- docs/fa/development/architecture.md | 8 +- docs/fa/development/browse-code.md | 13 +- docs/fa/development/build-cross-arm.md | 8 +- docs/fa/development/build-cross-osx.md | 2 +- docs/fa/development/build-osx.md | 2 +- docs/fa/development/build.md | 2 +- docs/fa/development/contrib.md | 2 +- docs/fa/development/developer-instruction.md | 26 +- docs/fa/development/index.md | 4 +- docs/fa/development/style.md | 12 +- docs/fa/development/tests.md | 47 +- docs/fa/engines/database-engines/index.md | 5 +- docs/fa/engines/database-engines/lazy.md | 2 +- docs/fa/engines/database-engines/mysql.md | 6 +- docs/fa/engines/index.md | 4 +- docs/fa/engines/table-engines/index.md | 9 +- .../table-engines/integrations/hdfs.md | 2 +- .../table-engines/integrations/index.md | 4 +- .../table-engines/integrations/jdbc.md | 2 +- .../table-engines/integrations/kafka.md | 16 +- .../table-engines/integrations/mysql.md | 4 +- .../table-engines/integrations/odbc.md | 4 +- .../engines/table-engines/log-family/index.md | 4 +- .../table-engines/log-family/log-family.md | 2 +- .../engines/table-engines/log-family/log.md | 2 +- .../table-engines/log-family/stripelog.md | 4 +- .../table-engines/log-family/tinylog.md | 2 +- .../mergetree-family/aggregatingmergetree.md | 7 +- .../mergetree-family/collapsingmergetree.md | 5 +- .../custom-partitioning-key.md | 2 +- .../mergetree-family/graphitemergetree.md | 2 +- .../table-engines/mergetree-family/index.md | 5 +- .../mergetree-family/mergetree.md | 12 +- .../mergetree-family/replacingmergetree.md | 2 +- .../mergetree-family/replication.md | 2 +- .../mergetree-family/summingmergetree.md | 2 +- .../versionedcollapsingmergetree.md | 4 +- .../engines/table-engines/special/buffer.md | 2 +- .../table-engines/special/dictionary.md | 4 +- .../table-engines/special/distributed.md | 12 +- .../table-engines/special/external-data.md | 2 +- docs/fa/engines/table-engines/special/file.md | 4 +- .../engines/table-engines/special/generate.md | 2 +- .../fa/engines/table-engines/special/index.md | 4 +- docs/fa/engines/table-engines/special/join.md | 10 +- .../table-engines/special/materializedview.md | 2 +- .../engines/table-engines/special/memory.md | 2 +- .../fa/engines/table-engines/special/merge.md | 6 +- docs/fa/engines/table-engines/special/null.md | 2 +- docs/fa/engines/table-engines/special/set.md | 2 +- docs/fa/engines/table-engines/special/url.md | 4 +- docs/fa/engines/table-engines/special/view.md | 2 +- docs/fa/faq/general.md | 10 +- docs/fa/faq/index.md | 2 +- .../example-datasets/amplab-benchmark.md | 2 +- .../example-datasets/criteo.md | 2 +- .../getting-started/example-datasets/index.md | 5 +- .../example-datasets/metrica.md | 4 +- .../example-datasets/nyc-taxi.md | 4 +- .../example-datasets/ontime.md | 2 +- .../example-datasets/star-schema.md | 2 +- .../example-datasets/wikistat.md | 2 +- docs/fa/getting-started/index.md | 4 +- docs/fa/getting-started/install.md | 14 +- docs/fa/getting-started/playground.md | 2 +- docs/fa/getting-started/tutorial.md | 11 +- docs/fa/guides/apply-catboost-model.md | 4 +- docs/fa/guides/index.md | 4 +- docs/fa/interfaces/cli.md | 2 +- docs/fa/interfaces/cpp.md | 4 +- docs/fa/interfaces/formats.md | 12 +- docs/fa/interfaces/http.md | 344 ++- docs/fa/interfaces/index.md | 6 +- docs/fa/interfaces/jdbc.md | 2 +- docs/fa/interfaces/mysql.md | 4 +- docs/fa/interfaces/odbc.md | 2 +- docs/fa/interfaces/tcp.md | 4 +- .../third-party/client-libraries.md | 5 +- docs/fa/interfaces/third-party/gui.md | 8 +- docs/fa/interfaces/third-party/index.md | 4 +- .../fa/interfaces/third-party/integrations.md | 13 +- docs/fa/interfaces/third-party/proxy.md | 2 +- docs/fa/introduction/adopters.md | 142 +- docs/fa/introduction/history.md | 8 +- docs/fa/introduction/index.md | 4 +- docs/fa/operations/access-rights.md | 191 +- docs/fa/operations/backup.md | 2 +- docs/fa/operations/configuration-files.md | 4 +- docs/fa/operations/index.md | 6 +- docs/fa/operations/monitoring.md | 4 +- .../optimizing-performance/index.md | 5 +- .../sampling-query-profiler.md | 2 +- docs/fa/operations/performance-test.md | 10 +- docs/fa/operations/quotas.md | 18 +- docs/fa/operations/requirements.md | 16 +- .../server-configuration-parameters/index.md | 7 +- .../settings.md | 46 +- .../settings/constraints-on-settings.md | 2 +- docs/fa/operations/settings/index.md | 9 +- .../settings/permissions-for-queries.md | 2 +- .../operations/settings/query-complexity.md | 22 +- .../operations/settings/settings-profiles.md | 18 +- docs/fa/operations/settings/settings-users.md | 18 +- docs/fa/operations/settings/settings.md | 125 +- docs/fa/operations/system-tables.md | 165 +- docs/fa/operations/tips.md | 6 +- docs/fa/operations/troubleshooting.md | 2 +- docs/fa/operations/update.md | 2 +- .../utilities/clickhouse-benchmark.md | 4 +- .../operations/utilities/clickhouse-copier.md | 2 +- .../operations/utilities/clickhouse-local.md | 4 +- docs/fa/operations/utilities/index.md | 4 +- .../aggregate-functions/combinators.md | 106 +- .../aggregate-functions/index.md | 4 +- .../parametric-functions.md | 15 +- .../aggregate-functions/reference.md | 244 +- docs/fa/sql-reference/ansi.md | 181 +- .../data-types/aggregatefunction.md | 6 +- docs/fa/sql-reference/data-types/array.md | 4 +- docs/fa/sql-reference/data-types/boolean.md | 2 +- docs/fa/sql-reference/data-types/date.md | 2 +- docs/fa/sql-reference/data-types/datetime.md | 8 +- .../fa/sql-reference/data-types/datetime64.md | 6 +- docs/fa/sql-reference/data-types/decimal.md | 10 +- .../sql-reference/data-types/domains/index.md | 4 +- .../sql-reference/data-types/domains/ipv4.md | 4 +- .../sql-reference/data-types/domains/ipv6.md | 2 +- .../data-types/domains/overview.md | 2 +- docs/fa/sql-reference/data-types/enum.md | 6 +- .../sql-reference/data-types/fixedstring.md | 4 +- docs/fa/sql-reference/data-types/float.md | 4 +- docs/fa/sql-reference/data-types/index.md | 4 +- docs/fa/sql-reference/data-types/int-uint.md | 2 +- .../nested-data-structures/index.md | 5 +- .../nested-data-structures/nested.md | 2 +- docs/fa/sql-reference/data-types/nullable.md | 2 +- .../data-types/simpleaggregatefunction.md | 39 +- .../special-data-types/expression.md | 2 +- .../data-types/special-data-types/index.md | 5 +- .../data-types/special-data-types/interval.md | 12 +- .../data-types/special-data-types/nothing.md | 4 +- .../data-types/special-data-types/set.md | 4 +- docs/fa/sql-reference/data-types/string.md | 2 +- docs/fa/sql-reference/data-types/tuple.md | 4 +- docs/fa/sql-reference/data-types/uuid.md | 2 +- .../external-dicts-dict-hierarchical.md | 2 +- .../external-dicts-dict-layout.md | 27 +- .../external-dicts-dict-lifetime.md | 7 +- .../external-dicts-dict-sources.md | 28 +- .../external-dicts-dict-structure.md | 2 +- .../external-dicts-dict.md | 2 +- .../external-dictionaries/external-dicts.md | 8 +- .../external-dictionaries/index.md | 5 +- docs/fa/sql-reference/dictionaries/index.md | 6 +- .../dictionaries/internal-dicts.md | 4 +- .../functions/arithmetic-functions.md | 4 +- .../functions/array-functions.md | 46 +- docs/fa/sql-reference/functions/array-join.md | 2 +- .../sql-reference/functions/bit-functions.md | 4 +- .../functions/bitmap-functions.md | 12 +- .../functions/comparison-functions.md | 2 +- .../functions/conditional-functions.md | 6 +- .../functions/date-time-functions.md | 24 +- .../functions/encoding-functions.md | 4 +- .../functions/ext-dict-functions.md | 8 +- .../functions/functions-for-nulls.md | 4 +- docs/fa/sql-reference/functions/geo.md | 10 +- .../sql-reference/functions/hash-functions.md | 46 +- .../functions/higher-order-functions.md | 2 +- .../sql-reference/functions/in-functions.md | 4 +- docs/fa/sql-reference/functions/index.md | 10 +- .../sql-reference/functions/introspection.md | 22 +- .../functions/ip-address-functions.md | 14 +- .../sql-reference/functions/json-functions.md | 88 +- .../functions/logical-functions.md | 2 +- .../functions/machine-learning-functions.md | 2 +- .../sql-reference/functions/math-functions.md | 4 +- .../functions/other-functions.md | 144 +- .../functions/random-functions.md | 45 +- .../functions/rounding-functions.md | 8 +- .../functions/splitting-merging-functions.md | 6 +- .../functions/string-functions.md | 26 +- .../functions/string-replace-functions.md | 4 +- .../functions/string-search-functions.md | 6 +- .../functions/type-conversion-functions.md | 16 +- .../sql-reference/functions/url-functions.md | 8 +- .../sql-reference/functions/uuid-functions.md | 4 +- .../functions/ym-dict-functions.md | 4 +- docs/fa/sql-reference/index.md | 10 +- docs/fa/sql-reference/operators/in.md | 204 ++ .../{operators.md => operators/index.md} | 31 +- docs/fa/sql-reference/statements/alter.md | 109 +- docs/fa/sql-reference/statements/create.md | 231 +- docs/fa/sql-reference/statements/grant.md | 476 +++ docs/fa/sql-reference/statements/index.md | 4 +- .../sql-reference/statements/insert-into.md | 2 +- docs/fa/sql-reference/statements/misc.md | 132 +- docs/fa/sql-reference/statements/revoke.md | 50 + docs/fa/sql-reference/statements/select.md | 1379 --------- .../statements/select/array-join.md | 282 ++ .../statements/select/distinct.md | 63 + .../sql-reference/statements/select/format.md | 18 + .../sql-reference/statements/select/from.md | 44 + .../statements/select/group-by.md | 132 + .../sql-reference/statements/select/having.md | 14 + .../sql-reference/statements/select/index.md | 158 + .../statements/select/into-outfile.md | 14 + .../sql-reference/statements/select/join.md | 191 ++ .../statements/select/limit-by.md | 71 + .../sql-reference/statements/select/limit.md | 14 + .../statements/select/order-by.md | 72 + .../statements/select/prewhere.md | 22 + .../sql-reference/statements/select/sample.md | 113 + .../statements/select/union-all.md | 35 + .../sql-reference/statements/select/where.md | 15 + .../sql-reference/statements/select/with.md | 80 + docs/fa/sql-reference/statements/show.md | 66 +- docs/fa/sql-reference/statements/system.md | 4 +- docs/fa/sql-reference/syntax.md | 48 +- docs/fa/sql-reference/table-functions/file.md | 4 +- .../sql-reference/table-functions/generate.md | 5 +- docs/fa/sql-reference/table-functions/hdfs.md | 4 +- .../fa/sql-reference/table-functions/index.md | 6 +- .../fa/sql-reference/table-functions/input.md | 2 +- docs/fa/sql-reference/table-functions/jdbc.md | 4 +- .../fa/sql-reference/table-functions/merge.md | 2 +- .../fa/sql-reference/table-functions/mysql.md | 2 +- .../sql-reference/table-functions/numbers.md | 2 +- docs/fa/sql-reference/table-functions/odbc.md | 4 +- .../sql-reference/table-functions/remote.md | 6 +- docs/fa/sql-reference/table-functions/url.md | 2 +- docs/fa/whats-new/changelog/2017.md | 40 +- docs/fa/whats-new/changelog/2018.md | 122 +- docs/fa/whats-new/changelog/2019.md | 178 +- docs/fa/whats-new/changelog/index.md | 665 +--- docs/fa/whats-new/index.md | 4 +- docs/fa/whats-new/roadmap.md | 2 +- docs/fa/whats-new/security-changelog.md | 18 +- docs/fr/commercial/cloud.md | 8 +- docs/fr/commercial/index.md | 3 +- docs/fr/commercial/support.md | 24 +- docs/fr/development/architecture.md | 142 +- docs/fr/development/browse-code.md | 10 +- docs/fr/development/build-cross-arm.md | 12 +- docs/fr/development/build-cross-osx.md | 12 +- docs/fr/development/build-osx.md | 12 +- docs/fr/development/build.md | 28 +- docs/fr/development/contrib.md | 2 +- docs/fr/development/developer-instruction.md | 108 +- docs/fr/development/index.md | 4 +- docs/fr/development/style.md | 226 +- docs/fr/development/tests.md | 123 +- docs/fr/engines/database-engines/index.md | 4 +- docs/fr/engines/database-engines/lazy.md | 4 +- docs/fr/engines/database-engines/mysql.md | 10 +- docs/fr/engines/index.md | 4 +- docs/fr/engines/table-engines/index.md | 20 +- .../table-engines/integrations/hdfs.md | 16 +- .../table-engines/integrations/index.md | 4 +- .../table-engines/integrations/jdbc.md | 10 +- .../table-engines/integrations/kafka.md | 38 +- .../table-engines/integrations/mysql.md | 18 +- .../table-engines/integrations/odbc.md | 14 +- .../engines/table-engines/log-family/index.md | 4 +- .../table-engines/log-family/log-family.md | 16 +- .../engines/table-engines/log-family/log.md | 8 +- .../table-engines/log-family/stripelog.md | 18 +- .../table-engines/log-family/tinylog.md | 4 +- .../mergetree-family/aggregatingmergetree.md | 31 +- .../mergetree-family/collapsingmergetree.md | 85 +- .../custom-partitioning-key.md | 28 +- .../mergetree-family/graphitemergetree.md | 16 +- .../table-engines/mergetree-family/index.md | 4 +- .../mergetree-family/mergetree.md | 196 +- .../mergetree-family/replacingmergetree.md | 18 +- .../mergetree-family/replication.md | 98 +- .../mergetree-family/summingmergetree.md | 46 +- .../versionedcollapsingmergetree.md | 72 +- .../engines/table-engines/special/buffer.md | 34 +- .../table-engines/special/dictionary.md | 10 +- .../table-engines/special/distributed.md | 52 +- .../table-engines/special/external-data.md | 16 +- docs/fr/engines/table-engines/special/file.md | 28 +- .../engines/table-engines/special/generate.md | 8 +- .../fr/engines/table-engines/special/index.md | 4 +- docs/fr/engines/table-engines/special/join.md | 30 +- .../table-engines/special/materializedview.md | 4 +- .../engines/table-engines/special/memory.md | 12 +- .../fr/engines/table-engines/special/merge.md | 18 +- docs/fr/engines/table-engines/special/null.md | 4 +- docs/fr/engines/table-engines/special/set.md | 10 +- docs/fr/engines/table-engines/special/url.md | 12 +- docs/fr/engines/table-engines/special/view.md | 4 +- docs/fr/faq/general.md | 22 +- docs/fr/faq/index.md | 2 +- .../example-datasets/amplab-benchmark.md | 4 +- .../example-datasets/criteo.md | 4 +- .../getting-started/example-datasets/index.md | 4 +- .../example-datasets/metrica.md | 12 +- .../example-datasets/nyc-taxi.md | 38 +- .../example-datasets/ontime.md | 16 +- .../example-datasets/star-schema.md | 2 +- .../example-datasets/wikistat.md | 4 +- docs/fr/getting-started/index.md | 6 +- docs/fr/getting-started/install.md | 60 +- docs/fr/getting-started/playground.md | 14 +- docs/fr/getting-started/tutorial.md | 63 +- docs/fr/guides/apply-catboost-model.md | 48 +- docs/fr/guides/index.md | 6 +- docs/fr/index.md | 46 +- docs/fr/interfaces/cli.md | 46 +- docs/fr/interfaces/cpp.md | 4 +- docs/fr/interfaces/formats.md | 250 +- docs/fr/interfaces/http.md | 430 ++- docs/fr/interfaces/index.md | 8 +- docs/fr/interfaces/jdbc.md | 2 +- docs/fr/interfaces/mysql.md | 6 +- docs/fr/interfaces/odbc.md | 2 +- docs/fr/interfaces/tcp.md | 4 +- .../third-party/client-libraries.md | 11 +- docs/fr/interfaces/third-party/gui.md | 28 +- docs/fr/interfaces/third-party/index.md | 4 +- .../fr/interfaces/third-party/integrations.md | 17 +- docs/fr/interfaces/third-party/proxy.md | 10 +- docs/fr/introduction/adopters.md | 144 +- docs/fr/introduction/distinctive-features.md | 58 +- docs/fr/introduction/history.md | 44 +- docs/fr/introduction/index.md | 2 +- docs/fr/introduction/performance.md | 20 +- docs/fr/operations/access-rights.md | 190 +- docs/fr/operations/backup.md | 18 +- docs/fr/operations/configuration-files.md | 16 +- docs/fr/operations/index.md | 12 +- docs/fr/operations/monitoring.md | 22 +- .../optimizing-performance/index.md | 4 +- .../sampling-query-profiler.md | 18 +- docs/fr/operations/performance-test.md | 12 +- docs/fr/operations/quotas.md | 24 +- docs/fr/operations/requirements.md | 28 +- .../server-configuration-parameters/index.md | 8 +- .../settings.md | 134 +- .../settings/constraints-on-settings.md | 10 +- docs/fr/operations/settings/index.md | 15 +- .../settings/permissions-for-queries.md | 10 +- .../operations/settings/query-complexity.md | 122 +- .../operations/settings/settings-profiles.md | 22 +- docs/fr/operations/settings/settings-users.md | 52 +- docs/fr/operations/settings/settings.md | 389 ++- docs/fr/operations/system-tables.md | 321 +- docs/fr/operations/tips.md | 52 +- docs/fr/operations/troubleshooting.md | 38 +- docs/fr/operations/update.md | 4 +- .../utilities/clickhouse-benchmark.md | 22 +- .../operations/utilities/clickhouse-copier.md | 12 +- .../operations/utilities/clickhouse-local.md | 10 +- docs/fr/operations/utilities/index.md | 4 +- .../aggregate-functions/combinators.md | 131 +- .../aggregate-functions/index.md | 16 +- .../parametric-functions.md | 88 +- .../aggregate-functions/reference.md | 450 ++- docs/fr/sql-reference/ansi.md | 181 +- .../data-types/aggregatefunction.md | 22 +- docs/fr/sql-reference/data-types/array.md | 14 +- docs/fr/sql-reference/data-types/boolean.md | 4 +- docs/fr/sql-reference/data-types/date.md | 4 +- docs/fr/sql-reference/data-types/datetime.md | 24 +- .../fr/sql-reference/data-types/datetime64.md | 14 +- docs/fr/sql-reference/data-types/decimal.md | 22 +- .../sql-reference/data-types/domains/index.md | 4 +- .../sql-reference/data-types/domains/ipv4.md | 6 +- .../sql-reference/data-types/domains/ipv6.md | 8 +- .../data-types/domains/overview.md | 20 +- docs/fr/sql-reference/data-types/enum.md | 30 +- .../sql-reference/data-types/fixedstring.md | 12 +- docs/fr/sql-reference/data-types/float.md | 16 +- docs/fr/sql-reference/data-types/index.md | 4 +- docs/fr/sql-reference/data-types/int-uint.md | 2 +- .../nested-data-structures/index.md | 4 +- .../nested-data-structures/nested.md | 18 +- docs/fr/sql-reference/data-types/nullable.md | 10 +- .../data-types/simpleaggregatefunction.md | 39 +- .../special-data-types/expression.md | 4 +- .../data-types/special-data-types/index.md | 6 +- .../data-types/special-data-types/interval.md | 18 +- .../data-types/special-data-types/nothing.md | 4 +- .../data-types/special-data-types/set.md | 4 +- docs/fr/sql-reference/data-types/string.md | 14 +- docs/fr/sql-reference/data-types/tuple.md | 16 +- docs/fr/sql-reference/data-types/uuid.md | 18 +- .../external-dicts-dict-hierarchical.md | 6 +- .../external-dicts-dict-layout.md | 68 +- .../external-dicts-dict-lifetime.md | 19 +- .../external-dicts-dict-sources.md | 40 +- .../external-dicts-dict-structure.md | 24 +- .../external-dicts-dict.md | 6 +- .../external-dictionaries/external-dicts.md | 20 +- .../external-dictionaries/index.md | 4 +- docs/fr/sql-reference/dictionaries/index.md | 8 +- .../dictionaries/internal-dicts.md | 32 +- .../functions/arithmetic-functions.md | 24 +- .../functions/array-functions.md | 142 +- docs/fr/sql-reference/functions/array-join.md | 8 +- .../sql-reference/functions/bit-functions.md | 14 +- .../functions/bitmap-functions.md | 22 +- .../functions/comparison-functions.md | 4 +- .../functions/conditional-functions.md | 22 +- .../functions/date-time-functions.md | 116 +- .../functions/encoding-functions.md | 26 +- .../functions/ext-dict-functions.md | 48 +- .../functions/functions-for-nulls.md | 36 +- docs/fr/sql-reference/functions/geo.md | 60 +- .../sql-reference/functions/hash-functions.md | 112 +- .../functions/higher-order-functions.md | 36 +- .../sql-reference/functions/in-functions.md | 14 +- docs/fr/sql-reference/functions/index.md | 48 +- .../sql-reference/functions/introspection.md | 46 +- .../functions/ip-address-functions.md | 24 +- .../sql-reference/functions/json-functions.md | 126 +- .../functions/logical-functions.md | 2 +- .../functions/machine-learning-functions.md | 8 +- .../sql-reference/functions/math-functions.md | 24 +- .../functions/other-functions.md | 246 +- .../functions/random-functions.md | 45 +- .../functions/rounding-functions.md | 44 +- .../functions/splitting-merging-functions.md | 16 +- .../functions/string-functions.md | 76 +- .../functions/string-replace-functions.md | 18 +- .../functions/string-search-functions.md | 62 +- .../functions/type-conversion-functions.md | 88 +- .../sql-reference/functions/url-functions.md | 66 +- .../sql-reference/functions/uuid-functions.md | 16 +- .../functions/ym-dict-functions.md | 24 +- docs/fr/sql-reference/index.md | 10 +- docs/fr/sql-reference/operators/in.md | 204 ++ .../{operators.md => operators/index.md} | 57 +- docs/fr/sql-reference/statements/alter.md | 267 +- docs/fr/sql-reference/statements/create.md | 307 +- docs/fr/sql-reference/statements/grant.md | 476 +++ docs/fr/sql-reference/statements/index.md | 4 +- .../sql-reference/statements/insert-into.md | 20 +- docs/fr/sql-reference/statements/misc.md | 188 +- docs/fr/sql-reference/statements/revoke.md | 50 + docs/fr/sql-reference/statements/select.md | 1379 --------- .../statements/select/array-join.md | 282 ++ .../statements/select/distinct.md | 63 + .../sql-reference/statements/select/format.md | 18 + .../sql-reference/statements/select/from.md | 44 + .../statements/select/group-by.md | 132 + .../sql-reference/statements/select/having.md | 14 + .../sql-reference/statements/select/index.md | 158 + .../statements/select/into-outfile.md | 14 + .../sql-reference/statements/select/join.md | 191 ++ .../statements/select/limit-by.md | 71 + .../sql-reference/statements/select/limit.md | 14 + .../statements/select/order-by.md | 72 + .../statements/select/prewhere.md | 22 + .../sql-reference/statements/select/sample.md | 113 + .../statements/select/union-all.md | 35 + .../sql-reference/statements/select/where.md | 15 + .../sql-reference/statements/select/with.md | 80 + docs/fr/sql-reference/statements/show.md | 80 +- docs/fr/sql-reference/statements/system.md | 26 +- docs/fr/sql-reference/syntax.md | 102 +- docs/fr/sql-reference/table-functions/file.md | 12 +- .../sql-reference/table-functions/generate.md | 7 +- docs/fr/sql-reference/table-functions/hdfs.md | 8 +- .../fr/sql-reference/table-functions/index.md | 8 +- .../fr/sql-reference/table-functions/input.md | 8 +- docs/fr/sql-reference/table-functions/jdbc.md | 4 +- .../fr/sql-reference/table-functions/merge.md | 4 +- .../fr/sql-reference/table-functions/mysql.md | 8 +- .../sql-reference/table-functions/numbers.md | 2 +- docs/fr/sql-reference/table-functions/odbc.md | 12 +- .../sql-reference/table-functions/remote.md | 24 +- docs/fr/sql-reference/table-functions/url.md | 2 +- docs/fr/whats-new/changelog/2017.md | 196 +- docs/fr/whats-new/changelog/2018.md | 874 +++--- docs/fr/whats-new/changelog/2019.md | 1316 ++++---- docs/fr/whats-new/changelog/index.md | 663 +--- docs/fr/whats-new/index.md | 4 +- docs/fr/whats-new/roadmap.md | 8 +- docs/fr/whats-new/security-changelog.md | 34 +- docs/ja/commercial/cloud.md | 16 +- docs/ja/commercial/index.md | 5 +- docs/ja/development/architecture.md | 164 +- docs/ja/development/browse-code.md | 10 +- docs/ja/development/build-cross-arm.md | 22 +- docs/ja/development/build-cross-osx.md | 24 +- docs/ja/development/build-osx.md | 18 +- docs/ja/development/build.md | 54 +- docs/ja/development/contrib.md | 72 +- docs/ja/development/developer-instruction.md | 159 +- docs/ja/development/index.md | 6 +- docs/ja/development/style.md | 324 +- docs/ja/development/tests.md | 197 +- docs/ja/engines/database-engines/index.md | 10 +- docs/ja/engines/database-engines/lazy.md | 4 +- docs/ja/engines/database-engines/mysql.md | 14 +- docs/ja/engines/index.md | 4 +- docs/ja/engines/table-engines/index.md | 52 +- .../table-engines/integrations/hdfs.md | 38 +- .../table-engines/integrations/index.md | 4 +- .../table-engines/integrations/jdbc.md | 16 +- .../table-engines/integrations/kafka.md | 50 +- .../table-engines/integrations/mysql.md | 26 +- .../table-engines/integrations/odbc.md | 24 +- .../engines/table-engines/log-family/index.md | 4 +- .../table-engines/log-family/log-family.md | 26 +- .../engines/table-engines/log-family/log.md | 10 +- .../table-engines/log-family/stripelog.md | 22 +- .../table-engines/log-family/tinylog.md | 8 +- .../mergetree-family/aggregatingmergetree.md | 38 +- .../mergetree-family/collapsingmergetree.md | 89 +- .../custom-partitioning-key.md | 44 +- .../mergetree-family/graphitemergetree.md | 37 +- .../table-engines/mergetree-family/index.md | 4 +- .../mergetree-family/mergetree.md | 316 +- .../mergetree-family/replacingmergetree.md | 29 +- .../mergetree-family/replication.md | 148 +- .../mergetree-family/summingmergetree.md | 48 +- .../versionedcollapsingmergetree.md | 79 +- .../engines/table-engines/special/buffer.md | 46 +- .../table-engines/special/dictionary.md | 14 +- .../table-engines/special/distributed.md | 96 +- .../table-engines/special/external-data.md | 32 +- docs/ja/engines/table-engines/special/file.md | 44 +- .../engines/table-engines/special/generate.md | 22 +- .../ja/engines/table-engines/special/index.md | 4 +- docs/ja/engines/table-engines/special/join.md | 34 +- .../table-engines/special/materializedview.md | 8 +- .../engines/table-engines/special/memory.md | 10 +- .../ja/engines/table-engines/special/merge.md | 28 +- docs/ja/engines/table-engines/special/null.md | 8 +- docs/ja/engines/table-engines/special/set.md | 12 +- docs/ja/engines/table-engines/special/url.md | 36 +- docs/ja/engines/table-engines/special/view.md | 8 +- docs/ja/faq/general.md | 24 +- docs/ja/faq/index.md | 2 +- .../example-datasets/amplab-benchmark.md | 8 +- .../example-datasets/criteo.md | 11 +- .../getting-started/example-datasets/index.md | 16 +- .../example-datasets/metrica.md | 26 +- .../example-datasets/nyc-taxi.md | 78 +- .../example-datasets/ontime.md | 22 +- .../example-datasets/star-schema.md | 11 +- .../example-datasets/wikistat.md | 6 +- docs/ja/getting-started/index.md | 8 +- docs/ja/getting-started/install.md | 68 +- docs/ja/getting-started/playground.md | 38 +- docs/ja/getting-started/tutorial.md | 109 +- docs/ja/guides/apply-catboost-model.md | 72 +- docs/ja/guides/index.md | 10 +- docs/ja/interfaces/cli.md | 74 +- docs/ja/interfaces/cpp.md | 8 +- docs/ja/interfaces/formats.md | 553 ++-- docs/ja/interfaces/http.md | 454 +-- docs/ja/interfaces/index.md | 24 +- docs/ja/interfaces/jdbc.md | 4 +- docs/ja/interfaces/mysql.md | 12 +- docs/ja/interfaces/odbc.md | 2 +- docs/ja/interfaces/tcp.md | 9 +- .../third-party/client-libraries.md | 49 +- docs/ja/interfaces/third-party/gui.md | 108 +- docs/ja/interfaces/third-party/index.md | 4 +- .../ja/interfaces/third-party/integrations.md | 69 +- docs/ja/interfaces/third-party/proxy.md | 26 +- docs/ja/introduction/adopters.md | 148 +- docs/ja/introduction/index.md | 4 +- docs/ja/operations/access-rights.md | 191 +- docs/ja/operations/backup.md | 34 +- docs/ja/operations/configuration-files.md | 32 +- docs/ja/operations/index.md | 16 +- docs/ja/operations/monitoring.md | 28 +- .../optimizing-performance/index.md | 4 +- .../sampling-query-profiler.md | 36 +- docs/ja/operations/performance-test.md | 20 +- docs/ja/operations/quotas.md | 32 +- docs/ja/operations/requirements.md | 38 +- .../server-configuration-parameters/index.md | 12 +- .../settings.md | 350 ++- .../settings/constraints-on-settings.md | 10 +- docs/ja/operations/settings/index.md | 31 +- .../settings/permissions-for-queries.md | 22 +- .../operations/settings/query-complexity.md | 156 +- .../operations/settings/settings-profiles.md | 24 +- docs/ja/operations/settings/settings-users.md | 62 +- docs/ja/operations/settings/settings.md | 663 ++-- docs/ja/operations/system-tables.md | 585 ++-- docs/ja/operations/tips.md | 90 +- docs/ja/operations/troubleshooting.md | 48 +- docs/ja/operations/update.md | 12 +- .../utilities/clickhouse-benchmark.md | 44 +- .../operations/utilities/clickhouse-copier.md | 28 +- .../operations/utilities/clickhouse-local.md | 28 +- docs/ja/operations/utilities/index.md | 10 +- .../aggregate-functions/combinators.md | 153 +- .../aggregate-functions/index.md | 22 +- .../parametric-functions.md | 116 +- .../aggregate-functions/reference.md | 730 +++-- docs/ja/sql-reference/ansi.md | 181 +- .../data-types/aggregatefunction.md | 26 +- docs/ja/sql-reference/data-types/array.md | 10 +- docs/ja/sql-reference/data-types/boolean.md | 4 +- docs/ja/sql-reference/data-types/date.md | 6 +- docs/ja/sql-reference/data-types/datetime.md | 38 +- .../ja/sql-reference/data-types/datetime64.md | 26 +- docs/ja/sql-reference/data-types/decimal.md | 52 +- .../sql-reference/data-types/domains/index.md | 4 +- .../sql-reference/data-types/domains/ipv4.md | 12 +- .../sql-reference/data-types/domains/ipv6.md | 14 +- .../data-types/domains/overview.md | 18 +- docs/ja/sql-reference/data-types/enum.md | 48 +- .../sql-reference/data-types/fixedstring.md | 34 +- docs/ja/sql-reference/data-types/float.md | 22 +- docs/ja/sql-reference/data-types/index.md | 6 +- docs/ja/sql-reference/data-types/int-uint.md | 6 +- .../nested-data-structures/index.md | 8 +- .../nested-data-structures/nested.md | 28 +- docs/ja/sql-reference/data-types/nullable.md | 16 +- .../special-data-types/expression.md | 4 +- .../data-types/special-data-types/index.md | 10 +- .../data-types/special-data-types/interval.md | 16 +- .../data-types/special-data-types/nothing.md | 8 +- .../data-types/special-data-types/set.md | 4 +- docs/ja/sql-reference/data-types/string.md | 10 +- docs/ja/sql-reference/data-types/tuple.md | 14 +- docs/ja/sql-reference/data-types/uuid.md | 16 +- .../external-dicts-dict-hierarchical.md | 12 +- .../external-dicts-dict-layout.md | 129 +- .../external-dicts-dict-lifetime.md | 27 +- .../external-dicts-dict-sources.md | 80 +- .../external-dicts-dict-structure.md | 52 +- .../external-dicts-dict.md | 12 +- .../external-dictionaries/external-dicts.md | 32 +- .../external-dictionaries/index.md | 4 +- docs/ja/sql-reference/dictionaries/index.md | 14 +- .../dictionaries/internal-dicts.md | 38 +- .../functions/arithmetic-functions.md | 50 +- .../functions/array-functions.md | 294 +- docs/ja/sql-reference/functions/array-join.md | 18 +- .../sql-reference/functions/bit-functions.md | 62 +- .../functions/bitmap-functions.md | 84 +- .../functions/comparison-functions.md | 16 +- .../functions/conditional-functions.md | 44 +- .../functions/date-time-functions.md | 280 +- .../functions/encoding-functions.md | 52 +- .../functions/ext-dict-functions.md | 58 +- .../functions/functions-for-nulls.md | 58 +- docs/ja/sql-reference/functions/geo.md | 120 +- .../sql-reference/functions/hash-functions.md | 200 +- .../functions/higher-order-functions.md | 78 +- .../sql-reference/functions/in-functions.md | 14 +- docs/ja/sql-reference/functions/index.md | 56 +- .../sql-reference/functions/introspection.md | 66 +- .../functions/ip-address-functions.md | 38 +- .../sql-reference/functions/json-functions.md | 158 +- .../functions/logical-functions.md | 10 +- .../functions/machine-learning-functions.md | 14 +- .../sql-reference/functions/math-functions.md | 50 +- .../functions/other-functions.md | 356 ++- .../functions/random-functions.md | 49 +- .../functions/rounding-functions.md | 60 +- .../functions/splitting-merging-functions.md | 40 +- .../functions/string-functions.md | 144 +- .../functions/string-replace-functions.md | 38 +- .../functions/string-search-functions.md | 178 +- .../functions/type-conversion-functions.md | 172 +- .../sql-reference/functions/url-functions.md | 62 +- .../sql-reference/functions/uuid-functions.md | 18 +- .../functions/ym-dict-functions.md | 65 +- docs/ja/sql-reference/index.md | 14 +- docs/ja/sql-reference/operators/in.md | 1 + .../{operators.md => operators/index.md} | 69 +- docs/ja/sql-reference/statements/alter.md | 331 +- docs/ja/sql-reference/statements/create.md | 355 ++- docs/ja/sql-reference/statements/grant.md | 1 + docs/ja/sql-reference/statements/index.md | 4 +- .../sql-reference/statements/insert-into.md | 30 +- docs/ja/sql-reference/statements/misc.md | 216 +- docs/ja/sql-reference/statements/revoke.md | 1 + docs/ja/sql-reference/statements/select.md | 1379 --------- .../statements/select/array-join.md | 1 + .../statements/select/distinct.md | 1 + .../sql-reference/statements/select/format.md | 1 + .../sql-reference/statements/select/from.md | 1 + .../statements/select/group-by.md | 1 + .../sql-reference/statements/select/having.md | 1 + .../sql-reference/statements/select/index.md | 1 + .../statements/select/into-outfile.md | 1 + .../sql-reference/statements/select/join.md | 1 + .../statements/select/limit-by.md | 1 + .../sql-reference/statements/select/limit.md | 1 + .../statements/select/order-by.md | 1 + .../statements/select/prewhere.md | 1 + .../sql-reference/statements/select/sample.md | 1 + .../statements/select/union-all.md | 1 + .../sql-reference/statements/select/where.md | 1 + .../sql-reference/statements/select/with.md | 1 + docs/ja/sql-reference/statements/show.md | 92 +- docs/ja/sql-reference/statements/system.md | 40 +- docs/ja/sql-reference/syntax.md | 114 +- docs/ja/sql-reference/table-functions/file.md | 28 +- .../sql-reference/table-functions/generate.md | 5 +- docs/ja/sql-reference/table-functions/hdfs.md | 30 +- .../ja/sql-reference/table-functions/index.md | 34 +- .../ja/sql-reference/table-functions/input.md | 28 +- docs/ja/sql-reference/table-functions/jdbc.md | 6 +- .../ja/sql-reference/table-functions/merge.md | 4 +- .../ja/sql-reference/table-functions/mysql.md | 10 +- .../sql-reference/table-functions/numbers.md | 6 +- docs/ja/sql-reference/table-functions/odbc.md | 20 +- .../sql-reference/table-functions/remote.md | 36 +- docs/ja/sql-reference/table-functions/url.md | 12 +- docs/ja/whats-new/changelog/2017.md | 282 +- docs/ja/whats-new/changelog/2018.md | 1260 ++++---- docs/ja/whats-new/changelog/2019.md | 2720 ++++++++--------- docs/ja/whats-new/changelog/index.md | 665 +--- docs/ja/whats-new/index.md | 4 +- docs/ja/whats-new/roadmap.md | 6 +- docs/ja/whats-new/security-changelog.md | 48 +- docs/ru/faq/general.md | 4 +- docs/ru/operations/requirements.md | 4 +- .../operations/settings/query-complexity.md | 10 +- docs/ru/operations/settings/settings.md | 8 +- .../parametric-functions.md | 2 +- .../aggregate-functions/reference.md | 4 +- .../functions/conditional-functions.md | 2 +- .../sql-reference/functions/in-functions.md | 2 +- docs/ru/sql-reference/index.md | 2 +- docs/ru/sql-reference/operators/in.md | 1 + .../{operators.md => operators/index.md} | 16 +- docs/ru/sql-reference/statements/grant.md | 1 + docs/ru/sql-reference/statements/revoke.md | 1 + docs/ru/sql-reference/statements/select.md | 1406 --------- .../statements/select/array-join.md | 277 ++ .../statements/select/distinct.md | 58 + .../sql-reference/statements/select/format.md | 13 + .../sql-reference/statements/select/from.md | 40 + .../statements/select/group-by.md | 119 + .../sql-reference/statements/select/having.md | 9 + .../sql-reference/statements/select/index.md | 162 + .../statements/select/into-outfile.md | 9 + .../sql-reference/statements/select/join.md | 187 ++ .../statements/select/limit-by.md | 69 + .../sql-reference/statements/select/limit.md | 9 + .../statements/select/order-by.md | 68 + .../statements/select/prewhere.md | 17 + .../sql-reference/statements/select/sample.md | 113 + .../statements/select/union-all.md | 30 + .../sql-reference/statements/select/where.md | 25 + .../sql-reference/statements/select/with.md | 76 + docs/ru/sql-reference/syntax.md | 2 +- .../ru/sql-reference/table-functions/index.md | 2 +- docs/ru/whats-new/changelog/index.md | 661 +--- docs/tools/make_links.sh | 12 +- docs/tools/test.py | 3 +- docs/tools/translate/translate.py | 14 +- .../update-all-machine-translated.sh | 26 + docs/tr/commercial/cloud.md | 4 +- docs/tr/commercial/index.md | 2 +- docs/tr/development/architecture.md | 18 +- docs/tr/development/browse-code.md | 8 +- docs/tr/development/build-cross-arm.md | 10 +- docs/tr/development/build-cross-osx.md | 10 +- docs/tr/development/build-osx.md | 4 +- docs/tr/development/build.md | 18 +- docs/tr/development/contrib.md | 2 +- docs/tr/development/developer-instruction.md | 60 +- docs/tr/development/index.md | 2 +- docs/tr/development/style.md | 22 +- docs/tr/development/tests.md | 52 +- docs/tr/engines/database-engines/index.md | 2 +- docs/tr/engines/database-engines/lazy.md | 4 +- docs/tr/engines/database-engines/mysql.md | 6 +- docs/tr/engines/index.md | 2 +- docs/tr/engines/table-engines/index.md | 8 +- .../table-engines/integrations/hdfs.md | 6 +- .../table-engines/integrations/index.md | 2 +- .../table-engines/integrations/jdbc.md | 4 +- .../table-engines/integrations/kafka.md | 14 +- .../table-engines/integrations/mysql.md | 6 +- .../table-engines/integrations/odbc.md | 4 +- .../engines/table-engines/log-family/index.md | 2 +- .../table-engines/log-family/log-family.md | 2 +- .../engines/table-engines/log-family/log.md | 4 +- .../table-engines/log-family/stripelog.md | 12 +- .../table-engines/log-family/tinylog.md | 2 +- .../mergetree-family/aggregatingmergetree.md | 11 +- .../mergetree-family/collapsingmergetree.md | 9 +- .../custom-partitioning-key.md | 4 +- .../mergetree-family/graphitemergetree.md | 4 +- .../table-engines/mergetree-family/index.md | 2 +- .../mergetree-family/mergetree.md | 48 +- .../mergetree-family/replacingmergetree.md | 2 +- .../mergetree-family/replication.md | 22 +- .../mergetree-family/summingmergetree.md | 6 +- .../versionedcollapsingmergetree.md | 10 +- .../engines/table-engines/special/buffer.md | 10 +- .../table-engines/special/dictionary.md | 4 +- .../table-engines/special/distributed.md | 18 +- .../table-engines/special/external-data.md | 8 +- docs/tr/engines/table-engines/special/file.md | 8 +- .../engines/table-engines/special/generate.md | 4 +- .../tr/engines/table-engines/special/index.md | 2 +- docs/tr/engines/table-engines/special/join.md | 18 +- .../table-engines/special/materializedview.md | 2 +- .../engines/table-engines/special/memory.md | 10 +- .../tr/engines/table-engines/special/merge.md | 2 +- docs/tr/engines/table-engines/special/null.md | 2 +- docs/tr/engines/table-engines/special/set.md | 4 +- docs/tr/engines/table-engines/special/url.md | 6 +- docs/tr/engines/table-engines/special/view.md | 2 +- docs/tr/faq/general.md | 20 +- docs/tr/faq/index.md | 2 +- .../example-datasets/amplab-benchmark.md | 2 +- .../example-datasets/criteo.md | 4 +- .../getting-started/example-datasets/index.md | 6 +- .../example-datasets/metrica.md | 8 +- .../example-datasets/nyc-taxi.md | 20 +- .../example-datasets/ontime.md | 6 +- .../example-datasets/star-schema.md | 4 +- .../example-datasets/wikistat.md | 2 +- docs/tr/getting-started/index.md | 2 +- docs/tr/getting-started/install.md | 41 +- docs/tr/getting-started/playground.md | 4 +- docs/tr/getting-started/tutorial.md | 25 +- docs/tr/guides/apply-catboost-model.md | 20 +- docs/tr/guides/index.md | 4 +- docs/tr/interfaces/cli.md | 8 +- docs/tr/interfaces/cpp.md | 2 +- docs/tr/interfaces/formats.md | 50 +- docs/tr/interfaces/http.md | 360 ++- docs/tr/interfaces/index.md | 4 +- docs/tr/interfaces/jdbc.md | 2 +- docs/tr/interfaces/mysql.md | 2 +- docs/tr/interfaces/odbc.md | 2 +- docs/tr/interfaces/tcp.md | 4 +- .../third-party/client-libraries.md | 5 +- docs/tr/interfaces/third-party/gui.md | 10 +- docs/tr/interfaces/third-party/index.md | 2 +- .../tr/interfaces/third-party/integrations.md | 11 +- docs/tr/interfaces/third-party/proxy.md | 4 +- docs/tr/introduction/adopters.md | 145 +- docs/tr/introduction/distinctive-features.md | 30 +- docs/tr/introduction/history.md | 22 +- docs/tr/introduction/index.md | 2 +- docs/tr/introduction/performance.md | 10 +- docs/tr/operations/access-rights.md | 190 +- docs/tr/operations/backup.md | 8 +- docs/tr/operations/configuration-files.md | 4 +- docs/tr/operations/index.md | 4 +- docs/tr/operations/monitoring.md | 6 +- .../optimizing-performance/index.md | 2 +- .../sampling-query-profiler.md | 6 +- docs/tr/operations/performance-test.md | 6 +- docs/tr/operations/quotas.md | 2 +- docs/tr/operations/requirements.md | 16 +- .../server-configuration-parameters/index.md | 6 +- .../settings.md | 22 +- .../settings/constraints-on-settings.md | 2 +- docs/tr/operations/settings/index.md | 9 +- .../settings/permissions-for-queries.md | 6 +- .../operations/settings/query-complexity.md | 26 +- .../operations/settings/settings-profiles.md | 18 +- docs/tr/operations/settings/settings-users.md | 24 +- docs/tr/operations/settings/settings.md | 98 +- docs/tr/operations/system-tables.md | 44 +- docs/tr/operations/tips.md | 26 +- docs/tr/operations/troubleshooting.md | 12 +- docs/tr/operations/update.md | 2 +- .../utilities/clickhouse-benchmark.md | 12 +- .../operations/utilities/clickhouse-copier.md | 2 +- .../operations/utilities/clickhouse-local.md | 4 +- docs/tr/operations/utilities/index.md | 2 +- .../aggregate-functions/combinators.md | 107 +- .../aggregate-functions/index.md | 4 +- .../parametric-functions.md | 22 +- .../aggregate-functions/reference.md | 205 +- docs/tr/sql-reference/ansi.md | 181 +- .../data-types/aggregatefunction.md | 4 +- docs/tr/sql-reference/data-types/array.md | 6 +- docs/tr/sql-reference/data-types/boolean.md | 2 +- docs/tr/sql-reference/data-types/date.md | 4 +- docs/tr/sql-reference/data-types/datetime.md | 12 +- .../tr/sql-reference/data-types/datetime64.md | 4 +- docs/tr/sql-reference/data-types/decimal.md | 8 +- .../sql-reference/data-types/domains/index.md | 2 +- .../sql-reference/data-types/domains/ipv4.md | 2 +- .../sql-reference/data-types/domains/ipv6.md | 2 +- .../data-types/domains/overview.md | 4 +- docs/tr/sql-reference/data-types/enum.md | 6 +- .../sql-reference/data-types/fixedstring.md | 2 +- docs/tr/sql-reference/data-types/float.md | 8 +- docs/tr/sql-reference/data-types/index.md | 2 +- docs/tr/sql-reference/data-types/int-uint.md | 2 +- .../nested-data-structures/index.md | 2 +- .../nested-data-structures/nested.md | 2 +- docs/tr/sql-reference/data-types/nullable.md | 4 +- .../special-data-types/expression.md | 4 +- .../data-types/special-data-types/index.md | 2 +- .../data-types/special-data-types/interval.md | 6 +- .../data-types/special-data-types/nothing.md | 2 +- .../data-types/special-data-types/set.md | 4 +- docs/tr/sql-reference/data-types/string.md | 2 +- docs/tr/sql-reference/data-types/tuple.md | 6 +- docs/tr/sql-reference/data-types/uuid.md | 6 +- .../external-dicts-dict-hierarchical.md | 2 +- .../external-dicts-dict-layout.md | 35 +- .../external-dicts-dict-lifetime.md | 2 +- .../external-dicts-dict-sources.md | 42 +- .../external-dicts-dict-structure.md | 6 +- .../external-dicts-dict.md | 4 +- .../external-dictionaries/external-dicts.md | 10 +- .../external-dictionaries/index.md | 2 +- docs/tr/sql-reference/dictionaries/index.md | 4 +- .../dictionaries/internal-dicts.md | 10 +- .../functions/arithmetic-functions.md | 6 +- .../functions/array-functions.md | 24 +- docs/tr/sql-reference/functions/array-join.md | 2 +- .../sql-reference/functions/bit-functions.md | 10 +- .../functions/bitmap-functions.md | 16 +- .../functions/comparison-functions.md | 4 +- .../functions/conditional-functions.md | 12 +- .../functions/date-time-functions.md | 24 +- .../functions/encoding-functions.md | 8 +- .../functions/ext-dict-functions.md | 8 +- .../functions/functions-for-nulls.md | 6 +- docs/tr/sql-reference/functions/geo.md | 8 +- .../sql-reference/functions/hash-functions.md | 52 +- .../functions/higher-order-functions.md | 42 +- .../sql-reference/functions/in-functions.md | 8 +- docs/tr/sql-reference/functions/index.md | 16 +- .../sql-reference/functions/introspection.md | 10 +- .../functions/ip-address-functions.md | 14 +- .../sql-reference/functions/json-functions.md | 90 +- .../functions/logical-functions.md | 4 +- .../functions/machine-learning-functions.md | 4 +- .../sql-reference/functions/math-functions.md | 14 +- .../functions/other-functions.md | 156 +- .../functions/random-functions.md | 41 +- .../functions/rounding-functions.md | 8 +- .../functions/splitting-merging-functions.md | 6 +- .../functions/string-functions.md | 28 +- .../functions/string-replace-functions.md | 10 +- .../functions/string-search-functions.md | 44 +- .../functions/type-conversion-functions.md | 16 +- .../sql-reference/functions/url-functions.md | 34 +- .../sql-reference/functions/uuid-functions.md | 6 +- .../functions/ym-dict-functions.md | 16 +- docs/tr/sql-reference/index.md | 8 +- docs/tr/sql-reference/operators/in.md | 1 + .../{operators.md => operators/index.md} | 82 +- docs/tr/sql-reference/statements/alter.md | 142 +- docs/tr/sql-reference/statements/create.md | 221 +- docs/tr/sql-reference/statements/grant.md | 1 + docs/tr/sql-reference/statements/index.md | 2 +- .../sql-reference/statements/insert-into.md | 6 +- docs/tr/sql-reference/statements/misc.md | 130 +- docs/tr/sql-reference/statements/revoke.md | 1 + docs/tr/sql-reference/statements/select.md | 1379 --------- .../statements/select/array-join.md | 1 + .../statements/select/distinct.md | 1 + .../sql-reference/statements/select/format.md | 1 + .../sql-reference/statements/select/from.md | 1 + .../statements/select/group-by.md | 1 + .../sql-reference/statements/select/having.md | 1 + .../sql-reference/statements/select/index.md | 1 + .../statements/select/into-outfile.md | 1 + .../sql-reference/statements/select/join.md | 1 + .../statements/select/limit-by.md | 1 + .../sql-reference/statements/select/limit.md | 1 + .../statements/select/order-by.md | 1 + .../statements/select/prewhere.md | 1 + .../sql-reference/statements/select/sample.md | 1 + .../statements/select/union-all.md | 1 + .../sql-reference/statements/select/where.md | 1 + .../sql-reference/statements/select/with.md | 1 + docs/tr/sql-reference/statements/show.md | 70 +- docs/tr/sql-reference/statements/system.md | 10 +- docs/tr/sql-reference/syntax.md | 22 +- docs/tr/sql-reference/table-functions/file.md | 6 +- .../sql-reference/table-functions/generate.md | 5 +- docs/tr/sql-reference/table-functions/hdfs.md | 8 +- .../tr/sql-reference/table-functions/index.md | 6 +- .../tr/sql-reference/table-functions/input.md | 2 +- docs/tr/sql-reference/table-functions/jdbc.md | 2 +- .../tr/sql-reference/table-functions/merge.md | 2 +- .../tr/sql-reference/table-functions/mysql.md | 6 +- .../sql-reference/table-functions/numbers.md | 6 +- docs/tr/sql-reference/table-functions/odbc.md | 8 +- .../sql-reference/table-functions/remote.md | 4 +- docs/tr/sql-reference/table-functions/url.md | 2 +- docs/tr/whats-new/changelog/2017.md | 66 +- docs/tr/whats-new/changelog/2018.md | 268 +- docs/tr/whats-new/changelog/index.md | 663 +--- docs/tr/whats-new/index.md | 2 +- docs/tr/whats-new/roadmap.md | 2 +- docs/tr/whats-new/security-changelog.md | 18 +- docs/zh/commercial/cloud.md | 4 +- docs/zh/commercial/index.md | 2 +- docs/zh/development/browse-code.md | 6 +- docs/zh/development/build-cross-arm.md | 2 +- docs/zh/engines/database-engines/lazy.md | 2 +- docs/zh/engines/index.md | 2 +- .../table-engines/integrations/hdfs.md | 2 +- .../table-engines/integrations/index.md | 2 +- .../table-engines/integrations/jdbc.md | 2 +- .../table-engines/integrations/odbc.md | 2 +- .../engines/table-engines/log-family/index.md | 2 +- .../mergetree-family/graphitemergetree.md | 2 +- .../table-engines/mergetree-family/index.md | 2 +- .../versionedcollapsingmergetree.md | 4 +- .../engines/table-engines/special/generate.md | 2 +- .../zh/engines/table-engines/special/index.md | 2 +- docs/zh/faq/index.md | 2 +- docs/zh/getting-started/tutorial.md | 7 +- docs/zh/guides/apply-catboost-model.md | 2 +- docs/zh/guides/index.md | 2 +- docs/zh/interfaces/mysql.md | 2 +- docs/zh/interfaces/third-party/index.md | 2 +- docs/zh/introduction/index.md | 2 +- docs/zh/operations/backup.md | 2 +- .../optimizing-performance/index.md | 2 +- .../sampling-query-profiler.md | 2 +- docs/zh/operations/performance-test.md | 6 +- docs/zh/operations/requirements.md | 6 +- .../settings/constraints-on-settings.md | 2 +- .../settings/permissions-for-queries.md | 2 +- .../operations/settings/query-complexity.md | 10 +- .../operations/settings/settings-profiles.md | 18 +- docs/zh/operations/settings/settings-users.md | 18 +- docs/zh/operations/settings/settings.md | 104 +- docs/zh/operations/system-tables.md | 104 +- docs/zh/operations/troubleshooting.md | 2 +- docs/zh/operations/update.md | 2 +- .../utilities/clickhouse-benchmark.md | 2 +- .../aggregate-functions/combinators.md | 99 +- .../aggregate-functions/index.md | 2 +- .../parametric-functions.md | 10 +- .../aggregate-functions/reference.md | 155 +- docs/zh/sql-reference/ansi.md | 181 +- .../zh/sql-reference/data-types/datetime64.md | 4 +- .../sql-reference/data-types/domains/index.md | 2 +- .../data-types/special-data-types/interval.md | 6 +- docs/zh/sql-reference/data-types/uuid.md | 2 +- .../external-dicts-dict-hierarchical.md | 2 +- .../external-dicts-dict-layout.md | 27 +- .../external-dicts-dict-lifetime.md | 7 +- .../external-dicts-dict-sources.md | 24 +- .../external-dicts-dict-structure.md | 2 +- .../external-dicts-dict.md | 2 +- .../external-dictionaries/external-dicts.md | 8 +- .../external-dictionaries/index.md | 2 +- docs/zh/sql-reference/dictionaries/index.md | 4 +- .../dictionaries/internal-dicts.md | 4 +- .../functions/conditional-functions.md | 2 +- .../sql-reference/functions/in-functions.md | 2 +- .../sql-reference/functions/introspection.md | 2 +- docs/zh/sql-reference/index.md | 8 +- docs/zh/sql-reference/operators/in.md | 1 + .../{operators.md => operators/index.md} | 16 +- docs/zh/sql-reference/statements/alter.md | 107 +- docs/zh/sql-reference/statements/grant.md | 1 + docs/zh/sql-reference/statements/index.md | 2 +- docs/zh/sql-reference/statements/misc.md | 118 +- docs/zh/sql-reference/statements/revoke.md | 1 + docs/zh/sql-reference/statements/select.md | 1379 --------- .../statements/select/array-join.md | 1 + .../statements/select/distinct.md | 1 + .../sql-reference/statements/select/format.md | 1 + .../sql-reference/statements/select/from.md | 1 + .../statements/select/group-by.md | 1 + .../sql-reference/statements/select/having.md | 1 + .../sql-reference/statements/select/index.md | 1 + .../statements/select/into-outfile.md | 1 + .../sql-reference/statements/select/join.md | 1 + .../statements/select/limit-by.md | 1 + .../sql-reference/statements/select/limit.md | 1 + .../statements/select/order-by.md | 1 + .../statements/select/prewhere.md | 1 + .../sql-reference/statements/select/sample.md | 1 + .../statements/select/union-all.md | 1 + .../sql-reference/statements/select/where.md | 1 + .../sql-reference/statements/select/with.md | 1 + docs/zh/sql-reference/statements/show.md | 66 +- docs/zh/sql-reference/statements/system.md | 2 +- docs/zh/sql-reference/syntax.md | 46 +- docs/zh/sql-reference/table-functions/file.md | 2 +- .../sql-reference/table-functions/generate.md | 5 +- docs/zh/sql-reference/table-functions/hdfs.md | 2 +- .../zh/sql-reference/table-functions/index.md | 4 +- .../zh/sql-reference/table-functions/input.md | 2 +- docs/zh/sql-reference/table-functions/jdbc.md | 2 +- .../zh/sql-reference/table-functions/merge.md | 2 +- .../zh/sql-reference/table-functions/mysql.md | 2 +- .../sql-reference/table-functions/numbers.md | 2 +- docs/zh/sql-reference/table-functions/odbc.md | 2 +- .../sql-reference/table-functions/remote.md | 2 +- docs/zh/sql-reference/table-functions/url.md | 2 +- docs/zh/whats-new/changelog/2017.md | 6 +- docs/zh/whats-new/changelog/2018.md | 4 +- docs/zh/whats-new/changelog/2019.md | 28 +- docs/zh/whats-new/changelog/index.md | 666 +--- docs/zh/whats-new/index.md | 2 +- 1394 files changed, 35831 insertions(+), 33966 deletions(-) create mode 100644 docs/en/sql-reference/operators/in.md rename docs/en/sql-reference/{operators.md => operators/index.md} (87%) delete mode 100644 docs/en/sql-reference/statements/select.md create mode 100644 docs/en/sql-reference/statements/select/array-join.md create mode 100644 docs/en/sql-reference/statements/select/distinct.md create mode 100644 docs/en/sql-reference/statements/select/format.md create mode 100644 docs/en/sql-reference/statements/select/from.md create mode 100644 docs/en/sql-reference/statements/select/group-by.md create mode 100644 docs/en/sql-reference/statements/select/having.md create mode 100644 docs/en/sql-reference/statements/select/index.md create mode 100644 docs/en/sql-reference/statements/select/into-outfile.md create mode 100644 docs/en/sql-reference/statements/select/join.md create mode 100644 docs/en/sql-reference/statements/select/limit-by.md create mode 100644 docs/en/sql-reference/statements/select/limit.md create mode 100644 docs/en/sql-reference/statements/select/order-by.md create mode 100644 docs/en/sql-reference/statements/select/prewhere.md create mode 100644 docs/en/sql-reference/statements/select/sample.md create mode 100644 docs/en/sql-reference/statements/select/union-all.md create mode 100644 docs/en/sql-reference/statements/select/where.md create mode 100644 docs/en/sql-reference/statements/select/with.md mode change 120000 => 100644 docs/es/commercial/support.md mode change 120000 => 100644 docs/es/sql-reference/ansi.md mode change 120000 => 100644 docs/es/sql-reference/data-types/simpleaggregatefunction.md create mode 100644 docs/es/sql-reference/operators/in.md rename docs/es/sql-reference/{operators.md => operators/index.md} (68%) create mode 100644 docs/es/sql-reference/statements/grant.md create mode 100644 docs/es/sql-reference/statements/revoke.md delete mode 100644 docs/es/sql-reference/statements/select.md create mode 100644 docs/es/sql-reference/statements/select/array-join.md create mode 100644 docs/es/sql-reference/statements/select/distinct.md create mode 100644 docs/es/sql-reference/statements/select/format.md create mode 100644 docs/es/sql-reference/statements/select/from.md create mode 100644 docs/es/sql-reference/statements/select/group-by.md create mode 100644 docs/es/sql-reference/statements/select/having.md create mode 100644 docs/es/sql-reference/statements/select/index.md create mode 100644 docs/es/sql-reference/statements/select/into-outfile.md create mode 100644 docs/es/sql-reference/statements/select/join.md create mode 100644 docs/es/sql-reference/statements/select/limit-by.md create mode 100644 docs/es/sql-reference/statements/select/limit.md create mode 100644 docs/es/sql-reference/statements/select/order-by.md create mode 100644 docs/es/sql-reference/statements/select/prewhere.md create mode 100644 docs/es/sql-reference/statements/select/sample.md create mode 100644 docs/es/sql-reference/statements/select/union-all.md create mode 100644 docs/es/sql-reference/statements/select/where.md create mode 100644 docs/es/sql-reference/statements/select/with.md mode change 120000 => 100644 docs/fa/commercial/support.md mode change 120000 => 100644 docs/fa/sql-reference/ansi.md mode change 120000 => 100644 docs/fa/sql-reference/data-types/simpleaggregatefunction.md create mode 100644 docs/fa/sql-reference/operators/in.md rename docs/fa/sql-reference/{operators.md => operators/index.md} (76%) create mode 100644 docs/fa/sql-reference/statements/grant.md create mode 100644 docs/fa/sql-reference/statements/revoke.md delete mode 100644 docs/fa/sql-reference/statements/select.md create mode 100644 docs/fa/sql-reference/statements/select/array-join.md create mode 100644 docs/fa/sql-reference/statements/select/distinct.md create mode 100644 docs/fa/sql-reference/statements/select/format.md create mode 100644 docs/fa/sql-reference/statements/select/from.md create mode 100644 docs/fa/sql-reference/statements/select/group-by.md create mode 100644 docs/fa/sql-reference/statements/select/having.md create mode 100644 docs/fa/sql-reference/statements/select/index.md create mode 100644 docs/fa/sql-reference/statements/select/into-outfile.md create mode 100644 docs/fa/sql-reference/statements/select/join.md create mode 100644 docs/fa/sql-reference/statements/select/limit-by.md create mode 100644 docs/fa/sql-reference/statements/select/limit.md create mode 100644 docs/fa/sql-reference/statements/select/order-by.md create mode 100644 docs/fa/sql-reference/statements/select/prewhere.md create mode 100644 docs/fa/sql-reference/statements/select/sample.md create mode 100644 docs/fa/sql-reference/statements/select/union-all.md create mode 100644 docs/fa/sql-reference/statements/select/where.md create mode 100644 docs/fa/sql-reference/statements/select/with.md mode change 120000 => 100644 docs/fr/commercial/support.md mode change 120000 => 100644 docs/fr/sql-reference/ansi.md mode change 120000 => 100644 docs/fr/sql-reference/data-types/simpleaggregatefunction.md create mode 100644 docs/fr/sql-reference/operators/in.md rename docs/fr/sql-reference/{operators.md => operators/index.md} (64%) create mode 100644 docs/fr/sql-reference/statements/grant.md create mode 100644 docs/fr/sql-reference/statements/revoke.md delete mode 100644 docs/fr/sql-reference/statements/select.md create mode 100644 docs/fr/sql-reference/statements/select/array-join.md create mode 100644 docs/fr/sql-reference/statements/select/distinct.md create mode 100644 docs/fr/sql-reference/statements/select/format.md create mode 100644 docs/fr/sql-reference/statements/select/from.md create mode 100644 docs/fr/sql-reference/statements/select/group-by.md create mode 100644 docs/fr/sql-reference/statements/select/having.md create mode 100644 docs/fr/sql-reference/statements/select/index.md create mode 100644 docs/fr/sql-reference/statements/select/into-outfile.md create mode 100644 docs/fr/sql-reference/statements/select/join.md create mode 100644 docs/fr/sql-reference/statements/select/limit-by.md create mode 100644 docs/fr/sql-reference/statements/select/limit.md create mode 100644 docs/fr/sql-reference/statements/select/order-by.md create mode 100644 docs/fr/sql-reference/statements/select/prewhere.md create mode 100644 docs/fr/sql-reference/statements/select/sample.md create mode 100644 docs/fr/sql-reference/statements/select/union-all.md create mode 100644 docs/fr/sql-reference/statements/select/where.md create mode 100644 docs/fr/sql-reference/statements/select/with.md mode change 120000 => 100644 docs/ja/sql-reference/ansi.md create mode 120000 docs/ja/sql-reference/operators/in.md rename docs/ja/sql-reference/{operators.md => operators/index.md} (61%) create mode 120000 docs/ja/sql-reference/statements/grant.md create mode 120000 docs/ja/sql-reference/statements/revoke.md delete mode 100644 docs/ja/sql-reference/statements/select.md create mode 120000 docs/ja/sql-reference/statements/select/array-join.md create mode 120000 docs/ja/sql-reference/statements/select/distinct.md create mode 120000 docs/ja/sql-reference/statements/select/format.md create mode 120000 docs/ja/sql-reference/statements/select/from.md create mode 120000 docs/ja/sql-reference/statements/select/group-by.md create mode 120000 docs/ja/sql-reference/statements/select/having.md create mode 120000 docs/ja/sql-reference/statements/select/index.md create mode 120000 docs/ja/sql-reference/statements/select/into-outfile.md create mode 120000 docs/ja/sql-reference/statements/select/join.md create mode 120000 docs/ja/sql-reference/statements/select/limit-by.md create mode 120000 docs/ja/sql-reference/statements/select/limit.md create mode 120000 docs/ja/sql-reference/statements/select/order-by.md create mode 120000 docs/ja/sql-reference/statements/select/prewhere.md create mode 120000 docs/ja/sql-reference/statements/select/sample.md create mode 120000 docs/ja/sql-reference/statements/select/union-all.md create mode 120000 docs/ja/sql-reference/statements/select/where.md create mode 120000 docs/ja/sql-reference/statements/select/with.md create mode 120000 docs/ru/sql-reference/operators/in.md rename docs/ru/sql-reference/{operators.md => operators/index.md} (88%) create mode 120000 docs/ru/sql-reference/statements/grant.md create mode 120000 docs/ru/sql-reference/statements/revoke.md delete mode 100644 docs/ru/sql-reference/statements/select.md create mode 100644 docs/ru/sql-reference/statements/select/array-join.md create mode 100644 docs/ru/sql-reference/statements/select/distinct.md create mode 100644 docs/ru/sql-reference/statements/select/format.md create mode 100644 docs/ru/sql-reference/statements/select/from.md create mode 100644 docs/ru/sql-reference/statements/select/group-by.md create mode 100644 docs/ru/sql-reference/statements/select/having.md create mode 100644 docs/ru/sql-reference/statements/select/index.md create mode 100644 docs/ru/sql-reference/statements/select/into-outfile.md create mode 100644 docs/ru/sql-reference/statements/select/join.md create mode 100644 docs/ru/sql-reference/statements/select/limit-by.md create mode 100644 docs/ru/sql-reference/statements/select/limit.md create mode 100644 docs/ru/sql-reference/statements/select/order-by.md create mode 100644 docs/ru/sql-reference/statements/select/prewhere.md create mode 100644 docs/ru/sql-reference/statements/select/sample.md create mode 100644 docs/ru/sql-reference/statements/select/union-all.md create mode 100644 docs/ru/sql-reference/statements/select/where.md create mode 100644 docs/ru/sql-reference/statements/select/with.md create mode 100755 docs/tools/translate/update-all-machine-translated.sh mode change 120000 => 100644 docs/tr/sql-reference/ansi.md create mode 120000 docs/tr/sql-reference/operators/in.md rename docs/tr/sql-reference/{operators.md => operators/index.md} (71%) create mode 120000 docs/tr/sql-reference/statements/grant.md create mode 120000 docs/tr/sql-reference/statements/revoke.md delete mode 100644 docs/tr/sql-reference/statements/select.md create mode 120000 docs/tr/sql-reference/statements/select/array-join.md create mode 120000 docs/tr/sql-reference/statements/select/distinct.md create mode 120000 docs/tr/sql-reference/statements/select/format.md create mode 120000 docs/tr/sql-reference/statements/select/from.md create mode 120000 docs/tr/sql-reference/statements/select/group-by.md create mode 120000 docs/tr/sql-reference/statements/select/having.md create mode 120000 docs/tr/sql-reference/statements/select/index.md create mode 120000 docs/tr/sql-reference/statements/select/into-outfile.md create mode 120000 docs/tr/sql-reference/statements/select/join.md create mode 120000 docs/tr/sql-reference/statements/select/limit-by.md create mode 120000 docs/tr/sql-reference/statements/select/limit.md create mode 120000 docs/tr/sql-reference/statements/select/order-by.md create mode 120000 docs/tr/sql-reference/statements/select/prewhere.md create mode 120000 docs/tr/sql-reference/statements/select/sample.md create mode 120000 docs/tr/sql-reference/statements/select/union-all.md create mode 120000 docs/tr/sql-reference/statements/select/where.md create mode 120000 docs/tr/sql-reference/statements/select/with.md mode change 120000 => 100644 docs/zh/sql-reference/ansi.md create mode 120000 docs/zh/sql-reference/operators/in.md rename docs/zh/sql-reference/{operators.md => operators/index.md} (87%) create mode 120000 docs/zh/sql-reference/statements/grant.md create mode 120000 docs/zh/sql-reference/statements/revoke.md delete mode 100644 docs/zh/sql-reference/statements/select.md create mode 120000 docs/zh/sql-reference/statements/select/array-join.md create mode 120000 docs/zh/sql-reference/statements/select/distinct.md create mode 120000 docs/zh/sql-reference/statements/select/format.md create mode 120000 docs/zh/sql-reference/statements/select/from.md create mode 120000 docs/zh/sql-reference/statements/select/group-by.md create mode 120000 docs/zh/sql-reference/statements/select/having.md create mode 120000 docs/zh/sql-reference/statements/select/index.md create mode 120000 docs/zh/sql-reference/statements/select/into-outfile.md create mode 120000 docs/zh/sql-reference/statements/select/join.md create mode 120000 docs/zh/sql-reference/statements/select/limit-by.md create mode 120000 docs/zh/sql-reference/statements/select/limit.md create mode 120000 docs/zh/sql-reference/statements/select/order-by.md create mode 120000 docs/zh/sql-reference/statements/select/prewhere.md create mode 120000 docs/zh/sql-reference/statements/select/sample.md create mode 120000 docs/zh/sql-reference/statements/select/union-all.md create mode 120000 docs/zh/sql-reference/statements/select/where.md create mode 120000 docs/zh/sql-reference/statements/select/with.md diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index 366188f4b7e..20710941a67 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -104,7 +104,7 @@ ENGINE MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDa In the example, we set partitioning by month. -We also set an expression for sampling as a hash by the user ID. This allows you to pseudorandomize the data in the table for each `CounterID` and `EventDate`. If you define a [SAMPLE](../../../sql-reference/statements/select.md#select-sample-clause) clause when selecting the data, ClickHouse will return an evenly pseudorandom data sample for a subset of users. +We also set an expression for sampling as a hash by the user ID. This allows you to pseudorandomize the data in the table for each `CounterID` and `EventDate`. If you define a [SAMPLE](../../../sql-reference/statements/select/sample.md#select-sample-clause) clause when selecting the data, ClickHouse will return an evenly pseudorandom data sample for a subset of users. The `index_granularity` setting can be omitted because 8192 is the default value. @@ -385,7 +385,7 @@ TTL time_column TTL time_column + interval ``` -To define `interval`, use [time interval](../../../sql-reference/operators.md#operators-datetime) operators. +To define `interval`, use [time interval](../../../sql-reference/operators/index.md#operators-datetime) operators. ``` sql TTL date_time + INTERVAL 1 MONTH diff --git a/docs/en/engines/table-engines/special/join.md b/docs/en/engines/table-engines/special/join.md index adc88115be3..d0e685f9c48 100644 --- a/docs/en/engines/table-engines/special/join.md +++ b/docs/en/engines/table-engines/special/join.md @@ -5,7 +5,7 @@ toc_title: Join # Join {#join} -Prepared data structure for using in [JOIN](../../../sql-reference/statements/select.md#select-join) operations. +Prepared data structure for using in [JOIN](../../../sql-reference/statements/select/join.md#select-join) operations. ## Creating a Table {#creating-a-table} @@ -21,8 +21,8 @@ See the detailed description of the [CREATE TABLE](../../../sql-reference/statem **Engine Parameters** -- `join_strictness` – [JOIN strictness](../../../sql-reference/statements/select.md#select-join-strictness). -- `join_type` – [JOIN type](../../../sql-reference/statements/select.md#select-join-types). +- `join_strictness` – [JOIN strictness](../../../sql-reference/statements/select/join.md#select-join-strictness). +- `join_type` – [JOIN type](../../../sql-reference/statements/select/join.md#select-join-types). - `k1[, k2, ...]` – Key columns from the `USING` clause that the `JOIN` operation is made with. Enter `join_strictness` and `join_type` parameters without quotes, for example, `Join(ANY, LEFT, col1)`. They must match the `JOIN` operation that the table will be used for. If the parameters don’t match, ClickHouse doesn’t throw an exception and may return incorrect data. @@ -98,7 +98,7 @@ When creating a table, the following settings are applied: The `Join`-engine tables can’t be used in `GLOBAL JOIN` operations. -The `Join`-engine allows use [join\_use\_nulls](../../../operations/settings/settings.md#join_use_nulls) setting in the `CREATE TABLE` statement. And [SELECT](../../../sql-reference/statements/select.md) query allows use `join_use_nulls` too. If you have different `join_use_nulls` settings, you can get an error joining table. It depends on kind of JOIN. When you use [joinGet](../../../sql-reference/functions/other-functions.md#joinget) function, you have to use the same `join_use_nulls` setting in `CRATE TABLE` and `SELECT` statements. +The `Join`-engine allows use [join\_use\_nulls](../../../operations/settings/settings.md#join_use_nulls) setting in the `CREATE TABLE` statement. And [SELECT](../../../sql-reference/statements/select/index.md) query allows use `join_use_nulls` too. If you have different `join_use_nulls` settings, you can get an error joining table. It depends on kind of JOIN. When you use [joinGet](../../../sql-reference/functions/other-functions.md#joinget) function, you have to use the same `join_use_nulls` setting in `CRATE TABLE` and `SELECT` statements. ## Data Storage {#data-storage} diff --git a/docs/en/faq/general.md b/docs/en/faq/general.md index 860356e46f6..53cb583e25f 100644 --- a/docs/en/faq/general.md +++ b/docs/en/faq/general.md @@ -27,7 +27,7 @@ NLS_LANG=RUSSIAN_RUSSIA.UTF8 ### Using INTO OUTFILE Clause {#using-into-outfile-clause} -Add an [INTO OUTFILE](../sql-reference/statements/select.md#into-outfile-clause) clause to your query. +Add an [INTO OUTFILE](../sql-reference/statements/select/into-outfile.md#into-outfile-clause) clause to your query. For example: @@ -35,7 +35,7 @@ For example: SELECT * FROM table INTO OUTFILE 'file' ``` -By default, ClickHouse uses the [TabSeparated](../interfaces/formats.md#tabseparated) format for output data. To select the [data format](../interfaces/formats.md), use the [FORMAT clause](../sql-reference/statements/select.md#format-clause). +By default, ClickHouse uses the [TabSeparated](../interfaces/formats.md#tabseparated) format for output data. To select the [data format](../interfaces/formats.md), use the [FORMAT clause](../sql-reference/statements/select/format.md#format-clause). For example: diff --git a/docs/en/operations/requirements.md b/docs/en/operations/requirements.md index 5f2d2de3535..e759849eaaa 100644 --- a/docs/en/operations/requirements.md +++ b/docs/en/operations/requirements.md @@ -22,9 +22,9 @@ The required volume of RAM depends on: - The complexity of queries. - The amount of data that is processed in queries. -To calculate the required volume of RAM, you should estimate the size of temporary data for [GROUP BY](../sql-reference/statements/select.md#select-group-by-clause), [DISTINCT](../sql-reference/statements/select.md#select-distinct), [JOIN](../sql-reference/statements/select.md#select-join) and other operations you use. +To calculate the required volume of RAM, you should estimate the size of temporary data for [GROUP BY](../sql-reference/statements/select/group-by.md#select-group-by-clause), [DISTINCT](../sql-reference/statements/select/distinct.md#select-distinct), [JOIN](../sql-reference/statements/select/join.md#select-join) and other operations you use. -ClickHouse can use external memory for temporary data. See [GROUP BY in External Memory](../sql-reference/statements/select.md#select-group-by-in-external-memory) for details. +ClickHouse can use external memory for temporary data. See [GROUP BY in External Memory](../sql-reference/statements/select/group-by.md#select-group-by-in-external-memory) for details. ## Swap File {#swap-file} diff --git a/docs/en/operations/settings/query-complexity.md b/docs/en/operations/settings/query-complexity.md index f463b18995d..83b6054d642 100644 --- a/docs/en/operations/settings/query-complexity.md +++ b/docs/en/operations/settings/query-complexity.md @@ -80,11 +80,11 @@ Using the ‘any’ value lets you run an approximation of GROUP BY. The quality ## max\_bytes\_before\_external\_group\_by {#settings-max_bytes_before_external_group_by} -Enables or disables execution of `GROUP BY` clauses in external memory. See [GROUP BY in external memory](../../sql-reference/statements/select.md#select-group-by-in-external-memory). +Enables or disables execution of `GROUP BY` clauses in external memory. See [GROUP BY in external memory](../../sql-reference/statements/select/group-by.md#select-group-by-in-external-memory). Possible values: -- Maximum volume of RAM (in bytes) that can be used by the single [GROUP BY](../../sql-reference/statements/select.md#select-group-by-clause) operation. +- Maximum volume of RAM (in bytes) that can be used by the single [GROUP BY](../../sql-reference/statements/select/group-by.md#select-group-by-clause) operation. - 0 — `GROUP BY` in external memory disabled. Default value: 0. @@ -232,7 +232,7 @@ What to do when the amount of data exceeds one of the limits: ‘throw’ or ‘ Limits the number of rows in the hash table that is used when joining tables. -This settings applies to [SELECT … JOIN](../../sql-reference/statements/select.md#select-join) operations and the [Join](../../engines/table-engines/special/join.md) table engine. +This settings applies to [SELECT … JOIN](../../sql-reference/statements/select/join.md#select-join) operations and the [Join](../../engines/table-engines/special/join.md) table engine. If a query contains multiple joins, ClickHouse checks this setting for every intermediate result. @@ -249,7 +249,7 @@ Default value: 0. Limits the size in bytes of the hash table used when joining tables. -This settings applies to [SELECT … JOIN](../../sql-reference/statements/select.md#select-join) operations and [Join table engine](../../engines/table-engines/special/join.md). +This settings applies to [SELECT … JOIN](../../sql-reference/statements/select/join.md#select-join) operations and [Join table engine](../../engines/table-engines/special/join.md). If the query contains joins, ClickHouse checks this setting for every intermediate result. @@ -278,7 +278,7 @@ Default value: `THROW`. **See Also** -- [JOIN clause](../../sql-reference/statements/select.md#select-join) +- [JOIN clause](../../sql-reference/statements/select/join.md#select-join) - [Join table engine](../../engines/table-engines/special/join.md) ## max\_partitions\_per\_insert\_block {#max-partitions-per-insert-block} diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index e97c0dc8373..67097103cc9 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -2,7 +2,7 @@ ## distributed\_product\_mode {#distributed-product-mode} -Changes the behavior of [distributed subqueries](../../sql-reference/statements/select.md). +Changes the behavior of [distributed subqueries](../../sql-reference/operators/in.md). ClickHouse applies this setting when the query contains the product of distributed tables, i.e. when the query for a distributed table contains a non-GLOBAL subquery for the distributed table. @@ -362,7 +362,7 @@ See also: ## join\_default\_strictness {#settings-join_default_strictness} -Sets default strictness for [JOIN clauses](../../sql-reference/statements/select.md#select-join). +Sets default strictness for [JOIN clauses](../../sql-reference/statements/select/join.md#select-join). Possible values: @@ -389,13 +389,13 @@ Default value: 0. See also: -- [JOIN clause](../../sql-reference/statements/select.md#select-join) +- [JOIN clause](../../sql-reference/statements/select/join.md#select-join) - [Join table engine](../../engines/table-engines/special/join.md) - [join\_default\_strictness](#settings-join_default_strictness) ## join\_use\_nulls {#join_use_nulls} -Sets the type of [JOIN](../../sql-reference/statements/select.md) behavior. When merging tables, empty cells may appear. ClickHouse fills them differently based on this setting. +Sets the type of [JOIN](../../sql-reference/statements/select/join.md) behavior. When merging tables, empty cells may appear. ClickHouse fills them differently based on this setting. Possible values: diff --git a/docs/en/sql-reference/aggregate-functions/combinators.md b/docs/en/sql-reference/aggregate-functions/combinators.md index f5733076802..8573d55a33c 100644 --- a/docs/en/sql-reference/aggregate-functions/combinators.md +++ b/docs/en/sql-reference/aggregate-functions/combinators.md @@ -1,6 +1,6 @@ --- toc_priority: 37 -toc_title: Aggregate function combinators +toc_title: Combinators --- # Aggregate Function Combinators {#aggregate_functions_combinators} diff --git a/docs/en/sql-reference/aggregate-functions/parametric-functions.md b/docs/en/sql-reference/aggregate-functions/parametric-functions.md index 01516a4bf51..3dec141d736 100644 --- a/docs/en/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/en/sql-reference/aggregate-functions/parametric-functions.md @@ -1,6 +1,6 @@ --- toc_priority: 38 -toc_title: Parametric aggregate functions +toc_title: Parametric --- # Parametric Aggregate Functions {#aggregate_functions_parametric} @@ -314,7 +314,7 @@ Result: ## retention {#retention} The function takes as arguments a set of conditions from 1 to 32 arguments of type `UInt8` that indicate whether a certain condition was met for the event. -Any condition can be specified as an argument (as in [WHERE](../../sql-reference/statements/select.md#select-where)). +Any condition can be specified as an argument (as in [WHERE](../../sql-reference/statements/select/where.md#select-where)). The conditions, except the first, apply in pairs: the result of the second will be true if the first and second are true, of the third if the first and fird are true, etc. diff --git a/docs/en/sql-reference/aggregate-functions/reference.md b/docs/en/sql-reference/aggregate-functions/reference.md index 3bfaae8e696..7aa37d67dbe 100644 --- a/docs/en/sql-reference/aggregate-functions/reference.md +++ b/docs/en/sql-reference/aggregate-functions/reference.md @@ -3,7 +3,7 @@ toc_priority: 36 toc_title: Reference --- -# Function Reference {#aggregate-functions-reference} +# Aggregate Function Reference {#aggregate-functions-reference} ## count {#agg_function-count} @@ -1259,7 +1259,7 @@ Otherwise, the result of the calculation is rounded to the nearest multiple of 1 Type: `Float32`. !!! note "Note" - If no values are passed to the function (when using `quantileTimingIf`), [NaN](../../sql-reference/data-types/float.md#data_type-float-nan-inf) is returned. The purpose of this is to differentiate these cases from cases that result in zero. See [ORDER BY clause](../statements/select.md#select-order-by) for notes on sorting `NaN` values. + If no values are passed to the function (when using `quantileTimingIf`), [NaN](../../sql-reference/data-types/float.md#data_type-float-nan-inf) is returned. The purpose of this is to differentiate these cases from cases that result in zero. See [ORDER BY clause](../statements/select/order-by.md#select-order-by) for notes on sorting `NaN` values. **Example** @@ -1344,7 +1344,7 @@ Otherwise, the result of the calculation is rounded to the nearest multiple of 1 Type: `Float32`. !!! note "Note" - If no values are passed to the function (when using `quantileTimingIf`), [NaN](../../sql-reference/data-types/float.md#data_type-float-nan-inf) is returned. The purpose of this is to differentiate these cases from cases that result in zero. See [ORDER BY clause](../statements/select.md#select-order-by) for notes on sorting `NaN` values. + If no values are passed to the function (when using `quantileTimingIf`), [NaN](../../sql-reference/data-types/float.md#data_type-float-nan-inf) is returned. The purpose of this is to differentiate these cases from cases that result in zero. See [ORDER BY clause](../statements/select/order-by.md#select-order-by) for notes on sorting `NaN` values. **Example** diff --git a/docs/en/sql-reference/data-types/aggregatefunction.md b/docs/en/sql-reference/data-types/aggregatefunction.md index f745bd7cf5a..842b5a87578 100644 --- a/docs/en/sql-reference/data-types/aggregatefunction.md +++ b/docs/en/sql-reference/data-types/aggregatefunction.md @@ -5,7 +5,7 @@ toc_title: AggregateFunction # AggregateFunction {#data-type-aggregatefunction} -Aggregate functions can have an implementation-defined intermediate state that can be serialized to an AggregateFunction(…) data type and stored in a table, usually, by means of [a materialized view](../../sql-reference/statements/select.md#create-view). The common way to produce an aggregate function state is by calling the aggregate function with the `-State` suffix. To get the final result of aggregation in the future, you must use the same aggregate function with the `-Merge`suffix. +Aggregate functions can have an implementation-defined intermediate state that can be serialized to an AggregateFunction(…) data type and stored in a table, usually, by means of [a materialized view](../../sql-reference/statements/create.md#create-view). The common way to produce an aggregate function state is by calling the aggregate function with the `-State` suffix. To get the final result of aggregation in the future, you must use the same aggregate function with the `-Merge`suffix. `AggregateFunction(name, types_of_arguments…)` — parametric data type. diff --git a/docs/en/sql-reference/data-types/datetime.md b/docs/en/sql-reference/data-types/datetime.md index d5b6377e658..4e14f90c029 100644 --- a/docs/en/sql-reference/data-types/datetime.md +++ b/docs/en/sql-reference/data-types/datetime.md @@ -121,7 +121,7 @@ FROM dt - [Functions for working with arrays](../../sql-reference/functions/array-functions.md) - [The `date_time_input_format` setting](../../operations/settings/settings.md#settings-date_time_input_format) - [The `timezone` server configuration parameter](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) -- [Operators for working with dates and times](../../sql-reference/operators.md#operators-datetime) +- [Operators for working with dates and times](../../sql-reference/operators/index.md#operators-datetime) - [The `Date` data type](date.md) [Original article](https://clickhouse.tech/docs/en/data_types/datetime/) diff --git a/docs/en/sql-reference/data-types/datetime64.md b/docs/en/sql-reference/data-types/datetime64.md index ae964263a59..0529dcbf6a3 100644 --- a/docs/en/sql-reference/data-types/datetime64.md +++ b/docs/en/sql-reference/data-types/datetime64.md @@ -97,6 +97,6 @@ FROM dt - [Functions for working with arrays](../../sql-reference/functions/array-functions.md) - [The `date_time_input_format` setting](../../operations/settings/settings.md#settings-date_time_input_format) - [The `timezone` server configuration parameter](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) -- [Operators for working with dates and times](../../sql-reference/operators.md#operators-datetime) +- [Operators for working with dates and times](../../sql-reference/operators/index.md#operators-datetime) - [`Date` data type](date.md) - [`DateTime` data type](datetime.md) diff --git a/docs/en/sql-reference/data-types/float.md b/docs/en/sql-reference/data-types/float.md index 646161d34fd..767001c51d2 100644 --- a/docs/en/sql-reference/data-types/float.md +++ b/docs/en/sql-reference/data-types/float.md @@ -80,6 +80,6 @@ SELECT 0 / 0 └──────────────┘ ``` - See the rules for `NaN` sorting in the section [ORDER BY clause](../sql_reference/statements/select.md). + See the rules for `NaN` sorting in the section [ORDER BY clause](../sql_reference/statements/select/order-by.md). [Original article](https://clickhouse.tech/docs/en/data_types/float/) diff --git a/docs/en/sql-reference/data-types/special-data-types/interval.md b/docs/en/sql-reference/data-types/special-data-types/interval.md index 8e1127d2d81..8a4b9ae7886 100644 --- a/docs/en/sql-reference/data-types/special-data-types/interval.md +++ b/docs/en/sql-reference/data-types/special-data-types/interval.md @@ -5,7 +5,7 @@ toc_title: Interval # Interval {#data-type-interval} -The family of data types representing time and date intervals. The resulting types of the [INTERVAL](../../../sql-reference/operators.md#operator-interval) operator. +The family of data types representing time and date intervals. The resulting types of the [INTERVAL](../../../sql-reference/operators/index.md#operator-interval) operator. !!! warning "Warning" `Interval` data type values can’t be stored in tables. @@ -79,5 +79,5 @@ Code: 43. DB::Exception: Received from localhost:9000. DB::Exception: Wrong argu ## See Also {#see-also} -- [INTERVAL](../../../sql-reference/operators.md#operator-interval) operator +- [INTERVAL](../../../sql-reference/operators/index.md#operator-interval) operator - [toInterval](../../../sql-reference/functions/type-conversion-functions.md#function-tointerval) type convertion functions diff --git a/docs/en/sql-reference/data-types/special-data-types/set.md b/docs/en/sql-reference/data-types/special-data-types/set.md index 05ca97910b6..ad15b5d33b9 100644 --- a/docs/en/sql-reference/data-types/special-data-types/set.md +++ b/docs/en/sql-reference/data-types/special-data-types/set.md @@ -5,6 +5,6 @@ toc_title: Set # Set {#set} -Used for the right half of an [IN](../../statements/select.md#select-in-operators) expression. +Used for the right half of an [IN](../../operators/in.md#select-in-operators) expression. [Original article](https://clickhouse.tech/docs/en/data_types/special_data_types/set/) diff --git a/docs/en/sql-reference/data-types/tuple.md b/docs/en/sql-reference/data-types/tuple.md index 39ca918a38a..dce267fb781 100644 --- a/docs/en/sql-reference/data-types/tuple.md +++ b/docs/en/sql-reference/data-types/tuple.md @@ -7,7 +7,7 @@ toc_title: Tuple(T1, T2, ...) A tuple of elements, each having an individual [type](index.md#data_types). -Tuples are used for temporary column grouping. Columns can be grouped when an IN expression is used in a query, and for specifying certain formal parameters of lambda functions. For more information, see the sections [IN operators](../../sql-reference/statements/select.md) and [Higher order functions](../../sql-reference/functions/higher-order-functions.md). +Tuples are used for temporary column grouping. Columns can be grouped when an IN expression is used in a query, and for specifying certain formal parameters of lambda functions. For more information, see the sections [IN operators](../../sql-reference/operators/in.md) and [Higher order functions](../../sql-reference/functions/higher-order-functions.md). Tuples can be the result of a query. In this case, for text formats other than JSON, values are comma-separated in brackets. In JSON formats, tuples are output as arrays (in square brackets). diff --git a/docs/en/sql-reference/functions/conditional-functions.md b/docs/en/sql-reference/functions/conditional-functions.md index ca6e1b70781..b253a699c1f 100644 --- a/docs/en/sql-reference/functions/conditional-functions.md +++ b/docs/en/sql-reference/functions/conditional-functions.md @@ -113,7 +113,7 @@ Returns `then` if the `cond` evaluates to be true (greater than zero), otherwise ## multiIf {#multiif} -Allows you to write the [CASE](../operators.md#operator_case) operator more compactly in the query. +Allows you to write the [CASE](../operators/index.md#operator_case) operator more compactly in the query. Syntax: `multiIf(cond_1, then_1, cond_2, then_2, ..., else)` diff --git a/docs/en/sql-reference/functions/in-functions.md b/docs/en/sql-reference/functions/in-functions.md index 0b197165429..6465c59e916 100644 --- a/docs/en/sql-reference/functions/in-functions.md +++ b/docs/en/sql-reference/functions/in-functions.md @@ -7,7 +7,7 @@ toc_title: Implementing the IN Operator ## in, notIn, globalIn, globalNotIn {#in-functions} -See the section [IN operators](../statements/select.md#select-in-operators). +See the section [IN operators](../operators/in.md#select-in-operators). ## tuple(x, y, …), operator (x, y, …) {#tuplex-y-operator-x-y} diff --git a/docs/en/sql-reference/index.md b/docs/en/sql-reference/index.md index ef316ee26eb..8b460fddd6b 100644 --- a/docs/en/sql-reference/index.md +++ b/docs/en/sql-reference/index.md @@ -7,10 +7,12 @@ toc_title: hidden # SQL Reference {#sql-reference} -- [SELECT](statements/select.md) +ClickHouse supports the following types of queries: + +- [SELECT](statements/select/index.md) - [INSERT INTO](statements/insert-into.md) - [CREATE](statements/create.md) - [ALTER](statements/alter.md#query_language_queries_alter) - [Other types of queries](statements/misc.md) -[Original article](https://clickhouse.tech/docs/en/query_language/) +[Original article](https://clickhouse.tech/docs/en/sql-reference/) diff --git a/docs/en/sql-reference/operators/in.md b/docs/en/sql-reference/operators/in.md new file mode 100644 index 00000000000..e08eb540b1e --- /dev/null +++ b/docs/en/sql-reference/operators/in.md @@ -0,0 +1,199 @@ +### IN Operators {#select-in-operators} + +The `IN`, `NOT IN`, `GLOBAL IN`, and `GLOBAL NOT IN` operators are covered separately, since their functionality is quite rich. + +The left side of the operator is either a single column or a tuple. + +Examples: + +``` sql +SELECT UserID IN (123, 456) FROM ... +SELECT (CounterID, UserID) IN ((34, 123), (101500, 456)) FROM ... +``` + +If the left side is a single column that is in the index, and the right side is a set of constants, the system uses the index for processing the query. + +Don’t list too many values explicitly (i.e. millions). If a data set is large, put it in a temporary table (for example, see the section “External data for query processing”), then use a subquery. + +The right side of the operator can be a set of constant expressions, a set of tuples with constant expressions (shown in the examples above), or the name of a database table or SELECT subquery in brackets. + +If the right side of the operator is the name of a table (for example, `UserID IN users`), this is equivalent to the subquery `UserID IN (SELECT * FROM users)`. Use this when working with external data that is sent along with the query. For example, the query can be sent together with a set of user IDs loaded to the ‘users’ temporary table, which should be filtered. + +If the right side of the operator is a table name that has the Set engine (a prepared data set that is always in RAM), the data set will not be created over again for each query. + +The subquery may specify more than one column for filtering tuples. +Example: + +``` sql +SELECT (CounterID, UserID) IN (SELECT CounterID, UserID FROM ...) FROM ... +``` + +The columns to the left and right of the IN operator should have the same type. + +The IN operator and subquery may occur in any part of the query, including in aggregate functions and lambda functions. +Example: + +``` sql +SELECT + EventDate, + avg(UserID IN + ( + SELECT UserID + FROM test.hits + WHERE EventDate = toDate('2014-03-17') + )) AS ratio +FROM test.hits +GROUP BY EventDate +ORDER BY EventDate ASC +``` + +``` text +┌──EventDate─┬────ratio─┐ +│ 2014-03-17 │ 1 │ +│ 2014-03-18 │ 0.807696 │ +│ 2014-03-19 │ 0.755406 │ +│ 2014-03-20 │ 0.723218 │ +│ 2014-03-21 │ 0.697021 │ +│ 2014-03-22 │ 0.647851 │ +│ 2014-03-23 │ 0.648416 │ +└────────────┴──────────┘ +``` + +For each day after March 17th, count the percentage of pageviews made by users who visited the site on March 17th. +A subquery in the IN clause is always run just one time on a single server. There are no dependent subqueries. + +## NULL Processing {#null-processing-1} + +During request processing, the IN operator assumes that the result of an operation with [NULL](../syntax.md#null-literal) is always equal to `0`, regardless of whether `NULL` is on the right or left side of the operator. `NULL` values are not included in any dataset, do not correspond to each other and cannot be compared. + +Here is an example with the `t_null` table: + +``` text +┌─x─┬────y─┐ +│ 1 │ ᴺᵁᴸᴸ │ +│ 2 │ 3 │ +└───┴──────┘ +``` + +Running the query `SELECT x FROM t_null WHERE y IN (NULL,3)` gives you the following result: + +``` text +┌─x─┐ +│ 2 │ +└───┘ +``` + +You can see that the row in which `y = NULL` is thrown out of the query results. This is because ClickHouse can’t decide whether `NULL` is included in the `(NULL,3)` set, returns `0` as the result of the operation, and `SELECT` excludes this row from the final output. + +``` sql +SELECT y IN (NULL, 3) +FROM t_null +``` + +``` text +┌─in(y, tuple(NULL, 3))─┐ +│ 0 │ +│ 1 │ +└───────────────────────┘ +``` + +## Distributed Subqueries {#select-distributed-subqueries} + +There are two options for IN-s with subqueries (similar to JOINs): normal `IN` / `JOIN` and `GLOBAL IN` / `GLOBAL JOIN`. They differ in how they are run for distributed query processing. + +!!! attention "Attention" + Remember that the algorithms described below may work differently depending on the [settings](../../operations/settings/settings.md) `distributed_product_mode` setting. + +When using the regular IN, the query is sent to remote servers, and each of them runs the subqueries in the `IN` or `JOIN` clause. + +When using `GLOBAL IN` / `GLOBAL JOINs`, first all the subqueries are run for `GLOBAL IN` / `GLOBAL JOINs`, and the results are collected in temporary tables. Then the temporary tables are sent to each remote server, where the queries are run using this temporary data. + +For a non-distributed query, use the regular `IN` / `JOIN`. + +Be careful when using subqueries in the `IN` / `JOIN` clauses for distributed query processing. + +Let’s look at some examples. Assume that each server in the cluster has a normal **local\_table**. Each server also has a **distributed\_table** table with the **Distributed** type, which looks at all the servers in the cluster. + +For a query to the **distributed\_table**, the query will be sent to all the remote servers and run on them using the **local\_table**. + +For example, the query + +``` sql +SELECT uniq(UserID) FROM distributed_table +``` + +will be sent to all remote servers as + +``` sql +SELECT uniq(UserID) FROM local_table +``` + +and run on each of them in parallel, until it reaches the stage where intermediate results can be combined. Then the intermediate results will be returned to the requestor server and merged on it, and the final result will be sent to the client. + +Now let’s examine a query with IN: + +``` sql +SELECT uniq(UserID) FROM distributed_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM local_table WHERE CounterID = 34) +``` + +- Calculation of the intersection of audiences of two sites. + +This query will be sent to all remote servers as + +``` sql +SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM local_table WHERE CounterID = 34) +``` + +In other words, the data set in the IN clause will be collected on each server independently, only across the data that is stored locally on each of the servers. + +This will work correctly and optimally if you are prepared for this case and have spread data across the cluster servers such that the data for a single UserID resides entirely on a single server. In this case, all the necessary data will be available locally on each server. Otherwise, the result will be inaccurate. We refer to this variation of the query as “local IN”. + +To correct how the query works when data is spread randomly across the cluster servers, you could specify **distributed\_table** inside a subquery. The query would look like this: + +``` sql +SELECT uniq(UserID) FROM distributed_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM distributed_table WHERE CounterID = 34) +``` + +This query will be sent to all remote servers as + +``` sql +SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM distributed_table WHERE CounterID = 34) +``` + +The subquery will begin running on each remote server. Since the subquery uses a distributed table, the subquery that is on each remote server will be resent to every remote server as + +``` sql +SELECT UserID FROM local_table WHERE CounterID = 34 +``` + +For example, if you have a cluster of 100 servers, executing the entire query will require 10,000 elementary requests, which is generally considered unacceptable. + +In such cases, you should always use GLOBAL IN instead of IN. Let’s look at how it works for the query + +``` sql +SELECT uniq(UserID) FROM distributed_table WHERE CounterID = 101500 AND UserID GLOBAL IN (SELECT UserID FROM distributed_table WHERE CounterID = 34) +``` + +The requestor server will run the subquery + +``` sql +SELECT UserID FROM distributed_table WHERE CounterID = 34 +``` + +and the result will be put in a temporary table in RAM. Then the request will be sent to each remote server as + +``` sql +SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID GLOBAL IN _data1 +``` + +and the temporary table `_data1` will be sent to every remote server with the query (the name of the temporary table is implementation-defined). + +This is more optimal than using the normal IN. However, keep the following points in mind: + +1. When creating a temporary table, data is not made unique. To reduce the volume of data transmitted over the network, specify DISTINCT in the subquery. (You don’t need to do this for a normal IN.) +2. The temporary table will be sent to all the remote servers. Transmission does not account for network topology. For example, if 10 remote servers reside in a datacenter that is very remote in relation to the requestor server, the data will be sent 10 times over the channel to the remote datacenter. Try to avoid large data sets when using GLOBAL IN. +3. When transmitting data to remote servers, restrictions on network bandwidth are not configurable. You might overload the network. +4. Try to distribute data across servers so that you don’t need to use GLOBAL IN on a regular basis. +5. If you need to use GLOBAL IN often, plan the location of the ClickHouse cluster so that a single group of replicas resides in no more than one data center with a fast network between them, so that a query can be processed entirely within a single data center. + +It also makes sense to specify a local table in the `GLOBAL IN` clause, in case this local table is only available on the requestor server and you want to use data from it on remote servers. diff --git a/docs/en/sql-reference/operators.md b/docs/en/sql-reference/operators/index.md similarity index 87% rename from docs/en/sql-reference/operators.md rename to docs/en/sql-reference/operators/index.md index bae38d01c6a..1758cfa22d9 100644 --- a/docs/en/sql-reference/operators.md +++ b/docs/en/sql-reference/operators/index.md @@ -59,7 +59,7 @@ ClickHouse transforms operators to their corresponding functions at the query pa ## Operators for Working with Data Sets {#operators-for-working-with-data-sets} -*See [IN operators](statements/select.md#select-in-operators).* +*See [IN operators](in.md).* `a IN ...` – The `in(a, b)` function. @@ -90,7 +90,7 @@ The `part` parameter specifies which part of the date to retrieve. The following The `part` parameter is case-insensitive. -The `date` parameter specifies the date or the time to process. Either [Date](../sql-reference/data-types/date.md) or [DateTime](../sql-reference/data-types/datetime.md) type is supported. +The `date` parameter specifies the date or the time to process. Either [Date](../../sql-reference/data-types/date.md) or [DateTime](../../sql-reference/data-types/datetime.md) type is supported. Examples: @@ -137,7 +137,7 @@ You can see more examples in [tests](https://github.com/ClickHouse/ClickHouse/bl ### INTERVAL {#operator-interval} -Creates an [Interval](../sql-reference/data-types/special-data-types/interval.md)-type value that should be used in arithmetical operations with [Date](../sql-reference/data-types/date.md) and [DateTime](../sql-reference/data-types/datetime.md)-type values. +Creates an [Interval](../../sql-reference/data-types/special-data-types/interval.md)-type value that should be used in arithmetical operations with [Date](../../sql-reference/data-types/date.md) and [DateTime](../../sql-reference/data-types/datetime.md)-type values. Types of intervals: - `SECOND` @@ -166,8 +166,8 @@ SELECT now() AS current_date_time, current_date_time + INTERVAL 4 DAY + INTERVAL **See Also** -- [Interval](../sql-reference/data-types/special-data-types/interval.md) data type -- [toInterval](../sql-reference/functions/type-conversion-functions.md#function-tointerval) type convertion functions +- [Interval](../../sql-reference/data-types/special-data-types/interval.md) data type +- [toInterval](../../sql-reference/functions/type-conversion-functions.md#function-tointerval) type convertion functions ## Logical Negation Operator {#logical-negation-operator} @@ -187,7 +187,7 @@ SELECT now() AS current_date_time, current_date_time + INTERVAL 4 DAY + INTERVAL Note: -The conditional operator calculates the values of b and c, then checks whether condition a is met, and then returns the corresponding value. If `b` or `C` is an [arrayJoin()](../sql-reference/functions/array-join.md#functions_arrayjoin) function, each row will be replicated regardless of the “a” condition. +The conditional operator calculates the values of b and c, then checks whether condition a is met, and then returns the corresponding value. If `b` or `C` is an [arrayJoin()](../../sql-reference/functions/array-join.md#functions_arrayjoin) function, each row will be replicated regardless of the “a” condition. ## Conditional Expression {#operator_case} @@ -236,7 +236,7 @@ ClickHouse supports the `IS NULL` and `IS NOT NULL` operators. ### IS NULL {#operator-is-null} -- For [Nullable](../sql-reference/data-types/nullable.md) type values, the `IS NULL` operator returns: +- For [Nullable](../../sql-reference/data-types/nullable.md) type values, the `IS NULL` operator returns: - `1`, if the value is `NULL`. - `0` otherwise. - For other values, the `IS NULL` operator always returns `0`. @@ -255,7 +255,7 @@ SELECT x+100 FROM t_null WHERE y IS NULL ### IS NOT NULL {#is-not-null} -- For [Nullable](../sql-reference/data-types/nullable.md) type values, the `IS NOT NULL` operator returns: +- For [Nullable](../../sql-reference/data-types/nullable.md) type values, the `IS NOT NULL` operator returns: - `0`, if the value is `NULL`. - `1` otherwise. - For other values, the `IS NOT NULL` operator always returns `1`. diff --git a/docs/en/sql-reference/statements/grant.md b/docs/en/sql-reference/statements/grant.md index 5fc534cf05a..f0a1508f1e9 100644 --- a/docs/en/sql-reference/statements/grant.md +++ b/docs/en/sql-reference/statements/grant.md @@ -219,7 +219,7 @@ Some queries by their implementation require a set of privileges. For example, t ### SELECT {#grant-select} -Allows to perform [SELECT](select.md) queries. +Allows to perform [SELECT](select/index.md) queries. Privilege level: `COLUMN`. diff --git a/docs/en/sql-reference/statements/select.md b/docs/en/sql-reference/statements/select.md deleted file mode 100644 index 6be4efebae1..00000000000 --- a/docs/en/sql-reference/statements/select.md +++ /dev/null @@ -1,1377 +0,0 @@ ---- -toc_priority: 33 -toc_title: SELECT ---- - -# SELECT Queries Syntax {#select-queries-syntax} - -`SELECT` performs data retrieval. - -``` sql -[WITH expr_list|(subquery)] -SELECT [DISTINCT] expr_list -[FROM [db.]table | (subquery) | table_function] [FINAL] -[SAMPLE sample_coeff] -[ARRAY JOIN ...] -[GLOBAL] [ANY|ALL] [INNER|LEFT|RIGHT|FULL|CROSS] [OUTER] JOIN (subquery)|table USING columns_list -[PREWHERE expr] -[WHERE expr] -[GROUP BY expr_list] [WITH TOTALS] -[HAVING expr] -[ORDER BY expr_list] -[LIMIT [offset_value, ]n BY columns] -[LIMIT [n, ]m] -[UNION ALL ...] -[INTO OUTFILE filename] -[FORMAT format] -``` - -All the clauses are optional, except for the required list of expressions immediately after SELECT. -The clauses below are described in almost the same order as in the query execution conveyor. - -If the query omits the `DISTINCT`, `GROUP BY` and `ORDER BY` clauses and the `IN` and `JOIN` subqueries, the query will be completely stream processed, using O(1) amount of RAM. -Otherwise, the query might consume a lot of RAM if the appropriate restrictions are not specified: `max_memory_usage`, `max_rows_to_group_by`, `max_rows_to_sort`, `max_rows_in_distinct`, `max_bytes_in_distinct`, `max_rows_in_set`, `max_bytes_in_set`, `max_rows_in_join`, `max_bytes_in_join`, `max_bytes_before_external_sort`, `max_bytes_before_external_group_by`. For more information, see the section “Settings”. It is possible to use external sorting (saving temporary tables to a disk) and external aggregation. `The system does not have "merge join"`. - -### WITH Clause {#with-clause} - -This section provides support for Common Table Expressions ([CTE](https://en.wikipedia.org/wiki/Hierarchical_and_recursive_queries_in_SQL)), with some limitations: -1. Recursive queries are not supported -2. When subquery is used inside WITH section, it’s result should be scalar with exactly one row -3. Expression’s results are not available in subqueries -Results of WITH clause expressions can be used inside SELECT clause. - -Example 1: Using constant expression as “variable” - -``` sql -WITH '2019-08-01 15:23:00' as ts_upper_bound -SELECT * -FROM hits -WHERE - EventDate = toDate(ts_upper_bound) AND - EventTime <= ts_upper_bound -``` - -Example 2: Evicting sum(bytes) expression result from SELECT clause column list - -``` sql -WITH sum(bytes) as s -SELECT - formatReadableSize(s), - table -FROM system.parts -GROUP BY table -ORDER BY s -``` - -Example 3: Using results of scalar subquery - -``` sql -/* this example would return TOP 10 of most huge tables */ -WITH - ( - SELECT sum(bytes) - FROM system.parts - WHERE active - ) AS total_disk_usage -SELECT - (sum(bytes) / total_disk_usage) * 100 AS table_disk_usage, - table -FROM system.parts -GROUP BY table -ORDER BY table_disk_usage DESC -LIMIT 10 -``` - -Example 4: Re-using expression in subquery -As a workaround for current limitation for expression usage in subqueries, you may duplicate it. - -``` sql -WITH ['hello'] AS hello -SELECT - hello, - * -FROM -( - WITH ['hello'] AS hello - SELECT hello -) -``` - -``` text -┌─hello─────┬─hello─────┐ -│ ['hello'] │ ['hello'] │ -└───────────┴───────────┘ -``` - -### FROM Clause {#select-from} - -If the FROM clause is omitted, data will be read from the `system.one` table. -The `system.one` table contains exactly one row (this table fulfills the same purpose as the DUAL table found in other DBMSs). - -The `FROM` clause specifies the source to read data from: - -- Table -- Subquery -- [Table function](../table-functions/index.md#table-functions) - -`ARRAY JOIN` and the regular `JOIN` may also be included (see below). - -Instead of a table, the `SELECT` subquery may be specified in parenthesis. -In contrast to standard SQL, a synonym does not need to be specified after a subquery. - -To execute a query, all the columns listed in the query are extracted from the appropriate table. Any columns not needed for the external query are thrown out of the subqueries. -If a query does not list any columns (for example, `SELECT count() FROM t`), some column is extracted from the table anyway (the smallest one is preferred), in order to calculate the number of rows. - -#### FINAL Modifier {#select-from-final} - -Applicable when selecting data from tables from the [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md)-engine family other than `GraphiteMergeTree`. When `FINAL` is specified, ClickHouse fully merges the data before returning the result and thus performs all data transformations that happen during merges for the given table engine. - -Also supported for: -- [Replicated](../../engines/table-engines/mergetree-family/replication.md) versions of `MergeTree` engines. -- [View](../../engines/table-engines/special/view.md), [Buffer](../../engines/table-engines/special/buffer.md), [Distributed](../../engines/table-engines/special/distributed.md), and [MaterializedView](../../engines/table-engines/special/materializedview.md) engines that operate over other engines, provided they were created over `MergeTree`-engine tables. - -Queries that use `FINAL` are executed not as fast as similar queries that don’t, because: - -- Query is executed in a single thread and data is merged during query execution. -- Queries with `FINAL` read primary key columns in addition to the columns specified in the query. - -In most cases, avoid using `FINAL`. - -### SAMPLE Clause {#select-sample-clause} - -The `SAMPLE` clause allows for approximated query processing. - -When data sampling is enabled, the query is not performed on all the data, but only on a certain fraction of data (sample). For example, if you need to calculate statistics for all the visits, it is enough to execute the query on the 1/10 fraction of all the visits and then multiply the result by 10. - -Approximated query processing can be useful in the following cases: - -- When you have strict timing requirements (like \<100ms) but you can’t justify the cost of additional hardware resources to meet them. -- When your raw data is not accurate, so approximation doesn’t noticeably degrade the quality. -- Business requirements target approximate results (for cost-effectiveness, or in order to market exact results to premium users). - -!!! note "Note" - You can only use sampling with the tables in the [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) family, and only if the sampling expression was specified during table creation (see [MergeTree engine](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table)). - -The features of data sampling are listed below: - -- Data sampling is a deterministic mechanism. The result of the same `SELECT .. SAMPLE` query is always the same. -- Sampling works consistently for different tables. For tables with a single sampling key, a sample with the same coefficient always selects the same subset of possible data. For example, a sample of user IDs takes rows with the same subset of all the possible user IDs from different tables. This means that you can use the sample in subqueries in the [IN](#select-in-operators) clause. Also, you can join samples using the [JOIN](#select-join) clause. -- Sampling allows reading less data from a disk. Note that you must specify the sampling key correctly. For more information, see [Creating a MergeTree Table](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table). - -For the `SAMPLE` clause the following syntax is supported: - -| SAMPLE Clause Syntax | Description | -|----------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `SAMPLE k` | Here `k` is the number from 0 to 1.
The query is executed on `k` fraction of data. For example, `SAMPLE 0.1` runs the query on 10% of data. [Read more](#select-sample-k) | -| `SAMPLE n` | Here `n` is a sufficiently large integer.
The query is executed on a sample of at least `n` rows (but not significantly more than this). For example, `SAMPLE 10000000` runs the query on a minimum of 10,000,000 rows. [Read more](#select-sample-n) | -| `SAMPLE k OFFSET m` | Here `k` and `m` are the numbers from 0 to 1.
The query is executed on a sample of `k` fraction of the data. The data used for the sample is offset by `m` fraction. [Read more](#select-sample-offset) | - -#### SAMPLE K {#select-sample-k} - -Here `k` is the number from 0 to 1 (both fractional and decimal notations are supported). For example, `SAMPLE 1/2` or `SAMPLE 0.5`. - -In a `SAMPLE k` clause, the sample is taken from the `k` fraction of data. The example is shown below: - -``` sql -SELECT - Title, - count() * 10 AS PageViews -FROM hits_distributed -SAMPLE 0.1 -WHERE - CounterID = 34 -GROUP BY Title -ORDER BY PageViews DESC LIMIT 1000 -``` - -In this example, the query is executed on a sample from 0.1 (10%) of data. Values of aggregate functions are not corrected automatically, so to get an approximate result, the value `count()` is manually multiplied by 10. - -#### SAMPLE N {#select-sample-n} - -Here `n` is a sufficiently large integer. For example, `SAMPLE 10000000`. - -In this case, the query is executed on a sample of at least `n` rows (but not significantly more than this). For example, `SAMPLE 10000000` runs the query on a minimum of 10,000,000 rows. - -Since the minimum unit for data reading is one granule (its size is set by the `index_granularity` setting), it makes sense to set a sample that is much larger than the size of the granule. - -When using the `SAMPLE n` clause, you don’t know which relative percent of data was processed. So you don’t know the coefficient the aggregate functions should be multiplied by. Use the `_sample_factor` virtual column to get the approximate result. - -The `_sample_factor` column contains relative coefficients that are calculated dynamically. This column is created automatically when you [create](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table) a table with the specified sampling key. The usage examples of the `_sample_factor` column are shown below. - -Let’s consider the table `visits`, which contains the statistics about site visits. The first example shows how to calculate the number of page views: - -``` sql -SELECT sum(PageViews * _sample_factor) -FROM visits -SAMPLE 10000000 -``` - -The next example shows how to calculate the total number of visits: - -``` sql -SELECT sum(_sample_factor) -FROM visits -SAMPLE 10000000 -``` - -The example below shows how to calculate the average session duration. Note that you don’t need to use the relative coefficient to calculate the average values. - -``` sql -SELECT avg(Duration) -FROM visits -SAMPLE 10000000 -``` - -#### SAMPLE K OFFSET M {#select-sample-offset} - -Here `k` and `m` are numbers from 0 to 1. Examples are shown below. - -**Example 1** - -``` sql -SAMPLE 1/10 -``` - -In this example, the sample is 1/10th of all data: - -`[++------------]` - -**Example 2** - -``` sql -SAMPLE 1/10 OFFSET 1/2 -``` - -Here, a sample of 10% is taken from the second half of the data. - -`[------++------]` - -### ARRAY JOIN Clause {#select-array-join-clause} - -Allows executing `JOIN` with an array or nested data structure. The intent is similar to the [arrayJoin](../functions/array-join.md#functions_arrayjoin) function, but its functionality is broader. - -``` sql -SELECT -FROM -[LEFT] ARRAY JOIN -[WHERE|PREWHERE ] -... -``` - -You can specify only a single `ARRAY JOIN` clause in a query. - -The query execution order is optimized when running `ARRAY JOIN`. Although `ARRAY JOIN` must always be specified before the `WHERE/PREWHERE` clause, it can be performed either before `WHERE/PREWHERE` (if the result is needed in this clause), or after completing it (to reduce the volume of calculations). The processing order is controlled by the query optimizer. - -Supported types of `ARRAY JOIN` are listed below: - -- `ARRAY JOIN` - In this case, empty arrays are not included in the result of `JOIN`. -- `LEFT ARRAY JOIN` - The result of `JOIN` contains rows with empty arrays. The value for an empty array is set to the default value for the array element type (usually 0, empty string or NULL). - -The examples below demonstrate the usage of the `ARRAY JOIN` and `LEFT ARRAY JOIN` clauses. Let’s create a table with an [Array](../../sql-reference/data-types/array.md) type column and insert values into it: - -``` sql -CREATE TABLE arrays_test -( - s String, - arr Array(UInt8) -) ENGINE = Memory; - -INSERT INTO arrays_test -VALUES ('Hello', [1,2]), ('World', [3,4,5]), ('Goodbye', []); -``` - -``` text -┌─s───────────┬─arr─────┐ -│ Hello │ [1,2] │ -│ World │ [3,4,5] │ -│ Goodbye │ [] │ -└─────────────┴─────────┘ -``` - -The example below uses the `ARRAY JOIN` clause: - -``` sql -SELECT s, arr -FROM arrays_test -ARRAY JOIN arr; -``` - -``` text -┌─s─────┬─arr─┐ -│ Hello │ 1 │ -│ Hello │ 2 │ -│ World │ 3 │ -│ World │ 4 │ -│ World │ 5 │ -└───────┴─────┘ -``` - -The next example uses the `LEFT ARRAY JOIN` clause: - -``` sql -SELECT s, arr -FROM arrays_test -LEFT ARRAY JOIN arr; -``` - -``` text -┌─s───────────┬─arr─┐ -│ Hello │ 1 │ -│ Hello │ 2 │ -│ World │ 3 │ -│ World │ 4 │ -│ World │ 5 │ -│ Goodbye │ 0 │ -└─────────────┴─────┘ -``` - -#### Using Aliases {#using-aliases} - -An alias can be specified for an array in the `ARRAY JOIN` clause. In this case, an array item can be accessed by this alias, but the array itself is accessed by the original name. Example: - -``` sql -SELECT s, arr, a -FROM arrays_test -ARRAY JOIN arr AS a; -``` - -``` text -┌─s─────┬─arr─────┬─a─┐ -│ Hello │ [1,2] │ 1 │ -│ Hello │ [1,2] │ 2 │ -│ World │ [3,4,5] │ 3 │ -│ World │ [3,4,5] │ 4 │ -│ World │ [3,4,5] │ 5 │ -└───────┴─────────┴───┘ -``` - -Using aliases, you can perform `ARRAY JOIN` with an external array. For example: - -``` sql -SELECT s, arr_external -FROM arrays_test -ARRAY JOIN [1, 2, 3] AS arr_external; -``` - -``` text -┌─s───────────┬─arr_external─┐ -│ Hello │ 1 │ -│ Hello │ 2 │ -│ Hello │ 3 │ -│ World │ 1 │ -│ World │ 2 │ -│ World │ 3 │ -│ Goodbye │ 1 │ -│ Goodbye │ 2 │ -│ Goodbye │ 3 │ -└─────────────┴──────────────┘ -``` - -Multiple arrays can be comma-separated in the `ARRAY JOIN` clause. In this case, `JOIN` is performed with them simultaneously (the direct sum, not the cartesian product). Note that all the arrays must have the same size. Example: - -``` sql -SELECT s, arr, a, num, mapped -FROM arrays_test -ARRAY JOIN arr AS a, arrayEnumerate(arr) AS num, arrayMap(x -> x + 1, arr) AS mapped; -``` - -``` text -┌─s─────┬─arr─────┬─a─┬─num─┬─mapped─┐ -│ Hello │ [1,2] │ 1 │ 1 │ 2 │ -│ Hello │ [1,2] │ 2 │ 2 │ 3 │ -│ World │ [3,4,5] │ 3 │ 1 │ 4 │ -│ World │ [3,4,5] │ 4 │ 2 │ 5 │ -│ World │ [3,4,5] │ 5 │ 3 │ 6 │ -└───────┴─────────┴───┴─────┴────────┘ -``` - -The example below uses the [arrayEnumerate](../../sql-reference/functions/array-functions.md#array_functions-arrayenumerate) function: - -``` sql -SELECT s, arr, a, num, arrayEnumerate(arr) -FROM arrays_test -ARRAY JOIN arr AS a, arrayEnumerate(arr) AS num; -``` - -``` text -┌─s─────┬─arr─────┬─a─┬─num─┬─arrayEnumerate(arr)─┐ -│ Hello │ [1,2] │ 1 │ 1 │ [1,2] │ -│ Hello │ [1,2] │ 2 │ 2 │ [1,2] │ -│ World │ [3,4,5] │ 3 │ 1 │ [1,2,3] │ -│ World │ [3,4,5] │ 4 │ 2 │ [1,2,3] │ -│ World │ [3,4,5] │ 5 │ 3 │ [1,2,3] │ -└───────┴─────────┴───┴─────┴─────────────────────┘ -``` - -#### ARRAY JOIN with Nested Data Structure {#array-join-with-nested-data-structure} - -`ARRAY`JOIN\`\` also works with [nested data structures](../../sql-reference/data-types/nested-data-structures/nested.md). Example: - -``` sql -CREATE TABLE nested_test -( - s String, - nest Nested( - x UInt8, - y UInt32) -) ENGINE = Memory; - -INSERT INTO nested_test -VALUES ('Hello', [1,2], [10,20]), ('World', [3,4,5], [30,40,50]), ('Goodbye', [], []); -``` - -``` text -┌─s───────┬─nest.x──┬─nest.y─────┐ -│ Hello │ [1,2] │ [10,20] │ -│ World │ [3,4,5] │ [30,40,50] │ -│ Goodbye │ [] │ [] │ -└─────────┴─────────┴────────────┘ -``` - -``` sql -SELECT s, `nest.x`, `nest.y` -FROM nested_test -ARRAY JOIN nest; -``` - -``` text -┌─s─────┬─nest.x─┬─nest.y─┐ -│ Hello │ 1 │ 10 │ -│ Hello │ 2 │ 20 │ -│ World │ 3 │ 30 │ -│ World │ 4 │ 40 │ -│ World │ 5 │ 50 │ -└───────┴────────┴────────┘ -``` - -When specifying names of nested data structures in `ARRAY JOIN`, the meaning is the same as `ARRAY JOIN` with all the array elements that it consists of. Examples are listed below: - -``` sql -SELECT s, `nest.x`, `nest.y` -FROM nested_test -ARRAY JOIN `nest.x`, `nest.y`; -``` - -``` text -┌─s─────┬─nest.x─┬─nest.y─┐ -│ Hello │ 1 │ 10 │ -│ Hello │ 2 │ 20 │ -│ World │ 3 │ 30 │ -│ World │ 4 │ 40 │ -│ World │ 5 │ 50 │ -└───────┴────────┴────────┘ -``` - -This variation also makes sense: - -``` sql -SELECT s, `nest.x`, `nest.y` -FROM nested_test -ARRAY JOIN `nest.x`; -``` - -``` text -┌─s─────┬─nest.x─┬─nest.y─────┐ -│ Hello │ 1 │ [10,20] │ -│ Hello │ 2 │ [10,20] │ -│ World │ 3 │ [30,40,50] │ -│ World │ 4 │ [30,40,50] │ -│ World │ 5 │ [30,40,50] │ -└───────┴────────┴────────────┘ -``` - -An alias may be used for a nested data structure, in order to select either the `JOIN` result or the source array. Example: - -``` sql -SELECT s, `n.x`, `n.y`, `nest.x`, `nest.y` -FROM nested_test -ARRAY JOIN nest AS n; -``` - -``` text -┌─s─────┬─n.x─┬─n.y─┬─nest.x──┬─nest.y─────┐ -│ Hello │ 1 │ 10 │ [1,2] │ [10,20] │ -│ Hello │ 2 │ 20 │ [1,2] │ [10,20] │ -│ World │ 3 │ 30 │ [3,4,5] │ [30,40,50] │ -│ World │ 4 │ 40 │ [3,4,5] │ [30,40,50] │ -│ World │ 5 │ 50 │ [3,4,5] │ [30,40,50] │ -└───────┴─────┴─────┴─────────┴────────────┘ -``` - -Example of using the [arrayEnumerate](../../sql-reference/functions/array-functions.md#array_functions-arrayenumerate) function: - -``` sql -SELECT s, `n.x`, `n.y`, `nest.x`, `nest.y`, num -FROM nested_test -ARRAY JOIN nest AS n, arrayEnumerate(`nest.x`) AS num; -``` - -``` text -┌─s─────┬─n.x─┬─n.y─┬─nest.x──┬─nest.y─────┬─num─┐ -│ Hello │ 1 │ 10 │ [1,2] │ [10,20] │ 1 │ -│ Hello │ 2 │ 20 │ [1,2] │ [10,20] │ 2 │ -│ World │ 3 │ 30 │ [3,4,5] │ [30,40,50] │ 1 │ -│ World │ 4 │ 40 │ [3,4,5] │ [30,40,50] │ 2 │ -│ World │ 5 │ 50 │ [3,4,5] │ [30,40,50] │ 3 │ -└───────┴─────┴─────┴─────────┴────────────┴─────┘ -``` - -### JOIN Clause {#select-join} - -Joins the data in the normal [SQL JOIN](https://en.wikipedia.org/wiki/Join_(SQL)) sense. - -!!! info "Note" - Not related to [ARRAY JOIN](#select-array-join-clause). - -``` sql -SELECT -FROM -[GLOBAL] [ANY|ALL] [INNER|LEFT|RIGHT|FULL|CROSS] [OUTER] JOIN -(ON )|(USING ) ... -``` - -The table names can be specified instead of `` and ``. This is equivalent to the `SELECT * FROM table` subquery, except in a special case when the table has the [Join](../../engines/table-engines/special/join.md) engine – an array prepared for joining. - -#### Supported Types of `JOIN` {#select-join-types} - -- `INNER JOIN` (or `JOIN`) -- `LEFT JOIN` (or `LEFT OUTER JOIN`) -- `RIGHT JOIN` (or `RIGHT OUTER JOIN`) -- `FULL JOIN` (or `FULL OUTER JOIN`) -- `CROSS JOIN` (or `,` ) - -See the standard [SQL JOIN](https://en.wikipedia.org/wiki/Join_(SQL)) description. - -#### Multiple JOIN {#multiple-join} - -Performing queries, ClickHouse rewrites multi-table joins into the sequence of two-table joins. For example, if there are four tables for join ClickHouse joins the first and the second, then joins the result with the third table, and at the last step, it joins the fourth one. - -If a query contains the `WHERE` clause, ClickHouse tries to pushdown filters from this clause through the intermediate join. If it cannot apply the filter to each intermediate join, ClickHouse applies the filters after all joins are completed. - -We recommend the `JOIN ON` or `JOIN USING` syntax for creating queries. For example: - -``` sql -SELECT * FROM t1 JOIN t2 ON t1.a = t2.a JOIN t3 ON t1.a = t3.a -``` - -You can use comma-separated lists of tables in the `FROM` clause. For example: - -``` sql -SELECT * FROM t1, t2, t3 WHERE t1.a = t2.a AND t1.a = t3.a -``` - -Don’t mix these syntaxes. - -ClickHouse doesn’t directly support syntax with commas, so we don’t recommend using them. The algorithm tries to rewrite the query in terms of `CROSS JOIN` and `INNER JOIN` clauses and then proceeds to query processing. When rewriting the query, ClickHouse tries to optimize performance and memory consumption. By default, ClickHouse treats commas as an `INNER JOIN` clause and converts `INNER JOIN` to `CROSS JOIN` when the algorithm cannot guarantee that `INNER JOIN` returns the required data. - -#### Strictness {#select-join-strictness} - -- `ALL` — If the right table has several matching rows, ClickHouse creates a [Cartesian product](https://en.wikipedia.org/wiki/Cartesian_product) from matching rows. This is the standard `JOIN` behavior in SQL. -- `ANY` — If the right table has several matching rows, only the first one found is joined. If the right table has only one matching row, the results of queries with `ANY` and `ALL` keywords are the same. -- `ASOF` — For joining sequences with a non-exact match. `ASOF JOIN` usage is described below. - -**ASOF JOIN Usage** - -`ASOF JOIN` is useful when you need to join records that have no exact match. - -Tables for `ASOF JOIN` must have an ordered sequence column. This column cannot be alone in a table, and should be one of the data types: `UInt32`, `UInt64`, `Float32`, `Float64`, `Date`, and `DateTime`. - -Syntax `ASOF JOIN ... ON`: - -``` sql -SELECT expressions_list -FROM table_1 -ASOF LEFT JOIN table_2 -ON equi_cond AND closest_match_cond -``` - -You can use any number of equality conditions and exactly one closest match condition. For example, `SELECT count() FROM table_1 ASOF LEFT JOIN table_2 ON table_1.a == table_2.b AND table_2.t <= table_1.t`. - -Conditions supported for the closest match: `>`, `>=`, `<`, `<=`. - -Syntax `ASOF JOIN ... USING`: - -``` sql -SELECT expressions_list -FROM table_1 -ASOF JOIN table_2 -USING (equi_column1, ... equi_columnN, asof_column) -``` - -`ASOF JOIN` uses `equi_columnX` for joining on equality and `asof_column` for joining on the closest match with the `table_1.asof_column >= table_2.asof_column` condition. The `asof_column` column always the last one in the `USING` clause. - -For example, consider the following tables: - - table_1 table_2 - event | ev_time | user_id event | ev_time | user_id - ----------|---------|---------- ----------|---------|---------- - ... ... - event_1_1 | 12:00 | 42 event_2_1 | 11:59 | 42 - ... event_2_2 | 12:30 | 42 - event_1_2 | 13:00 | 42 event_2_3 | 13:00 | 42 - ... ... - -`ASOF JOIN` can take the timestamp of a user event from `table_1` and find an event in `table_2` where the timestamp is closest to the timestamp of the event from `table_1` corresponding to the closest match condition. Equal timestamp values are the closest if available. Here, the `user_id` column can be used for joining on equality and the `ev_time` column can be used for joining on the closest match. In our example, `event_1_1` can be joined with `event_2_1` and `event_1_2` can be joined with `event_2_3`, but `event_2_2` can’t be joined. - -!!! note "Note" - `ASOF` join is **not** supported in the [Join](../../engines/table-engines/special/join.md) table engine. - -To set the default strictness value, use the session configuration parameter [join\_default\_strictness](../../operations/settings/settings.md#settings-join_default_strictness). - -#### GLOBAL JOIN {#global-join} - -When using a normal `JOIN`, the query is sent to remote servers. Subqueries are run on each of them in order to make the right table, and the join is performed with this table. In other words, the right table is formed on each server separately. - -When using `GLOBAL ... JOIN`, first the requestor server runs a subquery to calculate the right table. This temporary table is passed to each remote server, and queries are run on them using the temporary data that was transmitted. - -Be careful when using `GLOBAL`. For more information, see the section [Distributed subqueries](#select-distributed-subqueries). - -#### Usage Recommendations {#usage-recommendations} - -When running a `JOIN`, there is no optimization of the order of execution in relation to other stages of the query. The join (a search in the right table) is run before filtering in `WHERE` and before aggregation. In order to explicitly set the processing order, we recommend running a `JOIN` subquery with a subquery. - -Example: - -``` sql -SELECT - CounterID, - hits, - visits -FROM -( - SELECT - CounterID, - count() AS hits - FROM test.hits - GROUP BY CounterID -) ANY LEFT JOIN -( - SELECT - CounterID, - sum(Sign) AS visits - FROM test.visits - GROUP BY CounterID -) USING CounterID -ORDER BY hits DESC -LIMIT 10 -``` - -``` text -┌─CounterID─┬───hits─┬─visits─┐ -│ 1143050 │ 523264 │ 13665 │ -│ 731962 │ 475698 │ 102716 │ -│ 722545 │ 337212 │ 108187 │ -│ 722889 │ 252197 │ 10547 │ -│ 2237260 │ 196036 │ 9522 │ -│ 23057320 │ 147211 │ 7689 │ -│ 722818 │ 90109 │ 17847 │ -│ 48221 │ 85379 │ 4652 │ -│ 19762435 │ 77807 │ 7026 │ -│ 722884 │ 77492 │ 11056 │ -└───────────┴────────┴────────┘ -``` - -Subqueries don’t allow you to set names or use them for referencing a column from a specific subquery. -The columns specified in `USING` must have the same names in both subqueries, and the other columns must be named differently. You can use aliases to change the names of columns in subqueries (the example uses the aliases `hits` and `visits`). - -The `USING` clause specifies one or more columns to join, which establishes the equality of these columns. The list of columns is set without brackets. More complex join conditions are not supported. - -The right table (the subquery result) resides in RAM. If there isn’t enough memory, you can’t run a `JOIN`. - -Each time a query is run with the same `JOIN`, the subquery is run again because the result is not cached. To avoid this, use the special [Join](../../engines/table-engines/special/join.md) table engine, which is a prepared array for joining that is always in RAM. - -In some cases, it is more efficient to use `IN` instead of `JOIN`. -Among the various types of `JOIN`, the most efficient is `ANY LEFT JOIN`, then `ANY INNER JOIN`. The least efficient are `ALL LEFT JOIN` and `ALL INNER JOIN`. - -If you need a `JOIN` for joining with dimension tables (these are relatively small tables that contain dimension properties, such as names for advertising campaigns), a `JOIN` might not be very convenient due to the fact that the right table is re-accessed for every query. For such cases, there is an “external dictionaries” feature that you should use instead of `JOIN`. For more information, see the section [External dictionaries](../dictionaries/external-dictionaries/external-dicts.md). - -**Memory Limitations** - -ClickHouse uses the [hash join](https://en.wikipedia.org/wiki/Hash_join) algorithm. ClickHouse takes the `` and creates a hash table for it in RAM. If you need to restrict join operation memory consumption use the following settings: - -- [max\_rows\_in\_join](../../operations/settings/query-complexity.md#settings-max_rows_in_join) — Limits number of rows in the hash table. -- [max\_bytes\_in\_join](../../operations/settings/query-complexity.md#settings-max_bytes_in_join) — Limits size of the hash table. - -When any of these limits is reached, ClickHouse acts as the [join\_overflow\_mode](../../operations/settings/query-complexity.md#settings-join_overflow_mode) setting instructs. - -#### Processing of Empty or NULL Cells {#processing-of-empty-or-null-cells} - -While joining tables, the empty cells may appear. The setting [join\_use\_nulls](../../operations/settings/settings.md#join_use_nulls) define how ClickHouse fills these cells. - -If the `JOIN` keys are [Nullable](../data-types/nullable.md) fields, the rows where at least one of the keys has the value [NULL](../../sql-reference/syntax.md#null-literal) are not joined. - -#### Syntax Limitations {#syntax-limitations} - -For multiple `JOIN` clauses in a single `SELECT` query: - -- Taking all the columns via `*` is available only if tables are joined, not subqueries. -- The `PREWHERE` clause is not available. - -For `ON`, `WHERE`, and `GROUP BY` clauses: - -- Arbitrary expressions cannot be used in `ON`, `WHERE`, and `GROUP BY` clauses, but you can define an expression in a `SELECT` clause and then use it in these clauses via an alias. - -### WHERE Clause {#select-where} - -If there is a WHERE clause, it must contain an expression with the UInt8 type. This is usually an expression with comparison and logical operators. -This expression will be used for filtering data before all other transformations. - -If indexes are supported by the database table engine, the expression is evaluated on the ability to use indexes. - -### PREWHERE Clause {#prewhere-clause} - -This clause has the same meaning as the WHERE clause. The difference is in which data is read from the table. -When using PREWHERE, first only the columns necessary for executing PREWHERE are read. Then the other columns are read that are needed for running the query, but only those blocks where the PREWHERE expression is true. - -It makes sense to use PREWHERE if there are filtration conditions that are used by a minority of the columns in the query, but that provide strong data filtration. This reduces the volume of data to read. - -For example, it is useful to write PREWHERE for queries that extract a large number of columns, but that only have filtration for a few columns. - -PREWHERE is only supported by tables from the `*MergeTree` family. - -A query may simultaneously specify PREWHERE and WHERE. In this case, PREWHERE precedes WHERE. - -If the ‘optimize\_move\_to\_prewhere’ setting is set to 1 and PREWHERE is omitted, the system uses heuristics to automatically move parts of expressions from WHERE to PREWHERE. - -### GROUP BY Clause {#select-group-by-clause} - -This is one of the most important parts of a column-oriented DBMS. - -If there is a GROUP BY clause, it must contain a list of expressions. Each expression will be referred to here as a “key”. -All the expressions in the SELECT, HAVING, and ORDER BY clauses must be calculated from keys or from aggregate functions. In other words, each column selected from the table must be used either in keys or inside aggregate functions. - -If a query contains only table columns inside aggregate functions, the GROUP BY clause can be omitted, and aggregation by an empty set of keys is assumed. - -Example: - -``` sql -SELECT - count(), - median(FetchTiming > 60 ? 60 : FetchTiming), - count() - sum(Refresh) -FROM hits -``` - -However, in contrast to standard SQL, if the table doesn’t have any rows (either there aren’t any at all, or there aren’t any after using WHERE to filter), an empty result is returned, and not the result from one of the rows containing the initial values of aggregate functions. - -As opposed to MySQL (and conforming to standard SQL), you can’t get some value of some column that is not in a key or aggregate function (except constant expressions). To work around this, you can use the ‘any’ aggregate function (get the first encountered value) or ‘min/max’. - -Example: - -``` sql -SELECT - domainWithoutWWW(URL) AS domain, - count(), - any(Title) AS title -- getting the first occurred page header for each domain. -FROM hits -GROUP BY domain -``` - -For every different key value encountered, GROUP BY calculates a set of aggregate function values. - -GROUP BY is not supported for array columns. - -A constant can’t be specified as arguments for aggregate functions. Example: sum(1). Instead of this, you can get rid of the constant. Example: `count()`. - -#### NULL Processing {#null-processing} - -For grouping, ClickHouse interprets [NULL](../syntax.md#null-literal) as a value, and `NULL=NULL`. - -Here’s an example to show what this means. - -Assume you have this table: - -``` text -┌─x─┬────y─┐ -│ 1 │ 2 │ -│ 2 │ ᴺᵁᴸᴸ │ -│ 3 │ 2 │ -│ 3 │ 3 │ -│ 3 │ ᴺᵁᴸᴸ │ -└───┴──────┘ -``` - -The query `SELECT sum(x), y FROM t_null_big GROUP BY y` results in: - -``` text -┌─sum(x)─┬────y─┐ -│ 4 │ 2 │ -│ 3 │ 3 │ -│ 5 │ ᴺᵁᴸᴸ │ -└────────┴──────┘ -``` - -You can see that `GROUP BY` for `y = NULL` summed up `x`, as if `NULL` is this value. - -If you pass several keys to `GROUP BY`, the result will give you all the combinations of the selection, as if `NULL` were a specific value. - -#### WITH TOTALS Modifier {#with-totals-modifier} - -If the WITH TOTALS modifier is specified, another row will be calculated. This row will have key columns containing default values (zeros or empty lines), and columns of aggregate functions with the values calculated across all the rows (the “total” values). - -This extra row is output in JSON\*, TabSeparated\*, and Pretty\* formats, separately from the other rows. In the other formats, this row is not output. - -In JSON\* formats, this row is output as a separate ‘totals’ field. In TabSeparated\* formats, the row comes after the main result, preceded by an empty row (after the other data). In Pretty\* formats, the row is output as a separate table after the main result. - -`WITH TOTALS` can be run in different ways when HAVING is present. The behavior depends on the ‘totals\_mode’ setting. -By default, `totals_mode = 'before_having'`. In this case, ‘totals’ is calculated across all rows, including the ones that don’t pass through HAVING and ‘max\_rows\_to\_group\_by’. - -The other alternatives include only the rows that pass through HAVING in ‘totals’, and behave differently with the setting `max_rows_to_group_by` and `group_by_overflow_mode = 'any'`. - -`after_having_exclusive` – Don’t include rows that didn’t pass through `max_rows_to_group_by`. In other words, ‘totals’ will have less than or the same number of rows as it would if `max_rows_to_group_by` were omitted. - -`after_having_inclusive` – Include all the rows that didn’t pass through ‘max\_rows\_to\_group\_by’ in ‘totals’. In other words, ‘totals’ will have more than or the same number of rows as it would if `max_rows_to_group_by` were omitted. - -`after_having_auto` – Count the number of rows that passed through HAVING. If it is more than a certain amount (by default, 50%), include all the rows that didn’t pass through ‘max\_rows\_to\_group\_by’ in ‘totals’. Otherwise, do not include them. - -`totals_auto_threshold` – By default, 0.5. The coefficient for `after_having_auto`. - -If `max_rows_to_group_by` and `group_by_overflow_mode = 'any'` are not used, all variations of `after_having` are the same, and you can use any of them (for example, `after_having_auto`). - -You can use WITH TOTALS in subqueries, including subqueries in the JOIN clause (in this case, the respective total values are combined). - -#### GROUP BY in External Memory {#select-group-by-in-external-memory} - -You can enable dumping temporary data to the disk to restrict memory usage during `GROUP BY`. -The [max\_bytes\_before\_external\_group\_by](../../operations/settings/settings.md#settings-max_bytes_before_external_group_by) setting determines the threshold RAM consumption for dumping `GROUP BY` temporary data to the file system. If set to 0 (the default), it is disabled. - -When using `max_bytes_before_external_group_by`, we recommend that you set `max_memory_usage` about twice as high. This is necessary because there are two stages to aggregation: reading the data and forming intermediate data (1) and merging the intermediate data (2). Dumping data to the file system can only occur during stage 1. If the temporary data wasn’t dumped, then stage 2 might require up to the same amount of memory as in stage 1. - -For example, if [max\_memory\_usage](../../operations/settings/settings.md#settings_max_memory_usage) was set to 10000000000 and you want to use external aggregation, it makes sense to set `max_bytes_before_external_group_by` to 10000000000, and max\_memory\_usage to 20000000000. When external aggregation is triggered (if there was at least one dump of temporary data), maximum consumption of RAM is only slightly more than `max_bytes_before_external_group_by`. - -With distributed query processing, external aggregation is performed on remote servers. In order for the requester server to use only a small amount of RAM, set `distributed_aggregation_memory_efficient` to 1. - -When merging data flushed to the disk, as well as when merging results from remote servers when the `distributed_aggregation_memory_efficient` setting is enabled, consumes up to `1/256 * the_number_of_threads` from the total amount of RAM. - -When external aggregation is enabled, if there was less than `max_bytes_before_external_group_by` of data (i.e. data was not flushed), the query runs just as fast as without external aggregation. If any temporary data was flushed, the run time will be several times longer (approximately three times). - -If you have an `ORDER BY` with a `LIMIT` after `GROUP BY`, then the amount of used RAM depends on the amount of data in `LIMIT`, not in the whole table. But if the `ORDER BY` doesn’t have `LIMIT`, don’t forget to enable external sorting (`max_bytes_before_external_sort`). - -### LIMIT BY Clause {#limit-by-clause} - -A query with the `LIMIT n BY expressions` clause selects the first `n` rows for each distinct value of `expressions`. The key for `LIMIT BY` can contain any number of [expressions](../syntax.md#syntax-expressions). - -ClickHouse supports the following syntax: - -- `LIMIT [offset_value, ]n BY expressions` -- `LIMIT n OFFSET offset_value BY expressions` - -During query processing, ClickHouse selects data ordered by sorting key. The sorting key is set explicitly using an [ORDER BY](#select-order-by) clause or implicitly as a property of the table engine. Then ClickHouse applies `LIMIT n BY expressions` and returns the first `n` rows for each distinct combination of `expressions`. If `OFFSET` is specified, then for each data block that belongs to a distinct combination of `expressions`, ClickHouse skips `offset_value` number of rows from the beginning of the block and returns a maximum of `n` rows as a result. If `offset_value` is bigger than the number of rows in the data block, ClickHouse returns zero rows from the block. - -`LIMIT BY` is not related to `LIMIT`. They can both be used in the same query. - -**Examples** - -Sample table: - -``` sql -CREATE TABLE limit_by(id Int, val Int) ENGINE = Memory; -INSERT INTO limit_by values(1, 10), (1, 11), (1, 12), (2, 20), (2, 21); -``` - -Queries: - -``` sql -SELECT * FROM limit_by ORDER BY id, val LIMIT 2 BY id -``` - -``` text -┌─id─┬─val─┐ -│ 1 │ 10 │ -│ 1 │ 11 │ -│ 2 │ 20 │ -│ 2 │ 21 │ -└────┴─────┘ -``` - -``` sql -SELECT * FROM limit_by ORDER BY id, val LIMIT 1, 2 BY id -``` - -``` text -┌─id─┬─val─┐ -│ 1 │ 11 │ -│ 1 │ 12 │ -│ 2 │ 21 │ -└────┴─────┘ -``` - -The `SELECT * FROM limit_by ORDER BY id, val LIMIT 2 OFFSET 1 BY id` query returns the same result. - -The following query returns the top 5 referrers for each `domain, device_type` pair with a maximum of 100 rows in total (`LIMIT n BY + LIMIT`). - -``` sql -SELECT - domainWithoutWWW(URL) AS domain, - domainWithoutWWW(REFERRER_URL) AS referrer, - device_type, - count() cnt -FROM hits -GROUP BY domain, referrer, device_type -ORDER BY cnt DESC -LIMIT 5 BY domain, device_type -LIMIT 100 -``` - -### HAVING Clause {#having-clause} - -Allows filtering the result received after GROUP BY, similar to the WHERE clause. -WHERE and HAVING differ in that WHERE is performed before aggregation (GROUP BY), while HAVING is performed after it. -If aggregation is not performed, HAVING can’t be used. - -### ORDER BY Clause {#select-order-by} - -The ORDER BY clause contains a list of expressions, which can each be assigned DESC or ASC (the sorting direction). If the direction is not specified, ASC is assumed. ASC is sorted in ascending order, and DESC in descending order. The sorting direction applies to a single expression, not to the entire list. Example: `ORDER BY Visits DESC, SearchPhrase` - -For sorting by String values, you can specify collation (comparison). Example: `ORDER BY SearchPhrase COLLATE 'tr'` - for sorting by keyword in ascending order, using the Turkish alphabet, case insensitive, assuming that strings are UTF-8 encoded. COLLATE can be specified or not for each expression in ORDER BY independently. If ASC or DESC is specified, COLLATE is specified after it. When using COLLATE, sorting is always case-insensitive. - -We only recommend using COLLATE for final sorting of a small number of rows, since sorting with COLLATE is less efficient than normal sorting by bytes. - -Rows that have identical values for the list of sorting expressions are output in an arbitrary order, which can also be nondeterministic (different each time). -If the ORDER BY clause is omitted, the order of the rows is also undefined, and may be nondeterministic as well. - -`NaN` and `NULL` sorting order: - -- With the modifier `NULLS FIRST` — First `NULL`, then `NaN`, then other values. -- With the modifier `NULLS LAST` — First the values, then `NaN`, then `NULL`. -- Default — The same as with the `NULLS LAST` modifier. - -Example: - -For the table - -``` text -┌─x─┬────y─┐ -│ 1 │ ᴺᵁᴸᴸ │ -│ 2 │ 2 │ -│ 1 │ nan │ -│ 2 │ 2 │ -│ 3 │ 4 │ -│ 5 │ 6 │ -│ 6 │ nan │ -│ 7 │ ᴺᵁᴸᴸ │ -│ 6 │ 7 │ -│ 8 │ 9 │ -└───┴──────┘ -``` - -Run the query `SELECT * FROM t_null_nan ORDER BY y NULLS FIRST` to get: - -``` text -┌─x─┬────y─┐ -│ 1 │ ᴺᵁᴸᴸ │ -│ 7 │ ᴺᵁᴸᴸ │ -│ 1 │ nan │ -│ 6 │ nan │ -│ 2 │ 2 │ -│ 2 │ 2 │ -│ 3 │ 4 │ -│ 5 │ 6 │ -│ 6 │ 7 │ -│ 8 │ 9 │ -└───┴──────┘ -``` - -When floating point numbers are sorted, NaNs are separate from the other values. Regardless of the sorting order, NaNs come at the end. In other words, for ascending sorting they are placed as if they are larger than all the other numbers, while for descending sorting they are placed as if they are smaller than the rest. - -Less RAM is used if a small enough LIMIT is specified in addition to ORDER BY. Otherwise, the amount of memory spent is proportional to the volume of data for sorting. For distributed query processing, if GROUP BY is omitted, sorting is partially done on remote servers, and the results are merged on the requestor server. This means that for distributed sorting, the volume of data to sort can be greater than the amount of memory on a single server. - -If there is not enough RAM, it is possible to perform sorting in external memory (creating temporary files on a disk). Use the setting `max_bytes_before_external_sort` for this purpose. If it is set to 0 (the default), external sorting is disabled. If it is enabled, when the volume of data to sort reaches the specified number of bytes, the collected data is sorted and dumped into a temporary file. After all data is read, all the sorted files are merged and the results are output. Files are written to the /var/lib/clickhouse/tmp/ directory in the config (by default, but you can use the ‘tmp\_path’ parameter to change this setting). - -Running a query may use more memory than ‘max\_bytes\_before\_external\_sort’. For this reason, this setting must have a value significantly smaller than ‘max\_memory\_usage’. As an example, if your server has 128 GB of RAM and you need to run a single query, set ‘max\_memory\_usage’ to 100 GB, and ‘max\_bytes\_before\_external\_sort’ to 80 GB. - -External sorting works much less effectively than sorting in RAM. - -### SELECT Clause {#select-select} - -[Expressions](../syntax.md#syntax-expressions) specified in the `SELECT` clause are calculated after all the operations in the clauses described above are finished. These expressions work as if they apply to separate rows in the result. If expressions in the `SELECT` clause contain aggregate functions, then ClickHouse processes aggregate functions and expressions used as their arguments during the [GROUP BY](#select-group-by-clause) aggregation. - -If you want to include all columns in the result, use the asterisk (`*`) symbol. For example, `SELECT * FROM ...`. - -To match some columns in the result with a [re2](https://en.wikipedia.org/wiki/RE2_(software)) regular expression, you can use the `COLUMNS` expression. - -``` sql -COLUMNS('regexp') -``` - -For example, consider the table: - -``` sql -CREATE TABLE default.col_names (aa Int8, ab Int8, bc Int8) ENGINE = TinyLog -``` - -The following query selects data from all the columns containing the `a` symbol in their name. - -``` sql -SELECT COLUMNS('a') FROM col_names -``` - -``` text -┌─aa─┬─ab─┐ -│ 1 │ 1 │ -└────┴────┘ -``` - -The selected columns are returned not in the alphabetical order. - -You can use multiple `COLUMNS` expressions in a query and apply functions to them. - -For example: - -``` sql -SELECT COLUMNS('a'), COLUMNS('c'), toTypeName(COLUMNS('c')) FROM col_names -``` - -``` text -┌─aa─┬─ab─┬─bc─┬─toTypeName(bc)─┐ -│ 1 │ 1 │ 1 │ Int8 │ -└────┴────┴────┴────────────────┘ -``` - -Each column returned by the `COLUMNS` expression is passed to the function as a separate argument. Also you can pass other arguments to the function if it supports them. Be careful when using functions. If a function doesn’t support the number of arguments you have passed to it, ClickHouse throws an exception. - -For example: - -``` sql -SELECT COLUMNS('a') + COLUMNS('c') FROM col_names -``` - -``` text -Received exception from server (version 19.14.1): -Code: 42. DB::Exception: Received from localhost:9000. DB::Exception: Number of arguments for function plus doesn't match: passed 3, should be 2. -``` - -In this example, `COLUMNS('a')` returns two columns: `aa` and `ab`. `COLUMNS('c')` returns the `bc` column. The `+` operator can’t apply to 3 arguments, so ClickHouse throws an exception with the relevant message. - -Columns that matched the `COLUMNS` expression can have different data types. If `COLUMNS` doesn’t match any columns and is the only expression in `SELECT`, ClickHouse throws an exception. - -### DISTINCT Clause {#select-distinct} - -If DISTINCT is specified, only a single row will remain out of all the sets of fully matching rows in the result. -The result will be the same as if GROUP BY were specified across all the fields specified in SELECT without aggregate functions. But there are several differences from GROUP BY: - -- DISTINCT can be applied together with GROUP BY. -- When ORDER BY is omitted and LIMIT is defined, the query stops running immediately after the required number of different rows has been read. -- Data blocks are output as they are processed, without waiting for the entire query to finish running. - -DISTINCT is not supported if SELECT has at least one array column. - -`DISTINCT` works with [NULL](../syntax.md#null-literal) as if `NULL` were a specific value, and `NULL=NULL`. In other words, in the `DISTINCT` results, different combinations with `NULL` only occur once. - -ClickHouse supports using the `DISTINCT` and `ORDER BY` clauses for different columns in one query. The `DISTINCT` clause is executed before the `ORDER BY` clause. - -Example table: - -``` text -┌─a─┬─b─┐ -│ 2 │ 1 │ -│ 1 │ 2 │ -│ 3 │ 3 │ -│ 2 │ 4 │ -└───┴───┘ -``` - -When selecting data with the `SELECT DISTINCT a FROM t1 ORDER BY b ASC` query, we get the following result: - -``` text -┌─a─┐ -│ 2 │ -│ 1 │ -│ 3 │ -└───┘ -``` - -If we change the sorting direction `SELECT DISTINCT a FROM t1 ORDER BY b DESC`, we get the following result: - -``` text -┌─a─┐ -│ 3 │ -│ 1 │ -│ 2 │ -└───┘ -``` - -Row `2, 4` was cut before sorting. - -Take this implementation specificity into account when programming queries. - -### LIMIT Clause {#limit-clause} - -`LIMIT m` allows you to select the first `m` rows from the result. - -`LIMIT n, m` allows you to select the first `m` rows from the result after skipping the first `n` rows. The `LIMIT m OFFSET n` syntax is also supported. - -`n` and `m` must be non-negative integers. - -If there isn’t an `ORDER BY` clause that explicitly sorts results, the result may be arbitrary and nondeterministic. - -### UNION ALL Clause {#union-all-clause} - -You can use UNION ALL to combine any number of queries. Example: - -``` sql -SELECT CounterID, 1 AS table, toInt64(count()) AS c - FROM test.hits - GROUP BY CounterID - -UNION ALL - -SELECT CounterID, 2 AS table, sum(Sign) AS c - FROM test.visits - GROUP BY CounterID - HAVING c > 0 -``` - -Only UNION ALL is supported. The regular UNION (UNION DISTINCT) is not supported. If you need UNION DISTINCT, you can write SELECT DISTINCT from a subquery containing UNION ALL. - -Queries that are parts of UNION ALL can be run simultaneously, and their results can be mixed together. - -The structure of results (the number and type of columns) must match for the queries. But the column names can differ. In this case, the column names for the final result will be taken from the first query. Type casting is performed for unions. For example, if two queries being combined have the same field with non-`Nullable` and `Nullable` types from a compatible type, the resulting `UNION ALL` has a `Nullable` type field. - -Queries that are parts of UNION ALL can’t be enclosed in brackets. ORDER BY and LIMIT are applied to separate queries, not to the final result. If you need to apply a conversion to the final result, you can put all the queries with UNION ALL in a subquery in the FROM clause. - -### INTO OUTFILE Clause {#into-outfile-clause} - -Add the `INTO OUTFILE filename` clause (where filename is a string literal) to redirect query output to the specified file. -In contrast to MySQL, the file is created on the client side. The query will fail if a file with the same filename already exists. -This functionality is available in the command-line client and clickhouse-local (a query sent via HTTP interface will fail). - -The default output format is TabSeparated (the same as in the command-line client batch mode). - -### FORMAT Clause {#format-clause} - -Specify ‘FORMAT format’ to get data in any specified format. -You can use this for convenience, or for creating dumps. -For more information, see the section “Formats”. -If the FORMAT clause is omitted, the default format is used, which depends on both the settings and the interface used for accessing the DB. For the HTTP interface and the command-line client in batch mode, the default format is TabSeparated. For the command-line client in interactive mode, the default format is PrettyCompact (it has attractive and compact tables). - -When using the command-line client, data is passed to the client in an internal efficient format. The client independently interprets the FORMAT clause of the query and formats the data itself (thus relieving the network and the server from the load). - -### IN Operators {#select-in-operators} - -The `IN`, `NOT IN`, `GLOBAL IN`, and `GLOBAL NOT IN` operators are covered separately, since their functionality is quite rich. - -The left side of the operator is either a single column or a tuple. - -Examples: - -``` sql -SELECT UserID IN (123, 456) FROM ... -SELECT (CounterID, UserID) IN ((34, 123), (101500, 456)) FROM ... -``` - -If the left side is a single column that is in the index, and the right side is a set of constants, the system uses the index for processing the query. - -Don’t list too many values explicitly (i.e. millions). If a data set is large, put it in a temporary table (for example, see the section “External data for query processing”), then use a subquery. - -The right side of the operator can be a set of constant expressions, a set of tuples with constant expressions (shown in the examples above), or the name of a database table or SELECT subquery in brackets. - -If the right side of the operator is the name of a table (for example, `UserID IN users`), this is equivalent to the subquery `UserID IN (SELECT * FROM users)`. Use this when working with external data that is sent along with the query. For example, the query can be sent together with a set of user IDs loaded to the ‘users’ temporary table, which should be filtered. - -If the right side of the operator is a table name that has the Set engine (a prepared data set that is always in RAM), the data set will not be created over again for each query. - -The subquery may specify more than one column for filtering tuples. -Example: - -``` sql -SELECT (CounterID, UserID) IN (SELECT CounterID, UserID FROM ...) FROM ... -``` - -The columns to the left and right of the IN operator should have the same type. - -The IN operator and subquery may occur in any part of the query, including in aggregate functions and lambda functions. -Example: - -``` sql -SELECT - EventDate, - avg(UserID IN - ( - SELECT UserID - FROM test.hits - WHERE EventDate = toDate('2014-03-17') - )) AS ratio -FROM test.hits -GROUP BY EventDate -ORDER BY EventDate ASC -``` - -``` text -┌──EventDate─┬────ratio─┐ -│ 2014-03-17 │ 1 │ -│ 2014-03-18 │ 0.807696 │ -│ 2014-03-19 │ 0.755406 │ -│ 2014-03-20 │ 0.723218 │ -│ 2014-03-21 │ 0.697021 │ -│ 2014-03-22 │ 0.647851 │ -│ 2014-03-23 │ 0.648416 │ -└────────────┴──────────┘ -``` - -For each day after March 17th, count the percentage of pageviews made by users who visited the site on March 17th. -A subquery in the IN clause is always run just one time on a single server. There are no dependent subqueries. - -#### NULL Processing {#null-processing-1} - -During request processing, the IN operator assumes that the result of an operation with [NULL](../syntax.md#null-literal) is always equal to `0`, regardless of whether `NULL` is on the right or left side of the operator. `NULL` values are not included in any dataset, do not correspond to each other and cannot be compared. - -Here is an example with the `t_null` table: - -``` text -┌─x─┬────y─┐ -│ 1 │ ᴺᵁᴸᴸ │ -│ 2 │ 3 │ -└───┴──────┘ -``` - -Running the query `SELECT x FROM t_null WHERE y IN (NULL,3)` gives you the following result: - -``` text -┌─x─┐ -│ 2 │ -└───┘ -``` - -You can see that the row in which `y = NULL` is thrown out of the query results. This is because ClickHouse can’t decide whether `NULL` is included in the `(NULL,3)` set, returns `0` as the result of the operation, and `SELECT` excludes this row from the final output. - -``` sql -SELECT y IN (NULL, 3) -FROM t_null -``` - -``` text -┌─in(y, tuple(NULL, 3))─┐ -│ 0 │ -│ 1 │ -└───────────────────────┘ -``` - -#### Distributed Subqueries {#select-distributed-subqueries} - -There are two options for IN-s with subqueries (similar to JOINs): normal `IN` / `JOIN` and `GLOBAL IN` / `GLOBAL JOIN`. They differ in how they are run for distributed query processing. - -!!! attention "Attention" - Remember that the algorithms described below may work differently depending on the [settings](../../operations/settings/settings.md) `distributed_product_mode` setting. - -When using the regular IN, the query is sent to remote servers, and each of them runs the subqueries in the `IN` or `JOIN` clause. - -When using `GLOBAL IN` / `GLOBAL JOINs`, first all the subqueries are run for `GLOBAL IN` / `GLOBAL JOINs`, and the results are collected in temporary tables. Then the temporary tables are sent to each remote server, where the queries are run using this temporary data. - -For a non-distributed query, use the regular `IN` / `JOIN`. - -Be careful when using subqueries in the `IN` / `JOIN` clauses for distributed query processing. - -Let’s look at some examples. Assume that each server in the cluster has a normal **local\_table**. Each server also has a **distributed\_table** table with the **Distributed** type, which looks at all the servers in the cluster. - -For a query to the **distributed\_table**, the query will be sent to all the remote servers and run on them using the **local\_table**. - -For example, the query - -``` sql -SELECT uniq(UserID) FROM distributed_table -``` - -will be sent to all remote servers as - -``` sql -SELECT uniq(UserID) FROM local_table -``` - -and run on each of them in parallel, until it reaches the stage where intermediate results can be combined. Then the intermediate results will be returned to the requestor server and merged on it, and the final result will be sent to the client. - -Now let’s examine a query with IN: - -``` sql -SELECT uniq(UserID) FROM distributed_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM local_table WHERE CounterID = 34) -``` - -- Calculation of the intersection of audiences of two sites. - -This query will be sent to all remote servers as - -``` sql -SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM local_table WHERE CounterID = 34) -``` - -In other words, the data set in the IN clause will be collected on each server independently, only across the data that is stored locally on each of the servers. - -This will work correctly and optimally if you are prepared for this case and have spread data across the cluster servers such that the data for a single UserID resides entirely on a single server. In this case, all the necessary data will be available locally on each server. Otherwise, the result will be inaccurate. We refer to this variation of the query as “local IN”. - -To correct how the query works when data is spread randomly across the cluster servers, you could specify **distributed\_table** inside a subquery. The query would look like this: - -``` sql -SELECT uniq(UserID) FROM distributed_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM distributed_table WHERE CounterID = 34) -``` - -This query will be sent to all remote servers as - -``` sql -SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM distributed_table WHERE CounterID = 34) -``` - -The subquery will begin running on each remote server. Since the subquery uses a distributed table, the subquery that is on each remote server will be resent to every remote server as - -``` sql -SELECT UserID FROM local_table WHERE CounterID = 34 -``` - -For example, if you have a cluster of 100 servers, executing the entire query will require 10,000 elementary requests, which is generally considered unacceptable. - -In such cases, you should always use GLOBAL IN instead of IN. Let’s look at how it works for the query - -``` sql -SELECT uniq(UserID) FROM distributed_table WHERE CounterID = 101500 AND UserID GLOBAL IN (SELECT UserID FROM distributed_table WHERE CounterID = 34) -``` - -The requestor server will run the subquery - -``` sql -SELECT UserID FROM distributed_table WHERE CounterID = 34 -``` - -and the result will be put in a temporary table in RAM. Then the request will be sent to each remote server as - -``` sql -SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID GLOBAL IN _data1 -``` - -and the temporary table `_data1` will be sent to every remote server with the query (the name of the temporary table is implementation-defined). - -This is more optimal than using the normal IN. However, keep the following points in mind: - -1. When creating a temporary table, data is not made unique. To reduce the volume of data transmitted over the network, specify DISTINCT in the subquery. (You don’t need to do this for a normal IN.) -2. The temporary table will be sent to all the remote servers. Transmission does not account for network topology. For example, if 10 remote servers reside in a datacenter that is very remote in relation to the requestor server, the data will be sent 10 times over the channel to the remote datacenter. Try to avoid large data sets when using GLOBAL IN. -3. When transmitting data to remote servers, restrictions on network bandwidth are not configurable. You might overload the network. -4. Try to distribute data across servers so that you don’t need to use GLOBAL IN on a regular basis. -5. If you need to use GLOBAL IN often, plan the location of the ClickHouse cluster so that a single group of replicas resides in no more than one data center with a fast network between them, so that a query can be processed entirely within a single data center. - -It also makes sense to specify a local table in the `GLOBAL IN` clause, in case this local table is only available on the requestor server and you want to use data from it on remote servers. - -### Extreme Values {#extreme-values} - -In addition to results, you can also get minimum and maximum values for the results columns. To do this, set the **extremes** setting to 1. Minimums and maximums are calculated for numeric types, dates, and dates with times. For other columns, the default values are output. - -An extra two rows are calculated – the minimums and maximums, respectively. These extra two rows are output in `JSON*`, `TabSeparated*`, and `Pretty*` [formats](../../interfaces/formats.md), separate from the other rows. They are not output for other formats. - -In `JSON*` formats, the extreme values are output in a separate ‘extremes’ field. In `TabSeparated*` formats, the row comes after the main result, and after ‘totals’ if present. It is preceded by an empty row (after the other data). In `Pretty*` formats, the row is output as a separate table after the main result, and after `totals` if present. - -Extreme values are calculated for rows before `LIMIT`, but after `LIMIT BY`. However, when using `LIMIT offset, size`, the rows before `offset` are included in `extremes`. In stream requests, the result may also include a small number of rows that passed through `LIMIT`. - -### Notes {#notes} - -The `GROUP BY` and `ORDER BY` clauses do not support positional arguments. This contradicts MySQL, but conforms to standard SQL. -For example, `GROUP BY 1, 2` will be interpreted as grouping by constants (i.e. aggregation of all rows into one). - -You can use synonyms (`AS` aliases) in any part of a query. - -You can put an asterisk in any part of a query instead of an expression. When the query is analyzed, the asterisk is expanded to a list of all table columns (excluding the `MATERIALIZED` and `ALIAS` columns). There are only a few cases when using an asterisk is justified: - -- When creating a table dump. -- For tables containing just a few columns, such as system tables. -- For getting information about what columns are in a table. In this case, set `LIMIT 1`. But it is better to use the `DESC TABLE` query. -- When there is strong filtration on a small number of columns using `PREWHERE`. -- In subqueries (since columns that aren’t needed for the external query are excluded from subqueries). - -In all other cases, we don’t recommend using the asterisk, since it only gives you the drawbacks of a columnar DBMS instead of the advantages. In other words using the asterisk is not recommended. - -[Original article](https://clickhouse.tech/docs/en/query_language/select/) diff --git a/docs/en/sql-reference/statements/select/array-join.md b/docs/en/sql-reference/statements/select/array-join.md new file mode 100644 index 00000000000..8e05cf51232 --- /dev/null +++ b/docs/en/sql-reference/statements/select/array-join.md @@ -0,0 +1,277 @@ +# ARRAY JOIN Clause {#select-array-join-clause} + +It is a common operation for tables that contain an array column to produce a new table that has a column with each individual array element of that initial column, while values of other columns are duplicated. This is the basic case of what `ARRAY JOIN` clause does. + +Its name comes from the fact that it can be looked at as executing `JOIN` with an array or nested data structure. The intent is similar to the [arrayJoin](../../functions/array-join.md#functions_arrayjoin) function, but the clause functionality is broader. + +Syntax: + +``` sql +SELECT +FROM +[LEFT] ARRAY JOIN +[WHERE|PREWHERE ] +... +``` + +You can specify only one `ARRAY JOIN` clause in a `SELECT` query. + +Supported types of `ARRAY JOIN` are listed below: + +- `ARRAY JOIN` - In base case, empty arrays are not included in the result of `JOIN`. +- `LEFT ARRAY JOIN` - The result of `JOIN` contains rows with empty arrays. The value for an empty array is set to the default value for the array element type (usually 0, empty string or NULL). + +## Basic ARRAY JOIN Examples + +The examples below demonstrate the usage of the `ARRAY JOIN` and `LEFT ARRAY JOIN` clauses. Let’s create a table with an [Array](../../../sql-reference/data-types/array.md) type column and insert values into it: + +``` sql +CREATE TABLE arrays_test +( + s String, + arr Array(UInt8) +) ENGINE = Memory; + +INSERT INTO arrays_test +VALUES ('Hello', [1,2]), ('World', [3,4,5]), ('Goodbye', []); +``` + +``` text +┌─s───────────┬─arr─────┐ +│ Hello │ [1,2] │ +│ World │ [3,4,5] │ +│ Goodbye │ [] │ +└─────────────┴─────────┘ +``` + +The example below uses the `ARRAY JOIN` clause: + +``` sql +SELECT s, arr +FROM arrays_test +ARRAY JOIN arr; +``` + +``` text +┌─s─────┬─arr─┐ +│ Hello │ 1 │ +│ Hello │ 2 │ +│ World │ 3 │ +│ World │ 4 │ +│ World │ 5 │ +└───────┴─────┘ +``` + +The next example uses the `LEFT ARRAY JOIN` clause: + +``` sql +SELECT s, arr +FROM arrays_test +LEFT ARRAY JOIN arr; +``` + +``` text +┌─s───────────┬─arr─┐ +│ Hello │ 1 │ +│ Hello │ 2 │ +│ World │ 3 │ +│ World │ 4 │ +│ World │ 5 │ +│ Goodbye │ 0 │ +└─────────────┴─────┘ +``` + +## Using Aliases {#using-aliases} + +An alias can be specified for an array in the `ARRAY JOIN` clause. In this case, an array item can be accessed by this alias, but the array itself is accessed by the original name. Example: + +``` sql +SELECT s, arr, a +FROM arrays_test +ARRAY JOIN arr AS a; +``` + +``` text +┌─s─────┬─arr─────┬─a─┐ +│ Hello │ [1,2] │ 1 │ +│ Hello │ [1,2] │ 2 │ +│ World │ [3,4,5] │ 3 │ +│ World │ [3,4,5] │ 4 │ +│ World │ [3,4,5] │ 5 │ +└───────┴─────────┴───┘ +``` + +Using aliases, you can perform `ARRAY JOIN` with an external array. For example: + +``` sql +SELECT s, arr_external +FROM arrays_test +ARRAY JOIN [1, 2, 3] AS arr_external; +``` + +``` text +┌─s───────────┬─arr_external─┐ +│ Hello │ 1 │ +│ Hello │ 2 │ +│ Hello │ 3 │ +│ World │ 1 │ +│ World │ 2 │ +│ World │ 3 │ +│ Goodbye │ 1 │ +│ Goodbye │ 2 │ +│ Goodbye │ 3 │ +└─────────────┴──────────────┘ +``` + +Multiple arrays can be comma-separated in the `ARRAY JOIN` clause. In this case, `JOIN` is performed with them simultaneously (the direct sum, not the cartesian product). Note that all the arrays must have the same size. Example: + +``` sql +SELECT s, arr, a, num, mapped +FROM arrays_test +ARRAY JOIN arr AS a, arrayEnumerate(arr) AS num, arrayMap(x -> x + 1, arr) AS mapped; +``` + +``` text +┌─s─────┬─arr─────┬─a─┬─num─┬─mapped─┐ +│ Hello │ [1,2] │ 1 │ 1 │ 2 │ +│ Hello │ [1,2] │ 2 │ 2 │ 3 │ +│ World │ [3,4,5] │ 3 │ 1 │ 4 │ +│ World │ [3,4,5] │ 4 │ 2 │ 5 │ +│ World │ [3,4,5] │ 5 │ 3 │ 6 │ +└───────┴─────────┴───┴─────┴────────┘ +``` + +The example below uses the [arrayEnumerate](../../../sql-reference/functions/array-functions.md#array_functions-arrayenumerate) function: + +``` sql +SELECT s, arr, a, num, arrayEnumerate(arr) +FROM arrays_test +ARRAY JOIN arr AS a, arrayEnumerate(arr) AS num; +``` + +``` text +┌─s─────┬─arr─────┬─a─┬─num─┬─arrayEnumerate(arr)─┐ +│ Hello │ [1,2] │ 1 │ 1 │ [1,2] │ +│ Hello │ [1,2] │ 2 │ 2 │ [1,2] │ +│ World │ [3,4,5] │ 3 │ 1 │ [1,2,3] │ +│ World │ [3,4,5] │ 4 │ 2 │ [1,2,3] │ +│ World │ [3,4,5] │ 5 │ 3 │ [1,2,3] │ +└───────┴─────────┴───┴─────┴─────────────────────┘ +``` + +## ARRAY JOIN with Nested Data Structure {#array-join-with-nested-data-structure} + +`ARRAY JOIN` also works with [nested data structures](../../../sql-reference/data-types/nested-data-structures/nested.md): + +``` sql +CREATE TABLE nested_test +( + s String, + nest Nested( + x UInt8, + y UInt32) +) ENGINE = Memory; + +INSERT INTO nested_test +VALUES ('Hello', [1,2], [10,20]), ('World', [3,4,5], [30,40,50]), ('Goodbye', [], []); +``` + +``` text +┌─s───────┬─nest.x──┬─nest.y─────┐ +│ Hello │ [1,2] │ [10,20] │ +│ World │ [3,4,5] │ [30,40,50] │ +│ Goodbye │ [] │ [] │ +└─────────┴─────────┴────────────┘ +``` + +``` sql +SELECT s, `nest.x`, `nest.y` +FROM nested_test +ARRAY JOIN nest; +``` + +``` text +┌─s─────┬─nest.x─┬─nest.y─┐ +│ Hello │ 1 │ 10 │ +│ Hello │ 2 │ 20 │ +│ World │ 3 │ 30 │ +│ World │ 4 │ 40 │ +│ World │ 5 │ 50 │ +└───────┴────────┴────────┘ +``` + +When specifying names of nested data structures in `ARRAY JOIN`, the meaning is the same as `ARRAY JOIN` with all the array elements that it consists of. Examples are listed below: + +``` sql +SELECT s, `nest.x`, `nest.y` +FROM nested_test +ARRAY JOIN `nest.x`, `nest.y`; +``` + +``` text +┌─s─────┬─nest.x─┬─nest.y─┐ +│ Hello │ 1 │ 10 │ +│ Hello │ 2 │ 20 │ +│ World │ 3 │ 30 │ +│ World │ 4 │ 40 │ +│ World │ 5 │ 50 │ +└───────┴────────┴────────┘ +``` + +This variation also makes sense: + +``` sql +SELECT s, `nest.x`, `nest.y` +FROM nested_test +ARRAY JOIN `nest.x`; +``` + +``` text +┌─s─────┬─nest.x─┬─nest.y─────┐ +│ Hello │ 1 │ [10,20] │ +│ Hello │ 2 │ [10,20] │ +│ World │ 3 │ [30,40,50] │ +│ World │ 4 │ [30,40,50] │ +│ World │ 5 │ [30,40,50] │ +└───────┴────────┴────────────┘ +``` + +An alias may be used for a nested data structure, in order to select either the `JOIN` result or the source array. Example: + +``` sql +SELECT s, `n.x`, `n.y`, `nest.x`, `nest.y` +FROM nested_test +ARRAY JOIN nest AS n; +``` + +``` text +┌─s─────┬─n.x─┬─n.y─┬─nest.x──┬─nest.y─────┐ +│ Hello │ 1 │ 10 │ [1,2] │ [10,20] │ +│ Hello │ 2 │ 20 │ [1,2] │ [10,20] │ +│ World │ 3 │ 30 │ [3,4,5] │ [30,40,50] │ +│ World │ 4 │ 40 │ [3,4,5] │ [30,40,50] │ +│ World │ 5 │ 50 │ [3,4,5] │ [30,40,50] │ +└───────┴─────┴─────┴─────────┴────────────┘ +``` + +Example of using the [arrayEnumerate](../../../sql-reference/functions/array-functions.md#array_functions-arrayenumerate) function: + +``` sql +SELECT s, `n.x`, `n.y`, `nest.x`, `nest.y`, num +FROM nested_test +ARRAY JOIN nest AS n, arrayEnumerate(`nest.x`) AS num; +``` + +``` text +┌─s─────┬─n.x─┬─n.y─┬─nest.x──┬─nest.y─────┬─num─┐ +│ Hello │ 1 │ 10 │ [1,2] │ [10,20] │ 1 │ +│ Hello │ 2 │ 20 │ [1,2] │ [10,20] │ 2 │ +│ World │ 3 │ 30 │ [3,4,5] │ [30,40,50] │ 1 │ +│ World │ 4 │ 40 │ [3,4,5] │ [30,40,50] │ 2 │ +│ World │ 5 │ 50 │ [3,4,5] │ [30,40,50] │ 3 │ +└───────┴─────┴─────┴─────────┴────────────┴─────┘ +``` + +## Implementation Details + +The query execution order is optimized when running `ARRAY JOIN`. Although `ARRAY JOIN` must always be specified before the [WHERE](where.md)/[PREWHERE](prewhere.md) clause in a query, technically they can be performed in any order, unless result of `ARRAY JOIN` is used for filtering. The processing order is controlled by the query optimizer. diff --git a/docs/en/sql-reference/statements/select/distinct.md b/docs/en/sql-reference/statements/select/distinct.md new file mode 100644 index 00000000000..1f500bf469b --- /dev/null +++ b/docs/en/sql-reference/statements/select/distinct.md @@ -0,0 +1,58 @@ +# DISTINCT Clause {#select-distinct} + +If `SELECT DISTINCT` is specified, only unique rows will remain in a query result. Thus only a single row will remain out of all the sets of fully matching rows in the result. + +## Null Processing + +`DISTINCT` works with [NULL](../../syntax.md#null-literal) as if `NULL` were a specific value, and `NULL==NULL`. In other words, in the `DISTINCT` results, different combinations with `NULL` occur only once. It differs from `NULL` processing in most other contexts. + +## Alternatives + +It is possible to obtain the same result by applying [GROUP BY](group-by.md) across the same set of values as specified as `SELECT` clause, without using any aggregate functions. But there are few differences from `GROUP BY` approach: + +- `DISTINCT` can be applied together with `GROUP BY`. +- When [ORDER BY](order-by.md) is omitted and [LIMIT](limit.md) is defined, the query stops running immediately after the required number of different rows has been read. +- Data blocks are output as they are processed, without waiting for the entire query to finish running. + +## Limitations + +`DISTINCT` is not supported if `SELECT` has at least one array column. + +## Examples + +ClickHouse supports using the `DISTINCT` and `ORDER BY` clauses for different columns in one query. The `DISTINCT` clause is executed before the `ORDER BY` clause. + +Example table: + +``` text +┌─a─┬─b─┐ +│ 2 │ 1 │ +│ 1 │ 2 │ +│ 3 │ 3 │ +│ 2 │ 4 │ +└───┴───┘ +``` + +When selecting data with the `SELECT DISTINCT a FROM t1 ORDER BY b ASC` query, we get the following result: + +``` text +┌─a─┐ +│ 2 │ +│ 1 │ +│ 3 │ +└───┘ +``` + +If we change the sorting direction `SELECT DISTINCT a FROM t1 ORDER BY b DESC`, we get the following result: + +``` text +┌─a─┐ +│ 3 │ +│ 1 │ +│ 2 │ +└───┘ +``` + +Row `2, 4` was cut before sorting. + +Take this implementation specificity into account when programming queries. diff --git a/docs/en/sql-reference/statements/select/format.md b/docs/en/sql-reference/statements/select/format.md new file mode 100644 index 00000000000..75fd7256bfc --- /dev/null +++ b/docs/en/sql-reference/statements/select/format.md @@ -0,0 +1,13 @@ +# FORMAT Clause {#format-clause} + +ClickHouse supports a wide range of [serialization formats](../../../interfaces/formats.md) that can be used on query results among other things. There are multiple ways to choose a format for `SELECT` output, one of them is to specify `FORMAT format` at the end of query to get resulting data in any specific format. + +Specific format might be used either for convenience, integration with other systems or performance gain. + +## Default Format + +If the `FORMAT` clause is omitted, the default format is used, which depends on both the settings and the interface used for accessing the ClickHouse server. For the [HTTP interface](../../../interfaces/http.md) and the [command-line client ](../../../interfaces/cli.md) in batch mode, the default format is `TabSeparated`. For the command-line client in interactive mode, the default format is `PrettyCompact` (it produces compact human-readable tables). + +## Implementation Details + +When using the command-line client, data is always passed over the network in an internal efficient format (`Native`). The client independently interprets the `FORMAT` clause of the query and formats the data itself (thus relieving the network and the server from the extra load). diff --git a/docs/en/sql-reference/statements/select/from.md b/docs/en/sql-reference/statements/select/from.md new file mode 100644 index 00000000000..d017537557d --- /dev/null +++ b/docs/en/sql-reference/statements/select/from.md @@ -0,0 +1,39 @@ +# FROM Clause {#select-from} + +The `FROM` clause specifies the source to read data from: + +- [Table](../../../engines/table-engines/index.md) +- [Subquery](index.md) {## TODO: better link ##} +- [Table function](../../table-functions/index.md#table-functions) + +[JOIN](join.md) and [ARRAY JOIN](array-join.md) clauses may also be used to extend the functionality of the `FROM` clause. + +Subquery is another `SELECT` query that may be specified in parenthesis inside `FROM` clause. + +`FROM` clause can contain multiple data sources, separated by commas, which is equivalent of performing [CROSS JOIN](join.md) on them. + +## FINAL Modifier {#select-from-final} + +When `FINAL` is specified, ClickHouse fully merges the data before returning the result and thus performs all data transformations that happen during merges for the given table engine. + +It is applicable when selecting data from tables that use the [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md)-engine family (except `GraphiteMergeTree`). Also supported for: + +- [Replicated](../../../engines/table-engines/mergetree-family/replication.md) versions of `MergeTree` engines. +- [View](../../../engines/table-engines/special/view.md), [Buffer](../../../engines/table-engines/special/buffer.md), [Distributed](../../../engines/table-engines/special/distributed.md), and [MaterializedView](../../../engines/table-engines/special/materializedview.md) engines that operate over other engines, provided they were created over `MergeTree`-engine tables. + +### Drawbacks + +Queries that use `FINAL` are executed not as fast as similar queries that don’t, because: + +- Query is executed in a single thread and data is merged during query execution. +- Queries with `FINAL` read primary key columns in addition to the columns specified in the query. + +**In most cases, avoid using `FINAL`.** The common approach is to use different queries that assume the background processes of the `MergeTree` engine have't happened yet and deal with it by applying aggregation (for example, to discard duplicates). {## TODO: examples ##} + +## Implementation Details + +If the `FROM` clause is omitted, data will be read from the `system.one` table. +The `system.one` table contains exactly one row (this table fulfills the same purpose as the DUAL table found in other DBMSs). + +To execute a query, all the columns listed in the query are extracted from the appropriate table. Any columns not needed for the external query are thrown out of the subqueries. +If a query does not list any columns (for example, `SELECT count() FROM t`), some column is extracted from the table anyway (the smallest one is preferred), in order to calculate the number of rows. diff --git a/docs/en/sql-reference/statements/select/group-by.md b/docs/en/sql-reference/statements/select/group-by.md new file mode 100644 index 00000000000..c9cb275129e --- /dev/null +++ b/docs/en/sql-reference/statements/select/group-by.md @@ -0,0 +1,127 @@ +# GROUP BY Clause {#select-group-by-clause} + +`GROUP BY` clause switches the `SELECT` query into an aggregation mode, which works as follows: + +- `GROUP BY` clause contains a list of expressions (or a single expression, which is considered to be the list of length one). This list acts as a “grouping key”, while each individual expression will be referred to as a “key expressions”. +- All the expressions in the [SELECT](index.md), [HAVING](having.md), and [ORDER BY](order-by.md) clauses **must** be calculated based on key expressions **or** on [aggregate functions](../../../sql-reference/aggregate-functions/index.md) over non-key expressions (including plain columns). In other words, each column selected from the table must be used either in a key expression or inside an aggregate function, but not both. +- Result of aggregating `SELECT` query will contain as many rows as there were unique values of “grouping key” in source table. Usually this signficantly reduces the row count, often by orders of magnitude, but not necessarily: row count stays the same if all “grouping key” values were distinct. + +!!! note "Note" + There's an additional way to run aggregation over a table. If a query contains table columns only inside aggregate functions, the `GROUP BY clause` can be omitted, and aggregation by an empty set of keys is assumed. Such queries always return exactly one row. + +## NULL Processing {#null-processing} + +For grouping, ClickHouse interprets [NULL](../../syntax.md#null-literal) as a value, and `NULL==NULL`. It differs from `NULL` processing in most other contexts. + +Here’s an example to show what this means. + +Assume you have this table: + +``` text +┌─x─┬────y─┐ +│ 1 │ 2 │ +│ 2 │ ᴺᵁᴸᴸ │ +│ 3 │ 2 │ +│ 3 │ 3 │ +│ 3 │ ᴺᵁᴸᴸ │ +└───┴──────┘ +``` + +The query `SELECT sum(x), y FROM t_null_big GROUP BY y` results in: + +``` text +┌─sum(x)─┬────y─┐ +│ 4 │ 2 │ +│ 3 │ 3 │ +│ 5 │ ᴺᵁᴸᴸ │ +└────────┴──────┘ +``` + +You can see that `GROUP BY` for `y = NULL` summed up `x`, as if `NULL` is this value. + +If you pass several keys to `GROUP BY`, the result will give you all the combinations of the selection, as if `NULL` were a specific value. + +## WITH TOTALS Modifier {#with-totals-modifier} + +If the `WITH TOTALS` modifier is specified, another row will be calculated. This row will have key columns containing default values (zeros or empty lines), and columns of aggregate functions with the values calculated across all the rows (the “total” values). + +This extra row is only produced in `JSON*`, `TabSeparated*`, and `Pretty*` formats, separately from the other rows: + +- In `JSON*` formats, this row is output as a separate ‘totals’ field. +- In `TabSeparated*` formats, the row comes after the main result, preceded by an empty row (after the other data). +- In `Pretty*` formats, the row is output as a separate table after the main result. +- In the other formats it is not available. + +`WITH TOTALS` can be run in different ways when [HAVING](having.md) is present. The behavior depends on the `totals_mode` setting. + +### Configuring Totals Processing + +By default, `totals_mode = 'before_having'`. In this case, ‘totals’ is calculated across all rows, including the ones that don’t pass through HAVING and `max_rows_to_group_by`. + +The other alternatives include only the rows that pass through HAVING in ‘totals’, and behave differently with the setting `max_rows_to_group_by` and `group_by_overflow_mode = 'any'`. + +`after_having_exclusive` – Don’t include rows that didn’t pass through `max_rows_to_group_by`. In other words, ‘totals’ will have less than or the same number of rows as it would if `max_rows_to_group_by` were omitted. + +`after_having_inclusive` – Include all the rows that didn’t pass through ‘max\_rows\_to\_group\_by’ in ‘totals’. In other words, ‘totals’ will have more than or the same number of rows as it would if `max_rows_to_group_by` were omitted. + +`after_having_auto` – Count the number of rows that passed through HAVING. If it is more than a certain amount (by default, 50%), include all the rows that didn’t pass through ‘max\_rows\_to\_group\_by’ in ‘totals’. Otherwise, do not include them. + +`totals_auto_threshold` – By default, 0.5. The coefficient for `after_having_auto`. + +If `max_rows_to_group_by` and `group_by_overflow_mode = 'any'` are not used, all variations of `after_having` are the same, and you can use any of them (for example, `after_having_auto`). + +You can use `WITH TOTALS` in subqueries, including subqueries in the [JOIN](join.md) clause (in this case, the respective total values are combined). + +## Examples + +Example: + +``` sql +SELECT + count(), + median(FetchTiming > 60 ? 60 : FetchTiming), + count() - sum(Refresh) +FROM hits +``` + +However, in contrast to standard SQL, if the table doesn’t have any rows (either there aren’t any at all, or there aren’t any after using WHERE to filter), an empty result is returned, and not the result from one of the rows containing the initial values of aggregate functions. + +As opposed to MySQL (and conforming to standard SQL), you can’t get some value of some column that is not in a key or aggregate function (except constant expressions). To work around this, you can use the ‘any’ aggregate function (get the first encountered value) or ‘min/max’. + +Example: + +``` sql +SELECT + domainWithoutWWW(URL) AS domain, + count(), + any(Title) AS title -- getting the first occurred page header for each domain. +FROM hits +GROUP BY domain +``` + +For every different key value encountered, `GROUP BY` calculates a set of aggregate function values. + +`GROUP BY` is not supported for array columns. + +A constant can’t be specified as arguments for aggregate functions. Example: `sum(1)`. Instead of this, you can get rid of the constant. Example: `count()`. + +## Implementation Details + +Aggregation is one of the most important features of a column-oriented DBMS, and thus it's implementation is one of the most heavily optimized parts of ClickHouse. By default, aggregation is done in memory using a hash-table. It has 40+ specializations that are chosen automatically depending on “grouping key” data types. + +### GROUP BY in External Memory {#select-group-by-in-external-memory} + +You can enable dumping temporary data to the disk to restrict memory usage during `GROUP BY`. +The [max\_bytes\_before\_external\_group\_by](../../../operations/settings/settings.md#settings-max_bytes_before_external_group_by) setting determines the threshold RAM consumption for dumping `GROUP BY` temporary data to the file system. If set to 0 (the default), it is disabled. + +When using `max_bytes_before_external_group_by`, we recommend that you set `max_memory_usage` about twice as high. This is necessary because there are two stages to aggregation: reading the data and forming intermediate data (1) and merging the intermediate data (2). Dumping data to the file system can only occur during stage 1. If the temporary data wasn’t dumped, then stage 2 might require up to the same amount of memory as in stage 1. + +For example, if [max\_memory\_usage](../../../operations/settings/settings.md#settings_max_memory_usage) was set to 10000000000 and you want to use external aggregation, it makes sense to set `max_bytes_before_external_group_by` to 10000000000, and `max_memory_usage` to 20000000000. When external aggregation is triggered (if there was at least one dump of temporary data), maximum consumption of RAM is only slightly more than `max_bytes_before_external_group_by`. + +With distributed query processing, external aggregation is performed on remote servers. In order for the requester server to use only a small amount of RAM, set `distributed_aggregation_memory_efficient` to 1. + +When merging data flushed to the disk, as well as when merging results from remote servers when the `distributed_aggregation_memory_efficient` setting is enabled, consumes up to `1/256 * the_number_of_threads` from the total amount of RAM. + +When external aggregation is enabled, if there was less than `max_bytes_before_external_group_by` of data (i.e. data was not flushed), the query runs just as fast as without external aggregation. If any temporary data was flushed, the run time will be several times longer (approximately three times). + +If you have an [ORDER BY](order-by.md) with a [LIMIT](limit.md) after `GROUP BY`, then the amount of used RAM depends on the amount of data in `LIMIT`, not in the whole table. But if the `ORDER BY` doesn’t have `LIMIT`, don’t forget to enable external sorting (`max_bytes_before_external_sort`). diff --git a/docs/en/sql-reference/statements/select/having.md b/docs/en/sql-reference/statements/select/having.md new file mode 100644 index 00000000000..17413645449 --- /dev/null +++ b/docs/en/sql-reference/statements/select/having.md @@ -0,0 +1,9 @@ +# HAVING Clause {#having-clause} + +Allows filtering the aggregation results produced by [GROUP BY](group-by.md). It is similar to the [WHERE](where.md) clause, but the difference is that `WHERE` is performed before aggregation, while `HAVING` is performed after it. + +It is possible to reference aggregation results from `SELECT` clause in `HAVING` clause by their alias. Alternatively, `HAVING` clause can filter on results of additional aggregates that are not returned in query results. + +## Limitations + +`HAVING` can’t be used if aggregation is not performed. Use `WHERE` instead. diff --git a/docs/en/sql-reference/statements/select/index.md b/docs/en/sql-reference/statements/select/index.md new file mode 100644 index 00000000000..4ee75f7b575 --- /dev/null +++ b/docs/en/sql-reference/statements/select/index.md @@ -0,0 +1,160 @@ +--- +toc_priority: 33 +toc_title: SELECT +--- + +# SELECT Queries Syntax {#select-queries-syntax} + +`SELECT` performs data retrieval. + +``` sql +[WITH expr_list|(subquery)] +SELECT [DISTINCT] expr_list +[FROM [db.]table | (subquery) | table_function] [FINAL] +[SAMPLE sample_coeff] +[ARRAY JOIN ...] +[GLOBAL] [ANY|ALL] [INNER|LEFT|RIGHT|FULL|CROSS] [OUTER] JOIN (subquery)|table USING columns_list +[PREWHERE expr] +[WHERE expr] +[GROUP BY expr_list] [WITH TOTALS] +[HAVING expr] +[ORDER BY expr_list] +[LIMIT [offset_value, ]n BY columns] +[LIMIT [n, ]m] +[UNION ALL ...] +[INTO OUTFILE filename] +[FORMAT format] +``` + +All clauses are optional, except for the required list of expressions immediately after `SELECT` which is covered in more detail [below](#select-clause). + +Specifics of each optional clause are covered in separate sections, which are listed in the same order as they are executed: + +- [WITH clause](with.md) +- [FROM clause](from.md) +- [SAMPLE clause](sample.md) +- [JOIN clause](join.md) +- [PREWHERE clause](prewhere.md) +- [WHERE clause](where.md) +- [GROUP BY clause](group-by.md) +- [LIMIT BY clause](limit-by.md) +- [HAVING clause](having.md) +- [SELECT clause](#select-clause) +- [DISTINCT clause](distinct.md) +- [LIMIT clause](limit.md) +- [UNION ALL clause](union-all.md) +- [INTO OUTFILE clause](into-outfile.md) +- [FORMAT clause](format.md) + +## SELECT Clause {#select-clause} + +[Expressions](../../syntax.md#syntax-expressions) specified in the `SELECT` clause are calculated after all the operations in the clauses described above are finished. These expressions work as if they apply to separate rows in the result. If expressions in the `SELECT` clause contain aggregate functions, then ClickHouse processes aggregate functions and expressions used as their arguments during the [GROUP BY](group-by.md) aggregation. + +If you want to include all columns in the result, use the asterisk (`*`) symbol. For example, `SELECT * FROM ...`. + +To match some columns in the result with a [re2](https://en.wikipedia.org/wiki/RE2_(software)) regular expression, you can use the `COLUMNS` expression. + +``` sql +COLUMNS('regexp') +``` + +For example, consider the table: + +``` sql +CREATE TABLE default.col_names (aa Int8, ab Int8, bc Int8) ENGINE = TinyLog +``` + +The following query selects data from all the columns containing the `a` symbol in their name. + +``` sql +SELECT COLUMNS('a') FROM col_names +``` + +``` text +┌─aa─┬─ab─┐ +│ 1 │ 1 │ +└────┴────┘ +``` + +The selected columns are returned not in the alphabetical order. + +You can use multiple `COLUMNS` expressions in a query and apply functions to them. + +For example: + +``` sql +SELECT COLUMNS('a'), COLUMNS('c'), toTypeName(COLUMNS('c')) FROM col_names +``` + +``` text +┌─aa─┬─ab─┬─bc─┬─toTypeName(bc)─┐ +│ 1 │ 1 │ 1 │ Int8 │ +└────┴────┴────┴────────────────┘ +``` + +Each column returned by the `COLUMNS` expression is passed to the function as a separate argument. Also you can pass other arguments to the function if it supports them. Be careful when using functions. If a function doesn’t support the number of arguments you have passed to it, ClickHouse throws an exception. + +For example: + +``` sql +SELECT COLUMNS('a') + COLUMNS('c') FROM col_names +``` + +``` text +Received exception from server (version 19.14.1): +Code: 42. DB::Exception: Received from localhost:9000. DB::Exception: Number of arguments for function plus doesn't match: passed 3, should be 2. +``` + +In this example, `COLUMNS('a')` returns two columns: `aa` and `ab`. `COLUMNS('c')` returns the `bc` column. The `+` operator can’t apply to 3 arguments, so ClickHouse throws an exception with the relevant message. + +Columns that matched the `COLUMNS` expression can have different data types. If `COLUMNS` doesn’t match any columns and is the only expression in `SELECT`, ClickHouse throws an exception. + +### Asterisk + +You can put an asterisk in any part of a query instead of an expression. When the query is analyzed, the asterisk is expanded to a list of all table columns (excluding the `MATERIALIZED` and `ALIAS` columns). There are only a few cases when using an asterisk is justified: + +- When creating a table dump. +- For tables containing just a few columns, such as system tables. +- For getting information about what columns are in a table. In this case, set `LIMIT 1`. But it is better to use the `DESC TABLE` query. +- When there is strong filtration on a small number of columns using `PREWHERE`. +- In subqueries (since columns that aren’t needed for the external query are excluded from subqueries). + +In all other cases, we don’t recommend using the asterisk, since it only gives you the drawbacks of a columnar DBMS instead of the advantages. In other words using the asterisk is not recommended. + +### Extreme Values {#extreme-values} + +In addition to results, you can also get minimum and maximum values for the results columns. To do this, set the **extremes** setting to 1. Minimums and maximums are calculated for numeric types, dates, and dates with times. For other columns, the default values are output. + +An extra two rows are calculated – the minimums and maximums, respectively. These extra two rows are output in `JSON*`, `TabSeparated*`, and `Pretty*` [formats](../../../interfaces/formats.md), separate from the other rows. They are not output for other formats. + +In `JSON*` formats, the extreme values are output in a separate ‘extremes’ field. In `TabSeparated*` formats, the row comes after the main result, and after ‘totals’ if present. It is preceded by an empty row (after the other data). In `Pretty*` formats, the row is output as a separate table after the main result, and after `totals` if present. + +Extreme values are calculated for rows before `LIMIT`, but after `LIMIT BY`. However, when using `LIMIT offset, size`, the rows before `offset` are included in `extremes`. In stream requests, the result may also include a small number of rows that passed through `LIMIT`. + +### Notes {#notes} + +You can use synonyms (`AS` aliases) in any part of a query. + +The `GROUP BY` and `ORDER BY` clauses do not support positional arguments. This contradicts MySQL, but conforms to standard SQL. For example, `GROUP BY 1, 2` will be interpreted as grouping by constants (i.e. aggregation of all rows into one). + + +## Implementation Details + +If the query omits the `DISTINCT`, `GROUP BY` and `ORDER BY` clauses and the `IN` and `JOIN` subqueries, the query will be completely stream processed, using O(1) amount of RAM. Otherwise, the query might consume a lot of RAM if the appropriate restrictions are not specified: + +- `max_memory_usage` +- `max_rows_to_group_by` +- `max_rows_to_sort` +- `max_rows_in_distinct` +- `max_bytes_in_distinct` +- `max_rows_in_set` +- `max_bytes_in_set` +- `max_rows_in_join` +- `max_bytes_in_join` +- `max_bytes_before_external_sort` +- `max_bytes_before_external_group_by` + +For more information, see the section “Settings”. It is possible to use external sorting (saving temporary tables to a disk) and external aggregation. + + +{## [Original article](https://clickhouse.tech/docs/en/sql-reference/statements/select/) ##} diff --git a/docs/en/sql-reference/statements/select/into-outfile.md b/docs/en/sql-reference/statements/select/into-outfile.md new file mode 100644 index 00000000000..4385d31438a --- /dev/null +++ b/docs/en/sql-reference/statements/select/into-outfile.md @@ -0,0 +1,9 @@ +# INTO OUTFILE Clause {#into-outfile-clause} + +Add the `INTO OUTFILE filename` clause (where filename is a string literal) to `SELECT query` to redirect its output to the specified file on the client-side. + +## Implementation Details + +- This functionality is available in the [command-line client](../../../interfaces/cli.md) and [clickhouse-local](../../../operations/utilities/clickhouse-local.md). Thus a query sent via [HTTP interface](../../../interfaces/http.md) will fail. +- The query will fail if a file with the same filename already exists. +- The default [output format](../../../interfaces/formats.md) is `TabSeparated` (like in the command-line client batch mode). diff --git a/docs/en/sql-reference/statements/select/join.md b/docs/en/sql-reference/statements/select/join.md new file mode 100644 index 00000000000..bc160d1fc8d --- /dev/null +++ b/docs/en/sql-reference/statements/select/join.md @@ -0,0 +1,186 @@ +# JOIN Clause {#select-join} + +Join produces a new table by combining columns from one or multiple tables by using values common to each. It is a common operation in databases with SQL support, which corresponds to [relational algebra](https://en.wikipedia.org/wiki/Relational_algebra#Joins_and_join-like_operators) join. The special case of one table join is often referred to as "self-join". + +Syntax: +``` sql +SELECT +FROM +[GLOBAL] [ANY|ALL|ASOF] [INNER|LEFT|RIGHT|FULL|CROSS] [OUTER|SEMI|ANTI] JOIN +(ON )|(USING ) ... +``` + +Expressions from `ON` clause and columns from `USING` clause are called "join keys". Unless otherwise stated, join produces a [Cartesian product](https://en.wikipedia.org/wiki/Cartesian_product) from rows with matching "join keys", which might produce results with much more rows than the source tables. + +## Supported Types of JOIN {#select-join-types} + +All standard [SQL JOIN](https://en.wikipedia.org/wiki/Join_(SQL)) types are supported: + +- `INNER JOIN`, only matching rows are returned. +- `LEFT OUTER JOIN`, non-matching rows from left table are returned in addition to matching rows. +- `RIGHT OUTER JOIN`, non-matching rows from right table are returned in addition to matching rows. +- `FULL OUTER JOIN`, non-matching rows from both tables are returned in addition to matching rows. +- `CROSS JOIN`, produces cartesian product of whole tables, "join keys" are **not** specified. + +`JOIN` without specified type implies `INNER`. Keyword `OUTER` can be safely omitted. Alternative syntax for `CROSS JOIN` is specifying multiple tables in [FROM clause](from.md) separated by commas. + +Additional join types available in ClickHouse: + +- `LEFT SEMI JOIN` and `RIGHT SEMI JOIN`, a whitelist on "join keys", without producing a cartesian product. +- `LEFT ANTI JOIN` and `RIGHT ANTI JOIN`, a blacklist on "join keys", without producing a cartesian product. + +## Strictness {#select-join-strictness} + +Modifies how matching by "join keys" is performed + +- `ALL` — The standard `JOIN` behavior in SQL as described above. The default. +- `ANY` — Partially (for opposite side of `LEFT` and `RIGHT`) or completely (for `INNER` and `FULL`) disables the cartesian product for standard `JOIN` types. +- `ASOF` — For joining sequences with a non-exact match. `ASOF JOIN` usage is described below. + +!!! note "Note" + The default strictness value can be overriden using [join\_default\_strictness](../../../operations/settings/settings.md#settings-join_default_strictness) setting. + + +### ASOF JOIN Usage + +`ASOF JOIN` is useful when you need to join records that have no exact match. + +Tables for `ASOF JOIN` must have an ordered sequence column. This column cannot be alone in a table, and should be one of the data types: `UInt32`, `UInt64`, `Float32`, `Float64`, `Date`, and `DateTime`. + +Syntax `ASOF JOIN ... ON`: + +``` sql +SELECT expressions_list +FROM table_1 +ASOF LEFT JOIN table_2 +ON equi_cond AND closest_match_cond +``` + +You can use any number of equality conditions and exactly one closest match condition. For example, `SELECT count() FROM table_1 ASOF LEFT JOIN table_2 ON table_1.a == table_2.b AND table_2.t <= table_1.t`. + +Conditions supported for the closest match: `>`, `>=`, `<`, `<=`. + +Syntax `ASOF JOIN ... USING`: + +``` sql +SELECT expressions_list +FROM table_1 +ASOF JOIN table_2 +USING (equi_column1, ... equi_columnN, asof_column) +``` + +`ASOF JOIN` uses `equi_columnX` for joining on equality and `asof_column` for joining on the closest match with the `table_1.asof_column >= table_2.asof_column` condition. The `asof_column` column always the last one in the `USING` clause. + +For example, consider the following tables: + + table_1 table_2 + event | ev_time | user_id event | ev_time | user_id + ----------|---------|---------- ----------|---------|---------- + ... ... + event_1_1 | 12:00 | 42 event_2_1 | 11:59 | 42 + ... event_2_2 | 12:30 | 42 + event_1_2 | 13:00 | 42 event_2_3 | 13:00 | 42 + ... ... + +`ASOF JOIN` can take the timestamp of a user event from `table_1` and find an event in `table_2` where the timestamp is closest to the timestamp of the event from `table_1` corresponding to the closest match condition. Equal timestamp values are the closest if available. Here, the `user_id` column can be used for joining on equality and the `ev_time` column can be used for joining on the closest match. In our example, `event_1_1` can be joined with `event_2_1` and `event_1_2` can be joined with `event_2_3`, but `event_2_2` can’t be joined. + +!!! note "Note" + `ASOF` join is **not** supported in the [Join](../../../engines/table-engines/special/join.md) table engine. + +## Distributed Join {#global-join} + +There are two ways to execute join involving distributed tables: + +- When using a normal `JOIN`, the query is sent to remote servers. Subqueries are run on each of them in order to make the right table, and the join is performed with this table. In other words, the right table is formed on each server separately. +- When using `GLOBAL ... JOIN`, first the requestor server runs a subquery to calculate the right table. This temporary table is passed to each remote server, and queries are run on them using the temporary data that was transmitted. + +Be careful when using `GLOBAL`. For more information, see the [Distributed subqueries](../../operators/in.md#select-distributed-subqueries) section. + +## Usage Recommendations {#usage-recommendations} + +### Processing of Empty or NULL Cells {#processing-of-empty-or-null-cells} + +While joining tables, the empty cells may appear. The setting [join\_use\_nulls](../../../operations/settings/settings.md#join_use_nulls) define how ClickHouse fills these cells. + +If the `JOIN` keys are [Nullable](../../data-types/nullable.md) fields, the rows where at least one of the keys has the value [NULL](../../../sql-reference/syntax.md#null-literal) are not joined. + +### Syntax + +The columns specified in `USING` must have the same names in both subqueries, and the other columns must be named differently. You can use aliases to change the names of columns in subqueries. + +The `USING` clause specifies one or more columns to join, which establishes the equality of these columns. The list of columns is set without brackets. More complex join conditions are not supported. + +### Syntax Limitations {#syntax-limitations} + +For multiple `JOIN` clauses in a single `SELECT` query: + +- Taking all the columns via `*` is available only if tables are joined, not subqueries. +- The `PREWHERE` clause is not available. + +For `ON`, `WHERE`, and `GROUP BY` clauses: + +- Arbitrary expressions cannot be used in `ON`, `WHERE`, and `GROUP BY` clauses, but you can define an expression in a `SELECT` clause and then use it in these clauses via an alias. + +### Performance + +When running a `JOIN`, there is no optimization of the order of execution in relation to other stages of the query. The join (a search in the right table) is run before filtering in `WHERE` and before aggregation. + +Each time a query is run with the same `JOIN`, the subquery is run again because the result is not cached. To avoid this, use the special [Join](../../../engines/table-engines/special/join.md) table engine, which is a prepared array for joining that is always in RAM. + +In some cases, it is more efficient to use [IN](../../operators/in.md) instead of `JOIN`. + +If you need a `JOIN` for joining with dimension tables (these are relatively small tables that contain dimension properties, such as names for advertising campaigns), a `JOIN` might not be very convenient due to the fact that the right table is re-accessed for every query. For such cases, there is an “external dictionaries” feature that you should use instead of `JOIN`. For more information, see the [External dictionaries](../../dictionaries/external-dictionaries/external-dicts.md) section. + +### Memory Limitations + +By default, ClickHouse uses the [hash join](https://en.wikipedia.org/wiki/Hash_join) algorithm. ClickHouse takes the `` and creates a hash table for it in RAM. After some threshold of memory consumption, ClickHouse falls back to merge join algorithm. + +If you need to restrict join operation memory consumption use the following settings: + +- [max\_rows\_in\_join](../../../operations/settings/query-complexity.md#settings-max_rows_in_join) — Limits number of rows in the hash table. +- [max\_bytes\_in\_join](../../../operations/settings/query-complexity.md#settings-max_bytes_in_join) — Limits size of the hash table. + +When any of these limits is reached, ClickHouse acts as the [join\_overflow\_mode](../../../operations/settings/query-complexity.md#settings-join_overflow_mode) setting instructs. + +## Examples + +Example: + +``` sql +SELECT + CounterID, + hits, + visits +FROM +( + SELECT + CounterID, + count() AS hits + FROM test.hits + GROUP BY CounterID +) ANY LEFT JOIN +( + SELECT + CounterID, + sum(Sign) AS visits + FROM test.visits + GROUP BY CounterID +) USING CounterID +ORDER BY hits DESC +LIMIT 10 +``` + +``` text +┌─CounterID─┬───hits─┬─visits─┐ +│ 1143050 │ 523264 │ 13665 │ +│ 731962 │ 475698 │ 102716 │ +│ 722545 │ 337212 │ 108187 │ +│ 722889 │ 252197 │ 10547 │ +│ 2237260 │ 196036 │ 9522 │ +│ 23057320 │ 147211 │ 7689 │ +│ 722818 │ 90109 │ 17847 │ +│ 48221 │ 85379 │ 4652 │ +│ 19762435 │ 77807 │ 7026 │ +│ 722884 │ 77492 │ 11056 │ +└───────────┴────────┴────────┘ +``` diff --git a/docs/en/sql-reference/statements/select/limit-by.md b/docs/en/sql-reference/statements/select/limit-by.md new file mode 100644 index 00000000000..b6af13fb8d2 --- /dev/null +++ b/docs/en/sql-reference/statements/select/limit-by.md @@ -0,0 +1,66 @@ +# LIMIT BY Clause {#limit-by-clause} + +A query with the `LIMIT n BY expressions` clause selects the first `n` rows for each distinct value of `expressions`. The key for `LIMIT BY` can contain any number of [expressions](../../syntax.md#syntax-expressions). + +ClickHouse supports the following syntax variants: + +- `LIMIT [offset_value, ]n BY expressions` +- `LIMIT n OFFSET offset_value BY expressions` + +During query processing, ClickHouse selects data ordered by sorting key. The sorting key is set explicitly using an [ORDER BY](order-by.md) clause or implicitly as a property of the table engine. Then ClickHouse applies `LIMIT n BY expressions` and returns the first `n` rows for each distinct combination of `expressions`. If `OFFSET` is specified, then for each data block that belongs to a distinct combination of `expressions`, ClickHouse skips `offset_value` number of rows from the beginning of the block and returns a maximum of `n` rows as a result. If `offset_value` is bigger than the number of rows in the data block, ClickHouse returns zero rows from the block. + +!!! note "Note" + `LIMIT BY` is not related to [LIMIT](limit.md). They can both be used in the same query. + +## Examples + +Sample table: + +``` sql +CREATE TABLE limit_by(id Int, val Int) ENGINE = Memory; +INSERT INTO limit_by VALUES (1, 10), (1, 11), (1, 12), (2, 20), (2, 21); +``` + +Queries: + +``` sql +SELECT * FROM limit_by ORDER BY id, val LIMIT 2 BY id +``` + +``` text +┌─id─┬─val─┐ +│ 1 │ 10 │ +│ 1 │ 11 │ +│ 2 │ 20 │ +│ 2 │ 21 │ +└────┴─────┘ +``` + +``` sql +SELECT * FROM limit_by ORDER BY id, val LIMIT 1, 2 BY id +``` + +``` text +┌─id─┬─val─┐ +│ 1 │ 11 │ +│ 1 │ 12 │ +│ 2 │ 21 │ +└────┴─────┘ +``` + +The `SELECT * FROM limit_by ORDER BY id, val LIMIT 2 OFFSET 1 BY id` query returns the same result. + +The following query returns the top 5 referrers for each `domain, device_type` pair with a maximum of 100 rows in total (`LIMIT n BY + LIMIT`). + +``` sql +SELECT + domainWithoutWWW(URL) AS domain, + domainWithoutWWW(REFERRER_URL) AS referrer, + device_type, + count() cnt +FROM hits +GROUP BY domain, referrer, device_type +ORDER BY cnt DESC +LIMIT 5 BY domain, device_type +LIMIT 100 +``` diff --git a/docs/en/sql-reference/statements/select/limit.md b/docs/en/sql-reference/statements/select/limit.md new file mode 100644 index 00000000000..8f47904cecb --- /dev/null +++ b/docs/en/sql-reference/statements/select/limit.md @@ -0,0 +1,9 @@ +# LIMIT Clause {#limit-clause} + +`LIMIT m` allows to select the first `m` rows from the result. + +`LIMIT n, m` allows to select the `m` rows from the result after skipping the first `n` rows. The `LIMIT m OFFSET n` syntax is equivalent. + +`n` and `m` must be non-negative integers. + +If there is no [ORDER BY](order-by.md) clause that explicitly sorts results, the choice of rows for the result may be arbitrary and non-deterministic. diff --git a/docs/en/sql-reference/statements/select/order-by.md b/docs/en/sql-reference/statements/select/order-by.md new file mode 100644 index 00000000000..39bd19f8eab --- /dev/null +++ b/docs/en/sql-reference/statements/select/order-by.md @@ -0,0 +1,67 @@ +# ORDER BY Clause {#select-order-by} + +The `ORDER BY` clause contains a list of expressions, which can each be attributed with `DESC` (descending) or `ASC` (ascending) modifier which determine the sorting direction. If the direction is not specified, `ASC` is assumed, so it's usually omitted. The sorting direction applies to a single expression, not to the entire list. Example: `ORDER BY Visits DESC, SearchPhrase` + +Rows that have identical values for the list of sorting expressions are output in an arbitrary order, which can also be non-deterministic (different each time). +If the ORDER BY clause is omitted, the order of the rows is also undefined, and may be non-deterministic as well. + +## Sorting of Special Values + +There are two approaches to `NaN` and `NULL` sorting order: + +- By default or with the `NULLS LAST` modifier: first the values, then `NaN`, then `NULL`. +- With the `NULLS FIRST` modifier: first `NULL`, then `NaN`, then other values. + +### Example + +For the table + +``` text +┌─x─┬────y─┐ +│ 1 │ ᴺᵁᴸᴸ │ +│ 2 │ 2 │ +│ 1 │ nan │ +│ 2 │ 2 │ +│ 3 │ 4 │ +│ 5 │ 6 │ +│ 6 │ nan │ +│ 7 │ ᴺᵁᴸᴸ │ +│ 6 │ 7 │ +│ 8 │ 9 │ +└───┴──────┘ +``` + +Run the query `SELECT * FROM t_null_nan ORDER BY y NULLS FIRST` to get: + +``` text +┌─x─┬────y─┐ +│ 1 │ ᴺᵁᴸᴸ │ +│ 7 │ ᴺᵁᴸᴸ │ +│ 1 │ nan │ +│ 6 │ nan │ +│ 2 │ 2 │ +│ 2 │ 2 │ +│ 3 │ 4 │ +│ 5 │ 6 │ +│ 6 │ 7 │ +│ 8 │ 9 │ +└───┴──────┘ +``` + +When floating point numbers are sorted, NaNs are separate from the other values. Regardless of the sorting order, NaNs come at the end. In other words, for ascending sorting they are placed as if they are larger than all the other numbers, while for descending sorting they are placed as if they are smaller than the rest. + +## Collation Support + +For sorting by String values, you can specify collation (comparison). Example: `ORDER BY SearchPhrase COLLATE 'tr'` - for sorting by keyword in ascending order, using the Turkish alphabet, case insensitive, assuming that strings are UTF-8 encoded. `COLLATE` can be specified or not for each expression in ORDER BY independently. If `ASC` or `DESC` is specified, `COLLATE` is specified after it. When using `COLLATE`, sorting is always case-insensitive. + +We only recommend using `COLLATE` for final sorting of a small number of rows, since sorting with `COLLATE` is less efficient than normal sorting by bytes. + +## Implementation Details + +Less RAM is used if a small enough [LIMIT](limit.md) is specified in addition to `ORDER BY`. Otherwise, the amount of memory spent is proportional to the volume of data for sorting. For distributed query processing, if [GROUP BY](group-by.md) is omitted, sorting is partially done on remote servers, and the results are merged on the requestor server. This means that for distributed sorting, the volume of data to sort can be greater than the amount of memory on a single server. + +If there is not enough RAM, it is possible to perform sorting in external memory (creating temporary files on a disk). Use the setting `max_bytes_before_external_sort` for this purpose. If it is set to 0 (the default), external sorting is disabled. If it is enabled, when the volume of data to sort reaches the specified number of bytes, the collected data is sorted and dumped into a temporary file. After all data is read, all the sorted files are merged and the results are output. Files are written to the `/var/lib/clickhouse/tmp/` directory in the config (by default, but you can use the `tmp_path` parameter to change this setting). + +Running a query may use more memory than `max_bytes_before_external_sort`. For this reason, this setting must have a value significantly smaller than `max_memory_usage`. As an example, if your server has 128 GB of RAM and you need to run a single query, set `max_memory_usage` to 100 GB, and `max_bytes_before_external_sort` to 80 GB. + +External sorting works much less effectively than sorting in RAM. diff --git a/docs/en/sql-reference/statements/select/prewhere.md b/docs/en/sql-reference/statements/select/prewhere.md new file mode 100644 index 00000000000..765514a6835 --- /dev/null +++ b/docs/en/sql-reference/statements/select/prewhere.md @@ -0,0 +1,18 @@ +# PREWHERE Clause {#prewhere-clause} + +Prewhere is an optimization to apply filtering more efficiently. It is enabled by default even if `PREWHERE` clause is not specified explicitly. It works by automatically moving part of [WHERE](where.md) condition to prewhere stage. The role of `PREWHERE` clause is only to control this optimization if you think that you know how to do it better than it happens by default. + +With prewhere optimization, at first only the columns necessary for executing prewhere expression are read. Then the other columns are read that are needed for running the rest of the query, but only those blocks where the prewhere expression is "true" at least for some rows. If there are a lot of blocks where prewhere expression is "false" for all rows and prewhere needs less columns than other parts of query, this often allows to read a lot less data from disk for query execution. + +## Controlling Prewhere Manually + +The clause has the same meaning as the `WHERE` clause. The difference is in which data is read from the table. When manually controlling `PREWHERE` for filtration conditions that are used by a minority of the columns in the query, but that provide strong data filtration. This reduces the volume of data to read. + +A query may simultaneously specify `PREWHERE` and `WHERE`. In this case, `PREWHERE` precedes `WHERE`. + +If the `optimize_move_to_prewhere` setting is set to 0, heuristics to automatically move parts of expressions from `WHERE` to `PREWHERE` are disabled. + +## Limitations + +`PREWHERE` is only supported by tables from the `*MergeTree` family. + diff --git a/docs/en/sql-reference/statements/select/sample.md b/docs/en/sql-reference/statements/select/sample.md new file mode 100644 index 00000000000..955711e74c0 --- /dev/null +++ b/docs/en/sql-reference/statements/select/sample.md @@ -0,0 +1,108 @@ +# SAMPLE Clause {#select-sample-clause} + +The `SAMPLE` clause allows for approximated `SELECT` query processing. + +When data sampling is enabled, the query is not performed on all the data, but only on a certain fraction of data (sample). For example, if you need to calculate statistics for all the visits, it is enough to execute the query on the 1/10 fraction of all the visits and then multiply the result by 10. + +Approximated query processing can be useful in the following cases: + +- When you have strict timing requirements (like \<100ms) but you can’t justify the cost of additional hardware resources to meet them. +- When your raw data is not accurate, so approximation doesn’t noticeably degrade the quality. +- Business requirements target approximate results (for cost-effectiveness, or to market exact results to premium users). + +!!! note "Note" + You can only use sampling with the tables in the [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md) family, and only if the sampling expression was specified during table creation (see [MergeTree engine](../../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table)). + +The features of data sampling are listed below: + +- Data sampling is a deterministic mechanism. The result of the same `SELECT .. SAMPLE` query is always the same. +- Sampling works consistently for different tables. For tables with a single sampling key, a sample with the same coefficient always selects the same subset of possible data. For example, a sample of user IDs takes rows with the same subset of all the possible user IDs from different tables. This means that you can use the sample in subqueries in the [IN](../../operators/in.md) clause. Also, you can join samples using the [JOIN](join.md) clause. +- Sampling allows reading less data from a disk. Note that you must specify the sampling key correctly. For more information, see [Creating a MergeTree Table](../../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table). + +For the `SAMPLE` clause the following syntax is supported: + +| SAMPLE Clause Syntax | Description | +|----------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `SAMPLE k` | Here `k` is the number from 0 to 1.
The query is executed on `k` fraction of data. For example, `SAMPLE 0.1` runs the query on 10% of data. [Read more](#select-sample-k) | +| `SAMPLE n` | Here `n` is a sufficiently large integer.
The query is executed on a sample of at least `n` rows (but not significantly more than this). For example, `SAMPLE 10000000` runs the query on a minimum of 10,000,000 rows. [Read more](#select-sample-n) | +| `SAMPLE k OFFSET m` | Here `k` and `m` are the numbers from 0 to 1.
The query is executed on a sample of `k` fraction of the data. The data used for the sample is offset by `m` fraction. [Read more](#select-sample-offset) | + +## SAMPLE K {#select-sample-k} + +Here `k` is the number from 0 to 1 (both fractional and decimal notations are supported). For example, `SAMPLE 1/2` or `SAMPLE 0.5`. + +In a `SAMPLE k` clause, the sample is taken from the `k` fraction of data. The example is shown below: + +``` sql +SELECT + Title, + count() * 10 AS PageViews +FROM hits_distributed +SAMPLE 0.1 +WHERE + CounterID = 34 +GROUP BY Title +ORDER BY PageViews DESC LIMIT 1000 +``` + +In this example, the query is executed on a sample from 0.1 (10%) of data. Values of aggregate functions are not corrected automatically, so to get an approximate result, the value `count()` is manually multiplied by 10. + +## SAMPLE N {#select-sample-n} + +Here `n` is a sufficiently large integer. For example, `SAMPLE 10000000`. + +In this case, the query is executed on a sample of at least `n` rows (but not significantly more than this). For example, `SAMPLE 10000000` runs the query on a minimum of 10,000,000 rows. + +Since the minimum unit for data reading is one granule (its size is set by the `index_granularity` setting), it makes sense to set a sample that is much larger than the size of the granule. + +When using the `SAMPLE n` clause, you don’t know which relative percent of data was processed. So you don’t know the coefficient the aggregate functions should be multiplied by. Use the `_sample_factor` virtual column to get the approximate result. + +The `_sample_factor` column contains relative coefficients that are calculated dynamically. This column is created automatically when you [create](../../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table) a table with the specified sampling key. The usage examples of the `_sample_factor` column are shown below. + +Let’s consider the table `visits`, which contains the statistics about site visits. The first example shows how to calculate the number of page views: + +``` sql +SELECT sum(PageViews * _sample_factor) +FROM visits +SAMPLE 10000000 +``` + +The next example shows how to calculate the total number of visits: + +``` sql +SELECT sum(_sample_factor) +FROM visits +SAMPLE 10000000 +``` + +The example below shows how to calculate the average session duration. Note that you don’t need to use the relative coefficient to calculate the average values. + +``` sql +SELECT avg(Duration) +FROM visits +SAMPLE 10000000 +``` + +## SAMPLE K OFFSET M {#select-sample-offset} + +Here `k` and `m` are numbers from 0 to 1. Examples are shown below. + +**Example 1** + +``` sql +SAMPLE 1/10 +``` + +In this example, the sample is 1/10th of all data: + +`[++------------]` + +**Example 2** + +``` sql +SAMPLE 1/10 OFFSET 1/2 +``` + +Here, a sample of 10% is taken from the second half of the data. + +`[------++------]` diff --git a/docs/en/sql-reference/statements/select/union-all.md b/docs/en/sql-reference/statements/select/union-all.md new file mode 100644 index 00000000000..f89d15e0539 --- /dev/null +++ b/docs/en/sql-reference/statements/select/union-all.md @@ -0,0 +1,30 @@ +# UNION ALL Clause {#union-all-clause} + +You can use `UNION ALL` to combine any number of `SELECT` queries by extending their results. Example: + +``` sql +SELECT CounterID, 1 AS table, toInt64(count()) AS c + FROM test.hits + GROUP BY CounterID + +UNION ALL + +SELECT CounterID, 2 AS table, sum(Sign) AS c + FROM test.visits + GROUP BY CounterID + HAVING c > 0 +``` + +Result columns are matched by their index (order inside `SELECT`). If column names do not match, names for the final result are taken from the first query. + +Type casting is performed for unions. For example, if two queries being combined have the same field with non-`Nullable` and `Nullable` types from a compatible type, the resulting `UNION ALL` has a `Nullable` type field. + +Queries that are parts of `UNION ALL` can’t be enclosed in round brackets. [ORDER BY](order-by.md) and [LIMIT](limit.md) are applied to separate queries, not to the final result. If you need to apply a conversion to the final result, you can put all the queries with `UNION ALL` in a subquery in the [FROM](from.md) clause. + +## Limitations + +Only `UNION ALL` is supported. The regular `UNION` (`UNION DISTINCT`) is not supported. If you need `UNION DISTINCT`, you can write `SELECT DISTINCT` from a subquery containing `UNION ALL`. + +## Implementation Details + +Queries that are parts of `UNION ALL` can be run simultaneously, and their results can be mixed together. diff --git a/docs/en/sql-reference/statements/select/where.md b/docs/en/sql-reference/statements/select/where.md new file mode 100644 index 00000000000..c5a6c6add0f --- /dev/null +++ b/docs/en/sql-reference/statements/select/where.md @@ -0,0 +1,10 @@ +# WHERE Clause {#select-where} + +`WHERE` clause allows to filter the data that is coming from [FROM](from.md) clause of `SELECT`. + +If there is a `WHERE` clause, it must contain an expression with the `UInt8` type. This is usually an expression with comparison and logical operators. Rows where this expression evaluates to 0 are expluded from further transformations or result. + +`WHERE` expression is evaluated on the ability to use indexes and partition pruning, if the underlying table engine supports that. + +!!! note "Note" + There's a filtering optimization called [prewhere](prewhere.md). diff --git a/docs/en/sql-reference/statements/select/with.md b/docs/en/sql-reference/statements/select/with.md new file mode 100644 index 00000000000..d6530964e64 --- /dev/null +++ b/docs/en/sql-reference/statements/select/with.md @@ -0,0 +1,75 @@ +# WITH Clause {#with-clause} + +This section provides support for Common Table Expressions ([CTE](https://en.wikipedia.org/wiki/Hierarchical_and_recursive_queries_in_SQL)), so the results of `WITH` clause can be used in the rest of `SELECT` query. + +## Limitations + +1. Recursive queries are not supported. +2. When subquery is used inside WITH section, it’s result should be scalar with exactly one row. +3. Expression’s results are not available in subqueries. + +## Examples + +**Example 1:** Using constant expression as “variable” + +``` sql +WITH '2019-08-01 15:23:00' as ts_upper_bound +SELECT * +FROM hits +WHERE + EventDate = toDate(ts_upper_bound) AND + EventTime <= ts_upper_bound +``` + +**Example 2:** Evicting sum(bytes) expression result from SELECT clause column list + +``` sql +WITH sum(bytes) as s +SELECT + formatReadableSize(s), + table +FROM system.parts +GROUP BY table +ORDER BY s +``` + +**Example 3:** Using results of scalar subquery + +``` sql +/* this example would return TOP 10 of most huge tables */ +WITH + ( + SELECT sum(bytes) + FROM system.parts + WHERE active + ) AS total_disk_usage +SELECT + (sum(bytes) / total_disk_usage) * 100 AS table_disk_usage, + table +FROM system.parts +GROUP BY table +ORDER BY table_disk_usage DESC +LIMIT 10 +``` + +**Example 4:** Re-using expression in subquery + +As a workaround for current limitation for expression usage in subqueries, you may duplicate it. + +``` sql +WITH ['hello'] AS hello +SELECT + hello, + * +FROM +( + WITH ['hello'] AS hello + SELECT hello +) +``` + +``` text +┌─hello─────┬─hello─────┐ +│ ['hello'] │ ['hello'] │ +└───────────┴───────────┘ +``` diff --git a/docs/en/sql-reference/statements/show.md b/docs/en/sql-reference/statements/show.md index f9e35bb63c4..24db87c6171 100644 --- a/docs/en/sql-reference/statements/show.md +++ b/docs/en/sql-reference/statements/show.md @@ -101,7 +101,6 @@ SHOW DICTIONARIES FROM db LIKE '%reg%' LIMIT 2 ``` - ## SHOW GRANTS {#show-grants-statement} Shows privileges for a user. diff --git a/docs/en/sql-reference/syntax.md b/docs/en/sql-reference/syntax.md index 26a2945740a..9ff5a388ad3 100644 --- a/docs/en/sql-reference/syntax.md +++ b/docs/en/sql-reference/syntax.md @@ -101,7 +101,7 @@ Depending on the data format (input or output), `NULL` may have a different repr There are many nuances to processing `NULL`. For example, if at least one of the arguments of a comparison operation is `NULL`, the result of this operation is also `NULL`. The same is true for multiplication, addition, and other operations. For more information, read the documentation for each operation. -In queries, you can check `NULL` using the [IS NULL](operators.md#operator-is-null) and [IS NOT NULL](operators.md) operators and the related functions `isNull` and `isNotNull`. +In queries, you can check `NULL` using the [IS NULL](operators/index.md#operator-is-null) and [IS NOT NULL](operators/index.md) operators and the related functions `isNull` and `isNotNull`. ## Functions {#functions} diff --git a/docs/en/sql-reference/table-functions/index.md b/docs/en/sql-reference/table-functions/index.md index ee23b56e7c4..515e7083e5e 100644 --- a/docs/en/sql-reference/table-functions/index.md +++ b/docs/en/sql-reference/table-functions/index.md @@ -10,7 +10,7 @@ Table functions are methods for constructing tables. You can use table functions in: -- [FROM](../statements/select.md#select-from) clause of the `SELECT` query. +- [FROM](../statements/select/from.md) clause of the `SELECT` query. The method for creating a temporary table that is available only in the current query. The table is deleted when the query finishes. diff --git a/docs/es/commercial/cloud.md b/docs/es/commercial/cloud.md index e4d61390b22..bc593a82ad7 100644 --- a/docs/es/commercial/cloud.md +++ b/docs/es/commercial/cloud.md @@ -1,14 +1,16 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_priority: 1 +toc_title: Nube --- -# Proveedores De Servicios En La Nube De ClickHouse {#clickhouse-cloud-service-providers} +# Proveedores de servicios en la nube de ClickHouse {#clickhouse-cloud-service-providers} !!! info "INFO" Si ha lanzado una nube pública con el servicio ClickHouse administrado, no dude en [abrir una solicitud de extracción](https://github.com/ClickHouse/ClickHouse/edit/master/docs/en/commercial/cloud.md) añadiéndolo a la siguiente lista. -## Nube De Yandex {#yandex-cloud} +## Nube de Yandex {#yandex-cloud} [Servicio administrado de Yandex para ClickHouse](https://cloud.yandex.com/services/managed-clickhouse?utm_source=referrals&utm_medium=clickhouseofficialsite&utm_campaign=link3) proporciona las siguientes características clave: diff --git a/docs/es/commercial/index.md b/docs/es/commercial/index.md index 80bb81700f5..b367631ae1c 100644 --- a/docs/es/commercial/index.md +++ b/docs/es/commercial/index.md @@ -1,8 +1,9 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa -toc_folder_title: Commercial +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: Comercial toc_priority: 70 +toc_title: Comercial --- diff --git a/docs/es/commercial/support.md b/docs/es/commercial/support.md deleted file mode 120000 index 1eb20ccf36a..00000000000 --- a/docs/es/commercial/support.md +++ /dev/null @@ -1 +0,0 @@ -../../en/commercial/support.md \ No newline at end of file diff --git a/docs/es/commercial/support.md b/docs/es/commercial/support.md new file mode 100644 index 00000000000..a817d90dcb5 --- /dev/null +++ b/docs/es/commercial/support.md @@ -0,0 +1,23 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_priority: 3 +toc_title: Apoyo +--- + +# Proveedores de servicios de soporte comercial ClickHouse {#clickhouse-commercial-support-service-providers} + +!!! info "INFO" + Si ha lanzado un servicio de soporte comercial ClickHouse, no dude en [abrir una solicitud de extracción](https://github.com/ClickHouse/ClickHouse/edit/master/docs/en/commercial/support.md) añadiéndolo a la siguiente lista. + +## Altinidad {#altinity} + +Altinity ha ofrecido soporte y servicios empresariales ClickHouse desde 2017. Los clientes de Altinity van desde empresas Fortune 100 hasta startups. Visitar [Más información](https://www.altinity.com/) para más información. + +## Mafiree {#mafiree} + +[Descripción del servicio](http://mafiree.com/clickhouse-analytics-services.php) + +## MinervaDB {#minervadb} + +[Descripción del servicio](https://minervadb.com/index.php/clickhouse-consulting-and-support-by-minervadb/) diff --git a/docs/es/development/architecture.md b/docs/es/development/architecture.md index 384a1dddb70..90f028541f3 100644 --- a/docs/es/development/architecture.md +++ b/docs/es/development/architecture.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 62 toc_title: "Descripci\xF3n general de la arquitectura ClickHouse" --- -# Descripción General De La Arquitectura ClickHouse {#overview-of-clickhouse-architecture} +# Descripción general de la arquitectura ClickHouse {#overview-of-clickhouse-architecture} ClickHouse es un verdadero DBMS orientado a columnas. Los datos se almacenan por columnas y durante la ejecución de matrices (vectores o fragmentos de columnas). Siempre que sea posible, las operaciones se envían en matrices, en lugar de en valores individuales. Se llama “vectorized query execution,” y ayuda a reducir el costo del procesamiento de datos real. @@ -25,13 +25,13 @@ Sin embargo, también es posible trabajar con valores individuales. Para represe `Field` no tiene suficiente información sobre un tipo de datos específico para una tabla. Por ejemplo, `UInt8`, `UInt16`, `UInt32`, y `UInt64` todos están representados como `UInt64` en una `Field`. -## Abstracciones Con Fugas {#leaky-abstractions} +## Abstracciones con fugas {#leaky-abstractions} `IColumn` tiene métodos para transformaciones relacionales comunes de datos, pero no satisfacen todas las necesidades. Por ejemplo, `ColumnUInt64` no tiene un método para calcular la suma de dos columnas, y `ColumnString` no tiene un método para ejecutar una búsqueda de subcadena. Estas innumerables rutinas se implementan fuera de `IColumn`. Varias funciones en columnas se pueden implementar de una manera genérica, no eficiente utilizando `IColumn` para extraer `Field` valores, o de una manera especializada utilizando el conocimiento del diseño de la memoria interna de los datos en un `IColumn` aplicación. Se implementa mediante la conversión de funciones a un `IColumn` escriba y trate con la representación interna directamente. Por ejemplo, `ColumnUInt64` tiene el `getData` método que devuelve una referencia a una matriz interna, luego una rutina separada lee o llena esa matriz directamente. Tenemos “leaky abstractions” para permitir especializaciones eficientes de varias rutinas. -## Tipos De Datos {#data_types} +## Tipos de datos {#data_types} `IDataType` es responsable de la serialización y deserialización: para leer y escribir fragmentos de columnas o valores individuales en formato binario o de texto. `IDataType` corresponde directamente a los tipos de datos en las tablas. Por ejemplo, hay `DataTypeUInt32`, `DataTypeDateTime`, `DataTypeString` y así sucesivamente. @@ -49,7 +49,7 @@ Cuando calculamos alguna función sobre columnas en un bloque, agregamos otra co Se crean bloques para cada fragmento de datos procesado. Tenga en cuenta que para el mismo tipo de cálculo, los nombres y tipos de columna siguen siendo los mismos para diferentes bloques y solo cambian los datos de columna. Es mejor dividir los datos del bloque desde el encabezado del bloque porque los tamaños de bloque pequeños tienen una gran sobrecarga de cadenas temporales para copiar shared\_ptrs y nombres de columna. -## Bloquear Flujos {#block-streams} +## Bloquear flujos {#block-streams} Los flujos de bloques son para procesar datos. Usamos flujos de bloques para leer datos de algún lugar, realizar transformaciones de datos o escribir datos en algún lugar. `IBlockInputStream` tiene el `read` método para buscar el siguiente bloque mientras esté disponible. `IBlockOutputStream` tiene el `write` método para empujar el bloque en alguna parte. @@ -120,7 +120,7 @@ Los intérpretes son responsables de crear la canalización de ejecución de con Hay funciones ordinarias y funciones agregadas. Para las funciones agregadas, consulte la siguiente sección. -Ordinary functions don’t change the number of rows – they work as if they are processing each row independently. In fact, functions are not called for individual rows, but for `Block`de datos para implementar la ejecución de consultas vectorizadas. +Ordinary functions don't change the number of rows – they work as if they are processing each row independently. In fact, functions are not called for individual rows, but for `Block`de datos para implementar la ejecución de consultas vectorizadas. Hay algunas funciones diversas, como [BlockSize](../sql-reference/functions/other-functions.md#function-blocksize), [rowNumberInBlock](../sql-reference/functions/other-functions.md#function-rownumberinblock), y [runningAccumulate](../sql-reference/functions/other-functions.md#function-runningaccumulate), que explotan el procesamiento de bloques y violan la independencia de las filas. @@ -132,7 +132,7 @@ Es un excelente lugar para implementar la generación de código en tiempo de ej Debido a la ejecución de consultas vectorizadas, las funciones no se cortocircuitan. Por ejemplo, si escribe `WHERE f(x) AND g(y)`, ambos lados se calculan, incluso para las filas, cuando `f(x)` es cero (excepto cuando `f(x)` es una expresión constante cero). Pero si la selectividad del `f(x)` la condición es alta, y el cálculo de `f(x)` es mucho más barato que `g(y)`, es mejor implementar el cálculo de paso múltiple. Primero calcularía `f(x)`, a continuación, filtrar columnas por el resultado, y luego calcular `g(y)` solo para trozos de datos más pequeños y filtrados. -## Funciones Agregadas {#aggregate-functions} +## Funciones agregadas {#aggregate-functions} Las funciones agregadas son funciones con estado. Acumulan valores pasados en algún estado y le permiten obtener resultados de ese estado. Se gestionan con el `IAggregateFunction` interfaz. Los estados pueden ser bastante simples (el estado para `AggregateFunctionCount` es sólo una sola `UInt64` valor) o bastante complejo (el estado de `AggregateFunctionUniqCombined` es una combinación de una matriz lineal, una tabla hash, y un `HyperLogLog` estructura de datos probabilística). @@ -159,7 +159,7 @@ Mantenemos una compatibilidad total con versiones anteriores y posteriores para !!! note "Nota" Para la mayoría de las aplicaciones externas, recomendamos usar la interfaz HTTP porque es simple y fácil de usar. El protocolo TCP está más estrechamente vinculado a las estructuras de datos internas: utiliza un formato interno para pasar bloques de datos y utiliza marcos personalizados para datos comprimidos. No hemos lanzado una biblioteca C para ese protocolo porque requiere vincular la mayor parte de la base de código ClickHouse, lo cual no es práctico. -## Ejecución De Consultas Distribuidas {#distributed-query-execution} +## Ejecución de consultas distribuidas {#distributed-query-execution} Los servidores de una configuración de clúster son en su mayoría independientes. Puede crear un `Distributed` en uno o todos los servidores de un clúster. El `Distributed` table does not store data itself – it only provides a “view” a todas las tablas locales en varios nodos de un clúster. Cuando se SELECCIONA desde un `Distributed` tabla, reescribe esa consulta, elige nodos remotos de acuerdo con la configuración de equilibrio de carga y les envía la consulta. El `Distributed` table solicita a los servidores remotos que procesen una consulta hasta una etapa en la que se pueden fusionar resultados intermedios de diferentes servidores. Luego recibe los resultados intermedios y los fusiona. La tabla distribuida intenta distribuir tanto trabajo como sea posible a servidores remotos y no envía muchos datos intermedios a través de la red. @@ -167,7 +167,7 @@ Las cosas se vuelven más complicadas cuando tiene subconsultas en cláusulas IN No existe un plan de consulta global para la ejecución de consultas distribuidas. Cada nodo tiene su plan de consulta local para su parte del trabajo. Solo tenemos una ejecución simple de consultas distribuidas de un solo paso: enviamos consultas para nodos remotos y luego fusionamos los resultados. Pero esto no es factible para consultas complicadas con alta cardinalidad GROUP BY o con una gran cantidad de datos temporales para JOIN. En tales casos, necesitamos “reshuffle” datos entre servidores, lo que requiere una coordinación adicional. ClickHouse no admite ese tipo de ejecución de consultas, y tenemos que trabajar en ello. -## Árbol De fusión {#merge-tree} +## Árbol de fusión {#merge-tree} `MergeTree` es una familia de motores de almacenamiento que admite la indexación por clave principal. La clave principal puede ser una tupla arbitraria de columnas o expresiones. Datos en un `MergeTree` se almacena en “parts”. Cada parte almacena los datos en el orden de la clave principal, por lo que la tupla de la clave principal ordena los datos lexicográficamente. Todas las columnas de la tabla se almacenan en `column.bin` archivos en estas partes. Los archivos consisten en bloques comprimidos. Cada bloque suele ser de 64 KB a 1 MB de datos sin comprimir, dependiendo del tamaño del valor promedio. Los bloques constan de valores de columna colocados contiguamente uno tras otro. Los valores de columna están en el mismo orden para cada columna (la clave principal define el orden), por lo que cuando itera por muchas columnas, obtiene valores para las filas correspondientes. @@ -177,7 +177,7 @@ Cuando vamos a leer algo de una parte en `MergeTree` miramos `primary.idx` datos Cuando `INSERT` un montón de datos en `MergeTree`, ese grupo está ordenado por orden de clave primaria y forma una nueva parte. Hay subprocesos de fondo que seleccionan periódicamente algunas partes y las fusionan en una sola parte ordenada para mantener el número de partes relativamente bajo. Es por eso que se llama `MergeTree`. Por supuesto, la fusión conduce a “write amplification”. Todas las partes son inmutables: solo se crean y eliminan, pero no se modifican. Cuando se ejecuta SELECT, contiene una instantánea de la tabla (un conjunto de partes). Después de la fusión, también mantenemos las piezas viejas durante algún tiempo para facilitar la recuperación después de la falla, por lo que si vemos que alguna parte fusionada probablemente esté rota, podemos reemplazarla con sus partes de origen. -`MergeTree` no es un árbol de LSM porque no contiene “memtable” y “log”: inserted data is written directly to the filesystem. This makes it suitable only to INSERT data in batches, not by individual row and not very frequently – about once per second is ok, but a thousand times a second is not. We did it this way for simplicity’s sake, and because we are already inserting data in batches in our applications. +`MergeTree` no es un árbol de LSM porque no contiene “memtable” y “log”: inserted data is written directly to the filesystem. This makes it suitable only to INSERT data in batches, not by individual row and not very frequently – about once per second is ok, but a thousand times a second is not. We did it this way for simplicity's sake, and because we are already inserting data in batches in our applications. > Las tablas MergeTree solo pueden tener un índice (primario): no hay índices secundarios. Sería bueno permitir múltiples representaciones físicas bajo una tabla lógica, por ejemplo, para almacenar datos en más de un orden físico o incluso para permitir representaciones con datos preagregados junto con datos originales. diff --git a/docs/es/development/browse-code.md b/docs/es/development/browse-code.md index dfa20e016c1..ca031ad03f3 100644 --- a/docs/es/development/browse-code.md +++ b/docs/es/development/browse-code.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 63 -toc_title: "Examinar el c\xF3digo fuente de ClickHouse" +toc_title: "Buscar c\xF3digo fuente" --- -# Examinar El código Fuente De ClickHouse {#browse-clickhouse-source-code} +# Examinar el código fuente de ClickHouse {#browse-clickhouse-source-code} -Usted puede utilizar **Woboq** navegador de código en línea disponible [aqui](https://clickhouse.tech/codebrowser/html_report///ClickHouse/src/index.html). Proporciona navegación de código y resaltado semántico, búsqueda e indexación. La instantánea de código se actualiza diariamente. +Usted puede utilizar **Woboq** navegador de código en línea disponible [aqui](https://clickhouse.tech/codebrowser/html_report/ClickHouse/src/index.html). Proporciona navegación de código y resaltado semántico, búsqueda e indexación. La instantánea de código se actualiza diariamente. Además, puede navegar por las fuentes en [GitHub](https://github.com/ClickHouse/ClickHouse) como de costumbre. diff --git a/docs/es/development/build-cross-arm.md b/docs/es/development/build-cross-arm.md index 7c9b6decd14..2758e9a0e94 100644 --- a/docs/es/development/build-cross-arm.md +++ b/docs/es/development/build-cross-arm.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 67 toc_title: "C\xF3mo construir ClickHouse en Linux para AARCH64 (ARM64)" --- -# Cómo Construir ClickHouse En Linux Para La Arquitectura AARCH64 (ARM64 {#how-to-build-clickhouse-on-linux-for-aarch64-arm64-architecture} +# Cómo construir ClickHouse en Linux para la arquitectura AARCH64 (ARM64) {#how-to-build-clickhouse-on-linux-for-aarch64-arm64-architecture} Esto es para el caso cuando tiene una máquina Linux y desea usarla para compilar `clickhouse` binario que se ejecutará en otra máquina Linux con arquitectura de CPU AARCH64. Esto está destinado a las comprobaciones de integración continua que se ejecutan en servidores Linux. @@ -22,7 +22,7 @@ sudo apt-get update sudo apt-get install clang-8 ``` -# Instalar Conjunto De Herramientas De compilación Cruzada {#install-cross-compilation-toolset} +# Instalar conjunto de herramientas de compilación cruzada {#install-cross-compilation-toolset} ``` bash cd ClickHouse diff --git a/docs/es/development/build-cross-osx.md b/docs/es/development/build-cross-osx.md index a0d85fea899..35cf01a8ce3 100644 --- a/docs/es/development/build-cross-osx.md +++ b/docs/es/development/build-cross-osx.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 66 toc_title: "C\xF3mo construir ClickHouse en Linux para Mac OS X" --- -# Cómo Construir ClickHouse En Linux Para Mac OS X {#how-to-build-clickhouse-on-linux-for-mac-os-x} +# Cómo construir ClickHouse en Linux para Mac OS X {#how-to-build-clickhouse-on-linux-for-mac-os-x} Esto es para el caso cuando tiene una máquina Linux y desea usarla para compilar `clickhouse` Esto está destinado a las comprobaciones de integración continuas que se ejecutan en servidores Linux. Si desea crear ClickHouse directamente en Mac OS X, continúe con [otra instrucción](build-osx.md). @@ -21,7 +21,7 @@ sudo echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic-8 sudo apt-get install clang-8 ``` -# Instalar Conjunto De Herramientas De compilación Cruzada {#install-cross-compilation-toolset} +# Instalar conjunto de herramientas de compilación cruzada {#install-cross-compilation-toolset} Recordemos la ruta donde instalamos `cctools` como ${CCTOOLS} diff --git a/docs/es/development/build-osx.md b/docs/es/development/build-osx.md index 7c3d50593ac..39eba389798 100644 --- a/docs/es/development/build-osx.md +++ b/docs/es/development/build-osx.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 65 toc_title: "C\xF3mo crear ClickHouse en Mac OS X" --- -# Cómo Crear ClickHouse En Mac OS X {#how-to-build-clickhouse-on-mac-os-x} +# Cómo crear ClickHouse en Mac OS X {#how-to-build-clickhouse-on-mac-os-x} Build debería funcionar en Mac OS X 10.15 (Catalina) @@ -15,13 +15,13 @@ Build debería funcionar en Mac OS X 10.15 (Catalina) $ /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" ``` -## Instalar Compiladores, Herramientas y Bibliotecas Necesarios {#install-required-compilers-tools-and-libraries} +## Instalar compiladores, herramientas y bibliotecas necesarios {#install-required-compilers-tools-and-libraries} ``` bash $ brew install cmake ninja libtool gettext ``` -## Fuentes De ClickHouse De Pago {#checkout-clickhouse-sources} +## Fuentes de ClickHouse de pago {#checkout-clickhouse-sources} ``` bash $ git clone --recursive git@github.com:ClickHouse/ClickHouse.git diff --git a/docs/es/development/build.md b/docs/es/development/build.md index 24d20581e42..861ceb53e51 100644 --- a/docs/es/development/build.md +++ b/docs/es/development/build.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 64 toc_title: "C\xF3mo crear ClickHouse en Linux" --- -# Cómo Construir ClickHouse Para El Desarrollo {#how-to-build-clickhouse-for-development} +# Cómo construir ClickHouse para el desarrollo {#how-to-build-clickhouse-for-development} El siguiente tutorial se basa en el sistema Ubuntu Linux. Con los cambios apropiados, también debería funcionar en cualquier otra distribución de Linux. @@ -23,7 +23,7 @@ O cmake3 en lugar de cmake en sistemas más antiguos. Hay varias formas de hacer esto. -### Instalar Desde Un Paquete PPA {#install-from-a-ppa-package} +### Instalar desde un paquete PPA {#install-from-a-ppa-package} ``` bash $ sudo apt-get install software-properties-common @@ -32,18 +32,18 @@ $ sudo apt-get update $ sudo apt-get install gcc-9 g++-9 ``` -### Instalar Desde Fuentes {#install-from-sources} +### Instalar desde fuentes {#install-from-sources} Mira [Sistema abierto.](https://github.com/ClickHouse/ClickHouse/blob/master/utils/ci/build-gcc-from-sources.sh) -## Usar GCC 9 Para Compilaciones {#use-gcc-9-for-builds} +## Usar GCC 9 para compilaciones {#use-gcc-9-for-builds} ``` bash $ export CC=gcc-9 $ export CXX=g++-9 ``` -## Fuentes De ClickHouse De Pago {#checkout-clickhouse-sources} +## Fuentes de ClickHouse de pago {#checkout-clickhouse-sources} ``` bash $ git clone --recursive git@github.com:ClickHouse/ClickHouse.git @@ -69,7 +69,7 @@ $ cd .. Para crear un ejecutable, ejecute `ninja clickhouse`. Esto creará el `programs/clickhouse` ejecutable, que se puede usar con `client` o `server` argumento. -# Cómo Construir ClickHouse En Cualquier Linux {#how-to-build-clickhouse-on-any-linux} +# Cómo construir ClickHouse en cualquier Linux {#how-to-build-clickhouse-on-any-linux} La compilación requiere los siguientes componentes: @@ -108,7 +108,7 @@ Ejemplo de Fedora Rawhide: cmake ../ClickHouse make -j $(nproc) -# No Tienes Que Construir ClickHouse {#you-dont-have-to-build-clickhouse} +# No tienes que construir ClickHouse {#you-dont-have-to-build-clickhouse} ClickHouse está disponible en binarios y paquetes preconstruidos. Los binarios son portátiles y se pueden ejecutar en cualquier tipo de Linux. @@ -116,7 +116,7 @@ Están diseñados para lanzamientos estables, preestablecidos y de prueba, siemp Para encontrar la construcción más fresca de `master`, ir a [se compromete página](https://github.com/ClickHouse/ClickHouse/commits/master), haga clic en la primera marca de verificación verde o cruz roja cerca de confirmar, y haga clic en “Details” enlace justo después “ClickHouse Build Check”. -# Cómo Construir El Paquete Debian ClickHouse {#how-to-build-clickhouse-debian-package} +# Cómo construir el paquete Debian ClickHouse {#how-to-build-clickhouse-debian-package} ## Instalar Git y Pbuilder {#install-git-and-pbuilder} @@ -125,14 +125,14 @@ $ sudo apt-get update $ sudo apt-get install git python pbuilder debhelper lsb-release fakeroot sudo debian-archive-keyring debian-keyring ``` -## Fuentes De ClickHouse De Pago {#checkout-clickhouse-sources-1} +## Fuentes de ClickHouse de pago {#checkout-clickhouse-sources-1} ``` bash $ git clone --recursive --branch master https://github.com/ClickHouse/ClickHouse.git $ cd ClickHouse ``` -## Ejecutar Secuencia De Comandos De Lanzamiento {#run-release-script} +## Ejecutar secuencia de comandos de lanzamiento {#run-release-script} ``` bash $ ./release diff --git a/docs/es/development/contrib.md b/docs/es/development/contrib.md index 6b8b1f6aaf0..a510c9e80a5 100644 --- a/docs/es/development/contrib.md +++ b/docs/es/development/contrib.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 70 toc_title: Bibliotecas de terceros utilizadas --- -# Bibliotecas De Terceros Utilizadas {#third-party-libraries-used} +# Bibliotecas de terceros utilizadas {#third-party-libraries-used} | Biblioteca | Licencia | |--------------------|--------------------------------------------------------------------------------------------------------------------------------------------------| diff --git a/docs/es/development/developer-instruction.md b/docs/es/development/developer-instruction.md index ccdeeb0b95b..9c56abe33a1 100644 --- a/docs/es/development/developer-instruction.md +++ b/docs/es/development/developer-instruction.md @@ -1,21 +1,21 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 61 toc_title: "La instrucci\xF3n para desarrolladores de ClickHouse para principiantes" --- La construcción de ClickHouse es compatible con Linux, FreeBSD y Mac OS X. -# Si Utiliza Windows {#if-you-use-windows} +# Si utiliza Windows {#if-you-use-windows} Si usa Windows, necesita crear una máquina virtual con Ubuntu. Para comenzar a trabajar con una máquina virtual, instale VirtualBox. Puede descargar Ubuntu desde el sitio web: https://www.ubuntu.com/\#download. Por favor, cree una máquina virtual a partir de la imagen descargada (debe reservar al menos 4 GB de RAM para ello). Para ejecutar un terminal de línea de comandos en Ubuntu, busque un programa que contenga la palabra “terminal” en su nombre (gnome-terminal, konsole etc.) o simplemente presione Ctrl + Alt + T. -# Si Utiliza Un Sistema De 32 Bits {#if-you-use-a-32-bit-system} +# Si utiliza un sistema de 32 bits {#if-you-use-a-32-bit-system} ClickHouse no puede funcionar ni construir en un sistema de 32 bits. Debe adquirir acceso a un sistema de 64 bits y puede continuar leyendo. -# Crear Un Repositorio En GitHub {#creating-a-repository-on-github} +# Creación de un repositorio en GitHub {#creating-a-repository-on-github} Para comenzar a trabajar con el repositorio de ClickHouse, necesitará una cuenta de GitHub. @@ -35,13 +35,13 @@ Para hacer eso en Ubuntu, ejecutaría en la terminal de línea de comandos: Puede encontrar un breve manual sobre el uso de Git aquí: https://services.github.com/on-demand/downloads/github-git-cheat-sheet.pdf . Para obtener un manual detallado sobre Git, consulte https://git-scm.com/book/en/v2 . -# Clonación De Un Repositorio En Su máquina De Desarrollo {#cloning-a-repository-to-your-development-machine} +# Clonación de un repositorio en su máquina de desarrollo {#cloning-a-repository-to-your-development-machine} A continuación, debe descargar los archivos fuente en su máquina de trabajo. Esto se llama “to clone a repository” porque crea una copia local del repositorio en su máquina de trabajo. En el terminal de línea de comandos, ejecute: - git clone --recursive git@guthub.com:your_github_username/ClickHouse.git + git clone --recursive git@github.com:your_github_username/ClickHouse.git cd ClickHouse Nota: por favor, sustituye *your\_github\_username* con lo que es apropiado! @@ -79,7 +79,7 @@ También puede agregar la dirección original del repositorio de ClickHouse a su Después de ejecutar con éxito este comando, podrá extraer actualizaciones del repositorio principal de ClickHouse ejecutando `git pull upstream master`. -## Trabajar Con submódulos {#working-with-submodules} +## Trabajar con submódulos {#working-with-submodules} Trabajar con submódulos en git podría ser doloroso. Los siguientes comandos ayudarán a administrarlo: @@ -109,7 +109,7 @@ Los siguientes comandos le ayudarían a restablecer todos los submódulos al est git submodule foreach git submodule foreach git reset --hard git submodule foreach git submodule foreach git clean -xfd -# Sistema De construcción {#build-system} +# Sistema de construcción {#build-system} ClickHouse utiliza CMake y Ninja para la construcción. @@ -129,11 +129,11 @@ Para instalar CMake y Ninja en Mac OS X, primero instale Homebrew y luego instal A continuación, verifique la versión de CMake: `cmake --version`. Si está por debajo de 3.3, debe instalar una versión más reciente desde el sitio web: https://cmake.org/download/. -# Bibliotecas Externas Opcionales {#optional-external-libraries} +# Bibliotecas externas opcionales {#optional-external-libraries} ClickHouse utiliza varias bibliotecas externas para la construcción. Todos ellos no necesitan ser instalados por separado, ya que se construyen junto con ClickHouse a partir de las fuentes ubicadas en los submódulos. Puede consultar la lista en `contrib`. -# Compilador De C ++ {#c-compiler} +# Compilador de C ++ {#c-compiler} Los compiladores GCC a partir de la versión 9 y Clang versión 8 o superior son compatibles para construir ClickHouse. @@ -147,7 +147,7 @@ La compilación de Mac OS X solo es compatible con Clang. Sólo tiene que ejecut Si decide utilizar Clang, también puede instalar `libc++` y `lld` si usted sabe lo que es. Utilizar `ccache` también se recomienda. -# El Proceso De construcción {#the-building-process} +# El proceso de construcción {#the-building-process} Ahora que está listo para construir ClickHouse, le recomendamos que cree un directorio separado `build` dentro `ClickHouse` que contendrá todos los de la generación de artefactos: @@ -204,11 +204,11 @@ Tras la compilación exitosa, obtienes un archivo ejecutable `ClickHouse/ diff --git a/docs/es/development/style.md b/docs/es/development/style.md index 384c7189934..7bf3dfc02d6 100644 --- a/docs/es/development/style.md +++ b/docs/es/development/style.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 68 toc_title: "C\xF3mo escribir c\xF3digo C ++" --- -# Cómo Escribir código C ++ {#how-to-write-c-code} +# Cómo escribir código C ++ {#how-to-write-c-code} -## Recomendaciones Generales {#general-recommendations} +## Recomendaciones generales {#general-recommendations} **1.** Las siguientes son recomendaciones, no requisitos. @@ -422,7 +422,7 @@ También puede usar una abreviatura si el nombre completo se incluye junto a él **17.** Los nombres de archivo con código fuente de C++ deben tener `.cpp` ampliación. Los archivos de encabezado deben tener `.h` ampliación. -## Cómo Escribir código {#how-to-write-code} +## Cómo escribir código {#how-to-write-code} **1.** Gestión de la memoria. @@ -689,7 +689,7 @@ auto s = std::string{"Hello"}; **26.** Para funciones virtuales, escriba `virtual` en la clase base, pero escribe `override` en lugar de `virtual` en las clases descendientes. -## Características No Utilizadas De C ++ {#unused-features-of-c} +## Características no utilizadas de C ++ {#unused-features-of-c} **1.** La herencia virtual no se utiliza. @@ -763,7 +763,7 @@ Si ya hay una buena solución disponible, úsela, incluso si eso significa que d **5.** Siempre se da preferencia a las bibliotecas que ya están en uso. -## Recomendaciones Generales {#general-recommendations-1} +## Recomendaciones generales {#general-recommendations-1} **1.** Escribe el menor código posible. @@ -777,7 +777,7 @@ Si ya hay una buena solución disponible, úsela, incluso si eso significa que d **6.** Se fomenta la simplificación del código. Reduzca el tamaño de su código siempre que sea posible. -## Recomendaciones Adicionales {#additional-recommendations} +## Recomendaciones adicionales {#additional-recommendations} **1.** Especificar explícitamente `std::` para tipos de `stddef.h` diff --git a/docs/es/development/tests.md b/docs/es/development/tests.md index b889c7be6af..0d2931c97fa 100644 --- a/docs/es/development/tests.md +++ b/docs/es/development/tests.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 69 toc_title: "C\xF3mo ejecutar pruebas de ClickHouse" --- -# Pruebas De ClickHouse {#clickhouse-testing} +# Pruebas de ClickHouse {#clickhouse-testing} -## Pruebas Funcionales {#functional-tests} +## Pruebas funcionales {#functional-tests} Las pruebas funcionales son las más simples y cómodas de usar. La mayoría de las características de ClickHouse se pueden probar con pruebas funcionales y son obligatorias para cada cambio en el código de ClickHouse que se puede probar de esa manera. @@ -38,7 +38,7 @@ deshabilitar estos grupos de pruebas utilizando `--no-zookeeper`, `--no-shard` y Si conocemos algunos errores que se pueden reproducir fácilmente mediante pruebas funcionales, colocamos pruebas funcionales preparadas en `tests/queries/bugs` directorio. Estas pruebas se moverán a `tests/queries/0_stateless` cuando se corrigen errores. -## Pruebas De integración {#integration-tests} +## Pruebas de integración {#integration-tests} Las pruebas de integración permiten probar ClickHouse en la configuración agrupada y la interacción de ClickHouse con otros servidores como MySQL, Postgres, MongoDB. Son útiles para emular divisiones de red, caídas de paquetes, etc. Estas pruebas se ejecutan bajo Docker y crean múltiples contenedores con varios software. @@ -46,27 +46,27 @@ Ver `tests/integration/README.md` sobre cómo ejecutar estas pruebas. Tenga en cuenta que la integración de ClickHouse con controladores de terceros no se ha probado. Además, actualmente no tenemos pruebas de integración con nuestros controladores JDBC y ODBC. -## Pruebas Unitarias {#unit-tests} +## Pruebas unitarias {#unit-tests} Las pruebas unitarias son útiles cuando desea probar no ClickHouse como un todo, sino una sola biblioteca o clase aislada. Puede habilitar o deshabilitar la compilación de pruebas con `ENABLE_TESTS` Opción CMake. Las pruebas unitarias (y otros programas de prueba) se encuentran en `tests` subdirectorios en todo el código. Para ejecutar pruebas unitarias, escriba `ninja test`. Algunas pruebas usan `gtest`, pero algunos son solo programas que devuelven un código de salida distinto de cero en caso de fallo de prueba. No es necesariamente tener pruebas unitarias si el código ya está cubierto por pruebas funcionales (y las pruebas funcionales suelen ser mucho más simples de usar). -## Pruebas De Rendimiento {#performance-tests} +## Pruebas de rendimiento {#performance-tests} Las pruebas de rendimiento permiten medir y comparar el rendimiento de alguna parte aislada de ClickHouse en consultas sintéticas. Las pruebas se encuentran en `tests/performance`. Cada prueba está representada por `.xml` archivo con la descripción del caso de prueba. Las pruebas se ejecutan con `clickhouse performance-test` herramienta (que está incrustada en `clickhouse` binario). Ver `--help` para la invocación. -Cada prueba ejecuta una o múltiples consultas (posiblemente con combinaciones de parámetros) en un bucle con algunas condiciones para stop (como “maximum execution speed is not changing in three seconds”) y medir algunas métricas sobre el rendimiento de las consultas (como “maximum execution speed”). Algunas pruebas pueden contener condiciones previas en el conjunto de datos de pruebas precargado. +Cada prueba ejecuta una o varias consultas (posiblemente con combinaciones de parámetros) en un bucle con algunas condiciones para detener (como “maximum execution speed is not changing in three seconds”) y medir algunas métricas sobre el rendimiento de las consultas (como “maximum execution speed”). Algunas pruebas pueden contener condiciones previas en el conjunto de datos de pruebas precargado. Si desea mejorar el rendimiento de ClickHouse en algún escenario, y si se pueden observar mejoras en consultas simples, se recomienda encarecidamente escribir una prueba de rendimiento. Siempre tiene sentido usar `perf top` u otras herramientas de perf durante sus pruebas. -## Herramientas De Prueba y Secuencias De Comandos {#test-tools-and-scripts} +## Herramientas de prueba y secuencias de comandos {#test-tools-and-scripts} -Algunos programas en `tests` directorio no son pruebas preparadas, pero son herramientas de prueba. Por ejemplo, para `Lexer` hay una herramienta `dbms/Parsers/tests/lexer` que solo hacen la tokenización de stdin y escriben el resultado coloreado en stdout. Puede usar este tipo de herramientas como ejemplos de código y para exploración y pruebas manuales. +Algunos programas en `tests` directorio no son pruebas preparadas, pero son herramientas de prueba. Por ejemplo, para `Lexer` hay una herramienta `src/Parsers/tests/lexer` que solo hacen la tokenización de stdin y escriben el resultado coloreado en stdout. Puede usar este tipo de herramientas como ejemplos de código y para exploración y pruebas manuales. También puede colocar un par de archivos `.sh` y `.reference` junto con la herramienta para ejecutarlo en alguna entrada predefinida, entonces el resultado del script se puede comparar con `.reference` file. Este tipo de pruebas no están automatizadas. -## Miscellanous Pruebas {#miscellanous-tests} +## Pruebas diversas {#miscellaneous-tests} Hay pruebas para diccionarios externos ubicados en `tests/external_dictionaries` y para modelos aprendidos a máquina en `tests/external_models`. Estas pruebas no se actualizan y deben transferirse a pruebas de integración. @@ -74,7 +74,7 @@ Hay una prueba separada para inserciones de quórum. Esta prueba ejecuta el clú La prueba de quórum fue escrita por un equipo separado antes de que ClickHouse fuera de código abierto. Este equipo ya no trabaja con ClickHouse. La prueba fue escrita accidentalmente en Java. Por estas razones, la prueba de quórum debe reescribirse y trasladarse a pruebas de integración. -## Pruebas Manuales {#manual-testing} +## Pruebas manuales {#manual-testing} Cuando desarrolla una nueva característica, es razonable probarla también manualmente. Puede hacerlo con los siguientes pasos: @@ -109,7 +109,7 @@ Si el servidor de clickhouse del sistema ya se está ejecutando y no desea deten `clickhouse` binary casi no tiene dependencias y funciona en una amplia gama de distribuciones de Linux. Para probar rápidamente y sucio sus cambios en un servidor, simplemente puede `scp` su fresco construido `clickhouse` binario a su servidor y luego ejecútelo como en los ejemplos anteriores. -## Entorno De Prueba {#testing-environment} +## Entorno de prueba {#testing-environment} Antes de publicar la versión como estable, la implementamos en el entorno de prueba. El entorno de prueba es un clúster que procesa 1/39 parte de [El Yandex.Métrica](https://metrica.yandex.com/) datos. Compartimos nuestro entorno de pruebas con Yandex.Equipo de Metrica. ClickHouse se actualiza sin tiempo de inactividad sobre los datos existentes. Nos fijamos en un primer momento que los datos se procesan con éxito sin retraso de tiempo real, la replicación continúan trabajando y no hay problemas visibles para Yandex.Equipo de Metrica. La primera comprobación se puede hacer de la siguiente manera: @@ -119,7 +119,7 @@ SELECT hostName() AS h, any(version()), any(uptime()), max(UTCEventTime), count( En algunos casos también implementamos en el entorno de prueba de nuestros equipos de amigos en Yandex: Market, Cloud, etc. También tenemos algunos servidores de hardware que se utilizan con fines de desarrollo. -## Pruebas De Carga {#load-testing} +## Pruebas de carga {#load-testing} Después de implementar en el entorno de prueba, ejecutamos pruebas de carga con consultas del clúster de producción. Esto se hace manualmente. @@ -147,7 +147,7 @@ Usted debe comprobar que `clickhouse-server` no se bloquea, la huella de memoria Los tiempos de ejecución de consultas precisos no se registran y no se comparan debido a la alta variabilidad de las consultas y el entorno. -## Pruebas De construcción {#build-tests} +## Pruebas de construcción {#build-tests} Las pruebas de compilación permiten verificar que la compilación no esté rota en varias configuraciones alternativas y en algunos sistemas extranjeros. Las pruebas se encuentran en `ci` directorio. Ejecutan compilación desde la fuente dentro de Docker, Vagrant y, a veces, con `qemu-user-static` dentro de Docker. Estas pruebas están en desarrollo y las ejecuciones de pruebas no están automatizadas. @@ -165,11 +165,11 @@ Por ejemplo, construir con paquetes del sistema es una mala práctica, porque no Aunque no podemos ejecutar todas las pruebas en todas las variantes de compilaciones, queremos verificar al menos que varias variantes de compilación no estén rotas. Para este propósito utilizamos pruebas de construcción. -## Pruebas De Compatibilidad De Protocolos {#testing-for-protocol-compatibility} +## Pruebas de Compatibilidad de protocolos {#testing-for-protocol-compatibility} Cuando ampliamos el protocolo de red ClickHouse, probamos manualmente que el antiguo clickhouse-client funciona con el nuevo clickhouse-server y el nuevo clickhouse-client funciona con el antiguo clickhouse-server (simplemente ejecutando binarios de los paquetes correspondientes). -## Ayuda Del Compilador {#help-from-the-compiler} +## Ayuda del compilador {#help-from-the-compiler} Código principal de ClickHouse (que se encuentra en `dbms` directorio) se construye con `-Wall -Wextra -Werror` y con algunas advertencias habilitadas adicionales. Aunque estas opciones no están habilitadas para bibliotecas de terceros. @@ -199,13 +199,23 @@ Versión de depuración de `jemalloc` se utiliza para la compilación de depurac ## Fuzzing {#fuzzing} -Usamos una prueba de fuzz simple para generar consultas SQL aleatorias y verificar que el servidor no muera. Las pruebas de pelusa se realizan con el desinfectante Address. Lo puedes encontrar en `00746_sql_fuzzy.pl`. Esta prueba debe ejecutarse de forma continua (de la noche a la mañana y más). +ClickHouse fuzzing se implementa tanto usando [LibFuzzer](https://llvm.org/docs/LibFuzzer.html) y consultas SQL aleatorias. +Todas las pruebas de fuzz deben realizarse con desinfectantes (Dirección y Undefined). -A partir de diciembre de 2018, todavía no usamos pruebas de fuzz aisladas del código de la biblioteca. +LibFuzzer se usa para pruebas de fuzz aisladas del código de la biblioteca. Fuzzers se implementan como parte del código de prueba y tienen “\_fuzzer” nombre postfixes. +El ejemplo de Fuzzer se puede encontrar en `src/Parsers/tests/lexer_fuzzer.cpp`. Las configuraciones, diccionarios y corpus específicos de LibFuzzer se almacenan en `tests/fuzz`. +Le recomendamos que escriba pruebas fuzz para cada funcionalidad que maneje la entrada del usuario. -## Auditoría De Seguridad {#security-audit} +Fuzzers no se construyen de forma predeterminada. Para construir fuzzers ambos `-DENABLE_FUZZING=1` y `-DENABLE_TESTS=1` se deben establecer opciones. +Recomendamos deshabilitar Jemalloc mientras se construyen fuzzers. Configuración utilizada para integrar +Google OSS-Fuzz se puede encontrar en `docker/fuzz`. -La gente del departamento de Yandex Cloud hace una visión general básica de las capacidades de ClickHouse desde el punto de vista de la seguridad. +También usamos una prueba de fuzz simple para generar consultas SQL aleatorias y verificar que el servidor no muera al ejecutarlas. +Lo puedes encontrar en `00746_sql_fuzzy.pl`. Esta prueba debe ejecutarse de forma continua (de la noche a la mañana y más). + +## Auditoría de seguridad {#security-audit} + +La gente de Yandex Security Team hace una visión general básica de las capacidades de ClickHouse desde el punto de vista de la seguridad. ## Analizadores estáticos {#static-analyzers} @@ -217,7 +227,7 @@ Si usted usa `CLion` como IDE, puede aprovechar algunos `clang-tidy` comprueba f `FORTIFY_SOURCE` se utiliza de forma predeterminada. Es casi inútil, pero todavía tiene sentido en casos raros y no lo desactivamos. -## Estilo De código {#code-style} +## Estilo de código {#code-style} Se describen las reglas de estilo de código [aqui](https://clickhouse.tech/docs/en/development/style/). @@ -235,11 +245,11 @@ Cada lanzamiento de ClickHouse se prueba con los motores Yandex Metrica y AppMet Estas pruebas son automatizadas por un equipo separado. Debido a la gran cantidad de piezas móviles, las pruebas fallan la mayor parte del tiempo por razones completamente no relacionadas, que son muy difíciles de descubrir. Lo más probable es que estas pruebas tengan un valor negativo para nosotros. Sin embargo, se demostró que estas pruebas son útiles en aproximadamente una o dos veces de cada cientos. -## Cobertura De Prueba {#test-coverage} +## Cobertura de prueba {#test-coverage} A partir de julio de 2018, no realizamos un seguimiento de la cobertura de las pruebas. -## Automatización De Pruebas {#test-automation} +## Automatización de pruebas {#test-automation} Realizamos pruebas con el CI interno de Yandex y el sistema de automatización de trabajos llamado “Sandbox”. @@ -249,4 +259,3 @@ No usamos Travis CI debido al límite de tiempo y potencia computacional. No usamos Jenkins. Se usó antes y ahora estamos felices de no estar usando Jenkins. [Artículo Original](https://clickhouse.tech/docs/en/development/tests/) -pruebas/) diff --git a/docs/es/engines/database-engines/index.md b/docs/es/engines/database-engines/index.md index c524b2a2094..8784b9bd02b 100644 --- a/docs/es/engines/database-engines/index.md +++ b/docs/es/engines/database-engines/index.md @@ -1,12 +1,12 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa -toc_folder_title: Database Engines +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: Motores de base de datos toc_priority: 27 toc_title: "Implantaci\xF3n" --- -# Motores De Base De Datos {#database-engines} +# Motores de base de datos {#database-engines} Los motores de bases de datos le permiten trabajar con tablas. diff --git a/docs/es/engines/database-engines/lazy.md b/docs/es/engines/database-engines/lazy.md index 3622f45a00c..0988c4cb395 100644 --- a/docs/es/engines/database-engines/lazy.md +++ b/docs/es/engines/database-engines/lazy.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 31 toc_title: Perezoso --- @@ -11,7 +11,7 @@ Mantiene las tablas en RAM solamente `expiration_time_in_seconds` segundos despu Está optimizado para almacenar muchas tablas pequeñas \* Log, para las cuales hay un largo intervalo de tiempo entre los accesos. -## Creación De Una Base De Datos {#creating-a-database} +## Creación de una base de datos {#creating-a-database} CREATE DATABASE testlazy ENGINE = Lazy(expiration_time_in_seconds); diff --git a/docs/es/engines/database-engines/mysql.md b/docs/es/engines/database-engines/mysql.md index dd6855ba3f5..5f1dec97f35 100644 --- a/docs/es/engines/database-engines/mysql.md +++ b/docs/es/engines/database-engines/mysql.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 30 toc_title: MySQL --- -# Mysql {#mysql} +# MySQL {#mysql} Permite conectarse a bases de datos en un servidor MySQL remoto y realizar `INSERT` y `SELECT` consultas para intercambiar datos entre ClickHouse y MySQL. @@ -17,11 +17,11 @@ No puede realizar las siguientes consultas: - `CREATE TABLE` - `ALTER` -## Creación De Una Base De Datos {#creating-a-database} +## Creación de una base de datos {#creating-a-database} ``` sql CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster] -ENGINE = MySQL('host:port', 'database', 'user', 'password') +ENGINE = MySQL('host:port', ['database' | database], 'user', 'password') ``` **Parámetros del motor** @@ -31,7 +31,7 @@ ENGINE = MySQL('host:port', 'database', 'user', 'password') - `user` — MySQL user. - `password` — User password. -## Soporte De Tipos De Datos {#data_types-support} +## Soporte de tipos de datos {#data_types-support} | MySQL | Haga clic en Casa | |----------------------------------|--------------------------------------------------------------| @@ -53,7 +53,7 @@ Todos los demás tipos de datos MySQL se convierten en [Cadena](../../sql-refere [NULL](../../sql-reference/data-types/nullable.md) se admite. -## Ejemplos De Uso {#examples-of-use} +## Ejemplos de uso {#examples-of-use} Tabla en MySQL: diff --git a/docs/es/engines/index.md b/docs/es/engines/index.md index c2a7f33b49f..03e4426dd8d 100644 --- a/docs/es/engines/index.md +++ b/docs/es/engines/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa -toc_folder_title: Engines +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: Motor toc_priority: 25 --- diff --git a/docs/es/engines/table-engines/index.md b/docs/es/engines/table-engines/index.md index 972bd180d0c..cbc7cc5aaa8 100644 --- a/docs/es/engines/table-engines/index.md +++ b/docs/es/engines/table-engines/index.md @@ -1,12 +1,12 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa -toc_folder_title: Table Engines +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: Motores de mesa toc_priority: 26 toc_title: "Implantaci\xF3n" --- -# Motores De Mesa {#table_engines} +# Motores de mesa {#table_engines} El motor de tabla (tipo de tabla) determina: @@ -17,11 +17,11 @@ El motor de tabla (tipo de tabla) determina: - Si es posible la ejecución de solicitudes multiproceso. - Parámetros de replicación de datos. -## Familias De Motores {#engine-families} +## Familias de motores {#engine-families} -### Mergetree {#mergetree} +### Método de codificación de datos: {#mergetree} -Los motores de mesa más universales y funcionales para tareas de alta carga. La propiedad compartida por estos motores es la inserción rápida de datos con el posterior procesamiento de datos en segundo plano. `MergeTree` Los motores familiares admiten la replicación de datos (con [Replicado\*](mergetree-family/replication.md#replication) versiones de motores), particionamiento y otras características no admitidas en otros motores. +Los motores de mesa más universales y funcionales para tareas de alta carga. La propiedad compartida por estos motores es la inserción rápida de datos con el posterior procesamiento de datos en segundo plano. `MergeTree` Los motores familiares admiten la replicación de datos (con [Replicado\*](mergetree-family/replication.md#table_engines-replication) versiones de motores), particionamiento y otras características no admitidas en otros motores. Motores en la familia: @@ -43,7 +43,7 @@ Motores en la familia: - [StripeLog](log-family/stripelog.md#stripelog) - [Registro](log-family/log.md#log) -### Motores De integración {#integration-engines} +### Motores de integración {#integration-engines} Motores para comunicarse con otros sistemas de almacenamiento y procesamiento de datos. @@ -55,14 +55,14 @@ Motores en la familia: - [JDBC](integrations/jdbc.md#table-engine-jdbc) - [HDFS](integrations/hdfs.md#hdfs) -### Motores Especiales {#special-engines} +### Motores especiales {#special-engines} Motores en la familia: - [Distribuido](special/distributed.md#distributed) - [Método de codificación de datos:](special/materializedview.md#materializedview) - [Diccionario](special/dictionary.md#dictionary) -- [Fusionar](special/merge.md#merge +- \[Fusión\](special/merge.md\#merge - [File](special/file.md#file) - [Nulo](special/null.md#null) - [Establecer](special/set.md#set) diff --git a/docs/es/engines/table-engines/integrations/hdfs.md b/docs/es/engines/table-engines/integrations/hdfs.md index 4ccf75e3f89..500d6e91d90 100644 --- a/docs/es/engines/table-engines/integrations/hdfs.md +++ b/docs/es/engines/table-engines/integrations/hdfs.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 36 toc_title: HDFS --- @@ -50,7 +50,7 @@ SELECT * FROM hdfs_engine_table LIMIT 2 └──────┴───────┘ ``` -## Detalles De implementación {#implementation-details} +## Detalles de implementación {#implementation-details} - Las lecturas y escrituras pueden ser paralelas - No soportado: diff --git a/docs/es/engines/table-engines/integrations/index.md b/docs/es/engines/table-engines/integrations/index.md index 4145d8b3d7e..e57aaf88744 100644 --- a/docs/es/engines/table-engines/integrations/index.md +++ b/docs/es/engines/table-engines/integrations/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa -toc_folder_title: Integrations +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "Integraci\xF3n" toc_priority: 30 --- diff --git a/docs/es/engines/table-engines/integrations/jdbc.md b/docs/es/engines/table-engines/integrations/jdbc.md index dd514468bc5..fd3450cef7c 100644 --- a/docs/es/engines/table-engines/integrations/jdbc.md +++ b/docs/es/engines/table-engines/integrations/jdbc.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 34 toc_title: JDBC --- @@ -13,7 +13,7 @@ Para implementar la conexión JDBC, ClickHouse utiliza el programa independiente Este motor soporta el [NULL](../../../sql-reference/data-types/nullable.md) tipo de datos. -## Creación De Una Tabla {#creating-a-table} +## Creación de una tabla {#creating-a-table} ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name @@ -34,7 +34,7 @@ ENGINE = JDBC(dbms_uri, external_database, external_table) - `external_table` — Name of the table in `external_database`. -## Ejemplo De Uso {#usage-example} +## Ejemplo de uso {#usage-example} Crear una tabla en el servidor MySQL conectándose directamente con su cliente de consola: diff --git a/docs/es/engines/table-engines/integrations/kafka.md b/docs/es/engines/table-engines/integrations/kafka.md index 3d7b82a7838..508ac72b8e7 100644 --- a/docs/es/engines/table-engines/integrations/kafka.md +++ b/docs/es/engines/table-engines/integrations/kafka.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 32 toc_title: Kafka --- @@ -15,7 +15,7 @@ Kafka te permite: - Organice el almacenamiento tolerante a fallos. - Secuencias de proceso a medida que estén disponibles. -## Creación De Una Tabla {#table_engine-kafka-creating-a-table} +## Creación de una tabla {#table_engine-kafka-creating-a-table} ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] @@ -32,22 +32,26 @@ SETTINGS [kafka_row_delimiter = 'delimiter_symbol',] [kafka_schema = '',] [kafka_num_consumers = N,] - [kafka_skip_broken_messages = N] + [kafka_max_block_size = 0,] + [kafka_skip_broken_messages = N,] + [kafka_commit_every_batch = 0] ``` Parámetros requeridos: - `kafka_broker_list` – A comma-separated list of brokers (for example, `localhost:9092`). - `kafka_topic_list` – A list of Kafka topics. -- `kafka_group_name` – A group of Kafka consumers. Reading margins are tracked for each group separately. If you don’t want messages to be duplicated in the cluster, use the same group name everywhere. +- `kafka_group_name` – A group of Kafka consumers. Reading margins are tracked for each group separately. If you don't want messages to be duplicated in the cluster, use the same group name everywhere. - `kafka_format` – Message format. Uses the same notation as the SQL `FORMAT` función, tal como `JSONEachRow`. Para obtener más información, consulte [Formato](../../../interfaces/formats.md) apartado. Parámetros opcionales: - `kafka_row_delimiter` – Delimiter character, which ends the message. -- `kafka_schema` – Parameter that must be used if the format requires a schema definition. For example, [Cap’n Proto](https://capnproto.org/) requiere la ruta de acceso al archivo de esquema y el nombre de la raíz `schema.capnp:Message` objeto. +- `kafka_schema` – Parameter that must be used if the format requires a schema definition. For example, [Cap'n Proto](https://capnproto.org/) requiere la ruta de acceso al archivo de esquema y el nombre de la raíz `schema.capnp:Message` objeto. - `kafka_num_consumers` – The number of consumers per table. Default: `1`. Especifique más consumidores si el rendimiento de un consumidor es insuficiente. El número total de consumidores no debe exceder el número de particiones en el tema, ya que solo se puede asignar un consumidor por partición. +- `kafka_max_block_size` - El tamaño máximo de lote (en mensajes) para la encuesta (predeterminado: `max_block_size`). - `kafka_skip_broken_messages` – Kafka message parser tolerance to schema-incompatible messages per block. Default: `0`. Si `kafka_skip_broken_messages = N` entonces el motor salta *N* Mensajes de Kafka que no se pueden analizar (un mensaje es igual a una fila de datos). +- `kafka_commit_every_batch` - Confirmar cada lote consumido y manejado en lugar de una única confirmación después de escribir un bloque completo (predeterminado: `0`). Ejemplos: diff --git a/docs/es/engines/table-engines/integrations/mysql.md b/docs/es/engines/table-engines/integrations/mysql.md index 64b15ff1898..52799117255 100644 --- a/docs/es/engines/table-engines/integrations/mysql.md +++ b/docs/es/engines/table-engines/integrations/mysql.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 33 toc_title: MySQL --- @@ -9,7 +9,7 @@ toc_title: MySQL El motor MySQL le permite realizar `SELECT` consultas sobre datos almacenados en un servidor MySQL remoto. -## Creación De Una Tabla {#creating-a-table} +## Creación de una tabla {#creating-a-table} ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] @@ -51,7 +51,7 @@ Simple `WHERE` cláusulas tales como `=, !=, >, >=, <, <=` se ejecutan en el ser El resto de las condiciones y el `LIMIT` La restricción de muestreo se ejecuta en ClickHouse solo después de que finalice la consulta a MySQL. -## Ejemplo De Uso {#usage-example} +## Ejemplo de uso {#usage-example} Tabla en MySQL: diff --git a/docs/es/engines/table-engines/integrations/odbc.md b/docs/es/engines/table-engines/integrations/odbc.md index 385faac9010..75c79484d61 100644 --- a/docs/es/engines/table-engines/integrations/odbc.md +++ b/docs/es/engines/table-engines/integrations/odbc.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 35 toc_title: ODBC --- @@ -13,7 +13,7 @@ Para implementar con seguridad conexiones ODBC, ClickHouse usa un programa separ Este motor soporta el [NULL](../../../sql-reference/data-types/nullable.md) tipo de datos. -## Creación De Una Tabla {#creating-a-table} +## Creación de una tabla {#creating-a-table} ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] @@ -38,7 +38,7 @@ La estructura de la tabla puede diferir de la estructura de la tabla de origen: - `external_database` — Name of a database in an external DBMS. - `external_table` — Name of a table in the `external_database`. -## Ejemplo De Uso {#usage-example} +## Ejemplo de uso {#usage-example} **Recuperación de datos de la instalación local de MySQL a través de ODBC** diff --git a/docs/es/engines/table-engines/log-family/index.md b/docs/es/engines/table-engines/log-family/index.md index 53f85f95043..42fd671a063 100644 --- a/docs/es/engines/table-engines/log-family/index.md +++ b/docs/es/engines/table-engines/log-family/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa -toc_folder_title: Log Family +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: Familia de registro toc_priority: 29 --- diff --git a/docs/es/engines/table-engines/log-family/log-family.md b/docs/es/engines/table-engines/log-family/log-family.md index f73b8cfa8db..ee264f88d79 100644 --- a/docs/es/engines/table-engines/log-family/log-family.md +++ b/docs/es/engines/table-engines/log-family/log-family.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 31 toc_title: "Implantaci\xF3n" --- -# Familia Del Motor De Registro {#log-engine-family} +# Familia del motor de registro {#log-engine-family} Estos motores fueron desarrollados para escenarios en los que necesita escribir rápidamente muchas tablas pequeñas (hasta aproximadamente 1 millón de filas) y leerlas más tarde en su conjunto. @@ -15,7 +15,7 @@ Motores de la familia: - [Registro](log.md) - [TinyLog](tinylog.md) -## Propiedades Comunes {#common-properties} +## Propiedades comunes {#common-properties} Motor: diff --git a/docs/es/engines/table-engines/log-family/log.md b/docs/es/engines/table-engines/log-family/log.md index d361389e8da..e9ce17214eb 100644 --- a/docs/es/engines/table-engines/log-family/log.md +++ b/docs/es/engines/table-engines/log-family/log.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 33 toc_title: Registro --- diff --git a/docs/es/engines/table-engines/log-family/stripelog.md b/docs/es/engines/table-engines/log-family/stripelog.md index f96f3f73923..59a981a2fa6 100644 --- a/docs/es/engines/table-engines/log-family/stripelog.md +++ b/docs/es/engines/table-engines/log-family/stripelog.md @@ -1,17 +1,17 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 32 toc_title: StripeLog --- -# Lista De Stripelog {#stripelog} +# Lista de Stripelog {#stripelog} Este motor pertenece a la familia de motores de registro. Consulte las propiedades comunes de los motores de registro y sus diferencias en [Familia del motor de registro](log-family.md) artículo. Utilice este motor en escenarios en los que necesite escribir muchas tablas con una pequeña cantidad de datos (menos de 1 millón de filas). -## Creación De Una Tabla {#table_engines-stripelog-creating-a-table} +## Creación de una tabla {#table_engines-stripelog-creating-a-table} ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] @@ -24,7 +24,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] Vea la descripción detallada del [CREATE TABLE](../../../sql-reference/statements/create.md#create-table-query) consulta. -## Escribir Los Datos {#table_engines-stripelog-writing-the-data} +## Escribir los datos {#table_engines-stripelog-writing-the-data} El `StripeLog` el motor almacena todas las columnas en un archivo. Para cada `INSERT` consulta, ClickHouse agrega el bloque de datos al final de un archivo de tabla, escribiendo columnas una por una. @@ -35,11 +35,11 @@ Para cada tabla, ClickHouse escribe los archivos: El `StripeLog` el motor no soporta el `ALTER UPDATE` y `ALTER DELETE` operación. -## Lectura De Los Datos {#table_engines-stripelog-reading-the-data} +## Lectura de los datos {#table_engines-stripelog-reading-the-data} El archivo con marcas permite ClickHouse paralelizar la lectura de datos. Esto significa que un `SELECT` query devuelve filas en un orden impredecible. Utilice el `ORDER BY` cláusula para ordenar filas. -## Ejemplo De Uso {#table_engines-stripelog-example-of-use} +## Ejemplo de uso {#table_engines-stripelog-example-of-use} Creación de una tabla: diff --git a/docs/es/engines/table-engines/log-family/tinylog.md b/docs/es/engines/table-engines/log-family/tinylog.md index f3a5a525925..1aa392fb831 100644 --- a/docs/es/engines/table-engines/log-family/tinylog.md +++ b/docs/es/engines/table-engines/log-family/tinylog.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 34 toc_title: TinyLog --- diff --git a/docs/es/engines/table-engines/mergetree-family/aggregatingmergetree.md b/docs/es/engines/table-engines/mergetree-family/aggregatingmergetree.md index 9be6ed08246..2aedfbd2317 100644 --- a/docs/es/engines/table-engines/mergetree-family/aggregatingmergetree.md +++ b/docs/es/engines/table-engines/mergetree-family/aggregatingmergetree.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 35 toc_title: "Agregaci\xF3nMergeTree" --- @@ -11,11 +11,14 @@ El motor hereda de [Método de codificación de datos:](mergetree.md#table_engin Usted puede utilizar `AggregatingMergeTree` tablas para la agregación de datos incrementales, incluidas las vistas materializadas agregadas. -El motor procesa todas las columnas con [AggregateFunction](../../../sql-reference/data-types/aggregatefunction.md) tipo. +El motor procesa todas las columnas con los siguientes tipos: + +- [AggregateFunction](../../../sql-reference/data-types/aggregatefunction.md) +- [SimpleAggregateFunction](../../../sql-reference/data-types/simpleaggregatefunction.md) Es apropiado usar `AggregatingMergeTree` si reduce el número de filas por pedidos. -## Creación De Una Tabla {#creating-a-table} +## Creación de una tabla {#creating-a-table} ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] @@ -63,7 +66,7 @@ Al seleccionar datos de `AggregatingMergeTree` mesa, uso `GROUP BY` cláusula y En los resultados de `SELECT` consulta, los valores de `AggregateFunction` tipo tiene representación binaria específica de la implementación para todos los formatos de salida de ClickHouse. Si volcar datos en, por ejemplo, `TabSeparated` formato con `SELECT` consulta entonces este volcado se puede cargar de nuevo usando `INSERT` consulta. -## Ejemplo De Una Vista Materializada Agregada {#example-of-an-aggregated-materialized-view} +## Ejemplo de una vista materializada agregada {#example-of-an-aggregated-materialized-view} `AggregatingMergeTree` vista materializada que mira el `test.visits` tabla: diff --git a/docs/es/engines/table-engines/mergetree-family/collapsingmergetree.md b/docs/es/engines/table-engines/mergetree-family/collapsingmergetree.md index 0afe9edd051..027d5c2adf7 100644 --- a/docs/es/engines/table-engines/mergetree-family/collapsingmergetree.md +++ b/docs/es/engines/table-engines/mergetree-family/collapsingmergetree.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 36 toc_title: ColapsarMergeTree --- -# Colapsarmergetree {#table_engine-collapsingmergetree} +# ColapsarMergeTree {#table_engine-collapsingmergetree} El motor hereda de [Método de codificación de datos:](mergetree.md) y agrega la lógica de las filas que colapsan al algoritmo de fusión de partes de datos. @@ -13,7 +13,7 @@ El motor hereda de [Método de codificación de datos:](mergetree.md) y agrega l El motor puede reducir significativamente el volumen de almacenamiento y aumentar la eficiencia de `SELECT` consulta como consecuencia. -## Creación De Una Tabla {#creating-a-table} +## Creación de una tabla {#creating-a-table} ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] @@ -119,11 +119,8 @@ Cuando ClickHouse combina partes de datos, cada grupo de filas consecutivas tien Para cada parte de datos resultante, ClickHouse guarda: 1. El primero “cancel” y el último “state” si el número de “state” y “cancel” y la última fila es una “state” fila. - 2. El último “state” fila, si hay más “state” filas que “cancel” filas. - 3. El primero “cancel” fila, si hay más “cancel” filas que “state” filas. - 4. Ninguna de las filas, en todos los demás casos. También cuando hay al menos 2 más “state” filas que “cancel” filas, o al menos 2 más “cancel” filas entonces “state” fila, la fusión continúa, pero ClickHouse trata esta situación como un error lógico y la registra en el registro del servidor. Este error puede producirse si se insertan los mismos datos más de una vez. @@ -139,7 +136,7 @@ Los agregados `count`, `sum` y `avg` podría calcularse de esta manera. El agreg Si necesita extraer datos sin agregación (por ejemplo, para comprobar si hay filas presentes cuyos valores más recientes coinciden con ciertas condiciones), puede utilizar el `FINAL` modificador para el `FROM` clausula. Este enfoque es significativamente menos eficiente. -## Ejemplo De Uso {#example-of-use} +## Ejemplo de uso {#example-of-use} Datos de ejemplo: @@ -193,7 +190,7 @@ SELECT * FROM UAct └─────────────────────┴───────────┴──────────┴──────┘ ``` -¿qué vemos y dónde está colapsando? +¿Qué vemos y dónde está colapsando? Con dos `INSERT` consultas, hemos creado 2 partes de datos. El `SELECT` la consulta se realizó en 2 hilos, y obtuvimos un orden aleatorio de filas. No se ha producido un colapso porque todavía no se había fusionado las partes de datos. ClickHouse fusiona parte de datos en un momento desconocido que no podemos predecir. @@ -229,7 +226,7 @@ SELECT * FROM UAct FINAL Esta forma de seleccionar los datos es muy ineficiente. No lo use para mesas grandes. -## Ejemplo De Otro Enfoque {#example-of-another-approach} +## Ejemplo de otro enfoque {#example-of-another-approach} Datos de ejemplo: diff --git a/docs/es/engines/table-engines/mergetree-family/custom-partitioning-key.md b/docs/es/engines/table-engines/mergetree-family/custom-partitioning-key.md index c7bb60633fd..49829ee864f 100644 --- a/docs/es/engines/table-engines/mergetree-family/custom-partitioning-key.md +++ b/docs/es/engines/table-engines/mergetree-family/custom-partitioning-key.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 32 toc_title: "Clave de partici\xF3n personalizada" --- -# Clave De partición Personalizada {#custom-partitioning-key} +# Clave de partición personalizada {#custom-partitioning-key} La partición está disponible para el [Método de codificación de datos:](mergetree.md) mesas familiares (incluyendo [repetición](replication.md) tabla). [Vistas materializadas](../special/materializedview.md#materializedview) basado en tablas MergeTree soporte de particionamiento, también. diff --git a/docs/es/engines/table-engines/mergetree-family/graphitemergetree.md b/docs/es/engines/table-engines/mergetree-family/graphitemergetree.md index e8696fc3466..66438fc4bf9 100644 --- a/docs/es/engines/table-engines/mergetree-family/graphitemergetree.md +++ b/docs/es/engines/table-engines/mergetree-family/graphitemergetree.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 38 toc_title: GraphiteMergeTree --- -# Graphitemergetree {#graphitemergetree} +# GraphiteMergeTree {#graphitemergetree} Este motor está diseñado para el adelgazamiento y la agregación / promedio (rollup) [Grafito](http://graphite.readthedocs.io/en/latest/index.html) datos. Puede ser útil para los desarrolladores que desean usar ClickHouse como almacén de datos para Graphite. @@ -13,7 +13,7 @@ Puede usar cualquier motor de tabla ClickHouse para almacenar los datos de Graph El motor hereda propiedades de [Método de codificación de datos:](mergetree.md). -## Creación De Una Tabla {#creating-table} +## Creación de una tabla {#creating-table} ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] @@ -79,7 +79,7 @@ Todos los parámetros excepto `config_section` el mismo significado que en `Merg -## Configuración Acumulativa {#rollup-configuration} +## Configuración acumulativa {#rollup-configuration} La configuración del paquete acumulativo está definida por [graphite\_rollup](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-graphite) parámetro en la configuración del servidor. El nombre del parámetro podría ser cualquiera. Puede crear varias configuraciones y usarlas para diferentes tablas. @@ -88,7 +88,7 @@ Estructura de configuración Rollup: required-columns patterns -### Columnas Requeridas {#required-columns} +### Columnas requeridas {#required-columns} - `path_column_name` — The name of the column storing the metric name (Graphite sensor). Default value: `Path`. - `time_column_name` — The name of the column storing the time of measuring the metric. Default value: `Time`. @@ -136,7 +136,7 @@ Campos para `pattern` y `default` apartado: - `precision`– How precisely to define the age of the data in seconds. Should be a divisor for 86400 (seconds in a day). - `function` – The name of the aggregating function to apply to data whose age falls within the range `[age, age + precision]`. -### Ejemplo De configuración {#configuration-example} +### Ejemplo de configuración {#configuration-example} ``` xml diff --git a/docs/es/engines/table-engines/mergetree-family/index.md b/docs/es/engines/table-engines/mergetree-family/index.md index 339e1dd8eb3..359d58b2ff1 100644 --- a/docs/es/engines/table-engines/mergetree-family/index.md +++ b/docs/es/engines/table-engines/mergetree-family/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa -toc_folder_title: MergeTree Family +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: Familia MergeTree toc_priority: 28 --- diff --git a/docs/es/engines/table-engines/mergetree-family/mergetree.md b/docs/es/engines/table-engines/mergetree-family/mergetree.md index f05b2a3a9c7..9f7dc5a63f8 100644 --- a/docs/es/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/es/engines/table-engines/mergetree-family/mergetree.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 30 toc_title: "M\xE9todo de codificaci\xF3n de datos:" --- -# Mergetree {#table_engines-mergetree} +# Método de codificación de datos: {#table_engines-mergetree} El `MergeTree` motor y otros motores de esta familia (`*MergeTree`) son los motores de mesa ClickHouse más robustos. @@ -32,7 +32,7 @@ Principales características: !!! info "INFO" El [Fusionar](../special/merge.md#merge) el motor no pertenece al `*MergeTree` familia. -## Creación De Una Tabla {#table_engine-mergetree-creating-a-table} +## Creación de una tabla {#table_engine-mergetree-creating-a-table} ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] @@ -56,7 +56,7 @@ Para obtener una descripción de los parámetros, consulte [Descripción de la c !!! note "Nota" `INDEX` es una característica experimental, ver [Índices de saltos de datos](#table_engine-mergetree-data_skipping-indexes). -### Cláusulas De Consulta {#mergetree-query-clauses} +### Cláusulas de consulta {#mergetree-query-clauses} - `ENGINE` — Name and parameters of the engine. `ENGINE = MergeTree()`. El `MergeTree` el motor no tiene parámetros. @@ -94,7 +94,7 @@ Para obtener una descripción de los parámetros, consulte [Descripción de la c - `min_merge_bytes_to_use_direct_io` — The minimum data volume for merge operation that is required for using direct I/O access to the storage disk. When merging data parts, ClickHouse calculates the total storage volume of all the data to be merged. If the volume exceeds `min_merge_bytes_to_use_direct_io` bytes, ClickHouse lee y escribe los datos en el disco de almacenamiento utilizando la interfaz de E / S directa (`O_DIRECT` opcion). Si `min_merge_bytes_to_use_direct_io = 0`, entonces la E/S directa está deshabilitada. Valor predeterminado: `10 * 1024 * 1024 * 1024` byte. - `merge_with_ttl_timeout` — Minimum delay in seconds before repeating a merge with TTL. Default value: 86400 (1 day). - - `write_final_mark` — Enables or disables writing the final index mark at the end of data part (after the last byte). Default value: 1. Don’t turn it off. + - `write_final_mark` — Enables or disables writing the final index mark at the end of data part (after the last byte). Default value: 1. Don't turn it off. - `merge_max_block_size` — Maximum number of rows in block for merge operations. Default value: 8192. - `storage_policy` — Storage policy. See [Uso de varios dispositivos de bloque para el almacenamiento de datos](#table_engine-mergetree-multiple-volumes). @@ -106,7 +106,7 @@ ENGINE MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDa En el ejemplo, configuramos la partición por mes. -También establecemos una expresión para el muestreo como un hash por el ID de usuario. Esto le permite pseudoaleatorizar los datos en la tabla para cada `CounterID` y `EventDate`. Si define un [SAMPLE](../../../sql-reference/statements/select.md#select-sample-clause) cláusula al seleccionar los datos, ClickHouse devolverá una muestra de datos pseudoaleatoria uniforme para un subconjunto de usuarios. +También establecemos una expresión para el muestreo como un hash por el ID de usuario. Esto le permite pseudoaleatorizar los datos en la tabla para cada `CounterID` y `EventDate`. Si define un [SAMPLE](../../../sql-reference/statements/select/sample.md#select-sample-clause) cláusula al seleccionar los datos, ClickHouse devolverá una muestra de datos pseudoaleatoria uniforme para un subconjunto de usuarios. El `index_granularity` se puede omitir porque 8192 es el valor predeterminado. @@ -142,7 +142,7 @@ MergeTree(EventDate, intHash32(UserID), (CounterID, EventDate, intHash32(UserID) El `MergeTree` engine se configura de la misma manera que en el ejemplo anterior para el método de configuración del motor principal. -## Almacenamiento De Datos {#mergetree-data-storage} +## Almacenamiento de datos {#mergetree-data-storage} Una tabla consta de partes de datos ordenadas por clave principal. @@ -154,7 +154,7 @@ Cada parte de datos se divide lógicamente en gránulos. Un gránulo es el conju El tamaño del gránulo es restringido por `index_granularity` y `index_granularity_bytes` configuración del motor de tabla. El número de filas en un gránulo se encuentra en el `[1, index_granularity]` rango, dependiendo del tamaño de las filas. El tamaño de un gránulo puede exceder `index_granularity_bytes` si el tamaño de una sola fila es mayor que el valor de la configuración. En este caso, el tamaño del gránulo es igual al tamaño de la fila. -## Claves e índices Principales En Consultas {#primary-keys-and-indexes-in-queries} +## Claves e índices principales en consultas {#primary-keys-and-indexes-in-queries} Tome el `(CounterID, Date)` clave primaria como ejemplo. En este caso, la clasificación y el índice se pueden ilustrar de la siguiente manera: @@ -179,7 +179,7 @@ Los índices dispersos le permiten trabajar con una gran cantidad de filas de ta ClickHouse no requiere una clave principal única. Puede insertar varias filas con la misma clave principal. -### Selección De La Clave Principal {#selecting-the-primary-key} +### Selección de la clave principal {#selecting-the-primary-key} El número de columnas en la clave principal no está explícitamente limitado. Dependiendo de la estructura de datos, puede incluir más o menos columnas en la clave principal. Esto puede: @@ -200,7 +200,7 @@ El número de columnas en la clave principal no está explícitamente limitado. Una clave principal larga afectará negativamente al rendimiento de la inserción y al consumo de memoria, pero las columnas adicionales de la clave principal no afectarán al rendimiento de ClickHouse durante `SELECT` consulta. -### Elegir Una Clave Principal Que Difiere De La Clave De ordenación {#choosing-a-primary-key-that-differs-from-the-sorting-key} +### Elegir una clave principal que difiere de la clave de ordenación {#choosing-a-primary-key-that-differs-from-the-sorting-key} Es posible especificar una clave principal (una expresión con valores que se escriben en el archivo de índice para cada marca) que es diferente de la clave de ordenación (una expresión para ordenar las filas en partes de datos). En este caso, la tupla de expresión de clave primaria debe ser un prefijo de la tupla de expresión de clave de ordenación. @@ -211,7 +211,7 @@ En este caso, tiene sentido dejar solo unas pocas columnas en la clave principal [ALTER](../../../sql-reference/statements/alter.md) de la clave de ordenación es una operación ligera porque cuando se agrega una nueva columna simultáneamente a la tabla y a la clave de ordenación, las partes de datos existentes no necesitan ser cambiadas. Dado que la clave de ordenación anterior es un prefijo de la nueva clave de ordenación y no hay datos en la columna recién agregada, los datos se ordenan tanto por las claves de ordenación antiguas como por las nuevas en el momento de la modificación de la tabla. -### Uso De índices y Particiones En Consultas {#use-of-indexes-and-partitions-in-queries} +### Uso de índices y particiones en consultas {#use-of-indexes-and-partitions-in-queries} Para `SELECT` consultas, ClickHouse analiza si se puede usar un índice. Se puede usar un índice si el `WHERE/PREWHERE` clause tiene una expresión (como uno de los elementos de conjunción, o enteramente) que representa una operación de comparación de igualdad o desigualdad, o si tiene `IN` o `LIKE` con un prefijo fijo en columnas o expresiones que están en la clave principal o clave de partición, o en ciertas funciones parcialmente repetitivas de estas columnas, o relaciones lógicas de estas expresiones. @@ -243,7 +243,7 @@ Para comprobar si ClickHouse puede usar el índice al ejecutar una consulta, use La clave para particionar por mes permite leer solo aquellos bloques de datos que contienen fechas del rango adecuado. En este caso, el bloque de datos puede contener datos para muchas fechas (hasta un mes). Dentro de un bloque, los datos se ordenan por clave principal, que puede no contener la fecha como la primera columna. Debido a esto, el uso de una consulta con solo una condición de fecha que no especifica el prefijo de clave principal hará que se lean más datos que para una sola fecha. -### Uso Del índice Para Claves Primarias Parcialmente monotónicas {#use-of-index-for-partially-monotonic-primary-keys} +### Uso del índice para claves primarias parcialmente monótonas {#use-of-index-for-partially-monotonic-primary-keys} Considere, por ejemplo, los días del mes. Ellos forman un [monótona secuencia](https://en.wikipedia.org/wiki/Monotonic_function) durante un mes, pero no monótono durante períodos más prolongados. Esta es una secuencia parcialmente monotónica. Si un usuario crea la tabla con clave primaria parcialmente monótona, ClickHouse crea un índice disperso como de costumbre. Cuando un usuario selecciona datos de este tipo de tabla, ClickHouse analiza las condiciones de consulta. Si el usuario desea obtener datos entre dos marcas del índice y ambas marcas caen dentro de un mes, ClickHouse puede usar el índice en este caso particular porque puede calcular la distancia entre los parámetros de una consulta y las marcas de índice. @@ -251,7 +251,7 @@ ClickHouse no puede usar un índice si los valores de la clave principal en el r ClickHouse usa esta lógica no solo para secuencias de días del mes, sino para cualquier clave principal que represente una secuencia parcialmente monotónica. -### Índices De Saltos De Datos (experimental) {#table_engine-mergetree-data_skipping-indexes} +### Índices de saltos de datos (experimental) {#table_engine-mergetree-data_skipping-indexes} La declaración de índice se encuentra en la sección de columnas del `CREATE` consulta. @@ -285,7 +285,7 @@ SELECT count() FROM table WHERE s < 'z' SELECT count() FROM table WHERE u64 * i32 == 10 AND u64 * length(s) >= 1234 ``` -#### Tipos De índices Disponibles {#available-types-of-indices} +#### Tipos de índices disponibles {#available-types-of-indices} - `minmax` @@ -297,7 +297,7 @@ SELECT count() FROM table WHERE u64 * i32 == 10 AND u64 * length(s) >= 1234 - `ngrambf_v1(n, size_of_bloom_filter_in_bytes, number_of_hash_functions, random_seed)` - Tiendas a [Bloom filter](https://en.wikipedia.org/wiki/Bloom_filter) que contiene todos los ngrams de un bloque de datos. Funciona solo con cadenas. Puede ser utilizado para la optimización de `equals`, `like` y `in` expresiones. + Tiendas a [Filtro de floración](https://en.wikipedia.org/wiki/Bloom_filter) que contiene todos los ngrams de un bloque de datos. Funciona solo con cadenas. Puede ser utilizado para la optimización de `equals`, `like` y `in` expresiones. - `n` — ngram size, - `size_of_bloom_filter_in_bytes` — Bloom filter size in bytes (you can use large values here, for example, 256 or 512, because it can be compressed well). @@ -324,7 +324,7 @@ INDEX sample_index2 (u64 * length(str), i32 + f64 * 100, date, str) TYPE set(100 INDEX sample_index3 (lower(str), str) TYPE ngrambf_v1(3, 256, 2, 0) GRANULARITY 4 ``` -#### Funciones De Apoyo {#functions-support} +#### Funciones de apoyo {#functions-support} Condiciones en el `WHERE` cláusula contiene llamadas de las funciones que operan con columnas. Si la columna forma parte de un índice, ClickHouse intenta usar este índice al realizar las funciones. ClickHouse admite diferentes subconjuntos de funciones para usar índices. @@ -366,13 +366,13 @@ Los filtros Bloom pueden tener coincidencias falsas positivas, por lo que `ngram - `s != 1` - `NOT startsWith(s, 'test')` -## Acceso a Datos simultáneos {#concurrent-data-access} +## Acceso a datos simultáneos {#concurrent-data-access} Para el acceso simultáneo a tablas, usamos versiones múltiples. En otras palabras, cuando una tabla se lee y actualiza simultáneamente, los datos se leen de un conjunto de partes que está actualizado en el momento de la consulta. No hay cerraduras largas. Las inserciones no se interponen en el camino de las operaciones de lectura. La lectura de una tabla se paralela automáticamente. -## TTL Para Columnas y Tablas {#table_engine-mergetree-ttl} +## TTL para columnas y tablas {#table_engine-mergetree-ttl} Determina la duración de los valores. @@ -387,7 +387,7 @@ TTL time_column TTL time_column + interval ``` -Definir `interval`, utilizar [intervalo de tiempo](../../../sql-reference/operators.md#operators-datetime) operador. +Definir `interval`, utilizar [intervalo de tiempo](../../../sql-reference/operators/index.md#operators-datetime) operador. ``` sql TTL date_time + INTERVAL 1 MONTH @@ -476,11 +476,11 @@ ALTER TABLE example_table Los datos con un TTL caducado se eliminan cuando ClickHouse fusiona partes de datos. -Cuando ClickHouse ve que los datos han caducado, realiza una combinación fuera de programación. Para controlar la frecuencia de tales fusiones, puede establecer [Método de codificación de datos:](#mergetree_setting-merge_with_ttl_timeout). Si el valor es demasiado bajo, realizará muchas fusiones fuera de horario que pueden consumir muchos recursos. +Cuando ClickHouse ve que los datos han caducado, realiza una combinación fuera de programación. Para controlar la frecuencia de tales fusiones, puede establecer `merge_with_ttl_timeout`. Si el valor es demasiado bajo, realizará muchas fusiones fuera de horario que pueden consumir muchos recursos. Si realiza el `SELECT` consulta entre fusiones, puede obtener datos caducados. Para evitarlo, use el [OPTIMIZE](../../../sql-reference/statements/misc.md#misc_operations-optimize) consulta antes `SELECT`. -## Uso De múltiples Dispositivos De Bloque Para El Almacenamiento De Datos {#table_engine-mergetree-multiple-volumes} +## Uso de varios dispositivos de bloque para el almacenamiento de datos {#table_engine-mergetree-multiple-volumes} ### Implantación {#introduction} @@ -495,13 +495,13 @@ La parte de datos es la unidad móvil mínima para `MergeTree`-mesas de motor. L - Volume — Ordered set of equal disks (similar to [JBOD](https://en.wikipedia.org/wiki/Non-RAID_drive_architectures)). - Storage policy — Set of volumes and the rules for moving data between them. -Los nombres dados a las entidades descritas se pueden encontrar en las tablas del sistema, [sistema.almacenamiento\_policies](../../../operations/system-tables.md#system_tables-storage_policies) y [sistema.disco](../../../operations/system-tables.md#system_tables-disks). Para aplicar una de las directivas de almacenamiento configuradas para una tabla, `storage_policy` establecimiento de `MergeTree`-motor de la familia de las tablas. +Los nombres dados a las entidades descritas se pueden encontrar en las tablas del sistema, [sistema.almacenamiento\_policies](../../../operations/system-tables.md#system_tables-storage_policies) y [sistema.disco](../../../operations/system-tables.md#system_tables-disks). Para aplicar una de las directivas de almacenamiento configuradas para una tabla, `storage_policy` establecimiento de `MergeTree`-mesas de la familia del motor. ### Configuración {#table_engine-mergetree-multiple-volumes_configure} -Discos, volúmenes y políticas de almacenamiento deben ser declaradas dentro de la `` etiqueta ya sea en el archivo principal `config.xml` o en un archivo distinto en el `config.d` directorio. +Los discos, los volúmenes y las políticas de almacenamiento deben declararse `` etiqueta ya sea en el archivo principal `config.xml` o en un archivo distinto en el `config.d` directorio. -Configuración de la estructura: +Estructura de configuración: ``` xml @@ -567,7 +567,7 @@ Tags: - `policy_name_N` — Policy name. Policy names must be unique. - `volume_name_N` — Volume name. Volume names must be unique. - `disk` — a disk within a volume. -- `max_data_part_size_bytes` — the maximum size of a part that can be stored on any of the volume’s disks. +- `max_data_part_size_bytes` — the maximum size of a part that can be stored on any of the volume's disks. - `move_factor` — when the amount of available space gets lower than this factor, data automatically start to move on the next volume if any (by default, 0.1). Cofiguration ejemplos: @@ -602,9 +602,9 @@ Cofiguration ejemplos: ``` -En un ejemplo dado, el `hdd_in_order` implementa la política de [Ronda-robin](https://en.wikipedia.org/wiki/Round-robin_scheduling) enfoque. Por lo tanto esta política define un sólo volumen (`single`), las partes de datos se almacenan en todos sus discos en orden circular. Dicha política puede ser bastante útil si hay varios discos similares montados en el sistema, pero RAID no está configurado. Tenga en cuenta que cada unidad de disco individual no es confiable y es posible que desee compensarlo con un factor de replicación de 3 o más. +En un ejemplo dado, el `hdd_in_order` la política implementa el [Ronda-robin](https://en.wikipedia.org/wiki/Round-robin_scheduling) enfoque. Por lo tanto, esta política define solo un volumen (`single`), las partes de datos se almacenan en todos sus discos en orden circular. Dicha política puede ser bastante útil si hay varios discos similares montados en el sistema, pero RAID no está configurado. Tenga en cuenta que cada unidad de disco individual no es confiable y es posible que desee compensarlo con un factor de replicación de 3 o más. -Si hay diferentes tipos de discos disponibles en el sistema, `moving_from_ssd_to_hdd` la política puede ser utilizado en su lugar. Volumen `hot` consta de un disco SSD (`fast_ssd`), y el tamaño máximo de una parte que puede ser almacenado en este volumen es de 1GB. Todas las piezas con el tamaño de más de 1GB se almacenan directamente en el `cold` volumen, que contiene un disco duro `disk1`. +Si hay diferentes tipos de discos disponibles en el sistema, `moving_from_ssd_to_hdd` política se puede utilizar en su lugar. Volumen `hot` consta de un disco SSD (`fast_ssd`), y el tamaño máximo de una pieza que se puede almacenar en este volumen es de 1 GB. Todas las piezas con el tamaño más grande que 1GB serán almacenadas directamente en `cold` volumen, que contiene un disco duro `disk1`. Además, una vez que el disco `fast_ssd` se llena en más del 80%, los datos se transferirán al `disk1` por un proceso en segundo plano. El orden de enumeración de volúmenes dentro de una directiva de almacenamiento es importante. Una vez que un volumen está sobrellenado, los datos se mueven al siguiente. El orden de la enumeración del disco también es importante porque los datos se almacenan en ellos por turnos. @@ -642,13 +642,13 @@ En todos estos casos, excepto las mutaciones y la congelación de particiones, u Bajo el capó, las mutaciones y la congelación de particiones hacen uso de [enlaces duros](https://en.wikipedia.org/wiki/Hard_link). Los enlaces duros entre diferentes discos no son compatibles, por lo tanto, en tales casos las partes resultantes se almacenan en los mismos discos que los iniciales. En el fondo, las partes se mueven entre volúmenes en función de la cantidad de espacio libre (`move_factor` parámetro) según el orden en que se declaran los volúmenes en el archivo de configuración. -Los datos nunca se transfieren desde el último y al primero. Uno puede usar tablas del sistema [sistema.part\_log](../../../operations/system-tables.md#system_tables-part-log) (campo `type = MOVE_PART`) y [sistema.parte](../../../operations/system-tables.md#system_tables-parts) (campo `path` y `disk`) para monitorear el fondo se mueve. Además, la información detallada se puede encontrar en los registros del servidor. +Los datos nunca se transfieren desde el último y al primero. Uno puede usar tablas del sistema [sistema.part\_log](../../../operations/system-tables.md#system_tables-part-log) (campo `type = MOVE_PART`) y [sistema.parte](../../../operations/system-tables.md#system_tables-parts) (campo `path` y `disk`) para monitorear movimientos de fondo. Además, la información detallada se puede encontrar en los registros del servidor. El usuario puede forzar el movimiento de una pieza o una partición de un volumen a otro mediante la consulta [ALTER TABLE … MOVE PART\|PARTITION … TO VOLUME\|DISK …](../../../sql-reference/statements/alter.md#alter_move-partition), todas las restricciones para las operaciones en segundo plano se tienen en cuenta. La consulta inicia un movimiento por sí misma y no espera a que se completen las operaciones en segundo plano. El usuario recibirá un mensaje de error si no hay suficiente espacio libre disponible o si no se cumple alguna de las condiciones requeridas. Mover datos no interfiere con la replicación de datos. Por lo tanto, se pueden especificar diferentes directivas de almacenamiento para la misma tabla en diferentes réplicas. Después de la finalización de las fusiones y mutaciones de fondo, las partes viejas se eliminan solo después de un cierto período de tiempo (`old_parts_lifetime`). -Durante este tiempo, no se mueven a otros volúmenes o discos. Por lo tanto, hasta que finalmente se retira, se tomará en cuenta para la evaluación de los ocupados de espacio en disco. +Durante este tiempo, no se mueven a otros volúmenes o discos. Por lo tanto, hasta que las partes finalmente se eliminen, aún se tienen en cuenta para la evaluación del espacio en disco ocupado. [Artículo Original](https://clickhouse.tech/docs/ru/operations/table_engines/mergetree/) diff --git a/docs/es/engines/table-engines/mergetree-family/replacingmergetree.md b/docs/es/engines/table-engines/mergetree-family/replacingmergetree.md index e93a5cffd00..a1e95c5b5f4 100644 --- a/docs/es/engines/table-engines/mergetree-family/replacingmergetree.md +++ b/docs/es/engines/table-engines/mergetree-family/replacingmergetree.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 33 toc_title: ReplacingMergeTree --- -# Replacingmergetree {#replacingmergetree} +# ReplacingMergeTree {#replacingmergetree} El motor difiere de [Método de codificación de datos:](mergetree.md#table_engines-mergetree) en que elimina las entradas duplicadas con el mismo valor de clave principal (o más exactamente, con el mismo [clave de clasificación](mergetree.md) valor). @@ -13,7 +13,7 @@ La desduplicación de datos solo se produce durante una fusión. La fusión ocur Así, `ReplacingMergeTree` es adecuado para borrar datos duplicados en segundo plano para ahorrar espacio, pero no garantiza la ausencia de duplicados. -## Creación De Una Tabla {#creating-a-table} +## Creación de una tabla {#creating-a-table} ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] diff --git a/docs/es/engines/table-engines/mergetree-family/replication.md b/docs/es/engines/table-engines/mergetree-family/replication.md index a76a8f68a9a..52b4029fd82 100644 --- a/docs/es/engines/table-engines/mergetree-family/replication.md +++ b/docs/es/engines/table-engines/mergetree-family/replication.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 31 toc_title: "Replicaci\xF3n de datos" --- -# Replicación De Datos {#table_engines-replication} +# Replicación de datos {#table_engines-replication} La replicación solo se admite para tablas de la familia MergeTree: @@ -79,7 +79,7 @@ Puede tener cualquier número de réplicas de los mismos datos. El Yandex.Metric El sistema supervisa la sincronicidad de los datos en las réplicas y puede recuperarse después de un fallo. La conmutación por error es automática (para pequeñas diferencias en los datos) o semiautomática (cuando los datos difieren demasiado, lo que puede indicar un error de configuración). -## Creación De Tablas Replicadas {#creating-replicated-tables} +## Creación de tablas replicadas {#creating-replicated-tables} El `Replicated` prefijo se agrega al nombre del motor de tabla. Por ejemplo:`ReplicatedMergeTree`. @@ -149,7 +149,7 @@ Si agrega una nueva réplica después de que la tabla ya contenga algunos datos Para eliminar una réplica, ejecute `DROP TABLE`. However, only one replica is deleted – the one that resides on the server where you run the query. -## Recuperación después De Fallos {#recovery-after-failures} +## Recuperación después de fallos {#recovery-after-failures} Si ZooKeeper no está disponible cuando se inicia un servidor, las tablas replicadas cambian al modo de solo lectura. El sistema intenta conectarse periódicamente a ZooKeeper. @@ -173,7 +173,7 @@ sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data A continuación, reinicie el servidor. Al iniciar, el servidor elimina estos indicadores e inicia la recuperación. -## Recuperación después De La pérdida Completa De Datos {#recovery-after-complete-data-loss} +## Recuperación después de la pérdida completa de datos {#recovery-after-complete-data-loss} Si todos los datos y metadatos desaparecieron de uno de los servidores, siga estos pasos para la recuperación: @@ -188,7 +188,7 @@ Una opción de recuperación alternativa es eliminar información sobre la répl No hay restricción en el ancho de banda de la red durante la recuperación. Tenga esto en cuenta si está restaurando muchas réplicas a la vez. -## La Conversión De Mergetree A Replicatedmergetree {#converting-from-mergetree-to-replicatedmergetree} +## La conversión de MergeTree a ReplicatedMergeTree {#converting-from-mergetree-to-replicatedmergetree} Usamos el término `MergeTree` para referirse a todos los motores de mesa en el `MergeTree family`, lo mismo que para `ReplicatedMergeTree`. @@ -200,7 +200,7 @@ Cambie el nombre de la tabla MergeTree existente y, a continuación, cree un `Re Mueva los datos de la tabla antigua a la `detached` subdirectorio dentro del directorio con los nuevos datos de la tabla (`/var/lib/clickhouse/data/db_name/table_name/`). Luego ejecuta `ALTER TABLE ATTACH PARTITION` en una de las réplicas para agregar estas partes de datos al conjunto de trabajo. -## La Conversión De Replicatedmergetree A Mergetree {#converting-from-replicatedmergetree-to-mergetree} +## La conversión de ReplicatedMergeTree a MergeTree {#converting-from-replicatedmergetree-to-mergetree} Cree una tabla MergeTree con un nombre diferente. Mueva todos los datos del directorio con el `ReplicatedMergeTree` datos de la tabla al directorio de datos de la nueva tabla. A continuación, elimine el `ReplicatedMergeTree` y reinicie el servidor. @@ -211,7 +211,7 @@ Si desea deshacerse de un `ReplicatedMergeTree` sin iniciar el servidor: Después de esto, puede iniciar el servidor, crear un `MergeTree` tabla, mueva los datos a su directorio y, a continuación, reinicie el servidor. -## Recuperación Cuando Los Metadatos En El clúster De Zookeeper Se Pierden o Se dañan {#recovery-when-metadata-in-the-zookeeper-cluster-is-lost-or-damaged} +## Recuperación cuando se pierden o se dañan los metadatos del clúster Zookeeper {#recovery-when-metadata-in-the-zookeeper-cluster-is-lost-or-damaged} Si los datos de ZooKeeper se perdieron o se dañaron, puede guardar los datos moviéndolos a una tabla no duplicada como se describió anteriormente. diff --git a/docs/es/engines/table-engines/mergetree-family/summingmergetree.md b/docs/es/engines/table-engines/mergetree-family/summingmergetree.md index 3f0acdcf4d0..3ae9a1515c0 100644 --- a/docs/es/engines/table-engines/mergetree-family/summingmergetree.md +++ b/docs/es/engines/table-engines/mergetree-family/summingmergetree.md @@ -1,17 +1,17 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 34 toc_title: SummingMergeTree --- -# Summingmergetree {#summingmergetree} +# SummingMergeTree {#summingmergetree} El motor hereda de [Método de codificación de datos:](mergetree.md#table_engines-mergetree). La diferencia es que al fusionar partes de datos para `SummingMergeTree` ClickHouse reemplaza todas las filas con la misma clave primaria (o más exactamente, con la misma [clave de clasificación](mergetree.md)) con una fila que contiene valores resumidos para las columnas con el tipo de datos numérico. Si la clave de ordenación está compuesta de manera que un solo valor de clave corresponde a un gran número de filas, esto reduce significativamente el volumen de almacenamiento y acelera la selección de datos. Recomendamos usar el motor junto con `MergeTree`. Almacenar datos completos en `MergeTree` mesa, y el uso `SummingMergeTree` para el almacenamiento de datos agregados, por ejemplo, al preparar informes. Tal enfoque evitará que pierda datos valiosos debido a una clave primaria compuesta incorrectamente. -## Creación De Una Tabla {#creating-a-table} +## Creación de una tabla {#creating-a-table} ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] @@ -61,7 +61,7 @@ Todos los parámetros excepto `columns` el mismo significado que en `MergeTree`. -## Ejemplo De Uso {#usage-example} +## Ejemplo de uso {#usage-example} Considere la siguiente tabla: @@ -94,13 +94,13 @@ SELECT key, sum(value) FROM summtt GROUP BY key └─────┴────────────┘ ``` -## Procesamiento De Datos {#data-processing} +## Procesamiento de datos {#data-processing} -Cuando los datos se insertan en una tabla, se guardan tal cual. Clickhouse fusiona las partes insertadas de datos periódicamente y esto es cuando las filas con la misma clave principal se suman y se reemplazan con una para cada parte resultante de los datos. +Cuando los datos se insertan en una tabla, se guardan tal cual. ClickHouse combina las partes insertadas de los datos periódicamente y esto es cuando las filas con la misma clave principal se suman y se reemplazan con una para cada parte resultante de los datos. ClickHouse can merge the data parts so that different resulting parts of data cat consist rows with the same primary key, i.e. the summation will be incomplete. Therefore (`SELECT`) una función agregada [resumir()](../../../sql-reference/aggregate-functions/reference.md#agg_function-sum) y `GROUP BY` cláusula se debe utilizar en una consulta como se describe en el ejemplo anterior. -### Reglas Comunes Para La Suma {#common-rules-for-summation} +### Reglas comunes para la suma {#common-rules-for-summation} Se resumen los valores de las columnas con el tipo de datos numérico. El conjunto de columnas está definido por el parámetro `columns`. @@ -110,11 +110,11 @@ Si la columna no está en la clave principal y no se resume, se selecciona un va Los valores no se resumen para las columnas de la clave principal. -### La Suma En Las Columnas De función Agregada {#the-summation-in-the-aggregatefunction-columns} +### La suma en las columnas de función agregada {#the-summation-in-the-aggregatefunction-columns} Para columnas de [Tipo AggregateFunction](../../../sql-reference/data-types/aggregatefunction.md) ClickHouse se comporta como [AgregaciónMergeTree](aggregatingmergetree.md) agregación del motor según la función. -### Estructuras Anidadas {#nested-structures} +### Estructuras anidadas {#nested-structures} La tabla puede tener estructuras de datos anidadas que se procesan de una manera especial. diff --git a/docs/es/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md b/docs/es/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md index 7101823001f..d69bfe9440e 100644 --- a/docs/es/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md +++ b/docs/es/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 37 toc_title: VersionedCollapsingMergeTree --- -# Versionedcollapsingmergetree {#versionedcollapsingmergetree} +# VersionedCollapsingMergeTree {#versionedcollapsingmergetree} Este motor: @@ -16,7 +16,7 @@ Vea la sección [Derrumbar](#table_engines_versionedcollapsingmergetree) para m El motor hereda de [Método de codificación de datos:](mergetree.md#table_engines-mergetree) y agrega la lógica para colapsar filas al algoritmo para fusionar partes de datos. `VersionedCollapsingMergeTree` tiene el mismo propósito que [ColapsarMergeTree](collapsingmergetree.md) pero usa un algoritmo de colapso diferente que permite insertar los datos en cualquier orden con múltiples hilos. En particular, el `Version` columna ayuda a contraer las filas correctamente, incluso si se insertan en el orden incorrecto. En contraste, `CollapsingMergeTree` sólo permite la inserción estrictamente consecutiva. -## Creación De Una Tabla {#creating-a-table} +## Creación de una tabla {#creating-a-table} ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] @@ -64,7 +64,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], ... -) ENGINE [=] VersionedCollapsingMergeTree(date-column [, sampling_expression], (primary, key), index_granularity, sign, version) +) ENGINE [=] VersionedCollapsingMergeTree(date-column [, samp#table_engines_versionedcollapsingmergetreeling_expression], (primary, key), index_granularity, sign, version) ``` Todos los parámetros excepto `sign` y `version` el mismo significado que en `MergeTree`. @@ -133,7 +133,7 @@ Cuando ClickHouse combina partes de datos, elimina cada par de filas que tienen Cuando ClickHouse inserta datos, ordena filas por la clave principal. Si el `Version` la columna no está en la clave principal, ClickHouse la agrega a la clave principal implícitamente como el último campo y la usa para ordenar. -## Selección De Datos {#selecting-data} +## Selección de datos {#selecting-data} ClickHouse no garantiza que todas las filas con la misma clave principal estén en la misma parte de datos resultante o incluso en el mismo servidor físico. Esto es cierto tanto para escribir los datos como para la posterior fusión de las partes de datos. Además, ClickHouse procesa `SELECT` consultas con múltiples subprocesos, y no puede predecir el orden de las filas en el resultado. Esto significa que la agregación es necesaria si hay una necesidad de obtener completamente “collapsed” datos de un `VersionedCollapsingMergeTree` tabla. @@ -143,7 +143,7 @@ Los agregados `count`, `sum` y `avg` se puede calcular de esta manera. El agrega Si necesita extraer los datos con “collapsing” pero sin agregación (por ejemplo, para verificar si hay filas presentes cuyos valores más nuevos coinciden con ciertas condiciones), puede usar el `FINAL` modificador para el `FROM` clausula. Este enfoque es ineficiente y no debe usarse con tablas grandes. -## Ejemplo De Uso {#example-of-use} +## Ejemplo de uso {#example-of-use} Datos de ejemplo: @@ -198,7 +198,7 @@ SELECT * FROM UAct └─────────────────────┴───────────┴──────────┴──────┴─────────┘ ``` -¿qué vemos aquí y dónde están las partes colapsadas? +¿Qué vemos aquí y dónde están las partes colapsadas? Creamos dos partes de datos usando dos `INSERT` consulta. El `SELECT` la consulta se realizó en dos subprocesos, y el resultado es un orden aleatorio de filas. No se produjo el colapso porque las partes de datos aún no se han fusionado. ClickHouse fusiona partes de datos en un punto desconocido en el tiempo que no podemos predecir. diff --git a/docs/es/engines/table-engines/special/buffer.md b/docs/es/engines/table-engines/special/buffer.md index 0a869d556b8..3c8a20beaf3 100644 --- a/docs/es/engines/table-engines/special/buffer.md +++ b/docs/es/engines/table-engines/special/buffer.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 45 toc_title: "B\xFAfer" --- diff --git a/docs/es/engines/table-engines/special/dictionary.md b/docs/es/engines/table-engines/special/dictionary.md index bd1d0ce1bb5..6d9136a6a23 100644 --- a/docs/es/engines/table-engines/special/dictionary.md +++ b/docs/es/engines/table-engines/special/dictionary.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 35 toc_title: Diccionario --- diff --git a/docs/es/engines/table-engines/special/distributed.md b/docs/es/engines/table-engines/special/distributed.md index 98f9f52338a..df773c7aec4 100644 --- a/docs/es/engines/table-engines/special/distributed.md +++ b/docs/es/engines/table-engines/special/distributed.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 33 toc_title: Distribuido --- @@ -39,7 +39,7 @@ Por ejemplo, para una consulta con GROUP BY, los datos se agregarán en servidor En lugar del nombre de la base de datos, puede usar una expresión constante que devuelva una cadena. Por ejemplo: currentDatabase(). -logs – The cluster name in the server’s config file. +logs – The cluster name in the server's config file. Los clústeres se establecen así: @@ -84,7 +84,7 @@ Las réplicas están duplicando servidores (para leer todos los datos, puede acc Los nombres de clúster no deben contener puntos. Los parámetros `host`, `port`, y opcionalmente `user`, `password`, `secure`, `compression` se especifican para cada servidor: -- `host` – The address of the remote server. You can use either the domain or the IPv4 or IPv6 address. If you specify the domain, the server makes a DNS request when it starts, and the result is stored as long as the server is running. If the DNS request fails, the server doesn’t start. If you change the DNS record, restart the server. +- `host` – The address of the remote server. You can use either the domain or the IPv4 or IPv6 address. If you specify the domain, the server makes a DNS request when it starts, and the result is stored as long as the server is running. If the DNS request fails, the server doesn't start. If you change the DNS record, restart the server. - `port` – The TCP port for messenger activity (‘tcp\_port’ en la configuración, generalmente establecido en 9000). No lo confundas con http\_port. - `user` – Name of the user for connecting to a remote server. Default value: default. This user must have access to connect to the specified server. Access is configured in the users.xml file. For more information, see the section [Derechos de acceso](../../../operations/access-rights.md). - `password` – The password for connecting to a remote server (not masked). Default value: empty string. @@ -103,7 +103,7 @@ Para ver los clústeres, utilice el ‘system.clusters’ tabla. El motor distribuido permite trabajar con un clúster como un servidor local. Sin embargo, el clúster es inextensible: debe escribir su configuración en el archivo de configuración del servidor (mejor aún, para todos los servidores del clúster). -The Distributed engine requires writing clusters to the config file. Clusters from the config file are updated on the fly, without restarting the server. If you need to send a query to an unknown set of shards and replicas each time, you don’t need to create a Distributed table – use the ‘remote’ función de tabla en su lugar. Vea la sección [Funciones de tabla](../../../sql-reference/table-functions/index.md). +The Distributed engine requires writing clusters to the config file. Clusters from the config file are updated on the fly, without restarting the server. If you need to send a query to an unknown set of shards and replicas each time, you don't need to create a Distributed table – use the ‘remote’ función de tabla en su lugar. Vea la sección [Funciones de tabla](../../../sql-reference/table-functions/index.md). Hay dos métodos para escribir datos en un clúster: @@ -125,7 +125,7 @@ La expresión de fragmentación puede ser cualquier expresión de constantes y c Un simple recordatorio de la división es una solución limitada para sharding y no siempre es apropiado. Funciona para volúmenes medianos y grandes de datos (docenas de servidores), pero no para volúmenes muy grandes de datos (cientos de servidores o más). En este último caso, use el esquema de fragmentación requerido por el área asunto, en lugar de usar entradas en Tablas distribuidas. -SELECT queries are sent to all the shards and work regardless of how data is distributed across the shards (they can be distributed completely randomly). When you add a new shard, you don’t have to transfer the old data to it. You can write new data with a heavier weight – the data will be distributed slightly unevenly, but queries will work correctly and efficiently. +SELECT queries are sent to all the shards and work regardless of how data is distributed across the shards (they can be distributed completely randomly). When you add a new shard, you don't have to transfer the old data to it. You can write new data with a heavier weight – the data will be distributed slightly unevenly, but queries will work correctly and efficiently. Debería preocuparse por el esquema de fragmentación en los siguientes casos: diff --git a/docs/es/engines/table-engines/special/external-data.md b/docs/es/engines/table-engines/special/external-data.md index 914abf6ddb5..ce64b375ba2 100644 --- a/docs/es/engines/table-engines/special/external-data.md +++ b/docs/es/engines/table-engines/special/external-data.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 34 toc_title: Datos externos --- -# Datos Externos Para El Procesamiento De Consultas {#external-data-for-query-processing} +# Datos externos para el procesamiento de consultas {#external-data-for-query-processing} ClickHouse permite enviar a un servidor los datos necesarios para procesar una consulta, junto con una consulta SELECT. Estos datos se colocan en una tabla temporal (consulte la sección “Temporary tables”) y se puede utilizar en la consulta (por ejemplo, en operadores IN). diff --git a/docs/es/engines/table-engines/special/file.md b/docs/es/engines/table-engines/special/file.md index b366c72a335..fb739506a22 100644 --- a/docs/es/engines/table-engines/special/file.md +++ b/docs/es/engines/table-engines/special/file.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 37 toc_title: File --- @@ -16,7 +16,7 @@ Ejemplos de uso: - Convertir datos de un formato a otro. - Actualización de datos en ClickHouse mediante la edición de un archivo en un disco. -## Uso En El Servidor De Clickhouse {#usage-in-clickhouse-server} +## Uso en el servidor ClickHouse {#usage-in-clickhouse-server} ``` sql File(Format) @@ -67,16 +67,16 @@ SELECT * FROM file_engine_table └──────┴───────┘ ``` -## Uso En Clickhouse-local {#usage-in-clickhouse-local} +## Uso en ClickHouse-local {#usage-in-clickhouse-local} -En [Sistema abierto.](../../../operations/utilities/clickhouse-local.md) El motor de archivos acepta la ruta del archivo además de `Format`. Los flujos de entrada / salida predeterminados se pueden especificar utilizando nombres numéricos o legibles por humanos como `0` o `stdin`, `1` o `stdout`. +En [Sistema abierto.](../../../operations/utilities/clickhouse-local.md#clickhouse-local) El motor de archivos acepta la ruta del archivo además de `Format`. Los flujos de entrada / salida predeterminados se pueden especificar utilizando nombres numéricos o legibles por humanos como `0` o `stdin`, `1` o `stdout`. **Ejemplo:** ``` bash $ echo -e "1,2\n3,4" | clickhouse-local -q "CREATE TABLE table (a Int64, b Int64) ENGINE = File(CSV, stdin); SELECT a, b FROM table; DROP TABLE table" ``` -## Detalles De La implementación {#details-of-implementation} +## Detalles de la implementación {#details-of-implementation} - Multiple `SELECT` las consultas se pueden realizar simultáneamente, pero `INSERT` las consultas se esperarán entre sí. - Apoyado la creación de nuevos archivos por `INSERT` consulta. diff --git a/docs/es/engines/table-engines/special/generate.md b/docs/es/engines/table-engines/special/generate.md index 008acd15f7a..67e664284b4 100644 --- a/docs/es/engines/table-engines/special/generate.md +++ b/docs/es/engines/table-engines/special/generate.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 46 toc_title: GenerateRandom --- @@ -14,7 +14,7 @@ Ejemplos de uso: - Se usa en la prueba para poblar una tabla grande reproducible. - Generar entrada aleatoria para pruebas de fuzzing. -## Uso En El Servidor De Clickhouse {#usage-in-clickhouse-server} +## Uso en el servidor ClickHouse {#usage-in-clickhouse-server} ``` sql ENGINE = GenerateRandom(random_seed, max_string_length, max_array_length) @@ -49,7 +49,7 @@ SELECT * FROM generate_engine_table LIMIT 3 └──────┴────────────┘ ``` -## Detalles De La implementación {#details-of-implementation} +## Detalles de la implementación {#details-of-implementation} - No soportado: - `ALTER` diff --git a/docs/es/engines/table-engines/special/index.md b/docs/es/engines/table-engines/special/index.md index 9770e3fb6c5..9927a1f61d9 100644 --- a/docs/es/engines/table-engines/special/index.md +++ b/docs/es/engines/table-engines/special/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa -toc_folder_title: Special +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: Especial toc_priority: 31 --- diff --git a/docs/es/engines/table-engines/special/join.md b/docs/es/engines/table-engines/special/join.md index 75825cfc97f..bb8b0e513d9 100644 --- a/docs/es/engines/table-engines/special/join.md +++ b/docs/es/engines/table-engines/special/join.md @@ -1,15 +1,15 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 40 toc_title: Unir --- # Unir {#join} -Estructura de datos preparada para usar en [JOIN](../../../sql-reference/statements/select.md#select-join) operación. +Estructura de datos preparada para usar en [JOIN](../../../sql-reference/statements/select/join.md#select-join) operación. -## Creación De Una Tabla {#creating-a-table} +## Creación de una tabla {#creating-a-table} ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] @@ -23,13 +23,13 @@ Vea la descripción detallada del [CREATE TABLE](../../../sql-reference/statemen **Parámetros del motor** -- `join_strictness` – [ÚNETE a la rigurosidad](../../../sql-reference/statements/select.md#select-join-strictness). -- `join_type` – [Tipo de unión](../../../sql-reference/statements/select.md#select-join-types). +- `join_strictness` – [ÚNETE a la rigurosidad](../../../sql-reference/statements/select/join.md#select-join-strictness). +- `join_type` – [Tipo de unión](../../../sql-reference/statements/select/join.md#select-join-types). - `k1[, k2, ...]` – Key columns from the `USING` cláusula que el `JOIN` operación se hace con. Entrar `join_strictness` y `join_type` parámetros sin comillas, por ejemplo, `Join(ANY, LEFT, col1)`. Deben coincidir con el `JOIN` operación para la que se utilizará la tabla. Si los parámetros no coinciden, ClickHouse no lanza una excepción y puede devolver datos incorrectos. -## Uso De La Tabla {#table-usage} +## Uso de la tabla {#table-usage} ### Ejemplo {#example} @@ -79,7 +79,7 @@ SELECT joinGet('id_val_join', 'val', toUInt32(1)) └────────────────────────────────────────────┘ ``` -### Selección e inserción De Datos {#selecting-and-inserting-data} +### Selección e inserción de datos {#selecting-and-inserting-data} Usted puede utilizar `INSERT` consultas para agregar datos al `Join`-mesas de motor. Si la tabla se creó con el `ANY` estricta, se ignoran los datos de las claves duplicadas. Con el `ALL` estricta, se agregan todas las filas. @@ -88,7 +88,7 @@ No se puede realizar una `SELECT` consulta directamente desde la tabla. En su lu - Coloque la mesa hacia el lado derecho en un `JOIN` clausula. - Llame al [joinGet](../../../sql-reference/functions/other-functions.md#joinget) función, que le permite extraer datos de la tabla de la misma manera que de un diccionario. -### Limitaciones y Ajustes {#join-limitations-and-settings} +### Limitaciones y ajustes {#join-limitations-and-settings} Al crear una tabla, se aplican los siguientes valores: @@ -100,9 +100,9 @@ Al crear una tabla, se aplican los siguientes valores: El `Join`-las tablas del motor no se pueden usar en `GLOBAL JOIN` operación. -El `Join`-motor permite el uso [Sistema abierto.](../../../operations/settings/settings.md#join_use_nulls) ajuste en el `CREATE TABLE` instrucción. Y [SELECT](../../../sql-reference/statements/select.md) consulta permite el uso `join_use_nulls` demasiado. Si tienes diferentes `join_use_nulls` configuración, puede obtener un error al unirse a la tabla. Depende del tipo de JOIN. Cuando se utiliza [joinGet](../../../sql-reference/functions/other-functions.md#joinget) función, usted tiene que utilizar el mismo `join_use_nulls` ajuste en `CRATE TABLE` y `SELECT` instrucción. +El `Join`-motor permite el uso [Sistema abierto.](../../../operations/settings/settings.md#join_use_nulls) ajuste en el `CREATE TABLE` instrucción. Y [SELECT](../../../sql-reference/statements/select/index.md) consulta permite el uso `join_use_nulls` demasiado. Si tienes diferentes `join_use_nulls` configuración, puede obtener un error al unirse a la tabla. Depende del tipo de JOIN. Cuando se utiliza [joinGet](../../../sql-reference/functions/other-functions.md#joinget) función, usted tiene que utilizar el mismo `join_use_nulls` ajuste en `CRATE TABLE` y `SELECT` instrucción. -## Almacenamiento De Datos {#data-storage} +## Almacenamiento de datos {#data-storage} `Join` datos de la tabla siempre se encuentra en la memoria RAM. Al insertar filas en una tabla, ClickHouse escribe bloques de datos en el directorio del disco para que puedan restaurarse cuando se reinicie el servidor. diff --git a/docs/es/engines/table-engines/special/materializedview.md b/docs/es/engines/table-engines/special/materializedview.md index 3da6fb5000a..8b130ab141e 100644 --- a/docs/es/engines/table-engines/special/materializedview.md +++ b/docs/es/engines/table-engines/special/materializedview.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 43 toc_title: "M\xE9todo de codificaci\xF3n de datos:" --- -# Método De codificación De Datos: {#materializedview} +# Método de codificación de datos: {#materializedview} Se utiliza para implementar vistas materializadas (para obtener más información, consulte [CREATE TABLE](../../../sql-reference/statements/create.md)). Para almacenar datos, utiliza un motor diferente que se especificó al crear la vista. Al leer desde una tabla, solo usa este motor. diff --git a/docs/es/engines/table-engines/special/memory.md b/docs/es/engines/table-engines/special/memory.md index f7a2f81ff94..3d4f8ddff54 100644 --- a/docs/es/engines/table-engines/special/memory.md +++ b/docs/es/engines/table-engines/special/memory.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 44 toc_title: Memoria --- diff --git a/docs/es/engines/table-engines/special/merge.md b/docs/es/engines/table-engines/special/merge.md index 14d4c649471..0c19fd7e2b0 100644 --- a/docs/es/engines/table-engines/special/merge.md +++ b/docs/es/engines/table-engines/special/merge.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 36 toc_title: Fusionar --- diff --git a/docs/es/engines/table-engines/special/null.md b/docs/es/engines/table-engines/special/null.md index 53abadefb31..cc05e7839c9 100644 --- a/docs/es/engines/table-engines/special/null.md +++ b/docs/es/engines/table-engines/special/null.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 38 toc_title: Nulo --- diff --git a/docs/es/engines/table-engines/special/set.md b/docs/es/engines/table-engines/special/set.md index 4afc3fad85d..4ff23202443 100644 --- a/docs/es/engines/table-engines/special/set.md +++ b/docs/es/engines/table-engines/special/set.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 39 toc_title: Establecer --- diff --git a/docs/es/engines/table-engines/special/url.md b/docs/es/engines/table-engines/special/url.md index 0197b5e9a4a..654b8e99a4e 100644 --- a/docs/es/engines/table-engines/special/url.md +++ b/docs/es/engines/table-engines/special/url.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 41 toc_title: URL --- @@ -10,7 +10,7 @@ toc_title: URL Administra datos en un servidor HTTP/HTTPS remoto. Este motor es similar a la [File](file.md) motor. -## Uso Del Motor En El Servidor De Clickhouse {#using-the-engine-in-the-clickhouse-server} +## Uso del motor en el servidor ClickHouse {#using-the-engine-in-the-clickhouse-server} El `format` debe ser uno que ClickHouse pueda usar en `SELECT` consultas y, si es necesario, en `INSERTs`. Para obtener la lista completa de formatos admitidos, consulte @@ -71,7 +71,7 @@ SELECT * FROM url_engine_table └───────┴───────┘ ``` -## Detalles De La implementación {#details-of-implementation} +## Detalles de la implementación {#details-of-implementation} - Las lecturas y escrituras pueden ser paralelas - No soportado: diff --git a/docs/es/engines/table-engines/special/view.md b/docs/es/engines/table-engines/special/view.md index 3dae42d1b06..dbb496bcca4 100644 --- a/docs/es/engines/table-engines/special/view.md +++ b/docs/es/engines/table-engines/special/view.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 42 toc_title: Vista --- diff --git a/docs/es/faq/general.md b/docs/es/faq/general.md index c94babf1b94..dc6ba5b5915 100644 --- a/docs/es/faq/general.md +++ b/docs/es/faq/general.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 78 toc_title: Preguntas generales --- -# Preguntas Generales {#general-questions} +# Preguntas generales {#general-questions} -## ¿por qué no usar algo como mapreduce? {#why-not-use-something-like-mapreduce} +## ¿Por qué no usar algo como MapReduce? {#why-not-use-something-like-mapreduce} Podemos referirnos a sistemas como MapReduce como sistemas informáticos distribuidos en los que la operación de reducción se basa en la clasificación distribuida. La solución de código abierto más común en esta clase es [Acerca de nosotros](http://hadoop.apache.org). Yandex utiliza su solución interna, YT. @@ -15,7 +15,7 @@ Estos sistemas no son apropiados para consultas en línea debido a su alta laten La mayoría de las implementaciones de MapReduce le permiten ejecutar código arbitrario en un clúster. Pero un lenguaje de consulta declarativo es más adecuado para OLAP para ejecutar experimentos rápidamente. Por ejemplo, Hadoop tiene Hive y Pig. También considere Cloudera Impala o Shark (obsoleto) para Spark, así como Spark SQL, Presto y Apache Drill. El rendimiento cuando se ejecutan tales tareas es muy subóptimo en comparación con los sistemas especializados, pero la latencia relativamente alta hace que sea poco realista utilizar estos sistemas como back-end para una interfaz web. -## ¿qué sucede si tengo un problema con las codificaciones al usar oracle a través de odbc? {#oracle-odbc-encodings} +## ¿Qué sucede si tengo un problema con las codificaciones al usar Oracle a través de ODBC? {#oracle-odbc-encodings} Si utiliza Oracle a través del controlador ODBC como fuente de diccionarios externos, debe establecer el valor `NLS_LANG` variable de entorno en `/etc/default/clickhouse`. Para obtener más información, consulte [Oracle NLS\_LANG Preguntas frecuentes](https://www.oracle.com/technetwork/products/globalization/nls-lang-099431.html). @@ -25,11 +25,11 @@ Si utiliza Oracle a través del controlador ODBC como fuente de diccionarios ext NLS_LANG=RUSSIAN_RUSSIA.UTF8 ``` -## Cómo Exporto Datos De ClickHouse a Un Archivo? {#how-to-export-to-file} +## Cómo exporto datos de ClickHouse a un archivo? {#how-to-export-to-file} -### Uso De La cláusula INTO OUTFILE {#using-into-outfile-clause} +### Uso de la cláusula INTO OUTFILE {#using-into-outfile-clause} -Añadir un [INTO OUTFILE](../query_language/select/#into-outfile-clause) cláusula a su consulta. +Añadir un [INTO OUTFILE](../sql-reference/statements/select/into-outfile.md#into-outfile-clause) cláusula a su consulta. Por ejemplo: @@ -37,7 +37,7 @@ Por ejemplo: SELECT * FROM table INTO OUTFILE 'file' ``` -De forma predeterminada, ClickHouse usa el [TabSeparated](../interfaces/formats.md#tabseparated) formato de datos de salida. Para seleccionar el [formato de datos](../interfaces/formats.md), utilizar el [Cláusula FORMAT](../query_language/select/#format-clause). +De forma predeterminada, ClickHouse usa el [TabSeparated](../interfaces/formats.md#tabseparated) formato de datos de salida. Para seleccionar el [formato de datos](../interfaces/formats.md), utilizar el [Cláusula FORMAT](../sql-reference/statements/select/format.md#format-clause). Por ejemplo: @@ -45,11 +45,11 @@ Por ejemplo: SELECT * FROM table INTO OUTFILE 'file' FORMAT CSV ``` -### Uso De Una Tabla De Motor De Archivo {#using-a-file-engine-table} +### Uso de una tabla de motor de archivo {#using-a-file-engine-table} Ver [File](../engines/table-engines/special/file.md). -### Uso De La redirección De línea De Comandos {#using-command-line-redirection} +### Uso de la redirección de línea de comandos {#using-command-line-redirection} ``` sql $ clickhouse-client --query "SELECT * from table" --format FormatName > result.txt diff --git a/docs/es/faq/index.md b/docs/es/faq/index.md index 05863c71f9a..a44dbb31e89 100644 --- a/docs/es/faq/index.md +++ b/docs/es/faq/index.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_folder_title: F.A.Q. toc_priority: 76 --- diff --git a/docs/es/getting-started/example-datasets/amplab-benchmark.md b/docs/es/getting-started/example-datasets/amplab-benchmark.md index 3f9ec5bef61..c8fa77953cb 100644 --- a/docs/es/getting-started/example-datasets/amplab-benchmark.md +++ b/docs/es/getting-started/example-datasets/amplab-benchmark.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 17 toc_title: Referencia de Big Data de AMPLab --- -# Referencia De Big Data De AMPLab {#amplab-big-data-benchmark} +# Referencia de Big Data de AMPLab {#amplab-big-data-benchmark} Ver https://amplab.cs.berkeley.edu/benchmark/ diff --git a/docs/es/getting-started/example-datasets/criteo.md b/docs/es/getting-started/example-datasets/criteo.md index d95ea234962..79203b0276d 100644 --- a/docs/es/getting-started/example-datasets/criteo.md +++ b/docs/es/getting-started/example-datasets/criteo.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 19 toc_title: Registros de clics de Terabyte de Criteo --- -# Terabyte De Registros De Clics De Criteo {#terabyte-of-click-logs-from-criteo} +# Terabyte de registros de clics de Criteo {#terabyte-of-click-logs-from-criteo} Descargue los datos de http://labs.criteo.com/downloads/download-terabyte-click-logs/ diff --git a/docs/es/getting-started/example-datasets/index.md b/docs/es/getting-started/example-datasets/index.md index 5877a749ec4..28e06987af1 100644 --- a/docs/es/getting-started/example-datasets/index.md +++ b/docs/es/getting-started/example-datasets/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa -toc_folder_title: Example Datasets +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: Datos De Ejemplo toc_priority: 12 toc_title: "Implantaci\xF3n" --- diff --git a/docs/es/getting-started/example-datasets/metrica.md b/docs/es/getting-started/example-datasets/metrica.md index 39c4bd17a68..e1c6a2a361e 100644 --- a/docs/es/getting-started/example-datasets/metrica.md +++ b/docs/es/getting-started/example-datasets/metrica.md @@ -1,17 +1,17 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa -toc_priority: 21 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_priority: 14 toc_title: El Yandex.Metrica Datos --- -# Yandex Anonimizado.Metrica Datos {#anonymized-yandex-metrica-data} +# Yandex anonimizado.Metrica Datos {#anonymized-yandex-metrica-data} El conjunto de datos consta de dos tablas que contienen datos anónimos sobre los hits (`hits_v1`) y visitas (`visits_v1`) el Yandex.Métrica. Puedes leer más sobre Yandex.Metrica en [Historial de ClickHouse](../../introduction/history.md) apartado. El conjunto de datos consta de dos tablas, cualquiera de ellas se puede descargar como `tsv.xz` o como particiones preparadas. Además, una versión extendida de la `hits` La tabla que contiene 100 millones de filas está disponible como TSV en https://clickhouse-datasets.s3.yandex.net/hits/tsv/hits\_100m\_obfuscated\_v1.tsv.xz y como particiones preparadas en https://clickhouse-datasets.s3.yandex.net/hits/partitions/hits\_100m\_obfuscated\_v1.tar.xz. -## Obtención De Tablas a Partir De Particiones Preparadas {#obtaining-tables-from-prepared-partitions} +## Obtención de tablas a partir de particiones preparadas {#obtaining-tables-from-prepared-partitions} Descargar e importar tabla de hits: @@ -33,7 +33,7 @@ sudo service clickhouse-server restart clickhouse-client --query "SELECT COUNT(*) FROM datasets.visits_v1" ``` -## Obtención De Tablas a Partir De Un Archivo TSV Comprimido {#obtaining-tables-from-compressed-tsv-file} +## Obtención de tablas a partir de un archivo TSV comprimido {#obtaining-tables-from-compressed-tsv-file} Descargar e importar hits desde un archivo TSV comprimido: @@ -63,7 +63,7 @@ clickhouse-client --query "OPTIMIZE TABLE datasets.visits_v1 FINAL" clickhouse-client --query "SELECT COUNT(*) FROM datasets.visits_v1" ``` -## Consultas De Ejemplo {#example-queries} +## Consultas de ejemplo {#example-queries} [Tutorial de ClickHouse](../../getting-started/tutorial.md) se basa en Yandex.El conjunto de datos de Metrica y la forma recomendada de comenzar con este conjunto de datos es simplemente pasar por el tutorial. diff --git a/docs/es/getting-started/example-datasets/nyc-taxi.md b/docs/es/getting-started/example-datasets/nyc-taxi.md index 067e9de337f..e7dfe0898fe 100644 --- a/docs/es/getting-started/example-datasets/nyc-taxi.md +++ b/docs/es/getting-started/example-datasets/nyc-taxi.md @@ -1,18 +1,18 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 16 toc_title: Datos de taxis de Nueva York --- -# Datos De Taxis De Nueva York {#new-york-taxi-data} +# Datos de taxis de Nueva York {#new-york-taxi-data} Este conjunto de datos se puede obtener de dos maneras: - importación de datos sin procesar - descarga de particiones preparadas -## Cómo Importar Los Datos Sin Procesar {#how-to-import-the-raw-data} +## Cómo importar los datos sin procesar {#how-to-import-the-raw-data} Consulte https://github.com/toddwschneider/nyc-taxi-data y http://tech.marksblogg.com/billion-nyc-taxi-rides-redshift.html para obtener la descripción de un conjunto de datos e instrucciones para descargar. @@ -282,7 +282,7 @@ SELECT formatReadableSize(sum(bytes)) FROM system.parts WHERE table = 'trips_mer Entre otras cosas, puede ejecutar la consulta OPTIMIZE en MergeTree. Pero no es necesario ya que todo estará bien sin él. -## Descarga De Prepared Partitions {#download-of-prepared-partitions} +## Descarga de Prepared Partitions {#download-of-prepared-partitions} ``` bash $ curl -O https://clickhouse-datasets.s3.yandex.net/trips_mergetree/partitions/trips_mergetree.tar @@ -295,7 +295,7 @@ $ clickhouse-client --query "select count(*) from datasets.trips_mergetree" !!! info "INFO" Si va a ejecutar las consultas que se describen a continuación, debe usar el nombre completo de la tabla, `datasets.trips_mergetree`. -## Resultados En Un Solo Servidor {#results-on-single-server} +## Resultados en un solo servidor {#results-on-single-server} Q1: diff --git a/docs/es/getting-started/example-datasets/ontime.md b/docs/es/getting-started/example-datasets/ontime.md index 78403fce0f7..b0662ef8b53 100644 --- a/docs/es/getting-started/example-datasets/ontime.md +++ b/docs/es/getting-started/example-datasets/ontime.md @@ -1,18 +1,18 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 15 toc_title: A tiempo --- -# A Tiempo {#ontime} +# A tiempo {#ontime} Este conjunto de datos se puede obtener de dos maneras: - importación de datos sin procesar - descarga de particiones preparadas -## Importar Desde Datos Sin Procesar {#import-from-raw-data} +## Importar desde datos sin procesar {#import-from-raw-data} Descarga de datos: @@ -153,7 +153,7 @@ Carga de datos: $ for i in *.zip; do echo $i; unzip -cq $i '*.csv' | sed 's/\.00//g' | clickhouse-client --host=example-perftest01j --query="INSERT INTO ontime FORMAT CSVWithNames"; done ``` -## Descarga De Prepared Partitions {#download-of-prepared-partitions} +## Descarga de Prepared Partitions {#download-of-prepared-partitions} ``` bash $ curl -O https://clickhouse-datasets.s3.yandex.net/ontime/partitions/ontime.tar @@ -257,7 +257,7 @@ GROUP BY Carrier ORDER BY c3 DESC ``` -¿por qué? la solicitud anterior de una gama más amplia de años, 2000-2008 +¿Por qué? La solicitud anterior de una gama más amplia de años, 2000-2008 ``` sql SELECT Carrier, c, c2, c*100/c2 as c3 @@ -326,7 +326,7 @@ GROUP BY Year ORDER BY Year; ``` -¿por qué? los destinos más populares por el número de ciudades conectadas directamente para varios rangos de año +¿Por qué? Los destinos más populares por el número de ciudades conectadas directamente para varios rangos de año ``` sql SELECT DestCityName, uniqExact(OriginCityName) AS u diff --git a/docs/es/getting-started/example-datasets/star-schema.md b/docs/es/getting-started/example-datasets/star-schema.md index ebc18b94fd6..43f878eb205 100644 --- a/docs/es/getting-started/example-datasets/star-schema.md +++ b/docs/es/getting-started/example-datasets/star-schema.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 20 toc_title: Estrella Schema Benchmark --- diff --git a/docs/es/getting-started/example-datasets/wikistat.md b/docs/es/getting-started/example-datasets/wikistat.md index 938ed0c25d4..49d7263cdec 100644 --- a/docs/es/getting-started/example-datasets/wikistat.md +++ b/docs/es/getting-started/example-datasets/wikistat.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 18 toc_title: "Nombre de la red inal\xE1mbrica (SSID):" --- -# Nombre De La Red inalámbrica (SSID): {#wikistat} +# Nombre de la red inalámbrica (SSID): {#wikistat} Ver: http://dumps.wikimedia.org/other/pagecounts-raw/ diff --git a/docs/es/getting-started/index.md b/docs/es/getting-started/index.md index 10964e22817..681c2017ac1 100644 --- a/docs/es/getting-started/index.md +++ b/docs/es/getting-started/index.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa -toc_folder_title: Getting Started +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: Primeros pasos toc_hidden: true toc_priority: 8 toc_title: oculto --- -# Primeros Pasos {#getting-started} +# Primeros pasos {#getting-started} Si eres nuevo en ClickHouse y quieres tener una sensación práctica de su rendimiento, antes que nada, debes pasar por el [proceso de instalación](install.md). Después de eso puedes: diff --git a/docs/es/getting-started/install.md b/docs/es/getting-started/install.md index 6e6feee8464..83f5fff8af2 100644 --- a/docs/es/getting-started/install.md +++ b/docs/es/getting-started/install.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 11 toc_title: "Instalaci\xF3n" --- # Instalación {#installation} -## Requisitos Del Sistema {#system-requirements} +## Requisitos del sistema {#system-requirements} ClickHouse puede ejecutarse en cualquier Linux, FreeBSD o Mac OS X con arquitectura de CPU x86\_64, AArch64 o PowerPC64LE. @@ -19,9 +19,9 @@ $ grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not Para ejecutar ClickHouse en procesadores que no admiten SSE 4.2 o tienen arquitectura AArch64 o PowerPC64LE, debe [construir ClickHouse a partir de fuentes](#from-sources) con los ajustes de configuración adecuados. -## Opciones De instalación Disponibles {#available-installation-options} +## Opciones de instalación disponibles {#available-installation-options} -### De Paquetes DEB {#install-from-deb-packages} +### De paquetes DEB {#install-from-deb-packages} Se recomienda utilizar pre-compilado oficial `deb` Paquetes para Debian o Ubuntu. Ejecute estos comandos para instalar paquetes: @@ -31,7 +31,7 @@ Se recomienda utilizar pre-compilado oficial `deb` Paquetes para Debian o Ubuntu Si desea utilizar la versión más reciente, reemplace `stable` con `testing` (esto se recomienda para sus entornos de prueba). -También puede descargar e instalar paquetes manualmente desde aquí: https://repo.clickhouse.tech/deb/stable/main/. +También puede descargar e instalar paquetes manualmente desde [aqui](https://repo.clickhouse.tech/deb/stable/main/). #### Paquete {#packages} @@ -40,7 +40,7 @@ También puede descargar e instalar paquetes manualmente desde aquí: https://re - `clickhouse-client` — Creates a symbolic link for `clickhouse-client` y otras herramientas relacionadas con el cliente. e instala los archivos de configuración del cliente. - `clickhouse-common-static-dbg` — Installs ClickHouse compiled binary files with debug info. -### De Paquetes RPM {#from-rpm-packages} +### De paquetes RPM {#from-rpm-packages} Se recomienda utilizar pre-compilado oficial `rpm` También puede utilizar los paquetes para CentOS, RedHat y todas las demás distribuciones de Linux basadas en rpm. @@ -60,9 +60,9 @@ A continuación, ejecute estos comandos para instalar paquetes: sudo yum install clickhouse-server clickhouse-client ``` -También puede descargar e instalar paquetes manualmente desde aquí: https://repo.casa de clic.tecnología / rpm / estable / x86\_64. +También puede descargar e instalar paquetes manualmente desde [aqui](https://repo.clickhouse.tech/rpm/stable/x86_64). -### De Archivos Tgz {#from-tgz-archives} +### De archivos Tgz {#from-tgz-archives} Se recomienda utilizar pre-compilado oficial `tgz` para todas las distribuciones de Linux, donde la instalación de `deb` o `rpm` paquetes no es posible. @@ -96,7 +96,7 @@ Para los entornos de producción, se recomienda utilizar las últimas `stable`-v Para ejecutar ClickHouse dentro de Docker, siga la guía en [Eje de acoplador](https://hub.docker.com/r/yandex/clickhouse-server/). Esas imágenes usan oficial `deb` paquetes dentro. -### De Fuentes {#from-sources} +### De fuentes {#from-sources} Para compilar manualmente ClickHouse, siga las instrucciones para [Linux](../development/build.md) o [Mac OS X](../development/build-osx.md). diff --git a/docs/es/getting-started/playground.md b/docs/es/getting-started/playground.md index f71ae4672ed..1ab7246e2d4 100644 --- a/docs/es/getting-started/playground.md +++ b/docs/es/getting-started/playground.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 14 toc_title: Infantil --- -# Zona De Juegos ClickHouse {#clickhouse-playground} +# Zona de juegos ClickHouse {#clickhouse-playground} [Zona de juegos ClickHouse](https://play.clickhouse.tech?file=welcome) permite a las personas experimentar con ClickHouse ejecutando consultas al instante, sin configurar su servidor o clúster. Varios conjuntos de datos de ejemplo están disponibles en Playground, así como consultas de ejemplo que muestran las características de ClickHouse. diff --git a/docs/es/getting-started/tutorial.md b/docs/es/getting-started/tutorial.md index 43e2a88f0fd..ccc07e50468 100644 --- a/docs/es/getting-started/tutorial.md +++ b/docs/es/getting-started/tutorial.md @@ -1,17 +1,17 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 12 toc_title: Tutorial --- -# Tutorial De ClickHouse {#clickhouse-tutorial} +# Tutorial de ClickHouse {#clickhouse-tutorial} -## Qué Esperar De Este Tutorial? {#what-to-expect-from-this-tutorial} +## Qué Esperar de Este Tutorial? {#what-to-expect-from-this-tutorial} Al pasar por este tutorial, aprenderá cómo configurar un clúster de ClickHouse simple. Será pequeño, pero tolerante a fallos y escalable. Luego usaremos uno de los conjuntos de datos de ejemplo para llenarlo con datos y ejecutar algunas consultas de demostración. -## Configuración De Nodo único {#single-node-setup} +## Configuración de nodo único {#single-node-setup} Para posponer las complejidades de un entorno distribuido, comenzaremos con la implementación de ClickHouse en un único servidor o máquina virtual. ClickHouse generalmente se instala desde [deb](install.md#install-from-deb-packages) o [RPM](install.md#from-rpm-packages) paquetes, pero hay [alternativa](install.md#from-docker-image) para los sistemas operativos que no los admiten. @@ -21,7 +21,7 @@ Por ejemplo, ha elegido `deb` paquetes y ejecutado: {% include 'install/deb.sh' %} ``` -¿qué tenemos en los paquetes que tengo instalados: +¿Qué tenemos en los paquetes que tengo instalados: - `clickhouse-client` el paquete contiene [Casa de clics-cliente](../interfaces/cli.md) aplicación, cliente interactivo de la consola ClickHouse. - `clickhouse-common` El paquete contiene un archivo ejecutable ClickHouse. @@ -48,6 +48,7 @@ Una vez que el `clickhouse-server` está en funcionamiento, podemos usar `clickh
Consejos rápidos para clickhouse-cliente + Modo interactivo: ``` bash @@ -79,11 +80,11 @@ clickhouse-client --query='INSERT INTO table FORMAT TabSeparated' < data.tsv
-## Importar Conjunto De Datos De Muestra {#import-sample-dataset} +## Importar conjunto de datos de muestra {#import-sample-dataset} Ahora es el momento de llenar nuestro servidor ClickHouse con algunos datos de muestra. En este tutorial, usaremos los datos anónimos de Yandex.Metrica, el primer servicio que ejecuta ClickHouse en forma de producción antes de que se convirtiera en código abierto (más sobre eso en [sección de historia](../introduction/history.md)). Hay [múltiples formas de importar Yandex.Conjunto de datos de Metrica](example-datasets/metrica.md), y por el bien del tutorial, iremos con el más realista. -### Descargar y Extraer Datos De Tabla {#download-and-extract-table-data} +### Descargar y extraer datos de tabla {#download-and-extract-table-data} ``` bash curl https://clickhouse-datasets.s3.yandex.net/hits/tsv/hits_v1.tsv.xz | unxz --threads=`nproc` > hits_v1.tsv @@ -92,7 +93,7 @@ curl https://clickhouse-datasets.s3.yandex.net/visits/tsv/visits_v1.tsv.xz | unx Los archivos extraídos tienen un tamaño de aproximadamente 10 GB. -### Crear Tablas {#create-tables} +### Crear tablas {#create-tables} Como en la mayoría de los sistemas de gestión de bases de datos, ClickHouse agrupa lógicamente las tablas en “databases”. Hay un `default` base de datos, pero crearemos una nueva llamada `tutorial`: @@ -458,7 +459,7 @@ Puede ejecutar esas consultas utilizando el modo interactivo de `clickhouse-clie Como podemos ver, `hits_v1` utiliza el [motor básico MergeTree](../engines/table-engines/mergetree-family/mergetree.md), mientras que el `visits_v1` utiliza el [Derrumbar](../engines/table-engines/mergetree-family/collapsingmergetree.md) variante. -### Importar Datos {#import-data} +### Importar datos {#import-data} La importación de datos a ClickHouse se realiza a través de [INSERT INTO](../sql-reference/statements/insert-into.md) consulta como en muchas otras bases de datos SQL. Sin embargo, los datos generalmente se proporcionan en uno de los [Formatos de serialización compatibles](../interfaces/formats.md) en lugar de `VALUES` cláusula (que también es compatible). @@ -480,7 +481,7 @@ FORMAT TSV max_insert_block_size 1048576 0 "The maximum block size for insertion, if we control the creation of blocks for insertion." ``` -Opcionalmente se puede [OPTIMIZE](../query_language/misc/#misc_operations-optimize) las tablas después de la importación. Las tablas que están configuradas con un motor de la familia MergeTree siempre fusionan partes de datos en segundo plano para optimizar el almacenamiento de datos (o al menos verificar si tiene sentido). Estas consultas obligan al motor de tablas a realizar la optimización del almacenamiento en este momento en lugar de algún tiempo después: +Opcionalmente se puede [OPTIMIZE](../sql-reference/statements/misc.md#misc_operations-optimize) las tablas después de la importación. Las tablas que están configuradas con un motor de la familia MergeTree siempre fusionan partes de datos en segundo plano para optimizar el almacenamiento de datos (o al menos verificar si tiene sentido). Estas consultas obligan al motor de tablas a realizar la optimización del almacenamiento en este momento en lugar de algún tiempo después: ``` bash clickhouse-client --query "OPTIMIZE TABLE tutorial.hits_v1 FINAL" @@ -496,7 +497,7 @@ clickhouse-client --query "SELECT COUNT(*) FROM tutorial.hits_v1" clickhouse-client --query "SELECT COUNT(*) FROM tutorial.visits_v1" ``` -## Consultas De Ejemplo {#example-queries} +## Consultas de ejemplo {#example-queries} ``` sql SELECT @@ -518,7 +519,7 @@ FROM tutorial.visits_v1 WHERE (CounterID = 912887) AND (toYYYYMM(StartDate) = 201403) AND (domain(StartURL) = 'yandex.ru') ``` -## Implementación De clúster {#cluster-deployment} +## Implementación de clúster {#cluster-deployment} El clúster ClickHouse es un clúster homogéneo. Pasos para configurar: diff --git a/docs/es/guides/apply-catboost-model.md b/docs/es/guides/apply-catboost-model.md index 3f9039f68fd..b1fe50f3276 100644 --- a/docs/es/guides/apply-catboost-model.md +++ b/docs/es/guides/apply-catboost-model.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 41 toc_title: "Aplicaci\xF3n de modelos CatBoost" --- -# Aplicación De Un Modelo Catboost En ClickHouse {#applying-catboost-model-in-clickhouse} +# Aplicación de un modelo Catboost en ClickHouse {#applying-catboost-model-in-clickhouse} [CatBoost](https://catboost.ai) es una biblioteca de impulso de gradiente libre y de código abierto desarrollada en [Yandex](https://yandex.com/company/) para el aprendizaje automático. diff --git a/docs/es/guides/index.md b/docs/es/guides/index.md index e23b26d09ce..c8332ac7846 100644 --- a/docs/es/guides/index.md +++ b/docs/es/guides/index.md @@ -1,12 +1,12 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa -toc_folder_title: Guides +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: Guiar toc_priority: 38 toc_title: "Descripci\xF3n" --- -# Guías De ClickHouse {#clickhouse-guides} +# Guías de ClickHouse {#clickhouse-guides} Lista de instrucciones detalladas paso a paso que ayudan a resolver varias tareas usando ClickHouse: diff --git a/docs/es/index.md b/docs/es/index.md index 73432517622..9a2918ff7c8 100644 --- a/docs/es/index.md +++ b/docs/es/index.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa -toc_priority: 3 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_priority: 0 toc_title: "Descripci\xF3n" --- -# ¿qué es clickhouse? {#what-is-clickhouse} +# ¿Qué es ClickHouse? {#what-is-clickhouse} ClickHouse es un sistema de gestión de bases de datos orientado a columnas (DBMS) para el procesamiento analítico en línea de consultas (OLAP). @@ -40,7 +40,7 @@ Different orders for storing data are better suited to different scenarios. The Cuanto mayor sea la carga en el sistema, más importante es personalizar el sistema configurado para que coincida con los requisitos del escenario de uso, y más fino será esta personalización. No existe un sistema que sea igualmente adecuado para escenarios significativamente diferentes. Si un sistema es adaptable a un amplio conjunto de escenarios, bajo una carga alta, el sistema manejará todos los escenarios igualmente mal, o funcionará bien para solo uno o algunos de los escenarios posibles. -## Propiedades Clave Del Escenario OLAP {#key-properties-of-olap-scenario} +## Propiedades clave del escenario OLAP {#key-properties-of-olap-scenario} - La gran mayoría de las solicitudes son para acceso de lectura. - Los datos se actualizan en lotes bastante grandes (\> 1000 filas), no por filas individuales; o no se actualiza en absoluto. @@ -58,7 +58,7 @@ Cuanto mayor sea la carga en el sistema, más importante es personalizar el sist Es fácil ver que el escenario OLAP es muy diferente de otros escenarios populares (como el acceso OLTP o Key-Value). Por lo tanto, no tiene sentido intentar usar OLTP o una base de datos de valor clave para procesar consultas analíticas si desea obtener un rendimiento decente. Por ejemplo, si intenta usar MongoDB o Redis para análisis, obtendrá un rendimiento muy bajo en comparación con las bases de datos OLAP. -## Por qué Las Bases De Datos Orientadas a Columnas Funcionan Mejor En El Escenario OLAP {#why-column-oriented-databases-work-better-in-the-olap-scenario} +## Por qué las bases de datos orientadas a columnas funcionan mejor en el escenario OLAP {#why-column-oriented-databases-work-better-in-the-olap-scenario} Las bases de datos orientadas a columnas son más adecuadas para los escenarios OLAP: son al menos 100 veces más rápidas en el procesamiento de la mayoría de las consultas. Las razones se explican en detalle a continuación, pero el hecho es más fácil de demostrar visualmente: diff --git a/docs/es/interfaces/cli.md b/docs/es/interfaces/cli.md index 625827df333..1e4852be53f 100644 --- a/docs/es/interfaces/cli.md +++ b/docs/es/interfaces/cli.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 17 toc_title: "Cliente de l\xEDnea de comandos" --- -# Cliente De línea De Comandos {#command-line-client} +# Cliente de línea de comandos {#command-line-client} ClickHouse proporciona un cliente de línea de comandos nativo: `clickhouse-client`. El cliente admite opciones de línea de comandos y archivos de configuración. Para obtener más información, consulte [Configuración](#interfaces_cli_configuration). @@ -72,7 +72,7 @@ Puede cancelar una consulta larga presionando Ctrl + C. Sin embargo, aún tendr El cliente de línea de comandos permite pasar datos externos (tablas temporales externas) para consultar. Para obtener más información, consulte la sección “External data for query processing”. -### Consultas Con parámetros {#cli-queries-with-parameters} +### Consultas con parámetros {#cli-queries-with-parameters} Puede crear una consulta con parámetros y pasarles valores desde la aplicación cliente. Esto permite evitar formatear consultas con valores dinámicos específicos en el lado del cliente. Por ejemplo: @@ -80,7 +80,7 @@ Puede crear una consulta con parámetros y pasarles valores desde la aplicación $ clickhouse-client --param_parName="[1, 2]" -q "SELECT * FROM table WHERE a = {parName:Array(UInt16)}" ``` -#### Sintaxis De Consulta {#cli-queries-with-parameters-syntax} +#### Sintaxis de consulta {#cli-queries-with-parameters-syntax} Formatee una consulta como de costumbre, luego coloque los valores que desea pasar de los parámetros de la aplicación a la consulta entre llaves en el siguiente formato: @@ -109,7 +109,7 @@ Puede pasar parámetros a `clickhouse-client` (todos los parámetros tienen un v Los valores de los archivos de configuración anulan los valores predeterminados. -### Opciones De línea De Comandos {#command-line-options} +### Opciones de línea de comandos {#command-line-options} - `--host, -h` -– The server name, ‘localhost’ predeterminada. Puede utilizar el nombre o la dirección IPv4 o IPv6. - `--port` – The port to connect to. Default value: 9000. Note that the HTTP interface and the native interface use different ports. @@ -127,7 +127,7 @@ Puede pasar parámetros a `clickhouse-client` (todos los parámetros tienen un v - `--secure` – If specified, will connect to server over secure connection. - `--param_` — Value for a [consulta con parámetros](#cli-queries-with-parameters). -### Archivos De configuración {#configuration_files} +### Archivos de configuración {#configuration_files} `clickhouse-client` utiliza el primer archivo existente de los siguientes: diff --git a/docs/es/interfaces/cpp.md b/docs/es/interfaces/cpp.md index 91630ea95af..bc5dc3dbc24 100644 --- a/docs/es/interfaces/cpp.md +++ b/docs/es/interfaces/cpp.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 24 toc_title: Biblioteca de clientes de C++ --- -# Biblioteca De Clientes De C++ {#c-client-library} +# Biblioteca de clientes de C++ {#c-client-library} Ver README en [Bienvenidos](https://github.com/ClickHouse/clickhouse-cpp) repositorio. diff --git a/docs/es/interfaces/formats.md b/docs/es/interfaces/formats.md index c6a7a93f0f7..f84c78d6e0b 100644 --- a/docs/es/interfaces/formats.md +++ b/docs/es/interfaces/formats.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 21 toc_title: Formatos de entrada y salida --- -# Formatos Para Datos De Entrada y Salida {#formats} +# Formatos para datos de entrada y salida {#formats} ClickHouse puede aceptar y devolver datos en varios formatos. Se puede utilizar un formato admitido para la entrada para analizar los datos proporcionados a `INSERT`s, para llevar a cabo `SELECT`s de una tabla respaldada por archivos como File, URL o HDFS, o para leer un diccionario externo. Se puede utilizar un formato compatible con la salida para organizar el resultados de un `SELECT`, y realizar `INSERT`s en una tabla respaldada por archivos. @@ -78,7 +78,7 @@ SELECT EventDate, count() AS c FROM test.hits GROUP BY EventDate WITH TOTALS ORD 2014-03-23 1406958 ``` -### Formato De Datos {#data-formatting} +### Formato de datos {#data-formatting} Los números enteros se escriben en forma decimal. Los números pueden contener un extra “+” carácter al principio (ignorado al analizar y no grabado al formatear). Los números no negativos no pueden contener el signo negativo. Al leer, se permite analizar una cadena vacía como cero, o (para tipos con signo) una cadena que consiste en solo un signo menos como cero. Los números que no encajan en el tipo de datos correspondiente se pueden analizar como un número diferente, sin un mensaje de error. @@ -317,7 +317,7 @@ format_template_resultset = '/some/path/resultset.format', format_template_row = ## TSKV {#tskv} -Similar a TabSeparated, pero las salidas de un valor en nombre=valor de formato. Los nombres se escapó de la misma manera como en TabSeparated formato, y el símbolo = es también escapó. +Similar a TabSeparated , pero genera un valor en formato name=value . Los nombres se escapan de la misma manera que en el formato TabSeparated, y el símbolo = también se escapa. ``` text SearchPhrase= count()=8267016 @@ -526,7 +526,7 @@ Al usar este formato, ClickHouse genera filas como objetos JSON separados, delim Al insertar los datos, debe proporcionar un objeto JSON independiente para cada fila. -### Insertar Datos {#inserting-data} +### Insertar datos {#inserting-data} ``` sql INSERT INTO UserActivity FORMAT JSONEachRow {"PageViews":5, "UserID":"4324182021466249494", "Duration":146,"Sign":-1} {"UserID":"4324182021466249494","PageViews":6,"Duration":185,"Sign":1} @@ -561,7 +561,7 @@ CREATE TABLE IF NOT EXISTS example_table !!! note "Advertencia" Al insertar datos con `insert_sample_with_metadata = 1`, ClickHouse consume más recursos computacionales, en comparación con la inserción con `insert_sample_with_metadata = 0`. -### Selección De Datos {#selecting-data} +### Selección de datos {#selecting-data} Considere el `UserActivity` tabla como un ejemplo: @@ -584,7 +584,7 @@ A diferencia de la [JSON](#json) formato, no hay sustitución de secuencias UTF- !!! note "Nota" Cualquier conjunto de bytes se puede generar en las cadenas. Utilice el `JSONEachRow` si está seguro de que los datos de la tabla se pueden formatear como JSON sin perder ninguna información. -### Uso De Estructuras Anidadas {#jsoneachrow-nested} +### Uso de estructuras anidadas {#jsoneachrow-nested} Si tienes una mesa con [Anidar](../sql-reference/data-types/nested-data-structures/nested.md) columnas de tipo de datos, puede insertar datos JSON con la misma estructura. Habilite esta función con el [Entrada\_format\_import\_nested\_json](../operations/settings/settings.md#settings-input_format_import_nested_json) configuración. @@ -645,7 +645,7 @@ SELECT * FROM json_each_row_nested ## Nativo {#native} -El formato más eficiente. Los datos son escritos y leídos por bloques en formato binario. Para cada bloque, el número de filas, número de columnas, nombres y tipos de columnas y partes de columnas de este bloque se registran una tras otra. En otras palabras, este formato es “columnar” – it doesn’t convert columns to rows. This is the format used in the native interface for interaction between servers, for using the command-line client, and for C++ clients. +El formato más eficiente. Los datos son escritos y leídos por bloques en formato binario. Para cada bloque, el número de filas, número de columnas, nombres y tipos de columnas y partes de columnas de este bloque se registran una tras otra. En otras palabras, este formato es “columnar” – it doesn't convert columns to rows. This is the format used in the native interface for interaction between servers, for using the command-line client, and for C++ clients. Puede utilizar este formato para generar rápidamente volcados que sólo pueden ser leídos por el DBMS de ClickHouse. No tiene sentido trabajar con este formato usted mismo. @@ -786,7 +786,7 @@ Ver también: [input\_format\_values\_interpret\_expressions](../operations/sett ## Vertical {#vertical} -Imprime cada valor en una línea independiente con el nombre de la columna especificada. Este formato es conveniente para imprimir solo una o varias filas si cada fila consta de un gran número de columnas. +Imprime cada valor en una línea independiente con el nombre de columna especificado. Este formato es conveniente para imprimir solo una o varias filas si cada fila consta de un gran número de columnas. [NULL](../sql-reference/syntax.md) se emite como `ᴺᵁᴸᴸ`. @@ -897,9 +897,9 @@ Las matrices se emiten como `HelloWorld...?param_id=2¶m_phrase=test" -d "SELECT * FROM table WHERE int_column = {id:UInt8} and string_column = {phrase:String}" ``` -## Interfaz HTTP Predefinida {#predefined_http_interface} +## Interfaz HTTP predefinida {#predefined_http_interface} ClickHouse admite consultas específicas a través de la interfaz HTTP. Por ejemplo, puede escribir datos en una tabla de la siguiente manera: @@ -303,13 +303,16 @@ Ejemplo: ``` xml - - /metrics - GET - + + /predefined_query + POST,GET + + predefined_query_handler SELECT * FROM system.metrics LIMIT 5 FORMAT Template SETTINGS format_template_resultset = 'prometheus_template_output_format_resultset', format_template_row = 'prometheus_template_output_format_row', format_template_rows_between_delimiter = '\n' - - + + + ... + ... ``` @@ -318,21 +321,23 @@ Ejemplo: ``` bash -curl -vvv 'http://localhost:8123/metrics' +$ curl -v 'http://localhost:8123/predefined_query' * Trying ::1... * Connected to localhost (::1) port 8123 (#0) -> GET /metrics HTTP/1.1 +> GET /predefined_query HTTP/1.1 > Host: localhost:8123 > User-Agent: curl/7.47.0 > Accept: */* > < HTTP/1.1 200 OK -< Date: Wed, 27 Nov 2019 08:54:25 GMT +< Date: Tue, 28 Apr 2020 08:52:56 GMT < Connection: Keep-Alive < Content-Type: text/plain; charset=UTF-8 -< X-ClickHouse-Server-Display-Name: i-tl62qd0o +< X-ClickHouse-Server-Display-Name: i-mloy5trc < Transfer-Encoding: chunked -< X-ClickHouse-Query-Id: f39235f6-6ed7-488c-ae07-c7ceafb960f6 +< X-ClickHouse-Query-Id: 96fe0052-01e6-43ce-b12a-6b7370de6e8a +< X-ClickHouse-Format: Template +< X-ClickHouse-Timezone: Asia/Shanghai < Keep-Alive: timeout=3 < X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} < @@ -356,117 +361,62 @@ curl -vvv 'http://localhost:8123/metrics' # TYPE "ReplicatedSend" counter "ReplicatedSend" 0 +* Connection #0 to host localhost left intact + + * Connection #0 to host localhost left intact ``` -Como puede ver en el ejemplo, si `` está configurado en la configuración.archivo xml, ClickHouse coincidirá con las solicitudes HTTP recibidas con el tipo predefinido en ``, entonces ClickHouse ejecutará la consulta predefinida correspondiente si la coincidencia es exitosa. +Como puede ver en el ejemplo, si `` está configurado en la configuración.archivo xml y `` puede contener muchos `s`. ClickHouse coincidirá con las solicitudes HTTP recibidas con el tipo predefinido en `` y el primer emparejado ejecuta el controlador. Luego, ClickHouse ejecutará la consulta predefinida correspondiente si la coincidencia es exitosa. -Ahora `` puede configurar ``, ``, ``, `` y `` . +> Ahora `` puede configurar ``, ``, ``,``: +> `` es responsable de hacer coincidir la parte del método de la solicitud HTTP. `` se ajusta plenamente a la definición de [método](https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods) en el protocolo HTTP. Es una configuración opcional. Si no está definido en el archivo de configuración, no coincide con la parte del método de la solicitud HTTP. +> +> `` es responsable de hacer coincidir la parte url de la solicitud HTTP. Es compatible con [RE2](https://github.com/google/re2)expresiones regulares. Es una configuración opcional. Si no está definido en el archivo de configuración, no coincide con la parte url de la solicitud HTTP. +> +> `` es responsable de hacer coincidir la parte del encabezado de la solicitud HTTP. Es compatible con las expresiones regulares de RE2. Es una configuración opcional. Si no está definido en el archivo de configuración, no coincide con la parte de encabezado de la solicitud HTTP. +> +> `` contiene la parte de procesamiento principal. Ahora `` puede configurar ``, ``, ``, ``, ``, ``. +> \> `` Actualmente soporta tres tipos: **Dirección de correo electrónico**, **Nombre de la red inalámbrica (SSID):**, **estática**. +> \> +> \> `` - utilizar con el tipo predefined\_query\_handler, ejecuta la consulta cuando se llama al controlador. +> \> +> \> `` - utilizar con el tipo dynamic\_query\_handler, extrae y ejecuta el valor correspondiente al `` valor en parámetros de solicitud HTTP. +> \> +> \> `` - uso con tipo estático, código de estado de respuesta. +> \> +> \> `` - uso con tipo estático, respuesta [tipo de contenido](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type). +> \> +> \> `` - uso con tipo estático, contenido de respuesta enviado al cliente, cuando se usa el prefijo ‘file://’ o ‘config://’, encontrar el contenido del archivo o configuración enviar al cliente. -## root\_handler {#root_handler} +A continuación están los métodos de configuración para los diferentes ``. -`` devuelve el contenido especificado para la solicitud de ruta de acceso raíz. El contenido devuelto específico se configura mediante `http_server_default_response` en la configuración.XML. si no se especifica, devolver **Ok.** +## Dirección de correo electrónico {#predefined_query_handler} -`http_server_default_response` no está definido y se envía una solicitud HTTP a ClickHouse. El resultado es el siguiente: +`` admite la configuración de valores Settings y query\_params. Puede configurar `` en el tipo de ``. -``` xml - - - -``` - - $ curl 'http://localhost:8123' - Ok. - -`http_server_default_response` se define y se envía una solicitud HTTP a ClickHouse. El resultado es el siguiente: - -``` xml -
]]> - - - - -``` - - $ curl 'http://localhost:8123' -
% - -## Método De codificación De Datos: {#ping_handler} - -`` se puede utilizar para sondear el estado del servidor ClickHouse actual. Cuando el servidor HTTP ClickHouse es normal, acceder a ClickHouse a través de `` volverá **Ok.**. - -Ejemplo: - -``` xml - - /ping - -``` - -``` bash -$ curl 'http://localhost:8123/ping' -Ok. -``` - -## Sistema Abierto {#replicas_status_handler} - -`` se utiliza para detectar el estado de la réplica y el nodo de retorno **Ok.** si el nodo de réplica no tiene retraso. Si hay un retraso, devuelva el retraso específico. El valor de `` admite personalización. Si no especifica ``, Configuración predeterminada de ClickHouse `` ser **/replicas\_status**. - -Ejemplo: - -``` xml - - /replicas_status - -``` - -Ningún caso del retraso: - -``` bash -$ curl 'http://localhost:8123/replicas_status' -Ok. -``` - -Caso retrasado: - -``` bash -$ curl 'http://localhost:8123/replicas_status' -db.stats: Absolute delay: 22. Relative delay: 22. -``` - -## Dirección De Correo electrónico {#predefined_query_handler} - -Puede configurar ``, ``, `` y `` en ``. - -`` es responsable de hacer coincidir la parte del método de la solicitud HTTP. `` se ajusta plenamente a la definición de [método](https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods) en el protocolo HTTP. Es una configuración opcional. Si no está definido en el archivo de configuración, no coincide con la parte del método de la solicitud HTTP - -`` es responsable de hacer coincidir la parte url de la solicitud HTTP. Es compatible con [RE2](https://github.com/google/re2)expresiones regulares. Es una configuración opcional. Si no está definido en el archivo de configuración, no coincide con la parte url de la solicitud HTTP - -`` es responsable de hacer coincidir la parte del encabezado de la solicitud HTTP. Es compatible con las expresiones regulares de RE2. Es una configuración opcional. Si no está definido en el archivo de configuración, no coincide con la parte de encabezado de la solicitud HTTP - -`` valor es una consulta predefinida de ``, que es ejecutado por ClickHouse cuando se hace coincidir una solicitud HTTP y se devuelve el resultado de la consulta. Es una configuración imprescindible. - -`` admite la configuración de valores Settings y query\_params. +`` valor es una consulta predefinida de ``, que es ejecutado por ClickHouse cuando se hace coincidir una solicitud HTTP y se devuelve el resultado de la consulta. Es una configuración imprescindible. En el ejemplo siguiente se definen los valores de `max_threads` y `max_alter_threads` configuración, a continuación, consulta la tabla del sistema para comprobar si estos ajustes se han establecido correctamente. Ejemplo: ``` xml - - + + + [^/]+)(/(?P[^/]+))?]]> GET TEST_HEADER_VALUE [^/]+)(/(?P[^/]+))?]]> - [^/]+)(/(?P[^/]+))?]]> - + + predefined_query_handler SELECT value FROM system.settings WHERE name = {name_1:String} SELECT name, value FROM system.settings WHERE name = {name_2:String} - - - + + + ``` ``` bash @@ -475,37 +425,193 @@ $ curl -H 'XXX:TEST_HEADER_VALUE' -H 'PARAMS_XXX:max_threads' 'http://localhost: max_alter_threads 2 ``` -!!! note "Nota" - En uno ``, una `` sólo es compatible con uno `` de un tipo de plaquita. +!!! note "precaución" + En uno `` sólo es compatible con uno `` de un tipo de plaquita. -## Nombre De La Red inalámbrica (SSID): {#dynamic_query_handler} +## Nombre de la red inalámbrica (SSID): {#dynamic_query_handler} -`` que `` aumentar `` . +En ``, consulta se escribe en forma de param de la solicitud HTTP. La diferencia es que en ``, consulta se escribe en el archivo de configuración. Puede configurar `` en ``. -ClickHouse extrae y ejecuta el valor correspondiente al `` valor en la url de la petición HTTP. -Configuración predeterminada de ClickHouse `` ser `/query` . Es una configuración opcional. Si no hay una definición en el archivo de configuración, el parámetro no se pasa. +ClickHouse extrae y ejecuta el valor correspondiente al `` valor en la url de la solicitud HTTP. El valor predeterminado de `` ser `/query` . Es una configuración opcional. Si no hay una definición en el archivo de configuración, el parámetro no se pasa. Para experimentar con esta funcionalidad, el ejemplo define los valores de max\_threads y max\_alter\_threads y consulta si la configuración se estableció correctamente. -La diferencia es que en ``, consulta se escribe en el archivo de configuración. Pero en ``, consulta se escribe en forma de param de la solicitud HTTP. Ejemplo: ``` xml - - - - TEST_HEADER_VALUE_DYNAMIC - [^/]+)(/(?P[^/]+))?]]> - + + + + TEST_HEADER_VALUE_DYNAMIC + + dynamic_query_handler query_param - - + + + ``` ``` bash -$ curl -H 'XXX:TEST_HEADER_VALUE_DYNAMIC' -H 'PARAMS_XXX:max_threads' 'http://localhost:8123/?query_param=SELECT%20value%20FROM%20system.settings%20where%20name%20=%20%7Bname_1:String%7D%20OR%20name%20=%20%7Bname_2:String%7D&max_threads=1&max_alter_threads=2¶m_name_2=max_alter_threads' -1 -2 +$ curl -H 'XXX:TEST_HEADER_VALUE_DYNAMIC' 'http://localhost:8123/own?max_threads=1&max_alter_threads=2¶m_name_1=max_threads¶m_name_2=max_alter_threads&query_param=SELECT%20name,value%20FROM%20system.settings%20where%20name%20=%20%7Bname_1:String%7D%20OR%20name%20=%20%7Bname_2:String%7D' +max_threads 1 +max_alter_threads 2 +``` + +## estática {#static} + +`` puede volver [Content\_type](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type), [estatus](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status) y response\_content. response\_content puede devolver el contenido especificado + +Ejemplo: + +Devuelve un mensaje. + +``` xml + + + GET + xxx + /hi + + static + 402 + text/html; charset=UTF-8 + Say Hi! + + + +``` + +``` bash +$ curl -vv -H 'XXX:xxx' 'http://localhost:8123/hi' +* Trying ::1... +* Connected to localhost (::1) port 8123 (#0) +> GET /hi HTTP/1.1 +> Host: localhost:8123 +> User-Agent: curl/7.47.0 +> Accept: */* +> XXX:xxx +> +< HTTP/1.1 402 Payment Required +< Date: Wed, 29 Apr 2020 03:51:26 GMT +< Connection: Keep-Alive +< Content-Type: text/html; charset=UTF-8 +< Transfer-Encoding: chunked +< Keep-Alive: timeout=3 +< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} +< +* Connection #0 to host localhost left intact +Say Hi!% +``` + +Busque el contenido de la configuración enviada al cliente. + +``` xml +
]]> + + + + GET + xxx + /get_config_static_handler + + static + config://get_config_static_handler + + + +``` + +``` bash +$ curl -v -H 'XXX:xxx' 'http://localhost:8123/get_config_static_handler' +* Trying ::1... +* Connected to localhost (::1) port 8123 (#0) +> GET /get_config_static_handler HTTP/1.1 +> Host: localhost:8123 +> User-Agent: curl/7.47.0 +> Accept: */* +> XXX:xxx +> +< HTTP/1.1 200 OK +< Date: Wed, 29 Apr 2020 04:01:24 GMT +< Connection: Keep-Alive +< Content-Type: text/plain; charset=UTF-8 +< Transfer-Encoding: chunked +< Keep-Alive: timeout=3 +< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} +< +* Connection #0 to host localhost left intact +
% +``` + +Encuentra el contenido del archivo enviado al cliente. + +``` xml + + + GET + xxx + /get_absolute_path_static_handler + + static + text/html; charset=UTF-8 + file:///absolute_path_file.html + + + + GET + xxx + /get_relative_path_static_handler + + static + text/html; charset=UTF-8 + file://./relative_path_file.html + + + +``` + +``` bash +$ user_files_path='/var/lib/clickhouse/user_files' +$ sudo echo "Relative Path File" > $user_files_path/relative_path_file.html +$ sudo echo "Absolute Path File" > $user_files_path/absolute_path_file.html +$ curl -vv -H 'XXX:xxx' 'http://localhost:8123/get_absolute_path_static_handler' +* Trying ::1... +* Connected to localhost (::1) port 8123 (#0) +> GET /get_absolute_path_static_handler HTTP/1.1 +> Host: localhost:8123 +> User-Agent: curl/7.47.0 +> Accept: */* +> XXX:xxx +> +< HTTP/1.1 200 OK +< Date: Wed, 29 Apr 2020 04:18:16 GMT +< Connection: Keep-Alive +< Content-Type: text/html; charset=UTF-8 +< Transfer-Encoding: chunked +< Keep-Alive: timeout=3 +< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} +< +Absolute Path File +* Connection #0 to host localhost left intact +$ curl -vv -H 'XXX:xxx' 'http://localhost:8123/get_relative_path_static_handler' +* Trying ::1... +* Connected to localhost (::1) port 8123 (#0) +> GET /get_relative_path_static_handler HTTP/1.1 +> Host: localhost:8123 +> User-Agent: curl/7.47.0 +> Accept: */* +> XXX:xxx +> +< HTTP/1.1 200 OK +< Date: Wed, 29 Apr 2020 04:18:31 GMT +< Connection: Keep-Alive +< Content-Type: text/html; charset=UTF-8 +< Transfer-Encoding: chunked +< Keep-Alive: timeout=3 +< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} +< +Relative Path File +* Connection #0 to host localhost left intact ``` [Artículo Original](https://clickhouse.tech/docs/en/interfaces/http_interface/) diff --git a/docs/es/interfaces/index.md b/docs/es/interfaces/index.md index 5b65e44e46e..3632c8a9e29 100644 --- a/docs/es/interfaces/index.md +++ b/docs/es/interfaces/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa -toc_folder_title: Interfaces +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: Interfaz toc_priority: 14 toc_title: "Implantaci\xF3n" --- diff --git a/docs/es/interfaces/jdbc.md b/docs/es/interfaces/jdbc.md index fd3f26c946a..7303dec8960 100644 --- a/docs/es/interfaces/jdbc.md +++ b/docs/es/interfaces/jdbc.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 22 toc_title: Controlador JDBC --- diff --git a/docs/es/interfaces/mysql.md b/docs/es/interfaces/mysql.md index f47b75dadb0..796bfbcf1a4 100644 --- a/docs/es/interfaces/mysql.md +++ b/docs/es/interfaces/mysql.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 20 toc_title: Interfaz MySQL --- -# Interfaz De MySQL {#mysql-interface} +# Interfaz MySQL {#mysql-interface} ClickHouse soporta el protocolo de cable MySQL. Puede ser habilitado por [mysql\_port](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-mysql_port) configuración en el archivo de configuración: diff --git a/docs/es/interfaces/odbc.md b/docs/es/interfaces/odbc.md index 9d1a755122b..6ccb979c7f7 100644 --- a/docs/es/interfaces/odbc.md +++ b/docs/es/interfaces/odbc.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 23 toc_title: Conductor ODBC --- diff --git a/docs/es/interfaces/tcp.md b/docs/es/interfaces/tcp.md index 47395124b13..47df0d12829 100644 --- a/docs/es/interfaces/tcp.md +++ b/docs/es/interfaces/tcp.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 18 toc_title: Interfaz nativa (TCP) --- -# Interfaz Nativa (TCP) {#native-interface-tcp} +# Interfaz nativa (TCP) {#native-interface-tcp} El protocolo nativo se utiliza en el [cliente de línea de comandos](cli.md), para la comunicación entre servidores durante el procesamiento de consultas distribuidas, y también en otros programas de C, Desafortunadamente, el protocolo nativo de ClickHouse aún no tiene especificaciones formales, pero puede ser diseñado de manera inversa desde el código fuente de ClickHouse (comenzando [por aquí](https://github.com/ClickHouse/ClickHouse/tree/master/src/Client)) y/o mediante la interceptación y el análisis del tráfico TCP. diff --git a/docs/es/interfaces/third-party/client-libraries.md b/docs/es/interfaces/third-party/client-libraries.md index ad7faab8b96..44186ec9135 100644 --- a/docs/es/interfaces/third-party/client-libraries.md +++ b/docs/es/interfaces/third-party/client-libraries.md @@ -1,59 +1,58 @@ --- -machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa toc_priority: 26 -toc_title: Bibliotecas de clientes +toc_title: Client Libraries --- -# Bibliotecas De Clientes De Desarrolladores De Terceros {#client-libraries-from-third-party-developers} +# Client Libraries from Third-party Developers {#client-libraries-from-third-party-developers} -!!! warning "Descargo" - Yandex hace **ni** mantenga las bibliotecas enumeradas a continuación y no haya realizado ninguna prueba extensa para garantizar su calidad. +!!! warning "Disclaimer" + Yandex does **not** maintain the libraries listed below and haven’t done any extensive testing to ensure their quality. -- Película - - [InformaciónSistema abierto.](https://github.com/Infinidat/infi.clickhouse_orm) - - [Casa de clics-conductor](https://github.com/mymarilyn/clickhouse-driver) - - [Casa de clics-cliente](https://github.com/yurial/clickhouse-client) - - [Aiochclient](https://github.com/maximdanilchenko/aiochclient) +- Python + - [infi.clickhouse\_orm](https://github.com/Infinidat/infi.clickhouse_orm) + - [clickhouse-driver](https://github.com/mymarilyn/clickhouse-driver) + - [clickhouse-client](https://github.com/yurial/clickhouse-client) + - [aiochclient](https://github.com/maximdanilchenko/aiochclient) - PHP - - [Método de codificación de datos:](https://packagist.org/packages/smi2/phpClickHouse) - - [Sistema abierto.](https://packagist.org/packages/8bitov/clickhouse-php-client) - - [Sistema abierto.](https://packagist.org/packages/bozerkins/clickhouse-client) - - [Sistema abierto.](https://packagist.org/packages/simpod/clickhouse-client) - - [Seva-code/php-click-house-cliente](https://packagist.org/packages/seva-code/php-click-house-client) - - [Cliente de SeasClick C++](https://github.com/SeasX/SeasClick) -- Ve - - [Casa de clics](https://github.com/kshvakov/clickhouse/) - - [Sistema abierto.](https://github.com/roistat/go-clickhouse) - - [Bienvenidos al Portal de Licitación Electrónica de Licitación Electrónica](https://github.com/mailru/go-clickhouse) - - [Golang-clickhouse](https://github.com/leprosus/golang-clickhouse) + - [smi2/phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse) + - [8bitov/clickhouse-php-client](https://packagist.org/packages/8bitov/clickhouse-php-client) + - [bozerkins/clickhouse-client](https://packagist.org/packages/bozerkins/clickhouse-client) + - [simpod/clickhouse-client](https://packagist.org/packages/simpod/clickhouse-client) + - [seva-code/php-click-house-client](https://packagist.org/packages/seva-code/php-click-house-client) + - [SeasClick C++ client](https://github.com/SeasX/SeasClick) +- Go + - [clickhouse](https://github.com/kshvakov/clickhouse/) + - [go-clickhouse](https://github.com/roistat/go-clickhouse) + - [mailrugo-clickhouse](https://github.com/mailru/go-clickhouse) + - [golang-clickhouse](https://github.com/leprosus/golang-clickhouse) - NodeJs - - [Casa de clic (NodeJs)](https://github.com/TimonKK/clickhouse) - - [Inicio](https://github.com/apla/node-clickhouse) + - [clickhouse (NodeJs)](https://github.com/TimonKK/clickhouse) + - [node-clickhouse](https://github.com/apla/node-clickhouse) - Perl - - [Bienvenidos al Portal de Licitación Electrónica de Licitación Electrónica](https://github.com/elcamlost/perl-DBD-ClickHouse) - - [Bienvenidos al Portal de Licitación Electrónica de Licitación Electrónica](https://metacpan.org/release/HTTP-ClickHouse) - - [Cualquier evento-ClickHouse](https://metacpan.org/release/AnyEvent-ClickHouse) -- Rubí - - [Haga clic en Casa (Ruby)](https://github.com/shlima/click_house) + - [perl-DBD-ClickHouse](https://github.com/elcamlost/perl-DBD-ClickHouse) + - [HTTP-ClickHouse](https://metacpan.org/release/HTTP-ClickHouse) + - [AnyEvent-ClickHouse](https://metacpan.org/release/AnyEvent-ClickHouse) +- Ruby + - [ClickHouse (Ruby)](https://github.com/shlima/click_house) - [clickhouse-activerecord](https://github.com/PNixx/clickhouse-activerecord) - R - - [Sistema abierto.](https://github.com/hannesmuehleisen/clickhouse-r) - - [Bienvenidos al Portal de Licitación Electrónica de Licitación Electrónica](https://github.com/IMSMWU/RClickhouse) + - [clickhouse-r](https://github.com/hannesmuehleisen/clickhouse-r) + - [RClickHouse](https://github.com/IMSMWU/RClickHouse) - Java - - [Casa de clic-cliente-java](https://github.com/VirtusAI/clickhouse-client-java) - - [Casa de clics-cliente](https://github.com/Ecwid/clickhouse-client) -- Ciudad - - [Sistema abierto.](https://github.com/crobox/clickhouse-scala-client) -- Bienvenido + - [clickhouse-client-java](https://github.com/VirtusAI/clickhouse-client-java) + - [clickhouse-client](https://github.com/Ecwid/clickhouse-client) +- Scala + - [clickhouse-scala-client](https://github.com/crobox/clickhouse-scala-client) +- Kotlin - [AORM](https://github.com/TanVD/AORM) - C\# - - [Sistema abierto.Ado](https://github.com/killwort/ClickHouse-Net) - - [Sistema abierto.Cliente](https://github.com/DarkWanderer/ClickHouse.Client) - - [Sistema abierto.](https://github.com/ilyabreev/ClickHouse.Net) + - [ClickHouse.Ado](https://github.com/killwort/ClickHouse-Net) + - [ClickHouse.Client](https://github.com/DarkWanderer/ClickHouse.Client) + - [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net) - Elixir - [clickhousex](https://github.com/appodeal/clickhousex/) + - [pillar](https://github.com/sofakingworld/pillar) - Nim - - [Bienvenidos al Portal de Licitación Electrónica de Licitación Electrónica](https://github.com/leonardoce/nim-clickhouse) + - [nim-clickhouse](https://github.com/leonardoce/nim-clickhouse) -[Artículo Original](https://clickhouse.tech/docs/en/interfaces/third-party/client_libraries/) +[Original article](https://clickhouse.tech/docs/en/interfaces/third-party/client_libraries/) diff --git a/docs/es/interfaces/third-party/gui.md b/docs/es/interfaces/third-party/gui.md index c76a8adba5e..754c0f68c69 100644 --- a/docs/es/interfaces/third-party/gui.md +++ b/docs/es/interfaces/third-party/gui.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 28 toc_title: Interfaces Visuales --- -# Interfaces Visuales De Desarrolladores De Terceros {#visual-interfaces-from-third-party-developers} +# Interfaces visuales de desarrolladores de terceros {#visual-interfaces-from-third-party-developers} -## De código Abierto {#open-source} +## De código abierto {#open-source} ### Tabix {#tabix} @@ -23,7 +23,7 @@ Función: [Documentación de Tabix](https://tabix.io/doc/). -### Sistema Abierto {#houseops} +### Sistema abierto {#houseops} [Sistema abierto.](https://github.com/HouseOps/HouseOps) Es una interfaz de usuario / IDE para OSX, Linux y Windows. @@ -78,7 +78,7 @@ Función: - Vista previa de datos de tabla. - Búsqueda de texto completo. -### Sistema Abierto {#clickhouse-cli} +### Sistema abierto {#clickhouse-cli} [Sistema abierto.](https://github.com/hatarist/clickhouse-cli) es un cliente de línea de comandos alternativo para ClickHouse, escrito en Python 3. @@ -89,10 +89,14 @@ Función: - Soporte de buscapersonas para la salida de datos. - Comandos similares a PostgreSQL personalizados. -### Sistema Abierto {#clickhouse-flamegraph} +### Sistema abierto {#clickhouse-flamegraph} [Sistema abierto.](https://github.com/Slach/clickhouse-flamegraph) es una herramienta especializada para visualizar el `system.trace_log` como [Flamegraph](http://www.brendangregg.com/flamegraphs.html). +### Bienvenidos al Portal de Licitación Electrónica de Licitación Electrónica {#clickhouse-plantuml} + +[Método de codificación de datos:](https://pypi.org/project/clickhouse-plantuml/) es un script para generar [PlantUML](https://plantuml.com/) diagrama de esquemas de tablas. + ## Comercial {#commercial} ### DataGrip {#datagrip} @@ -124,7 +128,7 @@ Nivel de Cifrado WEP [disponible de forma gratuita](https://cloud.yandex.com/doc - [Documentación de DataLens](https://cloud.yandex.com/docs/datalens/). - [Tutorial](https://cloud.yandex.com/docs/solutions/datalens/data-from-ch-visualization) en la visualización de datos de una base de datos ClickHouse. -### Software De Holística {#holistics-software} +### Software de Holística {#holistics-software} [Holística](https://www.holistics.io/) es una plataforma de datos de pila completa y una herramienta de inteligencia de negocios. @@ -145,7 +149,7 @@ Función: - Desarrollo fácil y ágil utilizando LookML, un lenguaje que soporta curado [Modelado de datos](https://looker.com/platform/data-modeling) para apoyar a los redactores de informes y a los usuarios finales. -- Potente integración de flujo de trabajo a través de Looker’s [Acciones de datos](https://looker.com/platform/actions). +- Potente integración de flujo de trabajo a través de Looker's [Acciones de datos](https://looker.com/platform/actions). [Cómo configurar ClickHouse en Looker.](https://docs.looker.com/setup-and-management/database-config/clickhouse) diff --git a/docs/es/interfaces/third-party/index.md b/docs/es/interfaces/third-party/index.md index 821867a3ca7..adf50b05cdf 100644 --- a/docs/es/interfaces/third-party/index.md +++ b/docs/es/interfaces/third-party/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa -toc_folder_title: Third-Party +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: tercero toc_priority: 24 --- diff --git a/docs/es/interfaces/third-party/integrations.md b/docs/es/interfaces/third-party/integrations.md index 83c3204c596..5e9e8f841bf 100644 --- a/docs/es/interfaces/third-party/integrations.md +++ b/docs/es/interfaces/third-party/integrations.md @@ -1,102 +1,100 @@ --- -machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa toc_priority: 27 -toc_title: "Integraci\xF3n" +toc_title: Integrations --- -# Bibliotecas De integración De Desarrolladores Externos {#integration-libraries-from-third-party-developers} +# Integration Libraries from Third-party Developers {#integration-libraries-from-third-party-developers} -!!! warning "Descargo" - Yandex hace **ni** mantenga las herramientas y bibliotecas que se enumeran a continuación y no haya realizado ninguna prueba extensa para garantizar su calidad. +!!! warning "Disclaimer" + Yandex does **not** maintain the tools and libraries listed below and haven’t done any extensive testing to ensure their quality. -## Productos De Infraestructura {#infrastructure-products} +## Infrastructure Products {#infrastructure-products} -- Sistemas de gestión de bases de datos relacionales +- Relational database management systems - [MySQL](https://www.mysql.com) - - [Nombre de la red inalámbrica (SSID):](https://github.com/sysown/proxysql/wiki/ClickHouse-Support) - - [Casa de clic-mysql-lector de datos](https://github.com/Altinity/clickhouse-mysql-data-reader) - - [Horgh-replicador](https://github.com/larsnovikov/horgh-replicator) + - [ProxySQL](https://github.com/sysown/proxysql/wiki/ClickHouse-Support) + - [clickhouse-mysql-data-reader](https://github.com/Altinity/clickhouse-mysql-data-reader) + - [horgh-replicator](https://github.com/larsnovikov/horgh-replicator) - [PostgreSQL](https://www.postgresql.org) - - [Haga clickhousedb\_fdw](https://github.com/Percona-Lab/clickhousedb_fdw) - - [InformaciónSistema abierto.](https://github.com/Infinidat/infi.clickhouse_fdw) (utilizar [InformaciónSistema abierto.](https://github.com/Infinidat/infi.clickhouse_orm)) - - [Descripción](https://github.com/mkabilov/pg2ch) - - [Sistema abierto.](https://github.com/adjust/clickhouse_fdw) + - [clickhousedb\_fdw](https://github.com/Percona-Lab/clickhousedb_fdw) + - [infi.clickhouse\_fdw](https://github.com/Infinidat/infi.clickhouse_fdw) (uses [infi.clickhouse\_orm](https://github.com/Infinidat/infi.clickhouse_orm)) + - [pg2ch](https://github.com/mkabilov/pg2ch) + - [clickhouse\_fdw](https://github.com/adjust/clickhouse_fdw) - [MSSQL](https://en.wikipedia.org/wiki/Microsoft_SQL_Server) - - [Método de codificación de datos:](https://github.com/zlzforever/ClickHouseMigrator) -- Colas de mensajes + - [ClickHouseMigrator](https://github.com/zlzforever/ClickHouseMigrator) +- Message queues - [Kafka](https://kafka.apache.org) - - [clickhouse\_sinker](https://github.com/housepower/clickhouse_sinker) (usos [Go client](https://github.com/ClickHouse/clickhouse-go/)) -- Procesamiento de flujo + - [clickhouse\_sinker](https://github.com/housepower/clickhouse_sinker) (uses [Go client](https://github.com/ClickHouse/clickhouse-go/)) +- Stream processing - [Flink](https://flink.apache.org) - [flink-clickhouse-sink](https://github.com/ivi-ru/flink-clickhouse-sink) -- Almacenamiento de objetos +- Object storages - [S3](https://en.wikipedia.org/wiki/Amazon_S3) - - [Haga clic en el botón de copia de seguridad](https://github.com/AlexAkulov/clickhouse-backup) -- Orquestación de contenedores + - [clickhouse-backup](https://github.com/AlexAkulov/clickhouse-backup) +- Container orchestration - [Kubernetes](https://kubernetes.io) - - [Operador de clickhouse](https://github.com/Altinity/clickhouse-operator) -- Gestión de configuración - - [marioneta](https://puppet.com) - - [Bienvenidos al Portal de Licitación Electrónica de Licitación Electrónica](https://forge.puppet.com/innogames/clickhouse) - - [Sistema abierto.](https://forge.puppet.com/mfedotov/clickhouse) -- Monitoreo - - [Grafito](https://graphiteapp.org) + - [clickhouse-operator](https://github.com/Altinity/clickhouse-operator) +- Configuration management + - [puppet](https://puppet.com) + - [innogames/clickhouse](https://forge.puppet.com/innogames/clickhouse) + - [mfedotov/clickhouse](https://forge.puppet.com/mfedotov/clickhouse) +- Monitoring + - [Graphite](https://graphiteapp.org) - [graphouse](https://github.com/yandex/graphouse) - - [de carbono-clickhouse](https://github.com/lomik/carbon-clickhouse) + - - [Sistema abierto.](https://github.com/lomik/graphite-clickhouse) - - [Grafito-ch-optimizador](https://github.com/innogames/graphite-ch-optimizer) - optimiza las particiones [\*GraphiteMergeTree](../../engines/table-engines/mergetree-family/graphitemergetree.md#graphitemergetree) reglas de [Configuración de rollup](../../engines/table-engines/mergetree-family/graphitemergetree.md#rollup-configuration) podría ser aplicado + - [carbon-clickhouse](https://github.com/lomik/carbon-clickhouse) + + - [graphite-clickhouse](https://github.com/lomik/graphite-clickhouse) + - [graphite-ch-optimizer](https://github.com/innogames/graphite-ch-optimizer) - optimizes staled partitions in [\*GraphiteMergeTree](../../engines/table-engines/mergetree-family/graphitemergetree.md#graphitemergetree) if rules from [rollup configuration](../../engines/table-engines/mergetree-family/graphitemergetree.md#rollup-configuration) could be applied - [Grafana](https://grafana.com/) - - [Bienvenidos al Portal de Licitación Electrónica de Licitación Electrónica](https://github.com/Vertamedia/clickhouse-grafana) - - [Prometeo](https://prometheus.io/) - - [Sistema abierto.](https://github.com/f1yegor/clickhouse_exporter) - - [Bienvenido](https://github.com/Percona-Lab/PromHouse) - - [Sistema abierto.](https://github.com/hot-wifi/clickhouse_exporter) (utilizar [Ir cliente](https://github.com/kshvakov/clickhouse/)) + - [clickhouse-grafana](https://github.com/Vertamedia/clickhouse-grafana) + - [Prometheus](https://prometheus.io/) + - [clickhouse\_exporter](https://github.com/f1yegor/clickhouse_exporter) + - [PromHouse](https://github.com/Percona-Lab/PromHouse) + - [clickhouse\_exporter](https://github.com/hot-wifi/clickhouse_exporter) (uses [Go client](https://github.com/kshvakov/clickhouse/)) - [Nagios](https://www.nagios.org/) - - [Bienvenidos al Portal de Licitación Electrónica de Licitación Electrónica](https://github.com/exogroup/check_clickhouse/) - - [Inicio](https://github.com/innogames/igmonplugins/blob/master/src/check_clickhouse.py) + - [check\_clickhouse](https://github.com/exogroup/check_clickhouse/) + - [check\_clickhouse.py](https://github.com/innogames/igmonplugins/blob/master/src/check_clickhouse.py) - [Zabbix](https://www.zabbix.com) - - [Sistema abierto.](https://github.com/Altinity/clickhouse-zabbix-template) + - [clickhouse-zabbix-template](https://github.com/Altinity/clickhouse-zabbix-template) - [Sematext](https://sematext.com/) - - [integración clickhouse](https://github.com/sematext/sematext-agent-integrations/tree/master/clickhouse) -- Tala + - [clickhouse integration](https://github.com/sematext/sematext-agent-integrations/tree/master/clickhouse) +- Logging - [rsyslog](https://www.rsyslog.com/) - - [Bienvenidos al Portal de Licitación Electrónica de Licitación Electrónica](https://www.rsyslog.com/doc/master/configuration/modules/omclickhouse.html) + - [omclickhouse](https://www.rsyslog.com/doc/master/configuration/modules/omclickhouse.html) - [fluentd](https://www.fluentd.org) - - [casa de campo](https://github.com/flant/loghouse) (para [Kubernetes](https://kubernetes.io)) - - [Información](https://www.sematext.com/logagent) - - [Sistema de tabiquería interior y exterior](https://sematext.com/docs/logagent/output-plugin-clickhouse/) + - [loghouse](https://github.com/flant/loghouse) (for [Kubernetes](https://kubernetes.io)) + - [logagent](https://www.sematext.com/logagent) + - [logagent output-plugin-clickhouse](https://sematext.com/docs/logagent/output-plugin-clickhouse/) - Geo - [MaxMind](https://dev.maxmind.com/geoip/) - - [Para que usted pueda encontrar](https://github.com/AlexeyKupershtokh/clickhouse-maxmind-geoip) + - [clickhouse-maxmind-geoip](https://github.com/AlexeyKupershtokh/clickhouse-maxmind-geoip) -## Programación De Ecosistemas De Lenguaje {#programming-language-ecosystems} +## Programming Language Ecosystems {#programming-language-ecosystems} -- Película +- Python - [SQLAlchemy](https://www.sqlalchemy.org) - - [sqlalchemy-clickhouse](https://github.com/cloudflare/sqlalchemy-clickhouse) (utilizar [InformaciónSistema abierto.](https://github.com/Infinidat/infi.clickhouse_orm)) + - [sqlalchemy-clickhouse](https://github.com/cloudflare/sqlalchemy-clickhouse) (uses [infi.clickhouse\_orm](https://github.com/Infinidat/infi.clickhouse_orm)) - [pandas](https://pandas.pydata.org) - - [Pandahouse](https://github.com/kszucs/pandahouse) + - [pandahouse](https://github.com/kszucs/pandahouse) - PHP - [Doctrine](https://www.doctrine-project.org/) - [dbal-clickhouse](https://packagist.org/packages/friendsofdoctrine/dbal-clickhouse) - R - - [Dplyr](https://db.rstudio.com/dplyr/) - - [Bienvenidos al Portal de Licitación Electrónica de Licitación Electrónica](https://github.com/IMSMWU/RClickhouse) (utilizar [Bienvenidos](https://github.com/artpaul/clickhouse-cpp)) + - [dplyr](https://db.rstudio.com/dplyr/) + - [RClickHouse](https://github.com/IMSMWU/RClickHouse) (uses [clickhouse-cpp](https://github.com/artpaul/clickhouse-cpp)) - Java - [Hadoop](http://hadoop.apache.org) - - [Sistema abierto.](https://github.com/jaykelin/clickhouse-hdfs-loader) (utilizar [JDBC](../../sql-reference/table-functions/jdbc.md)) -- Ciudad + - [clickhouse-hdfs-loader](https://github.com/jaykelin/clickhouse-hdfs-loader) (uses [JDBC](../../sql-reference/table-functions/jdbc.md)) +- Scala - [Akka](https://akka.io) - - [Sistema abierto.](https://github.com/crobox/clickhouse-scala-client) + - [clickhouse-scala-client](https://github.com/crobox/clickhouse-scala-client) - C\# - [ADO.NET](https://docs.microsoft.com/en-us/dotnet/framework/data/adonet/ado-net-overview) - - [Sistema abierto.Ado](https://github.com/killwort/ClickHouse-Net) - - [Sistema abierto.Cliente](https://github.com/DarkWanderer/ClickHouse.Client) - - [Sistema abierto.](https://github.com/ilyabreev/ClickHouse.Net) - - [Bienvenidos al Portal de Licitación Electrónica de Licitación Electrónica](https://github.com/ilyabreev/ClickHouse.Net.Migrations) + - [ClickHouse.Ado](https://github.com/killwort/ClickHouse-Net) + - [ClickHouse.Client](https://github.com/DarkWanderer/ClickHouse.Client) + - [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net) + - [ClickHouse.Net.Migrations](https://github.com/ilyabreev/ClickHouse.Net.Migrations) - Elixir - [Ecto](https://github.com/elixir-ecto/ecto) - - [Método de codificación de datos:](https://github.com/appodeal/clickhouse_ecto) + - [clickhouse\_ecto](https://github.com/appodeal/clickhouse_ecto) -[Artículo Original](https://clickhouse.tech/docs/en/interfaces/third-party/integrations/) +[Original article](https://clickhouse.tech/docs/en/interfaces/third-party/integrations/) diff --git a/docs/es/interfaces/third-party/proxy.md b/docs/es/interfaces/third-party/proxy.md index c4b1c02b83a..e1aabf8fce4 100644 --- a/docs/es/interfaces/third-party/proxy.md +++ b/docs/es/interfaces/third-party/proxy.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 29 toc_title: Proxy --- -# Servidores Proxy De Desarrolladores De Terceros {#proxy-servers-from-third-party-developers} +# Servidores proxy de desarrolladores de terceros {#proxy-servers-from-third-party-developers} ## chproxy {#chproxy} @@ -31,7 +31,7 @@ Función: Implementado en Go. -## Bienvenidos Al Portal De Licitación Electrónica De Licitación Electrónica {#clickhouse-bulk} +## Bienvenidos al Portal de Licitación Electrónica de Licitación Electrónica {#clickhouse-bulk} [Bienvenidos al Portal de Licitación Electrónica de Licitación Electrónica](https://github.com/nikepan/clickhouse-bulk) es un simple colector de insertos ClickHouse. diff --git a/docs/es/introduction/adopters.md b/docs/es/introduction/adopters.md index 4f87a8db46c..e41e8005cc7 100644 --- a/docs/es/introduction/adopters.md +++ b/docs/es/introduction/adopters.md @@ -1,82 +1,86 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 8 toc_title: Adoptante --- -# Adoptadores De ClickHouse {#clickhouse-adopters} +# Adoptadores de ClickHouse {#clickhouse-adopters} !!! warning "Descargo" La siguiente lista de empresas que utilizan ClickHouse y sus historias de éxito se recopila a partir de fuentes públicas, por lo que podría diferir de la realidad actual. Le agradeceríamos que compartiera la historia de adoptar ClickHouse en su empresa y [agregarlo a la lista](https://github.com/ClickHouse/ClickHouse/edit/master/docs/en/introduction/adopters.md), pero por favor asegúrese de que usted no tendrá ningunos problemas de NDA haciendo así. Proporcionar actualizaciones con publicaciones de otras compañías también es útil. -| Empresa | Industria | Usecase | Tamaño de clúster | (Un)Tamaño de datos comprimidos\* | Referencia | -|--------------------------------------------------------------------------------------------|------------------------------------|-----------------------------|------------------------------------------------------------------|-------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [2gis](https://2gis.ru) | Asignar | Monitoreo | — | — | [Charla en ruso, julio 2019](https://youtu.be/58sPkXfq6nw) | -| [Navegador Aloha](https://alohabrowser.com/) | Aplicación móvil | Backend del navegador | — | — | [Diapositivas en ruso, mayo 2019](https://github.com/yandex/clickhouse-presentations/blob/master/meetup22/aloha.pdf) | -| [Amadeus](https://amadeus.com/) | Viaje | Analítica | — | — | [Comunicado de prensa, abril de 2018](https://www.altinity.com/blog/2018/4/5/amadeus-technologies-launches-investment-and-insights-tool-based-on-machine-learning-and-strategy-algorithms) | -| [Appsflyer](https://www.appsflyer.com) | Análisis móvil | Producto principal | — | — | [Charla en ruso, julio 2019](https://www.youtube.com/watch?v=M3wbRlcpBbY) | -| [ArenaData](https://arenadata.tech/) | Plataforma de datos | Producto principal | — | — | [Diapositivas en ruso, diciembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup38/indexes.pdf) | -| [Badoo](https://badoo.com) | Citas | Serie de tiempo | — | — | [Diapositivas en ruso, diciembre 2019](https://presentations.clickhouse.tech/meetup38/forecast.pdf) | -| [Benocs](https://www.benocs.com/) | Telemetría y análisis de red | Producto principal | — | — | [Diapositivas en español, octubre de 2017](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup9/lpm.pdf) | -| [Bloomberg](https://www.bloomberg.com/) | Finanzas, Medios | Monitoreo | 102 servidores | — | [Diapositivas, Mayo 2018](https://www.slideshare.net/Altinity/http-analytics-for-6m-requests-per-second-using-clickhouse-by-alexander-bocharov) | -| [Bloxy](https://bloxy.info) | Blockchain | Analítica | — | — | [Diapositivas en ruso, agosto 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/4_bloxy.pptx) | -| `Dataliance/UltraPower` | Telecomunicaciones | Analítica | — | — | [Diapositivas en chino, enero 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup12/telecom.pdf) | -| [CARTO](https://carto.com/) | Inteligencia de negocios | Análisis geográfico | — | — | [Procesamiento geoespacial con Clickhouse](https://carto.com/blog/geospatial-processing-with-clickhouse/) | -| [CERN](http://public.web.cern.ch/public/) | Investigación | Experimento | — | — | [Comunicado de prensa, abril de 2012](https://www.yandex.com/company/press_center/press_releases/2012/2012-04-10/) | -| [Cisco](http://cisco.com/) | Red | Análisis de tráfico | — | — | [Charla relámpago, octubre 2019](https://youtu.be/-hI1vDR2oPY?t=5057) | -| [Valores de la ciudadela](https://www.citadelsecurities.com/) | Financiación | — | — | — | [Contribución, marzo 2019](https://github.com/ClickHouse/ClickHouse/pull/4774) | -| [Más información](https://city-mobil.ru) | Taxi | Analítica | — | — | [Blog Post en ruso, marzo 2020](https://habr.com/en/company/citymobil/blog/490660/) | -| [ContentSquare](https://contentsquare.com) | Análisis web | Producto principal | — | — | [Publicación de blog en francés, noviembre 2018](http://souslecapot.net/2018/11/21/patrick-chatain-vp-engineering-chez-contentsquare-penser-davantage-amelioration-continue-que-revolution-constante/) | -| [Cloudflare](https://cloudflare.com) | CDN | Análisis de tráfico | 36 servidores | — | [Mensaje del blog, Mayo 2017](https://blog.cloudflare.com/how-cloudflare-analyzes-1m-dns-queries-per-second/), [Mensaje del blog, marzo 2018](https://blog.cloudflare.com/http-analytics-for-6m-requests-per-second-using-clickhouse/) | -| [Corunet](https://coru.net/) | Analítica | Producto principal | — | — | [Diapositivas en español, Abril 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup21/predictive_models.pdf) | -| [CraiditX 氪信](https://creditx.com) | Finanzas AI | Análisis | — | — | [Diapositivas en español, noviembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/udf.pptx) | -| [Criteo/Storetail](https://www.criteo.com/) | Menor | Producto principal | — | — | [Diapositivas en español, octubre 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/3_storetail.pptx) | -| [Banco de Deutsche](https://db.com) | Financiación | BI Analytics | — | — | [Diapositivas en español, octubre 2019](https://bigdatadays.ru/wp-content/uploads/2019/10/D2-H3-3_Yakunin-Goihburg.pdf) | -| [Diva-e](https://www.diva-e.com) | Consultoría digital | Producto principal | — | — | [Diapositivas en español, septiembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup29/ClickHouse-MeetUp-Unusual-Applications-sd-2019-09-17.pdf) | -| [Exness](https://www.exness.com) | Comercio | Métricas, Registro | — | — | [Charla en ruso, mayo 2019](https://youtu.be/_rpU-TvSfZ8?t=3215) | -| [Sistema abierto.](https://geniee.co.jp) | Red Ad | Producto principal | — | — | [Publicación de blog en japonés, julio 2017](https://tech.geniee.co.jp/entry/2017/07/20/160100) | -| [HUYA](https://www.huya.com/) | Video Streaming | Analítica | — | — | [Diapositivas en chino, octubre 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/7.%20ClickHouse万亿数据分析实践%20李本旺(sundy-li)%20虎牙.pdf) | -| [Idealista](https://www.idealista.com) | Inmobiliario | Analítica | — | — | [Blog Post en Inglés, Abril 2019](https://clickhouse.yandex/blog/en/clickhouse-meetup-in-madrid-on-april-2-2019) | -| [Infovista](https://www.infovista.com/) | Red | Analítica | — | — | [Diapositivas en español, octubre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup30/infovista.pdf) | -| [InnoGames](https://www.innogames.com) | Juego | Métricas, Registro | — | — | [Diapositivas en ruso, septiembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/graphite_and_clickHouse.pdf) | -| [Integros](https://integros.com) | Plataforma para servicios de video | Analítica | — | — | [Diapositivas en ruso, mayo 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | -| [Datos de Kodiak](https://www.kodiakdata.com/) | Nube | Producto principal | — | — | [Diapositivas en Engish, Abril 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup13/kodiak_data.pdf) | -| [Kontur](https://kontur.ru) | Desarrollo de software | Métricas | — | — | [Charla en ruso, noviembre 2018](https://www.youtube.com/watch?v=U4u4Bd0FtrY) | -| [Sistema abierto.](https://lifestreet.com/) | Red Ad | Producto principal | 75 servidores (3 réplicas) | 5.27 PiB | [Publicación de blog en ruso, febrero 2017](https://habr.com/en/post/322620/) | -| [Soluciones en la nube de Mail.ru](https://mcs.mail.ru/) | Servicios en la nube | Producto principal | — | — | [Ejecución de ClickHouse Instance, en ruso](https://mcs.mail.ru/help/db-create/clickhouse#) | -| [Mensaje de pájaro](https://www.messagebird.com) | Telecomunicaciones | Estadísticas | — | — | [Diapositivas en español, noviembre 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup20/messagebird.pdf) | -| [MGID](https://www.mgid.com/) | Red Ad | Analítica Web | — | — | [Nuestra experiencia en la implementación analítica DBMS ClickHouse, en ruso](http://gs-studio.com/news-about-it/32777----clickhouse---c) | -| [UnoAPM](https://www.oneapm.com/) | Supervisión y análisis de datos | Producto principal | — | — | [Diapositivas en chino, octubre 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/8.%20clickhouse在OneAPM的应用%20杜龙.pdf) | -| [Pragma Innovación](http://www.pragma-innovation.fr/) | Telemetría y Análisis de Big Data | Producto principal | — | — | [Diapositivas en español, octubre 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/4_pragma_innovation.pdf) | -| [QINGCLOUD](https://www.qingcloud.com/) | Servicios en la nube | Producto principal | — | — | [Diapositivas en chino, octubre 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/4.%20Cloud%20%2B%20TSDB%20for%20ClickHouse%20张健%20QingCloud.pdf) | -| [Qrator](https://qrator.net) | Protección DDoS | Producto principal | — | — | [Blog Post, marzo 2019](https://blog.qrator.net/en/clickhouse-ddos-mitigation_37/) | -| [Tecnología de la información del PORCIMIENTO de Pekín Co., Ltd.](https://www.percent.cn/) | Analítica | Producto principal | — | — | [Diapositivas en chino, junio 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/4.%20ClickHouse万亿数据双中心的设计与实践%20.pdf) | -| [Rambler](https://rambler.ru) | Servicios de Internet | Analítica | — | — | [Charla en ruso, abril 2018](https://medium.com/@ramblertop/разработка-api-clickhouse-для-рамблер-топ-100-f4c7e56f3141) | -| [Tencent](https://www.tencent.com) | Mensajería | Tala | — | — | [Charla en chino, noviembre 2019](https://youtu.be/T-iVQRuw-QY?t=5050) | -| [Estrellas de tráfico](https://trafficstars.com/) | Red AD | — | — | — | [Diapositivas en ruso, mayo 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup15/lightning/ninja.pdf) | -| [S7 Aerolíneas](https://www.s7.ru) | Aérea | Métricas, Registro | — | — | [Charla en ruso, marzo 2019](https://www.youtube.com/watch?v=nwG68klRpPg&t=15s) | -| [SEMrush](https://www.semrush.com/) | Marketing | Producto principal | — | — | [Diapositivas en ruso, agosto 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/5_semrush.pdf) | -| [Inicio](https://www.scireum.de/) | Comercio electrónico | Producto principal | — | — | [Charla en alemán, febrero de 2020](https://www.youtube.com/watch?v=7QWAn5RbyR4) | -| [Centinela](https://sentry.io/) | Desarrollador de software | Backend para el producto | — | — | [Publicación de blog en inglés, mayo 2019](https://blog.sentry.io/2019/05/16/introducing-snuba-sentrys-new-search-infrastructure) | -| [SGK](http://www.sgk.gov.tr/wps/portal/sgk/tr) | Gobierno Seguridad Social | Analítica | — | — | [Diapositivas en español, noviembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup35/ClickHouse%20Meetup-Ramazan%20POLAT.pdf) | -| [el seo.¿](https://seo.do/) | Analítica | Producto principal | — | — | [Diapositivas en español, noviembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup35/CH%20Presentation-%20Metehan%20Çetinkaya.pdf) | -| [Sina](http://english.sina.com/index.html) | Noticia | — | — | — | [Diapositivas en chino, octubre 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/6.%20ClickHouse最佳实践%20高鹏_新浪.pdf) | -| [SMI2](https://smi2.ru/) | Noticia | Analítica | — | — | [Blog Post en ruso, noviembre 2017](https://habr.com/ru/company/smi2/blog/314558/) | -| [Salto](https://www.splunk.com/) | Análisis de negocios | Producto principal | — | — | [Diapositivas en español, enero 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup12/splunk.pdf) | -| [Spotify](https://www.spotify.com) | Sica | Experimentación | — | — | [Diapositivas, julio 2018](https://www.slideshare.net/glebus/using-clickhouse-for-experimentation-104247173) | -| [Tencent](https://www.tencent.com) | Grandes Datos | Procesamiento de datos | — | — | [Diapositivas en chino, octubre 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/5.%20ClickHouse大数据集群应用_李俊飞腾讯网媒事业部.pdf) | -| [Más información](https://www.uber.com) | Taxi | Tala | — | — | [Diapositivas, febrero de 2020](https://presentations.clickhouse.tech/meetup40/uber.pdf) | -| [VKontakte](https://vk.com) | Red social | Estadísticas, Registro | — | — | [Diapositivas en ruso, agosto 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/3_vk.pdf) | -| [Método de codificación de datos:](https://wisebits.com/) | Soluciones de TI | Analítica | — | — | [Diapositivas en ruso, mayo 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | -| [Tecnología de Xiaoxin.](https://www.xiaoheiban.cn/) | Educación | Propósito común | — | — | [Diapositivas en español, noviembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/sync-clickhouse-with-mysql-mongodb.pptx) | -| [Ximalaya](https://www.ximalaya.com/) | Compartir audio | OLAP | — | — | [Diapositivas en español, noviembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/ximalaya.pdf) | -| [Nube de Yandex](https://cloud.yandex.ru/services/managed-clickhouse) | Nube pública | Producto principal | — | — | [Charla en ruso, diciembre 2019](https://www.youtube.com/watch?v=pgnak9e_E0o) | -| [Yandex DataLens](https://cloud.yandex.ru/services/datalens) | Inteligencia de negocios | Producto principal | — | — | [Diapositivas en ruso, diciembre 2019](https://presentations.clickhouse.tech/meetup38/datalens.pdf) | -| [Mercado de Yandex](https://market.yandex.ru/) | Comercio electrónico | Métricas, Registro | — | — | [Charla en ruso, enero 2019](https://youtu.be/_l1qP0DyBcA?t=478) | -| [Yandex Metrica](https://metrica.yandex.com) | Análisis web | Producto principal | 360 servidores en un clúster, 1862 servidores en un departamento | 66.41 PiB / 5.68 PiB | [Diapositivas, febrero de 2020](https://presentations.clickhouse.tech/meetup40/introduction/#13) | -| [ЦВТ](https://htc-cs.ru/) | Desarrollo de software | Métricas, Registro | — | — | [Blog Post, marzo 2019, en ruso](https://vc.ru/dev/62715-kak-my-stroili-monitoring-na-prometheus-clickhouse-i-elk) | -| [МКБ](https://mkb.ru/) | Banco | Supervisión del sistema web | — | — | [Diapositivas en ruso, septiembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/mkb.pdf) | -| [金数据](https://jinshuju.net) | BI Analytics | Producto principal | — | — | [Diapositivas en chino, octubre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/3.%20金数据数据架构调整方案Public.pdf) | +| Empresa | Industria | Usecase | Tamaño de clúster | (Un)Tamaño de datos comprimidos\* | Referencia | +|-------------------------------------------------------------------------------------------------|------------------------------------|-----------------------------|------------------------------------------------------------------|-------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 2gis | Asignar | Monitoreo | — | — | [Charla en ruso, julio 2019](https://youtu.be/58sPkXfq6nw) | +| Aloha Browser | Aplicación móvil | Backend del navegador | — | — | [Diapositivas en ruso, mayo 2019](https://github.com/yandex/clickhouse-presentations/blob/master/meetup22/aloha.pdf) | +| Amadeus | Viaje | Analítica | — | — | [Comunicado de prensa, abril de 2018](https://www.altinity.com/blog/2018/4/5/amadeus-technologies-launches-investment-and-insights-tool-based-on-machine-learning-and-strategy-algorithms) | +| Appsflyer | Análisis móvil | Producto principal | — | — | [Charla en ruso, julio 2019](https://www.youtube.com/watch?v=M3wbRlcpBbY) | +| ArenaData | Plataforma de datos | Producto principal | — | — | [Diapositivas en ruso, diciembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup38/indexes.pdf) | +| Badoo | Citas | Serie de tiempo | — | — | [Diapositivas en ruso, diciembre 2019](https://presentations.clickhouse.tech/meetup38/forecast.pdf) | +| Benocs | Telemetría y análisis de red | Producto principal | — | — | [Diapositivas en español, octubre de 2017](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup9/lpm.pdf) | +| Bloomberg | Finanzas, Medios | Monitoreo | 102 servidores | — | [Diapositivas, Mayo 2018](https://www.slideshare.net/Altinity/http-analytics-for-6m-requests-per-second-using-clickhouse-by-alexander-bocharov) | +| Bloxy | Blockchain | Analítica | — | — | [Diapositivas en ruso, agosto 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/4_bloxy.pptx) | +| Dataliance para China Telecom | Telecomunicaciones | Analítica | — | — | [Diapositivas en chino, enero 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup12/telecom.pdf) | +| CARTO | Inteligencia de negocios | Análisis geográfico | — | — | [Procesamiento geoespacial con ClickHouse](https://carto.com/blog/geospatial-processing-with-clickhouse/) | +| CERN | Investigación | Experimento | — | — | [Comunicado de prensa, abril de 2012](https://www.yandex.com/company/press_center/press_releases/2012/2012-04-10/) | +| Cisco | Red | Análisis de tráfico | — | — | [Charla relámpago, octubre 2019](https://youtu.be/-hI1vDR2oPY?t=5057) | +| Citadel Securities | Financiación | — | — | — | [Contribución, marzo 2019](https://github.com/ClickHouse/ClickHouse/pull/4774) | +| Más información | Taxi | Analítica | — | — | [Blog Post en ruso, marzo 2020](https://habr.com/en/company/citymobil/blog/490660/) | +| ContentSquare | Análisis web | Producto principal | — | — | [Publicación de blog en francés, noviembre 2018](http://souslecapot.net/2018/11/21/patrick-chatain-vp-engineering-chez-contentsquare-penser-davantage-amelioration-continue-que-revolution-constante/) | +| Cloudflare | CDN | Análisis de tráfico | 36 servidores | — | [Mensaje del blog, Mayo 2017](https://blog.cloudflare.com/how-cloudflare-analyzes-1m-dns-queries-per-second/), [Mensaje del blog, marzo 2018](https://blog.cloudflare.com/http-analytics-for-6m-requests-per-second-using-clickhouse/) | +| Corunet | Analítica | Producto principal | — | — | [Diapositivas en español, Abril 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup21/predictive_models.pdf) | +| CraiditX 氪信 | Finanzas AI | Análisis | — | — | [Diapositivas en español, noviembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/udf.pptx) | +| Criteo | Menor | Producto principal | — | — | [Diapositivas en español, octubre 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/3_storetail.pptx) | +| Deutsche Bank | Financiación | BI Analytics | — | — | [Diapositivas en español, octubre 2019](https://bigdatadays.ru/wp-content/uploads/2019/10/D2-H3-3_Yakunin-Goihburg.pdf) | +| Diva-e | Consultoría digital | Producto principal | — | — | [Diapositivas en español, septiembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup29/ClickHouse-MeetUp-Unusual-Applications-sd-2019-09-17.pdf) | +| Exness | Comercio | Métricas, Registro | — | — | [Charla en ruso, mayo 2019](https://youtu.be/_rpU-TvSfZ8?t=3215) | +| Sistema abierto. | Red Ad | Producto principal | — | — | [Publicación de blog en japonés, julio 2017](https://tech.geniee.co.jp/entry/2017/07/20/160100) | +| HUYA | Video Streaming | Analítica | — | — | [Diapositivas en chino, octubre 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/7.%20ClickHouse万亿数据分析实践%20李本旺(sundy-li)%20虎牙.pdf) | +| Idealista | Inmobiliario | Analítica | — | — | [Blog Post en Inglés, Abril 2019](https://clickhouse.yandex/blog/en/clickhouse-meetup-in-madrid-on-april-2-2019) | +| Infovista | Red | Analítica | — | — | [Diapositivas en español, octubre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup30/infovista.pdf) | +| InnoGames | Juego | Métricas, Registro | — | — | [Diapositivas en ruso, septiembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/graphite_and_clickHouse.pdf) | +| Integros | Plataforma para servicios de video | Analítica | — | — | [Diapositivas en ruso, mayo 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | +| Datos de Kodiak | Nube | Producto principal | — | — | [Diapositivas en Engish, Abril 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup13/kodiak_data.pdf) | +| Kontur | Desarrollo de software | Métricas | — | — | [Charla en ruso, noviembre 2018](https://www.youtube.com/watch?v=U4u4Bd0FtrY) | +| Sistema abierto. | Red Ad | Producto principal | 75 servidores (3 réplicas) | 5.27 PiB | [Publicación de blog en ruso, febrero 2017](https://habr.com/en/post/322620/) | +| Soluciones en la nube de Mail.ru | Servicios en la nube | Producto principal | — | — | [Artículo en ruso](https://mcs.mail.ru/help/db-create/clickhouse#) | +| Mensaje de pájaro | Telecomunicaciones | Estadísticas | — | — | [Diapositivas en español, noviembre 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup20/messagebird.pdf) | +| MGID | Red Ad | Analítica Web | — | — | [Publicación de blog en ruso, abril 2020](http://gs-studio.com/news-about-it/32777----clickhouse---c) | +| UnoAPM | Supervisión y análisis de datos | Producto principal | — | — | [Diapositivas en chino, octubre 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/8.%20clickhouse在OneAPM的应用%20杜龙.pdf) | +| Pragma Innovation | Telemetría y Análisis de Big Data | Producto principal | — | — | [Diapositivas en español, octubre 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/4_pragma_innovation.pdf) | +| QINGCLOUD | Servicios en la nube | Producto principal | — | — | [Diapositivas en chino, octubre 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/4.%20Cloud%20%2B%20TSDB%20for%20ClickHouse%20张健%20QingCloud.pdf) | +| Qrator | Protección DDoS | Producto principal | — | — | [Blog Post, marzo 2019](https://blog.qrator.net/en/clickhouse-ddos-mitigation_37/) | +| Percent 百分点 | Analítica | Producto principal | — | — | [Diapositivas en chino, junio 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/4.%20ClickHouse万亿数据双中心的设计与实践%20.pdf) | +| Rambler | Servicios de Internet | Analítica | — | — | [Charla en ruso, abril 2018](https://medium.com/@ramblertop/разработка-api-clickhouse-для-рамблер-топ-100-f4c7e56f3141) | +| Tencent | Mensajería | Tala | — | — | [Charla en chino, noviembre 2019](https://youtu.be/T-iVQRuw-QY?t=5050) | +| Traffic Stars | Red AD | — | — | — | [Diapositivas en ruso, mayo 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup15/lightning/ninja.pdf) | +| S7 Airlines | Aérea | Métricas, Registro | — | — | [Charla en ruso, marzo 2019](https://www.youtube.com/watch?v=nwG68klRpPg&t=15s) | +| SEMrush | Marketing | Producto principal | — | — | [Diapositivas en ruso, agosto 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/5_semrush.pdf) | +| scireum GmbH | Comercio electrónico | Producto principal | — | — | [Charla en alemán, febrero de 2020](https://www.youtube.com/watch?v=7QWAn5RbyR4) | +| Centinela | Desarrollador de software | Backend para el producto | — | — | [Publicación de blog en inglés, mayo 2019](https://blog.sentry.io/2019/05/16/introducing-snuba-sentrys-new-search-infrastructure) | +| SGK | Gobierno Seguridad Social | Analítica | — | — | [Diapositivas en español, noviembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup35/ClickHouse%20Meetup-Ramazan%20POLAT.pdf) | +| el seo.¿ | Analítica | Producto principal | — | — | [Diapositivas en español, noviembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup35/CH%20Presentation-%20Metehan%20Çetinkaya.pdf) | +| Sina | Noticia | — | — | — | [Diapositivas en chino, octubre 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/6.%20ClickHouse最佳实践%20高鹏_新浪.pdf) | +| SMI2 | Noticia | Analítica | — | — | [Blog Post en ruso, noviembre 2017](https://habr.com/ru/company/smi2/blog/314558/) | +| Salto | Análisis de negocios | Producto principal | — | — | [Diapositivas en español, enero 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup12/splunk.pdf) | +| Spotify | Sica | Experimentación | — | — | [Diapositivas, julio 2018](https://www.slideshare.net/glebus/using-clickhouse-for-experimentation-104247173) | +| Tencent | Grandes Datos | Procesamiento de datos | — | — | [Diapositivas en chino, octubre 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/5.%20ClickHouse大数据集群应用_李俊飞腾讯网媒事业部.pdf) | +| Más información | Taxi | Tala | — | — | [Diapositivas, febrero de 2020](https://presentations.clickhouse.tech/meetup40/uber.pdf) | +| VKontakte | Red social | Estadísticas, Registro | — | — | [Diapositivas en ruso, agosto 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/3_vk.pdf) | +| Método de codificación de datos: | Soluciones de TI | Analítica | — | — | [Diapositivas en ruso, mayo 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | +| Xiaoxin Tech | Educación | Propósito común | — | — | [Diapositivas en español, noviembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/sync-clickhouse-with-mysql-mongodb.pptx) | +| Ximalaya | Compartir audio | OLAP | — | — | [Diapositivas en español, noviembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/ximalaya.pdf) | +| Yandex Cloud | Nube pública | Producto principal | — | — | [Charla en ruso, diciembre 2019](https://www.youtube.com/watch?v=pgnak9e_E0o) | +| Yandex DataLens | Inteligencia de negocios | Producto principal | — | — | [Diapositivas en ruso, diciembre 2019](https://presentations.clickhouse.tech/meetup38/datalens.pdf) | +| Yandex Market | Comercio electrónico | Métricas, Registro | — | — | [Charla en ruso, enero 2019](https://youtu.be/_l1qP0DyBcA?t=478) | +| Yandex Metrica | Análisis web | Producto principal | 360 servidores en un clúster, 1862 servidores en un departamento | 66.41 PiB / 5.68 PiB | [Diapositivas, febrero de 2020](https://presentations.clickhouse.tech/meetup40/introduction/#13) | +| ЦВТ | Desarrollo de software | Métricas, Registro | — | — | [Blog Post, marzo 2019, en ruso](https://vc.ru/dev/62715-kak-my-stroili-monitoring-na-prometheus-clickhouse-i-elk) | +| МКБ | Banco | Supervisión del sistema web | — | — | [Diapositivas en ruso, septiembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/mkb.pdf) | +| Jinshuju 金数据 | BI Analytics | Producto principal | — | — | [Diapositivas en chino, octubre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/3.%20金数据数据架构调整方案Public.pdf) | +| Instana | Plataforma APM | Producto principal | — | — | [Publicación de Twitter](https://twitter.com/mieldonkers/status/1248884119158882304) | +| Wargaming | Juego | | — | — | [Entrevista](https://habr.com/en/post/496954/) | +| Crazypanda | Juego | | — | — | Sesión en vivo en ClickHouse meetup | +| FunCorp | Juego | | — | — | [Artículo](https://www.altinity.com/blog/migrating-from-redshift-to-clickhouse) | [Artículo Original](https://clickhouse.tech/docs/en/introduction/adopters/) diff --git a/docs/es/introduction/distinctive-features.md b/docs/es/introduction/distinctive-features.md index 370d85e7c1b..154b12a65e9 100644 --- a/docs/es/introduction/distinctive-features.md +++ b/docs/es/introduction/distinctive-features.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 4 toc_title: "Caracter\xEDsticas distintivas" --- -# Características Distintivas De ClickHouse {#distinctive-features-of-clickhouse} +# Características distintivas de ClickHouse {#distinctive-features-of-clickhouse} -## DBMS Orientado a Columnas Verdaderas {#true-column-oriented-dbms} +## DBMS orientado a columnas verdaderas {#true-column-oriented-dbms} En un verdadero DBMS orientado a columnas, no se almacenan datos adicionales con los valores. Entre otras cosas, esto significa que los valores de longitud constante deben ser compatibles, para evitar almacenar su longitud “number” al lado de los valores. Como ejemplo, mil millones de valores de tipo UInt8 deberían consumir alrededor de 1 GB sin comprimir, o esto afecta fuertemente el uso de la CPU. Es esencial almacenar los datos de forma compacta (sin “garbage”) incluso sin comprimir, ya que la velocidad de descompresión (uso de CPU) depende principalmente del volumen de datos sin comprimir. @@ -15,19 +15,19 @@ Vale la pena señalar porque hay sistemas que pueden almacenar valores de difere También vale la pena señalar que ClickHouse es un sistema de administración de bases de datos, no una sola base de datos. ClickHouse permite crear tablas y bases de datos en tiempo de ejecución, cargar datos y ejecutar consultas sin volver a configurar y reiniciar el servidor. -## Compresión De Datos {#data-compression} +## Compresión de datos {#data-compression} Algunos DBMS orientados a columnas (InfiniDB CE y MonetDB) no utilizan la compresión de datos. Sin embargo, la compresión de datos juega un papel clave para lograr un rendimiento excelente. -## Almacenamiento En Disco De Datos {#disk-storage-of-data} +## Almacenamiento en disco de datos {#disk-storage-of-data} Mantener los datos físicamente ordenados por clave principal permite extraer datos para sus valores específicos o rangos de valores con baja latencia, menos de unas pocas docenas de milisegundos. Algunos DBMS orientados a columnas (como SAP HANA y Google PowerDrill) solo pueden funcionar en RAM. Este enfoque fomenta la asignación de un presupuesto de hardware más grande que el necesario para el análisis en tiempo real. ClickHouse está diseñado para funcionar en discos duros normales, lo que significa que el costo por GB de almacenamiento de datos es bajo, pero SSD y RAM adicional también se utilizan completamente si están disponibles. -## Procesamiento Paralelo En Varios núcleos {#parallel-processing-on-multiple-cores} +## Procesamiento paralelo en varios núcleos {#parallel-processing-on-multiple-cores} Las consultas grandes se paralelizan naturalmente, tomando todos los recursos necesarios disponibles en el servidor actual. -## Procesamiento Distribuido En Varios Servidores {#distributed-processing-on-multiple-servers} +## Procesamiento distribuido en varios servidores {#distributed-processing-on-multiple-servers} Casi ninguno de los DBMS columnar mencionados anteriormente tiene soporte para el procesamiento de consultas distribuidas. En ClickHouse, los datos pueden residir en diferentes fragmentos. Cada fragmento puede ser un grupo de réplicas utilizadas para la tolerancia a errores. Todos los fragmentos se utilizan para ejecutar una consulta en paralelo, de forma transparente para el usuario. @@ -38,11 +38,11 @@ ClickHouse admite un lenguaje de consulta declarativo basado en SQL que es idén Las consultas admitidas incluyen GROUP BY, ORDER BY, subconsultas en cláusulas FROM, IN y JOIN y subconsultas escalares. No se admiten subconsultas y funciones de ventana dependientes. -## Motor Del Vector {#vector-engine} +## Motor del vector {#vector-engine} Los datos no solo se almacenan mediante columnas, sino que se procesan mediante vectores (partes de columnas), lo que permite lograr una alta eficiencia de CPU. -## Actualizaciones De Datos En Tiempo Real {#real-time-data-updates} +## Actualizaciones de datos en tiempo real {#real-time-data-updates} ClickHouse admite tablas con una clave principal. Para realizar consultas rápidamente en el rango de la clave principal, los datos se ordenan de forma incremental utilizando el árbol de combinación. Debido a esto, los datos se pueden agregar continuamente a la tabla. No se toman bloqueos cuando se ingieren nuevos datos. @@ -50,11 +50,11 @@ ClickHouse admite tablas con una clave principal. Para realizar consultas rápid Tener un dato ordenado físicamente por clave principal permite extraer datos para sus valores específicos o rangos de valores con baja latencia, menos de unas pocas docenas de milisegundos. -## Adecuado Para Consultas En línea {#suitable-for-online-queries} +## Adecuado para consultas en línea {#suitable-for-online-queries} La baja latencia significa que las consultas se pueden procesar sin demora y sin intentar preparar una respuesta por adelantado, justo en el mismo momento mientras se carga la página de la interfaz de usuario. En otras palabras, en línea. -## Soporte Para cálculos Aproximados {#support-for-approximated-calculations} +## Soporte para cálculos aproximados {#support-for-approximated-calculations} ClickHouse proporciona varias formas de intercambiar precisión por rendimiento: @@ -62,13 +62,13 @@ ClickHouse proporciona varias formas de intercambiar precisión por rendimiento: 2. Ejecutar una consulta basada en una parte (muestra) de datos y obtener un resultado aproximado. En este caso, se recuperan proporcionalmente menos datos del disco. 3. Ejecutar una agregación para un número limitado de claves aleatorias, en lugar de para todas las claves. Bajo ciertas condiciones para la distribución de claves en los datos, esto proporciona un resultado razonablemente preciso mientras se utilizan menos recursos. -## Replicación De Datos y Soporte De Integridad De Datos {#data-replication-and-data-integrity-support} +## Replicación de datos e integridad de datos {#data-replication-and-data-integrity-support} ClickHouse utiliza la replicación multi-maestro asincrónica. Después de escribir en cualquier réplica disponible, todas las réplicas restantes recuperan su copia en segundo plano. El sistema mantiene datos idénticos en diferentes réplicas. La recuperación después de la mayoría de las fallas se realiza automáticamente, o semiautomáticamente en casos complejos. Para obtener más información, consulte la sección [Replicación de datos](../engines/table-engines/mergetree-family/replication.md). -## Características Que Pueden Considerarse Desventajas {#clickhouse-features-that-can-be-considered-disadvantages} +## Características que pueden considerarse desventajas {#clickhouse-features-that-can-be-considered-disadvantages} 1. No hay transacciones completas. 2. Falta de capacidad para modificar o eliminar datos ya insertados con alta tasa y baja latencia. Hay eliminaciones y actualizaciones por lotes disponibles para limpiar o modificar datos, por ejemplo, para cumplir con [GDPR](https://gdpr-info.eu). diff --git a/docs/es/introduction/history.md b/docs/es/introduction/history.md index 5c24bac9405..7311fa01959 100644 --- a/docs/es/introduction/history.md +++ b/docs/es/introduction/history.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 7 toc_title: Historia --- -# Historial De ClickHouse {#clickhouse-history} +# Historial de ClickHouse {#clickhouse-history} ClickHouse se ha desarrollado inicialmente para alimentar [El Yandex.Métrica](https://metrica.yandex.com/), [la segunda plataforma de análisis web más grande del mundo](http://w3techs.com/technologies/overview/traffic_analysis/all), y sigue siendo el componente central de este sistema. Con más de 13 billones de registros en la base de datos y más de 20 mil millones de eventos diarios, ClickHouse permite generar informes personalizados sobre la marcha directamente a partir de datos no agregados. Este artículo cubre brevemente los objetivos de ClickHouse en las primeras etapas de su desarrollo. @@ -13,7 +13,7 @@ El Yandex.Metrica construye informes personalizados sobre la marcha basados en h A partir de abril de 2014, Yandex.Metrica estaba rastreando alrededor de 12 mil millones de eventos (vistas de páginas y clics) diariamente. Todos estos eventos deben almacenarse para crear informes personalizados. Una sola consulta puede requerir escanear millones de filas en unos pocos cientos de milisegundos, o cientos de millones de filas en solo unos segundos. -## Uso En Yandex.Metrica y Otros Servicios De Yandex {#usage-in-yandex-metrica-and-other-yandex-services} +## Uso en Yandex.Metrica y otros servicios de Yandex {#usage-in-yandex-metrica-and-other-yandex-services} ClickHouse sirve para múltiples propósitos en Yandex.Métrica. Su tarea principal es crear informes en modo en línea utilizando datos no agregados. Utiliza un clúster de 374 servidores, que almacenan más de 20,3 billones de filas en la base de datos. El volumen de datos comprimidos es de aproximadamente 2 PB, sin tener en cuenta duplicados y réplicas. El volumen de datos sin comprimir (en formato TSV) sería de aproximadamente 17 PB. @@ -28,7 +28,7 @@ ClickHouse también juega un papel clave en los siguientes procesos: Hoy en día, hay varias docenas de instalaciones de ClickHouse en otros servicios y departamentos de Yandex: verticales de búsqueda, comercio electrónico, publicidad, análisis de negocios, desarrollo móvil, servicios personales y otros. -## Datos Agregados y No Agregados {#aggregated-and-non-aggregated-data} +## Datos agregados y no agregados {#aggregated-and-non-aggregated-data} Existe una opinión generalizada de que para calcular las estadísticas de manera efectiva, debe agregar datos ya que esto reduce el volumen de datos. diff --git a/docs/es/introduction/index.md b/docs/es/introduction/index.md index 4488eeed6fc..7026dc800e4 100644 --- a/docs/es/introduction/index.md +++ b/docs/es/introduction/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa -toc_folder_title: Introduction +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "Implantaci\xF3n" toc_priority: 1 --- diff --git a/docs/es/introduction/performance.md b/docs/es/introduction/performance.md index 00fe622cf46..01640439128 100644 --- a/docs/es/introduction/performance.md +++ b/docs/es/introduction/performance.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 6 toc_title: Rendimiento --- @@ -11,21 +11,21 @@ De acuerdo con los resultados de las pruebas internas en Yandex, ClickHouse mues Numerosos puntos de referencia independientes llegaron a conclusiones similares. No son difíciles de encontrar mediante una búsqueda en Internet, o se puede ver [nuestra pequeña colección de enlaces relacionados](https://clickhouse.tech/#independent-benchmarks). -## Rendimiento Para Una única Consulta Grande {#throughput-for-a-single-large-query} +## Rendimiento para una única consulta grande {#throughput-for-a-single-large-query} El rendimiento se puede medir en filas por segundo o megabytes por segundo. Si los datos se colocan en la caché de la página, una consulta que no es demasiado compleja se procesa en hardware moderno a una velocidad de aproximadamente 2-10 GB / s de datos sin comprimir en un solo servidor (para los casos más sencillos, la velocidad puede alcanzar 30 GB / s). Si los datos no se colocan en la memoria caché de la página, la velocidad depende del subsistema de disco y la velocidad de compresión de datos. Por ejemplo, si el subsistema de disco permite leer datos a 400 MB/s y la tasa de compresión de datos es 3, se espera que la velocidad sea de alrededor de 1,2 GB/s. Para obtener la velocidad en filas por segundo, divida la velocidad en bytes por segundo por el tamaño total de las columnas utilizadas en la consulta. Por ejemplo, si se extraen 10 bytes de columnas, se espera que la velocidad sea de alrededor de 100-200 millones de filas por segundo. La velocidad de procesamiento aumenta casi linealmente para el procesamiento distribuido, pero solo si el número de filas resultantes de la agregación o la clasificación no es demasiado grande. -## Latencia Al Procesar Consultas Cortas {#latency-when-processing-short-queries} +## Latencia al procesar consultas cortas {#latency-when-processing-short-queries} Si una consulta usa una clave principal y no selecciona demasiadas columnas y filas para procesar (cientos de miles), puede esperar menos de 50 milisegundos de latencia (dígitos individuales de milisegundos en el mejor de los casos) si los datos se colocan en la memoria caché de la página. De lo contrario, la latencia está dominada principalmente por el número de búsquedas. Si utiliza unidades de disco giratorias, para un sistema que no está sobrecargado, la latencia se puede estimar con esta fórmula: `seek time (10 ms) * count of columns queried * count of data parts`. -## Rendimiento Al Procesar Una Gran Cantidad De Consultas Cortas {#throughput-when-processing-a-large-quantity-of-short-queries} +## Rendimiento al procesar una gran cantidad de consultas cortas {#throughput-when-processing-a-large-quantity-of-short-queries} En las mismas condiciones, ClickHouse puede manejar varios cientos de consultas por segundo en un solo servidor (hasta varios miles en el mejor de los casos). Dado que este escenario no es típico para DBMS analíticos, se recomienda esperar un máximo de 100 consultas por segundo. -## Rendimiento Al Insertar Datos {#performance-when-inserting-data} +## Rendimiento al insertar datos {#performance-when-inserting-data} Recomendamos insertar datos en paquetes de al menos 1000 filas o no más de una sola solicitud por segundo. Al insertar en una tabla MergeTree desde un volcado separado por tabuladores, la velocidad de inserción puede ser de 50 a 200 MB/s. Si las filas insertadas tienen alrededor de 1 Kb de tamaño, la velocidad será de 50,000 a 200,000 filas por segundo. Si las filas son pequeñas, el rendimiento puede ser mayor en filas por segundo (en los datos del sistema Banner -`>` 500.000 filas por segundo; en datos de grafito -`>` 1.000.000 de filas por segundo). Para mejorar el rendimiento, puede realizar varias consultas INSERT en paralelo, que se escala linealmente. diff --git a/docs/es/operations/access-rights.md b/docs/es/operations/access-rights.md index a0c8540d310..5f00dc901d8 100644 --- a/docs/es/operations/access-rights.md +++ b/docs/es/operations/access-rights.md @@ -1,113 +1,143 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 48 -toc_title: Derechos de acceso +toc_title: "Control de acceso y gesti\xF3n de cuentas" --- -# Derechos De Acceso {#access-rights} +# Control de acceso y gestión de cuentas {#access-control} -Los usuarios y los derechos de acceso se configuran en la configuración del usuario. Esto suele ser `users.xml`. +ClickHouse admite la administración de control de acceso basada en [RBAC](https://en.wikipedia.org/wiki/Role-based_access_control) enfoque. -Los usuarios se registran en el `users` apartado. Aquí hay un fragmento de la `users.xml` file: +Entidades de acceso de ClickHouse: +- [Cuenta de usuario](#user-account-management) +- [Rol](#role-management) +- [Política de fila](#row-policy-management) +- [Perfil de configuración](#settings-profiles-management) +- [Cuota](#quotas-management) -``` xml - - - - - - +- Servidor [archivos de configuración](configuration-files.md) `users.xml` y `config.xml`. - - +!!! note "Advertencia" + No puede administrar la misma entidad de acceso mediante ambos métodos de configuración simultáneamente. - - default +## Uso {#access-control-usage} - - default - +De forma predeterminada, el servidor ClickHouse proporciona la cuenta de usuario `default` que no está permitido usar control de acceso controlado por SQL y administración de cuentas, pero tiene todos los derechos y permisos. El `default` cuenta de usuario se utiliza en cualquier caso cuando el nombre de usuario no está definido, por ejemplo, al iniciar sesión desde el cliente o en consultas distribuidas. En el procesamiento de consultas distribuidas se utiliza una cuenta de usuario predeterminada, si la configuración del servidor o clúster no [usuario y contraseña](../engines/table-engines/special/distributed.md) propiedad. - - - - - web - default - - test - - - test - - - -``` +Si acaba de comenzar a usar ClickHouse, puede usar el siguiente escenario: -Puede ver una declaración de dos usuarios: `default`y`web`. Hemos añadido el `web` usuario por separado. +1. [Permitir](#enabling-access-control) Control de acceso basado en SQL y gestión de cuentas `default` usuario. +2. Inicie sesión bajo el `default` cuenta de usuario y crear todos los usuarios. No olvides crear una cuenta de administrador (`GRANT ALL ON *.* WITH GRANT OPTION TO admin_user_account`). +3. [Restringir permisos](settings/permissions-for-queries.md#permissions_for_queries) para el `default` usuario y deshabilitar el control de acceso impulsado por SQL y la administración de cuentas para ello. -El `default` usuario se elige en los casos en que no se pasa el nombre de usuario. El `default` usuario también se utiliza para el procesamiento de consultas distribuidas, si la configuración del servidor o clúster no `user` y `password` (véase la sección sobre el [Distribuido](../engines/table-engines/special/distributed.md) motor). +### Propiedades de la solución actual {#access-control-properties} -The user that is used for exchanging information between servers combined in a cluster must not have substantial restrictions or quotas – otherwise, distributed queries will fail. +- Puede conceder permisos para bases de datos y tablas incluso si no existen. +- Si se eliminó una tabla, no se revocarán todos los privilegios que corresponden a esta tabla. Por lo tanto, si se crea una nueva tabla más tarde con el mismo nombre, todos los privilegios vuelven a ser reales. Para revocar los privilegios correspondientes a la tabla eliminada, debe realizar, por ejemplo, el `REVOKE ALL PRIVILEGES ON db.table FROM ALL` consulta. +- No hay ninguna configuración de por vida para los privilegios. -La contraseña se especifica en texto sin cifrar (no recomendado) o en SHA-256. El hash no está salado. En este sentido, no debe considerar estas contraseñas como proporcionar seguridad contra posibles ataques maliciosos. Más bien, son necesarios para la protección de los empleados. +## Cuenta de usuario {#user-account-management} -Se especifica una lista de redes desde las que se permite el acceso. En este ejemplo, la lista de redes para ambos usuarios se carga desde un archivo independiente (`/etc/metrika.xml`) que contiene el `networks` sustitución. Aquí hay un fragmento de eso: +Una cuenta de usuario es una entidad de acceso que permite autorizar a alguien en ClickHouse. Una cuenta de usuario contiene: -``` xml - - ... - - ::/64 - 203.0.113.0/24 - 2001:DB8::/32 - ... - - -``` +- Información de identificación. +- [Privilegio](../sql-reference/statements/grant.md#grant-privileges) que definen un ámbito de consultas que el usuario puede realizar. +- Hosts desde los que se permite la conexión al servidor ClickHouse. +- Roles otorgados y predeterminados. +- Configuración con sus restricciones que se aplican de forma predeterminada en el inicio de sesión del usuario. +- Perfiles de configuración asignados. -Puede definir esta lista de redes directamente en `users.xml` o en un archivo en el `users.d` (para obtener más información, consulte la sección “[Archivos de configuración](configuration-files.md#configuration_files)”). +Los privilegios a una cuenta de usuario pueden ser otorgados por el [GRANT](../sql-reference/statements/grant.md) consulta o asignando [rol](#role-management). Para revocar privilegios de un usuario, ClickHouse proporciona el [REVOKE](../sql-reference/statements/revoke.md) consulta. Para listar los privilegios de un usuario, utilice - [SHOW GRANTS](../sql-reference/statements/show.md#show-grants-statement) instrucción. -La configuración incluye comentarios que explican cómo abrir el acceso desde todas partes. +Consultas de gestión: -Para su uso en producción, sólo especifique `ip` elementos (direcciones IP y sus máscaras), ya que usan `host` y `hoost_regexp` podría causar latencia adicional. +- [CREATE USER](../sql-reference/statements/create.md#create-user-statement) +- [ALTER USER](../sql-reference/statements/alter.md#alter-user-statement) +- [DROP USER](../sql-reference/statements/misc.md#drop-user-statement) +- [SHOW CREATE USER](../sql-reference/statements/show.md#show-create-user-statement) -A continuación se especifica el perfil de configuración de usuario (consulte la sección “[Perfiles de configuración](settings/settings-profiles.md)”. Puede especificar el perfil predeterminado, `default'`. El perfil puede tener cualquier nombre. Puede especificar el mismo perfil para diferentes usuarios. Lo más importante que puede escribir en el perfil de configuración es `readonly=1`, que asegura el acceso de sólo lectura. A continuación, especifique la cuota que se utilizará (consulte la sección “[Cuota](quotas.md#quotas)”). Puede especificar la cuota predeterminada: `default`. It is set in the config by default to only count resource usage, without restricting it. The quota can have any name. You can specify the same quota for different users – in this case, resource usage is calculated for each user individually. +### Ajustes Aplicación {#access-control-settings-applying} -En el opcional `` sección, también puede especificar una lista de bases de datos a las que el usuario puede acceder. De forma predeterminada, todas las bases de datos están disponibles para el usuario. Puede especificar el `default` base. En este caso, el usuario recibirá acceso a la base de datos de forma predeterminada. +Los ajustes se pueden establecer de diferentes maneras: para una cuenta de usuario, en sus roles y perfiles de configuración concedidos. En un inicio de sesión de usuario, si se establece una configuración en diferentes entidades de acceso, el valor y las restricciones de esta configuración se aplican mediante las siguientes prioridades (de mayor a menor): -En el opcional `` sección, también puede especificar una lista de diccionarios a los que el usuario puede acceder. De forma predeterminada, todos los diccionarios están disponibles para el usuario. +1. Configuración de la cuenta de usuario. +2. La configuración de los roles predeterminados de la cuenta de usuario. Si se establece una configuración en algunos roles, el orden de la configuración que se aplica no está definido. +3. La configuración de los perfiles de configuración asignados a un usuario o a sus roles predeterminados. Si se establece una configuración en algunos perfiles, el orden de aplicación de la configuración no está definido. +4. Ajustes aplicados a todo el servidor de forma predeterminada o desde el [perfil predeterminado](server-configuration-parameters/settings.md#default-profile). -Acceso a la `system` base de datos siempre está permitida (ya que esta base de datos se utiliza para procesar consultas). +## Rol {#role-management} -El usuario puede obtener una lista de todas las bases de datos y tablas en ellos mediante el uso de `SHOW` consultas o tablas del sistema, incluso si no se permite el acceso a bases de datos individuales. +Role es un contenedor para las entidades de acceso que se pueden conceder a una cuenta de usuario. -El acceso a la base de datos no está [sólo lectura](settings/permissions-for-queries.md#settings_readonly) configuración. No puede conceder acceso completo a una base de datos y `readonly` acceso a otro. +El rol contiene: + +- [Privilegio](../sql-reference/statements/grant.md#grant-privileges) +- Configuración y restricciones +- Lista de funciones concedidas + +Consultas de gestión: + +- [CREATE ROLE](../sql-reference/statements/create.md#create-role-statement) +- [ALTER ROLE](../sql-reference/statements/alter.md#alter-role-statement) +- [DROP ROLE](../sql-reference/statements/misc.md#drop-role-statement) +- [SET ROLE](../sql-reference/statements/misc.md#set-role-statement) +- [SET DEFAULT ROLE](../sql-reference/statements/misc.md#set-default-role-statement) +- [SHOW CREATE ROLE](../sql-reference/statements/show.md#show-create-role-statement) + +Los privilegios a un rol pueden ser otorgados por el [GRANT](../sql-reference/statements/grant.md) consulta. Para revocar privilegios de un rol, ClickHouse proporciona el [REVOKE](../sql-reference/statements/revoke.md) consulta. + +## Política de fila {#row-policy-management} + +La directiva de filas es un filtro que define qué filas está disponible para un usuario o para un rol. La directiva de filas contiene filtros para una tabla específica y una lista de roles y/o usuarios que deben usar esta directiva de filas. + +Consultas de gestión: + +- [CREATE ROW POLICY](../sql-reference/statements/create.md#create-row-policy-statement) +- [ALTER ROW POLICY](../sql-reference/statements/alter.md#alter-row-policy-statement) +- [DROP ROW POLICY](../sql-reference/statements/misc.md#drop-row-policy-statement) +- [SHOW CREATE ROW POLICY](../sql-reference/statements/show.md#show-create-row-policy-statement) + +## Perfil de configuración {#settings-profiles-management} + +El perfil de configuración es una colección de [configuración](settings/index.md). El perfil de configuración contiene configuraciones y restricciones, y una lista de roles y/o usuarios a los que se aplica esta cuota. + +Consultas de gestión: + +- [CREATE SETTINGS PROFILE](../sql-reference/statements/create.md#create-settings-profile-statement) +- [ALTER SETTINGS PROFILE](../sql-reference/statements/alter.md#alter-settings-profile-statement) +- [DROP SETTINGS PROFILE](../sql-reference/statements/misc.md#drop-settings-profile-statement) +- [SHOW CREATE SETTINGS PROFILE](../sql-reference/statements/show.md#show-create-settings-profile-statement) + +## Cuota {#quotas-management} + +La cuota limita el uso de recursos. Ver [Cuota](quotas.md). + +La cuota contiene un conjunto de límites para algunas duraciones y una lista de roles y / o usuarios que deben usar esta cuota. + +Consultas de gestión: + +- [CREATE QUOTA](../sql-reference/statements/create.md#create-quota-statement) +- [ALTER QUOTA](../sql-reference/statements/alter.md#alter-quota-statement) +- [DROP QUOTA](../sql-reference/statements/misc.md#drop-quota-statement) +- [SHOW CREATE QUOTA](../sql-reference/statements/show.md#show-create-quota-statement) + +## Habilitación del control de acceso basado en SQL y la administración de cuentas {#enabling-access-control} + +- Configure un directorio para el almacenamiento de configuraciones. + + ClickHouse almacena las configuraciones de entidades de acceso en la carpeta [access\_control\_path](server-configuration-parameters/settings.md#access_control_path) parámetro de configuración del servidor. + +- Habilite el control de acceso controlado por SQL y la administración de cuentas para al menos una cuenta de usuario. + + De forma predeterminada, el control de acceso controlado por SQL y la administración de cuentas se activan para todos los usuarios. Debe configurar al menos un usuario en el `users.xml` archivo de configuración y asigne 1 al [access\_management](settings/settings-users.md#access_management-user-setting) configuración. [Artículo Original](https://clickhouse.tech/docs/en/operations/access_rights/) diff --git a/docs/es/operations/backup.md b/docs/es/operations/backup.md index 8f47ef3f9bd..f1e5b3d3e09 100644 --- a/docs/es/operations/backup.md +++ b/docs/es/operations/backup.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 49 toc_title: Copia de seguridad de datos --- -# Copia De Seguridad De Datos {#data-backup} +# Copia de seguridad de datos {#data-backup} Mientras [replicación](../engines/table-engines/mergetree-family/replication.md) provides protection from hardware failures, it does not protect against human errors: accidental deletion of data, deletion of the wrong table or a table on the wrong cluster, and software bugs that result in incorrect data processing or data corruption. In many cases mistakes like these will affect all replicas. ClickHouse has built-in safeguards to prevent some types of mistakes — for example, by default [no puede simplemente eliminar tablas con un motor similar a MergeTree que contenga más de 50 Gb de datos](https://github.com/ClickHouse/ClickHouse/blob/v18.14.18-stable/programs/server/config.xml#L322-L330). Sin embargo, estas garantías no cubren todos los casos posibles y pueden eludirse. @@ -16,21 +16,21 @@ Cada empresa tiene diferentes recursos disponibles y requisitos comerciales, por !!! note "Nota" Tenga en cuenta que si realizó una copia de seguridad de algo y nunca intentó restaurarlo, es probable que la restauración no funcione correctamente cuando realmente la necesite (o al menos tomará más tiempo de lo que las empresas pueden tolerar). Por lo tanto, cualquiera que sea el enfoque de copia de seguridad que elija, asegúrese de automatizar el proceso de restauración también y practicarlo en un clúster de ClickHouse de repuesto regularmente. -## Duplicar Datos De Origen En Otro Lugar {#duplicating-source-data-somewhere-else} +## Duplicar datos de origen en otro lugar {#duplicating-source-data-somewhere-else} A menudo, los datos que se ingieren en ClickHouse se entregan a través de algún tipo de cola persistente, como [Acerca de nosotros](https://kafka.apache.org). En este caso, es posible configurar un conjunto adicional de suscriptores que leerá el mismo flujo de datos mientras se escribe en ClickHouse y lo almacenará en almacenamiento en frío en algún lugar. La mayoría de las empresas ya tienen algún almacenamiento en frío recomendado por defecto, que podría ser un almacén de objetos o un sistema de archivos distribuido como [HDFS](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HdfsDesign.html). -## Instantáneas Del Sistema De Archivos {#filesystem-snapshots} +## Instantáneas del sistema de archivos {#filesystem-snapshots} Algunos sistemas de archivos locales proporcionan funcionalidad de instantánea (por ejemplo, [ZFS](https://en.wikipedia.org/wiki/ZFS)), pero podrían no ser la mejor opción para servir consultas en vivo. Una posible solución es crear réplicas adicionales con este tipo de sistema de archivos y excluirlas del [Distribuido](../engines/table-engines/special/distributed.md) tablas que se utilizan para `SELECT` consulta. Las instantáneas en tales réplicas estarán fuera del alcance de cualquier consulta que modifique los datos. Como beneficio adicional, estas réplicas podrían tener configuraciones de hardware especiales con más discos conectados por servidor, lo que sería rentable. -## Método De codificación De Datos: {#clickhouse-copier} +## Método de codificación de datos: {#clickhouse-copier} [Método de codificación de datos:](utilities/clickhouse-copier.md) es una herramienta versátil que se creó inicialmente para volver a dividir tablas de tamaño petabyte. También se puede usar con fines de copia de seguridad y restauración porque copia datos de forma fiable entre tablas y clústeres de ClickHouse. Para volúmenes de datos más pequeños, un simple `INSERT INTO ... SELECT ...` a tablas remotas podría funcionar también. -## Manipulaciones Con Piezas {#manipulations-with-parts} +## Manipulaciones con piezas {#manipulations-with-parts} ClickHouse permite usar el `ALTER TABLE ... FREEZE PARTITION ...` consulta para crear una copia local de particiones de tabla. Esto se implementa utilizando enlaces duros al `/var/lib/clickhouse/shadow/` carpeta, por lo que generalmente no consume espacio adicional en disco para datos antiguos. Las copias creadas de archivos no son manejadas por el servidor ClickHouse, por lo que puede dejarlas allí: tendrá una copia de seguridad simple que no requiere ningún sistema externo adicional, pero seguirá siendo propenso a problemas de hardware. Por esta razón, es mejor copiarlos de forma remota en otra ubicación y luego eliminar las copias locales. Los sistemas de archivos distribuidos y los almacenes de objetos siguen siendo una buena opción para esto, pero los servidores de archivos conectados normales con una capacidad lo suficientemente grande podrían funcionar también (en este caso, la transferencia ocurrirá a través del sistema de archivos de red o tal vez [rsync](https://en.wikipedia.org/wiki/Rsync)). diff --git a/docs/es/operations/configuration-files.md b/docs/es/operations/configuration-files.md index fa5e30af42e..578d7b3f9ff 100644 --- a/docs/es/operations/configuration-files.md +++ b/docs/es/operations/configuration-files.md @@ -1,18 +1,18 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 50 toc_title: "Archivos de configuraci\xF3n" --- -# Archivos De configuración {#configuration_files} +# Archivos de configuración {#configuration_files} ClickHouse admite la administración de configuración de varios archivos. El archivo de configuración del servidor principal es `/etc/clickhouse-server/config.xml`. Otros archivos deben estar en el `/etc/clickhouse-server/config.d` directorio. !!! note "Nota" Todos los archivos de configuración deben estar en formato XML. Además, deben tener el mismo elemento raíz, generalmente ``. -Algunos valores especificados en el archivo de configuración principal se pueden anular en otros archivos de configuración. El `replace` o `remove` los atributos pueden ser especificada para los elementos de estos archivos de configuración. +Algunos valores especificados en el archivo de configuración principal se pueden anular en otros archivos de configuración. El `replace` o `remove` se pueden especificar atributos para los elementos de estos archivos de configuración. Si no se especifica ninguno, combina el contenido de los elementos de forma recursiva, reemplazando los valores de los elementos secundarios duplicados. @@ -22,11 +22,11 @@ Si `remove` se especifica, elimina el elemento. La configuración también puede definir “substitutions”. Si un elemento tiene el `incl` atributo, la sustitución correspondiente del archivo se utilizará como el valor. De forma predeterminada, la ruta al archivo con sustituciones es `/etc/metrika.xml`. Esto se puede cambiar en el [include\_from](server-configuration-parameters/settings.md#server_configuration_parameters-include_from) elemento en la configuración del servidor. Los valores de sustitución se especifican en `/yandex/substitution_name` elementos en este archivo. Si una sustitución especificada en `incl` no existe, se registra en el registro. Para evitar que ClickHouse registre las sustituciones que faltan, especifique `optional="true"` atributo (por ejemplo, ajustes para [macro](server-configuration-parameters/settings.md)). -Las sustituciones pueden también ser realizada a partir de Cuidador. Para ello, especifique el atributo `from_zk = "/path/to/node"`. El valor del elemento se sustituye por el contenido del nodo en `/path/to/node` en ZooKeeper. También puede colocar un subárbol XML completo en el nodo ZooKeeper y se insertará completamente en el elemento de origen. +Las sustituciones también se pueden realizar desde ZooKeeper. Para hacer esto, especifique el atributo `from_zk = "/path/to/node"`. El valor del elemento se sustituye por el contenido del nodo en `/path/to/node` en ZooKeeper. También puede colocar un subárbol XML completo en el nodo ZooKeeper y se insertará completamente en el elemento de origen. El `config.xml` file puede especificar una configuración separada con configuraciones de usuario, perfiles y cuotas. La ruta relativa a esta configuración se establece en el `users_config` elemento. Por defecto, es `users.xml`. Si `users_config` se omite, la configuración de usuario, los perfiles y las cuotas se especifican directamente en `config.xml`. -Configuración de usuarios puede ser dividido en archivos separados similar a `config.xml` y `config.d/`. +La configuración de los usuarios se puede dividir en archivos separados similares a `config.xml` y `config.d/`. El nombre del directorio se define como `users_config` sin `.xml` postfix concatenado con `.d`. Directorio `users.d` se utiliza por defecto, como `users_config` por defecto `users.xml`. Por ejemplo, puede tener un archivo de configuración separado para cada usuario como este: diff --git a/docs/es/operations/index.md b/docs/es/operations/index.md index ba1e0717d0c..9a928fa0f01 100644 --- a/docs/es/operations/index.md +++ b/docs/es/operations/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa -toc_folder_title: Operations +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "Operaci\xF3n" toc_priority: 41 toc_title: "Implantaci\xF3n" --- @@ -25,4 +25,4 @@ El manual de operaciones de ClickHouse consta de las siguientes secciones princi - [Configuración](settings/index.md) - [Utilidad](utilities/index.md) -[Artículo Original](https://clickhouse.tech/docs/en/operations/) +{## [Artículo Original](https://clickhouse.tech/docs/en/operations/) ##} diff --git a/docs/es/operations/monitoring.md b/docs/es/operations/monitoring.md index 91eddb7fbff..173d23aa041 100644 --- a/docs/es/operations/monitoring.md +++ b/docs/es/operations/monitoring.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 45 toc_title: Monitoreo --- @@ -12,7 +12,7 @@ Usted puede monitorear: - Utilización de recursos de hardware. - Métricas del servidor ClickHouse. -## Utilización De Recursos {#resource-utilization} +## Utilización de recursos {#resource-utilization} ClickHouse no supervisa el estado de los recursos de hardware por sí mismo. @@ -24,7 +24,7 @@ Se recomienda encarecidamente configurar la supervisión para: - Utilización del sistema de almacenamiento, RAM y red. -## Métricas Del Servidor De Clickhouse {#clickhouse-server-metrics} +## Métricas del servidor ClickHouse {#clickhouse-server-metrics} El servidor ClickHouse tiene instrumentos integrados para el monitoreo de estado propio. @@ -33,12 +33,14 @@ Para realizar un seguimiento de los eventos del servidor, use los registros del ClickHouse recoge: - Diferentes métricas de cómo el servidor utiliza recursos computacionales. -- Común de la estadística en el procesamiento de la consulta. +- Estadísticas comunes sobre el procesamiento de consultas. Puede encontrar métricas en el [sistema.métricas](../operations/system-tables.md#system_tables-metrics), [sistema.evento](../operations/system-tables.md#system_tables-events), y [sistema.asynchronous\_metrics](../operations/system-tables.md#system_tables-asynchronous_metrics) tabla. Puede configurar ClickHouse para exportar métricas a [Grafito](https://github.com/graphite-project). Ver el [Sección de grafito](server-configuration-parameters/settings.md#server_configuration_parameters-graphite) en el archivo de configuración del servidor ClickHouse. Antes de configurar la exportación de métricas, debe configurar Graphite siguiendo sus [guiar](https://graphite.readthedocs.io/en/latest/install.html). +Puede configurar ClickHouse para exportar métricas a [Prometeo](https://prometheus.io). Ver el [Sección Prometheus](server-configuration-parameters/settings.md#server_configuration_parameters-prometheus) en el archivo de configuración del servidor ClickHouse. Antes de configurar la exportación de métricas, debe configurar Prometheus siguiendo su oficial [guiar](https://prometheus.io/docs/prometheus/latest/installation/). + Además, puede supervisar la disponibilidad del servidor a través de la API HTTP. Enviar el `HTTP GET` solicitud de `/ping`. Si el servidor está disponible, responde con `200 OK`. Para supervisar servidores en una configuración de clúster, debe establecer [max\_replica\_delay\_for\_distributed\_queries](settings/settings.md#settings-max_replica_delay_for_distributed_queries) parámetro y utilizar el recurso HTTP `/replicas_status`. Una solicitud para `/replicas_status` devoluciones `200 OK` si la réplica está disponible y no se retrasa detrás de las otras réplicas. Si una réplica se retrasa, devuelve `503 HTTP_SERVICE_UNAVAILABLE` con información sobre la brecha. diff --git a/docs/es/operations/optimizing-performance/index.md b/docs/es/operations/optimizing-performance/index.md index 126096db767..d2796c6e0d3 100644 --- a/docs/es/operations/optimizing-performance/index.md +++ b/docs/es/operations/optimizing-performance/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa -toc_folder_title: Optimizing Performance +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "Optimizaci\xF3n del rendimiento" toc_priority: 52 --- diff --git a/docs/es/operations/optimizing-performance/sampling-query-profiler.md b/docs/es/operations/optimizing-performance/sampling-query-profiler.md index 98434f582b9..00ea86b6e8c 100644 --- a/docs/es/operations/optimizing-performance/sampling-query-profiler.md +++ b/docs/es/operations/optimizing-performance/sampling-query-profiler.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 54 toc_title: "Generaci\xF3n de perfiles de consultas" --- -# Analizador De Consultas De Muestreo {#sampling-query-profiler} +# Analizador de consultas de muestreo {#sampling-query-profiler} ClickHouse ejecuta el generador de perfiles de muestreo que permite analizar la ejecución de consultas. Utilizando el generador de perfiles puede encontrar rutinas de código fuente que se utilizan con más frecuencia durante la ejecución de la consulta. Puede rastrear el tiempo de CPU y el tiempo de reloj de pared invertido, incluido el tiempo de inactividad. diff --git a/docs/es/operations/performance-test.md b/docs/es/operations/performance-test.md index a65d196509b..d0beb7bd2b4 100644 --- a/docs/es/operations/performance-test.md +++ b/docs/es/operations/performance-test.md @@ -1,17 +1,17 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 54 toc_title: Prueba de hardware --- -# Cómo Probar Su Hardware Con ClickHouse {#how-to-test-your-hardware-with-clickhouse} +# Cómo probar su hardware con ClickHouse {#how-to-test-your-hardware-with-clickhouse} Con esta instrucción, puede ejecutar una prueba de rendimiento básica de ClickHouse en cualquier servidor sin instalar paquetes de ClickHouse. 1. Ir a “commits” página: https://github.com/ClickHouse/ClickHouse/commits/master -2. Haga clic en la primera marca de verificación verde o cruz roja con verde “ClickHouse Build Check” y haga clic en el “Details” enlace cerca “ClickHouse Build Check”. +2. Haga clic en la primera marca de verificación verde o cruz roja con verde “ClickHouse Build Check” y haga clic en el “Details” enlace cerca “ClickHouse Build Check”. No existe tal enlace en algunas confirmaciones, por ejemplo, confirmaciones con documentación. En este caso, elija la confirmación más cercana que tenga este enlace. 3. Copie el enlace a “clickhouse” binario para amd64 o aarch64. @@ -65,7 +65,7 @@ Con esta instrucción, puede ejecutar una prueba de rendimiento básica de Click ./clickhouse client --query "SELECT count() FROM hits_100m_obfuscated" 100000000 -1. Edite el benchmark-new.sh, cambie “clickhouse-client” a “./clickhouse client” y añadir “–max\_memory\_usage 100000000000” parámetro. +1. Edite el benchmark-new.sh, cambie `clickhouse-client` a `./clickhouse client` y añadir `–-max_memory_usage 100000000000` parámetro. @@ -79,4 +79,4 @@ Con esta instrucción, puede ejecutar una prueba de rendimiento básica de Click 1. Envíe los números y la información sobre la configuración de su hardware a clickhouse-feedback@yandex-team.com -Todos los resultados se publican aquí: https://clickhouse.tecnología/benchmark\_hardware.HTML +Todos los resultados se publican aquí: https://clickhouse.tecnología/punto de referencia/hardware/ diff --git a/docs/es/operations/quotas.md b/docs/es/operations/quotas.md index af6d09a54ce..9d84ce21339 100644 --- a/docs/es/operations/quotas.md +++ b/docs/es/operations/quotas.md @@ -1,14 +1,14 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 51 toc_title: Cuota --- # Cuota {#quotas} -Las cuotas le permiten limitar el uso de recursos durante un período de tiempo, o simplemente realizar un seguimiento del uso de recursos. -Las cuotas se configuran en la configuración del usuario. Esto suele ser ‘users.xml’. +Las cuotas le permiten limitar el uso de recursos durante un período de tiempo o realizar un seguimiento del uso de recursos. +Las cuotas se configuran en la configuración del usuario, que generalmente ‘users.xml’. El sistema también tiene una característica para limitar la complejidad de una sola consulta. Vea la sección “Restrictions on query complexity”). @@ -39,7 +39,7 @@ Veamos la sección del ‘users.xml’ fichero que define las cuotas. ``` -De forma predeterminada, la cuota sólo pistas de consumo de recursos por cada hora, sin limitación de uso. +De forma predeterminada, la cuota realiza un seguimiento del consumo de recursos para cada hora, sin limitar el uso. El consumo de recursos calculado para cada intervalo se envía al registro del servidor después de cada solicitud. ``` xml @@ -68,7 +68,7 @@ El consumo de recursos calculado para cada intervalo se envía al registro del s ``` -Para el ‘statbox’ Las restricciones se establecen por cada hora y por cada 24 horas (86.400 segundos). El intervalo de tiempo se cuenta a partir de un momento fijo definido por la implementación en el tiempo. En otras palabras, el intervalo de 24 horas no necesariamente comienza a medianoche. +Para el ‘statbox’ Las restricciones se establecen por cada hora y por cada 24 horas (86.400 segundos). El intervalo de tiempo se cuenta, a partir de un momento fijo definido por la implementación en el tiempo. En otras palabras, el intervalo de 24 horas no necesariamente comienza a medianoche. Cuando finaliza el intervalo, se borran todos los valores recopilados. Para la siguiente hora, el cálculo de la cuota comienza de nuevo. @@ -78,15 +78,15 @@ Estas son las cantidades que se pueden restringir: `errors` – The number of queries that threw an exception. -`result_rows` – The total number of rows given as the result. +`result_rows` – The total number of rows given as a result. -`read_rows` – The total number of source rows read from tables for running the query, on all remote servers. +`read_rows` – The total number of source rows read from tables for running the query on all remote servers. `execution_time` – The total query execution time, in seconds (wall time). Si se excede el límite durante al menos un intervalo de tiempo, se lanza una excepción con un texto sobre qué restricción se excedió, para qué intervalo y cuándo comienza el nuevo intervalo (cuando se pueden enviar consultas nuevamente). -Las cuotas pueden usar el “quota key” característica con el fin de informar sobre los recursos para múltiples claves de forma independiente. Aquí hay un ejemplo de esto: +Las cuotas pueden usar el “quota key” característica para informar sobre los recursos para múltiples claves de forma independiente. Aquí hay un ejemplo de esto: ``` xml @@ -97,7 +97,7 @@ Las cuotas pueden usar el “quota key” característica con el fin de informar so the quota will be counted separately for each username. Using keys makes sense only if quota_key is transmitted by the program, not by a user. - You can also write so the IP address is used as the quota key. + You can also write , so the IP address is used as the quota key. (But keep in mind that users can change the IPv6 address fairly easily.) --> diff --git a/docs/es/operations/requirements.md b/docs/es/operations/requirements.md index 4f8577ce717..29ed7a655a7 100644 --- a/docs/es/operations/requirements.md +++ b/docs/es/operations/requirements.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 44 toc_title: Requisito --- @@ -13,7 +13,7 @@ Para la instalación desde paquetes deb precompilados, utilice una CPU con arqui ClickHouse implementa el procesamiento de datos paralelo y utiliza todos los recursos de hardware disponibles. Al elegir un procesador, tenga en cuenta que ClickHouse funciona de manera más eficiente en configuraciones con un gran número de núcleos pero con una velocidad de reloj más baja que en configuraciones con menos núcleos y una velocidad de reloj más alta. Por ejemplo, 16 núcleos con 2600 MHz es preferible a 8 núcleos con 3600 MHz. -Uso de **Impulso de Turbo** y **hiper-threading** tecnologías se recomienda. Mejora significativamente el rendimiento con una carga típica. +Se recomienda usar **Impulso de Turbo** y **hiper-threading** tecnología. Mejora significativamente el rendimiento con una carga de trabajo típica. ## RAM {#ram} @@ -24,15 +24,15 @@ El volumen requerido de RAM depende de: - La complejidad de las consultas. - La cantidad de datos que se procesan en las consultas. -Para calcular el volumen requerido de RAM, debe estimar el tamaño de los datos temporales para [GROUP BY](../sql-reference/statements/select.md#select-group-by-clause), [DISTINCT](../sql-reference/statements/select.md#select-distinct), [JOIN](../sql-reference/statements/select.md#select-join) y otras operaciones que utilice. +Para calcular el volumen requerido de RAM, debe estimar el tamaño de los datos temporales para [GROUP BY](../sql-reference/statements/select/group-by.md#select-group-by-clause), [DISTINCT](../sql-reference/statements/select/distinct.md#select-distinct), [JOIN](../sql-reference/statements/select/join.md#select-join) y otras operaciones que utilice. -ClickHouse puede usar memoria externa para datos temporales. Ver [GROUP BY en memoria externa](../sql-reference/statements/select.md#select-group-by-in-external-memory) para más detalles. +ClickHouse puede usar memoria externa para datos temporales. Ver [GROUP BY en memoria externa](../sql-reference/statements/select/group-by.md#select-group-by-in-external-memory) para más detalles. -## Archivo De Intercambio {#swap-file} +## Archivo de intercambio {#swap-file} Deshabilite el archivo de intercambio para entornos de producción. -## Subsistema De Almacenamiento {#storage-subsystem} +## Subsistema de almacenamiento {#storage-subsystem} Necesita tener 2 GB de espacio libre en disco para instalar ClickHouse. @@ -56,6 +56,6 @@ El ancho de banda de la red es fundamental para procesar consultas distribuidas ## Software {#software} -ClickHouse está desarrollado para la familia de sistemas operativos Linux. La distribución de Linux recomendada es Ubuntu. El `tzdata` paquete debe ser instalado en el sistema. +ClickHouse está desarrollado principalmente para la familia de sistemas operativos Linux. La distribución de Linux recomendada es Ubuntu. El `tzdata` paquete debe ser instalado en el sistema. ClickHouse también puede funcionar en otras familias de sistemas operativos. Ver detalles en el [Primeros pasos](../getting-started/index.md) sección de la documentación. diff --git a/docs/es/operations/server-configuration-parameters/index.md b/docs/es/operations/server-configuration-parameters/index.md index 29a9550799c..e1e2e777b94 100644 --- a/docs/es/operations/server-configuration-parameters/index.md +++ b/docs/es/operations/server-configuration-parameters/index.md @@ -1,18 +1,18 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa -toc_folder_title: Server Configuration Parameters +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "Par\xE1metros de configuraci\xF3n del servidor" toc_priority: 54 toc_title: "Implantaci\xF3n" --- -# Parámetros De configuración Del Servidor {#server-settings} +# Parámetros de configuración del servidor {#server-settings} Esta sección contiene descripciones de la configuración del servidor que no se puede cambiar en el nivel de sesión o consulta. Estos ajustes se almacenan en el `config.xml` archivo en el servidor ClickHouse. -Otros ajustes se describen en el “[Configuración](../settings/index.md#settings)” apartado. +Otros ajustes se describen en el “[Configuración](../settings/index.md#session-settings-intro)” apartado. Antes de estudiar la configuración, lea el [Archivos de configuración](../configuration-files.md#configuration_files) sección y tomar nota del uso de sustituciones (el `incl` y `optional` atributo). diff --git a/docs/es/operations/server-configuration-parameters/settings.md b/docs/es/operations/server-configuration-parameters/settings.md index 80f36783c78..7ca3b628bbc 100644 --- a/docs/es/operations/server-configuration-parameters/settings.md +++ b/docs/es/operations/server-configuration-parameters/settings.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 57 toc_title: "Configuraci\xF3n del servidor" --- -# Configuración Del Servidor {#server-settings} +# Configuración del servidor {#server-settings} ## builtin\_dictionaries\_reload\_interval {#builtin-dictionaries-reload-interval} @@ -249,7 +249,7 @@ Puerto para el intercambio de datos entre servidores ClickHouse. 9009 ``` -## Sistema Abierto {#interserver-http-host} +## Sistema abierto {#interserver-http-host} El nombre de host que pueden utilizar otros servidores para acceder a este servidor. @@ -364,7 +364,7 @@ Para obtener más información, consulte la sección “[Creación de tablas rep ``` -## Método De codificación De Datos: {#server-mark-cache-size} +## Método de codificación de datos: {#server-mark-cache-size} Tamaño aproximado (en bytes) de la memoria caché de marcas utilizadas por los motores de [Método de codificación de datos:](../../engines/table-engines/mergetree-family/mergetree.md) familia. @@ -402,7 +402,7 @@ El número máximo de archivos abiertos. Predeterminada: `maximum`. -Recomendamos el uso de esta opción en Mac OS X desde la `getrlimit()` la función devuelve un valor incorrecto. +Recomendamos usar esta opción en Mac OS X desde el `getrlimit()` función devuelve un valor incorrecto. **Ejemplo** @@ -414,7 +414,7 @@ Recomendamos el uso de esta opción en Mac OS X desde la `getrlimit()` la funci Restricción en la eliminación de tablas. -Si el tamaño de un [Método de codificación de datos:](../../engines/table-engines/mergetree-family/mergetree.md) tabla supera `max_table_size_to_drop` (en bytes), no puede eliminarlo usando una consulta DROP. +Si el tamaño de un [Método de codificación de datos:](../../engines/table-engines/mergetree-family/mergetree.md) mesa excede `max_table_size_to_drop` (en bytes), no puede eliminarlo usando una consulta DROP. Si aún necesita eliminar la tabla sin reiniciar el servidor ClickHouse, cree el `/flags/force_drop_table` y ejecute la consulta DROP. @@ -453,7 +453,7 @@ Claves para la configuración del servidor/cliente: - privateKeyFile – The path to the file with the secret key of the PEM certificate. The file may contain a key and certificate at the same time. - certificateFile – The path to the client/server certificate file in PEM format. You can omit it if `privateKeyFile` contiene el certificado. - caConfig – The path to the file or directory that contains trusted root certificates. -- verificationMode – The method for checking the node’s certificates. Details are in the description of the [Contexto](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/Context.h) clase. Valores posibles: `none`, `relaxed`, `strict`, `once`. +- verificationMode – The method for checking the node's certificates. Details are in the description of the [Contexto](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/Context.h) clase. Valores posibles: `none`, `relaxed`, `strict`, `once`. - verificationDepth – The maximum length of the verification chain. Verification will fail if the certificate chain length exceeds the set value. - loadDefaultCAFile – Indicates that built-in CA certificates for OpenSSL will be used. Acceptable values: `true`, `false`. \| - cipherList – Supported OpenSSL encryptions. For example: `ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH`. @@ -465,7 +465,7 @@ Claves para la configuración del servidor/cliente: - requireTLSv1 – Require a TLSv1 connection. Acceptable values: `true`, `false`. - requireTLSv1\_1 – Require a TLSv1.1 connection. Acceptable values: `true`, `false`. - requireTLSv1 – Require a TLSv1.2 connection. Acceptable values: `true`, `false`. -- fips – Activates OpenSSL FIPS mode. Supported if the library’s OpenSSL version supports FIPS. +- fips – Activates OpenSSL FIPS mode. Supported if the library's OpenSSL version supports FIPS. - privateKeyPassphraseHandler – Class (PrivateKeyPassphraseHandler subclass) that requests the passphrase for accessing the private key. For example: ``, `KeyFileHandler`, `test`, ``. - invalidCertificateHandler – Class (a subclass of CertificateHandler) for verifying invalid certificates. For example: ` ConsoleCertificateHandler ` . - disableProtocols – Protocols that are not allowed to use. @@ -538,13 +538,37 @@ La ruta de acceso al directorio que contiene los datos. /var/lib/clickhouse/ ``` +## prometeo {#server_configuration_parameters-prometheus} + +Exponer datos de métricas para raspar desde [Prometeo](https://prometheus.io). + +Configuración: + +- `endpoint` – HTTP endpoint for scraping metrics by prometheus server. Start from ‘/’. +- `port` – Port for `endpoint`. +- `metrics` – Flag that sets to expose metrics from the [sistema.métricas](../system-tables.md#system_tables-metrics) tabla. +- `events` – Flag that sets to expose metrics from the [sistema.evento](../system-tables.md#system_tables-events) tabla. +- `asynchronous_metrics` – Flag that sets to expose current metrics values from the [sistema.asynchronous\_metrics](../system-tables.md#system_tables-asynchronous_metrics) tabla. + +**Ejemplo** + +``` xml + + /metrics + 8001 + true + true + true + +``` + ## query\_log {#server_configuration_parameters-query-log} Configuración de las consultas de registro recibidas con [log\_queries=1](../settings/settings.md) configuración. Las consultas se registran en el [sistema.query\_log](../../operations/system-tables.md#system_tables-query_log) tabla, no en un archivo separado. Puede cambiar el nombre de la tabla en el `table` parámetro (ver más abajo). -Utilice los siguientes parámetros para configurar el inicio de sesión: +Utilice los siguientes parámetros para configurar el registro: - `database` – Name of the database. - `table` – Name of the system table the queries will be logged in. @@ -564,13 +588,13 @@ Si la tabla no existe, ClickHouse la creará. Si la estructura del registro de c ``` -## Sistema Abierto {#server_configuration_parameters-query-thread-log} +## Sistema abierto {#server_configuration_parameters-query-thread-log} Configuración de subprocesos de registro de consultas recibidas con [Log\_query\_threads = 1](../settings/settings.md#settings-log-query-threads) configuración. Las consultas se registran en el [sistema.Sistema abierto.](../../operations/system-tables.md#system_tables-query-thread-log) tabla, no en un archivo separado. Puede cambiar el nombre de la tabla en el `table` parámetro (ver más abajo). -Utilice los siguientes parámetros para configurar el inicio de sesión: +Utilice los siguientes parámetros para configurar el registro: - `database` – Name of the database. - `table` – Name of the system table the queries will be logged in. @@ -592,7 +616,7 @@ Si la tabla no existe, ClickHouse la creará. Si la estructura del registro de s ## trace\_log {#server_configuration_parameters-trace_log} -La configuración para el [trace\_log](../../operations/system-tables.md#system_tables-trace_log) operación de la tabla del sistema. +Ajustes para el [trace\_log](../../operations/system-tables.md#system_tables-trace_log) operación de la tabla del sistema. Parámetros: @@ -659,7 +683,7 @@ Para el valor de la `incl` atributo, consulte la sección “[Archivos de config - [skip\_unavailable\_shards](../settings/settings.md#settings-skip_unavailable_shards) -## Zona Horaria {#server_configuration_parameters-timezone} +## Zona horaria {#server_configuration_parameters-timezone} La zona horaria del servidor. @@ -869,4 +893,14 @@ La actualización se realiza de forma asíncrona, en un subproceso del sistema s **Valor predeterminado**: 15. +## access\_control\_path {#access_control_path} + +Ruta de acceso a una carpeta donde un servidor ClickHouse almacena configuraciones de usuario y rol creadas por comandos SQL. + +Valor predeterminado: `/var/lib/clickhouse/access/`. + +**Ver también** + +- [Control de acceso y gestión de cuentas](../access-rights.md#access-control) + [Artículo Original](https://clickhouse.tech/docs/en/operations/server_configuration_parameters/settings/) diff --git a/docs/es/operations/settings/constraints-on-settings.md b/docs/es/operations/settings/constraints-on-settings.md index 6adc0fa9488..fe385f6ddbb 100644 --- a/docs/es/operations/settings/constraints-on-settings.md +++ b/docs/es/operations/settings/constraints-on-settings.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 62 toc_title: "Restricciones en la configuraci\xF3n" --- -# Restricciones En La configuración {#constraints-on-settings} +# Restricciones en la configuración {#constraints-on-settings} Las restricciones en los ajustes se pueden definir en el `profiles` sección de la `user.xml` el archivo de configuración y prohíba a los usuarios cambiar algunos de los ajustes `SET` consulta. Las restricciones se definen como las siguientes: diff --git a/docs/es/operations/settings/index.md b/docs/es/operations/settings/index.md index 7f135801098..37aab0a7e1b 100644 --- a/docs/es/operations/settings/index.md +++ b/docs/es/operations/settings/index.md @@ -1,14 +1,15 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa -toc_folder_title: Settings +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "Configuraci\xF3n" toc_priority: 55 toc_title: "Implantaci\xF3n" --- -# Configuración {#settings} +# Configuración {#session-settings-intro} + +Hay varias maneras de realizar todos los ajustes descritos en esta sección de documentación. -Hay varias formas de realizar todos los ajustes que se describen a continuación. Los ajustes se configuran en capas, por lo que cada capa subsiguiente redefine los ajustes anteriores. Formas de configurar los ajustes, por orden de prioridad: diff --git a/docs/es/operations/settings/permissions-for-queries.md b/docs/es/operations/settings/permissions-for-queries.md index 2737f5d2f97..f9f669b876e 100644 --- a/docs/es/operations/settings/permissions-for-queries.md +++ b/docs/es/operations/settings/permissions-for-queries.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 58 toc_title: Permisos para consultas --- -# Permisos Para Consultas {#permissions_for_queries} +# Permisos para consultas {#permissions_for_queries} Las consultas en ClickHouse se pueden dividir en varios tipos: @@ -43,7 +43,7 @@ de cambiar sólo ajustes específicos, para más detalles ver [restricciones en Valor predeterminado: 0 -## Método De codificación De Datos: {#settings_allow_ddl} +## Método de codificación de datos: {#settings_allow_ddl} Permite o niega [DDL](https://en.wikipedia.org/wiki/Data_definition_language) consulta. diff --git a/docs/es/operations/settings/query-complexity.md b/docs/es/operations/settings/query-complexity.md index fff8cf3cc15..218952dff1f 100644 --- a/docs/es/operations/settings/query-complexity.md +++ b/docs/es/operations/settings/query-complexity.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 59 toc_title: Restricciones en la complejidad de consultas --- -# Restricciones En La Complejidad De Consultas {#restrictions-on-query-complexity} +# Restricciones en la complejidad de consultas {#restrictions-on-query-complexity} Las restricciones en la complejidad de la consulta forman parte de la configuración. Se utilizan para proporcionar una ejecución más segura desde la interfaz de usuario. @@ -21,9 +21,9 @@ Puede tomar uno de dos valores: `throw` o `break`. Las restricciones en la agreg `break` – Stop executing the query and return the partial result, as if the source data ran out. -`any (only for group_by_overflow_mode)` – Continuing aggregation for the keys that got into the set, but don’t add new keys to the set. +`any (only for group_by_overflow_mode)` – Continuing aggregation for the keys that got into the set, but don't add new keys to the set. -## Método De codificación De Datos: {#settings_max_memory_usage} +## Método de codificación de datos: {#settings_max_memory_usage} La cantidad máxima de RAM que se utiliza para ejecutar una consulta en un único servidor. @@ -48,7 +48,7 @@ Los valores predeterminados se definen en [Configuración.h](https://github.com/ Ver también la descripción de [Método de codificación de datos:](#settings_max_memory_usage). -## Todos Los Derechos Reservados {#max-memory-usage-for-all-queries} +## Todos los derechos reservados {#max-memory-usage-for-all-queries} La cantidad máxima de RAM que se utilizará para ejecutar todas las consultas en un único servidor. @@ -67,11 +67,11 @@ Un número máximo de filas que se pueden leer de una tabla al ejecutar una cons Un número máximo de bytes (datos sin comprimir) que se pueden leer de una tabla al ejecutar una consulta. -## Método De codificación De Datos: {#read-overflow-mode} +## Método de codificación de datos: {#read-overflow-mode} Qué hacer cuando el volumen de datos leídos excede uno de los límites: ‘throw’ o ‘break’. Por defecto, throw. -## Método De codificación De Datos: {#settings-max-rows-to-group-by} +## Método de codificación de datos: {#settings-max-rows-to-group-by} Un número máximo de claves únicas recibidas de la agregación. Esta configuración le permite limitar el consumo de memoria al agregar. @@ -82,20 +82,20 @@ Uso de la ‘any’ valor le permite ejecutar una aproximación de GROUP BY. La ## max\_bytes\_before\_external\_group\_by {#settings-max_bytes_before_external_group_by} -Habilita o deshabilita la ejecución de `GROUP BY` en la memoria externa. Ver [GROUP BY en memoria externa](../../sql-reference/statements/select.md#select-group-by-in-external-memory). +Habilita o deshabilita la ejecución de `GROUP BY` en la memoria externa. Ver [GROUP BY en memoria externa](../../sql-reference/statements/select/group-by.md#select-group-by-in-external-memory). Valores posibles: -- Volumen máximo de RAM (en bytes) que puede ser utilizado por el único [GROUP BY](../../sql-reference/statements/select.md#select-group-by-clause) operación. +- Volumen máximo de RAM (en bytes) que puede ser utilizado por el único [GROUP BY](../../sql-reference/statements/select/group-by.md#select-group-by-clause) operación. - 0 — `GROUP BY` en la memoria externa deshabilitada. Valor predeterminado: 0. -## Método De codificación De Datos: {#max-rows-to-sort} +## Método de codificación de datos: {#max-rows-to-sort} Un número máximo de filas antes de ordenar. Esto le permite limitar el consumo de memoria al ordenar. -## Método De codificación De Datos: {#max-bytes-to-sort} +## Método de codificación de datos: {#max-bytes-to-sort} Un número máximo de bytes antes de ordenar. @@ -143,11 +143,11 @@ En este momento, no se comprueba una de las etapas de clasificación, o al fusio Qué hacer si la consulta se ejecuta más de ‘max\_execution\_time’: ‘throw’ o ‘break’. Por defecto, throw. -## Método De codificación De Datos: {#min-execution-speed} +## Método de codificación de datos: {#min-execution-speed} Velocidad de ejecución mínima en filas por segundo. Comprobado en cada bloque de datos cuando ‘timeout\_before\_checking\_execution\_speed’ expirar. Si la velocidad de ejecución es menor, se produce una excepción. -## Todos Los Derechos Reservados {#min-execution-speed-bytes} +## Todos los derechos reservados {#min-execution-speed-bytes} Un número mínimo de bytes de ejecución por segundo. Comprobado en cada bloque de datos cuando ‘timeout\_before\_checking\_execution\_speed’ expirar. Si la velocidad de ejecución es menor, se produce una excepción. @@ -159,7 +159,7 @@ Un número máximo de filas de ejecución por segundo. Comprobado en cada bloque Un número máximo de bytes de ejecución por segundo. Comprobado en cada bloque de datos cuando ‘timeout\_before\_checking\_execution\_speed’ expirar. Si la velocidad de ejecución es alta, la velocidad de ejecución se reducirá. -## Tiempo De Espera Antes De comprobar\_ejecución\_velocidad {#timeout-before-checking-execution-speed} +## Tiempo de espera antes de comprobar\_ejecución\_velocidad {#timeout-before-checking-execution-speed} Comprueba que la velocidad de ejecución no sea demasiado lenta (no menos de ‘min\_execution\_speed’), después de que el tiempo especificado en segundos haya expirado. @@ -194,11 +194,11 @@ En este momento, no se verifica durante el análisis, sino solo después de anal Un número máximo de elementos en un árbol sintáctico de consulta. Si se supera, se produce una excepción. De la misma manera que la configuración anterior, se verifica solo después de analizar la consulta. De forma predeterminada, 50.000. -## Método De codificación De Datos: {#max-rows-in-set} +## Método de codificación de datos: {#max-rows-in-set} Un número máximo de filas para un conjunto de datos en la cláusula IN creada a partir de una subconsulta. -## Método De codificación De Datos: {#max-bytes-in-set} +## Método de codificación de datos: {#max-bytes-in-set} Número máximo de bytes (datos sin comprimir) utilizados por un conjunto en la cláusula IN creada a partir de una subconsulta. @@ -206,11 +206,11 @@ Número máximo de bytes (datos sin comprimir) utilizados por un conjunto en la Qué hacer cuando la cantidad de datos excede uno de los límites: ‘throw’ o ‘break’. Por defecto, throw. -## Método De codificación De Datos: {#max-rows-in-distinct} +## Método de codificación de datos: {#max-rows-in-distinct} Un número máximo de filas diferentes al usar DISTINCT. -## Método De codificación De Datos: {#max-bytes-in-distinct} +## Método de codificación de datos: {#max-bytes-in-distinct} Un número máximo de bytes utilizados por una tabla hash cuando se utiliza DISTINCT. @@ -230,11 +230,11 @@ Un número máximo de bytes (datos sin comprimir) que se pueden pasar a un servi Qué hacer cuando la cantidad de datos excede uno de los límites: ‘throw’ o ‘break’. Por defecto, throw. -## Método De codificación De Datos: {#settings-max_rows_in_join} +## Método de codificación de datos: {#settings-max_rows_in_join} Limita el número de filas de la tabla hash que se utiliza al unir tablas. -Esta configuración se aplica a [SELECT … JOIN](../../sql-reference/statements/select.md#select-join) operaciones y la [Unir](../../engines/table-engines/special/join.md) motor de mesa. +Esta configuración se aplica a [SELECT … JOIN](../../sql-reference/statements/select/join.md#select-join) operaciones y la [Unir](../../engines/table-engines/special/join.md) motor de mesa. Si una consulta contiene varias combinaciones, ClickHouse comprueba esta configuración para cada resultado intermedio. @@ -247,11 +247,11 @@ Valores posibles: Valor predeterminado: 0. -## Método De codificación De Datos: {#settings-max_bytes_in_join} +## Método de codificación de datos: {#settings-max_bytes_in_join} Limita el tamaño en bytes de la tabla hash utilizada al unir tablas. -Esta configuración se aplica a [SELECT … JOIN](../../sql-reference/statements/select.md#select-join) operaciones y [Unirse al motor de tabla](../../engines/table-engines/special/join.md). +Esta configuración se aplica a [SELECT … JOIN](../../sql-reference/statements/select/join.md#select-join) operaciones y [Unirse al motor de tabla](../../engines/table-engines/special/join.md). Si la consulta contiene combinaciones, ClickHouse comprueba esta configuración para cada resultado intermedio. @@ -274,13 +274,13 @@ Define qué acción realiza ClickHouse cuando se alcanza cualquiera de los sigui Valores posibles: - `THROW` — ClickHouse throws an exception and breaks operation. -- `BREAK` — ClickHouse breaks operation and doesn’t throw an exception. +- `BREAK` — ClickHouse breaks operation and doesn't throw an exception. Valor predeterminado: `THROW`. **Ver también** -- [Cláusula JOIN](../../sql-reference/statements/select.md#select-join) +- [Cláusula JOIN](../../sql-reference/statements/select/join.md#select-join) - [Unirse al motor de tabla](../../engines/table-engines/special/join.md) ## max\_partitions\_per\_insert\_block {#max-partitions-per-insert-block} diff --git a/docs/es/operations/settings/settings-profiles.md b/docs/es/operations/settings/settings-profiles.md index fdde0976986..3d96a2c8fba 100644 --- a/docs/es/operations/settings/settings-profiles.md +++ b/docs/es/operations/settings/settings-profiles.md @@ -1,13 +1,21 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 61 toc_title: "Perfiles de configuraci\xF3n" --- -# Perfiles De configuración {#settings-profiles} +# Perfiles de configuración {#settings-profiles} + +Un perfil de configuración es una colección de configuraciones agrupadas con el mismo nombre. + +!!! note "Información" + ClickHouse también es compatible [Flujo de trabajo controlado por SQL](../access-rights.md#access-control) para administrar perfiles de configuración. Recomendamos usarlo. + +Un perfil puede tener cualquier nombre. El perfil puede tener cualquier nombre. Puede especificar el mismo perfil para diferentes usuarios. Lo más importante que puede escribir en el perfil de configuración es `readonly=1`, que asegura el acceso de sólo lectura. + +Los perfiles de configuración pueden heredar unos de otros. Para usar la herencia, indique una o varias `profile` configuraciones antes de las demás configuraciones que se enumeran en el perfil. En caso de que se defina una configuración en diferentes perfiles, se utiliza la última definida. -Un perfil de configuración es una colección de configuraciones agrupadas con el mismo nombre. Cada usuario de ClickHouse tiene un perfil. Para aplicar todos los ajustes de un perfil, establezca el `profile` configuración. Ejemplo: @@ -64,8 +72,10 @@ Ejemplo: ``` -El ejemplo especifica dos perfiles: `default` y `web`. El `default` tiene un propósito especial: siempre debe estar presente y se aplica al iniciar el servidor. En otras palabras, el `default` perfil contiene la configuración predeterminada. El `web` profile es un perfil regular que se puede establecer utilizando el `SET` consulta o utilizando un parámetro URL en una consulta HTTP. +El ejemplo especifica dos perfiles: `default` y `web`. -Los perfiles de configuración pueden heredar unos de otros. Para usar la herencia, indique una o varias `profile` configuraciones antes de las demás configuraciones que se enumeran en el perfil. En caso de que se defina una configuración en diferentes perfiles, se utiliza la última definida. +El `default` tiene un propósito especial: siempre debe estar presente y se aplica al iniciar el servidor. En otras palabras, el `default` perfil contiene la configuración predeterminada. + +El `web` profile es un perfil regular que se puede establecer utilizando el `SET` consulta o utilizando un parámetro URL en una consulta HTTP. [Artículo Original](https://clickhouse.tech/docs/en/operations/settings/settings_profiles/) diff --git a/docs/es/operations/settings/settings-users.md b/docs/es/operations/settings/settings-users.md index 86db864ff92..2f767c959ae 100644 --- a/docs/es/operations/settings/settings-users.md +++ b/docs/es/operations/settings/settings-users.md @@ -1,14 +1,17 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 63 toc_title: "Configuraci\xF3n del usuario" --- -# Configuración Del Usuario {#user-settings} +# Configuración del usuario {#user-settings} El `users` sección de la `user.xml` el archivo de configuración contiene la configuración del usuario. +!!! note "Información" + ClickHouse también es compatible [Flujo de trabajo controlado por SQL](../access-rights.md#access-control) para la gestión de usuarios. Recomendamos usarlo. + Estructura del `users` apartado: ``` xml @@ -19,6 +22,8 @@ Estructura del `users` apartado: + 0|1 + @@ -70,6 +75,17 @@ La contraseña se puede especificar en texto sin formato o en SHA256 (formato he La primera línea del resultado es la contraseña. La segunda línea es el hash SHA1 doble correspondiente. +### access\_management {#access_management-user-setting} + +Esta configuración habilita deshabilita el uso de [control de acceso y gestión de cuentas](../access-rights.md#access-control) para el usuario. + +Valores posibles: + +- 0 — Disabled. +- 1 — Enabled. + +Valor predeterminado: 0. + ### user\_name/redes {#user-namenetworks} Lista de redes desde las que el usuario puede conectarse al servidor ClickHouse. diff --git a/docs/es/operations/settings/settings.md b/docs/es/operations/settings/settings.md index 71849c6b3d1..1989bb71036 100644 --- a/docs/es/operations/settings/settings.md +++ b/docs/es/operations/settings/settings.md @@ -1,15 +1,13 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa -toc_priority: 60 -toc_title: "Configuraci\xF3n" +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd --- # Configuración {#settings} ## distributed\_product\_mode {#distributed-product-mode} -Cambia el comportamiento de [subconsultas distribuidas](../../sql-reference/statements/select.md). +Cambia el comportamiento de [subconsultas distribuidas](../../sql-reference/operators/in.md). ClickHouse applies this setting when the query contains the product of distributed tables, i.e. when the query for a distributed table contains a non-GLOBAL subquery for the distributed table. @@ -61,7 +59,7 @@ Se utiliza al realizar `SELECT` desde una tabla distribuida que apunta a tablas De forma predeterminada, 1 (habilitado). -## Fecha De Nacimiento {#settings-force_index_by_date} +## Fecha de nacimiento {#settings-force_index_by_date} Deshabilita la ejecución de consultas si el índice no se puede usar por fecha. @@ -79,7 +77,7 @@ Si `force_primary_key=1`, ClickHouse comprueba si la consulta tiene una condici ## Formato\_esquema {#format-schema} -Este parámetro es útil cuando se utilizan formatos que requieren una definición de esquema, como [Cap’n Proto](https://capnproto.org/) o [Protobuf](https://developers.google.com/protocol-buffers/). El valor depende del formato. +Este parámetro es útil cuando se utilizan formatos que requieren una definición de esquema, como [Cap'n Proto](https://capnproto.org/) o [Protobuf](https://developers.google.com/protocol-buffers/). El valor depende del formato. ## fsync\_metadata {#fsync-metadata} @@ -134,9 +132,9 @@ Valores posibles: Valor predeterminado: 0. -## Nombre De La Red inalámbrica (SSID): {#setting-max_http_get_redirects} +## Nombre de la red inalámbrica (SSID): {#setting-max_http_get_redirects} -Limita el número máximo de saltos de redirección HTTP GET para [URL](../../engines/table-engines/special/url.md)-mesas de motor. La configuración se aplica a ambos tipos de tablas: las creadas por [CREATE TABLE](../../query_language/create/#create-table-query) consulta y por el [URL](../../sql-reference/table-functions/url.md) función de la tabla. +Limita el número máximo de saltos de redirección HTTP GET para [URL](../../engines/table-engines/special/url.md)-mesas de motor. La configuración se aplica a ambos tipos de tablas: las creadas por [CREATE TABLE](../../sql-reference/statements/create.md#create-table-query) consulta y por el [URL](../../sql-reference/table-functions/url.md) función de la tabla. Valores posibles: @@ -222,17 +220,24 @@ Ok. ## input\_format\_values\_deduce\_templates\_of\_expressions {#settings-input_format_values_deduce_templates_of_expressions} -Habilita o deshabilita la deducción de plantilla para expresiones SQL en [Valor](../../interfaces/formats.md#data-format-values) formato. Permite analizar e interpretar expresiones en `Values` mucho más rápido si las expresiones en filas consecutivas tienen la misma estructura. ClickHouse intentará deducir la plantilla de una expresión, analizar las siguientes filas utilizando esta plantilla y evaluar la expresión en un lote de filas analizadas correctamente. Para la siguiente consulta: +Habilita o deshabilita la deducción de plantilla para expresiones SQL en [Valor](../../interfaces/formats.md#data-format-values) formato. Permite analizar e interpretar expresiones en `Values` mucho más rápido si las expresiones en filas consecutivas tienen la misma estructura. ClickHouse intenta deducir la plantilla de una expresión, analizar las siguientes filas utilizando esta plantilla y evaluar la expresión en un lote de filas analizadas correctamente. + +Valores posibles: + +- 0 — Disabled. +- 1 — Enabled. + +Valor predeterminado: 1. + +Para la siguiente consulta: ``` sql INSERT INTO test VALUES (lower('Hello')), (lower('world')), (lower('INSERT')), (upper('Values')), ... ``` -- si `input_format_values_interpret_expressions=1` y `format_values_deduce_templates_of_expressions=0` Las expresiones se interpretarán por separado para cada fila (esto es muy lento para un gran número de filas) -- si `input_format_values_interpret_expressions=0` y `format_values_deduce_templates_of_expressions=1` Las expresiones en la primera, segunda y tercera filas se analizarán usando la plantilla `lower(String)` e interpretados juntos, la expresión es la cuarta fila se analizará con otra plantilla (`upper(String)`) -- si `input_format_values_interpret_expressions=1` y `format_values_deduce_templates_of_expressions=1` - lo mismo que en el caso anterior, pero también permite la alternativa a la interpretación de expresiones por separado si no es posible deducir la plantilla. - -Habilitado de forma predeterminada. +- Si `input_format_values_interpret_expressions=1` y `format_values_deduce_templates_of_expressions=0`, las expresiones se interpretan por separado para cada fila (esto es muy lento para un gran número de filas). +- Si `input_format_values_interpret_expressions=0` y `format_values_deduce_templates_of_expressions=1`, las expresiones en la primera, segunda y tercera filas se analizan usando la plantilla `lower(String)` e interpretados juntos, la expresión en la cuarta fila se analiza con otra plantilla (`upper(String)`). +- Si `input_format_values_interpret_expressions=1` y `format_values_deduce_templates_of_expressions=1`, lo mismo que en el caso anterior, pero también permite la alternativa a la interpretación de expresiones por separado si no es posible deducir la plantilla. ## Entrada\_format\_values\_accurate\_types\_of\_literals {#settings-input-format-values-accurate-types-of-literals} @@ -244,9 +249,17 @@ Esta configuración sólo se utiliza cuando `input_format_values_deduce_template (..., abs(-1), ...), -- Int64 literal ``` -Cuando esta configuración está habilitada, ClickHouse comprobará el tipo real de literal y utilizará una plantilla de expresión del tipo correspondiente. En algunos casos, puede ralentizar significativamente la evaluación de expresiones en `Values`. -When disabled, ClickHouse may use more general type for some literals (e.g. `Float64` o `Int64` en lugar de `UInt64` para `42`), pero puede causar problemas de desbordamiento y precisión. -Habilitado de forma predeterminada. +Valores posibles: + +- 0 — Disabled. + + In this case, ClickHouse may use a more general type for some literals (e.g., `Float64` o `Int64` en lugar de `UInt64` para `42`), pero puede causar problemas de desbordamiento y precisión. + +- 1 — Enabled. + + En este caso, ClickHouse comprueba el tipo real de literal y utiliza una plantilla de expresión del tipo correspondiente. En algunos casos, puede ralentizar significativamente la evaluación de expresiones en `Values`. + +Valor predeterminado: 1. ## Entrada\_format\_defaults\_for\_omitted\_fields {#session_settings-input_format_defaults_for_omitted_fields} @@ -274,7 +287,7 @@ Habilita o deshabilita el uso de valores predeterminados si los datos de entrada ## input\_format\_skip\_unknown\_fields {#settings-input-format-skip-unknown-fields} -Habilita o deshabilita saltarse la inserción de datos adicionales. +Habilita o deshabilita omitir la inserción de datos adicionales. Al escribir datos, ClickHouse produce una excepción si los datos de entrada contienen columnas que no existen en la tabla de destino. Si la omisión está habilitada, ClickHouse no inserta datos adicionales y no lanza una excepción. @@ -296,7 +309,7 @@ Valor predeterminado: 0. Habilita o deshabilita la inserción de datos JSON con objetos anidados. -Formatos compatibles: +Formatos soportados: - [JSONEachRow](../../interfaces/formats.md#jsoneachrow) @@ -311,7 +324,7 @@ Ver también: - [Uso de estructuras anidadas](../../interfaces/formats.md#jsoneachrow-nested) con el `JSONEachRow` formato. -## input\_format\_with\_names\_use\_header {#settings-input-format-with-names-use-header} +## Entrada\_format\_with\_names\_use\_header {#settings-input-format-with-names-use-header} Habilita o deshabilita la comprobación del orden de las columnas al insertar datos. @@ -352,9 +365,9 @@ Ver también: - [Tipo de datos DateTime.](../../sql-reference/data-types/datetime.md) - [Funciones para trabajar con fechas y horas.](../../sql-reference/functions/date-time-functions.md) -## Por Favor, Introduzca Su dirección De Correo electrónico {#settings-join_default_strictness} +## Por favor, introduzca su dirección de correo electrónico {#settings-join_default_strictness} -Establece el rigor predeterminado para [Cláusulas JOIN](../../sql-reference/statements/select.md#select-join). +Establece el rigor predeterminado para [Cláusulas JOIN](../../sql-reference/statements/select/join.md#select-join). Valores posibles: @@ -381,13 +394,13 @@ Valor predeterminado: 0. Ver también: -- [Cláusula JOIN](../../sql-reference/statements/select.md#select-join) +- [Cláusula JOIN](../../sql-reference/statements/select/join.md#select-join) - [Unirse al motor de tabla](../../engines/table-engines/special/join.md) - [Por favor, introduzca su dirección de correo electrónico](#settings-join_default_strictness) -## Sistema Abierto {#join_use_nulls} +## Sistema abierto {#join_use_nulls} -Establece el tipo de [JOIN](../../sql-reference/statements/select.md) comportamiento. Al fusionar tablas, pueden aparecer celdas vacías. ClickHouse los rellena de manera diferente según esta configuración. +Establece el tipo de [JOIN](../../sql-reference/statements/select/join.md) comportamiento. Al fusionar tablas, pueden aparecer celdas vacías. ClickHouse los rellena de manera diferente según esta configuración. Valores posibles: @@ -412,7 +425,7 @@ Por defecto: 1,000,000. Solo funciona cuando se lee desde los motores MergeTree. ## merge\_tree\_min\_rows\_for\_concurrent\_read {#setting-merge-tree-min-rows-for-concurrent-read} -Si el número de filas que se leerán de un fichero [Método de codificación de datos:](../../engines/table-engines/mergetree-family/mergetree.md) tabla supera `merge_tree_min_rows_for_concurrent_read` luego ClickHouse intenta realizar una lectura simultánea de este archivo en varios hilos. +Si el número de filas que se leerán de un fichero [Método de codificación de datos:](../../engines/table-engines/mergetree-family/mergetree.md) mesa excede `merge_tree_min_rows_for_concurrent_read` luego ClickHouse intenta realizar una lectura simultánea de este archivo en varios hilos. Valores posibles: @@ -430,7 +443,7 @@ Valor posible: Valor predeterminado: 251658240. -## Método De codificación De Datos: {#setting-merge-tree-min-rows-for-seek} +## Método de codificación de datos: {#setting-merge-tree-min-rows-for-seek} Si la distancia entre dos bloques de datos que se leen en un archivo es menor que `merge_tree_min_rows_for_seek` filas, luego ClickHouse no busca a través del archivo, sino que lee los datos secuencialmente. @@ -484,7 +497,7 @@ Valor posible: Valor predeterminado: 2013265920. -## Todos Los Derechos Reservados {#settings-min-bytes-to-use-direct-io} +## Todos los derechos reservados {#settings-min-bytes-to-use-direct-io} El volumen de datos mínimo necesario para utilizar el acceso directo de E/S al disco de almacenamiento. @@ -509,6 +522,24 @@ Ejemplo: log_queries=1 ``` +## Nombre de la red inalámbrica (SSID): {#settings-log-queries-min-type} + +`query_log` tipo mínimo para iniciar sesión. + +Valores posibles: +- `QUERY_START` (`=1`) +- `QUERY_FINISH` (`=2`) +- `EXCEPTION_BEFORE_START` (`=3`) +- `EXCEPTION_WHILE_PROCESSING` (`=4`) + +Valor predeterminado: `QUERY_START`. + +Se puede usar para limitar a qué entiries va `query_log`, digamos que eres interesante solo en errores, entonces puedes usar `EXCEPTION_WHILE_PROCESSING`: + +``` text +log_queries_min_type='EXCEPTION_WHILE_PROCESSING' +``` + ## Log\_query\_threads {#settings-log-query-threads} Configuración del registro de subprocesos de consulta. @@ -533,6 +564,28 @@ Valor predeterminado: 1.048.576. El valor predeterminado es ligeramente más que `max_block_size`. La razón de esto se debe a que ciertos motores de mesa (`*MergeTree`) formar una parte de datos en el disco para cada bloque insertado, que es una entidad bastante grande. Similar, `*MergeTree` las tablas ordenan los datos durante la inserción y un tamaño de bloque lo suficientemente grande permiten clasificar más datos en la RAM. +## Nombre de la red inalámbrica (SSID): {#min-insert-block-size-rows} + +Establece el número mínimo de filas en el bloque que se pueden insertar en una tabla `INSERT` consulta. Los bloques de menor tamaño se aplastan en otros más grandes. + +Valores posibles: + +- Entero positivo. +- 0 — Squashing disabled. + +Valor predeterminado: 1048576. + +## Todos los derechos reservados {#min-insert-block-size-bytes} + +Establece el número mínimo de bytes en el bloque que se pueden insertar en una tabla `INSERT` consulta. Los bloques de menor tamaño se aplastan en otros más grandes. + +Valores posibles: + +- Entero positivo. +- 0 — Squashing disabled. + +Valor predeterminado: 268435456. + ## max\_replica\_delay\_for\_distributed\_queries {#settings-max_replica_delay_for_distributed_queries} Deshabilita las réplicas rezagadas para consultas distribuidas. Ver [Replicación](../../engines/table-engines/mergetree-family/replication.md). @@ -558,7 +611,7 @@ Para las consultas que se completan rápidamente debido a un LIMIT, puede establ Cuanto menor sea el `max_threads` valor, menos memoria se consume. -## Método De codificación De Datos: {#settings-max-insert-threads} +## Método de codificación de datos: {#settings-max-insert-threads} El número máximo de subprocesos para ejecutar el `INSERT SELECT` consulta. @@ -578,7 +631,7 @@ El tamaño máximo de bloques de datos sin comprimir antes de comprimir para esc No confunda bloques para la compresión (un fragmento de memoria que consta de bytes) con bloques para el procesamiento de consultas (un conjunto de filas de una tabla). -## Descripción Del Producto {#min-compress-block-size} +## Descripción del producto {#min-compress-block-size} Para [Método de codificación de datos:](../../engines/table-engines/mergetree-family/mergetree.md)" tabla. Para reducir la latencia al procesar consultas, un bloque se comprime al escribir la siguiente marca si su tamaño es al menos ‘min\_compress\_block\_size’. De forma predeterminada, 65.536. @@ -667,13 +720,13 @@ Para consultas que leen al menos un volumen algo grande de datos (un millón de Cuando se utiliza la interfaz HTTP, el ‘query\_id’ parámetro puede ser pasado. Se trata de cualquier cadena que sirva como identificador de consulta. Si una consulta del mismo usuario ‘query\_id’ que ya existe en este momento, el comportamiento depende de la ‘replace\_running\_query’ parámetro. -`0` (default) – Throw an exception (don’t allow the query to run if a query with the same ‘query\_id’ ya se está ejecutando). +`0` (default) – Throw an exception (don't allow the query to run if a query with the same ‘query\_id’ ya se está ejecutando). `1` – Cancel the old query and start running the new one. El Yandex.Metrica utiliza este parámetro establecido en 1 para implementar sugerencias para las condiciones de segmentación. Después de ingresar el siguiente carácter, si la consulta anterior aún no ha finalizado, debe cancelarse. -## Nombre De La Red inalámbrica (SSID): {#stream-flush-interval-ms} +## Nombre de la red inalámbrica (SSID): {#stream-flush-interval-ms} Funciona para tablas con streaming en el caso de un tiempo de espera, o cuando un subproceso genera [Max\_insert\_block\_size](#settings-max_insert_block_size) filas. @@ -692,7 +745,7 @@ ClickHouse admite los siguientes algoritmos para elegir réplicas: - [En orden](#load_balancing-in_order) - [Primero o aleatorio](#load_balancing-first_or_random) -### Aleatorio (por Defecto) {#load_balancing-random} +### Aleatorio (por defecto) {#load_balancing-random} ``` sql load_balancing = random @@ -701,13 +754,13 @@ load_balancing = random El número de errores se cuenta para cada réplica. La consulta se envía a la réplica con el menor número de errores, y si hay varios de estos, a cualquiera de ellos. Desventajas: La proximidad del servidor no se tiene en cuenta; si las réplicas tienen datos diferentes, también obtendrá datos diferentes. -### Nombre De Host más Cercano {#load_balancing-nearest_hostname} +### Nombre de host más cercano {#load_balancing-nearest_hostname} ``` sql load_balancing = nearest_hostname ``` -The number of errors is counted for each replica. Every 5 minutes, the number of errors is integrally divided by 2. Thus, the number of errors is calculated for a recent time with exponential smoothing. If there is one replica with a minimal number of errors (i.e. errors occurred recently on the other replicas), the query is sent to it. If there are multiple replicas with the same minimal number of errors, the query is sent to the replica with a hostname that is most similar to the server’s hostname in the config file (for the number of different characters in identical positions, up to the minimum length of both hostnames). +The number of errors is counted for each replica. Every 5 minutes, the number of errors is integrally divided by 2. Thus, the number of errors is calculated for a recent time with exponential smoothing. If there is one replica with a minimal number of errors (i.e. errors occurred recently on the other replicas), the query is sent to it. If there are multiple replicas with the same minimal number of errors, the query is sent to the replica with a hostname that is most similar to the server's hostname in the config file (for the number of different characters in identical positions, up to the minimum length of both hostnames). Por ejemplo, example01-01-1 y example01-01-2.yandex.ru son diferentes en una posición, mientras que example01-01-1 y example01-02-2 difieren en dos lugares. Este método puede parecer primitivo, pero no requiere datos externos sobre la topología de red, y no compara las direcciones IP, lo que sería complicado para nuestras direcciones IPv6. @@ -715,7 +768,7 @@ Este método puede parecer primitivo, pero no requiere datos externos sobre la t Por lo tanto, si hay réplicas equivalentes, se prefiere la más cercana por nombre. También podemos suponer que al enviar una consulta al mismo servidor, en ausencia de fallas, una consulta distribuida también irá a los mismos servidores. Por lo tanto, incluso si se colocan datos diferentes en las réplicas, la consulta devolverá principalmente los mismos resultados. -### En Orden {#load_balancing-in_order} +### En orden {#load_balancing-in_order} ``` sql load_balancing = in_order @@ -724,7 +777,7 @@ load_balancing = in_order Se accede a las réplicas con el mismo número de errores en el mismo orden en que se especifican en la configuración. Este método es apropiado cuando se sabe exactamente qué réplica es preferible. -### Primero o Aleatorio {#load_balancing-first_or_random} +### Primero o aleatorio {#load_balancing-first_or_random} ``` sql load_balancing = first_or_random @@ -773,12 +826,12 @@ Si se compiló esta parte de la canalización, la consulta puede ejecutarse más ## min\_count\_to\_compile {#min-count-to-compile} -¿cuántas veces usar potencialmente un fragmento de código compilado antes de ejecutar la compilación? por defecto, 3. +¿Cuántas veces usar potencialmente un fragmento de código compilado antes de ejecutar la compilación? Por defecto, 3. For testing, the value can be set to 0: compilation runs synchronously and the query waits for the end of the compilation process before continuing execution. For all other cases, use values ​​starting with 1. Compilation normally takes about 5-10 seconds. Si el valor es 1 o más, la compilación se produce de forma asíncrona en un subproceso independiente. El resultado se utilizará tan pronto como esté listo, incluidas las consultas que se están ejecutando actualmente. Se requiere código compilado para cada combinación diferente de funciones agregadas utilizadas en la consulta y el tipo de claves en la cláusula GROUP BY. -The results of the compilation are saved in the build directory in the form of .so files. There is no restriction on the number of compilation results since they don’t use very much space. Old results will be used after server restarts, except in the case of a server upgrade – in this case, the old results are deleted. +The results of the compilation are saved in the build directory in the form of .so files. There is no restriction on the number of compilation results since they don't use very much space. Old results will be used after server restarts, except in the case of a server upgrade – in this case, the old results are deleted. ## output\_format\_json\_quote\_64bit\_integers {#session_settings-output_format_json_quote_64bit_integers} @@ -869,7 +922,7 @@ Valores posibles: Valor predeterminado: 1. -De forma predeterminada, los bloques insertados en tablas replicadas `INSERT` (consulte \[Replicación de datos\] (../engines/table\_engines/mergetree\_family/replication.md). +De forma predeterminada, los bloques insertados en tablas replicadas `INSERT` declaración se deduplican (ver [Replicación de datos](../../engines/table-engines/mergetree-family/replication.md)). ## deduplicate\_blocks\_in\_dependent\_materialized\_views {#settings-deduplicate-blocks-in-dependent-materialized-views} @@ -886,10 +939,10 @@ Uso De forma predeterminada, la desduplicación no se realiza para las vistas materializadas, sino que se realiza en sentido ascendente, en la tabla de origen. Si se omite un bloque INSERTed debido a la desduplicación en la tabla de origen, no habrá inserción en las vistas materializadas adjuntas. Este comportamiento existe para permitir la inserción de datos altamente agregados en vistas materializadas, para los casos en que los bloques insertados son los mismos después de la agregación de vistas materializadas pero derivados de diferentes INSERT en la tabla de origen. -Al mismo tiempo, este comportamiento “breaks” `INSERT` idempotencia. Si una `INSERT` en la mesa principal fue exitoso y `INSERT` into a materialized view failed (e.g. because of communication failure with Zookeeper) a client will get an error and can retry the operation. However, the materialized view won’t receive the second insert because it will be discarded by deduplication in the main (source) table. The setting `deduplicate_blocks_in_dependent_materialized_views` permite cambiar este comportamiento. Al reintentar, una vista materializada recibirá la inserción de repetición y realizará la comprobación de desduplicación por sí misma, +Al mismo tiempo, este comportamiento “breaks” `INSERT` idempotencia. Si una `INSERT` en la mesa principal fue exitoso y `INSERT` into a materialized view failed (e.g. because of communication failure with Zookeeper) a client will get an error and can retry the operation. However, the materialized view won't receive the second insert because it will be discarded by deduplication in the main (source) table. The setting `deduplicate_blocks_in_dependent_materialized_views` permite cambiar este comportamiento. Al reintentar, una vista materializada recibirá la inserción de repetición y realizará la comprobación de desduplicación por sí misma, ignorando el resultado de la comprobación para la tabla de origen, e insertará filas perdidas debido a la primera falla. -## Método De codificación De Datos: {#settings-max-network-bytes} +## Método de codificación de datos: {#settings-max-network-bytes} Limita el volumen de datos (en bytes) que se recibe o se transmite a través de la red al ejecutar una consulta. Esta configuración se aplica a cada consulta individual. @@ -900,7 +953,7 @@ Valores posibles: Valor predeterminado: 0. -## Método De codificación De Datos: {#settings-max-network-bandwidth} +## Método de codificación de datos: {#settings-max-network-bandwidth} Limita la velocidad del intercambio de datos a través de la red en bytes por segundo. Esta configuración se aplica a todas las consultas. @@ -911,7 +964,7 @@ Valores posibles: Valor predeterminado: 0. -## Todos Los Derechos Reservados {#settings-max-network-bandwidth-for-user} +## Todos los derechos reservados {#settings-max-network-bandwidth-for-user} Limita la velocidad del intercambio de datos a través de la red en bytes por segundo. Esta configuración se aplica a todas las consultas que se ejecutan simultáneamente realizadas por un único usuario. @@ -922,7 +975,7 @@ Valores posibles: Valor predeterminado: 0. -## Todos Los Derechos Reservados {#settings-max-network-bandwidth-for-all-users} +## Todos los derechos reservados {#settings-max-network-bandwidth-for-all-users} Limita la velocidad a la que se intercambian datos a través de la red en bytes por segundo. Esta configuración se aplica a todas las consultas que se ejecutan simultáneamente en el servidor. @@ -1114,7 +1167,7 @@ Ver también: - Tabla del sistema [trace\_log](../../operations/system-tables.md#system_tables-trace_log) -## Los Resultados De La Prueba {#query_profiler_cpu_time_period_ns} +## Los resultados de la prueba {#query_profiler_cpu_time_period_ns} Establece el período para un temporizador de reloj de CPU [perfilador de consultas](../../operations/optimizing-performance/sampling-query-profiler.md). Este temporizador solo cuenta el tiempo de CPU. @@ -1160,14 +1213,14 @@ Valor predeterminado: 0. Habilitar el análisis paralelo de los formatos de datos para preservar el orden. Solo se admite para los formatos TSV, TKSV, CSV y JSONEachRow. -## También Puede Utilizar Los Siguientes métodos De envío: {#min-chunk-bytes-for-parallel-parsing} +## También puede utilizar los siguientes métodos de envío: {#min-chunk-bytes-for-parallel-parsing} - Tipo: unsigned int - Valor predeterminado: 1 MiB El tamaño mínimo de fragmento en bytes, que cada subproceso analizará en paralelo. -## Sistema Abierto {#settings-output_format_avro_codec} +## Sistema abierto {#settings-output_format_avro_codec} Establece el códec de compresión utilizado para el archivo Avro de salida. @@ -1181,7 +1234,7 @@ Valores posibles: Valor predeterminado: `snappy` (si está disponible) o `deflate`. -## Sistema Abierto {#settings-output_format_avro_sync_interval} +## Sistema abierto {#settings-output_format_avro_sync_interval} Establece el tamaño mínimo de datos (en bytes) entre los marcadores de sincronización para el archivo Avro de salida. @@ -1191,7 +1244,7 @@ Valores posibles: 32 (32 bytes) - 1073741824 (1 GiB) Valor predeterminado: 32768 (32 KiB) -## Todos Los Derechos Reservados {#settings-format_avro_schema_registry_url} +## Todos los derechos reservados {#settings-format_avro_schema_registry_url} Establece la URL del Registro de esquemas confluentes para usar con [AvroConfluent](../../interfaces/formats.md#data-format-avro-confluent) formato @@ -1199,4 +1252,14 @@ Tipo: URL Valor predeterminado: Vacío +## background\_pool\_size {#background_pool_size} + +Establece el número de subprocesos que realizan operaciones en segundo plano en motores de tabla (por ejemplo, fusiona [Motor MergeTree](../../engines/table-engines/mergetree-family/index.md) tabla). Esta configuración se aplica al inicio del servidor ClickHouse y no se puede cambiar en una sesión de usuario. Al ajustar esta configuración, puede administrar la carga de la CPU y el disco. Un tamaño de grupo más pequeño utiliza menos recursos de CPU y disco, pero los procesos en segundo plano avanzan más lentamente, lo que eventualmente podría afectar el rendimiento de la consulta. + +Valores posibles: + +- Cualquier entero positivo. + +Valor predeterminado: 16. + [Artículo Original](https://clickhouse.tech/docs/en/operations/settings/settings/) diff --git a/docs/es/operations/system-tables.md b/docs/es/operations/system-tables.md index 7fc0f3123dd..6b3ddf06bd9 100644 --- a/docs/es/operations/system-tables.md +++ b/docs/es/operations/system-tables.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 52 toc_title: Tablas del sistema --- -# Tablas Del Sistema {#system-tables} +# Tablas del sistema {#system-tables} Las tablas del sistema se utilizan para implementar parte de la funcionalidad del sistema y para proporcionar acceso a información sobre cómo funciona el sistema. No puede eliminar una tabla del sistema (pero puede realizar DETACH). @@ -147,29 +147,69 @@ Esta tabla del sistema se utiliza para implementar el `SHOW DATABASES` consulta. ## sistema.detached\_parts {#system_tables-detached_parts} -Contiene información sobre piezas separadas de [Método de codificación de datos:](../engines/table-engines/mergetree-family/mergetree.md) tabla. El `reason` columna especifica por qué se separó la pieza. Para las piezas separadas por el usuario, el motivo está vacío. Tales partes se pueden unir con [ALTER TABLE ATTACH PARTITION\|PART](../query_language/query_language/alter/#alter_attach-partition) comando. Para obtener la descripción de otras columnas, consulte [sistema.parte](#system_tables-parts). Si el nombre de la pieza no es válido, los valores de algunas columnas pueden ser `NULL`. Tales partes se pueden eliminar con [ALTER TABLE DROP DETACHED PART](../query_language/query_language/alter/#alter_drop-detached). +Contiene información sobre piezas separadas de [Método de codificación de datos:](../engines/table-engines/mergetree-family/mergetree.md) tabla. El `reason` columna especifica por qué se separó la pieza. Para las piezas separadas por el usuario, el motivo está vacío. Tales partes se pueden unir con [ALTER TABLE ATTACH PARTITION\|PART](../sql-reference/statements/alter.md#alter_attach-partition) comando. Para obtener la descripción de otras columnas, consulte [sistema.parte](#system_tables-parts). Si el nombre de la pieza no es válido, los valores de algunas columnas pueden ser `NULL`. Tales partes se pueden eliminar con [ALTER TABLE DROP DETACHED PART](../sql-reference/statements/alter.md#alter_drop-detached). -## sistema.diccionario {#system-dictionaries} +## sistema.diccionario {#system_tables-dictionaries} -Contiene información sobre diccionarios externos. +Contiene información sobre [diccionarios externos](../sql-reference/dictionaries/external-dictionaries/external-dicts.md). Columna: -- `name` (String) — Dictionary name. -- `type` (String) — Dictionary type: Flat, Hashed, Cache. -- `origin` (String) — Path to the configuration file that describes the dictionary. -- `attribute.names` (Array(String)) — Array of attribute names provided by the dictionary. -- `attribute.types` (Array(String)) — Corresponding array of attribute types that are provided by the dictionary. -- `has_hierarchy` (UInt8) — Whether the dictionary is hierarchical. -- `bytes_allocated` (UInt64) — The amount of RAM the dictionary uses. -- `hit_rate` (Float64) — For cache dictionaries, the percentage of uses for which the value was in the cache. -- `element_count` (UInt64) — The number of items stored in the dictionary. -- `load_factor` (Float64) — The percentage filled in the dictionary (for a hashed dictionary, the percentage filled in the hash table). -- `creation_time` (DateTime) — The time when the dictionary was created or last successfully reloaded. -- `last_exception` (String) — Text of the error that occurs when creating or reloading the dictionary if the dictionary couldn’t be created. -- `source` (String) — Text describing the data source for the dictionary. +- `database` ([Cadena](../sql-reference/data-types/string.md)) — Name of the database containing the dictionary created by DDL query. Empty string for other dictionaries. +- `name` ([Cadena](../sql-reference/data-types/string.md)) — [Nombre del diccionario](../sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md). +- `status` ([Enum8](../sql-reference/data-types/enum.md)) — Dictionary status. Possible values: + - `NOT_LOADED` — Dictionary was not loaded because it was not used. + - `LOADED` — Dictionary loaded successfully. + - `FAILED` — Unable to load the dictionary as a result of an error. + - `LOADING` — Dictionary is loading now. + - `LOADED_AND_RELOADING` — Dictionary is loaded successfully, and is being reloaded right now (frequent reasons: [SYSTEM RELOAD DICTIONARY](../sql-reference/statements/system.md#query_language-system-reload-dictionary) consulta, tiempo de espera, configuración del diccionario ha cambiado). + - `FAILED_AND_RELOADING` — Could not load the dictionary as a result of an error and is loading now. +- `origin` ([Cadena](../sql-reference/data-types/string.md)) — Path to the configuration file that describes the dictionary. +- `type` ([Cadena](../sql-reference/data-types/string.md)) — Type of a dictionary allocation. [Almacenamiento de diccionarios en la memoria](../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md). +- `key` — [Tipo de llave](../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md#ext_dict_structure-key): Clave numérica ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) or Сomposite key ([Cadena](../sql-reference/data-types/string.md)) — form “(type 1, type 2, …, type n)”. +- `attribute.names` ([Matriz](../sql-reference/data-types/array.md)([Cadena](../sql-reference/data-types/string.md))) — Array of [nombres de atributos](../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md#ext_dict_structure-attributes) proporcionada por el diccionario. +- `attribute.types` ([Matriz](../sql-reference/data-types/array.md)([Cadena](../sql-reference/data-types/string.md))) — Corresponding array of [tipos de atributos](../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md#ext_dict_structure-attributes) que son proporcionados por el diccionario. +- `bytes_allocated` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Amount of RAM allocated for the dictionary. +- `query_count` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Number of queries since the dictionary was loaded or since the last successful reboot. +- `hit_rate` ([Float64](../sql-reference/data-types/float.md)) — For cache dictionaries, the percentage of uses for which the value was in the cache. +- `element_count` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Number of items stored in the dictionary. +- `load_factor` ([Float64](../sql-reference/data-types/float.md)) — Percentage filled in the dictionary (for a hashed dictionary, the percentage filled in the hash table). +- `source` ([Cadena](../sql-reference/data-types/string.md)) — Text describing the [fuente de datos](../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md) para el diccionario. +- `lifetime_min` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Minimum [vida](../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md) del diccionario en la memoria, después de lo cual ClickHouse intenta volver a cargar el diccionario (si `invalidate_query` está configurado, entonces solo si ha cambiado). Establecer en segundos. +- `lifetime_max` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Maximum [vida](../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md) del diccionario en la memoria, después de lo cual ClickHouse intenta volver a cargar el diccionario (si `invalidate_query` está configurado, entonces solo si ha cambiado). Establecer en segundos. +- `loading_start_time` ([FechaHora](../sql-reference/data-types/datetime.md)) — Start time for loading the dictionary. +- `last_successful_update_time` ([FechaHora](../sql-reference/data-types/datetime.md)) — End time for loading or updating the dictionary. Helps to monitor some troubles with external sources and investigate causes. +- `loading_duration` ([Float32](../sql-reference/data-types/float.md)) — Duration of a dictionary loading. +- `last_exception` ([Cadena](../sql-reference/data-types/string.md)) — Text of the error that occurs when creating or reloading the dictionary if the dictionary couldn't be created. -Tenga en cuenta que la cantidad de memoria utilizada por el diccionario no es proporcional a la cantidad de elementos almacenados en él. Por lo tanto, para los diccionarios planos y en caché, todas las celdas de memoria se asignan previamente, independientemente de qué tan lleno esté realmente el diccionario. +**Ejemplo** + +Configurar el diccionario. + +``` sql +CREATE DICTIONARY dictdb.dict +( + `key` Int64 DEFAULT -1, + `value_default` String DEFAULT 'world', + `value_expression` String DEFAULT 'xxx' EXPRESSION 'toString(127 * 172)' +) +PRIMARY KEY key +SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'dicttbl' DB 'dictdb')) +LIFETIME(MIN 0 MAX 1) +LAYOUT(FLAT()) +``` + +Asegúrese de que el diccionario esté cargado. + +``` sql +SELECT * FROM system.dictionaries +``` + +``` text +┌─database─┬─name─┬─status─┬─origin──────┬─type─┬─key────┬─attribute.names──────────────────────┬─attribute.types─────┬─bytes_allocated─┬─query_count─┬─hit_rate─┬─element_count─┬───────────load_factor─┬─source─────────────────────┬─lifetime_min─┬─lifetime_max─┬──loading_start_time─┌──last_successful_update_time─┬──────loading_duration─┬─last_exception─┐ +│ dictdb │ dict │ LOADED │ dictdb.dict │ Flat │ UInt64 │ ['value_default','value_expression'] │ ['String','String'] │ 74032 │ 0 │ 1 │ 1 │ 0.0004887585532746823 │ ClickHouse: dictdb.dicttbl │ 0 │ 1 │ 2020-03-04 04:17:34 │ 2020-03-04 04:30:34 │ 0.002 │ │ +└──────────┴──────┴────────┴─────────────┴──────┴────────┴──────────────────────────────────────┴─────────────────────┴─────────────────┴─────────────┴──────────┴───────────────┴───────────────────────┴────────────────────────────┴──────────────┴──────────────┴─────────────────────┴──────────────────────────────┘───────────────────────┴────────────────┘ +``` ## sistema.evento {#system_tables-events} @@ -366,7 +406,7 @@ Esto es similar a la tabla DUAL que se encuentra en otros DBMS. Contiene información sobre partes de [Método de codificación de datos:](../engines/table-engines/mergetree-family/mergetree.md) tabla. -Cada fila describe una parte de los datos. +Cada fila describe una parte de datos. Columna: @@ -379,7 +419,7 @@ Columna: - `name` (`String`) – Name of the data part. -- `active` (`UInt8`) – Flag that indicates whether the data part is active. If a data part is active, it’s used in a table. Otherwise, it’s deleted. Inactive data parts remain after merging. +- `active` (`UInt8`) – Flag that indicates whether the data part is active. If a data part is active, it's used in a table. Otherwise, it's deleted. Inactive data parts remain after merging. - `marks` (`UInt64`) – The number of marks. To get the approximate number of rows in a data part, multiply `marks` por la granularidad del índice (generalmente 8192) (esta sugerencia no funciona para la granularidad adaptativa). @@ -421,7 +461,7 @@ Columna: - `primary_key_bytes_in_memory_allocated` (`UInt64`) – The amount of memory (in bytes) reserved for primary key values. -- `is_frozen` (`UInt8`) – Flag that shows that a partition data backup exists. 1, the backup exists. 0, the backup doesn’t exist. For more details, see [FREEZE PARTITION](../sql-reference/statements/alter.md#alter_freeze-partition) +- `is_frozen` (`UInt8`) – Flag that shows that a partition data backup exists. 1, the backup exists. 0, the backup doesn't exist. For more details, see [FREEZE PARTITION](../sql-reference/statements/alter.md#alter_freeze-partition) - `database` (`String`) – Name of the database. @@ -499,7 +539,7 @@ Contiene entradas de registro. El nivel de registro que va a esta tabla se puede Columna: - `event_date` (`Date`) - Fecha de la entrada. -- `event_time` (`DateTime`) - Tiempo de la entrada. +- `event_time` (`DateTime`) - Hora de la entrada. - `microseconds` (`UInt32`) - Microsegundos de la entrada. - `thread_name` (String) — Name of the thread from which the logging was done. - `thread_id` (UInt64) — OS thread ID. @@ -533,7 +573,7 @@ Para habilitar el registro de consultas, [Log\_queries](settings/settings.md#set El `system.query_log` tabla registra dos tipos de consultas: 1. Consultas iniciales ejecutadas directamente por el cliente. -2. Niño consultas que fueron iniciados por otras consultas (distribuida de la ejecución de la consulta). Para estos tipos de consultas, la información sobre el padre de las consultas se muestra en la `initial_*` columna. +2. Consultas secundarias iniciadas por otras consultas (para la ejecución de consultas distribuidas). Para estos tipos de consultas, la información sobre las consultas principales se muestra en el `initial_*` columna. Columna: @@ -570,7 +610,7 @@ Columna: - `interface` (UInt8) — Interface that the query was initiated from. Possible values: - 1 — TCP. - 2 — HTTP. -- `os_user` (String) — OS’s username who runs [Casa de clics-cliente](../interfaces/cli.md). +- `os_user` (String) — OS's username who runs [Casa de clics-cliente](../interfaces/cli.md). - `client_hostname` (String) — Hostname of the client machine where the [Casa de clics-cliente](../interfaces/cli.md) o se ejecuta otro cliente TCP. - `client_name` (String) — The [Casa de clics-cliente](../interfaces/cli.md) o otro nombre de cliente TCP. - `client_revision` (UInt32) — Revision of the [Casa de clics-cliente](../interfaces/cli.md) o otro cliente TCP. @@ -644,7 +684,7 @@ Columna: - `interface` (UInt8) — Interface that the query was initiated from. Possible values: - 1 — TCP. - 2 — HTTP. -- `os_user` (String) — OS’s username who runs [Casa de clics-cliente](../interfaces/cli.md). +- `os_user` (String) — OS's username who runs [Casa de clics-cliente](../interfaces/cli.md). - `client_hostname` (String) — Hostname of the client machine where the [Casa de clics-cliente](../interfaces/cli.md) o se ejecuta otro cliente TCP. - `client_name` (String) — The [Casa de clics-cliente](../interfaces/cli.md) o otro nombre de cliente TCP. - `client_revision` (UInt32) — Revision of the [Casa de clics-cliente](../interfaces/cli.md) o otro cliente TCP. @@ -680,24 +720,26 @@ Para analizar los registros, utilice el `addressToLine`, `addressToSymbol` y `de Columna: -- `event_date`([Fecha](../sql-reference/data-types/date.md)) — Date of sampling moment. +- `event_date` ([Fecha](../sql-reference/data-types/date.md)) — Date of sampling moment. -- `event_time`([FechaHora](../sql-reference/data-types/datetime.md)) — Timestamp of sampling moment. +- `event_time` ([FechaHora](../sql-reference/data-types/datetime.md)) — Timestamp of the sampling moment. -- `revision`([UInt32](../sql-reference/data-types/int-uint.md)) — ClickHouse server build revision. +- `timestamp_ns` ([UInt64](../sql-reference/data-types/int-uint.md)) — Timestamp of the sampling moment in nanoseconds. + +- `revision` ([UInt32](../sql-reference/data-types/int-uint.md)) — ClickHouse server build revision. Cuando se conecta al servidor por `clickhouse-client`, ves la cadena similar a `Connected to ClickHouse server version 19.18.1 revision 54429.`. Este campo contiene el `revision`, pero no el `version` de un servidor. -- `timer_type`([Enum8](../sql-reference/data-types/enum.md)) — Timer type: +- `timer_type` ([Enum8](../sql-reference/data-types/enum.md)) — Timer type: - `Real` representa el tiempo del reloj de pared. - `CPU` representa el tiempo de CPU. -- `thread_number`([UInt32](../sql-reference/data-types/int-uint.md)) — Thread identifier. +- `thread_number` ([UInt32](../sql-reference/data-types/int-uint.md)) — Thread identifier. -- `query_id`([Cadena](../sql-reference/data-types/string.md)) — Query identifier that can be used to get details about a query that was running from the [query\_log](#system_tables-query_log) tabla del sistema. +- `query_id` ([Cadena](../sql-reference/data-types/string.md)) — Query identifier that can be used to get details about a query that was running from the [query\_log](#system_tables-query_log) tabla del sistema. -- `trace`([Matriz (UInt64)](../sql-reference/data-types/array.md)) — Stack trace at the moment of sampling. Each element is a virtual memory address inside ClickHouse server process. +- `trace` ([Matriz (UInt64)](../sql-reference/data-types/array.md)) — Stack trace at the moment of sampling. Each element is a virtual memory address inside ClickHouse server process. **Ejemplo** @@ -839,30 +881,59 @@ WHERE Si esta consulta no devuelve nada, significa que todo está bien. -## sistema.configuración {#system-settings} +## sistema.configuración {#system-tables-system-settings} -Contiene información sobre la configuración actualmente en uso. -Es decir, se usa para ejecutar la consulta que está utilizando para leer del sistema.tabla de configuración. +Contiene información sobre la configuración de sesión para el usuario actual. Columna: -- `name` (String) — Setting name. -- `value` (String) — Setting value. -- `description` (String) — Setting description. -- `type` (String) — Setting type (implementation specific string value). -- `changed` (UInt8) — Whether the setting was explicitly defined in the config or explicitly changed. -- `min` (Nullable(String)) — Get minimum allowed value (if any is set via [limitación](settings/constraints-on-settings.md#constraints-on-settings)). -- `max` (Nullable(String)) — Get maximum allowed value (if any is set via [limitación](settings/constraints-on-settings.md#constraints-on-settings)). -- `readonly` (UInt8) — Can user change this setting (for more info, look into [limitación](settings/constraints-on-settings.md#constraints-on-settings)). +- `name` ([Cadena](../sql-reference/data-types/string.md)) — Setting name. +- `value` ([Cadena](../sql-reference/data-types/string.md)) — Setting value. +- `changed` ([UInt8](../sql-reference/data-types/int-uint.md#uint-ranges)) — Shows whether a setting is changed from its default value. +- `description` ([Cadena](../sql-reference/data-types/string.md)) — Short setting description. +- `min` ([NULL](../sql-reference/data-types/nullable.md)([Cadena](../sql-reference/data-types/string.md))) — Minimum value of the setting, if any is set via [limitación](settings/constraints-on-settings.md#constraints-on-settings). Si la configuración no tiene ningún valor mínimo, contiene [NULL](../sql-reference/syntax.md#null-literal). +- `max` ([NULL](../sql-reference/data-types/nullable.md)([Cadena](../sql-reference/data-types/string.md))) — Maximum value of the setting, if any is set via [limitación](settings/constraints-on-settings.md#constraints-on-settings). Si la configuración no tiene ningún valor máximo, contiene [NULL](../sql-reference/syntax.md#null-literal). +- `readonly` ([UInt8](../sql-reference/data-types/int-uint.md#uint-ranges)) — Shows whether the current user can change the setting: + - `0` — Current user can change the setting. + - `1` — Current user can't change the setting. -Ejemplo: +**Ejemplo** + +En el ejemplo siguiente se muestra cómo obtener información sobre la configuración cuyo nombre contiene `min_i`. ``` sql -SELECT name, value +SELECT * FROM system.settings -WHERE changed +WHERE name LIKE '%min_i%' ``` +``` text +┌─name────────────────────────────────────────┬─value─────┬─changed─┬─description───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┬─min──┬─max──┬─readonly─┐ +│ min_insert_block_size_rows │ 1048576 │ 0 │ Squash blocks passed to INSERT query to specified size in rows, if blocks are not big enough. │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ 0 │ +│ min_insert_block_size_bytes │ 268435456 │ 0 │ Squash blocks passed to INSERT query to specified size in bytes, if blocks are not big enough. │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ 0 │ +│ read_backoff_min_interval_between_events_ms │ 1000 │ 0 │ Settings to reduce the number of threads in case of slow reads. Do not pay attention to the event, if the previous one has passed less than a certain amount of time. │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ 0 │ +└─────────────────────────────────────────────┴───────────┴─────────┴───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┴──────┴──────┴──────────┘ +``` + +Uso de `WHERE changed` puede ser útil, por ejemplo, cuando se desea comprobar: + +- Si los ajustes de los archivos de configuración se cargan correctamente y están en uso. +- Configuración que cambió en la sesión actual. + + + +``` sql +SELECT * FROM system.settings WHERE changed AND name='load_balancing' +``` + +**Ver también** + +- [Configuración](settings/index.md#session-settings-intro) +- [Permisos para consultas](settings/permissions-for-queries.md#settings_readonly) +- [Restricciones en la configuración](settings/constraints-on-settings.md) + +## sistema.table\_engines {#system.table_engines} + ``` text ┌─name───────────────────┬─value───────┐ │ max_threads │ 8 │ @@ -950,7 +1021,7 @@ Esta tabla contiene las siguientes columnas (el tipo de columna se muestra entre - `partition_key` (String) - La expresión de clave de partición especificada en la tabla. -- `sorting_key` (Cadena) - La clave de clasificación de la expresión especificada en la tabla. +- `sorting_key` (String) - La expresión de clave de ordenación especificada en la tabla. - `primary_key` (String) - La expresión de clave principal especificada en la tabla. @@ -968,7 +1039,7 @@ Esta tabla contiene las siguientes columnas (el tipo de columna se muestra entre - If the table stores data on disk, returns used space on disk (i.e. compressed). - Si la tabla almacena datos en la memoria, devuelve el número aproximado de bytes utilizados en la memoria. -El `system.tables` se utiliza la tabla en `SHOW TABLES` implementación de consultas. +El `system.tables` se utiliza en `SHOW TABLES` implementación de consultas. ## sistema.Zookeeper {#system-zookeeper} diff --git a/docs/es/operations/tips.md b/docs/es/operations/tips.md index d45bc9bcda4..4d3e76c7d62 100644 --- a/docs/es/operations/tips.md +++ b/docs/es/operations/tips.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 58 toc_title: Recomendaciones de uso --- -# Recomendaciones De Uso {#usage-recommendations} +# Recomendaciones de uso {#usage-recommendations} ## CPU Scaling Governor {#cpu-scaling-governor} @@ -15,7 +15,7 @@ Utilice siempre el `performance` gobernador de escala. El `on-demand` regulador $ echo 'performance' | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor ``` -## Limitaciones De La CPU {#cpu-limitations} +## Limitaciones de la CPU {#cpu-limitations} Los procesadores pueden sobrecalentarse. Utilizar `dmesg` para ver si la velocidad de reloj de la CPU era limitada debido al sobrecalentamiento. La restricción también se puede establecer externamente en el nivel del centro de datos. Usted puede utilizar `turbostat` para controlarlo bajo una carga. @@ -32,7 +32,7 @@ No deshabilite el sobrecompromiso. Valor `cat /proc/sys/vm/overcommit_memory` de $ echo 0 | sudo tee /proc/sys/vm/overcommit_memory ``` -## Páginas Enormes {#huge-pages} +## Páginas enormes {#huge-pages} Siempre deshabilite las páginas enormes transparentes. Interfiere con los asignadores de memoria, lo que conduce a una degradación significativa del rendimiento. @@ -43,10 +43,10 @@ $ echo 'never' | sudo tee /sys/kernel/mm/transparent_hugepage/enabled Utilizar `perf top` para ver el tiempo pasado en el kernel para la administración de memoria. Las páginas enormes permanentes tampoco necesitan ser asignadas. -## Subsistema De Almacenamiento {#storage-subsystem} +## Subsistema de almacenamiento {#storage-subsystem} Si su presupuesto le permite usar SSD, use SSD. -Si no, utilice el disco duro. Discos Duros SATA de 7200 RPM va a hacer. +Si no, use HDD. Los discos duros SATA 7200 RPM servirán. Dar preferencia a una gran cantidad de servidores con discos duros locales sobre un número menor de servidores con estantes de discos conectados. Pero para almacenar archivos con consultas raras, los estantes funcionarán. @@ -76,13 +76,13 @@ Independientemente del uso de RAID, utilice siempre la replicación para la segu Habilite NCQ con una cola larga. Para HDD, elija el programador CFQ, y para SSD, elija noop. No reduzca el ‘readahead’ configuración. Para HDD, habilite la memoria caché de escritura. -## Sistema De Archivos {#file-system} +## Sistema de archivos {#file-system} Ext4 es la opción más confiable. Establecer las opciones de montaje `noatime, nobarrier`. XFS también es adecuado, pero no ha sido probado tan a fondo con ClickHouse. La mayoría de los otros sistemas de archivos también deberían funcionar bien. Los sistemas de archivos con asignación retrasada funcionan mejor. -## Núcleo De Linux {#linux-kernel} +## Núcleo de Linux {#linux-kernel} No use un kernel de Linux obsoleto. @@ -97,7 +97,7 @@ Utilice al menos una red de 10 GB, si es posible. 1 Gb también funcionará, per Probablemente ya esté utilizando ZooKeeper para otros fines. Puede usar la misma instalación de ZooKeeper, si aún no está sobrecargada. -It’s best to use a fresh version of ZooKeeper – 3.4.9 or later. The version in stable Linux distributions may be outdated. +It's best to use a fresh version of ZooKeeper – 3.4.9 or later. The version in stable Linux distributions may be outdated. Nunca debe usar scripts escritos manualmente para transferir datos entre diferentes clústeres de ZooKeeper, ya que el resultado será incorrecto para los nodos secuenciales. Nunca utilice el “zkcopy” utilidad por la misma razón: https://github.com/ksprojects/zkcopy/issues/15 @@ -105,7 +105,7 @@ Si desea dividir un clúster ZooKeeper existente en dos, la forma correcta es au No ejecute ZooKeeper en los mismos servidores que ClickHouse. Porque ZooKeeper es muy sensible a la latencia y ClickHouse puede utilizar todos los recursos del sistema disponibles. -Con la configuración predeterminada, Cuidador es una bomba de tiempo: +Con la configuración predeterminada, ZooKeeper es una bomba de tiempo: > El servidor ZooKeeper no eliminará archivos de instantáneas y registros antiguos cuando utilice la configuración predeterminada (consulte autopurge), y esto es responsabilidad del operador. diff --git a/docs/es/operations/troubleshooting.md b/docs/es/operations/troubleshooting.md index 82cda7b9c5e..04b00e4a5cd 100644 --- a/docs/es/operations/troubleshooting.md +++ b/docs/es/operations/troubleshooting.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 46 toc_title: "Soluci\xF3n de problemas" --- -# Solución De Problemas {#troubleshooting} +# Solución de problemas {#troubleshooting} - [Instalación](#troubleshooting-installation-errors) - [Conexión al servidor](#troubleshooting-accepts-no-connections) @@ -14,19 +14,19 @@ toc_title: "Soluci\xF3n de problemas" ## Instalación {#troubleshooting-installation-errors} -### No Puede Obtener Paquetes Deb Del Repositorio De Clickhouse Con Apt-get {#you-cannot-get-deb-packages-from-clickhouse-repository-with-apt-get} +### No puede obtener paquetes Deb del repositorio ClickHouse con Apt-get {#you-cannot-get-deb-packages-from-clickhouse-repository-with-apt-get} - Compruebe la configuración del firewall. - Si no puede acceder al repositorio por cualquier motivo, descargue los paquetes como se describe en el [Primeros pasos](../getting-started/index.md) artículo e instálelos manualmente usando el `sudo dpkg -i ` comando. También necesitará el `tzdata` paquete. -## Conexión Al Servidor {#troubleshooting-accepts-no-connections} +## Conexión al servidor {#troubleshooting-accepts-no-connections} Posibles problemas: - El servidor no se está ejecutando. - Parámetros de configuración inesperados o incorrectos. -### El Servidor No Se está Ejecutando {#server-is-not-running} +### El servidor no se está ejecutando {#server-is-not-running} **Compruebe si el servidor está ejecutado** @@ -95,7 +95,7 @@ $ sudo -u clickhouse /usr/bin/clickhouse-server --config-file /etc/clickhouse-se Este comando inicia el servidor como una aplicación interactiva con parámetros estándar del script de inicio automático. En este modo `clickhouse-server` imprime todos los mensajes de eventos en la consola. -### Parámetros De configuración {#configuration-parameters} +### Parámetros de configuración {#configuration-parameters} Comprobar: @@ -126,7 +126,7 @@ Comprobar: Es posible que esté utilizando el nombre de usuario o la contraseña incorrectos. -## Procesamiento De Consultas {#troubleshooting-does-not-process-queries} +## Procesamiento de consultas {#troubleshooting-does-not-process-queries} Si ClickHouse no puede procesar la consulta, envía una descripción de error al cliente. En el `clickhouse-client` obtienes una descripción del error en la consola. Si está utilizando la interfaz HTTP, ClickHouse envía la descripción del error en el cuerpo de la respuesta. Por ejemplo: @@ -139,7 +139,7 @@ Si empiezas `clickhouse-client` con el `stack-trace` parámetro, ClickHouse devu Es posible que vea un mensaje sobre una conexión rota. En este caso, puede repetir la consulta. Si la conexión se rompe cada vez que realiza la consulta, compruebe si hay errores en los registros del servidor. -## Eficiencia Del Procesamiento De Consultas {#troubleshooting-too-slow} +## Eficiencia del procesamiento de consultas {#troubleshooting-too-slow} Si ve que ClickHouse funciona demasiado lentamente, debe perfilar la carga en los recursos del servidor y la red para sus consultas. diff --git a/docs/es/operations/update.md b/docs/es/operations/update.md index 159e94a870e..11d15381d72 100644 --- a/docs/es/operations/update.md +++ b/docs/es/operations/update.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 47 toc_title: "Actualizaci\xF3n de ClickHouse" --- -# Actualización De ClickHouse {#clickhouse-update} +# Actualización de ClickHouse {#clickhouse-update} Si se instaló ClickHouse desde paquetes deb, ejecute los siguientes comandos en el servidor: diff --git a/docs/es/operations/utilities/clickhouse-benchmark.md b/docs/es/operations/utilities/clickhouse-benchmark.md index 6153add3127..9bcafa40dfe 100644 --- a/docs/es/operations/utilities/clickhouse-benchmark.md +++ b/docs/es/operations/utilities/clickhouse-benchmark.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 61 toc_title: Sistema abierto. --- -# Sistema Abierto {#clickhouse-benchmark} +# Sistema abierto {#clickhouse-benchmark} Se conecta a un servidor ClickHouse y envía repetidamente las consultas especificadas. @@ -38,7 +38,7 @@ clickhouse-benchmark [keys] < queries_file - `-c N`, `--concurrency=N` — Number of queries that `clickhouse-benchmark` se envía simultáneamente. Valor predeterminado: 1. - `-d N`, `--delay=N` — Interval in seconds between intermediate reports (set 0 to disable reports). Default value: 1. -- `-h WORD`, `--host=WORD` — Server host. Default value: `localhost`. Para el [modo de comparación](#clickhouse-benchmark-comparison-mode) se pueden utilizar varios `-h` claves. +- `-h WORD`, `--host=WORD` — Server host. Default value: `localhost`. Para el [modo de comparación](#clickhouse-benchmark-comparison-mode) puedes usar múltiples `-h` claves. - `-p N`, `--port=N` — Server port. Default value: 9000. For the [modo de comparación](#clickhouse-benchmark-comparison-mode) puedes usar múltiples `-p` claves. - `-i N`, `--iterations=N` — Total number of queries. Default value: 0. - `-r`, `--randomize` — Random order of queries execution if there is more then one input query. @@ -91,7 +91,7 @@ En el informe puedes encontrar: - Punto final del servidor ClickHouse. - Número de consultas procesadas. - - QPS: QPS: ¿cuántas consultas servidor realizó por segundo durante un período de tiempo especificado en el `--delay` argumento. + - QPS: QPS: ¿Cuántas consultas realizó el servidor por segundo durante un período `--delay` argumento. - RPS: ¿Cuántas filas lee el servidor por segundo durante un período `--delay` argumento. - MiB/s: ¿Cuántos mebibytes servidor leído por segundo durante un período especificado en el `--delay` argumento. - resultado RPS: ¿Cuántas filas colocadas por el servidor al resultado de una consulta por segundo durante un período `--delay` argumento. @@ -99,7 +99,7 @@ En el informe puedes encontrar: - Percentiles de tiempo de ejecución de consultas. -## Modo De comparación {#clickhouse-benchmark-comparison-mode} +## Modo de comparación {#clickhouse-benchmark-comparison-mode} `clickhouse-benchmark` puede comparar el rendimiento de dos servidores ClickHouse en ejecución. diff --git a/docs/es/operations/utilities/clickhouse-copier.md b/docs/es/operations/utilities/clickhouse-copier.md index 0fe432336f2..5717ffaa737 100644 --- a/docs/es/operations/utilities/clickhouse-copier.md +++ b/docs/es/operations/utilities/clickhouse-copier.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 59 toc_title: "M\xE9todo de codificaci\xF3n de datos:" --- -# Método De codificación De Datos: {#clickhouse-copier} +# Método de codificación de datos: {#clickhouse-copier} Copia datos de las tablas de un clúster en tablas de otro (o del mismo) clúster. @@ -26,7 +26,7 @@ Después de comenzar, `clickhouse-copier`: Para reducir el tráfico de red, recomendamos ejecutar `clickhouse-copier` en el mismo servidor donde se encuentran los datos de origen. -## Ejecución De Clickhouse-copiadora {#running-clickhouse-copier} +## Ejecución de Clickhouse-copiadora {#running-clickhouse-copier} La utilidad debe ejecutarse manualmente: @@ -43,7 +43,7 @@ Parámetros: - `task-upload-force` — Force upload `task-file` incluso si el nodo ya existe. - `base-dir` — The path to logs and auxiliary files. When it starts, `clickhouse-copier` crear `clickhouse-copier_YYYYMMHHSS_` subdirectorios en `$base-dir`. Si se omite este parámetro, los directorios se crean en el directorio donde `clickhouse-copier` se puso en marcha. -## Formato De Zookeeper.XML {#format-of-zookeeper-xml} +## Formato de Zookeeper.XML {#format-of-zookeeper-xml} ``` xml @@ -62,7 +62,7 @@ Parámetros: ``` -## Configuración De Tareas De Copia {#configuration-of-copying-tasks} +## Configuración de tareas de copia {#configuration-of-copying-tasks} ``` xml diff --git a/docs/es/operations/utilities/clickhouse-local.md b/docs/es/operations/utilities/clickhouse-local.md index fc41bc9af3c..e122f668f53 100644 --- a/docs/es/operations/utilities/clickhouse-local.md +++ b/docs/es/operations/utilities/clickhouse-local.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 60 toc_title: clickhouse-local --- -# Sistema Abierto {#clickhouse-local} +# clickhouse-local {#clickhouse-local} El `clickhouse-local` El programa le permite realizar un procesamiento rápido en archivos locales, sin tener que implementar y configurar el servidor ClickHouse. diff --git a/docs/es/operations/utilities/index.md b/docs/es/operations/utilities/index.md index 221b043986a..a69397a326c 100644 --- a/docs/es/operations/utilities/index.md +++ b/docs/es/operations/utilities/index.md @@ -1,14 +1,14 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa -toc_folder_title: Utilities +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: Utilidad toc_priority: 56 toc_title: "Descripci\xF3n" --- # Utilidad ClickHouse {#clickhouse-utility} -- [Sistema abierto.](clickhouse-local.md) — Allows running SQL queries on data without stopping the ClickHouse server, similar to how `awk` hace esto. +- [Sistema abierto.](clickhouse-local.md#clickhouse-local) — Allows running SQL queries on data without stopping the ClickHouse server, similar to how `awk` hace esto. - [Método de codificación de datos:](clickhouse-copier.md) — Copies (and reshards) data from one cluster to another cluster. - [Sistema abierto.](clickhouse-benchmark.md) — Loads server with the custom queries and settings. diff --git a/docs/es/sql-reference/aggregate-functions/combinators.md b/docs/es/sql-reference/aggregate-functions/combinators.md index dc0f5cd0181..c9fdcb9478f 100644 --- a/docs/es/sql-reference/aggregate-functions/combinators.md +++ b/docs/es/sql-reference/aggregate-functions/combinators.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 37 -toc_title: Combinadores de funciones agregadas +toc_title: Combinadores --- -# Combinadores De Funciones Agregadas {#aggregate_functions_combinators} +# Combinadores de funciones agregadas {#aggregate_functions_combinators} El nombre de una función agregada puede tener un sufijo anexado. Esto cambia la forma en que funciona la función de agregado. @@ -53,33 +53,110 @@ Convierte una función de agregado para tablas en una función de agregado para ## -OPor defecto {#agg-functions-combinator-ordefault} -Rellena el valor predeterminado del tipo devuelto de la función de agregado si no hay nada que agregar. +Cambia el comportamiento de una función agregada. + +Si una función agregada no tiene valores de entrada, con este combinador devuelve el valor predeterminado para su tipo de datos de retorno. Se aplica a las funciones agregadas que pueden tomar datos de entrada vacíos. + +`-OrDefault` se puede utilizar con otros combinadores. + +**Sintaxis** + +``` sql +OrDefault(x) +``` + +**Parámetros** + +- `x` — Aggregate function parameters. + +**Valores devueltos** + +Devuelve el valor predeterminado del tipo devuelto de una función de agregado si no hay nada que agregar. + +El tipo depende de la función de agregado utilizada. + +**Ejemplo** + +Consulta: ``` sql SELECT avg(number), avgOrDefault(number) FROM numbers(0) ``` +Resultado: + ``` text ┌─avg(number)─┬─avgOrDefault(number)─┐ │ nan │ 0 │ └─────────────┴──────────────────────┘ ``` -## -OrNull {#agg-functions-combinator-ornull} +También `-OrDefault` se puede utilizar con otros combinadores. Es útil cuando la función de agregado no acepta la entrada vacía. -Llenar `null` si no hay nada que agregar. La columna de retorno será anulable. +Consulta: ``` sql -SELECT avg(number), avgOrNull(number) FROM numbers(0) +SELECT avgOrDefaultIf(x, x > 10) +FROM +( + SELECT toDecimal32(1.23, 2) AS x +) ``` +Resultado: + ``` text -┌─avg(number)─┬─avgOrNull(number)─┐ -│ nan │ ᴺᵁᴸᴸ │ -└─────────────┴───────────────────┘ +┌─avgOrDefaultIf(x, greater(x, 10))─┐ +│ 0.00 │ +└───────────────────────────────────┘ ``` --OrDefault y -OrNull se pueden combinar con otros combinadores. Es útil cuando la función de agregado no acepta la entrada vacía. +## -OrNull {#agg-functions-combinator-ornull} + +Cambia el comportamiento de una función agregada. + +Este combinador convierte un resultado de una función agregada en [NULL](../data-types/nullable.md) tipo de datos. Si la función de agregado no tiene valores para calcular devuelve [NULL](../syntax.md#null-literal). + +`-OrNull` se puede utilizar con otros combinadores. + +**Sintaxis** + +``` sql +OrNull(x) +``` + +**Parámetros** + +- `x` — Aggregate function parameters. + +**Valores devueltos** + +- El resultado de la función de agregado, convertida a la `Nullable` tipo de datos. +- `NULL`, si no hay nada que agregar. + +Tipo: `Nullable(aggregate function return type)`. + +**Ejemplo** + +Añadir `-orNull` hasta el final de la función agregada. + +Consulta: + +``` sql +SELECT sumOrNull(number), toTypeName(sumOrNull(number)) FROM numbers(10) WHERE number > 10 +``` + +Resultado: + +``` text +┌─sumOrNull(number)─┬─toTypeName(sumOrNull(number))─┐ +│ ᴺᵁᴸᴸ │ Nullable(UInt64) │ +└───────────────────┴───────────────────────────────┘ +``` + +También `-OrNull` se puede utilizar con otros combinadores. Es útil cuando la función de agregado no acepta la entrada vacía. + +Consulta: ``` sql SELECT avgOrNullIf(x, x > 10) @@ -89,6 +166,8 @@ FROM ) ``` +Resultado: + ``` text ┌─avgOrNullIf(x, greater(x, 10))─┐ │ ᴺᵁᴸᴸ │ diff --git a/docs/es/sql-reference/aggregate-functions/index.md b/docs/es/sql-reference/aggregate-functions/index.md index c944330e9d4..7c7d58d5f94 100644 --- a/docs/es/sql-reference/aggregate-functions/index.md +++ b/docs/es/sql-reference/aggregate-functions/index.md @@ -1,12 +1,12 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa -toc_folder_title: Aggregate Functions +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: Funciones agregadas toc_priority: 33 toc_title: "Implantaci\xF3n" --- -# Funciones Agregadas {#aggregate-functions} +# Funciones agregadas {#aggregate-functions} Las funciones agregadas funcionan en el [normal](http://www.sql-tutorial.com/sql-aggregate-functions-sql-tutorial) forma esperada por los expertos en bases de datos. diff --git a/docs/es/sql-reference/aggregate-functions/parametric-functions.md b/docs/es/sql-reference/aggregate-functions/parametric-functions.md index 4b9a695f119..9aac66ae47e 100644 --- a/docs/es/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/es/sql-reference/aggregate-functions/parametric-functions.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 38 -toc_title: "Funciones agregadas param\xE9tricas" +toc_title: "Param\xE9trico" --- -# Funciones Agregadas paramétricas {#aggregate_functions_parametric} +# Funciones agregadas paramétricas {#aggregate_functions_parametric} Some aggregate functions can accept not only argument columns (used for compression), but a set of parameters – constants for initialization. The syntax is two pairs of brackets instead of one. The first is for parameters, and the second is for arguments. @@ -111,7 +111,7 @@ Tipo: `UInt8`. - `(?N)` — Matches the condition argument at position `N`. Las condiciones están numeradas en el `[1, 32]` gama. Por ejemplo, `(?1)` coincide con el argumento pasado al `cond1` parámetro. -- `.*` — Matches any number of events. You don’t need conditional arguments to match this element of the pattern. +- `.*` — Matches any number of events. You don't need conditional arguments to match this element of the pattern. - `(?t operator value)` — Sets the time in seconds that should separate two events. For example, pattern `(?1)(?t>1800)(?2)` coincide con los eventos que ocurren a más de 1800 segundos el uno del otro. Un número arbitrario de cualquier evento puede estar entre estos eventos. Puede usar el `>=`, `>`, `<`, `<=` operador. @@ -316,7 +316,7 @@ Resultado: ## retención {#retention} La función toma como argumentos un conjunto de condiciones de 1 a 32 argumentos de tipo `UInt8` que indican si se cumplió una determinada condición para el evento. -Cualquier condición se puede especificar como un argumento (como en [WHERE](../../sql-reference/statements/select.md#select-where)). +Cualquier condición se puede especificar como un argumento (como en [WHERE](../../sql-reference/statements/select/where.md#select-where)). Las condiciones, excepto la primera, se aplican en pares: el resultado del segundo será verdadero si el primero y el segundo son verdaderos, del tercero si el primero y el fird son verdaderos, etc. @@ -335,7 +335,7 @@ retention(cond1, cond2, ..., cond32); La matriz de 1 o 0. - 1 — condition was met for the event. -- 0 — condition wasn’t met for the event. +- 0 — condition wasn't met for the event. Tipo: `UInt8`. diff --git a/docs/es/sql-reference/aggregate-functions/reference.md b/docs/es/sql-reference/aggregate-functions/reference.md index 6d65bc2a643..b7722ecd221 100644 --- a/docs/es/sql-reference/aggregate-functions/reference.md +++ b/docs/es/sql-reference/aggregate-functions/reference.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 36 toc_title: Referencia --- -# Referencia De La función {#function-reference} +# Referencia de función agregada {#aggregate-functions-reference} ## contar {#agg_function-count} @@ -19,7 +19,7 @@ ClickHouse admite las siguientes sintaxis para `count`: La función puede tomar: -- Cero de los parámetros. +- Cero parámetros. - Una [expresion](../syntax.md#syntax-expressions). **Valor devuelto** @@ -83,7 +83,7 @@ En algunos casos, puede confiar en el orden de ejecución. Esto se aplica a los Cuando un `SELECT` consulta tiene el `GROUP BY` cláusula o al menos una función agregada, ClickHouse (en contraste con MySQL) requiere que todas las expresiones `SELECT`, `HAVING`, y `ORDER BY` las cláusulas pueden calcularse a partir de claves o de funciones agregadas. En otras palabras, cada columna seleccionada de la tabla debe usarse en claves o dentro de funciones agregadas. Para obtener un comportamiento como en MySQL, puede colocar las otras columnas en el `any` función de agregado. -## Cualquier Pesado (x) {#anyheavyx} +## Cualquier pesado (x) {#anyheavyx} Selecciona un valor que ocurre con frecuencia [pesos pesados](http://www.cs.umd.edu/~samir/498/karp.pdf) algoritmo. Si hay un valor que se produce más de la mitad de los casos en cada uno de los subprocesos de ejecución de la consulta, se devuelve este valor. Normalmente, el resultado es no determinista. @@ -115,7 +115,7 @@ FROM ontime Selecciona el último valor encontrado. El resultado es tan indeterminado como para el `any` función. -## Método De codificación De Datos: {#groupbitand} +## Método de codificación de datos: {#groupbitand} Se aplica bit a bit `AND` para la serie de números. @@ -244,7 +244,7 @@ binary decimal 01101000 = 104 ``` -## Método De codificación De Datos: {#groupbitmap} +## Método de codificación de datos: {#groupbitmap} Mapa de bits o cálculos agregados de una columna entera sin signo, devuelve cardinalidad de tipo UInt64, si agrega el sufijo -State, luego devuelve [objeto de mapa de bits](../../sql-reference/functions/bitmap-functions.md). @@ -332,9 +332,10 @@ Calcula la suma de los números, utilizando el mismo tipo de datos para el resul Solo funciona para números. -## sumMap(clave, valor) {#agg_functions-summap} +## Por ejemplo, el valor es el siguiente:)) {#agg_functions-summap} Totals el ‘value’ matriz de acuerdo con las claves especificadas en el ‘key’ matriz. +Pasar una tupla de matrices de claves y valores es sinónimo de pasar dos matrices de claves y valores. El número de elementos en ‘key’ y ‘value’ debe ser el mismo para cada fila que se sume. Returns a tuple of two arrays: keys in sorted order, and values ​​summed for the corresponding keys. @@ -347,25 +348,28 @@ CREATE TABLE sum_map( statusMap Nested( status UInt16, requests UInt64 - ) + ), + statusMapTuple Tuple(Array(Int32), Array(Int32)) ) ENGINE = Log; INSERT INTO sum_map VALUES - ('2000-01-01', '2000-01-01 00:00:00', [1, 2, 3], [10, 10, 10]), - ('2000-01-01', '2000-01-01 00:00:00', [3, 4, 5], [10, 10, 10]), - ('2000-01-01', '2000-01-01 00:01:00', [4, 5, 6], [10, 10, 10]), - ('2000-01-01', '2000-01-01 00:01:00', [6, 7, 8], [10, 10, 10]); + ('2000-01-01', '2000-01-01 00:00:00', [1, 2, 3], [10, 10, 10], ([1, 2, 3], [10, 10, 10])), + ('2000-01-01', '2000-01-01 00:00:00', [3, 4, 5], [10, 10, 10], ([3, 4, 5], [10, 10, 10])), + ('2000-01-01', '2000-01-01 00:01:00', [4, 5, 6], [10, 10, 10], ([4, 5, 6], [10, 10, 10])), + ('2000-01-01', '2000-01-01 00:01:00', [6, 7, 8], [10, 10, 10], ([6, 7, 8], [10, 10, 10])); + SELECT timeslot, - sumMap(statusMap.status, statusMap.requests) + sumMap(statusMap.status, statusMap.requests), + sumMap(statusMapTuple) FROM sum_map GROUP BY timeslot ``` ``` text -┌────────────timeslot─┬─sumMap(statusMap.status, statusMap.requests)─┐ -│ 2000-01-01 00:00:00 │ ([1,2,3,4,5],[10,10,20,10,10]) │ -│ 2000-01-01 00:01:00 │ ([4,5,6,7,8],[10,10,20,10,10]) │ -└─────────────────────┴──────────────────────────────────────────────┘ +┌────────────timeslot─┬─sumMap(statusMap.status, statusMap.requests)─┬─sumMap(statusMapTuple)─────────┐ +│ 2000-01-01 00:00:00 │ ([1,2,3,4,5],[10,10,20,10,10]) │ ([1,2,3,4,5],[10,10,20,10,10]) │ +│ 2000-01-01 00:01:00 │ ([4,5,6,7,8],[10,10,20,10,10]) │ ([4,5,6,7,8],[10,10,20,10,10]) │ +└─────────────────────┴──────────────────────────────────────────────┴────────────────────────────────┘ ``` ## SkewPop {#skewpop} @@ -390,7 +394,7 @@ The skewness of the given distribution. Type — [Float64](../../sql-reference/d SELECT skewPop(value) FROM series_with_value_column ``` -## Sistema Abierto {#skewsamp} +## Sistema abierto {#skewsamp} Calcula el [asimetría de la muestra](https://en.wikipedia.org/wiki/Skewness) de una secuencia. @@ -460,7 +464,7 @@ The kurtosis of the given distribution. Type — [Float64](../../sql-reference/d SELECT kurtSamp(value) FROM series_with_value_column ``` -## Para Obtener más información, Consulta Nuestra Política De Privacidad y Nuestras Condiciones De Uso) {#agg-function-timeseriesgroupsum} +## Para obtener más información, consulta nuestra Política de privacidad y nuestras Condiciones de uso) {#agg-function-timeseriesgroupsum} `timeSeriesGroupSum` puede agregar diferentes series de tiempo que muestran la marca de tiempo no la alineación. Utilizará la interpolación lineal entre dos marcas de tiempo de muestra y luego sumará series temporales juntas. @@ -512,23 +516,64 @@ Y el resultado será: [(2,0.2),(3,0.9),(7,2.1),(8,2.4),(12,3.6),(17,5.1),(18,5.4),(24,7.2),(25,2.5)] ``` -## También Puede Utilizar El Siguiente Ejemplo:) {#agg-function-timeseriesgroupratesum} +## También puede utilizar el siguiente ejemplo:) {#agg-function-timeseriesgroupratesum} -Del mismo modo timeSeriesGroupRateSum, timeSeriesGroupRateSum calculará la tasa de series temporales y luego sumará las tasas juntas. +De manera similar a `timeSeriesGroupSum`, `timeSeriesGroupRateSum` calcula la tasa de series temporales y luego suma las tasas juntas. Además, la marca de tiempo debe estar en orden ascendente antes de usar esta función. -Use esta función, el resultado anterior será: +Aplicando esta función a los datos del `timeSeriesGroupSum` ejemplo, se obtiene el siguiente resultado: ``` text [(2,0),(3,0.1),(7,0.3),(8,0.3),(12,0.3),(17,0.3),(18,0.3),(24,0.3),(25,0.1)] ``` -## Acerca De) {#agg_function-avg} +## Acerca de) {#agg_function-avg} Calcula el promedio. Solo funciona para números. El resultado es siempre Float64. +## avgPonderado {#avgweighted} + +Calcula el [media aritmética ponderada](https://en.wikipedia.org/wiki/Weighted_arithmetic_mean). + +**Sintaxis** + +``` sql +avgWeighted(x, weight) +``` + +**Parámetros** + +- `x` — Values. [Entero](../data-types/int-uint.md) o [punto flotante](../data-types/float.md). +- `weight` — Weights of the values. [Entero](../data-types/int-uint.md) o [punto flotante](../data-types/float.md). + +Tipo de `x` y `weight` debe ser el mismo. + +**Valor devuelto** + +- Media ponderada. +- `NaN`. Si todos los pesos son iguales a 0. + +Tipo: [Float64](../data-types/float.md). + +**Ejemplo** + +Consulta: + +``` sql +SELECT avgWeighted(x, w) +FROM values('x Int8, w Int8', (4, 1), (1, 0), (10, 2)) +``` + +Resultado: + +``` text +┌─avgWeighted(x, weight)─┐ +│ 8 │ +└────────────────────────┘ +``` + ## uniq {#agg_function-uniq} Calcula el número aproximado de diferentes valores del argumento. @@ -584,7 +629,7 @@ La función toma un número variable de parámetros. Los parámetros pueden ser **Valor devuelto** -- Un número [UInt64](../../sql-reference/data-types/int-uint.md)-tipo número. +- Numero [UInt64](../../sql-reference/data-types/int-uint.md)-tipo número. **Detalles de implementación** @@ -686,19 +731,93 @@ Por ejemplo, `groupArray (1) (x)` es equivalente a `[any (x)]`. En algunos casos, aún puede confiar en el orden de ejecución. Esto se aplica a los casos en que `SELECT` procede de una subconsulta que utiliza `ORDER BY`. -## Para Obtener más información, Consulte El Siguiente Enlace:) {#grouparrayinsertatvalue-position} +## GrupoArrayInsertAt {#grouparrayinsertat} Inserta un valor en la matriz en la posición especificada. -!!! note "Nota" - Esta función utiliza posiciones de base cero, contrariamente a las posiciones de base única convencionales para matrices SQL. +**Sintaxis** -Accepts the value and position as input. If several values ​​are inserted into the same position, any of them might end up in the resulting array (the first one will be used in the case of single-threaded execution). If no value is inserted into a position, the position is assigned the default value. +``` sql +groupArrayInsertAt(default_x, size)(x, pos); +``` -Parámetros opcionales: +Si en una consulta se insertan varios valores en la misma posición, la función se comporta de las siguientes maneras: -- El valor predeterminado para sustituir en posiciones vacías. -- La longitud de la matriz resultante. Esto le permite recibir matrices del mismo tamaño para todas las claves agregadas. Al utilizar este parámetro, se debe especificar el valor predeterminado. +- Si se ejecuta una consulta en un solo subproceso, se utiliza el primero de los valores insertados. +- Si una consulta se ejecuta en varios subprocesos, el valor resultante es uno indeterminado de los valores insertados. + +**Parámetros** + +- `x` — Value to be inserted. [Expresion](../syntax.md#syntax-expressions) lo que resulta en uno de los [tipos de datos compatibles](../../sql-reference/data-types/index.md). +- `pos` — Position at which the specified element `x` se va a insertar. La numeración de índices en la matriz comienza desde cero. [UInt32](../../sql-reference/data-types/int-uint.md#uint-ranges). +- `default_x`— Default value for substituting in empty positions. Optional parameter. [Expresion](../syntax.md#syntax-expressions) dando como resultado el tipo de datos configurado para `x` parámetro. Si `default_x` no está definido, el [valores predeterminados](../../sql-reference/statements/create.md#create-default-values) se utilizan. +- `size`— Length of the resulting array. Optional parameter. When using this parameter, the default value `default_x` debe ser especificado. [UInt32](../../sql-reference/data-types/int-uint.md#uint-ranges). + +**Valor devuelto** + +- Matriz con valores insertados. + +Tipo: [Matriz](../../sql-reference/data-types/array.md#data-type-array). + +**Ejemplo** + +Consulta: + +``` sql +SELECT groupArrayInsertAt(toString(number), number * 2) FROM numbers(5); +``` + +Resultado: + +``` text +┌─groupArrayInsertAt(toString(number), multiply(number, 2))─┐ +│ ['0','','1','','2','','3','','4'] │ +└───────────────────────────────────────────────────────────┘ +``` + +Consulta: + +``` sql +SELECT groupArrayInsertAt('-')(toString(number), number * 2) FROM numbers(5); +``` + +Resultado: + +``` text +┌─groupArrayInsertAt('-')(toString(number), multiply(number, 2))─┐ +│ ['0','-','1','-','2','-','3','-','4'] │ +└────────────────────────────────────────────────────────────────┘ +``` + +Consulta: + +``` sql +SELECT groupArrayInsertAt('-', 5)(toString(number), number * 2) FROM numbers(5); +``` + +Resultado: + +``` text +┌─groupArrayInsertAt('-', 5)(toString(number), multiply(number, 2))─┐ +│ ['0','-','1','-','2'] │ +└───────────────────────────────────────────────────────────────────┘ +``` + +Inserción multihilo de elementos en una posición. + +Consulta: + +``` sql +SELECT groupArrayInsertAt(number, 0) FROM numbers_mt(10) SETTINGS max_block_size = 1; +``` + +Como resultado de esta consulta, obtiene un entero aleatorio en el `[0,9]` gama. Por ejemplo: + +``` text +┌─groupArrayInsertAt(number, 0)─┐ +│ [7] │ +└───────────────────────────────┘ +``` ## groupArrayMovingSum {#agg_function-grouparraymovingsum} @@ -773,7 +892,7 @@ FROM t └────────────┴─────────────────────────────────┴────────────────────────┘ ``` -## Método De codificación De Datos: {#agg_function-grouparraymovingavg} +## Método de codificación de datos: {#agg_function-grouparraymovingavg} Calcula la media móvil de los valores de entrada. @@ -942,11 +1061,11 @@ Apodo: `medianDeterministic`. **Valor devuelto** -- Aproximado cuantil del nivel especificado. +- Cuantil aproximado del nivel especificado. Tipo: -- [Float64](../../sql-reference/data-types/float.md) para el tipo de datos numérico de entrada. +- [Float64](../../sql-reference/data-types/float.md) para la entrada de tipo de datos numéricos. - [Fecha](../../sql-reference/data-types/date.md) si los valores de entrada tienen `Date` tipo. - [FechaHora](../../sql-reference/data-types/datetime.md) si los valores de entrada tienen `DateTime` tipo. @@ -1009,7 +1128,7 @@ Apodo: `medianExact`. Tipo: -- [Float64](../../sql-reference/data-types/float.md) para el tipo de datos numérico de entrada. +- [Float64](../../sql-reference/data-types/float.md) para la entrada de tipo de datos numéricos. - [Fecha](../../sql-reference/data-types/date.md) si los valores de entrada tienen `Date` tipo. - [FechaHora](../../sql-reference/data-types/datetime.md) si los valores de entrada tienen `DateTime` tipo. @@ -1142,7 +1261,7 @@ De lo contrario, el resultado del cálculo se redondea al múltiplo más cercano Tipo: `Float32`. !!! note "Nota" - Si no se pasan valores a la función (cuando se `quantileTimingIf`), [NaN](../../sql-reference/data-types/float.md#data_type-float-nan-inf) se devuelve. El propósito de esto es diferenciar estos casos de los casos que resultan en cero. Ver [ORDER BY cláusula](../statements/select.md#select-order-by) para notas sobre la clasificación `NaN` valor. + Si no se pasan valores a la función (cuando se `quantileTimingIf`), [NaN](../../sql-reference/data-types/float.md#data_type-float-nan-inf) se devuelve. El propósito de esto es diferenciar estos casos de los casos que resultan en cero. Ver [ORDER BY cláusula](../statements/select/order-by.md#select-order-by) para notas sobre la clasificación `NaN` valor. **Ejemplo** @@ -1227,7 +1346,7 @@ De lo contrario, el resultado del cálculo se redondea al múltiplo más cercano Tipo: `Float32`. !!! note "Nota" - Si no se pasan valores a la función (cuando se `quantileTimingIf`), [NaN](../../sql-reference/data-types/float.md#data_type-float-nan-inf) se devuelve. El propósito de esto es diferenciar estos casos de los casos que resultan en cero. Ver [ORDER BY cláusula](../statements/select.md#select-order-by) para notas sobre la clasificación `NaN` valor. + Si no se pasan valores a la función (cuando se `quantileTimingIf`), [NaN](../../sql-reference/data-types/float.md#data_type-float-nan-inf) se devuelve. El propósito de esto es diferenciar estos casos de los casos que resultan en cero. Ver [ORDER BY cláusula](../statements/select/order-by.md#select-order-by) para notas sobre la clasificación `NaN` valor. **Ejemplo** @@ -1418,7 +1537,7 @@ Resultado: Todas las funciones de cuantiles también tienen funciones de cuantiles correspondientes: `quantiles`, `quantilesDeterministic`, `quantilesTiming`, `quantilesTimingWeighted`, `quantilesExact`, `quantilesExactWeighted`, `quantilesTDigest`. Estas funciones calculan todos los cuantiles de los niveles enumerados en una sola pasada y devuelven una matriz de los valores resultantes. -## Acerca De Nosotros) {#varsampx} +## Acerca de Nosotros) {#varsampx} Calcula la cantidad `Σ((x - x̅)^2) / (n - 1)`, donde `n` es el tamaño de la muestra y `x̅`es el valor promedio de `x`. @@ -1426,20 +1545,32 @@ Representa una estimación imparcial de la varianza de una variable aleatoria si Devoluciones `Float64`. Cuando `n <= 1`, devoluciones `+∞`. -## Nombre De La Red inalámbrica (SSID):) {#varpopx} +!!! note "Nota" + Esta función utiliza un algoritmo numéricamente inestable. Si necesita [estabilidad numérica](https://en.wikipedia.org/wiki/Numerical_stability) en los cálculos, utilice el `varSampStable` función. Funciona más lento, pero proporciona un menor error computacional. + +## Nombre de la red inalámbrica (SSID):) {#varpopx} Calcula la cantidad `Σ((x - x̅)^2) / n`, donde `n` es el tamaño de la muestra y `x̅`es el valor promedio de `x`. En otras palabras, dispersión para un conjunto de valores. Devoluciones `Float64`. +!!! note "Nota" + Esta función utiliza un algoritmo numéricamente inestable. Si necesita [estabilidad numérica](https://en.wikipedia.org/wiki/Numerical_stability) en los cálculos, utilice el `varPopStable` función. Funciona más lento, pero proporciona un menor error computacional. + ## Soporte técnico) {#stddevsampx} El resultado es igual a la raíz cuadrada de `varSamp(x)`. +!!! note "Nota" + Esta función utiliza un algoritmo numéricamente inestable. Si necesita [estabilidad numérica](https://en.wikipedia.org/wiki/Numerical_stability) en los cálculos, utilice el `stddevSampStable` función. Funciona más lento, pero proporciona un menor error computacional. + ## stddevPop(x) {#stddevpopx} El resultado es igual a la raíz cuadrada de `varPop(x)`. +!!! note "Nota" + Esta función utiliza un algoritmo numéricamente inestable. Si necesita [estabilidad numérica](https://en.wikipedia.org/wiki/Numerical_stability) en los cálculos, utilice el `stddevPopStable` función. Funciona más lento, pero proporciona un menor error computacional. + ## topK(N)(x) {#topknx} Devuelve una matriz de los valores aproximadamente más frecuentes de la columna especificada. La matriz resultante se ordena en orden descendente de frecuencia aproximada de valores (no por los valores mismos). @@ -1462,7 +1593,7 @@ Si se omite el parámetro, se utiliza el valor predeterminado 10. **Argumento** -- ’ x ’ – The value to calculate frequency. +- ' x ' – The value to calculate frequency. **Ejemplo** @@ -1524,14 +1655,23 @@ Calcula el valor de `Σ((x - x̅)(y - y̅)) / (n - 1)`. Devuelve Float64. Cuando `n <= 1`, returns +∞. +!!! note "Nota" + Esta función utiliza un algoritmo numéricamente inestable. Si necesita [estabilidad numérica](https://en.wikipedia.org/wiki/Numerical_stability) en los cálculos, utilice el `covarSampStable` función. Funciona más lento, pero proporciona un menor error computacional. + ## covarPop(x, y) {#covarpopx-y} Calcula el valor de `Σ((x - x̅)(y - y̅)) / n`. +!!! note "Nota" + Esta función utiliza un algoritmo numéricamente inestable. Si necesita [estabilidad numérica](https://en.wikipedia.org/wiki/Numerical_stability) en los cálculos, utilice el `covarPopStable` función. Funciona más lento pero proporciona un menor error computacional. + ## corr(x, y) {#corrx-y} Calcula el coeficiente de correlación de Pearson: `Σ((x - x̅)(y - y̅)) / sqrt(Σ((x - x̅)^2) * Σ((y - y̅)^2))`. +!!! note "Nota" + Esta función utiliza un algoritmo numéricamente inestable. Si necesita [estabilidad numérica](https://en.wikipedia.org/wiki/Numerical_stability) en los cálculos, utilice el `corrStable` función. Funciona más lento, pero proporciona un menor error computacional. + ## categoricalInformationValue {#categoricalinformationvalue} Calcula el valor de `(P(tag = 1) - P(tag = 0))(log(P(tag = 1)) - log(P(tag = 0)))` para cada categoría. @@ -1705,7 +1845,7 @@ stochasticLogisticRegression(1.0, 1.0, 10, 'SGD') - [stochasticLinearRegression](#agg_functions-stochasticlinearregression) - [Diferencia entre regresiones lineales y logísticas.](https://stackoverflow.com/questions/12146914/what-is-the-difference-between-linear-regression-and-logistic-regression) -## Método De codificación De Datos: {#groupbitmapand} +## Método de codificación de datos: {#groupbitmapand} Calcula el AND de una columna de mapa de bits, devuelve la cardinalidad del tipo UInt64, si agrega el sufijo -State, luego devuelve [objeto de mapa de bits](../../sql-reference/functions/bitmap-functions.md). @@ -1748,7 +1888,7 @@ SELECT arraySort(bitmapToArray(groupBitmapAndState(z))) FROM bitmap_column_expr_ └──────────────────────────────────────────────────┘ ``` -## Método De codificación De Datos: {#groupbitmapor} +## Método de codificación de datos: {#groupbitmapor} Calcula el OR de una columna de mapa de bits, devuelve la cardinalidad del tipo UInt64, si agrega el sufijo -State, luego devuelve [objeto de mapa de bits](../../sql-reference/functions/bitmap-functions.md). Esto es equivalente a `groupBitmapMerge`. @@ -1791,7 +1931,7 @@ SELECT arraySort(bitmapToArray(groupBitmapOrState(z))) FROM bitmap_column_expr_t └─────────────────────────────────────────────────┘ ``` -## Método De codificación De Datos: {#groupbitmapxor} +## Método de codificación de datos: {#groupbitmapxor} Calcula el XOR de una columna de mapa de bits, devuelve la cardinalidad del tipo UInt64, si agrega el sufijo -State, luego devuelve [objeto de mapa de bits](../../sql-reference/functions/bitmap-functions.md). diff --git a/docs/es/sql-reference/ansi.md b/docs/es/sql-reference/ansi.md deleted file mode 120000 index 3cf6bffed67..00000000000 --- a/docs/es/sql-reference/ansi.md +++ /dev/null @@ -1 +0,0 @@ -../../en/sql-reference/ansi.md \ No newline at end of file diff --git a/docs/es/sql-reference/ansi.md b/docs/es/sql-reference/ansi.md new file mode 100644 index 00000000000..3bf06d4419b --- /dev/null +++ b/docs/es/sql-reference/ansi.md @@ -0,0 +1,180 @@ +--- +machine_translated: true +machine_translated_rev: ad252bbb4f7e2899c448eb42ecc39ff195c8faa1 +toc_priority: 40 +toc_title: Compatibilidad con ANSI +--- + +# Compatibilidad de SQL ANSI de ClickHouse SQL Dialect {#ansi-sql-compatibility-of-clickhouse-sql-dialect} + +!!! note "Nota" + Este artículo se basa en la Tabla 38, “Feature taxonomy and definition for mandatory features”, Annex F of ISO/IEC CD 9075-2:2013. + +## Diferencias en el comportamiento {#differences-in-behaviour} + +En la tabla siguiente se enumeran los casos en que la característica de consulta funciona en ClickHouse, pero no se comporta como se especifica en ANSI SQL. + +| Feature ID | Nombre de la función | Diferencia | +|------------|----------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------| +| E011 | Tipos de datos numéricos | El literal numérico con punto se interpreta como aproximado (`Float64`) en lugar de exacta (`Decimal`) | +| E051-05 | Los elementos seleccionados pueden ser renombrados | Los cambios de nombre de los elementos tienen un alcance de visibilidad más amplio que solo el resultado SELECT | +| E141-01 | Restricciones NOT NULL | `NOT NULL` está implícito para las columnas de tabla de forma predeterminada | +| E011-04 | Operadores aritméticos | ClickHouse se desborda en lugar de la aritmética comprobada y cambia el tipo de datos de resultado en función de las reglas personalizadas | + +## Estado de la función {#feature-status} + +| Feature ID | Nombre de la función | Estatus | Comentario | +|------------|---------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **E011** | **Tipos de datos numéricos** | **Parcial**{.text-warning} | | +| E011-01 | Tipos de datos INTEGER y SMALLINT | Sí{.text-success} | | +| E011-02 | REAL, DOUBLE PRECISION y FLOAT tipos de datos tipos de datos | Parcial{.text-warning} | `FLOAT()`, `REAL` y `DOUBLE PRECISION` no son compatibles | +| E011-03 | Tipos de datos DECIMAL y NUMERIC | Parcial{.text-warning} | Solo `DECIMAL(p,s)` es compatible, no `NUMERIC` | +| E011-04 | Operadores aritméticos | Sí{.text-success} | | +| E011-05 | Comparación numérica | Sí{.text-success} | | +| E011-06 | Conversión implícita entre los tipos de datos numéricos | No{.text-danger} | ANSI SQL permite la conversión implícita arbitraria entre tipos numéricos, mientras que ClickHouse se basa en funciones que tienen múltiples sobrecargas en lugar de conversión implícita | +| **E021** | **Tipos de cadena de caracteres** | **Parcial**{.text-warning} | | +| E021-01 | Tipo de datos CHARACTER | No{.text-danger} | | +| E021-02 | Tipo de datos CHARACTER VARYING | No{.text-danger} | `String` se comporta de manera similar, pero sin límite de longitud entre paréntesis | +| E021-03 | Literales de caracteres | Parcial{.text-warning} | Sin concatenación automática de literales consecutivos y compatibilidad con el conjunto de caracteres | +| E021-04 | Función CHARACTER\_LENGTH | Parcial{.text-warning} | No `USING` clausula | +| E021-05 | Función OCTET\_LENGTH | No{.text-danger} | `LENGTH` se comporta de manera similar | +| E021-06 | SUBSTRING | Parcial{.text-warning} | No hay soporte para `SIMILAR` y `ESCAPE` cláusulas, no `SUBSTRING_REGEX` variante | +| E021-07 | Concatenación de caracteres | Parcial{.text-warning} | No `COLLATE` clausula | +| E021-08 | Funciones SUPERIOR e INFERIOR | Sí{.text-success} | | +| E021-09 | Función TRIM | Sí{.text-success} | | +| E021-10 | Conversión implícita entre los tipos de cadena de caracteres de longitud fija y longitud variable | No{.text-danger} | ANSI SQL permite la conversión implícita arbitraria entre tipos de cadena, mientras que ClickHouse se basa en funciones que tienen múltiples sobrecargas en lugar de conversión implícita | +| E021-11 | Función POSITION | Parcial{.text-warning} | No hay soporte para `IN` y `USING` cláusulas, no `POSITION_REGEX` variante | +| E021-12 | Comparación de caracteres | Sí{.text-success} | | +| **E031** | **Identificador** | **Parcial**{.text-warning} | | +| E031-01 | Identificadores delimitados | Parcial{.text-warning} | El soporte literal Unicode es limitado | +| E031-02 | Identificadores de minúsculas | Sí{.text-success} | | +| E031-03 | Trailing subrayado | Sí{.text-success} | | +| **E051** | **Especificación básica de la consulta** | **Parcial**{.text-warning} | | +| E051-01 | SELECT DISTINCT | Sí{.text-success} | | +| E051-02 | Cláusula GROUP BY | Sí{.text-success} | | +| E051-04 | GROUP BY puede contener columnas que no estén en `` | بله{.text-success} | | +| E051-05 | انتخاب موارد را می توان تغییر نام داد | بله{.text-success} | | +| E051-06 | داشتن بند | بله{.text-success} | | +| E051-07 | واجد شرایط \* در انتخاب لیست | بله{.text-success} | | +| E051-08 | نام همبستگی در بند | بله{.text-success} | | +| E051-09 | تغییر نام ستون ها در بند | نه{.text-danger} | | +| **E061** | **مخمصه عمومی و شرایط جستجو** | **نسبی**{.text-warning} | | +| E061-01 | پیش فرض مقایسه | بله{.text-success} | | +| E061-02 | بین پیش فرض | نسبی{.text-warning} | نه `SYMMETRIC` و `ASYMMETRIC` بند | +| E061-03 | در گزاره با لیستی از ارزش ها | بله{.text-success} | | +| E061-04 | مثل گزاره | بله{.text-success} | | +| E061-05 | مانند گزاره: فرار بند | نه{.text-danger} | | +| E061-06 | پیش فرض پوچ | بله{.text-success} | | +| E061-07 | گزاره مقایسه کمی | نه{.text-danger} | | +| E061-08 | پیش فرض وجود دارد | نه{.text-danger} | | +| E061-09 | Subqueries در مقایسه گزاره | بله{.text-success} | | +| E061-11 | در حال بارگذاری | بله{.text-success} | | +| E061-12 | زیرمجموعه ها در پیش بینی مقایسه اندازه گیری شده | نه{.text-danger} | | +| E061-13 | ارتباط subqueries | نه{.text-danger} | | +| E061-14 | وضعیت جستجو | بله{.text-success} | | +| **E071** | **عبارتهای پرسوجو پایه** | **نسبی**{.text-warning} | | +| E071-01 | اتحادیه اپراتور جدول مجزا | نه{.text-danger} | | +| E071-02 | اتحادیه تمام اپراتور جدول | بله{.text-success} | | +| E071-03 | به جز اپراتور جدول مجزا | نه{.text-danger} | | +| E071-05 | ستون ترکیب از طریق اپراتورهای جدول نیاز دقیقا همان نوع داده ندارد | بله{.text-success} | | +| E071-06 | اپراتورهای جدول در زیرمجموعه | بله{.text-success} | | +| **E081** | **امتیازات پایه** | **نسبی**{.text-warning} | کار در حال پیشرفت | +| **E091** | **تنظیم توابع** | **بله**{.text-success} | | +| E091-01 | AVG | بله{.text-success} | | +| E091-02 | COUNT | بله{.text-success} | | +| E091-03 | MAX | بله{.text-success} | | +| E091-04 | MIN | بله{.text-success} | | +| E091-05 | SUM | بله{.text-success} | | +| E091-06 | همه کمی | نه{.text-danger} | | +| E091-07 | کمی متمایز | نسبی{.text-warning} | همه توابع مجموع پشتیبانی | +| **E101** | **دستکاری داده های پایه** | **نسبی**{.text-warning} | | +| E101-01 | درج بیانیه | بله{.text-success} | توجه داشته باشید: کلید اصلی در خانه کلیک می کند به این معنی نیست `UNIQUE` محدودیت | +| E101-03 | بیانیه به روز رسانی جستجو | نه{.text-danger} | یک `ALTER UPDATE` بیانیه ای برای اصلاح داده های دسته ای | +| E101-04 | جستجو حذف بیانیه | نه{.text-danger} | یک `ALTER DELETE` بیانیه ای برای حذف داده های دسته ای | +| **E111** | **تک ردیف انتخاب بیانیه** | **نه**{.text-danger} | | +| **E121** | **پشتیبانی عمومی مکان نما** | **نه**{.text-danger} | | +| E121-01 | DECLARE CURSOR | نه{.text-danger} | | +| E121-02 | سفارش ستون ها در لیست انتخاب نمی شود | نه{.text-danger} | | +| E121-03 | عبارات ارزش به ترتیب توسط بند | نه{.text-danger} | | +| E121-04 | بیانیه باز | نه{.text-danger} | | +| E121-06 | بیانیه به روز رسانی موقعیت | نه{.text-danger} | | +| E121-07 | موقعیت حذف بیانیه | نه{.text-danger} | | +| E121-08 | بستن بیانیه | نه{.text-danger} | | +| E121-10 | واکشی بیانیه: ضمنی بعدی | نه{.text-danger} | | +| E121-17 | با نشانگر نگه دارید | نه{.text-danger} | | +| **E131** | **پشتیبانی ارزش صفر (صفر به جای ارزش)** | **نسبی**{.text-warning} | برخی از محدودیت ها اعمال می شود | +| **E141** | **محدودیت یکپارچگی عمومی** | **نسبی**{.text-warning} | | +| E141-01 | محدودیت NOT NULL | بله{.text-success} | یادداشت: `NOT NULL` برای ستون های جدول به طور پیش فرض ضمنی | +| E141-02 | محدودیت منحصر به فرد از ستون تهی نیست | نه{.text-danger} | | +| E141-03 | محدودیت های کلیدی اولیه | نه{.text-danger} | | +| E141-04 | محدودیت کلید خارجی عمومی با هیچ پیش فرض اقدام برای هر دو عمل حذف ارجاعی و عمل به روز رسانی ارجاعی | نه{.text-danger} | | +| E141-06 | بررسی محدودیت | بله{.text-success} | | +| E141-07 | پیشفرض ستون | بله{.text-success} | | +| E141-08 | تهی نیست استنباط در کلید اولیه | بله{.text-success} | | +| E141-10 | نام در یک کلید خارجی را می توان در هر سفارش مشخص شده است | نه{.text-danger} | | +| **E151** | **پشتیبانی تراکنش** | **نه**{.text-danger} | | +| E151-01 | بیانیه متعهد | نه{.text-danger} | | +| E151-02 | بیانیه عقبگرد | نه{.text-danger} | | +| **E152** | **بیانیه معامله عمومی مجموعه** | **نه**{.text-danger} | | +| E152-01 | مجموعه بیانیه معامله: جداسازی سطح SERIALIZABLE بند | نه{.text-danger} | | +| E152-02 | تنظیم بیانیه معامله: فقط خواندن و خواندن نوشتن جملات | نه{.text-danger} | | +| **E153** | **نمایش داده شد بهروز با زیرمجموعه** | **نه**{.text-danger} | | +| **E161** | **گذاشتن نظرات با استفاده از منجر منهای دو** | **بله**{.text-success} | | +| **E171** | **SQLSTATE پشتیبانی** | **نه**{.text-danger} | | +| **E182** | **اتصال زبان میزبان** | **نه**{.text-danger} | | +| **F031** | **دستکاری طرح اولیه** | **نسبی**{.text-warning} | | +| F031-01 | ایجاد بیانیه جدول برای ایجاد جداول پایه مداوم | نسبی{.text-warning} | نه `SYSTEM VERSIONING`, `ON COMMIT`, `GLOBAL`, `LOCAL`, `PRESERVE`, `DELETE`, `REF IS`, `WITH OPTIONS`, `UNDER`, `LIKE`, `PERIOD FOR` بند و هیچ پشتیبانی برای کاربر حل و فصل انواع داده ها | +| F031-02 | ایجاد نمایش بیانیه | نسبی{.text-warning} | نه `RECURSIVE`, `CHECK`, `UNDER`, `WITH OPTIONS` بند و هیچ پشتیبانی برای کاربر حل و فصل انواع داده ها | +| F031-03 | بیانیه گرانت | بله{.text-success} | | +| F031-04 | تغییر بیانیه جدول: اضافه کردن بند ستون | نسبی{.text-warning} | هیچ پشتیبانی برای `GENERATED` بند و مدت زمان سیستم | +| F031-13 | بیانیه جدول قطره: محدود کردن بند | نه{.text-danger} | | +| F031-16 | قطره مشاهده بیانیه: محدود بند | نه{.text-danger} | | +| F031-19 | لغو بیانیه: محدود کردن بند | نه{.text-danger} | | +| **F041** | **جدول پیوست عمومی** | **نسبی**{.text-warning} | | +| F041-01 | عضویت داخلی (اما نه لزوما کلمه کلیدی درونی) | بله{.text-success} | | +| F041-02 | کلیدواژه داخلی | بله{.text-success} | | +| F041-03 | LEFT OUTER JOIN | بله{.text-success} | | +| F041-04 | RIGHT OUTER JOIN | بله{.text-success} | | +| F041-05 | بیرونی می پیوندد می توان تو در تو | بله{.text-success} | | +| F041-07 | جدول درونی در بیرونی چپ یا راست عضویت نیز می تواند مورد استفاده قرار گیرد در عضویت درونی | بله{.text-success} | | +| F041-08 | همه اپراتورهای مقایسه پشتیبانی می شوند (و نه فقط =) | نه{.text-danger} | | +| **F051** | **تاریخ پایه و زمان** | **نسبی**{.text-warning} | | +| F051-01 | تاریخ نوع داده (از جمله پشتیبانی از تاریخ تحت اللفظی) | نسبی{.text-warning} | بدون تحت اللفظی | +| F051-02 | نوع داده زمان (از جمله پشتیبانی از زمان تحت اللفظی) با دقت ثانیه کسری حداقل 0 | نه{.text-danger} | | +| F051-03 | نوع داده برچسب زمان (از جمله پشتیبانی از تحت اللفظی برچسب زمان) با دقت ثانیه کسری از حداقل 0 و 6 | نه{.text-danger} | `DateTime64` زمان فراهم می کند قابلیت های مشابه | +| F051-04 | مقایسه گزاره در تاریخ, زمان, و انواع داده های برچسب زمان | نسبی{.text-warning} | فقط یک نوع داده موجود است | +| F051-05 | بازیگران صریح و روشن بین انواع تاریخ ساعت و انواع رشته شخصیت | بله{.text-success} | | +| F051-06 | CURRENT\_DATE | نه{.text-danger} | `today()` مشابه است | +| F051-07 | LOCALTIME | نه{.text-danger} | `now()` مشابه است | +| F051-08 | LOCALTIMESTAMP | نه{.text-danger} | | +| **F081** | **اتحادیه و به جز در دیدگاه** | **نسبی**{.text-warning} | | +| **F131** | **عملیات گروه بندی شده** | **نسبی**{.text-warning} | | +| F131-01 | جایی که, گروه های, و داشتن بند در نمایش داده شد با نمایش گروه بندی پشتیبانی | بله{.text-success} | | +| F131-02 | جداول چندگانه در نمایش داده شد با نمایش گروه بندی پشتیبانی می شود | بله{.text-success} | | +| F131-03 | تنظیم توابع پشتیبانی شده در نمایش داده شد با نمایش گروه بندی می شوند | بله{.text-success} | | +| F131-04 | Subqueries با گروه و داشتن بند و گروه بندی views | بله{.text-success} | | +| F131-05 | تک ردیف با گروه و داشتن بند و دیدگاه های گروه بندی شده را انتخاب کنید | نه{.text-danger} | | +| **F181** | **پشتیبانی از ماژول های متعدد** | **نه**{.text-danger} | | +| **F201** | **تابع بازیگران** | **بله**{.text-success} | | +| **F221** | **پیش فرض های صریح** | **نه**{.text-danger} | | +| **F261** | **عبارت مورد** | **بله**{.text-success} | | +| F261-01 | مورد ساده | بله{.text-success} | | +| F261-02 | مورد جستجو | بله{.text-success} | | +| F261-03 | NULLIF | بله{.text-success} | | +| F261-04 | COALESCE | بله{.text-success} | | +| **F311** | **بیانیه تعریف طرح** | **نسبی**{.text-warning} | | +| F311-01 | CREATE SCHEMA | نه{.text-danger} | | +| F311-02 | ایجاد جدول برای جداول پایه مداوم | بله{.text-success} | | +| F311-03 | CREATE VIEW | بله{.text-success} | | +| F311-04 | CREATE VIEW: WITH CHECK OPTION | نه{.text-danger} | | +| F311-05 | بیانیه گرانت | بله{.text-success} | | +| **F471** | **مقادیر زیر مقیاس** | **بله**{.text-success} | | +| **F481** | **پیش فرض صفر گسترش یافته است** | **بله**{.text-success} | | +| **F812** | **عمومی ضعیف** | **نه**{.text-danger} | | +| **T321** | **روال عمومی گذاشتن استناد** | **نه**{.text-danger} | | +| T321-01 | توابع تعریف شده توسط کاربر بدون اضافه بار | نه{.text-danger} | | +| T321-02 | روش های ذخیره شده تعریف شده توسط کاربر بدون اضافه بار | نه{.text-danger} | | +| T321-03 | فراخوانی تابع | نه{.text-danger} | | +| T321-04 | بیانیه تماس | نه{.text-danger} | | +| T321-05 | بیانیه بازگشت | نه{.text-danger} | | +| **T631** | **در گزاره با یک عنصر لیست** | **بله**{.text-success} | | diff --git a/docs/fa/sql-reference/data-types/aggregatefunction.md b/docs/fa/sql-reference/data-types/aggregatefunction.md index dd36147b8b7..34a3634cd6e 100644 --- a/docs/fa/sql-reference/data-types/aggregatefunction.md +++ b/docs/fa/sql-reference/data-types/aggregatefunction.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 52 toc_title: "\u06A9\u0627\u0631\u06A9\u0631\u062F(\u0646\u0627\u0645 \u0648 \u0646\u0627\ \u0645 \u062E\u0627\u0646\u0648\u0627\u062F\u06AF\u06CC..)" @@ -8,7 +8,7 @@ toc_title: "\u06A9\u0627\u0631\u06A9\u0631\u062F(\u0646\u0627\u0645 \u0648 \u064 # AggregateFunction(name, types\_of\_arguments…) {#data-type-aggregatefunction} -Aggregate functions can have an implementation-defined intermediate state that can be serialized to an AggregateFunction(…) data type and stored in a table, usually, by means of [مشاهده محقق](../../sql-reference/statements/select.md#create-view). راه معمول برای تولید یک دولت تابع جمع است با فراخوانی تابع جمع با `-State` پسوند. برای دریافت نتیجه نهایی از تجمع در اینده, شما باید همان تابع کل با استفاده از `-Merge`پسوند. +Aggregate functions can have an implementation-defined intermediate state that can be serialized to an AggregateFunction(…) data type and stored in a table, usually, by means of [مشاهده محقق](../../sql-reference/statements/create.md#create-view). راه معمول برای تولید یک دولت تابع جمع است با فراخوانی تابع جمع با `-State` پسوند. برای دریافت نتیجه نهایی از تجمع در اینده, شما باید همان تابع کل با استفاده از `-Merge`پسوند. `AggregateFunction` — parametric data type. @@ -52,7 +52,7 @@ quantilesState(0.5, 0.9)(SendTiming) ### گزینش داده {#data-selection} -هنگام انتخاب داده ها از `AggregatingMergeTree` جدول استفاده کنید `GROUP BY` بند و همان مجموع توابع به عنوان زمانی که قرار دادن داده اما با استفاده از `-Merge`پسوند. +هنگام انتخاب داده ها از `AggregatingMergeTree` جدول استفاده کنید `GROUP BY` بند و توابع کل همان هنگام قرار دادن داده, اما با استفاده از `-Merge`پسوند. یک تابع جمع با `-Merge` پسوند مجموعه ای از ایالت ها را ترکیب می کند و نتیجه تجمع کامل داده ها را باز می گرداند. diff --git a/docs/fa/sql-reference/data-types/array.md b/docs/fa/sql-reference/data-types/array.md index 6786a661904..218b0587e0a 100644 --- a/docs/fa/sql-reference/data-types/array.md +++ b/docs/fa/sql-reference/data-types/array.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 51 toc_title: "& \u062A\u0648\u0631\u06CC)" --- @@ -63,7 +63,7 @@ SELECT array(1, 2, NULL) AS x, toTypeName(x) └────────────┴───────────────────────────────┘ ``` -اگر شما سعی می کنید برای ایجاد یک آرایه از ناسازگار انواع داده ها clickhouse پرتاب یک استثنا: +اگر شما سعی می کنید برای ایجاد مجموعه ای از انواع داده های ناسازگار, تاتر می اندازد یک استثنا: ``` sql SELECT array(1, 'a') diff --git a/docs/fa/sql-reference/data-types/boolean.md b/docs/fa/sql-reference/data-types/boolean.md index a906de86943..d5136604738 100644 --- a/docs/fa/sql-reference/data-types/boolean.md +++ b/docs/fa/sql-reference/data-types/boolean.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 43 toc_title: "\u0628\u0648\u0644\u06CC" --- diff --git a/docs/fa/sql-reference/data-types/date.md b/docs/fa/sql-reference/data-types/date.md index 2a9de86cfb5..401bc1a2e21 100644 --- a/docs/fa/sql-reference/data-types/date.md +++ b/docs/fa/sql-reference/data-types/date.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 47 toc_title: "\u062A\u0627\u0631\u06CC\u062E" --- diff --git a/docs/fa/sql-reference/data-types/datetime.md b/docs/fa/sql-reference/data-types/datetime.md index 708e5b091c0..9de82d07420 100644 --- a/docs/fa/sql-reference/data-types/datetime.md +++ b/docs/fa/sql-reference/data-types/datetime.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 48 toc_title: DateTime --- @@ -19,7 +19,7 @@ DateTime([timezone]) حل: 1 ثانیه. -## استفاده از سخنان {#usage-remarks} +## اظهارات طریقه استفاده {#usage-remarks} نقطه در زمان به عنوان یک ذخیره می شود [برچسب زمان یونیکس](https://en.wikipedia.org/wiki/Unix_time), صرف نظر از منطقه زمانی و یا صرفه جویی در زمان نور روز. علاوه بر این `DateTime` نوع می توانید منطقه زمانی است که همین کار را برای کل ستون ذخیره, که تحت تاثیر قرار چگونه ارزش های `DateTime` مقادیر نوع در قالب متن نمایش داده می شود و چگونه مقادیر مشخص شده به عنوان رشته تجزیه می شوند (‘2020-01-01 05:00:01’). منطقه زمانی در ردیف جدول ذخیره نمی شود (و یا در نتیجه), اما در ابرداده ستون ذخیره می شود. لیستی از مناطق زمانی پشتیبانی شده را می توان در [اانا پایگاه منطقه زمانی](https://www.iana.org/time-zones). @@ -120,10 +120,10 @@ FROM dt - [توابع تبدیل نوع](../../sql-reference/functions/type-conversion-functions.md) - [توابع برای کار با تاریخ و زمان](../../sql-reference/functions/date-time-functions.md) -- [توابع کار با آرایه ها](../../sql-reference/functions/array-functions.md) +- [توابع برای کار با ارریس](../../sql-reference/functions/array-functions.md) - [این `date_time_input_format` تنظیم](../../operations/settings/settings.md#settings-date_time_input_format) - [این `timezone` پارامتر پیکربندی سرور](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) -- [اپراتورها برای کار با تاریخ و زمان](../../sql-reference/operators.md#operators-datetime) +- [اپراتورها برای کار با تاریخ و زمان](../../sql-reference/operators/index.md#operators-datetime) - [این `Date` نوع داده](date.md) [مقاله اصلی](https://clickhouse.tech/docs/en/data_types/datetime/) diff --git a/docs/fa/sql-reference/data-types/datetime64.md b/docs/fa/sql-reference/data-types/datetime64.md index 3c7bc236357..f168ef9444b 100644 --- a/docs/fa/sql-reference/data-types/datetime64.md +++ b/docs/fa/sql-reference/data-types/datetime64.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 49 toc_title: "\u0637\u0648\u0644 \u062A\u0627\u0631\u06CC\u062E 64" --- @@ -92,13 +92,13 @@ FROM dt └─────────────────────────┴─────────────────────────┘ ``` -## همچنین نگاه کنید {#see-also} +## همچنین نگاه کنید به {#see-also} - [توابع تبدیل نوع](../../sql-reference/functions/type-conversion-functions.md) - [توابع برای کار با تاریخ و زمان](../../sql-reference/functions/date-time-functions.md) - [توابع برای کار با ارریس](../../sql-reference/functions/array-functions.md) - [این `date_time_input_format` تنظیم](../../operations/settings/settings.md#settings-date_time_input_format) - [این `timezone` پارامتر پیکربندی سرور](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) -- [اپراتورها برای کار با تاریخ و زمان](../../sql-reference/operators.md#operators-datetime) +- [اپراتورها برای کار با تاریخ و زمان](../../sql-reference/operators/index.md#operators-datetime) - [`Date` نوع داده](date.md) - [`DateTime` نوع داده](datetime.md) diff --git a/docs/fa/sql-reference/data-types/decimal.md b/docs/fa/sql-reference/data-types/decimal.md index cd20f2ae1ec..722734ec6af 100644 --- a/docs/fa/sql-reference/data-types/decimal.md +++ b/docs/fa/sql-reference/data-types/decimal.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 42 toc_title: "\u062F\u0647\u062F\u0647\u06CC" --- @@ -25,13 +25,13 @@ toc_title: "\u062F\u0647\u062F\u0647\u06CC" - اعشار64 (بازدید کنندگان) - ( -1 \* 10^(18 - س), 1 \* 10^(18 بازدید کنندگان) ) - اعشار128 (بازدید کنندگان) - ( -1 \* 10^(38 - بازدید کنندگان), 1 \* 10^(38 بازدید کنندگان) ) -برای مثال decimal32(4) می تواند شامل اعداد از -99999.9999 به 99999.9999 با 0.0001 گام. +برای مثال Decimal32(4) می تواند شامل اعداد از -99999.9999 به 99999.9999 با 0.0001 گام. ## نمایندگی داخلی {#internal-representation} داخلی داده ها به عنوان اعداد صحیح امضا نرمال با عرض بیت مربوطه نشان داده است. محدوده ارزش واقعی است که می تواند در حافظه ذخیره می شود کمی بزرگتر از بالا مشخص, که تنها در تبدیل از یک رشته بررسی. -چرا که پردازنده مدرن اعداد صحیح 128 بیتی بومی را پشتیبانی نمی کند, عملیات بر روی اعشار128 شبیه سازی. از آنجا که این decimal128 با این نسخهها کار به طور قابل توجهی کندتر از decimal32/decimal64. +چرا که پردازنده مدرن اعداد صحیح 128 بیتی بومی را پشتیبانی نمی کند, عملیات بر روی اعشار128 شبیه سازی. از آنجا که این Decimal128 با این نسخهها کار به طور قابل توجهی کندتر از Decimal32/Decimal64. ## عملیات و نوع نتیجه {#operations-and-result-type} @@ -45,11 +45,11 @@ toc_title: "\u062F\u0647\u062F\u0647\u06CC" - اضافه کردن به, تفریق کردن: بازدید کنندگان = حداکثر(بازدید کنندگان 1, بازدید کنندگان2). - multuply: S = S1 + S2. -- تقسیم: S = S1. +- شکاف: بازدید کنندگان = س1. برای عملیات مشابه بین دهدهی و اعداد صحیح, نتیجه اعشاری از همان اندازه به عنوان یک استدلال است. -عملیات بین دهدهی و float32/float64 تعریف نشده. اگر شما به آنها نیاز دارید, شما می توانید به صراحت بازیگران یکی از استدلال با استفاده از todecimal32, todecimal64, todecimal128 یا tofloat32, tofloat64 برنامهنویسی. به خاطر داشته باشید که نتیجه دقت از دست دادن و تبدیل نوع یک عملیات محاسباتی گران است. +عملیات بین دهدهی و Float32/Float64 تعریف نشده. اگر شما به آنها نیاز دارید, شما می توانید به صراحت بازیگران یکی از استدلال با استفاده از toDecimal32, toDecimal64, toDecimal128 یا toFloat32, toFloat64 برنامهنویسی. به خاطر داشته باشید که نتیجه دقت از دست دادن و تبدیل نوع یک عملیات محاسباتی گران است. برخی از توابع در نتیجه بازگشت اعشاری به عنوان شناور64 (مثلا, ور یا استودف). محاسبات متوسط هنوز هم ممکن است در دهدهی انجام شود, که ممکن است به نتایج مختلف بین نت64 و ورودی اعشاری با ارزش های مشابه منجر شود. diff --git a/docs/fa/sql-reference/data-types/domains/index.md b/docs/fa/sql-reference/data-types/domains/index.md index 3a743f84290..089e1c43eed 100644 --- a/docs/fa/sql-reference/data-types/domains/index.md +++ b/docs/fa/sql-reference/data-types/domains/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_folder_title: Domains +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "\u062F\u0627\u0645\u0646\u0647" toc_priority: 56 --- diff --git a/docs/fa/sql-reference/data-types/domains/ipv4.md b/docs/fa/sql-reference/data-types/domains/ipv4.md index cd33df6acc9..645e839f6d8 100644 --- a/docs/fa/sql-reference/data-types/domains/ipv4.md +++ b/docs/fa/sql-reference/data-types/domains/ipv4.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 59 toc_title: IPv4 --- @@ -69,7 +69,7 @@ SELECT toTypeName(s), IPv4NumToString(from) as s FROM hits LIMIT 1; │ String │ 183.247.232.58 │ └───────────────────────────────────┴────────────────┘ -یا بازیگران به `UInt32` مقدار: +یا بازیگران به یک `UInt32` مقدار: ``` sql SELECT toTypeName(i), CAST(from as UInt32) as i FROM hits LIMIT 1; diff --git a/docs/fa/sql-reference/data-types/domains/ipv6.md b/docs/fa/sql-reference/data-types/domains/ipv6.md index 8c2779777de..6677916c49b 100644 --- a/docs/fa/sql-reference/data-types/domains/ipv6.md +++ b/docs/fa/sql-reference/data-types/domains/ipv6.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 60 toc_title: IPv6 --- diff --git a/docs/fa/sql-reference/data-types/domains/overview.md b/docs/fa/sql-reference/data-types/domains/overview.md index ab6155caf64..4507ca850ef 100644 --- a/docs/fa/sql-reference/data-types/domains/overview.md +++ b/docs/fa/sql-reference/data-types/domains/overview.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 58 toc_title: "\u0628\u0631\u0631\u0633\u06CC \u0627\u062C\u0645\u0627\u0644\u06CC" --- diff --git a/docs/fa/sql-reference/data-types/enum.md b/docs/fa/sql-reference/data-types/enum.md index b79ffa1cea0..3ededf871b4 100644 --- a/docs/fa/sql-reference/data-types/enum.md +++ b/docs/fa/sql-reference/data-types/enum.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 50 toc_title: "\u0634\u0645\u0627\u0631\u0634\u06CC" --- @@ -114,7 +114,7 @@ INSERT INTO t_enum_nullable Values('hello'),('world'),(NULL) در رم `Enum` ستون به همان شیوه ذخیره می شود `Int8` یا `Int16` از مقادیر عددی مربوطه. هنگام خواندن در فرم متن, تاتر تجزیه ارزش به عنوان یک رشته و جستجو برای رشته مربوطه را از مجموعه ای از ارزش شمارشی. اگر یافت نشد, یک استثنا پرتاب می شود. هنگام خواندن در قالب متن, رشته به عنوان خوانده شده و مقدار عددی مربوطه نگاه کردن. یک استثنا پرتاب خواهد شد اگر یافت نشد. -هنگام نوشتن در فرم متن, این ارزش به عنوان رشته مربوطه می نویسد. اگر داده های ستون شامل زباله (اعداد است که از مجموعه معتبر نیست), یک استثنا پرتاب می شود. زمانی که خواندن و نوشتن در فایل باینری فرم آن را با این نسخهها کار به همان روش برای int8 و int16 انواع داده ها. +هنگام نوشتن در فرم متن, این ارزش به عنوان رشته مربوطه می نویسد. اگر داده های ستون شامل زباله (اعداد است که از مجموعه معتبر نیست), یک استثنا پرتاب می شود. زمانی که خواندن و نوشتن در فایل باینری فرم آن را با این نسخهها کار به همان روش برای Int8 و Int16 انواع داده ها. مقدار پیش فرض ضمنی ارزش با کمترین تعداد است. در طول `ORDER BY`, `GROUP BY`, `IN`, `DISTINCT` و به همین ترتیب, مادران رفتار به همان شیوه به عنوان اعداد مربوطه. مثلا, سفارش شده توسط انواع عددی. اپراتورهای برابری و مقایسه کار به همان شیوه در مادران به عنوان در مقادیر عددی اساسی انجام. @@ -127,6 +127,6 @@ Most numeric and string operations are not defined for Enum values, e.g. adding مقادیر شمارشی نیز قابل تبدیل به انواع عددی با استفاده از `toT` تابع, که تی یک نوع عددی است. هنگامی که تی مربوط به نوع عددی اساسی شمارشی است, این تبدیل صفر هزینه است. نوع شمارشی را می توان بدون هزینه با استفاده از تغییر تغییر, اگر تنها مجموعه ای از ارزش تغییر است. ممکن است که به هر دو اضافه کردن و حذف اعضای شمارشی با استفاده از تغییر (از بین بردن امن است تنها در صورتی که مقدار حذف شده است هرگز در جدول استفاده می شود). به عنوان یک حفاظت, تغییر مقدار عددی از یک عضو شمارشی که قبلا تعریف یک استثنا پرتاب. -با استفاده از تغییر آن را ممکن است برای تغییر یک enum8 به enum16 یا بالعکس فقط مانند تغییر یک int8 به int16. +با استفاده از تغییر آن را ممکن است برای تغییر یک Enum8 به Enum16 یا بالعکس فقط مانند تغییر یک Int8 به Int16. [مقاله اصلی](https://clickhouse.tech/docs/en/data_types/enum/) diff --git a/docs/fa/sql-reference/data-types/fixedstring.md b/docs/fa/sql-reference/data-types/fixedstring.md index cb1aaf3c583..9824acb0d2f 100644 --- a/docs/fa/sql-reference/data-types/fixedstring.md +++ b/docs/fa/sql-reference/data-types/fixedstring.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 45 toc_title: "\u0631\u0634\u062A\u0647 \u062B\u0627\u0628\u062A)" --- @@ -30,7 +30,7 @@ toc_title: "\u0631\u0634\u062A\u0647 \u062B\u0627\u0628\u062A)" هنگام قرار دادن داده ها, تاتر: -- مکمل یک رشته با null بایت اگر رشته شامل کمتر از `N` بایت +- مکمل یک رشته با بایت پوچ اگر رشته شامل کمتر از `N` بایت - پرتاب `Too large value for FixedString(N)` استثنا اگر رشته شامل بیش از `N` بایت در هنگام انتخاب داده, تاتر می کند بایت پوچ در پایان رشته را حذف کنید. اگر شما استفاده از `WHERE` بند, شما باید بایت پوچ دستی اضافه برای مطابقت با `FixedString` ارزش. مثال زیر نشان میدهد که چگونه به استفاده از `WHERE` بند با `FixedString`. diff --git a/docs/fa/sql-reference/data-types/float.md b/docs/fa/sql-reference/data-types/float.md index 99f25f4e810..25ce81c3645 100644 --- a/docs/fa/sql-reference/data-types/float.md +++ b/docs/fa/sql-reference/data-types/float.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 41 toc_title: Float32, Float64 --- @@ -82,6 +82,6 @@ SELECT 0 / 0 └──────────────┘ ``` - See the rules for `NaN` sorting in the section [ORDER BY clause](../sql_reference/statements/select.md). + See the rules for `NaN` sorting in the section [ORDER BY clause](../sql_reference/statements/select/order-by.md). [مقاله اصلی](https://clickhouse.tech/docs/en/data_types/float/) diff --git a/docs/fa/sql-reference/data-types/index.md b/docs/fa/sql-reference/data-types/index.md index cb5a702ddbd..8ec0ba03469 100644 --- a/docs/fa/sql-reference/data-types/index.md +++ b/docs/fa/sql-reference/data-types/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_folder_title: Data Types +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "\u0627\u0646\u0648\u0627\u0639 \u062F\u0627\u062F\u0647 \u0647\u0627" toc_priority: 37 toc_title: "\u0645\u0639\u0631\u0641\u06CC \u0634\u0631\u06A9\u062A" --- diff --git a/docs/fa/sql-reference/data-types/int-uint.md b/docs/fa/sql-reference/data-types/int-uint.md index 600ffbc56e9..749c1b52a1b 100644 --- a/docs/fa/sql-reference/data-types/int-uint.md +++ b/docs/fa/sql-reference/data-types/int-uint.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 40 toc_title: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64 --- diff --git a/docs/fa/sql-reference/data-types/nested-data-structures/index.md b/docs/fa/sql-reference/data-types/nested-data-structures/index.md index f2885dc8a16..b6cbdb6dc16 100644 --- a/docs/fa/sql-reference/data-types/nested-data-structures/index.md +++ b/docs/fa/sql-reference/data-types/nested-data-structures/index.md @@ -1,7 +1,8 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_folder_title: Nested Data Structures +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "\u0633\u0627\u062E\u062A\u0627\u0631\u0647\u0627\u06CC \u062F\u0627\ + \u062F\u0647 \u062A\u0648 \u062F\u0631 \u062A\u0648" toc_hidden: true toc_priority: 54 toc_title: "\u0645\u062E\u0641\u06CC" diff --git a/docs/fa/sql-reference/data-types/nested-data-structures/nested.md b/docs/fa/sql-reference/data-types/nested-data-structures/nested.md index 6422fabff53..4857274f47e 100644 --- a/docs/fa/sql-reference/data-types/nested-data-structures/nested.md +++ b/docs/fa/sql-reference/data-types/nested-data-structures/nested.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 57 toc_title: "\u062A\u0648 \u062F\u0631 \u062A\u0648(Name1 Type1, Name2 Type2, ...)" --- diff --git a/docs/fa/sql-reference/data-types/nullable.md b/docs/fa/sql-reference/data-types/nullable.md index 65e6faa0c07..96643e8ac6c 100644 --- a/docs/fa/sql-reference/data-types/nullable.md +++ b/docs/fa/sql-reference/data-types/nullable.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 54 toc_title: Nullable --- diff --git a/docs/fa/sql-reference/data-types/simpleaggregatefunction.md b/docs/fa/sql-reference/data-types/simpleaggregatefunction.md deleted file mode 120000 index 76a7ef3b802..00000000000 --- a/docs/fa/sql-reference/data-types/simpleaggregatefunction.md +++ /dev/null @@ -1 +0,0 @@ -../../../en/sql-reference/data-types/simpleaggregatefunction.md \ No newline at end of file diff --git a/docs/fa/sql-reference/data-types/simpleaggregatefunction.md b/docs/fa/sql-reference/data-types/simpleaggregatefunction.md new file mode 100644 index 00000000000..ddb1cd56be0 --- /dev/null +++ b/docs/fa/sql-reference/data-types/simpleaggregatefunction.md @@ -0,0 +1,38 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +--- + +# عملکرد پلاکتی {#data-type-simpleaggregatefunction} + +`SimpleAggregateFunction(name, types_of_arguments…)` نوع داده ها ارزش فعلی عملکرد کل را ذخیره می کند و حالت کامل خود را ذخیره نمی کند [`AggregateFunction`](aggregatefunction.md) چرا این بهینه سازی را می توان به توابع که اموال زیر را نگه می دارد اعمال می شود: در نتیجه استفاده از یک تابع `f` به مجموعه ردیف `S1 UNION ALL S2` می توان با استفاده از `f` به بخش هایی از مجموعه ردیف به طور جداگانه, و سپس دوباره استفاده `f` به نتایج: `f(S1 UNION ALL S2) = f(f(S1) UNION ALL f(S2))`. این ویژگی تضمین می کند که نتایج تجمع بخشی به اندازه کافی برای محاسبه یک ترکیب هستند, بنابراین ما لازم نیست برای ذخیره و پردازش هر گونه اطلاعات اضافی. + +توابع زیر مجموع پشتیبانی می شوند: + +- [`any`](../../sql-reference/aggregate-functions/reference.md#agg_function-any) +- [`anyLast`](../../sql-reference/aggregate-functions/reference.md#anylastx) +- [`min`](../../sql-reference/aggregate-functions/reference.md#agg_function-min) +- [`max`](../../sql-reference/aggregate-functions/reference.md#agg_function-max) +- [`sum`](../../sql-reference/aggregate-functions/reference.md#agg_function-sum) +- [`groupBitAnd`](../../sql-reference/aggregate-functions/reference.md#groupbitand) +- [`groupBitOr`](../../sql-reference/aggregate-functions/reference.md#groupbitor) +- [`groupBitXor`](../../sql-reference/aggregate-functions/reference.md#groupbitxor) + +مقادیر `SimpleAggregateFunction(func, Type)` نگاه و ذخیره شده به همان شیوه به عنوان `Type` بنابراین شما نیازی به اعمال توابع با `-Merge`/`-State` پسوندها. `SimpleAggregateFunction` عملکرد بهتر از `AggregateFunction` با همان تابع تجمع. + +**پارامترها** + +- نام تابع جمع. +- انواع استدلال تابع جمع. + +**مثال** + +``` sql +CREATE TABLE t +( + column1 SimpleAggregateFunction(sum, UInt64), + column2 SimpleAggregateFunction(any, String) +) ENGINE = ... +``` + +[مقاله اصلی](https://clickhouse.tech/docs/en/data_types/simpleaggregatefunction/) diff --git a/docs/fa/sql-reference/data-types/special-data-types/expression.md b/docs/fa/sql-reference/data-types/special-data-types/expression.md index 102136441b6..5efa0708cf5 100644 --- a/docs/fa/sql-reference/data-types/special-data-types/expression.md +++ b/docs/fa/sql-reference/data-types/special-data-types/expression.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 58 toc_title: "\u0639\u0628\u0627\u0631\u062A" --- diff --git a/docs/fa/sql-reference/data-types/special-data-types/index.md b/docs/fa/sql-reference/data-types/special-data-types/index.md index d77e3bd93d2..901a39a5335 100644 --- a/docs/fa/sql-reference/data-types/special-data-types/index.md +++ b/docs/fa/sql-reference/data-types/special-data-types/index.md @@ -1,7 +1,8 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_folder_title: Special Data Types +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "\u0627\u0646\u0648\u0627\u0639 \u062F\u0627\u062F\u0647 \u0647\u0627\ + \u06CC \u0648\u06CC\u0698\u0647" toc_hidden: true toc_priority: 55 toc_title: "\u0645\u062E\u0641\u06CC" diff --git a/docs/fa/sql-reference/data-types/special-data-types/interval.md b/docs/fa/sql-reference/data-types/special-data-types/interval.md index f9be0f45a46..7fcf722e0e5 100644 --- a/docs/fa/sql-reference/data-types/special-data-types/interval.md +++ b/docs/fa/sql-reference/data-types/special-data-types/interval.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 61 toc_title: "\u0641\u0627\u0635\u0644\u0647" --- # فاصله {#data-type-interval} -خانواده از انواع داده ها به نمایندگی از فواصل زمان و تاریخ. انواع حاصل از [INTERVAL](../../../sql-reference/operators.md#operator-interval) اپراتور +خانواده از انواع داده ها به نمایندگی از فواصل زمان و تاریخ. انواع حاصل از [INTERVAL](../../../sql-reference/operators/index.md#operator-interval) اپراتور !!! warning "اخطار" `Interval` مقادیر نوع داده را نمی توان در جداول ذخیره کرد. @@ -15,7 +15,7 @@ toc_title: "\u0641\u0627\u0635\u0644\u0647" ساختار: - فاصله زمانی به عنوان یک مقدار عدد صحیح بدون علامت. -- نوع یک بازه ی زمانی. +- نوع فاصله. انواع فاصله پشتیبانی شده: @@ -54,7 +54,7 @@ SELECT now() as current_date_time, current_date_time + INTERVAL 4 DAY └─────────────────────┴───────────────────────────────┘ ``` -فواصل با انواع مختلف نمی تواند ترکیب شود. شما می توانید فواصل مانند استفاده کنید `4 DAY 1 HOUR`. تعیین فواصل در واحد هایی که کوچکتر یا مساوی به کوچکترین واحد از فاصله مثلا فاصله `1 day and an hour` فاصله را می توان به عنوان بیان شده است `25 HOUR` یا `90000 SECOND`. +فواصل با انواع مختلف نمی تواند ترکیب شود. شما می توانید فواصل مانند استفاده کنید `4 DAY 1 HOUR`. مشخص فواصل در واحد است که کوچکتر یا برابر با کوچکترین واحد فاصله, مثلا, فاصله `1 day and an hour` فاصله را می توان به عنوان بیان شده است `25 HOUR` یا `90000 SECOND`. شما می توانید عملیات ریاضی با انجام نمی `Interval`- ارزش نوع, اما شما می توانید فواصل از انواع مختلف در نتیجه به ارزش در اضافه `Date` یا `DateTime` انواع داده ها. به عنوان مثال: @@ -68,7 +68,7 @@ SELECT now() AS current_date_time, current_date_time + INTERVAL 4 DAY + INTERVAL └─────────────────────┴────────────────────────────────────────────────────────┘ ``` -پرس و جوی زیر باعث می شود یک استثنا: +پرس و جو زیر باعث یک استثنا: ``` sql select now() AS current_date_time, current_date_time + (INTERVAL 4 DAY + INTERVAL 3 HOUR) @@ -81,5 +81,5 @@ Code: 43. DB::Exception: Received from localhost:9000. DB::Exception: Wrong argu ## همچنین نگاه کنید به {#see-also} -- [INTERVAL](../../../sql-reference/operators.md#operator-interval) اپراتور +- [INTERVAL](../../../sql-reference/operators/index.md#operator-interval) اپراتور - [توینتروال](../../../sql-reference/functions/type-conversion-functions.md#function-tointerval) توابع تبدیل نوع diff --git a/docs/fa/sql-reference/data-types/special-data-types/nothing.md b/docs/fa/sql-reference/data-types/special-data-types/nothing.md index 1b8140bf809..4bc900edb67 100644 --- a/docs/fa/sql-reference/data-types/special-data-types/nothing.md +++ b/docs/fa/sql-reference/data-types/special-data-types/nothing.md @@ -1,8 +1,8 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 60 -toc_title: "\u0647\u06CC\u0686 \u0686\u06CC\u0632" +toc_title: "\u0647\u06CC\u0686\u06CC" --- # هیچی {#nothing} diff --git a/docs/fa/sql-reference/data-types/special-data-types/set.md b/docs/fa/sql-reference/data-types/special-data-types/set.md index 8f5cc9be947..9a5fcee4855 100644 --- a/docs/fa/sql-reference/data-types/special-data-types/set.md +++ b/docs/fa/sql-reference/data-types/special-data-types/set.md @@ -1,12 +1,12 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 59 toc_title: "\u062A\u0646\u0638\u06CC\u0645" --- # تنظیم {#set} -مورد استفاده برای نیمه راست یک [IN](../../../sql-reference/statements/select.md#select-in-operators) اصطلاح. +مورد استفاده برای نیمه راست یک [IN](../../operators/in.md#select-in-operators) اصطلاح. [مقاله اصلی](https://clickhouse.tech/docs/en/data_types/special_data_types/set/) diff --git a/docs/fa/sql-reference/data-types/string.md b/docs/fa/sql-reference/data-types/string.md index 4cd48b2a2f2..e260b681e0d 100644 --- a/docs/fa/sql-reference/data-types/string.md +++ b/docs/fa/sql-reference/data-types/string.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 44 toc_title: "\u0631\u0634\u062A\u0647" --- diff --git a/docs/fa/sql-reference/data-types/tuple.md b/docs/fa/sql-reference/data-types/tuple.md index d99b3085ef8..fc4aeb92fe0 100644 --- a/docs/fa/sql-reference/data-types/tuple.md +++ b/docs/fa/sql-reference/data-types/tuple.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 53 toc_title: "\u062A\u0627\u067E\u0644 (\u062A\u06CC1, \u062A\u06CC2,...)" --- @@ -9,7 +9,7 @@ toc_title: "\u062A\u0627\u067E\u0644 (\u062A\u06CC1, \u062A\u06CC2,...)" یک تاپل از عناصر, هر یک با داشتن یک فرد [نوع](index.md#data_types). -تاپل برای گروه بندی ستون موقت استفاده می شود. ستون ها را می توان گروه بندی کرد زمانی که یک عبارت در یک پرس و جو استفاده می شود, و برای مشخص کردن پارامترهای رسمی خاصی از توابع لامبدا. برای کسب اطلاعات بیشتر به بخش ها مراجعه کنید [در اپراتورها](../../sql-reference/statements/select.md) و [توابع سفارش بالاتر](../../sql-reference/functions/higher-order-functions.md). +تاپل برای گروه بندی ستون موقت استفاده می شود. ستون ها را می توان گروه بندی کرد زمانی که یک عبارت در یک پرس و جو استفاده می شود, و برای مشخص کردن پارامترهای رسمی خاصی از توابع لامبدا. برای کسب اطلاعات بیشتر به بخش ها مراجعه کنید [در اپراتورها](../../sql-reference/operators/in.md) و [توابع سفارش بالاتر](../../sql-reference/functions/higher-order-functions.md). تاپل می تواند در نتیجه یک پرس و جو. در این مورد, برای فرمت های متنی غیر از جانسون, ارزش کاما از هم جدا در براکت. در فرمت های جوسون, تاپل خروجی به عنوان ارریس هستند (در براکت مربع). diff --git a/docs/fa/sql-reference/data-types/uuid.md b/docs/fa/sql-reference/data-types/uuid.md index bec3e996f70..bc414b3b7e3 100644 --- a/docs/fa/sql-reference/data-types/uuid.md +++ b/docs/fa/sql-reference/data-types/uuid.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 46 toc_title: UUID --- diff --git a/docs/fa/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md b/docs/fa/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md index 055e3ad1204..0cb27e5ec39 100644 --- a/docs/fa/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md +++ b/docs/fa/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 45 toc_title: "\u0644\u063A\u062A\u0646\u0627\u0645\u0647\u0647\u0627 \u0633\u0644\u0633\ \u0644\u0647 \u0645\u0631\u0627\u062A\u0628\u06CC" diff --git a/docs/fa/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md b/docs/fa/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md index 5fbf88291a8..eee74368127 100644 --- a/docs/fa/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md +++ b/docs/fa/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 41 toc_title: "\u0630\u062E\u06CC\u0631\u0647 \u0648\u0627\u0698\u0647\u0646\u0627\u0645\ \u0647\u0647\u0627 \u062F\u0631 \u062D\u0627\u0641\u0638\u0647" @@ -57,6 +57,7 @@ LAYOUT(LAYOUT_TYPE(param value)) -- layout settings - [درهم](#dicts-external_dicts_dict_layout-hashed) - [فشردهسازی](#dicts-external_dicts_dict_layout-sparse_hashed) - [نهانگاه](#cache) +- [مستقیم](#direct) - [رنگها](#range-hashed) - [\_ساخت مجتمع](#complex-key-hashed) - [\_پیچید\_چهای پیچیده](#complex-key-cache) @@ -297,6 +298,28 @@ LAYOUT(CACHE(SIZE_IN_CELLS 1000000000)) این نوع ذخیره سازی برای استفاده با کامپوزیت است [کلید](external-dicts-dict-structure.md). مشابه به `cache`. +### مستقیم {#direct} + +فرهنگ لغت در حافظه ذخیره نمی شود و به طور مستقیم به منبع می رود در طول پردازش یک درخواست. + +کلید فرهنگ لغت است `UInt64` نوع. + +همه انواع [منابع](external-dicts-dict-sources.md), به جز فایل های محلی, پشتیبانی می شوند. + +مثال پیکربندی: + +``` xml + + + +``` + +یا + +``` sql +LAYOUT(DIRECT()) +``` + ### شمال اروپا {#ip-trie} این نوع ذخیره سازی برای پیشوندهای نقشه برداری شبکه (نشانی های اینترنتی) به فراداده مانند ان است. @@ -355,7 +378,7 @@ PRIMARY KEY prefix کلید باید تنها یک ویژگی نوع رشته ای داشته باشد که شامل یک پیشوند مجاز است. انواع دیگر هنوز پشتیبانی نمی شوند. -برای نمایش داده شد, شما باید توابع مشابه استفاده کنید (`dictGetT` با یک تاپل) به لغت نامه ها با کلید های ترکیبی: +برای نمایش داده شد, شما باید توابع مشابه استفاده کنید (`dictGetT` با یک تاپل) به عنوان لغت نامه با کلید های کامپوزیت: ``` sql dictGetT('dict_name', 'attr_name', tuple(ip)) diff --git a/docs/fa/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md b/docs/fa/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md index 4080580a329..ed4485d4392 100644 --- a/docs/fa/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md +++ b/docs/fa/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 42 toc_title: "\u0628\u0647 \u0631\u0648\u0632 \u0631\u0633\u0627\u0646\u06CC \u0641\u0631\ \u0647\u0646\u06AF \u0644\u063A\u062A" @@ -52,6 +52,11 @@ LIFETIME(300) LIFETIME(MIN 300 MAX 360) ``` +اگر `0` و `0`, کلیک می کند فرهنگ لغت توسط ایست بارگذاری مجدد نیست. +در این مورد, تاتر می توانید فرهنگ لغت زودتر بارگذاری مجدد اگر فایل پیکربندی فرهنگ لغت تغییر یافت و یا `SYSTEM RELOAD DICTIONARY` فرمان اعدام شد. + +هنگام به روز رسانی لغت نامه, سرور کلیک اعمال منطق مختلف بسته به نوع [متن](external-dicts-dict-sources.md): + هنگام به روز رسانی لغت نامه, سرور کلیک اعمال منطق مختلف بسته به نوع [متن](external-dicts-dict-sources.md): - برای یک فایل متنی زمان اصلاح را بررسی می کند. اگر زمان از زمان قبلا ثبت شده متفاوت, فرهنگ لغت به روز شده است. diff --git a/docs/fa/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md b/docs/fa/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md index 8a784f4b3e0..81e0055d92e 100644 --- a/docs/fa/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md +++ b/docs/fa/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 43 toc_title: "\u0645\u0646\u0627\u0628\u0639 \u0644\u063A\u062A \u0646\u0627\u0645\u0647\ \ \u0647\u0627\u06CC \u062E\u0627\u0631\u062C\u06CC" @@ -38,6 +38,28 @@ SOURCE(SOURCE_TYPE(param1 val1 ... paramN valN)) -- Source configuration منبع در پیکربندی `source` بخش. +برای انواع منبع [پرونده محلی](#dicts-external_dicts_dict_sources-local_file), [پرونده اجرایی](#dicts-external_dicts_dict_sources-executable), [HTTP(s)](#dicts-external_dicts_dict_sources-http), [فاحشه خانه](#dicts-external_dicts_dict_sources-clickhouse) +تنظیمات اختیاری در دسترس هستند: + +``` xml + + + /opt/dictionaries/os.tsv + TabSeparated + + + 0 + + +``` + +یا + +``` sql +SOURCE(FILE(path '/opt/dictionaries/os.tsv' format 'TabSeparated')) +SETTINGS(format_csv_allow_single_quotes = 0) +``` + انواع منابع (`source_type`): - [پرونده محلی](#dicts-external_dicts_dict_sources-local_file) @@ -196,7 +218,7 @@ SOURCE(ODBC( **نمونه ای از استفاده نا امن** -اجازه می دهد تا پیکربندی unixodbc برای postgresql. محتوای `/etc/odbc.ini`: +اجازه می دهد تا پیکربندی unixODBC برای PostgreSQL. محتوای `/etc/odbc.ini`: ``` text [gregtest] @@ -221,7 +243,7 @@ SELECT * FROM odbc('DSN=gregtest;Servername=some-server.com', 'test_db'); سیستم عامل اوبونتو. -نصب unixodbc و odbc driver for postgresql: +نصب unixODBC و ODBC driver for PostgreSQL: ``` bash $ sudo apt-get install -y unixodbc odbcinst odbc-postgresql diff --git a/docs/fa/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md b/docs/fa/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md index c952372656d..4df4e774bb4 100644 --- a/docs/fa/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md +++ b/docs/fa/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 44 toc_title: "\u06A9\u0644\u06CC\u062F \u0641\u0631\u0647\u0646\u06AF \u0644\u063A\u062A\ \ \u0648 \u0632\u0645\u06CC\u0646\u0647 \u0647\u0627\u06CC" diff --git a/docs/fa/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md b/docs/fa/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md index 96aee6503e8..9bd07022190 100644 --- a/docs/fa/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md +++ b/docs/fa/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 40 toc_title: "\u067E\u06CC\u06A9\u0631\u0628\u0646\u062F\u06CC \u06CC\u06A9 \u0641\u0631\ \u0647\u0646\u06AF \u0644\u063A\u062A \u062E\u0627\u0631\u062C\u06CC" diff --git a/docs/fa/sql-reference/dictionaries/external-dictionaries/external-dicts.md b/docs/fa/sql-reference/dictionaries/external-dictionaries/external-dicts.md index 75ee505e630..3640c07a092 100644 --- a/docs/fa/sql-reference/dictionaries/external-dictionaries/external-dicts.md +++ b/docs/fa/sql-reference/dictionaries/external-dictionaries/external-dicts.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 39 toc_title: "\u062A\u0648\u0636\u06CC\u062D\u0627\u062A \u06A9\u0644\u06CC" --- @@ -19,6 +19,12 @@ toc_title: "\u062A\u0648\u0636\u06CC\u062D\u0627\u062A \u06A9\u0644\u06CC" واژهنامهها را می توان در هنگام راه اندازی سرور و یا در اولین استفاده لود, بسته به [\_بارگیری کامل](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-dictionaries_lazy_load) تنظیمات. +این [واژهنامهها](../../../operations/system-tables.md#system_tables-dictionaries) جدول سیستم شامل اطلاعات در مورد لغت نامه پیکربندی در سرور. برای هر فرهنگ لغت شما می توانید وجود دارد: + +- وضعیت فرهنگ لغت. +- پارامترهای پیکربندی. +- معیارهای مانند مقدار رم اختصاص داده شده برای فرهنگ لغت و یا تعدادی از نمایش داده شد از فرهنگ لغت با موفقیت لود شد. + فایل پیکربندی فرهنگ لغت دارای فرمت زیر است: ``` xml diff --git a/docs/fa/sql-reference/dictionaries/external-dictionaries/index.md b/docs/fa/sql-reference/dictionaries/external-dictionaries/index.md index b5c506f5d93..28403e5355c 100644 --- a/docs/fa/sql-reference/dictionaries/external-dictionaries/index.md +++ b/docs/fa/sql-reference/dictionaries/external-dictionaries/index.md @@ -1,7 +1,8 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_folder_title: External Dictionaries +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "\u0648\u0627\u0698\u0647\u0646\u0627\u0645\u0647\u0647\u0627 \u062E\ + \u0627\u0631\u062C\u06CC" toc_priority: 37 --- diff --git a/docs/fa/sql-reference/dictionaries/index.md b/docs/fa/sql-reference/dictionaries/index.md index fa41fbaf9ab..ac575a9c885 100644 --- a/docs/fa/sql-reference/dictionaries/index.md +++ b/docs/fa/sql-reference/dictionaries/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_folder_title: Dictionaries +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "\u0648\u0627\u0698\u0647\u0646\u0627\u0645\u0647\u0647\u0627" toc_priority: 35 toc_title: "\u0645\u0639\u0631\u0641\u06CC \u0634\u0631\u06A9\u062A" --- @@ -17,6 +17,6 @@ toc_title: "\u0645\u0639\u0631\u0641\u06CC \u0634\u0631\u06A9\u062A" پشتیبانی از کلیک: - [ساخته شده در لغت نامه](internal-dicts.md#internal_dicts) با یک خاص [مجموعه ای از توابع](../../sql-reference/functions/ym-dict-functions.md). -- [افزونه لغت نامه (خارجی)](external-dictionaries/external-dicts.md#dicts-external-dicts) با یک [خالص توابع](../../sql-reference/functions/ext-dict-functions.md). +- [افزونه لغت نامه (خارجی) ](external-dictionaries/external-dicts.md#dicts-external-dicts) با یک [مجموعه ای از توابع](../../sql-reference/functions/ext-dict-functions.md). [مقاله اصلی](https://clickhouse.tech/docs/en/query_language/dicts/) diff --git a/docs/fa/sql-reference/dictionaries/internal-dicts.md b/docs/fa/sql-reference/dictionaries/internal-dicts.md index 33700b53f64..6e849733d20 100644 --- a/docs/fa/sql-reference/dictionaries/internal-dicts.md +++ b/docs/fa/sql-reference/dictionaries/internal-dicts.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 39 toc_title: "\u0648\u0627\u0698\u0647\u0646\u0627\u0645\u0647\u0647\u0627 \u062F\u0627\ \u062E\u0644\u06CC" @@ -40,7 +40,7 @@ ClickHouse شامل ساخته شده است در ویژگی برای کار ب `regions_names_*.txt`: ستون (بدون هدر): - شناسه منطقه (`UInt32`) -- نام منطقه (`String`) — Can’t contain tabs or line feeds, even escaped ones. +- نام منطقه (`String`) — Can't contain tabs or line feeds, even escaped ones. مجموعه تخت برای ذخیره سازی در رم استفاده می شود. به همین دلیل شناسه نباید بیش از یک میلیون. diff --git a/docs/fa/sql-reference/functions/arithmetic-functions.md b/docs/fa/sql-reference/functions/arithmetic-functions.md index 3d5af8b3536..be1fa3497c8 100644 --- a/docs/fa/sql-reference/functions/arithmetic-functions.md +++ b/docs/fa/sql-reference/functions/arithmetic-functions.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 35 toc_title: "\u062D\u0633\u0627\u0628" --- @@ -21,7 +21,7 @@ SELECT toTypeName(0), toTypeName(0 + 0), toTypeName(0 + 0 + 0), toTypeName(0 + 0 └───────────────┴────────────────────────┴─────────────────────────────────┴──────────────────────────────────────────┘ ``` -حساب توابع کار برای هر جفت از انواع از uint8, uint16, uint32, uint64, int8, int16, int32, int64, float32 یا float64. +حساب توابع کار برای هر جفت از انواع از UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32 یا Float64. سرریز به همان شیوه که در ج تولید++. diff --git a/docs/fa/sql-reference/functions/array-functions.md b/docs/fa/sql-reference/functions/array-functions.md index 3a191bfe80e..1988ed4266e 100644 --- a/docs/fa/sql-reference/functions/array-functions.md +++ b/docs/fa/sql-reference/functions/array-functions.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 46 toc_title: "\u06A9\u0627\u0631 \u0628\u0627 \u0627\u0631\u0631\u06CC\u0633" --- @@ -10,19 +10,19 @@ toc_title: "\u06A9\u0627\u0631 \u0628\u0627 \u0627\u0631\u0631\u06CC\u0633" ## خالی {#function-empty} بازده 1 برای یک مجموعه خالی, یا 0 برای یک مجموعه غیر خالی. -نتیجه این نوع uint8. +نتیجه این نوع UInt8. این تابع نیز برای رشته کار می کند. ## notEmpty {#function-notempty} بازده 0 برای یک مجموعه خالی, یا 1 برای یک مجموعه غیر خالی. -نتیجه این نوع uint8. +نتیجه این نوع UInt8. این تابع نیز برای رشته کار می کند. ## طول {#array_functions-length} بازگرداندن تعداد اقلام در مجموعه. -نتیجه این نوع uint64. +نتیجه این نوع UInt64. این تابع نیز برای رشته کار می کند. ## emptyArrayUInt8, emptyArrayUInt16, emptyArrayUInt32, emptyArrayUInt64 {#emptyarrayuint8-emptyarrayuint16-emptyarrayuint32-emptyarrayuint64} @@ -84,7 +84,7 @@ SELECT arrayConcat([1, 2], [3, 4], [5, 6]) AS res عنصر را با شاخص دریافت کنید `n` از مجموعه `arr`. `n` باید هر نوع عدد صحیح باشد. شاخص ها در مجموعه ای از یک شروع می شوند. -شاخص های منفی پشتیبانی می شوند. در این مورد آن را انتخاب می کند که عنصر مربوطه شماره از پایان. به عنوان مثال, `arr[-1]` اخرین وسیله ست +شاخص های منفی پشتیبانی می شوند. در این مورد, این انتخاب عنصر مربوطه شماره از پایان. به عنوان مثال, `arr[-1]` اخرین وسیله ست اگر شاخص می افتد در خارج از مرزهای مجموعه, این گرداند برخی از مقدار پیش فرض (0 برای اعداد, یک رشته خالی برای رشته, و غیره.), به جز برای مورد با یک مجموعه غیر ثابت و یک شاخص ثابت 0 (در این مورد وجود خواهد داشت یک خطا `Array indices are 1-based`). @@ -156,7 +156,7 @@ hasAny(array1, array2) - `array1` – Array of any type with a set of elements. - `array2` – Array of any type with a set of elements. -**بازگشت ارزش** +**مقادیر بازگشتی** - `1` اگر `array1` و `array2` حداقل یک عنصر مشابه داشته باشید. - `0` وگرنه @@ -198,7 +198,7 @@ SELECT indexOf([1, 3, NULL, NULL], NULL) ## هشدار داده می شود) {#countequalarr-x} -بازده تعداد عناصر موجود در آرایه برابر با x. معادل arraycount (elem -\> elem = x arr). +بازده تعداد عناصر موجود در آرایه برابر با x. معادل arrayCount (elem -\> elem = x arr). `NULL` عناصر به عنوان مقادیر جداگانه به کار گرفته. @@ -294,7 +294,7 @@ LIMIT 10 └─────────┴─────────┴────────┘ ``` -در این مثال هر هدف شناسه محاسبه تعداد تبدیل (هر عنصر در اهداف تو در تو ساختار داده ها یک هدف است که رسیده بود که ما اشاره به عنوان یک تبدیل) و تعداد جلسات. بدون مجموعه ملحق, ما می خواهیم تعداد جلسات به عنوان مجموع شمارش (امضا کردن). اما در این مورد خاص ردیف شد ضرب در تو در تو در اهداف و ساختار آن در سفارش به تعداد هر جلسه یک بار بعد از این ما اعمال یک شرط به ارزش arrayenumerateuniq(اهداف است.id) تابع. +در این مثال هر هدف شناسه محاسبه تعداد تبدیل (هر عنصر در اهداف تو در تو ساختار داده ها یک هدف است که رسیده بود که ما اشاره به عنوان یک تبدیل) و تعداد جلسات. بدون مجموعه ملحق, ما می خواهیم تعداد جلسات به عنوان مجموع شمارش (امضا کردن). اما در این مورد خاص ردیف شد ضرب در تو در تو در اهداف و ساختار آن در سفارش به تعداد هر جلسه یک بار بعد از این ما اعمال یک شرط به ارزش arrayEnumerateUniq(اهداف است.ID) تابع. تابع ارریینومراتونیک می تواند چندین بار از همان اندازه به عنوان استدلال استفاده کند. در این مورد, منحصر به فرد است برای تاپل از عناصر در موقعیت های مشابه در تمام ارریس در نظر گرفته. @@ -369,7 +369,7 @@ arrayPushBack(array, single_value) **پارامترها** - `array` – Array. -- `single_value` – A single value. Only numbers can be added to an array with numbers, and only strings can be added to an array of strings. When adding numbers, ClickHouse automatically sets the `single_value` نوع داده مجموعه را تایپ کنید. برای کسب اطلاعات بیشتر در مورد انواع داده ها در خانه کلیک کنید “[انواع داده ها](../../sql-reference/data-types/index.md#data_types)”. می توان `NULL`. تابع می افزاید: `NULL` عنصر به مجموعه ای, و نوع عناصر مجموعه ای تبدیل به `Nullable`. +- `single_value` – A single value. Only numbers can be added to an array with numbers, and only strings can be added to an array of strings. When adding numbers, ClickHouse automatically sets the `single_value` نوع داده مجموعه را تایپ کنید. برای کسب اطلاعات بیشتر در مورد انواع داده ها در خانه کلیک کنید “[انواع داده ها](../../sql-reference/data-types/index.md#data_types)”. می تواند باشد `NULL`. تابع می افزاید: `NULL` عنصر به مجموعه ای, و نوع عناصر مجموعه ای تبدیل به `Nullable`. **مثال** @@ -394,7 +394,7 @@ arrayPushFront(array, single_value) **پارامترها** - `array` – Array. -- `single_value` – A single value. Only numbers can be added to an array with numbers, and only strings can be added to an array of strings. When adding numbers, ClickHouse automatically sets the `single_value` نوع داده مجموعه را تایپ کنید. برای کسب اطلاعات بیشتر در مورد انواع داده ها در خانه کلیک کنید “[انواع داده ها](../../sql-reference/data-types/index.md#data_types)”. می تواند باشد `NULL`. این تابع می افزاید: `NULL` عنصر به مجموعه ای, و نوع عناصر مجموعه ای تبدیل به `Nullable`. +- `single_value` – A single value. Only numbers can be added to an array with numbers, and only strings can be added to an array of strings. When adding numbers, ClickHouse automatically sets the `single_value` نوع داده مجموعه را تایپ کنید. برای کسب اطلاعات بیشتر در مورد انواع داده ها در خانه کلیک کنید “[انواع داده ها](../../sql-reference/data-types/index.md#data_types)”. می تواند باشد `NULL`. تابع می افزاید: `NULL` عنصر به مجموعه ای, و نوع عناصر مجموعه ای تبدیل به `Nullable`. **مثال** @@ -450,7 +450,7 @@ SELECT arrayResize([1], 3, NULL) └───────────────────────────┘ ``` -## arraySlice {#arrayslice} +## بند {#arrayslice} یک تکه از مجموعه را برمی گرداند. @@ -523,7 +523,7 @@ SELECT arraySort([1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf]); - `NaN` مقادیر درست قبل هستند `NULL`. - `Inf` مقادیر درست قبل هستند `NaN`. -توجه داشته باشید که `arraySort` یک [عملکرد عالی مرتبه](higher-order-functions.md). شما می توانید یک تابع لامبدا را به عنوان اولین استدلال منتقل کنید. در این مورد مرتب سازی سفارش تعیین می شود در نتیجه از lambda تابع اعمال شده به عناصر آرایه است. +توجه داشته باشید که `arraySort` یک [عملکرد عالی مرتبه](higher-order-functions.md). شما می توانید یک تابع لامبدا را به عنوان اولین استدلال منتقل کنید. در این مورد ترتیب مرتب سازی بر اساس نتیجه تابع لامبدا اعمال شده به عناصر مجموعه تعیین می شود. بیایید مثال زیر را در نظر بگیریم: @@ -551,7 +551,7 @@ SELECT arraySort((x, y) -> y, ['hello', 'world'], [2, 1]) as res; └────────────────────┘ ``` -در اینجا عناصر موجود در مجموعه دوم (\[2, 1\]) تعریف یک کلید مرتب سازی برای عنصر مربوطه از مجموعه منبع (\[‘hello’, ‘world’\]), به این معنا که, \[‘hello’ –\> 2, ‘world’ –\> 1\]. Since the lambda function doesn’t use `x` مقادیر واقعی مجموعه منبع بر نظم در نتیجه تاثیر نمی گذارد. پس, ‘hello’ خواهد بود که عنصر دوم در نتیجه, و ‘world’ خواهد بود که برای اولین بار. +در اینجا عناصر موجود در مجموعه دوم (\[2, 1\]) تعریف یک کلید مرتب سازی برای عنصر مربوطه از مجموعه منبع (\[‘hello’, ‘world’\]), به این معنا که, \[‘hello’ –\> 2, ‘world’ –\> 1\]. Since the lambda function doesn't use `x` مقادیر واقعی مجموعه منبع بر نظم در نتیجه تاثیر نمی گذارد. پس, ‘hello’ خواهد بود که عنصر دوم در نتیجه, و ‘world’ خواهد بود که برای اولین بار. نمونه های دیگر در زیر نشان داده شده. @@ -990,7 +990,7 @@ SELECT arrayCompact([1, 1, nan, nan, 2, 3, 3, 3]) ## ارریزیپ {#arrayzip} -Combine multiple Array type columns into one Array\[Tuple(…)\] column +ترکیبی از چندین ردیف به یک مجموعه واحد. مجموعه حاصل شامل عناصر مربوطه را از ارریس منبع به تاپل در جهت ذکر شده از استدلال گروه بندی می شوند. **نحو** @@ -1000,31 +1000,35 @@ arrayZip(arr1, arr2, ..., arrN) **پارامترها** -`arr` — Any number of [& حذف](../../sql-reference/data-types/array.md) ستون نوع به ترکیب. +- `arrN` — [& حذف](../data-types/array.md). + +این تابع می تواند هر تعداد از مجموعه ای از انواع مختلف را. تمام ورودی های ورودی باید با اندازه یکسان باشند. **مقدار بازگشتی** -The result of Array\[Tuple(…)\] type after the combination of these arrays +- مجموعه ای با عناصر از ارریس منبع به گروه بندی می شوند [توپلس](../data-types/tuple.md). انواع داده ها در تاپل همان نوع از بند ورودی هستند و در همان جهت به عنوان ارریس به تصویب می رسد. + +نوع: [& حذف](../data-types/array.md). **مثال** پرسوجو: ``` sql -SELECT arrayZip(['a', 'b', 'c'], ['d', 'e', 'f']); +SELECT arrayZip(['a', 'b', 'c'], [5, 2, 1]) ``` نتیجه: ``` text -┌─arrayZip(['a', 'b', 'c'], ['d', 'e', 'f'])─┐ -│ [('a','d'),('b','e'),('c','f')] │ -└────────────────────────────────────────────┘ +┌─arrayZip(['a', 'b', 'c'], [5, 2, 1])─┐ +│ [('a',5),('b',2),('c',1)] │ +└──────────────────────────────────────┘ ``` ## ارریایکو {#arrayauc} -محاسبه حراج (منطقه تحت منحنی, که یک مفهوم در یادگیری ماشین است, مشاهده اطلاعات بیشتر: https://en.wikipedia.org/wiki/receiver\_operating\_characteristic\#area\_under\_the\_curve). +محاسبه حراج (منطقه تحت منحنی, که یک مفهوم در یادگیری ماشین است, مشاهده اطلاعات بیشتر: https://en.wikipedia.org/wiki/Receiver\_operating\_characteristic\#Area\_under\_the\_curve). **نحو** diff --git a/docs/fa/sql-reference/functions/array-join.md b/docs/fa/sql-reference/functions/array-join.md index 9d36eba1b89..4f4ee3eab60 100644 --- a/docs/fa/sql-reference/functions/array-join.md +++ b/docs/fa/sql-reference/functions/array-join.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 61 toc_title: "\u0627\u0631\u0631\u06CC\u062C\u06CC\u0646" --- diff --git a/docs/fa/sql-reference/functions/bit-functions.md b/docs/fa/sql-reference/functions/bit-functions.md index c25f5470614..932b8b8656e 100644 --- a/docs/fa/sql-reference/functions/bit-functions.md +++ b/docs/fa/sql-reference/functions/bit-functions.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 48 toc_title: "\u0628\u06CC\u062A" --- # توابع بیت {#bit-functions} -بیت توابع کار برای هر جفت از انواع از uint8, uint16, uint32, uint64, int8, int16, int32, int64, float32 یا float64. +بیت توابع کار برای هر جفت از انواع از UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32 یا Float64. نوع نتیجه یک عدد صحیح با بیت به حداکثر بیت از استدلال خود را برابر است. اگر حداقل یکی از استدلال امضا شده است, نتیجه یک شماره امضا شده است. اگر استدلال یک عدد ممیز شناور است, این است که به درون بازیگران64. diff --git a/docs/fa/sql-reference/functions/bitmap-functions.md b/docs/fa/sql-reference/functions/bitmap-functions.md index 64d94dd9571..f7740cf02e9 100644 --- a/docs/fa/sql-reference/functions/bitmap-functions.md +++ b/docs/fa/sql-reference/functions/bitmap-functions.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 49 toc_title: "\u0646\u06AF\u0627\u0634\u062A \u0628\u06CC\u062A" --- @@ -173,7 +173,7 @@ bitmapHasAny(bitmap1, bitmap2) - `bitmap*` – bitmap object. -**بازگشت ارزش** +**مقادیر بازگشتی** - `1` اگر `bitmap1` و `bitmap2` حداقل یک عنصر مشابه داشته باشید. - `0` وگرنه @@ -399,7 +399,7 @@ SELECT bitmapToArray(bitmapAndnot(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS ## اطلاعات دقیق {#bitmapandcardinality} -دو بیت مپ و محاسبه بازگشت cardinality از نوع uint64. +دو بیت مپ و محاسبه بازگشت cardinality از نوع UInt64. ``` sql bitmapAndCardinality(bitmap,bitmap) @@ -423,7 +423,7 @@ SELECT bitmapAndCardinality(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res; ## کمبود سیگار {#bitmaporcardinality} -دو بیت مپ و یا محاسبه بازگشت cardinality از نوع uint64. +دو بیت مپ و یا محاسبه بازگشت cardinality از نوع UInt64. ``` sql bitmapOrCardinality(bitmap,bitmap) @@ -447,7 +447,7 @@ SELECT bitmapOrCardinality(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res; ## هشدار داده می شود {#bitmapxorcardinality} -دو بیت مپ xor محاسبه بازگشت cardinality از نوع uint64. +دو بیت مپ xor محاسبه بازگشت cardinality از نوع UInt64. ``` sql bitmapXorCardinality(bitmap,bitmap) @@ -471,7 +471,7 @@ SELECT bitmapXorCardinality(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res; ## اطلاعات دقیق {#bitmapandnotcardinality} -دو بیت مپ andnot محاسبه بازگشت cardinality از نوع uint64. +دو بیت مپ andnot محاسبه بازگشت cardinality از نوع UInt64. ``` sql bitmapAndnotCardinality(bitmap,bitmap) diff --git a/docs/fa/sql-reference/functions/comparison-functions.md b/docs/fa/sql-reference/functions/comparison-functions.md index aee5cfe664c..61102ec38a6 100644 --- a/docs/fa/sql-reference/functions/comparison-functions.md +++ b/docs/fa/sql-reference/functions/comparison-functions.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 36 toc_title: "\u0645\u0642\u0627\u06CC\u0633\u0647" --- diff --git a/docs/fa/sql-reference/functions/conditional-functions.md b/docs/fa/sql-reference/functions/conditional-functions.md index 6e1c651e6ca..47fa1f4773f 100644 --- a/docs/fa/sql-reference/functions/conditional-functions.md +++ b/docs/fa/sql-reference/functions/conditional-functions.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 43 toc_title: "\u0634\u0631\u0637\u06CC " --- @@ -115,7 +115,7 @@ WHERE isNotNull(left) AND isNotNull(right) ## چندف {#multiif} -اجازه می دهد تا شما را به نوشتن [CASE](../operators.md#operator_case) اپراتور فشرده تر در پرس و جو. +اجازه می دهد تا شما را به نوشتن [CASE](../operators/index.md#operator_case) اپراتور فشرده تر در پرس و جو. نحو: `multiIf(cond_1, then_1, cond_2, then_2, ..., else)` @@ -170,7 +170,7 @@ FROM LEFT_RIGHT ## ارزشهای پوچ در شرطی {#null-values-in-conditionals} -زمانی که `NULL` ارزش ها در شرطی درگیر, نتیجه نیز خواهد بود `NULL`. +چه زمانی `NULL` ارزش ها در شرطی درگیر, نتیجه نیز خواهد بود `NULL`. ``` sql SELECT diff --git a/docs/fa/sql-reference/functions/date-time-functions.md b/docs/fa/sql-reference/functions/date-time-functions.md index 70b764e9f58..e956b7b71fa 100644 --- a/docs/fa/sql-reference/functions/date-time-functions.md +++ b/docs/fa/sql-reference/functions/date-time-functions.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 39 toc_title: "\u06A9\u0627\u0631 \u0628\u0627 \u062A\u0627\u0631\u06CC\u062E \u0648\ \ \u0632\u0645\u0627\u0646" @@ -46,7 +46,7 @@ SELECT ## سال {#todayofyear} -تبدیل یک تاریخ و یا تاریخ با گذشت زمان به یک uint16 تعداد شامل تعداد روز از سال (1-366). +تبدیل یک تاریخ و یا تاریخ با گذشت زمان به یک UInt16 تعداد شامل تعداد روز از سال (1-366). ## تودیفمون {#todayofmonth} @@ -58,16 +58,16 @@ SELECT ## تمام {#tohour} -تبدیل تاریخ با هم به یک uint8 شماره حاوی تعداد ساعت در زمان 24 ساعته (0-23). +تبدیل تاریخ با هم به یک UInt8 شماره حاوی تعداد ساعت در زمان 24 ساعته (0-23). This function assumes that if clocks are moved ahead, it is by one hour and occurs at 2 a.m., and if clocks are moved back, it is by one hour and occurs at 3 a.m. (which is not always true – even in Moscow the clocks were twice changed at a different time). ## تامینوت {#tominute} -تبدیل تاریخ با هم به یک uint8 شماره حاوی تعداد دقیقه از ساعت (0-59). +تبدیل تاریخ با هم به یک UInt8 شماره حاوی تعداد دقیقه از ساعت (0-59). ## جای خالی {#tosecond} -تبدیل تاریخ با هم به یک uint8 شماره حاوی شماره دوم در دقیقه (0-59). +تبدیل تاریخ با هم به یک UInt8 شماره حاوی شماره دوم در دقیقه (0-59). ثانیه جهش برای به حساب نمی. ## تیونیتیمستمپ {#to-unix-timestamp} @@ -115,7 +115,7 @@ SELECT toUnixTimestamp('2017-11-05 08:07:47', 'Asia/Tokyo') AS unix_timestamp دور کردن تاریخ یا تاریخ با زمان به روز اول سال ایزو. تاریخ را برمی گرداند. -## toStartOfQuarter {#tostartofquarter} +## تاستارتوفارتر {#tostartofquarter} دور یک تاریخ یا تاریخ با زمان به روز اول سه ماهه. اولین روز از سه ماهه است یا 1 ژانویه, 1 مارس, 1 جولای, یا 1 اکتبر. @@ -160,7 +160,7 @@ SELECT toUnixTimestamp('2017-11-05 08:07:47', 'Asia/Tokyo') AS unix_timestamp دور پایین تاریخ با زمان به شروع فاصله ده دقیقه. -## toStartOfFifteenMinutes {#tostartoffifteenminutes} +## حفاظت از محیط زیست {#tostartoffifteenminutes} دور پایین تاریخ با زمان به شروع فاصله پانزده دقیقه. @@ -214,7 +214,7 @@ SELECT toUnixTimestamp('2017-11-05 08:07:47', 'Asia/Tokyo') AS unix_timestamp ## هشدار داده می شود {#toisoweek} -تبدیل یک تاریخ و یا تاریخ با گذشت زمان به یک uint8 تعداد شامل iso هفته شماره. +تبدیل یک تاریخ و یا تاریخ با گذشت زمان به یک UInt8 تعداد شامل ISO هفته شماره. ## تاریخ \[, حالت\]) {#toweekdatemode} @@ -300,7 +300,7 @@ SELECT toDate('2016-12-27') AS date, toYearWeek(date) AS yearWeek0, toYearWeek(d قبول صفر استدلال و بازده تاریخ دیروز در یکی از لحظات اجرای درخواست. همان ‘today() - 1’. -## سانس {#timeslot} +## بازه زمانی {#timeslot} دور زمان به نیم ساعت. این تابع خاص به یاندکس است.متریکا, از نیم ساعت حداقل مقدار زمان برای شکستن یک جلسه به دو جلسه است اگر یک تگ ردیابی نشان می دهد تعداد صفحات متوالی یک کاربر که در زمان به شدت بیش از این مقدار متفاوت. این به این معنی است که تاپل (شناسه برچسب, شناسه کاربری, و شکاف زمان) را می توان مورد استفاده قرار گیرد به جستجو برای تعداد صفحات که در جلسه مربوطه شامل. @@ -315,7 +315,7 @@ SELECT toDate('2016-12-27') AS date, toYearWeek(date) AS yearWeek0, toYearWeek(d ## اطلاعات دقیق {#toyyyymmddhhmmss} -تبدیل یک تاریخ و یا تاریخ با گذشت زمان به یک uint64 تعداد شامل سال و ماه شماره (yyyy \* 10000000000 + mm \* 100000000 + dd \* 1000000 + hh \* 10000 + mm \* 100 + ss) است. +تبدیل یک تاریخ و یا تاریخ با گذشت زمان به یک UInt64 تعداد شامل سال و ماه شماره (YYYY \* 10000000000 + MM \* 100000000 + DD \* 1000000 + hh \* 10000 + mm \* 100 + ss) است. ## addYears, addMonths, addWeeks, addDays, addHours, addMinutes, addSeconds, addQuarters {#addyears-addmonths-addweeks-adddays-addhours-addminutes-addseconds-addquarters} @@ -439,12 +439,12 @@ Function formats a Time according given Format string. N.B.: Format is a constan | \# پ | هستم یا بعد از ظهر تعیین | PM | | %R | 24-ساعت ساعت ساعت: زمان میلی متر, معادل %ساعت: % متر | 22:33 | | %S | دوم (00-59) | 44 | -| % تی | شخصیت افقی تب (’) | | +| % تی | شخصیت افقی تب (') | | | %T | ایزو 8601 فرمت زمان (ساعت:میلی متر:اس اس), معادل %ساعت:%متر:%بازدید کنندگان | 22:33:44 | | \# تو | ایزو 8601 روز هفته به عنوان شماره با دوشنبه به عنوان 1 (1-7) | 2 | | %V | ایزو 8601 هفته شماره (01-53) | 01 | | \# وات | روز هفته به عنوان یک عدد اعشاری با یکشنبه به عنوان 0 (0-6) | 2 | -| \#… | سال گذشته دو رقم (00-99) | 18 | +| \#... | سال گذشته دو رقم (00-99) | 18 | | %Y | سال | 2018 | | %% | یک % نشانه | % | diff --git a/docs/fa/sql-reference/functions/encoding-functions.md b/docs/fa/sql-reference/functions/encoding-functions.md index 7a7fe8c221b..761026075c9 100644 --- a/docs/fa/sql-reference/functions/encoding-functions.md +++ b/docs/fa/sql-reference/functions/encoding-functions.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 52 toc_title: "\u06A9\u062F\u0628\u0646\u062F\u06CC" --- @@ -9,7 +9,7 @@ toc_title: "\u06A9\u062F\u0628\u0646\u062F\u06CC" ## کاراکتر {#char} -بازگرداندن رشته با طول به عنوان تعدادی از استدلال گذشت و هر بایت دارای ارزش استدلال مربوطه. می پذیرد استدلال های متعدد از انواع عددی. اگر ارزش بحث خارج از محدوده uint8 نوع داده آن است که تبدیل به uint8 با امکان گرد کردن و سرریز. +بازگرداندن رشته با طول به عنوان تعدادی از استدلال گذشت و هر بایت دارای ارزش استدلال مربوطه. می پذیرد استدلال های متعدد از انواع عددی. اگر ارزش بحث خارج از محدوده UInt8 نوع داده آن است که تبدیل به UInt8 با امکان گرد کردن و سرریز. **نحو** diff --git a/docs/fa/sql-reference/functions/ext-dict-functions.md b/docs/fa/sql-reference/functions/ext-dict-functions.md index f553eccad0a..204182cf3c1 100644 --- a/docs/fa/sql-reference/functions/ext-dict-functions.md +++ b/docs/fa/sql-reference/functions/ext-dict-functions.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 58 toc_title: "\u06A9\u0627\u0631 \u0628\u0627 \u0648\u0627\u0698\u0647\u0646\u0627\u0645\ \u0647\u0647\u0627 \u062E\u0627\u0631\u062C\u06CC" @@ -24,7 +24,7 @@ dictGetOrDefault('dict_name', 'attr_name', id_expr, default_value_expr) - `dict_name` — Name of the dictionary. [رشته تحت اللفظی](../syntax.md#syntax-string-literal). - `attr_name` — Name of the column of the dictionary. [رشته تحت اللفظی](../syntax.md#syntax-string-literal). - `id_expr` — Key value. [عبارت](../syntax.md#syntax-expressions) بازگشت یک [UInt64](../../sql-reference/data-types/int-uint.md) یا [تاپل](../../sql-reference/data-types/tuple.md)- نوع ارزش بسته به پیکربندی فرهنگ لغت . -- `default_value_expr` — Value returned if the dictionary doesn’t contain a row with the `id_expr` کلید [عبارت](../syntax.md#syntax-expressions) بازگشت ارزش در نوع داده پیکربندی شده برای `attr_name` صفت کردن. +- `default_value_expr` — Value returned if the dictionary doesn't contain a row with the `id_expr` کلید [عبارت](../syntax.md#syntax-expressions) بازگشت ارزش در نوع داده پیکربندی شده برای `attr_name` صفت کردن. **مقدار بازگشتی** @@ -96,7 +96,7 @@ LIMIT 3 └─────┴────────┘ ``` -**همچنین نگاه کنید** +**همچنین نگاه کنید به** - [واژهنامهها خارجی](../../sql-reference/dictionaries/external-dictionaries/external-dicts.md) @@ -190,7 +190,7 @@ dictGet[Type]OrDefault('dict_name', 'attr_name', id_expr, default_value_expr) - `dict_name` — Name of the dictionary. [رشته تحت اللفظی](../syntax.md#syntax-string-literal). - `attr_name` — Name of the column of the dictionary. [رشته تحت اللفظی](../syntax.md#syntax-string-literal). - `id_expr` — Key value. [عبارت](../syntax.md#syntax-expressions) بازگشت یک [UInt64](../../sql-reference/data-types/int-uint.md)- نوع ارزش. -- `default_value_expr` — Value which is returned if the dictionary doesn’t contain a row with the `id_expr` کلید [عبارت](../syntax.md#syntax-expressions) بازگشت یک مقدار در نوع داده پیکربندی شده برای `attr_name` صفت کردن. +- `default_value_expr` — Value which is returned if the dictionary doesn't contain a row with the `id_expr` کلید [عبارت](../syntax.md#syntax-expressions) بازگشت یک مقدار در نوع داده پیکربندی شده برای `attr_name` صفت کردن. **مقدار بازگشتی** diff --git a/docs/fa/sql-reference/functions/functions-for-nulls.md b/docs/fa/sql-reference/functions/functions-for-nulls.md index aafa60569cc..0415fc420d0 100644 --- a/docs/fa/sql-reference/functions/functions-for-nulls.md +++ b/docs/fa/sql-reference/functions/functions-for-nulls.md @@ -1,12 +1,12 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 63 toc_title: "\u06A9\u0627\u0631 \u0628\u0627 Nullable \u0627\u0633\u062A\u062F\u0644\ \u0627\u0644" --- -# توابع برای کار با nullable مصالح {#functions-for-working-with-nullable-aggregates} +# توابع برای کار با Nullable مصالح {#functions-for-working-with-nullable-aggregates} ## isNull {#isnull} diff --git a/docs/fa/sql-reference/functions/geo.md b/docs/fa/sql-reference/functions/geo.md index 04912539622..31d13caba11 100644 --- a/docs/fa/sql-reference/functions/geo.md +++ b/docs/fa/sql-reference/functions/geo.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 62 toc_title: "\u06A9\u0627\u0631 \u0628\u0627 \u0645\u062E\u062A\u0635\u0627\u062A \u062C\ \u063A\u0631\u0627\u0641\u06CC\u0627\u06CC\u06CC" @@ -62,7 +62,7 @@ pointInEllipses(x, y, x₀, y₀, a₀, b₀,...,xₙ, yₙ, aₙ, bₙ) **مقادیر بازگشتی** -`1` اگر نقطه در داخل است حداقل یکی از بیضی; `0`اگر این طور نیست. +`1` اگر نقطه در داخل حداقل یکی از بیضی; `0`اگر این طور نیست. **مثال** @@ -109,7 +109,7 @@ SELECT pointInPolygon((3., 3.), [(6, 0), (8, 4), (5, 8), (0, 2)]) AS res ## کد جغرافیایی {#geohashencode} -کد طول و عرض جغرافیایی به عنوان یک geohash-رشته مراجعه کنید (http://geohash.org/, https://en.wikipedia.org/wiki/geohash). +کد طول و عرض جغرافیایی به عنوان یک geohash-رشته مراجعه کنید (http://geohash.org/, https://en.wikipedia.org/wiki/Geohash). ``` sql geohashEncode(longitude, latitude, [precision]) @@ -223,7 +223,7 @@ SELECT geoToH3(37.79506683, 55.71290588, 15) as h3Index - مجموعه ای از رشته های دقت طولانی از زمینهاش جعبه پوشش منطقه فراهم, شما باید به ترتیب از اقلام تکیه نمی. - \[\]- مجموعه خالی اگر *کمینه* ارزش *عرض جغرافیایی* و *طول جغرافیایی* کمتر از متناظر نیست *حداکثر* ارزشهای خبری عبارتند از: -لطفا توجه داشته باشید که عملکرد یک استثنا را پرتاب می کند اگر مجموعه ای بیش از 10’000’000 باشد. +لطفا توجه داشته باشید که عملکرد یک استثنا را پرتاب می کند اگر مجموعه ای بیش از 10'000'000 باشد. **مثال** @@ -444,7 +444,7 @@ SELECT h3ToString(617420388352917503) as h3_string ## استراینگتوه3 {#stringtoh3} -تبدیل رشته به نمایندگی h3index (uint64) نمایندگی. +تبدیل رشته به نمایندگی H3Index (UInt64) نمایندگی. ``` sql stringToH3(index_str) diff --git a/docs/fa/sql-reference/functions/hash-functions.md b/docs/fa/sql-reference/functions/hash-functions.md index 7741cf20e4c..fae02d22950 100644 --- a/docs/fa/sql-reference/functions/hash-functions.md +++ b/docs/fa/sql-reference/functions/hash-functions.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 50 toc_title: "\u0647\u0634" --- @@ -235,7 +235,7 @@ A `Int32` نوع داده مقدار هش. **مثال** -درست پرس و جو با utf-16le کد گذاری رشته است. +درست پرس و جو با UTF-16LE کد گذاری رشته است. پرسوجو: @@ -313,8 +313,8 @@ SELECT metroHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00: ## مورد احترام {#jumpconsistenthash} -محاسبه jumpconsistenthash فرم uint64. -می پذیرد دو استدلال: یک کلید بین 64 نوع و تعداد سطل. بازده int32. +محاسبه JumpConsistentHash فرم UInt64. +می پذیرد دو استدلال: یک کلید بین 64 نوع و تعداد سطل. بازده Int32. برای کسب اطلاعات بیشتر به لینک مراجعه کنید: [مورد احترام](https://arxiv.org/pdf/1406.2294.pdf) ## سوفلش2\_32, سوفلشه2\_64 {#murmurhash2-32-murmurhash2-64} @@ -347,6 +347,44 @@ SELECT murmurHash2_64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23: └──────────────────────┴────────┘ ``` +## اطلاعات دقیق {#gccmurmurhash} + +محاسبه 64 بیتی [زمزمه 2](https://github.com/aappleby/smhasher) مقدار هش با استفاده از همان دانه هش به عنوان [شورای همکاری خلیج فارس](https://github.com/gcc-mirror/gcc/blob/41d6b10e96a1de98e90a7c0378437c3255814b16/libstdc%2B%2B-v3/include/bits/functional_hash.h#L191). این قابل حمل بین کلانگ و شورای همکاری خلیج فارس ایجاد شده است. + +**نحو** + +``` sql +gccMurmurHash(par1, ...); +``` + +**پارامترها** + +- `par1, ...` — A variable number of parameters that can be any of the [انواع داده های پشتیبانی شده](../../sql-reference/data-types/index.md#data_types). + +**مقدار بازگشتی** + +- محاسبه مقدار هش. + +نوع: [UInt64](../../sql-reference/data-types/int-uint.md). + +**مثال** + +پرسوجو: + +``` sql +SELECT + gccMurmurHash(1, 2, 3) AS res1, + gccMurmurHash(('a', [1, 2, 3], 4, (4, ['foo', 'bar'], 1, (1, 2)))) AS res2 +``` + +نتیجه: + +``` text +┌─────────────────res1─┬────────────────res2─┐ +│ 12384823029245979431 │ 1188926775431157506 │ +└──────────────────────┴─────────────────────┘ +``` + ## سوفلش3\_32, سوفلشه3\_64 {#murmurhash3-32-murmurhash3-64} تولید یک [سوفلهاش3](https://github.com/aappleby/smhasher) مقدار هش. diff --git a/docs/fa/sql-reference/functions/higher-order-functions.md b/docs/fa/sql-reference/functions/higher-order-functions.md index e90929668a1..ca0f56bbafe 100644 --- a/docs/fa/sql-reference/functions/higher-order-functions.md +++ b/docs/fa/sql-reference/functions/higher-order-functions.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 57 toc_title: "\u0628\u0627\u0644\u0627\u062A\u0631 \u0633\u0641\u0627\u0631\u0634" --- diff --git a/docs/fa/sql-reference/functions/in-functions.md b/docs/fa/sql-reference/functions/in-functions.md index 5beb072fad6..9c47b25135c 100644 --- a/docs/fa/sql-reference/functions/in-functions.md +++ b/docs/fa/sql-reference/functions/in-functions.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 60 toc_title: "\u0627\u062C\u0631\u0627\u06CC \u0627\u067E\u0631\u0627\u062A\u0648\u0631\ \ \u062F\u0631" @@ -10,7 +10,7 @@ toc_title: "\u0627\u062C\u0631\u0627\u06CC \u0627\u067E\u0631\u0627\u062A\u0648\ ## در notIn, globalIn, globalNotIn {#in-functions} -بخش را ببینید [در اپراتورها](../statements/select.md#select-in-operators). +بخش را ببینید [در اپراتورها](../operators/in.md#select-in-operators). ## tuple(x, y, …), operator (x, y, …) {#tuplex-y-operator-x-y} diff --git a/docs/fa/sql-reference/functions/index.md b/docs/fa/sql-reference/functions/index.md index 6293d0cc3c5..e6e6c162f2a 100644 --- a/docs/fa/sql-reference/functions/index.md +++ b/docs/fa/sql-reference/functions/index.md @@ -1,14 +1,14 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_folder_title: Functions +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "\u062A\u0648\u0627\u0628\u0639" toc_priority: 32 toc_title: "\u0645\u0639\u0631\u0641\u06CC \u0634\u0631\u06A9\u062A" --- # توابع {#functions} -حداقل\* دو نوع از توابع وجود دارد - توابع به طور منظم (فقط به نام “functions”) and aggregate functions. These are completely different concepts. Regular functions work as if they are applied to each row separately (for each row, the result of the function doesn’t depend on the other rows). Aggregate functions accumulate a set of values from various rows (i.e. they depend on the entire set of rows). +حداقل\* دو نوع از توابع وجود دارد - توابع به طور منظم (فقط به نام “functions”) and aggregate functions. These are completely different concepts. Regular functions work as if they are applied to each row separately (for each row, the result of the function doesn't depend on the other rows). Aggregate functions accumulate a set of values from various rows (i.e. they depend on the entire set of rows). در این بخش ما در مورد توابع به طور منظم. برای توابع کل, بخش را ببینید “Aggregate functions”. @@ -24,7 +24,7 @@ toc_title: "\u0645\u0639\u0631\u0641\u06CC \u0634\u0631\u06A9\u062A" ## انواع نتایج {#types-of-results} -همه توابع بازگشت بازگشت بازگشت تنها به عنوان نتیجه (چند ارزش نیست, و نه صفر ارزش). نوع نتیجه است که معمولا تنها با انواع استدلال تعریف, نه با ارزش. استثنا هستند tupleelement function (a.n اپراتور) و tofixedstring تابع. +همه توابع بازگشت بازگشت بازگشت تنها به عنوان نتیجه (چند ارزش نیست, و نه صفر ارزش). نوع نتیجه است که معمولا تنها با انواع استدلال تعریف, نه با ارزش. استثنا هستند tupleElement function (a.N اپراتور) و toFixedString تابع. ## ثابتها {#constants} @@ -44,7 +44,7 @@ toc_title: "\u0645\u0639\u0631\u0641\u06CC \u0634\u0631\u06A9\u062A" ## پایداری {#constancy} -Functions can’t change the values of their arguments – any changes are returned as the result. Thus, the result of calculating separate functions does not depend on the order in which the functions are written in the query. +Functions can't change the values of their arguments – any changes are returned as the result. Thus, the result of calculating separate functions does not depend on the order in which the functions are written in the query. ## خطا {#error-handling} diff --git a/docs/fa/sql-reference/functions/introspection.md b/docs/fa/sql-reference/functions/introspection.md index 26e1ebd781e..3f47fa77943 100644 --- a/docs/fa/sql-reference/functions/introspection.md +++ b/docs/fa/sql-reference/functions/introspection.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 65 toc_title: "\u062F\u0631\u0648\u0646 \u0646\u06AF\u0631\u06CC" --- @@ -24,7 +24,7 @@ toc_title: "\u062F\u0631\u0648\u0646 \u0646\u06AF\u0631\u06CC" ## افزودن مدخل جدید {#addresstoline} -تبدیل آدرس حافظه مجازی در داخل clickhouse فرایند سرور به نام فایل و شماره خط در clickhouse کد منبع. +تبدیل آدرس حافظه مجازی در داخل ClickHouse فرایند سرور به نام فایل و شماره خط در ClickHouse کد منبع. اگر شما استفاده از بسته های رسمی تاتر, شما نیاز به نصب `clickhouse-common-static-dbg` بسته @@ -42,7 +42,7 @@ addressToLine(address_of_binary_instruction) - نام فایل کد منبع و شماره خط در این فایل حد و مرز مشخصی توسط روده بزرگ. - For example, `/build/obj-x86_64-linux-gnu/../dbms/Common/ThreadPool.cpp:199`, where `199` is a line number. + For example, `/build/obj-x86_64-linux-gnu/../src/Common/ThreadPool.cpp:199`, where `199` is a line number. - نام یک باینری, اگر تابع می تواند اطلاعات اشکال زدایی پیدا کنید. @@ -87,7 +87,7 @@ SELECT addressToLine(94784076370703) \G ``` text Row 1: ────── -addressToLine(94784076370703): /build/obj-x86_64-linux-gnu/../dbms/Common/ThreadPool.cpp:199 +addressToLine(94784076370703): /build/obj-x86_64-linux-gnu/../src/Common/ThreadPool.cpp:199 ``` استفاده از تابع به ردیابی کل پشته: @@ -100,15 +100,15 @@ LIMIT 1 \G ``` -این [اررایماپ](higher-order-functions.md#higher_order_functions-array-map) تابع اجازه می دهد تا برای پردازش هر عنصر منحصر به فرد از `trace` تنظیم توسط `addressToLine` تابع. در نتیجه این پردازش می بینید در `trace_source_code_lines` ستون خروجی. +این [اررایماپ](higher-order-functions.md#higher_order_functions-array-map) تابع اجازه می دهد تا برای پردازش هر عنصر منحصر به فرد از `trace` تنظیم توسط `addressToLine` تابع. نتیجه این پردازش شما در دیدن `trace_source_code_lines` ستون خروجی. ``` text Row 1: ────── trace_source_code_lines: /lib/x86_64-linux-gnu/libpthread-2.27.so /usr/lib/debug/usr/bin/clickhouse -/build/obj-x86_64-linux-gnu/../dbms/Common/ThreadPool.cpp:199 -/build/obj-x86_64-linux-gnu/../dbms/Common/ThreadPool.h:155 +/build/obj-x86_64-linux-gnu/../src/Common/ThreadPool.cpp:199 +/build/obj-x86_64-linux-gnu/../src/Common/ThreadPool.h:155 /usr/include/c++/9/bits/atomic_base.h:551 /usr/lib/debug/usr/bin/clickhouse /lib/x86_64-linux-gnu/libpthread-2.27.so @@ -117,7 +117,7 @@ trace_source_code_lines: /lib/x86_64-linux-gnu/libpthread-2.27.so ## افزودن موقعیت {#addresstosymbol} -تبدیل آدرس حافظه مجازی در داخل clickhouse سرور روند به نمادی از clickhouse شی فایل های. +تبدیل آدرس حافظه مجازی در داخل ClickHouse سرور روند به نمادی از ClickHouse شی فایل های. **نحو** @@ -131,7 +131,7 @@ addressToSymbol(address_of_binary_instruction) **مقدار بازگشتی** -- نمادی از clickhouse شی فایل های. +- نماد از فایل های شی کلیکهاوس. - رشته خالی, اگر نشانی معتبر نیست. نوع: [رشته](../../sql-reference/data-types/string.md). @@ -259,7 +259,7 @@ query_id: 724028bf-f550-45aa-910d-2af6212b94ac trace: [94138803686098,94138815010911,94138815096522,94138815101224,94138815102091,94138814222988,94138806823642,94138814457211,94138806823642,94138814457211,94138806823642,94138806795179,94138806796144,94138753770094,94138753771646,94138753760572,94138852407232,140399185266395,140399178045583] ``` -این `trace` زمینه شامل ردیابی پشته در لحظه نمونه برداری. +این `trace` درست شامل ردیابی پشته در حال حاضر نمونه برداری. گرفتن نام تابع برای یک نشانی واحد: @@ -283,7 +283,7 @@ LIMIT 1 \G ``` -این [اررایماپ](higher-order-functions.md#higher_order_functions-array-map) تابع اجازه می دهد تا برای پردازش هر عنصر منحصر به فرد از `trace` تنظیم توسط `demangle` تابع. در نتیجه این پردازش می بینید در `trace_functions` ستون خروجی. +این [اررایماپ](higher-order-functions.md#higher_order_functions-array-map) تابع اجازه می دهد تا برای پردازش هر عنصر منحصر به فرد از `trace` تنظیم توسط `demangle` تابع. نتیجه این پردازش شما در دیدن `trace_functions` ستون خروجی. ``` text Row 1: diff --git a/docs/fa/sql-reference/functions/ip-address-functions.md b/docs/fa/sql-reference/functions/ip-address-functions.md index 3b19334af4b..12a43878fdf 100644 --- a/docs/fa/sql-reference/functions/ip-address-functions.md +++ b/docs/fa/sql-reference/functions/ip-address-functions.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 55 toc_title: "\u06A9\u0627\u0631 \u0628\u0627 \u0646\u0634\u0627\u0646\u06CC\u0647\u0627\ \u06CC \u0627\u06CC\u0646\u062A\u0631\u0646\u062A\u06CC" @@ -10,7 +10,7 @@ toc_title: "\u06A9\u0627\u0631 \u0628\u0627 \u0646\u0634\u0627\u0646\u06CC\u0647 ## اطلاعات دقیق) {#ipv4numtostringnum} -طول می کشد یک uint32 شماره. به عنوان یک نشانی اینترنتی 4 در اندی بزرگ تفسیر می کند. بازده یک رشته حاوی مربوطه آدرس ipv4 در قالب a. b. c. d (نقطه جدا کردن اعداد در شکل اعشاری). +طول می کشد یک UInt32 شماره. به عنوان یک نشانی اینترنتی 4 در اندی بزرگ تفسیر می کند. بازده یک رشته حاوی مربوطه آدرس IPv4 در قالب A. B. C. d (نقطه جدا کردن اعداد در شکل اعشاری). ## مدت 4 ساعت) {#ipv4stringtonums} @@ -18,7 +18,7 @@ toc_title: "\u06A9\u0627\u0631 \u0628\u0627 \u0646\u0634\u0627\u0646\u06CC\u0647 ## اطلاعات دقیق) {#ipv4numtostringclasscnum} -شبیه به ipv4numtostring اما با استفاده از \ به جای گذشته هشت تایی. +شبیه به IPv4NumToString اما با استفاده از \ به جای گذشته هشت تایی. مثال: @@ -135,9 +135,9 @@ SELECT IPv6NumToString(IPv4ToIPv6(IPv4StringToNum('192.168.0.1'))) AS addr └────────────────────┘ ``` -## cutIPv6(x bitsToCutForIPv6, bitsToCutForIPv4) {#cutipv6x-bitstocutforipv6-bitstocutforipv4} +## cutIPv6(x bytesToCutForIPv6, bytesToCutForIPv4) {#cutipv6x-bytestocutforipv6-bytestocutforipv4} -یک رشته ثابت(16) مقدار حاوی نشانی اینترنتی6 را در قالب باینری می پذیرد. بازگرداندن یک رشته حاوی نشانی از تعداد مشخصی از بیت در قالب متن حذف. به عنوان مثال: +یک رشته ثابت(16) مقدار حاوی نشانی اینترنتی6 را در قالب باینری می پذیرد. بازگرداندن یک رشته حاوی نشانی از تعداد مشخصی از بایت حذف شده در قالب متن. به عنوان مثال: ``` sql WITH @@ -156,7 +156,7 @@ SELECT ## IPv4CIDRToRange(ipv4, Cidr), {#ipv4cidrtorangeipv4-cidr} -قبول یک ipv4 و uint8 ارزش شامل [CIDR](https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing). یک تاپل را با دو لیگ4 حاوی محدوده پایین تر و محدوده بالاتر زیر شبکه باز کنید. +قبول یک IPv4 و UInt8 ارزش شامل [CIDR](https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing). یک تاپل را با دو لیگ4 حاوی محدوده پایین تر و محدوده بالاتر زیر شبکه باز کنید. ``` sql SELECT IPv4CIDRToRange(toIPv4('192.168.5.2'), 16) @@ -170,7 +170,7 @@ SELECT IPv4CIDRToRange(toIPv4('192.168.5.2'), 16) ## IPv6CIDRToRange(ipv6 Cidr), {#ipv6cidrtorangeipv6-cidr} -قبول یک ipv6 و uint8 ارزش حاوی cidr. یک تاپل را با دو ایپو6 حاوی محدوده پایین تر و محدوده بالاتر زیر شبکه باز کنید. +قبول یک IPv6 و UInt8 ارزش حاوی CIDR. یک تاپل را با دو ایپو6 حاوی محدوده پایین تر و محدوده بالاتر زیر شبکه باز کنید. ``` sql SELECT IPv6CIDRToRange(toIPv6('2001:0db8:0000:85a3:0000:0000:ac1f:8001'), 32); diff --git a/docs/fa/sql-reference/functions/json-functions.md b/docs/fa/sql-reference/functions/json-functions.md index ef06b6d599d..0f53a751989 100644 --- a/docs/fa/sql-reference/functions/json-functions.md +++ b/docs/fa/sql-reference/functions/json-functions.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 56 -toc_title: "\u06A9\u0627\u0631 \u0628\u0627 \u062C\u0627\u0646\u0633\u0648\u0646." +toc_title: "\u06A9\u0627\u0631 \u0628\u0627 \u062C\u0627\u0646\u0633\u0648\u0646" --- # توابع برای کار با جانسون {#functions-for-working-with-json} -در یاندکسمتریکا جیسون توسط کاربران به عنوان پارامترهای جلسه منتقل می شود. برخی از توابع خاص برای کار با این جانسون وجود دارد. (اگر چه در بسیاری از موارد jsons هستند علاوه بر این قبل از پردازش و در نتیجه ارزش ها قرار داده و در ستون جداگانه در خود پردازش فرمت.) همه این توابع در فرضیات قوی در مورد چه جانسون می تواند بر اساس, اما سعی می کنند به عنوان کوچک که ممکن است به کار انجام می شود. +در یاندکسمتریکا جیسون توسط کاربران به عنوان پارامترهای جلسه منتقل می شود. برخی از توابع خاص برای کار با این جانسون وجود دارد. (اگر چه در بسیاری از موارد JSONs هستند علاوه بر این قبل از پردازش و در نتیجه ارزش ها قرار داده و در ستون جداگانه در خود پردازش فرمت.) همه این توابع در فرضیات قوی در مورد چه جانسون می تواند بر اساس, اما سعی می کنند به عنوان کوچک که ممکن است به کار انجام می شود. مفروضات زیر ساخته شده است: @@ -26,7 +26,7 @@ toc_title: "\u06A9\u0627\u0631 \u0628\u0627 \u062C\u0627\u0646\u0633\u0648\u0646 ## ویزیتپرامستراکتینت (پارامز, نام) {#visitparamextractintparams-name} -همان int64. +همان Int64. ## اطلاعات دقیق) {#visitparamextractfloatparams-name} @@ -34,7 +34,7 @@ toc_title: "\u06A9\u0627\u0631 \u0628\u0627 \u062C\u0627\u0646\u0633\u0648\u0646 ## ویسیتپرامسترکتبولبولول (پارامز, نام) {#visitparamextractboolparams-name} -تجزیه واقعی / ارزش کاذب. نتیجه این است uint8. +تجزیه واقعی / ارزش کاذب. نتیجه این است UInt8. ## ویسیتپرمککتراو (پارامز, نام) {#visitparamextractrawparams-name} @@ -79,7 +79,7 @@ SELECT isValidJSON('not a json') = 0 اگر مقدار در سند جسون وجود داشته باشد, `1` برگردانده خواهد شد. -اگر این مقدار وجود ندارد, `0` برگردانده خواهد شد. +اگر مقدار وجود ندارد, `0` برگردانده خواهد شد. مثالها: @@ -125,7 +125,7 @@ SELECT JSONLength('{"a": "hello", "b": [-100, 200.0, 300]}') = 2 بازگشت به نوع یک مقدار جانسون. -اگر این مقدار وجود ندارد, `Null` برگردانده خواهد شد. +اگر مقدار وجود ندارد, `Null` برگردانده خواهد شد. مثالها: @@ -196,17 +196,17 @@ SELECT JSONExtract('{"day": 5}', 'day', 'Enum8(\'Sunday\' = 0, \'Monday\' = 1, \ ## JSONExtractKeysAndValues(json\[, indices\_or\_keys…\], Value\_type) {#jsonextractkeysandvaluesjson-indices-or-keys-value-type} -پارسه جفت کلید ارزش از یک جانسون که ارزش از نوع داده داده خانه رعیتی هستند. +تجزیه جفت کلید ارزش از یک جانسون که ارزش از نوع داده داده داده خانه عروسکی هستند. مثال: ``` sql -SELECT JSONExtractKeysAndValues('{"x": {"a": 5, "b": 7, "c": 11}}', 'x', 'Int8') = [('a',5),('b',7),('c',11)]; +SELECT JSONExtractKeysAndValues('{"x": {"a": 5, "b": 7, "c": 11}}', 'x', 'Int8') = [('a',5),('b',7),('c',11)] ``` ## JSONExtractRaw(json\[, indices\_or\_keys\]…) {#jsonextractrawjson-indices-or-keys} -بازگرداندن بخشی از جانسون. +بازگرداندن بخشی از جانسون به عنوان رشته نامحدود. اگر بخش وجود ندارد و یا دارای یک نوع اشتباه, یک رشته خالی بازگردانده خواهد شد. @@ -216,7 +216,7 @@ SELECT JSONExtractKeysAndValues('{"x": {"a": 5, "b": 7, "c": 11}}', 'x', 'Int8') SELECT JSONExtractRaw('{"a": "hello", "b": [-100, 200.0, 300]}', 'b') = '[-100, 200.0, 300]' ``` -## JSONExtractArrayRaw(json\[, indices\_or\_keys\]…) {#jsonextractarrayrawjson-indices-or-keys} +## JSONExtractArrayRaw(json\[, indices\_or\_keys…\]) {#jsonextractarrayrawjson-indices-or-keys} بازگرداندن مجموعه ای با عناصر از مجموعه جانسون,هر یک به عنوان رشته نامحدود نشان. @@ -228,4 +228,70 @@ SELECT JSONExtractRaw('{"a": "hello", "b": [-100, 200.0, 300]}', 'b') = '[-100, SELECT JSONExtractArrayRaw('{"a": "hello", "b": [-100, 200.0, "hello"]}', 'b') = ['-100', '200.0', '"hello"']' ``` +## در حال بارگذاری {#json-extract-keys-and-values-raw} + +عصاره داده های خام از یک شی جانسون. + +**نحو** + +``` sql +JSONExtractKeysAndValuesRaw(json[, p, a, t, h]) +``` + +**پارامترها** + +- `json` — [رشته](../data-types/string.md) با جانسون معتبر. +- `p, a, t, h` — Comma-separated indices or keys that specify the path to the inner field in a nested JSON object. Each argument can be either a [رشته](../data-types/string.md) برای دریافت این زمینه توسط کلید یا یک [عدد صحیح](../data-types/int-uint.md) برای دریافت میدان ازت هفتم (نمایه شده از 1 عدد صحیح منفی از پایان تعداد). اگر تنظیم نشده, طیف جانسون به عنوان شی سطح بالا تجزیه. پارامتر اختیاری. + +**مقادیر بازگشتی** + +- & حذف با `('key', 'value')` توپلس هر دو عضو تاپل رشته ها. +- مجموعه خالی اگر جسم درخواست شده وجود ندارد, یا جانسون ورودی نامعتبر است. + +نوع: [& حذف](../data-types/array.md)([تاپل](../data-types/tuple.md)([رشته](../data-types/string.md), [رشته](../data-types/string.md)). + +**مثالها** + +پرسوجو: + +``` sql +SELECT JSONExtractKeysAndValuesRaw('{"a": [-100, 200.0], "b":{"c": {"d": "hello", "f": "world"}}}') +``` + +نتیجه: + +``` text +┌─JSONExtractKeysAndValuesRaw('{"a": [-100, 200.0], "b":{"c": {"d": "hello", "f": "world"}}}')─┐ +│ [('a','[-100,200]'),('b','{"c":{"d":"hello","f":"world"}}')] │ +└──────────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +پرسوجو: + +``` sql +SELECT JSONExtractKeysAndValuesRaw('{"a": [-100, 200.0], "b":{"c": {"d": "hello", "f": "world"}}}', 'b') +``` + +نتیجه: + +``` text +┌─JSONExtractKeysAndValuesRaw('{"a": [-100, 200.0], "b":{"c": {"d": "hello", "f": "world"}}}', 'b')─┐ +│ [('c','{"d":"hello","f":"world"}')] │ +└───────────────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +پرسوجو: + +``` sql +SELECT JSONExtractKeysAndValuesRaw('{"a": [-100, 200.0], "b":{"c": {"d": "hello", "f": "world"}}}', -1, 'c') +``` + +نتیجه: + +``` text +┌─JSONExtractKeysAndValuesRaw('{"a": [-100, 200.0], "b":{"c": {"d": "hello", "f": "world"}}}', -1, 'c')─┐ +│ [('d','"hello"'),('f','"world"')] │ +└───────────────────────────────────────────────────────────────────────────────────────────────────────┘ +``` + [مقاله اصلی](https://clickhouse.tech/docs/en/query_language/functions/json_functions/) diff --git a/docs/fa/sql-reference/functions/logical-functions.md b/docs/fa/sql-reference/functions/logical-functions.md index 47a7de029f0..3e462ab98d1 100644 --- a/docs/fa/sql-reference/functions/logical-functions.md +++ b/docs/fa/sql-reference/functions/logical-functions.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 37 toc_title: "\u0645\u0646\u0637\u0642\u06CC" --- diff --git a/docs/fa/sql-reference/functions/machine-learning-functions.md b/docs/fa/sql-reference/functions/machine-learning-functions.md index 75790643570..6cb2b7d8e0b 100644 --- a/docs/fa/sql-reference/functions/machine-learning-functions.md +++ b/docs/fa/sql-reference/functions/machine-learning-functions.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 64 toc_title: "\u062A\u0648\u0627\u0628\u0639 \u06CC\u0627\u062F\u06AF\u06CC\u0631\u06CC\ \ \u0645\u0627\u0634\u06CC\u0646" diff --git a/docs/fa/sql-reference/functions/math-functions.md b/docs/fa/sql-reference/functions/math-functions.md index 87398f07c29..1adb18c6949 100644 --- a/docs/fa/sql-reference/functions/math-functions.md +++ b/docs/fa/sql-reference/functions/math-functions.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 44 toc_title: "\u0631\u06CC\u0627\u0636\u06CC" --- @@ -51,7 +51,7 @@ Returns a Float64 number that is close to the number π. ## عارف) {#erfx} -اگر ‘x’ is non-negative, then `erf(x / σ√2)` احتمال این که یک متغیر تصادفی داشتن یک توزیع نرمال با انحراف استاندارد است ‘σ’ طول می کشد ارزش است که از مقدار مورد انتظار توسط بیش از هم جدا ‘x’. +اگر ‘x’ سپس غیر منفی است `erf(x / σ√2)` احتمال این که یک متغیر تصادفی داشتن یک توزیع نرمال با انحراف استاندارد است ‘σ’ طول می کشد ارزش است که از مقدار مورد انتظار توسط بیش از هم جدا ‘x’. مثال (قانون سه سیگما): diff --git a/docs/fa/sql-reference/functions/other-functions.md b/docs/fa/sql-reference/functions/other-functions.md index b0d3e097271..d18fac245c5 100644 --- a/docs/fa/sql-reference/functions/other-functions.md +++ b/docs/fa/sql-reference/functions/other-functions.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 66 toc_title: "\u063A\u06CC\u0631\u0647" --- @@ -11,6 +11,63 @@ toc_title: "\u063A\u06CC\u0631\u0647" بازگرداندن یک رشته با نام میزبان که این تابع در انجام شد. برای پردازش توزیع شده, این نام میزبان سرور از راه دور است, اگر تابع بر روی یک سرور از راه دور انجام. +## گدماکرو {#getmacro} + +می شود یک مقدار به نام از [& کلاندارها](../../operations/server-configuration-parameters/settings.md#macros) بخش پیکربندی سرور. + +**نحو** + +``` sql +getMacro(name); +``` + +**پارامترها** + +- `name` — Name to retrieve from the `macros` بخش. [رشته](../../sql-reference/data-types/string.md#string). + +**مقدار بازگشتی** + +- ارزش ماکرو مشخص. + +نوع: [رشته](../../sql-reference/data-types/string.md). + +**مثال** + +به عنوان مثال `macros` بخش در فایل پیکربندی سرور: + +``` xml + + Value + +``` + +پرسوجو: + +``` sql +SELECT getMacro('test'); +``` + +نتیجه: + +``` text +┌─getMacro('test')─┐ +│ Value │ +└──────────────────┘ +``` + +یک راه جایگزین برای دریافت همان مقدار: + +``` sql +SELECT * FROM system.macros +WHERE macro = 'test'; +``` + +``` text +┌─macro─┬─substitution─┐ +│ test │ Value │ +└───────┴──────────────┘ +``` + ## FQDN {#fqdn} بازگرداندن نام دامنه به طور کامل واجد شرایط. @@ -152,7 +209,7 @@ SELECT visibleWidth(NULL) ## currentUser() {#other-function-currentuser} -بازده ورود به سایت از کاربر فعلی. ورود کاربر که آغاز پرس و جو بازگردانده خواهد شد در مورد distibuted پرس و جو. +بازگرداندن ورود کاربر فعلی. ورود کاربر, که پرس و جو شروع, خواهد شد در پرس و جو مورد رقیق بازگشت. ``` sql SELECT currentUser(); @@ -183,13 +240,82 @@ SELECT currentUser(); └───────────────┘ ``` +## & ایستانت {#is-constant} + +بررسی اینکه استدلال بیان ثابت است. + +A constant expression means an expression whose resulting value is known at the query analysis (i.e. before execution). For example, expressions over [literals](../syntax.md#literals) عبارات ثابت هستند. + +این تابع برای توسعه در نظر گرفته شده, اشکال زدایی و تظاهرات. + +**نحو** + +``` sql +isConstant(x) +``` + +**پارامترها** + +- `x` — Expression to check. + +**مقادیر بازگشتی** + +- `1` — `x` ثابت است. +- `0` — `x` غیر ثابت است. + +نوع: [UInt8](../data-types/int-uint.md). + +**مثالها** + +پرسوجو: + +``` sql +SELECT isConstant(x + 1) FROM (SELECT 43 AS x) +``` + +نتیجه: + +``` text +┌─isConstant(plus(x, 1))─┐ +│ 1 │ +└────────────────────────┘ +``` + +پرسوجو: + +``` sql +WITH 3.14 AS pi SELECT isConstant(cos(pi)) +``` + +نتیجه: + +``` text +┌─isConstant(cos(pi))─┐ +│ 1 │ +└─────────────────────┘ +``` + +پرسوجو: + +``` sql +SELECT isConstant(number) FROM numbers(1) +``` + +نتیجه: + +``` text +┌─isConstant(number)─┐ +│ 0 │ +└────────────────────┘ +``` + ## اطلاعات دقیق) {#isfinitex} -قبول float32 و float64 و بازده uint8 برابر با 1 اگر این استدلال بی نهایت است و نه یک نان در غیر این صورت 0 است. +قبول Float32 و Float64 و بازده UInt8 برابر با 1 اگر این استدلال بی نهایت است و نه یک نان در غیر این صورت 0 است. ## اطلاعات دقیق) {#isinfinitex} -قبول float32 و float64 و بازده uint8 برابر با 1 اگر این استدلال بی نهایت است در غیر این صورت 0 است. توجه داشته باشید که 0 برای نان بازگشت. +قبول Float32 و Float64 و بازده UInt8 برابر با 1 اگر این استدلال بی نهایت است در غیر این صورت 0 است. توجه داشته باشید که 0 برای نان بازگشت. ## اطلاعات دقیق {#ifnotfinite} @@ -225,7 +351,7 @@ SELECT currentUser(); ## اطلاعات دقیق) {#isnanx} -قبول float32 و float64 و بازده uint8 برابر با 1 اگر استدلال این است که یک نان در غیر این صورت 0 است. +قبول Float32 و Float64 و بازده UInt8 برابر با 1 اگر استدلال این است که یک نان در غیر این صورت 0 است. ## قابل تنظیم(\[‘hostname’\[, ‘username’\[, ‘password’\]\],\] ‘database’, ‘table’, ‘column’) {#hascolumnintablehostname-username-password-database-table-column} @@ -311,7 +437,7 @@ ORDER BY h ASC `T` و `U` می تواند عددی, رشته,یا تاریخ و یا انواع تاریخ ساعت. از کجا همان نامه نشان داده شده است (تی یا تو), برای انواع عددی این ممکن است تطبیق انواع, اما انواع که یک نوع رایج. -برای مثال استدلال می توانید نوع int64 در حالی که دوم آرایه(uint16) نوع. +برای مثال استدلال می توانید نوع Int64 در حالی که دوم آرایه(UInt16) نوع. اگر ‘x’ ارزش به یکی از عناصر در برابر است ‘array\_from’ مجموعه, این بازگرداندن عنصر موجود (که شماره همان) از ‘array\_to’ صف کردن. در غیر این صورت, باز می گردد ‘default’. اگر عناصر تطبیق های متعدد در وجود دارد ‘array\_from’ این یکی از مسابقات را برمی گرداند. @@ -445,7 +571,7 @@ neighbor(column, offset[, default_value]) **مقادیر بازگشتی** - مقدار برای `column` داخل `offset` فاصله از ردیف فعلی اگر `offset` ارزش خارج از مرزهای بلوک نیست. -- مقدار پیشفرض برای `column` اگر `offset` ارزش مرزهای بلوک خارج است. اگر `default_value` داده می شود و سپس از آن استفاده خواهد شد. +- مقدار پیشفرض برای `column` اگر `offset` ارزش مرزهای بلوک خارج است. اگر `default_value` داده می شود و سپس استفاده می شود. نوع: نوع بلوک های داده را تحت تاثیر قرار و یا نوع مقدار پیش فرض. @@ -612,7 +738,7 @@ WHERE diff != 1 ## هشدار داده می شود) {#macnumtostringnum} -قبول uint64 شماره. تفسیر به عنوان نشانی مک در اندی بزرگ. بازگرداندن یک رشته حاوی نشانی مک مربوطه را در قالب قلمی: ب: ر. ن:دکتر: ف.ا: ف. ف. (تعداد کولون جدا شده در فرم هگزادسیمال). +قبول UInt64 شماره. تفسیر به عنوان نشانی مک در اندی بزرگ. بازگرداندن یک رشته حاوی نشانی مک مربوطه را در قالب قلمی: ب: ر. ن:دکتر: ف.ا: ف. ف. (تعداد کولون جدا شده در فرم هگزادسیمال). ## MACStringToNum(s) {#macstringtonums} @@ -929,7 +1055,7 @@ SELECT formatReadableSize(filesystemCapacity()) AS "Capacity", toTypeName(filesy ## خرابی اجرا {#function-runningaccumulate} طول می کشد کشورهای تابع جمع و یک ستون با ارزش را برمی گرداند, در نتیجه تجمع این کشورها برای مجموعه ای از خطوط بلوک هستند, از اول به خط فعلی. -برای مثال طول می کشد state of aggregate function (به عنوان مثال runningaccumulate(uniqstate(userid))) و برای هر ردیف از بلوک بازگشت نتیجه از مجموع عملکرد در ادغام دولت قبلی تمام ردیف و ردیف جاری است. +برای مثال طول می کشد state of aggregate function (به عنوان مثال runningAccumulate(uniqState(UserID))) و برای هر ردیف از بلوک بازگشت نتیجه از مجموع عملکرد در ادغام دولت قبلی تمام ردیف و ردیف جاری است. بنابراین نتیجه عملکرد بستگی به پارتیشن داده ها به بلوک ها و به ترتیب داده ها در بلوک دارد. ## جوینت {#joinget} diff --git a/docs/fa/sql-reference/functions/random-functions.md b/docs/fa/sql-reference/functions/random-functions.md index fd98d228bc5..772d24885aa 100644 --- a/docs/fa/sql-reference/functions/random-functions.md +++ b/docs/fa/sql-reference/functions/random-functions.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 51 toc_title: "\u062A\u0648\u0644\u06CC\u062F \u0627\u0639\u062F\u0627\u062F \u0634\u0628\ \u0647 \u062A\u0635\u0627\u062F\u0641\u06CC" @@ -16,16 +16,51 @@ toc_title: "\u062A\u0648\u0644\u06CC\u062F \u0627\u0639\u062F\u0627\u062F \u0634 ## رند {#rand} -بازده یک شبه تصادفی uint32 شماره به طور مساوی توزیع شده در میان تمام uint32 از نوع اعداد است. -با استفاده از یک خطی congruential ژنراتور. +بازده یک شبه تصادفی UInt32 شماره به طور مساوی توزیع شده در میان تمام UInt32 از نوع اعداد است. +با استفاده از یک ژنراتور همخوان خطی. ## رند64 {#rand64} -بازده یک شبه تصادفی uint64 شماره به طور مساوی توزیع شده در میان تمام uint64 از نوع اعداد است. +بازده یک شبه تصادفی UInt64 شماره به طور مساوی توزیع شده در میان تمام UInt64 از نوع اعداد است. با استفاده از یک ژنراتور همخوان خطی. ## شرکت رندکونستانت {#randconstant} -بازگرداندن یک عدد اوینت32 شبه تصادفی, ارزش یکی برای بلوک های مختلف است. +تولید یک ستون ثابت با یک مقدار تصادفی. + +**نحو** + +``` sql +randConstant([x]) +``` + +**پارامترها** + +- `x` — [عبارت](../syntax.md#syntax-expressions) در نتیجه در هر یک از [انواع داده های پشتیبانی شده](../data-types/index.md#data_types). ارزش حاصل دور انداخته می شود, اما بیان خود را اگر برای دور زدن استفاده [رفع ضعف مشترک](index.md#common-subexpression-elimination) اگر تابع چندین بار در یک پرس و جو نامیده می شود. پارامتر اختیاری. + +**مقدار بازگشتی** + +- عدد شبه تصادفی. + +نوع: [UInt32](../data-types/int-uint.md). + +**مثال** + +پرسوجو: + +``` sql +SELECT rand(), rand(1), rand(number), randConstant(), randConstant(1), randConstant(number) +FROM numbers(3) +``` + +نتیجه: + +``` text +┌─────rand()─┬────rand(1)─┬─rand(number)─┬─randConstant()─┬─randConstant(1)─┬─randConstant(number)─┐ +│ 3047369878 │ 4132449925 │ 4044508545 │ 2740811946 │ 4229401477 │ 1924032898 │ +│ 2938880146 │ 1267722397 │ 4154983056 │ 2740811946 │ 4229401477 │ 1924032898 │ +│ 956619638 │ 4238287282 │ 1104342490 │ 2740811946 │ 4229401477 │ 1924032898 │ +└────────────┴────────────┴──────────────┴────────────────┴─────────────────┴──────────────────────┘ +``` [مقاله اصلی](https://clickhouse.tech/docs/en/query_language/functions/random_functions/) diff --git a/docs/fa/sql-reference/functions/rounding-functions.md b/docs/fa/sql-reference/functions/rounding-functions.md index 1ded1b41d88..a11125290e4 100644 --- a/docs/fa/sql-reference/functions/rounding-functions.md +++ b/docs/fa/sql-reference/functions/rounding-functions.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 45 toc_title: "\u06AF\u0631\u062F \u06A9\u0631\u062F\u0646" --- @@ -47,7 +47,7 @@ round(expression [, decimal_places]) **مقدار بازگشتی:** -گرد شماره از همان نوع به عنوان ورودی شماره. +تعداد گرد از همان نوع به عنوان شماره ورودی. ### مثالها {#examples} @@ -88,7 +88,7 @@ round(3.65, 1) = 3.6 **همچنین نگاه کنید به** -- [roundBankers](#roundbankers) +- [سرباز](#roundbankers) ## سرباز {#roundbankers} @@ -104,7 +104,7 @@ round(3.65, 1) = 3.6 با استفاده از گرد کردن بانکدار, شما می توانید اثر است که گرد کردن اعداد در نتایج حاصل از جمع و یا کم کردن این اعداد را کاهش می دهد. -برای مثال مجموع اعداد 1.5, 2.5, 3.5, 4.5 مختلف گرد: +مثلا, تعداد مجموع 1.5, 2.5, 3.5, 4.5 با گرد کردن متفاوت: - بدون گرد کردن: 1.5 + 2.5 + 3.5 + 4.5 = 12. - گرد کردن بانکدار: 2 + 2 + 4 + 4 = 12. diff --git a/docs/fa/sql-reference/functions/splitting-merging-functions.md b/docs/fa/sql-reference/functions/splitting-merging-functions.md index 8cd8b3c6c24..75046b72307 100644 --- a/docs/fa/sql-reference/functions/splitting-merging-functions.md +++ b/docs/fa/sql-reference/functions/splitting-merging-functions.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 47 toc_title: "\u062A\u0642\u0633\u06CC\u0645 \u0648 \u0627\u062F\u063A\u0627\u0645 \u0631\ \u0634\u062A\u0647 \u0647\u0627 \u0648 \u0627\u0631\u0631\u06CC\u0633" @@ -95,12 +95,12 @@ SELECT splitByString('', 'abcde') ## حذف میانبر در صفحه خانه\]) {#arraystringconcatarr-separator} -رشته های ذکر شده در مجموعه را با جداساز مطابقت می دهد.’جدا کننده’ پارامتر اختیاری است: یک رشته ثابت, مجموعه ای به یک رشته خالی به طور پیش فرض. +رشته های ذکر شده در مجموعه را با جداساز مطابقت می دهد.'جدا کننده' پارامتر اختیاری است: یک رشته ثابت, مجموعه ای به یک رشته خالی به طور پیش فرض. رشته را برمی گرداند. ## اطلاعات دقیق) {#alphatokenss} -انتخاب substrings متوالی بایت از محدوده a-z و a-z. بازگرداندن یک آرایه از substrings. +انتخاب substrings متوالی بایت از محدوده a-z و A-Z. بازگرداندن یک آرایه از substrings. **مثال** diff --git a/docs/fa/sql-reference/functions/string-functions.md b/docs/fa/sql-reference/functions/string-functions.md index 86a0b838497..cc69b6fce32 100644 --- a/docs/fa/sql-reference/functions/string-functions.md +++ b/docs/fa/sql-reference/functions/string-functions.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 40 toc_title: "\u06A9\u0627\u0631 \u0628\u0627 \u0631\u0634\u062A\u0647 \u0647\u0627" --- @@ -10,36 +10,36 @@ toc_title: "\u06A9\u0627\u0631 \u0628\u0627 \u0631\u0634\u062A\u0647 \u0647\u062 ## خالی {#empty} بازده 1 برای یک رشته خالی و یا 0 برای یک رشته غیر خالی. -نتیجه این نوع uint8. +نتیجه این نوع UInt8. یک رشته در نظر گرفته شده است غیر خالی اگر شامل حداقل یک بایت, حتی اگر این یک فضا یا یک بایت پوچ است. این تابع همچنین برای ارریس کار می کند. ## notEmpty {#notempty} بازده 0 برای یک رشته خالی یا 1 برای یک رشته غیر خالی. -نتیجه این نوع uint8. +نتیجه این نوع UInt8. این تابع همچنین برای ارریس کار می کند. ## طول {#length} بازگرداندن طول یک رشته در بایت (نه در شخصیت, و نه در نقاط کد). -نتیجه این نوع uint64. +نتیجه این نوع UInt64. این تابع همچنین برای ارریس کار می کند. ## طول 8 {#lengthutf8} بازگرداندن طول یک رشته در نقاط کد یونیکد (نه در شخصیت), فرض کنید که رشته شامل مجموعه ای از بایت است که متن کد گذاری شده را تشکیل می دهند. اگر این فرض ملاقات نکرده است, این گرداند برخی از نتیجه (این یک استثنا پرتاب نمی کند). -نتیجه این نوع uint64. +نتیجه این نوع UInt64. ## \_شروع مجدد {#char-length} بازگرداندن طول یک رشته در نقاط کد یونیکد (نه در شخصیت), فرض کنید که رشته شامل مجموعه ای از بایت است که متن کد گذاری شده را تشکیل می دهند. اگر این فرض ملاقات نکرده است, این گرداند برخی از نتیجه (این یک استثنا پرتاب نمی کند). -نتیجه این نوع uint64. +نتیجه این نوع UInt64. ## \_شخصیت شناسی {#character-length} بازگرداندن طول یک رشته در نقاط کد یونیکد (نه در شخصیت), فرض کنید که رشته شامل مجموعه ای از بایت است که متن کد گذاری شده را تشکیل می دهند. اگر این فرض ملاقات نکرده است, این گرداند برخی از نتیجه (این یک استثنا پرتاب نمی کند). -نتیجه این نوع uint64. +نتیجه این نوع UInt64. ## پایین تر {#lower} @@ -356,7 +356,7 @@ SELECT trim(BOTH ' ()' FROM '( Hello, world! )') ## trimLeft {#trimleft} -حذف تمام رخدادهای متوالی از فضای سفید مشترک (شخصیت اسکی 32) از ابتدای یک رشته. آن را نمی کند, حذف انواع دیگر از کاراکترهای فضای سفید (برگه بدون شکستن فضا و غیره.). +حذف تمام رخدادهای متوالی از فضای سفید مشترک (شخصیت اسکی 32) از ابتدای یک رشته. این کار انواع دیگر از شخصیت های فضای سفید را حذف کنید (باریکه, فضای بدون استراحت, و غیره.). **نحو** @@ -394,7 +394,7 @@ SELECT trimLeft(' Hello, world! ') ## trimRight {#trimright} -حذف همه متوالی تکرار مشترک فضای خالی (ascii شخصیت 32) از پایان یک رشته است. آن را نمی کند, حذف انواع دیگر از کاراکترهای فضای سفید (برگه بدون شکستن فضا و غیره.). +حذف تمام رخدادهای متوالی از فضای سفید مشترک (شخصیت اسکی 32) از پایان یک رشته. این کار انواع دیگر از شخصیت های فضای سفید را حذف کنید (باریکه, فضای بدون استراحت, و غیره.). **نحو** @@ -472,18 +472,18 @@ SELECT trimBoth(' Hello, world! ') بازگرداندن کنترلی از یک رشته, با استفاده از کروک-32-یی 802.3 چند جملهای و مقدار اولیه `0xffffffff` هشدار داده می شود -نتیجه این نوع uint32. +نتیجه این نوع UInt32. ## CRC32IEEE(s) {#crc32ieee} را برمی گرداند کنترل از یک رشته, با استفاده از کروم-32-یی 802.3 چند جملهای. -نتیجه این نوع uint32. +نتیجه این نوع UInt32. ## CRC64(s) {#crc64} -بازده crc64 کنترلی از یک رشته با استفاده از crc-64-ecma چند جملهای. +بازده CRC64 کنترلی از یک رشته با استفاده از CRC-64-ECMA چند جملهای. -نتیجه این نوع uint64. +نتیجه این نوع UInt64. [مقاله اصلی](https://clickhouse.tech/docs/en/query_language/functions/string_functions/) diff --git a/docs/fa/sql-reference/functions/string-replace-functions.md b/docs/fa/sql-reference/functions/string-replace-functions.md index ecc0593ff38..37551549ab5 100644 --- a/docs/fa/sql-reference/functions/string-replace-functions.md +++ b/docs/fa/sql-reference/functions/string-replace-functions.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 42 toc_title: "\u0628\u0631\u0627\u06CC \u062C\u0627\u06CC\u06AF\u0632\u06CC\u0646\u06CC\ \ \u062F\u0631 \u0631\u0634\u062A\u0647\u0647\u0627" @@ -88,7 +88,7 @@ SELECT replaceRegexpAll('Hello, World!', '^', 'here: ') AS res ## سرویس پرداخت درونبرنامهای پلی) {#regexpquotemetas} تابع می افزاید: یک بک اسلش قبل از برخی از شخصیت های از پیش تعریف شده در رشته. -نویسههای از پیش تعریفشده: ‘0’, ‘\\’, ‘\|’, ‘(’, ‘)’, ‘^’, ‘$’, ‘.’, ‘\[’, ‘\]’, ‘?’, ‘\*‘,’+‘,’{‘,’:‘,’-’. +نویسههای از پیش تعریفشده: ‘0’, ‘\\’, ‘\|’, ‘(’, ‘)’, ‘^’, ‘$’, ‘.’, ‘\[’, '\]', ‘?’, '\*‘,’+‘,’{‘,’:‘,’-'. این اجرای کمی از دوباره متفاوت2::پاسخ2:: نقل قول. این فرار صفر بایت به عنوان \\ 0 بجای 00 و فرار شخصیت تنها مورد نیاز. برای کسب اطلاعات بیشتر به لینک مراجعه کنید: [RE2](https://github.com/google/re2/blob/master/re2/re2.cc#L473) diff --git a/docs/fa/sql-reference/functions/string-search-functions.md b/docs/fa/sql-reference/functions/string-search-functions.md index a7bf2e1b311..af68dee0afa 100644 --- a/docs/fa/sql-reference/functions/string-search-functions.md +++ b/docs/fa/sql-reference/functions/string-search-functions.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 41 toc_title: "\u0628\u0631\u0627\u06CC \u062C\u0633\u062A\u062C\u0648\u06CC \u0631\u0634\ \u062A\u0647\u0647\u0627" @@ -279,13 +279,13 @@ SELECT multiSearchAllPositions('Hello, World!', ['hello', '!', 'world']) ## مقالههای جدید مرتبط با تحقیق این نویسنده1 سوزن2, …, needleنه\]) {#multisearchfirstindexhaystack-needle1-needle2-needlen} -بازگرداندن شاخص `i` (شروع از 1) سوزن چپ پیدا شده استمن … در رشته `haystack` و 0 در غیر این صورت. +بازگرداندن شاخص `i` (شروع از 1) سوزن چپ پیدا شده استمن ... در رشته `haystack` و 0 در غیر این صورت. برای یک جستجوی غیر حساس مورد و یا / و در توابع استفاده از فرمت جی تی اف 8 `multiSearchFirstIndexCaseInsensitive, multiSearchFirstIndexUTF8, multiSearchFirstIndexCaseInsensitiveUTF8`. ## مولسیرچانی (هیستک, \[سوزن1 سوزن2, …, needleنه\]) {#function-multisearchany} -بازده 1, اگر حداقل یک سوزن رشتهمن … مسابقات رشته `haystack` و 0 در غیر این صورت. +بازده 1, اگر حداقل یک سوزن رشتهمن ... مسابقات رشته `haystack` و 0 در غیر این صورت. برای یک جستجوی غیر حساس مورد و یا / و در توابع استفاده از فرمت جی تی اف 8 `multiSearchAnyCaseInsensitive, multiSearchAnyUTF8, multiSearchAnyCaseInsensitiveUTF8`. diff --git a/docs/fa/sql-reference/functions/type-conversion-functions.md b/docs/fa/sql-reference/functions/type-conversion-functions.md index 85f681d028e..353701818b8 100644 --- a/docs/fa/sql-reference/functions/type-conversion-functions.md +++ b/docs/fa/sql-reference/functions/type-conversion-functions.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 38 toc_title: "\u062A\u0628\u062F\u06CC\u0644 \u0646\u0648\u0639" --- @@ -239,14 +239,14 @@ SELECT toDecimal32OrZero(toString(-1.111), 2) AS val, toTypeName(val) هنگام تبدیل تاریخ به اعداد و یا بالعکس, تاریخ مربوط به تعداد روز از ابتدای عصر یونیکس. هنگام تبدیل تاریخ با بار به اعداد و یا بالعکس, تاریخ با زمان مربوط به تعداد ثانیه از ابتدای عصر یونیکس. -تاریخ و تاریخ-با-فرمت زمان برای todate/todatetime توابع تعریف شده به شرح زیر است: +تاریخ و تاریخ-با-فرمت زمان برای toDate/toDateTime توابع تعریف شده به شرح زیر است: ``` text YYYY-MM-DD YYYY-MM-DD hh:mm:ss ``` -به عنوان یک استثنا اگر تبدیل از uint32, int32, uint64 یا int64 عددی انواع, به, تاریخ, و اگر عدد بزرگتر یا مساوی به 65536 تعدادی را به عنوان تفسیر یک زمان یونیکس (و نه به عنوان تعداد روز) و گرد است به تاریخ. این اجازه می دهد تا پشتیبانی از وقوع مشترک نوشتن ‘toDate(unix\_timestamp)’ که در غیر این صورت یک خطا خواهد بود و نیاز به نوشتن بیشتر دست و پا گیر ‘toDate(toDateTime(unix\_timestamp))’. +به عنوان یک استثنا اگر تبدیل از UInt32, Int32, UInt64 یا Int64 عددی انواع, به, تاریخ, و اگر عدد بزرگتر یا مساوی به 65536 تعدادی را به عنوان تفسیر یک زمان یونیکس (و نه به عنوان تعداد روز) و گرد است به تاریخ. این اجازه می دهد تا پشتیبانی از وقوع مشترک نوشتن ‘toDate(unix\_timestamp)’ که در غیر این صورت یک خطا خواهد بود و نیاز به نوشتن بیشتر دست و پا گیر ‘toDate(toDateTime(unix\_timestamp))’. تبدیل بین تاریخ و تاریخ با زمان انجام شده است راه طبیعی: با اضافه کردن یک زمان خالی و یا حذف زمان. @@ -271,7 +271,7 @@ SELECT ## وضعیت زیستشناختی رکورد) {#tofixedstrings-n} تبدیل یک استدلال نوع رشته به یک رشته (نفر) نوع (یک رشته با طول ثابت نفر). نفر باید ثابت باشد. -اگر رشته دارای بایت کمتر از نفر, این است که با بایت پوچ به سمت راست منتقل. اگر رشته دارای بایت بیش از نفر, یک استثنا پرتاب می شود. +اگر رشته دارای بایت کمتر از نفر, این است که با بایت پوچ به سمت راست خالی. اگر رشته دارای بایت بیش از نفر, یک استثنا پرتاب می شود. ## در حال بارگذاری) {#tostringcuttozeros} @@ -319,7 +319,7 @@ SELECT toFixedString('foo\0bar', 8) AS s, toStringCutToZero(s) AS s_cut این تابع یک شماره یا تاریخ و یا تاریخ با زمان می پذیرد, و یک رشته ثابت حاوی بایت به نمایندگی از ارزش مربوطه را در سفارش میزبان را برمی گرداند (اندی کمی). بایت پوچ از پایان کاهش یافته است. مثلا, ارزش نوع اوینت32 255 رشته ثابت است که یک بایت طولانی است. -## قالب (ایکس تی) {#type_conversion_function-cast} +## بازیگران(x, T) {#type_conversion_function-cast} تبدیل ‘x’ به ‘t’ نوع داده. بازیگران نحو (ایکس به عنوان تی) نیز پشتیبانی می کند. @@ -340,7 +340,7 @@ SELECT └─────────────────────┴─────────────────────┴────────────┴─────────────────────┴───────────────────────────┘ ``` -تبدیل به fixedstring(n) تنها با این نسخهها کار برای استدلال از نوع string یا fixedstring(n). +تبدیل به FixedString(N) تنها با این نسخهها کار برای استدلال از نوع String یا FixedString(N). تبدیل نوع به [Nullable](../../sql-reference/data-types/nullable.md) و پشت پشتیبانی می شود. مثال: @@ -516,9 +516,9 @@ SELECT parseDateTimeBestEffort('10 20:19') └─────────────────────────────────────┘ ``` -**همچنین نگاه کنید** +**همچنین نگاه کنید به** -- \[ISO 8601 announcement by @xkcd\])https://xkcd.com/1179/) +- \[ایزو 8601 اطلاعیه توسط @xkcd\] (https://xkcd.com/1179/) - [RFC 1123](https://tools.ietf.org/html/rfc1123) - [toDate](#todate) - [toDateTime](#todatetime) diff --git a/docs/fa/sql-reference/functions/url-functions.md b/docs/fa/sql-reference/functions/url-functions.md index 5ef0274e234..054caee4c4c 100644 --- a/docs/fa/sql-reference/functions/url-functions.md +++ b/docs/fa/sql-reference/functions/url-functions.md @@ -1,9 +1,9 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 54 toc_title: "\u06A9\u0627\u0631 \u0628\u0627 \u0646\u0634\u0627\u0646\u06CC\u0647\u0627\ - \ \u06CC \u0627\u06CC\u0646\u062A\u0631\u0646\u062A\u06CC" + \u06CC \u0627\u06CC\u0646\u062A\u0631\u0646\u062A\u06CC" --- # توابع برای کار با نشانیهای اینترنتی {#functions-for-working-with-urls} @@ -73,7 +73,7 @@ SELECT domain('svn+ssh://some.svn-hosting.com:80/repo/trunk') ### توپولدومین {#topleveldomain} -عصاره این دامنه سطح بالا از یک url. +دامنه سطح بالا را از یک نشانی اینترنتی استخراج می کند. ``` sql topLevelDomain(url) @@ -183,7 +183,7 @@ SELECT decodeURLComponent('http://127.0.0.1:8123/?query=SELECT%201%3B') AS Decod └────────────────────────────────────────┘ ``` -## توابع که حذف بخشی از یک نشانی وب {#functions-that-remove-part-of-a-url} +## توابع حذف بخشی از نشانی وب {#functions-that-remove-part-of-a-url} اگر نشانی وب هیچ چیز مشابه ندارد, نشانی اینترنتی بدون تغییر باقی می ماند. diff --git a/docs/fa/sql-reference/functions/uuid-functions.md b/docs/fa/sql-reference/functions/uuid-functions.md index 7c51b0065f7..50f4d5e20f0 100644 --- a/docs/fa/sql-reference/functions/uuid-functions.md +++ b/docs/fa/sql-reference/functions/uuid-functions.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 53 toc_title: "\u06A9\u0627\u0631 \u0628\u0627 \u06CC\u0648\u06CC\u062F" --- @@ -49,7 +49,7 @@ toUUID(String) **مقدار بازگشتی** -این uuid نوع ارزش است. +مقدار نوع شناسه. **مثال طریقه استفاده** diff --git a/docs/fa/sql-reference/functions/ym-dict-functions.md b/docs/fa/sql-reference/functions/ym-dict-functions.md index 625b36ec83e..9e08ee16516 100644 --- a/docs/fa/sql-reference/functions/ym-dict-functions.md +++ b/docs/fa/sql-reference/functions/ym-dict-functions.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 59 toc_title: "\u06A9\u0627\u0631 \u0628\u0627 \u06CC\u0627\u0646\u062F\u06A9\u0633.\u0648\ \u0627\u0698\u0647\u0646\u0627\u0645\u0647\u0647\u0627 \u0645\u062A\u0631\u06CC\u06A9\ @@ -150,7 +150,7 @@ Accepts a UInt32 number – the region ID from the Yandex geobase. Returns an ar ### شناسه بسته:\]) {#regiontonameid-lang} -Accepts a UInt32 number – the region ID from the Yandex geobase. A string with the name of the language can be passed as a second argument. Supported languages are: ru, en, ua, uk, by, kz, tr. If the second argument is omitted, the language ‘ru’ is used. If the language is not supported, an exception is thrown. Returns a string – the name of the region in the corresponding language. If the region with the specified ID doesn’t exist, an empty string is returned. +Accepts a UInt32 number – the region ID from the Yandex geobase. A string with the name of the language can be passed as a second argument. Supported languages are: ru, en, ua, uk, by, kz, tr. If the second argument is omitted, the language ‘ru’ is used. If the language is not supported, an exception is thrown. Returns a string – the name of the region in the corresponding language. If the region with the specified ID doesn't exist, an empty string is returned. `ua` و `uk` هر دو به معنای اوکراین. diff --git a/docs/fa/sql-reference/index.md b/docs/fa/sql-reference/index.md index bcd48cf709d..99095c9e47b 100644 --- a/docs/fa/sql-reference/index.md +++ b/docs/fa/sql-reference/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_folder_title: SQL Reference +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "\u0645\u0631\u062C\u0639 \u0645\u0631\u0628\u0639" toc_hidden: true toc_priority: 28 toc_title: "\u0645\u062E\u0641\u06CC" @@ -9,10 +9,12 @@ toc_title: "\u0645\u062E\u0641\u06CC" # مرجع مربع {#sql-reference} -- [SELECT](statements/select.md) +تاتر از انواع زیر نمایش داده شد: + +- [SELECT](statements/select/index.md) - [INSERT INTO](statements/insert-into.md) - [CREATE](statements/create.md) - [ALTER](statements/alter.md#query_language_queries_alter) - [انواع دیگر نمایش داده شد](statements/misc.md) -[مقاله اصلی](https://clickhouse.tech/docs/en/query_language/) +[مقاله اصلی](https://clickhouse.tech/docs/en/sql-reference/) diff --git a/docs/fa/sql-reference/operators/in.md b/docs/fa/sql-reference/operators/in.md new file mode 100644 index 00000000000..779d92e0174 --- /dev/null +++ b/docs/fa/sql-reference/operators/in.md @@ -0,0 +1,204 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +--- + +### در اپراتورها {#select-in-operators} + +این `IN`, `NOT IN`, `GLOBAL IN` و `GLOBAL NOT IN` اپراتورها به طور جداگانه تحت پوشش, از قابلیت های خود را کاملا غنی است. + +سمت چپ اپراتور یا یک ستون یا یک تاپل است. + +مثالها: + +``` sql +SELECT UserID IN (123, 456) FROM ... +SELECT (CounterID, UserID) IN ((34, 123), (101500, 456)) FROM ... +``` + +اگر سمت چپ یک ستون است که در شاخص است, و در سمت راست مجموعه ای از ثابت است, سیستم با استفاده از شاخص برای پردازش پرس و جو. + +Don't list too many values explicitly (i.e. millions). If a data set is large, put it in a temporary table (for example, see the section “External data for query processing”), سپس با استفاده از یک خرده فروشی. + +سمت راست اپراتور می تواند مجموعه ای از عبارات ثابت, مجموعه ای از تاپل با عبارات ثابت (نشان داده شده در نمونه های بالا), و یا به نام یک جدول پایگاه داده و یا زیرخاکی در داخل پرانتز را انتخاب کنید. + +اگر سمت راست اپراتور نام یک جدول است (مثلا, `UserID IN users`), این معادل به خرده فروشی است `UserID IN (SELECT * FROM users)`. با استفاده از این هنگام کار با داده های خارجی است که همراه با پرس و جو ارسال می شود. مثلا, پرس و جو را می توان همراه با مجموعه ای از شناسه کاربر لود شده به ارسال ‘users’ جدول موقت, که باید فیلتر شود. + +اگر در سمت راست اپراتور یک نام جدول است که موتور مجموعه ای است (مجموعه داده های تهیه شده است که همیشه در رم), مجموعه داده خواهد شد بیش از دوباره برای هر پرس و جو ایجاد نمی. + +زیرخاکری ممکن است بیش از یک ستون برای فیلتر کردن تاپل را مشخص کنید. +مثال: + +``` sql +SELECT (CounterID, UserID) IN (SELECT CounterID, UserID FROM ...) FROM ... +``` + +ستون به سمت چپ و راست اپراتور در باید همان نوع. + +اپراتور در و زیرخاکی ممکن است در هر بخشی از پرس و جو رخ می دهد, از جمله در توابع کل و توابع لامبدا. +مثال: + +``` sql +SELECT + EventDate, + avg(UserID IN + ( + SELECT UserID + FROM test.hits + WHERE EventDate = toDate('2014-03-17') + )) AS ratio +FROM test.hits +GROUP BY EventDate +ORDER BY EventDate ASC +``` + +``` text +┌──EventDate─┬────ratio─┐ +│ 2014-03-17 │ 1 │ +│ 2014-03-18 │ 0.807696 │ +│ 2014-03-19 │ 0.755406 │ +│ 2014-03-20 │ 0.723218 │ +│ 2014-03-21 │ 0.697021 │ +│ 2014-03-22 │ 0.647851 │ +│ 2014-03-23 │ 0.648416 │ +└────────────┴──────────┘ +``` + +برای هر روز پس از مارس 17, تعداد درصد تعداد بازدید از صفحات ساخته شده توسط کاربران که سایت در مارس 17 بازدید. +یک خرده فروشی در بند در همیشه اجرا فقط یک بار بر روی یک سرور. هیچ زیرمجموعه وابسته وجود دارد. + +## پردازش پوچ {#null-processing-1} + +در طول پردازش درخواست, در اپراتور فرض می شود که در نتیجه یک عملیات با [NULL](../syntax.md#null-literal) همیشه برابر است با `0`, صرف نظر از اینکه `NULL` است در سمت راست یا چپ اپراتور. `NULL` ارزش ها در هر مجموعه داده شامل نمی شود, به یکدیگر مربوط نیست و نمی توان در مقایسه. + +در اینجا یک مثال با است `t_null` جدول: + +``` text +┌─x─┬────y─┐ +│ 1 │ ᴺᵁᴸᴸ │ +│ 2 │ 3 │ +└───┴──────┘ +``` + +در حال اجرا پرس و جو `SELECT x FROM t_null WHERE y IN (NULL,3)` به شما نتیجه زیر را می دهد: + +``` text +┌─x─┐ +│ 2 │ +└───┘ +``` + +شما می توانید ببینید که ردیف که `y = NULL` از نتایج پرس و جو پرتاب می شود. دلیل این است که تاتر نمی توانید تصمیم بگیرید که چه `NULL` در `(NULL,3)` تنظیم, بازگشت `0` به عنوان نتیجه عملیات و `SELECT` این سطر را از خروجی نهایی حذف می کند. + +``` sql +SELECT y IN (NULL, 3) +FROM t_null +``` + +``` text +┌─in(y, tuple(NULL, 3))─┐ +│ 0 │ +│ 1 │ +└───────────────────────┘ +``` + +## توزیع Subqueries {#select-distributed-subqueries} + +دو گزینه برای در بازدید کنندگان با کارخانه های فرعی وجود دارد (شبیه به می پیوندد): طبیعی `IN` / `JOIN` و `GLOBAL IN` / `GLOBAL JOIN`. در نحوه اجرا برای پردازش پرس و جو توزیع شده متفاوت است. + +!!! attention "توجه" + به یاد داشته باشید که الگوریتم های زیر توضیح داده شده ممکن است متفاوت بسته به کار [تنظیمات](../../operations/settings/settings.md) `distributed_product_mode` تنظیمات. + +هنگام استفاده از به طور منظم در, پرس و جو به سرور از راه دور ارسال, و هر یک از اجرا می شود کارخانه های فرعی در `IN` یا `JOIN` بند بند. + +هنگام استفاده از `GLOBAL IN` / `GLOBAL JOINs`, اول همه زیرمجموعه ها برای اجرا `GLOBAL IN` / `GLOBAL JOINs` و نتایج در جداول موقت جمع می شوند. سپس جداول موقت به هر سرور از راه دور ارسال, جایی که نمایش داده شد با استفاده از این داده های موقت اجرا. + +برای پرس و جو غیر توزیع, استفاده از به طور منظم `IN` / `JOIN`. + +مراقب باشید در هنگام استفاده از کارخانه های فرعی در `IN` / `JOIN` بند برای پردازش پرس و جو توزیع. + +بیایید نگاهی به برخی از نمونه. فرض کنید که هر سرور در خوشه طبیعی است **\_تمل**. هر سرور همچنین دارای یک **توزیع \_تماس** جدول با **توزیع شده** نوع, که به نظر می رسد در تمام سرور در خوشه. + +برای پرس و جو به **توزیع \_تماس** پرس و جو به تمام سرورهای راه دور ارسال می شود و با استفاده از **\_تمل**. + +برای مثال پرس و جو + +``` sql +SELECT uniq(UserID) FROM distributed_table +``` + +به تمام سرورهای راه دور ارسال می شود + +``` sql +SELECT uniq(UserID) FROM local_table +``` + +و به صورت موازی اجرا می شود تا زمانی که به مرحله ای برسد که نتایج متوسط می تواند ترکیب شود. سپس نتایج میانی به سرور درخواست کننده بازگردانده می شود و با هم ادغام می شوند و نتیجه نهایی به مشتری ارسال می شود. + +حالا اجازه دهید به بررسی یک پرس و جو با در: + +``` sql +SELECT uniq(UserID) FROM distributed_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM local_table WHERE CounterID = 34) +``` + +- محاسبه تقاطع مخاطبان از دو سایت. + +این پرس و جو خواهد شد به تمام سرور از راه دور به عنوان ارسال می شود + +``` sql +SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM local_table WHERE CounterID = 34) +``` + +به عبارت دیگر, داده های تعیین شده در بند در خواهد شد بر روی هر سرور به طور مستقل جمع, تنها در سراسر داده است که به صورت محلی بر روی هر یک از سرور های ذخیره شده. + +این به درستی و بهینه کار خواهد کرد اگر شما برای این مورد تهیه و داده ها در سراسر سرورهای خوشه گسترش یافته اند به طوری که داده ها را برای یک شناسه تنها ساکن به طور کامل بر روی یک سرور واحد. در این مورد, تمام اطلاعات لازم در دسترس خواهد بود به صورت محلی بر روی هر سرور. در غیر این صورت نتیجه نادرست خواهد بود. ما به این تنوع از پرس و جو به عنوان مراجعه کنید “local IN”. + +برای اصلاح چگونه پرس و جو کار می کند زمانی که داده ها به طور تصادفی در سراسر سرور خوشه گسترش, شما می توانید مشخص **توزیع \_تماس** در داخل یک خرده فروشی. پرس و جو شبیه به این خواهد بود: + +``` sql +SELECT uniq(UserID) FROM distributed_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM distributed_table WHERE CounterID = 34) +``` + +این پرس و جو خواهد شد به تمام سرور از راه دور به عنوان ارسال می شود + +``` sql +SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM distributed_table WHERE CounterID = 34) +``` + +خرده فروشی شروع خواهد شد در حال اجرا بر روی هر سرور از راه دور. پس از زیرخاکری با استفاده از یک جدول توزیع, خرده فروشی است که در هر سرور از راه دور خواهد شد به هر سرور از راه دور به عنوان خشمگین + +``` sql +SELECT UserID FROM local_table WHERE CounterID = 34 +``` + +مثلا, اگر شما یک خوشه از 100 سرور, اجرای کل پرس و جو نیاز 10,000 درخواست ابتدایی, که به طور کلی در نظر گرفته غیر قابل قبول. + +در چنین مواردی, شما همیشه باید جهانی به جای در استفاده از. بیایید نگاه کنیم که چگونه برای پرس و جو کار می کند + +``` sql +SELECT uniq(UserID) FROM distributed_table WHERE CounterID = 101500 AND UserID GLOBAL IN (SELECT UserID FROM distributed_table WHERE CounterID = 34) +``` + +سرور درخواست کننده خرده فروشی را اجرا خواهد کرد + +``` sql +SELECT UserID FROM distributed_table WHERE CounterID = 34 +``` + +و در نتیجه خواهد شد در یک جدول موقت در رم قرار داده است. سپس درخواست خواهد شد به هر سرور از راه دور به عنوان ارسال + +``` sql +SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID GLOBAL IN _data1 +``` + +و جدول موقت `_data1` خواهد شد به هر سرور از راه دور با پرس و جو ارسال (نام جدول موقت پیاده سازی تعریف شده است). + +این مطلوب تر از استفاده از نرمال در است. با این حال, نگه داشتن نکات زیر را در ذهن: + +1. هنگام ایجاد یک جدول موقت داده های منحصر به فرد ساخته شده است. برای کاهش حجم داده های منتقل شده بر روی شبکه مشخص متمایز در زیرخاکری. (شما لازم نیست برای انجام این کار برای عادی در.) +2. جدول موقت خواهد شد به تمام سرور از راه دور ارسال. انتقال برای توپولوژی شبکه به حساب نمی. مثلا, اگر 10 سرور از راه دور در یک مرکز داده است که در رابطه با سرور درخواست بسیار از راه دور اقامت, داده ها ارسال خواهد شد 10 بار بیش از کانال به مرکز داده از راه دور. سعی کنید برای جلوگیری از مجموعه داده های بزرگ در هنگام استفاده از جهانی در. +3. هنگام انتقال داده ها به سرور از راه دور, محدودیت در پهنای باند شبکه قابل تنظیم نیست. شما ممکن است شبکه بیش از حد. +4. سعی کنید برای توزیع داده ها در سراسر سرور به طوری که شما لازم نیست که به استفاده از جهانی را به صورت منظم. +5. اگر شما نیاز به استفاده از جهانی در اغلب, برنامه ریزی محل خوشه خانه کلیک به طوری که یک گروه واحد از کپی ساکن در بیش از یک مرکز داده با یک شبکه سریع بین, به طوری که یک پرس و جو را می توان به طور کامل در یک مرکز داده واحد پردازش. + +همچنین حس می کند برای مشخص کردن یک جدول محلی در `GLOBAL IN` بند, در صورتی که این جدول محلی تنها بر روی سرور درخواست در دسترس است و شما می خواهید به استفاده از داده ها را از روی سرور از راه دور. diff --git a/docs/fa/sql-reference/operators.md b/docs/fa/sql-reference/operators/index.md similarity index 76% rename from docs/fa/sql-reference/operators.md rename to docs/fa/sql-reference/operators/index.md index 3cec824e880..1d9d890c2be 100644 --- a/docs/fa/sql-reference/operators.md +++ b/docs/fa/sql-reference/operators/index.md @@ -1,14 +1,13 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 37 toc_title: "\u0627\u067E\u0631\u0627\u062A\u0648\u0631\u0647\u0627" --- # اپراتورها {#operators} -همه اپراتورها در حال تبدیل به توابع مربوط به خود را در پرس و جو و تجزیه مرحله مطابق با اولویت و associativity. -گروه اپراتورهای به ترتیب اولویت ذکر شده (بالاتر در لیست است, زودتر اپراتور به استدلال خود متصل). +ClickHouse تبدیل اپراتورها به توابع مربوط به خود را در پرس و جو و تجزیه مرحله با توجه به اولویت اولویت و associativity. ## اپراتورهای دسترسی {#access-operators} @@ -62,7 +61,7 @@ toc_title: "\u0627\u067E\u0631\u0627\u062A\u0648\u0631\u0647\u0627" ## اپراتورها برای کار با مجموعه داده ها {#operators-for-working-with-data-sets} -*ببینید [در اپراتورها](statements/select.md#select-in-operators).* +*ببینید [در اپراتورها](in.md).* `a IN ...` – The `in(a, b)` تابع. @@ -80,7 +79,7 @@ toc_title: "\u0627\u067E\u0631\u0627\u062A\u0648\u0631\u0647\u0627" EXTRACT(part FROM date); ``` -عصاره بخشی از یک تاریخ معین. مثلا, شما می توانید یک ماه از یک تاریخ معین بازیابی, یا یک ثانیه از یک زمان. +استخراج قطعات از یک تاریخ معین. مثلا, شما می توانید یک ماه از یک تاریخ معین بازیابی, یا یک ثانیه از یک زمان. این `part` پارامتر مشخص می کند که بخشی از تاریخ برای بازیابی. مقادیر زیر در دسترس هستند: @@ -93,7 +92,7 @@ EXTRACT(part FROM date); این `part` پارامتر غیر حساس به حروف است. -این `date` پارامتر تاریخ یا زمان پردازش را مشخص می کند. هر دو [تاریخ](../sql-reference/data-types/date.md) یا [DateTime](../sql-reference/data-types/datetime.md) نوع پشتیبانی می شود. +این `date` پارامتر تاریخ یا زمان پردازش را مشخص می کند. هر دو [تاریخ](../../sql-reference/data-types/date.md) یا [DateTime](../../sql-reference/data-types/datetime.md) نوع پشتیبانی می شود. مثالها: @@ -103,7 +102,7 @@ SELECT EXTRACT(MONTH FROM toDate('2017-06-15')); SELECT EXTRACT(YEAR FROM toDate('2017-06-15')); ``` -در مثال زیر ما ایجاد یک جدول و قرار دادن به آن یک مقدار با `DateTime` نوع. +در مثال زیر ما یک جدول ایجاد می کنیم و با ارزش وارد می کنیم `DateTime` نوع. ``` sql CREATE TABLE test.Orders @@ -140,7 +139,7 @@ FROM test.Orders; ### INTERVAL {#operator-interval} -ایجاد یک [فاصله](../sql-reference/data-types/special-data-types/interval.md)- ارزش نوع است که باید در عملیات ریاضی با استفاده [تاریخ](../sql-reference/data-types/date.md) و [DateTime](../sql-reference/data-types/datetime.md)- ارزش نوع . +ایجاد یک [فاصله](../../sql-reference/data-types/special-data-types/interval.md)- ارزش نوع است که باید در عملیات ریاضی با استفاده [تاریخ](../../sql-reference/data-types/date.md) و [DateTime](../../sql-reference/data-types/datetime.md)- ارزش نوع . انواع فواصل: - `SECOND` @@ -153,7 +152,7 @@ FROM test.Orders; - `YEAR` !!! warning "اخطار" - فواصل با انواع مختلف نمی تواند ترکیب شود. شما می توانید عبارات مانند استفاده کنید `INTERVAL 4 DAY 1 HOUR`. بیان فواصل در واحد است که کوچکتر یا مساوی کوچکترین واحد فاصله برای مثال `INTERVAL 25 HOUR`. شما می توانید عملیات تبعی مانند مثال زیر استفاده کنید. + فواصل با انواع مختلف نمی تواند ترکیب شود. شما می توانید عبارات مانند استفاده کنید `INTERVAL 4 DAY 1 HOUR`. مشخص فواصل در واحد است که کوچکتر یا برابر با کوچکترین واحد فاصله, مثلا, `INTERVAL 25 HOUR`. شما می توانید عملیات متوالی مانند مثال زیر استفاده کنید. مثال: @@ -167,10 +166,10 @@ SELECT now() AS current_date_time, current_date_time + INTERVAL 4 DAY + INTERVAL └─────────────────────┴────────────────────────────────────────────────────────┘ ``` -**همچنین نگاه کنید** +**همچنین نگاه کنید به** -- [فاصله](../sql-reference/data-types/special-data-types/interval.md) نوع داده -- [توینتروال](../sql-reference/functions/type-conversion-functions.md#function-tointerval) توابع تبدیل نوع +- [فاصله](../../sql-reference/data-types/special-data-types/interval.md) نوع داده +- [توینتروال](../../sql-reference/functions/type-conversion-functions.md#function-tointerval) توابع تبدیل نوع ## اپراتور نفی منطقی {#logical-negation-operator} @@ -190,7 +189,7 @@ SELECT now() AS current_date_time, current_date_time + INTERVAL 4 DAY + INTERVAL یادداشت: -اپراتور مشروط محاسبه ارزش ب و ج, سپس چک چه شرایط ملاقات کرده است, و سپس مقدار مربوطه را برمی گرداند. اگر `b` یا `C` یک [ارریجین()](../sql-reference/functions/array-join.md#functions_arrayjoin) عملکرد هر سطر خواهد بود تکرار صرف نظر از “a” شرط. +اپراتور مشروط محاسبه ارزش ب و ج, سپس چک چه شرایط ملاقات کرده است, و سپس مقدار مربوطه را برمی گرداند. اگر `b` یا `C` یک [ارریجین()](../../sql-reference/functions/array-join.md#functions_arrayjoin) تابع, هر ردیف خواهد شد بدون در نظر گرفتن تکرار “a” شرط. ## عبارت شرطی {#operator_case} @@ -216,7 +215,7 @@ END `x -> expr` – The `lambda(x, expr) function.` -اپراتورهای زیر یک اولویت ندارد, از براکت هستند: +اپراتورهای زیر اولویت ندارند زیرا براکت هستند: ## اپراتور ایجاد مجموعه {#array-creation-operator} @@ -239,7 +238,7 @@ END ### IS NULL {#operator-is-null} -- برای [Nullable](../sql-reference/data-types/nullable.md) مقادیر نوع `IS NULL` بازگشت اپراتور: +- برای [Nullable](../../sql-reference/data-types/nullable.md) مقادیر نوع `IS NULL` بازگشت اپراتور: - `1` اگر مقدار باشد `NULL`. - `0` وگرنه - برای ارزش های دیگر `IS NULL` اپراتور همیشه باز می گردد `0`. @@ -258,7 +257,7 @@ SELECT x+100 FROM t_null WHERE y IS NULL ### IS NOT NULL {#is-not-null} -- برای [Nullable](../sql-reference/data-types/nullable.md) مقادیر نوع `IS NOT NULL` بازگشت اپراتور: +- برای [Nullable](../../sql-reference/data-types/nullable.md) مقادیر نوع `IS NOT NULL` بازگشت اپراتور: - `0` اگر مقدار باشد `NULL`. - `1` وگرنه - برای ارزش های دیگر `IS NOT NULL` اپراتور همیشه باز می گردد `1`. diff --git a/docs/fa/sql-reference/statements/alter.md b/docs/fa/sql-reference/statements/alter.md index 5d4ac5004cf..6afc311f74e 100644 --- a/docs/fa/sql-reference/statements/alter.md +++ b/docs/fa/sql-reference/statements/alter.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 36 toc_title: ALTER --- @@ -26,7 +26,7 @@ ALTER TABLE [db].name [ON CLUSTER cluster] ADD|DROP|CLEAR|COMMENT|MODIFY COLUMN - [DROP COLUMN](#alter_drop-column) — Deletes the column. - [CLEAR COLUMN](#alter_clear-column) — Resets column values. - [COMMENT COLUMN](#alter_comment-column) — Adds a text comment to the column. -- [MODIFY COLUMN](#alter_modify-column) — Changes column’s type, default expression and TTL. +- [MODIFY COLUMN](#alter_modify-column) — Changes column's type, default expression and TTL. این اقدامات در زیر توضیح داده شده است. @@ -42,7 +42,7 @@ ADD COLUMN [IF NOT EXISTS] name [type] [default_expr] [codec] [AFTER name_after] اضافه کردن یک ستون فقط تغییر ساختار جدول, بدون انجام هر گونه اقدامات با داده. داده ها بر روی دیسک پس از ظاهر نمی شود `ALTER`. اگر داده ها برای یک ستون از دست رفته در هنگام خواندن از جدول, این است که در با مقادیر پیش فرض پر (با انجام عبارت پیش فرض اگر یکی وجود دارد, و یا با استفاده از صفر یا رشته های خالی). ستون بر روی دیسک به نظر می رسد پس از ادغام قطعات داده (دیدن [ادغام](../../engines/table-engines/mergetree-family/mergetree.md)). -این رویکرد به ما اجازه می دهد برای تکمیل `ALTER` پرس و جو فورا, بدون افزایش حجم داده های قدیمی. +این رویکرد اجازه می دهد تا ما را به تکمیل `ALTER` پرس و جو فورا, بدون افزایش حجم داده های قدیمی. مثال: @@ -206,10 +206,9 @@ ALTER TABLE [db].name DROP CONSTRAINT constraint_name; - [DETACH PARTITION](#alter_detach-partition) – Moves a partition to the `detached` دایرکتوری و فراموش کرده ام. - [DROP PARTITION](#alter_drop-partition) – Deletes a partition. - [ATTACH PART\|PARTITION](#alter_attach-partition) – Adds a part or partition from the `detached` دایرکتوری به جدول. -- [REPLACE PARTITION](#alter_replace-partition) - پارتیشن داده ها را از یک جدول به دیگری کپی می کند. - [ATTACH PARTITION FROM](#alter_attach-partition-from) – Copies the data partition from one table to another and adds. - [REPLACE PARTITION](#alter_replace-partition) - پارتیشن داده ها را از یک جدول به دیگری کپی می کند و جایگزین می شود. -- [MOVE PARTITION TO TABLE](#alter_move_to_table-partition) (\#تغییر\_موف\_ قابل تنظیم-پارتیشن) - پارتیشن داده را از یک جدول به دیگری حرکت دهید. +- [MOVE PARTITION TO TABLE](#alter_move_to_table-partition)(\#تغییر\_موف\_ قابل تنظیم-پارتیشن) - پارتیشن داده را از یک جدول به دیگری حرکت دهید. - [CLEAR COLUMN IN PARTITION](#alter_clear-column-partition) - بازنشانی ارزش یک ستون مشخص شده در یک پارتیشن. - [CLEAR INDEX IN PARTITION](#alter_clear-index-partition) - بازنشانی شاخص ثانویه مشخص شده در یک پارتیشن. - [FREEZE PARTITION](#alter_freeze-partition) – Creates a backup of a partition. @@ -484,7 +483,7 @@ ALTER TABLE [db.]table DELETE WHERE filter_expr ALTER TABLE [db.]table UPDATE column1 = expr1 [, ...] WHERE filter_expr ``` -این `filter_expr` باید از نوع باشد `UInt8`. این پرس و جو به روز رسانی مقادیر ستون مشخص شده به ارزش عبارات مربوطه در ردیف که `filter_expr` طول می کشد یک مقدار غیر صفر. ارزش ها به نوع ستون با استفاده از قالبی `CAST` اپراتور. به روز رسانی ستون هایی که در محاسبه اولیه استفاده می شود و یا کلید پارتیشن پشتیبانی نمی شود. +این `filter_expr` باید از نوع باشد `UInt8`. این پرس و جو به روز رسانی مقادیر ستون مشخص شده به ارزش عبارات مربوطه در ردیف که `filter_expr` طول می کشد یک مقدار غیر صفر. ارزش ها به نوع ستون با استفاده از قالبی `CAST` اپراتور به روز رسانی ستون هایی که در محاسبه اولیه استفاده می شود و یا کلید پارتیشن پشتیبانی نمی شود. ``` sql ALTER TABLE [db.]table MATERIALIZE INDEX name IN PARTITION partition_name @@ -502,4 +501,102 @@ ALTER TABLE [db.]table MATERIALIZE INDEX name IN PARTITION partition_name مطالب برای جهش به پایان رسید حذف نمی حق دور (تعداد نوشته های حفظ شده توسط تعیین `finished_mutations_to_keep` پارامتر موتور ذخیره سازی). نوشته جهش قدیمی تر حذف می شوند. +## ALTER USER {#alter-user-statement} + +تغییرات حساب کاربر کلیک. + +### نحو {#alter-user-syntax} + +``` sql +ALTER USER [IF EXISTS] name [ON CLUSTER cluster_name] + [RENAME TO new_name] + [IDENTIFIED [WITH {PLAINTEXT_PASSWORD|SHA256_PASSWORD|DOUBLE_SHA1_PASSWORD}] BY {'password'|'hash'}] + [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] + [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] +``` + +### توصیف {#alter-user-dscr} + +برای استفاده `ALTER USER` شما باید [ALTER USER](grant.md#grant-access-management) امتیاز. + +### مثالها {#alter-user-examples} + +تنظیم نقش های اعطا شده به عنوان پیش فرض: + +``` sql +ALTER USER user DEFAULT ROLE role1, role2 +``` + +اگر نقش قبلا به یک کاربر اعطا نمی, تاتر می اندازد یک استثنا. + +تنظیم تمام نقش های اعطا شده به طور پیش فرض: + +``` sql +ALTER USER user DEFAULT ROLE ALL +``` + +اگر یک نقش به یک کاربر در اینده اعطا خواهد شد به طور پیش فرض به طور خودکار تبدیل خواهد شد. + +تنظیم تمام نقش های اعطا شده به استثنای پیش فرض `role1` و `role2`: + +``` sql +ALTER USER user DEFAULT ROLE ALL EXCEPT role1, role2 +``` + +## ALTER ROLE {#alter-role-statement} + +نقش ها را تغییر می دهد. + +### نحو {#alter-role-syntax} + +``` sql +ALTER ROLE [IF EXISTS] name [ON CLUSTER cluster_name] + [RENAME TO new_name] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] +``` + +## ALTER ROW POLICY {#alter-row-policy-statement} + +سیاست تغییرات ردیف. + +### نحو {#alter-row-policy-syntax} + +``` sql +ALTER [ROW] POLICY [IF EXISTS] name [ON CLUSTER cluster_name] ON [database.]table + [RENAME TO new_name] + [AS {PERMISSIVE | RESTRICTIVE}] + [FOR SELECT] + [USING {condition | NONE}][,...] + [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] +``` + +## ALTER QUOTA {#alter-quota-statement} + +سهمیه تغییرات. + +### نحو {#alter-quota-syntax} + +``` sql +ALTER QUOTA [IF EXISTS] name [ON CLUSTER cluster_name] + [RENAME TO new_name] + [KEYED BY {'none' | 'user name' | 'ip address' | 'client key' | 'client key or user name' | 'client key or ip address'}] + [FOR [RANDOMIZED] INTERVAL number {SECOND | MINUTE | HOUR | DAY} + {MAX { {QUERIES | ERRORS | RESULT ROWS | RESULT BYTES | READ ROWS | READ BYTES | EXECUTION TIME} = number } [,...] | + NO LIMITS | TRACKING ONLY} [,...]] + [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] +``` + +## ALTER SETTINGS PROFILE {#alter-settings-profile-statement} + +سهمیه تغییرات. + +### نحو {#alter-settings-profile-syntax} + +``` sql +ALTER SETTINGS PROFILE [IF EXISTS] name [ON CLUSTER cluster_name] + [RENAME TO new_name] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...] +``` + [مقاله اصلی](https://clickhouse.tech/docs/en/query_language/alter/) diff --git a/docs/fa/sql-reference/statements/create.md b/docs/fa/sql-reference/statements/create.md index ee765ccd6be..f6217b8e87f 100644 --- a/docs/fa/sql-reference/statements/create.md +++ b/docs/fa/sql-reference/statements/create.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 35 toc_title: CREATE --- @@ -18,23 +18,19 @@ CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster] [ENGINE = engine(.. ### بند {#clauses} - `IF NOT EXISTS` + اگر `db_name` پایگاه داده از قبل وجود دارد, سپس کلیک هاوس می کند یک پایگاه داده جدید ایجاد کنید و: - If the `db_name` database already exists, then ClickHouse doesn't create a new database and: - - - Doesn't throw an exception if clause is specified. - - Throws an exception if clause isn't specified. + - اگر بند مشخص شده است یک استثنا پرتاب نمی کند. + - می اندازد یک استثنا اگر بند مشخص نشده است. - `ON CLUSTER` - - ClickHouse creates the `db_name` database on all the servers of a specified cluster. + کلیک ایجاد می کند `db_name` پایگاه داده در تمام سرورهای یک خوشه مشخص. - `ENGINE` - - [MySQL](../engines/database_engines/mysql.md) - - Allows you to retrieve data from the remote MySQL server. - - By default, ClickHouse uses its own [database engine](../engines/database_engines/index.md). + - [MySQL](../../engines/database-engines/mysql.md) + به شما اجازه می دهد برای بازیابی اطلاعات از سرور خروجی زیر از راه دور. + به طور پیش فرض, تاتر با استفاده از خود [موتور دادگان](../../engines/database-engines/index.md). ## CREATE TABLE {#create-table-query} @@ -49,7 +45,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] ) ENGINE = engine ``` -ایجاد یک جدول به نام ‘name’ در ‘db’ پایگاه داده یا پایگاه داده فعلی اگر ‘db’ تنظیم نشده است, با ساختار مشخص شده در براکت و ‘engine’ موتور. +ایجاد یک جدول به نام ‘name’ در ‘db’ پایگاه داده یا پایگاه داده فعلی اگر ‘db’ تنظیم نشده است, با ساختار مشخص شده در براکت و ‘engine’ موتور ساختار جدول یک لیست از توصیف ستون است. اگر شاخص ها توسط موتور پشتیبانی می شوند به عنوان پارامتر برای موتور جدول نشان داده می شوند. شرح ستون است `name type` در ساده ترین حالت. مثال: `RegionID UInt32`. @@ -75,7 +71,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name ENGINE = engine AS SELECT ... در همه موارد اگر `IF NOT EXISTS` مشخص شده است, پرس و جو یک خطا بازگشت نیست اگر جدول در حال حاضر وجود دارد. در این مورد پرس و جو هیچ کاری انجام نخواهد داد. -می تواند بند دیگر پس از وجود دارد `ENGINE` بند در پرس و جو. دیدن مستندات دقیق در مورد چگونگی ایجاد جدول در شرح [موتورهای جدول](../../engines/table-engines/index.md#table_engines). +می تواند بند دیگر پس از وجود دارد `ENGINE` بند در پرس و جو. مشاهده مستندات دقیق در مورد چگونگی ایجاد جداول در شرح [موتورهای جدول](../../engines/table-engines/index.md#table_engines). ### مقادیر پیشفرض {#create-default-values} @@ -86,9 +82,9 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name ENGINE = engine AS SELECT ... اگر عبارت پیش فرض تعریف شده است, نوع ستون اختیاری است. در صورتی که یک نوع به صراحت تعریف شده وجود ندارد, نوع بیان پیش فرض استفاده شده است. مثال: `EventDate DEFAULT toDate(EventTime)` – the ‘Date’ نوع خواهد شد برای استفاده ‘EventDate’ ستون. -اگر نوع داده ها و پیش فرض بیان تعریف شده به صراحت این را بیان خواهد شد بازیگران به نوع مشخص شده با استفاده از نوع مصاحبه توابع. مثال: `Hits UInt32 DEFAULT 0` معنی همان چیزی که به عنوان `Hits UInt32 DEFAULT toUInt32(0)`. +اگر نوع داده ها و بیان پیش فرض به صراحت تعریف, این عبارت خواهد شد به نوع مشخص با استفاده از توابع نوع ریخته گری بازیگران. مثال: `Hits UInt32 DEFAULT 0` معنی همان چیزی که به عنوان `Hits UInt32 DEFAULT toUInt32(0)`. -Default expressions may be defined as an arbitrary expression from table constants and columns. When creating and changing the table structure, it checks that expressions don’t contain loops. For INSERT, it checks that expressions are resolvable – that all columns they can be calculated from have been passed. +Default expressions may be defined as an arbitrary expression from table constants and columns. When creating and changing the table structure, it checks that expressions don't contain loops. For INSERT, it checks that expressions are resolvable – that all columns they can be calculated from have been passed. `DEFAULT expr` @@ -187,7 +183,7 @@ CREATE TABLE codec_example ENGINE = MergeTree() ``` -#### کدک های هدف مشترک {#create-query-common-purpose-codecs} +#### کدکهای هدف عمومی {#create-query-general-purpose-codecs} کدکها: @@ -202,7 +198,7 @@ ENGINE = MergeTree() تاتر از جداول موقت که دارای ویژگی های زیر: -- جداول موقت ناپدید می شوند هنگامی که جلسه به پایان می رسد از جمله اگر اتصال از دست داده است. +- جداول موقت ناپدید می شوند زمانی که جلسه به پایان می رسد, از جمله اگر اتصال از دست داده است. - جدول موقت با استفاده از موتور حافظه تنها. - دسی بل را نمی توان برای یک جدول موقت مشخص شده است. این است که در خارج از پایگاه داده ایجاد شده است. - غیر ممکن است برای ایجاد یک جدول موقت با پرس و جو توزیع دی ال در تمام سرورهای خوشه (با استفاده از `ON CLUSTER`): این جدول تنها در جلسه فعلی وجود دارد. @@ -304,6 +300,203 @@ LIFETIME([MIN val1] MAX val2) بسته به فرهنگ لغت [طرحبندی](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md) یک یا چند ویژگی را می توان به عنوان کلید فرهنگ لغت مشخص شده است. -برای کسب اطلاعات بیشتر, دیدن [واژهنامهها خارجی](../../sql-reference/dictionaries/external-dictionaries/external-dicts.md) بخش. +برای کسب اطلاعات بیشتر, دیدن [واژهنامهها خارجی](../dictionaries/external-dictionaries/external-dicts.md) بخش. + +## CREATE USER {#create-user-statement} + +ایجاد یک [حساب کاربری](../../operations/access-rights.md#user-account-management). + +### نحو {#create-user-syntax} + +``` sql +CREATE USER [IF NOT EXISTS | OR REPLACE] name [ON CLUSTER cluster_name] + [IDENTIFIED [WITH {NO_PASSWORD|PLAINTEXT_PASSWORD|SHA256_PASSWORD|SHA256_HASH|DOUBLE_SHA1_PASSWORD|DOUBLE_SHA1_HASH}] BY {'password'|'hash'}] + [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] + [DEFAULT ROLE role [,...]] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] +``` + +#### شناسایی {#identification} + +چندین راه برای شناسایی کاربر وجود دارد: + +- `IDENTIFIED WITH no_password` +- `IDENTIFIED WITH plaintext_password BY 'qwerty'` +- `IDENTIFIED WITH sha256_password BY 'qwerty'` یا `IDENTIFIED BY 'password'` +- `IDENTIFIED WITH sha256_hash BY 'hash'` +- `IDENTIFIED WITH double_sha1_password BY 'qwerty'` +- `IDENTIFIED WITH double_sha1_hash BY 'hash'` + +#### میزبان کاربر {#user-host} + +میزبان کاربر یک میزبان است که می تواند اتصال به سرور کلیک ایجاد شود. میزبان را می توان در `HOST` بخش پرس و جو به روش های زیر: + +- `HOST IP 'ip_address_or_subnetwork'` — User can connect to ClickHouse server only from the specified IP address or a [subnetwork](https://en.wikipedia.org/wiki/Subnetwork). مثالها: `HOST IP '192.168.0.0/16'`, `HOST IP '2001:DB8::/32'`. برای استفاده در تولید فقط مشخص کنید `HOST IP` عناصر (نشانی اینترنتی و ماسک خود را), از زمان استفاده از `host` و `host_regexp` ممکن است تاخیر اضافی شود. +- `HOST ANY` — User can connect from any location. This is default option. +- `HOST LOCAL` — User can connect only locally. +- `HOST NAME 'fqdn'` — User host can be specified as FQDN. For example, `HOST NAME 'mysite.com'`. +- `HOST NAME REGEXP 'regexp'` — You can use [pcre](http://www.pcre.org/) عبارات منظم در هنگام مشخص میزبان کاربر. به عنوان مثال, `HOST NAME REGEXP '.*\.mysite\.com'`. +- `HOST LIKE 'template'` — Allows you use the [LIKE](../functions/string-search-functions.md#function-like) اپراتور برای فیلتر کردن میزبان کاربر. به عنوان مثال, `HOST LIKE '%'` معادل است `HOST ANY`, `HOST LIKE '%.mysite.com'` فیلتر تمام میزبان در `mysite.com` دامنه. + +یکی دیگر از راه تعیین میزبان استفاده است `@` نحو با نام کاربر. مثالها: + +- `CREATE USER mira@'127.0.0.1'` — Equivalent to the `HOST IP` نحو. +- `CREATE USER mira@'localhost'` — Equivalent to the `HOST LOCAL` نحو. +- `CREATE USER mira@'192.168.%.%'` — Equivalent to the `HOST LIKE` نحو. + +!!! info "اخطار" + درمان کلیک `user_name@'address'` به عنوان یک نام کاربری به عنوان یک کل. بدین ترتیب, از لحاظ فنی شما می توانید چند کاربر با ایجاد `user_name` و سازه های مختلف پس از `@`. ما توصیه به انجام این کار. + +### مثالها {#create-user-examples} + +ایجاد حساب کاربری `mira` حفاظت شده توسط رمز عبور `qwerty`: + +``` sql +CREATE USER mira HOST IP '127.0.0.1' IDENTIFIED WITH sha256_password BY 'qwerty' +``` + +`mira` باید برنامه مشتری در میزبان که سرور کلیک اجرا می شود شروع می شود. + +ایجاد حساب کاربری `john` نقش ها را به طور پیش فرض اختصاص دهید و این نقش ها را پیش فرض کنید: + +``` sql +CREATE USER john DEFAULT ROLE role1, role2 +``` + +ایجاد حساب کاربری `john` و همه نقش های بعدی خود را به طور پیش فرض: + +``` sql +ALTER USER user DEFAULT ROLE ALL +``` + +هنگامی که برخی از نقش اختصاص داده خواهد شد به `john` در اینده این تبدیل خواهد شد به طور پیش فرض به طور خودکار. + +ایجاد حساب کاربری `john` و همه نقش های بعدی خود را به استثنای پیش فرض `role1` و `role2`: + +``` sql +ALTER USER john DEFAULT ROLE ALL EXCEPT role1, role2 +``` + +## CREATE ROLE {#create-role-statement} + +ایجاد یک [نقش](../../operations/access-rights.md#role-management). + +### نحو {#create-role-syntax} + +``` sql +CREATE ROLE [IF NOT EXISTS | OR REPLACE] name + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] +``` + +### توصیف {#create-role-description} + +نقش مجموعه ای از [امتیازات](grant.md#grant-privileges). کاربر اعطا شده با نقش می شود تمام امتیازات این نقش. + +کاربر را می توان با نقش های متعدد اختصاص داده است. کاربران می توانند نقش اعطا خود را در ترکیب دلخواه توسط اعمال [SET ROLE](misc.md#set-role-statement) بیانیه. دامنه نهایی امتیازات مجموعه ای ترکیبی از تمام امتیازات تمام نقش های کاربردی است. اگر یک کاربر امتیازات اعطا شده به طور مستقیم به حساب کاربری نیز با امتیازات اعطا شده توسط نقش ترکیب شده است. + +کاربر می تواند نقش پیش فرض که در ورود کاربر اعمال می شود. برای تنظیم نقش های پیش فرض از [SET DEFAULT ROLE](misc.md#set-default-role-statement) بیانیه یا [ALTER USER](alter.md#alter-user-statement) بیانیه. + +برای لغو نقش از [REVOKE](revoke.md) بیانیه. + +برای حذف نقش از [DROP ROLE](misc.md#drop-role-statement) بیانیه. نقش حذف شده به طور خودکار از تمام کاربران و نقش هایی که اعطا شد لغو می شود. + +### مثالها {#create-role-examples} + +``` sql +CREATE ROLE accountant; +GRANT SELECT ON db.* TO accountant; +``` + +این دنباله ای از نمایش داده شد نقش ایجاد می کند `accountant` که دارای امتیاز از خواندن داده ها از `accounting` بانک اطلاعات. + +اعطای نقش به کاربر `mira`: + +``` sql +GRANT accountant TO mira; +``` + +کاربر پس از این نقش اعطا می تواند استفاده و انجام نمایش داده شد مجاز است. به عنوان مثال: + +``` sql +SET ROLE accountant; +SELECT * FROM db.*; +``` + +## CREATE ROW POLICY {#create-row-policy-statement} + +ایجاد یک [پالایه برای سطرها](../../operations/access-rights.md#row-policy-management), که یک کاربر می تواند از یک جدول به عنوان خوانده شده. + +### نحو {#create-row-policy-syntax} + +``` sql +CREATE [ROW] POLICY [IF NOT EXISTS | OR REPLACE] policy_name [ON CLUSTER cluster_name] ON [db.]table + [AS {PERMISSIVE | RESTRICTIVE}] + [FOR SELECT] + [USING condition] + [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] +``` + +#### بخش به عنوان {#create-row-policy-as} + +با استفاده از این بخش شما می توانید سیاست های مجاز و یا محدود ایجاد کنید. + +سیاست مجاز دسترسی به ردیف کمک های مالی. سیاست های مجاز که به همان جدول اعمال می شود با هم با استفاده از بولی ترکیب `OR` اپراتور سیاست های مجاز به طور پیش فرض. + +سیاست محدود کننده دسترسی به ردیف محدود می کند. سیاست های محدود که به همان جدول اعمال می شود با هم با استفاده از بولی ترکیب شده است `AND` اپراتور + +سیاست های محدود به ردیف که فیلترهای مجاز گذشت اعمال می شود. اگر شما در تنظیم سیاست های محدود اما هیچ سیاست مجاز کاربر هر سطر از جدول می توانید. + +#### بخش به {#create-row-policy-to} + +در بخش `TO` شما می توانید یک لیست ترکیبی از نقش ها و کاربران را, مثلا, `CREATE ROW POLICY ... TO accountant, john@localhost`. + +کلیدواژه `ALL` به معنی تمام کاربران کلیک از جمله کاربر فعلی. کلیدواژهها `ALL EXCEPT` اجازه می دهد به حذف برخی از کاربران از لیست تمام کاربران به عنوان مثال `CREATE ROW POLICY ... TO ALL EXCEPT accountant, john@localhost` + +### مثالها {#examples} + +- `CREATE ROW POLICY filter ON mydb.mytable FOR SELECT USING a<1000 TO accountant, john@localhost` +- `CREATE ROW POLICY filter ON mydb.mytable FOR SELECT USING a<1000 TO ALL EXCEPT mira` + +## CREATE QUOTA {#create-quota-statement} + +ایجاد یک [سهمیه](../../operations/access-rights.md#quotas-management) این را می توان به یک کاربر یا نقش اختصاص داد. + +### نحو {#create-quota-syntax} + +``` sql +CREATE QUOTA [IF NOT EXISTS | OR REPLACE] name [ON CLUSTER cluster_name] + [KEYED BY {'none' | 'user name' | 'ip address' | 'client key' | 'client key or user name' | 'client key or ip address'}] + [FOR [RANDOMIZED] INTERVAL number {SECOND | MINUTE | HOUR | DAY} + {MAX { {QUERIES | ERRORS | RESULT ROWS | RESULT BYTES | READ ROWS | READ BYTES | EXECUTION TIME} = number } [,...] | + NO LIMITS | TRACKING ONLY} [,...]] + [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] +``` + +### مثال {#create-quota-example} + +محدود کردن حداکثر تعداد نمایش داده شد برای کاربر فعلی با 123 نمایش داده شد در 15 ماه محدودیت: + +``` sql +CREATE QUOTA qA FOR INTERVAL 15 MONTH MAX QUERIES 123 TO CURRENT_USER +``` + +## CREATE SETTINGS PROFILE {#create-settings-profile-statement} + +ایجاد یک [تنظیمات](../../operations/access-rights.md#settings-profiles-management) این را می توان به یک کاربر یا نقش اختصاص داد. + +### نحو {#create-settings-profile-syntax} + +``` sql +CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] name [ON CLUSTER cluster_name] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...] +``` + +# مثال {#create-settings-profile-syntax} + +ایجاد `max_memory_usage_profile` مشخصات تنظیمات با ارزش و محدودیت برای `max_memory_usage` تنظیمات. انتساب به `robin`: + +``` sql +CREATE SETTINGS PROFILE max_memory_usage_profile SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000 TO robin +``` [مقاله اصلی](https://clickhouse.tech/docs/en/query_language/create/) diff --git a/docs/fa/sql-reference/statements/grant.md b/docs/fa/sql-reference/statements/grant.md new file mode 100644 index 00000000000..fd0274ea892 --- /dev/null +++ b/docs/fa/sql-reference/statements/grant.md @@ -0,0 +1,476 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_priority: 39 +toc_title: GRANT +--- + +# GRANT {#grant} + +- کمک های مالی [امتیازات](#grant-privileges) به حساب کاربر کلیک و یا نقش. +- اختصاص نقش به حساب های کاربری و یا به نقش های دیگر. + +برای لغو امتیازات از [REVOKE](revoke.md) بیانیه. همچنین شما می توانید لیست امتیازات اعطا شده توسط [SHOW GRANTS](show.md#show-grants-statement) بیانیه. + +## اعطای نحو امتیاز {#grant-privigele-syntax} + +``` sql +GRANT [ON CLUSTER cluster_name] privilege[(column_name [,...])] [,...] ON {db.table|db.*|*.*|table|*} TO {user | role | CURRENT_USER} [,...] [WITH GRANT OPTION] +``` + +- `privilege` — Type of privilege. +- `role` — ClickHouse user role. +- `user` — ClickHouse user account. + +این `WITH GRANT OPTION` کمک های مالی بند `user` یا `role` با اجازه به انجام `GRANT` پرس و جو. کاربران می توانند امتیازات از همان دامنه دارند و کمتر اعطا. + +## اعطای نحو نقش {#assign-role-syntax} + +``` sql +GRANT [ON CLUSTER cluster_name] role [,...] TO {user | another_role | CURRENT_USER} [,...] [WITH ADMIN OPTION] +``` + +- `role` — ClickHouse user role. +- `user` — ClickHouse user account. + +این `WITH ADMIN OPTION` مجموعه بند [ADMIN OPTION](#admin-option-privilege) امتیاز برای `user` یا `role`. + +## استفاده {#grant-usage} + +برای استفاده `GRANT` حساب شما باید `GRANT OPTION` امتیاز. شما می توانید امتیازات تنها در داخل دامنه امتیازات حساب خود را اعطا. + +مثلا, مدیر امتیازات به اعطا کرده است `john` حساب توسط پرس و جو: + +``` sql +GRANT SELECT(x,y) ON db.table TO john WITH GRANT OPTION +``` + +این بدان معنی است که `john` دارای مجوز برای انجام: + +- `SELECT x,y FROM db.table`. +- `SELECT x FROM db.table`. +- `SELECT y FROM db.table`. + +`john` نمی تواند انجام دهد `SELECT z FROM db.table`. این `SELECT * FROM db.table` همچنین در دسترس نیست. پردازش این پرس و جو, تاتر هیچ داده بازگشت نیست, حتی `x` و `y`. تنها استثنا است اگر یک جدول شامل تنها `x` و `y` ستون, در این مورد کلیکهاوس را برمی گرداند تمام داده ها. + +همچنین `john` دارد `GRANT OPTION` امتیاز, بنابراین می تواند کاربران دیگر با امتیازات از همان و یا دامنه کوچکتر اعطا. + +مشخص امتیازات شما می توانید ستاره استفاده کنید (`*`) به جای یک جدول و یا یک نام پایگاه داده. برای مثال `GRANT SELECT ON db.* TO john` پرسوجو اجازه میدهد `john` برای انجام `SELECT` پرس و جو بیش از همه جداول در `db` بانک اطلاعات. همچنین می توانید نام پایگاه داده را حذف کنید. برای مثال در این مورد امتیازات برای پایگاه داده فعلی اعطا می شود: `GRANT SELECT ON * TO john` اعطا امتیاز در تمام جداول در پایگاه داده فعلی, `GRANT SELECT ON mytable TO john` اعطا امتیاز در `mytable` جدول در پایگاه داده فعلی. + +دسترسی به `system` پایگاه داده همیشه مجاز (از این پایگاه داده برای پردازش نمایش داده شد استفاده می شود). + +شما می توانید امتیازات متعدد به حساب های متعدد در یک پرس و جو عطا کند. پرسوجو `GRANT SELECT, INSERT ON *.* TO john, robin` اجازه می دهد تا حساب `john` و `robin` برای انجام `INSERT` و `SELECT` نمایش داده شد بیش از همه جداول در تمام پایگاه داده بر روی سرور. + +## امتیازات {#grant-privileges} + +امتیاز اجازه به انجام نوع خاصی از نمایش داده شد است. + +امتیازات یک ساختار سلسله مراتبی. مجموعه ای از نمایش داده شد مجاز بستگی به دامنه امتیاز. + +سلسله مراتب امتیازات: + +- [SELECT](#grant-select) +- [INSERT](#grant-insert) +- [ALTER](#grant-alter) + - `ALTER TABLE` + - `ALTER UPDATE` + - `ALTER DELETE` + - `ALTER COLUMN` + - `ALTER ADD COLUMN` + - `ALTER DROP COLUMN` + - `ALTER MODIFY COLUMN` + - `ALTER COMMENT COLUMN` + - `ALTER CLEAR COLUMN` + - `ALTER RENAME COLUMN` + - `ALTER INDEX` + - `ALTER ORDER BY` + - `ALTER ADD INDEX` + - `ALTER DROP INDEX` + - `ALTER MATERIALIZE INDEX` + - `ALTER CLEAR INDEX` + - `ALTER CONSTRAINT` + - `ALTER ADD CONSTRAINT` + - `ALTER DROP CONSTRAINT` + - `ALTER TTL` + - `ALTER MATERIALIZE TTL` + - `ALTER SETTINGS` + - `ALTER MOVE PARTITION` + - `ALTER FETCH PARTITION` + - `ALTER FREEZE PARTITION` + - `ALTER VIEW` + - `ALTER VIEW REFRESH` + - `ALTER VIEW MODIFY QUERY` +- [CREATE](#grant-create) + - `CREATE DATABASE` + - `CREATE TABLE` + - `CREATE VIEW` + - `CREATE DICTIONARY` + - `CREATE TEMPORARY TABLE` +- [DROP](#grant-drop) + - `DROP DATABASE` + - `DROP TABLE` + - `DROP VIEW` + - `DROP DICTIONARY` +- [TRUNCATE](#grant-truncate) +- [OPTIMIZE](#grant-optimize) +- [SHOW](#grant-show) + - `SHOW DATABASES` + - `SHOW TABLES` + - `SHOW COLUMNS` + - `SHOW DICTIONARIES` +- [KILL QUERY](#grant-kill-query) +- [ACCESS MANAGEMENT](#grant-access-management) + - `CREATE USER` + - `ALTER USER` + - `DROP USER` + - `CREATE ROLE` + - `ALTER ROLE` + - `DROP ROLE` + - `CREATE ROW POLICY` + - `ALTER ROW POLICY` + - `DROP ROW POLICY` + - `CREATE QUOTA` + - `ALTER QUOTA` + - `DROP QUOTA` + - `CREATE SETTINGS PROFILE` + - `ALTER SETTINGS PROFILE` + - `DROP SETTINGS PROFILE` + - `SHOW ACCESS` + - `SHOW_USERS` + - `SHOW_ROLES` + - `SHOW_ROW_POLICIES` + - `SHOW_QUOTAS` + - `SHOW_SETTINGS_PROFILES` + - `ROLE ADMIN` +- [SYSTEM](#grant-system) + - `SYSTEM SHUTDOWN` + - `SYSTEM DROP CACHE` + - `SYSTEM DROP DNS CACHE` + - `SYSTEM DROP MARK CACHE` + - `SYSTEM DROP UNCOMPRESSED CACHE` + - `SYSTEM RELOAD` + - `SYSTEM RELOAD CONFIG` + - `SYSTEM RELOAD DICTIONARY` + - `SYSTEM RELOAD EMBEDDED DICTIONARIES` + - `SYSTEM MERGES` + - `SYSTEM TTL MERGES` + - `SYSTEM FETCHES` + - `SYSTEM MOVES` + - `SYSTEM SENDS` + - `SYSTEM DISTRIBUTED SENDS` + - `SYSTEM REPLICATED SENDS` + - `SYSTEM REPLICATION QUEUES` + - `SYSTEM SYNC REPLICA` + - `SYSTEM RESTART REPLICA` + - `SYSTEM FLUSH` + - `SYSTEM FLUSH DISTRIBUTED` + - `SYSTEM FLUSH LOGS` +- [INTROSPECTION](#grant-introspection) + - `addressToLine` + - `addressToSymbol` + - `demangle` +- [SOURCES](#grant-sources) + - `FILE` + - `URL` + - `REMOTE` + - `YSQL` + - `ODBC` + - `JDBC` + - `HDFS` + - `S3` +- [دیکته کردن](#grant-dictget) + +نمونه هایی از چگونه این سلسله مراتب درمان می شود: + +- این `ALTER` امتیاز شامل تمام دیگر `ALTER*` امتیازات. +- `ALTER CONSTRAINT` شامل `ALTER ADD CONSTRAINT` و `ALTER DROP CONSTRAINT` امتیازات. + +امتیازات در سطوح مختلف اعمال می شود. دانستن یک سطح نشان می دهد نحو در دسترس برای امتیاز. + +سطح (از پایین به بالاتر): + +- `COLUMN` — Privilege can be granted for column, table, database, or globally. +- `TABLE` — Privilege can be granted for table, database, or globally. +- `VIEW` — Privilege can be granted for view, database, or globally. +- `DICTIONARY` — Privilege can be granted for dictionary, database, or globally. +- `DATABASE` — Privilege can be granted for database or globally. +- `GLOBAL` — Privilege can be granted only globally. +- `GROUP` — Groups privileges of different levels. When `GROUP`- امتیاز سطح اعطا شده است, تنها که امتیازات از گروه اعطا که به نحو استفاده مطابقت. + +نمونه هایی از نحو مجاز: + +- `GRANT SELECT(x) ON db.table TO user` +- `GRANT SELECT ON db.* TO user` + +نمونه هایی از نحو غیر مجاز: + +- `GRANT CREATE USER(x) ON db.table TO user` +- `GRANT CREATE USER ON db.* TO user` + +امتیاز ویژه [ALL](#grant-all) اعطا تمام امتیازات به یک حساب کاربری و یا نقش. + +به طور پیش فرض, یک حساب کاربری و یا نقش هیچ امتیازات. + +اگر یک کاربر یا نقش هیچ امتیاز به عنوان نمایش داده می شود [NONE](#grant-none) امتیاز. + +برخی از نمایش داده شد با اجرای خود نیاز به مجموعه ای از امتیازات. برای مثال برای انجام [RENAME](misc.md#misc_operations-rename) پرس و جو شما نیاز به امتیازات زیر: `SELECT`, `CREATE TABLE`, `INSERT` و `DROP TABLE`. + +### SELECT {#grant-select} + +اجازه می دهد تا برای انجام [SELECT](select/index.md) نمایش داده شد. + +سطح امتیاز: `COLUMN`. + +**توصیف** + +کاربر اعطا شده با این امتیاز می تواند انجام دهد `SELECT` نمایش داده شد بیش از یک لیست مشخص از ستون ها در جدول و پایگاه داده مشخص شده است. اگر کاربر شامل ستون های دیگر و سپس مشخص پرس و جو هیچ داده گرداند. + +امتیاز زیر را در نظر بگیرید: + +``` sql +GRANT SELECT(x,y) ON db.table TO john +``` + +این امتیاز اجازه می دهد `john` برای انجام هر `SELECT` پرس و جو که شامل داده ها از `x` و / یا `y` ستونها در `db.table`. به عنوان مثال, `SELECT x FROM db.table`. `john` نمی تواند انجام دهد `SELECT z FROM db.table`. این `SELECT * FROM db.table` همچنین در دسترس نیست. پردازش این پرس و جو, تاتر هیچ داده بازگشت نیست, حتی `x` و `y`. تنها استثنا است اگر یک جدول شامل تنها `x` و `y` ستون, در این مورد کلیکهاوس را برمی گرداند تمام داده ها. + +### INSERT {#grant-insert} + +اجازه می دهد تا انجام [INSERT](insert-into.md) نمایش داده شد. + +سطح امتیاز: `COLUMN`. + +**توصیف** + +کاربر اعطا شده با این امتیاز می تواند انجام دهد `INSERT` نمایش داده شد بیش از یک لیست مشخص از ستون ها در جدول و پایگاه داده مشخص شده است. اگر کاربر شامل ستون های دیگر و سپس مشخص پرس و جو هیچ داده وارد کنید. + +**مثال** + +``` sql +GRANT INSERT(x,y) ON db.table TO john +``` + +امتیاز اعطا شده اجازه می دهد `john` برای وارد کردن داده ها به `x` و / یا `y` ستونها در `db.table`. + +### ALTER {#grant-alter} + +اجازه می دهد تا انجام [ALTER](alter.md) نمایش داده شد مربوط به سلسله مراتب زیر از امتیازات: + +- `ALTER`. سطح: `COLUMN`. + - `ALTER TABLE`. سطح: `GROUP` + - `ALTER UPDATE`. سطح: `COLUMN`. نامگردانها: `UPDATE` + - `ALTER DELETE`. سطح: `COLUMN`. نامگردانها: `DELETE` + - `ALTER COLUMN`. سطح: `GROUP` + - `ALTER ADD COLUMN`. سطح: `COLUMN`. نامگردانها: `ADD COLUMN` + - `ALTER DROP COLUMN`. سطح: `COLUMN`. نامگردانها: `DROP COLUMN` + - `ALTER MODIFY COLUMN`. سطح: `COLUMN`. نامگردانها: `MODIFY COLUMN` + - `ALTER COMMENT COLUMN`. سطح: `COLUMN`. نامگردانها: `COMMENT COLUMN` + - `ALTER CLEAR COLUMN`. سطح: `COLUMN`. نامگردانها: `CLEAR COLUMN` + - `ALTER RENAME COLUMN`. سطح: `COLUMN`. نامگردانها: `RENAME COLUMN` + - `ALTER INDEX`. سطح: `GROUP`. نامگردانها: `INDEX` + - `ALTER ORDER BY`. سطح: `TABLE`. نامگردانها: `ALTER MODIFY ORDER BY`, `MODIFY ORDER BY` + - `ALTER ADD INDEX`. سطح: `TABLE`. نامگردانها: `ADD INDEX` + - `ALTER DROP INDEX`. سطح: `TABLE`. نامگردانها: `DROP INDEX` + - `ALTER MATERIALIZE INDEX`. سطح: `TABLE`. نامگردانها: `MATERIALIZE INDEX` + - `ALTER CLEAR INDEX`. سطح: `TABLE`. نامگردانها: `CLEAR INDEX` + - `ALTER CONSTRAINT`. سطح: `GROUP`. نامگردانها: `CONSTRAINT` + - `ALTER ADD CONSTRAINT`. سطح: `TABLE`. نامگردانها: `ADD CONSTRAINT` + - `ALTER DROP CONSTRAINT`. سطح: `TABLE`. نامگردانها: `DROP CONSTRAINT` + - `ALTER TTL`. سطح: `TABLE`. نامگردانها: `ALTER MODIFY TTL`, `MODIFY TTL` + - `ALTER MATERIALIZE TTL`. سطح: `TABLE`. نامگردانها: `MATERIALIZE TTL` + - `ALTER SETTINGS`. سطح: `TABLE`. نامگردانها: `ALTER SETTING`, `ALTER MODIFY SETTING`, `MODIFY SETTING` + - `ALTER MOVE PARTITION`. سطح: `TABLE`. نامگردانها: `ALTER MOVE PART`, `MOVE PARTITION`, `MOVE PART` + - `ALTER FETCH PARTITION`. سطح: `TABLE`. نامگردانها: `FETCH PARTITION` + - `ALTER FREEZE PARTITION`. سطح: `TABLE`. نامگردانها: `FREEZE PARTITION` + - `ALTER VIEW` سطح: `GROUP` + - `ALTER VIEW REFRESH`. سطح: `VIEW`. نامگردانها: `ALTER LIVE VIEW REFRESH`, `REFRESH VIEW` + - `ALTER VIEW MODIFY QUERY`. سطح: `VIEW`. نامگردانها: `ALTER TABLE MODIFY QUERY` + +نمونه هایی از چگونه این سلسله مراتب درمان می شود: + +- این `ALTER` امتیاز شامل تمام دیگر `ALTER*` امتیازات. +- `ALTER CONSTRAINT` شامل `ALTER ADD CONSTRAINT` و `ALTER DROP CONSTRAINT` امتیازات. + +**یادداشتها** + +- این `MODIFY SETTING` امتیاز اجازه می دهد تا برای تغییر تنظیمات موتور جدول. در تنظیمات و یا پارامترهای پیکربندی سرور تاثیر نمی گذارد. +- این `ATTACH` عملیات نیاز دارد [CREATE](#grant-create) امتیاز. +- این `DETACH` عملیات نیاز دارد [DROP](#grant-drop) امتیاز. +- برای جلوگیری از جهش توسط [KILL MUTATION](misc.md#kill-mutation) پرس و جو, شما نیاز به یک امتیاز برای شروع این جهش. مثلا, اگر شما می خواهید برای جلوگیری از `ALTER UPDATE` پرس و جو, شما نیاز به `ALTER UPDATE`, `ALTER TABLE` یا `ALTER` امتیاز. + +### CREATE {#grant-create} + +اجازه می دهد تا برای انجام [CREATE](create.md) و [ATTACH](misc.md#attach) دی ال-پرس و جو مربوط به سلسله مراتب زیر از امتیازات: + +- `CREATE`. سطح: `GROUP` + - `CREATE DATABASE`. سطح: `DATABASE` + - `CREATE TABLE`. سطح: `TABLE` + - `CREATE VIEW`. سطح: `VIEW` + - `CREATE DICTIONARY`. سطح: `DICTIONARY` + - `CREATE TEMPORARY TABLE`. سطح: `GLOBAL` + +**یادداشتها** + +- برای حذف جدول ایجاد شده نیاز کاربر [DROP](#grant-drop). + +### DROP {#grant-drop} + +اجازه می دهد تا برای انجام [DROP](misc.md#drop) و [DETACH](misc.md#detach) نمایش داده شد مربوط به سلسله مراتب زیر از امتیازات: + +- `DROP`. سطح: + - `DROP DATABASE`. سطح: `DATABASE` + - `DROP TABLE`. سطح: `TABLE` + - `DROP VIEW`. سطح: `VIEW` + - `DROP DICTIONARY`. سطح: `DICTIONARY` + +### TRUNCATE {#grant-truncate} + +اجازه می دهد تا برای انجام [TRUNCATE](misc.md#truncate-statement) نمایش داده شد. + +سطح امتیاز: `TABLE`. + +### OPTIMIZE {#grant-optimize} + +اجازه می دهد تا برای انجام [OPTIMIZE TABLE](misc.md#misc_operations-optimize) نمایش داده شد. + +سطح امتیاز: `TABLE`. + +### SHOW {#grant-show} + +اجازه می دهد تا برای انجام `SHOW`, `DESCRIBE`, `USE` و `EXISTS` نمایش داده شد, مربوط به سلسله مراتب زیر از امتیازات: + +- `SHOW`. سطح: `GROUP` + - `SHOW DATABASES`. سطح: `DATABASE`. اجازه می دهد تا برای اجرای `SHOW DATABASES`, `SHOW CREATE DATABASE`, `USE ` نمایش داده شد. + - `SHOW TABLES`. سطح: `TABLE`. اجازه می دهد تا برای اجرای `SHOW TABLES`, `EXISTS `, `CHECK
` نمایش داده شد. + - `SHOW COLUMNS`. سطح: `COLUMN`. اجازه می دهد تا برای اجرای `SHOW CREATE TABLE`, `DESCRIBE` نمایش داده شد. + - `SHOW DICTIONARIES`. سطح: `DICTIONARY`. اجازه می دهد تا برای اجرای `SHOW DICTIONARIES`, `SHOW CREATE DICTIONARY`, `EXISTS ` نمایش داده شد. + +**یادداشتها** + +یک کاربر است `SHOW` امتیاز اگر هر امتیاز دیگری در مورد جدول مشخص, فرهنگ لغت و یا پایگاه داده. + +### KILL QUERY {#grant-kill-query} + +اجازه می دهد تا برای انجام [KILL](misc.md#kill-query-statement) نمایش داده شد مربوط به سلسله مراتب زیر از امتیازات: + +سطح امتیاز: `GLOBAL`. + +**یادداشتها** + +`KILL QUERY` امتیاز اجازه می دهد تا یک کاربر برای کشتن نمایش داده شد از کاربران دیگر. + +### ACCESS MANAGEMENT {#grant-access-management} + +اجازه می دهد تا کاربر را به انجام نمایش داده شد که مدیریت کاربران, نقش ها و سیاست های ردیف. + +- `ACCESS MANAGEMENT`. سطح: `GROUP` + - `CREATE USER`. سطح: `GLOBAL` + - `ALTER USER`. سطح: `GLOBAL` + - `DROP USER`. سطح: `GLOBAL` + - `CREATE ROLE`. سطح: `GLOBAL` + - `ALTER ROLE`. سطح: `GLOBAL` + - `DROP ROLE`. سطح: `GLOBAL` + - `ROLE ADMIN`. سطح: `GLOBAL` + - `CREATE ROW POLICY`. سطح: `GLOBAL`. نامگردانها: `CREATE POLICY` + - `ALTER ROW POLICY`. سطح: `GLOBAL`. نامگردانها: `ALTER POLICY` + - `DROP ROW POLICY`. سطح: `GLOBAL`. نامگردانها: `DROP POLICY` + - `CREATE QUOTA`. سطح: `GLOBAL` + - `ALTER QUOTA`. سطح: `GLOBAL` + - `DROP QUOTA`. سطح: `GLOBAL` + - `CREATE SETTINGS PROFILE`. سطح: `GLOBAL`. نامگردانها: `CREATE PROFILE` + - `ALTER SETTINGS PROFILE`. سطح: `GLOBAL`. نامگردانها: `ALTER PROFILE` + - `DROP SETTINGS PROFILE`. سطح: `GLOBAL`. نامگردانها: `DROP PROFILE` + - `SHOW ACCESS`. سطح: `GROUP` + - `SHOW_USERS`. سطح: `GLOBAL`. نامگردانها: `SHOW CREATE USER` + - `SHOW_ROLES`. سطح: `GLOBAL`. نامگردانها: `SHOW CREATE ROLE` + - `SHOW_ROW_POLICIES`. سطح: `GLOBAL`. نامگردانها: `SHOW POLICIES`, `SHOW CREATE ROW POLICY`, `SHOW CREATE POLICY` + - `SHOW_QUOTAS`. سطح: `GLOBAL`. نامگردانها: `SHOW CREATE QUOTA` + - `SHOW_SETTINGS_PROFILES`. سطح: `GLOBAL`. نامگردانها: `SHOW PROFILES`, `SHOW CREATE SETTINGS PROFILE`, `SHOW CREATE PROFILE` + +این `ROLE ADMIN` امتیاز اجازه می دهد تا کاربر را به اعطای و لغو هر نقش از جمله کسانی که به کاربر با گزینه مدیریت داده نمی شود. + +### SYSTEM {#grant-system} + +اجازه می دهد تا کاربر را به انجام [SYSTEM](system.md) نمایش داده شد مربوط به سلسله مراتب زیر از امتیازات. + +- `SYSTEM`. سطح: `GROUP` + - `SYSTEM SHUTDOWN`. سطح: `GLOBAL`. نامگردانها: `SYSTEM KILL`, `SHUTDOWN` + - `SYSTEM DROP CACHE`. نامگردانها: `DROP CACHE` + - `SYSTEM DROP DNS CACHE`. سطح: `GLOBAL`. نامگردانها: `SYSTEM DROP DNS`, `DROP DNS CACHE`, `DROP DNS` + - `SYSTEM DROP MARK CACHE`. سطح: `GLOBAL`. نامگردانها: `SYSTEM DROP MARK`, `DROP MARK CACHE`, `DROP MARKS` + - `SYSTEM DROP UNCOMPRESSED CACHE`. سطح: `GLOBAL`. نامگردانها: `SYSTEM DROP UNCOMPRESSED`, `DROP UNCOMPRESSED CACHE`, `DROP UNCOMPRESSED` + - `SYSTEM RELOAD`. سطح: `GROUP` + - `SYSTEM RELOAD CONFIG`. سطح: `GLOBAL`. نامگردانها: `RELOAD CONFIG` + - `SYSTEM RELOAD DICTIONARY`. سطح: `GLOBAL`. نامگردانها: `SYSTEM RELOAD DICTIONARIES`, `RELOAD DICTIONARY`, `RELOAD DICTIONARIES` + - `SYSTEM RELOAD EMBEDDED DICTIONARIES`. سطح: `GLOBAL`. نامگردانها: تحقیق`ELOAD EMBEDDED DICTIONARIES` + - `SYSTEM MERGES`. سطح: `TABLE`. نامگردانها: `SYSTEM STOP MERGES`, `SYSTEM START MERGES`, `STOP MERGES`, `START MERGES` + - `SYSTEM TTL MERGES`. سطح: `TABLE`. نامگردانها: `SYSTEM STOP TTL MERGES`, `SYSTEM START TTL MERGES`, `STOP TTL MERGES`, `START TTL MERGES` + - `SYSTEM FETCHES`. سطح: `TABLE`. نامگردانها: `SYSTEM STOP FETCHES`, `SYSTEM START FETCHES`, `STOP FETCHES`, `START FETCHES` + - `SYSTEM MOVES`. سطح: `TABLE`. نامگردانها: `SYSTEM STOP MOVES`, `SYSTEM START MOVES`, `STOP MOVES`, `START MOVES` + - `SYSTEM SENDS`. سطح: `GROUP`. نامگردانها: `SYSTEM STOP SENDS`, `SYSTEM START SENDS`, `STOP SENDS`, `START SENDS` + - `SYSTEM DISTRIBUTED SENDS`. سطح: `TABLE`. نامگردانها: `SYSTEM STOP DISTRIBUTED SENDS`, `SYSTEM START DISTRIBUTED SENDS`, `STOP DISTRIBUTED SENDS`, `START DISTRIBUTED SENDS` + - `SYSTEM REPLICATED SENDS`. سطح: `TABLE`. نامگردانها: `SYSTEM STOP REPLICATED SENDS`, `SYSTEM START REPLICATED SENDS`, `STOP REPLICATED SENDS`, `START REPLICATED SENDS` + - `SYSTEM REPLICATION QUEUES`. سطح: `TABLE`. نامگردانها: `SYSTEM STOP REPLICATION QUEUES`, `SYSTEM START REPLICATION QUEUES`, `STOP REPLICATION QUEUES`, `START REPLICATION QUEUES` + - `SYSTEM SYNC REPLICA`. سطح: `TABLE`. نامگردانها: `SYNC REPLICA` + - `SYSTEM RESTART REPLICA`. سطح: `TABLE`. نامگردانها: `RESTART REPLICA` + - `SYSTEM FLUSH`. سطح: `GROUP` + - `SYSTEM FLUSH DISTRIBUTED`. سطح: `TABLE`. نامگردانها: `FLUSH DISTRIBUTED` + - `SYSTEM FLUSH LOGS`. سطح: `GLOBAL`. نامگردانها: `FLUSH LOGS` + +این `SYSTEM RELOAD EMBEDDED DICTIONARIES` امتیاز به طور ضمنی توسط اعطا `SYSTEM RELOAD DICTIONARY ON *.*` امتیاز. + +### INTROSPECTION {#grant-introspection} + +اجازه می دهد تا با استفاده از [درون نگری](../../operations/optimizing-performance/sampling-query-profiler.md) توابع. + +- `INTROSPECTION`. سطح: `GROUP`. نامگردانها: `INTROSPECTION FUNCTIONS` + - `addressToLine`. سطح: `GLOBAL` + - `addressToSymbol`. سطح: `GLOBAL` + - `demangle`. سطح: `GLOBAL` + +### SOURCES {#grant-sources} + +اجازه می دهد تا با استفاده از منابع داده های خارجی. امر به [موتورهای جدول](../../engines/table-engines/index.md) و [توابع جدول](../table-functions/index.md#table-functions). + +- `SOURCES`. سطح: `GROUP` + - `FILE`. سطح: `GLOBAL` + - `URL`. سطح: `GLOBAL` + - `REMOTE`. سطح: `GLOBAL` + - `YSQL`. سطح: `GLOBAL` + - `ODBC`. سطح: `GLOBAL` + - `JDBC`. سطح: `GLOBAL` + - `HDFS`. سطح: `GLOBAL` + - `S3`. سطح: `GLOBAL` + +این `SOURCES` امتیاز را قادر می سازد استفاده از تمام منابع. همچنین شما می توانید یک امتیاز برای هر منبع به صورت جداگانه اعطا. برای استفاده از منابع, شما نیاز به امتیازات اضافی. + +مثالها: + +- برای ایجاد یک جدول با [خروجی زیر موتور جدول](../../engines/table-engines/integrations/mysql.md) شما نیاز دارید `CREATE TABLE (ON db.table_name)` و `MYSQL` امتیازات. +- برای استفاده از [تابع جدول خروجی زیر](../table-functions/mysql.md) شما نیاز دارید `CREATE TEMPORARY TABLE` و `MYSQL` امتیازات. + +### دیکته کردن {#grant-dictget} + +- `dictGet`. نامگردانها: `dictHas`, `dictGetHierarchy`, `dictIsIn` + +اجازه می دهد تا کاربر را به اجرا [دیکته کردن](../functions/ext-dict-functions.md#dictget), [دیکتس](../functions/ext-dict-functions.md#dicthas), [حکومت دیکتاتوری](../functions/ext-dict-functions.md#dictgethierarchy), [دیکتاتوری](../functions/ext-dict-functions.md#dictisin) توابع. + +سطح امتیاز: `DICTIONARY`. + +**مثالها** + +- `GRANT dictGet ON mydb.mydictionary TO john` +- `GRANT dictGet ON mydictionary TO john` + +### ALL {#grant-all} + +اعطا تمام امتیازات در نهاد تنظیم به یک حساب کاربری و یا نقش. + +### NONE {#grant-none} + +هیچ امتیازاتی به دست نمیاره + +### ADMIN OPTION {#admin-option-privilege} + +این `ADMIN OPTION` امتیاز اجازه می دهد تا کاربر اعطای نقش خود را به یک کاربر دیگر. + +[مقاله اصلی](https://clickhouse.tech/docs/en/query_language/grant/) diff --git a/docs/fa/sql-reference/statements/index.md b/docs/fa/sql-reference/statements/index.md index a2246679149..3af2f1603c9 100644 --- a/docs/fa/sql-reference/statements/index.md +++ b/docs/fa/sql-reference/statements/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_folder_title: Statements +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "\u0628\u06CC\u0627\u0646\u06CC\u0647 \u0647\u0627" toc_priority: 31 --- diff --git a/docs/fa/sql-reference/statements/insert-into.md b/docs/fa/sql-reference/statements/insert-into.md index 7a309cfeb0f..39950c99688 100644 --- a/docs/fa/sql-reference/statements/insert-into.md +++ b/docs/fa/sql-reference/statements/insert-into.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 34 toc_title: INSERT INTO --- diff --git a/docs/fa/sql-reference/statements/misc.md b/docs/fa/sql-reference/statements/misc.md index c279b634ce0..d67983bd5ba 100644 --- a/docs/fa/sql-reference/statements/misc.md +++ b/docs/fa/sql-reference/statements/misc.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_priority: 39 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_priority: 41 toc_title: "\u063A\u06CC\u0631\u0647" --- -# متفرقه نمایش داده شد {#miscellaneous-queries} +# نمایش داده شد دیگر {#miscellaneous-queries} ## ATTACH {#attach} @@ -13,7 +13,7 @@ toc_title: "\u063A\u06CC\u0631\u0647" - به جای کلمه `CREATE` با استفاده از این کلمه `ATTACH`. - پرس و جو می کند داده ها بر روی دیسک ایجاد کنید, اما فرض می شود که داده ها در حال حاضر در مکان های مناسب, و فقط می افزاید: اطلاعات در مورد جدول به سرور. - پس از اجرای یک ضمیمه پرس و جو در سرور خواهد شد در مورد وجود جدول. + پس از اجرای پرس و جو ضمیمه, سرور در مورد وجود جدول می دانم. اگر جدول قبلا جدا شد (`DETACH`), به این معنی که ساختار خود شناخته شده است, شما می توانید مختصر بدون تعریف ساختار استفاده. @@ -86,7 +86,7 @@ DETACH TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] ``` این داده ها و یا ابرداده جدول را حذف کنید. در راه اندازی سرور بعدی, سرور خواهد ابرداده به عنوان خوانده شده و پیدا کردن در مورد جدول دوباره. -به طور مشابه “detached” جدول را می توان دوباره متصل با استفاده از `ATTACH` پرس و جو (به غیر از جداول سیستم که لازم نیست metadata ذخیره شده برای آنها). +به طور مشابه “detached” جدول را می توان دوباره متصل با استفاده از `ATTACH` پرس و جو (به غیر از جداول سیستم, که ابرداده ذخیره شده برای ندارد). وجود ندارد `DETACH DATABASE` پرس و جو. @@ -113,7 +113,65 @@ DROP [TEMPORARY] TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] دلس فرهنگ لغت. اگر `IF EXISTS` مشخص شده است, این خطا را نمی گرداند اگر جدول وجود ندارد و یا پایگاه داده وجود ندارد. -## EXISTS {#exists} +## DROP USER {#drop-user-statement} + +حذف یک کاربر. + +### نحو {#drop-user-syntax} + +``` sql +DROP USER [IF EXISTS] name [,...] [ON CLUSTER cluster_name] +``` + +## DROP ROLE {#drop-role-statement} + +نقش را حذف می کند. + +نقش حذف شده از تمام نهادهایی که اعطا شد لغو می شود. + +### نحو {#drop-role-syntax} + +``` sql +DROP ROLE [IF EXISTS] name [,...] [ON CLUSTER cluster_name] +``` + +## DROP ROW POLICY {#drop-row-policy-statement} + +حذف یک سیاست ردیف. + +سیاست ردیف حذف شده است از تمام اشخاص لغو جایی که اختصاص داده شد. + +### نحو {#drop-row-policy-syntax} + +``` sql +DROP [ROW] POLICY [IF EXISTS] name [,...] ON [database.]table [,...] [ON CLUSTER cluster_name] +``` + +## DROP QUOTA {#drop-quota-statement} + +حذف سهمیه. + +سهمیه حذف شده است از تمام اشخاص لغو جایی که اختصاص داده شد. + +### نحو {#drop-quota-syntax} + +``` sql +DROP QUOTA [IF EXISTS] name [,...] [ON CLUSTER cluster_name] +``` + +## DROP SETTINGS PROFILE {#drop-settings-profile-statement} + +حذف سهمیه. + +سهمیه حذف شده است از تمام اشخاص لغو جایی که اختصاص داده شد. + +### نحو {#drop-settings-profile-syntax} + +``` sql +DROP [SETTINGS] PROFILE [IF EXISTS] name [,...] [ON CLUSTER cluster_name] +``` + +## EXISTS {#exists-statement} ``` sql EXISTS [TEMPORARY] [TABLE|DICTIONARY] [db.]name [INTO OUTFILE filename] [FORMAT format] @@ -121,7 +179,7 @@ EXISTS [TEMPORARY] [TABLE|DICTIONARY] [db.]name [INTO OUTFILE filename] [FORMAT بازگرداندن یک `UInt8`- نوع ستون, که شامل ارزش واحد `0` اگر جدول یا پایگاه داده وجود ندارد, یا `1` اگر جدول در پایگاه داده مشخص شده وجود دارد. -## KILL QUERY {#kill-query} +## KILL QUERY {#kill-query-statement} ``` sql KILL QUERY [ON CLUSTER cluster] @@ -152,7 +210,7 @@ KILL QUERY WHERE user='username' SYNC 1. ‘finished’ – The query was terminated successfully. 2. ‘waiting’ – Waiting for the query to end after sending it a signal to terminate. -3. The other values ​​explain why the query can’t be stopped. +3. The other values ​​explain why the query can't be stopped. پرسوجوی تست (`TEST`) فقط چک حقوق کاربر و نمایش یک لیست از نمایش داده شد برای متوقف کردن. @@ -167,7 +225,7 @@ KILL MUTATION [ON CLUSTER cluster] تلاش برای لغو و حذف [جهشها](alter.md#alter-mutations) که در حال حاضر اجرای. جهش به لغو از انتخاب [`system.mutations`](../../operations/system-tables.md#system_tables-mutations) جدول با استفاده از فیلتر مشخص شده توسط `WHERE` بند از `KILL` پرس و جو. -آزمون پرس و جو (`TEST`) فقط چک حقوق کاربر و نمایش یک لیست از نمایش داده شد برای متوقف کردن. +پرسوجوی تست (`TEST`) فقط چک حقوق کاربر و نمایش یک لیست از نمایش داده شد برای متوقف کردن. مثالها: @@ -193,7 +251,7 @@ OPTIMIZE TABLE [db.]name [ON CLUSTER cluster] [PARTITION partition | PARTITION I این `OPTMIZE` پرس و جو نیز برای پشتیبانی [ماده بینی](../../engines/table-engines/special/materializedview.md) و [بافر](../../engines/table-engines/special/buffer.md) موتورها. دیگر موتورهای جدول پشتیبانی نمی شوند. -زمانی که `OPTIMIZE` با استفاده از [تکرار غذای اصلی](../../engines/table-engines/mergetree-family/replication.md) خانواده از موتورهای جدول, تاتر ایجاد یک کار برای ادغام و منتظر اعدام در تمام گره (در صورتی که `replication_alter_partitions_sync` تنظیم فعال است). +چه زمانی `OPTIMIZE` با استفاده از [تکرار غذای اصلی](../../engines/table-engines/mergetree-family/replication.md) خانواده از موتورهای جدول, تاتر ایجاد یک کار برای ادغام و منتظر اعدام در تمام گره (در صورتی که `replication_alter_partitions_sync` تنظیم فعال است). - اگر `OPTIMIZE` یک ادغام به هر دلیلی انجام نمی, این کار مشتری اطلاع نیست. برای فعال کردن اعلان ها از [ا\_فزون\_ف\_کوپ](../../operations/settings/settings.md#setting-optimize_throw_if_noop) تنظیمات. - اگر شما یک مشخص `PARTITION` فقط پارتیشن مشخص شده بهینه شده است. [نحوه تنظیم بیان پارتیشن](alter.md#alter-how-to-specify-part-expr). @@ -219,7 +277,7 @@ RENAME TABLE [db11.]name11 TO [db12.]name12, [db21.]name21 TO [db22.]name22, ... SET param = value ``` -انتساب `value` به `param` [تنظیم](../../operations/settings/index.md) برای جلسه فعلی. شما نمی توانید تغییر دهید [تنظیمات سرور](../../operations/server-configuration-parameters/index.md) از این طرف +انتساب `value` به `param` [تنظیم](../../operations/settings/index.md) برای جلسه فعلی. شما نمی توانید تغییر دهید [تنظیمات کارگزار](../../operations/server-configuration-parameters/index.md) از این طرف شما همچنین می توانید تمام مقادیر را از مشخصات تنظیمات مشخص شده در یک پرس و جو واحد تنظیم کنید. @@ -229,7 +287,55 @@ SET profile = 'profile-name-from-the-settings-file' برای کسب اطلاعات بیشتر, دیدن [تنظیمات](../../operations/settings/settings.md). -## TRUNCATE {#truncate} +## SET ROLE {#set-role-statement} + +فعال نقش برای کاربر فعلی. + +### نحو {#set-role-syntax} + +``` sql +SET ROLE {DEFAULT | NONE | role [,...] | ALL | ALL EXCEPT role [,...]} +``` + +## SET DEFAULT ROLE {#set-default-role-statement} + +مجموعه نقش به طور پیش فرض به یک کاربر. + +نقش پیش فرض به طور خودکار در ورود کاربر فعال می شود. شما می توانید به عنوان پیش فرض تنها نقش قبلا اعطا تنظیم شده است. اگر نقش به یک کاربر اعطا نمی, خانه عروسکی می اندازد یک استثنا. + +### نحو {#set-default-role-syntax} + +``` sql +SET DEFAULT ROLE {NONE | role [,...] | ALL | ALL EXCEPT role [,...]} TO {user|CURRENT_USER} [,...] +``` + +### مثالها {#set-default-role-examples} + +تنظیم نقش های پیش فرض چندگانه به یک کاربر: + +``` sql +SET DEFAULT ROLE role1, role2, ... TO user +``` + +تنظیم تمام نقش های اعطا شده به عنوان پیش فرض به یک کاربر: + +``` sql +SET DEFAULT ROLE ALL TO user +``` + +خالی کردن نقش پیشفرض از یک کاربر: + +``` sql +SET DEFAULT ROLE NONE TO user +``` + +مجموعه ای از تمام نقش های اعطا شده به عنوان پیش فرض به استثنای برخی از: + +``` sql +SET DEFAULT ROLE ALL EXCEPT role1, role2 TO user +``` + +## TRUNCATE {#truncate-statement} ``` sql TRUNCATE TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] @@ -237,7 +343,7 @@ TRUNCATE TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] حذف تمام داده ها را از یک جدول. هنگامی که بند `IF EXISTS` حذف شده است, پرس و جو یک خطا می گرداند اگر جدول وجود ندارد. -این `TRUNCATE` پرسوجو برای پشتیبانی نمیشود [مشاهده](../../engines/table-engines/special/view.md), [پرونده](../../engines/table-engines/special/file.md), [URL](../../engines/table-engines/special/url.md) و [خالی](../../engines/table-engines/special/null.md) موتورهای جدول. +این `TRUNCATE` پرسوجو برای پشتیبانی نمیشود [نما](../../engines/table-engines/special/view.md), [پرونده](../../engines/table-engines/special/file.md), [URL](../../engines/table-engines/special/url.md) و [خالی](../../engines/table-engines/special/null.md) موتورهای جدول. ## USE {#use} diff --git a/docs/fa/sql-reference/statements/revoke.md b/docs/fa/sql-reference/statements/revoke.md new file mode 100644 index 00000000000..c9fb77a3d48 --- /dev/null +++ b/docs/fa/sql-reference/statements/revoke.md @@ -0,0 +1,50 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_priority: 40 +toc_title: REVOKE +--- + +# REVOKE {#revoke} + +لغو امتیازات از کاربران و یا نقش. + +## نحو {#revoke-syntax} + +**لغو امتیازات از کاربران** + +``` sql +REVOKE [ON CLUSTER cluster_name] privilege[(column_name [,...])] [,...] ON {db.table|db.*|*.*|table|*} FROM {user | CURRENT_USER} [,...] | ALL | ALL EXCEPT {user | CURRENT_USER} [,...] +``` + +**نقش لغو از کاربران** + +``` sql +REVOKE [ON CLUSTER cluster_name] [ADMIN OPTION FOR] role [,...] FROM {user | role | CURRENT_USER} [,...] | ALL | ALL EXCEPT {user_name | role_name | CURRENT_USER} [,...] +``` + +## توصیف {#revoke-description} + +برای لغو برخی از امتیاز شما می توانید یک امتیاز از دامنه گسترده تر استفاده کنید و سپس شما قصد لغو. برای مثال اگر یک کاربر `SELECT (x,y)` امتیاز مدیر می تواند انجام دهد `REVOKE SELECT(x,y) ...` یا `REVOKE SELECT * ...` یا حتی `REVOKE ALL PRIVILEGES ...` پرسوجو برای لغو این امتیاز. + +### لغو نسبی {#partial-revokes-dscr} + +شما می توانید بخشی از یک امتیاز لغو. برای مثال اگر یک کاربر `SELECT *.*` امتیاز شما می توانید از این امتیاز لغو به خواندن داده ها از برخی از جدول و یا یک پایگاه داده. + +## مثالها {#revoke-example} + +اعطای `john` حساب کاربری با یک امتیاز را انتخاب کنید از تمام پایگاه داده به استثنای `accounts` یک: + +``` sql +GRANT SELECT ON *.* TO john; +REVOKE SELECT ON accounts.* FROM john; +``` + +اعطای `mira` حساب کاربری با یک امتیاز را انتخاب کنید از تمام ستون ها از `accounts.staff` جدول به استثنای `wage` یک + +``` sql +GRANT SELECT ON accounts.staff TO mira; +REVOKE SELECT(wage) ON accounts.staff FROM mira; +``` + +{## [مقاله اصلی](https://clickhouse.tech/docs/en/operations/settings/settings/) ##} diff --git a/docs/fa/sql-reference/statements/select.md b/docs/fa/sql-reference/statements/select.md deleted file mode 100644 index 50a5141c88d..00000000000 --- a/docs/fa/sql-reference/statements/select.md +++ /dev/null @@ -1,1379 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 0f7ef7704d018700049223525bad4a63911b6e70 -toc_priority: 33 -toc_title: SELECT ---- - -# نحو نمایش داده شد را انتخاب کنید {#select-queries-syntax} - -`SELECT` بازیابی داده ها را انجام می دهد. - -``` sql -[WITH expr_list|(subquery)] -SELECT [DISTINCT] expr_list -[FROM [db.]table | (subquery) | table_function] [FINAL] -[SAMPLE sample_coeff] -[ARRAY JOIN ...] -[GLOBAL] [ANY|ALL] [INNER|LEFT|RIGHT|FULL|CROSS] [OUTER] JOIN (subquery)|table USING columns_list -[PREWHERE expr] -[WHERE expr] -[GROUP BY expr_list] [WITH TOTALS] -[HAVING expr] -[ORDER BY expr_list] -[LIMIT [offset_value, ]n BY columns] -[LIMIT [n, ]m] -[UNION ALL ...] -[INTO OUTFILE filename] -[FORMAT format] -``` - -همه بند اختیاری هستند, به جز برای لیست مورد نیاز از عبارات بلافاصله پس از انتخاب. -بند های زیر تقریبا به همان ترتیب در نوار نقاله اجرای پرس و جو توصیف می شوند. - -اگر پرس و جو حذف `DISTINCT`, `GROUP BY` و `ORDER BY` بند و `IN` و `JOIN` کارخانه های فرعی, پرس و جو خواهد شد به طور کامل جریان پردازش, با استفاده از ای(1) مقدار رم. -در غیر این صورت, پرس و جو ممکن است مقدار زیادی از رم مصرف اگر محدودیت های مناسب مشخص نشده است: `max_memory_usage`, `max_rows_to_group_by`, `max_rows_to_sort`, `max_rows_in_distinct`, `max_bytes_in_distinct`, `max_rows_in_set`, `max_bytes_in_set`, `max_rows_in_join`, `max_bytes_in_join`, `max_bytes_before_external_sort`, `max_bytes_before_external_group_by`. برای کسب اطلاعات بیشتر به بخش مراجعه کنید “Settings”. ممکن است که به استفاده از مرتب سازی خارجی (صرفه جویی در جداول موقت به یک دیسک) و تجمع خارجی. `The system does not have "merge join"`. - -### با بند {#with-clause} - -در این بخش پشتیبانی از عبارات جدول مشترک فراهم می کند ([CTE](https://en.wikipedia.org/wiki/Hierarchical_and_recursive_queries_in_SQL)), با برخی از محدودیت: -1. نمایش داده شد بازگشتی پشتیبانی نمی شوند -2. هنگامی که زیرخاکری در داخل با بخش استفاده می شود, این نتیجه باید اسکالر با دقیقا یک ردیف باشد -3. بیان نتایج در دسترس نیست در subqueries -نتایج با عبارات بند را می توان در داخل بند را انتخاب کنید استفاده می شود. - -مثال 1: با استفاده از عبارت ثابت به عنوان “variable” - -``` sql -WITH '2019-08-01 15:23:00' as ts_upper_bound -SELECT * -FROM hits -WHERE - EventDate = toDate(ts_upper_bound) AND - EventTime <= ts_upper_bound -``` - -مثال 2: جمع تخلیه(بایت) نتیجه بیان از بند لیست ستون را انتخاب کنید - -``` sql -WITH sum(bytes) as s -SELECT - formatReadableSize(s), - table -FROM system.parts -GROUP BY table -ORDER BY s -``` - -مثال 3: با استفاده از نتایج حاصل از زیرخاکری اسکالر - -``` sql -/* this example would return TOP 10 of most huge tables */ -WITH - ( - SELECT sum(bytes) - FROM system.parts - WHERE active - ) AS total_disk_usage -SELECT - (sum(bytes) / total_disk_usage) * 100 AS table_disk_usage, - table -FROM system.parts -GROUP BY table -ORDER BY table_disk_usage DESC -LIMIT 10 -``` - -مثال 4: استفاده مجدد از بیان در زیرخاکری -به عنوان یک راه حل برای محدودیت فعلی برای استفاده بیان در زیر مجموعه, شما ممکن است تکراری. - -``` sql -WITH ['hello'] AS hello -SELECT - hello, - * -FROM -( - WITH ['hello'] AS hello - SELECT hello -) -``` - -``` text -┌─hello─────┬─hello─────┐ -│ ['hello'] │ ['hello'] │ -└───────────┴───────────┘ -``` - -### از بند {#select-from} - -اگر از بند حذف شده است, داده خواهد شد از خواندن `system.one` جدول -این `system.one` جدول شامل دقیقا یک ردیف است (این جدول همان هدف را به عنوان جدول دوگانه موجود در سایر دساماسها انجام می دهد). - -این `FROM` بند منبع برای خواندن داده ها از مشخص: - -- جدول -- خرده فروشی -- [تابع جدول](../table-functions/index.md#table-functions) - -`ARRAY JOIN` و به طور منظم `JOIN` همچنین ممکن است شامل شود (پایین را ببینید). - -به جای یک جدول `SELECT` خرده فروشی ممکن است در پرانتز مشخص. -در مقابل به گذاشتن استاندارد, مترادف نیازی به پس از یک خرده فروشی مشخص شود. - -برای اجرای پرس و جو تمام ستون های ذکر شده در پرس و جو از جدول مناسب استخراج می شوند. هر ستون برای پرس و جو خارجی مورد نیاز نیست از کارخانه های فرعی پرتاب می شود. -اگر پرس و جو هیچ ستون لیست نیست (به عنوان مثال, `SELECT count() FROM t`), برخی از ستون از جدول استخراج به هر حال (کوچکترین ترجیح داده می شود), به منظور محاسبه تعداد ردیف. - -#### تغییردهنده نهایی {#select-from-final} - -قابل استفاده در هنگام انتخاب داده ها از جداول از [ادغام](../../engines/table-engines/mergetree-family/mergetree.md)- خانواده موتور غیر از `GraphiteMergeTree`. چه زمانی `FINAL` مشخص شده است, تاتر به طور کامل ادغام داده ها قبل از بازگشت به نتیجه و در نتیجه انجام تمام تحولات داده که در طول ادغام برای موتور جدول داده شده اتفاق می افتد. - -همچنین برای پشتیبانی: -- [تکرار](../../engines/table-engines/mergetree-family/replication.md) نسخه های `MergeTree` موتورها. -- [نما](../../engines/table-engines/special/view.md), [بافر](../../engines/table-engines/special/buffer.md), [توزیع شده](../../engines/table-engines/special/distributed.md) و [ماده بینی](../../engines/table-engines/special/materializedview.md) موتورها که بیش از موتورهای دیگر کار می کنند به شرطی که بیش از ایجاد شده اند `MergeTree`- جدول موتور . - -نمایش داده شد که با استفاده از `FINAL` اعدام به همان سرعتی که نمایش داده شد مشابه که نمی, زیرا: - -- پرس و جو در یک موضوع اجرا و داده ها در طول اجرای پرس و جو با هم ادغام شدند. -- نمایش داده شد با `FINAL` خوانده شده ستون کلید اولیه در علاوه بر این به ستون مشخص شده در پرس و جو. - -در بیشتر موارد, اجتناب از استفاده از `FINAL`. - -### بند نمونه {#select-sample-clause} - -این `SAMPLE` بند اجازه می دهد تا برای پردازش پرس و جو تقریبی. - -هنگامی که نمونه گیری داده ها فعال است, پرس و جو بر روی تمام داده ها انجام نمی, اما تنها در بخش خاصی از داده ها (نمونه). مثلا, اگر شما نیاز به محاسبه ارقام برای تمام بازدیدکننده داشته است, کافی است برای اجرای پرس و جو در 1/10 کسری از تمام بازدیدکننده داشته است و سپس ضرب در نتیجه 10. - -پردازش پرس و جو تقریبی می تواند در موارد زیر مفید باشد: - -- هنگامی که شما شرایط زمان بندی دقیق (مانند \<100 مگابایت) اما شما نمی توانید هزینه منابع سخت افزاری اضافی را برای دیدار با خود توجیه کنید. -- هنگامی که داده های خام خود را دقیق نیست, بنابراین تقریب می کند به طرز محسوسی کاهش کیفیت. -- کسب و کار مورد نیاز هدف قرار دادن نتایج تقریبی (برای مقرون به صرفه بودن, و یا به منظور بازار نتایج دقیق به کاربران حق بیمه). - -!!! note "یادداشت" - شما فقط می توانید نمونه برداری با استفاده از جداول در [ادغام](../../engines/table-engines/mergetree-family/mergetree.md) خانواده, و تنها در صورتی که بیان نمونه برداری در ایجاد جدول مشخص شد (دیدن [موتور ادغام](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table)). - -ویژگی های نمونه گیری داده ها به شرح زیر است: - -- نمونهگیری دادهها یک مکانیسم قطعی است. نتیجه همان `SELECT .. SAMPLE` پرس و جو همیشه یکسان است. -- نمونه گیری به طور مداوم برای جداول مختلف کار می کند. برای جداول با یک کلید نمونه برداری تک, یک نمونه با ضریب همان همیشه زیر مجموعه همان داده های ممکن را انتخاب. برای مثال یک نمونه از شناسه های کاربر طول می کشد ردیف با همان زیر مجموعه از همه ممکن است شناسه کاربر از جداول مختلف. این به این معنی است که شما می توانید نمونه در کارخانه های فرعی در استفاده از [IN](#select-in-operators) بند بند. همچنین شما می توانید نمونه ها را با استفاده از [JOIN](#select-join) بند بند. -- نمونه گیری اجازه می دهد تا خواندن اطلاعات کمتر از یک دیسک. توجه داشته باشید که شما باید کلید نمونه برداری به درستی مشخص کنید. برای کسب اطلاعات بیشتر, دیدن [ایجاد یک جدول ادغام](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table). - -برای `SAMPLE` بند نحو زیر پشتیبانی می شود: - -| SAMPLE Clause Syntax | توصیف | -|----------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `SAMPLE k` | اینجا `k` است تعداد از 0 به 1.
پرس و جو در اجرا `k` کسری از داده ها. به عنوان مثال, `SAMPLE 0.1` پرس و جو را در 10 درصد از داده ها اجرا می کند. [ادامه مطلب](#select-sample-k) | -| `SAMPLE n` | اینجا `n` عدد صحیح به اندازه کافی بزرگ است.
پرس و جو بر روی یک نمونه از حداقل اعدام `n` ردیف (اما نه به طور قابل توجهی بیشتر از این). به عنوان مثال, `SAMPLE 10000000` پرس و جو را در حداقل ردیف های 10000000 اجرا می کند. [ادامه مطلب](#select-sample-n) | -| `SAMPLE k OFFSET m` | اینجا `k` و `m` اعداد از 0 به 1.
پرس و جو بر روی یک نمونه از اعدام `k` کسری از داده ها. داده های مورد استفاده برای نمونه توسط جبران `m` کسر کردن. [ادامه مطلب](#select-sample-offset) | - -#### SAMPLE K {#select-sample-k} - -اینجا `k` است تعداد از 0 به 1 (هر دو نمادهای کسری و اعشاری پشتیبانی می شوند). به عنوان مثال, `SAMPLE 1/2` یا `SAMPLE 0.5`. - -در یک `SAMPLE k` بند, نمونه از گرفته `k` کسری از داده ها. مثال زیر نشان داده شده است: - -``` sql -SELECT - Title, - count() * 10 AS PageViews -FROM hits_distributed -SAMPLE 0.1 -WHERE - CounterID = 34 -GROUP BY Title -ORDER BY PageViews DESC LIMIT 1000 -``` - -در این مثال پرس و جو اجرا شده است در یک نمونه از 0.1 (10%) از داده ها. ارزش توابع دانه ها به طور خودکار اصلاح نمی, بنابراین برای دریافت یک نتیجه تقریبی, ارزش `count()` به صورت دستی توسط ضرب 10. - -#### SAMPLE N {#select-sample-n} - -اینجا `n` عدد صحیح به اندازه کافی بزرگ است. به عنوان مثال, `SAMPLE 10000000`. - -در این مورد, پرس و جو بر روی یک نمونه از حداقل اعدام `n` ردیف (اما نه به طور قابل توجهی بیشتر از این). به عنوان مثال, `SAMPLE 10000000` پرس و جو را در حداقل ردیف های 10000000 اجرا می کند. - -از حداقل واحد برای خواندن داده ها یک گرانول است (اندازه خود را توسط مجموعه `index_granularity` تنظیم), این را حس می کند به مجموعه ای از یک نمونه است که بسیار بزرگتر از اندازه گرانول. - -هنگام استفاده از `SAMPLE n` بند, شما نمی دانید که درصد نسبی داده پردازش شد. بنابراین شما نمی دانید ضریب توابع کل باید توسط ضرب. استفاده از `_sample_factor` ستون مجازی برای دریافت نتیجه تقریبی. - -این `_sample_factor` ستون شامل ضرایب نسبی است که به صورت پویا محاسبه می شود. این ستون به طور خودکار ایجاد زمانی که شما [ایجاد](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table) یک جدول با کلید نمونه گیری مشخص. نمونه های استفاده از `_sample_factor` ستون در زیر نشان داده شده. - -بیایید جدول را در نظر بگیریم `visits`, که شامل ارقام در مورد بازدیدکننده داشته است سایت. مثال اول نشان می دهد که چگونه برای محاسبه تعداد بازدید از صفحه: - -``` sql -SELECT sum(PageViews * _sample_factor) -FROM visits -SAMPLE 10000000 -``` - -مثال بعدی نشان می دهد که چگونه برای محاسبه تعداد کل بازدیدکننده داشته است: - -``` sql -SELECT sum(_sample_factor) -FROM visits -SAMPLE 10000000 -``` - -مثال زیر نشان می دهد که چگونه برای محاسبه مدت زمان جلسه به طور متوسط. توجه داشته باشید که شما لازم نیست به استفاده از ضریب نسبی برای محاسبه مقادیر متوسط. - -``` sql -SELECT avg(Duration) -FROM visits -SAMPLE 10000000 -``` - -#### SAMPLE K OFFSET M {#select-sample-offset} - -اینجا `k` و `m` اعداد از 0 به 1. نمونه های زیر نشان داده شده. - -**مثال 1** - -``` sql -SAMPLE 1/10 -``` - -در این مثال نمونه 1 / 10 از تمام داده ها است: - -`[++------------]` - -**مثال 2** - -``` sql -SAMPLE 1/10 OFFSET 1/2 -``` - -در اینجا یک نمونه از 10 درصد گرفته شده از نیمه دوم از داده ها. - -`[------++------]` - -### مجموعه پیوستن بند {#select-array-join-clause} - -اجازه می دهد تا اجرای `JOIN` با یک مجموعه یا ساختار داده های تو در تو. قصد شبیه به است [ارریجین](../functions/array-join.md#functions_arrayjoin) تابع, اما قابلیت های خود را گسترده تر است. - -``` sql -SELECT -FROM -[LEFT] ARRAY JOIN -[WHERE|PREWHERE ] -... -``` - -شما می توانید تنها یک مشخص `ARRAY JOIN` بند در یک پرس و جو. - -سفارش اجرای پرس و جو در هنگام اجرا بهینه شده است `ARRAY JOIN`. اگرچه `ARRAY JOIN` همیشه باید قبل از مشخص شود `WHERE/PREWHERE` بند, این می تواند انجام شود یا قبل از `WHERE/PREWHERE` (اگر نتیجه در این بند مورد نیاز است), و یا پس از اتمام (برای کاهش حجم محاسبات). سفارش پردازش توسط بهینه ساز پرس و جو کنترل می شود. - -انواع پشتیبانی شده از `ARRAY JOIN` به شرح زیر است: - -- `ARRAY JOIN` - در این مورد, بند خالی در نتیجه شامل نمی شود `JOIN`. -- `LEFT ARRAY JOIN` - نتیجه `JOIN` شامل ردیف با ارریس خالی است. مقدار برای یک مجموعه خالی است به مقدار پیش فرض برای نوع عنصر مجموعه (معمولا 0, رشته خالی و یا تهی). - -نمونه های زیر نشان می دهد استفاده از `ARRAY JOIN` و `LEFT ARRAY JOIN` بند. بیایید یک جدول با یک [& حذف](../../sql-reference/data-types/array.md) ستون را تایپ کنید و مقادیر را وارد کنید: - -``` sql -CREATE TABLE arrays_test -( - s String, - arr Array(UInt8) -) ENGINE = Memory; - -INSERT INTO arrays_test -VALUES ('Hello', [1,2]), ('World', [3,4,5]), ('Goodbye', []); -``` - -``` text -┌─s───────────┬─arr─────┐ -│ Hello │ [1,2] │ -│ World │ [3,4,5] │ -│ Goodbye │ [] │ -└─────────────┴─────────┘ -``` - -مثال زیر از `ARRAY JOIN` بند: - -``` sql -SELECT s, arr -FROM arrays_test -ARRAY JOIN arr; -``` - -``` text -┌─s─────┬─arr─┐ -│ Hello │ 1 │ -│ Hello │ 2 │ -│ World │ 3 │ -│ World │ 4 │ -│ World │ 5 │ -└───────┴─────┘ -``` - -مثال بعدی با استفاده از `LEFT ARRAY JOIN` بند: - -``` sql -SELECT s, arr -FROM arrays_test -LEFT ARRAY JOIN arr; -``` - -``` text -┌─s───────────┬─arr─┐ -│ Hello │ 1 │ -│ Hello │ 2 │ -│ World │ 3 │ -│ World │ 4 │ -│ World │ 5 │ -│ Goodbye │ 0 │ -└─────────────┴─────┘ -``` - -#### استفاده از نام مستعار {#using-aliases} - -یک نام مستعار می تواند برای مجموعه ای در `ARRAY JOIN` بند بند. در این مورد, یک مورد مجموعه ای را می توان با این نام مستعار دیده, اما مجموعه خود را با نام اصلی قابل دسترسی. مثال: - -``` sql -SELECT s, arr, a -FROM arrays_test -ARRAY JOIN arr AS a; -``` - -``` text -┌─s─────┬─arr─────┬─a─┐ -│ Hello │ [1,2] │ 1 │ -│ Hello │ [1,2] │ 2 │ -│ World │ [3,4,5] │ 3 │ -│ World │ [3,4,5] │ 4 │ -│ World │ [3,4,5] │ 5 │ -└───────┴─────────┴───┘ -``` - -با استفاده از نام مستعار, شما می توانید انجام `ARRAY JOIN` با یک مجموعه خارجی. به عنوان مثال: - -``` sql -SELECT s, arr_external -FROM arrays_test -ARRAY JOIN [1, 2, 3] AS arr_external; -``` - -``` text -┌─s───────────┬─arr_external─┐ -│ Hello │ 1 │ -│ Hello │ 2 │ -│ Hello │ 3 │ -│ World │ 1 │ -│ World │ 2 │ -│ World │ 3 │ -│ Goodbye │ 1 │ -│ Goodbye │ 2 │ -│ Goodbye │ 3 │ -└─────────────┴──────────────┘ -``` - -ارریس های متعدد را می توان با کاما از هم جدا در `ARRAY JOIN` بند بند. در این مورد, `JOIN` به طور همزمان (مجموع مستقیم و نه محصول دکارتی) انجام می شود. توجه داشته باشید که تمام ارریس باید به همان اندازه. مثال: - -``` sql -SELECT s, arr, a, num, mapped -FROM arrays_test -ARRAY JOIN arr AS a, arrayEnumerate(arr) AS num, arrayMap(x -> x + 1, arr) AS mapped; -``` - -``` text -┌─s─────┬─arr─────┬─a─┬─num─┬─mapped─┐ -│ Hello │ [1,2] │ 1 │ 1 │ 2 │ -│ Hello │ [1,2] │ 2 │ 2 │ 3 │ -│ World │ [3,4,5] │ 3 │ 1 │ 4 │ -│ World │ [3,4,5] │ 4 │ 2 │ 5 │ -│ World │ [3,4,5] │ 5 │ 3 │ 6 │ -└───────┴─────────┴───┴─────┴────────┘ -``` - -مثال زیر از [شناسه بسته:](../../sql-reference/functions/array-functions.md#array_functions-arrayenumerate) تابع: - -``` sql -SELECT s, arr, a, num, arrayEnumerate(arr) -FROM arrays_test -ARRAY JOIN arr AS a, arrayEnumerate(arr) AS num; -``` - -``` text -┌─s─────┬─arr─────┬─a─┬─num─┬─arrayEnumerate(arr)─┐ -│ Hello │ [1,2] │ 1 │ 1 │ [1,2] │ -│ Hello │ [1,2] │ 2 │ 2 │ [1,2] │ -│ World │ [3,4,5] │ 3 │ 1 │ [1,2,3] │ -│ World │ [3,4,5] │ 4 │ 2 │ [1,2,3] │ -│ World │ [3,4,5] │ 5 │ 3 │ [1,2,3] │ -└───────┴─────────┴───┴─────┴─────────────────────┘ -``` - -#### مجموعه پیوستن با ساختار داده های تو در تو {#array-join-with-nested-data-structure} - -`ARRAY`پیوستن " همچنین با این نسخهها کار [ساختارهای داده تو در تو](../../sql-reference/data-types/nested-data-structures/nested.md). مثال: - -``` sql -CREATE TABLE nested_test -( - s String, - nest Nested( - x UInt8, - y UInt32) -) ENGINE = Memory; - -INSERT INTO nested_test -VALUES ('Hello', [1,2], [10,20]), ('World', [3,4,5], [30,40,50]), ('Goodbye', [], []); -``` - -``` text -┌─s───────┬─nest.x──┬─nest.y─────┐ -│ Hello │ [1,2] │ [10,20] │ -│ World │ [3,4,5] │ [30,40,50] │ -│ Goodbye │ [] │ [] │ -└─────────┴─────────┴────────────┘ -``` - -``` sql -SELECT s, `nest.x`, `nest.y` -FROM nested_test -ARRAY JOIN nest; -``` - -``` text -┌─s─────┬─nest.x─┬─nest.y─┐ -│ Hello │ 1 │ 10 │ -│ Hello │ 2 │ 20 │ -│ World │ 3 │ 30 │ -│ World │ 4 │ 40 │ -│ World │ 5 │ 50 │ -└───────┴────────┴────────┘ -``` - -هنگام مشخص کردن نام ساختارهای داده های تو در تو در `ARRAY JOIN`, معنای همان است که `ARRAY JOIN` با تمام عناصر مجموعه ای که شامل. نمونه به شرح زیر است: - -``` sql -SELECT s, `nest.x`, `nest.y` -FROM nested_test -ARRAY JOIN `nest.x`, `nest.y`; -``` - -``` text -┌─s─────┬─nest.x─┬─nest.y─┐ -│ Hello │ 1 │ 10 │ -│ Hello │ 2 │ 20 │ -│ World │ 3 │ 30 │ -│ World │ 4 │ 40 │ -│ World │ 5 │ 50 │ -└───────┴────────┴────────┘ -``` - -این تنوع نیز حس می کند: - -``` sql -SELECT s, `nest.x`, `nest.y` -FROM nested_test -ARRAY JOIN `nest.x`; -``` - -``` text -┌─s─────┬─nest.x─┬─nest.y─────┐ -│ Hello │ 1 │ [10,20] │ -│ Hello │ 2 │ [10,20] │ -│ World │ 3 │ [30,40,50] │ -│ World │ 4 │ [30,40,50] │ -│ World │ 5 │ [30,40,50] │ -└───────┴────────┴────────────┘ -``` - -نام مستعار ممکن است برای یک ساختار داده های تو در تو استفاده می شود, به منظور انتخاب هر دو `JOIN` نتیجه یا مجموعه منبع. مثال: - -``` sql -SELECT s, `n.x`, `n.y`, `nest.x`, `nest.y` -FROM nested_test -ARRAY JOIN nest AS n; -``` - -``` text -┌─s─────┬─n.x─┬─n.y─┬─nest.x──┬─nest.y─────┐ -│ Hello │ 1 │ 10 │ [1,2] │ [10,20] │ -│ Hello │ 2 │ 20 │ [1,2] │ [10,20] │ -│ World │ 3 │ 30 │ [3,4,5] │ [30,40,50] │ -│ World │ 4 │ 40 │ [3,4,5] │ [30,40,50] │ -│ World │ 5 │ 50 │ [3,4,5] │ [30,40,50] │ -└───────┴─────┴─────┴─────────┴────────────┘ -``` - -نمونه ای از استفاده از [شناسه بسته:](../../sql-reference/functions/array-functions.md#array_functions-arrayenumerate) تابع: - -``` sql -SELECT s, `n.x`, `n.y`, `nest.x`, `nest.y`, num -FROM nested_test -ARRAY JOIN nest AS n, arrayEnumerate(`nest.x`) AS num; -``` - -``` text -┌─s─────┬─n.x─┬─n.y─┬─nest.x──┬─nest.y─────┬─num─┐ -│ Hello │ 1 │ 10 │ [1,2] │ [10,20] │ 1 │ -│ Hello │ 2 │ 20 │ [1,2] │ [10,20] │ 2 │ -│ World │ 3 │ 30 │ [3,4,5] │ [30,40,50] │ 1 │ -│ World │ 4 │ 40 │ [3,4,5] │ [30,40,50] │ 2 │ -│ World │ 5 │ 50 │ [3,4,5] │ [30,40,50] │ 3 │ -└───────┴─────┴─────┴─────────┴────────────┴─────┘ -``` - -### پیوستن بند {#select-join} - -می پیوندد داده ها در عادی [SQL JOIN](https://en.wikipedia.org/wiki/Join_(SQL)) با عقل. - -!!! info "یادداشت" - نه مربوط به [ARRAY JOIN](#select-array-join-clause). - -``` sql -SELECT -FROM -[GLOBAL] [ANY|ALL] [INNER|LEFT|RIGHT|FULL|CROSS] [OUTER] JOIN -(ON )|(USING ) ... -``` - -نام جدول را می توان به جای مشخص `` و ``. این معادل است `SELECT * FROM table` خرده فروشی, به جز در یک مورد خاص زمانی که جدول است [پیوستن](../../engines/table-engines/special/join.md) engine – an array prepared for joining. - -#### انواع پشتیبانی شده از `JOIN` {#select-join-types} - -- `INNER JOIN` (یا `JOIN`) -- `LEFT JOIN` (یا `LEFT OUTER JOIN`) -- `RIGHT JOIN` (یا `RIGHT OUTER JOIN`) -- `FULL JOIN` (یا `FULL OUTER JOIN`) -- `CROSS JOIN` (یا `,` ) - -مشاهده استاندارد [SQL JOIN](https://en.wikipedia.org/wiki/Join_(SQL)) توصیف. - -#### چند پیوستن {#multiple-join} - -انجام نمایش داده شد, بازنویسی کلیک خانه چند جدول می پیوندد به دنباله ای از دو جدول می پیوندد. مثلا, اگر چهار جدول برای عضویت کلیک خانه می پیوندد اول و دوم وجود دارد, سپس نتیجه می پیوندد با جدول سوم, و در مرحله گذشته, می پیوندد یک چهارم. - -اگر پرس و جو شامل `WHERE` بند ClickHouse تلاش می کند به pushdown فیلتر از این بند از طریق متوسط پیوستن به. اگر می تواند فیلتر به هر ملحق متوسط اعمال می شود, تاتر اعمال فیلتر بعد از همه می پیوندد به پایان رسید. - -ما توصیه می کنیم `JOIN ON` یا `JOIN USING` نحو برای ایجاد نمایش داده شد. به عنوان مثال: - -``` sql -SELECT * FROM t1 JOIN t2 ON t1.a = t2.a JOIN t3 ON t1.a = t3.a -``` - -شما می توانید لیست کاما از هم جدا از جداول در استفاده از `FROM` بند بند. به عنوان مثال: - -``` sql -SELECT * FROM t1, t2, t3 WHERE t1.a = t2.a AND t1.a = t3.a -``` - -این سینتکس را مخلوط نکنید. - -کلیکهاوس مستقیما از دستورات ارتباطی با کاما پشتیبانی نمی کند بنابراین توصیه نمی کنیم از انها استفاده کنید. الگوریتم تلاش می کند به بازنویسی پرس و جو از نظر `CROSS JOIN` و `INNER JOIN` بند و سپس به پرس و جو پردازش. هنگامی که بازنویسی پرس و جو, تاتر تلاش می کند برای بهینه سازی عملکرد و مصرف حافظه. به طور پیش فرض, تاتر رفتار کاما به عنوان یک `INNER JOIN` بند و تبدیل `INNER JOIN` به `CROSS JOIN` هنگامی که الگوریتم نمی تواند تضمین کند که `INNER JOIN` بازگرداندن اطلاعات مورد نیاز. - -#### سختی {#select-join-strictness} - -- `ALL` — If the right table has several matching rows, ClickHouse creates a [محصول دکارتی](https://en.wikipedia.org/wiki/Cartesian_product) از تطبیق ردیف. این استاندارد است `JOIN` رفتار در گذاشتن. -- `ANY` — If the right table has several matching rows, only the first one found is joined. If the right table has only one matching row, the results of queries with `ANY` و `ALL` کلمات کلیدی یکسان هستند. -- `ASOF` — For joining sequences with a non-exact match. `ASOF JOIN` استفاده در زیر توضیح داده شده است. - -**از این رو پیوستن به استفاده** - -`ASOF JOIN` مفید است زمانی که شما نیاز به پیوستن به سوابق که هیچ بازی دقیق. - -جداول برای `ASOF JOIN` باید یک ستون توالی دستور داده اند. این ستون نمی تواند به تنهایی در یک جدول باشد و باید یکی از انواع داده ها باشد: `UInt32`, `UInt64`, `Float32`, `Float64`, `Date` و `DateTime`. - -نحو `ASOF JOIN ... ON`: - -``` sql -SELECT expressions_list -FROM table_1 -ASOF LEFT JOIN table_2 -ON equi_cond AND closest_match_cond -``` - -شما می توانید هر تعداد از شرایط برابری و دقیقا یکی از نزدیک ترین شرایط بازی استفاده کنید. به عنوان مثال, `SELECT count() FROM table_1 ASOF LEFT JOIN table_2 ON table_1.a == table_2.b AND table_2.t <= table_1.t`. - -شرایط پشتیبانی شده برای نزدیک ترین بازی: `>`, `>=`, `<`, `<=`. - -نحو `ASOF JOIN ... USING`: - -``` sql -SELECT expressions_list -FROM table_1 -ASOF JOIN table_2 -USING (equi_column1, ... equi_columnN, asof_column) -``` - -`ASOF JOIN` استفاده `equi_columnX` برای پیوستن به برابری و `asof_column` برای پیوستن به نزدیک ترین مسابقه با `table_1.asof_column >= table_2.asof_column` شرط. این `asof_column` ستون همیشه یکی از گذشته در `USING` بند بند. - -مثلا, جداول زیر را در نظر بگیرید: - - table_1 table_2 - event | ev_time | user_id event | ev_time | user_id - ----------|---------|---------- ----------|---------|---------- - ... ... - event_1_1 | 12:00 | 42 event_2_1 | 11:59 | 42 - ... event_2_2 | 12:30 | 42 - event_1_2 | 13:00 | 42 event_2_3 | 13:00 | 42 - ... ... - -`ASOF JOIN` می توانید برچسب زمان از یک رویداد کاربر از را `table_1` و پیدا کردن یک رویداد در `table_2` جایی که برچسب زمان نزدیک به زمان این رویداد از `table_1` مربوط به نزدیک ترین شرایط بازی. مقادیر برچسب زمان برابر نزدیک ترین در صورت موجود بودن. اینجا `user_id` ستون را می توان برای پیوستن به برابری و `ev_time` ستون را می توان برای پیوستن به در نزدیک ترین بازی استفاده می شود. در مثال ما, `event_1_1` می توان با پیوست `event_2_1` و `event_1_2` می توان با پیوست `event_2_3` اما `event_2_2` نمیشه عضو شد - -!!! note "یادداشت" - `ASOF` پیوستن است **نه** پردازشگر پشتیبانی شده: [پیوستن](../../engines/table-engines/special/join.md) موتور جدول. - -برای تنظیم مقدار سختگیرانه پیش فرض, استفاده از پارامتر پیکربندی جلسه [بررسی اجمالی](../../operations/settings/settings.md#settings-join_default_strictness). - -#### GLOBAL JOIN {#global-join} - -هنگام استفاده از نرمال `JOIN` پرس و جو به سرورهای راه دور ارسال می شود. زیرکریزها روی هر کدام اجرا می شوند تا میز مناسب را بسازند و پیوستن با این جدول انجام می شود. به عبارت دیگر, جدول سمت راست بر روی هر سرور تشکیل به طور جداگانه. - -هنگام استفاده از `GLOBAL ... JOIN`, اول سرور درخواست کننده اجرا می شود یک خرده فروشی برای محاسبه جدول سمت راست. این جدول موقت به هر سرور از راه دور منتقل می شود و نمایش داده می شود با استفاده از داده های موقت منتقل می شود. - -مراقب باشید در هنگام استفاده از `GLOBAL`. برای کسب اطلاعات بیشتر به بخش مراجعه کنید [توزیع subqueries](#select-distributed-subqueries). - -#### توصیه های استفاده {#usage-recommendations} - -هنگامی که در حال اجرا `JOIN` بهینه سازی سفارش اعدام در رابطه با سایر مراحل پرس و جو وجود ندارد. پیوستن (جستجو در جدول سمت راست) قبل از فیلتر کردن در اجرا می شود `WHERE` و قبل از تجمع. به منظور صراحت تنظیم سفارش پردازش, توصیه می کنیم در حال اجرا یک `JOIN` خرده فروشی با یک خرده فروشی. - -مثال: - -``` sql -SELECT - CounterID, - hits, - visits -FROM -( - SELECT - CounterID, - count() AS hits - FROM test.hits - GROUP BY CounterID -) ANY LEFT JOIN -( - SELECT - CounterID, - sum(Sign) AS visits - FROM test.visits - GROUP BY CounterID -) USING CounterID -ORDER BY hits DESC -LIMIT 10 -``` - -``` text -┌─CounterID─┬───hits─┬─visits─┐ -│ 1143050 │ 523264 │ 13665 │ -│ 731962 │ 475698 │ 102716 │ -│ 722545 │ 337212 │ 108187 │ -│ 722889 │ 252197 │ 10547 │ -│ 2237260 │ 196036 │ 9522 │ -│ 23057320 │ 147211 │ 7689 │ -│ 722818 │ 90109 │ 17847 │ -│ 48221 │ 85379 │ 4652 │ -│ 19762435 │ 77807 │ 7026 │ -│ 722884 │ 77492 │ 11056 │ -└───────────┴────────┴────────┘ -``` - -زیرمجموعه ها به شما اجازه نمی دهند نام ها را تنظیم کنید یا از یک ستون از یک زیر اندازه خاص استفاده کنید. -ستون های مشخص شده در `USING` باید نام های مشابه در هر دو کارخانه های فرعی دارند, و ستون های دیگر باید متفاوت به نام. شما می توانید نام مستعار برای تغییر نام ستون در زیرکریز استفاده (به عنوان مثال با استفاده از نام مستعار `hits` و `visits`). - -این `USING` بند یک یا چند ستون برای پیوستن به مشخص, که ایجاد برابری این ستون. لیست ستون ها بدون براکت تنظیم شده است. شرایط پیوستن پیچیده تر پشتیبانی نمی شوند. - -جدول سمت راست (نتیجه زیرخاکی) ساکن در رم. اگر حافظه کافی وجود ندارد, شما می توانید یک اجرا کنید `JOIN`. - -هر بار که پرس و جو با همان اجرا شود `JOIN`, خرده فروشی است دوباره اجرا به دلیل نتیجه ذخیره سازی نیست. برای جلوگیری از این, استفاده از ویژه [پیوستن](../../engines/table-engines/special/join.md) موتور جدول, که مجموعه ای تهیه شده برای پیوستن است که همیشه در رم. - -در بعضی موارد کارایی بیشتری برای استفاده دارد `IN` به جای `JOIN`. -در میان انواع مختلف `JOIN`, موثر ترین است `ANY LEFT JOIN` پس `ANY INNER JOIN`. کمترین کارایی عبارتند از `ALL LEFT JOIN` و `ALL INNER JOIN`. - -اگر شما نیاز به یک `JOIN` برای پیوستن به جداول بعد (این جداول نسبتا کوچک است که شامل خواص ابعاد هستند, مانند نام برای کمپین های تبلیغاتی), یک `JOIN` ممکن است بسیار مناسب با توجه به این واقعیت است که جدول سمت راست برای هر پرس و جو دوباره قابل دسترسی است. برای چنین مواردی وجود دارد “external dictionaries” ویژگی است که شما باید به جای استفاده از `JOIN`. برای کسب اطلاعات بیشتر به بخش مراجعه کنید [واژهنامهها خارجی](../dictionaries/external-dictionaries/external-dicts.md). - -**محدودیت حافظه** - -تاتر با استفاده از [هش پیوستن](https://en.wikipedia.org/wiki/Hash_join) الگوریتم. تاتر طول می کشد `` و یک جدول هش را در رم ایجاد می کند. اگر شما نیاز به محدود کردن پیوستن به مصرف حافظه عملیات استفاده از تنظیمات زیر: - -- [\_پاک کردن \_روشن گرافیک](../../operations/settings/query-complexity.md#settings-max_rows_in_join) — Limits number of rows in the hash table. -- [\_پویش همیشگی](../../operations/settings/query-complexity.md#settings-max_bytes_in_join) — Limits size of the hash table. - -هنگامی که هر یک از این محدودیت رسیده است, کلیک به عنوان عمل می کند [\_شروع مجدد](../../operations/settings/query-complexity.md#settings-join_overflow_mode) تنظیم دستور. - -#### پردازش سلولهای خالی یا خالی {#processing-of-empty-or-null-cells} - -در حالی که پیوستن به جداول سلول های خالی ظاهر می شود. تنظیمات [ارزشهای خبری عبارتند از:](../../operations/settings/settings.md#join_use_nulls) تعریف چگونه خانه را پر می کند این سلول ها. - -اگر `JOIN` کلید ها [Nullable](../data-types/nullable.md) زمینه, ردیف که حداقل یکی از کلید های دارای ارزش [NULL](../syntax.md#null-literal) عضو نشده. - -#### محدودیت نحو {#syntax-limitations} - -برای چند `JOIN` بند در یک `SELECT` پرسوجو: - -- گرفتن تمام ستون ها از طریق `*` در دسترس است تنها اگر جداول پیوست, نمی فرعی. -- این `PREWHERE` بند در دسترس نیست. - -برای `ON`, `WHERE` و `GROUP BY` بند: - -- عبارات دلخواه را نمی توان در `ON`, `WHERE` و `GROUP BY` بند, اما شما می توانید یک عبارت در یک تعریف `SELECT` بند و سپس در این بند از طریق یک نام مستعار استفاده کنید. - -### بند کجا {#select-where} - -در صورتی که بند جایی وجود دارد, باید بیان با نوع زیرپوش شامل8. این است که معمولا بیان با مقایسه و اپراتورهای منطقی. -این عبارت خواهد شد برای فیلتر کردن داده ها قبل از همه تحولات دیگر استفاده می شود. - -اگر شاخص توسط موتور جدول پایگاه داده پشتیبانی, بیان بر توانایی استفاده از شاخص ارزیابی. - -### در بند {#prewhere-clause} - -این بند همان معنی که بند است. تفاوت در این است که داده ها از جدول خوانده می شوند. -هنگامی که با استفاده از PREWHERE اول تنها ستون لازم برای اجرای PREWHERE در حال خواندن. سپس ستون های دیگر خوانده می شوند که برای اجرای پرس و جو مورد نیاز است اما تنها بلوک هایی که بیان پیشین درست است. - -این را حس می کند به استفاده از همه جا اگر شرایط فیلتراسیون که توسط یک اقلیت از ستون ها در پرس و جو استفاده می شود وجود دارد, اما که فیلتراسیون داده های قوی. این باعث کاهش حجم داده ها به خواندن. - -مثلا, مفید است برای نوشتن از کجا برای نمایش داده شد که استخراج تعداد زیادی از ستون, اما این تنها فیلتراسیون برای چند ستون دارند. - -همه جا تنها با جداول از پشتیبانی `*MergeTree` خانواده - -یک پرس و جو ممکن است به طور همزمان مشخص PREWHERE و در آن. در این مورد PREWHERE پیش می آید که در آن. - -اگر ‘optimize\_move\_to\_prewhere’ تنظیم به 1 و PREWHERE حذف شده از سیستم با استفاده از ابتکارات به طور خودکار حرکت قطعات از عبارات از کجا PREWHERE. - -### گروه بر اساس بند {#select-group-by-clause} - -این یکی از مهم ترین بخش های سندرم تونل کارپ ستون گرا است. - -در صورتی که یک گروه بند وجود دارد, باید یک لیست از عبارات حاوی. هر عبارت خواهد شد به اینجا به عنوان یک اشاره “key”. -همه عبارات در انتخاب, داشتن, و سفارش توسط بند باید از کلید و یا از توابع کل محاسبه می شود. به عبارت دیگر, هر ستون انتخاب شده از جدول باید یا در کلید و یا در داخل توابع دانه استفاده می شود. - -اگر یک پرس و جو شامل تنها ستون جدول در داخل توابع کل, گروه بند را می توان حذف, و تجمع توسط یک مجموعه خالی از کلید فرض بر این است. - -مثال: - -``` sql -SELECT - count(), - median(FetchTiming > 60 ? 60 : FetchTiming), - count() - sum(Refresh) -FROM hits -``` - -با این حال, در مقابل به استاندارد گذاشتن, اگر جدول هیچ ردیف ندارد (یا هیچ در همه وجود ندارد, و یا هر پس از استفاده از جایی که برای فیلتر کردن وجود ندارد), یک نتیجه خالی بازگشته است, و نه در نتیجه از یکی از ردیف های حاوی مقادیر اولیه از توابع کل. - -همانطور که به خروجی زیر مخالف (و منطبق با استاندارد گذاشتن), شما می توانید برخی از ارزش برخی از ستون است که در یک کلید و یا کل تابع نیست (به جز عبارات ثابت). برای کار در اطراف این, شما می توانید با استفاده از ‘any’ تابع جمع (اولین مقدار مواجه می شوند) یا ‘min/max’. - -مثال: - -``` sql -SELECT - domainWithoutWWW(URL) AS domain, - count(), - any(Title) AS title -- getting the first occurred page header for each domain. -FROM hits -GROUP BY domain -``` - -برای هر مقدار کلیدی مختلف مواجه می شوند, گروه با محاسبه مجموعه ای از مقادیر تابع کل. - -گروه توسط ستون های مجموعه پشتیبانی نمی شود. - -ثابت را نمی توان به عنوان استدلال برای توابع کل مشخص شده است. مثال: مجموع(1). به جای این, شما می توانید از ثابت خلاص. مثال: `count()`. - -#### پردازش پوچ {#null-processing} - -برای گروه بندی, تفسیر کلیک [NULL](../syntax.md#null-literal) به عنوان یک ارزش, و `NULL=NULL`. - -در اینجا یک مثال برای نشان دادن این بدان معنی است. - -فرض کنید شما باید این جدول: - -``` text -┌─x─┬────y─┐ -│ 1 │ 2 │ -│ 2 │ ᴺᵁᴸᴸ │ -│ 3 │ 2 │ -│ 3 │ 3 │ -│ 3 │ ᴺᵁᴸᴸ │ -└───┴──────┘ -``` - -پرسوجو `SELECT sum(x), y FROM t_null_big GROUP BY y` نتایج در: - -``` text -┌─sum(x)─┬────y─┐ -│ 4 │ 2 │ -│ 3 │ 3 │ -│ 5 │ ᴺᵁᴸᴸ │ -└────────┴──────┘ -``` - -شما می توانید ببینید که `GROUP BY` برای `y = NULL` خلاصه تا `x`, به عنوان اگر `NULL` این مقدار است. - -اگر شما تصویب چند کلید به `GROUP BY`, نتیجه به شما تمام ترکیبی از انتخاب را, اگر `NULL` یک مقدار خاص بودند. - -#### با اصلاح کننده بالغ {#with-totals-modifier} - -اگر با اصلاح بالغ مشخص شده است, ردیف دیگر محاسبه خواهد شد. این ردیف ستون های کلیدی حاوی مقادیر پیش فرض (صفر یا خطوط خالی) و ستون های توابع جمع شده با مقادیر محاسبه شده در تمام ردیف ها ( “total” ارزش ها). - -این ردیف اضافی خروجی در جانسون است\*, تابسپار\*, و زیبا \* فرمت, به طور جداگانه از ردیف های دیگر. در فرمت های دیگر, این ردیف خروجی نیست. - -در جانسون \* فرمت های, این ردیف خروجی به عنوان یک جداگانه است ‘totals’ رشته. در جدولپار\* فرمت های, ردیف پس از نتیجه اصلی, قبل از یک ردیف خالی (پس از داده های دیگر). در زیبا \* فرمت های ردیف خروجی به عنوان یک جدول جداگانه پس از نتیجه اصلی است. - -`WITH TOTALS` می توان در راه های مختلف اجرا زمانی که داشتن حاضر است. رفتار بستگی به ‘totals\_mode’ تنظیمات. -به طور پیش فرض, `totals_mode = 'before_having'`. در این مورد, ‘totals’ محاسبه شده است در تمام ردیف از جمله کسانی که عبور نمی کند از طریق داشتن و ‘max\_rows\_to\_group\_by’. - -جایگزین های دیگر شامل تنها ردیف است که از طریق داشتن در عبور ‘totals’ و رفتار متفاوت با تنظیم `max_rows_to_group_by` و `group_by_overflow_mode = 'any'`. - -`after_having_exclusive` – Don't include rows that didn't pass through `max_rows_to_group_by`. به عبارت دیگر, ‘totals’ کمتر از و یا به همان تعداد از ردیف به عنوان اگر داشته باشد `max_rows_to_group_by` حذف شد. - -`after_having_inclusive` – Include all the rows that didn't pass through ‘max\_rows\_to\_group\_by’ داخل ‘totals’. به عبارت دیگر, ‘totals’ بیش از و یا به همان تعداد از ردیف به عنوان اگر داشته باشد `max_rows_to_group_by` حذف شد. - -`after_having_auto` – Count the number of rows that passed through HAVING. If it is more than a certain amount (by default, 50%), include all the rows that didn't pass through ‘max\_rows\_to\_group\_by’ داخل ‘totals’. در غیر این صورت, را شامل نمی شود. - -`totals_auto_threshold` – By default, 0.5. The coefficient for `after_having_auto`. - -اگر `max_rows_to_group_by` و `group_by_overflow_mode = 'any'` استفاده نمی شود, تمام تغییرات از `after_having` یکسان هستند و شما می توانید از هر یک از این موارد استفاده کنید, `after_having_auto`). - -شما می توانید با استفاده از مجموع در subqueries از جمله subqueries در پیوستن به بند (در این مورد مربوطه مجموع ارزش ترکیب می شوند). - -#### گروه در حافظه خارجی {#select-group-by-in-external-memory} - -شما می توانید اطلاعات موقت تخلیه به دیسک را قادر به محدود کردن استفاده از حافظه در طول `GROUP BY`. -این [ا\_فزون\_بر\_گونهی\_گونهی زیر\_گروهها](../../operations/settings/settings.md#settings-max_bytes_before_external_group_by) تنظیم تعیین کننده مصرف رم را برای تخلیه می کند `GROUP BY` اطلاعات موقت به سیستم فایل. اگر به 0 (به طور پیش فرض), غیر فعال است. - -هنگام استفاده از `max_bytes_before_external_group_by`, توصیه می کنیم که به شما در تنظیم `max_memory_usage` در مورد دو برابر بالا. این لازم است زیرا دو مرحله برای تجمع وجود دارد: خواندن تاریخ و تشکیل داده های متوسط (1) و ادغام داده های متوسط (2). واژگون اطلاعات به سیستم فایل تنها می تواند در طول مرحله رخ می دهد 1. اگر داده های موقت ریخته نمی شد, سپس مرحله 2 ممکن است نیاز به همان مقدار از حافظه در مرحله 1. - -برای مثال اگر [\_کاساژ بیشینه](../../operations/settings/settings.md#settings_max_memory_usage) به 1000000000 تنظیم شد و شما می خواهید به استفاده از تجمع خارجی, این را حس می کند به مجموعه `max_bytes_before_external_group_by` به 10000000000 و حداکثر\_موری\_اساژ به 20000000000. هنگامی که تجمع خارجی باعث شده است (اگر حداقل یک روگرفت از داده های موقت وجود دارد), حداکثر مصرف رم تنها کمی بیشتر از `max_bytes_before_external_group_by`. - -با پردازش پرس و جو توزیع, تجمع خارجی بر روی سرور از راه دور انجام. به منظور سرور درخواست به استفاده از تنها مقدار کمی از رم, تنظیم `distributed_aggregation_memory_efficient` به 1. - -هنگامی که ادغام داده ها به دیسک سرخ, و همچنین زمانی که ادغام نتایج حاصل از سرور از راه دور زمانی که `distributed_aggregation_memory_efficient` تنظیم فعال است, مصرف تا `1/256 * the_number_of_threads` از مقدار کل رم. - -هنگامی که تجمع خارجی فعال است, اگر کمتر از وجود دارد `max_bytes_before_external_group_by` of data (i.e. data was not flushed), the query runs just as fast as without external aggregation. If any temporary data was flushed, the run time will be several times longer (approximately three times). - -اگر شما یک `ORDER BY` با یک `LIMIT` پس از `GROUP BY`, سپس مقدار رم استفاده می شود بستگی به مقدار داده ها در `LIMIT` نه تو کل میز اما اگر `ORDER BY` ندارد `LIMIT` فراموش نکنید که مرتب سازی خارجی را فعال کنید (`max_bytes_before_external_sort`). - -### محدود کردن بند {#limit-by-clause} - -پرس و جو با `LIMIT n BY expressions` بند اول را انتخاب می کند `n` ردیف برای هر مقدار مجزا از `expressions`. کلید برای `LIMIT BY` می تواند شامل هر تعداد از [عبارتها](../syntax.md#syntax-expressions). - -تاتر از نحو زیر پشتیبانی می کند: - -- `LIMIT [offset_value, ]n BY expressions` -- `LIMIT n OFFSET offset_value BY expressions` - -در طول پردازش پرس و جو, خانه را انتخاب داده دستور داد با مرتب سازی کلید. کلید مرتب سازی به صراحت با استفاده از یک مجموعه [ORDER BY](#select-order-by) بند یا به طور ضمنی به عنوان یک ویژگی از موتور جدول. سپس کلیک کنیداوس اعمال می شود `LIMIT n BY expressions` و اولین را برمی گرداند `n` ردیف برای هر ترکیب مجزا از `expressions`. اگر `OFFSET` مشخص شده است, سپس برای هر بلوک داده که متعلق به یک ترکیب متمایز از `expressions`. `offset_value` تعداد ردیف از ابتدای بلوک و حداکثر می گرداند `n` ردیف به عنوان یک نتیجه. اگر `offset_value` بزرگتر از تعدادی از ردیف در بلوک داده است, کلیک بازگشت صفر ردیف از بلوک. - -`LIMIT BY` مربوط به `LIMIT`. هر دو را می توان در همان پرس و جو استفاده کرد. - -**مثالها** - -جدول نمونه: - -``` sql -CREATE TABLE limit_by(id Int, val Int) ENGINE = Memory; -INSERT INTO limit_by values(1, 10), (1, 11), (1, 12), (2, 20), (2, 21); -``` - -نمایش داده شد: - -``` sql -SELECT * FROM limit_by ORDER BY id, val LIMIT 2 BY id -``` - -``` text -┌─id─┬─val─┐ -│ 1 │ 10 │ -│ 1 │ 11 │ -│ 2 │ 20 │ -│ 2 │ 21 │ -└────┴─────┘ -``` - -``` sql -SELECT * FROM limit_by ORDER BY id, val LIMIT 1, 2 BY id -``` - -``` text -┌─id─┬─val─┐ -│ 1 │ 11 │ -│ 1 │ 12 │ -│ 2 │ 21 │ -└────┴─────┘ -``` - -این `SELECT * FROM limit_by ORDER BY id, val LIMIT 2 OFFSET 1 BY id` پرس و جو همان نتیجه را برمی گرداند. - -پرس و جو زیر را برمی گرداند بالا 5 ارجاع برای هر `domain, device_type` جفت با حداکثر 100 ردیف در مجموع (`LIMIT n BY + LIMIT`). - -``` sql -SELECT - domainWithoutWWW(URL) AS domain, - domainWithoutWWW(REFERRER_URL) AS referrer, - device_type, - count() cnt -FROM hits -GROUP BY domain, referrer, device_type -ORDER BY cnt DESC -LIMIT 5 BY domain, device_type -LIMIT 100 -``` - -### داشتن بند {#having-clause} - -اجازه می دهد تا فیلتر نتیجه پس از گروه های دریافت, شبیه به بند جایی که. -در کجا و در جایی که قبل از تجمع انجام متفاوت است (گروه های), در حالی که پس از انجام. -اگر تجمع انجام نشده است, داشتن نمی توان استفاده کرد. - -### ORDER BY {#select-order-by} - -سفارش توسط بند شامل یک لیست از عبارات, که می تواند هر یک اختصاص داده شود مجموعه خارج از محدوده و یا صعودی (جهت مرتب سازی). اگر جهت مشخص نشده است, صعودی فرض بر این است. صعودی و نزولی در نزولی مرتب شده اند. جهت مرتب سازی شامل یک عبارت واحد, نه به کل لیست. مثال: `ORDER BY Visits DESC, SearchPhrase` - -برای مرتب سازی بر اساس مقادیر رشته, شما می توانید میترا مشخص (مقایسه). مثال: `ORDER BY SearchPhrase COLLATE 'tr'` - برای مرتب سازی بر اساس کلمه کلیدی به ترتیب صعودی, با استفاده از الفبای ترکی, حساس به حروف, فرض کنید که رشته ها سخن گفتن-8 کد گذاری. تلفیق می تواند مشخص شود یا نه برای هر عبارت به منظور به طور مستقل. اگر مرکز کنترل و یا مرکز کنترل خارج رحمی مشخص شده است, تلفیقی بعد از مشخص. هنگام استفاده از برخورد, مرتب سازی است که همیشه غیر حساس به حروف. - -ما فقط توصیه می کنیم با استفاده از تلفیق برای مرتب سازی نهایی تعداد کمی از ردیف, از مرتب سازی با تلفیقی کمتر موثر تر از مرتب سازی طبیعی با بایت است. - -ردیف هایی که دارای مقادیر یکسان برای لیست عبارات مرتب سازی هستند خروجی در جهت دلخواه هستند که همچنین می توانند نامعین (هر بار متفاوت) باشند. -اگر ORDER BY حذف شده منظور از ردیف نیز تعریف نشده و ممکن است nondeterministic به عنوان به خوبی. - -`NaN` و `NULL` مرتب سازی سفارش: - -- با اصلاح کننده `NULLS FIRST` — First `NULL` پس `NaN`, سپس ارزش های دیگر. -- با اصلاح کننده `NULLS LAST` — First the values, then `NaN` پس `NULL`. -- Default — The same as with the `NULLS LAST` تغییردهنده. - -مثال: - -برای جدول - -``` text -┌─x─┬────y─┐ -│ 1 │ ᴺᵁᴸᴸ │ -│ 2 │ 2 │ -│ 1 │ nan │ -│ 2 │ 2 │ -│ 3 │ 4 │ -│ 5 │ 6 │ -│ 6 │ nan │ -│ 7 │ ᴺᵁᴸᴸ │ -│ 6 │ 7 │ -│ 8 │ 9 │ -└───┴──────┘ -``` - -اجرای پرس و جو `SELECT * FROM t_null_nan ORDER BY y NULLS FIRST` برای دریافت: - -``` text -┌─x─┬────y─┐ -│ 1 │ ᴺᵁᴸᴸ │ -│ 7 │ ᴺᵁᴸᴸ │ -│ 1 │ nan │ -│ 6 │ nan │ -│ 2 │ 2 │ -│ 2 │ 2 │ -│ 3 │ 4 │ -│ 5 │ 6 │ -│ 6 │ 7 │ -│ 8 │ 9 │ -└───┴──────┘ -``` - -هنگامی که اعداد ممیز شناور طبقه بندی شده اند, نان جدا از ارزش های دیگر هستند. صرف نظر از نظم مرتب سازی, نان در پایان. به عبارت دیگر برای مرتب سازی صعودی قرار می گیرند همانطور که بزرگتر از همه شماره های دیگر هستند در حالی که برای مرتب سازی کوچکتر از بقیه قرار می گیرند. - -رم کمتر استفاده می شود اگر یک محدودیت به اندازه کافی کوچک است علاوه بر سفارش های مشخص. در غیر این صورت, مقدار حافظه صرف متناسب با حجم داده ها برای مرتب سازی است. برای پردازش پرس و جو توزیع, اگر گروه حذف شده است, مرتب سازی تا حدی بر روی سرور از راه دور انجام, و نتایج در سرور درخواست با هم ادغام شدند. این به این معنی است که برای مرتب سازی توزیع, حجم داده ها برای مرتب کردن می تواند بیشتر از مقدار حافظه بر روی یک سرور واحد. - -در صورتی که رم به اندازه کافی وجود ندارد, ممکن است به انجام مرتب سازی در حافظه خارجی (ایجاد فایل های موقت بر روی یک دیسک). از تنظیمات استفاده کنید `max_bytes_before_external_sort` برای این منظور. اگر قرار است 0 (به طور پیش فرض), مرتب سازی خارجی غیر فعال است. اگر فعال باشد, زمانی که حجم داده ها برای مرتب کردن بر اساس تعداد مشخصی از بایت می رسد, اطلاعات جمع شده مرتب شده و ریخته را به یک فایل موقت. پس از همه داده ها خوانده شده است, تمام فایل های طبقه بندی شده اند با هم ادغام شدند و نتایج خروجی. فایل ها به نوشته /ور/معاونت/تاتر/تی ام پی/ دایرکتوری در پیکربندی (به طور پیش فرض, اما شما می توانید با استفاده از ‘tmp\_path’ پارامتر برای تغییر این تنظیم). - -در حال اجرا یک پرس و جو ممکن است حافظه بیش از استفاده ‘max\_bytes\_before\_external\_sort’. به همین دلیل این تنظیم باید یک مقدار قابل توجهی کوچکتر از ‘max\_memory\_usage’. به عنوان مثال, اگر سرور شما 128 گیگابایت رم و شما نیاز به اجرای یک پرس و جو واحد, تنظیم ‘max\_memory\_usage’ به 100 گیگابایت, و ‘max\_bytes\_before\_external\_sort’ به 80 گیگابایت. - -مرتب سازی خارجی کار می کند بسیار کمتر به طور موثر از مرتب سازی در رم. - -### انتخاب بند {#select-select} - -[عبارتها](../syntax.md#syntax-expressions) مشخص شده در `SELECT` بند بعد از تمام عملیات در بند بالا توضیح محاسبه به پایان رسید. این عبارات کار می کنند به عنوان اگر به ردیف جداگانه در نتیجه اعمال می شود. اگر عبارات در `SELECT` بند شامل مجموع توابع سپس ClickHouse فرآیندهای کل توابع و عبارات استفاده می شود به عنوان استدلال های خود را در طول [GROUP BY](#select-group-by-clause) تجمع. - -اگر شما می خواهید که شامل تمام ستون ها در نتیجه, استفاده از ستاره (`*`) نماد. به عنوان مثال, `SELECT * FROM ...`. - -برای مطابقت با برخی از ستون ها در نتیجه با یک [شماره 2](https://en.wikipedia.org/wiki/RE2_(software)) عبارت منظم می توانید از `COLUMNS` اصطلاح. - -``` sql -COLUMNS('regexp') -``` - -برای مثال جدول را در نظر بگیرید: - -``` sql -CREATE TABLE default.col_names (aa Int8, ab Int8, bc Int8) ENGINE = TinyLog -``` - -پرس و جو زیر داده ها را از تمام ستون های حاوی `a` نماد به نام خود. - -``` sql -SELECT COLUMNS('a') FROM col_names -``` - -``` text -┌─aa─┬─ab─┐ -│ 1 │ 1 │ -└────┴────┘ -``` - -ستون های انتخاب شده به ترتیب حروف الفبا بازگردانده نمی شوند. - -شما می توانید چند استفاده کنید `COLUMNS` عبارات در پرس و جو و اعمال توابع را به خود. - -به عنوان مثال: - -``` sql -SELECT COLUMNS('a'), COLUMNS('c'), toTypeName(COLUMNS('c')) FROM col_names -``` - -``` text -┌─aa─┬─ab─┬─bc─┬─toTypeName(bc)─┐ -│ 1 │ 1 │ 1 │ Int8 │ -└────┴────┴────┴────────────────┘ -``` - -هر ستون توسط `COLUMNS` بیان به تابع به عنوان یک استدلال جداگانه منتقل می شود. همچنین شما می توانید استدلال های دیگر به تابع منتقل می کند اگر از. مراقب باشید در هنگام استفاده از توابع. اگر یک تابع تعداد استدلال شما را به تصویب را پشتیبانی نمی کند, خانه عروسکی می اندازد یک استثنا. - -به عنوان مثال: - -``` sql -SELECT COLUMNS('a') + COLUMNS('c') FROM col_names -``` - -``` text -Received exception from server (version 19.14.1): -Code: 42. DB::Exception: Received from localhost:9000. DB::Exception: Number of arguments for function plus doesn't match: passed 3, should be 2. -``` - -در این مثال, `COLUMNS('a')` بازگرداندن دو ستون: `aa` و `ab`. `COLUMNS('c')` بازگرداندن `bc` ستون. این `+` اپراتور نمی تواند به اعمال 3 استدلال, بنابراین تاتر می اندازد یک استثنا با پیام مربوطه. - -ستون که همسان `COLUMNS` بیان می تواند انواع داده های مختلف داشته باشد. اگر `COLUMNS` هیچ ستون مطابقت ندارد و تنها بیان در است `SELECT`, فاحشه خانه می اندازد یک استثنا. - -### بند مجزا {#select-distinct} - -اگر مشخص شده است, تنها یک ردیف از تمام مجموعه از ردیف به طور کامل تطبیق در نتیجه باقی خواهد ماند. -نتیجه همان خواهد بود که اگر گروه توسط در تمام زمینه های مشخص شده در انتخاب بدون توابع دانه مشخص شد. اما تفاوت های مختلفی از گروه وجود دارد: - -- مجزا را می توان همراه با گروه توسط اعمال می شود. -- هنگامی که سفارش های حذف شده است و حد تعریف شده است, پرس و جو متوقف می شود در حال اجرا بلافاصله پس از تعداد مورد نیاز از ردیف های مختلف خوانده شده است. -- بلوک های داده خروجی به عنوان پردازش می شوند, بدون انتظار برای کل پرس و جو را به پایان برساند در حال اجرا. - -متمایز پشتیبانی نمی شود اگر انتخاب حداقل یک ستون مجموعه ای دارد. - -`DISTINCT` با این نسخهها کار میکند [NULL](../syntax.md#null-literal) همانطور که اگر `NULL` یک مقدار خاص بودند, و `NULL=NULL`. به عبارت دیگر در `DISTINCT` نتایج, ترکیب های مختلف با `NULL` فقط یک بار رخ می دهد. - -کلیک پشتیبانی با استفاده از `DISTINCT` و `ORDER BY` بند برای ستون های مختلف در یک پرس و جو. این `DISTINCT` بند قبل از اجرا `ORDER BY` بند بند. - -جدول نمونه: - -``` text -┌─a─┬─b─┐ -│ 2 │ 1 │ -│ 1 │ 2 │ -│ 3 │ 3 │ -│ 2 │ 4 │ -└───┴───┘ -``` - -هنگام انتخاب داده ها با `SELECT DISTINCT a FROM t1 ORDER BY b ASC` پرس و جو, ما نتیجه زیر را دریافت کنید: - -``` text -┌─a─┐ -│ 2 │ -│ 1 │ -│ 3 │ -└───┘ -``` - -اگر ما جهت مرتب سازی را تغییر دهیم `SELECT DISTINCT a FROM t1 ORDER BY b DESC` ما نتیجه زیر را دریافت می کنیم: - -``` text -┌─a─┐ -│ 3 │ -│ 1 │ -│ 2 │ -└───┘ -``` - -سطر `2, 4` قبل از مرتب سازی قطع شد. - -نگاهی به این ویژگی پیاده سازی به حساب زمانی که برنامه نویسی نمایش داده شد. - -### بند محدود {#limit-clause} - -`LIMIT m` اجازه می دهد تا شما را به انتخاب اولین `m` ردیف از نتیجه. - -`LIMIT n, m` اجازه می دهد تا شما را به انتخاب اولین `m` ردیف از نتیجه پس از پرش برای اولین بار `n` ردیف این `LIMIT m OFFSET n` نحو نیز پشتیبانی می کند. - -`n` و `m` باید اعداد صحیح غیر منفی باشد. - -اگر وجود ندارد `ORDER BY` بند که به صراحت انواع نتایج, نتیجه ممکن است خودسرانه و نامعین. - -### اتحادیه همه بند {#union-all-clause} - -شما می توانید اتحادیه همه به ترکیب هر تعداد از نمایش داده شد استفاده کنید. مثال: - -``` sql -SELECT CounterID, 1 AS table, toInt64(count()) AS c - FROM test.hits - GROUP BY CounterID - -UNION ALL - -SELECT CounterID, 2 AS table, sum(Sign) AS c - FROM test.visits - GROUP BY CounterID - HAVING c > 0 -``` - -فقط اتحادیه پشتیبانی می شود. اتحادیه به طور منظم (اتحادیه مجزا) پشتیبانی نمی شود. اگر شما نیاز به اتحادیه مجزا, شما می توانید ارسال انتخاب کنید متمایز از زیرخاکی حاوی اتحادیه همه. - -نمایش داده شد که بخش هایی از اتحادیه همه را می توان به طور همزمان اجرا, و نتایج خود را می توان با هم مخلوط. - -ساختار نتایج (تعداد و نوع ستون) باید برای نمایش داده شد مطابقت داشته باشد. اما نام ستون می تواند متفاوت باشد. در این مورد, نام ستون برای نتیجه نهایی خواهد شد از پرس و جو برای اولین بار گرفته شده. نوع ریخته گری برای اتحادیه انجام می شود. برای مثال اگر دو نمایش داده شد در حال ترکیب باید همین زمینه را با غیر-`Nullable` و `Nullable` انواع از یک نوع سازگار, در نتیجه `UNION ALL` دارای یک `Nullable` نوع درست. - -نمایش داده شد که بخش هایی از اتحادیه همه را نمی توان در داخل پرانتز محصور شده است. سفارش و محدودیت برای نمایش داده شد جداگانه به نتیجه نهایی اعمال می شود. اگر شما نیاز به اعمال یک تبدیل به نتیجه نهایی, شما می توانید تمام نمایش داده شد با اتحادیه همه در یک خرده فروشی در بند از قرار. - -### به OUTFILE بند {#into-outfile-clause} - -افزودن `INTO OUTFILE filename` بند (جایی که نام فایل یک رشته تحت اللفظی است) برای تغییر مسیر خروجی پرس و جو به فایل مشخص شده است. -در مقابل خروجی زیر, فایل در سمت سرویس گیرنده ایجاد. پرس و جو شکست مواجه خواهد شد اگر یک فایل با نام فایل مشابه در حال حاضر وجود دارد. -این قابلیت در خط فرمان مشتری و فاحشه خانه در دسترس است-محلی (پرس و جو ارسال شده از طریق رابط اچ.تی. پی شکست خواهد خورد). - -فرمت خروجی به طور پیش فرض جدول است (همان است که در خط فرمان حالت دسته ای مشتری). - -### بند فرمت {#format-clause} - -مشخص ‘FORMAT format’ برای دریافت اطلاعات در هر فرمت مشخص شده است. -شما می توانید این کار را برای راحتی و یا برای ایجاد افسردگی استفاده کنید. -برای کسب اطلاعات بیشتر به بخش مراجعه کنید “Formats”. -اگر بند فرمت حذف شده است, فرمت پیش فرض استفاده شده است, که بستگی به هر دو تنظیمات و رابط مورد استفاده برای دسترسی به دسی بل. برای رابط اچ تی پی و مشتری خط فرمان در حالت دسته ای, فرمت پیش فرض جدولبندی شده است. برای مشتری خط فرمان در حالت تعاملی فرمت پیش فرض است که پیش سازه (این جداول جذاب و جمع و جور). - -هنگام استفاده از خط فرمان مشتری داده ها به مشتری در فرمت موثر داخلی منتقل می شود. مشتری به طور مستقل تفسیر بند فرمت پرس و جو و فرمت های داده های خود را (در نتیجه تسکین شبکه و سرور از بار). - -### در اپراتورها {#select-in-operators} - -این `IN`, `NOT IN`, `GLOBAL IN` و `GLOBAL NOT IN` اپراتورها به طور جداگانه تحت پوشش, از قابلیت های خود را کاملا غنی است. - -سمت چپ اپراتور یا یک ستون یا یک تاپل است. - -مثالها: - -``` sql -SELECT UserID IN (123, 456) FROM ... -SELECT (CounterID, UserID) IN ((34, 123), (101500, 456)) FROM ... -``` - -اگر سمت چپ یک ستون است که در شاخص است, و در سمت راست مجموعه ای از ثابت است, سیستم با استفاده از شاخص برای پردازش پرس و جو. - -Don't list too many values explicitly (i.e. millions). If a data set is large, put it in a temporary table (for example, see the section “External data for query processing”), سپس با استفاده از یک خرده فروشی. - -سمت راست اپراتور می تواند مجموعه ای از عبارات ثابت, مجموعه ای از تاپل با عبارات ثابت (نشان داده شده در نمونه های بالا), و یا به نام یک جدول پایگاه داده و یا زیرخاکی در داخل پرانتز را انتخاب کنید. - -اگر سمت راست اپراتور نام یک جدول است (مثلا, `UserID IN users`), این معادل به خرده فروشی است `UserID IN (SELECT * FROM users)`. با استفاده از این هنگام کار با داده های خارجی است که همراه با پرس و جو ارسال می شود. مثلا, پرس و جو را می توان همراه با مجموعه ای از شناسه کاربر لود شده به ارسال ‘users’ جدول موقت, که باید فیلتر شود. - -اگر در سمت راست اپراتور یک نام جدول است که موتور مجموعه ای است (مجموعه داده های تهیه شده است که همیشه در رم), مجموعه داده خواهد شد بیش از دوباره برای هر پرس و جو ایجاد نمی. - -زیرخاکری ممکن است بیش از یک ستون برای فیلتر کردن تاپل را مشخص کنید. -مثال: - -``` sql -SELECT (CounterID, UserID) IN (SELECT CounterID, UserID FROM ...) FROM ... -``` - -ستون به سمت چپ و راست اپراتور در باید همان نوع. - -اپراتور در و زیرخاکی ممکن است در هر بخشی از پرس و جو رخ می دهد, از جمله در توابع کل و توابع لامبدا. -مثال: - -``` sql -SELECT - EventDate, - avg(UserID IN - ( - SELECT UserID - FROM test.hits - WHERE EventDate = toDate('2014-03-17') - )) AS ratio -FROM test.hits -GROUP BY EventDate -ORDER BY EventDate ASC -``` - -``` text -┌──EventDate─┬────ratio─┐ -│ 2014-03-17 │ 1 │ -│ 2014-03-18 │ 0.807696 │ -│ 2014-03-19 │ 0.755406 │ -│ 2014-03-20 │ 0.723218 │ -│ 2014-03-21 │ 0.697021 │ -│ 2014-03-22 │ 0.647851 │ -│ 2014-03-23 │ 0.648416 │ -└────────────┴──────────┘ -``` - -برای هر روز پس از مارس 17, تعداد درصد تعداد بازدید از صفحات ساخته شده توسط کاربران که سایت در مارس 17 بازدید. -یک خرده فروشی در بند در همیشه اجرا فقط یک بار بر روی یک سرور. هیچ زیرمجموعه وابسته وجود دارد. - -#### پردازش پوچ {#null-processing-1} - -در طول پردازش درخواست, در اپراتور فرض می شود که در نتیجه یک عملیات با [NULL](../syntax.md#null-literal) همیشه برابر است با `0`, صرف نظر از اینکه `NULL` است در سمت راست یا چپ اپراتور. `NULL` ارزش ها در هر مجموعه داده شامل نمی شود, به یکدیگر مربوط نیست و نمی توان در مقایسه. - -در اینجا یک مثال با است `t_null` جدول: - -``` text -┌─x─┬────y─┐ -│ 1 │ ᴺᵁᴸᴸ │ -│ 2 │ 3 │ -└───┴──────┘ -``` - -در حال اجرا پرس و جو `SELECT x FROM t_null WHERE y IN (NULL,3)` به شما نتیجه زیر را می دهد: - -``` text -┌─x─┐ -│ 2 │ -└───┘ -``` - -شما می توانید ببینید که ردیف که `y = NULL` از نتایج پرس و جو پرتاب می شود. دلیل این است که تاتر نمی توانید تصمیم بگیرید که چه `NULL` در `(NULL,3)` تنظیم, بازگشت `0` به عنوان نتیجه عملیات و `SELECT` این سطر را از خروجی نهایی حذف می کند. - -``` sql -SELECT y IN (NULL, 3) -FROM t_null -``` - -``` text -┌─in(y, tuple(NULL, 3))─┐ -│ 0 │ -│ 1 │ -└───────────────────────┘ -``` - -#### توزیع Subqueries {#select-distributed-subqueries} - -دو گزینه برای در بازدید کنندگان با کارخانه های فرعی وجود دارد (شبیه به می پیوندد): طبیعی `IN` / `JOIN` و `GLOBAL IN` / `GLOBAL JOIN`. در نحوه اجرا برای پردازش پرس و جو توزیع شده متفاوت است. - -!!! attention "توجه" - به یاد داشته باشید که الگوریتم های زیر توضیح داده شده ممکن است متفاوت بسته به کار [تنظیمات](../../operations/settings/settings.md) `distributed_product_mode` تنظیمات. - -هنگام استفاده از به طور منظم در, پرس و جو به سرور از راه دور ارسال, و هر یک از اجرا می شود کارخانه های فرعی در `IN` یا `JOIN` بند بند. - -هنگام استفاده از `GLOBAL IN` / `GLOBAL JOINs`, اول همه زیرمجموعه ها برای اجرا `GLOBAL IN` / `GLOBAL JOINs` و نتایج در جداول موقت جمع می شوند. سپس جداول موقت به هر سرور از راه دور ارسال, جایی که نمایش داده شد با استفاده از این داده های موقت اجرا. - -برای پرس و جو غیر توزیع, استفاده از به طور منظم `IN` / `JOIN`. - -مراقب باشید در هنگام استفاده از کارخانه های فرعی در `IN` / `JOIN` بند برای پردازش پرس و جو توزیع. - -بیایید نگاهی به برخی از نمونه. فرض کنید که هر سرور در خوشه طبیعی است **\_تمل**. هر سرور همچنین دارای یک **توزیع \_تماس** جدول با **توزیع شده** نوع, که به نظر می رسد در تمام سرور در خوشه. - -برای پرس و جو به **توزیع \_تماس** پرس و جو به تمام سرورهای راه دور ارسال می شود و با استفاده از **\_تمل**. - -برای مثال پرس و جو - -``` sql -SELECT uniq(UserID) FROM distributed_table -``` - -به تمام سرورهای راه دور ارسال می شود - -``` sql -SELECT uniq(UserID) FROM local_table -``` - -و به صورت موازی اجرا می شود تا زمانی که به مرحله ای برسد که نتایج متوسط می تواند ترکیب شود. سپس نتایج میانی به سرور درخواست کننده بازگردانده می شود و با هم ادغام می شوند و نتیجه نهایی به مشتری ارسال می شود. - -حالا اجازه دهید به بررسی یک پرس و جو با در: - -``` sql -SELECT uniq(UserID) FROM distributed_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM local_table WHERE CounterID = 34) -``` - -- محاسبه تقاطع مخاطبان از دو سایت. - -این پرس و جو خواهد شد به تمام سرور از راه دور به عنوان ارسال می شود - -``` sql -SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM local_table WHERE CounterID = 34) -``` - -به عبارت دیگر, داده های تعیین شده در بند در خواهد شد بر روی هر سرور به طور مستقل جمع, تنها در سراسر داده است که به صورت محلی بر روی هر یک از سرور های ذخیره شده. - -این به درستی و بهینه کار خواهد کرد اگر شما برای این مورد تهیه و داده ها در سراسر سرورهای خوشه گسترش یافته اند به طوری که داده ها را برای یک شناسه تنها ساکن به طور کامل بر روی یک سرور واحد. در این مورد, تمام اطلاعات لازم در دسترس خواهد بود به صورت محلی بر روی هر سرور. در غیر این صورت نتیجه نادرست خواهد بود. ما به این تنوع از پرس و جو به عنوان مراجعه کنید “local IN”. - -برای اصلاح چگونه پرس و جو کار می کند زمانی که داده ها به طور تصادفی در سراسر سرور خوشه گسترش, شما می توانید مشخص **توزیع \_تماس** در داخل یک خرده فروشی. پرس و جو شبیه به این خواهد بود: - -``` sql -SELECT uniq(UserID) FROM distributed_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM distributed_table WHERE CounterID = 34) -``` - -این پرس و جو خواهد شد به تمام سرور از راه دور به عنوان ارسال می شود - -``` sql -SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM distributed_table WHERE CounterID = 34) -``` - -خرده فروشی شروع خواهد شد در حال اجرا بر روی هر سرور از راه دور. پس از زیرخاکری با استفاده از یک جدول توزیع, خرده فروشی است که در هر سرور از راه دور خواهد شد به هر سرور از راه دور به عنوان خشمگین - -``` sql -SELECT UserID FROM local_table WHERE CounterID = 34 -``` - -مثلا, اگر شما یک خوشه از 100 سرور, اجرای کل پرس و جو نیاز 10,000 درخواست ابتدایی, که به طور کلی در نظر گرفته غیر قابل قبول. - -در چنین مواردی, شما همیشه باید جهانی به جای در استفاده از. بیایید نگاه کنیم که چگونه برای پرس و جو کار می کند - -``` sql -SELECT uniq(UserID) FROM distributed_table WHERE CounterID = 101500 AND UserID GLOBAL IN (SELECT UserID FROM distributed_table WHERE CounterID = 34) -``` - -سرور درخواست کننده خرده فروشی را اجرا خواهد کرد - -``` sql -SELECT UserID FROM distributed_table WHERE CounterID = 34 -``` - -و در نتیجه خواهد شد در یک جدول موقت در رم قرار داده است. سپس درخواست خواهد شد به هر سرور از راه دور به عنوان ارسال - -``` sql -SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID GLOBAL IN _data1 -``` - -و جدول موقت `_data1` خواهد شد به هر سرور از راه دور با پرس و جو ارسال (نام جدول موقت پیاده سازی تعریف شده است). - -این مطلوب تر از استفاده از نرمال در است. با این حال, نگه داشتن نکات زیر را در ذهن: - -1. هنگام ایجاد یک جدول موقت داده های منحصر به فرد ساخته شده است. برای کاهش حجم داده های منتقل شده بر روی شبکه مشخص متمایز در زیرخاکری. (شما لازم نیست برای انجام این کار برای عادی در.) -2. جدول موقت خواهد شد به تمام سرور از راه دور ارسال. انتقال برای توپولوژی شبکه به حساب نمی. مثلا, اگر 10 سرور از راه دور در یک مرکز داده است که در رابطه با سرور درخواست بسیار از راه دور اقامت, داده ها ارسال خواهد شد 10 بار بیش از کانال به مرکز داده از راه دور. سعی کنید برای جلوگیری از مجموعه داده های بزرگ در هنگام استفاده از جهانی در. -3. هنگام انتقال داده ها به سرور از راه دور, محدودیت در پهنای باند شبکه قابل تنظیم نیست. شما ممکن است شبکه بیش از حد. -4. سعی کنید برای توزیع داده ها در سراسر سرور به طوری که شما لازم نیست که به استفاده از جهانی را به صورت منظم. -5. اگر شما نیاز به استفاده از جهانی در اغلب, برنامه ریزی محل خوشه خانه کلیک به طوری که یک گروه واحد از کپی ساکن در بیش از یک مرکز داده با یک شبکه سریع بین, به طوری که یک پرس و جو را می توان به طور کامل در یک مرکز داده واحد پردازش. - -همچنین حس می کند برای مشخص کردن یک جدول محلی در `GLOBAL IN` بند, در صورتی که این جدول محلی تنها بر روی سرور درخواست در دسترس است و شما می خواهید به استفاده از داده ها را از روی سرور از راه دور. - -### ارزش های شدید {#extreme-values} - -علاوه بر نتایج, شما همچنین می توانید حداقل و حداکثر ارزش برای ستون نتایج از. برای انجام این کار, تنظیم **افراط** تنظیم به 1. حداقل و حداکثر برای انواع عددی محاسبه, تاریخ, و تاریخ با بار. برای ستون های دیگر مقادیر پیش فرض خروجی هستند. - -An extra two rows are calculated – the minimums and maximums, respectively. These extra two rows are output in `JSON*`, `TabSeparated*` و `Pretty*` [فرشها](../../interfaces/formats.md), جدا از ردیف های دیگر. خروجی برای فرمت های دیگر نیستند. - -داخل `JSON*` فرمت, ارزش شدید خروجی در یک جداگانه ‘extremes’ رشته. داخل `TabSeparated*` فرمت, ردیف پس از نتیجه اصلی, و بعد از ‘totals’ اگر در حال حاضر. این است که توسط یک ردیف خالی قبل (پس از داده های دیگر). داخل `Pretty*` فرمت, ردیف خروجی به عنوان یک جدول جداگانه پس از نتیجه اصلی است, و بعد از `totals` اگر در حال حاضر. - -مقادیر شدید برای ردیف قبل محاسبه می شود `LIMIT` اما بعد از `LIMIT BY`. با این حال, هنگام استفاده از `LIMIT offset, size`, ردیف قبل از `offset` در `extremes`. در درخواست جریان, نتیجه نیز ممکن است شامل تعداد کمی از ردیف که از طریق تصویب `LIMIT`. - -### یادداشتها {#notes} - -این `GROUP BY` و `ORDER BY` بند انجام استدلال موضعی را پشتیبانی نمی کند. این در تضاد خروجی زیر, اما مطابق با استاندارد گذاشتن. -به عنوان مثال, `GROUP BY 1, 2` will be interpreted as grouping by constants (i.e. aggregation of all rows into one). - -شما می توانید مترادف استفاده کنید (`AS` نام مستعار) در هر بخشی از یک پرس و جو. - -شما می توانید یک ستاره در هر بخشی از یک پرس و جو به جای بیان قرار داده است. هنگامی که پرس و جو تجزیه و تحلیل, ستاره به یک لیست از تمام ستون جدول گسترش (به استثنای `MATERIALIZED` و `ALIAS` ستون). تنها چند مورد وجود دارد که با استفاده از ستاره توجیه می شود: - -- هنگام ایجاد یک روگرفت جدول. -- برای جداول حاوی فقط چند ستون, مانند جداول سیستم. -- برای گرفتن اطلاعات در مورد چه ستون در یک جدول هستند. در این مورد, تنظیم `LIMIT 1`. اما بهتر است از `DESC TABLE` پرس و جو. -- هنگامی که فیلتراسیون قوی در تعداد کمی از ستون ها با استفاده از وجود دارد `PREWHERE`. -- در subqueries (از ستون های که مورد نیاز نیست برای خارجی پرس و جو از مطالعه حذف شدند subqueries). - -در تمام موارد دیگر, ما توصیه نمی با استفاده از ستاره, زیرا تنها به شما می دهد اشکالاتی از یک سندرم روده تحریک پذیر ستونی به جای مزایای. به عبارت دیگر با استفاده از ستاره توصیه نمی شود. - -[مقاله اصلی](https://clickhouse.tech/docs/en/query_language/select/) diff --git a/docs/fa/sql-reference/statements/select/array-join.md b/docs/fa/sql-reference/statements/select/array-join.md new file mode 100644 index 00000000000..28655e3554a --- /dev/null +++ b/docs/fa/sql-reference/statements/select/array-join.md @@ -0,0 +1,282 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +--- + +# مجموعه پیوستن بند {#select-array-join-clause} + +این یک عملیات مشترک برای جداول است که حاوی یک ستون جداگانه برای تولید یک جدول جدید است که دارای یک ستون با هر عنصر جداگانه ای از ستون اولیه است در حالی که مقادیر ستون های دیگر تکرار می شوند. این مورد اساسی چه چیزی است `ARRAY JOIN` بند کند. + +نام خود را از این واقعیت است که می تواند در به عنوان اجرای نگاه `JOIN` با یک مجموعه یا ساختار داده های تو در تو. قصد شبیه به است [ارریجین](../../functions/array-join.md#functions_arrayjoin) تابع, اما قابلیت بند گسترده تر است. + +نحو: + +``` sql +SELECT +FROM +[LEFT] ARRAY JOIN +[WHERE|PREWHERE ] +... +``` + +شما می توانید تنها یک مشخص `ARRAY JOIN` بند در یک `SELECT` پرس و جو. + +انواع پشتیبانی شده از `ARRAY JOIN` به شرح زیر است: + +- `ARRAY JOIN` - در مورد پایه, بند خالی در نتیجه شامل نمی شود `JOIN`. +- `LEFT ARRAY JOIN` - نتیجه `JOIN` شامل ردیف با ارریس خالی است. مقدار برای یک مجموعه خالی است به مقدار پیش فرض برای نوع عنصر مجموعه (معمولا 0, رشته خالی و یا تهی). + +## پایه صف پیوستن به نمونه {#basic-array-join-examples} + +نمونه های زیر نشان می دهد استفاده از `ARRAY JOIN` و `LEFT ARRAY JOIN` بند. بیایید یک جدول با یک [& حذف](../../../sql-reference/data-types/array.md) ستون را تایپ کنید و مقادیر را وارد کنید: + +``` sql +CREATE TABLE arrays_test +( + s String, + arr Array(UInt8) +) ENGINE = Memory; + +INSERT INTO arrays_test +VALUES ('Hello', [1,2]), ('World', [3,4,5]), ('Goodbye', []); +``` + +``` text +┌─s───────────┬─arr─────┐ +│ Hello │ [1,2] │ +│ World │ [3,4,5] │ +│ Goodbye │ [] │ +└─────────────┴─────────┘ +``` + +مثال زیر از `ARRAY JOIN` بند: + +``` sql +SELECT s, arr +FROM arrays_test +ARRAY JOIN arr; +``` + +``` text +┌─s─────┬─arr─┐ +│ Hello │ 1 │ +│ Hello │ 2 │ +│ World │ 3 │ +│ World │ 4 │ +│ World │ 5 │ +└───────┴─────┘ +``` + +مثال بعدی با استفاده از `LEFT ARRAY JOIN` بند: + +``` sql +SELECT s, arr +FROM arrays_test +LEFT ARRAY JOIN arr; +``` + +``` text +┌─s───────────┬─arr─┐ +│ Hello │ 1 │ +│ Hello │ 2 │ +│ World │ 3 │ +│ World │ 4 │ +│ World │ 5 │ +│ Goodbye │ 0 │ +└─────────────┴─────┘ +``` + +## استفاده از نام مستعار {#using-aliases} + +یک نام مستعار می تواند برای مجموعه ای در `ARRAY JOIN` بند بند. در این مورد, یک مورد مجموعه ای را می توان با این نام مستعار دیده, اما مجموعه خود را با نام اصلی قابل دسترسی. مثال: + +``` sql +SELECT s, arr, a +FROM arrays_test +ARRAY JOIN arr AS a; +``` + +``` text +┌─s─────┬─arr─────┬─a─┐ +│ Hello │ [1,2] │ 1 │ +│ Hello │ [1,2] │ 2 │ +│ World │ [3,4,5] │ 3 │ +│ World │ [3,4,5] │ 4 │ +│ World │ [3,4,5] │ 5 │ +└───────┴─────────┴───┘ +``` + +با استفاده از نام مستعار, شما می توانید انجام `ARRAY JOIN` با یک مجموعه خارجی. به عنوان مثال: + +``` sql +SELECT s, arr_external +FROM arrays_test +ARRAY JOIN [1, 2, 3] AS arr_external; +``` + +``` text +┌─s───────────┬─arr_external─┐ +│ Hello │ 1 │ +│ Hello │ 2 │ +│ Hello │ 3 │ +│ World │ 1 │ +│ World │ 2 │ +│ World │ 3 │ +│ Goodbye │ 1 │ +│ Goodbye │ 2 │ +│ Goodbye │ 3 │ +└─────────────┴──────────────┘ +``` + +ارریس های متعدد را می توان با کاما از هم جدا در `ARRAY JOIN` بند بند. در این مورد, `JOIN` به طور همزمان (مجموع مستقیم و نه محصول دکارتی) انجام می شود. توجه داشته باشید که تمام ارریس باید به همان اندازه. مثال: + +``` sql +SELECT s, arr, a, num, mapped +FROM arrays_test +ARRAY JOIN arr AS a, arrayEnumerate(arr) AS num, arrayMap(x -> x + 1, arr) AS mapped; +``` + +``` text +┌─s─────┬─arr─────┬─a─┬─num─┬─mapped─┐ +│ Hello │ [1,2] │ 1 │ 1 │ 2 │ +│ Hello │ [1,2] │ 2 │ 2 │ 3 │ +│ World │ [3,4,5] │ 3 │ 1 │ 4 │ +│ World │ [3,4,5] │ 4 │ 2 │ 5 │ +│ World │ [3,4,5] │ 5 │ 3 │ 6 │ +└───────┴─────────┴───┴─────┴────────┘ +``` + +مثال زیر از [شناسه بسته:](../../../sql-reference/functions/array-functions.md#array_functions-arrayenumerate) تابع: + +``` sql +SELECT s, arr, a, num, arrayEnumerate(arr) +FROM arrays_test +ARRAY JOIN arr AS a, arrayEnumerate(arr) AS num; +``` + +``` text +┌─s─────┬─arr─────┬─a─┬─num─┬─arrayEnumerate(arr)─┐ +│ Hello │ [1,2] │ 1 │ 1 │ [1,2] │ +│ Hello │ [1,2] │ 2 │ 2 │ [1,2] │ +│ World │ [3,4,5] │ 3 │ 1 │ [1,2,3] │ +│ World │ [3,4,5] │ 4 │ 2 │ [1,2,3] │ +│ World │ [3,4,5] │ 5 │ 3 │ [1,2,3] │ +└───────┴─────────┴───┴─────┴─────────────────────┘ +``` + +## مجموعه پیوستن با ساختار داده های تو در تو {#array-join-with-nested-data-structure} + +`ARRAY JOIN` همچنین با این نسخهها کار [ساختارهای داده تو در تو](../../../sql-reference/data-types/nested-data-structures/nested.md): + +``` sql +CREATE TABLE nested_test +( + s String, + nest Nested( + x UInt8, + y UInt32) +) ENGINE = Memory; + +INSERT INTO nested_test +VALUES ('Hello', [1,2], [10,20]), ('World', [3,4,5], [30,40,50]), ('Goodbye', [], []); +``` + +``` text +┌─s───────┬─nest.x──┬─nest.y─────┐ +│ Hello │ [1,2] │ [10,20] │ +│ World │ [3,4,5] │ [30,40,50] │ +│ Goodbye │ [] │ [] │ +└─────────┴─────────┴────────────┘ +``` + +``` sql +SELECT s, `nest.x`, `nest.y` +FROM nested_test +ARRAY JOIN nest; +``` + +``` text +┌─s─────┬─nest.x─┬─nest.y─┐ +│ Hello │ 1 │ 10 │ +│ Hello │ 2 │ 20 │ +│ World │ 3 │ 30 │ +│ World │ 4 │ 40 │ +│ World │ 5 │ 50 │ +└───────┴────────┴────────┘ +``` + +هنگام مشخص کردن نام ساختارهای داده های تو در تو در `ARRAY JOIN`, معنای همان است که `ARRAY JOIN` با تمام عناصر مجموعه ای که شامل. نمونه به شرح زیر است: + +``` sql +SELECT s, `nest.x`, `nest.y` +FROM nested_test +ARRAY JOIN `nest.x`, `nest.y`; +``` + +``` text +┌─s─────┬─nest.x─┬─nest.y─┐ +│ Hello │ 1 │ 10 │ +│ Hello │ 2 │ 20 │ +│ World │ 3 │ 30 │ +│ World │ 4 │ 40 │ +│ World │ 5 │ 50 │ +└───────┴────────┴────────┘ +``` + +این تنوع نیز حس می کند: + +``` sql +SELECT s, `nest.x`, `nest.y` +FROM nested_test +ARRAY JOIN `nest.x`; +``` + +``` text +┌─s─────┬─nest.x─┬─nest.y─────┐ +│ Hello │ 1 │ [10,20] │ +│ Hello │ 2 │ [10,20] │ +│ World │ 3 │ [30,40,50] │ +│ World │ 4 │ [30,40,50] │ +│ World │ 5 │ [30,40,50] │ +└───────┴────────┴────────────┘ +``` + +نام مستعار ممکن است برای یک ساختار داده های تو در تو استفاده می شود, به منظور انتخاب هر دو `JOIN` نتیجه یا مجموعه منبع. مثال: + +``` sql +SELECT s, `n.x`, `n.y`, `nest.x`, `nest.y` +FROM nested_test +ARRAY JOIN nest AS n; +``` + +``` text +┌─s─────┬─n.x─┬─n.y─┬─nest.x──┬─nest.y─────┐ +│ Hello │ 1 │ 10 │ [1,2] │ [10,20] │ +│ Hello │ 2 │ 20 │ [1,2] │ [10,20] │ +│ World │ 3 │ 30 │ [3,4,5] │ [30,40,50] │ +│ World │ 4 │ 40 │ [3,4,5] │ [30,40,50] │ +│ World │ 5 │ 50 │ [3,4,5] │ [30,40,50] │ +└───────┴─────┴─────┴─────────┴────────────┘ +``` + +نمونه ای از استفاده از [شناسه بسته:](../../../sql-reference/functions/array-functions.md#array_functions-arrayenumerate) تابع: + +``` sql +SELECT s, `n.x`, `n.y`, `nest.x`, `nest.y`, num +FROM nested_test +ARRAY JOIN nest AS n, arrayEnumerate(`nest.x`) AS num; +``` + +``` text +┌─s─────┬─n.x─┬─n.y─┬─nest.x──┬─nest.y─────┬─num─┐ +│ Hello │ 1 │ 10 │ [1,2] │ [10,20] │ 1 │ +│ Hello │ 2 │ 20 │ [1,2] │ [10,20] │ 2 │ +│ World │ 3 │ 30 │ [3,4,5] │ [30,40,50] │ 1 │ +│ World │ 4 │ 40 │ [3,4,5] │ [30,40,50] │ 2 │ +│ World │ 5 │ 50 │ [3,4,5] │ [30,40,50] │ 3 │ +└───────┴─────┴─────┴─────────┴────────────┴─────┘ +``` + +## پیاده سازی اطلاعات {#implementation-details} + +سفارش اجرای پرس و جو در هنگام اجرا بهینه شده است `ARRAY JOIN`. اگرچه `ARRAY JOIN` همیشه باید قبل از مشخص شود [WHERE](where.md)/[PREWHERE](prewhere.md) بند در پرس و جو, از لحاظ فنی می تواند در هر سفارش انجام, مگر اینکه نتیجه `ARRAY JOIN` برای فیلتر کردن استفاده می شود. سفارش پردازش توسط بهینه ساز پرس و جو کنترل می شود. diff --git a/docs/fa/sql-reference/statements/select/distinct.md b/docs/fa/sql-reference/statements/select/distinct.md new file mode 100644 index 00000000000..9c2a454a729 --- /dev/null +++ b/docs/fa/sql-reference/statements/select/distinct.md @@ -0,0 +1,63 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +--- + +# بند مجزا {#select-distinct} + +اگر `SELECT DISTINCT` مشخص شده است, تنها ردیف منحصر به فرد در یک نتیجه پرس و جو باقی خواهد ماند. بنابراین تنها یک ردیف از تمام مجموعه از ردیف به طور کامل تطبیق در نتیجه باقی خواهد ماند. + +## پردازش پوچ {#null-processing} + +`DISTINCT` با این نسخهها کار میکند [NULL](../../syntax.md#null-literal) همانطور که اگر `NULL` یک مقدار خاص بودند, و `NULL==NULL`. به عبارت دیگر در `DISTINCT` نتایج, ترکیب های مختلف با `NULL` رخ می دهد تنها یک بار. این متفاوت است `NULL` پردازش در بسیاری از زمینه های دیگر. + +## جایگزین ها {#alternatives} + +ممکن است با استفاده از همان نتیجه حاصل شود [GROUP BY](group-by.md) در سراسر همان مجموعه ای از ارزش ها به عنوان مشخص شده به عنوان `SELECT` بند, بدون استفاده از هر گونه توابع کل. اما چند تفاوت از وجود دارد `GROUP BY` رویکرد: + +- `DISTINCT` می توان همراه با `GROUP BY`. +- چه زمانی [ORDER BY](order-by.md) حذف شده است و [LIMIT](limit.md) تعریف شده است, پرس و جو متوقف می شود در حال اجرا بلافاصله پس از تعداد مورد نیاز از ردیف های مختلف خوانده شده است. +- بلوک های داده خروجی به عنوان پردازش می شوند, بدون انتظار برای کل پرس و جو را به پایان برساند در حال اجرا. + +## محدودیت ها {#limitations} + +`DISTINCT` پشتیبانی نمی شود اگر `SELECT` حداقل یک ستون جداگانه دارد. + +## مثالها {#examples} + +کلیک پشتیبانی با استفاده از `DISTINCT` و `ORDER BY` بند برای ستون های مختلف در یک پرس و جو. این `DISTINCT` بند قبل از اجرا `ORDER BY` بند بند. + +جدول نمونه: + +``` text +┌─a─┬─b─┐ +│ 2 │ 1 │ +│ 1 │ 2 │ +│ 3 │ 3 │ +│ 2 │ 4 │ +└───┴───┘ +``` + +هنگام انتخاب داده ها با `SELECT DISTINCT a FROM t1 ORDER BY b ASC` پرس و جو, ما نتیجه زیر را دریافت کنید: + +``` text +┌─a─┐ +│ 2 │ +│ 1 │ +│ 3 │ +└───┘ +``` + +اگر ما جهت مرتب سازی را تغییر دهیم `SELECT DISTINCT a FROM t1 ORDER BY b DESC` ما نتیجه زیر را دریافت می کنیم: + +``` text +┌─a─┐ +│ 3 │ +│ 1 │ +│ 2 │ +└───┘ +``` + +سطر `2, 4` قبل از مرتب سازی قطع شد. + +نگاهی به این ویژگی پیاده سازی به حساب زمانی که برنامه نویسی نمایش داده شد. diff --git a/docs/fa/sql-reference/statements/select/format.md b/docs/fa/sql-reference/statements/select/format.md new file mode 100644 index 00000000000..27ac2233375 --- /dev/null +++ b/docs/fa/sql-reference/statements/select/format.md @@ -0,0 +1,18 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +--- + +# بند فرمت {#format-clause} + +تاتر پشتیبانی از طیف گسترده ای از [فرمت های ترتیب](../../../interfaces/formats.md) که می تواند در نتایج پرس و جو در میان چیزهای دیگر استفاده می شود. راه های متعدد برای انتخاب یک فرمت برای وجود دارد `SELECT` خروجی یکی از این موارد مشخص است `FORMAT format` در پایان پرس و جو برای دریافت و در نتیجه داده ها در هر فرمت خاص. + +فرمت خاص ممکن است یا برای راحتی استفاده می شود, ادغام با سیستم های دیگر و یا افزایش عملکرد. + +## قالب پیشفرض {#default-format} + +اگر `FORMAT` بند حذف شده است, فرمت پیش فرض استفاده شده است, که بستگی به هر دو تنظیمات و رابط مورد استفاده برای دسترسی به سرور کلیک. برای [رابط قام](../../../interfaces/http.md) و [مشتری خط فرمان](../../../interfaces/cli.md) در حالت دسته ای فرمت پیش فرض است `TabSeparated`. برای مشتری خط فرمان در حالت تعاملی فرمت پیش فرض است `PrettyCompact` (این تولید جداول جمع و جور انسان قابل خواندن). + +## پیاده سازی اطلاعات {#implementation-details} + +هنگام استفاده از سرویس گیرنده خط فرمان داده ها همیشه بر روی شبکه در فرمت موثر داخلی منتقل می شود (`Native`). مشتری به طور مستقل تفسیر می کند `FORMAT` بند از پرس و جو و فرمت های داده های خود را (در نتیجه تسکین شبکه و سرور از بار اضافی). diff --git a/docs/fa/sql-reference/statements/select/from.md b/docs/fa/sql-reference/statements/select/from.md new file mode 100644 index 00000000000..d2ec8f4c6ff --- /dev/null +++ b/docs/fa/sql-reference/statements/select/from.md @@ -0,0 +1,44 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +--- + +# از بند {#select-from} + +این `FROM` بند منبع برای خواندن داده ها از مشخص: + +- [جدول](../../../engines/table-engines/index.md) +- [خرده فروشی](index.md) لینک بهتر ##} +- [تابع جدول](../../table-functions/index.md#table-functions) + +[JOIN](join.md) و [ARRAY JOIN](array-join.md) بند نیز ممکن است مورد استفاده قرار گیرد به گسترش قابلیت های `FROM` بند بند. + +خرده فروشی دیگر است `SELECT` پرسوجوی که ممکن است در داخل پرانتز مشخص شود `FROM` بند بند. + +`FROM` بند می تواند شامل منابع داده های متعدد, جدا شده توسط کاما, که معادل انجام است [CROSS JOIN](join.md) با اونا. + +## تغییردهنده نهایی {#select-from-final} + +چه زمانی `FINAL` مشخص شده است, تاتر به طور کامل ادغام داده ها قبل از بازگشت به نتیجه و در نتیجه انجام تمام تحولات داده که در طول ادغام برای موتور جدول داده شده اتفاق می افتد. + +این قابل اجرا است در هنگام انتخاب داده ها از جداول که با استفاده از [ادغام](../../../engines/table-engines/mergetree-family/mergetree.md)- خانواده موتور (به جز `GraphiteMergeTree`). همچنین برای پشتیبانی: + +- [تکرار](../../../engines/table-engines/mergetree-family/replication.md) نسخه های `MergeTree` موتورها. +- [نما](../../../engines/table-engines/special/view.md), [بافر](../../../engines/table-engines/special/buffer.md), [توزیع شده](../../../engines/table-engines/special/distributed.md) و [ماده بینی](../../../engines/table-engines/special/materializedview.md) موتورها که بیش از موتورهای دیگر کار می کنند به شرطی که بیش از ایجاد شده اند `MergeTree`- جدول موتور . + +### اشکالاتی {#drawbacks} + +نمایش داده شد که با استفاده از `FINAL` اعدام به همان سرعتی که نمایش داده شد مشابه که نمی, زیرا: + +- پرس و جو در یک موضوع اجرا و داده ها در طول اجرای پرس و جو با هم ادغام شدند. +- نمایش داده شد با `FINAL` خوانده شده ستون کلید اولیه در علاوه بر این به ستون مشخص شده در پرس و جو. + +**در بیشتر موارد, اجتناب از استفاده از `FINAL`.** روش معمول این است که از نمایش های مختلف استفاده کنید که فرایندهای پس زمینه را فرض می کنند `MergeTree` موتور هنوز اتفاق نیفتاده است و با استفاده از تجمع (به عنوان مثال برای دور انداختن تکراری) مقابله می کند. پردازشگر پشتیبانی شده: ##} + +## پیاده سازی اطلاعات {#implementation-details} + +اگر `FROM` بند حذف شده است, داده خواهد شد از خواندن `system.one` جدول +این `system.one` جدول شامل دقیقا یک ردیف است (این جدول همان هدف را به عنوان جدول دوگانه موجود در سایر دساماسها انجام می دهد). + +برای اجرای پرس و جو تمام ستون های ذکر شده در پرس و جو از جدول مناسب استخراج می شوند. هر ستون برای پرس و جو خارجی مورد نیاز نیست از کارخانه های فرعی پرتاب می شود. +اگر پرس و جو هیچ ستون لیست نیست (به عنوان مثال, `SELECT count() FROM t`), برخی از ستون از جدول استخراج به هر حال (کوچکترین ترجیح داده می شود), به منظور محاسبه تعداد ردیف. diff --git a/docs/fa/sql-reference/statements/select/group-by.md b/docs/fa/sql-reference/statements/select/group-by.md new file mode 100644 index 00000000000..9b543a4b606 --- /dev/null +++ b/docs/fa/sql-reference/statements/select/group-by.md @@ -0,0 +1,132 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +--- + +# گروه بر اساس بند {#select-group-by-clause} + +`GROUP BY` بند کلید `SELECT` پرس و جو به حالت تجمع, که به شرح زیر کار می کند: + +- `GROUP BY` بند شامل یک لیست از عبارات (و یا یک عبارت واحد, که در نظر گرفته می شود لیستی از طول یک). این لیست به عنوان یک عمل می کند “grouping key”, در حالی که هر بیان فردی خواهد شد به عنوان یک اشاره “key expressions”. +- تمام عبارات در [SELECT](index.md), [HAVING](having.md) و [ORDER BY](order-by.md) بند **باید** بر اساس عبارات کلیدی محاسبه می شود **یا** روشن [توابع مجموع](../../../sql-reference/aggregate-functions/index.md) بیش از عبارات غیر کلیدی (از جمله ستون ساده). به عبارت دیگر, هر ستون انتخاب شده از جدول باید یا در یک عبارت کلیدی و یا در داخل یک تابع جمع استفاده می شود, اما نه هر دو. +- نتیجه جمع `SELECT` پرس و جو شامل ردیف به عنوان بسیاری از ارزش های منحصر به فرد وجود دارد “grouping key” در جدول منبع. معمولا این نشانه را کاهش می دهد تعداد ردیف, اغلب توسط سفارشات از قدر, اما نه لزوما: تعداد ردیف همان باقی می ماند اگر همه “grouping key” ارزش متمایز بودند. + +!!! note "یادداشت" + یک راه اضافی برای اجرای تجمع بیش از یک جدول وجود دارد. اگر یک پرس و جو شامل ستون جدول تنها در داخل توابع کل, که `GROUP BY clause` می توان حذف, و تجمع توسط یک مجموعه خالی از کلید فرض بر این است. چنین نمایش داده شد همیشه دقیقا یک ردیف بازگشت. + +## پردازش پوچ {#null-processing} + +برای گروه بندی, تفسیر کلیک [NULL](../../syntax.md#null-literal) به عنوان یک ارزش, و `NULL==NULL`. این متفاوت است `NULL` پردازش در بسیاری از زمینه های دیگر. + +در اینجا یک مثال برای نشان دادن این بدان معنی است. + +فرض کنید شما باید این جدول: + +``` text +┌─x─┬────y─┐ +│ 1 │ 2 │ +│ 2 │ ᴺᵁᴸᴸ │ +│ 3 │ 2 │ +│ 3 │ 3 │ +│ 3 │ ᴺᵁᴸᴸ │ +└───┴──────┘ +``` + +پرسوجو `SELECT sum(x), y FROM t_null_big GROUP BY y` نتایج در: + +``` text +┌─sum(x)─┬────y─┐ +│ 4 │ 2 │ +│ 3 │ 3 │ +│ 5 │ ᴺᵁᴸᴸ │ +└────────┴──────┘ +``` + +شما می توانید ببینید که `GROUP BY` برای `y = NULL` خلاصه تا `x`, به عنوان اگر `NULL` این مقدار است. + +اگر شما تصویب چند کلید به `GROUP BY`, نتیجه به شما تمام ترکیبی از انتخاب را, اگر `NULL` یک مقدار خاص بودند. + +## با اصلاح کننده بالغ {#with-totals-modifier} + +اگر `WITH TOTALS` اصلاح مشخص شده است, ردیف دیگر محاسبه خواهد شد. این ردیف ستون های کلیدی حاوی مقادیر پیش فرض (صفر یا خطوط خالی) و ستون های توابع جمع شده با مقادیر محاسبه شده در تمام ردیف ها ( “total” ارزش ها). + +این ردیف اضافی تنها در تولید `JSON*`, `TabSeparated*` و `Pretty*` فرمت ها به طور جداگانه از ردیف های دیگر: + +- داخل `JSON*` فرمت, این ردیف خروجی به عنوان یک جداگانه است ‘totals’ رشته. +- داخل `TabSeparated*` فرمت, ردیف پس از نتیجه اصلی, قبل از یک ردیف خالی (پس از داده های دیگر). +- داخل `Pretty*` فرمت, ردیف خروجی به عنوان یک جدول جداگانه پس از نتیجه اصلی است. +- در فرمت های دیگر در دسترس نیست. + +`WITH TOTALS` می توان در راه های مختلف اجرا زمانی که داشتن حاضر است. رفتار بستگی به ‘totals\_mode’ تنظیمات. + +### پیکربندی پردازش بالغ {#configuring-totals-processing} + +به طور پیش فرض, `totals_mode = 'before_having'`. در این مورد, ‘totals’ محاسبه شده است در تمام ردیف از جمله کسانی که عبور نمی کند از طریق داشتن و `max_rows_to_group_by`. + +جایگزین های دیگر شامل تنها ردیف است که از طریق داشتن در عبور ‘totals’ و رفتار متفاوت با تنظیم `max_rows_to_group_by` و `group_by_overflow_mode = 'any'`. + +`after_having_exclusive` – Don't include rows that didn't pass through `max_rows_to_group_by`. به عبارت دیگر, ‘totals’ کمتر از و یا به همان تعداد از ردیف به عنوان اگر داشته باشد `max_rows_to_group_by` حذف شد. + +`after_having_inclusive` – Include all the rows that didn't pass through ‘max\_rows\_to\_group\_by’ داخل ‘totals’. به عبارت دیگر, ‘totals’ بیش از و یا به همان تعداد از ردیف به عنوان اگر داشته باشد `max_rows_to_group_by` حذف شد. + +`after_having_auto` – Count the number of rows that passed through HAVING. If it is more than a certain amount (by default, 50%), include all the rows that didn't pass through ‘max\_rows\_to\_group\_by’ داخل ‘totals’. در غیر این صورت, را شامل نمی شود. + +`totals_auto_threshold` – By default, 0.5. The coefficient for `after_having_auto`. + +اگر `max_rows_to_group_by` و `group_by_overflow_mode = 'any'` استفاده نمی شود, تمام تغییرات از `after_having` یکسان هستند و شما می توانید از هر یک از این موارد استفاده کنید, `after_having_auto`). + +شما می توانید با استفاده از مجموع در subqueries از جمله subqueries در پیوستن به بند (در این مورد مربوطه مجموع ارزش ترکیب می شوند). + +## مثالها {#examples} + +مثال: + +``` sql +SELECT + count(), + median(FetchTiming > 60 ? 60 : FetchTiming), + count() - sum(Refresh) +FROM hits +``` + +با این حال, در مقابل به استاندارد گذاشتن, اگر جدول هیچ ردیف ندارد (یا هیچ در همه وجود ندارد, و یا هر پس از استفاده از جایی که برای فیلتر کردن وجود ندارد), یک نتیجه خالی بازگشته است, و نه در نتیجه از یکی از ردیف های حاوی مقادیر اولیه از توابع کل. + +همانطور که به خروجی زیر مخالف (و منطبق با استاندارد گذاشتن), شما می توانید برخی از ارزش برخی از ستون است که در یک کلید و یا کل تابع نیست (به جز عبارات ثابت). برای کار در اطراف این, شما می توانید با استفاده از ‘any’ تابع جمع (اولین مقدار مواجه می شوند) یا ‘min/max’. + +مثال: + +``` sql +SELECT + domainWithoutWWW(URL) AS domain, + count(), + any(Title) AS title -- getting the first occurred page header for each domain. +FROM hits +GROUP BY domain +``` + +برای هر مقدار کلیدی مختلف مواجه می شوند, گروه با محاسبه مجموعه ای از مقادیر تابع کل. + +گروه توسط ستون های مجموعه پشتیبانی نمی شود. + +ثابت را نمی توان به عنوان استدلال برای توابع کل مشخص شده است. مثال: مجموع(1). به جای این, شما می توانید از ثابت خلاص. مثال: `count()`. + +## پیاده سازی اطلاعات {#implementation-details} + +تجمع یکی از مهم ترین ویژگی های یک سندرم تونل کارپ ستون گرا است, و به این ترتیب اجرای یکی از قطعات به شدت بهینه سازی شده از خانه رعیتی است. به طور پیش فرض, تجمع در حافظه با استفاده از یک هش جدول انجام. این 40 + تخصص است که به طور خودکار بسته به انتخاب “grouping key” انواع داده ها. + +### گروه در حافظه خارجی {#select-group-by-in-external-memory} + +شما می توانید اطلاعات موقت تخلیه به دیسک را قادر به محدود کردن استفاده از حافظه در طول `GROUP BY`. +این [ا\_فزون\_بر\_گونهی\_گونهی زیر\_گروهها](../../../operations/settings/settings.md#settings-max_bytes_before_external_group_by) تنظیم تعیین کننده مصرف رم را برای تخلیه می کند `GROUP BY` اطلاعات موقت به سیستم فایل. اگر به 0 (به طور پیش فرض), غیر فعال است. + +هنگام استفاده از `max_bytes_before_external_group_by`, توصیه می کنیم که به شما در تنظیم `max_memory_usage` در مورد دو برابر بالا. این لازم است زیرا دو مرحله برای تجمع وجود دارد: خواندن داده ها و تشکیل داده های متوسط (1) و ادغام داده های متوسط (2). واژگون اطلاعات به سیستم فایل تنها می تواند در طول مرحله رخ می دهد 1. اگر داده های موقت ریخته نمی شد, سپس مرحله 2 ممکن است نیاز به همان مقدار از حافظه در مرحله 1. + +برای مثال اگر [\_کاساژ بیشینه](../../../operations/settings/settings.md#settings_max_memory_usage) به 1000000000 تنظیم شد و شما می خواهید به استفاده از تجمع خارجی, این را حس می کند به مجموعه `max_bytes_before_external_group_by` به 10000000000 و `max_memory_usage` به 20000000000. هنگامی که تجمع خارجی باعث شده است (اگر حداقل یک روگرفت از داده های موقت وجود دارد), حداکثر مصرف رم تنها کمی بیشتر از `max_bytes_before_external_group_by`. + +با پردازش پرس و جو توزیع, تجمع خارجی بر روی سرور از راه دور انجام. به منظور سرور درخواست به استفاده از تنها مقدار کمی از رم, تنظیم `distributed_aggregation_memory_efficient` به 1. + +هنگامی که ادغام داده ها به دیسک سرخ, و همچنین زمانی که ادغام نتایج حاصل از سرور از راه دور زمانی که `distributed_aggregation_memory_efficient` تنظیم فعال است, مصرف تا `1/256 * the_number_of_threads` از مقدار کل رم. + +هنگامی که تجمع خارجی فعال است, اگر کمتر از وجود دارد `max_bytes_before_external_group_by` of data (i.e. data was not flushed), the query runs just as fast as without external aggregation. If any temporary data was flushed, the run time will be several times longer (approximately three times). + +اگر شما یک [ORDER BY](order-by.md) با یک [LIMIT](limit.md) پس از `GROUP BY`, سپس مقدار رم استفاده می شود بستگی به مقدار داده ها در `LIMIT` نه تو کل میز اما اگر `ORDER BY` ندارد `LIMIT` فراموش نکنید که مرتب سازی خارجی را فعال کنید (`max_bytes_before_external_sort`). diff --git a/docs/fa/sql-reference/statements/select/having.md b/docs/fa/sql-reference/statements/select/having.md new file mode 100644 index 00000000000..56578743339 --- /dev/null +++ b/docs/fa/sql-reference/statements/select/having.md @@ -0,0 +1,14 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +--- + +# داشتن بند {#having-clause} + +اجازه می دهد تا فیلتر کردن نتایج تجمع تولید شده توسط [GROUP BY](group-by.md). این شبیه به [WHERE](where.md) بند, اما تفاوت این است که `WHERE` قبل از تجمع انجام, در حالی که `HAVING` بعد از اون انجام میشه + +ممکن است که به مرجع نتایج تجمع از `SELECT` بند در `HAVING` بند با نام مستعار خود. متناوبا, `HAVING` بند می تواند بر روی نتایج مصالح اضافی است که در نتایج پرس و جو بازگشت فیلتر کنید. + +## محدودیت ها {#limitations} + +`HAVING` نمی توان مورد استفاده قرار گیرد اگر تجمع انجام نشده است. استفاده `WHERE` در عوض diff --git a/docs/fa/sql-reference/statements/select/index.md b/docs/fa/sql-reference/statements/select/index.md new file mode 100644 index 00000000000..d919751dd37 --- /dev/null +++ b/docs/fa/sql-reference/statements/select/index.md @@ -0,0 +1,158 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_priority: 33 +toc_title: SELECT +--- + +# نحو نمایش داده شد را انتخاب کنید {#select-queries-syntax} + +`SELECT` بازیابی داده ها را انجام می دهد. + +``` sql +[WITH expr_list|(subquery)] +SELECT [DISTINCT] expr_list +[FROM [db.]table | (subquery) | table_function] [FINAL] +[SAMPLE sample_coeff] +[ARRAY JOIN ...] +[GLOBAL] [ANY|ALL] [INNER|LEFT|RIGHT|FULL|CROSS] [OUTER] JOIN (subquery)|table USING columns_list +[PREWHERE expr] +[WHERE expr] +[GROUP BY expr_list] [WITH TOTALS] +[HAVING expr] +[ORDER BY expr_list] +[LIMIT [offset_value, ]n BY columns] +[LIMIT [n, ]m] +[UNION ALL ...] +[INTO OUTFILE filename] +[FORMAT format] +``` + +همه بند اختیاری هستند, به جز برای لیست مورد نیاز از عبارات بلافاصله پس از `SELECT` که بیشتر تحت پوشش است [در زیر](#select-clause). + +ویژگی های هر بند اختیاری در بخش های جداگانه تحت پوشش, که در همان جهت ذکر شده که اجرا می شوند: + +- [با بند](with.md) +- [بند مجزا](distinct.md) +- [از بند](from.md) +- [بند نمونه](sample.md) +- [پیوستن بند](join.md) +- [در بند](prewhere.md) +- [بند کجا](where.md) +- [گروه بر اساس بند](group-by.md) +- [محدود کردن بند](limit-by.md) +- [داشتن بند](having.md) +- [انتخاب بند](#select-clause) +- [بند محدود](limit.md) +- [اتحادیه همه بند](union-all.md) + +## انتخاب بند {#select-clause} + +[عبارتها](../../syntax.md#syntax-expressions) مشخص شده در `SELECT` بند بعد از تمام عملیات در بند بالا توضیح محاسبه به پایان رسید. این عبارات کار می کنند به عنوان اگر به ردیف جداگانه در نتیجه اعمال می شود. اگر عبارات در `SELECT` بند شامل مجموع توابع سپس ClickHouse فرآیندهای کل توابع و عبارات استفاده می شود به عنوان استدلال های خود را در طول [GROUP BY](group-by.md) تجمع. + +اگر شما می خواهید که شامل تمام ستون ها در نتیجه, استفاده از ستاره (`*`) نماد. به عنوان مثال, `SELECT * FROM ...`. + +برای مطابقت با برخی از ستون ها در نتیجه با یک [شماره 2](https://en.wikipedia.org/wiki/RE2_(software)) عبارت منظم می توانید از `COLUMNS` اصطلاح. + +``` sql +COLUMNS('regexp') +``` + +برای مثال جدول را در نظر بگیرید: + +``` sql +CREATE TABLE default.col_names (aa Int8, ab Int8, bc Int8) ENGINE = TinyLog +``` + +پرس و جو زیر داده ها را از تمام ستون های حاوی `a` نماد به نام خود. + +``` sql +SELECT COLUMNS('a') FROM col_names +``` + +``` text +┌─aa─┬─ab─┐ +│ 1 │ 1 │ +└────┴────┘ +``` + +ستون های انتخاب شده به ترتیب حروف الفبا بازگردانده نمی شوند. + +شما می توانید چند استفاده کنید `COLUMNS` عبارات در پرس و جو و اعمال توابع را به خود. + +به عنوان مثال: + +``` sql +SELECT COLUMNS('a'), COLUMNS('c'), toTypeName(COLUMNS('c')) FROM col_names +``` + +``` text +┌─aa─┬─ab─┬─bc─┬─toTypeName(bc)─┐ +│ 1 │ 1 │ 1 │ Int8 │ +└────┴────┴────┴────────────────┘ +``` + +هر ستون توسط `COLUMNS` بیان به تابع به عنوان یک استدلال جداگانه منتقل می شود. همچنین شما می توانید استدلال های دیگر به تابع منتقل می کند اگر از. مراقب باشید در هنگام استفاده از توابع. اگر یک تابع تعداد استدلال شما را به تصویب را پشتیبانی نمی کند, خانه عروسکی می اندازد یک استثنا. + +به عنوان مثال: + +``` sql +SELECT COLUMNS('a') + COLUMNS('c') FROM col_names +``` + +``` text +Received exception from server (version 19.14.1): +Code: 42. DB::Exception: Received from localhost:9000. DB::Exception: Number of arguments for function plus doesn't match: passed 3, should be 2. +``` + +در این مثال, `COLUMNS('a')` بازگرداندن دو ستون: `aa` و `ab`. `COLUMNS('c')` بازگرداندن `bc` ستون. این `+` اپراتور نمی تواند به اعمال 3 استدلال, بنابراین تاتر می اندازد یک استثنا با پیام مربوطه. + +ستون که همسان `COLUMNS` بیان می تواند انواع داده های مختلف داشته باشد. اگر `COLUMNS` هیچ ستون مطابقت ندارد و تنها بیان در است `SELECT`, فاحشه خانه می اندازد یک استثنا. + +### ستاره {#asterisk} + +شما می توانید یک ستاره در هر بخشی از یک پرس و جو به جای بیان قرار داده است. هنگامی که پرس و جو تجزیه و تحلیل, ستاره به یک لیست از تمام ستون جدول گسترش (به استثنای `MATERIALIZED` و `ALIAS` ستون). تنها چند مورد وجود دارد که با استفاده از ستاره توجیه می شود: + +- هنگام ایجاد یک روگرفت جدول. +- برای جداول حاوی فقط چند ستون, مانند جداول سیستم. +- برای گرفتن اطلاعات در مورد چه ستون در یک جدول هستند. در این مورد, تنظیم `LIMIT 1`. اما بهتر است از `DESC TABLE` پرس و جو. +- هنگامی که فیلتراسیون قوی در تعداد کمی از ستون ها با استفاده از وجود دارد `PREWHERE`. +- در subqueries (از ستون های که مورد نیاز نیست برای خارجی پرس و جو از مطالعه حذف شدند subqueries). + +در تمام موارد دیگر, ما توصیه نمی با استفاده از ستاره, زیرا تنها به شما می دهد اشکالاتی از یک سندرم روده تحریک پذیر ستونی به جای مزایای. به عبارت دیگر با استفاده از ستاره توصیه نمی شود. + +### ارزش های شدید {#extreme-values} + +علاوه بر نتایج, شما همچنین می توانید حداقل و حداکثر ارزش برای ستون نتایج از. برای انجام این کار, تنظیم **افراط** تنظیم به 1. حداقل و حداکثر برای انواع عددی محاسبه, تاریخ, و تاریخ با بار. برای ستون های دیگر مقادیر پیش فرض خروجی هستند. + +An extra two rows are calculated – the minimums and maximums, respectively. These extra two rows are output in `JSON*`, `TabSeparated*` و `Pretty*` [فرشها](../../../interfaces/formats.md), جدا از ردیف های دیگر. خروجی برای فرمت های دیگر نیستند. + +داخل `JSON*` فرمت, ارزش شدید خروجی در یک جداگانه ‘extremes’ رشته. داخل `TabSeparated*` فرمت, ردیف پس از نتیجه اصلی, و بعد از ‘totals’ اگر در حال حاضر. این است که توسط یک ردیف خالی قبل (پس از داده های دیگر). داخل `Pretty*` فرمت, ردیف خروجی به عنوان یک جدول جداگانه پس از نتیجه اصلی است, و بعد از `totals` اگر در حال حاضر. + +مقادیر شدید برای ردیف قبل محاسبه می شود `LIMIT` اما بعد از `LIMIT BY`. با این حال, هنگام استفاده از `LIMIT offset, size`, ردیف قبل از `offset` در `extremes`. در درخواست جریان, نتیجه نیز ممکن است شامل تعداد کمی از ردیف که از طریق تصویب `LIMIT`. + +### یادداشتها {#notes} + +شما می توانید مترادف استفاده کنید (`AS` نام مستعار) در هر بخشی از یک پرس و جو. + +این `GROUP BY` و `ORDER BY` بند انجام استدلال موضعی را پشتیبانی نمی کند. این در تضاد خروجی زیر, اما مطابق با استاندارد گذاشتن. به عنوان مثال, `GROUP BY 1, 2` will be interpreted as grouping by constants (i.e. aggregation of all rows into one). + +## پیاده سازی اطلاعات {#implementation-details} + +اگر پرس و جو حذف `DISTINCT`, `GROUP BY` و `ORDER BY` بند و `IN` و `JOIN` کارخانه های فرعی, پرس و جو خواهد شد به طور کامل جریان پردازش, با استفاده از ای(1) مقدار رم. در غیر این صورت, پرس و جو ممکن است مقدار زیادی از رم مصرف اگر محدودیت های مناسب مشخص نشده است: + +- `max_memory_usage` +- `max_rows_to_group_by` +- `max_rows_to_sort` +- `max_rows_in_distinct` +- `max_bytes_in_distinct` +- `max_rows_in_set` +- `max_bytes_in_set` +- `max_rows_in_join` +- `max_bytes_in_join` +- `max_bytes_before_external_sort` +- `max_bytes_before_external_group_by` + +برای کسب اطلاعات بیشتر به بخش مراجعه کنید “Settings”. ممکن است که به استفاده از مرتب سازی خارجی (صرفه جویی در جداول موقت به یک دیسک) و تجمع خارجی. + +{## [مقاله اصلی](https://clickhouse.tech/docs/en/sql-reference/statements/select/) ##} diff --git a/docs/fa/sql-reference/statements/select/into-outfile.md b/docs/fa/sql-reference/statements/select/into-outfile.md new file mode 100644 index 00000000000..cdb428b4d08 --- /dev/null +++ b/docs/fa/sql-reference/statements/select/into-outfile.md @@ -0,0 +1,14 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +--- + +# به OUTFILE بند {#into-outfile-clause} + +افزودن `INTO OUTFILE filename` بند (جایی که نام فایل یک رشته تحت اللفظی است) به `SELECT query` برای تغییر مسیر خروجی خود را به فایل مشخص شده در سمت سرویس گیرنده. + +## پیاده سازی اطلاعات {#implementation-details} + +- این قابلیت در دسترس است [مشتری خط فرمان](../../../interfaces/cli.md) و [کلیک-محلی](../../../operations/utilities/clickhouse-local.md). بنابراین پرس و جو از طریق ارسال [رابط قام](../../../interfaces/http.md) شکست مواجه خواهد شد. +- پرس و جو شکست مواجه خواهد شد اگر یک فایل با نام فایل مشابه در حال حاضر وجود دارد. +- پیشفرض [فرمت خروجی](../../../interfaces/formats.md) هست `TabSeparated` (مانند در خط فرمان حالت دسته ای مشتری). diff --git a/docs/fa/sql-reference/statements/select/join.md b/docs/fa/sql-reference/statements/select/join.md new file mode 100644 index 00000000000..21e19c124fd --- /dev/null +++ b/docs/fa/sql-reference/statements/select/join.md @@ -0,0 +1,191 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +--- + +# پیوستن بند {#select-join} + +اضافه کردن تولید یک جدول جدید با ترکیب ستون از یک یا چند جدول با استفاده از مقادیر مشترک به هر. این یک عملیات مشترک در پایگاه داده با پشتیبانی از گذاشتن است, که مربوط به [جبر رابطه ای](https://en.wikipedia.org/wiki/Relational_algebra#Joins_and_join-like_operators) ملحق شو مورد خاص یک جدول پیوستن است که اغلب به عنوان اشاره “self-join”. + +نحو: + +``` sql +SELECT +FROM +[GLOBAL] [ANY|ALL|ASOF] [INNER|LEFT|RIGHT|FULL|CROSS] [OUTER|SEMI|ANTI] JOIN +(ON )|(USING ) ... +``` + +عبارات از `ON` بند و ستون از `USING` بند نامیده می شوند “join keys”. مگر در مواردی که در غیر این صورت اعلام, پیوستن به تولید یک [محصول دکارتی](https://en.wikipedia.org/wiki/Cartesian_product) از ردیف با تطبیق “join keys”, که ممکن است نتایج را با ردیف خیلی بیشتر از جداول منبع تولید. + +## انواع پشتیبانی شده از پیوستن {#select-join-types} + +همه استاندارد [SQL JOIN](https://en.wikipedia.org/wiki/Join_(SQL)) انواع پشتیبانی می شوند: + +- `INNER JOIN`, تنها ردیف تطبیق برگردانده می شوند. +- `LEFT OUTER JOIN`, ردیف غیر تطبیق از جدول سمت چپ علاوه بر تطبیق ردیف بازگشت. +- `RIGHT OUTER JOIN`, ردیف غیر تطبیق از جدول سمت چپ علاوه بر تطبیق ردیف بازگشت. +- `FULL OUTER JOIN`, ردیف غیر تطبیق از هر دو جدول علاوه بر تطبیق ردیف بازگشت. +- `CROSS JOIN`, تولید محصول دکارتی از تمام جداول, “join keys” هستند **نه** مشخص. + +`JOIN` بدون نوع مشخص نشان میدهد `INNER`. کلیدواژه `OUTER` می توان با خیال راحت حذف. نحو جایگزین برای `CROSS JOIN` مشخص جداول چندگانه در [از بند](from.md) جدا شده توسط کاما. + +پیوستن به انواع اضافی موجود در کلیک: + +- `LEFT SEMI JOIN` و `RIGHT SEMI JOIN`, یک لیست سفید در “join keys”, بدون تولید محصول دکارتی. +- `LEFT ANTI JOIN` و `RIGHT ANTI JOIN`, لیست سیاه در “join keys”, بدون تولید محصول دکارتی. + +## سختی {#select-join-strictness} + +تغییر چگونگی تطبیق توسط “join keys” انجام شده است + +- `ALL` — The standard `JOIN` رفتار در گذاشتن همانطور که در بالا توضیح. به طور پیش فرض. +- `ANY` — Partially (for opposite side of `LEFT` و `RIGHT`) یا به طور کامل (برای `INNER` و `FULL`) غیر فعال محصول دکارتی برای استاندارد `JOIN` انواع. +- `ASOF` — For joining sequences with a non-exact match. `ASOF JOIN` استفاده در زیر توضیح داده شده است. + +!!! note "یادداشت" + مقدار سختگیرانه پیش فرض را می توان با استفاده از لغو [بررسی اجمالی](../../../operations/settings/settings.md#settings-join_default_strictness) تنظیمات. + +### از این رو پیوستن به استفاده {#asof-join-usage} + +`ASOF JOIN` مفید است زمانی که شما نیاز به پیوستن به سوابق که هیچ بازی دقیق. + +جداول برای `ASOF JOIN` باید یک ستون توالی دستور داده اند. این ستون نمی تواند به تنهایی در یک جدول باشد و باید یکی از انواع داده ها باشد: `UInt32`, `UInt64`, `Float32`, `Float64`, `Date` و `DateTime`. + +نحو `ASOF JOIN ... ON`: + +``` sql +SELECT expressions_list +FROM table_1 +ASOF LEFT JOIN table_2 +ON equi_cond AND closest_match_cond +``` + +شما می توانید هر تعداد از شرایط برابری و دقیقا یکی از نزدیک ترین شرایط بازی استفاده کنید. به عنوان مثال, `SELECT count() FROM table_1 ASOF LEFT JOIN table_2 ON table_1.a == table_2.b AND table_2.t <= table_1.t`. + +شرایط پشتیبانی شده برای نزدیک ترین بازی: `>`, `>=`, `<`, `<=`. + +نحو `ASOF JOIN ... USING`: + +``` sql +SELECT expressions_list +FROM table_1 +ASOF JOIN table_2 +USING (equi_column1, ... equi_columnN, asof_column) +``` + +`ASOF JOIN` استفاده `equi_columnX` برای پیوستن به برابری و `asof_column` برای پیوستن به نزدیک ترین مسابقه با `table_1.asof_column >= table_2.asof_column` شرط. این `asof_column` ستون همیشه یکی از گذشته در `USING` بند بند. + +مثلا, جداول زیر را در نظر بگیرید: + + table_1 table_2 + event | ev_time | user_id event | ev_time | user_id + ----------|---------|---------- ----------|---------|---------- + ... ... + event_1_1 | 12:00 | 42 event_2_1 | 11:59 | 42 + ... event_2_2 | 12:30 | 42 + event_1_2 | 13:00 | 42 event_2_3 | 13:00 | 42 + ... ... + +`ASOF JOIN` می توانید برچسب زمان از یک رویداد کاربر از را `table_1` و پیدا کردن یک رویداد در `table_2` جایی که برچسب زمان نزدیک به زمان این رویداد از `table_1` مربوط به نزدیک ترین شرایط بازی. مقادیر برچسب زمان برابر نزدیک ترین در صورت موجود بودن. اینجا `user_id` ستون را می توان برای پیوستن به برابری و `ev_time` ستون را می توان برای پیوستن به در نزدیک ترین بازی استفاده می شود. در مثال ما, `event_1_1` می توان با پیوست `event_2_1` و `event_1_2` می توان با پیوست `event_2_3` اما `event_2_2` نمیشه عضو شد + +!!! note "یادداشت" + `ASOF` پیوستن است **نه** پردازشگر پشتیبانی شده: [پیوستن](../../../engines/table-engines/special/join.md) موتور جدول. + +## توزیع پیوستن {#global-join} + +دو راه برای اجرای پیوستن به جداول توزیع شده وجود دارد: + +- هنگام استفاده از نرمال `JOIN` پرس و جو به سرورهای راه دور ارسال می شود. زیرکریزها روی هر کدام اجرا می شوند تا میز مناسب را بسازند و پیوستن با این جدول انجام می شود. به عبارت دیگر, جدول سمت راست بر روی هر سرور تشکیل به طور جداگانه. +- هنگام استفاده از `GLOBAL ... JOIN`, اول سرور درخواست کننده اجرا می شود یک خرده فروشی برای محاسبه جدول سمت راست. این جدول موقت به هر سرور از راه دور منتقل می شود و نمایش داده می شود با استفاده از داده های موقت منتقل می شود. + +مراقب باشید در هنگام استفاده از `GLOBAL`. برای کسب اطلاعات بیشتر, دیدن [توزیع subqueries](../../operators/in.md#select-distributed-subqueries) بخش. + +## توصیه های استفاده {#usage-recommendations} + +### پردازش سلولهای خالی یا خالی {#processing-of-empty-or-null-cells} + +در حالی که پیوستن به جداول سلول های خالی ظاهر می شود. تنظیمات [ارزشهای خبری عبارتند از:](../../../operations/settings/settings.md#join_use_nulls) تعریف چگونه خانه را پر می کند این سلول ها. + +اگر `JOIN` کلید ها [Nullable](../../data-types/nullable.md) زمینه, ردیف که حداقل یکی از کلید های دارای ارزش [NULL](../../../sql-reference/syntax.md#null-literal) عضو نشده. + +### نحو {#syntax} + +ستون های مشخص شده در `USING` باید نام های مشابه در هر دو کارخانه های فرعی دارند, و ستون های دیگر باید متفاوت به نام. شما می توانید نام مستعار برای تغییر نام ستون ها در زیرکریز استفاده کنید. + +این `USING` بند یک یا چند ستون برای پیوستن به مشخص, که ایجاد برابری این ستون. لیست ستون ها بدون براکت تنظیم شده است. شرایط پیوستن پیچیده تر پشتیبانی نمی شوند. + +### محدودیت نحو {#syntax-limitations} + +برای چند `JOIN` بند در یک `SELECT` پرسوجو: + +- گرفتن تمام ستون ها از طریق `*` در دسترس است تنها اگر جداول پیوست, نمی فرعی. +- این `PREWHERE` بند در دسترس نیست. + +برای `ON`, `WHERE` و `GROUP BY` بند: + +- عبارات دلخواه را نمی توان در `ON`, `WHERE` و `GROUP BY` بند, اما شما می توانید یک عبارت در یک تعریف `SELECT` بند و سپس در این بند از طریق یک نام مستعار استفاده کنید. + +### عملکرد {#performance} + +هنگامی که در حال اجرا `JOIN` بهینه سازی سفارش اعدام در رابطه با سایر مراحل پرس و جو وجود ندارد. پیوستن (جستجو در جدول سمت راست) قبل از فیلتر کردن در اجرا می شود `WHERE` و قبل از تجمع. + +هر بار که پرس و جو با همان اجرا شود `JOIN`, خرده فروشی است دوباره اجرا به دلیل نتیجه ذخیره سازی نیست. برای جلوگیری از این, استفاده از ویژه [پیوستن](../../../engines/table-engines/special/join.md) موتور جدول, که مجموعه ای تهیه شده برای پیوستن است که همیشه در رم. + +در بعضی موارد کارایی بیشتری برای استفاده دارد [IN](../../operators/in.md) به جای `JOIN`. + +اگر شما نیاز به یک `JOIN` برای پیوستن به جداول بعد (این جداول نسبتا کوچک است که شامل خواص ابعاد هستند, مانند نام برای کمپین های تبلیغاتی), یک `JOIN` ممکن است بسیار مناسب با توجه به این واقعیت است که جدول سمت راست برای هر پرس و جو دوباره قابل دسترسی است. برای چنین مواردی وجود دارد “external dictionaries” ویژگی است که شما باید به جای استفاده از `JOIN`. برای کسب اطلاعات بیشتر, دیدن [واژهنامهها خارجی](../../dictionaries/external-dictionaries/external-dicts.md) بخش. + +### محدودیت حافظه {#memory-limitations} + +به طور پیش فرض, تاتر با استفاده از [هش پیوستن](https://en.wikipedia.org/wiki/Hash_join) الگوریتم. تاتر طول می کشد `` و یک جدول هش را در رم ایجاد می کند. پس از برخی از حد مصرف حافظه, خانه رعیتی می افتد به ادغام پیوستن الگوریتم. + +اگر شما نیاز به محدود کردن پیوستن به مصرف حافظه عملیات استفاده از تنظیمات زیر: + +- [\_پاک کردن \_روشن گرافیک](../../../operations/settings/query-complexity.md#settings-max_rows_in_join) — Limits number of rows in the hash table. +- [\_پویش همیشگی](../../../operations/settings/query-complexity.md#settings-max_bytes_in_join) — Limits size of the hash table. + +هنگامی که هر یک از این محدودیت رسیده است, کلیک به عنوان عمل می کند [\_شروع مجدد](../../../operations/settings/query-complexity.md#settings-join_overflow_mode) تنظیم دستور. + +## مثالها {#examples} + +مثال: + +``` sql +SELECT + CounterID, + hits, + visits +FROM +( + SELECT + CounterID, + count() AS hits + FROM test.hits + GROUP BY CounterID +) ANY LEFT JOIN +( + SELECT + CounterID, + sum(Sign) AS visits + FROM test.visits + GROUP BY CounterID +) USING CounterID +ORDER BY hits DESC +LIMIT 10 +``` + +``` text +┌─CounterID─┬───hits─┬─visits─┐ +│ 1143050 │ 523264 │ 13665 │ +│ 731962 │ 475698 │ 102716 │ +│ 722545 │ 337212 │ 108187 │ +│ 722889 │ 252197 │ 10547 │ +│ 2237260 │ 196036 │ 9522 │ +│ 23057320 │ 147211 │ 7689 │ +│ 722818 │ 90109 │ 17847 │ +│ 48221 │ 85379 │ 4652 │ +│ 19762435 │ 77807 │ 7026 │ +│ 722884 │ 77492 │ 11056 │ +└───────────┴────────┴────────┘ +``` diff --git a/docs/fa/sql-reference/statements/select/limit-by.md b/docs/fa/sql-reference/statements/select/limit-by.md new file mode 100644 index 00000000000..295b98090bc --- /dev/null +++ b/docs/fa/sql-reference/statements/select/limit-by.md @@ -0,0 +1,71 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +--- + +# محدود کردن بند {#limit-by-clause} + +پرس و جو با `LIMIT n BY expressions` بند اول را انتخاب می کند `n` ردیف برای هر مقدار مجزا از `expressions`. کلید برای `LIMIT BY` می تواند شامل هر تعداد از [عبارتها](../../syntax.md#syntax-expressions). + +تاتر از انواع نحو زیر: + +- `LIMIT [offset_value, ]n BY expressions` +- `LIMIT n OFFSET offset_value BY expressions` + +در طول پردازش پرس و جو, خانه را انتخاب داده دستور داد با مرتب سازی کلید. کلید مرتب سازی به صراحت با استفاده از یک مجموعه [ORDER BY](order-by.md) بند یا به طور ضمنی به عنوان یک ویژگی از موتور جدول. سپس کلیک کنیداوس اعمال می شود `LIMIT n BY expressions` و اولین را برمی گرداند `n` ردیف برای هر ترکیب مجزا از `expressions`. اگر `OFFSET` مشخص شده است, سپس برای هر بلوک داده که متعلق به یک ترکیب متمایز از `expressions`. `offset_value` تعداد ردیف از ابتدای بلوک و حداکثر می گرداند `n` ردیف به عنوان یک نتیجه. اگر `offset_value` بزرگتر از تعدادی از ردیف در بلوک داده است, کلیک بازگشت صفر ردیف از بلوک. + +!!! note "یادداشت" + `LIMIT BY` مربوط به [LIMIT](limit.md). هر دو را می توان در همان پرس و جو استفاده کرد. + +## مثالها {#examples} + +جدول نمونه: + +``` sql +CREATE TABLE limit_by(id Int, val Int) ENGINE = Memory; +INSERT INTO limit_by VALUES (1, 10), (1, 11), (1, 12), (2, 20), (2, 21); +``` + +نمایش داده شد: + +``` sql +SELECT * FROM limit_by ORDER BY id, val LIMIT 2 BY id +``` + +``` text +┌─id─┬─val─┐ +│ 1 │ 10 │ +│ 1 │ 11 │ +│ 2 │ 20 │ +│ 2 │ 21 │ +└────┴─────┘ +``` + +``` sql +SELECT * FROM limit_by ORDER BY id, val LIMIT 1, 2 BY id +``` + +``` text +┌─id─┬─val─┐ +│ 1 │ 11 │ +│ 1 │ 12 │ +│ 2 │ 21 │ +└────┴─────┘ +``` + +این `SELECT * FROM limit_by ORDER BY id, val LIMIT 2 OFFSET 1 BY id` پرس و جو همان نتیجه را برمی گرداند. + +پرس و جو زیر را برمی گرداند بالا 5 ارجاع برای هر `domain, device_type` جفت با حداکثر 100 ردیف در مجموع (`LIMIT n BY + LIMIT`). + +``` sql +SELECT + domainWithoutWWW(URL) AS domain, + domainWithoutWWW(REFERRER_URL) AS referrer, + device_type, + count() cnt +FROM hits +GROUP BY domain, referrer, device_type +ORDER BY cnt DESC +LIMIT 5 BY domain, device_type +LIMIT 100 +``` diff --git a/docs/fa/sql-reference/statements/select/limit.md b/docs/fa/sql-reference/statements/select/limit.md new file mode 100644 index 00000000000..5e862b2ec34 --- /dev/null +++ b/docs/fa/sql-reference/statements/select/limit.md @@ -0,0 +1,14 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +--- + +# بند محدود {#limit-clause} + +`LIMIT m` اجازه می دهد برای انتخاب اولین `m` ردیف از نتیجه. + +`LIMIT n, m` اجازه می دهد تا انتخاب کنید `m` ردیف از نتیجه پس از پرش برای اولین بار `n` ردیف این `LIMIT m OFFSET n` نحو معادل است. + +`n` و `m` باید اعداد صحیح غیر منفی باشد. + +اگر وجود ندارد [ORDER BY](order-by.md) بند که به صراحت انواع نتایج, انتخاب ردیف برای نتیجه ممکن است خودسرانه و غیر قطعی. diff --git a/docs/fa/sql-reference/statements/select/order-by.md b/docs/fa/sql-reference/statements/select/order-by.md new file mode 100644 index 00000000000..3cc263efcad --- /dev/null +++ b/docs/fa/sql-reference/statements/select/order-by.md @@ -0,0 +1,72 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +--- + +# ORDER BY {#select-order-by} + +این `ORDER BY` بند شامل یک لیست از عبارات, که می تواند هر یک را با نسبت داده `DESC` (نزولی) یا `ASC` (صعودی) اصلاح که تعیین جهت مرتب سازی. اگر جهت مشخص نشده است, `ASC` فرض بر این است, بنابراین معمولا حذف. جهت مرتب سازی شامل یک عبارت واحد, نه به کل لیست. مثال: `ORDER BY Visits DESC, SearchPhrase` + +ردیف که مقادیر یکسان برای لیست عبارات مرتب سازی خروجی در جهت دلخواه, که همچنین می تواند غیر قطعی شود (مختلف در هر زمان). +اگر منظور توسط بند حذف شده است, منظور از ردیف نیز تعریف نشده, و ممکن است غیر قطعی و همچنین. + +## مرتب سازی مقادیر ویژه {#sorting-of-special-values} + +دو روش برای `NaN` و `NULL` مرتب سازی سفارش: + +- به طور پیش فرض و یا با `NULLS LAST` تغییردهنده: ابتدا مقادیر, سپس `NaN` پس `NULL`. +- با `NULLS FIRST` تغییردهنده: اول `NULL` پس `NaN`, سپس ارزش های دیگر. + +### مثال {#example} + +برای جدول + +``` text +┌─x─┬────y─┐ +│ 1 │ ᴺᵁᴸᴸ │ +│ 2 │ 2 │ +│ 1 │ nan │ +│ 2 │ 2 │ +│ 3 │ 4 │ +│ 5 │ 6 │ +│ 6 │ nan │ +│ 7 │ ᴺᵁᴸᴸ │ +│ 6 │ 7 │ +│ 8 │ 9 │ +└───┴──────┘ +``` + +اجرای پرس و جو `SELECT * FROM t_null_nan ORDER BY y NULLS FIRST` برای دریافت: + +``` text +┌─x─┬────y─┐ +│ 1 │ ᴺᵁᴸᴸ │ +│ 7 │ ᴺᵁᴸᴸ │ +│ 1 │ nan │ +│ 6 │ nan │ +│ 2 │ 2 │ +│ 2 │ 2 │ +│ 3 │ 4 │ +│ 5 │ 6 │ +│ 6 │ 7 │ +│ 8 │ 9 │ +└───┴──────┘ +``` + +هنگامی که اعداد ممیز شناور طبقه بندی شده اند, نان جدا از ارزش های دیگر هستند. صرف نظر از نظم مرتب سازی, نان در پایان. به عبارت دیگر برای مرتب سازی صعودی قرار می گیرند همانطور که بزرگتر از همه شماره های دیگر هستند در حالی که برای مرتب سازی کوچکتر از بقیه قرار می گیرند. + +## پشتیبانی تلفیقی {#collation-support} + +برای مرتب سازی بر اساس مقادیر رشته, شما می توانید میترا مشخص (مقایسه). مثال: `ORDER BY SearchPhrase COLLATE 'tr'` - برای مرتب سازی بر اساس کلمه کلیدی به ترتیب صعودی, با استفاده از الفبای ترکی, حساس به حروف, فرض کنید که رشته ها سخن گفتن-8 کد گذاری. تلفیق می تواند مشخص شود یا نه برای هر عبارت به منظور به طور مستقل. اگر مرکز کنترل و یا مرکز کنترل خارج رحمی مشخص شده است, تلفیقی بعد از مشخص. هنگام استفاده از برخورد, مرتب سازی است که همیشه غیر حساس به حروف. + +ما فقط توصیه می کنیم با استفاده از تلفیق برای مرتب سازی نهایی تعداد کمی از ردیف, از مرتب سازی با تلفیقی کمتر موثر تر از مرتب سازی طبیعی با بایت است. + +## پیاده سازی اطلاعات {#implementation-details} + +رم کمتر استفاده می شود اگر به اندازه کافی کوچک [LIMIT](limit.md) علاوه بر مشخص شده است `ORDER BY`. در غیر این صورت, مقدار حافظه صرف متناسب با حجم داده ها برای مرتب سازی است. برای پردازش پرس و جو توزیع, اگر [GROUP BY](group-by.md) حذف شده است, مرتب سازی تا حدی بر روی سرور از راه دور انجام, و نتایج در سرور درخواست با هم ادغام شدند. این به این معنی است که برای مرتب سازی توزیع, حجم داده ها برای مرتب کردن می تواند بیشتر از مقدار حافظه بر روی یک سرور واحد. + +در صورتی که رم به اندازه کافی وجود ندارد, ممکن است به انجام مرتب سازی در حافظه خارجی (ایجاد فایل های موقت بر روی یک دیسک). از تنظیمات استفاده کنید `max_bytes_before_external_sort` برای این منظور. اگر قرار است 0 (به طور پیش فرض), مرتب سازی خارجی غیر فعال است. اگر فعال باشد, زمانی که حجم داده ها برای مرتب کردن بر اساس تعداد مشخصی از بایت می رسد, اطلاعات جمع شده مرتب شده و ریخته را به یک فایل موقت. پس از همه داده ها خوانده شده است, تمام فایل های طبقه بندی شده اند با هم ادغام شدند و نتایج خروجی. فایل ها به نوشته `/var/lib/clickhouse/tmp/` دایرکتوری در پیکربندی (به طور پیش فرض, اما شما می توانید با استفاده از `tmp_path` پارامتر برای تغییر این تنظیم). + +در حال اجرا یک پرس و جو ممکن است حافظه بیش از استفاده `max_bytes_before_external_sort`. به همین دلیل این تنظیم باید یک مقدار قابل توجهی کوچکتر از `max_memory_usage`. به عنوان مثال, اگر سرور شما 128 گیگابایت رم و شما نیاز به اجرای یک پرس و جو واحد, تنظیم `max_memory_usage` به 100 گیگابایت, و `max_bytes_before_external_sort` به 80 گیگابایت. + +مرتب سازی خارجی کار می کند بسیار کمتر به طور موثر از مرتب سازی در رم. diff --git a/docs/fa/sql-reference/statements/select/prewhere.md b/docs/fa/sql-reference/statements/select/prewhere.md new file mode 100644 index 00000000000..a729b9df86e --- /dev/null +++ b/docs/fa/sql-reference/statements/select/prewhere.md @@ -0,0 +1,22 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +--- + +# در بند {#prewhere-clause} + +در اینجا بهینه سازی برای اعمال فیلتر موثر تر است. این به طور پیش فرض حتی اگر فعال باشد `PREWHERE` بند به صراحت مشخص نشده است. این به طور خودکار در حال حرکت بخشی از کار می کند [WHERE](where.md) شرط به مرحله قبل از کجا. نقش `PREWHERE` بند تنها برای کنترل این بهینه سازی اگر شما فکر می کنم که شما می دانید که چگونه به انجام این کار بهتر از پیش فرض اتفاق می افتد است. + +با prewhere بهینه سازی در ابتدا تنها ستون لازم برای اجرای prewhere بیان بخوانید. سپس ستون های دیگر خوانده می شوند که برای اجرای بقیه پرس و جو مورد نیاز است اما تنها بلوک هایی که بیان قبل از هر کجا است “true” حداقل برای برخی از ردیف. اگر بلوک های زیادی وجود دارد که بیان قبل از کجا است “false” برای تمام سطر و جایی نیاز ستون کمتر از سایر بخش های پرس و جو, این اغلب اجازه می دهد تا به خواندن اطلاعات بسیار کمتر از دیسک برای اجرای پرس و جو. + +## کنترل دستی {#controlling-prewhere-manually} + +بند همان معنی به عنوان `WHERE` بند بند. تفاوت در این است که داده ها از جدول خوانده می شوند. هنگامی که دستی کنترل می شود `PREWHERE` برای شرایط فیلتراسیون که توسط اقلیتی از ستون ها در پرس و جو استفاده می شود, اما که فیلتراسیون داده های قوی. این باعث کاهش حجم داده ها به خواندن. + +پرسوجو ممکن است همزمان مشخص شود `PREWHERE` و `WHERE`. در این مورد, `PREWHERE` مقدمها `WHERE`. + +اگر `optimize_move_to_prewhere` تنظیم تنظیم شده است به 0, اکتشافی به طور خودکار حرکت بخش هایی از عبارات از `WHERE` به `PREWHERE` غیرفعال می شوند. + +## محدودیت ها {#limitations} + +`PREWHERE` تنها با جداول از پشتیبانی `*MergeTree` خانواده diff --git a/docs/fa/sql-reference/statements/select/sample.md b/docs/fa/sql-reference/statements/select/sample.md new file mode 100644 index 00000000000..4f04d69b293 --- /dev/null +++ b/docs/fa/sql-reference/statements/select/sample.md @@ -0,0 +1,113 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +--- + +# بند نمونه {#select-sample-clause} + +این `SAMPLE` بند اجازه می دهد تا برای تقریب `SELECT` پردازش پرس و جو. + +هنگامی که نمونه گیری داده ها فعال است, پرس و جو بر روی تمام داده ها انجام نمی, اما تنها در بخش خاصی از داده ها (نمونه). مثلا, اگر شما نیاز به محاسبه ارقام برای تمام بازدیدکننده داشته است, کافی است برای اجرای پرس و جو در 1/10 کسری از تمام بازدیدکننده داشته است و سپس ضرب در نتیجه 10. + +پردازش پرس و جو تقریبی می تواند در موارد زیر مفید باشد: + +- هنگامی که شما شرایط زمان بندی دقیق (مانند \<100 مگابایت) اما شما نمی توانید هزینه منابع سخت افزاری اضافی را برای دیدار با خود توجیه کنید. +- هنگامی که داده های خام خود را دقیق نیست, بنابراین تقریب می کند به طرز محسوسی کاهش کیفیت. +- کسب و کار مورد نیاز هدف قرار دادن نتایج تقریبی (برای مقرون به صرفه بودن, و یا به بازار نتایج دقیق به کاربران حق بیمه). + +!!! note "یادداشت" + شما فقط می توانید نمونه برداری با استفاده از جداول در [ادغام](../../../engines/table-engines/mergetree-family/mergetree.md) خانواده, و تنها در صورتی که بیان نمونه برداری در ایجاد جدول مشخص شد (دیدن [موتور ادغام](../../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table)). + +ویژگی های نمونه گیری داده ها به شرح زیر است: + +- نمونهگیری دادهها یک مکانیسم قطعی است. نتیجه همان `SELECT .. SAMPLE` پرس و جو همیشه یکسان است. +- نمونه گیری به طور مداوم برای جداول مختلف کار می کند. برای جداول با یک کلید نمونه برداری تک, یک نمونه با ضریب همان همیشه زیر مجموعه همان داده های ممکن را انتخاب. برای مثال یک نمونه از شناسه های کاربر طول می کشد ردیف با همان زیر مجموعه از همه ممکن است شناسه کاربر از جداول مختلف. این به این معنی است که شما می توانید نمونه در کارخانه های فرعی در استفاده از [IN](../../operators/in.md) بند بند. همچنین شما می توانید نمونه ها را با استفاده از [JOIN](join.md) بند بند. +- نمونه گیری اجازه می دهد تا خواندن اطلاعات کمتر از یک دیسک. توجه داشته باشید که شما باید کلید نمونه برداری به درستی مشخص کنید. برای کسب اطلاعات بیشتر, دیدن [ایجاد یک جدول ادغام](../../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table). + +برای `SAMPLE` بند نحو زیر پشتیبانی می شود: + +| SAMPLE Clause Syntax | توصیف | +|----------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `SAMPLE k` | اینجا `k` است تعداد از 0 به 1.
پرس و جو در اجرا `k` کسری از داده ها. به عنوان مثال, `SAMPLE 0.1` پرس و جو را در 10 درصد از داده ها اجرا می کند. [ادامه مطلب](#select-sample-k) | +| `SAMPLE n` | اینجا `n` عدد صحیح به اندازه کافی بزرگ است.
پرس و جو بر روی یک نمونه از حداقل اعدام `n` ردیف (اما نه به طور قابل توجهی بیشتر از این). به عنوان مثال, `SAMPLE 10000000` پرس و جو را در حداقل ردیف های 10000000 اجرا می کند. [ادامه مطلب](#select-sample-n) | +| `SAMPLE k OFFSET m` | اینجا `k` و `m` اعداد از 0 به 1.
پرس و جو بر روی یک نمونه از اعدام `k` کسری از داده ها. داده های مورد استفاده برای نمونه توسط جبران `m` کسر کردن. [ادامه مطلب](#select-sample-offset) | + +## SAMPLE K {#select-sample-k} + +اینجا `k` است تعداد از 0 به 1 (هر دو نمادهای کسری و اعشاری پشتیبانی می شوند). به عنوان مثال, `SAMPLE 1/2` یا `SAMPLE 0.5`. + +در یک `SAMPLE k` بند, نمونه از گرفته `k` کسری از داده ها. مثال زیر نشان داده شده است: + +``` sql +SELECT + Title, + count() * 10 AS PageViews +FROM hits_distributed +SAMPLE 0.1 +WHERE + CounterID = 34 +GROUP BY Title +ORDER BY PageViews DESC LIMIT 1000 +``` + +در این مثال پرس و جو اجرا شده است در یک نمونه از 0.1 (10%) از داده ها. ارزش توابع دانه ها به طور خودکار اصلاح نمی, بنابراین برای دریافت یک نتیجه تقریبی, ارزش `count()` به صورت دستی توسط ضرب 10. + +## SAMPLE N {#select-sample-n} + +اینجا `n` عدد صحیح به اندازه کافی بزرگ است. به عنوان مثال, `SAMPLE 10000000`. + +در این مورد, پرس و جو بر روی یک نمونه از حداقل اعدام `n` ردیف (اما نه به طور قابل توجهی بیشتر از این). به عنوان مثال, `SAMPLE 10000000` پرس و جو را در حداقل ردیف های 10000000 اجرا می کند. + +از حداقل واحد برای خواندن داده ها یک گرانول است (اندازه خود را توسط مجموعه `index_granularity` تنظیم), این را حس می کند به مجموعه ای از یک نمونه است که بسیار بزرگتر از اندازه گرانول. + +هنگام استفاده از `SAMPLE n` بند, شما نمی دانید که درصد نسبی داده پردازش شد. بنابراین شما نمی دانید ضریب توابع کل باید توسط ضرب. استفاده از `_sample_factor` ستون مجازی برای دریافت نتیجه تقریبی. + +این `_sample_factor` ستون شامل ضرایب نسبی است که به صورت پویا محاسبه می شود. این ستون به طور خودکار ایجاد زمانی که شما [ایجاد](../../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table) یک جدول با کلید نمونه گیری مشخص. نمونه های استفاده از `_sample_factor` ستون در زیر نشان داده شده. + +بیایید جدول را در نظر بگیریم `visits`, که شامل ارقام در مورد بازدیدکننده داشته است سایت. مثال اول نشان می دهد که چگونه برای محاسبه تعداد بازدید از صفحه: + +``` sql +SELECT sum(PageViews * _sample_factor) +FROM visits +SAMPLE 10000000 +``` + +مثال بعدی نشان می دهد که چگونه برای محاسبه تعداد کل بازدیدکننده داشته است: + +``` sql +SELECT sum(_sample_factor) +FROM visits +SAMPLE 10000000 +``` + +مثال زیر نشان می دهد که چگونه برای محاسبه مدت زمان جلسه به طور متوسط. توجه داشته باشید که شما لازم نیست به استفاده از ضریب نسبی برای محاسبه مقادیر متوسط. + +``` sql +SELECT avg(Duration) +FROM visits +SAMPLE 10000000 +``` + +## SAMPLE K OFFSET M {#select-sample-offset} + +اینجا `k` و `m` اعداد از 0 به 1. نمونه های زیر نشان داده شده. + +**مثال 1** + +``` sql +SAMPLE 1/10 +``` + +در این مثال نمونه 1 / 10 از تمام داده ها است: + +`[++------------]` + +**مثال 2** + +``` sql +SAMPLE 1/10 OFFSET 1/2 +``` + +در اینجا یک نمونه از 10 درصد گرفته شده از نیمه دوم از داده ها. + +`[------++------]` diff --git a/docs/fa/sql-reference/statements/select/union-all.md b/docs/fa/sql-reference/statements/select/union-all.md new file mode 100644 index 00000000000..3c4fe5c1546 --- /dev/null +++ b/docs/fa/sql-reference/statements/select/union-all.md @@ -0,0 +1,35 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +--- + +# اتحادیه همه بند {#union-all-clause} + +شما می توانید استفاده کنید `UNION ALL` برای ترکیب هر تعداد از `SELECT` نمایش داده شد با گسترش نتایج خود را. مثال: + +``` sql +SELECT CounterID, 1 AS table, toInt64(count()) AS c + FROM test.hits + GROUP BY CounterID + +UNION ALL + +SELECT CounterID, 2 AS table, sum(Sign) AS c + FROM test.visits + GROUP BY CounterID + HAVING c > 0 +``` + +ستون نتیجه با شاخص خود را همسان (سفارش در داخل `SELECT`). اگر نام ستون مطابقت ندارند, نام برای نتیجه نهایی از پرس و جو برای اولین بار گرفته شده. + +نوع ریخته گری برای اتحادیه انجام می شود. برای مثال اگر دو نمایش داده شد در حال ترکیب باید همین زمینه را با غیر-`Nullable` و `Nullable` انواع از یک نوع سازگار, در نتیجه `UNION ALL` دارای یک `Nullable` نوع درست. + +نمایش داده شد که بخش هایی از `UNION ALL` نمی توان در براکت های گرد محصور کرد. [ORDER BY](order-by.md) و [LIMIT](limit.md) برای نمایش داده شد جداگانه به نتیجه نهایی اعمال می شود. اگر شما نیاز به اعمال تبدیل به نتیجه نهایی شما می توانید تمام نمایش داده شد با قرار دادن `UNION ALL` در یک خرده فروشی در [FROM](from.md) بند بند. + +## محدودیت ها {#limitations} + +فقط `UNION ALL` پشتیبانی می شود. منظم `UNION` (`UNION DISTINCT`) پشتیبانی نمی شود . اگر شما نیاز دارید `UNION DISTINCT`, شما می توانید ارسال `SELECT DISTINCT` از زیرخاکری حاوی `UNION ALL`. + +## پیاده سازی اطلاعات {#implementation-details} + +نمایش داده شد که بخش هایی از `UNION ALL` می توان به طور همزمان اجرا, و نتایج خود را می توان با هم مخلوط. diff --git a/docs/fa/sql-reference/statements/select/where.md b/docs/fa/sql-reference/statements/select/where.md new file mode 100644 index 00000000000..c9a55304421 --- /dev/null +++ b/docs/fa/sql-reference/statements/select/where.md @@ -0,0 +1,15 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +--- + +# بند کجا {#select-where} + +`WHERE` بند اجازه می دهد تا برای فیلتر کردن داده ها است که از [FROM](from.md) بند `SELECT`. + +اگر وجود دارد `WHERE` بند, باید حاوی یک عبارت با `UInt8` نوع. این است که معمولا بیان با مقایسه و اپراتورهای منطقی. ردیف هایی که این عبارت به 0 ارزیابی می شود از تحولات یا نتیجه بیشتر توضیح داده می شود. + +`WHERE` بیان بر توانایی استفاده از شاخص ها و هرس پارتیشن ارزیابی, اگر موتور جدول زمینه ای پشتیبانی می کند که. + +!!! note "یادداشت" + یک بهینه سازی فیلتر به نام وجود دارد [کجا](prewhere.md). diff --git a/docs/fa/sql-reference/statements/select/with.md b/docs/fa/sql-reference/statements/select/with.md new file mode 100644 index 00000000000..4c14386b881 --- /dev/null +++ b/docs/fa/sql-reference/statements/select/with.md @@ -0,0 +1,80 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +--- + +# با بند {#with-clause} + +در این بخش پشتیبانی از عبارات جدول مشترک فراهم می کند ([CTE](https://en.wikipedia.org/wiki/Hierarchical_and_recursive_queries_in_SQL)), بنابراین نتایج حاصل از `WITH` بند را می توان در داخل استفاده کرد `SELECT` بند بند. + +## محدودیت ها {#limitations} + +1. نمایش داده شد بازگشتی پشتیبانی نمی شوند. +2. هنگامی که زیرخاکری در داخل با بخش استفاده می شود, این نتیجه باید اسکالر با دقیقا یک ردیف باشد. +3. نتایج بیان در زیرمجموعه ها در دسترس نیست. + +## مثالها {#examples} + +**مثال 1:** استفاده از عبارت ثابت به عنوان “variable” + +``` sql +WITH '2019-08-01 15:23:00' as ts_upper_bound +SELECT * +FROM hits +WHERE + EventDate = toDate(ts_upper_bound) AND + EventTime <= ts_upper_bound +``` + +**مثال 2:** جمع مبلغ (بایت) بیان نتیجه از بند لیست ستون را انتخاب کنید + +``` sql +WITH sum(bytes) as s +SELECT + formatReadableSize(s), + table +FROM system.parts +GROUP BY table +ORDER BY s +``` + +**مثال 3:** استفاده از نتایج حاصل از زیرکواری اسکالر + +``` sql +/* this example would return TOP 10 of most huge tables */ +WITH + ( + SELECT sum(bytes) + FROM system.parts + WHERE active + ) AS total_disk_usage +SELECT + (sum(bytes) / total_disk_usage) * 100 AS table_disk_usage, + table +FROM system.parts +GROUP BY table +ORDER BY table_disk_usage DESC +LIMIT 10 +``` + +**مثال 4:** استفاده مجدد از عبارت در زیرخاکری + +به عنوان یک راه حل برای محدودیت فعلی برای استفاده بیان در زیر مجموعه, شما ممکن است تکراری. + +``` sql +WITH ['hello'] AS hello +SELECT + hello, + * +FROM +( + WITH ['hello'] AS hello + SELECT hello +) +``` + +``` text +┌─hello─────┬─hello─────┐ +│ ['hello'] │ ['hello'] │ +└───────────┴───────────┘ +``` diff --git a/docs/fa/sql-reference/statements/show.md b/docs/fa/sql-reference/statements/show.md index fd8a74ca50a..57e8b90361d 100644 --- a/docs/fa/sql-reference/statements/show.md +++ b/docs/fa/sql-reference/statements/show.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 38 toc_title: SHOW --- @@ -102,4 +102,68 @@ SHOW DICTIONARIES FROM db LIKE '%reg%' LIMIT 2 └──────────────┘ ``` +## SHOW GRANTS {#show-grants-statement} + +نشان می دهد امتیازات برای یک کاربر. + +### نحو {#show-grants-syntax} + +``` sql +SHOW GRANTS [FOR user] +``` + +اگر کاربر مشخص نشده است, پرس و جو امتیازات برای کاربر فعلی را برمی گرداند. + +## SHOW CREATE USER {#show-create-user-statement} + +پارامترهای که در یک مورد استفاده قرار گرفت را نشان می دهد [ایجاد کاربر](create.md#create-user-statement). + +`SHOW CREATE USER` رمزهای عبور کاربر خروجی نیست. + +### نحو {#show-create-user-syntax} + +``` sql +SHOW CREATE USER [name | CURRENT_USER] +``` + +## SHOW CREATE ROLE {#show-create-role-statement} + +پارامترهای که در یک مورد استفاده قرار گرفت را نشان می دهد [ایجاد نقش](create.md#create-role-statement) + +### نحو {#show-create-role-syntax} + +``` sql +SHOW CREATE ROLE name +``` + +## SHOW CREATE ROW POLICY {#show-create-row-policy-statement} + +پارامترهای که در یک مورد استفاده قرار گرفت را نشان می دهد [ایجاد خط مشی سطر](create.md#create-row-policy-statement) + +### نحو {#show-create-row-policy-syntax} + +``` sql +SHOW CREATE [ROW] POLICY name ON [database.]table +``` + +## SHOW CREATE QUOTA {#show-create-quota-statement} + +پارامترهای که در یک مورد استفاده قرار گرفت را نشان می دهد [ایجاد سهمیه](create.md#create-quota-statement) + +### نحو {#show-create-row-policy-syntax} + +``` sql +SHOW CREATE QUOTA [name | CURRENT] +``` + +## SHOW CREATE SETTINGS PROFILE {#show-create-settings-profile-statement} + +پارامترهای که در یک مورد استفاده قرار گرفت را نشان می دهد [تنظیمات ایجاد پروفایل](create.md#create-settings-profile-statement) + +### نحو {#show-create-row-policy-syntax} + +``` sql +SHOW CREATE [SETTINGS] PROFILE name +``` + [مقاله اصلی](https://clickhouse.tech/docs/en/query_language/show/) diff --git a/docs/fa/sql-reference/statements/system.md b/docs/fa/sql-reference/statements/system.md index 619a0b5c87c..386bf982235 100644 --- a/docs/fa/sql-reference/statements/system.md +++ b/docs/fa/sql-reference/statements/system.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 37 toc_title: SYSTEM --- @@ -39,7 +39,7 @@ SELECT name, status FROM system.dictionaries; ## DROP DNS CACHE {#query_language-system-drop-dns-cache} -کش دی ان اس داخلی بازنشانی را کلیک کنید. گاهی اوقات (برای clickhouse نسخه) لازم است برای استفاده از این دستور هنگامی که در حال تغییر زیرساخت ها (تغییر آدرس ip دیگر clickhouse سرور یا سرور استفاده شده توسط لغت نامه). +کش دی ان اس داخلی بازنشانی را کلیک کنید. گاهی اوقات (برای ClickHouse نسخه) لازم است برای استفاده از این دستور هنگامی که در حال تغییر زیرساخت ها (تغییر آدرس IP دیگر ClickHouse سرور یا سرور استفاده شده توسط لغت نامه). برای راحت تر (اتوماتیک) مدیریت کش دیدن disable\_internal\_dns\_cache, dns\_cache\_update\_period پارامترهای. diff --git a/docs/fa/sql-reference/syntax.md b/docs/fa/sql-reference/syntax.md index 2efdc20bc0b..795f8302961 100644 --- a/docs/fa/sql-reference/syntax.md +++ b/docs/fa/sql-reference/syntax.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 31 toc_title: "\u0646\u062D\u0648" --- @@ -18,11 +18,11 @@ INSERT INTO t VALUES (1, 'Hello, world'), (2, 'abc'), (3, 'def') این `INSERT INTO t VALUES` قطعه توسط تجزیه کننده کامل و داده ها تجزیه می شود `(1, 'Hello, world'), (2, 'abc'), (3, 'def')` توسط تجزیه کننده جریان سریع تجزیه می شود. شما همچنین می توانید تجزیه کننده کامل برای داده ها با استفاده از [در حال خواندن:](../operations/settings/settings.md#settings-input_format_values_interpret_expressions) تنظیمات. چه زمانی `input_format_values_interpret_expressions = 1` کلیک هاوس اول سعی می کند به تجزیه ارزش با تجزیه کننده جریان سریع. در صورت عدم موفقیت کلیک هاوس تلاش می کند تا از تجزیه کننده کامل برای داده ها استفاده کند و مانند یک مربع درمان شود [عبارت](#syntax-expressions). داده ها می توانند هر فرمت داشته باشند. هنگامی که یک پرس و جو را دریافت کرده است, سرور محاسبه بیش از [بیشینه\_کرکی\_سیز](../operations/settings/settings.md#settings-max_query_size) بایت از درخواست در رم (به طور پیش فرض, 1 مگابایت), و بقیه جریان تجزیه. -این به این معنی است که سیستم با مشکلات بزرگ ندارد `INSERT` نمایش داده شد, مانند خروجی زیر می کند. +این اجازه می دهد تا برای اجتناب از مشکلات بزرگ `INSERT` نمایش داده شد. هنگام استفاده از `Values` قالب در یک `INSERT` پرس و جو, ممکن است به نظر می رسد که داده ها همان عبارات در تجزیه `SELECT` پرس و جو, اما این درست نیست. این `Values` فرمت بسیار محدود تر است. -بعد ما تجزیه کننده کامل را پوشش خواهیم داد. برای کسب اطلاعات بیشتر در مورد تجزیه کننده فرمت, دیدن [فرشها](../interfaces/formats.md) بخش. +بقیه این مقاله تجزیه کننده کامل را پوشش می دهد. برای کسب اطلاعات بیشتر در مورد تجزیه کننده فرمت, دیدن [فرشها](../interfaces/formats.md) بخش. ## فاصلهها {#spaces} @@ -30,9 +30,9 @@ INSERT INTO t VALUES (1, 'Hello, world'), (2, 'abc'), (3, 'def') ## توضیحات {#comments} -گذاشتن به سبک و ج سبک نظرات پشتیبانی می شوند. -گذاشتن به سبک نظرات: از `--` به انتهای خط فضای پس از `--` می توان حذف. -نظرات در ج سبک: از `/*` به `*/`. این نظرات می تواند چند خطی باشد. فضاهای اینجا مورد نیاز نیست, هم. +تاتر پشتیبانی از هر دو گذاشتن به سبک و ج سبک نظرات. +گذاشتن به سبک نظرات با شروع `--` و در ادامه به پایان خط, یک فضای پس از `--` می توان حذف. +ج سبک از `/*` به `*/`و می تواند چند خطی, فضاهای مورد نیاز نیست یا. ## کلیدواژهها {#syntax-keywords} @@ -43,20 +43,20 @@ INSERT INTO t VALUES (1, 'Hello, world'), (2, 'abc'), (3, 'def') این که نام نوع داده حساس به حروف باشد را میتوان در `system.data_type_families` جدول -در مقایسه با استاندارد گذاشتن تمام کلمات کلیدی دیگر (از جمله نام توابع) عبارتند از **حساس به حالت**. +در مقایسه با استاندارد گذاشتن تمام کلمات کلیدی دیگر (از جمله توابع نام) هستند **حساس به حالت**. -کلمات کلیدی محفوظ نیست (فقط به عنوان کلمات کلیدی در زمینه مربوطه تجزیه می شوند). در صورت استفاده [شناسهها](#syntax-identifiers) همان کلمات کلیدی را به نقل قول محصور. برای مثال پرس و جو `SELECT "FROM" FROM table_name` معتبر است اگر جدول `table_name` دارای ستون با نام `"FROM"`. +کلمات کلیدی محفوظ نیست; به عنوان مثل تنها در زمینه مربوطه درمان می شوند. در صورت استفاده [شناسهها](#syntax-identifiers) با همین نام به عنوان کلمات کلیدی, محصور را به دو نقل قول و یا پشت. برای مثال پرس و جو `SELECT "FROM" FROM table_name` معتبر است اگر جدول `table_name` دارای ستون با نام `"FROM"`. ## شناسهها {#syntax-identifiers} شناسه ها عبارتند از: -- خوشه, پایگاه داده, جدول, پارتیشن و ستون نام. +- خوشه, پایگاه داده, جدول, پارتیشن, و نام ستون. - توابع. - انواع داده ها. - [نامگردانهای بیان](#syntax-expression_aliases). -شناسه را می توان به نقل و یا غیر نقل. توصیه می شود از شناسه های غیر نقل قول استفاده کنید. +شناسه را می توان به نقل و یا غیر نقل. در حالت دوم ترجیح داده است. شناسه های غیر نقل قول باید عبارت منظم مطابقت `^[a-zA-Z_][0-9a-zA-Z_]*$` و نمی تواند برابر باشد [کلیدواژهها](#syntax-keywords). مثالها: `x, _1, X_y__Z123_.` @@ -64,34 +64,34 @@ INSERT INTO t VALUES (1, 'Hello, world'), (2, 'abc'), (3, 'def') ## Literals {#literals} -وجود دارد: عددی, رشته, ترکیب و `NULL` literals. +عددی وجود دارد, رشته, ترکیب, و `NULL` literals. ### عددی {#numeric} تحت اللفظی عددی تلاش می کند به تجزیه شود: -- برای اولین بار به عنوان یک شماره امضا 64 بیتی, با استفاده از [استرتول](https://en.cppreference.com/w/cpp/string/byte/strtoul) تابع. +- اولین, به عنوان یک شماره امضا 64 بیتی, با استفاده از [استرتول](https://en.cppreference.com/w/cpp/string/byte/strtoul) تابع. - اگر ناموفق, به عنوان یک عدد بدون علامت 64 بیتی, با استفاده از [استرول](https://en.cppreference.com/w/cpp/string/byte/strtol) تابع. - اگر ناموفق, به عنوان یک عدد شناور نقطه با استفاده از [رشته](https://en.cppreference.com/w/cpp/string/byte/strtof) تابع. -- در غیر این صورت, یک خطا بازگشته است. +- در غیر این صورت, این خطا را برمی گرداند. -مقدار مربوطه کوچکترین نوع است که متناسب با ارزش داشته باشد. +ارزش تحت اللفظی کوچکترین نوع است که متناسب با ارزش است. مثلا, 1 به عنوان تجزیه `UInt8` اما 256 به عنوان تجزیه شده است `UInt16`. برای کسب اطلاعات بیشتر, دیدن [انواع داده ها](../sql-reference/data-types/index.md). مثالها: `1`, `18446744073709551615`, `0xDEADBEEF`, `01`, `0.1`, `1e100`, `-1e-100`, `inf`, `nan`. ### رشته {#syntax-string-literal} -فقط رشته های رشته ای در نقل قول های تک پشتیبانی می شوند. شخصیت های محصور می تواند بک اسلش فرار. توالی فرار زیر یک مقدار خاص مربوطه: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\a`, `\v`, `\xHH`. در تمام موارد دیگر, فرار توالی در قالب `\c` کجا `c` است هر شخصیت, به تبدیل `c`. این به این معنی است که شما می توانید توالی استفاده کنید `\'`و`\\`. ارزش خواهد شد که [رشته](../sql-reference/data-types/string.md) نوع. +فقط رشته های رشته ای در نقل قول های تک پشتیبانی می شوند. شخصیت های محصور می تواند بک اسلش فرار. توالی فرار زیر یک مقدار خاص مربوطه: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\a`, `\v`, `\xHH`. در تمام موارد دیگر, فرار توالی در قالب `\c` کجا `c` است هر شخصیت, به تبدیل `c`. این بدان معنی است که شما می توانید توالی استفاده کنید `\'`و`\\`. ارزش خواهد شد که [رشته](../sql-reference/data-types/string.md) نوع. -حداقل مجموعه ای از شخصیت های که شما نیاز به فرار در لیتر رشته: `'` و `\`. نقل قول تنها را می توان با نقل قول تنها فرار, لیتر `'It\'s'` و `'It''s'` برابر هستند. +در رشته لیتر, شما نیاز به فرار حداقل `'` و `\`. نقل قول تنها را می توان با نقل قول تنها فرار, لیتر `'It\'s'` و `'It''s'` برابر هستند. ### ترکیب {#compound} -سازه ها برای ارریس پشتیبانی می شوند: `[1, 2, 3]` و تاپل: `(1, 'Hello, world!', 2)`.. -در واقع این نیست literals اما عبارات با آرایه ایجاد اپراتور و چند تایی ایجاد اپراتور بود. +ارریس با براکت مربع ساخته شده است `[1, 2, 3]`. نوپل ها با براکت های گرد ساخته می شوند `(1, 'Hello, world!', 2)`. +مشخصات فنی این نیست literals اما عبارات با آرایه ایجاد اپراتور و چند تایی ایجاد اپراتور بود. مجموعه ای باید شامل حداقل یک مورد باشد و یک تاپل باید حداقل دو مورد داشته باشد. -تاپل یک هدف خاص برای استفاده در `IN` بند یک `SELECT` پرس و جو. توپلس می تواند به عنوان نتیجه یک پرس و جو به دست, اما نمی توان به یک پایگاه داده ذخیره (به غیر از [حافظه](../engines/table-engines/special/memory.md) جدول). +یک مورد جداگانه وجود دارد که تاپل در ظاهر `IN` بند یک `SELECT` پرس و جو. نتایج پرس و جو می تواند شامل تاپل, اما تاپل را نمی توان به یک پایگاه داده ذخیره (به جز از جداول با [حافظه](../engines/table-engines/special/memory.md) موتور). ### NULL {#null-literal} @@ -101,13 +101,13 @@ INSERT INTO t VALUES (1, 'Hello, world'), (2, 'abc'), (3, 'def') بسته به فرمت داده (ورودی یا خروجی), `NULL` ممکن است نمایندگی های مختلف. برای کسب اطلاعات بیشتر, اسناد و مدارک برای دیدن [قالبهای داده](../interfaces/formats.md#formats). -بسیاری از تفاوت های ظریف برای پردازش وجود دارد `NULL`. مثلا, اگر حداقل یکی از استدلال از یک عملیات مقایسه است `NULL` نتیجه این عملیات نیز خواهد بود `NULL`. همان درست است برای ضرب است, بعلاوه, و عملیات دیگر. برای کسب اطلاعات بیشتر, خواندن اسناد و مدارک برای هر عملیات. +بسیاری از تفاوت های ظریف برای پردازش وجود دارد `NULL`. مثلا, اگر حداقل یکی از استدلال از یک عملیات مقایسه است `NULL` نتیجه این عملیات نیز است `NULL`. همان درست است برای ضرب است, بعلاوه, و عملیات دیگر. برای کسب اطلاعات بیشتر, خواندن اسناد و مدارک برای هر عملیات. -در نمایش داده شد, شما می توانید بررسی کنید `NULL` با استفاده از [IS NULL](operators.md#operator-is-null) و [IS NOT NULL](operators.md) اپراتورها و توابع مرتبط `isNull` و `isNotNull`. +در نمایش داده شد, شما می توانید بررسی کنید `NULL` با استفاده از [IS NULL](operators/index.md#operator-is-null) و [IS NOT NULL](operators/index.md) اپراتورها و توابع مرتبط `isNull` و `isNotNull`. ## توابع {#functions} -توابع مانند یک شناسه با یک لیست از استدلال نوشته شده (احتمالا خالی) در داخل پرانتز. در مقابل به گذاشتن استاندارد, براکت مورد نیاز است, حتی برای یک لیست استدلال خالی. مثال: `now()`. +فراخوانی تابع مانند یک شناسه با یک لیست از استدلال نوشته شده (احتمالا خالی) در براکت دور. در مقابل به گذاشتن استاندارد, براکت مورد نیاز است, حتی برای یک لیست استدلال خالی. مثال: `now()`. توابع منظم و جمع وجود دارد (بخش را ببینید “Aggregate functions”). برخی از توابع دانه می تواند شامل دو لیست از استدلال در براکت. مثال: `quantile (0.9) (x)`. این توابع مجموع نامیده می شوند “parametric” توابع, و استدلال در لیست اول نامیده می شوند “parameters”. نحو توابع کل بدون پارامتر همان است که برای توابع به طور منظم است. ## اپراتورها {#operators} @@ -143,7 +143,7 @@ expr AS alias ### نکاتی در مورد استفاده {#notes-on-usage} -نام مستعار جهانی برای یک پرس و جو و یا زیرخاکی هستند و شما می توانید یک نام مستعار در هر بخشی از یک پرس و جو برای هر بیان تعریف. به عنوان مثال, `SELECT (1 AS n) + 2, n`. +نام مستعار جهانی برای یک پرس و جو و یا زیرخاکی هستند, و شما می توانید یک نام مستعار در هر بخشی از یک پرس و جو برای هر بیان تعریف. به عنوان مثال, `SELECT (1 AS n) + 2, n`. نام مستعار قابل مشاهده نیست در subqueries و بین subqueries. مثلا, در حالی که اجرای پرس و جو `SELECT (SELECT sum(b.a) + num FROM b) - a.a AS num FROM a` تاتر استثنا را تولید می کند `Unknown identifier: num`. @@ -184,4 +184,4 @@ Code: 184. DB::Exception: Received from localhost:9000, 127.0.0.1. DB::Exception لیستی از عبارات یک یا چند عبارت از هم جدا شده توسط کاما است. توابع و اپراتورهای, به نوبه خود, می توانید عبارات به عنوان استدلال دارند. -[مقاله اصلی](https://clickhouse.tech/docs/en/query_language/syntax/) +[مقاله اصلی](https://clickhouse.tech/docs/en/sql_reference/syntax/) diff --git a/docs/fa/sql-reference/table-functions/file.md b/docs/fa/sql-reference/table-functions/file.md index 65883b3e547..7bb676670a4 100644 --- a/docs/fa/sql-reference/table-functions/file.md +++ b/docs/fa/sql-reference/table-functions/file.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 37 toc_title: "\u067E\u0631\u0648\u0646\u062F\u0647" --- @@ -116,6 +116,6 @@ FROM file('big_dir/file{0..9}{0..9}{0..9}', 'CSV', 'name String, value UInt32') **همچنین نگاه کنید به** -- [مجازی ستون](https://clickhouse.tech/docs/en/operations/table_engines/#table_engines-virtual_columns) +- [ستونهای مجازی](https://clickhouse.tech/docs/en/operations/table_engines/#table_engines-virtual_columns) [مقاله اصلی](https://clickhouse.tech/docs/en/query_language/table_functions/file/) diff --git a/docs/fa/sql-reference/table-functions/generate.md b/docs/fa/sql-reference/table-functions/generate.md index 0236d99cf57..535c98f068b 100644 --- a/docs/fa/sql-reference/table-functions/generate.md +++ b/docs/fa/sql-reference/table-functions/generate.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 47 toc_title: "\u0698\u0646\u0631\u0627\u0644" --- @@ -19,7 +19,6 @@ generateRandom('name TypeName[, name TypeName]...', [, 'random_seed'[, 'max_stri - `name` — Name of corresponding column. - `TypeName` — Type of corresponding column. -- `limit` — Number of rows to generate. - `max_array_length` — Maximum array length for all generated arrays. Defaults to `10`. - `max_string_length` — Maximum string length for all generated strings. Defaults to `10`. - `random_seed` — Specify random seed manually to produce stable results. If NULL — seed is randomly generated. @@ -31,7 +30,7 @@ generateRandom('name TypeName[, name TypeName]...', [, 'random_seed'[, 'max_stri ## مثال طریقه استفاده {#usage-example} ``` sql -SELECT * FROM generateRandom('a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3), UUID)', 1, 10, 2); +SELECT * FROM generateRandom('a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3), UUID)', 1, 10, 2) LIMIT 3; ``` ``` text diff --git a/docs/fa/sql-reference/table-functions/hdfs.md b/docs/fa/sql-reference/table-functions/hdfs.md index 0f90031dd80..ae42eda5165 100644 --- a/docs/fa/sql-reference/table-functions/hdfs.md +++ b/docs/fa/sql-reference/table-functions/hdfs.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 45 toc_title: hdfs --- # hdfs {#hdfs} -ایجاد یک جدول از فایل ها در اچ دی. این جدول عملکرد شبیه به [نشانی وب](url.md) و [پرونده](file.md) یکی +ایجاد یک جدول از فایل ها در اچ دی. این تابع جدول شبیه به [نشانی وب](url.md) و [پرونده](file.md) یکی ``` sql hdfs(URI, format, structure) diff --git a/docs/fa/sql-reference/table-functions/index.md b/docs/fa/sql-reference/table-functions/index.md index a639bab05bd..0ed98d139b2 100644 --- a/docs/fa/sql-reference/table-functions/index.md +++ b/docs/fa/sql-reference/table-functions/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_folder_title: Table Functions +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "\u062A\u0648\u0627\u0628\u0639 \u062C\u062F\u0648\u0644" toc_priority: 34 toc_title: "\u0645\u0639\u0631\u0641\u06CC \u0634\u0631\u06A9\u062A" --- @@ -12,7 +12,7 @@ toc_title: "\u0645\u0639\u0631\u0641\u06CC \u0634\u0631\u06A9\u062A" شما می توانید توابع جدول در استفاده: -- [FROM](../statements/select.md#select-from) بند از `SELECT` پرس و جو. +- [FROM](../statements/select/from.md) بند از `SELECT` پرس و جو. The method for creating a temporary table that is available only in the current query. The table is deleted when the query finishes. diff --git a/docs/fa/sql-reference/table-functions/input.md b/docs/fa/sql-reference/table-functions/input.md index 0ab23171f73..0c63761e78f 100644 --- a/docs/fa/sql-reference/table-functions/input.md +++ b/docs/fa/sql-reference/table-functions/input.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 46 toc_title: "\u0648\u0631\u0648\u062F\u06CC" --- diff --git a/docs/fa/sql-reference/table-functions/jdbc.md b/docs/fa/sql-reference/table-functions/jdbc.md index f596c728fdd..2625c65e1da 100644 --- a/docs/fa/sql-reference/table-functions/jdbc.md +++ b/docs/fa/sql-reference/table-functions/jdbc.md @@ -1,8 +1,8 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 43 -toc_title: jdbc +toc_title: "jdbc" --- # جستجو {#table-function-jdbc} diff --git a/docs/fa/sql-reference/table-functions/merge.md b/docs/fa/sql-reference/table-functions/merge.md index 5e843f4e460..9893fb71838 100644 --- a/docs/fa/sql-reference/table-functions/merge.md +++ b/docs/fa/sql-reference/table-functions/merge.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 38 toc_title: "\u0627\u062F\u063A\u0627\u0645" --- diff --git a/docs/fa/sql-reference/table-functions/mysql.md b/docs/fa/sql-reference/table-functions/mysql.md index 7455ec398df..3b37a4ad88f 100644 --- a/docs/fa/sql-reference/table-functions/mysql.md +++ b/docs/fa/sql-reference/table-functions/mysql.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 42 toc_title: "\u062E\u0631\u0648\u062C\u06CC \u0632\u06CC\u0631" --- diff --git a/docs/fa/sql-reference/table-functions/numbers.md b/docs/fa/sql-reference/table-functions/numbers.md index 86a4829ef72..a09d2d9fcfc 100644 --- a/docs/fa/sql-reference/table-functions/numbers.md +++ b/docs/fa/sql-reference/table-functions/numbers.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 39 toc_title: "\u0627\u0639\u062F\u0627\u062F" --- diff --git a/docs/fa/sql-reference/table-functions/odbc.md b/docs/fa/sql-reference/table-functions/odbc.md index 7efbeea28eb..e5a8ad2af40 100644 --- a/docs/fa/sql-reference/table-functions/odbc.md +++ b/docs/fa/sql-reference/table-functions/odbc.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 44 toc_title: "\u062C\u0633\u062A\u062C\u0648" --- @@ -29,7 +29,7 @@ odbc(connection_settings, external_database, external_table) این مثال برای لینوکس اوبونتو 18.04 و سرور خروجی زیر 5.7 بررسی می شود. -اطمینان حاصل شود که unixodbc و mysql اتصال نصب شده است. +اطمینان حاصل شود که unixODBC و MySQL اتصال نصب شده است. به طور پیش فرض (در صورت نصب از بسته), کلیک خانه شروع می شود به عنوان کاربر `clickhouse`. بنابراین شما نیاز به ایجاد و پیکربندی این کاربر در سرور خروجی زیر. diff --git a/docs/fa/sql-reference/table-functions/remote.md b/docs/fa/sql-reference/table-functions/remote.md index cf490a936a9..23a6753fd26 100644 --- a/docs/fa/sql-reference/table-functions/remote.md +++ b/docs/fa/sql-reference/table-functions/remote.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 40 toc_title: "\u062F\u0648\u0631" --- -# از راه دور remoteSecure {#remote-remotesecure} +# دور افتاده {#remote-remotesecure} اجازه می دهد تا شما را به دسترسی به سرور از راه دور بدون ایجاد یک `Distributed` جدول @@ -64,7 +64,7 @@ example01-{01..02}-{1|2} این مثال دو تکه که هر کدام دو کپی مشخص. -تعدادی از آدرس های تولید شده محدود است توسط یک ثابت است. در حال حاضر این 1000 نشانی است. +تعداد نشانیهای تولید شده توسط یک ثابت محدود شده است. در حال حاضر این 1000 نشانی است. با استفاده از `remote` تابع جدول کمتر مطلوب تر از ایجاد یک است `Distributed` جدول, چرا که در این مورد, اتصال سرور دوباره تاسیس برای هر درخواست. علاوه بر این, اگر نام میزبان قرار است, نام حل و فصل, و خطا شمارش نیست در هنگام کار با کپی های مختلف. هنگامی که پردازش تعداد زیادی از نمایش داده شد, همیشه ایجاد `Distributed` جدول جلوتر از زمان, و استفاده نکنید `remote` تابع جدول. diff --git a/docs/fa/sql-reference/table-functions/url.md b/docs/fa/sql-reference/table-functions/url.md index 0b282bf6633..bb7f5bdf419 100644 --- a/docs/fa/sql-reference/table-functions/url.md +++ b/docs/fa/sql-reference/table-functions/url.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 41 toc_title: "\u0646\u0634\u0627\u0646\u06CC \u0648\u0628" --- diff --git a/docs/fa/whats-new/changelog/2017.md b/docs/fa/whats-new/changelog/2017.md index 41c103c2c9d..939ed966c22 100644 --- a/docs/fa/whats-new/changelog/2017.md +++ b/docs/fa/whats-new/changelog/2017.md @@ -1,17 +1,17 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 79 toc_title: '2017' --- -### انتشار کلیک 1.1.54327, 2017-12-21 {#clickhouse-release-1-1-54327-2017-12-21} +### ClickHouse انتشار 1.1.54327, 2017-12-21 {#clickhouse-release-1-1-54327-2017-12-21} این نسخه شامل رفع اشکال برای نسخه قبلی 1.1.54318: - اشکال ثابت با شرایط مسابقه ممکن است در تکرار است که می تواند به از دست دادن داده ها منجر شود. این مسئله تاثیر می گذارد نسخه 1.1.54310 و 1.1.54318. اگر شما استفاده از یکی از این نسخه ها با جداول تکرار, به روز رسانی است که به شدت توصیه می شود. این موضوع نشان می دهد در سیاهههای مربوط در پیام های هشدار دهنده مانند `Part ... from own log doesn't exist.` موضوع مربوط است حتی اگر شما این پیام ها در سیاهههای مربوط را نمی بینم. -### انتشار کلیک 1.1.54318, 2017-11-30 {#clickhouse-release-1-1-54318-2017-11-30} +### ClickHouse انتشار 1.1.54318, 2017-11-30 {#clickhouse-release-1-1-54318-2017-11-30} این نسخه شامل رفع اشکال برای نسخه های قبلی 1.1.54310: @@ -21,7 +21,7 @@ toc_title: '2017' - ثابت است که شماره بود که باعث صف تکرار برای جلوگیری از در حال اجرا - چرخش ثابت و بایگانی سیاهههای مربوط به سرور -### انتشار کلیک 1.1.54310, 2017-11-01 {#clickhouse-release-1-1-54310-2017-11-01} +### ClickHouse انتشار 1.1.54310, 2017-11-01 {#clickhouse-release-1-1-54310-2017-11-01} #### ویژگی های جدید: {#new-features} @@ -33,7 +33,7 @@ toc_title: '2017' - طیف وسیعی از ارزش ها برای انواع تاریخ و تاریخ ساعت به سال گسترش 2105. - اضافه شدن `CREATE MATERIALIZED VIEW x TO y` پرس و جو (مشخص یک جدول موجود برای ذخیره سازی داده ها از یک نمایش محقق). - اضافه شدن `ATTACH TABLE` پرس و جو بدون استدلال. -- پردازش منطق به صورت تو در تو ستون با نام پایان در نقشه در یک summingmergetree جدول استخراج شد به summap مجموع عملکرد. شما هم اکنون می توانید این ستون ها به صراحت مشخص کنید. +- پردازش منطق به صورت تو در تو ستون با نام پایان در نقشه در یک SummingMergeTree جدول استخراج شد به sumMap مجموع عملکرد. شما هم اکنون می توانید این ستون ها به صراحت مشخص کنید. - حداکثر اندازه فرهنگ لغت ایپ تری به ورودی های 128 متری افزایش می یابد. - اضافه شدن تابع نوع گیرنده. - اضافه شده تابع جمع مجموع ورود. @@ -61,7 +61,7 @@ toc_title: '2017' - چندین کتابخانه شخص ثالث (به ویژه کم) به روز شد و تبدیل به دستی دستگاه گوارش. -### انتشار کلیک 1.1.54304, 2017-10-19 {#clickhouse-release-1-1-54304-2017-10-19} +### ClickHouse انتشار 1.1.54304, 2017-10-19 {#clickhouse-release-1-1-54304-2017-10-19} #### ویژگی های جدید: {#new-features-1} @@ -76,19 +76,19 @@ toc_title: '2017' - `FREEZE PARTITION` همیشه از نظر عام کار می کند در حال حاضر. - درخواست پست خالی در حال حاضر پاسخ با کد بازگشت 411. - خطاهای تفسیر ثابت برای عبارات مانند `CAST(1 AS Nullable(UInt8)).` -- ثابت خطا در هنگام خواندن `Array(Nullable(String))` ستون از `MergeTree` میز +- ثابت خطا در هنگام خواندن `Array(Nullable(String))` ستونها از `MergeTree` میز - ثابت توفنده زمانی که نمایش داده شد تجزیه مانند `SELECT dummy AS dummy, dummy AS b` - کاربران به درستی با نامعتبر به روز شد `users.xml` - دست زدن درست زمانی که یک فرهنگ لغت اجرایی یک کد پاسخ غیر صفر می گرداند. -### انتشار کلیک 1.1.54292, 2017-09-20 {#clickhouse-release-1-1-54292-2017-09-20} +### ClickHouse انتشار 1.1.54292, 2017-09-20 {#clickhouse-release-1-1-54292-2017-09-20} #### ویژگی های جدید: {#new-features-2} - اضافه شدن `pointInPolygon` تابع برای کار با مختصات در یک هواپیما مختصات. - اضافه شدن `sumMap` تابع جمع برای محاسبه مجموع ارریس, شبیه به `SummingMergeTree`. - اضافه شدن `trunc` تابع. بهبود عملکرد توابع گرد کردن (`round`, `floor`, `ceil`, `roundToExp2`) و اصلاح منطق چگونه کار می کنند. منطق را تغییر داد `roundToExp2` تابع برای کسر و اعداد منفی. -- این clickhouse فایل اجرایی است که در حال حاضر کمتر وابسته به libc نسخه. همان فایل اجرایی کلیک می تواند بر روی طیف گسترده ای از سیستم های لینوکس اجرا شود. هنوز وابستگی وجود دارد که با استفاده از نمایش داده شد وارد (با تنظیم `compile = 1` , است که به طور پیش فرض استفاده نمی شود). +- این ClickHouse فایل اجرایی است که در حال حاضر کمتر وابسته به libc نسخه. همان فایل اجرایی کلیک می تواند بر روی طیف گسترده ای از سیستم های لینوکس اجرا شود. هنوز وابستگی وجود دارد که با استفاده از نمایش داده شد وارد (با تنظیم `compile = 1` , است که به طور پیش فرض استفاده نمی شود). - کاهش زمان مورد نیاز برای تدوین پویا از نمایش داده شد. #### رفع اشکال: {#bug-fixes-2} @@ -100,7 +100,7 @@ toc_title: '2017' - ثابت خطا در تابع الحاق که در صورتی که ستون اول در یک بلوک رخ داده است نوع مجموعه ای. - پیشرفت در حال حاضر به درستی در سیستم نمایش داده می شود.ادغام جدول. -### انتشار کلیک 1.1.54289, 2017-09-13 {#clickhouse-release-1-1-54289-2017-09-13} +### ClickHouse انتشار 1.1.54289, 2017-09-13 {#clickhouse-release-1-1-54289-2017-09-13} #### ویژگی های جدید: {#new-features-3} @@ -109,10 +109,10 @@ toc_title: '2017' - اضافه شده `root` و `identity` پارامترهای پیکربندی باغ وحش. این اجازه می دهد تا شما را به منزوی کردن کاربران فردی در خوشه باغ وحش است. - اضافه شده توابع مجموع `groupBitAnd`, `groupBitOr` و `groupBitXor` (برای سازگاری, همچنین تحت نام در دسترس هستند `BIT_AND`, `BIT_OR` و `BIT_XOR`). - لغت نامه های خارجی را می توان از خروجی زیر با مشخص کردن یک سوکت در سیستم فایل لود می شود. -- خارجی دیکشنری می توان از mysql بیش از ssl (`ssl_cert`, `ssl_key`, `ssl_ca` پارامترهای). +- واژهنامهها خارجی را می توان از خروجی زیر بر اس اس ال لود می شود (`ssl_cert`, `ssl_key`, `ssl_ca` پارامترهای). - اضافه شدن `max_network_bandwidth_for_user` تنظیم برای محدود کردن استفاده از پهنای باند کلی برای نمایش داده شد در هر کاربر. - پشتیبانی از `DROP TABLE` برای جداول موقت. -- پشتیبانی برای خواندن `DateTime` مقادیر در قالب برچسب زمان یونیکس از `CSV` و `JSONEachRow` فرمتها. +- پشتیبانی از خواندن `DateTime` مقادیر در قالب برچسب زمان یونیکس از `CSV` و `JSONEachRow` فرمتها. - تاخیر کپی در نمایش داده شد توزیع در حال حاضر به طور پیش فرض حذف شدند (حد پیش فرض است 5 دقیقه). - قفل فیفو در طول تغییر استفاده می شود: پرس و جو را تغییر دهید به طور نامحدود برای نمایش داده شد به طور مداوم در حال اجرا مسدود شده است. - گزینه ای برای تنظیم `umask` در فایل پیکربندی. @@ -120,7 +120,7 @@ toc_title: '2017' #### رفع اشکال: {#bug-fixes-3} -- بهبود فرآیند برای حذف پیر گره ها در باغ وحش. قبلا, گره های قدیمی گاهی اوقات نمی حذف اگر درج بسیار مکرر وجود دارد, که باعث سرور به کند به تعطیل, در میان چیزهای دیگر. +- روند برای حذف گره های قدیمی در باغ وحش بهبود یافته است. قبلا, گره های قدیمی گاهی اوقات نمی حذف اگر درج بسیار مکرر وجود دارد, که باعث سرور به کند به تعطیل, در میان چیزهای دیگر. - تصادفی ثابت در هنگام انتخاب میزبان برای اتصال به باغ وحش. - رفع محرومیت از عقب انداختن کپی در نمایش داده شد توزیع اگر ماکت جایل هاست است. - خطایی را که یک بخش داده در یک `ReplicatedMergeTree` جدول را می توان پس از در حال اجرا شکسته `ALTER MODIFY` بر روی یک عنصر در یک `Nested` ساختار. @@ -142,11 +142,11 @@ toc_title: '2017' - در حال حاضر یک مقدار پیش فرض بالاتر برای تنظیم ادغام وجود دارد `max_bytes_to_merge_at_max_space_in_pool` (حداکثر اندازه کل قطعات داده به ادغام در بایت): از 100 دستگاه گوارش به 150 دستگاه گوارش افزایش یافته است. این ممکن است در ادغام بزرگ در حال اجرا پس از ارتقا سرور منجر, که می تواند افزایش بار بر روی زیر سیستم دیسک باعث. اگر فضای رایگان موجود بر روی سرور کمتر از دو برابر مقدار کل ادغام که در حال اجرا هستند, این باعث می شود همه ادغام دیگر برای جلوگیری از در حال اجرا, از جمله ادغام قطعات داده های کوچک. در نتیجه, قرار دادن نمایش داده شد با پیام شکست مواجه خواهد شد “Merges are processing significantly slower than inserts.” استفاده از `SELECT * FROM system.merges` پرس و جو برای نظارت بر وضعیت. شما همچنین می توانید بررسی کنید `DiskSpaceReservedForMerge` متریک در `system.metrics` جدول, و یا در گرافیت. شما لازم نیست برای انجام هر کاری برای رفع این مشکل خود را هنگامی که ادغام بزرگ پایان حل و فصل خواهد شد. اگر شما این غیر قابل قبول, شما می توانید مقدار قبلی برای بازگرداندن `max_bytes_to_merge_at_max_space_in_pool` تنظیمات. برای انجام این کار به `` بخش در پیکربندی.تنظیم ``` ``107374182400 ``` و راه اندازی مجدد سرور. -### انتشار کلیک 1.1.54284, 2017-08-29 {#clickhouse-release-1-1-54284-2017-08-29} +### ClickHouse انتشار 1.1.54284, 2017-08-29 {#clickhouse-release-1-1-54284-2017-08-29} - این یک نسخه رفع اشکال برای نسخه 1.1.54282 قبلی است. این رفع نشت در دایرکتوری قطعات در باغ وحش. -### انتشار کلیک 1.1.54282, 2017-08-23 {#clickhouse-release-1-1-54282-2017-08-23} +### ClickHouse انتشار 1.1.54282, 2017-08-23 {#clickhouse-release-1-1-54282-2017-08-23} این نسخه شامل رفع اشکال برای نسخه قبلی 1.1.54276: @@ -154,7 +154,7 @@ toc_title: '2017' - تجزیه ثابت در هنگام قرار دادن در فرمت مربوط به حوزه علمیه اگر داده های ورودی با شروع می شود. - Errors during runtime compilation of certain aggregate functions (e.g. `groupArray()`). -### انتشار کلیک 1.1.54276, 2017-08-16 {#clickhouse-release-1-1-54276-2017-08-16} +### ClickHouse انتشار 1.1.54276, 2017-08-16 {#clickhouse-release-1-1-54276-2017-08-16} #### ویژگی های جدید: {#new-features-4} @@ -162,7 +162,7 @@ toc_title: '2017' - درج را می توان همزمان در یک جدول توزیع انجام: خوب است تنها پس از بازگشت تمام داده ها بر روی تمام خرده ریز را نجات داد. این است که با تنظیم فعال می شود - اضافه شده نوع داده شناسه برای کار با شناسه 16 بایت. - نام مستعار اضافه شده از کاراکتر, شناور و انواع دیگر برای سازگاری با تابلو فرش. -- اضافه شده توابع toyyyymm, toyyyymmdd و toyyyymmddhhmmss برای تبدیل زمان به اعداد. +- اضافه شده توابع toYYYYMM, toYYYYMMDD و toYYYYMMDDhhmmss برای تبدیل زمان به اعداد. - شما می توانید از نشانی های اینترنتی (همراه با نام میزبان) برای شناسایی سرورها برای نمایش داده شد های هوشمند خوشه ای استفاده کنید. - اضافه شدن پشتیبانی برای استدلال غیر ثابت و شیپور خاموشی منفی در تابع `substring(str, pos, len).` - اضافه شدن پارامتر حداکثر `groupArray(max_size)(column)` عملکرد کلی و عملکرد خود را بهینه سازی کرد. @@ -184,9 +184,9 @@ toc_title: '2017' - اضافه شدن `output_format_json_quote_denormals` تنظیمات, را قادر می سازد خروجی نان و ارزشهای جبهه ملی در فرمت جانسون. - تخصیص جریان بهینه شده در هنگام خواندن از یک جدول توزیع شده است. - تنظیمات را می توان در حالت فقط خواندنی پیکربندی در صورتی که ارزش تغییر نمی کند. -- اضافه شده توانایی برای بازیابی غیر صحیح گرانول از mergetree موتور به منظور دیدار با محدودیت در اندازه بلوک مشخص شده در preferred\_block\_size\_bytes تنظیم. هدف این است که برای کاهش مصرف رم و افزایش محل کش در هنگام پردازش نمایش داده شد از جداول با ستون های بزرگ. +- اضافه شده توانایی برای بازیابی غیر صحیح گرانول از MergeTree موتور به منظور دیدار با محدودیت در اندازه بلوک مشخص شده در preferred\_block\_size\_bytes تنظیم. هدف این است که برای کاهش مصرف رم و افزایش محل کش در هنگام پردازش نمایش داده شد از جداول با ستون های بزرگ. - استفاده موثر از شاخص هایی که حاوی عبارات هستند `toStartOfHour(x)` برای شرایطی مانند `toStartOfHour(x) op сonstexpr.` -- اضافه شدن تنظیمات جدید برای mergetree موتورهای (به merge\_tree بخش در config.شمع): +- اضافه شدن تنظیمات جدید برای MergeTree موتورهای (به merge\_tree بخش در config.شمع): - replicated\_deduplication\_window\_seconds مجموعه تعدادی از ثانیه های مجاز برای deduplicating درج در تکرار جداول. - پاک کردن \_خروج \_پیروید تنظیم میکند که چگونه اغلب شروع به پاکسازی برای حذف اطلاعات منسوخ شده میکند. - از تبدیل شدن به رهبر (و اختصاص ادغام) می توانید یک کپی از تبدیل شدن به رهبر جلوگیری کنید. @@ -228,7 +228,7 @@ toc_title: '2017' - شما می توانید شورای همکاری خلیج فارس 7 به کامپایل خانه عروسکی استفاده کنید. - موازی ایجاد شده با استفاده از ccache+distcc سریع تر در حال حاضر. -### انتشار کلیک 1.1.54245, 2017-07-04 {#clickhouse-release-1-1-54245-2017-07-04} +### ClickHouse انتشار 1.1.54245, 2017-07-04 {#clickhouse-release-1-1-54245-2017-07-04} #### ویژگی های جدید: {#new-features-5} diff --git a/docs/fa/whats-new/changelog/2018.md b/docs/fa/whats-new/changelog/2018.md index b4077fb4db6..3ad0570d5a4 100644 --- a/docs/fa/whats-new/changelog/2018.md +++ b/docs/fa/whats-new/changelog/2018.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 78 toc_title: '2018' --- ## انتشار کلیک 18.16 {#clickhouse-release-18-16} -### انتشار کلیک 18.16.1, 2018-12-21 {#clickhouse-release-18-16-1-2018-12-21} +### ClickHouse انتشار 18.16.1, 2018-12-21 {#clickhouse-release-18-16-1-2018-12-21} #### رفع اشکال: {#bug-fixes} @@ -22,7 +22,7 @@ toc_title: '2018' - رفع برای ایجاد بر روی مکینتاش و بازو. -### انتشار کلیک 18.16.0, 2018-12-14 {#clickhouse-release-18-16-0-2018-12-14} +### ClickHouse انتشار 18.16.0, 2018-12-14 {#clickhouse-release-18-16-0-2018-12-14} #### ویژگی های جدید: {#new-features} @@ -58,13 +58,13 @@ toc_title: '2018' - پردازش پرس و جو ثابت زمانی که `compile_expressions` گزینه فعال است(به طور پیش فرض فعال است). عبارات ثابت نامشخص مانند `now` تابع دیگر گشوده. [\#3457](https://github.com/ClickHouse/ClickHouse/pull/3457) - ثابت تصادف در هنگام مشخص کردن یک استدلال مقیاس غیر ثابت در `toDecimal32/64/128` توابع. - ثابت خطا در هنگام تلاش برای وارد کردن مجموعه ای با `NULL` عناصر در `Values` قالب در یک ستون از نوع `Array` بدون `Nullable` (اگر `input_format_values_interpret_expressions` = 1). [\#3487](https://github.com/ClickHouse/ClickHouse/pull/3487) [\#3503](https://github.com/ClickHouse/ClickHouse/pull/3503) -- ثابت ورود خطا مداوم در `DDLWorker` اگر باغ وحش در دسترس نیست. [8f50c620](https://github.com/ClickHouse/ClickHouse/commit/8f50c620334988b28018213ec0092fe6423847e2) +- ثابت ورود خطا مداوم در `DDLWorker` اگر باغ وحش در دسترس نیست. [8 اف 50620](https://github.com/ClickHouse/ClickHouse/commit/8f50c620334988b28018213ec0092fe6423847e2) - ثابت نوع بازگشت برای `quantile*` توابع از `Date` و `DateTime` انواع استدلال. [\#3580](https://github.com/ClickHouse/ClickHouse/pull/3580) - ثابت `WITH` بند اگر یک نام مستعار ساده و بدون عبارات مشخص. [\#3570](https://github.com/ClickHouse/ClickHouse/pull/3570) - پردازش ثابت نمایش داده شد با نام زیر نمایش داده شد و نام ستون واجد شرایط زمانی که `enable_optimize_predicate_expression` فعال است. [زمستان ژانگ](https://github.com/ClickHouse/ClickHouse/pull/3588) - خطا را ثابت کرد `Attempt to attach to nullptr thread group` در هنگام کار با نمایش محقق. [Marek Vavruša](https://github.com/ClickHouse/ClickHouse/pull/3623) - ثابت تصادف در هنگام عبور استدلال نادرست خاص به `arrayReverse` تابع. [733ا7ب6](https://github.com/ClickHouse/ClickHouse/commit/73e3a7b662161d6005e7727d8a711b930386b871) -- ثابت سرریز بافر در `extractURLParameter` تابع. بهبود عملکرد. اضافه شده پردازش صحیح رشته حاوی صفر بایت. [141e9799](https://github.com/ClickHouse/ClickHouse/commit/141e9799e49201d84ea8e951d1bed4fb6d3dacb5) +- ثابت سرریز بافر در `extractURLParameter` تابع. بهبود عملکرد. اضافه شده پردازش صحیح رشته حاوی صفر بایت. [1419799](https://github.com/ClickHouse/ClickHouse/commit/141e9799e49201d84ea8e951d1bed4fb6d3dacb5) - سرریز بافر ثابت در `lowerUTF8` و `upperUTF8` توابع. حذف توانایی برای اجرای این توابع بیش از `FixedString` استدلال نوع. [\#3662](https://github.com/ClickHouse/ClickHouse/pull/3662) - ثابت شرایط مسابقه نادر در هنگام حذف `MergeTree` میز [\#3680](https://github.com/ClickHouse/ClickHouse/pull/3680) - ثابت شرایط مسابقه در هنگام خواندن از `Buffer` جداول و به طور همزمان انجام `ALTER` یا `DROP` در جداول هدف. [\#3719](https://github.com/ClickHouse/ClickHouse/pull/3719) @@ -119,7 +119,7 @@ toc_title: '2018' ## انتشار کلیک 18.14 {#clickhouse-release-18-14} -### انتشار کلیک 18.14.19, 2018-12-19 {#clickhouse-release-18-14-19-2018-12-19} +### ClickHouse انتشار 18.14.19, 2018-12-19 {#clickhouse-release-18-14-19-2018-12-19} #### رفع اشکال: {#bug-fixes-2} @@ -131,7 +131,7 @@ toc_title: '2018' - رفع برای ایجاد بر روی بازو. -### انتشار کلیک 18.14.18, 2018-12-04 {#clickhouse-release-18-14-18-2018-12-04} +### ClickHouse انتشار 18.14.18, 2018-12-04 {#clickhouse-release-18-14-18-2018-12-04} #### رفع اشکال: {#bug-fixes-3} @@ -144,7 +144,7 @@ toc_title: '2018' - ساخت ثابت با کتابخانه های لووم/کلنگ نسخه 7 از بسته های سیستم عامل (این کتابخانه ها برای تدوین پرس و جو در زمان اجرا استفاده می شود). [\#3582](https://github.com/ClickHouse/ClickHouse/pull/3582) -### انتشار کلیک 18.14.17, 2018-11-30 {#clickhouse-release-18-14-17-2018-11-30} +### ClickHouse انتشار 18.14.17, 2018-11-30 {#clickhouse-release-18-14-17-2018-11-30} #### رفع اشکال: {#bug-fixes-4} @@ -154,13 +154,13 @@ toc_title: '2018' - ثابت بن بست پرس و جو در مورد زمانی که ایجاد موضوع پرس و جو با شکست مواجه `Resource temporarily unavailable` خطا. [\#3643](https://github.com/ClickHouse/ClickHouse/pull/3643) - تجزیه ثابت از `ENGINE` بند زمانی که `CREATE AS table` نحو مورد استفاده قرار گرفت و `ENGINE` بند قبل از مشخص شد `AS table` (خطا منجر به نادیده گرفتن موتور مشخص شده). [\#3692](https://github.com/ClickHouse/ClickHouse/pull/3692) -### انتشار کلیک 18.14.15, 2018-11-21 {#clickhouse-release-18-14-15-2018-11-21} +### ClickHouse انتشار 18.14.15, 2018-11-21 {#clickhouse-release-18-14-15-2018-11-21} #### رفع اشکال: {#bug-fixes-5} - اندازه تکه حافظه دست بالا بود در حالی که غیرشخصی ستون از نوع `Array(String)` که منجر به “Memory limit exceeded” خطاها. این موضوع در نسخه 18.12.13 ظاهر شد. [\#3589](https://github.com/ClickHouse/ClickHouse/issues/3589) -### انتشار کلیک 18.14.14, 2018-11-20 {#clickhouse-release-18-14-14-2018-11-20} +### ClickHouse انتشار 18.14.14, 2018-11-20 {#clickhouse-release-18-14-14-2018-11-20} #### رفع اشکال: {#bug-fixes-6} @@ -170,7 +170,7 @@ toc_title: '2018' - مشکلات ثابت-7 از سیستم مکینتاش) [\#3582](https://github.com/ClickHouse/ClickHouse/pull/3582) -### انتشار کلیک 18.14.13, 2018-11-08 {#clickhouse-release-18-14-13-2018-11-08} +### ClickHouse انتشار 18.14.13, 2018-11-08 {#clickhouse-release-18-14-13-2018-11-08} #### رفع اشکال: {#bug-fixes-7} @@ -188,7 +188,7 @@ toc_title: '2018' - بهبود برای ساده سازی ساخت ارکادیا. [\#3475](https://github.com/ClickHouse/ClickHouse/pull/3475), [\#3535](https://github.com/ClickHouse/ClickHouse/pull/3535) -### انتشار کلیک 18.14.12, 2018-11-02 {#clickhouse-release-18-14-12-2018-11-02} +### ClickHouse انتشار 18.14.12, 2018-11-02 {#clickhouse-release-18-14-12-2018-11-02} #### رفع اشکال: {#bug-fixes-8} @@ -196,7 +196,7 @@ toc_title: '2018' - ثابت تولید نمایش داده شد نادرست (با خالی `WHERE` بند) هنگامی که پرس و جو پایگاه داده های خارجی. [هیلد](https://github.com/ClickHouse/ClickHouse/pull/3477) - ثابت با استفاده از یک مقدار ایست نادرست در لغت نامه او بی سی. [Marek Vavruša](https://github.com/ClickHouse/ClickHouse/pull/3511) -### انتشار کلیک 18.14.11, 2018-10-29 {#clickhouse-release-18-14-11-2018-10-29} +### ClickHouse انتشار 18.14.11, 2018-10-29 {#clickhouse-release-18-14-11-2018-10-29} #### رفع اشکال: {#bug-fixes-9} @@ -205,12 +205,12 @@ toc_title: '2018' - نتایج پرس و جو نادرست ثابت اگر `merge_tree_uniform_read_distribution` تنظیم غیر فعال است(به طور پیش فرض فعال). [\#3429](https://github.com/ClickHouse/ClickHouse/pull/3429) - ثابت خطا در درج به یک جدول توزیع شده در فرمت بومی. [\#3411](https://github.com/ClickHouse/ClickHouse/issues/3411) -### انتشار کلیک 18.14.10, 2018-10-23 {#clickhouse-release-18-14-10-2018-10-23} +### ClickHouse انتشار 18.14.10, 2018-10-23 {#clickhouse-release-18-14-10-2018-10-23} - این `compile_expressions` تنظیم (مجموعه دستگاه گوارش عبارات) به طور پیش فرض غیر فعال است. [\#3410](https://github.com/ClickHouse/ClickHouse/pull/3410) - این `enable_optimize_predicate_expression` تنظیم به طور پیش فرض غیر فعال است. -### انتشار کلیک 18.14.9, 2018-10-16 {#clickhouse-release-18-14-9-2018-10-16} +### ClickHouse انتشار 18.14.9, 2018-10-16 {#clickhouse-release-18-14-9-2018-10-16} #### ویژگی های جدید: {#new-features-1} @@ -227,7 +227,7 @@ toc_title: '2018' #### ویژگی های تجربی: {#experimental-features} -- بهینه سازی گروه بند برای `LowCardinality data types.` [\#3138](https://github.com/ClickHouse/ClickHouse/pull/3138) +- بهینه سازی گروه توسط بند برای `LowCardinality data types.` [\#3138](https://github.com/ClickHouse/ClickHouse/pull/3138) - محاسبه بهینه از عبارات برای `LowCardinality data types.` [\#3200](https://github.com/ClickHouse/ClickHouse/pull/3200) #### بهبود: {#improvements-2} @@ -272,7 +272,7 @@ toc_title: '2018' - تهیه غیر ضروری ثابت از ساختارهای داده برای `JOIN`بازدید کنندگان بر روی سرور که شروع پرس و جو در صورتی که `JOIN` تنها بر روی سرور از راه دور انجام می شود. [\#3340](https://github.com/ClickHouse/ClickHouse/pull/3340) - اشکالات ثابت در `Kafka` موتور: بن بست پس از استثنا در هنگام شروع به خواندن داده ها و قفل پس از اتمام [Marek Vavruša](https://github.com/ClickHouse/ClickHouse/pull/3215). - برای `Kafka` جداول اختیاری `schema` پارامتر تصویب نشد (طرح از `Cap'n'Proto` قالب). [اطلاعات دقیق](https://github.com/ClickHouse/ClickHouse/pull/3150) -- اگر این گروه از باغ وحش سرور است سرور است که اتصال را قبول اما پس از آن بلافاصله آن را به جای پاسخ به از دست دادن clickhouse انتخاب برای اتصال به سرور دیگری. قبلا این خطا را تولید کرد `Cannot read all data. Bytes read: 0. Bytes expected: 4.` و سرور نمی تواند شروع. [8218رف3](https://github.com/ClickHouse/ClickHouse/commit/8218cf3a5f39a43401953769d6d12a0bb8d29da9) +- اگر این گروه از باغ وحش سرور است سرور است که اتصال را قبول اما پس از آن بلافاصله آن را به جای پاسخ به از دست دادن ClickHouse انتخاب برای اتصال به سرور دیگری. قبلا این خطا را تولید کرد `Cannot read all data. Bytes read: 0. Bytes expected: 4.` و سرور نمی تواند شروع. [8218رف3](https://github.com/ClickHouse/ClickHouse/commit/8218cf3a5f39a43401953769d6d12a0bb8d29da9) - اگر این گروه از سرور باغ وحش شامل سرور که پرس و جو دی ان اس خطا می گرداند, این سرویس دهنده نادیده گرفته می شوند. [17ب8209](https://github.com/ClickHouse/ClickHouse/commit/17b8e209221061325ad7ba0539f03c6e65f87f29) - تبدیل نوع ثابت بین `Date` و `DateTime` هنگام وارد کردن داده ها در `VALUES` قالب (اگر `input_format_values_interpret_expressions = 1`). قبلا, تبدیل بین مقدار عددی تعداد روز در زمان عصر یونیکس و برچسب زمان یونیکس انجام شد, که منجر به نتایج غیر منتظره. [\#3229](https://github.com/ClickHouse/ClickHouse/pull/3229) - اصلاح نوع تبدیل بین `Decimal` و اعداد صحیح. [\#3211](https://github.com/ClickHouse/ClickHouse/pull/3211) @@ -289,7 +289,7 @@ toc_title: '2018' - این `log_query_threads` تنظیم (ورود اطلاعات در مورد هر موضوع اجرای پرس و جو) در حال حاضر اثر تنها در صورتی که `log_queries` گزینه (ورود اطلاعات در مورد نمایش داده شد) به 1 تنظیم شده است. از زمان `log_query_threads` گزینه به طور پیش فرض فعال, اطلاعات در مورد موضوعات قبلا وارد شده بود حتی اگر ورود به سیستم پرس و جو غیر فعال شد. [\#3241](https://github.com/ClickHouse/ClickHouse/pull/3241) - ثابت خطا در توزیع, بهره برداری از quantiles aggregate function (پیام خطا `Not found column quantile...`). [292ا8855](https://github.com/ClickHouse/ClickHouse/commit/292a885533b8e3b41ce8993867069d14cbd5a664) - ثابت مشکل سازگاری در هنگام کار بر روی یک خوشه از نسخه 18.12.17 سرور و سرور های قدیمی تر در همان زمان. برای نمایش داده شد توزیع شده با گروه های کلید از هر دو ثابت و غیر ثابت طول اگر وجود دارد مقدار زیادی از داده ها به کل بازگشت داده بود و نه همیشه به طور کامل جمع (دو ردیف مختلف شامل همان جمع کلید). [\#3254](https://github.com/ClickHouse/ClickHouse/pull/3254) -- ثابت دست زدن به تعویض در `clickhouse-performance-test`, اگر پرس و جو شامل تنها بخشی از تعویض اعلام شده در تست. [\#3263](https://github.com/ClickHouse/ClickHouse/pull/3263) +- دست زدن به ثابت تعویض در `clickhouse-performance-test`, اگر پرس و جو شامل تنها بخشی از تعویض اعلام شده در تست. [\#3263](https://github.com/ClickHouse/ClickHouse/pull/3263) - ثابت خطا در هنگام استفاده از `FINAL` با `PREWHERE`. [\#3298](https://github.com/ClickHouse/ClickHouse/pull/3298) - ثابت خطا در هنگام استفاده از `PREWHERE` بیش از ستون که در طول اضافه شد `ALTER`. [\#3298](https://github.com/ClickHouse/ClickHouse/pull/3298) - اضافه شدن یک چک برای عدم وجود `arrayJoin` برای `DEFAULT` و `MATERIALIZED` عبارات. قبلا, `arrayJoin` منجر به خطا در هنگام قرار دادن داده ها. [\#3337](https://github.com/ClickHouse/ClickHouse/pull/3337) @@ -303,7 +303,7 @@ toc_title: '2018' ## انتشار کلیک 18.12 {#clickhouse-release-18-12} -### انتشار کلیک 18.12.17, 2018-09-16 {#clickhouse-release-18-12-17-2018-09-16} +### ClickHouse انتشار 18.12.17, 2018-09-16 {#clickhouse-release-18-12-17-2018-09-16} #### ویژگی های جدید: {#new-features-2} @@ -324,7 +324,7 @@ toc_title: '2018' - این `enable_optimize_predicate_expression` گزینه به طور پیش فرض فعال (که است و نه خوش بینانه). اگر تجزیه و تحلیل پرس و جو خطا رخ می دهد که مربوط به جستجو برای نام ستون مجموعه `enable_optimize_predicate_expression` به 0. [زمستان ژانگ](https://github.com/ClickHouse/ClickHouse/pull/3107) -### انتشار کلیک 18.12.14, 2018-09-13 {#clickhouse-release-18-12-14-2018-09-13} +### ClickHouse انتشار 18.12.14, 2018-09-13 {#clickhouse-release-18-12-14-2018-09-13} #### ویژگی های جدید: {#new-features-3} @@ -345,7 +345,7 @@ toc_title: '2018' - ثابت تصادف در هنگام ایجاد یک جدول موقت از پرس و جو با یک `IN` شرط. [زمستان ژانگ](https://github.com/ClickHouse/ClickHouse/pull/3098) - ثابت خطا در توابع کل برای ارریس است که می تواند داشته باشد `NULL` عناصر. [زمستان ژانگ](https://github.com/ClickHouse/ClickHouse/pull/3097) -### انتشار کلیک 18.12.13, 2018-09-10 {#clickhouse-release-18-12-13-2018-09-10} +### ClickHouse انتشار 18.12.13, 2018-09-10 {#clickhouse-release-18-12-13-2018-09-10} #### ویژگی های جدید: {#new-features-4} @@ -401,7 +401,7 @@ toc_title: '2018' - این `odbc` تابع جدول در حال حاضر اجازه می دهد تا به شما برای مشخص کردن پایگاه داده/نام طرح. [ایموس پرنده](https://github.com/ClickHouse/ClickHouse/pull/2885) - توانایی استفاده از یک نام کاربری مشخص شده در `clickhouse-client` فایل پیکربندی. [ولادیمیر کوزبین](https://github.com/ClickHouse/ClickHouse/pull/2909) - این `ZooKeeperExceptions` شمارنده به سه شمارنده تقسیم شده است: `ZooKeeperUserExceptions`, `ZooKeeperHardwareExceptions` و `ZooKeeperOtherExceptions`. -- `ALTER DELETE` نمایش داده شد و کار را برای محقق views. +- `ALTER DELETE` نمایش داده شد برای نمایش محقق کار می کنند. - تصادفی اضافه شده در هنگام اجرای موضوع پاکسازی به صورت دوره ای برای `ReplicatedMergeTree` جداول به منظور جلوگیری از خوشه بار دوره ای زمانی که تعداد بسیار زیادی از وجود دارد `ReplicatedMergeTree` میز - پشتیبانی از `ATTACH TABLE ... ON CLUSTER` نمایش داده شد. [\#3025](https://github.com/ClickHouse/ClickHouse/pull/3025) @@ -443,7 +443,7 @@ toc_title: '2018' - اکثر تست های یکپارچه سازی هم اکنون می توانید توسط متعهد اجرا می شود. - چک سبک کد همچنین می توانید با ارتکاب اجرا می شود. -- این `memcpy` پیاده سازی به درستی در هنگام ساخت در لینوکس7/فدورا انتخاب شده است. [اتین champetier](https://github.com/ClickHouse/ClickHouse/pull/2912) +- این `memcpy` پیاده سازی به درستی در هنگام ساخت در لینوکس7/فدورا انتخاب شده است. [اتین Champetier](https://github.com/ClickHouse/ClickHouse/pull/2912) - هنگام استفاده از صدای شیپور برای ساخت, برخی از هشدارهای از `-Weverything` اضافه شده است, در علاوه بر این به طور منظم `-Wall-Wextra -Werror`. [\#2957](https://github.com/ClickHouse/ClickHouse/pull/2957) - اشکال زدایی ساخت با استفاده از `jemalloc` گزینه اشکال زدایی. - رابط کتابخانه برای تعامل با باغ وحش انتزاعی اعلام شده است. [\#2950](https://github.com/ClickHouse/ClickHouse/pull/2950) @@ -456,7 +456,7 @@ toc_title: '2018' - قام را می توان برای تکرار استفاده می شود. [\#2760](https://github.com/ClickHouse/ClickHouse/pull/2760) - اضافه شدن توابع `murmurHash2_64`, `murmurHash3_32`, `murmurHash3_64` و `murmurHash3_128` علاوه بر موجود `murmurHash2_32`. [\#2791](https://github.com/ClickHouse/ClickHouse/pull/2791) -- پشتیبانی از nullable types در clickhouse odbc driver (`ODBCDriver2` فرمت خروجی). [\#2834](https://github.com/ClickHouse/ClickHouse/pull/2834) +- پشتیبانی از Nullable types در ClickHouse ODBC driver (`ODBCDriver2` فرمت خروجی). [\#2834](https://github.com/ClickHouse/ClickHouse/pull/2834) - پشتیبانی از `UUID` در ستون های کلیدی. #### بهبود: {#improvements-5} @@ -500,7 +500,7 @@ toc_title: '2018' ## انتشار کلیک 18.6 {#clickhouse-release-18-6} -### انتشار کلیک 18.6.0, 2018-08-02 {#clickhouse-release-18-6-0-2018-08-02} +### ClickHouse انتشار 18.6.0, 2018-08-02 {#clickhouse-release-18-6-0-2018-08-02} #### ویژگی های جدید: {#new-features-6} @@ -532,7 +532,7 @@ toc_title: '2018' ## انتشار کلیک 18.4 {#clickhouse-release-18-4} -### انتشار کلیک 18.4.0, 2018-07-28 {#clickhouse-release-18-4-0-2018-07-28} +### ClickHouse انتشار 18.4.0, 2018-07-28 {#clickhouse-release-18-4-0-2018-07-28} #### ویژگی های جدید: {#new-features-8} @@ -570,11 +570,11 @@ toc_title: '2018' - پشتیبانی از انواع دلخواه در مقایسه اپراتورها ([\#2026](https://github.com/ClickHouse/ClickHouse/issues/2026)). - این `users.xml` فایل اجازه می دهد تا تنظیم یک ماسک زیر شبکه در قالب `10.0.0.1/255.255.255.0`. این برای استفاده از ماسک برای شبکه های اینترنتی6 با صفر در وسط ضروری است ([\#2637](https://github.com/ClickHouse/ClickHouse/pull/2637)). - اضافه شدن `arrayDistinct` تابع ([\#2670](https://github.com/ClickHouse/ClickHouse/pull/2670)). -- این summingmergetree موتور هم اکنون می توانید کار با aggregatefunction نوع ستون ([پان سنتانتین](https://github.com/ClickHouse/ClickHouse/pull/2566)). +- این SummingMergeTree موتور هم اکنون می توانید کار با AggregateFunction نوع ستون ([پان سنتانتین](https://github.com/ClickHouse/ClickHouse/pull/2566)). #### بهبود: {#improvements-9} -- طرح شماره برای نسخه های انتشار تغییر کرده است. در حال حاضر بخش اول شامل سال انتشار (a. d. مسکو منطقه زمانی منهای 2000) بخش دوم شامل تعدادی به صورت عمده تغییرات را افزایش می دهد برای بسیاری منتشر شده) و بخش سوم پچ نسخه. منتشر شده هنوز هم به عقب سازگار هستند, مگر اینکه در غیر این صورت در تغییرات اعلام. +- طرح شماره برای نسخه های انتشار تغییر کرده است. در حال حاضر بخش اول شامل سال انتشار (A. D. مسکو منطقه زمانی منهای 2000) بخش دوم شامل تعدادی به صورت عمده تغییرات را افزایش می دهد برای بسیاری منتشر شده) و بخش سوم پچ نسخه. منتشر شده هنوز هم به عقب سازگار هستند, مگر اینکه در غیر این صورت در تغییرات اعلام. - تبدیل سریع تر اعداد ممیز شناور به یک رشته ([ایموس پرنده](https://github.com/ClickHouse/ClickHouse/pull/2664)). - اگر برخی از ردیف در طول درج به دلیل خطاهای تجزیه قلم شد (این ممکن است با `input_allow_errors_num` و `input_allow_errors_ratio` تنظیمات را فعال کنید), تعداد ردیف قلم در حال حاضر به ورود به سیستم سرور نوشته شده است ([لوناردو سیسیچی](https://github.com/ClickHouse/ClickHouse/pull/2669)). @@ -582,7 +582,7 @@ toc_title: '2018' - ثابت فرمان کوتاه برای جداول موقت ([ایموس پرنده](https://github.com/ClickHouse/ClickHouse/pull/2624)). - ثابت بن بست نادر در کتابخانه مشتری باغ وحش که رخ داده است زمانی که یک خطای شبکه وجود دارد در حالی که خواندن پاسخ ([315200](https://github.com/ClickHouse/ClickHouse/commit/c315200e64b87e44bdf740707fc857d1fdf7e947)). -- ثابت خطا در طول بازیگران به nullable types ([\#1322](https://github.com/ClickHouse/ClickHouse/issues/1322)). +- ثابت خطا در طول بازیگران به Nullable types ([\#1322](https://github.com/ClickHouse/ClickHouse/issues/1322)). - ثابت نتیجه نادرست از `maxIntersection()` تابع زمانی که مرزهای فواصل همزمان ([مایکل فورمور](https://github.com/ClickHouse/ClickHouse/pull/2657)). - تحول نادرست ثابت از زنجیره بیان و یا در یک استدلال تابع ([chenxing-xc](https://github.com/ClickHouse/ClickHouse/pull/2663)). - تخریب عملکرد ثابت برای نمایش داده شد حاوی `IN (subquery)` عبارات در داخل یکی دیگر از خرده فروشی ([\#2571](https://github.com/ClickHouse/ClickHouse/issues/2571)). @@ -595,7 +595,7 @@ toc_title: '2018' ## انتشار کلیک 1.1 {#clickhouse-release-1-1} -### انتشار کلیک 1.1.54394, 2018-07-12 {#clickhouse-release-1-1-54394-2018-07-12} +### ClickHouse انتشار 1.1.54394, 2018-07-12 {#clickhouse-release-1-1-54394-2018-07-12} #### ویژگی های جدید: {#new-features-10} @@ -611,7 +611,7 @@ toc_title: '2018' - ثابت چگونه خالی `TinyLog` جدول پس از قرار دادن یک بلوک داده خالی کار می کند ([\#2563](https://github.com/ClickHouse/ClickHouse/issues/2563)). - این `system.zookeeper` جدول کار می کند اگر ارزش گره در باغ وحش تهی است. -### انتشار کلیک 1.1.54390, 2018-07-06 {#clickhouse-release-1-1-54390-2018-07-06} +### ClickHouse انتشار 1.1.54390, 2018-07-06 {#clickhouse-release-1-1-54390-2018-07-06} #### ویژگی های جدید: {#new-features-11} @@ -643,7 +643,7 @@ toc_title: '2018' - ارسال فایل های دیگر ممکن است زمانی که به خروجی زیر متصل می شود (`LOAD DATA LOCAL INFILE`). -### انتشار کلیک 1.1.54388, 2018-06-28 {#clickhouse-release-1-1-54388-2018-06-28} +### ClickHouse انتشار 1.1.54388, 2018-06-28 {#clickhouse-release-1-1-54388-2018-06-28} #### ویژگی های جدید: {#new-features-12} @@ -652,14 +652,14 @@ toc_title: '2018' - پشتیبانی از `TRUNCATE TABLE` پرسوجو ([زمستان ژانگ](https://github.com/ClickHouse/ClickHouse/pull/2260)) - چند جدید `SYSTEM` نمایش داده شد برای جداول تکرار (`RESTART REPLICAS`, `SYNC REPLICA`, `[STOP|START] [MERGES|FETCHES|SENDS REPLICATED|REPLICATION QUEUES]`). - توانایی نوشتن به یک جدول با موتور خروجی زیر و عملکرد جدول مربوطه اضافه شده است ([sundy-li](https://github.com/ClickHouse/ClickHouse/pull/2294)). -- اضافه شدن `url()` عملکرد جدول و `URL` موتور جدول ([الکساندر sapin](https://github.com/ClickHouse/ClickHouse/pull/2501)). +- اضافه شدن `url()` عملکرد جدول و `URL` موتور جدول ([الکساندر Sapin](https://github.com/ClickHouse/ClickHouse/pull/2501)). - اضافه شدن `windowFunnel` تابع جمع ([sundy-li](https://github.com/ClickHouse/ClickHouse/pull/2352)). - جدید `startsWith` و `endsWith` توابع برای رشته ها ([وادیم پلختینسکی](https://github.com/ClickHouse/ClickHouse/pull/2429)). - این `numbers()` تابع جدول در حال حاضر اجازه می دهد تا شما را به مشخص افست ([زمستان ژانگ](https://github.com/ClickHouse/ClickHouse/pull/2535)). - رمز عبور به `clickhouse-client` می توان تعاملی وارد شده است. - سیاهههای مربوط به سرور هم اکنون می توانید به وبلاگ ارسال می شود ([الکساندر کرشنینیکف](https://github.com/ClickHouse/ClickHouse/pull/2459)). -- پشتیبانی از ورود به لغت نامه ها با یک منبع کتابخانه مشترک ([الکساندر sapin](https://github.com/ClickHouse/ClickHouse/pull/2472)). -- پشتیبانی برای سفارشی csv delimiters ([ایوان ژوکوف](https://github.com/ClickHouse/ClickHouse/pull/2263)) +- پشتیبانی از ورود به لغت نامه ها با یک منبع کتابخانه مشترک ([الکساندر Sapin](https://github.com/ClickHouse/ClickHouse/pull/2472)). +- پشتیبانی برای سفارشی CSV delimiters ([ایوان ژوکوف](https://github.com/ClickHouse/ClickHouse/pull/2263)) - اضافه شدن `date_time_input_format` تنظیمات. اگر این تنظیم را تغییر دهید `'best_effort'`, ارزش تاریخ ساعت خواهد شد در طیف گسترده ای از فرمت های به عنوان خوانده شده. - اضافه شدن `clickhouse-obfuscator` ابزار برای مبهم و تاریک کردن داده ها. مثال طریقه استفاده: انتشار داده های مورد استفاده در تست عملکرد. @@ -675,7 +675,7 @@ toc_title: '2018' - هنگام خواندن یک ستون جداگانه از یک ساختار تو در تو خطایی رخ داد ([\#2066](https://github.com/ClickHouse/ClickHouse/issues/2066)). - ثابت خطا در هنگام تجزیه و تحلیل نمایش داده شد با داشتن بند مانند `HAVING tuple IN (...)`. - ثابت خطا در هنگام تجزیه و تحلیل نمایش داده شد با نام مستعار بازگشتی. -- ثابت خطا در هنگام خواندن از replacingmergetree با یک بیماری در prewhere فیلتر است که تمام ردیف ([\#2525](https://github.com/ClickHouse/ClickHouse/issues/2525)). +- ثابت خطا در هنگام خواندن از ReplacingMergeTree با یک بیماری در PREWHERE فیلتر است که تمام ردیف ([\#2525](https://github.com/ClickHouse/ClickHouse/issues/2525)). - هنگام استفاده از جلسات در رابط اچ تی پی تنظیمات پروفایل کاربر اعمال نشد. - ثابت چگونه تنظیمات از پارامترهای خط فرمان در خانه کلیک اعمال می شود-محلی. - کتابخانه مشتری باغ وحش در حال حاضر با استفاده از فاصله جلسه دریافت شده از سرور. @@ -686,7 +686,7 @@ toc_title: '2018' - مقایسه نوع برای ثابت شده است `DateTime` با و بدون منطقه زمانی ([الکساندر بوچاروف](https://github.com/ClickHouse/ClickHouse/pull/2400)). - تجزیه نحوی ثابت و قالب بندی از `CAST` اپراتور - درج ثابت به یک نمایش تحقق برای موتور جدول توزیع شده است ([Babacar Diassé](https://github.com/ClickHouse/ClickHouse/pull/2411)). -- ثابت شرایط مسابقه در هنگام نوشتن داده ها از `Kafka` موتور به نمایش تحقق ([Yangkuan لیو](https://github.com/ClickHouse/ClickHouse/pull/2448)). +- ثابت شرایط مسابقه در هنگام نوشتن داده ها از `Kafka` موتور به نمایش تحقق ([یانگکوان لیو](https://github.com/ClickHouse/ClickHouse/pull/2448)). - در از راه دور() تابع جدول ثابت شده است. - رفتار خروج ثابت از `clickhouse-client` در حالت چند خطی ([\#2510](https://github.com/ClickHouse/ClickHouse/issues/2510)). @@ -696,7 +696,7 @@ toc_title: '2018' - بهبود عملکرد فشرده سازی 2.4. - تجزیه و تحلیل سریع تر برای نمایش داده شد با تعداد زیادی از می پیوندد و زیر نمایش داده شد. - کش دی ان اس در حال حاضر به طور خودکار به روز هنگامی که بیش از حد بسیاری از خطاهای شبکه وجود دارد. -- جدول درج دیگر رخ می دهد, اگر وارد یکی از این محقق views امکان پذیر نیست به دلیل آن است بیش از حد بسیاری از قطعات. +- درج جدول دیگر رخ می دهد اگر درج به یکی از دیدگاه های تحقق ممکن نیست چرا که بیش از حد بسیاری از قطعات. - اصلاح اختلاف در شمارنده رویداد `Query`, `SelectQuery` و `InsertQuery`. - عبارات مانند `tuple IN (SELECT tuple)` مجاز اگر انواع تاپل مطابقت. - سرور با جداول تکرار می توانید شروع به حتی اگر شما باغ وحش پیکربندی نشده است. @@ -709,7 +709,7 @@ toc_title: '2018' - اضافه شده توانایی برای ساخت llvm از submodule. - این نسخه از librdkafka کتابخانه به روز شده است برای v0.11.4. - اضافه شدن توانایی استفاده از کتابخانه سیستم لیبکپویید. نسخه کتابخانه شده است به 0.4.0 به روز شد. -- ثابت ساخت با استفاده از vectorclass کتابخانه ([Babacar Diassé](https://github.com/ClickHouse/ClickHouse/pull/2274)). +- ثابت ساخت با استفاده از کتابخانه وکتورکلااس ([Babacar Diassé](https://github.com/ClickHouse/ClickHouse/pull/2274)). - در حال حاضر تولید فایل برای نینجا به طور پیش فرض (مانند هنگام استفاده از `-G Ninja`). - اضافه شدن قابلیت استفاده از کتابخانه لیبتاینفو به جای نوشیدن شراب ([جورجی کندراتیف](https://github.com/ClickHouse/ClickHouse/pull/2519)). - رفع یک درگیری فایل هدر در فدورا پوست دباغی نشده ([\#2520](https://github.com/ClickHouse/ClickHouse/issues/2520)). @@ -719,19 +719,19 @@ toc_title: '2018' - حذف فرار در `Vertical` و `Pretty*` فرمت ها و حذف `VerticalRaw` قالب. - اگر سرور با نسخه 1.1.54388 (یا جدیدتر) و سرور با نسخه های قدیمی تر به طور همزمان در یک پرس و جو توزیع استفاده می شود و پرس و جو است `cast(x, 'Type')` بیان بدون `AS` کلمه کلیدی و کلمه ندارد `cast` در بزرگ, یک استثنا خواهد شد با یک پیام مانند پرتاب `Not found column cast(0, 'UInt8') in block`. تخلیه: به روز رسانی سرور در کل خوشه. -### انتشار کلیک 1.1.54385, 2018-06-01 {#clickhouse-release-1-1-54385-2018-06-01} +### ClickHouse انتشار 1.1.54385, 2018-06-01 {#clickhouse-release-1-1-54385-2018-06-01} #### رفع اشکال: {#bug-fixes-21} - ثابت خطا که در برخی موارد باعث عملیات باغ وحش برای جلوگیری از. -### انتشار کلیک 1.1.54383, 2018-05-22 {#clickhouse-release-1-1-54383-2018-05-22} +### ClickHouse انتشار 1.1.54383, 2018-05-22 {#clickhouse-release-1-1-54383-2018-05-22} #### رفع اشکال: {#bug-fixes-22} - ثابت کاهش سرعت صف تکرار اگر یک جدول است بسیاری از کپی. -### انتشار کلیک 1.1.54381, 2018-05-14 {#clickhouse-release-1-1-54381-2018-05-14} +### ClickHouse انتشار 1.1.54381, 2018-05-14 {#clickhouse-release-1-1-54381-2018-05-14} #### رفع اشکال: {#bug-fixes-23} @@ -761,7 +761,7 @@ toc_title: '2018' - پشتیبانی حذف برای عبارات مانند `(a, b) IN (SELECT (a, b))` (شما می توانید بیان معادل استفاده کنید `(a, b) IN (SELECT a, b)`). در نسخه های قبلی, این عبارات منجر به نامشخص `WHERE` فیلتر کردن یا ایجاد خطا. -### انتشار کلیک 1.1.54378, 2018-04-16 {#clickhouse-release-1-1-54378-2018-04-16} +### ClickHouse انتشار 1.1.54378, 2018-04-16 {#clickhouse-release-1-1-54378-2018-04-16} #### ویژگی های جدید: {#new-features-14} @@ -774,7 +774,7 @@ toc_title: '2018' - پشتیبانی از رمزگذاری سرور به سرور برای جداول توزیع شده (`1` در پیکربندی ماکت در ``). - پیکربندی سطح جدول برای `ReplicatedMergeTree` خانواده به منظور به حداقل رساندن مقدار داده های ذخیره شده در باغ وحش: : `use_minimalistic_checksums_in_zookeeper = 1` - پیکربندی از `clickhouse-client` اعلان کردن. به طور پیش فرض, نام سرور در حال حاضر خروجی به اعلان. نام صفحه نمایش سرور را می توان تغییر داد. همچنین در ارسال `X-ClickHouse-Display-Name` HTTP header (Kirill Shvakov). -- چند کاما از هم جدا `topics` می توان برای مشخص `Kafka` موتور (توبیاس Adamson) +- چند کاما از هم جدا `topics` می توان برای مشخص `Kafka` موتور) - هنگامی که یک پرس و جو توسط متوقف `KILL QUERY` یا `replace_running_query` مشتری دریافت می کند `Query was canceled` استثنا به جای یک نتیجه ناقص. #### بهبود: {#improvements-13} @@ -814,7 +814,7 @@ toc_title: '2018' - حذف تفسیر خاص از یک عبارت در اگر مجموعه ای در سمت چپ مشخص شده است. قبلا بیان `arr IN (set)` به عنوان تفسیر شد “at least one `arr` element belongs to the `set`”. برای دریافت همان رفتار در نسخه جدید, نوشتن `arrayExists(x -> x IN (set), arr)`. - استفاده نادرست از گزینه سوکت را غیرفعال کرد `SO_REUSEPORT`, که به اشتباه به طور پیش فرض در کتابخانه کم فعال شد. توجه داشته باشید که در لینوکس دیگر هیچ دلیلی وجود ندارد که به طور همزمان نشانی ها را مشخص کند `::` و `0.0.0.0` for listen – use just `::`, که اجازه می دهد گوش دادن به اتصال هر دو بیش از لیگ4 و ایپو6 (با تنظیمات پیکربندی هسته به طور پیش فرض). شما همچنین می توانید به رفتار از نسخه های قبلی با مشخص برگرداندن `1` در پیکربندی. -### انتشار کلیک 1.1.54370, 2018-03-16 {#clickhouse-release-1-1-54370-2018-03-16} +### ClickHouse انتشار 1.1.54370, 2018-03-16 {#clickhouse-release-1-1-54370-2018-03-16} #### ویژگی های جدید: {#new-features-15} @@ -841,12 +841,12 @@ toc_title: '2018' - ترمیم رفتار برای نمایش داده شد مانند `SELECT * FROM remote('server2', default.table) WHERE col IN (SELECT col2 FROM default.table)` هنگامی که در سمت راست از `IN` باید از راه دور استفاده کنید `default.table` به جای یک محلی. این رفتار در نسخه 1.1.54358 شکسته شد. - حذف غیر اصلی ورود به سیستم سطح خطا از `Not found column ... in block`. -### انتشار کلیک 1.1.54362, 2018-03-11 {#clickhouse-release-1-1-54362-2018-03-11} +### ClickHouse انتشار 1.1.54362, 2018-03-11 {#clickhouse-release-1-1-54362-2018-03-11} #### ویژگی های جدید: {#new-features-16} - تجمع بدون `GROUP BY` برای یک مجموعه خالی (مانند `SELECT count(*) FROM table WHERE 0`) در حال حاضر در نتیجه با یک ردیف با ارزش تهی برای توابع کل گرداند, در انطباق با استاندارد گذاشتن. برای بازگرداندن رفتار قدیمی (بازگشت به نتیجه خالی), تنظیم `empty_result_for_aggregation_by_empty_set` به 1. -- تبدیل نوع اضافه شده برای `UNION ALL`. نام مستعار مختلف مجاز است `SELECT` موقعیت خود را در `UNION ALL`, در انطباق با استاندارد گذاشتن. +- تبدیل نوع اضافه شده برای `UNION ALL`. نام مستعار مختلف مجاز است `SELECT` موقعیت در `UNION ALL`, در انطباق با استاندارد گذاشتن. - عبارات دلخواه در پشتیبانی `LIMIT BY` بند. قبلا, تنها ممکن بود به استفاده از ستون ناشی از `SELECT`. - یک شاخص از `MergeTree` جداول استفاده می شود که `IN` به یک تاپل عبارات از ستون کلید اصلی اعمال می شود. مثال: `WHERE (UserID, EventDate) IN ((123, '2000-01-01'), ...)` هشدار داده می شود - اضافه شدن `clickhouse-copier` ابزار برای کپی کردن بین خوشه ها و داده های تغییر شکل (بتا). @@ -865,9 +865,9 @@ toc_title: '2018' - اضافه شده `DROP TEMPORARY TABLE` و `EXISTS TEMPORARY TABLE` نمایش داده شد (ژانگ2014). - پشتیبانی از `SHOW CREATE TABLE` برای جداول موقت (ژانگ2014). - اضافه شدن `system_profile` پارامتر پیکربندی برای تنظیمات مورد استفاده توسط فرایندهای داخلی. -- پشتیبانی برای بارگذاری `object_id` به عنوان یک ویژگی در `MongoDB` واژهنامهها (پاول لیتویننکو). +- پشتیبانی از بارگذاری `object_id` به عنوان یک ویژگی در `MongoDB` واژهنامهها (پاول لیتویننکو). - خواندن `null` به عنوان مقدار پیش فرض هنگام بارگذاری داده ها برای یک فرهنگ لغت خارجی با `MongoDB` منبع (پاول لیتویننکو). -- خواندن `DateTime` ارزش در `Values` فرمت از برچسب زمان یونیکس بدون نقل قول تنها. +- خواندن `DateTime` ارزش ها در `Values` فرمت از برچسب زمان یونیکس بدون نقل قول تنها. - عدم موفقیت در پشتیبانی `remote` توابع جدول برای موارد زمانی که برخی از کپی از دست رفته جدول درخواست. - تنظیمات پیکربندی را می توان در خط فرمان باطل زمانی که شما اجرا `clickhouse-server`. مثال: `clickhouse-server -- --logger.level=information`. - اجرا `empty` تابع از یک `FixedString` استدلال: تابع بازده 1 اگر رشته شامل به طور کامل از بایت پوچ (ژانگ2014). @@ -894,7 +894,7 @@ toc_title: '2018' - این `MkDocs` ژنراتور مستندات استفاده شده است. - هنگامی که شما سعی می کنید یک ستون جدول را حذف کنید که `DEFAULT`/`MATERIALIZED` عبارات از ستون های دیگر بستگی دارد, یک استثنا پرتاب می شود (ژانگ2014). - اضافه شدن توانایی تجزیه یک خط خالی در فرمت های متن به عنوان شماره 0 برای `Float` انواع داده ها. این ویژگی قبلا در دسترس بود اما در نسخه 1.1.54342 از دست داده بود. -- `Enum` مقادیر را می توان در استفاده `min`, `max`, `sum` و برخی دیگر توابع. در این موارد با استفاده از آن مربوط به مقادیر عددی. این ویژگی قبلا در دسترس بود اما از دست رفته در انتشار 1.1.54337. +- `Enum` مقادیر را می توان در استفاده `min`, `max`, `sum` و برخی از توابع دیگر. در این موارد از مقادیر عددی مربوطه استفاده می شود. این ویژگی قبلا در دسترس بود اما در نسخه 1.1.54337 از دست داده بود. - اضافه شده `max_expanded_ast_elements` برای محدود کردن اندازه از اس تی پس از نام مستعار به صورت بازگشتی در حال گسترش است. #### رفع اشکال: {#bug-fixes-27} @@ -913,7 +913,7 @@ toc_title: '2018' - ثابت تصادف رخ داده است که در حال اجرا `CHECK` پرسوجو برای `Distributed` جداول اگر تمام خرده ریز محلی هستند (فکس.اطلاعات دقیق - ثابت رگرسیون عملکرد کمی با توابع است که با استفاده از عبارات منظم. - ثابت رگرسیون عملکرد در هنگام ایجاد مجموعه های چند بعدی از عبارات پیچیده است. -- ثابت یک اشکال است که می تواند اضافی `FORMAT` بخش به نظر می رسد در یک `.sql` فایل با ابرداده. +- رفع اشکال که می تواند اضافی شود `FORMAT` بخش به نظر می رسد در یک `.sql` فایل با ابرداده. - رفع اشکال که باعث `max_table_size_to_drop` محدود به درخواست در هنگام تلاش برای حذف یک `MATERIALIZED VIEW` با نگاهی به یک جدول به صراحت مشخص. - ناسازگاری ثابت با مشتریان قدیمی (مشتریان قدیمی گاهی اوقات داده ها را با `DateTime('timezone')` نوع, که درک نمی کنند). - رفع اشکال در هنگام خواندن `Nested` عناصر ستون سازه هایی که با استفاده از اضافه شد `ALTER` اما این برای پارتیشن های قدیمی خالی است, زمانی که شرایط را برای این ستون ها به نقل مکان کرد `PREWHERE`. @@ -931,25 +931,25 @@ toc_title: '2018' - حذف `strict_insert_defaults` تنظیمات. اگر شما با استفاده از این قابلیت, ارسال به `clickhouse-feedback@yandex-team.com`. - حذف `UnsortedMergeTree` موتور -### انتشار کلیک 1.1.54343, 2018-02-05 {#clickhouse-release-1-1-54343-2018-02-05} +### ClickHouse انتشار 1.1.54343, 2018-02-05 {#clickhouse-release-1-1-54343-2018-02-05} - اضافه شدن پشتیبانی از ماکرو برای تعریف نام خوشه در نمایش داده شد ددل توزیع و سازنده جداول توزیع شده است: `CREATE TABLE distr ON CLUSTER '{cluster}' (...) ENGINE = Distributed('{cluster}', 'db', 'table')`. - در حال حاضر نمایش داده شد مانند `SELECT ... FROM table WHERE expr IN (subquery)` با استفاده از پردازش `table` نمایه. - پردازش تکراری بهبود یافته در هنگام قرار دادن به جداول تکرار, به طوری که دیگر کم کردن سرعت اجرای صف تکرار. -### انتشار کلیک 1.1.54342, 2018-01-22 {#clickhouse-release-1-1-54342-2018-01-22} +### ClickHouse انتشار 1.1.54342, 2018-01-22 {#clickhouse-release-1-1-54342-2018-01-22} این نسخه شامل رفع اشکال برای نسخه قبلی 1.1.54337: - ثابت رگرسیون در 1.1.54337: اگر کاربر به طور پیش فرض دسترسی خوانده است, سپس سرور حاضر به راه اندازی با پیام `Cannot create database in readonly mode`. - ثابت رگرسیون در 1.1.54337: در سیستم های با سیستم, سیاهههای مربوط همیشه به سای لاگ صرف نظر از پیکربندی نوشته شده; اسکریپت دیده بان هنوز هم با استفاده از اینیت.د - ثابت رگرسیون در 1.1.54337: پیکربندی پیش فرض اشتباه در تصویر کارگر بارانداز. -- ثابت nondeterministic رفتار graphitemergetree (شما می توانید آن را در ورود به سیستم پیام `Data after merge is not byte-identical to the data on another replicas`). +- ثابت nondeterministic رفتار GraphiteMergeTree (شما می توانید آن را در ورود به سیستم پیام `Data after merge is not byte-identical to the data on another replicas`). - رفع اشکال که ممکن است منجر به ادغام متناقض پس از بهینه سازی پرس و جو به تکرار جداول (شما ممکن است در پیام ورود به سیستم را ببینید `Part ... intersects the previous part`). - جداول بافر در حال حاضر به درستی کار زمانی که ستون محقق در جدول مقصد وجود دارد (توسط ژانگ2014). - رفع اشکال در اجرای پوچ. -### انتشار کلیک 1.1.54337, 2018-01-18 {#clickhouse-release-1-1-54337-2018-01-18} +### ClickHouse انتشار 1.1.54337, 2018-01-18 {#clickhouse-release-1-1-54337-2018-01-18} #### ویژگی های جدید: {#new-features-17} @@ -961,7 +961,7 @@ toc_title: '2018' - اضافه شدن `clickhouse format` ابزار برای قالب بندی نمایش داده شد. - اضافه شدن `format_schema_path` configuration parameter (Marek Vavruşa). It is used for specifying a schema in `Cap'n Proto` قالب. فایل های طرح را می توان تنها در دایرکتوری مشخص شده واقع شده است. - اضافه شدن پشتیبانی برای تعویض پیکربندی (`incl` و `conf.d`) برای پیکربندی لغت نامه ها و مدل های خارجی (پاول یاکونین). -- اضافه شدن یک ستون با اسناد و مدارک برای `system.settings` جدول (Kirill Shvakov). +- اضافه شدن یک ستون با اسناد و مدارک برای `system.settings` جدول (کریل شواکوف). - اضافه شدن `system.parts_columns` جدول با اطلاعات در مورد اندازه ستون در هر بخش داده ها از `MergeTree` میز - اضافه شدن `system.models` جدول با اطلاعات در مورد لود `CatBoost` مدل های یادگیری ماشین. - اضافه شدن `mysql` و `odbc` عملکرد جدول و متناظر `MySQL` و `ODBC` موتورهای جدول برای دسترسی به پایگاه داده از راه دور. این قابلیت در مرحله بتا است. @@ -975,7 +975,7 @@ toc_title: '2018' - اضافه شدن `intExp3` و `intExp4` توابع. - اضافه شدن `sumKahan` تابع جمع. - اضافه شده به \* شماره\* توابع پرنده, جایی که \* شماره \* یک نوع عددی است. -- اضافه شدن پشتیبانی برای `WITH` جملات برای `INSERT SELECT` پرس و جو (نویسنده: ژانگ2014). +- اضافه شدن پشتیبانی برای `WITH` بند برای `INSERT SELECT` پرس و جو (نویسنده: ژانگ2014). - تنظیمات اضافه شده: `http_connection_timeout`, `http_send_timeout`, `http_receive_timeout`. به خصوص این تنظیمات برای دانلود قطعات داده ها برای تکرار استفاده می شود. تغییر این تنظیمات اجازه می دهد تا برای عدم موفقیت سریع تر اگر شبکه غیرمنتظره است. - اضافه شدن پشتیبانی برای `ALTER` برای جداول نوع `Null` هشدار داده می شود - این `reinterpretAsString` تابع برای تمام انواع داده ها که به روشنی در حافظه ذخیره می شود گسترش یافته است. @@ -993,7 +993,7 @@ toc_title: '2018' #### رفع اشکال: {#bug-fixes-28} -- ثابت data deduplication را پس از اجرا `DROP` یا `DETACH PARTITION` پرس و جو. در نسخه های قبلی, حذف یک پارتیشن و قرار دادن داده های مشابه دوباره کار نمی کند چرا بلوک قرار داده تکراری در نظر گرفته شد. +- تکرار داده های ثابت پس از اجرای یک `DROP` یا `DETACH PARTITION` پرس و جو. در نسخه های قبلی, حذف یک پارتیشن و قرار دادن داده های مشابه دوباره کار نمی کند چرا بلوک قرار داده تکراری در نظر گرفته شد. - رفع اشکال که می تواند به تفسیر نادرست از منجر شود `WHERE` بند برای `CREATE MATERIALIZED VIEW` نمایش داده شد با `POPULATE` . - رفع اشکال در استفاده از `root_path` پارامتر در `zookeeper_servers` پیکربندی. - نتایج غیر منتظره ثابت از عبور از `Date` نشانوند به `toStartOfDay` . @@ -1006,12 +1006,12 @@ toc_title: '2018' - این `extractAll` تابع در حال حاضر مسابقات خالی پشتیبانی می کند. - ثابت خطا که استفاده از مسدود `libressl` به جای `openssl` . - ثابت `CREATE TABLE AS SELECT` پرس و جو از جداول موقت. -- ثابت غیر atomicity از به روز رسانی تکرار صف. این می تواند منجر به کپی بودن از همگام سازی تا سرور ری استارت. +- ثابت غیر اتمی به روز رسانی صف تکرار. این می تواند منجر به کپی بودن خارج از سنکرون تا سرور راه اندازی مجدد. - سرریز ممکن ثابت در `gcd` , `lcm` و `modulo` (`%` اپراتور) (ماکس اسکروخد). - `-preprocessed` فایل ها در حال حاضر پس از تغییر ایجاد شده است `umask` (`umask` را می توان در پیکربندی تغییر). - رفع اشکال در چک پس زمینه از قطعات (`MergeTreePartChecker` )هنگام استفاده از یک کلید پارتیشن سفارشی . - تجزیه ثابت از تاپل (ارزش های `Tuple` نوع داده) در فرمت های متن. -- پیام های خطا بهبود یافته در مورد انواع ناسازگار منتقل شده به `multiIf` , `array` و برخی دیگر توابع. +- پیام های خطا بهبود یافته در مورد انواع ناسازگار منتقل شده به `multiIf` , `array` و برخی از توابع دیگر. - پشتیبانی دوباره طراحی شده برای `Nullable` انواع. اشکالات ثابت که ممکن است به یک تصادف سرور منجر شود. ثابت تقریبا تمام اشکالات دیگر مربوط به `NULL` پشتیبانی: نادرست نوع تبدیل در وارد کردن را انتخاب کنید کافی برای حمایت از Nullable در داشتن و PREWHERE, `join_use_nulls` حالت, انواع قابل ابطال به عنوان استدلال `OR` اپراتور و غیره - اشکالات مختلف ثابت مربوط به معانی داخلی انواع داده ها. نمونه: جمع غیر ضروری از `Enum` فیلدهای تایپ شده `SummingMergeTree` ; تراز دلخواه `Enum` انواع در `Pretty` فرمت, و غیره. - چک سختگیرانه تر برای ترکیب مجاز از ستون کامپوزیت. @@ -1060,4 +1060,4 @@ toc_title: '2018' - هنگام انجام یک به روز رسانی نورد در یک خوشه, در نقطه ای که برخی از کپی در حال اجرا هستند نسخه های قدیمی از تاتر و برخی در حال اجرا هستند نسخه جدید, تکرار است به طور موقت متوقف و پیام `unknown parameter 'shard'` به نظر می رسد در ورود به سیستم. تکرار ادامه خواهد داد پس از همه کپی از خوشه به روز می شوند. - اگر نسخه های مختلف از تاتر در حال اجرا بر روی سرورهای خوشه, ممکن است که نمایش داده شد توزیع با استفاده از توابع زیر نتایج نادرست داشته باشد: `varSamp`, `varPop`, `stddevSamp`, `stddevPop`, `covarSamp`, `covarPop`, `corr`. شما باید تمام گره های خوشه ای به روز رسانی. -## [تغییرات برای 2017](https://github.com/ClickHouse/ClickHouse/blob/master/docs/en/changelog/2017.md) {#changelog-for-2017} +## [تغییرات برای 2017](./2017.md#clickhouse-release-1-1-54327-2017-12-21) {#changelog-for-2017} diff --git a/docs/fa/whats-new/changelog/2019.md b/docs/fa/whats-new/changelog/2019.md index 2039246a0e4..2cf0cfce7b3 100644 --- a/docs/fa/whats-new/changelog/2019.md +++ b/docs/fa/whats-new/changelog/2019.md @@ -1,20 +1,20 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 77 toc_title: '2019' --- -## انتشار کلیک و19. 17 {#clickhouse-release-v19-17} +## انتشار کلیک 19.17 {#clickhouse-release-v19-17} -### ClickHouse انتشار V19.17.6.36, 2019-12-27 {#clickhouse-release-v19-17-6-36-2019-12-27} +### ClickHouse انتشار 19.17.6.36, 2019-12-27 {#clickhouse-release-v19-17-6-36-2019-12-27} #### رفع اشکال {#bug-fix} - سرریز بافر بالقوه ثابت در حالت فشرده خارج. کاربر مخرب می تواند داده های فشرده ساخته شده است که می تواند باعث به عنوان خوانده شده پس از بافر منتقل می کند. این موضوع توسط الدار زیتوف از تیم امنیت اطلاعات یاندکس یافت شد. [\#8404](https://github.com/ClickHouse/ClickHouse/pull/8404) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - ثابت سقوط سرور ممکن است (`std::terminate`) هنگامی که سرور نمی تواند ارسال و یا ارسال داده ها در فرمت جسون یا میلی لیتر با ارزش از نوع داده رشته (که نیاز به اعتبار سنجی-8) و یا زمانی که فشرده سازی داده ها نتیجه با الگوریتم بروتلی و یا در برخی موارد نادر دیگر. [\#8384](https://github.com/ClickHouse/ClickHouse/pull/8384) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- ثابت لغت نامه با منبع از یک clickhouse `VIEW` در حال حاضر خواندن چنین واژهنامهها خطا ایجاد نمی کند `There is no query`. [\#8351](https://github.com/ClickHouse/ClickHouse/pull/8351) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) -- ثابت چک کردن اگر یک مشتری میزبان مجاز است با host\_regexp مشخص شده در کاربران.. [\#8241](https://github.com/ClickHouse/ClickHouse/pull/8241), [\#8342](https://github.com/ClickHouse/ClickHouse/pull/8342) ([ویتالی بارانو](https://github.com/vitlibar)) +- لغت نامه های ثابت با منبع از یک کلیک `VIEW` در حال حاضر خواندن چنین واژهنامهها خطا ایجاد نمی کند `There is no query`. [\#8351](https://github.com/ClickHouse/ClickHouse/pull/8351) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) +- چک کردن ثابت اگر یک میزبان مشتری توسط \_شخصی میزبان مشخص شده در کاربران مجاز باشد.. [\#8241](https://github.com/ClickHouse/ClickHouse/pull/8241), [\#8342](https://github.com/ClickHouse/ClickHouse/pull/8342) ([ویتالی بارانو](https://github.com/vitlibar)) - `RENAME TABLE` برای یک جدول توزیع در حال حاضر تغییر نام پوشه حاوی داده های درج شده قبل از ارسال به خرده ریز. این رفع یک موضوع را با تغییر نام های پی در پی `tableA->tableB`, `tableC->tableA`. [\#8306](https://github.com/ClickHouse/ClickHouse/pull/8306) ([تاولوبیکس](https://github.com/tavplubix)) - `range_hashed` واژهنامهها خارجی ایجاد شده توسط دی ال نمایش داده شد در حال حاضر اجازه می دهد محدوده از انواع عددی دلخواه. [\#8275](https://github.com/ClickHouse/ClickHouse/pull/8275) ([الساپین](https://github.com/alesapin)) - ثابت `INSERT INTO table SELECT ... FROM mysql(...)` تابع جدول. [\#8234](https://github.com/ClickHouse/ClickHouse/pull/8234) ([تاولوبیکس](https://github.com/tavplubix)) @@ -25,9 +25,9 @@ toc_title: '2019' - ثابت `DROP DICTIONARY IF EXISTS db.dict` در حال حاضر استثنا پرتاب نمی کند اگر `db` وجود نداره [\#8185](https://github.com/ClickHouse/ClickHouse/pull/8185) ([ویتالی بارانو](https://github.com/vitlibar)) - اگر یک جدول به دلیل سقوط سرور به طور کامل کاهش یافته است, سرور سعی خواهد کرد برای بازگرداندن و بارگذاری [\#8176](https://github.com/ClickHouse/ClickHouse/pull/8176) ([تاولوبیکس](https://github.com/tavplubix)) - ثابت پرس و جو تعداد بی اهمیت برای یک جدول توزیع اگر بیش از دو میز محلی سفال وجود دارد. [\#8164](https://github.com/ClickHouse/ClickHouse/pull/8164) ([小路](https://github.com/nicelulu)) -- اشکال ثابت که منجر به یک مسابقه داده در db::blockstreamprofileinfo::calculaterowsbeforelimit() [\#8143](https://github.com/ClickHouse/ClickHouse/pull/8143) ([الکساندر کازاکوف](https://github.com/Akazz)) +- اشکال ثابت که منجر به یک مسابقه داده در DB::BlockStreamProfileInfo::calculateRowsBeforeLimit() [\#8143](https://github.com/ClickHouse/ClickHouse/pull/8143) ([الکساندر کازاکوف](https://github.com/Akazz)) - ثابت `ALTER table MOVE part` اعدام بلافاصله پس از ادغام بخش مشخص, که می تواند باعث حرکت بخشی که بخش مشخص شده به هم ادغام شدند. در حال حاضر به درستی حرکت می کند بخش مشخص شده است. [\#8104](https://github.com/ClickHouse/ClickHouse/pull/8104) ([ولادیمیر چبوتراف](https://github.com/excitoon)) -- عبارات برای لغت نامه را می توان به عنوان رشته در حال حاضر مشخص شده است. این بسیار مفید است برای محاسبه ویژگی های در حالی که استخراج داده ها از غیر clickhouse منابع به دلیل آن اجازه می دهد تا به استفاده از غیر clickhouse نحو برای آن دسته از عبارات. [\#8098](https://github.com/ClickHouse/ClickHouse/pull/8098) ([الساپین](https://github.com/alesapin)) +- عبارات برای لغت نامه را می توان به عنوان رشته در حال حاضر مشخص شده است. این بسیار مفید است برای محاسبه ویژگی های در حالی که استخراج داده ها از غیر ClickHouse منابع به دلیل آن اجازه می دهد تا به استفاده از غیر ClickHouse نحو برای آن دسته از عبارات. [\#8098](https://github.com/ClickHouse/ClickHouse/pull/8098) ([الساپین](https://github.com/alesapin)) - ثابت یک مسابقه بسیار نادر در `clickhouse-copier` به دلیل سرریز در زکسید. [\#8088](https://github.com/ClickHouse/ClickHouse/pull/8088) ([هشدار داده می شود](https://github.com/dingxiangfei2009)) - رفع اشکال زمانی که پس از پرس و جو شکست خورده (با توجه به “Too many simultaneous queries” به عنوان مثال) این اطلاعات جداول خارجی را نمی خواند و درخواست بعدی این اطلاعات را به عنوان ابتدای پرس و جو بعدی تفسیر می کند که باعث خطا می شود `Unknown packet from client`. [\#8084](https://github.com/ClickHouse/ClickHouse/pull/8084) ([ازات خوژین](https://github.com/azat)) @@ -42,7 +42,7 @@ toc_title: '2019' - در حال حاضر یک استثنا خواهد شد در صورت استفاده با روابط در کنار محدودیت های پرتاب. و در حال حاضر امکان استفاده از بالا با محدودیت توسط. [\#7637](https://github.com/ClickHouse/ClickHouse/pull/7637) ([نیکیتا میخایلو](https://github.com/nikitamikhaylov)) - رفع بارگذاری مجدد فرهنگ لغت در صورتی که `invalidate_query` که متوقف به روز رسانی و برخی از استثنا در به روز رسانی قبلی تلاش می کند. [\#8029](https://github.com/ClickHouse/ClickHouse/pull/8029) ([الساپین](https://github.com/alesapin)) -### ClickHouse انتشار V19.17.4.11, 2019-11-22 {#clickhouse-release-v19-17-4-11-2019-11-22} +### ClickHouse انتشار 19.17.4.11, 2019-11-22 {#clickhouse-release-v19-17-4-11-2019-11-22} #### تغییر ناسازگار به عقب {#backward-incompatible-change} @@ -114,8 +114,8 @@ toc_title: '2019' #### ساخت/تست / بهبود بسته بندی {#buildtestingpackaging-improvement} -- اضافه کردن پشتیبانی برای متقابل کامپایل به معماری پردازنده عاشق64. refactor packager اسکریپت. [\#7370](https://github.com/ClickHouse/ClickHouse/pull/7370) [\#7539](https://github.com/ClickHouse/ClickHouse/pull/7539) ([ایوان](https://github.com/abyss7)) -- باز کردن داروین-x86\_64 و لینوکس-aarch64 toolchains به نصب docker دوره زمانی که ساختمان بسته [\#7534](https://github.com/ClickHouse/ClickHouse/pull/7534) ([ایوان](https://github.com/abyss7)) +- اضافه کردن پشتیبانی برای متقابل کامپایل به معماری پردازنده عاشق64. Refactor packager اسکریپت. [\#7370](https://github.com/ClickHouse/ClickHouse/pull/7370) [\#7539](https://github.com/ClickHouse/ClickHouse/pull/7539) ([ایوان](https://github.com/abyss7)) +- باز کردن داروین-x86\_64 و لینوکس-aarch64 toolchains به نصب Docker دوره زمانی که ساختمان بسته [\#7534](https://github.com/ClickHouse/ClickHouse/pull/7534) ([ایوان](https://github.com/abyss7)) - به روز رسانی تصویر کارگر بارانداز برای باینری بسته بندی [\#7474](https://github.com/ClickHouse/ClickHouse/pull/7474) ([ایوان](https://github.com/abyss7)) - خطاهای کامپایل ثابت در مکینتاش کاتالینا [\#7585](https://github.com/ClickHouse/ClickHouse/pull/7585) ([ارنست پلتایف](https://github.com/ernestp)) - برخی از فاکتورگیری مجدد در منطق تجزیه و تحلیل پرس و جو: تقسیم کلاس پیچیده را به چند ساده. [\#7454](https://github.com/ClickHouse/ClickHouse/pull/7454) ([زویکوف](https://github.com/4ertus2)) @@ -126,15 +126,15 @@ toc_title: '2019' #### غیره {#other} -- اضافه شده antlr4 گرامر برای clickhouse sql گویش [\#7595](https://github.com/ClickHouse/ClickHouse/issues/7595) [\#7596](https://github.com/ClickHouse/ClickHouse/pull/7596) ([الکسی میلویدو](https://github.com/alexey-milovidov)) +- اضافه شده ANTLR4 گرامر برای ClickHouse SQL گویش [\#7595](https://github.com/ClickHouse/ClickHouse/issues/7595) [\#7596](https://github.com/ClickHouse/ClickHouse/pull/7596) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -## انتشار کلیک و19. 16 {#clickhouse-release-v19-16} +## انتشار کلیک 19.16 {#clickhouse-release-v19-16} -#### انتشار کلیک و19.16. 14. 65, 2020-03-25 {#clickhouse-release-v19-16-14-65-2020-03-25} +#### انتشار کلیک 19.16.14.65, 2020-03-25 {#clickhouse-release-v19-16-14-65-2020-03-25} - رفع اشکال در محاسبات باریکش از عملیات منطقی سه تایی در استدلال های متعدد (بیش از 10). [\#8718](https://github.com/ClickHouse/ClickHouse/pull/8718) ([الکساندر کازاکوف](https://github.com/Akazz)) این bugfix شد backported به نسخه 19.16 توسط یک درخواست ویژه از Altinity. -#### Clickhouse انتشار V19.16.14.65 و 2020-03-05 {#clickhouse-release-v19-16-14-65-2020-03-05} +#### انتشار کلیک 19.16.14.65, 2020-03-05 {#clickhouse-release-v19-16-14-65-2020-03-05} - رفع ناسازگاری خرده فروشی توزیع با نسخه های قدیمی تر کانال. رفع [\#7851](https://github.com/ClickHouse/ClickHouse/issues/7851) [(تبلوبیکس)](https://github.com/tavplubix) @@ -154,11 +154,11 @@ toc_title: '2019' - افزودن `deduplicate_blocks_in_dependent_materialized_views` گزینه ای برای کنترل رفتار درج ژولیده به جداول با نمایش محقق. این ویژگی جدید توسط یک درخواست ویژه از التیت به نسخه رفع اشکال اضافه شد. [\#9070](https://github.com/ClickHouse/ClickHouse/pull/9070) [.)](https://github.com/urykhy) -### ClickHouse انتشار V19.16.2.2, 2019-10-30 {#clickhouse-release-v19-16-2-2-2019-10-30} +### انتشار کلیک خانه 19.16.2.2, 2019-10-30 {#clickhouse-release-v19-16-2-2-2019-10-30} #### تغییر ناسازگار به عقب {#backward-incompatible-change-1} -- اضافه کردن گم شده arity اعتبار برای تعداد/counif. +- اضافه کردن گم شده arity اعتبار برای تعداد/counIf. [\#7095](https://github.com/ClickHouse/ClickHouse/issues/7095) [\#7298](https://github.com/ClickHouse/ClickHouse/pull/7298) ([ولادیمیر](https://github.com/Vdimir)) - حذف میراث `asterisk_left_columns_only` تنظیم (به طور پیش فرض غیر فعال شد). @@ -170,7 +170,7 @@ toc_title: '2019' #### ویژگی جدید {#new-feature-2} -- معرفی uniqcombined64() برای محاسبه cardinality بیشتر از uint\_max. +- معرفی uniqCombined64() برای محاسبه cardinality بیشتر از UINT\_MAX. [\#7213](https://github.com/ClickHouse/ClickHouse/pull/7213), [\#7222](https://github.com/ClickHouse/ClickHouse/pull/7222) ([ازات خوژین](https://github.com/azat)) @@ -200,7 +200,7 @@ toc_title: '2019' واسیلیف](https://github.com/nikvas0)) - اضافه کردن توابع جمع گروهبیت مپند,- یا, - صخره نوردی برای ستون بیت مپ. [\#7109](https://github.com/ClickHouse/ClickHouse/pull/7109) ([ژیچنگ یو](https://github.com/yuzhichang)) -- اضافه کردن مجموع عملکرد combinators -ornull و ordefault که بازگشت تهی +- اضافه کردن مجموع عملکرد combinators -OrNull و OrDefault که بازگشت تهی یا مقادیر پیش فرض زمانی که هیچ چیز به جمع وجود دارد. [\#7331](https://github.com/ClickHouse/ClickHouse/pull/7331) ([هکز](https://github.com/hczhcz)) @@ -217,7 +217,7 @@ toc_title: '2019' پوپوف](https://github.com/CurtizJ)) - پلاگین احراز هویت ماریادب غیر فعال, که بستگی به فایل های خارج از پروژه. [\#7140](https://github.com/ClickHouse/ClickHouse/pull/7140) ([یوری - Baranov](https://github.com/yurriy)) + بارانوف](https://github.com/yurriy)) - رفع استثنا `Cannot convert column ... because it is constant but values of constants are different in source and result` که به ندرت می تواند رخ دهد زمانی که توابع `now()`, `today()`, `yesterday()`, `randConstant()` استفاده می شود. [\#7156](https://github.com/ClickHouse/ClickHouse/pull/7156) ([نیکولای @@ -235,11 +235,11 @@ toc_title: '2019' Chebotarev](https://github.com/excitoon)) - رفع سریع برای حل و فصل سقوط در جدول نمایش زنده و دوباره قادر می سازد تمام تست نمایش زنده. [\#7201](https://github.com/ClickHouse/ClickHouse/pull/7201) - ([vzakaznikov](https://github.com/vzakaznikov)) + ([وزکازنیکوف](https://github.com/vzakaznikov)) - مرتب کردن مقادیر صفر به درستی در شاخص دقیقه/حداکثر از قطعات ادغام. [\#7234](https://github.com/ClickHouse/ClickHouse/pull/7234) ([الکساندر کوزمنکوف](https://github.com/akuzm)) -- قرار نیست ستون مجازی به .sql ابرداده هنگامی که جدول ایجاد شده است `CREATE TABLE AS`. +- هنوز ستون مجازی به قرار داده نشده .ابرداده گذاشتن زمانی که جدول به عنوان ایجاد شده است `CREATE TABLE AS`. [\#7183](https://github.com/ClickHouse/ClickHouse/pull/7183) ([ایوان](https://github.com/abyss7)) - رفع گسل تقسیم بندی در `ATTACH PART` پرس و جو. [\#7185](https://github.com/ClickHouse/ClickHouse/pull/7185) @@ -247,9 +247,9 @@ toc_title: '2019' - رفع نتیجه اشتباه برای برخی از نمایش داده شد داده شده توسط بهینه سازی خالی در کارخانه های فرعی و خالی INNER/RIGHT JOIN. [\#7284](https://github.com/ClickHouse/ClickHouse/pull/7284) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) -- رفع addresssanitizer خطا در نمایش زنده getheader روش (). +- رفع AddressSanitizer خطا در نمایش زنده getHeader روش (). [\#7271](https://github.com/ClickHouse/ClickHouse/pull/7271) - ([vzakaznikov](https://github.com/vzakaznikov)) + ([وزکازنیکوف](https://github.com/vzakaznikov)) #### بهبود {#improvement-1} @@ -285,10 +285,10 @@ toc_title: '2019' - پشتیبانی از نابرابری های از دست رفته برای عضویت. این ممکن است برای پیوستن به نوع کمتر یا برابر و سخت انواع بیشتر و کمتر برای ستون اسوف در نحو. [\#7282](https://github.com/ClickHouse/ClickHouse/pull/7282) ([علم هنر - Zuikov](https://github.com/4ertus2)) + زویکوف](https://github.com/4ertus2)) - بهینه سازی بخشی ادغام اضافه کردن. [\#7070](https://github.com/ClickHouse/ClickHouse/pull/7070) ([زویکوف](https://github.com/4ertus2)) -- آیا استفاده از بیش از 98k حافظه در uniqcombined توابع. +- آیا استفاده از بیش از 98K حافظه در uniqCombined توابع. [\#7236](https://github.com/ClickHouse/ClickHouse/pull/7236), [\#7270](https://github.com/ClickHouse/ClickHouse/pull/7270) ([ازات خوژین](https://github.com/azat)) @@ -311,13 +311,13 @@ toc_title: '2019' - غیر فعال کردن برخی از موارد برای متقابل تلفیقی به سیستم عامل مک. [\#7101](https://github.com/ClickHouse/ClickHouse/pull/7101) ([ایوان](https://github.com/abyss7)) -- اضافه کردن گم شده ارتباط با pocoxml برای clickhouse\_common\_io. +- اضافه کردن گم شده ارتباط با PocoXML برای clickhouse\_common\_io. [\#7200](https://github.com/ClickHouse/ClickHouse/pull/7200) ([ازات خوژین](https://github.com/azat)) -- قبول متعدد تست فیلتر استدلال در clickhouse آزمون. +- قبول استدلال فیلتر تست های متعدد در فاحشه خانه تست. [\#7226](https://github.com/ClickHouse/ClickHouse/pull/7226) ([الکساندر کوزمنکوف](https://github.com/akuzm)) -- فعال musl و jemalloc برای arm. [\#7300](https://github.com/ClickHouse/ClickHouse/pull/7300) +- فعال musl و jemalloc برای ARM. [\#7300](https://github.com/ClickHouse/ClickHouse/pull/7300) ([ایموس پرنده](https://github.com/amosbird)) - اضافه شده `--client-option` پارامتر به `clickhouse-test` به تصویب پارامترهای اضافی به مشتری. [\#7277](https://github.com/ClickHouse/ClickHouse/pull/7277) ([نیکولای @@ -326,7 +326,7 @@ toc_title: '2019' [\#7103](https://github.com/ClickHouse/ClickHouse/pull/7103) ([فیلیمونف](https://github.com/filimonov)) - رفع خطاهای شناسایی شده توسط پوس. [\#7153](https://github.com/ClickHouse/ClickHouse/pull/7153) ([علم هنر - Zuikov](https://github.com/4ertus2)) + زویکوف](https://github.com/4ertus2)) - رفع ساخت برای داروین. [\#7149](https://github.com/ClickHouse/ClickHouse/pull/7149) ([ایوان](https://github.com/abyss7)) - glibc 2.29 سازگاری. [\#7142](https://github.com/ClickHouse/ClickHouse/pull/7142) ([ایموس @@ -348,7 +348,7 @@ toc_title: '2019' کازاکوف](https://github.com/Akazz)) - اضافه کردن ساخت با حفظ به سی. [\#7066](https://github.com/ClickHouse/ClickHouse/pull/7066) ([الکساندر کوزمنکوف](https://github.com/akuzm)) -- اجتناب از استفاده از آماده سازی نشده ارزش در metricstransmitter. +- اجتناب از استفاده از ارزش های بی قید و شرط در مترسستر. [\#7158](https://github.com/ClickHouse/ClickHouse/pull/7158) ([ازات خوژین](https://github.com/azat)) - رفع برخی از مشکلات در زمینه های پیدا شده توسط حفظ کننده. @@ -365,7 +365,7 @@ toc_title: '2019' تاریخ ساعت و امید. این رفع [\#7245](https://github.com/ClickHouse/ClickHouse/issues/7245) [\#7252](https://github.com/ClickHouse/ClickHouse/pull/7252) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- رفع threadsanitizer اطلاعات مسابقه خطا در نمایش زنده در هنگام دسترسی به no\_users\_thread متغیر است. +- رفع ThreadSanitizer اطلاعات مسابقه خطا در نمایش زنده در هنگام دسترسی به no\_users\_thread متغیر است. [\#7353](https://github.com/ClickHouse/ClickHouse/pull/7353) ([وزکازنیکوف](https://github.com/vzakaznikov)) - خلاص شدن از شر نمادها مالوک در لیبکمون @@ -399,7 +399,7 @@ toc_title: '2019' کوزمنکوف](https://github.com/akuzm)) - رفع گیرنده در صورتی که ریشه پارامتر خالی نیست. [\#7374](https://github.com/ClickHouse/ClickHouse/pull/7374) ([میخیل کوروتف](https://github.com/millb)) -- حذف برخی از کپی و چسباندن (temporaryfile و temporaryfilestream) +- حذف برخی از کپی و چسباندن (TemporaryFile و TemporaryFileStream) [\#7166](https://github.com/ClickHouse/ClickHouse/pull/7166) ([علم هنر زویکوف](https://github.com/4ertus2)) - بهبود خوانایی کد کمی (`MergeTreeData::getActiveContainingPart`). @@ -417,11 +417,11 @@ toc_title: '2019' #### رفع اشکال {#bug-fix-3} -- اضافه شده دست زدن به sql\_tinyint و sql\_bigint و ثابت دست زدن به sql\_float منبع داده در انواع odbc پل. +- اضافه شده دست زدن به SQL\_TINYINT و SQL\_BIGINT و ثابت دست زدن به SQL\_FLOAT منبع داده در انواع ODBC پل. [\#7491](https://github.com/ClickHouse/ClickHouse/pull/7491) ([دنیس گلازاشف](https://github.com/traceon)) - مجاز به برخی از قطعات بر روی دیسک مقصد و یا حجم در پارتیشن حرکت. [\#7434](https://github.com/ClickHouse/ClickHouse/pull/7434) ([ولادیمیر چبوتراف](https://github.com/excitoon)) -- ثابت null-ارزش در nullable ستون از طریق odbc-پل. +- ثابت NULL-ارزش در nullable ستون از طریق ODBC-پل. [\#7402](https://github.com/ClickHouse/ClickHouse/pull/7402) ([واسیلی نمکو](https://github.com/Enmk)) - درج ثابت به گره غیر محلی توزیع شده با ستون محقق. [\#7377](https://github.com/ClickHouse/ClickHouse/pull/7377) ([ازات خوژین](https://github.com/azat)) @@ -513,14 +513,14 @@ toc_title: '2019' ## انتشار کلیک 19.14 {#clickhouse-release-19-14} -### انتشار کلیک کنیدهاوس 19.14.7.15, 2019-10-02 {#clickhouse-release-19-14-7-15-2019-10-02} +### ClickHouse انتشار 19.14.7.15, 2019-10-02 {#clickhouse-release-19-14-7-15-2019-10-02} #### رفع اشکال {#bug-fix-6} - این نسخه همچنین شامل تمام رفع اشکال از 19.11.12.69. - سازگاری ثابت برای نمایش داده شد توزیع بین 19.14 و نسخه های قبلی. این رفع [\#7068](https://github.com/ClickHouse/ClickHouse/issues/7068). [\#7069](https://github.com/ClickHouse/ClickHouse/pull/7069) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -### انتشار کلیک کنیدهاوس 19.14.6.12, 2019-09-19 {#clickhouse-release-19-14-6-12-2019-09-19} +### ClickHouse انتشار 19.14.6.12, 2019-09-19 {#clickhouse-release-19-14-6-12-2019-09-19} #### رفع اشکال {#bug-fix-7} @@ -540,13 +540,13 @@ toc_title: '2019' - `WITH TIES` تغییردهنده برای `LIMIT`. (ادامه [\#5069](https://github.com/ClickHouse/ClickHouse/issues/5069)) [\#6610](https://github.com/ClickHouse/ClickHouse/pull/6610) ([انتون پوپوف](https://github.com/CurtizJ)) - تجزیه unquoted `NULL` تحت اللفظی به عنوان پوچ (اگر تنظیم `format_csv_unquoted_null_literal_as_null=1`). مقداردهی اولیه زمینه های تهی با مقادیر پیش فرض اگر نوع داده ها از این زمینه است قابل ابطال نیست (اگر تنظیم `input_format_null_as_default=1`). [\#5990](https://github.com/ClickHouse/ClickHouse/issues/5990) [\#6055](https://github.com/ClickHouse/ClickHouse/pull/6055) ([تاولوبیکس](https://github.com/tavplubix)) - پشتیبانی از نویسه عام در مسیرهای توابع جدول `file` و `hdfs`. اگر مسیر شامل نویسه عام, جدول خوانده خواهد شد. مثال استفاده: `select * from hdfs('hdfs://hdfs1:9000/some_dir/another_dir/*/file{0..9}{0..9}')` و `select * from file('some_dir/{some_file,another_file,yet_another}.tsv', 'TSV', 'value UInt32')`. [\#6092](https://github.com/ClickHouse/ClickHouse/pull/6092) ([اولگا خوستیکوا](https://github.com/stavrolia)) -- جدید `system.metric_log` جدول است که فروشگاه ها ارزش `system.events` و `system.metrics` با فاصله زمانی مشخص شده است. [\#6363](https://github.com/ClickHouse/ClickHouse/issues/6363) [\#6467](https://github.com/ClickHouse/ClickHouse/pull/6467) ([نیکیتا میخایلو](https://github.com/nikitamikhaylov)) [\#6530](https://github.com/ClickHouse/ClickHouse/pull/6530) ([الکسی میلویدو](https://github.com/alexey-milovidov)) +- جدید `system.metric_log` جدول که ارزش ها را ذخیره می کند `system.events` و `system.metrics` با فاصله زمانی مشخص شده است. [\#6363](https://github.com/ClickHouse/ClickHouse/issues/6363) [\#6467](https://github.com/ClickHouse/ClickHouse/pull/6467) ([نیکیتا میخایلو](https://github.com/nikitamikhaylov)) [\#6530](https://github.com/ClickHouse/ClickHouse/pull/6530) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - اجازه به نوشتن سیاهههای مربوط به متن کلیک به `system.text_log` جدول [\#6037](https://github.com/ClickHouse/ClickHouse/issues/6037) [\#6103](https://github.com/ClickHouse/ClickHouse/pull/6103) ([نیکیتا میخایلو](https://github.com/nikitamikhaylov)) [\#6164](https://github.com/ClickHouse/ClickHouse/pull/6164) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - نمایش علامت خصوصی در اثری پشته (این است که از طریق تجزیه جداول نماد فایل های جن انجام). اضافه شدن اطلاعات در مورد فایل و شماره خط در اثری پشته اگر اطلاعات اشکال زدایی وجود دارد. نام نماد افزایش سرعت مراجعه با علامت نمایه سازی در حال حاضر در برنامه. اضافه شده توابع جدید گذاشتن برای درون گرایی: `demangle` و `addressToLine`. تابع تغییر نام داد `symbolizeAddress` به `addressToSymbol` برای ثبات. تابع `addressToSymbol` خواهد نام لت و پار به دلایل عملکرد بازگشت و شما باید به درخواست `demangle`. اضافه شدن تنظیمات `allow_introspection_functions` که به طور پیش فرض خاموش است. [\#6201](https://github.com/ClickHouse/ClickHouse/pull/6201) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - تابع جدول `values` (نام غیر حساس به حروف است). این اجازه می دهد تا از خواندن `VALUES` فهرست پیشنهادی در [\#5984](https://github.com/ClickHouse/ClickHouse/issues/5984). مثال: `SELECT * FROM VALUES('a UInt64, s String', (1, 'one'), (2, 'two'), (3, 'three'))`. [\#6217](https://github.com/ClickHouse/ClickHouse/issues/6217). [\#6209](https://github.com/ClickHouse/ClickHouse/pull/6209) ([دیماروب2000](https://github.com/dimarub2000)) - توانایی تغییر تنظیمات ذخیره سازی اضافه شده است. نحو: `ALTER TABLE
MODIFY SETTING = `. [\#6366](https://github.com/ClickHouse/ClickHouse/pull/6366) [\#6669](https://github.com/ClickHouse/ClickHouse/pull/6669) [\#6685](https://github.com/ClickHouse/ClickHouse/pull/6685) ([الساپین](https://github.com/alesapin)) - پشتیبانی از حذف قطعات جدا شده. نحو: `ALTER TABLE DROP DETACHED PART ''`. [\#6158](https://github.com/ClickHouse/ClickHouse/pull/6158) ([تاولوبیکس](https://github.com/tavplubix)) -- محدودیت های جدول. اجازه می دهد تا برای اضافه کردن محدودیت به تعریف جدول خواهد شد که در درج بررسی می شود. [\#5273](https://github.com/ClickHouse/ClickHouse/pull/5273) ([گلب novikov](https://github.com/NanoBjorn)) [\#6652](https://github.com/ClickHouse/ClickHouse/pull/6652) ([الکسی میلویدو](https://github.com/alexey-milovidov)) +- محدودیت های جدول. اجازه می دهد تا برای اضافه کردن محدودیت به تعریف جدول خواهد شد که در درج بررسی می شود. [\#5273](https://github.com/ClickHouse/ClickHouse/pull/5273) ([گلب نویکوف](https://github.com/NanoBjorn)) [\#6652](https://github.com/ClickHouse/ClickHouse/pull/6652) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - پشتیبانی برای نمایش ساختگی محقق. [\#6324](https://github.com/ClickHouse/ClickHouse/pull/6324) ([ایموس پرنده](https://github.com/amosbird)) - روشن کردن پیشفیلتر پرس و جو به طور پیش فرض برای نمونه هر موضوع اعدام پرس و جو یک بار در ثانیه. [\#6283](https://github.com/ClickHouse/ClickHouse/pull/6283) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - قالب ورودی `ORC`. [\#6454](https://github.com/ClickHouse/ClickHouse/pull/6454) [\#6703](https://github.com/ClickHouse/ClickHouse/pull/6703) ([اکونیک 90](https://github.com/akonyaev90)) @@ -564,14 +564,14 @@ toc_title: '2019' #### ویژگی تجربی {#experimental-feature-2} - فرمت داده ورودی و خروجی `Template`. این اجازه می دهد برای مشخص رشته فرمت های سفارشی برای ورودی و خروجی. [\#4354](https://github.com/ClickHouse/ClickHouse/issues/4354) [\#6727](https://github.com/ClickHouse/ClickHouse/pull/6727) ([تاولوبیکس](https://github.com/tavplubix)) -- اجرای `LIVE VIEW` جداول که در اصل در پیشنهاد شد [\#2898](https://github.com/ClickHouse/ClickHouse/pull/2898) در حال بارگذاری [\#3925](https://github.com/ClickHouse/ClickHouse/issues/3925) و سپس به روز شده در [\#5541](https://github.com/ClickHouse/ClickHouse/issues/5541). ببینید [\#5541](https://github.com/ClickHouse/ClickHouse/issues/5541) برای شرح مفصلی. [\#5541](https://github.com/ClickHouse/ClickHouse/issues/5541) ([وزکازنیکوف](https://github.com/vzakaznikov)) [\#6425](https://github.com/ClickHouse/ClickHouse/pull/6425) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) [\#6656](https://github.com/ClickHouse/ClickHouse/pull/6656) ([vzakaznikov](https://github.com/vzakaznikov)) توجه داشته باشید که `LIVE VIEW` ویژگی ممکن است در نسخه های بعدی حذف شده است. +- اجرای `LIVE VIEW` جداول که در اصل در پیشنهاد شد [\#2898](https://github.com/ClickHouse/ClickHouse/pull/2898) در حال بارگذاری [\#3925](https://github.com/ClickHouse/ClickHouse/issues/3925) و سپس به روز شده در [\#5541](https://github.com/ClickHouse/ClickHouse/issues/5541). ببینید [\#5541](https://github.com/ClickHouse/ClickHouse/issues/5541) برای شرح مفصلی. [\#5541](https://github.com/ClickHouse/ClickHouse/issues/5541) ([وزکازنیکوف](https://github.com/vzakaznikov)) [\#6425](https://github.com/ClickHouse/ClickHouse/pull/6425) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) [\#6656](https://github.com/ClickHouse/ClickHouse/pull/6656) ([وزکازنیکوف](https://github.com/vzakaznikov)) توجه داشته باشید که `LIVE VIEW` ویژگی ممکن است در نسخه های بعدی حذف شده است. #### رفع اشکال {#bug-fix-8} - این نسخه همچنین شامل رفع اشکال از 19.13 و 19.11. - رفع گسل تقسیم بندی زمانی که جدول شاخص جست و خیز و ادغام عمودی اتفاق می افتد. [\#6723](https://github.com/ClickHouse/ClickHouse/pull/6723) ([الساپین](https://github.com/alesapin)) - رفع ستون در هر ستون با پیش فرض ستون غیر بدیهی است. پیش از این در مورد نیروی تی ال ادغام با `OPTIMIZE ... FINAL` پرس و جو, ارزش منقضی شده توسط پیش فرض نوع به جای پیش فرض ستون مشخص شده توسط کاربر جایگزین شد. [\#6796](https://github.com/ClickHouse/ClickHouse/pull/6796) ([انتون پوپوف](https://github.com/CurtizJ)) -- رفع کافکا پیام های تکراری مشکل در سرور معمولی راه اندازی مجدد. [\#6597](https://github.com/ClickHouse/ClickHouse/pull/6597) ([ایوان](https://github.com/abyss7)) +- رفع مشکل پیام های کافکا تقلید در راه اندازی مجدد سرور طبیعی است. [\#6597](https://github.com/ClickHouse/ClickHouse/pull/6597) ([ایوان](https://github.com/abyss7)) - حلقه بی نهایت ثابت در هنگام خواندن پیام کافکا. مکث نکنید / مصرف کننده رزومه در اشتراک در همه-در غیر این صورت ممکن است به طور نامحدود در برخی از حالات متوقف. [\#6354](https://github.com/ClickHouse/ClickHouse/pull/6354) ([ایوان](https://github.com/abyss7)) - ثابت `Key expression contains comparison between inconvertible types` استثنا در `bitmapContains` تابع. [\#6136](https://github.com/ClickHouse/ClickHouse/issues/6136) [\#6146](https://github.com/ClickHouse/ClickHouse/issues/6146) [\#6156](https://github.com/ClickHouse/ClickHouse/pull/6156) ([دیماروب2000](https://github.com/dimarub2000)) - اصلاح سگو با فعال `optimize_skip_unused_shards` و از دست رفته کلید شاردینگ. [\#6384](https://github.com/ClickHouse/ClickHouse/pull/6384) ([انتون پوپوف](https://github.com/CurtizJ)) @@ -611,7 +611,7 @@ toc_title: '2019' - ثابت امکان معلق نمایش داده شد زمانی که سرور غیرمنتظره است و استخر موضوع جهانی در نزدیکی کامل می شود. این بالاتر شانس اتفاق می افتد در خوشه با تعداد زیادی از خرده ریز (صدها), به دلیل نمایش داده شد توزیع اختصاص یک موضوع در هر اتصال به هر سفال. مثلا, این موضوع ممکن است تکثیر اگر یک خوشه از 330 خرده ریز در حال پردازش 30 نمایش داده شد توزیع همزمان. این موضوع بر تمام نسخه های با شروع از 19.2. [\#6301](https://github.com/ClickHouse/ClickHouse/pull/6301) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - منطق ثابت `arrayEnumerateUniqRanked` تابع. [\#6423](https://github.com/ClickHouse/ClickHouse/pull/6423) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - ثابت شده است که جدول نماد رمز گشایی. [\#6603](https://github.com/ClickHouse/ClickHouse/pull/6603) ([ایموس پرنده](https://github.com/amosbird)) -- استثنا بی ربط ثابت در بازیگران `LowCardinality(Nullable)` to not-Nullable column in case if it doesn’t contain Nulls (e.g. in query like `SELECT CAST(CAST('Hello' AS LowCardinality(Nullable(String))) AS String)`. [\#6094](https://github.com/ClickHouse/ClickHouse/issues/6094) [\#6119](https://github.com/ClickHouse/ClickHouse/pull/6119) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) +- استثنا بی ربط ثابت در بازیگران `LowCardinality(Nullable)` to not-Nullable column in case if it doesn't contain Nulls (e.g. in query like `SELECT CAST(CAST('Hello' AS LowCardinality(Nullable(String))) AS String)`. [\#6094](https://github.com/ClickHouse/ClickHouse/issues/6094) [\#6119](https://github.com/ClickHouse/ClickHouse/pull/6119) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) - حذف نقل قول اضافی از توضیحات در `system.settings` جدول [\#6696](https://github.com/ClickHouse/ClickHouse/issues/6696) [\#6699](https://github.com/ClickHouse/ClickHouse/pull/6699) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - اجتناب از بن بست ممکن است در `TRUNCATE` از جدول تکرار. [\#6695](https://github.com/ClickHouse/ClickHouse/pull/6695) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - رفع خواندن به منظور مرتب سازی کلید. [\#6189](https://github.com/ClickHouse/ClickHouse/pull/6189) ([انتون پوپوف](https://github.com/CurtizJ)) @@ -668,7 +668,7 @@ toc_title: '2019' - حرکت بسیاری از می پیوندد تهیه منطق از `ExpressionAction/ExpressionAnalyzer` به `AnalyzedJoin`. [\#6785](https://github.com/ClickHouse/ClickHouse/pull/6785) ([زویکوف](https://github.com/4ertus2)) - رفع تسان [اخطار](https://clickhouse-test-reports.s3.yandex.net/6399/c1c1d1daa98e199e620766f1bd06a5921050a00d/functional_stateful_tests_(thread).html) ‘lock-order-inversion’. [\#6740](https://github.com/ClickHouse/ClickHouse/pull/6740) ([واسیلی نمکو](https://github.com/Enmk)) - پیام های اطلاعات بهتر در مورد عدم قابلیت های لینوکس. ورود به سیستم خطاهای کشنده با “fatal” سطح, که ساده تر خواهد شد برای پیدا کردن در `system.text_log`. [\#6441](https://github.com/ClickHouse/ClickHouse/pull/6441) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- هنگام فعال کردن اطلاعات موقت تخلیه به دیسک برای محدود کردن استفاده از حافظه در طول `GROUP BY`, `ORDER BY` فضای دیسک رایگان را بررسی نکرد. ثابت اضافه کردن یک محیط جدید `min_free_disk_space`, هنگامی که فضای دیسک رایگان کوچکتر و سپس حد, پرس و جو را متوقف خواهد کرد و پرتاب `ErrorCodes::NOT_ENOUGH_SPACE`. [\#6678](https://github.com/ClickHouse/ClickHouse/pull/6678) ([Weiqing زو](https://github.com/weiqxu)) [\#6691](https://github.com/ClickHouse/ClickHouse/pull/6691) ([الکسی میلویدو](https://github.com/alexey-milovidov)) +- هنگام فعال کردن اطلاعات موقت تخلیه به دیسک برای محدود کردن استفاده از حافظه در طول `GROUP BY`, `ORDER BY` فضای دیسک رایگان را بررسی نکرد. ثابت اضافه کردن یک محیط جدید `min_free_disk_space`, هنگامی که فضای دیسک رایگان کوچکتر و سپس حد, پرس و جو را متوقف خواهد کرد و پرتاب `ErrorCodes::NOT_ENOUGH_SPACE`. [\#6678](https://github.com/ClickHouse/ClickHouse/pull/6678) ([در حال بارگذاری](https://github.com/weiqxu)) [\#6691](https://github.com/ClickHouse/ClickHouse/pull/6691) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - رواندا بازگشتی حذف شده توسط موضوع. این باعث می شود هیچ حس, چرا که موضوعات بین نمایش داده شد مورد استفاده مجدد قرار. `SELECT` پرس و جو ممکن است یک قفل در یک موضوع, یک قفل از موضوع دیگر نگه دارید و خروج از موضوع اول. در همان زمان اولین موضوع را می توان با استفاده مجدد `DROP` پرس و جو. این به نادرست منجر شود “Attempt to acquire exclusive lock recursively” پیام [\#6771](https://github.com/ClickHouse/ClickHouse/pull/6771) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - شکافتن `ExpressionAnalyzer.appendJoin()`. تهیه یک مکان در `ExpressionAnalyzer` برای `MergeJoin`. [\#6524](https://github.com/ClickHouse/ClickHouse/pull/6524) ([زویکوف](https://github.com/4ertus2)) - اضافه شده `mysql_native_password` پلاگین احراز هویت به خروجی زیر سرور سازگاری. [\#6194](https://github.com/ClickHouse/ClickHouse/pull/6194) ([یوری بارانوف](https://github.com/yurriy)) @@ -683,7 +683,7 @@ toc_title: '2019' - حالا `clickhouse-obfuscator` پرونده در دسترس است `clickhouse-client` بسته در نسخه های قبلی در دسترس بود `clickhouse obfuscator` (با فضای خالی). [\#5816](https://github.com/ClickHouse/ClickHouse/issues/5816) [\#6609](https://github.com/ClickHouse/ClickHouse/pull/6609) ([دیماروب2000](https://github.com/dimarub2000)) - بن بست ثابت زمانی که ما حداقل دو نمایش داده شد که حداقل دو جدول در جهت های مختلف و پرس و جو دیگری که انجام عملیات دسیدل در یکی از جداول به عنوان خوانده شده. ثابت دیگر بن بست بسیار نادر است. [\#6764](https://github.com/ClickHouse/ClickHouse/pull/6764) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - اضافه شده `os_thread_ids` ستون به `system.processes` و `system.query_log` برای احتمالات اشکال زدایی بهتر. [\#6763](https://github.com/ClickHouse/ClickHouse/pull/6763) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- یک راه حل برای اشکالات پسوند پی اچ پی میسورند که زمانی رخ می دهد `sha256_password` به عنوان یک پلاگین احراز هویت پیش فرض (شرح داده شده در [\#6031](https://github.com/ClickHouse/ClickHouse/issues/6031)). [\#6113](https://github.com/ClickHouse/ClickHouse/pull/6113) ([یوری baranov](https://github.com/yurriy)) +- یک راه حل برای اشکالات پسوند پی اچ پی میسورند که زمانی رخ می دهد `sha256_password` به عنوان یک پلاگین احراز هویت پیش فرض (شرح داده شده در [\#6031](https://github.com/ClickHouse/ClickHouse/issues/6031)). [\#6113](https://github.com/ClickHouse/ClickHouse/pull/6113) ([یوری بارانوف](https://github.com/yurriy)) - حذف محل غیر ضروری با ستون ابطال تغییر. [\#6693](https://github.com/ClickHouse/ClickHouse/pull/6693) ([زویکوف](https://github.com/4ertus2)) - تنظیم مقدار پیشفرض `queue_max_wait_ms` به صفر, به دلیل ارزش فعلی (پنج ثانیه) می سازد هیچ حس. شرایط نادر وجود دارد که این تنظیمات هر گونه استفاده. تنظیمات اضافه شده `replace_running_query_max_wait_ms`, `kafka_max_wait_ms` و `connection_pool_max_wait_ms` برای ابهامزدایی. [\#6692](https://github.com/ClickHouse/ClickHouse/pull/6692) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - استخراج `SelectQueryExpressionAnalyzer` از `ExpressionAnalyzer`. نگه داشتن یکی از گذشته برای نمایش داده شد غیر را انتخاب کنید. [\#6499](https://github.com/ClickHouse/ClickHouse/pull/6499) ([زویکوف](https://github.com/4ertus2)) @@ -695,7 +695,7 @@ toc_title: '2019' #### بهبود عملکرد {#performance-improvement-2} - بهینه سازی نمایش داده شد با `ORDER BY expressions` بند, جایی که `expressions` پیشوند همزمان با مرتب سازی کلید در `MergeTree` میز این بهینه سازی توسط کنترل `optimize_read_in_order` تنظیمات. [\#6054](https://github.com/ClickHouse/ClickHouse/pull/6054) [\#6629](https://github.com/ClickHouse/ClickHouse/pull/6629) ([انتون پوپوف](https://github.com/CurtizJ)) -- اجازه می دهد به استفاده از موضوعات مختلف در بخش بارگذاری و حذف. [\#6372](https://github.com/ClickHouse/ClickHouse/issues/6372) [\#6074](https://github.com/ClickHouse/ClickHouse/issues/6074) [\#6438](https://github.com/ClickHouse/ClickHouse/pull/6438) ([الکسی میلویدو](https://github.com/alexey-milovidov)) +- اجازه استفاده از موضوعات متعدد در طول بارگذاری قطعات و حذف. [\#6372](https://github.com/ClickHouse/ClickHouse/issues/6372) [\#6074](https://github.com/ClickHouse/ClickHouse/issues/6074) [\#6438](https://github.com/ClickHouse/ClickHouse/pull/6438) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - نوع دسته ای اجرا از به روز رسانی کشورهای تابع جمع. ممکن است به مزایای عملکرد منجر شود. [\#6435](https://github.com/ClickHouse/ClickHouse/pull/6435) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - با استفاده از `FastOps` کتابخانه برای توابع `exp`, `log`, `sigmoid`, `tanh`. FastOps سریع بردار ریاضی کتابخانه از مایکل Parakhin (Yandex CTO). بهبود عملکرد `exp` و `log` توابع بیش از 6 بار. توابع `exp` و `log` از `Float32` استدلال باز خواهد گشت `Float32` (در نسخه های قبلی همیشه باز می گردند `Float64`). حالا `exp(nan)` ممکن است بازگشت `inf`. نتیجه `exp` و `log` توابع ممکن است نزدیکترین ماشین تعداد نمایندگی به پاسخ واقعی نیست. [\#6254](https://github.com/ClickHouse/ClickHouse/pull/6254) ([الکسی میلویدو](https://github.com/alexey-milovidov)) با استفاده از Danila Kutenin نوع را fastops کار [\#6317](https://github.com/ClickHouse/ClickHouse/pull/6317) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - غیر فعال کردن بهینه سازی کلید متوالی برای `UInt8/16`. [\#6298](https://github.com/ClickHouse/ClickHouse/pull/6298) [\#6701](https://github.com/ClickHouse/ClickHouse/pull/6701) ([اکوزم](https://github.com/akuzm)) @@ -711,7 +711,7 @@ toc_title: '2019' - حالت مقایسه در `clickhouse-benchmark` [\#6220](https://github.com/ClickHouse/ClickHouse/issues/6220) [\#6343](https://github.com/ClickHouse/ClickHouse/pull/6343) ([دیماروب2000](https://github.com/dimarub2000)) - بهترین تلاش برای چاپ اثری پشته. همچنین اضافه شده است `SIGPROF` به عنوان یک سیگنال اشکال زدایی برای چاپ ردیابی پشته از یک موضوع در حال اجرا. [\#6529](https://github.com/ClickHouse/ClickHouse/pull/6529) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - هر تابع در فایل خود را, بخش 10. [\#6321](https://github.com/ClickHouse/ClickHouse/pull/6321) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- حذف دو برابر const `TABLE_IS_READ_ONLY`. [\#6566](https://github.com/ClickHouse/ClickHouse/pull/6566) ([فیلیمونف](https://github.com/filimonov)) +- حذف توایع دو برابر `TABLE_IS_READ_ONLY`. [\#6566](https://github.com/ClickHouse/ClickHouse/pull/6566) ([فیلیمونف](https://github.com/filimonov)) - تغییرات قالب بندی برای `StringHashMap` PR [\#5417](https://github.com/ClickHouse/ClickHouse/issues/5417). [\#6700](https://github.com/ClickHouse/ClickHouse/pull/6700) ([اکوزم](https://github.com/akuzm)) - خرده فروشی بهتر برای پیوستن به ایجاد در `ExpressionAnalyzer`. [\#6824](https://github.com/ClickHouse/ClickHouse/pull/6824) ([زویکوف](https://github.com/4ertus2)) - حذف یک وضعیت کار برکنار شده (پیدا شده توسط پوس استودیو). [\#6775](https://github.com/ClickHouse/ClickHouse/pull/6775) ([اکوزم](https://github.com/akuzm)) @@ -719,7 +719,7 @@ toc_title: '2019' - فاکتورگیری مجدد از تنظیمات. [\#6689](https://github.com/ClickHouse/ClickHouse/pull/6689) ([الساپین](https://github.com/alesapin)) - اضافه کردن نظر برای `set` توابع شاخص. [\#6319](https://github.com/ClickHouse/ClickHouse/pull/6319) ([نیکیتا واسیلیف](https://github.com/nikvas0)) - افزایش نمره اتم در نسخه اشکال زدایی در لینوکس. [\#6152](https://github.com/ClickHouse/ClickHouse/pull/6152) ([اکوزم](https://github.com/akuzm)) -- در حال حاضر در ساخت اشکال زدایی کار می کنند. [\#6650](https://github.com/ClickHouse/ClickHouse/pull/6650) ([Weiqing زو](https://github.com/weiqxu)) +- در حال حاضر در ساخت اشکال زدایی کار می کنند. [\#6650](https://github.com/ClickHouse/ClickHouse/pull/6650) ([در حال بارگذاری](https://github.com/weiqxu)) - اضافه شدن یک تست به `transform_query_for_external_database`. [\#6388](https://github.com/ClickHouse/ClickHouse/pull/6388) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - اضافه کردن تست برای نمایش چند مادی برای جدول کافکا. [\#6509](https://github.com/ClickHouse/ClickHouse/pull/6509) ([ایوان](https://github.com/abyss7)) - ایجاد یک طرح ساخت بهتر است. [\#6500](https://github.com/ClickHouse/ClickHouse/pull/6500) ([ایوان](https://github.com/abyss7)) @@ -734,7 +734,7 @@ toc_title: '2019' - حذف کد منسوخ برای `mimalloc` پشتیبانی [\#6715](https://github.com/ClickHouse/ClickHouse/pull/6715) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - `zlib-ng` قابلیت های ایکس86 را تعیین می کند و این اطلاعات را به متغیرهای جهانی ذخیره می کند. این است که در تماس دفالتینیت انجام, که ممکن است توسط موضوعات مختلف به طور همزمان ساخته شده. برای جلوگیری از چند رشته ای می نویسد, این کار را در هنگام راه اندازی کتابخانه. [\#6141](https://github.com/ClickHouse/ClickHouse/pull/6141) ([اکوزم](https://github.com/akuzm)) - تست رگرسیون برای یک اشکال که در پیوستن که در ثابت شد [\#5192](https://github.com/ClickHouse/ClickHouse/issues/5192). [\#6147](https://github.com/ClickHouse/ClickHouse/pull/6147) ([بختیاری روازیف](https://github.com/theruziev)) -- ثابت msan گزارش. [\#6144](https://github.com/ClickHouse/ClickHouse/pull/6144) ([الکسی میلویدو](https://github.com/alexey-milovidov)) +- ثابت MSan گزارش. [\#6144](https://github.com/ClickHouse/ClickHouse/pull/6144) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - رفع زدن تست فلش. [\#6782](https://github.com/ClickHouse/ClickHouse/pull/6782) ([انتون پوپوف](https://github.com/CurtizJ)) - مسابقه داده های نادرست ثابت در `MergeTreeDataPart::is_frozen` رشته. [\#6583](https://github.com/ClickHouse/ClickHouse/pull/6583) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - وقفه ثابت در تست ریش شدن. در نسخه های قبلی این موفق به پیدا کردن غلط معوق در پرس و جو `SELECT * FROM numbers_mt(gccMurmurHash(''))`. [\#6582](https://github.com/ClickHouse/ClickHouse/pull/6582) ([الکسی میلویدو](https://github.com/alexey-milovidov)) @@ -751,7 +751,7 @@ toc_title: '2019' - تست عملکرد اضافه شده برای گوریل ثابت و کدک مضاعف. [\#6179](https://github.com/ClickHouse/ClickHouse/pull/6179) ([واسیلی نمکو](https://github.com/Enmk)) - تقسیم تست ادغام `test_dictionaries` به 4 تست جداگانه. [\#6776](https://github.com/ClickHouse/ClickHouse/pull/6776) ([ویتالی بارانو](https://github.com/vitlibar)) - پردازشگر پشتیبانی شده: `PipelineExecutor`. [\#6777](https://github.com/ClickHouse/ClickHouse/pull/6777) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) -- اجازه می دهد به استفاده از `library` منبع فرهنگ لغت با اسان. [\#6482](https://github.com/ClickHouse/ClickHouse/pull/6482) ([الکسی میلویدو](https://github.com/alexey-milovidov)) +- مجاز به استفاده `library` منبع فرهنگ لغت با اسان. [\#6482](https://github.com/ClickHouse/ClickHouse/pull/6482) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - اضافه شدن گزینه برای تولید تغییرات از یک لیست از روابط عمومی. [\#6350](https://github.com/ClickHouse/ClickHouse/pull/6350) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - قفل `TinyLog` ذخیره سازی در هنگام خواندن. [\#6226](https://github.com/ClickHouse/ClickHouse/pull/6226) ([اکوزم](https://github.com/akuzm)) - بررسی برای پیوندهای نمادی شکسته در سی. [\#6634](https://github.com/ClickHouse/ClickHouse/pull/6634) ([الکسی میلویدو](https://github.com/alexey-milovidov)) @@ -763,10 +763,10 @@ toc_title: '2019' - گزارش ثابت اوبسان در `ProtobufWriter`. [\#6163](https://github.com/ClickHouse/ClickHouse/pull/6163) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - اجازه ندهید که از پیشفیلتر پرس و جو با ضدعفونی کننده استفاده کنید زیرا سازگار نیست. [\#6769](https://github.com/ClickHouse/ClickHouse/pull/6769) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - اضافه کردن تست برای بارگذاری یک فرهنگ لغت پس از شکست تایمر. [\#6114](https://github.com/ClickHouse/ClickHouse/pull/6114) ([ویتالی بارانو](https://github.com/vitlibar)) -- رفع تناقض در `PipelineExecutor::prepareProcessor` استدلال نوع. [\#6494](https://github.com/ClickHouse/ClickHouse/pull/6494) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) +- رفع تناقض در `PipelineExecutor::prepareProcessor` نوع استدلال. [\#6494](https://github.com/ClickHouse/ClickHouse/pull/6494) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) - اضافه شده یک تست برای ادرار بد. [\#6493](https://github.com/ClickHouse/ClickHouse/pull/6493) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- اضافه شدن چک بیشتر به `CAST` تابع. این باید دریافت اطلاعات بیشتر در مورد گسل تقسیم بندی فازی در تست. [\#6346](https://github.com/ClickHouse/ClickHouse/pull/6346) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) -- اضافه شده `gcc-9` پشتیبانی به `docker/builder` کانتینر که ایجاد تصویر به صورت محلی. [\#6333](https://github.com/ClickHouse/ClickHouse/pull/6333) ([گلب novikov](https://github.com/NanoBjorn)) +- اضافه شدن چک بیشتر به `CAST` تابع. این باید اطلاعات بیشتری در مورد گسل تقسیم بندی در تست فازی دریافت کنید. [\#6346](https://github.com/ClickHouse/ClickHouse/pull/6346) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) +- اضافه شده `gcc-9` پشتیبانی به `docker/builder` کانتینر که ایجاد تصویر به صورت محلی. [\#6333](https://github.com/ClickHouse/ClickHouse/pull/6333) ([گلب نویکوف](https://github.com/NanoBjorn)) - تست برای کلید اولیه با `LowCardinality(String)`. [\#5044](https://github.com/ClickHouse/ClickHouse/issues/5044) [\#6219](https://github.com/ClickHouse/ClickHouse/pull/6219) ([دیماروب2000](https://github.com/dimarub2000)) - تست های ثابت تحت تاثیر اثر کند پشته اثری چاپ. [\#6315](https://github.com/ClickHouse/ClickHouse/pull/6315) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - اضافه کردن یک مورد تست برای سقوط در `groupUniqArray` ثابت در [\#6029](https://github.com/ClickHouse/ClickHouse/pull/6029). [\#4402](https://github.com/ClickHouse/ClickHouse/issues/4402) [\#6129](https://github.com/ClickHouse/ClickHouse/pull/6129) ([اکوزم](https://github.com/akuzm)) @@ -809,7 +809,7 @@ toc_title: '2019' - تجزیه ثابت از `AggregateFunction` ارزش های جاسازی شده در پرس و جو. [\#6575](https://github.com/ClickHouse/ClickHouse/issues/6575) [\#6773](https://github.com/ClickHouse/ClickHouse/pull/6773) ([ژیچنگ یو](https://github.com/yuzhichang)) - رفتار اشتباه ثابت `trim` توابع خانواده. [\#6647](https://github.com/ClickHouse/ClickHouse/pull/6647) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -### ClickHouse انتشار 19.13.4.32, 2019-09-10 {#clickhouse-release-19-13-4-32-2019-09-10} +### انتشار کلیک خانه 19.13.4.32, 2019-09-10 {#clickhouse-release-19-13-4-32-2019-09-10} #### رفع اشکال {#bug-fix-11} @@ -827,7 +827,7 @@ toc_title: '2019' - رفع دو ناپایداری در کدک در فاز رفع فشار (کاربر مخرب می تواند داده های فشرده که منجر به سرریز بافر در رفع فشار ساخت). [\#6670](https://github.com/ClickHouse/ClickHouse/pull/6670) ([زویکوف](https://github.com/4ertus2)) -### انتشار کلیک کنیدهاوس 19.13.3.26, 2019-08-22 {#clickhouse-release-19-13-3-26-2019-08-22} +### ClickHouse انتشار 19.13.3.26, 2019-08-22 {#clickhouse-release-19-13-3-26-2019-08-22} #### رفع اشکال {#bug-fix-12} @@ -841,7 +841,7 @@ toc_title: '2019' #### تعمیر امنیتی {#security-fix-2} -- اگر مهاجم دارای دسترسی نوشتن به باغ وحش است و قادر به اجرای سفارشی سرور موجود در شبکه که در آن clickhouse اجرای آن می توانید ایجاد و سفارشی-ساخته شده در سرور های مخرب است که به عنوان clickhouse المثنی و ثبت آن در باغ وحش. هنگامی که ماکت دیگر بخش داده ها از ماکت های مخرب واکشی, می تواند فاحشه خانه و سرور را مجبور به ارسال به مسیر دلخواه در فایل سیستم. پیدا شده توسط الدار زیتوف, تیم امنیت اطلاعات در یاندکس. [\#6247](https://github.com/ClickHouse/ClickHouse/pull/6247) ([الکسی میلویدو](https://github.com/alexey-milovidov)) +- اگر مهاجم دارای دسترسی نوشتن به باغ وحش است و قادر به اجرای سفارشی سرور موجود در شبکه که در آن ClickHouse اجرای آن می توانید ایجاد و سفارشی-ساخته شده در سرور های مخرب است که به عنوان ClickHouse المثنی و ثبت آن در باغ وحش. هنگامی که ماکت دیگر بخش داده ها از ماکت های مخرب واکشی, می تواند فاحشه خانه و سرور را مجبور به ارسال به مسیر دلخواه در فایل سیستم. پیدا شده توسط الدار زیتوف, تیم امنیت اطلاعات در یاندکس. [\#6247](https://github.com/ClickHouse/ClickHouse/pull/6247) ([الکسی میلویدو](https://github.com/alexey-milovidov)) ### انتشار کلیک خانه 19.13.2.19, 2019-08-14 {#clickhouse-release-19-13-2-19-2019-08-14} @@ -901,7 +901,7 @@ toc_title: '2019' - اجتناب از سیگزگف نادر در حالی که ارسال داده ها در جداول با موتور توزیع شده (`Failed to send batch: file with index XXXXX is absent`). [\#7032](https://github.com/ClickHouse/ClickHouse/pull/7032) ([ازات خوژین](https://github.com/azat)) - ثابت `Unknown identifier` با چند می پیوندد. این رفع [\#5254](https://github.com/ClickHouse/ClickHouse/issues/5254). [\#7022](https://github.com/ClickHouse/ClickHouse/pull/7022) ([زویکوف](https://github.com/4ertus2)) -### انتشار تاتر 19.11.11.57, 2019-09-13 {#clickhouse-release-19-11-11-57-2019-09-13} +### ClickHouse انتشار 19.11.11.57, 2019-09-13 {#clickhouse-release-19-11-11-57-2019-09-13} - رفع خطا منطقی باعث حملات در هنگام انتخاب از کافکا موضوع خالی. [\#6902](https://github.com/ClickHouse/ClickHouse/issues/6902) [\#6909](https://github.com/ClickHouse/ClickHouse/pull/6909) ([ایوان](https://github.com/abyss7)) - ثابت برای عملکرد `АrrayEnumerateUniqRanked` با بند خالی در پارامز. [\#6928](https://github.com/ClickHouse/ClickHouse/pull/6928) ([پرولر](https://github.com/proller)) @@ -910,9 +910,9 @@ toc_title: '2019' #### رفع اشکال {#bug-fix-16} -- فراموش فروشگاه شیپور خاموشی برای کافکا پیام های دستی قادر به ارتکاب آنها را همه در یک بار و برای همه پارتیشن. رفع بالقوه تقلید در “one consumer - many partitions” سناریو [\#6872](https://github.com/ClickHouse/ClickHouse/pull/6872) ([ایوان](https://github.com/abyss7)) +- پیام های کافکا را به صورت دستی ذخیره کنید تا بتوانید همه را در یک زمان برای تمام پارتیشن ها انجام دهید. رفع تقلید بالقوه در “one consumer - many partitions” سناریو [\#6872](https://github.com/ClickHouse/ClickHouse/pull/6872) ([ایوان](https://github.com/abyss7)) -### انتشار کلیک کنیدهاوس 19.11.9.52, 2019-09-6 {#clickhouse-release-19-11-9-52-2019-09-6} +### ClickHouse انتشار 19.11.9.52, 2019-09-6 {#clickhouse-release-19-11-9-52-2019-09-6} - بهبود دست زدن به خطا در لغت نامه کش. [\#6737](https://github.com/ClickHouse/ClickHouse/pull/6737) ([ویتالی بارانو](https://github.com/vitlibar)) - اشکال ثابت در عملکرد `arrayEnumerateUniqRanked`. [\#6779](https://github.com/ClickHouse/ClickHouse/pull/6779) ([پرولر](https://github.com/proller)) @@ -928,9 +928,9 @@ toc_title: '2019' #### تعمیر امنیتی {#security-fix-3} -- اگر مهاجم دارای دسترسی نوشتن به باغ وحش است و قادر به اجرای سفارشی سرور موجود در شبکه که در آن clickhouse اجرا می شود, آن می تواند ایجاد سفارشی-ساخته شده در سرور های مخرب است که به عنوان clickhouse المثنی و ثبت آن در باغ وحش. هنگامی که ماکت دیگر بخش داده ها از ماکت های مخرب واکشی, می تواند فاحشه خانه و سرور را مجبور به ارسال به مسیر دلخواه در فایل سیستم. پیدا شده توسط الدار زیتوف, تیم امنیت اطلاعات در یاندکس. [\#6247](https://github.com/ClickHouse/ClickHouse/pull/6247) ([الکسی میلویدو](https://github.com/alexey-milovidov)) +- اگر مهاجم دارای دسترسی نوشتن به باغ وحش است و قادر به اجرای سفارشی سرور موجود در شبکه که در آن ClickHouse اجرا می شود, آن می تواند ایجاد سفارشی-ساخته شده در سرور های مخرب است که به عنوان ClickHouse المثنی و ثبت آن در باغ وحش. هنگامی که ماکت دیگر بخش داده ها از ماکت های مخرب واکشی, می تواند فاحشه خانه و سرور را مجبور به ارسال به مسیر دلخواه در فایل سیستم. پیدا شده توسط الدار زیتوف, تیم امنیت اطلاعات در یاندکس. [\#6247](https://github.com/ClickHouse/ClickHouse/pull/6247) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -### انتشار کلیک کنیدهاوس 19.11.8.46, 2019-08-22 {#clickhouse-release-19-11-8-46-2019-08-22} +### ClickHouse انتشار 19.11.8.46, 2019-08-22 {#clickhouse-release-19-11-8-46-2019-08-22} #### رفع اشکال {#bug-fix-17} @@ -939,9 +939,9 @@ toc_title: '2019' - ثابت یک موضوع است که اگر یک ماکت کهنه زنده می شود, هنوز هم ممکن است قطعات داده که توسط پارتیشن قطره حذف شد. [\#6522](https://github.com/ClickHouse/ClickHouse/issues/6522) [\#6523](https://github.com/ClickHouse/ClickHouse/pull/6523) ([تاولوبیکس](https://github.com/tavplubix)) - موضوع ثابت با تجزیه سی اس وی [\#6426](https://github.com/ClickHouse/ClickHouse/issues/6426) [\#6559](https://github.com/ClickHouse/ClickHouse/pull/6559) ([تاولوبیکس](https://github.com/tavplubix)) - مسابقه داده ثابت در سیستم.جدول قطعات و تغییر پرس و جو. این رفع [\#6245](https://github.com/ClickHouse/ClickHouse/issues/6245). [\#6513](https://github.com/ClickHouse/ClickHouse/pull/6513) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- ثابت کد اشتباه در جهش است که ممکن است منجر به حافظه فساد است. ثابت segfault با خواندن آدرس `0x14c0` که ممکن است به دلیل همزمان اتفاق `DROP TABLE` و `SELECT` از `system.parts` یا `system.parts_columns`. شرایط مسابقه ثابت در تهیه نمایش داده شد جهش. بن بست ثابت ناشی از `OPTIMIZE` از جداول تکرار و عملیات اصلاح همزمان مانند تغییر. [\#6514](https://github.com/ClickHouse/ClickHouse/pull/6514) ([الکسی میلویدو](https://github.com/alexey-milovidov)) +- کد اشتباه ثابت در جهش است که ممکن است به فساد حافظه منجر شود. سگو ثابت با خواندن نشانی `0x14c0` که ممکن است به دلیل همزمان اتفاق `DROP TABLE` و `SELECT` از `system.parts` یا `system.parts_columns`. شرایط مسابقه ثابت در تهیه نمایش داده شد جهش. بن بست ثابت ناشی از `OPTIMIZE` از جداول تکرار و عملیات اصلاح همزمان مانند تغییر. [\#6514](https://github.com/ClickHouse/ClickHouse/pull/6514) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -### انتشار کلیک کنیدهاوس 19.11.7.40, 2019-08-14 {#clickhouse-release-19-11-7-40-2019-08-14} +### ClickHouse انتشار 19.11.7.40, 2019-08-14 {#clickhouse-release-19-11-7-40-2019-08-14} #### رفع اشکال {#bug-fix-18} @@ -985,10 +985,10 @@ toc_title: '2019' - رفع اشکال با نوشتن شاخص ثانویه نشانه با دانه دانه تطبیقی. [\#6126](https://github.com/ClickHouse/ClickHouse/pull/6126) ([الساپین](https://github.com/alesapin)) - ثابت `WITH ROLLUP` و `WITH CUBE` اصلاح کننده های `GROUP BY` با تجمع دو سطح. [\#6225](https://github.com/ClickHouse/ClickHouse/pull/6225) ([انتون پوپوف](https://github.com/CurtizJ)) - ثابت قطع در `JSONExtractRaw` تابع. ثابت [\#6195](https://github.com/ClickHouse/ClickHouse/issues/6195) [\#6198](https://github.com/ClickHouse/ClickHouse/pull/6198) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- رفع segfault در externalloader::reloadoutdated(). [\#6082](https://github.com/ClickHouse/ClickHouse/pull/6082) ([ویتالی بارانو](https://github.com/vitlibar)) +- رفع segfault در ExternalLoader::reloadOutdated(). [\#6082](https://github.com/ClickHouse/ClickHouse/pull/6082) ([ویتالی بارانو](https://github.com/vitlibar)) - ثابت مورد زمانی که سرور ممکن است گوش دادن سوکت اما خاموش نمی بستن و ادامه خدمت نمایش داده شد باقی مانده است. شما ممکن است در نهایت با دو در حال اجرا فرایندهای فاحشه خانه و سرور. گاهی, سرور ممکن است یک خطا بازگشت `bad_function_call` برای نمایش داده شد باقی مانده است. [\#6231](https://github.com/ClickHouse/ClickHouse/pull/6231) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - شرایط بی فایده و نادرست ثابت در زمینه به روز رسانی برای بارگذاری اولیه از لغت نامه های خارجی از طریق ال بی سی, خروجی زیر, کلیک و قام. این رفع [\#6069](https://github.com/ClickHouse/ClickHouse/issues/6069) [\#6083](https://github.com/ClickHouse/ClickHouse/pull/6083) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- استثنا بی ربط ثابت در بازیگران `LowCardinality(Nullable)` to not-Nullable column in case if it doesn’t contain Nulls (e.g. in query like `SELECT CAST(CAST('Hello' AS LowCardinality(Nullable(String))) AS String)`. [\#6094](https://github.com/ClickHouse/ClickHouse/issues/6094) [\#6119](https://github.com/ClickHouse/ClickHouse/pull/6119) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) +- استثنا بی ربط ثابت در بازیگران `LowCardinality(Nullable)` to not-Nullable column in case if it doesn't contain Nulls (e.g. in query like `SELECT CAST(CAST('Hello' AS LowCardinality(Nullable(String))) AS String)`. [\#6094](https://github.com/ClickHouse/ClickHouse/issues/6094) [\#6119](https://github.com/ClickHouse/ClickHouse/pull/6119) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) - رفع نتیجه غیر قطعی از “uniq” تابع مجموع در موارد شدید نادر. اشکال در حال حاضر در تمام نسخه های تاتر بود. [\#6058](https://github.com/ClickHouse/ClickHouse/pull/6058) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - Segfault هنگامی که ما مجموعه ای کمی بیش از حد بالا CIDR در تابع `IPv6CIDRToRange`. [\#6068](https://github.com/ClickHouse/ClickHouse/pull/6068) ([کارخانه شراب سازی گیوم](https://github.com/YiuRULE)) - ثابت نشت حافظه کوچک زمانی که سرور پرتاب استثنا بسیاری از بسیاری از زمینه های مختلف. [\#6144](https://github.com/ClickHouse/ClickHouse/pull/6144) ([الکسی میلویدو](https://github.com/alexey-milovidov)) @@ -1031,20 +1031,20 @@ toc_title: '2019' - زیر جریان بافر ثابت در `visitParamExtractRaw`. این رفع [\#5901](https://github.com/ClickHouse/ClickHouse/issues/5901) [\#5902](https://github.com/ClickHouse/ClickHouse/pull/5902) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - در حال حاضر توزیع شده است `DROP/ALTER/TRUNCATE/OPTIMIZE ON CLUSTER` نمایش داده شد خواهد شد به طور مستقیم در ماکت رهبر اجرا شده است. [\#5757](https://github.com/ClickHouse/ClickHouse/pull/5757) ([الساپین](https://github.com/alesapin)) - ثابت `coalesce` برای `ColumnConst` با `ColumnNullable` + تغییرات مرتبط . [\#5755](https://github.com/ClickHouse/ClickHouse/pull/5755) ([زویکوف](https://github.com/4ertus2)) -- رفع `ReadBufferFromKafkaConsumer` به طوری که آن را نگه می دارد و خواندن پیام های جدید پس از `commit()` حتی اگر قبلا متوقف شده بود [\#5852](https://github.com/ClickHouse/ClickHouse/pull/5852) ([ایوان](https://github.com/abyss7)) +- رفع `ReadBufferFromKafkaConsumer` به طوری که نگه می دارد خواندن پیام های جدید پس از `commit()` حتی اگر قبلا متوقف شده بود [\#5852](https://github.com/ClickHouse/ClickHouse/pull/5852) ([ایوان](https://github.com/abyss7)) - ثابت `FULL` و `RIGHT` پیوستن به نتایج در هنگام پیوستن به `Nullable` کلید در جدول سمت راست. [\#5859](https://github.com/ClickHouse/ClickHouse/pull/5859) ([زویکوف](https://github.com/4ertus2)) - رفع احتمالی خواب بی نهایت نمایش داده شد اولویت پایین. [\#5842](https://github.com/ClickHouse/ClickHouse/pull/5842) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - رفع شرایط مسابقه, که باعث می شود که برخی از نمایش داده شد ممکن است در کواریلاگ پس از به نظر می رسد `SYSTEM FLUSH LOGS` پرس و جو. [\#5456](https://github.com/ClickHouse/ClickHouse/issues/5456) [\#5685](https://github.com/ClickHouse/ClickHouse/pull/5685) ([انتون پوپوف](https://github.com/CurtizJ)) - ثابت `heap-use-after-free` اسان هشدار در خوشه بندی ناشی از سازمان دیده بان که سعی کنید به استفاده از شی کپی در حال حاضر حذف شده است. [\#5871](https://github.com/ClickHouse/ClickHouse/pull/5871) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) -- اشتباه ثابت `StringRef` گر بازگشت توسط برخی از پیاده سازی `IColumn::deserializeAndInsertFromArena`. این اشکال تحت تاثیر قرار تنها واحد تست. [\#5973](https://github.com/ClickHouse/ClickHouse/pull/5973) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) +- اشتباه ثابت `StringRef` اشاره گر بازگشت توسط برخی از پیاده سازی از `IColumn::deserializeAndInsertFromArena`. این اشکال تحت تاثیر قرار تنها واحد تست. [\#5973](https://github.com/ClickHouse/ClickHouse/pull/5973) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) - جلوگیری از منبع و مجموعه ای متوسط پیوستن به ستون از پوشش ستون همان نام. [\#5941](https://github.com/ClickHouse/ClickHouse/pull/5941) ([زویکوف](https://github.com/4ertus2)) - رفع درج و پرس و جو را انتخاب کنید به خروجی زیر موتور با خروجی زیر شناسه سبک به نقل از. [\#5704](https://github.com/ClickHouse/ClickHouse/pull/5704) ([زمستان ژانگ](https://github.com/zhang2014)) - حالا `CHECK TABLE` پرس و جو می تواند با خانواده موتور ادغام کار می کنند. این گرداند بررسی وضعیت و پیام اگر هر برای هر بخش (و یا فایل در مورد موتورهای ساده تر). همچنین, رفع اشکال در واکشی از یک بخش شکسته. [\#5865](https://github.com/ClickHouse/ClickHouse/pull/5865) ([الساپین](https://github.com/alesapin)) -- رفع split\_shared\_libraries در زمان اجرا [\#5793](https://github.com/ClickHouse/ClickHouse/pull/5793) ([نام و نام خانوادگی](https://github.com/danlark1)) +- رفع SPLIT\_SHARED\_LIBRARIES در زمان اجرا [\#5793](https://github.com/ClickHouse/ClickHouse/pull/5793) ([نام و نام خانوادگی](https://github.com/danlark1)) - مقدار دهی اولیه منطقه زمانی ثابت زمانی که `/etc/localtime` پیوند نمادی نسبی مانند است `../usr/share/zoneinfo/Europe/Moscow` [\#5922](https://github.com/ClickHouse/ClickHouse/pull/5922) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- clickhouse-کپی: ثابت استفاده رایگان در خاموش کردن [\#5752](https://github.com/ClickHouse/ClickHouse/pull/5752) ([پرولر](https://github.com/proller)) +- تاتر-کپی: رفع استفاده-پس از رایگان در خاموش کردن [\#5752](https://github.com/ClickHouse/ClickHouse/pull/5752) ([پرولر](https://github.com/proller)) - به روز شده `simdjson`. ثابت موضوع است که برخی از پارسونز نامعتبر با صفر بایت موفقیت تجزیه. [\#5938](https://github.com/ClickHouse/ClickHouse/pull/5938) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- رفع خاموش کردن systemlogs [\#5802](https://github.com/ClickHouse/ClickHouse/pull/5802) ([انتون پوپوف](https://github.com/CurtizJ)) +- رفع خاموش شدن سیستم ها [\#5802](https://github.com/ClickHouse/ClickHouse/pull/5802) ([انتون پوپوف](https://github.com/CurtizJ)) - رفع زمانی که وضعیت در باطل کردن\_قرقمی بستگی به یک فرهنگ لغت. [\#6011](https://github.com/ClickHouse/ClickHouse/pull/6011) ([ویتالی بارانو](https://github.com/vitlibar)) #### بهبود {#improvement-6} @@ -1081,7 +1081,7 @@ toc_title: '2019' - سودهی به [اجرای رایگان](https://github.com/llvm-mirror/libunwind) برای ج++ دست زدن به استثنا و برای ردیابی پشته چاپ [\#4828](https://github.com/ClickHouse/ClickHouse/pull/4828) ([نیکیتا لکوف](https://github.com/laplab)) - اضافه کردن دو هشدار بیشتر از-ابزار [\#5923](https://github.com/ClickHouse/ClickHouse/pull/5923) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - اجازه می دهد برای ساخت تاتر با ضد عفونی کننده حافظه. [\#3949](https://github.com/ClickHouse/ClickHouse/pull/3949) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- ثابت ubsan گزارش مورد `bitTest` تابع در تست ریش شدن. [\#5943](https://github.com/ClickHouse/ClickHouse/pull/5943) ([الکسی میلویدو](https://github.com/alexey-milovidov)) +- گزارش ثابت اوبسان درباره `bitTest` تابع در تست ریش شدن. [\#5943](https://github.com/ClickHouse/ClickHouse/pull/5943) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - کارگر بارانداز: اضافه شدن امکان به داخل یک نمونه کلیک که نیاز به احراز هویت. [\#5727](https://github.com/ClickHouse/ClickHouse/pull/5727) ([هشدار داده می شود](https://github.com/shurshun)) - به روز رسانی کتابدار به نسخه 1.1.0 [\#5872](https://github.com/ClickHouse/ClickHouse/pull/5872) ([ایوان](https://github.com/abyss7)) - اضافه کردن ایست جهانی برای تست ادغام و غیر فعال کردن برخی از در کد تست. [\#5741](https://github.com/ClickHouse/ClickHouse/pull/5741) ([الساپین](https://github.com/alesapin)) @@ -1167,7 +1167,7 @@ toc_title: '2019' ## انتشار کلیک 19.9 {#clickhouse-release-19-9} -### انتشار تاتر 19.9.3.31, 2019-07-05 {#clickhouse-release-19-9-3-31-2019-07-05} +### ClickHouse انتشار 19.9.3.31, 2019-07-05 {#clickhouse-release-19-9-3-31-2019-07-05} #### رفع اشکال {#bug-fix-23} @@ -1201,7 +1201,7 @@ toc_title: '2019' - رفع از دست دادن داده های بالقوه در کافکا [\#5445](https://github.com/ClickHouse/ClickHouse/pull/5445) ([ایوان](https://github.com/abyss7)) - رفع حلقه بی نهایت بالقوه در `PrettySpace` فرمت زمانی که با صفر ستون نامیده می شود [\#5560](https://github.com/ClickHouse/ClickHouse/pull/5560) ([اولگا خوستیکوا](https://github.com/stavrolia)) - اشکال سرریز ثابت در مدل های خطی. اجازه مدل میلی لیتر اومال برای استدلال مدل غیر توایع. [\#5516](https://github.com/ClickHouse/ClickHouse/pull/5516) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) -- `ALTER TABLE ... DROP INDEX IF EXISTS ...` اگر شاخص موجود وجود ندارد باید یک استثنا را افزایش نمی دهد [\#5524](https://github.com/ClickHouse/ClickHouse/pull/5524) ([گلب novikov](https://github.com/NanoBjorn)) +- `ALTER TABLE ... DROP INDEX IF EXISTS ...` اگر شاخص موجود وجود ندارد باید یک استثنا را افزایش نمی دهد [\#5524](https://github.com/ClickHouse/ClickHouse/pull/5524) ([گلب نویکوف](https://github.com/NanoBjorn)) - رفع segfault با `bitmapHasAny` در زیر مقیاس [\#5528](https://github.com/ClickHouse/ClickHouse/pull/5528) ([ژیچنگ یو](https://github.com/yuzhichang)) - خطا ثابت زمانی که استخر اتصال تکرار کند سعی مجدد نیست برای حل و فصل میزبان, حتی زمانی که کش دی ان اس کاهش یافته بود. [\#5534](https://github.com/ClickHouse/ClickHouse/pull/5534) ([الساپین](https://github.com/alesapin)) - ثابت `ALTER ... MODIFY TTL` در تکرار غذای اصلی. [\#5539](https://github.com/ClickHouse/ClickHouse/pull/5539) ([انتون پوپوف](https://github.com/CurtizJ)) @@ -1209,7 +1209,7 @@ toc_title: '2019' - رفع تخصیص بد زمانی که کوتاه پیوستن به ذخیره سازی [\#5437](https://github.com/ClickHouse/ClickHouse/pull/5437) ([تسیسون](https://github.com/TCeason)) - در نسخه های اخیر از بسته tzdata برخی از فایل ها symlinks در حال حاضر. مکانیسم فعلی برای تشخیص منطقه زمانی پیش فرض شکسته می شود و نام اشتباه برای برخی از جغرافیایی می دهد. در حال حاضر حداقل ما نام منطقه زمانی را مجبور به محتویات انجمن تاز در صورت فراهم. [\#5443](https://github.com/ClickHouse/ClickHouse/pull/5443) ([ایوان](https://github.com/abyss7)) - رفع برخی از موارد بسیار نادر با جستجوگر چندگانه هنگامی که سوزن ثابت در مجموع حداقل 16 کیلوبایت طولانی است. الگوریتم از دست رفته و یا بیش از حد نتایج قبلی که می تواند به نتیجه نادرست منجر شود `multiSearchAny`. [\#5588](https://github.com/ClickHouse/ClickHouse/pull/5588) ([نام و نام خانوادگی](https://github.com/danlark1)) -- رفع مشکل زمانی که تنظیمات برای externaldata درخواست نمی تواند با استفاده از clickhouse تنظیمات. همچنین در حال حاضر تنظیمات `date_time_input_format` و `low_cardinality_allow_in_native_format` می تواند به دلیل ابهام از نام استفاده نمی شود (در داده های خارجی را می توان به عنوان فرمت جدول تفسیر و در پرس و جو می تواند یک محیط). [\#5455](https://github.com/ClickHouse/ClickHouse/pull/5455) ([نام و نام خانوادگی](https://github.com/danlark1)) +- رفع مشکل زمانی که تنظیمات برای ExternalData درخواست نمی تواند با استفاده از ClickHouse تنظیمات. همچنین در حال حاضر تنظیمات `date_time_input_format` و `low_cardinality_allow_in_native_format` می تواند به دلیل ابهام از نام استفاده نمی شود (در داده های خارجی را می توان به عنوان فرمت جدول تفسیر و در پرس و جو می تواند یک محیط). [\#5455](https://github.com/ClickHouse/ClickHouse/pull/5455) ([نام و نام خانوادگی](https://github.com/danlark1)) - رفع اشکال زمانی که قطعات تنها از فدراسیون فوتبال بدون رها کردن از باغ وحش حذف شد. [\#5520](https://github.com/ClickHouse/ClickHouse/pull/5520) ([الساپین](https://github.com/alesapin)) - حذف ورود اشکال زدایی از پروتکل خروجی زیر [\#5478](https://github.com/ClickHouse/ClickHouse/pull/5478) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - رد کردن زنود در طول پردازش پرس و جو دی ال [\#5489](https://github.com/ClickHouse/ClickHouse/pull/5489) ([ازات خوژین](https://github.com/azat)) @@ -1267,7 +1267,7 @@ toc_title: '2019' - حالا `if` و `multiIf` توابع به وضعیت تکیه نمی کنند `Nullable`, اما در شاخه برای سازگاری گذاشتن تکیه. [\#5238](https://github.com/ClickHouse/ClickHouse/pull/5238) ([جیان وو](https://github.com/janplus)) - `In` گزاره در حال حاضر تولید می کند `Null` نتیجه از `Null` ورودی مانند `Equal` تابع. [\#5152](https://github.com/ClickHouse/ClickHouse/pull/5152) ([جیان وو](https://github.com/janplus)) - بررسی محدودیت زمانی هر (flush\_interval / poll\_timeout) تعداد سطر از کافکا. این اجازه می دهد تا خواندن را از مصرف کننده کافکا بیشتر کند و محدودیت های زمانی جریان های سطح بالا را بررسی کند [\#5249](https://github.com/ClickHouse/ClickHouse/pull/5249) ([ایوان](https://github.com/abyss7)) -- لینک rdkafka با همراه sasl. باید اجازه می دهد به استفاده از sasl گریختن احراز هویت [\#5253](https://github.com/ClickHouse/ClickHouse/pull/5253) ([ایوان](https://github.com/abyss7)) +- لینک rdkafka با همراه SASL. باید اجازه می دهد به استفاده از SASL گریختن احراز هویت [\#5253](https://github.com/ClickHouse/ClickHouse/pull/5253) ([ایوان](https://github.com/abyss7)) - Batched نسخه RowRefList برای همه می پیوندد. [\#5267](https://github.com/ClickHouse/ClickHouse/pull/5267) ([زویکوف](https://github.com/4ertus2)) - کلیک سرور: اطلاعات بیشتر گوش پیام های خطا. [\#5268](https://github.com/ClickHouse/ClickHouse/pull/5268) ([پرولر](https://github.com/proller)) - واژهنامهها پشتیبانی در تاتر-کپی برای توابع در `` [\#5270](https://github.com/ClickHouse/ClickHouse/pull/5270) ([پرولر](https://github.com/proller)) @@ -1279,7 +1279,7 @@ toc_title: '2019' - پیام های خطا بهتر اگر عدم تطابق کنترلی است به احتمال زیاد توسط شکست سخت افزار ایجاد می شود. [\#5355](https://github.com/ClickHouse/ClickHouse/pull/5355) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - بررسی کنید که جداول پایه نمونه برداری را پشتیبانی می کنند `StorageMerge` [\#5366](https://github.com/ClickHouse/ClickHouse/pull/5366) ([ایوان](https://github.com/abyss7)) - Сlose MySQL connections after their usage in external dictionaries. It is related to issue \#893. [\#5395](https://github.com/ClickHouse/ClickHouse/pull/5395) ([Clément Rodriguez](https://github.com/clemrodriguez)) -- بهبود پروتکل سیم خروجی زیر. تغییر نام قالب به میسورقلوایر. با استفاده از raii برای تماس rsa\_free. غیر فعال کردن اس اس ال اگر زمینه نمی تواند ایجاد شود. [\#5419](https://github.com/ClickHouse/ClickHouse/pull/5419) ([یوری baranov](https://github.com/yurriy)) +- بهبود پروتکل سیم خروجی زیر. تغییر نام قالب به میسورقلوایر. با استفاده از RAII برای تماس RSA\_free. غیر فعال کردن اس اس ال اگر زمینه نمی تواند ایجاد شود. [\#5419](https://github.com/ClickHouse/ClickHouse/pull/5419) ([یوری بارانوف](https://github.com/yurriy)) - clickhouse-client: allow to run with unaccessable history file (read-only, no disk space, file is directory, …). [\#5431](https://github.com/ClickHouse/ClickHouse/pull/5431) ([پرولر](https://github.com/proller)) - تنظیمات پرس و جو احترام در درج ناهمزمان به جداول توزیع. [\#4936](https://github.com/ClickHouse/ClickHouse/pull/4936) ([تسیسون](https://github.com/TCeason)) - توابع تغییر نام داد `leastSqr` به `simpleLinearRegression`, `LinearRegression` به `linearRegression`, `LogisticRegression` به `logisticRegression`. [\#5391](https://github.com/ClickHouse/ClickHouse/pull/5391) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) @@ -1300,7 +1300,7 @@ toc_title: '2019' - رفع فشار نیاز به ستون با پیوستن [\#5192](https://github.com/ClickHouse/ClickHouse/pull/5192) ([زمستان ژانگ](https://github.com/zhang2014)) - اشکال ثابت, زمانی که تاتر توسط سیستم اجرا, فرمان `sudo service clickhouse-server forcerestart` کار نمی کند به عنوان انتظار می رود. [\#5204](https://github.com/ClickHouse/ClickHouse/pull/5204) ([پرولر](https://github.com/proller)) - رفع کدهای خطای اچ. تی. اچ. تی. پی در کد 9009 پورت همیشه کد 200 را حتی در خطاها برگرداند. [\#5216](https://github.com/ClickHouse/ClickHouse/pull/5216) ([پرولر](https://github.com/proller)) -- رفع simpleaggregatefunction برای رشته بیش از max\_small\_string\_size [\#5311](https://github.com/ClickHouse/ClickHouse/pull/5311) ([ازات خوژین](https://github.com/azat)) +- رفع SimpleAggregateFunction برای رشته بیش از MAX\_SMALL\_STRING\_SIZE [\#5311](https://github.com/ClickHouse/ClickHouse/pull/5311) ([ازات خوژین](https://github.com/azat)) - رفع خطا برای `Decimal` به `Nullable(Decimal)` تبدیل در. پشتیبانی از دیگر دهدهی به دهدهی تبدیل (از جمله مقیاس های مختلف). [\#5350](https://github.com/ClickHouse/ClickHouse/pull/5350) ([زویکوف](https://github.com/4ertus2)) - مسدود کردن فلش ثابت در کتابخانه سیمدجسون که منجر به محاسبه اشتباه از `uniqHLL` و `uniqCombined` تابع کلی و توابع ریاضی مانند `log`. [\#5354](https://github.com/ClickHouse/ClickHouse/pull/5354) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - دست زدن به ثابت موارد مخلوط ثابت/غیرنقدی در توابع جانسون. [\#5435](https://github.com/ClickHouse/ClickHouse/pull/5435) ([ویتالی بارانو](https://github.com/vitlibar)) @@ -1357,7 +1357,7 @@ toc_title: '2019' - خطا ثابت زمانی که استخر اتصال تکرار کند سعی مجدد نیست برای حل و فصل میزبان, حتی زمانی که کش دی ان اس کاهش یافته بود. [\#5534](https://github.com/ClickHouse/ClickHouse/pull/5534) ([الساپین](https://github.com/alesapin)) - ثابت `DROP INDEX IF EXISTS` پرس و جو. حالا `ALTER TABLE ... DROP INDEX IF EXISTS ...` پرس و جو یک استثنا را افزایش نمی دهد اگر شاخص وجود ندارد. [\#5524](https://github.com/ClickHouse/ClickHouse/pull/5524) ([گلب نویکوف](https://github.com/NanoBjorn)) - رفع اتحادیه تمام ستون نوع فوق. موارد با داده های متناقض و انواع ستون ستون نتیجه وجود دارد. [\#5503](https://github.com/ClickHouse/ClickHouse/pull/5503) ([زویکوف](https://github.com/4ertus2)) -- جست و خیز znonode در طول دستورات پرس و جو پردازش. قبل از اگر گره دیگر حذف زنود در صف کار, یکی که +- جست و خیز ZNONODE در طول دستورات پرس و جو پردازش. قبل از اگر گره دیگر حذف زنود در صف کار, یکی که اما در حال حاضر لیستی از کودکان را دریافت نمی کند موضوع کار کرم را خاتمه می دهد. [\#5489](https://github.com/ClickHouse/ClickHouse/pull/5489) ([ازات خوژین](https://github.com/azat)) - ثابت قرار دادن به توزیع() جدول با ستون محقق. [\#5429](https://github.com/ClickHouse/ClickHouse/pull/5429) ([ازات خوژین](https://github.com/azat)) @@ -1374,7 +1374,7 @@ toc_title: '2019' رفتار شبیه به `groupArray(max_size)(x)` تابع. [\#5026](https://github.com/ClickHouse/ClickHouse/pull/5026) ([گیمه کاسری](https://github.com/YiuRULE)) -- برای tsvwithnames/csvwithnames ورودی فرمت های فایل های ستون سفارش در حال حاضر می تواند +- برای TSVWithNames/CSVWithNames ورودی فرمت های فایل های ستون سفارش در حال حاضر می تواند تعیین شده از هدر فایل. این است که با کنترل `input_format_with_names_use_header` پارامتر. [\#5081](https://github.com/ClickHouse/ClickHouse/pull/5081) @@ -1422,7 +1422,7 @@ toc_title: '2019' - حل و فصل برخی از هشدارهای پوس استودیو. [\#5082](https://github.com/ClickHouse/ClickHouse/pull/5082) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- به روز رسانی lz4 [\#5040](https://github.com/ClickHouse/ClickHouse/pull/5040) ([دنیلا +- به روز رسانی LZ4 [\#5040](https://github.com/ClickHouse/ClickHouse/pull/5040) ([دنیلا کوتنین](https://github.com/danlark1)) - اضافه کردن پردازنده گرافیکی برای ساخت الزامات مورد نیاز برای درخواست پیش رو کشیدن \# 5030. [\#5110](https://github.com/ClickHouse/ClickHouse/pull/5110) @@ -1439,7 +1439,7 @@ toc_title: '2019' - اجازه اعشار نقل در سی سی. وی. [\#5284](https://github.com/ClickHouse/ClickHouse/pull/5284) ([زویکوف](https://github.com/4ertus2) - عدم تبدیل از اینف شناور / نان به اعشار (استثنا پرتاب). [\#5282](https://github.com/ClickHouse/ClickHouse/pull/5282) ([زویکوف](https://github.com/4ertus2)) - رفع مسابقه داده ها در تغییر نام پرس و جو. [\#5247](https://github.com/ClickHouse/ClickHouse/pull/5247) ([زمستان ژانگ](https://github.com/zhang2014)) -- به طور موقت غیر فعال کردن کلیک. استفاده از lfalloc ممکن است منجر به بسیاری از map\_failed در تخصیص uncompressedcache و در نتیجه به سقوط از نمایش داده شد در بالا لود سرور. [لامپ کم مصرف93](https://github.com/ClickHouse/ClickHouse/commit/cfdba938ce22f16efeec504f7f90206a515b1280)([نام و نام خانوادگی](https://github.com/danlark1)) +- به طور موقت غیر فعال کردن کلیک. استفاده از LFAlloc ممکن است منجر به بسیاری از MAP\_FAILED در تخصیص UncompressedCache و در نتیجه به سقوط از نمایش داده شد در بالا لود سرور. [لامپ کم مصرف93](https://github.com/ClickHouse/ClickHouse/commit/cfdba938ce22f16efeec504f7f90206a515b1280)([نام و نام خانوادگی](https://github.com/danlark1)) ### انتشار کلیک خانه 19.6.2.11, 2019-05-13 {#clickhouse-release-19-6-2-11-2019-05-13} @@ -1476,7 +1476,7 @@ toc_title: '2019' - اختلاف اشاره گر صفر پتانسیل ثابت در `clickhouse-copier`. [\#4900](https://github.com/ClickHouse/ClickHouse/pull/4900) ([پرولر](https://github.com/proller)) - خطای ثابت در پرس و جو با پیوستن + مجموعه پیوستن [\#4938](https://github.com/ClickHouse/ClickHouse/pull/4938) ([زویکوف](https://github.com/4ertus2)) - ثابت معلق در شروع سرور زمانی که یک فرهنگ لغت بستگی به فرهنگ لغت دیگری از طریق یک پایگاه داده با موتور=فرهنگ لغت. [\#4962](https://github.com/ClickHouse/ClickHouse/pull/4962) ([ویتالی بارانو](https://github.com/vitlibar)) -- Partially fix distributed\_product\_mode = local. It’s possible to allow columns of local tables in where/having/order by/… via table aliases. Throw exception if table does not have alias. There’s not possible to access to the columns without table aliases yet. [\#4986](https://github.com/ClickHouse/ClickHouse/pull/4986) ([زویکوف](https://github.com/4ertus2)) +- Partially fix distributed\_product\_mode = local. It's possible to allow columns of local tables in where/having/order by/… via table aliases. Throw exception if table does not have alias. There's not possible to access to the columns without table aliases yet. [\#4986](https://github.com/ClickHouse/ClickHouse/pull/4986) ([زویکوف](https://github.com/4ertus2)) - رفع نتیجه به طور بالقوه اشتباه برای `SELECT DISTINCT` با `JOIN` [\#5001](https://github.com/ClickHouse/ClickHouse/pull/5001) ([زویکوف](https://github.com/4ertus2)) - ثابت شرایط مسابقه داده بسیار نادر است که می تواند در هنگام اجرای یک پرس و جو با اتحادیه همه شامل حداقل دو انتخاب از سیستم اتفاق می افتد.ستون, سیستم.جداول, سیستم.قطعات, سیستم.\_شبردارها یا جداول ادغام خانواده و انجام تغییر ستون از جداول مرتبط به صورت همزمان. [\#5189](https://github.com/ClickHouse/ClickHouse/pull/5189) ([الکسی میلویدو](https://github.com/alexey-milovidov)) @@ -1504,7 +1504,7 @@ toc_title: '2019' #### رفع اشکال {#bug-fixes-6} -- نوع ثابت تنظیم `max_partitions_per_insert_block` از بولی به UInt64. [\#5028](https://github.com/ClickHouse/ClickHouse/pull/5028) ([محمد حسین sekhavat](https://github.com/mhsekhavat)) +- نوع ثابت تنظیم `max_partitions_per_insert_block` از بولی به UInt64. [\#5028](https://github.com/ClickHouse/ClickHouse/pull/5028) ([محمد حسین Sekhavat](https://github.com/mhsekhavat)) ### انتشار کلیک خانه 19.5.2.6, 2019-04-15 {#clickhouse-release-19-5-2-6-2019-04-15} @@ -1582,7 +1582,7 @@ toc_title: '2019' #### بهبود عملکرد {#performance-improvement-6} -- بهینه سازی volnitsky جستجوگر توسط inlining دادن حدود 5-10% بهبود جستجو برای نمایش داده شد و با بسیاری از سوزن و یا بسیاری از شبیه bigrams. [\#4862](https://github.com/ClickHouse/ClickHouse/pull/4862) ([نام و نام خانوادگی](https://github.com/danlark1)) +- بهینه سازی Volnitsky جستجوگر توسط inlining دادن حدود 5-10% بهبود جستجو برای نمایش داده شد و با بسیاری از سوزن و یا بسیاری از شبیه bigrams. [\#4862](https://github.com/ClickHouse/ClickHouse/pull/4862) ([نام و نام خانوادگی](https://github.com/danlark1)) - رفع مشکل عملکرد در هنگام تنظیم `use_uncompressed_cache` بزرگتر از صفر است, که به نظر می رسد زمانی که همه اطلاعات به عنوان خوانده شده موجود در کش. [\#4913](https://github.com/ClickHouse/ClickHouse/pull/4913) ([الساپین](https://github.com/alesapin)) #### ساخت/تست / بهبود بسته بندی {#buildtestingpackaging-improvement-10} @@ -1643,7 +1643,7 @@ toc_title: '2019' - عادی نگه دارید, `DEFAULT`, `MATERIALIZED` و `ALIAS` ستون ها در یک لیست واحد (رفع مشکل [\#2867](https://github.com/ClickHouse/ClickHouse/issues/2867)). [\#4707](https://github.com/ClickHouse/ClickHouse/pull/4707) ([الکس زتلپین](https://github.com/ztlpn)) -### انتشار کلیک کنیدهاوس 19.4.3.11, 2019-04-02 {#clickhouse-release-19-4-3-11-2019-04-02} +### ClickHouse انتشار 19.4.3.11, 2019-04-02 {#clickhouse-release-19-4-3-11-2019-04-02} #### رفع اشکال {#bug-fixes-8} @@ -1715,7 +1715,7 @@ toc_title: '2019' - استفاده از جداول مراجعه مناسب است که با استفاده از رابط برنامه کاربردی هش را برای کلید 8 بیتی و 16 بیتی. [\#4536](https://github.com/ClickHouse/ClickHouse/pull/4536) ([ایموس پرنده](https://github.com/amosbird)) - بهبود عملکرد مقایسه رشته. [\#4564](https://github.com/ClickHouse/ClickHouse/pull/4564) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - پاکسازی صف دی ال در یک موضوع جداگانه توزیع به طوری که کم کردن سرعت حلقه اصلی که پردازش توزیع وظایف دسیدال نیست. [\#4502](https://github.com/ClickHouse/ClickHouse/pull/4502) ([الکس زتلپین](https://github.com/ztlpn)) -- زمانی که `min_bytes_to_use_direct_io` تنظیم شده است 1, هر فایل با حالت اچ باز شد چرا که اندازه داده ها به خواندن گاهی اوقات به اندازه یک بلوک فشرده دست کم گرفت. [\#4526](https://github.com/ClickHouse/ClickHouse/pull/4526) ([الکسی میلویدو](https://github.com/alexey-milovidov)) +- چه زمانی `min_bytes_to_use_direct_io` تنظیم شده است 1, هر فایل با حالت اچ باز شد چرا که اندازه داده ها به خواندن گاهی اوقات به اندازه یک بلوک فشرده دست کم گرفت. [\#4526](https://github.com/ClickHouse/ClickHouse/pull/4526) ([الکسی میلویدو](https://github.com/alexey-milovidov)) #### ساخت/تست / بهبود بسته بندی {#buildtestingpackaging-improvement-12} @@ -1787,7 +1787,7 @@ toc_title: '2019' #### رفع اشکال {#bug-fixes-16} -- ثابت `WITH ROLLUP` نتیجه برای گروه های تک `LowCardinality` کلیدی است. [\#4384](https://github.com/ClickHouse/ClickHouse/pull/4384) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) +- ثابت `WITH ROLLUP` نتیجه برای گروه های تک `LowCardinality` کلید [\#4384](https://github.com/ClickHouse/ClickHouse/pull/4384) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) - اشکال ثابت در شاخص مجموعه (حذف گرانول اگر شامل بیش از `max_rows` ردیف). [\#4386](https://github.com/ClickHouse/ClickHouse/pull/4386) ([نیکیتا واسیلیف](https://github.com/nikvas0)) - بسیاری از رفع بورس ساخت. [\#4397](https://github.com/ClickHouse/ClickHouse/pull/4397) ([پرولر](https://github.com/proller)) - جایگزینی نام مستعار ثابت در نمایش داده شد با زیرخاکی حاوی همان نام مستعار (موضوع [\#4110](https://github.com/ClickHouse/ClickHouse/issues/4110)). [\#4351](https://github.com/ClickHouse/ClickHouse/pull/4351) ([زویکوف](https://github.com/4ertus2)) @@ -1796,7 +1796,7 @@ toc_title: '2019' - اضافه کردن قابلیت اجرا `clickhouse-server` برای تست بدون تابعیت در تصویر کارگر بارانداز. [\#4347](https://github.com/ClickHouse/ClickHouse/pull/4347) ([واسیلی نمکو](https://github.com/Enmk)) -### انتشار تاتر 19.3.3, 2019-02-13 {#clickhouse-release-19-3-3-2019-02-13} +### ClickHouse انتشار 19.3.3, 2019-02-13 {#clickhouse-release-19-3-3-2019-02-13} #### ویژگی های جدید {#new-features-6} @@ -1815,7 +1815,7 @@ toc_title: '2019' - اضافه شده `IPv4` و `IPv6` انواع داده ها. پیاده سازی موثر تر از `IPv*` توابع. [\#3669](https://github.com/ClickHouse/ClickHouse/pull/3669) ([واسیلی نمکو](https://github.com/Enmk)) - اضافه شدن تابع `toStartOfTenMinutes()`. [\#4298](https://github.com/ClickHouse/ClickHouse/pull/4298) ([ویتالی بارانو](https://github.com/vitlibar)) - اضافه شده `Protobuf` فرمت خروجی. [\#4005](https://github.com/ClickHouse/ClickHouse/pull/4005) [\#4158](https://github.com/ClickHouse/ClickHouse/pull/4158) ([ویتالی بارانو](https://github.com/vitlibar)) -- اضافه شده brotli پشتیبانی از http رابط برای واردات داده ها (درج). [\#4235](https://github.com/ClickHouse/ClickHouse/pull/4235) ([میخیل](https://github.com/fandyushin)) +- اضافه شده brotli پشتیبانی از HTTP رابط برای واردات داده ها (درج). [\#4235](https://github.com/ClickHouse/ClickHouse/pull/4235) ([میخیل](https://github.com/fandyushin)) - نکات اضافه شده در حالی که کاربر را تایپی در نام تابع و یا نوع در مشتری خط فرمان. [\#4239](https://github.com/ClickHouse/ClickHouse/pull/4239) ([نام و نام خانوادگی](https://github.com/danlark1)) - اضافه شده `Query-Id` به هدر پاسخ قام سرور. [\#4231](https://github.com/ClickHouse/ClickHouse/pull/4231) ([میخیل](https://github.com/fandyushin)) @@ -1898,7 +1898,7 @@ toc_title: '2019' - در حال حاضر گزارش سرور پیشرفت برای زنده نگه داشتن اتصالات مشتری. [\#4215](https://github.com/ClickHouse/ClickHouse/pull/4215) ([ایوان](https://github.com/abyss7)) - پیام کمی بهتر با دلیل برای بهینه سازی پرس و جو با `optimize_throw_if_noop` تنظیم را فعال کنید. [\#4294](https://github.com/ClickHouse/ClickHouse/pull/4294) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - اضافه شدن پشتیبانی از `--version` گزینه ای برای سرور کلیک. [\#4251](https://github.com/ClickHouse/ClickHouse/pull/4251) ([لوپاتین کنستانتین](https://github.com/k-lopatin)) -- اضافه شده `--help/-h` گزینه ای برای `clickhouse-server`. [\#4233](https://github.com/ClickHouse/ClickHouse/pull/4233) ([یوری baranov](https://github.com/yurriy)) +- اضافه شده `--help/-h` گزینه ای برای `clickhouse-server`. [\#4233](https://github.com/ClickHouse/ClickHouse/pull/4233) ([یوری بارانوف](https://github.com/yurriy)) - اضافه شدن پشتیبانی برای زیرمجموعه اسکالر با تابع کل نتیجه دولت است. [\#4348](https://github.com/ClickHouse/ClickHouse/pull/4348) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) - بهبود زمان خاموش کردن سرور و تغییر زمان انتظار. [\#4372](https://github.com/ClickHouse/ClickHouse/pull/4372) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - اضافه شدن اطلاعات در مورد تنظیمات \_شروع مجدد به سیستم.کپی و اضافه کردن ورود به سیستم اگر ماکت نمی خواهد سعی کنید برای تبدیل شدن به رهبر. [\#4379](https://github.com/ClickHouse/ClickHouse/pull/4379) ([الکس زتلپین](https://github.com/ztlpn)) @@ -2071,4 +2071,4 @@ toc_title: '2019' - ثابت misspells در نظرات و string literals تحت `dbms`. [\#4122](https://github.com/ClickHouse/ClickHouse/pull/4122) ([مامان](https://github.com/maiha)) - غلط املایی ثابت در نظرات. [\#4089](https://github.com/ClickHouse/ClickHouse/pull/4089) ([اوگنی پراودا](https://github.com/kvinty)) -## [تغییرات برای 2018](https://github.com/ClickHouse/ClickHouse/blob/master/docs/en/changelog/2018.md) {#changelog-for-2018} +## [تغییرات برای 2018](./2018.md#clickhouse-release-18-16) {#changelog-for-2018} diff --git a/docs/fa/whats-new/changelog/index.md b/docs/fa/whats-new/changelog/index.md index ebf1ee992e0..28acaeef587 100644 --- a/docs/fa/whats-new/changelog/index.md +++ b/docs/fa/whats-new/changelog/index.md @@ -1,668 +1,9 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_folder_title: Changelog +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "\u062A\u063A\u06CC\u06CC\u0631\u0627\u062A" toc_priority: 74 toc_title: '2020' --- -## انتشار کلیک و 20.3 {#clickhouse-release-v20-3} - -### ClickHouse انتشار V20.3.4.10, 2020-03-20 {#clickhouse-release-v20-3-4-10-2020-03-20} - -#### رفع اشکال {#bug-fix} - -- این نسخه همچنین شامل تمام رفع اشکال از 20.1.8.41 -- رفع از دست رفته `rows_before_limit_at_least` برای نمایش داده شد بیش از قام (با پردازنده خط لوله). این رفع [\#9730](https://github.com/ClickHouse/ClickHouse/issues/9730). [\#9757](https://github.com/ClickHouse/ClickHouse/pull/9757) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) - -### انتشار کلیک و 20.3.3.6, 2020-03-17 {#clickhouse-release-v20-3-3-6-2020-03-17} - -#### رفع اشکال {#bug-fix-1} - -- این نسخه همچنین شامل تمام رفع اشکال از 20.1.7.38 -- رفع اشکال در تکرار می کند که تکرار اجازه نمی دهد به کار اگر کاربر جهش در نسخه های قبلی اجرا کرده است. این رفع [\#9645](https://github.com/ClickHouse/ClickHouse/issues/9645). [\#9652](https://github.com/ClickHouse/ClickHouse/pull/9652) ([الساپین](https://github.com/alesapin)). این باعث می شود نسخه 20.3 به عقب سازگار دوباره. -- افزودن تنظیمات `use_compact_format_in_distributed_parts_names` که اجازه می دهد برای نوشتن فایل برای `INSERT` نمایش داده شد به `Distributed` جدول با فرمت جمع و جور تر. این رفع [\#9647](https://github.com/ClickHouse/ClickHouse/issues/9647). [\#9653](https://github.com/ClickHouse/ClickHouse/pull/9653) ([الساپین](https://github.com/alesapin)). این باعث می شود نسخه 20.3 به عقب سازگار دوباره. - -### انتشار کلیک و 20.3.2.1, 2020-03-12 {#clickhouse-release-v20-3-2-1-2020-03-12} - -#### تغییر ناسازگار به عقب {#backward-incompatible-change} - -- ثابت موضوع `file name too long` هنگام ارسال داده ها برای `Distributed` جداول برای تعداد زیادی از کپی. ثابت این موضوع که اعتبار ماکت در ورود به سیستم سرور قرار گرفتند. فرمت نام دایرکتوری بر روی دیسک به تغییر یافت `[shard{shard_index}[_replica{replica_index}]]`. [\#8911](https://github.com/ClickHouse/ClickHouse/pull/8911) ([میخیل کوروتف](https://github.com/millb)) پس از شما به نسخه جدید ارتقا دهید, شما نمی قادر خواهد بود به جمع و جور کردن بدون دخالت دستی, به دلیل نسخه سرور قدیمی می کند فرمت دایرکتوری جدید به رسمیت نمی شناسد. اگر شما می خواهید به جمع و جور کردن, شما باید به صورت دستی تغییر نام دایرکتوری مربوطه را به فرمت های قدیمی. این تغییر مربوط است تنها اگر شما ناهمزمان استفاده کرده اند `INSERT`اس به `Distributed` میز در نسخه 20.3.3 ما یک محیط است که به شما اجازه فعال کردن فرمت جدید به تدریج معرفی. -- فرمت ورودی ورود به سیستم تکرار برای دستورات جهش تغییر کرده است. شما باید صبر کنید برای جهش های قدیمی برای پردازش قبل از نصب نسخه جدید. -- پیاده سازی پیشفیلتر حافظه ساده است که افسردگی زخم به `system.trace_log` هر بایت بر سر حد تخصیص نرم [\#8765](https://github.com/ClickHouse/ClickHouse/pull/8765) ([ایوان](https://github.com/abyss7)) [\#9472](https://github.com/ClickHouse/ClickHouse/pull/9472) ([الکسی میلویدو](https://github.com/alexey-milovidov)) ستون `system.trace_log` از تغییر نام داد `timer_type` به `trace_type`. این تغییرات در تجزیه و تحلیل عملکرد شخص ثالث و ابزارهای پردازش فلامگراف نیاز دارد. -- استفاده از شناسه موضوع سیستم عامل در همه جا به جای شماره موضوع داخلی. این رفع [\#7477](https://github.com/ClickHouse/ClickHouse/issues/7477) قدیمی `clickhouse-client` می توانید سیاهههای مربوط که از سرور ارسال زمانی که تنظیم دریافت نمی `send_logs_level` فعال است, چرا که نام و نوع پیام ورود به سیستم ساختار تغییر یافت. از سوی دیگر نسخه های سرور مختلف سیاهههای مربوط با انواع مختلف به یکدیگر ارسال کنید. هنگامی که شما از `send_logs_level` محیط, شما باید مهم نیست. [\#8954](https://github.com/ClickHouse/ClickHouse/pull/8954) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- حذف `indexHint` تابع [\#9542](https://github.com/ClickHouse/ClickHouse/pull/9542) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- حذف `findClusterIndex`, `findClusterValue` توابع. این رفع [\#8641](https://github.com/ClickHouse/ClickHouse/issues/8641). اگر شما با استفاده از این توابع, ارسال یک ایمیل به `clickhouse-feedback@yandex-team.com` [\#9543](https://github.com/ClickHouse/ClickHouse/pull/9543) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- در حال حاضر مجاز به ایجاد ستون و یا اضافه کردن ستون با `SELECT` زیرخاکری به عنوان عبارت پیش فرض. [\#9481](https://github.com/ClickHouse/ClickHouse/pull/9481) ([الساپین](https://github.com/alesapin)) -- نیاز نام مستعار برای کارخانه های فرعی در عضویت. [\#9274](https://github.com/ClickHouse/ClickHouse/pull/9274) ([زویکوف](https://github.com/4ertus2)) -- بهبود `ALTER MODIFY/ADD` نمایش داده شد منطق. حالا شما نمی توانید `ADD` ستون بدون نوع, `MODIFY` عبارت پیش فرض نوع ستون را تغییر نمی دهد و `MODIFY` نوع ارزش بیان پیش فرض را از دست نمی دهد. رفع [\#8669](https://github.com/ClickHouse/ClickHouse/issues/8669). [\#9227](https://github.com/ClickHouse/ClickHouse/pull/9227) ([الساپین](https://github.com/alesapin)) -- نیاز به سرور برای راه اندازی مجدد برای اعمال تغییرات در پیکربندی ورود به سیستم. این یک راه حل موقت برای جلوگیری از اشکال که سرور سیاهههای مربوط به یک فایل ورود به سیستم حذف شده است (دیدن [\#8696](https://github.com/ClickHouse/ClickHouse/issues/8696)). [\#8707](https://github.com/ClickHouse/ClickHouse/pull/8707) ([الکساندر کوزمنکوف](https://github.com/akuzm)) -- تنظیمات `experimental_use_processors` به طور پیش فرض فعال است. این تنظیم را قادر می سازد استفاده از خط لوله پرس و جو جدید. این فاکتورگیری مجدد داخلی است و ما انتظار داریم هیچ تغییری قابل مشاهده. اگر مشکلی خواهید دید صفر را تنظیم کنید. [\#8768](https://github.com/ClickHouse/ClickHouse/pull/8768) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - -#### ویژگی جدید {#new-feature} - -- افزودن `Avro` و `AvroConfluent` فرمت های ورودی/خروجی [\#8571](https://github.com/ClickHouse/ClickHouse/pull/8571) ([اندرو انیشچوک](https://github.com/oandrew)) [\#8957](https://github.com/ClickHouse/ClickHouse/pull/8957) ([اندرو انیشچوک](https://github.com/oandrew)) [\#8717](https://github.com/ClickHouse/ClickHouse/pull/8717) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- به روز رسانی چند رشته ای و غیر مسدود کردن کلید منقضی شده در `cache` واژهنامهها (با اجازه اختیاری به خواندن قدیمی). [\#8303](https://github.com/ClickHouse/ClickHouse/pull/8303) ([نیکیتا میخایلو](https://github.com/nikitamikhaylov)) -- افزودن پرسوجو `ALTER ... MATERIALIZE TTL`. این اجرا می شود جهش است که نیروهای به حذف داده های منقضی شده توسط کنترل از راه دور و دوباره حساب متا اطلاعات در مورد کنترل از راه دور در تمام نقاط. [\#8775](https://github.com/ClickHouse/ClickHouse/pull/8775) ([انتون پوپوف](https://github.com/CurtizJ)) -- سوئیچ از hashjoin به mergejoin (بر روی دیسک) در صورت نیاز [\#9082](https://github.com/ClickHouse/ClickHouse/pull/9082) ([زویکوف](https://github.com/4ertus2)) -- اضافه شده `MOVE PARTITION` فرمان برای `ALTER TABLE` [\#4729](https://github.com/ClickHouse/ClickHouse/issues/4729) [\#6168](https://github.com/ClickHouse/ClickHouse/pull/6168) ([کارخانه شراب سازی گیوم](https://github.com/YiuRULE)) -- بارگذاری مجدد پیکربندی ذخیره سازی از فایل پیکربندی در پرواز. [\#8594](https://github.com/ClickHouse/ClickHouse/pull/8594) ([ولادیمیر چبوتراف](https://github.com/excitoon)) -- مجاز به تغییر `storage_policy` به یکی کمتر غنی. [\#8107](https://github.com/ClickHouse/ClickHouse/pull/8107) ([ولادیمیر چبوتراف](https://github.com/excitoon)) -- اضافه شدن پشتیبانی برای کره / نویسه عام برای ذخیره سازی اس3 و عملکرد جدول. [\#8851](https://github.com/ClickHouse/ClickHouse/pull/8851) ([ولادیمیر چبوتراف](https://github.com/excitoon)) -- پیاده سازی `bitAnd`, `bitOr`, `bitXor`, `bitNot` برای `FixedString(N)` نوع داده. [\#9091](https://github.com/ClickHouse/ClickHouse/pull/9091) ([کارخانه شراب سازی گیوم](https://github.com/YiuRULE)) -- اضافه شدن تابع `bitCount`. این رفع [\#8702](https://github.com/ClickHouse/ClickHouse/issues/8702). [\#8708](https://github.com/ClickHouse/ClickHouse/pull/8708) ([الکسی میلویدو](https://github.com/alexey-milovidov)) [\#8749](https://github.com/ClickHouse/ClickHouse/pull/8749) ([درباره ما](https://github.com/ikopylov)) -- افزودن `generateRandom` تابع جدول برای تولید ردیف تصادفی با طرح داده شده است. اجازه می دهد تا به جمعیت جدول تست دلخواه با داده ها. [\#8994](https://github.com/ClickHouse/ClickHouse/pull/8994) ([ایلیا یاتسیشین](https://github.com/qoega)) -- `JSONEachRowFormat`: پشتیبانی از موارد خاص زمانی که اشیا محصور شده در مجموعه سطح بالا. [\#8860](https://github.com/ClickHouse/ClickHouse/pull/8860) ([کروگلو پاول](https://github.com/Avogar)) -- در حال حاضر امکان ایجاد یک ستون با `DEFAULT` عبارت که به یک ستون با پیش فرض بستگی دارد `ALIAS` اصطلاح. [\#9489](https://github.com/ClickHouse/ClickHouse/pull/9489) ([الساپین](https://github.com/alesapin)) -- اجازه مشخص کردن `--limit` بیش از اندازه منبع داده در `clickhouse-obfuscator`. داده ها خود را با دانه های مختلف تصادفی تکرار. [\#9155](https://github.com/ClickHouse/ClickHouse/pull/9155) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- اضافه شده `groupArraySample` تابع (شبیه به `groupArray`) با الگوریتم نمونه برداری ذخیره. [\#8286](https://github.com/ClickHouse/ClickHouse/pull/8286) ([ایموس پرنده](https://github.com/amosbird)) -- حالا شما می توانید اندازه صف به روز رسانی در نظارت `cache`/`complex_key_cache` واژهنامهها از طریق معیارهای سیستم. [\#9413](https://github.com/ClickHouse/ClickHouse/pull/9413) ([نیکیتا میخایلو](https://github.com/nikitamikhaylov)) -- اجازه دهید به عنوان یک جداساز خط در فرمت خروجی سی. سی. وی با تنظیم استفاده کنید `output_format_csv_crlf_end_of_line` به 1 تنظیم شده است [\#8934](https://github.com/ClickHouse/ClickHouse/pull/8934) [\#8935](https://github.com/ClickHouse/ClickHouse/pull/8935) [\#8963](https://github.com/ClickHouse/ClickHouse/pull/8963) ([میخیل کوروتف](https://github.com/millb)) -- اجرای توابع بیشتری از [H3](https://github.com/uber/h3) API: `h3GetBaseCell`, `h3HexAreaM2`, `h3IndexesAreNeighbors`, `h3ToChildren`, `h3ToString` و `stringToH3` [\#8938](https://github.com/ClickHouse/ClickHouse/pull/8938) ([نیکو مان پودری](https://github.com/nmandery)) -- تنظیمات جدید معرفی شده است: `max_parser_depth` برای کنترل حداکثر اندازه پشته و اجازه می دهد نمایش داده شد پیچیده بزرگ است. این رفع [\#6681](https://github.com/ClickHouse/ClickHouse/issues/6681) و [\#7668](https://github.com/ClickHouse/ClickHouse/issues/7668). [\#8647](https://github.com/ClickHouse/ClickHouse/pull/8647) ([ماکسیم اسمیرنوف](https://github.com/qMBQx8GH)) -- افزودن یک تنظیم `force_optimize_skip_unused_shards` تنظیم به پرتاب اگر پرش از خرده ریز استفاده نشده امکان پذیر نیست [\#8805](https://github.com/ClickHouse/ClickHouse/pull/8805) ([ازات خوژین](https://github.com/azat)) -- مجاز به پیکربندی چندین دیسک / حجم برای ذخیره سازی داده ها برای ارسال در `Distributed` موتور [\#8756](https://github.com/ClickHouse/ClickHouse/pull/8756) ([ازات خوژین](https://github.com/azat)) -- سیاست ذخیره سازی پشتیبانی (``)برای ذخیره سازی داده های موقت . [\#8750](https://github.com/ClickHouse/ClickHouse/pull/8750) ([ازات خوژین](https://github.com/azat)) -- اضافه شده `X-ClickHouse-Exception-Code` هدر قام است که اگر استثنا قبل از ارسال داده ها پرتاب شد. این پیاده سازی [\#4971](https://github.com/ClickHouse/ClickHouse/issues/4971). [\#8786](https://github.com/ClickHouse/ClickHouse/pull/8786) ([میخیل کوروتف](https://github.com/millb)) -- اضافه شدن تابع `ifNotFinite`. این فقط یک شکر نحوی است: `ifNotFinite(x, y) = isFinite(x) ? x : y`. [\#8710](https://github.com/ClickHouse/ClickHouse/pull/8710) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- اضافه شده `last_successful_update_time` ستون در `system.dictionaries` جدول [\#9394](https://github.com/ClickHouse/ClickHouse/pull/9394) ([نیکیتا میخایلو](https://github.com/nikitamikhaylov)) -- افزودن `blockSerializedSize` تابع (اندازه بر روی دیسک بدون فشرده سازی) [\#8952](https://github.com/ClickHouse/ClickHouse/pull/8952) ([ازات خوژین](https://github.com/azat)) -- افزودن تابع `moduloOrZero` [\#9358](https://github.com/ClickHouse/ClickHouse/pull/9358) ([هکز](https://github.com/hczhcz)) -- جداول سیستم اضافه شده است `system.zeros` و `system.zeros_mt` و همچنین توابع داستان `zeros()` و `zeros_mt()`. جداول (و توابع جدول) شامل یک ستون با نام `zero` و نوع `UInt8`. این ستون حاوی صفر. این است که برای اهداف تست به عنوان سریع ترین روش برای تولید بسیاری از ردیف مورد نیاز است. این رفع [\#6604](https://github.com/ClickHouse/ClickHouse/issues/6604) [\#9593](https://github.com/ClickHouse/ClickHouse/pull/9593) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) - -#### ویژگی تجربی {#experimental-feature} - -- اضافه کردن فرمت جمع و جور جدید از قطعات در `MergeTree`- جداول خانواده که در تمام ستون ها در یک فایل ذخیره می شود . این کمک می کند برای افزایش عملکرد درج کوچک و مکرر. فرمت قدیمی (یک فایل در هر ستون) در حال حاضر گسترده ای نامیده می شود. فرمت ذخیره سازی داده ها توسط تنظیمات کنترل می شود `min_bytes_for_wide_part` و `min_rows_for_wide_part`. [\#8290](https://github.com/ClickHouse/ClickHouse/pull/8290) ([انتون پوپوف](https://github.com/CurtizJ)) -- پشتیبانی از ذخیره سازی اس 3 برای `Log`, `TinyLog` و `StripeLog` میز [\#8862](https://github.com/ClickHouse/ClickHouse/pull/8862) ([پاول کووالنکو](https://github.com/Jokser)) - -#### رفع اشکال {#bug-fix-2} - -- فضاهای خالی متناقض ثابت در پیام های ورود به سیستم. [\#9322](https://github.com/ClickHouse/ClickHouse/pull/9322) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- رفع اشکال که در مجموعه ای از تاپل نامش ذکر نشده به عنوان ساختارهای تو در تو در ایجاد جدول پهن شد. [\#8866](https://github.com/ClickHouse/ClickHouse/pull/8866) ([اچولکوف2](https://github.com/achulkov2)) -- ثابت موضوع زمانی که “Too many open files” خطا ممکن است رخ دهد اگر بیش از حد بسیاری از فایل های تطبیق الگوی لکه در وجود دارد `File` جدول یا `file` تابع جدول. در حال حاضر فایل ها باز تنبلی. این رفع [\#8857](https://github.com/ClickHouse/ClickHouse/issues/8857) [\#8861](https://github.com/ClickHouse/ClickHouse/pull/8861) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- قطره جدول موقت در حال حاضر قطره تنها جدول موقت. [\#8907](https://github.com/ClickHouse/ClickHouse/pull/8907) ([ویتالی بارانو](https://github.com/vitlibar)) -- حذف پارتیشن منسوخ شده زمانی که ما خاموش کردن سرور و یا جدا / ضمیمه یک جدول. [\#8602](https://github.com/ClickHouse/ClickHouse/pull/8602) ([کارخانه شراب سازی گیوم](https://github.com/YiuRULE)) -- برای چگونه دیسک پیش فرض محاسبه فضای رایگان از `data` شاخه فرعی. ثابت موضوع زمانی که مقدار فضای رایگان به درستی محاسبه نمی شود اگر `data` دایرکتوری به یک دستگاه جداگانه (مورد نادر) نصب شده است. این رفع [\#7441](https://github.com/ClickHouse/ClickHouse/issues/7441) [\#9257](https://github.com/ClickHouse/ClickHouse/pull/9257) ([میخیل کوروتف](https://github.com/millb)) -- اجازه کاما (صلیب) عضویت با در () داخل. [\#9251](https://github.com/ClickHouse/ClickHouse/pull/9251) ([زویکوف](https://github.com/4ertus2)) -- اجازه می دهد به بازنویسی صلیب به درونی ملحق اگر وجود دارد \[نه\] مانند اپراتور در جایی که بخش. [\#9229](https://github.com/ClickHouse/ClickHouse/pull/9229) ([زویکوف](https://github.com/4ertus2)) -- رفع نتیجه نادرست ممکن است پس از `GROUP BY` با تنظیم فعال `distributed_aggregation_memory_efficient`. رفع [\#9134](https://github.com/ClickHouse/ClickHouse/issues/9134). [\#9289](https://github.com/ClickHouse/ClickHouse/pull/9289) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) -- کلید پیدا شده است به عنوان در معیارهای لغت نامه کش از دست رفته شمارش شد. [\#9411](https://github.com/ClickHouse/ClickHouse/pull/9411) ([نیکیتا میخایلو](https://github.com/nikitamikhaylov)) -- رفع ناسازگاری پروتکل تکرار معرفی شده در [\#8598](https://github.com/ClickHouse/ClickHouse/issues/8598). [\#9412](https://github.com/ClickHouse/ClickHouse/pull/9412) ([الساپین](https://github.com/alesapin)) -- شرایط مسابقه ثابت در `queue_task_handle` در هنگام راه اندازی `ReplicatedMergeTree` میز [\#9552](https://github.com/ClickHouse/ClickHouse/pull/9552) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- رمز `NOT` جواب نداد `SHOW TABLES NOT LIKE` پرسوجو [\#8727](https://github.com/ClickHouse/ClickHouse/issues/8727) [\#8940](https://github.com/ClickHouse/ClickHouse/pull/8940) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- اضافه شدن محدوده چک به تابع `h3EdgeLengthM`. بدون این چک, سرریز بافر امکان پذیر است. [\#8945](https://github.com/ClickHouse/ClickHouse/pull/8945) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- رفع اشکال در محاسبات باریکش از عملیات منطقی سه تایی در استدلال های متعدد (بیش از 10). [\#8718](https://github.com/ClickHouse/ClickHouse/pull/8718) ([الکساندر کازاکوف](https://github.com/Akazz)) -- رفع خطای prewhere بهینه سازی است که می تواند منجر به segfaults یا `Inconsistent number of columns got from MergeTreeRangeReader` استثنا. [\#9024](https://github.com/ClickHouse/ClickHouse/pull/9024) ([انتون پوپوف](https://github.com/CurtizJ)) -- رفع غیر منتظره `Timeout exceeded while reading from socket` استثنا, که به طور تصادفی در اتصال امن اتفاق می افتد قبل از ایست در واقع بیش از حد و هنگامی که پیشفیلتر پرس و جو فعال است. همچنین اضافه کنید `connect_timeout_with_failover_secure_ms` تنظیمات (به طور پیش فرض 100 مگابایت), که شبیه به است `connect_timeout_with_failover_ms`, اما برای اتصالات امن استفاده می شود (به دلیل دست دادن اس اس ال کندتر است, از اتصال تی پی معمولی) [\#9026](https://github.com/ClickHouse/ClickHouse/pull/9026) ([تاولوبیکس](https://github.com/tavplubix)) -- رفع اشکال با جهش نهایی, زمانی که جهش ممکن است در حالت قطع با `parts_to_do=0` و `is_done=0`. [\#9022](https://github.com/ClickHouse/ClickHouse/pull/9022) ([الساپین](https://github.com/alesapin)) -- استفاده از جدید هر پیوستن به منطق با `partial_merge_join` تنظیمات. این ممکن است به `ANY|ALL|SEMI LEFT` و `ALL INNER` می پیوندد با `partial_merge_join=1` حالا [\#8932](https://github.com/ClickHouse/ClickHouse/pull/8932) ([زویکوف](https://github.com/4ertus2)) -- سفال در حال حاضر بست تنظیمات کردم از مبتکر به مواضع سفال به جای پرتاب یک استثنا. این ثابت اجازه می دهد تا برای ارسال نمایش داده شد به یک سفال با محدودیت های دیگر. [\#9447](https://github.com/ClickHouse/ClickHouse/pull/9447) ([ویتالی بارانو](https://github.com/vitlibar)) -- مشکل مدیریت حافظه ثابت در `MergeTreeReadPool`. [\#8791](https://github.com/ClickHouse/ClickHouse/pull/8791) ([ولادیمیر چبوتراف](https://github.com/excitoon)) -- ثابت `toDecimal*OrNull()` توابع خانواده که با رشته به نام `e`. رفع [\#8312](https://github.com/ClickHouse/ClickHouse/issues/8312) [\#8764](https://github.com/ClickHouse/ClickHouse/pull/8764) ([زویکوف](https://github.com/4ertus2)) -- اطمینان حاصل کنید که `FORMAT Null` هیچ اطلاعاتی را به مشتری ارسال نمی کند. [\#8767](https://github.com/ClickHouse/ClickHouse/pull/8767) ([الکساندر کوزمنکوف](https://github.com/akuzm)) -- رفع اشکال که برچسب زمان در `LiveViewBlockInputStream` به روز نمی. `LIVE VIEW` یکی از ویژگی های تجربی است. [\#8644](https://github.com/ClickHouse/ClickHouse/pull/8644) ([وکسیدر](https://github.com/Vxider)) [\#8625](https://github.com/ClickHouse/ClickHouse/pull/8625) ([وکسیدر](https://github.com/Vxider)) -- ثابت `ALTER MODIFY TTL` رفتار اشتباه است که اجازه نمی دهد به حذف عبارات قدیمی. [\#8422](https://github.com/ClickHouse/ClickHouse/pull/8422) ([ولادیمیر چبوتراف](https://github.com/excitoon)) -- گزارش ثابت اوبسان در ادغام. این رفع [\#9250](https://github.com/ClickHouse/ClickHouse/issues/9250) [\#9365](https://github.com/ClickHouse/ClickHouse/pull/9365) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- رفتار را ثابت کرد `match` و `extract` توابع زمانی که کومه علف خشک است صفر بایت. رفتار اشتباه بود که کومه علف خشک ثابت بود. این رفع [\#9160](https://github.com/ClickHouse/ClickHouse/issues/9160) [\#9163](https://github.com/ClickHouse/ClickHouse/pull/9163) ([الکسی میلویدو](https://github.com/alexey-milovidov)) [\#9345](https://github.com/ClickHouse/ClickHouse/pull/9345) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- اجتناب از پرتاب از تخریب کننده در کتابخانه ’ 3-حزب خود نمایی. [\#9066](https://github.com/ClickHouse/ClickHouse/pull/9066) ([اندرو انیشچوک](https://github.com/oandrew)) -- هنوز یک دسته نظرسنجی از مرتکب نشده `Kafka` تا حدی که می تواند به سوراخ در داده ها منجر شود. [\#8876](https://github.com/ClickHouse/ClickHouse/pull/8876) ([فیلیمونف](https://github.com/filimonov)) -- ثابت `joinGet` با انواع بازگشت باطل. https://github.com/ClickHouse/ClickHouse/issues/8919 [\#9014](https://github.com/ClickHouse/ClickHouse/pull/9014) ([ایموس پرنده](https://github.com/amosbird)) -- رفع ناسازگاری داده ها در هنگام فشرده با `T64` وابسته به کدک. [\#9016](https://github.com/ClickHouse/ClickHouse/pull/9016) ([زویکوف](https://github.com/4ertus2)) رفع شناسه نوع داده در `T64` کدک فشرده سازی است که منجر به اشتباه (د) فشرده سازی در نسخه های تحت تاثیر قرار. [\#9033](https://github.com/ClickHouse/ClickHouse/pull/9033) ([زویکوف](https://github.com/4ertus2)) -- افزودن تنظیمات `enable_early_constant_folding` و در برخی موارد که منجر به خطا غیر فعال کنید. [\#9010](https://github.com/ClickHouse/ClickHouse/pull/9010) ([زویکوف](https://github.com/4ertus2)) -- رفع pushdown گزاره optimizer با مشاهده و فعال کردن آزمون [\#9011](https://github.com/ClickHouse/ClickHouse/pull/9011) ([زمستان ژانگ](https://github.com/zhang2014)) -- رفع segfault در `Merge` جداول, که می تواند رخ دهد در هنگام خواندن از `File` ذخیره سازی [\#9387](https://github.com/ClickHouse/ClickHouse/pull/9387) ([تاولوبیکس](https://github.com/tavplubix)) -- اضافه شدن یک چک برای سیاست ذخیره سازی در `ATTACH PARTITION FROM`, `REPLACE PARTITION`, `MOVE TO TABLE`. در غیر این صورت می تواند داده ها از بخش غیر قابل دسترس پس از راه اندازی مجدد و جلوگیری از کلیک برای شروع. [\#9383](https://github.com/ClickHouse/ClickHouse/pull/9383) ([ولادیمیر چبوتراف](https://github.com/excitoon)) -- اصلاح تغییر می دهد اگر برای جدول تنظیم شده است. [\#8800](https://github.com/ClickHouse/ClickHouse/pull/8800) ([انتون پوپوف](https://github.com/CurtizJ)) -- رفع شرایط مسابقه است که می تواند رخ دهد که `SYSTEM RELOAD ALL DICTIONARIES` اعدام در حالی که برخی از فرهنگ لغت است که اصلاح/اضافه/حذف شده است. [\#8801](https://github.com/ClickHouse/ClickHouse/pull/8801) ([ویتالی بارانو](https://github.com/vitlibar)) -- در نسخه های قبلی `Memory` موتور پایگاه داده استفاده از مسیر داده خالی, بنابراین جداول در ایجاد `path` directory (e.g. `/var/lib/clickhouse/`), not in data directory of database (e.g. `/var/lib/clickhouse/db_name`). [\#8753](https://github.com/ClickHouse/ClickHouse/pull/8753) ([تاولوبیکس](https://github.com/tavplubix)) -- ثابت پیام ورود به سیستم اشتباه در مورد از دست رفته دیسک به طور پیش فرض و یا سیاست. [\#9530](https://github.com/ClickHouse/ClickHouse/pull/9530) ([ولادیمیر چبوتراف](https://github.com/excitoon)) -- ثابت نیست (است ()) برای شاخص نفخ\_فیلتر از انواع مجموعه. [\#9407](https://github.com/ClickHouse/ClickHouse/pull/9407) ([ایشیمب](https://github.com/achimbab)) -- اجازه دادن ستون اول (ها) در یک جدول با `Log` موتور یک نام مستعار است [\#9231](https://github.com/ClickHouse/ClickHouse/pull/9231) ([ایوان](https://github.com/abyss7)) -- رفع منظور از محدوده در حالی که خواندن از `MergeTree` جدول در یک موضوع. این می تواند به استثنا از منجر `MergeTreeRangeReader` یا نتایج پرس و جو اشتباه است. [\#9050](https://github.com/ClickHouse/ClickHouse/pull/9050) ([انتون پوپوف](https://github.com/CurtizJ)) -- ساخت `reinterpretAsFixedString` برای بازگشت `FixedString` به جای `String`. [\#9052](https://github.com/ClickHouse/ClickHouse/pull/9052) ([اندرو انیشچوک](https://github.com/oandrew)) -- اجتناب از موارد بسیار نادر زمانی که کاربر می تواند پیغام خطا اشتباه دریافت کنید (`Success` به جای شرح خطا دقیق). [\#9457](https://github.com/ClickHouse/ClickHouse/pull/9457) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- هنگام استفاده از تصادف نکنید `Template` قالب با قالب ردیف خالی. [\#8785](https://github.com/ClickHouse/ClickHouse/pull/8785) ([الکساندر کوزمنکوف](https://github.com/akuzm)) -- فایل های ابرداده برای جداول سیستم را می توان در جای اشتباه ایجاد شده است [\#8653](https://github.com/ClickHouse/ClickHouse/pull/8653) ([تاولوبیکس](https://github.com/tavplubix)) رفع [\#8581](https://github.com/ClickHouse/ClickHouse/issues/8581). -- رفع مسابقه داده ها به استثنای در فرهنگ لغت کش [\#8303](https://github.com/ClickHouse/ClickHouse/issues/8303). [\#9379](https://github.com/ClickHouse/ClickHouse/pull/9379) ([نیکیتا میخایلو](https://github.com/nikitamikhaylov)) -- یک استثنا برای پرس و جو پرتاب نکنید `ATTACH TABLE IF NOT EXISTS`. پیش از این پرتاب شد اگر جدول در حال حاضر وجود دارد, با وجود `IF NOT EXISTS` بند بند. [\#8967](https://github.com/ClickHouse/ClickHouse/pull/8967) ([انتون پوپوف](https://github.com/CurtizJ)) -- ثابت بسته شدن از دست رفته پین در پیام استثنا. [\#8811](https://github.com/ClickHouse/ClickHouse/pull/8811) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- اجتناب از پیام `Possible deadlock avoided` در راه اندازی تاتر مشتری در حالت تعاملی. [\#9455](https://github.com/ClickHouse/ClickHouse/pull/9455) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- ثابت موضوع زمانی که بالشتک در پایان پایگاه64 ارزش کد گذاری را می توان ناقص. به روز رسانی پایگاه64 کتابخانه. این رفع [\#9491](https://github.com/ClickHouse/ClickHouse/issues/9491) بسته [\#9492](https://github.com/ClickHouse/ClickHouse/issues/9492) [\#9500](https://github.com/ClickHouse/ClickHouse/pull/9500) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- جلوگیری از از دست دادن داده ها در `Kafka` در موارد نادر زمانی که استثنا اتفاق می افتد پس از خواندن پسوند اما قبل از ارتکاب. رفع [\#9378](https://github.com/ClickHouse/ClickHouse/issues/9378) [\#9507](https://github.com/ClickHouse/ClickHouse/pull/9507) ([فیلیمونف](https://github.com/filimonov)) -- استثنا ثابت در `DROP TABLE IF EXISTS` [\#8663](https://github.com/ClickHouse/ClickHouse/pull/8663) ([نیکیتا واسیلیف](https://github.com/nikvas0)) -- رفع سقوط زمانی که یک کاربر تلاش می کند `ALTER MODIFY SETTING` برای قدیمی شکل گرفته `MergeTree` موتورهای جدول خانواده. [\#9435](https://github.com/ClickHouse/ClickHouse/pull/9435) ([الساپین](https://github.com/alesapin)) -- پشتیبانی از uint64 اعداد است که مناسب نیست در int64 در json-توابع مربوط. به روز رسانی سیمدجسون به استاد. این رفع [\#9209](https://github.com/ClickHouse/ClickHouse/issues/9209) [\#9344](https://github.com/ClickHouse/ClickHouse/pull/9344) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- ثابت اعدام inversed predicates که غیر به شدت monotinic عملکردی شاخص استفاده شده است. [\#9223](https://github.com/ClickHouse/ClickHouse/pull/9223) ([الکساندر کازاکوف](https://github.com/Akazz)) -- سعی نکنید به برابر `IN` ثابت در `GROUP BY` [\#8868](https://github.com/ClickHouse/ClickHouse/pull/8868) ([ایموس پرنده](https://github.com/amosbird)) -- رفع اشکال در `ALTER DELETE` جهش که منجر به شاخص فساد. این رفع [\#9019](https://github.com/ClickHouse/ClickHouse/issues/9019) و [\#8982](https://github.com/ClickHouse/ClickHouse/issues/8982). علاوه بر این رفع شرایط مسابقه بسیار نادر در `ReplicatedMergeTree` `ALTER` نمایش داده شد. [\#9048](https://github.com/ClickHouse/ClickHouse/pull/9048) ([الساپین](https://github.com/alesapin)) -- هنگامی که تنظیمات `compile_expressions` فعال است, شما می توانید `unexpected column` داخل `LLVMExecutableFunction` هنگامی که ما با استفاده از `Nullable` نوع [\#8910](https://github.com/ClickHouse/ClickHouse/pull/8910) ([کارخانه شراب سازی گیوم](https://github.com/YiuRULE)) -- رفع چندگانه برای `Kafka` موتور: 1) ثابت تکراری که در طول توازن گروه مصرف کننده ظاهر شد. 2) رفع نادر ‘holes’ به نظر می رسد زمانی که داده ها از چندین پارتیشن با یک نظرسنجی نظرسنجی نظرسنجی نظرسنجی شد و تا حدی متعهد (در حال حاضر ما همیشه پردازش / مرتکب کل نظرسنجی بلوک از پیام). 3) رفع حملات گرگرفتگی با اندازه بلوک (قبل از که تنها با فاصله گرگرفتگی به درستی کار می کرد). 4) روش اشتراک بهتر (با بازخورد انتساب). 5) را تست کار سریع تر (با فواصل پیش فرض و وقفه). با توجه به این واقعیت است که داده ها توسط اندازه بلوک قبل از سرخ نیست (همانطور که باید با توجه به اسناد و مدارک), که روابط عمومی ممکن است به برخی از تخریب عملکرد با تنظیمات پیش فرض منجر شود (با توجه به بیشتر & حملات گرگرفتگی قلع که کمتر بهینه هستند). اگر موضوع عملکرد شما روبرو می شوند که پس از تغییر - لطفا افزایش می دهد `kafka_max_block_size` در جدول به ارزش بزرگتر ( به عنوان مثال `CREATE TABLE ...Engine=Kafka ... SETTINGS ... kafka_max_block_size=524288`). رفع [\#7259](https://github.com/ClickHouse/ClickHouse/issues/7259) [\#8917](https://github.com/ClickHouse/ClickHouse/pull/8917) ([فیلیمونف](https://github.com/filimonov)) -- ثابت `Parameter out of bound` استثنا در برخی از نمایش داده شد پس از بهینه سازی در همه جا. [\#8914](https://github.com/ClickHouse/ClickHouse/pull/8914) ([Baudouin Giard](https://github.com/bgiard)) -- ثابت مورد مخلوط ثابت استدلال از تابع `arrayZip`. [\#8705](https://github.com/ClickHouse/ClickHouse/pull/8705) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- هنگام اجرای `CREATE` پرس و جو, برابر عبارات ثابت در استدلال موتور ذخیره سازی. جایگزین کردن نام دادگان خالی با دادگان فعلی. رفع [\#6508](https://github.com/ClickHouse/ClickHouse/issues/6508), [\#3492](https://github.com/ClickHouse/ClickHouse/issues/3492) [\#9262](https://github.com/ClickHouse/ClickHouse/pull/9262) ([تاولوبیکس](https://github.com/tavplubix)) -- در حال حاضر امکان ایجاد یا اضافه کردن ستون ها با نام مستعار چرخه ای ساده مانند وجود ندارد `a DEFAULT b, b DEFAULT a`. [\#9603](https://github.com/ClickHouse/ClickHouse/pull/9603) ([الساپین](https://github.com/alesapin)) -- رفع اشکال با حرکت دو که ممکن است بخش اصلی فاسد است. این مربوط است اگر شما استفاده کنید `ALTER TABLE MOVE` [\#8680](https://github.com/ClickHouse/ClickHouse/pull/8680) ([ولادیمیر چبوتراف](https://github.com/excitoon)) -- اجازه داده شود `interval` شناسه به درستی تجزیه و تحلیل بدون پشت. موضوع ثابت زمانی که پرس و جو نمی تواند اجرا شود حتی اگر `interval` شناسه در پشت و یا به نقل از دو محصور. این رفع [\#9124](https://github.com/ClickHouse/ClickHouse/issues/9124). [\#9142](https://github.com/ClickHouse/ClickHouse/pull/9142) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- تست ریش ریش شدن ثابت و رفتار نادرست از `bitTestAll`/`bitTestAny` توابع. [\#9143](https://github.com/ClickHouse/ClickHouse/pull/9143) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- رفع سقوط احتمالی / تعداد اشتباه ردیف در `LIMIT n WITH TIES` هنگامی که بسیاری از ردیف به نفر ردیف برابر وجود دارد. [\#9464](https://github.com/ClickHouse/ClickHouse/pull/9464) ([تاولوبیکس](https://github.com/tavplubix)) -- رفع جهش با قطعات نوشته شده با فعال `insert_quorum`. [\#9463](https://github.com/ClickHouse/ClickHouse/pull/9463) ([الساپین](https://github.com/alesapin)) -- رفع مسابقه داده ها در نابودی `Poco::HTTPServer`. این می تواند رخ دهد زمانی که سرور شروع شده است و بلافاصله تعطیل. [\#9468](https://github.com/ClickHouse/ClickHouse/pull/9468) ([انتون پوپوف](https://github.com/CurtizJ)) -- رفع اشکال که در هنگام اجرای پیغام خطای گمراه کننده نشان داده شده است `SHOW CREATE TABLE a_table_that_does_not_exist`. [\#8899](https://github.com/ClickHouse/ClickHouse/pull/8899) ([اچولکوف2](https://github.com/achulkov2)) -- ثابت `Parameters are out of bound` استثنا در برخی موارد نادر زمانی که ما یک ثابت در `SELECT` بند زمانی که ما یک `ORDER BY` و یک `LIMIT` بند بند. [\#8892](https://github.com/ClickHouse/ClickHouse/pull/8892) ([کارخانه شراب سازی گیوم](https://github.com/YiuRULE)) -- رفع جهش نهایی, زمانی که در حال حاضر انجام جهش می تواند وضعیت را داشته باشد `is_done=0`. [\#9217](https://github.com/ClickHouse/ClickHouse/pull/9217) ([الساپین](https://github.com/alesapin)) -- جلوگیری از اجرای `ALTER ADD INDEX` برای جداول ادغام با نحو قدیمی, چرا که کار نمی کند. [\#8822](https://github.com/ClickHouse/ClickHouse/pull/8822) ([میخیل کوروتف](https://github.com/millb)) -- در هنگام راه اندازی سرور جدول دسترسی پیدا کنید, که `LIVE VIEW` بستگی دارد, بنابراین سرور قادر به شروع خواهد بود. همچنین حذف `LIVE VIEW` وابستگی زمانی که جدا `LIVE VIEW`. `LIVE VIEW` یکی از ویژگی های تجربی است. [\#8824](https://github.com/ClickHouse/ClickHouse/pull/8824) ([تاولوبیکس](https://github.com/tavplubix)) -- رفع اشکال احتمالی در `MergeTreeRangeReader`, در حالی که اجرای `PREWHERE`. [\#9106](https://github.com/ClickHouse/ClickHouse/pull/9106) ([انتون پوپوف](https://github.com/CurtizJ)) -- رفع چک عدم تطابق ممکن است با ستون. [\#9451](https://github.com/ClickHouse/ClickHouse/pull/9451) ([انتون پوپوف](https://github.com/CurtizJ)) -- رفع اشکال زمانی که قطعات شد که در پس زمینه با قوانین تغییر زمان در مورد نقل مکان کرد زمانی که تنها یک حجم وجود دارد. [\#8672](https://github.com/ClickHouse/ClickHouse/pull/8672) ([ولادیمیر چبوتراف](https://github.com/excitoon)) -- ثابت موضوع `Method createColumn() is not implemented for data type Set`. این رفع [\#7799](https://github.com/ClickHouse/ClickHouse/issues/7799). [\#8674](https://github.com/ClickHouse/ClickHouse/pull/8674) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- در حال حاضر ما سعی خواهد کرد نهایی جهش بیشتر. [\#9427](https://github.com/ClickHouse/ClickHouse/pull/9427) ([الساپین](https://github.com/alesapin)) -- ثابت `intDiv` توسط منهای یک ثابت [\#9351](https://github.com/ClickHouse/ClickHouse/pull/9351) ([هکز](https://github.com/hczhcz)) -- رفع شرایط مسابقه ممکن است در `BlockIO`. [\#9356](https://github.com/ClickHouse/ClickHouse/pull/9356) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) -- رفع اشکال منجر به ختم سرور در هنگام تلاش برای استفاده / رها کردن `Kafka` جدول ایجاد شده با اشتباه پارامترهای. [\#9513](https://github.com/ClickHouse/ClickHouse/pull/9513) ([فیلیمونف](https://github.com/filimonov)) -- اضافه شده راه حل اگر سیستم عامل نتیجه اشتباه را برمی گرداند `timer_create` تابع. [\#8837](https://github.com/ClickHouse/ClickHouse/pull/8837) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- خطای ثابت در استفاده از `min_marks_for_seek` پارامتر. ثابت پیام خطا زمانی که هیچ کلید شارژ در جدول توزیع وجود دارد و ما سعی می کنیم به جست و خیز خرده ریز استفاده نشده. [\#8908](https://github.com/ClickHouse/ClickHouse/pull/8908) ([ازات خوژین](https://github.com/azat)) - -#### بهبود {#improvement} - -- پیاده سازی `ALTER MODIFY/DROP` نمایش داده شد در بالای جهش برای `ReplicatedMergeTree*` خانواده موتور. حالا `ALTERS` بلوک تنها در مرحله به روز رسانی ابرداده, و بعد از که مسدود نمی. [\#8701](https://github.com/ClickHouse/ClickHouse/pull/8701) ([الساپین](https://github.com/alesapin)) -- اضافه کردن توانایی بازنویسی صلیب به درونی می پیوندد با `WHERE` بخش حاوی نام بدون تغییر. [\#9512](https://github.com/ClickHouse/ClickHouse/pull/9512) ([زویکوف](https://github.com/4ertus2)) -- ساخت `SHOW TABLES` و `SHOW DATABASES` نمایش داده شد حمایت از `WHERE` عبارات و `FROM`/`IN` [\#9076](https://github.com/ClickHouse/ClickHouse/pull/9076) ([بستنی و مغز گردو](https://github.com/sundy-li)) -- اضافه شدن یک تنظیم `deduplicate_blocks_in_dependent_materialized_views`. [\#9070](https://github.com/ClickHouse/ClickHouse/pull/9070) ([اوریخی](https://github.com/urykhy)) -- پس از تغییرات اخیر مشتری خروجی زیر شروع به چاپ رشته های باینری در سحر و جادو در نتیجه ساخت قابل خواندن نیست ([\#9032](https://github.com/ClickHouse/ClickHouse/issues/9032)). راه حل در محل کلیک است به علامت ستون رشته به عنوان سخن گفتن-8, که همیشه نمی, اما معمولا مورد. [\#9079](https://github.com/ClickHouse/ClickHouse/pull/9079) ([یوری بارانوف](https://github.com/yurriy)) -- اضافه کردن پشتیبانی از رشته و کلید رشته برای `sumMap` [\#8903](https://github.com/ClickHouse/ClickHouse/pull/8903) ([Baudouin Giard](https://github.com/bgiard)) -- کلید های رشته پشتیبانی در نقشه های خلاصه [\#8933](https://github.com/ClickHouse/ClickHouse/pull/8933) ([Baudouin Giard](https://github.com/bgiard)) -- سیگنال ختم موضوع به استخر موضوع حتی اگر موضوع استثنا پرتاب کرده است [\#8736](https://github.com/ClickHouse/ClickHouse/pull/8736) ([هشدار داده می شود](https://github.com/dingxiangfei2009)) -- مجاز به تنظیم `query_id` داخل `clickhouse-benchmark` [\#9416](https://github.com/ClickHouse/ClickHouse/pull/9416) ([انتون پوپوف](https://github.com/CurtizJ)) -- اجازه نمی دهد عبارات عجیب و غریب در `ALTER TABLE ... PARTITION partition` پرس و جو. این آدرس [\#7192](https://github.com/ClickHouse/ClickHouse/issues/7192) [\#8835](https://github.com/ClickHouse/ClickHouse/pull/8835) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- جدول `system.table_engines` در حال حاضر اطلاعات در مورد پشتیبانی از ویژگی فراهم می کند (مانند `supports_ttl` یا `supports_sort_order`). [\#8830](https://github.com/ClickHouse/ClickHouse/pull/8830) ([مکس اخمدوف](https://github.com/zlobober)) -- فعالسازی `system.metric_log` به طور پیش فرض. آن را شامل ردیف های با ارزش از ProfileEvents, CurrentMetrics جمع آوری شده با “collect\_interval\_milliseconds” فاصله (یک ثانیه به طور پیش فرض). جدول بسیار کوچک است (معمولا به ترتیب مگابایت) و جمع اوری این داده ها به طور پیش فرض معقول است. [\#9225](https://github.com/ClickHouse/ClickHouse/pull/9225) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- Initialize query profiler for all threads in a group, e.g. it allows to fully profile insert-queries. Fixes [\#6964](https://github.com/ClickHouse/ClickHouse/issues/6964) [\#8874](https://github.com/ClickHouse/ClickHouse/pull/8874) ([ایوان](https://github.com/abyss7)) -- در حال حاضر موقت `LIVE VIEW` ایجاد شده توسط `CREATE LIVE VIEW name WITH TIMEOUT [42] ...` به جای `CREATE TEMPORARY LIVE VIEW ...`, چرا که نحو قبلی بود سازگار با نیست `CREATE TEMPORARY TABLE ...` [\#9131](https://github.com/ClickHouse/ClickHouse/pull/9131) ([تاولوبیکس](https://github.com/tavplubix)) -- اضافه کردن \_خروج.پارامتر پیکربندی سطح برای محدود کردن ورودی که می رود به `system.text_log` جدول [\#8809](https://github.com/ClickHouse/ClickHouse/pull/8809) ([ازات خوژین](https://github.com/azat)) -- اجازه می دهد برای قرار دادن بخش دانلود شده به دیسک/حجم با توجه به قوانین تی ال [\#8598](https://github.com/ClickHouse/ClickHouse/pull/8598) ([ولادیمیر چبوتراف](https://github.com/excitoon)) -- برای لغت نامه خروجی زیر خارجی, اجازه می دهد برای تغییر دادن خروجی زیر استخر اتصال به “share” در میان واژهنامهها. این گزینه به طور قابل توجهی تعداد اتصالات به سرور خروجی زیر را کاهش می دهد. [\#9409](https://github.com/ClickHouse/ClickHouse/pull/9409) ([Clément Rodriguez](https://github.com/clemrodriguez)) -- نمایش نزدیکترین زمان اجرای پرس و جو برای کوانتوم در `clickhouse-benchmark` خروجی به جای مقادیر درونیابی. بهتر است برای نشان دادن ارزش هایی که مربوط به زمان اجرای برخی از نمایش داده شد. [\#8712](https://github.com/ClickHouse/ClickHouse/pull/8712) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- امکان اضافه کردن کلید و زمان بندی برای پیام هنگام قرار دادن داده ها به کافکا. رفع [\#7198](https://github.com/ClickHouse/ClickHouse/issues/7198) [\#8969](https://github.com/ClickHouse/ClickHouse/pull/8969) ([فیلیمونف](https://github.com/filimonov)) -- اگر سرور از ترمینال اجرا, تعداد موضوع برجسته, شناسه پرس و جو و اولویت ورود به سیستم با رنگ. این است که برای خوانایی بهبود یافته از پیام ورود به سیستم در ارتباط برای توسعه دهندگان. [\#8961](https://github.com/ClickHouse/ClickHouse/pull/8961) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- پیام استثنا بهتر در حالی که بارگذاری جداول برای `Ordinary` پایگاه داده است. [\#9527](https://github.com/ClickHouse/ClickHouse/pull/9527) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- پیاده سازی `arraySlice` برای ارریس با عملکرد کل ایالات. این رفع [\#9388](https://github.com/ClickHouse/ClickHouse/issues/9388) [\#9391](https://github.com/ClickHouse/ClickHouse/pull/9391) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- اجازه می دهد توابع ثابت و مجموعه های ثابت در سمت راست در اپراتور استفاده می شود. [\#8813](https://github.com/ClickHouse/ClickHouse/pull/8813) ([انتون پوپوف](https://github.com/CurtizJ)) -- اگر استثنا باغ وحش اتفاق افتاده است در حالی که واکشی داده ها برای سیستم.تکرار, نمایش در یک ستون جداگانه. این پیاده سازی [\#9137](https://github.com/ClickHouse/ClickHouse/issues/9137) [\#9138](https://github.com/ClickHouse/ClickHouse/pull/9138) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- بطور عام حذف قطعات داده ادغام در نابود کردن. [\#8402](https://github.com/ClickHouse/ClickHouse/pull/8402) ([ولادیمیر چبوتراف](https://github.com/excitoon)) -- پشتیبانی امنیت سطح ردیف برای جداول توزیع شده است. [\#8926](https://github.com/ClickHouse/ClickHouse/pull/8926) ([ایوان](https://github.com/abyss7)) -- Now we recognize suffix (like KB, KiB…) in settings values. [\#8072](https://github.com/ClickHouse/ClickHouse/pull/8072) ([میخیل کوروتف](https://github.com/millb)) -- جلوگیری از حافظه در حالی که ساخت نتیجه یک بزرگ بپیوندید. [\#8637](https://github.com/ClickHouse/ClickHouse/pull/8637) ([زویکوف](https://github.com/4ertus2)) -- اضافه شده نام خوشه به پیشنهادات در حالت تعاملی در `clickhouse-client`. [\#8709](https://github.com/ClickHouse/ClickHouse/pull/8709) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- Initialize query profiler for all threads in a group, e.g. it allows to fully profile insert-queries [\#8820](https://github.com/ClickHouse/ClickHouse/pull/8820) ([ایوان](https://github.com/abyss7)) -- ستون اضافه شده `exception_code` داخل `system.query_log` جدول [\#8770](https://github.com/ClickHouse/ClickHouse/pull/8770) ([میخیل کوروتف](https://github.com/millb)) -- فعال خروجی زیر سرور سازگاری در بندر `9004` در فایل پیکربندی سرور به طور پیش فرض. دستور تولید رمز عبور ثابت در مثال در پیکربندی. [\#8771](https://github.com/ClickHouse/ClickHouse/pull/8771) ([یوری بارانوف](https://github.com/yurriy)) -- جلوگیری از سقط جنین در خاموش کردن اگر سیستم فایل به صورت خوانده است. این رفع [\#9094](https://github.com/ClickHouse/ClickHouse/issues/9094) [\#9100](https://github.com/ClickHouse/ClickHouse/pull/9100) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- پیام استثنا بهتر زمانی که طول در قام پست پرس و جو مورد نیاز است. [\#9453](https://github.com/ClickHouse/ClickHouse/pull/9453) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- افزودن `_path` و `_file` ستون مجازی به `HDFS` و `File` موتور و `hdfs` و `file` توابع جدول [\#8489](https://github.com/ClickHouse/ClickHouse/pull/8489) ([اولگا خوستیکوا](https://github.com/stavrolia)) -- رفع خطا `Cannot find column` در حالی که قرار دادن به `MATERIALIZED VIEW` در صورتی که اگر ستون جدید برای مشاهده جدول داخلی اضافه شد. [\#8766](https://github.com/ClickHouse/ClickHouse/pull/8766) [\#8788](https://github.com/ClickHouse/ClickHouse/pull/8788) ([vzakaznikov](https://github.com/vzakaznikov)) [\#8788](https://github.com/ClickHouse/ClickHouse/issues/8788) [\#8806](https://github.com/ClickHouse/ClickHouse/pull/8806) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) [\#8803](https://github.com/ClickHouse/ClickHouse/pull/8803) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) -- رفع پیشرفت بیش از پروتکل کلاینت سرور بومی, با ارسال پیشرفت پس از به روز رسانی نهایی (مانند سیاهههای مربوط). این ممکن است تنها مربوط به برخی از ابزار های شخص ثالث که با استفاده از پروتکل های بومی. [\#9495](https://github.com/ClickHouse/ClickHouse/pull/9495) ([ازات خوژین](https://github.com/azat)) -- اضافه کردن یک سیستم متریک ردیابی تعداد اتصالات مشتری با استفاده از پروتکل خروجی زیر ([\#9013](https://github.com/ClickHouse/ClickHouse/issues/9013)). [\#9015](https://github.com/ClickHouse/ClickHouse/pull/9015) ([یوجین کلیموف](https://github.com/Slach)) -- از این به بعد پاسخ های اچ تی پی `X-ClickHouse-Timezone` هدر را به مقدار منطقه زمانی همان است که `SELECT timezone()` دوست گزارش. [\#9493](https://github.com/ClickHouse/ClickHouse/pull/9493) ([دنیس گلازاشف](https://github.com/traceon)) - -#### بهبود عملکرد {#performance-improvement} - -- بهبود عملکرد شاخص تجزیه و تحلیل با در [\#9261](https://github.com/ClickHouse/ClickHouse/pull/9261) ([انتون پوپوف](https://github.com/CurtizJ)) -- کد ساده تر و موثر تر در توابع منطقی + پاکسازی کد. پیگیری به [\#8718](https://github.com/ClickHouse/ClickHouse/issues/8718) [\#8728](https://github.com/ClickHouse/ClickHouse/pull/8728) ([الکساندر کازاکوف](https://github.com/Akazz)) -- بهبود عملکرد کلی (در محدوده 5%..200% برای نمایش داده شد تحت تاثیر قرار) با تضمین و حتی بایاس سخت تر با ج++20 ویژگی های. [\#9304](https://github.com/ClickHouse/ClickHouse/pull/9304) ([ایموس پرنده](https://github.com/amosbird)) -- مقایسه دقیق تر برای حلقه های داخلی توابع مقایسه. [\#9327](https://github.com/ClickHouse/ClickHouse/pull/9327) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- بایاس سخت تر برای حلقه داخلی از توابع ریاضی. [\#9325](https://github.com/ClickHouse/ClickHouse/pull/9325) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- A ~3 بار سریع تر برای اجرای ColumnVector::تکرار () از طریق آن ColumnConst::convertToFullColumn() اجرا شده است. همچنین در تست مفید خواهد بود که تحقق ثابت. [\#9293](https://github.com/ClickHouse/ClickHouse/pull/9293) ([الکساندر کازاکوف](https://github.com/Akazz)) -- یکی دیگر از بهبود عملکرد کوچک به `ColumnVector::replicate()` (این سرعت `materialize` عملکرد و توابع سفارش بالاتر) بهبود و حتی بیشتر به [\#9293](https://github.com/ClickHouse/ClickHouse/issues/9293) [\#9442](https://github.com/ClickHouse/ClickHouse/pull/9442) ([الکساندر کازاکوف](https://github.com/Akazz)) -- بهبود عملکرد `stochasticLinearRegression` تابع جمع. این پچ توسط اینتل کمک. [\#8652](https://github.com/ClickHouse/ClickHouse/pull/8652) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- بهبود عملکرد `reinterpretAsFixedString` تابع. [\#9342](https://github.com/ClickHouse/ClickHouse/pull/9342) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- بلوک ها را به مشتری ارسال نکنید `Null` فرمت در پردازنده خط لوله. [\#8797](https://github.com/ClickHouse/ClickHouse/pull/8797) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) [\#8767](https://github.com/ClickHouse/ClickHouse/pull/8767) ([الکساندر کوزمنکوف](https://github.com/akuzm)) - -#### ساخت/تست / بهبود بسته بندی {#buildtestingpackaging-improvement} - -- سیستم های انتقال مواد استثنا در حال حاضر به درستی کار می کند در زیر سیستم ویندوز برای لینوکس. ببینید https://github.com/clickhouse-extras/libunwind/pull/3 این رفع [\#6480](https://github.com/ClickHouse/ClickHouse/issues/6480) [\#9564](https://github.com/ClickHouse/ClickHouse/pull/9564) ([سوبولسو](https://github.com/sobolevsv)) -- جایگزینی `readline` با `replxx` برای ویرایش خط تعاملی در `clickhouse-client` [\#8416](https://github.com/ClickHouse/ClickHouse/pull/8416) ([ایوان](https://github.com/abyss7)) -- بهتر است زمان ساخت و کمتر در قالب instantiations در functionscomparison. [\#9324](https://github.com/ClickHouse/ClickHouse/pull/9324) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- ادغام اضافه شده با `clang-tidy` در سی. همچنین نگاه کنید به [\#6044](https://github.com/ClickHouse/ClickHouse/issues/6044) [\#9566](https://github.com/ClickHouse/ClickHouse/pull/9566) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- در حال حاضر ما لینک کلیک در سی با استفاده از `lld` حتی برای `gcc`. [\#9049](https://github.com/ClickHouse/ClickHouse/pull/9049) ([الساپین](https://github.com/alesapin)) -- اجازه می دهد به صورت تصادفی برنامه ریزی موضوع و درج اشکالات زمانی که `THREAD_FUZZER_*` متغیرهای محیطی تنظیم شده است. این کمک می کند تا تست. [\#9459](https://github.com/ClickHouse/ClickHouse/pull/9459) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- فعال کردن سوکت امن در تست بدون تابعیت [\#9288](https://github.com/ClickHouse/ClickHouse/pull/9288) ([تاولوبیکس](https://github.com/tavplubix)) -- را split\_shared\_libraries=خاموش قوی تر [\#9156](https://github.com/ClickHouse/ClickHouse/pull/9156) ([ازات خوژین](https://github.com/azat)) -- ساخت “performance\_introspection\_and\_logging” تست قابل اعتماد به سرور تصادفی گیر کرده است. این ممکن است در محیط زیست سی اتفاق می افتد. همچنین نگاه کنید به [\#9515](https://github.com/ClickHouse/ClickHouse/issues/9515) [\#9528](https://github.com/ClickHouse/ClickHouse/pull/9528) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- اعتبار میلی لیتر در چک سبک. [\#9550](https://github.com/ClickHouse/ClickHouse/pull/9550) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- شرایط مسابقه ثابت در تست `00738_lock_for_inner_table`. این تست به خواب متکی بود. [\#9555](https://github.com/ClickHouse/ClickHouse/pull/9555) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- تست عملکرد نوع را حذف کنید `once`. این مورد نیاز است برای اجرای تمام تست های عملکرد در حالت مقایسه استاتیک (قابل اعتماد تر). [\#9557](https://github.com/ClickHouse/ClickHouse/pull/9557) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- اضافه شده تست عملکرد برای توابع ریاضی. [\#9326](https://github.com/ClickHouse/ClickHouse/pull/9326) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- تست عملکرد اضافه شده برای `sumMap` و `sumMapWithOverflow` توابع مجموع. پیگیری برای [\#8933](https://github.com/ClickHouse/ClickHouse/issues/8933) [\#8947](https://github.com/ClickHouse/ClickHouse/pull/8947) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- اطمینان از سبک کد های خطا با چک سبک. [\#9370](https://github.com/ClickHouse/ClickHouse/pull/9370) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- اضافه کردن اسکریپت برای تست تاریخ. [\#8796](https://github.com/ClickHouse/ClickHouse/pull/8796) ([الساپین](https://github.com/alesapin)) -- افزودن هشدار شورای همکاری خلیج فارس `-Wsuggest-override` برای قرار دادن و رفع تمام مکان هایی که `override` کلمه کلیدی باید استفاده شود. [\#8760](https://github.com/ClickHouse/ClickHouse/pull/8760) ([کروزرکریگ](https://github.com/kreuzerkrieg)) -- نادیده گرفتن نماد ضعیف تحت سیستم عامل مک ایکس زیرا باید تعریف شود [\#9538](https://github.com/ClickHouse/ClickHouse/pull/9538) ([کاربر حذفشده](https://github.com/ghost)) -- عادی زمان در حال اجرا از برخی از نمایش داده شد در تست عملکرد. این است که در تهیه برای اجرای تمام تست های عملکرد در حالت مقایسه انجام می شود. [\#9565](https://github.com/ClickHouse/ClickHouse/pull/9565) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- رفع برخی از تست ها برای حمایت از تست های پرس و جو [\#9062](https://github.com/ClickHouse/ClickHouse/pull/9062) ([ایوان](https://github.com/abyss7)) -- فعال کردن اس اس ال در ساخت با ام اس ان, بنابراین سرور نمی خواهد شکست در هنگام راه اندازی در حال اجرا تست بدون تابعیت [\#9531](https://github.com/ClickHouse/ClickHouse/pull/9531) ([تاولوبیکس](https://github.com/tavplubix)) -- رفع جایگزینی پایگاه داده در نتایج تست [\#9384](https://github.com/ClickHouse/ClickHouse/pull/9384) ([ایلیا یاتسیشین](https://github.com/qoega)) -- ساخت رفع برای سیستم عامل های دیگر [\#9381](https://github.com/ClickHouse/ClickHouse/pull/9381) ([پرولر](https://github.com/proller)) [\#8755](https://github.com/ClickHouse/ClickHouse/pull/8755) ([پرولر](https://github.com/proller)) [\#8631](https://github.com/ClickHouse/ClickHouse/pull/8631) ([پرولر](https://github.com/proller)) -- اضافه شدن بخش دیسک به بدون تابعیت با پوشش تصویر داکر تست [\#9213](https://github.com/ClickHouse/ClickHouse/pull/9213) ([پاول کووالنکو](https://github.com/Jokser)) -- خلاص شدن از شر در منبع درخت فایل در هنگام ساخت با پتروشیمی [\#9588](https://github.com/ClickHouse/ClickHouse/pull/9588) ([ایموس پرنده](https://github.com/amosbird)) -- زمان ساخت کمی سریع تر با از بین بردن تجزیه طلبان از زمینه. کد را ساده تر کنید. [\#9232](https://github.com/ClickHouse/ClickHouse/pull/9232) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- به روز شده در چک کردن برای نمایش داده شد را قطع کرد در اسکریپت کلیک تست [\#8858](https://github.com/ClickHouse/ClickHouse/pull/8858) ([الکساندر کازاکوف](https://github.com/Akazz)) -- حذف برخی از فایل های بی فایده از مخزن. [\#8843](https://github.com/ClickHouse/ClickHouse/pull/8843) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- نوع تغییر کامل ریاضی از `once` به `loop`. [\#8783](https://github.com/ClickHouse/ClickHouse/pull/8783) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) -- اضافه کردن تصویر کارگر بارانداز که اجازه می دهد تا برای ساخت کد تعاملی مرورگر گزارش متنی برای کدهای ما. [\#8781](https://github.com/ClickHouse/ClickHouse/pull/8781) ([الساپین](https://github.com/alesapin)) ببینید [مرورگر کد ووبوک](https://clickhouse.tech/codebrowser/html_report///ClickHouse/dbms/src/index.html) -- سرکوب برخی از شکست تست تحت مسان. [\#8780](https://github.com/ClickHouse/ClickHouse/pull/8780) ([الکساندر کوزمنکوف](https://github.com/akuzm)) -- افزایش سرعت “exception while insert” امتحان این تست اغلب زمان در اشکال زدایی با پوشش ساخت. [\#8711](https://github.com/ClickHouse/ClickHouse/pull/8711) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- به روز شده `libcxx` و `libcxxabi` به سلامتی استاد در تهیه به [\#9304](https://github.com/ClickHouse/ClickHouse/issues/9304) [\#9308](https://github.com/ClickHouse/ClickHouse/pull/9308) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- رفع تست پوسته شدن `00910_zookeeper_test_alter_compression_codecs`. [\#9525](https://github.com/ClickHouse/ClickHouse/pull/9525) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- پاک کردن پرچم لینکر تکرار. اطمینان حاصل کنید که لینکر نمی خواهد نگاه کردن یک نماد غیر منتظره. [\#9433](https://github.com/ClickHouse/ClickHouse/pull/9433) ([ایموس پرنده](https://github.com/amosbird)) -- افزودن `clickhouse-odbc` درایور را به تصاویر تست. این اجازه می دهد تا به تست تعامل ClickHouse با ClickHouse از طریق خود ODBC driver. [\#9348](https://github.com/ClickHouse/ClickHouse/pull/9348) ([فیلیمونف](https://github.com/filimonov)) -- رفع چندین باگ در تست واحد. [\#9047](https://github.com/ClickHouse/ClickHouse/pull/9047) ([الساپین](https://github.com/alesapin)) -- فعالسازی `-Wmissing-include-dirs` هشدار شورای همکاری خلیج فارس برای از بین بردن تمام غیر موجود شامل-عمدتا به عنوان یک نتیجه از خطاهای برنامه نویسی کیک [\#8704](https://github.com/ClickHouse/ClickHouse/pull/8704) ([کروزرکریگ](https://github.com/kreuzerkrieg)) -- توصیف دلایل اگر پیشفیلتر پرس و جو نمی تواند کار کند. این است که برای در نظر گرفته شده [\#9049](https://github.com/ClickHouse/ClickHouse/issues/9049) [\#9144](https://github.com/ClickHouse/ClickHouse/pull/9144) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- به روز رسانی اپنسسل به بالادست استاد. ثابت موضوع زمانی که اتصالات ال اس ممکن است با پیام شکست `OpenSSL SSL_read: error:14094438:SSL routines:ssl3_read_bytes:tlsv1 alert internal error` و `SSL Exception: error:2400006E:random number generator::error retrieving entropy`. این موضوع در حال حاضر در نسخه 20.1 بود. [\#8956](https://github.com/ClickHouse/ClickHouse/pull/8956) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- به روز رسانی صفحه مدیریت برای سرور [\#8893](https://github.com/ClickHouse/ClickHouse/pull/8893) ([ایلیا مزایف](https://github.com/ne-ray)) -- اصلاحات جزیی در ساخت شورای همکاری خلیج فارس از منابع اسکریپت [\#8774](https://github.com/ClickHouse/ClickHouse/pull/8774) ([مایکل ناچاروف](https://github.com/mnach)) -- جایگزینی `numbers` به `zeros` در کامل کجا `number` ستون استفاده نمی شود. این به نتایج تست تمیز تر منجر شود. [\#9600](https://github.com/ClickHouse/ClickHouse/pull/9600) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) -- رفع مشکل سرریز پشته در هنگام استفاده از مقدار دهی اولیه در سازنده ستون. [\#9367](https://github.com/ClickHouse/ClickHouse/pull/9367) ([کاربر حذفشده](https://github.com/ghost)) -- ارتقا کتابدار به نسخه 1. 3.0. فعالسازی همراه `rdkafka` و `gsasl` کتابخانه ها در سیستم عامل مک ایکس. [\#9000](https://github.com/ClickHouse/ClickHouse/pull/9000) ([اندرو انیشچوک](https://github.com/oandrew)) -- ساخت ثابت در شورای همکاری خلیج فارس 9.2.0 [\#9306](https://github.com/ClickHouse/ClickHouse/pull/9306) ([وکسیدر](https://github.com/Vxider)) - -## انتشار کلیک و 20.1 {#clickhouse-release-v20-1} - -### ClickHouse انتشار V20.1.8.41, 2020-03-20 {#clickhouse-release-v20-1-8-41-2020-03-20} - -#### رفع اشکال {#bug-fix-3} - -- رفع همیشگی احتمالی `Cannot schedule a task` خطا (با توجه به استثنای بدون مانع در `ParallelAggregatingBlockInputStream::Handler::onFinish/onFinishThread`). این رفع [\#6833](https://github.com/ClickHouse/ClickHouse/issues/6833). [\#9154](https://github.com/ClickHouse/ClickHouse/pull/9154) ([ازات خوژین](https://github.com/azat)) -- رفع مصرف حافظه بیش از حد در `ALTER` نمایش داده شد (جهش). این رفع [\#9533](https://github.com/ClickHouse/ClickHouse/issues/9533) و [\#9670](https://github.com/ClickHouse/ClickHouse/issues/9670). [\#9754](https://github.com/ClickHouse/ClickHouse/pull/9754) ([الساپین](https://github.com/alesapin)) -- رفع اشکال در عقب نشینی در لغت نامه های خارجی. این رفع [\#9619](https://github.com/ClickHouse/ClickHouse/issues/9619). [\#9734](https://github.com/ClickHouse/ClickHouse/pull/9734) ([الساپین](https://github.com/alesapin)) - -### ClickHouse انتشار V20.1.7.38, 2020-03-18 {#clickhouse-release-v20-1-7-38-2020-03-18} - -#### رفع اشکال {#bug-fix-4} - -- ثابت نادرست نام تابع داخلی برای `sumKahan` و `sumWithOverflow`. من به استثنا منجر شود در حالی که با استفاده از این توابع در نمایش داده شد از راه دور. [\#9636](https://github.com/ClickHouse/ClickHouse/pull/9636) ([ازات خوژین](https://github.com/azat)). این موضوع در تمام نسخه های کلیک خانه بود. -- اجازه داده شود `ALTER ON CLUSTER` از `Distributed` جداول با تکرار داخلی. این رفع [\#3268](https://github.com/ClickHouse/ClickHouse/issues/3268). [\#9617](https://github.com/ClickHouse/ClickHouse/pull/9617) ([شینی2](https://github.com/shinoi2)). این موضوع در تمام نسخه های کلیک خانه بود. -- رفع استثناهای احتمالی `Size of filter doesn't match size of column` و `Invalid number of rows in Chunk` داخل `MergeTreeRangeReader`. می توانند هنگام اجرای ظاهر شوند `PREWHERE` در برخی موارد. رفع [\#9132](https://github.com/ClickHouse/ClickHouse/issues/9132). [\#9612](https://github.com/ClickHouse/ClickHouse/pull/9612) ([انتون پوپوف](https://github.com/CurtizJ)) -- ثابت موضوع: منطقه زمانی حفظ نشده بود اگر شما نوشتن یک عبارت ریاضی ساده مانند `time + 1` (در مقایسه با بیان مانند `time + INTERVAL 1 SECOND`). این رفع [\#5743](https://github.com/ClickHouse/ClickHouse/issues/5743). [\#9323](https://github.com/ClickHouse/ClickHouse/pull/9323) ([الکسی میلویدو](https://github.com/alexey-milovidov)). این موضوع در تمام نسخه های کلیک خانه بود. -- در حال حاضر امکان ایجاد یا اضافه کردن ستون ها با نام مستعار چرخه ای ساده مانند وجود ندارد `a DEFAULT b, b DEFAULT a`. [\#9603](https://github.com/ClickHouse/ClickHouse/pull/9603) ([الساپین](https://github.com/alesapin)) -- ثابت موضوع که بالشتک در پایان base64 کد گذاری ارزش می تواند ناقص. به روز رسانی پایگاه64 کتابخانه. این رفع [\#9491](https://github.com/ClickHouse/ClickHouse/issues/9491) بسته [\#9492](https://github.com/ClickHouse/ClickHouse/issues/9492) [\#9500](https://github.com/ClickHouse/ClickHouse/pull/9500) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- رفع مسابقه داده ها در نابودی `Poco::HTTPServer`. این می تواند رخ دهد زمانی که سرور شروع شده است و بلافاصله تعطیل. [\#9468](https://github.com/ClickHouse/ClickHouse/pull/9468) ([انتون پوپوف](https://github.com/CurtizJ)) -- رفع سقوط احتمالی / تعداد اشتباه ردیف در `LIMIT n WITH TIES` هنگامی که بسیاری از ردیف به نفر ردیف برابر وجود دارد. [\#9464](https://github.com/ClickHouse/ClickHouse/pull/9464) ([تاولوبیکس](https://github.com/tavplubix)) -- رفع چک عدم تطابق ممکن است با ستون. [\#9451](https://github.com/ClickHouse/ClickHouse/pull/9451) ([انتون پوپوف](https://github.com/CurtizJ)) -- رفع سقوط زمانی که یک کاربر تلاش می کند `ALTER MODIFY SETTING` برای قدیمی شکل گرفته `MergeTree` موتورهای جدول خانواده. [\#9435](https://github.com/ClickHouse/ClickHouse/pull/9435) ([الساپین](https://github.com/alesapin)) -- در حال حاضر ما سعی خواهد کرد نهایی جهش بیشتر. [\#9427](https://github.com/ClickHouse/ClickHouse/pull/9427) ([الساپین](https://github.com/alesapin)) -- رفع ناسازگاری پروتکل تکرار معرفی شده در [\#8598](https://github.com/ClickHouse/ClickHouse/issues/8598). [\#9412](https://github.com/ClickHouse/ClickHouse/pull/9412) ([الساپین](https://github.com/alesapin)) -- ثابت نیست (است ()) برای شاخص نفخ\_فیلتر از انواع مجموعه. [\#9407](https://github.com/ClickHouse/ClickHouse/pull/9407) ([ایشیمب](https://github.com/achimbab)) -- رفتار را ثابت کرد `match` و `extract` توابع زمانی که کومه علف خشک است صفر بایت. رفتار اشتباه بود که کومه علف خشک ثابت بود. این رفع [\#9160](https://github.com/ClickHouse/ClickHouse/issues/9160) [\#9163](https://github.com/ClickHouse/ClickHouse/pull/9163) ([الکسی میلویدو](https://github.com/alexey-milovidov)) [\#9345](https://github.com/ClickHouse/ClickHouse/pull/9345) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - -#### ساخت/تست / بهبود بسته بندی {#buildtestingpackaging-improvement-1} - -- سیستم های انتقال مواد استثنا در حال حاضر به درستی کار می کند در زیر سیستم ویندوز برای لینوکس. ببینید https://github.com/clickhouse-extras/libunwind/pull/3 این رفع [\#6480](https://github.com/ClickHouse/ClickHouse/issues/6480) [\#9564](https://github.com/ClickHouse/ClickHouse/pull/9564) ([سوبولسو](https://github.com/sobolevsv)) - -### ClickHouse انتشار V20.1.6.30, 2020-03-05 {#clickhouse-release-v20-1-6-30-2020-03-05} - -#### رفع اشکال {#bug-fix-5} - -- رفع ناسازگاری داده ها در هنگام فشرده با `T64` وابسته به کدک. - [\#9039](https://github.com/ClickHouse/ClickHouse/pull/9039) [(بی پایان 7)](https://github.com/abyss7) -- ثابت منظور از محدوده در حالی که خواندن از جدول ادغام در یک موضوع. رفع [\#8964](https://github.com/ClickHouse/ClickHouse/issues/8964). - [\#9050](https://github.com/ClickHouse/ClickHouse/pull/9050) [(کورتیزج)](https://github.com/CurtizJ) -- رفع اشکال احتمالی در `MergeTreeRangeReader`, در حالی که اجرای `PREWHERE`. رفع [\#9064](https://github.com/ClickHouse/ClickHouse/issues/9064). - [\#9106](https://github.com/ClickHouse/ClickHouse/pull/9106) [(کورتیزج)](https://github.com/CurtizJ) -- ثابت `reinterpretAsFixedString` برای بازگشت `FixedString` به جای `String`. - [\#9052](https://github.com/ClickHouse/ClickHouse/pull/9052) [(اوندرو)](https://github.com/oandrew) -- ثابت `joinGet` با انواع بازگشت باطل. رفع [\#8919](https://github.com/ClickHouse/ClickHouse/issues/8919) - [\#9014](https://github.com/ClickHouse/ClickHouse/pull/9014) [(طرقه)](https://github.com/amosbird) -- تعمیر ریش ریش شدن آزمون و نادرست رفتار bittestall/bittestany توابع. - [\#9143](https://github.com/ClickHouse/ClickHouse/pull/9143) [(الکسی میلویدو)](https://github.com/alexey-milovidov) -- رفع رفتار توابع بازی و استخراج زمانی که انبار کاه است صفر بایت. رفتار اشتباه بود که کومه علف خشک ثابت بود. رفع [\#9160](https://github.com/ClickHouse/ClickHouse/issues/9160) - [\#9163](https://github.com/ClickHouse/ClickHouse/pull/9163) [(الکسی میلویدو)](https://github.com/alexey-milovidov) -- ثابت اعدام inversed predicates که غیر به شدت monotinic عملکردی شاخص استفاده شده است. رفع [\#9034](https://github.com/ClickHouse/ClickHouse/issues/9034) - [\#9223](https://github.com/ClickHouse/ClickHouse/pull/9223) [(اکازز)](https://github.com/Akazz) -- اجازه به بازنویسی `CROSS` به `INNER JOIN` اگر وجود دارد `[NOT] LIKE` اپراتور در `WHERE` بخش. رفع [\#9191](https://github.com/ClickHouse/ClickHouse/issues/9191) - [\#9229](https://github.com/ClickHouse/ClickHouse/pull/9229) [(4رتوس2)](https://github.com/4ertus2) -- اجازه ستون اول (بازدید کنندگان) در یک جدول با موتور ورود به سیستم یک نام مستعار. - [\#9231](https://github.com/ClickHouse/ClickHouse/pull/9231) [(بی پایان 7)](https://github.com/abyss7) -- اجازه دادن به کاما از هم پیوستن با `IN()` داخل رفع [\#7314](https://github.com/ClickHouse/ClickHouse/issues/7314). - [\#9251](https://github.com/ClickHouse/ClickHouse/pull/9251) [(4رتوس2)](https://github.com/4ertus2) -- بهبود `ALTER MODIFY/ADD` نمایش داده شد منطق. حالا شما نمی توانید `ADD` ستون بدون نوع, `MODIFY` عبارت پیش فرض نوع ستون را تغییر نمی دهد و `MODIFY` نوع ارزش بیان پیش فرض را از دست نمی دهد. رفع [\#8669](https://github.com/ClickHouse/ClickHouse/issues/8669). - [\#9227](https://github.com/ClickHouse/ClickHouse/pull/9227) [(الساپین)](https://github.com/alesapin) -- رفع جهش نهایی, زمانی که جهش در حال حاضر انجام می شود می توانید وضعیت داشته باشد \_دون=0. - [\#9217](https://github.com/ClickHouse/ClickHouse/pull/9217) [(الساپین)](https://github.com/alesapin) -- پشتیبانی “Processors” خط لوله برای سیستم.اعداد و سیستم.\_شماره. این نیز رفع اشکال زمانی که `max_execution_time` محترم نیست. - [\#7796](https://github.com/ClickHouse/ClickHouse/pull/7796) [.)](https://github.com/KochetovNicolai) -- رفع شمارش اشتباه از `DictCacheKeysRequestedFound` متریک. - [\#9411](https://github.com/ClickHouse/ClickHouse/pull/9411) [(نیکیتامیخایلو)](https://github.com/nikitamikhaylov) -- اضافه شدن یک چک برای سیاست ذخیره سازی در `ATTACH PARTITION FROM`, `REPLACE PARTITION`, `MOVE TO TABLE` که در غیر این صورت می تواند داده ها از بخش غیر قابل دسترس پس از راه اندازی مجدد و جلوگیری از کلیک برای شروع. - [\#9383](https://github.com/ClickHouse/ClickHouse/pull/9383) [(هیجان)](https://github.com/excitoon) -- گزارش ثابت اوبسان در `MergeTreeIndexSet`. این رفع [\#9250](https://github.com/ClickHouse/ClickHouse/issues/9250) - [\#9365](https://github.com/ClickHouse/ClickHouse/pull/9365) [(الکسی میلویدو)](https://github.com/alexey-milovidov) -- رفع فضای داده ممکن در بلوکیو. - [\#9356](https://github.com/ClickHouse/ClickHouse/pull/9356) [.)](https://github.com/KochetovNicolai) -- پشتیبانی از `UInt64` اعداد است که متناسب نیست در درون64 در توابع مربوط به جانسون. بهروزرسانی `SIMDJSON` به سلامتی استاد این رفع [\#9209](https://github.com/ClickHouse/ClickHouse/issues/9209) - [\#9344](https://github.com/ClickHouse/ClickHouse/pull/9344) [(الکسی میلویدو)](https://github.com/alexey-milovidov) -- رفع مشکل زمانی که مقدار فضای رایگان به درستی محاسبه نمی شود اگر دایرکتوری داده ها به یک دستگاه جداگانه نصب شود. برای دیسک پیش فرض محاسبه فضای رایگان از دایرکتوری فرعی داده ها. این رفع [\#7441](https://github.com/ClickHouse/ClickHouse/issues/7441) - [\#9257](https://github.com/ClickHouse/ClickHouse/pull/9257) [(میلب)](https://github.com/millb) -- رفع مشکل زمانی که اتصالات ال اس ممکن است با پیام شکست `OpenSSL SSL_read: error:14094438:SSL routines:ssl3_read_bytes:tlsv1 alert internal error and SSL Exception: error:2400006E:random number generator::error retrieving entropy.` به روز رسانی اپنسسل به بالادست استاد. - [\#8956](https://github.com/ClickHouse/ClickHouse/pull/8956) [(الکسی میلویدو)](https://github.com/alexey-milovidov) -- هنگام اجرای `CREATE` پرس و جو, برابر عبارات ثابت در استدلال موتور ذخیره سازی. جایگزین کردن نام دادگان خالی با دادگان فعلی. رفع [\#6508](https://github.com/ClickHouse/ClickHouse/issues/6508), [\#3492](https://github.com/ClickHouse/ClickHouse/issues/3492). همچنین رفع بررسی برای نشانی های محلی در کلیک کنیدشاهارزش. - [\#9262](https://github.com/ClickHouse/ClickHouse/pull/9262) [(تبلوبیکس)](https://github.com/tavplubix) -- رفع segfault در `StorageMerge`, که می تواند در هنگام خواندن از داستان اتفاق می افتد. - [\#9387](https://github.com/ClickHouse/ClickHouse/pull/9387) [(تبلوبیکس)](https://github.com/tavplubix) -- جلوگیری از از دست دادن داده ها در `Kafka` در موارد نادر زمانی که استثنا اتفاق می افتد پس از خواندن پسوند اما قبل از ارتکاب. رفع [\#9378](https://github.com/ClickHouse/ClickHouse/issues/9378). مرتبط: [\#7175](https://github.com/ClickHouse/ClickHouse/issues/7175) - [\#9507](https://github.com/ClickHouse/ClickHouse/pull/9507) [(فیلیمونف)](https://github.com/filimonov) -- رفع اشکال منجر به ختم سرور در هنگام تلاش برای استفاده / رها کردن `Kafka` جدول ایجاد شده با پارامترهای اشتباه. رفع [\#9494](https://github.com/ClickHouse/ClickHouse/issues/9494). دارای [\#9507](https://github.com/ClickHouse/ClickHouse/issues/9507). - [\#9513](https://github.com/ClickHouse/ClickHouse/pull/9513) [(فیلیمونف)](https://github.com/filimonov) - -#### ویژگی جدید {#new-feature-1} - -- افزودن `deduplicate_blocks_in_dependent_materialized_views` گزینه ای برای کنترل رفتار درج ژولیده به جداول با نمایش محقق. این ویژگی جدید توسط یک درخواست ویژه از التیت به نسخه رفع اشکال اضافه شد. - [\#9070](https://github.com/ClickHouse/ClickHouse/pull/9070) [.)](https://github.com/urykhy) - -### ClickHouse انتشار V20.1.2.4, 2020-01-22 {#clickhouse-release-v20-1-2-4-2020-01-22} - -#### تغییر ناسازگار به عقب {#backward-incompatible-change-1} - -- ایجاد تنظیمات `merge_tree_uniform_read_distribution` منسوخ شده. سرور هنوز این تنظیم به رسمیت می شناسد اما هیچ اثر. [\#8308](https://github.com/ClickHouse/ClickHouse/pull/8308) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- نوع بازگشت تغییر از تابع `greatCircleDistance` به `Float32` چرا که در حال حاضر نتیجه محاسبه است `Float32`. [\#7993](https://github.com/ClickHouse/ClickHouse/pull/7993) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- در حال حاضر انتظار می رود که پارامترهای پرس و جو در “escaped” قالب. مثلا, به تصویب رشته `ab` شما باید برای نوشتن `a\tb` یا `a\b` و به ترتیب, `a%5Ctb` یا `a%5C%09b` در URL. این مورد نیاز است برای اضافه کردن امکان به تصویب تهی به عنوان `\N`. این رفع [\#7488](https://github.com/ClickHouse/ClickHouse/issues/7488). [\#8517](https://github.com/ClickHouse/ClickHouse/pull/8517) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- فعالسازی `use_minimalistic_part_header_in_zookeeper` تنظیم برای `ReplicatedMergeTree` به طور پیش فرض. این به طور قابل توجهی مقدار داده های ذخیره شده در باغ وحش را کاهش دهد. این تنظیم از نسخه 19.1 پشتیبانی می شود و ما در حال حاضر در تولید خدمات متعدد بدون هیچ مشکلی برای بیش از نیم سال استفاده می کنیم. غیر فعال کردن این تنظیم اگر شما یک فرصت برای جمع و جور کردن به نسخه های قدیمی تر از 19.1. [\#6850](https://github.com/ClickHouse/ClickHouse/pull/6850) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- شاخص های پرش داده ها تولید می شوند و به طور پیش فرض فعال می شوند. تنظیمات `allow_experimental_data_skipping_indices`, `allow_experimental_cross_to_join_conversion` و `allow_experimental_multiple_joins_emulation` در حال حاضر منسوخ و هیچ چیز. [\#7974](https://github.com/ClickHouse/ClickHouse/pull/7974) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- افزودن جدید `ANY JOIN` منطق برای `StorageJoin` سازگار با `JOIN` عمل به روز رسانی بدون تغییر در رفتار شما نیاز به اضافه کردن `SETTINGS any_join_distinct_right_table_keys = 1` به موتور پیوستن ابرداده جداول و یا از نو خلق این جداول پس از ارتقا دهید. [\#8400](https://github.com/ClickHouse/ClickHouse/pull/8400) ([زویکوف](https://github.com/4ertus2)) -- نیاز به سرور برای راه اندازی مجدد برای اعمال تغییرات در پیکربندی ورود به سیستم. این یک راه حل موقت برای جلوگیری از اشکال که سرور سیاهههای مربوط به یک فایل ورود به سیستم حذف شده است (دیدن [\#8696](https://github.com/ClickHouse/ClickHouse/issues/8696)). [\#8707](https://github.com/ClickHouse/ClickHouse/pull/8707) ([الکساندر کوزمنکوف](https://github.com/akuzm)) - -#### ویژگی جدید {#new-feature-2} - -- اطلاعات اضافه شده در مورد مسیرهای بخشی به `system.merges`. [\#8043](https://github.com/ClickHouse/ClickHouse/pull/8043) ([ولادیمیر چبوتراف](https://github.com/excitoon)) -- افزودن قابلیت اجرا `SYSTEM RELOAD DICTIONARY` پرسوجو در `ON CLUSTER` حالت. [\#8288](https://github.com/ClickHouse/ClickHouse/pull/8288) ([کارخانه شراب سازی گیوم](https://github.com/YiuRULE)) -- افزودن قابلیت اجرا `CREATE DICTIONARY` نمایش داده شد در `ON CLUSTER` حالت. [\#8163](https://github.com/ClickHouse/ClickHouse/pull/8163) ([الساپین](https://github.com/alesapin)) -- در حال حاضر مشخصات کاربر در `users.xml` می توانید پروفایل های متعدد به ارث می برند. [\#8343](https://github.com/ClickHouse/ClickHouse/pull/8343) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) -- اضافه شده `system.stack_trace` جدول که اجازه می دهد تا در اثری پشته از تمام موضوعات سرور نگاه. این برای توسعه دهندگان به درون نگری دولت سرور مفید است. این رفع [\#7576](https://github.com/ClickHouse/ClickHouse/issues/7576). [\#8344](https://github.com/ClickHouse/ClickHouse/pull/8344) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- افزودن `DateTime64` نوع داده با دقت زیر دوم قابل تنظیم. [\#7170](https://github.com/ClickHouse/ClickHouse/pull/7170) ([واسیلی نمکو](https://github.com/Enmk)) -- اضافه کردن تابع جدول `clusterAllReplicas` که اجازه می دهد به پرس و جو تمام گره در خوشه. [\#8493](https://github.com/ClickHouse/ClickHouse/pull/8493) ([کیران سونکری](https://github.com/kiransunkari)) -- اضافه کردن تابع جمع `categoricalInformationValue` که محاسبه ارزش اطلاعات از ویژگی های گسسته. [\#8117](https://github.com/ClickHouse/ClickHouse/pull/8117) ([هکز](https://github.com/hczhcz)) -- سرعت تجزیه فایل های داده در `CSV`, `TSV` و `JSONEachRow` قالب با انجام این کار به صورت موازی. [\#7780](https://github.com/ClickHouse/ClickHouse/pull/7780) ([الکساندر کوزمنکوف](https://github.com/akuzm)) -- افزودن تابع `bankerRound` که انجام گرد بانکدار. [\#8112](https://github.com/ClickHouse/ClickHouse/pull/8112) ([هکز](https://github.com/hczhcz)) -- پشتیبانی از زبان های بیشتر در فرهنگ لغت تعبیه شده برای نام منطقه: ‘ru’, ‘en’, ‘ua’, ‘uk’, ‘by’, ‘kz’, ‘tr’, ‘de’, ‘uz’, ‘lv’, ‘lt’, ‘et’, ‘pt’, ‘he’, ‘vi’. [\#8189](https://github.com/ClickHouse/ClickHouse/pull/8189) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- بهبود در سازگاری `ANY JOIN` منطق. حالا `t1 ANY LEFT JOIN t2` برابر `t2 ANY RIGHT JOIN t1`. [\#7665](https://github.com/ClickHouse/ClickHouse/pull/7665) ([زویکوف](https://github.com/4ertus2)) -- افزودن تنظیمات `any_join_distinct_right_table_keys` که رفتار قدیمی را قادر می سازد `ANY INNER JOIN`. [\#7665](https://github.com/ClickHouse/ClickHouse/pull/7665) ([زویکوف](https://github.com/4ertus2)) -- افزودن جدید `SEMI` و `ANTI JOIN`. قدیمی `ANY INNER JOIN` رفتار در حال حاضر به عنوان در دسترس `SEMI LEFT JOIN`. [\#7665](https://github.com/ClickHouse/ClickHouse/pull/7665) ([زویکوف](https://github.com/4ertus2)) -- اضافه شده `Distributed` قالب برای `File` موتور و `file` تابع جدول که اجازه می دهد تا از خواندن `.bin` فایل های تولید شده توسط درج ناهمزمان به `Distributed` جدول [\#8535](https://github.com/ClickHouse/ClickHouse/pull/8535) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) -- اضافه کردن استدلال ستون تنظیم مجدد اختیاری برای `runningAccumulate` که اجازه می دهد برای تنظیم مجدد نتایج تجمع برای هر مقدار کلید جدید. [\#8326](https://github.com/ClickHouse/ClickHouse/pull/8326) ([سرگی کنوننکو](https://github.com/kononencheg)) -- اضافه کردن توانایی استفاده از فاحشه خانه به عنوان نقطه پایانی پرومته. [\#7900](https://github.com/ClickHouse/ClickHouse/pull/7900) ([ولادیمیر](https://github.com/Vdimir)) -- اضافه کردن بخش `` داخل `config.xml` که میزبان مجاز برای موتورهای جدول از راه دور و توابع جدول را محدود می کند `URL`, `S3`, `HDFS`. [\#7154](https://github.com/ClickHouse/ClickHouse/pull/7154) ([میخیل کوروتف](https://github.com/millb)) -- اضافه شدن تابع `greatCircleAngle` که محاسبه فاصله در یک کره در درجه. [\#8105](https://github.com/ClickHouse/ClickHouse/pull/8105) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- شعاع زمین تغییر می شود سازگار با ساعت3 کتابخانه. [\#8105](https://github.com/ClickHouse/ClickHouse/pull/8105) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- اضافه شده `JSONCompactEachRow` و `JSONCompactEachRowWithNamesAndTypes` فرمت برای ورودی و خروجی. [\#7841](https://github.com/ClickHouse/ClickHouse/pull/7841) ([میخیل کوروتف](https://github.com/millb)) -- ویژگی اضافه شده برای موتورهای جدول مربوط به فایل و توابع جدول (`File`, `S3`, `URL`, `HDFS`) که اجازه می دهد به خواندن و نوشتن `gzip` فایل ها بر اساس پارامتر موتور اضافی و یا فرمت فایل. [\#7840](https://github.com/ClickHouse/ClickHouse/pull/7840) ([هشدار داده می شود](https://github.com/apbodrov)) -- اضافه شدن `randomASCII(length)` تابع, تولید یک رشته با یک مجموعه تصادفی از [ASCII](https://en.wikipedia.org/wiki/ASCII#Printable_characters) شخصیت های قابل چاپ. [\#8401](https://github.com/ClickHouse/ClickHouse/pull/8401) ([سرنیزه](https://github.com/BayoNet)) -- اضافه شدن تابع `JSONExtractArrayRaw` که مجموعه ای از عناصر مجموعه ای از جیسون نامحدود را باز می کند `JSON` رشته. [\#8081](https://github.com/ClickHouse/ClickHouse/pull/8081) ([اولگ متررخین](https://github.com/errx)) -- افزودن `arrayZip` تابع است که اجازه می دهد تا به ترکیب مجموعه ای از چند بند از طول برابر به یک مجموعه ای از تاپل. [\#8149](https://github.com/ClickHouse/ClickHouse/pull/8149) ([زمستان ژانگ](https://github.com/zhang2014)) -- اضافه کردن توانایی انتقال داده ها بین دیسک ها با توجه به پیکربندی `TTL`- عبارات برای `*MergeTree` موتورهای جدول خانواده. [\#8140](https://github.com/ClickHouse/ClickHouse/pull/8140) ([ولادیمیر چبوتراف](https://github.com/excitoon)) -- اضافه شدن تابع جمع جدید `avgWeighted` که اجازه می دهد برای محاسبه میانگین وزن. [\#7898](https://github.com/ClickHouse/ClickHouse/pull/7898) ([هشدار داده می شود](https://github.com/apbodrov)) -- در حال حاضر تجزیه موازی به طور پیش فرض برای فعال `TSV`, `TSKV`, `CSV` و `JSONEachRow` فرمتها. [\#7894](https://github.com/ClickHouse/ClickHouse/pull/7894) ([نیکیتا میخایلو](https://github.com/nikitamikhaylov)) -- اضافه کردن چندین توابع جغرافیایی از `H3` کتابخانه: `h3GetResolution`, `h3EdgeAngle`, `h3EdgeLength`, `h3IsValid` و `h3kRing`. [\#8034](https://github.com/ClickHouse/ClickHouse/pull/8034) ([کنستانتین مالانچو](https://github.com/hombit)) -- اضافه شدن پشتیبانی برای برتلی (`br`) فشرده سازی در ذخیره سازی مربوط به فایل و توابع جدول . این رفع [\#8156](https://github.com/ClickHouse/ClickHouse/issues/8156). [\#8526](https://github.com/ClickHouse/ClickHouse/pull/8526) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- افزودن `groupBit*` توابع برای `SimpleAggregationFunction` نوع. [\#8485](https://github.com/ClickHouse/ClickHouse/pull/8485) ([کارخانه شراب سازی گیوم](https://github.com/YiuRULE)) - -#### رفع اشکال {#bug-fix-6} - -- رفع تغییر نام جداول با `Distributed` موتور رفع مشکل [\#7868](https://github.com/ClickHouse/ClickHouse/issues/7868). [\#8306](https://github.com/ClickHouse/ClickHouse/pull/8306) ([تاولوبیکس](https://github.com/tavplubix)) -- در حال حاضر پشتیبانی لغت نامه `EXPRESSION` برای ویژگی های در رشته دلخواه در گویش غیر کلیک میدان. [\#8098](https://github.com/ClickHouse/ClickHouse/pull/8098) ([الساپین](https://github.com/alesapin)) -- رفع شکسته `INSERT SELECT FROM mysql(...)` پرس و جو. این رفع [\#8070](https://github.com/ClickHouse/ClickHouse/issues/8070) و [\#7960](https://github.com/ClickHouse/ClickHouse/issues/7960). [\#8234](https://github.com/ClickHouse/ClickHouse/pull/8234) ([تاولوبیکس](https://github.com/tavplubix)) -- رفع خطا “Mismatch column sizes” هنگام وارد کردن پیشفرض `Tuple` از `JSONEachRow`. این رفع [\#5653](https://github.com/ClickHouse/ClickHouse/issues/5653). [\#8606](https://github.com/ClickHouse/ClickHouse/pull/8606) ([تاولوبیکس](https://github.com/tavplubix)) -- در حال حاضر یک استثنا خواهد شد در صورت استفاده از پرتاب `WITH TIES` در کنار `LIMIT BY`. همچنین توانایی استفاده را اضافه کنید `TOP` با `LIMIT BY`. این رفع [\#7472](https://github.com/ClickHouse/ClickHouse/issues/7472). [\#7637](https://github.com/ClickHouse/ClickHouse/pull/7637) ([نیکیتا میخایلو](https://github.com/nikitamikhaylov)) -- رفع unintendent وابستگی از تازه glibc نسخه در `clickhouse-odbc-bridge` دودویی. [\#8046](https://github.com/ClickHouse/ClickHouse/pull/8046) ([ایموس پرنده](https://github.com/amosbird)) -- رفع اشکال در بررسی عملکرد `*MergeTree` خانواده موتور. در حال حاضر در صورتی که زمانی که ما مقدار مساوی از ردیف در گرانول گذشته و علامت گذشته (غیر نهایی) شکست نیست. [\#8047](https://github.com/ClickHouse/ClickHouse/pull/8047) ([الساپین](https://github.com/alesapin)) -- رفع درج در `Enum*` ستون پس از `ALTER` پرس و جو, زمانی که زمینه ای نوع عددی به جدول نوع مشخص برابر است. این رفع [\#7836](https://github.com/ClickHouse/ClickHouse/issues/7836). [\#7908](https://github.com/ClickHouse/ClickHouse/pull/7908) ([انتون پوپوف](https://github.com/CurtizJ)) -- مجاز منفی غیر ثابت “size” استدلال برای عملکرد `substring`. این اشتباه مجاز نیست. این رفع [\#4832](https://github.com/ClickHouse/ClickHouse/issues/4832). [\#7703](https://github.com/ClickHouse/ClickHouse/pull/7703) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- رفع اشکال تجزیه زمانی که تعداد اشتباه استدلال به تصویب رسید `(O|J)DBC` موتور جدول. [\#7709](https://github.com/ClickHouse/ClickHouse/pull/7709) ([الساپین](https://github.com/alesapin)) -- با استفاده از نام فرمان از روند در حال اجرا خانه کلیک در هنگام ارسال سیاهههای مربوط به سیسلوگ. در نسخه های قبلی, رشته خالی به جای نام فرمان مورد استفاده قرار گرفت. [\#8460](https://github.com/ClickHouse/ClickHouse/pull/8460) ([مایکل ناچاروف](https://github.com/mnach)) -- رفع چک میزبان مجاز برای `localhost`. این روابط عمومی رفع راه حل در [\#8241](https://github.com/ClickHouse/ClickHouse/pull/8241). [\#8342](https://github.com/ClickHouse/ClickHouse/pull/8342) ([ویتالی بارانو](https://github.com/vitlibar)) -- رفع سقوط نادر در `argMin` و `argMax` توابع برای استدلال رشته طولانی, زمانی که نتیجه در استفاده `runningAccumulate` تابع. این رفع [\#8325](https://github.com/ClickHouse/ClickHouse/issues/8325) [\#8341](https://github.com/ClickHouse/ClickHouse/pull/8341) ([دایناسور](https://github.com/769344359)) -- رفع بیش از حد حافظه برای جداول با `Buffer` موتور [\#8345](https://github.com/ClickHouse/ClickHouse/pull/8345) ([ازات خوژین](https://github.com/azat)) -- اشکال بالقوه ثابت در توابع است که می تواند `NULL` به عنوان یکی از استدلال و بازگشت غیر تهی. [\#8196](https://github.com/ClickHouse/ClickHouse/pull/8196) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- معیارهای محاسبات بهتر در استخر موضوع برای فرایندهای پس زمینه برای `MergeTree` موتورهای جدول. [\#8194](https://github.com/ClickHouse/ClickHouse/pull/8194) ([ولادیمیر چبوتراف](https://github.com/excitoon)) -- تابع ثابت `IN` داخل `WHERE` بیانیه ای که در سطح ردیف فیلتر جدول وجود دارد. رفع [\#6687](https://github.com/ClickHouse/ClickHouse/issues/6687) [\#8357](https://github.com/ClickHouse/ClickHouse/pull/8357) ([ایوان](https://github.com/abyss7)) -- در حال حاضر یک استثنا پرتاب می شود اگر ارزش جدایی ناپذیر است به طور کامل برای مقادیر تنظیمات تجزیه نشده است. [\#7678](https://github.com/ClickHouse/ClickHouse/pull/7678) ([میخیل کوروتف](https://github.com/millb)) -- رفع استثنا زمانی که عملکرد دانه ها در پرس و جو به جدول توزیع شده با بیش از دو خرده ریز محلی استفاده می شود. [\#8164](https://github.com/ClickHouse/ClickHouse/pull/8164) ([小路](https://github.com/nicelulu)) -- در حال حاضر فیلتر بلوم می توانید طول صفر عرایز رسیدگی می کند و محاسبات کار برکنار شده انجام نمی دهد. [\#8242](https://github.com/ClickHouse/ClickHouse/pull/8242) ([ایشیمب](https://github.com/achimbab)) -- چک کردن ثابت اگر یک میزبان مشتری با تطبیق میزبان مشتری به اجازه `host_regexp` مشخص شده در `users.xml`. [\#8241](https://github.com/ClickHouse/ClickHouse/pull/8241) ([ویتالی بارانو](https://github.com/vitlibar)) -- استراحت چک ستون مبهم است که منجر به مثبت کاذب در چند `JOIN ON` بخش. [\#8385](https://github.com/ClickHouse/ClickHouse/pull/8385) ([زویکوف](https://github.com/4ertus2)) -- ثابت سقوط سرور ممکن است (`std::terminate`) هنگامی که سرور نمی تواند ارسال و یا نوشتن داده ها در `JSON` یا `XML` قالب با مقادیر `String` نوع داده (که نیاز به `UTF-8` اعتبار سنجی) و یا زمانی که فشرده سازی داده های نتیجه با الگوریتم برتلی و یا در برخی موارد نادر دیگر. این رفع [\#7603](https://github.com/ClickHouse/ClickHouse/issues/7603) [\#8384](https://github.com/ClickHouse/ClickHouse/pull/8384) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- رفع شرایط مسابقه در `StorageDistributedDirectoryMonitor` پیدا شده توسط سی. این رفع [\#8364](https://github.com/ClickHouse/ClickHouse/issues/8364). [\#8383](https://github.com/ClickHouse/ClickHouse/pull/8383) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) -- در حال حاضر پس زمینه ادغام در `*MergeTree` جدول موتورهای خانواده حفظ سیاست ذخیره سازی حجم سفارش دقیق تر است. [\#8549](https://github.com/ClickHouse/ClickHouse/pull/8549) ([ولادیمیر چبوتراف](https://github.com/excitoon)) -- در حال حاضر موتور جدول `Kafka` به درستی کار می کند با `Native` قالب. این رفع [\#6731](https://github.com/ClickHouse/ClickHouse/issues/6731) [\#7337](https://github.com/ClickHouse/ClickHouse/issues/7337) [\#8003](https://github.com/ClickHouse/ClickHouse/issues/8003). [\#8016](https://github.com/ClickHouse/ClickHouse/pull/8016) ([فیلیمونف](https://github.com/filimonov)) -- فرمت های ثابت با هدر (مانند `CSVWithNames`) که پرتاب شد استثنا در مورد ایف برای موتور جدول `Kafka`. [\#8016](https://github.com/ClickHouse/ClickHouse/pull/8016) ([فیلیمونف](https://github.com/filimonov)) -- رفع اشکال با ساخت مجموعه ای از زیرخاکری در قسمت سمت راست از `IN` بخش. این رفع [\#5767](https://github.com/ClickHouse/ClickHouse/issues/5767) و [\#2542](https://github.com/ClickHouse/ClickHouse/issues/2542). [\#7755](https://github.com/ClickHouse/ClickHouse/pull/7755) ([نیکیتا میخایلو](https://github.com/nikitamikhaylov)) -- رفع سقوط احتمالی در هنگام خواندن از ذخیره سازی `File`. [\#7756](https://github.com/ClickHouse/ClickHouse/pull/7756) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) -- خواندن ثابت از فایل ها در `Parquet` قالب حاوی ستونهای نوع `list`. [\#8334](https://github.com/ClickHouse/ClickHouse/pull/8334) ([مکسولان](https://github.com/maxulan)) -- رفع خطا `Not found column` برای نمایش داده شد توزیع با `PREWHERE` شرایط وابسته به کلید نمونه برداری اگر `max_parallel_replicas > 1`. [\#7913](https://github.com/ClickHouse/ClickHouse/pull/7913) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) -- رفع خطا `Not found column` اگر پرسوجو استفاده شود `PREWHERE` وابسته به نام مستعار جدول و مجموعه نتیجه به دلیل شرایط کلیدی اولیه خالی بود. [\#7911](https://github.com/ClickHouse/ClickHouse/pull/7911) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) -- نوع بازگشت ثابت برای توابع `rand` و `randConstant` در صورت `Nullable` استدلال کردن. در حال حاضر توابع همیشه بازگشت `UInt32` و هرگز `Nullable(UInt32)`. [\#8204](https://github.com/ClickHouse/ClickHouse/pull/8204) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) -- پیش فرض غیر فعال فشار پایین برای `WITH FILL` اصطلاح. این رفع [\#7784](https://github.com/ClickHouse/ClickHouse/issues/7784). [\#7789](https://github.com/ClickHouse/ClickHouse/pull/7789) ([زمستان ژانگ](https://github.com/zhang2014)) -- نادرست ثابت `count()` نتیجه برای `SummingMergeTree` چه زمانی `FINAL` بخش استفاده شده است. [\#3280](https://github.com/ClickHouse/ClickHouse/issues/3280) [\#7786](https://github.com/ClickHouse/ClickHouse/pull/7786) ([نیکیتا میخایلو](https://github.com/nikitamikhaylov)) -- رفع نتیجه نادرست ممکن است برای توابع ثابت از سرور از راه دور. این برای نمایش داده شد با توابع مانند اتفاق افتاد `version()`, `uptime()`, و غیره. که برمی گرداند مقادیر ثابت مختلف برای سرورهای مختلف. این رفع [\#7666](https://github.com/ClickHouse/ClickHouse/issues/7666). [\#7689](https://github.com/ClickHouse/ClickHouse/pull/7689) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) -- رفع اشکال پیچیده در بهینه سازی پیش فرض فشار به پایین که منجر به نتایج اشتباه است. این مشکلات زیادی را در بهینه سازی پیش فرض پایین حل می کند. [\#8503](https://github.com/ClickHouse/ClickHouse/pull/8503) ([زمستان ژانگ](https://github.com/zhang2014)) -- رفع سقوط در `CREATE TABLE .. AS dictionary` پرس و جو. [\#8508](https://github.com/ClickHouse/ClickHouse/pull/8508) ([ازات خوژین](https://github.com/azat)) -- چند بهبود دستور زبان کلیک در `.g4` پرونده. [\#8294](https://github.com/ClickHouse/ClickHouse/pull/8294) ([taiyang-li](https://github.com/taiyang-li)) -- رفع اشکال که منجر به سقوط در `JOIN`با جداول با موتور `Join`. این رفع [\#7556](https://github.com/ClickHouse/ClickHouse/issues/7556) [\#8254](https://github.com/ClickHouse/ClickHouse/issues/8254) [\#7915](https://github.com/ClickHouse/ClickHouse/issues/7915) [\#8100](https://github.com/ClickHouse/ClickHouse/issues/8100). [\#8298](https://github.com/ClickHouse/ClickHouse/pull/8298) ([زویکوف](https://github.com/4ertus2)) -- رفع لغت نامه کار برکنار بارگذاری مجدد در `CREATE DATABASE`. [\#7916](https://github.com/ClickHouse/ClickHouse/pull/7916) ([ازات خوژین](https://github.com/azat)) -- محدود کردن حداکثر تعداد جریان برای خواندن از `StorageFile` و `StorageHDFS`. رفع https://github.com/ClickHouse/ClickHouse/issues/7650. [\#7981](https://github.com/ClickHouse/ClickHouse/pull/7981) ([الساپین](https://github.com/alesapin)) -- رفع اشکال در `ALTER ... MODIFY ... CODEC` پرس و جو, زمانی که کاربر هر دو عبارت به طور پیش فرض و کدک را مشخص کنید. رفع [8593](https://github.com/ClickHouse/ClickHouse/issues/8593). [\#8614](https://github.com/ClickHouse/ClickHouse/pull/8614) ([الساپین](https://github.com/alesapin)) -- رفع خطا در ادغام پس زمینه ستون ها با `SimpleAggregateFunction(LowCardinality)` نوع. [\#8613](https://github.com/ClickHouse/ClickHouse/pull/8613) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) -- بررسی نوع ثابت در عملکرد `toDateTime64`. [\#8375](https://github.com/ClickHouse/ClickHouse/pull/8375) ([واسیلی نمکو](https://github.com/Enmk)) -- در حال حاضر سرور در سقوط نیست `LEFT` یا `FULL JOIN` با و پیوستن به موتور و پشتیبانی نشده `join_use_nulls` تنظیمات. [\#8479](https://github.com/ClickHouse/ClickHouse/pull/8479) ([زویکوف](https://github.com/4ertus2)) -- حالا `DROP DICTIONARY IF EXISTS db.dict` پرس و جو می کند و پرتاب استثنا اگر `db` وجود نداره [\#8185](https://github.com/ClickHouse/ClickHouse/pull/8185) ([ویتالی بارانو](https://github.com/vitlibar)) -- رفع سقوط احتمالی در توابع جدول (`file`, `mysql`, `remote`) ناشی از استفاده از مرجع به حذف `IStorage` اعتراض. رفع تجزیه نادرست از ستون مشخص شده در درج به تابع جدول. [\#7762](https://github.com/ClickHouse/ClickHouse/pull/7762) ([تاولوبیکس](https://github.com/tavplubix)) -- اطمینان از شبکه قبل از شروع `clickhouse-server`. این رفع [\#7507](https://github.com/ClickHouse/ClickHouse/issues/7507). [\#8570](https://github.com/ClickHouse/ClickHouse/pull/8570) ([ژیچنگ یو](https://github.com/yuzhichang)) -- رفع وقفه دست زدن به برای اتصالات امن, بنابراین نمایش داده شد می کند بی تعریف قطع نمی. این رفع [\#8126](https://github.com/ClickHouse/ClickHouse/issues/8126). [\#8128](https://github.com/ClickHouse/ClickHouse/pull/8128) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- ثابت `clickhouse-copier`مشاجره کار برکنار بین کارگران همزمان. [\#7816](https://github.com/ClickHouse/ClickHouse/pull/7816) ([هشدار داده می شود](https://github.com/dingxiangfei2009)) -- در حال حاضر جهش می کند قطعات متصل جست و خیز نیست, حتی اگر نسخه جهش خود را بزرگتر از نسخه جهش فعلی بود. [\#7812](https://github.com/ClickHouse/ClickHouse/pull/7812) ([ژیچنگ یو](https://github.com/yuzhichang)) [\#8250](https://github.com/ClickHouse/ClickHouse/pull/8250) ([الساپین](https://github.com/alesapin)) -- چشمپوشی از نسخههای برکنار شده `*MergeTree` قطعات داده ها پس از حرکت به یکی دیگر از دیسک و سرور راه اندازی مجدد. [\#7810](https://github.com/ClickHouse/ClickHouse/pull/7810) ([ولادیمیر چبوتراف](https://github.com/excitoon)) -- رفع سقوط در `FULL JOIN` با `LowCardinality` داخل `JOIN` کلیدی است. [\#8252](https://github.com/ClickHouse/ClickHouse/pull/8252) ([زویکوف](https://github.com/4ertus2)) -- ممنوع به استفاده از نام ستون بیش از یک بار در قرار دادن پرس و جو مانند `INSERT INTO tbl (x, y, x)`. این رفع [\#5465](https://github.com/ClickHouse/ClickHouse/issues/5465), [\#7681](https://github.com/ClickHouse/ClickHouse/issues/7681). [\#7685](https://github.com/ClickHouse/ClickHouse/pull/7685) ([الساپین](https://github.com/alesapin)) -- اضافه شدن مجدد برای تشخیص تعداد هسته های پردازنده فیزیکی برای پردازنده ناشناخته (با استفاده از تعدادی از هسته های پردازنده منطقی). این رفع [\#5239](https://github.com/ClickHouse/ClickHouse/issues/5239). [\#7726](https://github.com/ClickHouse/ClickHouse/pull/7726) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- ثابت `There's no column` خطا برای مفردات و نام مستعار ستون. [\#8210](https://github.com/ClickHouse/ClickHouse/pull/8210) ([زویکوف](https://github.com/4ertus2)) -- تصادف شدید ثابت زمانی که `EXISTS` پرس و جو بدون استفاده شد `TABLE` یا `DICTIONARY` مقدماتی. درست مثل `EXISTS t`. این رفع [\#8172](https://github.com/ClickHouse/ClickHouse/issues/8172). این اشکال در نسخه 19.17 معرفی شد. [\#8213](https://github.com/ClickHouse/ClickHouse/pull/8213) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- رفع اشکال نادر با خطا `"Sizes of columns doesn't match"` که ممکن است به نظر می رسد در هنگام استفاده از `SimpleAggregateFunction` ستون. [\#7790](https://github.com/ClickHouse/ClickHouse/pull/7790) ([بوریس گرانویو](https://github.com/bgranvea)) -- رفع اشکال که کاربر با خالی است `allow_databases` دسترسی به تمام پایگاه های داده کردم (و همین کار را برای `allow_dictionaries`). [\#7793](https://github.com/ClickHouse/ClickHouse/pull/7793) ([دوفایت](https://github.com/DeifyTheGod)) -- رفع سقوط مشتری زمانی که سرور در حال حاضر از مشتری قطع شده است. [\#8071](https://github.com/ClickHouse/ClickHouse/pull/8071) ([ازات خوژین](https://github.com/azat)) -- ثابت `ORDER BY` رفتار در صورت مرتب سازی بر اساس پیشوند کلید اولیه و پسوند کلید غیر اولیه. [\#7759](https://github.com/ClickHouse/ClickHouse/pull/7759) ([انتون پوپوف](https://github.com/CurtizJ)) -- بررسی کنید که در حال حاضر ستون واجد شرایط در جدول. این رفع [\#6836](https://github.com/ClickHouse/ClickHouse/issues/6836). [\#7758](https://github.com/ClickHouse/ClickHouse/pull/7758) ([زویکوف](https://github.com/4ertus2)) -- رفتار ثابت با `ALTER MOVE` فرار بلافاصله پس از ادغام پایان حرکت سوپر شروع مشخص شده است. رفع [\#8103](https://github.com/ClickHouse/ClickHouse/issues/8103). [\#8104](https://github.com/ClickHouse/ClickHouse/pull/8104) ([ولادیمیر چبوتراف](https://github.com/excitoon)) -- رفع سقوط سرور ممکن است در حالی که با استفاده از `UNION` با شماره های مختلف ستون. رفع [\#7279](https://github.com/ClickHouse/ClickHouse/issues/7279). [\#7929](https://github.com/ClickHouse/ClickHouse/pull/7929) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) -- رفع اندازه زیر رشته نتیجه برای عملکرد `substr` با اندازه منفی. [\#8589](https://github.com/ClickHouse/ClickHouse/pull/8589) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) -- در حال حاضر سرور جهش بخشی در اجرا نیست `MergeTree` اگر موضوعات رایگان به اندازه کافی در استخر پس زمینه وجود ندارد. [\#8588](https://github.com/ClickHouse/ClickHouse/pull/8588) ([تاولوبیکس](https://github.com/tavplubix)) -- رفع خطای تایپی کوچک در قالب بندی `UNION ALL` AST. [\#7999](https://github.com/ClickHouse/ClickHouse/pull/7999) ([لیتا91](https://github.com/litao91)) -- ثابت نتایج فیلتر شکوفه نادرست برای اعداد منفی. این رفع [\#8317](https://github.com/ClickHouse/ClickHouse/issues/8317). [\#8566](https://github.com/ClickHouse/ClickHouse/pull/8566) ([زمستان ژانگ](https://github.com/zhang2014)) -- سرریز بافر بالقوه ثابت در حالت فشرده خارج. کاربر مخرب می تواند داده های فشرده ساخته شده است که باعث می شود پس از بافر به عنوان خوانده شده منتقل می کند. این موضوع توسط الدار زیتوف از تیم امنیت اطلاعات یاندکس یافت شد. [\#8404](https://github.com/ClickHouse/ClickHouse/pull/8404) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- رفع نتیجه نادرست به دلیل سرریز اعداد صحیح در `arrayIntersect`. [\#7777](https://github.com/ClickHouse/ClickHouse/pull/7777) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) -- حالا `OPTIMIZE TABLE` پرس و جو نمی خواهد برای کپی نیست منتظر به انجام عملیات. [\#8314](https://github.com/ClickHouse/ClickHouse/pull/8314) ([جوی سانتانا](https://github.com/javisantana)) -- ثابت `ALTER TTL` تجزیه کننده برای `Replicated*MergeTree` میز [\#8318](https://github.com/ClickHouse/ClickHouse/pull/8318) ([ولادیمیر چبوتراف](https://github.com/excitoon)) -- رفع ارتباط بین سرور و کلاینت, بنابراین سرور خواندن اطلاعات جداول موقت پس از شکست پرس و جو. [\#8084](https://github.com/ClickHouse/ClickHouse/pull/8084) ([ازات خوژین](https://github.com/azat)) -- ثابت `bitmapAnd` خطای تابع هنگام تقاطع کردن بیت مپ جمع و بیت مپ اسکالر. [\#8082](https://github.com/ClickHouse/ClickHouse/pull/8082) ([یو هوانگ](https://github.com/moon03432)) -- اصلاح تعریف `ZXid` با توجه به راهنمای برنامه نویس باغ وحش که رفع اشکال در `clickhouse-cluster-copier`. [\#8088](https://github.com/ClickHouse/ClickHouse/pull/8088) ([هشدار داده می شود](https://github.com/dingxiangfei2009)) -- `odbc` تابع جدول در حال حاضر احترام می گذارد `external_table_functions_use_nulls` تنظیمات. [\#7506](https://github.com/ClickHouse/ClickHouse/pull/7506) ([واسیلی نمکو](https://github.com/Enmk)) -- اشکال ثابت که منجر به یک مسابقه داده نادر است. [\#8143](https://github.com/ClickHouse/ClickHouse/pull/8143) ([الکساندر کازاکوف](https://github.com/Akazz)) -- حالا `SYSTEM RELOAD DICTIONARY` بارگذاری مجدد یک فرهنگ لغت به طور کامل, نادیده گرفتن `update_field`. این رفع [\#7440](https://github.com/ClickHouse/ClickHouse/issues/7440). [\#8037](https://github.com/ClickHouse/ClickHouse/pull/8037) ([ویتالی بارانو](https://github.com/vitlibar)) -- اضافه کردن توانایی برای بررسی اگر فرهنگ لغت در ایجاد پرس و جو وجود دارد. [\#8032](https://github.com/ClickHouse/ClickHouse/pull/8032) ([الساپین](https://github.com/alesapin)) -- ثابت `Float*` تجزیه در `Values` قالب. این رفع [\#7817](https://github.com/ClickHouse/ClickHouse/issues/7817). [\#7870](https://github.com/ClickHouse/ClickHouse/pull/7870) ([تاولوبیکس](https://github.com/tavplubix)) -- رفع سقوط زمانی که ما نمی توانیم فضای ذخیره در برخی از عملیات پس زمینه از `*MergeTree` موتورهای جدول خانواده. [\#7873](https://github.com/ClickHouse/ClickHouse/pull/7873) ([ولادیمیر چبوتراف](https://github.com/excitoon)) -- رفع سقوط عملیات ادغام زمانی که جدول شامل `SimpleAggregateFunction(LowCardinality)` ستون. این رفع [\#8515](https://github.com/ClickHouse/ClickHouse/issues/8515). [\#8522](https://github.com/ClickHouse/ClickHouse/pull/8522) ([ازات خوژین](https://github.com/azat)) -- بازگرداندن پشتیبانی از تمام مناطق فناوری اطلاعات و ارتباطات و اضافه کردن توانایی به درخواست تلفیقی برای عبارات ثابت. همچنین نام زبان را اضافه کنید `system.collations` جدول [\#8051](https://github.com/ClickHouse/ClickHouse/pull/8051) ([الساپین](https://github.com/alesapin)) -- رفع اشکال در هنگام خارجی لغت نامه صفر و حداقل طول عمر (`LIFETIME(MIN 0 MAX N)`, `LIFETIME(N)`) در پس زمینه به روز رسانی نیست . [\#7983](https://github.com/ClickHouse/ClickHouse/pull/7983) ([الساپین](https://github.com/alesapin)) -- رفع تصادف در هنگام فرهنگ لغت خارجی با منبع کلیک است زیرخاکی در پرس و جو. [\#8351](https://github.com/ClickHouse/ClickHouse/pull/8351) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) -- رفع نادرست تجزیه پسوند فایل در جدول با موتور `URL`. این رفع [\#8157](https://github.com/ClickHouse/ClickHouse/issues/8157). [\#8419](https://github.com/ClickHouse/ClickHouse/pull/8419) ([هشدار داده می شود](https://github.com/apbodrov)) -- ثابت `CHECK TABLE` پرسوجو برای `*MergeTree` جداول بدون کلید. رفع [\#7543](https://github.com/ClickHouse/ClickHouse/issues/7543). [\#7979](https://github.com/ClickHouse/ClickHouse/pull/7979) ([الساپین](https://github.com/alesapin)) -- تبدیل ثابت از `Float64` به نوع خروجی زیر. [\#8079](https://github.com/ClickHouse/ClickHouse/pull/8079) ([یوری بارانوف](https://github.com/yurriy)) -- در حال حاضر اگر جدول به طور کامل به دلیل سقوط سرور کاهش یافته است, سرور سعی خواهد کرد برای بازگرداندن و بارگذاری. [\#8176](https://github.com/ClickHouse/ClickHouse/pull/8176) ([تاولوبیکس](https://github.com/tavplubix)) -- تصادف ثابت در عملکرد جدول `file` در حالی که قرار دادن به فایل که وجود ندارد. در حال حاضر در این مورد فایل ایجاد می شود و سپس قرار دادن پردازش می شود. [\#8177](https://github.com/ClickHouse/ClickHouse/pull/8177) ([اولگا خوستیکوا](https://github.com/stavrolia)) -- رفع بن بست نادر است که می تواند زمانی اتفاق می افتد `trace_log` در را فعال کنید. [\#7838](https://github.com/ClickHouse/ClickHouse/pull/7838) ([فیلیمونف](https://github.com/filimonov)) -- اضافه کردن توانایی برای کار با انواع مختلف علاوه بر `Date` داخل `RangeHashed` فرهنگ لغت خارجی ایجاد شده از پرس و جو دی ال. رفع [7899](https://github.com/ClickHouse/ClickHouse/issues/7899). [\#8275](https://github.com/ClickHouse/ClickHouse/pull/8275) ([الساپین](https://github.com/alesapin)) -- رفع سقوط زمانی که `now64()` با نتیجه تابع دیگری نامیده می شود. [\#8270](https://github.com/ClickHouse/ClickHouse/pull/8270) ([واسیلی نمکو](https://github.com/Enmk)) -- اشکال ثابت با تشخیص قانون مجازات اسلامی مشتری برای اتصال به شبکه از طریق پروتکل سیم خروجی زیر. [\#7743](https://github.com/ClickHouse/ClickHouse/pull/7743) ([دیمیتری موزیکا](https://github.com/dmitriy-myz)) -- رفع دست زدن به مجموعه خالی در `arraySplit` تابع. این رفع [\#7708](https://github.com/ClickHouse/ClickHouse/issues/7708). [\#7747](https://github.com/ClickHouse/ClickHouse/pull/7747) ([هکز](https://github.com/hczhcz)) -- ثابت موضوع زمانی که `pid-file` از یکی دیگر از در حال اجرا `clickhouse-server` ممکن است حذف شود. [\#8487](https://github.com/ClickHouse/ClickHouse/pull/8487) ([Weiqing زو](https://github.com/weiqxu)) -- رفع بارگذاری مجدد فرهنگ لغت در صورتی که `invalidate_query` که متوقف به روز رسانی و برخی از استثنا در به روز رسانی قبلی تلاش می کند. [\#8029](https://github.com/ClickHouse/ClickHouse/pull/8029) ([الساپین](https://github.com/alesapin)) -- ثابت خطا در تابع `arrayReduce` که ممکن است منجر به “double free” و خطا در ترکیب تابع کل `Resample` که ممکن است به نشت حافظه منجر شود. اضافه شدن تابع جمع `aggThrow`. این تابع را می توان برای اهداف تست استفاده کرد. [\#8446](https://github.com/ClickHouse/ClickHouse/pull/8446) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - -#### بهبود {#improvement-1} - -- ورود به سیستم بهبود یافته در هنگام کار با `S3` موتور جدول. [\#8251](https://github.com/ClickHouse/ClickHouse/pull/8251) ([گریگوری پرواکوف](https://github.com/GrigoryPervakov)) -- پیام راهنما چاپ شده زمانی که هیچ استدلال در هنگام تماس گذشت `clickhouse-local`. این رفع [\#5335](https://github.com/ClickHouse/ClickHouse/issues/5335). [\#8230](https://github.com/ClickHouse/ClickHouse/pull/8230) ([هشدار داده می شود](https://github.com/Melancholic)) -- افزودن تنظیمات `mutations_sync` که اجازه می دهد تا صبر کنید `ALTER UPDATE/DELETE` نمایش داده شد همزمان. [\#8237](https://github.com/ClickHouse/ClickHouse/pull/8237) ([الساپین](https://github.com/alesapin)) -- اجازه برای راه اندازی نسبی `user_files_path` داخل `config.xml` (در راه شبیه به `format_schema_path`). [\#7632](https://github.com/ClickHouse/ClickHouse/pull/7632) ([هکز](https://github.com/hczhcz)) -- اضافه کردن استثنا برای انواع غیر قانونی برای توابع تبدیل با `-OrZero` پسوند. [\#7880](https://github.com/ClickHouse/ClickHouse/pull/7880) ([اندری کونیایف](https://github.com/akonyaev90)) -- ساده فرمت هدر داده ارسال به سفال در پرس و جو توزیع شده است. [\#8044](https://github.com/ClickHouse/ClickHouse/pull/8044) ([ویتالی بارانو](https://github.com/vitlibar)) -- `Live View` تعمیر مجدد موتور جدول. [\#8519](https://github.com/ClickHouse/ClickHouse/pull/8519) ([vzakaznikov](https://github.com/vzakaznikov)) -- اضافه کردن چک های اضافی برای لغت نامه های خارجی ایجاد شده از دی ال نمایش داده شد. [\#8127](https://github.com/ClickHouse/ClickHouse/pull/8127) ([الساپین](https://github.com/alesapin)) -- رفع خطا `Column ... already exists` در حالی که با استفاده از `FINAL` و `SAMPLE` together, e.g. `select count() from table final sample 1/2`. رفع [\#5186](https://github.com/ClickHouse/ClickHouse/issues/5186). [\#7907](https://github.com/ClickHouse/ClickHouse/pull/7907) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) -- در حال حاضر جدول استدلال اول `joinGet` تابع می تواند جدول شناسایی. [\#7707](https://github.com/ClickHouse/ClickHouse/pull/7707) ([ایموس پرنده](https://github.com/amosbird)) -- اجازه استفاده `MaterializedView` با subqueries بالا `Kafka` میز [\#8197](https://github.com/ClickHouse/ClickHouse/pull/8197) ([فیلیمونف](https://github.com/filimonov)) -- در حال حاضر پس زمینه بین دیسک ها حرکت می کند استخر تار شده را اجرا می کند. [\#7670](https://github.com/ClickHouse/ClickHouse/pull/7670) ([ولادیمیر چبوتراف](https://github.com/excitoon)) -- `SYSTEM RELOAD DICTIONARY` در حال حاضر اجرا همزمان. [\#8240](https://github.com/ClickHouse/ClickHouse/pull/8240) ([ویتالی بارانو](https://github.com/vitlibar)) -- ردیابی پشته در حال حاضر نمایش نشانی های فیزیکی (شیپور خاموشی در فایل شی) به جای نشانی های حافظه مجازی (جایی که فایل شی لود شد). که اجازه می دهد تا استفاده از `addr2line` هنگامی که باینری موقعیت مستقل است و اصل فعال است. این رفع [\#8360](https://github.com/ClickHouse/ClickHouse/issues/8360). [\#8387](https://github.com/ClickHouse/ClickHouse/pull/8387) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- پشتیبانی از نحو جدید برای فیلترهای امنیتی سطح ردیف: `
…
`. رفع [\#5779](https://github.com/ClickHouse/ClickHouse/issues/5779). [\#8381](https://github.com/ClickHouse/ClickHouse/pull/8381) ([ایوان](https://github.com/abyss7)) -- حالا `cityHash` تابع می تواند با کار `Decimal` و `UUID` انواع. رفع [\#5184](https://github.com/ClickHouse/ClickHouse/issues/5184). [\#7693](https://github.com/ClickHouse/ClickHouse/pull/7693) ([میخیل کوروتف](https://github.com/millb)) -- دانه دانه دانه دانه ثابت حذف (1024 بود) از سیاهههای مربوط به سیستم به دلیل منسوخ پس از اجرای دانه دانه تطبیقی. [\#7698](https://github.com/ClickHouse/ClickHouse/pull/7698) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- فعال خروجی زیر سرور سازگاری زمانی که تاتر بدون اس اس ال وارد شده است. [\#7852](https://github.com/ClickHouse/ClickHouse/pull/7852) ([یوری بارانوف](https://github.com/yurriy)) -- در حال حاضر چک سرور دسته توزیع, می دهد که خطاهای طولانی تر در مورد داده های خراب شده در دسته ای. [\#7914](https://github.com/ClickHouse/ClickHouse/pull/7914) ([ازات خوژین](https://github.com/azat)) -- پشتیبانی `DROP DATABASE`, `DETACH TABLE`, `DROP TABLE` و `ATTACH TABLE` برای `MySQL` موتور پایگاه داده. [\#8202](https://github.com/ClickHouse/ClickHouse/pull/8202) ([زمستان ژانگ](https://github.com/zhang2014)) -- اضافه کردن احراز هویت در اس 3 تابع جدول و موتور جدول. [\#7623](https://github.com/ClickHouse/ClickHouse/pull/7623) ([ولادیمیر چبوتراف](https://github.com/excitoon)) -- اضافه شدن چک برای قطعات اضافی از `MergeTree` در دیسک های مختلف, به منظور اجازه نمی دهد به دست قطعات داده در دیسک تعریف نشده. [\#8118](https://github.com/ClickHouse/ClickHouse/pull/8118) ([ولادیمیر چبوتراف](https://github.com/excitoon)) -- فعال کردن پشتیبانی اس اس ال برای مشتری مک و سرور. [\#8297](https://github.com/ClickHouse/ClickHouse/pull/8297) ([ایوان](https://github.com/abyss7)) -- در حال حاضر clickhouse کار می تواند به عنوان خروجی يکپارچه سرور (دیدن https://dev.mysql.com/doc/refman/5.7/en/federated-create-server.html). [\#7717](https://github.com/ClickHouse/ClickHouse/pull/7717) ([ماکسیم فدوتف](https://github.com/MaxFedotov)) -- `clickhouse-client` در حال حاضر تنها فعال `bracketed-paste` هنگامی که چند ضلعی روشن است و چند خطی خاموش است. این رفع (\#7757) \[https://github.com/ClickHouse/ClickHouse/issues/7757\]. [\#7761](https://github.com/ClickHouse/ClickHouse/pull/7761) ([ایموس پرنده](https://github.com/amosbird)) -- پشتیبانی `Array(Decimal)` داخل `if` تابع. [\#7721](https://github.com/ClickHouse/ClickHouse/pull/7721) ([زویکوف](https://github.com/4ertus2)) -- اعشار پشتیبانی در `arrayDifference`, `arrayCumSum` و `arrayCumSumNegative` توابع. [\#7724](https://github.com/ClickHouse/ClickHouse/pull/7724) ([زویکوف](https://github.com/4ertus2)) -- اضافه شده `lifetime` ستون به `system.dictionaries` جدول [\#6820](https://github.com/ClickHouse/ClickHouse/issues/6820) [\#7727](https://github.com/ClickHouse/ClickHouse/pull/7727) ([کیککول](https://github.com/kekekekule)) -- بررسی بهبود یافته برای قطعات موجود بر روی دیسک های مختلف برای `*MergeTree` موتورهای جدول. نشانیهای [\#7660](https://github.com/ClickHouse/ClickHouse/issues/7660). [\#8440](https://github.com/ClickHouse/ClickHouse/pull/8440) ([ولادیمیر چبوتراف](https://github.com/excitoon)) -- ادغام با `AWS SDK` برای `S3` فعل و انفعالات که اجازه می دهد تا به استفاده از تمام 3 ویژگی های خارج از جعبه. [\#8011](https://github.com/ClickHouse/ClickHouse/pull/8011) ([پاول کووالنکو](https://github.com/Jokser)) -- اضافه شدن پشتیبانی برای کارخانه های فرعی در `Live View` میز [\#7792](https://github.com/ClickHouse/ClickHouse/pull/7792) ([vzakaznikov](https://github.com/vzakaznikov)) -- بررسی برای استفاده از `Date` یا `DateTime` ستون از `TTL` عبارات حذف شد. [\#7920](https://github.com/ClickHouse/ClickHouse/pull/7920) ([ولادیمیر چبوتراف](https://github.com/excitoon)) -- اطلاعات در مورد دیسک به اضافه شد `system.detached_parts` جدول [\#7833](https://github.com/ClickHouse/ClickHouse/pull/7833) ([ولادیمیر چبوتراف](https://github.com/excitoon)) -- در حال حاضر تنظیمات `max_(table|partition)_size_to_drop` را می توان بدون راه اندازی مجدد تغییر کرده است. [\#7779](https://github.com/ClickHouse/ClickHouse/pull/7779) ([گریگوری پرواکوف](https://github.com/GrigoryPervakov)) -- قابلیت استفاده کمی بهتر از پیام های خطا. از کاربر بخواهید که خطوط زیر را حذف کند `Stack trace:`. [\#7897](https://github.com/ClickHouse/ClickHouse/pull/7897) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- خواندن بهتر پیام ها از `Kafka` موتور در فرمت های مختلف پس از [\#7935](https://github.com/ClickHouse/ClickHouse/issues/7935). [\#8035](https://github.com/ClickHouse/ClickHouse/pull/8035) ([ایوان](https://github.com/abyss7)) -- سازگاری بهتر با مشتریان خروجی زیر که پشتیبانی نمی کند `sha2_password` auth پلاگین. [\#8036](https://github.com/ClickHouse/ClickHouse/pull/8036) ([یوری baranov](https://github.com/yurriy)) -- پشتیبانی از انواع ستون بیشتر در خروجی زیر سرور سازگاری. [\#7975](https://github.com/ClickHouse/ClickHouse/pull/7975) ([یوری baranov](https://github.com/yurriy)) -- پیاده سازی `ORDER BY` بهینه سازی برای `Merge`, `Buffer` و `Materilized View` ذخیره سازی با زمینه `MergeTree` میز [\#8130](https://github.com/ClickHouse/ClickHouse/pull/8130) ([انتون پوپوف](https://github.com/CurtizJ)) -- در حال حاضر ما همیشه اجرای پسوند استفاده از `getrandom` برای سازگاری بهتر با دانه های قدیمی (\<3.17). [\#7940](https://github.com/ClickHouse/ClickHouse/pull/7940) ([ایموس پرنده](https://github.com/amosbird)) -- بررسی بهتر برای مقصد معتبر در یک قانون حرکت تغییر جنسیت. [\#8410](https://github.com/ClickHouse/ClickHouse/pull/8410) ([ولادیمیر چبوتراف](https://github.com/excitoon)) -- چک بهتر برای دسته درج شکسته برای `Distributed` موتور جدول. [\#7933](https://github.com/ClickHouse/ClickHouse/pull/7933) ([ازات خوژین](https://github.com/azat)) -- اضافه کردن ستون با مجموعه ای از نام قطعات که جهش باید در اینده پردازش به `system.mutations` جدول [\#8179](https://github.com/ClickHouse/ClickHouse/pull/8179) ([الساپین](https://github.com/alesapin)) -- موازی با مرتب کردن بر اساس ادغام بهینه سازی برای پردازنده. [\#8552](https://github.com/ClickHouse/ClickHouse/pull/8552) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) -- تنظیمات `mark_cache_min_lifetime` در حال حاضر منسوخ و هیچ کاری نمی کند. در نسخه های قبلی, علامت گذاری به عنوان کش می تواند در حافظه بزرگتر از رشد `mark_cache_size` به جای داده ها در `mark_cache_min_lifetime` چند ثانیه که منجر به سردرگمی و استفاده از حافظه بالاتر از حد انتظار است که به خصوص بد در حافظه محدود سیستم. اگر شما تخریب عملکرد پس از نصب این نسخه را ببینید, شما باید افزایش `mark_cache_size`. [\#8484](https://github.com/ClickHouse/ClickHouse/pull/8484) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- طرز تهیه برای استفاده `tid` همه جا این برای مورد نیاز است [\#7477](https://github.com/ClickHouse/ClickHouse/issues/7477). [\#8276](https://github.com/ClickHouse/ClickHouse/pull/8276) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - -#### بهبود عملکرد {#performance-improvement-1} - -- بهینه سازی عملکرد در خط لوله پردازنده. [\#7988](https://github.com/ClickHouse/ClickHouse/pull/7988) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) -- به روز رسانی غیر مسدود کردن کلید منقضی شده در لغت نامه کش (با اجازه خواندن قدیمی). [\#8303](https://github.com/ClickHouse/ClickHouse/pull/8303) ([نیکیتا میخایلو](https://github.com/nikitamikhaylov)) -- کامپایل تاتر بدون `-fno-omit-frame-pointer` در سطح جهانی به فراغت یک ثبت نام بیشتر. [\#8097](https://github.com/ClickHouse/ClickHouse/pull/8097) ([ایموس پرنده](https://github.com/amosbird)) -- افزایش سرعت `greatCircleDistance` عملکرد و تست عملکرد را اضافه کنید. [\#7307](https://github.com/ClickHouse/ClickHouse/pull/7307) ([اولگا خوستیکوا](https://github.com/stavrolia)) -- بهبود عملکرد عملکرد `roundDown`. [\#8465](https://github.com/ClickHouse/ClickHouse/pull/8465) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- بهبود عملکرد `max`, `min`, `argMin`, `argMax` برای `DateTime64` نوع داده. [\#8199](https://github.com/ClickHouse/ClickHouse/pull/8199) ([واسیلی نمکو](https://github.com/Enmk)) -- بهبود عملکرد مرتب سازی بدون محدودیت و یا با حد بزرگ و مرتب سازی خارجی. [\#8545](https://github.com/ClickHouse/ClickHouse/pull/8545) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- بهبود عملکرد قالب بندی اعداد ممیز شناور تا 6 بار. [\#8542](https://github.com/ClickHouse/ClickHouse/pull/8542) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- بهبود عملکرد `modulo` تابع. [\#7750](https://github.com/ClickHouse/ClickHouse/pull/7750) ([ایموس پرنده](https://github.com/amosbird)) -- بهینهسازی شده `ORDER BY` و ادغام با کلید ستون. [\#8335](https://github.com/ClickHouse/ClickHouse/pull/8335) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- اجرای بهتر برای `arrayReduce`, `-Array` و `-State` ترکیب کننده ها [\#7710](https://github.com/ClickHouse/ClickHouse/pull/7710) ([ایموس پرنده](https://github.com/amosbird)) -- حالا `PREWHERE` باید بهینه سازی شود حداقل به عنوان موثر به عنوان `WHERE`. [\#7769](https://github.com/ClickHouse/ClickHouse/pull/7769) ([ایموس پرنده](https://github.com/amosbird)) -- بهبود راه `round` و `roundBankers` دست زدن به اعداد منفی. [\#8229](https://github.com/ClickHouse/ClickHouse/pull/8229) ([هکز](https://github.com/hczhcz)) -- رمزگشایی بهبود عملکرد `DoubleDelta` و `Gorilla` کدک های تقریبا 30-40%. این رفع [\#7082](https://github.com/ClickHouse/ClickHouse/issues/7082). [\#8019](https://github.com/ClickHouse/ClickHouse/pull/8019) ([واسیلی نمکو](https://github.com/Enmk)) -- بهبود عملکرد `base64` توابع مرتبط. [\#8444](https://github.com/ClickHouse/ClickHouse/pull/8444) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- اضافه شدن یک تابع `geoDistance`. این شبیه به `greatCircleDistance` اما با استفاده از تقریب به دندان-84 مدل بیضی. عملکرد هر دو توابع در نزدیکی یکسان هستند. [\#8086](https://github.com/ClickHouse/ClickHouse/pull/8086) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- سریع تر `min` و `max` توابع تجمع برای `Decimal` نوع داده. [\#8144](https://github.com/ClickHouse/ClickHouse/pull/8144) ([زویکوف](https://github.com/4ertus2)) -- Vectorize پردازش `arrayReduce`. [\#7608](https://github.com/ClickHouse/ClickHouse/pull/7608) ([ایموس پرنده](https://github.com/amosbird)) -- `if` زنجیر در حال حاضر به عنوان بهینه سازی شده `multiIf`. [\#8355](https://github.com/ClickHouse/ClickHouse/pull/8355) ([کمالف روسلان](https://github.com/kamalov-ruslan)) -- رفع رگرسیون عملکرد `Kafka` موتور جدول معرفی شده در 19.15. این رفع [\#7261](https://github.com/ClickHouse/ClickHouse/issues/7261). [\#7935](https://github.com/ClickHouse/ClickHouse/pull/7935) ([فیلیمونف](https://github.com/filimonov)) -- حذف شد “pie” تولید کد که `gcc` از بسته های دبیان گاهی اوقات به طور پیش فرض به ارمغان بیاورد. [\#8483](https://github.com/ClickHouse/ClickHouse/pull/8483) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- موازی تجزیه فرمت های داده [\#6553](https://github.com/ClickHouse/ClickHouse/pull/6553) ([نیکیتا میخایلو](https://github.com/nikitamikhaylov)) -- به کار انداختن تجزیه کننده بهینه `Values` با عبارات به طور پیش فرض (`input_format_values_deduce_templates_of_expressions=1`). [\#8231](https://github.com/ClickHouse/ClickHouse/pull/8231) ([تاولوبیکس](https://github.com/tavplubix)) - -#### ساخت/تست / بهبود بسته بندی {#buildtestingpackaging-improvement-2} - -- ساخت رفع برای `ARM` و در حالت حداقل. [\#8304](https://github.com/ClickHouse/ClickHouse/pull/8304) ([پرولر](https://github.com/proller)) -- اضافه کردن پوشش خیط و پیت کردن فایل برای `clickhouse-server` وقتی بیماری مقاربتی:: اختلال نامیده می شود نیست. همچنین کمی بهبود یافته ورود به سیستم در تست بدون تابعیت با پوشش. [\#8267](https://github.com/ClickHouse/ClickHouse/pull/8267) ([الساپین](https://github.com/alesapin)) -- به روز رسانی llvm کتابخانه در contrib. اجتناب از استفاده از llvm از سیستم عامل بسته است. [\#8258](https://github.com/ClickHouse/ClickHouse/pull/8258) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- را همراه `curl` ساخت کاملا ساکت. [\#8232](https://github.com/ClickHouse/ClickHouse/pull/8232) [\#8203](https://github.com/ClickHouse/ClickHouse/pull/8203) ([پاول کووالنکو](https://github.com/Jokser)) -- رفع برخی از `MemorySanitizer` اخطار. [\#8235](https://github.com/ClickHouse/ClickHouse/pull/8235) ([الکساندر کوزمنکوف](https://github.com/akuzm)) -- استفاده `add_warning` و `no_warning` ماکروها در `CMakeLists.txt`. [\#8604](https://github.com/ClickHouse/ClickHouse/pull/8604) ([ایوان](https://github.com/abyss7)) -- اضافه کردن پشتیبانی از minio s3 سازگار شی (https://min.io/) برای ادغام بهتر شد. [\#7863](https://github.com/ClickHouse/ClickHouse/pull/7863) [\#7875](https://github.com/ClickHouse/ClickHouse/pull/7875) ([پاول کووالنکو](https://github.com/Jokser)) -- وارد شده `libc` سرصفحهها به کنترب. این اجازه می دهد تا به ایجاد بیشتر سازگار در سراسر سیستم های مختلف (فقط برای `x86_64-linux-gnu`). [\#5773](https://github.com/ClickHouse/ClickHouse/pull/5773) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- حذف `-fPIC` از برخی از کتابخانه ها. [\#8464](https://github.com/ClickHouse/ClickHouse/pull/8464) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- تمیز `CMakeLists.txt` برای حلقه. ببینید https://github.com/ClickHouse/ClickHouse/pull/8011\#issuecomment-569478910 [\#8459](https://github.com/ClickHouse/ClickHouse/pull/8459) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- هشدار خاموش در `CapNProto` کتابخونه. [\#8220](https://github.com/ClickHouse/ClickHouse/pull/8220) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- اضافه کردن تست عملکرد برای رشته کوتاه بهینه سازی جداول هش. [\#7679](https://github.com/ClickHouse/ClickHouse/pull/7679) ([ایموس پرنده](https://github.com/amosbird)) -- در حال حاضر خانه کلیک بر روی ساخت `AArch64` حتی اگر `MADV_FREE` در دسترس نیست. این رفع [\#8027](https://github.com/ClickHouse/ClickHouse/issues/8027). [\#8243](https://github.com/ClickHouse/ClickHouse/pull/8243) ([ایموس پرنده](https://github.com/amosbird)) -- بهروزرسانی `zlib-ng` برای رفع مشکلات ضدعفونی کننده حافظه. [\#7182](https://github.com/ClickHouse/ClickHouse/pull/7182) [\#8206](https://github.com/ClickHouse/ClickHouse/pull/8206) ([الکساندر کوزمنکوف](https://github.com/akuzm)) -- فعال کردن کتابخانه خروجی زیر داخلی در سیستم غیر لینوکس, به دلیل استفاده از بسته های سیستم عامل بسیار شکننده است و معمولا در همه کار نمی کند. این رفع [\#5765](https://github.com/ClickHouse/ClickHouse/issues/5765). [\#8426](https://github.com/ClickHouse/ClickHouse/pull/8426) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- ساخت ثابت در برخی از سیستم های پس از فعال کردن `libc++`. این جانشین [\#8374](https://github.com/ClickHouse/ClickHouse/issues/8374). [\#8380](https://github.com/ClickHouse/ClickHouse/pull/8380) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- ساخت `Field` مواد و روش ها نوع بیشتر-امن برای پیدا کردن خطاهای بیشتر. [\#7386](https://github.com/ClickHouse/ClickHouse/pull/7386) [\#8209](https://github.com/ClickHouse/ClickHouse/pull/8209) ([الکساندر کوزمنکوف](https://github.com/akuzm)) -- اضافه شده فایل های از دست رفته به `libc-headers` submodule. [\#8507](https://github.com/ClickHouse/ClickHouse/pull/8507) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- رفع اشتباه `JSON` نقل قول در خروجی تست عملکرد. [\#8497](https://github.com/ClickHouse/ClickHouse/pull/8497) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) -- در حال حاضر ردیابی پشته برای نمایش داده می شود `std::exception` و `Poco::Exception`. در نسخه های قبلی فقط برای `DB::Exception`. این باعث بهبود تشخیص می شود. [\#8501](https://github.com/ClickHouse/ClickHouse/pull/8501) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- بارینگ `clock_gettime` و `clock_nanosleep` برای نسخه های چرب زبان تازه. [\#8054](https://github.com/ClickHouse/ClickHouse/pull/8054) ([ایموس پرنده](https://github.com/amosbird)) -- فعالسازی `part_log` در مثال پیکربندی برای توسعه دهندگان. [\#8609](https://github.com/ClickHouse/ClickHouse/pull/8609) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- رفع ماهیت کالاهای کابل از بازنگری در `01036_no_superfluous_dict_reload_on_create_database*`. [\#8111](https://github.com/ClickHouse/ClickHouse/pull/8111) ([ازات خوژین](https://github.com/azat)) -- تست عملکرد کدک ثابت. [\#8615](https://github.com/ClickHouse/ClickHouse/pull/8615) ([واسیلی نمکو](https://github.com/Enmk)) -- اضافه کردن اسکریپت نصب برای `.tgz` ساخت و اسناد و مدارک برای خود. [\#8612](https://github.com/ClickHouse/ClickHouse/pull/8612) [\#8591](https://github.com/ClickHouse/ClickHouse/pull/8591) ([الساپین](https://github.com/alesapin)) -- حذف قدیمی `ZSTD` تست (این در سال 2016 ایجاد شد به تکثیر اشکال که قبل از 1.0 نسخه از زد تا به حال). این رفع [\#8618](https://github.com/ClickHouse/ClickHouse/issues/8618). [\#8619](https://github.com/ClickHouse/ClickHouse/pull/8619) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- ساخت ثابت در سیستم عامل مک کاتالینا. [\#8600](https://github.com/ClickHouse/ClickHouse/pull/8600) ([مو](https://github.com/meob)) -- افزایش تعداد ردیف در تست عملکرد کدک برای ایجاد نتایج قابل توجه است. [\#8574](https://github.com/ClickHouse/ClickHouse/pull/8574) ([واسیلی نمکو](https://github.com/Enmk)) -- در اشکال زدایی ایجاد, درمان `LOGICAL_ERROR` استثنا به عنوان شکست ادعای, به طوری که راحت تر به اطلاع. [\#8475](https://github.com/ClickHouse/ClickHouse/pull/8475) ([الکساندر کوزمنکوف](https://github.com/akuzm)) -- تست عملکرد مربوط به فرمت را قطعی تر کنید. [\#8477](https://github.com/ClickHouse/ClickHouse/pull/8477) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- بهروزرسانی `lz4` برای رفع یک شکست حفظ کننده. [\#8181](https://github.com/ClickHouse/ClickHouse/pull/8181) ([الکساندر کوزمنکوف](https://github.com/akuzm)) -- سرکوب یک یادداشت شناخته شده مثبت کاذب در دست زدن به استثنا. [\#8182](https://github.com/ClickHouse/ClickHouse/pull/8182) ([الکساندر کوزمنکوف](https://github.com/akuzm)) -- بهروزرسانی `gcc` و `g++` به نسخه 9 در `build/docker/build.sh` [\#7766](https://github.com/ClickHouse/ClickHouse/pull/7766) ([تلیتسکی](https://github.com/tlightsky)) -- اضافه کردن مورد تست عملکرد برای تست که `PREWHERE` بدتر از `WHERE`. [\#7768](https://github.com/ClickHouse/ClickHouse/pull/7768) ([ایموس پرنده](https://github.com/amosbird)) -- پیشرفت به سمت رفع یک تست فلکی. [\#8621](https://github.com/ClickHouse/ClickHouse/pull/8621) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- اجتناب از گزارش یادداشت برای داده ها از `libunwind`. [\#8539](https://github.com/ClickHouse/ClickHouse/pull/8539) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- به روز شده `libc++` به جدیدترین نسخه. [\#8324](https://github.com/ClickHouse/ClickHouse/pull/8324) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- ساخت کتابخانه ایکو از منابع. این رفع [\#6460](https://github.com/ClickHouse/ClickHouse/issues/6460). [\#8219](https://github.com/ClickHouse/ClickHouse/pull/8219) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- تغییر از `libressl` به `openssl`. ClickHouse باید پشتیبانی از TLS 1.3 و SNI پس از این تغییر. این رفع [\#8171](https://github.com/ClickHouse/ClickHouse/issues/8171). [\#8218](https://github.com/ClickHouse/ClickHouse/pull/8218) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- گزارش ثابت اوبسان هنگام استفاده `chacha20_poly1305` از اس اس ال (اتفاق می افتد در اتصال به https://yandex.ru/). [\#8214](https://github.com/ClickHouse/ClickHouse/pull/8214) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- حالت ثابت فایل رمز عبور پیش فرض برای `.deb` توزیع لینوکس. [\#8075](https://github.com/ClickHouse/ClickHouse/pull/8075) ([پرولر](https://github.com/proller)) -- بیان بهبود یافته برای گرفتن `clickhouse-server` PID در `clickhouse-test`. [\#8063](https://github.com/ClickHouse/ClickHouse/pull/8063) ([الکساندر کازاکوف](https://github.com/Akazz)) -- به روز شده در تماس/ماده چسبنده و لزج به نسخه برداری1.10.0. [\#8587](https://github.com/ClickHouse/ClickHouse/pull/8587) ([الکساندر بورمک](https://github.com/Alex-Burmak)) -- گزارش سه گانه ثابت در `base64` کتابخونه. همچنین این کتابخانه به جدیدترین نسخه به روز, اما مهم نیست. این رفع [\#8397](https://github.com/ClickHouse/ClickHouse/issues/8397). [\#8403](https://github.com/ClickHouse/ClickHouse/pull/8403) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- ثابت `00600_replace_running_query` برای پردازنده. [\#8272](https://github.com/ClickHouse/ClickHouse/pull/8272) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) -- حذف پشتیبانی برای `tcmalloc` برای ساخت `CMakeLists.txt` ساده تر [\#8310](https://github.com/ClickHouse/ClickHouse/pull/8310) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- انتشار شورای همکاری خلیج فارس ایجاد در حال حاضر استفاده کنید `libc++` به جای `libstdc++`. اخیرا `libc++` تنها با صدای جرنگ جرنگ مورد استفاده قرار گرفت. این ثبات تنظیمات ساخت و قابلیت حمل را بهبود بخشد. [\#8311](https://github.com/ClickHouse/ClickHouse/pull/8311) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- فعال کردن کتابخانه فناوری اطلاعات و ارتباطات برای ساخت با حفظ. [\#8222](https://github.com/ClickHouse/ClickHouse/pull/8222) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- سرکوب هشدارها از `CapNProto` کتابخونه. [\#8224](https://github.com/ClickHouse/ClickHouse/pull/8224) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- موارد خاص حذف کد برای `tcmalloc`, زیرا دیگر پشتیبانی. [\#8225](https://github.com/ClickHouse/ClickHouse/pull/8225) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- کشتن سرور برازنده اجازه می دهد تا برای ذخیره گزارش پوشش. این رفع گزارش پوشش ناقص ما به تازگی دیدن شده است. [\#8142](https://github.com/ClickHouse/ClickHouse/pull/8142) ([الساپین](https://github.com/alesapin)) -- تست عملکرد برای تمام کدک در برابر `Float64` و `UInt64` ارزشهای خبری عبارتند از: [\#8349](https://github.com/ClickHouse/ClickHouse/pull/8349) ([واسیلی نمکو](https://github.com/Enmk)) -- `termcap` بسیار توصیه شده و منجر به مشکلات مختلف (f.g. از دست رفته “up” کلاه و انعکاس `^J` به جای چند خط). به نفع `terminfo` یا همراه `ncurses`. [\#7737](https://github.com/ClickHouse/ClickHouse/pull/7737) ([ایموس پرنده](https://github.com/amosbird)) -- ثابت `test_storage_s3` تست ادغام. [\#7734](https://github.com/ClickHouse/ClickHouse/pull/7734) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) -- پشتیبانی `StorageFile(, null)` برای قرار دادن بلوک به فایل فرمت داده شده بدون در واقع به دیسک ارسال. این برای تست عملکرد مورد نیاز است. [\#8455](https://github.com/ClickHouse/ClickHouse/pull/8455) ([ایموس پرنده](https://github.com/amosbird)) -- استدلال اضافه شده `--print-time` به تست های عملکردی که زمان اجرای هر تست را چاپ می کند. [\#8001](https://github.com/ClickHouse/ClickHouse/pull/8001) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) -- اضافه ادعا به `KeyCondition` در حالی که ارزیابی مالیات بر ارزش افزوده. این هشدار را از شورای همکاری خلیج فارس-9 حل کنند. [\#8279](https://github.com/ClickHouse/ClickHouse/pull/8279) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- تخلیه گزینه های چوب کوره در سی ایجاد. [\#8273](https://github.com/ClickHouse/ClickHouse/pull/8273) ([الکساندر کوزمنکوف](https://github.com/akuzm)) -- اطلاعات اشکال زدایی برای برخی از کتابخانه های چربی تولید نمی کند. [\#8271](https://github.com/ClickHouse/ClickHouse/pull/8271) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- ساخت `log_to_console.xml` همیشه به خ ورود, صرف نظر از تعاملی است یا نه. [\#8395](https://github.com/ClickHouse/ClickHouse/pull/8395) ([الکساندر کوزمنکوف](https://github.com/akuzm)) -- حذف برخی از ویژگی های استفاده نشده از `clickhouse-performance-test` ابزار. [\#8555](https://github.com/ClickHouse/ClickHouse/pull/8555) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- در حال حاضر ما نیز برای جستجو `lld-X` با متناظر `clang-X` نسخه. [\#8092](https://github.com/ClickHouse/ClickHouse/pull/8092) ([الساپین](https://github.com/alesapin)) -- پارکت ساخت بهبود. [\#8421](https://github.com/ClickHouse/ClickHouse/pull/8421) ([مکسولان](https://github.com/maxulan)) -- هشدارهای شورای همکاری خلیج فارس بیشتر [\#8221](https://github.com/ClickHouse/ClickHouse/pull/8221) ([کروزرکریگ](https://github.com/kreuzerkrieg)) -- بسته بندی برای قوس لینوکس در حال حاضر اجازه می دهد تا برای اجرای سرور کلیک, و نه تنها مشتری. [\#8534](https://github.com/ClickHouse/ClickHouse/pull/8534) ([ولادیمیر چبوتراف](https://github.com/excitoon)) -- ثابت تست با پردازنده. رفع عملکرد کوچک. [\#7672](https://github.com/ClickHouse/ClickHouse/pull/7672) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) -- اطلاعات دقیق [\#8256](https://github.com/ClickHouse/ClickHouse/pull/8256) ([هشدار داده می شود](https://github.com/matwey)) -- در تهیه تعویض به سی++20 به عنوان یک جشن سال نو. “May the C++ force be with ClickHouse.” [\#8447](https://github.com/ClickHouse/ClickHouse/pull/8447) ([ایموس پرنده](https://github.com/amosbird)) - -#### ویژگی تجربی {#experimental-feature-1} - -- اضافه شدن تنظیمات تجربی `min_bytes_to_use_mmap_io`. این اجازه می دهد به خواندن فایل های بزرگ بدون کپی کردن داده ها از هسته به فضای کاربری. تنظیمات به طور پیش فرض غیر فعال. توصیه می شود در حدود 64 مگابایت است, چون اماسپ / مون مپ کند است. [\#8520](https://github.com/ClickHouse/ClickHouse/pull/8520) ([الکسی میلویدو](https://github.com/alexey-milovidov)) -- سهمیه به عنوان بخشی از سیستم کنترل دسترسی دوباره. جدول جدید اضافه شده است `system.quotas` توابع جدید `currentQuota`, `currentQuotaKey` نحو گذاشتن جدید `CREATE QUOTA`, `ALTER QUOTA`, `DROP QUOTA`, `SHOW QUOTA`. [\#7257](https://github.com/ClickHouse/ClickHouse/pull/7257) ([ویتالی بارانو](https://github.com/vitlibar)) -- اجازه پرش تنظیمات ناشناخته با هشدار به جای پرتاب استثنا. [\#7653](https://github.com/ClickHouse/ClickHouse/pull/7653) ([ویتالی بارانو](https://github.com/vitlibar)) -- سیاست های ردیف به عنوان بخشی از سیستم کنترل دسترسی دوباره. جدول جدید اضافه شده است `system.row_policies` تابع جدید `currentRowPolicies()` نحو گذاشتن جدید `CREATE POLICY`, `ALTER POLICY`, `DROP POLICY`, `SHOW CREATE POLICY`, `SHOW POLICIES`. [\#7808](https://github.com/ClickHouse/ClickHouse/pull/7808) ([ویتالی بارانو](https://github.com/vitlibar)) - -#### تعمیر امنیتی {#security-fix} - -- ثابت امکان خواندن ساختار دایرکتوری در جداول با `File` موتور جدول. این رفع [\#8536](https://github.com/ClickHouse/ClickHouse/issues/8536). [\#8537](https://github.com/ClickHouse/ClickHouse/pull/8537) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - -## [تغییرات برای 2019](https://github.com/ClickHouse/ClickHouse/blob/master/docs/en/changelog/2019.md) {#changelog-for-2019} +{% include "content/changelog.md" %} diff --git a/docs/fa/whats-new/index.md b/docs/fa/whats-new/index.md index ac27b70b8bd..2f4484f2bda 100644 --- a/docs/fa/whats-new/index.md +++ b/docs/fa/whats-new/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_folder_title: What's New +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "\u0686\u0647 \u062C\u062F\u06CC\u062F" toc_priority: 72 --- diff --git a/docs/fa/whats-new/roadmap.md b/docs/fa/whats-new/roadmap.md index f51b57e3a2a..e0e5f93c221 100644 --- a/docs/fa/whats-new/roadmap.md +++ b/docs/fa/whats-new/roadmap.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 74 toc_title: "\u0646\u0642\u0634\u0647 \u0631\u0627\u0647" --- diff --git a/docs/fa/whats-new/security-changelog.md b/docs/fa/whats-new/security-changelog.md index 79eb2230b03..2b0a0bcf846 100644 --- a/docs/fa/whats-new/security-changelog.md +++ b/docs/fa/whats-new/security-changelog.md @@ -1,12 +1,12 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 76 toc_title: "\u062A\u063A\u06CC\u06CC\u0631\u0627\u062A \u0627\u0645\u0646\u06CC\u062A\ \u06CC" --- -## ثابت در clickhouse انتشار 19.14.3.3, 2019-09-10 {#fixed-in-clickhouse-release-19-14-3-3-2019-09-10} +## ثابت در ClickHouse انتشار 19.14.3.3, 2019-09-10 {#fixed-in-clickhouse-release-19-14-3-3-2019-09-10} ### CVE-2019-15024 {#cve-2019-15024} @@ -26,7 +26,7 @@ toc_title: "\u062A\u063A\u06CC\u06CC\u0631\u0627\u062A \u0627\u0645\u0646\u06CC\ اعتبار: الدار زیتوف تیم امنیت اطلاعات یاندکس -## ثابت در clickhouse انتشار 19.13.6.1, 2019-09-20 {#fixed-in-clickhouse-release-19-13-6-1-2019-09-20} +## ثابت در ClickHouse انتشار 19.13.6.1, 2019-09-20 {#fixed-in-clickhouse-release-19-13-6-1-2019-09-20} ### CVE-2019-18657 {#cve-2019-18657} @@ -34,7 +34,7 @@ toc_title: "\u062A\u063A\u06CC\u06CC\u0631\u0627\u062A \u0627\u0645\u0646\u06CC\ اعتبار: [نیکیتا تیکومیرو](https://github.com/NSTikhomirov) -## ثابت در clickhouse انتشار 18.12.13, 2018-09-10 {#fixed-in-clickhouse-release-18-12-13-2018-09-10} +## ثابت در ClickHouse انتشار 18.12.13, 2018-09-10 {#fixed-in-clickhouse-release-18-12-13-2018-09-10} ### CVE-2018-14672 {#cve-2018-14672} @@ -42,7 +42,7 @@ toc_title: "\u062A\u063A\u06CC\u06CC\u0631\u0627\u062A \u0627\u0645\u0646\u06CC\ اعتبار: اندری کراسیچکوف از تیم امنیت اطلاعات یاندکس -## ثابت در clickhouse انتشار 18.10.3, 2018-08-13 {#fixed-in-clickhouse-release-18-10-3-2018-08-13} +## ثابت در ClickHouse انتشار 18.10.3, 2018-08-13 {#fixed-in-clickhouse-release-18-10-3-2018-08-13} ### CVE-2018-14671 {#cve-2018-14671} @@ -50,7 +50,7 @@ unixODBC اجازه بارگذاری دلخواه اشیاء مشترک از ف اعتبار: اندری کراسیچکوف و اوگنی سیدوروف از تیم امنیت اطلاعات یاندکس -## ثابت در clickhouse انتشار 1.1.54388, 2018-06-28 {#fixed-in-clickhouse-release-1-1-54388-2018-06-28} +## ثابت در ClickHouse انتشار 1.1.54388, 2018-06-28 {#fixed-in-clickhouse-release-1-1-54388-2018-06-28} ### CVE-2018-14668 {#cve-2018-14668} @@ -58,7 +58,7 @@ unixODBC اجازه بارگذاری دلخواه اشیاء مشترک از ف اعتبار: اندری کراسیچکوف از تیم امنیت اطلاعات یاندکس -## ثابت در clickhouse انتشار 1.1.54390, 2018-07-06 {#fixed-in-clickhouse-release-1-1-54390-2018-07-06} +## ثابت در ClickHouse انتشار 1.1.54390, 2018-07-06 {#fixed-in-clickhouse-release-1-1-54390-2018-07-06} ### CVE-2018-14669 {#cve-2018-14669} @@ -66,12 +66,12 @@ unixODBC اجازه بارگذاری دلخواه اشیاء مشترک از ف اعتبار: اندری کراسیچکوف و اوگنی سیدوروف از تیم امنیت اطلاعات یاندکس -## ثابت در clickhouse انتشار 1.1.54131, 2017-01-10 {#fixed-in-clickhouse-release-1-1-54131-2017-01-10} +## ثابت در ClickHouse انتشار 1.1.54131, 2017-01-10 {#fixed-in-clickhouse-release-1-1-54131-2017-01-10} ### CVE-2018-14670 {#cve-2018-14670} پیکربندی نادرست در بسته دب می تواند به استفاده غیر مجاز از پایگاه داده منجر شود. -اعتبار: انگلستان national cyber security centre (ncsc) +اعتبار: انگلستان National Cyber Security Centre (NCSC) {## [مقاله اصلی](https://clickhouse.tech/docs/en/security_changelog/) ##} diff --git a/docs/fr/commercial/cloud.md b/docs/fr/commercial/cloud.md index ea97c969b02..13013065bfe 100644 --- a/docs/fr/commercial/cloud.md +++ b/docs/fr/commercial/cloud.md @@ -1,12 +1,14 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_priority: 1 +toc_title: Nuage --- # Fournisseurs De Services Cloud ClickHouse {#clickhouse-cloud-service-providers} !!! info "Info" - Si vous avez lancé un cloud public avec un service clickhouse géré, n’hésitez pas à [ouvrir une demande d’extraction](https://github.com/ClickHouse/ClickHouse/edit/master/docs/en/commercial/cloud.md) ajouter à la liste suivante. + Si vous avez lancé un cloud public avec un service clickhouse géré, n'hésitez pas à [ouvrir une demande d'extraction](https://github.com/ClickHouse/ClickHouse/edit/master/docs/en/commercial/cloud.md) ajouter à la liste suivante. ## Yandex Cloud {#yandex-cloud} @@ -15,7 +17,7 @@ machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 - Service ZooKeeper entièrement géré pour [Réplication de ClickHouse](../engines/table-engines/mergetree-family/replication.md) - Choix multiples de type de stockage - Répliques dans différentes zones de disponibilité -- Le chiffrement et l’isolement +- Le chiffrement et l'isolement - Automatisation de la maintenance {## [Article Original](https://clickhouse.tech/docs/en/commercial/cloud/) ##} diff --git a/docs/fr/commercial/index.md b/docs/fr/commercial/index.md index 296ea571a59..388f9a47fc8 100644 --- a/docs/fr/commercial/index.md +++ b/docs/fr/commercial/index.md @@ -1,8 +1,9 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_folder_title: Commercial toc_priority: 70 +toc_title: Commercial --- diff --git a/docs/fr/commercial/support.md b/docs/fr/commercial/support.md deleted file mode 120000 index 1eb20ccf36a..00000000000 --- a/docs/fr/commercial/support.md +++ /dev/null @@ -1 +0,0 @@ -../../en/commercial/support.md \ No newline at end of file diff --git a/docs/fr/commercial/support.md b/docs/fr/commercial/support.md new file mode 100644 index 00000000000..f5ce091272f --- /dev/null +++ b/docs/fr/commercial/support.md @@ -0,0 +1,23 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_priority: 3 +toc_title: Soutien +--- + +# Fournisseurs De Services De Soutien Commercial ClickHouse {#clickhouse-commercial-support-service-providers} + +!!! info "Info" + Si vous avez lancé un service de support commercial ClickHouse, n'hésitez pas à [ouvrir une demande d'extraction](https://github.com/ClickHouse/ClickHouse/edit/master/docs/en/commercial/support.md) ajouter à la liste suivante. + +## Altinity {#altinity} + +Altinity offre le support et les services de ClickHouse d'entreprise depuis 2017. Les clients d'Altinity vont des entreprises Fortune 100 aux startups. Visiter [www.altinity.com](https://www.altinity.com/) pour plus d'informations. + +## Mafiree {#mafiree} + +[Description du Service](http://mafiree.com/clickhouse-analytics-services.php) + +## MinervaDB {#minervadb} + +[Description du Service](https://minervadb.com/index.php/clickhouse-consulting-and-support-by-minervadb/) diff --git a/docs/fr/development/architecture.md b/docs/fr/development/architecture.md index 4d6630403fa..97de3c8d1ce 100644 --- a/docs/fr/development/architecture.md +++ b/docs/fr/development/architecture.md @@ -1,203 +1,203 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 62 toc_title: Vue d'ensemble de L'Architecture ClickHouse --- -# Vue d’ensemble De L’Architecture ClickHouse {#overview-of-clickhouse-architecture} +# Vue d'ensemble de L'Architecture ClickHouse {#overview-of-clickhouse-architecture} -ClickHouse est un véritable SGBD orienté colonne. Les données sont stockées par colonnes et lors de l’exécution de tableaux (vecteurs ou morceaux de colonnes). Dans la mesure du possible, les opérations sont distribuées sur des tableaux, plutôt que sur des valeurs individuelles. Il est appelé “vectorized query execution,” et cela aide à réduire le coût du traitement des données réel. +ClickHouse est un véritable SGBD orienté colonne. Les données sont stockées par colonnes et lors de l'exécution de tableaux (vecteurs ou morceaux de colonnes). Dans la mesure du possible, les opérations sont distribuées sur des tableaux, plutôt que sur des valeurs individuelles. Il est appelé “vectorized query execution,” et cela aide à réduire le coût du traitement des données réel. -> Cette idée n’est pas nouvelle. Il remonte à la `APL` langage de programmation et ses descendants: `A +`, `J`, `K`, et `Q`. La programmation de tableau est utilisée dans le traitement des données scientifiques. Cette idée n’est pas non plus nouvelle dans les bases de données relationnelles: par exemple, elle est utilisée dans le `Vectorwise` système. +> Cette idée n'est pas nouvelle. Il remonte à la `APL` langage de programmation et ses descendants: `A +`, `J`, `K`, et `Q`. La programmation de tableau est utilisée dans le traitement des données scientifiques. Cette idée n'est pas non plus nouvelle dans les bases de données relationnelles: par exemple, elle est utilisée dans le `Vectorwise` système. -Il existe deux approches différentes pour accélérer le traitement des requêtes: l’exécution vectorisée des requêtes et la génération de code d’exécution. Ce dernier supprime toute indirection et expédition dynamique. Aucune de ces approches est strictement meilleure que l’autre. La génération de code d’exécution peut être meilleure lorsqu’elle fusionne de nombreuses opérations, utilisant ainsi pleinement les unités D’exécution du processeur et le pipeline. L’exécution de requête vectorisée peut être moins pratique car elle implique des vecteurs temporaires qui doivent être écrits dans le cache et lus. Si les données temporaires ne rentre pas dans le cache L2, cela devient un problème. Mais l’exécution de requête vectorisée utilise plus facilement les capacités SIMD de la CPU. Un [document de recherche](http://15721.courses.cs.cmu.edu/spring2016/papers/p5-sompolski.pdf) écrit par nos amis montre qu’il est préférable de combiner les deux approches. ClickHouse utilise l’exécution de requête vectorisée et a un support initial limité pour la génération de code d’exécution. +Il existe deux approches différentes pour accélérer le traitement des requêtes: l'exécution vectorisée des requêtes et la génération de code d'exécution. Ce dernier supprime toute indirection et expédition dynamique. Aucune de ces approches est strictement meilleure que l'autre. La génération de code d'exécution peut être meilleure lorsqu'elle fusionne de nombreuses opérations, utilisant ainsi pleinement les unités D'exécution du processeur et le pipeline. L'exécution de requête vectorisée peut être moins pratique car elle implique des vecteurs temporaires qui doivent être écrits dans le cache et lus. Si les données temporaires ne rentre pas dans le cache L2, cela devient un problème. Mais l'exécution de requête vectorisée utilise plus facilement les capacités SIMD de la CPU. Un [document de recherche](http://15721.courses.cs.cmu.edu/spring2016/papers/p5-sompolski.pdf) écrit par nos amis montre qu'il est préférable de combiner les deux approches. ClickHouse utilise l'exécution de requête vectorisée et a un support initial limité pour la génération de code d'exécution. ## Colonne {#columns} -`IColumn` l’interface est utilisée pour représenter des colonnes en mémoire (en fait, des morceaux de colonnes). Cette interface fournit des méthodes d’aide pour la mise en œuvre de divers opérateurs relationnels. Presque toutes les opérations sont immuables: elles ne modifient pas la colonne d’origine, mais en créent une nouvelle modifiée. Par exemple, l’ `IColumn :: filter` méthode accepte un masque d’octet de filtre. Il est utilisé pour le `WHERE` et `HAVING` opérateurs relationnels. Exemples supplémentaires: `IColumn :: permute` méthode de soutien `ORDER BY`, le `IColumn :: cut` méthode de soutien `LIMIT`. +`IColumn` l'interface est utilisée pour représenter des colonnes en mémoire (en fait, des morceaux de colonnes). Cette interface fournit des méthodes d'aide pour la mise en œuvre de divers opérateurs relationnels. Presque toutes les opérations sont immuables: elles ne modifient pas la colonne d'origine, mais en créent une nouvelle modifiée. Par exemple, l' `IColumn :: filter` méthode accepte un masque d'octet de filtre. Il est utilisé pour le `WHERE` et `HAVING` opérateurs relationnels. Exemples supplémentaires: `IColumn :: permute` méthode de soutien `ORDER BY`, le `IColumn :: cut` méthode de soutien `LIMIT`. -Divers `IColumn` application (`ColumnUInt8`, `ColumnString` et ainsi de suite) sont responsables de la mémoire disposition de colonnes. La disposition de la mémoire est généralement un tableau contigu. Pour le type entier de colonnes, c’est juste un contiguë tableau, comme `std :: vector`. Pour `String` et `Array` colonnes, il s’agit de deux vecteurs: Un pour tous les éléments du tableau, placé de manière contiguë, et un second pour les décalages au début de chaque tableau. Il y a aussi `ColumnConst` cela stocke une seule valeur en mémoire, mais ressemble à une colonne. +Divers `IColumn` application (`ColumnUInt8`, `ColumnString` et ainsi de suite) sont responsables de la mémoire disposition de colonnes. La disposition de la mémoire est généralement un tableau contigu. Pour le type entier de colonnes, c'est juste un contiguë tableau, comme `std :: vector`. Pour `String` et `Array` colonnes, il s'agit de deux vecteurs: Un pour tous les éléments du tableau, placé de manière contiguë, et un second pour les décalages au début de chaque tableau. Il y a aussi `ColumnConst` cela stocke une seule valeur en mémoire, mais ressemble à une colonne. ## Champ {#field} -Néanmoins, il est possible de travailler avec des valeurs individuelles ainsi. Pour représenter une valeur individuelle, la `Field` est utilisée. `Field` est juste une union discriminée de `UInt64`, `Int64`, `Float64`, `String` et `Array`. `IColumn` a l’ `operator[]` méthode pour obtenir la n-ème valeur en tant que `Field` et la `insert` méthode pour ajouter un `Field` à la fin d’une colonne. Ces méthodes ne sont pas très efficaces, car ils nécessitent de traiter avec temporaire `Field` des objets représentant une valeur individuelle. Il existe des méthodes plus efficaces, telles que `insertFrom`, `insertRangeFrom` et ainsi de suite. +Néanmoins, il est possible de travailler avec des valeurs individuelles ainsi. Pour représenter une valeur individuelle, la `Field` est utilisée. `Field` est juste une union discriminée de `UInt64`, `Int64`, `Float64`, `String` et `Array`. `IColumn` a l' `operator[]` méthode pour obtenir la n-ème valeur en tant que `Field` et la `insert` méthode pour ajouter un `Field` à la fin d'une colonne. Ces méthodes ne sont pas très efficaces, car ils nécessitent de traiter avec temporaire `Field` des objets représentant une valeur individuelle. Il existe des méthodes plus efficaces, telles que `insertFrom`, `insertRangeFrom` et ainsi de suite. -`Field` ne pas avoir assez d’informations sur un type de données spécifique pour une table. Exemple, `UInt8`, `UInt16`, `UInt32`, et `UInt64` tous sont représentés comme `UInt64` dans un `Field`. +`Field` ne pas avoir assez d'informations sur un type de données spécifique pour une table. Exemple, `UInt8`, `UInt16`, `UInt32`, et `UInt64` tous sont représentés comme `UInt64` dans un `Field`. ## Abstractions Qui Fuient {#leaky-abstractions} -`IColumn` a des méthodes pour les transformations relationnelles communes des données, mais elles ne répondent pas à tous les besoins. Exemple, `ColumnUInt64` ne pas avoir une méthode pour calculer la somme des deux colonnes, et `ColumnString` n’a pas de méthode pour exécuter une recherche de sous-chaîne. Ces innombrables routines sont mises en œuvre en dehors de `IColumn`. +`IColumn` a des méthodes pour les transformations relationnelles communes des données, mais elles ne répondent pas à tous les besoins. Exemple, `ColumnUInt64` ne pas avoir une méthode pour calculer la somme des deux colonnes, et `ColumnString` n'a pas de méthode pour exécuter une recherche de sous-chaîne. Ces innombrables routines sont mises en œuvre en dehors de `IColumn`. -Diverses fonctions sur les colonnes peuvent être implémentées de manière générique et non efficace en utilisant `IColumn` méthodes pour extraire `Field` valeurs, ou d’une manière spécialisée en utilisant la connaissance de la disposition de la mémoire interne des données dans un `IColumn` application. Il est implémenté en lançant des fonctions à un `IColumn` tapez et traitez directement la représentation interne. Exemple, `ColumnUInt64` a l’ `getData` méthode qui renvoie une référence à un tableau interne, puis une autre routine lit ou remplit ce tableau directement. Nous avons “leaky abstractions” permettent de spécialisations diverses routines. +Diverses fonctions sur les colonnes peuvent être implémentées de manière générique et non efficace en utilisant `IColumn` méthodes pour extraire `Field` valeurs, ou d'une manière spécialisée en utilisant la connaissance de la disposition de la mémoire interne des données dans un `IColumn` application. Il est implémenté en lançant des fonctions à un `IColumn` tapez et traitez directement la représentation interne. Exemple, `ColumnUInt64` a l' `getData` méthode qui renvoie une référence à un tableau interne, puis une autre routine lit ou remplit ce tableau directement. Nous avons “leaky abstractions” permettent de spécialisations diverses routines. ## Types De Données {#data_types} -`IDataType` est responsable de la sérialisation et de la désérialisation: pour la lecture et l’écriture de morceaux de colonnes ou de valeurs individuelles sous forme binaire ou de texte. `IDataType` correspond directement aux types de données dans les tables. Par exemple, il y a `DataTypeUInt32`, `DataTypeDateTime`, `DataTypeString` et ainsi de suite. +`IDataType` est responsable de la sérialisation et de la désérialisation: pour la lecture et l'écriture de morceaux de colonnes ou de valeurs individuelles sous forme binaire ou de texte. `IDataType` correspond directement aux types de données dans les tables. Par exemple, il y a `DataTypeUInt32`, `DataTypeDateTime`, `DataTypeString` et ainsi de suite. `IDataType` et `IColumn` ne sont que faiblement liés les uns aux autres. Différents types de données peuvent être représentés en mémoire par le même `IColumn` application. Exemple, `DataTypeUInt32` et `DataTypeDateTime` sont tous deux représentés par `ColumnUInt32` ou `ColumnConstUInt32`. En outre, le même type de données peut être représentée par différents `IColumn` application. Exemple, `DataTypeUInt8` peut être représenté par `ColumnUInt8` ou `ColumnConstUInt8`. `IDataType` stocke uniquement les métadonnées. Par exemple, `DataTypeUInt8` ne stocke rien du tout (sauf vptr) et `DataTypeFixedString` magasins juste `N` (la taille des chaînes de taille fixe). -`IDataType` a des méthodes d’aide pour différents formats de données. Des exemples sont des méthodes pour sérialiser une valeur avec des guillemets possibles, pour sérialiser une valeur pour JSON et pour sérialiser une valeur dans le format XML. Il n’y a pas de correspondance directe avec les formats de données. Par exemple, les différents formats de données `Pretty` et `TabSeparated` pouvez utiliser le même `serializeTextEscaped` méthode d’aide à partir de la `IDataType` interface. +`IDataType` a des méthodes d'aide pour différents formats de données. Des exemples sont des méthodes pour sérialiser une valeur avec des guillemets possibles, pour sérialiser une valeur pour JSON et pour sérialiser une valeur dans le format XML. Il n'y a pas de correspondance directe avec les formats de données. Par exemple, les différents formats de données `Pretty` et `TabSeparated` pouvez utiliser le même `serializeTextEscaped` méthode d'aide à partir de la `IDataType` interface. ## Bloc {#block} -A `Block` est un conteneur qui représente un sous-ensemble (morceau) d’une table en mémoire. C’est juste un ensemble de triplets: `(IColumn, IDataType, column name)`. Pendant l’exécution de la requête, les données sont traitées par `Block`s. Si nous avons un `Block`, nous disposons de données (dans le `IColumn` objet), nous avons des informations sur son type (dans `IDataType`) qui nous indique comment traiter cette colonne, et nous avons le nom de la colonne. Il peut s’agir du nom de colonne d’origine de la table ou d’un nom artificiel attribué pour obtenir des résultats temporaires de calculs. +A `Block` est un conteneur qui représente un sous-ensemble (morceau) d'une table en mémoire. C'est juste un ensemble de triplets: `(IColumn, IDataType, column name)`. Pendant l'exécution de la requête, les données sont traitées par `Block`s. Si nous avons un `Block`, nous disposons de données (dans le `IColumn` objet), nous avons des informations sur son type (dans `IDataType`) qui nous indique comment traiter cette colonne, et nous avons le nom de la colonne. Il peut s'agir du nom de colonne d'origine de la table ou d'un nom artificiel attribué pour obtenir des résultats temporaires de calculs. -Lorsque nous calculons une fonction sur des colonnes dans un bloc, nous ajoutons une autre colonne avec son résultat au bloc, et nous ne touchons pas les colonnes pour les arguments de la fonction car les opérations sont immuables. Plus tard, les colonnes inutiles peuvent être supprimées du bloc, mais pas modifiées. Il est pratique pour l’élimination des sous-expressions communes. +Lorsque nous calculons une fonction sur des colonnes dans un bloc, nous ajoutons une autre colonne avec son résultat au bloc, et nous ne touchons pas les colonnes pour les arguments de la fonction car les opérations sont immuables. Plus tard, les colonnes inutiles peuvent être supprimées du bloc, mais pas modifiées. Il est pratique pour l'élimination des sous-expressions communes. -Des blocs sont créés pour chaque bloc de données traité. Notez que pour le même type de calcul, les noms et les types de colonnes restent les mêmes pour différents blocs, et seules les données de colonne changent. Il est préférable de diviser les données de bloc de l’en-tête de bloc car les petites tailles de Bloc ont une surcharge élevée de chaînes temporaires pour copier shared\_ptrs et les noms de colonnes. +Des blocs sont créés pour chaque bloc de données traité. Notez que pour le même type de calcul, les noms et les types de colonnes restent les mêmes pour différents blocs, et seules les données de colonne changent. Il est préférable de diviser les données de bloc de l'en-tête de bloc car les petites tailles de Bloc ont une surcharge élevée de chaînes temporaires pour copier shared\_ptrs et les noms de colonnes. ## Bloquer Les Flux {#block-streams} -Les flux de blocs sont destinés au traitement des données. Nous utilisons des flux de blocs pour lire des données quelque part, effectuer des transformations de données ou écrire des données quelque part. `IBlockInputStream` a l’ `read` méthode pour récupérer le bloc suivant, tandis que des. `IBlockOutputStream` a l’ `write` méthode pour pousser le bloc quelque part. +Les flux de blocs sont destinés au traitement des données. Nous utilisons des flux de blocs pour lire des données quelque part, effectuer des transformations de données ou écrire des données quelque part. `IBlockInputStream` a l' `read` méthode pour récupérer le bloc suivant, tandis que des. `IBlockOutputStream` a l' `write` méthode pour pousser le bloc quelque part. Les flux sont responsables de: -1. De la lecture ou de l’écriture dans une table. La table renvoie simplement un flux pour lire ou écrire des blocs. +1. De la lecture ou de l'écriture dans une table. La table renvoie simplement un flux pour lire ou écrire des blocs. 2. Mise en œuvre des formats de données. Par exemple, si vous souhaitez envoyer des données vers un terminal `Pretty` format, vous créez un flux de sortie de bloc où vous poussez des blocs, et il les formate. -3. Effectuer des transformations de données. Disons que vous avez `IBlockInputStream` et veulent créer un flux filtré. Vous créez `FilterBlockInputStream` et l’initialiser avec votre flux de données. Puis quand vous tirez un bloc de `FilterBlockInputStream`, il extrait un bloc de votre flux, le filtre et vous renvoie le bloc filtré. Les pipelines d’exécution des requêtes sont représentés de cette façon. +3. Effectuer des transformations de données. Disons que vous avez `IBlockInputStream` et veulent créer un flux filtré. Vous créez `FilterBlockInputStream` et l'initialiser avec votre flux de données. Puis quand vous tirez un bloc de `FilterBlockInputStream`, il extrait un bloc de votre flux, le filtre et vous renvoie le bloc filtré. Les pipelines d'exécution des requêtes sont représentés de cette façon. -Il y a des transformations plus sophistiquées. Par exemple, lorsque vous tirez de `AggregatingBlockInputStream` il lit toutes les données à partir de sa source, agrégats, puis renvoie un flux de données agrégées pour vous. Un autre exemple: `UnionBlockInputStream` accepte de nombreuses sources d’entrée dans le constructeur et également un certain nombre de threads. Il lance plusieurs threads et lit à partir de plusieurs sources en parallèle. +Il y a des transformations plus sophistiquées. Par exemple, lorsque vous tirez de `AggregatingBlockInputStream` il lit toutes les données à partir de sa source, agrégats, puis renvoie un flux de données agrégées pour vous. Un autre exemple: `UnionBlockInputStream` accepte de nombreuses sources d'entrée dans le constructeur et également un certain nombre de threads. Il lance plusieurs threads et lit à partir de plusieurs sources en parallèle. -> Les flux de blocs utilisent le “pull” approche pour contrôler le flux: lorsque vous extrayez un bloc du premier flux, il extrait par conséquent les blocs requis des flux imbriqués, et l’ensemble du pipeline d’exécution fonctionnera. Ni “pull” ni “push” est la meilleure solution, car le flux de contrôle est implicite, ce qui limite l’implémentation de diverses fonctionnalités telles que l’exécution simultanée de plusieurs requêtes (fusion de plusieurs pipelines ensemble). Cette limitation pourrait être surmontée avec des coroutines ou simplement en exécutant des threads supplémentaires qui s’attendent les uns aux autres. Nous pouvons avoir plus de possibilités si nous rendons le flux de contrôle explicite: si nous localisons la logique pour passer des données d’une unité de calcul à une autre en dehors de ces unités de calcul. Lire ce [article](http://journal.stuffwithstuff.com/2013/01/13/iteration-inside-and-out/) pour plus de pensées. +> Les flux de blocs utilisent le “pull” approche pour contrôler le flux: lorsque vous extrayez un bloc du premier flux, il extrait par conséquent les blocs requis des flux imbriqués, et l'ensemble du pipeline d'exécution fonctionnera. Ni “pull” ni “push” est la meilleure solution, car le flux de contrôle est implicite, ce qui limite l'implémentation de diverses fonctionnalités telles que l'exécution simultanée de plusieurs requêtes (fusion de plusieurs pipelines ensemble). Cette limitation pourrait être surmontée avec des coroutines ou simplement en exécutant des threads supplémentaires qui s'attendent les uns aux autres. Nous pouvons avoir plus de possibilités si nous rendons le flux de contrôle explicite: si nous localisons la logique pour passer des données d'une unité de calcul à une autre en dehors de ces unités de calcul. Lire ce [article](http://journal.stuffwithstuff.com/2013/01/13/iteration-inside-and-out/) pour plus de pensées. -Il convient de noter que le pipeline d’exécution de la requête crée des données temporaires à chaque étape. Nous essayons de garder la taille du bloc suffisamment petite pour que les données temporaires tiennent dans le cache du processeur. Avec cette hypothèse, l’écriture et la lecture de données temporaires sont presque libres en comparaison avec d’autres calculs. Nous pourrions envisager une alternative, qui est de fusionner de nombreuses opérations dans le pipeline ensemble. Cela pourrait rendre le pipeline aussi court que possible et supprimer une grande partie des données temporaires, ce qui pourrait être un avantage, mais cela présente également des inconvénients. Par exemple, un pipeline divisé facilite l’implémentation de la mise en cache de données intermédiaires, le vol de données intermédiaires à partir de requêtes similaires exécutées en même temps et la fusion de pipelines pour des requêtes similaires. +Il convient de noter que le pipeline d'exécution de la requête crée des données temporaires à chaque étape. Nous essayons de garder la taille du bloc suffisamment petite pour que les données temporaires tiennent dans le cache du processeur. Avec cette hypothèse, l'écriture et la lecture de données temporaires sont presque libres en comparaison avec d'autres calculs. Nous pourrions envisager une alternative, qui est de fusionner de nombreuses opérations dans le pipeline ensemble. Cela pourrait rendre le pipeline aussi court que possible et supprimer une grande partie des données temporaires, ce qui pourrait être un avantage, mais cela présente également des inconvénients. Par exemple, un pipeline divisé facilite l'implémentation de la mise en cache de données intermédiaires, le vol de données intermédiaires à partir de requêtes similaires exécutées en même temps et la fusion de pipelines pour des requêtes similaires. ## Format {#formats} -Les formats de données sont implémentés avec des flux de blocs. Il y a “presentational” formats appropriés uniquement pour la sortie de données vers le client, tels que `Pretty` format, qui fournit seulement `IBlockOutputStream`. Et il existe des formats d’entrée / sortie, tels que `TabSeparated` ou `JSONEachRow`. +Les formats de données sont implémentés avec des flux de blocs. Il y a “presentational” formats appropriés uniquement pour la sortie de données vers le client, tels que `Pretty` format, qui fournit seulement `IBlockOutputStream`. Et il existe des formats d'entrée / sortie, tels que `TabSeparated` ou `JSONEachRow`. Il y a aussi des flux de lignes: `IRowInputStream` et `IRowOutputStream`. Ils vous permettent de tirer/pousser des données par des lignes individuelles, pas par des blocs. Et ils ne sont nécessaires que pour simplifier la mise en œuvre des formats orientés ligne. Wrapper `BlockInputStreamFromRowInputStream` et `BlockOutputStreamFromRowOutputStream` vous permet de convertir des flux orientés ligne en flux orientés blocs réguliers. ## I/O {#io} -Pour l’entrée/sortie orientée octet, il y a `ReadBuffer` et `WriteBuffer` les classes abstraites. Ils sont utilisés à la place de C++ `iostream`s. Ne vous inquiétez pas: chaque projet c++ mature utilise autre chose que `iostream`s pour de bonnes raisons. +Pour l'entrée/sortie orientée octet, il y a `ReadBuffer` et `WriteBuffer` les classes abstraites. Ils sont utilisés à la place de C++ `iostream`s. Ne vous inquiétez pas: chaque projet c++ mature utilise autre chose que `iostream`s pour de bonnes raisons. `ReadBuffer` et `WriteBuffer` sont juste un tampon contigu et un curseur pointant vers la position dans ce tampon. Les implémentations peuvent posséder ou non la mémoire du tampon. Il existe une méthode virtuelle pour remplir le tampon avec les données suivantes (pour `ReadBuffer`) ou pour vider le tampon quelque part (pour `WriteBuffer`). Les méthodes virtuelles sont rarement cités. Les implémentations de `ReadBuffer`/`WriteBuffer` sont utilisés pour travailler avec des fichiers et des descripteurs de fichiers et des sockets réseau, pour implémenter la compression (`CompressedWriteBuffer` is initialized with another WriteBuffer and performs compression before writing data to it), and for other purposes – the names `ConcatReadBuffer`, `LimitReadBuffer`, et `HashingWriteBuffer` parler pour eux-mêmes. -Read / WriteBuffers ne traite que les octets. Il y a des fonctions de `ReadHelpers` et `WriteHelpers` fichiers d’en-tête pour aider à formater l’entrée / sortie. Par exemple, il existe des assistants pour écrire un nombre au format décimal. +Read / WriteBuffers ne traite que les octets. Il y a des fonctions de `ReadHelpers` et `WriteHelpers` fichiers d'en-tête pour aider à formater l'entrée / sortie. Par exemple, il existe des assistants pour écrire un nombre au format décimal. Regardons ce qui se passe lorsque vous voulez écrire un ensemble de résultats dans `JSON` format de sortie standard (stdout). Vous avez un jeu de résultats prêt à être récupéré `IBlockInputStream`. Vous créez `WriteBufferFromFileDescriptor(STDOUT_FILENO)` pour écrire des octets dans stdout. Vous créez `JSONRowOutputStream`, initialisé avec qui `WriteBuffer`, pour écrire des lignes dans `JSON` à stdout. Vous créez `BlockOutputStreamFromRowOutputStream` de plus, pour la représenter comme `IBlockOutputStream`. Ensuite, vous appelez `copyData` pour transférer des données de `IBlockInputStream` de `IBlockOutputStream` et tout fonctionne. Interne, `JSONRowOutputStream` écrira divers délimiteurs JSON et appellera `IDataType::serializeTextJSON` méthode avec une référence à `IColumn` et le numéro de ligne comme arguments. Conséquent, `IDataType::serializeTextJSON` appellera une méthode de `WriteHelpers.h`: exemple, `writeText` pour les types numériques et `writeJSONString` pour `DataTypeString`. ## Table {#tables} -Le `IStorage` l’interface représente les tables. Différentes implémentations de cette interface sont des moteurs de table différents. Les exemples sont `StorageMergeTree`, `StorageMemory` et ainsi de suite. Les Instances de ces classes ne sont que des tables. +Le `IStorage` l'interface représente les tables. Différentes implémentations de cette interface sont des moteurs de table différents. Les exemples sont `StorageMergeTree`, `StorageMemory` et ainsi de suite. Les Instances de ces classes ne sont que des tables. -Clé `IStorage` les méthodes sont `read` et `write`. Il y a aussi des `alter`, `rename`, `drop` et ainsi de suite. Le `read` méthode accepte les arguments suivants: l’ensemble de colonnes à lire à partir d’un tableau, l’ `AST` requête à considérer, et le nombre souhaité de flux de retour. Il renvoie un ou plusieurs `IBlockInputStream` objets et informations sur l’étape de traitement des données qui a été effectuée dans un moteur de table lors de l’exécution de la requête. +Clé `IStorage` les méthodes sont `read` et `write`. Il y a aussi des `alter`, `rename`, `drop` et ainsi de suite. Le `read` méthode accepte les arguments suivants: l'ensemble de colonnes à lire à partir d'un tableau, l' `AST` requête à considérer, et le nombre souhaité de flux de retour. Il renvoie un ou plusieurs `IBlockInputStream` objets et informations sur l'étape de traitement des données qui a été effectuée dans un moteur de table lors de l'exécution de la requête. -Dans la plupart des cas, la méthode read n’est responsable que de la lecture des colonnes spécifiées à partir d’une table, et non d’un traitement ultérieur des données. Tout traitement ultérieur des données est effectué par l’interpréteur de requêtes et n’est pas de la responsabilité de `IStorage`. +Dans la plupart des cas, la méthode read n'est responsable que de la lecture des colonnes spécifiées à partir d'une table, et non d'un traitement ultérieur des données. Tout traitement ultérieur des données est effectué par l'interpréteur de requêtes et n'est pas de la responsabilité de `IStorage`. Mais il y a des exceptions notables: -- La requête AST est transmise au `read` et le moteur de table peut l’utiliser pour dériver l’utilisation de l’index et pour lire moins de données à partir d’une table. -- Parfois, le moteur de table peut traiter les données lui-même à une étape spécifique. Exemple, `StorageDistributed` peut envoyer une requête aux serveurs distants, leur demander de traiter les données à une étape où les données de différents serveurs distants peuvent être fusionnées, et renvoyer ces données prétraitées. L’interpréteur de requête termine ensuite le traitement des données. +- La requête AST est transmise au `read` et le moteur de table peut l'utiliser pour dériver l'utilisation de l'index et pour lire moins de données à partir d'une table. +- Parfois, le moteur de table peut traiter les données lui-même à une étape spécifique. Exemple, `StorageDistributed` peut envoyer une requête aux serveurs distants, leur demander de traiter les données à une étape où les données de différents serveurs distants peuvent être fusionnées, et renvoyer ces données prétraitées. L'interpréteur de requête termine ensuite le traitement des données. -Table `read` la méthode peut retourner plusieurs `IBlockInputStream` objets permettant le traitement parallèle des données. Ces flux d’entrée de bloc multiples peuvent lire à partir d’une table en parallèle. Ensuite, vous pouvez envelopper ces flux avec diverses transformations (telles que l’évaluation d’expression ou le filtrage) qui peuvent être calculées indépendamment et créer un `UnionBlockInputStream` en plus d’eux, pour lire à partir de plusieurs flux en parallèle. +Table `read` la méthode peut retourner plusieurs `IBlockInputStream` objets permettant le traitement parallèle des données. Ces flux d'entrée de bloc multiples peuvent lire à partir d'une table en parallèle. Ensuite, vous pouvez envelopper ces flux avec diverses transformations (telles que l'évaluation d'expression ou le filtrage) qui peuvent être calculées indépendamment et créer un `UnionBlockInputStream` en plus d'eux, pour lire à partir de plusieurs flux en parallèle. -Il y a aussi des `TableFunction`s. Ce sont des fonctions qui renvoient un `IStorage` objet à utiliser dans le `FROM` la clause d’une requête. +Il y a aussi des `TableFunction`s. Ce sont des fonctions qui renvoient un `IStorage` objet à utiliser dans le `FROM` la clause d'une requête. -Pour avoir une idée rapide de la façon d’implémenter votre moteur de table, regardez quelque chose de simple, comme `StorageMemory` ou `StorageTinyLog`. +Pour avoir une idée rapide de la façon d'implémenter votre moteur de table, regardez quelque chose de simple, comme `StorageMemory` ou `StorageTinyLog`. -> Comme le résultat de l’ `read` méthode, `IStorage` retourner `QueryProcessingStage` – information about what parts of the query were already calculated inside storage. +> Comme le résultat de l' `read` méthode, `IStorage` retourner `QueryProcessingStage` – information about what parts of the query were already calculated inside storage. ## Analyseur {#parsers} Un analyseur de descente récursif écrit à la main analyse une requête. Exemple, `ParserSelectQuery` appelle simplement récursivement les analyseurs sous-jacents pour diverses parties de la requête. Les analyseurs créent un `AST`. Le `AST` est représenté par des nœuds, qui sont des instances de `IAST`. -> Les générateurs d’analyseurs ne sont pas utilisés pour des raisons historiques. +> Les générateurs d'analyseurs ne sont pas utilisés pour des raisons historiques. ## Interprète {#interpreters} -Les interprètes sont responsables de la création du pipeline d’exécution des requêtes à partir `AST`. Il existe des interprètes simples, tels que `InterpreterExistsQuery` et `InterpreterDropQuery` ou le plus sophistiqué de `InterpreterSelectQuery`. Le pipeline d’exécution de requête est une combinaison de flux d’entrée ou de sortie de bloc. Par exemple, le résultat de l’interprétation de la `SELECT` la requête est la `IBlockInputStream` pour lire le jeu de résultats; le résultat de la requête d’INSERTION est l’ `IBlockOutputStream` pour écrire des données à insérer, et le résultat de l’interprétation `INSERT SELECT` la requête est la `IBlockInputStream` cela renvoie un jeu de résultats vide lors de la première lecture, mais qui copie `SELECT` de `INSERT` dans le même temps. +Les interprètes sont responsables de la création du pipeline d'exécution des requêtes à partir `AST`. Il existe des interprètes simples, tels que `InterpreterExistsQuery` et `InterpreterDropQuery` ou le plus sophistiqué de `InterpreterSelectQuery`. Le pipeline d'exécution de requête est une combinaison de flux d'entrée ou de sortie de bloc. Par exemple, le résultat de l'interprétation de la `SELECT` la requête est la `IBlockInputStream` pour lire le jeu de résultats; le résultat de la requête d'INSERTION est l' `IBlockOutputStream` pour écrire des données à insérer, et le résultat de l'interprétation `INSERT SELECT` la requête est la `IBlockInputStream` cela renvoie un jeu de résultats vide lors de la première lecture, mais qui copie `SELECT` de `INSERT` dans le même temps. -`InterpreterSelectQuery` utiliser `ExpressionAnalyzer` et `ExpressionActions` machines pour l’analyse des requêtes et des transformations. C’est là que la plupart des optimisations de requêtes basées sur des règles sont effectuées. `ExpressionAnalyzer` est assez désordonné et devrait être réécrit: diverses transformations et optimisations de requête doivent être extraites dans des classes séparées pour permettre des transformations modulaires ou une requête. +`InterpreterSelectQuery` utiliser `ExpressionAnalyzer` et `ExpressionActions` machines pour l'analyse des requêtes et des transformations. C'est là que la plupart des optimisations de requêtes basées sur des règles sont effectuées. `ExpressionAnalyzer` est assez désordonné et devrait être réécrit: diverses transformations et optimisations de requête doivent être extraites dans des classes séparées pour permettre des transformations modulaires ou une requête. ## Fonction {#functions} -Il y a des fonctions ordinaires et des fonctions agrégées. Pour les fonctions d’agrégation, voir la section suivante. +Il y a des fonctions ordinaires et des fonctions agrégées. Pour les fonctions d'agrégation, voir la section suivante. -Ordinary functions don’t change the number of rows – they work as if they are processing each row independently. In fact, functions are not called for individual rows, but for `Block`’s de données pour implémenter l’exécution de requête vectorisée. +Ordinary functions don't change the number of rows – they work as if they are processing each row independently. In fact, functions are not called for individual rows, but for `Block`'s de données pour implémenter l'exécution de requête vectorisée. -Il y a quelques fonctions diverses, comme [la taille de bloc](../sql-reference/functions/other-functions.md#function-blocksize), [rowNumberInBlock](../sql-reference/functions/other-functions.md#function-rownumberinblock), et [runningAccumulate](../sql-reference/functions/other-functions.md#function-runningaccumulate), qui exploitent le traitement de bloc et violent l’indépendance des lignes. +Il y a quelques fonctions diverses, comme [la taille de bloc](../sql-reference/functions/other-functions.md#function-blocksize), [rowNumberInBlock](../sql-reference/functions/other-functions.md#function-rownumberinblock), et [runningAccumulate](../sql-reference/functions/other-functions.md#function-runningaccumulate), qui exploitent le traitement de bloc et violent l'indépendance des lignes. -ClickHouse a un typage fort, donc il n’y a pas de conversion de type implicite. Si une fonction ne prend pas en charge une combinaison spécifique de types, elle lève une exception. Mais les fonctions peuvent fonctionner (être surchargées) pour de nombreuses combinaisons de types différentes. Par exemple, l’ `plus` fonction (pour mettre en œuvre la `+` opérateur) fonctionne pour toute combinaison de types numériques: `UInt8` + `Float32`, `UInt16` + `Int8` et ainsi de suite. En outre, certaines fonctions variadiques peuvent accepter n’importe quel nombre d’arguments, tels que `concat` fonction. +ClickHouse a un typage fort, donc il n'y a pas de conversion de type implicite. Si une fonction ne prend pas en charge une combinaison spécifique de types, elle lève une exception. Mais les fonctions peuvent fonctionner (être surchargées) pour de nombreuses combinaisons de types différentes. Par exemple, l' `plus` fonction (pour mettre en œuvre la `+` opérateur) fonctionne pour toute combinaison de types numériques: `UInt8` + `Float32`, `UInt16` + `Int8` et ainsi de suite. En outre, certaines fonctions variadiques peuvent accepter n'importe quel nombre d'arguments, tels que `concat` fonction. -L’implémentation d’une fonction peut être légèrement gênante car une fonction distribue explicitement les types de données pris en charge et pris en charge `IColumns`. Par exemple, l’ `plus` la fonction a du code généré par l’instanciation D’un modèle C++ pour chaque combinaison de types numériques, et des arguments gauche et droit constants ou non constants. +L'implémentation d'une fonction peut être légèrement gênante car une fonction distribue explicitement les types de données pris en charge et pris en charge `IColumns`. Par exemple, l' `plus` la fonction a du code généré par l'instanciation D'un modèle C++ pour chaque combinaison de types numériques, et des arguments gauche et droit constants ou non constants. -C’est un excellent endroit pour implémenter la génération de code d’exécution pour éviter le gonflement du code de modèle. En outre, il permet d’ajouter des fonctions fusionnées comme Fusionné Multiplier-Ajouter ou de faire plusieurs comparaisons dans une itération de boucle. +C'est un excellent endroit pour implémenter la génération de code d'exécution pour éviter le gonflement du code de modèle. En outre, il permet d'ajouter des fonctions fusionnées comme Fusionné Multiplier-Ajouter ou de faire plusieurs comparaisons dans une itération de boucle. -En raison de l’exécution de requête vectorisée, les fonctions ne sont pas court-circuitées. Par exemple, si vous écrivez `WHERE f(x) AND g(y)` les deux faces sont calculés, même pour les lignes, quand `f(x)` est égal à zéro (sauf quand `f(x)` est une expression constante nulle). Mais si la sélectivité de l’ `f(x)` la condition est élevée, et le calcul de `f(x)` est beaucoup moins cher que `g(y)`, il est préférable d’implémenter le calcul multi-pass. Il serait d’abord calculer `f(x)` puis filtrer les colonnes par la suite, puis de calculer `g(y)` uniquement pour les petits morceaux de données filtrés. +En raison de l'exécution de requête vectorisée, les fonctions ne sont pas court-circuitées. Par exemple, si vous écrivez `WHERE f(x) AND g(y)` les deux faces sont calculés, même pour les lignes, quand `f(x)` est égal à zéro (sauf quand `f(x)` est une expression constante nulle). Mais si la sélectivité de l' `f(x)` la condition est élevée, et le calcul de `f(x)` est beaucoup moins cher que `g(y)`, il est préférable d'implémenter le calcul multi-pass. Il serait d'abord calculer `f(x)` puis filtrer les colonnes par la suite, puis de calculer `g(y)` uniquement pour les petits morceaux de données filtrés. -## Les Fonctions D’Agrégation {#aggregate-functions} +## Les Fonctions D'Agrégation {#aggregate-functions} -Les fonctions d’agrégation sont des fonctions avec État. Ils accumulent les valeurs passées dans certains etats et vous permettent d’obtenir des résultats de cet état. Ils sont gérés avec le `IAggregateFunction` interface. Les États peuvent être assez simples (l’État pour `AggregateFunctionCount` est juste un seul `UInt64` valeur) ou très complexes (l’état de `AggregateFunctionUniqCombined` est une combinaison linéaire du tableau, d’une table de hachage, et un `HyperLogLog` structure probabiliste des données). +Les fonctions d'agrégation sont des fonctions avec État. Ils accumulent les valeurs passées dans certains etats et vous permettent d'obtenir des résultats de cet état. Ils sont gérés avec le `IAggregateFunction` interface. Les États peuvent être assez simples (l'État pour `AggregateFunctionCount` est juste un seul `UInt64` valeur) ou très complexes (l'état de `AggregateFunctionUniqCombined` est une combinaison linéaire du tableau, d'une table de hachage, et un `HyperLogLog` structure probabiliste des données). -Les États sont répartis en `Arena` (un pool de mémoire) pour traiter plusieurs états lors de l’exécution d’une cardinalité élevée `GROUP BY` requête. Les États peuvent avoir un constructeur et un destructeur non triviaux: par exemple, les États d’agrégation compliqués peuvent allouer eux-mêmes de la mémoire supplémentaire. Il faut accorder une certaine attention à la création et à la destruction des États et à la transmission appropriée de leur propriété et de leur ordre de destruction. +Les États sont répartis en `Arena` (un pool de mémoire) pour traiter plusieurs états lors de l'exécution d'une cardinalité élevée `GROUP BY` requête. Les États peuvent avoir un constructeur et un destructeur non triviaux: par exemple, les États d'agrégation compliqués peuvent allouer eux-mêmes de la mémoire supplémentaire. Il faut accorder une certaine attention à la création et à la destruction des États et à la transmission appropriée de leur propriété et de leur ordre de destruction. -Les États d’agrégation peuvent être sérialisés et désérialisés pour passer sur le réseau pendant l’exécution de la requête distribuée ou pour les écrire sur le disque où il n’y a pas assez de RAM. Ils peuvent même être stockés dans une table avec le `DataTypeAggregateFunction` pour permettre l’agrégation incrémentielle des données. +Les États d'agrégation peuvent être sérialisés et désérialisés pour passer sur le réseau pendant l'exécution de la requête distribuée ou pour les écrire sur le disque où il n'y a pas assez de RAM. Ils peuvent même être stockés dans une table avec le `DataTypeAggregateFunction` pour permettre l'agrégation incrémentielle des données. -> Le format de données sérialisé pour les états de fonction d’agrégat n’est pas versionné pour le moment. C’est ok si les États d’agrégat ne sont stockés que temporairement. Mais nous avons l’ `AggregatingMergeTree` moteur de table pour l’agrégation incrémentielle, et les gens l’utilisent déjà en production. C’est la raison pour laquelle la rétrocompatibilité est requise lors de la modification du format sérialisé pour toute fonction d’agrégat à l’avenir. +> Le format de données sérialisé pour les états de fonction d'agrégat n'est pas versionné pour le moment. C'est ok si les États d'agrégat ne sont stockés que temporairement. Mais nous avons l' `AggregatingMergeTree` moteur de table pour l'agrégation incrémentielle, et les gens l'utilisent déjà en production. C'est la raison pour laquelle la rétrocompatibilité est requise lors de la modification du format sérialisé pour toute fonction d'agrégat à l'avenir. ## Serveur {#server} Le serveur implémente plusieurs interfaces différentes: - Une interface HTTP pour tous les clients étrangers. -- Une interface TCP pour le client clickhouse natif et pour la communication inter-serveur lors de l’exécution de la requête distribuée. +- Une interface TCP pour le client clickhouse natif et pour la communication inter-serveur lors de l'exécution de la requête distribuée. - Une interface pour transférer des données pour la réplication. -En interne, il s’agit simplement d’un serveur multithread primitif sans coroutines ni fibres. Étant donné que le serveur n’est pas conçu pour traiter un taux élevé de requêtes simples, mais pour traiter un taux relativement faible de requêtes complexes, chacun d’eux peut traiter une grande quantité de données à des fins d’analyse. +En interne, il s'agit simplement d'un serveur multithread primitif sans coroutines ni fibres. Étant donné que le serveur n'est pas conçu pour traiter un taux élevé de requêtes simples, mais pour traiter un taux relativement faible de requêtes complexes, chacun d'eux peut traiter une grande quantité de données à des fins d'analyse. -Le serveur initialise le `Context` classe avec l’environnement nécessaire à l’exécution des requêtes: la liste des bases de données disponibles, des utilisateurs et des droits d’accès, des paramètres, des clusters, la liste des processus, le journal des requêtes, etc. Les interprètes utilisent cet environnement. +Le serveur initialise le `Context` classe avec l'environnement nécessaire à l'exécution des requêtes: la liste des bases de données disponibles, des utilisateurs et des droits d'accès, des paramètres, des clusters, la liste des processus, le journal des requêtes, etc. Les interprètes utilisent cet environnement. -Nous maintenons une compatibilité ascendante et descendante complète pour le protocole TCP du serveur: les anciens clients peuvent parler à de nouveaux serveurs, et les nouveaux clients peuvent parler à d’anciens serveurs. Mais nous ne voulons pas le maintenir éternellement, et nous supprimons le support pour les anciennes versions après environ un an. +Nous maintenons une compatibilité ascendante et descendante complète pour le protocole TCP du serveur: les anciens clients peuvent parler à de nouveaux serveurs, et les nouveaux clients peuvent parler à d'anciens serveurs. Mais nous ne voulons pas le maintenir éternellement, et nous supprimons le support pour les anciennes versions après environ un an. !!! note "Note" - Pour la plupart des applications externes, nous vous recommandons d’utiliser L’interface HTTP car elle est simple et facile à utiliser. Le protocole TCP est plus étroitement lié aux structures de données internes: il utilise un format interne pour passer des blocs de données, et il utilise un cadrage personnalisé pour les données compressées. Nous n’avons pas publié de bibliothèque C pour ce protocole car elle nécessite de lier la plupart de la base de code ClickHouse, ce qui n’est pas pratique. + Pour la plupart des applications externes, nous vous recommandons d'utiliser L'interface HTTP car elle est simple et facile à utiliser. Le protocole TCP est plus étroitement lié aux structures de données internes: il utilise un format interne pour passer des blocs de données, et il utilise un cadrage personnalisé pour les données compressées. Nous n'avons pas publié de bibliothèque C pour ce protocole car elle nécessite de lier la plupart de la base de code ClickHouse, ce qui n'est pas pratique. ## Exécution De Requête Distribuée {#distributed-query-execution} -Les serveurs d’une configuration de cluster sont pour la plupart indépendants. Vous pouvez créer un `Distributed` table sur un ou tous les serveurs dans un cluster. Le `Distributed` table does not store data itself – it only provides a “view” à toutes les tables sur plusieurs nœuds d’un cluster. Lorsque vous sélectionnez à partir d’un `Distributed` table, il réécrit cette requête, choisit les nœuds distants en fonction des paramètres d’équilibrage de charge et leur envoie la requête. Le `Distributed` table demande aux serveurs distants de traiter une requête jusqu’à une étape où les résultats intermédiaires de différents serveurs peuvent être fusionnés. Puis il reçoit les résultats intermédiaires et les fusionne. La table distribuée essaie de distribuer autant de travail que possible aux serveurs distants et n’envoie pas beaucoup de données intermédiaires sur le réseau. +Les serveurs d'une configuration de cluster sont pour la plupart indépendants. Vous pouvez créer un `Distributed` table sur un ou tous les serveurs dans un cluster. Le `Distributed` table does not store data itself – it only provides a “view” à toutes les tables sur plusieurs nœuds d'un cluster. Lorsque vous sélectionnez à partir d'un `Distributed` table, il réécrit cette requête, choisit les nœuds distants en fonction des paramètres d'équilibrage de charge et leur envoie la requête. Le `Distributed` table demande aux serveurs distants de traiter une requête jusqu'à une étape où les résultats intermédiaires de différents serveurs peuvent être fusionnés. Puis il reçoit les résultats intermédiaires et les fusionne. La table distribuée essaie de distribuer autant de travail que possible aux serveurs distants et n'envoie pas beaucoup de données intermédiaires sur le réseau. -Les choses deviennent plus compliquées lorsque vous avez des sous-requêtes dans des clauses IN ou JOIN, et que chacune d’elles utilise un `Distributed` table. Nous avons différentes stratégies pour l’exécution de ces requêtes. +Les choses deviennent plus compliquées lorsque vous avez des sous-requêtes dans des clauses IN ou JOIN, et que chacune d'elles utilise un `Distributed` table. Nous avons différentes stratégies pour l'exécution de ces requêtes. -Il n’existe pas de plan de requête global pour l’exécution des requêtes distribuées. Chaque nœud a son plan de requête local pour sa partie du travail. Nous n’avons qu’une simple exécution de requête distribuée en une seule passe: nous envoyons des requêtes pour les nœuds distants, puis fusionnons les résultats. Mais cela n’est pas possible pour les requêtes compliquées avec des groupes de cardinalité élevés ou avec une grande quantité de données temporaires pour la jointure. Dans de tels cas, nous avons besoin de “reshuffle” données entre les serveurs, ce qui nécessite une coordination supplémentaire. ClickHouse ne supporte pas ce type d’exécution de requête, et nous devons y travailler. +Il n'existe pas de plan de requête global pour l'exécution des requêtes distribuées. Chaque nœud a son plan de requête local pour sa partie du travail. Nous n'avons qu'une simple exécution de requête distribuée en une seule passe: nous envoyons des requêtes pour les nœuds distants, puis fusionnons les résultats. Mais cela n'est pas possible pour les requêtes compliquées avec des groupes de cardinalité élevés ou avec une grande quantité de données temporaires pour la jointure. Dans de tels cas, nous avons besoin de “reshuffle” données entre les serveurs, ce qui nécessite une coordination supplémentaire. ClickHouse ne supporte pas ce type d'exécution de requête, et nous devons y travailler. -## Fusion De L’Arbre {#merge-tree} +## Fusion De L'Arbre {#merge-tree} -`MergeTree` est une famille de moteurs de stockage qui prend en charge l’indexation par clé primaire. La clé primaire peut être un tuple arbitraire de colonnes ou d’expressions. De données dans un `MergeTree` la table est stockée dans “parts”. Chaque partie stocke les données dans l’ordre de la clé primaire, de sorte que les données sont ordonnées lexicographiquement par le tuple de clé primaire. Toutes les colonnes du tableau sont stockés dans différents `column.bin` les fichiers dans ces régions. Les fichiers sont constitués de blocs compressés. Chaque bloc est généralement de 64 KO à 1 Mo de données non compressées, en fonction de la taille de la valeur moyenne. Les blocs sont constitués de valeurs de colonne placées de manière contiguë l’une après l’autre. Les valeurs de colonne sont dans le même ordre pour chaque colonne (la clé primaire définit l’ordre), donc lorsque vous itérez par plusieurs colonnes, vous obtenez des valeurs pour les lignes correspondantes. +`MergeTree` est une famille de moteurs de stockage qui prend en charge l'indexation par clé primaire. La clé primaire peut être un tuple arbitraire de colonnes ou d'expressions. De données dans un `MergeTree` la table est stockée dans “parts”. Chaque partie stocke les données dans l'ordre de la clé primaire, de sorte que les données sont ordonnées lexicographiquement par le tuple de clé primaire. Toutes les colonnes du tableau sont stockés dans différents `column.bin` les fichiers dans ces régions. Les fichiers sont constitués de blocs compressés. Chaque bloc est généralement de 64 KO à 1 Mo de données non compressées, en fonction de la taille de la valeur moyenne. Les blocs sont constitués de valeurs de colonne placées de manière contiguë l'une après l'autre. Les valeurs de colonne sont dans le même ordre pour chaque colonne (la clé primaire définit l'ordre), donc lorsque vous itérez par plusieurs colonnes, vous obtenez des valeurs pour les lignes correspondantes. -La clé primaire elle-même est “sparse”. Il ne traite pas chaque ligne, mais seulement certaines plages de données. Séparé `primary.idx` fichier a la valeur de la clé primaire pour chaque N-ième ligne, où N est appelé `index_granularity` (habituellement, N = 8192). Aussi, pour chaque colonne, nous avons `column.mrk` les fichiers avec l’ “marks,” qui sont des décalages à chaque N-ème ligne dans le fichier de données. Chaque marque est une paire: le décalage dans le fichier au début du bloc compressé, et le décalage dans le bloc décompressé au début des données. Habituellement, les blocs compressés sont alignés par des marques, et le décalage dans le bloc décompressé est nul. Les données pour `primary.idx` réside toujours dans la mémoire, et les données pour `column.mrk` les fichiers sont mis en cache. +La clé primaire elle-même est “sparse”. Il ne traite pas chaque ligne, mais seulement certaines plages de données. Séparé `primary.idx` fichier a la valeur de la clé primaire pour chaque N-ième ligne, où N est appelé `index_granularity` (habituellement, N = 8192). Aussi, pour chaque colonne, nous avons `column.mrk` les fichiers avec l' “marks,” qui sont des décalages à chaque N-ème ligne dans le fichier de données. Chaque marque est une paire: le décalage dans le fichier au début du bloc compressé, et le décalage dans le bloc décompressé au début des données. Habituellement, les blocs compressés sont alignés par des marques, et le décalage dans le bloc décompressé est nul. Les données pour `primary.idx` réside toujours dans la mémoire, et les données pour `column.mrk` les fichiers sont mis en cache. -Quand nous allons lire quelque chose d’une partie dans `MergeTree` nous regardons `primary.idx` données et locate plages qui pourraient contenir des données demandées, puis regardez `column.mrk` données et calculer des décalages pour savoir où commencer à lire ces plages. En raison de la rareté, les données excédentaires peuvent être lues. ClickHouse ne convient pas à une charge élevée de requêtes ponctuelles simples, car toute la gamme avec `index_granularity` les lignes doivent être lues pour chaque clé, et le bloc compressé entier doit être décompressé pour chaque colonne. Nous avons rendu l’index clairsemé parce que nous devons être en mesure de maintenir des milliards de lignes par serveur unique sans consommation de mémoire notable pour l’index. De plus, comme la clé primaire est clairsemée, elle n’est pas unique: elle ne peut pas vérifier l’existence de la clé dans la table au moment de l’insertion. Vous pourriez avoir plusieurs lignes avec la même clé dans une table. +Quand nous allons lire quelque chose d'une partie dans `MergeTree` nous regardons `primary.idx` données et locate plages qui pourraient contenir des données demandées, puis regardez `column.mrk` données et calculer des décalages pour savoir où commencer à lire ces plages. En raison de la rareté, les données excédentaires peuvent être lues. ClickHouse ne convient pas à une charge élevée de requêtes ponctuelles simples, car toute la gamme avec `index_granularity` les lignes doivent être lues pour chaque clé, et le bloc compressé entier doit être décompressé pour chaque colonne. Nous avons rendu l'index clairsemé parce que nous devons être en mesure de maintenir des milliards de lignes par serveur unique sans consommation de mémoire notable pour l'index. De plus, comme la clé primaire est clairsemée, elle n'est pas unique: elle ne peut pas vérifier l'existence de la clé dans la table au moment de l'insertion. Vous pourriez avoir plusieurs lignes avec la même clé dans une table. -Lorsque vous `INSERT` un tas de données dans `MergeTree`, ce groupe est trié par ordre de clé primaire et forme une nouvelle partie. Il existe des threads d’arrière-plan qui sélectionnent périodiquement certaines parties et les fusionnent en une seule partie triée pour maintenir le nombre de parties relativement faible. C’est pourquoi il est appelé `MergeTree`. Bien sûr, la fusion conduit à “write amplification”. Toutes les parties sont immuables: elles sont seulement créées et supprimées, mais pas modifiées. Lorsque SELECT est exécuté, il contient un instantané de la table (un ensemble de parties). Après la Fusion, nous conservons également les anciennes pièces pendant un certain temps pour faciliter une récupération après une défaillance, donc si nous voyons qu’une partie fusionnée est probablement cassée, nous pouvons la remplacer par ses parties sources. +Lorsque vous `INSERT` un tas de données dans `MergeTree`, ce groupe est trié par ordre de clé primaire et forme une nouvelle partie. Il existe des threads d'arrière-plan qui sélectionnent périodiquement certaines parties et les fusionnent en une seule partie triée pour maintenir le nombre de parties relativement faible. C'est pourquoi il est appelé `MergeTree`. Bien sûr, la fusion conduit à “write amplification”. Toutes les parties sont immuables: elles sont seulement créées et supprimées, mais pas modifiées. Lorsque SELECT est exécuté, il contient un instantané de la table (un ensemble de parties). Après la Fusion, nous conservons également les anciennes pièces pendant un certain temps pour faciliter une récupération après une défaillance, donc si nous voyons qu'une partie fusionnée est probablement cassée, nous pouvons la remplacer par ses parties sources. -`MergeTree` n’est pas un arbre LSM car il ne contient pas “memtable” et “log”: inserted data is written directly to the filesystem. This makes it suitable only to INSERT data in batches, not by individual row and not very frequently – about once per second is ok, but a thousand times a second is not. We did it this way for simplicity’s sake, and because we are already inserting data in batches in our applications. +`MergeTree` n'est pas un arbre LSM car il ne contient pas “memtable” et “log”: inserted data is written directly to the filesystem. This makes it suitable only to INSERT data in batches, not by individual row and not very frequently – about once per second is ok, but a thousand times a second is not. We did it this way for simplicity's sake, and because we are already inserting data in batches in our applications. -> Les tables MergeTree ne peuvent avoir qu’un seul index (primaire): il n’y a pas d’index secondaires. Il serait bon d’autoriser plusieurs représentations physiques sous une table logique, par exemple, pour stocker des données dans plus d’un ordre physique ou même pour autoriser des représentations avec des données pré-agrégées avec des données originales. +> Les tables MergeTree ne peuvent avoir qu'un seul index (primaire): il n'y a pas d'index secondaires. Il serait bon d'autoriser plusieurs représentations physiques sous une table logique, par exemple, pour stocker des données dans plus d'un ordre physique ou même pour autoriser des représentations avec des données pré-agrégées avec des données originales. -Il existe des moteurs MergeTree qui effectuent un travail supplémentaire lors des fusions en arrière-plan. Les exemples sont `CollapsingMergeTree` et `AggregatingMergeTree`. Cela pourrait être traité comme un support spécial pour les mises à jour. Gardez à l’esprit que ce ne sont pas de vraies mises à jour car les utilisateurs n’ont généralement aucun contrôle sur le moment où les fusions en arrière-plan sont exécutées et les données dans un `MergeTree` la table est presque toujours stockée dans plus d’une partie, pas sous une forme complètement fusionnée. +Il existe des moteurs MergeTree qui effectuent un travail supplémentaire lors des fusions en arrière-plan. Les exemples sont `CollapsingMergeTree` et `AggregatingMergeTree`. Cela pourrait être traité comme un support spécial pour les mises à jour. Gardez à l'esprit que ce ne sont pas de vraies mises à jour car les utilisateurs n'ont généralement aucun contrôle sur le moment où les fusions en arrière-plan sont exécutées et les données dans un `MergeTree` la table est presque toujours stockée dans plus d'une partie, pas sous une forme complètement fusionnée. ## Réplication {#replication} La réplication dans ClickHouse peut être configurée sur une base par table. Vous pouvez avoir des tables répliquées et des tables non répliquées sur le même serveur. Vous pouvez également avoir des tables répliquées de différentes manières, comme une table avec une réplication à deux facteurs et une autre avec trois facteurs. -La réplication est implémentée dans le `ReplicatedMergeTree` moteur de stockage. Le chemin d’accès dans `ZooKeeper` est spécifié comme paramètre pour le moteur de stockage. Toutes les tables avec le même chemin dans `ZooKeeper` devenez des répliques les unes des autres: elles synchronisent leurs données et maintiennent la cohérence. Les répliques peuvent être ajoutées et supprimées dynamiquement simplement en créant ou en supprimant une table. +La réplication est implémentée dans le `ReplicatedMergeTree` moteur de stockage. Le chemin d'accès dans `ZooKeeper` est spécifié comme paramètre pour le moteur de stockage. Toutes les tables avec le même chemin dans `ZooKeeper` devenez des répliques les unes des autres: elles synchronisent leurs données et maintiennent la cohérence. Les répliques peuvent être ajoutées et supprimées dynamiquement simplement en créant ou en supprimant une table. -La réplication utilise un schéma multi-maître asynchrone. Vous pouvez insérer des données dans n’importe quel réplica qui a une session avec `ZooKeeper`, et les données sont répliquées à toutes les autres répliques de manière asynchrone. Parce que ClickHouse ne prend pas en charge les mises à jour, la réplication est sans conflit. Comme il n’y a pas d’accusé de réception de quorum des insertions, les données juste insérées peuvent être perdues si un nœud échoue. +La réplication utilise un schéma multi-maître asynchrone. Vous pouvez insérer des données dans n'importe quel réplica qui a une session avec `ZooKeeper`, et les données sont répliquées à toutes les autres répliques de manière asynchrone. Parce que ClickHouse ne prend pas en charge les mises à jour, la réplication est sans conflit. Comme il n'y a pas d'accusé de réception de quorum des insertions, les données juste insérées peuvent être perdues si un nœud échoue. -Les métadonnées pour la réplication sont stockées dans ZooKeeper. Il existe un journal de réplication qui répertorie les actions à effectuer. Les Actions sont: obtenir une partie; fusionner des parties; déposer une partition, et ainsi de suite. Chaque réplica copie le journal de réplication dans sa file d’attente, puis exécute les actions de la file d’attente. Par exemple, sur l’insertion, l’ “get the part” l’action est créée dans le journal, et chaque réplique téléchargements de la partie. Les fusions sont coordonnées entre les répliques pour obtenir des résultats identiques aux octets. Toutes les parties sont fusionnées de la même manière sur toutes les répliques. Il est réalisé en élisant une réplique en tant que leader, et cette réplique initie fusionne et écrit “merge parts” actions dans le journal. +Les métadonnées pour la réplication sont stockées dans ZooKeeper. Il existe un journal de réplication qui répertorie les actions à effectuer. Les Actions sont: obtenir une partie; fusionner des parties; déposer une partition, et ainsi de suite. Chaque réplica copie le journal de réplication dans sa file d'attente, puis exécute les actions de la file d'attente. Par exemple, sur l'insertion, l' “get the part” l'action est créée dans le journal, et chaque réplique téléchargements de la partie. Les fusions sont coordonnées entre les répliques pour obtenir des résultats identiques aux octets. Toutes les parties sont fusionnées de la même manière sur toutes les répliques. Il est réalisé en élisant une réplique en tant que leader, et cette réplique initie fusionne et écrit “merge parts” actions dans le journal. -La réplication est physique: seules les parties compressées sont transférées entre les nœuds, pas les requêtes. Les fusions sont traitées sur chaque réplique indépendamment dans la plupart des cas pour réduire les coûts du réseau en évitant l’amplification du réseau. Grand fusionné les pièces sont envoyées sur le réseau uniquement en cas de retard de réplication. +La réplication est physique: seules les parties compressées sont transférées entre les nœuds, pas les requêtes. Les fusions sont traitées sur chaque réplique indépendamment dans la plupart des cas pour réduire les coûts du réseau en évitant l'amplification du réseau. Grand fusionné les pièces sont envoyées sur le réseau uniquement en cas de retard de réplication. -En outre, chaque réplique stocke son état dans ZooKeeper comme l’ensemble des pièces et ses sommes de contrôle. Lorsque l’état sur le système de fichiers local diverge de l’état de référence dans ZooKeeper, le réplica restaure sa cohérence en téléchargeant les parties manquantes et brisées à partir d’autres réplicas. Lorsqu’il y a des données inattendues ou brisées dans le système de fichiers local, ClickHouse ne les supprime pas, mais les déplace dans un répertoire séparé et les oublie. +En outre, chaque réplique stocke son état dans ZooKeeper comme l'ensemble des pièces et ses sommes de contrôle. Lorsque l'état sur le système de fichiers local diverge de l'état de référence dans ZooKeeper, le réplica restaure sa cohérence en téléchargeant les parties manquantes et brisées à partir d'autres réplicas. Lorsqu'il y a des données inattendues ou brisées dans le système de fichiers local, ClickHouse ne les supprime pas, mais les déplace dans un répertoire séparé et les oublie. !!! note "Note" - Le cluster ClickHouse est constitué de fragments indépendants, et chaque fragment est constitué de répliques. Le cluster est **pas élastique**, donc, après avoir ajouté un nouveau fragment, les données ne sont pas rééquilibrées automatiquement entre les fragments. Au lieu de cela, la charge du cluster est censée être ajustée pour être inégale. Cette implémentation vous donne plus de contrôle, et c’est ok pour des clusters relativement petits, tels que des dizaines de nœuds. Mais pour les clusters avec des centaines de nœuds que nous utilisons en production, cette approche devient un inconvénient important. Nous devrions implémenter un moteur de table qui s’étend sur le cluster avec des régions répliquées dynamiquement qui pourraient être divisées et équilibrées automatiquement entre les clusters. + Le cluster ClickHouse est constitué de fragments indépendants, et chaque fragment est constitué de répliques. Le cluster est **pas élastique**, donc, après avoir ajouté un nouveau fragment, les données ne sont pas rééquilibrées automatiquement entre les fragments. Au lieu de cela, la charge du cluster est censée être ajustée pour être inégale. Cette implémentation vous donne plus de contrôle, et c'est ok pour des clusters relativement petits, tels que des dizaines de nœuds. Mais pour les clusters avec des centaines de nœuds que nous utilisons en production, cette approche devient un inconvénient important. Nous devrions implémenter un moteur de table qui s'étend sur le cluster avec des régions répliquées dynamiquement qui pourraient être divisées et équilibrées automatiquement entre les clusters. {## [Article Original](https://clickhouse.tech/docs/en/development/architecture/) ##} diff --git a/docs/fr/development/browse-code.md b/docs/fr/development/browse-code.md index e26a621b156..ef8beb600c6 100644 --- a/docs/fr/development/browse-code.md +++ b/docs/fr/development/browse-code.md @@ -1,14 +1,14 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 63 -toc_title: Parcourir Le Code Source De ClickHouse +toc_title: Parcourir Le Code Source --- # Parcourir Le Code Source De ClickHouse {#browse-clickhouse-source-code} -Vous pouvez utiliser **Woboq** navigateur de code en ligne disponible [ici](https://clickhouse.tech/codebrowser/html_report///ClickHouse/dbms/index.html). Il fournit la navigation de code et la mise en évidence sémantique, la recherche et l’indexation. L’instantané de code est mis à jour quotidiennement. +Vous pouvez utiliser **Woboq** navigateur de code en ligne disponible [ici](https://clickhouse.tech/codebrowser/html_report/ClickHouse/src/index.html). Il fournit la navigation de code et la mise en évidence sémantique, la recherche et l'indexation. L'instantané de code est mis à jour quotidiennement. -Aussi, vous pouvez parcourir les sources sur [GitHub](https://github.com/ClickHouse/ClickHouse) comme à l’habitude. +Aussi, vous pouvez parcourir les sources sur [GitHub](https://github.com/ClickHouse/ClickHouse) comme à l'habitude. -Si vous êtes intéressé par L’IDE à utiliser, nous vous recommandons CLion, Qt Creator, VS Code et KDevelop (avec des mises en garde). Vous pouvez utiliser N’importe quel IDE préféré. Vim et Emacs comptent également. +Si vous êtes intéressé par L'IDE à utiliser, nous vous recommandons CLion, Qt Creator, VS Code et KDevelop (avec des mises en garde). Vous pouvez utiliser N'importe quel IDE préféré. Vim et Emacs comptent également. diff --git a/docs/fr/development/build-cross-arm.md b/docs/fr/development/build-cross-arm.md index 668a5f03376..472862eabe5 100644 --- a/docs/fr/development/build-cross-arm.md +++ b/docs/fr/development/build-cross-arm.md @@ -1,15 +1,15 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 67 toc_title: Comment Construire ClickHouse sur Linux pour AARCH64 (ARM64) --- -# Comment Construire ClickHouse Sur Linux Pour l’architecture AARCH64 (ARM64) {#how-to-build-clickhouse-on-linux-for-aarch64-arm64-architecture} +# Comment Construire ClickHouse sur Linux pour L'Architecture AARCH64 (ARM64) {#how-to-build-clickhouse-on-linux-for-aarch64-arm64-architecture} -C’est pour le cas où vous avez machine Linux et que vous voulez utiliser pour construire `clickhouse` binaire qui fonctionnera sur une autre machine Linux avec une architecture CPU AARCH64. Ceci est destiné aux contrôles d’intégration continus qui s’exécutent sur des serveurs Linux. +C'est pour le cas où vous avez machine Linux et que vous voulez utiliser pour construire `clickhouse` binaire qui fonctionnera sur une autre machine Linux avec une architecture CPU AARCH64. Ceci est destiné aux contrôles d'intégration continus qui s'exécutent sur des serveurs Linux. -La construction croisée pour AARCH64 est basée sur [Instructions de construction](build.md), suivez d’abord. +La construction croisée pour AARCH64 est basée sur [Instructions de construction](build.md), suivez d'abord. # Installer Clang-8 {#install-clang-8} @@ -22,7 +22,7 @@ sudo apt-get update sudo apt-get install clang-8 ``` -# Installer Un Ensemble D’Outils De Compilation Croisée {#install-cross-compilation-toolset} +# Installer Un Ensemble D'Outils De Compilation Croisée {#install-cross-compilation-toolset} ``` bash cd ClickHouse @@ -40,4 +40,4 @@ CC=clang-8 CXX=clang++-8 cmake . -Bbuild-arm64 -DCMAKE_TOOLCHAIN_FILE=cmake/linu ninja -C build-arm64 ``` -Le binaire résultant s’exécutera uniquement sur Linux avec l’architecture CPU AARCH64. +Le binaire résultant s'exécutera uniquement sur Linux avec l'architecture CPU AARCH64. diff --git a/docs/fr/development/build-cross-osx.md b/docs/fr/development/build-cross-osx.md index 847c397c261..4fa97a65c7b 100644 --- a/docs/fr/development/build-cross-osx.md +++ b/docs/fr/development/build-cross-osx.md @@ -1,15 +1,15 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 66 toc_title: Comment Construire ClickHouse sur Linux pour Mac OS X --- -# Comment Construire ClickHouse Sur Linux Pour Mac OS X {#how-to-build-clickhouse-on-linux-for-mac-os-x} +# Comment Construire ClickHouse sur Linux pour Mac OS X {#how-to-build-clickhouse-on-linux-for-mac-os-x} -C’est pour le cas où vous avez machine Linux et que vous voulez utiliser pour construire `clickhouse` binaire qui s’exécutera sur OS X. Ceci est destiné aux contrôles d’intégration continus qui s’exécutent sur des serveurs Linux. Si vous voulez construire ClickHouse directement sur Mac OS X, puis procéder à [une autre instruction](build-osx.md). +C'est pour le cas où vous avez machine Linux et que vous voulez utiliser pour construire `clickhouse` binaire qui s'exécutera sur OS X. Ceci est destiné aux contrôles d'intégration continus qui s'exécutent sur des serveurs Linux. Si vous voulez construire ClickHouse directement sur Mac OS X, puis procéder à [une autre instruction](build-osx.md). -Le cross-build pour Mac OS X est basé sur le [Instructions de construction](build.md), suivez d’abord. +Le cross-build pour Mac OS X est basé sur le [Instructions de construction](build.md), suivez d'abord. # Installer Clang-8 {#install-clang-8} @@ -21,7 +21,7 @@ sudo echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic-8 sudo apt-get install clang-8 ``` -# Installer Un Ensemble D’Outils De Compilation Croisée {#install-cross-compilation-toolset} +# Installer Un Ensemble D'Outils De Compilation Croisée {#install-cross-compilation-toolset} Souvenons nous du chemin où nous installons `cctools` comme ${CCTOOLS} @@ -40,7 +40,7 @@ cd cctools-port/cctools make install ``` -En outre, nous devons télécharger macOS X SDK dans l’arbre de travail. +En outre, nous devons télécharger macOS X SDK dans l'arbre de travail. ``` bash cd ClickHouse diff --git a/docs/fr/development/build-osx.md b/docs/fr/development/build-osx.md index b8a76195b4b..677e257a637 100644 --- a/docs/fr/development/build-osx.md +++ b/docs/fr/development/build-osx.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 65 toc_title: Comment Construire ClickHouse sur Mac OS X --- -# Comment Construire ClickHouse Sur Mac OS X {#how-to-build-clickhouse-on-mac-os-x} +# Comment Construire ClickHouse sur Mac OS X {#how-to-build-clickhouse-on-mac-os-x} Build devrait fonctionner sur Mac OS X 10.15 (Catalina) @@ -15,7 +15,7 @@ Build devrait fonctionner sur Mac OS X 10.15 (Catalina) $ /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" ``` -## Installez Les Compilateurs, Outils Et bibliothèques Requis {#install-required-compilers-tools-and-libraries} +## Installez les compilateurs, outils et bibliothèques requis {#install-required-compilers-tools-and-libraries} ``` bash $ brew install cmake ninja libtool gettext @@ -45,12 +45,12 @@ $ ninja $ cd .. ``` -## Mises En Garde {#caveats} +## Mises en garde {#caveats} -Si vous avez l’intention d’exécuter clickhouse-server, assurez-vous d’augmenter la variable maxfiles du système. +Si vous avez l'intention d'exécuter clickhouse-server, assurez-vous d'augmenter la variable maxfiles du système. !!! info "Note" - Vous aurez besoin d’utiliser sudo. + Vous aurez besoin d'utiliser sudo. Pour ce faire, créez le fichier suivant: diff --git a/docs/fr/development/build.md b/docs/fr/development/build.md index f55b5451632..a57e0f20337 100644 --- a/docs/fr/development/build.md +++ b/docs/fr/development/build.md @@ -1,17 +1,17 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 64 toc_title: Comment Construire ClickHouse sur Linux --- -# Comment Construire ClickHouse Pour Le développement {#how-to-build-clickhouse-for-development} +# Comment Construire ClickHouse pour le développement {#how-to-build-clickhouse-for-development} Le tutoriel suivant est basé sur le système Linux Ubuntu. Avec les modifications appropriées, il devrait également fonctionner sur toute autre distribution Linux. Plates-formes prises en charge: x86\_64 et AArch64. La prise en charge de Power9 est expérimentale. -## Installez Git, CMake, Python Et Ninja {#install-git-cmake-python-and-ninja} +## Installez Git, CMake, Python et Ninja {#install-git-cmake-python-and-ninja} ``` bash $ sudo apt-get install git cmake python ninja-build @@ -23,7 +23,7 @@ Ou cmake3 au lieu de cmake sur les systèmes plus anciens. Il y a plusieurs façons de le faire. -### Installer à Partir d’un Paquet PPA {#install-from-a-ppa-package} +### Installer à partir d'un paquet PPA {#install-from-a-ppa-package} ``` bash $ sudo apt-get install software-properties-common @@ -32,11 +32,11 @@ $ sudo apt-get update $ sudo apt-get install gcc-9 g++-9 ``` -### Installer à Partir De Sources {#install-from-sources} +### Installer à partir de Sources {#install-from-sources} Regarder [utils/ci/build-gcc-from-sources.sh](https://github.com/ClickHouse/ClickHouse/blob/master/utils/ci/build-gcc-from-sources.sh) -## Utilisez GCC 9 Pour Les Builds {#use-gcc-9-for-builds} +## Utilisez GCC 9 pour les Builds {#use-gcc-9-for-builds} ``` bash $ export CC=gcc-9 @@ -67,13 +67,13 @@ $ cd .. ``` Pour créer un exécutable, exécutez `ninja clickhouse`. -Cela va créer de l’ `programs/clickhouse` exécutable, qui peut être utilisé avec `client` ou `server` argument. +Cela va créer de l' `programs/clickhouse` exécutable, qui peut être utilisé avec `client` ou `server` argument. -# Comment Construire ClickHouse Sur N’importe Quel Linux {#how-to-build-clickhouse-on-any-linux} +# Comment Construire ClickHouse sur N'importe quel Linux {#how-to-build-clickhouse-on-any-linux} La construction nécessite les composants suivants: -- Git (est utilisé uniquement pour extraire les sources, ce n’est pas nécessaire pour la construction) +- Git (est utilisé uniquement pour extraire les sources, ce n'est pas nécessaire pour la construction) - CMake 3.10 ou plus récent - Ninja (recommandé) ou faire - Compilateur C++: gcc 9 ou clang 8 ou plus récent @@ -108,17 +108,17 @@ Exemple Pour Fedora Rawhide: cmake ../ClickHouse make -j $(nproc) -# Vous N’avez Pas à Construire ClickHouse {#you-dont-have-to-build-clickhouse} +# Vous N'avez pas à construire ClickHouse {#you-dont-have-to-build-clickhouse} -ClickHouse est disponible dans des binaires et des paquets pré-construits. Les binaires sont portables et peuvent être exécutés sur N’importe quelle saveur Linux. +ClickHouse est disponible dans des binaires et des paquets pré-construits. Les binaires sont portables et peuvent être exécutés sur N'importe quelle saveur Linux. -Ils sont conçus pour les versions stables, préconfigurables et de test aussi longtemps que pour chaque commit à master et pour chaque requête d’extraction. +Ils sont conçus pour les versions stables, préconfigurables et de test aussi longtemps que pour chaque commit à master et pour chaque requête d'extraction. Pour trouver la construction la plus fraîche de `master`, aller à [page commits](https://github.com/ClickHouse/ClickHouse/commits/master), cliquez sur la première coche verte ou Croix Rouge près de commit, et cliquez sur le “Details” lien à droite après “ClickHouse Build Check”. -# Comment Construire Le Paquet ClickHouse Debian {#how-to-build-clickhouse-debian-package} +# Comment construire le paquet ClickHouse Debian {#how-to-build-clickhouse-debian-package} -## Installer Git Et Pbuilder {#install-git-and-pbuilder} +## Installer Git et Pbuilder {#install-git-and-pbuilder} ``` bash $ sudo apt-get update diff --git a/docs/fr/development/contrib.md b/docs/fr/development/contrib.md index 9750b348541..52eac7a71ac 100644 --- a/docs/fr/development/contrib.md +++ b/docs/fr/development/contrib.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 70 toc_title: "Biblioth\xE8ques Tierces Utilis\xE9es" --- diff --git a/docs/fr/development/developer-instruction.md b/docs/fr/development/developer-instruction.md index ef9a965e0f5..414cfc1d339 100644 --- a/docs/fr/development/developer-instruction.md +++ b/docs/fr/development/developer-instruction.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 61 toc_title: "Le D\xE9butant Clickhouse Developer Instruction" --- @@ -9,21 +9,21 @@ La construction de ClickHouse est prise en charge sous Linux, FreeBSD et Mac OS # Si Vous Utilisez Windows {#if-you-use-windows} -Si vous utilisez Windows, vous devez créer une machine virtuelle avec Ubuntu. Pour commencer à travailler avec une machine virtuelle, installez VirtualBox. Vous pouvez télécharger Ubuntu sur le site: https://www.ubuntu.com/\#download. veuillez créer une machine virtuelle à partir de l’image téléchargée (vous devez réserver au moins 4 Go de RAM pour cela). Pour exécuter un terminal de ligne de commande dans Ubuntu, recherchez un programme contenant le mot “terminal” dans son nom (gnome-terminal, konsole etc.) ou appuyez simplement sur Ctrl + Alt + T. +Si vous utilisez Windows, vous devez créer une machine virtuelle avec Ubuntu. Pour commencer à travailler avec une machine virtuelle, installez VirtualBox. Vous pouvez télécharger Ubuntu sur le site: https://www.ubuntu.com/\#download. veuillez créer une machine virtuelle à partir de l'image téléchargée (vous devez réserver au moins 4 Go de RAM pour cela). Pour exécuter un terminal de ligne de commande dans Ubuntu, recherchez un programme contenant le mot “terminal” dans son nom (gnome-terminal, konsole etc.) ou appuyez simplement sur Ctrl + Alt + T. -# Si Vous Utilisez Un système 32 Bits {#if-you-use-a-32-bit-system} +# Si vous utilisez un système 32 bits {#if-you-use-a-32-bit-system} -ClickHouse ne peut pas fonctionner ou construire sur un système 32 bits. Vous devez acquérir l’accès à un système 64 bits et vous pouvez continuer la lecture. +ClickHouse ne peut pas fonctionner ou construire sur un système 32 bits. Vous devez acquérir l'accès à un système 64 bits et vous pouvez continuer la lecture. -# Création d’un référentiel Sur GitHub {#creating-a-repository-on-github} +# Création d'un référentiel sur GitHub {#creating-a-repository-on-github} -Pour commencer à travailler avec clickhouse repository, vous aurez besoin d’un compte GitHub. +Pour commencer à travailler avec clickhouse repository, vous aurez besoin d'un compte GitHub. -Vous en avez probablement déjà un, mais si vous ne le faites pas, veuillez vous inscrire à https://github.com. dans le cas où vous n’avez pas de clés SSH, vous devez les générer, puis les télécharger sur GitHub. Il est nécessaire pour l’envoi de vos correctifs. Il est également possible d’utiliser les mêmes clés SSH que vous utilisez avec d’autres serveurs SSH - vous les avez probablement déjà. +Vous en avez probablement déjà un, mais si vous ne le faites pas, veuillez vous inscrire à https://github.com. dans le cas où vous n'avez pas de clés SSH, vous devez les générer, puis les télécharger sur GitHub. Il est nécessaire pour l'envoi de vos correctifs. Il est également possible d'utiliser les mêmes clés SSH que vous utilisez avec d'autres serveurs SSH - vous les avez probablement déjà. -Créer un fork de clickhouse repository. Pour ce faire, cliquez sur l’ “fork” bouton dans le coin supérieur droit à https://github.com/ClickHouse/ClickHouse. il fourche votre propre copie de ClickHouse / ClickHouse à votre compte. +Créer un fork de clickhouse repository. Pour ce faire, cliquez sur l' “fork” bouton dans le coin supérieur droit à https://github.com/ClickHouse/ClickHouse. il fourche votre propre copie de ClickHouse / ClickHouse à votre compte. -Le processus de développement consiste d’abord à valider les modifications prévues dans votre fork de ClickHouse, puis à créer un “pull request” pour que ces modifications soient acceptées dans le référentiel principal (ClickHouse/ClickHouse). +Le processus de développement consiste d'abord à valider les modifications prévues dans votre fork de ClickHouse, puis à créer un “pull request” pour que ces modifications soient acceptées dans le référentiel principal (ClickHouse/ClickHouse). Pour travailler avec les dépôts git, veuillez installer `git`. @@ -32,32 +32,32 @@ Pour ce faire dans Ubuntu vous exécutez dans le terminal de ligne de commande: sudo apt update sudo apt install git -Un bref manuel sur l’utilisation de Git peut être trouvé ici: https://services.github.com/on-demand/downloads/github-git-cheat-sheet.pdf. +Un bref manuel sur l'utilisation de Git peut être trouvé ici: https://services.github.com/on-demand/downloads/github-git-cheat-sheet.pdf. Pour un manuel détaillé sur Git voir https://git-scm.com/book/en/v2. -# Clonage d’un référentiel Sur Votre Machine De développement {#cloning-a-repository-to-your-development-machine} +# Clonage D'un référentiel sur votre machine de développement {#cloning-a-repository-to-your-development-machine} -Ensuite, vous devez télécharger les fichiers source sur votre machine de travail. Ceci est appelé “to clone a repository” parce qu’il crée une copie locale du dépôt sur votre machine de travail. +Ensuite, vous devez télécharger les fichiers source sur votre machine de travail. Ceci est appelé “to clone a repository” parce qu'il crée une copie locale du dépôt sur votre machine de travail. Dans le terminal de ligne de commande exécuter: - git clone --recursive git@guthub.com:your_github_username/ClickHouse.git + git clone --recursive git@github.com:your_github_username/ClickHouse.git cd ClickHouse Remarque: Veuillez remplacer *your\_github\_username* avec ce qui est approprié! Cette commande va créer un répertoire `ClickHouse` contenant la copie de travail du projet. -Il est important que le chemin d’accès au répertoire de travail ne contienne aucun espace, car cela peut entraîner des problèmes lors de l’exécution du système de construction. +Il est important que le chemin d'accès au répertoire de travail ne contienne aucun espace, car cela peut entraîner des problèmes lors de l'exécution du système de construction. -Veuillez noter que clickhouse repository utilise `submodules`. That is what the references to additional repositories are called (i.e. external libraries on which the project depends). It means that when cloning the repository you need to specify the `--recursive` drapeau comme dans l’exemple ci-dessus. Si le dépôt a été cloné sans submodules, pour télécharger, vous devez exécuter les opérations suivantes: +Veuillez noter que clickhouse repository utilise `submodules`. That is what the references to additional repositories are called (i.e. external libraries on which the project depends). It means that when cloning the repository you need to specify the `--recursive` drapeau comme dans l'exemple ci-dessus. Si le dépôt a été cloné sans submodules, pour télécharger, vous devez exécuter les opérations suivantes: git submodule init git submodule update -Vous pouvez vérifier l’état avec la commande: `git submodule status`. +Vous pouvez vérifier l'état avec la commande: `git submodule status`. -Si vous obtenez le message d’erreur suivantes: +Si vous obtenez le message d'erreur suivantes: Permission denied (publickey). fatal: Could not read from remote repository. @@ -65,21 +65,21 @@ Si vous obtenez le message d’erreur suivantes: Please make sure you have the correct access rights and the repository exists. -Cela signifie généralement que les clés SSH pour la connexion à GitHub sont manquantes. Ces clés sont normalement situés dans `~/.ssh`. Pour que les clés SSH soient acceptées, vous devez les télécharger dans la section Paramètres de L’interface utilisateur GitHub. +Cela signifie généralement que les clés SSH pour la connexion à GitHub sont manquantes. Ces clés sont normalement situés dans `~/.ssh`. Pour que les clés SSH soient acceptées, vous devez les télécharger dans la section Paramètres de L'interface utilisateur GitHub. Vous pouvez également cloner le référentiel via le protocole https: git clone https://github.com/ClickHouse/ClickHouse.git -Ceci, cependant, ne vous permettra pas d’envoyer vos modifications sur le serveur. Vous pouvez toujours l’utiliser temporairement et ajouter les clés SSH plus tard en remplaçant l’adresse distante du référentiel par `git remote` commande. +Ceci, cependant, ne vous permettra pas d'envoyer vos modifications sur le serveur. Vous pouvez toujours l'utiliser temporairement et ajouter les clés SSH plus tard en remplaçant l'adresse distante du référentiel par `git remote` commande. -Vous pouvez également ajouter l’adresse du dépôt clickhouse original à votre référentiel local pour extraire les mises à jour à partir de là: +Vous pouvez également ajouter l'adresse du dépôt clickhouse original à votre référentiel local pour extraire les mises à jour à partir de là: git remote add upstream git@github.com:ClickHouse/ClickHouse.git Après avoir exécuté avec succès cette commande vous serez en mesure de tirer les mises à jour du repo clickhouse principal en exécutant `git pull upstream master`. -## Travailler Avec Des Submodules {#working-with-submodules} +## Travailler avec des Submodules {#working-with-submodules} Travailler avec des sous-modules dans git pourrait être douloureux. Prochaines commandes aidera à gérer: @@ -93,7 +93,7 @@ Travailler avec des sous-modules dans git pourrait être douloureux. Prochaines # Two last commands could be merged together git submodule update --init -Les commandes suivantes vous aideront à réinitialiser tous les sous-modules à l’état initial (!AVERTISSEMENT! - tout changement à l’intérieur sera supprimé): +Les commandes suivantes vous aideront à réinitialiser tous les sous-modules à l'état initial (!AVERTISSEMENT! - tout changement à l'intérieur sera supprimé): # Synchronizes submodules' remote URL with .gitmodules git submodule sync --recursive @@ -122,7 +122,7 @@ Sur CentOS, RedHat run `sudo yum install cmake ninja-build`. Si vous utilisez Arch ou Gentoo, vous savez probablement vous - même comment installer CMake. -Pour installer CMake et Ninja sur Mac OS X installez D’abord Homebrew puis installez tout le reste via brew: +Pour installer CMake et Ninja sur Mac OS X installez D'abord Homebrew puis installez tout le reste via brew: /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" brew install cmake ninja @@ -131,32 +131,32 @@ Ensuite, vérifiez la version de CMake: `cmake --version`. Si elle est inférieu # Bibliothèques Externes Facultatives {#optional-external-libraries} -ClickHouse utilise plusieurs bibliothèques externes pour la construction. Tous n’ont pas besoin d’être installés séparément car ils sont construits avec ClickHouse à partir des sources situées dans les sous-modules. Vous pouvez vérifier la liste dans `contrib`. +ClickHouse utilise plusieurs bibliothèques externes pour la construction. Tous n'ont pas besoin d'être installés séparément car ils sont construits avec ClickHouse à partir des sources situées dans les sous-modules. Vous pouvez vérifier la liste dans `contrib`. -# Compilateur C++ {#c-compiler} +# Compilateur C++ {#c-compiler} Les compilateurs GCC à partir de la version 9 et Clang version 8 ou supérieure sont pris en charge pour construire ClickHouse. -Les builds officiels de Yandex utilisent actuellement GCC car ils génèrent du code machine de performances légèrement meilleures (ce qui donne une différence allant jusqu’à plusieurs pour cent selon nos benchmarks). Et Clang est plus pratique pour le développement habituellement. Cependant, notre plate-forme d’intégration continue (CI) vérifie environ une douzaine de combinaisons de construction. +Les builds officiels de Yandex utilisent actuellement GCC car ils génèrent du code machine de performances légèrement meilleures (ce qui donne une différence allant jusqu'à plusieurs pour cent selon nos benchmarks). Et Clang est plus pratique pour le développement habituellement. Cependant, notre plate-forme d'intégration continue (CI) vérifie environ une douzaine de combinaisons de construction. Pour installer GCC sur Ubuntu Exécutez: `sudo apt install gcc g++` Vérifiez la version de gcc: `gcc --version`. Si elle est inférieure à 9, suivez les instructions ici: https://clickhouse.tech/docs/fr/développement/construction/\#install-gcc-9. -Mac OS X build est pris en charge uniquement pour Clang. Il suffit d’exécuter `brew install llvm` +Mac OS X build est pris en charge uniquement pour Clang. Il suffit d'exécuter `brew install llvm` -Si vous décidez d’utiliser Clang, vous pouvez également installer `libc++` et `lld` si vous savez ce que c’est. Utiliser `ccache` est également recommandé. +Si vous décidez d'utiliser Clang, vous pouvez également installer `libc++` et `lld` si vous savez ce que c'est. Utiliser `ccache` est également recommandé. # Le Processus De Construction {#the-building-process} -Maintenant que vous êtes prêt à construire ClickHouse nous vous conseillons de créer un répertoire séparé `build` à l’intérieur de `ClickHouse` qui contiendra tous les artefacts de construction: +Maintenant que vous êtes prêt à construire ClickHouse nous vous conseillons de créer un répertoire séparé `build` à l'intérieur de `ClickHouse` qui contiendra tous les artefacts de construction: mkdir build cd build Vous pouvez avoir plusieurs répertoires différents (build\_release, build\_debug, etc.) pour les différents types de construction. -Tandis qu’à l’intérieur de la `build` répertoire, configurez votre build en exécutant CMake. Avant la première exécution, vous devez définir des variables d’environnement qui spécifient le compilateur (compilateur gcc version 9 dans cet exemple). +Tandis qu'à l'intérieur de la `build` répertoire, configurez votre build en exécutant CMake. Avant la première exécution, vous devez définir des variables d'environnement qui spécifient le compilateur (compilateur gcc version 9 dans cet exemple). Linux: @@ -170,7 +170,7 @@ Mac OS X: Le `CC` variable spécifie le compilateur pour C (abréviation de compilateur C), et `CXX` variable indique quel compilateur C++ doit être utilisé pour la construction. -Pour une version plus rapide, vous pouvez recourir à l’ `debug` build type - Une construction sans optimisations. Pour cela fournissez le paramètre suivant `-D CMAKE_BUILD_TYPE=Debug`: +Pour une version plus rapide, vous pouvez recourir à l' `debug` build type - Une construction sans optimisations. Pour cela fournissez le paramètre suivant `-D CMAKE_BUILD_TYPE=Debug`: cmake -D CMAKE_BUILD_TYPE=Debug .. @@ -186,35 +186,35 @@ Si vous avez besoin de construire tous les binaires (utilitaires et tests), vous ninja -La construction complète nécessite environ 30 Go d’espace disque libre ou 15 Go pour construire les binaires principaux. +La construction complète nécessite environ 30 Go d'espace disque libre ou 15 Go pour construire les binaires principaux. -Lorsqu’une grande quantité de RAM est disponible sur la machine de construction vous devez limiter le nombre de tâches de construction exécutées en parallèle avec `-j` param: +Lorsqu'une grande quantité de RAM est disponible sur la machine de construction vous devez limiter le nombre de tâches de construction exécutées en parallèle avec `-j` param: ninja -j 1 clickhouse-server clickhouse-client Sur les machines avec 4 Go de RAM, il est recommandé de spécifier 1, pour 8 Go de RAM `-j 2` est recommandé. -Si vous recevez le message: `ninja: error: loading 'build.ninja': No such file or directory`, cela signifie que la génération d’une configuration de construction a échoué et que vous devez inspecter le message ci-dessus. +Si vous recevez le message: `ninja: error: loading 'build.ninja': No such file or directory`, cela signifie que la génération d'une configuration de construction a échoué et que vous devez inspecter le message ci-dessus. Après le démarrage réussi du processus de construction, vous verrez la progression de la construction - le nombre de tâches traitées et le nombre total de tâches. -Lors de la construction de messages sur les fichiers protobuf dans la bibliothèque libhdfs2 comme `libprotobuf WARNING` peuvent apparaître. Ils touchent rien et sont sûrs d’être ignoré. +Lors de la construction de messages sur les fichiers protobuf dans la bibliothèque libhdfs2 comme `libprotobuf WARNING` peuvent apparaître. Ils touchent rien et sont sûrs d'être ignoré. Lors de la construction, vous obtenez un fichier exécutable `ClickHouse//programs/clickhouse`: ls -l programs/clickhouse -# Exécution De L’exécutable Construit De ClickHouse {#running-the-built-executable-of-clickhouse} +# Exécution de L'exécutable construit de ClickHouse {#running-the-built-executable-of-clickhouse} -Pour exécuter le serveur sous l’utilisateur actuel vous devez naviguer vers `ClickHouse/programs/server/` (situé à l’extérieur de `build` et les exécuter: +Pour exécuter le serveur sous l'utilisateur actuel vous devez naviguer vers `ClickHouse/programs/server/` (situé à l'extérieur de `build` et les exécuter: - ../../../build/programs/clickhouse server + ../../build/programs/clickhouse server -Dans ce cas, ClickHouse utilisera les fichiers de configuration situés dans le répertoire courant. Vous pouvez l’exécuter `clickhouse server` depuis n’importe quel répertoire spécifiant le chemin d’accès à un fichier de configuration en tant que paramètre de ligne de commande `--config-file`. +Dans ce cas, ClickHouse utilisera les fichiers de configuration situés dans le répertoire courant. Vous pouvez l'exécuter `clickhouse server` depuis n'importe quel répertoire spécifiant le chemin d'accès à un fichier de configuration en tant que paramètre de ligne de commande `--config-file`. Pour vous connecter à ClickHouse avec clickhouse-client dans un autre terminal, accédez à `ClickHouse/build/programs/` et exécuter `clickhouse client`. -Si vous obtenez `Connection refused` message sur Mac OS X ou FreeBSD, essayez de spécifier l’adresse hôte 127.0.0.1: +Si vous obtenez `Connection refused` message sur Mac OS X ou FreeBSD, essayez de spécifier l'adresse hôte 127.0.0.1: clickhouse client --host 127.0.0.1 @@ -224,26 +224,26 @@ Vous pouvez remplacer la version de production de clickhouse binary installée d sudo cp ClickHouse/build/programs/clickhouse /usr/bin/ sudo service clickhouse-server start -Notez que `clickhouse-client`, `clickhouse-server` et d’autres sont des liens symboliques à la commune `clickhouse` binaire. +Notez que `clickhouse-client`, `clickhouse-server` et d'autres sont des liens symboliques à la commune `clickhouse` binaire. Vous pouvez également exécuter votre binaire ClickHouse personnalisé avec le fichier de configuration du package clickhouse installé sur votre système: sudo service clickhouse-server stop sudo -u clickhouse ClickHouse/build/programs/clickhouse server --config-file /etc/clickhouse-server/config.xml -# IDE (environnement De développement intégré) {#ide-integrated-development-environment} +# IDE (environnement de développement intégré) {#ide-integrated-development-environment} -Si vous ne savez pas quel IDE utiliser, nous vous recommandons D’utiliser CLion. CLion est un logiciel commercial, mais il offre une période d’essai gratuite de 30 jours. Il est également gratuit pour les étudiants. CLion peut être utilisé à la fois sur Linux et sur Mac OS X. +Si vous ne savez pas quel IDE utiliser, nous vous recommandons D'utiliser CLion. CLion est un logiciel commercial, mais il offre une période d'essai gratuite de 30 jours. Il est également gratuit pour les étudiants. CLion peut être utilisé à la fois sur Linux et sur Mac OS X. -KDevelop et QTCreator sont d’autres grandes alternatives D’un IDE pour développer ClickHouse. KDevelop est un IDE très pratique bien qu’instable. Si KDevelop se bloque après un certain temps lors de l’ouverture du projet, vous devez cliquer sur “Stop All” bouton dès qu’il a ouvert la liste des fichiers du projet. Après cela, KDevelop devrait être bien pour travailler avec. +KDevelop et QTCreator sont d'autres grandes alternatives D'un IDE pour développer ClickHouse. KDevelop est un IDE très pratique bien qu'instable. Si KDevelop se bloque après un certain temps lors de l'ouverture du projet, vous devez cliquer sur “Stop All” bouton dès qu'il a ouvert la liste des fichiers du projet. Après cela, KDevelop devrait être bien pour travailler avec. -En tant qu’éditeurs de code simples, vous pouvez utiliser Sublime Text ou Visual Studio Code, ou Kate (qui sont tous disponibles sur Linux). +En tant qu'éditeurs de code simples, vous pouvez utiliser Sublime Text ou Visual Studio Code, ou Kate (qui sont tous disponibles sur Linux). -Juste au cas où, il convient de mentionner que CLion crée `build` chemin sur son propre, il aussi sur son propre sélectionne `debug` pour le type de construction, pour la configuration, il utilise une version de CMake définie dans CLion et non celle installée par vous, et enfin, CLion utilisera `make` pour exécuter construire des tâches au lieu de `ninja`. C’est un comportement normal, gardez cela à l’esprit pour éviter toute confusion. +Juste au cas où, il convient de mentionner que CLion crée `build` chemin sur son propre, il aussi sur son propre sélectionne `debug` pour le type de construction, pour la configuration, il utilise une version de CMake définie dans CLion et non celle installée par vous, et enfin, CLion utilisera `make` pour exécuter construire des tâches au lieu de `ninja`. C'est un comportement normal, gardez cela à l'esprit pour éviter toute confusion. -# L’Écriture De Code {#writing-code} +# L'Écriture De Code {#writing-code} -La description de l’architecture ClickHouse peut être trouvée ici: https://clickhouse.tech/docs/fr/développement/architecture/ +La description de l'architecture ClickHouse peut être trouvée ici: https://clickhouse.tech/docs/fr/développement/architecture/ Le code Style Guide: https://clickhouse.tech/docs/fr/développement/style/ @@ -253,7 +253,7 @@ Liste des tâches: https://github.com/ClickHouse/ClickHouse/blob/master/testsruc # Des Données De Test {#test-data} -Le développement de ClickHouse nécessite souvent le chargement d’ensembles de données réalistes. Il est particulièrement important pour les tests de performance. Nous avons un ensemble spécialement préparé de données anonymisées de Yandex.Metrica. Il nécessite en outre quelques 3 Go d’espace disque libre. Notez que ces données ne sont pas requises pour accomplir la plupart des tâches de développement. +Le développement de ClickHouse nécessite souvent le chargement d'ensembles de données réalistes. Il est particulièrement important pour les tests de performance. Nous avons un ensemble spécialement préparé de données anonymisées de Yandex.Metrica. Il nécessite en outre quelques 3 Go d'espace disque libre. Notez que ces données ne sont pas requises pour accomplir la plupart des tâches de développement. sudo apt install wget xz-utils @@ -265,6 +265,8 @@ Le développement de ClickHouse nécessite souvent le chargement d’ensembles d clickhouse-client + CREATE DATABASE IF NOT EXISTS test + CREATE TABLE test.hits ( WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String, RefererDomain String, Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), UTCEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), RemoteIP UInt32, RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming Int32, DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32, FetchTiming Int32, RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32, LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32, RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String, `ParsedParams.Key1` Array(String), `ParsedParams.Key2` Array(String), `ParsedParams.Key3` Array(String), `ParsedParams.Key4` Array(String), `ParsedParams.Key5` Array(String), `ParsedParams.ValueDouble` Array(Float64), IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree PARTITION BY toYYYYMM(EventDate) SAMPLE BY intHash32(UserID) ORDER BY (CounterID, EventDate, intHash32(UserID), EventTime); CREATE TABLE test.visits ( CounterID UInt32, StartDate Date, Sign Int8, IsNew UInt8, VisitID UInt64, UserID UInt64, StartTime DateTime, Duration UInt32, UTCStartTime DateTime, PageViews Int32, Hits Int32, IsBounce UInt8, Referer String, StartURL String, RefererDomain String, StartURLDomain String, EndURL String, LinkURL String, IsDownload UInt8, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, PlaceID Int32, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), IsYandex UInt8, GoalReachesDepth Int32, GoalReachesURL Int32, GoalReachesAny Int32, SocialSourceNetworkID UInt8, SocialSourcePage String, MobilePhoneModel String, ClientEventTime DateTime, RegionID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RemoteIP UInt32, RemoteIP6 FixedString(16), IPNetworkID UInt32, SilverlightVersion3 UInt32, CodeVersion UInt32, ResolutionWidth UInt16, ResolutionHeight UInt16, UserAgentMajor UInt16, UserAgentMinor UInt16, WindowClientWidth UInt16, WindowClientHeight UInt16, SilverlightVersion2 UInt8, SilverlightVersion4 UInt16, FlashVersion3 UInt16, FlashVersion4 UInt16, ClientTimeZone Int16, OS UInt8, UserAgent UInt8, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, NetMajor UInt8, NetMinor UInt8, MobilePhone UInt8, SilverlightVersion1 UInt8, Age UInt8, Sex UInt8, Income UInt8, JavaEnable UInt8, CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, BrowserLanguage UInt16, BrowserCountry UInt16, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), Params Array(String), `Goals.ID` Array(UInt32), `Goals.Serial` Array(UInt32), `Goals.EventTime` Array(DateTime), `Goals.Price` Array(Int64), `Goals.OrderID` Array(String), `Goals.CurrencyID` Array(UInt32), WatchIDs Array(UInt64), ParamSumPrice Int64, ParamCurrency FixedString(3), ParamCurrencyID UInt16, ClickLogID UInt64, ClickEventID Int32, ClickGoodEvent Int32, ClickEventTime DateTime, ClickPriorityID Int32, ClickPhraseID Int32, ClickPageID Int32, ClickPlaceID Int32, ClickTypeID Int32, ClickResourceID Int32, ClickCost UInt32, ClickClientIP UInt32, ClickDomainID UInt32, ClickURL String, ClickAttempt UInt8, ClickOrderID UInt32, ClickBannerID UInt32, ClickMarketCategoryID UInt32, ClickMarketPP UInt32, ClickMarketCategoryName String, ClickMarketPPName String, ClickAWAPSCampaignName String, ClickPageName String, ClickTargetType UInt16, ClickTargetPhraseID UInt64, ClickContextType UInt8, ClickSelectType Int8, ClickOptions String, ClickGroupBannerID Int32, OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, FirstVisit DateTime, PredLastVisit Date, LastVisit Date, TotalVisits UInt32, `TraficSource.ID` Array(Int8), `TraficSource.SearchEngineID` Array(UInt16), `TraficSource.AdvEngineID` Array(UInt8), `TraficSource.PlaceID` Array(UInt16), `TraficSource.SocialSourceNetworkID` Array(UInt8), `TraficSource.Domain` Array(String), `TraficSource.SearchPhrase` Array(String), `TraficSource.SocialSourcePage` Array(String), Attendance FixedString(16), CLID UInt32, YCLID UInt64, NormalizedRefererHash UInt64, SearchPhraseHash UInt64, RefererDomainHash UInt64, NormalizedStartURLHash UInt64, StartURLDomainHash UInt64, NormalizedEndURLHash UInt64, TopLevelDomain UInt64, URLScheme UInt64, OpenstatServiceNameHash UInt64, OpenstatCampaignIDHash UInt64, OpenstatAdIDHash UInt64, OpenstatSourceIDHash UInt64, UTMSourceHash UInt64, UTMMediumHash UInt64, UTMCampaignHash UInt64, UTMContentHash UInt64, UTMTermHash UInt64, FromHash UInt64, WebVisorEnabled UInt8, WebVisorActivity UInt32, `ParsedParams.Key1` Array(String), `ParsedParams.Key2` Array(String), `ParsedParams.Key3` Array(String), `ParsedParams.Key4` Array(String), `ParsedParams.Key5` Array(String), `ParsedParams.ValueDouble` Array(Float64), `Market.Type` Array(UInt8), `Market.GoalID` Array(UInt32), `Market.OrderID` Array(String), `Market.OrderPrice` Array(Int64), `Market.PP` Array(UInt32), `Market.DirectPlaceID` Array(UInt32), `Market.DirectOrderID` Array(UInt32), `Market.DirectBannerID` Array(UInt32), `Market.GoodID` Array(String), `Market.GoodName` Array(String), `Market.GoodQuantity` Array(Int32), `Market.GoodPrice` Array(Int64), IslandID FixedString(16)) ENGINE = CollapsingMergeTree(Sign) PARTITION BY toYYYYMM(StartDate) SAMPLE BY intHash32(UserID) ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID); @@ -274,12 +276,12 @@ Le développement de ClickHouse nécessite souvent le chargement d’ensembles d # La Création De Pull Request {#creating-pull-request} -Accédez à votre référentiel fork dans L’interface utilisateur de GitHub. Si vous avez développé dans une branche, vous devez sélectionner cette branche. Il y aura un “Pull request” bouton situé sur l’écran. En substance, cela signifie “create a request for accepting my changes into the main repository”. +Accédez à votre référentiel fork dans L'interface utilisateur de GitHub. Si vous avez développé dans une branche, vous devez sélectionner cette branche. Il y aura un “Pull request” bouton situé sur l'écran. En substance, cela signifie “create a request for accepting my changes into the main repository”. -Une demande d’extraction peuvent être créés, même si le travail n’est pas encore terminée. Dans ce cas veuillez mettre le mot “WIP” (travaux en cours) au début du titre, il peut être modifié plus tard. Ceci est utile pour l’examen coopératif et la discussion des changements ainsi que pour l’exécution de tous les tests disponibles. Il est important que vous fournissiez une brève description de vos modifications, il sera ensuite utilisé pour générer des journaux de modifications de version. +Une demande d'extraction peuvent être créés, même si le travail n'est pas encore terminée. Dans ce cas veuillez mettre le mot “WIP” (travaux en cours) au début du titre, il peut être modifié plus tard. Ceci est utile pour l'examen coopératif et la discussion des changements ainsi que pour l'exécution de tous les tests disponibles. Il est important que vous fournissiez une brève description de vos modifications, il sera ensuite utilisé pour générer des journaux de modifications de version. Les tests commenceront dès que les employés de Yandex étiqueteront votre PR avec une étiquette “can be tested”. The results of some first checks (e.g. code style) will come in within several minutes. Build check results will arrive within half an hour. And the main set of tests will report itself within an hour. -Le système préparera les builds binaires ClickHouse pour votre demande de tirage individuellement. Pour récupérer ces versions cliquez sur le “Details” lien à côté “ClickHouse build check” entrée dans la liste de vérifications. Vous y trouverez des liens directs vers les construit .paquets deb de ClickHouse que vous pouvez déployer même sur vos serveurs de production (si vous n’avez pas peur). +Le système préparera les builds binaires ClickHouse pour votre demande de tirage individuellement. Pour récupérer ces versions cliquez sur le “Details” lien à côté “ClickHouse build check” entrée dans la liste de vérifications. Vous y trouverez des liens directs vers les construit .paquets deb de ClickHouse que vous pouvez déployer même sur vos serveurs de production (si vous n'avez pas peur). -Très probablement, certaines des constructions échoueront à la première fois. Cela est dû au fait que nous avons vérifier s’appuie à la fois avec gcc, ainsi qu’avec clang, pratiquement tous les avertissements existants (toujours avec le `-Werror` drapeau) activé pour clang. Sur cette même page, vous pouvez trouver tous les journaux de construction afin que vous n’ayez pas à construire ClickHouse de toutes les manières possibles. +Très probablement, certaines des constructions échoueront à la première fois. Cela est dû au fait que nous avons vérifier s'appuie à la fois avec gcc, ainsi qu'avec clang, pratiquement tous les avertissements existants (toujours avec le `-Werror` drapeau) activé pour clang. Sur cette même page, vous pouvez trouver tous les journaux de construction afin que vous n'ayez pas à construire ClickHouse de toutes les manières possibles. diff --git a/docs/fr/development/index.md b/docs/fr/development/index.md index 1461fcae7bb..6a6dbe34dda 100644 --- a/docs/fr/development/index.md +++ b/docs/fr/development/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 -toc_folder_title: Development +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "D\xE9veloppement" toc_hidden: true toc_priority: 58 toc_title: "cach\xE9s" diff --git a/docs/fr/development/style.md b/docs/fr/development/style.md index 7d229b7e8f1..ef50570fbdf 100644 --- a/docs/fr/development/style.md +++ b/docs/fr/development/style.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 68 toc_title: "Comment \xE9crire du Code C++ " --- -# Comment écrire Du Code C++ {#how-to-write-c-code} +# Comment écrire du Code C++ {#how-to-write-c-code} ## Recommandations Générales {#general-recommendations} @@ -15,15 +15,15 @@ toc_title: "Comment \xE9crire du Code C++ " **3.** Le style de Code est nécessaire pour la cohérence. La cohérence facilite la lecture du code et facilite également la recherche du code. -**4.** Beaucoup de règles n’ont pas de raisons logiques; elles sont dictées par des pratiques établies. +**4.** Beaucoup de règles n'ont pas de raisons logiques; elles sont dictées par des pratiques établies. ## Formater {#formatting} **1.** La plupart du formatage se fera automatiquement par `clang-format`. -**2.** Les tirets sont 4 espaces. Configurez votre environnement de développement afin qu’un onglet ajoute quatre espaces. +**2.** Les tirets sont 4 espaces. Configurez votre environnement de développement afin qu'un onglet ajoute quatre espaces. -**3.** Les crochets d’ouverture et de fermeture doivent être sur une ligne séparée. +**3.** Les crochets d'ouverture et de fermeture doivent être sur une ligne séparée. ``` cpp inline void readBoolText(bool & x, ReadBuffer & buf) @@ -34,14 +34,14 @@ inline void readBoolText(bool & x, ReadBuffer & buf) } ``` -**4.** Si le corps entier de la fonction est un `statement` il peut donc être placé sur une seule ligne. Place des espaces autour des accolades (en plus de l’espace à la fin de la ligne). +**4.** Si le corps entier de la fonction est un `statement` il peut donc être placé sur une seule ligne. Place des espaces autour des accolades (en plus de l'espace à la fin de la ligne). ``` cpp inline size_t mask() const { return buf_size() - 1; } inline size_t place(HashValue x) const { return x & mask(); } ``` -**5.** Pour les fonctions. Ne mettez pas d’espaces entre parenthèses. +**5.** Pour les fonctions. Ne mettez pas d'espaces entre parenthèses. ``` cpp void reinsert(const Value & x) @@ -51,7 +51,7 @@ void reinsert(const Value & x) memcpy(&buf[place_value], &x, sizeof(x)); ``` -**6.** Dans `if`, `for`, `while` et d’autres expressions, un espace est inséré devant le support d’ouverture (par opposition aux appels de fonction). +**6.** Dans `if`, `for`, `while` et d'autres expressions, un espace est inséré devant le support d'ouverture (par opposition aux appels de fonction). ``` cpp for (size_t i = 0; i < rows; i += storage.index_granularity) @@ -65,7 +65,7 @@ UInt8 month = (s[5] - '0') * 10 + (s[6] - '0'); UInt8 day = (s[8] - '0') * 10 + (s[9] - '0'); ``` -**8.** Si un saut de ligne est entré, placez l’opérateur sur une nouvelle ligne et augmentez le retrait avant. +**8.** Si un saut de ligne est entré, placez l'opérateur sur une nouvelle ligne et augmentez le retrait avant. ``` cpp if (elapsed_ns) @@ -74,7 +74,7 @@ if (elapsed_ns) << bytes_read_on_server * 1000.0 / elapsed_ns << " MB/s.) "; ``` -**9.** Vous pouvez utiliser des espaces pour l’alignement dans une ligne, si vous le souhaitez. +**9.** Vous pouvez utiliser des espaces pour l'alignement dans une ligne, si vous le souhaitez. ``` cpp dst.ClickLogID = click.LogID; @@ -82,17 +82,17 @@ dst.ClickEventID = click.EventID; dst.ClickGoodEvent = click.GoodEvent; ``` -**10.** N’utilisez pas d’espaces autour des opérateurs `.`, `->`. +**10.** N'utilisez pas d'espaces autour des opérateurs `.`, `->`. -Si nécessaire, l’opérateur peut être renvoyé à la ligne suivante. Dans ce cas, le décalage devant celui-ci est augmenté. +Si nécessaire, l'opérateur peut être renvoyé à la ligne suivante. Dans ce cas, le décalage devant celui-ci est augmenté. -**11.** N’utilisez pas d’espace pour séparer les opérateurs unaires (`--`, `++`, `*`, `&`, …) from the argument. +**11.** N'utilisez pas d'espace pour séparer les opérateurs unaires (`--`, `++`, `*`, `&`, …) from the argument. -**12.** Mettre un espace après une virgule, mais pas avant. La même règle vaut pour un point-virgule à l’intérieur d’un `for` expression. +**12.** Mettre un espace après une virgule, mais pas avant. La même règle vaut pour un point-virgule à l'intérieur d'un `for` expression. **13.** Ne pas utiliser des espaces pour séparer les `[]` opérateur. -**14.** Dans un `template <...>` expression, utiliser un espace entre les `template` et `<`; pas d’espace après `<` ou avant `>`. +**14.** Dans un `template <...>` expression, utiliser un espace entre les `template` et `<`; pas d'espace après `<` ou avant `>`. ``` cpp template @@ -113,11 +113,11 @@ public: } ``` -**16.** Si le même `namespace` est utilisé pour l’ensemble du fichier, et il n’y a rien d’autre significatif, un décalage n’est pas nécessaire à l’intérieur `namespace`. +**16.** Si le même `namespace` est utilisé pour l'ensemble du fichier, et il n'y a rien d'autre significatif, un décalage n'est pas nécessaire à l'intérieur `namespace`. -**17.** Si le bloc pour un `if`, `for`, `while` ou autres expressions se compose d’un seul `statement`, les accolades sont facultatives. Place de la `statement` sur une ligne séparée, à la place. Cette règle est également valable pour les imbriqués `if`, `for`, `while`, … +**17.** Si le bloc pour un `if`, `for`, `while` ou autres expressions se compose d'un seul `statement`, les accolades sont facultatives. Place de la `statement` sur une ligne séparée, à la place. Cette règle est également valable pour les imbriqués `if`, `for`, `while`, … -Mais si l’intérieur `statement` contient des accolades ou `else` le bloc externe doit être écrit dans les accolades. +Mais si l'intérieur `statement` contient des accolades ou `else` le bloc externe doit être écrit dans les accolades. ``` cpp /// Finish write. @@ -125,7 +125,7 @@ for (auto & stream : streams) stream.second->finalize(); ``` -**18.** Il ne devrait pas y avoir d’espaces aux extrémités des lignes. +**18.** Il ne devrait pas y avoir d'espaces aux extrémités des lignes. **19.** Les fichiers Source sont encodés en UTF-8. @@ -135,9 +135,9 @@ for (auto & stream : streams) << ", " << (timer.elapsed() / chunks_stats.hits) << " μsec/hit."; ``` -**21.** N’écrivez pas plusieurs expressions sur une seule ligne. +**21.** N'écrivez pas plusieurs expressions sur une seule ligne. -**22.** Groupez les sections de code à l’intérieur des fonctions et séparez-les avec pas plus d’une ligne vide. +**22.** Groupez les sections de code à l'intérieur des fonctions et séparez-les avec pas plus d'une ligne vide. **23.** Séparez les fonctions, les classes, etc. avec une ou deux lignes vides. @@ -151,7 +151,7 @@ const std::string & s char const * pos ``` -**25.** Lors de la déclaration d’un pointeur ou d’une référence, le `*` et `&` les symboles doivent être séparés par des espaces des deux côtés. +**25.** Lors de la déclaration d'un pointeur ou d'une référence, le `*` et `&` les symboles doivent être séparés par des espaces des deux côtés. ``` cpp //correct @@ -161,9 +161,9 @@ const char* pos const char *pos ``` -**26.** Lors de l’utilisation de types de modèles, les alias avec le `using` mot-clé (sauf dans les cas les plus simples). +**26.** Lors de l'utilisation de types de modèles, les alias avec le `using` mot-clé (sauf dans les cas les plus simples). -En d’autres termes, les paramètres du modèle sont indiquées que dans `using` et ne sont pas répétés dans le code. +En d'autres termes, les paramètres du modèle sont indiquées que dans `using` et ne sont pas répétés dans le code. `using` peut être déclaré localement, comme dans une fonction. @@ -182,7 +182,7 @@ std::map> streams; int x, *y; ``` -**28.** N’utilisez pas de moulages de style C. +**28.** N'utilisez pas de moulages de style C. ``` cpp //incorrect @@ -193,15 +193,15 @@ std::cerr << static_cast(c) << std::endl; **29.** Dans les classes et les structures, groupez les membres et les fonctions séparément dans chaque portée de visibilité. -**30.** Pour les petites classes et structures, il n’est pas nécessaire de séparer la déclaration de méthode de l’implémentation. +**30.** Pour les petites classes et structures, il n'est pas nécessaire de séparer la déclaration de méthode de l'implémentation. La même chose est vraie pour les petites méthodes dans toutes les classes ou structures. -Pour les classes et les structures modélisées, ne séparez pas les déclarations de méthode de l’implémentation (car sinon elles doivent être définies dans la même unité de traduction). +Pour les classes et les structures modélisées, ne séparez pas les déclarations de méthode de l'implémentation (car sinon elles doivent être définies dans la même unité de traduction). **31.** Vous pouvez envelopper des lignes à 140 caractères, au lieu de 80. -**32.** Utilisez toujours les opérateurs d’incrémentation/décrémentation de préfixe si postfix n’est pas requis. +**32.** Utilisez toujours les opérateurs d'incrémentation/décrémentation de préfixe si postfix n'est pas requis. ``` cpp for (Names::const_iterator it = column_names.begin(); it != column_names.end(); ++it) @@ -209,9 +209,9 @@ for (Names::const_iterator it = column_names.begin(); it != column_names.end(); ## Commentaire {#comments} -**1.** Assurez-vous d’ajouter des commentaires pour toutes les parties non triviales du code. +**1.** Assurez-vous d'ajouter des commentaires pour toutes les parties non triviales du code. -C’est très important. Écrit le commentaire peut vous aider à réaliser que le code n’est pas nécessaire, ou qu’il est mal conçu. +C'est très important. Écrit le commentaire peut vous aider à réaliser que le code n'est pas nécessaire, ou qu'il est mal conçu. ``` cpp /** Part of piece of memory, that can be used. @@ -223,7 +223,7 @@ C’est très important. Écrit le commentaire peut vous aider à réaliser que **2.** Les commentaires peuvent être aussi détaillées que nécessaire. -**3.** Placez les commentaires avant le code qu’ils décrivent. Dans de rares cas, des commentaires peuvent venir après le code, sur la même ligne. +**3.** Placez les commentaires avant le code qu'ils décrivent. Dans de rares cas, des commentaires peuvent venir après le code, sur la même ligne. ``` cpp /** Parses and executes the query. @@ -239,9 +239,9 @@ void executeQuery( **4.** Les commentaires doivent être rédigés en anglais seulement. -**5.** Si vous écrivez une bibliothèque, incluez des commentaires détaillés l’expliquant dans le fichier d’en-tête principal. +**5.** Si vous écrivez une bibliothèque, incluez des commentaires détaillés l'expliquant dans le fichier d'en-tête principal. -**6.** N’ajoutez pas de commentaires qui ne fournissent pas d’informations supplémentaires. En particulier, ne laissez pas de commentaires vides comme celui-ci: +**6.** N'ajoutez pas de commentaires qui ne fournissent pas d'informations supplémentaires. En particulier, ne laissez pas de commentaires vides comme celui-ci: ``` cpp /* @@ -264,13 +264,13 @@ void executeQuery( */ ``` -L’exemple est emprunté à partir de la ressource http://home.tamk.fi/~jaalto/cours/coding-style/doc/désuète-code/. +L'exemple est emprunté à partir de la ressource http://home.tamk.fi/~jaalto/cours/coding-style/doc/désuète-code/. **7.** Ne pas écrire des commentaires de déchets (auteur, date de création .. au début de chaque fichier. **8.** Les commentaires sur une seule ligne commencent par trois barres obliques: `///` et les commentaires multi-lignes commencer avec `/**`. Ces commentaires sont pris en considération “documentation”. -REMARQUE: Vous pouvez utiliser Doxygen pour générer de la documentation à partir de ces commentaires. Mais Doxygen n’est généralement pas utilisé car il est plus pratique de naviguer dans le code dans L’IDE. +REMARQUE: Vous pouvez utiliser Doxygen pour générer de la documentation à partir de ces commentaires. Mais Doxygen n'est généralement pas utilisé car il est plus pratique de naviguer dans le code dans L'IDE. **9.** Les commentaires multilignes ne doivent pas avoir de lignes vides au début et à la fin (sauf la ligne qui ferme un commentaire multilignes). @@ -278,15 +278,15 @@ REMARQUE: Vous pouvez utiliser Doxygen pour générer de la documentation à par **11.** Supprimez les parties commentées du code avant de valider. -**12.** N’utilisez pas de blasphème dans les commentaires ou le code. +**12.** N'utilisez pas de blasphème dans les commentaires ou le code. -**13.** N’utilisez pas de majuscules. N’utilisez pas de ponctuation excessive. +**13.** N'utilisez pas de majuscules. N'utilisez pas de ponctuation excessive. ``` cpp /// WHAT THE FAIL??? ``` -**14.** N’utilisez pas de commentaires pour créer des délimiteurs. +**14.** N'utilisez pas de commentaires pour créer des délimiteurs. ``` cpp ///****************************************************** @@ -298,7 +298,7 @@ REMARQUE: Vous pouvez utiliser Doxygen pour générer de la documentation à par /// Why did you do this stuff? ``` -**16.** Il n’est pas nécessaire d’écrire un commentaire à la fin d’un bloc décrivant de quoi il s’agissait. +**16.** Il n'est pas nécessaire d'écrire un commentaire à la fin d'un bloc décrivant de quoi il s'agissait. ``` cpp /// for @@ -370,8 +370,8 @@ Si le fichier contient une seule fonction, nommez le fichier de la même manièr **11.** Si le nom contient une abréviation, puis: -- Pour les noms de variables, l’abréviation doit utiliser des lettres minuscules `mysql_connection` (pas `mySQL_connection`). -- Pour les noms de classes et de fonctions, conservez les majuscules dans l’abréviation`MySQLConnection` (pas `MySqlConnection`). +- Pour les noms de variables, l'abréviation doit utiliser des lettres minuscules `mysql_connection` (pas `mySQL_connection`). +- Pour les noms de classes et de fonctions, conservez les majuscules dans l'abréviation`MySQLConnection` (pas `MySqlConnection`). **12.** Les arguments du constructeur utilisés uniquement pour initialiser les membres de la classe doivent être nommés de la même manière que les membres de la classe, mais avec un trait de soulignement à la fin. @@ -388,15 +388,15 @@ FileQueueProcessor( } ``` -Le suffixe de soulignement peut être omis si l’argument n’est pas utilisé dans le corps du constructeur. +Le suffixe de soulignement peut être omis si l'argument n'est pas utilisé dans le corps du constructeur. -**13.** Il n’y a pas de différence dans les noms des variables locales et des membres de classe (aucun préfixe requis). +**13.** Il n'y a pas de différence dans les noms des variables locales et des membres de classe (aucun préfixe requis). ``` cpp timer (not m_timer) ``` -**14.** Pour les constantes dans un `enum`, utilisez CamelCase avec une lettre majuscule. ALL\_CAPS est également acceptable. Si l’ `enum` est non local, utilisez un `enum class`. +**14.** Pour les constantes dans un `enum`, utilisez CamelCase avec une lettre majuscule. ALL\_CAPS est également acceptable. Si l' `enum` est non local, utilisez un `enum class`. ``` cpp enum class CompressionMethod @@ -406,37 +406,37 @@ enum class CompressionMethod }; ``` -**15.** Tous les noms doivent être en anglais. La translittération des mots russes n’est pas autorisé. +**15.** Tous les noms doivent être en anglais. La translittération des mots russes n'est pas autorisé. not Stroka -**16.** Les abréviations sont acceptables si elles sont bien connues (quand vous pouvez facilement trouver la signification de l’abréviation dans Wikipédia ou dans un moteur de recherche). +**16.** Les abréviations sont acceptables si elles sont bien connues (quand vous pouvez facilement trouver la signification de l'abréviation dans Wikipédia ou dans un moteur de recherche). `AST`, `SQL`. Not `NVDH` (some random letters) -Les mots incomplets sont acceptables si la version abrégée est d’usage courant. +Les mots incomplets sont acceptables si la version abrégée est d'usage courant. Vous pouvez également utiliser une abréviation si le nom complet est ensuite incluse dans les commentaires. -**17.** Les noms de fichiers avec le code source C++ doivent avoir `.cpp` extension. Fichiers d’en-tête doit avoir la `.h` extension. +**17.** Les noms de fichiers avec le code source C++ doivent avoir `.cpp` extension. Fichiers d'en-tête doit avoir la `.h` extension. -## Comment écrire Du Code {#how-to-write-code} +## Comment écrire du Code {#how-to-write-code} **1.** Gestion de la mémoire. Désallocation manuelle de la mémoire (`delete`) ne peut être utilisé que dans le code de la bibliothèque. -Dans le code de la bibliothèque, de la `delete` l’opérateur ne peut être utilisé dans des destructeurs. +Dans le code de la bibliothèque, de la `delete` l'opérateur ne peut être utilisé dans des destructeurs. -Dans le code de l’application, la mémoire doit être libérée par l’objet qui la possède. +Dans le code de l'application, la mémoire doit être libérée par l'objet qui la possède. Exemple: -- Le plus simple est de placer un objet sur la pile, ou d’en faire un membre d’une autre classe. +- Le plus simple est de placer un objet sur la pile, ou d'en faire un membre d'une autre classe. - Pour un grand nombre de petits objets, utiliser des récipients. -- Pour la désallocation automatique d’un petit nombre d’objets qui résident dans le tas, utilisez `shared_ptr/unique_ptr`. +- Pour la désallocation automatique d'un petit nombre d'objets qui résident dans le tas, utilisez `shared_ptr/unique_ptr`. **2.** La gestion des ressources. @@ -444,11 +444,11 @@ Utiliser `RAII` et voir ci-dessus. **3.** La gestion des erreurs. -Utilisez des exceptions. Dans la plupart des cas, vous avez seulement besoin de lancer une exception, et n’avez pas besoin de l’attraper (à cause de `RAII`). +Utilisez des exceptions. Dans la plupart des cas, vous avez seulement besoin de lancer une exception, et n'avez pas besoin de l'attraper (à cause de `RAII`). -Dans les applications de traitement de données hors ligne, il est souvent acceptable de ne pas attraper d’exceptions. +Dans les applications de traitement de données hors ligne, il est souvent acceptable de ne pas attraper d'exceptions. -Dans les serveurs qui gèrent les demandes des utilisateurs, il suffit généralement d’attraper des exceptions au niveau supérieur du gestionnaire de connexion. +Dans les serveurs qui gèrent les demandes des utilisateurs, il suffit généralement d'attraper des exceptions au niveau supérieur du gestionnaire de connexion. Dans les fonctions de thread, vous devez attraper et conserver toutes les exceptions pour les repasser dans le thread principal après `join`. @@ -485,7 +485,7 @@ catch (const DB::Exception & e) } ``` -Lorsque vous utilisez des fonctions avec des codes de réponse ou `errno` toujours vérifier le résultat et de lever une exception en cas d’erreur. +Lorsque vous utilisez des fonctions avec des codes de réponse ou `errno` toujours vérifier le résultat et de lever une exception en cas d'erreur. ``` cpp if (0 != close(fd)) @@ -494,24 +494,24 @@ if (0 != close(fd)) `Do not use assert`. -**4.** Les types d’Exception. +**4.** Les types d'Exception. -Il n’est pas nécessaire d’utiliser une hiérarchie d’exceptions complexe dans le code de l’application. Le texte d’exception doit être compréhensible pour un administrateur système. +Il n'est pas nécessaire d'utiliser une hiérarchie d'exceptions complexe dans le code de l'application. Le texte d'exception doit être compréhensible pour un administrateur système. **5.** Lancer des exceptions de destructeurs. -Ce n’est pas recommandé, mais il est permis. +Ce n'est pas recommandé, mais il est permis. Utilisez les options suivantes: - Créer une fonction (`done()` ou `finalize()`) qui vont faire tout le travail en amont qui pourrait conduire à une exception. Si cette fonction a été appelée, il ne devrait y avoir aucune exception dans le destructeur plus tard. -- Les tâches trop complexes (comme l’envoi de messages sur le réseau) peuvent être placées dans une méthode distincte que l’utilisateur de la classe devra appeler avant la destruction. -- Si il y a une exception dans le destructeur, il est préférable de l’enregistrer que de le cacher (si l’enregistreur est disponible). +- Les tâches trop complexes (comme l'envoi de messages sur le réseau) peuvent être placées dans une méthode distincte que l'utilisateur de la classe devra appeler avant la destruction. +- Si il y a une exception dans le destructeur, il est préférable de l'enregistrer que de le cacher (si l'enregistreur est disponible). - Dans les applications simples, il est acceptable de compter sur `std::terminate` (pour les cas de `noexcept` par défaut en C++11) pour gérer les exceptions. **6.** Blocs de code anonymes. -Vous pouvez créer un bloc de code séparé à l’intérieur d’une seule fonction afin de rendre certaines variables locales, de sorte que les destructeurs sont appelés à la sortie du bloc. +Vous pouvez créer un bloc de code séparé à l'intérieur d'une seule fonction afin de rendre certaines variables locales, de sorte que les destructeurs sont appelés à la sortie du bloc. ``` cpp Block block = data.in->read(); @@ -529,25 +529,25 @@ ready_any.set(); Dans les programmes de traitement de données hors ligne: -- Essayez d’obtenir les meilleures performances possibles sur un seul noyau CPU. Vous pouvez ensuite paralléliser votre code si nécessaire. +- Essayez d'obtenir les meilleures performances possibles sur un seul noyau CPU. Vous pouvez ensuite paralléliser votre code si nécessaire. Dans les applications serveur: -- Utiliser le pool de threads pour traiter les demandes. À ce stade, nous n’avons pas eu de tâches nécessitant un changement de contexte dans l’espace utilisateur. +- Utiliser le pool de threads pour traiter les demandes. À ce stade, nous n'avons pas eu de tâches nécessitant un changement de contexte dans l'espace utilisateur. -La fourche n’est pas utilisé pour la parallélisation. +La fourche n'est pas utilisé pour la parallélisation. **8.** Synchronisation des threads. Souvent, il est possible de faire en sorte que différents threads utilisent différentes cellules de mémoire (encore mieux: différentes lignes de cache,) et de ne pas utiliser de synchronisation de thread (sauf `joinAll`). -Si la synchronisation est nécessaire, dans la plupart des cas, il suffit d’utiliser mutex sous `lock_guard`. +Si la synchronisation est nécessaire, dans la plupart des cas, il suffit d'utiliser mutex sous `lock_guard`. -Dans d’autres cas, utilisez des primitives de synchronisation système. Ne pas utiliser occupé attendre. +Dans d'autres cas, utilisez des primitives de synchronisation système. Ne pas utiliser occupé attendre. Les opérations atomiques ne doivent être utilisées que dans les cas les plus simples. -N’essayez pas d’implémenter des structures de données sans verrou à moins qu’il ne s’agisse de votre principal domaine d’expertise. +N'essayez pas d'implémenter des structures de données sans verrou à moins qu'il ne s'agisse de votre principal domaine d'expertise. **9.** Pointeurs vs références. @@ -557,7 +557,7 @@ Dans la plupart des cas, préférez les références. Utiliser des références constantes, des pointeurs vers des constantes, `const_iterator` et const méthodes. -Considérer `const` pour être par défaut et utiliser non-`const` seulement quand c’est nécessaire. +Considérer `const` pour être par défaut et utiliser non-`const` seulement quand c'est nécessaire. Lors du passage de variables par valeur, en utilisant `const` habituellement ne fait pas de sens. @@ -569,21 +569,21 @@ Utiliser `unsigned` si nécessaire. Utiliser les types `UInt8`, `UInt16`, `UInt32`, `UInt64`, `Int8`, `Int16`, `Int32`, et `Int64` ainsi que `size_t`, `ssize_t`, et `ptrdiff_t`. -N’utilisez pas ces types pour les nombres: `signed/unsigned long`, `long long`, `short`, `signed/unsigned char`, `char`. +N'utilisez pas ces types pour les nombres: `signed/unsigned long`, `long long`, `short`, `signed/unsigned char`, `char`. **13.** Passer des arguments. Passer des valeurs complexes par référence (y compris `std::string`). -Si une fonction capture la propriété d’un objet créé dans le tas, définissez le type d’argument `shared_ptr` ou `unique_ptr`. +Si une fonction capture la propriété d'un objet créé dans le tas, définissez le type d'argument `shared_ptr` ou `unique_ptr`. **14.** Les valeurs de retour. -Dans la plupart des cas, il suffit d’utiliser `return`. Ne pas écrire `[return std::move(res)]{.strike}`. +Dans la plupart des cas, il suffit d'utiliser `return`. Ne pas écrire `[return std::move(res)]{.strike}`. Si la fonction alloue un objet sur le tas et le renvoie, utilisez `shared_ptr` ou `unique_ptr`. -Dans de rares cas, vous devrez peut-être renvoyer la valeur via un argument. Dans ce cas, l’argument doit être une référence. +Dans de rares cas, vous devrez peut-être renvoyer la valeur via un argument. Dans ce cas, l'argument doit être une référence. ``` cpp using AggregateFunctionPtr = std::shared_ptr; @@ -599,23 +599,23 @@ public: **15.** espace de noms. -Il n’est pas nécessaire d’utiliser une `namespace` pour le code de l’application. +Il n'est pas nécessaire d'utiliser une `namespace` pour le code de l'application. -Les petites bibliothèques n’ont pas besoin de cela non plus. +Les petites bibliothèques n'ont pas besoin de cela non plus. Pour les bibliothèques moyennes et grandes, mettez tout dans un `namespace`. -Dans la bibliothèque `.h` fichier, vous pouvez utiliser `namespace detail` pour masquer les détails d’implémentation non nécessaires pour le code de l’application. +Dans la bibliothèque `.h` fichier, vous pouvez utiliser `namespace detail` pour masquer les détails d'implémentation non nécessaires pour le code de l'application. Dans un `.cpp` fichier, vous pouvez utiliser un `static` ou un espace de noms anonyme pour masquer les symboles. -Aussi, un `namespace` peut être utilisé pour un `enum` pour éviter que les noms correspondants ne tombent dans un `namespace` (mais il est préférable d’utiliser un `enum class`). +Aussi, un `namespace` peut être utilisé pour un `enum` pour éviter que les noms correspondants ne tombent dans un `namespace` (mais il est préférable d'utiliser un `enum class`). **16.** Initialisation différée. -Si des arguments sont requis pour l’initialisation, vous ne devriez normalement pas écrire de constructeur par défaut. +Si des arguments sont requis pour l'initialisation, vous ne devriez normalement pas écrire de constructeur par défaut. -Si plus tard, vous devez retarder l’initialisation, vous pouvez ajouter un constructeur par défaut qui créera un objet invalide. Ou, pour un petit nombre d’objets, vous pouvez utiliser `shared_ptr/unique_ptr`. +Si plus tard, vous devez retarder l'initialisation, vous pouvez ajouter un constructeur par défaut qui créera un objet invalide. Ou, pour un petit nombre d'objets, vous pouvez utiliser `shared_ptr/unique_ptr`. ``` cpp Loader(DB::Connection * connection_, const std::string & query, size_t max_block_size_); @@ -626,7 +626,7 @@ Loader() {} **17.** Des fonctions virtuelles. -Si la classe n’est pas destinée à une utilisation polymorphe, vous n’avez pas besoin de rendre les fonctions virtuelles. Ceci s’applique également pour le destructeur. +Si la classe n'est pas destinée à une utilisation polymorphe, vous n'avez pas besoin de rendre les fonctions virtuelles. Ceci s'applique également pour le destructeur. **18.** Encodage. @@ -638,37 +638,37 @@ Voir les exemples partout dans le code. Avant de valider, supprimez toute journalisation sans signification et de débogage, ainsi que tout autre type de sortie de débogage. -L’enregistrement des cycles doit être évité, même au niveau de la Trace. +L'enregistrement des cycles doit être évité, même au niveau de la Trace. -Les journaux doivent être lisibles à tout niveau d’enregistrement. +Les journaux doivent être lisibles à tout niveau d'enregistrement. -La journalisation ne doit être utilisée que dans le code de l’application, pour la plupart. +La journalisation ne doit être utilisée que dans le code de l'application, pour la plupart. Les messages du journal doivent être écrits en anglais. -Le journal devrait de préférence être compréhensible pour l’administrateur système. +Le journal devrait de préférence être compréhensible pour l'administrateur système. -N’utilisez pas de blasphème dans le journal. +N'utilisez pas de blasphème dans le journal. -Utilisez L’encodage UTF-8 dans le journal. Dans de rares cas, vous pouvez utiliser des caractères non-ASCII dans le journal. +Utilisez L'encodage UTF-8 dans le journal. Dans de rares cas, vous pouvez utiliser des caractères non-ASCII dans le journal. -**20.** D’entrée-sortie. +**20.** D'entrée-sortie. -Ne pas utiliser de `iostreams` dans les cycles internes qui sont critiques pour les performances de l’application (et ne jamais utiliser `stringstream`). +Ne pas utiliser de `iostreams` dans les cycles internes qui sont critiques pour les performances de l'application (et ne jamais utiliser `stringstream`). -L’utilisation de la `DB/IO` la bibliothèque la place. +L'utilisation de la `DB/IO` la bibliothèque la place. -**21.** La Date et l’heure. +**21.** La Date et l'heure. Voir la `DateLUT` bibliothèque. **22.** comprendre. -Toujours utiliser `#pragma once` au lieu d’inclure des gardes. +Toujours utiliser `#pragma once` au lieu d'inclure des gardes. **23.** utiliser. -`using namespace` n’est pas utilisé. Vous pouvez utiliser `using` avec quelque chose de spécifique. Mais faire local à l’intérieur d’une classe ou d’une fonction. +`using namespace` n'est pas utilisé. Vous pouvez utiliser `using` avec quelque chose de spécifique. Mais faire local à l'intérieur d'une classe ou d'une fonction. **24.** Ne pas utiliser de `trailing return type` pour les fonctions, sauf si nécessaire. @@ -687,13 +687,13 @@ std::string s{"Hello"}; auto s = std::string{"Hello"}; ``` -**26.** Pour les fonctions virtuelles, écrire `virtual` dans la classe de base, mais d’écrire `override` plutôt `virtual` dans les classes descendantes. +**26.** Pour les fonctions virtuelles, écrire `virtual` dans la classe de base, mais d'écrire `override` plutôt `virtual` dans les classes descendantes. -## Fonctionnalités inutilisées De C++ {#unused-features-of-c} +## Fonctionnalités inutilisées de C++ {#unused-features-of-c} -**1.** L’héritage virtuel n’est pas utilisé. +**1.** L'héritage virtuel n'est pas utilisé. -**2.** Les spécificateurs d’Exception de C++03 ne sont pas utilisés. +**2.** Les spécificateurs d'Exception de C++03 ne sont pas utilisés. ## Plate {#platform} @@ -709,9 +709,9 @@ La bibliothèque standard est utilisée (`libstdc++` ou `libc++`). **4.**OS: Linux Ubuntu, pas plus vieux que précis. -**5.**Le Code est écrit pour l’architecture CPU x86\_64. +**5.**Le Code est écrit pour l'architecture CPU x86\_64. -Le jeu D’instructions CPU est l’ensemble minimum pris en charge parmi nos serveurs. Actuellement, il s’agit de SSE 4.2. +Le jeu D'instructions CPU est l'ensemble minimum pris en charge parmi nos serveurs. Actuellement, il s'agit de SSE 4.2. **6.** Utiliser `-Wall -Wextra -Werror` drapeaux de compilation. @@ -737,11 +737,11 @@ Le jeu D’instructions CPU est l’ensemble minimum pris en charge parmi nos se Bien que seules les révisions sélectionnées soient considérées comme réalisables. -**8.** Faire s’engage aussi souvent que possible, même si le code n’est que partiellement prêt. +**8.** Faire s'engage aussi souvent que possible, même si le code n'est que partiellement prêt. Utilisez des branches à cet effet. -Si votre code dans le `master` la branche n’est pas constructible pourtant, l’exclure de la construction avant que le `push`. Vous devrez le terminer ou l’enlever dans quelques jours. +Si votre code dans le `master` la branche n'est pas constructible pourtant, l'exclure de la construction avant que le `push`. Vous devrez le terminer ou l'enlever dans quelques jours. **9.** Pour les modifications non triviales, utilisez les branches et publiez-les sur le serveur. @@ -753,13 +753,13 @@ Si votre code dans le `master` la branche n’est pas constructible pourtant, l **2.** Si nécessaire, vous pouvez utiliser toutes les bibliothèques bien connues disponibles dans le package OS. -S’il existe déjà une bonne solution, utilisez-la, même si cela signifie que vous devez installer une autre bibliothèque. +S'il existe déjà une bonne solution, utilisez-la, même si cela signifie que vous devez installer une autre bibliothèque. (Mais soyez prêt à supprimer les mauvaises bibliothèques du code.) -**3.** Vous pouvez installer une bibliothèque qui n’est pas dans les paquets, les paquets n’ont pas ce que vous souhaitez ou avez une version périmée ou le mauvais type de compilation. +**3.** Vous pouvez installer une bibliothèque qui n'est pas dans les paquets, les paquets n'ont pas ce que vous souhaitez ou avez une version périmée ou le mauvais type de compilation. -**4.** Si la Bibliothèque est petite et n’a pas son propre système de construction complexe, placez les fichiers source dans le `contrib` dossier. +**4.** Si la Bibliothèque est petite et n'a pas son propre système de construction complexe, placez les fichiers source dans le `contrib` dossier. **5.** La préférence est toujours donnée aux bibliothèques déjà utilisées. @@ -769,11 +769,11 @@ S’il existe déjà une bonne solution, utilisez-la, même si cela signifie que **2.** Essayez la solution la plus simple. -**3.** N’écrivez pas de code tant que vous ne savez pas comment cela va fonctionner et comment la boucle interne fonctionnera. +**3.** N'écrivez pas de code tant que vous ne savez pas comment cela va fonctionner et comment la boucle interne fonctionnera. **4.** Dans les cas les plus simples, utilisez `using` au lieu de classes ou des structures. -**5.** Si possible, n’écrivez pas de constructeurs de copie, d’opérateurs d’affectation, de destructeurs (autres que Virtuels, si la classe contient au moins une fonction virtuelle), de constructeurs de déplacement ou d’opérateurs d’affectation de déplacement. En d’autres termes, les fonctions générées par le compilateur doivent fonctionner correctement. Vous pouvez utiliser `default`. +**5.** Si possible, n'écrivez pas de constructeurs de copie, d'opérateurs d'affectation, de destructeurs (autres que Virtuels, si la classe contient au moins une fonction virtuelle), de constructeurs de déplacement ou d'opérateurs d'affectation de déplacement. En d'autres termes, les fonctions générées par le compilateur doivent fonctionner correctement. Vous pouvez utiliser `default`. **6.** La simplification du Code est encouragée. Réduire la taille de votre code si possible. @@ -781,15 +781,15 @@ S’il existe déjà une bonne solution, utilisez-la, même si cela signifie que **1.** Spécifier explicitement `std::` pour les types de `stddef.h` -n’est pas recommandé. En d’autres termes, nous vous recommandons d’écriture `size_t` plutôt `std::size_t` parce que c’est plus court. +n'est pas recommandé. En d'autres termes, nous vous recommandons d'écriture `size_t` plutôt `std::size_t` parce que c'est plus court. -Il est acceptable d’ajouter `std::`. +Il est acceptable d'ajouter `std::`. **2.** Spécifier explicitement `std::` pour les fonctions de la bibliothèque C standard -n’est pas recommandé. En d’autres termes, écrire `memcpy` plutôt `std::memcpy`. +n'est pas recommandé. En d'autres termes, écrire `memcpy` plutôt `std::memcpy`. -La raison en est qu’il existe des fonctions non standard similaires, telles que `memmem`. Nous utilisons ces fonctions à l’occasion. Ces fonctions n’existent pas dans `namespace std`. +La raison en est qu'il existe des fonctions non standard similaires, telles que `memmem`. Nous utilisons ces fonctions à l'occasion. Ces fonctions n'existent pas dans `namespace std`. Si vous écrivez `std::memcpy` plutôt `memcpy` partout, puis `memmem` sans `std::` va sembler étrange. @@ -797,13 +797,13 @@ Néanmoins, vous pouvez toujours utiliser `std::` si vous le souhaitez. **3.** Utilisation des fonctions de C lorsque les mêmes sont disponibles dans la bibliothèque C++ standard. -Ceci est acceptable s’il est plus efficace. +Ceci est acceptable s'il est plus efficace. -Par exemple, l’utilisation `memcpy` plutôt `std::copy` pour copier de gros morceaux de mémoire. +Par exemple, l'utilisation `memcpy` plutôt `std::copy` pour copier de gros morceaux de mémoire. **4.** Arguments de fonction multiligne. -L’un des styles d’emballage suivants est autorisé: +L'un des styles d'emballage suivants est autorisé: ``` cpp function( diff --git a/docs/fr/development/tests.md b/docs/fr/development/tests.md index 9a34cb5f61f..c059472bb0c 100644 --- a/docs/fr/development/tests.md +++ b/docs/fr/development/tests.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 69 toc_title: "Comment ex\xE9cuter des Tests ClickHouse" --- @@ -11,15 +11,15 @@ toc_title: "Comment ex\xE9cuter des Tests ClickHouse" Les tests fonctionnels sont les plus simples et pratiques à utiliser. La plupart des fonctionnalités de ClickHouse peuvent être testées avec des tests fonctionnels et elles sont obligatoires à utiliser pour chaque changement de code de ClickHouse qui peut être testé de cette façon. -Chaque test fonctionnel envoie une ou plusieurs requêtes au serveur clickhouse en cours d’exécution et compare le résultat avec la référence. +Chaque test fonctionnel envoie une ou plusieurs requêtes au serveur clickhouse en cours d'exécution et compare le résultat avec la référence. -Les Tests sont situés dans `queries` répertoire. Il y a deux sous-répertoires: `stateless` et `stateful`. Les tests sans état exécutent des requêtes sans données de test préchargées - ils créent souvent de petits ensembles de données synthétiques à la volée, dans le test lui-même. Les tests avec État nécessitent des données de test préchargées de Yandex.Metrica et non disponible pour le grand public. Nous avons tendance à utiliser uniquement `stateless` tests et éviter d’ajouter de nouveaux `stateful` test. +Les Tests sont situés dans `queries` répertoire. Il y a deux sous-répertoires: `stateless` et `stateful`. Les tests sans état exécutent des requêtes sans données de test préchargées - ils créent souvent de petits ensembles de données synthétiques à la volée, dans le test lui-même. Les tests avec État nécessitent des données de test préchargées de Yandex.Metrica et non disponible pour le grand public. Nous avons tendance à utiliser uniquement `stateless` tests et éviter d'ajouter de nouveaux `stateful` test. Chaque test peut être de deux types: `.sql` et `.sh`. `.sql` test est le script SQL simple qui est canalisé vers `clickhouse-client --multiquery --testmode`. `.sh` test est un script qui est exécuté par lui-même. Pour exécuter tous les tests, utilisez `clickhouse-test` outil. Regarder `--help` pour la liste des options possibles. Vous pouvez simplement exécuter tous les tests ou exécuter un sous ensemble de tests filtrés par sous chaîne dans le nom du test: `./clickhouse-test substring`. -Le moyen le plus simple d’invoquer des tests fonctionnels est de copier `clickhouse-client` de `/usr/bin/`, exécuter `clickhouse-server` et puis exécutez `./clickhouse-test` à partir de son propre répertoire. +Le moyen le plus simple d'invoquer des tests fonctionnels est de copier `clickhouse-client` de `/usr/bin/`, exécuter `clickhouse-server` et puis exécutez `./clickhouse-test` à partir de son propre répertoire. Pour ajouter un nouveau test, créez un `.sql` ou `.sh` fichier dans `queries/0_stateless` répertoire, vérifiez-le manuellement, puis générez `.reference` fichier de la façon suivante: `clickhouse-client -n --testmode < 00000_test.sql > 00000_test.reference` ou `./00000_test.sh > ./00000_test.reference`. @@ -29,8 +29,8 @@ Si vous souhaitez utiliser des requêtes distribuées dans les tests fonctionnel Certains tests sont marqués avec `zookeeper`, `shard` ou `long` en leurs noms. `zookeeper` est pour les tests qui utilisent ZooKeeper. `shard` est pour les tests -nécessite l’écoute du serveur `127.0.0.*`; `distributed` ou `global` avoir le même -sens. `long` est pour les tests qui s’exécutent légèrement plus longtemps qu’une seconde. Vous pouvez +nécessite l'écoute du serveur `127.0.0.*`; `distributed` ou `global` avoir le même +sens. `long` est pour les tests qui s'exécutent légèrement plus longtemps qu'une seconde. Vous pouvez désactivez ces groupes de tests en utilisant `--no-zookeeper`, `--no-shard` et `--no-long` options, respectivement. @@ -38,47 +38,47 @@ désactivez ces groupes de tests en utilisant `--no-zookeeper`, `--no-shard` et Si nous connaissons des bugs qui peuvent être facilement reproduits par des tests fonctionnels, nous plaçons des tests fonctionnels préparés dans `tests/queries/bugs` répertoire. Ces tests seront déplacés à `tests/queries/0_stateless` quand les bugs sont corrigés. -## Les Tests D’Intégration {#integration-tests} +## Les Tests D'Intégration {#integration-tests} -Les tests d’intégration permettent de tester ClickHouse en configuration cluster et clickhouse interaction avec D’autres serveurs comme MySQL, Postgres, MongoDB. Ils sont utiles pour émuler les splits réseau, les chutes de paquets, etc. Ces tests sont exécutés sous Docker et créent plusieurs conteneurs avec divers logiciels. +Les tests d'intégration permettent de tester ClickHouse en configuration cluster et clickhouse interaction avec D'autres serveurs comme MySQL, Postgres, MongoDB. Ils sont utiles pour émuler les splits réseau, les chutes de paquets, etc. Ces tests sont exécutés sous Docker et créent plusieurs conteneurs avec divers logiciels. -Voir `tests/integration/README.md` sur la façon d’exécuter ces tests. +Voir `tests/integration/README.md` sur la façon d'exécuter ces tests. -Notez que l’intégration de ClickHouse avec des pilotes tiers n’est pas testée. De plus, nous n’avons actuellement pas de tests d’intégration avec nos pilotes JDBC et ODBC. +Notez que l'intégration de ClickHouse avec des pilotes tiers n'est pas testée. De plus, nous n'avons actuellement pas de tests d'intégration avec nos pilotes JDBC et ODBC. ## Les Tests Unitaires {#unit-tests} -Les tests unitaires sont utiles lorsque vous voulez tester non pas le ClickHouse dans son ensemble, mais une seule bibliothèque ou classe isolée. Vous pouvez activer ou désactiver la génération de tests avec `ENABLE_TESTS` Option CMake. Les tests unitaires (et autres programmes de test) sont situés dans `tests` sous-répertoires à travers le code. Pour exécuter des tests unitaires, tapez `ninja test`. Certains tests utilisent `gtest`, mais certains ne sont que des programmes qui renvoient un code de sortie non nul en cas d’échec du test. +Les tests unitaires sont utiles lorsque vous voulez tester non pas le ClickHouse dans son ensemble, mais une seule bibliothèque ou classe isolée. Vous pouvez activer ou désactiver la génération de tests avec `ENABLE_TESTS` Option CMake. Les tests unitaires (et autres programmes de test) sont situés dans `tests` sous-répertoires à travers le code. Pour exécuter des tests unitaires, tapez `ninja test`. Certains tests utilisent `gtest`, mais certains ne sont que des programmes qui renvoient un code de sortie non nul en cas d'échec du test. -Ce n’est pas nécessairement d’avoir des tests unitaires si le code est déjà couvert par des tests fonctionnels (et les tests fonctionnels sont généralement beaucoup plus simples à utiliser). +Ce n'est pas nécessairement d'avoir des tests unitaires si le code est déjà couvert par des tests fonctionnels (et les tests fonctionnels sont généralement beaucoup plus simples à utiliser). ## Tests De Performance {#performance-tests} -Les tests de Performance permettent de mesurer et de comparer les performances d’une partie isolée de ClickHouse sur des requêtes synthétiques. Les Tests sont situés à `tests/performance`. Chaque test est représenté par `.xml` fichier avec description du cas de test. Les Tests sont exécutés avec `clickhouse performance-test` outil (qui est incorporé dans `clickhouse` binaire). Voir `--help` pour l’invocation. +Les tests de Performance permettent de mesurer et de comparer les performances d'une partie isolée de ClickHouse sur des requêtes synthétiques. Les Tests sont situés à `tests/performance`. Chaque test est représenté par `.xml` fichier avec description du cas de test. Les Tests sont exécutés avec `clickhouse performance-test` outil (qui est incorporé dans `clickhouse` binaire). Voir `--help` pour l'invocation. -Chaque essai un ou miltiple requêtes (éventuellement avec des combinaisons de paramètres) dans une boucle avec certaines conditions pour l’arrêt (comme “maximum execution speed is not changing in three seconds”) et mesurer certaines mesures sur les performances de la requête (comme “maximum execution speed”). Certains tests peuvent contenir des conditions préalables sur un ensemble de données de test préchargé. +Chaque essai d'exécuter une ou plusieurs requêtes (éventuellement avec des combinaisons de paramètres) dans une boucle avec certaines conditions pour l'arrêt (comme “maximum execution speed is not changing in three seconds”) et mesurer certaines mesures sur les performances de la requête (comme “maximum execution speed”). Certains tests peuvent contenir des conditions préalables sur un ensemble de données de test préchargé. -Si vous souhaitez améliorer les performances de ClickHouse dans certains scénarios, et si des améliorations peuvent être observées sur des requêtes simples, il est fortement recommandé d’écrire un test de performance. Il est toujours logique d’utiliser `perf top` ou d’autres outils perf pendant vos tests. +Si vous souhaitez améliorer les performances de ClickHouse dans certains scénarios, et si des améliorations peuvent être observées sur des requêtes simples, il est fortement recommandé d'écrire un test de performance. Il est toujours logique d'utiliser `perf top` ou d'autres outils perf pendant vos tests. -## Outils Et Scripts De Test {#test-tools-and-scripts} +## Outils et Scripts de Test {#test-tools-and-scripts} -Certains programmes dans `tests` directory ne sont pas des tests préparés, mais sont des outils de test. Par exemple, pour `Lexer` il est un outil `dbms/Parsers/tests/lexer` Cela fait juste la tokenisation de stdin et écrit le résultat colorisé dans stdout. Vous pouvez utiliser ce genre d’outils comme exemples de code et pour l’exploration et les tests manuels. +Certains programmes dans `tests` directory ne sont pas des tests préparés, mais sont des outils de test. Par exemple, pour `Lexer` il est un outil `src/Parsers/tests/lexer` Cela fait juste la tokenisation de stdin et écrit le résultat colorisé dans stdout. Vous pouvez utiliser ce genre d'outils comme exemples de code et pour l'exploration et les tests manuels. -Vous pouvez également placer une paire de fichiers `.sh` et `.reference` avec l’outil pour l’exécuter sur une entrée prédéfinie - alors le résultat du script peut être comparé à `.reference` fichier. Ce genre de tests ne sont pas automatisés. +Vous pouvez également placer une paire de fichiers `.sh` et `.reference` avec l'outil pour l'exécuter sur une entrée prédéfinie - alors le résultat du script peut être comparé à `.reference` fichier. Ce genre de tests ne sont pas automatisés. -## Tests Divers {#miscellanous-tests} +## Divers Tests {#miscellaneous-tests} -Il existe des tests pour les dictionnaires externes situés à `tests/external_dictionaries` et pour machine appris modèles dans `tests/external_models`. Ces tests ne sont pas mis à jour et doivent être transférés aux tests d’intégration. +Il existe des tests pour les dictionnaires externes situés à `tests/external_dictionaries` et pour machine appris modèles dans `tests/external_models`. Ces tests ne sont pas mis à jour et doivent être transférés aux tests d'intégration. -Il y a un test séparé pour les inserts de quorum. Ce test exécute le cluster ClickHouse sur des serveurs séparés et émule divers cas d’échec: scission réseau, chute de paquets (entre les nœuds ClickHouse, entre Clickhouse et ZooKeeper, entre le serveur ClickHouse et le client, etc.), `kill -9`, `kill -STOP` et `kill -CONT` , comme [Jepsen](https://aphyr.com/tags/Jepsen). Ensuite, le test vérifie que toutes les insertions reconnues ont été écrites et que toutes les insertions rejetées ne l’ont pas été. +Il y a un test séparé pour les inserts de quorum. Ce test exécute le cluster ClickHouse sur des serveurs séparés et émule divers cas d'échec: scission réseau, chute de paquets (entre les nœuds ClickHouse, entre Clickhouse et ZooKeeper, entre le serveur ClickHouse et le client, etc.), `kill -9`, `kill -STOP` et `kill -CONT` , comme [Jepsen](https://aphyr.com/tags/Jepsen). Ensuite, le test vérifie que toutes les insertions reconnues ont été écrites et que toutes les insertions rejetées ne l'ont pas été. -Le test de Quorum a été écrit par une équipe distincte avant que ClickHouse ne soit open-source. Cette équipe ne travaille plus avec ClickHouse. Test a été écrit accidentellement en Java. Pour ces raisons, quorum test doit être réécrit et déplacé vers tests d’intégration. +Le test de Quorum a été écrit par une équipe distincte avant que ClickHouse ne soit open-source. Cette équipe ne travaille plus avec ClickHouse. Test a été écrit accidentellement en Java. Pour ces raisons, quorum test doit être réécrit et déplacé vers tests d'intégration. ## Les Tests Manuels {#manual-testing} Lorsque vous développez une nouvelle fonctionnalité, il est raisonnable de tester également manuellement. Vous pouvez le faire avec les étapes suivantes: -Construire ClickHouse. Exécuter ClickHouse à partir du terminal: changer le répertoire à `programs/clickhouse-server` et de l’exécuter avec `./clickhouse-server`. Il utilisera la configuration (`config.xml`, `users.xml` et les fichiers à l’intérieur `config.d` et `users.d` répertoires) à partir du répertoire courant par défaut. Pour vous connecter au serveur ClickHouse, exécutez `programs/clickhouse-client/clickhouse-client`. +Construire ClickHouse. Exécuter ClickHouse à partir du terminal: changer le répertoire à `programs/clickhouse-server` et de l'exécuter avec `./clickhouse-server`. Il utilisera la configuration (`config.xml`, `users.xml` et les fichiers à l'intérieur `config.d` et `users.d` répertoires) à partir du répertoire courant par défaut. Pour vous connecter au serveur ClickHouse, exécutez `programs/clickhouse-client/clickhouse-client`. Notez que tous les outils clickhouse (serveur, client, etc.) ne sont que des liens symboliques vers un seul binaire nommé `clickhouse`. Vous pouvez trouver ce binaire à `programs/clickhouse`. Tous les outils peuvent également être invoquée comme `clickhouse tool` plutôt `clickhouse-tool`. @@ -105,23 +105,23 @@ Exemple avec gdb: $ sudo -u clickhouse gdb --args /usr/bin/clickhouse server --config-file /etc/clickhouse-server/config.xml ``` -Si le système clickhouse-server est déjà en cours d’exécution et que vous ne voulez pas l’arrêter, vous pouvez modifier les numéros de port dans votre `config.xml` (ou de les remplacer dans un fichier `config.d` répertoire), fournissez le chemin de données approprié, et exécutez-le. +Si le système clickhouse-server est déjà en cours d'exécution et que vous ne voulez pas l'arrêter, vous pouvez modifier les numéros de port dans votre `config.xml` (ou de les remplacer dans un fichier `config.d` répertoire), fournissez le chemin de données approprié, et exécutez-le. -`clickhouse` binary n’a presque aucune dépendance et fonctionne sur un large éventail de distributions Linux. Rapide et sale de tester vos modifications sur un serveur, vous pouvez simplement `scp` votre douce construite `clickhouse` binaire à votre serveur et ensuite l’exécuter comme dans les exemples ci-dessus. +`clickhouse` binary n'a presque aucune dépendance et fonctionne sur un large éventail de distributions Linux. Rapide et sale de tester vos modifications sur un serveur, vous pouvez simplement `scp` votre douce construite `clickhouse` binaire à votre serveur et ensuite l'exécuter comme dans les exemples ci-dessus. -## L’Environnement De Test {#testing-environment} +## L'Environnement De Test {#testing-environment} -Avant de publier la version stable, nous la déployons sur l’environnement de test. L’environnement de test est un cluster processus 1/39 partie de [Yandex.Metrica](https://metrica.yandex.com/) données. Nous partageons notre environnement de test avec Yandex.Metrica de l’équipe. ClickHouse est mis à niveau sans temps d’arrêt au-dessus des données existantes. Nous regardons d’abord que les données sont traitées avec succès sans retard par rapport au temps réel, la réplication continue à fonctionner et il n’y a pas de problèmes visibles pour Yandex.Metrica de l’équipe. Première vérification peut être effectuée de la façon suivante: +Avant de publier la version stable, nous la déployons sur l'environnement de test. L'environnement de test est un cluster processus 1/39 partie de [Yandex.Metrica](https://metrica.yandex.com/) données. Nous partageons notre environnement de test avec Yandex.Metrica de l'équipe. ClickHouse est mis à niveau sans temps d'arrêt au-dessus des données existantes. Nous regardons d'abord que les données sont traitées avec succès sans retard par rapport au temps réel, la réplication continue à fonctionner et il n'y a pas de problèmes visibles pour Yandex.Metrica de l'équipe. Première vérification peut être effectuée de la façon suivante: ``` sql SELECT hostName() AS h, any(version()), any(uptime()), max(UTCEventTime), count() FROM remote('example01-01-{1..3}t', merge, hits) WHERE EventDate >= today() - 2 GROUP BY h ORDER BY h; ``` -Dans certains cas, nous déployons également à l’environnement de test de nos équipes d’amis dans Yandex: marché, Cloud, etc. Nous avons également des serveurs matériels qui sont utilisés à des fins de développement. +Dans certains cas, nous déployons également à l'environnement de test de nos équipes d'amis dans Yandex: marché, Cloud, etc. Nous avons également des serveurs matériels qui sont utilisés à des fins de développement. ## Les Tests De Charge {#load-testing} -Après le déploiement dans l’environnement de test, nous exécutons des tests de charge avec des requêtes du cluster de production. Ceci est fait manuellement. +Après le déploiement dans l'environnement de test, nous exécutons des tests de charge avec des requêtes du cluster de production. Ceci est fait manuellement. Assurez-vous que vous avez activé `query_log` sur votre cluster de production. @@ -131,9 +131,9 @@ Recueillir le journal des requêtes pour une journée ou plus: $ clickhouse-client --query="SELECT DISTINCT query FROM system.query_log WHERE event_date = today() AND query LIKE '%ym:%' AND query NOT LIKE '%system.query_log%' AND type = 2 AND is_initial_query" > queries.tsv ``` -C’est une façon compliquée exemple. `type = 2` filtrera les requêtes exécutées avec succès. `query LIKE '%ym:%'` est de sélectionner les requêtes de Yandex.Metrica. `is_initial_query` est de sélectionner uniquement les requêtes initiées par le client, pas par ClickHouse lui-même (en tant que partie du traitement de requête distribué). +C'est une façon compliquée exemple. `type = 2` filtrera les requêtes exécutées avec succès. `query LIKE '%ym:%'` est de sélectionner les requêtes de Yandex.Metrica. `is_initial_query` est de sélectionner uniquement les requêtes initiées par le client, pas par ClickHouse lui-même (en tant que partie du traitement de requête distribué). -`scp` ce journal à votre cluster de test et l’exécuter comme suit: +`scp` ce journal à votre cluster de test et l'exécuter comme suit: ``` bash $ clickhouse benchmark --concurrency 16 < queries.tsv @@ -143,13 +143,13 @@ $ clickhouse benchmark --concurrency 16 < queries.tsv Ensuite, laissez-le pour une nuit ou un week-end et allez vous reposer. -Tu devrais vérifier ça `clickhouse-server` ne plante pas, l’empreinte mémoire est limitée et les performances ne se dégradent pas au fil du temps. +Tu devrais vérifier ça `clickhouse-server` ne plante pas, l'empreinte mémoire est limitée et les performances ne se dégradent pas au fil du temps. -Les délais précis d’exécution des requêtes ne sont pas enregistrés et ne sont pas comparés en raison de la grande variabilité des requêtes et de l’environnement. +Les délais précis d'exécution des requêtes ne sont pas enregistrés et ne sont pas comparés en raison de la grande variabilité des requêtes et de l'environnement. ## Essais De Construction {#build-tests} -Les tests de construction permettent de vérifier que la construction n’est pas interrompue sur diverses configurations alternatives et sur certains systèmes étrangers. Les Tests sont situés à `ci` répertoire. Ils exécutent build from source à L’intérieur de Docker, Vagrant, et parfois avec `qemu-user-static` à l’intérieur de Docker. Ces tests sont en cours de développement et les essais ne sont pas automatisées. +Les tests de construction permettent de vérifier que la construction n'est pas interrompue sur diverses configurations alternatives et sur certains systèmes étrangers. Les Tests sont situés à `ci` répertoire. Ils exécutent build from source à L'intérieur de Docker, Vagrant, et parfois avec `qemu-user-static` à l'intérieur de Docker. Ces tests sont en cours de développement et les essais ne sont pas automatisées. Motivation: @@ -161,61 +161,71 @@ Normalement, nous libérons et exécutons tous les tests sur une seule variante - construire sur la plate-forme AArch64; - construire sur la plate-forme PowerPc. -Par exemple, construire avec des paquets système est une mauvaise pratique, car nous ne pouvons pas garantir quelle version exacte des paquets un système aura. Mais c’est vraiment nécessaire pour les responsables Debian. Pour cette raison, nous devons au moins soutenir cette variante de construction. Un autre exemple: la liaison partagée est une source commune de problèmes, mais elle est nécessaire pour certains amateurs. +Par exemple, construire avec des paquets système est une mauvaise pratique, car nous ne pouvons pas garantir quelle version exacte des paquets un système aura. Mais c'est vraiment nécessaire pour les responsables Debian. Pour cette raison, nous devons au moins soutenir cette variante de construction. Un autre exemple: la liaison partagée est une source commune de problèmes, mais elle est nécessaire pour certains amateurs. Bien que nous ne puissions pas exécuter tous les tests sur toutes les variantes de builds, nous voulons vérifier au moins que les différentes variantes de build ne sont pas cassées. Pour cela nous utilisons les essais de construction. -## Test De Compatibilité Du Protocole {#testing-for-protocol-compatibility} +## Test de compatibilité du protocole {#testing-for-protocol-compatibility} -Lorsque nous étendons le protocole réseau ClickHouse, nous testons manuellement que l’ancien clickhouse-client fonctionne avec le nouveau clickhouse-server et que le nouveau clickhouse-client fonctionne avec l’ancien clickhouse-server (simplement en exécutant des binaires à partir des paquets correspondants). +Lorsque nous étendons le protocole réseau ClickHouse, nous testons manuellement que l'ancien clickhouse-client fonctionne avec le nouveau clickhouse-server et que le nouveau clickhouse-client fonctionne avec l'ancien clickhouse-server (simplement en exécutant des binaires à partir des paquets correspondants). -## L’Aide Du Compilateur {#help-from-the-compiler} +## L'aide du Compilateur {#help-from-the-compiler} Code ClickHouse principal (qui est situé dans `dbms` annuaire) est construit avec `-Wall -Wextra -Werror` et avec quelques avertissements supplémentaires activés. Bien que ces options ne soient pas activées pour les bibliothèques tierces. Clang a des avertissements encore plus utiles - vous pouvez les chercher avec `-Weverything` et choisissez quelque chose à construire par défaut. -Pour les builds de production, gcc est utilisé (il génère toujours un code légèrement plus efficace que clang). Pour le développement, clang est généralement plus pratique à utiliser. Vous pouvez construire sur votre propre machine avec le mode débogage (pour économiser la batterie de votre ordinateur portable), mais veuillez noter que le compilateur est capable de générer plus d’Avertissements avec `-O3` grâce à une meilleure analyse du flux de contrôle et de l’inter-procédure. Lors de la construction avec clang, `libc++` est utilisé au lieu de `libstdc++` et lors de la construction avec le mode débogage, la version de débogage de `libc++` est utilisé qui permet d’attraper plus d’erreurs à l’exécution. +Pour les builds de production, gcc est utilisé (il génère toujours un code légèrement plus efficace que clang). Pour le développement, clang est généralement plus pratique à utiliser. Vous pouvez construire sur votre propre machine avec le mode débogage (pour économiser la batterie de votre ordinateur portable), mais veuillez noter que le compilateur est capable de générer plus d'Avertissements avec `-O3` grâce à une meilleure analyse du flux de contrôle et de l'inter-procédure. Lors de la construction avec clang, `libc++` est utilisé au lieu de `libstdc++` et lors de la construction avec le mode débogage, la version de débogage de `libc++` est utilisé qui permet d'attraper plus d'erreurs à l'exécution. ## Désinfectant {#sanitizers} -**Désinfectant d’adresse**. -Nous exécutons des tests fonctionnels et d’intégration sous ASan sur la base de per-commit. +**Désinfectant d'adresse**. +Nous exécutons des tests fonctionnels et d'intégration sous ASan sur la base de per-commit. **Valgrind (Memcheck)**. Nous effectuons des tests fonctionnels sous Valgrind pendant la nuit. Cela prend plusieurs heures. Actuellement il y a un faux positif connu dans `re2` bibliothèque, consultez [cet article](https://research.swtch.com/sparse). **Désinfectant de comportement indéfini.** -Nous exécutons des tests fonctionnels et d’intégration sous ASan sur la base de per-commit. +Nous exécutons des tests fonctionnels et d'intégration sous ASan sur la base de per-commit. **Désinfectant pour filetage**. -Nous exécutons des tests fonctionnels sous TSan sur la base de per-commit. Nous n’exécutons toujours pas de tests D’intégration sous TSan sur la base de la validation. +Nous exécutons des tests fonctionnels sous TSan sur la base de per-commit. Nous n'exécutons toujours pas de tests D'intégration sous TSan sur la base de la validation. **Mémoire de désinfectant**. -Actuellement, nous n’utilisons toujours pas MSan. +Actuellement, nous n'utilisons toujours pas MSan. **Débogueur allocateur.** Version de débogage de `jemalloc` est utilisé pour la construction de débogage. ## Fuzzing {#fuzzing} -Nous utilisons un simple test fuzz pour générer des requêtes SQL aléatoires et vérifier que le serveur ne meurt pas. Le test de Fuzz est effectué avec un désinfectant D’adresse. Vous pouvez le trouver dans `00746_sql_fuzzy.pl`. Ce test doit être exécuté en continu (pendant la nuit et plus longtemps). +Clickhouse fuzzing est implémenté à la fois en utilisant [libFuzzer](https://llvm.org/docs/LibFuzzer.html) et des requêtes SQL aléatoires. +Tous les tests de fuzz doivent être effectués avec des désinfectants (adresse et indéfini). -En décembre 2018, nous n’utilisons toujours pas de tests fuzz isolés du code de la bibliothèque. +LibFuzzer est utilisé pour les tests de fuzz isolés du code de la bibliothèque. Les Fuzzers sont implémentés dans le cadre du code de test et ont “\_fuzzer” nom postfixes. +Exemple Fuzzer peut être trouvé à `src/Parsers/tests/lexer_fuzzer.cpp`. Les configs, dictionnaires et corpus spécifiques à LibFuzzer sont stockés à `tests/fuzz`. +Nous vous encourageons à écrire des tests fuzz pour chaque fonctionnalité qui gère l'entrée de l'utilisateur. + +Fuzzers ne sont pas construits par défaut. Pour construire fuzzers à la fois `-DENABLE_FUZZING=1` et `-DENABLE_TESTS=1` options doivent être définies. +Nous vous recommandons de désactiver Jemalloc lors de la construction de fuzzers. Configuration utilisée pour intégrer clickhouse fuzzing à +Google OSS-Fuzz peut être trouvé à `docker/fuzz`. + +Nous utilisons également un simple test fuzz pour générer des requêtes SQL aléatoires et vérifier que le serveur ne meurt pas en les exécutant. +Vous pouvez le trouver dans `00746_sql_fuzzy.pl`. Ce test doit être exécuté en continu (pendant la nuit et plus longtemps). ## Audit De Sécurité {#security-audit} -Les gens du Département Cloud de Yandex font un aperçu de base des capacités de ClickHouse du point de vue de la sécurité. +Les gens de L'équipe de sécurité Yandex font un aperçu de base des capacités de ClickHouse du point de vue de la sécurité. ## Analyseurs Statiques {#static-analyzers} -Nous courons `PVS-Studio` par commettre base. Nous avons évalué `clang-tidy`, `Coverity`, `cppcheck`, `PVS-Studio`, `tscancode`. Vous trouverez des instructions pour l’utilisation dans `tests/instructions/` répertoire. Aussi, vous pouvez lire [l’article en russe](https://habr.com/company/yandex/blog/342018/). +Nous courons `PVS-Studio` par commettre base. Nous avons évalué `clang-tidy`, `Coverity`, `cppcheck`, `PVS-Studio`, `tscancode`. Vous trouverez des instructions pour l'utilisation dans `tests/instructions/` répertoire. Aussi, vous pouvez lire [l'article en russe](https://habr.com/company/yandex/blog/342018/). -Si vous utilisez `CLion` en tant QU’IDE, vous pouvez tirer parti de certains `clang-tidy` contrôles de la boîte. +Si vous utilisez `CLion` en tant QU'IDE, vous pouvez tirer parti de certains `clang-tidy` contrôles de la boîte. ## Durcir {#hardening} -`FORTIFY_SOURCE` est utilisé par défaut. C’est presque inutile, mais cela a toujours du sens dans de rares cas et nous ne le désactivons pas. +`FORTIFY_SOURCE` est utilisé par défaut. C'est presque inutile, mais cela a toujours du sens dans de rares cas et nous ne le désactivons pas. ## Code De Style {#code-style} @@ -223,7 +233,7 @@ Les règles de style de Code sont décrites [ici](https://clickhouse.tech/docs/e Pour vérifier certaines violations de style courantes, vous pouvez utiliser `utils/check-style` script. -Pour forcer le style approprié de votre code, vous pouvez utiliser `clang-format`. Fichier `.clang-format` est situé à la racine des sources. Il correspond principalement à notre style de code réel. Mais il n’est pas recommandé d’appliquer `clang-format` pour les fichiers existants, car il rend le formatage pire. Vous pouvez utiliser `clang-format-diff` outil que vous pouvez trouver dans clang référentiel source. +Pour forcer le style approprié de votre code, vous pouvez utiliser `clang-format`. Fichier `.clang-format` est situé à la racine des sources. Il correspond principalement à notre style de code réel. Mais il n'est pas recommandé d'appliquer `clang-format` pour les fichiers existants, car il rend le formatage pire. Vous pouvez utiliser `clang-format-diff` outil que vous pouvez trouver dans clang référentiel source. Alternativement vous pouvez essayer `uncrustify` outil pour reformater votre code. La Configuration est en `uncrustify.cfg` dans la racine des sources. Il est moins testé que `clang-format`. @@ -231,7 +241,7 @@ Alternativement vous pouvez essayer `uncrustify` outil pour reformater votre cod ## Tests Metrica B2B {#metrica-b2b-tests} -Chaque version de ClickHouse est testée avec les moteurs Yandex Metrica et AppMetrica. Les versions de test et stables de ClickHouse sont déployées sur des machines virtuelles et exécutées avec une petite copie de metrica engine qui traite un échantillon fixe de données d’entrée. Ensuite, les résultats de deux instances de metrica engine sont comparés ensemble. +Chaque version de ClickHouse est testée avec les moteurs Yandex Metrica et AppMetrica. Les versions de test et stables de ClickHouse sont déployées sur des machines virtuelles et exécutées avec une petite copie de metrica engine qui traite un échantillon fixe de données d'entrée. Ensuite, les résultats de deux instances de metrica engine sont comparés ensemble. Ces tests sont automatisés par une équipe distincte. En raison du nombre élevé de pièces en mouvement, les tests échouent la plupart du temps complètement raisons, qui sont très difficiles à comprendre. Très probablement, ces tests ont une valeur négative pour nous. Néanmoins, ces tests se sont révélés utiles dans environ une ou deux fois sur des centaines. @@ -241,12 +251,11 @@ En juillet 2018, nous ne suivons pas la couverture des tests. ## Automatisation Des Tests {#test-automation} -Nous exécutons des tests avec Yandex CI interne et le système d’automatisation des tâches nommé “Sandbox”. +Nous exécutons des tests avec Yandex CI interne et le système d'automatisation des tâches nommé “Sandbox”. -Les travaux de construction et les tests sont exécutés dans Sandbox sur une base de validation. Les paquets résultants et les résultats des tests sont publiés dans GitHub et peuvent être téléchargés par des liens directs. Les artefacts sont stockés éternellement. Lorsque vous envoyez une demande de tirage sur GitHub, nous l’étiquetons comme “can be tested” et notre système CI construira des paquets ClickHouse (release, debug, avec un désinfectant d’adresse, etc.) pour vous. +Les travaux de construction et les tests sont exécutés dans Sandbox sur une base de validation. Les paquets résultants et les résultats des tests sont publiés dans GitHub et peuvent être téléchargés par des liens directs. Les artefacts sont stockés éternellement. Lorsque vous envoyez une demande de tirage sur GitHub, nous l'étiquetons comme “can be tested” et notre système CI construira des paquets ClickHouse (release, debug, avec un désinfectant d'adresse, etc.) pour vous. -Nous n’utilisons pas Travis CI en raison de la limite de temps et de puissance de calcul. -On n’utilise pas Jenkins. Il a été utilisé avant et maintenant nous sommes heureux de ne pas utiliser Jenkins. +Nous n'utilisons pas Travis CI en raison de la limite de temps et de puissance de calcul. +On n'utilise pas Jenkins. Il a été utilisé avant et maintenant nous sommes heureux de ne pas utiliser Jenkins. [Article Original](https://clickhouse.tech/docs/en/development/tests/) -développement/tests/) diff --git a/docs/fr/engines/database-engines/index.md b/docs/fr/engines/database-engines/index.md index fe44cdaf558..a5b08b63453 100644 --- a/docs/fr/engines/database-engines/index.md +++ b/docs/fr/engines/database-engines/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 -toc_folder_title: Database Engines +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "Moteurs De Base De Donn\xE9es" toc_priority: 27 toc_title: Introduction --- diff --git a/docs/fr/engines/database-engines/lazy.md b/docs/fr/engines/database-engines/lazy.md index 69cba3ebba3..545936ae861 100644 --- a/docs/fr/engines/database-engines/lazy.md +++ b/docs/fr/engines/database-engines/lazy.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 31 toc_title: Paresseux --- @@ -11,7 +11,7 @@ Conserve les tables en RAM uniquement `expiration_time_in_seconds` secondes apr Il est optimisé pour stocker de nombreuses petites tables \*Log, pour lesquelles il y a un long intervalle de temps entre les accès. -## La création d’une Base De données {#creating-a-database} +## La création d'une Base de données {#creating-a-database} CREATE DATABASE testlazy ENGINE = Lazy(expiration_time_in_seconds); diff --git a/docs/fr/engines/database-engines/mysql.md b/docs/fr/engines/database-engines/mysql.md index 8b76c870cc7..b40c8961b92 100644 --- a/docs/fr/engines/database-engines/mysql.md +++ b/docs/fr/engines/database-engines/mysql.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 30 toc_title: MySQL --- -# Mysql {#mysql} +# MySQL {#mysql} Permet de se connecter à des bases de données sur un serveur MySQL distant et `INSERT` et `SELECT` requêtes pour échanger des données entre Clickhouse et MySQL. @@ -17,11 +17,11 @@ Vous ne pouvez pas effectuer les requêtes suivantes: - `CREATE TABLE` - `ALTER` -## La création d’une Base De données {#creating-a-database} +## La création d'une Base de données {#creating-a-database} ``` sql CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster] -ENGINE = MySQL('host:port', 'database', 'user', 'password') +ENGINE = MySQL('host:port', ['database' | database], 'user', 'password') ``` **Les Paramètres Du Moteur** @@ -53,7 +53,7 @@ Tous les autres types de données MySQL sont convertis en [Chaîne](../../sql-re [Nullable](../../sql-reference/data-types/nullable.md) est pris en charge. -## Exemples D’Utilisation {#examples-of-use} +## Exemples D'utilisation {#examples-of-use} Table dans MySQL: diff --git a/docs/fr/engines/index.md b/docs/fr/engines/index.md index af36619876c..3dfb7bf3748 100644 --- a/docs/fr/engines/index.md +++ b/docs/fr/engines/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 -toc_folder_title: Engines +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: Moteur toc_priority: 25 --- diff --git a/docs/fr/engines/table-engines/index.md b/docs/fr/engines/table-engines/index.md index e672bc846fe..a05ab19868c 100644 --- a/docs/fr/engines/table-engines/index.md +++ b/docs/fr/engines/table-engines/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 -toc_folder_title: Table Engines +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: Moteurs De Table toc_priority: 26 toc_title: Introduction --- @@ -14,14 +14,14 @@ Le moteur de table (type de table) détermine: - Quelles requêtes sont prises en charge et comment. - Accès simultané aux données. - Utilisation des index, si elle est présente. -- Indique si l’exécution d’une requête multithread est possible. +- Indique si l'exécution d'une requête multithread est possible. - Paramètres de réplication des données. ## Familles De Moteurs {#engine-families} -### Mergetree {#mergetree} +### MergeTree {#mergetree} -Les moteurs de table les plus universels et fonctionnels pour les tâches à forte charge. La propriété partagée par ces moteurs est l’insertion rapide des données avec traitement ultérieur des données d’arrière-plan. `MergeTree` les moteurs de la famille prennent en charge la réplication des données (avec [Répliqué\*](mergetree-family/replication.md#replication) versions de moteurs), le partitionnement, et d’autres fonctionnalités non prises en charge dans d’autres moteurs. +Les moteurs de table les plus universels et fonctionnels pour les tâches à forte charge. La propriété partagée par ces moteurs est l'insertion rapide des données avec traitement ultérieur des données d'arrière-plan. `MergeTree` les moteurs de la famille prennent en charge la réplication des données (avec [Répliqué\*](mergetree-family/replication.md#table_engines-replication) versions de moteurs), le partitionnement, et d'autres fonctionnalités non prises en charge dans d'autres moteurs. Moteurs dans la famille: @@ -35,7 +35,7 @@ Moteurs dans la famille: ### Journal {#log} -Léger [moteur](log-family/index.md) avec une fonctionnalité minimale. Ils sont les plus efficaces lorsque vous devez écrire rapidement de nombreuses petites tables (jusqu’à environ 1 million de lignes) et les lire plus tard dans leur ensemble. +Léger [moteur](log-family/index.md) avec une fonctionnalité minimale. Ils sont les plus efficaces lorsque vous devez écrire rapidement de nombreuses petites tables (jusqu'à environ 1 million de lignes) et les lire plus tard dans leur ensemble. Moteurs dans la famille: @@ -43,9 +43,9 @@ Moteurs dans la famille: - [StripeLog](log-family/stripelog.md#stripelog) - [Journal](log-family/log.md#log) -### Moteurs D’Intégration {#integration-engines} +### Moteurs D'Intégration {#integration-engines} -Moteurs de communication avec d’autres systèmes de stockage et de traitement de données. +Moteurs de communication avec d'autres systèmes de stockage et de traitement de données. Moteurs dans la famille: @@ -62,7 +62,7 @@ Moteurs dans la famille: - [Distribué](special/distributed.md#distributed) - [MaterializedView](special/materializedview.md#materializedview) - [Dictionnaire](special/dictionary.md#dictionary) -- [Fusionner](special/merge.md#merge +- \[Fusion\](spécial/de fusion.md\#fusion - [Fichier](special/file.md#file) - [NULL](special/null.md#null) - [Définir](special/set.md#set) @@ -80,6 +80,6 @@ Vous ne devez pas spécifier de colonnes virtuelles dans `CREATE TABLE` requête Pour sélectionner des données dans une colonne virtuelle, vous devez spécifier son nom `SELECT` requête. `SELECT *` ne renvoie pas de valeurs à partir de colonnes virtuelles. -Si vous créez une table avec une colonne portant le même nom que l’une des colonnes virtuelles de la table, la colonne virtuelle devient inaccessible. Nous ne recommandons pas de faire cela. Pour éviter les conflits, les noms de colonnes virtuelles sont généralement précédés d’un trait de soulignement. +Si vous créez une table avec une colonne portant le même nom que l'une des colonnes virtuelles de la table, la colonne virtuelle devient inaccessible. Nous ne recommandons pas de faire cela. Pour éviter les conflits, les noms de colonnes virtuelles sont généralement précédés d'un trait de soulignement. [Article Original](https://clickhouse.tech/docs/en/operations/table_engines/) diff --git a/docs/fr/engines/table-engines/integrations/hdfs.md b/docs/fr/engines/table-engines/integrations/hdfs.md index 8f8da8f8126..fd207793b02 100644 --- a/docs/fr/engines/table-engines/integrations/hdfs.md +++ b/docs/fr/engines/table-engines/integrations/hdfs.md @@ -1,14 +1,14 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 36 toc_title: HDFS --- # HDFS {#table_engines-hdfs} -Ce moteur fournit l’intégration avec [Apache Hadoop](https://en.wikipedia.org/wiki/Apache_Hadoop) l’écosystème en permettant de gérer les données sur [HDFS](https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/HdfsDesign.html)via ClickHouse. Ce moteur est similaire -à l’ [Fichier](../special/file.md#table_engines-file) et [URL](../special/url.md#table_engines-url) moteurs, mais fournit des fonctionnalités spécifiques Hadoop. +Ce moteur fournit l'intégration avec [Apache Hadoop](https://en.wikipedia.org/wiki/Apache_Hadoop) l'écosystème en permettant de gérer les données sur [HDFS](https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/HdfsDesign.html)via ClickHouse. Ce moteur est similaire +à l' [Fichier](../special/file.md#table_engines-file) et [URL](../special/url.md#table_engines-url) moteurs, mais fournit des fonctionnalités spécifiques Hadoop. ## Utilisation {#usage} @@ -16,9 +16,9 @@ Ce moteur fournit l’intégration avec [Apache Hadoop](https://en.wikipedia.org ENGINE = HDFS(URI, format) ``` -Le `URI` paramètre est L’URI du fichier entier dans HDFS. -Le `format` paramètre spécifie l’un des formats de fichier disponibles. Effectuer -`SELECT` requêtes, le format doit être pris en charge pour l’entrée, et à effectuer +Le `URI` paramètre est L'URI du fichier entier dans HDFS. +Le `format` paramètre spécifie l'un des formats de fichier disponibles. Effectuer +`SELECT` requêtes, le format doit être pris en charge pour l'entrée, et à effectuer `INSERT` queries – for output. The available formats are listed in the [Format](../../../interfaces/formats.md#formats) section. Le chemin le cadre de `URI` peut contenir des globules. Dans ce cas, le tableau serait en lecture seule. @@ -60,14 +60,14 @@ SELECT * FROM hdfs_engine_table LIMIT 2 **Globs dans le chemin** -Plusieurs composants de chemin peuvent avoir des globs. Pour être traité, le fichier devrait exister et correspondre au modèle de chemin entier. Liste des fichiers détermine pendant `SELECT` (pas à l’ `CREATE` moment). +Plusieurs composants de chemin peuvent avoir des globs. Pour être traité, le fichier devrait exister et correspondre au modèle de chemin entier. Liste des fichiers détermine pendant `SELECT` (pas à l' `CREATE` moment). - `*` — Substitutes any number of any characters except `/` y compris la chaîne vide. - `?` — Substitutes any single character. - `{some_string,another_string,yet_another_one}` — Substitutes any of strings `'some_string', 'another_string', 'yet_another_one'`. - `{N..M}` — Substitutes any number in range from N to M including both borders. -Les Constructions avec `{}` sont similaires à l’ [distant](../../../sql-reference/table-functions/remote.md) table de fonction. +Les Constructions avec `{}` sont similaires à l' [distant](../../../sql-reference/table-functions/remote.md) table de fonction. **Exemple** diff --git a/docs/fr/engines/table-engines/integrations/index.md b/docs/fr/engines/table-engines/integrations/index.md index 705aa507d6a..205b631880b 100644 --- a/docs/fr/engines/table-engines/integrations/index.md +++ b/docs/fr/engines/table-engines/integrations/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 -toc_folder_title: Integrations +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "Int\xE9gration" toc_priority: 30 --- diff --git a/docs/fr/engines/table-engines/integrations/jdbc.md b/docs/fr/engines/table-engines/integrations/jdbc.md index 728dd873a27..e67209f2709 100644 --- a/docs/fr/engines/table-engines/integrations/jdbc.md +++ b/docs/fr/engines/table-engines/integrations/jdbc.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 34 toc_title: JDBC --- @@ -13,7 +13,7 @@ Pour implémenter la connexion JDBC, ClickHouse utilise le programme séparé [c Ce moteur prend en charge le [Nullable](../../../sql-reference/data-types/nullable.md) type de données. -## Création d’une Table {#creating-a-table} +## Création d'une Table {#creating-a-table} ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name @@ -34,9 +34,9 @@ ENGINE = JDBC(dbms_uri, external_database, external_table) - `external_table` — Name of the table in `external_database`. -## Exemple D’Utilisation {#usage-example} +## Exemple D'Utilisation {#usage-example} -Création d’une table dans le serveur MySQL en se connectant directement avec son client console: +Création d'une table dans le serveur MySQL en se connectant directement avec son client console: ``` text mysql> CREATE TABLE `test`.`test` ( @@ -59,7 +59,7 @@ mysql> select * from test; 1 row in set (0,00 sec) ``` -Création d’une table dans le serveur ClickHouse et sélection des données: +Création d'une table dans le serveur ClickHouse et sélection des données: ``` sql CREATE TABLE jdbc_table diff --git a/docs/fr/engines/table-engines/integrations/kafka.md b/docs/fr/engines/table-engines/integrations/kafka.md index ff91cf140e5..d4a2de42f22 100644 --- a/docs/fr/engines/table-engines/integrations/kafka.md +++ b/docs/fr/engines/table-engines/integrations/kafka.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 32 toc_title: Kafka --- @@ -11,11 +11,11 @@ Ce moteur fonctionne avec [Apache Kafka](http://kafka.apache.org/). Kafka vous permet de: -- Publier ou s’abonner aux flux de données. +- Publier ou s'abonner aux flux de données. - Organiser le stockage tolérant aux pannes. -- Traiter les flux à mesure qu’ils deviennent disponibles. +- Traiter les flux à mesure qu'ils deviennent disponibles. -## Création d’une Table {#table_engine-kafka-creating-a-table} +## Création d'une Table {#table_engine-kafka-creating-a-table} ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] @@ -32,22 +32,26 @@ SETTINGS [kafka_row_delimiter = 'delimiter_symbol',] [kafka_schema = '',] [kafka_num_consumers = N,] - [kafka_skip_broken_messages = N] + [kafka_max_block_size = 0,] + [kafka_skip_broken_messages = N,] + [kafka_commit_every_batch = 0] ``` Les paramètres requis: - `kafka_broker_list` – A comma-separated list of brokers (for example, `localhost:9092`). - `kafka_topic_list` – A list of Kafka topics. -- `kafka_group_name` – A group of Kafka consumers. Reading margins are tracked for each group separately. If you don’t want messages to be duplicated in the cluster, use the same group name everywhere. -- `kafka_format` – Message format. Uses the same notation as the SQL `FORMAT` la fonction, tels que `JSONEachRow`. Pour plus d’informations, voir le [Format](../../../interfaces/formats.md) section. +- `kafka_group_name` – A group of Kafka consumers. Reading margins are tracked for each group separately. If you don't want messages to be duplicated in the cluster, use the same group name everywhere. +- `kafka_format` – Message format. Uses the same notation as the SQL `FORMAT` la fonction, tels que `JSONEachRow`. Pour plus d'informations, voir le [Format](../../../interfaces/formats.md) section. Paramètres facultatifs: - `kafka_row_delimiter` – Delimiter character, which ends the message. -- `kafka_schema` – Parameter that must be used if the format requires a schema definition. For example, [Cap’n Proto](https://capnproto.org/) nécessite le chemin d’accès du fichier de schéma et le nom de la racine `schema.capnp:Message` objet. -- `kafka_num_consumers` – The number of consumers per table. Default: `1`. Spécifiez plus de consommateurs si le débit d’un consommateur est insuffisant. Le nombre total de consommateurs ne doit pas dépasser le nombre de partitions dans la rubrique, car un seul consommateur peut être affecté par partition. +- `kafka_schema` – Parameter that must be used if the format requires a schema definition. For example, [Cap'n Proto](https://capnproto.org/) nécessite le chemin d'accès du fichier de schéma et le nom de la racine `schema.capnp:Message` objet. +- `kafka_num_consumers` – The number of consumers per table. Default: `1`. Spécifiez plus de consommateurs si le débit d'un consommateur est insuffisant. Le nombre total de consommateurs ne doit pas dépasser le nombre de partitions dans la rubrique, car un seul consommateur peut être affecté par partition. +- `kafka_max_block_size` - La taille maximale du lot (dans les messages) pour le sondage (par défaut: `max_block_size`). - `kafka_skip_broken_messages` – Kafka message parser tolerance to schema-incompatible messages per block. Default: `0`. Si `kafka_skip_broken_messages = N` puis le moteur saute *N* Messages Kafka qui ne peuvent pas être analysés (un message est égal à une ligne de données). +- `kafka_commit_every_batch` - Commit chaque lot consommé et traité au lieu d'un seul commit après avoir écrit un bloc entier (par défaut: `0`). Exemple: @@ -84,7 +88,7 @@ Exemple: Méthode obsolète pour créer une Table !!! attention "Attention" - N’utilisez pas cette méthode dans les nouveaux projets. Si possible, optez anciens projets à la méthode décrite ci-dessus. + N'utilisez pas cette méthode dans les nouveaux projets. Si possible, optez anciens projets à la méthode décrite ci-dessus. ``` sql Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format @@ -95,17 +99,17 @@ Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format ## Description {#description} -Les messages livrés sont suivis automatiquement, de sorte que chaque message d’un groupe n’est compté qu’une seule fois. Si vous souhaitez obtenir les données deux fois, créez une copie de la table avec un autre nom de groupe. +Les messages livrés sont suivis automatiquement, de sorte que chaque message d'un groupe n'est compté qu'une seule fois. Si vous souhaitez obtenir les données deux fois, créez une copie de la table avec un autre nom de groupe. -Les groupes sont flexibles et synchronisés sur le cluster. Par exemple, si vous avez 10 thèmes et 5 copies d’une table dans un cluster, chaque copie obtient 2 sujets. Si le nombre de copies change, les rubriques sont redistribuées automatiquement entre les copies. En savoir plus à ce sujet à http://kafka.apache.org/intro. +Les groupes sont flexibles et synchronisés sur le cluster. Par exemple, si vous avez 10 thèmes et 5 copies d'une table dans un cluster, chaque copie obtient 2 sujets. Si le nombre de copies change, les rubriques sont redistribuées automatiquement entre les copies. En savoir plus à ce sujet à http://kafka.apache.org/intro. -`SELECT` n’est pas particulièrement utile pour la lecture de messages (sauf pour le débogage), car chaque message ne peut être lu qu’une seule fois. Il est plus pratique de créer des threads en temps réel à l’aide de vues matérialisées. Pour ce faire: +`SELECT` n'est pas particulièrement utile pour la lecture de messages (sauf pour le débogage), car chaque message ne peut être lu qu'une seule fois. Il est plus pratique de créer des threads en temps réel à l'aide de vues matérialisées. Pour ce faire: 1. Utilisez le moteur pour créer un consommateur Kafka et considérez-le comme un flux de données. 2. Créez une table avec la structure souhaitée. 3. Créer une vue matérialisée qui convertit les données du moteur et le met dans une table créée précédemment. -Lorsque l’ `MATERIALIZED VIEW` rejoint le moteur, il commence à collecter des données en arrière-plan. Cela vous permet de recevoir continuellement des messages de Kafka et de les convertir au format requis en utilisant `SELECT`. +Lorsque l' `MATERIALIZED VIEW` rejoint le moteur, il commence à collecter des données en arrière-plan. Cela vous permet de recevoir continuellement des messages de Kafka et de les convertir au format requis en utilisant `SELECT`. Une table kafka peut avoir autant de vues matérialisées que vous le souhaitez, elles ne lisent pas directement les données de la table kafka, mais reçoivent de nouveaux enregistrements( en blocs), de cette façon vous pouvez écrire sur plusieurs tables avec différents niveaux de détail (avec regroupement - agrégation et sans). Exemple: @@ -130,7 +134,7 @@ Exemple: SELECT level, sum(total) FROM daily GROUP BY level; ``` -Pour améliorer les performances, les messages reçus sont regroupées en blocs de la taille de [max\_insert\_block\_size](../../../operations/server-configuration-parameters/settings.md#settings-max_insert_block_size). Si le bloc n’a pas été formé à l’intérieur [stream\_flush\_interval\_ms](../../../operations/server-configuration-parameters/settings.md) millisecondes, les données seront vidées dans le tableau, indépendamment de l’intégralité du bloc. +Pour améliorer les performances, les messages reçus sont regroupées en blocs de la taille de [max\_insert\_block\_size](../../../operations/server-configuration-parameters/settings.md#settings-max_insert_block_size). Si le bloc n'a pas été formé à l'intérieur [stream\_flush\_interval\_ms](../../../operations/server-configuration-parameters/settings.md) millisecondes, les données seront vidées dans le tableau, indépendamment de l'intégralité du bloc. Pour arrêter de recevoir des données de rubrique ou pour modifier la logique de conversion, détachez la vue matérialisée: @@ -143,7 +147,7 @@ Si vous souhaitez modifier la table cible en utilisant `ALTER`, nous vous recomm ## Configuration {#configuration} -Similaire à GraphiteMergeTree, le moteur Kafka prend en charge la configuration étendue à l’aide du fichier de configuration ClickHouse. Il y a deux clés de configuration que vous pouvez utiliser: global (`kafka`) et des rubriques (`kafka_*`). La configuration globale est appliquée en premier, puis la configuration au niveau de la rubrique est appliquée (si elle existe). +Similaire à GraphiteMergeTree, le moteur Kafka prend en charge la configuration étendue à l'aide du fichier de configuration ClickHouse. Il y a deux clés de configuration que vous pouvez utiliser: global (`kafka`) et des rubriques (`kafka_*`). La configuration globale est appliquée en premier, puis la configuration au niveau de la rubrique est appliquée (si elle existe). ``` xml @@ -159,7 +163,7 @@ Similaire à GraphiteMergeTree, le moteur Kafka prend en charge la configuration ``` -Pour obtenir une liste des options de configuration possibles, consultez [librdkafka référence de configuration](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md). Utilisez le trait de soulignement (`_`) au lieu d’un point dans la configuration ClickHouse. Exemple, `check.crcs=true` sera `true`. +Pour obtenir une liste des options de configuration possibles, consultez [librdkafka référence de configuration](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md). Utilisez le trait de soulignement (`_`) au lieu d'un point dans la configuration ClickHouse. Exemple, `check.crcs=true` sera `true`. ## Les Colonnes Virtuelles {#virtual-columns} diff --git a/docs/fr/engines/table-engines/integrations/mysql.md b/docs/fr/engines/table-engines/integrations/mysql.md index d44ac762d0b..bc291e0aebe 100644 --- a/docs/fr/engines/table-engines/integrations/mysql.md +++ b/docs/fr/engines/table-engines/integrations/mysql.md @@ -1,15 +1,15 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 33 toc_title: MySQL --- # Mysql {#mysql} -Le moteur MySQL vous permet d’effectuer `SELECT` requêtes sur les données stockées sur un serveur MySQL distant. +Le moteur MySQL vous permet d'effectuer `SELECT` requêtes sur les données stockées sur un serveur MySQL distant. -## Création d’une Table {#creating-a-table} +## Création d'une Table {#creating-a-table} ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] @@ -22,10 +22,10 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] Voir une description détaillée de la [CREATE TABLE](../../../sql-reference/statements/create.md#create-table-query) requête. -La structure de la table peut différer de la structure de la table MySQL d’origine: +La structure de la table peut différer de la structure de la table MySQL d'origine: -- Les noms de colonnes doivent être les mêmes que dans la table MySQL d’origine, mais vous pouvez utiliser seulement certaines de ces colonnes et dans n’importe quel ordre. -- Les types de colonnes peuvent différer de ceux de la table MySQL d’origine. ClickHouse essaie de [jeter](../../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) valeurs des types de données ClickHouse. +- Les noms de colonnes doivent être les mêmes que dans la table MySQL d'origine, mais vous pouvez utiliser seulement certaines de ces colonnes et dans n'importe quel ordre. +- Les types de colonnes peuvent différer de ceux de la table MySQL d'origine. ClickHouse essaie de [jeter](../../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) valeurs des types de données ClickHouse. **Les Paramètres Du Moteur** @@ -45,13 +45,13 @@ La structure de la table peut différer de la structure de la table MySQL d’or Exemple: `INSERT INTO t (c1,c2) VALUES ('a', 2) ON DUPLICATE KEY UPDATE c2 = c2 + 1`, où `on_duplicate_clause` être `UPDATE c2 = c2 + 1`. Voir la [Documentation de MySQL](https://dev.mysql.com/doc/refman/8.0/en/insert-on-duplicate.html) pour trouver lequel `on_duplicate_clause` vous pouvez utiliser avec le `ON DUPLICATE KEY` clause. - Spécifier `on_duplicate_clause` vous avez besoin de passer `0` à l’ `replace_query` paramètre. Si vous passez simultanément `replace_query = 1` et `on_duplicate_clause`, Clickhouse génère une exception. + Spécifier `on_duplicate_clause` vous avez besoin de passer `0` à l' `replace_query` paramètre. Si vous passez simultanément `replace_query = 1` et `on_duplicate_clause`, Clickhouse génère une exception. Simple `WHERE` des clauses telles que `=, !=, >, >=, <, <=` sont exécutés sur le serveur MySQL. -Le reste des conditions et le `LIMIT` les contraintes d’échantillonnage sont exécutées dans ClickHouse uniquement après la fin de la requête à MySQL. +Le reste des conditions et le `LIMIT` les contraintes d'échantillonnage sont exécutées dans ClickHouse uniquement après la fin de la requête à MySQL. -## Exemple D’Utilisation {#usage-example} +## Exemple D'Utilisation {#usage-example} Table dans MySQL: diff --git a/docs/fr/engines/table-engines/integrations/odbc.md b/docs/fr/engines/table-engines/integrations/odbc.md index 8f24024602e..ee0ffb171d2 100644 --- a/docs/fr/engines/table-engines/integrations/odbc.md +++ b/docs/fr/engines/table-engines/integrations/odbc.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 35 toc_title: ODBC --- @@ -13,7 +13,7 @@ Pour implémenter en toute sécurité les connexions ODBC, ClickHouse utilise un Ce moteur prend en charge le [Nullable](../../../sql-reference/data-types/nullable.md) type de données. -## Création d’une Table {#creating-a-table} +## Création d'une Table {#creating-a-table} ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] @@ -29,7 +29,7 @@ Voir une description détaillée de la [CREATE TABLE](../../../sql-reference/sta La structure de la table peut différer de la structure de la table source: -- Les noms de colonnes doivent être les mêmes que dans la table source, mais vous pouvez utiliser quelques-unes de ces colonnes et dans n’importe quel ordre. +- Les noms de colonnes doivent être les mêmes que dans la table source, mais vous pouvez utiliser quelques-unes de ces colonnes et dans n'importe quel ordre. - Les types de colonnes peuvent différer de ceux de la table source. ClickHouse essaie de [jeter](../../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) valeurs des types de données ClickHouse. **Les Paramètres Du Moteur** @@ -38,15 +38,15 @@ La structure de la table peut différer de la structure de la table source: - `external_database` — Name of a database in an external DBMS. - `external_table` — Name of a table in the `external_database`. -## Exemple D’Utilisation {#usage-example} +## Exemple D'Utilisation {#usage-example} -**Récupération des données de L’installation MySQL locale via ODBC** +**Récupération des données de L'installation MySQL locale via ODBC** Cet exemple est vérifié pour Ubuntu Linux 18.04 et MySQL server 5.7. Assurez-vous que unixODBC et MySQL Connector sont installés. -Par défaut (si installé à partir de paquets), ClickHouse démarre en tant qu’utilisateur `clickhouse`. Ainsi, vous devez créer et configurer cet utilisateur dans le serveur MySQL. +Par défaut (si installé à partir de paquets), ClickHouse démarre en tant qu'utilisateur `clickhouse`. Ainsi, vous devez créer et configurer cet utilisateur dans le serveur MySQL. ``` bash $ sudo mysql @@ -70,7 +70,7 @@ USERNAME = clickhouse PASSWORD = clickhouse ``` -Vous pouvez vérifier la connexion en utilisant le `isql` utilitaire de l’installation unixODBC. +Vous pouvez vérifier la connexion en utilisant le `isql` utilitaire de l'installation unixODBC. ``` bash $ isql -v mysqlconn diff --git a/docs/fr/engines/table-engines/log-family/index.md b/docs/fr/engines/table-engines/log-family/index.md index c698303a8ac..9b94463744c 100644 --- a/docs/fr/engines/table-engines/log-family/index.md +++ b/docs/fr/engines/table-engines/log-family/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 -toc_folder_title: Log Family +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: Journal De La Famille toc_priority: 29 --- diff --git a/docs/fr/engines/table-engines/log-family/log-family.md b/docs/fr/engines/table-engines/log-family/log-family.md index b13155e2e8f..4ff840b9699 100644 --- a/docs/fr/engines/table-engines/log-family/log-family.md +++ b/docs/fr/engines/table-engines/log-family/log-family.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 31 toc_title: Introduction --- # Famille De Moteurs En Rondins {#log-engine-family} -Ces moteurs ont été développés pour les scénarios où vous devez écrire rapidement de nombreuses petites tables (jusqu’à environ 1 million de lignes) et les lire plus tard dans leur ensemble. +Ces moteurs ont été développés pour les scénarios où vous devez écrire rapidement de nombreuses petites tables (jusqu'à environ 1 million de lignes) et les lire plus tard dans leur ensemble. Les moteurs de la famille: @@ -21,11 +21,11 @@ Moteur: - Stocker des données sur un disque. -- Ajouter des données à la fin du fichier lors de l’écriture. +- Ajouter des données à la fin du fichier lors de l'écriture. -- Bloque simultanées dans l’accès aux données. +- Bloque simultanées dans l'accès aux données. - Lors `INSERT` requêtes, la table est verrouillée, et d’autres requêtes pour la lecture et l’écriture de données attendent que la table se déverrouille. S’il n’y a pas de requêtes d’écriture de données, un certain nombre de requêtes de lecture de données peuvent être effectuées simultanément. + Lors `INSERT` requêtes, la table est verrouillée, et d'autres requêtes pour la lecture et l'écriture de données attendent que la table se déverrouille. S'il n'y a pas de requêtes d'écriture de données, un certain nombre de requêtes de lecture de données peuvent être effectuées simultanément. - Ne prennent pas en charge [mutation](../../../sql-reference/statements/alter.md#alter-mutations) opérations. @@ -33,14 +33,14 @@ Moteur: Cela signifie que `SELECT` les requêtes pour les plages de données ne sont pas efficaces. -- N’écrivez pas de données de manière atomique. +- N'écrivez pas de données de manière atomique. - Vous pouvez obtenir une table avec des données corrompues si quelque chose interrompt l’opération d’écriture, par exemple, un arrêt anormal du serveur. + Vous pouvez obtenir une table avec des données corrompues si quelque chose interrompt l'opération d'écriture, par exemple, un arrêt anormal du serveur. ## Différence {#differences} Le `TinyLog` le moteur est le plus simple de la famille et offre la fonctionnalité la plus pauvre et la plus faible efficacité. Le `TinyLog` le moteur ne prend pas en charge la lecture de données parallèles par plusieurs threads. Il lit les données plus lentement que les autres moteurs de la famille qui prennent en charge la lecture parallèle et utilise presque autant de descripteurs que `Log` moteur, car il stocke chaque colonne dans un fichier séparé. Utilisez-le dans des scénarios simples à faible charge. -Le `Log` et `StripeLog` les moteurs prennent en charge la lecture de données parallèle. Lors de la lecture de données, ClickHouse utilise plusieurs threads. Chaque thread traite un bloc de données séparé. Le `Log` le moteur utilise un fichier distinct pour chaque colonne de la table. `StripeLog` stocke toutes les données dans un seul fichier. En conséquence, la `StripeLog` moteur utilise moins de descripteurs dans le système d’exploitation, mais le `Log` moteur fournit une plus grande efficacité lors de la lecture des données. +Le `Log` et `StripeLog` les moteurs prennent en charge la lecture de données parallèle. Lors de la lecture de données, ClickHouse utilise plusieurs threads. Chaque thread traite un bloc de données séparé. Le `Log` le moteur utilise un fichier distinct pour chaque colonne de la table. `StripeLog` stocke toutes les données dans un seul fichier. En conséquence, la `StripeLog` moteur utilise moins de descripteurs dans le système d'exploitation, mais le `Log` moteur fournit une plus grande efficacité lors de la lecture des données. [Article Original](https://clickhouse.tech/docs/en/operations/table_engines/log_family/) diff --git a/docs/fr/engines/table-engines/log-family/log.md b/docs/fr/engines/table-engines/log-family/log.md index 8389412dbec..33302e2b7f2 100644 --- a/docs/fr/engines/table-engines/log-family/log.md +++ b/docs/fr/engines/table-engines/log-family/log.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 33 toc_title: Journal --- @@ -9,8 +9,8 @@ toc_title: Journal Moteur appartient à la famille de journal des moteurs. Voir les propriétés communes des moteurs de journal et leurs différences dans le [Famille De Moteurs En Rondins](log-family.md) article. -Journal diffère de [TinyLog](tinylog.md) dans un petit fichier de “marks” réside avec les fichiers de colonne. Ces marques sont écrites sur chaque bloc de données et contiennent des décalages qui indiquent où commencer à lire le fichier afin d’ignorer le nombre de lignes spécifié. Cela permet de lire les données de table dans plusieurs threads. -Pour l’accès aux données simultanées, les opérations de lecture peuvent être effectuées simultanément, tandis que les opérations d’écriture bloc lit et l’autre. -Le moteur de journal ne prend pas en charge les index. De même, si l’écriture dans une table a échoué, la table est cassée et la lecture de celle-ci renvoie une erreur. Le moteur de journal est approprié pour les données temporaires, les tables en écriture unique, et à des fins de test ou de démonstration. +Journal diffère de [TinyLog](tinylog.md) dans un petit fichier de “marks” réside avec les fichiers de colonne. Ces marques sont écrites sur chaque bloc de données et contiennent des décalages qui indiquent où commencer à lire le fichier afin d'ignorer le nombre de lignes spécifié. Cela permet de lire les données de table dans plusieurs threads. +Pour l'accès aux données simultanées, les opérations de lecture peuvent être effectuées simultanément, tandis que les opérations d'écriture bloc lit et l'autre. +Le moteur de journal ne prend pas en charge les index. De même, si l'écriture dans une table a échoué, la table est cassée et la lecture de celle-ci renvoie une erreur. Le moteur de journal est approprié pour les données temporaires, les tables en écriture unique, et à des fins de test ou de démonstration. [Article Original](https://clickhouse.tech/docs/en/operations/table_engines/log/) diff --git a/docs/fr/engines/table-engines/log-family/stripelog.md b/docs/fr/engines/table-engines/log-family/stripelog.md index 53d790abaf4..390be1d34e3 100644 --- a/docs/fr/engines/table-engines/log-family/stripelog.md +++ b/docs/fr/engines/table-engines/log-family/stripelog.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 32 toc_title: StripeLog --- @@ -11,7 +11,7 @@ Ce moteur appartient à la famille des moteurs en rondins. Voir les propriétés Utilisez ce moteur dans des scénarios lorsque vous devez écrire de nombreuses tables avec une petite quantité de données (moins de 1 million de lignes). -## Création d’une Table {#table_engines-stripelog-creating-a-table} +## Création d'une Table {#table_engines-stripelog-creating-a-table} ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] @@ -24,9 +24,9 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] Voir la description détaillée de la [CREATE TABLE](../../../sql-reference/statements/create.md#create-table-query) requête. -## L’écriture Des Données {#table_engines-stripelog-writing-the-data} +## L'écriture des Données {#table_engines-stripelog-writing-the-data} -Le `StripeLog` moteur stocke toutes les colonnes dans un fichier. Pour chaque `INSERT` requête, ClickHouse ajoute le bloc de données à la fin d’un fichier de table, en écrivant des colonnes une par une. +Le `StripeLog` moteur stocke toutes les colonnes dans un fichier. Pour chaque `INSERT` requête, ClickHouse ajoute le bloc de données à la fin d'un fichier de table, en écrivant des colonnes une par une. Pour chaque table ClickHouse écrit les fichiers: @@ -35,13 +35,13 @@ Pour chaque table ClickHouse écrit les fichiers: Le `StripeLog` moteur ne prend pas en charge la `ALTER UPDATE` et `ALTER DELETE` opérations. -## La Lecture Des Données {#table_engines-stripelog-reading-the-data} +## La lecture des Données {#table_engines-stripelog-reading-the-data} -Le fichier avec des marques permet à ClickHouse de paralléliser la lecture des données. Cela signifie qu’une `SELECT` la requête renvoie des lignes dans un ordre imprévisible. L’utilisation de la `ORDER BY` clause pour trier les lignes. +Le fichier avec des marques permet à ClickHouse de paralléliser la lecture des données. Cela signifie qu'une `SELECT` la requête renvoie des lignes dans un ordre imprévisible. L'utilisation de la `ORDER BY` clause pour trier les lignes. -## Exemple D’Utilisation {#table_engines-stripelog-example-of-use} +## Exemple D'utilisation {#table_engines-stripelog-example-of-use} -Création d’une table: +Création d'une table: ``` sql CREATE TABLE stripe_log_table @@ -62,7 +62,7 @@ INSERT INTO stripe_log_table VALUES (now(),'REGULAR','The second regular message Nous avons utilisé deux `INSERT` requêtes pour créer deux blocs de données `data.bin` fichier. -ClickHouse utilise plusieurs threads lors de la sélection des données. Chaque thread lit un bloc de données séparé et renvoie les lignes résultantes indépendamment à la fin. En conséquence, l’ordre des blocs de lignes dans le résultat ne correspond pas à l’ordre des mêmes blocs dans l’entrée, dans la plupart des cas. Exemple: +ClickHouse utilise plusieurs threads lors de la sélection des données. Chaque thread lit un bloc de données séparé et renvoie les lignes résultantes indépendamment à la fin. En conséquence, l'ordre des blocs de lignes dans le résultat ne correspond pas à l'ordre des mêmes blocs dans l'entrée, dans la plupart des cas. Exemple: ``` sql SELECT * FROM stripe_log_table diff --git a/docs/fr/engines/table-engines/log-family/tinylog.md b/docs/fr/engines/table-engines/log-family/tinylog.md index 3cb2388ec23..54292730851 100644 --- a/docs/fr/engines/table-engines/log-family/tinylog.md +++ b/docs/fr/engines/table-engines/log-family/tinylog.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 34 toc_title: TinyLog --- @@ -11,6 +11,6 @@ Le moteur appartient à la famille de moteurs en rondins. Voir [Famille De Moteu Ce moteur de table est généralement utilisé avec la méthode write-once: écrivez des données une fois, puis lisez-les autant de fois que nécessaire. Par exemple, vous pouvez utiliser `TinyLog`- tapez des tables pour les données intermédiaires qui sont traitées en petits lots. Notez que le stockage des données dans un grand nombre de petites tables est inefficace. -Les requêtes sont exécutées dans un flux unique. En d’autres termes, ce moteur est destiné à des tables relativement petites (jusqu’à environ 1 000 000 de lignes). Il est logique d’utiliser ce moteur de table si vous avez beaucoup de petites tables, car il est plus simple que le [Journal](log.md) moteur (moins de fichiers doivent être ouverts). +Les requêtes sont exécutées dans un flux unique. En d'autres termes, ce moteur est destiné à des tables relativement petites (jusqu'à environ 1 000 000 de lignes). Il est logique d'utiliser ce moteur de table si vous avez beaucoup de petites tables, car il est plus simple que le [Journal](log.md) moteur (moins de fichiers doivent être ouverts). [Article Original](https://clickhouse.tech/docs/en/operations/table_engines/tinylog/) diff --git a/docs/fr/engines/table-engines/mergetree-family/aggregatingmergetree.md b/docs/fr/engines/table-engines/mergetree-family/aggregatingmergetree.md index 9f473daa26c..e8e74960545 100644 --- a/docs/fr/engines/table-engines/mergetree-family/aggregatingmergetree.md +++ b/docs/fr/engines/table-engines/mergetree-family/aggregatingmergetree.md @@ -1,21 +1,24 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 35 toc_title: AggregatingMergeTree --- # Aggregatingmergetree {#aggregatingmergetree} -Le moteur hérite de [MergeTree](mergetree.md#table_engines-mergetree), modifier la logique pour les parties de données Fusion. ClickHouse remplace toutes les lignes avec la même clé primaire (ou, plus précisément, avec la même [clé de tri](mergetree.md)) avec une seule ligne (dans un rayon d’une partie des données) qui stocke une combinaison d’états de fonctions d’agrégation. +Le moteur hérite de [MergeTree](mergetree.md#table_engines-mergetree), modifier la logique pour les parties de données Fusion. ClickHouse remplace toutes les lignes avec la même clé primaire (ou, plus précisément, avec la même [clé de tri](mergetree.md)) avec une seule ligne (dans un rayon d'une partie des données) qui stocke une combinaison d'états de fonctions d'agrégation. -Vous pouvez utiliser `AggregatingMergeTree` tables pour l’agrégation incrémentielle des données, y compris pour les vues matérialisées agrégées. +Vous pouvez utiliser `AggregatingMergeTree` tables pour l'agrégation incrémentielle des données, y compris pour les vues matérialisées agrégées. -Le moteur traite toutes les colonnes avec [AggregateFunction](../../../sql-reference/data-types/aggregatefunction.md) type. +Le moteur traite toutes les colonnes avec les types suivants: -Il est approprié d’utiliser `AggregatingMergeTree` si elle réduit le nombre de lignes par commande. +- [AggregateFunction](../../../sql-reference/data-types/aggregatefunction.md) +- [SimpleAggregateFunction](../../../sql-reference/data-types/simpleaggregatefunction.md) -## Création d’une Table {#creating-a-table} +Il est approprié d'utiliser `AggregatingMergeTree` si elle réduit le nombre de lignes par commande. + +## Création d'une Table {#creating-a-table} ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] @@ -35,14 +38,14 @@ Pour une description des paramètres de requête, voir [demande de description]( **Les clauses de requête** -Lors de la création d’un `AggregatingMergeTree` la table de la même [clause](mergetree.md) sont nécessaires, comme lors de la création d’un `MergeTree` table. +Lors de la création d'un `AggregatingMergeTree` la table de la même [clause](mergetree.md) sont nécessaires, comme lors de la création d'un `MergeTree` table.
Méthode obsolète pour créer une Table !!! attention "Attention" - N’utilisez pas cette méthode dans les nouveaux projets et, si possible, remplacez les anciens projets par la méthode décrite ci-dessus. + N'utilisez pas cette méthode dans les nouveaux projets et, si possible, remplacez les anciens projets par la méthode décrite ci-dessus. ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] @@ -56,14 +59,14 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] Tous les paramètres ont la même signification que dans `MergeTree`.
-## Sélectionner Et insérer {#select-and-insert} +## Sélectionner et insérer {#select-and-insert} -Pour insérer des données, utilisez [INSERT SELECT](../../../sql-reference/statements/insert-into.md) requête avec l’ensemble-l’État des fonctions. -Lors de la sélection des données `AggregatingMergeTree` table, utilisez `GROUP BY` et les mêmes fonctions d’agrégat que lors de l’insertion de données, mais en utilisant `-Merge` suffixe. +Pour insérer des données, utilisez [INSERT SELECT](../../../sql-reference/statements/insert-into.md) requête avec l'ensemble-l'État des fonctions. +Lors de la sélection des données `AggregatingMergeTree` table, utilisez `GROUP BY` et les mêmes fonctions d'agrégat que lors de l'insertion de données, mais en utilisant `-Merge` suffixe. -Dans les résultats de `SELECT` requête, les valeurs de `AggregateFunction` type ont une représentation binaire spécifique à l’implémentation pour tous les formats de sortie ClickHouse. Si les données de vidage dans, par exemple, `TabSeparated` format avec `SELECT` requête alors ce vidage peut être chargé en utilisant `INSERT` requête. +Dans les résultats de `SELECT` requête, les valeurs de `AggregateFunction` type ont une représentation binaire spécifique à l'implémentation pour tous les formats de sortie ClickHouse. Si les données de vidage dans, par exemple, `TabSeparated` format avec `SELECT` requête alors ce vidage peut être chargé en utilisant `INSERT` requête. -## Exemple D’une Vue matérialisée agrégée {#example-of-an-aggregated-materialized-view} +## Exemple D'une vue matérialisée agrégée {#example-of-an-aggregated-materialized-view} `AggregatingMergeTree` vue matérialisée qui regarde le `test.visits` table: @@ -85,7 +88,7 @@ Insertion de données dans la `test.visits` table. INSERT INTO test.visits ... ``` -Les données sont insérées dans la table et la vue `test.basic` que va effectuer l’agrégation. +Les données sont insérées dans la table et la vue `test.basic` que va effectuer l'agrégation. Pour obtenir les données agrégées, nous devons exécuter une requête telle que `SELECT ... GROUP BY ...` à partir de la vue `test.basic`: diff --git a/docs/fr/engines/table-engines/mergetree-family/collapsingmergetree.md b/docs/fr/engines/table-engines/mergetree-family/collapsingmergetree.md index 9f3fb2fe5a3..80f61622539 100644 --- a/docs/fr/engines/table-engines/mergetree-family/collapsingmergetree.md +++ b/docs/fr/engines/table-engines/mergetree-family/collapsingmergetree.md @@ -1,19 +1,19 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 36 toc_title: CollapsingMergeTree --- -# Collapsingmergetree {#table_engine-collapsingmergetree} +# CollapsingMergeTree {#table_engine-collapsingmergetree} -Le moteur hérite de [MergeTree](mergetree.md) et ajoute la logique de l’effondrement des lignes de données de pièces algorithme de fusion. +Le moteur hérite de [MergeTree](mergetree.md) et ajoute la logique de l'effondrement des lignes de données de pièces algorithme de fusion. -`CollapsingMergeTree` supprime de manière asynchrone (réduit) les paires de lignes si tous les champs d’une clé de tri (`ORDER BY`) sont équivalents à l’exception du champ particulier `Sign` ce qui peut avoir `1` et `-1` valeur. Les lignes sans paire sont conservées. Pour plus de détails, voir le [Effondrer](#table_engine-collapsingmergetree-collapsing) la section du document. +`CollapsingMergeTree` supprime de manière asynchrone (réduit) les paires de lignes si tous les champs d'une clé de tri (`ORDER BY`) sont équivalents à l'exception du champ particulier `Sign` ce qui peut avoir `1` et `-1` valeur. Les lignes sans paire sont conservées. Pour plus de détails, voir le [Effondrer](#table_engine-collapsingmergetree-collapsing) la section du document. -Le moteur peut réduire considérablement le volume de stockage et augmenter l’efficacité de `SELECT` requête en conséquence. +Le moteur peut réduire considérablement le volume de stockage et augmenter l'efficacité de `SELECT` requête en conséquence. -## Création d’une Table {#creating-a-table} +## Création d'une Table {#creating-a-table} ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] @@ -38,14 +38,14 @@ Pour une description des paramètres de requête, voir [description de la requê **Les clauses de requête** -Lors de la création d’un `CollapsingMergeTree` de table, de la même [les clauses de requête](mergetree.md#table_engine-mergetree-creating-a-table) sont nécessaires, comme lors de la création d’un `MergeTree` table. +Lors de la création d'un `CollapsingMergeTree` de table, de la même [les clauses de requête](mergetree.md#table_engine-mergetree-creating-a-table) sont nécessaires, comme lors de la création d'un `MergeTree` table.
Méthode obsolète pour créer une Table !!! attention "Attention" - N’utilisez pas cette méthode dans les nouveaux projets et, si possible, remplacez les anciens projets par la méthode décrite ci-dessus. + N'utilisez pas cette méthode dans les nouveaux projets et, si possible, remplacez les anciens projets par la méthode décrite ci-dessus. ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] @@ -68,11 +68,11 @@ Tous les paramètres excepté `sign` ont la même signification que dans `MergeT ### Données {#data} -Considérez la situation où vous devez enregistrer des données en constante évolution pour un objet. Il semble logique d’avoir une ligne pour un objet et de la mettre à jour à tout changement, mais l’opération de mise à jour est coûteuse et lente pour le SGBD car elle nécessite une réécriture des données dans le stockage. Si vous avez besoin d’écrire des données rapidement, la mise à jour n’est pas acceptable, mais vous pouvez écrire les modifications d’un objet de manière séquentielle comme suit. +Considérez la situation où vous devez enregistrer des données en constante évolution pour un objet. Il semble logique d'avoir une ligne pour un objet et de la mettre à jour à tout changement, mais l'opération de mise à jour est coûteuse et lente pour le SGBD car elle nécessite une réécriture des données dans le stockage. Si vous avez besoin d'écrire des données rapidement, la mise à jour n'est pas acceptable, mais vous pouvez écrire les modifications d'un objet de manière séquentielle comme suit. -Utilisez la colonne particulière `Sign`. Si `Sign = 1` cela signifie que la ligne est un état d’un objet, appelons-la “state” rangée. Si `Sign = -1` il signifie l’annulation de l’état d’un objet avec les mêmes attributs, nous allons l’appeler “cancel” rangée. +Utilisez la colonne particulière `Sign`. Si `Sign = 1` cela signifie que la ligne est un état d'un objet, appelons-la “state” rangée. Si `Sign = -1` il signifie l'annulation de l'état d'un objet avec les mêmes attributs, nous allons l'appeler “cancel” rangée. -Par exemple, nous voulons calculer combien de pages les utilisateurs ont vérifié sur un site et combien de temps ils étaient là. À un certain moment nous écrire la ligne suivante avec l’état de l’activité de l’utilisateur: +Par exemple, nous voulons calculer combien de pages les utilisateurs ont vérifié sur un site et combien de temps ils étaient là. À un certain moment nous écrire la ligne suivante avec l'état de l'activité de l'utilisateur: ``` text ┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ @@ -80,7 +80,7 @@ Par exemple, nous voulons calculer combien de pages les utilisateurs ont vérifi └─────────────────────┴───────────┴──────────┴──────┘ ``` -À un moment donné, nous enregistrons le changement d’activité de l’utilisateur et l’écrivons avec les deux lignes suivantes. +À un moment donné, nous enregistrons le changement d'activité de l'utilisateur et l'écrivons avec les deux lignes suivantes. ``` text ┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ @@ -89,11 +89,11 @@ Par exemple, nous voulons calculer combien de pages les utilisateurs ont vérifi └─────────────────────┴───────────┴──────────┴──────┘ ``` -La première ligne annule le précédent état de l’objet (utilisateur). Il doit copier les champs de clé de tri de l’état annulé sauf `Sign`. +La première ligne annule le précédent état de l'objet (utilisateur). Il doit copier les champs de clé de tri de l'état annulé sauf `Sign`. -La deuxième ligne contient l’état actuel. +La deuxième ligne contient l'état actuel. -Comme nous avons besoin seulement le dernier état de l’activité de l’utilisateur, les lignes +Comme nous avons besoin seulement le dernier état de l'activité de l'utilisateur, les lignes ``` text ┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ @@ -102,46 +102,43 @@ Comme nous avons besoin seulement le dernier état de l’activité de l’utili └─────────────────────┴───────────┴──────────┴──────┘ ``` -peut être supprimé en réduisant l’état invalide (ancien) d’un objet. `CollapsingMergeTree` fait cela lors de la fusion des parties de données. +peut être supprimé en réduisant l'état invalide (ancien) d'un objet. `CollapsingMergeTree` fait cela lors de la fusion des parties de données. Pourquoi nous avons besoin de 2 lignes pour chaque changement lu dans le [Algorithme](#table_engine-collapsingmergetree-collapsing-algorithm) paragraphe. -**Propriétés particulières d’une telle approche** +**Propriétés particulières d'une telle approche** -1. Le programme qui écrit les données doit se souvenir de l’état d’un objet pour pouvoir l’annuler. “Cancel” string doit contenir des copies des champs de clé de tri du “state” chaîne et le contraire `Sign`. Il augmente la taille initiale de stockage, mais permet d’écrire les données rapidement. -2. Les tableaux de plus en plus longs dans les colonnes réduisent l’efficacité du moteur en raison de la charge pour l’écriture. Plus les données sont simples, plus l’efficacité est élevée. -3. Le `SELECT` les résultats dépendent fortement de la cohérence de l’historique des modifications d’objet. Être précis lors de la préparation des données pour l’insertion. Vous pouvez obtenir des résultats imprévisibles dans des données incohérentes, par exemple des valeurs négatives pour des mesures non négatives telles que la profondeur de session. +1. Le programme qui écrit les données doit se souvenir de l'état d'un objet pour pouvoir l'annuler. “Cancel” string doit contenir des copies des champs de clé de tri du “state” chaîne et le contraire `Sign`. Il augmente la taille initiale de stockage, mais permet d'écrire les données rapidement. +2. Les tableaux de plus en plus longs dans les colonnes réduisent l'efficacité du moteur en raison de la charge pour l'écriture. Plus les données sont simples, plus l'efficacité est élevée. +3. Le `SELECT` les résultats dépendent fortement de la cohérence de l'historique des modifications d'objet. Être précis lors de la préparation des données pour l'insertion. Vous pouvez obtenir des résultats imprévisibles dans des données incohérentes, par exemple des valeurs négatives pour des mesures non négatives telles que la profondeur de session. ### Algorithme {#table_engine-collapsingmergetree-collapsing-algorithm} -Lorsque ClickHouse fusionne des parties de données, chaque groupe de lignes consécutives avec la même clé de tri (`ORDER BY`) est réduit à pas plus de deux rangées, une avec `Sign = 1` (“state” ligne) et l’autre avec `Sign = -1` (“cancel” rangée). En d’autres termes, les entrées de l’effondrement. +Lorsque ClickHouse fusionne des parties de données, chaque groupe de lignes consécutives avec la même clé de tri (`ORDER BY`) est réduit à pas plus de deux rangées, une avec `Sign = 1` (“state” ligne) et l'autre avec `Sign = -1` (“cancel” rangée). En d'autres termes, les entrées de l'effondrement. Pour chaque partie de données résultante clickhouse enregistre: 1. Première “cancel” et la dernière “state” lignes, si le nombre de “state” et “cancel” lignes correspond et la dernière ligne est un “state” rangée. - 2. La dernière “state” ligne, si il y a plus de “state” les lignes de “cancel” rangée. - 3. Première “cancel” ligne, si il y a plus de “cancel” les lignes de “state” rangée. - 4. Aucune des lignes, dans tous les autres cas. -Aussi quand il y a au moins 2 plus “state” les lignes de “cancel” les lignes, ou au moins 2 de plus “cancel” rangs puis “state” la fusion continue, mais ClickHouse traite cette situation comme une erreur logique et l’enregistre dans le journal du serveur. Cette erreur peut se produire si les mêmes données ont été insérées plus d’une fois. +Aussi quand il y a au moins 2 plus “state” les lignes de “cancel” les lignes, ou au moins 2 de plus “cancel” rangs puis “state” la fusion continue, mais ClickHouse traite cette situation comme une erreur logique et l'enregistre dans le journal du serveur. Cette erreur peut se produire si les mêmes données ont été insérées plus d'une fois. -Ainsi, l’effondrement ne devrait pas changer les résultats du calcul des statistiques. -Les changements se sont progressivement effondrés de sorte qu’à la fin seul le dernier état de presque tous les objets à gauche. +Ainsi, l'effondrement ne devrait pas changer les résultats du calcul des statistiques. +Les changements se sont progressivement effondrés de sorte qu'à la fin seul le dernier état de presque tous les objets à gauche. -Le `Sign` est nécessaire car l’algorithme de fusion ne garantit pas que toutes les lignes avec la même clé de tri seront dans la même partie de données résultante et même sur le même serveur physique. Processus de ClickHouse `SELECT` les requêtes avec plusieurs threads, et il ne peut pas prédire l’ordre des lignes dans le résultat. L’agrégation est nécessaire s’il y a un besoin d’obtenir complètement “collapsed” les données de `CollapsingMergeTree` table. +Le `Sign` est nécessaire car l'algorithme de fusion ne garantit pas que toutes les lignes avec la même clé de tri seront dans la même partie de données résultante et même sur le même serveur physique. Processus de ClickHouse `SELECT` les requêtes avec plusieurs threads, et il ne peut pas prédire l'ordre des lignes dans le résultat. L'agrégation est nécessaire s'il y a un besoin d'obtenir complètement “collapsed” les données de `CollapsingMergeTree` table. -Pour finaliser la réduction, écrivez une requête avec `GROUP BY` fonctions de clause et d’agrégation qui tiennent compte du signe. Par exemple, pour calculer la quantité, l’utilisation `sum(Sign)` plutôt `count()`. Pour calculer la somme de quelque chose, utilisez `sum(Sign * x)` plutôt `sum(x)` et ainsi de suite , et également ajouter `HAVING sum(Sign) > 0`. +Pour finaliser la réduction, écrivez une requête avec `GROUP BY` fonctions de clause et d'agrégation qui tiennent compte du signe. Par exemple, pour calculer la quantité, l'utilisation `sum(Sign)` plutôt `count()`. Pour calculer la somme de quelque chose, utilisez `sum(Sign * x)` plutôt `sum(x)` et ainsi de suite , et également ajouter `HAVING sum(Sign) > 0`. -Aggregate `count`, `sum` et `avg` pourrait être calculée de cette manière. Aggregate `uniq` peut être calculé si un objet a au moins un état non réduit. Aggregate `min` et `max` impossible de calculer parce que `CollapsingMergeTree` n’enregistre pas l’historique des valeurs des États réduits. +Aggregate `count`, `sum` et `avg` pourrait être calculée de cette manière. Aggregate `uniq` peut être calculé si un objet a au moins un état non réduit. Aggregate `min` et `max` impossible de calculer parce que `CollapsingMergeTree` n'enregistre pas l'historique des valeurs des États réduits. -Si vous avez besoin d’extraire des données sans agrégation (par exemple, pour vérifier si des lignes sont présentes dont les valeurs les plus récentes correspondent à certaines conditions), vous pouvez utiliser `FINAL` le modificateur du `FROM` clause. Cette approche est nettement moins efficace. +Si vous avez besoin d'extraire des données sans agrégation (par exemple, pour vérifier si des lignes sont présentes dont les valeurs les plus récentes correspondent à certaines conditions), vous pouvez utiliser `FINAL` le modificateur du `FROM` clause. Cette approche est nettement moins efficace. -## Exemple D’Utilisation {#example-of-use} +## Exemple D'utilisation {#example-of-use} -Les données de l’exemple: +Les données de l'exemple: ``` text ┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ @@ -175,9 +172,9 @@ INSERT INTO UAct VALUES (4324182021466249494, 5, 146, 1) INSERT INTO UAct VALUES (4324182021466249494, 5, 146, -1),(4324182021466249494, 6, 185, 1) ``` -Nous utilisons deux `INSERT` requêtes pour créer deux parties de données différentes. Si nous insérons les données avec une requête, ClickHouse crée une partie de données et n’effectuera aucune fusion. +Nous utilisons deux `INSERT` requêtes pour créer deux parties de données différentes. Si nous insérons les données avec une requête, ClickHouse crée une partie de données et n'effectuera aucune fusion. -L’obtention de données: +L'obtention de données: ``` sql SELECT * FROM UAct @@ -193,11 +190,11 @@ SELECT * FROM UAct └─────────────────────┴───────────┴──────────┴──────┘ ``` -Que voyons-nous et où s’effondre? +Que voyons-nous et où s'effondre? -Avec deux `INSERT` requêtes, nous avons créé 2 parties de données. Le `SELECT` la requête a été effectuée dans 2 threads, et nous avons obtenu un ordre aléatoire de lignes. L’effondrement n’a pas eu lieu car il n’y avait pas encore de fusion des parties de données. ClickHouse fusionne une partie des données dans un moment inconnu que nous ne pouvons pas prédire. +Avec deux `INSERT` requêtes, nous avons créé 2 parties de données. Le `SELECT` la requête a été effectuée dans 2 threads, et nous avons obtenu un ordre aléatoire de lignes. L'effondrement n'a pas eu lieu car il n'y avait pas encore de fusion des parties de données. ClickHouse fusionne une partie des données dans un moment inconnu que nous ne pouvons pas prédire. -Nous avons donc besoin d’agrégation: +Nous avons donc besoin d'agrégation: ``` sql SELECT @@ -215,7 +212,7 @@ HAVING sum(Sign) > 0 └─────────────────────┴───────────┴──────────┘ ``` -Si nous n’avons pas besoin d’agrégation et de vouloir forcer l’effondrement, nous pouvons utiliser `FINAL` le modificateur `FROM` clause. +Si nous n'avons pas besoin d'agrégation et de vouloir forcer l'effondrement, nous pouvons utiliser `FINAL` le modificateur `FROM` clause. ``` sql SELECT * FROM UAct FINAL @@ -227,11 +224,11 @@ SELECT * FROM UAct FINAL └─────────────────────┴───────────┴──────────┴──────┘ ``` -Cette façon de sélectionner les données est très inefficace. Ne l’utilisez pas pour les grandes tables. +Cette façon de sélectionner les données est très inefficace. Ne l'utilisez pas pour les grandes tables. -## Exemple D’Une Autre Approche {#example-of-another-approach} +## Exemple D'une autre approche {#example-of-another-approach} -Les données de l’exemple: +Les données de l'exemple: ``` text ┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ @@ -241,7 +238,7 @@ Les données de l’exemple: └─────────────────────┴───────────┴──────────┴──────┘ ``` -L’idée est que les fusions ne prennent en compte que les champs clés. Et dans le “Cancel” ligne nous pouvons spécifier des valeurs négatives qui égalisent la version précédente de la ligne lors de la sommation sans utiliser la colonne de signe. Pour cette approche, il est nécessaire de changer le type de données `PageViews`,`Duration` pour stocker les valeurs négatives de UInt8 - \> Int16. +L'idée est que les fusions ne prennent en compte que les champs clés. Et dans le “Cancel” ligne nous pouvons spécifier des valeurs négatives qui égalisent la version précédente de la ligne lors de la sommation sans utiliser la colonne de signe. Pour cette approche, il est nécessaire de changer le type de données `PageViews`,`Duration` pour stocker les valeurs négatives de UInt8 - \> Int16. ``` sql CREATE TABLE UAct @@ -255,7 +252,7 @@ ENGINE = CollapsingMergeTree(Sign) ORDER BY UserID ``` -Nous allons tester l’approche: +Nous allons tester l'approche: ``` sql insert into UAct values(4324182021466249494, 5, 146, 1); diff --git a/docs/fr/engines/table-engines/mergetree-family/custom-partitioning-key.md b/docs/fr/engines/table-engines/mergetree-family/custom-partitioning-key.md index a93657b4090..3617655adb1 100644 --- a/docs/fr/engines/table-engines/mergetree-family/custom-partitioning-key.md +++ b/docs/fr/engines/table-engines/mergetree-family/custom-partitioning-key.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 32 toc_title: "Cl\xE9 De Partitionnement Personnalis\xE9e" --- @@ -9,9 +9,9 @@ toc_title: "Cl\xE9 De Partitionnement Personnalis\xE9e" Le partitionnement est disponible pour [MergeTree](mergetree.md) table de famille (y compris les [répliqué](replication.md) table). [Les vues matérialisées](../special/materializedview.md#materializedview) basé sur les tables MergeTree prennent également en charge le partitionnement. -Une partition est une combinaison logique d’enregistrements dans une table selon un critère spécifié. Vous pouvez définir une partition par un critère arbitraire, comme, par mois, par jour, ou par type d’événement. Chaque partition est stockée séparément pour simplifier les manipulations de ces données. Lors de l’accès aux données, ClickHouse utilise le plus petit sous-ensemble de partitions possible. +Une partition est une combinaison logique d'enregistrements dans une table selon un critère spécifié. Vous pouvez définir une partition par un critère arbitraire, comme, par mois, par jour, ou par type d'événement. Chaque partition est stockée séparément pour simplifier les manipulations de ces données. Lors de l'accès aux données, ClickHouse utilise le plus petit sous-ensemble de partitions possible. -La partition est spécifiée dans le `PARTITION BY expr` clause lors de [création d’une table](mergetree.md#table_engine-mergetree-creating-a-table). La clé de partition peut être n’importe quelle expression des colonnes de la table. Par exemple, pour spécifier le partitionnement par mois, utilisez l’expression `toYYYYMM(date_column)`: +La partition est spécifiée dans le `PARTITION BY expr` clause lors de [création d'une table](mergetree.md#table_engine-mergetree-creating-a-table). La clé de partition peut être n'importe quelle expression des colonnes de la table. Par exemple, pour spécifier le partitionnement par mois, utilisez l'expression `toYYYYMM(date_column)`: ``` sql CREATE TABLE visits @@ -25,7 +25,7 @@ PARTITION BY toYYYYMM(VisitDate) ORDER BY Hour; ``` -La clé de partition peut également être un tuple d’expressions (similaire à la [clé primaire](mergetree.md#primary-keys-and-indexes-in-queries)). Exemple: +La clé de partition peut également être un tuple d'expressions (similaire à la [clé primaire](mergetree.md#primary-keys-and-indexes-in-queries)). Exemple: ``` sql ENGINE = ReplicatedCollapsingMergeTree('/clickhouse/tables/name', 'replica1', Sign) @@ -33,14 +33,14 @@ PARTITION BY (toMonday(StartDate), EventType) ORDER BY (CounterID, StartDate, intHash32(UserID)); ``` -Dans cet exemple, nous définissons le partitionnement par les types d’événements qui se sont produits au cours de la semaine en cours. +Dans cet exemple, nous définissons le partitionnement par les types d'événements qui se sont produits au cours de la semaine en cours. -Lors de l’insertion de nouvelles données dans une table, ces données sont stockées en tant que partie séparée (bloc) triée par la clé primaire. Dans 10-15 minutes après l’insertion, les parties de la même partition sont fusionnées dans la partie entière. +Lors de l'insertion de nouvelles données dans une table, ces données sont stockées en tant que partie séparée (bloc) triée par la clé primaire. Dans 10-15 minutes après l'insertion, les parties de la même partition sont fusionnées dans la partie entière. !!! info "Info" - Une fusion ne fonctionne que pour les parties de données qui ont la même valeur pour l’expression de partitionnement. Cela signifie **vous ne devriez pas faire des partitions trop granulaires** (plus d’un millier de partitions). Sinon, l’ `SELECT` la requête fonctionne mal en raison d’un nombre déraisonnablement élevé de fichiers dans le système de fichiers et des descripteurs de fichiers ouverts. + Une fusion ne fonctionne que pour les parties de données qui ont la même valeur pour l'expression de partitionnement. Cela signifie **vous ne devriez pas faire des partitions trop granulaires** (plus d'un millier de partitions). Sinon, l' `SELECT` la requête fonctionne mal en raison d'un nombre déraisonnablement élevé de fichiers dans le système de fichiers et des descripteurs de fichiers ouverts. -L’utilisation de la [système.partie](../../../operations/system-tables.md#system_tables-parts) table pour afficher les parties et les partitions de la table. Par exemple, supposons que nous avons une `visits` table avec partitionnement par mois. Nous allons effectuer le `SELECT` la requête pour l’ `system.parts` table: +L'utilisation de la [système.partie](../../../operations/system-tables.md#system_tables-parts) table pour afficher les parties et les partitions de la table. Par exemple, supposons que nous avons une `visits` table avec partitionnement par mois. Nous allons effectuer le `SELECT` la requête pour l' `system.parts` table: ``` sql SELECT @@ -72,14 +72,14 @@ Décomposons le nom de la première partie: `201901_1_3_1`: - `201901` est le nom de la partition. - `1` est le nombre minimum du bloc de données. - `3` est le nombre maximal de blocs de données. -- `1` est le niveau de bloc (la profondeur de l’arbre de fusion à partir duquel il est formé). +- `1` est le niveau de bloc (la profondeur de l'arbre de fusion à partir duquel il est formé). !!! info "Info" Les parties des tables de type ancien ont le nom: `20190117_20190123_2_2_0` (date minimale - date maximale - numéro de bloc minimum - numéro de bloc maximum-niveau). Le `active` colonne indique le statut de la partie. `1` est active; `0` est inactif. Les parties inactives sont, par exemple, des parties source restant après la fusion à une partie plus grande. Les parties de données corrompues sont également indiquées comme inactives. -Comme vous pouvez le voir dans l’exemple, il y a plusieurs parties séparées de la même partition (par exemple, `201901_1_3_1` et `201901_1_9_2`). Cela signifie que ces parties ne sont pas encore fusionnées. Clickhouse fusionne les parties insérées des données périodiquement, environ 15 minutes après l’insertion. En outre, vous pouvez effectuer une fusion non planifiée en utilisant [OPTIMIZE](../../../sql-reference/statements/misc.md#misc_operations-optimize) requête. Exemple: +Comme vous pouvez le voir dans l'exemple, il y a plusieurs parties séparées de la même partition (par exemple, `201901_1_3_1` et `201901_1_9_2`). Cela signifie que ces parties ne sont pas encore fusionnées. Clickhouse fusionne les parties insérées des données périodiquement, environ 15 minutes après l'insertion. En outre, vous pouvez effectuer une fusion non planifiée en utilisant [OPTIMIZE](../../../sql-reference/statements/misc.md#misc_operations-optimize) requête. Exemple: ``` sql OPTIMIZE TABLE visits PARTITION 201902; @@ -100,7 +100,7 @@ OPTIMIZE TABLE visits PARTITION 201902; Les parties inactives seront supprimées environ 10 minutes après la fusion. -Une autre façon d’afficher un ensemble de pièces et de partitions est d’aller dans le répertoire de la table: `/var/lib/clickhouse/data///`. Exemple: +Une autre façon d'afficher un ensemble de pièces et de partitions est d'aller dans le répertoire de la table: `/var/lib/clickhouse/data//
/`. Exemple: ``` bash /var/lib/clickhouse/data/default/visits$ ls -l @@ -118,10 +118,10 @@ drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 1 16:48 detached Dossier ‘201901\_1\_1\_0’, ‘201901\_1\_7\_1’ et ainsi de suite sont les répertoires des parties. Chaque partie se rapporte à une partition correspondante et contient des données juste pour un certain mois (la table dans cet exemple a partitionnement par mois). -Le `detached` le répertoire contient des parties qui ont été détachées de la table à l’aide [DETACH](../../../sql-reference/statements/alter.md#alter_detach-partition) requête. Les parties corrompues sont également déplacées dans ce répertoire, au lieu d’être supprimées. Le serveur n’utilise pas les pièces de la `detached` directory. You can add, delete, or modify the data in this directory at any time – the server will not know about this until you run the [ATTACH](../../../sql-reference/statements/alter.md#alter_attach-partition) requête. +Le `detached` le répertoire contient des parties qui ont été détachées de la table à l'aide [DETACH](../../../sql-reference/statements/alter.md#alter_detach-partition) requête. Les parties corrompues sont également déplacées dans ce répertoire, au lieu d'être supprimées. Le serveur n'utilise pas les pièces de la `detached` directory. You can add, delete, or modify the data in this directory at any time – the server will not know about this until you run the [ATTACH](../../../sql-reference/statements/alter.md#alter_attach-partition) requête. -Notez que sur le serveur d’exploitation, vous ne pouvez pas modifier manuellement l’ensemble de pièces ou leurs données sur le système de fichiers, car le serveur ne le saura pas. Pour les tables non répliquées, vous pouvez le faire lorsque le serveur est arrêté, mais ce n’est pas recommandé. Pour les tables répliquées, l’ensemble de pièces ne peut en aucun cas être modifié. +Notez que sur le serveur d'exploitation, vous ne pouvez pas modifier manuellement l'ensemble de pièces ou leurs données sur le système de fichiers, car le serveur ne le saura pas. Pour les tables non répliquées, vous pouvez le faire lorsque le serveur est arrêté, mais ce n'est pas recommandé. Pour les tables répliquées, l'ensemble de pièces ne peut en aucun cas être modifié. -ClickHouse vous permet d’effectuer des opérations avec les partitions: les supprimer, copier d’une table à une autre, ou créer une sauvegarde. Voir la liste de toutes les opérations de la section [Manipulations avec des Partitions et des pièces](../../../sql-reference/statements/alter.md#alter_manipulations-with-partitions). +ClickHouse vous permet d'effectuer des opérations avec les partitions: les supprimer, copier d'une table à une autre, ou créer une sauvegarde. Voir la liste de toutes les opérations de la section [Manipulations avec des Partitions et des pièces](../../../sql-reference/statements/alter.md#alter_manipulations-with-partitions). [Article Original](https://clickhouse.tech/docs/en/operations/table_engines/custom_partitioning_key/) diff --git a/docs/fr/engines/table-engines/mergetree-family/graphitemergetree.md b/docs/fr/engines/table-engines/mergetree-family/graphitemergetree.md index 38876ebb0d5..d1dc5e64a4f 100644 --- a/docs/fr/engines/table-engines/mergetree-family/graphitemergetree.md +++ b/docs/fr/engines/table-engines/mergetree-family/graphitemergetree.md @@ -1,19 +1,19 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 38 toc_title: GraphiteMergeTree --- -# Graphitemergetree {#graphitemergetree} +# GraphiteMergeTree {#graphitemergetree} -Ce moteur est conçu pour l’amincissement et l’agrégation / moyenne (cumul) [Graphite](http://graphite.readthedocs.io/en/latest/index.html) données. Il peut être utile aux développeurs qui veulent utiliser ClickHouse comme un magasin de données pour Graphite. +Ce moteur est conçu pour l'amincissement et l'agrégation / moyenne (cumul) [Graphite](http://graphite.readthedocs.io/en/latest/index.html) données. Il peut être utile aux développeurs qui veulent utiliser ClickHouse comme un magasin de données pour Graphite. -Vous pouvez utiliser N’importe quel moteur de table ClickHouse pour stocker les données Graphite si vous n’avez pas besoin de cumul, mais si vous avez besoin d’un cumul, utilisez `GraphiteMergeTree`. Le moteur réduit le volume de stockage et augmente l’efficacité des requêtes de Graphite. +Vous pouvez utiliser N'importe quel moteur de table ClickHouse pour stocker les données Graphite si vous n'avez pas besoin de cumul, mais si vous avez besoin d'un cumul, utilisez `GraphiteMergeTree`. Le moteur réduit le volume de stockage et augmente l'efficacité des requêtes de Graphite. Le moteur hérite des propriétés de [MergeTree](mergetree.md). -## Création d’une Table {#creating-table} +## Création d'une Table {#creating-table} ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] @@ -52,14 +52,14 @@ Les noms de ces colonnes doivent être définis dans la configuration de cumul. **Les clauses de requête** -Lors de la création d’un `GraphiteMergeTree` de table, de la même [clause](mergetree.md#table_engine-mergetree-creating-a-table) sont nécessaires, comme lors de la création d’un `MergeTree` table. +Lors de la création d'un `GraphiteMergeTree` de table, de la même [clause](mergetree.md#table_engine-mergetree-creating-a-table) sont nécessaires, comme lors de la création d'un `MergeTree` table.
Méthode obsolète pour créer une Table !!! attention "Attention" - N’utilisez pas cette méthode dans les nouveaux projets et, si possible, remplacez les anciens projets par la méthode décrite ci-dessus. + N'utilisez pas cette méthode dans les nouveaux projets et, si possible, remplacez les anciens projets par la méthode décrite ci-dessus. ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] @@ -127,7 +127,7 @@ default 1. Patterns with both `function` and `retention`. 1. Pattern `default`. -Lors du traitement d’une ligne, ClickHouse vérifie les règles `pattern` section. Chacun `pattern` (comprendre `default`) les articles peuvent contenir des `function` paramètre d’agrégation, `retention` les paramètres ou les deux à la fois. Si le nom de la métrique correspond `regexp` les règles de la `pattern` section (ou sections) sont appliquées; sinon, les règles de la `default` section sont utilisés. +Lors du traitement d'une ligne, ClickHouse vérifie les règles `pattern` section. Chacun `pattern` (comprendre `default`) les articles peuvent contenir des `function` paramètre d'agrégation, `retention` les paramètres ou les deux à la fois. Si le nom de la métrique correspond `regexp` les règles de la `pattern` section (ou sections) sont appliquées; sinon, les règles de la `default` section sont utilisés. Champs pour `pattern` et `default` section: diff --git a/docs/fr/engines/table-engines/mergetree-family/index.md b/docs/fr/engines/table-engines/mergetree-family/index.md index 82c36cfc29f..e2de11a7591 100644 --- a/docs/fr/engines/table-engines/mergetree-family/index.md +++ b/docs/fr/engines/table-engines/mergetree-family/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 -toc_folder_title: MergeTree Family +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: Famille MergeTree toc_priority: 28 --- diff --git a/docs/fr/engines/table-engines/mergetree-family/mergetree.md b/docs/fr/engines/table-engines/mergetree-family/mergetree.md index 9e6ef260040..3906eabfc47 100644 --- a/docs/fr/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/fr/engines/table-engines/mergetree-family/mergetree.md @@ -1,15 +1,15 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 30 toc_title: MergeTree --- -# Mergetree {#table_engines-mergetree} +# MergeTree {#table_engines-mergetree} Le `MergeTree` moteur et autres moteurs de cette famille (`*MergeTree`) sont les moteurs de table ClickHouse les plus robustes. -Les moteurs de la `MergeTree` famille sont conçus pour l’insertion d’une très grande quantité de données dans une table. Les données sont rapidement écrites dans la table partie par partie, puis des règles sont appliquées pour fusionner les parties en arrière-plan. Cette méthode est beaucoup plus efficace que de réécrire continuellement les données dans le stockage pendant l’insertion. +Les moteurs de la `MergeTree` famille sont conçus pour l'insertion d'une très grande quantité de données dans une table. Les données sont rapidement écrites dans la table partie par partie, puis des règles sont appliquées pour fusionner les parties en arrière-plan. Cette méthode est beaucoup plus efficace que de réécrire continuellement les données dans le stockage pendant l'insertion. Principales caractéristiques: @@ -23,16 +23,16 @@ Principales caractéristiques: - Prise en charge de la réplication des données. - La famille de `ReplicatedMergeTree` tables fournit la réplication des données. Pour plus d’informations, voir [Réplication des données](replication.md). + La famille de `ReplicatedMergeTree` tables fournit la réplication des données. Pour plus d'informations, voir [Réplication des données](replication.md). -- Appui d’échantillonnage de données. +- Appui d'échantillonnage de données. - Si nécessaire, vous pouvez définir la méthode d’échantillonnage des données dans le tableau. + Si nécessaire, vous pouvez définir la méthode d'échantillonnage des données dans le tableau. !!! info "Info" - Le [Fusionner](../special/merge.md#merge) le moteur n’appartient pas à la `*MergeTree` famille. + Le [Fusionner](../special/merge.md#merge) le moteur n'appartient pas à la `*MergeTree` famille. -## Création d’une Table {#table_engine-mergetree-creating-a-table} +## Création d'une Table {#table_engine-mergetree-creating-a-table} ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] @@ -58,19 +58,19 @@ Pour une description des paramètres, voir [Créer une description de requête]( ### Les Clauses De Requête {#mergetree-query-clauses} -- `ENGINE` — Name and parameters of the engine. `ENGINE = MergeTree()`. Le `MergeTree` le moteur n’a pas de paramètres. +- `ENGINE` — Name and parameters of the engine. `ENGINE = MergeTree()`. Le `MergeTree` le moteur n'a pas de paramètres. - `PARTITION BY` — The [clé de partitionnement](custom-partitioning-key.md). - Pour le partitionnement par mois, utilisez les `toYYYYMM(date_column)` l’expression, où `date_column` est une colonne avec une date du type [Date](../../../sql-reference/data-types/date.md). Les noms de partition ici ont le `"YYYYMM"` format. + Pour le partitionnement par mois, utilisez les `toYYYYMM(date_column)` l'expression, où `date_column` est une colonne avec une date du type [Date](../../../sql-reference/data-types/date.md). Les noms de partition ici ont le `"YYYYMM"` format. - `ORDER BY` — The sorting key. - Un tuple de colonnes ou d’expressions arbitraires. Exemple: `ORDER BY (CounterID, EventDate)`. + Un tuple de colonnes ou d'expressions arbitraires. Exemple: `ORDER BY (CounterID, EventDate)`. - `PRIMARY KEY` — The primary key if it [diffère de la clé de tri](#choosing-a-primary-key-that-differs-from-the-sorting-key). - Par défaut, la clé primaire est la même que la clé de tri (qui est spécifiée par `ORDER BY` clause). Ainsi dans la plupart des cas il n’est pas nécessaire de spécifier un `PRIMARY KEY` clause. + Par défaut, la clé primaire est la même que la clé de tri (qui est spécifiée par `ORDER BY` clause). Ainsi dans la plupart des cas il n'est pas nécessaire de spécifier un `PRIMARY KEY` clause. - `SAMPLE BY` — An expression for sampling. @@ -78,10 +78,10 @@ Pour une description des paramètres, voir [Créer une description de requête]( - `TTL` — A list of rules specifying storage duration of rows and defining logic of automatic parts movement [entre disques et volumes](#table_engine-mergetree-multiple-volumes). - L’Expression doit en avoir une `Date` ou `DateTime` colonne comme un résultat. Exemple: + L'Expression doit en avoir une `Date` ou `DateTime` colonne comme un résultat. Exemple: `TTL date + INTERVAL 1 DAY` - Le Type de la règle `DELETE|TO DISK 'xxx'|TO VOLUME 'xxx'` spécifie une action à effectuer avec la partie si l’expression est satisfaite (atteint l’heure actuelle): suppression des Lignes expirées, déplacement d’une partie (si l’expression est satisfaite pour toutes les lignes d’une partie) sur le disque spécifié (`TO DISK 'xxx'`) ou de volume (`TO VOLUME 'xxx'`). Le type par défaut de la règle est suppression (`DELETE`). Liste de règles multiples peut spécifié, mais il ne devrait pas y avoir plus d’un `DELETE` règle. + Le Type de la règle `DELETE|TO DISK 'xxx'|TO VOLUME 'xxx'` spécifie une action à effectuer avec la partie si l'expression est satisfaite (atteint l'heure actuelle): suppression des Lignes expirées, déplacement d'une partie (si l'expression est satisfaite pour toutes les lignes d'une partie) sur le disque spécifié (`TO DISK 'xxx'`) ou de volume (`TO VOLUME 'xxx'`). Le type par défaut de la règle est suppression (`DELETE`). Liste de règles multiples peut spécifié, mais il ne devrait pas y avoir plus d'un `DELETE` règle. Pour plus de détails, voir [TTL pour les colonnes et les tableaux](#table_engine-mergetree-ttl) @@ -89,12 +89,12 @@ Pour une description des paramètres, voir [Créer une description de requête]( - `index_granularity` — Maximum number of data rows between the marks of an index. Default value: 8192. See [Le Stockage De Données](#mergetree-data-storage). - `index_granularity_bytes` — Maximum size of data granules in bytes. Default value: 10Mb. To restrict the granule size only by number of rows, set to 0 (not recommended). See [Le Stockage De Données](#mergetree-data-storage). - - `enable_mixed_granularity_parts` — Enables or disables transitioning to control the granule size with the `index_granularity_bytes` paramètre. Avant la version 19.11, il n’y avait que le `index_granularity` réglage pour restreindre la taille des granules. Le `index_granularity_bytes` le paramètre améliore les performances de ClickHouse lors de la sélection de données à partir de tables avec de grandes lignes (des dizaines et des centaines de mégaoctets). Si vous avez des tables avec de grandes lignes, vous pouvez activer ce paramètre pour les tables d’améliorer l’efficacité de `SELECT` requête. - - `use_minimalistic_part_header_in_zookeeper` — Storage method of the data parts headers in ZooKeeper. If `use_minimalistic_part_header_in_zookeeper=1`, puis Zookeeper stocke moins de données. Pour plus d’informations, voir le [Description du réglage](../../../operations/server-configuration-parameters/settings.md#server-settings-use_minimalistic_part_header_in_zookeeper) dans “Server configuration parameters”. - - `min_merge_bytes_to_use_direct_io` — The minimum data volume for merge operation that is required for using direct I/O access to the storage disk. When merging data parts, ClickHouse calculates the total storage volume of all the data to be merged. If the volume exceeds `min_merge_bytes_to_use_direct_io` octets, ClickHouse lit et écrit les données sur le disque de stockage en utilisant l’interface d’E/S directe (`O_DIRECT` option). Si `min_merge_bytes_to_use_direct_io = 0`, puis les e/s directes sont désactivées. Valeur par défaut: `10 * 1024 * 1024 * 1024` octet. + - `enable_mixed_granularity_parts` — Enables or disables transitioning to control the granule size with the `index_granularity_bytes` paramètre. Avant la version 19.11, il n'y avait que le `index_granularity` réglage pour restreindre la taille des granules. Le `index_granularity_bytes` le paramètre améliore les performances de ClickHouse lors de la sélection de données à partir de tables avec de grandes lignes (des dizaines et des centaines de mégaoctets). Si vous avez des tables avec de grandes lignes, vous pouvez activer ce paramètre pour les tables d'améliorer l'efficacité de `SELECT` requête. + - `use_minimalistic_part_header_in_zookeeper` — Storage method of the data parts headers in ZooKeeper. If `use_minimalistic_part_header_in_zookeeper=1`, puis Zookeeper stocke moins de données. Pour plus d'informations, voir le [Description du réglage](../../../operations/server-configuration-parameters/settings.md#server-settings-use_minimalistic_part_header_in_zookeeper) dans “Server configuration parameters”. + - `min_merge_bytes_to_use_direct_io` — The minimum data volume for merge operation that is required for using direct I/O access to the storage disk. When merging data parts, ClickHouse calculates the total storage volume of all the data to be merged. If the volume exceeds `min_merge_bytes_to_use_direct_io` octets, ClickHouse lit et écrit les données sur le disque de stockage en utilisant l'interface d'E/S directe (`O_DIRECT` option). Si `min_merge_bytes_to_use_direct_io = 0`, puis les e/s directes sont désactivées. Valeur par défaut: `10 * 1024 * 1024 * 1024` octet. - `merge_with_ttl_timeout` — Minimum delay in seconds before repeating a merge with TTL. Default value: 86400 (1 day). - - `write_final_mark` — Enables or disables writing the final index mark at the end of data part (after the last byte). Default value: 1. Don’t turn it off. + - `write_final_mark` — Enables or disables writing the final index mark at the end of data part (after the last byte). Default value: 1. Don't turn it off. - `merge_max_block_size` — Maximum number of rows in block for merge operations. Default value: 8192. - `storage_policy` — Storage policy. See [Utilisation de plusieurs périphériques de bloc pour le stockage de données](#table_engine-mergetree-multiple-volumes). @@ -104,9 +104,9 @@ Pour une description des paramètres, voir [Créer une description de requête]( ENGINE MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity=8192 ``` -Dans l’exemple, nous définissons le partitionnement par mois. +Dans l'exemple, nous définissons le partitionnement par mois. -Nous définissons également une expression pour l’échantillonnage en tant que hachage par l’ID utilisateur. Cela vous permet de pseudorandomiser les données dans la table pour chaque `CounterID` et `EventDate`. Si vous définissez un [SAMPLE](../../../sql-reference/statements/select.md#select-sample-clause) clause lors de la sélection des données, ClickHouse retournera un échantillon de données uniformément pseudo-aléatoire pour un sous-ensemble d’utilisateurs. +Nous définissons également une expression pour l'échantillonnage en tant que hachage par l'ID utilisateur. Cela vous permet de pseudorandomiser les données dans la table pour chaque `CounterID` et `EventDate`. Si vous définissez un [SAMPLE](../../../sql-reference/statements/select/sample.md#select-sample-clause) clause lors de la sélection des données, ClickHouse retournera un échantillon de données uniformément pseudo-aléatoire pour un sous-ensemble d'utilisateurs. Le `index_granularity` paramètre peut être omis, car 8192 est la valeur par défaut. @@ -115,7 +115,7 @@ Le `index_granularity` paramètre peut être omis, car 8192 est la valeur par d Méthode obsolète pour créer une Table !!! attention "Attention" - N’utilisez pas cette méthode dans les nouveaux projets. Si possible, optez anciens projets à la méthode décrite ci-dessus. + N'utilisez pas cette méthode dans les nouveaux projets. Si possible, optez anciens projets à la méthode décrite ci-dessus. ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] @@ -131,7 +131,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] - `date-column` — The name of a column of the [Date](../../../sql-reference/data-types/date.md) type. ClickHouse crée automatiquement des partitions par mois en fonction de cette colonne. Les noms de partition sont dans le `"YYYYMM"` format. - `sampling_expression` — An expression for sampling. - `(primary, key)` — Primary key. Type: [Tuple()](../../../sql-reference/data-types/tuple.md) -- `index_granularity` — The granularity of an index. The number of data rows between the “marks” d’un index. La valeur 8192 est appropriée pour la plupart des tâches. +- `index_granularity` — The granularity of an index. The number of data rows between the “marks” d'un index. La valeur 8192 est appropriée pour la plupart des tâches. **Exemple** @@ -139,24 +139,24 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] MergeTree(EventDate, intHash32(UserID), (CounterID, EventDate, intHash32(UserID)), 8192) ``` -Le `MergeTree` le moteur est configuré de la même manière que dans l’exemple ci-dessus pour la méthode de configuration du moteur principal. +Le `MergeTree` le moteur est configuré de la même manière que dans l'exemple ci-dessus pour la méthode de configuration du moteur principal.
## Le Stockage De Données {#mergetree-data-storage} Une table se compose de parties de données triées par clé primaire. -Lorsque des données sont insérées dans une table, des parties de données distinctes sont créées et chacune d’elles est lexicographiquement triée par clé primaire. Par exemple, si la clé primaire est `(CounterID, Date)`, les données de la pièce sont triées par `CounterID` et au sein de chaque `CounterID` il est commandé par `Date`. +Lorsque des données sont insérées dans une table, des parties de données distinctes sont créées et chacune d'elles est lexicographiquement triée par clé primaire. Par exemple, si la clé primaire est `(CounterID, Date)`, les données de la pièce sont triées par `CounterID` et au sein de chaque `CounterID` il est commandé par `Date`. Les données appartenant à différentes partitions sont séparés en différentes parties. En arrière-plan, ClickHouse fusionne des parties de données pour un stockage plus efficace. Les parties appartenant à des partitions différentes ne sont pas fusionnées. La fusion mécanisme ne garantit pas que toutes les lignes avec la même clé primaire sera dans la même partie des données. -Chaque partie de données est logiquement divisée en granules. Un granule est le plus petit ensemble de données indivisible que ClickHouse lit lors de la sélection des données. ClickHouse ne divise pas les lignes ou les valeurs, de sorte que chaque granule contient toujours un nombre entier de lignes. La première rangée de granules est marqué avec la valeur de la clé primaire de la ligne. Pour chaque partie de données, ClickHouse crée un fichier d’index qui stocke les marques. Pour chaque colonne, que ce soit dans la clé primaire ou non, ClickHouse stocke également les mêmes marques. Ces marques vous permettent de trouver des données directement dans les fichiers de colonnes. +Chaque partie de données est logiquement divisée en granules. Un granule est le plus petit ensemble de données indivisible que ClickHouse lit lors de la sélection des données. ClickHouse ne divise pas les lignes ou les valeurs, de sorte que chaque granule contient toujours un nombre entier de lignes. La première rangée de granules est marqué avec la valeur de la clé primaire de la ligne. Pour chaque partie de données, ClickHouse crée un fichier d'index qui stocke les marques. Pour chaque colonne, que ce soit dans la clé primaire ou non, ClickHouse stocke également les mêmes marques. Ces marques vous permettent de trouver des données directement dans les fichiers de colonnes. -La taille de granule est limitée par `index_granularity` et `index_granularity_bytes` paramètres du moteur de table. Le nombre de lignes dans un granule jette dans la `[1, index_granularity]` gamme, en fonction de la taille des lignes. La taille des granulés peut dépasser `index_granularity_bytes` si la taille d’une seule ligne est supérieure à la valeur du paramètre. Dans ce cas, la taille du granule est égale à la taille de la ligne. +La taille de granule est limitée par `index_granularity` et `index_granularity_bytes` paramètres du moteur de table. Le nombre de lignes dans un granule jette dans la `[1, index_granularity]` gamme, en fonction de la taille des lignes. La taille des granulés peut dépasser `index_granularity_bytes` si la taille d'une seule ligne est supérieure à la valeur du paramètre. Dans ce cas, la taille du granule est égale à la taille de la ligne. -## Clés Primaires Et Index Dans Les requêtes {#primary-keys-and-indexes-in-queries} +## Clés primaires et Index dans les requêtes {#primary-keys-and-indexes-in-queries} -Prendre la `(CounterID, Date)` clé primaire comme un exemple. Dans ce cas, le tri et l’index peuvent être illustrés comme suit: +Prendre la `(CounterID, Date)` clé primaire comme un exemple. Dans ce cas, le tri et l'index peuvent être illustrés comme suit: Whole data: [---------------------------------------------] CounterID: [aaaaaaaaaaaaaaaaaabbbbcdeeeeeeeeeeeeefgggggggghhhhhhhhhiiiiiiiiikllllllll] @@ -171,24 +171,24 @@ Si la requête de données spécifie: - `CounterID IN ('a', 'h') AND Date = 3` le serveur lit les données dans les gammes des marques `[1, 3)` et `[7, 8)`. - `Date = 3`, le serveur lit les données de la plage de marque `[1, 10]`. -Les exemples ci-dessus montrent qu’il est toujours plus efficace d’utiliser un indice qu’une analyse complète. +Les exemples ci-dessus montrent qu'il est toujours plus efficace d'utiliser un indice qu'une analyse complète. -Un index clairsemé permet de lire des données supplémentaires. Lors de la lecture d’une plage unique de la clé primaire, jusqu’à `index_granularity * 2` lignes supplémentaires dans chaque bloc de données peut être lu. +Un index clairsemé permet de lire des données supplémentaires. Lors de la lecture d'une plage unique de la clé primaire, jusqu'à `index_granularity * 2` lignes supplémentaires dans chaque bloc de données peut être lu. -Les index clairsemés vous permettent de travailler avec un très grand nombre de lignes de table, car dans la plupart des cas, ces index tiennent dans la RAM de l’ordinateur. +Les index clairsemés vous permettent de travailler avec un très grand nombre de lignes de table, car dans la plupart des cas, ces index tiennent dans la RAM de l'ordinateur. ClickHouse ne nécessite pas de clé primaire unique. Vous pouvez insérer plusieurs lignes avec la même clé primaire. -### Sélection De La clé Primaire {#selecting-the-primary-key} +### Sélection de la clé primaire {#selecting-the-primary-key} -Le nombre de colonnes de la clé primaire n’est pas explicitement limitée. Selon la structure de données, vous pouvez inclure plus ou moins de colonnes dans la clé primaire. Cela peut: +Le nombre de colonnes de la clé primaire n'est pas explicitement limitée. Selon la structure de données, vous pouvez inclure plus ou moins de colonnes dans la clé primaire. Cela peut: -- Améliorer la performance d’un indice. +- Améliorer la performance d'un indice. Si la clé primaire est `(a, b)`, puis ajouter une autre colonne `c` pour améliorer les performances si les conditions suivantes sont réunies: - Il y a des requêtes avec une condition sur la colonne `c`. - - Longues plages de données (plusieurs fois plus longues que `index_granularity`) avec des valeurs identiques pour `(a, b)` sont communs. En d’autres termes, lors de l’ajout d’une autre colonne vous permet de passer très longues plages de données. + - Longues plages de données (plusieurs fois plus longues que `index_granularity`) avec des valeurs identiques pour `(a, b)` sont communs. En d'autres termes, lors de l'ajout d'une autre colonne vous permet de passer très longues plages de données. - Améliorer la compression des données. @@ -196,26 +196,26 @@ Le nombre de colonnes de la clé primaire n’est pas explicitement limitée. Se - Fournir une logique supplémentaire lors de la fusion de parties de [CollapsingMergeTree](collapsingmergetree.md#table_engine-collapsingmergetree) et [SummingMergeTree](summingmergetree.md) moteur. - Dans ce cas, on peut spécifier l’ *clé de tri* qui est différente de la clé primaire. + Dans ce cas, on peut spécifier l' *clé de tri* qui est différente de la clé primaire. -Une clé primaire longue affectera négativement les performances d’insertion et la consommation de mémoire, mais des colonnes supplémentaires dans la clé primaire n’affecteront pas les performances de ClickHouse pendant `SELECT` requête. +Une clé primaire longue affectera négativement les performances d'insertion et la consommation de mémoire, mais des colonnes supplémentaires dans la clé primaire n'affecteront pas les performances de ClickHouse pendant `SELECT` requête. -### Choisir Une clé Primaire Qui diffère De La clé De Tri {#choosing-a-primary-key-that-differs-from-the-sorting-key} +### Choisir une clé primaire qui diffère de la clé de tri {#choosing-a-primary-key-that-differs-from-the-sorting-key} -Il est possible de spécifier une clé primaire (une expression avec des valeurs qui sont écrites dans le fichier d’index pour chaque marque) qui est différente de la clé de tri (une expression pour trier les lignes dans les parties de données). Dans ce cas, le tuple d’expression de clé primaire doit être un préfixe du tuple d’expression de clé de tri. +Il est possible de spécifier une clé primaire (une expression avec des valeurs qui sont écrites dans le fichier d'index pour chaque marque) qui est différente de la clé de tri (une expression pour trier les lignes dans les parties de données). Dans ce cas, le tuple d'expression de clé primaire doit être un préfixe du tuple d'expression de clé de tri. Cette fonctionnalité est utile lorsque vous utilisez le [SummingMergeTree](summingmergetree.md) et -[AggregatingMergeTree](aggregatingmergetree.md) table des moteurs. Dans un cas courant lors de l’utilisation de ces moteurs, la table a deux types de colonnes: *dimension* et *mesure*. Les requêtes typiques agrégent les valeurs des colonnes de mesure avec arbitraire `GROUP BY` et filtrage par dimensions. Comme SummingMergeTree et AggregatingMergeTree regroupent des lignes avec la même valeur de la clé de tri, il est naturel d’y ajouter toutes les dimensions. En conséquence, l’expression se compose d’une longue liste de colonnes, et cette liste doit être mise à jour fréquemment avec nouvellement ajoutée. +[AggregatingMergeTree](aggregatingmergetree.md) table des moteurs. Dans un cas courant lors de l'utilisation de ces moteurs, la table a deux types de colonnes: *dimension* et *mesure*. Les requêtes typiques agrégent les valeurs des colonnes de mesure avec arbitraire `GROUP BY` et filtrage par dimensions. Comme SummingMergeTree et AggregatingMergeTree regroupent des lignes avec la même valeur de la clé de tri, il est naturel d'y ajouter toutes les dimensions. En conséquence, l'expression se compose d'une longue liste de colonnes, et cette liste doit être mise à jour fréquemment avec nouvellement ajoutée. Dans ce cas, il est logique de ne laisser que quelques colonnes dans la clé primaire qui fourniront des analyses de plage efficaces et ajouteront les colonnes de dimension restantes au tuple de clé de tri. -[ALTER](../../../sql-reference/statements/alter.md) la clé de tri est une opération légère car lorsqu’une nouvelle colonne est ajoutée simultanément à la table et à la clé de tri, les parties de données existantes n’ont pas besoin d’être modifiées. Comme l’ancienne clé de tri est un préfixe de la nouvelle clé de tri et qu’il n’y a pas de données dans la colonne nouvellement ajoutée, les données sont triées à la fois par l’ancienne et la nouvelle clé de tri au moment de la modification de la table. +[ALTER](../../../sql-reference/statements/alter.md) la clé de tri est une opération légère car lorsqu'une nouvelle colonne est ajoutée simultanément à la table et à la clé de tri, les parties de données existantes n'ont pas besoin d'être modifiées. Comme l'ancienne clé de tri est un préfixe de la nouvelle clé de tri et qu'il n'y a pas de données dans la colonne nouvellement ajoutée, les données sont triées à la fois par l'ancienne et la nouvelle clé de tri au moment de la modification de la table. -### Utilisation D’Index Et De Partitions Dans Les requêtes {#use-of-indexes-and-partitions-in-queries} +### Utilisation D'Index et de Partitions dans les requêtes {#use-of-indexes-and-partitions-in-queries} -Pour `SELECT` requêtes, clickhouse analyse si un index peut être utilisé. Un index peut être utilisé si le `WHERE/PREWHERE` clause a une expression (comme l’un des éléments de conjonction, ou entièrement) qui représente une opération de comparaison d’égalité ou d’inégalité, ou si elle a `IN` ou `LIKE` avec un préfixe fixe sur les colonnes ou les expressions qui sont dans la clé primaire ou la clé de partitionnement, ou sur certaines fonctions partiellement répétitives de ces colonnes, ou les relations logiques de ces expressions. +Pour `SELECT` requêtes, clickhouse analyse si un index peut être utilisé. Un index peut être utilisé si le `WHERE/PREWHERE` clause a une expression (comme l'un des éléments de conjonction, ou entièrement) qui représente une opération de comparaison d'égalité ou d'inégalité, ou si elle a `IN` ou `LIKE` avec un préfixe fixe sur les colonnes ou les expressions qui sont dans la clé primaire ou la clé de partitionnement, ou sur certaines fonctions partiellement répétitives de ces colonnes, ou les relations logiques de ces expressions. -Ainsi, il est possible d’exécuter des requêtes sur une ou plusieurs plages de la clé primaire. Dans cet exemple, les requêtes seront rapides lorsqu’elles sont exécutées pour une balise de suivi spécifique, pour une balise et une plage de dates spécifiques, pour une balise et une date spécifiques, pour plusieurs balises avec une plage de dates, etc. +Ainsi, il est possible d'exécuter des requêtes sur une ou plusieurs plages de la clé primaire. Dans cet exemple, les requêtes seront rapides lorsqu'elles sont exécutées pour une balise de suivi spécifique, pour une balise et une plage de dates spécifiques, pour une balise et une date spécifiques, pour plusieurs balises avec une plage de dates, etc. Regardons le moteur configuré comme suit: @@ -229,31 +229,31 @@ SELECT count() FROM table WHERE EventDate = toDate(now()) AND (CounterID = 34 OR SELECT count() FROM table WHERE ((EventDate >= toDate('2014-01-01') AND EventDate <= toDate('2014-01-31')) OR EventDate = toDate('2014-05-01')) AND CounterID IN (101500, 731962, 160656) AND (CounterID = 101500 OR EventDate != toDate('2014-05-01')) ``` -ClickHouse utilisera l’index de clé primaire pour supprimer les données incorrectes et la clé de partitionnement mensuel pour supprimer les partitions qui se trouvent dans des plages de dates incorrectes. +ClickHouse utilisera l'index de clé primaire pour supprimer les données incorrectes et la clé de partitionnement mensuel pour supprimer les partitions qui se trouvent dans des plages de dates incorrectes. -Les requêtes ci-dessus montrent que l’index est utilisé même pour les expressions complexes. La lecture de la table est organisée de sorte que l’utilisation de l’index ne peut pas être plus lente qu’une analyse complète. +Les requêtes ci-dessus montrent que l'index est utilisé même pour les expressions complexes. La lecture de la table est organisée de sorte que l'utilisation de l'index ne peut pas être plus lente qu'une analyse complète. -Dans l’exemple ci-dessous, l’index ne peut pas être utilisé. +Dans l'exemple ci-dessous, l'index ne peut pas être utilisé. ``` sql SELECT count() FROM table WHERE CounterID = 34 OR URL LIKE '%upyachka%' ``` -Pour vérifier si ClickHouse pouvez utiliser l’index lors de l’exécution d’une requête, utilisez les paramètres [force\_index\_by\_date](../../../operations/settings/settings.md#settings-force_index_by_date) et [force\_primary\_key](../../../operations/settings/settings.md). +Pour vérifier si ClickHouse pouvez utiliser l'index lors de l'exécution d'une requête, utilisez les paramètres [force\_index\_by\_date](../../../operations/settings/settings.md#settings-force_index_by_date) et [force\_primary\_key](../../../operations/settings/settings.md). -La clé de partitionnement par mois permet de lire uniquement les blocs de données qui contiennent des dates de la plage appropriée. Dans ce cas, le bloc de données peut contenir des données pour plusieurs dates (jusqu’à un mois entier). Dans un bloc, les données sont triées par clé primaire, qui peut ne pas contenir la date comme première colonne. Pour cette raison, l’utilisation d’une requête avec seulement une condition de date qui ne spécifie pas le préfixe de clé primaire entraînera la lecture de plus de données que pour une seule date. +La clé de partitionnement par mois permet de lire uniquement les blocs de données qui contiennent des dates de la plage appropriée. Dans ce cas, le bloc de données peut contenir des données pour plusieurs dates (jusqu'à un mois entier). Dans un bloc, les données sont triées par clé primaire, qui peut ne pas contenir la date comme première colonne. Pour cette raison, l'utilisation d'une requête avec seulement une condition de date qui ne spécifie pas le préfixe de clé primaire entraînera la lecture de plus de données que pour une seule date. -### Utilisation De L’Index Pour Les clés Primaires Partiellement Monotones {#use-of-index-for-partially-monotonic-primary-keys} +### Utilisation de L'Index pour les clés primaires partiellement monotones {#use-of-index-for-partially-monotonic-primary-keys} -Considérons, par exemple, les jours du mois. Ils forment un [monotone de la séquence](https://en.wikipedia.org/wiki/Monotonic_function) pendant un mois, mais pas monotone pendant des périodes plus longues. C’est une séquence partiellement monotone. Si un utilisateur crée la table avec une clé primaire partiellement monotone, ClickHouse crée un index clairsemé comme d’habitude. Lorsqu’un utilisateur sélectionne des données à partir de ce type de table, ClickHouse analyse les conditions de requête. Si L’utilisateur veut obtenir des données entre deux marques de l’index et que ces deux marques tombent dans un mois, ClickHouse peut utiliser l’index dans ce cas particulier car il peut calculer la distance entre les paramètres d’une requête et les marques d’index. +Considérons, par exemple, les jours du mois. Ils forment un [monotone de la séquence](https://en.wikipedia.org/wiki/Monotonic_function) pendant un mois, mais pas monotone pendant des périodes plus longues. C'est une séquence partiellement monotone. Si un utilisateur crée la table avec une clé primaire partiellement monotone, ClickHouse crée un index clairsemé comme d'habitude. Lorsqu'un utilisateur sélectionne des données à partir de ce type de table, ClickHouse analyse les conditions de requête. Si L'utilisateur veut obtenir des données entre deux marques de l'index et que ces deux marques tombent dans un mois, ClickHouse peut utiliser l'index dans ce cas particulier car il peut calculer la distance entre les paramètres d'une requête et les marques d'index. ClickHouse ne peut pas utiliser un index si les valeurs de la clé primaire dans la plage de paramètres de requête ne représentent pas une séquence monotone. Dans ce cas, ClickHouse utilise la méthode full scan. ClickHouse utilise cette logique non seulement pour les séquences de jours du mois, mais pour toute clé primaire qui représente une séquence partiellement monotone. -### Index De Saut De données (expérimental) {#table_engine-mergetree-data_skipping-indexes} +### Index de saut de données (expérimental) {#table_engine-mergetree-data_skipping-indexes} -La déclaration d’index se trouve dans la section colonnes du `CREATE` requête. +La déclaration d'index se trouve dans la section colonnes du `CREATE` requête. ``` sql INDEX index_name expr TYPE type(...) GRANULARITY granularity_value @@ -261,7 +261,7 @@ INDEX index_name expr TYPE type(...) GRANULARITY granularity_value Pour les tables de la `*MergeTree` famille, les indices de saut de données peuvent être spécifiés. -Ces indices agrégent certaines informations sur l’expression spécifiée sur les blocs, qui consistent en `granularity_value` granules (la taille du granule est spécifiée en utilisant `index_granularity` réglage dans le moteur de table). Ensuite, ces agrégats sont utilisés dans `SELECT` requêtes pour réduire la quantité de données à lire à partir du disque en ignorant de gros blocs de données `where` la requête ne peut pas être satisfait. +Ces indices agrégent certaines informations sur l'expression spécifiée sur les blocs, qui consistent en `granularity_value` granules (la taille du granule est spécifiée en utilisant `index_granularity` réglage dans le moteur de table). Ensuite, ces agrégats sont utilisés dans `SELECT` requêtes pour réduire la quantité de données à lire à partir du disque en ignorant de gros blocs de données `where` la requête ne peut pas être satisfait. **Exemple** @@ -278,26 +278,26 @@ CREATE TABLE table_name ... ``` -Les Indices de L’exemple peuvent être utilisés par ClickHouse pour réduire la quantité de données à lire à partir du disque dans les requêtes suivantes: +Les Indices de L'exemple peuvent être utilisés par ClickHouse pour réduire la quantité de données à lire à partir du disque dans les requêtes suivantes: ``` sql SELECT count() FROM table WHERE s < 'z' SELECT count() FROM table WHERE u64 * i32 == 10 AND u64 * length(s) >= 1234 ``` -#### Types D’Indices Disponibles {#available-types-of-indices} +#### Types d'Indices disponibles {#available-types-of-indices} - `minmax` - Magasins extrêmes de l’expression spécifiée (si l’expression est `tuple` puis il stocke les extrêmes pour chaque élément de `tuple`), utilise les informations stockées pour sauter des blocs de données comme la clé primaire. + Magasins extrêmes de l'expression spécifiée (si l'expression est `tuple` puis il stocke les extrêmes pour chaque élément de `tuple`), utilise les informations stockées pour sauter des blocs de données comme la clé primaire. - `set(max_rows)` - Stocke les valeurs uniques de l’expression spécifiée (pas plus de `max_rows` rangée, `max_rows=0` moyen “no limits”). Utilise les valeurs pour vérifier si le `WHERE` l’expression n’est pas satisfiable sur un bloc de données. + Stocke les valeurs uniques de l'expression spécifiée (pas plus de `max_rows` rangée, `max_rows=0` moyen “no limits”). Utilise les valeurs pour vérifier si le `WHERE` l'expression n'est pas satisfiable sur un bloc de données. - `ngrambf_v1(n, size_of_bloom_filter_in_bytes, number_of_hash_functions, random_seed)` - Magasins un [Filtre de Bloom](https://en.wikipedia.org/wiki/Bloom_filter) qui contient tous les ngrams d’un bloc de données. Fonctionne uniquement avec des chaînes. Peut être utilisé pour l’optimisation de `equals`, `like` et `in` expression. + Magasins un [Filtre de Bloom](https://en.wikipedia.org/wiki/Bloom_filter) qui contient tous les ngrams d'un bloc de données. Fonctionne uniquement avec des chaînes. Peut être utilisé pour l'optimisation de `equals`, `like` et `in` expression. - `n` — ngram size, - `size_of_bloom_filter_in_bytes` — Bloom filter size in bytes (you can use large values here, for example, 256 or 512, because it can be compressed well). @@ -314,7 +314,7 @@ SELECT count() FROM table WHERE u64 * i32 == 10 AND u64 * length(s) >= 1234 Types de données pris en charge: `Int*`, `UInt*`, `Float*`, `Enum`, `Date`, `DateTime`, `String`, `FixedString`, `Array`, `LowCardinality`, `Nullable`. - Les fonctions suivantes peuvent l’utiliser: [égal](../../../sql-reference/functions/comparison-functions.md), [notEquals](../../../sql-reference/functions/comparison-functions.md), [dans](../../../sql-reference/functions/in-functions.md), [notIn](../../../sql-reference/functions/in-functions.md), [avoir](../../../sql-reference/functions/array-functions.md). + Les fonctions suivantes peuvent l'utiliser: [égal](../../../sql-reference/functions/comparison-functions.md), [notEquals](../../../sql-reference/functions/comparison-functions.md), [dans](../../../sql-reference/functions/in-functions.md), [notIn](../../../sql-reference/functions/in-functions.md), [avoir](../../../sql-reference/functions/array-functions.md). @@ -326,9 +326,9 @@ INDEX sample_index3 (lower(str), str) TYPE ngrambf_v1(3, 256, 2, 0) GRANULARITY #### Les Fonctions De Soutien {#functions-support} -Les Conditions dans le `WHERE` la clause contient des appels des fonctions qui fonctionnent avec des colonnes. Si la colonne fait partie d’un index, ClickHouse essaie d’utiliser cet index lors de l’exécution des fonctions. ClickHouse prend en charge différents sous-ensembles de fonctions pour l’utilisation d’index. +Les Conditions dans le `WHERE` la clause contient des appels des fonctions qui fonctionnent avec des colonnes. Si la colonne fait partie d'un index, ClickHouse essaie d'utiliser cet index lors de l'exécution des fonctions. ClickHouse prend en charge différents sous-ensembles de fonctions pour l'utilisation d'index. -Le `set` l’indice peut être utilisé avec toutes les fonctions. Les sous-ensembles de fonctions pour les autres index sont présentés dans le tableau ci-dessous. +Le `set` l'indice peut être utilisé avec toutes les fonctions. Les sous-ensembles de fonctions pour les autres index sont présentés dans le tableau ci-dessous. | Fonction (opérateur) / Indice de | clé primaire | minmax | ngrambf\_v1 | tokenbf\_v1 | bloom\_filter | |--------------------------------------------------------------------------------------------------------------|--------------|--------|-------------|-------------|---------------| @@ -349,9 +349,9 @@ Le `set` l’indice peut être utilisé avec toutes les fonctions. Les sous-ense | [notEmpty](../../../sql-reference/functions/array-functions.md#function-notempty) | ✔ | ✔ | ✗ | ✗ | ✗ | | hasToken | ✗ | ✗ | ✗ | ✔ | ✗ | -Les fonctions avec un argument constant inférieur à la taille ngram ne peuvent pas être utilisées par `ngrambf_v1` pour l’optimisation de la requête. +Les fonctions avec un argument constant inférieur à la taille ngram ne peuvent pas être utilisées par `ngrambf_v1` pour l'optimisation de la requête. -Les filtres Bloom peuvent avoir des correspondances faussement positives, de sorte que le `ngrambf_v1`, `tokenbf_v1`, et `bloom_filter` les index ne peuvent pas être utilisés pour optimiser les requêtes où le résultat d’une fonction est censé être faux, par exemple: +Les filtres Bloom peuvent avoir des correspondances faussement positives, de sorte que le `ngrambf_v1`, `tokenbf_v1`, et `bloom_filter` les index ne peuvent pas être utilisés pour optimiser les requêtes où le résultat d'une fonction est censé être faux, par exemple: - Peut être optimisé: - `s LIKE '%test%'` @@ -368,11 +368,11 @@ Les filtres Bloom peuvent avoir des correspondances faussement positives, de sor ## Accès Simultané Aux Données {#concurrent-data-access} -Pour l’accès aux tables simultanées, nous utilisons le multi-versioning. En d’autres termes, lorsqu’une table est lue et mise à jour simultanément, les données sont lues à partir d’un ensemble de parties en cours au moment de la requête. Il n’y a pas de longues mèches. Les Inserts ne gênent pas les opérations de lecture. +Pour l'accès aux tables simultanées, nous utilisons le multi-versioning. En d'autres termes, lorsqu'une table est lue et mise à jour simultanément, les données sont lues à partir d'un ensemble de parties en cours au moment de la requête. Il n'y a pas de longues mèches. Les Inserts ne gênent pas les opérations de lecture. -Lecture à partir d’un tableau est automatiquement parallélisée. +Lecture à partir d'un tableau est automatiquement parallélisée. -## TTL Pour Les Colonnes Et Les Tableaux {#table_engine-mergetree-ttl} +## TTL pour les colonnes et les tableaux {#table_engine-mergetree-ttl} Détermine la durée de vie de des valeurs. @@ -387,7 +387,7 @@ TTL time_column TTL time_column + interval ``` -Définir `interval`, utiliser [intervalle](../../../sql-reference/operators.md#operators-datetime) opérateur. +Définir `interval`, utiliser [intervalle](../../../sql-reference/operators/index.md#operators-datetime) opérateur. ``` sql TTL date_time + INTERVAL 1 MONTH @@ -396,13 +396,13 @@ TTL date_time + INTERVAL 15 HOUR ### Colonne TTL {#mergetree-column-ttl} -Lorsque les valeurs de la colonne expirent, ClickHouse les remplace par les valeurs par défaut du type de données de la colonne. Si toutes les valeurs de colonne de la partie données expirent, ClickHouse supprime cette colonne de la partie données d’un système de fichiers. +Lorsque les valeurs de la colonne expirent, ClickHouse les remplace par les valeurs par défaut du type de données de la colonne. Si toutes les valeurs de colonne de la partie données expirent, ClickHouse supprime cette colonne de la partie données d'un système de fichiers. Le `TTL` la clause ne peut pas être utilisée pour les colonnes clés. Exemple: -Création d’une table avec TTL +Création d'une table avec TTL ``` sql CREATE TABLE example_table @@ -417,7 +417,7 @@ PARTITION BY toYYYYMM(d) ORDER BY d; ``` -Ajout de TTL à une colonne d’une table existante +Ajout de TTL à une colonne d'une table existante ``` sql ALTER TABLE example_table @@ -435,13 +435,13 @@ ALTER TABLE example_table ### Tableau TTL {#mergetree-table-ttl} -Table peut avoir une expression pour la suppression de Lignes expirées, et plusieurs expressions pour le déplacement automatique de pièces entre [disques ou volumes](#table_engine-mergetree-multiple-volumes). Lorsque les lignes de la table expirent, ClickHouse supprime toutes les lignes correspondantes. Pour les pièces en mouvement, toutes les lignes d’une pièce doivent satisfaire aux critères d’expression de mouvement. +Table peut avoir une expression pour la suppression de Lignes expirées, et plusieurs expressions pour le déplacement automatique de pièces entre [disques ou volumes](#table_engine-mergetree-multiple-volumes). Lorsque les lignes de la table expirent, ClickHouse supprime toutes les lignes correspondantes. Pour les pièces en mouvement, toutes les lignes d'une pièce doivent satisfaire aux critères d'expression de mouvement. ``` sql TTL expr [DELETE|TO DISK 'aaa'|TO VOLUME 'bbb'], ... ``` -Type de règle TTL peut suivre chaque expression TTL. Il affecte une action qui doit être faite une fois que l’expression est satisfaite (atteint l’heure actuelle): +Type de règle TTL peut suivre chaque expression TTL. Il affecte une action qui doit être faite une fois que l'expression est satisfaite (atteint l'heure actuelle): - `DELETE` - supprimer les Lignes expirées (action par défaut); - `TO DISK 'aaa'` - déplacer la partie sur le disque `aaa`; @@ -449,7 +449,7 @@ Type de règle TTL peut suivre chaque expression TTL. Il affecte une action qui Exemple: -Création d’une table avec TTL +Création d'une table avec TTL ``` sql CREATE TABLE example_table @@ -478,15 +478,15 @@ Les données avec un TTL expiré sont supprimées lorsque ClickHouse fusionne de Lorsque ClickHouse voit que les données sont expirées, il effectue une fusion hors calendrier. Pour contrôler la fréquence de ces fusions, vous pouvez définir `merge_with_ttl_timeout`. Si la valeur est trop faible, il effectuera de nombreuses fusions hors calendrier qui peuvent consommer beaucoup de ressources. -Si vous effectuez la `SELECT` requête entre les fusionne, vous pouvez obtenir des données expirées. Pour éviter cela, utilisez la [OPTIMIZE](../../../sql-reference/statements/misc.md#misc_operations-optimize) requête avant de l’ `SELECT`. +Si vous effectuez la `SELECT` requête entre les fusionne, vous pouvez obtenir des données expirées. Pour éviter cela, utilisez la [OPTIMIZE](../../../sql-reference/statements/misc.md#misc_operations-optimize) requête avant de l' `SELECT`. -## Utilisation De Plusieurs Périphériques De Bloc Pour Le Stockage De Données {#table_engine-mergetree-multiple-volumes} +## Utilisation de plusieurs périphériques de bloc pour le stockage de données {#table_engine-mergetree-multiple-volumes} ### Introduction {#introduction} -`MergeTree` les moteurs de table de famille peuvent stocker des données sur plusieurs périphériques de bloc. Par exemple, il peut être utile lorsque les données d’un tableau sont implicitement divisé en “hot” et “cold”. Les données les plus récentes sont régulièrement demandées mais ne nécessitent qu’une petite quantité d’espace. Au contraire, les données historiques à queue grasse sont rarement demandées. Si plusieurs disques sont disponibles, la “hot” les données peuvent être situées sur des disques rapides (par exemple, SSD NVMe ou en mémoire), tandis que le “cold” des données relativement lente (par exemple, disque dur). +`MergeTree` les moteurs de table de famille peuvent stocker des données sur plusieurs périphériques de bloc. Par exemple, il peut être utile lorsque les données d'un tableau sont implicitement divisé en “hot” et “cold”. Les données les plus récentes sont régulièrement demandées mais ne nécessitent qu'une petite quantité d'espace. Au contraire, les données historiques à queue grasse sont rarement demandées. Si plusieurs disques sont disponibles, la “hot” les données peuvent être situées sur des disques rapides (par exemple, SSD NVMe ou en mémoire), tandis que le “cold” des données relativement lente (par exemple, disque dur). -La partie de données est l’unité mobile minimum pour `MergeTree`-tables de moteur. Les données appartenant à une partie sont stockées sur un disque. Les parties de données peuvent être déplacées entre les disques en arrière-plan (selon les paramètres de l’utilisateur) ainsi qu’au moyen du [ALTER](../../../sql-reference/statements/alter.md#alter_move-partition) requête. +La partie de données est l'unité mobile minimum pour `MergeTree`-tables de moteur. Les données appartenant à une partie sont stockées sur un disque. Les parties de données peuvent être déplacées entre les disques en arrière-plan (selon les paramètres de l'utilisateur) ainsi qu'au moyen du [ALTER](../../../sql-reference/statements/alter.md#alter_move-partition) requête. ### Terme {#terms} @@ -495,7 +495,7 @@ La partie de données est l’unité mobile minimum pour `MergeTree`-tables de m - Volume — Ordered set of equal disks (similar to [JBOD](https://en.wikipedia.org/wiki/Non-RAID_drive_architectures)). - Storage policy — Set of volumes and the rules for moving data between them. -Les noms donnés aux entités décrites peuvent être trouvés dans les tables système, [système.storage\_policies](../../../operations/system-tables.md#system_tables-storage_policies) et [système.disque](../../../operations/system-tables.md#system_tables-disks). Pour appliquer l’une des stratégies de stockage configurées pour une table, utilisez `storage_policy` réglage de `MergeTree`-moteur de table de famille. +Les noms donnés aux entités décrites peuvent être trouvés dans les tables système, [système.storage\_policies](../../../operations/system-tables.md#system_tables-storage_policies) et [système.disque](../../../operations/system-tables.md#system_tables-disks). Pour appliquer l'une des stratégies de stockage configurées pour une table, utilisez `storage_policy` réglage de `MergeTree`-moteur de table de famille. ### Configuration {#table_engine-mergetree-multiple-volumes_configure} @@ -531,7 +531,7 @@ Balise: - `path` — path under which a server will store data (`data` et `shadow` des dossiers), doit être terminé par ‘/’. - `keep_free_space_bytes` — the amount of free disk space to be reserved. -L’ordre du disque définition n’est pas important. +L'ordre du disque définition n'est pas important. Stratégies de stockage balisage de configuration: @@ -567,7 +567,7 @@ Balise: - `policy_name_N` — Policy name. Policy names must be unique. - `volume_name_N` — Volume name. Volume names must be unique. - `disk` — a disk within a volume. -- `max_data_part_size_bytes` — the maximum size of a part that can be stored on any of the volume’s disks. +- `max_data_part_size_bytes` — the maximum size of a part that can be stored on any of the volume's disks. - `move_factor` — when the amount of available space gets lower than this factor, data automatically start to move on the next volume if any (by default, 0.1). Exemples de Cofiguration: @@ -602,14 +602,14 @@ Exemples de Cofiguration: ``` -Dans l’exemple donné, la `hdd_in_order` politique met en œuvre les [round-robin](https://en.wikipedia.org/wiki/Round-robin_scheduling) approche. Ainsi cette politique ne définit qu’un seul volume (`single`), les parties des données sont stockées sur tous ses disques dans l’ordre circulaire. Une telle politique peut être très utile s’il y a plusieurs disques similaires sont montés sur le système, mais RAID N’est pas configuré. Gardez à l’esprit que chaque lecteur de disque n’est pas fiable et vous pouvez compenser avec facteur de réplication de 3 ou plus. +Dans l'exemple donné, la `hdd_in_order` politique met en œuvre les [round-robin](https://en.wikipedia.org/wiki/Round-robin_scheduling) approche. Ainsi cette politique ne définit qu'un seul volume (`single`), les parties des données sont stockées sur tous ses disques dans l'ordre circulaire. Une telle politique peut être très utile s'il y a plusieurs disques similaires sont montés sur le système, mais RAID N'est pas configuré. Gardez à l'esprit que chaque lecteur de disque n'est pas fiable et vous pouvez compenser avec facteur de réplication de 3 ou plus. -S’il existe différents types de disques disponibles dans le système, `moving_from_ssd_to_hdd` la stratégie peut être utilisée à la place. Volume `hot` se compose d’un disque SSD (`fast_ssd`), et la taille maximale d’une pièce qui peut être stocké sur ce volume est de 1 go. Toutes les pièces avec la taille plus grande que 1 GB sera stocké directement sur le `cold` le volume, qui contient un disque dur de disque `disk1`. -Aussi, une fois le disque `fast_ssd` est complété par plus de 80%, les données seront transférées à la `disk1` par un processus d’arrière-plan. +S'il existe différents types de disques disponibles dans le système, `moving_from_ssd_to_hdd` la stratégie peut être utilisée à la place. Volume `hot` se compose d'un disque SSD (`fast_ssd`), et la taille maximale d'une pièce qui peut être stocké sur ce volume est de 1 go. Toutes les pièces avec la taille plus grande que 1 GB sera stocké directement sur le `cold` le volume, qui contient un disque dur de disque `disk1`. +Aussi, une fois le disque `fast_ssd` est complété par plus de 80%, les données seront transférées à la `disk1` par un processus d'arrière-plan. -L’ordre d’énumération des volumes dans une stratégie de stockage est important. Une fois qu’un volume est surchargé, les données sont déplacées vers le suivant. L’ordre d’énumération des disques est important parce que les données sont stockées dans les virages. +L'ordre d'énumération des volumes dans une stratégie de stockage est important. Une fois qu'un volume est surchargé, les données sont déplacées vers le suivant. L'ordre d'énumération des disques est important parce que les données sont stockées dans les virages. -Lors de la création d’une table, on peut lui appliquer l’une des stratégies de stockage configurées: +Lors de la création d'une table, on peut lui appliquer l'une des stratégies de stockage configurées: ``` sql CREATE TABLE table_with_non_default_policy ( @@ -623,32 +623,32 @@ PARTITION BY toYYYYMM(EventDate) SETTINGS storage_policy = 'moving_from_ssd_to_hdd' ``` -Le `default` la Politique de stockage implique d’utiliser un seul volume, qui se compose d’un seul disque donné dans ``. Une fois qu’une table est créée, sa stratégie de stockage ne peut pas être modifiée. +Le `default` la Politique de stockage implique d'utiliser un seul volume, qui se compose d'un seul disque donné dans ``. Une fois qu'une table est créée, sa stratégie de stockage ne peut pas être modifiée. ### Détail {#details} Dans le cas de `MergeTree` les tableaux, les données sont sur le disque de différentes façons: -- En tant que résultat d’un insert (`INSERT` requête). +- En tant que résultat d'un insert (`INSERT` requête). - En arrière-plan fusionne et [mutation](../../../sql-reference/statements/alter.md#alter-mutations). -- Lors du téléchargement à partir d’une autre réplique. +- Lors du téléchargement à partir d'une autre réplique. - À la suite du gel de la partition [ALTER TABLE … FREEZE PARTITION](../../../sql-reference/statements/alter.md#alter_freeze-partition). -Dans tous ces cas, à l’exception des mutations et du gel de partition, une pièce est stockée sur un volume et un disque selon la Politique de stockage donnée: +Dans tous ces cas, à l'exception des mutations et du gel de partition, une pièce est stockée sur un volume et un disque selon la Politique de stockage donnée: -1. Le premier volume (dans l’ordre de définition) qui a suffisamment d’espace disque pour stocker une pièce (`unreserved_space > current_part_size`) et permet de stocker des pièces d’une taille donnée (`max_data_part_size_bytes > current_part_size`) est choisi. -2. Dans ce volume, ce disque est choisi qui suit celui, qui a été utilisé pour stocker le bloc de données précédent, et qui a de l’espace libre plus que la taille de la pièce (`unreserved_space - keep_free_space_bytes > current_part_size`). +1. Le premier volume (dans l'ordre de définition) qui a suffisamment d'espace disque pour stocker une pièce (`unreserved_space > current_part_size`) et permet de stocker des pièces d'une taille donnée (`max_data_part_size_bytes > current_part_size`) est choisi. +2. Dans ce volume, ce disque est choisi qui suit celui, qui a été utilisé pour stocker le bloc de données précédent, et qui a de l'espace libre plus que la taille de la pièce (`unreserved_space - keep_free_space_bytes > current_part_size`). Sous le capot, les mutations et la congélation des cloisons utilisent [des liens en dur](https://en.wikipedia.org/wiki/Hard_link). Les liens durs entre différents disques ne sont pas pris en charge, donc dans de tels cas, les pièces résultantes sont stockées sur les mêmes disques que les disques initiaux. -En arrière - plan, les pièces sont déplacées entre les volumes en fonction de la quantité d’espace libre (`move_factor` paramètre) selon l’ordre les volumes sont déclarées dans le fichier de configuration. -Les données ne sont jamais transférées du dernier et dans le premier. On peut utiliser des tables système [système.part\_log](../../../operations/system-tables.md#system_tables-part-log) (champ `type = MOVE_PART`) et [système.partie](../../../operations/system-tables.md#system_tables-parts) (Fields `path` et `disk`) pour surveiller l’arrière-plan se déplace. Aussi, les informations détaillées peuvent être trouvées dans les journaux du serveur. +En arrière - plan, les pièces sont déplacées entre les volumes en fonction de la quantité d'espace libre (`move_factor` paramètre) selon l'ordre les volumes sont déclarées dans le fichier de configuration. +Les données ne sont jamais transférées du dernier et dans le premier. On peut utiliser des tables système [système.part\_log](../../../operations/system-tables.md#system_tables-part-log) (champ `type = MOVE_PART`) et [système.partie](../../../operations/system-tables.md#system_tables-parts) (Fields `path` et `disk`) pour surveiller l'arrière-plan se déplace. Aussi, les informations détaillées peuvent être trouvées dans les journaux du serveur. -L’utilisateur peut forcer le déplacement d’une partie ou d’une partition d’un volume à l’autre à l’aide de la requête [ALTER TABLE … MOVE PART\|PARTITION … TO VOLUME\|DISK …](../../../sql-reference/statements/alter.md#alter_move-partition), toutes les restrictions pour les opérations de fond sont prises en compte. La requête initie un mouvement seul et n’attend pas que les opérations d’arrière-plan soient terminées. L’utilisateur recevra un message d’erreur si pas assez d’espace libre est disponible ou si l’une des conditions requises ne sont pas remplies. +L'utilisateur peut forcer le déplacement d'une partie ou d'une partition d'un volume à l'autre à l'aide de la requête [ALTER TABLE … MOVE PART\|PARTITION … TO VOLUME\|DISK …](../../../sql-reference/statements/alter.md#alter_move-partition), toutes les restrictions pour les opérations de fond sont prises en compte. La requête initie un mouvement seul et n'attend pas que les opérations d'arrière-plan soient terminées. L'utilisateur recevra un message d'erreur si pas assez d'espace libre est disponible ou si l'une des conditions requises ne sont pas remplies. -Le déplacement des données n’interfère pas avec la réplication des données. Par conséquent, différentes stratégies de stockage peuvent être spécifiées pour la même table sur différents réplicas. +Le déplacement des données n'interfère pas avec la réplication des données. Par conséquent, différentes stratégies de stockage peuvent être spécifiées pour la même table sur différents réplicas. -Après l’achèvement des fusions d’arrière-plan et des mutations, les anciennes parties ne sont supprimées qu’après un certain temps (`old_parts_lifetime`). -Pendant ce temps, ils ne sont pas déplacés vers d’autres volumes ou des disques. Par conséquent, jusqu’à ce que les pièces soient finalement supprimées, elles sont toujours prises en compte pour l’évaluation de l’espace disque occupé. +Après l'achèvement des fusions d'arrière-plan et des mutations, les anciennes parties ne sont supprimées qu'après un certain temps (`old_parts_lifetime`). +Pendant ce temps, ils ne sont pas déplacés vers d'autres volumes ou des disques. Par conséquent, jusqu'à ce que les pièces soient finalement supprimées, elles sont toujours prises en compte pour l'évaluation de l'espace disque occupé. [Article Original](https://clickhouse.tech/docs/ru/operations/table_engines/mergetree/) diff --git a/docs/fr/engines/table-engines/mergetree-family/replacingmergetree.md b/docs/fr/engines/table-engines/mergetree-family/replacingmergetree.md index 85f8fc7b8ff..ac3c0f3b021 100644 --- a/docs/fr/engines/table-engines/mergetree-family/replacingmergetree.md +++ b/docs/fr/engines/table-engines/mergetree-family/replacingmergetree.md @@ -1,19 +1,19 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 33 toc_title: ReplacingMergeTree --- -# Replacingmergetree {#replacingmergetree} +# ReplacingMergeTree {#replacingmergetree} -Le moteur diffère de [MergeTree](mergetree.md#table_engines-mergetree) en ce qu’il supprime les doublons avec la même valeur de clé primaire (ou, plus précisément, avec la même [clé de tri](mergetree.md) valeur). +Le moteur diffère de [MergeTree](mergetree.md#table_engines-mergetree) en ce qu'il supprime les doublons avec la même valeur de clé primaire (ou, plus précisément, avec la même [clé de tri](mergetree.md) valeur). -La déduplication des données se produit uniquement lors d’une fusion. La fusion se produit en arrière-plan à un moment inconnu, vous ne pouvez donc pas le planifier. Certaines des données peuvent rester non traitées. Bien que vous puissiez exécuter une fusion imprévue en utilisant le `OPTIMIZE` requête, ne comptez pas l’utiliser, parce que la `OPTIMIZE` requête va lire et écrire une grande quantité de données. +La déduplication des données se produit uniquement lors d'une fusion. La fusion se produit en arrière-plan à un moment inconnu, vous ne pouvez donc pas le planifier. Certaines des données peuvent rester non traitées. Bien que vous puissiez exécuter une fusion imprévue en utilisant le `OPTIMIZE` requête, ne comptez pas l'utiliser, parce que la `OPTIMIZE` requête va lire et écrire une grande quantité de données. -Ainsi, `ReplacingMergeTree` convient pour effacer les données en double en arrière-plan afin d’économiser de l’espace, mais cela ne garantit pas l’absence de doublons. +Ainsi, `ReplacingMergeTree` convient pour effacer les données en double en arrière-plan afin d'économiser de l'espace, mais cela ne garantit pas l'absence de doublons. -## Création d’une Table {#creating-a-table} +## Création d'une Table {#creating-a-table} ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] @@ -35,21 +35,21 @@ Pour une description des paramètres de requête, voir [demande de description]( - `ver` — column with version. Type `UInt*`, `Date` ou `DateTime`. Paramètre facultatif. - Lors de la fusion, `ReplacingMergeTree` de toutes les lignes avec la même clé primaire ne laisse qu’un: + Lors de la fusion, `ReplacingMergeTree` de toutes les lignes avec la même clé primaire ne laisse qu'un: - Dernier dans la sélection, si `ver` pas ensemble. - Avec la version maximale, si `ver` défini. **Les clauses de requête** -Lors de la création d’un `ReplacingMergeTree` la table de la même [clause](mergetree.md) sont nécessaires, comme lors de la création d’un `MergeTree` table. +Lors de la création d'un `ReplacingMergeTree` la table de la même [clause](mergetree.md) sont nécessaires, comme lors de la création d'un `MergeTree` table.
Méthode obsolète pour créer une Table !!! attention "Attention" - N’utilisez pas cette méthode dans les nouveaux projets et, si possible, remplacez les anciens projets par la méthode décrite ci-dessus. + N'utilisez pas cette méthode dans les nouveaux projets et, si possible, remplacez les anciens projets par la méthode décrite ci-dessus. ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] diff --git a/docs/fr/engines/table-engines/mergetree-family/replication.md b/docs/fr/engines/table-engines/mergetree-family/replication.md index 6c444d689eb..49690204869 100644 --- a/docs/fr/engines/table-engines/mergetree-family/replication.md +++ b/docs/fr/engines/table-engines/mergetree-family/replication.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 31 toc_title: "R\xE9plication Des Donn\xE9es" --- # Réplication Des Données {#table_engines-replication} -La réplication n’est prise en charge que pour les tables de la famille MergeTree: +La réplication n'est prise en charge que pour les tables de la famille MergeTree: - ReplicatedMergeTree - ReplicatedSummingMergeTree @@ -17,17 +17,17 @@ La réplication n’est prise en charge que pour les tables de la famille MergeT - ReplicatedVersionedCollapsingMergetree - ReplicatedGraphiteMergeTree -La réplication fonctionne au niveau d’une table individuelle, Pas du serveur entier. Un serveur peut stocker des tables répliquées et non répliquées en même temps. +La réplication fonctionne au niveau d'une table individuelle, Pas du serveur entier. Un serveur peut stocker des tables répliquées et non répliquées en même temps. La réplication ne dépend pas de la fragmentation. Chaque fragment a sa propre réplication indépendante. -Données compressées pour `INSERT` et `ALTER` les requêtes sont répliquées (pour plus d’informations, consultez la documentation de [ALTER](../../../sql-reference/statements/alter.md#query_language_queries_alter)). +Données compressées pour `INSERT` et `ALTER` les requêtes sont répliquées (pour plus d'informations, consultez la documentation de [ALTER](../../../sql-reference/statements/alter.md#query_language_queries_alter)). `CREATE`, `DROP`, `ATTACH`, `DETACH` et `RENAME` les requêtes sont exécutées sur un seul serveur et ne sont pas répliquées: -- Le `CREATE TABLE` query crée une nouvelle table réplicable sur le serveur où la requête est exécutée. Si cette table existe déjà sur d’autres serveurs, il ajoute une nouvelle réplique. -- Le `DROP TABLE` requête supprime la réplique situé sur le serveur où l’exécution de la requête. -- Le `RENAME` requête renomme la table sur l’une des répliques. En d’autres termes, les tables répliquées peuvent avoir des noms différents sur différentes répliques. +- Le `CREATE TABLE` query crée une nouvelle table réplicable sur le serveur où la requête est exécutée. Si cette table existe déjà sur d'autres serveurs, il ajoute une nouvelle réplique. +- Le `DROP TABLE` requête supprime la réplique situé sur le serveur où l'exécution de la requête. +- Le `RENAME` requête renomme la table sur l'une des répliques. En d'autres termes, les tables répliquées peuvent avoir des noms différents sur différentes répliques. Clickhouse utilise [Apache ZooKeeper](https://zookeeper.apache.org) pour stocker des informations méta répliques. Utilisez ZooKeeper version 3.4.5 ou plus récente. @@ -55,27 +55,27 @@ Exemple de définition des adresses du cluster ZooKeeper: ``` -Vous pouvez spécifier N’importe quel cluster Zookeeper existant et le système utilisera un répertoire pour ses propres données (le répertoire est spécifié lors de la création d’une table réplicable). +Vous pouvez spécifier N'importe quel cluster Zookeeper existant et le système utilisera un répertoire pour ses propres données (le répertoire est spécifié lors de la création d'une table réplicable). -Si ZooKeeper n’est pas défini dans le fichier de configuration, vous ne pouvez pas créer de tables répliquées et toutes les tables répliquées existantes seront en lecture seule. +Si ZooKeeper n'est pas défini dans le fichier de configuration, vous ne pouvez pas créer de tables répliquées et toutes les tables répliquées existantes seront en lecture seule. -La gardienne n’est pas utilisé dans `SELECT` requêtes car la réplication n’affecte pas les performances de `SELECT` et les requêtes s’exécutent aussi vite que pour les tables non répliquées. Lors de l’interrogation de tables répliquées distribuées, le comportement de ClickHouse est contrôlé par les paramètres [max\_replica\_delay\_for\_distributed\_queries](../../../operations/settings/settings.md#settings-max_replica_delay_for_distributed_queries) et [fallback\_to\_stale\_replicas\_for\_distributed\_queries](../../../operations/settings/settings.md#settings-fallback_to_stale_replicas_for_distributed_queries). +La gardienne n'est pas utilisé dans `SELECT` requêtes car la réplication n'affecte pas les performances de `SELECT` et les requêtes s'exécutent aussi vite que pour les tables non répliquées. Lors de l'interrogation de tables répliquées distribuées, le comportement de ClickHouse est contrôlé par les paramètres [max\_replica\_delay\_for\_distributed\_queries](../../../operations/settings/settings.md#settings-max_replica_delay_for_distributed_queries) et [fallback\_to\_stale\_replicas\_for\_distributed\_queries](../../../operations/settings/settings.md#settings-fallback_to_stale_replicas_for_distributed_queries). -Pour chaque `INSERT` requête, environ dix entrées sont ajoutées à ZooKeeper par le biais de plusieurs transactions. (Pour être plus précis, c’est pour chaque bloc de données inséré; une requête D’insertion contient un bloc ou un bloc par `max_insert_block_size = 1048576` rangée.) Cela conduit à des latences légèrement plus longues pour `INSERT` par rapport aux tables non répliquées. Mais si vous suivez les recommandations pour insérer des données dans des lots de pas plus d’un `INSERT` par seconde, cela ne crée aucun problème. L’ensemble du cluster clickhouse utilisé pour coordonner un cluster ZooKeeper a un total de plusieurs centaines `INSERTs` par seconde. Le débit sur les insertions de données (le nombre de lignes par seconde) est aussi élevé que pour les non-données répliquées. +Pour chaque `INSERT` requête, environ dix entrées sont ajoutées à ZooKeeper par le biais de plusieurs transactions. (Pour être plus précis, c'est pour chaque bloc de données inséré; une requête D'insertion contient un bloc ou un bloc par `max_insert_block_size = 1048576` rangée.) Cela conduit à des latences légèrement plus longues pour `INSERT` par rapport aux tables non répliquées. Mais si vous suivez les recommandations pour insérer des données dans des lots de pas plus d'un `INSERT` par seconde, cela ne crée aucun problème. L'ensemble du cluster clickhouse utilisé pour coordonner un cluster ZooKeeper a un total de plusieurs centaines `INSERTs` par seconde. Le débit sur les insertions de données (le nombre de lignes par seconde) est aussi élevé que pour les non-données répliquées. -Pour les clusters très volumineux, vous pouvez utiliser différents clusters ZooKeeper pour différents fragments. Cependant, cela ne s’est pas avéré nécessaire sur le Yandex.Cluster Metrica (environ 300 serveurs). +Pour les clusters très volumineux, vous pouvez utiliser différents clusters ZooKeeper pour différents fragments. Cependant, cela ne s'est pas avéré nécessaire sur le Yandex.Cluster Metrica (environ 300 serveurs). -La réplication est asynchrone et multi-maître. `INSERT` les requêtes (ainsi que `ALTER`) peuvent être envoyés à n’importe quel serveur disponible. Les données sont insérées sur le serveur où la requête est exécutée, puis il est copié sur les autres serveurs. Comme il est asynchrone, les données récemment insérées apparaissent sur les autres répliques avec une certaine latence. Si une partie des répliques ne sont pas disponibles, les données sont écrites lorsqu’elles sont disponibles. Si une réplique est disponible, la latence correspond au temps nécessaire pour transférer le bloc de données compressées sur le réseau. +La réplication est asynchrone et multi-maître. `INSERT` les requêtes (ainsi que `ALTER`) peuvent être envoyés à n'importe quel serveur disponible. Les données sont insérées sur le serveur où la requête est exécutée, puis il est copié sur les autres serveurs. Comme il est asynchrone, les données récemment insérées apparaissent sur les autres répliques avec une certaine latence. Si une partie des répliques ne sont pas disponibles, les données sont écrites lorsqu'elles sont disponibles. Si une réplique est disponible, la latence correspond au temps nécessaire pour transférer le bloc de données compressées sur le réseau. -Par défaut, une requête INSERT attend la confirmation de l’écriture des données à partir d’un seul réplica. Si les données ont été correctement écrit sur une seule réplique et le serveur avec cette réplique cesse d’exister, les données enregistrées seront perdues. Pour activer la confirmation des Écritures de données à partir de plusieurs réplicas, utilisez `insert_quorum` option. +Par défaut, une requête INSERT attend la confirmation de l'écriture des données à partir d'un seul réplica. Si les données ont été correctement écrit sur une seule réplique et le serveur avec cette réplique cesse d'exister, les données enregistrées seront perdues. Pour activer la confirmation des Écritures de données à partir de plusieurs réplicas, utilisez `insert_quorum` option. -Chaque bloc de données est écrit de manière atomique. La requête D’insertion est divisée en blocs jusqu’à `max_insert_block_size = 1048576` rangée. En d’autres termes, si l’ `INSERT` la requête a moins de 1048576 lignes, elle est faite de manière atomique. +Chaque bloc de données est écrit de manière atomique. La requête D'insertion est divisée en blocs jusqu'à `max_insert_block_size = 1048576` rangée. En d'autres termes, si l' `INSERT` la requête a moins de 1048576 lignes, elle est faite de manière atomique. -Les blocs de données sont dédupliquées. Pour plusieurs écritures du même bloc de données (blocs de données de même taille contenant les mêmes lignes dans le même ordre), le bloc n’est écrit qu’une seule fois. La raison en est en cas de défaillance du réseau lorsque l’application cliente ne sait pas si les données ont été écrites dans la base de données, de sorte que le `INSERT` requête peut simplement être répété. Peu importe à quelles insertions de réplica ont été envoyées avec des données identiques. `INSERTs` sont idempotents. Les paramètres de déduplication sont contrôlés par [merge\_tree](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-merge_tree) les paramètres du serveur. +Les blocs de données sont dédupliquées. Pour plusieurs écritures du même bloc de données (blocs de données de même taille contenant les mêmes lignes dans le même ordre), le bloc n'est écrit qu'une seule fois. La raison en est en cas de défaillance du réseau lorsque l'application cliente ne sait pas si les données ont été écrites dans la base de données, de sorte que le `INSERT` requête peut simplement être répété. Peu importe à quelles insertions de réplica ont été envoyées avec des données identiques. `INSERTs` sont idempotents. Les paramètres de déduplication sont contrôlés par [merge\_tree](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-merge_tree) les paramètres du serveur. -Pendant la réplication, seules les données source à insérer sont transférées sur le réseau. D’autres transformations de données (fusion) sont coordonnées et effectuées sur toutes les répliques de la même manière. Cela minimise l’utilisation du réseau, ce qui signifie que la réplication fonctionne bien lorsque les répliques résident dans différents centres de données. (Notez que la duplication de données dans différents centres de données est l’objectif principal de la réplication.) +Pendant la réplication, seules les données source à insérer sont transférées sur le réseau. D'autres transformations de données (fusion) sont coordonnées et effectuées sur toutes les répliques de la même manière. Cela minimise l'utilisation du réseau, ce qui signifie que la réplication fonctionne bien lorsque les répliques résident dans différents centres de données. (Notez que la duplication de données dans différents centres de données est l'objectif principal de la réplication.) -Vous pouvez avoir n’importe quel nombre de répliques des mêmes données. Yandex.Metrica utilise la double réplication en production. Chaque serveur utilise RAID-5 ou RAID-6, et RAID-10 dans certains cas. C’est une solution relativement fiable et pratique. +Vous pouvez avoir n'importe quel nombre de répliques des mêmes données. Yandex.Metrica utilise la double réplication en production. Chaque serveur utilise RAID-5 ou RAID-6, et RAID-10 dans certains cas. C'est une solution relativement fiable et pratique. Le système surveille la synchronicité des données sur les répliques et est capable de récupérer après une défaillance. Le basculement est automatique (pour les petites différences de données) ou semi-automatique (lorsque les données diffèrent trop, ce qui peut indiquer une erreur de configuration). @@ -117,7 +117,7 @@ CREATE TABLE table_name
-Comme le montre l’exemple, ces paramètres peuvent contenir des substitutions entre crochets. Les valeurs substituées sont tirées de la ‘macros’ section du fichier de configuration. Exemple: +Comme le montre l'exemple, ces paramètres peuvent contenir des substitutions entre crochets. Les valeurs substituées sont tirées de la ‘macros’ section du fichier de configuration. Exemple: ``` xml @@ -127,45 +127,45 @@ Comme le montre l’exemple, ces paramètres peuvent contenir des substitutions ``` -Le chemin d’accès à la table dans ZooKeeper doit être unique pour chaque table répliquée. Les Tables sur différents fragments doivent avoir des chemins différents. +Le chemin d'accès à la table dans ZooKeeper doit être unique pour chaque table répliquée. Les Tables sur différents fragments doivent avoir des chemins différents. Dans ce cas, le chemin se compose des parties suivantes: -`/clickhouse/tables/` est le préfixe commun. Nous vous recommandons d’utiliser exactement celui-ci. +`/clickhouse/tables/` est le préfixe commun. Nous vous recommandons d'utiliser exactement celui-ci. -`{layer}-{shard}` est l’identificateur de fragment. Dans cet exemple, il se compose de deux parties, depuis l’Yandex.Le cluster Metrica utilise le sharding à deux niveaux. Pour la plupart des tâches, vous ne pouvez laisser que la substitution {shard}, qui sera étendue à l’Identificateur de partition. +`{layer}-{shard}` est l'identificateur de fragment. Dans cet exemple, il se compose de deux parties, depuis l'Yandex.Le cluster Metrica utilise le sharding à deux niveaux. Pour la plupart des tâches, vous ne pouvez laisser que la substitution {shard}, qui sera étendue à l'Identificateur de partition. -`table_name` est le nom du nœud de la table dans ZooKeeper. C’est une bonne idée de le rendre identique au nom de la table. Il est défini explicitement, car contrairement au nom de la table, il ne change pas après une requête de renommage. +`table_name` est le nom du nœud de la table dans ZooKeeper. C'est une bonne idée de le rendre identique au nom de la table. Il est défini explicitement, car contrairement au nom de la table, il ne change pas après une requête de renommage. *HINT*: vous pouvez ajouter un nom de base de données devant `table_name` Aussi. E. g. `db_name.table_name` -Le nom du réplica identifie différentes réplicas de la même table. Vous pouvez utiliser le nom de serveur pour cela, comme dans l’exemple. Le nom doit seulement être unique dans chaque fragment. +Le nom du réplica identifie différentes réplicas de la même table. Vous pouvez utiliser le nom de serveur pour cela, comme dans l'exemple. Le nom doit seulement être unique dans chaque fragment. -Vous pouvez définir les paramètres explicitement au lieu d’utiliser des substitutions. Cela peut être pratique pour tester et configurer de petits clusters. Cependant, vous ne pouvez pas utiliser de requêtes DDL distribuées (`ON CLUSTER`) dans ce cas. +Vous pouvez définir les paramètres explicitement au lieu d'utiliser des substitutions. Cela peut être pratique pour tester et configurer de petits clusters. Cependant, vous ne pouvez pas utiliser de requêtes DDL distribuées (`ON CLUSTER`) dans ce cas. -Lorsque vous travaillez avec de grands clusters, nous vous recommandons d’utiliser des substitutions car elles réduisent la probabilité d’erreur. +Lorsque vous travaillez avec de grands clusters, nous vous recommandons d'utiliser des substitutions car elles réduisent la probabilité d'erreur. -Exécutez l’ `CREATE TABLE` requête sur chaque réplique. Cette requête crée une nouvelle table répliquée, ou ajoute une nouvelle réplique à un existant. +Exécutez l' `CREATE TABLE` requête sur chaque réplique. Cette requête crée une nouvelle table répliquée, ou ajoute une nouvelle réplique à un existant. -Si vous ajoutez une nouvelle réplique après que la table contient déjà des données sur d’autres répliques, les données seront copiées des autres répliques vers la nouvelle après l’exécution de la requête. En d’autres termes, la nouvelle réplique synchronise avec les autres. +Si vous ajoutez une nouvelle réplique après que la table contient déjà des données sur d'autres répliques, les données seront copiées des autres répliques vers la nouvelle après l'exécution de la requête. En d'autres termes, la nouvelle réplique synchronise avec les autres. Pour supprimer une réplique, exécutez `DROP TABLE`. However, only one replica is deleted – the one that resides on the server where you run the query. ## Récupération Après Des Échecs {#recovery-after-failures} -Si ZooKeeper n’est pas disponible au démarrage d’un serveur, les tables répliquées passent en mode Lecture seule. Le système tente périodiquement de se connecter à ZooKeeper. +Si ZooKeeper n'est pas disponible au démarrage d'un serveur, les tables répliquées passent en mode Lecture seule. Le système tente périodiquement de se connecter à ZooKeeper. -Si ZooKeeper est indisponible pendant un `INSERT`, ou une erreur se produit lors de l’interaction avec ZooKeeper, une exception est levée. +Si ZooKeeper est indisponible pendant un `INSERT`, ou une erreur se produit lors de l'interaction avec ZooKeeper, une exception est levée. -Après la connexion à ZooKeeper, le système vérifie si l’ensemble de données du système de fichiers local correspond à l’ensemble de données attendu (ZooKeeper stocke ces informations). S’il y a des incohérences mineures, le système les résout en synchronisant les données avec les répliques. +Après la connexion à ZooKeeper, le système vérifie si l'ensemble de données du système de fichiers local correspond à l'ensemble de données attendu (ZooKeeper stocke ces informations). S'il y a des incohérences mineures, le système les résout en synchronisant les données avec les répliques. Si le système détecte des parties de données brisées (avec la mauvaise taille des fichiers) ou des parties non reconnues (parties écrites dans le système de fichiers mais non enregistrées dans ZooKeeper), il les déplace vers le `detached` sous-répertoire (ils ne sont pas supprimés). Toutes les pièces manquantes sont copiées à partir des répliques. -Notez que ClickHouse n’effectue aucune action destructrice telle que la suppression automatique d’une grande quantité de données. +Notez que ClickHouse n'effectue aucune action destructrice telle que la suppression automatique d'une grande quantité de données. -Lorsque le serveur démarre (ou établit une nouvelle session avec ZooKeeper), il vérifie uniquement la quantité et la taille de tous les fichiers. Si les tailles de fichier correspondent mais que les octets ont été modifiés quelque part au milieu, cela n’est pas détecté immédiatement, mais uniquement lorsque vous tentez de lire les données `SELECT` requête. La requête lève une exception concernant une somme de contrôle ou une taille non correspondante d’un bloc compressé. Dans ce cas, des parties de données sont ajoutées à la file d’attente de vérification et copiées à partir des répliques si nécessaire. +Lorsque le serveur démarre (ou établit une nouvelle session avec ZooKeeper), il vérifie uniquement la quantité et la taille de tous les fichiers. Si les tailles de fichier correspondent mais que les octets ont été modifiés quelque part au milieu, cela n'est pas détecté immédiatement, mais uniquement lorsque vous tentez de lire les données `SELECT` requête. La requête lève une exception concernant une somme de contrôle ou une taille non correspondante d'un bloc compressé. Dans ce cas, des parties de données sont ajoutées à la file d'attente de vérification et copiées à partir des répliques si nécessaire. Si la série de données diffère trop de celle attendue, un mécanisme de sécurité est déclenché. Le serveur entre cela dans le journal et refuse de lancer. La raison en est que ce cas peut indiquer une erreur de configuration, par exemple si une réplique sur un fragment a été accidentellement configurée comme une réplique sur un fragment différent. Cependant, les seuils pour ce mécanisme sont fixés assez bas, et cette situation peut se produire pendant la récupération normale de la défaillance. Dans ce cas, les données sont restaurées semi-automatiquement - par “pushing a button”. -Pour démarrer la récupération, créez le nœud `/path_to_table/replica_name/flags/force_restore_data` dans ZooKeeper avec n’importe quel contenu, ou exécutez la commande pour restaurer toutes les tables répliquées: +Pour démarrer la récupération, créez le nœud `/path_to_table/replica_name/flags/force_restore_data` dans ZooKeeper avec n'importe quel contenu, ou exécutez la commande pour restaurer toutes les tables répliquées: ``` bash sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data @@ -175,43 +175,43 @@ Puis redémarrez le serveur. Au démarrage, le serveur supprime ces indicateurs ## Récupération Après La Perte De Données Complète {#recovery-after-complete-data-loss} -Si toutes les données et métadonnées ont disparu de l’un des serveurs, procédez comme suit pour la récupération: +Si toutes les données et métadonnées ont disparu de l'un des serveurs, procédez comme suit pour la récupération: -1. Installez ClickHouse sur le serveur. Définissez correctement les substitutions dans le fichier de configuration qui contient l’Identificateur de fragment et les répliques, si vous les utilisez. -2. Si vous avez des tables non compliquées qui doivent être dupliquées manuellement sur les serveurs, copiez leurs données à partir d’un réplica (dans le répertoire `/var/lib/clickhouse/data/db_name/table_name/`). -3. Copier les définitions de table situées dans `/var/lib/clickhouse/metadata/` à partir d’une réplique. Si un identificateur de fragment ou de réplica est défini explicitement dans les définitions de table, corrigez-le de manière à ce qu’il corresponde à ce réplica. (Alternativement, démarrez le serveur et faites tous les `ATTACH TABLE` les requêtes qui auraient dû être dans les .les fichiers sql dans `/var/lib/clickhouse/metadata/`.) -4. Pour démarrer la récupération, créez le nœud ZooKeeper `/path_to_table/replica_name/flags/force_restore_data` tout contenu, ou d’exécuter la commande pour restaurer toutes les tables répliquées: `sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data` +1. Installez ClickHouse sur le serveur. Définissez correctement les substitutions dans le fichier de configuration qui contient l'Identificateur de fragment et les répliques, si vous les utilisez. +2. Si vous avez des tables non compliquées qui doivent être dupliquées manuellement sur les serveurs, copiez leurs données à partir d'un réplica (dans le répertoire `/var/lib/clickhouse/data/db_name/table_name/`). +3. Copier les définitions de table situées dans `/var/lib/clickhouse/metadata/` à partir d'une réplique. Si un identificateur de fragment ou de réplica est défini explicitement dans les définitions de table, corrigez-le de manière à ce qu'il corresponde à ce réplica. (Alternativement, démarrez le serveur et faites tous les `ATTACH TABLE` les requêtes qui auraient dû être dans les .les fichiers sql dans `/var/lib/clickhouse/metadata/`.) +4. Pour démarrer la récupération, créez le nœud ZooKeeper `/path_to_table/replica_name/flags/force_restore_data` tout contenu, ou d'exécuter la commande pour restaurer toutes les tables répliquées: `sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data` -Ensuite, démarrez le serveur (redémarrez, s’il est déjà en cours d’exécution). Les données seront téléchargées à partir de répliques. +Ensuite, démarrez le serveur (redémarrez, s'il est déjà en cours d'exécution). Les données seront téléchargées à partir de répliques. Une autre option de récupération consiste à supprimer des informations sur la réplique perdue de ZooKeeper (`/path_to_table/replica_name`), puis créez à nouveau la réplique comme décrit dans “[Création de tables répliquées](#creating-replicated-tables)”. -Il n’y a aucune restriction sur la bande passante réseau pendant la récupération. Gardez cela à l’esprit si vous restaurez de nombreuses répliques à la fois. +Il n'y a aucune restriction sur la bande passante réseau pendant la récupération. Gardez cela à l'esprit si vous restaurez de nombreuses répliques à la fois. -## Conversion De Mergetree En Replicatedmergetree {#converting-from-mergetree-to-replicatedmergetree} +## Conversion de MergeTree en ReplicatedMergeTree {#converting-from-mergetree-to-replicatedmergetree} Nous utilisons le terme `MergeTree` pour consulter tous les moteurs de la `MergeTree family` le même que pour `ReplicatedMergeTree`. Si vous avez eu une `MergeTree` table qui a été répliquée manuellement, vous pouvez le convertir en une table répliquée. Vous devrez peut-être le faire si vous avez déjà recueilli une grande quantité de données dans un `MergeTree` table et maintenant vous voulez activer la réplication. -Si les données diffèrent sur différentes répliques, synchronisez-les d’abord ou supprimez-les sur toutes les répliques sauf une. +Si les données diffèrent sur différentes répliques, synchronisez-les d'abord ou supprimez-les sur toutes les répliques sauf une. -Renommez la table mergetree existante, puis créez un `ReplicatedMergeTree` table avec l’ancien nom. -Déplacez les données de l’ancienne table vers `detached` sous-répertoire à l’intérieur du répertoire avec les nouvelles données de la table (`/var/lib/clickhouse/data/db_name/table_name/`). -Ensuite, exécutez `ALTER TABLE ATTACH PARTITION` sur l’une des répliques d’ajouter ces données à des parties de l’ensemble de travail. +Renommez la table mergetree existante, puis créez un `ReplicatedMergeTree` table avec l'ancien nom. +Déplacez les données de l'ancienne table vers `detached` sous-répertoire à l'intérieur du répertoire avec les nouvelles données de la table (`/var/lib/clickhouse/data/db_name/table_name/`). +Ensuite, exécutez `ALTER TABLE ATTACH PARTITION` sur l'une des répliques d'ajouter ces données à des parties de l'ensemble de travail. -## Conversion De Replicatedmergetree En Mergetree {#converting-from-replicatedmergetree-to-mergetree} +## Conversion de ReplicatedMergeTree en MergeTree {#converting-from-replicatedmergetree-to-mergetree} Créez une table MergeTree avec un nom différent. Déplacez toutes les données du répertoire avec le `ReplicatedMergeTree` données de la table dans le répertoire de données de la nouvelle table. Ensuite, supprimer le `ReplicatedMergeTree` table et redémarrez le serveur. -Si vous voulez vous débarrasser d’un `ReplicatedMergeTree` table sans lancer le serveur: +Si vous voulez vous débarrasser d'un `ReplicatedMergeTree` table sans lancer le serveur: - Supprimer la `.sql` fichier dans le répertoire de métadonnées (`/var/lib/clickhouse/metadata/`). - Supprimer le chemin correspondant dans ZooKeeper (`/path_to_table/replica_name`). Après cela, vous pouvez lancer le serveur, créer un `MergeTree` tableau, déplacer les données de son répertoire, puis redémarrez le serveur. -## Récupération Lorsque Les métadonnées Du Cluster Zookeeper Sont Perdues Ou endommagées {#recovery-when-metadata-in-the-zookeeper-cluster-is-lost-or-damaged} +## Récupération lorsque les métadonnées du Cluster Zookeeper sont perdues ou endommagées {#recovery-when-metadata-in-the-zookeeper-cluster-is-lost-or-damaged} Si les données de ZooKeeper ont été perdues ou endommagées, vous pouvez les enregistrer en les déplaçant dans une table non compliquée comme décrit ci-dessus. diff --git a/docs/fr/engines/table-engines/mergetree-family/summingmergetree.md b/docs/fr/engines/table-engines/mergetree-family/summingmergetree.md index 6be54f6dbef..f4f2b94a93f 100644 --- a/docs/fr/engines/table-engines/mergetree-family/summingmergetree.md +++ b/docs/fr/engines/table-engines/mergetree-family/summingmergetree.md @@ -1,17 +1,17 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 34 toc_title: SummingMergeTree --- -# Summingmergetree {#summingmergetree} +# SummingMergeTree {#summingmergetree} -Le moteur hérite de [MergeTree](mergetree.md#table_engines-mergetree). La différence est que lors de la fusion de parties de données pour `SummingMergeTree` tables ClickHouse remplace toutes les lignes avec la même clé primaire (ou, plus précisément, avec la même [clé de tri](mergetree.md)) avec une ligne qui contient des valeurs résumées pour les colonnes avec le type de données numériques. Si la clé de tri est composée de telle sorte qu’une seule valeur de clé correspond à un grand nombre de lignes, cela réduit considérablement le volume de stockage et accélère la sélection des données. +Le moteur hérite de [MergeTree](mergetree.md#table_engines-mergetree). La différence est que lors de la fusion de parties de données pour `SummingMergeTree` tables ClickHouse remplace toutes les lignes avec la même clé primaire (ou, plus précisément, avec la même [clé de tri](mergetree.md)) avec une ligne qui contient des valeurs résumées pour les colonnes avec le type de données numériques. Si la clé de tri est composée de telle sorte qu'une seule valeur de clé correspond à un grand nombre de lignes, cela réduit considérablement le volume de stockage et accélère la sélection des données. -Nous vous recommandons d’utiliser le moteur avec `MergeTree`. Stocker des données complètes dans `MergeTree` table, et l’utilisation `SummingMergeTree` pour le stockage de données agrégées, par exemple, lors de la préparation de rapports. Une telle approche vous empêchera de perdre des données précieuses en raison d’une clé primaire mal composée. +Nous vous recommandons d'utiliser le moteur avec `MergeTree`. Stocker des données complètes dans `MergeTree` table, et l'utilisation `SummingMergeTree` pour le stockage de données agrégées, par exemple, lors de la préparation de rapports. Une telle approche vous empêchera de perdre des données précieuses en raison d'une clé primaire mal composée. -## Création d’une Table {#creating-a-table} +## Création d'une Table {#creating-a-table} ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] @@ -31,20 +31,20 @@ Pour une description des paramètres de requête, voir [demande de description]( **Paramètres de SummingMergeTree** - `columns` - un n-uplet avec les noms de colonnes où les valeurs seront résumées. Paramètre facultatif. - Les colonnes doivent être d’un type numérique et ne doit pas être dans la clé primaire. + Les colonnes doivent être d'un type numérique et ne doit pas être dans la clé primaire. Si `columns` non spécifié, ClickHouse résume les valeurs dans toutes les colonnes avec un type de données numérique qui ne sont pas dans la clé primaire. **Les clauses de requête** -Lors de la création d’un `SummingMergeTree` la table de la même [clause](mergetree.md) sont nécessaires, comme lors de la création d’un `MergeTree` table. +Lors de la création d'un `SummingMergeTree` la table de la même [clause](mergetree.md) sont nécessaires, comme lors de la création d'un `MergeTree` table.
Méthode obsolète pour créer une Table !!! attention "Attention" - N’utilisez pas cette méthode dans les nouveaux projets et, si possible, remplacez les anciens projets par la méthode décrite ci-dessus. + N'utilisez pas cette méthode dans les nouveaux projets et, si possible, remplacez les anciens projets par la méthode décrite ci-dessus. ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] @@ -61,7 +61,7 @@ Tous les paramètres excepté `columns` ont la même signification que dans `Mer
-## Exemple D’Utilisation {#usage-example} +## Exemple D'Utilisation {#usage-example} Considérons le tableau suivant: @@ -81,7 +81,7 @@ Insérer des données: INSERT INTO summtt Values(1,1),(1,2),(2,1) ``` -ClickHouse peut résumer toutes les lignes pas complètement ([voir ci-dessous](#data-processing)), nous utilisons donc une fonction d’agrégation `sum` et `GROUP BY` la clause dans la requête. +ClickHouse peut résumer toutes les lignes pas complètement ([voir ci-dessous](#data-processing)), nous utilisons donc une fonction d'agrégation `sum` et `GROUP BY` la clause dans la requête. ``` sql SELECT key, sum(value) FROM summtt GROUP BY key @@ -96,32 +96,32 @@ SELECT key, sum(value) FROM summtt GROUP BY key ## Le Traitement Des Données {#data-processing} -Lorsque les données sont insérées dans une table, elles sont enregistrées telles quelles. Clickhouse fusionne périodiquement les parties de données insérées et c’est à ce moment que les lignes avec la même clé primaire sont additionnées et remplacées par une pour chaque partie de données résultante. +Lorsque les données sont insérées dans une table, elles sont enregistrées telles quelles. Clickhouse fusionne périodiquement les parties de données insérées et c'est à ce moment que les lignes avec la même clé primaire sont additionnées et remplacées par une pour chaque partie de données résultante. -ClickHouse can merge the data parts so that different resulting parts of data cat consist rows with the same primary key, i.e. the summation will be incomplete. Therefore (`SELECT`) une fonction d’agrégation [somme()](../../../sql-reference/aggregate-functions/reference.md#agg_function-sum) et `GROUP BY` la clause doit être utilisé dans une requête comme décrit dans l’exemple ci-dessus. +ClickHouse can merge the data parts so that different resulting parts of data cat consist rows with the same primary key, i.e. the summation will be incomplete. Therefore (`SELECT`) une fonction d'agrégation [somme()](../../../sql-reference/aggregate-functions/reference.md#agg_function-sum) et `GROUP BY` la clause doit être utilisé dans une requête comme décrit dans l'exemple ci-dessus. -### Règles Communes Pour La Sommation {#common-rules-for-summation} +### Règles communes pour la sommation {#common-rules-for-summation} -Les valeurs dans les colonnes avec le type de données numériques sont résumées. L’ensemble des colonnes est défini par le paramètre `columns`. +Les valeurs dans les colonnes avec le type de données numériques sont résumées. L'ensemble des colonnes est défini par le paramètre `columns`. Si les valeurs étaient 0 dans toutes les colonnes pour la sommation, la ligne est supprimée. -Si la colonne n’est pas dans la clé primaire et n’est pas résumée, une valeur arbitraire est sélectionnée parmi celles existantes. +Si la colonne n'est pas dans la clé primaire et n'est pas résumée, une valeur arbitraire est sélectionnée parmi celles existantes. Les valeurs ne sont pas résumés des colonnes de la clé primaire. -### La Somme Dans Les Colonnes Aggregatefunction {#the-summation-in-the-aggregatefunction-columns} +### La somme dans les colonnes Aggregatefunction {#the-summation-in-the-aggregatefunction-columns} -Pour les colonnes de [Type AggregateFunction](../../../sql-reference/data-types/aggregatefunction.md) ClickHouse se comporte comme [AggregatingMergeTree](aggregatingmergetree.md) moteur d’agrégation selon la fonction. +Pour les colonnes de [Type AggregateFunction](../../../sql-reference/data-types/aggregatefunction.md) ClickHouse se comporte comme [AggregatingMergeTree](aggregatingmergetree.md) moteur d'agrégation selon la fonction. ### Structures Imbriquées {#nested-structures} -Table peut avoir des structures de données imbriquées qui sont traitées d’une manière spéciale. +Table peut avoir des structures de données imbriquées qui sont traitées d'une manière spéciale. -Si le nom d’une table imbriquée se termine avec `Map` et il contient au moins deux colonnes qui répondent aux critères suivants: +Si le nom d'une table imbriquée se termine avec `Map` et il contient au moins deux colonnes qui répondent aux critères suivants: -- la première colonne est numérique `(*Int*, Date, DateTime)` ou une chaîne de caractères `(String, FixedString)`, nous allons l’appeler `key`, -- les autres colonnes sont arithmétique `(*Int*, Float32/64)`, nous allons l’appeler `(values...)`, +- la première colonne est numérique `(*Int*, Date, DateTime)` ou une chaîne de caractères `(String, FixedString)`, nous allons l'appeler `key`, +- les autres colonnes sont arithmétique `(*Int*, Float32/64)`, nous allons l'appeler `(values...)`, ensuite, cette table imbriquée est interprétée comme un mappage de `key => (values...)` et lors de la fusion de ses lignes, les éléments de deux ensembles de données sont regroupées par `key` avec une sommation du correspondant `(values...)`. @@ -134,8 +134,8 @@ Exemple: [(1, 100), (2, 150)] + [(1, -100)] -> [(2, 150)] ``` -Lorsque vous demandez des données, utilisez [sumMap (clé, valeur)](../../../sql-reference/aggregate-functions/reference.md) fonction pour l’agrégation de `Map`. +Lorsque vous demandez des données, utilisez [sumMap (clé, valeur)](../../../sql-reference/aggregate-functions/reference.md) fonction pour l'agrégation de `Map`. -Pour la structure de données imbriquée, vous n’avez pas besoin de spécifier ses colonnes dans le tuple de colonnes pour la sommation. +Pour la structure de données imbriquée, vous n'avez pas besoin de spécifier ses colonnes dans le tuple de colonnes pour la sommation. [Article Original](https://clickhouse.tech/docs/en/operations/table_engines/summingmergetree/) diff --git a/docs/fr/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md b/docs/fr/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md index a810069fb21..9be9fb5e76e 100644 --- a/docs/fr/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md +++ b/docs/fr/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md @@ -1,22 +1,22 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 37 toc_title: VersionedCollapsingMergeTree --- -# Versionedcollapsingmergetree {#versionedcollapsingmergetree} +# VersionedCollapsingMergeTree {#versionedcollapsingmergetree} Ce moteur: -- Permet l’écriture rapide des États d’objet qui changent continuellement. -- Supprime les anciens États d’objets en arrière-plan. Cela réduit considérablement le volume de stockage. +- Permet l'écriture rapide des États d'objet qui changent continuellement. +- Supprime les anciens États d'objets en arrière-plan. Cela réduit considérablement le volume de stockage. Voir la section [Effondrer](#table_engines_versionedcollapsingmergetree) pour plus de détails. -Le moteur hérite de [MergeTree](mergetree.md#table_engines-mergetree) et ajoute la logique de réduction des lignes à l’algorithme de fusion des parties de données. `VersionedCollapsingMergeTree` sert le même but que [CollapsingMergeTree](collapsingmergetree.md) mais utilise un autre effondrement algorithme qui permet d’insérer les données dans n’importe quel ordre avec plusieurs threads. En particulier, l’ `Version` la colonne aide à réduire correctement les lignes même si elles sont insérées dans le mauvais ordre. Contrairement, `CollapsingMergeTree` permet uniquement une insertion strictement consécutive. +Le moteur hérite de [MergeTree](mergetree.md#table_engines-mergetree) et ajoute la logique de réduction des lignes à l'algorithme de fusion des parties de données. `VersionedCollapsingMergeTree` sert le même but que [CollapsingMergeTree](collapsingmergetree.md) mais utilise un autre effondrement algorithme qui permet d'insérer les données dans n'importe quel ordre avec plusieurs threads. En particulier, l' `Version` la colonne aide à réduire correctement les lignes même si elles sont insérées dans le mauvais ordre. Contrairement, `CollapsingMergeTree` permet uniquement une insertion strictement consécutive. -## Création d’une Table {#creating-a-table} +## Création d'une Table {#creating-a-table} ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] @@ -49,14 +49,14 @@ VersionedCollapsingMergeTree(sign, version) **Les Clauses De Requête** -Lors de la création d’un `VersionedCollapsingMergeTree` de table, de la même [clause](mergetree.md) sont requis lors de la création d’un `MergeTree` table. +Lors de la création d'un `VersionedCollapsingMergeTree` de table, de la même [clause](mergetree.md) sont requis lors de la création d'un `MergeTree` table.
Méthode obsolète pour créer une Table !!! attention "Attention" - N’utilisez pas cette méthode dans les nouveaux projets. Si possible, passer les anciens projets à la méthode décrite ci-dessus. + N'utilisez pas cette méthode dans les nouveaux projets. Si possible, passer les anciens projets à la méthode décrite ci-dessus. ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] @@ -64,10 +64,10 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], ... -) ENGINE [=] VersionedCollapsingMergeTree(date-column [, sampling_expression], (primary, key), index_granularity, sign, version) +) ENGINE [=] VersionedCollapsingMergeTree(date-column [, samp#table_engines_versionedcollapsingmergetreeling_expression], (primary, key), index_granularity, sign, version) ``` -Tous les paramètres, à l’exception `sign` et `version` ont la même signification que dans `MergeTree`. +Tous les paramètres, à l'exception `sign` et `version` ont la même signification que dans `MergeTree`. - `sign` — Name of the column with the type of row: `1` est un “state” rangée, `-1` est un “cancel” rangée. @@ -83,11 +83,11 @@ Tous les paramètres, à l’exception `sign` et `version` ont la même signific ### Données {#data} -Considérez une situation où vous devez enregistrer des données en constante évolution pour un objet. Il est raisonnable d’avoir une ligne pour un objet et de mettre à jour la ligne chaque fois qu’il y a des modifications. Cependant, l’opération de mise à jour est coûteuse et lente pour un SGBD car elle nécessite la réécriture des données dans le stockage. La mise à jour n’est pas acceptable si vous devez écrire des données rapidement, mais vous pouvez écrire les modifications sur un objet de manière séquentielle comme suit. +Considérez une situation où vous devez enregistrer des données en constante évolution pour un objet. Il est raisonnable d'avoir une ligne pour un objet et de mettre à jour la ligne chaque fois qu'il y a des modifications. Cependant, l'opération de mise à jour est coûteuse et lente pour un SGBD car elle nécessite la réécriture des données dans le stockage. La mise à jour n'est pas acceptable si vous devez écrire des données rapidement, mais vous pouvez écrire les modifications sur un objet de manière séquentielle comme suit. -L’utilisation de la `Sign` colonne lors de l’écriture de la ligne. Si `Sign = 1` cela signifie que la ligne est un état d’un objet (appelons-la “state” rangée). Si `Sign = -1` il indique l’annulation de l’état d’un objet avec les mêmes attributs (appelons-la “cancel” rangée). Également utiliser l’ `Version` colonne, qui doit identifier chaque état d’un objet avec un numéro distinct. +L'utilisation de la `Sign` colonne lors de l'écriture de la ligne. Si `Sign = 1` cela signifie que la ligne est un état d'un objet (appelons-la “state” rangée). Si `Sign = -1` il indique l'annulation de l'état d'un objet avec les mêmes attributs (appelons-la “cancel” rangée). Également utiliser l' `Version` colonne, qui doit identifier chaque état d'un objet avec un numéro distinct. -Par exemple, nous voulons calculer le nombre de pages visitées sur le site et combien de temps ils étaient là. À un moment donné nous écrivons la ligne suivante avec l’état de l’activité de l’utilisateur: +Par exemple, nous voulons calculer le nombre de pages visitées sur le site et combien de temps ils étaient là. À un moment donné nous écrivons la ligne suivante avec l'état de l'activité de l'utilisateur: ``` text ┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┬─Version─┐ @@ -95,7 +95,7 @@ Par exemple, nous voulons calculer le nombre de pages visitées sur le site et c └─────────────────────┴───────────┴──────────┴──────┴─────────┘ ``` -À un moment donné, nous enregistrons le changement d’activité de l’utilisateur et l’écrivons avec les deux lignes suivantes. +À un moment donné, nous enregistrons le changement d'activité de l'utilisateur et l'écrivons avec les deux lignes suivantes. ``` text ┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┬─Version─┐ @@ -104,11 +104,11 @@ Par exemple, nous voulons calculer le nombre de pages visitées sur le site et c └─────────────────────┴───────────┴──────────┴──────┴─────────┘ ``` -La première ligne annule le précédent état de l’objet (utilisateur). Il doit copier tous les champs de l’état annulé sauf `Sign`. +La première ligne annule le précédent état de l'objet (utilisateur). Il doit copier tous les champs de l'état annulé sauf `Sign`. -La deuxième ligne contient l’état actuel. +La deuxième ligne contient l'état actuel. -Parce que nous avons besoin seulement le dernier état de l’activité de l’utilisateur, les lignes +Parce que nous avons besoin seulement le dernier état de l'activité de l'utilisateur, les lignes ``` text ┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┬─Version─┐ @@ -117,35 +117,35 @@ Parce que nous avons besoin seulement le dernier état de l’activité de l’u └─────────────────────┴───────────┴──────────┴──────┴─────────┘ ``` -peut être supprimé, réduisant l’état invalide (ancien) de l’objet. `VersionedCollapsingMergeTree` fait cela lors de la fusion des parties de données. +peut être supprimé, réduisant l'état invalide (ancien) de l'objet. `VersionedCollapsingMergeTree` fait cela lors de la fusion des parties de données. Pour savoir pourquoi nous avons besoin de deux lignes pour chaque changement, voir [Algorithme](#table_engines-versionedcollapsingmergetree-algorithm). -**Notes sur l’Utilisation de la** +**Notes sur l'Utilisation de la** -1. Le programme qui écrit les données devraient se souvenir de l’état d’un objet afin de l’annuler. Le “cancel” chaîne doit être une copie de la “state” chaîne avec le contraire `Sign`. Cela augmente la taille initiale de stockage, mais permet d’écrire les données rapidement. -2. Les tableaux de plus en plus longs dans les colonnes réduisent l’efficacité du moteur en raison de la charge d’écriture. Plus les données sont simples, meilleure est l’efficacité. -3. `SELECT` les résultats dépendent fortement de la cohérence de l’histoire de l’objet change. Être précis lors de la préparation des données pour l’insertion. Vous pouvez obtenir des résultats imprévisibles avec des données incohérentes, telles que des valeurs négatives pour des métriques non négatives telles que la profondeur de session. +1. Le programme qui écrit les données devraient se souvenir de l'état d'un objet afin de l'annuler. Le “cancel” chaîne doit être une copie de la “state” chaîne avec le contraire `Sign`. Cela augmente la taille initiale de stockage, mais permet d'écrire les données rapidement. +2. Les tableaux de plus en plus longs dans les colonnes réduisent l'efficacité du moteur en raison de la charge d'écriture. Plus les données sont simples, meilleure est l'efficacité. +3. `SELECT` les résultats dépendent fortement de la cohérence de l'histoire de l'objet change. Être précis lors de la préparation des données pour l'insertion. Vous pouvez obtenir des résultats imprévisibles avec des données incohérentes, telles que des valeurs négatives pour des métriques non négatives telles que la profondeur de session. ### Algorithme {#table_engines-versionedcollapsingmergetree-algorithm} -Lorsque ClickHouse fusionne des parties de données, il supprime chaque paire de lignes ayant la même clé primaire et la même version et différentes `Sign`. L’ordre des lignes n’a pas d’importance. +Lorsque ClickHouse fusionne des parties de données, il supprime chaque paire de lignes ayant la même clé primaire et la même version et différentes `Sign`. L'ordre des lignes n'a pas d'importance. -Lorsque ClickHouse insère des données, il ordonne les lignes par la clé primaire. Si l’ `Version` la colonne n’est pas dans la clé primaire, ClickHouse ajoute à la clé primaire implicitement que le dernier champ et l’utilise pour la commande. +Lorsque ClickHouse insère des données, il ordonne les lignes par la clé primaire. Si l' `Version` la colonne n'est pas dans la clé primaire, ClickHouse ajoute à la clé primaire implicitement que le dernier champ et l'utilise pour la commande. ## La Sélection De Données {#selecting-data} -ClickHouse ne garantit pas que toutes les lignes avec la même clé primaire sera dans la même partie des données ou même sur le même serveur physique. Cela est vrai à la fois pour l’écriture des données et pour la fusion ultérieure des parties de données. En outre, les processus ClickHouse `SELECT` requêtes avec plusieurs threads, et il ne peut pas prédire l’ordre des lignes dans le résultat. Cela signifie que le regroupement est nécessaire s’il est nécessaire pour obtenir complètement “collapsed” données à partir d’un `VersionedCollapsingMergeTree` table. +ClickHouse ne garantit pas que toutes les lignes avec la même clé primaire sera dans la même partie des données ou même sur le même serveur physique. Cela est vrai à la fois pour l'écriture des données et pour la fusion ultérieure des parties de données. En outre, les processus ClickHouse `SELECT` requêtes avec plusieurs threads, et il ne peut pas prédire l'ordre des lignes dans le résultat. Cela signifie que le regroupement est nécessaire s'il est nécessaire pour obtenir complètement “collapsed” données à partir d'un `VersionedCollapsingMergeTree` table. -Pour finaliser la réduction, écrivez une requête avec un `GROUP BY` fonctions de clause et d’agrégation qui tiennent compte du signe. Par exemple, pour calculer la quantité, l’utilisation `sum(Sign)` plutôt `count()`. Pour calculer la somme de quelque chose, utilisez `sum(Sign * x)` plutôt `sum(x)` et d’ajouter `HAVING sum(Sign) > 0`. +Pour finaliser la réduction, écrivez une requête avec un `GROUP BY` fonctions de clause et d'agrégation qui tiennent compte du signe. Par exemple, pour calculer la quantité, l'utilisation `sum(Sign)` plutôt `count()`. Pour calculer la somme de quelque chose, utilisez `sum(Sign * x)` plutôt `sum(x)` et d'ajouter `HAVING sum(Sign) > 0`. -Aggregate `count`, `sum` et `avg` peut être calculée de cette manière. Aggregate `uniq` peut être calculé si un objet a au moins un non-état effondré. Aggregate `min` et `max` ne peut pas être calculé, car `VersionedCollapsingMergeTree` ne sauvegarde pas l’historique des valeurs des États réduits. +Aggregate `count`, `sum` et `avg` peut être calculée de cette manière. Aggregate `uniq` peut être calculé si un objet a au moins un non-état effondré. Aggregate `min` et `max` ne peut pas être calculé, car `VersionedCollapsingMergeTree` ne sauvegarde pas l'historique des valeurs des États réduits. -Si vous avez besoin d’extraire les données avec “collapsing” mais sans agrégation (par exemple, pour vérifier si des lignes sont présentes dont les valeurs les plus récentes correspondent à certaines conditions), vous pouvez utiliser `FINAL` le modificateur du `FROM` clause. Cette approche est inefficace et ne devrait pas être utilisée avec de grandes tables. +Si vous avez besoin d'extraire les données avec “collapsing” mais sans agrégation (par exemple, pour vérifier si des lignes sont présentes dont les valeurs les plus récentes correspondent à certaines conditions), vous pouvez utiliser `FINAL` le modificateur du `FROM` clause. Cette approche est inefficace et ne devrait pas être utilisée avec de grandes tables. -## Exemple D’Utilisation {#example-of-use} +## Exemple D'utilisation {#example-of-use} -Les données de l’exemple: +Les données de l'exemple: ``` text ┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┬─Version─┐ @@ -180,9 +180,9 @@ INSERT INTO UAct VALUES (4324182021466249494, 5, 146, 1, 1) INSERT INTO UAct VALUES (4324182021466249494, 5, 146, -1, 1),(4324182021466249494, 6, 185, 1, 2) ``` -Nous utilisons deux `INSERT` requêtes pour créer deux parties de données différentes. Si nous insérons les données avec une seule requête, ClickHouse crée une partie de données et n’effectuera jamais de fusion. +Nous utilisons deux `INSERT` requêtes pour créer deux parties de données différentes. Si nous insérons les données avec une seule requête, ClickHouse crée une partie de données et n'effectuera jamais de fusion. -L’obtention de données: +L'obtention de données: ``` sql SELECT * FROM UAct @@ -200,9 +200,9 @@ SELECT * FROM UAct Que voyons-nous ici et où sont les parties effondrées? Nous avons créé deux parties de données en utilisant deux `INSERT` requête. Le `SELECT` la requête a été effectuée dans deux threads, et le résultat est un ordre aléatoire de lignes. -L’effondrement n’a pas eu lieu car les parties de données n’ont pas encore été fusionnées. ClickHouse fusionne des parties de données à un moment inconnu que nous ne pouvons pas prédire. +L'effondrement n'a pas eu lieu car les parties de données n'ont pas encore été fusionnées. ClickHouse fusionne des parties de données à un moment inconnu que nous ne pouvons pas prédire. -C’est pourquoi nous avons besoin de l’agrégation: +C'est pourquoi nous avons besoin de l'agrégation: ``` sql SELECT @@ -221,7 +221,7 @@ HAVING sum(Sign) > 0 └─────────────────────┴───────────┴──────────┴─────────┘ ``` -Si nous n’avons pas besoin d’agrégation et que nous voulons forcer l’effondrement, nous pouvons utiliser le `FINAL` le modificateur du `FROM` clause. +Si nous n'avons pas besoin d'agrégation et que nous voulons forcer l'effondrement, nous pouvons utiliser le `FINAL` le modificateur du `FROM` clause. ``` sql SELECT * FROM UAct FINAL @@ -233,6 +233,6 @@ SELECT * FROM UAct FINAL └─────────────────────┴───────────┴──────────┴──────┴─────────┘ ``` -C’est un moyen très inefficace de sélectionner des données. Ne l’utilisez pas pour les grandes tables. +C'est un moyen très inefficace de sélectionner des données. Ne l'utilisez pas pour les grandes tables. [Article Original](https://clickhouse.tech/docs/en/operations/table_engines/versionedcollapsingmergetree/) diff --git a/docs/fr/engines/table-engines/special/buffer.md b/docs/fr/engines/table-engines/special/buffer.md index 085e82a95e4..af26b32177b 100644 --- a/docs/fr/engines/table-engines/special/buffer.md +++ b/docs/fr/engines/table-engines/special/buffer.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 45 toc_title: Tampon --- # Tampon {#buffer} -Met en mémoire tampon les données à écrire dans la RAM, les vidant périodiquement dans une autre table. Pendant l’opération de lecture, les données sont lues à partir de la mémoire tampon, et l’autre simultanément. +Met en mémoire tampon les données à écrire dans la RAM, les vidant périodiquement dans une autre table. Pendant l'opération de lecture, les données sont lues à partir de la mémoire tampon, et l'autre simultanément. ``` sql Buffer(database, table, num_layers, min_time, max_time, min_rows, max_rows, min_bytes, max_bytes) @@ -26,7 +26,7 @@ Les données sont vidées du tampon et écrites dans la table de destination si - `min_rows`, `max_rows` – Condition for the number of rows in the buffer. - `min_bytes`, `max_bytes` – Condition for the number of bytes in the buffer. -Pendant l’opération d’écriture, les données sont insérées dans un `num_layers` nombre aléatoire de tampons. Ou, si la partie de données à insérer est suffisamment grande (supérieure à `max_rows` ou `max_bytes`), il est écrit directement dans la table de destination, en omettant le tampon. +Pendant l'opération d'écriture, les données sont insérées dans un `num_layers` nombre aléatoire de tampons. Ou, si la partie de données à insérer est suffisamment grande (supérieure à `max_rows` ou `max_bytes`), il est écrit directement dans la table de destination, en omettant le tampon. Les conditions de purger les données sont calculées séparément pour chacun des `num_layers` tampon. Par exemple, si `num_layers = 16` et `max_bytes = 100000000`, la consommation maximale de RAM est de 1,6 Go. @@ -36,36 +36,36 @@ Exemple: CREATE TABLE merge.hits_buffer AS merge.hits ENGINE = Buffer(merge, hits, 16, 10, 100, 10000, 1000000, 10000000, 100000000) ``` -La création d’un ‘merge.hits\_buffer’ table avec la même structure que ‘merge.hits’ et en utilisant le moteur tampon. Lors de l’écriture dans cette table, les données sont mises en mémoire tampon dans la RAM ‘merge.hits’ table. 16 tampons sont créés. Les données dans chacun d’entre eux est rincé si 100 secondes sont écoulées, ou un million de lignes ont été écrites, ou 100 MO de données ont été écrits; ou si, simultanément, 10 secondes et 10 000 lignes et 10 MO de données ont été écrites. Par exemple, si une ligne a été écrite, après 100 secondes, il sera vidé, n’importe quoi. Mais si plusieurs lignes ont été écrites, les données seront vidées plus tôt. +La création d'un ‘merge.hits\_buffer’ table avec la même structure que ‘merge.hits’ et en utilisant le moteur tampon. Lors de l'écriture dans cette table, les données sont mises en mémoire tampon dans la RAM ‘merge.hits’ table. 16 tampons sont créés. Les données dans chacun d'entre eux est rincé si 100 secondes sont écoulées, ou un million de lignes ont été écrites, ou 100 MO de données ont été écrits; ou si, simultanément, 10 secondes et 10 000 lignes et 10 MO de données ont été écrites. Par exemple, si une ligne a été écrite, après 100 secondes, il sera vidé, n'importe quoi. Mais si plusieurs lignes ont été écrites, les données seront vidées plus tôt. Lorsque le serveur est arrêté, avec DROP TABLE ou DETACH TABLE, les données du tampon sont également vidées vers la table de destination. -Vous pouvez définir des chaînes vides entre guillemets simples pour le nom de la base de données et de la table. Cela indique l’absence d’une table de destination. Dans ce cas, lorsque les conditions de vidage des données sont atteintes, le tampon est simplement effacé. Cela peut être utile pour garder une fenêtre de données dans la mémoire. +Vous pouvez définir des chaînes vides entre guillemets simples pour le nom de la base de données et de la table. Cela indique l'absence d'une table de destination. Dans ce cas, lorsque les conditions de vidage des données sont atteintes, le tampon est simplement effacé. Cela peut être utile pour garder une fenêtre de données dans la mémoire. -Lors de la lecture à partir d’une table tampon, les données sont traitées à la fois à partir du tampon et de la table de destination (s’il y en a une). -Notez que les tables de tampon ne prennent pas en charge un index. En d’autres termes, les données dans le tampon sont entièrement analysées, ce qui peut être lent pour les grands tampons. (Pour les données dans une table subordonnée, l’index qu’il prend en charge sera utilisé.) +Lors de la lecture à partir d'une table tampon, les données sont traitées à la fois à partir du tampon et de la table de destination (s'il y en a une). +Notez que les tables de tampon ne prennent pas en charge un index. En d'autres termes, les données dans le tampon sont entièrement analysées, ce qui peut être lent pour les grands tampons. (Pour les données dans une table subordonnée, l'index qu'il prend en charge sera utilisé.) -Si l’ensemble de colonnes de la table tampon ne correspond pas à l’ensemble de colonnes d’une table subordonnée, un sous-ensemble de colonnes existant dans les deux tables est inséré. +Si l'ensemble de colonnes de la table tampon ne correspond pas à l'ensemble de colonnes d'une table subordonnée, un sous-ensemble de colonnes existant dans les deux tables est inséré. -Si les types ne correspondent pas à l’une des colonnes de la table tampon et à une table subordonnée, un message d’erreur est entré dans le journal du serveur et le tampon est effacé. -La même chose se produit si la table subordonnée n’existe pas lorsque le tampon est vidé. +Si les types ne correspondent pas à l'une des colonnes de la table tampon et à une table subordonnée, un message d'erreur est entré dans le journal du serveur et le tampon est effacé. +La même chose se produit si la table subordonnée n'existe pas lorsque le tampon est vidé. -Si vous devez exécuter ALTER pour une table subordonnée et la table tampon, nous vous recommandons de supprimer d’abord la table tampon, d’exécuter ALTER pour la table subordonnée, puis de créer à nouveau la table tampon. +Si vous devez exécuter ALTER pour une table subordonnée et la table tampon, nous vous recommandons de supprimer d'abord la table tampon, d'exécuter ALTER pour la table subordonnée, puis de créer à nouveau la table tampon. Si le serveur est redémarré anormalement, les données dans le tampon sont perdues. -FINAL et SAMPLE ne fonctionnent pas correctement pour les tables tampon. Ces conditions sont transmises à la table de destination, mais ne sont pas utilisées pour traiter les données dans le tampon. Si ces fonctionnalités sont nécessaires, nous vous recommandons d’utiliser uniquement la table tampon pour l’écriture, lors de la lecture à partir de la table de destination. +FINAL et SAMPLE ne fonctionnent pas correctement pour les tables tampon. Ces conditions sont transmises à la table de destination, mais ne sont pas utilisées pour traiter les données dans le tampon. Si ces fonctionnalités sont nécessaires, nous vous recommandons d'utiliser uniquement la table tampon pour l'écriture, lors de la lecture à partir de la table de destination. -Lors de l’ajout de données à un Tampon, un des tampons est verrouillé. Cela entraîne des retards si une opération de lecture est effectuée simultanément à partir de la table. +Lors de l'ajout de données à un Tampon, un des tampons est verrouillé. Cela entraîne des retards si une opération de lecture est effectuée simultanément à partir de la table. Les données insérées dans une table tampon peuvent se retrouver dans la table subordonnée dans un ordre différent et dans des blocs différents. Pour cette raison, une table tampon est difficile à utiliser pour écrire correctement dans un CollapsingMergeTree. Pour éviter les problèmes, vous pouvez définir ‘num\_layers’ 1. -Si la table de destination est répliquée, certaines caractéristiques attendues des tables répliquées sont perdues lors de l’écriture dans une table tampon. Les modifications aléatoires apportées à l’ordre des lignes et des tailles des parties de données provoquent l’arrêt de la déduplication des données, ce qui signifie qu’il n’est pas possible d’avoir un ‘exactly once’ Ecrire dans des tables répliquées. +Si la table de destination est répliquée, certaines caractéristiques attendues des tables répliquées sont perdues lors de l'écriture dans une table tampon. Les modifications aléatoires apportées à l'ordre des lignes et des tailles des parties de données provoquent l'arrêt de la déduplication des données, ce qui signifie qu'il n'est pas possible d'avoir un ‘exactly once’ Ecrire dans des tables répliquées. -En raison de ces inconvénients, nous ne pouvons recommander l’utilisation d’une table tampon que dans de rares cas. +En raison de ces inconvénients, nous ne pouvons recommander l'utilisation d'une table tampon que dans de rares cas. -Une table tampon est utilisée lorsque trop D’insertions sont reçues d’un grand nombre de serveurs sur une unité de temps et que les données ne peuvent pas être mises en mémoire tampon avant l’insertion, ce qui signifie que les insertions ne peuvent pas s’exécuter assez rapidement. +Une table tampon est utilisée lorsque trop D'insertions sont reçues d'un grand nombre de serveurs sur une unité de temps et que les données ne peuvent pas être mises en mémoire tampon avant l'insertion, ce qui signifie que les insertions ne peuvent pas s'exécuter assez rapidement. -Notez qu’il n’est pas judicieux d’insérer des données d’une ligne de temps, même pour Tampon tables. Cela ne produira qu’une vitesse de quelques milliers de lignes par seconde, tandis que l’insertion de blocs de données plus grands peut produire plus d’un million de lignes par seconde (voir la section “Performance”). +Notez qu'il n'est pas judicieux d'insérer des données d'une ligne de temps, même pour Tampon tables. Cela ne produira qu'une vitesse de quelques milliers de lignes par seconde, tandis que l'insertion de blocs de données plus grands peut produire plus d'un million de lignes par seconde (voir la section “Performance”). [Article Original](https://clickhouse.tech/docs/en/operations/table_engines/buffer/) diff --git a/docs/fr/engines/table-engines/special/dictionary.md b/docs/fr/engines/table-engines/special/dictionary.md index a3db44b2064..530d6510d51 100644 --- a/docs/fr/engines/table-engines/special/dictionary.md +++ b/docs/fr/engines/table-engines/special/dictionary.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 35 toc_title: Dictionnaire --- @@ -9,7 +9,7 @@ toc_title: Dictionnaire Le `Dictionary` le moteur affiche le [dictionnaire](../../../sql-reference/dictionaries/external-dictionaries/external-dicts.md) données comme une table ClickHouse. -À titre d’exemple, considérons un dictionnaire de `products` avec la configuration suivante: +À titre d'exemple, considérons un dictionnaire de `products` avec la configuration suivante: ``` xml @@ -64,9 +64,9 @@ WHERE name = 'products' └──────────┴──────┴────────┴─────────────────┴─────────────────┴─────────────────┴───────────────┴─────────────────┘ ``` -Vous pouvez utiliser l’ [dictGet\*](../../../sql-reference/functions/ext-dict-functions.md#ext_dict_functions) fonction pour obtenir les données du dictionnaire dans ce format. +Vous pouvez utiliser l' [dictGet\*](../../../sql-reference/functions/ext-dict-functions.md#ext_dict_functions) fonction pour obtenir les données du dictionnaire dans ce format. -Cette vue n’est pas utile lorsque vous avez besoin d’obtenir des données brutes ou `JOIN` opération. Pour ces cas, vous pouvez utiliser le `Dictionary` moteur, qui affiche les données du dictionnaire dans une table. +Cette vue n'est pas utile lorsque vous avez besoin d'obtenir des données brutes ou `JOIN` opération. Pour ces cas, vous pouvez utiliser le `Dictionary` moteur, qui affiche les données du dictionnaire dans une table. Syntaxe: @@ -74,7 +74,7 @@ Syntaxe: CREATE TABLE %table_name% (%fields%) engine = Dictionary(%dictionary_name%)` ``` -Exemple d’utilisation: +Exemple d'utilisation: ``` sql create table products (product_id UInt64, title String) Engine = Dictionary(products); diff --git a/docs/fr/engines/table-engines/special/distributed.md b/docs/fr/engines/table-engines/special/distributed.md index 0920f662dff..566172bdb66 100644 --- a/docs/fr/engines/table-engines/special/distributed.md +++ b/docs/fr/engines/table-engines/special/distributed.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 33 toc_title: "Distribu\xE9" --- @@ -8,19 +8,19 @@ toc_title: "Distribu\xE9" # Distribué {#distributed} **Les Tables avec moteur distribué ne stockent aucune donnée par elles mêmes**, mais autoriser le traitement des requêtes distribuées sur plusieurs serveurs. -La lecture est automatiquement parallélisée. Lors d’une lecture, les index de table sur les serveurs distants sont utilisés, s’il y en a. +La lecture est automatiquement parallélisée. Lors d'une lecture, les index de table sur les serveurs distants sont utilisés, s'il y en a. Le moteur distribué accepte les paramètres: - le nom du cluster dans le fichier de configuration du serveur -- le nom d’une base de données distante +- le nom d'une base de données distante -- le nom d’une table distante +- le nom d'une table distante - (en option) sharding clé -- (éventuellement) nom de la stratégie, il sera utilisé pour stocker des fichiers temporaires pour l’envoi asynchrone +- (éventuellement) nom de la stratégie, il sera utilisé pour stocker des fichiers temporaires pour l'envoi asynchrone Voir aussi: @@ -35,11 +35,11 @@ Distributed(logs, default, hits[, sharding_key[, policy_name]]) Les données seront lues à partir de tous les serveurs ‘logs’ cluster, à partir de la valeur par défaut.hits table située sur chaque serveur du cluster. Les données ne sont pas seulement lues mais sont partiellement traitées sur les serveurs distants (dans la mesure du possible). -Par exemple, pour une requête avec GROUP BY, les données seront agrégées sur des serveurs distants et les états intermédiaires des fonctions d’agrégation seront envoyés au serveur demandeur. Ensuite, les données seront plus agrégées. +Par exemple, pour une requête avec GROUP BY, les données seront agrégées sur des serveurs distants et les états intermédiaires des fonctions d'agrégation seront envoyés au serveur demandeur. Ensuite, les données seront plus agrégées. Au lieu du nom de la base de données, vous pouvez utiliser une expression constante qui renvoie une chaîne. Par exemple: currentDatabase(). -logs – The cluster name in the server’s config file. +logs – The cluster name in the server's config file. Les Clusters sont définis comme ceci: @@ -79,23 +79,23 @@ Les Clusters sont définis comme ceci: Ici un cluster est défini avec le nom ‘logs’ qui se compose de deux fragments, dont chacune contient deux répliques. Les partitions se réfèrent aux serveurs qui contiennent différentes parties des données (pour lire toutes les données, vous devez accéder à tous les partitions). -Les répliques sont des serveurs de duplication (afin de lire toutes les données, vous pouvez accéder aux données sur l’une des répliques). +Les répliques sont des serveurs de duplication (afin de lire toutes les données, vous pouvez accéder aux données sur l'une des répliques). Les noms de Cluster ne doivent pas contenir de points. Paramètre `host`, `port` et , éventuellement, `user`, `password`, `secure`, `compression` sont spécifiés pour chaque serveur: -- `host` – The address of the remote server. You can use either the domain or the IPv4 or IPv6 address. If you specify the domain, the server makes a DNS request when it starts, and the result is stored as long as the server is running. If the DNS request fails, the server doesn’t start. If you change the DNS record, restart the server. +- `host` – The address of the remote server. You can use either the domain or the IPv4 or IPv6 address. If you specify the domain, the server makes a DNS request when it starts, and the result is stored as long as the server is running. If the DNS request fails, the server doesn't start. If you change the DNS record, restart the server. - `port` – The TCP port for messenger activity (‘tcp\_port’ dans la configuration, généralement définie sur 9000). Ne le confondez pas avec http\_port. -- `user` – Name of the user for connecting to a remote server. Default value: default. This user must have access to connect to the specified server. Access is configured in the users.xml file. For more information, see the section [Les droits d’accès](../../../operations/access-rights.md). +- `user` – Name of the user for connecting to a remote server. Default value: default. This user must have access to connect to the specified server. Access is configured in the users.xml file. For more information, see the section [Les droits d'accès](../../../operations/access-rights.md). - `password` – The password for connecting to a remote server (not masked). Default value: empty string. - `secure` - Utilisez ssl pour la connexion, généralement vous devez également définir `port` = 9440. Le serveur doit écouter `9440` et avoir des certificats corrects. - `compression` - Utiliser la compression de données. Valeur par défaut: true. When specifying replicas, one of the available replicas will be selected for each of the shards when reading. You can configure the algorithm for load balancing (the preference for which replica to access) – see the [équilibrage](../../../operations/settings/settings.md#settings-load_balancing) paramètre. -Si la connexion avec le serveur n’est pas établie, il y aura une tentative de connexion avec un court délai. Si la connexion échoue, la réplique suivante sera sélectionnée, et ainsi de suite pour toutes les répliques. Si la tentative de connexion a échoué pour toutes les répliques, la tentative sera répété de la même façon, plusieurs fois. +Si la connexion avec le serveur n'est pas établie, il y aura une tentative de connexion avec un court délai. Si la connexion échoue, la réplique suivante sera sélectionnée, et ainsi de suite pour toutes les répliques. Si la tentative de connexion a échoué pour toutes les répliques, la tentative sera répété de la même façon, plusieurs fois. Cela fonctionne en faveur de la résilience, mais ne fournit pas de tolérance aux pannes complète: un serveur distant peut accepter la connexion, mais peut ne pas fonctionner ou fonctionner mal. -Vous pouvez spécifier un seul des fragments (dans ce cas, le traitement de la requête doit être appelé distant, plutôt que distribué) ou jusqu’à un nombre quelconque de fragments. Dans chaque fragment, vous pouvez spécifier un nombre de répliques. Vous pouvez spécifier un nombre différent de répliques pour chaque fragment. +Vous pouvez spécifier un seul des fragments (dans ce cas, le traitement de la requête doit être appelé distant, plutôt que distribué) ou jusqu'à un nombre quelconque de fragments. Dans chaque fragment, vous pouvez spécifier un nombre de répliques. Vous pouvez spécifier un nombre différent de répliques pour chaque fragment. Vous pouvez spécifier autant de clusters que vous souhaitez dans la configuration. @@ -103,40 +103,40 @@ Pour afficher vos clusters, utilisez ‘system.clusters’ table. Le moteur distribué permet de travailler avec un cluster comme un serveur local. Cependant, le cluster est inextensible: vous devez écrire sa configuration dans le fichier de configuration du serveur (encore mieux, pour tous les serveurs du cluster). -The Distributed engine requires writing clusters to the config file. Clusters from the config file are updated on the fly, without restarting the server. If you need to send a query to an unknown set of shards and replicas each time, you don’t need to create a Distributed table – use the ‘remote’ fonction de table à la place. Voir la section [Les fonctions de Table](../../../sql-reference/table-functions/index.md). +The Distributed engine requires writing clusters to the config file. Clusters from the config file are updated on the fly, without restarting the server. If you need to send a query to an unknown set of shards and replicas each time, you don't need to create a Distributed table – use the ‘remote’ fonction de table à la place. Voir la section [Les fonctions de Table](../../../sql-reference/table-functions/index.md). Il existe deux méthodes pour écrire des données dans un cluster: -Tout d’abord, vous pouvez définir les serveurs d’écrire les données à et effectuer l’écriture directement sur chaque fragment. En d’autres termes, effectuez INSERT dans les tables que la table distribuée “looks at”. C’est la solution la plus flexible car vous pouvez utiliser n’importe quel schéma de sharding, qui pourrait être non trivial en raison des exigences du sujet. C’est également la solution la plus optimale puisque les données peuvent être écrites sur différents fragments de manière complètement indépendante. +Tout d'abord, vous pouvez définir les serveurs d'écrire les données à et effectuer l'écriture directement sur chaque fragment. En d'autres termes, effectuez INSERT dans les tables que la table distribuée “looks at”. C'est la solution la plus flexible car vous pouvez utiliser n'importe quel schéma de sharding, qui pourrait être non trivial en raison des exigences du sujet. C'est également la solution la plus optimale puisque les données peuvent être écrites sur différents fragments de manière complètement indépendante. -Deuxièmement, vous pouvez effectuer INSERT dans une table distribuée. Dans ce cas, la table distribuera les données insérées sur les serveurs eux-mêmes. Pour écrire dans une table distribuée, elle doit avoir un jeu de clés de sharding (le dernier paramètre). De plus, s’il n’y a qu’un seul fragment, l’opération d’écriture fonctionne sans spécifier la clé de sharding, car cela ne signifie rien dans ce cas. +Deuxièmement, vous pouvez effectuer INSERT dans une table distribuée. Dans ce cas, la table distribuera les données insérées sur les serveurs eux-mêmes. Pour écrire dans une table distribuée, elle doit avoir un jeu de clés de sharding (le dernier paramètre). De plus, s'il n'y a qu'un seul fragment, l'opération d'écriture fonctionne sans spécifier la clé de sharding, car cela ne signifie rien dans ce cas. Chaque fragment peut avoir un poids défini dans le fichier de configuration. Par défaut, le poids est égal à un. Les données sont réparties entre les fragments dans la quantité proportionnelle au poids des fragments. Par exemple, si il y a deux tessons et le premier a un poids de 9 tandis que la seconde a un poids de 10, le premier sera envoyé 9 / 19 parties de lignes, et le second sera envoyé 10 / 19. Chaque fragment peut avoir le ‘internal\_replication’ paramètre défini dans le fichier de configuration. -Si ce paramètre est défini à ‘true’, l’opération d’écriture sélectionne le premier saine réplique et écrit les données. Utilisez cette option si le tableau Distribué “looks at” tables répliquées. En d’autres termes, si la table où les données seront écrites va répliquer elle-même. +Si ce paramètre est défini à ‘true’, l'opération d'écriture sélectionne le premier saine réplique et écrit les données. Utilisez cette option si le tableau Distribué “looks at” tables répliquées. En d'autres termes, si la table où les données seront écrites va répliquer elle-même. -Si elle est définie sur ‘false’ (par défaut), les données sont écrites dans toutes les répliques. En substance, cela signifie que la table distribuée réplique les données elle-même. C’est pire que d’utiliser des tables répliquées, car la cohérence des répliques n’est pas vérifiée et, au fil du temps, elles contiendront des données légèrement différentes. +Si elle est définie sur ‘false’ (par défaut), les données sont écrites dans toutes les répliques. En substance, cela signifie que la table distribuée réplique les données elle-même. C'est pire que d'utiliser des tables répliquées, car la cohérence des répliques n'est pas vérifiée et, au fil du temps, elles contiendront des données légèrement différentes. -Pour sélectionner le fragment auquel une ligne de données est envoyée, l’expression de sharding est analysée et son reste est extrait de la diviser par le poids total des fragments. La ligne est envoyée au fragment qui correspond au demi-intervalle des restes de ‘prev\_weight’ de ‘prev\_weights + weight’, où ‘prev\_weights’ c’est le poids total des tessons avec le plus petit nombre, et ‘weight’ est le poids de cet éclat. Par exemple, s’il y a deux fragments, et que le premier a un poids de 9 tandis que le second a un poids de 10, la ligne sera envoyée au premier fragment pour les restes de la plage \[0, 9), et au second pour les restes de la plage \[9, 19). +Pour sélectionner le fragment auquel une ligne de données est envoyée, l'expression de sharding est analysée et son reste est extrait de la diviser par le poids total des fragments. La ligne est envoyée au fragment qui correspond au demi-intervalle des restes de ‘prev\_weight’ de ‘prev\_weights + weight’, où ‘prev\_weights’ c'est le poids total des tessons avec le plus petit nombre, et ‘weight’ est le poids de cet éclat. Par exemple, s'il y a deux fragments, et que le premier a un poids de 9 tandis que le second a un poids de 10, la ligne sera envoyée au premier fragment pour les restes de la plage \[0, 9), et au second pour les restes de la plage \[9, 19). -L’expression de sharding peut être n’importe quelle expression de constantes et de colonnes de table qui renvoie un entier. Par exemple, vous pouvez utiliser l’expression ‘rand()’ pour la distribution aléatoire des données, ou ‘UserID’ pour la distribution par le reste de la division de L’ID de l’utilisateur (alors les données d’un seul utilisateur résideront sur un seul fragment, ce qui simplifie l’exécution et la jointure par les utilisateurs). Si l’une des colonnes n’est pas assez répartie uniformément, vous pouvez l’envelopper dans une fonction de hachage: intHash64 (UserID). +L'expression de sharding peut être n'importe quelle expression de constantes et de colonnes de table qui renvoie un entier. Par exemple, vous pouvez utiliser l'expression ‘rand()’ pour la distribution aléatoire des données, ou ‘UserID’ pour la distribution par le reste de la division de L'ID de l'utilisateur (alors les données d'un seul utilisateur résideront sur un seul fragment, ce qui simplifie l'exécution et la jointure par les utilisateurs). Si l'une des colonnes n'est pas assez répartie uniformément, vous pouvez l'envelopper dans une fonction de hachage: intHash64 (UserID). -Un simple rappel de la division est une solution limitée pour le sharding et n’est pas toujours approprié. Cela fonctionne pour des volumes de données moyens et importants (des dizaines de serveurs), mais pas pour des volumes de données très importants (des centaines de serveurs ou plus). Dans ce dernier cas, utilisez le schéma de répartition requis par le domaine, plutôt que d’utiliser des entrées dans des tableaux distribués. +Un simple rappel de la division est une solution limitée pour le sharding et n'est pas toujours approprié. Cela fonctionne pour des volumes de données moyens et importants (des dizaines de serveurs), mais pas pour des volumes de données très importants (des centaines de serveurs ou plus). Dans ce dernier cas, utilisez le schéma de répartition requis par le domaine, plutôt que d'utiliser des entrées dans des tableaux distribués. -SELECT queries are sent to all the shards and work regardless of how data is distributed across the shards (they can be distributed completely randomly). When you add a new shard, you don’t have to transfer the old data to it. You can write new data with a heavier weight – the data will be distributed slightly unevenly, but queries will work correctly and efficiently. +SELECT queries are sent to all the shards and work regardless of how data is distributed across the shards (they can be distributed completely randomly). When you add a new shard, you don't have to transfer the old data to it. You can write new data with a heavier weight – the data will be distributed slightly unevenly, but queries will work correctly and efficiently. Vous devriez être préoccupé par le système de sharding dans les cas suivants: - Les requêtes sont utilisées qui nécessitent des données de jointure (IN ou JOIN) par une clé spécifique. Si les données sont partagées par cette clé, vous pouvez utiliser local in ou JOIN au lieu de GLOBAL IN ou global JOIN, ce qui est beaucoup plus efficace. -- Un grand nombre de serveurs est utilisé (des centaines ou plus) avec un grand nombre de petites requêtes (requêtes de clients individuels - sites Web, annonceurs ou partenaires). Pour que les petites requêtes n’affectent pas l’ensemble du cluster, il est logique de localiser les données d’un seul client sur un seul fragment. Alternativement, comme nous l’avons fait dans Yandex.Metrica, vous pouvez configurer le sharding à deux niveaux: divisez le cluster entier en “layers”, où une couche peut être constituée de plusieurs éclats. Les données d’un seul client sont situées sur une seule couche, mais des fragments peuvent être ajoutés à une couche si nécessaire, et les données sont distribuées aléatoirement à l’intérieur de celles-ci. Des tables distribuées sont créées pour chaque couche et une seule table distribuée partagée est créée pour les requêtes globales. +- Un grand nombre de serveurs est utilisé (des centaines ou plus) avec un grand nombre de petites requêtes (requêtes de clients individuels - sites Web, annonceurs ou partenaires). Pour que les petites requêtes n'affectent pas l'ensemble du cluster, il est logique de localiser les données d'un seul client sur un seul fragment. Alternativement, comme nous l'avons fait dans Yandex.Metrica, vous pouvez configurer le sharding à deux niveaux: divisez le cluster entier en “layers”, où une couche peut être constituée de plusieurs éclats. Les données d'un seul client sont situées sur une seule couche, mais des fragments peuvent être ajoutés à une couche si nécessaire, et les données sont distribuées aléatoirement à l'intérieur de celles-ci. Des tables distribuées sont créées pour chaque couche et une seule table distribuée partagée est créée pour les requêtes globales. -Les données sont écrites de manière asynchrone. Lorsqu’il est inséré dans la table, le bloc de données est simplement écrit dans le système de fichiers local. Les données sont envoyées aux serveurs distants en arrière-plan dès que possible. La période d’envoi des données est gérée par [distributed\_directory\_monitor\_sleep\_time\_ms](../../../operations/settings/settings.md#distributed_directory_monitor_sleep_time_ms) et [distributed\_directory\_monitor\_max\_sleep\_time\_ms](../../../operations/settings/settings.md#distributed_directory_monitor_max_sleep_time_ms) paramètre. Le `Distributed` moteur envoie chaque fichier de données insérées séparément, mais vous pouvez activer le lot envoi de fichiers avec l’ [distributed\_directory\_monitor\_batch\_inserts](../../../operations/settings/settings.md#distributed_directory_monitor_batch_inserts) paramètre. Ce paramètre améliore les performances du cluster en utilisant mieux les ressources réseau et serveur local. Vous devriez vérifier si les données sont envoyées avec succès en vérifiant la liste des fichiers (données en attente d’envoi) dans le répertoire de la table: `/var/lib/clickhouse/data/database/table/`. +Les données sont écrites de manière asynchrone. Lorsqu'il est inséré dans la table, le bloc de données est simplement écrit dans le système de fichiers local. Les données sont envoyées aux serveurs distants en arrière-plan dès que possible. La période d'envoi des données est gérée par [distributed\_directory\_monitor\_sleep\_time\_ms](../../../operations/settings/settings.md#distributed_directory_monitor_sleep_time_ms) et [distributed\_directory\_monitor\_max\_sleep\_time\_ms](../../../operations/settings/settings.md#distributed_directory_monitor_max_sleep_time_ms) paramètre. Le `Distributed` moteur envoie chaque fichier de données insérées séparément, mais vous pouvez activer le lot envoi de fichiers avec l' [distributed\_directory\_monitor\_batch\_inserts](../../../operations/settings/settings.md#distributed_directory_monitor_batch_inserts) paramètre. Ce paramètre améliore les performances du cluster en utilisant mieux les ressources réseau et serveur local. Vous devriez vérifier si les données sont envoyées avec succès en vérifiant la liste des fichiers (données en attente d'envoi) dans le répertoire de la table: `/var/lib/clickhouse/data/database/table/`. -Si le serveur a cessé d’exister ou a subi un redémarrage Brutal (par exemple, après une panne de périphérique) après une insertion dans une table distribuée, les données insérées peuvent être perdues. Si une partie de données endommagée est détectée dans le répertoire de la table, elle est transférée ‘broken’ sous-répertoire et n’est plus utilisé. +Si le serveur a cessé d'exister ou a subi un redémarrage Brutal (par exemple, après une panne de périphérique) après une insertion dans une table distribuée, les données insérées peuvent être perdues. Si une partie de données endommagée est détectée dans le répertoire de la table, elle est transférée ‘broken’ sous-répertoire et n'est plus utilisé. -Lorsque l’option max\_parallel\_replicas est activée, le traitement des requêtes est parallélisé entre toutes les répliques d’un seul fragment. Pour plus d’informations, consultez la section [max\_parallel\_replicas](../../../operations/settings/settings.md#settings-max_parallel_replicas). +Lorsque l'option max\_parallel\_replicas est activée, le traitement des requêtes est parallélisé entre toutes les répliques d'un seul fragment. Pour plus d'informations, consultez la section [max\_parallel\_replicas](../../../operations/settings/settings.md#settings-max_parallel_replicas). ## Les Colonnes Virtuelles {#virtual-columns} @@ -147,6 +147,6 @@ Lorsque l’option max\_parallel\_replicas est activée, le traitement des requ **Voir Aussi** -- [Les colonnes virtuelles](../index.md#table_engines-virtual_columns) +- [Les colonnes virtuelles](index.md#table_engines-virtual_columns) [Article Original](https://clickhouse.tech/docs/en/operations/table_engines/distributed/) diff --git a/docs/fr/engines/table-engines/special/external-data.md b/docs/fr/engines/table-engines/special/external-data.md index 58946f2bc5f..acf63600f61 100644 --- a/docs/fr/engines/table-engines/special/external-data.md +++ b/docs/fr/engines/table-engines/special/external-data.md @@ -1,19 +1,19 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 34 toc_title: "De donn\xE9es externes" --- -# Données Externes Pour Le Traitement Des requêtes {#external-data-for-query-processing} +# Données externes pour le traitement des requêtes {#external-data-for-query-processing} -ClickHouse permet d’Envoyer à un serveur les données nécessaires au traitement d’une requête, ainsi qu’une requête SELECT. Ces données sont placées dans une table temporaire (voir la section “Temporary tables”) et peut être utilisé dans la requête (par exemple, dans les DANS les opérateurs). +ClickHouse permet d'Envoyer à un serveur les données nécessaires au traitement d'une requête, ainsi qu'une requête SELECT. Ces données sont placées dans une table temporaire (voir la section “Temporary tables”) et peut être utilisé dans la requête (par exemple, dans les DANS les opérateurs). -Par exemple, si vous disposez d’un fichier texte avec important des identifiants d’utilisateur, vous pouvez le télécharger sur le serveur avec une requête qui utilise la filtration par cette liste. +Par exemple, si vous disposez d'un fichier texte avec important des identifiants d'utilisateur, vous pouvez le télécharger sur le serveur avec une requête qui utilise la filtration par cette liste. -Si vous devez exécuter plusieurs requêtes avec un volume important de données externes, n’utilisez pas cette fonctionnalité. Il est préférable de télécharger les données sur la base de données à l’avance. +Si vous devez exécuter plusieurs requêtes avec un volume important de données externes, n'utilisez pas cette fonctionnalité. Il est préférable de télécharger les données sur la base de données à l'avance. -Les données externes peuvent être téléchargées à l’aide du client de ligne de commande (en mode non interactif) ou à l’aide de L’interface HTTP. +Les données externes peuvent être téléchargées à l'aide du client de ligne de commande (en mode non interactif) ou à l'aide de L'interface HTTP. Dans le client de ligne de commande, vous pouvez spécifier une section paramètres du format @@ -30,7 +30,7 @@ Une seule table peut être récupérée à partir de stdin. Les paramètres suivants sont facultatifs: **–name**– Name of the table. If omitted, \_data is used. **–format** – Data format in the file. If omitted, TabSeparated is used. -L’un des paramètres suivants est requis:**–types** – A list of comma-separated column types. For example: `UInt64,String`. The columns will be named \_1, \_2, … +L'un des paramètres suivants est requis:**–types** – A list of comma-separated column types. For example: `UInt64,String`. The columns will be named \_1, \_2, … **–structure**– The table structure in the format`UserID UInt64`, `URL String`. Définit les noms et les types de colonnes. Les fichiers spécifiés dans ‘file’ sera analysé par le format spécifié dans ‘format’, en utilisant les types de données spécifié dans ‘types’ ou ‘structure’. La table sera téléchargée sur le serveur et accessible en tant que table temporaire avec le nom dans ‘name’. @@ -48,7 +48,7 @@ $ cat /etc/passwd | sed 's/:/\t/g' | clickhouse-client --query="SELECT shell, co /bin/sync 1 ``` -Lors de l’utilisation de L’interface HTTP, les données externes sont transmises au format multipart/form-data. Chaque tableau est transmis en tant que fichier séparé. Le nom de la table est tiré du nom du fichier. Le ‘query\_string’ est passé les paramètres ‘name\_format’, ‘name\_types’, et ‘name\_structure’, où ‘name’ est le nom de la table que ces paramètres correspondent. La signification des paramètres est la même que lors de l’utilisation du client de ligne de commande. +Lors de l'utilisation de L'interface HTTP, les données externes sont transmises au format multipart/form-data. Chaque tableau est transmis en tant que fichier séparé. Le nom de la table est tiré du nom du fichier. Le ‘query\_string’ est passé les paramètres ‘name\_format’, ‘name\_types’, et ‘name\_structure’, où ‘name’ est le nom de la table que ces paramètres correspondent. La signification des paramètres est la même que lors de l'utilisation du client de ligne de commande. Exemple: diff --git a/docs/fr/engines/table-engines/special/file.md b/docs/fr/engines/table-engines/special/file.md index 7b599f0820e..eedc1206527 100644 --- a/docs/fr/engines/table-engines/special/file.md +++ b/docs/fr/engines/table-engines/special/file.md @@ -1,29 +1,29 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 37 toc_title: Fichier --- # Fichier {#table_engines-file} -Le moteur de table de fichiers conserve les données dans un fichier dans l’un des [fichier +Le moteur de table de fichiers conserve les données dans un fichier dans l'un des [fichier format](../../../interfaces/formats.md#formats) (TabSeparated, Native, etc.). -Exemples d’utilisation: +Exemples d'utilisation: - Exportation de données de ClickHouse vers un fichier. -- Convertir des données d’un format à un autre. -- Mise à jour des données dans ClickHouse via l’édition d’un fichier sur un disque. +- Convertir des données d'un format à un autre. +- Mise à jour des données dans ClickHouse via l'édition d'un fichier sur un disque. -## Utilisation Dans Le Serveur Clickhouse {#usage-in-clickhouse-server} +## Utilisation dans le serveur ClickHouse {#usage-in-clickhouse-server} ``` sql File(Format) ``` -Le `Format` paramètre spécifie l’un des formats de fichier disponibles. Effectuer -`SELECT` requêtes, le format doit être pris en charge pour l’entrée, et à effectuer +Le `Format` paramètre spécifie l'un des formats de fichier disponibles. Effectuer +`SELECT` requêtes, le format doit être pris en charge pour l'entrée, et à effectuer `INSERT` queries – for output. The available formats are listed in the [Format](../../../interfaces/formats.md#formats) section. @@ -34,7 +34,7 @@ Lors de la création de la table en utilisant `File(Format)` il crée sous-répe Vous pouvez créer manuellement ce sous dossier et ce fichier dans le système de fichiers [ATTACH](../../../sql-reference/statements/misc.md) il à la table des informations avec le nom correspondant, de sorte que vous pouvez interroger les données de ce fichier. !!! warning "Avertissement" - Soyez prudent avec cette fonctionnalité, car ClickHouse ne garde pas trace des modifications externes apportées à ces fichiers. Le résultat des Écritures simultanées via ClickHouse et en dehors de ClickHouse n’est pas défini. + Soyez prudent avec cette fonctionnalité, car ClickHouse ne garde pas trace des modifications externes apportées à ces fichiers. Le résultat des Écritures simultanées via ClickHouse et en dehors de ClickHouse n'est pas défini. **Exemple:** @@ -67,19 +67,19 @@ SELECT * FROM file_engine_table └──────┴───────┘ ``` -## Utilisation Dans Clickhouse-local {#usage-in-clickhouse-local} +## Utilisation dans ClickHouse-local {#usage-in-clickhouse-local} -Dans [clickhouse-local](../../../operations/utilities/clickhouse-local.md) Fichier moteur accepte chemin d’accès au fichier en plus `Format`. Les flux d’entrée / sortie par défaut peuvent être spécifiés en utilisant des noms numériques ou lisibles par l’homme comme `0` ou `stdin`, `1` ou `stdout`. +Dans [clickhouse-local](../../../operations/utilities/clickhouse-local.md) Fichier moteur accepte chemin d'accès au fichier en plus `Format`. Les flux d'entrée / sortie par défaut peuvent être spécifiés en utilisant des noms numériques ou lisibles par l'homme comme `0` ou `stdin`, `1` ou `stdout`. **Exemple:** ``` bash $ echo -e "1,2\n3,4" | clickhouse-local -q "CREATE TABLE table (a Int64, b Int64) ENGINE = File(CSV, stdin); SELECT a, b FROM table; DROP TABLE table" ``` -## Les Détails De Mise En Œuvre {#details-of-implementation} +## Les détails de mise en Œuvre {#details-of-implementation} -- Plusieurs `SELECT` les requêtes peuvent être effectuées simultanément, mais `INSERT` requêtes s’attendre les uns les autres. -- Prise en charge de la création d’un nouveau fichier par `INSERT` requête. +- Plusieurs `SELECT` les requêtes peuvent être effectuées simultanément, mais `INSERT` requêtes s'attendre les uns les autres. +- Prise en charge de la création d'un nouveau fichier par `INSERT` requête. - Si le fichier existe, `INSERT` ajouterait de nouvelles valeurs dedans. - Pas pris en charge: - `ALTER` diff --git a/docs/fr/engines/table-engines/special/generate.md b/docs/fr/engines/table-engines/special/generate.md index 74e3c6fb7bb..9b2aa57c3e5 100644 --- a/docs/fr/engines/table-engines/special/generate.md +++ b/docs/fr/engines/table-engines/special/generate.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 46 toc_title: GenerateRandom --- @@ -9,12 +9,12 @@ toc_title: GenerateRandom Le moteur de table GenerateRandom produit des données aléatoires pour un schéma de table donné. -Exemples d’utilisation: +Exemples d'utilisation: - Utiliser dans le test pour remplir une grande table reproductible. - Générer une entrée aléatoire pour les tests de fuzzing. -## Utilisation Dans Le Serveur Clickhouse {#usage-in-clickhouse-server} +## Utilisation dans le serveur ClickHouse {#usage-in-clickhouse-server} ``` sql ENGINE = GenerateRandom(random_seed, max_string_length, max_array_length) @@ -49,7 +49,7 @@ SELECT * FROM generate_engine_table LIMIT 3 └──────┴────────────┘ ``` -## Les Détails De Mise En Œuvre {#details-of-implementation} +## Les détails de mise en Œuvre {#details-of-implementation} - Pas pris en charge: - `ALTER` diff --git a/docs/fr/engines/table-engines/special/index.md b/docs/fr/engines/table-engines/special/index.md index 481be47314c..424a80c511d 100644 --- a/docs/fr/engines/table-engines/special/index.md +++ b/docs/fr/engines/table-engines/special/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 -toc_folder_title: Special +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "Sp\xE9cial" toc_priority: 31 --- diff --git a/docs/fr/engines/table-engines/special/join.md b/docs/fr/engines/table-engines/special/join.md index 7af77d21fb2..a3b33a7aa32 100644 --- a/docs/fr/engines/table-engines/special/join.md +++ b/docs/fr/engines/table-engines/special/join.md @@ -1,15 +1,15 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 40 toc_title: Rejoindre --- # Rejoindre {#join} -Structure de données préparée pour l’utilisation dans [JOIN](../../../sql-reference/statements/select.md#select-join) opérations. +Structure de données préparée pour l'utilisation dans [JOIN](../../../sql-reference/statements/select/join.md#select-join) opérations. -## Création d’une Table {#creating-a-table} +## Création d'une Table {#creating-a-table} ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] @@ -23,11 +23,11 @@ Voir la description détaillée de la [CREATE TABLE](../../../sql-reference/stat **Les Paramètres Du Moteur** -- `join_strictness` – [ADHÉRER à la rigueur](../../../sql-reference/statements/select.md#select-join-strictness). -- `join_type` – [Type de jointure](../../../sql-reference/statements/select.md#select-join-types). -- `k1[, k2, ...]` – Key columns from the `USING` la clause que l’ `JOIN` l’opération est faite avec de la. +- `join_strictness` – [ADHÉRER à la rigueur](../../../sql-reference/statements/select/join.md#select-join-strictness). +- `join_type` – [Type de jointure](../../../sql-reference/statements/select/join.md#select-join-types). +- `k1[, k2, ...]` – Key columns from the `USING` la clause que l' `JOIN` l'opération est faite avec de la. -Entrer `join_strictness` et `join_type` paramètres sans guillemets, par exemple, `Join(ANY, LEFT, col1)`. Ils doivent correspondre à la `JOIN` fonctionnement que le tableau sera utilisé pour. Si les paramètres ne correspondent pas, ClickHouse ne lance pas d’exception et peut renvoyer des données incorrectes. +Entrer `join_strictness` et `join_type` paramètres sans guillemets, par exemple, `Join(ANY, LEFT, col1)`. Ils doivent correspondre à la `JOIN` fonctionnement que le tableau sera utilisé pour. Si les paramètres ne correspondent pas, ClickHouse ne lance pas d'exception et peut renvoyer des données incorrectes. ## Utilisation Du Tableau {#table-usage} @@ -79,18 +79,18 @@ SELECT joinGet('id_val_join', 'val', toUInt32(1)) └────────────────────────────────────────────┘ ``` -### Sélection Et Insertion De données {#selecting-and-inserting-data} +### Sélection et insertion de données {#selecting-and-inserting-data} -Vous pouvez utiliser `INSERT` requêtes pour ajouter des données au `Join`-tables de moteur. Si la table a été créée avec `ANY` rigueur, les données pour les clés en double sont ignorées. Avec l’ `ALL` rigueur, toutes les lignes sont ajoutées. +Vous pouvez utiliser `INSERT` requêtes pour ajouter des données au `Join`-tables de moteur. Si la table a été créée avec `ANY` rigueur, les données pour les clés en double sont ignorées. Avec l' `ALL` rigueur, toutes les lignes sont ajoutées. -Vous ne pouvez pas effectuer un `SELECT` requête directement à partir de la table. Au lieu de cela, utilisez l’une des méthodes suivantes: +Vous ne pouvez pas effectuer un `SELECT` requête directement à partir de la table. Au lieu de cela, utilisez l'une des méthodes suivantes: - Placez la table sur le côté droit dans un `JOIN` clause. -- Appelez le [joinGet](../../../sql-reference/functions/other-functions.md#joinget) fonction, qui vous permet d’extraire des données de la table de la même manière que d’un dictionnaire. +- Appelez le [joinGet](../../../sql-reference/functions/other-functions.md#joinget) fonction, qui vous permet d'extraire des données de la table de la même manière que d'un dictionnaire. -### Limitations Et paramètres {#join-limitations-and-settings} +### Limitations et paramètres {#join-limitations-and-settings} -Lors de la création d’un tableau, les paramètres suivants sont appliqués: +Lors de la création d'un tableau, les paramètres suivants sont appliqués: - [join\_use\_nulls](../../../operations/settings/settings.md#join_use_nulls) - [max\_rows\_in\_join](../../../operations/settings/query-complexity.md#settings-max_rows_in_join) @@ -100,11 +100,11 @@ Lors de la création d’un tableau, les paramètres suivants sont appliqués: Le `Join`- les tables de moteur ne peuvent pas être utilisées dans `GLOBAL JOIN` opérations. -Le `Join`-moteur permet d’utiliser [join\_use\_nulls](../../../operations/settings/settings.md#join_use_nulls) réglage de la `CREATE TABLE` déclaration. Et [SELECT](../../../sql-reference/statements/select.md) requête permet d’utiliser `join_use_nulls` trop. Si vous avez différents `join_use_nulls` paramètres, vous pouvez obtenir une table de jointure d’erreur. Il dépend de type de JOINTURE. Lorsque vous utilisez [joinGet](../../../sql-reference/functions/other-functions.md#joinget) fonction, vous devez utiliser le même `join_use_nulls` réglage en `CRATE TABLE` et `SELECT` déclaration. +Le `Join`-moteur permet d'utiliser [join\_use\_nulls](../../../operations/settings/settings.md#join_use_nulls) réglage de la `CREATE TABLE` déclaration. Et [SELECT](../../../sql-reference/statements/select/index.md) requête permet d'utiliser `join_use_nulls` trop. Si vous avez différents `join_use_nulls` paramètres, vous pouvez obtenir une table de jointure d'erreur. Il dépend de type de JOINTURE. Lorsque vous utilisez [joinGet](../../../sql-reference/functions/other-functions.md#joinget) fonction, vous devez utiliser le même `join_use_nulls` réglage en `CRATE TABLE` et `SELECT` déclaration. ## Le Stockage De Données {#data-storage} -`Join` les données de la table sont toujours situées dans la RAM. Lors de l’insertion de lignes dans une table, ClickHouse écrit des blocs de données dans le répertoire du disque afin qu’ils puissent être restaurés lorsque le serveur redémarre. +`Join` les données de la table sont toujours situées dans la RAM. Lors de l'insertion de lignes dans une table, ClickHouse écrit des blocs de données dans le répertoire du disque afin qu'ils puissent être restaurés lorsque le serveur redémarre. Si le serveur redémarre incorrectement, le bloc de données sur le disque peut être perdu ou endommagé. Dans ce cas, vous devrez peut-être supprimer manuellement le fichier contenant des données endommagées. diff --git a/docs/fr/engines/table-engines/special/materializedview.md b/docs/fr/engines/table-engines/special/materializedview.md index 84e5a94c9fe..e1b4d4a708d 100644 --- a/docs/fr/engines/table-engines/special/materializedview.md +++ b/docs/fr/engines/table-engines/special/materializedview.md @@ -1,12 +1,12 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 43 toc_title: MaterializedView --- # Materializedview {#materializedview} -Utilisé pour implémenter des vues matérialisées (pour plus d’informations, voir [CREATE TABLE](../../../sql-reference/statements/create.md)). Pour stocker des données, il utilise un moteur différent qui a été spécifié lors de la création de la vue. Lors de la lecture d’une table, il utilise juste ce moteur. +Utilisé pour implémenter des vues matérialisées (pour plus d'informations, voir [CREATE TABLE](../../../sql-reference/statements/create.md)). Pour stocker des données, il utilise un moteur différent qui a été spécifié lors de la création de la vue. Lors de la lecture d'une table, il utilise juste ce moteur. [Article Original](https://clickhouse.tech/docs/en/operations/table_engines/materializedview/) diff --git a/docs/fr/engines/table-engines/special/memory.md b/docs/fr/engines/table-engines/special/memory.md index 06c727a4eb0..94e619eae07 100644 --- a/docs/fr/engines/table-engines/special/memory.md +++ b/docs/fr/engines/table-engines/special/memory.md @@ -1,18 +1,18 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 44 toc_title: "M\xE9moire" --- # Mémoire {#memory} -Le moteur de mémoire stocke les données en RAM, sous forme non compressée. Les données sont stockées exactement sous la même forme qu’elles sont reçues lors de la lecture. En d’autres termes, la lecture de ce tableau est entièrement gratuit. -L’accès aux données simultanées est synchronisé. Les verrous sont courts: les opérations de lecture et d’écriture ne se bloquent pas. +Le moteur de mémoire stocke les données en RAM, sous forme non compressée. Les données sont stockées exactement sous la même forme qu'elles sont reçues lors de la lecture. En d'autres termes, la lecture de ce tableau est entièrement gratuit. +L'accès aux données simultanées est synchronisé. Les verrous sont courts: les opérations de lecture et d'écriture ne se bloquent pas. Les index ne sont pas pris en charge. La lecture est parallélisée. -La productivité maximale (plus de 10 Go / s) est atteinte sur les requêtes simples, car il n’y a pas de lecture à partir du disque, de décompression ou de désérialisation des données. (Il convient de noter que dans de nombreux cas, la productivité du moteur MergeTree est presque aussi élevée.) -Lors du redémarrage d’un serveur, les données disparaissent de la table et la table devient vide. -Normalement, l’utilisation de ce moteur de table n’est pas justifiée. Cependant, il peut être utilisé pour des tests, et pour des tâches où la vitesse maximale est requise sur un nombre relativement faible de lignes (jusqu’à environ 100 000 000). +La productivité maximale (plus de 10 Go / s) est atteinte sur les requêtes simples, car il n'y a pas de lecture à partir du disque, de décompression ou de désérialisation des données. (Il convient de noter que dans de nombreux cas, la productivité du moteur MergeTree est presque aussi élevée.) +Lors du redémarrage d'un serveur, les données disparaissent de la table et la table devient vide. +Normalement, l'utilisation de ce moteur de table n'est pas justifiée. Cependant, il peut être utilisé pour des tests, et pour des tâches où la vitesse maximale est requise sur un nombre relativement faible de lignes (jusqu'à environ 100 000 000). Le moteur de mémoire est utilisé par le système pour les tables temporaires avec des données de requête externes (voir la section “External data for processing a query”), et pour la mise en œuvre globale dans (voir la section “IN operators”). diff --git a/docs/fr/engines/table-engines/special/merge.md b/docs/fr/engines/table-engines/special/merge.md index d534f30b6c9..991204c7dd6 100644 --- a/docs/fr/engines/table-engines/special/merge.md +++ b/docs/fr/engines/table-engines/special/merge.md @@ -1,14 +1,14 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 36 toc_title: Fusionner --- # Fusionner {#merge} -Le `Merge` moteur (à ne pas confondre avec `MergeTree`) ne stocke pas les données elles-mêmes, mais permet la lecture de n’importe quel nombre d’autres tables simultanément. -La lecture est automatiquement parallélisée. L’écriture dans une table n’est pas prise en charge. Lors de la lecture, les index des tables en cours de lecture sont utilisés, s’ils existent. +Le `Merge` moteur (à ne pas confondre avec `MergeTree`) ne stocke pas les données elles-mêmes, mais permet la lecture de n'importe quel nombre d'autres tables simultanément. +La lecture est automatiquement parallélisée. L'écriture dans une table n'est pas prise en charge. Lors de la lecture, les index des tables en cours de lecture sont utilisés, s'ils existent. Le `Merge` engine accepte les paramètres: le nom de la base de données et une expression régulière pour les tables. Exemple: @@ -17,17 +17,17 @@ Exemple: Merge(hits, '^WatchLog') ``` -Les données seront lues à partir des tableaux du `hits` base de données dont les noms correspondent à l’expression régulière ‘`^WatchLog`’. +Les données seront lues à partir des tableaux du `hits` base de données dont les noms correspondent à l'expression régulière ‘`^WatchLog`’. Au lieu du nom de la base de données, vous pouvez utiliser une expression constante qui renvoie une chaîne. Exemple, `currentDatabase()`. Regular expressions — [re2](https://github.com/google/re2) (prend en charge un sous-ensemble de PCRE), sensible à la casse. -Voir les notes sur les symboles d’échappement dans les expressions régulières “match” section. +Voir les notes sur les symboles d'échappement dans les expressions régulières “match” section. -Lors de la sélection des tables à lire, le `Merge` le tableau lui-même ne sera pas choisie, même si elle correspond à l’expression rationnelle. C’est pour éviter les boucles. -Il est possible de créer deux `Merge` des tables qui essaieront sans cesse de lire les données des autres, mais ce n’est pas une bonne idée. +Lors de la sélection des tables à lire, le `Merge` le tableau lui-même ne sera pas choisie, même si elle correspond à l'expression rationnelle. C'est pour éviter les boucles. +Il est possible de créer deux `Merge` des tables qui essaieront sans cesse de lire les données des autres, mais ce n'est pas une bonne idée. -L’utilisation traditionnelle de la `Merge` moteur pour travailler avec un grand nombre de `TinyLog` les tables comme si avec une seule table. +L'utilisation traditionnelle de la `Merge` moteur pour travailler avec un grand nombre de `TinyLog` les tables comme si avec une seule table. Exemple 2: @@ -61,7 +61,7 @@ FROM WatchLog - `_table` — Contains the name of the table from which data was read. Type: [Chaîne](../../../sql-reference/data-types/string.md). - Vous pouvez définir les conditions constantes sur `_table` dans le `WHERE/PREWHERE` clause (par exemple, `WHERE _table='xyz'`). Dans ce cas l’opération de lecture est effectuée uniquement pour les tables où la condition sur `_table` est satisfaite, pour le `_table` colonne agit comme un index. + Vous pouvez définir les conditions constantes sur `_table` dans le `WHERE/PREWHERE` clause (par exemple, `WHERE _table='xyz'`). Dans ce cas l'opération de lecture est effectuée uniquement pour les tables où la condition sur `_table` est satisfaite, pour le `_table` colonne agit comme un index. **Voir Aussi** diff --git a/docs/fr/engines/table-engines/special/null.md b/docs/fr/engines/table-engines/special/null.md index bffd4267b79..5215fc753be 100644 --- a/docs/fr/engines/table-engines/special/null.md +++ b/docs/fr/engines/table-engines/special/null.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 38 toc_title: 'NULL' --- # NULL {#null} -Lors de l’écriture dans une table Null, Les données sont ignorées. Lors de la lecture à partir d’une table Null, la réponse est vide. +Lors de l'écriture dans une table Null, Les données sont ignorées. Lors de la lecture à partir d'une table Null, la réponse est vide. Toutefois, vous pouvez créer une vue matérialisée sur une table Nuls. Ainsi, les données écrites dans la table finira dans la vue. diff --git a/docs/fr/engines/table-engines/special/set.md b/docs/fr/engines/table-engines/special/set.md index 5f81f2b0a30..32bfde63698 100644 --- a/docs/fr/engines/table-engines/special/set.md +++ b/docs/fr/engines/table-engines/special/set.md @@ -1,18 +1,18 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 39 toc_title: "D\xE9finir" --- # Définir {#set} -Un ensemble de données qui est toujours en RAM. Il est conçu pour être utilisé sur le côté droit de l’opérateur (voir la section “IN operators”). +Un ensemble de données qui est toujours en RAM. Il est conçu pour être utilisé sur le côté droit de l'opérateur (voir la section “IN operators”). -Vous pouvez utiliser INSERT pour insérer des données dans la table. De nouveaux éléments seront ajoutés à l’ensemble de données, tandis que les doublons seront ignorés. -Mais vous ne pouvez pas effectuer SELECT à partir de la table. La seule façon de récupérer des données est en l’utilisant dans la moitié droite de l’opérateur. +Vous pouvez utiliser INSERT pour insérer des données dans la table. De nouveaux éléments seront ajoutés à l'ensemble de données, tandis que les doublons seront ignorés. +Mais vous ne pouvez pas effectuer SELECT à partir de la table. La seule façon de récupérer des données est en l'utilisant dans la moitié droite de l'opérateur. -Les données sont toujours situées dans la RAM. Pour INSERT, les blocs de données insérées sont également écrits dans le répertoire des tables sur le disque. Lors du démarrage du serveur, ces données sont chargées dans la RAM. En d’autres termes, après le redémarrage, les données restent en place. +Les données sont toujours situées dans la RAM. Pour INSERT, les blocs de données insérées sont également écrits dans le répertoire des tables sur le disque. Lors du démarrage du serveur, ces données sont chargées dans la RAM. En d'autres termes, après le redémarrage, les données restent en place. Pour un redémarrage brutal du serveur, le bloc de données sur le disque peut être perdu ou endommagé. Dans ce dernier cas, vous devrez peut-être supprimer manuellement le fichier contenant des données endommagées. diff --git a/docs/fr/engines/table-engines/special/url.md b/docs/fr/engines/table-engines/special/url.md index 30b1d0b7e8b..36068845546 100644 --- a/docs/fr/engines/table-engines/special/url.md +++ b/docs/fr/engines/table-engines/special/url.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 41 toc_title: URL --- @@ -8,15 +8,15 @@ toc_title: URL # URL (URL, Format) {#table_engines-url} Gère les données sur un serveur HTTP / HTTPS distant. Ce moteur est similaire -à l’ [Fichier](file.md) moteur. +à l' [Fichier](file.md) moteur. -## Utilisation Du Moteur Dans Le Serveur Clickhouse {#using-the-engine-in-the-clickhouse-server} +## Utilisation du moteur dans le serveur ClickHouse {#using-the-engine-in-the-clickhouse-server} Le `format` doit être celui que ClickHouse peut utiliser dans `SELECT` les requêtes et, si nécessaire, en `INSERTs`. Pour la liste complète des formats pris en charge, voir [Format](../../../interfaces/formats.md#formats). -Le `URL` doit être conforme à la structure D’un Localisateur de ressources uniforme. L’URL spécifiée doit pointer vers un serveur +Le `URL` doit être conforme à la structure D'un Localisateur de ressources uniforme. L'URL spécifiée doit pointer vers un serveur qui utilise le protocole HTTP ou HTTPS. Cela ne nécessite pas de en-têtes supplémentaires pour obtenir une réponse du serveur. @@ -35,7 +35,7 @@ CREATE TABLE url_engine_table (word String, value UInt64) ENGINE=URL('http://127.0.0.1:12345/', CSV) ``` -**2.** Créez un serveur HTTP de base à l’aide des outils Python 3 standard et +**2.** Créez un serveur HTTP de base à l'aide des outils Python 3 standard et démarrer: ``` python3 @@ -71,7 +71,7 @@ SELECT * FROM url_engine_table └───────┴───────┘ ``` -## Les Détails De Mise En Œuvre {#details-of-implementation} +## Les détails de mise en Œuvre {#details-of-implementation} - Les lectures et les écritures peuvent être parallèles - Pas pris en charge: diff --git a/docs/fr/engines/table-engines/special/view.md b/docs/fr/engines/table-engines/special/view.md index 6711f0817d7..aaeb6ce90cb 100644 --- a/docs/fr/engines/table-engines/special/view.md +++ b/docs/fr/engines/table-engines/special/view.md @@ -1,12 +1,12 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 42 toc_title: Vue --- # Vue {#table_engines-view} -Utilisé pour implémenter des vues (pour plus d’informations, voir `CREATE VIEW query`). Il ne stocke pas de données, mais stocke uniquement les `SELECT` requête. Lors de la lecture d’une table, il exécute cette requête (et supprime toutes les colonnes inutiles de la requête). +Utilisé pour implémenter des vues (pour plus d'informations, voir `CREATE VIEW query`). Il ne stocke pas de données, mais stocke uniquement les `SELECT` requête. Lors de la lecture d'une table, il exécute cette requête (et supprime toutes les colonnes inutiles de la requête). [Article Original](https://clickhouse.tech/docs/en/operations/table_engines/view/) diff --git a/docs/fr/faq/general.md b/docs/fr/faq/general.md index 443eea4bf49..bd2765be2d2 100644 --- a/docs/fr/faq/general.md +++ b/docs/fr/faq/general.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 78 toc_title: "Questions G\xE9n\xE9rales" --- @@ -9,15 +9,15 @@ toc_title: "Questions G\xE9n\xE9rales" ## Pourquoi Ne Pas Utiliser Quelque Chose Comme MapReduce? {#why-not-use-something-like-mapreduce} -Nous pouvons nous référer à des systèmes comme MapReduce en tant que systèmes informatiques distribués dans lesquels l’opération de réduction est basée sur le tri distribué. La solution open-source la plus courante dans cette classe est [Apache Hadoop](http://hadoop.apache.org). Yandex utilise sa solution interne, YT. +Nous pouvons nous référer à des systèmes comme MapReduce en tant que systèmes informatiques distribués dans lesquels l'opération de réduction est basée sur le tri distribué. La solution open-source la plus courante dans cette classe est [Apache Hadoop](http://hadoop.apache.org). Yandex utilise sa solution interne, YT. -Ces systèmes ne sont pas appropriés pour les requêtes en ligne en raison de leur latence élevée. En d’autres termes, ils ne peuvent pas être utilisés comme back-end pour une interface web. Ces types de systèmes ne sont pas utiles pour les mises à jour de données en temps réel. Le tri distribué n’est pas la meilleure façon d’effectuer des opérations de réduction si le résultat de l’opération et tous les résultats intermédiaires (s’il y en a) sont situés dans la RAM d’un seul serveur, ce qui est généralement le cas pour les requêtes en ligne. Dans un tel cas, une table de hachage est un moyen optimal d’effectuer des opérations de réduction. Une approche courante pour optimiser les tâches map-reduce est la pré-agrégation (réduction partielle) à l’aide d’une table de hachage en RAM. L’utilisateur effectue cette optimisation manuellement. Le tri distribué est l’une des principales causes de réduction des performances lors de l’exécution de tâches simples de réduction de la carte. +Ces systèmes ne sont pas appropriés pour les requêtes en ligne en raison de leur latence élevée. En d'autres termes, ils ne peuvent pas être utilisés comme back-end pour une interface web. Ces types de systèmes ne sont pas utiles pour les mises à jour de données en temps réel. Le tri distribué n'est pas la meilleure façon d'effectuer des opérations de réduction si le résultat de l'opération et tous les résultats intermédiaires (s'il y en a) sont situés dans la RAM d'un seul serveur, ce qui est généralement le cas pour les requêtes en ligne. Dans un tel cas, une table de hachage est un moyen optimal d'effectuer des opérations de réduction. Une approche courante pour optimiser les tâches map-reduce est la pré-agrégation (réduction partielle) à l'aide d'une table de hachage en RAM. L'utilisateur effectue cette optimisation manuellement. Le tri distribué est l'une des principales causes de réduction des performances lors de l'exécution de tâches simples de réduction de la carte. -La plupart des implémentations MapReduce vous permettent d’exécuter du code arbitraire sur un cluster. Mais un langage de requête déclaratif est mieux adapté à OLAP pour exécuter des expériences rapidement. Par exemple, Hadoop a ruche et Cochon. Considérez également Cloudera Impala ou Shark (obsolète) pour Spark, ainsi que Spark SQL, Presto et Apache Drill. Les performances lors de l’exécution de telles tâches sont très sous-optimales par rapport aux systèmes spécialisés, mais une latence relativement élevée rend irréaliste l’utilisation de ces systèmes comme backend pour une interface web. +La plupart des implémentations MapReduce vous permettent d'exécuter du code arbitraire sur un cluster. Mais un langage de requête déclaratif est mieux adapté à OLAP pour exécuter des expériences rapidement. Par exemple, Hadoop a ruche et Cochon. Considérez également Cloudera Impala ou Shark (obsolète) pour Spark, ainsi que Spark SQL, Presto et Apache Drill. Les performances lors de l'exécution de telles tâches sont très sous-optimales par rapport aux systèmes spécialisés, mais une latence relativement élevée rend irréaliste l'utilisation de ces systèmes comme backend pour une interface web. -## Que Faire Si j’ai Un problème Avec Les Encodages Lors De l’utilisation D’Oracle Via ODBC? {#oracle-odbc-encodings} +## Que Faire si j'ai un problème avec les encodages lors de l'utilisation D'Oracle via ODBC? {#oracle-odbc-encodings} -Si vous utilisez Oracle via le pilote ODBC comme source de dictionnaires externes, vous devez définir la valeur correcte pour `NLS_LANG` variable d’environnement dans `/etc/default/clickhouse`. Pour plus d’informations, voir le [FAQ Oracle NLS\_LANG](https://www.oracle.com/technetwork/products/globalization/nls-lang-099431.html). +Si vous utilisez Oracle via le pilote ODBC comme source de dictionnaires externes, vous devez définir la valeur correcte pour `NLS_LANG` variable d'environnement dans `/etc/default/clickhouse`. Pour plus d'informations, voir le [FAQ Oracle NLS\_LANG](https://www.oracle.com/technetwork/products/globalization/nls-lang-099431.html). **Exemple** @@ -25,11 +25,11 @@ Si vous utilisez Oracle via le pilote ODBC comme source de dictionnaires externe NLS_LANG=RUSSIAN_RUSSIA.UTF8 ``` -## Comment Exporter Des données De ClickHouse Vers Un Fichier? {#how-to-export-to-file} +## Comment exporter des données de ClickHouse vers un fichier? {#how-to-export-to-file} -### Utilisation De La Clause INTO OUTFILE {#using-into-outfile-clause} +### Utilisation de la Clause INTO OUTFILE {#using-into-outfile-clause} -Ajouter un [INTO OUTFILE](../query_language/select/#into-outfile-clause) clause à votre requête. +Ajouter un [INTO OUTFILE](../sql-reference/statements/select/into-outfile.md#into-outfile-clause) clause à votre requête. Exemple: @@ -37,7 +37,7 @@ Exemple: SELECT * FROM table INTO OUTFILE 'file' ``` -Par défaut, ClickHouse utilise [TabSeparated](../interfaces/formats.md#tabseparated) format pour les données de sortie. Pour sélectionner le [format de données](../interfaces/formats.md), utiliser le [FORMAT de la clause](../query_language/select/#format-clause). +Par défaut, ClickHouse utilise [TabSeparated](../interfaces/formats.md#tabseparated) format pour les données de sortie. Pour sélectionner le [format de données](../interfaces/formats.md), utiliser le [FORMAT de la clause](../sql-reference/statements/select/format.md#format-clause). Exemple: @@ -45,7 +45,7 @@ Exemple: SELECT * FROM table INTO OUTFILE 'file' FORMAT CSV ``` -### Utilisation d’une Table De Moteur De Fichiers {#using-a-file-engine-table} +### Utilisation d'une Table de moteur de fichiers {#using-a-file-engine-table} Voir [Fichier](../engines/table-engines/special/file.md). diff --git a/docs/fr/faq/index.md b/docs/fr/faq/index.md index 062df2298ea..a44dbb31e89 100644 --- a/docs/fr/faq/index.md +++ b/docs/fr/faq/index.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_folder_title: F.A.Q. toc_priority: 76 --- diff --git a/docs/fr/getting-started/example-datasets/amplab-benchmark.md b/docs/fr/getting-started/example-datasets/amplab-benchmark.md index 6a5f8d53c72..066bd128ff6 100644 --- a/docs/fr/getting-started/example-datasets/amplab-benchmark.md +++ b/docs/fr/getting-started/example-datasets/amplab-benchmark.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 17 toc_title: AMPLab Big Data Benchmark --- @@ -9,7 +9,7 @@ toc_title: AMPLab Big Data Benchmark Tu vois https://amplab.cs.berkeley.edu/benchmark/ -Inscrivez-vous pour un compte GRATUIT à https://aws.amazon.com. il nécessite une carte de crédit, e-mail et numéro de téléphone. Obtenir une nouvelle clé d’accès à https://console.aws.amazon.com/iam/home?nc2=h\_m\_sc\#security\_credential +Inscrivez-vous pour un compte GRATUIT à https://aws.amazon.com. il nécessite une carte de crédit, e-mail et numéro de téléphone. Obtenir une nouvelle clé d'accès à https://console.aws.amazon.com/iam/home?nc2=h\_m\_sc\#security\_credential Exécutez ce qui suit dans la console: diff --git a/docs/fr/getting-started/example-datasets/criteo.md b/docs/fr/getting-started/example-datasets/criteo.md index 1ddc9da6313..8b52390cdcf 100644 --- a/docs/fr/getting-started/example-datasets/criteo.md +++ b/docs/fr/getting-started/example-datasets/criteo.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 19 toc_title: "T\xE9raoctet click Logs de Criteo" --- -# Téraoctet De Journaux De Clics De Criteo {#terabyte-of-click-logs-from-criteo} +# Téraoctet de journaux de clics de Criteo {#terabyte-of-click-logs-from-criteo} Télécharger les données à partir de http://labs.criteo.com/downloads/download-terabyte-click-logs/ diff --git a/docs/fr/getting-started/example-datasets/index.md b/docs/fr/getting-started/example-datasets/index.md index 93a8ca9c699..03b2f15890b 100644 --- a/docs/fr/getting-started/example-datasets/index.md +++ b/docs/fr/getting-started/example-datasets/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 -toc_folder_title: Example Datasets +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "Exemple De Jeux De Donn\xE9es" toc_priority: 12 toc_title: Introduction --- diff --git a/docs/fr/getting-started/example-datasets/metrica.md b/docs/fr/getting-started/example-datasets/metrica.md index e4b37429b67..6cc8f93b1c2 100644 --- a/docs/fr/getting-started/example-datasets/metrica.md +++ b/docs/fr/getting-started/example-datasets/metrica.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 -toc_priority: 21 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_priority: 14 toc_title: "Yandex.Metrica De Donn\xE9es" --- @@ -9,9 +9,9 @@ toc_title: "Yandex.Metrica De Donn\xE9es" Dataset se compose de deux tables contenant des données anonymisées sur les hits (`hits_v1`) et les visites (`visits_v1`) de Yandex.Metrica. Vous pouvez en savoir plus sur Yandex.Metrica dans [Histoire de ClickHouse](../../introduction/history.md) section. -L’ensemble de données se compose de deux tables, l’une d’elles peut être téléchargée sous forme compressée `tsv.xz` fichier ou comme partitions préparées. En outre, une version étendue de l’ `hits` table contenant 100 millions de lignes est disponible comme TSV à https://clickhouse-datasets.s3.yandex.net/hits/tsv/hits\_100m\_obfuscated\_v1.tsv.xz et comme partitions préparées à https://clickhouse-datasets.s3.yandex.net/hits/partitions/hits\_100m\_obfuscated\_v1.tar.xz. +L'ensemble de données se compose de deux tables, l'une d'elles peut être téléchargée sous forme compressée `tsv.xz` fichier ou comme partitions préparées. En outre, une version étendue de l' `hits` table contenant 100 millions de lignes est disponible comme TSV à https://clickhouse-datasets.s3.yandex.net/hits/tsv/hits\_100m\_obfuscated\_v1.tsv.xz et comme partitions préparées à https://clickhouse-datasets.s3.yandex.net/hits/partitions/hits\_100m\_obfuscated\_v1.tar.xz. -## Obtention De Tables à Partir De Partitions préparées {#obtaining-tables-from-prepared-partitions} +## Obtention de Tables à partir de Partitions préparées {#obtaining-tables-from-prepared-partitions} Télécharger et importer la table hits: @@ -33,7 +33,7 @@ sudo service clickhouse-server restart clickhouse-client --query "SELECT COUNT(*) FROM datasets.visits_v1" ``` -## Obtention De Tables à Partir D’un Fichier TSV compressé {#obtaining-tables-from-compressed-tsv-file} +## Obtention de Tables à partir D'un fichier TSV compressé {#obtaining-tables-from-compressed-tsv-file} Télécharger et importer des hits à partir du fichier TSV compressé: @@ -67,4 +67,4 @@ clickhouse-client --query "SELECT COUNT(*) FROM datasets.visits_v1" [Tutoriel ClickHouse](../../getting-started/tutorial.md) est basé sur Yandex.Metrica dataset et la façon recommandée pour commencer avec cet ensemble de données est de simplement passer par tutoriel. -D’autres exemples de requêtes pour ces tables peuvent être trouvés parmi [tests avec État](https://github.com/ClickHouse/ClickHouse/tree/master/tests/queries/1_stateful) de ClickHouse (ils sont nommés `test.hists` et `test.visits` y). +D'autres exemples de requêtes pour ces tables peuvent être trouvés parmi [tests avec État](https://github.com/ClickHouse/ClickHouse/tree/master/tests/queries/1_stateful) de ClickHouse (ils sont nommés `test.hists` et `test.visits` y). diff --git a/docs/fr/getting-started/example-datasets/nyc-taxi.md b/docs/fr/getting-started/example-datasets/nyc-taxi.md index 6ddc5f17190..92818ac9d3f 100644 --- a/docs/fr/getting-started/example-datasets/nyc-taxi.md +++ b/docs/fr/getting-started/example-datasets/nyc-taxi.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 16 toc_title: "New York Taxi Donn\xE9es" --- @@ -12,11 +12,11 @@ Ce jeu de données peut être obtenu de deux façons: - importation à partir de données brutes - téléchargement de partitions -## Comment Importer Les données Brutes {#how-to-import-the-raw-data} +## Comment importer les données brutes {#how-to-import-the-raw-data} -Tu vois https://github.com/toddwschneider/nyc-taxi-data et http://tech.marksblogg.com/billion-nyc-taxi-rides-redshift.html pour la description d’un ensemble de données et les instructions de téléchargement. +Tu vois https://github.com/toddwschneider/nyc-taxi-data et http://tech.marksblogg.com/billion-nyc-taxi-rides-redshift.html pour la description d'un ensemble de données et les instructions de téléchargement. -Le téléchargement entraînera environ 227 Go de données non compressées dans des fichiers CSV. Le téléchargement prend environ une heure sur une connexion 1 Gbit (téléchargement parallèle depuis s3.amazonaws.com récupère au moins la moitié d’un canal 1 Gbit). +Le téléchargement entraînera environ 227 Go de données non compressées dans des fichiers CSV. Le téléchargement prend environ une heure sur une connexion 1 Gbit (téléchargement parallèle depuis s3.amazonaws.com récupère au moins la moitié d'un canal 1 Gbit). Certains fichiers peuvent ne pas télécharger entièrement. Vérifiez la taille des fichiers et re-télécharger tout ce qui semble douteux. Certains fichiers peuvent contenir des lignes invalides. Vous pouvez les corriger comme suit: @@ -28,11 +28,11 @@ mv data/yellow_tripdata_2010-02.csv_ data/yellow_tripdata_2010-02.csv mv data/yellow_tripdata_2010-03.csv_ data/yellow_tripdata_2010-03.csv ``` -Ensuite, les données doivent être pré-traitées dans PostgreSQL. Cela créera des sélections de points dans les polygones (pour faire correspondre les points sur la carte avec les arrondissements de New York) et combinera toutes les données en une seule table plate dénormalisée à l’aide d’une jointure. Pour ce faire, vous devrez installer PostgreSQL avec le support PostGIS. +Ensuite, les données doivent être pré-traitées dans PostgreSQL. Cela créera des sélections de points dans les polygones (pour faire correspondre les points sur la carte avec les arrondissements de New York) et combinera toutes les données en une seule table plate dénormalisée à l'aide d'une jointure. Pour ce faire, vous devrez installer PostgreSQL avec le support PostGIS. -Soyez prudent lors de l’exécution `initialize_database.sh` et vérifiez à nouveau manuellement que toutes les tables ont été créées correctement. +Soyez prudent lors de l'exécution `initialize_database.sh` et vérifiez à nouveau manuellement que toutes les tables ont été créées correctement. -Il faut environ 20-30 minutes pour traiter la valeur de chaque mois de données dans PostgreSQL, pour un total d’environ 48 heures. +Il faut environ 20-30 minutes pour traiter la valeur de chaque mois de données dans PostgreSQL, pour un total d'environ 48 heures. Vous pouvez vérifier le nombre de téléchargé lignes comme suit: @@ -45,9 +45,9 @@ $ time psql nyc-taxi-data -c "SELECT count(*) FROM trips;" real 7m9.164s ``` -(C’est un peu plus de 1,1 milliard de lignes rapportées par Mark Litwintschik dans une série de billets de blog.) +(C'est un peu plus de 1,1 milliard de lignes rapportées par Mark Litwintschik dans une série de billets de blog.) -Les données de PostgreSQL utilisent 370 GO d’espace. +Les données de PostgreSQL utilisent 370 GO d'espace. Exportation des données depuis PostgreSQL: @@ -121,7 +121,7 @@ COPY ) TO '/opt/milovidov/nyc-taxi-data/trips.tsv'; ``` -L’instantané de données est créé à une vitesse d’environ 50 Mo par seconde. Lors de la création de l’instantané, PostgreSQL lit à partir du disque à une vitesse d’environ 28 Mo par seconde. +L'instantané de données est créé à une vitesse d'environ 50 Mo par seconde. Lors de la création de l'instantané, PostgreSQL lit à partir du disque à une vitesse d'environ 28 Mo par seconde. Cela prend environ 5 heures. Le fichier TSV résultant est 590612904969 octets. Créer une table temporaire dans ClickHouse: @@ -195,7 +195,7 @@ Les données sont lues à une vitesse de 112-140 Mo/seconde. Le chargement de données dans une table de type de journal dans un flux a pris 76 minutes. Les données de ce tableau utilisent 142 GO. -(L’importation de données directement depuis Postgres est également possible en utilisant `COPY ... TO PROGRAM`.) +(L'importation de données directement depuis Postgres est également possible en utilisant `COPY ... TO PROGRAM`.) Unfortunately, all the fields associated with the weather (precipitation…average\_wind\_speed) were filled with NULL. Because of this, we will remove them from the final data set. @@ -265,10 +265,10 @@ toUInt16(ifNull(dropoff_puma, '0')) AS dropoff_puma FROM trips ``` -Cela prend 3030 secondes à une vitesse d’environ 428 000 lignes par seconde. +Cela prend 3030 secondes à une vitesse d'environ 428 000 lignes par seconde. Pour le charger plus rapidement, vous pouvez créer la table avec le `Log` le moteur de `MergeTree`. Dans ce cas, le téléchargement fonctionne plus rapidement que 200 secondes. -La table utilise 126 GO d’espace disque. +La table utilise 126 GO d'espace disque. ``` sql SELECT formatReadableSize(sum(bytes)) FROM system.parts WHERE table = 'trips_mergetree' AND active @@ -280,9 +280,9 @@ SELECT formatReadableSize(sum(bytes)) FROM system.parts WHERE table = 'trips_mer └────────────────────────────────┘ ``` -Entre autres choses, vous pouvez exécuter la requête OPTIMIZE sur MergeTree. Mais ce n’est pas nécessaire puisque tout ira bien sans elle. +Entre autres choses, vous pouvez exécuter la requête OPTIMIZE sur MergeTree. Mais ce n'est pas nécessaire puisque tout ira bien sans elle. -## Téléchargement Des Partitions préparées {#download-of-prepared-partitions} +## Téléchargement des Partitions préparées {#download-of-prepared-partitions} ``` bash $ curl -O https://clickhouse-datasets.s3.yandex.net/trips_mergetree/partitions/trips_mergetree.tar @@ -295,7 +295,7 @@ $ clickhouse-client --query "select count(*) from datasets.trips_mergetree" !!! info "Info" Si vous exécutez les requêtes décrites ci-dessous, vous devez utiliser le nom complet de la table, `datasets.trips_mergetree`. -## Résultats Sur Un Seul Serveur {#results-on-single-server} +## Résultats sur un seul serveur {#results-on-single-server} Q1: @@ -336,9 +336,9 @@ Le serveur suivant a été utilisé: Deux Intel (R) Xeon (R) CPU E5-2650 v2 @ 2.60 GHz, 16 noyaux physiques total, 128 GiB RAM, 8x6 TB HD sur le matériel RAID-5 -Temps d’exécution est le meilleur des trois pistes. Mais à partir de la deuxième exécution, les requêtes lisent les données du cache du système de fichiers. Aucune autre mise en cache ne se produit: les données sont lues et traitées à chaque exécution. +Temps d'exécution est le meilleur des trois pistes. Mais à partir de la deuxième exécution, les requêtes lisent les données du cache du système de fichiers. Aucune autre mise en cache ne se produit: les données sont lues et traitées à chaque exécution. -La création d’un tableau sur trois serveurs: +La création d'un tableau sur trois serveurs: Sur chaque serveur: @@ -369,7 +369,7 @@ Q4: 1.241 secondes. Pas de surprise ici, depuis les requêtes sont réparties linéairement. -Nous avons également les résultats d’un cluster de 140 serveurs: +Nous avons également les résultats d'un cluster de 140 serveurs: Q1: 0,028 sec. Q2: 0,043 sec. diff --git a/docs/fr/getting-started/example-datasets/ontime.md b/docs/fr/getting-started/example-datasets/ontime.md index 668e2a04ac0..4040e2a1900 100644 --- a/docs/fr/getting-started/example-datasets/ontime.md +++ b/docs/fr/getting-started/example-datasets/ontime.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 15 toc_title: OnTime --- @@ -12,7 +12,7 @@ Ce jeu de données peut être obtenu de deux façons: - importation à partir de données brutes - téléchargement de partitions -## Importation À Partir De Données Brutes {#import-from-raw-data} +## Importation à partir de données brutes {#import-from-raw-data} Téléchargement de données: @@ -28,7 +28,7 @@ done (à partir de https://github.com/Percona-Lab/ontime-airline-performance/blob/master/download.sh ) -Création d’une table: +Création d'une table: ``` sql CREATE TABLE `ontime` ( @@ -153,7 +153,7 @@ Le chargement des données: $ for i in *.zip; do echo $i; unzip -cq $i '*.csv' | sed 's/\.00//g' | clickhouse-client --host=example-perftest01j --query="INSERT INTO ontime FORMAT CSVWithNames"; done ``` -## Téléchargement Des Partitions préparées {#download-of-prepared-partitions} +## Téléchargement des Partitions préparées {#download-of-prepared-partitions} ``` bash $ curl -O https://clickhouse-datasets.s3.yandex.net/ontime/partitions/ontime.tar @@ -180,7 +180,7 @@ FROM ); ``` -T1. Le nombre de vols par jour de l’année 2000 à 2008 +T1. Le nombre de vols par jour de l'année 2000 à 2008 ``` sql SELECT DayOfWeek, count(*) AS c @@ -200,7 +200,7 @@ GROUP BY DayOfWeek ORDER BY c DESC; ``` -T3. Le nombre de retards par l’aéroport pour 2000-2008 +T3. Le nombre de retards par l'aéroport pour 2000-2008 ``` sql SELECT Origin, count(*) AS c @@ -257,7 +257,7 @@ GROUP BY Carrier ORDER BY c3 DESC ``` -Q6. La demande précédente pour une plus large gamme d’années, 2000-2008 +Q6. La demande précédente pour une plus large gamme d'années, 2000-2008 ``` sql SELECT Carrier, c, c2, c*100/c2 as c3 @@ -326,7 +326,7 @@ GROUP BY Year ORDER BY Year; ``` -Q8. Les destinations les plus populaires par le nombre de villes directement connectées pour différentes plages d’années +Q8. Les destinations les plus populaires par le nombre de villes directement connectées pour différentes plages d'années ``` sql SELECT DestCityName, uniqExact(OriginCityName) AS u diff --git a/docs/fr/getting-started/example-datasets/star-schema.md b/docs/fr/getting-started/example-datasets/star-schema.md index f39d810623b..6a32faaa357 100644 --- a/docs/fr/getting-started/example-datasets/star-schema.md +++ b/docs/fr/getting-started/example-datasets/star-schema.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 20 toc_title: "R\xE9f\xE9rence Du Sch\xE9ma En \xC9toile" --- diff --git a/docs/fr/getting-started/example-datasets/wikistat.md b/docs/fr/getting-started/example-datasets/wikistat.md index 266d5c6da1c..cecd5ef0728 100644 --- a/docs/fr/getting-started/example-datasets/wikistat.md +++ b/docs/fr/getting-started/example-datasets/wikistat.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 18 toc_title: WikiStat --- @@ -9,7 +9,7 @@ toc_title: WikiStat Voir: http://dumps.wikimedia.org/other/pagecounts-raw/ -Création d’une table: +Création d'une table: ``` sql CREATE TABLE wikistat diff --git a/docs/fr/getting-started/index.md b/docs/fr/getting-started/index.md index ec0cf7da48b..3b90ff92b89 100644 --- a/docs/fr/getting-started/index.md +++ b/docs/fr/getting-started/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 -toc_folder_title: Getting Started +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: Prise En Main toc_hidden: true toc_priority: 8 toc_title: "cach\xE9s" @@ -9,7 +9,7 @@ toc_title: "cach\xE9s" # Prise En Main {#getting-started} -Si vous êtes nouveau à ClickHouse et que vous voulez obtenir un sentiment pratique de sa performance, tout d’abord, vous devez passer par le [processus d’installation](install.md). Après cela, vous pouvez: +Si vous êtes nouveau à ClickHouse et que vous voulez obtenir un sentiment pratique de sa performance, tout d'abord, vous devez passer par le [processus d'installation](install.md). Après cela, vous pouvez: - [Passez par tutoriel détaillé](tutorial.md) - [Expérience avec des exemples de jeux de données](example-datasets/ontime.md) diff --git a/docs/fr/getting-started/install.md b/docs/fr/getting-started/install.md index ea97b5a63de..770c4cf8e42 100644 --- a/docs/fr/getting-started/install.md +++ b/docs/fr/getting-started/install.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 11 toc_title: Installation --- @@ -9,9 +9,9 @@ toc_title: Installation ## Configuration Système Requise {#system-requirements} -ClickHouse peut fonctionner sur N’importe quel Linux, FreeBSD ou Mac OS X avec une architecture CPU x86\_64, AArch64 ou PowerPC64LE. +ClickHouse peut fonctionner sur N'importe quel Linux, FreeBSD ou Mac OS X avec une architecture CPU x86\_64, AArch64 ou PowerPC64LE. -Les binaires pré-construits officiels sont généralement compilés pour le jeu d’instructions x86\_64 et leverage SSE 4.2, donc sauf indication contraire, l’utilisation du processeur qui le prend en charge devient une exigence système supplémentaire. Voici la commande pour vérifier si le processeur actuel prend en charge SSE 4.2: +Les binaires pré-construits officiels sont généralement compilés pour le jeu d'instructions x86\_64 et leverage SSE 4.2, donc sauf indication contraire, l'utilisation du processeur qui le prend en charge devient une exigence système supplémentaire. Voici la commande pour vérifier si le processeur actuel prend en charge SSE 4.2: ``` bash $ grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not supported" @@ -19,13 +19,11 @@ $ grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not Pour exécuter ClickHouse sur des processeurs qui ne prennent pas en charge SSE 4.2 ou qui ont une architecture AArch64 ou PowerPC64LE, vous devez [construire ClickHouse à partir de sources](#from-sources) avec des ajustements de configuration appropriés. -## Options D’Installation Disponibles {#available-installation-options} +## Options D'Installation Disponibles {#available-installation-options} -### À Partir De Paquets DEB {#install-from-deb-packages} +### À partir de paquets DEB {#install-from-deb-packages} -Il est recommandé d’utiliser officiel pré-compilé `deb` Paquets Pour Debian ou Ubuntu. - -Exécutez ensuite ces commandes pour installer les paquets: +Il est recommandé d'utiliser officiel pré-compilé `deb` Paquets Pour Debian ou Ubuntu. Exécutez ces commandes pour installer les paquets: ``` bash {% include 'install/deb.sh' %} @@ -33,20 +31,20 @@ Exécutez ensuite ces commandes pour installer les paquets: Si vous souhaitez utiliser la version la plus récente, remplacer `stable` avec `testing` (ceci est recommandé pour vos environnements de test). -Vous pouvez également télécharger et installer des paquets manuellement à partir d’ici: https://repo.clickhouse.tech/deb/stable/main/. +Vous pouvez également télécharger et installer des paquets manuellement à partir de [ici](https://repo.clickhouse.tech/deb/stable/main/). #### Paquet {#packages} - `clickhouse-common-static` — Installs ClickHouse compiled binary files. - `clickhouse-server` — Creates a symbolic link for `clickhouse-server` et installe la configuration du serveur par défaut. -- `clickhouse-client` — Creates a symbolic link for `clickhouse-client` et d’autres outils. et installe les fichiers de configuration du client. +- `clickhouse-client` — Creates a symbolic link for `clickhouse-client` et d'autres outils. et installe les fichiers de configuration du client. - `clickhouse-common-static-dbg` — Installs ClickHouse compiled binary files with debug info. -### À Partir De Paquets RPM {#from-rpm-packages} +### À partir de paquets RPM {#from-rpm-packages} -Il est recommandé d’utiliser officiel pré-compilé `rpm` packages pour CentOS, RedHat et toutes les autres distributions Linux basées sur rpm. +Il est recommandé d'utiliser officiel pré-compilé `rpm` packages pour CentOS, RedHat et toutes les autres distributions Linux basées sur rpm. -Tout d’abord, vous devez ajouter le dépôt officiel: +Tout d'abord, vous devez ajouter le dépôt officiel: ``` bash sudo yum install yum-utils @@ -62,14 +60,14 @@ Exécutez ensuite ces commandes pour installer les paquets: sudo yum install clickhouse-server clickhouse-client ``` -Vous pouvez également télécharger et installer des paquets manuellement à partir d’ici: https://repo.clickhouse.tech / rpm / stable / x86\_64. +Vous pouvez également télécharger et installer des paquets manuellement à partir de [ici](https://repo.clickhouse.tech/rpm/stable/x86_64). -### À Partir d’archives Tgz {#from-tgz-archives} +### À Partir D'Archives Tgz {#from-tgz-archives} -Il est recommandé d’utiliser officiel pré-compilé `tgz` archives pour toutes les distributions Linux, où l’installation de `deb` ou `rpm` les emballages n’est pas possible. +Il est recommandé d'utiliser officiel pré-compilé `tgz` archives pour toutes les distributions Linux, où l'installation de `deb` ou `rpm` les emballages n'est pas possible. La version requise peut être téléchargée avec `curl` ou `wget` depuis le référentiel https://repo.yandex.ru/clickhouse/tgz/. -Après cela, les archives téléchargées doivent être décompressées et installées avec des scripts d’installation. Exemple pour la dernière version: +Après cela, les archives téléchargées doivent être décompressées et installées avec des scripts d'installation. Exemple pour la dernière version: ``` bash export LATEST_VERSION=`curl https://api.github.com/repos/ClickHouse/ClickHouse/tags 2>/dev/null | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -n 1` @@ -92,22 +90,22 @@ tar -xzvf clickhouse-client-$LATEST_VERSION.tgz sudo clickhouse-client-$LATEST_VERSION/install/doinst.sh ``` -Pour les environnements de production, il est recommandé d’utiliser la dernière `stable`-version. Vous pouvez trouver son numéro sur la page GitHub https://github.com/ClickHouse/ClickHouse/tags avec postfix `-stable`. +Pour les environnements de production, il est recommandé d'utiliser la dernière `stable`-version. Vous pouvez trouver son numéro sur la page GitHub https://github.com/ClickHouse/ClickHouse/tags avec postfix `-stable`. -### À Partir De L’Image Docker {#from-docker-image} +### À Partir De L'Image Docker {#from-docker-image} -Pour exécuter Clickhouse à L’intérieur Docker suivez le guide sur [Hub Docker](https://hub.docker.com/r/yandex/clickhouse-server/). Ces images utilisent officiel `deb` les paquets à l’intérieur. +Pour exécuter Clickhouse à L'intérieur Docker suivez le guide sur [Hub Docker](https://hub.docker.com/r/yandex/clickhouse-server/). Ces images utilisent officiel `deb` les paquets à l'intérieur. ### À Partir De Sources {#from-sources} Pour compiler manuellement ClickHouse, suivez les instructions pour [Linux](../development/build.md) ou [Mac OS X](../development/build-osx.md). -Vous pouvez compiler des paquets et les installer ou utiliser des programmes sans installer de paquets. En outre, en construisant manuellement, vous pouvez désactiver L’exigence SSE 4.2 ou construire pour les processeurs AArch64. +Vous pouvez compiler des paquets et les installer ou utiliser des programmes sans installer de paquets. En outre, en construisant manuellement, vous pouvez désactiver L'exigence SSE 4.2 ou construire pour les processeurs AArch64. Client: programs/clickhouse-client Server: programs/clickhouse-server -Vous devrez créer un dossier de données et de métadonnées et `chown` pour l’utilisateur souhaité. Leurs chemins peuvent être modifiés dans la configuration du serveur (src / programs / server / config.xml), par défaut, ils sont: +Vous devrez créer un dossier de données et de métadonnées et `chown` pour l'utilisateur souhaité. Leurs chemins peuvent être modifiés dans la configuration du serveur (src / programs / server / config.xml), par défaut, ils sont: /opt/clickhouse/data/default/ /opt/clickhouse/metadata/default/ @@ -122,7 +120,7 @@ Pour démarrer le serveur en tant que démon, exécutez: $ sudo service clickhouse-server start ``` -Si vous n’avez pas `service` commande, exécuter comme +Si vous n'avez pas `service` commande, exécuter comme ``` bash $ sudo /etc/init.d/clickhouse-server start @@ -139,11 +137,11 @@ $ clickhouse-server --config-file=/etc/clickhouse-server/config.xml ``` Dans ce cas, le journal sera imprimé sur la console, ce qui est pratique lors du développement. -Si le fichier de configuration se trouve dans le répertoire courant, vous n’avez pas besoin `--config-file` paramètre. Par défaut, il utilise `./config.xml`. +Si le fichier de configuration se trouve dans le répertoire courant, vous n'avez pas besoin `--config-file` paramètre. Par défaut, il utilise `./config.xml`. -ClickHouse prend en charge les paramètres de restriction d’accès. Ils sont situés dans la `users.xml` fichier (à côté de `config.xml`). -Par défaut, l’accès est autorisé depuis n’importe où pour `default` l’utilisateur, sans un mot de passe. Voir `user/default/networks`. -Pour plus d’informations, consultez la section [“Configuration Files”](../operations/configuration-files.md). +ClickHouse prend en charge les paramètres de restriction d'accès. Ils sont situés dans la `users.xml` fichier (à côté de `config.xml`). +Par défaut, l'accès est autorisé depuis n'importe où pour `default` l'utilisateur, sans un mot de passe. Voir `user/default/networks`. +Pour plus d'informations, consultez la section [“Configuration Files”](../operations/configuration-files.md). Après le lancement du serveur, vous pouvez utiliser le client de ligne de commande pour vous y connecter: @@ -151,10 +149,10 @@ Après le lancement du serveur, vous pouvez utiliser le client de ligne de comma $ clickhouse-client ``` -Par défaut, il se connecte à `localhost:9000` au nom de l’utilisateur `default` sans un mot de passe. Il peut également être utilisé pour se connecter à un serveur distant en utilisant `--host` argument. +Par défaut, il se connecte à `localhost:9000` au nom de l'utilisateur `default` sans un mot de passe. Il peut également être utilisé pour se connecter à un serveur distant en utilisant `--host` argument. -Le terminal doit utiliser L’encodage UTF-8. -Pour plus d’informations, consultez la section [“Command-line client”](../interfaces/cli.md). +Le terminal doit utiliser L'encodage UTF-8. +Pour plus d'informations, consultez la section [“Command-line client”](../interfaces/cli.md). Exemple: @@ -179,6 +177,6 @@ SELECT 1 **Félicitations, le système fonctionne!** -Pour continuer à expérimenter, vous pouvez télécharger l’un des jeux de données de test ou passer par [tutoriel](https://clickhouse.tech/tutorial.html). +Pour continuer à expérimenter, vous pouvez télécharger l'un des jeux de données de test ou passer par [tutoriel](https://clickhouse.tech/tutorial.html). [Article Original](https://clickhouse.tech/docs/en/getting_started/install/) diff --git a/docs/fr/getting-started/playground.md b/docs/fr/getting-started/playground.md index c68d2281ad2..6ddded13782 100644 --- a/docs/fr/getting-started/playground.md +++ b/docs/fr/getting-started/playground.md @@ -1,19 +1,19 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 14 toc_title: "R\xE9cr\xE9ation" --- # Clickhouse Aire De Jeux {#clickhouse-playground} -[Clickhouse Aire De Jeux](https://play.clickhouse.tech?file=welcome) permet aux utilisateurs d’expérimenter avec ClickHouse en exécutant des requêtes instantanément, sans configurer leur serveur ou leur cluster. +[Clickhouse Aire De Jeux](https://play.clickhouse.tech?file=welcome) permet aux utilisateurs d'expérimenter avec ClickHouse en exécutant des requêtes instantanément, sans configurer leur serveur ou leur cluster. Plusieurs exemples de jeux de données sont disponibles dans le terrain de jeu ainsi que des exemples de requêtes qui montrent les fonctionnalités de ClickHouse. Les requêtes sont exécutées comme un utilisateur en lecture seule. Cela implique certaines limites: - Les requêtes DDL ne sont pas autorisées -- Les requêtes D’insertion ne sont pas autorisées +- Les requêtes D'insertion ne sont pas autorisées Les paramètres suivants sont également appliquées: - [`max_result_bytes=10485760`](../operations/settings/query_complexity/#max-result-bytes) @@ -21,17 +21,17 @@ Les paramètres suivants sont également appliquées: - [`result_overflow_mode=break`](../operations/settings/query_complexity/#result-overflow-mode) - [`max_execution_time=60000`](../operations/settings/query_complexity/#max-execution-time) -Clickhouse Playground donne l’expérience du m2.Petite +Clickhouse Playground donne l'expérience du m2.Petite [Service géré pour ClickHouse](https://cloud.yandex.com/services/managed-clickhouse) exemple hébergé dans [Yandex.Nuage](https://cloud.yandex.com/). -Plus d’informations sur [les fournisseurs de cloud](../commercial/cloud.md). +Plus d'informations sur [les fournisseurs de cloud](../commercial/cloud.md). Clickhouse Playground interface web fait des demandes via ClickHouse [HTTP API](../interfaces/http.md). Le backend Playground est juste un cluster ClickHouse sans aucune application Côté Serveur supplémentaire. ClickHouse HTTPS endpoint est également disponible dans le cadre du terrain de jeu. -Vous pouvez effectuer des requêtes sur playground en utilisant n’importe quel client HTTP, par exemple [curl](https://curl.haxx.se) ou [wget](https://www.gnu.org/software/wget/), ou configurer une connexion en utilisant [JDBC](../interfaces/jdbc.md) ou [ODBC](../interfaces/odbc.md) pilote. -Plus d’informations sur les produits logiciels qui prennent en charge ClickHouse est disponible [ici](../interfaces/index.md). +Vous pouvez effectuer des requêtes sur playground en utilisant n'importe quel client HTTP, par exemple [curl](https://curl.haxx.se) ou [wget](https://www.gnu.org/software/wget/), ou configurer une connexion en utilisant [JDBC](../interfaces/jdbc.md) ou [ODBC](../interfaces/odbc.md) pilote. +Plus d'informations sur les produits logiciels qui prennent en charge ClickHouse est disponible [ici](../interfaces/index.md). | Paramètre | Valeur | |:-------------|:----------------------------------------------| diff --git a/docs/fr/getting-started/tutorial.md b/docs/fr/getting-started/tutorial.md index e8434310029..e8a0e8d81cb 100644 --- a/docs/fr/getting-started/tutorial.md +++ b/docs/fr/getting-started/tutorial.md @@ -1,19 +1,19 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 12 toc_title: Tutoriel --- # Tutoriel ClickHouse {#clickhouse-tutorial} -## À Quoi S’attendre De Ce Tutoriel? {#what-to-expect-from-this-tutorial} +## À quoi S'attendre de ce tutoriel? {#what-to-expect-from-this-tutorial} -En parcourant ce tutoriel, vous apprendrez à configurer un cluster ClickHouse simple. Ce sera petit, mais tolérant aux pannes et évolutif. Ensuite, nous utiliserons l’un des exemples de jeux de données pour le remplir de données et exécuter des requêtes de démonstration. +En parcourant ce tutoriel, vous apprendrez à configurer un cluster ClickHouse simple. Ce sera petit, mais tolérant aux pannes et évolutif. Ensuite, nous utiliserons l'un des exemples de jeux de données pour le remplir de données et exécuter des requêtes de démonstration. ## Configuration De Noeud Unique {#single-node-setup} -Pour retarder les complexités d’un environnement distribué, nous allons commencer par déployer ClickHouse sur un seul serveur ou une machine virtuelle. ClickHouse est généralement installé à partir de [deb](install.md#install-from-deb-packages) ou [rpm](install.md#from-rpm-packages) les paquets, mais il y a [alternative](install.md#from-docker-image) pour les systèmes d’exploitation qui ne sont pas les soutenir. +Pour retarder les complexités d'un environnement distribué, nous allons commencer par déployer ClickHouse sur un seul serveur ou une machine virtuelle. ClickHouse est généralement installé à partir de [deb](install.md#install-from-deb-packages) ou [tr / min](install.md#from-rpm-packages) les paquets, mais il y a [alternative](install.md#from-docker-image) pour les systèmes d'exploitation qui ne sont pas les soutenir. Par exemple, vous avez choisi `deb` paquets et exécutés: @@ -27,9 +27,9 @@ Quelles sont les paquets installés: - `clickhouse-common` paquet contient un fichier exécutable ClickHouse. - `clickhouse-server` package contient des fichiers de configuration pour exécuter ClickHouse en tant que serveur. -Les fichiers de configuration du serveur sont `/etc/clickhouse-server/`. Avant d’aller plus loin, notez le `` élément `config.xml`. Path détermine l’emplacement pour le stockage des données, il doit donc être situé sur le volume avec une grande capacité de disque; la valeur par défaut est `/var/lib/clickhouse/`. Si vous souhaitez ajuster la configuration, il n’est pas pratique de modifier directement `config.xml` fichier, considérant qu’il pourrait obtenir réécrit sur les futures mises à jour du progiciel. La façon recommandée de remplacer les éléments de configuration est de créer [fichiers dans config.d: répertoire](../operations/configuration-files.md) qui servent de “patches” config.XML. +Les fichiers de configuration du serveur sont `/etc/clickhouse-server/`. Avant d'aller plus loin, notez le `` élément `config.xml`. Path détermine l'emplacement pour le stockage des données, il doit donc être situé sur le volume avec une grande capacité de disque; la valeur par défaut est `/var/lib/clickhouse/`. Si vous souhaitez ajuster la configuration, il n'est pas pratique de modifier directement `config.xml` fichier, considérant qu'il pourrait obtenir réécrit sur les futures mises à jour du progiciel. La façon recommandée de remplacer les éléments de configuration est de créer [fichiers dans config.d: répertoire](../operations/configuration-files.md) qui servent de “patches” config.XML. -Comme vous l’avez peut-être remarqué, `clickhouse-server` n’est pas lancé automatiquement après l’installation du paquet. Il ne sera pas redémarré automatiquement après les mises à jour, non plus. La façon dont vous démarrez le serveur dépend de votre système d’initialisation, généralement, c’est: +Comme vous l'avez peut-être remarqué, `clickhouse-server` n'est pas lancé automatiquement après l'installation du paquet. Il ne sera pas redémarré automatiquement après les mises à jour, non plus. La façon dont vous démarrez le serveur dépend de votre système d'initialisation, généralement, c'est: ``` bash sudo service clickhouse-server start @@ -41,13 +41,14 @@ ou sudo /etc/init.d/clickhouse-server start ``` -L’emplacement par défaut pour les journaux du serveur est `/var/log/clickhouse-server/`. Le serveur est prêt à gérer les connexions client une fois `Ready for connections` message. +L'emplacement par défaut pour les journaux du serveur est `/var/log/clickhouse-server/`. Le serveur est prêt à gérer les connexions client une fois `Ready for connections` message. -Une fois l’ `clickhouse-server` est opérationnel, nous pouvons utiliser `clickhouse-client` pour se connecter au serveur et effectuer des tests de requêtes comme `SELECT "Hello, world!";`. +Une fois l' `clickhouse-server` est opérationnel, nous pouvons utiliser `clickhouse-client` pour se connecter au serveur et effectuer des tests de requêtes comme `SELECT "Hello, world!";`.
Conseils rapides pour clickhouse-client + Mode interactif: ``` bash @@ -70,7 +71,7 @@ echo 'SELECT 1' | clickhouse-client clickhouse-client <<< 'SELECT 1' ``` -Insérer des données à partir d’un fichier au format spécifié: +Insérer des données à partir d'un fichier au format spécifié: ``` bash clickhouse-client --query='INSERT INTO table VALUES' < data.txt @@ -81,16 +82,16 @@ clickhouse-client --query='INSERT INTO table FORMAT TabSeparated' < data.tsv ## Importer Un Échantillon De Données {#import-sample-dataset} -Maintenant, il est temps de remplir notre serveur ClickHouse avec quelques exemples de données. Dans ce tutoriel, nous allons utiliser les données anonymisées de Yandex.Metrica, le premier service qui exécute ClickHouse en production avant de devenir open-source (plus à ce sujet dans [section d’histoire](../introduction/history.md)). Il y a [plusieurs façons d’importer Yandex.Metrica dataset](example-datasets/metrica.md), et pour le bien du tutoriel, nous irons avec le plus réaliste. +Maintenant, il est temps de remplir notre serveur ClickHouse avec quelques exemples de données. Dans ce tutoriel, nous allons utiliser les données anonymisées de Yandex.Metrica, le premier service qui exécute ClickHouse en production avant de devenir open-source (plus à ce sujet dans [section d'histoire](../introduction/history.md)). Il y a [plusieurs façons d'importer Yandex.Metrica dataset](example-datasets/metrica.md), et pour le bien du tutoriel, nous irons avec le plus réaliste. -### Télécharger Et Extraire Les données De La Table {#download-and-extract-table-data} +### Télécharger et extraire les données de la Table {#download-and-extract-table-data} ``` bash curl https://clickhouse-datasets.s3.yandex.net/hits/tsv/hits_v1.tsv.xz | unxz --threads=`nproc` > hits_v1.tsv curl https://clickhouse-datasets.s3.yandex.net/visits/tsv/visits_v1.tsv.xz | unxz --threads=`nproc` > visits_v1.tsv ``` -Les fichiers extraits ont une taille d’environ 10 Go. +Les fichiers extraits ont une taille d'environ 10 Go. ### Créer Des Tables {#create-tables} @@ -104,12 +105,12 @@ La syntaxe pour créer des tables est beaucoup plus compliquée par rapport aux 1. Nom de la table à créer. 2. Table schema, i.e. list of columns and their [types de données](../sql-reference/data-types/index.md). -3. [Tableau moteur](../engines/table-engines/index.md) et ce sont les paramètres, qui déterminent tous les détails sur la façon dont les requêtes à cette table seront physiquement exécutées. +3. [Tableau moteur](../engines/table-engines/index.md) et ses paramètres, qui détermine tous les détails sur la façon dont les requêtes à cette table seront physiquement exécutées. -Yandex.Metrica est un service d’analyse web, et l’exemple de jeu de données ne couvre pas toutes ses fonctionnalités, il n’y a donc que deux tables à créer: +Yandex.Metrica est un service d'analyse web, et l'exemple de jeu de données ne couvre pas toutes ses fonctionnalités, il n'y a donc que deux tables à créer: - `hits` est un tableau avec chaque action effectuée par tous les utilisateurs sur tous les sites couverts par le service. -- `visits` est une table qui contient des sessions pré-construites au lieu d’actions individuelles. +- `visits` est une table qui contient des sessions pré-construites au lieu d'actions individuelles. Voyons et exécutons les vraies requêtes create table pour ces tables: @@ -454,13 +455,13 @@ SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192 ``` -Vous pouvez exécuter ces requêtes en utilisant le mode interactif de `clickhouse-client` (lancez - le simplement dans un terminal sans spécifier une requête à l’avance) ou essayez-en [interface de rechange](../interfaces/index.md) Si tu veux. +Vous pouvez exécuter ces requêtes en utilisant le mode interactif de `clickhouse-client` (lancez - le simplement dans un terminal sans spécifier une requête à l'avance) ou essayez-en [interface de rechange](../interfaces/index.md) Si tu veux. Comme nous pouvons le voir, `hits_v1` utilise la [moteur MergeTree de base](../engines/table-engines/mergetree-family/mergetree.md) tandis que le `visits_v1` utilise la [Effondrer](../engines/table-engines/mergetree-family/collapsingmergetree.md) variante. ### Importer Des Données {#import-data} -L’importation de données vers ClickHouse se fait via [INSERT INTO](../sql-reference/statements/insert-into.md) requête comme dans de nombreuses autres bases de données SQL. Toutefois, les données sont généralement fournies dans l’une des [formats de sérialisation pris en charge](../interfaces/formats.md) plutôt `VALUES` clause (qui est également pris en charge). +L'importation de données vers ClickHouse se fait via [INSERT INTO](../sql-reference/statements/insert-into.md) requête comme dans de nombreuses autres bases de données SQL. Toutefois, les données sont généralement fournies dans l'une des [formats de sérialisation pris en charge](../interfaces/formats.md) plutôt `VALUES` clause (qui est également pris en charge). Les fichiers que nous avons téléchargés plus tôt sont au format séparé par des onglets, alors voici comment les importer via le client console: @@ -469,7 +470,7 @@ clickhouse-client --query "INSERT INTO tutorial.hits_v1 FORMAT TSV" --max_insert clickhouse-client --query "INSERT INTO tutorial.visits_v1 FORMAT TSV" --max_insert_block_size=100000 < visits_v1.tsv ``` -ClickHouse a beaucoup de [les paramètres de tune](../operations/settings/index.md) et une façon de Les spécifier dans le client console est via des arguments, comme nous pouvons le voir avec `--max_insert_block_size`. La façon la plus simple de comprendre quels paramètres sont disponibles, que signifient-ils et quelles sont les valeurs par défaut est d’interroger le `system.settings` table: +ClickHouse a beaucoup de [les paramètres de tune](../operations/settings/index.md) et une façon de Les spécifier dans le client console est via des arguments, comme nous pouvons le voir avec `--max_insert_block_size`. La façon la plus simple de comprendre quels paramètres sont disponibles, que signifient-ils et quelles sont les valeurs par défaut est d'interroger le `system.settings` table: ``` sql SELECT name, value, changed, description @@ -480,16 +481,16 @@ FORMAT TSV max_insert_block_size 1048576 0 "The maximum block size for insertion, if we control the creation of blocks for insertion." ``` -Optionnellement, vous pouvez [OPTIMIZE](../query_language/misc/#misc_operations-optimize) les tables après l’importation. Les Tables configurées avec un moteur de MergeTree-family font toujours des fusions de parties de données en arrière-plan pour optimiser le stockage des données (ou au moins vérifier si cela a du sens). Ces requêtes forcent le moteur de table à optimiser le stockage dès maintenant au lieu d’un certain temps plus tard: +Optionnellement, vous pouvez [OPTIMIZE](../sql-reference/statements/misc.md#misc_operations-optimize) les tables après l'importation. Les Tables configurées avec un moteur de MergeTree-family font toujours des fusions de parties de données en arrière-plan pour optimiser le stockage des données (ou au moins vérifier si cela a du sens). Ces requêtes forcent le moteur de table à optimiser le stockage dès maintenant au lieu d'un certain temps plus tard: ``` bash clickhouse-client --query "OPTIMIZE TABLE tutorial.hits_v1 FINAL" clickhouse-client --query "OPTIMIZE TABLE tutorial.visits_v1 FINAL" ``` -Ces requêtes démarrent une opération intensive D’E/S et de CPU, donc si la table reçoit systématiquement de nouvelles données, il est préférable de la laisser seule et de laisser les fusions s’exécuter en arrière-plan. +Ces requêtes démarrent une opération intensive D'E/S et de CPU, donc si la table reçoit systématiquement de nouvelles données, il est préférable de la laisser seule et de laisser les fusions s'exécuter en arrière-plan. -Maintenant, nous pouvons vérifier si l’importation de table a réussi: +Maintenant, nous pouvons vérifier si l'importation de table a réussi: ``` bash clickhouse-client --query "SELECT COUNT(*) FROM tutorial.hits_v1" @@ -527,7 +528,7 @@ Clickhouse cluster est un cluster homogène. Étapes pour configurer: 3. Créer des tables locales sur chaque instance 4. Créer un [Distribué table](../engines/table-engines/special/distributed.md) -[Distribué table](../engines/table-engines/special/distributed.md) est en fait une sorte de “view” aux tables locales du cluster ClickHouse. SELECT query from a distributed table s’exécute à l’aide des ressources de tous les fragments du cluster. Vous pouvez spécifier des configurations pour plusieurs clusters et créer plusieurs tables distribuées fournissant des vues à différents clusters. +[Distribué table](../engines/table-engines/special/distributed.md) est en fait une sorte de “view” aux tables locales du cluster ClickHouse. SELECT query from a distributed table s'exécute à l'aide des ressources de tous les fragments du cluster. Vous pouvez spécifier des configurations pour plusieurs clusters et créer plusieurs tables distribuées fournissant des vues à différents clusters. Exemple de configuration pour un cluster avec trois fragments, une réplique chacun: @@ -562,16 +563,16 @@ Pour plus de démonstration, créons une nouvelle table locale avec le même `CR CREATE TABLE tutorial.hits_local (...) ENGINE = MergeTree() ... ``` -Création d’une table distribuée fournissant une vue dans les tables locales du cluster: +Création d'une table distribuée fournissant une vue dans les tables locales du cluster: ``` sql CREATE TABLE tutorial.hits_all AS tutorial.hits_local ENGINE = Distributed(perftest_3shards_1replicas, tutorial, hits_local, rand()); ``` -Une pratique courante consiste à créer des tables distribuées similaires sur toutes les machines du cluster. Il permet d’exécuter des requêtes distribuées sur n’importe quelle machine du cluster. Il existe également une autre option pour créer une table distribuée temporaire pour une requête SELECT donnée en utilisant [distant](../sql-reference/table-functions/remote.md) table de fonction. +Une pratique courante consiste à créer des tables distribuées similaires sur toutes les machines du cluster. Il permet d'exécuter des requêtes distribuées sur n'importe quelle machine du cluster. Il existe également une autre option pour créer une table distribuée temporaire pour une requête SELECT donnée en utilisant [distant](../sql-reference/table-functions/remote.md) table de fonction. -Passons à l’exécution de [INSERT SELECT](../sql-reference/statements/insert-into.md) dans les Distribué table la table à plusieurs serveurs. +Passons à l'exécution de [INSERT SELECT](../sql-reference/statements/insert-into.md) dans les Distribué table la table à plusieurs serveurs. ``` sql INSERT INTO tutorial.hits_all SELECT * FROM tutorial.hits_v1; @@ -580,13 +581,13 @@ INSERT INTO tutorial.hits_all SELECT * FROM tutorial.hits_v1; !!! warning "Avis" Cette approche ne convient pas au sharding de grandes tables. Il y a un outil séparé [clickhouse-copieur](../operations/utilities/clickhouse-copier.md) cela peut re-fragmenter de grandes tables arbitraires. -Comme vous pouvez vous y attendre, les requêtes lourdes de calcul s’exécutent N fois plus vite si elles utilisent 3 serveurs au lieu d’un. +Comme vous pouvez vous y attendre, les requêtes lourdes de calcul s'exécutent N fois plus vite si elles utilisent 3 serveurs au lieu d'un. Dans ce cas, nous avons utilisé un cluster avec 3 fragments, et chacun contient une seule réplique. Pour assurer la résilience dans un environnement de production, nous recommandons que chaque fragment contienne 2-3 répliques réparties entre plusieurs zones de disponibilité ou centres de données (ou au moins des racks). Notez que ClickHouse prend en charge un nombre illimité de répliques. -Exemple de configuration pour un cluster d’un fragment contenant trois répliques: +Exemple de configuration pour un cluster d'un fragment contenant trois répliques: ``` xml @@ -610,10 +611,10 @@ Exemple de configuration pour un cluster d’un fragment contenant trois répliq ``` -Pour activer la réplication native [ZooKeeper](http://zookeeper.apache.org/) est requis. ClickHouse s’occupe de la cohérence des données sur toutes les répliques et exécute automatiquement la procédure de restauration après l’échec. Il est recommandé de déployer le cluster ZooKeeper sur des serveurs séparés (où aucun autre processus, y compris ClickHouse, n’est en cours d’exécution). +Pour activer la réplication native [ZooKeeper](http://zookeeper.apache.org/) est requis. ClickHouse s'occupe de la cohérence des données sur toutes les répliques et exécute automatiquement la procédure de restauration après l'échec. Il est recommandé de déployer le cluster ZooKeeper sur des serveurs séparés (où aucun autre processus, y compris ClickHouse, n'est en cours d'exécution). !!! note "Note" - ZooKeeper est pas une exigence stricte: dans certains cas simples, vous pouvez dupliquer les données par écrit dans tous les réplicas de votre code d’application. Cette approche est **pas** recommandé, dans ce cas, ClickHouse ne sera pas en mesure de garantir la cohérence des données sur toutes les répliques. Ainsi, il devient la responsabilité de votre application. + ZooKeeper est pas une exigence stricte: dans certains cas simples, vous pouvez dupliquer les données par écrit dans tous les réplicas de votre code d'application. Cette approche est **pas** recommandé, dans ce cas, ClickHouse ne sera pas en mesure de garantir la cohérence des données sur toutes les répliques. Ainsi, il devient la responsabilité de votre application. Les emplacements ZooKeeper sont spécifiés dans le fichier de configuration: @@ -643,7 +644,7 @@ En outre, nous devons définir des macros pour identifier chaque fragment et cha ``` -S’il n’y a pas de répliques pour le moment lors de la création de la table répliquée, une nouvelle première réplique est instanciée. S’il existe déjà des répliques en direct, la nouvelle réplique clone les données de celles existantes. Vous avez la possibilité de créer toutes les tables répliquées d’abord, et ensuite insérer les données. Une autre option consiste à créer des répliques et à en ajouter d’autres Après ou pendant l’insertion des données. +S'il n'y a pas de répliques pour le moment lors de la création de la table répliquée, une nouvelle première réplique est instanciée. S'il existe déjà des répliques en direct, la nouvelle réplique clone les données de celles existantes. Vous avez la possibilité de créer toutes les tables répliquées d'abord, et ensuite insérer les données. Une autre option consiste à créer des répliques et à en ajouter d'autres Après ou pendant l'insertion des données. ``` sql CREATE TABLE tutorial.hits_replica (...) @@ -660,6 +661,6 @@ Ici, nous utilisons [ReplicatedMergeTree](../engines/table-engines/mergetree-fam INSERT INTO tutorial.hits_replica SELECT * FROM tutorial.hits_local; ``` -La réplication fonctionne en mode multi-maître. Les données peuvent être chargées dans n’importe quel réplica, et le système les synchronise ensuite automatiquement avec d’autres instances. La réplication est asynchrone, donc à un moment donné, toutes les répliques ne peuvent pas contenir de données récemment insérées. Au moins une réplique devrait être en place pour permettre l’ingestion de données. D’autres synchroniseront les données et répareront la cohérence une fois qu’ils redeviendront actifs. Notez que cette approche permet une faible possibilité de perte de données récemment insérées. +La réplication fonctionne en mode multi-maître. Les données peuvent être chargées dans n'importe quel réplica, et le système les synchronise ensuite automatiquement avec d'autres instances. La réplication est asynchrone, donc à un moment donné, toutes les répliques ne peuvent pas contenir de données récemment insérées. Au moins une réplique devrait être en place pour permettre l'ingestion de données. D'autres synchroniseront les données et répareront la cohérence une fois qu'ils redeviendront actifs. Notez que cette approche permet une faible possibilité de perte de données récemment insérées. [Article Original](https://clickhouse.tech/docs/en/getting_started/tutorial/) diff --git a/docs/fr/guides/apply-catboost-model.md b/docs/fr/guides/apply-catboost-model.md index 05293c451ad..1b6aae9bbbd 100644 --- a/docs/fr/guides/apply-catboost-model.md +++ b/docs/fr/guides/apply-catboost-model.md @@ -1,33 +1,33 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 41 toc_title: "Application Des Mod\xE8les CatBoost" --- -# Application D’un modèle Catboost Dans ClickHouse {#applying-catboost-model-in-clickhouse} +# Application D'un modèle Catboost dans ClickHouse {#applying-catboost-model-in-clickhouse} -[CatBoost](https://catboost.ai) est une bibliothèque de dynamisation de gradient libre et open-source développée à [Yandex](https://yandex.com/company/) pour l’apprentissage automatique. +[CatBoost](https://catboost.ai) est une bibliothèque de dynamisation de gradient libre et open-source développée à [Yandex](https://yandex.com/company/) pour l'apprentissage automatique. -Avec cette instruction, vous apprendrez à appliquer des modèles pré-formés dans ClickHouse en exécutant l’inférence de modèle à partir de SQL. +Avec cette instruction, vous apprendrez à appliquer des modèles pré-formés dans ClickHouse en exécutant l'inférence de modèle à partir de SQL. Pour appliquer un modèle CatBoost dans ClickHouse: 1. [Créer une Table](#create-table). 2. [Insérez les données dans la Table](#insert-data-to-table). 3. [Intégrer CatBoost dans ClickHouse](#integrate-catboost-into-clickhouse) (Étape facultative). -4. [Exécutez L’inférence du modèle à partir de SQL](#run-model-inference). +4. [Exécutez L'inférence du modèle à partir de SQL](#run-model-inference). -Pour plus d’informations sur la formation des modèles CatBoost, voir [Formation et application de modèles](https://catboost.ai/docs/features/training.html#training). +Pour plus d'informations sur la formation des modèles CatBoost, voir [Formation et application de modèles](https://catboost.ai/docs/features/training.html#training). ## Préalable {#prerequisites} -Si vous n’avez pas le [Docker](https://docs.docker.com/install/) pourtant, l’installer. +Si vous n'avez pas le [Docker](https://docs.docker.com/install/) pourtant, l'installer. !!! note "Note" [Docker](https://www.docker.com) est une plate-forme logicielle qui vous permet de créer des conteneurs qui isolent une installation CatBoost et ClickHouse du reste du système. -Avant d’appliquer un modèle CatBoost: +Avant d'appliquer un modèle CatBoost: **1.** Tirez la [Docker image](https://hub.docker.com/r/yandex/tutorial-catboost-clickhouse) à partir du registre: @@ -35,9 +35,9 @@ Avant d’appliquer un modèle CatBoost: $ docker pull yandex/tutorial-catboost-clickhouse ``` -Cette image Docker contient tout ce dont vous avez besoin pour exécuter CatBoost et ClickHouse: code, runtime, bibliothèques, variables d’environnement et fichiers de configuration. +Cette image Docker contient tout ce dont vous avez besoin pour exécuter CatBoost et ClickHouse: code, runtime, bibliothèques, variables d'environnement et fichiers de configuration. -**2.** Assurez-vous que l’image Docker a été tirée avec succès: +**2.** Assurez-vous que l'image Docker a été tirée avec succès: ``` bash $ docker image ls @@ -53,7 +53,7 @@ $ docker run -it -p 8888:8888 yandex/tutorial-catboost-clickhouse ## 1. Créer une Table {#create-table} -Pour créer une table ClickHouse pour l’exemple de formation: +Pour créer une table ClickHouse pour l'exemple de formation: **1.** Démarrez clickhouse console client en mode interactif: @@ -62,9 +62,9 @@ $ clickhouse client ``` !!! note "Note" - Le serveur ClickHouse est déjà en cours d’exécution dans le conteneur Docker. + Le serveur ClickHouse est déjà en cours d'exécution dans le conteneur Docker. -**2.** Créer la table à l’aide de la commande: +**2.** Créer la table à l'aide de la commande: ``` sql :) CREATE TABLE amazon_train @@ -122,19 +122,19 @@ FROM amazon_train ## 3. Intégrer CatBoost dans ClickHouse {#integrate-catboost-into-clickhouse} !!! note "Note" - **Étape facultative.** L’image Docker contient tout ce dont vous avez besoin pour exécuter CatBoost et ClickHouse. + **Étape facultative.** L'image Docker contient tout ce dont vous avez besoin pour exécuter CatBoost et ClickHouse. Pour intégrer CatBoost dans ClickHouse: -**1.** Construire la bibliothèque d’évaluation. +**1.** Construire la bibliothèque d'évaluation. -Le moyen le plus rapide d’évaluer un modèle CatBoost est la compilation `libcatboostmodel.` bibliothèque. Pour plus d’informations sur la création de la bibliothèque, voir [Documentation CatBoost](https://catboost.ai/docs/concepts/c-plus-plus-api_dynamic-c-pluplus-wrapper.html). +Le moyen le plus rapide d'évaluer un modèle CatBoost est la compilation `libcatboostmodel.` bibliothèque. Pour plus d'informations sur la création de la bibliothèque, voir [Documentation CatBoost](https://catboost.ai/docs/concepts/c-plus-plus-api_dynamic-c-pluplus-wrapper.html). -**2.** Créez un nouveau répertoire n’importe où et avec n’importe quel nom, par exemple, `data` et mettez la bibliothèque créée dedans. L’image Docker contient déjà la bibliothèque `data/libcatboostmodel.so`. +**2.** Créez un nouveau répertoire n'importe où et avec n'importe quel nom, par exemple, `data` et mettez la bibliothèque créée dedans. L'image Docker contient déjà la bibliothèque `data/libcatboostmodel.so`. -**3.** Créez un nouveau répertoire pour le modèle de configuration n’importe où et avec n’importe quel nom, par exemple, `models`. +**3.** Créez un nouveau répertoire pour le modèle de configuration n'importe où et avec n'importe quel nom, par exemple, `models`. -**4.** Créez un fichier de configuration de modèle avec n’importe quel nom, par exemple, `models/amazon_model.xml`. +**4.** Créez un fichier de configuration de modèle avec n'importe quel nom, par exemple, `models/amazon_model.xml`. **5.** Décrire la configuration du modèle: @@ -153,7 +153,7 @@ Le moyen le plus rapide d’évaluer un modèle CatBoost est la compilation `lib ``` -**6.** Ajoutez le chemin D’accès à CatBoost et la configuration du modèle à la configuration de ClickHouse: +**6.** Ajoutez le chemin D'accès à CatBoost et la configuration du modèle à la configuration de ClickHouse: ``` xml @@ -161,7 +161,7 @@ Le moyen le plus rapide d’évaluer un modèle CatBoost est la compilation `lib /home/catboost/models/*_model.xml ``` -## 4. Exécutez L’inférence du modèle à partir de SQL {#run-model-inference} +## 4. Exécutez L'inférence du modèle à partir de SQL {#run-model-inference} Pour le modèle de test exécutez le client ClickHouse `$ clickhouse client`. @@ -208,9 +208,9 @@ LIMIT 10 ``` !!! note "Note" - Plus d’infos sur [exp()](../sql-reference/functions/math-functions.md) fonction. + Plus d'infos sur [exp()](../sql-reference/functions/math-functions.md) fonction. -Calculons LogLoss sur l’échantillon: +Calculons LogLoss sur l'échantillon: ``` sql :) SELECT -avg(tg * log(prob) + (1 - tg) * log(1 - prob)) AS logloss @@ -234,6 +234,6 @@ FROM ``` !!! note "Note" - Plus d’infos sur [avg()](../sql-reference/aggregate-functions/reference.md#agg_function-avg) et [journal()](../sql-reference/functions/math-functions.md) fonction. + Plus d'infos sur [avg()](../sql-reference/aggregate-functions/reference.md#agg_function-avg) et [journal()](../sql-reference/functions/math-functions.md) fonction. [Article Original](https://clickhouse.tech/docs/en/guides/apply_catboost_model/) diff --git a/docs/fr/guides/index.md b/docs/fr/guides/index.md index f33c0cccb6f..5f6552fdb45 100644 --- a/docs/fr/guides/index.md +++ b/docs/fr/guides/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 -toc_folder_title: Guides +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: Guide toc_priority: 38 toc_title: "Aper\xE7u" --- @@ -11,6 +11,6 @@ toc_title: "Aper\xE7u" Liste des instructions détaillées étape par étape qui aident à résoudre diverses tâches en utilisant ClickHouse: - [Tutoriel sur la configuration simple du cluster](../getting-started/tutorial.md) -- [Application D’un modèle CatBoost dans ClickHouse](apply-catboost-model.md) +- [Application D'un modèle CatBoost dans ClickHouse](apply-catboost-model.md) [Article Original](https://clickhouse.tech/docs/en/guides/) diff --git a/docs/fr/index.md b/docs/fr/index.md index 8d29d755185..fe714af8d49 100644 --- a/docs/fr/index.md +++ b/docs/fr/index.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 -toc_priority: 3 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_priority: 0 toc_title: "Aper\xE7u" --- -# Qu’est-ce Que ClickHouse? {#what-is-clickhouse} +# Qu'Est-Ce Que ClickHouse? {#what-is-clickhouse} ClickHouse est un système de gestion de base de données orienté colonne (SGBD) pour le traitement analytique en ligne des requêtes (OLAP). @@ -18,9 +18,9 @@ Dans un “normal” SGBD orienté ligne, les données sont stockées dans cet o | \#2 | 89953706054 | 1 | Mission | 1 | 2016-05-18 07:38:00 | | \#N | … | … | … | … | … | -En d’autres termes, toutes les valeurs liées à une ligne sont physiquement stockées l’une à côté de l’autre. +En d'autres termes, toutes les valeurs liées à une ligne sont physiquement stockées l'une à côté de l'autre. -Des exemples d’un SGBD orienté ligne sont MySQL, Postgres et MS SQL Server. +Des exemples d'un SGBD orienté ligne sont MySQL, Postgres et MS SQL Server. Dans un SGBD orienté colonne, les données sont stockées comme ceci: @@ -32,33 +32,33 @@ Dans un SGBD orienté colonne, les données sont stockées comme ceci: | GoodEvent: | 1 | 1 | 1 | … | | EventTime: | 2016-05-18 05:19:20 | 2016-05-18 08:10:20 | 2016-05-18 07:38:00 | … | -Ces exemples montrent l’ordre que les données sont organisées en. Les valeurs de différentes colonnes sont stockés séparément, et les données de la même colonne sont stockées ensemble. +Ces exemples montrent l'ordre que les données sont organisées en. Les valeurs de différentes colonnes sont stockés séparément, et les données de la même colonne sont stockées ensemble. -Exemples D’un SGBD orienté colonne: Vertica, Paraccel (matrice Actian et Amazon Redshift), Sybase IQ, Exasol, Infobright, InfiniDB, MonetDB (VectorWise et Actian Vector), LucidDB, SAP HANA, Google Dremel, Google PowerDrill, Druid et kdb+. +Exemples D'un SGBD orienté colonne: Vertica, Paraccel (matrice Actian et Amazon Redshift), Sybase IQ, Exasol, Infobright, InfiniDB, MonetDB (VectorWise et Actian Vector), LucidDB, SAP HANA, Google Dremel, Google PowerDrill, Druid et kdb+. Different orders for storing data are better suited to different scenarios. The data access scenario refers to what queries are made, how often, and in what proportion; how much data is read for each type of query – rows, columns, and bytes; the relationship between reading and updating data; the working size of the data and how locally it is used; whether transactions are used, and how isolated they are; requirements for data replication and logical integrity; requirements for latency and throughput for each type of query, and so on. -Plus la charge sur le système est élevée, plus il est important de personnaliser le système configuré pour correspondre aux exigences du scénario d’utilisation, et plus cette personnalisation devient fine. Il n’y a pas de système qui soit aussi bien adapté à des scénarios significativement différents. Si un système est adaptable à un large ensemble de scénarios, sous une charge élevée, le système traitera tous les scénarios de manière également médiocre, ou fonctionnera bien pour un ou quelques-uns des scénarios possibles. +Plus la charge sur le système est élevée, plus il est important de personnaliser le système configuré pour correspondre aux exigences du scénario d'utilisation, et plus cette personnalisation devient fine. Il n'y a pas de système qui soit aussi bien adapté à des scénarios significativement différents. Si un système est adaptable à un large ensemble de scénarios, sous une charge élevée, le système traitera tous les scénarios de manière également médiocre, ou fonctionnera bien pour un ou quelques-uns des scénarios possibles. -## Propriétés clés Du scénario OLAP {#key-properties-of-olap-scenario} +## Propriétés clés du scénario OLAP {#key-properties-of-olap-scenario} -- La grande majorité des demandes concernent l’accès en lecture. +- La grande majorité des demandes concernent l'accès en lecture. - Les données sont mises à jour en lots assez importants (\> 1000 lignes), pas par des lignes simples; ou elles ne sont pas mises à jour du tout. - Les données sont ajoutées à la base de données mais ne sont pas modifiées. - Pour les lectures, un assez grand nombre de lignes sont extraites de la base de données, mais seulement un petit sous-ensemble de colonnes. -- Les Tables sont “wide,” ce qui signifie qu’ils contiennent un grand nombre de colonnes. +- Les Tables sont “wide,” ce qui signifie qu'ils contiennent un grand nombre de colonnes. - Les requêtes sont relativement rares (généralement des centaines de requêtes par serveur ou moins par seconde). - Pour les requêtes simples, les latences autour de 50 ms sont autorisées. - Les valeurs de colonne sont assez petites: nombres et chaînes courtes (par exemple, 60 octets par URL). -- Nécessite un débit élevé lors du traitement d’une seule requête (jusqu’à des milliards de lignes par seconde par serveur). +- Nécessite un débit élevé lors du traitement d'une seule requête (jusqu'à des milliards de lignes par seconde par serveur). - Les Transactions ne sont pas nécessaires. - Faibles exigences en matière de cohérence des données. - Il y a une grande table par requête. Toutes les tables sont petites, sauf une. -- Un résultat de requête est significativement plus petit que les données source. En d’autres termes, les données sont filtrées ou agrégées, de sorte que le résultat s’intègre dans la RAM d’un seul serveur. +- Un résultat de requête est significativement plus petit que les données source. En d'autres termes, les données sont filtrées ou agrégées, de sorte que le résultat s'intègre dans la RAM d'un seul serveur. -Il est facile de voir que le scénario OLAP est très différent des autres scénarios populaires (tels que OLTP ou key-Value access). Il n’est donc pas logique d’essayer D’utiliser OLTP ou une base de données clé-valeur pour traiter les requêtes analytiques si vous voulez obtenir des performances décentes. Par exemple, si vous essayez D’utiliser MongoDB ou Redis pour l’analyse, vous obtiendrez des performances très médiocres par rapport aux bases de données OLAP. +Il est facile de voir que le scénario OLAP est très différent des autres scénarios populaires (tels que OLTP ou key-Value access). Il n'est donc pas logique d'essayer D'utiliser OLTP ou une base de données clé-valeur pour traiter les requêtes analytiques si vous voulez obtenir des performances décentes. Par exemple, si vous essayez D'utiliser MongoDB ou Redis pour l'analyse, vous obtiendrez des performances très médiocres par rapport aux bases de données OLAP. -## Pourquoi Les Bases De données orientées Colonne Fonctionnent Mieux Dans Le scénario OLAP {#why-column-oriented-databases-work-better-in-the-olap-scenario} +## Pourquoi les bases de données orientées colonne fonctionnent mieux dans le scénario OLAP {#why-column-oriented-databases-work-better-in-the-olap-scenario} Les bases de données orientées colonne sont mieux adaptées aux scénarios OLAP: elles sont au moins 100 fois plus rapides dans le traitement de la plupart des requêtes. Les raisons sont expliquées en détail ci-dessous, mais le fait est plus facile de démontrer visuellement: @@ -72,26 +72,26 @@ Les bases de données orientées colonne sont mieux adaptées aux scénarios OLA Vous voyez la différence? -### D’entrée/sortie {#inputoutput} +### D'entrée/sortie {#inputoutput} 1. Pour une requête analytique, seul un petit nombre de colonnes de table doit être lu. Dans une base de données orientée colonne, vous pouvez lire uniquement les données dont vous avez besoin. Par exemple, si vous avez besoin de 5 colonnes sur 100, Vous pouvez vous attendre à une réduction de 20 fois des e / s. -2. Puisque les données sont lues en paquets, il est plus facile de les compresser. Les données dans les colonnes sont également plus faciles à compresser. Cela réduit d’autant le volume d’e/S. -3. En raison de la réduction des E / S, Plus de données s’insèrent dans le cache du système. +2. Puisque les données sont lues en paquets, il est plus facile de les compresser. Les données dans les colonnes sont également plus faciles à compresser. Cela réduit d'autant le volume d'e/S. +3. En raison de la réduction des E / S, Plus de données s'insèrent dans le cache du système. -Par exemple, la requête “count the number of records for each advertising platform” nécessite la lecture d’un “advertising platform ID” colonne, qui prend 1 octet non compressé. Si la majeure partie du trafic ne provenait pas de plates-formes publicitaires, vous pouvez vous attendre à une compression d’au moins 10 fois de cette colonne. Lors de l’utilisation d’un algorithme de compression rapide, la décompression des données est possible à une vitesse d’au moins plusieurs gigaoctets de données non compressées par seconde. En d’autres termes, cette requête ne peut être traitée qu’à une vitesse d’environ plusieurs milliards de lignes par seconde sur un seul serveur. Cette vitesse est effectivement atteinte dans la pratique. +Par exemple, la requête “count the number of records for each advertising platform” nécessite la lecture d'un “advertising platform ID” colonne, qui prend 1 octet non compressé. Si la majeure partie du trafic ne provenait pas de plates-formes publicitaires, vous pouvez vous attendre à une compression d'au moins 10 fois de cette colonne. Lors de l'utilisation d'un algorithme de compression rapide, la décompression des données est possible à une vitesse d'au moins plusieurs gigaoctets de données non compressées par seconde. En d'autres termes, cette requête ne peut être traitée qu'à une vitesse d'environ plusieurs milliards de lignes par seconde sur un seul serveur. Cette vitesse est effectivement atteinte dans la pratique. ### CPU {#cpu} -Étant donné que l’exécution d’une requête nécessite le traitement d’un grand nombre de lignes, il est utile de répartir toutes les opérations pour des vecteurs entiers au lieu de lignes séparées, ou d’implémenter le moteur de requête de sorte qu’il n’y ait presque aucun coût d’expédition. Si vous ne le faites pas, avec un sous-système de disque à moitié décent, l’interpréteur de requête bloque inévitablement le processeur. Il est logique de stocker des données dans des colonnes et de les traiter, si possible, par des colonnes. +Étant donné que l'exécution d'une requête nécessite le traitement d'un grand nombre de lignes, il est utile de répartir toutes les opérations pour des vecteurs entiers au lieu de lignes séparées, ou d'implémenter le moteur de requête de sorte qu'il n'y ait presque aucun coût d'expédition. Si vous ne le faites pas, avec un sous-système de disque à moitié décent, l'interpréteur de requête bloque inévitablement le processeur. Il est logique de stocker des données dans des colonnes et de les traiter, si possible, par des colonnes. Il y a deux façons de le faire: -1. Un moteur vectoriel. Toutes les opérations sont écrites pour les vecteurs, au lieu de valeurs séparées. Cela signifie que vous n’avez pas besoin d’appeler les opérations très souvent, et les coûts d’expédition sont négligeables. Le code d’opération contient un cycle interne optimisé. +1. Un moteur vectoriel. Toutes les opérations sont écrites pour les vecteurs, au lieu de valeurs séparées. Cela signifie que vous n'avez pas besoin d'appeler les opérations très souvent, et les coûts d'expédition sont négligeables. Le code d'opération contient un cycle interne optimisé. 2. La génération de Code. Le code généré pour la requête contient tous les appels indirects. -Ce n’est pas fait dans “normal” bases de données, car cela n’a pas de sens lors de l’exécution de requêtes simples. Cependant, il y a des exceptions. Par exemple, MemSQL utilise la génération de code pour réduire la latence lors du traitement des requêtes SQL. (À titre de comparaison, les SGBD analytiques nécessitent une optimisation du débit, et non une latence.) +Ce n'est pas fait dans “normal” bases de données, car cela n'a pas de sens lors de l'exécution de requêtes simples. Cependant, il y a des exceptions. Par exemple, MemSQL utilise la génération de code pour réduire la latence lors du traitement des requêtes SQL. (À titre de comparaison, les SGBD analytiques nécessitent une optimisation du débit, et non une latence.) -Notez que pour l’efficacité du processeur, le langage de requête doit être déclaratif (SQL ou MDX), ou au moins un vecteur (J, K). La requête ne doit contenir que des boucles implicites, permettant une optimisation. +Notez que pour l'efficacité du processeur, le langage de requête doit être déclaratif (SQL ou MDX), ou au moins un vecteur (J, K). La requête ne doit contenir que des boucles implicites, permettant une optimisation. {## [Article Original](https://clickhouse.tech/docs/en/) ##} diff --git a/docs/fr/interfaces/cli.md b/docs/fr/interfaces/cli.md index 69c7d8ba0b1..5b8ee67c31a 100644 --- a/docs/fr/interfaces/cli.md +++ b/docs/fr/interfaces/cli.md @@ -1,15 +1,15 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 17 toc_title: Client De Ligne De Commande --- -# Client De Ligne De Commande {#command-line-client} +# Client de ligne de commande {#command-line-client} -ClickHouse fournit un client de ligne de commande natif: `clickhouse-client`. Le client prend en charge les options de ligne de commande et les fichiers de configuration. Pour plus d’informations, voir [Configuration](#interfaces_cli_configuration). +ClickHouse fournit un client de ligne de commande natif: `clickhouse-client`. Le client prend en charge les options de ligne de commande et les fichiers de configuration. Pour plus d'informations, voir [Configuration](#interfaces_cli_configuration). -[Installer](../getting-started/index.md) à partir de la `clickhouse-client` package et l’exécuter avec la commande `clickhouse-client`. +[Installer](../getting-started/index.md) à partir de la `clickhouse-client` package et l'exécuter avec la commande `clickhouse-client`. ``` bash $ clickhouse-client @@ -20,15 +20,15 @@ Connected to ClickHouse server version 19.17.1 revision 54428. :) ``` -Différentes versions client et serveur sont compatibles les unes avec les autres, mais certaines fonctionnalités peuvent ne pas être disponibles dans les anciens clients. Nous vous recommandons d’utiliser la même version du client que l’application serveur. Lorsque vous essayez d’utiliser un client de l’ancienne version, puis le serveur, `clickhouse-client` affiche le message: +Différentes versions client et serveur sont compatibles les unes avec les autres, mais certaines fonctionnalités peuvent ne pas être disponibles dans les anciens clients. Nous vous recommandons d'utiliser la même version du client que l'application serveur. Lorsque vous essayez d'utiliser un client de l'ancienne version, puis le serveur, `clickhouse-client` affiche le message: ClickHouse client version is older than ClickHouse server. It may lack support for new features. ## Utilisation {#cli_usage} -Le client peut être utilisé en mode interactif et non interactif (batch). Pour utiliser le mode batch, spécifiez ‘query’ d’un paramètre ou d’envoyer des données à ‘stdin’ (il vérifie que ‘stdin’ n’est pas un terminal), ou les deux. Similaire à L’interface HTTP, lors de l’utilisation ‘query’ paramètre et envoi de données à ‘stdin’ la demande est une concaténation de la ‘query’ paramètre, un saut de ligne, et les données de ‘stdin’. Ceci est pratique pour les grandes requêtes D’insertion. +Le client peut être utilisé en mode interactif et non interactif (batch). Pour utiliser le mode batch, spécifiez ‘query’ d'un paramètre ou d'envoyer des données à ‘stdin’ (il vérifie que ‘stdin’ n'est pas un terminal), ou les deux. Similaire à L'interface HTTP, lors de l'utilisation ‘query’ paramètre et envoi de données à ‘stdin’ la demande est une concaténation de la ‘query’ paramètre, un saut de ligne, et les données de ‘stdin’. Ceci est pratique pour les grandes requêtes D'insertion. -Exemple d’utilisation du client pour insérer des données: +Exemple d'utilisation du client pour insérer des données: ``` bash $ echo -ne "1, 'some text', '2016-08-14 00:00:00'\n2, 'some more text', '2016-08-14 00:00:01'" | clickhouse-client --database=test --query="INSERT INTO test FORMAT CSV"; @@ -43,38 +43,38 @@ $ cat file.csv | clickhouse-client --database=test --query="INSERT INTO test FOR En mode batch, le format de données par défaut est TabSeparated. Vous pouvez définir le format dans la clause FORMAT de la requête. -Par défaut, vous ne pouvez traiter une seule requête en mode batch. De faire plusieurs requêtes à partir d’un “script,” l’utilisation de la `--multiquery` paramètre. Cela fonctionne pour toutes les requêtes sauf INSERT. Les résultats de la requête sont produits consécutivement sans séparateurs supplémentaires. De même, pour traiter un grand nombre de requêtes, vous pouvez exécuter ‘clickhouse-client’ pour chaque requête. Notez que cela peut prendre des dizaines de millisecondes pour lancer le ‘clickhouse-client’ programme. +Par défaut, vous ne pouvez traiter une seule requête en mode batch. De faire plusieurs requêtes à partir d'un “script,” l'utilisation de la `--multiquery` paramètre. Cela fonctionne pour toutes les requêtes sauf INSERT. Les résultats de la requête sont produits consécutivement sans séparateurs supplémentaires. De même, pour traiter un grand nombre de requêtes, vous pouvez exécuter ‘clickhouse-client’ pour chaque requête. Notez que cela peut prendre des dizaines de millisecondes pour lancer le ‘clickhouse-client’ programme. En mode interactif, vous obtenez une ligne de commande où vous pouvez entrer des requêtes. -Si ‘multiline’ n’est pas spécifié (par défaut): Pour exécuter la requête, appuyez sur Entrée. Le point-virgule n’est pas nécessaire à la fin de la requête. Pour entrer une requête multiligne, entrez une barre oblique inverse `\` avant le saut de ligne. Après avoir appuyé sur Entrée, il vous sera demandé d’entrer la ligne suivante de la requête. +Si ‘multiline’ n'est pas spécifié (par défaut): Pour exécuter la requête, appuyez sur Entrée. Le point-virgule n'est pas nécessaire à la fin de la requête. Pour entrer une requête multiligne, entrez une barre oblique inverse `\` avant le saut de ligne. Après avoir appuyé sur Entrée, il vous sera demandé d'entrer la ligne suivante de la requête. -Si multiline est spécifié: pour exécuter une requête, terminez-la par un point-virgule et appuyez sur Entrée. Si le point-virgule a été omis à la fin de l’entrée ligne, vous serez invité à entrer la ligne suivante de la requête. +Si multiline est spécifié: pour exécuter une requête, terminez-la par un point-virgule et appuyez sur Entrée. Si le point-virgule a été omis à la fin de l'entrée ligne, vous serez invité à entrer la ligne suivante de la requête. Une seule requête est exécutée, donc tout après le point-virgule est ignoré. Vous pouvez spécifier `\G` au lieu de le ou après le point-virgule. Cela indique le format Vertical. Dans ce format, chaque valeur est imprimée sur une ligne distincte, ce qui est pratique pour les grandes tables. Cette fonctionnalité inhabituelle a été ajoutée pour la compatibilité avec la CLI MySQL. -La ligne de commande est basé sur ‘replxx’ (similaire à ‘readline’). En d’autres termes, il utilise les raccourcis clavier familiers et conserve un historique. L’histoire est écrite à `~/.clickhouse-client-history`. +La ligne de commande est basé sur ‘replxx’ (similaire à ‘readline’). En d'autres termes, il utilise les raccourcis clavier familiers et conserve un historique. L'histoire est écrite à `~/.clickhouse-client-history`. Par défaut, le format utilisé est Jolicompact. Vous pouvez modifier le format dans la clause FORMAT de la requête, ou en spécifiant `\G` à la fin de la requête, en utilisant le `--format` ou `--vertical` dans la ligne de commande, ou en utilisant le fichier de configuration client. -Pour quitter le client, appuyez sur Ctrl + D (ou Ctrl + C), ou entrez l’une des options suivantes au lieu d’une requête: “exit”, “quit”, “logout”, “exit;”, “quit;”, “logout;”, “q”, “Q”, “:q” +Pour quitter le client, appuyez sur Ctrl + D (ou Ctrl + C), ou entrez l'une des options suivantes au lieu d'une requête: “exit”, “quit”, “logout”, “exit;”, “quit;”, “logout;”, “q”, “Q”, “:q” -Lors du traitement d’une requête, le client affiche: +Lors du traitement d'une requête, le client affiche: -1. Progrès, qui est mis à jour pas plus de 10 fois par seconde (par défaut). Pour les requêtes rapides, la progression peut ne pas avoir le temps d’être affichée. -2. La requête formatée après l’analyse, pour le débogage. +1. Progrès, qui est mis à jour pas plus de 10 fois par seconde (par défaut). Pour les requêtes rapides, la progression peut ne pas avoir le temps d'être affichée. +2. La requête formatée après l'analyse, pour le débogage. 3. Le résultat dans le format spécifié. 4. Le nombre de lignes dans le résultat, le temps passé et la vitesse moyenne de traitement des requêtes. -Vous pouvez annuler une requête longue en appuyant sur Ctrl + C. Cependant, vous devrez toujours attendre un peu pour que le serveur abandonne la requête. Il n’est pas possible d’annuler une requête à certaines étapes. Si vous n’attendez pas et appuyez une seconde fois sur Ctrl+C, le client quittera. +Vous pouvez annuler une requête longue en appuyant sur Ctrl + C. Cependant, vous devrez toujours attendre un peu pour que le serveur abandonne la requête. Il n'est pas possible d'annuler une requête à certaines étapes. Si vous n'attendez pas et appuyez une seconde fois sur Ctrl+C, le client quittera. -Le client de ligne de commande permet de passer des données externes (tables temporaires externes) pour l’interrogation. Pour plus d’informations, consultez la section “External data for query processing”. +Le client de ligne de commande permet de passer des données externes (tables temporaires externes) pour l'interrogation. Pour plus d'informations, consultez la section “External data for query processing”. -### Requêtes Avec paramètres {#cli-queries-with-parameters} +### Requêtes avec paramètres {#cli-queries-with-parameters} -Vous pouvez créer une requête avec des paramètres et leur transmettre des valeurs à partir de l’application cliente. Cela permet d’éviter le formatage de la requête avec des valeurs dynamiques spécifiques côté client. Exemple: +Vous pouvez créer une requête avec des paramètres et leur transmettre des valeurs à partir de l'application cliente. Cela permet d'éviter le formatage de la requête avec des valeurs dynamiques spécifiques côté client. Exemple: ``` bash $ clickhouse-client --param_parName="[1, 2]" -q "SELECT * FROM table WHERE a = {parName:Array(UInt16)}" @@ -82,14 +82,14 @@ $ clickhouse-client --param_parName="[1, 2]" -q "SELECT * FROM table WHERE a = #### La Syntaxe De La Requête {#cli-queries-with-parameters-syntax} -Formatez une requête comme d’habitude, puis placez les valeurs que vous souhaitez transmettre des paramètres de l’application à la requête entre accolades au format suivant: +Formatez une requête comme d'habitude, puis placez les valeurs que vous souhaitez transmettre des paramètres de l'application à la requête entre accolades au format suivant: ``` sql {:} ``` - `name` — Placeholder identifier. In the console client it should be used in app parameters as `--param_ = value`. -- `data type` — [Type de données](../sql-reference/data-types/index.md) de l’application valeur de paramètre. Par exemple, une structure de données comme `(integer, ('string', integer))` peut avoir la `Tuple(UInt8, Tuple(String, UInt8))` type de données (vous pouvez également utiliser un autre [entier](../sql-reference/data-types/int-uint.md) type). +- `data type` — [Type de données](../sql-reference/data-types/index.md) de l'application valeur de paramètre. Par exemple, une structure de données comme `(integer, ('string', integer))` peut avoir la `Tuple(UInt8, Tuple(String, UInt8))` type de données (vous pouvez également utiliser un autre [entier](../sql-reference/data-types/int-uint.md) type). #### Exemple {#example} @@ -111,7 +111,7 @@ Vous pouvez passer des paramètres à `clickhouse-client` (tous les paramètres ### Options De Ligne De Commande {#command-line-options} -- `--host, -h` -– The server name, ‘localhost’ par défaut. Vous pouvez utiliser le nom ou L’adresse IPv4 ou IPv6. +- `--host, -h` -– The server name, ‘localhost’ par défaut. Vous pouvez utiliser le nom ou L'adresse IPv4 ou IPv6. - `--port` – The port to connect to. Default value: 9000. Note that the HTTP interface and the native interface use different ports. - `--user, -u` – The username. Default value: default. - `--password` – The password. Default value: empty string. @@ -120,7 +120,7 @@ Vous pouvez passer des paramètres à `clickhouse-client` (tous les paramètres - `--multiline, -m` – If specified, allow multiline queries (do not send the query on Enter). - `--multiquery, -n` – If specified, allow processing multiple queries separated by semicolons. - `--format, -f` – Use the specified default format to output the result. -- `--vertical, -E` – If specified, use the Vertical format by default to output the result. This is the same as ‘–format=Vertical’. Dans ce format, chaque valeur est imprimée sur une ligne séparée, ce qui est utile lors de l’affichage de tables larges. +- `--vertical, -E` – If specified, use the Vertical format by default to output the result. This is the same as ‘–format=Vertical’. Dans ce format, chaque valeur est imprimée sur une ligne séparée, ce qui est utile lors de l'affichage de tables larges. - `--time, -t` – If specified, print the query execution time to ‘stderr’ en mode non interactif. - `--stacktrace` – If specified, also print the stack trace if an exception occurs. - `--config-file` – The name of the configuration file. diff --git a/docs/fr/interfaces/cpp.md b/docs/fr/interfaces/cpp.md index 32009fc42c0..0451b0722aa 100644 --- a/docs/fr/interfaces/cpp.md +++ b/docs/fr/interfaces/cpp.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 24 toc_title: "Biblioth\xE8que Client C++ " --- -# Bibliothèque Client C++ {#c-client-library} +# Bibliothèque Client C++ {#c-client-library} Voir le fichier README à [clickhouse-cpp](https://github.com/ClickHouse/clickhouse-cpp) référentiel. diff --git a/docs/fr/interfaces/formats.md b/docs/fr/interfaces/formats.md index a97eaf3f372..13419cf6c35 100644 --- a/docs/fr/interfaces/formats.md +++ b/docs/fr/interfaces/formats.md @@ -1,14 +1,14 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 21 toc_title: "Formats d'entr\xE9e et de sortie" --- -# Formats Pour Les données D’entrée Et De Sortie {#formats} +# Formats pour les données D'entrée et de sortie {#formats} -ClickHouse peut accepter et renvoyer des données dans différents formats. Un format pris en charge pour l’entrée peut être utilisée pour analyser les données fournies à `INSERT`s, pour effectuer `SELECT`s à partir d’une table sauvegardée par fichier, telle Qu’un fichier, une URL ou un HDFS, ou pour lire un dictionnaire externe. Un format pris en charge pour la sortie peut être utilisé pour organiser -les résultats d’une `SELECT` et pour effectuer `INSERT`s dans une table sauvegardée par fichier. +ClickHouse peut accepter et renvoyer des données dans différents formats. Un format pris en charge pour l'entrée peut être utilisée pour analyser les données fournies à `INSERT`s, pour effectuer `SELECT`s à partir d'une table sauvegardée par fichier, telle Qu'un fichier, une URL ou un HDFS, ou pour lire un dictionnaire externe. Un format pris en charge pour la sortie peut être utilisé pour organiser +les résultats d'une `SELECT` et pour effectuer `INSERT`s dans une table sauvegardée par fichier. Les formats pris en charge sont: @@ -47,17 +47,17 @@ Les formats pris en charge sont: | [XML](#xml) | ✗ | ✔ | | [CapnProto](#capnproto) | ✔ | ✗ | -Vous pouvez contrôler certains paramètres de traitement de format avec les paramètres ClickHouse. Pour plus d’informations, lisez le [Paramètre](../operations/settings/settings.md) section. +Vous pouvez contrôler certains paramètres de traitement de format avec les paramètres ClickHouse. Pour plus d'informations, lisez le [Paramètre](../operations/settings/settings.md) section. ## TabSeparated {#tabseparated} -Au format TabSeparated, les données sont écrites par ligne. Chaque ligne contient des valeurs séparées par des onglets. Chaque valeur est suivie par un onglet, à l’exception de la dernière valeur dans la ligne, qui est suivi par un saut de ligne. Les flux de ligne strictement Unix sont supposés partout. La dernière ligne doit également contenir un saut de ligne à la fin. Les valeurs sont écrites au format texte, sans guillemets et avec des caractères spéciaux échappés. +Au format TabSeparated, les données sont écrites par ligne. Chaque ligne contient des valeurs séparées par des onglets. Chaque valeur est suivie par un onglet, à l'exception de la dernière valeur dans la ligne, qui est suivi par un saut de ligne. Les flux de ligne strictement Unix sont supposés partout. La dernière ligne doit également contenir un saut de ligne à la fin. Les valeurs sont écrites au format texte, sans guillemets et avec des caractères spéciaux échappés. Ce format est également disponible sous le nom de `TSV`. -Le `TabSeparated` le format est pratique pour le traitement des données à l’aide de programmes et de scripts personnalisés. Il est utilisé par défaut dans L’interface HTTP et dans le mode batch du client de ligne de commande. Ce format permet également de transférer des données entre différents SGBD. Par exemple, vous pouvez obtenir un dump de MySQL et le télécharger sur ClickHouse, ou vice versa. +Le `TabSeparated` le format est pratique pour le traitement des données à l'aide de programmes et de scripts personnalisés. Il est utilisé par défaut dans L'interface HTTP et dans le mode batch du client de ligne de commande. Ce format permet également de transférer des données entre différents SGBD. Par exemple, vous pouvez obtenir un dump de MySQL et le télécharger sur ClickHouse, ou vice versa. -Le `TabSeparated` format prend en charge la sortie des valeurs totales (lors de L’utilisation avec des totaux) et des valeurs extrêmes (lorsque ‘extremes’ est réglé sur 1). Dans ces cas, les valeurs totales et les extrêmes sont sorties après les données principales. Le résultat principal, les valeurs totales et les extrêmes sont séparés les uns des autres par une ligne vide. Exemple: +Le `TabSeparated` format prend en charge la sortie des valeurs totales (lors de L'utilisation avec des totaux) et des valeurs extrêmes (lorsque ‘extremes’ est réglé sur 1). Dans ces cas, les valeurs totales et les extrêmes sont sorties après les données principales. Le résultat principal, les valeurs totales et les extrêmes sont séparés les uns des autres par une ligne vide. Exemple: ``` sql SELECT EventDate, count() AS c FROM test.hits GROUP BY EventDate WITH TOTALS ORDER BY EventDate FORMAT TabSeparated`` @@ -80,20 +80,20 @@ SELECT EventDate, count() AS c FROM test.hits GROUP BY EventDate WITH TOTALS ORD ### Mise En Forme Des Données {#data-formatting} -Les nombres entiers sont écrits sous forme décimale. Les numéros peuvent contenir un supplément “+” caractère au début (ignoré lors de l’analyse, et non enregistré lors du formatage). Les nombres non négatifs ne peuvent pas contenir le signe négatif. Lors de la lecture, il est permis d’analyser une chaîne vide en tant que zéro, ou (pour les types signés) une chaîne composée d’un signe moins en tant que zéro. Les nombres qui ne correspondent pas au type de données correspondant peuvent être analysés comme un nombre différent, sans message d’erreur. +Les nombres entiers sont écrits sous forme décimale. Les numéros peuvent contenir un supplément “+” caractère au début (ignoré lors de l'analyse, et non enregistré lors du formatage). Les nombres non négatifs ne peuvent pas contenir le signe négatif. Lors de la lecture, il est permis d'analyser une chaîne vide en tant que zéro, ou (pour les types signés) une chaîne composée d'un signe moins en tant que zéro. Les nombres qui ne correspondent pas au type de données correspondant peuvent être analysés comme un nombre différent, sans message d'erreur. Les nombres à virgule flottante sont écrits sous forme décimale. Le point est utilisé comme séparateur décimal. Les entrées exponentielles sont prises en charge, tout comme ‘inf’, ‘+inf’, ‘-inf’, et ‘nan’. Une entrée de nombres à virgule flottante peut commencer ou se terminer par une virgule décimale. Pendant le formatage, la précision peut être perdue sur les nombres à virgule flottante. -Pendant l’analyse, il n’est pas strictement nécessaire de lire le nombre représentable de la machine le plus proche. +Pendant l'analyse, il n'est pas strictement nécessaire de lire le nombre représentable de la machine le plus proche. Les Dates sont écrites au format AAAA-MM-JJ et analysées dans le même format, mais avec tous les caractères comme séparateurs. Les Dates avec les heures sont écrites dans le format `YYYY-MM-DD hh:mm:ss` et analysé dans le même format, mais avec des caractères comme séparateurs. -Tout cela se produit dans le fuseau horaire du système au moment où le client ou le serveur démarre (selon lequel d’entre eux formate les données). Pour les dates avec des heures, l’heure d’été n’est pas spécifiée. Donc, si un vidage a des temps pendant l’heure d’été, le vidage ne correspond pas sans équivoque aux données, et l’analyse sélectionnera l’une des deux fois. -Lors d’une opération de lecture, les dates et dates incorrectes avec des heures peuvent être analysées avec un débordement naturel ou en tant que dates et heures nulles, sans message d’erreur. +Tout cela se produit dans le fuseau horaire du système au moment où le client ou le serveur démarre (selon lequel d'entre eux formate les données). Pour les dates avec des heures, l'heure d'été n'est pas spécifiée. Donc, si un vidage a des temps pendant l'heure d'été, le vidage ne correspond pas sans équivoque aux données, et l'analyse sélectionnera l'une des deux fois. +Lors d'une opération de lecture, les dates et dates incorrectes avec des heures peuvent être analysées avec un débordement naturel ou en tant que dates et heures nulles, sans message d'erreur. -À titre d’exception, l’analyse des dates avec des heures est également prise en charge au format d’horodatage Unix, si elle se compose exactement de 10 chiffres décimaux. Le résultat n’est pas dépendant du fuseau horaire. Les formats AAAA-MM-JJ hh:mm:ss et NNNNNNNNNN sont différenciés automatiquement. +À titre d'exception, l'analyse des dates avec des heures est également prise en charge au format d'horodatage Unix, si elle se compose exactement de 10 chiffres décimaux. Le résultat n'est pas dépendant du fuseau horaire. Les formats AAAA-MM-JJ hh:mm:ss et NNNNNNNNNN sont différenciés automatiquement. -Les chaînes sont sorties avec des caractères spéciaux échappés par une barre oblique inverse. Les séquences d’échappement suivantes sont utilisées pour la sortie: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\'`, `\\`. L’analyse prend également en charge les séquences `\a`, `\v`, et `\xHH` (séquences d’échappement hexadécimales) et `\c` séquences, où `c` est un caractère (ces séquences sont convertis à `c`). Ainsi, la lecture de données prend en charge les formats où un saut de ligne peut être écrit comme `\n` ou `\` ou comme un saut de ligne. Par exemple, la chaîne `Hello world` avec un saut de ligne entre les mots au lieu de l’espace peut être analysé dans l’une des variantes suivantes: +Les chaînes sont sorties avec des caractères spéciaux échappés par une barre oblique inverse. Les séquences d'échappement suivantes sont utilisées pour la sortie: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\'`, `\\`. L'analyse prend également en charge les séquences `\a`, `\v`, et `\xHH` (séquences d'échappement hexadécimales) et `\c` séquences, où `c` est un caractère (ces séquences sont convertis à `c`). Ainsi, la lecture de données prend en charge les formats où un saut de ligne peut être écrit comme `\n` ou `\` ou comme un saut de ligne. Par exemple, la chaîne `Hello world` avec un saut de ligne entre les mots au lieu de l'espace peut être analysé dans l'une des variantes suivantes: ``` text Hello\nworld @@ -102,15 +102,15 @@ Hello\ world ``` -La deuxième variante est prise en charge car MySQL l’utilise lors de l’écriture de vidages séparés par des tabulations. +La deuxième variante est prise en charge car MySQL l'utilise lors de l'écriture de vidages séparés par des tabulations. -L’ensemble minimum de caractères que vous devez échapper lors du passage de données au format TabSeparated: tab, saut de ligne (LF) et barre oblique inverse. +L'ensemble minimum de caractères que vous devez échapper lors du passage de données au format TabSeparated: tab, saut de ligne (LF) et barre oblique inverse. Seul un petit ensemble de symboles sont échappés. Vous pouvez facilement tomber sur une valeur de chaîne que votre terminal va ruiner en sortie. -Les tableaux sont écrits sous la forme d’une liste de valeurs séparées par des virgules entre crochets. Le nombre d’éléments dans le tableau sont formatés comme normalement. `Date` et `DateTime` les types sont écrits entre guillemets simples. Les chaînes sont écrites entre guillemets simples avec les mêmes règles d’échappement que ci-dessus. +Les tableaux sont écrits sous la forme d'une liste de valeurs séparées par des virgules entre crochets. Le nombre d'éléments dans le tableau sont formatés comme normalement. `Date` et `DateTime` les types sont écrits entre guillemets simples. Les chaînes sont écrites entre guillemets simples avec les mêmes règles d'échappement que ci-dessus. -[NULL](../sql-reference/syntax.md) est formaté en tant qu’ `\N`. +[NULL](../sql-reference/syntax.md) est formaté en tant qu' `\N`. Chaque élément de [Imbriqué](../sql-reference/data-types/nested-data-structures/nested.md) structures est représenté sous forme de tableau. @@ -143,46 +143,46 @@ SELECT * FROM nestedt FORMAT TSV ## TabSeparatedRaw {#tabseparatedraw} Diffère de `TabSeparated` format en ce que les lignes sont écrites sans échappement. -Ce format n’est approprié que pour la sortie d’un résultat de requête, mais pas pour l’analyse (récupération des données à insérer dans une table). +Ce format n'est approprié que pour la sortie d'un résultat de requête, mais pas pour l'analyse (récupération des données à insérer dans une table). Ce format est également disponible sous le nom de `TSVRaw`. ## TabSeparatedWithNames {#tabseparatedwithnames} Diffère de la `TabSeparated` formater en ce que les noms de colonne sont écrits dans la première ligne. -Pendant l’analyse, la première ligne est complètement ignorée. Vous ne pouvez pas utiliser les noms de colonnes pour déterminer leur position ou vérifier leur exactitude. -(La prise en charge de l’analyse de la ligne d’en-tête peut être ajoutée à l’avenir.) +Pendant l'analyse, la première ligne est complètement ignorée. Vous ne pouvez pas utiliser les noms de colonnes pour déterminer leur position ou vérifier leur exactitude. +(La prise en charge de l'analyse de la ligne d'en-tête peut être ajoutée à l'avenir.) Ce format est également disponible sous le nom de `TSVWithNames`. ## TabSeparatedWithNamesAndTypes {#tabseparatedwithnamesandtypes} Diffère de la `TabSeparated` format que les noms de colonne sont écrits à la première ligne, tandis que les types de colonne sont dans la deuxième rangée. -Pendant l’analyse, les première et deuxième lignes sont complètement ignorées. +Pendant l'analyse, les première et deuxième lignes sont complètement ignorées. Ce format est également disponible sous le nom de `TSVWithNamesAndTypes`. ## Modèle {#format-template} -Ce format permet de spécifier une chaîne de format personnalisée avec des espaces réservés pour les valeurs avec une règle d’échappement spécifiée. +Ce format permet de spécifier une chaîne de format personnalisée avec des espaces réservés pour les valeurs avec une règle d'échappement spécifiée. -Il utilise les paramètres `format_template_resultset`, `format_template_row`, `format_template_rows_between_delimiter` and some settings of other formats (e.g. `output_format_json_quote_64bit_integers` lors de l’utilisation de `JSON` s’échapper, voir plus loin) +Il utilise les paramètres `format_template_resultset`, `format_template_row`, `format_template_rows_between_delimiter` and some settings of other formats (e.g. `output_format_json_quote_64bit_integers` lors de l'utilisation de `JSON` s'échapper, voir plus loin) -Paramètre `format_template_row` spécifie le chemin d’accès au fichier, qui contient une chaîne de format pour les lignes avec la syntaxe suivante: +Paramètre `format_template_row` spécifie le chemin d'accès au fichier, qui contient une chaîne de format pour les lignes avec la syntaxe suivante: `delimiter_1${column_1:serializeAs_1}delimiter_2${column_2:serializeAs_2} ... delimiter_N`, où `delimiter_i` est un délimiteur entre les valeurs (`$` symbole peut être échappé comme `$$`), -`column_i` est un nom ou un index d’une colonne dont les valeurs sont choisies ou inséré (si vide, alors la colonne sera ignoré), -`serializeAs_i` est une règle d’échappement pour les valeurs de colonne. Les règles d’échappement suivantes sont prises en charge: +`column_i` est un nom ou un index d'une colonne dont les valeurs sont choisies ou inséré (si vide, alors la colonne sera ignoré), +`serializeAs_i` est une règle d'échappement pour les valeurs de colonne. Les règles d'échappement suivantes sont prises en charge: - `CSV`, `JSON`, `XML` (de même pour les formats des mêmes noms) - `Escaped` (de la même manière à `TSV`) - `Quoted` (de la même manière à `Values`) -- `Raw` (sans s’échapper, de même pour `TSVRaw`) -- `None` (pas de règle d’échappement, voir plus loin) +- `Raw` (sans s'échapper, de même pour `TSVRaw`) +- `None` (pas de règle d'échappement, voir plus loin) -Si une règle d’échappement est omise, alors `None` sera utilisé. `XML` et `Raw` sont adaptés uniquement pour la sortie. +Si une règle d'échappement est omise, alors `None` sera utilisé. `XML` et `Raw` sont adaptés uniquement pour la sortie. Donc, pour la chaîne de format suivante: @@ -194,20 +194,20 @@ les valeurs de `SearchPhrase`, `c` et `price` colonnes, qui sont échappées com Le `format_template_rows_between_delimiter` paramètre spécifie délimiteur entre les lignes, qui est imprimé (ou attendu) après chaque ligne, sauf la dernière (`\n` par défaut) -Paramètre `format_template_resultset` spécifie le chemin d’accès au fichier, qui contient une chaîne de format pour resultset. Format string for resultset a la même syntaxe qu’une chaîne de format pour row et permet de spécifier un préfixe, un suffixe et un moyen d’imprimer des informations supplémentaires. Il contient les espaces réservés suivants au lieu des noms de colonnes: +Paramètre `format_template_resultset` spécifie le chemin d'accès au fichier, qui contient une chaîne de format pour resultset. Format string for resultset a la même syntaxe qu'une chaîne de format pour row et permet de spécifier un préfixe, un suffixe et un moyen d'imprimer des informations supplémentaires. Il contient les espaces réservés suivants au lieu des noms de colonnes: - `data` est les lignes avec des données dans `format_template_row` format, séparés par des `format_template_rows_between_delimiter`. Cet espace doit être le premier espace réservé dans la chaîne de format. -- `totals` est la ligne avec des valeurs totales dans `format_template_row` format (lors de L’utilisation avec des totaux) +- `totals` est la ligne avec des valeurs totales dans `format_template_row` format (lors de L'utilisation avec des totaux) - `min` est la ligne avec des valeurs minimales dans `format_template_row` format (lorsque les extrêmes sont définis sur 1) - `max` est la ligne avec des valeurs maximales en `format_template_row` format (lorsque les extrêmes sont définis sur 1) - `rows` le nombre total de lignes de sortie -- `rows_before_limit` est le nombre minimal de lignes qu’il y aurait eu sans limite. Sortie uniquement si la requête contient LIMIT. Si la requête contient GROUP BY, rows\_before\_limit\_at\_least est le nombre exact de lignes qu’il y aurait eu sans limite. -- `time` est le temps d’exécution de la requête en secondes +- `rows_before_limit` est le nombre minimal de lignes qu'il y aurait eu sans limite. Sortie uniquement si la requête contient LIMIT. Si la requête contient GROUP BY, rows\_before\_limit\_at\_least est le nombre exact de lignes qu'il y aurait eu sans limite. +- `time` est le temps d'exécution de la requête en secondes - `rows_read` est le nombre de lignes a été lu -- `bytes_read` est le nombre d’octets (non compressé) a été lu +- `bytes_read` est le nombre d'octets (non compressé) a été lu -Réservé `data`, `totals`, `min` et `max` ne doit pas avoir de règle d’échappement spécifiée (ou `None` doit être spécifié explicitement). Les espaces réservés restants peuvent avoir une règle d’échappement spécifiée. -Si l’ `format_template_resultset` paramètre est une chaîne vide, `${data}` est utilisé comme valeur par défaut. +Réservé `data`, `totals`, `min` et `max` ne doit pas avoir de règle d'échappement spécifiée (ou `None` doit être spécifié explicitement). Les espaces réservés restants peuvent avoir une règle d'échappement spécifiée. +Si l' `format_template_resultset` paramètre est une chaîne vide, `${data}` est utilisé comme valeur par défaut. Pour insérer des requêtes format permet de sauter certaines colonnes ou certains champs si préfixe ou suffixe (voir Exemple). Sélectionnez exemple: @@ -289,14 +289,14 @@ Some header\n${data}\nTotal rows: ${:CSV}\n Page views: ${PageViews:CSV}, User id: ${UserID:CSV}, Useless field: ${:CSV}, Duration: ${Duration:CSV}, Sign: ${Sign:CSV} ``` -`PageViews`, `UserID`, `Duration` et `Sign` à l’intérieur des espaces réservés sont des noms de colonnes dans la table. Les valeurs après `Useless field` en rangées et après `\nTotal rows:` dans le suffixe sera ignoré. -Tous les séparateurs dans les données d’entrée doivent être strictement égal à délimiteurs dans les chaînes de format. +`PageViews`, `UserID`, `Duration` et `Sign` à l'intérieur des espaces réservés sont des noms de colonnes dans la table. Les valeurs après `Useless field` en rangées et après `\nTotal rows:` dans le suffixe sera ignoré. +Tous les séparateurs dans les données d'entrée doivent être strictement égal à délimiteurs dans les chaînes de format. ## TemplateIgnoreSpaces {#templateignorespaces} -Ce format est adapté uniquement pour l’entrée. -Semblable à `Template`, mais ignore les espaces entre les séparateurs et les valeurs dans le flux d’entrée. Toutefois, si les chaînes de format contiennent des espaces, ces caractères seront attendus dans le flux d’entrée. Permet également de spécifier des espaces réservés vides (`${}` ou `${:None}`) pour diviser un délimiteur en parties séparées pour ignorer les espaces entre eux. Ces espaces réservés sont utilisés uniquement pour ignorer les caractères d’espace. -Il est possible de lire `JSON` l’utilisation de ce format, si les valeurs des colonnes ont le même ordre dans toutes les lignes. Par exemple, la requête suivante peut être utilisée pour insérer des données à partir d’un exemple de format de sortie [JSON](#json): +Ce format est adapté uniquement pour l'entrée. +Semblable à `Template`, mais ignore les espaces entre les séparateurs et les valeurs dans le flux d'entrée. Toutefois, si les chaînes de format contiennent des espaces, ces caractères seront attendus dans le flux d'entrée. Permet également de spécifier des espaces réservés vides (`${}` ou `${:None}`) pour diviser un délimiteur en parties séparées pour ignorer les espaces entre eux. Ces espaces réservés sont utilisés uniquement pour ignorer les caractères d'espace. +Il est possible de lire `JSON` l'utilisation de ce format, si les valeurs des colonnes ont le même ordre dans toutes les lignes. Par exemple, la requête suivante peut être utilisée pour insérer des données à partir d'un exemple de format de sortie [JSON](#json): ``` sql INSERT INTO table_name FORMAT TemplateIgnoreSpaces SETTINGS @@ -317,7 +317,7 @@ format_template_resultset = '/some/path/resultset.format', format_template_row = ## TSKV {#tskv} -Similaire à TabSeparated, mais affiche une valeur au format name = value. Les noms sont échappés de la même manière qu’au format TabSeparated, et le symbole = est également échappé. +Similaire à TabSeparated, mais affiche une valeur au format name = value. Les noms sont échappés de la même manière qu'au format TabSeparated, et le symbole = est également échappé. ``` text SearchPhrase= count()=8267016 @@ -332,7 +332,7 @@ SearchPhrase=curtain designs count()=1064 SearchPhrase=baku count()=1000 ``` -[NULL](../sql-reference/syntax.md) est formaté en tant qu’ `\N`. +[NULL](../sql-reference/syntax.md) est formaté en tant qu' `\N`. ``` sql SELECT * FROM t_null FORMAT TSKV @@ -342,46 +342,46 @@ SELECT * FROM t_null FORMAT TSKV x=1 y=\N ``` -Quand il y a un grand nombre de petites colonnes, ce format est inefficace, et il n’y a généralement pas de raison de l’utiliser. Néanmoins, ce n’est pas pire que JSONEachRow en termes d’efficacité. +Quand il y a un grand nombre de petites colonnes, ce format est inefficace, et il n'y a généralement pas de raison de l'utiliser. Néanmoins, ce n'est pas pire que JSONEachRow en termes d'efficacité. Both data output and parsing are supported in this format. For parsing, any order is supported for the values of different columns. It is acceptable for some values to be omitted – they are treated as equal to their default values. In this case, zeros and blank rows are used as default values. Complex values that could be specified in the table are not supported as defaults. -L’analyse permet la présence du champ supplémentaire `tskv` sans le signe égal ou de valeur. Ce champ est ignoré. +L'analyse permet la présence du champ supplémentaire `tskv` sans le signe égal ou de valeur. Ce champ est ignoré. ## CSV {#csv} Format des valeurs séparées par des virgules ([RFC](https://tools.ietf.org/html/rfc4180)). -Lors du formatage, les lignes sont entourées de guillemets doubles. Un guillemet double à l’intérieur d’une chaîne est affiché sous la forme de deux guillemets doubles dans une rangée. Il n’y a pas d’autres règles pour échapper les caractères. Date et date-heure sont entre guillemets. Les nombres sont produits sans guillemets. Les valeurs sont séparées par un caractère délimiteur, qui est `,` par défaut. Le caractère délimiteur est défini dans le paramètre [format\_csv\_delimiter](../operations/settings/settings.md#settings-format_csv_delimiter). Les lignes sont séparées à L’aide du saut de ligne Unix (LF). Les tableaux sont sérialisés au format CSV comme suit: tout d’abord, le tableau est sérialisé en une chaîne comme au format TabSeparated, puis la chaîne résultante est sortie au format CSV entre guillemets doubles. Les Tuples au format CSV sont sérialisés en tant que colonnes séparées (c’est-à-dire que leur imbrication dans le tuple est perdue). +Lors du formatage, les lignes sont entourées de guillemets doubles. Un guillemet double à l'intérieur d'une chaîne est affiché sous la forme de deux guillemets doubles dans une rangée. Il n'y a pas d'autres règles pour échapper les caractères. Date et date-heure sont entre guillemets. Les nombres sont produits sans guillemets. Les valeurs sont séparées par un caractère délimiteur, qui est `,` par défaut. Le caractère délimiteur est défini dans le paramètre [format\_csv\_delimiter](../operations/settings/settings.md#settings-format_csv_delimiter). Les lignes sont séparées à L'aide du saut de ligne Unix (LF). Les tableaux sont sérialisés au format CSV comme suit: tout d'abord, le tableau est sérialisé en une chaîne comme au format TabSeparated, puis la chaîne résultante est sortie au format CSV entre guillemets doubles. Les Tuples au format CSV sont sérialisés en tant que colonnes séparées (c'est-à-dire que leur imbrication dans le tuple est perdue). ``` bash $ clickhouse-client --format_csv_delimiter="|" --query="INSERT INTO test.csv FORMAT CSV" < data.csv ``` -\*Par défaut, le délimiteur est `,`. Voir la [format\_csv\_delimiter](../operations/settings/settings.md#settings-format_csv_delimiter) réglage pour plus d’informations. +\*Par défaut, le délimiteur est `,`. Voir la [format\_csv\_delimiter](../operations/settings/settings.md#settings-format_csv_delimiter) réglage pour plus d'informations. -Lors de l’analyse, toutes les valeurs peuvent être analysés avec ou sans guillemets. Les guillemets doubles et simples sont pris en charge. Les lignes peuvent également être organisées sans guillemets. Dans ce cas, ils sont analysés jusqu’au caractère délimiteur ou au saut de ligne (CR ou LF). En violation de la RFC, lors de l’analyse des lignes sans guillemets, les espaces et les onglets de début et de fin sont ignorés. Pour le saut de ligne, les types Unix (LF), Windows (CR LF) et Mac OS Classic (CR LF) sont tous pris en charge. +Lors de l'analyse, toutes les valeurs peuvent être analysés avec ou sans guillemets. Les guillemets doubles et simples sont pris en charge. Les lignes peuvent également être organisées sans guillemets. Dans ce cas, ils sont analysés jusqu'au caractère délimiteur ou au saut de ligne (CR ou LF). En violation de la RFC, lors de l'analyse des lignes sans guillemets, les espaces et les onglets de début et de fin sont ignorés. Pour le saut de ligne, les types Unix (LF), Windows (CR LF) et Mac OS Classic (CR LF) sont tous pris en charge. -Les valeurs d’entrée non cotées vides sont remplacées par des valeurs par défaut pour les colonnes respectives, si +Les valeurs d'entrée non cotées vides sont remplacées par des valeurs par défaut pour les colonnes respectives, si [input\_format\_defaults\_for\_omitted\_fields](../operations/settings/settings.md#session_settings-input_format_defaults_for_omitted_fields) est activé. -`NULL` est formaté en tant qu’ `\N` ou `NULL` ou une chaîne vide non cotée (voir paramètres [input\_format\_csv\_unquoted\_null\_literal\_as\_null](../operations/settings/settings.md#settings-input_format_csv_unquoted_null_literal_as_null) et [input\_format\_defaults\_for\_omitted\_fields](../operations/settings/settings.md#session_settings-input_format_defaults_for_omitted_fields)). +`NULL` est formaté en tant qu' `\N` ou `NULL` ou une chaîne vide non cotée (voir paramètres [input\_format\_csv\_unquoted\_null\_literal\_as\_null](../operations/settings/settings.md#settings-input_format_csv_unquoted_null_literal_as_null) et [input\_format\_defaults\_for\_omitted\_fields](../operations/settings/settings.md#session_settings-input_format_defaults_for_omitted_fields)). Le format CSV prend en charge la sortie des totaux et des extrêmes de la même manière que `TabSeparated`. ## CSVWithNames {#csvwithnames} -Imprime également la ligne d’en-tête, semblable à `TabSeparatedWithNames`. +Imprime également la ligne d'en-tête, semblable à `TabSeparatedWithNames`. ## CustomSeparated {#format-customseparated} -Semblable à [Modèle](#format-template), mais il imprime ou lit toutes les colonnes et utilise la règle d’échappement du paramètre `format_custom_escaping_rule` et délimiteurs de paramètres `format_custom_field_delimiter`, `format_custom_row_before_delimiter`, `format_custom_row_after_delimiter`, `format_custom_row_between_delimiter`, `format_custom_result_before_delimiter` et `format_custom_result_after_delimiter`, pas à partir de chaînes de format. +Semblable à [Modèle](#format-template), mais il imprime ou lit toutes les colonnes et utilise la règle d'échappement du paramètre `format_custom_escaping_rule` et délimiteurs de paramètres `format_custom_field_delimiter`, `format_custom_row_before_delimiter`, `format_custom_row_after_delimiter`, `format_custom_row_between_delimiter`, `format_custom_result_before_delimiter` et `format_custom_result_after_delimiter`, pas à partir de chaînes de format. Il y a aussi `CustomSeparatedIgnoreSpaces` le format, qui est similaire à `TemplateIgnoreSpaces`. ## JSON {#json} -Sorties de données au format JSON. Outre les tables de données, il génère également des noms et des types de colonnes, ainsi que des informations supplémentaires: le nombre total de lignes de sortie et le nombre de lignes qui auraient pu être sorties s’il n’y avait pas de limite. Exemple: +Sorties de données au format JSON. Outre les tables de données, il génère également des noms et des types de colonnes, ainsi que des informations supplémentaires: le nombre total de lignes de sortie et le nombre de lignes qui auraient pu être sorties s'il n'y avait pas de limite. Exemple: ``` sql SELECT SearchPhrase, count() AS c FROM test.hits GROUP BY SearchPhrase WITH TOTALS ORDER BY c DESC LIMIT 5 FORMAT JSON @@ -451,22 +451,22 @@ SELECT SearchPhrase, count() AS c FROM test.hits GROUP BY SearchPhrase WITH TOTA } ``` -Le JSON est compatible avec JavaScript. Pour ce faire, certains caractères sont en outre échappés: la barre oblique `/` s’est échappée comme l’ `\/`; sauts de ligne alternatifs `U+2028` et `U+2029`, qui cassent certains navigateurs, sont échappés comme `\uXXXX`. Les caractères de contrôle ASCII sont échappés: retour arrière, flux de formulaire, saut de ligne, retour chariot et tabulation horizontale sont remplacés par `\b`, `\f`, `\n`, `\r`, `\t` , ainsi que les octets restants dans la plage 00-1F en utilisant `\uXXXX` sequences. Invalid UTF-8 sequences are changed to the replacement character � so the output text will consist of valid UTF-8 sequences. For compatibility with JavaScript, Int64 and UInt64 integers are enclosed in double-quotes by default. To remove the quotes, you can set the configuration parameter [output\_format\_json\_quote\_64bit\_integers](../operations/settings/settings.md#session_settings-output_format_json_quote_64bit_integers) à 0. +Le JSON est compatible avec JavaScript. Pour ce faire, certains caractères sont en outre échappés: la barre oblique `/` s'est échappée comme l' `\/`; sauts de ligne alternatifs `U+2028` et `U+2029`, qui cassent certains navigateurs, sont échappés comme `\uXXXX`. Les caractères de contrôle ASCII sont échappés: retour arrière, flux de formulaire, saut de ligne, retour chariot et tabulation horizontale sont remplacés par `\b`, `\f`, `\n`, `\r`, `\t` , ainsi que les octets restants dans la plage 00-1F en utilisant `\uXXXX` sequences. Invalid UTF-8 sequences are changed to the replacement character � so the output text will consist of valid UTF-8 sequences. For compatibility with JavaScript, Int64 and UInt64 integers are enclosed in double-quotes by default. To remove the quotes, you can set the configuration parameter [output\_format\_json\_quote\_64bit\_integers](../operations/settings/settings.md#session_settings-output_format_json_quote_64bit_integers) à 0. `rows` – The total number of output rows. `rows_before_limit_at_least` Le nombre minimal de lignes aurait été sans limite. Sortie uniquement si la requête contient LIMIT. -Si la requête contient GROUP BY, rows\_before\_limit\_at\_least est le nombre exact de lignes qu’il y aurait eu sans limite. +Si la requête contient GROUP BY, rows\_before\_limit\_at\_least est le nombre exact de lignes qu'il y aurait eu sans limite. `totals` – Total values (when using WITH TOTALS). `extremes` – Extreme values (when extremes are set to 1). -Ce format n’est approprié que pour la sortie d’un résultat de requête, mais pas pour l’analyse (récupération des données à insérer dans une table). +Ce format n'est approprié que pour la sortie d'un résultat de requête, mais pas pour l'analyse (récupération des données à insérer dans une table). -Supports ClickHouse [NULL](../sql-reference/syntax.md) s’affiche à l’écran `null` dans la sortie JSON. +Supports ClickHouse [NULL](../sql-reference/syntax.md) s'affiche à l'écran `null` dans la sortie JSON. -Voir aussi l’ [JSONEachRow](#jsoneachrow) format. +Voir aussi l' [JSONEachRow](#jsoneachrow) format. ## JSONCompact {#jsoncompact} @@ -511,12 +511,12 @@ Exemple: } ``` -Ce format n’est approprié que pour la sortie d’un résultat de requête, mais pas pour l’analyse (récupération des données à insérer dans une table). -Voir aussi l’ `JSONEachRow` format. +Ce format n'est approprié que pour la sortie d'un résultat de requête, mais pas pour l'analyse (récupération des données à insérer dans une table). +Voir aussi l' `JSONEachRow` format. ## JSONEachRow {#jsoneachrow} -Lorsque vous utilisez ce format, ClickHouse affiche les lignes en tant qu’objets JSON séparés et délimités par des retours à la ligne, mais les données dans leur ensemble ne sont pas JSON valides. +Lorsque vous utilisez ce format, ClickHouse affiche les lignes en tant qu'objets JSON séparés et délimités par des retours à la ligne, mais les données dans leur ensemble ne sont pas JSON valides. ``` json {"SearchPhrase":"curtain designs","count()":"1064"} @@ -524,7 +524,7 @@ Lorsque vous utilisez ce format, ClickHouse affiche les lignes en tant qu’obje {"SearchPhrase":"","count()":"8267016"} ``` -Lors de l’insertion des données, vous devez fournir un objet JSON distinct pour chaque ligne. +Lors de l'insertion des données, vous devez fournir un objet JSON distinct pour chaque ligne. ### Insertion De Données {#inserting-data} @@ -534,10 +534,10 @@ INSERT INTO UserActivity FORMAT JSONEachRow {"PageViews":5, "UserID":"4324182021 Clickhouse permet: -- Toute commande de paires clé-valeur dans l’objet. +- Toute commande de paires clé-valeur dans l'objet. - Omettre certaines valeurs. -ClickHouse ignore les espaces entre les éléments et les virgules après les objets. Vous pouvez passer tous les objets en une seule ligne. Vous n’avez pas à les séparer avec des sauts de ligne. +ClickHouse ignore les espaces entre les éléments et les virgules après les objets. Vous pouvez passer tous les objets en une seule ligne. Vous n'avez pas à les séparer avec des sauts de ligne. **Valeurs omises traitement** @@ -559,11 +559,11 @@ CREATE TABLE IF NOT EXISTS example_table - Si `input_format_defaults_for_omitted_fields = 1`, alors la valeur par défaut pour `x` égal `0` mais la valeur par défaut de `a` égal `x * 2`. !!! note "Avertissement" - Lors de l’insertion de données avec `insert_sample_with_metadata = 1`, ClickHouse consomme plus de ressources de calcul, par rapport à l’insertion avec `insert_sample_with_metadata = 0`. + Lors de l'insertion de données avec `insert_sample_with_metadata = 1`, ClickHouse consomme plus de ressources de calcul, par rapport à l'insertion avec `insert_sample_with_metadata = 0`. ### La Sélection De Données {#selecting-data} -Envisager l’ `UserActivity` table comme un exemple: +Envisager l' `UserActivity` table comme un exemple: ``` text ┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ @@ -579,12 +579,12 @@ Requête `SELECT * FROM UserActivity FORMAT JSONEachRow` retourner: {"UserID":"4324182021466249494","PageViews":6,"Duration":185,"Sign":1} ``` -Contrairement à l’ [JSON](#json) format, il n’y a pas de substitution de séquences UTF-8 non valides. Les valeurs sont échappés de la même manière que pour `JSON`. +Contrairement à l' [JSON](#json) format, il n'y a pas de substitution de séquences UTF-8 non valides. Les valeurs sont échappés de la même manière que pour `JSON`. !!! note "Note" - Tout ensemble d’octets peut être sortie dans les cordes. L’utilisation de la `JSONEachRow` formater si vous êtes sûr que les données de la table peuvent être formatées en tant que JSON sans perdre aucune information. + Tout ensemble d'octets peut être sortie dans les cordes. L'utilisation de la `JSONEachRow` formater si vous êtes sûr que les données de la table peuvent être formatées en tant que JSON sans perdre aucune information. -### Utilisation De Structures imbriquées {#jsoneachrow-nested} +### Utilisation de Structures imbriquées {#jsoneachrow-nested} Si vous avez une table avec [Imbriqué](../sql-reference/data-types/nested-data-structures/nested.md) colonnes de type de données, vous pouvez insérer des données JSON avec la même structure. Activer cette fonctionnalité avec le [input\_format\_import\_nested\_json](../operations/settings/settings.md#settings-input_format_import_nested_json) paramètre. @@ -600,7 +600,7 @@ Comme vous pouvez le voir dans la `Nested` description du type de données, Clic INSERT INTO json_each_row_nested FORMAT JSONEachRow {"n.s": ["abc", "def"], "n.i": [1, 23]} ``` -Pour insérer des données en tant qu’objet JSON hiérarchique, définissez [input\_format\_import\_nested\_json=1](../operations/settings/settings.md#settings-input_format_import_nested_json). +Pour insérer des données en tant qu'objet JSON hiérarchique, définissez [input\_format\_import\_nested\_json=1](../operations/settings/settings.md#settings-input_format_import_nested_json). ``` json { @@ -645,20 +645,20 @@ SELECT * FROM json_each_row_nested ## Natif {#native} -Le format le plus efficace. Les données sont écrites et lues par des blocs au format binaire. Pour chaque bloc, le nombre de lignes, le nombre de colonnes, les noms et types de colonnes et les parties de colonnes de ce bloc sont enregistrés les uns après les autres. En d’autres termes, ce format est “columnar” – it doesn’t convert columns to rows. This is the format used in the native interface for interaction between servers, for using the command-line client, and for C++ clients. +Le format le plus efficace. Les données sont écrites et lues par des blocs au format binaire. Pour chaque bloc, le nombre de lignes, le nombre de colonnes, les noms et types de colonnes et les parties de colonnes de ce bloc sont enregistrés les uns après les autres. En d'autres termes, ce format est “columnar” – it doesn't convert columns to rows. This is the format used in the native interface for interaction between servers, for using the command-line client, and for C++ clients. -Vous pouvez utiliser ce format pour générer rapidement des vidages qui ne peuvent être lus que par le SGBD ClickHouse. Cela n’a pas de sens de travailler avec ce format vous-même. +Vous pouvez utiliser ce format pour générer rapidement des vidages qui ne peuvent être lus que par le SGBD ClickHouse. Cela n'a pas de sens de travailler avec ce format vous-même. ## NULL {#null} -Rien n’est sortie. Cependant, la requête est traitée et, lors de l’utilisation du client de ligne de commande, les données sont transmises au client. Ceci est utilisé pour les tests, y compris les tests de performance. -Évidemment, ce format n’est approprié que pour la sortie, pas pour l’analyse. +Rien n'est sortie. Cependant, la requête est traitée et, lors de l'utilisation du client de ligne de commande, les données sont transmises au client. Ceci est utilisé pour les tests, y compris les tests de performance. +Évidemment, ce format n'est approprié que pour la sortie, pas pour l'analyse. ## Joli {#pretty} -Affiche les données sous forme de tables Unicode-art, en utilisant également des séquences d’échappement ANSI pour définir les couleurs dans le terminal. +Affiche les données sous forme de tables Unicode-art, en utilisant également des séquences d'échappement ANSI pour définir les couleurs dans le terminal. Une grille complète de la table est dessinée, et chaque ligne occupe deux lignes dans le terminal. -Chaque bloc de résultat est sorti sous la forme d’une table séparée. Ceci est nécessaire pour que les blocs puissent être sortis sans résultats de mise en mémoire tampon (la mise en mémoire tampon serait nécessaire pour pré-calculer la largeur visible de toutes les valeurs). +Chaque bloc de résultat est sorti sous la forme d'une table séparée. Ceci est nécessaire pour que les blocs puissent être sortis sans résultats de mise en mémoire tampon (la mise en mémoire tampon serait nécessaire pour pré-calculer la largeur visible de toutes les valeurs). [NULL](../sql-reference/syntax.md) est sortie `ᴺᵁᴸᴸ`. @@ -687,9 +687,9 @@ SELECT 'String with \'quotes\' and \t character' AS Escaping_test ``` Pour éviter de déverser trop de données sur le terminal, seules les 10 000 premières lignes sont imprimées. Si le nombre de lignes est supérieur ou égal à 10 000, le message “Showed first 10 000” est imprimé. -Ce format n’est approprié que pour la sortie d’un résultat de requête, mais pas pour l’analyse (récupération des données à insérer dans une table). +Ce format n'est approprié que pour la sortie d'un résultat de requête, mais pas pour l'analyse (récupération des données à insérer dans une table). -Le joli format prend en charge la sortie des valeurs totales (lors de L’utilisation avec des totaux) et des extrêmes (lorsque ‘extremes’ est réglé sur 1). Dans ces cas, les valeurs totales et les valeurs extrêmes sont sorties après les données principales, dans des tableaux séparés. Exemple (montré pour le [PrettyCompact](#prettycompact) format): +Le joli format prend en charge la sortie des valeurs totales (lors de L'utilisation avec des totaux) et des extrêmes (lorsque ‘extremes’ est réglé sur 1). Dans ces cas, les valeurs totales et les valeurs extrêmes sont sorties après les données principales, dans des tableaux séparés. Exemple (montré pour le [PrettyCompact](#prettycompact) format): ``` sql SELECT EventDate, count() AS c FROM test.hits GROUP BY EventDate WITH TOTALS ORDER BY EventDate FORMAT PrettyCompact @@ -725,11 +725,11 @@ Ce format est utilisé par défaut dans le client de ligne de commande en mode i ## PrettyCompactMonoBlock {#prettycompactmonoblock} -Diffère de [PrettyCompact](#prettycompact) dans ce cas, jusqu’à 10 000 lignes sont mises en mémoire tampon, puis sorties en tant que table unique, pas par blocs. +Diffère de [PrettyCompact](#prettycompact) dans ce cas, jusqu'à 10 000 lignes sont mises en mémoire tampon, puis sorties en tant que table unique, pas par blocs. ## PrettyNoEscapes {#prettynoescapes} -Diffère de Pretty en ce que les séquences d’échappement ANSI ne sont pas utilisées. Ceci est nécessaire pour afficher ce format dans un navigateur, ainsi que pour utiliser le ‘watch’ utilitaire de ligne de commande. +Diffère de Pretty en ce que les séquences d'échappement ANSI ne sont pas utilisées. Ceci est nécessaire pour afficher ce format dans un navigateur, ainsi que pour utiliser le ‘watch’ utilitaire de ligne de commande. Exemple: @@ -737,7 +737,7 @@ Exemple: $ watch -n1 "clickhouse-client --query='SELECT event, value FROM system.events FORMAT PrettyCompactNoEscapes'" ``` -Vous pouvez utiliser L’interface HTTP pour afficher dans le navigateur. +Vous pouvez utiliser L'interface HTTP pour afficher dans le navigateur. ### Joliscompactnoescapes {#prettycompactnoescapes} @@ -749,7 +749,7 @@ Le même que le réglage précédent. ## PrettySpace {#prettyspace} -Diffère de [PrettyCompact](#prettycompact) dans cet espace (caractères d’espace) est utilisé à la place de la grille. +Diffère de [PrettyCompact](#prettycompact) dans cet espace (caractères d'espace) est utilisé à la place de la grille. ## RowBinary {#rowbinary} @@ -757,18 +757,18 @@ Formats et analyse les données par ligne au format binaire. Les lignes et les v Ce format est moins efficace que le format natif car il est basé sur des lignes. Les entiers utilisent une représentation little-endian de longueur fixe. Par exemple, UInt64 utilise 8 octets. -DateTime est représenté par UInt32 contenant L’horodatage Unix comme valeur. +DateTime est représenté par UInt32 contenant L'horodatage Unix comme valeur. Date est représenté comme un objet UInt16 qui contient le nombre de jours depuis 1970-01-01 comme valeur. La chaîne est représentée par une longueur varint (non signée [LEB128](https://en.wikipedia.org/wiki/LEB128)), suivie par les octets de la chaîne. -FixedString est représenté simplement comme une séquence d’octets. +FixedString est représenté simplement comme une séquence d'octets. -Le tableau est représenté sous la forme d’une longueur varint (non signée [LEB128](https://en.wikipedia.org/wiki/LEB128)), suivie par les éléments de la matrice. +Le tableau est représenté sous la forme d'une longueur varint (non signée [LEB128](https://en.wikipedia.org/wiki/LEB128)), suivie par les éléments de la matrice. -Pour [NULL](../sql-reference/syntax.md#null-literal) un soutien, un octet supplémentaire contenant 1 ou 0 est ajouté avant chaque [Nullable](../sql-reference/data-types/nullable.md) valeur. Si la valeur est 1, alors la valeur est `NULL` et cet octet est interprétée comme une valeur distincte. Si 0, la valeur après l’octet n’est pas `NULL`. +Pour [NULL](../sql-reference/syntax.md#null-literal) un soutien, un octet supplémentaire contenant 1 ou 0 est ajouté avant chaque [Nullable](../sql-reference/data-types/nullable.md) valeur. Si la valeur est 1, alors la valeur est `NULL` et cet octet est interprétée comme une valeur distincte. Si 0, la valeur après l'octet n'est pas `NULL`. ## Rowbinarywithnamesettypes {#rowbinarywithnamesandtypes} -Semblable à [RowBinary](#rowbinary) mais avec l’ajout de l’en-tête: +Semblable à [RowBinary](#rowbinary) mais avec l'ajout de l'en-tête: - [LEB128](https://en.wikipedia.org/wiki/LEB128)- nombre codé de colonnes (N) - N `String`s spécification des noms de colonnes @@ -776,17 +776,17 @@ Semblable à [RowBinary](#rowbinary) mais avec l’ajout de l’en-tête: ## Valeur {#data-format-values} -Imprime chaque ligne entre parenthèses. Les lignes sont séparées par des virgules. Il n’y a pas de virgule après la dernière ligne. Les valeurs entre parenthèses sont également séparées par des virgules. Les nombres sont produits dans un format décimal sans guillemets. Les tableaux sont affichés entre crochets. Les chaînes, les dates et les dates avec des heures sont affichées entre guillemets. Les règles d’échappement et l’analyse sont similaires à [TabSeparated](#tabseparated) format. Pendant le formatage, les espaces supplémentaires ne sont pas insérés, mais pendant l’analyse, ils sont autorisés et ignorés (sauf pour les espaces à l’intérieur des valeurs de tableau, qui ne sont pas autorisés). [NULL](../sql-reference/syntax.md) est représentée comme `NULL`. +Imprime chaque ligne entre parenthèses. Les lignes sont séparées par des virgules. Il n'y a pas de virgule après la dernière ligne. Les valeurs entre parenthèses sont également séparées par des virgules. Les nombres sont produits dans un format décimal sans guillemets. Les tableaux sont affichés entre crochets. Les chaînes, les dates et les dates avec des heures sont affichées entre guillemets. Les règles d'échappement et l'analyse sont similaires à [TabSeparated](#tabseparated) format. Pendant le formatage, les espaces supplémentaires ne sont pas insérés, mais pendant l'analyse, ils sont autorisés et ignorés (sauf pour les espaces à l'intérieur des valeurs de tableau, qui ne sont pas autorisés). [NULL](../sql-reference/syntax.md) est représentée comme `NULL`. The minimum set of characters that you need to escape when passing data in Values ​​format: single quotes and backslashes. -C’est le format qui est utilisé dans `INSERT INTO t VALUES ...`, mais vous pouvez également l’utiliser pour le formatage des résultats de requête. +C'est le format qui est utilisé dans `INSERT INTO t VALUES ...`, mais vous pouvez également l'utiliser pour le formatage des résultats de requête. Voir aussi: [input\_format\_values\_interpret\_expressions](../operations/settings/settings.md#settings-input_format_values_interpret_expressions) et [input\_format\_values\_deduce\_templates\_of\_expressions](../operations/settings/settings.md#settings-input_format_values_deduce_templates_of_expressions) paramètre. ## Vertical {#vertical} -Imprime chaque valeur sur une ligne distincte avec le nom de colonne spécifié. Ce format est pratique pour imprimer une ou plusieurs lignes si chaque ligne est constituée d’un grand nombre de colonnes. +Imprime chaque valeur sur une ligne distincte avec le nom de colonne spécifié. Ce format est pratique pour imprimer une ou plusieurs lignes si chaque ligne est constituée d'un grand nombre de colonnes. [NULL](../sql-reference/syntax.md) est sortie `ᴺᵁᴸᴸ`. @@ -816,15 +816,15 @@ test: string with 'quotes' and with some special characters ``` -Ce format n’est approprié que pour la sortie d’un résultat de requête, mais pas pour l’analyse (récupération des données à insérer dans une table). +Ce format n'est approprié que pour la sortie d'un résultat de requête, mais pas pour l'analyse (récupération des données à insérer dans une table). ## VerticalRaw {#verticalraw} -Semblable à [Vertical](#vertical), mais avec échapper désactivé. Ce format ne convient que pour la sortie des résultats de requête, pas pour l’analyse (recevoir des données et les insérer dans la table). +Semblable à [Vertical](#vertical), mais avec échapper désactivé. Ce format ne convient que pour la sortie des résultats de requête, pas pour l'analyse (recevoir des données et les insérer dans la table). ## XML {#xml} -Le format XML ne convient que pour la sortie, pas pour l’analyse. Exemple: +Le format XML ne convient que pour la sortie, pas pour l'analyse. Exemple: ``` xml @@ -888,18 +888,18 @@ Le format XML ne convient que pour la sortie, pas pour l’analyse. Exemple: ``` -Si le nom de colonne n’a pas un format acceptable, juste ‘field’ est utilisé comme le nom de l’élément. En général, la structure XML suit la structure JSON. +Si le nom de colonne n'a pas un format acceptable, juste ‘field’ est utilisé comme le nom de l'élément. En général, la structure XML suit la structure JSON. Just as for JSON, invalid UTF-8 sequences are changed to the replacement character � so the output text will consist of valid UTF-8 sequences. Dans les valeurs de chaîne, les caractères `<` et `&` sont échappés comme `<` et `&`. -Les tableaux sont produits comme `HelloWorld...`,et n-uplets d’ `HelloWorld...`. +Les tableaux sont produits comme `HelloWorld...`,et n-uplets d' `HelloWorld...`. ## CapnProto {#capnproto} -Cap’n Proto est un format de message binaire similaire aux tampons de protocole et Thrift, mais pas comme JSON ou MessagePack. +Cap'n Proto est un format de message binaire similaire aux tampons de protocole et Thrift, mais pas comme JSON ou MessagePack. -Les messages Cap’n Proto sont strictement typés et ne sont pas auto-descriptifs, ce qui signifie qu’ils ont besoin d’une description de schéma externe. Le schéma est appliqué à la volée et mise en cache pour chaque requête. +Les messages Cap'n Proto sont strictement typés et ne sont pas auto-descriptifs, ce qui signifie qu'ils ont besoin d'une description de schéma externe. Le schéma est appliqué à la volée et mise en cache pour chaque requête. ``` bash $ cat capnproto_messages.bin | clickhouse-client --query "INSERT INTO test.hits FORMAT CapnProto SETTINGS format_schema='schema:Message'" @@ -914,7 +914,7 @@ struct Message { } ``` -La désérialisation est efficace et n’augmente généralement pas la charge du système. +La désérialisation est efficace et n'augmente généralement pas la charge du système. Voir aussi [Schéma De Format](#formatschema). @@ -925,7 +925,7 @@ Protobuf-est un [Protocol Buffers](https://developers.google.com/protocol-buffer Ce format nécessite un schéma de format externe. Le schéma est mis en cache entre les requêtes. Clickhouse prend en charge les deux `proto2` et `proto3` syntaxe. Les champs répétés/optionnels/obligatoires sont pris en charge. -Exemples d’utilisation: +Exemples d'utilisation: ``` sql SELECT * FROM test.table FORMAT Protobuf SETTINGS format_schema = 'schemafile:MessageType' @@ -950,7 +950,7 @@ message MessageType { Pour trouver la correspondance entre les colonnes de table et les champs du type de message des tampons de protocole, ClickHouse compare leurs noms. Cette comparaison est insensible à la casse et les caractères `_` (trait de soulignement) et `.` (dot) sont considérés comme égaux. -Si les types d’une colonne et d’un champ de message des tampons de protocole sont différents, la conversion nécessaire est appliquée. +Si les types d'une colonne et d'un champ de message des tampons de protocole sont différents, la conversion nécessaire est appliquée. Les messages imbriqués sont pris en charge. Par exemple, pour le champ `z` dans le type de message suivant @@ -967,7 +967,7 @@ message MessageType { ``` ClickHouse tente de trouver une colonne nommée `x.y.z` (ou `x_y_z` ou `X.y_Z` et ainsi de suite). -Les messages imbriqués conviennent à l’entrée ou à la sortie d’un [structures de données imbriquées](../sql-reference/data-types/nested-data-structures/nested.md). +Les messages imbriqués conviennent à l'entrée ou à la sortie d'un [structures de données imbriquées](../sql-reference/data-types/nested-data-structures/nested.md). Valeurs par défaut définies dans un schéma protobuf comme ceci @@ -987,9 +987,9 @@ Voir aussi [comment lire / écrire des messages protobuf délimités par la long ## Avro {#data-format-avro} -[Apache Avro](http://avro.apache.org/) est un cadre de sérialisation de données orienté ligne développé dans le projet Hadoop D’Apache. +[Apache Avro](http://avro.apache.org/) est un cadre de sérialisation de données orienté ligne développé dans le projet Hadoop D'Apache. -ClickHouse Avro format prend en charge la lecture et l’écriture [Fichiers de données Avro](http://avro.apache.org/docs/current/spec.html#Object+Container+Files). +ClickHouse Avro format prend en charge la lecture et l'écriture [Fichiers de données Avro](http://avro.apache.org/docs/current/spec.html#Object+Container+Files). ### Types De Données Correspondant {#data_types-matching} @@ -1019,18 +1019,18 @@ Types de données logiques Avro non pris en charge: `uuid`, `time-millis`, `time ### Insertion De Données {#inserting-data-1} -Pour insérer des données d’un fichier Avro dans la table ClickHouse: +Pour insérer des données d'un fichier Avro dans la table ClickHouse: ``` bash $ cat file.avro | clickhouse-client --query="INSERT INTO {some_table} FORMAT Avro" ``` -Le schéma racine du fichier Avro d’entrée doit être de `record` type. +Le schéma racine du fichier Avro d'entrée doit être de `record` type. Pour trouver la correspondance entre les colonnes de table et les champs du schéma Avro ClickHouse compare leurs noms. Cette comparaison est sensible à la casse. Les champs inutilisés sont ignorés. -Les types de données des colonnes de la table ClickHouse peuvent différer des champs correspondants des données Avro insérées. Lors de l’insertion de données, ClickHouse interprète les types de données selon le tableau ci-dessus, puis [jeter](../query_language/functions/type_conversion_functions/#type_conversion_function-cast) les données au type de colonne correspondant. +Les types de données des colonnes de la table ClickHouse peuvent différer des champs correspondants des données Avro insérées. Lors de l'insertion de données, ClickHouse interprète les types de données selon le tableau ci-dessus, puis [jeter](../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) les données au type de colonne correspondant. ### La Sélection De Données {#selecting-data-1} @@ -1051,11 +1051,11 @@ Sortie Avro fichier de compression et sync intervalle peut être configuré avec Avroconfluent prend en charge le décodage des messages Avro à objet unique couramment utilisés avec [Kafka](https://kafka.apache.org/) et [Confluentes Schéma De Registre](https://docs.confluent.io/current/schema-registry/index.html). -Chaque message Avro intègre un id de schéma qui peut être résolu dans le schéma réel à l’aide du registre de schéma. +Chaque message Avro intègre un id de schéma qui peut être résolu dans le schéma réel à l'aide du registre de schéma. Les schémas sont mis en cache une fois résolus. -L’URL du registre de schéma est configurée avec [format\_avro\_schema\_registry\_url](../operations/settings/settings.md#settings-format_avro_schema_registry_url) +L'URL du registre de schéma est configurée avec [format\_avro\_schema\_registry\_url](../operations/settings/settings.md#settings-format_avro_schema_registry_url) ### Types De Données Correspondant {#data_types-matching-1} @@ -1097,7 +1097,7 @@ SELECT * FROM topic1_stream; ## Parquet {#data-format-parquet} -[Apache Parquet](http://parquet.apache.org/) est un format de stockage colonnaire répandu dans L’écosystème Hadoop. ClickHouse prend en charge les opérations de lecture et d’écriture pour ce format. +[Apache Parquet](http://parquet.apache.org/) est un format de stockage colonnaire répandu dans L'écosystème Hadoop. ClickHouse prend en charge les opérations de lecture et d'écriture pour ce format. ### Types De Données Correspondant {#data_types-matching-2} @@ -1125,17 +1125,17 @@ Clickhouse prend en charge la précision configurable de `Decimal` type. Le `INS Types de données Parquet non pris en charge: `DATE32`, `TIME32`, `FIXED_SIZE_BINARY`, `JSON`, `UUID`, `ENUM`. -Les types de données des colonnes de table ClickHouse peuvent différer des champs correspondants des données de Parquet insérées. Lors de l’insertion de données, ClickHouse interprète les types de données selon le tableau ci-dessus, puis [jeter](../query_language/functions/type_conversion_functions/#type_conversion_function-cast) les données de ce type de données qui est défini pour la colonne de la table ClickHouse. +Les types de données des colonnes de table ClickHouse peuvent différer des champs correspondants des données de Parquet insérées. Lors de l'insertion de données, ClickHouse interprète les types de données selon le tableau ci-dessus, puis [jeter](../query_language/functions/type_conversion_functions/#type_conversion_function-cast) les données de ce type de données qui est défini pour la colonne de la table ClickHouse. -### Insertion Et sélection De données {#inserting-and-selecting-data} +### Insertion et sélection de données {#inserting-and-selecting-data} -Vous pouvez insérer des données Parquet à partir d’un fichier dans la table ClickHouse par la commande suivante: +Vous pouvez insérer des données Parquet à partir d'un fichier dans la table ClickHouse par la commande suivante: ``` bash $ cat {filename} | clickhouse-client --query="INSERT INTO {some_table} FORMAT Parquet" ``` -Vous pouvez sélectionner des données à partir d’une table de ClickHouse et les enregistrer dans un fichier au format Parquet par la commande suivante: +Vous pouvez sélectionner des données à partir d'une table de ClickHouse et les enregistrer dans un fichier au format Parquet par la commande suivante: ``` bash $ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Parquet" > {some_file.pq} @@ -1145,7 +1145,7 @@ Pour échanger des données avec Hadoop, vous pouvez utiliser [Moteur de table H ## ORC {#data-format-orc} -[Apache ORC](https://orc.apache.org/) est un format de stockage colonnaire répandu dans L’écosystème Hadoop. Vous ne pouvez insérer des données dans ce format à ClickHouse. +[Apache ORC](https://orc.apache.org/) est un format de stockage colonnaire répandu dans L'écosystème Hadoop. Vous ne pouvez insérer des données dans ce format à ClickHouse. ### Types De Données Correspondant {#data_types-matching-3} @@ -1168,15 +1168,15 @@ Le tableau ci-dessous montre les types de données pris en charge et comment ils | `STRING`, `BINARY` | [Chaîne](../sql-reference/data-types/string.md) | | `DECIMAL` | [Décimal](../sql-reference/data-types/decimal.md) | -Clickhouse prend en charge la précision configurable de la `Decimal` type. Le `INSERT` requête traite de l’ORC `DECIMAL` tapez comme le ClickHouse `Decimal128` type. +Clickhouse prend en charge la précision configurable de la `Decimal` type. Le `INSERT` requête traite de l'ORC `DECIMAL` tapez comme le ClickHouse `Decimal128` type. Types de données ORC non pris en charge: `DATE32`, `TIME32`, `FIXED_SIZE_BINARY`, `JSON`, `UUID`, `ENUM`. -Les types de données des colonnes de la table ClickHouse ne doivent pas correspondre aux champs de données Orc correspondants. Lors de l’insertion de données, ClickHouse interprète les types de données selon le tableau ci-dessus, puis [jeter](../query_language/functions/type_conversion_functions/#type_conversion_function-cast) les données du type de données défini pour la colonne clickhouse table. +Les types de données des colonnes de la table ClickHouse ne doivent pas correspondre aux champs de données Orc correspondants. Lors de l'insertion de données, ClickHouse interprète les types de données selon le tableau ci-dessus, puis [jeter](../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) les données du type de données défini pour la colonne clickhouse table. ### Insertion De Données {#inserting-data-2} -Vous pouvez insérer des données ORC à partir d’un fichier dans la table ClickHouse par la commande suivante: +Vous pouvez insérer des données ORC à partir d'un fichier dans la table ClickHouse par la commande suivante: ``` bash $ cat filename.orc | clickhouse-client --query="INSERT INTO some_table FORMAT ORC" @@ -1187,15 +1187,15 @@ Pour échanger des données avec Hadoop, vous pouvez utiliser [Moteur de table H ## Schéma De Format {#formatschema} Le nom du fichier contenant le schéma de format est défini par le paramètre `format_schema`. -Il est nécessaire de définir ce paramètre lorsqu’il est utilisé dans l’un des formats `Cap'n Proto` et `Protobuf`. -Le format de schéma est une combinaison d’un nom de fichier et le nom d’un type de message dans ce fichier, délimité par une virgule, +Il est nécessaire de définir ce paramètre lorsqu'il est utilisé dans l'un des formats `Cap'n Proto` et `Protobuf`. +Le format de schéma est une combinaison d'un nom de fichier et le nom d'un type de message dans ce fichier, délimité par une virgule, e.g. `schemafile.proto:MessageType`. -Si le fichier possède l’extension standard pour le format (par exemple, `.proto` pour `Protobuf`), +Si le fichier possède l'extension standard pour le format (par exemple, `.proto` pour `Protobuf`), il peut être omis et dans ce cas, le format de schéma ressemble `schemafile:MessageType`. Si vous entrez ou sortez des données via le [client](../interfaces/cli.md) dans le [mode interactif](../interfaces/cli.md#cli_usage) le nom de fichier spécifié dans le format de schéma peut contenir un chemin absolu, soit un chemin relatif au répertoire courant sur le client. -Si vous utilisez le client dans le [mode batch](../interfaces/cli.md#cli_usage), le chemin d’accès au schéma doit être relatif pour des raisons de sécurité. +Si vous utilisez le client dans le [mode batch](../interfaces/cli.md#cli_usage), le chemin d'accès au schéma doit être relatif pour des raisons de sécurité. Si vous entrez ou sortez des données via le [Interface HTTP](../interfaces/http.md) le nom de fichier spécifié dans le format de schéma doit être situé dans le répertoire spécifié dans [format\_schema\_path](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-format_schema_path) @@ -1203,10 +1203,10 @@ dans la configuration du serveur. ## Sauter Les Erreurs {#skippingerrors} -Certains formats tels que `CSV`, `TabSeparated`, `TSKV`, `JSONEachRow`, `Template`, `CustomSeparated` et `Protobuf` pouvez ignorer brisé ligne si erreur d’analyse s’est produite et poursuivre l’analyse à partir du début de la ligne suivante. Voir [input\_format\_allow\_errors\_num](../operations/settings/settings.md#settings-input_format_allow_errors_num) et +Certains formats tels que `CSV`, `TabSeparated`, `TSKV`, `JSONEachRow`, `Template`, `CustomSeparated` et `Protobuf` pouvez ignorer brisé ligne si erreur d'analyse s'est produite et poursuivre l'analyse à partir du début de la ligne suivante. Voir [input\_format\_allow\_errors\_num](../operations/settings/settings.md#settings-input_format_allow_errors_num) et [input\_format\_allow\_errors\_ratio](../operations/settings/settings.md#settings-input_format_allow_errors_ratio) paramètre. Limitation: -- En cas d’erreur d’analyse `JSONEachRow` ignore toutes les données jusqu’à la nouvelle ligne (ou EOF), donc les lignes doivent être délimitées par `\n` pour compter les erreurs correctement. -- `Template` et `CustomSeparated` utilisez delimiter après la dernière colonne et delimiter entre les lignes pour trouver le début de la ligne suivante, donc sauter les erreurs ne fonctionne que si au moins l’une d’entre elles n’est pas vide. +- En cas d'erreur d'analyse `JSONEachRow` ignore toutes les données jusqu'à la nouvelle ligne (ou EOF), donc les lignes doivent être délimitées par `\n` pour compter les erreurs correctement. +- `Template` et `CustomSeparated` utilisez delimiter après la dernière colonne et delimiter entre les lignes pour trouver le début de la ligne suivante, donc sauter les erreurs ne fonctionne que si au moins l'une d'entre elles n'est pas vide. [Article Original](https://clickhouse.tech/docs/en/interfaces/formats/) diff --git a/docs/fr/interfaces/http.md b/docs/fr/interfaces/http.md index 0e6bb538a96..2de32747d4a 100644 --- a/docs/fr/interfaces/http.md +++ b/docs/fr/interfaces/http.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 19 toc_title: Interface HTTP --- # Interface HTTP {#http-interface} -L’interface HTTP vous permet D’utiliser ClickHouse sur n’importe quelle plate-forme à partir de n’importe quel langage de programmation. Nous l’utilisons pour travailler à partir de Java et Perl, ainsi que des scripts shell. Dans d’autres départements, L’interface HTTP est utilisée à partir de Perl, Python et Go. L’interface HTTP est plus limitée que l’interface native, mais elle a une meilleure compatibilité. +L'interface HTTP vous permet D'utiliser ClickHouse sur n'importe quelle plate-forme à partir de n'importe quel langage de programmation. Nous l'utilisons pour travailler à partir de Java et Perl, ainsi que des scripts shell. Dans d'autres départements, L'interface HTTP est utilisée à partir de Perl, Python et Go. L'interface HTTP est plus limitée que l'interface native, mais elle a une meilleure compatibilité. Par défaut, clickhouse-server écoute HTTP sur le port 8123 (cela peut être modifié dans la configuration). @@ -25,12 +25,12 @@ $ curl 'http://localhost:8123/ping' Ok. ``` -Envoyer la demande sous forme D’URL ‘query’ paramètre, ou comme un POSTE. Ou envoyer le début de la requête dans l’ ‘query’ paramètre, et le reste dans le POST (nous expliquerons plus tard pourquoi cela est nécessaire). La taille de L’URL est limitée à 16 Ko, alors gardez cela à l’esprit lors de l’envoi de requêtes volumineuses. +Envoyer la demande sous forme D'URL ‘query’ paramètre, ou comme un POSTE. Ou envoyer le début de la requête dans l' ‘query’ paramètre, et le reste dans le POST (nous expliquerons plus tard pourquoi cela est nécessaire). La taille de L'URL est limitée à 16 Ko, alors gardez cela à l'esprit lors de l'envoi de requêtes volumineuses. En cas de succès, vous recevez le code de réponse 200 et le résultat dans le corps de réponse. -Si une erreur se produit, vous recevez le code de réponse 500 et un texte de description de l’erreur dans le corps de la réponse. +Si une erreur se produit, vous recevez le code de réponse 500 et un texte de description de l'erreur dans le corps de la réponse. -Lorsque vous utilisez la méthode GET, ‘readonly’ est définie. En d’autres termes, pour les requêtes qui modifient les données, vous ne pouvez utiliser que la méthode POST. Vous pouvez envoyer la requête elle-même dans le corps du message ou dans le paramètre URL. +Lorsque vous utilisez la méthode GET, ‘readonly’ est définie. En d'autres termes, pour les requêtes qui modifient les données, vous ne pouvez utiliser que la méthode POST. Vous pouvez envoyer la requête elle-même dans le corps du message ou dans le paramètre URL. Exemple: @@ -54,7 +54,7 @@ X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","writ ``` Comme vous pouvez le voir, curl est un peu gênant en ce sens que les espaces doivent être échappés URL. -Bien que wget échappe à tout lui-même, nous ne recommandons pas de l’utiliser car il ne fonctionne pas bien sur HTTP 1.1 lors de l’utilisation de keep-alive et Transfer-Encoding: chunked. +Bien que wget échappe à tout lui-même, nous ne recommandons pas de l'utiliser car il ne fonctionne pas bien sur HTTP 1.1 lors de l'utilisation de keep-alive et Transfer-Encoding: chunked. ``` bash $ echo 'SELECT 1' | curl 'http://localhost:8123/' --data-binary @- @@ -77,7 +77,7 @@ ECT 1 , expected One of: SHOW TABLES, SHOW DATABASES, SELECT, INSERT, CREATE, ATTACH, RENAME, DROP, DETACH, USE, SET, OPTIMIZE., e.what() = DB::Exception ``` -Par défaut, les données sont renvoyées au format TabSeparated (pour plus d’informations, voir “Formats” section). +Par défaut, les données sont renvoyées au format TabSeparated (pour plus d'informations, voir “Formats” section). Vous utilisez la clause FORMAT de la requête pour demander tout autre format. ``` bash @@ -91,13 +91,13 @@ $ echo 'SELECT 1 FORMAT Pretty' | curl 'http://localhost:8123/?' --data-binary @ La méthode POST de transmission des données est nécessaire pour les requêtes INSERT. Dans ce cas, vous pouvez écrire le début de la requête dans le paramètre URL et utiliser POST pour transmettre les données à insérer. Les données à insérer pourraient être, par exemple, un vidage séparé par tabulation de MySQL. De cette façon, la requête INSERT remplace LOAD DATA LOCAL INFILE de MySQL. -Exemples: création d’une table: +Exemples: création d'une table: ``` bash $ echo 'CREATE TABLE t (a UInt8) ENGINE = Memory' | curl 'http://localhost:8123/' --data-binary @- ``` -Utilisation de la requête D’insertion familière pour l’insertion de données: +Utilisation de la requête D'insertion familière pour l'insertion de données: ``` bash $ echo 'INSERT INTO t VALUES (1),(2),(3)' | curl 'http://localhost:8123/' --data-binary @- @@ -109,19 +109,19 @@ Les données peuvent être envoyées séparément de la requête: $ echo '(4),(5),(6)' | curl 'http://localhost:8123/?query=INSERT%20INTO%20t%20VALUES' --data-binary @- ``` -Vous pouvez spécifier n’importe quel format de données. Le ‘Values’ le format est le même que ce qui est utilisé lors de L’écriture INSERT dans les valeurs t: +Vous pouvez spécifier n'importe quel format de données. Le ‘Values’ le format est le même que ce qui est utilisé lors de L'écriture INSERT dans les valeurs t: ``` bash $ echo '(7),(8),(9)' | curl 'http://localhost:8123/?query=INSERT%20INTO%20t%20FORMAT%20Values' --data-binary @- ``` -Pour insérer des données à partir d’un vidage séparé par des tabulations, spécifiez le format correspondant: +Pour insérer des données à partir d'un vidage séparé par des tabulations, spécifiez le format correspondant: ``` bash $ echo -ne '10\n11\n12\n' | curl 'http://localhost:8123/?query=INSERT%20INTO%20t%20FORMAT%20TabSeparated' --data-binary @- ``` -La lecture de la table des matières. Les données sont sorties dans un ordre aléatoire en raison d’un traitement de requête parallèle: +La lecture de la table des matières. Les données sont sorties dans un ordre aléatoire en raison d'un traitement de requête parallèle: ``` bash $ curl 'http://localhost:8123/?query=SELECT%20a%20FROM%20t' @@ -147,16 +147,16 @@ $ echo 'DROP TABLE t' | curl 'http://localhost:8123/' --data-binary @- Pour les requêtes réussies qui ne renvoient pas de table de données, un corps de réponse vide est renvoyé. -Vous pouvez utiliser le format de compression ClickHouse interne lors de la transmission de données. Les données compressées ont un format non standard, et vous devrez utiliser le spécial `clickhouse-compressor` programme de travail (il s’est installé avec le `clickhouse-client` paquet). Pour augmenter l’efficacité de l’insertion de données, vous pouvez désactiver la vérification de la somme de contrôle côté serveur en utilisant [http\_native\_compression\_disable\_checksumming\_on\_decompress](../operations/settings/settings.md#settings-http_native_compression_disable_checksumming_on_decompress) paramètre. +Vous pouvez utiliser le format de compression ClickHouse interne lors de la transmission de données. Les données compressées ont un format non standard, et vous devrez utiliser le spécial `clickhouse-compressor` programme de travail (il s'est installé avec le `clickhouse-client` paquet). Pour augmenter l'efficacité de l'insertion de données, vous pouvez désactiver la vérification de la somme de contrôle côté serveur en utilisant [http\_native\_compression\_disable\_checksumming\_on\_decompress](../operations/settings/settings.md#settings-http_native_compression_disable_checksumming_on_decompress) paramètre. -Si vous avez spécifié `compress=1` dans l’URL, le serveur compresse les données qu’il vous envoie. -Si vous avez spécifié `decompress=1` dans L’URL, le serveur décompresse les mêmes données que vous transmettez `POST` méthode. +Si vous avez spécifié `compress=1` dans l'URL, le serveur compresse les données qu'il vous envoie. +Si vous avez spécifié `decompress=1` dans L'URL, le serveur décompresse les mêmes données que vous transmettez `POST` méthode. -Vous pouvez également choisir d’utiliser [La compression HTTP](https://en.wikipedia.org/wiki/HTTP_compression). Pour envoyer un compressé `POST` demande, ajouter l’en-tête de requête `Content-Encoding: compression_method`. Pour que ClickHouse compresse la réponse, vous devez ajouter `Accept-Encoding: compression_method`. Supports ClickHouse `gzip`, `br`, et `deflate` [méthodes de compression](https://en.wikipedia.org/wiki/HTTP_compression#Content-Encoding_tokens). Pour activer la compression HTTP, vous devez utiliser le ClickHouse [enable\_http\_compression](../operations/settings/settings.md#settings-enable_http_compression) paramètre. Vous pouvez configurer le niveau de compression des données dans le [http\_zlib\_compression\_level](#settings-http_zlib_compression_level) pour toutes les méthodes de compression. +Vous pouvez également choisir d'utiliser [La compression HTTP](https://en.wikipedia.org/wiki/HTTP_compression). Pour envoyer un compressé `POST` demande, ajouter l'en-tête de requête `Content-Encoding: compression_method`. Pour que ClickHouse compresse la réponse, vous devez ajouter `Accept-Encoding: compression_method`. Supports ClickHouse `gzip`, `br`, et `deflate` [méthodes de compression](https://en.wikipedia.org/wiki/HTTP_compression#Content-Encoding_tokens). Pour activer la compression HTTP, vous devez utiliser le ClickHouse [enable\_http\_compression](../operations/settings/settings.md#settings-enable_http_compression) paramètre. Vous pouvez configurer le niveau de compression des données dans le [http\_zlib\_compression\_level](#settings-http_zlib_compression_level) pour toutes les méthodes de compression. -Vous pouvez l’utiliser pour réduire le trafic réseau lors de la transmission d’une grande quantité de données, ou pour créer des vidages qui sont immédiatement compressés. +Vous pouvez l'utiliser pour réduire le trafic réseau lors de la transmission d'une grande quantité de données, ou pour créer des vidages qui sont immédiatement compressés. -Exemples d’envoi de données avec compression: +Exemples d'envoi de données avec compression: ``` bash #Sending data to the server: @@ -169,7 +169,7 @@ $ echo "SELECT 1" | gzip -c | curl -sS --data-binary @- -H 'Content-Encoding: gz !!! note "Note" Certains clients HTTP peuvent décompresser les données du serveur par défaut (avec `gzip` et `deflate`) et vous pouvez obtenir des données décompressées même si vous utilisez les paramètres de compression correctement. -Vous pouvez utiliser l’ ‘database’ Paramètre URL pour spécifier la base de données par défaut. +Vous pouvez utiliser l' ‘database’ Paramètre URL pour spécifier la base de données par défaut. ``` bash $ echo 'SELECT number FROM numbers LIMIT 10' | curl 'http://localhost:8123/?database=system' --data-binary @- @@ -185,11 +185,11 @@ $ echo 'SELECT number FROM numbers LIMIT 10' | curl 'http://localhost:8123/?data 9 ``` -Par défaut, la base de données enregistrée dans les paramètres du serveur est utilisée comme base de données par défaut. Par défaut, c’est la base de données appelée ‘default’. Alternativement, vous pouvez toujours spécifier la base de données en utilisant un point avant le nom de la table. +Par défaut, la base de données enregistrée dans les paramètres du serveur est utilisée comme base de données par défaut. Par défaut, c'est la base de données appelée ‘default’. Alternativement, vous pouvez toujours spécifier la base de données en utilisant un point avant le nom de la table. -Le nom d’utilisateur et le mot de passe peuvent être indiqués de l’une des trois façons suivantes: +Le nom d'utilisateur et le mot de passe peuvent être indiqués de l'une des trois façons suivantes: -1. Utilisation de L’authentification de base HTTP. Exemple: +1. Utilisation de L'authentification de base HTTP. Exemple: @@ -197,7 +197,7 @@ Le nom d’utilisateur et le mot de passe peuvent être indiqués de l’une des $ echo 'SELECT 1' | curl 'http://user:password@localhost:8123/' -d @- ``` -1. Dans le ‘user’ et ‘password’ Les paramètres d’URL. Exemple: +1. Dans le ‘user’ et ‘password’ Les paramètres d'URL. Exemple: @@ -213,10 +213,10 @@ $ echo 'SELECT 1' | curl 'http://localhost:8123/?user=user&password=password' -d $ echo 'SELECT 1' | curl -H 'X-ClickHouse-User: user' -H 'X-ClickHouse-Key: password' 'http://localhost:8123/' -d @- ``` -Si le nom d’utilisateur n’est spécifié, le `default` le nom est utilisé. Si le mot de passe n’est spécifié, le mot de passe vide est utilisé. -Vous pouvez également utiliser les paramètres D’URL pour spécifier des paramètres pour le traitement d’une seule requête ou de profils entiers de paramètres. Exemple: http: / / localhost: 8123/?profil = web & max\_rows\_to\_read=1000000000 & query=sélectionner + 1 +Si le nom d'utilisateur n'est spécifié, le `default` le nom est utilisé. Si le mot de passe n'est spécifié, le mot de passe vide est utilisé. +Vous pouvez également utiliser les paramètres D'URL pour spécifier des paramètres pour le traitement d'une seule requête ou de profils entiers de paramètres. Exemple: http: / / localhost: 8123/?profil = web & max\_rows\_to\_read=1000000000 & query=sélectionner + 1 -Pour plus d’informations, voir le [Paramètre](../operations/settings/index.md) section. +Pour plus d'informations, voir le [Paramètre](../operations/settings/index.md) section. ``` bash $ echo 'SELECT number FROM system.numbers LIMIT 10' | curl 'http://localhost:8123/?' --data-binary @- @@ -232,11 +232,11 @@ $ echo 'SELECT number FROM system.numbers LIMIT 10' | curl 'http://localhost:812 9 ``` -Pour plus d’informations sur les autres paramètres, consultez la section “SET”. +Pour plus d'informations sur les autres paramètres, consultez la section “SET”. -De même, vous pouvez utiliser des sessions ClickHouse dans le protocole HTTP. Pour ce faire, vous devez ajouter l’ `session_id` GET paramètre à la demande. Vous pouvez utiliser n’importe quelle chaîne comme ID de session. Par défaut, la session est terminée après 60 secondes d’inactivité. Pour modifier ce délai d’attente, de modifier la `default_session_timeout` dans la configuration du serveur, ou ajoutez le `session_timeout` GET paramètre à la demande. Pour vérifier l’état de la session, utilisez `session_check=1` paramètre. Une seule requête à la fois peut être exécutée dans une seule session. +De même, vous pouvez utiliser des sessions ClickHouse dans le protocole HTTP. Pour ce faire, vous devez ajouter l' `session_id` GET paramètre à la demande. Vous pouvez utiliser n'importe quelle chaîne comme ID de session. Par défaut, la session est terminée après 60 secondes d'inactivité. Pour modifier ce délai d'attente, de modifier la `default_session_timeout` dans la configuration du serveur, ou ajoutez le `session_timeout` GET paramètre à la demande. Pour vérifier l'état de la session, utilisez `session_check=1` paramètre. Une seule requête à la fois peut être exécutée dans une seule session. -Vous pouvez recevoir des informations sur le déroulement d’une requête en `X-ClickHouse-Progress` en-têtes de réponse. Pour ce faire, activez [send\_progress\_in\_http\_headers](../operations/settings/settings.md#settings-send_progress_in_http_headers). Exemple de l’en-tête de séquence: +Vous pouvez recevoir des informations sur le déroulement d'une requête en `X-ClickHouse-Progress` en-têtes de réponse. Pour ce faire, activez [send\_progress\_in\_http\_headers](../operations/settings/settings.md#settings-send_progress_in_http_headers). Exemple de l'en-tête de séquence: ``` text X-ClickHouse-Progress: {"read_rows":"2752512","read_bytes":"240570816","total_rows_to_read":"8880128"} @@ -244,7 +244,7 @@ X-ClickHouse-Progress: {"read_rows":"5439488","read_bytes":"482285394","total_ro X-ClickHouse-Progress: {"read_rows":"8783786","read_bytes":"819092887","total_rows_to_read":"8880128"} ``` -Possibles champs d’en-tête: +Possibles champs d'en-tête: - `read_rows` — Number of rows read. - `read_bytes` — Volume of data read in bytes. @@ -252,18 +252,18 @@ Possibles champs d’en-tête: - `written_rows` — Number of rows written. - `written_bytes` — Volume of data written in bytes. -Les requêtes en cours d’exécution ne s’arrêtent pas automatiquement si la connexion HTTP est perdue. L’analyse et le formatage des données sont effectués côté serveur et l’utilisation du réseau peut s’avérer inefficace. -Facultatif ‘query\_id’ le paramètre peut être passé comme ID de requête (n’importe quelle chaîne). Pour plus d’informations, consultez la section “Settings, replace\_running\_query”. +Les requêtes en cours d'exécution ne s'arrêtent pas automatiquement si la connexion HTTP est perdue. L'analyse et le formatage des données sont effectués côté serveur et l'utilisation du réseau peut s'avérer inefficace. +Facultatif ‘query\_id’ le paramètre peut être passé comme ID de requête (n'importe quelle chaîne). Pour plus d'informations, consultez la section “Settings, replace\_running\_query”. -Facultatif ‘quota\_key’ le paramètre peut être passé comme clé de quota (n’importe quelle chaîne). Pour plus d’informations, consultez la section “Quotas”. +Facultatif ‘quota\_key’ le paramètre peut être passé comme clé de quota (n'importe quelle chaîne). Pour plus d'informations, consultez la section “Quotas”. -L’interface HTTP permet de transmettre des données externes (tables temporaires externes) pour l’interrogation. Pour plus d’informations, consultez la section “External data for query processing”. +L'interface HTTP permet de transmettre des données externes (tables temporaires externes) pour l'interrogation. Pour plus d'informations, consultez la section “External data for query processing”. ## Tampon De Réponse {#response-buffering} -Vous pouvez activer la mise en mémoire tampon des réponses côté serveur. Le `buffer_size` et `wait_end_of_query` Les paramètres D’URL sont fournis à cette fin. +Vous pouvez activer la mise en mémoire tampon des réponses côté serveur. Le `buffer_size` et `wait_end_of_query` Les paramètres D'URL sont fournis à cette fin. -`buffer_size` détermine le nombre d’octets dans le résultat de tampon dans la mémoire du serveur. Si un corps de résultat est supérieur à ce seuil, le tampon est écrit sur le canal HTTP et les données restantes sont envoyées directement au canal HTTP. +`buffer_size` détermine le nombre d'octets dans le résultat de tampon dans la mémoire du serveur. Si un corps de résultat est supérieur à ce seuil, le tampon est écrit sur le canal HTTP et les données restantes sont envoyées directement au canal HTTP. Pour vous assurer que la réponse entière est mise en mémoire tampon, définissez `wait_end_of_query=1`. Dans ce cas, les données ne sont pas stockées dans la mémoire tampon temporaire du serveur de fichiers. @@ -273,11 +273,11 @@ Exemple: $ curl -sS 'http://localhost:8123/?max_result_bytes=4000000&buffer_size=3000000&wait_end_of_query=1' -d 'SELECT toUInt8(number) FROM system.numbers LIMIT 9000000 FORMAT RowBinary' ``` -Utilisez la mise en mémoire tampon pour éviter les situations où une erreur de traitement de requête s’est produite après l’envoi du code de réponse et des en-têtes HTTP au client. Dans cette situation, un message d’erreur est écrit à la fin du corps de la réponse, et du côté client, l’erreur ne peut être détectée qu’à l’étape d’analyse. +Utilisez la mise en mémoire tampon pour éviter les situations où une erreur de traitement de requête s'est produite après l'envoi du code de réponse et des en-têtes HTTP au client. Dans cette situation, un message d'erreur est écrit à la fin du corps de la réponse, et du côté client, l'erreur ne peut être détectée qu'à l'étape d'analyse. -### Requêtes Avec paramètres {#cli-queries-with-parameters} +### Requêtes avec paramètres {#cli-queries-with-parameters} -Vous pouvez créer une requête avec paramètres et transmettre des valeurs des paramètres de la requête HTTP. Pour plus d’informations, voir [Requêtes avec des paramètres pour CLI](cli.md#cli-queries-with-parameters). +Vous pouvez créer une requête avec paramètres et transmettre des valeurs des paramètres de la requête HTTP. Pour plus d'informations, voir [Requêtes avec des paramètres pour CLI](cli.md#cli-queries-with-parameters). ### Exemple {#example} @@ -287,52 +287,57 @@ $ curl -sS "
?param_id=2¶m_phrase=test" -d "SELECT * FROM table WHER ## Interface HTTP prédéfinie {#predefined_http_interface} -ClickHouse prend en charge des requêtes spécifiques via L’interface HTTP. Par exemple, vous pouvez écrire des données dans un tableau comme suit: +ClickHouse prend en charge des requêtes spécifiques via L'interface HTTP. Par exemple, vous pouvez écrire des données dans un tableau comme suit: ``` bash $ echo '(4),(5),(6)' | curl 'http://localhost:8123/?query=INSERT%20INTO%20t%20VALUES' --data-binary @- ``` -ClickHouse prend également en charge L’Interface HTTP prédéfinie qui peut vous aider à une intégration plus facile avec des outils tiers tels que [Prometheus exportateur](https://github.com/percona-lab/clickhouse_exporter). +ClickHouse prend également en charge L'Interface HTTP prédéfinie qui peut vous aider à une intégration plus facile avec des outils tiers tels que [Prometheus exportateur](https://github.com/percona-lab/clickhouse_exporter). Exemple: -- Tout d’abord, ajoutez cette section au fichier de configuration du serveur: +- Tout d'abord, ajoutez cette section au fichier de configuration du serveur: ``` xml - - /metrics - GET - + + /predefined_query + POST,GET + + predefined_query_handler SELECT * FROM system.metrics LIMIT 5 FORMAT Template SETTINGS format_template_resultset = 'prometheus_template_output_format_resultset', format_template_row = 'prometheus_template_output_format_row', format_template_rows_between_delimiter = '\n' - - + + + ... + ... ``` -- Vous pouvez maintenant demander l’url directement pour les données au format Prometheus: +- Vous pouvez maintenant demander l'url directement pour les données au format Prometheus: ``` bash -curl -vvv 'http://localhost:8123/metrics' +$ curl -v 'http://localhost:8123/predefined_query' * Trying ::1... * Connected to localhost (::1) port 8123 (#0) -> GET /metrics HTTP/1.1 +> GET /predefined_query HTTP/1.1 > Host: localhost:8123 > User-Agent: curl/7.47.0 > Accept: */* > < HTTP/1.1 200 OK -< Date: Wed, 27 Nov 2019 08:54:25 GMT +< Date: Tue, 28 Apr 2020 08:52:56 GMT < Connection: Keep-Alive < Content-Type: text/plain; charset=UTF-8 -< X-ClickHouse-Server-Display-Name: i-tl62qd0o +< X-ClickHouse-Server-Display-Name: i-mloy5trc < Transfer-Encoding: chunked -< X-ClickHouse-Query-Id: f39235f6-6ed7-488c-ae07-c7ceafb960f6 +< X-ClickHouse-Query-Id: 96fe0052-01e6-43ce-b12a-6b7370de6e8a +< X-ClickHouse-Format: Template +< X-ClickHouse-Timezone: Asia/Shanghai < Keep-Alive: timeout=3 < X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} < @@ -356,117 +361,62 @@ curl -vvv 'http://localhost:8123/metrics' # TYPE "ReplicatedSend" counter "ReplicatedSend" 0 +* Connection #0 to host localhost left intact + + * Connection #0 to host localhost left intact ``` -Comme vous pouvez le voir dans l’exemple, si `` est configuré dans la configuration.fichier xml, ClickHouse fera correspondre les requêtes HTTP reçues au type prédéfini dans ``, puis ClickHouse exécutera la requête prédéfinie correspondante si la correspondance est réussie. +Comme vous pouvez le voir dans l'exemple, si `` est configuré dans la configuration.fichier xml et `` peut contenir beaucoup `s`. ClickHouse fera correspondre les requêtes HTTP reçues au type prédéfini `` et la première appariés exécute le gestionnaire. Ensuite, ClickHouse exécutera la requête prédéfinie correspondante si la correspondance est réussie. -Maintenant `` pouvez configurer ``, ``, ``, `` et `` . +> Maintenant `` pouvez configurer ``, ``, ``,``: +> `` est responsable de la correspondance de la partie méthode de la requête HTTP. `` entièrement conforme à la définition de [méthode](https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods) dans le protocole HTTP. C'est une option de configuration. S'il n'est pas défini dans le fichier de configuration, il ne correspond pas à la partie méthode de la requête HTTP. +> +> `` est responsable de la correspondance de la partie url de la requête HTTP. Il est compatible avec [RE2](https://github.com/google/re2)s 'expressions régulières. C'est une option de configuration. S'il n'est pas défini dans le fichier de configuration, il ne correspond pas à la partie url de la requête HTTP. +> +> `` est responsable de la correspondance de la partie d'en-tête de la requête HTTP. Il est compatible avec les expressions régulières de RE2. C'est une option de configuration. S'il n'est pas défini dans le fichier de configuration, il ne correspond pas à la partie d'en-tête de la requête HTTP. +> +> `` contient la partie de traitement principale. Maintenant `` pouvez configurer ``, ``, ``, ``, ``, ``. +> \> `` prend actuellement en charge trois types: **predefined\_query\_handler**, **dynamic\_query\_handler**, **statique**. +> \> +> \> `` - utiliser avec le type predefined\_query\_handler, exécute la requête lorsque le gestionnaire est appelé. +> \> +> \> `` - utiliser avec le type dynamic\_query\_handler, extrait et exécute la valeur correspondant au `` valeur dans les paramètres de requête HTTP. +> \> +> \> `` - utiliser avec le type statique, code d'état de réponse. +> \> +> \> `` - utiliser avec statique type, réponse [content-type](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type). +> \> +> \> `` - utiliser avec le type statique, le contenu de la réponse envoyé au client, lors de l'utilisation du préfixe ‘file://’ ou ‘config://’, trouver le contenu du fichier ou de la configuration envoyer au client. -## root\_handler {#root_handler} - -`` renvoie le contenu spécifié pour la requête de chemin racine. Le contenu de retour spécifique est configuré par `http_server_default_response` dans la configuration.XML. si non spécifié, le retour **OK.** - -`http_server_default_response` n’est pas défini et une requête HTTP est envoyée à ClickHouse. Le résultat est comme suit: - -``` xml - - - -``` - - $ curl 'http://localhost:8123' - Ok. - -`http_server_default_response` est défini et une requête HTTP est envoyée à ClickHouse. Le résultat est comme suit: - -``` xml -
]]> - - - - -``` - - $ curl 'http://localhost:8123' -
% - -## ping\_handler {#ping_handler} - -`` peut être utilisé pour sonder la santé du serveur clickhouse actuel. Lorsque le serveur HTTP ClickHouse est normal, l’accès à ClickHouse via `` sera de retour **OK.**. - -Exemple: - -``` xml - - /ping - -``` - -``` bash -$ curl 'http://localhost:8123/ping' -Ok. -``` - -## replicas\_status\_handler {#replicas_status_handler} - -`` est utilisé pour détecter l’état du nœud de réplica et le retour **OK.** si le nœud réplique n’a pas de délai. S’il y a un retard, renvoyez le retard spécifique. La valeur de `` prend en charge la personnalisation. Si vous ne spécifiez pas ``, ClickHouse réglage par défaut `` être **/ replicas\_status**. - -Exemple: - -``` xml - - /replicas_status - -``` - -Aucun retard de cas: - -``` bash -$ curl 'http://localhost:8123/replicas_status' -Ok. -``` - -Retard de cas: - -``` bash -$ curl 'http://localhost:8123/replicas_status' -db.stats: Absolute delay: 22. Relative delay: 22. -``` +Viennent ensuite les méthodes de configuration pour les différents ``. ## predefined\_query\_handler {#predefined_query_handler} -Vous pouvez configurer ``, ``, `` et `` dans ``. +`` prend en charge les paramètres de réglage et les valeurs query\_params. Vous pouvez configurer `` dans le type de ``. -`` est responsable de la correspondance de la partie méthode de la requête HTTP. `` entièrement conforme à la définition de [méthode](https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods) dans le protocole HTTP. C’est une option de configuration. S’il n’est pas défini dans le fichier de configuration, il ne correspond pas à la partie méthode de la requête HTTP +`` la valeur est une requête prédéfinie de ``, qui est exécuté par ClickHouse lorsqu'une requête HTTP est mise en correspondance et que le résultat de la requête est renvoyé. C'est une configuration incontournable. -`` est responsable de la correspondance de la partie url de la requête HTTP. Il est compatible avec [RE2](https://github.com/google/re2)s ’expressions régulières. C’est une option de configuration. S’il n’est pas défini dans le fichier de configuration, il ne correspond pas à la partie url de la requête HTTP - -`` est responsable de la correspondance de la partie d’en-tête de la requête HTTP. Il est compatible avec les expressions régulières de RE2. C’est une option de configuration. S’il n’est pas défini dans le fichier de configuration, il ne correspond pas à la partie d’en-tête de la requête HTTP - -`` la valeur est une requête prédéfinie de ``, qui est exécuté par ClickHouse lorsqu’une requête HTTP est mise en correspondance et que le résultat de la requête est renvoyé. C’est une configuration incontournable. - -`` prend en charge les paramètres de réglage et les valeurs query\_params. - -L’exemple suivant définit les valeurs de `max_threads` et `max_alter_threads` Paramètres, puis interroge la table système pour vérifier si ces paramètres ont été définis avec succès. +L'exemple suivant définit les valeurs de `max_threads` et `max_alter_threads` Paramètres, puis interroge la table système pour vérifier si ces paramètres ont été définis avec succès. Exemple: ``` xml - - + + + [^/]+)(/(?P[^/]+))?]]> GET TEST_HEADER_VALUE [^/]+)(/(?P[^/]+))?]]> - [^/]+)(/(?P[^/]+))?]]> - + + predefined_query_handler SELECT value FROM system.settings WHERE name = {name_1:String} SELECT name, value FROM system.settings WHERE name = {name_2:String} - - - + + + ``` ``` bash @@ -475,37 +425,193 @@ $ curl -H 'XXX:TEST_HEADER_VALUE' -H 'PARAMS_XXX:max_threads' 'http://localhost: max_alter_threads 2 ``` -!!! note "Note" - Dans un ``, un `` prend en charge un seul `` d’un type d’insertion. +!!! note "précaution" + Dans un `` prend en charge un seul `` d'un type d'insertion. ## dynamic\_query\_handler {#dynamic_query_handler} -`` que `` augmenter `` . +Dans ``, la requête est écrite sous la forme de param de la requête HTTP. La différence est que dans ``, la requête est écrite dans le fichier de configuration. Vous pouvez configurer `` dans ``. -Clickhouse extrait et exécute la valeur correspondant au `` valeur dans l’url de la requête HTTP. -ClickHouse réglage par défaut `` être `/query` . C’est une option de configuration. Si il n’y a pas de définition dans le fichier de configuration, le paramètre n’est pas passé. +Clickhouse extrait et exécute la valeur correspondant au `` valeur dans l'url de la requête HTTP. La valeur par défaut de `` être `/query` . C'est une option de configuration. Si il n'y a pas de définition dans le fichier de configuration, le paramètre n'est pas passé. -Pour expérimenter cette fonctionnalité, l’exemple définit les valeurs de max\_threads et max\_alter\_threads et demande si les paramètres ont été définis avec succès. -La différence est que dans ``, la requête est écrite dans le fichier de configuration. Mais dans ``, la requête est écrite sous la forme de param de la requête HTTP. +Pour expérimenter cette fonctionnalité, l'exemple définit les valeurs de max\_threads et max\_alter\_threads et demande si les paramètres ont été définis avec succès. Exemple: ``` xml - - - - TEST_HEADER_VALUE_DYNAMIC - [^/]+)(/(?P[^/]+))?]]> - + + + + TEST_HEADER_VALUE_DYNAMIC + + dynamic_query_handler query_param - - + + + ``` ``` bash -$ curl -H 'XXX:TEST_HEADER_VALUE_DYNAMIC' -H 'PARAMS_XXX:max_threads' 'http://localhost:8123/?query_param=SELECT%20value%20FROM%20system.settings%20where%20name%20=%20%7Bname_1:String%7D%20OR%20name%20=%20%7Bname_2:String%7D&max_threads=1&max_alter_threads=2¶m_name_2=max_alter_threads' -1 -2 +$ curl -H 'XXX:TEST_HEADER_VALUE_DYNAMIC' 'http://localhost:8123/own?max_threads=1&max_alter_threads=2¶m_name_1=max_threads¶m_name_2=max_alter_threads&query_param=SELECT%20name,value%20FROM%20system.settings%20where%20name%20=%20%7Bname_1:String%7D%20OR%20name%20=%20%7Bname_2:String%7D' +max_threads 1 +max_alter_threads 2 +``` + +## statique {#static} + +`` peut-retour [content\_type](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type), [statut](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status) et response\_content. response\_content peut renvoyer le contenu spécifié + +Exemple: + +De retour d'un message. + +``` xml + + + GET + xxx + /hi + + static + 402 + text/html; charset=UTF-8 + Say Hi! + + + +``` + +``` bash +$ curl -vv -H 'XXX:xxx' 'http://localhost:8123/hi' +* Trying ::1... +* Connected to localhost (::1) port 8123 (#0) +> GET /hi HTTP/1.1 +> Host: localhost:8123 +> User-Agent: curl/7.47.0 +> Accept: */* +> XXX:xxx +> +< HTTP/1.1 402 Payment Required +< Date: Wed, 29 Apr 2020 03:51:26 GMT +< Connection: Keep-Alive +< Content-Type: text/html; charset=UTF-8 +< Transfer-Encoding: chunked +< Keep-Alive: timeout=3 +< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} +< +* Connection #0 to host localhost left intact +Say Hi!% +``` + +Trouvez le contenu de la configuration envoyer au client. + +``` xml +
]]> + + + + GET + xxx + /get_config_static_handler + + static + config://get_config_static_handler + + + +``` + +``` bash +$ curl -v -H 'XXX:xxx' 'http://localhost:8123/get_config_static_handler' +* Trying ::1... +* Connected to localhost (::1) port 8123 (#0) +> GET /get_config_static_handler HTTP/1.1 +> Host: localhost:8123 +> User-Agent: curl/7.47.0 +> Accept: */* +> XXX:xxx +> +< HTTP/1.1 200 OK +< Date: Wed, 29 Apr 2020 04:01:24 GMT +< Connection: Keep-Alive +< Content-Type: text/plain; charset=UTF-8 +< Transfer-Encoding: chunked +< Keep-Alive: timeout=3 +< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} +< +* Connection #0 to host localhost left intact +
% +``` + +Trouvez le contenu du fichier envoyé au client. + +``` xml + + + GET + xxx + /get_absolute_path_static_handler + + static + text/html; charset=UTF-8 + file:///absolute_path_file.html + + + + GET + xxx + /get_relative_path_static_handler + + static + text/html; charset=UTF-8 + file://./relative_path_file.html + + + +``` + +``` bash +$ user_files_path='/var/lib/clickhouse/user_files' +$ sudo echo "Relative Path File" > $user_files_path/relative_path_file.html +$ sudo echo "Absolute Path File" > $user_files_path/absolute_path_file.html +$ curl -vv -H 'XXX:xxx' 'http://localhost:8123/get_absolute_path_static_handler' +* Trying ::1... +* Connected to localhost (::1) port 8123 (#0) +> GET /get_absolute_path_static_handler HTTP/1.1 +> Host: localhost:8123 +> User-Agent: curl/7.47.0 +> Accept: */* +> XXX:xxx +> +< HTTP/1.1 200 OK +< Date: Wed, 29 Apr 2020 04:18:16 GMT +< Connection: Keep-Alive +< Content-Type: text/html; charset=UTF-8 +< Transfer-Encoding: chunked +< Keep-Alive: timeout=3 +< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} +< +Absolute Path File +* Connection #0 to host localhost left intact +$ curl -vv -H 'XXX:xxx' 'http://localhost:8123/get_relative_path_static_handler' +* Trying ::1... +* Connected to localhost (::1) port 8123 (#0) +> GET /get_relative_path_static_handler HTTP/1.1 +> Host: localhost:8123 +> User-Agent: curl/7.47.0 +> Accept: */* +> XXX:xxx +> +< HTTP/1.1 200 OK +< Date: Wed, 29 Apr 2020 04:18:31 GMT +< Connection: Keep-Alive +< Content-Type: text/html; charset=UTF-8 +< Transfer-Encoding: chunked +< Keep-Alive: timeout=3 +< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} +< +Relative Path File +* Connection #0 to host localhost left intact ``` [Article Original](https://clickhouse.tech/docs/en/interfaces/http_interface/) diff --git a/docs/fr/interfaces/index.md b/docs/fr/interfaces/index.md index 8c3e1c82403..bba875be1f2 100644 --- a/docs/fr/interfaces/index.md +++ b/docs/fr/interfaces/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 -toc_folder_title: Interfaces +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: Interface toc_priority: 14 toc_title: Introduction --- @@ -13,12 +13,12 @@ ClickHouse fournit deux interfaces réseau (les deux peuvent être encapsulées - [HTTP](http.md) qui est documenté et facile à utiliser directement. - [Natif de TCP](tcp.md) qui a moins de frais généraux. -Dans la plupart des cas, il est recommandé d’utiliser un outil ou une bibliothèque approprié au lieu d’interagir directement avec ceux-ci. Officiellement pris en charge par Yandex sont les suivants: +Dans la plupart des cas, il est recommandé d'utiliser un outil ou une bibliothèque approprié au lieu d'interagir directement avec ceux-ci. Officiellement pris en charge par Yandex sont les suivants: - [Client de ligne de commande](cli.md) - [JDBC](jdbc.md) - [Pilote ODBC](odbc.md) -- [Bibliothèque client c++](cpp.md) +- [Bibliothèque client c++ ](cpp.md) Il existe également un large éventail de bibliothèques tierces pour travailler avec ClickHouse: diff --git a/docs/fr/interfaces/jdbc.md b/docs/fr/interfaces/jdbc.md index 1bd9dfe8e9f..dc68dcb9831 100644 --- a/docs/fr/interfaces/jdbc.md +++ b/docs/fr/interfaces/jdbc.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 22 toc_title: JDBC --- diff --git a/docs/fr/interfaces/mysql.md b/docs/fr/interfaces/mysql.md index c20d88ca905..b3ec2e30e26 100644 --- a/docs/fr/interfaces/mysql.md +++ b/docs/fr/interfaces/mysql.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 20 toc_title: Interface MySQL --- @@ -13,7 +13,7 @@ ClickHouse prend en charge le protocole de fil MySQL. Il peut être activé par 9004 ``` -Exemple de connexion à l’aide d’outil de ligne de commande `mysql`: +Exemple de connexion à l'aide d'outil de ligne de commande `mysql`: ``` bash $ mysql --protocol tcp -u default -P 9004 @@ -38,7 +38,7 @@ mysql> ``` Pour la compatibilité avec tous les clients MySQL, il est recommandé de spécifier le mot de passe utilisateur avec [double SHA1](../operations/settings/settings-users.md#password_double_sha1_hex) dans le fichier de configuration. -Si le mot de passe de l’utilisateur est spécifié [SHA256](../operations/settings/settings-users.md#password_sha256_hex), certains clients ne pourront pas s’authentifier (mysqljs et anciennes versions de l’outil de ligne de commande mysql). +Si le mot de passe de l'utilisateur est spécifié [SHA256](../operations/settings/settings-users.md#password_sha256_hex), certains clients ne pourront pas s'authentifier (mysqljs et anciennes versions de l'outil de ligne de commande mysql). Restriction: diff --git a/docs/fr/interfaces/odbc.md b/docs/fr/interfaces/odbc.md index 1f366f77d51..991dc833502 100644 --- a/docs/fr/interfaces/odbc.md +++ b/docs/fr/interfaces/odbc.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 23 toc_title: Pilote ODBC --- diff --git a/docs/fr/interfaces/tcp.md b/docs/fr/interfaces/tcp.md index 3103b1c105f..d1fa251e0e4 100644 --- a/docs/fr/interfaces/tcp.md +++ b/docs/fr/interfaces/tcp.md @@ -1,12 +1,12 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 18 toc_title: Interface Native (TCP) --- # Interface Native (TCP) {#native-interface-tcp} -Le protocole natif est utilisé dans le [client de ligne de commande](cli.md), pour la communication inter-serveur pendant le traitement de requête distribué, et également dans d’autres programmes C++. Malheureusement, le protocole clickhouse natif n’a pas encore de spécification formelle, mais il peut être rétro-conçu à partir du code source ClickHouse (démarrage [ici](https://github.com/ClickHouse/ClickHouse/tree/master/dbms/Client)) et/ou en interceptant et en analysant le trafic TCP. +Le protocole natif est utilisé dans le [client de ligne de commande](cli.md), pour la communication inter-serveur pendant le traitement de requête distribué, et également dans d'autres programmes C++. Malheureusement, le protocole clickhouse natif n'a pas encore de spécification formelle, mais il peut être rétro-conçu à partir du code source ClickHouse (démarrage [ici](https://github.com/ClickHouse/ClickHouse/tree/master/src/Client)) et/ou en interceptant et en analysant le trafic TCP. [Article Original](https://clickhouse.tech/docs/en/interfaces/tcp/) diff --git a/docs/fr/interfaces/third-party/client-libraries.md b/docs/fr/interfaces/third-party/client-libraries.md index ec2fe4e7002..f07a27aec6c 100644 --- a/docs/fr/interfaces/third-party/client-libraries.md +++ b/docs/fr/interfaces/third-party/client-libraries.md @@ -1,14 +1,14 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 26 toc_title: "Biblioth\xE8ques Clientes" --- -# Bibliothèques Clientes De développeurs Tiers {#client-libraries-from-third-party-developers} +# Bibliothèques clientes de développeurs tiers {#client-libraries-from-third-party-developers} !!! warning "Avertissement" - Yandex ne **pas** maintenir les bibliothèques énumérées ci-dessous et n’ont pas fait de tests approfondis pour assurer leur qualité. + Yandex ne **pas** maintenir les bibliothèques énumérées ci-dessous et n'ont pas fait de tests approfondis pour assurer leur qualité. - Python - [infi.clickhouse\_orm](https://github.com/Infinidat/infi.clickhouse_orm) @@ -21,7 +21,7 @@ toc_title: "Biblioth\xE8ques Clientes" - [bozerkins / clickhouse-client](https://packagist.org/packages/bozerkins/clickhouse-client) - [simpod / clickhouse-client](https://packagist.org/packages/simpod/clickhouse-client) - [seva-code/php-cliquez-maison-client](https://packagist.org/packages/seva-code/php-click-house-client) - - [Client Seasclick c++](https://github.com/SeasX/SeasClick) + - [Client Seasclick c++ ](https://github.com/SeasX/SeasClick) - Aller - [clickhouse](https://github.com/kshvakov/clickhouse/) - [aller-clickhouse](https://github.com/roistat/go-clickhouse) @@ -39,7 +39,7 @@ toc_title: "Biblioth\xE8ques Clientes" - [clickhouse-activerecord](https://github.com/PNixx/clickhouse-activerecord) - R - [clickhouse-r](https://github.com/hannesmuehleisen/clickhouse-r) - - [RClickhouse](https://github.com/IMSMWU/RClickhouse) + - [RClickHouse](https://github.com/IMSMWU/RClickHouse) - Java - [clickhouse-client-java](https://github.com/VirtusAI/clickhouse-client-java) - [clickhouse-client](https://github.com/Ecwid/clickhouse-client) @@ -53,6 +53,7 @@ toc_title: "Biblioth\xE8ques Clientes" - [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net) - Elixir - [clickhousex](https://github.com/appodeal/clickhousex/) + - [pilier](https://github.com/sofakingworld/pillar) - Nim - [nim-clickhouse](https://github.com/leonardoce/nim-clickhouse) diff --git a/docs/fr/interfaces/third-party/gui.md b/docs/fr/interfaces/third-party/gui.md index 341784f582f..af25f86ee54 100644 --- a/docs/fr/interfaces/third-party/gui.md +++ b/docs/fr/interfaces/third-party/gui.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 28 toc_title: Les Interfaces Visuelles --- -# Interfaces Visuelles De développeurs Tiers {#visual-interfaces-from-third-party-developers} +# Interfaces visuelles de développeurs tiers {#visual-interfaces-from-third-party-developers} ## Open-Source {#open-source} @@ -15,10 +15,10 @@ Interface Web pour ClickHouse dans le [Tabix](https://github.com/tabixio/tabix) Caractéristique: -- Fonctionne avec ClickHouse directement à partir du navigateur, sans avoir besoin d’installer un logiciel supplémentaire. +- Fonctionne avec ClickHouse directement à partir du navigateur, sans avoir besoin d'installer un logiciel supplémentaire. - Éditeur de requête avec coloration syntaxique. -- L’Auto-complétion des commandes. -- Outils d’analyse graphique de l’exécution des requêtes. +- L'Auto-complétion des commandes. +- Outils d'analyse graphique de l'exécution des requêtes. - Options de jeu de couleurs. [Documentation Tabix](https://tabix.io/doc/). @@ -31,7 +31,7 @@ Caractéristique: - Générateur de requêtes avec coloration syntaxique. Affichez la réponse dans une table ou une vue JSON. - Exporter les résultats de la requête au format CSV ou JSON. -- Liste des processus avec des descriptions. Le mode d’écriture. Capacité à arrêter (`KILL`) processus. +- Liste des processus avec des descriptions. Le mode d'écriture. Capacité à arrêter (`KILL`) processus. - Base de données du graphique. Affiche toutes les tables et leurs colonnes avec des informations supplémentaires. - Une vue rapide de la taille de la colonne. - La configuration du serveur. @@ -40,7 +40,7 @@ Les fonctionnalités suivantes sont prévues pour le développement: - Gestion de base de données. - La gestion des utilisateurs. -- En temps réel l’analyse des données. +- En temps réel l'analyse des données. - Surveillance de Cluster. - La gestion de Cluster. - Suivi des tables répliquées et Kafka. @@ -93,11 +93,15 @@ Caractéristique: [clickhouse-flamegraph](https://github.com/Slach/clickhouse-flamegraph) est un outil spécialisé pour visualiser la `system.trace_log` comme [flamegraph](http://www.brendangregg.com/flamegraphs.html). +### clickhouse-plantuml {#clickhouse-plantuml} + +[cickhouse-plantuml](https://pypi.org/project/clickhouse-plantuml/) est un script à générer [PlantUML](https://plantuml.com/) schéma des schémas des tableaux. + ## Commercial {#commercial} ### DataGrip {#datagrip} -[DataGrip](https://www.jetbrains.com/datagrip/) est un IDE de base de données de JetBrains avec un support dédié pour ClickHouse. Il est également intégré dans D’autres outils basés sur IntelliJ: PyCharm, IntelliJ IDEA, GoLand, PhpStorm et autres. +[DataGrip](https://www.jetbrains.com/datagrip/) est un IDE de base de données de JetBrains avec un support dédié pour ClickHouse. Il est également intégré dans D'autres outils basés sur IntelliJ: PyCharm, IntelliJ IDEA, GoLand, PhpStorm et autres. Caractéristique: @@ -110,7 +114,7 @@ Caractéristique: ### Yandex DataLens {#yandex-datalens} -[Yandex DataLens](https://cloud.yandex.ru/services/datalens) est un service de visualisation et d’analyse de données. +[Yandex DataLens](https://cloud.yandex.ru/services/datalens) est un service de visualisation et d'analyse de données. Caractéristique: @@ -122,7 +126,7 @@ Caractéristique: DataLens est [disponible gratuitement](https://cloud.yandex.com/docs/datalens/pricing) pour les projets à faible charge, même pour un usage commercial. - [Documentation DataLens](https://cloud.yandex.com/docs/datalens/). -- [Tutoriel](https://cloud.yandex.com/docs/solutions/datalens/data-from-ch-visualization) sur la visualisation des données à partir d’une base de données ClickHouse. +- [Tutoriel](https://cloud.yandex.com/docs/solutions/datalens/data-from-ch-visualization) sur la visualisation des données à partir d'une base de données ClickHouse. ### Logiciel Holistics {#holistics-software} @@ -138,8 +142,8 @@ Caractéristique: ### Looker {#looker} -[Looker](https://looker.com) est une plate-forme de données et un outil de business intelligence avec prise en charge de plus de 50 dialectes de base de données, y compris ClickHouse. Looker est disponible en tant que plate-forme SaaS et auto-hébergé. Les utilisateurs peuvent utiliser Looker via le navigateur pour explorer les données, créer des visualisations et des tableaux de bord, planifier des rapports et partager leurs idées avec des collègues. Looker fournit un riche ensemble d’outils pour intégrer ces fonctionnalités dans d’autres applications, et une API -pour intégrer les données avec d’autres applications. +[Looker](https://looker.com) est une plate-forme de données et un outil de business intelligence avec prise en charge de plus de 50 dialectes de base de données, y compris ClickHouse. Looker est disponible en tant que plate-forme SaaS et auto-hébergé. Les utilisateurs peuvent utiliser Looker via le navigateur pour explorer les données, créer des visualisations et des tableaux de bord, planifier des rapports et partager leurs idées avec des collègues. Looker fournit un riche ensemble d'outils pour intégrer ces fonctionnalités dans d'autres applications, et une API +pour intégrer les données avec d'autres applications. Caractéristique: diff --git a/docs/fr/interfaces/third-party/index.md b/docs/fr/interfaces/third-party/index.md index 75062f3bad4..fd721a39c3e 100644 --- a/docs/fr/interfaces/third-party/index.md +++ b/docs/fr/interfaces/third-party/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 -toc_folder_title: Third-Party +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: tiers toc_priority: 24 --- diff --git a/docs/fr/interfaces/third-party/integrations.md b/docs/fr/interfaces/third-party/integrations.md index 797b6971ad1..6edb04ce518 100644 --- a/docs/fr/interfaces/third-party/integrations.md +++ b/docs/fr/interfaces/third-party/integrations.md @@ -1,16 +1,16 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 27 toc_title: "Int\xE9gration" --- -# Bibliothèques d’intégration De développeurs Tiers {#integration-libraries-from-third-party-developers} +# Bibliothèques d'intégration de développeurs tiers {#integration-libraries-from-third-party-developers} !!! warning "Avertissement" - Yandex ne **pas** maintenir les outils et les bibliothèques énumérés ci-dessous et n’ont pas fait de tests approfondis pour assurer leur qualité. + Yandex ne **pas** maintenir les outils et les bibliothèques énumérés ci-dessous et n'ont pas fait de tests approfondis pour assurer leur qualité. -## Produits D’Infrastructure {#infrastructure-products} +## Produits D'Infrastructure {#infrastructure-products} - Systèmes de gestion de bases de données relationnelles - [MySQL](https://www.mysql.com) @@ -24,9 +24,12 @@ toc_title: "Int\xE9gration" - [clickhouse\_fdw](https://github.com/adjust/clickhouse_fdw) - [MSSQL](https://en.wikipedia.org/wiki/Microsoft_SQL_Server) - [ClickHouseMigrator](https://github.com/zlzforever/ClickHouseMigrator) -- Files d’attente de messages +- Files d'attente de messages - [Kafka](https://kafka.apache.org) - - [clickhouse\_sinker](https://github.com/housepower/clickhouse_sinker) (utiliser [Allez client](https://github.com/kshvakov/clickhouse/)) + - [clickhouse\_sinker](https://github.com/housepower/clickhouse_sinker) (utiliser [Allez client](https://github.com/ClickHouse/clickhouse-go/)) +- Traitement de flux + - [Flink](https://flink.apache.org) + - [flink-clickhouse-évier](https://github.com/ivi-ru/flink-clickhouse-sink) - Objet de stockages - [S3](https://en.wikipedia.org/wiki/Amazon_S3) - [clickhouse-sauvegarde](https://github.com/AlexAkulov/clickhouse-backup) @@ -79,7 +82,7 @@ toc_title: "Int\xE9gration" - [dbal-clickhouse](https://packagist.org/packages/friendsofdoctrine/dbal-clickhouse) - R - [dplyr](https://db.rstudio.com/dplyr/) - - [RClickhouse](https://github.com/IMSMWU/RClickhouse) (utiliser [clickhouse-cpp](https://github.com/artpaul/clickhouse-cpp)) + - [RClickHouse](https://github.com/IMSMWU/RClickHouse) (utiliser [clickhouse-cpp](https://github.com/artpaul/clickhouse-cpp)) - Java - [Hadoop](http://hadoop.apache.org) - [clickhouse-HDFS-chargeur](https://github.com/jaykelin/clickhouse-hdfs-loader) (utiliser [JDBC](../../sql-reference/table-functions/jdbc.md)) diff --git a/docs/fr/interfaces/third-party/proxy.md b/docs/fr/interfaces/third-party/proxy.md index 30a813d5515..0283cc7fbe6 100644 --- a/docs/fr/interfaces/third-party/proxy.md +++ b/docs/fr/interfaces/third-party/proxy.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 29 toc_title: Proxy --- -# Serveurs Proxy De développeurs Tiers {#proxy-servers-from-third-party-developers} +# Serveurs Proxy de développeurs tiers {#proxy-servers-from-third-party-developers} ## chproxy {#chproxy} @@ -21,7 +21,7 @@ Mis en œuvre dans Go. ## KittenHouse {#kittenhouse} -[KittenHouse](https://github.com/VKCOM/kittenhouse) est conçu pour être un proxy local entre ClickHouse et serveur d’applications dans le cas où il est impossible ou gênant d’insérer des données en mémoire tampon du côté de votre application. +[KittenHouse](https://github.com/VKCOM/kittenhouse) est conçu pour être un proxy local entre ClickHouse et serveur d'applications dans le cas où il est impossible ou gênant d'insérer des données en mémoire tampon du côté de votre application. Caractéristique: @@ -33,13 +33,13 @@ Mis en œuvre dans Go. ## ClickHouse-Vrac {#clickhouse-bulk} -[ClickHouse-Vrac](https://github.com/nikepan/clickhouse-bulk) est un collecteur simple D’insertion de ClickHouse. +[ClickHouse-Vrac](https://github.com/nikepan/clickhouse-bulk) est un collecteur simple D'insertion de ClickHouse. Caractéristique: - Groupez les demandes et envoyez-les par seuil ou intervalle. - Plusieurs serveurs distants. -- L’authentification de base. +- L'authentification de base. Mis en œuvre dans Go. diff --git a/docs/fr/introduction/adopters.md b/docs/fr/introduction/adopters.md index 5b9caeb1e7f..833fc111fbe 100644 --- a/docs/fr/introduction/adopters.md +++ b/docs/fr/introduction/adopters.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 8 toc_title: Adoptant --- @@ -8,75 +8,79 @@ toc_title: Adoptant # Clickhouse Adopteurs {#clickhouse-adopters} !!! warning "Avertissement" - La liste suivante des entreprises utilisant ClickHouse et leurs histoires de réussite est assemblé à partir de sources publiques, pourrait donc différer de la réalité actuelle. Nous vous serions reconnaissants si vous partager l’histoire de l’adoption de ClickHouse dans votre entreprise et [ajouter à la liste](https://github.com/ClickHouse/ClickHouse/edit/master/docs/en/introduction/adopters.md), mais assurez-vous que vous n’aurez aucun problème de NDA en le faisant. Fournir des mises à jour avec des publications d’autres entreprises est également utile. + La liste suivante des entreprises utilisant ClickHouse et leurs histoires de réussite est assemblé à partir de sources publiques, pourrait donc différer de la réalité actuelle. Nous vous serions reconnaissants si vous partager l'histoire de l'adoption de ClickHouse dans votre entreprise et [ajouter à la liste](https://github.com/ClickHouse/ClickHouse/edit/master/docs/en/introduction/adopters.md), mais assurez-vous que vous n'aurez aucun problème de NDA en le faisant. Fournir des mises à jour avec des publications d'autres entreprises est également utile. -| Entreprise | Industrie | Cas d’utilisation | La Taille De Cluster | (Onu)Taille Des Données Compressées\* | Référence | -|-------------------------------------------------------------------------------|-------------------------------------|------------------------------|-----------------------------------------------------------------|-----------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [2gis](https://2gis.ru) | Cartographie | Surveiller | — | — | [Parler en russe, juillet 2019](https://youtu.be/58sPkXfq6nw) | -| [Aloha Navigateur](https://alohabrowser.com/) | Mobile App | Navigateur backend | — | — | [Diapositives en russe, mai 2019](https://github.com/yandex/clickhouse-presentations/blob/master/meetup22/aloha.pdf) | -| [Amadeus](https://amadeus.com/) | Voyage | Analytics | — | — | [Communiqué De Presse, Avril 2018](https://www.altinity.com/blog/2018/4/5/amadeus-technologies-launches-investment-and-insights-tool-based-on-machine-learning-and-strategy-algorithms) | -| [Appsflyer](https://www.appsflyer.com) | Mobile analytics | Produit principal | — | — | [Parler en russe, juillet 2019](https://www.youtube.com/watch?v=M3wbRlcpBbY) | -| [ArenaData](https://arenadata.tech/) | Plate-Forme De Données | Produit principal | — | — | [Diapositives en russe, décembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup38/indexes.pdf) | -| [Badoo](https://badoo.com) | Rencontres | Timeseries | — | — | [Diapositives en russe, décembre 2019](https://presentations.clickhouse.tech/meetup38/forecast.pdf) | -| [Benocs](https://www.benocs.com/) | Télémétrie et analyse de réseau | Produit Principal | — | — | [Diapositives en anglais, octobre 2017](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup9/lpm.pdf) | -| [Bloomberg](https://www.bloomberg.com/) | Les Finances, Les Médias | Surveiller | 102 serveurs | — | [Diapositives, Mai 2018](https://www.slideshare.net/Altinity/http-analytics-for-6m-requests-per-second-using-clickhouse-by-alexander-bocharov) | -| [Bloxy](https://bloxy.info) | Blockchain | Analytics | — | — | [Diapositives en russe, août 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/4_bloxy.pptx) | -| `Dataliance/UltraPower` | Télécommunication | Analytics | — | — | [Diapositives en chinois, janvier 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup12/telecom.pdf) | -| [CARTO](https://carto.com/) | Business Intelligence | GEO analytics | — | — | [Traitement géospatial avec Clickhouse](https://carto.com/blog/geospatial-processing-with-clickhouse/) | -| [CERN](http://public.web.cern.ch/public/) | Recherche | Expérience | — | — | [Communiqué De Presse, avril 2012](https://www.yandex.com/company/press_center/press_releases/2012/2012-04-10/) | -| [Cisco](http://cisco.com/) | Réseau | L’analyse de trafic | — | — | [Lightning talk, octobre 2019](https://youtu.be/-hI1vDR2oPY?t=5057) | -| [Citadelle Titres](https://www.citadelsecurities.com/) | Finance | — | — | — | [Contribution, Mars 2019](https://github.com/ClickHouse/ClickHouse/pull/4774) | -| [Citymobil](https://city-mobil.ru) | Taxi | Analytics | — | — | [Billet de Blog en russe, mars 2020](https://habr.com/en/company/citymobil/blog/490660/) | -| [ContentSquare](https://contentsquare.com) | Web analytics | Produit principal | — | — | [Billet de Blog, novembre 2018](http://souslecapot.net/2018/11/21/patrick-chatain-vp-engineering-chez-contentsquare-penser-davantage-amelioration-continue-que-revolution-constante/) | -| [Cloudflare](https://cloudflare.com) | CDN | L’analyse de trafic | 36 serveurs | — | [Billet de Blog, mai 2017](https://blog.cloudflare.com/how-cloudflare-analyzes-1m-dns-queries-per-second/), [Billet de Blog, mars 2018](https://blog.cloudflare.com/http-analytics-for-6m-requests-per-second-using-clickhouse/) | -| [Corunet](https://coru.net/) | Analytics | Produit principal | — | — | [Diapositives en anglais, avril 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup21/predictive_models.pdf) | -| [CraiditX 氪信](https://creditx.com) | Les finances de l’IA | Analyse | — | — | [Diapositives en anglais, novembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/udf.pptx) | -| [Criteo / Storetail](https://www.criteo.com/) | Détail | Produit principal | — | — | [Diapositives en anglais, octobre 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/3_storetail.pptx) | -| [La Deutsche Bank](https://db.com) | Finance | BI Analytics | — | — | [Diapositives en anglais, octobre 2019](https://bigdatadays.ru/wp-content/uploads/2019/10/D2-H3-3_Yakunin-Goihburg.pdf) | -| [Diva-e](https://www.diva-e.com) | Conseil Digital | Produit Principal | — | — | [Diapositives en anglais, septembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup29/ClickHouse-MeetUp-Unusual-Applications-sd-2019-09-17.pdf) | -| [Exness](https://www.exness.com) | Trading | Métriques, Journalisation | — | — | [Parler en russe, mai 2019](https://youtu.be/_rpU-TvSfZ8?t=3215) | -| [Geniee](https://geniee.co.jp) | Réseau publicitaire | Produit principal | — | — | [Billet de Blog en japonais, juillet 2017](https://tech.geniee.co.jp/entry/2017/07/20/160100) | -| [HUYA](https://www.huya.com/) | Le Streaming Vidéo | Analytics | — | — | [Diapositives en chinois, octobre 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/7.%20ClickHouse万亿数据分析实践%20李本旺(sundy-li)%20虎牙.pdf) | -| [Idealista](https://www.idealista.com) | Immobilier | Analytics | — | — | [Billet de Blog en anglais, avril 2019](https://clickhouse.yandex/blog/en/clickhouse-meetup-in-madrid-on-april-2-2019) | -| [Infovista](https://www.infovista.com/) | Réseau | Analytics | — | — | [Diapositives en anglais, octobre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup30/infovista.pdf) | -| [InnoGames](https://www.innogames.com) | Jeu | Métriques, Journalisation | — | — | [Diapositives en russe, septembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/graphite_and_clickHouse.pdf) | -| [Integros](https://integros.com) | Plate-forme pour les services vidéo | Analytics | — | — | [Diapositives en russe, mai 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | -| [Données Kodiak](https://www.kodiakdata.com/) | Nuage | Produit principal | — | — | [Diapositives en Engish, avril 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup13/kodiak_data.pdf) | -| [Kontur](https://kontur.ru) | Le Développement De Logiciels | Métrique | — | — | [Parler en russe, novembre 2018](https://www.youtube.com/watch?v=U4u4Bd0FtrY) | -| [LifeStreet](https://lifestreet.com/) | Réseau publicitaire | Produit principal | 75 serveurs (3 répliques) | 5.27 FRP | [Billet de Blog en russe, février 2017](https://habr.com/en/post/322620/) | -| [Mail.ru Solutions Cloud](https://mcs.mail.ru/) | Les services de Cloud | Produit principal | — | — | [Exécution de L’Instance ClickHouse, en russe](https://mcs.mail.ru/help/db-create/clickhouse#) | -| [MessageBird](https://www.messagebird.com) | Télécommunication | Statistique | — | — | [Diapositives en anglais, novembre 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup20/messagebird.pdf) | -| [MGID](https://www.mgid.com/) | Réseau publicitaire | Web-analytics | — | — | [Notre expérience dans la mise en œuvre de SGBD analytique ClickHouse, en russe](http://gs-studio.com/news-about-it/32777----clickhouse---c) | -| [OneAPM](https://www.oneapm.com/) | Monitorings et analyse des données | Produit principal | — | — | [Diapositives en chinois, octobre 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/8.%20clickhouse在OneAPM的应用%20杜龙.pdf) | -| [Pragma L’Innovation](http://www.pragma-innovation.fr/) | Télémétrie et analyse Big Data | Produit principal | — | — | [Diapositives en anglais, octobre 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/4_pragma_innovation.pdf) | -| [QINGCLOUD](https://www.qingcloud.com/) | Les services de Cloud | Produit principal | — | — | [Diapositives en chinois, octobre 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/4.%20Cloud%20%2B%20TSDB%20for%20ClickHouse%20张健%20QingCloud.pdf) | -| [Qrator](https://qrator.net) | Protection DDoS | Produit principal | — | — | [Billet De Blog, Mars 2019](https://blog.qrator.net/en/clickhouse-ddos-mitigation_37/) | -| [Beijing pour cent Information Technology Co., Ltd.](https://www.percent.cn/) | Analytics | Produit Principal | — | — | [Diapositives en chinois, juin 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/4.%20ClickHouse万亿数据双中心的设计与实践%20.pdf) | -| [Rambler](https://rambler.ru) | Services Internet | Analytics | — | — | [Parler en russe, avril 2018](https://medium.com/@ramblertop/разработка-api-clickhouse-для-рамблер-топ-100-f4c7e56f3141) | -| [Tencent](https://www.tencent.com) | Messagerie | Journalisation | — | — | [Parler en chinois, novembre 2019](https://youtu.be/T-iVQRuw-QY?t=5050) | -| [Étoiles De Circulation](https://trafficstars.com/) | Réseau publicitaire | — | — | — | [Diapositives en russe, mai 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup15/lightning/ninja.pdf) | -| [S7 Airlines](https://www.s7.ru) | Avion | Métriques, Journalisation | — | — | [Parler en russe, mars 2019](https://www.youtube.com/watch?v=nwG68klRpPg&t=15s) | -| [SEMrush](https://www.semrush.com/) | Marketing | Produit principal | — | — | [Diapositives en russe, août 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/5_semrush.pdf) | -| [scireum GmbH](https://www.scireum.de/) | Ecommerce | Produit principal | — | — | [Présentation en allemand, février 2020](https://www.youtube.com/watch?v=7QWAn5RbyR4) | -| [Sentry](https://sentry.io/) | Développeur de logiciels | Backend pour le produit | — | — | [Billet de Blog en anglais, mai 2019](https://blog.sentry.io/2019/05/16/introducing-snuba-sentrys-new-search-infrastructure) | -| [SGK](http://www.sgk.gov.tr/wps/portal/sgk/tr) | Gouvernement Sécurité Sociale | Analytics | — | — | [Diapositives en anglais, novembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup35/ClickHouse%20Meetup-Ramazan%20POLAT.pdf) | -| [seo.do](https://seo.do/) | Analytics | Produit principal | — | — | [Diapositives en anglais, novembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup35/CH%20Presentation-%20Metehan%20Çetinkaya.pdf) | -| [Sina](http://english.sina.com/index.html) | Nouvelles | — | — | — | [Diapositives en chinois, octobre 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/6.%20ClickHouse最佳实践%20高鹏_新浪.pdf) | -| [SMI2](https://smi2.ru/) | Nouvelles | Analytics | — | — | [Billet de Blog en russe, novembre 2017](https://habr.com/ru/company/smi2/blog/314558/) | -| [Splunk](https://www.splunk.com/) | Business Analytics | Produit principal | — | — | [Diapositives en anglais, janvier 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup12/splunk.pdf) | -| [Spotify](https://www.spotify.com) | Musical | Expérimentation | — | — | [Diapositives, Juillet 2018](https://www.slideshare.net/glebus/using-clickhouse-for-experimentation-104247173) | -| [Tencent](https://www.tencent.com) | Big Data | Le traitement des données | — | — | [Diapositives en chinois, octobre 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/5.%20ClickHouse大数据集群应用_李俊飞腾讯网媒事业部.pdf) | -| [Uber](https://www.uber.com) | Taxi | Journalisation | — | — | [Diapositives, Février 2020](https://presentations.clickhouse.tech/meetup40/uber.pdf) | -| [VKontakte](https://vk.com) | Réseau Social | Statistiques, Journalisation | — | — | [Diapositives en russe, août 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/3_vk.pdf) | -| [Wisebits](https://wisebits.com/) | Solutions informatiques | Analytics | — | — | [Diapositives en russe, mai 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | -| [Xiaoxin Tech.](https://www.xiaoheiban.cn/) | Éducation | But commun | — | — | [Diapositives en anglais, novembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/sync-clickhouse-with-mysql-mongodb.pptx) | -| [Ximalaya](https://www.ximalaya.com/) | Partage Audio | OLAP | — | — | [Diapositives en anglais, novembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/ximalaya.pdf) | -| [Yandex Cloud](https://cloud.yandex.ru/services/managed-clickhouse) | Le Cloud Public | Produit principal | — | — | [Parler en russe, décembre 2019](https://www.youtube.com/watch?v=pgnak9e_E0o) | -| [Yandex DataLens](https://cloud.yandex.ru/services/datalens) | Business Intelligence | Produit principal | — | — | [Diapositives en russe, décembre 2019](https://presentations.clickhouse.tech/meetup38/datalens.pdf) | -| [Yandex Marché](https://market.yandex.ru/) | Ecommerce | Métriques, Journalisation | — | — | [Parler en russe, janvier 2019](https://youtu.be/_l1qP0DyBcA?t=478) | -| [Yandex Metrica](https://metrica.yandex.com) | Web analytics | Produit principal | 360 serveurs dans un cluster, 1862 serveurs dans un département | 66,41 FRP / 5,68 FRP | [Diapositives, Février 2020](https://presentations.clickhouse.tech/meetup40/introduction/#13) | -| [ЦВТ](https://htc-cs.ru/) | Le Développement De Logiciels | Métriques, Journalisation | — | — | [Billet de Blog, mars 2019, en russe](https://vc.ru/dev/62715-kak-my-stroili-monitoring-na-prometheus-clickhouse-i-elk) | -| [МКБ](https://mkb.ru/) | Banque | Surveillance du système Web | — | — | [Diapositives en russe, septembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/mkb.pdf) | -| [金数据](https://jinshuju.net) | BI Analytics | Produit principal | — | — | [Diapositives en chinois, octobre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/3.%20金数据数据架构调整方案Public.pdf) | +| Entreprise | Industrie | Cas d'utilisation | La Taille De Cluster | (Onu)Taille Des Données Compressées\* | Référence | +|-------------------------------------------------------------------------------------------------|-------------------------------------|------------------------------|-----------------------------------------------------------------|-----------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 2gis | Cartographie | Surveiller | — | — | [Parler en russe, juillet 2019](https://youtu.be/58sPkXfq6nw) | +| Aloha Browser | Mobile App | Navigateur backend | — | — | [Diapositives en russe, mai 2019](https://github.com/yandex/clickhouse-presentations/blob/master/meetup22/aloha.pdf) | +| Amadeus | Voyage | Analytics | — | — | [Communiqué De Presse, Avril 2018](https://www.altinity.com/blog/2018/4/5/amadeus-technologies-launches-investment-and-insights-tool-based-on-machine-learning-and-strategy-algorithms) | +| Appsflyer | Mobile analytics | Produit principal | — | — | [Parler en russe, juillet 2019](https://www.youtube.com/watch?v=M3wbRlcpBbY) | +| ArenaData | Plate-Forme De Données | Produit principal | — | — | [Diapositives en russe, décembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup38/indexes.pdf) | +| Badoo | Rencontres | Timeseries | — | — | [Diapositives en russe, décembre 2019](https://presentations.clickhouse.tech/meetup38/forecast.pdf) | +| Benocs | Télémétrie et analyse de réseau | Produit Principal | — | — | [Diapositives en anglais, octobre 2017](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup9/lpm.pdf) | +| Bloomberg | Les Finances, Les Médias | Surveiller | 102 serveurs | — | [Diapositives, Mai 2018](https://www.slideshare.net/Altinity/http-analytics-for-6m-requests-per-second-using-clickhouse-by-alexander-bocharov) | +| Bloxy | Blockchain | Analytics | — | — | [Diapositives en russe, août 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/4_bloxy.pptx) | +| Dataliance pour China Telecom | Télécommunication | Analytics | — | — | [Diapositives en chinois, janvier 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup12/telecom.pdf) | +| CARTO | Business Intelligence | GEO analytics | — | — | [Traitement géospatial avec ClickHouse](https://carto.com/blog/geospatial-processing-with-clickhouse/) | +| CERN | Recherche | Expérience | — | — | [Communiqué De Presse, avril 2012](https://www.yandex.com/company/press_center/press_releases/2012/2012-04-10/) | +| Cisco | Réseau | L'analyse de trafic | — | — | [Lightning talk, octobre 2019](https://youtu.be/-hI1vDR2oPY?t=5057) | +| Citadel Securities | Finance | — | — | — | [Contribution, Mars 2019](https://github.com/ClickHouse/ClickHouse/pull/4774) | +| Citymobil | Taxi | Analytics | — | — | [Billet de Blog en russe, mars 2020](https://habr.com/en/company/citymobil/blog/490660/) | +| ContentSquare | Web analytics | Produit principal | — | — | [Billet de Blog, novembre 2018](http://souslecapot.net/2018/11/21/patrick-chatain-vp-engineering-chez-contentsquare-penser-davantage-amelioration-continue-que-revolution-constante/) | +| Cloudflare | CDN | L'analyse de trafic | 36 serveurs | — | [Billet de Blog, mai 2017](https://blog.cloudflare.com/how-cloudflare-analyzes-1m-dns-queries-per-second/), [Billet de Blog, mars 2018](https://blog.cloudflare.com/http-analytics-for-6m-requests-per-second-using-clickhouse/) | +| Corunet | Analytics | Produit principal | — | — | [Diapositives en anglais, avril 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup21/predictive_models.pdf) | +| CraiditX 氪信 | Les finances de l'IA | Analyse | — | — | [Diapositives en anglais, novembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/udf.pptx) | +| Criteo | Détail | Produit principal | — | — | [Diapositives en anglais, octobre 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/3_storetail.pptx) | +| Deutsche Bank | Finance | BI Analytics | — | — | [Diapositives en anglais, octobre 2019](https://bigdatadays.ru/wp-content/uploads/2019/10/D2-H3-3_Yakunin-Goihburg.pdf) | +| Diva-e | Conseil Digital | Produit Principal | — | — | [Diapositives en anglais, septembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup29/ClickHouse-MeetUp-Unusual-Applications-sd-2019-09-17.pdf) | +| Exness | Trading | Métriques, Journalisation | — | — | [Parler en russe, mai 2019](https://youtu.be/_rpU-TvSfZ8?t=3215) | +| Geniee | Réseau publicitaire | Produit principal | — | — | [Billet de Blog en japonais, juillet 2017](https://tech.geniee.co.jp/entry/2017/07/20/160100) | +| HUYA | Le Streaming Vidéo | Analytics | — | — | [Diapositives en chinois, octobre 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/7.%20ClickHouse万亿数据分析实践%20李本旺(sundy-li)%20虎牙.pdf) | +| Idealista | Immobilier | Analytics | — | — | [Billet de Blog en anglais, avril 2019](https://clickhouse.yandex/blog/en/clickhouse-meetup-in-madrid-on-april-2-2019) | +| Infovista | Réseau | Analytics | — | — | [Diapositives en anglais, octobre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup30/infovista.pdf) | +| InnoGames | Jeu | Métriques, Journalisation | — | — | [Diapositives en russe, septembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/graphite_and_clickHouse.pdf) | +| Integros | Plate-forme pour les services vidéo | Analytics | — | — | [Diapositives en russe, mai 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | +| Données Kodiak | Nuage | Produit principal | — | — | [Diapositives en Engish, avril 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup13/kodiak_data.pdf) | +| Kontur | Le Développement De Logiciels | Métrique | — | — | [Parler en russe, novembre 2018](https://www.youtube.com/watch?v=U4u4Bd0FtrY) | +| LifeStreet | Réseau publicitaire | Produit principal | 75 serveurs (3 répliques) | 5.27 FRP | [Billet de Blog en russe, février 2017](https://habr.com/en/post/322620/) | +| Mail.ru Solutions Cloud | Les services de Cloud | Produit principal | — | — | [Article en russe](https://mcs.mail.ru/help/db-create/clickhouse#) | +| MessageBird | Télécommunication | Statistique | — | — | [Diapositives en anglais, novembre 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup20/messagebird.pdf) | +| MGID | Réseau publicitaire | Web-analytics | — | — | [Billet de Blog en russe, avril 2020](http://gs-studio.com/news-about-it/32777----clickhouse---c) | +| OneAPM | Monitorings et analyse des données | Produit principal | — | — | [Diapositives en chinois, octobre 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/8.%20clickhouse在OneAPM的应用%20杜龙.pdf) | +| Pragma Innovation | Télémétrie et analyse Big Data | Produit principal | — | — | [Diapositives en anglais, octobre 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/4_pragma_innovation.pdf) | +| QINGCLOUD | Les services de Cloud | Produit principal | — | — | [Diapositives en chinois, octobre 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/4.%20Cloud%20%2B%20TSDB%20for%20ClickHouse%20张健%20QingCloud.pdf) | +| Qrator | Protection DDoS | Produit principal | — | — | [Billet De Blog, Mars 2019](https://blog.qrator.net/en/clickhouse-ddos-mitigation_37/) | +| Percent 百分点 | Analytics | Produit Principal | — | — | [Diapositives en chinois, juin 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/4.%20ClickHouse万亿数据双中心的设计与实践%20.pdf) | +| Rambler | Services Internet | Analytics | — | — | [Parler en russe, avril 2018](https://medium.com/@ramblertop/разработка-api-clickhouse-для-рамблер-топ-100-f4c7e56f3141) | +| Tencent | Messagerie | Journalisation | — | — | [Parler en chinois, novembre 2019](https://youtu.be/T-iVQRuw-QY?t=5050) | +| Traffic Stars | Réseau publicitaire | — | — | — | [Diapositives en russe, mai 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup15/lightning/ninja.pdf) | +| S7 Airlines | Avion | Métriques, Journalisation | — | — | [Parler en russe, mars 2019](https://www.youtube.com/watch?v=nwG68klRpPg&t=15s) | +| SEMrush | Marketing | Produit principal | — | — | [Diapositives en russe, août 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/5_semrush.pdf) | +| scireum GmbH | Ecommerce | Produit principal | — | — | [Présentation en allemand, février 2020](https://www.youtube.com/watch?v=7QWAn5RbyR4) | +| Sentry | Développeur de logiciels | Backend pour le produit | — | — | [Billet de Blog en anglais, mai 2019](https://blog.sentry.io/2019/05/16/introducing-snuba-sentrys-new-search-infrastructure) | +| SGK | Gouvernement Sécurité Sociale | Analytics | — | — | [Diapositives en anglais, novembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup35/ClickHouse%20Meetup-Ramazan%20POLAT.pdf) | +| seo.do | Analytics | Produit principal | — | — | [Diapositives en anglais, novembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup35/CH%20Presentation-%20Metehan%20Çetinkaya.pdf) | +| Sina | Nouvelles | — | — | — | [Diapositives en chinois, octobre 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/6.%20ClickHouse最佳实践%20高鹏_新浪.pdf) | +| SMI2 | Nouvelles | Analytics | — | — | [Billet de Blog en russe, novembre 2017](https://habr.com/ru/company/smi2/blog/314558/) | +| Splunk | Business Analytics | Produit principal | — | — | [Diapositives en anglais, janvier 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup12/splunk.pdf) | +| Spotify | Musical | Expérimentation | — | — | [Diapositives, Juillet 2018](https://www.slideshare.net/glebus/using-clickhouse-for-experimentation-104247173) | +| Tencent | Big Data | Le traitement des données | — | — | [Diapositives en chinois, octobre 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/5.%20ClickHouse大数据集群应用_李俊飞腾讯网媒事业部.pdf) | +| Uber | Taxi | Journalisation | — | — | [Diapositives, Février 2020](https://presentations.clickhouse.tech/meetup40/uber.pdf) | +| VKontakte | Réseau Social | Statistiques, Journalisation | — | — | [Diapositives en russe, août 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/3_vk.pdf) | +| Wisebits | Solutions informatiques | Analytics | — | — | [Diapositives en russe, mai 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | +| Xiaoxin Tech | Éducation | But commun | — | — | [Diapositives en anglais, novembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/sync-clickhouse-with-mysql-mongodb.pptx) | +| Ximalaya | Partage Audio | OLAP | — | — | [Diapositives en anglais, novembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/ximalaya.pdf) | +| Yandex Cloud | Le Cloud Public | Produit principal | — | — | [Parler en russe, décembre 2019](https://www.youtube.com/watch?v=pgnak9e_E0o) | +| Yandex DataLens | Business Intelligence | Produit principal | — | — | [Diapositives en russe, décembre 2019](https://presentations.clickhouse.tech/meetup38/datalens.pdf) | +| Yandex Market | Ecommerce | Métriques, Journalisation | — | — | [Parler en russe, janvier 2019](https://youtu.be/_l1qP0DyBcA?t=478) | +| Yandex Metrica | Web analytics | Produit principal | 360 serveurs dans un cluster, 1862 serveurs dans un département | 66,41 FRP / 5,68 FRP | [Diapositives, Février 2020](https://presentations.clickhouse.tech/meetup40/introduction/#13) | +| ЦВТ | Le Développement De Logiciels | Métriques, Journalisation | — | — | [Billet de Blog, mars 2019, en russe](https://vc.ru/dev/62715-kak-my-stroili-monitoring-na-prometheus-clickhouse-i-elk) | +| МКБ | Banque | Surveillance du système Web | — | — | [Diapositives en russe, septembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/mkb.pdf) | +| Jinshuju 金数据 | BI Analytics | Produit principal | — | — | [Diapositives en chinois, octobre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/3.%20金数据数据架构调整方案Public.pdf) | +| Instana | Plate-forme APM | Produit principal | — | — | [Après Twitter](https://twitter.com/mieldonkers/status/1248884119158882304) | +| Wargaming | Jeu | | — | — | [Entrevue](https://habr.com/en/post/496954/) | +| Crazypanda | Jeu | | — | — | Session en direct sur clickhouse meetup | +| FunCorp | Jeu | | — | — | [Article](https://www.altinity.com/blog/migrating-from-redshift-to-clickhouse) | [Article Original](https://clickhouse.tech/docs/en/introduction/adopters/) diff --git a/docs/fr/introduction/distinctive-features.md b/docs/fr/introduction/distinctive-features.md index 68cc1655b4b..9d01d245c4e 100644 --- a/docs/fr/introduction/distinctive-features.md +++ b/docs/fr/introduction/distinctive-features.md @@ -1,38 +1,38 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 4 toc_title: "particularit\xE9" --- -# Caractéristiques Distinctives De ClickHouse {#distinctive-features-of-clickhouse} +# Caractéristiques distinctives de ClickHouse {#distinctive-features-of-clickhouse} -## Vrai SGBD orienté Colonne {#true-column-oriented-dbms} +## Vrai SGBD orienté colonne {#true-column-oriented-dbms} -Dans un vrai SGBD orienté colonne, aucune donnée supplémentaire n’est stockée avec les valeurs. Entre autres choses, cela signifie que les valeurs de longueur constante doivent être prises en charge, pour éviter de stocker leur longueur “number” à côté de ces valeurs. Par exemple, un milliard de valeurs de type UInt8 devrait consommer environ 1 Go non compressé, ou cela affecte fortement l’utilisation du processeur. Il est essentiel de stocker des données de manière compacte (sans “garbage”) même lorsqu’il n’est pas compressé, puisque la vitesse de décompression (utilisation du processeur) dépend principalement du volume de données non compressées. +Dans un vrai SGBD orienté colonne, aucune donnée supplémentaire n'est stockée avec les valeurs. Entre autres choses, cela signifie que les valeurs de longueur constante doivent être prises en charge, pour éviter de stocker leur longueur “number” à côté de ces valeurs. Par exemple, un milliard de valeurs de type UInt8 devrait consommer environ 1 Go non compressé, ou cela affecte fortement l'utilisation du processeur. Il est essentiel de stocker des données de manière compacte (sans “garbage”) même lorsqu'il n'est pas compressé, puisque la vitesse de décompression (utilisation du processeur) dépend principalement du volume de données non compressées. -Il est à noter car il existe des systèmes qui peuvent stocker des valeurs de différentes colonnes séparément, mais qui ne peuvent pas traiter efficacement les requêtes analytiques en raison de leur optimisation pour d’autres scénarios. Les exemples sont HBase, BigTable, Cassandra et HyperTable. Dans ces systèmes, vous obtiendriez un débit d’environ cent mille lignes par seconde, mais pas des centaines de millions de lignes par seconde. +Il est à noter car il existe des systèmes qui peuvent stocker des valeurs de différentes colonnes séparément, mais qui ne peuvent pas traiter efficacement les requêtes analytiques en raison de leur optimisation pour d'autres scénarios. Les exemples sont HBase, BigTable, Cassandra et HyperTable. Dans ces systèmes, vous obtiendriez un débit d'environ cent mille lignes par seconde, mais pas des centaines de millions de lignes par seconde. -Il est également intéressant de noter que ClickHouse est un système de gestion de base de données, pas une seule base de données. ClickHouse permet de créer des tables et des bases de données en cours d’exécution, de charger des données et d’exécuter des requêtes sans reconfigurer et redémarrer le serveur. +Il est également intéressant de noter que ClickHouse est un système de gestion de base de données, pas une seule base de données. ClickHouse permet de créer des tables et des bases de données en cours d'exécution, de charger des données et d'exécuter des requêtes sans reconfigurer et redémarrer le serveur. ## Compression De Données {#data-compression} -Certains SGBD orientés colonne (InfiniDB CE et MonetDB) n’utilisent pas la compression de données. Cependant, la compression des données joue un rôle clé dans la réalisation d’excellentes performances. +Certains SGBD orientés colonne (InfiniDB CE et MonetDB) n'utilisent pas la compression de données. Cependant, la compression des données joue un rôle clé dans la réalisation d'excellentes performances. -## Stockage De données Sur Disque {#disk-storage-of-data} +## Stockage de données sur disque {#disk-storage-of-data} -Garder les données physiquement triées par clé primaire permet d’extraire des données pour ses valeurs spécifiques ou plages de valeurs avec une faible latence, moins de quelques dizaines de millisecondes. Certains SGBD orientés colonne (tels que SAP HANA et Google PowerDrill) ne peuvent fonctionner qu’en RAM. Cette approche encourage l’allocation d’un budget matériel plus important que ce qui est nécessaire pour l’analyse en temps réel. ClickHouse est conçu pour fonctionner sur des disques durs réguliers, ce qui signifie que le coût par Go de stockage de données est faible, mais SSD et RAM supplémentaire sont également entièrement utilisés si disponible. +Garder les données physiquement triées par clé primaire permet d'extraire des données pour ses valeurs spécifiques ou plages de valeurs avec une faible latence, moins de quelques dizaines de millisecondes. Certains SGBD orientés colonne (tels que SAP HANA et Google PowerDrill) ne peuvent fonctionner qu'en RAM. Cette approche encourage l'allocation d'un budget matériel plus important que ce qui est nécessaire pour l'analyse en temps réel. ClickHouse est conçu pour fonctionner sur des disques durs réguliers, ce qui signifie que le coût par Go de stockage de données est faible, mais SSD et RAM supplémentaire sont également entièrement utilisés si disponible. -## Traitement parallèle Sur Plusieurs cœurs {#parallel-processing-on-multiple-cores} +## Traitement parallèle sur plusieurs cœurs {#parallel-processing-on-multiple-cores} Les grandes requêtes sont parallélisées naturellement, en prenant toutes les ressources nécessaires disponibles sur le serveur actuel. -## Traitement distribué Sur Plusieurs Serveurs {#distributed-processing-on-multiple-servers} +## Traitement distribué sur plusieurs serveurs {#distributed-processing-on-multiple-servers} Presque aucun des SGBD en colonnes mentionnés ci-dessus ne prend en charge le traitement des requêtes distribuées. -Dans ClickHouse, les données peuvent résider sur différents fragments. Chaque fragment peut être un groupe de répliques utilisées pour la tolérance aux pannes. Tous les fragments sont utilisés pour exécuter une requête en parallèle, de façon transparente pour l’utilisateur. +Dans ClickHouse, les données peuvent résider sur différents fragments. Chaque fragment peut être un groupe de répliques utilisées pour la tolérance aux pannes. Tous les fragments sont utilisés pour exécuter une requête en parallèle, de façon transparente pour l'utilisateur. -## Prise En Charge SQL {#sql-support} +## Prise en charge SQL {#sql-support} ClickHouse prend en charge un langage de requête déclarative basé sur SQL qui est identique à la norme SQL dans de nombreux cas. Les requêtes prises en charge incluent les clauses GROUP BY, ORDER BY, les sous-requêtes in FROM, IN et JOIN, ainsi que les sous-requêtes scalaires. @@ -40,38 +40,38 @@ Les sous-requêtes dépendantes et les fonctions de fenêtre ne sont pas prises ## Moteur Vectoriel {#vector-engine} -Les données ne sont pas seulement stockées par des colonnes, mais sont traitées par des vecteurs (parties de colonnes), ce qui permet d’atteindre une efficacité élevée du processeur. +Les données ne sont pas seulement stockées par des colonnes, mais sont traitées par des vecteurs (parties de colonnes), ce qui permet d'atteindre une efficacité élevée du processeur. -## Données En Temps réel Des Mises à Jour {#real-time-data-updates} +## Données en temps réel des Mises à jour {#real-time-data-updates} -ClickHouse prend en charge les tables avec une clé primaire. Pour effectuer rapidement des requêtes sur la plage de la clé primaire, les données sont triées progressivement à l’aide de l’arborescence de fusion. Pour cette raison, les données peuvent être continuellement ajoutées à la table. Pas de verrouillage lorsque de nouvelles données sont ingérés. +ClickHouse prend en charge les tables avec une clé primaire. Pour effectuer rapidement des requêtes sur la plage de la clé primaire, les données sont triées progressivement à l'aide de l'arborescence de fusion. Pour cette raison, les données peuvent être continuellement ajoutées à la table. Pas de verrouillage lorsque de nouvelles données sont ingérés. ## Index {#index} -Avoir une donnée physiquement triée par clé primaire permet d’extraire des données pour ses valeurs spécifiques ou plages de valeurs avec une faible latence, moins de quelques dizaines de millisecondes. +Avoir une donnée physiquement triée par clé primaire permet d'extraire des données pour ses valeurs spécifiques ou plages de valeurs avec une faible latence, moins de quelques dizaines de millisecondes. -## Convient Pour Les requêtes En Ligne {#suitable-for-online-queries} +## Convient pour les requêtes en ligne {#suitable-for-online-queries} -Faible latence signifie que les requêtes peuvent être traitées sans délai et sans essayer de préparer une réponse à l’avance, au même moment pendant le chargement de la page de l’interface utilisateur. En d’autres termes, en ligne. +Faible latence signifie que les requêtes peuvent être traitées sans délai et sans essayer de préparer une réponse à l'avance, au même moment pendant le chargement de la page de l'interface utilisateur. En d'autres termes, en ligne. -## Prise En Charge Des Calculs Approximatifs {#support-for-approximated-calculations} +## Prise en charge des calculs approximatifs {#support-for-approximated-calculations} -ClickHouse offre différentes façons d’échanger la précision pour la performance: +ClickHouse offre différentes façons d'échanger la précision pour la performance: -1. Fonctions d’agrégation pour le calcul approximatif du nombre de valeurs distinctes, de médianes et de quantiles. -2. L’exécution d’une requête basée sur une partie (échantillon) de données et obtenir un pseudo résultat. Dans ce cas, proportionnellement, moins de données sont récupérées à partir du disque. -3. L’exécution d’une agrégation pour un nombre limité de clés aléatoires, au lieu de toutes les clés. Sous certaines conditions pour la distribution des clés dans les données, cela fournit un résultat raisonnablement précis tout en utilisant moins de ressources. +1. Fonctions d'agrégation pour le calcul approximatif du nombre de valeurs distinctes, de médianes et de quantiles. +2. L'exécution d'une requête basée sur une partie (échantillon) de données et obtenir un pseudo résultat. Dans ce cas, proportionnellement, moins de données sont récupérées à partir du disque. +3. L'exécution d'une agrégation pour un nombre limité de clés aléatoires, au lieu de toutes les clés. Sous certaines conditions pour la distribution des clés dans les données, cela fournit un résultat raisonnablement précis tout en utilisant moins de ressources. -## Prise En Charge De La réplication Et De l’intégrité Des données {#data-replication-and-data-integrity-support} +## Prise en charge de la réplication et de l'intégrité des données {#data-replication-and-data-integrity-support} -ClickHouse utilise la réplication multi-maître asynchrone. Après avoir été écrit dans n’importe quelle réplique disponible, toutes les répliques restantes récupèrent leur copie en arrière-plan. Le système conserve des données identiques sur différentes répliques. La récupération après la plupart des échecs est effectuée automatiquement ou semi-automatiquement dans les cas complexes. +ClickHouse utilise la réplication multi-maître asynchrone. Après avoir été écrit dans n'importe quelle réplique disponible, toutes les répliques restantes récupèrent leur copie en arrière-plan. Le système conserve des données identiques sur différentes répliques. La récupération après la plupart des échecs est effectuée automatiquement ou semi-automatiquement dans les cas complexes. -Pour plus d’informations, consultez la section [Réplication des données](../engines/table-engines/mergetree-family/replication.md). +Pour plus d'informations, consultez la section [Réplication des données](../engines/table-engines/mergetree-family/replication.md). -## Caractéristiques De ClickHouse Qui Peuvent être considérées Comme Des inconvénients {#clickhouse-features-that-can-be-considered-disadvantages} +## Caractéristiques qui peuvent être considérées comme des inconvénients {#clickhouse-features-that-can-be-considered-disadvantages} 1. Pas de transactions à part entière. 2. Manque de capacité à modifier ou supprimer des données déjà insérées avec un taux élevé et une faible latence. Des suppressions et des mises à jour par lots sont disponibles pour nettoyer ou modifier les données, par exemple pour [GDPR](https://gdpr-info.eu). -3. L’index clairsemé rend ClickHouse pas si approprié pour les requêtes ponctuelles récupérant des lignes simples par leurs clés. +3. L'index clairsemé rend ClickHouse pas si approprié pour les requêtes ponctuelles récupérant des lignes simples par leurs clés. [Article Original](https://clickhouse.tech/docs/en/introduction/distinctive_features/) diff --git a/docs/fr/introduction/history.md b/docs/fr/introduction/history.md index 707c3439e56..8bf1400d7f7 100644 --- a/docs/fr/introduction/history.md +++ b/docs/fr/introduction/history.md @@ -1,22 +1,22 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 7 toc_title: Histoire --- # Histoire De ClickHouse {#clickhouse-history} -ClickHouse a été développé initialement au pouvoir [Yandex.Metrica](https://metrica.yandex.com/), [la deuxième plus grande plateforme d’analyse dans le monde](http://w3techs.com/technologies/overview/traffic_analysis/all) et continue à être le composant de base de ce système. Avec plus de 13 Billions d’enregistrements dans la base de données et plus de 20 milliards d’événements par jour, ClickHouse permet de générer des rapports personnalisés à la volée directement à partir de données non agrégées. Cet article couvre brièvement les objectifs de ClickHouse dans les premiers stades de son développement. +ClickHouse a été développé initialement au pouvoir [Yandex.Metrica](https://metrica.yandex.com/), [la deuxième plus grande plateforme d'analyse dans le monde](http://w3techs.com/technologies/overview/traffic_analysis/all) et continue à être le composant de base de ce système. Avec plus de 13 Billions d'enregistrements dans la base de données et plus de 20 milliards d'événements par jour, ClickHouse permet de générer des rapports personnalisés à la volée directement à partir de données non agrégées. Cet article couvre brièvement les objectifs de ClickHouse dans les premiers stades de son développement. -Yandex.Metrica construit des rapports personnalisés à la volée en fonction des hits et des sessions, avec des segments arbitraires définis par l’utilisateur. Faisant souvent requiert la construction d’agrégats complexes, tels que le nombre d’utilisateurs uniques. De nouvelles données pour la création d’un rapport arrivent en temps réel. +Yandex.Metrica construit des rapports personnalisés à la volée en fonction des hits et des sessions, avec des segments arbitraires définis par l'utilisateur. Faisant souvent requiert la construction d'agrégats complexes, tels que le nombre d'utilisateurs uniques. De nouvelles données pour la création d'un rapport arrivent en temps réel. -En avril 2014, Yandex.Metrica suivait environ 12 milliards d’événements (pages vues et clics) par jour. Tous ces événements doivent être stockés à créer des rapports personnalisés. Une seule requête peut exiger de la numérisation de millions de lignes en quelques centaines de millisecondes, ou des centaines de millions de lignes en quelques secondes. +En avril 2014, Yandex.Metrica suivait environ 12 milliards d'événements (pages vues et clics) par jour. Tous ces événements doivent être stockés à créer des rapports personnalisés. Une seule requête peut exiger de la numérisation de millions de lignes en quelques centaines de millisecondes, ou des centaines de millions de lignes en quelques secondes. -## Utilisation Dans Yandex.Metrica Et Autres Services Yandex {#usage-in-yandex-metrica-and-other-yandex-services} +## Utilisation dans Yandex.Metrica et autres Services Yandex {#usage-in-yandex-metrica-and-other-yandex-services} ClickHouse sert à des fins multiples dans Yandex.Metrica. -Sa tâche principale est de créer des rapports en mode en ligne en utilisant des données non agrégées. Il utilise un cluster de 374 serveurs qui stockent plus de 20,3 billions de lignes dans la base de données. Le volume de données compressées est d’environ 2 PB, sans tenir compte des doublons et des répliques. Le volume de données non compressées (au format TSV) serait d’environ 17 PB. +Sa tâche principale est de créer des rapports en mode en ligne en utilisant des données non agrégées. Il utilise un cluster de 374 serveurs qui stockent plus de 20,3 billions de lignes dans la base de données. Le volume de données compressées est d'environ 2 PB, sans tenir compte des doublons et des répliques. Le volume de données non compressées (au format TSV) serait d'environ 17 PB. ClickHouse joue également un rôle clé dans les processus suivants: @@ -24,33 +24,33 @@ ClickHouse joue également un rôle clé dans les processus suivants: - Traitement des données intermédiaires. - Création de rapports globaux avec Analytics. - Exécution de requêtes pour le débogage du Yandex.Moteur Metrica. -- Analyse des journaux de L’API et de l’interface utilisateur. +- Analyse des journaux de L'API et de l'interface utilisateur. -De nos jours, il existe plusieurs dizaines d’installations ClickHouse dans D’autres services et départements Yandex: recherche verticale, E-commerce, Publicité, business analytics, développement mobile, Services personnels et autres. +De nos jours, il existe plusieurs dizaines d'installations ClickHouse dans D'autres services et départements Yandex: recherche verticale, E-commerce, Publicité, business analytics, développement mobile, Services personnels et autres. -## Données agrégées Et Non agrégées {#aggregated-and-non-aggregated-data} +## Données agrégées et non agrégées {#aggregated-and-non-aggregated-data} Il y a une opinion répandue que pour calculer efficacement les statistiques, vous devez agréger les données car cela réduit le volume de données. -Mais l’agrégation de données est livré avec beaucoup de limitations: +Mais l'agrégation de données est livré avec beaucoup de limitations: -- Vous devez disposer d’une liste prédéfinie des rapports requis. -- L’utilisateur ne peut pas créer de rapports personnalisés. -- Lors de l’agrégation sur un grand nombre de clés distinctes, le volume de données est à peine réduit, l’agrégation est donc inutile. -- Pour un grand nombre de rapports, il y a trop de variations d’agrégation (explosion combinatoire). -- Lors de l’agrégation de clés avec une cardinalité élevée (telles que les URL), le volume de données n’est pas réduit de beaucoup (moins de deux fois). -- Pour cette raison, le volume de données avec l’agrégation peut augmenter au lieu de diminuer. +- Vous devez disposer d'une liste prédéfinie des rapports requis. +- L'utilisateur ne peut pas créer de rapports personnalisés. +- Lors de l'agrégation sur un grand nombre de clés distinctes, le volume de données est à peine réduit, l'agrégation est donc inutile. +- Pour un grand nombre de rapports, il y a trop de variations d'agrégation (explosion combinatoire). +- Lors de l'agrégation de clés avec une cardinalité élevée (telles que les URL), le volume de données n'est pas réduit de beaucoup (moins de deux fois). +- Pour cette raison, le volume de données avec l'agrégation peut augmenter au lieu de diminuer. - Les utilisateurs ne voient pas tous les rapports que nous générons pour eux. Une grande partie de ces calculs est inutile. -- L’intégrité logique des données peut être violée pour diverses agrégations. +- L'intégrité logique des données peut être violée pour diverses agrégations. -Si nous n’agrégeons rien et travaillons avec des données non agrégées, cela pourrait réduire le volume des calculs. +Si nous n'agrégeons rien et travaillons avec des données non agrégées, cela pourrait réduire le volume des calculs. -Cependant, avec l’agrégation, une partie importante du travail est déconnectée et achevée relativement calmement. En revanche, les calculs en ligne nécessitent un calcul aussi rapide que possible, car l’utilisateur attend le résultat. +Cependant, avec l'agrégation, une partie importante du travail est déconnectée et achevée relativement calmement. En revanche, les calculs en ligne nécessitent un calcul aussi rapide que possible, car l'utilisateur attend le résultat. -Yandex.Metrica dispose d’un système spécialisé d’agrégation des données appelé Metrage, qui a été utilisé pour la majorité des rapports. +Yandex.Metrica dispose d'un système spécialisé d'agrégation des données appelé Metrage, qui a été utilisé pour la majorité des rapports. À partir de 2009, Yandex.Metrica a également utilisé une base de données OLAP spécialisée pour les données non agrégées appelée OLAPServer, qui était auparavant utilisée pour le générateur de rapports. -OLAPServer a bien fonctionné pour les données non agrégées, mais il avait de nombreuses restrictions qui ne lui permettaient pas d’être utilisé pour tous les rapports comme souhaité. Ceux-ci comprenaient le manque de prise en charge des types de données (uniquement des nombres) et l’incapacité de mettre à jour progressivement les données en temps réel (cela ne pouvait être fait qu’en réécrivant les données quotidiennement). OLAPServer n’est pas un SGBD, mais une base de données spécialisée. +OLAPServer a bien fonctionné pour les données non agrégées, mais il avait de nombreuses restrictions qui ne lui permettaient pas d'être utilisé pour tous les rapports comme souhaité. Ceux-ci comprenaient le manque de prise en charge des types de données (uniquement des nombres) et l'incapacité de mettre à jour progressivement les données en temps réel (cela ne pouvait être fait qu'en réécrivant les données quotidiennement). OLAPServer n'est pas un SGBD, mais une base de données spécialisée. -L’objectif initial de ClickHouse était de supprimer les limites D’OLAPServer et de résoudre le problème du travail avec des données non agrégées pour tous les rapports, mais au fil des ans, il est devenu un système de gestion de base de données polyvalent adapté à un large éventail de tâches analytiques. +L'objectif initial de ClickHouse était de supprimer les limites D'OLAPServer et de résoudre le problème du travail avec des données non agrégées pour tous les rapports, mais au fil des ans, il est devenu un système de gestion de base de données polyvalent adapté à un large éventail de tâches analytiques. [Article Original](https://clickhouse.tech/docs/en/introduction/history/) diff --git a/docs/fr/introduction/index.md b/docs/fr/introduction/index.md index 365784abfca..e0496b30f00 100644 --- a/docs/fr/introduction/index.md +++ b/docs/fr/introduction/index.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_folder_title: Introduction toc_priority: 1 --- diff --git a/docs/fr/introduction/performance.md b/docs/fr/introduction/performance.md index 0c0852a961d..30d521e11cd 100644 --- a/docs/fr/introduction/performance.md +++ b/docs/fr/introduction/performance.md @@ -1,32 +1,32 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 6 toc_title: Performance --- # Performance {#performance} -Selon les résultats des tests internes chez Yandex, ClickHouse affiche les meilleures performances (à la fois le débit le plus élevé pour les requêtes longues et la latence la plus faible pour les requêtes courtes) pour des scénarios d’exploitation comparables parmi les systèmes de sa classe disponibles pour les tests. Vous pouvez afficher les résultats du test sur un [page séparée](https://clickhouse.tech/benchmark/dbms/). +Selon les résultats des tests internes chez Yandex, ClickHouse affiche les meilleures performances (à la fois le débit le plus élevé pour les requêtes longues et la latence la plus faible pour les requêtes courtes) pour des scénarios d'exploitation comparables parmi les systèmes de sa classe disponibles pour les tests. Vous pouvez afficher les résultats du test sur un [page séparée](https://clickhouse.tech/benchmark/dbms/). De nombreux points de repère indépendants sont arrivés à des conclusions similaires. Ils ne sont pas difficiles à trouver en utilisant une recherche sur internet, ou vous pouvez voir [notre petite collection de liens](https://clickhouse.tech/#independent-benchmarks). -## Débit Pour Une Seule Grande requête {#throughput-for-a-single-large-query} +## Débit pour une seule grande requête {#throughput-for-a-single-large-query} -Le débit peut être mesuré en lignes par seconde ou en mégaoctets par seconde. Si les données sont placées dans le cache de page, une requête pas trop complexe est traitée sur du matériel moderne à une vitesse d’environ 2-10 GB / s de données non compressées sur un seul serveur (pour les cas les plus simples, la vitesse peut atteindre 30 GB/s). Si les données ne sont pas placées dans le cache de page, la vitesse dépend du sous-système de disque et du taux de compression des données. Par exemple, si le sous-système de disque permet de lire des données à 400 Mo/s et que le taux de compression des données est de 3, la vitesse devrait être d’environ 1,2 Go/s. Pour obtenir la vitesse en lignes par seconde, divisez la vitesse en octets par seconde par la taille totale des colonnes utilisées dans la requête. Par exemple, si 10 octets de colonnes sont extraites, la vitesse devrait être d’environ 100 à 200 millions de lignes par seconde. +Le débit peut être mesuré en lignes par seconde ou en mégaoctets par seconde. Si les données sont placées dans le cache de page, une requête pas trop complexe est traitée sur du matériel moderne à une vitesse d'environ 2-10 GB / s de données non compressées sur un seul serveur (pour les cas les plus simples, la vitesse peut atteindre 30 GB/s). Si les données ne sont pas placées dans le cache de page, la vitesse dépend du sous-système de disque et du taux de compression des données. Par exemple, si le sous-système de disque permet de lire des données à 400 Mo/s et que le taux de compression des données est de 3, la vitesse devrait être d'environ 1,2 Go/s. Pour obtenir la vitesse en lignes par seconde, divisez la vitesse en octets par seconde par la taille totale des colonnes utilisées dans la requête. Par exemple, si 10 octets de colonnes sont extraites, la vitesse devrait être d'environ 100 à 200 millions de lignes par seconde. -La vitesse de traitement augmente presque linéairement pour le traitement distribué, mais seulement si le nombre de lignes résultant de l’agrégation ou du tri n’est pas trop important. +La vitesse de traitement augmente presque linéairement pour le traitement distribué, mais seulement si le nombre de lignes résultant de l'agrégation ou du tri n'est pas trop important. ## Latence Lors Du Traitement Des Requêtes Courtes {#latency-when-processing-short-queries} -Si une requête utilise une clé primaire et ne sélectionne pas trop de colonnes et de lignes à traiter (des centaines de milliers), Vous pouvez vous attendre à moins de 50 millisecondes de latence (un seul chiffre de millisecondes dans le meilleur des cas) si les données sont placées dans le cache de page. Sinon, la latence est principalement dominée par le nombre de recherches. Si vous utilisez des lecteurs de disque rotatifs, pour un système qui n’est pas surchargé, la latence peut être estimée avec cette formule: `seek time (10 ms) * count of columns queried * count of data parts`. +Si une requête utilise une clé primaire et ne sélectionne pas trop de colonnes et de lignes à traiter (des centaines de milliers), Vous pouvez vous attendre à moins de 50 millisecondes de latence (un seul chiffre de millisecondes dans le meilleur des cas) si les données sont placées dans le cache de page. Sinon, la latence est principalement dominée par le nombre de recherches. Si vous utilisez des lecteurs de disque rotatifs, pour un système qui n'est pas surchargé, la latence peut être estimée avec cette formule: `seek time (10 ms) * count of columns queried * count of data parts`. -## Débit Lors Du Traitement d’une Grande quantité De requêtes Courtes {#throughput-when-processing-a-large-quantity-of-short-queries} +## Débit lors du traitement d'une grande quantité de requêtes courtes {#throughput-when-processing-a-large-quantity-of-short-queries} -Dans les mêmes conditions, ClickHouse peut traiter plusieurs centaines de requêtes par seconde sur un seul serveur (jusqu’à plusieurs milliers dans le meilleur des cas). Étant donné que ce scénario n’est pas typique pour les SGBD analytiques, nous vous recommandons d’attendre un maximum de 100 requêtes par seconde. +Dans les mêmes conditions, ClickHouse peut traiter plusieurs centaines de requêtes par seconde sur un seul serveur (jusqu'à plusieurs milliers dans le meilleur des cas). Étant donné que ce scénario n'est pas typique pour les SGBD analytiques, nous vous recommandons d'attendre un maximum de 100 requêtes par seconde. -## Performances Lors De L’Insertion De Données {#performance-when-inserting-data} +## Performances Lors De L'Insertion De Données {#performance-when-inserting-data} -Nous vous recommandons d’insérer des données dans des paquets d’au moins 1000 lignes, ou pas plus qu’une seule demande par seconde. Lors de l’insertion dans une table MergeTree à partir d’un dump séparé par des tabulations, la vitesse d’insertion peut être de 50 à 200 Mo/s. Si les lignes insérées ont une taille d’environ 1 KO, La vitesse sera de 50 000 à 200 000 lignes par seconde. Si les lignes sont petites, les performances peuvent être plus élevées en lignes par seconde (sur les données du système de bannière -`>` 500 000 lignes par seconde; sur les données de Graphite -`>` 1 000 000 lignes par seconde). Pour améliorer les performances, vous pouvez effectuer plusieurs requêtes D’insertion en parallèle, qui s’adaptent linéairement. +Nous vous recommandons d'insérer des données dans des paquets d'au moins 1000 lignes, ou pas plus qu'une seule demande par seconde. Lors de l'insertion dans une table MergeTree à partir d'un dump séparé par des tabulations, la vitesse d'insertion peut être de 50 à 200 Mo/s. Si les lignes insérées ont une taille d'environ 1 KO, La vitesse sera de 50 000 à 200 000 lignes par seconde. Si les lignes sont petites, les performances peuvent être plus élevées en lignes par seconde (sur les données du système de bannière -`>` 500 000 lignes par seconde; sur les données de Graphite -`>` 1 000 000 lignes par seconde). Pour améliorer les performances, vous pouvez effectuer plusieurs requêtes D'insertion en parallèle, qui s'adaptent linéairement. [Article Original](https://clickhouse.tech/docs/en/introduction/performance/) diff --git a/docs/fr/operations/access-rights.md b/docs/fr/operations/access-rights.md index 8e0429be454..350a19bb7de 100644 --- a/docs/fr/operations/access-rights.md +++ b/docs/fr/operations/access-rights.md @@ -1,113 +1,143 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 48 -toc_title: "Les Droits D'Acc\xE8s" +toc_title: "Le Contr\xF4le d'acc\xE8s et de Gestion de Compte" --- -# Les Droits D’Accès {#access-rights} +# Le Contrôle d'accès et de Gestion de Compte {#access-control} -Les utilisateurs et les droits d’accès sont configurés dans la configuration utilisateur. Ce n’est généralement `users.xml`. +Clickhouse prend en charge la gestion du contrôle d'accès basée sur [RBAC](https://en.wikipedia.org/wiki/Role-based_access_control) approche. -Les utilisateurs sont enregistrés dans le `users` section. Voici un fragment de la `users.xml` fichier: +Entités d'accès ClickHouse: +- [Compte d'utilisateur](#user-account-management) +- [Rôle](#role-management) +- [La Ligne Politique](#row-policy-management) +- [Les Paramètres De Profil](#settings-profiles-management) +- [Quota](#quotas-management) -``` xml - - - - - - +- Serveur [les fichiers de configuration](configuration-files.md) `users.xml` et `config.xml`. - - +!!! note "Avertissement" + Vous ne pouvez pas gérer la même entité d'accès par les deux méthodes de configuration simultanément. - - default +## Utilisation {#access-control-usage} - - default - +Par défaut, le serveur ClickHouse fournit le compte utilisateur `default` ce qui n'est pas autorisé à utiliser le contrôle D'accès piloté par SQL et la gestion de compte, mais a tous les droits et autorisations. Le `default` compte d'utilisateur est utilisé dans tous les cas, lorsque l'utilisateur n'est pas défini, par exemple, lors de la connexion du client ou dans les requêtes distribuées. Dans le traitement des requêtes distribuées, un compte utilisateur par défaut est utilisé si la configuration du serveur ou du cluster ne spécifie pas [d'utilisateur et mot de passe](../engines/table-engines/special/distributed.md) propriété. - - - - - web - default - - test - - - test - - - -``` +Si vous commencez simplement à utiliser ClickHouse, vous pouvez utiliser le scénario suivant: -Vous pouvez voir une déclaration de deux utilisateurs: `default`et`web`. Nous avons ajouté l’ `web` utilisateur séparément. +1. [Permettre](#enabling-access-control) Contrôle D'accès piloté par SQL et gestion de compte pour le `default` utilisateur. +2. Connexion en vertu de la `default` compte d'utilisateur et de créer tous les utilisateurs. N'oubliez pas de créer un compte d'administrateur (`GRANT ALL ON *.* WITH GRANT OPTION TO admin_user_account`). +3. [Restreindre les autorisations](settings/permissions-for-queries.md#permissions_for_queries) pour l' `default` utilisateur et désactiver le contrôle D'accès piloté par SQL et la gestion des comptes pour elle. -Le `default` l’utilisateur est choisi dans les cas où le nom d’utilisateur n’est pas passé. Le `default` l’utilisateur est également utilisé pour le traitement des requêtes distribuées, si la configuration du serveur ou du cluster `user` et `password` (voir la section sur les [Distribué](../engines/table-engines/special/distributed.md) moteur). +### Propriétés de la Solution actuelle {#access-control-properties} -The user that is used for exchanging information between servers combined in a cluster must not have substantial restrictions or quotas – otherwise, distributed queries will fail. +- Vous pouvez accorder des autorisations pour les bases de données et les tables même si elles n'existent pas. +- Si une table a été supprimée, tous les privilèges correspondant à cette table ne sont pas révoqués. Ainsi, si une nouvelle table est créée plus tard avec le même nom, tous les privilèges redeviennent réels. Pour révoquer les privilèges correspondant à la table supprimée, vous devez effectuer, par exemple, l' `REVOKE ALL PRIVILEGES ON db.table FROM ALL` requête. +- Il n'y a pas de paramètres de durée de vie pour les privilèges. -Le mot de passe est spécifié en texte clair (non recommandé) ou en SHA-256. Le hash n’est pas salé. À cet égard, vous ne devez pas considérer ces mots de passe comme assurant la sécurité contre les attaques malveillantes potentielles. Au contraire, ils sont nécessaires pour la protection contre les employés. +## Compte d'utilisateur {#user-account-management} -Une liste de réseaux est précisé que l’accès est autorisé à partir. Dans cet exemple, la liste des réseaux pour les utilisateurs est chargé à partir d’un fichier séparé (`/etc/metrika.xml`) contenant les `networks` substitution. Voici un fragment de: +Un compte d'utilisateur est une entité qui permet d'autoriser quelqu'un à ClickHouse. Un compte utilisateur contient: -``` xml - - ... - - ::/64 - 203.0.113.0/24 - 2001:DB8::/32 - ... - - -``` +- Informations d'Identification. +- [Privilège](../sql-reference/statements/grant.md#grant-privileges) qui définissent l'étendue des requêtes que l'utilisateur peut effectuer. +- Hôtes à partir desquels la connexion au serveur ClickHouse est autorisée. +- Rôles accordés et par défaut. +- Paramètres avec leurs contraintes qui s'appliquent par défaut lors de la connexion de l'utilisateur. +- Profils de paramètres assignés. -Vous pouvez définir cette liste de réseaux directement dans `users.xml` ou dans un fichier dans le `users.d` répertoire (pour plus d’informations, consultez la section “[Fichiers de Configuration](configuration-files.md#configuration_files)”). +Des privilèges à un compte d'utilisateur peuvent être accordés par [GRANT](../sql-reference/statements/grant.md) requête ou en attribuant [rôle](#role-management). Pour révoquer les privilèges d'un utilisateur, ClickHouse fournit [REVOKE](../sql-reference/statements/revoke.md) requête. Pour lister les privilèges d'un utilisateur, utilisez - [SHOW GRANTS](../sql-reference/statements/show.md#show-grants-statement) déclaration. -La configuration comprend des commentaires expliquant comment ouvrir l’accès de partout. +Gestion des requêtes: -Pour une utilisation en production, spécifiez uniquement `ip` (adresses IP et leurs masques), depuis l’utilisation `host` et `hoost_regexp` peut causer une latence supplémentaire. +- [CREATE USER](../sql-reference/statements/create.md#create-user-statement) +- [ALTER USER](../sql-reference/statements/alter.md#alter-user-statement) +- [DROP USER](../sql-reference/statements/misc.md#drop-user-statement) +- [SHOW CREATE USER](../sql-reference/statements/show.md#show-create-user-statement) -Ensuite, le profil des paramètres utilisateur est spécifié (voir la section “[Les paramètres des profils](settings/settings-profiles.md)”. Vous pouvez spécifier le profil par défaut, `default'`. Le profil peut avoir n’importe quel nom. Vous pouvez spécifier le même profil pour différents utilisateurs. La chose la plus importante que vous pouvez écrire dans les paramètres de profil `readonly=1` qui assure un accès en lecture seule. Spécifiez ensuite le quota à utiliser (voir la section “[Quota](quotas.md#quotas)”). Vous pouvez spécifier le quota par défaut: `default`. It is set in the config by default to only count resource usage, without restricting it. The quota can have any name. You can specify the same quota for different users – in this case, resource usage is calculated for each user individually. +### Paramètres Application {#access-control-settings-applying} -Dans le facultatif `` section, vous pouvez également spécifier une liste de bases de données que l’utilisateur peut accéder. Par défaut, toutes les bases de données sont disponibles pour l’utilisateur. Vous pouvez spécifier l’ `default` la base de données. Dans ce cas, l’utilisateur recevra l’accès à la base de données par défaut. +Les paramètres peuvent être définis de différentes manières: pour un compte utilisateur, dans ses profils de rôles et de paramètres accordés. Lors d'une connexion utilisateur, si un paramètre est défini dans différentes entités d'accès, la valeur et les contraintes de ce paramètre sont appliquées par les priorités suivantes (de plus haut à plus bas): -Dans le facultatif `` section, vous pouvez également spécifier une liste de dictionnaires que l’utilisateur peut accéder. Par défaut, tous les dictionnaires sont disponibles pour l’utilisateur. +1. Paramètre de compte utilisateur. +2. Les paramètres de rôles par défaut du compte d'utilisateur. Si un paramètre est défini dans certains rôles, l'ordre de la mise en application n'est pas défini. +3. Les paramètres dans les profils de paramètres attribués à un utilisateur ou à ses rôles par défaut. Si un paramètre est défini dans certains profils, l'ordre d'application des paramètres n'est pas défini. +4. Paramètres appliqués à l'ensemble du serveur par défaut [profil par défaut](server-configuration-parameters/settings.md#default-profile). -L’accès à la `system` la base de données est toujours autorisée (puisque cette base de données est utilisée pour traiter les requêtes). +## Rôle {#role-management} -L’utilisateur peut obtenir une liste de toutes les bases de données et tables en utilisant `SHOW` requêtes ou tables système, même si l’accès aux bases de données individuelles n’est pas autorisé. +Le rôle est un conteneur pour l'accès des entités qui peuvent être accordées à un compte d'utilisateur. -Accès de base de données n’est pas liée à la [ReadOnly](settings/permissions-for-queries.md#settings_readonly) paramètre. Vous ne pouvez pas accorder un accès complet à une base de données et `readonly` l’accès à un autre. +Rôle contient: + +- [Privilège](../sql-reference/statements/grant.md#grant-privileges) +- Paramètres et contraintes +- Liste des rôles attribués + +Gestion des requêtes: + +- [CREATE ROLE](../sql-reference/statements/create.md#create-role-statement) +- [ALTER ROLE](../sql-reference/statements/alter.md#alter-role-statement) +- [DROP ROLE](../sql-reference/statements/misc.md#drop-role-statement) +- [SET ROLE](../sql-reference/statements/misc.md#set-role-statement) +- [SET DEFAULT ROLE](../sql-reference/statements/misc.md#set-default-role-statement) +- [SHOW CREATE ROLE](../sql-reference/statements/show.md#show-create-role-statement) + +Les privilèges d'un rôle peuvent être accordés par [GRANT](../sql-reference/statements/grant.md) requête. Pour révoquer les privilèges D'un rôle ClickHouse fournit [REVOKE](../sql-reference/statements/revoke.md) requête. + +## La Ligne Politique {#row-policy-management} + +La stratégie de ligne est un filtre qui définit les lignes disponibles pour un utilisateur ou pour un rôle. La stratégie de ligne contient des filtres pour une table spécifique et une liste de rôles et / ou d'utilisateurs qui doivent utiliser cette stratégie de ligne. + +Gestion des requêtes: + +- [CREATE ROW POLICY](../sql-reference/statements/create.md#create-row-policy-statement) +- [ALTER ROW POLICY](../sql-reference/statements/alter.md#alter-row-policy-statement) +- [DROP ROW POLICY](../sql-reference/statements/misc.md#drop-row-policy-statement) +- [SHOW CREATE ROW POLICY](../sql-reference/statements/show.md#show-create-row-policy-statement) + +## Les Paramètres De Profil {#settings-profiles-management} + +Paramètres profil est une collection de [paramètre](settings/index.md). Le profil paramètres contient les paramètres et les contraintes, ainsi que la liste des rôles et/ou des utilisateurs auxquels ce quota est appliqué. + +Gestion des requêtes: + +- [CREATE SETTINGS PROFILE](../sql-reference/statements/create.md#create-settings-profile-statement) +- [ALTER SETTINGS PROFILE](../sql-reference/statements/alter.md#alter-settings-profile-statement) +- [DROP SETTINGS PROFILE](../sql-reference/statements/misc.md#drop-settings-profile-statement) +- [SHOW CREATE SETTINGS PROFILE](../sql-reference/statements/show.md#show-create-settings-profile-statement) + +## Quota {#quotas-management} + +Le Quota limite l'utilisation des ressources. Voir [Quota](quotas.md). + +Quota contient un ensemble de limites pour certaines durées, et la liste des rôles et/ou des utilisateurs qui devrait utiliser ce quota. + +Gestion des requêtes: + +- [CREATE QUOTA](../sql-reference/statements/create.md#create-quota-statement) +- [ALTER QUOTA](../sql-reference/statements/alter.md#alter-quota-statement) +- [DROP QUOTA](../sql-reference/statements/misc.md#drop-quota-statement) +- [SHOW CREATE QUOTA](../sql-reference/statements/show.md#show-create-quota-statement) + +## Activation du contrôle D'accès piloté par SQL et de la gestion de Compte {#enabling-access-control} + +- Configurez un répertoire pour le stockage des configurations. + + Clickhouse stocke les configurations d'entité d'accès dans le dossier défini dans [access\_control\_path](server-configuration-parameters/settings.md#access_control_path) paramètre de configuration du serveur. + +- Activez le contrôle D'accès piloté par SQL et la gestion de compte pour au moins un compte d'utilisateur. + + Par défaut, le contrôle D'accès piloté par SQL et la gestion des comptes sont activés pour tous les utilisateurs. Vous devez configurer au moins un utilisateur dans le `users.xml` fichier de configuration et affecter 1 au [access\_management](settings/settings-users.md#access_management-user-setting) paramètre. [Article Original](https://clickhouse.tech/docs/en/operations/access_rights/) diff --git a/docs/fr/operations/backup.md b/docs/fr/operations/backup.md index 36e42e150b4..9a463372947 100644 --- a/docs/fr/operations/backup.md +++ b/docs/fr/operations/backup.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 49 toc_title: "La Sauvegarde Des Donn\xE9es" --- @@ -9,20 +9,20 @@ toc_title: "La Sauvegarde Des Donn\xE9es" Alors [réplication](../engines/table-engines/mergetree-family/replication.md) provides protection from hardware failures, it does not protect against human errors: accidental deletion of data, deletion of the wrong table or a table on the wrong cluster, and software bugs that result in incorrect data processing or data corruption. In many cases mistakes like these will affect all replicas. ClickHouse has built-in safeguards to prevent some types of mistakes — for example, by default [vous ne pouvez pas simplement supprimer des tables avec un moteur de type MergeTree contenant plus de 50 Go de données](https://github.com/ClickHouse/ClickHouse/blob/v18.14.18-stable/programs/server/config.xml#L322-L330). Toutefois, ces garanties ne couvrent pas tous les cas possibles et peuvent être contournés. -Afin d’atténuer efficacement les erreurs humaines possibles, vous devez préparer soigneusement une stratégie de sauvegarde et de restauration de vos données **préalablement**. +Afin d'atténuer efficacement les erreurs humaines possibles, vous devez préparer soigneusement une stratégie de sauvegarde et de restauration de vos données **préalablement**. -Chaque entreprise a différentes ressources disponibles et les exigences de l’entreprise, donc il n’y a pas de solution universelle pour les sauvegardes et restaurations ClickHouse qui s’adaptera à toutes les situations. Ce qui fonctionne pour un gigaoctet de données ne fonctionnera probablement pas pour des dizaines de pétaoctets. Il existe une variété d’approches possibles avec leurs propres avantages et inconvénients, qui sera discuté ci-dessous. C’est une bonne idée d’utiliser plusieurs approches au lieu d’un seul, afin de compenser leurs lacunes. +Chaque entreprise a différentes ressources disponibles et les exigences de l'entreprise, donc il n'y a pas de solution universelle pour les sauvegardes et restaurations ClickHouse qui s'adaptera à toutes les situations. Ce qui fonctionne pour un gigaoctet de données ne fonctionnera probablement pas pour des dizaines de pétaoctets. Il existe une variété d'approches possibles avec leurs propres avantages et inconvénients, qui sera discuté ci-dessous. C'est une bonne idée d'utiliser plusieurs approches au lieu d'un seul, afin de compenser leurs lacunes. !!! note "Note" - Gardez à l’esprit que si vous avez sauvegardé quelque chose et que vous n’avez jamais essayé de le restaurer, il est probable que la restauration ne fonctionnera pas correctement lorsque vous en avez réellement besoin (ou du moins cela prendra plus de temps que ce que les entreprises peuvent tolérer). Donc, quelle que soit l’approche de sauvegarde que vous choisissez, assurez-vous d’automatiser le processus de restauration et de le pratiquer sur un cluster clickhouse de rechange régulièrement. + Gardez à l'esprit que si vous avez sauvegardé quelque chose et que vous n'avez jamais essayé de le restaurer, il est probable que la restauration ne fonctionnera pas correctement lorsque vous en avez réellement besoin (ou du moins cela prendra plus de temps que ce que les entreprises peuvent tolérer). Donc, quelle que soit l'approche de sauvegarde que vous choisissez, assurez-vous d'automatiser le processus de restauration et de le pratiquer sur un cluster clickhouse de rechange régulièrement. ## Dupliquer Les Données Sources Ailleurs {#duplicating-source-data-somewhere-else} -Souvent, les données qui sont ingérées dans ClickHouse sont livrées via une sorte de file d’attente persistante, telle que [Apache Kafka](https://kafka.apache.org). Dans ce cas, il est possible de configurer un ensemble supplémentaire d’abonnés qui liront le même flux de données pendant qu’il est écrit dans ClickHouse et le stockeront dans un stockage à froid quelque part. La plupart des entreprises ont déjà un stockage à froid recommandé par défaut, qui pourrait être un magasin d’objets ou un système de fichiers distribué comme [HDFS](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HdfsDesign.html). +Souvent, les données qui sont ingérées dans ClickHouse sont livrées via une sorte de file d'attente persistante, telle que [Apache Kafka](https://kafka.apache.org). Dans ce cas, il est possible de configurer un ensemble supplémentaire d'abonnés qui liront le même flux de données pendant qu'il est écrit dans ClickHouse et le stockeront dans un stockage à froid quelque part. La plupart des entreprises ont déjà un stockage à froid recommandé par défaut, qui pourrait être un magasin d'objets ou un système de fichiers distribué comme [HDFS](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HdfsDesign.html). ## Instantanés Du Système De Fichiers {#filesystem-snapshots} -Certains systèmes de fichiers locaux fournissent des fonctionnalités d’instantané (par exemple, [ZFS](https://en.wikipedia.org/wiki/ZFS)), mais ils pourraient ne pas être le meilleur choix pour servir les requêtes actives. Une solution possible consiste à créer des répliques supplémentaires avec ce type de système de fichiers et à les exclure du [Distribué](../engines/table-engines/special/distributed.md) les tables qui sont utilisés pour `SELECT` requête. Les instantanés sur ces répliques seront hors de portée des requêtes qui modifient les données. En prime, ces répliques pourraient avoir des configurations matérielles spéciales avec plus de disques attachés par serveur, ce qui serait rentable. +Certains systèmes de fichiers locaux fournissent des fonctionnalités d'instantané (par exemple, [ZFS](https://en.wikipedia.org/wiki/ZFS)), mais ils pourraient ne pas être le meilleur choix pour servir les requêtes actives. Une solution possible consiste à créer des répliques supplémentaires avec ce type de système de fichiers et à les exclure du [Distribué](../engines/table-engines/special/distributed.md) les tables qui sont utilisés pour `SELECT` requête. Les instantanés sur ces répliques seront hors de portée des requêtes qui modifient les données. En prime, ces répliques pourraient avoir des configurations matérielles spéciales avec plus de disques attachés par serveur, ce qui serait rentable. ## clickhouse-copieur {#clickhouse-copier} @@ -30,11 +30,11 @@ Certains systèmes de fichiers locaux fournissent des fonctionnalités d’insta Pour de plus petits volumes de données, un simple `INSERT INTO ... SELECT ...` les tables distantes peuvent également fonctionner. -## Manipulations Avec Des Pièces {#manipulations-with-parts} +## Manipulations avec des pièces {#manipulations-with-parts} -ClickHouse permet d’utiliser le `ALTER TABLE ... FREEZE PARTITION ...` requête pour créer une copie locale des partitions de table. Ceci est implémenté en utilisant des liens durs vers le `/var/lib/clickhouse/shadow/` dossier, donc il ne consomme généralement pas d’espace disque supplémentaire pour les anciennes données. Les copies créées des fichiers ne sont pas gérées par clickhouse server, vous pouvez donc les laisser là: vous aurez une sauvegarde simple qui ne nécessite aucun système externe supplémentaire, mais elle sera toujours sujette à des problèmes matériels. Pour cette raison, il est préférable de les copier à distance vers un autre emplacement, puis de supprimer les copies locales. Les systèmes de fichiers distribués et les magasins d’objets sont toujours une bonne option pour cela, mais les serveurs de fichiers attachés normaux avec une capacité suffisante peuvent également fonctionner (dans ce cas, le transfert se fera via le système de fichiers réseau ou peut-être [rsync](https://en.wikipedia.org/wiki/Rsync)). +ClickHouse permet d'utiliser le `ALTER TABLE ... FREEZE PARTITION ...` requête pour créer une copie locale des partitions de table. Ceci est implémenté en utilisant des liens durs vers le `/var/lib/clickhouse/shadow/` dossier, donc il ne consomme généralement pas d'espace disque supplémentaire pour les anciennes données. Les copies créées des fichiers ne sont pas gérées par clickhouse server, vous pouvez donc les laisser là: vous aurez une sauvegarde simple qui ne nécessite aucun système externe supplémentaire, mais elle sera toujours sujette à des problèmes matériels. Pour cette raison, il est préférable de les copier à distance vers un autre emplacement, puis de supprimer les copies locales. Les systèmes de fichiers distribués et les magasins d'objets sont toujours une bonne option pour cela, mais les serveurs de fichiers attachés normaux avec une capacité suffisante peuvent également fonctionner (dans ce cas, le transfert se fera via le système de fichiers réseau ou peut-être [rsync](https://en.wikipedia.org/wiki/Rsync)). -Pour plus d’informations sur les requêtes liées aux manipulations de [Modifier la documentation](../sql-reference/statements/alter.md#alter_manipulations-with-partitions). +Pour plus d'informations sur les requêtes liées aux manipulations de [Modifier la documentation](../sql-reference/statements/alter.md#alter_manipulations-with-partitions). Un outil tiers est disponible pour automatiser cette approche: [clickhouse-sauvegarde](https://github.com/AlexAkulov/clickhouse-backup). diff --git a/docs/fr/operations/configuration-files.md b/docs/fr/operations/configuration-files.md index 82ffbcd9ea7..71569cc9385 100644 --- a/docs/fr/operations/configuration-files.md +++ b/docs/fr/operations/configuration-files.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 50 toc_title: Fichiers De Configuration --- @@ -12,17 +12,17 @@ ClickHouse prend en charge la gestion de la configuration multi-fichiers. Le fic !!! note "Note" Tous les fichiers de configuration doivent être au format XML. Aussi, ils doivent avoir le même élément racine, généralement ``. -Certains paramètres spécifiés dans le fichier de configuration principal peuvent être remplacés dans d’autres fichiers de configuration. Le `replace` ou `remove` les attributs peuvent être spécifiés pour les éléments de ces fichiers de configuration. +Certains paramètres spécifiés dans le fichier de configuration principal peuvent être remplacés dans d'autres fichiers de configuration. Le `replace` ou `remove` les attributs peuvent être spécifiés pour les éléments de ces fichiers de configuration. -Si ni l’un ni l’autre n’est spécifié, il combine le contenu des éléments de manière récursive, remplaçant les valeurs des enfants en double. +Si ni l'un ni l'autre n'est spécifié, il combine le contenu des éléments de manière récursive, remplaçant les valeurs des enfants en double. -Si `replace` est spécifié, il remplace l’élément entier par celui spécifié. +Si `replace` est spécifié, il remplace l'élément entier par celui spécifié. -Si `remove` est spécifié, il supprime l’élément. +Si `remove` est spécifié, il supprime l'élément. -La configuration peut également définir “substitutions”. Si un élément a le `incl` attribut, la substitution correspondante du fichier sera utilisée comme valeur. Par défaut, le chemin d’accès au fichier avec des substitutions est `/etc/metrika.xml`. Ceci peut être changé dans le [include\_from](server-configuration-parameters/settings.md#server_configuration_parameters-include_from) élément dans la configuration du serveur. Les valeurs de substitution sont spécifiées dans `/yandex/substitution_name` les éléments de ce fichier. Si une substitution spécifiée dans `incl` n’existe pas, il est enregistré dans le journal. Pour empêcher ClickHouse de consigner les substitutions manquantes, spécifiez `optional="true"` attribut (par exemple, les paramètres de [macro](server-configuration-parameters/settings.md)). +La configuration peut également définir “substitutions”. Si un élément a le `incl` attribut, la substitution correspondante du fichier sera utilisée comme valeur. Par défaut, le chemin d'accès au fichier avec des substitutions est `/etc/metrika.xml`. Ceci peut être changé dans le [include\_from](server-configuration-parameters/settings.md#server_configuration_parameters-include_from) élément dans la configuration du serveur. Les valeurs de substitution sont spécifiées dans `/yandex/substitution_name` les éléments de ce fichier. Si une substitution spécifiée dans `incl` n'existe pas, il est enregistré dans le journal. Pour empêcher ClickHouse de consigner les substitutions manquantes, spécifiez `optional="true"` attribut (par exemple, les paramètres de [macro](server-configuration-parameters/settings.md)). -Les Substitutions peuvent également être effectuées à partir de ZooKeeper. Pour ce faire, spécifiez l’attribut `from_zk = "/path/to/node"`. La valeur de l’élément est remplacé par le contenu du noeud au `/path/to/node` dans ZooKeeper. Vous pouvez également placer un sous-arbre XML entier sur le nœud ZooKeeper et il sera entièrement inséré dans l’élément source. +Les Substitutions peuvent également être effectuées à partir de ZooKeeper. Pour ce faire, spécifiez l'attribut `from_zk = "/path/to/node"`. La valeur de l'élément est remplacé par le contenu du noeud au `/path/to/node` dans ZooKeeper. Vous pouvez également placer un sous-arbre XML entier sur le nœud ZooKeeper et il sera entièrement inséré dans l'élément source. Le `config.xml` le fichier peut spécifier une configuration distincte avec les paramètres utilisateur, les profils et les quotas. Le chemin relatif à cette configuration est défini dans `users_config` élément. Par défaut, il est `users.xml`. Si `users_config` est omis, les paramètres utilisateur, les profils et les quotas sont `config.xml`. @@ -50,7 +50,7 @@ $ cat /etc/clickhouse-server/users.d/alice.xml ``` -Pour chaque fichier de configuration, le serveur génère également `file-preprocessed.xml` les fichiers lors du démarrage. Ces fichiers contiennent toutes les remplacements et des remplacements, et ils sont destinés à l’usage informatif. Si des substitutions ZooKeeper ont été utilisées dans les fichiers de configuration mais que ZooKeeper n’est pas disponible au démarrage du serveur, le serveur charge la configuration à partir du fichier prétraité. +Pour chaque fichier de configuration, le serveur génère également `file-preprocessed.xml` les fichiers lors du démarrage. Ces fichiers contiennent toutes les remplacements et des remplacements, et ils sont destinés à l'usage informatif. Si des substitutions ZooKeeper ont été utilisées dans les fichiers de configuration mais que ZooKeeper n'est pas disponible au démarrage du serveur, le serveur charge la configuration à partir du fichier prétraité. Le serveur suit les changements dans les fichiers de configuration, ainsi que les fichiers et les nœuds ZooKeeper utilisés lors des substitutions et des remplacements, et recharge les paramètres pour les utilisateurs et les clusters à la volée. Cela signifie que vous pouvez modifier le cluster, les utilisateurs et leurs paramètres sans redémarrer le serveur. diff --git a/docs/fr/operations/index.md b/docs/fr/operations/index.md index f680cbcf9ee..23f98bdaed2 100644 --- a/docs/fr/operations/index.md +++ b/docs/fr/operations/index.md @@ -1,21 +1,21 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 -toc_folder_title: Operations +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "Op\xE9rations" toc_priority: 41 toc_title: Introduction --- # Opérations {#operations} -Le manuel d’exploitation de ClickHouse comprend les principales sections suivantes: +Le manuel d'exploitation de ClickHouse comprend les principales sections suivantes: - [Exigence](requirements.md) - [Surveiller](monitoring.md) - [Dépannage](troubleshooting.md) -- [Recommandations D’Utilisation](tips.md) +- [Recommandations D'Utilisation](tips.md) - [Procédure De Mise À Jour](update.md) -- [Les Droits D’Accès](access-rights.md) +- [Les Droits D'Accès](access-rights.md) - [La Sauvegarde Des Données](backup.md) - [Fichiers De Configuration](configuration-files.md) - [Quota](quotas.md) @@ -25,4 +25,4 @@ Le manuel d’exploitation de ClickHouse comprend les principales sections suiva - [Paramètre](settings/index.md) - [Utilitaire](utilities/index.md) -[Article Original](https://clickhouse.tech/docs/en/operations/) +{## [Article Original](https://clickhouse.tech/docs/en/operations/) ##} diff --git a/docs/fr/operations/monitoring.md b/docs/fr/operations/monitoring.md index fc0eaaa4bc6..275e8a5d8ac 100644 --- a/docs/fr/operations/monitoring.md +++ b/docs/fr/operations/monitoring.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 45 toc_title: Surveiller --- @@ -9,24 +9,24 @@ toc_title: Surveiller Vous pouvez surveiller: -- L’utilisation des ressources matérielles. +- L'utilisation des ressources matérielles. - Statistiques du serveur ClickHouse. -## L’Utilisation Des Ressources {#resource-utilization} +## L'Utilisation Des Ressources {#resource-utilization} -ClickHouse ne surveille pas l’état des ressources matérielles par lui-même. +ClickHouse ne surveille pas l'état des ressources matérielles par lui-même. Il est fortement recommandé de configurer la surveillance de: - Charge et température sur les processeurs. - Vous pouvez utiliser [dmesg](https://en.wikipedia.org/wiki/Dmesg), [turbostat](https://www.linux.org/docs/man8/turbostat.html) ou d’autres instruments. + Vous pouvez utiliser [dmesg](https://en.wikipedia.org/wiki/Dmesg), [turbostat](https://www.linux.org/docs/man8/turbostat.html) ou d'autres instruments. - Utilisation du système de stockage, de la RAM et du réseau. -## Métriques Du Serveur Clickhouse {#clickhouse-server-metrics} +## Métriques Du Serveur ClickHouse {#clickhouse-server-metrics} -Clickhouse server a des instruments embarqués pour la surveillance de l’auto-état. +Clickhouse server a des instruments embarqués pour la surveillance de l'auto-état. Pour suivre les événements du serveur, utilisez les journaux du serveur. Voir la [enregistreur](server-configuration-parameters/settings.md#server_configuration_parameters-logger) section du fichier de configuration. @@ -37,8 +37,10 @@ Clickhouse recueille: Vous pouvez trouver des mesures dans le [système.métrique](../operations/system-tables.md#system_tables-metrics), [système.événement](../operations/system-tables.md#system_tables-events), et [système.asynchronous\_metrics](../operations/system-tables.md#system_tables-asynchronous_metrics) table. -Vous pouvez configurer ClickHouse pour exporter des métriques vers [Graphite](https://github.com/graphite-project). Voir la [Graphite section](server-configuration-parameters/settings.md#server_configuration_parameters-graphite) dans le fichier de configuration du serveur ClickHouse. Avant de configurer l’exportation des métriques, vous devez configurer Graphite en suivant leur [guide](https://graphite.readthedocs.io/en/latest/install.html). +Vous pouvez configurer ClickHouse pour exporter des métriques vers [Graphite](https://github.com/graphite-project). Voir la [Graphite section](server-configuration-parameters/settings.md#server_configuration_parameters-graphite) dans le fichier de configuration du serveur ClickHouse. Avant de configurer l'exportation des métriques, vous devez configurer Graphite en suivant leur [guide](https://graphite.readthedocs.io/en/latest/install.html). -De plus, vous pouvez surveiller la disponibilité du serveur via L’API HTTP. Envoyer la `HTTP GET` demande à `/ping`. Si le serveur est disponible, il répond avec `200 OK`. +Vous pouvez configurer ClickHouse pour exporter des métriques vers [Prometheus](https://prometheus.io). Voir la [Prometheus section](server-configuration-parameters/settings.md#server_configuration_parameters-prometheus) dans le fichier de configuration du serveur ClickHouse. Avant de configurer l'exportation des métriques, vous devez configurer Prometheus en suivant leur [guide](https://prometheus.io/docs/prometheus/latest/installation/). -Pour surveiller les serveurs dans une configuration de cluster, vous devez [max\_replica\_delay\_for\_distributed\_queries](settings/settings.md#settings-max_replica_delay_for_distributed_queries) paramètre et utiliser la ressource HTTP `/replicas_status`. Une demande de `/replicas_status` retourner `200 OK` si la réplique est disponible et n’est pas retardé derrière les autres réplicas. Si une réplique est retardée, elle revient `503 HTTP_SERVICE_UNAVAILABLE` avec des informations sur l’écart. +De plus, vous pouvez surveiller la disponibilité du serveur via L'API HTTP. Envoyer la `HTTP GET` demande à `/ping`. Si le serveur est disponible, il répond avec `200 OK`. + +Pour surveiller les serveurs dans une configuration de cluster, vous devez [max\_replica\_delay\_for\_distributed\_queries](settings/settings.md#settings-max_replica_delay_for_distributed_queries) paramètre et utiliser la ressource HTTP `/replicas_status`. Une demande de `/replicas_status` retourner `200 OK` si la réplique est disponible et n'est pas retardé derrière les autres réplicas. Si une réplique est retardée, elle revient `503 HTTP_SERVICE_UNAVAILABLE` avec des informations sur l'écart. diff --git a/docs/fr/operations/optimizing-performance/index.md b/docs/fr/operations/optimizing-performance/index.md index a596d617d8f..3877d558eca 100644 --- a/docs/fr/operations/optimizing-performance/index.md +++ b/docs/fr/operations/optimizing-performance/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 -toc_folder_title: Optimizing Performance +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: Optimisation Des Performances toc_priority: 52 --- diff --git a/docs/fr/operations/optimizing-performance/sampling-query-profiler.md b/docs/fr/operations/optimizing-performance/sampling-query-profiler.md index bebb4a1087f..ba8f3ecd110 100644 --- a/docs/fr/operations/optimizing-performance/sampling-query-profiler.md +++ b/docs/fr/operations/optimizing-performance/sampling-query-profiler.md @@ -1,35 +1,35 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 54 toc_title: "Profilage De Requ\xEAte" --- # Échantillonnage Du Profileur De Requête {#sampling-query-profiler} -ClickHouse exécute un profileur d’échantillonnage qui permet d’analyser l’exécution des requêtes. En utilisant profiler, vous pouvez trouver des routines de code source qui ont utilisé le plus fréquemment lors de l’exécution de la requête. Vous pouvez suivre le temps CPU et le temps d’horloge murale passé, y compris le temps d’inactivité. +ClickHouse exécute un profileur d'échantillonnage qui permet d'analyser l'exécution des requêtes. En utilisant profiler, vous pouvez trouver des routines de code source qui ont utilisé le plus fréquemment lors de l'exécution de la requête. Vous pouvez suivre le temps CPU et le temps d'horloge murale passé, y compris le temps d'inactivité. Utilisation du générateur de profils: - Installation de la [trace\_log](../server-configuration-parameters/settings.md#server_configuration_parameters-trace_log) la section de la configuration du serveur. - Cette section configure le [trace\_log](../../operations/system-tables.md#system_tables-trace_log) tableau système contenant les résultats du fonctionnement du profileur. Il est configuré par défaut. Rappelez-vous que les données de ce tableau est valable que pour un serveur en cours d’exécution. Après le redémarrage du serveur, ClickHouse ne nettoie pas la table et toute l’adresse de mémoire virtuelle stockée peut devenir invalide. + Cette section configure le [trace\_log](../../operations/system-tables.md#system_tables-trace_log) tableau système contenant les résultats du fonctionnement du profileur. Il est configuré par défaut. Rappelez-vous que les données de ce tableau est valable que pour un serveur en cours d'exécution. Après le redémarrage du serveur, ClickHouse ne nettoie pas la table et toute l'adresse de mémoire virtuelle stockée peut devenir invalide. - Installation de la [query\_profiler\_cpu\_time\_period\_ns](../settings/settings.md#query_profiler_cpu_time_period_ns) ou [query\_profiler\_real\_time\_period\_ns](../settings/settings.md#query_profiler_real_time_period_ns) paramètre. Les deux paramètres peuvent être utilisés simultanément. - Ces paramètres vous permettent de configurer les minuteries du profileur. Comme il s’agit des paramètres de session, vous pouvez obtenir une fréquence d’échantillonnage différente pour l’ensemble du serveur, les utilisateurs individuels ou les profils d’utilisateurs, pour votre session interactive et pour chaque requête individuelle. + Ces paramètres vous permettent de configurer les minuteries du profileur. Comme il s'agit des paramètres de session, vous pouvez obtenir une fréquence d'échantillonnage différente pour l'ensemble du serveur, les utilisateurs individuels ou les profils d'utilisateurs, pour votre session interactive et pour chaque requête individuelle. -La fréquence d’échantillonnage par défaut est d’un échantillon par seconde et le processeur et les minuteries réelles sont activés. Cette fréquence permet de collecter suffisamment d’informations sur le cluster ClickHouse. En même temps, en travaillant avec cette fréquence, profiler n’affecte pas les performances du serveur ClickHouse. Si vous avez besoin de profiler chaque requête individuelle, essayez d’utiliser une fréquence d’échantillonnage plus élevée. +La fréquence d'échantillonnage par défaut est d'un échantillon par seconde et le processeur et les minuteries réelles sont activés. Cette fréquence permet de collecter suffisamment d'informations sur le cluster ClickHouse. En même temps, en travaillant avec cette fréquence, profiler n'affecte pas les performances du serveur ClickHouse. Si vous avez besoin de profiler chaque requête individuelle, essayez d'utiliser une fréquence d'échantillonnage plus élevée. Pour analyser les `trace_log` système de table: - Installer le `clickhouse-common-static-dbg` paquet. Voir [Installer à partir de paquets DEB](../../getting-started/install.md#install-from-deb-packages). -- Autoriser les fonctions d’introspection par [allow\_introspection\_functions](../settings/settings.md#settings-allow_introspection_functions) paramètre. +- Autoriser les fonctions d'introspection par [allow\_introspection\_functions](../settings/settings.md#settings-allow_introspection_functions) paramètre. - Pour des raisons de sécurité, les fonctions d’introspection sont désactivées par défaut. + Pour des raisons de sécurité, les fonctions d'introspection sont désactivées par défaut. -- L’utilisation de la `addressToLine`, `addressToSymbol` et `demangle` [fonctions d’introspection](../../sql-reference/functions/introspection.md) pour obtenir les noms de fonctions et leurs positions dans le code ClickHouse. Pour obtenir un profil pour une requête, vous devez agréger les données du `trace_log` table. Vous pouvez agréger des données par des fonctions individuelles ou par l’ensemble des traces de la pile. +- L'utilisation de la `addressToLine`, `addressToSymbol` et `demangle` [fonctions d'introspection](../../sql-reference/functions/introspection.md) pour obtenir les noms de fonctions et leurs positions dans le code ClickHouse. Pour obtenir un profil pour une requête, vous devez agréger les données du `trace_log` table. Vous pouvez agréger des données par des fonctions individuelles ou par l'ensemble des traces de la pile. Si vous avez besoin de visualiser `trace_log` info, essayez [flamegraph](../../interfaces/third-party/gui/#clickhouse-flamegraph) et [speedscope](https://github.com/laplab/clickhouse-speedscope). @@ -41,7 +41,7 @@ Dans cet exemple, nous: - Agrégation par trace de pile. -- En utilisant les fonctions d’introspection, nous obtiendrons un rapport de: +- En utilisant les fonctions d'introspection, nous obtiendrons un rapport de: - Noms des symboles et des fonctions de code source correspondantes. - Emplacements de code Source de ces fonctions. diff --git a/docs/fr/operations/performance-test.md b/docs/fr/operations/performance-test.md index cd4732e8f96..6093772aefe 100644 --- a/docs/fr/operations/performance-test.md +++ b/docs/fr/operations/performance-test.md @@ -1,17 +1,17 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 54 toc_title: "Tester Le Mat\xE9riel" --- -# Comment Tester Votre Matériel Avec ClickHouse {#how-to-test-your-hardware-with-clickhouse} +# Comment tester votre matériel avec ClickHouse {#how-to-test-your-hardware-with-clickhouse} -Avec cette instruction, vous pouvez exécuter le test de performance clickhouse de base sur n’importe quel serveur sans installation de paquets ClickHouse. +Avec cette instruction, vous pouvez exécuter le test de performance clickhouse de base sur n'importe quel serveur sans installation de paquets ClickHouse. 1. Aller à “commits” page: https://github.com/ClickHouse/ClickHouse/commits/master -2. Cliquez sur la première coche verte ou croix rouge avec vert “ClickHouse Build Check” et cliquez sur le “Details” lien de proximité “ClickHouse Build Check”. +2. Cliquez sur la première coche verte ou croix rouge avec vert “ClickHouse Build Check” et cliquez sur le “Details” lien de proximité “ClickHouse Build Check”. Il n'y a pas un tel lien dans certains commits, par exemple des commits avec de la documentation. Dans ce cas, choisissez le commit le plus proche ayant ce lien. 3. Copiez le lien à “clickhouse” binaire pour amd64 ou aarch64. @@ -65,7 +65,7 @@ Avec cette instruction, vous pouvez exécuter le test de performance clickhouse ./clickhouse client --query "SELECT count() FROM hits_100m_obfuscated" 100000000 -1. Modifier le benchmark-new.sh, changement “clickhouse-client” de “./clickhouse client” et d’ajouter “–max\_memory\_usage 100000000000” paramètre. +1. Modifier le benchmark-new.sh, changement `clickhouse-client` de `./clickhouse client` et d'ajouter `–-max_memory_usage 100000000000` paramètre. @@ -79,4 +79,4 @@ Avec cette instruction, vous pouvez exécuter le test de performance clickhouse 1. Envoyez les numéros et les informations sur votre configuration matérielle à clickhouse-feedback@yandex-team.com -Tous les résultats sont publiés ici: https://clickhouse.tech / benchmark\_hardware.HTML +Tous les résultats sont publiés ici: https://clickhouse.tech/de référence/de matériel/ diff --git a/docs/fr/operations/quotas.md b/docs/fr/operations/quotas.md index 95da8e13fc2..ae15c774eaf 100644 --- a/docs/fr/operations/quotas.md +++ b/docs/fr/operations/quotas.md @@ -1,16 +1,16 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 51 toc_title: Quota --- # Quota {#quotas} -Les Quotas permettent de limiter l’utilisation des ressources au cours d’une période de temps, ou tout simplement suivre l’utilisation des ressources. -Les Quotas sont configurés dans la configuration utilisateur. Ce n’est généralement ‘users.xml’. +Les Quotas permettent de limiter l'utilisation des ressources au cours d'une période de temps ou de suivre l'utilisation des ressources. +Les Quotas sont configurés dans la configuration utilisateur, qui est généralement ‘users.xml’. -Le système dispose également d’une fonctionnalité pour limiter la complexité d’une seule requête. Voir la section “Restrictions on query complexity”). +Le système dispose également d'une fonctionnalité pour limiter la complexité d'une seule requête. Voir la section “Restrictions on query complexity”). Contrairement aux restrictions de complexité des requêtes, les quotas: @@ -39,7 +39,7 @@ Regardons la section de la ‘users.xml’ fichier qui définit les quotas. ``` -Par défaut, le quota suit simplement la consommation de ressources pour chaque heure, sans limiter l’utilisation. +Par défaut, le quota suit la consommation de ressources pour chaque heure, sans limiter l'utilisation. La consommation de ressources calculé pour chaque intervalle est sortie dans le journal du serveur après chaque demande. ``` xml @@ -68,9 +68,9 @@ La consommation de ressources calculé pour chaque intervalle est sortie dans le ``` -Pour l’ ‘statbox’ quota, restrictions sont fixées pour toutes les heures et pour toutes les 24 heures (86 400 secondes). L’intervalle de temps est compté à partir d’un moment fixe défini par l’implémentation. En d’autres termes, l’intervalle de 24 heures ne commence pas nécessairement à minuit. +Pour l' ‘statbox’ quota, restrictions sont fixées pour toutes les heures et pour toutes les 24 heures (86 400 secondes). L'intervalle de temps est compté, à partir d'un moment fixe défini par l'implémentation. En d'autres termes, l'intervalle de 24 heures ne commence pas nécessairement à minuit. -Lorsque l’intervalle se termine, toutes les valeurs collectées sont effacées. Pour l’heure suivante, le calcul du quota recommence. +Lorsque l'intervalle se termine, toutes les valeurs collectées sont effacées. Pour l'heure suivante, le calcul du quota recommence. Voici les montants qui peuvent être restreint: @@ -78,15 +78,15 @@ Voici les montants qui peuvent être restreint: `errors` – The number of queries that threw an exception. -`result_rows` – The total number of rows given as the result. +`result_rows` – The total number of rows given as a result. -`read_rows` – The total number of source rows read from tables for running the query, on all remote servers. +`read_rows` – The total number of source rows read from tables for running the query on all remote servers. `execution_time` – The total query execution time, in seconds (wall time). Si la limite est dépassée pendant au moins un intervalle de temps, une exception est levée avec un texte indiquant quelle restriction a été dépassée, pour quel intervalle et quand le nouvel intervalle commence (lorsque les requêtes peuvent être envoyées à nouveau). -Les Quotas peuvent utiliser le “quota key” fonctionnalité afin de rendre compte des ressources pour plusieurs clés indépendamment. Voici un exemple de ce: +Les Quotas peuvent utiliser le “quota key” fonctionnalité de rapport sur les ressources pour plusieurs clés indépendamment. Voici un exemple de ce: ``` xml @@ -97,7 +97,7 @@ Les Quotas peuvent utiliser le “quota key” fonctionnalité afin de rendre co so the quota will be counted separately for each username. Using keys makes sense only if quota_key is transmitted by the program, not by a user. - You can also write so the IP address is used as the quota key. + You can also write , so the IP address is used as the quota key. (But keep in mind that users can change the IPv6 address fairly easily.) --> @@ -105,7 +105,7 @@ Les Quotas peuvent utiliser le “quota key” fonctionnalité afin de rendre co Le quota est attribué aux utilisateurs dans le ‘users’ section de la configuration. Voir la section “Access rights”. -Pour le traitement des requêtes distribuées, les montants accumulés sont stockés sur le serveur demandeur. Donc, si l’utilisateur se rend sur un autre serveur, le quota y sera “start over”. +Pour le traitement des requêtes distribuées, les montants accumulés sont stockés sur le serveur demandeur. Donc, si l'utilisateur se rend sur un autre serveur, le quota y sera “start over”. Lorsque le serveur est redémarré, les quotas sont réinitialisés. diff --git a/docs/fr/operations/requirements.md b/docs/fr/operations/requirements.md index 88b06e9c44e..936d6f57f5b 100644 --- a/docs/fr/operations/requirements.md +++ b/docs/fr/operations/requirements.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 44 toc_title: Exigence --- @@ -9,38 +9,38 @@ toc_title: Exigence ## CPU {#cpu} -Pour l’installation à partir de paquets deb prédéfinis, utilisez un processeur avec l’architecture x86\_64 et la prise en charge des instructions SSE 4.2. Pour exécuter ClickHouse avec des processeurs qui ne prennent pas en charge SSE 4.2 ou qui ont une architecture AArch64 ou PowerPC64LE, vous devez créer ClickHouse à partir de sources. +Pour l'installation à partir de paquets deb prédéfinis, utilisez un processeur avec l'architecture x86\_64 et la prise en charge des instructions SSE 4.2. Pour exécuter ClickHouse avec des processeurs qui ne prennent pas en charge SSE 4.2 ou qui ont une architecture AArch64 ou PowerPC64LE, vous devez créer ClickHouse à partir de sources. -ClickHouse implémente le traitement parallèle des données et utilise toutes les ressources matérielles disponibles. Lors du choix d’un processeur, tenez compte du fait que ClickHouse fonctionne plus efficacement dans les configurations avec un grand nombre de cœurs mais une fréquence d’horloge plus faible que dans les configurations avec moins de cœurs et une fréquence d’horloge plus élevée. Par exemple, 16 cœurs avec 2600 MHz est préférable à 8 cœurs avec 3600 MHz. +ClickHouse implémente le traitement parallèle des données et utilise toutes les ressources matérielles disponibles. Lors du choix d'un processeur, tenez compte du fait que ClickHouse fonctionne plus efficacement dans les configurations avec un grand nombre de cœurs mais une fréquence d'horloge plus faible que dans les configurations avec moins de cœurs et une fréquence d'horloge plus élevée. Par exemple, 16 cœurs avec 2600 MHz est préférable à 8 cœurs avec 3600 MHz. -L’utilisation de **Turbo Boost** et **la technologie hyper-threading** technologies est recommandé. Il améliore sensiblement les performances avec une charge typique. +Il est recommandé d'utiliser **Turbo Boost** et **la technologie hyper-threading** technologie. Il améliore considérablement les performances avec une charge de travail typique. ## RAM {#ram} -Nous vous recommandons d’utiliser un minimum de 4 Go de RAM afin d’effectuer des requêtes non triviales. Le serveur ClickHouse peut fonctionner avec une quantité beaucoup plus petite de RAM, mais il nécessite de la mémoire pour traiter les requêtes. +Nous vous recommandons d'utiliser un minimum de 4 Go de RAM pour effectuer des requêtes non triviales. Le serveur ClickHouse peut fonctionner avec une quantité beaucoup plus petite de RAM, mais il nécessite de la mémoire pour traiter les requêtes. Le volume de RAM requis dépend de: - La complexité des requêtes. - La quantité de données traitées dans les requêtes. -Pour calculer le volume de RAM requis, vous devez estimer la taille des données temporaires pour [GROUP BY](../sql-reference/statements/select.md#select-group-by-clause), [DISTINCT](../sql-reference/statements/select.md#select-distinct), [JOIN](../sql-reference/statements/select.md#select-join) et d’autres opérations que vous utilisez. +Pour calculer le volume de RAM requis, vous devez estimer la taille des données temporaires pour [GROUP BY](../sql-reference/statements/select/group-by.md#select-group-by-clause), [DISTINCT](../sql-reference/statements/select/distinct.md#select-distinct), [JOIN](../sql-reference/statements/select/join.md#select-join) et d'autres opérations que vous utilisez. -ClickHouse peut utiliser la mémoire externe pour les données temporaires. Voir [Groupe par dans la mémoire externe](../sql-reference/statements/select.md#select-group-by-in-external-memory) pour plus de détails. +ClickHouse peut utiliser la mémoire externe pour les données temporaires. Voir [Groupe par dans la mémoire externe](../sql-reference/statements/select/group-by.md#select-group-by-in-external-memory) pour plus de détails. -## Fichier D’Échange {#swap-file} +## Fichier D'Échange {#swap-file} -Désactiver le fichier d’échange pour les environnements de production. +Désactiver le fichier d'échange pour les environnements de production. ## Sous-Système De Stockage {#storage-subsystem} -Vous devez avoir 2 Go d’espace disque libre pour installer ClickHouse. +Vous devez avoir 2 Go d'espace disque libre pour installer ClickHouse. -Le volume de stockage requis pour vos données doit être calculé séparément. L’évaluation devrait inclure: +Le volume de stockage requis pour vos données doit être calculé séparément. L'évaluation devrait inclure: - Estimation du volume de données. - Vous pouvez prendre un échantillon des données et obtenir la taille moyenne d’une ligne. Ensuite, multipliez la valeur par le nombre de lignes que vous souhaitez stocker. + Vous pouvez prendre un échantillon des données et obtenir la taille moyenne d'une ligne. Ensuite, multipliez la valeur par le nombre de lignes que vous souhaitez stocker. - Le coefficient de compression des données. @@ -56,6 +56,6 @@ La bande passante du réseau est essentielle pour traiter les requêtes distribu ## Logiciel {#software} -ClickHouse est développé pour la famille de systèmes D’exploitation Linux. La distribution Linux recommandée est Ubuntu. Le `tzdata` paquet doit être installé dans le système. +ClickHouse est développé principalement pour la famille de systèmes D'exploitation Linux. La distribution Linux recommandée est Ubuntu. Le `tzdata` paquet doit être installé dans le système. -ClickHouse peut également fonctionner dans d’autres familles de systèmes d’exploitation. Voir les détails dans le [Prise en main](../getting-started/index.md) section de la documentation. +ClickHouse peut également fonctionner dans d'autres familles de systèmes d'exploitation. Voir les détails dans le [Prise en main](../getting-started/index.md) section de la documentation. diff --git a/docs/fr/operations/server-configuration-parameters/index.md b/docs/fr/operations/server-configuration-parameters/index.md index d72d3762209..0ecfb4a44cc 100644 --- a/docs/fr/operations/server-configuration-parameters/index.md +++ b/docs/fr/operations/server-configuration-parameters/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 -toc_folder_title: Server Configuration Parameters +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "Param\xE8tres De Configuration Du Serveur" toc_priority: 54 toc_title: Introduction --- @@ -12,8 +12,8 @@ Cette section contient des descriptions des paramètres du serveur qui ne peuven Ces paramètres sont stockés dans la `config.xml` fichier sur le serveur ClickHouse. -D’autres paramètres sont décrits dans le “[Paramètre](../settings/index.md#settings)” section. +D'autres paramètres sont décrits dans le “[Paramètre](../settings/index.md#session-settings-intro)” section. -Avant d’étudier les paramètres, lire la [Fichiers de Configuration](../configuration-files.md#configuration_files) section et notez l’utilisation de substitutions (le `incl` et `optional` attribut). +Avant d'étudier les paramètres, lire la [Fichiers de Configuration](../configuration-files.md#configuration_files) section et notez l'utilisation de substitutions (le `incl` et `optional` attribut). [Article Original](https://clickhouse.tech/docs/en/operations/server_configuration_parameters/) diff --git a/docs/fr/operations/server-configuration-parameters/settings.md b/docs/fr/operations/server-configuration-parameters/settings.md index d54b59a253a..741bec44421 100644 --- a/docs/fr/operations/server-configuration-parameters/settings.md +++ b/docs/fr/operations/server-configuration-parameters/settings.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 57 toc_title: "Les Param\xE8tres Du Serveur" --- @@ -9,9 +9,9 @@ toc_title: "Les Param\xE8tres Du Serveur" ## builtin\_dictionaries\_reload\_interval {#builtin-dictionaries-reload-interval} -L’intervalle en secondes avant de recharger les dictionnaires intégrés. +L'intervalle en secondes avant de recharger les dictionnaires intégrés. -Clickhouse recharge les dictionnaires intégrés toutes les X secondes. Cela permet d’éditer des dictionnaires “on the fly” sans redémarrer le serveur. +Clickhouse recharge les dictionnaires intégrés toutes les X secondes. Cela permet d'éditer des dictionnaires “on the fly” sans redémarrer le serveur. Valeur par défaut: 3600. @@ -26,7 +26,7 @@ Valeur par défaut: 3600. Paramètres de compression de données pour [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md)-tables de moteur. !!! warning "Avertissement" - Ne l’utilisez pas si vous venez de commencer à utiliser ClickHouse. + Ne l'utilisez pas si vous venez de commencer à utiliser ClickHouse. Modèle de Configuration: @@ -54,7 +54,7 @@ Actions lorsque les conditions sont remplies: - Si une partie de données correspond à un ensemble de conditions, ClickHouse utilise la méthode de compression spécifiée. - Si une partie de données correspond à plusieurs ensembles de conditions, ClickHouse utilise le premier ensemble de conditions correspondant. -Si aucune condition n’est remplie pour une partie de données, ClickHouse utilise `lz4` compression. +Si aucune condition n'est remplie pour une partie de données, ClickHouse utilise `lz4` compression. **Exemple** @@ -94,7 +94,7 @@ Les paramètres des profils sont situés dans le fichier spécifié dans le para ## dictionaries\_config {#server_configuration_parameters-dictionaries_config} -Chemin d’accès au fichier de configuration des dictionnaires externes. +Chemin d'accès au fichier de configuration des dictionnaires externes. Chemin: @@ -115,7 +115,7 @@ Chargement paresseux des dictionnaires. Si `true` chaque dictionnaire est créé lors de la première utilisation. Si la création du dictionnaire a échoué, la fonction qui utilisait le dictionnaire lève une exception. -Si `false`, tous les dictionnaires sont créés lorsque le serveur démarre, et si il y a une erreur, le serveur s’arrête. +Si `false`, tous les dictionnaires sont créés lorsque le serveur démarre, et si il y a une erreur, le serveur s'arrête. La valeur par défaut est `true`. @@ -127,7 +127,7 @@ La valeur par défaut est `true`. ## format\_schema\_path {#server_configuration_parameters-format_schema_path} -Le chemin d’accès au répertoire avec des régimes pour l’entrée de données, tels que les schémas pour l’ [CapnProto](../../interfaces/formats.md#capnproto) format. +Le chemin d'accès au répertoire avec des régimes pour l'entrée de données, tels que les schémas pour l' [CapnProto](../../interfaces/formats.md#capnproto) format. **Exemple** @@ -152,7 +152,7 @@ Paramètre: - events\_cumulative – Sending cumulative data from the [système.événement](../../operations/system-tables.md#system_tables-events) table. - asynchronous\_metrics – Sending data from the [système.asynchronous\_metrics](../../operations/system-tables.md#system_tables-asynchronous_metrics) table. -Vous pouvez configurer plusieurs `` clause. Par exemple, vous pouvez l’utiliser pour envoyer des données différentes à différents intervalles. +Vous pouvez configurer plusieurs `` clause. Par exemple, vous pouvez l'utiliser pour envoyer des données différentes à différents intervalles. **Exemple** @@ -172,7 +172,7 @@ Vous pouvez configurer plusieurs `` clause. Par exemple, vous pouvez l ## graphite\_rollup {#server_configuration_parameters-graphite-rollup} -Paramètres pour l’amincissement des données pour le Graphite. +Paramètres pour l'amincissement des données pour le Graphite. Pour plus de détails, voir [GraphiteMergeTree](../../engines/table-engines/mergetree-family/graphitemergetree.md). @@ -219,7 +219,7 @@ La valeur par défaut est “Ok.” (avec un saut de ligne à la fin) **Exemple** -Ouvrir `https://tabix.io/` lors de l’accès à `http://localhost: http_port`. +Ouvrir `https://tabix.io/` lors de l'accès à `http://localhost: http_port`. ``` xml @@ -229,9 +229,9 @@ Ouvrir `https://tabix.io/` lors de l’accès à `http://localhost: http_port`. ## include\_from {#server_configuration_parameters-include_from} -Le chemin d’accès au fichier avec des substitutions. +Le chemin d'accès au fichier avec des substitutions. -Pour plus d’informations, consultez la section “[Fichiers de Configuration](../configuration-files.md#configuration_files)”. +Pour plus d'informations, consultez la section “[Fichiers de Configuration](../configuration-files.md#configuration_files)”. **Exemple** @@ -241,7 +241,7 @@ Pour plus d’informations, consultez la section “[Fichiers de Configuration]( ## interserver\_http\_port {#interserver-http-port} -Port pour l’échange de données entre les serveurs ClickHouse. +Port pour l'échange de données entre les serveurs ClickHouse. **Exemple** @@ -251,7 +251,7 @@ Port pour l’échange de données entre les serveurs ClickHouse. ## interserver\_http\_host {#interserver-http-host} -Le nom d’hôte qui peut être utilisé par d’autres serveurs pour accéder à ce serveur. +Le nom d'hôte qui peut être utilisé par d'autres serveurs pour accéder à ce serveur. Si elle est omise, elle est définie de la même manière que `hostname-f` commande. @@ -265,8 +265,8 @@ Utile pour rompre avec une interface réseau spécifique. ## interserver\_http\_credentials {#server-settings-interserver-http-credentials} -Le nom d’utilisateur et le mot de passe utilisés pour [réplication](../../engines/table-engines/mergetree-family/replication.md) avec les moteurs \* répliqués. Ces informations d’identification sont utilisées uniquement pour la communication entre les répliques et ne sont pas liées aux informations d’identification des clients ClickHouse. Le serveur vérifie ces informations d’identification pour la connexion de répliques et utilise les mêmes informations d’identification lors de la connexion à d’autres répliques. Donc, ces informations d’identification doivent être identiques pour tous les réplicas dans un cluster. -Par défaut, l’authentification n’est pas utilisé. +Le nom d'utilisateur et le mot de passe utilisés pour [réplication](../../engines/table-engines/mergetree-family/replication.md) avec les moteurs \* répliqués. Ces informations d'identification sont utilisées uniquement pour la communication entre les répliques et ne sont pas liées aux informations d'identification des clients ClickHouse. Le serveur vérifie ces informations d'identification pour la connexion de répliques et utilise les mêmes informations d'identification lors de la connexion à d'autres répliques. Donc, ces informations d'identification doivent être identiques pour tous les réplicas dans un cluster. +Par défaut, l'authentification n'est pas utilisé. Cette section contient les paramètres suivants: @@ -327,7 +327,7 @@ Touches: ``` -L’écriture dans le syslog est également prise en charge. Exemple de Config: +L'écriture dans le syslog est également prise en charge. Exemple de Config: ``` xml @@ -356,7 +356,7 @@ Substitutions de paramètres pour les tables répliquées. Peut être omis si les tables répliquées ne sont pas utilisées. -Pour plus d’informations, consultez la section “[Création de tables répliquées](../../engines/table-engines/mergetree-family/replication.md)”. +Pour plus d'informations, consultez la section “[Création de tables répliquées](../../engines/table-engines/mergetree-family/replication.md)”. **Exemple** @@ -368,7 +368,7 @@ Pour plus d’informations, consultez la section “[Création de tables répliq Taille approximative (en octets) du cache des marques utilisées par les [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) famille. -Le cache est partagé pour le serveur et la mémoire est allouée au besoin. La taille du cache doit être d’au moins 5368709120. +Le cache est partagé pour le serveur et la mémoire est allouée au besoin. La taille du cache doit être d'au moins 5368709120. **Exemple** @@ -402,7 +402,7 @@ Le nombre maximal de fichiers ouverts. Par défaut: `maximum`. -Nous vous recommandons d’utiliser cette option sous Mac OS X depuis le `getrlimit()` la fonction renvoie une valeur incorrecte. +Nous vous recommandons d'utiliser cette option sous Mac OS X depuis le `getrlimit()` la fonction renvoie une valeur incorrecte. **Exemple** @@ -414,7 +414,7 @@ Nous vous recommandons d’utiliser cette option sous Mac OS X depuis le `getrli Restriction sur la suppression de tables. -Si la taille d’un [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) table dépasse `max_table_size_to_drop` (en octets), vous ne pouvez pas le supprimer à l’aide d’une requête DROP. +Si la taille d'un [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) table dépasse `max_table_size_to_drop` (en octets), vous ne pouvez pas le supprimer à l'aide d'une requête DROP. Si vous devez toujours supprimer la table sans redémarrer le serveur ClickHouse, créez le `/flags/force_drop_table` fichier et exécutez la requête DROP. @@ -432,7 +432,7 @@ La valeur 0 signifie que vous pouvez supprimer toutes les tables sans aucune res Réglage fin des tables dans le [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md). -Pour plus d’informations, consultez MergeTreeSettings.h fichier d’en-tête. +Pour plus d'informations, consultez MergeTreeSettings.h fichier d'en-tête. **Exemple** @@ -446,26 +446,26 @@ Pour plus d’informations, consultez MergeTreeSettings.h fichier d’en-tête. Configuration client/serveur SSL. -Le Support pour SSL est fourni par le `libpoco` bibliothèque. L’interface est décrite dans le fichier [SSLManager.h](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h) +Le Support pour SSL est fourni par le `libpoco` bibliothèque. L'interface est décrite dans le fichier [SSLManager.h](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h) Clés pour les paramètres Serveur/client: - privateKeyFile – The path to the file with the secret key of the PEM certificate. The file may contain a key and certificate at the same time. - certificateFile – The path to the client/server certificate file in PEM format. You can omit it if `privateKeyFile` contient le certificat. - caConfig – The path to the file or directory that contains trusted root certificates. -- verificationMode – The method for checking the node’s certificates. Details are in the description of the [Cadre](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/Context.h) classe. Valeurs possibles: `none`, `relaxed`, `strict`, `once`. +- verificationMode – The method for checking the node's certificates. Details are in the description of the [Cadre](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/Context.h) classe. Valeurs possibles: `none`, `relaxed`, `strict`, `once`. - verificationDepth – The maximum length of the verification chain. Verification will fail if the certificate chain length exceeds the set value. - loadDefaultCAFile – Indicates that built-in CA certificates for OpenSSL will be used. Acceptable values: `true`, `false`. \| - cipherList – Supported OpenSSL encryptions. For example: `ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH`. - cacheSessions – Enables or disables caching sessions. Must be used in combination with `sessionIdContext`. Les valeurs acceptables: `true`, `false`. -- sessionIdContext – A unique set of random characters that the server appends to each generated identifier. The length of the string must not exceed `SSL_MAX_SSL_SESSION_ID_LENGTH`. Ce paramètre est toujours recommandé car il permet d’éviter les problèmes à la fois si le serveur met en cache la session et si le client demande la mise en cache. Valeur par défaut: `${application.name}`. +- sessionIdContext – A unique set of random characters that the server appends to each generated identifier. The length of the string must not exceed `SSL_MAX_SSL_SESSION_ID_LENGTH`. Ce paramètre est toujours recommandé car il permet d'éviter les problèmes à la fois si le serveur met en cache la session et si le client demande la mise en cache. Valeur par défaut: `${application.name}`. - sessionCacheSize – The maximum number of sessions that the server caches. Default value: 1024\*20. 0 – Unlimited sessions. - sessionTimeout – Time for caching the session on the server. - extendedVerification – Automatically extended verification of certificates after the session ends. Acceptable values: `true`, `false`. - requireTLSv1 – Require a TLSv1 connection. Acceptable values: `true`, `false`. - requireTLSv1\_1 – Require a TLSv1.1 connection. Acceptable values: `true`, `false`. - requireTLSv1 – Require a TLSv1.2 connection. Acceptable values: `true`, `false`. -- fips – Activates OpenSSL FIPS mode. Supported if the library’s OpenSSL version supports FIPS. +- fips – Activates OpenSSL FIPS mode. Supported if the library's OpenSSL version supports FIPS. - privateKeyPassphraseHandler – Class (PrivateKeyPassphraseHandler subclass) that requests the passphrase for accessing the private key. For example: ``, `KeyFileHandler`, `test`, ``. - invalidCertificateHandler – Class (a subclass of CertificateHandler) for verifying invalid certificates. For example: ` ConsoleCertificateHandler ` . - disableProtocols – Protocols that are not allowed to use. @@ -527,7 +527,7 @@ Utilisez les paramètres suivants pour configurer la journalisation: ## chemin {#server_configuration_parameters-path} -Chemin d’accès au répertoire contenant des données. +Chemin d'accès au répertoire contenant des données. !!! note "Note" La barre oblique de fin est obligatoire. @@ -538,6 +538,30 @@ Chemin d’accès au répertoire contenant des données. /var/lib/clickhouse/ ``` +## prometheus {#server_configuration_parameters-prometheus} + +Exposer les données de métriques pour le raclage à partir [Prometheus](https://prometheus.io). + +Paramètre: + +- `endpoint` – HTTP endpoint for scraping metrics by prometheus server. Start from ‘/’. +- `port` – Port for `endpoint`. +- `metrics` – Flag that sets to expose metrics from the [système.métrique](../system-tables.md#system_tables-metrics) table. +- `events` – Flag that sets to expose metrics from the [système.événement](../system-tables.md#system_tables-events) table. +- `asynchronous_metrics` – Flag that sets to expose current metrics values from the [système.asynchronous\_metrics](../system-tables.md#system_tables-asynchronous_metrics) table. + +**Exemple** + +``` xml + + /metrics + 8001 + true + true + true + +``` + ## query\_log {#server_configuration_parameters-query-log} Réglage de la journalisation des requêtes reçues avec [log\_queries=1](../settings/settings.md) paramètre. @@ -551,7 +575,7 @@ Utilisez les paramètres suivants pour configurer la journalisation: - `partition_by` – Sets a [partitionnement personnalisé clé](../../engines/table-engines/mergetree-family/custom-partitioning-key.md) pour une table. - `flush_interval_milliseconds` – Interval for flushing data from the buffer in memory to the table. -Si la table n’existe pas, ClickHouse la créera. Si la structure du journal des requêtes a été modifiée lors de la mise à jour du serveur ClickHouse, la table avec l’ancienne structure est renommée et une nouvelle table est créée automatiquement. +Si la table n'existe pas, ClickHouse la créera. Si la structure du journal des requêtes a été modifiée lors de la mise à jour du serveur ClickHouse, la table avec l'ancienne structure est renommée et une nouvelle table est créée automatiquement. **Exemple** @@ -577,7 +601,7 @@ Utilisez les paramètres suivants pour configurer la journalisation: - `partition_by` – Sets a [partitionnement personnalisé clé](../../engines/table-engines/mergetree-family/custom-partitioning-key.md) pour un système de tableau. - `flush_interval_milliseconds` – Interval for flushing data from the buffer in memory to the table. -Si la table n’existe pas, ClickHouse la créera. Si la structure du journal des threads de requête a été modifiée lors de la mise à jour du serveur ClickHouse, la table avec l’ancienne structure est renommée et une nouvelle table est créée automatiquement. +Si la table n'existe pas, ClickHouse la créera. Si la structure du journal des threads de requête a été modifiée lors de la mise à jour du serveur ClickHouse, la table avec l'ancienne structure est renommée et une nouvelle table est créée automatiquement. **Exemple** @@ -614,7 +638,7 @@ Le fichier de configuration du serveur par défaut `config.xml` contient la sect ## query\_masking\_rules {#query-masking-rules} -Règles basées sur Regexp, qui seront appliquées aux requêtes ainsi qu’à tous les messages de journal avant de les stocker dans les journaux du serveur, +Règles basées sur Regexp, qui seront appliquées aux requêtes ainsi qu'à tous les messages de journal avant de les stocker dans les journaux du serveur, `system.query_log`, `system.text_log`, `system.processes` table, et dans les journaux envoyés au client. Qui permet à la prévention de fuite de données sensibles à partir de requêtes SQL (comme les noms, e-mails, identificateurs ou numéros de carte de crédit) aux journaux. @@ -636,11 +660,11 @@ Config champs: - `regexp` - Expression régulière compatible RE2 (obligatoire) - `replace` - chaîne de substitution pour les données sensibles (facultatif, par défaut - six astérisques) -Les règles de masquage sont appliquées à l’ensemble de la requête (pour éviter les fuites de données sensibles provenant de requêtes malformées / Non analysables). +Les règles de masquage sont appliquées à l'ensemble de la requête (pour éviter les fuites de données sensibles provenant de requêtes malformées / Non analysables). `system.events` table ont compteur `QueryMaskingRulesMatch` qui ont un nombre global de requête de masquage des règles de correspondances. -Pour les requêtes distribuées chaque serveur doivent être configurés séparément, sinon, les sous-requêtes transmises à d’autres +Pour les requêtes distribuées chaque serveur doivent être configurés séparément, sinon, les sous-requêtes transmises à d'autres les nœuds seront stockés sans masquage. ## remote\_servers {#server-settings-remote-servers} @@ -653,7 +677,7 @@ Configuration des clusters utilisés par le [Distribué](../../engines/table-eng ``` -Pour la valeur de l’ `incl` attribut, voir la section “[Fichiers de Configuration](../configuration-files.md#configuration_files)”. +Pour la valeur de l' `incl` attribut, voir la section “[Fichiers de Configuration](../configuration-files.md#configuration_files)”. **Voir Aussi** @@ -663,9 +687,9 @@ Pour la valeur de l’ `incl` attribut, voir la section “[Fichiers de Configur Le fuseau horaire du serveur. -Spécifié comme identifiant IANA pour le fuseau horaire UTC ou l’emplacement géographique (par exemple, Afrique / Abidjan). +Spécifié comme identifiant IANA pour le fuseau horaire UTC ou l'emplacement géographique (par exemple, Afrique / Abidjan). -Le fuseau horaire est nécessaire pour les conversions entre les formats String et DateTime lorsque les champs DateTime sont sortis au format texte (imprimés à l’écran ou dans un fichier) et lors de L’obtention de DateTime à partir d’une chaîne. En outre, le fuseau horaire est utilisé dans les fonctions qui fonctionnent avec l’heure et la date si elles ne reçoivent pas le fuseau horaire dans les paramètres d’entrée. +Le fuseau horaire est nécessaire pour les conversions entre les formats String et DateTime lorsque les champs DateTime sont sortis au format texte (imprimés à l'écran ou dans un fichier) et lors de L'obtention de DateTime à partir d'une chaîne. En outre, le fuseau horaire est utilisé dans les fonctions qui fonctionnent avec l'heure et la date si elles ne reçoivent pas le fuseau horaire dans les paramètres d'entrée. **Exemple** @@ -713,7 +737,7 @@ Exemple ## tmp\_path {#server-settings-tmp_path} -Chemin d’accès aux données temporaires pour le traitement des requêtes volumineuses. +Chemin d'accès aux données temporaires pour le traitement des requêtes volumineuses. !!! note "Note" La barre oblique de fin est obligatoire. @@ -727,7 +751,7 @@ Chemin d’accès aux données temporaires pour le traitement des requêtes volu ## tmp\_policy {#server-settings-tmp-policy} La politique de [`storage_configuration`](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes) pour stocker des fichiers temporaires. -Si cela n’est pas [`tmp_path`](#server-settings-tmp_path) est utilisé, sinon elle est ignorée. +Si cela n'est pas [`tmp_path`](#server-settings-tmp_path) est utilisé, sinon elle est ignorée. !!! note "Note" - `move_factor` est ignoré @@ -739,7 +763,7 @@ Si cela n’est pas [`tmp_path`](#server-settings-tmp_path) est utilisé, sinon Taille du Cache (en octets) pour les données non compressées utilisées par les [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md). -Il y a un cache partagé pour le serveur. La mémoire est allouée à la demande. Le cache est utilisé si l’option [use\_uncompressed\_cache](../settings/settings.md#setting-use_uncompressed_cache) est activé. +Il y a un cache partagé pour le serveur. La mémoire est allouée à la demande. Le cache est utilisé si l'option [use\_uncompressed\_cache](../settings/settings.md#setting-use_uncompressed_cache) est activé. Le cache non compressé est avantageux pour les requêtes très courtes dans des cas individuels. @@ -761,10 +785,10 @@ Le répertoire avec les fichiers utilisateur. Utilisé dans la fonction de table ## users\_config {#users-config} -Chemin d’accès au fichier qui contient: +Chemin d'accès au fichier qui contient: -- Les configurations de l’utilisateur. -- Les droits d’accès. +- Les configurations de l'utilisateur. +- Les droits d'accès. - Les paramètres des profils. - Les paramètres de Quota. @@ -776,9 +800,9 @@ Chemin d’accès au fichier qui contient: ## zookeeper {#server-settings_zookeeper} -Contient des paramètres qui permettent à ClickHouse d’interagir avec [ZooKeeper](http://zookeeper.apache.org/) cluster. +Contient des paramètres qui permettent à ClickHouse d'interagir avec [ZooKeeper](http://zookeeper.apache.org/) cluster. -ClickHouse utilise ZooKeeper pour stocker les métadonnées des répliques lors de l’utilisation de tables répliquées. Si les tables répliquées ne sont pas utilisées, cette section de paramètres peut être omise. +ClickHouse utilise ZooKeeper pour stocker les métadonnées des répliques lors de l'utilisation de tables répliquées. Si les tables répliquées ne sont pas utilisées, cette section de paramètres peut être omise. Cette section contient les paramètres suivants: @@ -831,25 +855,25 @@ Cette section contient les paramètres suivants: Méthode de stockage pour les en-têtes de partie de données dans ZooKeeper. -Ce paramètre s’applique uniquement à l’ `MergeTree` famille. Il peut être spécifié: +Ce paramètre s'applique uniquement à l' `MergeTree` famille. Il peut être spécifié: -- À l’échelle mondiale dans le [merge\_tree](#server_configuration_parameters-merge_tree) la section de la `config.xml` fichier. +- À l'échelle mondiale dans le [merge\_tree](#server_configuration_parameters-merge_tree) la section de la `config.xml` fichier. ClickHouse utilise le paramètre pour toutes les tables du serveur. Vous pouvez modifier le réglage à tout moment. Les tables existantes changent de comportement lorsque le paramètre change. - Pour chaque table. - Lors de la création d’un tableau, indiquer la [moteur de réglage](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table). Le comportement d’une table existante avec ce paramètre ne change pas, même si le paramètre global des changements. + Lors de la création d'un tableau, indiquer la [moteur de réglage](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table). Le comportement d'une table existante avec ce paramètre ne change pas, même si le paramètre global des changements. **Valeurs possibles** - 0 — Functionality is turned off. - 1 — Functionality is turned on. -Si `use_minimalistic_part_header_in_zookeeper = 1`, puis [répliqué](../../engines/table-engines/mergetree-family/replication.md) les tables stockent les en-têtes des parties de données de manière compacte à l’aide `znode`. Si la table contient plusieurs colonnes, cette méthode de stockage réduit considérablement le volume des données stockées dans Zookeeper. +Si `use_minimalistic_part_header_in_zookeeper = 1`, puis [répliqué](../../engines/table-engines/mergetree-family/replication.md) les tables stockent les en-têtes des parties de données de manière compacte à l'aide `znode`. Si la table contient plusieurs colonnes, cette méthode de stockage réduit considérablement le volume des données stockées dans Zookeeper. !!! attention "Attention" - Après l’application de `use_minimalistic_part_header_in_zookeeper = 1`, vous ne pouvez pas rétrograder le serveur ClickHouse vers une version qui ne prend pas en charge ce paramètre. Soyez prudent lors de la mise à niveau de ClickHouse sur les serveurs d’un cluster. Ne mettez pas à niveau tous les serveurs à la fois. Il est plus sûr de tester de nouvelles versions de ClickHouse dans un environnement de test, ou sur quelques serveurs d’un cluster. + Après l'application de `use_minimalistic_part_header_in_zookeeper = 1`, vous ne pouvez pas rétrograder le serveur ClickHouse vers une version qui ne prend pas en charge ce paramètre. Soyez prudent lors de la mise à niveau de ClickHouse sur les serveurs d'un cluster. Ne mettez pas à niveau tous les serveurs à la fois. Il est plus sûr de tester de nouvelles versions de ClickHouse dans un environnement de test, ou sur quelques serveurs d'un cluster. Data part headers already stored with this setting can't be restored to their previous (non-compact) representation. @@ -857,7 +881,7 @@ Si `use_minimalistic_part_header_in_zookeeper = 1`, puis [répliqué](../../engi ## disable\_internal\_dns\_cache {#server-settings-disable-internal-dns-cache} -Désactive le cache DNS interne. Recommandé pour l’utilisation de ClickHouse dans les systèmes +Désactive le cache DNS interne. Recommandé pour l'utilisation de ClickHouse dans les systèmes avec des infrastructures en constante évolution telles que Kubernetes. **Valeur par défaut:** 0. @@ -869,4 +893,14 @@ La mise à jour est effectuée de manière asynchrone, dans un thread système s **Valeur par défaut**: 15. +## access\_control\_path {#access_control_path} + +Chemin d'accès à un dossier dans lequel un serveur clickhouse stocke les configurations utilisateur et rôle créées par les commandes SQL. + +Valeur par défaut: `/var/lib/clickhouse/access/`. + +**Voir aussi** + +- [Le Contrôle d'accès et de Gestion de Compte](../access-rights.md#access-control) + [Article Original](https://clickhouse.tech/docs/en/operations/server_configuration_parameters/settings/) diff --git a/docs/fr/operations/settings/constraints-on-settings.md b/docs/fr/operations/settings/constraints-on-settings.md index b7e9e2cf79d..b9c34841da1 100644 --- a/docs/fr/operations/settings/constraints-on-settings.md +++ b/docs/fr/operations/settings/constraints-on-settings.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 62 toc_title: "Contraintes sur les param\xE8tres" --- -# Contraintes Sur Les paramètres {#constraints-on-settings} +# Contraintes sur les paramètres {#constraints-on-settings} Les contraintes sur les paramètres peuvent être définis dans le `profiles` la section de la `user.xml` fichier de configuration et interdire aux utilisateurs de modifier certains `SET` requête. Les contraintes sont définies comme suit: @@ -32,8 +32,8 @@ Les contraintes sont définies comme suit: ``` -Si l’utilisateur tente de violer les contraintes une exception est levée et le réglage n’est pas modifié. -Trois types de contraintes sont pris en charge: `min`, `max`, `readonly`. Le `min` et `max` les contraintes spécifient les limites supérieure et inférieure pour un paramètre numérique et peuvent être utilisées en combinaison. Le `readonly` contrainte spécifie que l’utilisateur ne peut pas modifier le paramètre correspondant à tous. +Si l'utilisateur tente de violer les contraintes une exception est levée et le réglage n'est pas modifié. +Trois types de contraintes sont pris en charge: `min`, `max`, `readonly`. Le `min` et `max` les contraintes spécifient les limites supérieure et inférieure pour un paramètre numérique et peuvent être utilisées en combinaison. Le `readonly` contrainte spécifie que l'utilisateur ne peut pas modifier le paramètre correspondant à tous. **Exemple:** Laisser `users.xml` comprend des lignes: @@ -70,6 +70,6 @@ Code: 452, e.displayText() = DB::Exception: Setting max_memory_usage should not Code: 452, e.displayText() = DB::Exception: Setting force_index_by_date should not be changed. ``` -**Note:** le `default` le profil a une manipulation particulière: toutes les contraintes définies pour `default` le profil devient les contraintes par défaut, de sorte qu’ils restreignent tous les utilisateurs jusqu’à ce qu’ils soient explicitement remplacés pour ces utilisateurs. +**Note:** le `default` le profil a une manipulation particulière: toutes les contraintes définies pour `default` le profil devient les contraintes par défaut, de sorte qu'ils restreignent tous les utilisateurs jusqu'à ce qu'ils soient explicitement remplacés pour ces utilisateurs. [Article Original](https://clickhouse.tech/docs/en/operations/settings/constraints_on_settings/) diff --git a/docs/fr/operations/settings/index.md b/docs/fr/operations/settings/index.md index 1d10d57a2da..0c81a2487c4 100644 --- a/docs/fr/operations/settings/index.md +++ b/docs/fr/operations/settings/index.md @@ -1,21 +1,22 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 -toc_folder_title: Settings +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "Param\xE8tre" toc_priority: 55 toc_title: Introduction --- -# Paramètre {#settings} +# Paramètre {#session-settings-intro} + +Il existe plusieurs façons d'effectuer tous les paramètres décrits dans cette section de la documentation. -Il existe plusieurs façons de faire tous les paramètres décrits ci-dessous. Les paramètres sont configurés en couches, de sorte que chaque couche suivante redéfinit les paramètres précédents. Façons de configurer les paramètres, par ordre de priorité: -- Paramètres dans l’ `users.xml` fichier de configuration du serveur. +- Paramètres dans l' `users.xml` fichier de configuration du serveur. - Situé dans l’élément ``. + Situé dans l'élément ``. - Les paramètres de la Session. @@ -25,7 +26,7 @@ Façons de configurer les paramètres, par ordre de priorité: - Les paramètres de requête. - Lorsque vous démarrez le client clickhouse console en mode non interactif, définissez le paramètre startup `--setting=value`. - - Lors de l’utilisation de L’API HTTP, passez les paramètres CGI (`URL?setting_1=value&setting_2=value...`). + - Lors de l'utilisation de L'API HTTP, passez les paramètres CGI (`URL?setting_1=value&setting_2=value...`). Les paramètres qui ne peuvent être effectués que dans le fichier de configuration du serveur ne sont pas couverts dans cette section. diff --git a/docs/fr/operations/settings/permissions-for-queries.md b/docs/fr/operations/settings/permissions-for-queries.md index 4d014a3937a..9107438e4b5 100644 --- a/docs/fr/operations/settings/permissions-for-queries.md +++ b/docs/fr/operations/settings/permissions-for-queries.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 58 toc_title: "Autorisations pour les requ\xEAtes" --- -# Autorisations Pour Les requêtes {#permissions_for_queries} +# Autorisations pour les requêtes {#permissions_for_queries} Les requêtes dans ClickHouse peuvent être divisées en plusieurs types: @@ -34,11 +34,11 @@ Valeurs possibles: - 1 — Only read data queries are allowed. - 2 — Read data and change settings queries are allowed. -Après le réglage de `readonly = 1` l’utilisateur ne peut pas changer `readonly` et `allow_ddl` les paramètres de la session en cours. +Après le réglage de `readonly = 1` l'utilisateur ne peut pas changer `readonly` et `allow_ddl` les paramètres de la session en cours. -Lors de l’utilisation de la `GET` méthode dans le [Interface HTTP](../../interfaces/http.md), `readonly = 1` est définie automatiquement. Pour modifier les données, utilisez `POST` méthode. +Lors de l'utilisation de la `GET` méthode dans le [Interface HTTP](../../interfaces/http.md), `readonly = 1` est définie automatiquement. Pour modifier les données, utilisez `POST` méthode. -Paramètre `readonly = 1` interdire à l’utilisateur de modifier tous les paramètres. Il y a un moyen d’interdire à l’utilisateur +Paramètre `readonly = 1` interdire à l'utilisateur de modifier tous les paramètres. Il y a un moyen d'interdire à l'utilisateur de modifier uniquement des paramètres spécifiques, pour plus de détails, voir [contraintes sur les paramètres](constraints-on-settings.md). Valeur par défaut: 0 diff --git a/docs/fr/operations/settings/query-complexity.md b/docs/fr/operations/settings/query-complexity.md index c7cd26e1c16..7242c2e0222 100644 --- a/docs/fr/operations/settings/query-complexity.md +++ b/docs/fr/operations/settings/query-complexity.md @@ -1,27 +1,27 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 59 toc_title: "Restrictions sur la complexit\xE9 des requ\xEAtes" --- -# Restrictions Sur La Complexité Des Requêtes {#restrictions-on-query-complexity} +# Restrictions sur la complexité des requêtes {#restrictions-on-query-complexity} Les Restrictions sur la complexité des requêtes font partie des paramètres. -Ils sont utilisés pour fournir une exécution plus sûre à partir de l’interface utilisateur. -Presque toutes les restrictions ne s’appliquent qu’à `SELECT`. Pour le traitement des requêtes distribuées, des restrictions sont appliquées sur chaque serveur séparément. +Ils sont utilisés pour fournir une exécution plus sûre à partir de l'interface utilisateur. +Presque toutes les restrictions ne s'appliquent qu'à `SELECT`. Pour le traitement des requêtes distribuées, des restrictions sont appliquées sur chaque serveur séparément. ClickHouse vérifie les restrictions pour les parties de données, pas pour chaque ligne. Cela signifie que vous pouvez dépasser la valeur de restriction de la taille de la partie données. -Restrictions sur l’ “maximum amount of something” peut prendre la valeur 0, ce qui signifie “unrestricted”. +Restrictions sur l' “maximum amount of something” peut prendre la valeur 0, ce qui signifie “unrestricted”. La plupart des restrictions ont également un ‘overflow\_mode’ paramètre signification que faire lorsque la limite est dépassée. -Il peut prendre deux valeurs: `throw` ou `break`. Les Restrictions sur l’agrégation (group\_by\_overflow\_mode) ont également la valeur `any`. +Il peut prendre deux valeurs: `throw` ou `break`. Les Restrictions sur l'agrégation (group\_by\_overflow\_mode) ont également la valeur `any`. `throw` – Throw an exception (default). `break` – Stop executing the query and return the partial result, as if the source data ran out. -`any (only for group_by_overflow_mode)` – Continuing aggregation for the keys that got into the set, but don’t add new keys to the set. +`any (only for group_by_overflow_mode)` – Continuing aggregation for the keys that got into the set, but don't add new keys to the set. ## max\_memory\_usage {#settings_max_memory_usage} @@ -30,21 +30,21 @@ La quantité maximale de RAM à utiliser pour exécuter une requête sur un seul Dans le fichier de configuration par défaut, le maximum est de 10 Go. Le réglage ne tient pas compte du volume de mémoire disponible ou du volume total de mémoire sur la machine. -La restriction s’applique à une seule requête au sein d’un seul serveur. +La restriction s'applique à une seule requête au sein d'un seul serveur. Vous pouvez utiliser `SHOW PROCESSLIST` pour vérifier la consommation de mémoire pour chaque requête. En outre, la consommation de mémoire maximale est suivie pour chaque requête et écrite dans le journal. -L’utilisation de la mémoire n’est pas surveillée pour les membres de certaines fonctions d’agrégation. +L'utilisation de la mémoire n'est pas surveillée pour les membres de certaines fonctions d'agrégation. -L’utilisation de la mémoire n’est pas totalement suivies pour les états des fonctions d’agrégation `min`, `max`, `any`, `anyLast`, `argMin`, `argMax` de `String` et `Array` argument. +L'utilisation de la mémoire n'est pas totalement suivies pour les états des fonctions d'agrégation `min`, `max`, `any`, `anyLast`, `argMin`, `argMax` de `String` et `Array` argument. La consommation de mémoire est également limitée par les paramètres `max_memory_usage_for_user` et `max_memory_usage_for_all_queries`. ## max\_memory\_usage\_for\_user {#max-memory-usage-for-user} -Quantité maximale de RAM à utiliser pour exécuter les requêtes d’un utilisateur sur un seul serveur. +Quantité maximale de RAM à utiliser pour exécuter les requêtes d'un utilisateur sur un seul serveur. -Les valeurs par défaut sont définies dans [Paramètre.h](https://github.com/ClickHouse/ClickHouse/blob/master/dbms/Core/Settings.h#L288). Par défaut, le montant n’est pas limité (`max_memory_usage_for_user = 0`). +Les valeurs par défaut sont définies dans [Paramètre.h](https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/Settings.h#L288). Par défaut, le montant n'est pas limité (`max_memory_usage_for_user = 0`). Voir aussi la description de [max\_memory\_usage](#settings_max_memory_usage). @@ -52,41 +52,41 @@ Voir aussi la description de [max\_memory\_usage](#settings_max_memory_usage). La quantité maximale de RAM à utiliser pour exécuter toutes les requêtes sur un seul serveur. -Les valeurs par défaut sont définies dans [Paramètre.h](https://github.com/ClickHouse/ClickHouse/blob/master/dbms/Core/Settings.h#L289). Par défaut, le montant n’est pas limité (`max_memory_usage_for_all_queries = 0`). +Les valeurs par défaut sont définies dans [Paramètre.h](https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/Settings.h#L289). Par défaut, le montant n'est pas limité (`max_memory_usage_for_all_queries = 0`). Voir aussi la description de [max\_memory\_usage](#settings_max_memory_usage). ## max\_rows\_to\_read {#max-rows-to-read} Les restrictions suivantes peut être vérifiée sur chaque bloc (au lieu de sur chaque ligne). Autrement dit, les restrictions peuvent être brisées un peu. -Lors de l’exécution d’une requête dans plusieurs threads, les restrictions suivantes s’appliquent à chaque thread séparément. +Lors de l'exécution d'une requête dans plusieurs threads, les restrictions suivantes s'appliquent à chaque thread séparément. -Un nombre maximum de lignes pouvant être lues à partir d’un tableau lors de l’exécution d’une requête. +Un nombre maximum de lignes pouvant être lues à partir d'un tableau lors de l'exécution d'une requête. ## max\_bytes\_to\_read {#max-bytes-to-read} -Nombre maximal d’octets (données non compressées) pouvant être lus à partir d’une table lors de l’exécution d’une requête. +Nombre maximal d'octets (données non compressées) pouvant être lus à partir d'une table lors de l'exécution d'une requête. ## read\_overflow\_mode {#read-overflow-mode} -Que faire lorsque le volume de lecture de données dépasse l’une des limites: ‘throw’ ou ‘break’. Par défaut, les jeter. +Que faire lorsque le volume de lecture de données dépasse l'une des limites: ‘throw’ ou ‘break’. Par défaut, les jeter. ## max\_rows\_to\_group\_by {#settings-max-rows-to-group-by} -Un nombre maximum de clés uniques reçues de l’agrégation. Ce paramètre permet de limiter la consommation de mémoire lors de l’agrégation. +Un nombre maximum de clés uniques reçues de l'agrégation. Ce paramètre permet de limiter la consommation de mémoire lors de l'agrégation. ## group\_by\_overflow\_mode {#group-by-overflow-mode} -Que faire lorsque le nombre de clés uniques pour l’agrégation dépasse la limite: ‘throw’, ‘break’, ou ‘any’. Par défaut, les jeter. -À l’aide de la ‘any’ valeur vous permet d’exécuter une approximation de GROUP BY. La qualité de cette approximation dépend de la nature statistique des données. +Que faire lorsque le nombre de clés uniques pour l'agrégation dépasse la limite: ‘throw’, ‘break’, ou ‘any’. Par défaut, les jeter. +À l'aide de la ‘any’ valeur vous permet d'exécuter une approximation de GROUP BY. La qualité de cette approximation dépend de la nature statistique des données. ## max\_bytes\_before\_external\_group\_by {#settings-max_bytes_before_external_group_by} -Active ou désactive l’exécution de `GROUP BY` clauses dans la mémoire externe. Voir [Groupe par dans la mémoire externe](../../sql-reference/statements/select.md#select-group-by-in-external-memory). +Active ou désactive l'exécution de `GROUP BY` clauses dans la mémoire externe. Voir [Groupe par dans la mémoire externe](../../sql-reference/statements/select/group-by.md#select-group-by-in-external-memory). Valeurs possibles: -- Volume maximal de RAM (en octets) pouvant être utilisé par le [GROUP BY](../../sql-reference/statements/select.md#select-group-by-clause) opération. +- Volume maximal de RAM (en octets) pouvant être utilisé par le [GROUP BY](../../sql-reference/statements/select/group-by.md#select-group-by-clause) opération. - 0 — `GROUP BY` dans la mémoire externe désactivé. Valeur par défaut: 0. @@ -97,25 +97,25 @@ Un nombre maximum de lignes avant le tri. Cela vous permet de limiter la consomm ## max\_bytes\_to\_sort {#max-bytes-to-sort} -Un nombre maximal d’octets avant le tri. +Un nombre maximal d'octets avant le tri. ## sort\_overflow\_mode {#sort-overflow-mode} -Que faire si le nombre de lignes reçues avant le tri dépasse l’une des limites: ‘throw’ ou ‘break’. Par défaut, les jeter. +Que faire si le nombre de lignes reçues avant le tri dépasse l'une des limites: ‘throw’ ou ‘break’. Par défaut, les jeter. ## max\_result\_rows {#setting-max_result_rows} -Limite sur le nombre de lignes dans le résultat. Également vérifié pour les sous-requêtes, et sur des serveurs distants lors de l’exécution de parties d’une requête distribuée. +Limite sur le nombre de lignes dans le résultat. Également vérifié pour les sous-requêtes, et sur des serveurs distants lors de l'exécution de parties d'une requête distribuée. ## max\_result\_bytes {#max-result-bytes} -Limite sur le nombre d’octets dans le résultat. Le même que le réglage précédent. +Limite sur le nombre d'octets dans le résultat. Le même que le réglage précédent. ## result\_overflow\_mode {#result-overflow-mode} -Que faire si le volume du résultat dépasse l’une des limites: ‘throw’ ou ‘break’. Par défaut, les jeter. +Que faire si le volume du résultat dépasse l'une des limites: ‘throw’ ou ‘break’. Par défaut, les jeter. -Utiliser ‘break’ est similaire à L’utilisation de LIMIT. `Break` interrompt l’exécution seulement au niveau du bloc. Cela signifie que la quantité de lignes renvoyées est supérieure à [max\_result\_rows](#setting-max_result_rows) multiples de [max\_block\_size](settings.md#setting-max_block_size) et dépend de l’ [max\_threads](settings.md#settings-max_threads). +Utiliser ‘break’ est similaire à L'utilisation de LIMIT. `Break` interrompt l'exécution seulement au niveau du bloc. Cela signifie que la quantité de lignes renvoyées est supérieure à [max\_result\_rows](#setting-max_result_rows) multiples de [max\_block\_size](settings.md#setting-max_block_size) et dépend de l' [max\_threads](settings.md#settings-max_threads). Exemple: @@ -136,8 +136,8 @@ Résultat: ## max\_execution\_time {#max-execution-time} -Durée maximale d’exécution de la requête en secondes. -Pour le moment, il n’est pas vérifié pour l’une des étapes de tri, ni lors de la fusion et de la finalisation des fonctions d’agrégat. +Durée maximale d'exécution de la requête en secondes. +Pour le moment, il n'est pas vérifié pour l'une des étapes de tri, ni lors de la fusion et de la finalisation des fonctions d'agrégat. ## timeout\_overflow\_mode {#timeout-overflow-mode} @@ -145,36 +145,36 @@ Que faire si la requête est exécutée plus de ‘max\_execution\_time’: ‘t ## min\_execution\_speed {#min-execution-speed} -Vitesse d’exécution minimale en lignes par seconde. Vérifié sur chaque bloc de données quand ‘timeout\_before\_checking\_execution\_speed’ expirer. Si la vitesse d’exécution est inférieure, une exception est levée. +Vitesse d'exécution minimale en lignes par seconde. Vérifié sur chaque bloc de données quand ‘timeout\_before\_checking\_execution\_speed’ expirer. Si la vitesse d'exécution est inférieure, une exception est levée. ## min\_execution\_speed\_bytes {#min-execution-speed-bytes} -Un nombre minimum d’exécution d’octets par seconde. Vérifié sur chaque bloc de données quand ‘timeout\_before\_checking\_execution\_speed’ expirer. Si la vitesse d’exécution est inférieure, une exception est levée. +Un nombre minimum d'exécution d'octets par seconde. Vérifié sur chaque bloc de données quand ‘timeout\_before\_checking\_execution\_speed’ expirer. Si la vitesse d'exécution est inférieure, une exception est levée. ## max\_execution\_speed {#max-execution-speed} -Un nombre maximal d’exécution de lignes par seconde. Vérifié sur chaque bloc de données quand ‘timeout\_before\_checking\_execution\_speed’ expirer. Si la vitesse d’exécution est élevée, la vitesse d’exécution sera réduit. +Un nombre maximal d'exécution de lignes par seconde. Vérifié sur chaque bloc de données quand ‘timeout\_before\_checking\_execution\_speed’ expirer. Si la vitesse d'exécution est élevée, la vitesse d'exécution sera réduit. ## max\_execution\_speed\_bytes {#max-execution-speed-bytes} -Un nombre maximal d’exécution d’octets par seconde. Vérifié sur chaque bloc de données quand ‘timeout\_before\_checking\_execution\_speed’ expirer. Si la vitesse d’exécution est élevée, la vitesse d’exécution sera réduit. +Un nombre maximal d'exécution d'octets par seconde. Vérifié sur chaque bloc de données quand ‘timeout\_before\_checking\_execution\_speed’ expirer. Si la vitesse d'exécution est élevée, la vitesse d'exécution sera réduit. ## timeout\_before\_checking\_execution\_speed {#timeout-before-checking-execution-speed} -Vérifie que la vitesse d’exécution n’est pas trop lent (pas moins de ‘min\_execution\_speed’), après l’expiration du temps spécifié en secondes. +Vérifie que la vitesse d'exécution n'est pas trop lent (pas moins de ‘min\_execution\_speed’), après l'expiration du temps spécifié en secondes. ## max\_columns\_to\_read {#max-columns-to-read} -Nombre maximal de colonnes pouvant être lues à partir d’une table dans une seule requête. Si une requête nécessite la lecture d’un plus grand nombre de colonnes, il lève une exception. +Nombre maximal de colonnes pouvant être lues à partir d'une table dans une seule requête. Si une requête nécessite la lecture d'un plus grand nombre de colonnes, il lève une exception. ## max\_temporary\_columns {#max-temporary-columns} -Nombre maximal de colonnes temporaires qui doivent être conservées en RAM en même temps lors de l’exécution d’une requête, y compris les colonnes constantes. S’il y a plus de colonnes temporaires que cela, il lève une exception. +Nombre maximal de colonnes temporaires qui doivent être conservées en RAM en même temps lors de l'exécution d'une requête, y compris les colonnes constantes. S'il y a plus de colonnes temporaires que cela, il lève une exception. ## max\_temporary\_non\_const\_columns {#max-temporary-non-const-columns} La même chose que ‘max\_temporary\_columns’ mais sans compter constante colonnes. -Notez que les colonnes constantes sont formées assez souvent lors de l’exécution d’une requête, mais elles nécessitent environ zéro ressource informatique. +Notez que les colonnes constantes sont formées assez souvent lors de l'exécution d'une requête, mais elles nécessitent environ zéro ressource informatique. ## max\_subquery\_depth {#max-subquery-depth} @@ -182,63 +182,63 @@ Profondeur maximale de sous-requêtes. Si les sous-requêtes sont plus profondes ## max\_pipeline\_depth {#max-pipeline-depth} -Profondeur maximale du pipeline. Correspond au nombre de transformations que chaque bloc de données lors du traitement des requêtes. Compté dans les limites d’un seul serveur. Si la profondeur du pipeline est supérieure, une exception est levée. Par défaut, 1000. +Profondeur maximale du pipeline. Correspond au nombre de transformations que chaque bloc de données lors du traitement des requêtes. Compté dans les limites d'un seul serveur. Si la profondeur du pipeline est supérieure, une exception est levée. Par défaut, 1000. ## max\_ast\_depth {#max-ast-depth} -Profondeur maximale d’une requête arbre syntaxique. En cas de dépassement, une exception est levée. -À ce moment, il n’est pas vérifié pendant l’analyse, mais seulement après l’analyse de la requête. Autrement dit, un arbre syntaxique trop profond peut être créé pendant l’analyse, mais la requête échouera. Par défaut, 1000. +Profondeur maximale d'une requête arbre syntaxique. En cas de dépassement, une exception est levée. +À ce moment, il n'est pas vérifié pendant l'analyse, mais seulement après l'analyse de la requête. Autrement dit, un arbre syntaxique trop profond peut être créé pendant l'analyse, mais la requête échouera. Par défaut, 1000. ## max\_ast\_elements {#max-ast-elements} -Un nombre maximal d’éléments dans une requête arbre syntaxique. En cas de dépassement, une exception est levée. -De la même manière que le paramètre précédent, il est vérifié qu’après l’analyse de la requête. Par défaut, 50 000. +Un nombre maximal d'éléments dans une requête arbre syntaxique. En cas de dépassement, une exception est levée. +De la même manière que le paramètre précédent, il est vérifié qu'après l'analyse de la requête. Par défaut, 50 000. ## max\_rows\_in\_set {#max-rows-in-set} -Nombre maximal de lignes pour un ensemble de données dans la clause in créée à partir d’une sous-requête. +Nombre maximal de lignes pour un ensemble de données dans la clause in créée à partir d'une sous-requête. ## max\_bytes\_in\_set {#max-bytes-in-set} -Nombre maximal d’octets (données non compressées) utilisés par un ensemble de la clause in créé à partir d’une sous-requête. +Nombre maximal d'octets (données non compressées) utilisés par un ensemble de la clause in créé à partir d'une sous-requête. ## set\_overflow\_mode {#set-overflow-mode} -Que faire lorsque la quantité de données dépasse l’une des limites: ‘throw’ ou ‘break’. Par défaut, les jeter. +Que faire lorsque la quantité de données dépasse l'une des limites: ‘throw’ ou ‘break’. Par défaut, les jeter. ## max\_rows\_in\_distinct {#max-rows-in-distinct} -Un nombre maximum de lignes différentes lors de L’utilisation de DISTINCT. +Un nombre maximum de lignes différentes lors de L'utilisation de DISTINCT. ## max\_bytes\_in\_distinct {#max-bytes-in-distinct} -Nombre maximal d’octets utilisés par une table de hachage lors de L’utilisation de DISTINCT. +Nombre maximal d'octets utilisés par une table de hachage lors de L'utilisation de DISTINCT. ## distinct\_overflow\_mode {#distinct-overflow-mode} -Que faire lorsque la quantité de données dépasse l’une des limites: ‘throw’ ou ‘break’. Par défaut, les jeter. +Que faire lorsque la quantité de données dépasse l'une des limites: ‘throw’ ou ‘break’. Par défaut, les jeter. ## max\_rows\_to\_transfer {#max-rows-to-transfer} -Nombre maximal de lignes pouvant être transmises à un serveur distant ou enregistrées dans une table temporaire lors de L’utilisation de GLOBAL IN. +Nombre maximal de lignes pouvant être transmises à un serveur distant ou enregistrées dans une table temporaire lors de L'utilisation de GLOBAL IN. ## max\_bytes\_to\_transfer {#max-bytes-to-transfer} -Nombre maximal d’octets (données non compressées) pouvant être transmis à un serveur distant ou enregistrés dans une table temporaire lors de L’utilisation de GLOBAL IN. +Nombre maximal d'octets (données non compressées) pouvant être transmis à un serveur distant ou enregistrés dans une table temporaire lors de L'utilisation de GLOBAL IN. ## transfer\_overflow\_mode {#transfer-overflow-mode} -Que faire lorsque la quantité de données dépasse l’une des limites: ‘throw’ ou ‘break’. Par défaut, les jeter. +Que faire lorsque la quantité de données dépasse l'une des limites: ‘throw’ ou ‘break’. Par défaut, les jeter. ## max\_rows\_in\_join {#settings-max_rows_in_join} Limite le nombre de lignes dans la table de hachage utilisée lors de la jonction de tables. -Ce réglage s’applique à [SELECT … JOIN](../../sql-reference/statements/select.md#select-join) les opérations et les [Rejoindre](../../engines/table-engines/special/join.md) tableau moteur. +Ce réglage s'applique à [SELECT … JOIN](../../sql-reference/statements/select/join.md#select-join) les opérations et les [Rejoindre](../../engines/table-engines/special/join.md) tableau moteur. Si une requête contient plusieurs jointures, ClickHouse vérifie ce paramètre pour chaque résultat intermédiaire. -ClickHouse peut procéder à différentes actions lorsque la limite est atteinte. L’utilisation de la [join\_overflow\_mode](#settings-join_overflow_mode) réglage pour choisir l’action. +ClickHouse peut procéder à différentes actions lorsque la limite est atteinte. L'utilisation de la [join\_overflow\_mode](#settings-join_overflow_mode) réglage pour choisir l'action. Valeurs possibles: @@ -249,13 +249,13 @@ Valeur par défaut: 0. ## max\_bytes\_in\_join {#settings-max_bytes_in_join} -Limite la taille en octets de la table de hachage utilisée lors de l’assemblage de tables. +Limite la taille en octets de la table de hachage utilisée lors de l'assemblage de tables. -Ce réglage s’applique à [SELECT … JOIN](../../sql-reference/statements/select.md#select-join) les opérations et les [Rejoindre le moteur de table](../../engines/table-engines/special/join.md). +Ce réglage s'applique à [SELECT … JOIN](../../sql-reference/statements/select/join.md#select-join) les opérations et les [Rejoindre le moteur de table](../../engines/table-engines/special/join.md). Si la requête contient des jointures, ClickHouse vérifie ce paramètre pour chaque résultat intermédiaire. -ClickHouse peut procéder à différentes actions lorsque la limite est atteinte. Utiliser [join\_overflow\_mode](#settings-join_overflow_mode) paramètres pour choisir l’action. +ClickHouse peut procéder à différentes actions lorsque la limite est atteinte. Utiliser [join\_overflow\_mode](#settings-join_overflow_mode) paramètres pour choisir l'action. Valeurs possibles: @@ -266,7 +266,7 @@ Valeur par défaut: 0. ## join\_overflow\_mode {#settings-join_overflow_mode} -Définit l’action que ClickHouse effectue lorsque l’une des limites de jointure suivantes est atteinte: +Définit l'action que ClickHouse effectue lorsque l'une des limites de jointure suivantes est atteinte: - [max\_bytes\_in\_join](#settings-max_bytes_in_join) - [max\_rows\_in\_join](#settings-max_rows_in_join) @@ -274,13 +274,13 @@ Définit l’action que ClickHouse effectue lorsque l’une des limites de joint Valeurs possibles: - `THROW` — ClickHouse throws an exception and breaks operation. -- `BREAK` — ClickHouse breaks operation and doesn’t throw an exception. +- `BREAK` — ClickHouse breaks operation and doesn't throw an exception. Valeur par défaut: `THROW`. **Voir Aussi** -- [Clause de JOINTURE](../../sql-reference/statements/select.md#select-join) +- [Clause de JOINTURE](../../sql-reference/statements/select/join.md#select-join) - [Rejoindre le moteur de table](../../engines/table-engines/special/join.md) ## max\_partitions\_per\_insert\_block {#max-partitions-per-insert-block} @@ -294,7 +294,7 @@ Valeur par défaut: 100. **Détail** -Lors de l’insertion de données, ClickHouse calcule le nombre de partitions dans le bloc inséré. Si le nombre de partitions est plus que `max_partitions_per_insert_block`, ClickHouse lève une exception avec le texte suivant: +Lors de l'insertion de données, ClickHouse calcule le nombre de partitions dans le bloc inséré. Si le nombre de partitions est plus que `max_partitions_per_insert_block`, ClickHouse lève une exception avec le texte suivant: > “Too many partitions for single INSERT block (more than” + toString (max\_parts) + “). The limit is controlled by ‘max\_partitions\_per\_insert\_block’ setting. A large number of partitions is a common misconception. It will lead to severe negative performance impact, including slow server startup, slow INSERT queries and slow SELECT queries. Recommended total number of partitions for a table is under 1000..10000. Please note, that partitioning is not intended to speed up SELECT queries (ORDER BY key is sufficient to make range queries fast). Partitions are intended for data manipulation (DROP PARTITION, etc).” diff --git a/docs/fr/operations/settings/settings-profiles.md b/docs/fr/operations/settings/settings-profiles.md index 93da554cee5..2a0338e7873 100644 --- a/docs/fr/operations/settings/settings-profiles.md +++ b/docs/fr/operations/settings/settings-profiles.md @@ -1,14 +1,22 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 61 toc_title: "Les Param\xE8tres Des Profils" --- # Les Paramètres Des Profils {#settings-profiles} -Un profil de paramètres est une collection de paramètres regroupés sous le même nom. Chaque utilisateur de ClickHouse a un profil. -Pour appliquer tous les paramètres d’un profil, définissez `profile` paramètre. +Un profil de paramètres est une collection de paramètres regroupés sous le même nom. + +!!! note "Information" + Clickhouse prend également en charge [Flux de travail piloté par SQL](../access-rights.md#access-control) pour gérer les profils de paramètres. Nous vous conseillons de l'utiliser. + +Un profil peut avoir n'importe quel nom. Le profil peut avoir n'importe quel nom. Vous pouvez spécifier le même profil pour différents utilisateurs. La chose la plus importante que vous pouvez écrire dans les paramètres de profil `readonly=1` qui assure un accès en lecture seule. + +Paramètres les profils peuvent hériter les uns des autres. Pour utiliser l'héritage, indiquer un ou plusieurs `profile` paramètres avant les autres paramètres répertoriés dans le profil. Dans le cas où un paramètre est défini dans les différents profils, les dernières définie est utilisée. + +Pour appliquer tous les paramètres d'un profil, définissez `profile` paramètre. Exemple: @@ -18,7 +26,7 @@ Installer le `web` profil. SET profile = 'web' ``` -Les profils de paramètres sont déclarés dans le fichier de configuration utilisateur. Ce n’est généralement `users.xml`. +Les profils de paramètres sont déclarés dans le fichier de configuration utilisateur. Ce n'est généralement `users.xml`. Exemple: @@ -64,8 +72,10 @@ Exemple: ``` -L’exemple spécifie deux profils: `default` et `web`. Le `default` profil a un but particulier: il doit toujours être présent et est appliquée lors du démarrage du serveur. En d’autres termes, l’ `default` profil contient les paramètres par défaut. Le `web` profil est un profil régulier qui peut être défini à l’aide `SET` requête ou en utilisant un paramètre URL dans une requête HTTP. +L'exemple spécifie deux profils: `default` et `web`. -Paramètres les profils peuvent hériter les uns des autres. Pour utiliser l’héritage, indiquer un ou plusieurs `profile` paramètres avant les autres paramètres répertoriés dans le profil. Dans le cas où un paramètre est défini dans les différents profils, les dernières définie est utilisée. +Le `default` profil a un but particulier: il doit toujours être présent et est appliquée lors du démarrage du serveur. En d'autres termes, l' `default` profil contient les paramètres par défaut. + +Le `web` profil est un profil régulier qui peut être défini à l'aide `SET` requête ou en utilisant un paramètre URL dans une requête HTTP. [Article Original](https://clickhouse.tech/docs/en/operations/settings/settings_profiles/) diff --git a/docs/fr/operations/settings/settings-users.md b/docs/fr/operations/settings/settings-users.md index 6256c6002e5..2be41d90533 100644 --- a/docs/fr/operations/settings/settings-users.md +++ b/docs/fr/operations/settings/settings-users.md @@ -1,14 +1,17 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 63 toc_title: "Les Param\xE8tres De L'Utilisateur" --- -# Les Paramètres De L’Utilisateur {#user-settings} +# Les Paramètres De L'Utilisateur {#user-settings} Le `users` la section de la `user.xml` le fichier de configuration contient les paramètres utilisateur. +!!! note "Information" + Clickhouse prend également en charge [Flux de travail piloté par SQL](../access-rights.md#access-control) pour gérer les utilisateurs. Nous vous conseillons de l'utiliser. + La Structure de la `users` section: ``` xml @@ -19,6 +22,8 @@ La Structure de la `users` section: + 0|1 + @@ -48,11 +53,11 @@ Le mot de passe peut être spécifié en texte clair ou en SHA256 (format hexad -- Pour attribuer un mot de passe à l’aide de son hachage SHA256, placez-le dans un `password_sha256_hex` élément. +- Pour attribuer un mot de passe à l'aide de son hachage SHA256, placez-le dans un `password_sha256_hex` élément. Exemple, `65e84be33532fb784c48129675f9eff3a682b27168c0ea744b2cf58ee02337c5`. - Exemple de génération d’un mot de passe à partir du shell: + Exemple de génération d'un mot de passe à partir du shell: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha256sum | tr -d '-' @@ -64,17 +69,28 @@ Le mot de passe peut être spécifié en texte clair ou en SHA256 (format hexad Exemple, `08b4a0f1de6ad37da17359e592c8d74788a83eb0`. - Exemple de génération d’un mot de passe à partir du shell: + Exemple de génération d'un mot de passe à partir du shell: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha1sum | tr -d '-' | xxd -r -p | sha1sum | tr -d '-' La première ligne du résultat est le mot de passe. La deuxième ligne est le double hachage SHA1 correspondant. +### access\_management {#access_management-user-setting} + +Ce paramètre active de désactive l'utilisation de SQL-driven [le contrôle d'accès et de gestion de compte](../access-rights.md#access-control) pour l'utilisateur. + +Valeurs possibles: + +- 0 — Disabled. +- 1 — Enabled. + +Valeur par défaut: 0. + ### nom\_utilisateur / réseaux {#user-namenetworks} -Liste des réseaux à partir desquels L’utilisateur peut se connecter au serveur ClickHouse. +Liste des réseaux à partir desquels L'utilisateur peut se connecter au serveur ClickHouse. -Chaque élément de la liste peut avoir l’une des formes suivantes: +Chaque élément de la liste peut avoir l'une des formes suivantes: - `` — IP address or network mask. @@ -84,28 +100,28 @@ Chaque élément de la liste peut avoir l’une des formes suivantes: Exemple: `example01.host.ru`. - Pour vérifier l’accès, une requête DNS est effectuée et toutes les adresses IP renvoyées sont comparées à l’adresse homologue. + Pour vérifier l'accès, une requête DNS est effectuée et toutes les adresses IP renvoyées sont comparées à l'adresse homologue. - `` — Regular expression for hostnames. Exemple, `^example\d\d-\d\d-\d\.host\.ru$` - Pour vérifier l’accès, un [Requête DNS PTR](https://en.wikipedia.org/wiki/Reverse_DNS_lookup) est effectuée pour l’adresse homologue, puis l’expression rationnelle spécifiée est appliquée. Ensuite, une autre requête DNS est effectuée pour les résultats de la requête PTR et toutes les adresses reçues sont comparées à l’adresse homologue. Nous recommandons fortement que regexp se termine avec $. + Pour vérifier l'accès, un [Requête DNS PTR](https://en.wikipedia.org/wiki/Reverse_DNS_lookup) est effectuée pour l'adresse homologue, puis l'expression rationnelle spécifiée est appliquée. Ensuite, une autre requête DNS est effectuée pour les résultats de la requête PTR et toutes les adresses reçues sont comparées à l'adresse homologue. Nous recommandons fortement que regexp se termine avec $. -Tous les résultats des requêtes DNS sont mis en cache jusqu’au redémarrage du serveur. +Tous les résultats des requêtes DNS sont mis en cache jusqu'au redémarrage du serveur. **Exemple** -Pour ouvrir l’accès de l’utilisateur à partir de n’importe quel réseau, spécifiez: +Pour ouvrir l'accès de l'utilisateur à partir de n'importe quel réseau, spécifiez: ``` xml ::/0 ``` !!! warning "Avertissement" - Il n’est pas sûr d’ouvrir l’accès à partir de n’importe quel réseau, sauf si vous avez un pare-feu correctement configuré ou si le serveur n’est pas directement connecté à Internet. + Il n'est pas sûr d'ouvrir l'accès à partir de n'importe quel réseau, sauf si vous avez un pare-feu correctement configuré ou si le serveur n'est pas directement connecté à Internet. -Pour ouvrir l’accès uniquement à partir de localhost, spécifier: +Pour ouvrir l'accès uniquement à partir de localhost, spécifier: ``` xml ::1 @@ -114,18 +130,18 @@ Pour ouvrir l’accès uniquement à partir de localhost, spécifier: ### nom\_utilisateur / profil {#user-nameprofile} -Vous pouvez attribuer un profil des paramètres pour l’utilisateur. Les profils de paramètres sont configurés dans une section distincte du `users.xml` fichier. Pour plus d’informations, voir [Profils des paramètres](settings-profiles.md). +Vous pouvez attribuer un profil des paramètres pour l'utilisateur. Les profils de paramètres sont configurés dans une section distincte du `users.xml` fichier. Pour plus d'informations, voir [Profils des paramètres](settings-profiles.md). ### nom\_utilisateur / quota {#user-namequota} -Les Quotas vous permettent de suivre ou de limiter l’utilisation des ressources sur une période donnée. Les Quotas sont configurés dans le `quotas` +Les Quotas vous permettent de suivre ou de limiter l'utilisation des ressources sur une période donnée. Les Quotas sont configurés dans le `quotas` la section de la `users.xml` fichier de configuration. -Vous pouvez attribuer un jeu de quotas à l’utilisateur. Pour une description détaillée de la configuration des quotas, voir [Quota](../quotas.md#quotas). +Vous pouvez attribuer un jeu de quotas à l'utilisateur. Pour une description détaillée de la configuration des quotas, voir [Quota](../quotas.md#quotas). ### nom\_utilisateur/bases de données {#user-namedatabases} -Dans cette section, vous pouvez limiter les lignes renvoyées par ClickHouse pour `SELECT` requêtes faites par l’utilisateur actuel, implémentant ainsi la sécurité de base au niveau de la ligne. +Dans cette section, vous pouvez limiter les lignes renvoyées par ClickHouse pour `SELECT` requêtes faites par l'utilisateur actuel, implémentant ainsi la sécurité de base au niveau de la ligne. **Exemple** @@ -143,6 +159,6 @@ La configuration suivante force cet utilisateur `user1` ne peut voir les lignes ``` -Le `filter` peut être n’importe quelle expression résultant en un [UInt8](../../sql-reference/data-types/int-uint.md)-le type de la valeur. Il contient généralement des comparaisons et des opérateurs logiques. Les lignes de `database_name.table1` où filtrer les résultats à 0 ne sont pas retournés pour cet utilisateur. Le filtrage est incompatible avec `PREWHERE` opérations et désactive `WHERE→PREWHERE` optimisation. +Le `filter` peut être n'importe quelle expression résultant en un [UInt8](../../sql-reference/data-types/int-uint.md)-le type de la valeur. Il contient généralement des comparaisons et des opérateurs logiques. Les lignes de `database_name.table1` où filtrer les résultats à 0 ne sont pas retournés pour cet utilisateur. Le filtrage est incompatible avec `PREWHERE` opérations et désactive `WHERE→PREWHERE` optimisation. [Article Original](https://clickhouse.tech/docs/en/operations/settings/settings_users/) diff --git a/docs/fr/operations/settings/settings.md b/docs/fr/operations/settings/settings.md index 92855487d2a..ab26a114bcf 100644 --- a/docs/fr/operations/settings/settings.md +++ b/docs/fr/operations/settings/settings.md @@ -1,23 +1,21 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 -toc_priority: 60 -toc_title: "Param\xE8tre" +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd --- # Paramètre {#settings} ## distributed\_product\_mode {#distributed-product-mode} -Modifie le comportement de [distribués sous-requêtes](../../sql-reference/statements/select.md). +Modifie le comportement de [distribués sous-requêtes](../../sql-reference/operators/in.md). ClickHouse applies this setting when the query contains the product of distributed tables, i.e. when the query for a distributed table contains a non-GLOBAL subquery for the distributed table. Restriction: - Uniquement appliqué pour les sous-requêtes IN et JOIN. -- Uniquement si la section FROM utilise une table distribuée contenant plus d’un fragment. -- Si la sous-requête concerne un distribué tableau contenant plus d’un fragment. +- Uniquement si la section FROM utilise une table distribuée contenant plus d'un fragment. +- Si la sous-requête concerne un distribué tableau contenant plus d'un fragment. - Pas utilisé pour une table [distant](../../sql-reference/table-functions/remote.md) fonction. Valeurs possibles: @@ -47,9 +45,9 @@ Considérez les requêtes suivantes: 1. `SELECT count() FROM test_table WHERE date = '2018-10-10'` 2. `SELECT count() FROM (SELECT * FROM test_table) WHERE date = '2018-10-10'` -Si `enable_optimize_predicate_expression = 1`, alors le temps d’exécution de ces requêtes est égal car ClickHouse s’applique `WHERE` à la sous-requête lors du traitement. +Si `enable_optimize_predicate_expression = 1`, alors le temps d'exécution de ces requêtes est égal car ClickHouse s'applique `WHERE` à la sous-requête lors du traitement. -Si `enable_optimize_predicate_expression = 0` puis le temps d’exécution de la deuxième requête est beaucoup plus long, parce que le `WHERE` la clause s’applique à toutes les données après la sous-requête des finitions. +Si `enable_optimize_predicate_expression = 0` puis le temps d'exécution de la deuxième requête est beaucoup plus long, parce que le `WHERE` la clause s'applique à toutes les données après la sous-requête des finitions. ## fallback\_to\_stale\_replicas\_for\_distributed\_queries {#settings-fallback_to_stale_replicas_for_distributed_queries} @@ -57,33 +55,33 @@ Force une requête à un réplica obsolète si les données mises à jour ne son ClickHouse sélectionne le plus pertinent parmi les répliques obsolètes de la table. -Utilisé lors de l’exécution `SELECT` à partir d’une table distribuée qui pointe vers des tables répliquées. +Utilisé lors de l'exécution `SELECT` à partir d'une table distribuée qui pointe vers des tables répliquées. Par défaut, 1 (activé). ## force\_index\_by\_date {#settings-force_index_by_date} -Désactive l’exécution de la requête si l’index ne peut pas être utilisé par jour. +Désactive l'exécution de la requête si l'index ne peut pas être utilisé par jour. Fonctionne avec les tables de la famille MergeTree. -Si `force_index_by_date=1`, Clickhouse vérifie si la requête a une condition de clé de date qui peut être utilisée pour restreindre les plages de données. S’il n’y a pas de condition appropriée, il lève une exception. Cependant, il ne vérifie pas si la condition réduit la quantité de données à lire. Par exemple, la condition `Date != ' 2000-01-01 '` est acceptable même lorsqu’il correspond à toutes les données de la table (c’est-à-dire que l’exécution de la requête nécessite une analyse complète). Pour plus d’informations sur les plages de données dans les tables MergeTree, voir [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md). +Si `force_index_by_date=1`, Clickhouse vérifie si la requête a une condition de clé de date qui peut être utilisée pour restreindre les plages de données. S'il n'y a pas de condition appropriée, il lève une exception. Cependant, il ne vérifie pas si la condition réduit la quantité de données à lire. Par exemple, la condition `Date != ' 2000-01-01 '` est acceptable même lorsqu'il correspond à toutes les données de la table (c'est-à-dire que l'exécution de la requête nécessite une analyse complète). Pour plus d'informations sur les plages de données dans les tables MergeTree, voir [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md). ## force\_primary\_key {#force-primary-key} -Désactive l’exécution de la requête si l’indexation par la clé primaire n’est pas possible. +Désactive l'exécution de la requête si l'indexation par la clé primaire n'est pas possible. Fonctionne avec les tables de la famille MergeTree. -Si `force_primary_key=1`, Clickhouse vérifie si la requête a une condition de clé primaire qui peut être utilisée pour restreindre les plages de données. S’il n’y a pas de condition appropriée, il lève une exception. Cependant, il ne vérifie pas si la condition réduit la quantité de données à lire. Pour plus d’informations sur les plages de données dans les tables MergeTree, voir [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md). +Si `force_primary_key=1`, Clickhouse vérifie si la requête a une condition de clé primaire qui peut être utilisée pour restreindre les plages de données. S'il n'y a pas de condition appropriée, il lève une exception. Cependant, il ne vérifie pas si la condition réduit la quantité de données à lire. Pour plus d'informations sur les plages de données dans les tables MergeTree, voir [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md). ## format\_schema {#format-schema} -Ce paramètre est utile lorsque vous utilisez des formats nécessitant une définition de schéma, tels que [Cap’n Proto](https://capnproto.org/) ou [Protobuf](https://developers.google.com/protocol-buffers/). La valeur dépend du format. +Ce paramètre est utile lorsque vous utilisez des formats nécessitant une définition de schéma, tels que [Cap'n Proto](https://capnproto.org/) ou [Protobuf](https://developers.google.com/protocol-buffers/). La valeur dépend du format. ## fsync\_metadata {#fsync-metadata} -Active ou désactive [fsync](http://pubs.opengroup.org/onlinepubs/9699919799/functions/fsync.html) lors de l’écriture `.sql` fichier. Activé par défaut. +Active ou désactive [fsync](http://pubs.opengroup.org/onlinepubs/9699919799/functions/fsync.html) lors de l'écriture `.sql` fichier. Activé par défaut. Il est logique de le désactiver si le serveur a des millions de tables minuscules qui sont constamment créées et détruites. @@ -91,7 +89,7 @@ Il est logique de le désactiver si le serveur a des millions de tables minuscul Active ou désactive la compression de données dans la réponse à une requête HTTP. -Pour plus d’informations, lire l’ [Description de L’interface HTTP](../../interfaces/http.md). +Pour plus d'informations, lire l' [Description de L'interface HTTP](../../interfaces/http.md). Valeurs possibles: @@ -112,7 +110,7 @@ Valeur par défaut: 3. Active ou désactive la vérification de la somme de contrôle lors de la décompression des données HTTP POST du client. Utilisé uniquement pour le format de compression natif ClickHouse (non utilisé avec `gzip` ou `deflate`). -Pour plus d’informations, lire l’ [Description de L’interface HTTP](../../interfaces/http.md). +Pour plus d'informations, lire l' [Description de L'interface HTTP](../../interfaces/http.md). Valeurs possibles: @@ -125,7 +123,7 @@ Valeur par défaut: 0. Active ou désactive `X-ClickHouse-Progress` - Têtes de réponse HTTP dans `clickhouse-server` réponse. -Pour plus d’informations, lire l’ [Description de L’interface HTTP](../../interfaces/http.md). +Pour plus d'informations, lire l' [Description de L'interface HTTP](../../interfaces/http.md). Valeurs possibles: @@ -136,7 +134,7 @@ Valeur par défaut: 0. ## max\_http\_get\_redirects {#setting-max_http_get_redirects} -Limite le nombre maximal de sauts de redirection HTTP GET pour [URL](../../engines/table-engines/special/url.md)-tables de moteur. Le paramètre s’applique aux deux types de tables: celles créées par [CREATE TABLE](../../query_language/create/#create-table-query) requête et par la [URL](../../sql-reference/table-functions/url.md) table de fonction. +Limite le nombre maximal de sauts de redirection HTTP GET pour [URL](../../engines/table-engines/special/url.md)-tables de moteur. Le paramètre s'applique aux deux types de tables: celles créées par [CREATE TABLE](../../sql-reference/statements/create.md#create-table-query) requête et par la [URL](../../sql-reference/table-functions/url.md) table de fonction. Valeurs possibles: @@ -147,32 +145,32 @@ Valeur par défaut: 0. ## input\_format\_allow\_errors\_num {#settings-input_format_allow_errors_num} -Définit le nombre maximal d’erreurs acceptables lors de la lecture à partir de formats de texte (CSV, TSV, etc.). +Définit le nombre maximal d'erreurs acceptables lors de la lecture à partir de formats de texte (CSV, TSV, etc.). La valeur par défaut est de 0. Toujours le coupler avec `input_format_allow_errors_ratio`. -Si une erreur s’est produite lors de la lecture de lignes mais que le compteur d’erreurs est toujours inférieur à `input_format_allow_errors_num`, ClickHouse ignore la ligne et passe à la suivante. +Si une erreur s'est produite lors de la lecture de lignes mais que le compteur d'erreurs est toujours inférieur à `input_format_allow_errors_num`, ClickHouse ignore la ligne et passe à la suivante. Si les deux `input_format_allow_errors_num` et `input_format_allow_errors_ratio` sont dépassés, ClickHouse lève une exception. ## input\_format\_allow\_errors\_ratio {#settings-input_format_allow_errors_ratio} -Définit le pourcentage maximal d’erreurs autorisées lors de la lecture à partir de formats de texte (CSV, TSV, etc.). -Le pourcentage d’erreurs est défini comme un nombre à virgule flottante compris entre 0 et 1. +Définit le pourcentage maximal d'erreurs autorisées lors de la lecture à partir de formats de texte (CSV, TSV, etc.). +Le pourcentage d'erreurs est défini comme un nombre à virgule flottante compris entre 0 et 1. La valeur par défaut est de 0. Toujours le coupler avec `input_format_allow_errors_num`. -Si une erreur s’est produite lors de la lecture de lignes mais que le compteur d’erreurs est toujours inférieur à `input_format_allow_errors_ratio`, ClickHouse ignore la ligne et passe à la suivante. +Si une erreur s'est produite lors de la lecture de lignes mais que le compteur d'erreurs est toujours inférieur à `input_format_allow_errors_ratio`, ClickHouse ignore la ligne et passe à la suivante. Si les deux `input_format_allow_errors_num` et `input_format_allow_errors_ratio` sont dépassés, ClickHouse lève une exception. ## input\_format\_values\_interpret\_expressions {#settings-input_format_values_interpret_expressions} -Active ou désactive L’analyseur SQL complet si l’analyseur de flux rapide ne peut pas analyser les données. Ce paramètre est utilisé uniquement pour la [Valeur](../../interfaces/formats.md#data-format-values) format lors de l’insertion des données. Pour plus d’informations sur l’analyse syntaxique, consultez [Syntaxe](../../sql-reference/syntax.md) section. +Active ou désactive L'analyseur SQL complet si l'analyseur de flux rapide ne peut pas analyser les données. Ce paramètre est utilisé uniquement pour la [Valeur](../../interfaces/formats.md#data-format-values) format lors de l'insertion des données. Pour plus d'informations sur l'analyse syntaxique, consultez [Syntaxe](../../sql-reference/syntax.md) section. Valeurs possibles: @@ -182,11 +180,11 @@ Valeurs possibles: - 1 — Enabled. - Dans ce cas, vous pouvez utiliser une expression SQL en tant que valeur, mais l’insertion de données est beaucoup plus lente de cette façon. Si vous insérez uniquement des données formatées, ClickHouse se comporte comme si la valeur de réglage était 0. + Dans ce cas, vous pouvez utiliser une expression SQL en tant que valeur, mais l'insertion de données est beaucoup plus lente de cette façon. Si vous insérez uniquement des données formatées, ClickHouse se comporte comme si la valeur de réglage était 0. Valeur par défaut: 1. -Exemple D’utilisation +Exemple D'utilisation Insérez le [DateTime](../../sql-reference/data-types/datetime.md) tapez valeur avec les différents paramètres. @@ -222,17 +220,24 @@ Ok. ## input\_format\_values\_deduce\_templates\_of\_expressions {#settings-input_format_values_deduce_templates_of_expressions} -Active ou désactive la déduction de modèle pour une expression SQL dans [Valeur](../../interfaces/formats.md#data-format-values) format. Il permet d’analyser et d’interpréter les expressions dans `Values` beaucoup plus rapide si les expressions dans des lignes consécutives ont la même structure. ClickHouse va essayer de déduire le modèle d’une expression, analyser les lignes suivantes en utilisant ce modèle et évaluer l’expression sur un lot de lignes analysées avec succès. Pour la requête suivante: +Active ou désactive la déduction de modèle pour les expressions SQL dans [Valeur](../../interfaces/formats.md#data-format-values) format. Il permet d'analyser et d'interpréter des expressions dans `Values` beaucoup plus rapide si les expressions dans des lignes consécutives ont la même structure. ClickHouse tente de déduire le modèle d'une expression, d'analyser les lignes suivantes à l'aide de ce modèle et d'évaluer l'expression sur un lot de lignes analysées avec succès. + +Valeurs possibles: + +- 0 — Disabled. +- 1 — Enabled. + +Valeur par défaut: 1. + +Pour la requête suivante: ``` sql INSERT INTO test VALUES (lower('Hello')), (lower('world')), (lower('INSERT')), (upper('Values')), ... ``` -- si `input_format_values_interpret_expressions=1` et `format_values_deduce_templates_of_expressions=0` les expressions seront interprétées séparément pour chaque ligne (Ceci est très lent pour un grand nombre de lignes) -- si `input_format_values_interpret_expressions=0` et `format_values_deduce_templates_of_expressions=1` expressions dans les première, deuxième et troisième lignes seront analysées à l’aide du modèle `lower(String)` et interprété ensemble, l’expression est la quatrième ligne sera analysée avec un autre modèle (`upper(String)`) -- si `input_format_values_interpret_expressions=1` et `format_values_deduce_templates_of_expressions=1` - la même chose que dans le cas précédent, mais permet également d’interpréter les expressions séparément s’il n’est pas possible de déduire le modèle. - -Activé par défaut. +- Si `input_format_values_interpret_expressions=1` et `format_values_deduce_templates_of_expressions=0`, les expressions sont interprétées séparément pour chaque ligne (c'est très lent pour un grand nombre de lignes). +- Si `input_format_values_interpret_expressions=0` et `format_values_deduce_templates_of_expressions=1`, les expressions des première, deuxième et troisième lignes sont analysées à l'aide de template `lower(String)` et interprété ensemble, l'expression dans la quatrième ligne est analysée avec un autre modèle (`upper(String)`). +- Si `input_format_values_interpret_expressions=1` et `format_values_deduce_templates_of_expressions=1`, le même que dans le cas précédent, mais permet également d'interpréter les expressions séparément s'il n'est pas possible de déduire le modèle. ## input\_format\_values\_accurate\_types\_of\_literals {#settings-input-format-values-accurate-types-of-literals} @@ -244,13 +249,21 @@ Ce paramètre est utilisé uniquement lorsque `input_format_values_deduce_templa (..., abs(-1), ...), -- Int64 literal ``` -Lorsque ce paramètre est activé, ClickHouse vérifie le type réel de littéral et utilise un modèle d’expression du type correspondant. Dans certains cas, cela peut considérablement ralentir l’évaluation de l’expression dans `Values`. -When disabled, ClickHouse may use more general type for some literals (e.g. `Float64` ou `Int64` plutôt `UInt64` pour `42`), mais cela peut causer des problèmes de débordement et de précision. -Activé par défaut. +Valeurs possibles: + +- 0 — Disabled. + + In this case, ClickHouse may use a more general type for some literals (e.g., `Float64` ou `Int64` plutôt `UInt64` pour `42`), mais cela peut causer des problèmes de débordement et de précision. + +- 1 — Enabled. + + Dans ce cas, ClickHouse vérifie le type réel de littéral et utilise un modèle d'expression du type correspondant. Dans certains cas, cela peut considérablement ralentir l'évaluation de l'expression dans `Values`. + +Valeur par défaut: 1. ## input\_format\_defaults\_for\_omitted\_fields {#session_settings-input_format_defaults_for_omitted_fields} -Lors de l’exécution de `INSERT` requêtes, remplacez les valeurs de colonne d’entrée omises par les valeurs par défaut des colonnes respectives. Cette option s’applique uniquement aux [JSONEachRow](../../interfaces/formats.md#jsoneachrow), [CSV](../../interfaces/formats.md#csv) et [TabSeparated](../../interfaces/formats.md#tabseparated) format. +Lors de l'exécution de `INSERT` requêtes, remplacez les valeurs de colonne d'entrée omises par les valeurs par défaut des colonnes respectives. Cette option s'applique uniquement aux [JSONEachRow](../../interfaces/formats.md#jsoneachrow), [CSV](../../interfaces/formats.md#csv) et [TabSeparated](../../interfaces/formats.md#tabseparated) format. !!! note "Note" Lorsque cette option est activée, les métadonnées de table étendues sont envoyées du serveur au client. Il consomme des ressources informatiques supplémentaires sur le serveur et peut réduire les performances. @@ -270,13 +283,13 @@ Désactivé par défaut. ## input\_format\_null\_as\_default {#settings-input-format-null-as-default} -Active ou désactive l’utilisation des valeurs par défaut si les données `NULL` mais le type de données de la colonne correspondante dans pas `Nullable(T)` (pour les formats de saisie de texte). +Active ou désactive l'utilisation des valeurs par défaut si les données `NULL` mais le type de données de la colonne correspondante dans pas `Nullable(T)` (pour les formats de saisie de texte). ## input\_format\_skip\_unknown\_fields {#settings-input-format-skip-unknown-fields} -Active ou désactive le saut d’insertion de données supplémentaires. +Active ou désactive le saut d'insertion de données supplémentaires. -Lors de l’écriture de données, ClickHouse lève une exception si les données d’entrée contiennent des colonnes qui n’existent pas dans la table cible. Si le saut est activé, ClickHouse n’insère pas de données supplémentaires et ne lance pas d’exception. +Lors de l'écriture de données, ClickHouse lève une exception si les données d'entrée contiennent des colonnes qui n'existent pas dans la table cible. Si le saut est activé, ClickHouse n'insère pas de données supplémentaires et ne lance pas d'exception. Formats pris en charge: @@ -294,7 +307,7 @@ Valeur par défaut: 0. ## input\_format\_import\_nested\_json {#settings-input_format_import_nested_json} -Active ou désactive l’insertion de données JSON avec des objets imbriqués. +Active ou désactive l'insertion de données JSON avec des objets imbriqués. Formats pris en charge: @@ -309,13 +322,13 @@ Valeur par défaut: 0. Voir aussi: -- [Utilisation de Structures imbriquées](../../interfaces/formats.md#jsoneachrow-nested) avec l’ `JSONEachRow` format. +- [Utilisation de Structures imbriquées](../../interfaces/formats.md#jsoneachrow-nested) avec l' `JSONEachRow` format. ## input\_format\_with\_names\_use\_header {#settings-input-format-with-names-use-header} -Active ou désactive la vérification de l’ordre des colonnes lors de l’insertion de données. +Active ou désactive la vérification de l'ordre des colonnes lors de l'insertion de données. -Pour améliorer les performances d’insertion, nous vous recommandons de désactiver cette vérification si vous êtes sûr que l’ordre des colonnes des données d’entrée est le même que dans la table cible. +Pour améliorer les performances d'insertion, nous vous recommandons de désactiver cette vérification si vous êtes sûr que l'ordre des colonnes des données d'entrée est le même que dans la table cible. Formats pris en charge: @@ -331,9 +344,9 @@ Valeur par défaut: 1. ## date\_time\_input\_format {#settings-date_time_input_format} -Permet de choisir un analyseur de la représentation textuelle de la date et de l’heure. +Permet de choisir un analyseur de la représentation textuelle de la date et de l'heure. -Le réglage ne s’applique pas à [fonctions date et heure](../../sql-reference/functions/date-time-functions.md). +Le réglage ne s'applique pas à [fonctions date et heure](../../sql-reference/functions/date-time-functions.md). Valeurs possibles: @@ -354,14 +367,14 @@ Voir aussi: ## join\_default\_strictness {#settings-join_default_strictness} -Définit la rigueur par défaut pour [JOIN clauses](../../sql-reference/statements/select.md#select-join). +Définit la rigueur par défaut pour [JOIN clauses](../../sql-reference/statements/select/join.md#select-join). Valeurs possibles: -- `ALL` — If the right table has several matching rows, ClickHouse creates a [Produit cartésien](https://en.wikipedia.org/wiki/Cartesian_product) à partir des lignes correspondantes. C’est normal `JOIN` comportement de SQL standard. +- `ALL` — If the right table has several matching rows, ClickHouse creates a [Produit cartésien](https://en.wikipedia.org/wiki/Cartesian_product) à partir des lignes correspondantes. C'est normal `JOIN` comportement de SQL standard. - `ANY` — If the right table has several matching rows, only the first one found is joined. If the right table has only one matching row, the results of `ANY` et `ALL` sont les mêmes. - `ASOF` — For joining sequences with an uncertain match. -- `Empty string` — If `ALL` ou `ANY` n’est pas spécifié dans la requête, ClickHouse lève une exception. +- `Empty string` — If `ALL` ou `ANY` n'est pas spécifié dans la requête, ClickHouse lève une exception. Valeur par défaut: `ALL`. @@ -370,7 +383,7 @@ Valeur par défaut: `ALL`. Modifie le comportement des opérations de jointure avec `ANY` rigueur. !!! warning "Attention" - Ce paramètre s’applique uniquement pour `JOIN` opérations avec [Rejoindre](../../engines/table-engines/special/join.md) le moteur de tables. + Ce paramètre s'applique uniquement pour `JOIN` opérations avec [Rejoindre](../../engines/table-engines/special/join.md) le moteur de tables. Valeurs possibles: @@ -381,13 +394,13 @@ Valeur par défaut: 0. Voir aussi: -- [Clause de JOINTURE](../../sql-reference/statements/select.md#select-join) +- [Clause de JOINTURE](../../sql-reference/statements/select/join.md#select-join) - [Rejoindre le moteur de table](../../engines/table-engines/special/join.md) - [join\_default\_strictness](#settings-join_default_strictness) ## join\_use\_nulls {#join_use_nulls} -Définit le type de [JOIN](../../sql-reference/statements/select.md) comportement. Lors de la fusion de tables, des cellules vides peuvent apparaître. ClickHouse les remplit différemment en fonction de ce paramètre. +Définit le type de [JOIN](../../sql-reference/statements/select/join.md) comportement. Lors de la fusion de tables, des cellules vides peuvent apparaître. ClickHouse les remplit différemment en fonction de ce paramètre. Valeurs possibles: @@ -398,7 +411,7 @@ Valeur par défaut: 0. ## max\_block\_size {#setting-max_block_size} -Dans ClickHouse, les données sont traitées par Blocs (Ensembles de parties de colonne). Les cycles de traitement internes pour un seul bloc sont assez efficaces, mais il y a des dépenses notables sur chaque bloc. Le `max_block_size` le paramètre est une recommandation pour la taille du bloc (dans un nombre de lignes) à charger à partir des tables. La taille du bloc ne doit pas être trop petite, de sorte que les dépenses sur chaque bloc sont toujours perceptibles, mais pas trop grande pour que la requête avec limite qui est terminée après le premier bloc soit traitée rapidement. L’objectif est d’éviter de consommer trop de mémoire lors de l’extraction d’un grand nombre de colonnes dans plusieurs threads et de préserver au moins certains localité de cache. +Dans ClickHouse, les données sont traitées par Blocs (Ensembles de parties de colonne). Les cycles de traitement internes pour un seul bloc sont assez efficaces, mais il y a des dépenses notables sur chaque bloc. Le `max_block_size` le paramètre est une recommandation pour la taille du bloc (dans un nombre de lignes) à charger à partir des tables. La taille du bloc ne doit pas être trop petite, de sorte que les dépenses sur chaque bloc sont toujours perceptibles, mais pas trop grande pour que la requête avec limite qui est terminée après le premier bloc soit traitée rapidement. L'objectif est d'éviter de consommer trop de mémoire lors de l'extraction d'un grand nombre de colonnes dans plusieurs threads et de préserver au moins certains localité de cache. Valeur par défaut: de 65 536. @@ -406,13 +419,13 @@ Les blocs de la taille de `max_block_size` ne sont pas toujours chargées de la ## preferred\_block\_size\_bytes {#preferred-block-size-bytes} -Utilisé dans le même but que `max_block_size`, mais il définit la taille de bloc recommandée en octets en l’adaptant au nombre de lignes dans le bloc. +Utilisé dans le même but que `max_block_size`, mais il définit la taille de bloc recommandée en octets en l'adaptant au nombre de lignes dans le bloc. Cependant, la taille du bloc ne peut pas être supérieure à `max_block_size` rangée. Par défaut: 1 000 000. Cela ne fonctionne que lors de la lecture des moteurs MergeTree. ## merge\_tree\_min\_rows\_for\_concurrent\_read {#setting-merge-tree-min-rows-for-concurrent-read} -Si le nombre de lignes à lire à partir d’un fichier d’un [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) table dépasse `merge_tree_min_rows_for_concurrent_read` ensuite, ClickHouse essaie d’effectuer une lecture simultanée de ce fichier sur plusieurs threads. +Si le nombre de lignes à lire à partir d'un fichier d'un [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) table dépasse `merge_tree_min_rows_for_concurrent_read` ensuite, ClickHouse essaie d'effectuer une lecture simultanée de ce fichier sur plusieurs threads. Valeurs possibles: @@ -422,7 +435,7 @@ Valeur par défaut: 163840. ## merge\_tree\_min\_bytes\_for\_concurrent\_read {#setting-merge-tree-min-bytes-for-concurrent-read} -Si le nombre d’octets à lire à partir d’un fichier d’un [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md)- table de moteur dépasse `merge_tree_min_bytes_for_concurrent_read` puis ClickHouse essaie de lire simultanément à partir de ce fichier dans plusieurs threads. +Si le nombre d'octets à lire à partir d'un fichier d'un [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md)- table de moteur dépasse `merge_tree_min_bytes_for_concurrent_read` puis ClickHouse essaie de lire simultanément à partir de ce fichier dans plusieurs threads. Valeur Possible: @@ -452,7 +465,7 @@ Valeur par défaut: 0. ## merge\_tree\_coarse\_index\_granularité {#setting-merge-tree-coarse-index-granularity} -Lors de la recherche de données, ClickHouse vérifie les marques de données dans le fichier d’index. Si ClickHouse trouve que les clés requises sont dans une certaine plage, il divise cette plage en `merge_tree_coarse_index_granularity` subranges et recherche les clés requises récursivement. +Lors de la recherche de données, ClickHouse vérifie les marques de données dans le fichier d'index. Si ClickHouse trouve que les clés requises sont dans une certaine plage, il divise cette plage en `merge_tree_coarse_index_granularity` subranges et recherche les clés requises récursivement. Valeurs possibles: @@ -462,7 +475,7 @@ Valeur par défaut: 8. ## merge\_tree\_max\_rows\_to\_use\_cache {#setting-merge-tree-max-rows-to-use-cache} -Si ClickHouse devrait lire plus de `merge_tree_max_rows_to_use_cache` lignes dans une requête, il n’utilise pas le cache des blocs non compressés. +Si ClickHouse devrait lire plus de `merge_tree_max_rows_to_use_cache` lignes dans une requête, il n'utilise pas le cache des blocs non compressés. Le cache des blocs non compressés stocke les données extraites pour les requêtes. ClickHouse utilise ce cache pour accélérer les réponses aux petites requêtes répétées. Ce paramètre protège le cache contre le saccage par les requêtes qui lisent une grande quantité de données. Le [uncompressed\_cache\_size](../server-configuration-parameters/settings.md#server-settings-uncompressed_cache_size) le paramètre serveur définit la taille du cache des blocs non compressés. @@ -474,7 +487,7 @@ Default value: 128 ✕ 8192. ## merge\_tree\_max\_bytes\_to\_use\_cache {#setting-merge-tree-max-bytes-to-use-cache} -Si ClickHouse devrait lire plus de `merge_tree_max_bytes_to_use_cache` octets dans une requête, il n’utilise pas le cache de non compressé blocs. +Si ClickHouse devrait lire plus de `merge_tree_max_bytes_to_use_cache` octets dans une requête, il n'utilise pas le cache de non compressé blocs. Le cache des blocs non compressés stocke les données extraites pour les requêtes. ClickHouse utilise ce cache pour accélérer les réponses aux petites requêtes répétées. Ce paramètre protège le cache contre le saccage par les requêtes qui lisent une grande quantité de données. Le [uncompressed\_cache\_size](../server-configuration-parameters/settings.md#server-settings-uncompressed_cache_size) le paramètre serveur définit la taille du cache des blocs non compressés. @@ -486,7 +499,7 @@ Valeur par défaut: 2013265920. ## min\_bytes\_to\_use\_direct\_io {#settings-min-bytes-to-use-direct-io} -Volume de données minimum requis pour utiliser l’accès direct aux E/S sur le disque de stockage. +Volume de données minimum requis pour utiliser l'accès direct aux E/S sur le disque de stockage. ClickHouse utilise ce paramètre lors de la lecture de données à partir de tables. Si le volume total de stockage de toutes les données à lire dépasse `min_bytes_to_use_direct_io` octets, puis ClickHouse lit les données du disque de stockage avec le `O_DIRECT` option. @@ -509,6 +522,24 @@ Exemple: log_queries=1 ``` +## log\_queries\_min\_type {#settings-log-queries-min-type} + +`query_log` type minimal à enregistrer. + +Valeurs possibles: +- `QUERY_START` (`=1`) +- `QUERY_FINISH` (`=2`) +- `EXCEPTION_BEFORE_START` (`=3`) +- `EXCEPTION_WHILE_PROCESSING` (`=4`) + +Valeur par défaut: `QUERY_START`. + +Peut être utilisé pour limiter le nombre de entiries va va `query_log`, dites que vous êtes intéressant que dans les erreurs, alors vous pouvez utiliser `EXCEPTION_WHILE_PROCESSING`: + +``` text +log_queries_min_type='EXCEPTION_WHILE_PROCESSING' +``` + ## log\_query\_threads {#settings-log-query-threads} Configuration de la journalisation des threads de requête. @@ -523,38 +554,60 @@ log_query_threads=1 ## max\_insert\_block\_size {#settings-max_insert_block_size} -La taille des blocs à former pour l’insertion dans une table. -Ce paramètre s’applique uniquement dans les cas où le serveur formes les blocs. -Par exemple, pour une insertion via L’interface HTTP, le serveur analyse le format de données et forme des blocs de la taille spécifiée. -Mais lors de l’utilisation de clickhouse-client, le client analyse les données ‘max\_insert\_block\_size’ le réglage sur le serveur n’affecte pas la taille des blocs insérés. -Le paramètre n’a pas non plus de but lors de L’utilisation D’INSERT SELECT, car les données sont insérées à l’aide des mêmes blocs qui sont formés après SELECT. +La taille des blocs à former pour l'insertion dans une table. +Ce paramètre s'applique uniquement dans les cas où le serveur formes les blocs. +Par exemple, pour une insertion via L'interface HTTP, le serveur analyse le format de données et forme des blocs de la taille spécifiée. +Mais lors de l'utilisation de clickhouse-client, le client analyse les données ‘max\_insert\_block\_size’ le réglage sur le serveur n'affecte pas la taille des blocs insérés. +Le paramètre n'a pas non plus de but lors de L'utilisation D'INSERT SELECT, car les données sont insérées à l'aide des mêmes blocs qui sont formés après SELECT. Valeur par défaut: 1 048 576 octets. -La valeur par défaut est légèrement supérieure à `max_block_size`. La raison en est que certains moteurs de table (`*MergeTree`) former une partie de données sur le disque pour chaque bloc inséré, qui est une entité assez grande. Pareillement, `*MergeTree` les tables trient les données lors de l’insertion et une taille de bloc suffisamment grande permet de trier plus de données dans la RAM. +La valeur par défaut est légèrement supérieure à `max_block_size`. La raison en est que certains moteurs de table (`*MergeTree`) former une partie de données sur le disque pour chaque bloc inséré, qui est une entité assez grande. Pareillement, `*MergeTree` les tables trient les données lors de l'insertion et une taille de bloc suffisamment grande permet de trier plus de données dans la RAM. + +## min\_insert\_block\_size\_rows {#min-insert-block-size-rows} + +Définit le nombre minimum de lignes dans le bloc qui peut être inséré dans un tableau par un `INSERT` requête. Les blocs de plus petite taille sont écrasés en plus gros. + +Valeurs possibles: + +- Entier positif. +- 0 — Squashing disabled. + +Valeur par défaut: 1048576. + +## min\_insert\_block\_size\_bytes {#min-insert-block-size-bytes} + +Définit le nombre minimal d'octets dans le bloc qui peut être inséré dans un tableau par un `INSERT` requête. Les blocs de plus petite taille sont écrasés en plus gros. + +Valeurs possibles: + +- Entier positif. +- 0 — Squashing disabled. + +Valeur par défaut: 268435456. ## max\_replica\_delay\_for\_distributed\_queries {#settings-max_replica_delay_for_distributed_queries} Désactive les répliques en retard pour les requêtes distribuées. Voir [Réplication](../../engines/table-engines/mergetree-family/replication.md). -Définit le temps en secondes. Si une réplique accuse plus de retard que la valeur définie, cette réplique n’est pas utilisée. +Définit le temps en secondes. Si une réplique accuse plus de retard que la valeur définie, cette réplique n'est pas utilisée. Valeur par défaut: 300. -Utilisé lors de l’exécution `SELECT` à partir d’une table distribuée qui pointe vers des tables répliquées. +Utilisé lors de l'exécution `SELECT` à partir d'une table distribuée qui pointe vers des tables répliquées. ## max\_threads {#settings-max_threads} -Nombre maximal de threads de traitement des requêtes, à l’exclusion des threads de récupération de données à partir de serveurs distants (voir ‘max\_distributed\_connections’ paramètre). +Nombre maximal de threads de traitement des requêtes, à l'exclusion des threads de récupération de données à partir de serveurs distants (voir ‘max\_distributed\_connections’ paramètre). -Ce paramètre s’applique aux threads qui effectuent les mêmes étapes du pipeline de traitement des requêtes en parallèle. -Par exemple, lors de la lecture d’une table, s’il est possible d’évaluer des expressions avec des fonctions, filtrer avec WHERE et pré-agréger pour GROUP BY en parallèle en utilisant au moins ‘max\_threads’ nombre de threads, puis ‘max\_threads’ sont utilisés. +Ce paramètre s'applique aux threads qui effectuent les mêmes étapes du pipeline de traitement des requêtes en parallèle. +Par exemple, lors de la lecture d'une table, s'il est possible d'évaluer des expressions avec des fonctions, filtrer avec WHERE et pré-agréger pour GROUP BY en parallèle en utilisant au moins ‘max\_threads’ nombre de threads, puis ‘max\_threads’ sont utilisés. Valeur par défaut: nombre de cœurs de processeur physiques. -Si moins d’une requête SELECT est normalement exécutée sur un serveur à la fois, définissez ce paramètre sur une valeur légèrement inférieure au nombre réel de cœurs de processeur. +Si moins d'une requête SELECT est normalement exécutée sur un serveur à la fois, définissez ce paramètre sur une valeur légèrement inférieure au nombre réel de cœurs de processeur. -Pour les requêtes qui sont terminées rapidement en raison d’une limite, vous pouvez définir une valeur inférieure ‘max\_threads’. Par exemple, si le nombre d’entrées se trouvent dans chaque bloc et max\_threads = 8, 8 blocs sont récupérées, même s’il aurait été suffisante pour lire un seul. +Pour les requêtes qui sont terminées rapidement en raison d'une limite, vous pouvez définir une valeur inférieure ‘max\_threads’. Par exemple, si le nombre d'entrées se trouvent dans chaque bloc et max\_threads = 8, 8 blocs sont récupérées, même s'il aurait été suffisante pour lire un seul. Le plus petit de la `max_threads` valeur, moins la mémoire est consommée. @@ -564,50 +617,50 @@ Nombre maximal de threads à exécuter `INSERT SELECT` requête. Valeurs possibles: -- 0 (or 1) — `INSERT SELECT` pas d’exécution parallèle. +- 0 (or 1) — `INSERT SELECT` pas d'exécution parallèle. - Entier positif. Plus grand que 1. Valeur par défaut: 0. -Parallèle `INSERT SELECT` n’a d’effet que si l’ `SELECT` une partie est exécutée en parallèle, voir [max\_threads](#settings-max_threads) paramètre. +Parallèle `INSERT SELECT` n'a d'effet que si l' `SELECT` une partie est exécutée en parallèle, voir [max\_threads](#settings-max_threads) paramètre. Des valeurs plus élevées conduiront à une utilisation de la mémoire plus élevée. ## max\_compress\_block\_size {#max-compress-block-size} -La taille maximale des blocs de données non compressées avant la compression pour l’écriture dans une table. Par défaut, 1 048 576 (1 MiB). Si la taille est réduite, le taux de compression est considérablement réduit, la vitesse de compression et de décompression augmente légèrement en raison de la localisation du cache, et la consommation de mémoire est réduite. Il n’y aucune raison de modifier ce paramètre. +La taille maximale des blocs de données non compressées avant la compression pour l'écriture dans une table. Par défaut, 1 048 576 (1 MiB). Si la taille est réduite, le taux de compression est considérablement réduit, la vitesse de compression et de décompression augmente légèrement en raison de la localisation du cache, et la consommation de mémoire est réduite. Il n'y aucune raison de modifier ce paramètre. -Ne confondez pas les blocs pour la compression (un morceau de mémoire constitué d’octets) avec des blocs pour le traitement des requêtes (Un ensemble de lignes d’une table). +Ne confondez pas les blocs pour la compression (un morceau de mémoire constitué d'octets) avec des blocs pour le traitement des requêtes (Un ensemble de lignes d'une table). ## min\_compress\_block\_size {#min-compress-block-size} -Pour [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md)" table. Afin de réduire la latence lors du traitement des requêtes, un bloc est compressé lors de l’écriture de la marque suivante si sa taille est au moins ‘min\_compress\_block\_size’. Par défaut, 65 536. +Pour [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md)" table. Afin de réduire la latence lors du traitement des requêtes, un bloc est compressé lors de l'écriture de la marque suivante si sa taille est au moins ‘min\_compress\_block\_size’. Par défaut, 65 536. La taille réelle du bloc, si les données non compressées sont inférieures à ‘max\_compress\_block\_size’ pas moins de cette valeur et pas moins que le volume de données pour une marque. Regardons un exemple. Supposons que ‘index\_granularity’ a 8192 lors de la création de la table. -Nous écrivons une colonne de type UInt32 (4 octets par valeur). Lors de l’écriture de 8192 lignes, le total sera de 32 KO de données. Puisque min\_compress\_block\_size = 65 536, un bloc compressé sera formé pour toutes les deux marques. +Nous écrivons une colonne de type UInt32 (4 octets par valeur). Lors de l'écriture de 8192 lignes, le total sera de 32 KO de données. Puisque min\_compress\_block\_size = 65 536, un bloc compressé sera formé pour toutes les deux marques. -Nous écrivons une colonne URL avec le type de chaîne (taille moyenne de 60 octets par valeur). Lors de l’écriture de 8192 lignes, la moyenne sera légèrement inférieure à 500 Ko de données. Comme il s’agit de plus de 65 536, un bloc compressé sera formé pour chaque marque. Dans ce cas, lors de la lecture de données du disque dans la plage d’une seule marque, les données supplémentaires ne seront pas décompressées. +Nous écrivons une colonne URL avec le type de chaîne (taille moyenne de 60 octets par valeur). Lors de l'écriture de 8192 lignes, la moyenne sera légèrement inférieure à 500 Ko de données. Comme il s'agit de plus de 65 536, un bloc compressé sera formé pour chaque marque. Dans ce cas, lors de la lecture de données du disque dans la plage d'une seule marque, les données supplémentaires ne seront pas décompressées. -Il n’y aucune raison de modifier ce paramètre. +Il n'y aucune raison de modifier ce paramètre. ## max\_query\_size {#settings-max_query_size} -La partie maximale d’une requête qui peut être prise en RAM pour l’analyse avec L’analyseur SQL. -La requête INSERT contient également des données pour INSERT qui sont traitées par un analyseur de flux séparé (qui consomme O (1) RAM), qui n’est pas inclus dans cette restriction. +La partie maximale d'une requête qui peut être prise en RAM pour l'analyse avec L'analyseur SQL. +La requête INSERT contient également des données pour INSERT qui sont traitées par un analyseur de flux séparé (qui consomme O (1) RAM), qui n'est pas inclus dans cette restriction. Valeur par défaut: 256 Ko. ## interactive\_delay {#interactive-delay} -Intervalle en microsecondes pour vérifier si l’exécution de la requête a été annulée et envoyer la progression. +Intervalle en microsecondes pour vérifier si l'exécution de la requête a été annulée et envoyer la progression. -Valeur par défaut: 100 000 (vérifie l’Annulation et envoie la progression dix fois par seconde). +Valeur par défaut: 100 000 (vérifie l'Annulation et envoie la progression dix fois par seconde). ## connect\_timeout, receive\_timeout, send\_timeout {#connect-timeout-receive-timeout-send-timeout} -Délais d’attente en secondes sur le socket utilisé pour communiquer avec le client. +Délais d'attente en secondes sur le socket utilisé pour communiquer avec le client. Valeur par défaut: 10, 300, 300. @@ -619,17 +672,17 @@ Valeur par défaut: 0 ## poll\_interval {#poll-interval} -Verrouillez une boucle d’attente pendant le nombre de secondes spécifié. +Verrouillez une boucle d'attente pendant le nombre de secondes spécifié. Valeur par défaut: 10. ## max\_distributed\_connections {#max-distributed-connections} -Nombre maximal de connexions simultanées avec des serveurs distants pour le traitement distribué d’une seule requête vers une seule table distribuée. Nous vous recommandons de définir une valeur au moins égale au nombre de serveurs dans le cluster. +Nombre maximal de connexions simultanées avec des serveurs distants pour le traitement distribué d'une seule requête vers une seule table distribuée. Nous vous recommandons de définir une valeur au moins égale au nombre de serveurs dans le cluster. Valeur par défaut: 1024. -Les paramètres suivants ne sont utilisés que lors de la création de tables distribuées (et lors du lancement d’un serveur), il n’y a donc aucune raison de les modifier lors de l’exécution. +Les paramètres suivants ne sont utilisés que lors de la création de tables distribuées (et lors du lancement d'un serveur), il n'y a donc aucune raison de les modifier lors de l'exécution. ## distributed\_connections\_pool\_size {#distributed-connections-pool-size} @@ -639,8 +692,8 @@ Valeur par défaut: 1024. ## connect\_timeout\_with\_failover\_ms {#connect-timeout-with-failover-ms} -Délai d’attente en millisecondes pour la connexion à un serveur distant pour un moteur de table distribué, si ‘shard’ et ‘replica’ les sections sont utilisées dans la définition du cluster. -En cas d’échec, plusieurs tentatives sont faites pour se connecter à diverses répliques. +Délai d'attente en millisecondes pour la connexion à un serveur distant pour un moteur de table distribué, si ‘shard’ et ‘replica’ les sections sont utilisées dans la définition du cluster. +En cas d'échec, plusieurs tentatives sont faites pour se connecter à diverses répliques. Valeur par défaut: 50. @@ -652,30 +705,30 @@ Valeur par défaut: 3. ## extrême {#extremes} -Indique s’il faut compter les valeurs extrêmes (les minimums et les maximums dans les colonnes d’un résultat de requête). Accepte 0 ou 1. Par défaut, 0 (désactivé). -Pour plus d’informations, consultez la section “Extreme values”. +Indique s'il faut compter les valeurs extrêmes (les minimums et les maximums dans les colonnes d'un résultat de requête). Accepte 0 ou 1. Par défaut, 0 (désactivé). +Pour plus d'informations, consultez la section “Extreme values”. ## use\_uncompressed\_cache {#setting-use_uncompressed_cache} -Indique s’il faut utiliser un cache de blocs non compressés. Accepte 0 ou 1. Par défaut, 0 (désactivé). -L’utilisation du cache non compressé (uniquement pour les tables de la famille MergeTree) peut réduire considérablement la latence et augmenter le débit lorsque vous travaillez avec un grand nombre de requêtes courtes. Activez ce paramètre pour les utilisateurs qui envoient des requêtes courtes fréquentes. Faites également attention à la [uncompressed\_cache\_size](../server-configuration-parameters/settings.md#server-settings-uncompressed_cache_size) configuration parameter (only set in the config file) – the size of uncompressed cache blocks. By default, it is 8 GiB. The uncompressed cache is filled in as needed and the least-used data is automatically deleted. +Indique s'il faut utiliser un cache de blocs non compressés. Accepte 0 ou 1. Par défaut, 0 (désactivé). +L'utilisation du cache non compressé (uniquement pour les tables de la famille MergeTree) peut réduire considérablement la latence et augmenter le débit lorsque vous travaillez avec un grand nombre de requêtes courtes. Activez ce paramètre pour les utilisateurs qui envoient des requêtes courtes fréquentes. Faites également attention à la [uncompressed\_cache\_size](../server-configuration-parameters/settings.md#server-settings-uncompressed_cache_size) configuration parameter (only set in the config file) – the size of uncompressed cache blocks. By default, it is 8 GiB. The uncompressed cache is filled in as needed and the least-used data is automatically deleted. -Pour les requêtes qui lisent au moins un volume de données assez important (un million de lignes ou plus), le cache non compressé est désactivé automatiquement pour économiser de l’espace pour les requêtes vraiment petites. Cela signifie que vous pouvez garder la ‘use\_uncompressed\_cache’ toujours la valeur 1. +Pour les requêtes qui lisent au moins un volume de données assez important (un million de lignes ou plus), le cache non compressé est désactivé automatiquement pour économiser de l'espace pour les requêtes vraiment petites. Cela signifie que vous pouvez garder la ‘use\_uncompressed\_cache’ toujours la valeur 1. ## replace\_running\_query {#replace-running-query} -Lors de l’utilisation de L’interface HTTP, le ‘query\_id’ le paramètre peut être passé. C’est n’importe quelle chaîne qui sert d’Identificateur de requête. -Si une requête d’un utilisateur avec le même ‘query\_id’ il existe déjà à ce moment, le comportement dépend de la ‘replace\_running\_query’ paramètre. +Lors de l'utilisation de L'interface HTTP, le ‘query\_id’ le paramètre peut être passé. C'est n'importe quelle chaîne qui sert d'Identificateur de requête. +Si une requête d'un utilisateur avec le même ‘query\_id’ il existe déjà à ce moment, le comportement dépend de la ‘replace\_running\_query’ paramètre. -`0` (default) – Throw an exception (don’t allow the query to run if a query with the same ‘query\_id’ est déjà en cours d’exécution). +`0` (default) – Throw an exception (don't allow the query to run if a query with the same ‘query\_id’ est déjà en cours d'exécution). `1` – Cancel the old query and start running the new one. -Yandex.Metrica utilise ce paramètre défini sur 1 pour implémenter des suggestions de conditions de segmentation. Après avoir entré le caractère suivant, si l’ancienne requête n’est pas encore terminée, elle doit être annulée. +Yandex.Metrica utilise ce paramètre défini sur 1 pour implémenter des suggestions de conditions de segmentation. Après avoir entré le caractère suivant, si l'ancienne requête n'est pas encore terminée, elle doit être annulée. ## stream\_flush\_interval\_ms {#stream-flush-interval-ms} -Fonctionne pour les tables avec des flux dans le cas d’une expiration, ou lorsqu’un thread génère [max\_insert\_block\_size](#settings-max_insert_block_size) rangée. +Fonctionne pour les tables avec des flux dans le cas d'une expiration, ou lorsqu'un thread génère [max\_insert\_block\_size](#settings-max_insert_block_size) rangée. La valeur par défaut est 7500. @@ -683,12 +736,12 @@ Plus la valeur est petite, plus les données sont vidées dans la table. Régler ## équilibrage {#settings-load_balancing} -Spécifie l’algorithme de sélection des réplicas utilisé pour le traitement des requêtes distribuées. +Spécifie l'algorithme de sélection des réplicas utilisé pour le traitement des requêtes distribuées. ClickHouse prend en charge les algorithmes suivants de choix des répliques: - [Aléatoire](#load_balancing-random) (par défaut) -- [Nom d’hôte le plus proche](#load_balancing-nearest_hostname) +- [Nom d'hôte le plus proche](#load_balancing-nearest_hostname) - [Afin](#load_balancing-in_order) - [Premier ou aléatoire](#load_balancing-first_or_random) @@ -698,22 +751,22 @@ ClickHouse prend en charge les algorithmes suivants de choix des répliques: load_balancing = random ``` -Le nombre d’erreurs est compté pour chaque réplique. La requête est envoyée au réplica avec le moins d’erreurs, et s’il y en a plusieurs, à n’importe qui d’entre eux. -Inconvénients: la proximité du serveur n’est pas prise en compte; si les répliques ont des données différentes, vous obtiendrez également des données différentes. +Le nombre d'erreurs est compté pour chaque réplique. La requête est envoyée au réplica avec le moins d'erreurs, et s'il y en a plusieurs, à n'importe qui d'entre eux. +Inconvénients: la proximité du serveur n'est pas prise en compte; si les répliques ont des données différentes, vous obtiendrez également des données différentes. -### Nom D’Hôte Le Plus Proche {#load_balancing-nearest_hostname} +### Nom D'Hôte Le Plus Proche {#load_balancing-nearest_hostname} ``` sql load_balancing = nearest_hostname ``` -The number of errors is counted for each replica. Every 5 minutes, the number of errors is integrally divided by 2. Thus, the number of errors is calculated for a recent time with exponential smoothing. If there is one replica with a minimal number of errors (i.e. errors occurred recently on the other replicas), the query is sent to it. If there are multiple replicas with the same minimal number of errors, the query is sent to the replica with a hostname that is most similar to the server’s hostname in the config file (for the number of different characters in identical positions, up to the minimum length of both hostnames). +The number of errors is counted for each replica. Every 5 minutes, the number of errors is integrally divided by 2. Thus, the number of errors is calculated for a recent time with exponential smoothing. If there is one replica with a minimal number of errors (i.e. errors occurred recently on the other replicas), the query is sent to it. If there are multiple replicas with the same minimal number of errors, the query is sent to the replica with a hostname that is most similar to the server's hostname in the config file (for the number of different characters in identical positions, up to the minimum length of both hostnames). -Par exemple, exemple01-01-1 et example01-01-2.yandex.ru sont différents dans une position, tandis que l’exemple01-01-1 et l’exemple01-02-2 diffèrent dans deux endroits. +Par exemple, exemple01-01-1 et example01-01-2.yandex.ru sont différents dans une position, tandis que l'exemple01-01-1 et l'exemple01-02-2 diffèrent dans deux endroits. Cette méthode peut sembler primitive, mais elle ne nécessite pas de données externes sur la topologie du réseau, et elle ne compare pas les adresses IP, ce qui serait compliqué pour nos adresses IPv6. -Ainsi, s’il existe des répliques équivalentes, la plus proche par son nom est préférée. -Nous pouvons également supposer que lors de l’envoi d’une requête au même serveur, en l’absence d’Échecs, une requête distribuée ira également aux mêmes serveurs. Ainsi, même si des données différentes sont placées sur les répliques, la requête retournera principalement les mêmes résultats. +Ainsi, s'il existe des répliques équivalentes, la plus proche par son nom est préférée. +Nous pouvons également supposer que lors de l'envoi d'une requête au même serveur, en l'absence d'Échecs, une requête distribuée ira également aux mêmes serveurs. Ainsi, même si des données différentes sont placées sur les répliques, la requête retournera principalement les mêmes résultats. ### Afin {#load_balancing-in_order} @@ -721,22 +774,22 @@ Nous pouvons également supposer que lors de l’envoi d’une requête au même load_balancing = in_order ``` -Répliques avec le même nombre d’erreurs sont accessibles dans le même ordre qu’ils sont définis dans la configuration. +Répliques avec le même nombre d'erreurs sont accessibles dans le même ordre qu'ils sont définis dans la configuration. Cette méthode est appropriée lorsque vous savez exactement quelle réplique est préférable. -### Premier Ou aléatoire {#load_balancing-first_or_random} +### Premier ou aléatoire {#load_balancing-first_or_random} ``` sql load_balancing = first_or_random ``` -Cet algorithme choisit la première réplique de l’ensemble ou une réplique aléatoire si la première n’est pas disponible. Il est efficace dans les configurations de topologie de réplication croisée, mais inutile dans d’autres configurations. +Cet algorithme choisit la première réplique de l'ensemble ou une réplique aléatoire si la première n'est pas disponible. Il est efficace dans les configurations de topologie de réplication croisée, mais inutile dans d'autres configurations. -Le `first_or_random` algorithme résout le problème de la `in_order` algorithme. Avec `in_order`, si une réplique tombe en panne, la suivante obtient une double charge tandis que les répliques restantes gèrent la quantité habituelle de trafic. Lors de l’utilisation de la `first_or_random` algorithme, la charge est répartie uniformément entre les répliques qui sont encore disponibles. +Le `first_or_random` algorithme résout le problème de la `in_order` algorithme. Avec `in_order`, si une réplique tombe en panne, la suivante obtient une double charge tandis que les répliques restantes gèrent la quantité habituelle de trafic. Lors de l'utilisation de la `first_or_random` algorithme, la charge est répartie uniformément entre les répliques qui sont encore disponibles. ## prefer\_localhost\_replica {#settings-prefer-localhost-replica} -Active / désactive préférable d’utiliser le réplica localhost lors du traitement des requêtes distribuées. +Active / désactive préférable d'utiliser le réplica localhost lors du traitement des requêtes distribuées. Valeurs possibles: @@ -760,29 +813,29 @@ Voir la section “WITH TOTALS modifier”. ## max\_parallel\_replicas {#settings-max_parallel_replicas} -Nombre maximal de répliques pour chaque fragment lors de l’exécution d’une requête. -Par souci de cohérence (pour obtenir différentes parties du même partage de données), Cette option ne fonctionne que lorsque la clé d’échantillonnage est définie. -Le retard de réplique n’est pas contrôlé. +Nombre maximal de répliques pour chaque fragment lors de l'exécution d'une requête. +Par souci de cohérence (pour obtenir différentes parties du même partage de données), Cette option ne fonctionne que lorsque la clé d'échantillonnage est définie. +Le retard de réplique n'est pas contrôlé. ## compiler {#compile} Activer la compilation des requêtes. Par défaut, 0 (désactivé). -La compilation n’est utilisée que pour une partie du pipeline de traitement des requêtes: pour la première étape de l’agrégation (GROUP BY). -Si cette partie du pipeline a été compilée, la requête peut s’exécuter plus rapidement en raison du déploiement de cycles courts et des appels de fonction d’agrégation intégrés. L’amélioration maximale des performances (jusqu’à quatre fois plus rapide dans de rares cas) est observée pour les requêtes avec plusieurs fonctions d’agrégat simples. Typiquement, le gain de performance est insignifiant. Dans de très rares cas, il peut ralentir l’exécution de la requête. +La compilation n'est utilisée que pour une partie du pipeline de traitement des requêtes: pour la première étape de l'agrégation (GROUP BY). +Si cette partie du pipeline a été compilée, la requête peut s'exécuter plus rapidement en raison du déploiement de cycles courts et des appels de fonction d'agrégation intégrés. L'amélioration maximale des performances (jusqu'à quatre fois plus rapide dans de rares cas) est observée pour les requêtes avec plusieurs fonctions d'agrégat simples. Typiquement, le gain de performance est insignifiant. Dans de très rares cas, il peut ralentir l'exécution de la requête. ## min\_count\_to\_compile {#min-count-to-compile} -Combien de fois utiliser potentiellement un morceau de code compilé avant d’exécuter la compilation. Par défaut, 3. +Combien de fois utiliser potentiellement un morceau de code compilé avant d'exécuter la compilation. Par défaut, 3. For testing, the value can be set to 0: compilation runs synchronously and the query waits for the end of the compilation process before continuing execution. For all other cases, use values ​​starting with 1. Compilation normally takes about 5-10 seconds. -Si la valeur est 1 ou plus, la compilation se produit de manière asynchrone dans un thread séparé. Le résultat sera utilisé dès qu’il sera prêt, y compris les requêtes en cours d’exécution. +Si la valeur est 1 ou plus, la compilation se produit de manière asynchrone dans un thread séparé. Le résultat sera utilisé dès qu'il sera prêt, y compris les requêtes en cours d'exécution. -Le code compilé est requis pour chaque combinaison différente de fonctions d’agrégat utilisées dans la requête et le type de clés dans la clause GROUP BY. -The results of the compilation are saved in the build directory in the form of .so files. There is no restriction on the number of compilation results since they don’t use very much space. Old results will be used after server restarts, except in the case of a server upgrade – in this case, the old results are deleted. +Le code compilé est requis pour chaque combinaison différente de fonctions d'agrégat utilisées dans la requête et le type de clés dans la clause GROUP BY. +The results of the compilation are saved in the build directory in the form of .so files. There is no restriction on the number of compilation results since they don't use very much space. Old results will be used after server restarts, except in the case of a server upgrade – in this case, the old results are deleted. ## output\_format\_json\_quote\_64bit\_integers {#session_settings-output_format_json_quote_64bit_integers} -Si la valeur est true, les entiers apparaissent entre guillemets lors de l’utilisation des formats JSON\* Int64 et UInt64 (pour la compatibilité avec la plupart des implémentations JavaScript); sinon, les entiers sont sortis sans les guillemets. +Si la valeur est true, les entiers apparaissent entre guillemets lors de l'utilisation des formats JSON\* Int64 et UInt64 (pour la compatibilité avec la plupart des implémentations JavaScript); sinon, les entiers sont sortis sans les guillemets. ## format\_csv\_delimiter {#settings-format_csv_delimiter} @@ -790,7 +843,7 @@ Caractère interprété comme un délimiteur dans les données CSV. Par défaut, ## input\_format\_csv\_unquoted\_null\_literal\_as\_null {#settings-input_format_csv_unquoted_null_literal_as_null} -Pour le format D’entrée CSV active ou désactive l’analyse des `NULL` comme littéral (synonyme de `\N`). +Pour le format D'entrée CSV active ou désactive l'analyse des `NULL` comme littéral (synonyme de `\N`). ## output\_format\_csv\_crlf\_end\_of\_line {#settings-output-format-csv-crlf-end-of-line} @@ -811,16 +864,16 @@ Valeur par défaut: 0. Quorum écrit -`INSERT` ne réussit que lorsque ClickHouse parvient à écrire correctement les données `insert_quorum` des répliques au cours de la `insert_quorum_timeout`. Si, pour une raison quelconque, le nombre de répliques avec succès écrit n’atteint pas le `insert_quorum`, l’écriture est considérée comme ayant échoué et ClickHouse supprimera le bloc inséré de toutes les répliques où les données ont déjà été écrites. +`INSERT` ne réussit que lorsque ClickHouse parvient à écrire correctement les données `insert_quorum` des répliques au cours de la `insert_quorum_timeout`. Si, pour une raison quelconque, le nombre de répliques avec succès écrit n'atteint pas le `insert_quorum`, l'écriture est considérée comme ayant échoué et ClickHouse supprimera le bloc inséré de toutes les répliques où les données ont déjà été écrites. -Toutes les répliques du quorum sont cohérentes, c’est-à-dire qu’elles contiennent des données de toutes les `INSERT` requête. Le `INSERT` la séquence est linéarisé. +Toutes les répliques du quorum sont cohérentes, c'est-à-dire qu'elles contiennent des données de toutes les `INSERT` requête. Le `INSERT` la séquence est linéarisé. Lors de la lecture des données écrites à partir du `insert_quorum`, vous pouvez utiliser le [select\_sequential\_consistency](#settings-select_sequential_consistency) option. Clickhouse génère une exception - Si le nombre de répliques au moment de la requête est inférieure à la `insert_quorum`. -- Lors d’une tentative d’écriture de données lorsque le bloc précédent n’a pas encore été inséré dans le `insert_quorum` des répliques. Cette situation peut se produire si l’utilisateur tente d’effectuer une `INSERT` avant le précédent avec le `insert_quorum` est terminé. +- Lors d'une tentative d'écriture de données lorsque le bloc précédent n'a pas encore été inséré dans le `insert_quorum` des répliques. Cette situation peut se produire si l'utilisateur tente d'effectuer une `INSERT` avant le précédent avec le `insert_quorum` est terminé. Voir aussi: @@ -829,7 +882,7 @@ Voir aussi: ## insert\_quorum\_timeout {#settings-insert_quorum_timeout} -Ecrire dans quorum timeout en secondes. Si le délai d’attente est passé et qu’aucune écriture n’a encore eu lieu, ClickHouse génère une exception et le client doit répéter la requête pour écrire le même bloc dans le même réplica ou tout autre réplica. +Ecrire dans quorum timeout en secondes. Si le délai d'attente est passé et qu'aucune écriture n'a encore eu lieu, ClickHouse génère une exception et le client doit répéter la requête pour écrire le même bloc dans le même réplica ou tout autre réplica. Valeur par défaut: 60 secondes. @@ -851,7 +904,7 @@ Valeur par défaut: 0. Utilisation -Lorsque la cohérence séquentielle est activée, ClickHouse permet au client d’exécuter `SELECT` requête uniquement pour les répliques qui contiennent des données de toutes les `INSERT` requêtes exécutées avec `insert_quorum`. Si le client fait référence à une réplique partielle, ClickHouse génère une exception. La requête SELECT n’inclut pas les données qui n’ont pas encore été écrites dans le quorum des répliques. +Lorsque la cohérence séquentielle est activée, ClickHouse permet au client d'exécuter `SELECT` requête uniquement pour les répliques qui contiennent des données de toutes les `INSERT` requêtes exécutées avec `insert_quorum`. Si le client fait référence à une réplique partielle, ClickHouse génère une exception. La requête SELECT n'inclut pas les données qui n'ont pas encore été écrites dans le quorum des répliques. Voir aussi: @@ -869,7 +922,7 @@ Valeurs possibles: Valeur par défaut: 1. -Par défaut, les blocs insérés dans les tables répliquées `INSERT` déclaration sont dédupliquées (voir \[Réplication de Données\] (../moteurs/table\_engines/mergetree\_family/réplication.md). +Par défaut, les blocs insérés dans les tables répliquées `INSERT` déclaration sont dédupliquées (voir [Réplication Des Données](../../engines/table-engines/mergetree-family/replication.md)). ## déduplicate\_blocks\_in\_dependent\_materialized\_views {#settings-deduplicate-blocks-in-dependent-materialized-views} @@ -884,14 +937,14 @@ Valeur par défaut: 0. Utilisation -Par défaut, la déduplication n’est pas effectuée pour les vues matérialisées mais en amont, dans la table source. -Si un bloc inséré est ignoré en raison de la déduplication dans la table source, il n’y aura pas d’insertion dans les vues matérialisées attachées. Ce comportement existe pour permettre l’insertion de données hautement agrégées dans des vues matérialisées, dans les cas où les blocs insérés sont les mêmes après l’agrégation de vues matérialisées mais dérivés de différentes insertions dans la table source. -Dans le même temps, ce comportement “breaks” `INSERT` idempotence. Si un `INSERT` dans la table principale a été un succès et `INSERT` into a materialized view failed (e.g. because of communication failure with Zookeeper) a client will get an error and can retry the operation. However, the materialized view won’t receive the second insert because it will be discarded by deduplication in the main (source) table. The setting `deduplicate_blocks_in_dependent_materialized_views` permet de changer ce comportement. Lors d’une nouvelle tentative, une vue matérialisée recevra l’insertion répétée et effectuera une vérification de déduplication par elle-même, +Par défaut, la déduplication n'est pas effectuée pour les vues matérialisées mais en amont, dans la table source. +Si un bloc inséré est ignoré en raison de la déduplication dans la table source, il n'y aura pas d'insertion dans les vues matérialisées attachées. Ce comportement existe pour permettre l'insertion de données hautement agrégées dans des vues matérialisées, dans les cas où les blocs insérés sont les mêmes après l'agrégation de vues matérialisées mais dérivés de différentes insertions dans la table source. +Dans le même temps, ce comportement “breaks” `INSERT` idempotence. Si un `INSERT` dans la table principale a été un succès et `INSERT` into a materialized view failed (e.g. because of communication failure with Zookeeper) a client will get an error and can retry the operation. However, the materialized view won't receive the second insert because it will be discarded by deduplication in the main (source) table. The setting `deduplicate_blocks_in_dependent_materialized_views` permet de changer ce comportement. Lors d'une nouvelle tentative, une vue matérialisée recevra l'insertion répétée et effectuera une vérification de déduplication par elle-même, ignorant le résultat de la vérification pour la table source, et insérera des lignes perdues en raison de la première défaillance. ## max\_network\_bytes {#settings-max-network-bytes} -Limite le volume de données (en octets) qui est reçu ou transmis sur le réseau lors de l’exécution d’une requête. Ce paramètre s’applique à chaque individu requête. +Limite le volume de données (en octets) qui est reçu ou transmis sur le réseau lors de l'exécution d'une requête. Ce paramètre s'applique à chaque individu requête. Valeurs possibles: @@ -902,7 +955,7 @@ Valeur par défaut: 0. ## max\_network\_bandwidth {#settings-max-network-bandwidth} -Limite la vitesse de l’échange de données sur le réseau en octets par seconde. Ce paramètre s’applique à toutes les requêtes. +Limite la vitesse de l'échange de données sur le réseau en octets par seconde. Ce paramètre s'applique à toutes les requêtes. Valeurs possibles: @@ -913,7 +966,7 @@ Valeur par défaut: 0. ## max\_network\_bandwidth\_for\_user {#settings-max-network-bandwidth-for-user} -Limite la vitesse de l’échange de données sur le réseau en octets par seconde. Ce paramètre s’applique à toutes les requêtes exécutées simultanément par un seul utilisateur. +Limite la vitesse de l'échange de données sur le réseau en octets par seconde. Ce paramètre s'applique à toutes les requêtes exécutées simultanément par un seul utilisateur. Valeurs possibles: @@ -924,7 +977,7 @@ Valeur par défaut: 0. ## max\_network\_bandwidth\_for\_all\_users {#settings-max-network-bandwidth-for-all-users} -Limite la vitesse à laquelle les données sont échangées sur le réseau en octets par seconde. Ce paramètre s’applique à toutes les requêtes exécutées simultanément sur le serveur. +Limite la vitesse à laquelle les données sont échangées sur le réseau en octets par seconde. Ce paramètre s'applique à toutes les requêtes exécutées simultanément sur le serveur. Valeurs possibles: @@ -935,7 +988,7 @@ Valeur par défaut: 0. ## count\_distinct\_implementation {#settings-count_distinct_implementation} -Spécifie de l’ `uniq*` les fonctions doivent être utilisées pour [COUNT(DISTINCT …)](../../sql-reference/aggregate-functions/reference.md#agg_function-count) construction. +Spécifie de l' `uniq*` les fonctions doivent être utilisées pour [COUNT(DISTINCT …)](../../sql-reference/aggregate-functions/reference.md#agg_function-count) construction. Valeurs possibles: @@ -951,7 +1004,7 @@ Valeur par défaut: `uniqExact`. Active ou désactive le saut silencieux des fragments indisponibles. -Tesson est considéré comme indisponible si toutes ses répliques ne sont pas disponibles. Une réplique n’est pas disponible dans les cas suivants: +Tesson est considéré comme indisponible si toutes ses répliques ne sont pas disponibles. Une réplique n'est pas disponible dans les cas suivants: - ClickHouse ne peut pas se connecter à la réplique pour une raison quelconque. @@ -959,21 +1012,21 @@ Tesson est considéré comme indisponible si toutes ses répliques ne sont pas d - La réplique ne peut pas être résolue via le DNS. - Si le nom d’hôte du réplica ne peut pas être résolu via DNS, il peut indiquer les situations suivantes: + Si le nom d'hôte du réplica ne peut pas être résolu via DNS, il peut indiquer les situations suivantes: - - L’hôte de la réplique n’a pas d’enregistrement DNS. Il peut se produire dans les systèmes avec DNS dynamique, par exemple, [Kubernetes](https://kubernetes.io), où les nœuds peuvent être insolubles pendant les temps d’arrêt, et ce n’est pas une erreur. + - L'hôte de la réplique n'a pas d'enregistrement DNS. Il peut se produire dans les systèmes avec DNS dynamique, par exemple, [Kubernetes](https://kubernetes.io), où les nœuds peuvent être insolubles pendant les temps d'arrêt, et ce n'est pas une erreur. - - Erreur de Configuration. Le fichier de configuration ClickHouse contient un mauvais nom d’hôte. + - Erreur de Configuration. Le fichier de configuration ClickHouse contient un mauvais nom d'hôte. Valeurs possibles: - 1 — skipping enabled. - Si un fragment n’est pas disponible, ClickHouse renvoie un résultat basé sur des données partielles et ne signale pas les problèmes de disponibilité des nœuds. + Si un fragment n'est pas disponible, ClickHouse renvoie un résultat basé sur des données partielles et ne signale pas les problèmes de disponibilité des nœuds. - 0 — skipping disabled. - Si un fragment n’est pas disponible, ClickHouse lève une exception. + Si un fragment n'est pas disponible, ClickHouse lève une exception. Valeur par défaut: 0. @@ -985,13 +1038,13 @@ Valeur par défaut: 0 ## force\_optimize\_skip\_unused\_shards {#settings-force_optimize_skip_unused_shards} -Active ou désactive l’exécution de la requête si [`optimize_skip_unused_shards`](#settings-optimize_skip_unused_shards) activé et sauter des fragments inutilisés n’est pas possible. Si le saut n’est pas possible et le paramètre est activé, une exception sera levée. +Active ou désactive l'exécution de la requête si [`optimize_skip_unused_shards`](#settings-optimize_skip_unused_shards) activé et sauter des fragments inutilisés n'est pas possible. Si le saut n'est pas possible et le paramètre est activé, une exception sera levée. Valeurs possibles: - 0 - Désactivé (ne jette) -- 1-Désactiver l’exécution de la requête uniquement si la table a une clé de sharding -- 2-Désactiver l’exécution de la requête quelle que soit la clé de sharding est définie pour la table +- 1-Désactiver l'exécution de la requête uniquement si la table a une clé de sharding +- 2-Désactiver l'exécution de la requête quelle que soit la clé de sharding est définie pour la table Valeur par défaut: 0 @@ -1008,9 +1061,9 @@ Valeur par défaut: 0. ## optimize\_throw\_if\_noop {#setting-optimize_throw_if_noop} -Active ou désactive le lancement d’une exception si [OPTIMIZE](../../sql-reference/statements/misc.md#misc_operations-optimize) la requête n’a pas effectué de fusion. +Active ou désactive le lancement d'une exception si [OPTIMIZE](../../sql-reference/statements/misc.md#misc_operations-optimize) la requête n'a pas effectué de fusion. -Par défaut, `OPTIMIZE` retourne avec succès même s’il n’a rien fait. Ce paramètre vous permet de différencier ces situations et d’obtenir la raison dans un message d’exception. +Par défaut, `OPTIMIZE` retourne avec succès même s'il n'a rien fait. Ce paramètre vous permet de différencier ces situations et d'obtenir la raison dans un message d'exception. Valeurs possibles: @@ -1036,7 +1089,7 @@ Voir aussi: - Type: unsigned int - Valeur par défaut: 1000 -Le nombre d’erreurs de chaque réplique est plafonné à cette valeur, empêchant une seule réplique d’accumuler trop d’erreurs. +Le nombre d'erreurs de chaque réplique est plafonné à cette valeur, empêchant une seule réplique d'accumuler trop d'erreurs. Voir aussi: @@ -1045,7 +1098,7 @@ Voir aussi: ## distributed\_directory\_monitor\_sleep\_time\_ms {#distributed_directory_monitor_sleep_time_ms} -Intervalle de Base pour le [Distribué](../../engines/table-engines/special/distributed.md) tableau moteur à envoyer des données. L’intervalle réel augmente de façon exponentielle en cas d’erreurs. +Intervalle de Base pour le [Distribué](../../engines/table-engines/special/distributed.md) tableau moteur à envoyer des données. L'intervalle réel augmente de façon exponentielle en cas d'erreurs. Valeurs possibles: @@ -1055,7 +1108,7 @@ Valeur par défaut: 100 millisecondes. ## distributed\_directory\_monitor\_max\_sleep\_time\_ms {#distributed_directory_monitor_max_sleep_time_ms} -Intervalle maximal pour le [Distribué](../../engines/table-engines/special/distributed.md) tableau moteur à envoyer des données. Limite la croissance exponentielle de l’intervalle défini dans [distributed\_directory\_monitor\_sleep\_time\_ms](#distributed_directory_monitor_sleep_time_ms) paramètre. +Intervalle maximal pour le [Distribué](../../engines/table-engines/special/distributed.md) tableau moteur à envoyer des données. Limite la croissance exponentielle de l'intervalle défini dans [distributed\_directory\_monitor\_sleep\_time\_ms](#distributed_directory_monitor_sleep_time_ms) paramètre. Valeurs possibles: @@ -1065,9 +1118,9 @@ Valeur par défaut: 30000 millisecondes (30 secondes). ## distributed\_directory\_monitor\_batch\_inserts {#distributed_directory_monitor_batch_inserts} -Active / désactive l’envoi des données insérées par lots. +Active / désactive l'envoi des données insérées par lots. -Lorsque l’envoi par lots est activé, le [Distribué](../../engines/table-engines/special/distributed.md) tableau moteur essaie d’envoyer plusieurs fichiers de données insérées dans une seule opération au lieu de les envoyer séparément. L’envoi par lots améliore les performances du cluster en utilisant mieux les ressources du serveur et du réseau. +Lorsque l'envoi par lots est activé, le [Distribué](../../engines/table-engines/special/distributed.md) tableau moteur essaie d'envoyer plusieurs fichiers de données insérées dans une seule opération au lieu de les envoyer séparément. L'envoi par lots améliore les performances du cluster en utilisant mieux les ressources du serveur et du réseau. Valeurs possibles: @@ -1078,22 +1131,22 @@ Valeur par défaut: 0. ## os\_thread\_priority {#setting-os-thread-priority} -Définit la priorité ([beau](https://en.wikipedia.org/wiki/Nice_(Unix))) pour les threads qui exécutent des requêtes. Le planificateur du système d’exploitation considère cette priorité lors du choix du prochain thread à exécuter sur chaque noyau CPU disponible. +Définit la priorité ([beau](https://en.wikipedia.org/wiki/Nice_(Unix))) pour les threads qui exécutent des requêtes. Le planificateur du système d'exploitation considère cette priorité lors du choix du prochain thread à exécuter sur chaque noyau CPU disponible. !!! warning "Avertissement" - Pour utiliser ce paramètre, vous devez définir l’ `CAP_SYS_NICE` capacité. Le `clickhouse-server` paquet configure lors de l’installation. Certains environnements virtuels ne vous permettent pas de définir `CAP_SYS_NICE` capacité. Dans ce cas, `clickhouse-server` affiche un message à ce sujet au début. + Pour utiliser ce paramètre, vous devez définir l' `CAP_SYS_NICE` capacité. Le `clickhouse-server` paquet configure lors de l'installation. Certains environnements virtuels ne vous permettent pas de définir `CAP_SYS_NICE` capacité. Dans ce cas, `clickhouse-server` affiche un message à ce sujet au début. Valeurs possibles: - Vous pouvez définir des valeurs dans la gamme `[-20, 19]`. -Des valeurs plus faibles signifient une priorité plus élevée. Les discussions avec des bas `nice` les valeurs de priorité sont effectués plus fréquemment que les discussions avec des valeurs élevées. Les valeurs élevées sont préférables pour les requêtes non interactives de longue durée, car elles leur permettent d’abandonner rapidement des ressources au profit de requêtes interactives courtes lorsqu’elles arrivent. +Des valeurs plus faibles signifient une priorité plus élevée. Les discussions avec des bas `nice` les valeurs de priorité sont effectués plus fréquemment que les discussions avec des valeurs élevées. Les valeurs élevées sont préférables pour les requêtes non interactives de longue durée, car elles leur permettent d'abandonner rapidement des ressources au profit de requêtes interactives courtes lorsqu'elles arrivent. Valeur par défaut: 0. ## query\_profiler\_real\_time\_period\_ns {#query_profiler_real_time_period_ns} -Définit la période pour une horloge réelle de la [requête profiler](../../operations/optimizing-performance/sampling-query-profiler.md). La vraie minuterie d’horloge compte le temps d’horloge murale. +Définit la période pour une horloge réelle de la [requête profiler](../../operations/optimizing-performance/sampling-query-profiler.md). La vraie minuterie d'horloge compte le temps d'horloge murale. Valeurs possibles: @@ -1116,7 +1169,7 @@ Voir aussi: ## query\_profiler\_cpu\_time\_period\_ns {#query_profiler_cpu_time_period_ns} -Définit la période pour une minuterie D’horloge CPU du [requête profiler](../../operations/optimizing-performance/sampling-query-profiler.md). Cette minuterie ne compte que le temps CPU. +Définit la période pour une minuterie D'horloge CPU du [requête profiler](../../operations/optimizing-performance/sampling-query-profiler.md). Cette minuterie ne compte que le temps CPU. Valeurs possibles: @@ -1158,7 +1211,7 @@ Valeur par défaut: 0. - Type: bool - Valeur par défaut: True -Activer l’analyse parallèle des formats de données en préservant l’ordre. Pris en charge uniquement pour les formats TSV, TKSV, CSV et jsoneachrow. +Activer l'analyse parallèle des formats de données en préservant l'ordre. Pris en charge uniquement pour les formats TSV, TKSV, CSV et jsoneachrow. ## min\_chunk\_bytes\_for\_parallel\_parsing {#min-chunk-bytes-for-parallel-parsing} @@ -1193,10 +1246,20 @@ Valeur par défaut: 32768 (32 Ko) ## format\_avro\_schema\_registry\_url {#settings-format_avro_schema_registry_url} -Définit L’URL de Registre de schéma Confluent à utiliser avec [AvroConfluent](../../interfaces/formats.md#data-format-avro-confluent) format +Définit L'URL de Registre de schéma Confluent à utiliser avec [AvroConfluent](../../interfaces/formats.md#data-format-avro-confluent) format Type: URL Valeur par défaut: vide +## background\_pool\_size {#background_pool_size} + +Définit le nombre de threads effectuant des opérations d'arrière-plan dans les moteurs de table (par exemple, fusionne dans [Moteur MergeTree](../../engines/table-engines/mergetree-family/index.md) table). Ce paramètre est appliqué au démarrage du serveur ClickHouse et ne peut pas être modifié dans une session utilisateur. En ajustant ce paramètre, vous gérez la charge du processeur et du disque. Une taille de pool plus petite utilise moins de ressources CPU et disque, mais les processus d'arrière-plan avancent plus lentement, ce qui pourrait éventuellement avoir un impact sur les performances des requêtes. + +Valeurs possibles: + +- Tout nombre entier positif. + +Valeur par défaut: 16. + [Article Original](https://clickhouse.tech/docs/en/operations/settings/settings/) diff --git a/docs/fr/operations/system-tables.md b/docs/fr/operations/system-tables.md index 49b14698af7..3a92eca20bb 100644 --- a/docs/fr/operations/system-tables.md +++ b/docs/fr/operations/system-tables.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 52 toc_title: "Les Tables Syst\xE8me" --- @@ -9,7 +9,7 @@ toc_title: "Les Tables Syst\xE8me" Les tables système sont utilisées pour implémenter une partie des fonctionnalités du système et pour fournir un accès à des informations sur le fonctionnement du système. Vous ne pouvez pas supprimer une table système (mais vous pouvez effectuer un détachement). -Les tables système n’ont pas de fichiers avec des données sur le disque ou de fichiers avec des métadonnées. Le serveur crée toutes les tables système au démarrage. +Les tables système n'ont pas de fichiers avec des données sur le disque ou de fichiers avec des métadonnées. Le serveur crée toutes les tables système au démarrage. Les tables système sont en lecture seule. Ils sont situés dans la ‘system’ la base de données. @@ -64,10 +64,10 @@ Colonne: - `host_address` (String) — The host IP address obtained from DNS. - `port` (UInt16) — The port to use for connecting to the server. - `user` (String) — The name of the user for connecting to the server. -- `errors_count` (UInt32) - nombre de fois que cet hôte n’a pas atteint le réplica. -- `estimated_recovery_time` (UInt32) - secondes restantes jusqu’à ce que le nombre d’erreurs de réplique soit remis à zéro et qu’il soit considéré comme revenu à la normale. +- `errors_count` (UInt32) - nombre de fois que cet hôte n'a pas atteint le réplica. +- `estimated_recovery_time` (UInt32) - secondes restantes jusqu'à ce que le nombre d'erreurs de réplique soit remis à zéro et qu'il soit considéré comme revenu à la normale. -Veuillez noter que `errors_count` est mise à jour une fois par requête à la grappe, mais `estimated_recovery_time` est recalculé sur-demande. Il pourrait donc y avoir un cas de non-zéro `errors_count` et zéro `estimated_recovery_time`, cette requête suivante sera nulle `errors_count` et essayez d’utiliser des répliques comme si elle ne comporte pas d’erreurs. +Veuillez noter que `errors_count` est mise à jour une fois par requête à la grappe, mais `estimated_recovery_time` est recalculé sur-demande. Il pourrait donc y avoir un cas de non-zéro `errors_count` et zéro `estimated_recovery_time`, cette requête suivante sera nulle `errors_count` et essayez d'utiliser des répliques comme si elle ne comporte pas d'erreurs. **Voir aussi** @@ -79,7 +79,7 @@ Veuillez noter que `errors_count` est mise à jour une fois par requête à la g Contient des informations sur les colonnes de toutes les tables. -Vous pouvez utiliser ce tableau pour obtenir des informations similaires à l’ [DESCRIBE TABLE](../sql-reference/statements/misc.md#misc-describe-table) requête, mais pour plusieurs tables à la fois. +Vous pouvez utiliser ce tableau pour obtenir des informations similaires à l' [DESCRIBE TABLE](../sql-reference/statements/misc.md#misc-describe-table) requête, mais pour plusieurs tables à la fois. Le `system.columns` le tableau contient les colonnes suivantes (la colonne type est indiqué entre parenthèses): @@ -87,7 +87,7 @@ Le `system.columns` le tableau contient les colonnes suivantes (la colonne type - `table` (String) — Table name. - `name` (String) — Column name. - `type` (String) — Column type. -- `default_kind` (String) — Expression type (`DEFAULT`, `MATERIALIZED`, `ALIAS`) pour la valeur par défaut, ou une chaîne vide si elle n’est pas définie. +- `default_kind` (String) — Expression type (`DEFAULT`, `MATERIALIZED`, `ALIAS`) pour la valeur par défaut, ou une chaîne vide si elle n'est pas définie. - `default_expression` (String) — Expression for the default value, or an empty string if it is not defined. - `data_compressed_bytes` (UInt64) — The size of compressed data, in bytes. - `data_uncompressed_bytes` (UInt64) — The size of decompressed data, in bytes. @@ -100,7 +100,7 @@ Le `system.columns` le tableau contient les colonnes suivantes (la colonne type ## système.contributeur {#system-contributors} -Contient des informations sur les donateurs. Tous les constributors dans un ordre aléatoire. L’ordre est aléatoire au moment de l’exécution de la requête. +Contient des informations sur les donateurs. Tous les constributors dans un ordre aléatoire. L'ordre est aléatoire au moment de l'exécution de la requête. Colonne: @@ -147,33 +147,73 @@ Cette table système est utilisée pour implémenter `SHOW DATABASES` requête. ## système.detached\_parts {#system_tables-detached_parts} -Contient des informations sur les pièces détachées de [MergeTree](../engines/table-engines/mergetree-family/mergetree.md) table. Le `reason` colonne spécifie pourquoi la pièce a été détachée. Pour les pièces détachées par l’utilisateur, la raison est vide. De telles pièces peuvent être attachées avec [ALTER TABLE ATTACH PARTITION\|PART](../query_language/query_language/alter/#alter_attach-partition) commande. Pour la description des autres colonnes, voir [système.partie](#system_tables-parts). Si le nom de pièce n’est pas valide, les valeurs de certaines colonnes peuvent être `NULL`. Ces pièces peuvent être supprimés avec [ALTER TABLE DROP DETACHED PART](../query_language/query_language/alter/#alter_drop-detached). +Contient des informations sur les pièces détachées de [MergeTree](../engines/table-engines/mergetree-family/mergetree.md) table. Le `reason` colonne spécifie pourquoi la pièce a été détachée. Pour les pièces détachées par l'utilisateur, la raison est vide. De telles pièces peuvent être attachées avec [ALTER TABLE ATTACH PARTITION\|PART](../sql-reference/statements/alter.md#alter_attach-partition) commande. Pour la description des autres colonnes, voir [système.partie](#system_tables-parts). Si le nom de pièce n'est pas valide, les valeurs de certaines colonnes peuvent être `NULL`. Ces pièces peuvent être supprimés avec [ALTER TABLE DROP DETACHED PART](../sql-reference/statements/alter.md#alter_drop-detached). -## système.dictionnaire {#system-dictionaries} +## système.dictionnaire {#system_tables-dictionaries} -Contient des informations sur les dictionnaires externes. +Contient des informations sur [dictionnaires externes](../sql-reference/dictionaries/external-dictionaries/external-dicts.md). Colonne: -- `name` (String) — Dictionary name. -- `type` (String) — Dictionary type: Flat, Hashed, Cache. -- `origin` (String) — Path to the configuration file that describes the dictionary. -- `attribute.names` (Array(String)) — Array of attribute names provided by the dictionary. -- `attribute.types` (Array(String)) — Corresponding array of attribute types that are provided by the dictionary. -- `has_hierarchy` (UInt8) — Whether the dictionary is hierarchical. -- `bytes_allocated` (UInt64) — The amount of RAM the dictionary uses. -- `hit_rate` (Float64) — For cache dictionaries, the percentage of uses for which the value was in the cache. -- `element_count` (UInt64) — The number of items stored in the dictionary. -- `load_factor` (Float64) — The percentage filled in the dictionary (for a hashed dictionary, the percentage filled in the hash table). -- `creation_time` (DateTime) — The time when the dictionary was created or last successfully reloaded. -- `last_exception` (String) — Text of the error that occurs when creating or reloading the dictionary if the dictionary couldn’t be created. -- `source` (String) — Text describing the data source for the dictionary. +- `database` ([Chaîne](../sql-reference/data-types/string.md)) — Name of the database containing the dictionary created by DDL query. Empty string for other dictionaries. +- `name` ([Chaîne](../sql-reference/data-types/string.md)) — [Nom du dictionnaire](../sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md). +- `status` ([Enum8](../sql-reference/data-types/enum.md)) — Dictionary status. Possible values: + - `NOT_LOADED` — Dictionary was not loaded because it was not used. + - `LOADED` — Dictionary loaded successfully. + - `FAILED` — Unable to load the dictionary as a result of an error. + - `LOADING` — Dictionary is loading now. + - `LOADED_AND_RELOADING` — Dictionary is loaded successfully, and is being reloaded right now (frequent reasons: [SYSTEM RELOAD DICTIONARY](../sql-reference/statements/system.md#query_language-system-reload-dictionary) requête, délai d'attente, configuration du dictionnaire a changé). + - `FAILED_AND_RELOADING` — Could not load the dictionary as a result of an error and is loading now. +- `origin` ([Chaîne](../sql-reference/data-types/string.md)) — Path to the configuration file that describes the dictionary. +- `type` ([Chaîne](../sql-reference/data-types/string.md)) — Type of a dictionary allocation. [Stockage des dictionnaires en mémoire](../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md). +- `key` — [Type de clé](../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md#ext_dict_structure-key): Touche Numérique ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) or Сomposite key ([Chaîne](../sql-reference/data-types/string.md)) — form “(type 1, type 2, …, type n)”. +- `attribute.names` ([Tableau](../sql-reference/data-types/array.md)([Chaîne](../sql-reference/data-types/string.md))) — Array of [les noms d'attribut](../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md#ext_dict_structure-attributes) fournis par le dictionnaire. +- `attribute.types` ([Tableau](../sql-reference/data-types/array.md)([Chaîne](../sql-reference/data-types/string.md))) — Corresponding array of [les types d'attribut](../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md#ext_dict_structure-attributes) qui sont fournis par le dictionnaire. +- `bytes_allocated` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Amount of RAM allocated for the dictionary. +- `query_count` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Number of queries since the dictionary was loaded or since the last successful reboot. +- `hit_rate` ([Float64](../sql-reference/data-types/float.md)) — For cache dictionaries, the percentage of uses for which the value was in the cache. +- `element_count` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Number of items stored in the dictionary. +- `load_factor` ([Float64](../sql-reference/data-types/float.md)) — Percentage filled in the dictionary (for a hashed dictionary, the percentage filled in the hash table). +- `source` ([Chaîne](../sql-reference/data-types/string.md)) — Text describing the [source de données](../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md) pour le dictionnaire. +- `lifetime_min` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Minimum [vie](../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md) du dictionnaire en mémoire, après quoi ClickHouse tente de recharger le dictionnaire (si `invalidate_query` est définie, alors que si elle a changé). Réglez en quelques secondes. +- `lifetime_max` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Maximum [vie](../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md) du dictionnaire en mémoire, après quoi ClickHouse tente de recharger le dictionnaire (si `invalidate_query` est définie, alors que si elle a changé). Réglez en quelques secondes. +- `loading_start_time` ([DateTime](../sql-reference/data-types/datetime.md)) — Start time for loading the dictionary. +- `last_successful_update_time` ([DateTime](../sql-reference/data-types/datetime.md)) — End time for loading or updating the dictionary. Helps to monitor some troubles with external sources and investigate causes. +- `loading_duration` ([Float32](../sql-reference/data-types/float.md)) — Duration of a dictionary loading. +- `last_exception` ([Chaîne](../sql-reference/data-types/string.md)) — Text of the error that occurs when creating or reloading the dictionary if the dictionary couldn't be created. -Notez que la quantité de mémoire utilisée par le dictionnaire n’est pas proportionnel au nombre d’articles qui s’y trouvent. Ainsi, pour les dictionnaires plats et mis en cache, toutes les cellules de mémoire sont pré-assignées, quelle que soit la capacité du dictionnaire. +**Exemple** + +Configurez le dictionnaire. + +``` sql +CREATE DICTIONARY dictdb.dict +( + `key` Int64 DEFAULT -1, + `value_default` String DEFAULT 'world', + `value_expression` String DEFAULT 'xxx' EXPRESSION 'toString(127 * 172)' +) +PRIMARY KEY key +SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'dicttbl' DB 'dictdb')) +LIFETIME(MIN 0 MAX 1) +LAYOUT(FLAT()) +``` + +Assurez-vous que le dictionnaire est chargé. + +``` sql +SELECT * FROM system.dictionaries +``` + +``` text +┌─database─┬─name─┬─status─┬─origin──────┬─type─┬─key────┬─attribute.names──────────────────────┬─attribute.types─────┬─bytes_allocated─┬─query_count─┬─hit_rate─┬─element_count─┬───────────load_factor─┬─source─────────────────────┬─lifetime_min─┬─lifetime_max─┬──loading_start_time─┌──last_successful_update_time─┬──────loading_duration─┬─last_exception─┐ +│ dictdb │ dict │ LOADED │ dictdb.dict │ Flat │ UInt64 │ ['value_default','value_expression'] │ ['String','String'] │ 74032 │ 0 │ 1 │ 1 │ 0.0004887585532746823 │ ClickHouse: dictdb.dicttbl │ 0 │ 1 │ 2020-03-04 04:17:34 │ 2020-03-04 04:30:34 │ 0.002 │ │ +└──────────┴──────┴────────┴─────────────┴──────┴────────┴──────────────────────────────────────┴─────────────────────┴─────────────────┴─────────────┴──────────┴───────────────┴───────────────────────┴────────────────────────────┴──────────────┴──────────────┴─────────────────────┴──────────────────────────────┘───────────────────────┴────────────────┘ +``` ## système.événement {#system_tables-events} -Contient des informations sur le nombre d’événements survenus dans le système. Par exemple, dans le tableau, vous pouvez trouver combien `SELECT` les requêtes ont été traitées depuis le démarrage du serveur ClickHouse. +Contient des informations sur le nombre d'événements survenus dans le système. Par exemple, dans le tableau, vous pouvez trouver combien `SELECT` les requêtes ont été traitées depuis le démarrage du serveur ClickHouse. Colonne: @@ -221,9 +261,9 @@ Colonne: - `config_name` (Chaîne) - `graphite_rollup` nom du paramètre. - `regexp` (Chaîne) - un modèle pour le nom de la métrique. -- `function` (Chaîne) - le nom de la fonction d’agrégation. -- `age` (UInt64) - l’âge minimum des données en secondes. -- `precision` (UInt64) - comment définir précisément l’âge des données en secondes. +- `function` (Chaîne) - le nom de la fonction d'agrégation. +- `age` (UInt64) - l'âge minimum des données en secondes. +- `precision` (UInt64) - comment définir précisément l'âge des données en secondes. - `priority` (UInt16) - priorité de motif. - `is_default` (UInt8) - indique si le motif est la valeur par défaut. - `Tables.database` (Array (String)) - tableau de noms de tables de base de données qui utilisent `config_name` paramètre. @@ -259,7 +299,7 @@ Colonne: - `value` ([Int64](../sql-reference/data-types/int-uint.md)) — Metric value. - `description` ([Chaîne](../sql-reference/data-types/string.md)) — Metric description. -La liste des mesures que vous pouvez trouver dans le [SGBD / commun / CurrentMetrics.rpc](https://github.com/ClickHouse/ClickHouse/blob/master/dbms/Common/CurrentMetrics.cpp) fichier source de ClickHouse. +La liste des mesures que vous pouvez trouver dans le [src / Common / CurrentMetrics.rpc](https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/CurrentMetrics.cpp) fichier source de ClickHouse. **Exemple** @@ -291,8 +331,8 @@ SELECT * FROM system.metrics LIMIT 10 ## système.metric\_log {#system_tables-metric_log} -Contient l’historique des valeurs de métriques des tables `system.metrics` et `system.events` périodiquement vidé sur le disque. -Pour activer la collection d’historique des métriques `system.metric_log`, créer `/etc/clickhouse-server/config.d/metric_log.xml` avec le contenu suivant: +Contient l'historique des valeurs de métriques des tables `system.metrics` et `system.events` périodiquement vidé sur le disque. +Pour activer la collection d'historique des métriques `system.metric_log`, créer `/etc/clickhouse-server/config.d/metric_log.xml` avec le contenu suivant: ``` xml @@ -353,14 +393,14 @@ Les lectures de cette table ne sont pas parallélisées. ## système.numbers\_mt {#system-numbers-mt} -Le même que ‘system.numbers’ mais les lectures sont parallélisées. Les nombres peuvent être retournés dans n’importe quel ordre. +Le même que ‘system.numbers’ mais les lectures sont parallélisées. Les nombres peuvent être retournés dans n'importe quel ordre. Utilisé pour les tests. ## système.un {#system-one} Cette table contient une seule ligne avec un ‘dummy’ Colonne UInt8 contenant la valeur 0. Cette table est utilisée si une requête SELECT ne spécifie pas la clause FROM. -Ceci est similaire à la table double trouvée dans d’autres SGBD. +Ceci est similaire à la table double trouvée dans d'autres SGBD. ## système.partie {#system_tables-parts} @@ -379,9 +419,9 @@ Colonne: - `name` (`String`) – Name of the data part. -- `active` (`UInt8`) – Flag that indicates whether the data part is active. If a data part is active, it’s used in a table. Otherwise, it’s deleted. Inactive data parts remain after merging. +- `active` (`UInt8`) – Flag that indicates whether the data part is active. If a data part is active, it's used in a table. Otherwise, it's deleted. Inactive data parts remain after merging. -- `marks` (`UInt64`) – The number of marks. To get the approximate number of rows in a data part, multiply `marks` par la granularité d’index (généralement 8192) (cet indice ne fonctionne pas pour la granularité adaptative). +- `marks` (`UInt64`) – The number of marks. To get the approximate number of rows in a data part, multiply `marks` par la granularité d'index (généralement 8192) (cet indice ne fonctionne pas pour la granularité adaptative). - `rows` (`UInt64`) – The number of rows. @@ -421,7 +461,7 @@ Colonne: - `primary_key_bytes_in_memory_allocated` (`UInt64`) – The amount of memory (in bytes) reserved for primary key values. -- `is_frozen` (`UInt8`) – Flag that shows that a partition data backup exists. 1, the backup exists. 0, the backup doesn’t exist. For more details, see [FREEZE PARTITION](../sql-reference/statements/alter.md#alter_freeze-partition) +- `is_frozen` (`UInt8`) – Flag that shows that a partition data backup exists. 1, the backup exists. 0, the backup doesn't exist. For more details, see [FREEZE PARTITION](../sql-reference/statements/alter.md#alter_freeze-partition) - `database` (`String`) – Name of the database. @@ -435,9 +475,9 @@ Colonne: - `hash_of_all_files` (`String`) – [sipHash128](../sql-reference/functions/hash-functions.md#hash_functions-siphash128) de fichiers compressés. -- `hash_of_uncompressed_files` (`String`) – [sipHash128](../sql-reference/functions/hash-functions.md#hash_functions-siphash128) de fichiers non compressés (fichiers avec des marques, fichier d’index, etc.). +- `hash_of_uncompressed_files` (`String`) – [sipHash128](../sql-reference/functions/hash-functions.md#hash_functions-siphash128) de fichiers non compressés (fichiers avec des marques, fichier d'index, etc.). -- `uncompressed_hash_of_compressed_files` (`String`) – [sipHash128](../sql-reference/functions/hash-functions.md#hash_functions-siphash128) des données dans les fichiers compressés comme s’ils étaient non compressé. +- `uncompressed_hash_of_compressed_files` (`String`) – [sipHash128](../sql-reference/functions/hash-functions.md#hash_functions-siphash128) des données dans les fichiers compressés comme s'ils étaient non compressé. - `bytes` (`UInt64`) – Alias for `bytes_on_disk`. @@ -447,7 +487,7 @@ Colonne: Le `system.part_log` la table est créée uniquement si [part\_log](server-configuration-parameters/settings.md#server_configuration_parameters-part-log) serveur paramètre est spécifié. -Ce tableau contient des informations sur les événements survenus avec [les parties de données](../engines/table-engines/mergetree-family/custom-partitioning-key.md) dans le [MergeTree](../engines/table-engines/mergetree-family/mergetree.md) table de famille, telles que l’ajout ou la fusion de données. +Ce tableau contient des informations sur les événements survenus avec [les parties de données](../engines/table-engines/mergetree-family/custom-partitioning-key.md) dans le [MergeTree](../engines/table-engines/mergetree-family/mergetree.md) table de famille, telles que l'ajout ou la fusion de données. Le `system.part_log` le tableau contient les colonnes suivantes: @@ -482,14 +522,14 @@ Cette table système est utilisée pour implémenter `SHOW PROCESSLIST` requête Colonne: -- `user` (String) – The user who made the query. Keep in mind that for distributed processing, queries are sent to remote servers under the `default` utilisateur. Le champ contient le nom d’utilisateur pour une requête spécifique, pas pour une requête que cette requête lancée. +- `user` (String) – The user who made the query. Keep in mind that for distributed processing, queries are sent to remote servers under the `default` utilisateur. Le champ contient le nom d'utilisateur pour une requête spécifique, pas pour une requête que cette requête lancée. - `address` (String) – The IP address the request was made from. The same for distributed processing. To track where a distributed query was originally made from, look at `system.processes` sur le serveur du demandeur de requête. - `elapsed` (Float64) – The time in seconds since request execution started. - `rows_read` (UInt64) – The number of rows read from the table. For distributed processing, on the requestor server, this is the total for all remote servers. - `bytes_read` (UInt64) – The number of uncompressed bytes read from the table. For distributed processing, on the requestor server, this is the total for all remote servers. - `total_rows_approx` (UInt64) – The approximation of the total number of rows that should be read. For distributed processing, on the requestor server, this is the total for all remote servers. It can be updated during request processing, when new sources to process become known. - `memory_usage` (UInt64) – Amount of RAM the request uses. It might not include some types of dedicated memory. See the [max\_memory\_usage](../operations/settings/query-complexity.md#settings_max_memory_usage) paramètre. -- `query` (String) – The query text. For `INSERT` il n’inclut pas les données à insérer. +- `query` (String) – The query text. For `INSERT` il n'inclut pas les données à insérer. - `query_id` (String) – Query ID, if defined. ## système.text\_log {#system-tables-text-log} @@ -498,12 +538,12 @@ Contient des entrées de journalisation. Niveau de journalisation qui va à cett Colonne: -- `event_date` (`Date`) - Date de l’entrée. -- `event_time` (`DateTime`) - Temps de l’entrée. -- `microseconds` (`UInt32`) - Microsecondes de l’entrée. +- `event_date` (`Date`) - Date de l'entrée. +- `event_time` (`DateTime`) - Temps de l'entrée. +- `microseconds` (`UInt32`) - Microsecondes de l'entrée. - `thread_name` (String) — Name of the thread from which the logging was done. - `thread_id` (UInt64) — OS thread ID. -- `level` (`Enum8`) - Niveau d’entrée. +- `level` (`Enum8`) - Niveau d'entrée. - `'Fatal' = 1` - `'Critical' = 2` - `'Error' = 3` @@ -521,19 +561,19 @@ Colonne: ## système.query\_log {#system_tables-query_log} -Contient des informations sur l’exécution de requêtes. Pour chaque requête, vous pouvez voir l’Heure de début du traitement, la durée du traitement, les messages d’erreur et d’autres informations. +Contient des informations sur l'exécution de requêtes. Pour chaque requête, vous pouvez voir l'Heure de début du traitement, la durée du traitement, les messages d'erreur et d'autres informations. !!! note "Note" - Le tableau ne contient pas les données d’entrée pour `INSERT` requête. + Le tableau ne contient pas les données d'entrée pour `INSERT` requête. -Clickhouse crée cette table uniquement si [query\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) serveur paramètre est spécifié. Ce paramètre définit les règles de journalisation, tels que l’intervalle d’enregistrement ou le nom de la table, la requête sera connecté. +Clickhouse crée cette table uniquement si [query\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) serveur paramètre est spécifié. Ce paramètre définit les règles de journalisation, tels que l'intervalle d'enregistrement ou le nom de la table, la requête sera connecté. Pour activer la journalisation des requêtes, définissez [log\_queries](settings/settings.md#settings-log-queries) paramètre 1. Pour plus de détails, voir le [Paramètre](settings/settings.md) section. Le `system.query_log` table enregistre deux types de requêtes: 1. Requêtes initiales qui ont été exécutées directement par le client. -2. Requêtes enfants initiées par d’autres requêtes (pour l’exécution de requêtes distribuées). Pour ces types de requêtes, des informations sur les requêtes parentes sont affichées dans `initial_*` colonne. +2. Requêtes enfants initiées par d'autres requêtes (pour l'exécution de requêtes distribuées). Pour ces types de requêtes, des informations sur les requêtes parentes sont affichées dans `initial_*` colonne. Colonne: @@ -549,7 +589,7 @@ Colonne: - `read_rows` (UInt64) — Number of read rows. - `read_bytes` (UInt64) — Number of read bytes. - `written_rows` (UInt64) — For `INSERT` des requêtes, le nombre de lignes. Pour les autres requêtes, la valeur de la colonne est 0. -- `written_bytes` (UInt64) — For `INSERT` des requêtes, le nombre d’octets écrits. Pour les autres requêtes, la valeur de la colonne est 0. +- `written_bytes` (UInt64) — For `INSERT` des requêtes, le nombre d'octets écrits. Pour les autres requêtes, la valeur de la colonne est 0. - `result_rows` (UInt64) — Number of rows in the result. - `result_bytes` (UInt64) — Number of bytes in the result. - `memory_usage` (UInt64) — Memory consumption by the query. @@ -570,7 +610,7 @@ Colonne: - `interface` (UInt8) — Interface that the query was initiated from. Possible values: - 1 — TCP. - 2 — HTTP. -- `os_user` (String) — OS’s username who runs [clickhouse-client](../interfaces/cli.md). +- `os_user` (String) — OS's username who runs [clickhouse-client](../interfaces/cli.md). - `client_hostname` (String) — Hostname of the client machine where the [clickhouse-client](../interfaces/cli.md) ou un autre client TCP est exécuté. - `client_name` (String) — The [clickhouse-client](../interfaces/cli.md) ou un autre nom de client TCP. - `client_revision` (UInt32) — Revision of the [clickhouse-client](../interfaces/cli.md) ou un autre client TCP. @@ -590,11 +630,11 @@ Colonne: - `Settings.Names` (Array(String)) — Names of settings that were changed when the client ran the query. To enable logging changes to settings, set the `log_query_settings` paramètre 1. - `Settings.Values` (Array(String)) — Values of settings that are listed in the `Settings.Names` colonne. -Chaque requête crée une ou deux lignes dans le `query_log` le tableau, en fonction de l’état de la requête: +Chaque requête crée une ou deux lignes dans le `query_log` le tableau, en fonction de l'état de la requête: -1. Si l’exécution de la requête est réussie, deux événements de type 1 et 2 sont créés (voir `type` colonne). -2. Si une erreur s’est produite pendant le traitement de la requête, deux événements avec les types 1 et 4 sont créés. -3. Si une erreur s’est produite avant le lancement de la requête, un seul événement de type 3 est créé. +1. Si l'exécution de la requête est réussie, deux événements de type 1 et 2 sont créés (voir `type` colonne). +2. Si une erreur s'est produite pendant le traitement de la requête, deux événements avec les types 1 et 4 sont créés. +3. Si une erreur s'est produite avant le lancement de la requête, un seul événement de type 3 est créé. Par défaut, les journaux sont ajoutés à la table à des intervalles de 7,5 secondes. Vous pouvez définir cet intervalle dans la [query\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) configuration du serveur (voir `flush_interval_milliseconds` paramètre). Pour vider les journaux de force du tampon mémoire dans la table, utilisez le `SYSTEM FLUSH LOGS` requête. @@ -607,9 +647,9 @@ Vous pouvez spécifier une clé de partitionnement arbitraire pour `system.query ## système.query\_thread\_log {#system_tables-query-thread-log} -La table contient des informations sur chaque thread d’exécution de requête. +La table contient des informations sur chaque thread d'exécution de requête. -Clickhouse crée cette table uniquement si [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-thread-log) serveur paramètre est spécifié. Ce paramètre définit les règles de journalisation, tels que l’intervalle d’enregistrement ou le nom de la table, la requête sera connecté. +Clickhouse crée cette table uniquement si [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-thread-log) serveur paramètre est spécifié. Ce paramètre définit les règles de journalisation, tels que l'intervalle d'enregistrement ou le nom de la table, la requête sera connecté. Pour activer la journalisation des requêtes, définissez [log\_query\_threads](settings/settings.md#settings-log-query-threads) paramètre 1. Pour plus de détails, voir le [Paramètre](settings/settings.md) section. @@ -622,7 +662,7 @@ Colonne: - `read_rows` (UInt64) — Number of read rows. - `read_bytes` (UInt64) — Number of read bytes. - `written_rows` (UInt64) — For `INSERT` des requêtes, le nombre de lignes. Pour les autres requêtes, la valeur de la colonne est 0. -- `written_bytes` (UInt64) — For `INSERT` des requêtes, le nombre d’octets écrits. Pour les autres requêtes, la valeur de la colonne est 0. +- `written_bytes` (UInt64) — For `INSERT` des requêtes, le nombre d'octets écrits. Pour les autres requêtes, la valeur de la colonne est 0. - `memory_usage` (Int64) — The difference between the amount of allocated and freed memory in context of this thread. - `peak_memory_usage` (Int64) — The maximum difference between the amount of allocated and freed memory in context of this thread. - `thread_name` (String) — Name of the thread. @@ -644,7 +684,7 @@ Colonne: - `interface` (UInt8) — Interface that the query was initiated from. Possible values: - 1 — TCP. - 2 — HTTP. -- `os_user` (String) — OS’s username who runs [clickhouse-client](../interfaces/cli.md). +- `os_user` (String) — OS's username who runs [clickhouse-client](../interfaces/cli.md). - `client_hostname` (String) — Hostname of the client machine where the [clickhouse-client](../interfaces/cli.md) ou un autre client TCP est exécuté. - `client_name` (String) — The [clickhouse-client](../interfaces/cli.md) ou un autre nom de client TCP. - `client_revision` (UInt32) — Revision of the [clickhouse-client](../interfaces/cli.md) ou un autre client TCP. @@ -672,32 +712,34 @@ Vous pouvez spécifier une clé de partitionnement arbitraire pour `system.query ## système.trace\_log {#system_tables-trace_log} -Contient des traces de pile collectées par le profileur de requête d’échantillonnage. +Contient des traces de pile collectées par le profileur de requête d'échantillonnage. -Clickhouse crée cette table lorsque le [trace\_log](server-configuration-parameters/settings.md#server_configuration_parameters-trace_log) la section de configuration du serveur est définie. Aussi l’ [query\_profiler\_real\_time\_period\_ns](settings/settings.md#query_profiler_real_time_period_ns) et [query\_profiler\_cpu\_time\_period\_ns](settings/settings.md#query_profiler_cpu_time_period_ns) paramètres doivent être définis. +Clickhouse crée cette table lorsque le [trace\_log](server-configuration-parameters/settings.md#server_configuration_parameters-trace_log) la section de configuration du serveur est définie. Aussi l' [query\_profiler\_real\_time\_period\_ns](settings/settings.md#query_profiler_real_time_period_ns) et [query\_profiler\_cpu\_time\_period\_ns](settings/settings.md#query_profiler_cpu_time_period_ns) paramètres doivent être définis. -Pour analyser les journaux, utilisez `addressToLine`, `addressToSymbol` et `demangle` fonctions d’introspection. +Pour analyser les journaux, utilisez `addressToLine`, `addressToSymbol` et `demangle` fonctions d'introspection. Colonne: -- `event_date`([Date](../sql-reference/data-types/date.md)) — Date of sampling moment. +- `event_date` ([Date](../sql-reference/data-types/date.md)) — Date of sampling moment. -- `event_time`([DateTime](../sql-reference/data-types/datetime.md)) — Timestamp of sampling moment. +- `event_time` ([DateTime](../sql-reference/data-types/datetime.md)) — Timestamp of the sampling moment. -- `revision`([UInt32](../sql-reference/data-types/int-uint.md)) — ClickHouse server build revision. +- `timestamp_ns` ([UInt64](../sql-reference/data-types/int-uint.md)) — Timestamp of the sampling moment in nanoseconds. - Lors de la connexion au serveur par `clickhouse-client`, vous voyez la chaîne similaire à `Connected to ClickHouse server version 19.18.1 revision 54429.`. Ce champ contient le `revision` mais pas le `version` d’un serveur. +- `revision` ([UInt32](../sql-reference/data-types/int-uint.md)) — ClickHouse server build revision. -- `timer_type`([Enum8](../sql-reference/data-types/enum.md)) — Timer type: + Lors de la connexion au serveur par `clickhouse-client`, vous voyez la chaîne similaire à `Connected to ClickHouse server version 19.18.1 revision 54429.`. Ce champ contient le `revision` mais pas le `version` d'un serveur. - - `Real` représente l’horloge murale. +- `timer_type` ([Enum8](../sql-reference/data-types/enum.md)) — Timer type: + + - `Real` représente l'horloge murale. - `CPU` représente le temps CPU. -- `thread_number`([UInt32](../sql-reference/data-types/int-uint.md)) — Thread identifier. +- `thread_number` ([UInt32](../sql-reference/data-types/int-uint.md)) — Thread identifier. -- `query_id`([Chaîne](../sql-reference/data-types/string.md)) — Query identifier that can be used to get details about a query that was running from the [query\_log](#system_tables-query_log) système de table. +- `query_id` ([Chaîne](../sql-reference/data-types/string.md)) — Query identifier that can be used to get details about a query that was running from the [query\_log](#system_tables-query_log) système de table. -- `trace`([Tableau (UInt64)](../sql-reference/data-types/array.md)) — Stack trace at the moment of sampling. Each element is a virtual memory address inside ClickHouse server process. +- `trace` ([Tableau (UInt64)](../sql-reference/data-types/array.md)) — Stack trace at the moment of sampling. Each element is a virtual memory address inside ClickHouse server process. **Exemple** @@ -719,7 +761,7 @@ trace: [94222141367858,94222152240175,94222152325351,94222152329944,9422 ## système.réplique {#system_tables-replicas} -Contient des informations et l’état des tables répliquées résidant sur le serveur local. +Contient des informations et l'état des tables répliquées résidant sur le serveur local. Ce tableau peut être utilisé pour la surveillance. La table contient une ligne pour chaque Répliqué\* table. Exemple: @@ -772,35 +814,35 @@ Colonne: - `table` (`String`)- Nom de la Table - `engine` (`String`)- Nom du moteur de Table - `is_leader` (`UInt8`) - Si la réplique est le chef de file. - Une seule réplique à la fois peut être le leader. Le leader est responsable de la sélection des fusions d’arrière-plan à effectuer. - Notez que les Écritures peuvent être effectuées sur n’importe quel réplica disponible et ayant une session dans ZK, qu’il s’agisse d’un leader. + Une seule réplique à la fois peut être le leader. Le leader est responsable de la sélection des fusions d'arrière-plan à effectuer. + Notez que les Écritures peuvent être effectuées sur n'importe quel réplica disponible et ayant une session dans ZK, qu'il s'agisse d'un leader. - `can_become_leader` (`UInt8`)- Si la réplique peut être élue en tant que leader. - `is_readonly` (`UInt8`) - Si la réplique est en mode lecture seule. - Ce mode est activé si la configuration n’a pas de sections avec ZooKeeper, si une erreur inconnue s’est produite lors de la réinitialisation des sessions dans ZooKeeper et lors de la réinitialisation des sessions dans ZooKeeper. + Ce mode est activé si la configuration n'a pas de sections avec ZooKeeper, si une erreur inconnue s'est produite lors de la réinitialisation des sessions dans ZooKeeper et lors de la réinitialisation des sessions dans ZooKeeper. - `is_session_expired` (`UInt8`)- la session avec ZooKeeper a expiré. Fondamentalement le même que `is_readonly`. -- `future_parts` (`UInt32`)- Le nombre de parties de données qui apparaîtront à la suite D’insertions ou de fusions qui n’ont pas encore été effectuées. -- `parts_to_check` (`UInt32`) - Le nombre de parties des données dans la file d’attente pour la vérification. Une pièce est placée dans la file d’attente de vérification s’il y a un soupçon qu’elle pourrait être endommagée. -- `zookeeper_path` (`String`)- Chemin d’accès aux données de la table dans ZooKeeper. -- `replica_name` (`String`) - Réplique nom de la Gardienne. Différentes répliques d’une même table ont des noms différents. +- `future_parts` (`UInt32`)- Le nombre de parties de données qui apparaîtront à la suite D'insertions ou de fusions qui n'ont pas encore été effectuées. +- `parts_to_check` (`UInt32`) - Le nombre de parties des données dans la file d'attente pour la vérification. Une pièce est placée dans la file d'attente de vérification s'il y a un soupçon qu'elle pourrait être endommagée. +- `zookeeper_path` (`String`)- Chemin d'accès aux données de la table dans ZooKeeper. +- `replica_name` (`String`) - Réplique nom de la Gardienne. Différentes répliques d'une même table ont des noms différents. - `replica_path` (`String`)- Chemin vers les données de réplique dans ZooKeeper. La même chose que la concaténation ‘zookeeper\_path/replicas/replica\_path’. -- `columns_version` (`Int32`)- Numéro de Version de la structure de la table. Indique combien de fois ALTER a été effectué. Si les répliques ont des versions différentes, cela signifie que certaines répliques n’ont pas encore Toutes les modifications. -- `queue_size` (`UInt32`),- La taille de la file d’attente pour les opérations en attente d’être exécuté. Les opérations comprennent l’insertion de blocs de données, les fusions et certaines autres actions. Il coïncide généralement avec `future_parts`. -- `inserts_in_queue` (`UInt32`) - Nombre d’insertions de blocs de données qui doivent être faits. Les Insertions sont généralement répliquées assez rapidement. Si ce nombre est grand, cela signifie que quelque chose est faux. -- `merges_in_queue` (`UInt32`) - Le nombre de fusions en attente d’être fait. Parfois, les fusions sont longues, donc cette valeur peut être supérieure à zéro pendant une longue période. -- `part_mutations_in_queue` (`UInt32`) - Le nombre de mutations en attente d’être fait. -- `queue_oldest_time` (`DateTime`) - Si `queue_size` supérieur à 0, indique quand l’opération la plus ancienne a été ajoutée à la file d’attente. +- `columns_version` (`Int32`)- Numéro de Version de la structure de la table. Indique combien de fois ALTER a été effectué. Si les répliques ont des versions différentes, cela signifie que certaines répliques n'ont pas encore Toutes les modifications. +- `queue_size` (`UInt32`),- La taille de la file d'attente pour les opérations en attente d'être exécuté. Les opérations comprennent l'insertion de blocs de données, les fusions et certaines autres actions. Il coïncide généralement avec `future_parts`. +- `inserts_in_queue` (`UInt32`) - Nombre d'insertions de blocs de données qui doivent être faits. Les Insertions sont généralement répliquées assez rapidement. Si ce nombre est grand, cela signifie que quelque chose est faux. +- `merges_in_queue` (`UInt32`) - Le nombre de fusions en attente d'être fait. Parfois, les fusions sont longues, donc cette valeur peut être supérieure à zéro pendant une longue période. +- `part_mutations_in_queue` (`UInt32`) - Le nombre de mutations en attente d'être fait. +- `queue_oldest_time` (`DateTime`) - Si `queue_size` supérieur à 0, indique quand l'opération la plus ancienne a été ajoutée à la file d'attente. - `inserts_oldest_time` (`DateTime`) - Voir `queue_oldest_time` - `merges_oldest_time` (`DateTime`) - Voir `queue_oldest_time` - `part_mutations_oldest_time` (`DateTime`) - Voir `queue_oldest_time` -Les 4 colonnes suivantes ont une valeur non nulle uniquement lorsqu’il y a une session active avec ZK. +Les 4 colonnes suivantes ont une valeur non nulle uniquement lorsqu'il y a une session active avec ZK. -- `log_max_index` (`UInt64`) - Maximum nombre d’entrées dans le journal de l’activité générale. -- `log_pointer` (`UInt64`)- Numéro d’entrée Maximum dans le journal de l’activité générale que le réplica a copié dans sa file d’attente d’exécution, plus un. Si `log_pointer` est beaucoup plus petite que `log_max_index` quelque chose ne va pas. -- `last_queue_update` (`DateTime`) - Lorsque la file d’attente a été mise à jour la dernière fois. +- `log_max_index` (`UInt64`) - Maximum nombre d'entrées dans le journal de l'activité générale. +- `log_pointer` (`UInt64`)- Numéro d'entrée Maximum dans le journal de l'activité générale que le réplica a copié dans sa file d'attente d'exécution, plus un. Si `log_pointer` est beaucoup plus petite que `log_max_index` quelque chose ne va pas. +- `last_queue_update` (`DateTime`) - Lorsque la file d'attente a été mise à jour la dernière fois. - `absolute_delay` (`UInt64`)- Combien de décalage en secondes la réplique actuelle A. - `total_replicas` (`UInt8`) - Le nombre total de répliques connues de ce tableau. -- `active_replicas` (`UInt8`) - Le nombre de répliques de cette table qui ont une session dans ZooKeeper (c’est-à-dire le nombre de répliques fonctionnelles). +- `active_replicas` (`UInt8`) - Le nombre de répliques de cette table qui ont une session dans ZooKeeper (c'est-à-dire le nombre de répliques fonctionnelles). Si vous demandez toutes les colonnes, la table peut fonctionner un peu lentement, car plusieurs lectures de ZooKeeper sont faites pour chaque ligne. Si vous ne demandez pas les 4 dernières colonnes (log\_max\_index, log\_pointer, total\_replicas, active\_replicas), la table fonctionne rapidement. @@ -839,30 +881,59 @@ WHERE Si cette requête ne retourne rien, cela signifie que tout va bien. -## système.paramètre {#system-settings} +## système.paramètre {#system-tables-system-settings} -Contient des informations sur les paramètres actuellement utilisés. -I. e. utilisé pour l’exécution de la requête que vous utilisez pour lire à partir du système.les paramètres de la table. +Contient des informations sur les paramètres de session pour l'utilisateur actuel. Colonne: -- `name` (String) — Setting name. -- `value` (String) — Setting value. -- `description` (String) — Setting description. -- `type` (String) — Setting type (implementation specific string value). -- `changed` (UInt8) — Whether the setting was explicitly defined in the config or explicitly changed. -- `min` (Nullable(String)) — Get minimum allowed value (if any is set via [contraintes](settings/constraints-on-settings.md#constraints-on-settings)). -- `max` (Nullable(String)) — Get maximum allowed value (if any is set via [contraintes](settings/constraints-on-settings.md#constraints-on-settings)). -- `readonly` (UInt8) — Can user change this setting (for more info, look into [contraintes](settings/constraints-on-settings.md#constraints-on-settings)). +- `name` ([Chaîne](../sql-reference/data-types/string.md)) — Setting name. +- `value` ([Chaîne](../sql-reference/data-types/string.md)) — Setting value. +- `changed` ([UInt8](../sql-reference/data-types/int-uint.md#uint-ranges)) — Shows whether a setting is changed from its default value. +- `description` ([Chaîne](../sql-reference/data-types/string.md)) — Short setting description. +- `min` ([Nullable](../sql-reference/data-types/nullable.md)([Chaîne](../sql-reference/data-types/string.md))) — Minimum value of the setting, if any is set via [contraintes](settings/constraints-on-settings.md#constraints-on-settings). Si le réglage n'a pas de valeur minimale, contient [NULL](../sql-reference/syntax.md#null-literal). +- `max` ([Nullable](../sql-reference/data-types/nullable.md)([Chaîne](../sql-reference/data-types/string.md))) — Maximum value of the setting, if any is set via [contraintes](settings/constraints-on-settings.md#constraints-on-settings). Si le réglage n'a pas de valeur maximale, contient [NULL](../sql-reference/syntax.md#null-literal). +- `readonly` ([UInt8](../sql-reference/data-types/int-uint.md#uint-ranges)) — Shows whether the current user can change the setting: + - `0` — Current user can change the setting. + - `1` — Current user can't change the setting. -Exemple: +**Exemple** + +L'exemple suivant montre comment obtenir des informations sur les paramètres dont le nom contient `min_i`. ``` sql -SELECT name, value +SELECT * FROM system.settings -WHERE changed +WHERE name LIKE '%min_i%' ``` +``` text +┌─name────────────────────────────────────────┬─value─────┬─changed─┬─description───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┬─min──┬─max──┬─readonly─┐ +│ min_insert_block_size_rows │ 1048576 │ 0 │ Squash blocks passed to INSERT query to specified size in rows, if blocks are not big enough. │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ 0 │ +│ min_insert_block_size_bytes │ 268435456 │ 0 │ Squash blocks passed to INSERT query to specified size in bytes, if blocks are not big enough. │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ 0 │ +│ read_backoff_min_interval_between_events_ms │ 1000 │ 0 │ Settings to reduce the number of threads in case of slow reads. Do not pay attention to the event, if the previous one has passed less than a certain amount of time. │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ 0 │ +└─────────────────────────────────────────────┴───────────┴─────────┴───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┴──────┴──────┴──────────┘ +``` + +À l'aide de `WHERE changed` peut être utile, par exemple, lorsque vous voulez vérifier: + +- Indique si les paramètres des fichiers de configuration sont chargés correctement et sont utilisés. +- Paramètres modifiés dans la session en cours. + + + +``` sql +SELECT * FROM system.settings WHERE changed AND name='load_balancing' +``` + +**Voir aussi** + +- [Paramètre](settings/index.md#session-settings-intro) +- [Autorisations pour les requêtes](settings/permissions-for-queries.md#settings_readonly) +- [Contraintes sur les paramètres](settings/constraints-on-settings.md) + +## système.tableau\_moteurs {#system.table_engines} + ``` text ┌─name───────────────────┬─value───────┐ │ max_threads │ 8 │ @@ -934,9 +1005,9 @@ Ce tableau contient les colonnes suivantes (le type de colonne est indiqué entr - `is_temporary` (UInt8) - indicateur qui indique si la table est temporaire. -- `data_path` (Chaîne) - chemin d’accès aux données de la table dans le système de fichiers. +- `data_path` (Chaîne) - chemin d'accès aux données de la table dans le système de fichiers. -- `metadata_path` (Chaîne) - chemin d’accès aux métadonnées de la table dans le système de fichiers. +- `metadata_path` (Chaîne) - chemin d'accès aux métadonnées de la table dans le système de fichiers. - `metadata_modification_time` (DateTime) - Heure de la dernière modification des métadonnées de la table. @@ -948,36 +1019,36 @@ Ce tableau contient les colonnes suivantes (le type de colonne est indiqué entr - `engine_full` (Chaîne) - paramètres du moteur de table. -- `partition_key` (String) - l’expression de clé de partition spécifiée dans le tableau. +- `partition_key` (String) - l'expression de clé de partition spécifiée dans le tableau. -- `sorting_key` (String) - l’expression de clé de tri spécifiée dans la table. +- `sorting_key` (String) - l'expression de clé de tri spécifiée dans la table. -- `primary_key` (String) - l’expression de clé primaire spécifiée dans la table. +- `primary_key` (String) - l'expression de clé primaire spécifiée dans la table. -- `sampling_key` (String) - l’expression de clé d’échantillonnage spécifiée dans la table. +- `sampling_key` (String) - l'expression de clé d'échantillonnage spécifiée dans la table. - `storage_policy` (String) - La politique de stockage: - [MergeTree](../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes) - [Distribué](../engines/table-engines/special/distributed.md#distributed) -- `total_rows` (Nullable (UInt64)) - nombre Total de lignes, s’il est possible de déterminer rapidement le nombre exact de lignes dans la table, sinon `Null` (y compris underying `Buffer` table). +- `total_rows` (Nullable (UInt64)) - nombre Total de lignes, s'il est possible de déterminer rapidement le nombre exact de lignes dans la table, sinon `Null` (y compris underying `Buffer` table). -- `total_bytes` (Nullable (UInt64)) - nombre Total d’octets, s’il est possible de déterminer rapidement le nombre exact d’octets pour la table sur le stockage, sinon `Null` (**ne pas** comprend tout de stockage sous-jacent). +- `total_bytes` (Nullable (UInt64)) - nombre Total d'octets, s'il est possible de déterminer rapidement le nombre exact d'octets pour la table sur le stockage, sinon `Null` (**ne pas** comprend tout de stockage sous-jacent). - If the table stores data on disk, returns used space on disk (i.e. compressed). - - Si la table stocke des données en mémoire, renvoie un nombre approximatif d’octets utilisés en mémoire. + - Si la table stocke des données en mémoire, renvoie un nombre approximatif d'octets utilisés en mémoire. Le `system.tables` le tableau est utilisé dans `SHOW TABLES` implémentation de requête. ## système.zookeeper {#system-zookeeper} -La table n’existe pas si ZooKeeper n’est pas configuré. Permet de lire les données du cluster Zookeeper défini dans la configuration. -La requête doit avoir un ‘path’ condition d’égalité dans la clause WHERE. C’est le chemin dans ZooKeeper pour les enfants pour lesquels vous souhaitez obtenir des données. +La table n'existe pas si ZooKeeper n'est pas configuré. Permet de lire les données du cluster Zookeeper défini dans la configuration. +La requête doit avoir un ‘path’ condition d'égalité dans la clause WHERE. C'est le chemin dans ZooKeeper pour les enfants pour lesquels vous souhaitez obtenir des données. Requête `SELECT * FROM system.zookeeper WHERE path = '/clickhouse'` données de sortie pour tous les enfants `/clickhouse` nœud. Pour générer des données pour tous les nœuds racine, écrivez path = ‘/’. -Si le chemin d’accès spécifié dans ‘path’ n’existe pas, une exception sera levée. +Si le chemin d'accès spécifié dans ‘path’ n'existe pas, une exception sera levée. Colonne: @@ -1047,25 +1118,25 @@ Le tableau contient des informations sur [mutation](../sql-reference/statements/ **base de données**, **table** - Le nom de la base de données et de la table à laquelle la mutation a été appliquée. -**mutation\_id** - Le numéro d’identification de la mutation. Pour les tables répliquées ces ID correspondent aux noms znode dans le `/mutations/` répertoire de la Gardienne. Pour les tables non compliquées, Les Id correspondent aux noms de fichiers dans le répertoire de données de la table. +**mutation\_id** - Le numéro d'identification de la mutation. Pour les tables répliquées ces ID correspondent aux noms znode dans le `/mutations/` répertoire de la Gardienne. Pour les tables non compliquées, Les Id correspondent aux noms de fichiers dans le répertoire de données de la table. **commande** - La chaîne de commande mutation (la partie de la requête après `ALTER TABLE [db.]table`). **create\_time** - Quand cette commande de mutation a été soumise pour exécution. -**block\_numbers.partition\_id**, **block\_numbers.nombre** - Une colonne imbriquée. Pour les mutations de tables répliquées, il contient un enregistrement pour chaque partition: l’ID de partition et le numéro de bloc acquis par la mutation (dans chaque partition, seules les parties contenant des blocs avec des nombres inférieurs au numéro de bloc acquis par la mutation dans cette partition seront mutées). Dans les tables non répliquées, les numéros de bloc de toutes les partitions forment une seule séquence. Cela signifie que pour les mutations de tables non répliquées, la colonne contiendra un enregistrement avec un seul numéro de bloc acquis par la mutation. +**block\_numbers.partition\_id**, **block\_numbers.nombre** - Une colonne imbriquée. Pour les mutations de tables répliquées, il contient un enregistrement pour chaque partition: l'ID de partition et le numéro de bloc acquis par la mutation (dans chaque partition, seules les parties contenant des blocs avec des nombres inférieurs au numéro de bloc acquis par la mutation dans cette partition seront mutées). Dans les tables non répliquées, les numéros de bloc de toutes les partitions forment une seule séquence. Cela signifie que pour les mutations de tables non répliquées, la colonne contiendra un enregistrement avec un seul numéro de bloc acquis par la mutation. **parts\_to\_do** - Le nombre de parties de données qui doivent être mutées pour que la mutation se termine. -**\_done** - La mutation est faite? Notez que même si `parts_to_do = 0` il est possible qu’une mutation d’une table répliquée ne soit pas encore effectuée en raison d’un INSERT de longue durée qui créera une nouvelle partie de données qui devra être mutée. +**\_done** - La mutation est faite? Notez que même si `parts_to_do = 0` il est possible qu'une mutation d'une table répliquée ne soit pas encore effectuée en raison d'un INSERT de longue durée qui créera une nouvelle partie de données qui devra être mutée. -S’il y avait des problèmes avec la mutation de certaines parties, les colonnes suivantes contiennent des informations supplémentaires: +S'il y avait des problèmes avec la mutation de certaines parties, les colonnes suivantes contiennent des informations supplémentaires: -**latest\_failed\_part** - Le nom de la partie la plus récente qui n’a pas pu être mutée. +**latest\_failed\_part** - Le nom de la partie la plus récente qui n'a pas pu être mutée. -**latest\_fail\_time** - Le temps de la partie la plus récente mutation de l’échec. +**latest\_fail\_time** - Le temps de la partie la plus récente mutation de l'échec. -**latest\_fail\_reason** - Le message d’exception qui a provoqué l’échec de la mutation de pièce la plus récente. +**latest\_fail\_reason** - Le message d'exception qui a provoqué l'échec de la mutation de pièce la plus récente. ## système.disque {#system_tables-disks} @@ -1092,6 +1163,6 @@ Colonne: - `max_data_part_size` ([UInt64](../sql-reference/data-types/int-uint.md)) — Maximum size of a data part that can be stored on volume disks (0 — no limit). - `move_factor` ([Float64](../sql-reference/data-types/float.md)) — Ratio of free disk space. When the ratio exceeds the value of configuration parameter, ClickHouse start to move data to the next volume in order. -Si la stratégie de stockage contient plus d’un volume, les informations pour chaque volume sont stockées dans la ligne individuelle de la table. +Si la stratégie de stockage contient plus d'un volume, les informations pour chaque volume sont stockées dans la ligne individuelle de la table. [Article Original](https://clickhouse.tech/docs/en/operations/system_tables/) diff --git a/docs/fr/operations/tips.md b/docs/fr/operations/tips.md index 7cc94af0a10..a4fe5f844de 100644 --- a/docs/fr/operations/tips.md +++ b/docs/fr/operations/tips.md @@ -1,15 +1,15 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 58 toc_title: Recommandations D'Utilisation --- -# Recommandations D’Utilisation {#usage-recommendations} +# Recommandations D'Utilisation {#usage-recommendations} -## Gouverneur De Mise à L’échelle Du Processeur {#cpu-scaling-governor} +## Gouverneur de mise à L'échelle du processeur {#cpu-scaling-governor} -Utilisez toujours la `performance` mise à l’échelle gouverneur. Le `on-demand` gouverneur de mise à l’échelle fonctionne bien pire avec une demande constamment élevée. +Utilisez toujours la `performance` mise à l'échelle gouverneur. Le `on-demand` gouverneur de mise à l'échelle fonctionne bien pire avec une demande constamment élevée. ``` bash $ echo 'performance' | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor @@ -17,14 +17,14 @@ $ echo 'performance' | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_gov ## CPU Limitations {#cpu-limitations} -Les processeurs peuvent surchauffer. Utiliser `dmesg` pour voir si la fréquence D’horloge du processeur était limitée en raison de la surchauffe. +Les processeurs peuvent surchauffer. Utiliser `dmesg` pour voir si la fréquence D'horloge du processeur était limitée en raison de la surchauffe. La restriction peut également être définie en externe au niveau du centre de données. Vous pouvez utiliser `turbostat` à surveiller sous une charge. ## RAM {#ram} -Pour de petites quantités de données (jusqu’à ~200 GO en mode compressé), il est préférable d’utiliser autant de mémoire que le volume de données. -Pour de grandes quantités de données et lors du traitement de requêtes interactives (en ligne), vous devez utiliser une quantité raisonnable de RAM (128 Go ou plus) afin que le sous-ensemble de données chaudes s’intègre dans le cache des pages. -Même pour des volumes de données d’environ 50 To par serveur, l’utilisation de 128 Go de RAM améliore considérablement les performances des requêtes par rapport à 64 Go. +Pour de petites quantités de données (jusqu'à ~200 GO en mode compressé), il est préférable d'utiliser autant de mémoire que le volume de données. +Pour de grandes quantités de données et lors du traitement de requêtes interactives (en ligne), vous devez utiliser une quantité raisonnable de RAM (128 Go ou plus) afin que le sous-ensemble de données chaudes s'intègre dans le cache des pages. +Même pour des volumes de données d'environ 50 To par serveur, l'utilisation de 128 Go de RAM améliore considérablement les performances des requêtes par rapport à 64 Go. Ne désactivez pas de surcharge. Valeur `cat /proc/sys/vm/overcommit_memory` devrait être 0 ou 1. Exécuter @@ -41,12 +41,12 @@ $ echo 'never' | sudo tee /sys/kernel/mm/transparent_hugepage/enabled ``` Utiliser `perf top` pour regarder le temps passé dans le noyau pour la gestion de la mémoire. -Les pages énormes permanentes n’ont pas non plus besoin d’être allouées. +Les pages énormes permanentes n'ont pas non plus besoin d'être allouées. ## Sous-Système De Stockage {#storage-subsystem} -Si votre budget vous permet D’utiliser SSD, utilisez SSD. -Sinon, utilisez un disque dur. Disques durs SATA 7200 RPM fera l’affaire. +Si votre budget vous permet D'utiliser SSD, utilisez SSD. +Sinon, utilisez un disque dur. Disques durs SATA 7200 RPM fera l'affaire. Donner la préférence à un grand nombre de serveurs avec des disques durs locaux sur un plus petit nombre de serveurs avec un disque attaché étagères. Mais pour stocker des archives avec des requêtes rares, les étagères fonctionneront. @@ -54,12 +54,12 @@ Mais pour stocker des archives avec des requêtes rares, les étagères fonction ## RAID {#raid} Lorsque vous utilisez le disque dur, vous pouvez combiner leur RAID-10, RAID-5, RAID-6 ou RAID-50. -Pour Linux, le RAID logiciel est meilleur (avec `mdadm`). Nous ne recommandons pas d’utiliser LVM. +Pour Linux, le RAID logiciel est meilleur (avec `mdadm`). Nous ne recommandons pas d'utiliser LVM. Lors de la création de RAID-10, sélectionnez `far` disposition. Si votre budget le permet, choisissez RAID-10. Si vous avez plus de 4 disques, Utilisez RAID-6 (préféré) ou RAID-50, au lieu de RAID-5. -Lorsque vous utilisez RAID-5, RAID-6 ou RAID-50, augmentez toujours stripe\_cache\_size, car la valeur par défaut n’est généralement pas le meilleur choix. +Lorsque vous utilisez RAID-5, RAID-6 ou RAID-50, augmentez toujours stripe\_cache\_size, car la valeur par défaut n'est généralement pas le meilleur choix. ``` bash $ echo 4096 | sudo tee /sys/block/md2/md/stripe_cache_size @@ -71,43 +71,43 @@ Une taille de bloc de 1024 KO est suffisante pour toutes les configurations RAID Ne définissez jamais la taille du bloc trop petite ou trop grande. Vous pouvez utiliser RAID-0 sur SSD. -Quelle que soit L’utilisation du RAID, utilisez toujours la réplication pour la sécurité des données. +Quelle que soit L'utilisation du RAID, utilisez toujours la réplication pour la sécurité des données. -Activer NCQ avec une longue file d’attente. Pour HDD, choisissez le planificateur CFQ, et pour SSD, choisissez noop. Ne pas réduire le ‘readahead’ paramètre. -Pour le disque dur, activez le cache d’écriture. +Activer NCQ avec une longue file d'attente. Pour HDD, choisissez le planificateur CFQ, et pour SSD, choisissez noop. Ne pas réduire le ‘readahead’ paramètre. +Pour le disque dur, activez le cache d'écriture. ## Système De Fichiers {#file-system} -Ext4 est l’option la plus fiable. Définir les options de montage `noatime, nobarrier`. -XFS est également adapté, mais il n’a pas été aussi soigneusement testé avec ClickHouse. +Ext4 est l'option la plus fiable. Définir les options de montage `noatime, nobarrier`. +XFS est également adapté, mais il n'a pas été aussi soigneusement testé avec ClickHouse. La plupart des autres systèmes de fichiers devraient également fonctionner correctement. Les systèmes de fichiers avec allocation retardée fonctionnent mieux. ## Le Noyau Linux {#linux-kernel} -N’utilisez pas un noyau Linux obsolète. +N'utilisez pas un noyau Linux obsolète. ## Réseau {#network} Si vous utilisez IPv6, augmenter la taille du cache. -Le noyau Linux avant 3.2 avait une multitude de problèmes avec l’implémentation D’IPv6. +Le noyau Linux avant 3.2 avait une multitude de problèmes avec l'implémentation D'IPv6. Utilisez au moins un réseau de 10 Go, si possible. 1 Go fonctionnera également, mais ce sera bien pire pour patcher des répliques avec des dizaines de téraoctets de données, ou pour traiter des requêtes distribuées avec une grande quantité de données intermédiaires. ## ZooKeeper {#zookeeper} -Vous utilisez probablement déjà ZooKeeper à d’autres fins. Vous pouvez utiliser la même installation de ZooKeeper, si elle n’est pas déjà surchargée. +Vous utilisez probablement déjà ZooKeeper à d'autres fins. Vous pouvez utiliser la même installation de ZooKeeper, si elle n'est pas déjà surchargée. -It’s best to use a fresh version of ZooKeeper – 3.4.9 or later. The version in stable Linux distributions may be outdated. +It's best to use a fresh version of ZooKeeper – 3.4.9 or later. The version in stable Linux distributions may be outdated. -Vous ne devez jamais utiliser de scripts écrits manuellement pour transférer des données entre différents clusters ZooKeeper, car le résultat sera incorrect pour les nœuds séquentiels. Ne jamais utiliser de l’ “zkcopy” utilitaire pour la même raison: https://github.com/ksprojects/zkcopy/issues/15 +Vous ne devez jamais utiliser de scripts écrits manuellement pour transférer des données entre différents clusters ZooKeeper, car le résultat sera incorrect pour les nœuds séquentiels. Ne jamais utiliser de l' “zkcopy” utilitaire pour la même raison: https://github.com/ksprojects/zkcopy/issues/15 -Si vous souhaitez diviser un cluster Zookeeper existant en deux, le bon moyen est d’augmenter le nombre de ses répliques, puis de le reconfigurer en deux clusters indépendants. +Si vous souhaitez diviser un cluster Zookeeper existant en deux, le bon moyen est d'augmenter le nombre de ses répliques, puis de le reconfigurer en deux clusters indépendants. -N’exécutez pas ZooKeeper sur les mêmes serveurs que ClickHouse. Parce que ZooKeeper est très sensible à la latence et ClickHouse peut utiliser toutes les ressources système disponibles. +N'exécutez pas ZooKeeper sur les mêmes serveurs que ClickHouse. Parce que ZooKeeper est très sensible à la latence et ClickHouse peut utiliser toutes les ressources système disponibles. Avec les paramètres par défaut, ZooKeeper est une bombe à retardement: -> Le serveur ZooKeeper ne supprime pas les fichiers des anciens snapshots et journaux lors de l’utilisation de la configuration par défaut (voir autopurge), et c’est la responsabilité de l’opérateur. +> Le serveur ZooKeeper ne supprime pas les fichiers des anciens snapshots et journaux lors de l'utilisation de la configuration par défaut (voir autopurge), et c'est la responsabilité de l'opérateur. Cette bombe doit être désamorcée. diff --git a/docs/fr/operations/troubleshooting.md b/docs/fr/operations/troubleshooting.md index 206f383ffe9..0608569c61c 100644 --- a/docs/fr/operations/troubleshooting.md +++ b/docs/fr/operations/troubleshooting.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 46 toc_title: "D\xE9pannage" --- @@ -14,19 +14,19 @@ toc_title: "D\xE9pannage" ## Installation {#troubleshooting-installation-errors} -### Vous Ne Pouvez Pas Obtenir De Paquets Deb à Partir Du référentiel Clickhouse Avec Apt-get {#you-cannot-get-deb-packages-from-clickhouse-repository-with-apt-get} +### Vous ne pouvez pas obtenir de paquets deb à partir du référentiel ClickHouse avec Apt-get {#you-cannot-get-deb-packages-from-clickhouse-repository-with-apt-get} - Vérifiez les paramètres du pare-feu. -- Si vous ne pouvez pas accéder au référentiel pour quelque raison que ce soit, téléchargez les packages comme décrit dans [Prise en main](../getting-started/index.md) article et les installer manuellement en utilisant le `sudo dpkg -i ` commande. Vous aurez aussi besoin d’ `tzdata` paquet. +- Si vous ne pouvez pas accéder au référentiel pour quelque raison que ce soit, téléchargez les packages comme décrit dans [Prise en main](../getting-started/index.md) article et les installer manuellement en utilisant le `sudo dpkg -i ` commande. Vous aurez aussi besoin d' `tzdata` paquet. -## Connexion Au Serveur {#troubleshooting-accepts-no-connections} +## Connexion au Serveur {#troubleshooting-accepts-no-connections} Problèmes possibles: -- Le serveur n’est pas en cours d’exécution. +- Le serveur n'est pas en cours d'exécution. - Paramètres de configuration inattendus ou incorrects. -### Le Serveur N’Est Pas En Cours D’Exécution {#server-is-not-running} +### Le Serveur N'Est Pas En Cours D'Exécution {#server-is-not-running} **Vérifiez si le serveur est runnnig** @@ -36,7 +36,7 @@ Commande: $ sudo service clickhouse-server status ``` -Si le serveur n’est pas en cours d’exécution, démarrez-le avec la commande: +Si le serveur n'est pas en cours d'exécution, démarrez-le avec la commande: ``` bash $ sudo service clickhouse-server start @@ -51,13 +51,13 @@ Si le serveur a démarré avec succès, vous devriez voir les chaînes: - ` Application: starting up.` — Server started. - ` Application: Ready for connections.` — Server is running and ready for connections. -Si `clickhouse-server` démarrage a échoué avec une erreur de configuration, vous devriez voir la `` chaîne avec une description de l’erreur. Exemple: +Si `clickhouse-server` démarrage a échoué avec une erreur de configuration, vous devriez voir la `` chaîne avec une description de l'erreur. Exemple: ``` text 2019.01.11 15:23:25.549505 [ 45 ] {} ExternalDictionaries: Failed reloading 'event2id' external dictionary: Poco::Exception. Code: 1000, e.code() = 111, e.displayText() = Connection refused, e.what() = Connection refused ``` -Si vous ne voyez pas d’erreur à la fin du fichier, parcourez le fichier entier à partir de la chaîne: +Si vous ne voyez pas d'erreur à la fin du fichier, parcourez le fichier entier à partir de la chaîne: ``` text Application: starting up. @@ -81,7 +81,7 @@ Revision: 54413 **Voir système.d les journaux** -Si vous ne trouvez aucune information utile dans `clickhouse-server` journaux ou il n’y a pas de journaux, vous pouvez afficher `system.d` journaux à l’aide de la commande: +Si vous ne trouvez aucune information utile dans `clickhouse-server` journaux ou il n'y a pas de journaux, vous pouvez afficher `system.d` journaux à l'aide de la commande: ``` bash $ sudo journalctl -u clickhouse-server @@ -93,7 +93,7 @@ $ sudo journalctl -u clickhouse-server $ sudo -u clickhouse /usr/bin/clickhouse-server --config-file /etc/clickhouse-server/config.xml ``` -Cette commande démarre le serveur en tant qu’application interactive avec les paramètres standard du script de démarrage automatique. Dans ce mode `clickhouse-server` imprime tous les messages d’événement dans la console. +Cette commande démarre le serveur en tant qu'application interactive avec les paramètres standard du script de démarrage automatique. Dans ce mode `clickhouse-server` imprime tous les messages d'événement dans la console. ### Paramètres De Configuration {#configuration-parameters} @@ -111,7 +111,7 @@ Vérifier: - Paramètres du protocole HTTP. - Vérifiez les paramètres de protocole pour L’API HTTP. + Vérifiez les paramètres de protocole pour L'API HTTP. - Paramètres de connexion sécurisés. @@ -120,27 +120,27 @@ Vérifier: - Le [tcp\_port\_secure](server-configuration-parameters/settings.md#server_configuration_parameters-tcp_port_secure) paramètre. - Paramètres pour [SSL sertificates](server-configuration-parameters/settings.md#server_configuration_parameters-openssl). - Utilisez les paramètres appropriés lors de la connexion. Par exemple, l’utilisation de la `port_secure` paramètre avec `clickhouse_client`. + Utilisez les paramètres appropriés lors de la connexion. Par exemple, l'utilisation de la `port_secure` paramètre avec `clickhouse_client`. -- Les paramètres de l’utilisateur. +- Les paramètres de l'utilisateur. - Vous utilisez peut-être un mauvais nom d’utilisateur ou mot de passe. + Vous utilisez peut-être un mauvais nom d'utilisateur ou mot de passe. ## Traitement Des Requêtes {#troubleshooting-does-not-process-queries} -Si ClickHouse ne peut pas traiter la requête, il envoie une description d’erreur au client. Dans le `clickhouse-client` vous obtenez une description de l’erreur dans la console. Si vous utilisez L’interface HTTP, ClickHouse envoie la description de l’erreur dans le corps de la réponse. Exemple: +Si ClickHouse ne peut pas traiter la requête, il envoie une description d'erreur au client. Dans le `clickhouse-client` vous obtenez une description de l'erreur dans la console. Si vous utilisez L'interface HTTP, ClickHouse envoie la description de l'erreur dans le corps de la réponse. Exemple: ``` bash $ curl 'http://localhost:8123/' --data-binary "SELECT a" Code: 47, e.displayText() = DB::Exception: Unknown identifier: a. Note that there are no tables (FROM clause) in your query, context: required_names: 'a' source_tables: table_aliases: private_aliases: column_aliases: public_columns: 'a' masked_columns: array_join_columns: source_columns: , e.what() = DB::Exception ``` -Si vous commencez à `clickhouse-client` avec l’ `stack-trace` paramètre, ClickHouse renvoie la trace de la pile du serveur avec la description d’une erreur. +Si vous commencez à `clickhouse-client` avec l' `stack-trace` paramètre, ClickHouse renvoie la trace de la pile du serveur avec la description d'une erreur. Vous pouvez voir un message sur une connexion rompue. Dans ce cas, vous pouvez répéter la requête. Si la connexion se rompt chaque fois que vous effectuez la requête, vérifiez les journaux du serveur pour détecter les erreurs. -## Efficacité Du Traitement Des Requêtes {#troubleshooting-too-slow} +## Efficacité du traitement des requêtes {#troubleshooting-too-slow} Si vous voyez que ClickHouse fonctionne trop lentement, vous devez profiler la charge sur les ressources du serveur et le réseau pour vos requêtes. -Vous pouvez utiliser l’utilitaire clickhouse-benchmark pour profiler les requêtes. Il indique le nombre de requêtes traitées par seconde, le nombre de lignes traitées par seconde, et les percentiles de temps de traitement des requêtes. +Vous pouvez utiliser l'utilitaire clickhouse-benchmark pour profiler les requêtes. Il indique le nombre de requêtes traitées par seconde, le nombre de lignes traitées par seconde, et les percentiles de temps de traitement des requêtes. diff --git a/docs/fr/operations/update.md b/docs/fr/operations/update.md index f8df4ff478f..e1bfe57b0a3 100644 --- a/docs/fr/operations/update.md +++ b/docs/fr/operations/update.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 47 toc_title: "Mise \xC0 Jour De ClickHouse" --- @@ -17,4 +17,4 @@ $ sudo service clickhouse-server restart Si vous avez installé ClickHouse en utilisant autre chose que les paquets deb recommandés, utilisez la méthode de mise à jour appropriée. -ClickHouse ne prend pas en charge une mise à jour distribuée. L’opération doit être effectuée consécutivement sur chaque serveur séparé. Ne pas mettre à jour tous les serveurs d’un cluster simultanément, ou le cluster sera indisponible pendant un certain temps. +ClickHouse ne prend pas en charge une mise à jour distribuée. L'opération doit être effectuée consécutivement sur chaque serveur séparé. Ne pas mettre à jour tous les serveurs d'un cluster simultanément, ou le cluster sera indisponible pendant un certain temps. diff --git a/docs/fr/operations/utilities/clickhouse-benchmark.md b/docs/fr/operations/utilities/clickhouse-benchmark.md index 670aee97394..975cb09814e 100644 --- a/docs/fr/operations/utilities/clickhouse-benchmark.md +++ b/docs/fr/operations/utilities/clickhouse-benchmark.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 61 toc_title: clickhouse-benchmark --- @@ -38,20 +38,20 @@ clickhouse-benchmark [keys] < queries_file - `-c N`, `--concurrency=N` — Number of queries that `clickhouse-benchmark` envoie simultanément. Valeur par défaut: 1. - `-d N`, `--delay=N` — Interval in seconds between intermediate reports (set 0 to disable reports). Default value: 1. -- `-h WORD`, `--host=WORD` — Server host. Default value: `localhost`. Pour l’ [mode de comparaison](#clickhouse-benchmark-comparison-mode) vous pouvez utiliser plusieurs `-h` touches. +- `-h WORD`, `--host=WORD` — Server host. Default value: `localhost`. Pour l' [mode de comparaison](#clickhouse-benchmark-comparison-mode) vous pouvez utiliser plusieurs `-h` touches. - `-p N`, `--port=N` — Server port. Default value: 9000. For the [mode de comparaison](#clickhouse-benchmark-comparison-mode) vous pouvez utiliser plusieurs `-p` touches. - `-i N`, `--iterations=N` — Total number of queries. Default value: 0. - `-r`, `--randomize` — Random order of queries execution if there is more then one input query. - `-s`, `--secure` — Using TLS connection. -- `-t N`, `--timelimit=N` — Time limit in seconds. `clickhouse-benchmark` arrête d’envoyer des requêtes lorsque le délai spécifié est atteint. Valeur par défaut: 0 (limite de temps désactivée). -- `--confidence=N` — Level of confidence for T-test. Possible values: 0 (80%), 1 (90%), 2 (95%), 3 (98%), 4 (99%), 5 (99.5%). Default value: 5. In the [mode de comparaison](#clickhouse-benchmark-comparison-mode) `clickhouse-benchmark` effectue les [Test T de L’étudiant indépendant à deux échantillons](https://en.wikipedia.org/wiki/Student%27s_t-test#Independent_two-sample_t-test) tester pour déterminer si les deux distributions ne sont pas différentes avec le niveau de confiance sélectionné. +- `-t N`, `--timelimit=N` — Time limit in seconds. `clickhouse-benchmark` arrête d'envoyer des requêtes lorsque le délai spécifié est atteint. Valeur par défaut: 0 (limite de temps désactivée). +- `--confidence=N` — Level of confidence for T-test. Possible values: 0 (80%), 1 (90%), 2 (95%), 3 (98%), 4 (99%), 5 (99.5%). Default value: 5. In the [mode de comparaison](#clickhouse-benchmark-comparison-mode) `clickhouse-benchmark` effectue les [Test T de L'étudiant indépendant à deux échantillons](https://en.wikipedia.org/wiki/Student%27s_t-test#Independent_two-sample_t-test) tester pour déterminer si les deux distributions ne sont pas différentes avec le niveau de confiance sélectionné. - `--cumulative` — Printing cumulative data instead of data per interval. - `--database=DATABASE_NAME` — ClickHouse database name. Default value: `default`. - `--json=FILEPATH` — JSON output. When the key is set, `clickhouse-benchmark` génère un rapport dans le fichier JSON spécifié. - `--user=USERNAME` — ClickHouse user name. Default value: `default`. - `--password=PSWD` — ClickHouse user password. Default value: empty string. -- `--stacktrace` — Stack traces output. When the key is set, `clickhouse-bencmark` affiche les traces d’exceptions de la pile. -- `--stage=WORD` — Query processing stage at server. ClickHouse stops query processing and returns answer to `clickhouse-benchmark` à l’étape spécifiée. Valeurs possibles: `complete`, `fetch_columns`, `with_mergeable_state`. Valeur par défaut: `complete`. +- `--stacktrace` — Stack traces output. When the key is set, `clickhouse-bencmark` affiche les traces d'exceptions de la pile. +- `--stage=WORD` — Query processing stage at server. ClickHouse stops query processing and returns answer to `clickhouse-benchmark` à l'étape spécifiée. Valeurs possibles: `complete`, `fetch_columns`, `with_mergeable_state`. Valeur par défaut: `complete`. - `--help` — Shows the help message. Si vous voulez appliquer des [paramètre](../../operations/settings/index.md) pour les requêtes, les passer comme une clé `--= SETTING_VALUE`. Exemple, `--max_memory_usage=1048576`. @@ -87,21 +87,21 @@ Dans le rapport, vous pouvez trouver: - Nombre de requêtes dans le `Queries executed:` champ. -- Le statut de chaîne de caractères contenant (dans l’ordre): +- Le statut de chaîne de caractères contenant (dans l'ordre): - Point de terminaison du serveur ClickHouse. - Nombre de requêtes traitées. - QPS: QPS: combien de requêtes serveur effectuées par seconde pendant une période spécifiée dans le `--delay` argument. - RPS: combien de lignes le serveur a lues par seconde pendant une période spécifiée dans `--delay` argument. - MiB/ s: combien de mebibytes serveur lus par seconde pendant une période spécifiée dans le `--delay` argument. - - résultat RPS: combien de lignes placées par le serveur au résultat d’une requête par seconde pendant une période spécifiée dans le `--delay` argument. - - MiB / s de résultat. combien de mibibytes placés par le serveur au résultat d’une requête par seconde pendant une période spécifiée dans `--delay` argument. + - résultat RPS: combien de lignes placées par le serveur au résultat d'une requête par seconde pendant une période spécifiée dans le `--delay` argument. + - MiB / s de résultat. combien de mibibytes placés par le serveur au résultat d'une requête par seconde pendant une période spécifiée dans `--delay` argument. -- Percentiles du temps d’exécution des requêtes. +- Percentiles du temps d'exécution des requêtes. ## Mode De Comparaison {#clickhouse-benchmark-comparison-mode} -`clickhouse-benchmark` peut comparer les performances pour deux serveurs clickhouse en cours d’exécution. +`clickhouse-benchmark` peut comparer les performances pour deux serveurs clickhouse en cours d'exécution. Pour utiliser le mode de comparaison, spécifiez les points de terminaison des deux serveurs par deux paires de `--host`, `--port` touches. Clés appariées ensemble par position dans la liste des arguments, la première `--host` est assorti avec le premier `--port` et ainsi de suite. `clickhouse-benchmark` établit les connexions aux serveurs, puis envoie des requêtes. Chaque requête adressée à un serveur sélectionné au hasard. Les résultats sont présentés pour chaque serveur séparément. diff --git a/docs/fr/operations/utilities/clickhouse-copier.md b/docs/fr/operations/utilities/clickhouse-copier.md index 8e90aac104b..da96c41e8f2 100644 --- a/docs/fr/operations/utilities/clickhouse-copier.md +++ b/docs/fr/operations/utilities/clickhouse-copier.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 59 toc_title: clickhouse-copieur --- # clickhouse-copieur {#clickhouse-copier} -Copie les données des tables d’un cluster vers des tables d’un autre cluster (ou du même cluster). +Copie les données des tables d'un cluster vers des tables d'un autre cluster (ou du même cluster). Vous pouvez exécuter plusieurs `clickhouse-copier` instances sur différents serveurs pour effectuer le même travail. ZooKeeper est utilisé pour synchroniser les processus. @@ -16,7 +16,7 @@ Après le démarrage de, `clickhouse-copier`: - Se connecte à ZooKeeper et reçoit: - La copie de tâches. - - L’état de la copie d’emplois. + - L'état de la copie d'emplois. - Il effectue les travaux. @@ -28,7 +28,7 @@ Pour réduire le trafic réseau, nous vous recommandons de `clickhouse-copier` s ## Course Clickhouse-copieur {#running-clickhouse-copier} -L’utilitaire doit être exécuté manuellement: +L'utilitaire doit être exécuté manuellement: ``` bash $ clickhouse-copier copier --daemon --config zookeeper.xml --task-path /task/path --base-dir /path/to/dir @@ -43,7 +43,7 @@ Paramètre: - `task-upload-force` — Force upload `task-file` même si le nœud existe déjà. - `base-dir` — The path to logs and auxiliary files. When it starts, `clickhouse-copier` crée `clickhouse-copier_YYYYMMHHSS_` les sous-répertoires `$base-dir`. Si ce paramètre est omis, les répertoires sont créés dans le répertoire où `clickhouse-copier` a été lancé. -## Format De Zookeeper.XML {#format-of-zookeeper-xml} +## Format de Zookeeper.XML {#format-of-zookeeper-xml} ``` xml @@ -62,7 +62,7 @@ Paramètre: ``` -## Configuration Des tâches De Copie {#configuration-of-copying-tasks} +## Configuration des tâches de copie {#configuration-of-copying-tasks} ``` xml diff --git a/docs/fr/operations/utilities/clickhouse-local.md b/docs/fr/operations/utilities/clickhouse-local.md index 495f87eee57..5dfac0d89ca 100644 --- a/docs/fr/operations/utilities/clickhouse-local.md +++ b/docs/fr/operations/utilities/clickhouse-local.md @@ -1,22 +1,22 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 60 toc_title: clickhouse-local --- # clickhouse-local {#clickhouse-local} -Le `clickhouse-local` programme vous permet d’effectuer un traitement rapide sur les fichiers locaux, sans avoir à déployer et configurer le serveur ClickHouse. +Le `clickhouse-local` programme vous permet d'effectuer un traitement rapide sur les fichiers locaux, sans avoir à déployer et configurer le serveur ClickHouse. Accepte les données qui représentent des tables et les interroge en utilisant [Clickhouse dialecte SQL](../../sql-reference/index.md). -`clickhouse-local` utilise le même noyau que clickhouse server, de sorte qu’il prend en charge la plupart des fonctionnalités et le même ensemble de formats et de moteurs de table. +`clickhouse-local` utilise le même noyau que clickhouse server, de sorte qu'il prend en charge la plupart des fonctionnalités et le même ensemble de formats et de moteurs de table. -Par défaut `clickhouse-local` n’a pas accès aux données sur le même hôte, mais il prend en charge le chargement de la configuration du serveur à l’aide `--config-file` argument. +Par défaut `clickhouse-local` n'a pas accès aux données sur le même hôte, mais il prend en charge le chargement de la configuration du serveur à l'aide `--config-file` argument. !!! warning "Avertissement" - Il n’est pas recommandé de charger la configuration du serveur de production dans `clickhouse-local` parce que les données peuvent être endommagées en cas d’erreur humaine. + Il n'est pas recommandé de charger la configuration du serveur de production dans `clickhouse-local` parce que les données peuvent être endommagées en cas d'erreur humaine. ## Utilisation {#usage} diff --git a/docs/fr/operations/utilities/index.md b/docs/fr/operations/utilities/index.md index 7523d5dd216..51a250bb996 100644 --- a/docs/fr/operations/utilities/index.md +++ b/docs/fr/operations/utilities/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 -toc_folder_title: Utilities +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: Utilitaire toc_priority: 56 toc_title: "Aper\xE7u" --- diff --git a/docs/fr/sql-reference/aggregate-functions/combinators.md b/docs/fr/sql-reference/aggregate-functions/combinators.md index 85ff7ea3e38..1aabd34051b 100644 --- a/docs/fr/sql-reference/aggregate-functions/combinators.md +++ b/docs/fr/sql-reference/aggregate-functions/combinators.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 37 -toc_title: "Combinateurs de fonction d'agr\xE9gat" +toc_title: Combinators --- -# Combinateurs De Fonction D’Agrégat {#aggregate_functions_combinators} +# Combinateurs De Fonction D'Agrégat {#aggregate_functions_combinators} -Le nom d’une fonction d’agrégat peut avoir un suffixe ajouté. Cela change la façon dont la fonction d’agrégation fonctionne. +Le nom d'une fonction d'agrégat peut avoir un suffixe ajouté. Cela change la façon dont la fonction d'agrégation fonctionne. ## -Si {#agg-functions-combinator-if} @@ -15,21 +15,21 @@ The suffix -If can be appended to the name of any aggregate function. In this ca Exemple: `sumIf(column, cond)`, `countIf(cond)`, `avgIf(x, cond)`, `quantilesTimingIf(level1, level2)(x, cond)`, `argMinIf(arg, val, cond)` et ainsi de suite. -Avec les fonctions d’agrégat conditionnel, vous pouvez calculer des agrégats pour plusieurs conditions à la fois, sans utiliser de sous-requêtes et `JOIN`s. Par exemple, dans Yandex.Metrica, les fonctions d’agrégat conditionnel sont utilisées pour implémenter la fonctionnalité de comparaison de segment. +Avec les fonctions d'agrégat conditionnel, vous pouvez calculer des agrégats pour plusieurs conditions à la fois, sans utiliser de sous-requêtes et `JOIN`s. Par exemple, dans Yandex.Metrica, les fonctions d'agrégat conditionnel sont utilisées pour implémenter la fonctionnalité de comparaison de segment. ## -Tableau {#agg-functions-combinator-array} -Le Tableau suffixe peut être ajouté à toute fonction d’agrégation. Dans ce cas, la fonction d’agrégation des arguments de la ‘Array(T)’ type (tableaux) au lieu de ‘T’ tapez les arguments. Si la fonction aggregate accepte plusieurs arguments, il doit s’agir de tableaux de longueurs égales. Lors du traitement des tableaux, la fonction d’agrégation fonctionne comme la fonction d’agrégation d’origine sur tous les éléments du tableau. +Le Tableau suffixe peut être ajouté à toute fonction d'agrégation. Dans ce cas, la fonction d'agrégation des arguments de la ‘Array(T)’ type (tableaux) au lieu de ‘T’ tapez les arguments. Si la fonction aggregate accepte plusieurs arguments, il doit s'agir de tableaux de longueurs égales. Lors du traitement des tableaux, la fonction d'agrégation fonctionne comme la fonction d'agrégation d'origine sur tous les éléments du tableau. Exemple 1: `sumArray(arr)` - Totalise tous les éléments de tous ‘arr’ tableau. Dans cet exemple, il aurait pu être écrit plus simplement: `sum(arraySum(arr))`. -Exemple 2: `uniqArray(arr)` – Counts the number of unique elements in all ‘arr’ tableau. Cela pourrait être fait d’une manière plus facile: `uniq(arrayJoin(arr))` mais ce n’est pas toujours possible d’ajouter des ‘arrayJoin’ pour une requête. +Exemple 2: `uniqArray(arr)` – Counts the number of unique elements in all ‘arr’ tableau. Cela pourrait être fait d'une manière plus facile: `uniq(arrayJoin(arr))` mais ce n'est pas toujours possible d'ajouter des ‘arrayJoin’ pour une requête. \- Si et-tableau peut être combiné. Cependant, ‘Array’ doit venir en premier, puis ‘If’. Exemple: `uniqArrayIf(arr, cond)`, `quantilesTimingArrayIf(level1, level2)(arr, cond)`. En raison de cet ordre, le ‘cond’ argument ne sera pas un tableau. ## -État {#agg-functions-combinator-state} -Si vous appliquez ce combinateur, la fonction d’agrégation ne renvoie pas la valeur résultante (par exemple le nombre de valeurs uniques pour [uniq](reference.md#agg_function-uniq) la fonction), mais un état intermédiaire de l’agrégation (pour `uniq`, c’est la table de hachage pour calculer le nombre de valeurs uniques). C’est un `AggregateFunction(...)` qui peuvent être utilisés pour un traitement ultérieur ou stockés dans un tableau pour terminer l’agrégation plus tard. +Si vous appliquez ce combinateur, la fonction d'agrégation ne renvoie pas la valeur résultante (par exemple le nombre de valeurs uniques pour [uniq](reference.md#agg_function-uniq) la fonction), mais un état intermédiaire de l'agrégation (pour `uniq`, c'est la table de hachage pour calculer le nombre de valeurs uniques). C'est un `AggregateFunction(...)` qui peuvent être utilisés pour un traitement ultérieur ou stockés dans un tableau pour terminer l'agrégation plus tard. Pour travailler avec ces états, utilisez: @@ -41,45 +41,122 @@ Pour travailler avec ces états, utilisez: ## -Fusionner {#aggregate_functions_combinators-merge} -Si vous appliquez ce combinateur, la fonction d’agrégation prend l’état d’agrégation intermédiaire comme argument, combine les États pour terminer l’agrégation et renvoie la valeur résultante. +Si vous appliquez ce combinateur, la fonction d'agrégation prend l'état d'agrégation intermédiaire comme argument, combine les États pour terminer l'agrégation et renvoie la valeur résultante. ## -MergeState {#aggregate_functions_combinators-mergestate} -Fusionne les États d’agrégation intermédiaires de la même manière que le combinateur-Merge. Cependant, il ne renvoie pas la valeur résultante, mais un État d’agrégation intermédiaire, similaire au combinateur-State. +Fusionne les États d'agrégation intermédiaires de la même manière que le combinateur-Merge. Cependant, il ne renvoie pas la valeur résultante, mais un État d'agrégation intermédiaire, similaire au combinateur-State. ## - ForEach {#agg-functions-combinator-foreach} -Convertit une fonction d’agrégation pour les tables en une fonction d’agrégation pour les tableaux qui agrège les éléments de tableau correspondants et renvoie un tableau de résultats. Exemple, `sumForEach` pour les tableaux `[1, 2]`, `[3, 4, 5]`et`[6, 7]`renvoie le résultat `[10, 13, 5]` après avoir additionné les éléments de tableau correspondants. +Convertit une fonction d'agrégation pour les tables en une fonction d'agrégation pour les tableaux qui agrège les éléments de tableau correspondants et renvoie un tableau de résultats. Exemple, `sumForEach` pour les tableaux `[1, 2]`, `[3, 4, 5]`et`[6, 7]`renvoie le résultat `[10, 13, 5]` après avoir additionné les éléments de tableau correspondants. ## - OrDefault {#agg-functions-combinator-ordefault} -Remplit la valeur par défaut du type de retour de la fonction d’agrégation s’il n’y a rien à agréger. +Modifie le comportement d'une fonction d'agrégat. + +Si une fonction d'agrégation n'a pas de valeurs d'entrée, avec ce combinateur, elle renvoie la valeur par défaut pour son type de données de retour. S'applique aux fonctions d'agrégation qui peuvent prendre des données d'entrée vides. + +`-OrDefault` peut être utilisé avec d'autres combinators. + +**Syntaxe** + +``` sql +OrDefault(x) +``` + +**Paramètre** + +- `x` — Aggregate function parameters. + +**Valeurs renvoyées** + +Renvoie la valeur par défaut du type de retour d'une fonction d'agrégation s'il n'y a rien à agréger. + +Le Type dépend de la fonction d'agrégation utilisée. + +**Exemple** + +Requête: ``` sql SELECT avg(number), avgOrDefault(number) FROM numbers(0) ``` +Résultat: + ``` text ┌─avg(number)─┬─avgOrDefault(number)─┐ │ nan │ 0 │ └─────────────┴──────────────────────┘ ``` -## - OrNull {#agg-functions-combinator-ornull} +Également `-OrDefault` peut être utilisé avec un autre combinateur. Il est utile lorsque la fonction d'agrégation n'accepte pas l'entrée vide. -Remplir `null` si il n’y a rien à s’agréger. La colonne de retour sera nullable. +Requête: ``` sql -SELECT avg(number), avgOrNull(number) FROM numbers(0) +SELECT avgOrDefaultIf(x, x > 10) +FROM +( + SELECT toDecimal32(1.23, 2) AS x +) ``` +Résultat: + ``` text -┌─avg(number)─┬─avgOrNull(number)─┐ -│ nan │ ᴺᵁᴸᴸ │ -└─────────────┴───────────────────┘ +┌─avgOrDefaultIf(x, greater(x, 10))─┐ +│ 0.00 │ +└───────────────────────────────────┘ ``` -\- OrDefault et-OrNull peuvent être combinés avec d’autres combinateurs. Il est utile lorsque la fonction d’agrégation n’accepte pas l’entrée vide. +## - OrNull {#agg-functions-combinator-ornull} + +Modifie le comportement d'une fonction d'agrégat. + +Ce combinateur convertit un résultat d'une fonction d'agrégation à l' [Nullable](../data-types/nullable.md) type de données. Si la fonction d'agrégation n'a pas de valeurs à calculer elle renvoie [NULL](../syntax.md#null-literal). + +`-OrNull` peut être utilisé avec d'autres combinators. + +**Syntaxe** + +``` sql +OrNull(x) +``` + +**Paramètre** + +- `x` — Aggregate function parameters. + +**Valeurs renvoyées** + +- Le résultat de la fonction d'agrégat, converti en `Nullable` type de données. +- `NULL` si il n'y a rien à s'agréger. + +Type: `Nullable(aggregate function return type)`. + +**Exemple** + +Ajouter `-orNull` à la fin de la fonction d'agrégation. + +Requête: + +``` sql +SELECT sumOrNull(number), toTypeName(sumOrNull(number)) FROM numbers(10) WHERE number > 10 +``` + +Résultat: + +``` text +┌─sumOrNull(number)─┬─toTypeName(sumOrNull(number))─┐ +│ ᴺᵁᴸᴸ │ Nullable(UInt64) │ +└───────────────────┴───────────────────────────────┘ +``` + +Également `-OrNull` peut être utilisé avec un autre combinateur. Il est utile lorsque la fonction d'agrégation n'accepte pas l'entrée vide. + +Requête: ``` sql SELECT avgOrNullIf(x, x > 10) @@ -89,6 +166,8 @@ FROM ) ``` +Résultat: + ``` text ┌─avgOrNullIf(x, greater(x, 10))─┐ │ ᴺᵁᴸᴸ │ @@ -97,7 +176,7 @@ FROM ## -Resample {#agg-functions-combinator-resample} -Permet de diviser les données en groupes, puis séparément agrège les données de ces groupes. Les groupes sont créés en divisant les valeurs d’une colonne en intervalles. +Permet de diviser les données en groupes, puis séparément agrège les données de ces groupes. Les groupes sont créés en divisant les valeurs d'une colonne en intervalles. ``` sql Resample(start, end, step)(, resampling_key) @@ -106,7 +185,7 @@ Permet de diviser les données en groupes, puis séparément agrège les donnée **Paramètre** - `start` — Starting value of the whole required interval for `resampling_key` valeur. -- `stop` — Ending value of the whole required interval for `resampling_key` valeur. L’ensemble de l’intervalle ne comprend pas les `stop` valeur `[start, stop)`. +- `stop` — Ending value of the whole required interval for `resampling_key` valeur. L'ensemble de l'intervalle ne comprend pas les `stop` valeur `[start, stop)`. - `step` — Step for separating the whole interval into subintervals. The `aggFunction` est exécuté sur chacun de ces sous-intervalles indépendamment. - `resampling_key` — Column whose values are used for separating data into intervals. - `aggFunction_params` — `aggFunction` paramètre. @@ -117,7 +196,7 @@ Permet de diviser les données en groupes, puis séparément agrège les donnée **Exemple** -Envisager l’ `people` le tableau avec les données suivantes: +Envisager l' `people` le tableau avec les données suivantes: ``` text ┌─name───┬─age─┬─wage─┐ @@ -130,9 +209,9 @@ Envisager l’ `people` le tableau avec les données suivantes: └────────┴─────┴──────┘ ``` -Obtenons les noms des personnes dont l’âge se trouve dans les intervalles de `[30,60)` et `[60,75)`. Puisque nous utilisons la représentation entière pour l’âge, nous obtenons des âges dans le `[30, 59]` et `[60,74]` intervalle. +Obtenons les noms des personnes dont l'âge se trouve dans les intervalles de `[30,60)` et `[60,75)`. Puisque nous utilisons la représentation entière pour l'âge, nous obtenons des âges dans le `[30, 59]` et `[60,74]` intervalle. -Pour agréger des noms dans un tableau, nous utilisons [grouperay](reference.md#agg_function-grouparray) fonction d’agrégation. Il faut un argument. Dans notre cas, c’est l’ `name` colonne. Le `groupArrayResample` fonction devrait utiliser le `age` colonne pour agréger les noms par âge. Pour définir les intervalles requis, nous passons le `30, 75, 30` des arguments dans la `groupArrayResample` fonction. +Pour agréger des noms dans un tableau, nous utilisons [grouperay](reference.md#agg_function-grouparray) fonction d'agrégation. Il faut un argument. Dans notre cas, c'est l' `name` colonne. Le `groupArrayResample` fonction devrait utiliser le `age` colonne pour agréger les noms par âge. Pour définir les intervalles requis, nous passons le `30, 75, 30` des arguments dans la `groupArrayResample` fonction. ``` sql SELECT groupArrayResample(30, 75, 30)(name, age) FROM people @@ -146,9 +225,9 @@ SELECT groupArrayResample(30, 75, 30)(name, age) FROM people Considérez les résultats. -`Jonh` est hors de l’échantillon parce qu’il est trop jeune. D’autres personnes sont distribués selon les intervalles d’âge. +`Jonh` est hors de l'échantillon parce qu'il est trop jeune. D'autres personnes sont distribués selon les intervalles d'âge. -Maintenant, nous allons compter le nombre total de personnes et leur salaire moyen dans les intervalles d’âge. +Maintenant, nous allons compter le nombre total de personnes et leur salaire moyen dans les intervalles d'âge. ``` sql SELECT diff --git a/docs/fr/sql-reference/aggregate-functions/index.md b/docs/fr/sql-reference/aggregate-functions/index.md index d279d4fe214..4f4e3f057ae 100644 --- a/docs/fr/sql-reference/aggregate-functions/index.md +++ b/docs/fr/sql-reference/aggregate-functions/index.md @@ -1,23 +1,23 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 -toc_folder_title: Aggregate Functions +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "Les Fonctions D'Agr\xE9gation" toc_priority: 33 toc_title: Introduction --- -# Les Fonctions d’agrégation {#aggregate-functions} +# Les Fonctions D'Agrégation {#aggregate-functions} -Les fonctions d’agrégation fonctionnent dans le [normal](http://www.sql-tutorial.com/sql-aggregate-functions-sql-tutorial) comme prévu par les experts de la base de données. +Les fonctions d'agrégation fonctionnent dans le [normal](http://www.sql-tutorial.com/sql-aggregate-functions-sql-tutorial) comme prévu par les experts de la base de données. Clickhouse prend également en charge: -- [Fonctions d’agrégat paramétriques](parametric-functions.md#aggregate_functions_parametric) qui acceptent d’autres paramètres en plus des colonnes. -- [Combinators](combinators.md#aggregate_functions_combinators), qui modifient le comportement des fonctions d’agrégation. +- [Fonctions d'agrégat paramétriques](parametric-functions.md#aggregate_functions_parametric) qui acceptent d'autres paramètres en plus des colonnes. +- [Combinators](combinators.md#aggregate_functions_combinators), qui modifient le comportement des fonctions d'agrégation. ## Le Traitement NULL {#null-processing} -Au cours de l’agrégation, tous les `NULL`s sont ignorés. +Au cours de l'agrégation, tous les `NULL`s sont ignorés. **Exemple:** @@ -43,7 +43,7 @@ SELECT sum(y) FROM t_null_big │ 7 │ └────────┘ -Le `sum` la fonction d’interprète `NULL` comme `0`. En particulier, cela signifie que si la fonction reçoit en entrée d’une sélection où toutes les valeurs sont `NULL`, alors le résultat sera `0`, pas `NULL`. +Le `sum` la fonction d'interprète `NULL` comme `0`. En particulier, cela signifie que si la fonction reçoit en entrée d'une sélection où toutes les valeurs sont `NULL`, alors le résultat sera `0`, pas `NULL`. Maintenant, vous pouvez utiliser le `groupArray` fonction pour créer un tableau à partir `y` colonne: diff --git a/docs/fr/sql-reference/aggregate-functions/parametric-functions.md b/docs/fr/sql-reference/aggregate-functions/parametric-functions.md index e16b38771de..a8d6d533cb8 100644 --- a/docs/fr/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/fr/sql-reference/aggregate-functions/parametric-functions.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 38 -toc_title: "Fonctions d'agr\xE9gat param\xE9triques" +toc_title: "Param\xE9trique" --- -# Fonctions D’Agrégat Paramétriques {#aggregate_functions_parametric} +# Fonctions D'Agrégat Paramétriques {#aggregate_functions_parametric} Some aggregate functions can accept not only argument columns (used for compression), but a set of parameters – constants for initialization. The syntax is two pairs of brackets instead of one. The first is for parameters, and the second is for arguments. @@ -17,12 +17,12 @@ Calcule un histogramme adaptatif. Cela ne garantit pas des résultats précis. histogram(number_of_bins)(values) ``` -Les fonctions utilise [Un Algorithme D’Arbre De Décision Parallèle En Continu](http://jmlr.org/papers/volume11/ben-haim10a/ben-haim10a.pdf). Les bordures des bacs d’histogramme sont ajustées au fur et à mesure que de nouvelles données entrent dans une fonction. Dans le cas courant, les largeurs des bacs ne sont pas égales. +Les fonctions utilise [Un Algorithme D'Arbre De Décision Parallèle En Continu](http://jmlr.org/papers/volume11/ben-haim10a/ben-haim10a.pdf). Les bordures des bacs d'histogramme sont ajustées au fur et à mesure que de nouvelles données entrent dans une fonction. Dans le cas courant, les largeurs des bacs ne sont pas égales. **Paramètre** `number_of_bins` — Upper limit for the number of bins in the histogram. The function automatically calculates the number of bins. It tries to reach the specified number of bins, but if it fails, it uses fewer bins. -`values` — [Expression](../syntax.md#syntax-expressions) résultant en valeurs d’entrée. +`values` — [Expression](../syntax.md#syntax-expressions) résultant en valeurs d'entrée. **Valeurs renvoyées** @@ -78,11 +78,11 @@ FROM └────────┴───────┘ ``` -Dans ce cas, vous devez vous rappeler que vous ne connaissez pas les frontières de la corbeille d’histogramme. +Dans ce cas, vous devez vous rappeler que vous ne connaissez pas les frontières de la corbeille d'histogramme. ## sequenceMatch(pattern)(timestamp, cond1, cond2, …) {#function-sequencematch} -Vérifie si la séquence contient une chaîne d’événements qui correspond au modèle. +Vérifie si la séquence contient une chaîne d'événements qui correspond au modèle. ``` sql sequenceMatch(pattern)(timestamp, cond1, cond2, ...) @@ -97,7 +97,7 @@ sequenceMatch(pattern)(timestamp, cond1, cond2, ...) - `timestamp` — Column considered to contain time data. Typical data types are `Date` et `DateTime`. Vous pouvez également utiliser les prises en charge [UInt](../../sql-reference/data-types/int-uint.md) types de données. -- `cond1`, `cond2` — Conditions that describe the chain of events. Data type: `UInt8`. Vous pouvez passer jusqu’à 32 arguments de condition. La fonction ne prend en compte que les événements décrits dans ces conditions. Si la séquence contient des données qui ne sont pas décrites dans une condition, la fonction les ignore. +- `cond1`, `cond2` — Conditions that describe the chain of events. Data type: `UInt8`. Vous pouvez passer jusqu'à 32 arguments de condition. La fonction ne prend en compte que les événements décrits dans ces conditions. Si la séquence contient des données qui ne sont pas décrites dans une condition, la fonction les ignore. **Valeurs renvoyées** @@ -109,11 +109,11 @@ Type: `UInt8`. **Syntaxe du motif** -- `(?N)` — Matches the condition argument at position `N`. Les Conditions sont numérotées dans le `[1, 32]` gamme. Exemple, `(?1)` correspond à l’argument passé au `cond1` paramètre. +- `(?N)` — Matches the condition argument at position `N`. Les Conditions sont numérotées dans le `[1, 32]` gamme. Exemple, `(?1)` correspond à l'argument passé au `cond1` paramètre. -- `.*` — Matches any number of events. You don’t need conditional arguments to match this element of the pattern. +- `.*` — Matches any number of events. You don't need conditional arguments to match this element of the pattern. -- `(?t operator value)` — Sets the time in seconds that should separate two events. For example, pattern `(?1)(?t>1800)(?2)` correspond à des événements qui se produisent plus de 1800 secondes les uns des autres. Un nombre arbitraire d’événements peut se trouver entre ces événements. Vous pouvez utiliser l’ `>=`, `>`, `<`, `<=` opérateur. +- `(?t operator value)` — Sets the time in seconds that should separate two events. For example, pattern `(?1)(?t>1800)(?2)` correspond à des événements qui se produisent plus de 1800 secondes les uns des autres. Un nombre arbitraire d'événements peut se trouver entre ces événements. Vous pouvez utiliser l' `>=`, `>`, `<`, `<=` opérateur. **Exemple** @@ -139,7 +139,7 @@ SELECT sequenceMatch('(?1)(?2)')(time, number = 1, number = 2) FROM t └───────────────────────────────────────────────────────────────────────┘ ``` -La fonction a trouvé la chaîne d’événements où le numéro 2 suit le numéro 1. Il a sauté le numéro 3 entre eux, car le nombre n’est pas décrit comme un événement. Si nous voulons prendre ce nombre en compte lors de la recherche de l’événement de la chaîne donnée dans l’exemple, nous devrions en faire une condition. +La fonction a trouvé la chaîne d'événements où le numéro 2 suit le numéro 1. Il a sauté le numéro 3 entre eux, car le nombre n'est pas décrit comme un événement. Si nous voulons prendre ce nombre en compte lors de la recherche de l'événement de la chaîne donnée dans l'exemple, nous devrions en faire une condition. ``` sql SELECT sequenceMatch('(?1)(?2)')(time, number = 1, number = 2, number = 3) FROM t @@ -151,7 +151,7 @@ SELECT sequenceMatch('(?1)(?2)')(time, number = 1, number = 2, number = 3) FROM └──────────────────────────────────────────────────────────────────────────────────────────┘ ``` -Dans ce cas, la fonction n’a pas pu trouver la chaîne d’événements correspondant au modèle, car l’événement pour le numéro 3 s’est produit entre 1 et 2. Si dans le même cas nous vérifions la condition pour le numéro 4, la séquence correspondrait au motif. +Dans ce cas, la fonction n'a pas pu trouver la chaîne d'événements correspondant au modèle, car l'événement pour le numéro 3 s'est produit entre 1 et 2. Si dans le même cas nous vérifions la condition pour le numéro 4, la séquence correspondrait au motif. ``` sql SELECT sequenceMatch('(?1)(?2)')(time, number = 1, number = 2, number = 4) FROM t @@ -169,7 +169,7 @@ SELECT sequenceMatch('(?1)(?2)')(time, number = 1, number = 2, number = 4) FROM ## sequenceCount(pattern)(time, cond1, cond2, …) {#function-sequencecount} -Compte le nombre de chaînes d’événements correspondant au motif. La fonction recherche les chaînes d’événements qui ne se chevauchent pas. Il commence à rechercher la chaîne suivante après que la chaîne actuelle est appariée. +Compte le nombre de chaînes d'événements correspondant au motif. La fonction recherche les chaînes d'événements qui ne se chevauchent pas. Il commence à rechercher la chaîne suivante après que la chaîne actuelle est appariée. !!! warning "Avertissement" Les événements qui se produisent à la même seconde peuvent se situer dans la séquence dans un ordre indéfini affectant le résultat. @@ -184,11 +184,11 @@ sequenceCount(pattern)(timestamp, cond1, cond2, ...) - `timestamp` — Column considered to contain time data. Typical data types are `Date` et `DateTime`. Vous pouvez également utiliser les prises en charge [UInt](../../sql-reference/data-types/int-uint.md) types de données. -- `cond1`, `cond2` — Conditions that describe the chain of events. Data type: `UInt8`. Vous pouvez passer jusqu’à 32 arguments de condition. La fonction ne prend en compte que les événements décrits dans ces conditions. Si la séquence contient des données qui ne sont pas décrites dans une condition, la fonction les ignore. +- `cond1`, `cond2` — Conditions that describe the chain of events. Data type: `UInt8`. Vous pouvez passer jusqu'à 32 arguments de condition. La fonction ne prend en compte que les événements décrits dans ces conditions. Si la séquence contient des données qui ne sont pas décrites dans une condition, la fonction les ignore. **Valeurs renvoyées** -- Nombre de chaînes d’événements qui ne se chevauchent pas et qui sont mises en correspondance. +- Nombre de chaînes d'événements qui ne se chevauchent pas et qui sont mises en correspondance. Type: `UInt64`. @@ -207,7 +207,7 @@ Considérer les données dans le `t` table: └──────┴────────┘ ``` -Comptez combien de fois le nombre 2 se produit après le nombre 1 avec n’importe quelle quantité d’autres nombres entre eux: +Comptez combien de fois le nombre 2 se produit après le nombre 1 avec n'importe quelle quantité d'autres nombres entre eux: ``` sql SELECT sequenceCount('(?1).*(?2)')(time, number = 1, number = 2) FROM t @@ -225,15 +225,15 @@ SELECT sequenceCount('(?1).*(?2)')(time, number = 1, number = 2) FROM t ## fenêtrefunnel {#windowfunnel} -Recherche les chaînes d’événements dans une fenêtre de temps coulissante et calcule le nombre maximum d’événements qui se sont produits à partir de la chaîne. +Recherche les chaînes d'événements dans une fenêtre de temps coulissante et calcule le nombre maximum d'événements qui se sont produits à partir de la chaîne. -La fonction fonctionne selon l’algorithme: +La fonction fonctionne selon l'algorithme: -- La fonction recherche les données qui déclenchent la première condition de la chaîne et définit le compteur d’événements sur 1. C’est le moment où la fenêtre coulissante commence. +- La fonction recherche les données qui déclenchent la première condition de la chaîne et définit le compteur d'événements sur 1. C'est le moment où la fenêtre coulissante commence. -- Si les événements de la chaîne se produisent séquentiellement dans la fenêtre, le compteur est incrémenté. Si la séquence d’événements est perturbée, le compteur n’est pas incrémenté. +- Si les événements de la chaîne se produisent séquentiellement dans la fenêtre, le compteur est incrémenté. Si la séquence d'événements est perturbée, le compteur n'est pas incrémenté. -- Si les données ont plusieurs chaînes d’événements à différents points d’achèvement, la fonction affichera uniquement la taille de la chaîne la plus longue. +- Si les données ont plusieurs chaînes d'événements à différents points d'achèvement, la fonction affichera uniquement la taille de la chaîne la plus longue. **Syntaxe** @@ -244,9 +244,9 @@ windowFunnel(window, [mode])(timestamp, cond1, cond2, ..., condN) **Paramètre** - `window` — Length of the sliding window in seconds. -- `mode` - C’est un argument facultatif. +- `mode` - C'est un argument facultatif. - `'strict'` - Lorsque le `'strict'` est défini, le windowFunnel() applique des conditions uniquement pour les valeurs uniques. -- `timestamp` — Name of the column containing the timestamp. Data types supported: [Date](../../sql-reference/data-types/date.md), [DateTime](../../sql-reference/data-types/datetime.md#data_type-datetime) et d’autres types entiers non signés (notez que même si timestamp prend en charge le `UInt64` type, sa valeur ne peut pas dépasser le maximum Int64, qui est 2^63 - 1). +- `timestamp` — Name of the column containing the timestamp. Data types supported: [Date](../../sql-reference/data-types/date.md), [DateTime](../../sql-reference/data-types/datetime.md#data_type-datetime) et d'autres types entiers non signés (notez que même si timestamp prend en charge le `UInt64` type, sa valeur ne peut pas dépasser le maximum Int64, qui est 2^63 - 1). - `cond` — Conditions or data describing the chain of events. [UInt8](../../sql-reference/data-types/int-uint.md). **Valeur renvoyée** @@ -258,16 +258,16 @@ Type: `Integer`. **Exemple** -Déterminer si une période de temps est suffisant pour l’utilisateur de sélectionner un téléphone et d’acheter deux fois dans la boutique en ligne. +Déterminer si une période de temps est suffisant pour l'utilisateur de sélectionner un téléphone et d'acheter deux fois dans la boutique en ligne. -Définissez la chaîne d’événements suivante: +Définissez la chaîne d'événements suivante: -1. L’utilisateur s’est connecté à son compte sur le magasin (`eventID = 1003`). -2. L’utilisateur recherche un téléphone (`eventID = 1007, product = 'phone'`). -3. Toute commande de l’utilisateur (`eventID = 1009`). -4. L’Utilisateur a fait la commande à nouveau (`eventID = 1010`). +1. L'utilisateur s'est connecté à son compte sur le magasin (`eventID = 1003`). +2. L'utilisateur recherche un téléphone (`eventID = 1007, product = 'phone'`). +3. Toute commande de l'utilisateur (`eventID = 1009`). +4. L'Utilisateur a fait la commande à nouveau (`eventID = 1010`). -Table d’entrée: +Table d'entrée: ``` text ┌─event_date─┬─user_id─┬───────────timestamp─┬─eventID─┬─product─┐ @@ -284,7 +284,7 @@ Table d’entrée: └────────────┴─────────┴─────────────────────┴─────────┴─────────┘ ``` -Savoir dans quelle mesure l’utilisateur `user_id` pourrait passer à travers la chaîne dans une période en Janvier-Février de 2019. +Savoir dans quelle mesure l'utilisateur `user_id` pourrait passer à travers la chaîne dans une période en Janvier-Février de 2019. Requête: @@ -315,10 +315,10 @@ Résultat: ## rétention {#retention} -La fonction prend comme arguments un ensemble de conditions de 1 à 32 arguments de type `UInt8` qui indiquent si une certaine condition est remplie pour l’événement. -Toute condition peut être spécifiée comme argument (comme dans [WHERE](../../sql-reference/statements/select.md#select-where)). +La fonction prend comme arguments un ensemble de conditions de 1 à 32 arguments de type `UInt8` qui indiquent si une certaine condition est remplie pour l'événement. +Toute condition peut être spécifiée comme argument (comme dans [WHERE](../../sql-reference/statements/select/where.md#select-where)). -Les conditions, à l’exception de la première, s’appliquent par paires: le résultat de la seconde sera vrai si la première et la deuxième sont remplies, le troisième si la première et la fird sont vraies, etc. +Les conditions, à l'exception de la première, s'appliquent par paires: le résultat de la seconde sera vrai si la première et la deuxième sont remplies, le troisième si la première et la fird sont vraies, etc. **Syntaxe** @@ -335,7 +335,7 @@ retention(cond1, cond2, ..., cond32); Le tableau de 1 ou 0. - 1 — condition was met for the event. -- 0 — condition wasn’t met for the event. +- 0 — condition wasn't met for the event. Type: `UInt8`. @@ -353,7 +353,7 @@ INSERT INTO retention_test SELECT '2020-01-02', number FROM numbers(10); INSERT INTO retention_test SELECT '2020-01-03', number FROM numbers(15); ``` -Table d’entrée: +Table d'entrée: Requête: @@ -402,7 +402,7 @@ Résultat: └────────────┴─────┘ ``` -**2.** Grouper les utilisateurs par ID unique `uid` à l’aide de la `retention` fonction. +**2.** Grouper les utilisateurs par ID unique `uid` à l'aide de la `retention` fonction. Requête: @@ -469,23 +469,23 @@ Résultat: Où: - `r1`- le nombre de visiteurs uniques qui ont visité le site au cours du 2020-01-01 (le `cond1` condition). -- `r2`- le nombre de visiteurs uniques qui ont visité le site au cours d’une période donnée entre 2020-01-01 et 2020-01-02 (`cond1` et `cond2` condition). -- `r3`- le nombre de visiteurs uniques qui ont visité le site au cours d’une période donnée entre 2020-01-01 et 2020-01-03 (`cond1` et `cond3` condition). +- `r2`- le nombre de visiteurs uniques qui ont visité le site au cours d'une période donnée entre 2020-01-01 et 2020-01-02 (`cond1` et `cond2` condition). +- `r3`- le nombre de visiteurs uniques qui ont visité le site au cours d'une période donnée entre 2020-01-01 et 2020-01-03 (`cond1` et `cond3` condition). ## uniqUpTo (N) (x) {#uniquptonx} Calculates the number of different argument values ​​if it is less than or equal to N. If the number of different argument values is greater than N, it returns N + 1. -Recommandé pour une utilisation avec de petites Ns, jusqu’à 10. La valeur maximale de N est de 100. +Recommandé pour une utilisation avec de petites Ns, jusqu'à 10. La valeur maximale de N est de 100. -Pour l’état d’une fonction d’agrégation, il utilise la quantité de mémoire égale à 1 + N \* de la taille d’une valeur d’octets. +Pour l'état d'une fonction d'agrégation, il utilise la quantité de mémoire égale à 1 + N \* de la taille d'une valeur d'octets. Pour les chaînes, il stocke un hachage non cryptographique de 8 octets. Soit le calcul est approchée pour les chaînes. La fonction fonctionne également pour plusieurs arguments. Cela fonctionne aussi vite que possible, sauf dans les cas où une grande valeur N est utilisée et le nombre de valeurs uniques est légèrement inférieur à N. -Exemple d’utilisation: +Exemple d'utilisation: ``` text Problem: Generate a report that shows only keywords that produced at least 5 unique users. @@ -496,4 +496,4 @@ Solution: Write in the GROUP BY query SearchPhrase HAVING uniqUpTo(4)(UserID) >= ## sumMapFiltered(keys\_to\_keep) (clés, valeurs) {#summapfilteredkeys-to-keepkeys-values} -Même comportement que [sumMap](reference.md#agg_functions-summap) sauf qu’un tableau de clés est passé en paramètre. Cela peut être particulièrement utile lorsque vous travaillez avec une forte cardinalité de touches. +Même comportement que [sumMap](reference.md#agg_functions-summap) sauf qu'un tableau de clés est passé en paramètre. Cela peut être particulièrement utile lorsque vous travaillez avec une forte cardinalité de touches. diff --git a/docs/fr/sql-reference/aggregate-functions/reference.md b/docs/fr/sql-reference/aggregate-functions/reference.md index 29704eb0d2f..456a9e9c4b3 100644 --- a/docs/fr/sql-reference/aggregate-functions/reference.md +++ b/docs/fr/sql-reference/aggregate-functions/reference.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 36 toc_title: "R\xE9f\xE9rence" --- -# La Fonction De Référence {#function-reference} +# Référence De La Fonction Agrégée {#aggregate-functions-reference} ## compter {#agg_function-count} @@ -25,15 +25,15 @@ La fonction peut prendre: **Valeur renvoyée** - Si la fonction est appelée sans paramètres, il compte le nombre de lignes. -- Si l’ [expression](../syntax.md#syntax-expressions) est passé, alors la fonction compte combien de fois cette expression retournée not null. Si l’expression renvoie un [Nullable](../../sql-reference/data-types/nullable.md)- tapez la valeur, puis le résultat de `count` séjours pas `Nullable`. La fonction renvoie 0 si l’expression est retournée `NULL` pour toutes les lignes. +- Si l' [expression](../syntax.md#syntax-expressions) est passé, alors la fonction compte combien de fois cette expression retournée not null. Si l'expression renvoie un [Nullable](../../sql-reference/data-types/nullable.md)- tapez la valeur, puis le résultat de `count` séjours pas `Nullable`. La fonction renvoie 0 si l'expression est retournée `NULL` pour toutes les lignes. Dans les deux cas le type de la valeur renvoyée est [UInt64](../../sql-reference/data-types/int-uint.md). **Détail** -Clickhouse soutient le `COUNT(DISTINCT ...)` syntaxe. Le comportement de cette construction dépend de la [count\_distinct\_implementation](../../operations/settings/settings.md#settings-count_distinct_implementation) paramètre. Il définit lequel des [uniq\*](#agg_function-uniq) fonctions est utilisée pour effectuer l’opération. La valeur par défaut est la [uniqExact](#agg_function-uniqexact) fonction. +Clickhouse soutient le `COUNT(DISTINCT ...)` syntaxe. Le comportement de cette construction dépend de la [count\_distinct\_implementation](../../operations/settings/settings.md#settings-count_distinct_implementation) paramètre. Il définit lequel des [uniq\*](#agg_function-uniq) fonctions est utilisée pour effectuer l'opération. La valeur par défaut est la [uniqExact](#agg_function-uniqexact) fonction. -Le `SELECT count() FROM table` la requête n’est pas optimisé, car le nombre d’entrées dans la table n’est pas stockée séparément. Il choisit une petite colonne de la table et compte le nombre de valeurs qu’il contient. +Le `SELECT count() FROM table` la requête n'est pas optimisé, car le nombre d'entrées dans la table n'est pas stockée séparément. Il choisit une petite colonne de la table et compte le nombre de valeurs qu'il contient. **Exemple** @@ -71,21 +71,21 @@ SELECT count(DISTINCT num) FROM t └────────────────┘ ``` -Cet exemple montre que `count(DISTINCT num)` est effectuée par le `uniqExact` en fonction de l’ `count_distinct_implementation` valeur de réglage. +Cet exemple montre que `count(DISTINCT num)` est effectuée par le `uniqExact` en fonction de l' `count_distinct_implementation` valeur de réglage. ## tout(x) {#agg_function-any} Sélectionne la première valeur rencontrée. -La requête peut être exécutée dans n’importe quel ordre, et même dans un ordre différent à chaque fois, de sorte que le résultat de cette fonction est indéterminée. +La requête peut être exécutée dans n'importe quel ordre, et même dans un ordre différent à chaque fois, de sorte que le résultat de cette fonction est indéterminée. Pour obtenir un résultat déterminé, vous pouvez utiliser le ‘min’ ou ‘max’ fonction au lieu de ‘any’. -Dans certains cas, vous pouvez compter sur l’ordre de l’exécution. Cela s’applique aux cas où SELECT provient d’une sous-requête qui utilise ORDER BY. +Dans certains cas, vous pouvez compter sur l'ordre de l'exécution. Cela s'applique aux cas où SELECT provient d'une sous-requête qui utilise ORDER BY. -Lorsqu’un `SELECT` la requête a l’ `GROUP BY` ou au moins une fonction d’agrégat, ClickHouse (contrairement à MySQL) exige que toutes les expressions du `SELECT`, `HAVING`, et `ORDER BY` clauses être calculée à partir de clés ou de fonctions d’agrégation. En d’autres termes, chaque colonne sélectionnée dans la table doit être utilisée soit dans les clés, soit dans les fonctions d’agrégation. Pour obtenir un comportement comme dans MySQL, vous pouvez mettre les autres colonnes dans le `any` fonction d’agrégation. +Lorsqu'un `SELECT` la requête a l' `GROUP BY` ou au moins une fonction d'agrégat, ClickHouse (contrairement à MySQL) exige que toutes les expressions du `SELECT`, `HAVING`, et `ORDER BY` clauses être calculée à partir de clés ou de fonctions d'agrégation. En d'autres termes, chaque colonne sélectionnée dans la table doit être utilisée soit dans les clés, soit dans les fonctions d'agrégation. Pour obtenir un comportement comme dans MySQL, vous pouvez mettre les autres colonnes dans le `any` fonction d'agrégation. ## anyHeavy (x) {#anyheavyx} -Sélectionne une valeur fréquente à l’aide [poids lourds](http://www.cs.umd.edu/~samir/498/karp.pdf) algorithme. S’il y a une valeur qui se produit plus de la moitié des cas dans chacun des threads d’exécution de la requête, cette valeur est renvoyée. Normalement, le résultat est non déterministe. +Sélectionne une valeur fréquente à l'aide [poids lourds](http://www.cs.umd.edu/~samir/498/karp.pdf) algorithme. S'il y a une valeur qui se produit plus de la moitié des cas dans chacun des threads d'exécution de la requête, cette valeur est renvoyée. Normalement, le résultat est non déterministe. ``` sql anyHeavy(column) @@ -97,7 +97,7 @@ anyHeavy(column) **Exemple** -Prendre la [OnTime](../../getting-started/example-datasets/ontime.md) ensemble de données et sélectionnez n’importe quelle valeur `AirlineID` colonne. +Prendre la [OnTime](../../getting-started/example-datasets/ontime.md) ensemble de données et sélectionnez n'importe quelle valeur `AirlineID` colonne. ``` sql SELECT anyHeavy(AirlineID) AS res @@ -117,7 +117,7 @@ Le résultat est tout aussi indéterminé que pour le `any` fonction. ## groupBitAnd {#groupbitand} -S’applique au niveau du BIT `AND` pour les séries de nombres. +S'applique au niveau du BIT `AND` pour les séries de nombres. ``` sql groupBitAnd(expr) @@ -160,7 +160,7 @@ binary decimal ## groupBitOr {#groupbitor} -S’applique au niveau du BIT `OR` pour les séries de nombres. +S'applique au niveau du BIT `OR` pour les séries de nombres. ``` sql groupBitOr(expr) @@ -203,7 +203,7 @@ binary decimal ## groupBitXor {#groupbitxor} -S’applique au niveau du BIT `XOR` pour les séries de nombres. +S'applique au niveau du BIT `XOR` pour les séries de nombres. ``` sql groupBitXor(expr) @@ -246,7 +246,7 @@ binary decimal ## groupBitmap {#groupbitmap} -Calculs Bitmap ou agrégés à partir d’une colonne entière non signée, retour cardinalité de type UInt64, si Ajouter suffixe-State, puis retour [objet bitmap](../../sql-reference/functions/bitmap-functions.md). +Calculs Bitmap ou agrégés à partir d'une colonne entière non signée, retour cardinalité de type UInt64, si Ajouter suffixe-State, puis retour [objet bitmap](../../sql-reference/functions/bitmap-functions.md). ``` sql groupBitmap(expr) @@ -295,7 +295,7 @@ Calcule le maximum. ## argMin (arg, val) {#agg-function-argmin} -Calcule la ‘arg’ valeur pour un minimum ‘val’ valeur. S’il y a plusieurs valeurs différentes de ‘arg’ pour des valeurs minimales de ‘val’ la première de ces valeurs rencontrées est de sortie. +Calcule la ‘arg’ valeur pour un minimum ‘val’ valeur. S'il y a plusieurs valeurs différentes de ‘arg’ pour des valeurs minimales de ‘val’ la première de ces valeurs rencontrées est de sortie. **Exemple:** @@ -319,7 +319,7 @@ SELECT argMin(user, salary) FROM salary ## argMax(arg, val) {#agg-function-argmax} -Calcule la ‘arg’ valeur pour un maximum ‘val’ valeur. S’il y a plusieurs valeurs différentes de ‘arg’ pour les valeurs maximales de ‘val’ la première de ces valeurs rencontrées est de sortie. +Calcule la ‘arg’ valeur pour un maximum ‘val’ valeur. S'il y a plusieurs valeurs différentes de ‘arg’ pour les valeurs maximales de ‘val’ la première de ces valeurs rencontrées est de sortie. ## sum(x) {#agg_function-sum} @@ -328,14 +328,15 @@ Ne fonctionne que pour les numéros. ## sumWithOverflow (x) {#sumwithoverflowx} -Calcule la somme des nombres, en utilisant le même type de données pour le résultat que pour les paramètres d’entrée. Si la somme dépasse la valeur maximale pour ce type de données, la fonction renvoie une erreur. +Calcule la somme des nombres, en utilisant le même type de données pour le résultat que pour les paramètres d'entrée. Si la somme dépasse la valeur maximale pour ce type de données, la fonction renvoie une erreur. Ne fonctionne que pour les numéros. -## sumMap (clé, valeur) {#agg_functions-summap} +## sumMap(clé, valeur), sumMap(Tuple(clé, valeur)) {#agg_functions-summap} Les totaux de la ‘value’ tableau selon les clés spécifiés dans le ‘key’ tableau. -Le nombre d’éléments dans ‘key’ et ‘value’ doit être identique pour chaque ligne totalisée. +Le passage du tuple des tableaux de clés et de valeurs est synonyme du passage de deux tableaux de clés et de valeurs. +Le nombre d'éléments dans ‘key’ et ‘value’ doit être identique pour chaque ligne totalisée. Returns a tuple of two arrays: keys in sorted order, and values ​​summed for the corresponding keys. Exemple: @@ -347,30 +348,33 @@ CREATE TABLE sum_map( statusMap Nested( status UInt16, requests UInt64 - ) + ), + statusMapTuple Tuple(Array(Int32), Array(Int32)) ) ENGINE = Log; INSERT INTO sum_map VALUES - ('2000-01-01', '2000-01-01 00:00:00', [1, 2, 3], [10, 10, 10]), - ('2000-01-01', '2000-01-01 00:00:00', [3, 4, 5], [10, 10, 10]), - ('2000-01-01', '2000-01-01 00:01:00', [4, 5, 6], [10, 10, 10]), - ('2000-01-01', '2000-01-01 00:01:00', [6, 7, 8], [10, 10, 10]); + ('2000-01-01', '2000-01-01 00:00:00', [1, 2, 3], [10, 10, 10], ([1, 2, 3], [10, 10, 10])), + ('2000-01-01', '2000-01-01 00:00:00', [3, 4, 5], [10, 10, 10], ([3, 4, 5], [10, 10, 10])), + ('2000-01-01', '2000-01-01 00:01:00', [4, 5, 6], [10, 10, 10], ([4, 5, 6], [10, 10, 10])), + ('2000-01-01', '2000-01-01 00:01:00', [6, 7, 8], [10, 10, 10], ([6, 7, 8], [10, 10, 10])); + SELECT timeslot, - sumMap(statusMap.status, statusMap.requests) + sumMap(statusMap.status, statusMap.requests), + sumMap(statusMapTuple) FROM sum_map GROUP BY timeslot ``` ``` text -┌────────────timeslot─┬─sumMap(statusMap.status, statusMap.requests)─┐ -│ 2000-01-01 00:00:00 │ ([1,2,3,4,5],[10,10,20,10,10]) │ -│ 2000-01-01 00:01:00 │ ([4,5,6,7,8],[10,10,20,10,10]) │ -└─────────────────────┴──────────────────────────────────────────────┘ +┌────────────timeslot─┬─sumMap(statusMap.status, statusMap.requests)─┬─sumMap(statusMapTuple)─────────┐ +│ 2000-01-01 00:00:00 │ ([1,2,3,4,5],[10,10,20,10,10]) │ ([1,2,3,4,5],[10,10,20,10,10]) │ +│ 2000-01-01 00:01:00 │ ([4,5,6,7,8],[10,10,20,10,10]) │ ([4,5,6,7,8],[10,10,20,10,10]) │ +└─────────────────────┴──────────────────────────────────────────────┴────────────────────────────────┘ ``` ## skewPop {#skewpop} -Calcule la [asymétrie](https://en.wikipedia.org/wiki/Skewness) d’une séquence. +Calcule la [asymétrie](https://en.wikipedia.org/wiki/Skewness) d'une séquence. ``` sql skewPop(expr) @@ -378,7 +382,7 @@ skewPop(expr) **Paramètre** -`expr` — [Expression](../syntax.md#syntax-expressions) retour d’un nombre. +`expr` — [Expression](../syntax.md#syntax-expressions) retour d'un nombre. **Valeur renvoyée** @@ -392,9 +396,9 @@ SELECT skewPop(value) FROM series_with_value_column ## skewSamp {#skewsamp} -Calcule la [asymétrie de l’échantillon](https://en.wikipedia.org/wiki/Skewness) d’une séquence. +Calcule la [asymétrie de l'échantillon](https://en.wikipedia.org/wiki/Skewness) d'une séquence. -Il représente une estimation non biaisée de l’asymétrie d’une variable aléatoire si les valeurs passées forme de son échantillon. +Il représente une estimation non biaisée de l'asymétrie d'une variable aléatoire si les valeurs passées forme de son échantillon. ``` sql skewSamp(expr) @@ -402,11 +406,11 @@ skewSamp(expr) **Paramètre** -`expr` — [Expression](../syntax.md#syntax-expressions) retour d’un nombre. +`expr` — [Expression](../syntax.md#syntax-expressions) retour d'un nombre. **Valeur renvoyée** -The skewness of the given distribution. Type — [Float64](../../sql-reference/data-types/float.md). Si `n <= 1` (`n` est la taille de l’échantillon), alors la fonction renvoie `nan`. +The skewness of the given distribution. Type — [Float64](../../sql-reference/data-types/float.md). Si `n <= 1` (`n` est la taille de l'échantillon), alors la fonction renvoie `nan`. **Exemple** @@ -416,7 +420,7 @@ SELECT skewSamp(value) FROM series_with_value_column ## kurtPop {#kurtpop} -Calcule la [kurtosis](https://en.wikipedia.org/wiki/Kurtosis) d’une séquence. +Calcule la [kurtosis](https://en.wikipedia.org/wiki/Kurtosis) d'une séquence. ``` sql kurtPop(expr) @@ -424,7 +428,7 @@ kurtPop(expr) **Paramètre** -`expr` — [Expression](../syntax.md#syntax-expressions) retour d’un nombre. +`expr` — [Expression](../syntax.md#syntax-expressions) retour d'un nombre. **Valeur renvoyée** @@ -438,9 +442,9 @@ SELECT kurtPop(value) FROM series_with_value_column ## kurtSamp {#kurtsamp} -Calcule la [l’échantillon le coefficient d’aplatissement](https://en.wikipedia.org/wiki/Kurtosis) d’une séquence. +Calcule la [l'échantillon le coefficient d'aplatissement](https://en.wikipedia.org/wiki/Kurtosis) d'une séquence. -Il représente une estimation non biaisée de la kurtose d’une variable aléatoire si les valeurs passées forment son échantillon. +Il représente une estimation non biaisée de la kurtose d'une variable aléatoire si les valeurs passées forment son échantillon. ``` sql kurtSamp(expr) @@ -448,11 +452,11 @@ kurtSamp(expr) **Paramètre** -`expr` — [Expression](../syntax.md#syntax-expressions) retour d’un nombre. +`expr` — [Expression](../syntax.md#syntax-expressions) retour d'un nombre. **Valeur renvoyée** -The kurtosis of the given distribution. Type — [Float64](../../sql-reference/data-types/float.md). Si `n <= 1` (`n` la taille de l’échantillon), alors la fonction renvoie `nan`. +The kurtosis of the given distribution. Type — [Float64](../../sql-reference/data-types/float.md). Si `n <= 1` (`n` la taille de l'échantillon), alors la fonction renvoie `nan`. **Exemple** @@ -462,8 +466,8 @@ SELECT kurtSamp(value) FROM series_with_value_column ## timeSeriesGroupSum(uid, horodatage, valeur) {#agg-function-timeseriesgroupsum} -`timeSeriesGroupSum` peut agréger différentes séries temporelles qui échantillonnent l’horodatage et non l’alignement. -Il utilisera une interpolation linéaire entre deux échantillons d’horodatage, puis additionnera les séries temporelles ensemble. +`timeSeriesGroupSum` peut agréger différentes séries temporelles qui échantillonnent l'horodatage et non l'alignement. +Il utilisera une interpolation linéaire entre deux échantillons d'horodatage, puis additionnera les séries temporelles ensemble. - `uid` la série temporelle est elle unique, `UInt64`. - `timestamp` est de type Int64 afin de prendre en charge la milliseconde ou la microseconde. @@ -471,7 +475,7 @@ Il utilisera une interpolation linéaire entre deux échantillons d’horodatage La fonction renvoie un tableau de tuples avec `(timestamp, aggregated_value)` pair. -Avant d’utiliser cette fonction, assurez-vous `timestamp` est dans l’ordre croissant. +Avant d'utiliser cette fonction, assurez-vous `timestamp` est dans l'ordre croissant. Exemple: @@ -514,10 +518,10 @@ Et le résultat sera: ## timeSeriesGroupRateSum(uid, ts, val) {#agg-function-timeseriesgroupratesum} -De même timeSeriesGroupRateSum, timeSeriesGroupRateSum calculera le taux de séries temporelles, puis additionnera les taux ensemble. -En outre, l’horodatage doit être dans l’ordre croissant avant d’utiliser cette fonction. +De la même manière à `timeSeriesGroupSum`, `timeSeriesGroupRateSum` calcule le taux de séries chronologiques, puis additionne les taux ensemble. +En outre, l'horodatage doit être dans l'ordre croissant avant d'utiliser cette fonction. -Utilisez cette fonction, le résultat ci-dessus sera: +Application de cette fonction aux données du `timeSeriesGroupSum` exemple, vous obtenez le résultat suivant: ``` text [(2,0),(3,0.1),(7,0.3),(8,0.3),(12,0.3),(17,0.3),(18,0.3),(24,0.3),(25,0.1)] @@ -529,9 +533,50 @@ Calcule la moyenne. Ne fonctionne que pour les numéros. Le résultat est toujours Float64. +## avgWeighted {#avgweighted} + +Calcule la [moyenne arithmétique pondérée](https://en.wikipedia.org/wiki/Weighted_arithmetic_mean). + +**Syntaxe** + +``` sql +avgWeighted(x, weight) +``` + +**Paramètre** + +- `x` — Values. [Entier](../data-types/int-uint.md) ou [virgule flottante](../data-types/float.md). +- `weight` — Weights of the values. [Entier](../data-types/int-uint.md) ou [virgule flottante](../data-types/float.md). + +Type de `x` et `weight` doit être le même. + +**Valeur renvoyée** + +- Moyenne pondérée. +- `NaN`. Si tous les poids sont égaux à 0. + +Type: [Float64](../data-types/float.md). + +**Exemple** + +Requête: + +``` sql +SELECT avgWeighted(x, w) +FROM values('x Int8, w Int8', (4, 1), (1, 0), (10, 2)) +``` + +Résultat: + +``` text +┌─avgWeighted(x, weight)─┐ +│ 8 │ +└────────────────────────┘ +``` + ## uniq {#agg_function-uniq} -Calcule le nombre approximatif des différentes valeurs de l’argument. +Calcule le nombre approximatif des différentes valeurs de l'argument. ``` sql uniq(x[, ...]) @@ -549,15 +594,15 @@ La fonction prend un nombre variable de paramètres. Les paramètres peuvent êt Fonction: -- Calcule un hachage pour tous les paramètres de l’agrégat, puis l’utilise dans les calculs. +- Calcule un hachage pour tous les paramètres de l'agrégat, puis l'utilise dans les calculs. -- Utilise un algorithme d’échantillonnage adaptatif. Pour l’état de calcul, La fonction utilise un échantillon de valeurs de hachage d’éléments jusqu’à 65536. +- Utilise un algorithme d'échantillonnage adaptatif. Pour l'état de calcul, La fonction utilise un échantillon de valeurs de hachage d'éléments jusqu'à 65536. This algorithm is very accurate and very efficient on the CPU. When the query contains several of these functions, using `uniq` is almost as fast as using other aggregate functions. -- Fournit le résultat de manière déterministe (cela ne dépend pas de l’ordre de traitement de la requête). +- Fournit le résultat de manière déterministe (cela ne dépend pas de l'ordre de traitement de la requête). -Nous vous recommandons d’utiliser cette fonction dans presque tous les scénarios. +Nous vous recommandons d'utiliser cette fonction dans presque tous les scénarios. **Voir Aussi** @@ -568,7 +613,7 @@ Nous vous recommandons d’utiliser cette fonction dans presque tous les scénar ## uniqcombiné {#agg_function-uniqcombined} -Calcule le nombre approximatif de différentes valeurs d’argument. +Calcule le nombre approximatif de différentes valeurs d'argument. ``` sql uniqCombined(HLL_precision)(x[, ...]) @@ -580,7 +625,7 @@ Le `uniqCombined` la fonction est un bon choix pour calculer le nombre de valeur La fonction prend un nombre variable de paramètres. Les paramètres peuvent être `Tuple`, `Array`, `Date`, `DateTime`, `String` ou des types numériques. -`HLL_precision` est le logarithme en base 2 du nombre de cellules dans [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog). Facultatif, vous pouvez utiliser la fonction comme `uniqCombined(x[, ...])`. La valeur par défaut pour `HLL_precision` est 17, qui est effectivement 96 Ko d’espace(2 ^ 17 cellules, 6 bits chacune). +`HLL_precision` est le logarithme en base 2 du nombre de cellules dans [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog). Facultatif, vous pouvez utiliser la fonction comme `uniqCombined(x[, ...])`. La valeur par défaut pour `HLL_precision` est 17, qui est effectivement 96 Ko d'espace(2 ^ 17 cellules, 6 bits chacune). **Valeur renvoyée** @@ -590,13 +635,13 @@ La fonction prend un nombre variable de paramètres. Les paramètres peuvent êt Fonction: -- Calcule un hachage (hachage 64 bits pour `String` et 32 bits sinon) pour tous les paramètres dans l’agrégat, puis l’utilise dans les calculs. +- Calcule un hachage (hachage 64 bits pour `String` et 32 bits sinon) pour tous les paramètres dans l'agrégat, puis l'utilise dans les calculs. -- Utilise une combinaison de trois algorithmes: tableau, table de hachage et HyperLogLog avec une table de correction d’erreur. +- Utilise une combinaison de trois algorithmes: tableau, table de hachage et HyperLogLog avec une table de correction d'erreur. For a small number of distinct elements, an array is used. When the set size is larger, a hash table is used. For a larger number of elements, HyperLogLog is used, which will occupy a fixed amount of memory. -- Fournit le résultat de manière déterministe (cela ne dépend pas de l’ordre de traitement de la requête). +- Fournit le résultat de manière déterministe (cela ne dépend pas de l'ordre de traitement de la requête). !!! note "Note" Comme il utilise le hachage 32 bits pour non-`String` type, le résultat aura une erreur très élevée pour les cardinalités significativement plus grandes que `UINT_MAX` (erreur va augmenter rapidement après quelques dizaines de milliards de valeurs distinctes), donc dans ce cas, vous devez utiliser [uniqCombined64](#agg_function-uniqcombined64) @@ -605,7 +650,7 @@ Par rapport à la [uniq](#agg_function-uniq) la fonction, la `uniqCombined`: - Consomme plusieurs fois moins de mémoire. - Calcule avec plusieurs fois plus de précision. -- A généralement des performances légèrement inférieures. Dans certains scénarios, `uniqCombined` peut faire mieux que `uniq` par exemple, avec des requêtes distribuées qui transmettent un grand nombre d’agrégation des états sur le réseau. +- A généralement des performances légèrement inférieures. Dans certains scénarios, `uniqCombined` peut faire mieux que `uniq` par exemple, avec des requêtes distribuées qui transmettent un grand nombre d'agrégation des états sur le réseau. **Voir Aussi** @@ -620,7 +665,7 @@ Même que [uniqcombiné](#agg_function-uniqcombined), mais utilise le hachage 64 ## uniqHLL12 {#agg_function-uniqhll12} -Calcule le nombre approximatif de différentes valeurs d’argument, en utilisant [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog) algorithme. +Calcule le nombre approximatif de différentes valeurs d'argument, en utilisant [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog) algorithme. ``` sql uniqHLL12(x[, ...]) @@ -638,15 +683,15 @@ La fonction prend un nombre variable de paramètres. Les paramètres peuvent êt Fonction: -- Calcule un hachage pour tous les paramètres de l’agrégat, puis l’utilise dans les calculs. +- Calcule un hachage pour tous les paramètres de l'agrégat, puis l'utilise dans les calculs. -- Utilise L’algorithme HyperLogLog pour approximer le nombre de valeurs d’argument différentes. +- Utilise L'algorithme HyperLogLog pour approximer le nombre de valeurs d'argument différentes. 212 5-bit cells are used. The size of the state is slightly more than 2.5 KB. The result is not very accurate (up to ~10% error) for small data sets (<10K elements). However, the result is fairly accurate for high-cardinality data sets (10K-100M), with a maximum error of ~1.6%. Starting from 100M, the estimation error increases, and the function will return very inaccurate results for data sets with extremely high cardinality (1B+ elements). -- Fournit le résultat déterminé (il ne dépend pas de l’ordre de traitement de la requête). +- Fournit le résultat déterminé (il ne dépend pas de l'ordre de traitement de la requête). -Nous ne recommandons pas d’utiliser cette fonction. Dans la plupart des cas, l’utilisation de la [uniq](#agg_function-uniq) ou [uniqcombiné](#agg_function-uniqcombined) fonction. +Nous ne recommandons pas d'utiliser cette fonction. Dans la plupart des cas, l'utilisation de la [uniq](#agg_function-uniq) ou [uniqcombiné](#agg_function-uniqcombined) fonction. **Voir Aussi** @@ -656,15 +701,15 @@ Nous ne recommandons pas d’utiliser cette fonction. Dans la plupart des cas, l ## uniqExact {#agg_function-uniqexact} -Calcule le nombre exact de différentes valeurs d’argument. +Calcule le nombre exact de différentes valeurs d'argument. ``` sql uniqExact(x[, ...]) ``` -L’utilisation de la `uniqExact` fonction si vous avez absolument besoin d’un résultat exact. Sinon l’utilisation de la [uniq](#agg_function-uniq) fonction. +L'utilisation de la `uniqExact` fonction si vous avez absolument besoin d'un résultat exact. Sinon l'utilisation de la [uniq](#agg_function-uniq) fonction. -Le `uniqExact` la fonction utilise plus de mémoire que `uniq` parce que la taille de l’état a surabondance de croissance que le nombre de valeurs différentes augmente. +Le `uniqExact` la fonction utilise plus de mémoire que `uniq` parce que la taille de l'état a surabondance de croissance que le nombre de valeurs différentes augmente. **Paramètre** @@ -678,31 +723,105 @@ La fonction prend un nombre variable de paramètres. Les paramètres peuvent êt ## groupArray(x), groupArray (max\_size) (x) {#agg_function-grouparray} -Crée un tableau de valeurs de l’argument. +Crée un tableau de valeurs de l'argument. Les valeurs peuvent être ajoutées au tableau dans une (indéterminée) de commande. La deuxième version (avec le `max_size` paramètre) limite la taille du tableau résultant à `max_size` élément. Exemple, `groupArray (1) (x)` est équivalent à `[any (x)]`. -Dans certains cas, vous pouvez toujours compter sur l’ordre de l’exécution. Cela s’applique aux cas où `SELECT` provient d’une sous-requête qui utilise `ORDER BY`. +Dans certains cas, vous pouvez toujours compter sur l'ordre de l'exécution. Cela s'applique aux cas où `SELECT` provient d'une sous-requête qui utilise `ORDER BY`. -## groupeparrayinsertat(valeur, position) {#grouparrayinsertatvalue-position} +## groupeparrayinsertat {#grouparrayinsertat} Insère une valeur dans le tableau à la position spécifiée. -!!! note "Note" - Cette fonction utilise des positions à base zéro, contrairement aux positions à base unique classiques pour les tableaux SQL. +**Syntaxe** -Accepts the value and position as input. If several values ​​are inserted into the same position, any of them might end up in the resulting array (the first one will be used in the case of single-threaded execution). If no value is inserted into a position, the position is assigned the default value. +``` sql +groupArrayInsertAt(default_x, size)(x, pos); +``` -Paramètres facultatifs: +Si dans une requête plusieurs valeurs sont insérées dans la même position, la fonction se comporte de la manière suivante: -- Valeur par défaut pour la substitution dans des positions vides. -- La longueur du tableau résultant. Cela vous permet de recevoir des tableaux de la même taille pour tous les agrégats clés. Lorsque vous utilisez ce paramètre, la valeur par défaut doit être spécifiée. +- Si une requête est exécutée dans un seul thread, la première des valeurs insérées est utilisée. +- Si une requête est exécutée dans plusieurs threads, le résultat est indéterminé l'une des valeurs insérées. + +**Paramètre** + +- `x` — Value to be inserted. [Expression](../syntax.md#syntax-expressions) résultant dans l'un des [types de données pris en charge](../../sql-reference/data-types/index.md). +- `pos` — Position at which the specified element `x` doit être inséré. L'indice de numérotation dans le tableau commence à partir de zéro. [UInt32](../../sql-reference/data-types/int-uint.md#uint-ranges). +- `default_x`— Default value for substituting in empty positions. Optional parameter. [Expression](../syntax.md#syntax-expressions) résultant dans le type de données configuré pour le `x` paramètre. Si `default_x` n'est pas définie, la [les valeurs par défaut](../../sql-reference/statements/create.md#create-default-values) sont utilisés. +- `size`— Length of the resulting array. Optional parameter. When using this parameter, the default value `default_x` doit être spécifié. [UInt32](../../sql-reference/data-types/int-uint.md#uint-ranges). + +**Valeur renvoyée** + +- Tableau avec des valeurs insérées. + +Type: [Tableau](../../sql-reference/data-types/array.md#data-type-array). + +**Exemple** + +Requête: + +``` sql +SELECT groupArrayInsertAt(toString(number), number * 2) FROM numbers(5); +``` + +Résultat: + +``` text +┌─groupArrayInsertAt(toString(number), multiply(number, 2))─┐ +│ ['0','','1','','2','','3','','4'] │ +└───────────────────────────────────────────────────────────┘ +``` + +Requête: + +``` sql +SELECT groupArrayInsertAt('-')(toString(number), number * 2) FROM numbers(5); +``` + +Résultat: + +``` text +┌─groupArrayInsertAt('-')(toString(number), multiply(number, 2))─┐ +│ ['0','-','1','-','2','-','3','-','4'] │ +└────────────────────────────────────────────────────────────────┘ +``` + +Requête: + +``` sql +SELECT groupArrayInsertAt('-', 5)(toString(number), number * 2) FROM numbers(5); +``` + +Résultat: + +``` text +┌─groupArrayInsertAt('-', 5)(toString(number), multiply(number, 2))─┐ +│ ['0','-','1','-','2'] │ +└───────────────────────────────────────────────────────────────────┘ +``` + +Insertion multi-thread d'éléments dans une position. + +Requête: + +``` sql +SELECT groupArrayInsertAt(number, 0) FROM numbers_mt(10) SETTINGS max_block_size = 1; +``` + +Comme un résultat de cette requête, vous obtenez entier aléatoire dans le `[0,9]` gamme. Exemple: + +``` text +┌─groupArrayInsertAt(number, 0)─┐ +│ [7] │ +└───────────────────────────────┘ +``` ## groupeparraymovingsum {#agg_function-grouparraymovingsum} -Calcule la somme mobile des valeurs d’entrée. +Calcule la somme mobile des valeurs d'entrée. ``` sql groupArrayMovingSum(numbers_for_summing) @@ -718,11 +837,11 @@ La fonction peut prendre la taille de la fenêtre comme paramètre. Si spécifi **Valeurs renvoyées** -- Tableau de la même taille et de même type que les données d’entrée. +- Tableau de la même taille et de même type que les données d'entrée. **Exemple** -La table d’échantillon: +La table d'échantillon: ``` sql CREATE TABLE t @@ -775,7 +894,7 @@ FROM t ## groupArrayMovingAvg {#agg_function-grouparraymovingavg} -Calcule la moyenne mobile des valeurs d’entrée. +Calcule la moyenne mobile des valeurs d'entrée. ``` sql groupArrayMovingAvg(numbers_for_summing) @@ -791,13 +910,13 @@ La fonction peut prendre la taille de la fenêtre comme paramètre. Si spécifi **Valeurs renvoyées** -- Tableau de la même taille et de même type que les données d’entrée. +- Tableau de la même taille et de même type que les données d'entrée. -La fonction utilise [l’arrondi vers zéro](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero). Il tronque les décimales insignifiantes pour le type de données résultant. +La fonction utilise [l'arrondi vers zéro](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero). Il tronque les décimales insignifiantes pour le type de données résultant. **Exemple** -La table d’échantillon `b`: +La table d'échantillon `b`: ``` sql CREATE TABLE t @@ -850,7 +969,7 @@ FROM t ## groupUniqArray(x), groupUniqArray (max\_size) (x) {#groupuniqarrayx-groupuniqarraymax-sizex} -Crée un tableau à partir de différentes valeurs d’argument. La consommation de mémoire est la même que pour la `uniqExact` fonction. +Crée un tableau à partir de différentes valeurs d'argument. La consommation de mémoire est la même que pour la `uniqExact` fonction. La deuxième version (avec le `max_size` paramètre) limite la taille du tableau résultant à `max_size` élément. Exemple, `groupUniqArray(1)(x)` est équivalent à `[any(x)]`. @@ -859,9 +978,9 @@ Exemple, `groupUniqArray(1)(x)` est équivalent à `[any(x)]`. Calcule une approximation [quantile](https://en.wikipedia.org/wiki/Quantile) des données numériques de la séquence. -Cette fonction s’applique [réservoir d’échantillonnage](https://en.wikipedia.org/wiki/Reservoir_sampling) avec une taille de réservoir jusqu’à 8192 et un générateur de nombres aléatoires pour l’échantillonnage. Le résultat est non-déterministe. Pour obtenir un quantile exact, Utilisez le [quantileExact](#quantileexact) fonction. +Cette fonction s'applique [réservoir d'échantillonnage](https://en.wikipedia.org/wiki/Reservoir_sampling) avec une taille de réservoir jusqu'à 8192 et un générateur de nombres aléatoires pour l'échantillonnage. Le résultat est non-déterministe. Pour obtenir un quantile exact, Utilisez le [quantileExact](#quantileexact) fonction. -Lorsque vous utilisez plusieurs `quantile*` fonctionne avec différents niveaux dans une requête, les états internes ne sont pas combinées (qui est, la requête fonctionne moins efficacement qu’il le pouvait). Dans ce cas, utilisez la [les quantiles](#quantiles) fonction. +Lorsque vous utilisez plusieurs `quantile*` fonctionne avec différents niveaux dans une requête, les états internes ne sont pas combinées (qui est, la requête fonctionne moins efficacement qu'il le pouvait). Dans ce cas, utilisez la [les quantiles](#quantiles) fonction. **Syntaxe** @@ -882,13 +1001,13 @@ Alias: `median`. Type: -- [Float64](../../sql-reference/data-types/float.md) pour l’entrée de type de données numériques. -- [Date](../../sql-reference/data-types/date.md) si les valeurs d’entrée ont le `Date` type. -- [DateTime](../../sql-reference/data-types/datetime.md) si les valeurs d’entrée ont le `DateTime` type. +- [Float64](../../sql-reference/data-types/float.md) pour l'entrée de type de données numériques. +- [Date](../../sql-reference/data-types/date.md) si les valeurs d'entrée ont le `Date` type. +- [DateTime](../../sql-reference/data-types/datetime.md) si les valeurs d'entrée ont le `DateTime` type. **Exemple** -Table d’entrée: +Table d'entrée: ``` text ┌─val─┐ @@ -922,9 +1041,9 @@ Résultat: Calcule une approximation [quantile](https://en.wikipedia.org/wiki/Quantile) des données numériques de la séquence. -Cette fonction s’applique [réservoir d’échantillonnage](https://en.wikipedia.org/wiki/Reservoir_sampling) avec une taille de réservoir jusqu’à 8192 et un algorithme déterministe d’échantillonnage. Le résultat est déterministe. Pour obtenir un quantile exact, Utilisez le [quantileExact](#quantileexact) fonction. +Cette fonction s'applique [réservoir d'échantillonnage](https://en.wikipedia.org/wiki/Reservoir_sampling) avec une taille de réservoir jusqu'à 8192 et un algorithme déterministe d'échantillonnage. Le résultat est déterministe. Pour obtenir un quantile exact, Utilisez le [quantileExact](#quantileexact) fonction. -Lorsque vous utilisez plusieurs `quantile*` fonctionne avec différents niveaux dans une requête, les états internes ne sont pas combinées (qui est, la requête fonctionne moins efficacement qu’il le pouvait). Dans ce cas, utilisez la [les quantiles](#quantiles) fonction. +Lorsque vous utilisez plusieurs `quantile*` fonctionne avec différents niveaux dans une requête, les états internes ne sont pas combinées (qui est, la requête fonctionne moins efficacement qu'il le pouvait). Dans ce cas, utilisez la [les quantiles](#quantiles) fonction. **Syntaxe** @@ -946,13 +1065,13 @@ Alias: `medianDeterministic`. Type: -- [Float64](../../sql-reference/data-types/float.md) pour l’entrée de type de données numériques. -- [Date](../../sql-reference/data-types/date.md) si les valeurs d’entrée ont le `Date` type. -- [DateTime](../../sql-reference/data-types/datetime.md) si les valeurs d’entrée ont le `DateTime` type. +- [Float64](../../sql-reference/data-types/float.md) pour l'entrée de type de données numériques. +- [Date](../../sql-reference/data-types/date.md) si les valeurs d'entrée ont le `Date` type. +- [DateTime](../../sql-reference/data-types/datetime.md) si les valeurs d'entrée ont le `DateTime` type. **Exemple** -Table d’entrée: +Table d'entrée: ``` text ┌─val─┐ @@ -984,11 +1103,11 @@ Résultat: ## quantileExact {#quantileexact} -Exactement calcule l’ [quantile](https://en.wikipedia.org/wiki/Quantile) des données numériques de la séquence. +Exactement calcule l' [quantile](https://en.wikipedia.org/wiki/Quantile) des données numériques de la séquence. To get exact value, all the passed values ​​are combined into an array, which is then partially sorted. Therefore, the function consumes `O(n)` de mémoire, où `n` est un nombre de valeurs qui ont été passées. Cependant, pour un petit nombre de valeurs, la fonction est très efficace. -Lorsque vous utilisez plusieurs `quantile*` fonctionne avec différents niveaux dans une requête, les états internes ne sont pas combinées (qui est, la requête fonctionne moins efficacement qu’il le pouvait). Dans ce cas, utilisez la [les quantiles](#quantiles) fonction. +Lorsque vous utilisez plusieurs `quantile*` fonctionne avec différents niveaux dans une requête, les états internes ne sont pas combinées (qui est, la requête fonctionne moins efficacement qu'il le pouvait). Dans ce cas, utilisez la [les quantiles](#quantiles) fonction. **Syntaxe** @@ -1009,9 +1128,9 @@ Alias: `medianExact`. Type: -- [Float64](../../sql-reference/data-types/float.md) pour l’entrée de type de données numériques. -- [Date](../../sql-reference/data-types/date.md) si les valeurs d’entrée ont le `Date` type. -- [DateTime](../../sql-reference/data-types/datetime.md) si les valeurs d’entrée ont le `DateTime` type. +- [Float64](../../sql-reference/data-types/float.md) pour l'entrée de type de données numériques. +- [Date](../../sql-reference/data-types/date.md) si les valeurs d'entrée ont le `Date` type. +- [DateTime](../../sql-reference/data-types/datetime.md) si les valeurs d'entrée ont le `DateTime` type. **Exemple** @@ -1036,11 +1155,11 @@ Résultat: ## quantileExactWeighted {#quantileexactweighted} -Exactement calcule l’ [quantile](https://en.wikipedia.org/wiki/Quantile) d’une séquence de données numériques, en tenant compte du poids de chaque élément. +Exactement calcule l' [quantile](https://en.wikipedia.org/wiki/Quantile) d'une séquence de données numériques, en tenant compte du poids de chaque élément. To get exact value, all the passed values ​​are combined into an array, which is then partially sorted. Each value is counted with its weight, as if it is present `weight` times. A hash table is used in the algorithm. Because of this, if the passed values ​​are frequently repeated, the function consumes less RAM than [quantileExact](#quantileexact). Vous pouvez utiliser cette fonction au lieu de `quantileExact` et spécifiez le poids 1. -Lorsque vous utilisez plusieurs `quantile*` fonctionne avec différents niveaux dans une requête, les états internes ne sont pas combinées (qui est, la requête fonctionne moins efficacement qu’il le pouvait). Dans ce cas, utilisez la [les quantiles](#quantiles) fonction. +Lorsque vous utilisez plusieurs `quantile*` fonctionne avec différents niveaux dans une requête, les états internes ne sont pas combinées (qui est, la requête fonctionne moins efficacement qu'il le pouvait). Dans ce cas, utilisez la [les quantiles](#quantiles) fonction. **Syntaxe** @@ -1062,13 +1181,13 @@ Alias: `medianExactWeighted`. Type: -- [Float64](../../sql-reference/data-types/float.md) pour l’entrée de type de données numériques. -- [Date](../../sql-reference/data-types/date.md) si les valeurs d’entrée ont le `Date` type. -- [DateTime](../../sql-reference/data-types/datetime.md) si les valeurs d’entrée ont le `DateTime` type. +- [Float64](../../sql-reference/data-types/float.md) pour l'entrée de type de données numériques. +- [Date](../../sql-reference/data-types/date.md) si les valeurs d'entrée ont le `Date` type. +- [DateTime](../../sql-reference/data-types/datetime.md) si les valeurs d'entrée ont le `DateTime` type. **Exemple** -Table d’entrée: +Table d'entrée: ``` text ┌─n─┬─val─┐ @@ -1102,9 +1221,9 @@ Résultat: Avec la précision déterminée calcule le [quantile](https://en.wikipedia.org/wiki/Quantile) des données numériques de la séquence. -Le résultat est déterministe (il ne dépend pas de l’ordre de traitement de la requête). La fonction est optimisée pour travailler avec des séquences qui décrivent des distributions comme les temps de chargement des pages web ou les temps de réponse du backend. +Le résultat est déterministe (il ne dépend pas de l'ordre de traitement de la requête). La fonction est optimisée pour travailler avec des séquences qui décrivent des distributions comme les temps de chargement des pages web ou les temps de réponse du backend. -Lorsque vous utilisez plusieurs `quantile*` fonctionne avec différents niveaux dans une requête, les états internes ne sont pas combinées (qui est, la requête fonctionne moins efficacement qu’il le pouvait). Dans ce cas, utilisez la [les quantiles](#quantiles) fonction. +Lorsque vous utilisez plusieurs `quantile*` fonctionne avec différents niveaux dans une requête, les états internes ne sont pas combinées (qui est, la requête fonctionne moins efficacement qu'il le pouvait). Dans ce cas, utilisez la [les quantiles](#quantiles) fonction. **Syntaxe** @@ -1142,11 +1261,11 @@ Sinon, le résultat du calcul est arrondi au plus proche multiple de 16 ms. Type: `Float32`. !!! note "Note" - Si aucune valeur n’est transmise à la fonction (lors de l’utilisation de `quantileTimingIf`), [Nan](../../sql-reference/data-types/float.md#data_type-float-nan-inf) est retourné. Le but est de différencier ces cas de cas qui aboutissent à zéro. Voir [Clause ORDER BY](../statements/select.md#select-order-by) pour des notes sur le tri `NaN` valeur. + Si aucune valeur n'est transmise à la fonction (lors de l'utilisation de `quantileTimingIf`), [Nan](../../sql-reference/data-types/float.md#data_type-float-nan-inf) est retourné. Le but est de différencier ces cas de cas qui aboutissent à zéro. Voir [Clause ORDER BY](../statements/select/order-by.md#select-order-by) pour des notes sur le tri `NaN` valeur. **Exemple** -Table d’entrée: +Table d'entrée: ``` text ┌─response_time─┐ @@ -1183,11 +1302,11 @@ Résultat: ## quantileTimingWeighted {#quantiletimingweighted} -Avec la précision déterminée calcule le [quantile](https://en.wikipedia.org/wiki/Quantile) d’une séquence de données numériques en fonction du poids de chaque élément de séquence. +Avec la précision déterminée calcule le [quantile](https://en.wikipedia.org/wiki/Quantile) d'une séquence de données numériques en fonction du poids de chaque élément de séquence. -Le résultat est déterministe (il ne dépend pas de l’ordre de traitement de la requête). La fonction est optimisée pour travailler avec des séquences qui décrivent des distributions comme les temps de chargement des pages web ou les temps de réponse du backend. +Le résultat est déterministe (il ne dépend pas de l'ordre de traitement de la requête). La fonction est optimisée pour travailler avec des séquences qui décrivent des distributions comme les temps de chargement des pages web ou les temps de réponse du backend. -Lorsque vous utilisez plusieurs `quantile*` fonctionne avec différents niveaux dans une requête, les états internes ne sont pas combinées (qui est, la requête fonctionne moins efficacement qu’il le pouvait). Dans ce cas, utilisez la [les quantiles](#quantiles) fonction. +Lorsque vous utilisez plusieurs `quantile*` fonctionne avec différents niveaux dans une requête, les états internes ne sont pas combinées (qui est, la requête fonctionne moins efficacement qu'il le pouvait). Dans ce cas, utilisez la [les quantiles](#quantiles) fonction. **Syntaxe** @@ -1227,11 +1346,11 @@ Sinon, le résultat du calcul est arrondi au plus proche multiple de 16 ms. Type: `Float32`. !!! note "Note" - Si aucune valeur n’est transmise à la fonction (lors de l’utilisation de `quantileTimingIf`), [Nan](../../sql-reference/data-types/float.md#data_type-float-nan-inf) est retourné. Le but est de différencier ces cas de cas qui aboutissent à zéro. Voir [Clause ORDER BY](../statements/select.md#select-order-by) pour des notes sur le tri `NaN` valeur. + Si aucune valeur n'est transmise à la fonction (lors de l'utilisation de `quantileTimingIf`), [Nan](../../sql-reference/data-types/float.md#data_type-float-nan-inf) est retourné. Le but est de différencier ces cas de cas qui aboutissent à zéro. Voir [Clause ORDER BY](../statements/select/order-by.md#select-order-by) pour des notes sur le tri `NaN` valeur. **Exemple** -Table d’entrée: +Table d'entrée: ``` text ┌─response_time─┬─weight─┐ @@ -1265,13 +1384,13 @@ Résultat: ## quantileTDigest {#quantiletdigest} -Calcule une approximation [quantile](https://en.wikipedia.org/wiki/Quantile) d’une séquence de données numériques utilisant [t-digest](https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf) algorithme. +Calcule une approximation [quantile](https://en.wikipedia.org/wiki/Quantile) d'une séquence de données numériques utilisant [t-digest](https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf) algorithme. -L’erreur maximale est de 1%. La consommation de mémoire est `log(n)`, où `n` est un certain nombre de valeurs. Le résultat dépend de l’ordre d’exécution de la requête et n’est pas déterministe. +L'erreur maximale est de 1%. La consommation de mémoire est `log(n)`, où `n` est un certain nombre de valeurs. Le résultat dépend de l'ordre d'exécution de la requête et n'est pas déterministe. -La performance de la fonction est inférieure à la performance de [quantile](#quantile) ou [quantileTiming](#quantiletiming). En termes de rapport entre la taille de L’état et la précision, cette fonction est bien meilleure que `quantile`. +La performance de la fonction est inférieure à la performance de [quantile](#quantile) ou [quantileTiming](#quantiletiming). En termes de rapport entre la taille de L'état et la précision, cette fonction est bien meilleure que `quantile`. -Lorsque vous utilisez plusieurs `quantile*` fonctionne avec différents niveaux dans une requête, les états internes ne sont pas combinées (qui est, la requête fonctionne moins efficacement qu’il le pouvait). Dans ce cas, utilisez la [les quantiles](#quantiles) fonction. +Lorsque vous utilisez plusieurs `quantile*` fonctionne avec différents niveaux dans une requête, les états internes ne sont pas combinées (qui est, la requête fonctionne moins efficacement qu'il le pouvait). Dans ce cas, utilisez la [les quantiles](#quantiles) fonction. **Syntaxe** @@ -1292,9 +1411,9 @@ Alias: `medianTDigest`. Type: -- [Float64](../../sql-reference/data-types/float.md) pour l’entrée de type de données numériques. -- [Date](../../sql-reference/data-types/date.md) si les valeurs d’entrée ont le `Date` type. -- [DateTime](../../sql-reference/data-types/datetime.md) si les valeurs d’entrée ont le `DateTime` type. +- [Float64](../../sql-reference/data-types/float.md) pour l'entrée de type de données numériques. +- [Date](../../sql-reference/data-types/date.md) si les valeurs d'entrée ont le `Date` type. +- [DateTime](../../sql-reference/data-types/datetime.md) si les valeurs d'entrée ont le `DateTime` type. **Exemple** @@ -1319,13 +1438,13 @@ Résultat: ## quantileTDigestWeighted {#quantiletdigestweighted} -Calcule une approximation [quantile](https://en.wikipedia.org/wiki/Quantile) d’une séquence de données numériques utilisant [t-digest](https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf) algorithme. La fonction prend en compte le poids de chaque séquence de membre. L’erreur maximale est de 1%. La consommation de mémoire est `log(n)`, où `n` est un certain nombre de valeurs. +Calcule une approximation [quantile](https://en.wikipedia.org/wiki/Quantile) d'une séquence de données numériques utilisant [t-digest](https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf) algorithme. La fonction prend en compte le poids de chaque séquence de membre. L'erreur maximale est de 1%. La consommation de mémoire est `log(n)`, où `n` est un certain nombre de valeurs. -La performance de la fonction est inférieure à la performance de [quantile](#quantile) ou [quantileTiming](#quantiletiming). En termes de rapport entre la taille de L’état et la précision, cette fonction est bien meilleure que `quantile`. +La performance de la fonction est inférieure à la performance de [quantile](#quantile) ou [quantileTiming](#quantiletiming). En termes de rapport entre la taille de L'état et la précision, cette fonction est bien meilleure que `quantile`. -Le résultat dépend de l’ordre d’exécution de la requête et n’est pas déterministe. +Le résultat dépend de l'ordre d'exécution de la requête et n'est pas déterministe. -Lorsque vous utilisez plusieurs `quantile*` fonctionne avec différents niveaux dans une requête, les états internes ne sont pas combinées (qui est, la requête fonctionne moins efficacement qu’il le pouvait). Dans ce cas, utilisez la [les quantiles](#quantiles) fonction. +Lorsque vous utilisez plusieurs `quantile*` fonctionne avec différents niveaux dans une requête, les états internes ne sont pas combinées (qui est, la requête fonctionne moins efficacement qu'il le pouvait). Dans ce cas, utilisez la [les quantiles](#quantiles) fonction. **Syntaxe** @@ -1347,9 +1466,9 @@ Alias: `medianTDigest`. Type: -- [Float64](../../sql-reference/data-types/float.md) pour l’entrée de type de données numériques. -- [Date](../../sql-reference/data-types/date.md) si les valeurs d’entrée ont le `Date` type. -- [DateTime](../../sql-reference/data-types/datetime.md) si les valeurs d’entrée ont le `DateTime` type. +- [Float64](../../sql-reference/data-types/float.md) pour l'entrée de type de données numériques. +- [Date](../../sql-reference/data-types/date.md) si les valeurs d'entrée ont le `Date` type. +- [DateTime](../../sql-reference/data-types/datetime.md) si les valeurs d'entrée ont le `DateTime` type. **Exemple** @@ -1374,7 +1493,7 @@ Résultat: ## médian {#median} -Le `median*` les fonctions sont les Alias pour le correspondant `quantile*` fonction. Ils calculent la médiane d’un échantillon de données numériques. +Le `median*` les fonctions sont les Alias pour le correspondant `quantile*` fonction. Ils calculent la médiane d'un échantillon de données numériques. Fonction: @@ -1389,7 +1508,7 @@ Fonction: **Exemple** -Table d’entrée: +Table d'entrée: ``` text ┌─val─┐ @@ -1420,31 +1539,43 @@ Toutes les fonctions quantiles ont également des fonctions quantiles correspond ## varSamp (x) {#varsampx} -Calcule le montant `Σ((x - x̅)^2) / (n - 1)`, où `n` est la taille de l’échantillon et `x̅`est la valeur moyenne de `x`. +Calcule le montant `Σ((x - x̅)^2) / (n - 1)`, où `n` est la taille de l'échantillon et `x̅`est la valeur moyenne de `x`. -Il représente une estimation non biaisée de la variance d’une variable aléatoire si les valeurs passées forment son échantillon. +Il représente une estimation non biaisée de la variance d'une variable aléatoire si les valeurs passées forment son échantillon. Retourner `Float64`. Lorsque `n <= 1`, retourner `+∞`. +!!! note "Note" + Cette fonction utilise un algorithme numériquement instable. Si vous avez besoin d' [stabilité numérique](https://en.wikipedia.org/wiki/Numerical_stability) dans les calculs, utiliser le `varSampStable` fonction. Il fonctionne plus lentement, mais fournit une erreur de calcul inférieure. + ## varPop (x) {#varpopx} -Calcule le montant `Σ((x - x̅)^2) / n`, où `n` est la taille de l’échantillon et `x̅`est la valeur moyenne de `x`. +Calcule le montant `Σ((x - x̅)^2) / n`, où `n` est la taille de l'échantillon et `x̅`est la valeur moyenne de `x`. -En d’autres termes, dispersion pour un ensemble de valeurs. Retourner `Float64`. +En d'autres termes, dispersion pour un ensemble de valeurs. Retourner `Float64`. + +!!! note "Note" + Cette fonction utilise un algorithme numériquement instable. Si vous avez besoin d' [stabilité numérique](https://en.wikipedia.org/wiki/Numerical_stability) dans les calculs, utiliser le `varPopStable` fonction. Il fonctionne plus lentement, mais fournit une erreur de calcul inférieure. ## stddevSamp (x) {#stddevsampx} Le résultat est égal à la racine carrée de `varSamp(x)`. +!!! note "Note" + Cette fonction utilise un algorithme numériquement instable. Si vous avez besoin d' [stabilité numérique](https://en.wikipedia.org/wiki/Numerical_stability) dans les calculs, utiliser le `stddevSampStable` fonction. Il fonctionne plus lentement, mais fournit une erreur de calcul inférieure. + ## stddevPop (x) {#stddevpopx} Le résultat est égal à la racine carrée de `varPop(x)`. +!!! note "Note" + Cette fonction utilise un algorithme numériquement instable. Si vous avez besoin d' [stabilité numérique](https://en.wikipedia.org/wiki/Numerical_stability) dans les calculs, utiliser le `stddevPopStable` fonction. Il fonctionne plus lentement, mais fournit une erreur de calcul inférieure. + ## topK (N) (x) {#topknx} Renvoie un tableau des valeurs approximativement les plus fréquentes dans la colonne spécifiée. Le tableau est trié par ordre décroissant de fréquence approximative des valeurs (et non par les valeurs elles-mêmes). -Met en œuvre la [Gain De Place Filtré](http://www.l2f.inesc-id.pt/~fmmb/wiki/uploads/Work/misnis.ref0a.pdf) algorithme d’analyse de TopK, basé sur l’algorithme de réduction et de combinaison de [Économie D’Espace Parallèle](https://arxiv.org/pdf/1401.0702.pdf). +Met en œuvre la [Gain De Place Filtré](http://www.l2f.inesc-id.pt/~fmmb/wiki/uploads/Work/misnis.ref0a.pdf) algorithme d'analyse de TopK, basé sur l'algorithme de réduction et de combinaison de [Économie D'Espace Parallèle](https://arxiv.org/pdf/1401.0702.pdf). ``` sql topK(N)(column) @@ -1452,17 +1583,17 @@ topK(N)(column) Cette fonction ne fournit pas un résultat garanti. Dans certaines situations, des erreurs peuvent se produire et renvoyer des valeurs fréquentes qui ne sont pas les valeurs les plus fréquentes. -Nous vous recommandons d’utiliser l’ `N < 10` valeur; performance est réduite avec grand `N` valeur. Valeur maximale de `N = 65536`. +Nous vous recommandons d'utiliser l' `N < 10` valeur; performance est réduite avec grand `N` valeur. Valeur maximale de `N = 65536`. **Paramètre** -- ‘N’ est le nombre d’éléments de retour. +- ‘N’ est le nombre d'éléments de retour. Si le paramètre est omis, la valeur par défaut 10 est utilisé. **Argument** -- ’ x ’ – The value to calculate frequency. +- ' x ' – The value to calculate frequency. **Exemple** @@ -1524,14 +1655,23 @@ Calcule la valeur de `Σ((x - x̅)(y - y̅)) / (n - 1)`. Renvoie Float64. Lorsque `n <= 1`, returns +∞. +!!! note "Note" + Cette fonction utilise un algorithme numériquement instable. Si vous avez besoin d' [stabilité numérique](https://en.wikipedia.org/wiki/Numerical_stability) dans les calculs, utiliser le `covarSampStable` fonction. Il fonctionne plus lentement, mais fournit une erreur de calcul inférieure. + ## covarPop (x, y) {#covarpopx-y} Calcule la valeur de `Σ((x - x̅)(y - y̅)) / n`. +!!! note "Note" + Cette fonction utilise un algorithme numériquement instable. Si vous avez besoin d' [stabilité numérique](https://en.wikipedia.org/wiki/Numerical_stability) dans les calculs, utiliser le `covarPopStable` fonction. Cela fonctionne plus lentement mais fournit une erreur de calcul inférieure. + ## corr (x, y) {#corrx-y} Calcule le coefficient de corrélation de Pearson: `Σ((x - x̅)(y - y̅)) / sqrt(Σ((x - x̅)^2) * Σ((y - y̅)^2))`. +!!! note "Note" + Cette fonction utilise un algorithme numériquement instable. Si vous avez besoin d' [stabilité numérique](https://en.wikipedia.org/wiki/Numerical_stability) dans les calculs, utiliser le `corrStable` fonction. Il fonctionne plus lentement, mais fournit une erreur de calcul inférieure. + ## categoricalInformationValue {#categoricalinformationvalue} Calcule la valeur de `(P(tag = 1) - P(tag = 0))(log(P(tag = 1)) - log(P(tag = 0)))` pour chaque catégorie. @@ -1540,7 +1680,7 @@ Calcule la valeur de `(P(tag = 1) - P(tag = 0))(log(P(tag = 1)) - log(P(tag = 0) categoricalInformationValue(category1, category2, ..., tag) ``` -Le résultat indique comment une caractéristique discrète (catégorique) `[category1, category2, ...]` contribuer à un modèle d’apprentissage qui prédit la valeur de `tag`. +Le résultat indique comment une caractéristique discrète (catégorique) `[category1, category2, ...]` contribuer à un modèle d'apprentissage qui prédit la valeur de `tag`. ## simplelineearregression {#simplelinearregression} @@ -1583,24 +1723,24 @@ SELECT arrayReduce('simpleLinearRegression', [0, 1, 2, 3], [3, 4, 5, 6]) ## stochasticLinearRegression {#agg_functions-stochasticlinearregression} -Cette fonction implémente la régression linéaire stochastique. Il prend en charge les paramètres personnalisés pour le taux d’apprentissage, le coefficient de régularisation L2, la taille de mini-lot et a peu de méthodes pour mettre à jour les poids ([Adam](https://en.wikipedia.org/wiki/Stochastic_gradient_descent#Adam) (utilisé par défaut), [simple SGD](https://en.wikipedia.org/wiki/Stochastic_gradient_descent), [Élan](https://en.wikipedia.org/wiki/Stochastic_gradient_descent#Momentum), [Nesterov](https://mipt.ru/upload/medialibrary/d7e/41-91.pdf)). +Cette fonction implémente la régression linéaire stochastique. Il prend en charge les paramètres personnalisés pour le taux d'apprentissage, le coefficient de régularisation L2, la taille de mini-lot et a peu de méthodes pour mettre à jour les poids ([Adam](https://en.wikipedia.org/wiki/Stochastic_gradient_descent#Adam) (utilisé par défaut), [simple SGD](https://en.wikipedia.org/wiki/Stochastic_gradient_descent), [Élan](https://en.wikipedia.org/wiki/Stochastic_gradient_descent#Momentum), [Nesterov](https://mipt.ru/upload/medialibrary/d7e/41-91.pdf)). ### Paramètre {#agg_functions-stochasticlinearregression-parameters} -Il y a 4 paramètres personnalisables. Ils sont passés à la fonction séquentiellement, mais il n’est pas nécessaire de passer tous les quatre-les valeurs par défaut seront utilisées, mais un bon modèle nécessite un réglage des paramètres. +Il y a 4 paramètres personnalisables. Ils sont passés à la fonction séquentiellement, mais il n'est pas nécessaire de passer tous les quatre-les valeurs par défaut seront utilisées, mais un bon modèle nécessite un réglage des paramètres. ``` text stochasticLinearRegression(1.0, 1.0, 10, 'SGD') ``` -1. `learning rate` est le coefficient sur la longueur de l’étape, lorsque l’étape de descente de gradient est effectuée. Un taux d’apprentissage trop élevé peut entraîner des poids infinis du modèle. Par défaut est `0.00001`. +1. `learning rate` est le coefficient sur la longueur de l'étape, lorsque l'étape de descente de gradient est effectuée. Un taux d'apprentissage trop élevé peut entraîner des poids infinis du modèle. Par défaut est `0.00001`. 2. `l2 regularization coefficient` ce qui peut aider à éviter le surajustement. Par défaut est `0.1`. -3. `mini-batch size` définit le nombre d’éléments, dont les gradients seront calculés et additionnés pour effectuer une étape de descente de gradient. La descente stochastique Pure utilise un élément, mais avoir de petits lots (environ 10 éléments) rend les étapes de gradient plus stables. Par défaut est `15`. +3. `mini-batch size` définit le nombre d'éléments, dont les gradients seront calculés et additionnés pour effectuer une étape de descente de gradient. La descente stochastique Pure utilise un élément, mais avoir de petits lots (environ 10 éléments) rend les étapes de gradient plus stables. Par défaut est `15`. 4. `method for updating weights` ils sont: `Adam` (par défaut), `SGD`, `Momentum`, `Nesterov`. `Momentum` et `Nesterov` nécessitent un peu plus de calculs et de mémoire, mais ils sont utiles en termes de vitesse de convergence et de stabilité des méthodes de gradient stochastique. ### Utilisation {#agg_functions-stochasticlinearregression-usage} -`stochasticLinearRegression` est utilisé en deux étapes: ajustement du modèle et prédiction sur de nouvelles données. Afin de correspondre le modèle et l’enregistrer son état pour utilisation ultérieure nous utilisons `-State` combinator, qui enregistre essentiellement l’état (poids du modèle, etc.). +`stochasticLinearRegression` est utilisé en deux étapes: ajustement du modèle et prédiction sur de nouvelles données. Afin de correspondre le modèle et l'enregistrer son état pour utilisation ultérieure nous utilisons `-State` combinator, qui enregistre essentiellement l'état (poids du modèle, etc.). Pour prédire nous utilisons la fonction [evalMLMethod](../functions/machine-learning-functions.md#machine_learning_methods-evalmlmethod) qui prend un état comme un argument ainsi que des fonctionnalités à prévoir sur. @@ -1622,12 +1762,12 @@ stochasticLinearRegressionState(0.1, 0.0, 5, 'SGD')(target, param1, param2) AS state FROM train_data; ``` -Ici, nous devons également insérer des données dans `train_data` table. Le nombre de paramètres n’est pas fixe, il dépend uniquement du nombre d’arguments, passés dans `linearRegressionState`. Ils doivent tous être des valeurs numériques. +Ici, nous devons également insérer des données dans `train_data` table. Le nombre de paramètres n'est pas fixe, il dépend uniquement du nombre d'arguments, passés dans `linearRegressionState`. Ils doivent tous être des valeurs numériques. Notez que la colonne avec la valeur cible (que nous aimerions apprendre à prédire) est insérée comme premier argument. **2.** Prédire -Après avoir enregistré un État dans la table, nous pouvons l’utiliser plusieurs fois pour la prédiction, ou même fusionner avec d’autres États et créer de nouveaux modèles encore meilleurs. +Après avoir enregistré un État dans la table, nous pouvons l'utiliser plusieurs fois pour la prédiction, ou même fusionner avec d'autres États et créer de nouveaux modèles encore meilleurs. ``` sql WITH (SELECT state FROM your_model) AS model SELECT @@ -1640,13 +1780,13 @@ La requête renvoie une colonne de valeurs prédites. Notez que le premier argum ### Note {#agg_functions-stochasticlinearregression-notes} -1. Pour fusionner deux modèles l’utilisateur peut créer une telle requête: +1. Pour fusionner deux modèles l'utilisateur peut créer une telle requête: `sql SELECT state1 + state2 FROM your_models` où `your_models` le tableau contient les deux modèles. Cette requête renvoie la nouvelle `AggregateFunctionState` objet. -2. L’utilisateur peut récupérer les poids du modèle pour ses propres fins, sans enregistrer le modèle, si aucune `-State` combinator est utilisé. +2. L'utilisateur peut récupérer les poids du modèle pour ses propres fins, sans enregistrer le modèle, si aucune `-State` combinator est utilisé. `sql SELECT stochasticLinearRegression(0.01)(target, param1, param2) FROM train_data` - Une telle requête s’adaptera au Modèle et retournera ses poids-d’abord sont des poids, qui correspondent aux paramètres du modèle, le dernier est un biais. Ainsi, dans l’exemple ci-dessus, la requête renvoie une colonne avec 3 valeurs. + Une telle requête s'adaptera au Modèle et retournera ses poids-d'abord sont des poids, qui correspondent aux paramètres du modèle, le dernier est un biais. Ainsi, dans l'exemple ci-dessus, la requête renvoie une colonne avec 3 valeurs. **Voir Aussi** @@ -1661,7 +1801,7 @@ Cette fonction implémente la régression logistique stochastique. Il peut être Les paramètres sont exactement les mêmes que dans stochasticLinearRegression: `learning rate`, `l2 regularization coefficient`, `mini-batch size`, `method for updating weights`. -Pour plus d’informations, voir [paramètre](#agg_functions-stochasticlinearregression-parameters). +Pour plus d'informations, voir [paramètre](#agg_functions-stochasticlinearregression-parameters). ``` text stochasticLogisticRegression(1.0, 1.0, 10, 'SGD') @@ -1707,7 +1847,7 @@ stochasticLogisticRegression(1.0, 1.0, 10, 'SGD') ## groupBitmapAnd {#groupbitmapand} -Calculs le et d’une colonne bitmap, retour cardinalité de type UInt64, si Ajouter suffixe-État, puis retour [objet bitmap](../../sql-reference/functions/bitmap-functions.md). +Calculs le et d'une colonne bitmap, retour cardinalité de type UInt64, si Ajouter suffixe-État, puis retour [objet bitmap](../../sql-reference/functions/bitmap-functions.md). ``` sql groupBitmapAnd(expr) @@ -1750,7 +1890,7 @@ SELECT arraySort(bitmapToArray(groupBitmapAndState(z))) FROM bitmap_column_expr_ ## groupBitmapOr {#groupbitmapor} -Calculs le ou d’une colonne bitmap, retour cardinalité de type UInt64, si Ajouter suffixe-État, puis retour [objet bitmap](../../sql-reference/functions/bitmap-functions.md). C’est l’équivalent de `groupBitmapMerge`. +Calculs le ou d'une colonne bitmap, retour cardinalité de type UInt64, si Ajouter suffixe-État, puis retour [objet bitmap](../../sql-reference/functions/bitmap-functions.md). C'est l'équivalent de `groupBitmapMerge`. ``` sql groupBitmapOr(expr) @@ -1793,7 +1933,7 @@ SELECT arraySort(bitmapToArray(groupBitmapOrState(z))) FROM bitmap_column_expr_t ## groupBitmapXor {#groupbitmapxor} -Calculs le XOR d’une colonne bitmap, retour cardinalité de type UInt64, si Ajouter suffixe-État, puis retour [objet bitmap](../../sql-reference/functions/bitmap-functions.md). +Calculs le XOR d'une colonne bitmap, retour cardinalité de type UInt64, si Ajouter suffixe-État, puis retour [objet bitmap](../../sql-reference/functions/bitmap-functions.md). ``` sql groupBitmapOr(expr) diff --git a/docs/fr/sql-reference/ansi.md b/docs/fr/sql-reference/ansi.md deleted file mode 120000 index 3cf6bffed67..00000000000 --- a/docs/fr/sql-reference/ansi.md +++ /dev/null @@ -1 +0,0 @@ -../../en/sql-reference/ansi.md \ No newline at end of file diff --git a/docs/fr/sql-reference/ansi.md b/docs/fr/sql-reference/ansi.md new file mode 100644 index 00000000000..8fa5e272085 --- /dev/null +++ b/docs/fr/sql-reference/ansi.md @@ -0,0 +1,180 @@ +--- +machine_translated: true +machine_translated_rev: ad252bbb4f7e2899c448eb42ecc39ff195c8faa1 +toc_priority: 40 +toc_title: "La Compatibilit\xE9 ANSI" +--- + +# Compatibilité ANSI SQL du dialecte CLICKHOUSE SQL {#ansi-sql-compatibility-of-clickhouse-sql-dialect} + +!!! note "Note" + Cet article s'appuie sur le tableau 38, “Feature taxonomy and definition for mandatory features”, Annex F of ISO/IEC CD 9075-2:2013. + +## Différences de comportement {#differences-in-behaviour} + +Le tableau suivant répertorie les cas où la fonctionnalité de requête fonctionne dans ClickHouse, mais ne se comporte pas comme spécifié dans ANSI SQL. + +| Feature ID | Nom De La Fonctionnalité | Différence | +|------------|-------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------| +| E011 | Types de données numériques | Le littéral numérique avec période est interprété comme approximatif (`Float64`) au lieu de exact (`Decimal`) | +| E051-05 | Les éléments sélectionnés peuvent être renommés | Les renommages d'élément ont une portée de visibilité plus large que le simple résultat de sélection | +| E141-01 | Contraintes non nulles | `NOT NULL` est implicite pour les colonnes de table par défaut | +| E011-04 | Opérateurs arithmétiques | Clickhouse déborde au lieu de l'arithmétique vérifiée et modifie le type de données de résultat en fonction des règles personnalisées | + +## Fonction D'État {#feature-status} + +| Feature ID | Nom De La Fonctionnalité | Statut | Commentaire | +|------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **E011** | **Types de données numériques** | **Partiel**{.text-warning} | | +| E011-01 | Types de données INTEGER et SMALLINT | Oui{.text-success} | | +| E011-02 | Types de données réel, double précision et flottant types de données | Partiel{.text-warning} | `FLOAT()`, `REAL` et `DOUBLE PRECISION` ne sont pas pris en charge | +| E011-03 | Types de données décimales et numériques | Partiel{.text-warning} | Seulement `DECIMAL(p,s)` est pris en charge, pas `NUMERIC` | +| E011-04 | Opérateurs arithmétiques | Oui{.text-success} | | +| E011-05 | Comparaison numérique | Oui{.text-success} | | +| E011-06 | Casting implicite parmi les types de données numériques | Aucun{.text-danger} | ANSI SQL permet la distribution implicite arbitraire entre les types numériques, tandis que ClickHouse repose sur des fonctions ayant plusieurs surcharges au lieu de la distribution implicite | +| **E021** | **Types de chaînes de caractères** | **Partiel**{.text-warning} | | +| E021-01 | Type de données CARACTÈRE | Aucun{.text-danger} | | +| E021-02 | TYPE DE DONNÉES variable de caractère | Aucun{.text-danger} | `String` se comporte de la même manière, mais sans limite de longueur entre parenthèses | +| E021-03 | Littéraux de caractères | Partiel{.text-warning} | Aucune concaténation automatique de littéraux consécutifs et prise en charge du jeu de caractères | +| E021-04 | Fonction CHARACTER\_LENGTH | Partiel{.text-warning} | Aucun `USING` clause | +| E021-05 | Fonction OCTET\_LENGTH | Aucun{.text-danger} | `LENGTH` se comporte de la même façon | +| E021-06 | SUBSTRING | Partiel{.text-warning} | Pas de support pour `SIMILAR` et `ESCAPE` clauses, pas de `SUBSTRING_REGEX` variante | +| E021-07 | Concaténation de caractères | Partiel{.text-warning} | Aucun `COLLATE` clause | +| E021-08 | Fonctions supérieures et inférieures | Oui{.text-success} | | +| E021-09 | La fonction TRIM | Oui{.text-success} | | +| E021-10 | Conversion implicite entre les types de chaînes de caractères de longueur fixe et de longueur variable | Aucun{.text-danger} | ANSI SQL permet la distribution implicite arbitraire entre les types de chaîne, tandis que ClickHouse repose sur des fonctions ayant plusieurs surcharges au lieu de la distribution implicite | +| E021-11 | La POSITION de la fonction | Partiel{.text-warning} | Pas de support pour `IN` et `USING` clauses, pas de `POSITION_REGEX` variante | +| E021-12 | Comparaison de caractères | Oui{.text-success} | | +| **E031** | **Identificateur** | **Partiel**{.text-warning} | | +| E031-01 | Identificateurs délimités | Partiel{.text-warning} | Le support littéral Unicode est limité | +| E031-02 | Identificateurs minuscules | Oui{.text-success} | | +| E031-03 | Fuite de soulignement | Oui{.text-success} | | +| **E051** | **Spécification de requête de base** | **Partiel**{.text-warning} | | +| E051-01 | SELECT DISTINCT | Oui{.text-success} | | +| E051-02 | Groupe par clause | Oui{.text-success} | | +| E051-04 | GROUP BY peut contenir des colonnes `
`, `CHECK
` requête. + - `SHOW COLUMNS`. Niveau: `COLUMN`. Permet d'exécuter des `SHOW CREATE TABLE`, `DESCRIBE` requête. + - `SHOW DICTIONARIES`. Niveau: `DICTIONARY`. Permet d'exécuter des `SHOW DICTIONARIES`, `SHOW CREATE DICTIONARY`, `EXISTS ` requête. + +**Note** + +Un utilisateur a le `SHOW` privilège s'il a un autre privilège concernant la table, le dictionnaire ou la base de données spécifiés. + +### KILL QUERY {#grant-kill-query} + +Permet d'effectuer les [KILL](misc.md#kill-query-statement) requêtes correspondant à la hiérarchie de privilèges suivante: + +Le niveau de privilège: `GLOBAL`. + +**Note** + +`KILL QUERY` privilège permet à un utilisateur de tuer les requêtes des autres utilisateurs. + +### ACCESS MANAGEMENT {#grant-access-management} + +Permet à un utilisateur d'effectuer des requêtes qui gèrent les utilisateurs, les rôles et les stratégies de ligne. + +- `ACCESS MANAGEMENT`. Niveau: `GROUP` + - `CREATE USER`. Niveau: `GLOBAL` + - `ALTER USER`. Niveau: `GLOBAL` + - `DROP USER`. Niveau: `GLOBAL` + - `CREATE ROLE`. Niveau: `GLOBAL` + - `ALTER ROLE`. Niveau: `GLOBAL` + - `DROP ROLE`. Niveau: `GLOBAL` + - `ROLE ADMIN`. Niveau: `GLOBAL` + - `CREATE ROW POLICY`. Niveau: `GLOBAL`. Alias: `CREATE POLICY` + - `ALTER ROW POLICY`. Niveau: `GLOBAL`. Alias: `ALTER POLICY` + - `DROP ROW POLICY`. Niveau: `GLOBAL`. Alias: `DROP POLICY` + - `CREATE QUOTA`. Niveau: `GLOBAL` + - `ALTER QUOTA`. Niveau: `GLOBAL` + - `DROP QUOTA`. Niveau: `GLOBAL` + - `CREATE SETTINGS PROFILE`. Niveau: `GLOBAL`. Alias: `CREATE PROFILE` + - `ALTER SETTINGS PROFILE`. Niveau: `GLOBAL`. Alias: `ALTER PROFILE` + - `DROP SETTINGS PROFILE`. Niveau: `GLOBAL`. Alias: `DROP PROFILE` + - `SHOW ACCESS`. Niveau: `GROUP` + - `SHOW_USERS`. Niveau: `GLOBAL`. Alias: `SHOW CREATE USER` + - `SHOW_ROLES`. Niveau: `GLOBAL`. Alias: `SHOW CREATE ROLE` + - `SHOW_ROW_POLICIES`. Niveau: `GLOBAL`. Alias: `SHOW POLICIES`, `SHOW CREATE ROW POLICY`, `SHOW CREATE POLICY` + - `SHOW_QUOTAS`. Niveau: `GLOBAL`. Alias: `SHOW CREATE QUOTA` + - `SHOW_SETTINGS_PROFILES`. Niveau: `GLOBAL`. Alias: `SHOW PROFILES`, `SHOW CREATE SETTINGS PROFILE`, `SHOW CREATE PROFILE` + +Le `ROLE ADMIN` le privilège permet à un utilisateur d'accorder et de révoquer tous les rôles, y compris ceux qui ne lui sont pas accordés avec l'option admin. + +### SYSTEM {#grant-system} + +Permet à un utilisateur d'effectuer la [SYSTEM](system.md) requêtes correspondant à la hiérarchie de privilèges suivante. + +- `SYSTEM`. Niveau: `GROUP` + - `SYSTEM SHUTDOWN`. Niveau: `GLOBAL`. Alias: `SYSTEM KILL`, `SHUTDOWN` + - `SYSTEM DROP CACHE`. Alias: `DROP CACHE` + - `SYSTEM DROP DNS CACHE`. Niveau: `GLOBAL`. Alias: `SYSTEM DROP DNS`, `DROP DNS CACHE`, `DROP DNS` + - `SYSTEM DROP MARK CACHE`. Niveau: `GLOBAL`. Alias: `SYSTEM DROP MARK`, `DROP MARK CACHE`, `DROP MARKS` + - `SYSTEM DROP UNCOMPRESSED CACHE`. Niveau: `GLOBAL`. Alias: `SYSTEM DROP UNCOMPRESSED`, `DROP UNCOMPRESSED CACHE`, `DROP UNCOMPRESSED` + - `SYSTEM RELOAD`. Niveau: `GROUP` + - `SYSTEM RELOAD CONFIG`. Niveau: `GLOBAL`. Alias: `RELOAD CONFIG` + - `SYSTEM RELOAD DICTIONARY`. Niveau: `GLOBAL`. Alias: `SYSTEM RELOAD DICTIONARIES`, `RELOAD DICTIONARY`, `RELOAD DICTIONARIES` + - `SYSTEM RELOAD EMBEDDED DICTIONARIES`. Niveau: `GLOBAL`. Alias: R`ELOAD EMBEDDED DICTIONARIES` + - `SYSTEM MERGES`. Niveau: `TABLE`. Alias: `SYSTEM STOP MERGES`, `SYSTEM START MERGES`, `STOP MERGES`, `START MERGES` + - `SYSTEM TTL MERGES`. Niveau: `TABLE`. Alias: `SYSTEM STOP TTL MERGES`, `SYSTEM START TTL MERGES`, `STOP TTL MERGES`, `START TTL MERGES` + - `SYSTEM FETCHES`. Niveau: `TABLE`. Alias: `SYSTEM STOP FETCHES`, `SYSTEM START FETCHES`, `STOP FETCHES`, `START FETCHES` + - `SYSTEM MOVES`. Niveau: `TABLE`. Alias: `SYSTEM STOP MOVES`, `SYSTEM START MOVES`, `STOP MOVES`, `START MOVES` + - `SYSTEM SENDS`. Niveau: `GROUP`. Alias: `SYSTEM STOP SENDS`, `SYSTEM START SENDS`, `STOP SENDS`, `START SENDS` + - `SYSTEM DISTRIBUTED SENDS`. Niveau: `TABLE`. Alias: `SYSTEM STOP DISTRIBUTED SENDS`, `SYSTEM START DISTRIBUTED SENDS`, `STOP DISTRIBUTED SENDS`, `START DISTRIBUTED SENDS` + - `SYSTEM REPLICATED SENDS`. Niveau: `TABLE`. Alias: `SYSTEM STOP REPLICATED SENDS`, `SYSTEM START REPLICATED SENDS`, `STOP REPLICATED SENDS`, `START REPLICATED SENDS` + - `SYSTEM REPLICATION QUEUES`. Niveau: `TABLE`. Alias: `SYSTEM STOP REPLICATION QUEUES`, `SYSTEM START REPLICATION QUEUES`, `STOP REPLICATION QUEUES`, `START REPLICATION QUEUES` + - `SYSTEM SYNC REPLICA`. Niveau: `TABLE`. Alias: `SYNC REPLICA` + - `SYSTEM RESTART REPLICA`. Niveau: `TABLE`. Alias: `RESTART REPLICA` + - `SYSTEM FLUSH`. Niveau: `GROUP` + - `SYSTEM FLUSH DISTRIBUTED`. Niveau: `TABLE`. Alias: `FLUSH DISTRIBUTED` + - `SYSTEM FLUSH LOGS`. Niveau: `GLOBAL`. Alias: `FLUSH LOGS` + +Le `SYSTEM RELOAD EMBEDDED DICTIONARIES` privilège implicitement accordé par le `SYSTEM RELOAD DICTIONARY ON *.*` privilège. + +### INTROSPECTION {#grant-introspection} + +Permet l'utilisation de [introspection](../../operations/optimizing-performance/sampling-query-profiler.md) fonction. + +- `INTROSPECTION`. Niveau: `GROUP`. Alias: `INTROSPECTION FUNCTIONS` + - `addressToLine`. Niveau: `GLOBAL` + - `addressToSymbol`. Niveau: `GLOBAL` + - `demangle`. Niveau: `GLOBAL` + +### SOURCES {#grant-sources} + +Permet d'utiliser des sources de données externes. S'applique à [moteurs de table](../../engines/table-engines/index.md) et [les fonctions de table](../table-functions/index.md#table-functions). + +- `SOURCES`. Niveau: `GROUP` + - `FILE`. Niveau: `GLOBAL` + - `URL`. Niveau: `GLOBAL` + - `REMOTE`. Niveau: `GLOBAL` + - `YSQL`. Niveau: `GLOBAL` + - `ODBC`. Niveau: `GLOBAL` + - `JDBC`. Niveau: `GLOBAL` + - `HDFS`. Niveau: `GLOBAL` + - `S3`. Niveau: `GLOBAL` + +Le `SOURCES` privilège permet l'utilisation de toutes les sources. Vous pouvez également accorder un privilège pour chaque source individuellement. Pour utiliser les sources, vous avez besoin de privilèges supplémentaires. + +Exemple: + +- Pour créer une table avec [Moteur de table MySQL](../../engines/table-engines/integrations/mysql.md), vous avez besoin `CREATE TABLE (ON db.table_name)` et `MYSQL` privilège. +- L'utilisation de la [fonction de table mysql](../table-functions/mysql.md), vous avez besoin `CREATE TEMPORARY TABLE` et `MYSQL` privilège. + +### dictGet {#grant-dictget} + +- `dictGet`. Alias: `dictHas`, `dictGetHierarchy`, `dictIsIn` + +Permet à un utilisateur d'exécuter [dictGet](../functions/ext-dict-functions.md#dictget), [dictHas](../functions/ext-dict-functions.md#dicthas), [dictGetHierarchy](../functions/ext-dict-functions.md#dictgethierarchy), [dictisine](../functions/ext-dict-functions.md#dictisin) fonction. + +Niveau de privilège: `DICTIONARY`. + +**Exemple** + +- `GRANT dictGet ON mydb.mydictionary TO john` +- `GRANT dictGet ON mydictionary TO john` + +### ALL {#grant-all} + +Les subventions de tous les privilèges sur l'entité réglementée à un compte d'utilisateur ou un rôle. + +### NONE {#grant-none} + +N'accorde pas de privilèges. + +### ADMIN OPTION {#admin-option-privilege} + +Le `ADMIN OPTION` le privilège permet à un utilisateur d'accorder son rôle à un autre utilisateur. + +[Article Original](https://clickhouse.tech/docs/en/query_language/grant/) diff --git a/docs/fr/sql-reference/statements/index.md b/docs/fr/sql-reference/statements/index.md index eef857c8cdb..f08d64cee39 100644 --- a/docs/fr/sql-reference/statements/index.md +++ b/docs/fr/sql-reference/statements/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 -toc_folder_title: Statements +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "D\xE9claration" toc_priority: 31 --- diff --git a/docs/fr/sql-reference/statements/insert-into.md b/docs/fr/sql-reference/statements/insert-into.md index 99e80bfb796..95026d142e3 100644 --- a/docs/fr/sql-reference/statements/insert-into.md +++ b/docs/fr/sql-reference/statements/insert-into.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 34 toc_title: INSERT INTO --- ## INSERT {#insert} -L’ajout de données. +L'ajout de données. Format de requête de base: @@ -20,9 +20,9 @@ La requête peut spécifier une liste de colonnes à insérer `[(c1, c2, c3)]`. - Les valeurs calculées à partir `DEFAULT` expressions spécifiées dans la définition de la table. - Zéros et chaînes vides, si `DEFAULT` les expressions ne sont pas définies. -Si [strict\_insert\_defaults=1](../../operations/settings/settings.md), les colonnes qui n’ont pas `DEFAULT` défini doit être répertorié dans la requête. +Si [strict\_insert\_defaults=1](../../operations/settings/settings.md), les colonnes qui n'ont pas `DEFAULT` défini doit être répertorié dans la requête. -Les données peuvent être transmises à L’INSERT dans n’importe quel [format](../../interfaces/formats.md#formats) soutenu par ClickHouse. Le format doit être spécifié explicitement dans la requête: +Les données peuvent être transmises à L'INSERT dans n'importe quel [format](../../interfaces/formats.md#formats) soutenu par ClickHouse. Le format doit être spécifié explicitement dans la requête: ``` sql INSERT INTO [db.]table [(c1, c2, c3)] FORMAT format_name data_set @@ -34,7 +34,7 @@ For example, the following query format is identical to the basic version of INS INSERT INTO [db.]table [(c1, c2, c3)] FORMAT Values (v11, v12, v13), (v21, v22, v23), ... ``` -ClickHouse supprime tous les espaces et un saut de ligne (s’il y en a un) avant les données. Lors de la formation d’une requête, nous recommandons de placer les données sur une nouvelle ligne après les opérateurs de requête (ceci est important si les données commencent par des espaces). +ClickHouse supprime tous les espaces et un saut de ligne (s'il y en a un) avant les données. Lors de la formation d'une requête, nous recommandons de placer les données sur une nouvelle ligne après les opérateurs de requête (ceci est important si les données commencent par des espaces). Exemple: @@ -44,21 +44,21 @@ INSERT INTO t FORMAT TabSeparated 22 Qwerty ``` -Vous pouvez insérer des données séparément de la requête à l’aide du client de ligne de commande ou de L’interface HTTP. Pour plus d’informations, consultez la section “[Interface](../../interfaces/index.md#interfaces)”. +Vous pouvez insérer des données séparément de la requête à l'aide du client de ligne de commande ou de L'interface HTTP. Pour plus d'informations, consultez la section “[Interface](../../interfaces/index.md#interfaces)”. ### Contraintes {#constraints} Si la table a [contraintes](create.md#constraints), their expressions will be checked for each row of inserted data. If any of those constraints is not satisfied — server will raise an exception containing constraint name and expression, the query will be stopped. -### Insertion Des Résultats De `SELECT` {#insert_query_insert-select} +### Insertion des résultats de `SELECT` {#insert_query_insert-select} ``` sql INSERT INTO [db.]table [(c1, c2, c3)] SELECT ... ``` -Les colonnes sont mappées en fonction de leur position dans la clause SELECT. Cependant, leurs noms dans L’expression SELECT et la table pour INSERT peuvent différer. Si nécessaire, la coulée de type est effectuée. +Les colonnes sont mappées en fonction de leur position dans la clause SELECT. Cependant, leurs noms dans L'expression SELECT et la table pour INSERT peuvent différer. Si nécessaire, la coulée de type est effectuée. -Aucun des formats de données à l’exception des Valeurs permettent de définir des valeurs d’expressions telles que `now()`, `1 + 2` et ainsi de suite. Le format des valeurs permet une utilisation limitée des expressions, mais ce n’est pas recommandé, car dans ce cas, un code inefficace est utilisé pour leur exécution. +Aucun des formats de données à l'exception des Valeurs permettent de définir des valeurs d'expressions telles que `now()`, `1 + 2` et ainsi de suite. Le format des valeurs permet une utilisation limitée des expressions, mais ce n'est pas recommandé, car dans ce cas, un code inefficace est utilisé pour leur exécution. Les autres requêtes de modification des parties de données ne sont pas prises en charge: `UPDATE`, `DELETE`, `REPLACE`, `MERGE`, `UPSERT`, `INSERT UPDATE`. Cependant, vous pouvez supprimer les anciennes données en utilisant `ALTER TABLE ... DROP PARTITION`. @@ -67,7 +67,7 @@ Cependant, vous pouvez supprimer les anciennes données en utilisant `ALTER TABL ### Considérations De Performance {#performance-considerations} -`INSERT` trie les données d’entrée par la clé primaire et les divise en partitions par une clé de partition. Si vous insérez des données dans plusieurs partitions à la fois, cela peut réduire considérablement les performances de l’ `INSERT` requête. Pour éviter cela: +`INSERT` trie les données d'entrée par la clé primaire et les divise en partitions par une clé de partition. Si vous insérez des données dans plusieurs partitions à la fois, cela peut réduire considérablement les performances de l' `INSERT` requête. Pour éviter cela: - Ajoutez des données en lots assez importants, tels que 100 000 lignes à la fois. - Groupez les données par une clé de partition avant de les télécharger sur ClickHouse. diff --git a/docs/fr/sql-reference/statements/misc.md b/docs/fr/sql-reference/statements/misc.md index a25b61705e9..bd8375c3ec0 100644 --- a/docs/fr/sql-reference/statements/misc.md +++ b/docs/fr/sql-reference/statements/misc.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 -toc_priority: 39 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_priority: 41 toc_title: Autre --- @@ -13,7 +13,7 @@ Cette requête est exactement la même que `CREATE`, mais - Au lieu de la parole `CREATE` il utilise le mot `ATTACH`. - La requête ne crée pas de données sur le disque, mais suppose que les données sont déjà aux endroits appropriés, et ajoute simplement des informations sur la table au serveur. - Après avoir exécuté une requête ATTACH, le serveur connaîtra l’existence de la table. + Après avoir exécuté une requête ATTACH, le serveur connaîtra l'existence de la table. Si la table a été précédemment détachée (`DETACH`), ce qui signifie que sa structure est connue, vous pouvez utiliser un raccourci sans définir la structure. @@ -21,7 +21,7 @@ Si la table a été précédemment détachée (`DETACH`), ce qui signifie que sa ATTACH TABLE [IF NOT EXISTS] [db.]name [ON CLUSTER cluster] ``` -Cette requête est utilisée lors du démarrage du serveur. Le serveur stocke les métadonnées de la table sous forme de fichiers avec `ATTACH` requêtes, qu’il exécute simplement au lancement (à l’exception des tables système, qui sont explicitement créées sur le serveur). +Cette requête est utilisée lors du démarrage du serveur. Le serveur stocke les métadonnées de la table sous forme de fichiers avec `ATTACH` requêtes, qu'il exécute simplement au lancement (à l'exception des tables système, qui sont explicitement créées sur le serveur). ## CHECK TABLE {#check-table} @@ -31,13 +31,13 @@ Vérifie si les données de la table sont corrompues. CHECK TABLE [db.]name ``` -Le `CHECK TABLE` requête compare réelle des tailles de fichier avec les valeurs attendues qui sont stockés sur le serveur. Si le fichier tailles ne correspondent pas aux valeurs stockées, cela signifie que les données sont endommagées. Cela peut être causé, par exemple, par un plantage du système lors de l’exécution de la requête. +Le `CHECK TABLE` requête compare réelle des tailles de fichier avec les valeurs attendues qui sont stockés sur le serveur. Si le fichier tailles ne correspondent pas aux valeurs stockées, cela signifie que les données sont endommagées. Cela peut être causé, par exemple, par un plantage du système lors de l'exécution de la requête. La réponse de la requête contient `result` colonne avec une seule ligne. La ligne a une valeur de [Booléen](../../sql-reference/data-types/boolean.md) type: - 0 - les données de la table sont corrompues. -- 1 - les données maintiennent l’intégrité. +- 1 - les données maintiennent l'intégrité. Le `CHECK TABLE` query prend en charge les moteurs de table suivants: @@ -48,9 +48,9 @@ Le `CHECK TABLE` query prend en charge les moteurs de table suivants: Effectué sur les tables avec un autre moteur de table provoque une exception. -Les moteurs de la `*Log` la famille ne fournit pas de récupération automatique des données en cas d’échec. L’utilisation de la `CHECK TABLE` requête pour suivre la perte de données en temps opportun. +Les moteurs de la `*Log` la famille ne fournit pas de récupération automatique des données en cas d'échec. L'utilisation de la `CHECK TABLE` requête pour suivre la perte de données en temps opportun. -Pour `MergeTree` moteurs de la famille, le `CHECK TABLE` query affiche un État de vérification pour chaque partie de données individuelle d’une table sur le serveur local. +Pour `MergeTree` moteurs de la famille, le `CHECK TABLE` query affiche un État de vérification pour chaque partie de données individuelle d'une table sur le serveur local. **Si les données sont corrompues** @@ -59,7 +59,7 @@ Si la table est corrompue, vous pouvez copier les données non corrompues dans u 1. Créez une nouvelle table avec la même structure que la table endommagée. Pour ce faire exécutez la requête `CREATE TABLE AS `. 2. Définir le [max\_threads](../../operations/settings/settings.md#settings-max_threads) la valeur 1 pour traiter la requête suivante dans un seul thread. Pour ce faire, exécutez la requête `SET max_threads = 1`. 3. Exécuter la requête `INSERT INTO SELECT * FROM `. Cette demande copie les données non corrompues de la table endommagée vers une autre table. Seules les données avant la partie corrompue seront copiées. -4. Redémarrez l’ `clickhouse-client` pour réinitialiser l’ `max_threads` valeur. +4. Redémarrez l' `clickhouse-client` pour réinitialiser l' `max_threads` valeur. ## DESCRIBE TABLE {#misc-describe-table} @@ -71,7 +71,7 @@ Renvoie ce qui suit `String` les colonnes de type: - `name` — Column name. - `type`— Column type. -- `default_type` — Clause that is used in [expression par défaut](create.md#create-default-values) (`DEFAULT`, `MATERIALIZED` ou `ALIAS`). Column contient une chaîne vide, si l’expression par défaut n’est pas spécifiée. +- `default_type` — Clause that is used in [expression par défaut](create.md#create-default-values) (`DEFAULT`, `MATERIALIZED` ou `ALIAS`). Column contient une chaîne vide, si l'expression par défaut n'est pas spécifiée. - `default_expression` — Value specified in the `DEFAULT` clause. - `comment_expression` — Comment text. @@ -79,16 +79,16 @@ Les structures de données imbriquées sont sorties dans “expanded” format. ## DETACH {#detach} -Supprime les informations sur le ‘name’ table du serveur. Le serveur cesse de connaître l’existence de la table. +Supprime les informations sur le ‘name’ table du serveur. Le serveur cesse de connaître l'existence de la table. ``` sql DETACH TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] ``` Cela ne supprime pas les données ou les métadonnées de la table. Lors du prochain lancement du serveur, le serveur Lira les métadonnées et découvrira à nouveau la table. -De même, un “detached” tableau peut être re-attaché en utilisant le `ATTACH` requête (à l’exception des tables système, qui n’ont pas de stocker les métadonnées pour eux). +De même, un “detached” tableau peut être re-attaché en utilisant le `ATTACH` requête (à l'exception des tables système, qui n'ont pas de stocker les métadonnées pour eux). -Il n’y a pas de `DETACH DATABASE` requête. +Il n'y a pas de `DETACH DATABASE` requête. ## DROP {#drop} @@ -98,30 +98,88 @@ Cette requête a deux types: `DROP DATABASE` et `DROP TABLE`. DROP DATABASE [IF EXISTS] db [ON CLUSTER cluster] ``` -Supprime toutes les tables à l’intérieur de la ‘db’ la base de données, puis supprime le ‘db’ la base de données elle-même. -Si `IF EXISTS` est spécifié, il ne renvoie pas d’erreur si la base de données n’existe pas. +Supprime toutes les tables à l'intérieur de la ‘db’ la base de données, puis supprime le ‘db’ la base de données elle-même. +Si `IF EXISTS` est spécifié, il ne renvoie pas d'erreur si la base de données n'existe pas. ``` sql DROP [TEMPORARY] TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] ``` Supprime la table. -Si `IF EXISTS` est spécifié, il ne renvoie pas d’erreur si la table n’existe pas ou si la base de données n’existe pas. +Si `IF EXISTS` est spécifié, il ne renvoie pas d'erreur si la table n'existe pas ou si la base de données n'existe pas. DROP DICTIONARY [IF EXISTS] [db.]name Delets le dictionnaire. -Si `IF EXISTS` est spécifié, il ne renvoie pas d’erreur si la table n’existe pas ou si la base de données n’existe pas. +Si `IF EXISTS` est spécifié, il ne renvoie pas d'erreur si la table n'existe pas ou si la base de données n'existe pas. -## EXISTS {#exists} +## DROP USER {#drop-user-statement} + +Supprime un utilisateur. + +### Syntaxe {#drop-user-syntax} + +``` sql +DROP USER [IF EXISTS] name [,...] [ON CLUSTER cluster_name] +``` + +## DROP ROLE {#drop-role-statement} + +Supprime un rôle. + +Le rôle supprimé est révoqué de toutes les entités où il a été accordé. + +### Syntaxe {#drop-role-syntax} + +``` sql +DROP ROLE [IF EXISTS] name [,...] [ON CLUSTER cluster_name] +``` + +## DROP ROW POLICY {#drop-row-policy-statement} + +Supprime une stratégie de ligne. + +La stratégie de ligne supprimée est révoquée de toutes les entités sur lesquelles elle a été affectée. + +### Syntaxe {#drop-row-policy-syntax} + +``` sql +DROP [ROW] POLICY [IF EXISTS] name [,...] ON [database.]table [,...] [ON CLUSTER cluster_name] +``` + +## DROP QUOTA {#drop-quota-statement} + +Supprime un quota. + +Le quota supprimé est révoqué de toutes les entités où il a été affecté. + +### Syntaxe {#drop-quota-syntax} + +``` sql +DROP QUOTA [IF EXISTS] name [,...] [ON CLUSTER cluster_name] +``` + +## DROP SETTINGS PROFILE {#drop-settings-profile-statement} + +Supprime un quota. + +Le quota supprimé est révoqué de toutes les entités où il a été affecté. + +### Syntaxe {#drop-settings-profile-syntax} + +``` sql +DROP [SETTINGS] PROFILE [IF EXISTS] name [,...] [ON CLUSTER cluster_name] +``` + +## EXISTS {#exists-statement} ``` sql EXISTS [TEMPORARY] [TABLE|DICTIONARY] [db.]name [INTO OUTFILE filename] [FORMAT format] ``` -Renvoie un seul `UInt8`- type colonne, qui contient la valeur unique `0` si la table ou base de données n’existe pas, ou `1` si la table existe dans la base de données spécifiée. +Renvoie un seul `UInt8`- type colonne, qui contient la valeur unique `0` si la table ou base de données n'existe pas, ou `1` si la table existe dans la base de données spécifiée. -## KILL QUERY {#kill-query} +## KILL QUERY {#kill-query-statement} ``` sql KILL QUERY [ON CLUSTER cluster] @@ -130,7 +188,7 @@ KILL QUERY [ON CLUSTER cluster] [FORMAT format] ``` -Tente de mettre fin de force aux requêtes en cours d’exécution. +Tente de mettre fin de force aux requêtes en cours d'exécution. Les requêtes à terminer sont sélectionnées dans le système.processus en utilisant les critères définis dans le `WHERE` la clause de la `KILL` requête. Exemple: @@ -145,16 +203,16 @@ KILL QUERY WHERE user='username' SYNC Les utilisateurs en lecture seule peuvent uniquement arrêter leurs propres requêtes. -Par défaut, la version asynchrone des requêtes est utilisé (`ASYNC`), qui n’attend pas la confirmation que les requêtes se sont arrêtées. +Par défaut, la version asynchrone des requêtes est utilisé (`ASYNC`), qui n'attend pas la confirmation que les requêtes se sont arrêtées. -La version synchrone (`SYNC`) attend que toutes les requêtes d’arrêter et affiche des informations sur chaque processus s’arrête. -La réponse contient l’ `kill_status` la colonne, qui peut prendre les valeurs suivantes: +La version synchrone (`SYNC`) attend que toutes les requêtes d'arrêter et affiche des informations sur chaque processus s'arrête. +La réponse contient l' `kill_status` la colonne, qui peut prendre les valeurs suivantes: 1. ‘finished’ – The query was terminated successfully. 2. ‘waiting’ – Waiting for the query to end after sending it a signal to terminate. -3. The other values ​​explain why the query can’t be stopped. +3. The other values ​​explain why the query can't be stopped. -Une requête de test (`TEST`) vérifie uniquement les droits de l’utilisateur et affiche une liste de requêtes à arrêter. +Une requête de test (`TEST`) vérifie uniquement les droits de l'utilisateur et affiche une liste de requêtes à arrêter. ## KILL MUTATION {#kill-mutation} @@ -165,9 +223,9 @@ KILL MUTATION [ON CLUSTER cluster] [FORMAT format] ``` -Essaie d’annuler et supprimer [mutation](alter.md#alter-mutations) actuellement en cours d’exécution. Les Mutations à annuler sont sélectionnées parmi [`system.mutations`](../../operations/system-tables.md#system_tables-mutations) tableau à l’aide du filtre spécifié par le `WHERE` la clause de la `KILL` requête. +Essaie d'annuler et supprimer [mutation](alter.md#alter-mutations) actuellement en cours d'exécution. Les Mutations à annuler sont sélectionnées parmi [`system.mutations`](../../operations/system-tables.md#system_tables-mutations) tableau à l'aide du filtre spécifié par le `WHERE` la clause de la `KILL` requête. -Une requête de test (`TEST`) vérifie uniquement les droits de l’utilisateur et affiche une liste de requêtes à arrêter. +Une requête de test (`TEST`) vérifie uniquement les droits de l'utilisateur et affiche une liste de requêtes à arrêter. Exemple: @@ -189,16 +247,16 @@ Les modifications déjà apportées par la mutation ne sont pas annulées. OPTIMIZE TABLE [db.]name [ON CLUSTER cluster] [PARTITION partition | PARTITION ID 'partition_id'] [FINAL] [DEDUPLICATE] ``` -Cette requête tente d’initialiser une fusion non programmée de parties de données pour les tables avec un moteur de [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) famille. +Cette requête tente d'initialiser une fusion non programmée de parties de données pour les tables avec un moteur de [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) famille. Le `OPTMIZE` la requête est également prise en charge pour [MaterializedView](../../engines/table-engines/special/materializedview.md) et la [Tampon](../../engines/table-engines/special/buffer.md) moteur. Les autres moteurs de table ne sont pas pris en charge. -Lorsque `OPTIMIZE` est utilisé avec le [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/replication.md) famille de moteurs de table, ClickHouse crée une tâche pour la fusion et attend l’exécution sur tous les nœuds (si le `replication_alter_partitions_sync` paramètre est activé). +Lorsque `OPTIMIZE` est utilisé avec le [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/replication.md) famille de moteurs de table, ClickHouse crée une tâche pour la fusion et attend l'exécution sur tous les nœuds (si le `replication_alter_partitions_sync` paramètre est activé). -- Si `OPTIMIZE` n’effectue pas de fusion pour une raison quelconque, il ne notifie pas le client. Pour activer les notifications, utilisez [optimize\_throw\_if\_noop](../../operations/settings/settings.md#setting-optimize_throw_if_noop) paramètre. -- Si vous spécifiez un `PARTITION`, seule la partition spécifiée est optimisé. [Comment définir l’expression de la partition](alter.md#alter-how-to-specify-part-expr). -- Si vous spécifiez `FINAL`, l’optimisation est effectuée, même lorsque toutes les données sont déjà dans une partie. -- Si vous spécifiez `DEDUPLICATE`, alors des lignes complètement identiques seront dédupliquées (toutes les colonnes sont comparées), cela n’a de sens que pour le moteur MergeTree. +- Si `OPTIMIZE` n'effectue pas de fusion pour une raison quelconque, il ne notifie pas le client. Pour activer les notifications, utilisez [optimize\_throw\_if\_noop](../../operations/settings/settings.md#setting-optimize_throw_if_noop) paramètre. +- Si vous spécifiez un `PARTITION`, seule la partition spécifiée est optimisé. [Comment définir l'expression de la partition](alter.md#alter-how-to-specify-part-expr). +- Si vous spécifiez `FINAL`, l'optimisation est effectuée, même lorsque toutes les données sont déjà dans une partie. +- Si vous spécifiez `DEDUPLICATE`, alors des lignes complètement identiques seront dédupliquées (toutes les colonnes sont comparées), cela n'a de sens que pour le moteur MergeTree. !!! warning "Avertissement" `OPTIMIZE` ne peut pas réparer le “Too many parts” erreur. @@ -219,7 +277,7 @@ Toutes les tables sont renommées sous verrouillage global. Renommer des tables SET param = value ``` -Assigner `value` à l’ `param` [paramètre](../../operations/settings/index.md) pour la session en cours. Vous ne pouvez pas modifier [les paramètres du serveur](../../operations/server-configuration-parameters/index.md) de cette façon. +Assigner `value` à l' `param` [paramètre](../../operations/settings/index.md) pour la session en cours. Vous ne pouvez pas modifier [les paramètres du serveur](../../operations/server-configuration-parameters/index.md) de cette façon. Vous pouvez également définir toutes les valeurs de certains paramètres de profil dans une seule requête. @@ -227,17 +285,65 @@ Vous pouvez également définir toutes les valeurs de certains paramètres de pr SET profile = 'profile-name-from-the-settings-file' ``` -Pour plus d’informations, voir [Paramètre](../../operations/settings/settings.md). +Pour plus d'informations, voir [Paramètre](../../operations/settings/settings.md). -## TRUNCATE {#truncate} +## SET ROLE {#set-role-statement} + +Active les rôles pour l'utilisateur actuel. + +### Syntaxe {#set-role-syntax} + +``` sql +SET ROLE {DEFAULT | NONE | role [,...] | ALL | ALL EXCEPT role [,...]} +``` + +## SET DEFAULT ROLE {#set-default-role-statement} + +Définit les rôles par défaut à un utilisateur. + +Les rôles par défaut sont automatiquement activés lors de la connexion de l'utilisateur. Vous pouvez définir par défaut uniquement les rôles précédemment accordés. Si le rôle n'est pas accordé à un utilisateur, ClickHouse lève une exception. + +### Syntaxe {#set-default-role-syntax} + +``` sql +SET DEFAULT ROLE {NONE | role [,...] | ALL | ALL EXCEPT role [,...]} TO {user|CURRENT_USER} [,...] +``` + +### Exemple {#set-default-role-examples} + +Définir plusieurs rôles par défaut à un utilisateur: + +``` sql +SET DEFAULT ROLE role1, role2, ... TO user +``` + +Définissez tous les rôles accordés par défaut sur un utilisateur: + +``` sql +SET DEFAULT ROLE ALL TO user +``` + +Purger les rôles par défaut d'un utilisateur: + +``` sql +SET DEFAULT ROLE NONE TO user +``` + +Définissez tous les rôles accordés par défaut à l'exception de certains d'entre eux: + +``` sql +SET DEFAULT ROLE ALL EXCEPT role1, role2 TO user +``` + +## TRUNCATE {#truncate-statement} ``` sql TRUNCATE TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] ``` -Supprime toutes les données d’une table. Lorsque la clause `IF EXISTS` est omis, la requête renvoie une erreur si la table n’existe pas. +Supprime toutes les données d'une table. Lorsque la clause `IF EXISTS` est omis, la requête renvoie une erreur si la table n'existe pas. -Le `TRUNCATE` la requête n’est pas prise en charge pour [Vue](../../engines/table-engines/special/view.md), [Fichier](../../engines/table-engines/special/file.md), [URL](../../engines/table-engines/special/url.md) et [NULL](../../engines/table-engines/special/null.md) table des moteurs. +Le `TRUNCATE` la requête n'est pas prise en charge pour [Vue](../../engines/table-engines/special/view.md), [Fichier](../../engines/table-engines/special/file.md), [URL](../../engines/table-engines/special/url.md) et [NULL](../../engines/table-engines/special/null.md) table des moteurs. ## USE {#use} @@ -246,7 +352,7 @@ USE db ``` Vous permet de définir la base de données actuelle pour la session. -La base de données actuelle est utilisée pour rechercher des tables si la base de données n’est pas explicitement définie dans la requête avec un point avant le nom de la table. -Cette requête ne peut pas être faite lors de l’utilisation du protocole HTTP, car il n’y a pas de concept de session. +La base de données actuelle est utilisée pour rechercher des tables si la base de données n'est pas explicitement définie dans la requête avec un point avant le nom de la table. +Cette requête ne peut pas être faite lors de l'utilisation du protocole HTTP, car il n'y a pas de concept de session. [Article Original](https://clickhouse.tech/docs/en/query_language/misc/) diff --git a/docs/fr/sql-reference/statements/revoke.md b/docs/fr/sql-reference/statements/revoke.md new file mode 100644 index 00000000000..6137cc30f8c --- /dev/null +++ b/docs/fr/sql-reference/statements/revoke.md @@ -0,0 +1,50 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_priority: 40 +toc_title: REVOKE +--- + +# REVOKE {#revoke} + +Révoque les privilèges des utilisateurs ou rôles. + +## Syntaxe {#revoke-syntax} + +**Révocation des privilèges des utilisateurs** + +``` sql +REVOKE [ON CLUSTER cluster_name] privilege[(column_name [,...])] [,...] ON {db.table|db.*|*.*|table|*} FROM {user | CURRENT_USER} [,...] | ALL | ALL EXCEPT {user | CURRENT_USER} [,...] +``` + +**Révocation des rôles des utilisateurs** + +``` sql +REVOKE [ON CLUSTER cluster_name] [ADMIN OPTION FOR] role [,...] FROM {user | role | CURRENT_USER} [,...] | ALL | ALL EXCEPT {user_name | role_name | CURRENT_USER} [,...] +``` + +## Description {#revoke-description} + +Pour révoquer certains privilèges, vous pouvez utiliser un privilège de portée plus large que vous envisagez de révoquer. Par exemple, si un utilisateur a la `SELECT (x,y)` privilège, administrateur peut effectuer `REVOKE SELECT(x,y) ...`, ou `REVOKE SELECT * ...` ou même `REVOKE ALL PRIVILEGES ...` requête de révoquer ce privilège. + +### Révocations Partielles {#partial-revokes-dscr} + +Vous pouvez révoquer une partie d'un privilège. Par exemple, si un utilisateur a la `SELECT *.*` Privilège vous pouvez révoquer un privilège pour lire les données d'une table ou d'une base de données. + +## Exemple {#revoke-example} + +Subvention de l' `john` compte utilisateur avec le privilège de sélectionner parmi toutes les bases de données `accounts` un: + +``` sql +GRANT SELECT ON *.* TO john; +REVOKE SELECT ON accounts.* FROM john; +``` + +Subvention de l' `mira` compte utilisateur avec le privilège de sélectionner parmi toutes les colonnes `accounts.staff` tableau à l'exception de la `wage` un. + +``` sql +GRANT SELECT ON accounts.staff TO mira; +REVOKE SELECT(wage) ON accounts.staff FROM mira; +``` + +{## [Article Original](https://clickhouse.tech/docs/en/operations/settings/settings/) ##} diff --git a/docs/fr/sql-reference/statements/select.md b/docs/fr/sql-reference/statements/select.md deleted file mode 100644 index 7cb5587f014..00000000000 --- a/docs/fr/sql-reference/statements/select.md +++ /dev/null @@ -1,1379 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 0f7ef7704d018700049223525bad4a63911b6e70 -toc_priority: 33 -toc_title: SELECT ---- - -# Sélectionnez la syntaxe des requêtes {#select-queries-syntax} - -`SELECT` effectue la récupération des données. - -``` sql -[WITH expr_list|(subquery)] -SELECT [DISTINCT] expr_list -[FROM [db.]table | (subquery) | table_function] [FINAL] -[SAMPLE sample_coeff] -[ARRAY JOIN ...] -[GLOBAL] [ANY|ALL] [INNER|LEFT|RIGHT|FULL|CROSS] [OUTER] JOIN (subquery)|table USING columns_list -[PREWHERE expr] -[WHERE expr] -[GROUP BY expr_list] [WITH TOTALS] -[HAVING expr] -[ORDER BY expr_list] -[LIMIT [offset_value, ]n BY columns] -[LIMIT [n, ]m] -[UNION ALL ...] -[INTO OUTFILE filename] -[FORMAT format] -``` - -Toutes les clauses sont facultatives, à l'exception de la liste d'expressions requise immédiatement après SELECT. -Les clauses ci-dessous sont décrites dans presque le même ordre que dans l'exécution de la requête convoyeur. - -Si la requête omet le `DISTINCT`, `GROUP BY` et `ORDER BY` les clauses et les `IN` et `JOIN` sous-requêtes, la requête sera complètement traitée en flux, en utilisant O (1) quantité de RAM. -Sinon, la requête peut consommer beaucoup de RAM si les restrictions appropriées ne sont pas spécifiées: `max_memory_usage`, `max_rows_to_group_by`, `max_rows_to_sort`, `max_rows_in_distinct`, `max_bytes_in_distinct`, `max_rows_in_set`, `max_bytes_in_set`, `max_rows_in_join`, `max_bytes_in_join`, `max_bytes_before_external_sort`, `max_bytes_before_external_group_by`. Pour plus d'informations, consultez la section “Settings”. Il est possible d'utiliser le tri externe (sauvegarde des tables temporaires sur un disque) et l'agrégation externe. `The system does not have "merge join"`. - -### AVEC la Clause {#with-clause} - -Cette section prend en charge les Expressions de Table courantes ([CTE](https://en.wikipedia.org/wiki/Hierarchical_and_recursive_queries_in_SQL)), avec quelques limitations: -1. Les requêtes récursives ne sont pas prises en charge -2. Lorsque la sous-requête est utilisée à l'intérieur avec section, son résultat doit être scalaire avec exactement une ligne -3. Les résultats d'Expression ne sont pas disponibles dans les sous requêtes -Les résultats des expressions de clause WITH peuvent être utilisés dans la clause SELECT. - -Exemple 1: Utilisation d'une expression constante comme “variable” - -``` sql -WITH '2019-08-01 15:23:00' as ts_upper_bound -SELECT * -FROM hits -WHERE - EventDate = toDate(ts_upper_bound) AND - EventTime <= ts_upper_bound -``` - -Exemple 2: Expulsion de la somme(octets) résultat de l'expression de clause SELECT de la liste de colonnes - -``` sql -WITH sum(bytes) as s -SELECT - formatReadableSize(s), - table -FROM system.parts -GROUP BY table -ORDER BY s -``` - -Exemple 3: Utilisation des résultats de la sous-requête scalaire - -``` sql -/* this example would return TOP 10 of most huge tables */ -WITH - ( - SELECT sum(bytes) - FROM system.parts - WHERE active - ) AS total_disk_usage -SELECT - (sum(bytes) / total_disk_usage) * 100 AS table_disk_usage, - table -FROM system.parts -GROUP BY table -ORDER BY table_disk_usage DESC -LIMIT 10 -``` - -Exemple 4: réutilisation de l'expression dans la sous-requête -Comme solution de contournement pour la limitation actuelle de l'utilisation de l'expression dans les sous-requêtes, Vous pouvez la dupliquer. - -``` sql -WITH ['hello'] AS hello -SELECT - hello, - * -FROM -( - WITH ['hello'] AS hello - SELECT hello -) -``` - -``` text -┌─hello─────┬─hello─────┐ -│ ['hello'] │ ['hello'] │ -└───────────┴───────────┘ -``` - -### De la Clause {#select-from} - -Si la clause FROM est omise, les données seront lues à partir `system.one` table. -Le `system.one` table contient exactement une ligne (cette table remplit le même but que la table double trouvée dans d'autres SGBD). - -Le `FROM` clause spécifie la source à partir de laquelle lire les données: - -- Table -- Sous-requête -- [Fonction de Table](../table-functions/index.md#table-functions) - -`ARRAY JOIN` et le régulier `JOIN` peuvent également être inclus (voir ci-dessous). - -Au lieu d'une table, l' `SELECT` sous-requête peut être spécifiée entre parenthèses. -Contrairement à SQL standard, un synonyme n'a pas besoin d'être spécifié après une sous-requête. - -Pour exécuter une requête, toutes les colonnes mentionnées dans la requête sont extraites de la table appropriée. Toutes les colonnes non nécessaires pour la requête externe sont rejetées des sous-requêtes. -Si une requête ne répertorie aucune colonne (par exemple, `SELECT count() FROM t`), une colonne est extraite de la table de toute façon (la plus petite est préférée), afin de calculer le nombre de lignes. - -#### Modificateur FINAL {#select-from-final} - -Applicable lors de la sélection de données à partir de tables [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md)-famille de moteurs autres que `GraphiteMergeTree`. Lorsque `FINAL` est spécifié, ClickHouse fusionne complètement les données avant de renvoyer le résultat et effectue ainsi toutes les transformations de données qui se produisent lors des fusions pour le moteur de table donné. - -Également pris en charge pour: -- [Répliqué](../../engines/table-engines/mergetree-family/replication.md) les versions de `MergeTree` moteur. -- [Vue](../../engines/table-engines/special/view.md), [Tampon](../../engines/table-engines/special/buffer.md), [Distribué](../../engines/table-engines/special/distributed.md), et [MaterializedView](../../engines/table-engines/special/materializedview.md) moteurs qui fonctionnent sur d'autres moteurs, à condition qu'ils aient été créés sur `MergeTree`-tables de moteur. - -Requêtes qui utilisent `FINAL` sont exécutés pas aussi vite que les requêtes similaires qui ne le font pas, car: - -- La requête est exécutée dans un seul thread et les données sont fusionnées lors de l'exécution de la requête. -- Les requêtes avec `FINAL` lire les colonnes de clé primaire en plus des colonnes spécifiées dans la requête. - -Dans la plupart des cas, évitez d'utiliser `FINAL`. - -### Exemple de Clause {#select-sample-clause} - -Le `SAMPLE` la clause permet un traitement de requête approximatif. - -Lorsque l'échantillonnage de données est activé, la requête n'est pas effectuée sur toutes les données, mais uniquement sur une certaine fraction de données (échantillon). Par exemple, si vous avez besoin de calculer des statistiques pour toutes les visites, il suffit d'exécuter la requête sur le 1/10 de la fraction de toutes les visites, puis multiplier le résultat par 10. - -Le traitement approximatif des requêtes peut être utile dans les cas suivants: - -- Lorsque vous avez des exigences de synchronisation strictes (comme \<100ms), mais que vous ne pouvez pas justifier le coût des ressources matérielles supplémentaires pour y répondre. -- Lorsque vos données brutes ne sont pas précises, l'approximation ne dégrade pas sensiblement la qualité. -- Les exigences commerciales ciblent des résultats approximatifs (pour la rentabilité, ou afin de commercialiser des résultats exacts aux utilisateurs premium). - -!!! note "Note" - Vous ne pouvez utiliser l'échantillonnage qu'avec les tables [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) famille, et seulement si l'expression d'échantillonnage a été spécifiée lors de la création de la table (voir [Moteur MergeTree](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table)). - -Les caractéristiques de l'échantillonnage des données sont énumérées ci-dessous: - -- L'échantillonnage de données est un mécanisme déterministe. Le résultat de la même `SELECT .. SAMPLE` la requête est toujours le même. -- L'échantillonnage fonctionne de manière cohérente pour différentes tables. Pour les tables avec une seule clé d'échantillonnage, un échantillon avec le même coefficient sélectionne toujours le même sous-ensemble de données possibles. Par exemple, un exemple d'ID utilisateur prend des lignes avec le même sous-ensemble de tous les ID utilisateur possibles de différentes tables. Cela signifie que vous pouvez utiliser l'exemple dans les sous-requêtes dans la [IN](#select-in-operators) clause. En outre, vous pouvez joindre des échantillons en utilisant le [JOIN](#select-join) clause. -- L'échantillonnage permet de lire moins de données à partir d'un disque. Notez que vous devez spécifier l'échantillonnage clé correctement. Pour plus d'informations, voir [Création d'une Table MergeTree](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table). - -Pour l' `SAMPLE` clause la syntaxe suivante est prise en charge: - -| SAMPLE Clause Syntax | Description | -|----------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `SAMPLE k` | Ici `k` est le nombre de 0 à 1.
La requête est exécutée sur `k` fraction des données. Exemple, `SAMPLE 0.1` exécute la requête sur 10% des données. [Lire plus](#select-sample-k) | -| `SAMPLE n` | Ici `n` est un entier suffisamment grand.
La requête est exécutée sur un échantillon d'au moins `n` lignes (mais pas significativement plus que cela). Exemple, `SAMPLE 10000000` exécute la requête sur un minimum de 10 000 000 lignes. [Lire plus](#select-sample-n) | -| `SAMPLE k OFFSET m` | Ici `k` et `m` sont les nombres de 0 à 1.
La requête est exécutée sur un échantillon de `k` fraction des données. Les données utilisées pour l'échantillon est compensée par `m` fraction. [Lire plus](#select-sample-offset) | - -#### SAMPLE K {#select-sample-k} - -Ici `k` est le nombre de 0 à 1 (les notations fractionnaires et décimales sont prises en charge). Exemple, `SAMPLE 1/2` ou `SAMPLE 0.5`. - -Dans un `SAMPLE k` clause, l'échantillon est prélevé à partir de la `k` fraction des données. L'exemple est illustré ci-dessous: - -``` sql -SELECT - Title, - count() * 10 AS PageViews -FROM hits_distributed -SAMPLE 0.1 -WHERE - CounterID = 34 -GROUP BY Title -ORDER BY PageViews DESC LIMIT 1000 -``` - -Dans cet exemple, la requête est exécutée sur un échantillon de 0,1 (10%) de données. Les valeurs des fonctions d'agrégat ne sont pas corrigées automatiquement, donc pour obtenir un résultat approximatif, la valeur `count()` est multiplié manuellement par 10. - -#### SAMPLE N {#select-sample-n} - -Ici `n` est un entier suffisamment grand. Exemple, `SAMPLE 10000000`. - -Dans ce cas, la requête est exécutée sur un échantillon d'au moins `n` lignes (mais pas significativement plus que cela). Exemple, `SAMPLE 10000000` exécute la requête sur un minimum de 10 000 000 lignes. - -Puisque l'unité minimale pour la lecture des données est un granule (sa taille est définie par le `index_granularity` de réglage), il est logique de définir un échantillon beaucoup plus grand que la taille du granule. - -Lors de l'utilisation de la `SAMPLE n` clause, vous ne savez pas quel pourcentage relatif de données a été traité. Donc, vous ne connaissez pas le coefficient par lequel les fonctions agrégées doivent être multipliées. L'utilisation de la `_sample_factor` colonne virtuelle pour obtenir le résultat approximatif. - -Le `_sample_factor` colonne contient des coefficients relatifs qui sont calculés dynamiquement. Cette colonne est créée automatiquement lorsque vous [créer](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table) une table avec la clé d'échantillonnage spécifiée. Les exemples d'utilisation de la `_sample_factor` colonne sont indiqués ci-dessous. - -Considérons la table `visits` qui contient des statistiques sur les visites de site. Le premier exemple montre comment calculer le nombre de pages vues: - -``` sql -SELECT sum(PageViews * _sample_factor) -FROM visits -SAMPLE 10000000 -``` - -L'exemple suivant montre comment calculer le nombre total de visites: - -``` sql -SELECT sum(_sample_factor) -FROM visits -SAMPLE 10000000 -``` - -L'exemple ci-dessous montre comment calculer la durée moyenne de la session. Notez que vous n'avez pas besoin d'utiliser le coefficient relatif pour calculer les valeurs moyennes. - -``` sql -SELECT avg(Duration) -FROM visits -SAMPLE 10000000 -``` - -#### SAMPLE K OFFSET M {#select-sample-offset} - -Ici `k` et `m` sont des nombres de 0 à 1. Des exemples sont présentés ci-dessous. - -**Exemple 1** - -``` sql -SAMPLE 1/10 -``` - -Dans cet exemple, l'échantillon représente 1 / 10e de toutes les données: - -`[++------------]` - -**Exemple 2** - -``` sql -SAMPLE 1/10 OFFSET 1/2 -``` - -Ici, un échantillon de 10% est prélevé à partir de la seconde moitié des données. - -`[------++------]` - -### Clause de jointure de tableau {#select-array-join-clause} - -Permet l'exécution de `JOIN` avec un tableau ou une structure de données imbriquée. L'intention est similaire à la [arrayJoin](../functions/array-join.md#functions_arrayjoin) la fonction, mais sa fonctionnalité est plus large. - -``` sql -SELECT -FROM -[LEFT] ARRAY JOIN -[WHERE|PREWHERE ] -... -``` - -Vous pouvez spécifier qu'un seul `ARRAY JOIN` la clause dans une requête. - -L'ordre d'exécution de la requête est optimisé lors de l'exécution `ARRAY JOIN`. Bien `ARRAY JOIN` doit toujours être spécifié avant l' `WHERE/PREWHERE` clause, il peut être effectué soit avant `WHERE/PREWHERE` (si le résultat est nécessaire dans cette clause), ou après l'avoir terminé (pour réduire le volume de calculs). L'ordre de traitement est contrôlée par l'optimiseur de requête. - -Types pris en charge de `ARRAY JOIN` sont énumérés ci-dessous: - -- `ARRAY JOIN` - Dans ce cas, des tableaux vides ne sont pas inclus dans le résultat de `JOIN`. -- `LEFT ARRAY JOIN` - Le résultat de `JOIN` contient des lignes avec des tableaux vides. La valeur d'un tableau vide est définie sur la valeur par défaut pour le type d'élément de tableau (généralement 0, chaîne vide ou NULL). - -Les exemples ci-dessous illustrent l'utilisation de la `ARRAY JOIN` et `LEFT ARRAY JOIN` clause. Créons une table avec un [Tableau](../../sql-reference/data-types/array.md) tapez colonne et insérez des valeurs dedans: - -``` sql -CREATE TABLE arrays_test -( - s String, - arr Array(UInt8) -) ENGINE = Memory; - -INSERT INTO arrays_test -VALUES ('Hello', [1,2]), ('World', [3,4,5]), ('Goodbye', []); -``` - -``` text -┌─s───────────┬─arr─────┐ -│ Hello │ [1,2] │ -│ World │ [3,4,5] │ -│ Goodbye │ [] │ -└─────────────┴─────────┘ -``` - -L'exemple ci-dessous utilise la `ARRAY JOIN` clause: - -``` sql -SELECT s, arr -FROM arrays_test -ARRAY JOIN arr; -``` - -``` text -┌─s─────┬─arr─┐ -│ Hello │ 1 │ -│ Hello │ 2 │ -│ World │ 3 │ -│ World │ 4 │ -│ World │ 5 │ -└───────┴─────┘ -``` - -L'exemple suivant utilise l' `LEFT ARRAY JOIN` clause: - -``` sql -SELECT s, arr -FROM arrays_test -LEFT ARRAY JOIN arr; -``` - -``` text -┌─s───────────┬─arr─┐ -│ Hello │ 1 │ -│ Hello │ 2 │ -│ World │ 3 │ -│ World │ 4 │ -│ World │ 5 │ -│ Goodbye │ 0 │ -└─────────────┴─────┘ -``` - -#### À L'Aide D'Alias {#using-aliases} - -Un alias peut être spécifié pour un tableau `ARRAY JOIN` clause. Dans ce cas, un élément de tableau peut être consulté par ce pseudonyme, mais le tableau lui-même est accessible par le nom d'origine. Exemple: - -``` sql -SELECT s, arr, a -FROM arrays_test -ARRAY JOIN arr AS a; -``` - -``` text -┌─s─────┬─arr─────┬─a─┐ -│ Hello │ [1,2] │ 1 │ -│ Hello │ [1,2] │ 2 │ -│ World │ [3,4,5] │ 3 │ -│ World │ [3,4,5] │ 4 │ -│ World │ [3,4,5] │ 5 │ -└───────┴─────────┴───┘ -``` - -En utilisant des alias, vous pouvez effectuer `ARRAY JOIN` avec un groupe externe. Exemple: - -``` sql -SELECT s, arr_external -FROM arrays_test -ARRAY JOIN [1, 2, 3] AS arr_external; -``` - -``` text -┌─s───────────┬─arr_external─┐ -│ Hello │ 1 │ -│ Hello │ 2 │ -│ Hello │ 3 │ -│ World │ 1 │ -│ World │ 2 │ -│ World │ 3 │ -│ Goodbye │ 1 │ -│ Goodbye │ 2 │ -│ Goodbye │ 3 │ -└─────────────┴──────────────┘ -``` - -Plusieurs tableaux peuvent être séparés par des virgules `ARRAY JOIN` clause. Dans ce cas, `JOIN` est effectuée avec eux simultanément (la somme directe, pas le produit cartésien). Notez que tous les tableaux doivent avoir la même taille. Exemple: - -``` sql -SELECT s, arr, a, num, mapped -FROM arrays_test -ARRAY JOIN arr AS a, arrayEnumerate(arr) AS num, arrayMap(x -> x + 1, arr) AS mapped; -``` - -``` text -┌─s─────┬─arr─────┬─a─┬─num─┬─mapped─┐ -│ Hello │ [1,2] │ 1 │ 1 │ 2 │ -│ Hello │ [1,2] │ 2 │ 2 │ 3 │ -│ World │ [3,4,5] │ 3 │ 1 │ 4 │ -│ World │ [3,4,5] │ 4 │ 2 │ 5 │ -│ World │ [3,4,5] │ 5 │ 3 │ 6 │ -└───────┴─────────┴───┴─────┴────────┘ -``` - -L'exemple ci-dessous utilise la [arrayEnumerate](../../sql-reference/functions/array-functions.md#array_functions-arrayenumerate) fonction: - -``` sql -SELECT s, arr, a, num, arrayEnumerate(arr) -FROM arrays_test -ARRAY JOIN arr AS a, arrayEnumerate(arr) AS num; -``` - -``` text -┌─s─────┬─arr─────┬─a─┬─num─┬─arrayEnumerate(arr)─┐ -│ Hello │ [1,2] │ 1 │ 1 │ [1,2] │ -│ Hello │ [1,2] │ 2 │ 2 │ [1,2] │ -│ World │ [3,4,5] │ 3 │ 1 │ [1,2,3] │ -│ World │ [3,4,5] │ 4 │ 2 │ [1,2,3] │ -│ World │ [3,4,5] │ 5 │ 3 │ [1,2,3] │ -└───────┴─────────┴───┴─────┴─────────────────────┘ -``` - -#### Jointure de tableau avec la Structure de données imbriquée {#array-join-with-nested-data-structure} - -`ARRAY`Rejoindre " fonctionne également avec [structures de données imbriquées](../../sql-reference/data-types/nested-data-structures/nested.md). Exemple: - -``` sql -CREATE TABLE nested_test -( - s String, - nest Nested( - x UInt8, - y UInt32) -) ENGINE = Memory; - -INSERT INTO nested_test -VALUES ('Hello', [1,2], [10,20]), ('World', [3,4,5], [30,40,50]), ('Goodbye', [], []); -``` - -``` text -┌─s───────┬─nest.x──┬─nest.y─────┐ -│ Hello │ [1,2] │ [10,20] │ -│ World │ [3,4,5] │ [30,40,50] │ -│ Goodbye │ [] │ [] │ -└─────────┴─────────┴────────────┘ -``` - -``` sql -SELECT s, `nest.x`, `nest.y` -FROM nested_test -ARRAY JOIN nest; -``` - -``` text -┌─s─────┬─nest.x─┬─nest.y─┐ -│ Hello │ 1 │ 10 │ -│ Hello │ 2 │ 20 │ -│ World │ 3 │ 30 │ -│ World │ 4 │ 40 │ -│ World │ 5 │ 50 │ -└───────┴────────┴────────┘ -``` - -Lorsque vous spécifiez des noms de structures de données imbriquées dans `ARRAY JOIN` le sens est le même que `ARRAY JOIN` avec tous les éléments du tableau qui la compose. Des exemples sont énumérés ci-dessous: - -``` sql -SELECT s, `nest.x`, `nest.y` -FROM nested_test -ARRAY JOIN `nest.x`, `nest.y`; -``` - -``` text -┌─s─────┬─nest.x─┬─nest.y─┐ -│ Hello │ 1 │ 10 │ -│ Hello │ 2 │ 20 │ -│ World │ 3 │ 30 │ -│ World │ 4 │ 40 │ -│ World │ 5 │ 50 │ -└───────┴────────┴────────┘ -``` - -Cette variation a également du sens: - -``` sql -SELECT s, `nest.x`, `nest.y` -FROM nested_test -ARRAY JOIN `nest.x`; -``` - -``` text -┌─s─────┬─nest.x─┬─nest.y─────┐ -│ Hello │ 1 │ [10,20] │ -│ Hello │ 2 │ [10,20] │ -│ World │ 3 │ [30,40,50] │ -│ World │ 4 │ [30,40,50] │ -│ World │ 5 │ [30,40,50] │ -└───────┴────────┴────────────┘ -``` - -Un alias peut être utilisé pour une structure de données imbriquée, afin de sélectionner `JOIN` le résultat ou le tableau source. Exemple: - -``` sql -SELECT s, `n.x`, `n.y`, `nest.x`, `nest.y` -FROM nested_test -ARRAY JOIN nest AS n; -``` - -``` text -┌─s─────┬─n.x─┬─n.y─┬─nest.x──┬─nest.y─────┐ -│ Hello │ 1 │ 10 │ [1,2] │ [10,20] │ -│ Hello │ 2 │ 20 │ [1,2] │ [10,20] │ -│ World │ 3 │ 30 │ [3,4,5] │ [30,40,50] │ -│ World │ 4 │ 40 │ [3,4,5] │ [30,40,50] │ -│ World │ 5 │ 50 │ [3,4,5] │ [30,40,50] │ -└───────┴─────┴─────┴─────────┴────────────┘ -``` - -Exemple d'utilisation de l' [arrayEnumerate](../../sql-reference/functions/array-functions.md#array_functions-arrayenumerate) fonction: - -``` sql -SELECT s, `n.x`, `n.y`, `nest.x`, `nest.y`, num -FROM nested_test -ARRAY JOIN nest AS n, arrayEnumerate(`nest.x`) AS num; -``` - -``` text -┌─s─────┬─n.x─┬─n.y─┬─nest.x──┬─nest.y─────┬─num─┐ -│ Hello │ 1 │ 10 │ [1,2] │ [10,20] │ 1 │ -│ Hello │ 2 │ 20 │ [1,2] │ [10,20] │ 2 │ -│ World │ 3 │ 30 │ [3,4,5] │ [30,40,50] │ 1 │ -│ World │ 4 │ 40 │ [3,4,5] │ [30,40,50] │ 2 │ -│ World │ 5 │ 50 │ [3,4,5] │ [30,40,50] │ 3 │ -└───────┴─────┴─────┴─────────┴────────────┴─────┘ -``` - -### Clause de JOINTURE {#select-join} - -Rejoint les données dans la normale [SQL JOIN](https://en.wikipedia.org/wiki/Join_(SQL)) sens. - -!!! info "Note" - Pas liées à [ARRAY JOIN](#select-array-join-clause). - -``` sql -SELECT -FROM -[GLOBAL] [ANY|ALL] [INNER|LEFT|RIGHT|FULL|CROSS] [OUTER] JOIN -(ON )|(USING ) ... -``` - -Les noms de table peuvent être spécifiés au lieu de `` et ``. Ceci est équivalent à la `SELECT * FROM table` sous-requête, sauf dans un cas particulier lorsque la table a [Rejoindre](../../engines/table-engines/special/join.md) engine – an array prepared for joining. - -#### Types pris en charge de `JOIN` {#select-join-types} - -- `INNER JOIN` (ou `JOIN`) -- `LEFT JOIN` (ou `LEFT OUTER JOIN`) -- `RIGHT JOIN` (ou `RIGHT OUTER JOIN`) -- `FULL JOIN` (ou `FULL OUTER JOIN`) -- `CROSS JOIN` (ou `,` ) - -Voir la norme [SQL JOIN](https://en.wikipedia.org/wiki/Join_(SQL)) Description. - -#### Plusieurs REJOINDRE {#multiple-join} - -En effectuant des requêtes, ClickHouse réécrit les jointures multi-tables dans la séquence des jointures à deux tables. Par exemple, S'il y a quatre tables pour join clickhouse rejoint la première et la seconde, puis rejoint le résultat avec la troisième table, et à la dernière étape, il rejoint la quatrième. - -Si une requête contient l' `WHERE` clickhouse essaie de pousser les filtres de cette clause à travers la jointure intermédiaire. S'il ne peut pas appliquer le filtre à chaque jointure intermédiaire, ClickHouse applique les filtres une fois toutes les jointures terminées. - -Nous recommandons l' `JOIN ON` ou `JOIN USING` syntaxe pour créer des requêtes. Exemple: - -``` sql -SELECT * FROM t1 JOIN t2 ON t1.a = t2.a JOIN t3 ON t1.a = t3.a -``` - -Vous pouvez utiliser des listes de tables séparées par des virgules `FROM` clause. Exemple: - -``` sql -SELECT * FROM t1, t2, t3 WHERE t1.a = t2.a AND t1.a = t3.a -``` - -Ne mélangez pas ces syntaxes. - -ClickHouse ne supporte pas directement la syntaxe avec des virgules, Nous ne recommandons donc pas de les utiliser. L'algorithme tente de réécrire la requête en termes de `CROSS JOIN` et `INNER JOIN` clauses et procède ensuite au traitement des requêtes. Lors de la réécriture de la requête, ClickHouse tente d'optimiser les performances et la consommation de mémoire. Par défaut, ClickHouse traite les virgules comme `INNER JOIN` clause et convertit `INNER JOIN` de `CROSS JOIN` lorsque l'algorithme ne peut pas garantir que `INNER JOIN` retourne les données requises. - -#### Rigueur {#select-join-strictness} - -- `ALL` — If the right table has several matching rows, ClickHouse creates a [Produit cartésien](https://en.wikipedia.org/wiki/Cartesian_product) à partir des lignes correspondantes. C'est la norme `JOIN` comportement en SQL. -- `ANY` — If the right table has several matching rows, only the first one found is joined. If the right table has only one matching row, the results of queries with `ANY` et `ALL` les mots clés sont les mêmes. -- `ASOF` — For joining sequences with a non-exact match. `ASOF JOIN` l'utilisation est décrite ci-dessous. - -**ASOF joindre L'utilisation** - -`ASOF JOIN` est utile lorsque vous devez joindre des enregistrements qui n'ont pas de correspondance exacte. - -Tables pour `ASOF JOIN` doit avoir une colonne de séquence ordonnée. Cette colonne ne peut pas être seule dans une table et doit être l'un des types de données: `UInt32`, `UInt64`, `Float32`, `Float64`, `Date`, et `DateTime`. - -Syntaxe `ASOF JOIN ... ON`: - -``` sql -SELECT expressions_list -FROM table_1 -ASOF LEFT JOIN table_2 -ON equi_cond AND closest_match_cond -``` - -Vous pouvez utiliser n'importe quel nombre de conditions d'égalité et exactement une condition de correspondance la plus proche. Exemple, `SELECT count() FROM table_1 ASOF LEFT JOIN table_2 ON table_1.a == table_2.b AND table_2.t <= table_1.t`. - -Conditions prises en charge pour la correspondance la plus proche: `>`, `>=`, `<`, `<=`. - -Syntaxe `ASOF JOIN ... USING`: - -``` sql -SELECT expressions_list -FROM table_1 -ASOF JOIN table_2 -USING (equi_column1, ... equi_columnN, asof_column) -``` - -`ASOF JOIN` utiliser `equi_columnX` pour rejoindre sur l'égalité et `asof_column` pour rejoindre le match le plus proche avec le `table_1.asof_column >= table_2.asof_column` condition. Le `asof_column` colonne toujours la dernière dans le `USING` clause. - -Par exemple, considérez les tableaux suivants: - - table_1 table_2 - event | ev_time | user_id event | ev_time | user_id - ----------|---------|---------- ----------|---------|---------- - ... ... - event_1_1 | 12:00 | 42 event_2_1 | 11:59 | 42 - ... event_2_2 | 12:30 | 42 - event_1_2 | 13:00 | 42 event_2_3 | 13:00 | 42 - ... ... - -`ASOF JOIN` peut prendre la date d'un événement utilisateur de `table_1` et trouver un événement dans `table_2` où le timestamp est plus proche de l'horodatage de l'événement à partir de `table_1` correspondant à la condition de correspondance la plus proche. Les valeurs d'horodatage égales sont les plus proches si elles sont disponibles. Ici, l' `user_id` la colonne peut être utilisée pour joindre sur l'égalité et le `ev_time` la colonne peut être utilisée pour se joindre à la correspondance la plus proche. Dans notre exemple, `event_1_1` peut être jointe à `event_2_1` et `event_1_2` peut être jointe à `event_2_3`, mais `event_2_2` ne peut pas être rejoint. - -!!! note "Note" - `ASOF` jointure est **pas** pris en charge dans le [Rejoindre](../../engines/table-engines/special/join.md) tableau moteur. - -Pour définir la valeur de rigueur par défaut, utilisez le paramètre de configuration de session [join\_default\_strictness](../../operations/settings/settings.md#settings-join_default_strictness). - -#### GLOBAL JOIN {#global-join} - -Lors de l'utilisation normale `JOIN` la requête est envoyée aux serveurs distants. Les sous-requêtes sont exécutées sur chacune d'elles afin de créer la bonne table, et la jointure est effectuée avec cette table. En d'autres termes, la table de droite est formée sur chaque serveur séparément. - -Lors de l'utilisation de `GLOBAL ... JOIN`, d'abord le serveur demandeur exécute une sous-requête pour calculer la bonne table. Cette table temporaire est transmise à chaque serveur distant, et les requêtes sont exécutées sur eux en utilisant les données temporaires qui ont été transmises. - -Soyez prudent lorsque vous utilisez `GLOBAL`. Pour plus d'informations, consultez la section [Sous-requêtes distribuées](#select-distributed-subqueries). - -#### Recommandations D'Utilisation {#usage-recommendations} - -Lors de l'exécution d'un `JOIN`, il n'y a pas d'optimisation de la commande d'exécution par rapport aux autres stades de la requête. La jointure (une recherche dans la table de droite) est exécutée avant de filtrer `WHERE` et avant l'agrégation. Afin de définir explicitement l'ordre de traitement, nous vous recommandons d'exécuter une `JOIN` sous-requête avec une sous-requête. - -Exemple: - -``` sql -SELECT - CounterID, - hits, - visits -FROM -( - SELECT - CounterID, - count() AS hits - FROM test.hits - GROUP BY CounterID -) ANY LEFT JOIN -( - SELECT - CounterID, - sum(Sign) AS visits - FROM test.visits - GROUP BY CounterID -) USING CounterID -ORDER BY hits DESC -LIMIT 10 -``` - -``` text -┌─CounterID─┬───hits─┬─visits─┐ -│ 1143050 │ 523264 │ 13665 │ -│ 731962 │ 475698 │ 102716 │ -│ 722545 │ 337212 │ 108187 │ -│ 722889 │ 252197 │ 10547 │ -│ 2237260 │ 196036 │ 9522 │ -│ 23057320 │ 147211 │ 7689 │ -│ 722818 │ 90109 │ 17847 │ -│ 48221 │ 85379 │ 4652 │ -│ 19762435 │ 77807 │ 7026 │ -│ 722884 │ 77492 │ 11056 │ -└───────────┴────────┴────────┘ -``` - -Les sous-requêtes ne vous permettent pas de définir des noms ou de les utiliser pour référencer une colonne à partir d'une sous-requête spécifique. -Les colonnes spécifiées dans `USING` doit avoir les mêmes noms dans les deux sous-requêtes, et les autres colonnes doivent être nommées différemment. Vous pouvez utiliser des alias pour les noms des colonnes dans les sous-requêtes (l'exemple utilise l'alias `hits` et `visits`). - -Le `USING` clause spécifie une ou plusieurs colonnes de jointure, qui établit l'égalité de ces colonnes. La liste des colonnes est définie sans crochets. Les conditions de jointure plus complexes ne sont pas prises en charge. - -La table de droite (le résultat de la sous-requête) réside dans la RAM. S'il n'y a pas assez de mémoire, vous ne pouvez pas exécuter un `JOIN`. - -Chaque fois qu'une requête est exécutée avec la même `JOIN`, la sous-requête est exécutée à nouveau car le résultat n'est pas mis en cache. Pour éviter cela, utilisez la spéciale [Rejoindre](../../engines/table-engines/special/join.md) table engine, qui est un tableau préparé pour l'assemblage qui est toujours en RAM. - -Dans certains cas, il est plus efficace d'utiliser `IN` plutôt `JOIN`. -Parmi les différents types de `JOIN`, le plus efficace est d' `ANY LEFT JOIN`, puis `ANY INNER JOIN`. Les moins efficaces sont `ALL LEFT JOIN` et `ALL INNER JOIN`. - -Si vous avez besoin d'un `JOIN` pour se joindre à des tables de dimension (ce sont des tables relativement petites qui contiennent des propriétés de dimension, telles que des noms pour des campagnes publicitaires), un `JOIN` peut-être pas très pratique en raison du fait que la bonne table est ré-accédée pour chaque requête. Pour de tels cas, il y a un “external dictionaries” la fonctionnalité que vous devez utiliser à la place de `JOIN`. Pour plus d'informations, consultez la section [Dictionnaires externes](../dictionaries/external-dictionaries/external-dicts.md). - -**Limitations De Mémoire** - -ClickHouse utilise le [jointure de hachage](https://en.wikipedia.org/wiki/Hash_join) algorithme. ClickHouse prend le `` et crée une table de hachage pour cela dans la RAM. Si vous devez restreindre la consommation de mémoire de l'opération join utilisez les paramètres suivants: - -- [max\_rows\_in\_join](../../operations/settings/query-complexity.md#settings-max_rows_in_join) — Limits number of rows in the hash table. -- [max\_bytes\_in\_join](../../operations/settings/query-complexity.md#settings-max_bytes_in_join) — Limits size of the hash table. - -Lorsque l'une de ces limites est atteinte, ClickHouse agit comme [join\_overflow\_mode](../../operations/settings/query-complexity.md#settings-join_overflow_mode) réglage des instructions. - -#### Traitement des cellules vides ou nulles {#processing-of-empty-or-null-cells} - -Lors de la jonction de tables, les cellules vides peuvent apparaître. Paramètre [join\_use\_nulls](../../operations/settings/settings.md#join_use_nulls) définir comment clickhouse remplit ces cellules. - -Si l' `JOIN` les touches sont [Nullable](../data-types/nullable.md) champs, les lignes où au moins une des clés a la valeur [NULL](../syntax.md#null-literal) ne sont pas jointes. - -#### Limitations De Syntaxe {#syntax-limitations} - -Pour plusieurs `JOIN` clauses dans un seul `SELECT` requête: - -- Prendre toutes les colonnes via `*` n'est disponible que si les tables sont jointes, pas les sous-requêtes. -- Le `PREWHERE` la clause n'est pas disponible. - -Pour `ON`, `WHERE`, et `GROUP BY` clause: - -- Les expressions arbitraires ne peuvent pas être utilisées dans `ON`, `WHERE`, et `GROUP BY` mais vous pouvez définir une expression dans un `SELECT` clause et ensuite l'utiliser dans ces clauses via un alias. - -### Clause where {#select-where} - -S'il existe une clause WHERE, elle doit contenir une expression de type UInt8. C'est généralement une expression avec comparaison et opérateurs logiques. -Cette expression est utilisée pour filtrer les données avant toutes les autres transformations. - -Si les index sont pris en charge par le moteur de table de base de données, l'expression est évaluée sur la possibilité d'utiliser des index. - -### Clause PREWHERE {#prewhere-clause} - -Cette clause a le même sens que la clause WHERE. La différence est dans laquelle les données sont lues à partir de la table. -Lors de L'utilisation de PREWHERE, d'abord, seules les colonnes nécessaires à L'exécution de PREWHERE sont lues. Ensuite, les autres colonnes sont lues qui sont nécessaires pour exécuter la requête, mais seulement les blocs où L'expression PREWHERE est vraie. - -Il est logique d'utiliser PREWHERE s'il existe des conditions de filtration qui sont utilisées par une minorité de colonnes dans la requête, mais qui fournissent une filtration de données forte. Cela réduit le volume de données à lire. - -Par exemple, il est utile d'écrire PREWHERE pour les requêtes qui extraient un grand nombre de colonnes, mais qui n'ont que la filtration pour quelques colonnes. - -PREWHERE est uniquement pris en charge par les tables `*MergeTree` famille. - -Une requête peut spécifier simultanément PREWHERE et WHERE. Dans ce cas, PREWHERE précède WHERE. - -Si l' ‘optimize\_move\_to\_prewhere’ le paramètre est défini sur 1 et PREWHERE est omis, le système utilise des heuristiques pour déplacer automatiquement des parties d'expressions D'où vers PREWHERE. - -### Clause GROUP BY {#select-group-by-clause} - -C'est l'une des parties les plus importantes d'un SGBD orienté colonne. - -S'il existe une clause GROUP BY, elle doit contenir une liste d'expressions. Chaque expression sera appelée ici comme “key”. -Toutes les expressions des clauses SELECT, HAVING et ORDER BY doivent être calculées à partir de clés ou de fonctions d'agrégation. En d'autres termes, chaque colonne sélectionnée dans la table doit être utilisée soit dans les clés, soit dans les fonctions d'agrégation. - -Si une requête ne contient que des colonnes de table dans les fonctions d'agrégation, la clause GROUP BY peut être omise et l'agrégation par un ensemble de clés vide est supposée. - -Exemple: - -``` sql -SELECT - count(), - median(FetchTiming > 60 ? 60 : FetchTiming), - count() - sum(Refresh) -FROM hits -``` - -Cependant, contrairement au SQL standard, si la table n'a pas de lignes (soit il n'y en a pas du tout, soit il n'y en a pas après avoir utilisé WHERE to filter), un résultat vide est renvoyé, et non le résultat d'une des lignes contenant les valeurs initiales des fonctions d'agrégat. - -Contrairement à MySQL (et conforme à SQL standard), vous ne pouvez pas obtenir une valeur d'une colonne qui n'est pas dans une fonction clé ou agrégée (sauf les expressions constantes). Pour contourner ce problème, vous pouvez utiliser le ‘any’ fonction d'agrégation (récupère la première valeur rencontrée) ou ‘min/max’. - -Exemple: - -``` sql -SELECT - domainWithoutWWW(URL) AS domain, - count(), - any(Title) AS title -- getting the first occurred page header for each domain. -FROM hits -GROUP BY domain -``` - -Pour chaque valeur de clé différente rencontrée, GROUP BY calcule un ensemble de valeurs de fonction d'agrégation. - -GROUP BY n'est pas pris en charge pour les colonnes de tableau. - -Une constante ne peut pas être spécifiée comme arguments pour les fonctions d'agrégation. Exemple: somme(1). Au lieu de cela, vous pouvez vous débarrasser de la constante. Exemple: `count()`. - -#### Le Traitement NULL {#null-processing} - -Pour le regroupement, ClickHouse interprète [NULL](../syntax.md#null-literal) comme une valeur, et `NULL=NULL`. - -Voici un exemple pour montrer ce que cela signifie. - -Supposons que vous avez cette table: - -``` text -┌─x─┬────y─┐ -│ 1 │ 2 │ -│ 2 │ ᴺᵁᴸᴸ │ -│ 3 │ 2 │ -│ 3 │ 3 │ -│ 3 │ ᴺᵁᴸᴸ │ -└───┴──────┘ -``` - -Requête `SELECT sum(x), y FROM t_null_big GROUP BY y` résultats dans: - -``` text -┌─sum(x)─┬────y─┐ -│ 4 │ 2 │ -│ 3 │ 3 │ -│ 5 │ ᴺᵁᴸᴸ │ -└────────┴──────┘ -``` - -Vous pouvez voir que `GROUP BY` pour `y = NULL` résumer `x` comme si `NULL` a cette valeur. - -Si vous passez plusieurs clés `GROUP BY` le résultat vous donnera toutes les combinaisons de la sélection, comme si `NULL` ont une valeur spécifique. - -#### Avec modificateur de totaux {#with-totals-modifier} - -Si le modificateur avec totaux est spécifié, une autre ligne sera calculée. Cette ligne aura des colonnes clés contenant des valeurs par défaut (zéros ou lignes vides), et des colonnes de fonctions d'agrégat avec les valeurs calculées sur toutes les lignes (le “total” valeur). - -Cette ligne supplémentaire est sortie dans les formats JSON\*, TabSeparated\* et Pretty\*, séparément des autres lignes. Dans les autres formats, cette ligne n'est pas sortie. - -Dans les formats JSON\*, cette ligne est sortie en tant que ‘totals’ champ. Dans les formats TabSeparated\*, la ligne vient après le résultat principal, précédée d'une ligne vide (après les autres données). Dans les formats Pretty\*, la ligne est sortie sous forme de table séparée après le résultat principal. - -`WITH TOTALS` peut être exécuté de différentes manières lorsqu'il est présent. Le comportement dépend de l' ‘totals\_mode’ paramètre. -Par défaut, `totals_mode = 'before_having'`. Dans ce cas, ‘totals’ est calculé sur toutes les lignes, y compris celles qui ne passent pas par ‘max\_rows\_to\_group\_by’. - -Les autres alternatives incluent uniquement les lignes qui passent à travers avoir dans ‘totals’, et se comporter différemment avec le réglage `max_rows_to_group_by` et `group_by_overflow_mode = 'any'`. - -`after_having_exclusive` – Don't include rows that didn't pass through `max_rows_to_group_by`. En d'autres termes, ‘totals’ aura moins ou le même nombre de lignes que si `max_rows_to_group_by` ont été omis. - -`after_having_inclusive` – Include all the rows that didn't pass through ‘max\_rows\_to\_group\_by’ dans ‘totals’. En d'autres termes, ‘totals’ aura plus ou le même nombre de lignes que si `max_rows_to_group_by` ont été omis. - -`after_having_auto` – Count the number of rows that passed through HAVING. If it is more than a certain amount (by default, 50%), include all the rows that didn't pass through ‘max\_rows\_to\_group\_by’ dans ‘totals’. Sinon, ne pas les inclure. - -`totals_auto_threshold` – By default, 0.5. The coefficient for `after_having_auto`. - -Si `max_rows_to_group_by` et `group_by_overflow_mode = 'any'` ne sont pas utilisés, toutes les variations de `after_having` sont les mêmes, et vous pouvez utiliser l'un d'eux (par exemple, `after_having_auto`). - -Vous pouvez utiliser avec les totaux dans les sous-requêtes, y compris les sous-requêtes dans la clause JOIN (dans ce cas, les valeurs totales respectives sont combinées). - -#### Groupe par dans la mémoire externe {#select-group-by-in-external-memory} - -Vous pouvez activer le dumping des données temporaires sur le disque pour limiter l'utilisation de la mémoire pendant `GROUP BY`. -Le [max\_bytes\_before\_external\_group\_by](../../operations/settings/settings.md#settings-max_bytes_before_external_group_by) réglage détermine le seuil de consommation de RAM pour le dumping `GROUP BY` données temporaires dans le système de fichiers. Si elle est définie sur 0 (valeur par défaut), elle est désactivée. - -Lors de l'utilisation de `max_bytes_before_external_group_by`, nous vous recommandons de définir `max_memory_usage` environ deux fois plus élevé. Ceci est nécessaire car il y a deux étapes à l'agrégation: la lecture de la date et la formation des données intermédiaires (1) et la fusion des données intermédiaires (2). Le Dumping des données dans le système de fichiers ne peut se produire qu'au cours de l'étape 1. Si les données temporaires n'ont pas été vidées, l'étape 2 peut nécessiter jusqu'à la même quantité de mémoire qu'à l'étape 1. - -Par exemple, si [max\_memory\_usage](../../operations/settings/settings.md#settings_max_memory_usage) a été défini sur 10000000000 et que vous souhaitez utiliser l'agrégation externe, il est logique de définir `max_bytes_before_external_group_by` à 10000000000, et max\_memory\_usage à 20000000000. Lorsque l'agrégation externe est déclenchée (s'il y a eu au moins un vidage de données temporaires), la consommation maximale de RAM n'est que légèrement supérieure à `max_bytes_before_external_group_by`. - -Avec le traitement des requêtes distribuées, l'agrégation externe est effectuée sur des serveurs distants. Pour que le serveur demandeur n'utilise qu'une petite quantité de RAM, définissez `distributed_aggregation_memory_efficient` 1. - -Lors de la fusion de données vidées sur le disque, ainsi que lors de la fusion des résultats de serveurs distants lorsque `distributed_aggregation_memory_efficient` paramètre est activé, consomme jusqu'à `1/256 * the_number_of_threads` à partir de la quantité totale de mémoire RAM. - -Lorsque l'agrégation externe est activée, s'il y a moins de `max_bytes_before_external_group_by` of data (i.e. data was not flushed), the query runs just as fast as without external aggregation. If any temporary data was flushed, the run time will be several times longer (approximately three times). - -Si vous avez un `ORDER BY` avec un `LIMIT` après `GROUP BY` puis la quantité de RAM dépend de la quantité de données dans `LIMIT`, pas dans l'ensemble de la table. Mais si l' `ORDER BY` n'a pas `LIMIT`, n'oubliez pas d'activer externe de tri (`max_bytes_before_external_sort`). - -### Limite par Clause {#limit-by-clause} - -Une requête avec l' `LIMIT n BY expressions` la clause sélectionne le premier `n` lignes pour chaque valeur distincte de `expressions`. La clé pour `LIMIT BY` peut contenir n'importe quel nombre de [expression](../syntax.md#syntax-expressions). - -ClickHouse prend en charge la syntaxe suivante: - -- `LIMIT [offset_value, ]n BY expressions` -- `LIMIT n OFFSET offset_value BY expressions` - -Pendant le traitement de la requête, ClickHouse sélectionne les données classées par clé de tri. La clé de tri est définie explicitement à l'aide [ORDER BY](#select-order-by) clause ou implicitement en tant que propriété du moteur de table. Puis clickhouse s'applique `LIMIT n BY expressions` et renvoie le premier `n` lignes pour chaque combinaison distincte de `expressions`. Si `OFFSET` est spécifié, puis pour chaque bloc de données qui appartient à une combinaison particulière de `expressions`, Clickhouse saute `offset_value` nombre de lignes depuis le début du bloc et renvoie un maximum de `n` les lignes en conséquence. Si `offset_value` est plus grand que le nombre de lignes dans le bloc de données, ClickHouse renvoie zéro lignes du bloc. - -`LIMIT BY` n'est pas liée à `LIMIT`. Ils peuvent tous deux être utilisés dans la même requête. - -**Exemple** - -Exemple de table: - -``` sql -CREATE TABLE limit_by(id Int, val Int) ENGINE = Memory; -INSERT INTO limit_by values(1, 10), (1, 11), (1, 12), (2, 20), (2, 21); -``` - -Requête: - -``` sql -SELECT * FROM limit_by ORDER BY id, val LIMIT 2 BY id -``` - -``` text -┌─id─┬─val─┐ -│ 1 │ 10 │ -│ 1 │ 11 │ -│ 2 │ 20 │ -│ 2 │ 21 │ -└────┴─────┘ -``` - -``` sql -SELECT * FROM limit_by ORDER BY id, val LIMIT 1, 2 BY id -``` - -``` text -┌─id─┬─val─┐ -│ 1 │ 11 │ -│ 1 │ 12 │ -│ 2 │ 21 │ -└────┴─────┘ -``` - -Le `SELECT * FROM limit_by ORDER BY id, val LIMIT 2 OFFSET 1 BY id` requête renvoie le même résultat. - -La requête suivante renvoie les 5 principaux référents pour chaque `domain, device_type` paire avec un maximum de 100 lignes au total (`LIMIT n BY + LIMIT`). - -``` sql -SELECT - domainWithoutWWW(URL) AS domain, - domainWithoutWWW(REFERRER_URL) AS referrer, - device_type, - count() cnt -FROM hits -GROUP BY domain, referrer, device_type -ORDER BY cnt DESC -LIMIT 5 BY domain, device_type -LIMIT 100 -``` - -### Clause HAVING {#having-clause} - -Permet de filtrer le résultat reçu après GROUP BY, similaire à la clause WHERE. -Où et ayant diffèrent en ce que Où est effectué avant l'agrégation (GROUP BY), tout en ayant est effectué après. -Si l'agrégation n'est pas effectuée, HAVING ne peut pas être utilisé. - -### Clause ORDER BY {#select-order-by} - -La clause ORDER BY contient une liste d'expressions, qui peuvent chacune être affectées à DESC ou ASC (la direction de tri). Si la direction n'est pas spécifiée, ASC est supposé. ASC est trié dans l'ordre croissant, et DESC dans l'ordre décroissant. La direction de tri s'applique à une seule expression, pas à la liste entière. Exemple: `ORDER BY Visits DESC, SearchPhrase` - -Pour le tri par valeurs de chaîne, vous pouvez spécifier le classement (comparaison). Exemple: `ORDER BY SearchPhrase COLLATE 'tr'` - pour le tri par mot-clé dans l'ordre croissant, en utilisant l'alphabet turc, insensible à la casse, en supposant que les chaînes sont encodées en UTF-8. COLLATE peut être spécifié ou non pour chaque expression dans L'ordre par indépendamment. Si ASC ou DESC est spécifié, COLLATE est spécifié après. Lors de L'utilisation de COLLATE, le tri est toujours insensible à la casse. - -Nous recommandons uniquement D'utiliser COLLATE pour le tri final d'un petit nombre de lignes, car le tri avec COLLATE est moins efficace que le tri normal par octets. - -Les lignes qui ont des valeurs identiques pour la liste des expressions de tri sont sorties dans un ordre arbitraire, qui peut également être non déterministe (différent à chaque fois). -Si la clause ORDER BY est omise, l'ordre des lignes est également indéfini et peut également être non déterministe. - -`NaN` et `NULL` ordre de tri: - -- Avec le modificateur `NULLS FIRST` — First `NULL`, puis `NaN` puis d'autres valeurs. -- Avec le modificateur `NULLS LAST` — First the values, then `NaN`, puis `NULL`. -- Default — The same as with the `NULLS LAST` modificateur. - -Exemple: - -Pour la table - -``` text -┌─x─┬────y─┐ -│ 1 │ ᴺᵁᴸᴸ │ -│ 2 │ 2 │ -│ 1 │ nan │ -│ 2 │ 2 │ -│ 3 │ 4 │ -│ 5 │ 6 │ -│ 6 │ nan │ -│ 7 │ ᴺᵁᴸᴸ │ -│ 6 │ 7 │ -│ 8 │ 9 │ -└───┴──────┘ -``` - -Exécuter la requête `SELECT * FROM t_null_nan ORDER BY y NULLS FIRST` obtenir: - -``` text -┌─x─┬────y─┐ -│ 1 │ ᴺᵁᴸᴸ │ -│ 7 │ ᴺᵁᴸᴸ │ -│ 1 │ nan │ -│ 6 │ nan │ -│ 2 │ 2 │ -│ 2 │ 2 │ -│ 3 │ 4 │ -│ 5 │ 6 │ -│ 6 │ 7 │ -│ 8 │ 9 │ -└───┴──────┘ -``` - -Lorsque les nombres à virgule flottante sont triés, les Nan sont séparés des autres valeurs. Quel que soit l'ordre de tri, NaNs viennent à la fin. En d'autres termes, pour le Tri ascendant, ils sont placés comme s'ils étaient plus grands que tous les autres nombres, tandis que pour le Tri descendant, ils sont placés comme s'ils étaient plus petits que les autres. - -Moins de RAM est utilisé si une limite assez petite est spécifiée en plus de ORDER BY. Sinon, la quantité de mémoire dépensée est proportionnelle au volume de données à trier. Pour le traitement des requêtes distribuées, si GROUP BY est omis, le tri est partiellement effectué sur des serveurs distants et les résultats sont fusionnés sur le serveur demandeur. Cela signifie que pour le tri distribué, le volume de données à trier peut être supérieur à la quantité de mémoire sur un seul serveur. - -S'il N'y a pas assez de RAM, il est possible d'effectuer un tri dans la mémoire externe (création de fichiers temporaires sur un disque). Utilisez le paramètre `max_bytes_before_external_sort` pour ce but. S'il est défini sur 0 (par défaut), le tri externe est désactivé. Si elle est activée, lorsque le volume de données à trier atteint le nombre spécifié d'octets, les données collectées sont triés et déposés dans un fichier temporaire. Une fois toutes les données lues, tous les fichiers triés sont fusionnés et les résultats sont générés. Les fichiers sont écrits dans le répertoire/var/lib / clickhouse / tmp / dans la configuration (par défaut, mais vous pouvez ‘tmp\_path’ paramètre pour modifier ce paramètre). - -L'exécution d'une requête peut utiliser plus de mémoire que ‘max\_bytes\_before\_external\_sort’. Pour cette raison, ce paramètre doit avoir une valeur significativement inférieure à ‘max\_memory\_usage’. Par exemple, si votre serveur dispose de 128 Go de RAM et que vous devez exécuter une seule requête, définissez ‘max\_memory\_usage’ à 100 Go, et ‘max\_bytes\_before\_external\_sort’ à 80 Go. - -Le tri externe fonctionne beaucoup moins efficacement que le tri dans la RAM. - -### Clause SELECT {#select-select} - -[Expression](../syntax.md#syntax-expressions) spécifié dans le `SELECT` clause sont calculés après toutes les opérations dans les clauses décrites ci-dessus sont terminés. Ces expressions fonctionnent comme si elles s'appliquaient à des lignes séparées dans le résultat. Si les expressions dans le `SELECT` la clause contient des fonctions d'agrégation, puis clickhouse traite les fonctions d'agrégation et les expressions utilisées [GROUP BY](#select-group-by-clause) agrégation. - -Si vous souhaitez inclure toutes les colonnes dans le résultat, utilisez l'astérisque (`*`) symbole. Exemple, `SELECT * FROM ...`. - -Pour correspondre à certaines colonnes dans le résultat avec un [re2](https://en.wikipedia.org/wiki/RE2_(software)) expression régulière, vous pouvez utiliser le `COLUMNS` expression. - -``` sql -COLUMNS('regexp') -``` - -Par exemple, considérez le tableau: - -``` sql -CREATE TABLE default.col_names (aa Int8, ab Int8, bc Int8) ENGINE = TinyLog -``` - -La requête suivante sélectionne les données de toutes les colonnes contenant les `a` symbole dans leur nom. - -``` sql -SELECT COLUMNS('a') FROM col_names -``` - -``` text -┌─aa─┬─ab─┐ -│ 1 │ 1 │ -└────┴────┘ -``` - -Les colonnes sélectionnées sont retournés pas dans l'ordre alphabétique. - -Vous pouvez utiliser plusieurs `COLUMNS` expressions dans une requête et leur appliquer des fonctions. - -Exemple: - -``` sql -SELECT COLUMNS('a'), COLUMNS('c'), toTypeName(COLUMNS('c')) FROM col_names -``` - -``` text -┌─aa─┬─ab─┬─bc─┬─toTypeName(bc)─┐ -│ 1 │ 1 │ 1 │ Int8 │ -└────┴────┴────┴────────────────┘ -``` - -Chaque colonne renvoyée par le `COLUMNS` expression est passée à la fonction en tant qu'argument séparé. Vous pouvez également passer d'autres arguments à la fonction si elle les supporte. Soyez prudent lorsque vous utilisez des fonctions. Si une fonction ne prend pas en charge le nombre d'arguments que vous lui avez transmis, ClickHouse lève une exception. - -Exemple: - -``` sql -SELECT COLUMNS('a') + COLUMNS('c') FROM col_names -``` - -``` text -Received exception from server (version 19.14.1): -Code: 42. DB::Exception: Received from localhost:9000. DB::Exception: Number of arguments for function plus doesn't match: passed 3, should be 2. -``` - -Dans cet exemple, `COLUMNS('a')` retourne deux colonnes: `aa` et `ab`. `COLUMNS('c')` renvoie la `bc` colonne. Le `+` l'opérateur ne peut pas s'appliquer à 3 arguments, donc ClickHouse lève une exception avec le message pertinent. - -Colonnes qui correspondent à la `COLUMNS` l'expression peut avoir différents types de données. Si `COLUMNS` ne correspond à aucune colonne et est la seule expression dans `SELECT`, ClickHouse lance une exception. - -### La Clause DISTINCT {#select-distinct} - -Si DISTINCT est spécifié, une seule ligne restera hors de tous les ensembles de lignes entièrement correspondantes dans le résultat. -Le résultat sera le même que si GROUP BY était spécifié dans tous les champs spécifiés dans SELECT without aggregate functions. Mais il y a plusieurs différences de GROUP BY: - -- DISTINCT peut être appliqué avec GROUP BY. -- Lorsque ORDER BY est omis et que LIMIT est défini, la requête s'arrête immédiatement après la lecture du nombre requis de lignes différentes. -- Les blocs de données sont produits au fur et à mesure qu'ils sont traités, sans attendre que la requête entière se termine. - -DISTINCT n'est pas pris en charge si SELECT a au moins une colonne de tableau. - -`DISTINCT` fonctionne avec [NULL](../syntax.md#null-literal) comme si `NULL` ont une valeur spécifique, et `NULL=NULL`. En d'autres termes, dans le `DISTINCT` résultats, différentes combinaisons avec `NULL` qu'une seule fois. - -Clickhouse prend en charge l'utilisation du `DISTINCT` et `ORDER BY` clauses pour différentes colonnes dans une requête. Le `DISTINCT` la clause est exécutée avant la `ORDER BY` clause. - -Exemple de table: - -``` text -┌─a─┬─b─┐ -│ 2 │ 1 │ -│ 1 │ 2 │ -│ 3 │ 3 │ -│ 2 │ 4 │ -└───┴───┘ -``` - -Lors de la sélection de données avec le `SELECT DISTINCT a FROM t1 ORDER BY b ASC` requête, nous obtenons le résultat suivant: - -``` text -┌─a─┐ -│ 2 │ -│ 1 │ -│ 3 │ -└───┘ -``` - -Si nous changeons la direction de tri `SELECT DISTINCT a FROM t1 ORDER BY b DESC`, nous obtenons le résultat suivant: - -``` text -┌─a─┐ -│ 3 │ -│ 1 │ -│ 2 │ -└───┘ -``` - -Rangée `2, 4` a été coupé avant de les trier. - -Prenez en compte cette spécificité d'implémentation lors de la programmation des requêtes. - -### Clause LIMIT {#limit-clause} - -`LIMIT m` vous permet de sélectionner la première `m` lignes du résultat. - -`LIMIT n, m` vous permet de sélectionner la première `m` lignes du résultat après avoir sauté le premier `n` rangée. Le `LIMIT m OFFSET n` la syntaxe est également prise en charge. - -`n` et `m` doivent être des entiers non négatifs. - -Si il n'y a pas un `ORDER BY` clause qui trie explicitement les résultats, le résultat peut être arbitraire et non déterministe. - -### Clause UNION ALL {#union-all-clause} - -Vous pouvez utiliser UNION ALL pour combiner n'importe quel nombre de requêtes. Exemple: - -``` sql -SELECT CounterID, 1 AS table, toInt64(count()) AS c - FROM test.hits - GROUP BY CounterID - -UNION ALL - -SELECT CounterID, 2 AS table, sum(Sign) AS c - FROM test.visits - GROUP BY CounterID - HAVING c > 0 -``` - -Seule UNION ALL est prise en charge. L'UNION régulière (Union distincte) n'est pas prise en charge. Si vous avez besoin D'UNION DISTINCT, vous pouvez écrire SELECT DISTINCT à partir d'une sous-requête contenant UNION ALL. - -Les requêtes qui font partie de L'UNION peuvent toutes être exécutées simultanément et leurs résultats peuvent être mélangés. - -La structure des résultats (le nombre et le type de colonnes) doit correspondre aux requêtes. Mais les noms des colonnes peuvent différer. Dans ce cas, les noms de colonne pour le résultat final seront tirés de la première requête. La coulée de Type est effectuée pour les syndicats. Par exemple, si deux requêtes combinées ont le même champ avec non-`Nullable` et `Nullable` types d'un type compatible, la `UNION ALL` a un `Nullable` type de champ. - -Les requêtes qui font partie de UNION ALL ne peuvent pas être placées entre crochets. ORDER BY et LIMIT sont appliqués à des requêtes distinctes, pas au résultat final. Si vous devez appliquer une conversion au résultat final, vous pouvez placer toutes les requêtes avec UNION ALL dans une sous-requête de la clause FROM. - -### Dans OUTFILE Clause {#into-outfile-clause} - -Ajouter l' `INTO OUTFILE filename` clause (où filename est un littéral de chaîne) pour rediriger la sortie de la requête vers le fichier spécifié. -Contrairement à MySQL, le fichier est créé du côté client. La requête échouera si un fichier portant le même nom existe déjà. -Cette fonctionnalité est disponible dans le client de ligne de commande et clickhouse-local (une requête envoyée via L'interface HTTP échouera). - -Le format de sortie par défaut est TabSeparated (le même que dans le mode batch client de ligne de commande). - -### FORMAT de la Clause {#format-clause} - -Spécifier ‘FORMAT format’ pour obtenir des données dans n'importe quel format spécifié. -Vous pouvez l'utiliser pour plus de commodité, ou pour créer des vidages. -Pour plus d'informations, consultez la section “Formats”. -Si la clause FORMAT est omise, le format par défaut est utilisé, ce qui dépend à la fois des paramètres et de l'interface utilisée pour accéder à la base de données. Pour L'interface HTTP et le client de ligne de commande en mode batch, le format par défaut est TabSeparated. Pour le client de ligne de commande en mode interactif, le format par défaut est PrettyCompact (il a des tables attrayantes et compactes). - -Lors de l'utilisation du client de ligne de commande, les données sont transmises au client dans un format efficace interne. Le client interprète indépendamment la clause de FORMAT de la requête et formate les données elles-mêmes (soulageant ainsi le réseau et le serveur de la charge). - -### Dans les opérateurs {#select-in-operators} - -Le `IN`, `NOT IN`, `GLOBAL IN`, et `GLOBAL NOT IN` les opérateurs sont traitées séparément, car leur fonctionnalité est assez riche. - -Le côté gauche de l'opérateur, soit une seule colonne ou un tuple. - -Exemple: - -``` sql -SELECT UserID IN (123, 456) FROM ... -SELECT (CounterID, UserID) IN ((34, 123), (101500, 456)) FROM ... -``` - -Si le côté gauche est une colonne unique qui est dans l'index, et le côté droit est un ensemble de constantes, le système utilise l'index pour le traitement de la requête. - -Don't list too many values explicitly (i.e. millions). If a data set is large, put it in a temporary table (for example, see the section “External data for query processing”), puis utiliser une sous-requête. - -Le côté droit de l'opérateur peut être un ensemble d'expressions constantes, un ensemble de tuples avec des expressions constantes (illustrées dans les exemples ci-dessus), ou le nom d'une table de base de données ou une sous-requête SELECT entre parenthèses. - -Si le côté droit de l'opérateur est le nom d'une table (par exemple, `UserID IN users`), ceci est équivalent à la sous-requête `UserID IN (SELECT * FROM users)`. Utilisez ceci lorsque vous travaillez avec des données externes envoyées avec la requête. Par exemple, la requête peut être envoyée avec un ensemble d'ID utilisateur chargés dans le ‘users’ table temporaire, qui doit être filtrée. - -Si le côté droit de l'opérateur est un nom de table qui a le moteur Set (un ensemble de données préparé qui est toujours en RAM), l'ensemble de données ne sera pas créé à nouveau pour chaque requête. - -La sous-requête peut spécifier plusieurs colonnes pour filtrer les tuples. -Exemple: - -``` sql -SELECT (CounterID, UserID) IN (SELECT CounterID, UserID FROM ...) FROM ... -``` - -Les colonnes à gauche et à droite de l'opérateur doit avoir le même type. - -L'opérateur IN et la sous-requête peuvent se produire dans n'importe quelle partie de la requête, y compris dans les fonctions d'agrégation et les fonctions lambda. -Exemple: - -``` sql -SELECT - EventDate, - avg(UserID IN - ( - SELECT UserID - FROM test.hits - WHERE EventDate = toDate('2014-03-17') - )) AS ratio -FROM test.hits -GROUP BY EventDate -ORDER BY EventDate ASC -``` - -``` text -┌──EventDate─┬────ratio─┐ -│ 2014-03-17 │ 1 │ -│ 2014-03-18 │ 0.807696 │ -│ 2014-03-19 │ 0.755406 │ -│ 2014-03-20 │ 0.723218 │ -│ 2014-03-21 │ 0.697021 │ -│ 2014-03-22 │ 0.647851 │ -│ 2014-03-23 │ 0.648416 │ -└────────────┴──────────┘ -``` - -Pour chaque jour après le 17 mars, comptez le pourcentage de pages vues par les utilisateurs qui ont visité le site le 17 mars. -Une sous-requête dans la clause est toujours exécuter une seule fois sur un seul serveur. Il n'y a pas de sous-requêtes dépendantes. - -#### Le Traitement NULL {#null-processing-1} - -Pendant le traitement de la demande, l'opérateur n'assume que le résultat d'une opération avec [NULL](../syntax.md#null-literal) est toujours égale à `0` indépendamment de savoir si `NULL` est sur le côté droit ou gauche de l'opérateur. `NULL` les valeurs ne sont incluses dans aucun jeu de données, ne correspondent pas entre elles et ne peuvent pas être comparées. - -Voici un exemple avec le `t_null` table: - -``` text -┌─x─┬────y─┐ -│ 1 │ ᴺᵁᴸᴸ │ -│ 2 │ 3 │ -└───┴──────┘ -``` - -L'exécution de la requête `SELECT x FROM t_null WHERE y IN (NULL,3)` vous donne le résultat suivant: - -``` text -┌─x─┐ -│ 2 │ -└───┘ -``` - -Vous pouvez voir que la ligne dans laquelle `y = NULL` est jeté hors de résultats de la requête. C'est parce que ClickHouse ne peut pas décider si `NULL` est inclus dans le `(NULL,3)` ensemble, les retours `0` comme le résultat de l'opération, et `SELECT` exclut cette ligne de la sortie finale. - -``` sql -SELECT y IN (NULL, 3) -FROM t_null -``` - -``` text -┌─in(y, tuple(NULL, 3))─┐ -│ 0 │ -│ 1 │ -└───────────────────────┘ -``` - -#### Sous-Requêtes Distribuées {#select-distributed-subqueries} - -Il y a deux options pour IN-S avec des sous-requêtes (similaires aux jointures): normal `IN` / `JOIN` et `GLOBAL IN` / `GLOBAL JOIN`. Ils diffèrent dans la façon dont ils sont exécutés pour le traitement des requêtes distribuées. - -!!! attention "Attention" - Rappelez-vous que les algorithmes décrits ci-dessous peuvent travailler différemment en fonction de la [paramètre](../../operations/settings/settings.md) `distributed_product_mode` paramètre. - -Lors de l'utilisation de l'IN régulier, la requête est envoyée à des serveurs distants, et chacun d'eux exécute les sous-requêtes dans le `IN` ou `JOIN` clause. - -Lors de l'utilisation de `GLOBAL IN` / `GLOBAL JOINs`, d'abord toutes les sous-requêtes sont exécutées pour `GLOBAL IN` / `GLOBAL JOINs`, et les résultats sont recueillis dans des tableaux temporaires. Ensuite, les tables temporaires sont envoyés à chaque serveur distant, où les requêtes sont exécutées à l'aide temporaire de données. - -Pour une requête non distribuée, utilisez `IN` / `JOIN`. - -Soyez prudent lorsque vous utilisez des sous-requêtes dans le `IN` / `JOIN` clauses pour le traitement des requêtes distribuées. - -Regardons quelques exemples. Supposons que chaque serveur du cluster a un **local\_table**. Chaque serveur dispose également d'une **table distributed\_table** table avec le **Distribué** type, qui regarde tous les serveurs du cluster. - -Pour une requête à l' **table distributed\_table**, la requête sera envoyée à tous les serveurs distants et exécutée sur eux en utilisant le **local\_table**. - -Par exemple, la requête - -``` sql -SELECT uniq(UserID) FROM distributed_table -``` - -sera envoyé à tous les serveurs distants - -``` sql -SELECT uniq(UserID) FROM local_table -``` - -et l'exécuter sur chacun d'eux en parallèle, jusqu'à ce qu'il atteigne le stade où les résultats intermédiaires peuvent être combinés. Ensuite, les résultats intermédiaires seront retournés au demandeur de serveur et de fusion, et le résultat final sera envoyé au client. - -Examinons maintenant une requête avec IN: - -``` sql -SELECT uniq(UserID) FROM distributed_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM local_table WHERE CounterID = 34) -``` - -- Calcul de l'intersection des audiences de deux sites. - -Cette requête sera envoyée à tous les serveurs distants - -``` sql -SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM local_table WHERE CounterID = 34) -``` - -En d'autres termes, l'ensemble de données de la clause IN sera collecté sur chaque serveur indépendamment, uniquement à travers les données stockées localement sur chacun des serveurs. - -Cela fonctionnera correctement et de manière optimale si vous êtes prêt pour ce cas et que vous avez réparti les données entre les serveurs de cluster de telle sorte que les données d'un seul ID utilisateur résident entièrement sur un seul serveur. Dans ce cas, toutes les données nécessaires seront disponibles localement sur chaque serveur. Sinon, le résultat sera erroné. Nous nous référons à cette variation de la requête que “local IN”. - -Pour corriger le fonctionnement de la requête lorsque les données sont réparties aléatoirement sur les serveurs de cluster, vous pouvez spécifier **table distributed\_table** à l'intérieur d'une sous-requête. La requête ressemblerait à ceci: - -``` sql -SELECT uniq(UserID) FROM distributed_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM distributed_table WHERE CounterID = 34) -``` - -Cette requête sera envoyée à tous les serveurs distants - -``` sql -SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM distributed_table WHERE CounterID = 34) -``` - -La sous-requête commencera à s'exécuter sur chaque serveur distant. Étant donné que la sous-requête utilise une table distribuée, la sous-requête qui se trouve sur chaque serveur distant sera renvoyée à chaque serveur distant comme - -``` sql -SELECT UserID FROM local_table WHERE CounterID = 34 -``` - -Par exemple, si vous avez un cluster de 100 SERVEURS, l'exécution de la requête entière nécessitera 10 000 requêtes élémentaires, ce qui est généralement considéré comme inacceptable. - -Dans de tels cas, vous devez toujours utiliser GLOBAL IN au lieu de IN. Voyons comment cela fonctionne pour la requête - -``` sql -SELECT uniq(UserID) FROM distributed_table WHERE CounterID = 101500 AND UserID GLOBAL IN (SELECT UserID FROM distributed_table WHERE CounterID = 34) -``` - -Le serveur demandeur exécutera la sous requête - -``` sql -SELECT UserID FROM distributed_table WHERE CounterID = 34 -``` - -et le résultat sera mis dans une table temporaire en RAM. Ensuite, la demande sera envoyée à chaque serveur distant - -``` sql -SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID GLOBAL IN _data1 -``` - -et la table temporaire `_data1` sera envoyé à chaque serveur distant avec la requête (le nom de la table temporaire est défini par l'implémentation). - -Ceci est plus optimal que d'utiliser la normale dans. Cependant, gardez les points suivants à l'esprit: - -1. Lors de la création d'une table temporaire, les données ne sont pas uniques. Pour réduire le volume de données transmises sur le réseau, spécifiez DISTINCT dans la sous-requête. (Vous n'avez pas besoin de le faire pour un IN normal.) -2. La table temporaire sera envoyé à tous les serveurs distants. La Transmission ne tient pas compte de la topologie du réseau. Par exemple, si 10 serveurs distants résident dans un centre de données très distant par rapport au serveur demandeur, les données seront envoyées 10 fois sur le canal au centre de données distant. Essayez d'éviter les grands ensembles de données lorsque vous utilisez GLOBAL IN. -3. Lors de la transmission de données à des serveurs distants, les restrictions sur la bande passante réseau ne sont pas configurables. Vous pourriez surcharger le réseau. -4. Essayez de distribuer les données entre les serveurs afin que vous n'ayez pas besoin D'utiliser GLOBAL IN sur une base régulière. -5. Si vous devez utiliser GLOBAL in souvent, planifiez l'emplacement du cluster ClickHouse de sorte qu'un seul groupe de répliques ne réside pas dans plus d'un centre de données avec un réseau rapide entre eux, de sorte qu'une requête puisse être traitée entièrement dans un seul centre de données. - -Il est également judicieux de spécifier une table locale dans le `GLOBAL IN` clause, dans le cas où cette table locale est uniquement disponible sur le serveur demandeur et que vous souhaitez utiliser les données de celui-ci sur des serveurs distants. - -### Les Valeurs Extrêmes {#extreme-values} - -En plus des résultats, vous pouvez également obtenir des valeurs minimales et maximales pour les colonnes de résultats. Pour ce faire, définissez la **extrême** réglage sur 1. Les Minimums et les maximums sont calculés pour les types numériques, les dates et les dates avec des heures. Pour les autres colonnes, les valeurs par défaut sont sorties. - -An extra two rows are calculated – the minimums and maximums, respectively. These extra two rows are output in `JSON*`, `TabSeparated*`, et `Pretty*` [format](../../interfaces/formats.md), séparés des autres lignes. Ils ne sont pas Produits pour d'autres formats. - -Dans `JSON*` formats, les valeurs extrêmes sont sorties dans un ‘extremes’ champ. Dans `TabSeparated*` formats, la ligne vient après le résultat principal, et après ‘totals’ si elle est présente. Elle est précédée par une ligne vide (après les autres données). Dans `Pretty*` formats, la ligne est sortie comme une table séparée après le résultat principal, et après `totals` si elle est présente. - -Les valeurs extrêmes sont calculées pour les lignes avant `LIMIT` mais après `LIMIT BY`. Cependant, lors de l'utilisation de `LIMIT offset, size`, les lignes avant de les `offset` sont inclus dans `extremes`. Dans les requêtes de flux, le résultat peut également inclure un petit nombre de lignes qui ont traversé `LIMIT`. - -### Note {#notes} - -Le `GROUP BY` et `ORDER BY` les clauses ne supportent pas les arguments positionnels. Cela contredit MySQL, mais est conforme à SQL standard. -Exemple, `GROUP BY 1, 2` will be interpreted as grouping by constants (i.e. aggregation of all rows into one). - -Vous pouvez utiliser des synonymes (`AS` alias) dans n'importe quelle partie d'une requête. - -Vous pouvez mettre un astérisque dans quelque partie de la requête au lieu d'une expression. Lorsque la requête est analysée, l'astérisque est étendu à une liste de toutes les colonnes `MATERIALIZED` et `ALIAS` colonne). Il n'y a que quelques cas où l'utilisation d'un astérisque est justifiée: - -- Lors de la création d'un vidage de table. -- Pour les tables contenant seulement quelques colonnes, comme les tables système. -- Pour obtenir des informations sur ce que sont les colonnes dans une table. Dans ce cas, la valeur `LIMIT 1`. Mais il est préférable d'utiliser la `DESC TABLE` requête. -- Quand il y a une forte filtration sur un petit nombre de colonnes en utilisant `PREWHERE`. -- Dans les sous-requêtes (puisque les colonnes qui ne sont pas nécessaires pour la requête externe sont exclues des sous-requêtes). - -Dans tous les autres cas, nous ne recommandons pas d'utiliser l'astérisque, car il ne vous donne que les inconvénients d'un SGBD colonnaire au lieu des avantages. En d'autres termes, l'utilisation de l'astérisque n'est pas recommandée. - -[Article Original](https://clickhouse.tech/docs/en/query_language/select/) diff --git a/docs/fr/sql-reference/statements/select/array-join.md b/docs/fr/sql-reference/statements/select/array-join.md new file mode 100644 index 00000000000..07b27d5d16c --- /dev/null +++ b/docs/fr/sql-reference/statements/select/array-join.md @@ -0,0 +1,282 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +--- + +# Clause de jointure de tableau {#select-array-join-clause} + +C'est une opération courante pour les tables qui contiennent une colonne de tableau pour produire une nouvelle table qui a une colonne avec chaque élément de tableau individuel de cette colonne initiale, tandis que les valeurs des autres colonnes sont dupliquées. C'est le cas de fond de ce `ARRAY JOIN` la clause le fait. + +Son nom vient du fait qu'il peut être regardé comme l'exécution de `JOIN` avec un tableau ou une structure de données imbriquée. L'intention est similaire à la [arrayJoin](../../functions/array-join.md#functions_arrayjoin) fonction, mais la fonctionnalité de la clause est plus large. + +Syntaxe: + +``` sql +SELECT +FROM +[LEFT] ARRAY JOIN +[WHERE|PREWHERE ] +... +``` + +Vous ne pouvez en spécifier qu'un `ARRAY JOIN` la clause dans un `SELECT` requête. + +Types pris en charge de `ARRAY JOIN` sont énumérés ci-dessous: + +- `ARRAY JOIN` - Dans le cas de base, les tableaux vides ne sont pas inclus dans le résultat de `JOIN`. +- `LEFT ARRAY JOIN` - Le résultat de `JOIN` contient des lignes avec des tableaux vides. La valeur d'un tableau vide est définie sur la valeur par défaut pour le type d'élément de tableau (généralement 0, chaîne vide ou NULL). + +## Exemples de jointure de tableau de base {#basic-array-join-examples} + +Les exemples ci-dessous illustrent l'utilisation de la `ARRAY JOIN` et `LEFT ARRAY JOIN` clause. Créons une table avec un [Tableau](../../../sql-reference/data-types/array.md) tapez colonne et insérez des valeurs dedans: + +``` sql +CREATE TABLE arrays_test +( + s String, + arr Array(UInt8) +) ENGINE = Memory; + +INSERT INTO arrays_test +VALUES ('Hello', [1,2]), ('World', [3,4,5]), ('Goodbye', []); +``` + +``` text +┌─s───────────┬─arr─────┐ +│ Hello │ [1,2] │ +│ World │ [3,4,5] │ +│ Goodbye │ [] │ +└─────────────┴─────────┘ +``` + +L'exemple ci-dessous utilise la `ARRAY JOIN` clause: + +``` sql +SELECT s, arr +FROM arrays_test +ARRAY JOIN arr; +``` + +``` text +┌─s─────┬─arr─┐ +│ Hello │ 1 │ +│ Hello │ 2 │ +│ World │ 3 │ +│ World │ 4 │ +│ World │ 5 │ +└───────┴─────┘ +``` + +L'exemple suivant utilise l' `LEFT ARRAY JOIN` clause: + +``` sql +SELECT s, arr +FROM arrays_test +LEFT ARRAY JOIN arr; +``` + +``` text +┌─s───────────┬─arr─┐ +│ Hello │ 1 │ +│ Hello │ 2 │ +│ World │ 3 │ +│ World │ 4 │ +│ World │ 5 │ +│ Goodbye │ 0 │ +└─────────────┴─────┘ +``` + +## À L'Aide D'Alias {#using-aliases} + +Un alias peut être spécifié pour un tableau `ARRAY JOIN` clause. Dans ce cas, un élément de tableau peut être consulté par ce pseudonyme, mais le tableau lui-même est accessible par le nom d'origine. Exemple: + +``` sql +SELECT s, arr, a +FROM arrays_test +ARRAY JOIN arr AS a; +``` + +``` text +┌─s─────┬─arr─────┬─a─┐ +│ Hello │ [1,2] │ 1 │ +│ Hello │ [1,2] │ 2 │ +│ World │ [3,4,5] │ 3 │ +│ World │ [3,4,5] │ 4 │ +│ World │ [3,4,5] │ 5 │ +└───────┴─────────┴───┘ +``` + +En utilisant des alias, vous pouvez effectuer `ARRAY JOIN` avec un groupe externe. Exemple: + +``` sql +SELECT s, arr_external +FROM arrays_test +ARRAY JOIN [1, 2, 3] AS arr_external; +``` + +``` text +┌─s───────────┬─arr_external─┐ +│ Hello │ 1 │ +│ Hello │ 2 │ +│ Hello │ 3 │ +│ World │ 1 │ +│ World │ 2 │ +│ World │ 3 │ +│ Goodbye │ 1 │ +│ Goodbye │ 2 │ +│ Goodbye │ 3 │ +└─────────────┴──────────────┘ +``` + +Plusieurs tableaux peuvent être séparés par des virgules `ARRAY JOIN` clause. Dans ce cas, `JOIN` est effectuée avec eux simultanément (la somme directe, pas le produit cartésien). Notez que tous les tableaux doivent avoir la même taille. Exemple: + +``` sql +SELECT s, arr, a, num, mapped +FROM arrays_test +ARRAY JOIN arr AS a, arrayEnumerate(arr) AS num, arrayMap(x -> x + 1, arr) AS mapped; +``` + +``` text +┌─s─────┬─arr─────┬─a─┬─num─┬─mapped─┐ +│ Hello │ [1,2] │ 1 │ 1 │ 2 │ +│ Hello │ [1,2] │ 2 │ 2 │ 3 │ +│ World │ [3,4,5] │ 3 │ 1 │ 4 │ +│ World │ [3,4,5] │ 4 │ 2 │ 5 │ +│ World │ [3,4,5] │ 5 │ 3 │ 6 │ +└───────┴─────────┴───┴─────┴────────┘ +``` + +L'exemple ci-dessous utilise la [arrayEnumerate](../../../sql-reference/functions/array-functions.md#array_functions-arrayenumerate) fonction: + +``` sql +SELECT s, arr, a, num, arrayEnumerate(arr) +FROM arrays_test +ARRAY JOIN arr AS a, arrayEnumerate(arr) AS num; +``` + +``` text +┌─s─────┬─arr─────┬─a─┬─num─┬─arrayEnumerate(arr)─┐ +│ Hello │ [1,2] │ 1 │ 1 │ [1,2] │ +│ Hello │ [1,2] │ 2 │ 2 │ [1,2] │ +│ World │ [3,4,5] │ 3 │ 1 │ [1,2,3] │ +│ World │ [3,4,5] │ 4 │ 2 │ [1,2,3] │ +│ World │ [3,4,5] │ 5 │ 3 │ [1,2,3] │ +└───────┴─────────┴───┴─────┴─────────────────────┘ +``` + +## Jointure de tableau avec la Structure de données imbriquée {#array-join-with-nested-data-structure} + +`ARRAY JOIN` fonctionne également avec [structures de données imbriquées](../../../sql-reference/data-types/nested-data-structures/nested.md): + +``` sql +CREATE TABLE nested_test +( + s String, + nest Nested( + x UInt8, + y UInt32) +) ENGINE = Memory; + +INSERT INTO nested_test +VALUES ('Hello', [1,2], [10,20]), ('World', [3,4,5], [30,40,50]), ('Goodbye', [], []); +``` + +``` text +┌─s───────┬─nest.x──┬─nest.y─────┐ +│ Hello │ [1,2] │ [10,20] │ +│ World │ [3,4,5] │ [30,40,50] │ +│ Goodbye │ [] │ [] │ +└─────────┴─────────┴────────────┘ +``` + +``` sql +SELECT s, `nest.x`, `nest.y` +FROM nested_test +ARRAY JOIN nest; +``` + +``` text +┌─s─────┬─nest.x─┬─nest.y─┐ +│ Hello │ 1 │ 10 │ +│ Hello │ 2 │ 20 │ +│ World │ 3 │ 30 │ +│ World │ 4 │ 40 │ +│ World │ 5 │ 50 │ +└───────┴────────┴────────┘ +``` + +Lorsque vous spécifiez des noms de structures de données imbriquées dans `ARRAY JOIN` le sens est le même que `ARRAY JOIN` avec tous les éléments du tableau qui la compose. Des exemples sont énumérés ci-dessous: + +``` sql +SELECT s, `nest.x`, `nest.y` +FROM nested_test +ARRAY JOIN `nest.x`, `nest.y`; +``` + +``` text +┌─s─────┬─nest.x─┬─nest.y─┐ +│ Hello │ 1 │ 10 │ +│ Hello │ 2 │ 20 │ +│ World │ 3 │ 30 │ +│ World │ 4 │ 40 │ +│ World │ 5 │ 50 │ +└───────┴────────┴────────┘ +``` + +Cette variation a également du sens: + +``` sql +SELECT s, `nest.x`, `nest.y` +FROM nested_test +ARRAY JOIN `nest.x`; +``` + +``` text +┌─s─────┬─nest.x─┬─nest.y─────┐ +│ Hello │ 1 │ [10,20] │ +│ Hello │ 2 │ [10,20] │ +│ World │ 3 │ [30,40,50] │ +│ World │ 4 │ [30,40,50] │ +│ World │ 5 │ [30,40,50] │ +└───────┴────────┴────────────┘ +``` + +Un alias peut être utilisé pour une structure de données imbriquée, afin de sélectionner `JOIN` le résultat ou le tableau source. Exemple: + +``` sql +SELECT s, `n.x`, `n.y`, `nest.x`, `nest.y` +FROM nested_test +ARRAY JOIN nest AS n; +``` + +``` text +┌─s─────┬─n.x─┬─n.y─┬─nest.x──┬─nest.y─────┐ +│ Hello │ 1 │ 10 │ [1,2] │ [10,20] │ +│ Hello │ 2 │ 20 │ [1,2] │ [10,20] │ +│ World │ 3 │ 30 │ [3,4,5] │ [30,40,50] │ +│ World │ 4 │ 40 │ [3,4,5] │ [30,40,50] │ +│ World │ 5 │ 50 │ [3,4,5] │ [30,40,50] │ +└───────┴─────┴─────┴─────────┴────────────┘ +``` + +Exemple d'utilisation de l' [arrayEnumerate](../../../sql-reference/functions/array-functions.md#array_functions-arrayenumerate) fonction: + +``` sql +SELECT s, `n.x`, `n.y`, `nest.x`, `nest.y`, num +FROM nested_test +ARRAY JOIN nest AS n, arrayEnumerate(`nest.x`) AS num; +``` + +``` text +┌─s─────┬─n.x─┬─n.y─┬─nest.x──┬─nest.y─────┬─num─┐ +│ Hello │ 1 │ 10 │ [1,2] │ [10,20] │ 1 │ +│ Hello │ 2 │ 20 │ [1,2] │ [10,20] │ 2 │ +│ World │ 3 │ 30 │ [3,4,5] │ [30,40,50] │ 1 │ +│ World │ 4 │ 40 │ [3,4,5] │ [30,40,50] │ 2 │ +│ World │ 5 │ 50 │ [3,4,5] │ [30,40,50] │ 3 │ +└───────┴─────┴─────┴─────────┴────────────┴─────┘ +``` + +## Détails De Mise En Œuvre {#implementation-details} + +L'ordre d'exécution de la requête est optimisé lors de l'exécution `ARRAY JOIN`. Bien `ARRAY JOIN` doit toujours être spécifié avant l' [WHERE](where.md)/[PREWHERE](prewhere.md) dans une requête, techniquement, ils peuvent être exécutés dans n'importe quel ordre, sauf résultat de `ARRAY JOIN` est utilisé pour le filtrage. L'ordre de traitement est contrôlée par l'optimiseur de requête. diff --git a/docs/fr/sql-reference/statements/select/distinct.md b/docs/fr/sql-reference/statements/select/distinct.md new file mode 100644 index 00000000000..94552018c98 --- /dev/null +++ b/docs/fr/sql-reference/statements/select/distinct.md @@ -0,0 +1,63 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +--- + +# La Clause DISTINCT {#select-distinct} + +Si `SELECT DISTINCT` est spécifié, seules les lignes uniques restera un résultat de requête. Ainsi, une seule ligne restera hors de tous les ensembles de lignes entièrement correspondantes dans le résultat. + +## Le Traitement Null {#null-processing} + +`DISTINCT` fonctionne avec [NULL](../../syntax.md#null-literal) comme si `NULL` ont une valeur spécifique, et `NULL==NULL`. En d'autres termes, dans le `DISTINCT` résultats, différentes combinaisons avec `NULL` une fois seulement. Elle diffère de `NULL` traitement dans la plupart des autres contextes. + +## Alternative {#alternatives} + +Il est possible d'obtenir le même résultat en appliquant [GROUP BY](group-by.md) sur le même ensemble de valeurs, comme spécifié comme `SELECT` clause, sans utiliser de fonctions d'agrégation. Mais il y a peu de différences de `GROUP BY` approche: + +- `DISTINCT` peut être utilisé avec d' `GROUP BY`. +- Lorsque [ORDER BY](order-by.md) est omis et [LIMIT](limit.md) est définie, la requête s'arrête immédiatement après le nombre de lignes différentes, a été lu. +- Les blocs de données sont produits au fur et à mesure qu'ils sont traités, sans attendre que la requête entière se termine. + +## Limitation {#limitations} + +`DISTINCT` n'est pas pris en charge si `SELECT` a au moins une colonne de tableau. + +## Exemple {#examples} + +Clickhouse prend en charge l'utilisation du `DISTINCT` et `ORDER BY` clauses pour différentes colonnes dans une requête. Le `DISTINCT` la clause est exécutée avant la `ORDER BY` clause. + +Exemple de table: + +``` text +┌─a─┬─b─┐ +│ 2 │ 1 │ +│ 1 │ 2 │ +│ 3 │ 3 │ +│ 2 │ 4 │ +└───┴───┘ +``` + +Lors de la sélection de données avec le `SELECT DISTINCT a FROM t1 ORDER BY b ASC` requête, nous obtenons le résultat suivant: + +``` text +┌─a─┐ +│ 2 │ +│ 1 │ +│ 3 │ +└───┘ +``` + +Si nous changeons la direction de tri `SELECT DISTINCT a FROM t1 ORDER BY b DESC`, nous obtenons le résultat suivant: + +``` text +┌─a─┐ +│ 3 │ +│ 1 │ +│ 2 │ +└───┘ +``` + +Rangée `2, 4` a été coupé avant de les trier. + +Prenez en compte cette spécificité d'implémentation lors de la programmation des requêtes. diff --git a/docs/fr/sql-reference/statements/select/format.md b/docs/fr/sql-reference/statements/select/format.md new file mode 100644 index 00000000000..a88bb7831ba --- /dev/null +++ b/docs/fr/sql-reference/statements/select/format.md @@ -0,0 +1,18 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +--- + +# FORMAT de la Clause {#format-clause} + +Clickhouse prend en charge une large gamme de [formats de sérialisation](../../../interfaces/formats.md) qui peut être utilisé sur les résultats de la requête entre autres choses. Il existe plusieurs façons de choisir un format pour `SELECT` de sortie, l'un d'eux est de spécifier `FORMAT format` à la fin de la requête pour obtenir les données résultantes dans tout format spécifique. + +Un format spécifique peut être utilisé pour des raisons de commodité, d'intégration avec d'autres systèmes ou d'amélioration des performances. + +## Format Par Défaut {#default-format} + +Si l' `FORMAT` la clause est omise, le format par défaut est utilisé, ce qui dépend à la fois des paramètres et de l'interface utilisée pour accéder au serveur ClickHouse. Pour l' [Interface HTTP](../../../interfaces/http.md) et la [client de ligne de commande](../../../interfaces/cli.md) en mode batch, le format par défaut est `TabSeparated`. Pour le client de ligne de commande en mode interactif, le format par défaut est `PrettyCompact` (il produit des tables compactes lisibles par l'homme). + +## Détails De Mise En Œuvre {#implementation-details} + +Lors de l'utilisation du client de ligne de commande, les données sont toujours transmises sur le réseau dans un format efficace interne (`Native`). Le client interprète indépendamment le `FORMAT` clause de la requête et formate les données elles-mêmes (soulageant ainsi le réseau et le serveur de la charge supplémentaire). diff --git a/docs/fr/sql-reference/statements/select/from.md b/docs/fr/sql-reference/statements/select/from.md new file mode 100644 index 00000000000..964ffdd13fb --- /dev/null +++ b/docs/fr/sql-reference/statements/select/from.md @@ -0,0 +1,44 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +--- + +# De la Clause {#select-from} + +Le `FROM` clause spécifie la source à partir de laquelle lire les données: + +- [Table](../../../engines/table-engines/index.md) +- [Sous-requête](index.md) {##TODO: meilleur lien ##} +- [Fonction de Table](../../table-functions/index.md#table-functions) + +[JOIN](join.md) et [ARRAY JOIN](array-join.md) les clauses peuvent également être utilisées pour étendre la fonctionnalité de la `FROM` clause. + +Subquery est un autre `SELECT` requête qui peut être spécifié entre parenthèses à l'intérieur `FROM` clause. + +`FROM` la clause peut contenir plusieurs sources de données, séparées par des virgules, ce qui équivaut à effectuer [CROSS JOIN](join.md) sur eux. + +## Modificateur FINAL {#select-from-final} + +Lorsque `FINAL` est spécifié, ClickHouse fusionne complètement les données avant de renvoyer le résultat et effectue ainsi toutes les transformations de données qui se produisent lors des fusions pour le moteur de table donné. + +Il est applicable lors de la sélection de données à partir de tables qui utilisent [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md)-la famille de moteurs (à l'exception de `GraphiteMergeTree`). Également pris en charge pour: + +- [Répliqué](../../../engines/table-engines/mergetree-family/replication.md) les versions de `MergeTree` moteur. +- [Vue](../../../engines/table-engines/special/view.md), [Tampon](../../../engines/table-engines/special/buffer.md), [Distribué](../../../engines/table-engines/special/distributed.md), et [MaterializedView](../../../engines/table-engines/special/materializedview.md) moteurs qui fonctionnent sur d'autres moteurs, à condition qu'ils aient été créés sur `MergeTree`-tables de moteur. + +### Inconvénient {#drawbacks} + +Requêtes qui utilisent `FINAL` sont exécutés pas aussi vite que les requêtes similaires qui ne le font pas, car: + +- La requête est exécutée dans un seul thread et les données sont fusionnées lors de l'exécution de la requête. +- Les requêtes avec `FINAL` lire les colonnes de clé primaire en plus des colonnes spécifiées dans la requête. + +**Dans la plupart des cas, évitez d'utiliser `FINAL`.** L'approche commune consiste à utiliser différentes requêtes qui supposent les processus d'arrière-plan du `MergeTree` le moteur n'est pas encore arrivé et y faire face en appliquant l'agrégation (par exemple, pour éliminer les doublons). {##TODO: exemples ##} + +## Détails De Mise En Œuvre {#implementation-details} + +Si l' `FROM` la clause est omise, les données seront lues à partir `system.one` table. +Le `system.one` table contient exactement une ligne (cette table remplit le même but que la table double trouvée dans d'autres SGBD). + +Pour exécuter une requête, toutes les colonnes mentionnées dans la requête sont extraites de la table appropriée. Toutes les colonnes non nécessaires pour la requête externe sont rejetées des sous-requêtes. +Si une requête ne répertorie aucune colonne (par exemple, `SELECT count() FROM t`), une colonne est extraite de la table de toute façon (la plus petite est préférée), afin de calculer le nombre de lignes. diff --git a/docs/fr/sql-reference/statements/select/group-by.md b/docs/fr/sql-reference/statements/select/group-by.md new file mode 100644 index 00000000000..059e313fed7 --- /dev/null +++ b/docs/fr/sql-reference/statements/select/group-by.md @@ -0,0 +1,132 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +--- + +# Clause GROUP BY {#select-group-by-clause} + +`GROUP BY` la clause change le `SELECT` requête dans un mode d'agrégation, qui fonctionne comme suit: + +- `GROUP BY` clause contient une liste des expressions (ou une seule expression, qui est considéré comme la liste de longueur). Cette liste agit comme un “grouping key”, tandis que chaque expression individuelle sera appelée “key expressions”. +- Toutes les expressions dans le [SELECT](index.md), [HAVING](having.md), et [ORDER BY](order-by.md) clause **devoir** être calculé sur la base d'expressions clés **ou** sur [les fonctions d'agrégation](../../../sql-reference/aggregate-functions/index.md) sur les expressions non-clés (y compris les colonnes simples). En d'autres termes, chaque colonne sélectionnée dans la table doit être utilisée soit dans une expression de clé, soit dans une fonction d'agrégat, mais pas les deux. +- Résultat de l'agrégation de `SELECT` la requête contiendra autant de lignes qu'il y avait des valeurs uniques de “grouping key” dans la table source. Habituellement, cela réduit considérablement le nombre de lignes, souvent par ordre de grandeur, mais pas nécessairement: le nombre de lignes reste le même si tous “grouping key” les valeurs sont distinctes. + +!!! note "Note" + Il existe un moyen supplémentaire d'exécuter l'agrégation sur une table. Si une requête ne contient que des colonnes de table à l'intérieur des fonctions `GROUP BY clause` peut être omis, et l'agrégation par un ensemble vide de touches est supposé. Ces interrogations renvoient toujours exactement une ligne. + +## Le Traitement NULL {#null-processing} + +Pour le regroupement, ClickHouse interprète [NULL](../../syntax.md#null-literal) comme une valeur, et `NULL==NULL`. Elle diffère de `NULL` traitement dans la plupart des autres contextes. + +Voici un exemple pour montrer ce que cela signifie. + +Supposons que vous avez cette table: + +``` text +┌─x─┬────y─┐ +│ 1 │ 2 │ +│ 2 │ ᴺᵁᴸᴸ │ +│ 3 │ 2 │ +│ 3 │ 3 │ +│ 3 │ ᴺᵁᴸᴸ │ +└───┴──────┘ +``` + +Requête `SELECT sum(x), y FROM t_null_big GROUP BY y` résultats dans: + +``` text +┌─sum(x)─┬────y─┐ +│ 4 │ 2 │ +│ 3 │ 3 │ +│ 5 │ ᴺᵁᴸᴸ │ +└────────┴──────┘ +``` + +Vous pouvez voir que `GROUP BY` pour `y = NULL` résumer `x` comme si `NULL` a cette valeur. + +Si vous passez plusieurs clés `GROUP BY` le résultat vous donnera toutes les combinaisons de la sélection, comme si `NULL` ont une valeur spécifique. + +## Avec modificateur de totaux {#with-totals-modifier} + +Si l' `WITH TOTALS` modificateur est spécifié, une autre ligne sera calculée. Cette ligne aura des colonnes clés contenant des valeurs par défaut (zéros ou lignes vides), et des colonnes de fonctions d'agrégat avec les valeurs calculées sur toutes les lignes (le “total” valeur). + +Cette ligne supplémentaire est uniquement produite en `JSON*`, `TabSeparated*`, et `Pretty*` formats, séparément des autres lignes: + +- Dans `JSON*` formats, cette ligne est sortie en tant que distinct ‘totals’ champ. +- Dans `TabSeparated*` formats, la ligne vient après le résultat principal, précédé par une ligne vide (après les autres données). +- Dans `Pretty*` formats, la ligne est sortie comme une table séparée après le résultat principal. +- Dans les autres formats, il n'est pas disponible. + +`WITH TOTALS` peut être exécuté de différentes manières lorsqu'il est présent. Le comportement dépend de l' ‘totals\_mode’ paramètre. + +### Configuration Du Traitement Des Totaux {#configuring-totals-processing} + +Par défaut, `totals_mode = 'before_having'`. Dans ce cas, ‘totals’ est calculé sur toutes les lignes, y compris celles qui ne passent pas par `max_rows_to_group_by`. + +Les autres alternatives incluent uniquement les lignes qui passent à travers avoir dans ‘totals’, et se comporter différemment avec le réglage `max_rows_to_group_by` et `group_by_overflow_mode = 'any'`. + +`after_having_exclusive` – Don't include rows that didn't pass through `max_rows_to_group_by`. En d'autres termes, ‘totals’ aura moins ou le même nombre de lignes que si `max_rows_to_group_by` ont été omis. + +`after_having_inclusive` – Include all the rows that didn't pass through ‘max\_rows\_to\_group\_by’ dans ‘totals’. En d'autres termes, ‘totals’ aura plus ou le même nombre de lignes que si `max_rows_to_group_by` ont été omis. + +`after_having_auto` – Count the number of rows that passed through HAVING. If it is more than a certain amount (by default, 50%), include all the rows that didn't pass through ‘max\_rows\_to\_group\_by’ dans ‘totals’. Sinon, ne pas les inclure. + +`totals_auto_threshold` – By default, 0.5. The coefficient for `after_having_auto`. + +Si `max_rows_to_group_by` et `group_by_overflow_mode = 'any'` ne sont pas utilisés, toutes les variations de `after_having` sont les mêmes, et vous pouvez utiliser l'un d'eux (par exemple, `after_having_auto`). + +Vous pouvez utiliser avec les totaux dans les sous-requêtes, y compris les sous-requêtes dans la clause JOIN (dans ce cas, les valeurs totales respectives sont combinées). + +## Exemple {#examples} + +Exemple: + +``` sql +SELECT + count(), + median(FetchTiming > 60 ? 60 : FetchTiming), + count() - sum(Refresh) +FROM hits +``` + +Cependant, contrairement au SQL standard, si la table n'a pas de lignes (soit il n'y en a pas du tout, soit il n'y en a pas après avoir utilisé WHERE to filter), un résultat vide est renvoyé, et non le résultat d'une des lignes contenant les valeurs initiales des fonctions d'agrégat. + +Contrairement à MySQL (et conforme à SQL standard), vous ne pouvez pas obtenir une valeur d'une colonne qui n'est pas dans une fonction clé ou agrégée (sauf les expressions constantes). Pour contourner ce problème, vous pouvez utiliser le ‘any’ fonction d'agrégation (récupère la première valeur rencontrée) ou ‘min/max’. + +Exemple: + +``` sql +SELECT + domainWithoutWWW(URL) AS domain, + count(), + any(Title) AS title -- getting the first occurred page header for each domain. +FROM hits +GROUP BY domain +``` + +Pour chaque valeur de clé différente rencontrée, GROUP BY calcule un ensemble de valeurs de fonction d'agrégation. + +GROUP BY n'est pas pris en charge pour les colonnes de tableau. + +Une constante ne peut pas être spécifiée comme arguments pour les fonctions d'agrégation. Exemple: somme(1). Au lieu de cela, vous pouvez vous débarrasser de la constante. Exemple: `count()`. + +## Détails De Mise En Œuvre {#implementation-details} + +L'agrégation est l'une des caractéristiques les plus importantes d'un SGBD orienté colonne, et donc son implémentation est l'une des parties les plus optimisées de ClickHouse. Par défaut, l'agrégation se fait en mémoire à l'aide d'une table de hachage. Il a plus de 40 spécialisations qui sont choisies automatiquement en fonction de “grouping key” types de données. + +### Groupe par dans la mémoire externe {#select-group-by-in-external-memory} + +Vous pouvez activer le dumping des données temporaires sur le disque pour limiter l'utilisation de la mémoire pendant `GROUP BY`. +Le [max\_bytes\_before\_external\_group\_by](../../../operations/settings/settings.md#settings-max_bytes_before_external_group_by) réglage détermine le seuil de consommation de RAM pour le dumping `GROUP BY` données temporaires dans le système de fichiers. Si elle est définie sur 0 (valeur par défaut), elle est désactivée. + +Lors de l'utilisation de `max_bytes_before_external_group_by`, nous vous recommandons de définir `max_memory_usage` environ deux fois plus élevé. Ceci est nécessaire car il y a deux étapes à l'agrégation: la lecture des données et la formation des données intermédiaires (1) et la fusion des données intermédiaires (2). Le Dumping des données dans le système de fichiers ne peut se produire qu'au cours de l'étape 1. Si les données temporaires n'ont pas été vidées, l'étape 2 peut nécessiter jusqu'à la même quantité de mémoire qu'à l'étape 1. + +Par exemple, si [max\_memory\_usage](../../../operations/settings/settings.md#settings_max_memory_usage) a été défini sur 10000000000 et que vous souhaitez utiliser l'agrégation externe, il est logique de définir `max_bytes_before_external_group_by` à 10000000000, et `max_memory_usage` à 20000000000. Lorsque l'agrégation externe est déclenchée (s'il y a eu au moins un vidage de données temporaires), la consommation maximale de RAM n'est que légèrement supérieure à `max_bytes_before_external_group_by`. + +Avec le traitement des requêtes distribuées, l'agrégation externe est effectuée sur des serveurs distants. Pour que le serveur demandeur n'utilise qu'une petite quantité de RAM, définissez `distributed_aggregation_memory_efficient` 1. + +Lors de la fusion de données vidées sur le disque, ainsi que lors de la fusion des résultats de serveurs distants lorsque `distributed_aggregation_memory_efficient` paramètre est activé, consomme jusqu'à `1/256 * the_number_of_threads` à partir de la quantité totale de mémoire RAM. + +Lorsque l'agrégation externe est activée, s'il y a moins de `max_bytes_before_external_group_by` of data (i.e. data was not flushed), the query runs just as fast as without external aggregation. If any temporary data was flushed, the run time will be several times longer (approximately three times). + +Si vous avez un [ORDER BY](order-by.md) avec un [LIMIT](limit.md) après `GROUP BY` puis la quantité de RAM dépend de la quantité de données dans `LIMIT`, pas dans l'ensemble de la table. Mais si l' `ORDER BY` n'a pas `LIMIT`, n'oubliez pas d'activer externe de tri (`max_bytes_before_external_sort`). diff --git a/docs/fr/sql-reference/statements/select/having.md b/docs/fr/sql-reference/statements/select/having.md new file mode 100644 index 00000000000..9425830c3d4 --- /dev/null +++ b/docs/fr/sql-reference/statements/select/having.md @@ -0,0 +1,14 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +--- + +# Clause HAVING {#having-clause} + +Permet de filtrer les résultats d'agrégation produits par [GROUP BY](group-by.md). Il est similaire à la [WHERE](where.md) la clause, mais la différence est que `WHERE` est effectuée avant l'agrégation, tandis que `HAVING` est effectué d'après elle. + +Il est possible de référencer les résultats d'agrégation à partir de `SELECT` la clause dans `HAVING` clause par leur alias. Alternativement, `HAVING` clause peut filtrer sur les résultats d'agrégats supplémentaires qui ne sont pas retournés dans les résultats de la requête. + +## Limitation {#limitations} + +`HAVING` ne peut pas être utilisé si le regroupement n'est pas effectuée. Utiliser `WHERE` plutôt. diff --git a/docs/fr/sql-reference/statements/select/index.md b/docs/fr/sql-reference/statements/select/index.md new file mode 100644 index 00000000000..0a3e6116d08 --- /dev/null +++ b/docs/fr/sql-reference/statements/select/index.md @@ -0,0 +1,158 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_priority: 33 +toc_title: SELECT +--- + +# Sélectionnez la syntaxe des requêtes {#select-queries-syntax} + +`SELECT` effectue la récupération des données. + +``` sql +[WITH expr_list|(subquery)] +SELECT [DISTINCT] expr_list +[FROM [db.]table | (subquery) | table_function] [FINAL] +[SAMPLE sample_coeff] +[ARRAY JOIN ...] +[GLOBAL] [ANY|ALL] [INNER|LEFT|RIGHT|FULL|CROSS] [OUTER] JOIN (subquery)|table USING columns_list +[PREWHERE expr] +[WHERE expr] +[GROUP BY expr_list] [WITH TOTALS] +[HAVING expr] +[ORDER BY expr_list] +[LIMIT [offset_value, ]n BY columns] +[LIMIT [n, ]m] +[UNION ALL ...] +[INTO OUTFILE filename] +[FORMAT format] +``` + +Toutes les clauses sont facultatives, à l'exception de la liste d'expressions requise immédiatement après `SELECT` qui est abordée plus en détail [dessous](#select-clause). + +Spécificités de chaque clause facultative, sont couverts dans des sections distinctes, qui sont énumérés dans le même ordre qu'elles sont exécutées: + +- [AVEC la clause](with.md) +- [La clause DISTINCT](distinct.md) +- [De la clause](from.md) +- [Exemple de clause](sample.md) +- [Clause de JOINTURE](join.md) +- [Clause PREWHERE](prewhere.md) +- [Clause where](where.md) +- [Groupe par clause](group-by.md) +- [Limite par clause](limit-by.md) +- [Clause HAVING](having.md) +- [Clause SELECT](#select-clause) +- [Clause LIMIT](limit.md) +- [Clause UNION ALL](union-all.md) + +## Clause SELECT {#select-clause} + +[Expression](../../syntax.md#syntax-expressions) spécifié dans le `SELECT` clause sont calculés après toutes les opérations dans les clauses décrites ci-dessus sont terminés. Ces expressions fonctionnent comme si elles s'appliquaient à des lignes séparées dans le résultat. Si les expressions dans le `SELECT` la clause contient des fonctions d'agrégation, puis clickhouse traite les fonctions d'agrégation et les expressions utilisées [GROUP BY](group-by.md) agrégation. + +Si vous souhaitez inclure toutes les colonnes dans le résultat, utilisez l'astérisque (`*`) symbole. Exemple, `SELECT * FROM ...`. + +Pour correspondre à certaines colonnes dans le résultat avec un [re2](https://en.wikipedia.org/wiki/RE2_(software)) expression régulière, vous pouvez utiliser le `COLUMNS` expression. + +``` sql +COLUMNS('regexp') +``` + +Par exemple, considérez le tableau: + +``` sql +CREATE TABLE default.col_names (aa Int8, ab Int8, bc Int8) ENGINE = TinyLog +``` + +La requête suivante sélectionne les données de toutes les colonnes contenant les `a` symbole dans leur nom. + +``` sql +SELECT COLUMNS('a') FROM col_names +``` + +``` text +┌─aa─┬─ab─┐ +│ 1 │ 1 │ +└────┴────┘ +``` + +Les colonnes sélectionnées sont retournés pas dans l'ordre alphabétique. + +Vous pouvez utiliser plusieurs `COLUMNS` expressions dans une requête et leur appliquer des fonctions. + +Exemple: + +``` sql +SELECT COLUMNS('a'), COLUMNS('c'), toTypeName(COLUMNS('c')) FROM col_names +``` + +``` text +┌─aa─┬─ab─┬─bc─┬─toTypeName(bc)─┐ +│ 1 │ 1 │ 1 │ Int8 │ +└────┴────┴────┴────────────────┘ +``` + +Chaque colonne renvoyée par le `COLUMNS` expression est passée à la fonction en tant qu'argument séparé. Vous pouvez également passer d'autres arguments à la fonction si elle les supporte. Soyez prudent lorsque vous utilisez des fonctions. Si une fonction ne prend pas en charge le nombre d'arguments que vous lui avez transmis, ClickHouse lève une exception. + +Exemple: + +``` sql +SELECT COLUMNS('a') + COLUMNS('c') FROM col_names +``` + +``` text +Received exception from server (version 19.14.1): +Code: 42. DB::Exception: Received from localhost:9000. DB::Exception: Number of arguments for function plus doesn't match: passed 3, should be 2. +``` + +Dans cet exemple, `COLUMNS('a')` retourne deux colonnes: `aa` et `ab`. `COLUMNS('c')` renvoie la `bc` colonne. Le `+` l'opérateur ne peut pas s'appliquer à 3 arguments, donc ClickHouse lève une exception avec le message pertinent. + +Colonnes qui correspondent à la `COLUMNS` l'expression peut avoir différents types de données. Si `COLUMNS` ne correspond à aucune colonne et est la seule expression dans `SELECT`, ClickHouse lance une exception. + +### Astérisque {#asterisk} + +Vous pouvez mettre un astérisque dans quelque partie de la requête au lieu d'une expression. Lorsque la requête est analysée, l'astérisque est étendu à une liste de toutes les colonnes `MATERIALIZED` et `ALIAS` colonne). Il n'y a que quelques cas où l'utilisation d'un astérisque est justifiée: + +- Lors de la création d'un vidage de table. +- Pour les tables contenant seulement quelques colonnes, comme les tables système. +- Pour obtenir des informations sur ce que sont les colonnes dans une table. Dans ce cas, la valeur `LIMIT 1`. Mais il est préférable d'utiliser la `DESC TABLE` requête. +- Quand il y a une forte filtration sur un petit nombre de colonnes en utilisant `PREWHERE`. +- Dans les sous-requêtes (puisque les colonnes qui ne sont pas nécessaires pour la requête externe sont exclues des sous-requêtes). + +Dans tous les autres cas, nous ne recommandons pas d'utiliser l'astérisque, car il ne vous donne que les inconvénients d'un SGBD colonnaire au lieu des avantages. En d'autres termes, l'utilisation de l'astérisque n'est pas recommandée. + +### Les Valeurs Extrêmes {#extreme-values} + +En plus des résultats, vous pouvez également obtenir des valeurs minimales et maximales pour les colonnes de résultats. Pour ce faire, définissez la **extrême** réglage sur 1. Les Minimums et les maximums sont calculés pour les types numériques, les dates et les dates avec des heures. Pour les autres colonnes, les valeurs par défaut sont sorties. + +An extra two rows are calculated – the minimums and maximums, respectively. These extra two rows are output in `JSON*`, `TabSeparated*`, et `Pretty*` [format](../../../interfaces/formats.md), séparés des autres lignes. Ils ne sont pas Produits pour d'autres formats. + +Dans `JSON*` formats, les valeurs extrêmes sont sorties dans un ‘extremes’ champ. Dans `TabSeparated*` formats, la ligne vient après le résultat principal, et après ‘totals’ si elle est présente. Elle est précédée par une ligne vide (après les autres données). Dans `Pretty*` formats, la ligne est sortie comme une table séparée après le résultat principal, et après `totals` si elle est présente. + +Les valeurs extrêmes sont calculées pour les lignes avant `LIMIT` mais après `LIMIT BY`. Cependant, lors de l'utilisation de `LIMIT offset, size`, les lignes avant de les `offset` sont inclus dans `extremes`. Dans les requêtes de flux, le résultat peut également inclure un petit nombre de lignes qui ont traversé `LIMIT`. + +### Note {#notes} + +Vous pouvez utiliser des synonymes (`AS` alias) dans n'importe quelle partie d'une requête. + +Le `GROUP BY` et `ORDER BY` les clauses ne supportent pas les arguments positionnels. Cela contredit MySQL, mais est conforme à SQL standard. Exemple, `GROUP BY 1, 2` will be interpreted as grouping by constants (i.e. aggregation of all rows into one). + +## Détails De Mise En Œuvre {#implementation-details} + +Si la requête omet le `DISTINCT`, `GROUP BY` et `ORDER BY` les clauses et les `IN` et `JOIN` sous-requêtes, la requête sera complètement traitée en flux, en utilisant O (1) quantité de RAM. Sinon, la requête peut consommer beaucoup de RAM si les restrictions appropriées ne sont pas spécifiées: + +- `max_memory_usage` +- `max_rows_to_group_by` +- `max_rows_to_sort` +- `max_rows_in_distinct` +- `max_bytes_in_distinct` +- `max_rows_in_set` +- `max_bytes_in_set` +- `max_rows_in_join` +- `max_bytes_in_join` +- `max_bytes_before_external_sort` +- `max_bytes_before_external_group_by` + +Pour plus d'informations, consultez la section “Settings”. Il est possible d'utiliser le tri externe (sauvegarde des tables temporaires sur un disque) et l'agrégation externe. + +{## [Article Original](https://clickhouse.tech/docs/en/sql-reference/statements/select/) ##} diff --git a/docs/fr/sql-reference/statements/select/into-outfile.md b/docs/fr/sql-reference/statements/select/into-outfile.md new file mode 100644 index 00000000000..0150de7cb97 --- /dev/null +++ b/docs/fr/sql-reference/statements/select/into-outfile.md @@ -0,0 +1,14 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +--- + +# Dans OUTFILE Clause {#into-outfile-clause} + +Ajouter l' `INTO OUTFILE filename` clause (où filename est un littéral de chaîne) pour `SELECT query` pour rediriger sa sortie vers le fichier spécifié côté client. + +## Détails De Mise En Œuvre {#implementation-details} + +- Cette fonctionnalité est disponible dans les [client de ligne de commande](../../../interfaces/cli.md) et [clickhouse-local](../../../operations/utilities/clickhouse-local.md). Ainsi, une requête envoyée par [Interface HTTP](../../../interfaces/http.md) va échouer. +- La requête échouera si un fichier portant le même nom existe déjà. +- Défaut [le format de sortie](../../../interfaces/formats.md) être `TabSeparated` (comme dans le mode batch client en ligne de commande). diff --git a/docs/fr/sql-reference/statements/select/join.md b/docs/fr/sql-reference/statements/select/join.md new file mode 100644 index 00000000000..d802f68e4bf --- /dev/null +++ b/docs/fr/sql-reference/statements/select/join.md @@ -0,0 +1,191 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +--- + +# Clause de JOINTURE {#select-join} + +Join produit une nouvelle table en combinant des colonnes d'une ou plusieurs tables en utilisant des valeurs communes à chacune. C'est une opération courante dans les bases de données avec support SQL, ce qui correspond à [l'algèbre relationnelle](https://en.wikipedia.org/wiki/Relational_algebra#Joins_and_join-like_operators) rejoindre. Le cas particulier d'une jointure de table est souvent appelé “self-join”. + +Syntaxe: + +``` sql +SELECT +FROM +[GLOBAL] [ANY|ALL|ASOF] [INNER|LEFT|RIGHT|FULL|CROSS] [OUTER|SEMI|ANTI] JOIN +(ON )|(USING ) ... +``` + +Les Expressions de `ON` clause et colonnes de `USING` clause sont appelés “join keys”. Sauf indication contraire, joindre un produit [Produit cartésien](https://en.wikipedia.org/wiki/Cartesian_product) des lignes, avec correspondance “join keys”, ce qui pourrait produire des résultats avec beaucoup plus de lignes que les tables source. + +## Types de jointure pris en charge {#select-join-types} + +Tous les standard [SQL JOIN](https://en.wikipedia.org/wiki/Join_(SQL)) les types sont pris en charge: + +- `INNER JOIN`, seules les lignes correspondantes sont retournés. +- `LEFT OUTER JOIN`, les lignes non correspondantes de la table de gauche sont retournées en plus des lignes correspondantes. +- `RIGHT OUTER JOIN`, les lignes non correspondantes de la table de gauche sont retournées en plus des lignes correspondantes. +- `FULL OUTER JOIN`, les lignes non correspondantes des deux tables sont renvoyées en plus des lignes correspondantes. +- `CROSS JOIN`, produit le produit cartésien des tables entières, “join keys” être **pas** défini. + +`JOIN` sans type spécifié implique `INNER`. Mot `OUTER` peut les oublier. Syntaxe Alternative pour `CROSS JOIN` spécifie plusieurs tables dans [De la clause](from.md) séparés par des virgules. + +Autres types de jointure disponibles dans ClickHouse: + +- `LEFT SEMI JOIN` et `RIGHT SEMI JOIN` une liste blanche sur “join keys”, sans produire un produit cartésien. +- `LEFT ANTI JOIN` et `RIGHT ANTI JOIN` une liste noire sur “join keys”, sans produire un produit cartésien. + +## Rigueur {#select-join-strictness} + +Modifie la façon dont la correspondance par “join keys” est effectué + +- `ALL` — The standard `JOIN` comportement en SQL comme décrit ci-dessus. Défaut. +- `ANY` — Partially (for opposite side of `LEFT` et `RIGHT`) ou complètement (pour `INNER` et `FULL`) désactive le produit cartésien de la norme `JOIN` type. +- `ASOF` — For joining sequences with a non-exact match. `ASOF JOIN` l'utilisation est décrite ci-dessous. + +!!! note "Note" + La valeur de rigueur par défaut peut être remplacée à l'aide [join\_default\_strictness](../../../operations/settings/settings.md#settings-join_default_strictness) paramètre. + +### ASOF joindre L'utilisation {#asof-join-usage} + +`ASOF JOIN` est utile lorsque vous devez joindre des enregistrements qui n'ont pas de correspondance exacte. + +Tables pour `ASOF JOIN` doit avoir une colonne de séquence ordonnée. Cette colonne ne peut pas être seule dans une table et doit être l'un des types de données: `UInt32`, `UInt64`, `Float32`, `Float64`, `Date`, et `DateTime`. + +Syntaxe `ASOF JOIN ... ON`: + +``` sql +SELECT expressions_list +FROM table_1 +ASOF LEFT JOIN table_2 +ON equi_cond AND closest_match_cond +``` + +Vous pouvez utiliser n'importe quel nombre de conditions d'égalité et exactement une condition de correspondance la plus proche. Exemple, `SELECT count() FROM table_1 ASOF LEFT JOIN table_2 ON table_1.a == table_2.b AND table_2.t <= table_1.t`. + +Conditions prises en charge pour la correspondance la plus proche: `>`, `>=`, `<`, `<=`. + +Syntaxe `ASOF JOIN ... USING`: + +``` sql +SELECT expressions_list +FROM table_1 +ASOF JOIN table_2 +USING (equi_column1, ... equi_columnN, asof_column) +``` + +`ASOF JOIN` utiliser `equi_columnX` pour rejoindre sur l'égalité et `asof_column` pour rejoindre le match le plus proche avec le `table_1.asof_column >= table_2.asof_column` condition. Le `asof_column` colonne toujours la dernière dans le `USING` clause. + +Par exemple, considérez les tableaux suivants: + + table_1 table_2 + event | ev_time | user_id event | ev_time | user_id + ----------|---------|---------- ----------|---------|---------- + ... ... + event_1_1 | 12:00 | 42 event_2_1 | 11:59 | 42 + ... event_2_2 | 12:30 | 42 + event_1_2 | 13:00 | 42 event_2_3 | 13:00 | 42 + ... ... + +`ASOF JOIN` peut prendre la date d'un événement utilisateur de `table_1` et trouver un événement dans `table_2` où le timestamp est plus proche de l'horodatage de l'événement à partir de `table_1` correspondant à la condition de correspondance la plus proche. Les valeurs d'horodatage égales sont les plus proches si elles sont disponibles. Ici, l' `user_id` la colonne peut être utilisée pour joindre sur l'égalité et le `ev_time` la colonne peut être utilisée pour se joindre à la correspondance la plus proche. Dans notre exemple, `event_1_1` peut être jointe à `event_2_1` et `event_1_2` peut être jointe à `event_2_3`, mais `event_2_2` ne peut pas être rejoint. + +!!! note "Note" + `ASOF` jointure est **pas** pris en charge dans le [Rejoindre](../../../engines/table-engines/special/join.md) tableau moteur. + +## Jointure Distribuée {#global-join} + +Il existe deux façons d'exécuter join impliquant des tables distribuées: + +- Lors de l'utilisation normale `JOIN` la requête est envoyée aux serveurs distants. Les sous-requêtes sont exécutées sur chacune d'elles afin de créer la bonne table, et la jointure est effectuée avec cette table. En d'autres termes, la table de droite est formée sur chaque serveur séparément. +- Lors de l'utilisation de `GLOBAL ... JOIN`, d'abord le serveur demandeur exécute une sous-requête pour calculer la bonne table. Cette table temporaire est transmise à chaque serveur distant, et les requêtes sont exécutées sur eux en utilisant les données temporaires qui ont été transmises. + +Soyez prudent lorsque vous utilisez `GLOBAL`. Pour plus d'informations, voir le [Sous-requêtes distribuées](../../operators/in.md#select-distributed-subqueries) section. + +## Recommandations D'Utilisation {#usage-recommendations} + +### Traitement des cellules vides ou nulles {#processing-of-empty-or-null-cells} + +Lors de la jonction de tables, les cellules vides peuvent apparaître. Paramètre [join\_use\_nulls](../../../operations/settings/settings.md#join_use_nulls) définir comment clickhouse remplit ces cellules. + +Si l' `JOIN` les touches sont [Nullable](../../data-types/nullable.md) champs, les lignes où au moins une des clés a la valeur [NULL](../../../sql-reference/syntax.md#null-literal) ne sont pas jointes. + +### Syntaxe {#syntax} + +Les colonnes spécifiées dans `USING` doit avoir les mêmes noms dans les deux sous-requêtes, et les autres colonnes doivent être nommées différemment. Vous pouvez utiliser des alias pour les noms des colonnes dans les sous-requêtes. + +Le `USING` clause spécifie une ou plusieurs colonnes de jointure, qui établit l'égalité de ces colonnes. La liste des colonnes est définie sans crochets. Les conditions de jointure plus complexes ne sont pas prises en charge. + +### Limitations De Syntaxe {#syntax-limitations} + +Pour plusieurs `JOIN` clauses dans un seul `SELECT` requête: + +- Prendre toutes les colonnes via `*` n'est disponible que si les tables sont jointes, pas les sous-requêtes. +- Le `PREWHERE` la clause n'est pas disponible. + +Pour `ON`, `WHERE`, et `GROUP BY` clause: + +- Les expressions arbitraires ne peuvent pas être utilisées dans `ON`, `WHERE`, et `GROUP BY` mais vous pouvez définir une expression dans un `SELECT` clause et ensuite l'utiliser dans ces clauses via un alias. + +### Performance {#performance} + +Lors de l'exécution d'un `JOIN`, il n'y a pas d'optimisation de la commande d'exécution par rapport aux autres stades de la requête. La jointure (une recherche dans la table de droite) est exécutée avant de filtrer `WHERE` et avant l'agrégation. + +Chaque fois qu'une requête est exécutée avec la même `JOIN`, la sous-requête est exécutée à nouveau car le résultat n'est pas mis en cache. Pour éviter cela, utilisez la spéciale [Rejoindre](../../../engines/table-engines/special/join.md) table engine, qui est un tableau préparé pour l'assemblage qui est toujours en RAM. + +Dans certains cas, il est plus efficace d'utiliser [IN](../../operators/in.md) plutôt `JOIN`. + +Si vous avez besoin d'un `JOIN` pour se joindre à des tables de dimension (ce sont des tables relativement petites qui contiennent des propriétés de dimension, telles que des noms pour des campagnes publicitaires), un `JOIN` peut-être pas très pratique en raison du fait que la bonne table est ré-accédée pour chaque requête. Pour de tels cas, il y a un “external dictionaries” la fonctionnalité que vous devez utiliser à la place de `JOIN`. Pour plus d'informations, voir le [Dictionnaires externes](../../dictionaries/external-dictionaries/external-dicts.md) section. + +### Limitations De Mémoire {#memory-limitations} + +Par défaut, ClickHouse utilise [jointure de hachage](https://en.wikipedia.org/wiki/Hash_join) algorithme. ClickHouse prend le `` et crée une table de hachage pour cela dans la RAM. Après un certain seuil de consommation de mémoire, ClickHouse revient à fusionner l'algorithme de jointure. + +Si vous devez restreindre la consommation de mémoire de l'opération join utilisez les paramètres suivants: + +- [max\_rows\_in\_join](../../../operations/settings/query-complexity.md#settings-max_rows_in_join) — Limits number of rows in the hash table. +- [max\_bytes\_in\_join](../../../operations/settings/query-complexity.md#settings-max_bytes_in_join) — Limits size of the hash table. + +Lorsque l'une de ces limites est atteinte, ClickHouse agit comme [join\_overflow\_mode](../../../operations/settings/query-complexity.md#settings-join_overflow_mode) réglage des instructions. + +## Exemple {#examples} + +Exemple: + +``` sql +SELECT + CounterID, + hits, + visits +FROM +( + SELECT + CounterID, + count() AS hits + FROM test.hits + GROUP BY CounterID +) ANY LEFT JOIN +( + SELECT + CounterID, + sum(Sign) AS visits + FROM test.visits + GROUP BY CounterID +) USING CounterID +ORDER BY hits DESC +LIMIT 10 +``` + +``` text +┌─CounterID─┬───hits─┬─visits─┐ +│ 1143050 │ 523264 │ 13665 │ +│ 731962 │ 475698 │ 102716 │ +│ 722545 │ 337212 │ 108187 │ +│ 722889 │ 252197 │ 10547 │ +│ 2237260 │ 196036 │ 9522 │ +│ 23057320 │ 147211 │ 7689 │ +│ 722818 │ 90109 │ 17847 │ +│ 48221 │ 85379 │ 4652 │ +│ 19762435 │ 77807 │ 7026 │ +│ 722884 │ 77492 │ 11056 │ +└───────────┴────────┴────────┘ +``` diff --git a/docs/fr/sql-reference/statements/select/limit-by.md b/docs/fr/sql-reference/statements/select/limit-by.md new file mode 100644 index 00000000000..4d1bd766ef1 --- /dev/null +++ b/docs/fr/sql-reference/statements/select/limit-by.md @@ -0,0 +1,71 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +--- + +# Limite par Clause {#limit-by-clause} + +Une requête avec l' `LIMIT n BY expressions` la clause sélectionne le premier `n` lignes pour chaque valeur distincte de `expressions`. La clé pour `LIMIT BY` peut contenir n'importe quel nombre de [expression](../../syntax.md#syntax-expressions). + +ClickHouse prend en charge les variantes de syntaxe suivantes: + +- `LIMIT [offset_value, ]n BY expressions` +- `LIMIT n OFFSET offset_value BY expressions` + +Pendant le traitement de la requête, ClickHouse sélectionne les données classées par clé de tri. La clé de tri est définie explicitement à l'aide [ORDER BY](order-by.md) clause ou implicitement en tant que propriété du moteur de table. Puis clickhouse s'applique `LIMIT n BY expressions` et renvoie le premier `n` lignes pour chaque combinaison distincte de `expressions`. Si `OFFSET` est spécifié, puis pour chaque bloc de données qui appartient à une combinaison particulière de `expressions`, Clickhouse saute `offset_value` nombre de lignes depuis le début du bloc et renvoie un maximum de `n` les lignes en conséquence. Si `offset_value` est plus grand que le nombre de lignes dans le bloc de données, ClickHouse renvoie zéro lignes du bloc. + +!!! note "Note" + `LIMIT BY` n'est pas liée à [LIMIT](limit.md). Ils peuvent tous deux être utilisés dans la même requête. + +## Exemple {#examples} + +Exemple de table: + +``` sql +CREATE TABLE limit_by(id Int, val Int) ENGINE = Memory; +INSERT INTO limit_by VALUES (1, 10), (1, 11), (1, 12), (2, 20), (2, 21); +``` + +Requête: + +``` sql +SELECT * FROM limit_by ORDER BY id, val LIMIT 2 BY id +``` + +``` text +┌─id─┬─val─┐ +│ 1 │ 10 │ +│ 1 │ 11 │ +│ 2 │ 20 │ +│ 2 │ 21 │ +└────┴─────┘ +``` + +``` sql +SELECT * FROM limit_by ORDER BY id, val LIMIT 1, 2 BY id +``` + +``` text +┌─id─┬─val─┐ +│ 1 │ 11 │ +│ 1 │ 12 │ +│ 2 │ 21 │ +└────┴─────┘ +``` + +Le `SELECT * FROM limit_by ORDER BY id, val LIMIT 2 OFFSET 1 BY id` requête renvoie le même résultat. + +La requête suivante renvoie les 5 principaux référents pour chaque `domain, device_type` paire avec un maximum de 100 lignes au total (`LIMIT n BY + LIMIT`). + +``` sql +SELECT + domainWithoutWWW(URL) AS domain, + domainWithoutWWW(REFERRER_URL) AS referrer, + device_type, + count() cnt +FROM hits +GROUP BY domain, referrer, device_type +ORDER BY cnt DESC +LIMIT 5 BY domain, device_type +LIMIT 100 +``` diff --git a/docs/fr/sql-reference/statements/select/limit.md b/docs/fr/sql-reference/statements/select/limit.md new file mode 100644 index 00000000000..69334c32cc9 --- /dev/null +++ b/docs/fr/sql-reference/statements/select/limit.md @@ -0,0 +1,14 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +--- + +# Clause LIMIT {#limit-clause} + +`LIMIT m` permet de sélectionner la première `m` lignes du résultat. + +`LIMIT n, m` permet de sélectionner le `m` lignes du résultat après avoir sauté le premier `n` rangée. Le `LIMIT m OFFSET n` la syntaxe est équivalente. + +`n` et `m` doivent être des entiers non négatifs. + +Si il n'y a pas de [ORDER BY](order-by.md) clause qui trie explicitement les résultats, le choix des lignes pour le résultat peut être arbitraire et non déterministe. diff --git a/docs/fr/sql-reference/statements/select/order-by.md b/docs/fr/sql-reference/statements/select/order-by.md new file mode 100644 index 00000000000..2a4ef58d7ad --- /dev/null +++ b/docs/fr/sql-reference/statements/select/order-by.md @@ -0,0 +1,72 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +--- + +# Clause ORDER BY {#select-order-by} + +Le `ORDER BY` clause contient une liste des expressions, qui peuvent être attribuées avec `DESC` (décroissant) ou `ASC` modificateur (ascendant) qui détermine la direction de tri. Si la direction n'est pas spécifié, `ASC` est supposé, donc il est généralement omis. La direction de tri s'applique à une seule expression, pas à la liste entière. Exemple: `ORDER BY Visits DESC, SearchPhrase` + +Les lignes qui ont des valeurs identiques pour la liste des expressions de tri sont sorties dans un ordre arbitraire, qui peut également être non déterministe (différent à chaque fois). +Si la clause ORDER BY est omise, l'ordre des lignes est également indéfini et peut également être non déterministe. + +## Tri des valeurs spéciales {#sorting-of-special-values} + +Il existe deux approches pour `NaN` et `NULL` ordre de tri: + +- Par défaut ou avec le `NULLS LAST` modificateur: d'abord les valeurs, puis `NaN`, puis `NULL`. +- Avec l' `NULLS FIRST` modificateur: première `NULL`, puis `NaN` puis d'autres valeurs. + +### Exemple {#example} + +Pour la table + +``` text +┌─x─┬────y─┐ +│ 1 │ ᴺᵁᴸᴸ │ +│ 2 │ 2 │ +│ 1 │ nan │ +│ 2 │ 2 │ +│ 3 │ 4 │ +│ 5 │ 6 │ +│ 6 │ nan │ +│ 7 │ ᴺᵁᴸᴸ │ +│ 6 │ 7 │ +│ 8 │ 9 │ +└───┴──────┘ +``` + +Exécuter la requête `SELECT * FROM t_null_nan ORDER BY y NULLS FIRST` obtenir: + +``` text +┌─x─┬────y─┐ +│ 1 │ ᴺᵁᴸᴸ │ +│ 7 │ ᴺᵁᴸᴸ │ +│ 1 │ nan │ +│ 6 │ nan │ +│ 2 │ 2 │ +│ 2 │ 2 │ +│ 3 │ 4 │ +│ 5 │ 6 │ +│ 6 │ 7 │ +│ 8 │ 9 │ +└───┴──────┘ +``` + +Lorsque les nombres à virgule flottante sont triés, les Nan sont séparés des autres valeurs. Quel que soit l'ordre de tri, NaNs viennent à la fin. En d'autres termes, pour le Tri ascendant, ils sont placés comme s'ils étaient plus grands que tous les autres nombres, tandis que pour le Tri descendant, ils sont placés comme s'ils étaient plus petits que les autres. + +## Classement De Soutien {#collation-support} + +Pour le tri par valeurs de chaîne, vous pouvez spécifier le classement (comparaison). Exemple: `ORDER BY SearchPhrase COLLATE 'tr'` - pour le tri par mot-clé dans l'ordre croissant, en utilisant l'alphabet turc, insensible à la casse, en supposant que les chaînes sont encodées en UTF-8. COLLATE peut être spécifié ou non pour chaque expression dans L'ordre par indépendamment. Si ASC ou DESC est spécifié, COLLATE est spécifié après. Lors de L'utilisation de COLLATE, le tri est toujours insensible à la casse. + +Nous recommandons uniquement D'utiliser COLLATE pour le tri final d'un petit nombre de lignes, car le tri avec COLLATE est moins efficace que le tri normal par octets. + +## Détails De Mise En Œuvre {#implementation-details} + +Moins de RAM est utilisé si un assez petit [LIMIT](limit.md) est précisée en plus `ORDER BY`. Sinon, la quantité de mémoire dépensée est proportionnelle au volume de données à trier. Pour le traitement des requêtes distribuées, si [GROUP BY](group-by.md) est omis, le tri est partiellement effectué sur les serveurs distants et les résultats sont fusionnés Sur le serveur demandeur. Cela signifie que pour le tri distribué, le volume de données à trier peut être supérieur à la quantité de mémoire sur un seul serveur. + +S'il N'y a pas assez de RAM, il est possible d'effectuer un tri dans la mémoire externe (création de fichiers temporaires sur un disque). Utilisez le paramètre `max_bytes_before_external_sort` pour ce but. S'il est défini sur 0 (par défaut), le tri externe est désactivé. Si elle est activée, lorsque le volume de données à trier atteint le nombre spécifié d'octets, les données collectées sont triés et déposés dans un fichier temporaire. Une fois toutes les données lues, tous les fichiers triés sont fusionnés et les résultats sont générés. Les fichiers sont écrits dans le `/var/lib/clickhouse/tmp/` dans la configuration (par défaut, mais vous pouvez `tmp_path` paramètre pour modifier ce paramètre). + +L'exécution d'une requête peut utiliser plus de mémoire que `max_bytes_before_external_sort`. Pour cette raison, ce paramètre doit avoir une valeur significativement inférieure à `max_memory_usage`. Par exemple, si votre serveur dispose de 128 Go de RAM et que vous devez exécuter une seule requête, définissez `max_memory_usage` à 100 Go, et `max_bytes_before_external_sort` à 80 Go. + +Le tri externe fonctionne beaucoup moins efficacement que le tri dans la RAM. diff --git a/docs/fr/sql-reference/statements/select/prewhere.md b/docs/fr/sql-reference/statements/select/prewhere.md new file mode 100644 index 00000000000..2c825d050f4 --- /dev/null +++ b/docs/fr/sql-reference/statements/select/prewhere.md @@ -0,0 +1,22 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +--- + +# Clause PREWHERE {#prewhere-clause} + +Prewhere est une optimisation pour appliquer le filtrage plus efficacement. Il est activé par défaut, même si `PREWHERE` la clause n'est pas explicitement spécifié. Il fonctionne en déplaçant automatiquement une partie de [WHERE](where.md) condition à prewhere étape. Le rôle de `PREWHERE` la clause est seulement pour contrôler cette optimisation si vous pensez que vous savez comment le faire mieux que par défaut. + +Avec l'optimisation prewhere, au début, seules les colonnes nécessaires à l'exécution de l'expression prewhere sont lues. Ensuite, les autres colonnes sont lues qui sont nécessaires pour exécuter le reste de la requête, mais seulement les blocs où l'expression prewhere est “true” au moins pour certaines lignes. S'il y a beaucoup de blocs où prewhere expression est “false” pour toutes les lignes et prewhere a besoin de moins de colonnes que les autres parties de la requête, cela permet souvent de lire beaucoup moins de données à partir du disque pour l'exécution de la requête. + +## Contrôle Manuel De Prewhere {#controlling-prewhere-manually} + +La clause a le même sens que la `WHERE` clause. La différence est dans laquelle les données sont lues à partir de la table. Quand à commander manuellement `PREWHERE` pour les conditions de filtration qui sont utilisées par une minorité des colonnes de la requête, mais qui fournissent une filtration de données forte. Cela réduit le volume de données à lire. + +Une requête peut spécifier simultanément `PREWHERE` et `WHERE`. Dans ce cas, `PREWHERE` précéder `WHERE`. + +Si l' `optimize_move_to_prewhere` le paramètre est défini sur 0, heuristiques pour déplacer automatiquement des parties d'expressions `WHERE` de `PREWHERE` sont désactivés. + +## Limitation {#limitations} + +`PREWHERE` est uniquement pris en charge par les tables `*MergeTree` famille. diff --git a/docs/fr/sql-reference/statements/select/sample.md b/docs/fr/sql-reference/statements/select/sample.md new file mode 100644 index 00000000000..b2ddc060a19 --- /dev/null +++ b/docs/fr/sql-reference/statements/select/sample.md @@ -0,0 +1,113 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +--- + +# Exemple de Clause {#select-sample-clause} + +Le `SAMPLE` clause permet approchée `SELECT` le traitement de la requête. + +Lorsque l'échantillonnage de données est activé, la requête n'est pas effectuée sur toutes les données, mais uniquement sur une certaine fraction de données (échantillon). Par exemple, si vous avez besoin de calculer des statistiques pour toutes les visites, il suffit d'exécuter la requête sur le 1/10 de la fraction de toutes les visites, puis multiplier le résultat par 10. + +Le traitement approximatif des requêtes peut être utile dans les cas suivants: + +- Lorsque vous avez des exigences de synchronisation strictes (comme \<100ms), mais que vous ne pouvez pas justifier le coût des ressources matérielles supplémentaires pour y répondre. +- Lorsque vos données brutes ne sont pas précises, l'approximation ne dégrade pas sensiblement la qualité. +- Les exigences commerciales ciblent des résultats approximatifs (pour la rentabilité, ou pour commercialiser des résultats exacts aux utilisateurs premium). + +!!! note "Note" + Vous ne pouvez utiliser l'échantillonnage qu'avec les tables [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md) famille, et seulement si l'expression d'échantillonnage a été spécifiée lors de la création de la table (voir [Moteur MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table)). + +Les caractéristiques de l'échantillonnage des données sont énumérées ci-dessous: + +- L'échantillonnage de données est un mécanisme déterministe. Le résultat de la même `SELECT .. SAMPLE` la requête est toujours le même. +- L'échantillonnage fonctionne de manière cohérente pour différentes tables. Pour les tables avec une seule clé d'échantillonnage, un échantillon avec le même coefficient sélectionne toujours le même sous-ensemble de données possibles. Par exemple, un exemple d'ID utilisateur prend des lignes avec le même sous-ensemble de tous les ID utilisateur possibles de différentes tables. Cela signifie que vous pouvez utiliser l'exemple dans les sous-requêtes dans la [IN](../../operators/in.md) clause. En outre, vous pouvez joindre des échantillons en utilisant le [JOIN](join.md) clause. +- L'échantillonnage permet de lire moins de données à partir d'un disque. Notez que vous devez spécifier l'échantillonnage clé correctement. Pour plus d'informations, voir [Création d'une Table MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table). + +Pour l' `SAMPLE` clause la syntaxe suivante est prise en charge: + +| SAMPLE Clause Syntax | Description | +|----------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `SAMPLE k` | Ici `k` est le nombre de 0 à 1.
La requête est exécutée sur `k` fraction des données. Exemple, `SAMPLE 0.1` exécute la requête sur 10% des données. [Lire plus](#select-sample-k) | +| `SAMPLE n` | Ici `n` est un entier suffisamment grand.
La requête est exécutée sur un échantillon d'au moins `n` lignes (mais pas significativement plus que cela). Exemple, `SAMPLE 10000000` exécute la requête sur un minimum de 10 000 000 lignes. [Lire plus](#select-sample-n) | +| `SAMPLE k OFFSET m` | Ici `k` et `m` sont les nombres de 0 à 1.
La requête est exécutée sur un échantillon de `k` fraction des données. Les données utilisées pour l'échantillon est compensée par `m` fraction. [Lire plus](#select-sample-offset) | + +## SAMPLE K {#select-sample-k} + +Ici `k` est le nombre de 0 à 1 (les notations fractionnaires et décimales sont prises en charge). Exemple, `SAMPLE 1/2` ou `SAMPLE 0.5`. + +Dans un `SAMPLE k` clause, l'échantillon est prélevé à partir de la `k` fraction des données. L'exemple est illustré ci-dessous: + +``` sql +SELECT + Title, + count() * 10 AS PageViews +FROM hits_distributed +SAMPLE 0.1 +WHERE + CounterID = 34 +GROUP BY Title +ORDER BY PageViews DESC LIMIT 1000 +``` + +Dans cet exemple, la requête est exécutée sur un échantillon de 0,1 (10%) de données. Les valeurs des fonctions d'agrégat ne sont pas corrigées automatiquement, donc pour obtenir un résultat approximatif, la valeur `count()` est multiplié manuellement par 10. + +## SAMPLE N {#select-sample-n} + +Ici `n` est un entier suffisamment grand. Exemple, `SAMPLE 10000000`. + +Dans ce cas, la requête est exécutée sur un échantillon d'au moins `n` lignes (mais pas significativement plus que cela). Exemple, `SAMPLE 10000000` exécute la requête sur un minimum de 10 000 000 lignes. + +Puisque l'unité minimale pour la lecture des données est un granule (sa taille est définie par le `index_granularity` de réglage), il est logique de définir un échantillon beaucoup plus grand que la taille du granule. + +Lors de l'utilisation de la `SAMPLE n` clause, vous ne savez pas quel pourcentage relatif de données a été traité. Donc, vous ne connaissez pas le coefficient par lequel les fonctions agrégées doivent être multipliées. L'utilisation de la `_sample_factor` colonne virtuelle pour obtenir le résultat approximatif. + +Le `_sample_factor` colonne contient des coefficients relatifs qui sont calculés dynamiquement. Cette colonne est créée automatiquement lorsque vous [créer](../../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table) une table avec la clé d'échantillonnage spécifiée. Les exemples d'utilisation de la `_sample_factor` colonne sont indiqués ci-dessous. + +Considérons la table `visits` qui contient des statistiques sur les visites de site. Le premier exemple montre comment calculer le nombre de pages vues: + +``` sql +SELECT sum(PageViews * _sample_factor) +FROM visits +SAMPLE 10000000 +``` + +L'exemple suivant montre comment calculer le nombre total de visites: + +``` sql +SELECT sum(_sample_factor) +FROM visits +SAMPLE 10000000 +``` + +L'exemple ci-dessous montre comment calculer la durée moyenne de la session. Notez que vous n'avez pas besoin d'utiliser le coefficient relatif pour calculer les valeurs moyennes. + +``` sql +SELECT avg(Duration) +FROM visits +SAMPLE 10000000 +``` + +## SAMPLE K OFFSET M {#select-sample-offset} + +Ici `k` et `m` sont des nombres de 0 à 1. Des exemples sont présentés ci-dessous. + +**Exemple 1** + +``` sql +SAMPLE 1/10 +``` + +Dans cet exemple, l'échantillon représente 1 / 10e de toutes les données: + +`[++------------]` + +**Exemple 2** + +``` sql +SAMPLE 1/10 OFFSET 1/2 +``` + +Ici, un échantillon de 10% est prélevé à partir de la seconde moitié des données. + +`[------++------]` diff --git a/docs/fr/sql-reference/statements/select/union-all.md b/docs/fr/sql-reference/statements/select/union-all.md new file mode 100644 index 00000000000..63e9987965f --- /dev/null +++ b/docs/fr/sql-reference/statements/select/union-all.md @@ -0,0 +1,35 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +--- + +# Clause UNION ALL {#union-all-clause} + +Vous pouvez utiliser `UNION ALL` à combiner `SELECT` requêtes en étendant leurs résultats. Exemple: + +``` sql +SELECT CounterID, 1 AS table, toInt64(count()) AS c + FROM test.hits + GROUP BY CounterID + +UNION ALL + +SELECT CounterID, 2 AS table, sum(Sign) AS c + FROM test.visits + GROUP BY CounterID + HAVING c > 0 +``` + +Les colonnes de résultat sont appariées par leur index (ordre intérieur `SELECT`). Si les noms de colonne ne correspondent pas, les noms du résultat final sont tirés de la première requête. + +La coulée de Type est effectuée pour les syndicats. Par exemple, si deux requêtes combinées ont le même champ avec non-`Nullable` et `Nullable` types d'un type compatible, la `UNION ALL` a un `Nullable` type de champ. + +Requêtes qui font partie de `UNION ALL` ne peut pas être placée entre parenthèses. [ORDER BY](order-by.md) et [LIMIT](limit.md) sont appliqués à des requêtes séparées, pas au résultat final. Si vous devez appliquer une conversion au résultat final, vous pouvez mettre toutes les requêtes avec `UNION ALL` dans une sous-requête dans la [FROM](from.md) clause. + +## Limitation {#limitations} + +Seulement `UNION ALL` est pris en charge. Régulier `UNION` (`UNION DISTINCT`) n'est pas pris en charge. Si vous avez besoin d' `UNION DISTINCT`, vous pouvez écrire `SELECT DISTINCT` à partir d'une sous-requête contenant `UNION ALL`. + +## Détails De Mise En Œuvre {#implementation-details} + +Requêtes qui font partie de `UNION ALL` peuvent être exécutées simultanément, et leurs résultats peuvent être mélangés ensemble. diff --git a/docs/fr/sql-reference/statements/select/where.md b/docs/fr/sql-reference/statements/select/where.md new file mode 100644 index 00000000000..a4d7bc5e87a --- /dev/null +++ b/docs/fr/sql-reference/statements/select/where.md @@ -0,0 +1,15 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +--- + +# Clause where {#select-where} + +`WHERE` clause permet de filtrer les données en provenance de [FROM](from.md) la clause de `SELECT`. + +Si il y a un `WHERE` , il doit contenir une expression avec la `UInt8` type. C'est généralement une expression avec comparaison et opérateurs logiques. Les lignes où cette expression est évaluée à 0 sont exclues des transformations ou des résultats ultérieurs. + +`WHERE` expression est évaluée sur la possibilité d'utiliser des index et l'élagage de partition, si le moteur de table sous-jacent le prend en charge. + +!!! note "Note" + Il y a une optimisation de filtrage appelée [prewhere](prewhere.md). diff --git a/docs/fr/sql-reference/statements/select/with.md b/docs/fr/sql-reference/statements/select/with.md new file mode 100644 index 00000000000..a42aedf460b --- /dev/null +++ b/docs/fr/sql-reference/statements/select/with.md @@ -0,0 +1,80 @@ +--- +machine_translated: true +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +--- + +# AVEC la Clause {#with-clause} + +Cette section prend en charge les Expressions de Table courantes ([CTE](https://en.wikipedia.org/wiki/Hierarchical_and_recursive_queries_in_SQL)), de sorte que les résultats de `WITH` la clause peut être utilisé à l'intérieur `SELECT` clause. + +## Limitation {#limitations} + +1. Les requêtes récursives ne sont pas prises en charge. +2. Lorsque la sous-requête est utilisée à l'intérieur avec section, son résultat doit être scalaire avec exactement une ligne. +3. Les résultats d'Expression ne sont pas disponibles dans les sous-requêtes. + +## Exemple {#examples} + +**Exemple 1:** Utilisation d'une expression constante comme “variable” + +``` sql +WITH '2019-08-01 15:23:00' as ts_upper_bound +SELECT * +FROM hits +WHERE + EventDate = toDate(ts_upper_bound) AND + EventTime <= ts_upper_bound +``` + +**Exemple 2:** De les expulser, somme(octets) résultat de l'expression de clause SELECT de la liste de colonnes + +``` sql +WITH sum(bytes) as s +SELECT + formatReadableSize(s), + table +FROM system.parts +GROUP BY table +ORDER BY s +``` + +**Exemple 3:** Utilisation des résultats de la sous-requête scalaire + +``` sql +/* this example would return TOP 10 of most huge tables */ +WITH + ( + SELECT sum(bytes) + FROM system.parts + WHERE active + ) AS total_disk_usage +SELECT + (sum(bytes) / total_disk_usage) * 100 AS table_disk_usage, + table +FROM system.parts +GROUP BY table +ORDER BY table_disk_usage DESC +LIMIT 10 +``` + +**Exemple 4:** Réutilisation de l'expression dans la sous-requête + +Comme solution de contournement pour la limitation actuelle de l'utilisation de l'expression dans les sous-requêtes, Vous pouvez la dupliquer. + +``` sql +WITH ['hello'] AS hello +SELECT + hello, + * +FROM +( + WITH ['hello'] AS hello + SELECT hello +) +``` + +``` text +┌─hello─────┬─hello─────┐ +│ ['hello'] │ ['hello'] │ +└───────────┴───────────┘ +``` diff --git a/docs/fr/sql-reference/statements/show.md b/docs/fr/sql-reference/statements/show.md index 23a21e6a82b..129c6e30d1c 100644 --- a/docs/fr/sql-reference/statements/show.md +++ b/docs/fr/sql-reference/statements/show.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 38 toc_title: SHOW --- -# Afficher Les requêtes {#show-queries} +# Afficher les requêtes {#show-queries} ## SHOW CREATE TABLE {#show-create-table} @@ -13,7 +13,7 @@ toc_title: SHOW SHOW CREATE [TEMPORARY] [TABLE|DICTIONARY] [db.]table [INTO OUTFILE filename] [FORMAT format] ``` -Renvoie un seul `String`-type ‘statement’ column, which contains a single value – the `CREATE` requête utilisée pour créer l’objet spécifié. +Renvoie un seul `String`-type ‘statement’ column, which contains a single value – the `CREATE` requête utilisée pour créer l'objet spécifié. ## SHOW DATABASES {#show-databases} @@ -30,7 +30,7 @@ Cette requête est identique à `SELECT name FROM system.databases [INTO OUTFILE SHOW PROCESSLIST [INTO OUTFILE filename] [FORMAT format] ``` -Sorties le contenu de la [système.processus](../../operations/system-tables.md#system_tables-processes) table, qui contient une liste de requêtes en cours de traitement en ce moment, à l’exception `SHOW PROCESSLIST` requête. +Sorties le contenu de la [système.processus](../../operations/system-tables.md#system_tables-processes) table, qui contient une liste de requêtes en cours de traitement en ce moment, à l'exception `SHOW PROCESSLIST` requête. Le `SELECT * FROM system.processes` requête renvoie des données sur toutes les requêtes en cours. @@ -48,9 +48,9 @@ Affiche une liste de tableaux. SHOW [TEMPORARY] TABLES [{FROM | IN} ] [LIKE '' | WHERE expr] [LIMIT ] [INTO OUTFILE ] [FORMAT ] ``` -Si l’ `FROM` la clause n’est pas spécifié, la requête renvoie la liste des tables de la base de données actuelle. +Si l' `FROM` la clause n'est pas spécifié, la requête renvoie la liste des tables de la base de données actuelle. -Vous pouvez obtenir les mêmes résultats que l’ `SHOW TABLES` requête de la façon suivante: +Vous pouvez obtenir les mêmes résultats que l' `SHOW TABLES` requête de la façon suivante: ``` sql SELECT name FROM system.tables WHERE database = [AND name LIKE ] [LIMIT ] [INTO OUTFILE ] [FORMAT ] @@ -79,9 +79,9 @@ Affiche une liste de [dictionnaires externes](../../sql-reference/dictionaries/e SHOW DICTIONARIES [FROM ] [LIKE ''] [LIMIT ] [INTO OUTFILE ] [FORMAT ] ``` -Si l’ `FROM` la clause n’est pas spécifié, la requête retourne la liste des dictionnaires de la base de données actuelle. +Si l' `FROM` la clause n'est pas spécifié, la requête retourne la liste des dictionnaires de la base de données actuelle. -Vous pouvez obtenir les mêmes résultats que l’ `SHOW DICTIONARIES` requête de la façon suivante: +Vous pouvez obtenir les mêmes résultats que l' `SHOW DICTIONARIES` requête de la façon suivante: ``` sql SELECT name FROM system.dictionaries WHERE database = [AND name LIKE ] [LIMIT ] [INTO OUTFILE ] [FORMAT ] @@ -102,4 +102,68 @@ SHOW DICTIONARIES FROM db LIKE '%reg%' LIMIT 2 └──────────────┘ ``` +## SHOW GRANTS {#show-grants-statement} + +Montre les privilèges d'un utilisateur. + +### Syntaxe {#show-grants-syntax} + +``` sql +SHOW GRANTS [FOR user] +``` + +Si l'utilisateur n'est pas spécifié, la requête renvoie les privilèges de l'utilisateur actuel. + +## SHOW CREATE USER {#show-create-user-statement} + +Affiche les paramètres qui ont été utilisés [la création d'un utilisateur](create.md#create-user-statement). + +`SHOW CREATE USER` ne produit pas de mots de passe utilisateur. + +### Syntaxe {#show-create-user-syntax} + +``` sql +SHOW CREATE USER [name | CURRENT_USER] +``` + +## SHOW CREATE ROLE {#show-create-role-statement} + +Affiche les paramètres qui ont été utilisés [la création de rôle](create.md#create-role-statement) + +### Syntaxe {#show-create-role-syntax} + +``` sql +SHOW CREATE ROLE name +``` + +## SHOW CREATE ROW POLICY {#show-create-row-policy-statement} + +Affiche les paramètres qui ont été utilisés [création de stratégie de ligne](create.md#create-row-policy-statement) + +### Syntaxe {#show-create-row-policy-syntax} + +``` sql +SHOW CREATE [ROW] POLICY name ON [database.]table +``` + +## SHOW CREATE QUOTA {#show-create-quota-statement} + +Affiche les paramètres qui ont été utilisés [quota de création](create.md#create-quota-statement) + +### Syntaxe {#show-create-row-policy-syntax} + +``` sql +SHOW CREATE QUOTA [name | CURRENT] +``` + +## SHOW CREATE SETTINGS PROFILE {#show-create-settings-profile-statement} + +Affiche les paramètres qui ont été utilisés [configuration création de profil](create.md#create-settings-profile-statement) + +### Syntaxe {#show-create-row-policy-syntax} + +``` sql +SHOW CREATE [SETTINGS] PROFILE name +``` + [Article Original](https://clickhouse.tech/docs/en/query_language/show/) diff --git a/docs/fr/sql-reference/statements/system.md b/docs/fr/sql-reference/statements/system.md index 1f064f830c0..6b957eb9695 100644 --- a/docs/fr/sql-reference/statements/system.md +++ b/docs/fr/sql-reference/statements/system.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 37 toc_title: SYSTEM --- -# SYSTÈME De Requêtes {#query-language-system} +# SYSTÈME de Requêtes {#query-language-system} - [RELOAD DICTIONARIES](#query_language-system-reload-dictionaries) - [RELOAD DICTIONARY](#query_language-system-reload-dictionary) @@ -24,14 +24,14 @@ toc_title: SYSTEM ## RELOAD DICTIONARIES {#query_language-system-reload-dictionaries} Recharge tous les dictionnaires qui ont déjà été chargés avec succès. -Par défaut, les dictionnaires sont chargés paresseusement (voir [dictionaries\_lazy\_load](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-dictionaries_lazy_load)), donc au lieu d’être chargés automatiquement au démarrage, ils sont initialisés lors du premier accès via la fonction dictGet ou sélectionnez dans les tables avec ENGINE = Dictionary . Le `SYSTEM RELOAD DICTIONARIES` query recharge ces dictionnaires (chargés). +Par défaut, les dictionnaires sont chargés paresseusement (voir [dictionaries\_lazy\_load](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-dictionaries_lazy_load)), donc au lieu d'être chargés automatiquement au démarrage, ils sont initialisés lors du premier accès via la fonction dictGet ou sélectionnez dans les tables avec ENGINE = Dictionary . Le `SYSTEM RELOAD DICTIONARIES` query recharge ces dictionnaires (chargés). Retourne toujours `Ok.` quel que soit le résultat de la mise à jour du dictionnaire. -## Recharger Le Dictionnaire Dictionary\_name {#query_language-system-reload-dictionary} +## Recharger le dictionnaire Dictionary\_name {#query_language-system-reload-dictionary} -Recharge complètement un dictionnaire `dictionary_name`, quel que soit l’état du dictionnaire (LOADED / NOT\_LOADED / FAILED). +Recharge complètement un dictionnaire `dictionary_name`, quel que soit l'état du dictionnaire (LOADED / NOT\_LOADED / FAILED). Retourne toujours `Ok.` quel que soit le résultat de la mise à jour du dictionnaire. -L’état du dictionnaire peut être vérifié en interrogeant le `system.dictionaries` table. +L'état du dictionnaire peut être vérifié en interrogeant le `system.dictionaries` table. ``` sql SELECT name, status FROM system.dictionaries; @@ -39,7 +39,7 @@ SELECT name, status FROM system.dictionaries; ## DROP DNS CACHE {#query_language-system-drop-dns-cache} -Réinitialise le cache DNS interne de ClickHouse. Parfois (pour les anciennes versions de ClickHouse), il est nécessaire d’utiliser cette commande lors de la modification de l’infrastructure (modification de l’adresse IP d’un autre serveur ClickHouse ou du serveur utilisé par les dictionnaires). +Réinitialise le cache DNS interne de ClickHouse. Parfois (pour les anciennes versions de ClickHouse), il est nécessaire d'utiliser cette commande lors de la modification de l'infrastructure (modification de l'adresse IP d'un autre serveur ClickHouse ou du serveur utilisé par les dictionnaires). Pour une gestion du cache plus pratique (automatique), voir paramètres disable\_internal\_dns\_cache, dns\_cache\_update\_period. @@ -65,11 +65,11 @@ Annule le processus de ClickHouse (comme `kill -9 {$ pid_clickhouse-server}`) ## Gestion Des Tables Distribuées {#query-language-system-distributed} -ClickHouse peut gérer [distribué](../../engines/table-engines/special/distributed.md) table. Lorsqu’un utilisateur insère des données dans ces tables, ClickHouse crée d’abord une file d’attente des données qui doivent être envoyées aux nœuds de cluster, puis l’envoie de manière asynchrone. Vous pouvez gérer le traitement des files d’attente avec [STOP DISTRIBUTED SENDS](#query_language-system-stop-distributed-sends), [FLUSH DISTRIBUTED](#query_language-system-flush-distributed), et [START DISTRIBUTED SENDS](#query_language-system-start-distributed-sends) requête. Vous pouvez également insérer de manière synchrone des données distribuées avec `insert_distributed_sync` paramètre. +ClickHouse peut gérer [distribué](../../engines/table-engines/special/distributed.md) table. Lorsqu'un utilisateur insère des données dans ces tables, ClickHouse crée d'abord une file d'attente des données qui doivent être envoyées aux nœuds de cluster, puis l'envoie de manière asynchrone. Vous pouvez gérer le traitement des files d'attente avec [STOP DISTRIBUTED SENDS](#query_language-system-stop-distributed-sends), [FLUSH DISTRIBUTED](#query_language-system-flush-distributed), et [START DISTRIBUTED SENDS](#query_language-system-start-distributed-sends) requête. Vous pouvez également insérer de manière synchrone des données distribuées avec `insert_distributed_sync` paramètre. ### STOP DISTRIBUTED SENDS {#query_language-system-stop-distributed-sends} -Désactive la distribution de données en arrière-plan lors de l’insertion de données dans des tables distribuées. +Désactive la distribution de données en arrière-plan lors de l'insertion de données dans des tables distribuées. ``` sql SYSTEM STOP DISTRIBUTED SENDS [db.] @@ -77,7 +77,7 @@ SYSTEM STOP DISTRIBUTED SENDS [db.] ### FLUSH DISTRIBUTED {#query_language-system-flush-distributed} -Force ClickHouse à envoyer des données aux nœuds de cluster de manière synchrone. Si des nœuds ne sont pas disponibles, ClickHouse lève une exception et arrête l’exécution de la requête. Vous pouvez réessayer la requête jusqu’à ce qu’elle réussisse, ce qui se produira lorsque tous les nœuds seront de nouveau en ligne. +Force ClickHouse à envoyer des données aux nœuds de cluster de manière synchrone. Si des nœuds ne sont pas disponibles, ClickHouse lève une exception et arrête l'exécution de la requête. Vous pouvez réessayer la requête jusqu'à ce qu'elle réussisse, ce qui se produira lorsque tous les nœuds seront de nouveau en ligne. ``` sql SYSTEM FLUSH DISTRIBUTED [db.] @@ -85,7 +85,7 @@ SYSTEM FLUSH DISTRIBUTED [db.] ### START DISTRIBUTED SENDS {#query_language-system-start-distributed-sends} -Active la distribution de données en arrière-plan lors de l’insertion de données dans des tables distribuées. +Active la distribution de données en arrière-plan lors de l'insertion de données dans des tables distribuées. ``` sql SYSTEM START DISTRIBUTED SENDS [db.] @@ -93,14 +93,14 @@ SYSTEM START DISTRIBUTED SENDS [db.] ### STOP MERGES {#query_language-system-stop-merges} -Offre la possibilité d’arrêter les fusions d’arrière-plan pour les tables de la famille MergeTree: +Offre la possibilité d'arrêter les fusions d'arrière-plan pour les tables de la famille MergeTree: ``` sql SYSTEM STOP MERGES [[db.]merge_tree_family_table_name] ``` !!! note "Note" - `DETACH / ATTACH` table va commencer les fusions d’arrière-plan pour la table même dans le cas où les fusions ont été arrêtées pour toutes les tables MergeTree auparavant. + `DETACH / ATTACH` table va commencer les fusions d'arrière-plan pour la table même dans le cas où les fusions ont été arrêtées pour toutes les tables MergeTree auparavant. ### START MERGES {#query_language-system-start-merges} diff --git a/docs/fr/sql-reference/syntax.md b/docs/fr/sql-reference/syntax.md index 8c50eb40b8f..27a2e231956 100644 --- a/docs/fr/sql-reference/syntax.md +++ b/docs/fr/sql-reference/syntax.md @@ -1,51 +1,51 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 31 toc_title: Syntaxe --- # Syntaxe {#syntax} -Il existe deux types d’analyseurs dans le système: L’analyseur SQL complet (un analyseur de descente récursif) et l’analyseur de format de données (un analyseur de flux rapide). -Dans tous les cas à l’exception de la `INSERT` requête, seul L’analyseur SQL complet est utilisé. +Il existe deux types d'analyseurs dans le système: L'analyseur SQL complet (un analyseur de descente récursif) et l'analyseur de format de données (un analyseur de flux rapide). +Dans tous les cas à l'exception de la `INSERT` requête, seul L'analyseur SQL complet est utilisé. Le `INSERT` requête utilise les deux analyseurs: ``` sql INSERT INTO t VALUES (1, 'Hello, world'), (2, 'abc'), (3, 'def') ``` -Le `INSERT INTO t VALUES` fragment est analysé par l’analyseur complet, et les données `(1, 'Hello, world'), (2, 'abc'), (3, 'def')` est analysé par l’analyseur de flux rapide. Vous pouvez également activer l’analyseur complet pour les données à l’aide de la [input\_format\_values\_interpret\_expressions](../operations/settings/settings.md#settings-input_format_values_interpret_expressions) paramètre. Lorsque `input_format_values_interpret_expressions = 1`, ClickHouse essaie d’abord d’analyser les valeurs avec l’analyseur de flux rapide. S’il échoue, ClickHouse essaie d’utiliser l’analyseur complet pour les données, en le traitant comme un SQL [expression](#syntax-expressions). +Le `INSERT INTO t VALUES` fragment est analysé par l'analyseur complet, et les données `(1, 'Hello, world'), (2, 'abc'), (3, 'def')` est analysé par l'analyseur de flux rapide. Vous pouvez également activer l'analyseur complet pour les données à l'aide de la [input\_format\_values\_interpret\_expressions](../operations/settings/settings.md#settings-input_format_values_interpret_expressions) paramètre. Lorsque `input_format_values_interpret_expressions = 1`, ClickHouse essaie d'abord d'analyser les valeurs avec l'analyseur de flux rapide. S'il échoue, ClickHouse essaie d'utiliser l'analyseur complet pour les données, en le traitant comme un SQL [expression](#syntax-expressions). -Les données peuvent avoir n’importe quel format. Lorsqu’une requête est reçue, le serveur calcule pas plus que [max\_query\_size](../operations/settings/settings.md#settings-max_query_size) octets de la requête en RAM (par défaut, 1 Mo), et le reste est analysé en flux. -Cela signifie que le système n’a pas de problèmes avec de grandes `INSERT` requêtes, comme le fait MySQL. +Les données peuvent avoir n'importe quel format. Lorsqu'une requête est reçue, le serveur calcule pas plus que [max\_query\_size](../operations/settings/settings.md#settings-max_query_size) octets de la requête en RAM (par défaut, 1 Mo), et le reste est analysé en flux. +Il permet d'éviter les problèmes avec de grandes `INSERT` requête. -Lors de l’utilisation de la `Values` format dans un `INSERT` de la requête, il peut sembler que les données sont analysées de même que les expressions dans un `SELECT` requête, mais ce n’est pas vrai. Le `Values` le format est beaucoup plus limitée. +Lors de l'utilisation de la `Values` format dans un `INSERT` de la requête, il peut sembler que les données sont analysées de même que les expressions dans un `SELECT` requête, mais ce n'est pas vrai. Le `Values` le format est beaucoup plus limitée. -Ensuite, nous allons couvrir l’analyseur complet. Pour plus d’informations sur les analyseurs de format, consultez [Format](../interfaces/formats.md) section. +Le reste de cet article couvre l'analyseur complet. Pour plus d'informations sur les analyseurs de format, consultez [Format](../interfaces/formats.md) section. ## Espace {#spaces} -Il peut y avoir n’importe quel nombre de symboles d’espace entre les constructions syntaxiques (y compris le début et la fin d’une requête). Les symboles d’espace incluent l’espace, l’onglet, le saut de ligne, Le CR et le flux de formulaire. +Il peut y avoir n'importe quel nombre de symboles d'espace entre les constructions syntaxiques (y compris le début et la fin d'une requête). Les symboles d'espace incluent l'espace, l'onglet, le saut de ligne, Le CR et le flux de formulaire. ## Commentaire {#comments} -Les commentaires de style SQL et C sont pris en charge. -Commentaires de style SQL: de `--` à la fin de la ligne. L’espace après `--` peut être omis. -Commentaires dans C-style: de `/*` de `*/`. Ces commentaires peuvent être multilignes. Les espaces ne sont pas requis ici non plus. +ClickHouse prend en charge les commentaires de style SQL et de style C. +Les commentaires de style SQL commencent par `--` et continuer jusqu'à la fin de la ligne, un espace après `--` peut être omis. +C-style sont de `/*` de `*/`et peut être multiligne, les espaces ne sont pas requis non plus. ## Mot {#syntax-keywords} -Les mots clés sont insensibles à la casse lorsqu’ils correspondent à: +Les mots clés sont insensibles à la casse lorsqu'ils correspondent à: - La norme SQL. Exemple, `SELECT`, `select` et `SeLeCt` sont toutes valides. -- Implémentation dans certains SGBD populaires (MySQL ou Postgres). Exemple, `DateTime` est la même que `datetime`. +- Implémentation dans certains SGBD populaires (MySQL ou Postgres). Exemple, `DateTime` est le même que `datetime`. Si le nom du type de données est sensible à la casse peut être vérifié `system.data_type_families` table. Contrairement à SQL standard, tous les autres mots clés (y compris les noms de fonctions) sont **sensible à la casse**. -Mots-clés ne sont pas réservés (ils sont simplement considérés comme des mots-clés dans le contexte correspondant). Si vous utilisez [identificateur](#syntax-identifiers) de même que les mots clés, les placer entre guillemets. Par exemple, la requête `SELECT "FROM" FROM table_name` est valide si la table `table_name` a colonne avec le nom de `"FROM"`. +Mots-clés ne sont pas réservés; ils sont traités comme tels que dans le contexte correspondant. Si vous utilisez [identificateur](#syntax-identifiers) avec le même nom que les mots-clés, placez-les entre guillemets doubles ou backticks. Par exemple, la requête `SELECT "FROM" FROM table_name` est valide si la table `table_name` a colonne avec le nom de `"FROM"`. ## Identificateur {#syntax-identifiers} @@ -56,42 +56,42 @@ Les identificateurs sont: - Types de données. - [Expression des alias](#syntax-expression_aliases). -Les identificateurs peuvent être cités ou non cités. Il est recommandé d’utiliser des identificateurs sans guillemets. +Les identificateurs peuvent être cités ou non cités. Ce dernier est préféré. -Non identificateurs doivent correspondre à l’expression régulière `^[a-zA-Z_][0-9a-zA-Z_]*$` et ne peut pas être égale à [mot](#syntax-keywords). Exemple: `x, _1, X_y__Z123_.` +Non identificateurs doivent correspondre à l'expression régulière `^[a-zA-Z_][0-9a-zA-Z_]*$` et ne peut pas être égale à [mot](#syntax-keywords). Exemple: `x, _1, X_y__Z123_.` -Si vous souhaitez utiliser les identifiants de la même manière que les mots-clés ou si vous souhaitez utiliser d’autres symboles dans les identifiants, citez-le en utilisant des guillemets doubles ou des backticks, par exemple, `"id"`, `` `id` ``. +Si vous souhaitez utiliser les identifiants de la même manière que les mots-clés ou si vous souhaitez utiliser d'autres symboles dans les identifiants, citez-le en utilisant des guillemets doubles ou des backticks, par exemple, `"id"`, `` `id` ``. ## Littéral {#literals} -Il y a: Numérique, chaîne, composé et `NULL` littéral. +Il y a numérique, chaîne de caractères, composé, et `NULL` littéral. ### Numérique {#numeric} -Un littéral numérique tente d’être analysé: +Littéral numérique tente d'être analysé: -- D’abord comme un nombre signé 64 bits, en utilisant le [strtoull](https://en.cppreference.com/w/cpp/string/byte/strtoul) fonction. -- En cas d’échec, en tant que nombre non signé 64 bits, [strtoll](https://en.cppreference.com/w/cpp/string/byte/strtol) fonction. -- En cas d’échec, en tant que nombre à virgule flottante [strtod](https://en.cppreference.com/w/cpp/string/byte/strtof) fonction. -- Sinon, une erreur est renvoyée. +- Tout d'abord, comme un nombre signé 64 bits, en utilisant le [strtoull](https://en.cppreference.com/w/cpp/string/byte/strtoul) fonction. +- En cas d'échec, en tant que nombre non signé 64 bits, [strtoll](https://en.cppreference.com/w/cpp/string/byte/strtol) fonction. +- En cas d'échec, en tant que nombre à virgule flottante [strtod](https://en.cppreference.com/w/cpp/string/byte/strtof) fonction. +- Sinon, elle renvoie une erreur. -La valeur correspondante aura le plus petit type dans lequel la valeur correspond. -Par exemple, 1 est analysé comme `UInt8`, mais 256 est analysé comme `UInt16`. Pour plus d’informations, voir [Types de données](../sql-reference/data-types/index.md). +La valeur littérale a le plus petit type dans lequel la valeur correspond. +Par exemple, 1 est analysé comme `UInt8`, mais 256 est analysé comme `UInt16`. Pour plus d'informations, voir [Types de données](../sql-reference/data-types/index.md). Exemple: `1`, `18446744073709551615`, `0xDEADBEEF`, `01`, `0.1`, `1e100`, `-1e-100`, `inf`, `nan`. ### Chaîne {#syntax-string-literal} -Seuls les littéraux de chaîne entre guillemets simples sont pris en charge. Le clos de caractères barre oblique inverse échappé. Les séquences d’échappement suivantes ont une valeur spéciale correspondante: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\a`, `\v`, `\xHH`. Dans tous les autres cas, des séquences d’échappement au format `\c`, où `c` est un caractère, sont convertis à `c`. Cela signifie que vous pouvez utiliser les séquences `\'`et`\\`. La valeur aurez l’ [Chaîne](../sql-reference/data-types/string.md) type. +Seuls les littéraux de chaîne entre guillemets simples sont pris en charge. Le clos de caractères barre oblique inverse échappé. Les séquences d'échappement suivantes ont une valeur spéciale correspondante: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\a`, `\v`, `\xHH`. Dans tous les autres cas, des séquences d'échappement au format `\c`, où `c` est un caractère, sont convertis à `c`. Cela signifie que vous pouvez utiliser les séquences `\'`et`\\`. La valeur aurez l' [Chaîne](../sql-reference/data-types/string.md) type. -L’ensemble minimum de caractères que vous devez échapper dans les littéraux de chaîne: `'` et `\`. Apostrophe peut être échappé avec l’apostrophe, les littéraux `'It\'s'` et `'It''s'` sont égaux. +Dans les littéraux de chaîne, vous devez vous échapper d'au moins `'` et `\`. Les guillemets simples peuvent être échappés avec le guillemet simple, littéraux `'It\'s'` et `'It''s'` sont égaux. ### Composé {#compound} -Les Constructions sont prises en charge pour les tableaux: `[1, 2, 3]` et les tuples: `(1, 'Hello, world!', 2)`.. -En fait, ce ne sont pas des littéraux, mais des expressions avec l’opérateur de création de tableau et l’opérateur de création de tuple, respectivement. -Un tableau doit être composé d’au moins un élément, et un tuple doit avoir au moins deux éléments. -Les Tuples ont un but spécial pour l’usage dans le `IN` clause de a `SELECT` requête. Les Tuples peuvent être obtenus à la suite d’une requête, mais ils ne peuvent pas être enregistrées dans une base de données (à l’exception de [Mémoire](../engines/table-engines/special/memory.md) table). +Les tableaux sont construits avec des crochets `[1, 2, 3]`. Nuples sont construits avec des supports ronds `(1, 'Hello, world!', 2)`. +Techniquement, ce ne sont pas des littéraux, mais des expressions avec l'opérateur de création de tableau et l'opérateur de création de tuple, respectivement. +Un tableau doit être composé d'au moins un élément, et un tuple doit avoir au moins deux éléments. +Il y a un cas distinct lorsque les tuples apparaissent dans le `IN` clause de a `SELECT` requête. Les résultats de la requête peuvent inclure des tuples, mais les tuples ne peuvent pas être enregistrés dans une base de données (à l'exception des tables avec [Mémoire](../engines/table-engines/special/memory.md) moteur). ### NULL {#null-literal} @@ -99,29 +99,29 @@ Indique que la valeur est manquante. Afin de stocker `NULL` dans un champ de table, il doit être de la [Nullable](../sql-reference/data-types/nullable.md) type. -Selon le format de données (entrée ou sortie), `NULL` peut avoir une représentation différente. Pour plus d’informations, consultez la documentation de [formats de données](../interfaces/formats.md#formats). +Selon le format de données (entrée ou sortie), `NULL` peut avoir une représentation différente. Pour plus d'informations, consultez la documentation de [formats de données](../interfaces/formats.md#formats). -Il y a beaucoup de nuances au traitement `NULL`. Par exemple, si au moins l’un des arguments d’une opération de comparaison est `NULL` le résultat de cette opération sera également `NULL`. Il en va de même pour la multiplication, l’addition et d’autres opérations. Pour plus d’informations, lisez la documentation pour chaque opération. +Il y a beaucoup de nuances au traitement `NULL`. Par exemple, si au moins l'un des arguments d'une opération de comparaison est `NULL` le résultat de cette opération est également `NULL`. Il en va de même pour la multiplication, l'addition et d'autres opérations. Pour plus d'informations, lisez la documentation pour chaque opération. -Dans les requêtes, vous pouvez vérifier `NULL` à l’aide de la [IS NULL](operators.md#operator-is-null) et [IS NOT NULL](operators.md) opérateurs et les fonctions connexes `isNull` et `isNotNull`. +Dans les requêtes, vous pouvez vérifier `NULL` à l'aide de la [IS NULL](operators/index.md#operator-is-null) et [IS NOT NULL](operators/index.md) opérateurs et les fonctions connexes `isNull` et `isNotNull`. ## Fonction {#functions} -Les fonctions sont écrites comme un identifiant avec une liste d’arguments (éventuellement vide) entre parenthèses. Contrairement au SQL standard, les crochets sont requis, même pour une liste d’arguments vide. Exemple: `now()`. -Il existe des fonctions régulières et agrégées (voir la section “Aggregate functions”). Certaines fonctions d’agrégat peut contenir deux listes d’arguments entre parenthèses. Exemple: `quantile (0.9) (x)`. Ces fonctions d’agrégation sont appelés “parametric” fonctions, et les arguments dans la première liste sont appelés “parameters”. La syntaxe des fonctions d’agrégation sans paramètres est la même que pour les fonctions régulières. +Les appels de fonction sont écrits comme un identifiant avec une liste d'arguments (éventuellement vide) entre parenthèses. Contrairement à SQL standard, les crochets sont requis, même pour une liste d'arguments vide. Exemple: `now()`. +Il existe des fonctions régulières et agrégées (voir la section “Aggregate functions”). Certaines fonctions d'agrégat peut contenir deux listes d'arguments entre parenthèses. Exemple: `quantile (0.9) (x)`. Ces fonctions d'agrégation sont appelés “parametric” fonctions, et les arguments dans la première liste sont appelés “parameters”. La syntaxe des fonctions d'agrégation sans paramètres est la même que pour les fonctions régulières. ## Opérateur {#operators} -Les opérateurs sont convertis en leurs fonctions correspondantes lors de l’analyse des requêtes, en tenant compte de leur priorité et de leur associativité. -Par exemple, l’expression `1 + 2 * 3 + 4` est transformé à `plus(plus(1, multiply(2, 3)), 4)`. +Les opérateurs sont convertis en leurs fonctions correspondantes lors de l'analyse des requêtes, en tenant compte de leur priorité et de leur associativité. +Par exemple, l'expression `1 + 2 * 3 + 4` est transformé à `plus(plus(1, multiply(2, 3)), 4)`. -## Types De données Et Moteurs De Table De Base De données {#data_types-and-database-table-engines} +## Types de données et moteurs de Table de base de données {#data_types-and-database-table-engines} -Types de données et moteurs de table dans `CREATE` les requêtes sont écrites de la même manière que les identifiants ou les fonctions. En d’autres termes, ils peuvent ou non contenir une liste d’arguments entre parenthèses. Pour plus d’informations, voir les sections “Data types,” “Table engines,” et “CREATE”. +Types de données et moteurs de table dans `CREATE` les requêtes sont écrites de la même manière que les identifiants ou les fonctions. En d'autres termes, ils peuvent ou ne peuvent pas contenir une liste d'arguments entre parenthèses. Pour plus d'informations, voir les sections “Data types,” “Table engines,” et “CREATE”. ## Expression Des Alias {#syntax-expression_aliases} -Un alias est un nom défini par l’utilisateur pour une expression dans une requête. +Un alias est un nom défini par l'utilisateur pour l'expression dans une requête. ``` sql expr AS alias @@ -141,15 +141,15 @@ expr AS alias For example, `SELECT "table t".column_name FROM table_name AS "table t"`. -### Notes Sur l’Utilisation De La {#notes-on-usage} +### Notes sur l'Utilisation de la {#notes-on-usage} -Les alias sont globaux pour une requête ou d’une sous-requête et vous pouvez définir un alias dans n’importe quelle partie d’une requête de toute expression. Exemple, `SELECT (1 AS n) + 2, n`. +Les alias sont globaux pour une requête ou d'une sous-requête, vous pouvez définir un alias dans n'importe quelle partie d'une requête de toute expression. Exemple, `SELECT (1 AS n) + 2, n`. -Les alias ne sont pas visibles dans les sous-requêtes et entre les sous-requêtes. Par exemple, lors de l’exécution de la requête `SELECT (SELECT sum(b.a) + num FROM b) - a.a AS num FROM a` Clickhouse génère l’exception `Unknown identifier: num`. +Les alias ne sont pas visibles dans les sous-requêtes et entre les sous-requêtes. Par exemple, lors de l'exécution de la requête `SELECT (SELECT sum(b.a) + num FROM b) - a.a AS num FROM a` Clickhouse génère l'exception `Unknown identifier: num`. -Si un alias est défini pour les colonnes de `SELECT` la clause d’une sous-requête, ces colonnes sont visibles dans la requête externe. Exemple, `SELECT n + m FROM (SELECT 1 AS n, 2 AS m)`. +Si un alias est défini pour les colonnes de `SELECT` la clause d'une sous-requête, ces colonnes sont visibles dans la requête externe. Exemple, `SELECT n + m FROM (SELECT 1 AS n, 2 AS m)`. -Soyez prudent avec les Alias qui sont les mêmes que les noms de colonnes ou de tables. Considérons l’exemple suivant: +Soyez prudent avec les Alias qui sont les mêmes que les noms de colonnes ou de tables. Considérons l'exemple suivant: ``` sql CREATE TABLE t @@ -172,16 +172,16 @@ Received exception from server (version 18.14.17): Code: 184. DB::Exception: Received from localhost:9000, 127.0.0.1. DB::Exception: Aggregate function sum(b) is found inside another aggregate function in query. ``` -Dans cet exemple, nous avons déclaré table `t` avec la colonne `b`. Ensuite, lors de la sélection des données, nous avons défini le `sum(b) AS b` alias. Comme les alias sont globaux, ClickHouse a substitué le littéral `b` dans l’expression `argMax(a, b)` avec l’expression `sum(b)`. Cette substitution a provoqué l’exception. +Dans cet exemple, nous avons déclaré table `t` avec la colonne `b`. Ensuite, lors de la sélection des données, nous avons défini le `sum(b) AS b` alias. Comme les alias sont globaux, ClickHouse a substitué le littéral `b` dans l'expression `argMax(a, b)` avec l'expression `sum(b)`. Cette substitution a provoqué l'exception. ## Astérisque {#asterisk} -Dans un `SELECT` requête, un astérisque peut remplacer l’expression. Pour plus d’informations, consultez la section “SELECT”. +Dans un `SELECT` requête, un astérisque peut remplacer l'expression. Pour plus d'informations, consultez la section “SELECT”. ## Expression {#syntax-expressions} -Une expression est une fonction, un identifiant, un littéral, une application d’un opérateur, une expression entre parenthèses, une sous-requête ou un astérisque. Il peut également contenir un alias. +Une expression est une fonction, un identifiant, un littéral, une application d'un opérateur, une expression entre parenthèses, une sous-requête ou un astérisque. Il peut également contenir un alias. Une liste des expressions est une ou plusieurs expressions séparées par des virgules. Les fonctions et les opérateurs, à leur tour, peuvent avoir des expressions comme arguments. -[Article Original](https://clickhouse.tech/docs/en/query_language/syntax/) +[Article Original](https://clickhouse.tech/docs/en/sql_reference/syntax/) diff --git a/docs/fr/sql-reference/table-functions/file.md b/docs/fr/sql-reference/table-functions/file.md index 51ae818e5df..620ac6b2786 100644 --- a/docs/fr/sql-reference/table-functions/file.md +++ b/docs/fr/sql-reference/table-functions/file.md @@ -1,21 +1,21 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 37 toc_title: fichier --- # fichier {#file} -Crée un tableau à partir d’un fichier. Cette fonction de table est similaire à [URL](url.md) et [hdfs](hdfs.md) ceux. +Crée un tableau à partir d'un fichier. Cette fonction de table est similaire à [URL](url.md) et [hdfs](hdfs.md) ceux. ``` sql file(path, format, structure) ``` -**Les paramètres d’entrée** +**Les paramètres d'entrée** -- `path` — The relative path to the file from [user\_files\_path](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-user_files_path). Chemin d’accès à la prise en charge des fichiers suivant les globs en mode Lecture seule: `*`, `?`, `{abc,def}` et `{N..M}` où `N`, `M` — numbers, \``'abc', 'def'` — strings. +- `path` — The relative path to the file from [user\_files\_path](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-user_files_path). Chemin d'accès à la prise en charge des fichiers suivant les globs en mode Lecture seule: `*`, `?`, `{abc,def}` et `{N..M}` où `N`, `M` — numbers, \``'abc', 'def'` — strings. - `format` — The [format](../../interfaces/formats.md#formats) de le fichier. - `structure` — Structure of the table. Format `'column1_name column1_type, column2_name column2_type, ...'`. @@ -59,14 +59,14 @@ SELECT * FROM file('test.csv', 'CSV', 'column1 UInt32, column2 UInt32, column3 U **Globs dans le chemin** -Plusieurs composants de chemin peuvent avoir des globs. Pour être traité, le fichier doit exister et correspondre à l’ensemble du modèle de chemin (pas seulement le suffixe ou le préfixe). +Plusieurs composants de chemin peuvent avoir des globs. Pour être traité, le fichier doit exister et correspondre à l'ensemble du modèle de chemin (pas seulement le suffixe ou le préfixe). - `*` — Substitutes any number of any characters except `/` y compris la chaîne vide. - `?` — Substitutes any single character. - `{some_string,another_string,yet_another_one}` — Substitutes any of strings `'some_string', 'another_string', 'yet_another_one'`. - `{N..M}` — Substitutes any number in range from N to M including both borders. -Les Constructions avec `{}` sont similaires à l’ [fonction de table à distance](../../sql-reference/table-functions/remote.md)). +Les Constructions avec `{}` sont similaires à l' [fonction de table à distance](../../sql-reference/table-functions/remote.md)). **Exemple** diff --git a/docs/fr/sql-reference/table-functions/generate.md b/docs/fr/sql-reference/table-functions/generate.md index 1b5d0ca8f26..1f7eeddd0e1 100644 --- a/docs/fr/sql-reference/table-functions/generate.md +++ b/docs/fr/sql-reference/table-functions/generate.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 47 toc_title: generateRandom --- @@ -19,7 +19,6 @@ generateRandom('name TypeName[, name TypeName]...', [, 'random_seed'[, 'max_stri - `name` — Name of corresponding column. - `TypeName` — Type of corresponding column. -- `limit` — Number of rows to generate. - `max_array_length` — Maximum array length for all generated arrays. Defaults to `10`. - `max_string_length` — Maximum string length for all generated strings. Defaults to `10`. - `random_seed` — Specify random seed manually to produce stable results. If NULL — seed is randomly generated. @@ -28,10 +27,10 @@ generateRandom('name TypeName[, name TypeName]...', [, 'random_seed'[, 'max_stri Un objet de table avec le schéma demandé. -## Exemple D’Utilisation {#usage-example} +## Exemple D'Utilisation {#usage-example} ``` sql -SELECT * FROM generateRandom('a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3), UUID)', 1, 10, 2); +SELECT * FROM generateRandom('a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3), UUID)', 1, 10, 2) LIMIT 3; ``` ``` text diff --git a/docs/fr/sql-reference/table-functions/hdfs.md b/docs/fr/sql-reference/table-functions/hdfs.md index 8df5d0cd94d..67c892e0de9 100644 --- a/docs/fr/sql-reference/table-functions/hdfs.md +++ b/docs/fr/sql-reference/table-functions/hdfs.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 45 toc_title: hdfs --- @@ -13,7 +13,7 @@ Crée une table à partir de fichiers dans HDFS. Cette fonction de table est sim hdfs(URI, format, structure) ``` -**Les paramètres d’entrée** +**Les paramètres d'entrée** - `URI` — The relative URI to the file in HDFS. Path to file support following globs in readonly mode: `*`, `?`, `{abc,def}` et `{N..M}` où `N`, `M` — numbers, \``'abc', 'def'` — strings. - `format` — The [format](../../interfaces/formats.md#formats) de le fichier. @@ -42,14 +42,14 @@ LIMIT 2 **Globs dans le chemin** -Plusieurs composants de chemin peuvent avoir des globs. Pour être traité, le fichier doit exister et correspondre à l’ensemble du modèle de chemin (pas seulement le suffixe ou le préfixe). +Plusieurs composants de chemin peuvent avoir des globs. Pour être traité, le fichier doit exister et correspondre à l'ensemble du modèle de chemin (pas seulement le suffixe ou le préfixe). - `*` — Substitutes any number of any characters except `/` y compris la chaîne vide. - `?` — Substitutes any single character. - `{some_string,another_string,yet_another_one}` — Substitutes any of strings `'some_string', 'another_string', 'yet_another_one'`. - `{N..M}` — Substitutes any number in range from N to M including both borders. -Les Constructions avec `{}` sont similaires à l’ [fonction de table à distance](../../sql-reference/table-functions/remote.md)). +Les Constructions avec `{}` sont similaires à l' [fonction de table à distance](../../sql-reference/table-functions/remote.md)). **Exemple** diff --git a/docs/fr/sql-reference/table-functions/index.md b/docs/fr/sql-reference/table-functions/index.md index 2f71bd509c1..ed25e5d6d5d 100644 --- a/docs/fr/sql-reference/table-functions/index.md +++ b/docs/fr/sql-reference/table-functions/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 -toc_folder_title: Table Functions +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: Les Fonctions De Table toc_priority: 34 toc_title: Introduction --- @@ -12,7 +12,7 @@ Les fonctions de Table sont des méthodes pour construire des tables. Vous pouvez utiliser les fonctions de table dans: -- [FROM](../statements/select.md#select-from) la clause de la `SELECT` requête. +- [FROM](../statements/select/from.md) la clause de la `SELECT` requête. The method for creating a temporary table that is available only in the current query. The table is deleted when the query finishes. @@ -28,7 +28,7 @@ Vous pouvez utiliser les fonctions de table dans: | [fichier](file.md) | Crée un [Fichier](../../engines/table-engines/special/file.md)-moteur de table. | | [fusionner](merge.md) | Crée un [Fusionner](../../engines/table-engines/special/merge.md)-moteur de table. | | [nombre](numbers.md) | Crée une table avec une seule colonne remplie de nombres entiers. | -| [distant](remote.md) | Vous permet d’accéder à des serveurs distants sans [Distribué](../../engines/table-engines/special/distributed.md)-moteur de table. | +| [distant](remote.md) | Vous permet d'accéder à des serveurs distants sans [Distribué](../../engines/table-engines/special/distributed.md)-moteur de table. | | [URL](url.md) | Crée un [URL](../../engines/table-engines/special/url.md)-moteur de table. | | [mysql](mysql.md) | Crée un [MySQL](../../engines/table-engines/integrations/mysql.md)-moteur de table. | | [jdbc](jdbc.md) | Crée un [JDBC](../../engines/table-engines/integrations/jdbc.md)-moteur de table. | diff --git a/docs/fr/sql-reference/table-functions/input.md b/docs/fr/sql-reference/table-functions/input.md index f6babf097a0..21e0eacb5c1 100644 --- a/docs/fr/sql-reference/table-functions/input.md +++ b/docs/fr/sql-reference/table-functions/input.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 46 toc_title: "entr\xE9e" --- @@ -17,17 +17,17 @@ Cette fonction peut être utilisée uniquement dans `INSERT SELECT` requête et (par exemple, il peut être utilisé dans la sous-requête, etc.). Les données peuvent être envoyées de quelque manière que ce soit comme pour ordinaire `INSERT` requête et passé dans tout disponible [format](../../interfaces/formats.md#formats) -qui doit être spécifié à la fin de la requête (contrairement à l’ordinaire `INSERT SELECT`). +qui doit être spécifié à la fin de la requête (contrairement à l'ordinaire `INSERT SELECT`). La caractéristique principale de cette fonction est que lorsque le serveur reçoit des données du client il les convertit simultanément selon la liste des expressions dans le `SELECT` clause et insère dans la table cible. Table temporaire -avec toutes les données transférées n’est pas créé. +avec toutes les données transférées n'est pas créé. **Exemple** - Laissez le `test` le tableau a la structure suivante `(a String, b String)` et les données `data.csv` a une structure différente `(col1 String, col2 Date, col3 Int32)`. Requête pour insérer - les données de l’ `data.csv` dans le `test` table avec conversion simultanée ressemble à ceci: + les données de l' `data.csv` dans le `test` table avec conversion simultanée ressemble à ceci: diff --git a/docs/fr/sql-reference/table-functions/jdbc.md b/docs/fr/sql-reference/table-functions/jdbc.md index caebd4ef00f..76dea0e0930 100644 --- a/docs/fr/sql-reference/table-functions/jdbc.md +++ b/docs/fr/sql-reference/table-functions/jdbc.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 43 toc_title: jdbc --- @@ -9,7 +9,7 @@ toc_title: jdbc `jdbc(jdbc_connection_uri, schema, table)` - retourne la table qui est connectée via le pilote JDBC. -Ce tableau fonction nécessite séparé `clickhouse-jdbc-bridge` programme en cours d’exécution. +Ce tableau fonction nécessite séparé `clickhouse-jdbc-bridge` programme en cours d'exécution. Il prend en charge les types Nullable (basé sur DDL de la table distante qui est interrogée). **Exemple** diff --git a/docs/fr/sql-reference/table-functions/merge.md b/docs/fr/sql-reference/table-functions/merge.md index 0e24a6eb882..1ec264b06bd 100644 --- a/docs/fr/sql-reference/table-functions/merge.md +++ b/docs/fr/sql-reference/table-functions/merge.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 38 toc_title: fusionner --- @@ -9,6 +9,6 @@ toc_title: fusionner `merge(db_name, 'tables_regexp')` – Creates a temporary Merge table. For more information, see the section “Table engines, Merge”. -La structure de la table est tirée de la première table rencontrée qui correspond à l’expression régulière. +La structure de la table est tirée de la première table rencontrée qui correspond à l'expression régulière. [Article Original](https://clickhouse.tech/docs/en/query_language/table_functions/merge/) diff --git a/docs/fr/sql-reference/table-functions/mysql.md b/docs/fr/sql-reference/table-functions/mysql.md index e3f8123e4d8..295456914f0 100644 --- a/docs/fr/sql-reference/table-functions/mysql.md +++ b/docs/fr/sql-reference/table-functions/mysql.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 42 toc_title: mysql --- @@ -35,13 +35,13 @@ mysql('host:port', 'database', 'table', 'user', 'password'[, replace_query, 'on_ Simple `WHERE` des clauses telles que `=, !=, >, >=, <, <=` sont actuellement exécutés sur le serveur MySQL. -Le reste des conditions et le `LIMIT` les contraintes d’échantillonnage sont exécutées dans ClickHouse uniquement après la fin de la requête à MySQL. +Le reste des conditions et le `LIMIT` les contraintes d'échantillonnage sont exécutées dans ClickHouse uniquement après la fin de la requête à MySQL. **Valeur Renvoyée** -Un objet table avec les mêmes colonnes que la table MySQL d’origine. +Un objet table avec les mêmes colonnes que la table MySQL d'origine. -## Exemple D’Utilisation {#usage-example} +## Exemple D'Utilisation {#usage-example} Table dans MySQL: diff --git a/docs/fr/sql-reference/table-functions/numbers.md b/docs/fr/sql-reference/table-functions/numbers.md index 6e44a04184a..50a5ad61002 100644 --- a/docs/fr/sql-reference/table-functions/numbers.md +++ b/docs/fr/sql-reference/table-functions/numbers.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 39 toc_title: nombre --- diff --git a/docs/fr/sql-reference/table-functions/odbc.md b/docs/fr/sql-reference/table-functions/odbc.md index 0cb9330ab1e..aae636a5eb2 100644 --- a/docs/fr/sql-reference/table-functions/odbc.md +++ b/docs/fr/sql-reference/table-functions/odbc.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 44 toc_title: ODBC --- @@ -21,17 +21,17 @@ Paramètre: Pour implémenter en toute sécurité les connexions ODBC, ClickHouse utilise un programme distinct `clickhouse-odbc-bridge`. Si le pilote ODBC est chargé directement depuis `clickhouse-server`, les problèmes de pilote peuvent planter le serveur ClickHouse. Clickhouse démarre automatiquement `clickhouse-odbc-bridge` lorsque cela est nécessaire. Le programme ODBC bridge est installé à partir du même package que `clickhouse-server`. -Les champs avec l’ `NULL` les valeurs de la table externe sont converties en valeurs par défaut pour le type de données de base. Par exemple, si un champ de table MySQL distant a `INT NULL` type il est converti en 0 (la valeur par défaut pour ClickHouse `Int32` type de données). +Les champs avec l' `NULL` les valeurs de la table externe sont converties en valeurs par défaut pour le type de données de base. Par exemple, si un champ de table MySQL distant a `INT NULL` type il est converti en 0 (la valeur par défaut pour ClickHouse `Int32` type de données). -## Exemple d’utilisation {#usage-example} +## Exemple D'Utilisation {#usage-example} -**Obtenir des données de L’installation MySQL locale via ODBC** +**Obtenir des données de L'installation MySQL locale via ODBC** Cet exemple est vérifié pour Ubuntu Linux 18.04 et MySQL server 5.7. Assurez-vous que unixODBC et MySQL Connector sont installés. -Par défaut (si installé à partir de paquets), ClickHouse démarre en tant qu’utilisateur `clickhouse`. Ainsi, vous devez créer et configurer cet utilisateur dans le serveur MySQL. +Par défaut (si installé à partir de paquets), ClickHouse démarre en tant qu'utilisateur `clickhouse`. Ainsi, vous devez créer et configurer cet utilisateur dans le serveur MySQL. ``` bash $ sudo mysql @@ -55,7 +55,7 @@ USERNAME = clickhouse PASSWORD = clickhouse ``` -Vous pouvez vérifier la connexion en utilisant le `isql` utilitaire de l’installation unixODBC. +Vous pouvez vérifier la connexion en utilisant le `isql` utilitaire de l'installation unixODBC. ``` bash $ isql -v mysqlconn diff --git a/docs/fr/sql-reference/table-functions/remote.md b/docs/fr/sql-reference/table-functions/remote.md index c7c541a4afa..2a2fa7d829d 100644 --- a/docs/fr/sql-reference/table-functions/remote.md +++ b/docs/fr/sql-reference/table-functions/remote.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 40 toc_title: distant --- # à distance, remoteSecure {#remote-remotesecure} -Vous permet d’accéder à des serveurs distants sans `Distributed` table. +Vous permet d'accéder à des serveurs distants sans `Distributed` table. Signature: @@ -16,7 +16,7 @@ remote('addresses_expr', db, table[, 'user'[, 'password']]) remote('addresses_expr', db.table[, 'user'[, 'password']]) ``` -`addresses_expr` – An expression that generates addresses of remote servers. This may be just one server address. The server address is `host:port` ou juste `host`. L’hôte peut être spécifié comme nom de serveur ou l’adresse IPv4 ou IPv6. Une adresse IPv6 est indiquée entre crochets. Le port est le port TCP sur le serveur distant. Si le port est omis, il utilise `tcp_port` à partir du fichier de configuration du serveur (par défaut, 9000). +`addresses_expr` – An expression that generates addresses of remote servers. This may be just one server address. The server address is `host:port` ou juste `host`. L'hôte peut être spécifié comme nom de serveur ou l'adresse IPv4 ou IPv6. Une adresse IPv6 est indiquée entre crochets. Le port est le port TCP sur le serveur distant. Si le port est omis, il utilise `tcp_port` à partir du fichier de configuration du serveur (par défaut, 9000). !!! important "Important" Le port est requis pour une adresse IPv6. @@ -40,21 +40,21 @@ Exemple: example01-01-1,example01-02-1 ``` -Une partie de l’expression peut être spécifiée entre crochets. L’exemple précédent peut être écrite comme suit: +Une partie de l'expression peut être spécifiée entre crochets. L'exemple précédent peut être écrite comme suit: ``` text example01-0{1,2}-1 ``` -Les accolades peuvent contenir une plage de Nombres séparés par deux points (entiers non négatifs). Dans ce cas, la gamme est étendue à un ensemble de valeurs qui génèrent fragment d’adresses. Si le premier nombre commence par zéro, les valeurs sont formées avec le même alignement zéro. L’exemple précédent peut être écrite comme suit: +Les accolades peuvent contenir une plage de Nombres séparés par deux points (entiers non négatifs). Dans ce cas, la gamme est étendue à un ensemble de valeurs qui génèrent fragment d'adresses. Si le premier nombre commence par zéro, les valeurs sont formées avec le même alignement zéro. L'exemple précédent peut être écrite comme suit: ``` text example01-{01..02}-1 ``` -Si vous avez plusieurs paires d’accolades, il génère le produit direct des ensembles correspondants. +Si vous avez plusieurs paires d'accolades, il génère le produit direct des ensembles correspondants. -Les adresses et les parties d’adresses entre crochets peuvent être séparées par le symbole de tuyau (\|). Dans ce cas, les ensembles correspondants de adresses sont interprétés comme des répliques, et la requête sera envoyée à la première sain réplique. Cependant, les répliques sont itérées dans l’ordre actuellement défini dans [équilibrage](../../operations/settings/settings.md) paramètre. +Les adresses et les parties d'adresses entre crochets peuvent être séparées par le symbole de tuyau (\|). Dans ce cas, les ensembles correspondants de adresses sont interprétés comme des répliques, et la requête sera envoyée à la première sain réplique. Cependant, les répliques sont itérées dans l'ordre actuellement défini dans [équilibrage](../../operations/settings/settings.md) paramètre. Exemple: @@ -64,19 +64,19 @@ example01-{01..02}-{1|2} Cet exemple spécifie deux fragments qui ont chacun deux répliques. -Le nombre d’adresses générées est limitée par une constante. En ce moment, c’est 1000 adresses. +Le nombre d'adresses générées est limitée par une constante. En ce moment, c'est 1000 adresses. -À l’aide de la `remote` la fonction de table est moins optimale que la création d’un `Distributed` table, car dans ce cas, la connexion au serveur est rétablie pour chaque requête. En outre, si des noms d’hôte, les noms sont résolus, et les erreurs ne sont pas comptés lors de travail avec diverses répliques. Lors du traitement d’un grand nombre de requêtes, créez toujours `Distributed` table à l’avance, et ne pas utiliser la `remote` table de fonction. +À l'aide de la `remote` la fonction de table est moins optimale que la création d'un `Distributed` table, car dans ce cas, la connexion au serveur est rétablie pour chaque requête. En outre, si des noms d'hôte, les noms sont résolus, et les erreurs ne sont pas comptés lors de travail avec diverses répliques. Lors du traitement d'un grand nombre de requêtes, créez toujours `Distributed` table à l'avance, et ne pas utiliser la `remote` table de fonction. Le `remote` table de fonction peut être utile dans les cas suivants: - Accès à un serveur spécifique pour la comparaison de données, le débogage et les tests. - Requêtes entre différents clusters ClickHouse à des fins de recherche. - Demandes distribuées peu fréquentes qui sont faites manuellement. -- Distribué demandes où l’ensemble des serveurs est redéfinie à chaque fois. +- Distribué demandes où l'ensemble des serveurs est redéfinie à chaque fois. -Si l’utilisateur n’est pas spécifié, `default` est utilisée. -Si le mot de passe n’est spécifié, un mot de passe vide est utilisé. +Si l'utilisateur n'est pas spécifié, `default` est utilisée. +Si le mot de passe n'est spécifié, un mot de passe vide est utilisé. `remoteSecure` - la même chose que `remote` but with secured connection. Default port — [tcp\_port\_secure](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-tcp_port_secure) de config ou 9440. diff --git a/docs/fr/sql-reference/table-functions/url.md b/docs/fr/sql-reference/table-functions/url.md index a932c1fccce..1df5cf55526 100644 --- a/docs/fr/sql-reference/table-functions/url.md +++ b/docs/fr/sql-reference/table-functions/url.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 41 toc_title: URL --- diff --git a/docs/fr/whats-new/changelog/2017.md b/docs/fr/whats-new/changelog/2017.md index dc8131bcb79..be2cb7de9f4 100644 --- a/docs/fr/whats-new/changelog/2017.md +++ b/docs/fr/whats-new/changelog/2017.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 79 toc_title: '2017' --- @@ -9,16 +9,16 @@ toc_title: '2017' Cette version contient des corrections de bugs pour la version précédente 1.1.54318: -- Correction d’un bug avec condition de concurrence possible dans la réplication qui pourrait conduire à la perte de données. Ce problème affecte les versions 1.1.54310 et 1.1.54318. Si vous utilisez l’une de ces versions avec des tables répliquées, la mise à jour est fortement recommandée. Ce problème apparaît dans les journaux dans les messages d’avertissement comme `Part ... from own log doesn't exist.` Le problème est pertinent même si vous ne voyez pas ces messages dans les journaux. +- Correction d'un bug avec condition de concurrence possible dans la réplication qui pourrait conduire à la perte de données. Ce problème affecte les versions 1.1.54310 et 1.1.54318. Si vous utilisez l'une de ces versions avec des tables répliquées, la mise à jour est fortement recommandée. Ce problème apparaît dans les journaux dans les messages d'avertissement comme `Part ... from own log doesn't exist.` Le problème est pertinent même si vous ne voyez pas ces messages dans les journaux. ### Clickhouse Version 1.1.54318, 2017-11-30 {#clickhouse-release-1-1-54318-2017-11-30} Cette version contient des corrections de bugs pour la version précédente 1.1.54310: - Correction de suppressions de ligne incorrectes lors des fusions dans le moteur SummingMergeTree -- Correction d’une fuite de mémoire dans les moteurs mergetree non compliqués +- Correction d'une fuite de mémoire dans les moteurs mergetree non compliqués - Correction de la dégradation des performances avec des inserts fréquents dans les moteurs MergeTree -- Correction d’un problème qui causait la file d’attente de réplication pour arrêter l’exécution +- Correction d'un problème qui causait la file d'attente de réplication pour arrêter l'exécution - Rotation fixe et archivage des journaux du serveur ### Clickhouse Version 1.1.54310, 2017-11-01 {#clickhouse-release-1-1-54310-2017-11-01} @@ -30,34 +30,34 @@ Cette version contient des corrections de bugs pour la version précédente 1.1. - Ajout du support pour le chargement [CatBoost](https://catboost.yandex/) modèles et les appliquer aux données stockées dans ClickHouse. - Ajout du support pour les fuseaux horaires avec des décalages non entiers de UTC. - Ajout du support pour les opérations arithmétiques avec des intervalles de temps. -- La plage de valeurs pour les types Date et DateTime est étendue à l’année 2105. -- Ajouté le `CREATE MATERIALIZED VIEW x TO y` requête (spécifie une table pour stocker les données d’une vue matérialisée). +- La plage de valeurs pour les types Date et DateTime est étendue à l'année 2105. +- Ajouté le `CREATE MATERIALIZED VIEW x TO y` requête (spécifie une table pour stocker les données d'une vue matérialisée). - Ajouté le `ATTACH TABLE` requête sans arguments. -- La logique de traitement des colonnes imbriquées avec des noms se terminant par-Map dans une table SummingMergeTree a été extraite dans la fonction d’agrégat sumMap. Vous pouvez maintenant spécifier ces colonnes explicitement. +- La logique de traitement des colonnes imbriquées avec des noms se terminant par-Map dans une table SummingMergeTree a été extraite dans la fonction d'agrégat sumMap. Vous pouvez maintenant spécifier ces colonnes explicitement. - La taille maximale du dictionnaire IP trie est augmentée à 128M entrées. - Ajout de la fonction getSizeOfEnumType. -- Ajout de la fonction d’agrégat sumWithOverflow. -- Ajout du support pour le format D’entrée Cap’n Proto. -- Vous pouvez maintenant personnaliser le niveau de compression lors de l’utilisation de l’algorithme zstd. +- Ajout de la fonction d'agrégat sumWithOverflow. +- Ajout du support pour le format D'entrée Cap'n Proto. +- Vous pouvez maintenant personnaliser le niveau de compression lors de l'utilisation de l'algorithme zstd. -#### Modifications Incompatibles En arrière: {#backward-incompatible-changes} +#### Modifications Incompatibles En Arrière: {#backward-incompatible-changes} -- Création de tables temporaires avec un moteur autre que la Mémoire n’est pas autorisé. -- La création explicite de tables avec le moteur View ou MaterializedView n’est pas autorisée. -- Lors de la création de la table, une nouvelle vérification vérifie que l’expression de clé d’échantillonnage est incluse dans la clé primaire. +- Création de tables temporaires avec un moteur autre que la Mémoire n'est pas autorisé. +- La création explicite de tables avec le moteur View ou MaterializedView n'est pas autorisée. +- Lors de la création de la table, une nouvelle vérification vérifie que l'expression de clé d'échantillonnage est incluse dans la clé primaire. #### Corrections De Bugs: {#bug-fixes} -- Correction des accrochages lors de l’insertion synchrone dans une table distribuée. +- Correction des accrochages lors de l'insertion synchrone dans une table distribuée. - Ajout et retrait non atomiques fixes de pièces dans des tables répliquées. - Les données insérées dans une vue matérialisée ne sont pas soumises à une déduplication inutile. -- L’exécution d’une requête vers une table distribuée pour laquelle le réplica local est en retard et les réplicas distants ne sont pas disponibles n’entraîne plus d’erreur. -- Les utilisateurs n’ont pas besoin d’autorisations d’accès au `default` base de données pour créer des tables temporaires plus. -- Correction d’un plantage lors de la spécification du type de tableau sans arguments. +- L'exécution d'une requête vers une table distribuée pour laquelle le réplica local est en retard et les réplicas distants ne sont pas disponibles n'entraîne plus d'erreur. +- Les utilisateurs n'ont pas besoin d'autorisations d'accès au `default` base de données pour créer des tables temporaires plus. +- Correction d'un plantage lors de la spécification du type de tableau sans arguments. - Correction des interruptions lorsque le volume du disque contenant les journaux du serveur est plein. -- Correction d’un débordement dans la fonction toRelativeWeekNum pour la première semaine de L’époque Unix. +- Correction d'un débordement dans la fonction toRelativeWeekNum pour la première semaine de L'époque Unix. -#### Construire Des améliorations: {#build-improvements} +#### Construire Des Améliorations: {#build-improvements} - Plusieurs bibliothèques tierces (notamment Poco) ont été mises à jour et converties en sous-modules git. @@ -69,130 +69,130 @@ Cette version contient des corrections de bugs pour la version précédente 1.1. #### Corrections De Bugs: {#bug-fixes-1} -- `ALTER` pour les tables répliquées essaie maintenant de commencer à s’exécuter dès que possible. +- `ALTER` pour les tables répliquées essaie maintenant de commencer à s'exécuter dès que possible. - Fixe plante lors de la lecture des données avec le paramètre `preferred_block_size_bytes=0.` - Plantages fixes de `clickhouse-client` lorsque vous appuyez sur `Page Down` -- L’interprétation correcte de certaines requêtes complexes avec `GLOBAL IN` et `UNION ALL` +- L'interprétation correcte de certaines requêtes complexes avec `GLOBAL IN` et `UNION ALL` - `FREEZE PARTITION` fonctionne toujours atomiquement maintenant. - Les requêtes POST vides renvoient maintenant une réponse avec le code 411. -- Correction d’erreurs d’interprétation pour des expressions comme `CAST(1 AS Nullable(UInt8)).` -- Correction d’une erreur lors de la lecture `Array(Nullable(String))` les colonnes de `MergeTree` table. -- Fixe s’écraser lors de l’analyse des requêtes comme `SELECT dummy AS dummy, dummy AS b` +- Correction d'erreurs d'interprétation pour des expressions comme `CAST(1 AS Nullable(UInt8)).` +- Correction d'une erreur lors de la lecture `Array(Nullable(String))` les colonnes de `MergeTree` table. +- Fixe s'écraser lors de l'analyse des requêtes comme `SELECT dummy AS dummy, dummy AS b` - Les utilisateurs sont mis à jour correctement avec invalide `users.xml` -- Manipulation correcte lorsqu’un dictionnaire exécutable renvoie un code de réponse différent de zéro. +- Manipulation correcte lorsqu'un dictionnaire exécutable renvoie un code de réponse différent de zéro. ### Clickhouse Version 1.1.54292, 2017-09-20 {#clickhouse-release-1-1-54292-2017-09-20} #### Nouveauté: {#new-features-2} - Ajouté le `pointInPolygon` fonction pour travailler avec des coordonnées sur un plan de coordonnées. -- Ajouté le `sumMap` fonction d’agrégation pour calculer la somme des tableaux, similaire à `SummingMergeTree`. -- Ajouté le `trunc` fonction. Amélioration des performances des fonctions d’arrondi (`round`, `floor`, `ceil`, `roundToExp2`) et corrigé la logique de la façon dont ils fonctionnent. Changé la logique de la `roundToExp2` fonction pour les fractions et les nombres négatifs. -- Le fichier exécutable ClickHouse dépend maintenant moins de la version libc. Le même fichier exécutable ClickHouse peut fonctionner sur une grande variété de systèmes Linux. Il existe toujours une dépendance lors de l’utilisation de requêtes compilées (avec le paramètre `compile = 1` qui n’est pas utilisé par défaut). +- Ajouté le `sumMap` fonction d'agrégation pour calculer la somme des tableaux, similaire à `SummingMergeTree`. +- Ajouté le `trunc` fonction. Amélioration des performances des fonctions d'arrondi (`round`, `floor`, `ceil`, `roundToExp2`) et corrigé la logique de la façon dont ils fonctionnent. Changé la logique de la `roundToExp2` fonction pour les fractions et les nombres négatifs. +- Le fichier exécutable ClickHouse dépend maintenant moins de la version libc. Le même fichier exécutable ClickHouse peut fonctionner sur une grande variété de systèmes Linux. Il existe toujours une dépendance lors de l'utilisation de requêtes compilées (avec le paramètre `compile = 1` qui n'est pas utilisé par défaut). - Réduit le temps nécessaire à la compilation dynamique des requêtes. #### Corrections De Bugs: {#bug-fixes-2} -- Correction d’une erreur qui produisait parfois `part ... intersects previous part` messages et cohérence des répliques affaiblies. -- Correction d’une erreur qui a provoqué le verrouillage du serveur si ZooKeeper n’était pas disponible pendant l’arrêt. +- Correction d'une erreur qui produisait parfois `part ... intersects previous part` messages et cohérence des répliques affaiblies. +- Correction d'une erreur qui a provoqué le verrouillage du serveur si ZooKeeper n'était pas disponible pendant l'arrêt. - Suppression de la journalisation excessive lors de la restauration des répliques. -- Correction d’une erreur dans L’Union toute la mise en œuvre. -- Correction d’une erreur dans la fonction concat qui s’est produite si la première colonne d’un bloc a le type de tableau. +- Correction d'une erreur dans L'Union toute la mise en œuvre. +- Correction d'une erreur dans la fonction concat qui s'est produite si la première colonne d'un bloc a le type de tableau. - La progression est maintenant affichée correctement dans le système.fusionne table. ### Clickhouse Version 1.1.54289, 2017-09-13 {#clickhouse-release-1-1-54289-2017-09-13} #### Nouveauté: {#new-features-3} -- `SYSTEM` les requêtes pour l’administration du serveur: `SYSTEM RELOAD DICTIONARY`, `SYSTEM RELOAD DICTIONARIES`, `SYSTEM DROP DNS CACHE`, `SYSTEM SHUTDOWN`, `SYSTEM KILL`. +- `SYSTEM` les requêtes pour l'administration du serveur: `SYSTEM RELOAD DICTIONARY`, `SYSTEM RELOAD DICTIONARIES`, `SYSTEM DROP DNS CACHE`, `SYSTEM SHUTDOWN`, `SYSTEM KILL`. - Ajout de fonctions pour travailler avec des tableaux: `concat`, `arraySlice`, `arrayPushBack`, `arrayPushFront`, `arrayPopBack`, `arrayPopFront`. -- Ajouter `root` et `identity` paramètres pour la configuration de ZooKeeper. Cela vous permet d’isoler des utilisateurs individuels sur le même cluster ZooKeeper. -- Fonctions d’agrégation ajoutées `groupBitAnd`, `groupBitOr`, et `groupBitXor` (pour la compatibilité, ils sont également disponibles sous les noms de `BIT_AND`, `BIT_OR`, et `BIT_XOR`). +- Ajouter `root` et `identity` paramètres pour la configuration de ZooKeeper. Cela vous permet d'isoler des utilisateurs individuels sur le même cluster ZooKeeper. +- Fonctions d'agrégation ajoutées `groupBitAnd`, `groupBitOr`, et `groupBitXor` (pour la compatibilité, ils sont également disponibles sous les noms de `BIT_AND`, `BIT_OR`, et `BIT_XOR`). - Les dictionnaires externes peuvent être chargés à partir de MySQL en spécifiant un socket dans le système de fichiers. - Les dictionnaires externes peuvent être chargés à partir de MySQL sur SSL (`ssl_cert`, `ssl_key`, `ssl_ca` paramètre). -- Ajouté le `max_network_bandwidth_for_user` paramètre pour limiter l’utilisation globale de la bande passante pour les requêtes par utilisateur. +- Ajouté le `max_network_bandwidth_for_user` paramètre pour limiter l'utilisation globale de la bande passante pour les requêtes par utilisateur. - Soutien pour `DROP TABLE` pour les tables temporaires. -- Support pour la lecture `DateTime` valeurs au format d’horodatage Unix de `CSV` et `JSONEachRow` format. +- Support pour la lecture `DateTime` valeurs au format d'horodatage Unix de `CSV` et `JSONEachRow` format. - Les répliques en retard dans les requêtes distribuées sont maintenant exclues par défaut (le seuil par défaut est de 5 minutes). -- Le verrouillage FIFO est utilisé pendant ALTER: une requête ALTER n’est pas bloquée indéfiniment pour les requêtes en cours d’exécution en continu. +- Le verrouillage FIFO est utilisé pendant ALTER: une requête ALTER n'est pas bloquée indéfiniment pour les requêtes en cours d'exécution en continu. - Option à définir `umask` dans le fichier de configuration. - Amélioration des performances pour les requêtes avec `DISTINCT` . #### Corrections De Bugs: {#bug-fixes-3} -- Amélioration du processus de suppression des anciens nœuds dans ZooKeeper. Auparavant, les anciens nœuds n’étaient parfois pas supprimés s’il y avait des insertions très fréquentes, ce qui faisait que le serveur était lent à s’arrêter, entre autres choses. +- Amélioration du processus de suppression des anciens nœuds dans ZooKeeper. Auparavant, les anciens nœuds n'étaient parfois pas supprimés s'il y avait des insertions très fréquentes, ce qui faisait que le serveur était lent à s'arrêter, entre autres choses. - Correction de la randomisation lors du choix des hôtes pour la connexion à ZooKeeper. -- Correction de l’exclusion des répliques en retard dans les requêtes distribuées si la réplique est localhost. -- Correction d’une erreur où une partie des données dans un `ReplicatedMergeTree` table pourrait être cassé après l’exécution `ALTER MODIFY` sur un élément dans un `Nested` structure. -- Correction d’une erreur qui pourrait provoquer des requêtes SELECT “hang”. +- Correction de l'exclusion des répliques en retard dans les requêtes distribuées si la réplique est localhost. +- Correction d'une erreur où une partie des données dans un `ReplicatedMergeTree` table pourrait être cassé après l'exécution `ALTER MODIFY` sur un élément dans un `Nested` structure. +- Correction d'une erreur qui pourrait provoquer des requêtes SELECT “hang”. - Améliorations apportées aux requêtes DDL distribuées. - Correction de la requête `CREATE TABLE ... AS `. -- Résolu à l’impasse dans l’ `ALTER ... CLEAR COLUMN IN PARTITION` requête pour `Buffer` table. -- Correction de la valeur par défaut invalide pour `Enum` s (0 au lieu du minimum) lors de l’utilisation du `JSONEachRow` et `TSKV` format. -- Résolu l’apparition de processus zombies lors de l’utilisation d’un dictionnaire avec un `executable` source. +- Résolu à l'impasse dans l' `ALTER ... CLEAR COLUMN IN PARTITION` requête pour `Buffer` table. +- Correction de la valeur par défaut invalide pour `Enum` s (0 au lieu du minimum) lors de l'utilisation du `JSONEachRow` et `TSKV` format. +- Résolu l'apparition de processus zombies lors de l'utilisation d'un dictionnaire avec un `executable` source. - Correction de segfault pour la requête HEAD. -#### Workflow amélioré Pour développer Et Assembler ClickHouse: {#improved-workflow-for-developing-and-assembling-clickhouse} +#### Workflow amélioré pour développer et assembler ClickHouse: {#improved-workflow-for-developing-and-assembling-clickhouse} - Vous pouvez utiliser `pbuilder` pour construire ClickHouse. - Vous pouvez utiliser `libc++` plutôt `libstdc++` pour construit sur Linux. -- Ajout d’instructions pour l’utilisation d’outils d’analyse de code statique: `Coverage`, `clang-tidy`, `cppcheck`. +- Ajout d'instructions pour l'utilisation d'outils d'analyse de code statique: `Coverage`, `clang-tidy`, `cppcheck`. -#### Veuillez Noter Lors De La Mise à Niveau: {#please-note-when-upgrading} +#### Veuillez Noter Lors De La Mise À Niveau: {#please-note-when-upgrading} -- Il y a maintenant une valeur par défaut plus élevée pour le paramètre MergeTree `max_bytes_to_merge_at_max_space_in_pool` (la taille totale maximale des parties de données à fusionner, en octets): elle est passée de 100 GiB à 150 GiB. Cela peut entraîner de grandes fusions s’exécutant après la mise à niveau du serveur, ce qui pourrait entraîner une charge accrue sur le sous-système de disque. Si l’espace libre disponible sur le serveur est inférieur au double de la quantité totale des fusions en cours d’exécution, toutes les autres fusions s’arrêteront, y compris les fusions de petites parties de données. Par conséquent, les requêtes D’insertion échoueront avec le message “Merges are processing significantly slower than inserts.” L’utilisation de la `SELECT * FROM system.merges` requête pour surveiller la situation. Vous pouvez également vérifier le `DiskSpaceReservedForMerge` métrique dans l’ `system.metrics` table, ou en Graphite. Vous n’avez rien à faire pour résoudre ce problème, car le problème se résoudra une fois les grandes fusions terminées. Si vous trouvez cela inacceptable, vous pouvez restaurer la valeur précédente pour le `max_bytes_to_merge_at_max_space_in_pool` paramètre. Pour ce faire, allez à l’ `` section dans config.xml, ensemble ``` ``107374182400 ``` et redémarrer le serveur. +- Il y a maintenant une valeur par défaut plus élevée pour le paramètre MergeTree `max_bytes_to_merge_at_max_space_in_pool` (la taille totale maximale des parties de données à fusionner, en octets): elle est passée de 100 GiB à 150 GiB. Cela peut entraîner de grandes fusions s'exécutant après la mise à niveau du serveur, ce qui pourrait entraîner une charge accrue sur le sous-système de disque. Si l'espace libre disponible sur le serveur est inférieur au double de la quantité totale des fusions en cours d'exécution, toutes les autres fusions s'arrêteront, y compris les fusions de petites parties de données. Par conséquent, les requêtes D'insertion échoueront avec le message “Merges are processing significantly slower than inserts.” L'utilisation de la `SELECT * FROM system.merges` requête pour surveiller la situation. Vous pouvez également vérifier le `DiskSpaceReservedForMerge` métrique dans l' `system.metrics` table, ou en Graphite. Vous n'avez rien à faire pour résoudre ce problème, car le problème se résoudra une fois les grandes fusions terminées. Si vous trouvez cela inacceptable, vous pouvez restaurer la valeur précédente pour le `max_bytes_to_merge_at_max_space_in_pool` paramètre. Pour ce faire, allez à l' `` section dans config.xml, ensemble ``` ``107374182400 ``` et redémarrer le serveur. ### Clickhouse Version 1.1.54284, 2017-08-29 {#clickhouse-release-1-1-54284-2017-08-29} -- C’est une version de correction de la précédente 1.1.54282 libération. Il corrige les fuites dans le répertoire des pièces dans ZooKeeper. +- C'est une version de correction de la précédente 1.1.54282 libération. Il corrige les fuites dans le répertoire des pièces dans ZooKeeper. ### Clickhouse Version 1.1.54282, 2017-08-23 {#clickhouse-release-1-1-54282-2017-08-23} Cette version contient des corrections de bugs pour la version précédente 1.1.54276: -- Fixe `DB::Exception: Assertion violation: !_path.empty()` lors de l’insertion dans une table distribuée. -- Correction de l’analyse lors de l’insertion au format RowBinary si les données d’entrée commencent par’;’. +- Fixe `DB::Exception: Assertion violation: !_path.empty()` lors de l'insertion dans une table distribuée. +- Correction de l'analyse lors de l'insertion au format RowBinary si les données d'entrée commencent par';'. - Errors during runtime compilation of certain aggregate functions (e.g. `groupArray()`). ### Clickhouse Version 1.1.54276, 2017-08-16 {#clickhouse-release-1-1-54276-2017-08-16} #### Nouveauté: {#new-features-4} -- Ajout d’une section facultative avec pour une requête SELECT. Exemple de requête: `WITH 1+1 AS a SELECT a, a*a` -- INSERT peut être effectué de manière synchrone dans une table distribuée: OK n’est retourné qu’une fois toutes les données enregistrées sur tous les fragments. Ceci est activé par le paramètre insert\_distributed\_sync = 1. +- Ajout d'une section facultative avec pour une requête SELECT. Exemple de requête: `WITH 1+1 AS a SELECT a, a*a` +- INSERT peut être effectué de manière synchrone dans une table distribuée: OK n'est retourné qu'une fois toutes les données enregistrées sur tous les fragments. Ceci est activé par le paramètre insert\_distributed\_sync = 1. - Ajout du type de données UUID pour travailler avec des identifiants de 16 octets. -- Ajout D’alias de CHAR, FLOAT et d’autres types pour la compatibilité avec Tableau. +- Ajout D'alias de CHAR, FLOAT et d'autres types pour la compatibilité avec Tableau. - Ajout des fonctions toYYYYMM, toYYYYMMDD et toYYYYMMDDhhmmss pour convertir le temps en nombres. -- Vous pouvez utiliser les adresses IP (avec le nom d’hôte) pour identifier les serveurs pour les requêtes DDL en cluster. +- Vous pouvez utiliser les adresses IP (avec le nom d'hôte) pour identifier les serveurs pour les requêtes DDL en cluster. - Ajout du support pour les arguments non constants et les décalages négatifs dans la fonction `substring(str, pos, len).` -- Ajout du paramètre max\_size pour `groupArray(max_size)(column)` fonction d’agrégation, et optimisé ses performances. +- Ajout du paramètre max\_size pour `groupArray(max_size)(column)` fonction d'agrégation, et optimisé ses performances. #### Les Principaux Changements: {#main-changes} - Améliorations de la sécurité: tous les fichiers du serveur sont créés avec des autorisations 0640 (peuvent être modifiés via `` paramètre de configuration). -- Amélioration des messages d’erreur pour les requêtes avec une syntaxe invalide. +- Amélioration des messages d'erreur pour les requêtes avec une syntaxe invalide. - Réduction significative de la consommation de mémoire et amélioration des performances lors de la fusion de grandes sections de données MergeTree. - Augmentation significative des performances des fusions de données pour le Remplacementmergetree engine. -- Amélioration des performances pour les insertions asynchrones à partir d’une table distribuée en combinant plusieurs insertions sources. Pour activer cette fonctionnalité, utilisez le paramètre distributed\_directory\_monitor\_batch\_inserts=1. +- Amélioration des performances pour les insertions asynchrones à partir d'une table distribuée en combinant plusieurs insertions sources. Pour activer cette fonctionnalité, utilisez le paramètre distributed\_directory\_monitor\_batch\_inserts=1. -#### Modifications Incompatibles En arrière: {#backward-incompatible-changes-1} +#### Modifications Incompatibles En Arrière: {#backward-incompatible-changes-1} - Modification du format binaire des états agrégés de `groupArray(array_column)` fonctions pour les tableaux. -#### Liste complète Des Modifications: {#complete-list-of-changes} +#### Liste complète des modifications: {#complete-list-of-changes} - Ajouté le `output_format_json_quote_denormals` paramètre, qui permet la sortie des valeurs NaN et inf au format JSON. -- Allocation de flux optimisée lors de la lecture à partir d’une table distribuée. +- Allocation de flux optimisée lors de la lecture à partir d'une table distribuée. - Les paramètres peuvent être configurés en mode Lecture seule si la valeur ne change pas. -- Ajout de la possibilité de récupérer des granules non entiers du moteur MergeTree afin de respecter les restrictions sur la taille de bloc spécifiée dans le paramètre preferred\_block\_size\_bytes. Le but est de réduire la consommation de RAM et d’augmenter la localisation du cache lors du traitement des requêtes à partir de tables avec de grandes colonnes. +- Ajout de la possibilité de récupérer des granules non entiers du moteur MergeTree afin de respecter les restrictions sur la taille de bloc spécifiée dans le paramètre preferred\_block\_size\_bytes. Le but est de réduire la consommation de RAM et d'augmenter la localisation du cache lors du traitement des requêtes à partir de tables avec de grandes colonnes. - Utilisation efficace des index qui contiennent des expressions comme `toStartOfHour(x)` pour des conditions comme `toStartOfHour(x) op сonstexpr.` - Ajout de nouveaux paramètres pour les moteurs MergeTree (la section merge\_tree dans config.XML): - replicated\_deduplication\_window\_seconds définit le nombre de secondes autorisé pour la déduplication des insertions dans les tables répliquées. - cleanup\_delay\_period définit la fréquence de démarrage du nettoyage pour supprimer les données obsolètes. - - replicated\_can\_become\_leader peut empêcher une réplique de devenir le leader (et d’affecter des fusions). + - replicated\_can\_become\_leader peut empêcher une réplique de devenir le leader (et d'affecter des fusions). - Nettoyage accéléré pour supprimer les données obsolètes de ZooKeeper. -- Plusieurs améliorations et corrections pour les requêtes ddl en cluster. Un intérêt particulier est le nouveau paramètre distributed\_ddl\_task\_timeout, qui limite le temps d’attente d’une réponse des serveurs du cluster. Si une requête ddl n’a pas été effectuée sur tous les hôtes, une réponse contiendra une erreur de délai d’attente et une requête sera exécutée en mode asynchrone. -- Amélioration de l’affichage des traces de pile dans les journaux du serveur. +- Plusieurs améliorations et corrections pour les requêtes ddl en cluster. Un intérêt particulier est le nouveau paramètre distributed\_ddl\_task\_timeout, qui limite le temps d'attente d'une réponse des serveurs du cluster. Si une requête ddl n'a pas été effectuée sur tous les hôtes, une réponse contiendra une erreur de délai d'attente et une requête sera exécutée en mode asynchrone. +- Amélioration de l'affichage des traces de pile dans les journaux du serveur. - Ajouté le “none” valeur pour la méthode de compression. - Vous pouvez utiliser plusieurs sections dictionaries\_config dans config.XML. - Il est possible de se connecter à MySQL via un socket dans le système de fichiers. @@ -201,28 +201,28 @@ Cette version contient des corrections de bugs pour la version précédente 1.1. #### Corrections De Bugs: {#bug-fixes-4} - Les tables distribuées utilisant une table de fusion fonctionnent maintenant correctement pour une requête SELECT avec une condition sur le `_table` champ. -- Correction d’une condition de course rare dans ReplicatedMergeTree lors de la vérification des pièces de données. -- Fixe le gel sur “leader election” lors du démarrage d’un serveur. -- Le paramètre max\_replica\_delay\_for\_distributed\_queries a été ignoré lors de l’utilisation d’une réplique locale de la source de données. Ce problème a été corrigé. -- Correction d’un comportement incorrect de `ALTER TABLE CLEAR COLUMN IN PARTITION` lorsque vous tentez de nettoyer une colonne non existante. -- Correction d’une exception dans la fonction multiIf lors de l’utilisation de tableaux ou de chaînes vides. -- Correction d’allocations de mémoire excessives lors de la désérialisation du format natif. -- Correction d’une mise à jour automatique incorrecte des dictionnaires Trie. -- Correction d’une exception lors de l’exécution de requêtes avec une clause GROUP BY à partir d’une table de fusion lors de l’utilisation D’un exemple. -- Correction d’un plantage de GROUP BY lors de l’utilisation de distributed\_aggregation\_memory\_efficient=1. +- Correction d'une condition de course rare dans ReplicatedMergeTree lors de la vérification des pièces de données. +- Fixe le gel sur “leader election” lors du démarrage d'un serveur. +- Le paramètre max\_replica\_delay\_for\_distributed\_queries a été ignoré lors de l'utilisation d'une réplique locale de la source de données. Ce problème a été corrigé. +- Correction d'un comportement incorrect de `ALTER TABLE CLEAR COLUMN IN PARTITION` lorsque vous tentez de nettoyer une colonne non existante. +- Correction d'une exception dans la fonction multiIf lors de l'utilisation de tableaux ou de chaînes vides. +- Correction d'allocations de mémoire excessives lors de la désérialisation du format natif. +- Correction d'une mise à jour automatique incorrecte des dictionnaires Trie. +- Correction d'une exception lors de l'exécution de requêtes avec une clause GROUP BY à partir d'une table de fusion lors de l'utilisation D'un exemple. +- Correction d'un plantage de GROUP BY lors de l'utilisation de distributed\_aggregation\_memory\_efficient=1. - Vous pouvez maintenant spécifier la base de données.table dans le côté droit de IN et JOIN. -- Trop de threads ont été utilisés pour l’agrégation parallèle. Ce problème a été corrigé. +- Trop de threads ont été utilisés pour l'agrégation parallèle. Ce problème a été corrigé. - Correction de la façon dont le “if” fonction fonctionne avec des arguments FixedString. -- SELECT a mal fonctionné à partir d’une table distribuée pour les fragments avec un poids de 0. Ce problème a été corrigé. +- SELECT a mal fonctionné à partir d'une table distribuée pour les fragments avec un poids de 0. Ce problème a été corrigé. - Exécuter `CREATE VIEW IF EXISTS no longer causes crashes.` -- Correction d’un comportement incorrect lorsque input\_format\_skip\_unknown\_fields = 1 est défini et qu’il existe des nombres négatifs. -- Correction d’une boucle infinie dans le `dictGetHierarchy()` fonction s’il y a des données non valides dans le dictionnaire. -- Fixe `Syntax error: unexpected (...)` erreurs lors de l’exécution de requêtes distribuées avec des sous-requêtes dans une clause IN ou JOIN et des tables de fusion. -- Correction d’une interprétation incorrecte D’une requête SELECT à partir de tables de dictionnaire. -- Correction de l’ “Cannot mremap” erreur lors de l’utilisation de tableaux dans In et JOIN clauses avec plus de 2 milliards d’éléments. +- Correction d'un comportement incorrect lorsque input\_format\_skip\_unknown\_fields = 1 est défini et qu'il existe des nombres négatifs. +- Correction d'une boucle infinie dans le `dictGetHierarchy()` fonction s'il y a des données non valides dans le dictionnaire. +- Fixe `Syntax error: unexpected (...)` erreurs lors de l'exécution de requêtes distribuées avec des sous-requêtes dans une clause IN ou JOIN et des tables de fusion. +- Correction d'une interprétation incorrecte D'une requête SELECT à partir de tables de dictionnaire. +- Correction de l' “Cannot mremap” erreur lors de l'utilisation de tableaux dans In et JOIN clauses avec plus de 2 milliards d'éléments. - Correction du basculement pour les dictionnaires avec MySQL comme source. -#### Workflow amélioré Pour développer Et Assembler ClickHouse: {#improved-workflow-for-developing-and-assembling-clickhouse-1} +#### Workflow amélioré pour développer et assembler ClickHouse: {#improved-workflow-for-developing-and-assembling-clickhouse-1} - Construit peuvent être assemblés en Arcadie. - Vous pouvez utiliser gcc 7 pour compiler ClickHouse. @@ -234,35 +234,35 @@ Cette version contient des corrections de bugs pour la version précédente 1.1. - DDL distribué (par exemple, `CREATE TABLE ON CLUSTER`) - La réplication de la requête `ALTER TABLE CLEAR COLUMN IN PARTITION.` -- Le moteur pour les tables de dictionnaire (accès aux données du dictionnaire sous la forme d’une table). +- Le moteur pour les tables de dictionnaire (accès aux données du dictionnaire sous la forme d'une table). - Moteur de base de données de dictionnaire (ce type de base de données a automatiquement des tables de dictionnaire disponibles pour tous les dictionnaires externes connectés). - Vous pouvez vérifier les mises à jour du dictionnaire en envoyant une demande à la source. - Noms de colonnes qualifiés - Les identificateurs entre des guillemets doubles. -- Sessions dans L’interface HTTP. -- La requête OPTIMIZE pour une table répliquée peut s’exécuter non seulement sur le leader. +- Sessions dans L'interface HTTP. +- La requête OPTIMIZE pour une table répliquée peut s'exécuter non seulement sur le leader. -#### Modifications Incompatibles En arrière: {#backward-incompatible-changes-2} +#### Modifications Incompatibles En Arrière: {#backward-incompatible-changes-2} - Supprimé ensemble GLOBAL. #### Des Modifications Mineures: {#minor-changes} -- Maintenant, après le déclenchement d’une alerte, le journal imprime la trace complète de la pile. +- Maintenant, après le déclenchement d'une alerte, le journal imprime la trace complète de la pile. - Détendu la vérification du nombre de pièces de données endommagées/supplémentaires au démarrage (il y avait trop de faux positifs). #### Corrections De Bugs: {#bug-fixes-5} -- Correction d’une mauvaise connexion “sticking” lors de l’insertion dans une table distribuée. -- GLOBAL in fonctionne maintenant pour une requête à partir d’une table de fusion qui regarde une table distribuée. +- Correction d'une mauvaise connexion “sticking” lors de l'insertion dans une table distribuée. +- GLOBAL in fonctionne maintenant pour une requête à partir d'une table de fusion qui regarde une table distribuée. - Le nombre incorrect de cœurs a été détecté sur une machine virtuelle Google Compute Engine. Ce problème a été corrigé. -- Changements dans le fonctionnement d’une source exécutable de dictionnaires externes mis en cache. +- Changements dans le fonctionnement d'une source exécutable de dictionnaires externes mis en cache. - Correction de la comparaison des chaînes contenant des caractères nuls. - Correction de la comparaison des champs de clé primaire Float32 avec des constantes. -- Auparavant, une estimation incorrecte de la taille d’un champ pouvait entraîner des allocations trop importantes. -- Correction d’un plantage lors de l’interrogation d’une colonne Nullable ajoutée à une table en utilisant ALTER. -- Correction d’un plantage lors du tri par une colonne Nullable, si le nombre de lignes est inférieur à la limite. -- Correction d’une commande par sous-requête composée uniquement de valeurs constantes. -- Auparavant, une table répliquée pouvait rester dans l’état non valide après l’échec d’une table de suppression. +- Auparavant, une estimation incorrecte de la taille d'un champ pouvait entraîner des allocations trop importantes. +- Correction d'un plantage lors de l'interrogation d'une colonne Nullable ajoutée à une table en utilisant ALTER. +- Correction d'un plantage lors du tri par une colonne Nullable, si le nombre de lignes est inférieur à la limite. +- Correction d'une commande par sous-requête composée uniquement de valeurs constantes. +- Auparavant, une table répliquée pouvait rester dans l'état non valide après l'échec d'une table de suppression. - Les alias des sous-requêtes scalaires avec des résultats vides ne sont plus perdus. -- Maintenant, une requête qui a utilisé la compilation n’échoue pas avec une erreur si le fichier. so est endommagé. +- Maintenant, une requête qui a utilisé la compilation n'échoue pas avec une erreur si le fichier. so est endommagé. diff --git a/docs/fr/whats-new/changelog/2018.md b/docs/fr/whats-new/changelog/2018.md index be6112f108e..1d901948376 100644 --- a/docs/fr/whats-new/changelog/2018.md +++ b/docs/fr/whats-new/changelog/2018.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 78 toc_title: '2018' --- @@ -11,14 +11,14 @@ toc_title: '2018' #### Corrections De Bugs: {#bug-fixes} -- Correction d’une erreur qui a conduit à des problèmes avec la mise à jour des dictionnaires avec la source ODBC. [\#3825](https://github.com/ClickHouse/ClickHouse/issues/3825), [\#3829](https://github.com/ClickHouse/ClickHouse/issues/3829) -- La compilation JIT des fonctions d’agrégat fonctionne maintenant avec des colonnes LowCardinality. [\#3838](https://github.com/ClickHouse/ClickHouse/issues/3838) +- Correction d'une erreur qui a conduit à des problèmes avec la mise à jour des dictionnaires avec la source ODBC. [\#3825](https://github.com/ClickHouse/ClickHouse/issues/3825), [\#3829](https://github.com/ClickHouse/ClickHouse/issues/3829) +- La compilation JIT des fonctions d'agrégat fonctionne maintenant avec des colonnes LowCardinality. [\#3838](https://github.com/ClickHouse/ClickHouse/issues/3838) #### Amélioration: {#improvements} -- Ajouté le `low_cardinality_allow_in_native_format` paramètre enabled (activé, option par défaut). Lorsqu’elles sont désactivées, les colonnes LowCardinality seront converties en colonnes ordinaires pour les requêtes SELECT et les colonnes ordinaires seront attendues pour les requêtes INSERT. [\#3879](https://github.com/ClickHouse/ClickHouse/pull/3879) +- Ajouté le `low_cardinality_allow_in_native_format` paramètre enabled (activé, option par défaut). Lorsqu'elles sont désactivées, les colonnes LowCardinality seront converties en colonnes ordinaires pour les requêtes SELECT et les colonnes ordinaires seront attendues pour les requêtes INSERT. [\#3879](https://github.com/ClickHouse/ClickHouse/pull/3879) -#### Construire Des améliorations: {#build-improvements} +#### Construire Des Améliorations: {#build-improvements} - Corrections pour les builds sur macOS et ARM. @@ -26,94 +26,94 @@ toc_title: '2018' #### Nouveauté: {#new-features} -- `DEFAULT` les expressions sont évaluées pour les champs manquants lors du chargement de données dans des formats d’entrée semi-structurés (`JSONEachRow`, `TSKV`). La fonction est activée avec le `insert_sample_with_metadata` paramètre. [\#3555](https://github.com/ClickHouse/ClickHouse/pull/3555) -- Le `ALTER TABLE` la requête a maintenant la `MODIFY ORDER BY` action pour changer la clé de tri lors de l’ajout ou de la suppression d’une colonne de table. Ceci est utile pour les tables dans la `MergeTree` famille qui effectuent des tâches supplémentaires lors de la fusion en fonction de cette clé de tri, telles que `SummingMergeTree`, `AggregatingMergeTree` et ainsi de suite. [\#3581](https://github.com/ClickHouse/ClickHouse/pull/3581) [\#3755](https://github.com/ClickHouse/ClickHouse/pull/3755) -- Pour les tableaux dans le `MergeTree` famille, maintenant vous pouvez spécifier une clé de tri différente (`ORDER BY`) et de l’indice de (`PRIMARY KEY`). La clé de tri peut être plus longue que l’index. [\#3581](https://github.com/ClickHouse/ClickHouse/pull/3581) -- Ajouté le `hdfs` fonction de table et le `HDFS` moteur de table pour l’importation et l’exportation de données vers HDFS. [chenxing-xc](https://github.com/ClickHouse/ClickHouse/pull/3617) +- `DEFAULT` les expressions sont évaluées pour les champs manquants lors du chargement de données dans des formats d'entrée semi-structurés (`JSONEachRow`, `TSKV`). La fonction est activée avec le `insert_sample_with_metadata` paramètre. [\#3555](https://github.com/ClickHouse/ClickHouse/pull/3555) +- Le `ALTER TABLE` la requête a maintenant la `MODIFY ORDER BY` action pour changer la clé de tri lors de l'ajout ou de la suppression d'une colonne de table. Ceci est utile pour les tables dans la `MergeTree` famille qui effectuent des tâches supplémentaires lors de la fusion en fonction de cette clé de tri, telles que `SummingMergeTree`, `AggregatingMergeTree` et ainsi de suite. [\#3581](https://github.com/ClickHouse/ClickHouse/pull/3581) [\#3755](https://github.com/ClickHouse/ClickHouse/pull/3755) +- Pour les tableaux dans le `MergeTree` famille, maintenant vous pouvez spécifier une clé de tri différente (`ORDER BY`) et de l'indice de (`PRIMARY KEY`). La clé de tri peut être plus longue que l'index. [\#3581](https://github.com/ClickHouse/ClickHouse/pull/3581) +- Ajouté le `hdfs` fonction de table et le `HDFS` moteur de table pour l'importation et l'exportation de données vers HDFS. [chenxing-xc](https://github.com/ClickHouse/ClickHouse/pull/3617) - Ajout de fonctions pour travailler avec base64: `base64Encode`, `base64Decode`, `tryBase64Decode`. [Alexander Krasheninnikov](https://github.com/ClickHouse/ClickHouse/pull/3350) -- Vous pouvez maintenant utiliser un paramètre pour configurer la précision du `uniqCombined` fonction d’agrégation (sélectionnez le nombre de cellules HyperLogLog). [\#3406](https://github.com/ClickHouse/ClickHouse/pull/3406) +- Vous pouvez maintenant utiliser un paramètre pour configurer la précision du `uniqCombined` fonction d'agrégation (sélectionnez le nombre de cellules HyperLogLog). [\#3406](https://github.com/ClickHouse/ClickHouse/pull/3406) - Ajouté le `system.contributors` table qui contient les noms de tous ceux qui ont fait des commits dans ClickHouse. [\#3452](https://github.com/ClickHouse/ClickHouse/pull/3452) -- Ajout de la possibilité d’omettre la partition de l’ `ALTER TABLE ... FREEZE` requête en vue de sauvegarder toutes les partitions à la fois. [\#3514](https://github.com/ClickHouse/ClickHouse/pull/3514) +- Ajout de la possibilité d'omettre la partition de l' `ALTER TABLE ... FREEZE` requête en vue de sauvegarder toutes les partitions à la fois. [\#3514](https://github.com/ClickHouse/ClickHouse/pull/3514) - Ajouter `dictGet` et `dictGetOrDefault` fonctions qui ne nécessitent pas de spécifier le type de valeur de retour. Le type est déterminé automatiquement à partir de la description du dictionnaire. [Amos Oiseau](https://github.com/ClickHouse/ClickHouse/pull/3564) - Vous pouvez maintenant spécifier des commentaires pour une colonne dans la description de la table et la modifier en utilisant `ALTER`. [\#3377](https://github.com/ClickHouse/ClickHouse/pull/3377) - La lecture est prise en charge pour `Join` tapez des tables avec des touches simples. [Amos Oiseau](https://github.com/ClickHouse/ClickHouse/pull/3728) -- Vous pouvez maintenant spécifier les options `join_use_nulls`, `max_rows_in_join`, `max_bytes_in_join`, et `join_overflow_mode` lors de la création d’un `Join` type de table. [Amos Oiseau](https://github.com/ClickHouse/ClickHouse/pull/3728) -- Ajouté le `joinGet` fonction qui permet d’utiliser un `Join` tapez table comme un dictionnaire. [Amos Oiseau](https://github.com/ClickHouse/ClickHouse/pull/3728) +- Vous pouvez maintenant spécifier les options `join_use_nulls`, `max_rows_in_join`, `max_bytes_in_join`, et `join_overflow_mode` lors de la création d'un `Join` type de table. [Amos Oiseau](https://github.com/ClickHouse/ClickHouse/pull/3728) +- Ajouté le `joinGet` fonction qui permet d'utiliser un `Join` tapez table comme un dictionnaire. [Amos Oiseau](https://github.com/ClickHouse/ClickHouse/pull/3728) - Ajouté le `partition_key`, `sorting_key`, `primary_key`, et `sampling_key` les colonnes de la `system.tables` table afin de fournir des informations sur les clés de table. [\#3609](https://github.com/ClickHouse/ClickHouse/pull/3609) - Ajouté le `is_in_partition_key`, `is_in_sorting_key`, `is_in_primary_key`, et `is_in_sampling_key` les colonnes de la `system.columns` table. [\#3609](https://github.com/ClickHouse/ClickHouse/pull/3609) - Ajouté le `min_time` et `max_time` les colonnes de la `system.parts` table. Ces colonnes sont remplies lorsque la clé de partitionnement est une expression composée de `DateTime` colonne. [Emmanuel Donin de Rosière](https://github.com/ClickHouse/ClickHouse/pull/3800) #### Corrections De Bugs: {#bug-fixes-1} -- Corrections et améliorations des performances pour `LowCardinality` type de données. `GROUP BY` utiliser `LowCardinality(Nullable(...))`. Obtenir les valeurs de `extremes`. Traitement des fonctions d’ordre Élevé. `LEFT ARRAY JOIN`. Distribué `GROUP BY`. Fonctions qui renvoient `Array`. L’exécution de `ORDER BY`. Écrit à `Distributed` tableaux (nicelulu). Rétrocompatibilité pour `INSERT` requêtes provenant d’anciens clients qui implémentent `Native` protocole. Soutien pour `LowCardinality` pour `JOIN`. Amélioration des performances lorsque vous travaillez dans un flux unique. [\#3823](https://github.com/ClickHouse/ClickHouse/pull/3823) [\#3803](https://github.com/ClickHouse/ClickHouse/pull/3803) [\#3799](https://github.com/ClickHouse/ClickHouse/pull/3799) [\#3769](https://github.com/ClickHouse/ClickHouse/pull/3769) [\#3744](https://github.com/ClickHouse/ClickHouse/pull/3744) [\#3681](https://github.com/ClickHouse/ClickHouse/pull/3681) [\#3651](https://github.com/ClickHouse/ClickHouse/pull/3651) [\#3649](https://github.com/ClickHouse/ClickHouse/pull/3649) [\#3641](https://github.com/ClickHouse/ClickHouse/pull/3641) [\#3632](https://github.com/ClickHouse/ClickHouse/pull/3632) [\#3568](https://github.com/ClickHouse/ClickHouse/pull/3568) [\#3523](https://github.com/ClickHouse/ClickHouse/pull/3523) [\#3518](https://github.com/ClickHouse/ClickHouse/pull/3518) -- Correction de la façon dont le `select_sequential_consistency` l’option fonctionne. Auparavant, lorsque ce paramètre était activé, un résultat incomplet était parfois renvoyé après avoir commencé à écrire sur une nouvelle partition. [\#2863](https://github.com/ClickHouse/ClickHouse/pull/2863) -- Les bases de données sont correctement spécifiées lors de L’exécution de DDL `ON CLUSTER` les requêtes et `ALTER UPDATE/DELETE`. [\#3772](https://github.com/ClickHouse/ClickHouse/pull/3772) [\#3460](https://github.com/ClickHouse/ClickHouse/pull/3460) -- Les bases de données sont correctement spécifiées pour les sous-requêtes à l’intérieur d’une vue. [\#3521](https://github.com/ClickHouse/ClickHouse/pull/3521) -- Correction d’un bug dans `PREWHERE` avec `FINAL` pour `VersionedCollapsingMergeTree`. [7167bfd7](https://github.com/ClickHouse/ClickHouse/commit/7167bfd7b365538f7a91c4307ad77e552ab4e8c1) -- Maintenant, vous pouvez utiliser `KILL QUERY` pour annuler les requêtes qui n’ont pas encore démarré car elles attendent que la table soit verrouillée. [\#3517](https://github.com/ClickHouse/ClickHouse/pull/3517) -- Correction des calculs de date et d’heure si les horloges ont été déplacées à minuit (cela se produit en Iran, et est arrivé à Moscou de 1981 à 1983). Auparavant, cela a conduit à la réinitialisation de l’heure un jour plus tôt que nécessaire, et a également provoqué un formatage incorrect de la date et de l’heure au format texte. [\#3819](https://github.com/ClickHouse/ClickHouse/pull/3819) -- Correction de bugs dans certains cas, de `VIEW` et les sous-requêtes qui omettent la base de données. [L’Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/3521) -- Correction d’une condition de course lors de la lecture simultanée d’un `MATERIALIZED VIEW` et la suppression d’un `MATERIALIZED VIEW` en raison de ne pas verrouiller l’interne `MATERIALIZED VIEW`. [\#3404](https://github.com/ClickHouse/ClickHouse/pull/3404) [\#3694](https://github.com/ClickHouse/ClickHouse/pull/3694) -- Correction de l’erreur `Lock handler cannot be nullptr.` [\#3689](https://github.com/ClickHouse/ClickHouse/pull/3689) +- Corrections et améliorations des performances pour `LowCardinality` type de données. `GROUP BY` utiliser `LowCardinality(Nullable(...))`. Obtenir les valeurs de `extremes`. Traitement des fonctions d'ordre Élevé. `LEFT ARRAY JOIN`. Distribué `GROUP BY`. Fonctions qui renvoient `Array`. L'exécution de `ORDER BY`. Écrit à `Distributed` tableaux (nicelulu). Rétrocompatibilité pour `INSERT` requêtes provenant d'anciens clients qui implémentent `Native` protocole. Soutien pour `LowCardinality` pour `JOIN`. Amélioration des performances lorsque vous travaillez dans un flux unique. [\#3823](https://github.com/ClickHouse/ClickHouse/pull/3823) [\#3803](https://github.com/ClickHouse/ClickHouse/pull/3803) [\#3799](https://github.com/ClickHouse/ClickHouse/pull/3799) [\#3769](https://github.com/ClickHouse/ClickHouse/pull/3769) [\#3744](https://github.com/ClickHouse/ClickHouse/pull/3744) [\#3681](https://github.com/ClickHouse/ClickHouse/pull/3681) [\#3651](https://github.com/ClickHouse/ClickHouse/pull/3651) [\#3649](https://github.com/ClickHouse/ClickHouse/pull/3649) [\#3641](https://github.com/ClickHouse/ClickHouse/pull/3641) [\#3632](https://github.com/ClickHouse/ClickHouse/pull/3632) [\#3568](https://github.com/ClickHouse/ClickHouse/pull/3568) [\#3523](https://github.com/ClickHouse/ClickHouse/pull/3523) [\#3518](https://github.com/ClickHouse/ClickHouse/pull/3518) +- Correction de la façon dont le `select_sequential_consistency` l'option fonctionne. Auparavant, lorsque ce paramètre était activé, un résultat incomplet était parfois renvoyé après avoir commencé à écrire sur une nouvelle partition. [\#2863](https://github.com/ClickHouse/ClickHouse/pull/2863) +- Les bases de données sont correctement spécifiées lors de L'exécution de DDL `ON CLUSTER` les requêtes et `ALTER UPDATE/DELETE`. [\#3772](https://github.com/ClickHouse/ClickHouse/pull/3772) [\#3460](https://github.com/ClickHouse/ClickHouse/pull/3460) +- Les bases de données sont correctement spécifiées pour les sous-requêtes à l'intérieur d'une vue. [\#3521](https://github.com/ClickHouse/ClickHouse/pull/3521) +- Correction d'un bug dans `PREWHERE` avec `FINAL` pour `VersionedCollapsingMergeTree`. [7167bfd7](https://github.com/ClickHouse/ClickHouse/commit/7167bfd7b365538f7a91c4307ad77e552ab4e8c1) +- Maintenant, vous pouvez utiliser `KILL QUERY` pour annuler les requêtes qui n'ont pas encore démarré car elles attendent que la table soit verrouillée. [\#3517](https://github.com/ClickHouse/ClickHouse/pull/3517) +- Correction des calculs de date et d'heure si les horloges ont été déplacées à minuit (cela se produit en Iran, et est arrivé à Moscou de 1981 à 1983). Auparavant, cela a conduit à la réinitialisation de l'heure un jour plus tôt que nécessaire, et a également provoqué un formatage incorrect de la date et de l'heure au format texte. [\#3819](https://github.com/ClickHouse/ClickHouse/pull/3819) +- Correction de bugs dans certains cas, de `VIEW` et les sous-requêtes qui omettent la base de données. [L'Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/3521) +- Correction d'une condition de course lors de la lecture simultanée d'un `MATERIALIZED VIEW` et la suppression d'un `MATERIALIZED VIEW` en raison de ne pas verrouiller l'interne `MATERIALIZED VIEW`. [\#3404](https://github.com/ClickHouse/ClickHouse/pull/3404) [\#3694](https://github.com/ClickHouse/ClickHouse/pull/3694) +- Correction de l'erreur `Lock handler cannot be nullptr.` [\#3689](https://github.com/ClickHouse/ClickHouse/pull/3689) - Correction du traitement des requêtes lorsque le `compile_expressions` option est activée (elle est activée par défaut). Expressions constantes non déterministes comme le `now` fonction ne sont plus déplié. [\#3457](https://github.com/ClickHouse/ClickHouse/pull/3457) -- Correction d’un plantage lors de la spécification d’un argument d’échelle non constant dans `toDecimal32/64/128` fonction. -- Correction d’une erreur lors de l’insertion d’un tableau avec `NULL` éléments dans le `Values` formater dans une colonne de type `Array` sans `Nullable` (si `input_format_values_interpret_expressions` = 1). [\#3487](https://github.com/ClickHouse/ClickHouse/pull/3487) [\#3503](https://github.com/ClickHouse/ClickHouse/pull/3503) -- Fixe continue de journalisation des erreurs dans `DDLWorker` si la Gardienne n’est pas disponible. [8f50c620](https://github.com/ClickHouse/ClickHouse/commit/8f50c620334988b28018213ec0092fe6423847e2) -- Correction du type de retour pour `quantile*` les fonctions de `Date` et `DateTime` les types d’arguments. [\#3580](https://github.com/ClickHouse/ClickHouse/pull/3580) -- Correction de l’ `WITH` clause si elle spécifie un alias simple sans expressions. [\#3570](https://github.com/ClickHouse/ClickHouse/pull/3570) -- Correction du traitement des requêtes avec des sous-requêtes nommées et des noms de colonnes qualifiés lorsque `enable_optimize_predicate_expression` est activé. [L’Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/3588) -- Correction de l’erreur `Attempt to attach to nullptr thread group` lorsque vous travaillez avec des vues matérialisées. [Marek Vavruša](https://github.com/ClickHouse/ClickHouse/pull/3623) -- Correction d’un plantage lors du passage de certains arguments incorrects `arrayReverse` fonction. [73e3a7b6](https://github.com/ClickHouse/ClickHouse/commit/73e3a7b662161d6005e7727d8a711b930386b871) -- Correction du débordement de tampon dans le `extractURLParameter` fonction. Amélioration de la performance. Ajout d’un traitement correct des chaînes contenant zéro octet. [141e9799](https://github.com/ClickHouse/ClickHouse/commit/141e9799e49201d84ea8e951d1bed4fb6d3dacb5) -- Dépassement de tampon fixe dans le `lowerUTF8` et `upperUTF8` fonction. Retiré la possibilité d’exécuter ces fonctions sur `FixedString` tapez les arguments. [\#3662](https://github.com/ClickHouse/ClickHouse/pull/3662) -- Correction d’une condition de course rare lors de la suppression `MergeTree` table. [\#3680](https://github.com/ClickHouse/ClickHouse/pull/3680) -- Correction d’une condition de course lors de la lecture de `Buffer` tables et effectuer simultanément `ALTER` ou `DROP` sur les tables cibles. [\#3719](https://github.com/ClickHouse/ClickHouse/pull/3719) -- Correction d’un segfault si le `max_temporary_non_const_columns` limite a été dépassée. [\#3788](https://github.com/ClickHouse/ClickHouse/pull/3788) +- Correction d'un plantage lors de la spécification d'un argument d'échelle non constant dans `toDecimal32/64/128` fonction. +- Correction d'une erreur lors de l'insertion d'un tableau avec `NULL` éléments dans le `Values` formater dans une colonne de type `Array` sans `Nullable` (si `input_format_values_interpret_expressions` = 1). [\#3487](https://github.com/ClickHouse/ClickHouse/pull/3487) [\#3503](https://github.com/ClickHouse/ClickHouse/pull/3503) +- Fixe continue de journalisation des erreurs dans `DDLWorker` si la Gardienne n'est pas disponible. [8f50c620](https://github.com/ClickHouse/ClickHouse/commit/8f50c620334988b28018213ec0092fe6423847e2) +- Correction du type de retour pour `quantile*` les fonctions de `Date` et `DateTime` les types d'arguments. [\#3580](https://github.com/ClickHouse/ClickHouse/pull/3580) +- Correction de l' `WITH` clause si elle spécifie un alias simple sans expressions. [\#3570](https://github.com/ClickHouse/ClickHouse/pull/3570) +- Correction du traitement des requêtes avec des sous-requêtes nommées et des noms de colonnes qualifiés lorsque `enable_optimize_predicate_expression` est activé. [L'Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/3588) +- Correction de l'erreur `Attempt to attach to nullptr thread group` lorsque vous travaillez avec des vues matérialisées. [Marek Vavruša](https://github.com/ClickHouse/ClickHouse/pull/3623) +- Correction d'un plantage lors du passage de certains arguments incorrects `arrayReverse` fonction. [73e3a7b6](https://github.com/ClickHouse/ClickHouse/commit/73e3a7b662161d6005e7727d8a711b930386b871) +- Correction du débordement de tampon dans le `extractURLParameter` fonction. Amélioration de la performance. Ajout d'un traitement correct des chaînes contenant zéro octet. [141e9799](https://github.com/ClickHouse/ClickHouse/commit/141e9799e49201d84ea8e951d1bed4fb6d3dacb5) +- Dépassement de tampon fixe dans le `lowerUTF8` et `upperUTF8` fonction. Retiré la possibilité d'exécuter ces fonctions sur `FixedString` tapez les arguments. [\#3662](https://github.com/ClickHouse/ClickHouse/pull/3662) +- Correction d'une condition de course rare lors de la suppression `MergeTree` table. [\#3680](https://github.com/ClickHouse/ClickHouse/pull/3680) +- Correction d'une condition de course lors de la lecture de `Buffer` tables et effectuer simultanément `ALTER` ou `DROP` sur les tables cibles. [\#3719](https://github.com/ClickHouse/ClickHouse/pull/3719) +- Correction d'un segfault si le `max_temporary_non_const_columns` limite a été dépassée. [\#3788](https://github.com/ClickHouse/ClickHouse/pull/3788) #### Amélioration: {#improvements-1} -- Le serveur n’écrit pas les fichiers de configuration traités `/etc/clickhouse-server/` répertoire. Au lieu de cela, il les enregistre dans la `preprocessed_configs` répertoire à l’intérieur `path`. Cela signifie que l’ `/etc/clickhouse-server/` répertoire n’ont pas d’accès en écriture pour le `clickhouse` de l’utilisateur, ce qui améliore la sécurité. [\#2443](https://github.com/ClickHouse/ClickHouse/pull/2443) -- Le `min_merge_bytes_to_use_direct_io` l’option est définie sur 10 GiB par défaut. Une fusion qui forme de grandes parties de tables de la famille MergeTree sera effectuée dans `O_DIRECT` mode, qui empêche l’expulsion excessive du cache de page. [\#3504](https://github.com/ClickHouse/ClickHouse/pull/3504) -- Démarrage accéléré du serveur lorsqu’il y a un très grand nombre de tables. [\#3398](https://github.com/ClickHouse/ClickHouse/pull/3398) -- Ajout d’un pool de connexion et HTTP `Keep-Alive` pour les connexions entre les répliques. [\#3594](https://github.com/ClickHouse/ClickHouse/pull/3594) -- Si la syntaxe de la requête n’est pas `400 Bad Request` le code est renvoyé dans la `HTTP` interface (500 a été retourné précédemment). [31bc680a](https://github.com/ClickHouse/ClickHouse/commit/31bc680ac5f4bb1d0360a8ba4696fa84bb47d6ab) -- Le `join_default_strictness` l’option est définie sur `ALL` par défaut, pour la compatibilité. [120e2cbe](https://github.com/ClickHouse/ClickHouse/commit/120e2cbe2ff4fbad626c28042d9b28781c805afe) +- Le serveur n'écrit pas les fichiers de configuration traités `/etc/clickhouse-server/` répertoire. Au lieu de cela, il les enregistre dans la `preprocessed_configs` répertoire à l'intérieur `path`. Cela signifie que l' `/etc/clickhouse-server/` répertoire n'ont pas d'accès en écriture pour le `clickhouse` de l'utilisateur, ce qui améliore la sécurité. [\#2443](https://github.com/ClickHouse/ClickHouse/pull/2443) +- Le `min_merge_bytes_to_use_direct_io` l'option est définie sur 10 GiB par défaut. Une fusion qui forme de grandes parties de tables de la famille MergeTree sera effectuée dans `O_DIRECT` mode, qui empêche l'expulsion excessive du cache de page. [\#3504](https://github.com/ClickHouse/ClickHouse/pull/3504) +- Démarrage accéléré du serveur lorsqu'il y a un très grand nombre de tables. [\#3398](https://github.com/ClickHouse/ClickHouse/pull/3398) +- Ajout d'un pool de connexion et HTTP `Keep-Alive` pour les connexions entre les répliques. [\#3594](https://github.com/ClickHouse/ClickHouse/pull/3594) +- Si la syntaxe de la requête n'est pas `400 Bad Request` le code est renvoyé dans la `HTTP` interface (500 a été retourné précédemment). [31bc680a](https://github.com/ClickHouse/ClickHouse/commit/31bc680ac5f4bb1d0360a8ba4696fa84bb47d6ab) +- Le `join_default_strictness` l'option est définie sur `ALL` par défaut, pour la compatibilité. [120e2cbe](https://github.com/ClickHouse/ClickHouse/commit/120e2cbe2ff4fbad626c28042d9b28781c805afe) - Suppression de la journalisation vers `stderr` à partir de la `re2` bibliothèque pour les expressions régulières non valides ou complexes. [\#3723](https://github.com/ClickHouse/ClickHouse/pull/3723) - Ajouté pour la `Kafka` moteur de table: vérifie les abonnements avant de commencer à lire à partir de Kafka; le paramètre kafka\_max\_block\_size pour la table. [Marek Vavruša](https://github.com/ClickHouse/ClickHouse/pull/3396) -- Le `cityHash64`, `farmHash64`, `metroHash64`, `sipHash64`, `halfMD5`, `murmurHash2_32`, `murmurHash2_64`, `murmurHash3_32`, et `murmurHash3_64` fonctions maintenant travailler pour n’importe quel nombre d’arguments et des arguments sous la forme de tuples. [\#3451](https://github.com/ClickHouse/ClickHouse/pull/3451) [\#3519](https://github.com/ClickHouse/ClickHouse/pull/3519) +- Le `cityHash64`, `farmHash64`, `metroHash64`, `sipHash64`, `halfMD5`, `murmurHash2_32`, `murmurHash2_64`, `murmurHash3_32`, et `murmurHash3_64` fonctions maintenant travailler pour n'importe quel nombre d'arguments et des arguments sous la forme de tuples. [\#3451](https://github.com/ClickHouse/ClickHouse/pull/3451) [\#3519](https://github.com/ClickHouse/ClickHouse/pull/3519) - Le `arrayReverse` fonction fonctionne maintenant avec tous les types de tableaux. [73e3a7b6](https://github.com/ClickHouse/ClickHouse/commit/73e3a7b662161d6005e7727d8a711b930386b871) -- Ajout d’un paramètre optionnel: la taille de l’emplacement pour le `timeSlots` fonction. [Kirill Shvakov](https://github.com/ClickHouse/ClickHouse/pull/3724) +- Ajout d'un paramètre optionnel: la taille de l'emplacement pour le `timeSlots` fonction. [Kirill Shvakov](https://github.com/ClickHouse/ClickHouse/pull/3724) - Pour `FULL` et `RIGHT JOIN`, le `max_block_size` le paramètre est utilisé pour un flux de données non jointes à partir de la table de droite. [Amos Oiseau](https://github.com/ClickHouse/ClickHouse/pull/3699) - Ajouté le `--secure` paramètre de ligne de commande dans `clickhouse-benchmark` et `clickhouse-performance-test` pour activer TLS. [\#3688](https://github.com/ClickHouse/ClickHouse/pull/3688) [\#3690](https://github.com/ClickHouse/ClickHouse/pull/3690) -- Conversion de Type lorsque la structure d’un `Buffer` type table ne correspond pas à la structure de la table de destination. [Vitaly Baranov](https://github.com/ClickHouse/ClickHouse/pull/3603) -- Ajouté le `tcp_keep_alive_timeout` option pour activer les paquets persistant après une inactivité de l’intervalle de temps spécifié. [\#3441](https://github.com/ClickHouse/ClickHouse/pull/3441) -- Suppression des guillemets inutiles de valeurs pour la clé de partition dans le `system.parts` le tableau si il se compose d’une seule colonne. [\#3652](https://github.com/ClickHouse/ClickHouse/pull/3652) +- Conversion de Type lorsque la structure d'un `Buffer` type table ne correspond pas à la structure de la table de destination. [Vitaly Baranov](https://github.com/ClickHouse/ClickHouse/pull/3603) +- Ajouté le `tcp_keep_alive_timeout` option pour activer les paquets persistant après une inactivité de l'intervalle de temps spécifié. [\#3441](https://github.com/ClickHouse/ClickHouse/pull/3441) +- Suppression des guillemets inutiles de valeurs pour la clé de partition dans le `system.parts` le tableau si il se compose d'une seule colonne. [\#3652](https://github.com/ClickHouse/ClickHouse/pull/3652) - La fonction modulo fonctionne pour `Date` et `DateTime` types de données. [\#3385](https://github.com/ClickHouse/ClickHouse/pull/3385) - Ajouté synonymes pour le `POWER`, `LN`, `LCASE`, `UCASE`, `REPLACE`, `LOCATE`, `SUBSTR`, et `MID` fonction. [\#3774](https://github.com/ClickHouse/ClickHouse/pull/3774) [\#3763](https://github.com/ClickHouse/ClickHouse/pull/3763) Certains noms de fonctions sont insensibles à la casse pour la compatibilité avec le standard SQL. Sucre syntaxique ajouté `SUBSTRING(expr FROM start FOR length)` pour la compatibilité avec SQL. [\#3804](https://github.com/ClickHouse/ClickHouse/pull/3804) -- Ajout de la possibilité de `mlock` pages mémoire correspondant à `clickhouse-server` code exécutable pour l’empêcher d’être forcé hors de la mémoire. Cette fonctionnalité est désactivée par défaut. [\#3553](https://github.com/ClickHouse/ClickHouse/pull/3553) -- Amélioration des performances lors de la lecture de `O_DIRECT` (avec l’ `min_bytes_to_use_direct_io` option activée). [\#3405](https://github.com/ClickHouse/ClickHouse/pull/3405) -- Amélioration de la performance de l’ `dictGet...OrDefault` fonction pour un argument clé constant et un argument par défaut non constant. [Amos Oiseau](https://github.com/ClickHouse/ClickHouse/pull/3563) +- Ajout de la possibilité de `mlock` pages mémoire correspondant à `clickhouse-server` code exécutable pour l'empêcher d'être forcé hors de la mémoire. Cette fonctionnalité est désactivée par défaut. [\#3553](https://github.com/ClickHouse/ClickHouse/pull/3553) +- Amélioration des performances lors de la lecture de `O_DIRECT` (avec l' `min_bytes_to_use_direct_io` option activée). [\#3405](https://github.com/ClickHouse/ClickHouse/pull/3405) +- Amélioration de la performance de l' `dictGet...OrDefault` fonction pour un argument clé constant et un argument par défaut non constant. [Amos Oiseau](https://github.com/ClickHouse/ClickHouse/pull/3563) - Le `firstSignificantSubdomain` la fonction traite maintenant les domaines `gov`, `mil`, et `edu`. [Igor Hatarist](https://github.com/ClickHouse/ClickHouse/pull/3601) Amélioration de la performance. [\#3628](https://github.com/ClickHouse/ClickHouse/pull/3628) -- Possibilité de spécifier des variables d’environnement personnalisées pour le démarrage `clickhouse-server` à l’aide de la `SYS-V init.d` script en définissant `CLICKHOUSE_PROGRAM_ENV` dans `/etc/default/clickhouse`. +- Possibilité de spécifier des variables d'environnement personnalisées pour le démarrage `clickhouse-server` à l'aide de la `SYS-V init.d` script en définissant `CLICKHOUSE_PROGRAM_ENV` dans `/etc/default/clickhouse`. [Pavlo Bashynskyi](https://github.com/ClickHouse/ClickHouse/pull/3612) -- Code de retour Correct pour le script d’initialisation clickhouse-server. [\#3516](https://github.com/ClickHouse/ClickHouse/pull/3516) +- Code de retour Correct pour le script d'initialisation clickhouse-server. [\#3516](https://github.com/ClickHouse/ClickHouse/pull/3516) - Le `system.metrics` la table a maintenant le `VersionInteger` métrique, et `system.build_options` a la ligne ajoutée `VERSION_INTEGER`, qui contient la forme numérique de la version ClickHouse, telle que `18016000`. [\#3644](https://github.com/ClickHouse/ClickHouse/pull/3644) - Retiré la possibilité de comparer la `Date` tapez avec un nombre pour éviter les erreurs potentielles comme `date = 2018-12-17`, où les citations autour de la date sont omises par erreur. [\#3687](https://github.com/ClickHouse/ClickHouse/pull/3687) -- Correction du comportement des fonctions avec État comme `rowNumberInAllBlocks`. Ils ont précédemment sorti un résultat qui était un nombre plus grand en raison du démarrage lors de l’analyse de la requête. [Amos Oiseau](https://github.com/ClickHouse/ClickHouse/pull/3729) -- Si l’ `force_restore_data` le fichier ne peut pas être supprimé, un message d’erreur est affiché. [Amos Oiseau](https://github.com/ClickHouse/ClickHouse/pull/3794) +- Correction du comportement des fonctions avec État comme `rowNumberInAllBlocks`. Ils ont précédemment sorti un résultat qui était un nombre plus grand en raison du démarrage lors de l'analyse de la requête. [Amos Oiseau](https://github.com/ClickHouse/ClickHouse/pull/3729) +- Si l' `force_restore_data` le fichier ne peut pas être supprimé, un message d'erreur est affiché. [Amos Oiseau](https://github.com/ClickHouse/ClickHouse/pull/3794) -#### Construire Des améliorations: {#build-improvements-1} +#### Construire Des Améliorations: {#build-improvements-1} - Mise à jour le `jemalloc` bibliothèque, qui corrige une fuite de mémoire potentielle. [Amos Oiseau](https://github.com/ClickHouse/ClickHouse/pull/3557) - Profilage avec `jemalloc` est activé par défaut pour les versions de débogage. [2cc82f5c](https://github.com/ClickHouse/ClickHouse/commit/2cc82f5cbe266421cd4c1165286c2c47e5ffcb15) -- Ajout de la possibilité d’exécuter des tests d’intégration lorsque `Docker` est installé sur le système. [\#3650](https://github.com/ClickHouse/ClickHouse/pull/3650) -- Ajout du test d’expression fuzz dans les requêtes SELECT. [\#3442](https://github.com/ClickHouse/ClickHouse/pull/3442) -- Ajout d’un test de stress pour les commits, qui effectue des tests fonctionnels en parallèle et dans un ordre aléatoire pour détecter plus de conditions de course. [\#3438](https://github.com/ClickHouse/ClickHouse/pull/3438) +- Ajout de la possibilité d'exécuter des tests d'intégration lorsque `Docker` est installé sur le système. [\#3650](https://github.com/ClickHouse/ClickHouse/pull/3650) +- Ajout du test d'expression fuzz dans les requêtes SELECT. [\#3442](https://github.com/ClickHouse/ClickHouse/pull/3442) +- Ajout d'un test de stress pour les commits, qui effectue des tests fonctionnels en parallèle et dans un ordre aléatoire pour détecter plus de conditions de course. [\#3438](https://github.com/ClickHouse/ClickHouse/pull/3438) - Amélioration de la méthode de démarrage de clickhouse-server dans une image Docker. [Elghazal Ahmed](https://github.com/ClickHouse/ClickHouse/pull/3663) -- Pour une image Docker, ajout du support pour l’initialisation des bases de données à l’aide de fichiers dans le `/docker-entrypoint-initdb.d` répertoire. [Konstantin Lebedev](https://github.com/ClickHouse/ClickHouse/pull/3695) +- Pour une image Docker, ajout du support pour l'initialisation des bases de données à l'aide de fichiers dans le `/docker-entrypoint-initdb.d` répertoire. [Konstantin Lebedev](https://github.com/ClickHouse/ClickHouse/pull/3695) - Corrections pour les builds sur ARM. [\#3709](https://github.com/ClickHouse/ClickHouse/pull/3709) -#### Modifications Incompatibles En arrière: {#backward-incompatible-changes} +#### Modifications Incompatibles En Arrière: {#backward-incompatible-changes} - Retiré la possibilité de comparer la `Date` tapez avec un numéro. Plutôt `toDate('2018-12-18') = 17883`, vous devez utiliser la conversion de type explicite `= toDate(17883)` [\#3687](https://github.com/ClickHouse/ClickHouse/pull/3687) @@ -123,11 +123,11 @@ toc_title: '2018' #### Corrections De Bugs: {#bug-fixes-2} -- Correction d’une erreur qui a conduit à des problèmes avec la mise à jour des dictionnaires avec la source ODBC. [\#3825](https://github.com/ClickHouse/ClickHouse/issues/3825), [\#3829](https://github.com/ClickHouse/ClickHouse/issues/3829) -- Les bases de données sont correctement spécifiées lors de L’exécution de DDL `ON CLUSTER` requête. [\#3460](https://github.com/ClickHouse/ClickHouse/pull/3460) -- Correction d’un segfault si le `max_temporary_non_const_columns` limite a été dépassée. [\#3788](https://github.com/ClickHouse/ClickHouse/pull/3788) +- Correction d'une erreur qui a conduit à des problèmes avec la mise à jour des dictionnaires avec la source ODBC. [\#3825](https://github.com/ClickHouse/ClickHouse/issues/3825), [\#3829](https://github.com/ClickHouse/ClickHouse/issues/3829) +- Les bases de données sont correctement spécifiées lors de L'exécution de DDL `ON CLUSTER` requête. [\#3460](https://github.com/ClickHouse/ClickHouse/pull/3460) +- Correction d'un segfault si le `max_temporary_non_const_columns` limite a été dépassée. [\#3788](https://github.com/ClickHouse/ClickHouse/pull/3788) -#### Construire Des améliorations: {#build-improvements-2} +#### Construire Des Améliorations: {#build-improvements-2} - Corrections pour les builds sur ARM. @@ -135,24 +135,24 @@ toc_title: '2018' #### Corrections De Bugs: {#bug-fixes-3} -- Correction d’une erreur dans `dictGet...` fonction pour les dictionnaires de type `range` si un des arguments est constante et l’autre ne l’est pas. [\#3751](https://github.com/ClickHouse/ClickHouse/pull/3751) -- Correction d’une erreur qui a causé des messages `netlink: '...': attribute type 1 has an invalid length` pour être imprimé dans le journal du noyau Linux, cela ne se passait que sur des versions assez fraîches du noyau Linux. [\#3749](https://github.com/ClickHouse/ClickHouse/pull/3749) -- Fixe erreur de segmentation en fonction `empty` pour l’argument de `FixedString` type. [Daniel, Dao Quang Minh](https://github.com/ClickHouse/ClickHouse/pull/3703) -- Correction d’une allocation de mémoire excessive lors de l’utilisation d’une grande valeur de `max_query_size` (a la mémoire de morceau de `max_query_size` octets a été préalloué à la fois). [\#3720](https://github.com/ClickHouse/ClickHouse/pull/3720) +- Correction d'une erreur dans `dictGet...` fonction pour les dictionnaires de type `range` si un des arguments est constante et l'autre ne l'est pas. [\#3751](https://github.com/ClickHouse/ClickHouse/pull/3751) +- Correction d'une erreur qui a causé des messages `netlink: '...': attribute type 1 has an invalid length` pour être imprimé dans le journal du noyau Linux, cela ne se passait que sur des versions assez fraîches du noyau Linux. [\#3749](https://github.com/ClickHouse/ClickHouse/pull/3749) +- Fixe erreur de segmentation en fonction `empty` pour l'argument de `FixedString` type. [Daniel, Dao Quang Minh](https://github.com/ClickHouse/ClickHouse/pull/3703) +- Correction d'une allocation de mémoire excessive lors de l'utilisation d'une grande valeur de `max_query_size` (a la mémoire de morceau de `max_query_size` octets a été préalloué à la fois). [\#3720](https://github.com/ClickHouse/ClickHouse/pull/3720) #### Construire Des Changements: {#build-changes} -- Correction de la construction avec les bibliothèques LLVM/Clang de la version 7 à partir des paquets du système D’exploitation (ces bibliothèques sont utilisées pour la compilation de requêtes d’exécution). [\#3582](https://github.com/ClickHouse/ClickHouse/pull/3582) +- Correction de la construction avec les bibliothèques LLVM/Clang de la version 7 à partir des paquets du système D'exploitation (ces bibliothèques sont utilisées pour la compilation de requêtes d'exécution). [\#3582](https://github.com/ClickHouse/ClickHouse/pull/3582) ### Clickhouse Version 18.14.17, 2018-11-30 {#clickhouse-release-18-14-17-2018-11-30} #### Corrections De Bugs: {#bug-fixes-4} -- Correction de cas où le processus de pont ODBC ne s’est pas terminé avec le processus du serveur principal. [\#3642](https://github.com/ClickHouse/ClickHouse/pull/3642) +- Correction de cas où le processus de pont ODBC ne s'est pas terminé avec le processus du serveur principal. [\#3642](https://github.com/ClickHouse/ClickHouse/pull/3642) - Insertion synchrone fixe dans le `Distributed` table avec une liste des colonnes qui diffère de la liste des colonnes de la table distante. [\#3673](https://github.com/ClickHouse/ClickHouse/pull/3673) -- Correction d’une condition de concurrence rare qui peut conduire à un crash lors de la suppression D’une table MergeTree. [\#3643](https://github.com/ClickHouse/ClickHouse/pull/3643) -- Correction d’un blocage de requête dans le cas où la création de thread de requête échoue avec le `Resource temporarily unavailable` erreur. [\#3643](https://github.com/ClickHouse/ClickHouse/pull/3643) -- Fixe de l’analyse de l’ `ENGINE` clause lorsque le `CREATE AS table` la syntaxe a été utilisée et `ENGINE` la clause a été spécifiée avant le `AS table` (l’erreur a entraîné en ignorant le moteur). [\#3692](https://github.com/ClickHouse/ClickHouse/pull/3692) +- Correction d'une condition de concurrence rare qui peut conduire à un crash lors de la suppression D'une table MergeTree. [\#3643](https://github.com/ClickHouse/ClickHouse/pull/3643) +- Correction d'un blocage de requête dans le cas où la création de thread de requête échoue avec le `Resource temporarily unavailable` erreur. [\#3643](https://github.com/ClickHouse/ClickHouse/pull/3643) +- Fixe de l'analyse de l' `ENGINE` clause lorsque le `CREATE AS table` la syntaxe a été utilisée et `ENGINE` la clause a été spécifiée avant le `AS table` (l'erreur a entraîné en ignorant le moteur). [\#3692](https://github.com/ClickHouse/ClickHouse/pull/3692) ### Clickhouse Version 18.14.15, 2018-11-21 {#clickhouse-release-18-14-15-2018-11-21} @@ -174,17 +174,17 @@ toc_title: '2018' #### Corrections De Bugs: {#bug-fixes-7} -- Correction de l’ `Block structure mismatch in MergingSorted stream` erreur. [\#3162](https://github.com/ClickHouse/ClickHouse/issues/3162) +- Correction de l' `Block structure mismatch in MergingSorted stream` erreur. [\#3162](https://github.com/ClickHouse/ClickHouse/issues/3162) - Fixe `ON CLUSTER` requêtes dans le cas où les connexions sécurisées ont été activées dans la configuration du cluster (le `` drapeau). [\#3465](https://github.com/ClickHouse/ClickHouse/pull/3465) -- Correction d’une erreur dans les requêtes utilisées `SAMPLE`, `PREWHERE` et les colonnes alias. [\#3543](https://github.com/ClickHouse/ClickHouse/pull/3543) -- Correction d’un rare `unknown compression method` erreur lors de la `min_bytes_to_use_direct_io` le réglage a été activé. [3544](https://github.com/ClickHouse/ClickHouse/pull/3544) +- Correction d'une erreur dans les requêtes utilisées `SAMPLE`, `PREWHERE` et les colonnes alias. [\#3543](https://github.com/ClickHouse/ClickHouse/pull/3543) +- Correction d'un rare `unknown compression method` erreur lors de la `min_bytes_to_use_direct_io` le réglage a été activé. [3544](https://github.com/ClickHouse/ClickHouse/pull/3544) #### Amélioration Des Performances: {#performance-improvements} -- Régression de performance fixe des requêtes avec `GROUP BY` de colonnes de type UInt16 ou Date lors de l’exécution sur les processeurs AMD EPYC. [Igor Lapko](https://github.com/ClickHouse/ClickHouse/pull/3512) +- Régression de performance fixe des requêtes avec `GROUP BY` de colonnes de type UInt16 ou Date lors de l'exécution sur les processeurs AMD EPYC. [Igor Lapko](https://github.com/ClickHouse/ClickHouse/pull/3512) - Correction de la régression des performances des requêtes qui traitent les chaînes longues. [\#3530](https://github.com/ClickHouse/ClickHouse/pull/3530) -#### Construire Des améliorations: {#build-improvements-3} +#### Construire Des Améliorations: {#build-improvements-3} - Améliorations pour simplifier la construction Arcadia. [\#3475](https://github.com/ClickHouse/ClickHouse/pull/3475), [\#3535](https://github.com/ClickHouse/ClickHouse/pull/3535) @@ -192,22 +192,22 @@ toc_title: '2018' #### Corrections De Bugs: {#bug-fixes-8} -- Correction d’un plantage lors de la jonction de deux sous-requêtes sans nom. [\#3505](https://github.com/ClickHouse/ClickHouse/pull/3505) -- Fixe générant des requêtes incorrectes (avec un vide `WHERE` clause) lors de l’interrogation de bases de données externes. [hotid](https://github.com/ClickHouse/ClickHouse/pull/3477) -- Correction en utilisant une valeur de délai d’attente incorrecte Dans les dictionnaires ODBC. [Marek Vavruša](https://github.com/ClickHouse/ClickHouse/pull/3511) +- Correction d'un plantage lors de la jonction de deux sous-requêtes sans nom. [\#3505](https://github.com/ClickHouse/ClickHouse/pull/3505) +- Fixe générant des requêtes incorrectes (avec un vide `WHERE` clause) lors de l'interrogation de bases de données externes. [hotid](https://github.com/ClickHouse/ClickHouse/pull/3477) +- Correction en utilisant une valeur de délai d'attente incorrecte Dans les dictionnaires ODBC. [Marek Vavruša](https://github.com/ClickHouse/ClickHouse/pull/3511) ### Clickhouse Version 18.14.11, 2018-10-29 {#clickhouse-release-18-14-11-2018-10-29} #### Corrections De Bugs: {#bug-fixes-9} -- Correction de l’erreur `Block structure mismatch in UNION stream: different number of columns` dans les requêtes LIMIT. [\#2156](https://github.com/ClickHouse/ClickHouse/issues/2156) -- Correction d’erreurs lors de la fusion de données dans des tables contenant des tableaux à l’intérieur de structures imbriquées. [\#3397](https://github.com/ClickHouse/ClickHouse/pull/3397) +- Correction de l'erreur `Block structure mismatch in UNION stream: different number of columns` dans les requêtes LIMIT. [\#2156](https://github.com/ClickHouse/ClickHouse/issues/2156) +- Correction d'erreurs lors de la fusion de données dans des tables contenant des tableaux à l'intérieur de structures imbriquées. [\#3397](https://github.com/ClickHouse/ClickHouse/pull/3397) - Correction de résultats de requête incorrects si le `merge_tree_uniform_read_distribution` paramètre est désactivé (il est activé par défaut). [\#3429](https://github.com/ClickHouse/ClickHouse/pull/3429) -- Correction d’une erreur sur les insertions à une table distribuée au format natif. [\#3411](https://github.com/ClickHouse/ClickHouse/issues/3411) +- Correction d'une erreur sur les insertions à une table distribuée au format natif. [\#3411](https://github.com/ClickHouse/ClickHouse/issues/3411) ### Clickhouse Version 18.14.10, 2018-10-23 {#clickhouse-release-18-14-10-2018-10-23} -- Le `compile_expressions` le paramètre (compilation JIT d’expressions) est désactivé par défaut. [\#3410](https://github.com/ClickHouse/ClickHouse/pull/3410) +- Le `compile_expressions` le paramètre (compilation JIT d'expressions) est désactivé par défaut. [\#3410](https://github.com/ClickHouse/ClickHouse/pull/3410) - Le `enable_optimize_predicate_expression` paramètre est désactivé par défaut. ### Clickhouse Version 18.14.9, 2018-10-16 {#clickhouse-release-18-14-9-2018-10-16} @@ -216,16 +216,16 @@ toc_title: '2018' - Le `WITH CUBE` le modificateur `GROUP BY` (la syntaxe alternative `GROUP BY CUBE(...)` est également disponible). [\#3172](https://github.com/ClickHouse/ClickHouse/pull/3172) - Ajouté le `formatDateTime` fonction. [Alexandr Krasheninnikov](https://github.com/ClickHouse/ClickHouse/pull/2770) -- Ajouté le `JDBC` tableau moteur et `jdbc` fonction table (nécessite l’installation de clickhouse-JDBC-bridge). [Alexandr Krasheninnikov](https://github.com/ClickHouse/ClickHouse/pull/3210) +- Ajouté le `JDBC` tableau moteur et `jdbc` fonction table (nécessite l'installation de clickhouse-JDBC-bridge). [Alexandr Krasheninnikov](https://github.com/ClickHouse/ClickHouse/pull/3210) - Ajout de fonctions pour travailler avec le numéro de semaine ISO: `toISOWeek`, `toISOYear`, `toStartOfISOYear`, et `toDayOfYear`. [\#3146](https://github.com/ClickHouse/ClickHouse/pull/3146) - Maintenant, vous pouvez utiliser `Nullable` colonnes pour `MySQL` et `ODBC` table. [\#3362](https://github.com/ClickHouse/ClickHouse/pull/3362) - Imbriquée structures de données peuvent être lues comme des objets imbriqués dans `JSONEachRow` format. Ajouté le `input_format_import_nested_json` paramètre. [Veloman Yunkan](https://github.com/ClickHouse/ClickHouse/pull/3144) -- Le traitement parallèle est disponible pour beaucoup `MATERIALIZED VIEW`s lors de l’insertion de données. Voir la `parallel_view_processing` paramètre. [Marek Vavruša](https://github.com/ClickHouse/ClickHouse/pull/3208) +- Le traitement parallèle est disponible pour beaucoup `MATERIALIZED VIEW`s lors de l'insertion de données. Voir la `parallel_view_processing` paramètre. [Marek Vavruša](https://github.com/ClickHouse/ClickHouse/pull/3208) - Ajouté le `SYSTEM FLUSH LOGS` requête (vidage forcé des journaux sur les tables système telles que `query_log`) [\#3321](https://github.com/ClickHouse/ClickHouse/pull/3321) - Maintenant, vous pouvez utiliser prédéfinis `database` et `table` macros lors de la déclaration `Replicated` table. [\#3251](https://github.com/ClickHouse/ClickHouse/pull/3251) -- A ajouté la capacité de lire `Decimal` valeurs de type en notation d’ingénierie (indiquant des puissances de dix). [\#3153](https://github.com/ClickHouse/ClickHouse/pull/3153) +- A ajouté la capacité de lire `Decimal` valeurs de type en notation d'ingénierie (indiquant des puissances de dix). [\#3153](https://github.com/ClickHouse/ClickHouse/pull/3153) -#### Caractéristiques expérimentales: {#experimental-features} +#### Caractéristiques Expérimentales: {#experimental-features} - Optimisation de la clause GROUP BY pour `LowCardinality data types.` [\#3138](https://github.com/ClickHouse/ClickHouse/pull/3138) - Calcul optimisé des expressions pour `LowCardinality data types.` [\#3200](https://github.com/ClickHouse/ClickHouse/pull/3200) @@ -233,71 +233,71 @@ toc_title: '2018' #### Amélioration: {#improvements-2} - Consommation de mémoire considérablement réduite pour les requêtes avec `ORDER BY` et `LIMIT`. Voir la `max_bytes_before_remerge_sort` paramètre. [\#3205](https://github.com/ClickHouse/ClickHouse/pull/3205) -- En l’absence de `JOIN` (`LEFT`, `INNER`, …), `INNER JOIN` est supposé. [\#3147](https://github.com/ClickHouse/ClickHouse/pull/3147) -- Qualifié astérisques fonctionner correctement dans les requêtes avec `JOIN`. [L’Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/3202) -- Le `ODBC` table engine choisit correctement la méthode de citation des identifiants dans le dialecte SQL d’une base de données distante. [Alexandr Krasheninnikov](https://github.com/ClickHouse/ClickHouse/pull/3210) +- En l'absence de `JOIN` (`LEFT`, `INNER`, …), `INNER JOIN` est supposé. [\#3147](https://github.com/ClickHouse/ClickHouse/pull/3147) +- Qualifié astérisques fonctionner correctement dans les requêtes avec `JOIN`. [L'Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/3202) +- Le `ODBC` table engine choisit correctement la méthode de citation des identifiants dans le dialecte SQL d'une base de données distante. [Alexandr Krasheninnikov](https://github.com/ClickHouse/ClickHouse/pull/3210) - Le `compile_expressions` le paramètre (compilation JIT des expressions) est activé par défaut. -- Correction du comportement pour la base de données/TABLE DROP simultanée si existe et créer une base de données/TABLE si N’existe pas. Auparavant, un `CREATE DATABASE ... IF NOT EXISTS` requête peut renvoyer le message d’erreur “File … already exists” et la `CREATE TABLE ... IF NOT EXISTS` et `DROP TABLE IF EXISTS` les requêtes peuvent revenir `Table ... is creating or attaching right now`. [\#3101](https://github.com/ClickHouse/ClickHouse/pull/3101) -- Les expressions LIKE ET IN avec une moitié droite constante sont passées au serveur distant lors de l’interrogation à partir de tables MySQL ou ODBC. [\#3182](https://github.com/ClickHouse/ClickHouse/pull/3182) -- Les comparaisons avec des expressions constantes dans une clause WHERE sont transmises au serveur distant lors de l’interrogation à partir de tables MySQL et ODBC. Auparavant, seules les comparaisons avec les constantes étaient passées. [\#3182](https://github.com/ClickHouse/ClickHouse/pull/3182) +- Correction du comportement pour la base de données/TABLE DROP simultanée si existe et créer une base de données/TABLE si N'existe pas. Auparavant, un `CREATE DATABASE ... IF NOT EXISTS` requête peut renvoyer le message d'erreur “File … already exists” et la `CREATE TABLE ... IF NOT EXISTS` et `DROP TABLE IF EXISTS` les requêtes peuvent revenir `Table ... is creating or attaching right now`. [\#3101](https://github.com/ClickHouse/ClickHouse/pull/3101) +- Les expressions LIKE ET IN avec une moitié droite constante sont passées au serveur distant lors de l'interrogation à partir de tables MySQL ou ODBC. [\#3182](https://github.com/ClickHouse/ClickHouse/pull/3182) +- Les comparaisons avec des expressions constantes dans une clause WHERE sont transmises au serveur distant lors de l'interrogation à partir de tables MySQL et ODBC. Auparavant, seules les comparaisons avec les constantes étaient passées. [\#3182](https://github.com/ClickHouse/ClickHouse/pull/3182) - Calcul Correct de la largeur de ligne dans le terminal pour `Pretty` formats, y compris les chaînes avec des hiéroglyphes. [Amos Oiseau](https://github.com/ClickHouse/ClickHouse/pull/3257). - `ON CLUSTER` peut être spécifié pour `ALTER UPDATE` requête. - Amélioration des performances de lecture des données `JSONEachRow` format. [\#3332](https://github.com/ClickHouse/ClickHouse/pull/3332) -- Ajouté synonymes pour le `LENGTH` et `CHARACTER_LENGTH` fonctions de compatibilité. Le `CONCAT` la fonction n’est plus sensible à la casse. [\#3306](https://github.com/ClickHouse/ClickHouse/pull/3306) +- Ajouté synonymes pour le `LENGTH` et `CHARACTER_LENGTH` fonctions de compatibilité. Le `CONCAT` la fonction n'est plus sensible à la casse. [\#3306](https://github.com/ClickHouse/ClickHouse/pull/3306) - Ajouté le `TIMESTAMP` synonyme de la `DateTime` type. [\#3390](https://github.com/ClickHouse/ClickHouse/pull/3390) -- Il y a toujours de l’espace réservé pour query\_id dans les journaux du serveur, même si la ligne de journal n’est pas liée à une requête. Cela facilite l’analyse des journaux de texte du serveur avec des outils tiers. -- La consommation de mémoire par une requête est enregistrée lorsqu’il dépasse le niveau suivant d’un nombre entier de gigaoctets. [\#3205](https://github.com/ClickHouse/ClickHouse/pull/3205) -- Ajout du mode de compatibilité pour le cas où la bibliothèque cliente qui utilise le protocole natif envoie moins de colonnes par erreur que ce que le serveur attend pour la requête D’insertion. Ce scénario était possible lors de l’utilisation de la bibliothèque clickhouse-cpp. Auparavant, ce scénario provoquait le plantage du serveur. [\#3171](https://github.com/ClickHouse/ClickHouse/pull/3171) -- Dans une expression WHERE définie par l’utilisateur `clickhouse-copier` vous pouvez maintenant utiliser un `partition_key` alias (pour un filtrage supplémentaire par partition de table source). Ceci est utile si le schéma de partitionnement change pendant la copie, mais ne change que légèrement. [\#3166](https://github.com/ClickHouse/ClickHouse/pull/3166) +- Il y a toujours de l'espace réservé pour query\_id dans les journaux du serveur, même si la ligne de journal n'est pas liée à une requête. Cela facilite l'analyse des journaux de texte du serveur avec des outils tiers. +- La consommation de mémoire par une requête est enregistrée lorsqu'il dépasse le niveau suivant d'un nombre entier de gigaoctets. [\#3205](https://github.com/ClickHouse/ClickHouse/pull/3205) +- Ajout du mode de compatibilité pour le cas où la bibliothèque cliente qui utilise le protocole natif envoie moins de colonnes par erreur que ce que le serveur attend pour la requête D'insertion. Ce scénario était possible lors de l'utilisation de la bibliothèque clickhouse-cpp. Auparavant, ce scénario provoquait le plantage du serveur. [\#3171](https://github.com/ClickHouse/ClickHouse/pull/3171) +- Dans une expression WHERE définie par l'utilisateur `clickhouse-copier` vous pouvez maintenant utiliser un `partition_key` alias (pour un filtrage supplémentaire par partition de table source). Ceci est utile si le schéma de partitionnement change pendant la copie, mais ne change que légèrement. [\#3166](https://github.com/ClickHouse/ClickHouse/pull/3166) - Le flux de travail du `Kafka` le moteur a été déplacé vers un pool de threads en arrière-plan afin de réduire automatiquement la vitesse de lecture des données à des charges élevées. [Marek Vavruša](https://github.com/ClickHouse/ClickHouse/pull/3215). - Support pour la lecture `Tuple` et `Nested` valeurs de structures comme `struct` dans le `Cap'n'Proto format`. [Marek Vavruša](https://github.com/ClickHouse/ClickHouse/pull/3216) - La liste des domaines de premier niveau pour les `firstSignificantSubdomain` la fonction inclut maintenant le domaine `biz`. [décaséale](https://github.com/ClickHouse/ClickHouse/pull/3219) - Dans la configuration des dictionnaires externes, `null_value` est interprétée comme la valeur du type de données par défaut. [\#3330](https://github.com/ClickHouse/ClickHouse/pull/3330) - Soutien pour le `intDiv` et `intDivOrZero` fonctions pour `Decimal`. [b48402e8](https://github.com/ClickHouse/ClickHouse/commit/b48402e8712e2b9b151e0eef8193811d433a1264) -- Soutien pour le `Date`, `DateTime`, `UUID`, et `Decimal` types comme clé pour le `sumMap` fonction d’agrégation. [\#3281](https://github.com/ClickHouse/ClickHouse/pull/3281) +- Soutien pour le `Date`, `DateTime`, `UUID`, et `Decimal` types comme clé pour le `sumMap` fonction d'agrégation. [\#3281](https://github.com/ClickHouse/ClickHouse/pull/3281) - Soutien pour le `Decimal` type de données dans les dictionnaires externes. [\#3324](https://github.com/ClickHouse/ClickHouse/pull/3324) - Soutien pour le `Decimal` type de données dans `SummingMergeTree` table. [\#3348](https://github.com/ClickHouse/ClickHouse/pull/3348) - Ajouté spécialisations pour `UUID` dans `if`. [\#3366](https://github.com/ClickHouse/ClickHouse/pull/3366) -- Réduit le nombre de `open` et `close` les appels système lors de la lecture d’un `MergeTree table`. [\#3283](https://github.com/ClickHouse/ClickHouse/pull/3283) -- A `TRUNCATE TABLE` requête peut être exécutée sur n’importe quel réplica (la requête est transmise au chef de réplique). [Kirill Shvakov](https://github.com/ClickHouse/ClickHouse/pull/3375) +- Réduit le nombre de `open` et `close` les appels système lors de la lecture d'un `MergeTree table`. [\#3283](https://github.com/ClickHouse/ClickHouse/pull/3283) +- A `TRUNCATE TABLE` requête peut être exécutée sur n'importe quel réplica (la requête est transmise au chef de réplique). [Kirill Shvakov](https://github.com/ClickHouse/ClickHouse/pull/3375) #### Corrections De Bugs: {#bug-fixes-10} -- Correction d’un problème avec `Dictionary` tables pour `range_hashed` dictionnaire. Cette erreur s’est produite dans la version 18.12.17. [\#1702](https://github.com/ClickHouse/ClickHouse/pull/1702) -- Correction d’une erreur lors du chargement `range_hashed` les dictionnaires (le message `Unsupported type Nullable (...)`). Cette erreur s’est produite dans la version 18.12.17. [\#3362](https://github.com/ClickHouse/ClickHouse/pull/3362) -- Corrigé des erreurs dans la `pointInPolygon` fonction due à l’accumulation de calculs inexacts pour les polygones avec un grand nombre de sommets situés à proximité les uns des autres. [\#3331](https://github.com/ClickHouse/ClickHouse/pull/3331) [\#3341](https://github.com/ClickHouse/ClickHouse/pull/3341) -- Si, après la fusion de parties de données, la somme de contrôle de la partie résultante diffère du résultat de la même fusion dans une autre réplique, le résultat de la fusion est supprimé et la partie de données est téléchargée à partir de l’autre réplique (c’est le comportement correct). Mais après avoir téléchargé la partie data, elle n’a pas pu être ajoutée à l’ensemble de travail en raison d’une erreur indiquant que la partie existe déjà (car la partie data a été supprimée avec un certain retard après la fusion). Cela a conduit à cycliques tente de télécharger les mêmes données. [\#3194](https://github.com/ClickHouse/ClickHouse/pull/3194) -- Correction d’un calcul incorrect de la consommation totale de mémoire par les requêtes (en raison d’un calcul incorrect, le `max_memory_usage_for_all_queries` le réglage n’a pas fonctionné correctement et le `MemoryTracking` métrique a une valeur incorrecte). Cette erreur s’est produite dans la version 18.12.13. [Marek Vavruša](https://github.com/ClickHouse/ClickHouse/pull/3344) -- Correction de la fonctionnalité de `CREATE TABLE ... ON CLUSTER ... AS SELECT ...` Cette erreur s’est produite dans la version 18.12.13. [\#3247](https://github.com/ClickHouse/ClickHouse/pull/3247) +- Correction d'un problème avec `Dictionary` tables pour `range_hashed` dictionnaire. Cette erreur s'est produite dans la version 18.12.17. [\#1702](https://github.com/ClickHouse/ClickHouse/pull/1702) +- Correction d'une erreur lors du chargement `range_hashed` les dictionnaires (le message `Unsupported type Nullable (...)`). Cette erreur s'est produite dans la version 18.12.17. [\#3362](https://github.com/ClickHouse/ClickHouse/pull/3362) +- Corrigé des erreurs dans la `pointInPolygon` fonction due à l'accumulation de calculs inexacts pour les polygones avec un grand nombre de sommets situés à proximité les uns des autres. [\#3331](https://github.com/ClickHouse/ClickHouse/pull/3331) [\#3341](https://github.com/ClickHouse/ClickHouse/pull/3341) +- Si, après la fusion de parties de données, la somme de contrôle de la partie résultante diffère du résultat de la même fusion dans une autre réplique, le résultat de la fusion est supprimé et la partie de données est téléchargée à partir de l'autre réplique (c'est le comportement correct). Mais après avoir téléchargé la partie data, elle n'a pas pu être ajoutée à l'ensemble de travail en raison d'une erreur indiquant que la partie existe déjà (car la partie data a été supprimée avec un certain retard après la fusion). Cela a conduit à cycliques tente de télécharger les mêmes données. [\#3194](https://github.com/ClickHouse/ClickHouse/pull/3194) +- Correction d'un calcul incorrect de la consommation totale de mémoire par les requêtes (en raison d'un calcul incorrect, le `max_memory_usage_for_all_queries` le réglage n'a pas fonctionné correctement et le `MemoryTracking` métrique a une valeur incorrecte). Cette erreur s'est produite dans la version 18.12.13. [Marek Vavruša](https://github.com/ClickHouse/ClickHouse/pull/3344) +- Correction de la fonctionnalité de `CREATE TABLE ... ON CLUSTER ... AS SELECT ...` Cette erreur s'est produite dans la version 18.12.13. [\#3247](https://github.com/ClickHouse/ClickHouse/pull/3247) - Correction de la préparation inutile des structures de données pour `JOIN`s sur le serveur qui initie la requête si `JOIN` est effectué uniquement sur des serveurs distants. [\#3340](https://github.com/ClickHouse/ClickHouse/pull/3340) - Correction de bugs dans le `Kafka` engine: blocages après les exceptions lors du démarrage de la lecture des données, et verrous à la fin [Marek Vavruša](https://github.com/ClickHouse/ClickHouse/pull/3215). -- Pour `Kafka` tableaux, le facultatif `schema` le paramètre n’a pas été passé (le schéma du `Cap'n'Proto` format). [Vojtech Splichal](https://github.com/ClickHouse/ClickHouse/pull/3150) -- Si L’ensemble des serveurs ZooKeeper a des serveurs qui acceptent la connexion mais la ferment immédiatement au lieu de répondre à la prise de contact, ClickHouse choisit de connecter un autre serveur. Auparavant, cela produisait l’erreur `Cannot read all data. Bytes read: 0. Bytes expected: 4.` et le serveur ne pouvait pas commencer. [8218cf3a](https://github.com/ClickHouse/ClickHouse/commit/8218cf3a5f39a43401953769d6d12a0bb8d29da9) -- Si L’ensemble des serveurs ZooKeeper contient des serveurs pour lesquels la requête DNS renvoie une erreur, ces serveurs sont ignorés. [17b8e209](https://github.com/ClickHouse/ClickHouse/commit/17b8e209221061325ad7ba0539f03c6e65f87f29) -- Conversion de type fixe entre `Date` et `DateTime` lors de l’insertion de données dans le `VALUES` format (si `input_format_values_interpret_expressions = 1`). Auparavant, la conversion était effectuée entre la valeur numérique du nombre de jours dans Unix Epoch time et L’horodatage Unix, ce qui conduisait à des résultats inattendus. [\#3229](https://github.com/ClickHouse/ClickHouse/pull/3229) +- Pour `Kafka` tableaux, le facultatif `schema` le paramètre n'a pas été passé (le schéma du `Cap'n'Proto` format). [Vojtech Splichal](https://github.com/ClickHouse/ClickHouse/pull/3150) +- Si L'ensemble des serveurs ZooKeeper a des serveurs qui acceptent la connexion mais la ferment immédiatement au lieu de répondre à la prise de contact, ClickHouse choisit de connecter un autre serveur. Auparavant, cela produisait l'erreur `Cannot read all data. Bytes read: 0. Bytes expected: 4.` et le serveur ne pouvait pas commencer. [8218cf3a](https://github.com/ClickHouse/ClickHouse/commit/8218cf3a5f39a43401953769d6d12a0bb8d29da9) +- Si L'ensemble des serveurs ZooKeeper contient des serveurs pour lesquels la requête DNS renvoie une erreur, ces serveurs sont ignorés. [17b8e209](https://github.com/ClickHouse/ClickHouse/commit/17b8e209221061325ad7ba0539f03c6e65f87f29) +- Conversion de type fixe entre `Date` et `DateTime` lors de l'insertion de données dans le `VALUES` format (si `input_format_values_interpret_expressions = 1`). Auparavant, la conversion était effectuée entre la valeur numérique du nombre de jours dans Unix Epoch time et L'horodatage Unix, ce qui conduisait à des résultats inattendus. [\#3229](https://github.com/ClickHouse/ClickHouse/pull/3229) - Conversion de type corrigée entre `Decimal` et des nombres entiers. [\#3211](https://github.com/ClickHouse/ClickHouse/pull/3211) -- Corrigé des erreurs dans la `enable_optimize_predicate_expression` paramètre. [L’Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/3231) -- Correction d’une erreur D’analyse au format CSV avec des nombres à virgule flottante si un séparateur CSV non par défaut est utilisé, tel que `;` [\#3155](https://github.com/ClickHouse/ClickHouse/pull/3155) -- Correction de l’ `arrayCumSumNonNegative` fonction (il n’accumule pas de valeurs négatives si l’accumulateur est inférieur à zéro). [Aleksey Studnev](https://github.com/ClickHouse/ClickHouse/pull/3163) -- Fixe comment `Merge` les tables de travail sur le dessus de `Distributed` tables lors de l’utilisation `PREWHERE`. [\#3165](https://github.com/ClickHouse/ClickHouse/pull/3165) -- Corrections de bugs dans l’ `ALTER UPDATE` requête. +- Corrigé des erreurs dans la `enable_optimize_predicate_expression` paramètre. [L'Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/3231) +- Correction d'une erreur D'analyse au format CSV avec des nombres à virgule flottante si un séparateur CSV non par défaut est utilisé, tel que `;` [\#3155](https://github.com/ClickHouse/ClickHouse/pull/3155) +- Correction de l' `arrayCumSumNonNegative` fonction (il n'accumule pas de valeurs négatives si l'accumulateur est inférieur à zéro). [Aleksey Studnev](https://github.com/ClickHouse/ClickHouse/pull/3163) +- Fixe comment `Merge` les tables de travail sur le dessus de `Distributed` tables lors de l'utilisation `PREWHERE`. [\#3165](https://github.com/ClickHouse/ClickHouse/pull/3165) +- Corrections de bugs dans l' `ALTER UPDATE` requête. - Correction de bugs dans le `odbc` fonction de table apparue dans la version 18.12. [\#3197](https://github.com/ClickHouse/ClickHouse/pull/3197) -- Correction du fonctionnement des fonctions d’agrégat avec `StateArray` combinators. [\#3188](https://github.com/ClickHouse/ClickHouse/pull/3188) -- Correction d’un crash lors de la division d’une `Decimal` valeur par zéro. [69dd6609](https://github.com/ClickHouse/ClickHouse/commit/69dd6609193beb4e7acd3e6ad216eca0ccfb8179) +- Correction du fonctionnement des fonctions d'agrégat avec `StateArray` combinators. [\#3188](https://github.com/ClickHouse/ClickHouse/pull/3188) +- Correction d'un crash lors de la division d'une `Decimal` valeur par zéro. [69dd6609](https://github.com/ClickHouse/ClickHouse/commit/69dd6609193beb4e7acd3e6ad216eca0ccfb8179) - Sortie fixe des types pour les opérations utilisant `Decimal` et des arguments entiers. [\#3224](https://github.com/ClickHouse/ClickHouse/pull/3224) - Correction du segfault pendant `GROUP BY` sur `Decimal128`. [3359ba06](https://github.com/ClickHouse/ClickHouse/commit/3359ba06c39fcd05bfdb87d6c64154819621e13a) -- Le `log_query_threads` le paramètre (journalisation des informations sur chaque thread d’exécution de la requête) ne prend effet que si `log_queries` l’option (journalisation des informations sur les requêtes) est définie sur 1. Depuis le `log_query_threads` l’option est activée par défaut, les informations sur les threads ont déjà été enregistrées même si la journalisation des requêtes a été désactivée. [\#3241](https://github.com/ClickHouse/ClickHouse/pull/3241) -- Correction d’une erreur dans le fonctionnement distribué de la fonction d’agrégat quantiles (le message d’erreur `Not found column quantile...`). [292a8855](https://github.com/ClickHouse/ClickHouse/commit/292a885533b8e3b41ce8993867069d14cbd5a664) -- Correction du problème de compatibilité lorsque vous travaillez sur un cluster de serveurs de version 18.12.17 et de serveurs plus anciens en même temps. Pour les requêtes distribuées avec des clés GROUP BY de longueur fixe et non fixe, s’il y avait une grande quantité de données à agréger, les données renvoyées n’étaient pas toujours entièrement agrégées (deux lignes différentes contenaient les mêmes clés d’agrégation). [\#3254](https://github.com/ClickHouse/ClickHouse/pull/3254) -- Manipulation fixe des substitutions dans `clickhouse-performance-test` si la requête ne contient qu’une partie des substitutions déclaré dans le test. [\#3263](https://github.com/ClickHouse/ClickHouse/pull/3263) -- Correction d’une erreur lors de l’utilisation `FINAL` avec `PREWHERE`. [\#3298](https://github.com/ClickHouse/ClickHouse/pull/3298) -- Correction d’une erreur lors de l’utilisation `PREWHERE` sur les colonnes qui ont été ajoutées pendant `ALTER`. [\#3298](https://github.com/ClickHouse/ClickHouse/pull/3298) -- Ajout d’une vérification de l’absence de `arrayJoin` pour `DEFAULT` et `MATERIALIZED` expression. Précédemment, `arrayJoin` conduit à une erreur lors de l’insertion de données. [\#3337](https://github.com/ClickHouse/ClickHouse/pull/3337) -- Ajout d’une vérification de l’absence de `arrayJoin` dans un `PREWHERE` clause. Auparavant, cela a conduit à des messages comme `Size ... doesn't match` ou `Unknown compression method` lors de l’exécution de requêtes. [\#3357](https://github.com/ClickHouse/ClickHouse/pull/3357) -- Correction de segfault qui pourrait se produire dans de rares cas après l’optimisation qui a remplacé et chaînes des évaluations d’égalité avec l’expression IN correspondante. [liuyimin-bytedance](https://github.com/ClickHouse/ClickHouse/pull/3339) -- Corrections mineures à `clickhouse-benchmark`: auparavant, les informations client n’étaient pas envoyées au serveur; maintenant, le nombre de requêtes exécutées est calculé plus précisément lors de l’arrêt et pour limiter le nombre d’itérations. [\#3351](https://github.com/ClickHouse/ClickHouse/pull/3351) [\#3352](https://github.com/ClickHouse/ClickHouse/pull/3352) +- Le `log_query_threads` le paramètre (journalisation des informations sur chaque thread d'exécution de la requête) ne prend effet que si `log_queries` l'option (journalisation des informations sur les requêtes) est définie sur 1. Depuis le `log_query_threads` l'option est activée par défaut, les informations sur les threads ont déjà été enregistrées même si la journalisation des requêtes a été désactivée. [\#3241](https://github.com/ClickHouse/ClickHouse/pull/3241) +- Correction d'une erreur dans le fonctionnement distribué de la fonction d'agrégat quantiles (le message d'erreur `Not found column quantile...`). [292a8855](https://github.com/ClickHouse/ClickHouse/commit/292a885533b8e3b41ce8993867069d14cbd5a664) +- Correction du problème de compatibilité lorsque vous travaillez sur un cluster de serveurs de version 18.12.17 et de serveurs plus anciens en même temps. Pour les requêtes distribuées avec des clés GROUP BY de longueur fixe et non fixe, s'il y avait une grande quantité de données à agréger, les données renvoyées n'étaient pas toujours entièrement agrégées (deux lignes différentes contenaient les mêmes clés d'agrégation). [\#3254](https://github.com/ClickHouse/ClickHouse/pull/3254) +- Manipulation fixe des substitutions dans `clickhouse-performance-test` si la requête ne contient qu'une partie des substitutions déclaré dans le test. [\#3263](https://github.com/ClickHouse/ClickHouse/pull/3263) +- Correction d'une erreur lors de l'utilisation `FINAL` avec `PREWHERE`. [\#3298](https://github.com/ClickHouse/ClickHouse/pull/3298) +- Correction d'une erreur lors de l'utilisation `PREWHERE` sur les colonnes qui ont été ajoutées pendant `ALTER`. [\#3298](https://github.com/ClickHouse/ClickHouse/pull/3298) +- Ajout d'une vérification de l'absence de `arrayJoin` pour `DEFAULT` et `MATERIALIZED` expression. Précédemment, `arrayJoin` conduit à une erreur lors de l'insertion de données. [\#3337](https://github.com/ClickHouse/ClickHouse/pull/3337) +- Ajout d'une vérification de l'absence de `arrayJoin` dans un `PREWHERE` clause. Auparavant, cela a conduit à des messages comme `Size ... doesn't match` ou `Unknown compression method` lors de l'exécution de requêtes. [\#3357](https://github.com/ClickHouse/ClickHouse/pull/3357) +- Correction de segfault qui pourrait se produire dans de rares cas après l'optimisation qui a remplacé et chaînes des évaluations d'égalité avec l'expression IN correspondante. [liuyimin-bytedance](https://github.com/ClickHouse/ClickHouse/pull/3339) +- Corrections mineures à `clickhouse-benchmark`: auparavant, les informations client n'étaient pas envoyées au serveur; maintenant, le nombre de requêtes exécutées est calculé plus précisément lors de l'arrêt et pour limiter le nombre d'itérations. [\#3351](https://github.com/ClickHouse/ClickHouse/pull/3351) [\#3352](https://github.com/ClickHouse/ClickHouse/pull/3352) -#### Modifications Incompatibles En arrière: {#backward-incompatible-changes-1} +#### Modifications Incompatibles En Arrière: {#backward-incompatible-changes-1} - Enlevé le `allow_experimental_decimal_type` option. Le `Decimal` type de données est disponible pour utilisation par défaut. [\#3329](https://github.com/ClickHouse/ClickHouse/pull/3329) @@ -308,145 +308,145 @@ toc_title: '2018' #### Nouveauté: {#new-features-2} - `invalidate_query` (la possibilité de spécifier une requête pour vérifier si un dictionnaire externe doit être mis à jour) est implémentée pour `clickhouse` source. [\#3126](https://github.com/ClickHouse/ClickHouse/pull/3126) -- Ajout de la possibilité d’utiliser `UInt*`, `Int*`, et `DateTime` types de données (avec le `Date` le type) comme un `range_hashed` clé de dictionnaire externe qui définit les limites des plages. Maintenant `NULL` peut être utilisé pour désigner un intervalle ouvert. [Vasily Nemkov](https://github.com/ClickHouse/ClickHouse/pull/3123) -- Le `Decimal` type prend maintenant en charge `var*` et `stddev*` les fonctions d’agrégation. [\#3129](https://github.com/ClickHouse/ClickHouse/pull/3129) +- Ajout de la possibilité d'utiliser `UInt*`, `Int*`, et `DateTime` types de données (avec le `Date` le type) comme un `range_hashed` clé de dictionnaire externe qui définit les limites des plages. Maintenant `NULL` peut être utilisé pour désigner un intervalle ouvert. [Vasily Nemkov](https://github.com/ClickHouse/ClickHouse/pull/3123) +- Le `Decimal` type prend maintenant en charge `var*` et `stddev*` les fonctions d'agrégation. [\#3129](https://github.com/ClickHouse/ClickHouse/pull/3129) - Le `Decimal` type prend désormais en charge les fonctions mathématiques (`exp`, `sin` et ainsi de suite.) [\#3129](https://github.com/ClickHouse/ClickHouse/pull/3129) - Le `system.part_log` la table a maintenant le `partition_id` colonne. [\#3089](https://github.com/ClickHouse/ClickHouse/pull/3089) #### Corrections De Bugs: {#bug-fixes-11} -- `Merge` fonctionne maintenant correctement sur `Distributed` table. [L’Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/3159) -- Incompatibilité fixe (dépendance inutile sur le `glibc` version) qui a rendu impossible l’exécution de ClickHouse sur `Ubuntu Precise` et les anciennes versions. L’incompatibilité est apparue dans la version 18.12.13. [\#3130](https://github.com/ClickHouse/ClickHouse/pull/3130) -- Corrigé des erreurs dans la `enable_optimize_predicate_expression` paramètre. [L’Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/3107) -- Correction d’un problème mineur de rétrocompatibilité apparu lors de l’utilisation d’un cluster de répliques sur des versions antérieures au 18.12.13 et de la création simultanée d’une nouvelle réplique d’une table sur un serveur avec une version plus récente (indiquée dans le message `Can not clone replica, because the ... updated to new ClickHouse version`, ce qui est logique, mais ne devrait pas arriver). [\#3122](https://github.com/ClickHouse/ClickHouse/pull/3122) +- `Merge` fonctionne maintenant correctement sur `Distributed` table. [L'Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/3159) +- Incompatibilité fixe (dépendance inutile sur le `glibc` version) qui a rendu impossible l'exécution de ClickHouse sur `Ubuntu Precise` et les anciennes versions. L'incompatibilité est apparue dans la version 18.12.13. [\#3130](https://github.com/ClickHouse/ClickHouse/pull/3130) +- Corrigé des erreurs dans la `enable_optimize_predicate_expression` paramètre. [L'Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/3107) +- Correction d'un problème mineur de rétrocompatibilité apparu lors de l'utilisation d'un cluster de répliques sur des versions antérieures au 18.12.13 et de la création simultanée d'une nouvelle réplique d'une table sur un serveur avec une version plus récente (indiquée dans le message `Can not clone replica, because the ... updated to new ClickHouse version`, ce qui est logique, mais ne devrait pas arriver). [\#3122](https://github.com/ClickHouse/ClickHouse/pull/3122) -#### Modifications Incompatibles En arrière: {#backward-incompatible-changes-2} +#### Modifications Incompatibles En Arrière: {#backward-incompatible-changes-2} -- Le `enable_optimize_predicate_expression` option est activée par défaut (ce qui est plutôt optimiste). Si des erreurs d’analyse de requête se produisent liées à la recherche des noms de colonnes, définissez `enable_optimize_predicate_expression` à 0. [L’Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/3107) +- Le `enable_optimize_predicate_expression` option est activée par défaut (ce qui est plutôt optimiste). Si des erreurs d'analyse de requête se produisent liées à la recherche des noms de colonnes, définissez `enable_optimize_predicate_expression` à 0. [L'Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/3107) ### Clickhouse Version 18.12.14, 2018-09-13 {#clickhouse-release-18-12-14-2018-09-13} #### Nouveauté: {#new-features-3} - Ajout du support pour `ALTER UPDATE` requête. [\#3035](https://github.com/ClickHouse/ClickHouse/pull/3035) -- Ajouté le `allow_ddl` option, qui limite l’accès de L’utilisateur aux requêtes DDL. [\#3104](https://github.com/ClickHouse/ClickHouse/pull/3104) +- Ajouté le `allow_ddl` option, qui limite l'accès de L'utilisateur aux requêtes DDL. [\#3104](https://github.com/ClickHouse/ClickHouse/pull/3104) - Ajouté le `min_merge_bytes_to_use_direct_io` option pour `MergeTree` moteurs, qui vous permet de définir un seuil pour la taille totale de la fusion (quand au-dessus du seuil, les fichiers de partie de données seront traités en utilisant O\_DIRECT). [\#3117](https://github.com/ClickHouse/ClickHouse/pull/3117) - Le `system.merges` la table système contient maintenant `partition_id` colonne. [\#3099](https://github.com/ClickHouse/ClickHouse/pull/3099) #### Amélioration {#improvements-3} -- Si une partie de données reste inchangée pendant la mutation, elle n’est pas téléchargée par les répliques. [\#3103](https://github.com/ClickHouse/ClickHouse/pull/3103) +- Si une partie de données reste inchangée pendant la mutation, elle n'est pas téléchargée par les répliques. [\#3103](https://github.com/ClickHouse/ClickHouse/pull/3103) - La saisie semi-automatique est disponible pour les noms de paramètres lorsque vous travaillez avec `clickhouse-client`. [\#3106](https://github.com/ClickHouse/ClickHouse/pull/3106) #### Corrections De Bugs: {#bug-fixes-12} -- Ajouter un chèque pour les montants des tableaux sont des éléments de `Nested` les champs de type lors de l’insertion. [\#3118](https://github.com/ClickHouse/ClickHouse/pull/3118) -- Correction d’une erreur de mise à jour des dictionnaires externes `ODBC` source et `hashed` stockage. Cette erreur s’est produite dans la version 18.12.13. -- Correction d’un plantage lors de la création d’une table temporaire à partir d’une requête `IN` condition. [L’Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/3098) -- Correction d’une erreur dans les fonctions d’agrégation pour les tableaux peuvent avoir `NULL` élément. [L’Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/3097) +- Ajouter un chèque pour les montants des tableaux sont des éléments de `Nested` les champs de type lors de l'insertion. [\#3118](https://github.com/ClickHouse/ClickHouse/pull/3118) +- Correction d'une erreur de mise à jour des dictionnaires externes `ODBC` source et `hashed` stockage. Cette erreur s'est produite dans la version 18.12.13. +- Correction d'un plantage lors de la création d'une table temporaire à partir d'une requête `IN` condition. [L'Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/3098) +- Correction d'une erreur dans les fonctions d'agrégation pour les tableaux peuvent avoir `NULL` élément. [L'Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/3097) ### Clickhouse Version 18.12.13, 2018-09-10 {#clickhouse-release-18-12-13-2018-09-10} #### Nouveauté: {#new-features-4} -- Ajouté le `DECIMAL(digits, scale)` type de données (`Decimal32(scale)`, `Decimal64(scale)`, `Decimal128(scale)`). Pour l’activer, utilisez le paramètre `allow_experimental_decimal_type`. [\#2846](https://github.com/ClickHouse/ClickHouse/pull/2846) [\#2970](https://github.com/ClickHouse/ClickHouse/pull/2970) [\#3008](https://github.com/ClickHouse/ClickHouse/pull/3008) [\#3047](https://github.com/ClickHouse/ClickHouse/pull/3047) +- Ajouté le `DECIMAL(digits, scale)` type de données (`Decimal32(scale)`, `Decimal64(scale)`, `Decimal128(scale)`). Pour l'activer, utilisez le paramètre `allow_experimental_decimal_type`. [\#2846](https://github.com/ClickHouse/ClickHouse/pull/2846) [\#2970](https://github.com/ClickHouse/ClickHouse/pull/2970) [\#3008](https://github.com/ClickHouse/ClickHouse/pull/3008) [\#3047](https://github.com/ClickHouse/ClickHouse/pull/3047) - Nouveau `WITH ROLLUP` le modificateur `GROUP BY` (syntaxe alternative: `GROUP BY ROLLUP(...)`). [\#2948](https://github.com/ClickHouse/ClickHouse/pull/2948) -- Dans les requêtes avec jointure, le caractère étoile se développe en une liste de colonnes dans toutes les tables, conformément à la norme SQL. Vous pouvez restaurer l’ancien comportement en paramètre `asterisk_left_columns_only` à 1 au niveau de la configuration utilisateur. [L’Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/2787) -- Ajout du support pour joindre avec les fonctions de table. [L’Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/2907) +- Dans les requêtes avec jointure, le caractère étoile se développe en une liste de colonnes dans toutes les tables, conformément à la norme SQL. Vous pouvez restaurer l'ancien comportement en paramètre `asterisk_left_columns_only` à 1 au niveau de la configuration utilisateur. [L'Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/2787) +- Ajout du support pour joindre avec les fonctions de table. [L'Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/2907) - Saisie semi-automatique en appuyant sur Tab dans clickhouse-client. [Sergey Shcherbin](https://github.com/ClickHouse/ClickHouse/pull/2447) - Ctrl + C dans clickhouse-client efface une requête qui a été entrée. [\#2877](https://github.com/ClickHouse/ClickHouse/pull/2877) - Ajouté le `join_default_strictness` paramètre (valeurs: `"`, `'any'`, `'all'`). Cela vous permet de ne pas préciser `ANY` ou `ALL` pour `JOIN`. [\#2982](https://github.com/ClickHouse/ClickHouse/pull/2982) -- Chaque ligne du journal du serveur associée au traitement de la requête affiche L’ID de la requête. [\#2482](https://github.com/ClickHouse/ClickHouse/pull/2482) -- Maintenant, vous pouvez obtenir des journaux d’exécution de requête dans clickhouse-client (utilisez le `send_logs_level` paramètre). Avec le traitement des requêtes distribuées, les journaux sont cascadés à partir de tous les serveurs. [\#2482](https://github.com/ClickHouse/ClickHouse/pull/2482) -- Le `system.query_log` et `system.processes` (`SHOW PROCESSLIST` les tableaux ont maintenant des informations sur tous les paramètres modifiés lorsque vous exécutez une requête (la structure imbriquée de l’ `Settings` données). Ajouté le `log_query_settings` paramètre. [\#2482](https://github.com/ClickHouse/ClickHouse/pull/2482) -- Le `system.query_log` et `system.processes` les tables affichent désormais des informations sur le nombre de threads participant à l’exécution de la requête (voir `thread_numbers` colonne). [\#2482](https://github.com/ClickHouse/ClickHouse/pull/2482) -- Ajouter `ProfileEvents` compteurs qui mesurent le temps passé à lire et à écrire sur le réseau et à lire et à écrire sur le disque, le nombre d’erreurs réseau et le temps passé à attendre lorsque la bande passante réseau est limitée. [\#2482](https://github.com/ClickHouse/ClickHouse/pull/2482) -- Ajouter `ProfileEvents`compteurs qui contiennent les métriques système de rusage (vous pouvez les utiliser pour obtenir des informations sur L’utilisation du processeur dans l’espace utilisateur et le noyau, les erreurs de page et les commutateurs de contexte), ainsi que les métriques taskstats (utilisez-les pour obtenir des informations sur le temps d’attente d’E/S, le temps D’attente du processeur et [\#2482](https://github.com/ClickHouse/ClickHouse/pull/2482) -- Le `ProfileEvents` les compteurs sont appliqués globalement et pour chaque requête, ainsi que pour chaque thread d’exécution de requête, ce qui vous permet de profiler la consommation de ressources par requête en détail. [\#2482](https://github.com/ClickHouse/ClickHouse/pull/2482) -- Ajouté le `system.query_thread_log` table, qui contient des informations sur chaque thread d’exécution de requête. Ajouté le `log_query_threads` paramètre. [\#2482](https://github.com/ClickHouse/ClickHouse/pull/2482) +- Chaque ligne du journal du serveur associée au traitement de la requête affiche L'ID de la requête. [\#2482](https://github.com/ClickHouse/ClickHouse/pull/2482) +- Maintenant, vous pouvez obtenir des journaux d'exécution de requête dans clickhouse-client (utilisez le `send_logs_level` paramètre). Avec le traitement des requêtes distribuées, les journaux sont cascadés à partir de tous les serveurs. [\#2482](https://github.com/ClickHouse/ClickHouse/pull/2482) +- Le `system.query_log` et `system.processes` (`SHOW PROCESSLIST` les tableaux ont maintenant des informations sur tous les paramètres modifiés lorsque vous exécutez une requête (la structure imbriquée de l' `Settings` données). Ajouté le `log_query_settings` paramètre. [\#2482](https://github.com/ClickHouse/ClickHouse/pull/2482) +- Le `system.query_log` et `system.processes` les tables affichent désormais des informations sur le nombre de threads participant à l'exécution de la requête (voir `thread_numbers` colonne). [\#2482](https://github.com/ClickHouse/ClickHouse/pull/2482) +- Ajouter `ProfileEvents` compteurs qui mesurent le temps passé à lire et à écrire sur le réseau et à lire et à écrire sur le disque, le nombre d'erreurs réseau et le temps passé à attendre lorsque la bande passante réseau est limitée. [\#2482](https://github.com/ClickHouse/ClickHouse/pull/2482) +- Ajouter `ProfileEvents`compteurs qui contiennent les métriques système de rusage (vous pouvez les utiliser pour obtenir des informations sur L'utilisation du processeur dans l'espace utilisateur et le noyau, les erreurs de page et les commutateurs de contexte), ainsi que les métriques taskstats (utilisez-les pour obtenir des informations sur le temps d'attente d'E/S, le temps D'attente du processeur et [\#2482](https://github.com/ClickHouse/ClickHouse/pull/2482) +- Le `ProfileEvents` les compteurs sont appliqués globalement et pour chaque requête, ainsi que pour chaque thread d'exécution de requête, ce qui vous permet de profiler la consommation de ressources par requête en détail. [\#2482](https://github.com/ClickHouse/ClickHouse/pull/2482) +- Ajouté le `system.query_thread_log` table, qui contient des informations sur chaque thread d'exécution de requête. Ajouté le `log_query_threads` paramètre. [\#2482](https://github.com/ClickHouse/ClickHouse/pull/2482) - Le `system.metrics` et `system.events` les tables ont maintenant une documentation intégrée. [\#3016](https://github.com/ClickHouse/ClickHouse/pull/3016) - Ajouté le `arrayEnumerateDense` fonction. [Amos Oiseau](https://github.com/ClickHouse/ClickHouse/pull/2975) - Ajouté le `arrayCumSumNonNegative` et `arrayDifference` fonction. [Aleksey Studnev](https://github.com/ClickHouse/ClickHouse/pull/2942) -- Ajouté le `retention` fonction d’agrégation. [Sundy Li](https://github.com/ClickHouse/ClickHouse/pull/2887) -- Vous pouvez maintenant ajouter (fusionner) des états de fonctions d’agrégat en utilisant l’opérateur plus et multiplier les états de fonctions d’agrégat par une constante non négative. [\#3062](https://github.com/ClickHouse/ClickHouse/pull/3062) [\#3034](https://github.com/ClickHouse/ClickHouse/pull/3034) +- Ajouté le `retention` fonction d'agrégation. [Sundy Li](https://github.com/ClickHouse/ClickHouse/pull/2887) +- Vous pouvez maintenant ajouter (fusionner) des états de fonctions d'agrégat en utilisant l'opérateur plus et multiplier les états de fonctions d'agrégat par une constante non négative. [\#3062](https://github.com/ClickHouse/ClickHouse/pull/3062) [\#3034](https://github.com/ClickHouse/ClickHouse/pull/3034) - Les Tables de la famille MergeTree ont maintenant la colonne virtuelle `_partition_id`. [\#3089](https://github.com/ClickHouse/ClickHouse/pull/3089) -#### Caractéristiques expérimentales: {#experimental-features-1} +#### Caractéristiques Expérimentales: {#experimental-features-1} - Ajouté le `LowCardinality(T)` type de données. Ce type de données crée automatiquement un dictionnaire local de valeurs et permet le traitement des données sans déballer le dictionnaire. [\#2830](https://github.com/ClickHouse/ClickHouse/pull/2830) -- Ajout d’un cache de fonctions compilées JIT et d’un compteur pour le nombre d’utilisations avant la compilation. Pour compiler des expressions JIT, activez `compile_expressions` paramètre. [\#2990](https://github.com/ClickHouse/ClickHouse/pull/2990) [\#3077](https://github.com/ClickHouse/ClickHouse/pull/3077) +- Ajout d'un cache de fonctions compilées JIT et d'un compteur pour le nombre d'utilisations avant la compilation. Pour compiler des expressions JIT, activez `compile_expressions` paramètre. [\#2990](https://github.com/ClickHouse/ClickHouse/pull/2990) [\#3077](https://github.com/ClickHouse/ClickHouse/pull/3077) #### Amélioration: {#improvements-4} -- Correction du problème avec l’accumulation illimitée du journal de réplication quand il y a des répliques abandonnées. Ajout d’un mode de récupération efficace pour les répliques avec un long décalage. -- Amélioration des performances de `GROUP BY` avec l’agrégation de plusieurs champs lorsque l’un d’eux est une chaîne et les autres sont de longueur fixe. -- Amélioration des performances lors de l’utilisation `PREWHERE` et avec transfert implicite d’expressions dans `PREWHERE`. -- Amélioration des performances d’analyse pour les formats de texte (`CSV`, `TSV`). [Amos Oiseau](https://github.com/ClickHouse/ClickHouse/pull/2977) [\#2980](https://github.com/ClickHouse/ClickHouse/pull/2980) +- Correction du problème avec l'accumulation illimitée du journal de réplication quand il y a des répliques abandonnées. Ajout d'un mode de récupération efficace pour les répliques avec un long décalage. +- Amélioration des performances de `GROUP BY` avec l'agrégation de plusieurs champs lorsque l'un d'eux est une chaîne et les autres sont de longueur fixe. +- Amélioration des performances lors de l'utilisation `PREWHERE` et avec transfert implicite d'expressions dans `PREWHERE`. +- Amélioration des performances d'analyse pour les formats de texte (`CSV`, `TSV`). [Amos Oiseau](https://github.com/ClickHouse/ClickHouse/pull/2977) [\#2980](https://github.com/ClickHouse/ClickHouse/pull/2980) - Amélioration des performances de lecture des chaînes et des tableaux dans les formats binaires. [Amos Oiseau](https://github.com/ClickHouse/ClickHouse/pull/2955) - Augmentation des performances et réduction de la consommation de mémoire pour les requêtes `system.tables` et `system.columns` quand il y a un très grand nombre de tables sur un seul serveur. [\#2953](https://github.com/ClickHouse/ClickHouse/pull/2953) -- Correction d’un problème de performances dans le cas d’un grand flux de requêtes résultant en une erreur (la `_dl_addr` la fonction est visible dans `perf top` mais le serveur n’utilise pas beaucoup de CPU). [\#2938](https://github.com/ClickHouse/ClickHouse/pull/2938) -- Les Conditions sont converties dans la vue (lorsque `enable_optimize_predicate_expression` est activé). [L’Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/2907) +- Correction d'un problème de performances dans le cas d'un grand flux de requêtes résultant en une erreur (la `_dl_addr` la fonction est visible dans `perf top` mais le serveur n'utilise pas beaucoup de CPU). [\#2938](https://github.com/ClickHouse/ClickHouse/pull/2938) +- Les Conditions sont converties dans la vue (lorsque `enable_optimize_predicate_expression` est activé). [L'Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/2907) - Améliorations de la fonctionnalité pour le `UUID` type de données. [\#3074](https://github.com/ClickHouse/ClickHouse/pull/3074) [\#2985](https://github.com/ClickHouse/ClickHouse/pull/2985) - Le `UUID` le type de données est pris en charge dans les dictionnaires-Alchemist. [\#2822](https://github.com/ClickHouse/ClickHouse/pull/2822) -- Le `visitParamExtractRaw` la fonction fonctionne correctement avec les structures imbriquées. [L’Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/2974) -- Lorsque l’ `input_format_skip_unknown_fields` paramètre est activé, les champs d’objet dans `JSONEachRow` format sont ignorés correctement. [BlahGeek](https://github.com/ClickHouse/ClickHouse/pull/2958) +- Le `visitParamExtractRaw` la fonction fonctionne correctement avec les structures imbriquées. [L'Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/2974) +- Lorsque l' `input_format_skip_unknown_fields` paramètre est activé, les champs d'objet dans `JSONEachRow` format sont ignorés correctement. [BlahGeek](https://github.com/ClickHouse/ClickHouse/pull/2958) - Pour un `CASE` expression avec conditions, vous pouvez maintenant omettre `ELSE`, ce qui est équivalent à `ELSE NULL`. [\#2920](https://github.com/ClickHouse/ClickHouse/pull/2920) -- Le délai d’attente de l’opération peut maintenant être configuré lorsque vous travaillez avec ZooKeeper. [urykhy](https://github.com/ClickHouse/ClickHouse/pull/2971) +- Le délai d'attente de l'opération peut maintenant être configuré lorsque vous travaillez avec ZooKeeper. [urykhy](https://github.com/ClickHouse/ClickHouse/pull/2971) - Vous pouvez spécifier un décalage pour `LIMIT n, m` comme `LIMIT n OFFSET m`. [\#2840](https://github.com/ClickHouse/ClickHouse/pull/2840) -- Vous pouvez utiliser l’ `SELECT TOP n` syntaxe comme alternative pour `LIMIT`. [\#2840](https://github.com/ClickHouse/ClickHouse/pull/2840) -- Augmentation de la taille de la file d’attente pour écrire dans les tables `SystemLog parameter queue is full` l’erreur ne se produit pas aussi souvent. -- Le `windowFunnel` fonction d’agrégation prend désormais en charge les événements qui répondent à plusieurs conditions. [Amos Oiseau](https://github.com/ClickHouse/ClickHouse/pull/2801) +- Vous pouvez utiliser l' `SELECT TOP n` syntaxe comme alternative pour `LIMIT`. [\#2840](https://github.com/ClickHouse/ClickHouse/pull/2840) +- Augmentation de la taille de la file d'attente pour écrire dans les tables `SystemLog parameter queue is full` l'erreur ne se produit pas aussi souvent. +- Le `windowFunnel` fonction d'agrégation prend désormais en charge les événements qui répondent à plusieurs conditions. [Amos Oiseau](https://github.com/ClickHouse/ClickHouse/pull/2801) - Les colonnes dupliquées peuvent être utilisées dans un `USING` la clause de `JOIN`. [\#3006](https://github.com/ClickHouse/ClickHouse/pull/3006) -- `Pretty` formats maintenant avoir une limite sur l’alignement des colonnes de largeur. L’utilisation de la `output_format_pretty_max_column_pad_width` paramètre. Si une valeur est plus large, il sera toujours affichée dans son intégralité, mais les autres cellules dans le tableau ne sera pas trop large. [\#3003](https://github.com/ClickHouse/ClickHouse/pull/3003) +- `Pretty` formats maintenant avoir une limite sur l'alignement des colonnes de largeur. L'utilisation de la `output_format_pretty_max_column_pad_width` paramètre. Si une valeur est plus large, il sera toujours affichée dans son intégralité, mais les autres cellules dans le tableau ne sera pas trop large. [\#3003](https://github.com/ClickHouse/ClickHouse/pull/3003) - Le `odbc` la fonction table vous permet maintenant de spécifier le nom de la base de données / schéma. [Amos Oiseau](https://github.com/ClickHouse/ClickHouse/pull/2885) -- Ajout de la possibilité d’utiliser un nom d’utilisateur spécifié dans le `clickhouse-client` fichier de configuration. [Vladimir Kozbin](https://github.com/ClickHouse/ClickHouse/pull/2909) +- Ajout de la possibilité d'utiliser un nom d'utilisateur spécifié dans le `clickhouse-client` fichier de configuration. [Vladimir Kozbin](https://github.com/ClickHouse/ClickHouse/pull/2909) - Le `ZooKeeperExceptions` compteur a été divisé en trois compteurs: `ZooKeeperUserExceptions`, `ZooKeeperHardwareExceptions`, et `ZooKeeperOtherExceptions`. - `ALTER DELETE` les requêtes fonctionnent pour les vues matérialisées. -- Ajouté randomisation lors de l’exécution du thread de nettoyage périodiquement pour `ReplicatedMergeTree` afin d’éviter les pics de charge périodiques lorsqu’il y a un très grand nombre de `ReplicatedMergeTree` table. +- Ajouté randomisation lors de l'exécution du thread de nettoyage périodiquement pour `ReplicatedMergeTree` afin d'éviter les pics de charge périodiques lorsqu'il y a un très grand nombre de `ReplicatedMergeTree` table. - Soutien pour `ATTACH TABLE ... ON CLUSTER` requête. [\#3025](https://github.com/ClickHouse/ClickHouse/pull/3025) #### Corrections De Bugs: {#bug-fixes-13} -- Correction d’un problème avec `Dictionary` tables (jette le `Size of offsets doesn't match size of column` ou `Unknown compression method` exception). Ce bug est apparu dans la version 18.10.3. [\#2913](https://github.com/ClickHouse/ClickHouse/issues/2913) -- Correction d’un bug lors de la fusion `CollapsingMergeTree` tables si l’une des parties de données est vide (ces parties sont formées lors de la fusion ou `ALTER DELETE` si toutes les données ont été supprimées), et le `vertical` l’algorithme a été utilisé pour la fusion. [\#3049](https://github.com/ClickHouse/ClickHouse/pull/3049) -- Correction d’une condition de course pendant `DROP` ou `TRUNCATE` pour `Memory` tables simultanément `SELECT`, ce qui pourrait conduire à des pannes de serveur. Ce bug est apparu dans la version 1.1.54388. [\#3038](https://github.com/ClickHouse/ClickHouse/pull/3038) -- Correction de la possibilité de perte de données lors de l’insertion dans `Replicated` des tables, si la `Session is expired` l’erreur est renvoyée (la perte de données peut être détectée par le `ReplicatedDataLoss` métrique). Cette erreur s’est produite dans la version 1.1.54378. [\#2939](https://github.com/ClickHouse/ClickHouse/pull/2939) [\#2949](https://github.com/ClickHouse/ClickHouse/pull/2949) [\#2964](https://github.com/ClickHouse/ClickHouse/pull/2964) -- Correction d’un segfault pendant `JOIN ... ON`. [\#3000](https://github.com/ClickHouse/ClickHouse/pull/3000) -- Correction de l’erreur de recherche des noms de colonne lorsque le `WHERE` expression se compose entièrement d’un nom de colonne qualifié, tel que `WHERE table.column`. [\#2994](https://github.com/ClickHouse/ClickHouse/pull/2994) -- Correction de l’ “Not found column” erreur survenue lors de l’exécution de requêtes distribuées si une seule colonne composée d’une expression avec une sous-requête est demandée à partir d’un serveur distant. [\#3087](https://github.com/ClickHouse/ClickHouse/pull/3087) -- Correction de l’ `Block structure mismatch in UNION stream: different number of columns` erreur qui s’est produite pour les requêtes distribuées si l’un des fragments est locale et l’autre ne l’est pas, et l’optimisation de la déplacer à `PREWHERE` est déclenchée. [\#2226](https://github.com/ClickHouse/ClickHouse/pull/2226) [\#3037](https://github.com/ClickHouse/ClickHouse/pull/3037) [\#3055](https://github.com/ClickHouse/ClickHouse/pull/3055) [\#3065](https://github.com/ClickHouse/ClickHouse/pull/3065) [\#3073](https://github.com/ClickHouse/ClickHouse/pull/3073) [\#3090](https://github.com/ClickHouse/ClickHouse/pull/3090) [\#3093](https://github.com/ClickHouse/ClickHouse/pull/3093) -- Correction de l’ `pointInPolygon` fonction pour certains cas de polygones non convexes. [\#2910](https://github.com/ClickHouse/ClickHouse/pull/2910) +- Correction d'un problème avec `Dictionary` tables (jette le `Size of offsets doesn't match size of column` ou `Unknown compression method` exception). Ce bug est apparu dans la version 18.10.3. [\#2913](https://github.com/ClickHouse/ClickHouse/issues/2913) +- Correction d'un bug lors de la fusion `CollapsingMergeTree` tables si l'une des parties de données est vide (ces parties sont formées lors de la fusion ou `ALTER DELETE` si toutes les données ont été supprimées), et le `vertical` l'algorithme a été utilisé pour la fusion. [\#3049](https://github.com/ClickHouse/ClickHouse/pull/3049) +- Correction d'une condition de course pendant `DROP` ou `TRUNCATE` pour `Memory` tables simultanément `SELECT`, ce qui pourrait conduire à des pannes de serveur. Ce bug est apparu dans la version 1.1.54388. [\#3038](https://github.com/ClickHouse/ClickHouse/pull/3038) +- Correction de la possibilité de perte de données lors de l'insertion dans `Replicated` des tables, si la `Session is expired` l'erreur est renvoyée (la perte de données peut être détectée par le `ReplicatedDataLoss` métrique). Cette erreur s'est produite dans la version 1.1.54378. [\#2939](https://github.com/ClickHouse/ClickHouse/pull/2939) [\#2949](https://github.com/ClickHouse/ClickHouse/pull/2949) [\#2964](https://github.com/ClickHouse/ClickHouse/pull/2964) +- Correction d'un segfault pendant `JOIN ... ON`. [\#3000](https://github.com/ClickHouse/ClickHouse/pull/3000) +- Correction de l'erreur de recherche des noms de colonne lorsque le `WHERE` expression se compose entièrement d'un nom de colonne qualifié, tel que `WHERE table.column`. [\#2994](https://github.com/ClickHouse/ClickHouse/pull/2994) +- Correction de l' “Not found column” erreur survenue lors de l'exécution de requêtes distribuées si une seule colonne composée d'une expression avec une sous-requête est demandée à partir d'un serveur distant. [\#3087](https://github.com/ClickHouse/ClickHouse/pull/3087) +- Correction de l' `Block structure mismatch in UNION stream: different number of columns` erreur qui s'est produite pour les requêtes distribuées si l'un des fragments est locale et l'autre ne l'est pas, et l'optimisation de la déplacer à `PREWHERE` est déclenchée. [\#2226](https://github.com/ClickHouse/ClickHouse/pull/2226) [\#3037](https://github.com/ClickHouse/ClickHouse/pull/3037) [\#3055](https://github.com/ClickHouse/ClickHouse/pull/3055) [\#3065](https://github.com/ClickHouse/ClickHouse/pull/3065) [\#3073](https://github.com/ClickHouse/ClickHouse/pull/3073) [\#3090](https://github.com/ClickHouse/ClickHouse/pull/3090) [\#3093](https://github.com/ClickHouse/ClickHouse/pull/3093) +- Correction de l' `pointInPolygon` fonction pour certains cas de polygones non convexes. [\#2910](https://github.com/ClickHouse/ClickHouse/pull/2910) - Correction du résultat incorrect lors de la comparaison `nan` avec des entiers. [\#3024](https://github.com/ClickHouse/ClickHouse/pull/3024) -- Correction d’une erreur dans le `zlib-ng` bibliothèque qui pourrait conduire à segfault dans de rares cas. [\#2854](https://github.com/ClickHouse/ClickHouse/pull/2854) -- Correction d’une fuite de mémoire lors de l’insertion dans une table avec `AggregateFunction` colonnes, si l’état de la fonction d’agrégat n’est pas simple (alloue la mémoire séparément), et si une seule demande d’insertion entraîne plusieurs petits blocs. [\#3084](https://github.com/ClickHouse/ClickHouse/pull/3084) -- Correction d’une condition de concurrence lors de la création et la suppression de la même `Buffer` ou `MergeTree` table en même temps. -- Correction de la possibilité d’un segfault lors de la comparaison des tuples constitués de certains types non triviaux, tels que les tuples. [\#2989](https://github.com/ClickHouse/ClickHouse/pull/2989) -- Correction de la possibilité d’un segfault lors de l’exécution de certains `ON CLUSTER` requête. [L’Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/2960) -- Correction d’une erreur dans le `arrayDistinct` fonction pour `Nullable` les éléments du tableau. [\#2845](https://github.com/ClickHouse/ClickHouse/pull/2845) [\#2937](https://github.com/ClickHouse/ClickHouse/pull/2937) -- Le `enable_optimize_predicate_expression` option prend en charge correctement les cas avec `SELECT *`. [L’Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/2929) +- Correction d'une erreur dans le `zlib-ng` bibliothèque qui pourrait conduire à segfault dans de rares cas. [\#2854](https://github.com/ClickHouse/ClickHouse/pull/2854) +- Correction d'une fuite de mémoire lors de l'insertion dans une table avec `AggregateFunction` colonnes, si l'état de la fonction d'agrégat n'est pas simple (alloue la mémoire séparément), et si une seule demande d'insertion entraîne plusieurs petits blocs. [\#3084](https://github.com/ClickHouse/ClickHouse/pull/3084) +- Correction d'une condition de concurrence lors de la création et la suppression de la même `Buffer` ou `MergeTree` table en même temps. +- Correction de la possibilité d'un segfault lors de la comparaison des tuples constitués de certains types non triviaux, tels que les tuples. [\#2989](https://github.com/ClickHouse/ClickHouse/pull/2989) +- Correction de la possibilité d'un segfault lors de l'exécution de certains `ON CLUSTER` requête. [L'Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/2960) +- Correction d'une erreur dans le `arrayDistinct` fonction pour `Nullable` les éléments du tableau. [\#2845](https://github.com/ClickHouse/ClickHouse/pull/2845) [\#2937](https://github.com/ClickHouse/ClickHouse/pull/2937) +- Le `enable_optimize_predicate_expression` option prend en charge correctement les cas avec `SELECT *`. [L'Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/2929) - Correction du segfault lors de la réinitialisation de la session ZooKeeper. [\#2917](https://github.com/ClickHouse/ClickHouse/pull/2917) - Blocage potentiel fixe lorsque vous travaillez avec ZooKeeper. -- Correction d’un code incorrect pour ajouter des structures de données imbriquées dans un `SummingMergeTree`. -- Lors de l’allocation de mémoire pour les états de fonctions d’agrégat, l’alignement est correctement pris en compte, ce qui permet d’utiliser des opérations nécessitant un alignement lors de la mise en œuvre des états de fonctions d’agrégat. [chenxing-xc](https://github.com/ClickHouse/ClickHouse/pull/2808) +- Correction d'un code incorrect pour ajouter des structures de données imbriquées dans un `SummingMergeTree`. +- Lors de l'allocation de mémoire pour les états de fonctions d'agrégat, l'alignement est correctement pris en compte, ce qui permet d'utiliser des opérations nécessitant un alignement lors de la mise en œuvre des états de fonctions d'agrégat. [chenxing-xc](https://github.com/ClickHouse/ClickHouse/pull/2808) -#### Correction De sécurité: {#security-fix} +#### Correction De Sécurité: {#security-fix} -- Utilisation sûre des sources de données ODBC. L’Interaction avec les pilotes ODBC utilise un `clickhouse-odbc-bridge` processus. Les erreurs dans les pilotes ODBC tiers ne causent plus de problèmes de stabilité du serveur ou de vulnérabilités. [\#2828](https://github.com/ClickHouse/ClickHouse/pull/2828) [\#2879](https://github.com/ClickHouse/ClickHouse/pull/2879) [\#2886](https://github.com/ClickHouse/ClickHouse/pull/2886) [\#2893](https://github.com/ClickHouse/ClickHouse/pull/2893) [\#2921](https://github.com/ClickHouse/ClickHouse/pull/2921) -- Correction d’une validation incorrecte du chemin du fichier dans le `catBoostPool` table de fonction. [\#2894](https://github.com/ClickHouse/ClickHouse/pull/2894) -- Le contenu des tableaux du système (`tables`, `databases`, `parts`, `columns`, `parts_columns`, `merges`, `mutations`, `replicas`, et `replication_queue`) sont filtrés en fonction de l’accès configuré de l’utilisateur aux bases de données (`allow_databases`). [L’Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/2856) +- Utilisation sûre des sources de données ODBC. L'Interaction avec les pilotes ODBC utilise un `clickhouse-odbc-bridge` processus. Les erreurs dans les pilotes ODBC tiers ne causent plus de problèmes de stabilité du serveur ou de vulnérabilités. [\#2828](https://github.com/ClickHouse/ClickHouse/pull/2828) [\#2879](https://github.com/ClickHouse/ClickHouse/pull/2879) [\#2886](https://github.com/ClickHouse/ClickHouse/pull/2886) [\#2893](https://github.com/ClickHouse/ClickHouse/pull/2893) [\#2921](https://github.com/ClickHouse/ClickHouse/pull/2921) +- Correction d'une validation incorrecte du chemin du fichier dans le `catBoostPool` table de fonction. [\#2894](https://github.com/ClickHouse/ClickHouse/pull/2894) +- Le contenu des tableaux du système (`tables`, `databases`, `parts`, `columns`, `parts_columns`, `merges`, `mutations`, `replicas`, et `replication_queue`) sont filtrés en fonction de l'accès configuré de l'utilisateur aux bases de données (`allow_databases`). [L'Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/2856) -#### Modifications Incompatibles En arrière: {#backward-incompatible-changes-3} +#### Modifications Incompatibles En Arrière: {#backward-incompatible-changes-3} -- Dans les requêtes avec jointure, le caractère étoile se développe en une liste de colonnes dans toutes les tables, conformément à la norme SQL. Vous pouvez restaurer l’ancien comportement en paramètre `asterisk_left_columns_only` à 1 au niveau de la configuration utilisateur. +- Dans les requêtes avec jointure, le caractère étoile se développe en une liste de colonnes dans toutes les tables, conformément à la norme SQL. Vous pouvez restaurer l'ancien comportement en paramètre `asterisk_left_columns_only` à 1 au niveau de la configuration utilisateur. #### Construire Des Changements: {#build-changes-2} -- La plupart des tests d’intégration peuvent maintenant être exécutés par commit. +- La plupart des tests d'intégration peuvent maintenant être exécutés par commit. - Les contrôles de style de Code peuvent également être exécutés par commit. -- Le `memcpy` l’implémentation est choisie correctement lors de la construction sur CentOS7/Fedora. [Etienne Champetier](https://github.com/ClickHouse/ClickHouse/pull/2912) +- Le `memcpy` l'implémentation est choisie correctement lors de la construction sur CentOS7/Fedora. [Etienne Champetier](https://github.com/ClickHouse/ClickHouse/pull/2912) - Lorsque vous utilisez clang pour construire, certains avertissements de `-Weverything` ont été ajoutées, en plus de la `-Wall-Wextra -Werror`. [\#2957](https://github.com/ClickHouse/ClickHouse/pull/2957) - Débogage de la construction utilise le `jemalloc` option de débogage. -- L’interface de la bibliothèque pour interagir avec ZooKeeper est déclarée abstraite. [\#2950](https://github.com/ClickHouse/ClickHouse/pull/2950) +- L'interface de la bibliothèque pour interagir avec ZooKeeper est déclarée abstraite. [\#2950](https://github.com/ClickHouse/ClickHouse/pull/2950) ## Clickhouse Version 18.10 {#clickhouse-release-18-10} @@ -455,47 +455,47 @@ toc_title: '2018' #### Nouveauté: {#new-features-5} - HTTPS peut être utilisé pour la réplication. [\#2760](https://github.com/ClickHouse/ClickHouse/pull/2760) -- Ajout des fonctions `murmurHash2_64`, `murmurHash3_32`, `murmurHash3_64`, et `murmurHash3_128` en plus de l’existant `murmurHash2_32`. [\#2791](https://github.com/ClickHouse/ClickHouse/pull/2791) +- Ajout des fonctions `murmurHash2_64`, `murmurHash3_32`, `murmurHash3_64`, et `murmurHash3_128` en plus de l'existant `murmurHash2_32`. [\#2791](https://github.com/ClickHouse/ClickHouse/pull/2791) - Prise en charge des types Nullable dans le pilote ODBC ClickHouse (`ODBCDriver2` le format de sortie). [\#2834](https://github.com/ClickHouse/ClickHouse/pull/2834) - Soutien pour `UUID` dans les colonnes de clé. #### Amélioration: {#improvements-5} -- Les Clusters peuvent être supprimés sans redémarrer le serveur lorsqu’ils sont supprimés des fichiers de configuration. [\#2777](https://github.com/ClickHouse/ClickHouse/pull/2777) -- Les dictionnaires externes peuvent être supprimés sans redémarrer le serveur lorsqu’ils sont supprimés des fichiers de configuration. [\#2779](https://github.com/ClickHouse/ClickHouse/pull/2779) +- Les Clusters peuvent être supprimés sans redémarrer le serveur lorsqu'ils sont supprimés des fichiers de configuration. [\#2777](https://github.com/ClickHouse/ClickHouse/pull/2777) +- Les dictionnaires externes peuvent être supprimés sans redémarrer le serveur lorsqu'ils sont supprimés des fichiers de configuration. [\#2779](https://github.com/ClickHouse/ClickHouse/pull/2779) - Ajouter `SETTINGS` soutien pour le `Kafka` tableau moteur. [Alexander Marshalov](https://github.com/ClickHouse/ClickHouse/pull/2781) -- Des améliorations pour l’ `UUID` type de données (pas encore terminée). [\#2618](https://github.com/ClickHouse/ClickHouse/pull/2618) +- Des améliorations pour l' `UUID` type de données (pas encore terminée). [\#2618](https://github.com/ClickHouse/ClickHouse/pull/2618) - Prise en charge des pièces vides après fusion dans le `SummingMergeTree`, `CollapsingMergeTree` et `VersionedCollapsingMergeTree` moteur. [\#2815](https://github.com/ClickHouse/ClickHouse/pull/2815) - Les anciens enregistrements de mutations terminées sont supprimés (`ALTER DELETE`). [\#2784](https://github.com/ClickHouse/ClickHouse/pull/2784) - Ajouté le `system.merge_tree_settings` table. [Kirill Shvakov](https://github.com/ClickHouse/ClickHouse/pull/2841) -- Le `system.tables` la table a maintenant des colonnes de dépendance: `dependencies_database` et `dependencies_table`. [L’Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/2851) +- Le `system.tables` la table a maintenant des colonnes de dépendance: `dependencies_database` et `dependencies_table`. [L'Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/2851) - Ajouté le `max_partition_size_to_drop` option de configuration. [\#2782](https://github.com/ClickHouse/ClickHouse/pull/2782) - Ajouté le `output_format_json_escape_forward_slashes` option. [Alexandre Botcharov](https://github.com/ClickHouse/ClickHouse/pull/2812) - Ajouté le `max_fetch_partition_retries_count` paramètre. [\#2831](https://github.com/ClickHouse/ClickHouse/pull/2831) -- Ajouté le `prefer_localhost_replica` paramètre permettant de désactiver la préférence pour une réplique locale et d’accéder à une réplique locale sans interaction entre processus. [\#2832](https://github.com/ClickHouse/ClickHouse/pull/2832) -- Le `quantileExact` fonction d’agrégation retourne `nan` dans le cas de l’agrégation sur un vide `Float32` ou `Float64` définir. [Sundy Li](https://github.com/ClickHouse/ClickHouse/pull/2855) +- Ajouté le `prefer_localhost_replica` paramètre permettant de désactiver la préférence pour une réplique locale et d'accéder à une réplique locale sans interaction entre processus. [\#2832](https://github.com/ClickHouse/ClickHouse/pull/2832) +- Le `quantileExact` fonction d'agrégation retourne `nan` dans le cas de l'agrégation sur un vide `Float32` ou `Float64` définir. [Sundy Li](https://github.com/ClickHouse/ClickHouse/pull/2855) #### Corrections De Bugs: {#bug-fixes-14} -- Suppression de l’échappement inutile des paramètres de chaîne de connexion pour ODBC, ce qui rendait impossible l’établissement d’une connexion. Cette erreur s’est produite dans la version 18.6.0. -- Fixe la logique de traitement `REPLACE PARTITION` les commandes dans la file d’attente de réplication. Si il y a deux `REPLACE` pour la même partition, la logique incorrecte pourrait entraîner l’un d’entre eux de rester dans la file d’attente de réplication et ne pas être exécuté. [\#2814](https://github.com/ClickHouse/ClickHouse/pull/2814) -- Correction d’un bug de fusion lorsque toutes les parties de données étaient vides (parties formées à partir d’une fusion ou `ALTER DELETE` si toutes les données ont été supprimées). Ce bug est apparu dans la version 18.1.0. [\#2930](https://github.com/ClickHouse/ClickHouse/pull/2930) -- Correction d’une erreur pour simultanées `Set` ou `Join`. [Amos Oiseau](https://github.com/ClickHouse/ClickHouse/pull/2823) -- Correction de l’ `Block structure mismatch in UNION stream: different number of columns` erreur qui s’est produite pour `UNION ALL` requêtes dans une sous-requête si l’un des `SELECT` les requêtes contiennent des noms de colonnes en double. [L’Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/2094) -- Correction d’une fuite de mémoire si une exception se produisait lors de la connexion à un serveur MySQL. -- Correction d’un code de réponse clickhouse-client incorrect en cas d’erreur de requête. -- Correction d’un comportement incorrect des vues matérialisées contenant DISTINCT. [\#2795](https://github.com/ClickHouse/ClickHouse/issues/2795) +- Suppression de l'échappement inutile des paramètres de chaîne de connexion pour ODBC, ce qui rendait impossible l'établissement d'une connexion. Cette erreur s'est produite dans la version 18.6.0. +- Fixe la logique de traitement `REPLACE PARTITION` les commandes dans la file d'attente de réplication. Si il y a deux `REPLACE` pour la même partition, la logique incorrecte pourrait entraîner l'un d'entre eux de rester dans la file d'attente de réplication et ne pas être exécuté. [\#2814](https://github.com/ClickHouse/ClickHouse/pull/2814) +- Correction d'un bug de fusion lorsque toutes les parties de données étaient vides (parties formées à partir d'une fusion ou `ALTER DELETE` si toutes les données ont été supprimées). Ce bug est apparu dans la version 18.1.0. [\#2930](https://github.com/ClickHouse/ClickHouse/pull/2930) +- Correction d'une erreur pour simultanées `Set` ou `Join`. [Amos Oiseau](https://github.com/ClickHouse/ClickHouse/pull/2823) +- Correction de l' `Block structure mismatch in UNION stream: different number of columns` erreur qui s'est produite pour `UNION ALL` requêtes dans une sous-requête si l'un des `SELECT` les requêtes contiennent des noms de colonnes en double. [L'Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/2094) +- Correction d'une fuite de mémoire si une exception se produisait lors de la connexion à un serveur MySQL. +- Correction d'un code de réponse clickhouse-client incorrect en cas d'erreur de requête. +- Correction d'un comportement incorrect des vues matérialisées contenant DISTINCT. [\#2795](https://github.com/ClickHouse/ClickHouse/issues/2795) -#### Modifications Incompatibles En arrière {#backward-incompatible-changes-4} +#### Modifications Incompatibles En Arrière {#backward-incompatible-changes-4} - Suppression de la prise en charge des requêtes CHECK TABLE pour les tables distribuées. #### Construire Des Changements: {#build-changes-3} -- L’allocateur a été remplacé: `jemalloc` est maintenant utilisé à la place de `tcmalloc`. Dans certains scénarios, cela augmente la vitesse jusqu’à 20%. Cependant, il y a des questions qui ont ralenti jusqu’à 20%. La consommation de mémoire a été réduite d’environ 10% dans certains scénarios, avec une meilleure stabilité. Avec des charges très compétitives, l’utilisation du processeur dans l’espace utilisateur et dans le système ne montre qu’une légère augmentation. [\#2773](https://github.com/ClickHouse/ClickHouse/pull/2773) -- L’utilisation de libressl à partir d’un sous-module. [\#1983](https://github.com/ClickHouse/ClickHouse/pull/1983) [\#2807](https://github.com/ClickHouse/ClickHouse/pull/2807) -- Utilisation d’unixodbc à partir d’un sous-module. [\#2789](https://github.com/ClickHouse/ClickHouse/pull/2789) -- L’utilisation de mariadb-connecteur-c à partir d’un sous-module. [\#2785](https://github.com/ClickHouse/ClickHouse/pull/2785) +- L'allocateur a été remplacé: `jemalloc` est maintenant utilisé à la place de `tcmalloc`. Dans certains scénarios, cela augmente la vitesse jusqu'à 20%. Cependant, il y a des questions qui ont ralenti jusqu'à 20%. La consommation de mémoire a été réduite d'environ 10% dans certains scénarios, avec une meilleure stabilité. Avec des charges très compétitives, l'utilisation du processeur dans l'espace utilisateur et dans le système ne montre qu'une légère augmentation. [\#2773](https://github.com/ClickHouse/ClickHouse/pull/2773) +- L'utilisation de libressl à partir d'un sous-module. [\#1983](https://github.com/ClickHouse/ClickHouse/pull/1983) [\#2807](https://github.com/ClickHouse/ClickHouse/pull/2807) +- Utilisation d'unixodbc à partir d'un sous-module. [\#2789](https://github.com/ClickHouse/ClickHouse/pull/2789) +- L'utilisation de mariadb-connecteur-c à partir d'un sous-module. [\#2785](https://github.com/ClickHouse/ClickHouse/pull/2785) - Ajout de fichiers de test fonctionnels au référentiel qui dépendent de la disponibilité des données de test (pour le moment, sans les données de test elles-mêmes). ## Clickhouse Version 18.6 {#clickhouse-release-18-6} @@ -506,7 +506,7 @@ toc_title: '2018' - Ajout du support pour les expressions on pour la syntaxe JOIN ON: `JOIN ON Expr([table.]column ...) = Expr([table.]column, ...) [AND Expr([table.]column, ...) = Expr([table.]column, ...) ...]` - L’expression doit être une chaîne d’égalités rejoint par l’opérateur ET. De chaque côté de l’égalité peut être une expression arbitraire sur les colonnes de l’une des tables. L’utilisation de noms de colonnes entièrement qualifiés est prise en charge (`table.name`, `database.table.name`, `table_alias.name`, `subquery_alias.name`) pour la bonne table. [\#2742](https://github.com/ClickHouse/ClickHouse/pull/2742) + L'expression doit être une chaîne d'égalités rejoint par l'opérateur ET. De chaque côté de l'égalité peut être une expression arbitraire sur les colonnes de l'une des tables. L'utilisation de noms de colonnes entièrement qualifiés est prise en charge (`table.name`, `database.table.name`, `table_alias.name`, `subquery_alias.name`) pour la bonne table. [\#2742](https://github.com/ClickHouse/ClickHouse/pull/2742) - HTTPS peut être activé pour la réplication. [\#2760](https://github.com/ClickHouse/ClickHouse/pull/2760) #### Amélioration: {#improvements-6} @@ -523,12 +523,12 @@ toc_title: '2018' #### Amélioration: {#improvements-7} -- Maintenant, vous pouvez utiliser le `from_env` [\#2741](https://github.com/ClickHouse/ClickHouse/pull/2741) attribut pour définir des valeurs dans les fichiers de configuration à partir de variables d’environnement. +- Maintenant, vous pouvez utiliser le `from_env` [\#2741](https://github.com/ClickHouse/ClickHouse/pull/2741) attribut pour définir des valeurs dans les fichiers de configuration à partir de variables d'environnement. - Ajout de versions insensibles à la casse `coalesce`, `ifNull`, et `nullIf functions` [\#2752](https://github.com/ClickHouse/ClickHouse/pull/2752). #### Corrections De Bugs: {#bug-fixes-15} -- Correction d’un bug possible lors du démarrage d’une réplique [\#2759](https://github.com/ClickHouse/ClickHouse/pull/2759). +- Correction d'un bug possible lors du démarrage d'une réplique [\#2759](https://github.com/ClickHouse/ClickHouse/pull/2759). ## Clickhouse Version 18.4 {#clickhouse-release-18-4} @@ -537,27 +537,27 @@ toc_title: '2018' #### Nouveauté: {#new-features-8} - Tables système ajoutées: `formats`, `data_type_families`, `aggregate_function_combinators`, `table_functions`, `table_engines`, `collations` [\#2721](https://github.com/ClickHouse/ClickHouse/pull/2721). -- Ajout de la possibilité d’utiliser une fonction de table au lieu d’un tableau en argument d’une `remote` ou `cluster table function` [\#2708](https://github.com/ClickHouse/ClickHouse/pull/2708). -- Soutien pour `HTTP Basic` l’authentification dans le protocole de réplication [\#2727](https://github.com/ClickHouse/ClickHouse/pull/2727). +- Ajout de la possibilité d'utiliser une fonction de table au lieu d'un tableau en argument d'une `remote` ou `cluster table function` [\#2708](https://github.com/ClickHouse/ClickHouse/pull/2708). +- Soutien pour `HTTP Basic` l'authentification dans le protocole de réplication [\#2727](https://github.com/ClickHouse/ClickHouse/pull/2727). - Le `has` fonction permet de rechercher une valeur numérique dans un tableau de `Enum` valeur [Maxim Khrisanfov](https://github.com/ClickHouse/ClickHouse/pull/2699). -- Prise en charge de l’ajout de séparateurs de messages arbitraires lors de la lecture de `Kafka` [Amos Oiseau](https://github.com/ClickHouse/ClickHouse/pull/2701). +- Prise en charge de l'ajout de séparateurs de messages arbitraires lors de la lecture de `Kafka` [Amos Oiseau](https://github.com/ClickHouse/ClickHouse/pull/2701). #### Amélioration: {#improvements-8} -- Le `ALTER TABLE t DELETE WHERE` la requête ne réécrit pas les parties de données qui n’ont pas été affectées par la condition WHERE [\#2694](https://github.com/ClickHouse/ClickHouse/pull/2694). +- Le `ALTER TABLE t DELETE WHERE` la requête ne réécrit pas les parties de données qui n'ont pas été affectées par la condition WHERE [\#2694](https://github.com/ClickHouse/ClickHouse/pull/2694). - Le `use_minimalistic_checksums_in_zookeeper` option pour `ReplicatedMergeTree` des tables est activé par défaut. Ce paramètre a été ajouté dans la version 1.1.54378, 2018-04-16. Les Versions antérieures à 1.1.54378 ne peuvent plus être installées. -- La prise en charge de `KILL` et `OPTIMIZE` requêtes qui spécifient `ON CLUSTER` [L’Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/2689). +- La prise en charge de `KILL` et `OPTIMIZE` requêtes qui spécifient `ON CLUSTER` [L'Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/2689). #### Corrections De Bugs: {#bug-fixes-16} -- Correction de l’erreur `Column ... is not under an aggregate function and not in GROUP BY` pour l’agrégation avec une expression. Ce bug est apparu dans la version 18.1.0. ([bbdd780b](https://github.com/ClickHouse/ClickHouse/commit/bbdd780be0be06a0f336775941cdd536878dd2c2)) -- Correction d’un bug dans l’ `windowFunnel aggregate function` [L’Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/2735). -- Correction d’un bug dans l’ `anyHeavy` fonction d’agrégation ([a2101df2](https://github.com/ClickHouse/ClickHouse/commit/a2101df25a6a0fba99aa71f8793d762af2b801ee)) -- Correction d’un crash du serveur lors de l’utilisation du `countArray()` fonction d’agrégation. +- Correction de l'erreur `Column ... is not under an aggregate function and not in GROUP BY` pour l'agrégation avec une expression. Ce bug est apparu dans la version 18.1.0. ([bbdd780b](https://github.com/ClickHouse/ClickHouse/commit/bbdd780be0be06a0f336775941cdd536878dd2c2)) +- Correction d'un bug dans l' `windowFunnel aggregate function` [L'Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/2735). +- Correction d'un bug dans l' `anyHeavy` fonction d'agrégation ([a2101df2](https://github.com/ClickHouse/ClickHouse/commit/a2101df25a6a0fba99aa71f8793d762af2b801ee)) +- Correction d'un crash du serveur lors de l'utilisation du `countArray()` fonction d'agrégation. -#### Modifications Incompatibles En arrière: {#backward-incompatible-changes-5} +#### Modifications Incompatibles En Arrière: {#backward-incompatible-changes-5} -- Paramètres pour `Kafka` moteur a été changé de `Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format[, kafka_schema, kafka_num_consumers])` de `Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format[, kafka_row_delimiter, kafka_schema, kafka_num_consumers])`. Si vos tables utilisent `kafka_schema` ou `kafka_num_consumers` paramètres, vous devez modifier manuellement les fichiers de métadonnées `path/metadata/database/table.sql` et d’ajouter `kafka_row_delimiter` paramètre avec `''` valeur. +- Paramètres pour `Kafka` moteur a été changé de `Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format[, kafka_schema, kafka_num_consumers])` de `Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format[, kafka_row_delimiter, kafka_schema, kafka_num_consumers])`. Si vos tables utilisent `kafka_schema` ou `kafka_num_consumers` paramètres, vous devez modifier manuellement les fichiers de métadonnées `path/metadata/database/table.sql` et d'ajouter `kafka_row_delimiter` paramètre avec `''` valeur. ## Clickhouse Version 18.1 {#clickhouse-release-18-1} @@ -574,24 +574,24 @@ toc_title: '2018' #### Amélioration: {#improvements-9} -- Modification du schéma de numérotation pour les versions de version. Maintenant, la première partie contient l’année de sortie (A. D., fuseau horaire de Moscou, moins 2000), la deuxième partie contient le nombre de changements majeurs (augmente pour la plupart des versions), et la troisième partie est la version patch. Les versions sont toujours rétrocompatibles, sauf indication contraire dans le changelog. +- Modification du schéma de numérotation pour les versions de version. Maintenant, la première partie contient l'année de sortie (A. D., fuseau horaire de Moscou, moins 2000), la deuxième partie contient le nombre de changements majeurs (augmente pour la plupart des versions), et la troisième partie est la version patch. Les versions sont toujours rétrocompatibles, sauf indication contraire dans le changelog. - Conversions plus rapides de nombres à virgule flottante en une chaîne ([Amos Oiseau](https://github.com/ClickHouse/ClickHouse/pull/2664)). -- Si certaines lignes ont été ignorées lors d’une insertion en raison d’erreurs d’analyse (ceci est possible avec `input_allow_errors_num` et `input_allow_errors_ratio` paramètres activé), le nombre de lignes ignorées est maintenant écrit dans le journal du serveur ([Leonardo Cecchi](https://github.com/ClickHouse/ClickHouse/pull/2669)). +- Si certaines lignes ont été ignorées lors d'une insertion en raison d'erreurs d'analyse (ceci est possible avec `input_allow_errors_num` et `input_allow_errors_ratio` paramètres activé), le nombre de lignes ignorées est maintenant écrit dans le journal du serveur ([Leonardo Cecchi](https://github.com/ClickHouse/ClickHouse/pull/2669)). #### Corrections De Bugs: {#bug-fixes-17} - Correction de la commande TRUNCATE pour les tables temporaires ([Amos Oiseau](https://github.com/ClickHouse/ClickHouse/pull/2624)). -- Correction d’une impasse rare dans la bibliothèque client ZooKeeper qui se produisait lorsqu’une erreur réseau se produisait lors de la lecture de la réponse ([c315200](https://github.com/ClickHouse/ClickHouse/commit/c315200e64b87e44bdf740707fc857d1fdf7e947)). -- Correction d’une erreur lors d’une conversion en types Nullable ([\#1322](https://github.com/ClickHouse/ClickHouse/issues/1322)). -- Correction du résultat incorrect de l’ `maxIntersection()` fonction lorsque les limites des intervalles de coïncidé ([Michael Furmur](https://github.com/ClickHouse/ClickHouse/pull/2657)). -- Correction d’une transformation incorrecte de la chaîne d’expression OR dans un argument de fonction ([chenxing-xc](https://github.com/ClickHouse/ClickHouse/pull/2663)). -- Fixe une dégradation des performances pour les requêtes contenant `IN (subquery)` expressions à l’intérieur d’une autre sous-requête ([\#2571](https://github.com/ClickHouse/ClickHouse/issues/2571)). -- Fixe incompatibilité entre les serveurs avec des versions différentes requêtes distribuées qui utilisent un `CAST` fonction qui n’est pas en majuscules ([fe8c4d6](https://github.com/ClickHouse/ClickHouse/commit/fe8c4d64e434cacd4ceef34faa9005129f2190a5)). -- Ajout de guillemets manquants d’identifiants pour les requêtes à un SGBD externe ([\#2635](https://github.com/ClickHouse/ClickHouse/issues/2635)). +- Correction d'une impasse rare dans la bibliothèque client ZooKeeper qui se produisait lorsqu'une erreur réseau se produisait lors de la lecture de la réponse ([c315200](https://github.com/ClickHouse/ClickHouse/commit/c315200e64b87e44bdf740707fc857d1fdf7e947)). +- Correction d'une erreur lors d'une conversion en types Nullable ([\#1322](https://github.com/ClickHouse/ClickHouse/issues/1322)). +- Correction du résultat incorrect de l' `maxIntersection()` fonction lorsque les limites des intervalles de coïncidé ([Michael Furmur](https://github.com/ClickHouse/ClickHouse/pull/2657)). +- Correction d'une transformation incorrecte de la chaîne d'expression OR dans un argument de fonction ([chenxing-xc](https://github.com/ClickHouse/ClickHouse/pull/2663)). +- Fixe une dégradation des performances pour les requêtes contenant `IN (subquery)` expressions à l'intérieur d'une autre sous-requête ([\#2571](https://github.com/ClickHouse/ClickHouse/issues/2571)). +- Fixe incompatibilité entre les serveurs avec des versions différentes requêtes distribuées qui utilisent un `CAST` fonction qui n'est pas en majuscules ([fe8c4d6](https://github.com/ClickHouse/ClickHouse/commit/fe8c4d64e434cacd4ceef34faa9005129f2190a5)). +- Ajout de guillemets manquants d'identifiants pour les requêtes à un SGBD externe ([\#2635](https://github.com/ClickHouse/ClickHouse/issues/2635)). -#### Modifications Incompatibles En arrière: {#backward-incompatible-changes-6} +#### Modifications Incompatibles En Arrière: {#backward-incompatible-changes-6} -- La conversion d’une chaîne contenant le nombre zéro en DateTime ne fonctionne pas. Exemple: `SELECT toDateTime('0')`. C’est aussi la raison pour laquelle `DateTime DEFAULT '0'` ne fonctionne pas dans les tableaux, ainsi que `0` dans les dictionnaires. Solution: remplacer `0` avec `0000-00-00 00:00:00`. +- La conversion d'une chaîne contenant le nombre zéro en DateTime ne fonctionne pas. Exemple: `SELECT toDateTime('0')`. C'est aussi la raison pour laquelle `DateTime DEFAULT '0'` ne fonctionne pas dans les tableaux, ainsi que `0` dans les dictionnaires. Solution: remplacer `0` avec `0000-00-00 00:00:00`. ## Clickhouse Version 1.1 {#clickhouse-release-1-1} @@ -599,16 +599,16 @@ toc_title: '2018' #### Nouveauté: {#new-features-10} -- Ajouté le `histogram` fonction d’agrégation ([Mikhail Surin](https://github.com/ClickHouse/ClickHouse/pull/2521)). +- Ajouté le `histogram` fonction d'agrégation ([Mikhail Surin](https://github.com/ClickHouse/ClickHouse/pull/2521)). - Maintenant `OPTIMIZE TABLE ... FINAL` peut être utilisé sans spécifier de partitions pour `ReplicatedMergeTree` ([Amos Oiseau](https://github.com/ClickHouse/ClickHouse/pull/2600)). #### Corrections De Bugs: {#bug-fixes-18} -- Correction d’un problème avec un très petit délai d’attente pour les sockets (une seconde) pour la lecture et l’écriture lors de l’envoi et du téléchargement de données répliquées, ce qui rendait impossible le téléchargement de pièces plus grandes s’il y avait une charge sur le réseau ou le disque (cela entraînait des tentatives cycliques de téléchargement de pièces). Cette erreur s’est produite dans la version 1.1.54388. -- Correction de problèmes lors de l’utilisation de chroot dans ZooKeeper si vous avez inséré des blocs de données en double dans la table. +- Correction d'un problème avec un très petit délai d'attente pour les sockets (une seconde) pour la lecture et l'écriture lors de l'envoi et du téléchargement de données répliquées, ce qui rendait impossible le téléchargement de pièces plus grandes s'il y avait une charge sur le réseau ou le disque (cela entraînait des tentatives cycliques de téléchargement de pièces). Cette erreur s'est produite dans la version 1.1.54388. +- Correction de problèmes lors de l'utilisation de chroot dans ZooKeeper si vous avez inséré des blocs de données en double dans la table. - Le `has` la fonction fonctionne maintenant correctement pour un tableau avec les valeurs null éléments ([\#2115](https://github.com/ClickHouse/ClickHouse/issues/2115)). -- Le `system.tables` table fonctionne maintenant correctement lorsqu’il est utilisé dans les requêtes distribuées. Le `metadata_modification_time` et `engine_full` les colonnes sont maintenant non-virtuel. Correction d’une erreur qui s’est produite si seules ces colonnes étaient interrogées à partir de la table. -- Correction de la façon dont un vide `TinyLog` table fonctionne après l’insertion d’un bloc de données vide ([\#2563](https://github.com/ClickHouse/ClickHouse/issues/2563)). +- Le `system.tables` table fonctionne maintenant correctement lorsqu'il est utilisé dans les requêtes distribuées. Le `metadata_modification_time` et `engine_full` les colonnes sont maintenant non-virtuel. Correction d'une erreur qui s'est produite si seules ces colonnes étaient interrogées à partir de la table. +- Correction de la façon dont un vide `TinyLog` table fonctionne après l'insertion d'un bloc de données vide ([\#2563](https://github.com/ClickHouse/ClickHouse/issues/2563)). - Le `system.zookeeper` table fonctionne si la valeur du nœud dans ZooKeeper est NULL. ### Clickhouse Version 1.1.54390, 2018-07-06 {#clickhouse-release-1-1-54390-2018-07-06} @@ -616,32 +616,32 @@ toc_title: '2018' #### Nouveauté: {#new-features-11} - Les requêtes peuvent être envoyées `multipart/form-data` format (dans le `query` champ), ce qui est utile si des données externes sont également envoyées pour le traitement de la requête ([Olga Hvostikova](https://github.com/ClickHouse/ClickHouse/pull/2490)). -- Ajout de la possibilité d’activer ou de désactiver le traitement des guillemets simples ou doubles lors de la lecture de données au format CSV. Vous pouvez configurer cela dans le `format_csv_allow_single_quotes` et `format_csv_allow_double_quotes` paramètre ([Amos Oiseau](https://github.com/ClickHouse/ClickHouse/pull/2574)). +- Ajout de la possibilité d'activer ou de désactiver le traitement des guillemets simples ou doubles lors de la lecture de données au format CSV. Vous pouvez configurer cela dans le `format_csv_allow_single_quotes` et `format_csv_allow_double_quotes` paramètre ([Amos Oiseau](https://github.com/ClickHouse/ClickHouse/pull/2574)). - Maintenant `OPTIMIZE TABLE ... FINAL` peut être utilisé sans spécifier la partition pour les variantes non répliquées de `MergeTree` ([Amos Oiseau](https://github.com/ClickHouse/ClickHouse/pull/2599)). #### Amélioration: {#improvements-10} -- Amélioration des performances, réduction de la consommation de mémoire et suivi correct de la consommation de mémoire avec l’utilisation de L’opérateur IN lorsqu’un index de table peut être utilisé ([\#2584](https://github.com/ClickHouse/ClickHouse/pull/2584)). -- Retiré redondant vérification des sommes de contrôle lors de l’ajout d’une partie des données. Ceci est important lorsqu’il y a un grand nombre de répliques, car dans ces cas, le nombre total de contrôles était égal à n^2. +- Amélioration des performances, réduction de la consommation de mémoire et suivi correct de la consommation de mémoire avec l'utilisation de L'opérateur IN lorsqu'un index de table peut être utilisé ([\#2584](https://github.com/ClickHouse/ClickHouse/pull/2584)). +- Retiré redondant vérification des sommes de contrôle lors de l'ajout d'une partie des données. Ceci est important lorsqu'il y a un grand nombre de répliques, car dans ces cas, le nombre total de contrôles était égal à n^2. - Ajout du support pour `Array(Tuple(...))` arguments en faveur de la `arrayEnumerateUniq` fonction ([\#2573](https://github.com/ClickHouse/ClickHouse/pull/2573)). - Ajouter `Nullable` soutien pour le `runningDifference` fonction ([\#2594](https://github.com/ClickHouse/ClickHouse/pull/2594)). -- Amélioration des performances d’analyse des requêtes lorsqu’il existe un très grand nombre d’expressions ([\#2572](https://github.com/ClickHouse/ClickHouse/pull/2572)). +- Amélioration des performances d'analyse des requêtes lorsqu'il existe un très grand nombre d'expressions ([\#2572](https://github.com/ClickHouse/ClickHouse/pull/2572)). - Sélection plus rapide des parties de données à fusionner `ReplicatedMergeTree` table. Récupération plus rapide de la session ZooKeeper ([\#2597](https://github.com/ClickHouse/ClickHouse/pull/2597)). - Le `format_version.txt` fichier pour `MergeTree` tables est recréée si elle est manquante, ce qui est logique si ClickHouse est lancé après avoir copié la structure de répertoire sans fichiers ([Ciprian Hacman](https://github.com/ClickHouse/ClickHouse/pull/2593)). #### Corrections De Bugs: {#bug-fixes-19} -- Correction d’un bug lorsque vous travaillez avec ZooKeeper qui pourrait rendre impossible la récupération de la session et des États en lecture seule des tables avant de redémarrer le serveur. -- Correction d’un bug lorsque vous travaillez avec ZooKeeper qui pourrait entraîner la suppression des anciens nœuds si la session est interrompue. -- Correction d’une erreur dans le `quantileTDigest` fonction pour les arguments Float (ce bug a été introduit dans la version 1.1.54388) ([Mikhail Surin](https://github.com/ClickHouse/ClickHouse/pull/2553)). -- Correction d’un bug dans l’index des tables MergeTree si la colonne de clé primaire est située à l’intérieur de la fonction de conversion des types entre des entiers signés et non signés de même taille ([\#2603](https://github.com/ClickHouse/ClickHouse/pull/2603)). +- Correction d'un bug lorsque vous travaillez avec ZooKeeper qui pourrait rendre impossible la récupération de la session et des États en lecture seule des tables avant de redémarrer le serveur. +- Correction d'un bug lorsque vous travaillez avec ZooKeeper qui pourrait entraîner la suppression des anciens nœuds si la session est interrompue. +- Correction d'une erreur dans le `quantileTDigest` fonction pour les arguments Float (ce bug a été introduit dans la version 1.1.54388) ([Mikhail Surin](https://github.com/ClickHouse/ClickHouse/pull/2553)). +- Correction d'un bug dans l'index des tables MergeTree si la colonne de clé primaire est située à l'intérieur de la fonction de conversion des types entre des entiers signés et non signés de même taille ([\#2603](https://github.com/ClickHouse/ClickHouse/pull/2603)). - Fixe erreur de segmentation si `macros` sont utilisés mais ils ne sont pas dans le fichier de configuration ([\#2570](https://github.com/ClickHouse/ClickHouse/pull/2570)). - Correction du passage à la base de données par défaut lors de la reconnexion du client ([\#2583](https://github.com/ClickHouse/ClickHouse/pull/2583)). -- Correction d’un bug qui se produisait lors de la `use_index_for_in_with_subqueries` paramètre a été désactivé. +- Correction d'un bug qui se produisait lors de la `use_index_for_in_with_subqueries` paramètre a été désactivé. -#### Correction De sécurité: {#security-fix-1} +#### Correction De Sécurité: {#security-fix-1} -- L’envoi de fichiers n’est plus possible lorsqu’il est connecté à MySQL (`LOAD DATA LOCAL INFILE`). +- L'envoi de fichiers n'est plus possible lorsqu'il est connecté à MySQL (`LOAD DATA LOCAL INFILE`). ### Clickhouse Version 1.1.54388, 2018-06-28 {#clickhouse-release-1-1-54388-2018-06-28} @@ -649,57 +649,57 @@ toc_title: '2018' - Soutien pour le `ALTER TABLE t DELETE WHERE` requête pour les tables répliquées. Ajouté le `system.mutations` tableau pour suivre la progression de ce type de requêtes. - Soutien pour le `ALTER TABLE t [REPLACE|ATTACH] PARTITION` requête pour les tables \* MergeTree. -- Soutien pour le `TRUNCATE TABLE` requête ([L’Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/2260)) +- Soutien pour le `TRUNCATE TABLE` requête ([L'Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/2260)) - Plusieurs nouveaux `SYSTEM` requêtes pour les tables répliquées (`RESTART REPLICAS`, `SYNC REPLICA`, `[STOP|START] [MERGES|FETCHES|SENDS REPLICATED|REPLICATION QUEUES]`). -- Ajout de la possibilité d’écrire dans une table avec le moteur MySQL et la fonction de table correspondante ([sundy-li](https://github.com/ClickHouse/ClickHouse/pull/2294)). +- Ajout de la possibilité d'écrire dans une table avec le moteur MySQL et la fonction de table correspondante ([sundy-li](https://github.com/ClickHouse/ClickHouse/pull/2294)). - Ajouté le `url()` fonction de table et le `URL` tableau moteur ([Alexander Sapin](https://github.com/ClickHouse/ClickHouse/pull/2501)). -- Ajouté le `windowFunnel` fonction d’agrégation ([sundy-li](https://github.com/ClickHouse/ClickHouse/pull/2352)). +- Ajouté le `windowFunnel` fonction d'agrégation ([sundy-li](https://github.com/ClickHouse/ClickHouse/pull/2352)). - Nouveau `startsWith` et `endsWith` fonctions pour les chaînes ([Vadim Plakhtinsky](https://github.com/ClickHouse/ClickHouse/pull/2429)). -- Le `numbers()` la fonction table vous permet maintenant de spécifier le décalage ([L’Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/2535)). +- Le `numbers()` la fonction table vous permet maintenant de spécifier le décalage ([L'Hiver Zhang](https://github.com/ClickHouse/ClickHouse/pull/2535)). - Le mot de passe pour `clickhouse-client` peut être saisi de manière interactive. - Les journaux du serveur peuvent maintenant être envoyés à syslog ([Alexander Krasheninnikov](https://github.com/ClickHouse/ClickHouse/pull/2459)). - Prise en charge de la connexion dans les dictionnaires avec une source de bibliothèque partagée ([Alexander Sapin](https://github.com/ClickHouse/ClickHouse/pull/2472)). - Prise en charge des délimiteurs CSV personnalisés ([Ivan Joukov](https://github.com/ClickHouse/ClickHouse/pull/2263)) - Ajouté le `date_time_input_format` paramètre. Si vous basculez ce paramètre sur `'best_effort'`, Les valeurs DateTime seront lues dans un large éventail de formats. -- Ajouté le `clickhouse-obfuscator` utilitaire pour l’obscurcissement des données. Exemple d’utilisation: publication des données utilisées dans les tests de performance. +- Ajouté le `clickhouse-obfuscator` utilitaire pour l'obscurcissement des données. Exemple d'utilisation: publication des données utilisées dans les tests de performance. -#### Caractéristiques expérimentales: {#experimental-features-2} +#### Caractéristiques Expérimentales: {#experimental-features-2} - Ajout de la possibilité de calculer `and` arguments uniquement là où ils sont nécessaires ([Anastasia Tsarkova](https://github.com/ClickHouse/ClickHouse/pull/2272)) - La compilation JIT en code natif est maintenant disponible pour certaines expressions ([pyos](https://github.com/ClickHouse/ClickHouse/pull/2277)). #### Corrections De Bugs: {#bug-fixes-20} -- Les doublons n’apparaissent plus pour une requête avec `DISTINCT` et `ORDER BY`. +- Les doublons n'apparaissent plus pour une requête avec `DISTINCT` et `ORDER BY`. - Les requêtes avec `ARRAY JOIN` et `arrayFilter` ne renvoie plus un résultat incorrect. -- Correction d’une erreur lors de la lecture d’une colonne de tableau à partir d’une structure Imbriquée ([\#2066](https://github.com/ClickHouse/ClickHouse/issues/2066)). -- Correction d’une erreur lors de l’analyse des requêtes avec une clause HAVING comme `HAVING tuple IN (...)`. -- Correction d’une erreur lors de l’analyse des requêtes avec récursive des alias. -- Correction d’une erreur lors de la lecture de ReplacingMergeTree avec une condition dans PREWHERE qui filtre Toutes les lignes ([\#2525](https://github.com/ClickHouse/ClickHouse/issues/2525)). -- Les paramètres de profil utilisateur n’ont pas été appliqués lors de l’utilisation de sessions dans L’interface HTTP. +- Correction d'une erreur lors de la lecture d'une colonne de tableau à partir d'une structure Imbriquée ([\#2066](https://github.com/ClickHouse/ClickHouse/issues/2066)). +- Correction d'une erreur lors de l'analyse des requêtes avec une clause HAVING comme `HAVING tuple IN (...)`. +- Correction d'une erreur lors de l'analyse des requêtes avec récursive des alias. +- Correction d'une erreur lors de la lecture de ReplacingMergeTree avec une condition dans PREWHERE qui filtre Toutes les lignes ([\#2525](https://github.com/ClickHouse/ClickHouse/issues/2525)). +- Les paramètres de profil utilisateur n'ont pas été appliqués lors de l'utilisation de sessions dans L'interface HTTP. - Correction de la façon dont les paramètres sont appliqués à partir des paramètres de ligne de commande dans clickhouse-local. -- La bibliothèque client ZooKeeper utilise maintenant le délai d’attente de session reçu du serveur. -- Correction d’un bug dans la bibliothèque client ZooKeeper lorsque le client attendait la réponse du serveur plus longtemps que le délai d’attente. -- Correction de l’élagage des pièces pour les requêtes avec des conditions sur les colonnes de clé de partition ([\#2342](https://github.com/ClickHouse/ClickHouse/issues/2342)). +- La bibliothèque client ZooKeeper utilise maintenant le délai d'attente de session reçu du serveur. +- Correction d'un bug dans la bibliothèque client ZooKeeper lorsque le client attendait la réponse du serveur plus longtemps que le délai d'attente. +- Correction de l'élagage des pièces pour les requêtes avec des conditions sur les colonnes de clé de partition ([\#2342](https://github.com/ClickHouse/ClickHouse/issues/2342)). - Les fusions sont maintenant possibles après `CLEAR COLUMN IN PARTITION` ([\#2315](https://github.com/ClickHouse/ClickHouse/issues/2315)). - Le mappage de Type dans la fonction de table ODBC a été corrigé ([sundy-li](https://github.com/ClickHouse/ClickHouse/pull/2268)). - Les comparaisons de Type ont été corrigées pour `DateTime` avec et sans le fuseau horaire ([Alexandre Botcharov](https://github.com/ClickHouse/ClickHouse/pull/2400)). -- Correction de l’analyse syntaxique et du formatage `CAST` opérateur. +- Correction de l'analyse syntaxique et du formatage `CAST` opérateur. - Insertion fixe dans une vue matérialisée pour le moteur de table distribué ([Babacar Diassé](https://github.com/ClickHouse/ClickHouse/pull/2411)). -- Correction d’une condition de concurrence lors de l’écriture de données `Kafka` moteur aux vues matérialisées ([Yangkuan Liu](https://github.com/ClickHouse/ClickHouse/pull/2448)). +- Correction d'une condition de concurrence lors de l'écriture de données `Kafka` moteur aux vues matérialisées ([Yangkuan Liu](https://github.com/ClickHouse/ClickHouse/pull/2448)). - Correction de SSRF dans la fonction de table remote (). - Comportement de sortie fixe de `clickhouse-client` en mode multi-lignes ([\#2510](https://github.com/ClickHouse/ClickHouse/issues/2510)). #### Amélioration: {#improvements-11} -- Les tâches d’arrière plan dans les tables répliquées sont maintenant effectuées dans un pool de threads plutôt que dans des threads séparés ([Silviu Caragea](https://github.com/ClickHouse/ClickHouse/pull/1722)). +- Les tâches d'arrière plan dans les tables répliquées sont maintenant effectuées dans un pool de threads plutôt que dans des threads séparés ([Silviu Caragea](https://github.com/ClickHouse/ClickHouse/pull/1722)). - Amélioration des performances de compression LZ4. - Analyse plus rapide pour les requêtes avec un grand nombre de jointures et de sous-requêtes. -- Le cache DNS est maintenant mis à jour automatiquement lorsqu’il y a trop d’erreurs réseau. -- Les insertions de Table ne se produisent plus si l’insertion dans l’une des vues matérialisées n’est pas possible car elle comporte trop de parties. -- Correction de l’écart dans les compteurs d’événements `Query`, `SelectQuery`, et `InsertQuery`. +- Le cache DNS est maintenant mis à jour automatiquement lorsqu'il y a trop d'erreurs réseau. +- Les insertions de Table ne se produisent plus si l'insertion dans l'une des vues matérialisées n'est pas possible car elle comporte trop de parties. +- Correction de l'écart dans les compteurs d'événements `Query`, `SelectQuery`, et `InsertQuery`. - Des Expressions comme `tuple IN (SELECT tuple)` sont autorisés si les types de tuple correspondent. -- Un serveur avec des tables répliquées peut démarrer même si vous n’avez pas configuré ZooKeeper. +- Un serveur avec des tables répliquées peut démarrer même si vous n'avez pas configuré ZooKeeper. - Lors du calcul du nombre de cœurs CPU disponibles, les limites sur les groupes cgroups sont maintenant prises en compte ([Atri Sharma](https://github.com/ClickHouse/ClickHouse/pull/2325)). - Ajouté chown pour les répertoires de configuration dans le fichier de configuration systemd ([Mikhaïl Shirjaeva](https://github.com/ClickHouse/ClickHouse/pull/2421)). @@ -708,111 +708,111 @@ toc_title: '2018' - Le compilateur gcc8 peut être utilisé pour les builds. - Ajout de la possibilité de construire llvm à partir du sous-module. - La version de la bibliothèque librdkafka a été mise à jour vers v0. 11. 4. -- Ajout de la possibilité d’utiliser la bibliothèque libcpuid du système. La version de la bibliothèque a été mise à jour à 0.4.0. +- Ajout de la possibilité d'utiliser la bibliothèque libcpuid du système. La version de la bibliothèque a été mise à jour à 0.4.0. - Correction de la construction en utilisant la bibliothèque vectorclass ([Babacar Diassé](https://github.com/ClickHouse/ClickHouse/pull/2274)). -- Cmake génère maintenant des fichiers pour ninja par défaut (comme lors de l’utilisation `-G Ninja`). -- Ajout de la possibilité d’utiliser la bibliothèque libtinfo au lieu de libtermcap ([Georgy Kondratiev](https://github.com/ClickHouse/ClickHouse/pull/2519)). -- Correction d’un conflit de fichier d’en-tête dans Fedora Rawhide ([\#2520](https://github.com/ClickHouse/ClickHouse/issues/2520)). +- Cmake génère maintenant des fichiers pour ninja par défaut (comme lors de l'utilisation `-G Ninja`). +- Ajout de la possibilité d'utiliser la bibliothèque libtinfo au lieu de libtermcap ([Georgy Kondratiev](https://github.com/ClickHouse/ClickHouse/pull/2519)). +- Correction d'un conflit de fichier d'en-tête dans Fedora Rawhide ([\#2520](https://github.com/ClickHouse/ClickHouse/issues/2520)). -#### Modifications Incompatibles En arrière: {#backward-incompatible-changes-7} +#### Modifications Incompatibles En Arrière: {#backward-incompatible-changes-7} - Retiré échapper dans `Vertical` et `Pretty*` formats et supprimé le `VerticalRaw` format. -- Si des serveurs avec la version 1.1.54388 (ou plus récente) et des serveurs avec une version plus ancienne sont utilisés simultanément dans une requête distribuée et la requête a le `cast(x, 'Type')` expression sans `AS` mot clé et n’a pas le mot `cast` en majuscules, une exception sera levée avec un message du genre `Not found column cast(0, 'UInt8') in block`. Solution: mettez à jour le serveur sur l’ensemble du cluster. +- Si des serveurs avec la version 1.1.54388 (ou plus récente) et des serveurs avec une version plus ancienne sont utilisés simultanément dans une requête distribuée et la requête a le `cast(x, 'Type')` expression sans `AS` mot clé et n'a pas le mot `cast` en majuscules, une exception sera levée avec un message du genre `Not found column cast(0, 'UInt8') in block`. Solution: mettez à jour le serveur sur l'ensemble du cluster. ### Clickhouse Version 1.1.54385, 2018-06-01 {#clickhouse-release-1-1-54385-2018-06-01} #### Corrections De Bugs: {#bug-fixes-21} -- Correction d’une erreur qui, dans certains cas, provoquait le blocage des opérations de ZooKeeper. +- Correction d'une erreur qui, dans certains cas, provoquait le blocage des opérations de ZooKeeper. ### Clickhouse Version 1.1.54383, 2018-05-22 {#clickhouse-release-1-1-54383-2018-05-22} #### Corrections De Bugs: {#bug-fixes-22} -- Correction d’un ralentissement de la file d’attente de réplication si une table a plusieurs répliques. +- Correction d'un ralentissement de la file d'attente de réplication si une table a plusieurs répliques. ### Clickhouse Version 1.1.54381, 2018-05-14 {#clickhouse-release-1-1-54381-2018-05-14} #### Corrections De Bugs: {#bug-fixes-23} -- Correction d’une fuite de nœuds dans ZooKeeper lorsque ClickHouse perd la connexion au serveur ZooKeeper. +- Correction d'une fuite de nœuds dans ZooKeeper lorsque ClickHouse perd la connexion au serveur ZooKeeper. ### Clickhouse Version 1.1.54380, 2018-04-21 {#clickhouse-release-1-1-54380-2018-04-21} #### Nouveauté: {#new-features-13} -- Ajout de la fonction table `file(path, format, structure)`. Un exemple de lecture d’octets depuis `/dev/urandom`: ``` ln -s /dev/urandom /var/lib/clickhouse/user_files/random``clickhouse-client -q "SELECT * FROM file('random', 'RowBinary', 'd UInt8') LIMIT 10" ```. +- Ajout de la fonction table `file(path, format, structure)`. Un exemple de lecture d'octets depuis `/dev/urandom`: ``` ln -s /dev/urandom /var/lib/clickhouse/user_files/random``clickhouse-client -q "SELECT * FROM file('random', 'RowBinary', 'd UInt8') LIMIT 10" ```. #### Amélioration: {#improvements-12} - Les sous-requêtes peuvent être encapsulées `()` crochets pour améliorer la lisibilité des requêtes. Exemple: `(SELECT 1) UNION ALL (SELECT 1)`. -- Simple `SELECT` les requêtes de l’ `system.processes` le tableau ne sont pas inclus dans le `max_concurrent_queries` limite. +- Simple `SELECT` les requêtes de l' `system.processes` le tableau ne sont pas inclus dans le `max_concurrent_queries` limite. #### Corrections De Bugs: {#bug-fixes-24} -- Correction d’un comportement incorrect de la `IN` opérateur quand sélectionner à partir de `MATERIALIZED VIEW`. -- Correction d’un filtrage incorrect par index de partition dans des expressions comme `partition_key_column IN (...)`. +- Correction d'un comportement incorrect de la `IN` opérateur quand sélectionner à partir de `MATERIALIZED VIEW`. +- Correction d'un filtrage incorrect par index de partition dans des expressions comme `partition_key_column IN (...)`. - Incapacité fixe à exécuter `OPTIMIZE` requête sur réplica non leader si `REANAME` a été effectuée sur la table. -- Correction de l’erreur d’autorisation lors de l’exécution `OPTIMIZE` ou `ALTER` requêtes sur une réplique non-leader. -- Fixe le gel de l’ `KILL QUERY`. -- Correction d’une erreur dans la bibliothèque client ZooKeeper qui a conduit à la perte de montres, le gel de la file d’attente DDL distribuée, et des ralentissements dans la file d’attente de réplication si un non vide `chroot` le préfixe est utilisé dans la configuration de ZooKeeper. +- Correction de l'erreur d'autorisation lors de l'exécution `OPTIMIZE` ou `ALTER` requêtes sur une réplique non-leader. +- Fixe le gel de l' `KILL QUERY`. +- Correction d'une erreur dans la bibliothèque client ZooKeeper qui a conduit à la perte de montres, le gel de la file d'attente DDL distribuée, et des ralentissements dans la file d'attente de réplication si un non vide `chroot` le préfixe est utilisé dans la configuration de ZooKeeper. -#### Modifications Incompatibles En arrière: {#backward-incompatible-changes-8} +#### Modifications Incompatibles En Arrière: {#backward-incompatible-changes-8} -- Suppression du support pour les expressions comme `(a, b) IN (SELECT (a, b))` (vous pouvez utiliser l’expression équivalente `(a, b) IN (SELECT a, b)`). Dans les versions précédentes, ces expressions ont conduit à indéterminé `WHERE` filtrage ou causé des erreurs. +- Suppression du support pour les expressions comme `(a, b) IN (SELECT (a, b))` (vous pouvez utiliser l'expression équivalente `(a, b) IN (SELECT a, b)`). Dans les versions précédentes, ces expressions ont conduit à indéterminé `WHERE` filtrage ou causé des erreurs. ### Clickhouse Version 1.1.54378, 2018-04-16 {#clickhouse-release-1-1-54378-2018-04-16} #### Nouveauté: {#new-features-14} -- Niveau d’enregistrement peut être modifié sans redémarrer le serveur. +- Niveau d'enregistrement peut être modifié sans redémarrer le serveur. - Ajouté le `SHOW CREATE DATABASE` requête. - Le `query_id` peut être passé à `clickhouse-client` (elBroom). - Nouveau paramètre: `max_network_bandwidth_for_all_users`. - Ajout du support pour `ALTER TABLE ... PARTITION ...` pour `MATERIALIZED VIEW`. -- Ajout d’informations sur la taille des parties de données sous forme non compressée dans la table système. +- Ajout d'informations sur la taille des parties de données sous forme non compressée dans la table système. - Prise en charge du chiffrement de serveur à serveur pour les tables distribuées (`1` dans la configuration de la réplique ``). - Configuration du niveau de la table pour `ReplicatedMergeTree` la famille afin de minimiser la quantité de données stockées dans Zookeeper: : `use_minimalistic_checksums_in_zookeeper = 1` -- Configuration de l’ `clickhouse-client` invite. Par défaut, les noms de serveur sont maintenant affichés à l’invite. Le nom d’affichage du serveur peut être modifié. Il est également envoyé dans le `X-ClickHouse-Display-Name` En-tête HTTP (Kirill Shvakov). -- Séparés par des virgules multiples `topics` peut être spécifié pour l’ `Kafka` moteur (Tobias Adamson) -- Quand une requête est arrêtée par `KILL QUERY` ou `replace_running_query` le client reçoit l’ `Query was canceled` exception au lieu d’un résultat incomplète. +- Configuration de l' `clickhouse-client` invite. Par défaut, les noms de serveur sont maintenant affichés à l'invite. Le nom d'affichage du serveur peut être modifié. Il est également envoyé dans le `X-ClickHouse-Display-Name` En-tête HTTP (Kirill Shvakov). +- Séparés par des virgules multiples `topics` peut être spécifié pour l' `Kafka` moteur (Tobias Adamson) +- Quand une requête est arrêtée par `KILL QUERY` ou `replace_running_query` le client reçoit l' `Query was canceled` exception au lieu d'un résultat incomplète. #### Amélioration: {#improvements-13} -- `ALTER TABLE ... DROP/DETACH PARTITION` les requêtes sont exécutées à l’avant de la file d’attente de réplication. +- `ALTER TABLE ... DROP/DETACH PARTITION` les requêtes sont exécutées à l'avant de la file d'attente de réplication. - `SELECT ... FINAL` et `OPTIMIZE ... FINAL` peut être utilisé même lorsque la table a une seule partie de données. - A `query_log` la table est recréée à la volée si elle a été supprimée manuellement (Kirill Shvakov). - Le `lengthUTF8` fonction fonctionne plus rapidement (zhang2014). -- Amélioration des performances des inserts synchrones dans `Distributed` table (`insert_distributed_sync = 1` lorsqu’il existe un très grand nombre de fragments. -- Le serveur accepte le `send_timeout` et `receive_timeout` les paramètres du client et les applique lors de la connexion au client (ils sont appliqués dans l’ordre inverse: le socket du serveur `send_timeout` est définie à l’ `receive_timeout` valeur reçue du client, et vice versa). +- Amélioration des performances des inserts synchrones dans `Distributed` table (`insert_distributed_sync = 1` lorsqu'il existe un très grand nombre de fragments. +- Le serveur accepte le `send_timeout` et `receive_timeout` les paramètres du client et les applique lors de la connexion au client (ils sont appliqués dans l'ordre inverse: le socket du serveur `send_timeout` est définie à l' `receive_timeout` valeur reçue du client, et vice versa). - Récupération de crash plus robuste pour une insertion asynchrone dans `Distributed` table. - Le type de retour de la `countEqual` la fonction a changé à partir de `UInt32` de `UInt64` (谢磊). #### Corrections De Bugs: {#bug-fixes-25} -- Correction d’une erreur avec `IN` lorsque le côté gauche de l’expression est `Nullable`. -- Les résultats corrects sont maintenant retournés lors de l’utilisation de tuples avec `IN` lorsque certains des composants tuple sont dans l’index de la table. +- Correction d'une erreur avec `IN` lorsque le côté gauche de l'expression est `Nullable`. +- Les résultats corrects sont maintenant retournés lors de l'utilisation de tuples avec `IN` lorsque certains des composants tuple sont dans l'index de la table. - Le `max_execution_time` limite fonctionne désormais correctement avec les requêtes distribuées. -- Correction d’erreurs lors du calcul de la taille des colonnes composites `system.columns` table. -- Correction d’une erreur lors de la création d’une table temporaire `CREATE TEMPORARY TABLE IF NOT EXISTS.` +- Correction d'erreurs lors du calcul de la taille des colonnes composites `system.columns` table. +- Correction d'une erreur lors de la création d'une table temporaire `CREATE TEMPORARY TABLE IF NOT EXISTS.` - Erreurs corrigées dans `StorageKafka` (\#\#2075) -- Le serveur fixe se bloque à partir d’arguments non valides de certaines fonctions d’agrégat. -- Correction de l’erreur qui a empêché l’ `DETACH DATABASE` requête de l’arrêt des tâches d’arrière-plan pour `ReplicatedMergeTree` table. -- `Too many parts` l’état est moins susceptible de se produire lors de l’insertion dans agrégées des vues matérialisées (\#\#2084). -- Correction de la gestion récursive des substitutions dans la configuration si une substitution doit être suivie d’une autre substitution au même niveau. -- Correction de la syntaxe dans le fichier de métadonnées lors de la création d’un `VIEW` qui utilise une requête avec `UNION ALL`. +- Le serveur fixe se bloque à partir d'arguments non valides de certaines fonctions d'agrégat. +- Correction de l'erreur qui a empêché l' `DETACH DATABASE` requête de l'arrêt des tâches d'arrière-plan pour `ReplicatedMergeTree` table. +- `Too many parts` l'état est moins susceptible de se produire lors de l'insertion dans agrégées des vues matérialisées (\#\#2084). +- Correction de la gestion récursive des substitutions dans la configuration si une substitution doit être suivie d'une autre substitution au même niveau. +- Correction de la syntaxe dans le fichier de métadonnées lors de la création d'un `VIEW` qui utilise une requête avec `UNION ALL`. - `SummingMergeTree` fonctionne maintenant correctement pour la sommation des structures de données imbriquées avec une clé composite. -- Correction de la possibilité d’une condition de course lors du choix du leader pour `ReplicatedMergeTree` table. +- Correction de la possibilité d'une condition de course lors du choix du leader pour `ReplicatedMergeTree` table. #### Construire Des Changements: {#build-changes-5} - La construction prend en charge `ninja` plutôt `make` et utilise `ninja` par défaut pour les versions de construction. - Paquets renommés: `clickhouse-server-base` dans `clickhouse-common-static`; `clickhouse-server-common` dans `clickhouse-server`; `clickhouse-common-dbg` dans `clickhouse-common-static-dbg`. Pour installer, utilisez `clickhouse-server clickhouse-client`. Les paquets avec les anciens noms seront toujours chargés dans les dépôts pour une compatibilité descendante. -#### Modifications Incompatibles En arrière: {#backward-incompatible-changes-9} +#### Modifications Incompatibles En Arrière: {#backward-incompatible-changes-9} -- Retiré de l’interprétation d’une expression si un tableau est spécifié sur le côté gauche. Auparavant, l’expression `arr IN (set)` a été interprété comme “at least one `arr` element belongs to the `set`”. Pour obtenir le même comportement dans la nouvelle version, écrire `arrayExists(x -> x IN (set), arr)`. -- Désactivé l’utilisation incorrecte de l’option socket `SO_REUSEPORT`, qui a été incorrectement activé par défaut dans la bibliothèque Poco. Notez que sous Linux il n’y a plus aucune raison de spécifier simultanément les adresses `::` et `0.0.0.0` for listen – use just `::`, qui permet d’écouter la connexion à la fois sur IPv4 et IPv6 (avec les paramètres de configuration du noyau par défaut). Vous pouvez également revenir au comportement des versions précédentes en spécifiant `1` dans la config. +- Retiré de l'interprétation d'une expression si un tableau est spécifié sur le côté gauche. Auparavant, l'expression `arr IN (set)` a été interprété comme “at least one `arr` element belongs to the `set`”. Pour obtenir le même comportement dans la nouvelle version, écrire `arrayExists(x -> x IN (set), arr)`. +- Désactivé l'utilisation incorrecte de l'option socket `SO_REUSEPORT`, qui a été incorrectement activé par défaut dans la bibliothèque Poco. Notez que sous Linux il n'y a plus aucune raison de spécifier simultanément les adresses `::` et `0.0.0.0` for listen – use just `::`, qui permet d'écouter la connexion à la fois sur IPv4 et IPv6 (avec les paramètres de configuration du noyau par défaut). Vous pouvez également revenir au comportement des versions précédentes en spécifiant `1` dans la config. ### Clickhouse Version 1.1.54370, 2018-03-16 {#clickhouse-release-1-1-54370-2018-03-16} @@ -820,46 +820,46 @@ toc_title: '2018' - Ajouté le `system.macros` table et mise à jour automatique des macros lorsque le fichier de configuration est modifié. - Ajouté le `SYSTEM RELOAD CONFIG` requête. -- Ajouté le `maxIntersections(left_col, right_col)` fonction d’agrégation, qui retourne le nombre maximal d’simultanément intersection d’intervalles `[left; right]`. Le `maxIntersectionsPosition(left, right)` fonction retourne le début de la “maximum” intervalle. ([Michael Furmur](https://github.com/ClickHouse/ClickHouse/pull/2012)). +- Ajouté le `maxIntersections(left_col, right_col)` fonction d'agrégation, qui retourne le nombre maximal d'simultanément intersection d'intervalles `[left; right]`. Le `maxIntersectionsPosition(left, right)` fonction retourne le début de la “maximum” intervalle. ([Michael Furmur](https://github.com/ClickHouse/ClickHouse/pull/2012)). #### Amélioration: {#improvements-14} -- Lors de l’insertion de données dans une `Replicated` tableau, moins de demandes sont faites à `ZooKeeper` (et la plupart des erreurs au niveau de l’utilisateur ont disparu de la `ZooKeeper` journal). +- Lors de l'insertion de données dans une `Replicated` tableau, moins de demandes sont faites à `ZooKeeper` (et la plupart des erreurs au niveau de l'utilisateur ont disparu de la `ZooKeeper` journal). - Ajout de la possibilité de créer des alias pour les ensembles de données. Exemple: `WITH (1, 2, 3) AS set SELECT number IN set FROM system.numbers LIMIT 10`. #### Corrections De Bugs: {#bug-fixes-26} -- Correction de l’ `Illegal PREWHERE` erreur lors de la lecture des tables de fusion pour `Distributed`table. +- Correction de l' `Illegal PREWHERE` erreur lors de la lecture des tables de fusion pour `Distributed`table. - Ajout de correctifs qui vous permettent de démarrer clickhouse-server dans des conteneurs Docker IPv4 uniquement. -- Correction d’une condition de course lors de la lecture du système `system.parts_columns tables.` -- Suppression de la double mise en mémoire tampon lors d’un insert synchrone `Distributed` table, ce qui aurait pu provoquer la connexion à timeout. -- Correction d’un bug qui a causé des attentes trop longues pour une réplique indisponible avant de commencer un `SELECT` requête. +- Correction d'une condition de course lors de la lecture du système `system.parts_columns tables.` +- Suppression de la double mise en mémoire tampon lors d'un insert synchrone `Distributed` table, ce qui aurait pu provoquer la connexion à timeout. +- Correction d'un bug qui a causé des attentes trop longues pour une réplique indisponible avant de commencer un `SELECT` requête. - Correction de dates incorrectes dans le `system.parts` table. -- Correction d’un bug qui rendait impossible l’insertion de données dans un `Replicated` le tableau si `chroot` était non vide dans la configuration du `ZooKeeper` cluster. -- Correction de l’algorithme de fusion verticale pour un `ORDER BY` table. -- Restauré la possibilité d’utiliser des dictionnaires dans les requêtes aux tables distantes, même si ces dictionnaires ne sont pas présents sur le serveur demandeur. Cette fonctionnalité a été perdue dans la version 1.1.54362. -- Restauré le comportement pour les requêtes comme `SELECT * FROM remote('server2', default.table) WHERE col IN (SELECT col2 FROM default.table)` lorsque le côté droit de la `IN` devrait utiliser une télécommande `default.table` au lieu d’un local. Ce comportement a été rompu dans la version 1.1.54358. +- Correction d'un bug qui rendait impossible l'insertion de données dans un `Replicated` le tableau si `chroot` était non vide dans la configuration du `ZooKeeper` cluster. +- Correction de l'algorithme de fusion verticale pour un `ORDER BY` table. +- Restauré la possibilité d'utiliser des dictionnaires dans les requêtes aux tables distantes, même si ces dictionnaires ne sont pas présents sur le serveur demandeur. Cette fonctionnalité a été perdue dans la version 1.1.54362. +- Restauré le comportement pour les requêtes comme `SELECT * FROM remote('server2', default.table) WHERE col IN (SELECT col2 FROM default.table)` lorsque le côté droit de la `IN` devrait utiliser une télécommande `default.table` au lieu d'un local. Ce comportement a été rompu dans la version 1.1.54358. - Suppression de la journalisation au niveau des erreurs `Not found column ... in block`. ### Clickhouse Version 1.1.54362, 2018-03-11 {#clickhouse-release-1-1-54362-2018-03-11} #### Nouveauté: {#new-features-16} -- Agrégation sans `GROUP BY` pour un ensemble vide (comme `SELECT count(*) FROM table WHERE 0`) renvoie maintenant un résultat avec une ligne avec des valeurs null pour les fonctions d’agrégation, conformément à la norme SQL. Pour restaurer l’ancien comportement (renvoyer un résultat vide), définissez `empty_result_for_aggregation_by_empty_set` 1. -- Conversion de type ajouté pour `UNION ALL`. Différents noms d’alias sont autorisés dans `SELECT` les positions dans `UNION ALL` en conformité avec le standard SQL. -- Les expressions arbitraires sont prises en charge dans `LIMIT BY` clause. Auparavant, il était seulement possible d’utiliser des colonnes résultant de `SELECT`. -- Un indice de `MergeTree` tables est utilisé lorsque `IN` est appliqué à un n-uplet d’expressions à partir des colonnes de la clé primaire. Exemple: `WHERE (UserID, EventDate) IN ((123, '2000-01-01'), ...)` (Anastasiya Tsarkova). +- Agrégation sans `GROUP BY` pour un ensemble vide (comme `SELECT count(*) FROM table WHERE 0`) renvoie maintenant un résultat avec une ligne avec des valeurs null pour les fonctions d'agrégation, conformément à la norme SQL. Pour restaurer l'ancien comportement (renvoyer un résultat vide), définissez `empty_result_for_aggregation_by_empty_set` 1. +- Conversion de type ajouté pour `UNION ALL`. Différents noms d'alias sont autorisés dans `SELECT` les positions dans `UNION ALL` en conformité avec le standard SQL. +- Les expressions arbitraires sont prises en charge dans `LIMIT BY` clause. Auparavant, il était seulement possible d'utiliser des colonnes résultant de `SELECT`. +- Un indice de `MergeTree` tables est utilisé lorsque `IN` est appliqué à un n-uplet d'expressions à partir des colonnes de la clé primaire. Exemple: `WHERE (UserID, EventDate) IN ((123, '2000-01-01'), ...)` (Anastasiya Tsarkova). - Ajouté le `clickhouse-copier` outil pour copier entre les clusters et remodeler les données (beta). - Ajout de fonctions de hachage cohérentes: `yandexConsistentHash`, `jumpConsistentHash`, `sumburConsistentHash`. Ils peuvent être utilisés comme une clé de sharding afin de réduire la quantité de trafic réseau lors de remaniements ultérieurs. -- L’ajout de fonctions: `arrayAny`, `arrayAll`, `hasAny`, `hasAll`, `arrayIntersect`, `arrayResize`. +- L'ajout de fonctions: `arrayAny`, `arrayAll`, `hasAny`, `hasAll`, `arrayIntersect`, `arrayResize`. - Ajouté le `arrayCumSum` fonction (Javi Santana). -- Ajouté le `parseDateTimeBestEffort`, `parseDateTimeBestEffortOrZero`, et `parseDateTimeBestEffortOrNull` fonctions pour lire le DateTime à partir d’une chaîne contenant du texte dans une grande variété de formats possibles. +- Ajouté le `parseDateTimeBestEffort`, `parseDateTimeBestEffortOrZero`, et `parseDateTimeBestEffortOrNull` fonctions pour lire le DateTime à partir d'une chaîne contenant du texte dans une grande variété de formats possibles. - Les données peuvent être partiellement rechargées à partir de dictionnaires externes lors de la mise à jour (charger uniquement les enregistrements dans lesquels la valeur du champ spécifié supérieure à celle du téléchargement précédent) (Arsen Hakobyan). -- Ajouté le `cluster` table de fonction. Exemple: `cluster(cluster_name, db, table)`. Le `remote` la fonction table peut accepter le nom du cluster comme premier argument, s’il est spécifié comme identifiant. +- Ajouté le `cluster` table de fonction. Exemple: `cluster(cluster_name, db, table)`. Le `remote` la fonction table peut accepter le nom du cluster comme premier argument, s'il est spécifié comme identifiant. - Le `remote` et `cluster` les fonctions de table peuvent être utilisées dans `INSERT` requête. - Ajouté le `create_table_query` et `engine_full` colonnes virtuelles au `system.tables`table . Le `metadata_modification_time` la colonne est virtuel. - Ajouté le `data_path` et `metadata_path` les colonnes à `system.tables`et`system.databases` tables, et a ajouté le `path` la colonne de la `system.parts` et `system.parts_columns` table. -- Ajout d’informations supplémentaires sur les fusions `system.part_log` table. +- Ajout d'informations supplémentaires sur les fusions `system.part_log` table. - Une clé de partitionnement arbitraire peut être utilisée pour `system.query_log` table (Kirill Shvakov). - Le `SHOW TABLES` query affiche maintenant également des tables temporaires. Ajout de tables temporaires et `is_temporary` colonne de `system.tables` (zhang2014). - Ajouter `DROP TEMPORARY TABLE` et `EXISTS TEMPORARY TABLE` les requêtes (zhang2014). @@ -867,65 +867,65 @@ toc_title: '2018' - Ajouté le `system_profile` paramètre de configuration pour les paramètres utilisés par les processus internes. - Soutien pour le chargement `object_id` comme un attribut de `MongoDB` dictionnaires (Pavel Litvinenko). - Lecture `null` comme valeur par défaut lors du chargement de données pour un dictionnaire externe `MongoDB` source (Pavel Litvinenko). -- Lecture `DateTime` les valeurs dans la `Values` formater à partir D’un horodatage Unix sans guillemets simples. +- Lecture `DateTime` les valeurs dans la `Values` formater à partir D'un horodatage Unix sans guillemets simples. - Le basculement est pris en charge dans `remote` fonctions de table pour les cas où certaines répliques manquent la table demandée. - Les paramètres de Configuration peuvent être remplacées dans la ligne de commande lorsque vous exécutez `clickhouse-server`. Exemple: `clickhouse-server -- --logger.level=information`. -- Mise en œuvre de la `empty` fonction à partir d’un `FixedString` argument: la fonction renvoie 1 si la chaîne est entièrement composée d’octets nuls (zhang2014). -- Ajouté le `listen_try`paramètre de configuration pour l’écoute d’au moins une des adresses listen sans quitter, si certaines adresses ne peuvent pas être écoutées (utile pour les systèmes avec prise en charge désactivée pour IPv4 ou IPv6). +- Mise en œuvre de la `empty` fonction à partir d'un `FixedString` argument: la fonction renvoie 1 si la chaîne est entièrement composée d'octets nuls (zhang2014). +- Ajouté le `listen_try`paramètre de configuration pour l'écoute d'au moins une des adresses listen sans quitter, si certaines adresses ne peuvent pas être écoutées (utile pour les systèmes avec prise en charge désactivée pour IPv4 ou IPv6). - Ajouté le `VersionedCollapsingMergeTree` tableau moteur. - Prise en charge des lignes et des types numériques arbitraires `library` source du dictionnaire. - `MergeTree` les tableaux peuvent être utilisés sans une clé primaire (vous devez spécifier `ORDER BY tuple()`). -- A `Nullable` peut être de type `CAST` pour un non-`Nullable` type si l’argument n’est pas `NULL`. +- A `Nullable` peut être de type `CAST` pour un non-`Nullable` type si l'argument n'est pas `NULL`. - `RENAME TABLE` peut être effectuée pour `VIEW`. - Ajouté le `throwIf` fonction. -- Ajouté le `odbc_default_field_size` option, qui vous permet d’étendre la taille maximale de la valeur chargée à partir D’une source ODBC (par défaut, il est 1024). +- Ajouté le `odbc_default_field_size` option, qui vous permet d'étendre la taille maximale de la valeur chargée à partir D'une source ODBC (par défaut, il est 1024). - Le `system.processes` table et `SHOW PROCESSLIST` ont maintenant la `is_cancelled` et `peak_memory_usage` colonne. #### Amélioration: {#improvements-15} - Les limites et quotas sur le résultat ne sont plus appliqués aux données intermédiaires pour `INSERT SELECT` les requêtes ou pour `SELECT` les sous-requêtes. -- Moins de faux déclencheurs de `force_restore_data` lors de la vérification de l’état de `Replicated` les tables lorsque le serveur démarre. +- Moins de faux déclencheurs de `force_restore_data` lors de la vérification de l'état de `Replicated` les tables lorsque le serveur démarre. - Ajouté le `allow_distributed_ddl` option. - Les fonctions non déterministes ne sont pas autorisées dans les expressions `MergeTree` table de clés. - Fichiers avec des substitutions de `config.d` les répertoires sont chargés par ordre alphabétique. -- Amélioration de la performance de l’ `arrayElement` fonction dans le cas d’une constante tableau multidimensionnel avec un tableau vide comme l’un des éléments. Exemple: `[[1], []][x]`. -- Le serveur démarre plus rapidement maintenant lors de l’utilisation de fichiers de configuration avec de très grandes substitutions (par exemple, de très grandes listes de réseaux IP). -- Lors de l’exécution d’une requête, les fonctions de valeur de table s’exécutent une fois. Précédemment, `remote` et `mysql` les fonctions à valeur de table ont effectué la même requête deux fois pour récupérer la structure de la table à partir d’un serveur distant. +- Amélioration de la performance de l' `arrayElement` fonction dans le cas d'une constante tableau multidimensionnel avec un tableau vide comme l'un des éléments. Exemple: `[[1], []][x]`. +- Le serveur démarre plus rapidement maintenant lors de l'utilisation de fichiers de configuration avec de très grandes substitutions (par exemple, de très grandes listes de réseaux IP). +- Lors de l'exécution d'une requête, les fonctions de valeur de table s'exécutent une fois. Précédemment, `remote` et `mysql` les fonctions à valeur de table ont effectué la même requête deux fois pour récupérer la structure de la table à partir d'un serveur distant. - Le `MkDocs` générateur de documentation est utilisé. - Lorsque vous essayez de supprimer une colonne de table `DEFAULT`/`MATERIALIZED` les expressions des autres colonnes dépendent, une exception est levée (zhang2014). -- Ajout de la possibilité d’analyser une ligne vide dans des formats de texte comme le nombre 0 pour `Float` types de données. Cette fonctionnalité était auparavant disponible mais a été perdue dans la version 1.1.54342. +- Ajout de la possibilité d'analyser une ligne vide dans des formats de texte comme le nombre 0 pour `Float` types de données. Cette fonctionnalité était auparavant disponible mais a été perdue dans la version 1.1.54342. - `Enum` les valeurs peuvent être utilisés dans `min`, `max`, `sum` et quelques autres fonctions. Dans ces cas, il utilise des valeurs numériques correspondantes. Cette fonctionnalité était auparavant disponible mais a été perdue dans la version 1.1.54337. -- Ajouter `max_expanded_ast_elements` pour limiter la taille de L’AST après l’expansion récursive des alias. +- Ajouter `max_expanded_ast_elements` pour limiter la taille de L'AST après l'expansion récursive des alias. #### Corrections De Bugs: {#bug-fixes-27} - Correction de cas où des colonnes inutiles ont été supprimées des sous-requêtes par erreur, ou non supprimées des sous-requêtes contenant `UNION ALL`. -- Correction d’un bug dans les fusions pour `ReplacingMergeTree` table. +- Correction d'un bug dans les fusions pour `ReplacingMergeTree` table. - Insertions synchrones fixes dans `Distributed` table (`insert_distributed_sync = 1`). - Fixe erreur de segmentation pour certaines utilisations de `FULL` et `RIGHT JOIN` avec des colonnes en double dans les sous-requêtes. - Fixe erreur de segmentation pour certaines utilisations de `replace_running_query` et `KILL QUERY`. -- Fixe l’ordre du `source` et `last_exception` les colonnes dans l’ `system.dictionaries` table. -- Correction d’un bug lors de l’ `DROP DATABASE` la requête n’a pas supprimé le fichier contenant des métadonnées. -- Correction de l’ `DROP DATABASE` requête pour `Dictionary` les bases de données. -- Fixe la faible précision de `uniqHLL12` et `uniqCombined` fonctions pour les cardinalités supérieures à 100 millions d’articles (Alex Bocharov). +- Fixe l'ordre du `source` et `last_exception` les colonnes dans l' `system.dictionaries` table. +- Correction d'un bug lors de l' `DROP DATABASE` la requête n'a pas supprimé le fichier contenant des métadonnées. +- Correction de l' `DROP DATABASE` requête pour `Dictionary` les bases de données. +- Fixe la faible précision de `uniqHLL12` et `uniqCombined` fonctions pour les cardinalités supérieures à 100 millions d'articles (Alex Bocharov). - Correction du calcul des valeurs par défaut implicites si nécessaire pour calculer simultanément des expressions explicites par défaut dans `INSERT` les requêtes (zhang2014). -- Correction d’un cas rare lorsqu’une requête à un `MergeTree` la table n’a pas pu finir (chenxing-xc). -- Correction d’un plantage survenu lors de l’exécution d’un `CHECK` requête pour `Distributed` tables si tous les fragments sont locaux (chenxing.xc). -- Correction d’une légère régression des performances avec des fonctions qui utilisent des expressions régulières. -- Correction d’une régression de performance lors de la création de tableaux multidimensionnels à partir d’expressions complexes. -- Correction d’un bug qui pourrait causer un supplément `FORMAT` article à paraître dans un `.sql` fichier de métadonnées. -- Correction d’un bug qui a causé la `max_table_size_to_drop` limite à appliquer lorsque vous essayez de supprimer un `MATERIALIZED VIEW` en regardant une table explicitement spécifiée. -- Correction de l’incompatibilité avec les anciens clients (les anciens clients étaient parfois envoyés `DateTime('timezone')` type, dont ils ne comprennent pas). -- Correction d’un bug lors de la lecture `Nested` éléments de colonne de structures qui ont été ajoutés en utilisant `ALTER` mais qui sont vides pour les anciennes partitions, lorsque les conditions de ces colonnes a déménagé à `PREWHERE`. -- Correction d’un bug lors du filtrage des tables virtuelles `_table` colonnes dans les requêtes à `Merge` table. -- Correction d’un bug lors de l’utilisation `ALIAS` les colonnes en `Distributed` table. -- Correction d’un bug qui rendait la compilation dynamique impossible pour les requêtes avec des fonctions `quantile` famille. -- Correction d’une condition de concurrence dans le pipeline d’exécution de requête qui s’est produite dans de très rares cas lors de l’utilisation `Merge` avec un grand nombre de tables, et lors de l’utilisation `GLOBAL` les sous-requêtes. -- Correction d’un crash lors du passage de tableaux de tailles différentes pour un `arrayReduce` fonction lors de l’utilisation de fonctions d’agrégation à partir de plusieurs arguments. -- Interdit l’utilisation de requêtes avec `UNION ALL` dans un `MATERIALIZED VIEW`. -- Correction d’une erreur lors de l’initialisation du `part_log` table système au démarrage du serveur (par défaut, `part_log` est désactivée). +- Correction d'un cas rare lorsqu'une requête à un `MergeTree` la table n'a pas pu finir (chenxing-xc). +- Correction d'un plantage survenu lors de l'exécution d'un `CHECK` requête pour `Distributed` tables si tous les fragments sont locaux (chenxing.xc). +- Correction d'une légère régression des performances avec des fonctions qui utilisent des expressions régulières. +- Correction d'une régression de performance lors de la création de tableaux multidimensionnels à partir d'expressions complexes. +- Correction d'un bug qui pourrait causer un supplément `FORMAT` article à paraître dans un `.sql` fichier de métadonnées. +- Correction d'un bug qui a causé la `max_table_size_to_drop` limite à appliquer lorsque vous essayez de supprimer un `MATERIALIZED VIEW` en regardant une table explicitement spécifiée. +- Correction de l'incompatibilité avec les anciens clients (les anciens clients étaient parfois envoyés `DateTime('timezone')` type, dont ils ne comprennent pas). +- Correction d'un bug lors de la lecture `Nested` éléments de colonne de structures qui ont été ajoutés en utilisant `ALTER` mais qui sont vides pour les anciennes partitions, lorsque les conditions de ces colonnes a déménagé à `PREWHERE`. +- Correction d'un bug lors du filtrage des tables virtuelles `_table` colonnes dans les requêtes à `Merge` table. +- Correction d'un bug lors de l'utilisation `ALIAS` les colonnes en `Distributed` table. +- Correction d'un bug qui rendait la compilation dynamique impossible pour les requêtes avec des fonctions `quantile` famille. +- Correction d'une condition de concurrence dans le pipeline d'exécution de requête qui s'est produite dans de très rares cas lors de l'utilisation `Merge` avec un grand nombre de tables, et lors de l'utilisation `GLOBAL` les sous-requêtes. +- Correction d'un crash lors du passage de tableaux de tailles différentes pour un `arrayReduce` fonction lors de l'utilisation de fonctions d'agrégation à partir de plusieurs arguments. +- Interdit l'utilisation de requêtes avec `UNION ALL` dans un `MATERIALIZED VIEW`. +- Correction d'une erreur lors de l'initialisation du `part_log` table système au démarrage du serveur (par défaut, `part_log` est désactivée). -#### Modifications Incompatibles En arrière: {#backward-incompatible-changes-10} +#### Modifications Incompatibles En Arrière: {#backward-incompatible-changes-10} - Enlevé le `distributed_ddl_allow_replicated_alter` option. Ce comportement est activé par défaut. - Enlevé le `strict_insert_defaults` paramètre. Si vous utilisez cette fonctionnalité, écrivez à `clickhouse-feedback@yandex-team.com`. @@ -934,20 +934,20 @@ toc_title: '2018' ### Clickhouse Version 1.1.54343, 2018-02-05 {#clickhouse-release-1-1-54343-2018-02-05} - Ajout de la prise en charge des macros pour définir les noms de cluster dans les requêtes DDL distribuées et les constructeurs de tables distribuées: `CREATE TABLE distr ON CLUSTER '{cluster}' (...) ENGINE = Distributed('{cluster}', 'db', 'table')`. -- Maintenant requêtes comme `SELECT ... FROM table WHERE expr IN (subquery)` sont traitées à l’aide de la `table` index. -- Amélioration du traitement des doublons lors de l’insertion dans des tables répliquées, de sorte qu’ils ne ralentissent plus l’exécution de la file d’attente de réplication. +- Maintenant requêtes comme `SELECT ... FROM table WHERE expr IN (subquery)` sont traitées à l'aide de la `table` index. +- Amélioration du traitement des doublons lors de l'insertion dans des tables répliquées, de sorte qu'ils ne ralentissent plus l'exécution de la file d'attente de réplication. ### Clickhouse Version 1.1.54342, 2018-01-22 {#clickhouse-release-1-1-54342-2018-01-22} Cette version contient des corrections de bugs pour la version précédente 1.1.54337: -- Correction d’une régression dans 1.1.54337: si l’utilisateur par défaut a un accès en lecture seule, le serveur refuse de démarrer avec le message `Cannot create database in readonly mode`. -- Correction d’une régression dans 1.1.54337: sur les systèmes avec systemd, les journaux sont toujours écrits dans syslog quelle que soit la configuration; le script watchdog utilise toujours init.d. -- Correction d’une régression dans 1.1.54337: mauvaise configuration par défaut dans L’image Docker. +- Correction d'une régression dans 1.1.54337: si l'utilisateur par défaut a un accès en lecture seule, le serveur refuse de démarrer avec le message `Cannot create database in readonly mode`. +- Correction d'une régression dans 1.1.54337: sur les systèmes avec systemd, les journaux sont toujours écrits dans syslog quelle que soit la configuration; le script watchdog utilise toujours init.d. +- Correction d'une régression dans 1.1.54337: mauvaise configuration par défaut dans L'image Docker. - Correction du comportement non déterministe de GraphiteMergeTree (vous pouvez le voir dans les messages de journal `Data after merge is not byte-identical to the data on another replicas`). -- Correction d’un bug qui peut conduire à des fusions incohérentes après optimiser la requête aux tables répliquées (vous pouvez le voir dans les messages de journal `Part ... intersects the previous part`). +- Correction d'un bug qui peut conduire à des fusions incohérentes après optimiser la requête aux tables répliquées (vous pouvez le voir dans les messages de journal `Part ... intersects the previous part`). - Les tables tampon fonctionnent maintenant correctement lorsque des colonnes matérialisées sont présentes dans la table de destination (par zhang2014). -- Correction d’un bug dans la mise en œuvre de NULL. +- Correction d'un bug dans la mise en œuvre de NULL. ### Clickhouse Version 1.1.54337, 2018-01-18 {#clickhouse-release-1-1-54337-2018-01-18} @@ -955,109 +955,109 @@ Cette version contient des corrections de bugs pour la version précédente 1.1. - Ajout du support pour le stockage de tableaux et de tuples multidimensionnels (`Tuple` type de données) dans les tableaux. - Prise en charge des fonctions de table pour `DESCRIBE` et `INSERT` requête. Ajout du support pour les sous-requêtes dans `DESCRIBE`. Exemple: `DESC TABLE remote('host', default.hits)`; `DESC TABLE (SELECT 1)`; `INSERT INTO TABLE FUNCTION remote('host', default.hits)`. Soutien pour `INSERT INTO TABLE` outre `INSERT INTO`. -- Amélioration du support pour les fuseaux horaires. Le `DateTime` le type de données peut être annoté avec le fuseau horaire utilisé pour l’analyse et le formatage dans les formats de texte. Exemple: `DateTime('Europe/Moscow')`. Lorsque les fuseaux horaires sont spécifiés dans les fonctions `DateTime` arguments, le type de retour pour suivre le fuseau horaire, et la valeur sera affichée comme prévu. +- Amélioration du support pour les fuseaux horaires. Le `DateTime` le type de données peut être annoté avec le fuseau horaire utilisé pour l'analyse et le formatage dans les formats de texte. Exemple: `DateTime('Europe/Moscow')`. Lorsque les fuseaux horaires sont spécifiés dans les fonctions `DateTime` arguments, le type de retour pour suivre le fuseau horaire, et la valeur sera affichée comme prévu. - Ajout des fonctions `toTimeZone`, `timeDiff`, `toQuarter`, `toRelativeQuarterNum`. Le `toRelativeHour`/`Minute`/`Second` les fonctions peuvent prendre une valeur de type `Date` comme argument. Le `now` nom de la fonction est sensible à la casse. - Ajouté le `toStartOfFifteenMinutes` fonction (Kirill Shvakov). - Ajouté le `clickhouse format` outil de formatage des requêtes. - Ajouté le `format_schema_path` configuration parameter (Marek Vavruşa). It is used for specifying a schema in `Cap'n Proto` format. Les fichiers de schéma peuvent être situés uniquement dans le répertoire spécifié. - Ajout du support pour les substitutions de configuration (`incl` et `conf.d`) pour la configuration de dictionnaires et de modèles externes (Pavel Yakunin). -- Ajout d’une colonne avec la documentation pour le `system.settings` table (Kirill Shvakov). +- Ajout d'une colonne avec la documentation pour le `system.settings` table (Kirill Shvakov). - Ajouté le `system.parts_columns` table avec des informations sur la taille des colonnes dans chaque partie `MergeTree` table. -- Ajouté le `system.models` tableau avec des informations sur chargé `CatBoost` machine de modèles d’apprentissage. +- Ajouté le `system.models` tableau avec des informations sur chargé `CatBoost` machine de modèles d'apprentissage. - Ajouté le `mysql` et `odbc` fonction de table et correspondant `MySQL` et `ODBC` moteurs de table pour accéder aux bases de données distantes. Cette fonctionnalité est en phase bêta. -- Ajout de la possibilité de passer un argument de type `AggregateFunction` pour l’ `groupArray` fonction d’agrégation (vous pouvez donc créer un tableau d’États d’une fonction d’agrégation). -- Suppression des restrictions sur diverses combinaisons de combinateurs de fonction d’agrégat. Par exemple, vous pouvez utiliser `avgForEachIf` ainsi que `avgIfForEach` les fonctions d’agrégation, qui ont des comportements différents. -- Le `-ForEach` fonction d’agrégation combinator est prolongée pour le cas des fonctions d’agrégation de plusieurs arguments. -- Ajout du support pour les fonctions d’agrégation de `Nullable` arguments même pour les cas où la fonction renvoie un non-`Nullable` résultat (ajouté avec la contribution de Silviu Caragea). Exemple: `groupArray`, `groupUniqArray`, `topK`. +- Ajout de la possibilité de passer un argument de type `AggregateFunction` pour l' `groupArray` fonction d'agrégation (vous pouvez donc créer un tableau d'États d'une fonction d'agrégation). +- Suppression des restrictions sur diverses combinaisons de combinateurs de fonction d'agrégat. Par exemple, vous pouvez utiliser `avgForEachIf` ainsi que `avgIfForEach` les fonctions d'agrégation, qui ont des comportements différents. +- Le `-ForEach` fonction d'agrégation combinator est prolongée pour le cas des fonctions d'agrégation de plusieurs arguments. +- Ajout du support pour les fonctions d'agrégation de `Nullable` arguments même pour les cas où la fonction renvoie un non-`Nullable` résultat (ajouté avec la contribution de Silviu Caragea). Exemple: `groupArray`, `groupUniqArray`, `topK`. - Ajouté le `max_client_network_bandwidth` pour `clickhouse-client` (Kirill Shvakov). - Les utilisateurs avec le `readonly = 2` setting are allowed to work with TEMPORARY tables (CREATE, DROP, INSERT…) (Kirill Shvakov). -- Ajout du support pour l’utilisation de plusieurs consommateurs avec le `Kafka` moteur. Options de configuration étendues pour `Kafka` (Marek Vavruša). +- Ajout du support pour l'utilisation de plusieurs consommateurs avec le `Kafka` moteur. Options de configuration étendues pour `Kafka` (Marek Vavruša). - Ajouté le `intExp3` et `intExp4` fonction. -- Ajouté le `sumKahan` fonction d’agrégation. +- Ajouté le `sumKahan` fonction d'agrégation. - Ajout des fonctions to\* Number \* OrNull, où \* Number \* est un type numérique. - Ajout du support pour `WITH` clauses pour un `INSERT SELECT` requête (auteur: zhang2014). - Ajout des paramètres de: `http_connection_timeout`, `http_send_timeout`, `http_receive_timeout`. En particulier, ces paramètres sont utilisés pour télécharger des parties de données pour la réplication. La modification de ces paramètres permet un basculement plus rapide si le réseau est surchargé. - Ajout du support pour `ALTER` pour les tables de type `Null` (Anastasiya Tsarkova). - Le `reinterpretAsString` la fonction est étendue pour tous les types de données stockés de manière contiguë en mémoire. -- Ajouté le `--silent` option pour le `clickhouse-local` outil. Il supprime les informations d’exécution de requête d’impression dans stderr. +- Ajouté le `--silent` option pour le `clickhouse-local` outil. Il supprime les informations d'exécution de requête d'impression dans stderr. - Ajout du support pour la lecture des valeurs de type `Date` à partir du texte dans un format où le mois et / ou le jour du mois est spécifié en utilisant un seul chiffre au lieu de deux chiffres (oiseau Amos). #### Optimisations Des Performances: {#performance-optimizations} -- Amélioration des performances des fonctions d’agrégation `min`, `max`, `any`, `anyLast`, `anyHeavy`, `argMin`, `argMax` à partir d’arguments de chaîne. +- Amélioration des performances des fonctions d'agrégation `min`, `max`, `any`, `anyLast`, `anyHeavy`, `argMin`, `argMax` à partir d'arguments de chaîne. - Amélioration des performances des fonctions `isInfinite`, `isFinite`, `isNaN`, `roundToExp2`. -- Amélioration des performances de l’analyse et du formatage `Date` et `DateTime` tapez les valeurs au format texte. -- Amélioration des performances et de la précision de l’analyse des nombres à virgule flottante. -- Abaissé l’utilisation de la mémoire pour `JOIN` dans le cas où les parties gauche et droite ont des colonnes avec des noms identiques qui ne sont pas contenus dans `USING` . -- Amélioration des performances des fonctions d’agrégation `varSamp`, `varPop`, `stddevSamp`, `stddevPop`, `covarSamp`, `covarPop`, `corr` en réduisant la stabilité de calcul. Les anciennes fonctions sont disponibles sous les noms de `varSampStable`, `varPopStable`, `stddevSampStable`, `stddevPopStable`, `covarSampStable`, `covarPopStable`, `corrStable`. +- Amélioration des performances de l'analyse et du formatage `Date` et `DateTime` tapez les valeurs au format texte. +- Amélioration des performances et de la précision de l'analyse des nombres à virgule flottante. +- Abaissé l'utilisation de la mémoire pour `JOIN` dans le cas où les parties gauche et droite ont des colonnes avec des noms identiques qui ne sont pas contenus dans `USING` . +- Amélioration des performances des fonctions d'agrégation `varSamp`, `varPop`, `stddevSamp`, `stddevPop`, `covarSamp`, `covarPop`, `corr` en réduisant la stabilité de calcul. Les anciennes fonctions sont disponibles sous les noms de `varSampStable`, `varPopStable`, `stddevSampStable`, `stddevPopStable`, `covarSampStable`, `covarPopStable`, `corrStable`. #### Corrections De Bugs: {#bug-fixes-28} -- Déduplication de données fixe après l’exécution d’un `DROP` ou `DETACH PARTITION` requête. Dans la version précédente, la suppression d’une partition et l’insertion à nouveau des mêmes données ne fonctionnaient pas car les blocs insérés étaient considérés comme des doublons. -- Correction d’un bug qui pourrait conduire à une interprétation incorrecte de la `WHERE` la clause de `CREATE MATERIALIZED VIEW` les requêtes avec `POPULATE` . -- Correction d’un bug dans l’utilisation de l’ `root_path` paramètre dans l’ `zookeeper_servers` configuration. +- Déduplication de données fixe après l'exécution d'un `DROP` ou `DETACH PARTITION` requête. Dans la version précédente, la suppression d'une partition et l'insertion à nouveau des mêmes données ne fonctionnaient pas car les blocs insérés étaient considérés comme des doublons. +- Correction d'un bug qui pourrait conduire à une interprétation incorrecte de la `WHERE` la clause de `CREATE MATERIALIZED VIEW` les requêtes avec `POPULATE` . +- Correction d'un bug dans l'utilisation de l' `root_path` paramètre dans l' `zookeeper_servers` configuration. - Correction des résultats inattendus du passage du `Date` argument `toStartOfDay` . -- Correction de l’ `addMonths` et `subtractMonths` fonctions et l’arithmétique pour `INTERVAL n MONTH` dans les cas où le résultat de l’année précédente. +- Correction de l' `addMonths` et `subtractMonths` fonctions et l'arithmétique pour `INTERVAL n MONTH` dans les cas où le résultat de l'année précédente. - Ajout du support manquant pour le `UUID` type de données pour `DISTINCT` , `JOIN` , et `uniq` fonctions agrégées et dictionnaires externes (Evgeniy Ivanov). Soutien pour `UUID` est encore incomplète. - Fixe `SummingMergeTree` comportement dans les cas où les lignes additionnées à zéro. - Diverses corrections pour le `Kafka` engine (Marek Vavruša). -- Correction d’un comportement incorrect de la `Join` moteur de table (oiseau Amos). -- Correction d’un comportement d’allocateur incorrect sous FreeBSD et OS X. +- Correction d'un comportement incorrect de la `Join` moteur de table (oiseau Amos). +- Correction d'un comportement d'allocateur incorrect sous FreeBSD et OS X. - Le `extractAll` fonction prend désormais en charge les correspondances vides. -- Correction d’une erreur qui bloquait l’utilisation de `libressl` plutôt `openssl` . -- Correction de l’ `CREATE TABLE AS SELECT` requête à partir de tables temporaires. -- Correction de la non-atomicité de la mise à jour de la file d’attente de réplication. Cela peut entraîner la désynchronisation des répliques jusqu’au redémarrage du serveur. -- Correction d’un débordement possible dans `gcd` , `lcm` et `modulo` (`%` de l’opérateur) (Mak Skorokhod). +- Correction d'une erreur qui bloquait l'utilisation de `libressl` plutôt `openssl` . +- Correction de l' `CREATE TABLE AS SELECT` requête à partir de tables temporaires. +- Correction de la non-atomicité de la mise à jour de la file d'attente de réplication. Cela peut entraîner la désynchronisation des répliques jusqu'au redémarrage du serveur. +- Correction d'un débordement possible dans `gcd` , `lcm` et `modulo` (`%` de l'opérateur) (Mak Skorokhod). - `-preprocessed` les fichiers sont maintenant créés après modification `umask` (`umask` peut être changé dans le fichier de configuration). -- Correction d’un bug dans la vérification des antécédents des pièces (`MergeTreePartChecker` ) lors de l’utilisation d’une coutume clé de partition. -- Correction de l’analyse des tuples (valeurs du `Tuple` type de données) dans des formats de texte. -- Amélioration des messages d’erreur sur les types incompatibles transmis à `multiIf` , `array` et quelques autres fonctions. -- Repensé de soutien pour `Nullable` type. Correction de bugs qui peuvent conduire à un plantage du serveur. Correction de presque tous les autres bugs liés à `NULL` support: conversions de type incorrectes dans INSERT SELECT, support insuffisant pour Nullable dans HAVING et PREWHERE, `join_use_nulls` mode, Nullable types comme arguments de `OR` l’opérateur, etc. -- Correction de divers bugs liés à la sémantique interne des types de données. Exemples: sommation inutile de `Enum` tapez les champs dans `SummingMergeTree` ; l’alignement de la `Enum` types de `Pretty` formats, etc. +- Correction d'un bug dans la vérification des antécédents des pièces (`MergeTreePartChecker` ) lors de l'utilisation d'une coutume clé de partition. +- Correction de l'analyse des tuples (valeurs du `Tuple` type de données) dans des formats de texte. +- Amélioration des messages d'erreur sur les types incompatibles transmis à `multiIf` , `array` et quelques autres fonctions. +- Repensé de soutien pour `Nullable` type. Correction de bugs qui peuvent conduire à un plantage du serveur. Correction de presque tous les autres bugs liés à `NULL` support: conversions de type incorrectes dans INSERT SELECT, support insuffisant pour Nullable dans HAVING et PREWHERE, `join_use_nulls` mode, Nullable types comme arguments de `OR` l'opérateur, etc. +- Correction de divers bugs liés à la sémantique interne des types de données. Exemples: sommation inutile de `Enum` tapez les champs dans `SummingMergeTree` ; l'alignement de la `Enum` types de `Pretty` formats, etc. - Contrôles plus stricts pour les combinaisons autorisées de colonnes composites. -- Correction du débordement lors de la spécification d’un très grand paramètre pour le `FixedString` type de données. -- Correction d’un bug dans l’ `topK` fonction d’agrégation dans un cas générique. -- Ajout de la vérification manquante pour l’égalité des tailles de tableau dans les arguments de n-ARY variantes de fonctions d’agrégation avec un `-Array` combinator. -- Correction d’un bug dans `--pager` pour `clickhouse-client` (auteur: ks1322). +- Correction du débordement lors de la spécification d'un très grand paramètre pour le `FixedString` type de données. +- Correction d'un bug dans l' `topK` fonction d'agrégation dans un cas générique. +- Ajout de la vérification manquante pour l'égalité des tailles de tableau dans les arguments de n-ARY variantes de fonctions d'agrégation avec un `-Array` combinator. +- Correction d'un bug dans `--pager` pour `clickhouse-client` (auteur: ks1322). - Fixe la précision de la `exp10` fonction. - Correction du comportement du `visitParamExtract` fonction pour une meilleure conformité avec la documentation. - Correction du crash lorsque des types de données incorrects sont spécifiés. - Correction du comportement de `DISTINCT` dans le cas lorsque toutes les colonnes sont des constantes. -- Correction du formatage de la requête dans le cas de l’utilisation du `tupleElement` fonction avec une expression constante complexe comme indice d’élément tuple. -- Correction d’un bug dans `Dictionary` tables pour `range_hashed` dictionnaire. -- Correction d’un bug qui conduit à des lignes excessives dans le résultat de `FULL` et `RIGHT JOIN` (Oiseau Amos). -- Correction d’un plantage du serveur lors de la création et de la suppression de fichiers temporaires `config.d` répertoires pendant le rechargement de la configuration. -- Correction de l’ `SYSTEM DROP DNS CACHE` requête: le cache a été vidé mais les adresses des nœuds de cluster n’ont pas été mises à jour. -- Correction du comportement de `MATERIALIZED VIEW` après l’exécution de `DETACH TABLE` for the table under the view (Marek Vavruša). +- Correction du formatage de la requête dans le cas de l'utilisation du `tupleElement` fonction avec une expression constante complexe comme indice d'élément tuple. +- Correction d'un bug dans `Dictionary` tables pour `range_hashed` dictionnaire. +- Correction d'un bug qui conduit à des lignes excessives dans le résultat de `FULL` et `RIGHT JOIN` (Oiseau Amos). +- Correction d'un plantage du serveur lors de la création et de la suppression de fichiers temporaires `config.d` répertoires pendant le rechargement de la configuration. +- Correction de l' `SYSTEM DROP DNS CACHE` requête: le cache a été vidé mais les adresses des nœuds de cluster n'ont pas été mises à jour. +- Correction du comportement de `MATERIALIZED VIEW` après l'exécution de `DETACH TABLE` for the table under the view (Marek Vavruša). -#### Construire Des améliorations: {#build-improvements-4} +#### Construire Des Améliorations: {#build-improvements-4} -- Le `pbuilder` l’outil est utilisé pour les versions. Le processus de construction est presque complètement indépendant de l’environnement hôte de construction. -- Une seule version est utilisée pour différentes versions du système d’exploitation. Les paquets et les binaires ont été rendus compatibles avec un large éventail de systèmes Linux. +- Le `pbuilder` l'outil est utilisé pour les versions. Le processus de construction est presque complètement indépendant de l'environnement hôte de construction. +- Une seule version est utilisée pour différentes versions du système d'exploitation. Les paquets et les binaires ont été rendus compatibles avec un large éventail de systèmes Linux. - Ajouté le `clickhouse-test` paquet. Il peut être utilisé pour exécuter des tests fonctionnels. -- L’archive source peut maintenant être publié dans le référentiel. Il peut être utilisé pour reproduire la construction sans utiliser GitHub. -- Ajout d’une intégration limitée avec Travis CI. En raison des limites de temps de construction dans Travis, seule la construction de débogage est testée et un sous-ensemble limité de tests est exécuté. +- L'archive source peut maintenant être publié dans le référentiel. Il peut être utilisé pour reproduire la construction sans utiliser GitHub. +- Ajout d'une intégration limitée avec Travis CI. En raison des limites de temps de construction dans Travis, seule la construction de débogage est testée et un sous-ensemble limité de tests est exécuté. - Ajout du support pour `Cap'n'Proto` dans la construction par défaut. - Modification du format des sources de documentation à partir de `Restricted Text` de `Markdown`. -- Ajout du support pour `systemd` (Vladimir Smirnov). Il est désactivé par défaut en raison d’une incompatibilité avec certaines images du système D’exploitation et peut être activé manuellement. +- Ajout du support pour `systemd` (Vladimir Smirnov). Il est désactivé par défaut en raison d'une incompatibilité avec certaines images du système D'exploitation et peut être activé manuellement. - Pour la génération de code dynamique, `clang` et `lld` sont intégrées dans le `clickhouse` binaire. Ils peuvent également être invoqués comme `clickhouse clang` et `clickhouse lld` . -- Suppression de l’utilisation des extensions GNU du code. Permis à l’ `-Wextra` option. Lors de la construction avec `clang` la valeur par défaut est `libc++` plutôt `libstdc++`. +- Suppression de l'utilisation des extensions GNU du code. Permis à l' `-Wextra` option. Lors de la construction avec `clang` la valeur par défaut est `libc++` plutôt `libstdc++`. - Extrait `clickhouse_parsers` et `clickhouse_common_io` les bibliothèques pour accélérer les constructions des différents outils. -#### Modifications Incompatibles En arrière: {#backward-incompatible-changes-11} +#### Modifications Incompatibles En Arrière: {#backward-incompatible-changes-11} -- Le format des marques dans `Log` tapez les tables qui contiennent `Nullable` les colonnes ont été modifiées d’une manière incompatible avec l’arrière. Si vous avez ces tables, vous devez les convertir en `TinyLog` tapez avant de démarrer la nouvelle version du serveur. Pour ce faire, remplacez `ENGINE = Log` avec `ENGINE = TinyLog` dans le correspondant `.sql` fichier dans le `metadata` répertoire. Si votre table n’a pas `Nullable` les colonnes ou si le type de votre table n’est pas `Log`, alors vous n’avez pas besoin de faire quoi que ce soit. +- Le format des marques dans `Log` tapez les tables qui contiennent `Nullable` les colonnes ont été modifiées d'une manière incompatible avec l'arrière. Si vous avez ces tables, vous devez les convertir en `TinyLog` tapez avant de démarrer la nouvelle version du serveur. Pour ce faire, remplacez `ENGINE = Log` avec `ENGINE = TinyLog` dans le correspondant `.sql` fichier dans le `metadata` répertoire. Si votre table n'a pas `Nullable` les colonnes ou si le type de votre table n'est pas `Log`, alors vous n'avez pas besoin de faire quoi que ce soit. - Enlevé le `experimental_allow_extended_storage_definition_syntax` paramètre. Maintenant cette fonctionnalité est activée par défaut. - Le `runningIncome` fonction a été renommée en `runningDifferenceStartingWithFirstvalue` pour éviter toute confusion. - Enlevé le `FROM ARRAY JOIN arr` syntaxe lorsque la jointure du tableau est spécifiée directement après FROM sans table (Amos Bird). - Enlevé le `BlockTabSeparated` format utilisé uniquement à des fins de démonstration. -- Modification du format d’État pour les fonctions d’agrégation `varSamp`, `varPop`, `stddevSamp`, `stddevPop`, `covarSamp`, `covarPop`, `corr`. Si vous avez stocké des états de ces fonctions d’agrégat dans des tables (en utilisant `AggregateFunction` type de données ou vues matérialisées avec les états correspondants), écrivez svp à clickhouse-feedback@yandex-team.com. -- Dans les versions précédentes du serveur, il y avait une fonctionnalité non documentée: si une fonction d’agrégation dépend de paramètres, vous pouvez toujours la spécifier sans paramètres dans le type de données AggregateFunction. Exemple: `AggregateFunction(quantiles, UInt64)` plutôt `AggregateFunction(quantiles(0.5, 0.9), UInt64)`. Cette fonctionnalité a été perdu. Bien qu’il ne soit pas documenté, nous prévoyons de le soutenir à nouveau dans les prochaines versions. -- Les types de données Enum ne peuvent pas être utilisés dans les fonctions d’agrégat min/max. Cette capacité sera rendu dans la prochaine version. +- Modification du format d'État pour les fonctions d'agrégation `varSamp`, `varPop`, `stddevSamp`, `stddevPop`, `covarSamp`, `covarPop`, `corr`. Si vous avez stocké des états de ces fonctions d'agrégat dans des tables (en utilisant `AggregateFunction` type de données ou vues matérialisées avec les états correspondants), écrivez svp à clickhouse-feedback@yandex-team.com. +- Dans les versions précédentes du serveur, il y avait une fonctionnalité non documentée: si une fonction d'agrégation dépend de paramètres, vous pouvez toujours la spécifier sans paramètres dans le type de données AggregateFunction. Exemple: `AggregateFunction(quantiles, UInt64)` plutôt `AggregateFunction(quantiles(0.5, 0.9), UInt64)`. Cette fonctionnalité a été perdu. Bien qu'il ne soit pas documenté, nous prévoyons de le soutenir à nouveau dans les prochaines versions. +- Les types de données Enum ne peuvent pas être utilisés dans les fonctions d'agrégat min/max. Cette capacité sera rendu dans la prochaine version. -#### Veuillez Noter Lors De La Mise à Niveau: {#please-note-when-upgrading} +#### Veuillez Noter Lors De La Mise À Niveau: {#please-note-when-upgrading} -- Lorsque vous effectuez une mise à jour continue sur un cluster, au moment où certaines répliques exécutent L’ancienne version de ClickHouse et d’autres la nouvelle version, la réplication est temporairement arrêtée et le message `unknown parameter 'shard'` apparaît dans le journal. La réplication se poursuivra après la mise à jour de toutes les répliques du cluster. -- Si différentes versions de ClickHouse sont en cours d’exécution sur les serveurs de cluster, il est possible que les requêtes distribuées utilisant les fonctions suivantes aient des résultats incorrects: `varSamp`, `varPop`, `stddevSamp`, `stddevPop`, `covarSamp`, `covarPop`, `corr`. Vous devez mettre à jour tous les nœuds de cluster. +- Lorsque vous effectuez une mise à jour continue sur un cluster, au moment où certaines répliques exécutent L'ancienne version de ClickHouse et d'autres la nouvelle version, la réplication est temporairement arrêtée et le message `unknown parameter 'shard'` apparaît dans le journal. La réplication se poursuivra après la mise à jour de toutes les répliques du cluster. +- Si différentes versions de ClickHouse sont en cours d'exécution sur les serveurs de cluster, il est possible que les requêtes distribuées utilisant les fonctions suivantes aient des résultats incorrects: `varSamp`, `varPop`, `stddevSamp`, `stddevPop`, `covarSamp`, `covarPop`, `corr`. Vous devez mettre à jour tous les nœuds de cluster. -## [Changelog pour 2017](https://github.com/ClickHouse/ClickHouse/blob/master/docs/en/changelog/2017.md) {#changelog-for-2017} +## [Changelog pour 2017](./2017.md#clickhouse-release-1-1-54327-2017-12-21) {#changelog-for-2017} diff --git a/docs/fr/whats-new/changelog/2019.md b/docs/fr/whats-new/changelog/2019.md index e7da17cc754..1e320d28084 100644 --- a/docs/fr/whats-new/changelog/2019.md +++ b/docs/fr/whats-new/changelog/2019.md @@ -1,48 +1,48 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 77 toc_title: '2019' --- -## Clickhouse Version V19. 17 {#clickhouse-release-v19-17} +## Clickhouse Version 19.17 {#clickhouse-release-v19-17} -### Clickhouse Version V19. 17. 6. 36, 2019-12-27 {#clickhouse-release-v19-17-6-36-2019-12-27} +### Clickhouse Version 19.17.6.36, 2019-12-27 {#clickhouse-release-v19-17-6-36-2019-12-27} #### Bug Fix {#bug-fix} -- Dépassement de tampon potentiel fixe en décompression. Un utilisateur malveillant peut transmettre des données compressées fabriquées qui pourraient provoquer une lecture après le tampon. Ce problème a été trouvé par Eldar Zaitov de l’équipe de sécurité de L’information Yandex. [\#8404](https://github.com/ClickHouse/ClickHouse/pull/8404) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction possible plantage du serveur (`std::terminate`) lorsque le serveur ne peut pas envoyer ou écrire des données au format JSON ou XML avec des valeurs de type string data (qui nécessitent une validation UTF-8) ou lors de la compression des données de résultat avec l’algorithme Brotli ou dans certains autres cas rares. [\#8384](https://github.com/ClickHouse/ClickHouse/pull/8384) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Dictionnaires fixes avec la source d’un clickhouse `VIEW`, maintenant la lecture de tels dictionnaires ne provoque pas l’erreur `There is no query`. [\#8351](https://github.com/ClickHouse/ClickHouse/pull/8351) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Dépassement de tampon potentiel fixe en décompression. Un utilisateur malveillant peut transmettre des données compressées fabriquées qui pourraient provoquer une lecture après le tampon. Ce problème a été trouvé par Eldar Zaitov de l'équipe de sécurité de L'information Yandex. [\#8404](https://github.com/ClickHouse/ClickHouse/pull/8404) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction possible plantage du serveur (`std::terminate`) lorsque le serveur ne peut pas envoyer ou écrire des données au format JSON ou XML avec des valeurs de type string data (qui nécessitent une validation UTF-8) ou lors de la compression des données de résultat avec l'algorithme Brotli ou dans certains autres cas rares. [\#8384](https://github.com/ClickHouse/ClickHouse/pull/8384) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Dictionnaires fixes avec la source d'un clickhouse `VIEW`, maintenant la lecture de tels dictionnaires ne provoque pas l'erreur `There is no query`. [\#8351](https://github.com/ClickHouse/ClickHouse/pull/8351) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) - Fixe vérifier si un hôte client est autorisé par host\_regexp spécifié dans les utilisateurs.XML. [\#8241](https://github.com/ClickHouse/ClickHouse/pull/8241), [\#8342](https://github.com/ClickHouse/ClickHouse/pull/8342) ([Vitaly Baranov](https://github.com/vitlibar)) -- `RENAME TABLE` pour une table distribuée renomme maintenant le dossier contenant les données insérées avant d’envoyer aux fragments. Cela résout un problème avec les renommages successifs `tableA->tableB`, `tableC->tableA`. [\#8306](https://github.com/ClickHouse/ClickHouse/pull/8306) ([tavplubix](https://github.com/tavplubix)) +- `RENAME TABLE` pour une table distribuée renomme maintenant le dossier contenant les données insérées avant d'envoyer aux fragments. Cela résout un problème avec les renommages successifs `tableA->tableB`, `tableC->tableA`. [\#8306](https://github.com/ClickHouse/ClickHouse/pull/8306) ([tavplubix](https://github.com/tavplubix)) - `range_hashed` les dictionnaires externes créés par des requêtes DDL autorisent désormais des plages de types numériques arbitraires. [\#8275](https://github.com/ClickHouse/ClickHouse/pull/8275) ([alésapine](https://github.com/alesapin)) - Fixe `INSERT INTO table SELECT ... FROM mysql(...)` table de fonction. [\#8234](https://github.com/ClickHouse/ClickHouse/pull/8234) ([tavplubix](https://github.com/tavplubix)) -- Fixe erreur de segmentation dans `INSERT INTO TABLE FUNCTION file()` lors de l’insertion dans un fichier qui n’existe pas. Maintenant, dans ce cas, le fichier sera créé et insérez seraient traités. [\#8177](https://github.com/ClickHouse/ClickHouse/pull/8177) ([Olga Khvostikova](https://github.com/stavrolia)) -- Correction d’une erreur bitmapAnd lors de l’intersection d’un bitmap agrégé et d’un bitmap scalaire. [\#8082](https://github.com/ClickHouse/ClickHouse/pull/8082) ([Yue Huang](https://github.com/moon03432)) +- Fixe erreur de segmentation dans `INSERT INTO TABLE FUNCTION file()` lors de l'insertion dans un fichier qui n'existe pas. Maintenant, dans ce cas, le fichier sera créé et insérez seraient traités. [\#8177](https://github.com/ClickHouse/ClickHouse/pull/8177) ([Olga Khvostikova](https://github.com/stavrolia)) +- Correction d'une erreur bitmapAnd lors de l'intersection d'un bitmap agrégé et d'un bitmap scalaire. [\#8082](https://github.com/ClickHouse/ClickHouse/pull/8082) ([Yue Huang](https://github.com/moon03432)) - Correction de segfault quand `EXISTS` la requête a été utilisé sans `TABLE` ou `DICTIONARY` qualificatif, tout comme `EXISTS t`. [\#8213](https://github.com/ClickHouse/ClickHouse/pull/8213) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Type de retour fixe pour les fonctions `rand` et `randConstant` en cas d’argument nullable. Maintenant renvoient toujours `UInt32` et jamais `Nullable(UInt32)`. [\#8204](https://github.com/ClickHouse/ClickHouse/pull/8204) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Fixe `DROP DICTIONARY IF EXISTS db.dict` maintenant il ne lance pas d’exception si `db` n’existe pas. [\#8185](https://github.com/ClickHouse/ClickHouse/pull/8185) ([Vitaly Baranov](https://github.com/vitlibar)) -- Si une table n’a pas été complètement abandonnée en raison d’un plantage du serveur, le serveur essaiera de la restaurer et de la charger [\#8176](https://github.com/ClickHouse/ClickHouse/pull/8176) ([tavplubix](https://github.com/tavplubix)) -- Correction d’une requête de comptage trivial pour une table distribuée s’il y a plus de deux tables locales de fragments. [\#8164](https://github.com/ClickHouse/ClickHouse/pull/8164) ([小路](https://github.com/nicelulu)) -- Correction d’un bug qui conduisait à une course de données dans DB:: BlockStreamProfileInfo:: calculateRowsBeforeLimit() [\#8143](https://github.com/ClickHouse/ClickHouse/pull/8143) ([Alexander Kazakov](https://github.com/Akazz)) -- Fixe `ALTER table MOVE part` exécuté immédiatement après la fusion de la partie spécifiée, ce qui pourrait provoquer le déplacement d’une partie la partie fusionné. Maintenant, il déplace correctement la partie spécifiée. [\#8104](https://github.com/ClickHouse/ClickHouse/pull/8104) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Les Expressions pour les dictionnaires peuvent maintenant être spécifiées en tant que chaînes. Ceci est utile pour le calcul des attributs lors de l’extraction de données à partir de sources non-ClickHouse, car il permet d’utiliser une syntaxe non-ClickHouse pour ces expressions. [\#8098](https://github.com/ClickHouse/ClickHouse/pull/8098) ([alésapine](https://github.com/alesapin)) -- Correction d’une course très rare dans `clickhouse-copier` en raison d’un débordement dans ZXid. [\#8088](https://github.com/ClickHouse/ClickHouse/pull/8088) ([Ding Xiang Fei](https://github.com/dingxiangfei2009)) +- Type de retour fixe pour les fonctions `rand` et `randConstant` en cas d'argument nullable. Maintenant renvoient toujours `UInt32` et jamais `Nullable(UInt32)`. [\#8204](https://github.com/ClickHouse/ClickHouse/pull/8204) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Fixe `DROP DICTIONARY IF EXISTS db.dict` maintenant il ne lance pas d'exception si `db` n'existe pas. [\#8185](https://github.com/ClickHouse/ClickHouse/pull/8185) ([Vitaly Baranov](https://github.com/vitlibar)) +- Si une table n'a pas été complètement abandonnée en raison d'un plantage du serveur, le serveur essaiera de la restaurer et de la charger [\#8176](https://github.com/ClickHouse/ClickHouse/pull/8176) ([tavplubix](https://github.com/tavplubix)) +- Correction d'une requête de comptage trivial pour une table distribuée s'il y a plus de deux tables locales de fragments. [\#8164](https://github.com/ClickHouse/ClickHouse/pull/8164) ([小路](https://github.com/nicelulu)) +- Correction d'un bug qui conduisait à une course de données dans DB:: BlockStreamProfileInfo:: calculateRowsBeforeLimit() [\#8143](https://github.com/ClickHouse/ClickHouse/pull/8143) ([Alexander Kazakov](https://github.com/Akazz)) +- Fixe `ALTER table MOVE part` exécuté immédiatement après la fusion de la partie spécifiée, ce qui pourrait provoquer le déplacement d'une partie la partie fusionné. Maintenant, il déplace correctement la partie spécifiée. [\#8104](https://github.com/ClickHouse/ClickHouse/pull/8104) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Les Expressions pour les dictionnaires peuvent maintenant être spécifiées en tant que chaînes. Ceci est utile pour le calcul des attributs lors de l'extraction de données à partir de sources non-ClickHouse, car il permet d'utiliser une syntaxe non-ClickHouse pour ces expressions. [\#8098](https://github.com/ClickHouse/ClickHouse/pull/8098) ([alésapine](https://github.com/alesapin)) +- Correction d'une course très rare dans `clickhouse-copier` en raison d'un débordement dans ZXid. [\#8088](https://github.com/ClickHouse/ClickHouse/pull/8088) ([Ding Xiang Fei](https://github.com/dingxiangfei2009)) - Correction du bug quand après la requête a échoué (en raison de “Too many simultaneous queries” par exemple) il ne lirait pas les informations des tables externes, et le la requête suivante interpréterait cette information comme le début de la requête suivante provoquant une erreur comme `Unknown packet from client`. [\#8084](https://github.com/ClickHouse/ClickHouse/pull/8084) ([Azat Khuzhin](https://github.com/azat)) - Éviter la déréférence nulle après “Unknown packet X from server” [\#8071](https://github.com/ClickHouse/ClickHouse/pull/8071) ([Azat Khuzhin](https://github.com/azat)) -- Restaurer la prise en charge de tous les paramètres régionaux ICU, ajouter la possibilité d’appliquer des collations pour les expressions constantes et ajouter le nom de la langue au système.tableau de collations. [\#8051](https://github.com/ClickHouse/ClickHouse/pull/8051) ([alésapine](https://github.com/alesapin)) +- Restaurer la prise en charge de tous les paramètres régionaux ICU, ajouter la possibilité d'appliquer des collations pour les expressions constantes et ajouter le nom de la langue au système.tableau de collations. [\#8051](https://github.com/ClickHouse/ClickHouse/pull/8051) ([alésapine](https://github.com/alesapin)) - Nombre de flux pour lire à partir `StorageFile` et `StorageHDFS` est maintenant limitée, pour éviter de dépasser la limite de mémoire. [\#7981](https://github.com/ClickHouse/ClickHouse/pull/7981) ([alésapine](https://github.com/alesapin)) - Fixe `CHECK TABLE` requête pour `*MergeTree` les tables sans clé. [\#7979](https://github.com/ClickHouse/ClickHouse/pull/7979) ([alésapine](https://github.com/alesapin)) -- Suppression du numéro de mutation d’un nom de pièce au cas où il n’y aurait pas de mutations. Cette suppression a amélioré la compatibilité avec les anciennes versions. [\#8250](https://github.com/ClickHouse/ClickHouse/pull/8250) ([alésapine](https://github.com/alesapin)) +- Suppression du numéro de mutation d'un nom de pièce au cas où il n'y aurait pas de mutations. Cette suppression a amélioré la compatibilité avec les anciennes versions. [\#8250](https://github.com/ClickHouse/ClickHouse/pull/8250) ([alésapine](https://github.com/alesapin)) - Correction du bug que les mutations sont ignorées pour certaines parties attachées en raison de leur data\_version sont plus grandes que la version de mutation de table. [\#7812](https://github.com/ClickHouse/ClickHouse/pull/7812) ([Zhichang Yu](https://github.com/yuzhichang)) - Autoriser le démarrage du serveur avec des copies redondantes des pièces après les avoir déplacées vers un autre périphérique. [\#7810](https://github.com/ClickHouse/ClickHouse/pull/7810) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Correction de l’erreur “Sizes of columns doesn’t match” qui pourraient apparaître lors de l’utilisation de fonction d’agrégation des colonnes. [\#7790](https://github.com/ClickHouse/ClickHouse/pull/7790) ([Boris Granveaud](https://github.com/bgranvea)) -- Maintenant, une exception sera levée en cas d’utilisation avec des liens à côté de LIMIT BY. Et maintenant, il est possible d’utiliser TOP avec LIMIT BY. [\#7637](https://github.com/ClickHouse/ClickHouse/pull/7637) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- Correction du rechargement du dictionnaire s’il a `invalidate_query`, qui a arrêté les mises à jour et une exception sur les tentatives de mise à jour précédentes. [\#8029](https://github.com/ClickHouse/ClickHouse/pull/8029) ([alésapine](https://github.com/alesapin)) +- Correction de l'erreur “Sizes of columns doesn’t match” qui pourraient apparaître lors de l'utilisation de fonction d'agrégation des colonnes. [\#7790](https://github.com/ClickHouse/ClickHouse/pull/7790) ([Boris Granveaud](https://github.com/bgranvea)) +- Maintenant, une exception sera levée en cas d'utilisation avec des liens à côté de LIMIT BY. Et maintenant, il est possible d'utiliser TOP avec LIMIT BY. [\#7637](https://github.com/ClickHouse/ClickHouse/pull/7637) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +- Correction du rechargement du dictionnaire s'il a `invalidate_query`, qui a arrêté les mises à jour et une exception sur les tentatives de mise à jour précédentes. [\#8029](https://github.com/ClickHouse/ClickHouse/pull/8029) ([alésapine](https://github.com/alesapin)) -### Clickhouse Version V19. 17. 4. 11, 2019-11-22 {#clickhouse-release-v19-17-4-11-2019-11-22} +### Clickhouse Version 19.17.4.11, 2019-11-22 {#clickhouse-release-v19-17-4-11-2019-11-22} #### Modification Incompatible En Arrière {#backward-incompatible-change} @@ -51,44 +51,44 @@ toc_title: '2019' #### Nouveauté {#new-feature} - Ajoutez la possibilité de créer des dictionnaires avec des requêtes DDL. [\#7360](https://github.com/ClickHouse/ClickHouse/pull/7360) ([alésapine](https://github.com/alesapin)) -- Faire `bloom_filter` type de support d’index `LowCardinality` et `Nullable` [\#7363](https://github.com/ClickHouse/ClickHouse/issues/7363) [\#7561](https://github.com/ClickHouse/ClickHouse/pull/7561) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Faire `bloom_filter` type de support d'index `LowCardinality` et `Nullable` [\#7363](https://github.com/ClickHouse/ClickHouse/issues/7363) [\#7561](https://github.com/ClickHouse/ClickHouse/pull/7561) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) - Ajouter une fonction `isValidJSON` pour vérifier que la chaîne est un json valide. [\#5910](https://github.com/ClickHouse/ClickHouse/issues/5910) [\#7293](https://github.com/ClickHouse/ClickHouse/pull/7293) ([Vdimir](https://github.com/Vdimir)) - Mettre `arrayCompact` fonction [\#7328](https://github.com/ClickHouse/ClickHouse/pull/7328) ([Mémo](https://github.com/Joeywzr)) - Créé fonction `hex` pour les nombres Décimaux. Il fonctionne comme `hex(reinterpretAsString())`, mais ne supprime pas les derniers octets zéro. [\#7355](https://github.com/ClickHouse/ClickHouse/pull/7355) ([Mikhail Korotov](https://github.com/millb)) -- Ajouter `arrayFill` et `arrayReverseFill` fonctions, qui remplacent les éléments par d’autres éléments en avant / arrière d’eux dans le tableau. [\#7380](https://github.com/ClickHouse/ClickHouse/pull/7380) ([hcz](https://github.com/hczhcz)) +- Ajouter `arrayFill` et `arrayReverseFill` fonctions, qui remplacent les éléments par d'autres éléments en avant / arrière d'eux dans le tableau. [\#7380](https://github.com/ClickHouse/ClickHouse/pull/7380) ([hcz](https://github.com/hczhcz)) - Ajouter `CRC32IEEE()`/`CRC64()` soutien [\#7480](https://github.com/ClickHouse/ClickHouse/pull/7480) ([Azat Khuzhin](https://github.com/azat)) - Mettre `char` fonction similaire à celle dans [mysql](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_char) [\#7486](https://github.com/ClickHouse/ClickHouse/pull/7486) ([sundyli](https://github.com/sundy-li)) -- Ajouter `bitmapTransform` fonction. Il transforme un tableau de valeurs d’une image bitmap dans un autre tableau de valeurs, le résultat est un nouveau bitmap [\#7598](https://github.com/ClickHouse/ClickHouse/pull/7598) ([Zhichang Yu](https://github.com/yuzhichang)) +- Ajouter `bitmapTransform` fonction. Il transforme un tableau de valeurs d'une image bitmap dans un autre tableau de valeurs, le résultat est un nouveau bitmap [\#7598](https://github.com/ClickHouse/ClickHouse/pull/7598) ([Zhichang Yu](https://github.com/yuzhichang)) - Mettre `javaHashUTF16LE()` fonction [\#7651](https://github.com/ClickHouse/ClickHouse/pull/7651) ([achimbab](https://github.com/achimbab)) - Ajouter `_shard_num` colonne virtuelle pour le moteur distribué [\#7624](https://github.com/ClickHouse/ClickHouse/pull/7624) ([Azat Khuzhin](https://github.com/azat)) #### Caractéristique Expérimentale {#experimental-feature} -- Prise en charge des processeurs (nouveau pipeline d’exécution de requêtes) dans `MergeTree`. [\#7181](https://github.com/ClickHouse/ClickHouse/pull/7181) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Prise en charge des processeurs (nouveau pipeline d'exécution de requêtes) dans `MergeTree`. [\#7181](https://github.com/ClickHouse/ClickHouse/pull/7181) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) #### Bug Fix {#bug-fix-1} -- Correction d’une analyse float incorrecte Dans `Values` [\#7817](https://github.com/ClickHouse/ClickHouse/issues/7817) [\#7870](https://github.com/ClickHouse/ClickHouse/pull/7870) ([tavplubix](https://github.com/tavplubix)) -- Correction d’un blocage rare qui peut se produire lorsque trace\_log est activé. [\#7838](https://github.com/ClickHouse/ClickHouse/pull/7838) ([filimonov](https://github.com/filimonov)) +- Correction d'une analyse float incorrecte Dans `Values` [\#7817](https://github.com/ClickHouse/ClickHouse/issues/7817) [\#7870](https://github.com/ClickHouse/ClickHouse/pull/7870) ([tavplubix](https://github.com/tavplubix)) +- Correction d'un blocage rare qui peut se produire lorsque trace\_log est activé. [\#7838](https://github.com/ClickHouse/ClickHouse/pull/7838) ([filimonov](https://github.com/filimonov)) - Empêcher la duplication des messages lors de la production de la table Kafka a tout MVS en sélectionnant [\#7265](https://github.com/ClickHouse/ClickHouse/pull/7265) ([Ivan](https://github.com/abyss7)) - Soutien pour `Array(LowCardinality(Nullable(String)))` dans `IN`. Résoudre [\#7364](https://github.com/ClickHouse/ClickHouse/issues/7364) [\#7366](https://github.com/ClickHouse/ClickHouse/pull/7366) ([achimbab](https://github.com/achimbab)) - Ajouter le traitement de `SQL_TINYINT` et `SQL_BIGINT` et correction de la gestion des `SQL_FLOAT` types de sources de données dans ODBC Bridge. [\#7491](https://github.com/ClickHouse/ClickHouse/pull/7491) ([Denis Glazachev](https://github.com/traceon)) -- Correction de l’agrégation (`avg` et quantiles) sur des colonnes décimales vides [\#7431](https://github.com/ClickHouse/ClickHouse/pull/7431) ([Andrey Konyaev](https://github.com/akonyaev90)) +- Correction de l'agrégation (`avg` et quantiles) sur des colonnes décimales vides [\#7431](https://github.com/ClickHouse/ClickHouse/pull/7431) ([Andrey Konyaev](https://github.com/akonyaev90)) - Fixer `INSERT` en Distribué avec `MATERIALIZED` colonne [\#7377](https://github.com/ClickHouse/ClickHouse/pull/7377) ([Azat Khuzhin](https://github.com/azat)) - Faire `MOVE PARTITION` fonctionne si certaines parties de la partition sont déjà sur le disque ou le volume de destination [\#7434](https://github.com/ClickHouse/ClickHouse/pull/7434) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Correction d’un bug avec hardlinks ne pas être créé lors de mutations dans `ReplicatedMergeTree` dans des configurations multi-disques. [\#7558](https://github.com/ClickHouse/ClickHouse/pull/7558) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Correction d’un bug avec une mutation sur un MergeTree lorsque la partie entière reste inchangée et le meilleur espace est trouvé sur un autre disque [\#7602](https://github.com/ClickHouse/ClickHouse/pull/7602) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Correction d’un bug avec `keep_free_space_ratio` ne pas être lu à partir de la configuration des disques [\#7645](https://github.com/ClickHouse/ClickHouse/pull/7645) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Correction d’un bug avec la table ne contient que `Tuple` colonnes ou colonnes avec des chemins complexes. Fixer [7541](https://github.com/ClickHouse/ClickHouse/issues/7541). [\#7545](https://github.com/ClickHouse/ClickHouse/pull/7545) ([alésapine](https://github.com/alesapin)) +- Correction d'un bug avec hardlinks ne pas être créé lors de mutations dans `ReplicatedMergeTree` dans des configurations multi-disques. [\#7558](https://github.com/ClickHouse/ClickHouse/pull/7558) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Correction d'un bug avec une mutation sur un MergeTree lorsque la partie entière reste inchangée et le meilleur espace est trouvé sur un autre disque [\#7602](https://github.com/ClickHouse/ClickHouse/pull/7602) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Correction d'un bug avec `keep_free_space_ratio` ne pas être lu à partir de la configuration des disques [\#7645](https://github.com/ClickHouse/ClickHouse/pull/7645) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Correction d'un bug avec la table ne contient que `Tuple` colonnes ou colonnes avec des chemins complexes. Fixer [7541](https://github.com/ClickHouse/ClickHouse/issues/7541). [\#7545](https://github.com/ClickHouse/ClickHouse/pull/7545) ([alésapine](https://github.com/alesapin)) - Ne pas tenir compte de la mémoire pour le moteur tampon dans la limite max\_memory\_usage [\#7552](https://github.com/ClickHouse/ClickHouse/pull/7552) ([Azat Khuzhin](https://github.com/azat)) -- Correction de l’utilisation finale de la marque dans `MergeTree` tableaux commandés par `tuple()`. Dans de rares cas cela pourrait conduire à `Can't adjust last granule` erreur lors de la sélection. [\#7639](https://github.com/ClickHouse/ClickHouse/pull/7639) ([Anton Popov](https://github.com/CurtizJ)) -- Correction d’un bug dans les mutations qui ont un prédicat avec des actions qui nécessitent un contexte (par exemple des fonctions pour json), ce qui peut entraîner des plantages ou des exceptions étranges. [\#7664](https://github.com/ClickHouse/ClickHouse/pull/7664) ([alésapine](https://github.com/alesapin)) -- Correction de l’inadéquation des noms de base de données et de table s’échappant dans `data/` et `shadow/` annuaire [\#7575](https://github.com/ClickHouse/ClickHouse/pull/7575) ([Alexander Burmak](https://github.com/Alex-Burmak)) -- Support duplicated keys in RIGHT\|FULL JOINs, e.g. `ON t.x = u.x AND t.x = u.y`. Correction d’un crash dans ce cas. [\#7586](https://github.com/ClickHouse/ClickHouse/pull/7586) ([Artem Zuikov](https://github.com/4ertus2)) -- Fixer `Not found column in block` lors de la jointure sur l’expression avec jointure droite ou complète. [\#7641](https://github.com/ClickHouse/ClickHouse/pull/7641) ([Artem Zuikov](https://github.com/4ertus2)) +- Correction de l'utilisation finale de la marque dans `MergeTree` tableaux commandés par `tuple()`. Dans de rares cas cela pourrait conduire à `Can't adjust last granule` erreur lors de la sélection. [\#7639](https://github.com/ClickHouse/ClickHouse/pull/7639) ([Anton Popov](https://github.com/CurtizJ)) +- Correction d'un bug dans les mutations qui ont un prédicat avec des actions qui nécessitent un contexte (par exemple des fonctions pour json), ce qui peut entraîner des plantages ou des exceptions étranges. [\#7664](https://github.com/ClickHouse/ClickHouse/pull/7664) ([alésapine](https://github.com/alesapin)) +- Correction de l'inadéquation des noms de base de données et de table s'échappant dans `data/` et `shadow/` annuaire [\#7575](https://github.com/ClickHouse/ClickHouse/pull/7575) ([Alexander Burmak](https://github.com/Alex-Burmak)) +- Support duplicated keys in RIGHT\|FULL JOINs, e.g. `ON t.x = u.x AND t.x = u.y`. Correction d'un crash dans ce cas. [\#7586](https://github.com/ClickHouse/ClickHouse/pull/7586) ([Artem Zuikov](https://github.com/4ertus2)) +- Fixer `Not found column in block` lors de la jointure sur l'expression avec jointure droite ou complète. [\#7641](https://github.com/ClickHouse/ClickHouse/pull/7641) ([Artem Zuikov](https://github.com/4ertus2)) - Une tentative de plus pour corriger la boucle infinie dans `PrettySpace` format [\#7591](https://github.com/ClickHouse/ClickHouse/pull/7591) ([Olga Khvostikova](https://github.com/stavrolia)) -- Correction d’un bug dans `concat` fonction lorsque tous les arguments étaient `FixedString` de la même taille. [\#7635](https://github.com/ClickHouse/ClickHouse/pull/7635) ([alésapine](https://github.com/alesapin)) -- Correction d’une exception en cas d’utilisation de 1 argument lors de la définition des stockages S3, URL et HDFS. [\#7618](https://github.com/ClickHouse/ClickHouse/pull/7618) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Correction d'un bug dans `concat` fonction lorsque tous les arguments étaient `FixedString` de la même taille. [\#7635](https://github.com/ClickHouse/ClickHouse/pull/7635) ([alésapine](https://github.com/alesapin)) +- Correction d'une exception en cas d'utilisation de 1 argument lors de la définition des stockages S3, URL et HDFS. [\#7618](https://github.com/ClickHouse/ClickHouse/pull/7618) ([Vladimir Chebotarev](https://github.com/excitoon)) - Correction de la portée de InterpreterSelectQuery pour les vues Avec requête [\#7601](https://github.com/ClickHouse/ClickHouse/pull/7601) ([Azat Khuzhin](https://github.com/azat)) #### Amélioration {#improvement} @@ -98,70 +98,70 @@ toc_title: '2019' - Lancez une exception si nous ne pouvons pas détecter la table pour le nom de la colonne dans la requête. [\#7358](https://github.com/ClickHouse/ClickHouse/pull/7358) ([Artem Zuikov](https://github.com/4ertus2)) - Ajouter `merge_max_block_size` réglage de `MergeTreeSettings` [\#7412](https://github.com/ClickHouse/ClickHouse/pull/7412) ([Artem Zuikov](https://github.com/4ertus2)) - Les requêtes avec `HAVING` et sans `GROUP BY` supposons groupe par constante. Si, `SELECT 1 HAVING 1` maintenant retourne un résultat. [\#7496](https://github.com/ClickHouse/ClickHouse/pull/7496) ([Amos Oiseau](https://github.com/amosbird)) -- Soutien à l’analyse `(X,)` comme tuple similaire à python. [\#7501](https://github.com/ClickHouse/ClickHouse/pull/7501), [\#7562](https://github.com/ClickHouse/ClickHouse/pull/7562) ([Amos Oiseau](https://github.com/amosbird)) +- Soutien à l'analyse `(X,)` comme tuple similaire à python. [\#7501](https://github.com/ClickHouse/ClickHouse/pull/7501), [\#7562](https://github.com/ClickHouse/ClickHouse/pull/7562) ([Amos Oiseau](https://github.com/amosbird)) - Faire `range` les comportements de fonction ressemblent presque à ceux de pythonic. [\#7518](https://github.com/ClickHouse/ClickHouse/pull/7518) ([sundyli](https://github.com/sundy-li)) - Ajouter `constraints` les colonnes de la table `system.settings` [\#7553](https://github.com/ClickHouse/ClickHouse/pull/7553) ([Vitaly Baranov](https://github.com/vitlibar)) -- Meilleur format Null pour le gestionnaire tcp, de sorte qu’il est possible d’utiliser `select ignore() from table format Null` pour perf mesure via clickhouse-client [\#7606](https://github.com/ClickHouse/ClickHouse/pull/7606) ([Amos Oiseau](https://github.com/amosbird)) +- Meilleur format Null pour le gestionnaire tcp, de sorte qu'il est possible d'utiliser `select ignore() from table format Null` pour perf mesure via clickhouse-client [\#7606](https://github.com/ClickHouse/ClickHouse/pull/7606) ([Amos Oiseau](https://github.com/amosbird)) - Les requêtes comme `CREATE TABLE ... AS (SELECT (1, 2))` sont analysés correctement [\#7542](https://github.com/ClickHouse/ClickHouse/pull/7542) ([hcz](https://github.com/hczhcz)) #### Amélioration Des Performances {#performance-improvement} -- Les performances de l’agrégation sur les clés de chaîne courte sont améliorées. [\#6243](https://github.com/ClickHouse/ClickHouse/pull/6243) ([Alexander Kuzmenkov](https://github.com/akuzm), [Amos Oiseau](https://github.com/amosbird)) -- Exécutez une autre passe d’analyse de syntaxe / expression pour obtenir des optimisations potentielles après que les prédicats constants sont pliés. [\#7497](https://github.com/ClickHouse/ClickHouse/pull/7497) ([Amos Oiseau](https://github.com/amosbird)) +- Les performances de l'agrégation sur les clés de chaîne courte sont améliorées. [\#6243](https://github.com/ClickHouse/ClickHouse/pull/6243) ([Alexander Kuzmenkov](https://github.com/akuzm), [Amos Oiseau](https://github.com/amosbird)) +- Exécutez une autre passe d'analyse de syntaxe / expression pour obtenir des optimisations potentielles après que les prédicats constants sont pliés. [\#7497](https://github.com/ClickHouse/ClickHouse/pull/7497) ([Amos Oiseau](https://github.com/amosbird)) - Utilisez les méta informations de stockage pour évaluer trivial `SELECT count() FROM table;` [\#7510](https://github.com/ClickHouse/ClickHouse/pull/7510) ([Amos Oiseau](https://github.com/amosbird), [alexeï-milovidov](https://github.com/alexey-milovidov)) - Vectoriser le traitement `arrayReduce` semblable à Agrégateur `addBatch`. [\#7608](https://github.com/ClickHouse/ClickHouse/pull/7608) ([Amos Oiseau](https://github.com/amosbird)) - Améliorations mineures des performances de `Kafka` consommation [\#7475](https://github.com/ClickHouse/ClickHouse/pull/7475) ([Ivan](https://github.com/abyss7)) -#### Construction / Test / Amélioration De L’Emballage {#buildtestingpackaging-improvement} +#### Construction / Test / Amélioration De L'Emballage {#buildtestingpackaging-improvement} -- Ajouter la prise en charge de la compilation croisée à L’architecture du processeur AARCH64. Refactoriser le code emballeur script. [\#7370](https://github.com/ClickHouse/ClickHouse/pull/7370) [\#7539](https://github.com/ClickHouse/ClickHouse/pull/7539) ([Ivan](https://github.com/abyss7)) -- Décompressez les chaînes d’outils darwin-x86\_64 et linux-aarch64 dans le volume Docker monté lors de la construction de paquets [\#7534](https://github.com/ClickHouse/ClickHouse/pull/7534) ([Ivan](https://github.com/abyss7)) -- Mise à jour de L’Image Docker pour le Packager binaire [\#7474](https://github.com/ClickHouse/ClickHouse/pull/7474) ([Ivan](https://github.com/abyss7)) +- Ajouter la prise en charge de la compilation croisée à L'architecture du processeur AARCH64. Refactoriser le code emballeur script. [\#7370](https://github.com/ClickHouse/ClickHouse/pull/7370) [\#7539](https://github.com/ClickHouse/ClickHouse/pull/7539) ([Ivan](https://github.com/abyss7)) +- Décompressez les chaînes d'outils darwin-x86\_64 et linux-aarch64 dans le volume Docker monté lors de la construction de paquets [\#7534](https://github.com/ClickHouse/ClickHouse/pull/7534) ([Ivan](https://github.com/abyss7)) +- Mise à jour de L'Image Docker pour le Packager binaire [\#7474](https://github.com/ClickHouse/ClickHouse/pull/7474) ([Ivan](https://github.com/abyss7)) - Correction des erreurs de compilation sur macOS Catalina [\#7585](https://github.com/ClickHouse/ClickHouse/pull/7585) ([Ernest Poletaev](https://github.com/ernestp)) -- Certains refactoring dans la logique d’analyse de requête: diviser la classe complexe en plusieurs classes simples. [\#7454](https://github.com/ClickHouse/ClickHouse/pull/7454) ([Artem Zuikov](https://github.com/4ertus2)) +- Certains refactoring dans la logique d'analyse de requête: diviser la classe complexe en plusieurs classes simples. [\#7454](https://github.com/ClickHouse/ClickHouse/pull/7454) ([Artem Zuikov](https://github.com/4ertus2)) - Fix construire sans submodules [\#7295](https://github.com/ClickHouse/ClickHouse/pull/7295) ([proller](https://github.com/proller)) - Mieux `add_globs` dans les fichiers CMake [\#7418](https://github.com/ClickHouse/ClickHouse/pull/7418) ([Amos Oiseau](https://github.com/amosbird)) - Supprimer les chemins codés en dur dans `unwind` cible [\#7460](https://github.com/ClickHouse/ClickHouse/pull/7460) ([Konstantin Podshumok](https://github.com/podshumok)) -- Permettre d’utiliser le format mysql sans ssl [\#7524](https://github.com/ClickHouse/ClickHouse/pull/7524) ([proller](https://github.com/proller)) +- Permettre d'utiliser le format mysql sans ssl [\#7524](https://github.com/ClickHouse/ClickHouse/pull/7524) ([proller](https://github.com/proller)) #### Autre {#other} - Ajout de la grammaire ANTLR4 pour le dialecte ClickHouse SQL [\#7595](https://github.com/ClickHouse/ClickHouse/issues/7595) [\#7596](https://github.com/ClickHouse/ClickHouse/pull/7596) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -## Clickhouse Version V19. 16 {#clickhouse-release-v19-16} +## Clickhouse Version 19.16 {#clickhouse-release-v19-16} -#### Clickhouse Version V19. 16. 14. 65, 2020-03-25 {#clickhouse-release-v19-16-14-65-2020-03-25} +#### Clickhouse Version 19.16.14.65, 2020-03-25 {#clickhouse-release-v19-16-14-65-2020-03-25} -- Correction d’un bug dans les calculs par lots des opérations logiques ternaires sur plusieurs arguments (plus de 10). [\#8718](https://github.com/ClickHouse/ClickHouse/pull/8718) ([Alexander Kazakov](https://github.com/Akazz)) Ce correctif a été rétroporté à la version 19.16 par une demande spéciale D’Altinity. +- Correction d'un bug dans les calculs par lots des opérations logiques ternaires sur plusieurs arguments (plus de 10). [\#8718](https://github.com/ClickHouse/ClickHouse/pull/8718) ([Alexander Kazakov](https://github.com/Akazz)) Ce correctif a été rétroporté à la version 19.16 par une demande spéciale D'Altinity. -#### Clickhouse Version V19. 16. 14. 65, 2020-03-05 {#clickhouse-release-v19-16-14-65-2020-03-05} +#### Clickhouse Version 19.16.14.65, 2020-03-05 {#clickhouse-release-v19-16-14-65-2020-03-05} -- Correction de l’incompatibilité des sous-requêtes distribuées avec les anciennes versions de CH. Fixer [\#7851](https://github.com/ClickHouse/ClickHouse/issues/7851) +- Correction de l'incompatibilité des sous-requêtes distribuées avec les anciennes versions de CH. Fixer [\#7851](https://github.com/ClickHouse/ClickHouse/issues/7851) [(tabplubix)](https://github.com/tavplubix) -- Lors de l’exécution de `CREATE` requête, plier les expressions constantes dans les arguments du moteur de stockage. Remplacez le nom de base de données vide par la base de données actuelle. Fixer [\#6508](https://github.com/ClickHouse/ClickHouse/issues/6508), [\#3492](https://github.com/ClickHouse/ClickHouse/issues/3492). Corrigez également la vérification de l’adresse locale dans `ClickHouseDictionarySource`. +- Lors de l'exécution de `CREATE` requête, plier les expressions constantes dans les arguments du moteur de stockage. Remplacez le nom de base de données vide par la base de données actuelle. Fixer [\#6508](https://github.com/ClickHouse/ClickHouse/issues/6508), [\#3492](https://github.com/ClickHouse/ClickHouse/issues/3492). Corrigez également la vérification de l'adresse locale dans `ClickHouseDictionarySource`. [\#9262](https://github.com/ClickHouse/ClickHouse/pull/9262) [(tabplubix)](https://github.com/tavplubix) -- Maintenant fond fusionne dans `*MergeTree` la famille des moteurs de table préserve l’ordre de volume de la Politique de stockage avec plus de précision. +- Maintenant fond fusionne dans `*MergeTree` la famille des moteurs de table préserve l'ordre de volume de la Politique de stockage avec plus de précision. [\#8549](https://github.com/ClickHouse/ClickHouse/pull/8549) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Empêcher la perte de données dans `Kafka` dans de rares cas, lorsque l’exception se produit après la lecture du suffixe mais avant la validation. Fixer [\#9378](https://github.com/ClickHouse/ClickHouse/issues/9378). Concerner: [\#7175](https://github.com/ClickHouse/ClickHouse/issues/7175) +- Empêcher la perte de données dans `Kafka` dans de rares cas, lorsque l'exception se produit après la lecture du suffixe mais avant la validation. Fixer [\#9378](https://github.com/ClickHouse/ClickHouse/issues/9378). Concerner: [\#7175](https://github.com/ClickHouse/ClickHouse/issues/7175) [\#9507](https://github.com/ClickHouse/ClickHouse/pull/9507) [(filimonov)](https://github.com/filimonov) -- Correction d’un bug menant à la résiliation du serveur lorsque vous essayez d’utiliser / drop `Kafka` tableau créé avec de mauvais paramètres. Fixer [\#9494](https://github.com/ClickHouse/ClickHouse/issues/9494). Incorporer [\#9507](https://github.com/ClickHouse/ClickHouse/issues/9507). +- Correction d'un bug menant à la résiliation du serveur lorsque vous essayez d'utiliser / drop `Kafka` tableau créé avec de mauvais paramètres. Fixer [\#9494](https://github.com/ClickHouse/ClickHouse/issues/9494). Incorporer [\#9507](https://github.com/ClickHouse/ClickHouse/issues/9507). [\#9513](https://github.com/ClickHouse/ClickHouse/pull/9513) [(filimonov)](https://github.com/filimonov) -- Autoriser l’utilisation des `MaterializedView` avec les sous-requêtes ci-dessus `Kafka` table. +- Autoriser l'utilisation des `MaterializedView` avec les sous-requêtes ci-dessus `Kafka` table. [\#8197](https://github.com/ClickHouse/ClickHouse/pull/8197) ([filimonov](https://github.com/filimonov)) #### Nouveauté {#new-feature-1} -- Ajouter `deduplicate_blocks_in_dependent_materialized_views` option pour contrôler le comportement des insertions idempotent dans des tables avec des vues matérialisées. Cette nouvelle fonctionnalité a été ajoutée à la version de bugfix par une demande spéciale D’Altinity. +- Ajouter `deduplicate_blocks_in_dependent_materialized_views` option pour contrôler le comportement des insertions idempotent dans des tables avec des vues matérialisées. Cette nouvelle fonctionnalité a été ajoutée à la version de bugfix par une demande spéciale D'Altinity. [\#9070](https://github.com/ClickHouse/ClickHouse/pull/9070) [(urykhy)](https://github.com/urykhy) -### Clickhouse Version V19. 16. 2. 2, 2019-10-30 {#clickhouse-release-v19-16-2-2-2019-10-30} +### Clickhouse Version 19.16.2.2, 2019-10-30 {#clickhouse-release-v19-16-2-2-2019-10-30} #### Modification Incompatible En Arrière {#backward-incompatible-change-1} -- Ajouter une validation d’arité manquante pour count / counIf. +- Ajouter une validation d'arité manquante pour count / counIf. [\#7095](https://github.com/ClickHouse/ClickHouse/issues/7095) [\#7298](https://github.com/ClickHouse/ClickHouse/pull/7298) ([Vdimir](https://github.com/Vdimir)) -- Supprimer l’héritage `asterisk_left_columns_only` paramètre (il est désactivé par défaut). +- Supprimer l'héritage `asterisk_left_columns_only` paramètre (il est désactivé par défaut). [\#7335](https://github.com/ClickHouse/ClickHouse/pull/7335) ([Artem Zuikov](https://github.com/4ertus2)) - Les chaînes de Format pour le format de données de modèle sont maintenant spécifiées dans les fichiers. @@ -183,7 +183,7 @@ toc_title: '2019' - Définissez deux options de configuration pour un dictionnaire basé sur une source HTTP: `credentials` et `http-headers`. [\#7092](https://github.com/ClickHouse/ClickHouse/pull/7092) ([Guillaume Tassery](https://github.com/YiuRULE)) -- Ajouter un nouveau ProfileEvent `Merge` cela compte le nombre de fusions d’arrière-plan lancées. +- Ajouter un nouveau ProfileEvent `Merge` cela compte le nombre de fusions d'arrière-plan lancées. [\#7093](https://github.com/ClickHouse/ClickHouse/pull/7093) ([Mikhail Korotov](https://github.com/millb)) - Ajouter la fonction fullHostName qui renvoie un nom de domaine complet. @@ -198,13 +198,13 @@ toc_title: '2019' - Ajouter un nouveau moteur de base de données `Lazy` qui est optimisé pour stocker un grand nombre de petits journaux table. [\#7171](https://github.com/ClickHouse/ClickHouse/pull/7171) ([Nikita Vasiliev](https://github.com/nikvas0)) -- Ajouter des fonctions d’agrégation groupBitmapAnd, - ou, - Xor pour les colonnes bitmap. [\#7109](https://github.com/ClickHouse/ClickHouse/pull/7109) ([Zhichang +- Ajouter des fonctions d'agrégation groupBitmapAnd, - ou, - Xor pour les colonnes bitmap. [\#7109](https://github.com/ClickHouse/ClickHouse/pull/7109) ([Zhichang Yu](https://github.com/yuzhichang)) -- Ajouter des combinateurs de fonctions d’agrégat-OrNull et-OrDefault, qui renvoient null - ou des valeurs par défaut lorsqu’il n’y a rien à agréger. +- Ajouter des combinateurs de fonctions d'agrégat-OrNull et-OrDefault, qui renvoient null + ou des valeurs par défaut lorsqu'il n'y a rien à agréger. [\#7331](https://github.com/ClickHouse/ClickHouse/pull/7331) ([hcz](https://github.com/hczhcz)) -- Introduire le format de données CustomSeparated qui prend en charge l’échappement personnalisé et +- Introduire le format de données CustomSeparated qui prend en charge l'échappement personnalisé et séparateur de règles. [\#7118](https://github.com/ClickHouse/ClickHouse/pull/7118) ([tavplubix](https://github.com/tavplubix)) - Soutien Redis comme source de dictionnaire externe. [\#4361](https://github.com/ClickHouse/ClickHouse/pull/4361) [\#6962](https://github.com/ClickHouse/ClickHouse/pull/6962) ([comunodi](https://github.com/comunodi), [Anton @@ -212,23 +212,23 @@ toc_title: '2019' #### Bug Fix {#bug-fix-2} -- Correction d’un résultat de requête incorrect s’il a `WHERE IN (SELECT ...)` la section et `optimize_read_in_order` être +- Correction d'un résultat de requête incorrect s'il a `WHERE IN (SELECT ...)` la section et `optimize_read_in_order` être utiliser. [\#7371](https://github.com/ClickHouse/ClickHouse/pull/7371) ([Anton Popov](https://github.com/CurtizJ)) -- Plugin D’authentification MariaDB désactivé, qui dépend des fichiers en dehors du projet. +- Plugin D'authentification MariaDB désactivé, qui dépend des fichiers en dehors du projet. [\#7140](https://github.com/ClickHouse/ClickHouse/pull/7140) ([Iouri Baranov](https://github.com/yurriy)) -- Correction d’une exception `Cannot convert column ... because it is constant but values of constants are different in source and result` ce qui pourrait rarement arriver lorsque les fonctions `now()`, `today()`, +- Correction d'une exception `Cannot convert column ... because it is constant but values of constants are different in source and result` ce qui pourrait rarement arriver lorsque les fonctions `now()`, `today()`, `yesterday()`, `randConstant()` sont utilisés. [\#7156](https://github.com/ClickHouse/ClickHouse/pull/7156) ([Nikolaï Kochetov](https://github.com/KochetovNicolai)) -- Correction d’un problème d’utilisation de HTTP keep alive timeout au lieu de TCP keep alive timeout. +- Correction d'un problème d'utilisation de HTTP keep alive timeout au lieu de TCP keep alive timeout. [\#7351](https://github.com/ClickHouse/ClickHouse/pull/7351) ([Vassili Nemkov](https://github.com/Enmk)) -- Correction d’un défaut de segmentation dans groupBitmapOr (problème [\#7109](https://github.com/ClickHouse/ClickHouse/issues/7109)). +- Correction d'un défaut de segmentation dans groupBitmapOr (problème [\#7109](https://github.com/ClickHouse/ClickHouse/issues/7109)). [\#7289](https://github.com/ClickHouse/ClickHouse/pull/7289) ([Zhichang Yu](https://github.com/yuzhichang)) -- Pour les vues matérialisées, le commit pour Kafka est appelé après l’écriture de toutes les données. +- Pour les vues matérialisées, le commit pour Kafka est appelé après l'écriture de toutes les données. [\#7175](https://github.com/ClickHouse/ClickHouse/pull/7175) ([Ivan](https://github.com/abyss7)) - Fixe de mal `duration_ms` valeur en `system.part_log` table. Il y a dix reprises. [\#7172](https://github.com/ClickHouse/ClickHouse/pull/7172) ([Vladimir @@ -241,19 +241,19 @@ toc_title: '2019' Kuzmenkov](https://github.com/akuzm)) - Ne mettez pas de colonnes virtuelles à .métadonnées sql lorsque la table est créée en tant que `CREATE TABLE AS`. [\#7183](https://github.com/ClickHouse/ClickHouse/pull/7183) ([Ivan](https://github.com/abyss7)) -- Correction d’un défaut de segmentation dans `ATTACH PART` requête. +- Correction d'un défaut de segmentation dans `ATTACH PART` requête. [\#7185](https://github.com/ClickHouse/ClickHouse/pull/7185) ([alésapine](https://github.com/alesapin)) -- Correction d’un mauvais résultat pour certaines requêtes données par l’optimisation de empty IN subqueries et empty +- Correction d'un mauvais résultat pour certaines requêtes données par l'optimisation de empty IN subqueries et empty INNER/RIGHT JOIN. [\#7284](https://github.com/ClickHouse/ClickHouse/pull/7284) ([Nikolaï Kochetov](https://github.com/KochetovNicolai)) -- Correction D’une erreur AddressSanitizer dans la méthode LIVE VIEW getHeader (). +- Correction D'une erreur AddressSanitizer dans la méthode LIVE VIEW getHeader (). [\#7271](https://github.com/ClickHouse/ClickHouse/pull/7271) ([vzakaznikov](https://github.com/vzakaznikov)) #### Amélioration {#improvement-1} -- Ajouter un message en cas d’attente queue\_wait\_max\_ms. +- Ajouter un message en cas d'attente queue\_wait\_max\_ms. [\#7390](https://github.com/ClickHouse/ClickHouse/pull/7390) ([Azat Khuzhin](https://github.com/azat)) - Faites le réglage de `s3_min_upload_part_size` au niveau de la table. @@ -264,12 +264,12 @@ toc_title: '2019' - Squash blocs de gauche en fusion partielle join (optimisation). [\#7122](https://github.com/ClickHouse/ClickHouse/pull/7122) ([Artem Zuikov](https://github.com/4ertus2)) -- N’autorisez pas les fonctions non déterministes dans les mutations des moteurs de table répliqués, car +- N'autorisez pas les fonctions non déterministes dans les mutations des moteurs de table répliqués, car peut introduire des incohérences entre les répliques. [\#7247](https://github.com/ClickHouse/ClickHouse/pull/7247) ([Alexander Kazakov](https://github.com/Akazz)) -- Désactivez le suivi de la mémoire lors de la conversion de trace de pile d’exception en chaîne. Il peut empêcher la perte - des messages d’erreur de type `Memory limit exceeded` sur le serveur, qui a causé la `Attempt to read after eof` exception sur le client. [\#7264](https://github.com/ClickHouse/ClickHouse/pull/7264) +- Désactivez le suivi de la mémoire lors de la conversion de trace de pile d'exception en chaîne. Il peut empêcher la perte + des messages d'erreur de type `Memory limit exceeded` sur le serveur, qui a causé la `Attempt to read after eof` exception sur le client. [\#7264](https://github.com/ClickHouse/ClickHouse/pull/7264) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) - Améliorations diverses du format. Résoudre [\#6033](https://github.com/ClickHouse/ClickHouse/issues/6033), @@ -278,7 +278,7 @@ toc_title: '2019' [\#6742](https://github.com/ClickHouse/ClickHouse/issues/6742) [\#7215](https://github.com/ClickHouse/ClickHouse/pull/7215) ([tavplubix](https://github.com/tavplubix)) -- ClickHouse ignore les valeurs du côté droit de L’opérateur IN qui ne sont pas convertibles vers la gauche +- ClickHouse ignore les valeurs du côté droit de L'opérateur IN qui ne sont pas convertibles vers la gauche side type. Make it work properly for compound types – Array and Tuple. [\#7283](https://github.com/ClickHouse/ClickHouse/pull/7283) ([Alexander Kuzmenkov](https://github.com/akuzm)) @@ -288,11 +288,11 @@ toc_title: '2019' Zuikov](https://github.com/4ertus2)) - Optimiser la fusion partielle jointure. [\#7070](https://github.com/ClickHouse/ClickHouse/pull/7070) ([Artem Zuikov](https://github.com/4ertus2)) -- N’utilisez pas plus de 98K de mémoire dans les fonctions uniqCombined. +- N'utilisez pas plus de 98K de mémoire dans les fonctions uniqCombined. [\#7236](https://github.com/ClickHouse/ClickHouse/pull/7236), [\#7270](https://github.com/ClickHouse/ClickHouse/pull/7270) ([Azat Khuzhin](https://github.com/azat)) -- Rincer les parties de la table de jonction de droite sur le disque dans PartialMergeJoin (s’il n’y en a pas assez +- Rincer les parties de la table de jonction de droite sur le disque dans PartialMergeJoin (s'il n'y en a pas assez mémoire). Chargez les données en arrière en cas de besoin. [\#7186](https://github.com/ClickHouse/ClickHouse/pull/7186) ([Artem Zuikov](https://github.com/4ertus2)) @@ -303,11 +303,11 @@ toc_title: '2019' Oiseau](https://github.com/amosbird)) - De retour plus tôt si la sous-requête est vide. [\#7007](https://github.com/ClickHouse/ClickHouse/pull/7007) ([小路](https://github.com/nicelulu)) -- Optimiser l’analyse de l’expression SQL dans les valeurs. +- Optimiser l'analyse de l'expression SQL dans les valeurs. [\#6781](https://github.com/ClickHouse/ClickHouse/pull/6781) ([tavplubix](https://github.com/tavplubix)) -#### Construction / Test / Amélioration De L’Emballage {#buildtestingpackaging-improvement-1} +#### Construction / Test / Amélioration De L'Emballage {#buildtestingpackaging-improvement-1} - Désactivez certaines contribs pour la compilation croisée sur Mac OS. [\#7101](https://github.com/ClickHouse/ClickHouse/pull/7101) ([Ivan](https://github.com/abyss7)) @@ -334,10 +334,10 @@ toc_title: '2019' - Assurez-vous que dh\_clean ne touche pas les fichiers sources potentiels. [\#7205](https://github.com/ClickHouse/ClickHouse/pull/7205) ([Amos Oiseau](https://github.com/amosbird)) -- Essayez d’éviter les conflits lors de la mise à jour à partir d’altinity rpm-le fichier de configuration est emballé séparément +- Essayez d'éviter les conflits lors de la mise à jour à partir d'altinity rpm-le fichier de configuration est emballé séparément dans clickhouse-serveur commun. [\#7073](https://github.com/ClickHouse/ClickHouse/pull/7073) ([filimonov](https://github.com/filimonov)) -- Optimisez certains fichiers d’en-tête pour des reconstructions plus rapides. +- Optimisez certains fichiers d'en-tête pour des reconstructions plus rapides. [\#7212](https://github.com/ClickHouse/ClickHouse/pull/7212), [\#7231](https://github.com/ClickHouse/ClickHouse/pull/7231) ([Alexander Kuzmenkov](https://github.com/akuzm)) @@ -348,7 +348,7 @@ toc_title: '2019' Kazakov](https://github.com/Akazz)) - Ajouter build avec MemorySanitizer à CI. [\#7066](https://github.com/ClickHouse/ClickHouse/pull/7066) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- Évitez l’utilisation de valeurs non initialisées dans MetricsTransmitter. +- Évitez l'utilisation de valeurs non initialisées dans MetricsTransmitter. [\#7158](https://github.com/ClickHouse/ClickHouse/pull/7158) ([Azat Khuzhin](https://github.com/azat)) - Correction de certains problèmes dans les champs trouvés par MemorySanitizer. @@ -356,23 +356,23 @@ toc_title: '2019' [\#7179](https://github.com/ClickHouse/ClickHouse/pull/7179) ([Alexander Kuzmenkov](https://github.com/akuzm)), [\#7376](https://github.com/ClickHouse/ClickHouse/pull/7376) ([Amos Oiseau](https://github.com/amosbird)) -- Correction d’un comportement indéfini dans murmurhash32. [\#7388](https://github.com/ClickHouse/ClickHouse/pull/7388) ([Amos +- Correction d'un comportement indéfini dans murmurhash32. [\#7388](https://github.com/ClickHouse/ClickHouse/pull/7388) ([Amos Oiseau](https://github.com/amosbird)) -- Correction d’un comportement indéfini dans StoragesInfoStream. [\#7384](https://github.com/ClickHouse/ClickHouse/pull/7384) +- Correction d'un comportement indéfini dans StoragesInfoStream. [\#7384](https://github.com/ClickHouse/ClickHouse/pull/7384) ([tavplubix](https://github.com/tavplubix)) -- Correction du pliage d’expressions constantes pour les moteurs de base de données externes (MySQL, ODBC, JDBC). Dans les précédents +- Correction du pliage d'expressions constantes pour les moteurs de base de données externes (MySQL, ODBC, JDBC). Dans les précédents versions il ne fonctionnait pas pour plusieurs expressions constantes et ne fonctionnait pas du tout pour la Date, DateTime et UUID. Cela corrige [\#7245](https://github.com/ClickHouse/ClickHouse/issues/7245) [\#7252](https://github.com/ClickHouse/ClickHouse/pull/7252) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction D’une erreur de course de données ThreadSanitizer dans la vue en direct lors de l’accès à la variable no\_users\_thread. +- Correction D'une erreur de course de données ThreadSanitizer dans la vue en direct lors de l'accès à la variable no\_users\_thread. [\#7353](https://github.com/ClickHouse/ClickHouse/pull/7353) ([vzakaznikov](https://github.com/vzakaznikov)) - Débarrassez-vous des symboles malloc dans libcommon [\#7134](https://github.com/ClickHouse/ClickHouse/pull/7134), [\#7065](https://github.com/ClickHouse/ClickHouse/pull/7065) ([Amos Oiseau](https://github.com/amosbird)) -- Ajoutez l’indicateur global ENABLE\_LIBRARIES pour désactiver toutes les bibliothèques. +- Ajoutez l'indicateur global ENABLE\_LIBRARIES pour désactiver toutes les bibliothèques. [\#7063](https://github.com/ClickHouse/ClickHouse/pull/7063) ([proller](https://github.com/proller)) @@ -389,15 +389,15 @@ toc_title: '2019' - Petit refactoring et renommage près de dictionnaires externes. [\#7111](https://github.com/ClickHouse/ClickHouse/pull/7111) ([alésapine](https://github.com/alesapin)) -- Refactorisez du code pour vous préparer au contrôle d’accès basé sur les rôles. [\#7235](https://github.com/ClickHouse/ClickHouse/pull/7235) ([Vitaly +- Refactorisez du code pour vous préparer au contrôle d'accès basé sur les rôles. [\#7235](https://github.com/ClickHouse/ClickHouse/pull/7235) ([Vitaly Baranov](https://github.com/vitlibar)) - Quelques améliorations dans Databasecode ordinaire. [\#7086](https://github.com/ClickHouse/ClickHouse/pull/7086) ([Nikita Vasiliev](https://github.com/nikvas0)) -- N’utilisez pas d’itérateurs dans les méthodes find() et emplace () des tables de hachage. +- N'utilisez pas d'itérateurs dans les méthodes find() et emplace () des tables de hachage. [\#7026](https://github.com/ClickHouse/ClickHouse/pull/7026) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- Fix getMultipleValuesFromConfig dans le cas où le paramètre root n’est pas vide. [\#7374](https://github.com/ClickHouse/ClickHouse/pull/7374) +- Fix getMultipleValuesFromConfig dans le cas où le paramètre root n'est pas vide. [\#7374](https://github.com/ClickHouse/ClickHouse/pull/7374) ([Mikhail Korotov](https://github.com/millb)) - Supprimer un copier-coller (TemporaryFile et TemporaryFileStream) [\#7166](https://github.com/ClickHouse/ClickHouse/pull/7166) ([Artem @@ -407,7 +407,7 @@ toc_title: '2019' Chebotarev](https://github.com/excitoon)) - Attendez tous les travaux planifiés, qui utilisent des objets locaux, si `ThreadPool::schedule(...)` jeter exception. Renommer `ThreadPool::schedule(...)` de `ThreadPool::scheduleOrThrowOnError(...)` et - correction des commentaires pour rendre évident qu’il peut jeter. + correction des commentaires pour rendre évident qu'il peut jeter. [\#7350](https://github.com/ClickHouse/ClickHouse/pull/7350) ([tavplubix](https://github.com/tavplubix)) @@ -427,19 +427,19 @@ toc_title: '2019' [\#7377](https://github.com/ClickHouse/ClickHouse/pull/7377) ([Azat Khuzhin](https://github.com/azat)) - Fonction fixe getMultipleValuesFromConfig. [\#7374](https://github.com/ClickHouse/ClickHouse/pull/7374) ([Mikhail Korotov](https://github.com/millb)) -- Correction d’un problème d’utilisation de HTTP keep alive timeout au lieu de TCP keep alive timeout. +- Correction d'un problème d'utilisation de HTTP keep alive timeout au lieu de TCP keep alive timeout. [\#7351](https://github.com/ClickHouse/ClickHouse/pull/7351) ([Vasily Nemkov](https://github.com/Enmk)) -- Attendez que tous les travaux se terminent à l’exception (corrige les segfaults rares). +- Attendez que tous les travaux se terminent à l'exception (corrige les segfaults rares). [\#7350](https://github.com/ClickHouse/ClickHouse/pull/7350) ([tavplubix](https://github.com/tavplubix)) -- Ne poussez pas vers MVs lors de l’insertion dans la table Kafka. +- Ne poussez pas vers MVs lors de l'insertion dans la table Kafka. [\#7265](https://github.com/ClickHouse/ClickHouse/pull/7265) ([Ivan](https://github.com/abyss7)) -- Désactiver le suivi de la mémoire pour la pile d’exception. +- Désactiver le suivi de la mémoire pour la pile d'exception. [\#7264](https://github.com/ClickHouse/ClickHouse/pull/7264) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Correction d’un mauvais code dans la transformation de la requête pour la base de données externe. +- Correction d'un mauvais code dans la transformation de la requête pour la base de données externe. [\#7252](https://github.com/ClickHouse/ClickHouse/pull/7252) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Évitez l’utilisation de valeurs non initialisées dans MetricsTransmitter. +- Évitez l'utilisation de valeurs non initialisées dans MetricsTransmitter. [\#7158](https://github.com/ClickHouse/ClickHouse/pull/7158) ([Azat Khuzhin](https://github.com/azat)) -- Ajout d’un exemple de configuration avec des macros pour les tests ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Ajout d'un exemple de configuration avec des macros pour les tests ([alexeï-milovidov](https://github.com/alexey-milovidov)) ### Clickhouse Version 19.15.3.6, 2019-10-09 {#clickhouse-release-19-15-3-6-2019-10-09} @@ -447,11 +447,11 @@ toc_title: '2019' - Correction de bad\_variant dans le dictionnaire haché. ([alésapine](https://github.com/alesapin)) -- Correction d’un bug avec défaut de segmentation dans la requête de pièce jointe. +- Correction d'un bug avec défaut de segmentation dans la requête de pièce jointe. ([alésapine](https://github.com/alesapin)) - Calcul du temps fixe en `MergeTreeData`. ([Vladimir Chebotarev](https://github.com/excitoon)) -- Commit à Kafka explicitement après la finalisation de l’écriture. +- Commit à Kafka explicitement après la finalisation de l'écriture. [\#7175](https://github.com/ClickHouse/ClickHouse/pull/7175) ([Ivan](https://github.com/abyss7)) - Sérialiser correctement les valeurs NULL dans les index min / max des parties MergeTree. [\#7234](https://github.com/ClickHouse/ClickHouse/pull/7234) ([Alexander Kuzmenkov](https://github.com/akuzm)) @@ -460,56 +460,56 @@ toc_title: '2019' #### Nouveauté {#new-feature-3} -- Stockage à plusieurs niveaux: prise en charge de l’utilisation de plusieurs volumes de stockage pour les tables avec mergetree engine. Il est possible de stocker de nouvelles données sur SSD et de déplacer automatiquement les anciennes données sur le disque dur. ([exemple](https://clickhouse.github.io/clickhouse-presentations/meetup30/new_features/#12)). [\#4918](https://github.com/ClickHouse/ClickHouse/pull/4918) ([Igr](https://github.com/ObjatieGroba)) [\#6489](https://github.com/ClickHouse/ClickHouse/pull/6489) ([alésapine](https://github.com/alesapin)) +- Stockage à plusieurs niveaux: prise en charge de l'utilisation de plusieurs volumes de stockage pour les tables avec mergetree engine. Il est possible de stocker de nouvelles données sur SSD et de déplacer automatiquement les anciennes données sur le disque dur. ([exemple](https://clickhouse.github.io/clickhouse-presentations/meetup30/new_features/#12)). [\#4918](https://github.com/ClickHouse/ClickHouse/pull/4918) ([Igr](https://github.com/ObjatieGroba)) [\#6489](https://github.com/ClickHouse/ClickHouse/pull/6489) ([alésapine](https://github.com/alesapin)) - Ajouter une fonction de table `input` pour lire les données entrantes dans `INSERT SELECT` requête. [\#5450](https://github.com/ClickHouse/ClickHouse/pull/5450) ([palasonique1](https://github.com/palasonic1)) [\#6832](https://github.com/ClickHouse/ClickHouse/pull/6832) ([Anton Popov](https://github.com/CurtizJ)) -- Ajouter un `sparse_hashed` mise en page du dictionnaire, qui est fonctionnellement équivalente à la `hashed` mise en page, mais est plus efficace en mémoire. Il utilise environ deux fois moins de mémoire au prix d’une récupération de valeur plus lente. [\#6894](https://github.com/ClickHouse/ClickHouse/pull/6894) ([Azat Khuzhin](https://github.com/azat)) -- Implémenter la capacité de définir la liste des utilisateurs pour l’accès aux dictionnaires. Seule la base de données connectée actuelle utilisant. [\#6907](https://github.com/ClickHouse/ClickHouse/pull/6907) ([Guillaume Tassery](https://github.com/YiuRULE)) +- Ajouter un `sparse_hashed` mise en page du dictionnaire, qui est fonctionnellement équivalente à la `hashed` mise en page, mais est plus efficace en mémoire. Il utilise environ deux fois moins de mémoire au prix d'une récupération de valeur plus lente. [\#6894](https://github.com/ClickHouse/ClickHouse/pull/6894) ([Azat Khuzhin](https://github.com/azat)) +- Implémenter la capacité de définir la liste des utilisateurs pour l'accès aux dictionnaires. Seule la base de données connectée actuelle utilisant. [\#6907](https://github.com/ClickHouse/ClickHouse/pull/6907) ([Guillaume Tassery](https://github.com/YiuRULE)) - Ajouter `LIMIT` option pour `SHOW` requête. [\#6944](https://github.com/ClickHouse/ClickHouse/pull/6944) ([Philipp Malkovsky](https://github.com/malkfilipp)) -- Ajouter `bitmapSubsetLimit(bitmap, range_start, limit)` fonction, qui renvoie le sous-ensemble du plus petit `limit` valeurs dans l’ensemble qui n’est pas inférieure à `range_start`. [\#6957](https://github.com/ClickHouse/ClickHouse/pull/6957) ([Zhichang Yu](https://github.com/yuzhichang)) +- Ajouter `bitmapSubsetLimit(bitmap, range_start, limit)` fonction, qui renvoie le sous-ensemble du plus petit `limit` valeurs dans l'ensemble qui n'est pas inférieure à `range_start`. [\#6957](https://github.com/ClickHouse/ClickHouse/pull/6957) ([Zhichang Yu](https://github.com/yuzhichang)) - Ajouter `bitmapMin` et `bitmapMax` fonction. [\#6970](https://github.com/ClickHouse/ClickHouse/pull/6970) ([Zhichang Yu](https://github.com/yuzhichang)) - Ajouter une fonction `repeat` liées à la [numéro-6648](https://github.com/ClickHouse/ClickHouse/issues/6648) [\#6999](https://github.com/ClickHouse/ClickHouse/pull/6999) ([Flynn](https://github.com/ucasFL)) #### Caractéristique Expérimentale {#experimental-feature-1} - Implémentez (en mémoire) une variante de jointure de fusion qui ne change pas le pipeline actuel. Le résultat est partiellement trié par clé de fusion. Définir `partial_merge_join = 1` pour utiliser cette fonctionnalité. La Jointure de Fusion est toujours en développement. [\#6940](https://github.com/ClickHouse/ClickHouse/pull/6940) ([Artem Zuikov](https://github.com/4ertus2)) -- Ajouter `S3` fonction de moteur et de table. Il est encore en développement (pas encore de support d’authentification). [\#5596](https://github.com/ClickHouse/ClickHouse/pull/5596) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Ajouter `S3` fonction de moteur et de table. Il est encore en développement (pas encore de support d'authentification). [\#5596](https://github.com/ClickHouse/ClickHouse/pull/5596) ([Vladimir Chebotarev](https://github.com/excitoon)) #### Amélioration {#improvement-2} - Chaque message lu à partir de Kafka est inséré atomiquement. Cela résout presque tous les problèmes connus avec Kafka engine. [\#6950](https://github.com/ClickHouse/ClickHouse/pull/6950) ([Ivan](https://github.com/abyss7)) - Améliorations pour le basculement des requêtes distribuées. Raccourcir le temps de récupération, il est maintenant configurable et peut être vu dans `system.clusters`. [\#6399](https://github.com/ClickHouse/ClickHouse/pull/6399) ([Vasily Nemkov](https://github.com/Enmk)) - Supporte les valeurs numériques pour les énumérations directement dans `IN` section. \#6766 [\#6941](https://github.com/ClickHouse/ClickHouse/pull/6941) ([dimarub2000](https://github.com/dimarub2000)) -- Support (facultatif, désactivé par défaut) redirige sur le stockage D’URL. [\#6914](https://github.com/ClickHouse/ClickHouse/pull/6914) ([maqroll](https://github.com/maqroll)) -- Ajouter un message d’information lorsque le client avec une ancienne version se connecte à un serveur. [\#6893](https://github.com/ClickHouse/ClickHouse/pull/6893) ([Philipp Malkovsky](https://github.com/malkfilipp)) -- Supprimer la limite de temps de veille maximale pour l’envoi de données dans les tables distribuées [\#6895](https://github.com/ClickHouse/ClickHouse/pull/6895) ([Azat Khuzhin](https://github.com/azat)) -- Ajouter la possibilité d’envoyer des événements de profil (compteurs) avec des valeurs cumulatives à graphite. Il peut être activé sous `` dans serveur `config.xml`. [\#6969](https://github.com/ClickHouse/ClickHouse/pull/6969) ([Azat Khuzhin](https://github.com/azat)) -- Ajouter automatiquement le type de fonte `T` de `LowCardinality(T)` lors de l’insertion de données dans la colonne de type `LowCardinality(T)` au format natif via HTTP. [\#6891](https://github.com/ClickHouse/ClickHouse/pull/6891) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Ajout de la capacité à utiliser la fonction `hex` sans l’aide de `reinterpretAsString` pour `Float32`, `Float64`. [\#7024](https://github.com/ClickHouse/ClickHouse/pull/7024) ([Mikhail Korotov](https://github.com/millb)) +- Support (facultatif, désactivé par défaut) redirige sur le stockage D'URL. [\#6914](https://github.com/ClickHouse/ClickHouse/pull/6914) ([maqroll](https://github.com/maqroll)) +- Ajouter un message d'information lorsque le client avec une ancienne version se connecte à un serveur. [\#6893](https://github.com/ClickHouse/ClickHouse/pull/6893) ([Philipp Malkovsky](https://github.com/malkfilipp)) +- Supprimer la limite de temps de veille maximale pour l'envoi de données dans les tables distribuées [\#6895](https://github.com/ClickHouse/ClickHouse/pull/6895) ([Azat Khuzhin](https://github.com/azat)) +- Ajouter la possibilité d'envoyer des événements de profil (compteurs) avec des valeurs cumulatives à graphite. Il peut être activé sous `` dans serveur `config.xml`. [\#6969](https://github.com/ClickHouse/ClickHouse/pull/6969) ([Azat Khuzhin](https://github.com/azat)) +- Ajouter automatiquement le type de fonte `T` de `LowCardinality(T)` lors de l'insertion de données dans la colonne de type `LowCardinality(T)` au format natif via HTTP. [\#6891](https://github.com/ClickHouse/ClickHouse/pull/6891) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Ajout de la capacité à utiliser la fonction `hex` sans l'aide de `reinterpretAsString` pour `Float32`, `Float64`. [\#7024](https://github.com/ClickHouse/ClickHouse/pull/7024) ([Mikhail Korotov](https://github.com/millb)) -#### Construction / Test / Amélioration De L’Emballage {#buildtestingpackaging-improvement-2} +#### Construction / Test / Amélioration De L'Emballage {#buildtestingpackaging-improvement-2} -- Ajouter gdb-index au binaire clickhouse avec des informations de débogage. Il permettra d’accélérer le temps de démarrage de `gdb`. [\#6947](https://github.com/ClickHouse/ClickHouse/pull/6947) ([alésapine](https://github.com/alesapin)) -- Accélérez l’emballage deb avec dpkg-deb patché qui utilise `pigz`. [\#6960](https://github.com/ClickHouse/ClickHouse/pull/6960) ([alésapine](https://github.com/alesapin)) -- Définir `enable_fuzzing = 1` pour activer l’instrumentation libfuzzer de tout le code du projet. [\#7042](https://github.com/ClickHouse/ClickHouse/pull/7042) ([kyprizel](https://github.com/kyprizel)) +- Ajouter gdb-index au binaire clickhouse avec des informations de débogage. Il permettra d'accélérer le temps de démarrage de `gdb`. [\#6947](https://github.com/ClickHouse/ClickHouse/pull/6947) ([alésapine](https://github.com/alesapin)) +- Accélérez l'emballage deb avec dpkg-deb patché qui utilise `pigz`. [\#6960](https://github.com/ClickHouse/ClickHouse/pull/6960) ([alésapine](https://github.com/alesapin)) +- Définir `enable_fuzzing = 1` pour activer l'instrumentation libfuzzer de tout le code du projet. [\#7042](https://github.com/ClickHouse/ClickHouse/pull/7042) ([kyprizel](https://github.com/kyprizel)) - Ajouter Split build smoke test dans CI. [\#7061](https://github.com/ClickHouse/ClickHouse/pull/7061) ([alésapine](https://github.com/alesapin)) - Ajouter build avec MemorySanitizer à CI. [\#7066](https://github.com/ClickHouse/ClickHouse/pull/7066) ([Alexander Kuzmenkov](https://github.com/akuzm)) - Remplacer `libsparsehash` avec `sparsehash-c11` [\#6965](https://github.com/ClickHouse/ClickHouse/pull/6965) ([Azat Khuzhin](https://github.com/azat)) #### Bug Fix {#bug-fix-5} -- Correction de la dégradation des performances de l’analyse d’index sur les clés complexes sur les grandes tables. Cela corrige \# 6924. [\#7075](https://github.com/ClickHouse/ClickHouse/pull/7075) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’une erreur logique provoquant segfaults lors de la sélection de Kafka sujet vide. [\#6909](https://github.com/ClickHouse/ClickHouse/pull/6909) ([Ivan](https://github.com/abyss7)) -- Correction d’une connexion MySQL trop tôt fermer `MySQLBlockInputStream.cpp`. [\#6882](https://github.com/ClickHouse/ClickHouse/pull/6882) ([Clément Rodriguez](https://github.com/clemrodriguez)) +- Correction de la dégradation des performances de l'analyse d'index sur les clés complexes sur les grandes tables. Cela corrige \# 6924. [\#7075](https://github.com/ClickHouse/ClickHouse/pull/7075) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction d'une erreur logique provoquant segfaults lors de la sélection de Kafka sujet vide. [\#6909](https://github.com/ClickHouse/ClickHouse/pull/6909) ([Ivan](https://github.com/abyss7)) +- Correction d'une connexion MySQL trop tôt fermer `MySQLBlockInputStream.cpp`. [\#6882](https://github.com/ClickHouse/ClickHouse/pull/6882) ([Clément Rodriguez](https://github.com/clemrodriguez)) - Retour du support pour les très anciens noyaux Linux (correction [\#6841](https://github.com/ClickHouse/ClickHouse/issues/6841)) [\#6853](https://github.com/ClickHouse/ClickHouse/pull/6853) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Corriger les éventuelles pertes de données dans `insert select` requête en cas de bloc vide dans le flux d’entrée. \#6834 \#6862 [\#6911](https://github.com/ClickHouse/ClickHouse/pull/6911) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Corriger les éventuelles pertes de données dans `insert select` requête en cas de bloc vide dans le flux d'entrée. \#6834 \#6862 [\#6911](https://github.com/ClickHouse/ClickHouse/pull/6911) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) - Correctif pour la fonction `АrrayEnumerateUniqRanked` avec des tableaux vides dans params [\#6928](https://github.com/ClickHouse/ClickHouse/pull/6928) ([proller](https://github.com/proller)) - Correction de requêtes complexes avec des jointures de tableau et des sous-requêtes globales. [\#6934](https://github.com/ClickHouse/ClickHouse/pull/6934) ([Ivan](https://github.com/abyss7)) - Fixer `Unknown identifier` erreur dans ORDER BY et GROUP BY avec plusieurs jointures [\#7022](https://github.com/ClickHouse/ClickHouse/pull/7022) ([Artem Zuikov](https://github.com/4ertus2)) -- Fixe `MSan` avertissement lors de l’exécution de la fonction avec `LowCardinality` argument. [\#7062](https://github.com/ClickHouse/ClickHouse/pull/7062) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Fixe `MSan` avertissement lors de l'exécution de la fonction avec `LowCardinality` argument. [\#7062](https://github.com/ClickHouse/ClickHouse/pull/7062) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) #### Modification Incompatible En Arrière {#backward-incompatible-change-2} -- Format de sérialisation modifié de bitmap \* états de fonction d’agrégation pour améliorer les performances. Les États sérialisés de bitmap \* des versions précédentes ne peuvent pas être lus. [\#6908](https://github.com/ClickHouse/ClickHouse/pull/6908) ([Zhichang Yu](https://github.com/yuzhichang)) +- Format de sérialisation modifié de bitmap \* états de fonction d'agrégation pour améliorer les performances. Les États sérialisés de bitmap \* des versions précédentes ne peuvent pas être lus. [\#6908](https://github.com/ClickHouse/ClickHouse/pull/6908) ([Zhichang Yu](https://github.com/yuzhichang)) ## Clickhouse Version 19.14 {#clickhouse-release-19-14} @@ -525,12 +525,12 @@ toc_title: '2019' #### Bug Fix {#bug-fix-7} - Correctif pour la fonction `АrrayEnumerateUniqRanked` avec des tableaux vides dans params. [\#6928](https://github.com/ClickHouse/ClickHouse/pull/6928) ([proller](https://github.com/proller)) -- Nom de sous-requête fixe dans les requêtes avec `ARRAY JOIN` et `GLOBAL IN subquery` avec alias. Utilisez l’alias de sous-requête pour le nom de table externe s’il est spécifié. [\#6934](https://github.com/ClickHouse/ClickHouse/pull/6934) ([Ivan](https://github.com/abyss7)) +- Nom de sous-requête fixe dans les requêtes avec `ARRAY JOIN` et `GLOBAL IN subquery` avec alias. Utilisez l'alias de sous-requête pour le nom de table externe s'il est spécifié. [\#6934](https://github.com/ClickHouse/ClickHouse/pull/6934) ([Ivan](https://github.com/abyss7)) -#### Construction / Test / Amélioration De L’Emballage {#buildtestingpackaging-improvement-3} +#### Construction / Test / Amélioration De L'Emballage {#buildtestingpackaging-improvement-3} -- Fixer [le battement](https://clickhouse-test-reports.s3.yandex.net/6944/aab95fd5175a513413c7395a73a82044bdafb906/functional_stateless_tests_(debug).html) test `00715_fetch_merged_or_mutated_part_zookeeper` en le réécrivant dans un script shell car il doit attendre que des mutations s’appliquent. [\#6977](https://github.com/ClickHouse/ClickHouse/pull/6977) ([Alexander Kazakov](https://github.com/Akazz)) -- Correction de L’échec UBSan et MemSan en fonction `groupUniqArray` avec l’argument de tableau emtpy. Il a été causé par le placement de vide `PaddedPODArray` dans la table de hachage zéro cellule parce que le constructeur pour la valeur de cellule Zéro n’a pas été appelé. [\#6937](https://github.com/ClickHouse/ClickHouse/pull/6937) ([Amos Oiseau](https://github.com/amosbird)) +- Fixer [le battement](https://clickhouse-test-reports.s3.yandex.net/6944/aab95fd5175a513413c7395a73a82044bdafb906/functional_stateless_tests_(debug).html) test `00715_fetch_merged_or_mutated_part_zookeeper` en le réécrivant dans un script shell car il doit attendre que des mutations s'appliquent. [\#6977](https://github.com/ClickHouse/ClickHouse/pull/6977) ([Alexander Kazakov](https://github.com/Akazz)) +- Correction de L'échec UBSan et MemSan en fonction `groupUniqArray` avec l'argument de tableau emtpy. Il a été causé par le placement de vide `PaddedPODArray` dans la table de hachage zéro cellule parce que le constructeur pour la valeur de cellule Zéro n'a pas été appelé. [\#6937](https://github.com/ClickHouse/ClickHouse/pull/6937) ([Amos Oiseau](https://github.com/amosbird)) ### Clickhouse Version 19.14.3.3, 2019-09-10 {#clickhouse-release-19-14-3-3-2019-09-10} @@ -538,23 +538,23 @@ toc_title: '2019' - `WITH FILL` le modificateur `ORDER BY`. (suite de la [\#5069](https://github.com/ClickHouse/ClickHouse/issues/5069)) [\#6610](https://github.com/ClickHouse/ClickHouse/pull/6610) ([Anton Popov](https://github.com/CurtizJ)) - `WITH TIES` le modificateur `LIMIT`. (suite de la [\#5069](https://github.com/ClickHouse/ClickHouse/issues/5069)) [\#6610](https://github.com/ClickHouse/ClickHouse/pull/6610) ([Anton Popov](https://github.com/CurtizJ)) -- Analyser non cotées `NULL` littéral comme NULL (si paramètre `format_csv_unquoted_null_literal_as_null=1`). Initialiser les champs null avec des valeurs par défaut si le type de données de ce champ n’est pas nullable (si `input_format_null_as_default=1`). [\#5990](https://github.com/ClickHouse/ClickHouse/issues/5990) [\#6055](https://github.com/ClickHouse/ClickHouse/pull/6055) ([tavplubix](https://github.com/tavplubix)) -- Prise en charge des caractères génériques dans les chemins des fonctions de table `file` et `hdfs`. Si le chemin contient des caractères génériques, la table sera en lecture seule. Exemple d’utilisation: `select * from hdfs('hdfs://hdfs1:9000/some_dir/another_dir/*/file{0..9}{0..9}')` et `select * from file('some_dir/{some_file,another_file,yet_another}.tsv', 'TSV', 'value UInt32')`. [\#6092](https://github.com/ClickHouse/ClickHouse/pull/6092) ([Olga Khvostikova](https://github.com/stavrolia)) -- Nouveau `system.metric_log` le tableau qui stocke les valeurs de `system.events` et `system.metrics` avec l’intervalle de temps spécifié. [\#6363](https://github.com/ClickHouse/ClickHouse/issues/6363) [\#6467](https://github.com/ClickHouse/ClickHouse/pull/6467) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) [\#6530](https://github.com/ClickHouse/ClickHouse/pull/6530) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Permettre d’écrire des journaux de texte ClickHouse à `system.text_log` table. [\#6037](https://github.com/ClickHouse/ClickHouse/issues/6037) [\#6103](https://github.com/ClickHouse/ClickHouse/pull/6103) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) [\#6164](https://github.com/ClickHouse/ClickHouse/pull/6164) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Afficher les symboles privés dans les traces de pile (cela se fait via l’analyse des tables de symboles des fichiers ELF). Ajout d’informations sur le numéro de fichier et de ligne dans les traces de pile si les informations de débogage sont présentes. Accélérer la recherche de nom de symbole avec des symboles d’indexation présents dans le programme. Ajout de nouvelles fonctions SQL pour l’introspection: `demangle` et `addressToLine`. Renommé fonction `symbolizeAddress` de `addressToSymbol` pour des raisons de cohérence. Fonction `addressToSymbol` retournera le nom mutilé pour des raisons de performance et vous devez appliquer `demangle`. Ajout d’un réglage `allow_introspection_functions` qui est désactivée par défaut. [\#6201](https://github.com/ClickHouse/ClickHouse/pull/6201) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Analyser non cotées `NULL` littéral comme NULL (si paramètre `format_csv_unquoted_null_literal_as_null=1`). Initialiser les champs null avec des valeurs par défaut si le type de données de ce champ n'est pas nullable (si `input_format_null_as_default=1`). [\#5990](https://github.com/ClickHouse/ClickHouse/issues/5990) [\#6055](https://github.com/ClickHouse/ClickHouse/pull/6055) ([tavplubix](https://github.com/tavplubix)) +- Prise en charge des caractères génériques dans les chemins des fonctions de table `file` et `hdfs`. Si le chemin contient des caractères génériques, la table sera en lecture seule. Exemple d'utilisation: `select * from hdfs('hdfs://hdfs1:9000/some_dir/another_dir/*/file{0..9}{0..9}')` et `select * from file('some_dir/{some_file,another_file,yet_another}.tsv', 'TSV', 'value UInt32')`. [\#6092](https://github.com/ClickHouse/ClickHouse/pull/6092) ([Olga Khvostikova](https://github.com/stavrolia)) +- Nouveau `system.metric_log` le tableau qui stocke les valeurs de `system.events` et `system.metrics` avec l'intervalle de temps spécifié. [\#6363](https://github.com/ClickHouse/ClickHouse/issues/6363) [\#6467](https://github.com/ClickHouse/ClickHouse/pull/6467) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) [\#6530](https://github.com/ClickHouse/ClickHouse/pull/6530) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Permettre d'écrire des journaux de texte ClickHouse à `system.text_log` table. [\#6037](https://github.com/ClickHouse/ClickHouse/issues/6037) [\#6103](https://github.com/ClickHouse/ClickHouse/pull/6103) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) [\#6164](https://github.com/ClickHouse/ClickHouse/pull/6164) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Afficher les symboles privés dans les traces de pile (cela se fait via l'analyse des tables de symboles des fichiers ELF). Ajout d'informations sur le numéro de fichier et de ligne dans les traces de pile si les informations de débogage sont présentes. Accélérer la recherche de nom de symbole avec des symboles d'indexation présents dans le programme. Ajout de nouvelles fonctions SQL pour l'introspection: `demangle` et `addressToLine`. Renommé fonction `symbolizeAddress` de `addressToSymbol` pour des raisons de cohérence. Fonction `addressToSymbol` retournera le nom mutilé pour des raisons de performance et vous devez appliquer `demangle`. Ajout d'un réglage `allow_introspection_functions` qui est désactivée par défaut. [\#6201](https://github.com/ClickHouse/ClickHouse/pull/6201) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Fonction de Table `values` (le nom est sensible à la casse). Il permet de lire à partir de `VALUES` la liste proposée dans [\#5984](https://github.com/ClickHouse/ClickHouse/issues/5984). Exemple: `SELECT * FROM VALUES('a UInt64, s String', (1, 'one'), (2, 'two'), (3, 'three'))`. [\#6217](https://github.com/ClickHouse/ClickHouse/issues/6217). [\#6209](https://github.com/ClickHouse/ClickHouse/pull/6209) ([dimarub2000](https://github.com/dimarub2000)) -- Ajout d’une capacité de modifier les paramètres de stockage. Syntaxe: `ALTER TABLE
MODIFY SETTING = `. [\#6366](https://github.com/ClickHouse/ClickHouse/pull/6366) [\#6669](https://github.com/ClickHouse/ClickHouse/pull/6669) [\#6685](https://github.com/ClickHouse/ClickHouse/pull/6685) ([alésapine](https://github.com/alesapin)) +- Ajout d'une capacité de modifier les paramètres de stockage. Syntaxe: `ALTER TABLE
MODIFY SETTING = `. [\#6366](https://github.com/ClickHouse/ClickHouse/pull/6366) [\#6669](https://github.com/ClickHouse/ClickHouse/pull/6669) [\#6685](https://github.com/ClickHouse/ClickHouse/pull/6685) ([alésapine](https://github.com/alesapin)) - Support pour enlever des pièces détachées. Syntaxe: `ALTER TABLE DROP DETACHED PART ''`. [\#6158](https://github.com/ClickHouse/ClickHouse/pull/6158) ([tavplubix](https://github.com/tavplubix)) -- Les contraintes de Table. Permet d’ajouter une contrainte à la définition de la table qui sera vérifiée lors de l’insertion. [\#5273](https://github.com/ClickHouse/ClickHouse/pull/5273) ([Gleb Novikov](https://github.com/NanoBjorn)) [\#6652](https://github.com/ClickHouse/ClickHouse/pull/6652) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Les contraintes de Table. Permet d'ajouter une contrainte à la définition de la table qui sera vérifiée lors de l'insertion. [\#5273](https://github.com/ClickHouse/ClickHouse/pull/5273) ([Gleb Novikov](https://github.com/NanoBjorn)) [\#6652](https://github.com/ClickHouse/ClickHouse/pull/6652) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Soutien en cascade des vues matérialisées. [\#6324](https://github.com/ClickHouse/ClickHouse/pull/6324) ([Amos Oiseau](https://github.com/amosbird)) -- Activez query profiler par défaut pour échantillonner chaque thread d’exécution de requête une fois par seconde. [\#6283](https://github.com/ClickHouse/ClickHouse/pull/6283) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Format d’entrée `ORC`. [\#6454](https://github.com/ClickHouse/ClickHouse/pull/6454) [\#6703](https://github.com/ClickHouse/ClickHouse/pull/6703) ([akonyaev90](https://github.com/akonyaev90)) -- Ajout de deux nouvelles fonctions: `sigmoid` et `tanh` (qui sont utiles pour les applications d’apprentissage automatique). [\#6254](https://github.com/ClickHouse/ClickHouse/pull/6254) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Fonction `hasToken(haystack, token)`, `hasTokenCaseInsensitive(haystack, token)` pour vérifier si un jeton est dans la botte de foin. Le jeton est une sous-chaîne de longueur maximale entre deux caractères ASCII non alphanumériques (ou limites de haystack). Le jeton doit être une chaîne constante. Pris en charge par tokenbf\_v1 spécialisation de l’index. [\#6596](https://github.com/ClickHouse/ClickHouse/pull/6596), [\#6662](https://github.com/ClickHouse/ClickHouse/pull/6662) ([Vasily Nemkov](https://github.com/Enmk)) -- Nouvelle fonction `neighbor(value, offset[, default_value])`. Permet d’atteindre la valeur prev / next dans la colonne d’un bloc de données. [\#5925](https://github.com/ClickHouse/ClickHouse/pull/5925) ([Alex Laquelle Le Krash](https://github.com/alex-krash)) [6685365ab8c5b74f9650492c88a012596eb1b0c6](https://github.com/ClickHouse/ClickHouse/commit/6685365ab8c5b74f9650492c88a012596eb1b0c6) [341e2e4587a18065c2da1ca888c73389f48ce36c](https://github.com/ClickHouse/ClickHouse/commit/341e2e4587a18065c2da1ca888c73389f48ce36c) [Alexey Milovidov](https://github.com/alexey-milovidov) -- Créé une fonction `currentUser()`, retour connexion de l’utilisateur autorisé. Ajout d’alias `user()` pour la compatibilité avec MySQL. [\#6470](https://github.com/ClickHouse/ClickHouse/pull/6470) ([Alex Laquelle Le Krash](https://github.com/alex-krash)) -- Nouvelles fonctions d’agrégation `quantilesExactInclusive` et `quantilesExactExclusive` qui étaient proposés dans [\#5885](https://github.com/ClickHouse/ClickHouse/issues/5885). [\#6477](https://github.com/ClickHouse/ClickHouse/pull/6477) ([dimarub2000](https://github.com/dimarub2000)) +- Activez query profiler par défaut pour échantillonner chaque thread d'exécution de requête une fois par seconde. [\#6283](https://github.com/ClickHouse/ClickHouse/pull/6283) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Format d'entrée `ORC`. [\#6454](https://github.com/ClickHouse/ClickHouse/pull/6454) [\#6703](https://github.com/ClickHouse/ClickHouse/pull/6703) ([akonyaev90](https://github.com/akonyaev90)) +- Ajout de deux nouvelles fonctions: `sigmoid` et `tanh` (qui sont utiles pour les applications d'apprentissage automatique). [\#6254](https://github.com/ClickHouse/ClickHouse/pull/6254) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Fonction `hasToken(haystack, token)`, `hasTokenCaseInsensitive(haystack, token)` pour vérifier si un jeton est dans la botte de foin. Le jeton est une sous-chaîne de longueur maximale entre deux caractères ASCII non alphanumériques (ou limites de haystack). Le jeton doit être une chaîne constante. Pris en charge par tokenbf\_v1 spécialisation de l'index. [\#6596](https://github.com/ClickHouse/ClickHouse/pull/6596), [\#6662](https://github.com/ClickHouse/ClickHouse/pull/6662) ([Vasily Nemkov](https://github.com/Enmk)) +- Nouvelle fonction `neighbor(value, offset[, default_value])`. Permet d'atteindre la valeur prev / next dans la colonne d'un bloc de données. [\#5925](https://github.com/ClickHouse/ClickHouse/pull/5925) ([Alex Laquelle Le Krash](https://github.com/alex-krash)) [6685365ab8c5b74f9650492c88a012596eb1b0c6](https://github.com/ClickHouse/ClickHouse/commit/6685365ab8c5b74f9650492c88a012596eb1b0c6) [341e2e4587a18065c2da1ca888c73389f48ce36c](https://github.com/ClickHouse/ClickHouse/commit/341e2e4587a18065c2da1ca888c73389f48ce36c) [Alexey Milovidov](https://github.com/alexey-milovidov) +- Créé une fonction `currentUser()`, retour connexion de l'utilisateur autorisé. Ajout d'alias `user()` pour la compatibilité avec MySQL. [\#6470](https://github.com/ClickHouse/ClickHouse/pull/6470) ([Alex Laquelle Le Krash](https://github.com/alex-krash)) +- Nouvelles fonctions d'agrégation `quantilesExactInclusive` et `quantilesExactExclusive` qui étaient proposés dans [\#5885](https://github.com/ClickHouse/ClickHouse/issues/5885). [\#6477](https://github.com/ClickHouse/ClickHouse/pull/6477) ([dimarub2000](https://github.com/dimarub2000)) - Fonction `bitmapRange(bitmap, range_begin, range_end)` qui renvoie un nouvel ensemble avec une plage spécifiée (ne pas inclure `range_end`). [\#6314](https://github.com/ClickHouse/ClickHouse/pull/6314) ([Zhichang Yu](https://github.com/yuzhichang)) - Fonction `geohashesInBox(longitude_min, latitude_min, longitude_max, latitude_max, precision)` ce qui crée un tableau de chaînes de précision longues de geohash-boîtes couvrant la zone fournie. [\#6127](https://github.com/ClickHouse/ClickHouse/pull/6127) ([Vasily Nemkov](https://github.com/Enmk)) - Implémenter la prise en charge de la requête INSERT avec `Kafka` table. [\#6012](https://github.com/ClickHouse/ClickHouse/pull/6012) ([Ivan](https://github.com/abyss7)) @@ -563,216 +563,216 @@ toc_title: '2019' #### Caractéristique Expérimentale {#experimental-feature-2} -- Format de données d’entrée et de sortie `Template`. Il permet de spécifier une chaîne de format personnalisée pour l’entrée et la sortie. [\#4354](https://github.com/ClickHouse/ClickHouse/issues/4354) [\#6727](https://github.com/ClickHouse/ClickHouse/pull/6727) ([tavplubix](https://github.com/tavplubix)) +- Format de données d'entrée et de sortie `Template`. Il permet de spécifier une chaîne de format personnalisée pour l'entrée et la sortie. [\#4354](https://github.com/ClickHouse/ClickHouse/issues/4354) [\#6727](https://github.com/ClickHouse/ClickHouse/pull/6727) ([tavplubix](https://github.com/tavplubix)) - La mise en œuvre de `LIVE VIEW` tableaux qui ont été initialement proposés dans [\#2898](https://github.com/ClickHouse/ClickHouse/pull/2898) préparés dans [\#3925](https://github.com/ClickHouse/ClickHouse/issues/3925), puis mis à jour dans [\#5541](https://github.com/ClickHouse/ClickHouse/issues/5541). Voir [\#5541](https://github.com/ClickHouse/ClickHouse/issues/5541) pour une description détaillée. [\#5541](https://github.com/ClickHouse/ClickHouse/issues/5541) ([vzakaznikov](https://github.com/vzakaznikov)) [\#6425](https://github.com/ClickHouse/ClickHouse/pull/6425) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) [\#6656](https://github.com/ClickHouse/ClickHouse/pull/6656) ([vzakaznikov](https://github.com/vzakaznikov)) Note que `LIVE VIEW` fonction peut être supprimée dans les prochaines versions. #### Bug Fix {#bug-fix-8} - Cette version contient également toutes les corrections de bugs de 19.13 et 19.11. -- Correction d’un défaut de segmentation lorsque la table a des indices de saut et que la fusion verticale se produit. [\#6723](https://github.com/ClickHouse/ClickHouse/pull/6723) ([alésapine](https://github.com/alesapin)) -- Correction de TTL par colonne avec des valeurs par défaut de colonne non triviales. Auparavant en cas de force TTL fusionner avec `OPTIMIZE ... FINAL` requête, les valeurs expirées ont été remplacées par des valeurs par défaut de type au lieu des valeurs par défaut de colonne spécifiées par l’utilisateur. [\#6796](https://github.com/ClickHouse/ClickHouse/pull/6796) ([Anton Popov](https://github.com/CurtizJ)) +- Correction d'un défaut de segmentation lorsque la table a des indices de saut et que la fusion verticale se produit. [\#6723](https://github.com/ClickHouse/ClickHouse/pull/6723) ([alésapine](https://github.com/alesapin)) +- Correction de TTL par colonne avec des valeurs par défaut de colonne non triviales. Auparavant en cas de force TTL fusionner avec `OPTIMIZE ... FINAL` requête, les valeurs expirées ont été remplacées par des valeurs par défaut de type au lieu des valeurs par défaut de colonne spécifiées par l'utilisateur. [\#6796](https://github.com/ClickHouse/ClickHouse/pull/6796) ([Anton Popov](https://github.com/CurtizJ)) - Correction du problème de duplication des messages Kafka lors du redémarrage normal du serveur. [\#6597](https://github.com/ClickHouse/ClickHouse/pull/6597) ([Ivan](https://github.com/abyss7)) -- Boucle infinie fixe lors de la lecture des messages Kafka. Ne pas mettre en pause/reprendre le consommateur sur l’abonnement du tout - sinon il peut être mis en pause indéfiniment dans certains scénarios. [\#6354](https://github.com/ClickHouse/ClickHouse/pull/6354) ([Ivan](https://github.com/abyss7)) +- Boucle infinie fixe lors de la lecture des messages Kafka. Ne pas mettre en pause/reprendre le consommateur sur l'abonnement du tout - sinon il peut être mis en pause indéfiniment dans certains scénarios. [\#6354](https://github.com/ClickHouse/ClickHouse/pull/6354) ([Ivan](https://github.com/abyss7)) - Fixer `Key expression contains comparison between inconvertible types` exception dans `bitmapContains` fonction. [\#6136](https://github.com/ClickHouse/ClickHouse/issues/6136) [\#6146](https://github.com/ClickHouse/ClickHouse/issues/6146) [\#6156](https://github.com/ClickHouse/ClickHouse/pull/6156) ([dimarub2000](https://github.com/dimarub2000)) - Correction de segfault avec activé `optimize_skip_unused_shards` et clé de sharding manquante. [\#6384](https://github.com/ClickHouse/ClickHouse/pull/6384) ([Anton Popov](https://github.com/CurtizJ)) -- Correction du mauvais code dans les mutations qui peuvent conduire à la corruption de la mémoire. Correction de segfault avec lecture de l’adresse `0x14c0` cela peut se produire en raison de simultané `DROP TABLE` et `SELECT` de `system.parts` ou `system.parts_columns`. Condition de course fixe dans la préparation des requêtes de mutation. Blocage fixe causé par `OPTIMIZE` des tables répliquées et des opérations de modification simultanées comme ALTERs. [\#6514](https://github.com/ClickHouse/ClickHouse/pull/6514) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Suppression de la journalisation supplémentaire dans L’interface MySQL [\#6389](https://github.com/ClickHouse/ClickHouse/pull/6389) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Renvoie la possibilité d’analyser les paramètres booléens depuis ‘true’ et ‘false’ dans le fichier de configuration. [\#6278](https://github.com/ClickHouse/ClickHouse/pull/6278) ([alésapine](https://github.com/alesapin)) -- Correction d’un crash dans l’ `quantile` et `median` la fonction sur `Nullable(Decimal128)`. [\#6378](https://github.com/ClickHouse/ClickHouse/pull/6378) ([Artem Zuikov](https://github.com/4ertus2)) +- Correction du mauvais code dans les mutations qui peuvent conduire à la corruption de la mémoire. Correction de segfault avec lecture de l'adresse `0x14c0` cela peut se produire en raison de simultané `DROP TABLE` et `SELECT` de `system.parts` ou `system.parts_columns`. Condition de course fixe dans la préparation des requêtes de mutation. Blocage fixe causé par `OPTIMIZE` des tables répliquées et des opérations de modification simultanées comme ALTERs. [\#6514](https://github.com/ClickHouse/ClickHouse/pull/6514) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Suppression de la journalisation supplémentaire dans L'interface MySQL [\#6389](https://github.com/ClickHouse/ClickHouse/pull/6389) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Renvoie la possibilité d'analyser les paramètres booléens depuis ‘true’ et ‘false’ dans le fichier de configuration. [\#6278](https://github.com/ClickHouse/ClickHouse/pull/6278) ([alésapine](https://github.com/alesapin)) +- Correction d'un crash dans l' `quantile` et `median` la fonction sur `Nullable(Decimal128)`. [\#6378](https://github.com/ClickHouse/ClickHouse/pull/6378) ([Artem Zuikov](https://github.com/4ertus2)) - Correction possible résultat incomplet retourné par `SELECT` requête avec `WHERE` condition sur la clé primaire contient la conversion en type Float. Il a été causé par une vérification incorrecte de la monotonie dans `toFloat` fonction. [\#6248](https://github.com/ClickHouse/ClickHouse/issues/6248) [\#6374](https://github.com/ClickHouse/ClickHouse/pull/6374) ([dimarub2000](https://github.com/dimarub2000)) -- Vérifier `max_expanded_ast_elements` réglage des mutations. Mutations claires après `TRUNCATE TABLE`. [\#6205](https://github.com/ClickHouse/ClickHouse/pull/6205) ([L’Hiver Zhang](https://github.com/zhang2014)) -- Correction des résultats de jointure pour les colonnes clés lorsqu’elles sont utilisées avec `join_use_nulls`. Attachez des valeurs NULL au lieu des valeurs par défaut des colonnes. [\#6249](https://github.com/ClickHouse/ClickHouse/pull/6249) ([Artem Zuikov](https://github.com/4ertus2)) +- Vérifier `max_expanded_ast_elements` réglage des mutations. Mutations claires après `TRUNCATE TABLE`. [\#6205](https://github.com/ClickHouse/ClickHouse/pull/6205) ([L'Hiver Zhang](https://github.com/zhang2014)) +- Correction des résultats de jointure pour les colonnes clés lorsqu'elles sont utilisées avec `join_use_nulls`. Attachez des valeurs NULL au lieu des valeurs par défaut des colonnes. [\#6249](https://github.com/ClickHouse/ClickHouse/pull/6249) ([Artem Zuikov](https://github.com/4ertus2)) - Correction des indices de saut avec Fusion verticale et modification. Correctif pour `Bad size of marks file` exception. [\#6594](https://github.com/ClickHouse/ClickHouse/issues/6594) [\#6713](https://github.com/ClickHouse/ClickHouse/pull/6713) ([alésapine](https://github.com/alesapin)) -- Correction plantage rare dans `ALTER MODIFY COLUMN` et fusion verticale lorsque l’une des parties fusionnées/modifiées est vide (0 lignes) [\#6746](https://github.com/ClickHouse/ClickHouse/issues/6746) [\#6780](https://github.com/ClickHouse/ClickHouse/pull/6780) ([alésapine](https://github.com/alesapin)) -- Correction d’un bug dans la conversion de `LowCardinality` types de `AggregateFunctionFactory`. Cela corrige [\#6257](https://github.com/ClickHouse/ClickHouse/issues/6257). [\#6281](https://github.com/ClickHouse/ClickHouse/pull/6281) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Correction d’un comportement incorrect et de segfaults possibles dans `topK` et `topKWeighted` agrégé fonctions. [\#6404](https://github.com/ClickHouse/ClickHouse/pull/6404) ([Anton Popov](https://github.com/CurtizJ)) +- Correction plantage rare dans `ALTER MODIFY COLUMN` et fusion verticale lorsque l'une des parties fusionnées/modifiées est vide (0 lignes) [\#6746](https://github.com/ClickHouse/ClickHouse/issues/6746) [\#6780](https://github.com/ClickHouse/ClickHouse/pull/6780) ([alésapine](https://github.com/alesapin)) +- Correction d'un bug dans la conversion de `LowCardinality` types de `AggregateFunctionFactory`. Cela corrige [\#6257](https://github.com/ClickHouse/ClickHouse/issues/6257). [\#6281](https://github.com/ClickHouse/ClickHouse/pull/6281) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Correction d'un comportement incorrect et de segfaults possibles dans `topK` et `topKWeighted` agrégé fonctions. [\#6404](https://github.com/ClickHouse/ClickHouse/pull/6404) ([Anton Popov](https://github.com/CurtizJ)) - Code dangereux fixe autour `getIdentifier` fonction. [\#6401](https://github.com/ClickHouse/ClickHouse/issues/6401) [\#6409](https://github.com/ClickHouse/ClickHouse/pull/6409) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’un bug dans le protocole de fil MySQL (est utilisé lors de la connexion à clickhouse forme client MySQL). Causé par un débordement de tampon de tas `PacketPayloadWriteBuffer`. [\#6212](https://github.com/ClickHouse/ClickHouse/pull/6212) ([Yuriy Baranov](https://github.com/yurriy)) +- Correction d'un bug dans le protocole de fil MySQL (est utilisé lors de la connexion à clickhouse forme client MySQL). Causé par un débordement de tampon de tas `PacketPayloadWriteBuffer`. [\#6212](https://github.com/ClickHouse/ClickHouse/pull/6212) ([Yuriy Baranov](https://github.com/yurriy)) - Fuite de mémoire fixe dans `bitmapSubsetInRange` fonction. [\#6819](https://github.com/ClickHouse/ClickHouse/pull/6819) ([Zhichang Yu](https://github.com/yuzhichang)) -- Correction d’un bug rare lorsque la mutation est exécutée après un changement de granularité. [\#6816](https://github.com/ClickHouse/ClickHouse/pull/6816) ([alésapine](https://github.com/alesapin)) +- Correction d'un bug rare lorsque la mutation est exécutée après un changement de granularité. [\#6816](https://github.com/ClickHouse/ClickHouse/pull/6816) ([alésapine](https://github.com/alesapin)) - Autoriser le message protobuf avec tous les champs par défaut. [\#6132](https://github.com/ClickHouse/ClickHouse/pull/6132) ([Vitaly Baranov](https://github.com/vitlibar)) -- Résoudre un bug avec `nullIf` fonction lorsque nous envoyer un `NULL` l’argument sur le deuxième argument. [\#6446](https://github.com/ClickHouse/ClickHouse/pull/6446) ([Guillaume Tassery](https://github.com/YiuRULE)) -- Correction d’un bug rare avec une mauvaise allocation/désallocation de la mémoire dans des dictionnaires de cache de clés Complexes avec des champs de chaîne qui conduit à une consommation de mémoire infinie (ressemble à une fuite de mémoire). Bug se reproduit lorsque la taille de la chaîne était une puissance de deux à partir de huit (8, 16, 32, etc.). [\#6447](https://github.com/ClickHouse/ClickHouse/pull/6447) ([alésapine](https://github.com/alesapin)) -- Correction de L’encodage Gorilla sur les petites séquences qui a provoqué une exception `Cannot write after end of buffer`. [\#6398](https://github.com/ClickHouse/ClickHouse/issues/6398) [\#6444](https://github.com/ClickHouse/ClickHouse/pull/6444) ([Vasily Nemkov](https://github.com/Enmk)) -- Permet d’utiliser des types Non nullables dans les jointures avec `join_use_nulls` permettre. [\#6705](https://github.com/ClickHouse/ClickHouse/pull/6705) ([Artem Zuikov](https://github.com/4ertus2)) +- Résoudre un bug avec `nullIf` fonction lorsque nous envoyer un `NULL` l'argument sur le deuxième argument. [\#6446](https://github.com/ClickHouse/ClickHouse/pull/6446) ([Guillaume Tassery](https://github.com/YiuRULE)) +- Correction d'un bug rare avec une mauvaise allocation/désallocation de la mémoire dans des dictionnaires de cache de clés Complexes avec des champs de chaîne qui conduit à une consommation de mémoire infinie (ressemble à une fuite de mémoire). Bug se reproduit lorsque la taille de la chaîne était une puissance de deux à partir de huit (8, 16, 32, etc.). [\#6447](https://github.com/ClickHouse/ClickHouse/pull/6447) ([alésapine](https://github.com/alesapin)) +- Correction de L'encodage Gorilla sur les petites séquences qui a provoqué une exception `Cannot write after end of buffer`. [\#6398](https://github.com/ClickHouse/ClickHouse/issues/6398) [\#6444](https://github.com/ClickHouse/ClickHouse/pull/6444) ([Vasily Nemkov](https://github.com/Enmk)) +- Permet d'utiliser des types Non nullables dans les jointures avec `join_use_nulls` permettre. [\#6705](https://github.com/ClickHouse/ClickHouse/pull/6705) ([Artem Zuikov](https://github.com/4ertus2)) - Désactiver `Poco::AbstractConfiguration` substitutions dans la requête dans `clickhouse-client`. [\#6706](https://github.com/ClickHouse/ClickHouse/pull/6706) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Éviter l’impasse dans `REPLACE PARTITION`. [\#6677](https://github.com/ClickHouse/ClickHouse/pull/6677) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Éviter l'impasse dans `REPLACE PARTITION`. [\#6677](https://github.com/ClickHouse/ClickHouse/pull/6677) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Utiliser `arrayReduce` pour des arguments constants peuvent conduire à segfault. [\#6242](https://github.com/ClickHouse/ClickHouse/issues/6242) [\#6326](https://github.com/ClickHouse/ClickHouse/pull/6326) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Correction des parties incohérentes qui peuvent apparaître si la réplique a été restaurée après `DROP PARTITION`. [\#6522](https://github.com/ClickHouse/ClickHouse/issues/6522) [\#6523](https://github.com/ClickHouse/ClickHouse/pull/6523) ([tavplubix](https://github.com/tavplubix)) - Correction du blocage dans `JSONExtractRaw` fonction. [\#6195](https://github.com/ClickHouse/ClickHouse/issues/6195) [\#6198](https://github.com/ClickHouse/ClickHouse/pull/6198) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’un bug avec des indices de saut incorrects sérialisation et agrégation avec granularité adaptative. [\#6594](https://github.com/ClickHouse/ClickHouse/issues/6594). [\#6748](https://github.com/ClickHouse/ClickHouse/pull/6748) ([alésapine](https://github.com/alesapin)) +- Correction d'un bug avec des indices de saut incorrects sérialisation et agrégation avec granularité adaptative. [\#6594](https://github.com/ClickHouse/ClickHouse/issues/6594). [\#6748](https://github.com/ClickHouse/ClickHouse/pull/6748) ([alésapine](https://github.com/alesapin)) - Fixer `WITH ROLLUP` et `WITH CUBE` les modificateurs de `GROUP BY` avec agrégation à deux niveaux. [\#6225](https://github.com/ClickHouse/ClickHouse/pull/6225) ([Anton Popov](https://github.com/CurtizJ)) -- Correction d’un bug avec l’écriture de marques d’indices secondaires avec une granularité adaptative. [\#6126](https://github.com/ClickHouse/ClickHouse/pull/6126) ([alésapine](https://github.com/alesapin)) -- Correction de l’ordre d’initialisation lors du démarrage du serveur. Depuis `StorageMergeTree::background_task_handle` est initialisée dans `startup()` le `MergeTreeBlockOutputStream::write()` peut tenter de l’utiliser avant l’initialisation. Vérifiez simplement s’il est initialisé. [\#6080](https://github.com/ClickHouse/ClickHouse/pull/6080) ([Ivan](https://github.com/abyss7)) -- Effacement du tampon de données de l’opération de lecture précédente terminée par une erreur. [\#6026](https://github.com/ClickHouse/ClickHouse/pull/6026) ([Nikolay](https://github.com/bopohaa)) -- Correction d’un bug avec l’activation de la granularité adaptative lors de la création d’une nouvelle réplique pour la table répliquée\*MergeTree. [\#6394](https://github.com/ClickHouse/ClickHouse/issues/6394) [\#6452](https://github.com/ClickHouse/ClickHouse/pull/6452) ([alésapine](https://github.com/alesapin)) -- Correction d’un crash possible lors du démarrage du serveur en cas d’exception `libunwind` au cours de l’exception à l’accès à uninitialized `ThreadStatus` structure. [\#6456](https://github.com/ClickHouse/ClickHouse/pull/6456) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- Correction d’un crash dans l’ `yandexConsistentHash` fonction. Trouvé par fuzz test. [\#6304](https://github.com/ClickHouse/ClickHouse/issues/6304) [\#6305](https://github.com/ClickHouse/ClickHouse/pull/6305) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction d'un bug avec l'écriture de marques d'indices secondaires avec une granularité adaptative. [\#6126](https://github.com/ClickHouse/ClickHouse/pull/6126) ([alésapine](https://github.com/alesapin)) +- Correction de l'ordre d'initialisation lors du démarrage du serveur. Depuis `StorageMergeTree::background_task_handle` est initialisée dans `startup()` le `MergeTreeBlockOutputStream::write()` peut tenter de l'utiliser avant l'initialisation. Vérifiez simplement s'il est initialisé. [\#6080](https://github.com/ClickHouse/ClickHouse/pull/6080) ([Ivan](https://github.com/abyss7)) +- Effacement du tampon de données de l'opération de lecture précédente terminée par une erreur. [\#6026](https://github.com/ClickHouse/ClickHouse/pull/6026) ([Nikolay](https://github.com/bopohaa)) +- Correction d'un bug avec l'activation de la granularité adaptative lors de la création d'une nouvelle réplique pour la table répliquée\*MergeTree. [\#6394](https://github.com/ClickHouse/ClickHouse/issues/6394) [\#6452](https://github.com/ClickHouse/ClickHouse/pull/6452) ([alésapine](https://github.com/alesapin)) +- Correction d'un crash possible lors du démarrage du serveur en cas d'exception `libunwind` au cours de l'exception à l'accès à uninitialized `ThreadStatus` structure. [\#6456](https://github.com/ClickHouse/ClickHouse/pull/6456) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +- Correction d'un crash dans l' `yandexConsistentHash` fonction. Trouvé par fuzz test. [\#6304](https://github.com/ClickHouse/ClickHouse/issues/6304) [\#6305](https://github.com/ClickHouse/ClickHouse/pull/6305) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Correction de la possibilité de suspendre les requêtes lorsque le serveur est surchargé et que le pool de threads global devient presque complet. Cela a plus de chances de se produire sur les clusters avec un grand nombre de fragments (des centaines), car les requêtes distribuées allouent un thread par connexion à chaque fragment. Par exemple, ce problème peut se reproduire si un cluster de 330 fragments traite 30 requêtes distribuées simultanées. Ce problème affecte toutes les versions à partir de 19.2. [\#6301](https://github.com/ClickHouse/ClickHouse/pull/6301) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Fixe la logique de `arrayEnumerateUniqRanked` fonction. [\#6423](https://github.com/ClickHouse/ClickHouse/pull/6423) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Correction de segfault lors du décodage de la table des symboles. [\#6603](https://github.com/ClickHouse/ClickHouse/pull/6603) ([Amos Oiseau](https://github.com/amosbird)) -- Correction d’une exception non pertinente dans la distribution de `LowCardinality(Nullable)` to not-Nullable column in case if it doesn’t contain Nulls (e.g. in query like `SELECT CAST(CAST('Hello' AS LowCardinality(Nullable(String))) AS String)`. [\#6094](https://github.com/ClickHouse/ClickHouse/issues/6094) [\#6119](https://github.com/ClickHouse/ClickHouse/pull/6119) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Correction d'une exception non pertinente dans la distribution de `LowCardinality(Nullable)` to not-Nullable column in case if it doesn't contain Nulls (e.g. in query like `SELECT CAST(CAST('Hello' AS LowCardinality(Nullable(String))) AS String)`. [\#6094](https://github.com/ClickHouse/ClickHouse/issues/6094) [\#6119](https://github.com/ClickHouse/ClickHouse/pull/6119) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) - Suppression de guillemets supplémentaires de description dans `system.settings` table. [\#6696](https://github.com/ClickHouse/ClickHouse/issues/6696) [\#6699](https://github.com/ClickHouse/ClickHouse/pull/6699) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Éviter l’impasse possible dans `TRUNCATE` de table Répliquée. [\#6695](https://github.com/ClickHouse/ClickHouse/pull/6695) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction de la lecture dans l’ordre de la clé de tri. [\#6189](https://github.com/ClickHouse/ClickHouse/pull/6189) ([Anton Popov](https://github.com/CurtizJ)) +- Éviter l'impasse possible dans `TRUNCATE` de table Répliquée. [\#6695](https://github.com/ClickHouse/ClickHouse/pull/6695) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction de la lecture dans l'ordre de la clé de tri. [\#6189](https://github.com/ClickHouse/ClickHouse/pull/6189) ([Anton Popov](https://github.com/CurtizJ)) - Fixer `ALTER TABLE ... UPDATE` requête pour les tables avec `enable_mixed_granularity_parts=1`. [\#6543](https://github.com/ClickHouse/ClickHouse/pull/6543) ([alésapine](https://github.com/alesapin)) -- Correction d’un bug ouvert par [\#4405](https://github.com/ClickHouse/ClickHouse/pull/4405) (depuis 19.4.0). Reproduit dans les requêtes aux tables distribuées sur les tables MergeTree lorsque nous n’interrogeons aucune colonne (`SELECT 1`). [\#6236](https://github.com/ClickHouse/ClickHouse/pull/6236) ([alésapine](https://github.com/alesapin)) -- Dépassement fixe dans la division entière du type signé au type non signé. Le comportement était exactement comme dans le langage C ou c++ (règles de promotion entières) qui peut être surprenant. Veuillez noter que le débordement est toujours possible lors de la division d’un grand nombre signé en un grand nombre non signé ou vice-versa (mais ce cas est moins habituel). Le problème existait dans toutes les versions du serveur. [\#6214](https://github.com/ClickHouse/ClickHouse/issues/6214) [\#6233](https://github.com/ClickHouse/ClickHouse/pull/6233) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Limiter le temps de sommeil maximal pour l’étranglement lorsque `max_execution_speed` ou `max_execution_speed_bytes` est définie. Correction de fausses erreurs comme `Estimated query execution time (inf seconds) is too long`. [\#5547](https://github.com/ClickHouse/ClickHouse/issues/5547) [\#6232](https://github.com/ClickHouse/ClickHouse/pull/6232) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction de problèmes d’utilisation `MATERIALIZED` colonnes et alias dans `MaterializedView`. [\#448](https://github.com/ClickHouse/ClickHouse/issues/448) [\#3484](https://github.com/ClickHouse/ClickHouse/issues/3484) [\#3450](https://github.com/ClickHouse/ClickHouse/issues/3450) [\#2878](https://github.com/ClickHouse/ClickHouse/issues/2878) [\#2285](https://github.com/ClickHouse/ClickHouse/issues/2285) [\#3796](https://github.com/ClickHouse/ClickHouse/pull/3796) ([Amos Oiseau](https://github.com/amosbird)) [\#6316](https://github.com/ClickHouse/ClickHouse/pull/6316) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Fixer `FormatFactory` comportement pour les flux d’entrée qui ne sont pas implémentés en tant que Processeur. [\#6495](https://github.com/ClickHouse/ClickHouse/pull/6495) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Correction d’une faute. [\#6631](https://github.com/ClickHouse/ClickHouse/pull/6631) ([Alex Ryndin](https://github.com/alexryndin)) -- Typo dans le message d’erreur (is - \> are). [\#6839](https://github.com/ClickHouse/ClickHouse/pull/6839) ([Denis Zhuravlev](https://github.com/den-crane)) -- Correction d’une erreur lors de l’analyse de la liste des colonnes de la chaîne si le type contenait une virgule (Ce problème était pertinent pour `File`, `URL`, `HDFS` stockage) [\#6217](https://github.com/ClickHouse/ClickHouse/issues/6217). [\#6209](https://github.com/ClickHouse/ClickHouse/pull/6209) ([dimarub2000](https://github.com/dimarub2000)) +- Correction d'un bug ouvert par [\#4405](https://github.com/ClickHouse/ClickHouse/pull/4405) (depuis 19.4.0). Reproduit dans les requêtes aux tables distribuées sur les tables MergeTree lorsque nous n'interrogeons aucune colonne (`SELECT 1`). [\#6236](https://github.com/ClickHouse/ClickHouse/pull/6236) ([alésapine](https://github.com/alesapin)) +- Dépassement fixe dans la division entière du type signé au type non signé. Le comportement était exactement comme dans le langage C ou c++ (règles de promotion entières) qui peut être surprenant. Veuillez noter que le débordement est toujours possible lors de la division d'un grand nombre signé en un grand nombre non signé ou vice-versa (mais ce cas est moins habituel). Le problème existait dans toutes les versions du serveur. [\#6214](https://github.com/ClickHouse/ClickHouse/issues/6214) [\#6233](https://github.com/ClickHouse/ClickHouse/pull/6233) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Limiter le temps de sommeil maximal pour l'étranglement lorsque `max_execution_speed` ou `max_execution_speed_bytes` est définie. Correction de fausses erreurs comme `Estimated query execution time (inf seconds) is too long`. [\#5547](https://github.com/ClickHouse/ClickHouse/issues/5547) [\#6232](https://github.com/ClickHouse/ClickHouse/pull/6232) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction de problèmes d'utilisation `MATERIALIZED` colonnes et alias dans `MaterializedView`. [\#448](https://github.com/ClickHouse/ClickHouse/issues/448) [\#3484](https://github.com/ClickHouse/ClickHouse/issues/3484) [\#3450](https://github.com/ClickHouse/ClickHouse/issues/3450) [\#2878](https://github.com/ClickHouse/ClickHouse/issues/2878) [\#2285](https://github.com/ClickHouse/ClickHouse/issues/2285) [\#3796](https://github.com/ClickHouse/ClickHouse/pull/3796) ([Amos Oiseau](https://github.com/amosbird)) [\#6316](https://github.com/ClickHouse/ClickHouse/pull/6316) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Fixer `FormatFactory` comportement pour les flux d'entrée qui ne sont pas implémentés en tant que Processeur. [\#6495](https://github.com/ClickHouse/ClickHouse/pull/6495) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Correction d'une faute. [\#6631](https://github.com/ClickHouse/ClickHouse/pull/6631) ([Alex Ryndin](https://github.com/alexryndin)) +- Typo dans le message d'erreur (is - \> are). [\#6839](https://github.com/ClickHouse/ClickHouse/pull/6839) ([Denis Zhuravlev](https://github.com/den-crane)) +- Correction d'une erreur lors de l'analyse de la liste des colonnes de la chaîne si le type contenait une virgule (Ce problème était pertinent pour `File`, `URL`, `HDFS` stockage) [\#6217](https://github.com/ClickHouse/ClickHouse/issues/6217). [\#6209](https://github.com/ClickHouse/ClickHouse/pull/6209) ([dimarub2000](https://github.com/dimarub2000)) #### Correction De Sécurité {#security-fix} - Cette version contient également tous les correctifs de sécurité de bugs de 19.13 et 19.11. -- Correction de la possibilité d’une requête fabriquée pour provoquer un crash du serveur en raison d’un débordement de pile dans L’analyseur SQL. Correction de la possibilité de débordement de pile dans les tables de fusion et distribuées, les vues matérialisées et les conditions de sécurité au niveau des lignes impliquant des sous-requêtes. [\#6433](https://github.com/ClickHouse/ClickHouse/pull/6433) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction de la possibilité d'une requête fabriquée pour provoquer un crash du serveur en raison d'un débordement de pile dans L'analyseur SQL. Correction de la possibilité de débordement de pile dans les tables de fusion et distribuées, les vues matérialisées et les conditions de sécurité au niveau des lignes impliquant des sous-requêtes. [\#6433](https://github.com/ClickHouse/ClickHouse/pull/6433) ([alexeï-milovidov](https://github.com/alexey-milovidov)) #### Amélioration {#improvement-3} - Mise en œuvre correcte de la logique ternaire pour `AND/OR`. [\#6048](https://github.com/ClickHouse/ClickHouse/pull/6048) ([Alexander Kazakov](https://github.com/Akazz)) -- Maintenant les valeurs et les lignes avec TTL expiré seront supprimées après `OPTIMIZE ... FINAL` query from old parts without TTL infos or with outdated TTL infos, e.g. after `ALTER ... MODIFY TTL` requête. Ajouté requêtes `SYSTEM STOP/START TTL MERGES` pour interdire / autoriser les fusions d’affectation avec TTL et filtrer les valeurs expirées dans toutes les fusions. [\#6274](https://github.com/ClickHouse/ClickHouse/pull/6274) ([Anton Popov](https://github.com/CurtizJ)) -- Possibilité de changer l’emplacement du fichier d’historique ClickHouse pour le client à l’aide `CLICKHOUSE_HISTORY_FILE` env. [\#6840](https://github.com/ClickHouse/ClickHouse/pull/6840) ([filimonov](https://github.com/filimonov)) +- Maintenant les valeurs et les lignes avec TTL expiré seront supprimées après `OPTIMIZE ... FINAL` query from old parts without TTL infos or with outdated TTL infos, e.g. after `ALTER ... MODIFY TTL` requête. Ajouté requêtes `SYSTEM STOP/START TTL MERGES` pour interdire / autoriser les fusions d'affectation avec TTL et filtrer les valeurs expirées dans toutes les fusions. [\#6274](https://github.com/ClickHouse/ClickHouse/pull/6274) ([Anton Popov](https://github.com/CurtizJ)) +- Possibilité de changer l'emplacement du fichier d'historique ClickHouse pour le client à l'aide `CLICKHOUSE_HISTORY_FILE` env. [\#6840](https://github.com/ClickHouse/ClickHouse/pull/6840) ([filimonov](https://github.com/filimonov)) - Supprimer `dry_run` drapeau de `InterpreterSelectQuery`. … [\#6375](https://github.com/ClickHouse/ClickHouse/pull/6375) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) - Soutien `ASOF JOIN` avec `ON` section. [\#6211](https://github.com/ClickHouse/ClickHouse/pull/6211) ([Artem Zuikov](https://github.com/4ertus2)) - Meilleure prise en charge des index de saut pour les mutations et la réplication. Soutien pour `MATERIALIZE/CLEAR INDEX ... IN PARTITION` requête. `UPDATE x = x` recalcule tous les indices qui utilisent la colonne `x`. [\#5053](https://github.com/ClickHouse/ClickHouse/pull/5053) ([Nikita Vasilev](https://github.com/nikvas0)) - Permettre à `ATTACH` affichages en direct (par exemple, au démarrage du serveur), indépendamment de `allow_experimental_live_view` paramètre. [\#6754](https://github.com/ClickHouse/ClickHouse/pull/6754) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Pour les traces de pile recueillies par le profileur de requête, n’incluez pas les trames de pile générées par le profileur de requête lui-même. [\#6250](https://github.com/ClickHouse/ClickHouse/pull/6250) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Pour les traces de pile recueillies par le profileur de requête, n'incluez pas les trames de pile générées par le profileur de requête lui-même. [\#6250](https://github.com/ClickHouse/ClickHouse/pull/6250) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Maintenant fonctions de table `values`, `file`, `url`, `hdfs` avoir un support pour les colonnes ALIAS. [\#6255](https://github.com/ClickHouse/ClickHouse/pull/6255) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Lancer une exception si `config.d` le fichier n’a pas l’élément racine correspondant comme fichier de configuration. [\#6123](https://github.com/ClickHouse/ClickHouse/pull/6123) ([dimarub2000](https://github.com/dimarub2000)) -- Imprimer des informations supplémentaires dans le message d’exception pour `no space left on device`. [\#6182](https://github.com/ClickHouse/ClickHouse/issues/6182), [\#6252](https://github.com/ClickHouse/ClickHouse/issues/6252) [\#6352](https://github.com/ClickHouse/ClickHouse/pull/6352) ([tavplubix](https://github.com/tavplubix)) -- Lors de la détermination des éclats d’un `Distributed` table à couvrir par une requête de lecture (pour `optimize_skip_unused_shards` = 1) ClickHouse vérifie maintenant les conditions des deux `prewhere` et `where` clauses de l’instruction select. [\#6521](https://github.com/ClickHouse/ClickHouse/pull/6521) ([Alexander Kazakov](https://github.com/Akazz)) -- Permettre `SIMDJSON` pour les machines sans AVX2 mais avec SSE 4.2 et pclmul jeu d’instructions. [\#6285](https://github.com/ClickHouse/ClickHouse/issues/6285) [\#6320](https://github.com/ClickHouse/ClickHouse/pull/6320) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Lancer une exception si `config.d` le fichier n'a pas l'élément racine correspondant comme fichier de configuration. [\#6123](https://github.com/ClickHouse/ClickHouse/pull/6123) ([dimarub2000](https://github.com/dimarub2000)) +- Imprimer des informations supplémentaires dans le message d'exception pour `no space left on device`. [\#6182](https://github.com/ClickHouse/ClickHouse/issues/6182), [\#6252](https://github.com/ClickHouse/ClickHouse/issues/6252) [\#6352](https://github.com/ClickHouse/ClickHouse/pull/6352) ([tavplubix](https://github.com/tavplubix)) +- Lors de la détermination des éclats d'un `Distributed` table à couvrir par une requête de lecture (pour `optimize_skip_unused_shards` = 1) ClickHouse vérifie maintenant les conditions des deux `prewhere` et `where` clauses de l'instruction select. [\#6521](https://github.com/ClickHouse/ClickHouse/pull/6521) ([Alexander Kazakov](https://github.com/Akazz)) +- Permettre `SIMDJSON` pour les machines sans AVX2 mais avec SSE 4.2 et pclmul jeu d'instructions. [\#6285](https://github.com/ClickHouse/ClickHouse/issues/6285) [\#6320](https://github.com/ClickHouse/ClickHouse/pull/6320) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - ClickHouse peut fonctionner sur les systèmes de fichiers sans `O_DIRECT` soutien (tels que ZFS et BtrFS) sans réglage supplémentaire. [\#4449](https://github.com/ClickHouse/ClickHouse/issues/4449) [\#6730](https://github.com/ClickHouse/ClickHouse/pull/6730) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Soutien pousser vers le bas prédicat pour la sous-requête finale. [\#6120](https://github.com/ClickHouse/ClickHouse/pull/6120) ([TCeason](https://github.com/TCeason)) [\#6162](https://github.com/ClickHouse/ClickHouse/pull/6162) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Mieux `JOIN ON` extraction des clés [\#6131](https://github.com/ClickHouse/ClickHouse/pull/6131) ([Artem Zuikov](https://github.com/4ertus2)) - Mise à jour `SIMDJSON`. [\#6285](https://github.com/ClickHouse/ClickHouse/issues/6285). [\#6306](https://github.com/ClickHouse/ClickHouse/pull/6306) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Optimiser la sélection de la plus petite colonne pour `SELECT count()` requête. [\#6344](https://github.com/ClickHouse/ClickHouse/pull/6344) ([Amos Oiseau](https://github.com/amosbird)) -- Ajouter `strict` paramètre `windowFunnel()`. Lorsque l’ `strict` est définie, le `windowFunnel()` applique des conditions uniquement pour les valeurs uniques. [\#6548](https://github.com/ClickHouse/ClickHouse/pull/6548) ([achimbab](https://github.com/achimbab)) +- Ajouter `strict` paramètre `windowFunnel()`. Lorsque l' `strict` est définie, le `windowFunnel()` applique des conditions uniquement pour les valeurs uniques. [\#6548](https://github.com/ClickHouse/ClickHouse/pull/6548) ([achimbab](https://github.com/achimbab)) - Interface plus sûre de `mysqlxx::Pool`. [\#6150](https://github.com/ClickHouse/ClickHouse/pull/6150) ([avasiliev](https://github.com/avasiliev)) -- Options taille de ligne lors de l’exécution avec `--help` l’option correspond maintenant à la taille du terminal. [\#6590](https://github.com/ClickHouse/ClickHouse/pull/6590) ([dimarub2000](https://github.com/dimarub2000)) -- Désactiver “read in order” optimisation pour l’agrégation, sans touches. [\#6599](https://github.com/ClickHouse/ClickHouse/pull/6599) ([Anton Popov](https://github.com/CurtizJ)) -- Code D’état HTTP pour `INCORRECT_DATA` et `TYPE_MISMATCH` les codes d’erreur ont été modifiés par défaut `500 Internal Server Error` de `400 Bad Request`. [\#6271](https://github.com/ClickHouse/ClickHouse/pull/6271) ([Alexander Rodin](https://github.com/a-rodin)) +- Options taille de ligne lors de l'exécution avec `--help` l'option correspond maintenant à la taille du terminal. [\#6590](https://github.com/ClickHouse/ClickHouse/pull/6590) ([dimarub2000](https://github.com/dimarub2000)) +- Désactiver “read in order” optimisation pour l'agrégation, sans touches. [\#6599](https://github.com/ClickHouse/ClickHouse/pull/6599) ([Anton Popov](https://github.com/CurtizJ)) +- Code D'état HTTP pour `INCORRECT_DATA` et `TYPE_MISMATCH` les codes d'erreur ont été modifiés par défaut `500 Internal Server Error` de `400 Bad Request`. [\#6271](https://github.com/ClickHouse/ClickHouse/pull/6271) ([Alexander Rodin](https://github.com/a-rodin)) - Déplacer Rejoindre objet de `ExpressionAction` dans `AnalyzedJoin`. `ExpressionAnalyzer` et `ExpressionAction` ne sais pas à propos de `Join` classe de plus. Sa logique est cachée par `AnalyzedJoin` iface. [\#6801](https://github.com/ClickHouse/ClickHouse/pull/6801) ([Artem Zuikov](https://github.com/4ertus2)) -- Correction d’un blocage possible des requêtes distribuées lorsque l’un des fragments est localhost mais que la requête est envoyée via une connexion réseau. [\#6759](https://github.com/ClickHouse/ClickHouse/pull/6759) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction d'un blocage possible des requêtes distribuées lorsque l'un des fragments est localhost mais que la requête est envoyée via une connexion réseau. [\#6759](https://github.com/ClickHouse/ClickHouse/pull/6759) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Sémantique modifiée de plusieurs tables `RENAME` pour éviter les blocages possibles. [\#6757](https://github.com/ClickHouse/ClickHouse/issues/6757). [\#6756](https://github.com/ClickHouse/ClickHouse/pull/6756) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Serveur de compatibilité MySQL réécrit pour empêcher le chargement de la charge utile de paquet complet en mémoire. Diminution de la consommation de mémoire pour chaque connexion à environ `2 * DBMS_DEFAULT_BUFFER_SIZE` (tampons de lecture/écriture). [\#5811](https://github.com/ClickHouse/ClickHouse/pull/5811) ([Yuriy Baranov](https://github.com/yurriy)) -- Déplacez l’alias AST interprétant la logique hors de l’analyseur qui n’a rien à savoir sur la sémantique des requêtes. [\#6108](https://github.com/ClickHouse/ClickHouse/pull/6108) ([Artem Zuikov](https://github.com/4ertus2)) +- Déplacez l'alias AST interprétant la logique hors de l'analyseur qui n'a rien à savoir sur la sémantique des requêtes. [\#6108](https://github.com/ClickHouse/ClickHouse/pull/6108) ([Artem Zuikov](https://github.com/4ertus2)) - Analyse légèrement plus sûre de `NamesAndTypesList`. [\#6408](https://github.com/ClickHouse/ClickHouse/issues/6408). [\#6410](https://github.com/ClickHouse/ClickHouse/pull/6410) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- `clickhouse-copier`: Permet d’utiliser les `where_condition` de config avec `partition_key` alias dans la requête pour vérifier l’existence de la partition (auparavant, il était utilisé uniquement dans la lecture des requêtes de données). [\#6577](https://github.com/ClickHouse/ClickHouse/pull/6577) ([proller](https://github.com/proller)) -- Ajout d’un argument de message Facultatif dans `throwIf`. ([\#5772](https://github.com/ClickHouse/ClickHouse/issues/5772)) [\#6329](https://github.com/ClickHouse/ClickHouse/pull/6329) ([Vdimir](https://github.com/Vdimir)) -- L’exception du serveur obtenue lors de l’envoi des données d’insertion est également en cours de traitement dans le client. [\#5891](https://github.com/ClickHouse/ClickHouse/issues/5891) [\#6711](https://github.com/ClickHouse/ClickHouse/pull/6711) ([dimarub2000](https://github.com/dimarub2000)) -- Ajout d’une métrique `DistributedFilesToInsert` cela montre le nombre total de fichiers dans le système de fichiers qui sont sélectionnés pour envoyer aux serveurs distants par des tables distribuées. Le nombre est additionné à travers tous les fragments. [\#6600](https://github.com/ClickHouse/ClickHouse/pull/6600) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- `clickhouse-copier`: Permet d'utiliser les `where_condition` de config avec `partition_key` alias dans la requête pour vérifier l'existence de la partition (auparavant, il était utilisé uniquement dans la lecture des requêtes de données). [\#6577](https://github.com/ClickHouse/ClickHouse/pull/6577) ([proller](https://github.com/proller)) +- Ajout d'un argument de message Facultatif dans `throwIf`. ([\#5772](https://github.com/ClickHouse/ClickHouse/issues/5772)) [\#6329](https://github.com/ClickHouse/ClickHouse/pull/6329) ([Vdimir](https://github.com/Vdimir)) +- L'exception du serveur obtenue lors de l'envoi des données d'insertion est également en cours de traitement dans le client. [\#5891](https://github.com/ClickHouse/ClickHouse/issues/5891) [\#6711](https://github.com/ClickHouse/ClickHouse/pull/6711) ([dimarub2000](https://github.com/dimarub2000)) +- Ajout d'une métrique `DistributedFilesToInsert` cela montre le nombre total de fichiers dans le système de fichiers qui sont sélectionnés pour envoyer aux serveurs distants par des tables distribuées. Le nombre est additionné à travers tous les fragments. [\#6600](https://github.com/ClickHouse/ClickHouse/pull/6600) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Déplacer la plupart des jointures préparer la logique de `ExpressionAction/ExpressionAnalyzer` de `AnalyzedJoin`. [\#6785](https://github.com/ClickHouse/ClickHouse/pull/6785) ([Artem Zuikov](https://github.com/4ertus2)) - Fix TSan [avertissement](https://clickhouse-test-reports.s3.yandex.net/6399/c1c1d1daa98e199e620766f1bd06a5921050a00d/functional_stateful_tests_(thread).html) ‘lock-order-inversion’. [\#6740](https://github.com/ClickHouse/ClickHouse/pull/6740) ([Vasily Nemkov](https://github.com/Enmk)) -- De meilleurs messages d’information sur le manque de capacités Linux. Journalisation des erreurs fatales avec “fatal” niveau, cela le rendra plus facile à trouver dans `system.text_log`. [\#6441](https://github.com/ClickHouse/ClickHouse/pull/6441) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Lorsque activer le dumping des données temporaires sur le disque pour limiter l’utilisation de la mémoire pendant `GROUP BY`, `ORDER BY` il n’a pas vérifier l’espace disque libre. Le correctif ajoute un nouveau paramètre `min_free_disk_space`, lorsque l’espace disque libre est plus petit que le seuil, la requête s’arrêtera et lancera `ErrorCodes::NOT_ENOUGH_SPACE`. [\#6678](https://github.com/ClickHouse/ClickHouse/pull/6678) ([Weiqing Xu](https://github.com/weiqxu)) [\#6691](https://github.com/ClickHouse/ClickHouse/pull/6691) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Supprimé rwlock récursif par thread. Cela n’a aucun sens, car les threads sont réutilisés entre les requêtes. `SELECT` la requête peut acquérir un verrou dans un thread, tenir un verrou d’un autre thread et quitter le premier thread. Dans le même temps, le premier fil peut être réutilisé par `DROP` requête. Cela mènera à la faux “Attempt to acquire exclusive lock recursively” message. [\#6771](https://github.com/ClickHouse/ClickHouse/pull/6771) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- De meilleurs messages d'information sur le manque de capacités Linux. Journalisation des erreurs fatales avec “fatal” niveau, cela le rendra plus facile à trouver dans `system.text_log`. [\#6441](https://github.com/ClickHouse/ClickHouse/pull/6441) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Lorsque activer le dumping des données temporaires sur le disque pour limiter l'utilisation de la mémoire pendant `GROUP BY`, `ORDER BY` il n'a pas vérifier l'espace disque libre. Le correctif ajoute un nouveau paramètre `min_free_disk_space`, lorsque l'espace disque libre est plus petit que le seuil, la requête s'arrêtera et lancera `ErrorCodes::NOT_ENOUGH_SPACE`. [\#6678](https://github.com/ClickHouse/ClickHouse/pull/6678) ([Weiqing Xu](https://github.com/weiqxu)) [\#6691](https://github.com/ClickHouse/ClickHouse/pull/6691) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Supprimé rwlock récursif par thread. Cela n'a aucun sens, car les threads sont réutilisés entre les requêtes. `SELECT` la requête peut acquérir un verrou dans un thread, tenir un verrou d'un autre thread et quitter le premier thread. Dans le même temps, le premier fil peut être réutilisé par `DROP` requête. Cela mènera à la faux “Attempt to acquire exclusive lock recursively” message. [\#6771](https://github.com/ClickHouse/ClickHouse/pull/6771) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Split `ExpressionAnalyzer.appendJoin()`. Préparer une place dans `ExpressionAnalyzer` pour `MergeJoin`. [\#6524](https://github.com/ClickHouse/ClickHouse/pull/6524) ([Artem Zuikov](https://github.com/4ertus2)) -- Ajouter `mysql_native_password` plugin d’authentification au serveur de compatibilité MySQL. [\#6194](https://github.com/ClickHouse/ClickHouse/pull/6194) ([Yuriy Baranov](https://github.com/yurriy)) +- Ajouter `mysql_native_password` plugin d'authentification au serveur de compatibilité MySQL. [\#6194](https://github.com/ClickHouse/ClickHouse/pull/6194) ([Yuriy Baranov](https://github.com/yurriy)) - Un moins grand nombre de `clock_gettime` appels; correction de la compatibilité ABI entre debug / release in `Allocator` (question insignifiante). [\#6197](https://github.com/ClickHouse/ClickHouse/pull/6197) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Déplacer `collectUsedColumns` de `ExpressionAnalyzer` de `SyntaxAnalyzer`. `SyntaxAnalyzer` faire `required_source_columns` lui-même maintenant. [\#6416](https://github.com/ClickHouse/ClickHouse/pull/6416) ([Artem Zuikov](https://github.com/4ertus2)) - Ajouter un paramètre `joined_subquery_requires_alias` pour exiger des alias pour les sous-sélections et les `FROM` that more than one table is present (i.e. queries with JOINs). [\#6733](https://github.com/ClickHouse/ClickHouse/pull/6733) ([Artem Zuikov](https://github.com/4ertus2)) - Extrait `GetAggregatesVisitor` classe de `ExpressionAnalyzer`. [\#6458](https://github.com/ClickHouse/ClickHouse/pull/6458) ([Artem Zuikov](https://github.com/4ertus2)) - `system.query_log`: modifier le type de données de `type` colonne de `Enum`. [\#6265](https://github.com/ClickHouse/ClickHouse/pull/6265) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- La liaison statique de `sha256_password` greffon d’authentification. [\#6512](https://github.com/ClickHouse/ClickHouse/pull/6512) ([Yuriy Baranov](https://github.com/yurriy)) -- Évitez la dépendance supplémentaire pour le paramètre `compile` travailler. Dans les versions précédentes, l’utilisateur peut obtenir une erreur comme `cannot open crti.o`, `unable to find library -lc` etc. [\#6309](https://github.com/ClickHouse/ClickHouse/pull/6309) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Plus de validation de l’entrée qui peut provenir d’une réplique malveillante. [\#6303](https://github.com/ClickHouse/ClickHouse/pull/6303) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- La liaison statique de `sha256_password` greffon d'authentification. [\#6512](https://github.com/ClickHouse/ClickHouse/pull/6512) ([Yuriy Baranov](https://github.com/yurriy)) +- Évitez la dépendance supplémentaire pour le paramètre `compile` travailler. Dans les versions précédentes, l'utilisateur peut obtenir une erreur comme `cannot open crti.o`, `unable to find library -lc` etc. [\#6309](https://github.com/ClickHouse/ClickHouse/pull/6309) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Plus de validation de l'entrée qui peut provenir d'une réplique malveillante. [\#6303](https://github.com/ClickHouse/ClickHouse/pull/6303) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Maintenant `clickhouse-obfuscator` le fichier est disponible dans `clickhouse-client` paquet. Dans les versions précédentes, il était disponible en tant que `clickhouse obfuscator` (avec des espaces). [\#5816](https://github.com/ClickHouse/ClickHouse/issues/5816) [\#6609](https://github.com/ClickHouse/ClickHouse/pull/6609) ([dimarub2000](https://github.com/dimarub2000)) -- Blocage fixe lorsque nous avons au moins deux requêtes qui lisent au moins deux tables dans un ordre différent et une autre requête qui effectue une opération DDL sur l’une des tables. Correction d’une autre impasse très rare. [\#6764](https://github.com/ClickHouse/ClickHouse/pull/6764) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Blocage fixe lorsque nous avons au moins deux requêtes qui lisent au moins deux tables dans un ordre différent et une autre requête qui effectue une opération DDL sur l'une des tables. Correction d'une autre impasse très rare. [\#6764](https://github.com/ClickHouse/ClickHouse/pull/6764) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Ajouter `os_thread_ids` colonne de `system.processes` et `system.query_log` pour une meilleure mise possibilités. [\#6763](https://github.com/ClickHouse/ClickHouse/pull/6763) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Une solution de contournement pour les bogues D’extension PHP mysqlnd qui se produisent lorsque `sha256_password` est utilisé comme un plugin d’authentification par défaut (décrit dans [\#6031](https://github.com/ClickHouse/ClickHouse/issues/6031)). [\#6113](https://github.com/ClickHouse/ClickHouse/pull/6113) ([Yuriy Baranov](https://github.com/yurriy)) +- Une solution de contournement pour les bogues D'extension PHP mysqlnd qui se produisent lorsque `sha256_password` est utilisé comme un plugin d'authentification par défaut (décrit dans [\#6031](https://github.com/ClickHouse/ClickHouse/issues/6031)). [\#6113](https://github.com/ClickHouse/ClickHouse/pull/6113) ([Yuriy Baranov](https://github.com/yurriy)) - Supprimez la place inutile avec les colonnes de nullité modifiées. [\#6693](https://github.com/ClickHouse/ClickHouse/pull/6693) ([Artem Zuikov](https://github.com/4ertus2)) -- Définir la valeur par défaut de `queue_max_wait_ms` à zéro, parce que la valeur actuelle (cinq secondes) n’a aucun sens. Il y a de rares circonstances où ce paramètre a une utilité. Ajout des paramètres de `replace_running_query_max_wait_ms`, `kafka_max_wait_ms` et `connection_pool_max_wait_ms` pour la désambiguïsation. [\#6692](https://github.com/ClickHouse/ClickHouse/pull/6692) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Définir la valeur par défaut de `queue_max_wait_ms` à zéro, parce que la valeur actuelle (cinq secondes) n'a aucun sens. Il y a de rares circonstances où ce paramètre a une utilité. Ajout des paramètres de `replace_running_query_max_wait_ms`, `kafka_max_wait_ms` et `connection_pool_max_wait_ms` pour la désambiguïsation. [\#6692](https://github.com/ClickHouse/ClickHouse/pull/6692) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Extrait `SelectQueryExpressionAnalyzer` de `ExpressionAnalyzer`. Conservez le dernier pour les requêtes non-select. [\#6499](https://github.com/ClickHouse/ClickHouse/pull/6499) ([Artem Zuikov](https://github.com/4ertus2)) -- Suppression de la duplication des formats d’entrée et de sortie. [\#6239](https://github.com/ClickHouse/ClickHouse/pull/6239) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Autoriser l’utilisateur à remplacer `poll_interval` et `idle_connection_timeout` paramètres de connexion. [\#6230](https://github.com/ClickHouse/ClickHouse/pull/6230) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- `MergeTree` a maintenant une option supplémentaire `ttl_only_drop_parts` (désactivé par défaut) pour éviter l’élagage partiel des pièces, afin qu’elles tombent complètement lorsque toutes les lignes d’une pièce sont expirées. [\#6191](https://github.com/ClickHouse/ClickHouse/pull/6191) ([Sergi Vladykin](https://github.com/svladykin)) +- Suppression de la duplication des formats d'entrée et de sortie. [\#6239](https://github.com/ClickHouse/ClickHouse/pull/6239) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Autoriser l'utilisateur à remplacer `poll_interval` et `idle_connection_timeout` paramètres de connexion. [\#6230](https://github.com/ClickHouse/ClickHouse/pull/6230) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- `MergeTree` a maintenant une option supplémentaire `ttl_only_drop_parts` (désactivé par défaut) pour éviter l'élagage partiel des pièces, afin qu'elles tombent complètement lorsque toutes les lignes d'une pièce sont expirées. [\#6191](https://github.com/ClickHouse/ClickHouse/pull/6191) ([Sergi Vladykin](https://github.com/svladykin)) - Type vérifie les fonctions set index. Throw exception si la fonction a un mauvais type. Cela corrige le test fuzz avec UBSan. [\#6511](https://github.com/ClickHouse/ClickHouse/pull/6511) ([Nikita Vasilev](https://github.com/nikvas0)) #### Amélioration Des Performances {#performance-improvement-2} - Optimiser les requêtes avec `ORDER BY expressions` clause, où `expressions` ont coïncidé préfixe avec clé de tri dans `MergeTree` table. Cette optimisation est contrôlée par `optimize_read_in_order` paramètre. [\#6054](https://github.com/ClickHouse/ClickHouse/pull/6054) [\#6629](https://github.com/ClickHouse/ClickHouse/pull/6629) ([Anton Popov](https://github.com/CurtizJ)) -- Permettre d’utiliser plusieurs threads pendant le chargement et le retrait des pièces. [\#6372](https://github.com/ClickHouse/ClickHouse/issues/6372) [\#6074](https://github.com/ClickHouse/ClickHouse/issues/6074) [\#6438](https://github.com/ClickHouse/ClickHouse/pull/6438) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Variante de lot implémentée de la mise à jour des états de fonction d’agrégat. Il peut conduire à des avantages de performance. [\#6435](https://github.com/ClickHouse/ClickHouse/pull/6435) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Utiliser `FastOps` bibliothèque de fonctions `exp`, `log`, `sigmoid`, `tanh`. FastOps est une bibliothèque mathématique vectorielle rapide de Michael Parakhin (Yandex CTO). Amélioration des performances de `exp` et `log` fonctions plus de 6 fois. Fonction `exp` et `log` de `Float32` l’argument retournera `Float32` (dans les versions précédentes, ils reviennent toujours `Float64`). Maintenant `exp(nan)` peut-retour `inf`. Le résultat de `exp` et `log` les fonctions peuvent ne pas être le nombre représentable de la machine le plus proche de la vraie réponse. [\#6254](https://github.com/ClickHouse/ClickHouse/pull/6254) ([alexeï-milovidov](https://github.com/alexey-milovidov)) Utilisation de la variante Danila Kutenin pour faire fonctionner les fasttops [\#6317](https://github.com/ClickHouse/ClickHouse/pull/6317) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Désactiver l’optimisation de clé consécutive pour `UInt8/16`. [\#6298](https://github.com/ClickHouse/ClickHouse/pull/6298) [\#6701](https://github.com/ClickHouse/ClickHouse/pull/6701) ([akuzm](https://github.com/akuzm)) -- Amélioration des performances de `simdjson` bibliothèque en se débarrassant de l’allocation dynamique dans `ParsedJson::Iterator`. [\#6479](https://github.com/ClickHouse/ClickHouse/pull/6479) ([Vitaly Baranov](https://github.com/vitlibar)) -- Pages de pré-défaut lors de l’allocation de mémoire avec `mmap()`. [\#6667](https://github.com/ClickHouse/ClickHouse/pull/6667) ([akuzm](https://github.com/akuzm)) -- Correction d’un bug de performance dans `Decimal` comparaison. [\#6380](https://github.com/ClickHouse/ClickHouse/pull/6380) ([Artem Zuikov](https://github.com/4ertus2)) +- Permettre d'utiliser plusieurs threads pendant le chargement et le retrait des pièces. [\#6372](https://github.com/ClickHouse/ClickHouse/issues/6372) [\#6074](https://github.com/ClickHouse/ClickHouse/issues/6074) [\#6438](https://github.com/ClickHouse/ClickHouse/pull/6438) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Variante de lot implémentée de la mise à jour des états de fonction d'agrégat. Il peut conduire à des avantages de performance. [\#6435](https://github.com/ClickHouse/ClickHouse/pull/6435) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Utiliser `FastOps` bibliothèque de fonctions `exp`, `log`, `sigmoid`, `tanh`. FastOps est une bibliothèque mathématique vectorielle rapide de Michael Parakhin (Yandex CTO). Amélioration des performances de `exp` et `log` fonctions plus de 6 fois. Fonction `exp` et `log` de `Float32` l'argument retournera `Float32` (dans les versions précédentes, ils reviennent toujours `Float64`). Maintenant `exp(nan)` peut-retour `inf`. Le résultat de `exp` et `log` les fonctions peuvent ne pas être le nombre représentable de la machine le plus proche de la vraie réponse. [\#6254](https://github.com/ClickHouse/ClickHouse/pull/6254) ([alexeï-milovidov](https://github.com/alexey-milovidov)) Utilisation de la variante Danila Kutenin pour faire fonctionner les fasttops [\#6317](https://github.com/ClickHouse/ClickHouse/pull/6317) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Désactiver l'optimisation de clé consécutive pour `UInt8/16`. [\#6298](https://github.com/ClickHouse/ClickHouse/pull/6298) [\#6701](https://github.com/ClickHouse/ClickHouse/pull/6701) ([akuzm](https://github.com/akuzm)) +- Amélioration des performances de `simdjson` bibliothèque en se débarrassant de l'allocation dynamique dans `ParsedJson::Iterator`. [\#6479](https://github.com/ClickHouse/ClickHouse/pull/6479) ([Vitaly Baranov](https://github.com/vitlibar)) +- Pages de pré-défaut lors de l'allocation de mémoire avec `mmap()`. [\#6667](https://github.com/ClickHouse/ClickHouse/pull/6667) ([akuzm](https://github.com/akuzm)) +- Correction d'un bug de performance dans `Decimal` comparaison. [\#6380](https://github.com/ClickHouse/ClickHouse/pull/6380) ([Artem Zuikov](https://github.com/4ertus2)) -#### Construction / Test / Amélioration De L’Emballage {#buildtestingpackaging-improvement-4} +#### Construction / Test / Amélioration De L'Emballage {#buildtestingpackaging-improvement-4} -- Supprimer le compilateur (instanciation du modèle d’exécution) car nous avons gagné sur ses performances. [\#6646](https://github.com/ClickHouse/ClickHouse/pull/6646) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Ajout d’un test de performance pour montrer la dégradation des performances dans gcc-9 de manière plus isolée. [\#6302](https://github.com/ClickHouse/ClickHouse/pull/6302) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Ajout d’un tableau de fonction `numbers_mt`, qui est la version multithread `numbers`. Mise à jour des tests de performance avec des fonctions de hachage. [\#6554](https://github.com/ClickHouse/ClickHouse/pull/6554) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Supprimer le compilateur (instanciation du modèle d'exécution) car nous avons gagné sur ses performances. [\#6646](https://github.com/ClickHouse/ClickHouse/pull/6646) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Ajout d'un test de performance pour montrer la dégradation des performances dans gcc-9 de manière plus isolée. [\#6302](https://github.com/ClickHouse/ClickHouse/pull/6302) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Ajout d'un tableau de fonction `numbers_mt`, qui est la version multithread `numbers`. Mise à jour des tests de performance avec des fonctions de hachage. [\#6554](https://github.com/ClickHouse/ClickHouse/pull/6554) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) - Mode de comparaison dans `clickhouse-benchmark` [\#6220](https://github.com/ClickHouse/ClickHouse/issues/6220) [\#6343](https://github.com/ClickHouse/ClickHouse/pull/6343) ([dimarub2000](https://github.com/dimarub2000)) -- Meilleur effort pour imprimer des traces de pile. Également ajouté `SIGPROF` comme un signal de débogage pour imprimer la trace de la pile d’un thread en cours d’exécution. [\#6529](https://github.com/ClickHouse/ClickHouse/pull/6529) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Meilleur effort pour imprimer des traces de pile. Également ajouté `SIGPROF` comme un signal de débogage pour imprimer la trace de la pile d'un thread en cours d'exécution. [\#6529](https://github.com/ClickHouse/ClickHouse/pull/6529) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Chaque fonction dans son propre fichier, partie 10. [\#6321](https://github.com/ClickHouse/ClickHouse/pull/6321) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Supprimer doublé const `TABLE_IS_READ_ONLY`. [\#6566](https://github.com/ClickHouse/ClickHouse/pull/6566) ([filimonov](https://github.com/filimonov)) - Changements de formatage pour `StringHashMap` PR [\#5417](https://github.com/ClickHouse/ClickHouse/issues/5417). [\#6700](https://github.com/ClickHouse/ClickHouse/pull/6700) ([akuzm](https://github.com/akuzm)) - Meilleure sous-requête pour la création de jointures dans `ExpressionAnalyzer`. [\#6824](https://github.com/ClickHouse/ClickHouse/pull/6824) ([Artem Zuikov](https://github.com/4ertus2)) - Supprimer une condition redondante (trouvée par PVS Studio). [\#6775](https://github.com/ClickHouse/ClickHouse/pull/6775) ([akuzm](https://github.com/akuzm)) -- Séparer l’interface de table de hachage pour `ReverseIndex`. [\#6672](https://github.com/ClickHouse/ClickHouse/pull/6672) ([akuzm](https://github.com/akuzm)) +- Séparer l'interface de table de hachage pour `ReverseIndex`. [\#6672](https://github.com/ClickHouse/ClickHouse/pull/6672) ([akuzm](https://github.com/akuzm)) - Refactoring des paramètres. [\#6689](https://github.com/ClickHouse/ClickHouse/pull/6689) ([alésapine](https://github.com/alesapin)) -- Ajouter des commentaires pour `set` fonctions d’index. [\#6319](https://github.com/ClickHouse/ClickHouse/pull/6319) ([Nikita Vasilev](https://github.com/nikvas0)) +- Ajouter des commentaires pour `set` fonctions d'index. [\#6319](https://github.com/ClickHouse/ClickHouse/pull/6319) ([Nikita Vasilev](https://github.com/nikvas0)) - Augmenter le score OOM dans la version de débogage sur Linux. [\#6152](https://github.com/ClickHouse/ClickHouse/pull/6152) ([akuzm](https://github.com/akuzm)) - HDFS HA fonctionne maintenant dans la construction de débogage. [\#6650](https://github.com/ClickHouse/ClickHouse/pull/6650) ([Weiqing Xu](https://github.com/weiqxu)) -- Ajout d’un test pour `transform_query_for_external_database`. [\#6388](https://github.com/ClickHouse/ClickHouse/pull/6388) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Ajout d'un test pour `transform_query_for_external_database`. [\#6388](https://github.com/ClickHouse/ClickHouse/pull/6388) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Ajouter un test pour plusieurs vues matérialisées pour la table Kafka. [\#6509](https://github.com/ClickHouse/ClickHouse/pull/6509) ([Ivan](https://github.com/abyss7)) - Faire mieux construire régime. [\#6500](https://github.com/ClickHouse/ClickHouse/pull/6500) ([Ivan](https://github.com/abyss7)) - Fixe `test_external_dictionaries` intégration dans le cas où il a été exécuté sous un utilisateur non root. [\#6507](https://github.com/ClickHouse/ClickHouse/pull/6507) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) - Le bogue se reproduit lorsque la taille totale des paquets écrits dépasse `DBMS_DEFAULT_BUFFER_SIZE`. [\#6204](https://github.com/ClickHouse/ClickHouse/pull/6204) ([Yuriy Baranov](https://github.com/yurriy)) -- Ajout d’un test pour `RENAME` tableau condition de course [\#6752](https://github.com/ClickHouse/ClickHouse/pull/6752) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Ajout d'un test pour `RENAME` tableau condition de course [\#6752](https://github.com/ClickHouse/ClickHouse/pull/6752) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Évitez la course de données sur les paramètres dans `KILL QUERY`. [\#6753](https://github.com/ClickHouse/ClickHouse/pull/6753) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Ajouter un test d’intégration pour gérer les erreurs par un dictionnaire de cache. [\#6755](https://github.com/ClickHouse/ClickHouse/pull/6755) ([Vitaly Baranov](https://github.com/vitlibar)) -- Désactiver l’analyse des fichiers objet ELF sur Mac OS, car cela n’a aucun sens. [\#6578](https://github.com/ClickHouse/ClickHouse/pull/6578) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Ajouter un test d'intégration pour gérer les erreurs par un dictionnaire de cache. [\#6755](https://github.com/ClickHouse/ClickHouse/pull/6755) ([Vitaly Baranov](https://github.com/vitlibar)) +- Désactiver l'analyse des fichiers objet ELF sur Mac OS, car cela n'a aucun sens. [\#6578](https://github.com/ClickHouse/ClickHouse/pull/6578) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Essayez de rendre le générateur de changelog meilleur. [\#6327](https://github.com/ClickHouse/ClickHouse/pull/6327) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Additionneur `-Wshadow` passer à la GCC. [\#6325](https://github.com/ClickHouse/ClickHouse/pull/6325) ([kreuzerkrieg](https://github.com/kreuzerkrieg)) - Suppression du code obsolète pour `mimalloc` soutien. [\#6715](https://github.com/ClickHouse/ClickHouse/pull/6715) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- `zlib-ng` détermine les capacités x86 et enregistre ces informations dans les variables globales. Ceci est fait dans l’appel defalteInit, qui peut être fait par différents threads simultanément. Pour éviter les Écritures multithread, faites-le au démarrage de la bibliothèque. [\#6141](https://github.com/ClickHouse/ClickHouse/pull/6141) ([akuzm](https://github.com/akuzm)) +- `zlib-ng` détermine les capacités x86 et enregistre ces informations dans les variables globales. Ceci est fait dans l'appel defalteInit, qui peut être fait par différents threads simultanément. Pour éviter les Écritures multithread, faites-le au démarrage de la bibliothèque. [\#6141](https://github.com/ClickHouse/ClickHouse/pull/6141) ([akuzm](https://github.com/akuzm)) - Test de régression pour un bug qui dans join qui a été corrigé dans [\#5192](https://github.com/ClickHouse/ClickHouse/issues/5192). [\#6147](https://github.com/ClickHouse/ClickHouse/pull/6147) ([Bakhtiyor Ruziev](https://github.com/theruziev)) - Rapport MSAN fixe. [\#6144](https://github.com/ClickHouse/ClickHouse/pull/6144) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Correction du test TTL battement. [\#6782](https://github.com/ClickHouse/ClickHouse/pull/6782) ([Anton Popov](https://github.com/CurtizJ)) - Correction de fausse course de données dans `MergeTreeDataPart::is_frozen` champ. [\#6583](https://github.com/ClickHouse/ClickHouse/pull/6583) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Délais d’attente fixes dans le test fuzz. Dans la version précédente, il a réussi à trouver false hangup dans la requête `SELECT * FROM numbers_mt(gccMurmurHash(''))`. [\#6582](https://github.com/ClickHouse/ClickHouse/pull/6582) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Délais d'attente fixes dans le test fuzz. Dans la version précédente, il a réussi à trouver false hangup dans la requête `SELECT * FROM numbers_mt(gccMurmurHash(''))`. [\#6582](https://github.com/ClickHouse/ClickHouse/pull/6582) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Ajouté debug contrôles `static_cast` des colonnes. [\#6581](https://github.com/ClickHouse/ClickHouse/pull/6581) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Prise en charge D’Oracle Linux dans les paquets RPM officiels. [\#6356](https://github.com/ClickHouse/ClickHouse/issues/6356) [\#6585](https://github.com/ClickHouse/ClickHouse/pull/6585) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Prise en charge D'Oracle Linux dans les paquets RPM officiels. [\#6356](https://github.com/ClickHouse/ClickHouse/issues/6356) [\#6585](https://github.com/ClickHouse/ClickHouse/pull/6585) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Changé JSON perftests de `once` de `loop` type. [\#6536](https://github.com/ClickHouse/ClickHouse/pull/6536) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) - `odbc-bridge.cpp` définit `main()` donc, il ne devrait pas être inclus dans `clickhouse-lib`. [\#6538](https://github.com/ClickHouse/ClickHouse/pull/6538) ([Orivej Desh](https://github.com/orivej)) -- Test de crash dans l’ `FULL|RIGHT JOIN` avec nulls dans les clés de la table de droite. [\#6362](https://github.com/ClickHouse/ClickHouse/pull/6362) ([Artem Zuikov](https://github.com/4ertus2)) -- Ajout d’un test pour la limite d’extension des alias, juste au cas où. [\#6442](https://github.com/ClickHouse/ClickHouse/pull/6442) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Test de crash dans l' `FULL|RIGHT JOIN` avec nulls dans les clés de la table de droite. [\#6362](https://github.com/ClickHouse/ClickHouse/pull/6362) ([Artem Zuikov](https://github.com/4ertus2)) +- Ajout d'un test pour la limite d'extension des alias, juste au cas où. [\#6442](https://github.com/ClickHouse/ClickHouse/pull/6442) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Commutation de `boost::filesystem` de `std::filesystem` échéant. [\#6253](https://github.com/ClickHouse/ClickHouse/pull/6253) [\#6385](https://github.com/ClickHouse/ClickHouse/pull/6385) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Ajout de paquets RPM au site web. [\#6251](https://github.com/ClickHouse/ClickHouse/pull/6251) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Ajouter un test pour fixe `Unknown identifier` exception dans `IN` section. [\#6708](https://github.com/ClickHouse/ClickHouse/pull/6708) ([Artem Zuikov](https://github.com/4ertus2)) - Simplifier `shared_ptr_helper` parce que les gens confrontés à des difficultés à le comprendre. [\#6675](https://github.com/ClickHouse/ClickHouse/pull/6675) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Ajout de tests de performance pour le codec Gorilla et DoubleDelta fixe. [\#6179](https://github.com/ClickHouse/ClickHouse/pull/6179) ([Vasily Nemkov](https://github.com/Enmk)) -- Diviser le test d’intégration `test_dictionaries` dans 4 tests distincts. [\#6776](https://github.com/ClickHouse/ClickHouse/pull/6776) ([Vitaly Baranov](https://github.com/vitlibar)) -- Correction de L’avertissement PVS-Studio dans `PipelineExecutor`. [\#6777](https://github.com/ClickHouse/ClickHouse/pull/6777) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Permettre d’utiliser `library` dictionnaire source avec ASan. [\#6482](https://github.com/ClickHouse/ClickHouse/pull/6482) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Ajout d’une option pour générer changelog à partir d’une liste de PRs. [\#6350](https://github.com/ClickHouse/ClickHouse/pull/6350) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Diviser le test d'intégration `test_dictionaries` dans 4 tests distincts. [\#6776](https://github.com/ClickHouse/ClickHouse/pull/6776) ([Vitaly Baranov](https://github.com/vitlibar)) +- Correction de L'avertissement PVS-Studio dans `PipelineExecutor`. [\#6777](https://github.com/ClickHouse/ClickHouse/pull/6777) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Permettre d'utiliser `library` dictionnaire source avec ASan. [\#6482](https://github.com/ClickHouse/ClickHouse/pull/6482) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Ajout d'une option pour générer changelog à partir d'une liste de PRs. [\#6350](https://github.com/ClickHouse/ClickHouse/pull/6350) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Verrouiller le `TinyLog` de stockage lors de la lecture. [\#6226](https://github.com/ClickHouse/ClickHouse/pull/6226) ([akuzm](https://github.com/akuzm)) - Vérifiez les liens symboliques brisés dans CI. [\#6634](https://github.com/ClickHouse/ClickHouse/pull/6634) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Augmentez le délai pour “stack overflow” test car cela peut prendre beaucoup de temps dans la construction de débogage. [\#6637](https://github.com/ClickHouse/ClickHouse/pull/6637) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Ajout d’une vérification pour les doubles espaces. [\#6643](https://github.com/ClickHouse/ClickHouse/pull/6643) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Fixer `new/delete` suivi de la mémoire lors de la construction avec des désinfectants. Le suivi n’est pas clair. Il empêche uniquement les exceptions de limite de mémoire dans les tests. [\#6450](https://github.com/ClickHouse/ClickHouse/pull/6450) ([Artem Zuikov](https://github.com/4ertus2)) +- Ajout d'une vérification pour les doubles espaces. [\#6643](https://github.com/ClickHouse/ClickHouse/pull/6643) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Fixer `new/delete` suivi de la mémoire lors de la construction avec des désinfectants. Le suivi n'est pas clair. Il empêche uniquement les exceptions de limite de mémoire dans les tests. [\#6450](https://github.com/ClickHouse/ClickHouse/pull/6450) ([Artem Zuikov](https://github.com/4ertus2)) - Activez la vérification des symboles non définis lors de la liaison. [\#6453](https://github.com/ClickHouse/ClickHouse/pull/6453) ([Ivan](https://github.com/abyss7)) - Éviter la reconstruction `hyperscan` quotidien. [\#6307](https://github.com/ClickHouse/ClickHouse/pull/6307) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Rapport UBSan fixe dans `ProtobufWriter`. [\#6163](https://github.com/ClickHouse/ClickHouse/pull/6163) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Ne permettez pas d’utiliser query profiler avec des désinfectants car il n’est pas compatible. [\#6769](https://github.com/ClickHouse/ClickHouse/pull/6769) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Ne permettez pas d'utiliser query profiler avec des désinfectants car il n'est pas compatible. [\#6769](https://github.com/ClickHouse/ClickHouse/pull/6769) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Ajouter un test pour recharger un dictionnaire après échec par minuterie. [\#6114](https://github.com/ClickHouse/ClickHouse/pull/6114) ([Vitaly Baranov](https://github.com/vitlibar)) -- Correction de l’incohérence dans `PipelineExecutor::prepareProcessor` type d’argument. [\#6494](https://github.com/ClickHouse/ClickHouse/pull/6494) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Ajout d’un test pour les mauvais URI. [\#6493](https://github.com/ClickHouse/ClickHouse/pull/6493) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Ajouté plus de contrôles `CAST` fonction. Cela devrait obtenir plus d’informations sur la faille de segmentation dans le test flou. [\#6346](https://github.com/ClickHouse/ClickHouse/pull/6346) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Ajouter `gcc-9` soutien à `docker/builder` conteneur qui construit l’image localement. [\#6333](https://github.com/ClickHouse/ClickHouse/pull/6333) ([Gleb Novikov](https://github.com/NanoBjorn)) +- Correction de l'incohérence dans `PipelineExecutor::prepareProcessor` type d'argument. [\#6494](https://github.com/ClickHouse/ClickHouse/pull/6494) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Ajout d'un test pour les mauvais URI. [\#6493](https://github.com/ClickHouse/ClickHouse/pull/6493) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Ajouté plus de contrôles `CAST` fonction. Cela devrait obtenir plus d'informations sur la faille de segmentation dans le test flou. [\#6346](https://github.com/ClickHouse/ClickHouse/pull/6346) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Ajouter `gcc-9` soutien à `docker/builder` conteneur qui construit l'image localement. [\#6333](https://github.com/ClickHouse/ClickHouse/pull/6333) ([Gleb Novikov](https://github.com/NanoBjorn)) - Test de la clé primaire avec `LowCardinality(String)`. [\#5044](https://github.com/ClickHouse/ClickHouse/issues/5044) [\#6219](https://github.com/ClickHouse/ClickHouse/pull/6219) ([dimarub2000](https://github.com/dimarub2000)) -- Correction des tests affectés par l’impression de traces de pile lente. [\#6315](https://github.com/ClickHouse/ClickHouse/pull/6315) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction des tests affectés par l'impression de traces de pile lente. [\#6315](https://github.com/ClickHouse/ClickHouse/pull/6315) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Ajouter un cas de test pour crash in `groupUniqArray` fixe dans [\#6029](https://github.com/ClickHouse/ClickHouse/pull/6029). [\#4402](https://github.com/ClickHouse/ClickHouse/issues/4402) [\#6129](https://github.com/ClickHouse/ClickHouse/pull/6129) ([akuzm](https://github.com/akuzm)) -- Tests de mutations d’indices fixes. [\#6645](https://github.com/ClickHouse/ClickHouse/pull/6645) ([Nikita Vasilev](https://github.com/nikvas0)) -- Dans le test de performance, ne lisez pas le journal des requêtes pour les requêtes que nous n’avons pas exécutées. [\#6427](https://github.com/ClickHouse/ClickHouse/pull/6427) ([akuzm](https://github.com/akuzm)) -- La vue matérialisée peut maintenant être créée avec n’importe quel type de cardinalité faible quel que soit le paramètre concernant les types de cardinalité faible suspects. [\#6428](https://github.com/ClickHouse/ClickHouse/pull/6428) ([Olga Khvostikova](https://github.com/stavrolia)) +- Tests de mutations d'indices fixes. [\#6645](https://github.com/ClickHouse/ClickHouse/pull/6645) ([Nikita Vasilev](https://github.com/nikvas0)) +- Dans le test de performance, ne lisez pas le journal des requêtes pour les requêtes que nous n'avons pas exécutées. [\#6427](https://github.com/ClickHouse/ClickHouse/pull/6427) ([akuzm](https://github.com/akuzm)) +- La vue matérialisée peut maintenant être créée avec n'importe quel type de cardinalité faible quel que soit le paramètre concernant les types de cardinalité faible suspects. [\#6428](https://github.com/ClickHouse/ClickHouse/pull/6428) ([Olga Khvostikova](https://github.com/stavrolia)) - Mise à jour des tests pour `send_logs_level` paramètre. [\#6207](https://github.com/ClickHouse/ClickHouse/pull/6207) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) - Correction de la construction sous gcc-8.2. [\#6196](https://github.com/ClickHouse/ClickHouse/pull/6196) ([Max Akhmedov](https://github.com/zlobober)) - Correction de la construction avec libc++interne. [\#6724](https://github.com/ClickHouse/ClickHouse/pull/6724) ([Ivan](https://github.com/abyss7)) @@ -783,7 +783,7 @@ toc_title: '2019' #### Modification Incompatible En Arrière {#backward-incompatible-change-3} -- Suppression de la fonction de table rarement utilisée `catBoostPool` et de stockage `CatBoostPool`. Si vous avez utilisé cette fonction de table, veuillez écrire un courriel à `clickhouse-feedback@yandex-team.com`. Notez que L’intégration CatBoost reste et sera prise en charge. [\#6279](https://github.com/ClickHouse/ClickHouse/pull/6279) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Suppression de la fonction de table rarement utilisée `catBoostPool` et de stockage `CatBoostPool`. Si vous avez utilisé cette fonction de table, veuillez écrire un courriel à `clickhouse-feedback@yandex-team.com`. Notez que L'intégration CatBoost reste et sera prise en charge. [\#6279](https://github.com/ClickHouse/ClickHouse/pull/6279) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Désactiver `ANY RIGHT JOIN` et `ANY FULL JOIN` par défaut. Définir `any_join_distinct_right_table_keys` réglage pour les activer. [\#5126](https://github.com/ClickHouse/ClickHouse/issues/5126) [\#6351](https://github.com/ClickHouse/ClickHouse/pull/6351) ([Artem Zuikov](https://github.com/4ertus2)) ## Clickhouse Version 19.13 {#clickhouse-release-19-13} @@ -799,14 +799,14 @@ toc_title: '2019' #### Bug Fix {#bug-fix-10} - Cette version contient également toutes les corrections de bugs de 19.14.6.12. -- Correction d’un état incohérent possible de la table lors de l’exécution `DROP` requête pour la table répliquée alors que zookeeper n’est pas accessible. [\#6045](https://github.com/ClickHouse/ClickHouse/issues/6045) [\#6413](https://github.com/ClickHouse/ClickHouse/pull/6413) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +- Correction d'un état incohérent possible de la table lors de l'exécution `DROP` requête pour la table répliquée alors que zookeeper n'est pas accessible. [\#6045](https://github.com/ClickHouse/ClickHouse/issues/6045) [\#6413](https://github.com/ClickHouse/ClickHouse/pull/6413) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) - Correction de la course de données dans StorageMerge [\#6717](https://github.com/ClickHouse/ClickHouse/pull/6717) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’un bug introduit dans query profiler qui conduit à recv sans fin de socket. [\#6386](https://github.com/ClickHouse/ClickHouse/pull/6386) ([alésapine](https://github.com/alesapin)) -- Correction de l’utilisation excessive du processeur lors de l’exécution `JSONExtractRaw` la fonction sur une valeur booléenne. [\#6208](https://github.com/ClickHouse/ClickHouse/pull/6208) ([Vitaly Baranov](https://github.com/vitlibar)) +- Correction d'un bug introduit dans query profiler qui conduit à recv sans fin de socket. [\#6386](https://github.com/ClickHouse/ClickHouse/pull/6386) ([alésapine](https://github.com/alesapin)) +- Correction de l'utilisation excessive du processeur lors de l'exécution `JSONExtractRaw` la fonction sur une valeur booléenne. [\#6208](https://github.com/ClickHouse/ClickHouse/pull/6208) ([Vitaly Baranov](https://github.com/vitlibar)) - Corrige la régression tout en poussant vers la vue matérialisée. [\#6415](https://github.com/ClickHouse/ClickHouse/pull/6415) ([Ivan](https://github.com/abyss7)) -- Fonction de Table `url` la vulnérabilité avait-elle permis à l’attaquant d’injecter des en-têtes HTTP arbitraires dans la requête. Ce problème a été trouvé par [Nikita Tikhomirov](https://github.com/NSTikhomirov). [\#6466](https://github.com/ClickHouse/ClickHouse/pull/6466) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Fix inutile `AST` vérifier dans L’index de jeu. [\#6510](https://github.com/ClickHouse/ClickHouse/issues/6510) [\#6651](https://github.com/ClickHouse/ClickHouse/pull/6651) ([Nikita Vasilev](https://github.com/nikvas0)) -- Fixe l’analyse de `AggregateFunction` valeurs ancrées dans la requête. [\#6575](https://github.com/ClickHouse/ClickHouse/issues/6575) [\#6773](https://github.com/ClickHouse/ClickHouse/pull/6773) ([Zhichang Yu](https://github.com/yuzhichang)) +- Fonction de Table `url` la vulnérabilité avait-elle permis à l'attaquant d'injecter des en-têtes HTTP arbitraires dans la requête. Ce problème a été trouvé par [Nikita Tikhomirov](https://github.com/NSTikhomirov). [\#6466](https://github.com/ClickHouse/ClickHouse/pull/6466) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Fix inutile `AST` vérifier dans L'index de jeu. [\#6510](https://github.com/ClickHouse/ClickHouse/issues/6510) [\#6651](https://github.com/ClickHouse/ClickHouse/pull/6651) ([Nikita Vasilev](https://github.com/nikvas0)) +- Fixe l'analyse de `AggregateFunction` valeurs ancrées dans la requête. [\#6575](https://github.com/ClickHouse/ClickHouse/issues/6575) [\#6773](https://github.com/ClickHouse/ClickHouse/pull/6773) ([Zhichang Yu](https://github.com/yuzhichang)) - Fixe mauvais comportement de `trim` les fonctions de la famille. [\#6647](https://github.com/ClickHouse/ClickHouse/pull/6647) ([alexeï-milovidov](https://github.com/alexey-milovidov)) ### Clickhouse Version 19.13.4.32, 2019-09-10 {#clickhouse-release-19-13-4-32-2019-09-10} @@ -815,33 +815,33 @@ toc_title: '2019' - Cette version contient également tous les correctifs de sécurité de bugs de 19.11.9.52 et 19.11.10.54. - Les données fixes de course `system.parts` table et `ALTER` requête. [\#6245](https://github.com/ClickHouse/ClickHouse/issues/6245) [\#6513](https://github.com/ClickHouse/ClickHouse/pull/6513) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’en-tête non apparié dans les flux se sont produits en cas de lecture à partir d’une table distribuée vide avec sample et prewhere. [\#6167](https://github.com/ClickHouse/ClickHouse/issues/6167) ([Lixiang Qian](https://github.com/fancyqlx)) [\#6823](https://github.com/ClickHouse/ClickHouse/pull/6823) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Correction d’un crash lors de l’utilisation de `IN` clause avec une sous-requête avec un tuple. [\#6125](https://github.com/ClickHouse/ClickHouse/issues/6125) [\#6550](https://github.com/ClickHouse/ClickHouse/pull/6550) ([tavplubix](https://github.com/tavplubix)) +- Correction d'en-tête non apparié dans les flux se sont produits en cas de lecture à partir d'une table distribuée vide avec sample et prewhere. [\#6167](https://github.com/ClickHouse/ClickHouse/issues/6167) ([Lixiang Qian](https://github.com/fancyqlx)) [\#6823](https://github.com/ClickHouse/ClickHouse/pull/6823) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Correction d'un crash lors de l'utilisation de `IN` clause avec une sous-requête avec un tuple. [\#6125](https://github.com/ClickHouse/ClickHouse/issues/6125) [\#6550](https://github.com/ClickHouse/ClickHouse/pull/6550) ([tavplubix](https://github.com/tavplubix)) - Corrigé cas avec les mêmes noms de colonnes dans `GLOBAL JOIN ON` section. [\#6181](https://github.com/ClickHouse/ClickHouse/pull/6181) ([Artem Zuikov](https://github.com/4ertus2)) -- Correction d’un crash lors de la coulée de types à `Decimal` qui ne la supportent pas. Jetez exception à la place. [\#6297](https://github.com/ClickHouse/ClickHouse/pull/6297) ([Artem Zuikov](https://github.com/4ertus2)) -- Correction d’un crash dans `extractAll()` fonction. [\#6644](https://github.com/ClickHouse/ClickHouse/pull/6644) ([Artem Zuikov](https://github.com/4ertus2)) +- Correction d'un crash lors de la coulée de types à `Decimal` qui ne la supportent pas. Jetez exception à la place. [\#6297](https://github.com/ClickHouse/ClickHouse/pull/6297) ([Artem Zuikov](https://github.com/4ertus2)) +- Correction d'un crash dans `extractAll()` fonction. [\#6644](https://github.com/ClickHouse/ClickHouse/pull/6644) ([Artem Zuikov](https://github.com/4ertus2)) - Transformation de requête pour `MySQL`, `ODBC`, `JDBC` fonctions de table fonctionne maintenant correctement pour `SELECT WHERE` requêtes avec plusieurs `AND` expression. [\#6381](https://github.com/ClickHouse/ClickHouse/issues/6381) [\#6676](https://github.com/ClickHouse/ClickHouse/pull/6676) ([dimarub2000](https://github.com/dimarub2000)) -- Ajout de vérifications de déclaration précédentes pour L’intégration de MySQL 8. [\#6569](https://github.com/ClickHouse/ClickHouse/pull/6569) ([Rafael David Tinoco](https://github.com/rafaeldtinoco)) +- Ajout de vérifications de déclaration précédentes pour L'intégration de MySQL 8. [\#6569](https://github.com/ClickHouse/ClickHouse/pull/6569) ([Rafael David Tinoco](https://github.com/rafaeldtinoco)) #### Correction De Sécurité {#security-fix-1} -- Correction de deux vulnérabilités dans les codecs en phase de décompression (l’utilisateur malveillant peut fabriquer des données compressées qui conduiront à un débordement de tampon en décompression). [\#6670](https://github.com/ClickHouse/ClickHouse/pull/6670) ([Artem Zuikov](https://github.com/4ertus2)) +- Correction de deux vulnérabilités dans les codecs en phase de décompression (l'utilisateur malveillant peut fabriquer des données compressées qui conduiront à un débordement de tampon en décompression). [\#6670](https://github.com/ClickHouse/ClickHouse/pull/6670) ([Artem Zuikov](https://github.com/4ertus2)) ### Clickhouse Version 19.13.3.26, 2019-08-22 {#clickhouse-release-19-13-3-26-2019-08-22} #### Bug Fix {#bug-fix-12} - Fixer `ALTER TABLE ... UPDATE` requête pour les tables avec `enable_mixed_granularity_parts=1`. [\#6543](https://github.com/ClickHouse/ClickHouse/pull/6543) ([alésapine](https://github.com/alesapin)) -- Correction de NPE lors de l’utilisation de la clause IN avec une sous-requête avec un tuple. [\#6125](https://github.com/ClickHouse/ClickHouse/issues/6125) [\#6550](https://github.com/ClickHouse/ClickHouse/pull/6550) ([tavplubix](https://github.com/tavplubix)) -- Correction d’un problème que si un réplica périmé devient vivant, il peut encore avoir des parties de données qui ont été supprimés par la partition DROP. [\#6522](https://github.com/ClickHouse/ClickHouse/issues/6522) [\#6523](https://github.com/ClickHouse/ClickHouse/pull/6523) ([tavplubix](https://github.com/tavplubix)) -- Correction d’un problème avec l’analyse CSV [\#6426](https://github.com/ClickHouse/ClickHouse/issues/6426) [\#6559](https://github.com/ClickHouse/ClickHouse/pull/6559) ([tavplubix](https://github.com/tavplubix)) +- Correction de NPE lors de l'utilisation de la clause IN avec une sous-requête avec un tuple. [\#6125](https://github.com/ClickHouse/ClickHouse/issues/6125) [\#6550](https://github.com/ClickHouse/ClickHouse/pull/6550) ([tavplubix](https://github.com/tavplubix)) +- Correction d'un problème que si un réplica périmé devient vivant, il peut encore avoir des parties de données qui ont été supprimés par la partition DROP. [\#6522](https://github.com/ClickHouse/ClickHouse/issues/6522) [\#6523](https://github.com/ClickHouse/ClickHouse/pull/6523) ([tavplubix](https://github.com/tavplubix)) +- Correction d'un problème avec l'analyse CSV [\#6426](https://github.com/ClickHouse/ClickHouse/issues/6426) [\#6559](https://github.com/ClickHouse/ClickHouse/pull/6559) ([tavplubix](https://github.com/tavplubix)) - Course de données fixe dans le système.table de pièces et ALTER query. Cela corrige [\#6245](https://github.com/ClickHouse/ClickHouse/issues/6245). [\#6513](https://github.com/ClickHouse/ClickHouse/pull/6513) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction du mauvais code dans les mutations qui peuvent conduire à la corruption de la mémoire. Correction de segfault avec lecture de l’adresse `0x14c0` cela peut se produire en raison de simultané `DROP TABLE` et `SELECT` de `system.parts` ou `system.parts_columns`. Condition de course fixe dans la préparation des requêtes de mutation. Blocage fixe causé par `OPTIMIZE` des tables répliquées et des opérations de modification simultanées comme ALTERs. [\#6514](https://github.com/ClickHouse/ClickHouse/pull/6514) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction possible perte de données après `ALTER DELETE` requête sur la table avec l’index de saut. [\#6224](https://github.com/ClickHouse/ClickHouse/issues/6224) [\#6282](https://github.com/ClickHouse/ClickHouse/pull/6282) ([Nikita Vasilev](https://github.com/nikvas0)) +- Correction du mauvais code dans les mutations qui peuvent conduire à la corruption de la mémoire. Correction de segfault avec lecture de l'adresse `0x14c0` cela peut se produire en raison de simultané `DROP TABLE` et `SELECT` de `system.parts` ou `system.parts_columns`. Condition de course fixe dans la préparation des requêtes de mutation. Blocage fixe causé par `OPTIMIZE` des tables répliquées et des opérations de modification simultanées comme ALTERs. [\#6514](https://github.com/ClickHouse/ClickHouse/pull/6514) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction possible perte de données après `ALTER DELETE` requête sur la table avec l'index de saut. [\#6224](https://github.com/ClickHouse/ClickHouse/issues/6224) [\#6282](https://github.com/ClickHouse/ClickHouse/pull/6282) ([Nikita Vasilev](https://github.com/nikvas0)) #### Correction De Sécurité {#security-fix-2} -- Si L’attaquant a un accès en écriture à ZooKeeper et est capable d’exécuter un serveur personnalisé disponible à partir du réseau où clickhouse s’exécute, il peut créer un serveur malveillant sur mesure qui agira comme réplique de ClickHouse et l’enregistrer dans ZooKeeper. Lorsqu’une autre réplique récupère une partie de données à partir d’une réplique malveillante, elle peut forcer clickhouse-server à écrire sur un chemin arbitraire sur le système de fichiers. Trouvé par Eldar Zaitov, équipe de sécurité de L’information chez Yandex. [\#6247](https://github.com/ClickHouse/ClickHouse/pull/6247) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Si L'attaquant a un accès en écriture à ZooKeeper et est capable d'exécuter un serveur personnalisé disponible à partir du réseau où clickhouse s'exécute, il peut créer un serveur malveillant sur mesure qui agira comme réplique de ClickHouse et l'enregistrer dans ZooKeeper. Lorsqu'une autre réplique récupère une partie de données à partir d'une réplique malveillante, elle peut forcer clickhouse-server à écrire sur un chemin arbitraire sur le système de fichiers. Trouvé par Eldar Zaitov, équipe de sécurité de L'information chez Yandex. [\#6247](https://github.com/ClickHouse/ClickHouse/pull/6247) ([alexeï-milovidov](https://github.com/alexey-milovidov)) ### Clickhouse Version 19.13.2.19, 2019-08-14 {#clickhouse-release-19-13-2-19-2019-08-14} @@ -850,36 +850,36 @@ toc_title: '2019' - Échantillonnage du profileur au niveau de la requête. [Exemple](https://gist.github.com/alexey-milovidov/92758583dd41c24c360fdb8d6a4da194). [\#4247](https://github.com/ClickHouse/ClickHouse/issues/4247) ([laplab](https://github.com/laplab)) [\#6124](https://github.com/ClickHouse/ClickHouse/pull/6124) ([alexeï-milovidov](https://github.com/alexey-milovidov)) [\#6250](https://github.com/ClickHouse/ClickHouse/pull/6250) [\#6283](https://github.com/ClickHouse/ClickHouse/pull/6283) [\#6386](https://github.com/ClickHouse/ClickHouse/pull/6386) - Permet de spécifier une liste de colonnes avec `COLUMNS('regexp')` expression qui fonctionne comme une variante plus sophistiquée de `*` astérisque. [\#5951](https://github.com/ClickHouse/ClickHouse/pull/5951) ([mfridental](https://github.com/mfridental)), ([alexeï-milovidov](https://github.com/alexey-milovidov)) - `CREATE TABLE AS table_function()` est maintenant possible [\#6057](https://github.com/ClickHouse/ClickHouse/pull/6057) ([dimarub2000](https://github.com/dimarub2000)) -- Adam optimizer pour la descente de gradient stochastique est utilisé par défaut dans `stochasticLinearRegression()` et `stochasticLogisticRegression()` fonctions d’agrégation, car il montre une bonne qualité sans presque aucun réglage. [\#6000](https://github.com/ClickHouse/ClickHouse/pull/6000) ([Quid37](https://github.com/Quid37)) +- Adam optimizer pour la descente de gradient stochastique est utilisé par défaut dans `stochasticLinearRegression()` et `stochasticLogisticRegression()` fonctions d'agrégation, car il montre une bonne qualité sans presque aucun réglage. [\#6000](https://github.com/ClickHouse/ClickHouse/pull/6000) ([Quid37](https://github.com/Quid37)) - Added functions for working with the сustom week number [\#5212](https://github.com/ClickHouse/ClickHouse/pull/5212) ([Andy Yang](https://github.com/andyyzh)) - `RENAME` les requêtes fonctionnent maintenant avec tous les stockages. [\#5953](https://github.com/ClickHouse/ClickHouse/pull/5953) ([Ivan](https://github.com/abyss7)) -- Maintenant client de recevoir les journaux du serveur avec n’importe quel niveau de `send_logs_level` quel que soit le niveau de journal spécifié dans les paramètres du serveur. [\#5964](https://github.com/ClickHouse/ClickHouse/pull/5964) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +- Maintenant client de recevoir les journaux du serveur avec n'importe quel niveau de `send_logs_level` quel que soit le niveau de journal spécifié dans les paramètres du serveur. [\#5964](https://github.com/ClickHouse/ClickHouse/pull/5964) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) #### Modification Incompatible En Arrière {#backward-incompatible-change-4} - Paramètre `input_format_defaults_for_omitted_fields` est activé par défaut. Les insertions dans les tables distribuées ont besoin que ce paramètre soit le même sur le cluster (vous devez le définir avant de lancer la mise à jour). Il permet de calculer des expressions par défaut Complexes pour les champs omis dans `JSONEachRow` et `CSV*` format. Il devrait être le comportement attendu, mais peut conduire à négligeable différence de performances. [\#6043](https://github.com/ClickHouse/ClickHouse/pull/6043) ([Artem Zuikov](https://github.com/4ertus2)), [\#5625](https://github.com/ClickHouse/ClickHouse/pull/5625) ([akuzm](https://github.com/akuzm)) -#### Caractéristiques expérimentales {#experimental-features} +#### Caractéristiques Expérimentales {#experimental-features} -- Nouveau pipeline de traitement des requêtes. Utiliser `experimental_use_processors=1` une option pour l’activer. Utilisez pour votre propre problème. [\#4914](https://github.com/ClickHouse/ClickHouse/pull/4914) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Nouveau pipeline de traitement des requêtes. Utiliser `experimental_use_processors=1` une option pour l'activer. Utilisez pour votre propre problème. [\#4914](https://github.com/ClickHouse/ClickHouse/pull/4914) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) #### Bug Fix {#bug-fix-13} -- L’intégration de Kafka a été corrigée dans cette version. -- Fixe `DoubleDelta` l’encodage de `Int64` pour les grands `DoubleDelta` les valeurs, l’amélioration de la `DoubleDelta` encodage de données aléatoires pour `Int32`. [\#5998](https://github.com/ClickHouse/ClickHouse/pull/5998) ([Vasily Nemkov](https://github.com/Enmk)) +- L'intégration de Kafka a été corrigée dans cette version. +- Fixe `DoubleDelta` l'encodage de `Int64` pour les grands `DoubleDelta` les valeurs, l'amélioration de la `DoubleDelta` encodage de données aléatoires pour `Int32`. [\#5998](https://github.com/ClickHouse/ClickHouse/pull/5998) ([Vasily Nemkov](https://github.com/Enmk)) - Surestimation fixe de `max_rows_to_read` si le paramètre `merge_tree_uniform_read_distribution` est réglé sur 0. [\#6019](https://github.com/ClickHouse/ClickHouse/pull/6019) ([alexeï-milovidov](https://github.com/alexey-milovidov)) #### Amélioration {#improvement-4} -- Lève une exception si `config.d` le fichier n’a pas l’élément racine correspondant comme fichier de configuration [\#6123](https://github.com/ClickHouse/ClickHouse/pull/6123) ([dimarub2000](https://github.com/dimarub2000)) +- Lève une exception si `config.d` le fichier n'a pas l'élément racine correspondant comme fichier de configuration [\#6123](https://github.com/ClickHouse/ClickHouse/pull/6123) ([dimarub2000](https://github.com/dimarub2000)) #### Amélioration Des Performances {#performance-improvement-3} - Optimiser `count()`. Maintenant, il utilise la plus petite colonne (si possible). [\#6028](https://github.com/ClickHouse/ClickHouse/pull/6028) ([Amos Oiseau](https://github.com/amosbird)) -#### Construction / Test / Amélioration De L’Emballage {#buildtestingpackaging-improvement-5} +#### Construction / Test / Amélioration De L'Emballage {#buildtestingpackaging-improvement-5} -- Signaler l’utilisation de la mémoire dans les tests de performance. [\#5899](https://github.com/ClickHouse/ClickHouse/pull/5899) ([akuzm](https://github.com/akuzm)) +- Signaler l'utilisation de la mémoire dans les tests de performance. [\#5899](https://github.com/ClickHouse/ClickHouse/pull/5899) ([akuzm](https://github.com/akuzm)) - Correction de la construction avec externe `libcxx` [\#6010](https://github.com/ClickHouse/ClickHouse/pull/6010) ([Ivan](https://github.com/abyss7)) - Correction de la construction partagée avec `rdkafka` bibliothèque [\#6101](https://github.com/ClickHouse/ClickHouse/pull/6101) ([Ivan](https://github.com/abyss7)) @@ -889,21 +889,21 @@ toc_title: '2019' #### Bug Fix {#bug-fix-14} -- Correction d’un crash rare dans `ALTER MODIFY COLUMN` et fusion verticale lorsque l’une des parties fusionnées/modifiées est vide (0 lignes). [\#6780](https://github.com/ClickHouse/ClickHouse/pull/6780) ([alésapine](https://github.com/alesapin)) -- Mise à jour manuelle de `SIMDJSON`. Cela corrige l’inondation possible des fichiers stderr avec des messages de diagnostic JSON faux. [\#7548](https://github.com/ClickHouse/ClickHouse/pull/7548) ([Alexander Kazakov](https://github.com/Akazz)) -- Correction d’un bug avec `mrk` extension de fichier pour mutations ([alésapine](https://github.com/alesapin)) +- Correction d'un crash rare dans `ALTER MODIFY COLUMN` et fusion verticale lorsque l'une des parties fusionnées/modifiées est vide (0 lignes). [\#6780](https://github.com/ClickHouse/ClickHouse/pull/6780) ([alésapine](https://github.com/alesapin)) +- Mise à jour manuelle de `SIMDJSON`. Cela corrige l'inondation possible des fichiers stderr avec des messages de diagnostic JSON faux. [\#7548](https://github.com/ClickHouse/ClickHouse/pull/7548) ([Alexander Kazakov](https://github.com/Akazz)) +- Correction d'un bug avec `mrk` extension de fichier pour mutations ([alésapine](https://github.com/alesapin)) ### Version De ClickHouse 19.11.12.69, 2019-10-02 {#clickhouse-release-19-11-12-69-2019-10-02} #### Bug Fix {#bug-fix-15} -- Correction de la dégradation des performances de l’analyse d’index sur les clés complexes sur les grandes tables. Cela corrige [\#6924](https://github.com/ClickHouse/ClickHouse/issues/6924). [\#7075](https://github.com/ClickHouse/ClickHouse/pull/7075) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Évitez SIGSEGV rare lors de l’envoi de données dans des tables avec moteur distribué (`Failed to send batch: file with index XXXXX is absent`). [\#7032](https://github.com/ClickHouse/ClickHouse/pull/7032) ([Azat Khuzhin](https://github.com/azat)) +- Correction de la dégradation des performances de l'analyse d'index sur les clés complexes sur les grandes tables. Cela corrige [\#6924](https://github.com/ClickHouse/ClickHouse/issues/6924). [\#7075](https://github.com/ClickHouse/ClickHouse/pull/7075) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Évitez SIGSEGV rare lors de l'envoi de données dans des tables avec moteur distribué (`Failed to send batch: file with index XXXXX is absent`). [\#7032](https://github.com/ClickHouse/ClickHouse/pull/7032) ([Azat Khuzhin](https://github.com/azat)) - Fixer `Unknown identifier` avec plusieurs jointures. Cela corrige [\#5254](https://github.com/ClickHouse/ClickHouse/issues/5254). [\#7022](https://github.com/ClickHouse/ClickHouse/pull/7022) ([Artem Zuikov](https://github.com/4ertus2)) ### Clickhouse Version 19.11.11.57, 2019-09-13 {#clickhouse-release-19-11-11-57-2019-09-13} -- Correction d’une erreur logique provoquant segfaults lors de la sélection de Kafka sujet vide. [\#6902](https://github.com/ClickHouse/ClickHouse/issues/6902) [\#6909](https://github.com/ClickHouse/ClickHouse/pull/6909) ([Ivan](https://github.com/abyss7)) +- Correction d'une erreur logique provoquant segfaults lors de la sélection de Kafka sujet vide. [\#6902](https://github.com/ClickHouse/ClickHouse/issues/6902) [\#6909](https://github.com/ClickHouse/ClickHouse/pull/6909) ([Ivan](https://github.com/abyss7)) - Correctif pour la fonction `АrrayEnumerateUniqRanked` avec des tableaux vides dans params. [\#6928](https://github.com/ClickHouse/ClickHouse/pull/6928) ([proller](https://github.com/proller)) ### Clickhouse Version 19.11.10.54, 2019-09-10 {#clickhouse-release-19-11-10-54-2019-09-10} @@ -915,53 +915,53 @@ toc_title: '2019' ### Clickhouse Version 19.11.9.52, 2019-09-6 {#clickhouse-release-19-11-9-52-2019-09-6} - Améliorer la gestion des erreurs dans les dictionnaires de cache. [\#6737](https://github.com/ClickHouse/ClickHouse/pull/6737) ([Vitaly Baranov](https://github.com/vitlibar)) -- Correction d’un bug dans la fonction `arrayEnumerateUniqRanked`. [\#6779](https://github.com/ClickHouse/ClickHouse/pull/6779) ([proller](https://github.com/proller)) -- Fixer `JSONExtract` fonction lors de l’extraction d’une `Tuple` de JSON. [\#6718](https://github.com/ClickHouse/ClickHouse/pull/6718) ([Vitaly Baranov](https://github.com/vitlibar)) -- Correction possible perte de données après `ALTER DELETE` requête sur la table avec l’index de saut. [\#6224](https://github.com/ClickHouse/ClickHouse/issues/6224) [\#6282](https://github.com/ClickHouse/ClickHouse/pull/6282) ([Nikita Vasilev](https://github.com/nikvas0)) +- Correction d'un bug dans la fonction `arrayEnumerateUniqRanked`. [\#6779](https://github.com/ClickHouse/ClickHouse/pull/6779) ([proller](https://github.com/proller)) +- Fixer `JSONExtract` fonction lors de l'extraction d'une `Tuple` de JSON. [\#6718](https://github.com/ClickHouse/ClickHouse/pull/6718) ([Vitaly Baranov](https://github.com/vitlibar)) +- Correction possible perte de données après `ALTER DELETE` requête sur la table avec l'index de saut. [\#6224](https://github.com/ClickHouse/ClickHouse/issues/6224) [\#6282](https://github.com/ClickHouse/ClickHouse/pull/6282) ([Nikita Vasilev](https://github.com/nikvas0)) - Test de performance fixe. [\#6392](https://github.com/ClickHouse/ClickHouse/pull/6392) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Parquet: Correction de la lecture des colonnes booléennes. [\#6579](https://github.com/ClickHouse/ClickHouse/pull/6579) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Fixe mauvais comportement de `nullIf` fonction pour les arguments constants. [\#6518](https://github.com/ClickHouse/ClickHouse/pull/6518) ([Guillaume Tassery](https://github.com/YiuRULE)) [\#6580](https://github.com/ClickHouse/ClickHouse/pull/6580) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Correction du problème de duplication des messages Kafka lors du redémarrage normal du serveur. [\#6597](https://github.com/ClickHouse/ClickHouse/pull/6597) ([Ivan](https://github.com/abyss7)) -- Correction d’un problème quand long `ALTER UPDATE` ou `ALTER DELETE` peut empêcher régulière fusionne à exécuter. Empêcher les mutations de s’exécuter s’il n’y a pas assez de threads libres disponibles. [\#6502](https://github.com/ClickHouse/ClickHouse/issues/6502) [\#6617](https://github.com/ClickHouse/ClickHouse/pull/6617) ([tavplubix](https://github.com/tavplubix)) -- Correction d’une erreur avec le traitement “timezone” dans le fichier de configuration du serveur. [\#6709](https://github.com/ClickHouse/ClickHouse/pull/6709) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction d'un problème quand long `ALTER UPDATE` ou `ALTER DELETE` peut empêcher régulière fusionne à exécuter. Empêcher les mutations de s'exécuter s'il n'y a pas assez de threads libres disponibles. [\#6502](https://github.com/ClickHouse/ClickHouse/issues/6502) [\#6617](https://github.com/ClickHouse/ClickHouse/pull/6617) ([tavplubix](https://github.com/tavplubix)) +- Correction d'une erreur avec le traitement “timezone” dans le fichier de configuration du serveur. [\#6709](https://github.com/ClickHouse/ClickHouse/pull/6709) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Correction des tests kafka. [\#6805](https://github.com/ClickHouse/ClickHouse/pull/6805) ([Ivan](https://github.com/abyss7)) #### Correction De Sécurité {#security-fix-3} -- Si L’attaquant a un accès en écriture à ZooKeeper et est capable d’exécuter un serveur personnalisé disponible à partir du réseau où clickhouse s’exécute, il peut créer un serveur malveillant personnalisé qui agira comme réplique de ClickHouse et l’enregistrer dans ZooKeeper. Lorsqu’une autre réplique récupère une partie de données à partir d’une réplique malveillante, elle peut forcer clickhouse-server à écrire sur un chemin arbitraire sur le système de fichiers. Trouvé par Eldar Zaitov, équipe de sécurité de L’information chez Yandex. [\#6247](https://github.com/ClickHouse/ClickHouse/pull/6247) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Si L'attaquant a un accès en écriture à ZooKeeper et est capable d'exécuter un serveur personnalisé disponible à partir du réseau où clickhouse s'exécute, il peut créer un serveur malveillant personnalisé qui agira comme réplique de ClickHouse et l'enregistrer dans ZooKeeper. Lorsqu'une autre réplique récupère une partie de données à partir d'une réplique malveillante, elle peut forcer clickhouse-server à écrire sur un chemin arbitraire sur le système de fichiers. Trouvé par Eldar Zaitov, équipe de sécurité de L'information chez Yandex. [\#6247](https://github.com/ClickHouse/ClickHouse/pull/6247) ([alexeï-milovidov](https://github.com/alexey-milovidov)) ### Clickhouse Version 19.11.8.46, 2019-08-22 {#clickhouse-release-19-11-8-46-2019-08-22} #### Bug Fix {#bug-fix-17} - Fixer `ALTER TABLE ... UPDATE` requête pour les tables avec `enable_mixed_granularity_parts=1`. [\#6543](https://github.com/ClickHouse/ClickHouse/pull/6543) ([alésapine](https://github.com/alesapin)) -- Correction de NPE lors de l’utilisation de la clause IN avec une sous-requête avec un tuple. [\#6125](https://github.com/ClickHouse/ClickHouse/issues/6125) [\#6550](https://github.com/ClickHouse/ClickHouse/pull/6550) ([tavplubix](https://github.com/tavplubix)) -- Correction d’un problème que si un réplica périmé devient vivant, il peut encore avoir des parties de données qui ont été supprimés par la partition DROP. [\#6522](https://github.com/ClickHouse/ClickHouse/issues/6522) [\#6523](https://github.com/ClickHouse/ClickHouse/pull/6523) ([tavplubix](https://github.com/tavplubix)) -- Correction d’un problème avec l’analyse CSV [\#6426](https://github.com/ClickHouse/ClickHouse/issues/6426) [\#6559](https://github.com/ClickHouse/ClickHouse/pull/6559) ([tavplubix](https://github.com/tavplubix)) +- Correction de NPE lors de l'utilisation de la clause IN avec une sous-requête avec un tuple. [\#6125](https://github.com/ClickHouse/ClickHouse/issues/6125) [\#6550](https://github.com/ClickHouse/ClickHouse/pull/6550) ([tavplubix](https://github.com/tavplubix)) +- Correction d'un problème que si un réplica périmé devient vivant, il peut encore avoir des parties de données qui ont été supprimés par la partition DROP. [\#6522](https://github.com/ClickHouse/ClickHouse/issues/6522) [\#6523](https://github.com/ClickHouse/ClickHouse/pull/6523) ([tavplubix](https://github.com/tavplubix)) +- Correction d'un problème avec l'analyse CSV [\#6426](https://github.com/ClickHouse/ClickHouse/issues/6426) [\#6559](https://github.com/ClickHouse/ClickHouse/pull/6559) ([tavplubix](https://github.com/tavplubix)) - Course de données fixe dans le système.table de pièces et ALTER query. Cela corrige [\#6245](https://github.com/ClickHouse/ClickHouse/issues/6245). [\#6513](https://github.com/ClickHouse/ClickHouse/pull/6513) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction du mauvais code dans les mutations qui peuvent conduire à la corruption de la mémoire. Correction de segfault avec lecture de l’adresse `0x14c0` cela peut se produire en raison de simultané `DROP TABLE` et `SELECT` de `system.parts` ou `system.parts_columns`. Condition de course fixe dans la préparation des requêtes de mutation. Blocage fixe causé par `OPTIMIZE` des tables répliquées et des opérations de modification simultanées comme ALTERs. [\#6514](https://github.com/ClickHouse/ClickHouse/pull/6514) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction du mauvais code dans les mutations qui peuvent conduire à la corruption de la mémoire. Correction de segfault avec lecture de l'adresse `0x14c0` cela peut se produire en raison de simultané `DROP TABLE` et `SELECT` de `system.parts` ou `system.parts_columns`. Condition de course fixe dans la préparation des requêtes de mutation. Blocage fixe causé par `OPTIMIZE` des tables répliquées et des opérations de modification simultanées comme ALTERs. [\#6514](https://github.com/ClickHouse/ClickHouse/pull/6514) ([alexeï-milovidov](https://github.com/alexey-milovidov)) ### Clickhouse Version 19.11.7.40, 2019-08-14 {#clickhouse-release-19-11-7-40-2019-08-14} #### Bug Fix {#bug-fix-18} -- L’intégration de Kafka a été corrigée dans cette version. -- Correction de segfault lors de l’utilisation `arrayReduce` pour les querelles constantes. [\#6326](https://github.com/ClickHouse/ClickHouse/pull/6326) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- L'intégration de Kafka a été corrigée dans cette version. +- Correction de segfault lors de l'utilisation `arrayReduce` pour les querelles constantes. [\#6326](https://github.com/ClickHouse/ClickHouse/pull/6326) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Fixe `toFloat()` monotonie. [\#6374](https://github.com/ClickHouse/ClickHouse/pull/6374) ([dimarub2000](https://github.com/dimarub2000)) - Correction de segfault avec activé `optimize_skip_unused_shards` et clé de sharding manquante. [\#6384](https://github.com/ClickHouse/ClickHouse/pull/6384) ([CurtizJ](https://github.com/CurtizJ)) - Fixe la logique de `arrayEnumerateUniqRanked` fonction. [\#6423](https://github.com/ClickHouse/ClickHouse/pull/6423) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Suppression de la journalisation supplémentaire du gestionnaire MySQL. [\#6389](https://github.com/ClickHouse/ClickHouse/pull/6389) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’un comportement incorrect et de segfaults possibles dans `topK` et `topKWeighted` agrégé fonctions. [\#6404](https://github.com/ClickHouse/ClickHouse/pull/6404) ([CurtizJ](https://github.com/CurtizJ)) -- N’exposez pas les colonnes virtuelles dans `system.columns` table. Ceci est nécessaire pour la compatibilité descendante. [\#6406](https://github.com/ClickHouse/ClickHouse/pull/6406) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’un bug avec l’allocation de mémoire pour les champs de chaîne dans le dictionnaire de cache de clé complexe. [\#6447](https://github.com/ClickHouse/ClickHouse/pull/6447) ([alésapine](https://github.com/alesapin)) -- Correction d’un bug avec l’activation de la granularité adaptative lors de la création d’une nouvelle réplique pour `Replicated*MergeTree` table. [\#6452](https://github.com/ClickHouse/ClickHouse/pull/6452) ([alésapine](https://github.com/alesapin)) +- Correction d'un comportement incorrect et de segfaults possibles dans `topK` et `topKWeighted` agrégé fonctions. [\#6404](https://github.com/ClickHouse/ClickHouse/pull/6404) ([CurtizJ](https://github.com/CurtizJ)) +- N'exposez pas les colonnes virtuelles dans `system.columns` table. Ceci est nécessaire pour la compatibilité descendante. [\#6406](https://github.com/ClickHouse/ClickHouse/pull/6406) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction d'un bug avec l'allocation de mémoire pour les champs de chaîne dans le dictionnaire de cache de clé complexe. [\#6447](https://github.com/ClickHouse/ClickHouse/pull/6447) ([alésapine](https://github.com/alesapin)) +- Correction d'un bug avec l'activation de la granularité adaptative lors de la création d'une nouvelle réplique pour `Replicated*MergeTree` table. [\#6452](https://github.com/ClickHouse/ClickHouse/pull/6452) ([alésapine](https://github.com/alesapin)) - Correction de la boucle infinie lors de la lecture des messages Kafka. [\#6354](https://github.com/ClickHouse/ClickHouse/pull/6354) ([abyss7](https://github.com/abyss7)) -- Correction de la possibilité d’une requête fabriquée pour provoquer un crash du serveur en raison d’un débordement de pile dans L’analyseur SQL et de la possibilité d’un débordement de pile dans `Merge` et `Distributed` table [\#6433](https://github.com/ClickHouse/ClickHouse/pull/6433) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction D’une erreur D’encodage de gorille sur les petites séquences. [\#6444](https://github.com/ClickHouse/ClickHouse/pull/6444) ([Enmk](https://github.com/Enmk)) +- Correction de la possibilité d'une requête fabriquée pour provoquer un crash du serveur en raison d'un débordement de pile dans L'analyseur SQL et de la possibilité d'un débordement de pile dans `Merge` et `Distributed` table [\#6433](https://github.com/ClickHouse/ClickHouse/pull/6433) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction D'une erreur D'encodage de gorille sur les petites séquences. [\#6444](https://github.com/ClickHouse/ClickHouse/pull/6444) ([Enmk](https://github.com/Enmk)) #### Amélioration {#improvement-5} -- Autoriser l’utilisateur à remplacer `poll_interval` et `idle_connection_timeout` paramètres de connexion. [\#6230](https://github.com/ClickHouse/ClickHouse/pull/6230) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Autoriser l'utilisateur à remplacer `poll_interval` et `idle_connection_timeout` paramètres de connexion. [\#6230](https://github.com/ClickHouse/ClickHouse/pull/6230) ([alexeï-milovidov](https://github.com/alexey-milovidov)) ### Clickhouse Version 19.11.5.28, 2019-08-05 {#clickhouse-release-19-11-5-28-2019-08-05} @@ -969,9 +969,9 @@ toc_title: '2019' - Correction de la possibilité de suspendre les requêtes lorsque le serveur est surchargé. [\#6301](https://github.com/ClickHouse/ClickHouse/pull/6301) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Correction de FPE dans la fonction yandexconsistenthash. Cela corrige [\#6304](https://github.com/ClickHouse/ClickHouse/issues/6304). [\#6126](https://github.com/ClickHouse/ClickHouse/pull/6126) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’un bug dans la conversion de `LowCardinality` types de `AggregateFunctionFactory`. Cela corrige [\#6257](https://github.com/ClickHouse/ClickHouse/issues/6257). [\#6281](https://github.com/ClickHouse/ClickHouse/pull/6281) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Corrigé de l’analyse de `bool` les paramètres de `true` et `false` chaînes de caractères dans les fichiers de configuration. [\#6278](https://github.com/ClickHouse/ClickHouse/pull/6278) ([alésapine](https://github.com/alesapin)) -- Correction d’un bug rare avec des en-têtes de flux incompatibles dans les requêtes `Distributed` table de `MergeTree` table quand une partie de `WHERE` se déplace à `PREWHERE`. [\#6236](https://github.com/ClickHouse/ClickHouse/pull/6236) ([alésapine](https://github.com/alesapin)) +- Correction d'un bug dans la conversion de `LowCardinality` types de `AggregateFunctionFactory`. Cela corrige [\#6257](https://github.com/ClickHouse/ClickHouse/issues/6257). [\#6281](https://github.com/ClickHouse/ClickHouse/pull/6281) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Corrigé de l'analyse de `bool` les paramètres de `true` et `false` chaînes de caractères dans les fichiers de configuration. [\#6278](https://github.com/ClickHouse/ClickHouse/pull/6278) ([alésapine](https://github.com/alesapin)) +- Correction d'un bug rare avec des en-têtes de flux incompatibles dans les requêtes `Distributed` table de `MergeTree` table quand une partie de `WHERE` se déplace à `PREWHERE`. [\#6236](https://github.com/ClickHouse/ClickHouse/pull/6236) ([alésapine](https://github.com/alesapin)) - Dépassement fixe dans la division entière du type signé au type non signé. Cela corrige [\#6214](https://github.com/ClickHouse/ClickHouse/issues/6214). [\#6233](https://github.com/ClickHouse/ClickHouse/pull/6233) ([alexeï-milovidov](https://github.com/alexey-milovidov)) #### Modification Incompatible En Arrière {#backward-incompatible-change-5} @@ -982,21 +982,21 @@ toc_title: '2019' #### Bug Fix {#bug-fix-20} -- Correction d’un bug avec l’écriture de marques d’indices secondaires avec une granularité adaptative. [\#6126](https://github.com/ClickHouse/ClickHouse/pull/6126) ([alésapine](https://github.com/alesapin)) +- Correction d'un bug avec l'écriture de marques d'indices secondaires avec une granularité adaptative. [\#6126](https://github.com/ClickHouse/ClickHouse/pull/6126) ([alésapine](https://github.com/alesapin)) - Fixer `WITH ROLLUP` et `WITH CUBE` les modificateurs de `GROUP BY` avec agrégation à deux niveaux. [\#6225](https://github.com/ClickHouse/ClickHouse/pull/6225) ([Anton Popov](https://github.com/CurtizJ)) - Correction du blocage dans `JSONExtractRaw` fonction. Fixe [\#6195](https://github.com/ClickHouse/ClickHouse/issues/6195) [\#6198](https://github.com/ClickHouse/ClickHouse/pull/6198) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Correction de segfault dans ExternalLoader::reloadOutdated (). [\#6082](https://github.com/ClickHouse/ClickHouse/pull/6082) ([Vitaly Baranov](https://github.com/vitlibar)) -- Correction du cas où le serveur peut fermer les sockets d’écoute mais ne pas arrêter et continuer à servir les requêtes restantes. Vous pouvez vous retrouver avec deux processus clickhouse-server en cours d’exécution. Parfois, le serveur peut renvoyer une erreur `bad_function_call` pour les requêtes restantes. [\#6231](https://github.com/ClickHouse/ClickHouse/pull/6231) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’une condition inutile et incorrecte sur le champ de mise à jour pour le chargement initial des dictionnaires externes via ODBC, MySQL, ClickHouse et HTTP. Cela corrige [\#6069](https://github.com/ClickHouse/ClickHouse/issues/6069) [\#6083](https://github.com/ClickHouse/ClickHouse/pull/6083) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’une exception non pertinente dans la distribution de `LowCardinality(Nullable)` to not-Nullable column in case if it doesn’t contain Nulls (e.g. in query like `SELECT CAST(CAST('Hello' AS LowCardinality(Nullable(String))) AS String)`. [\#6094](https://github.com/ClickHouse/ClickHouse/issues/6094) [\#6119](https://github.com/ClickHouse/ClickHouse/pull/6119) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Correction d’un résultat non déterministe de “uniq” fonction agrégée dans des cas extrêmement rares. Le bug était présent dans toutes les versions de ClickHouse. [\#6058](https://github.com/ClickHouse/ClickHouse/pull/6058) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction du cas où le serveur peut fermer les sockets d'écoute mais ne pas arrêter et continuer à servir les requêtes restantes. Vous pouvez vous retrouver avec deux processus clickhouse-server en cours d'exécution. Parfois, le serveur peut renvoyer une erreur `bad_function_call` pour les requêtes restantes. [\#6231](https://github.com/ClickHouse/ClickHouse/pull/6231) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction d'une condition inutile et incorrecte sur le champ de mise à jour pour le chargement initial des dictionnaires externes via ODBC, MySQL, ClickHouse et HTTP. Cela corrige [\#6069](https://github.com/ClickHouse/ClickHouse/issues/6069) [\#6083](https://github.com/ClickHouse/ClickHouse/pull/6083) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction d'une exception non pertinente dans la distribution de `LowCardinality(Nullable)` to not-Nullable column in case if it doesn't contain Nulls (e.g. in query like `SELECT CAST(CAST('Hello' AS LowCardinality(Nullable(String))) AS String)`. [\#6094](https://github.com/ClickHouse/ClickHouse/issues/6094) [\#6119](https://github.com/ClickHouse/ClickHouse/pull/6119) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Correction d'un résultat non déterministe de “uniq” fonction agrégée dans des cas extrêmement rares. Le bug était présent dans toutes les versions de ClickHouse. [\#6058](https://github.com/ClickHouse/ClickHouse/pull/6058) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Segfault lorsque nous définissons un peu trop haut CIDR sur la fonction `IPv6CIDRToRange`. [\#6068](https://github.com/ClickHouse/ClickHouse/pull/6068) ([Guillaume Tassery](https://github.com/YiuRULE)) -- Correction d’une petite fuite de mémoire lorsque le serveur lance de nombreuses exceptions dans de nombreux contextes différents. [\#6144](https://github.com/ClickHouse/ClickHouse/pull/6144) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Corrigez la situation lorsque le consommateur a été mis en pause avant l’abonnement et n’a pas repris après. [\#6075](https://github.com/ClickHouse/ClickHouse/pull/6075) ([Ivan](https://github.com/abyss7)) Notez que Kafka est cassé dans cette version. -- Effacement du tampon de données Kafka de l’opération de lecture précédente terminée par une erreur [\#6026](https://github.com/ClickHouse/ClickHouse/pull/6026) ([Nikolay](https://github.com/bopohaa)) Notez que Kafka est cassé dans cette version. -- Depuis `StorageMergeTree::background_task_handle` est initialisée dans `startup()` le `MergeTreeBlockOutputStream::write()` peut tenter de l’utiliser avant l’initialisation. Vérifiez simplement s’il est initialisé. [\#6080](https://github.com/ClickHouse/ClickHouse/pull/6080) ([Ivan](https://github.com/abyss7)) +- Correction d'une petite fuite de mémoire lorsque le serveur lance de nombreuses exceptions dans de nombreux contextes différents. [\#6144](https://github.com/ClickHouse/ClickHouse/pull/6144) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Corrigez la situation lorsque le consommateur a été mis en pause avant l'abonnement et n'a pas repris après. [\#6075](https://github.com/ClickHouse/ClickHouse/pull/6075) ([Ivan](https://github.com/abyss7)) Notez que Kafka est cassé dans cette version. +- Effacement du tampon de données Kafka de l'opération de lecture précédente terminée par une erreur [\#6026](https://github.com/ClickHouse/ClickHouse/pull/6026) ([Nikolay](https://github.com/bopohaa)) Notez que Kafka est cassé dans cette version. +- Depuis `StorageMergeTree::background_task_handle` est initialisée dans `startup()` le `MergeTreeBlockOutputStream::write()` peut tenter de l'utiliser avant l'initialisation. Vérifiez simplement s'il est initialisé. [\#6080](https://github.com/ClickHouse/ClickHouse/pull/6080) ([Ivan](https://github.com/abyss7)) -#### Construction / Test / Amélioration De L’Emballage {#buildtestingpackaging-improvement-6} +#### Construction / Test / Amélioration De L'Emballage {#buildtestingpackaging-improvement-6} - Ajouté officiel `rpm` paquet. [\#5740](https://github.com/ClickHouse/ClickHouse/pull/5740) ([proller](https://github.com/proller)) ([alésapine](https://github.com/alesapin)) - Ajouter une capacité à construire `.rpm` et `.tgz` les paquets avec `packager` script. [\#5769](https://github.com/ClickHouse/ClickHouse/pull/5769) ([alésapine](https://github.com/alesapin)) @@ -1012,101 +1012,101 @@ toc_title: '2019' - Ajout du support pour les déclarations préparées. [\#5331](https://github.com/ClickHouse/ClickHouse/pull/5331/) ([Alexander](https://github.com/sanych73)) [\#5630](https://github.com/ClickHouse/ClickHouse/pull/5630) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - `DoubleDelta` et `Gorilla` codecs de colonne [\#5600](https://github.com/ClickHouse/ClickHouse/pull/5600) ([Vasily Nemkov](https://github.com/Enmk)) -- Ajouter `os_thread_priority` paramètre qui permet de contrôler la “nice” valeur des threads de traitement de requête utilisés par le système d’exploitation pour ajuster la priorité de planification dynamique. Il exige `CAP_SYS_NICE` capacités à travailler. Cela met en œuvre [\#5858](https://github.com/ClickHouse/ClickHouse/issues/5858) [\#5909](https://github.com/ClickHouse/ClickHouse/pull/5909) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Ajouter `os_thread_priority` paramètre qui permet de contrôler la “nice” valeur des threads de traitement de requête utilisés par le système d'exploitation pour ajuster la priorité de planification dynamique. Il exige `CAP_SYS_NICE` capacités à travailler. Cela met en œuvre [\#5858](https://github.com/ClickHouse/ClickHouse/issues/5858) [\#5909](https://github.com/ClickHouse/ClickHouse/pull/5909) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Mettre `_topic`, `_offset`, `_key` colonnes pour moteur Kafka [\#5382](https://github.com/ClickHouse/ClickHouse/pull/5382) ([Ivan](https://github.com/abyss7)) Notez que Kafka est cassé dans cette version. - Ajouter agrégat fonction combinateur `-Resample` [\#5590](https://github.com/ClickHouse/ClickHouse/pull/5590) ([hcz](https://github.com/hczhcz)) -- Les fonctions d’agrégation `groupArrayMovingSum(win_size)(x)` et `groupArrayMovingAvg(win_size)(x)`, qui calculent la somme mobile / avg avec ou sans limitation de taille de fenêtre. [\#5595](https://github.com/ClickHouse/ClickHouse/pull/5595) ([inv2004](https://github.com/inv2004)) +- Les fonctions d'agrégation `groupArrayMovingSum(win_size)(x)` et `groupArrayMovingAvg(win_size)(x)`, qui calculent la somme mobile / avg avec ou sans limitation de taille de fenêtre. [\#5595](https://github.com/ClickHouse/ClickHouse/pull/5595) ([inv2004](https://github.com/inv2004)) - Ajouter synonim `arrayFlatten` \<-\> `flatten` [\#5764](https://github.com/ClickHouse/ClickHouse/pull/5764) ([hcz](https://github.com/hczhcz)) - Fonction Intergate H3 `geoToH3` de Uber. [\#4724](https://github.com/ClickHouse/ClickHouse/pull/4724) ([Remen Ivan](https://github.com/BHYCHIK)) [\#5805](https://github.com/ClickHouse/ClickHouse/pull/5805) ([alexeï-milovidov](https://github.com/alexey-milovidov)) #### Bug Fix {#bug-fix-21} -- Implémentez le cache DNS avec une mise à jour asynchrone. Thread séparé résout tous les hôtes et met à jour le cache DNS avec la période (Paramètre `dns_cache_update_period`). Cela devrait aider, lorsque l’adresse ip des hôtes change fréquemment. [\#5857](https://github.com/ClickHouse/ClickHouse/pull/5857) ([Anton Popov](https://github.com/CurtizJ)) +- Implémentez le cache DNS avec une mise à jour asynchrone. Thread séparé résout tous les hôtes et met à jour le cache DNS avec la période (Paramètre `dns_cache_update_period`). Cela devrait aider, lorsque l'adresse ip des hôtes change fréquemment. [\#5857](https://github.com/ClickHouse/ClickHouse/pull/5857) ([Anton Popov](https://github.com/CurtizJ)) - Fixer erreur de segmentation dans `Delta` codec qui affecte les colonnes avec des valeurs inférieures à 32 bits. Le bug a conduit à la corruption de la mémoire aléatoire. [\#5786](https://github.com/ClickHouse/ClickHouse/pull/5786) ([alésapine](https://github.com/alesapin)) - Correction de segfault dans la fusion TTL avec des colonnes non physiques en bloc. [\#5819](https://github.com/ClickHouse/ClickHouse/pull/5819) ([Anton Popov](https://github.com/CurtizJ)) -- Correction d’un bug rare dans la vérification de la pièce avec `LowCardinality` colonne. Précédemment `checkDataPart` échoue toujours pour une partie avec `LowCardinality` colonne. [\#5832](https://github.com/ClickHouse/ClickHouse/pull/5832) ([alésapine](https://github.com/alesapin)) -- Évitez de suspendre les connexions lorsque le pool de threads du serveur est plein. Il est important pour les connexions de `remote` fonction de table ou connexions à un fragment sans réplicas lorsqu’il y a un long délai de connexion. Cela corrige [\#5878](https://github.com/ClickHouse/ClickHouse/issues/5878) [\#5881](https://github.com/ClickHouse/ClickHouse/pull/5881) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction d'un bug rare dans la vérification de la pièce avec `LowCardinality` colonne. Précédemment `checkDataPart` échoue toujours pour une partie avec `LowCardinality` colonne. [\#5832](https://github.com/ClickHouse/ClickHouse/pull/5832) ([alésapine](https://github.com/alesapin)) +- Évitez de suspendre les connexions lorsque le pool de threads du serveur est plein. Il est important pour les connexions de `remote` fonction de table ou connexions à un fragment sans réplicas lorsqu'il y a un long délai de connexion. Cela corrige [\#5878](https://github.com/ClickHouse/ClickHouse/issues/5878) [\#5881](https://github.com/ClickHouse/ClickHouse/pull/5881) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Prise en charge des arguments constants pour `evalMLModel` fonction. Cela corrige [\#5817](https://github.com/ClickHouse/ClickHouse/issues/5817) [\#5820](https://github.com/ClickHouse/ClickHouse/pull/5820) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Correction du problème lorsque ClickHouse détermine le fuseau horaire par défaut comme `UCT` plutôt `UTC`. Cela corrige [\#5804](https://github.com/ClickHouse/ClickHouse/issues/5804). [\#5828](https://github.com/ClickHouse/ClickHouse/pull/5828) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Fixe de type dépassement de tampon dans `visitParamExtractRaw`. Cela corrige [\#5901](https://github.com/ClickHouse/ClickHouse/issues/5901) [\#5902](https://github.com/ClickHouse/ClickHouse/pull/5902) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Maintenant distribué `DROP/ALTER/TRUNCATE/OPTIMIZE ON CLUSTER` les requêtes seront exécutées directement sur la réplique leader. [\#5757](https://github.com/ClickHouse/ClickHouse/pull/5757) ([alésapine](https://github.com/alesapin)) - Fixer `coalesce` pour `ColumnConst` avec `ColumnNullable` + changements connexes. [\#5755](https://github.com/ClickHouse/ClickHouse/pull/5755) ([Artem Zuikov](https://github.com/4ertus2)) -- Fixer le `ReadBufferFromKafkaConsumer` alors qu’il continue à lire de nouveaux messages après `commit()` même si elle a été interrompue avant [\#5852](https://github.com/ClickHouse/ClickHouse/pull/5852) ([Ivan](https://github.com/abyss7)) -- Fixer `FULL` et `RIGHT` Joindre les résultats lors de l’adhésion sur `Nullable` clés dans la table de droite. [\#5859](https://github.com/ClickHouse/ClickHouse/pull/5859) ([Artem Zuikov](https://github.com/4ertus2)) +- Fixer le `ReadBufferFromKafkaConsumer` alors qu'il continue à lire de nouveaux messages après `commit()` même si elle a été interrompue avant [\#5852](https://github.com/ClickHouse/ClickHouse/pull/5852) ([Ivan](https://github.com/abyss7)) +- Fixer `FULL` et `RIGHT` Joindre les résultats lors de l'adhésion sur `Nullable` clés dans la table de droite. [\#5859](https://github.com/ClickHouse/ClickHouse/pull/5859) ([Artem Zuikov](https://github.com/4ertus2)) - Correction Possible du sommeil infini des requêtes de faible priorité. [\#5842](https://github.com/ClickHouse/ClickHouse/pull/5842) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Correction de la condition de concurrence, qui fait que certaines requêtes peuvent ne pas apparaître dans query\_log après `SYSTEM FLUSH LOGS` requête. [\#5456](https://github.com/ClickHouse/ClickHouse/issues/5456) [\#5685](https://github.com/ClickHouse/ClickHouse/pull/5685) ([Anton Popov](https://github.com/CurtizJ)) -- Fixe `heap-use-after-free` Avertissement ASan dans ClusterCopier causé par la montre qui essaie d’utiliser l’objet copieur déjà supprimé. [\#5871](https://github.com/ClickHouse/ClickHouse/pull/5871) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Fixe de mal `StringRef` pointeur retourné par certaines implémentations de `IColumn::deserializeAndInsertFromArena`. Ce bogue n’a affecté que les tests unitaires. [\#5973](https://github.com/ClickHouse/ClickHouse/pull/5973) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Fixe `heap-use-after-free` Avertissement ASan dans ClusterCopier causé par la montre qui essaie d'utiliser l'objet copieur déjà supprimé. [\#5871](https://github.com/ClickHouse/ClickHouse/pull/5871) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Fixe de mal `StringRef` pointeur retourné par certaines implémentations de `IColumn::deserializeAndInsertFromArena`. Ce bogue n'a affecté que les tests unitaires. [\#5973](https://github.com/ClickHouse/ClickHouse/pull/5973) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) - Empêcher les colonnes de jointure de tableau source et intermédiaire de masquer les colonnes de même nom. [\#5941](https://github.com/ClickHouse/ClickHouse/pull/5941) ([Artem Zuikov](https://github.com/4ertus2)) -- Fixer insérer et sélectionner la requête au moteur MySQL avec l’identifiant de style MySQL citant. [\#5704](https://github.com/ClickHouse/ClickHouse/pull/5704) ([L’Hiver Zhang](https://github.com/zhang2014)) -- Maintenant `CHECK TABLE` query peut fonctionner avec la famille de moteurs MergeTree. Il renvoie l’état de contrôle et le message le cas échéant pour chaque partie (ou fichier dans le cas de moteurs plus simples). Aussi, correction d’un bug dans l’extraction d’une partie cassée. [\#5865](https://github.com/ClickHouse/ClickHouse/pull/5865) ([alésapine](https://github.com/alesapin)) -- Correction de L’exécution SPLIT\_SHARED\_LIBRARIES [\#5793](https://github.com/ClickHouse/ClickHouse/pull/5793) ([Danila Kutenin](https://github.com/danlark1)) -- Correction de l’initialisation du fuseau horaire lorsque `/etc/localtime` est un lien symbolique comme `../usr/share/zoneinfo/Europe/Moscow` [\#5922](https://github.com/ClickHouse/ClickHouse/pull/5922) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- clickhouse-copieur: Fix utiliser-après livraison à l’arrêt [\#5752](https://github.com/ClickHouse/ClickHouse/pull/5752) ([proller](https://github.com/proller)) +- Fixer insérer et sélectionner la requête au moteur MySQL avec l'identifiant de style MySQL citant. [\#5704](https://github.com/ClickHouse/ClickHouse/pull/5704) ([L'Hiver Zhang](https://github.com/zhang2014)) +- Maintenant `CHECK TABLE` query peut fonctionner avec la famille de moteurs MergeTree. Il renvoie l'état de contrôle et le message le cas échéant pour chaque partie (ou fichier dans le cas de moteurs plus simples). Aussi, correction d'un bug dans l'extraction d'une partie cassée. [\#5865](https://github.com/ClickHouse/ClickHouse/pull/5865) ([alésapine](https://github.com/alesapin)) +- Correction de L'exécution SPLIT\_SHARED\_LIBRARIES [\#5793](https://github.com/ClickHouse/ClickHouse/pull/5793) ([Danila Kutenin](https://github.com/danlark1)) +- Correction de l'initialisation du fuseau horaire lorsque `/etc/localtime` est un lien symbolique comme `../usr/share/zoneinfo/Europe/Moscow` [\#5922](https://github.com/ClickHouse/ClickHouse/pull/5922) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- clickhouse-copieur: Fix utiliser-après livraison à l'arrêt [\#5752](https://github.com/ClickHouse/ClickHouse/pull/5752) ([proller](https://github.com/proller)) - Mettre `simdjson`. Correction du problème que certains JSONs invalides avec zéro octets analysent avec succès. [\#5938](https://github.com/ClickHouse/ClickHouse/pull/5938) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction de l’arrêt des SystemLogs [\#5802](https://github.com/ClickHouse/ClickHouse/pull/5802) ([Anton Popov](https://github.com/CurtizJ)) -- Correction de la suspension lorsque la condition dans invalidate\_query dépend d’un dictionnaire. [\#6011](https://github.com/ClickHouse/ClickHouse/pull/6011) ([Vitaly Baranov](https://github.com/vitlibar)) +- Correction de l'arrêt des SystemLogs [\#5802](https://github.com/ClickHouse/ClickHouse/pull/5802) ([Anton Popov](https://github.com/CurtizJ)) +- Correction de la suspension lorsque la condition dans invalidate\_query dépend d'un dictionnaire. [\#6011](https://github.com/ClickHouse/ClickHouse/pull/6011) ([Vitaly Baranov](https://github.com/vitlibar)) #### Amélioration {#improvement-6} - Autoriser les adresses insolubles dans la configuration du cluster. Ils seront considérés comme indisponibles et essayés de résoudre à chaque tentative de connexion. Ceci est particulièrement utile pour Kubernetes. Cela corrige [\#5714](https://github.com/ClickHouse/ClickHouse/issues/5714) [\#5924](https://github.com/ClickHouse/ClickHouse/pull/5924) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Fermez les connexions TCP inactives (avec un délai d’attente d’une heure par défaut). Ceci est particulièrement important pour les grands clusters avec plusieurs tables distribuées sur chaque serveur, car chaque serveur peut éventuellement conserver un pool de connexions à tous les autres serveurs, et après la concurrence maximale des requêtes, les connexions calent. Cela corrige [\#5879](https://github.com/ClickHouse/ClickHouse/issues/5879) [\#5880](https://github.com/ClickHouse/ClickHouse/pull/5880) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Meilleure qualité de `topK` fonction. Modification du comportement de L’ensemble SavingSpace pour supprimer le dernier élément si le nouvel élément a un poids plus important. [\#5833](https://github.com/ClickHouse/ClickHouse/issues/5833) [\#5850](https://github.com/ClickHouse/ClickHouse/pull/5850) ([Guillaume Tassery](https://github.com/YiuRULE)) -- Les fonctions D’URL pour travailler avec des domaines peuvent maintenant fonctionner pour les URL incomplètes sans schéma [\#5725](https://github.com/ClickHouse/ClickHouse/pull/5725) ([alésapine](https://github.com/alesapin)) +- Fermez les connexions TCP inactives (avec un délai d'attente d'une heure par défaut). Ceci est particulièrement important pour les grands clusters avec plusieurs tables distribuées sur chaque serveur, car chaque serveur peut éventuellement conserver un pool de connexions à tous les autres serveurs, et après la concurrence maximale des requêtes, les connexions calent. Cela corrige [\#5879](https://github.com/ClickHouse/ClickHouse/issues/5879) [\#5880](https://github.com/ClickHouse/ClickHouse/pull/5880) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Meilleure qualité de `topK` fonction. Modification du comportement de L'ensemble SavingSpace pour supprimer le dernier élément si le nouvel élément a un poids plus important. [\#5833](https://github.com/ClickHouse/ClickHouse/issues/5833) [\#5850](https://github.com/ClickHouse/ClickHouse/pull/5850) ([Guillaume Tassery](https://github.com/YiuRULE)) +- Les fonctions D'URL pour travailler avec des domaines peuvent maintenant fonctionner pour les URL incomplètes sans schéma [\#5725](https://github.com/ClickHouse/ClickHouse/pull/5725) ([alésapine](https://github.com/alesapin)) - Sommes de contrôle ajoutées à la `system.parts_columns` table. [\#5874](https://github.com/ClickHouse/ClickHouse/pull/5874) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) - Ajouter `Enum` type de données comme synonyme de `Enum8` ou `Enum16`. [\#5886](https://github.com/ClickHouse/ClickHouse/pull/5886) ([dimarub2000](https://github.com/dimarub2000)) - Variante de transposition de bits complète pour `T64` codec. Pourrait conduire à une meilleure compression avec `zstd`. [\#5742](https://github.com/ClickHouse/ClickHouse/pull/5742) ([Artem Zuikov](https://github.com/4ertus2)) - Condition sur `startsWith` fonction maintenant peut utilise la clé primaire. Cela corrige [\#5310](https://github.com/ClickHouse/ClickHouse/issues/5310) et [\#5882](https://github.com/ClickHouse/ClickHouse/issues/5882) [\#5919](https://github.com/ClickHouse/ClickHouse/pull/5919) ([dimarub2000](https://github.com/dimarub2000)) -- Permettre d’utiliser `clickhouse-copier` avec la topologie de cluster de réplication croisée en permettant le nom de base de données vide. [\#5745](https://github.com/ClickHouse/ClickHouse/pull/5745) ([nvartolomei](https://github.com/nvartolomei)) +- Permettre d'utiliser `clickhouse-copier` avec la topologie de cluster de réplication croisée en permettant le nom de base de données vide. [\#5745](https://github.com/ClickHouse/ClickHouse/pull/5745) ([nvartolomei](https://github.com/nvartolomei)) - Utiliser `UTC` comme fuseau horaire par défaut sur un système sans `tzdata` (e.g. bare Docker container). Before this patch, error message `Could not determine local time zone` a été imprimé et le serveur ou le client a refusé de démarrer. [\#5827](https://github.com/ClickHouse/ClickHouse/pull/5827) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Retourné soutien à virgule flottante argument dans la fonction `quantileTiming` pour la compatibilité descendante. [\#5911](https://github.com/ClickHouse/ClickHouse/pull/5911) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Afficher quelle table est manquante colonne dans les messages d’erreur. [\#5768](https://github.com/ClickHouse/ClickHouse/pull/5768) ([Ivan](https://github.com/abyss7)) -- Interdire l’exécution d’une requête avec le même query\_id par divers utilisateurs [\#5430](https://github.com/ClickHouse/ClickHouse/pull/5430) ([proller](https://github.com/proller)) +- Afficher quelle table est manquante colonne dans les messages d'erreur. [\#5768](https://github.com/ClickHouse/ClickHouse/pull/5768) ([Ivan](https://github.com/abyss7)) +- Interdire l'exécution d'une requête avec le même query\_id par divers utilisateurs [\#5430](https://github.com/ClickHouse/ClickHouse/pull/5430) ([proller](https://github.com/proller)) - Code plus robuste pour envoyer des métriques au Graphite. Cela fonctionnera même pendant de longues périodes multiples `RENAME TABLE` opération. [\#5875](https://github.com/ClickHouse/ClickHouse/pull/5875) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Des messages d’erreur plus informatifs seront affichés lorsque ThreadPool ne peut pas planifier une tâche pour l’exécution. Cela corrige [\#5305](https://github.com/ClickHouse/ClickHouse/issues/5305) [\#5801](https://github.com/ClickHouse/ClickHouse/pull/5801) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Des messages d'erreur plus informatifs seront affichés lorsque ThreadPool ne peut pas planifier une tâche pour l'exécution. Cela corrige [\#5305](https://github.com/ClickHouse/ClickHouse/issues/5305) [\#5801](https://github.com/ClickHouse/ClickHouse/pull/5801) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Inverser ngramSearch pour être plus intuitif [\#5807](https://github.com/ClickHouse/ClickHouse/pull/5807) ([Danila Kutenin](https://github.com/danlark1)) -- Ajouter l’analyse utilisateur dans HDFS engine builder [\#5946](https://github.com/ClickHouse/ClickHouse/pull/5946) ([akonyaev90](https://github.com/akonyaev90)) +- Ajouter l'analyse utilisateur dans HDFS engine builder [\#5946](https://github.com/ClickHouse/ClickHouse/pull/5946) ([akonyaev90](https://github.com/akonyaev90)) - Mettre à jour la valeur par défaut de `max_ast_elements parameter` [\#5933](https://github.com/ClickHouse/ClickHouse/pull/5933) ([Artem Konovalov](https://github.com/izebit)) -- Ajout d’une notion de paramètres obsolètes. Le paramètre obsolète `allow_experimental_low_cardinality_type` peut être utilisé avec aucun effet. [0f15c01c6802f7ce1a1494c12c846be8c98944cd](https://github.com/ClickHouse/ClickHouse/commit/0f15c01c6802f7ce1a1494c12c846be8c98944cd) [Alexey Milovidov](https://github.com/alexey-milovidov) +- Ajout d'une notion de paramètres obsolètes. Le paramètre obsolète `allow_experimental_low_cardinality_type` peut être utilisé avec aucun effet. [0f15c01c6802f7ce1a1494c12c846be8c98944cd](https://github.com/ClickHouse/ClickHouse/commit/0f15c01c6802f7ce1a1494c12c846be8c98944cd) [Alexey Milovidov](https://github.com/alexey-milovidov) #### Amélioration Des Performances {#performance-improvement-4} -- Augmentez le nombre de flux à sélectionner dans la table de fusion pour une distribution plus uniforme des threads. Ajout d’un réglage `max_streams_multiplier_for_merge_tables`. Cela corrige [\#5797](https://github.com/ClickHouse/ClickHouse/issues/5797) [\#5915](https://github.com/ClickHouse/ClickHouse/pull/5915) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Augmentez le nombre de flux à sélectionner dans la table de fusion pour une distribution plus uniforme des threads. Ajout d'un réglage `max_streams_multiplier_for_merge_tables`. Cela corrige [\#5797](https://github.com/ClickHouse/ClickHouse/issues/5797) [\#5915](https://github.com/ClickHouse/ClickHouse/pull/5915) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -#### Construction / Test / Amélioration De L’Emballage {#buildtestingpackaging-improvement-7} +#### Construction / Test / Amélioration De L'Emballage {#buildtestingpackaging-improvement-7} -- Ajoutez un test de rétrocompatibilité pour l’interaction client-serveur avec différentes versions de clickhouse. [\#5868](https://github.com/ClickHouse/ClickHouse/pull/5868) ([alésapine](https://github.com/alesapin)) +- Ajoutez un test de rétrocompatibilité pour l'interaction client-serveur avec différentes versions de clickhouse. [\#5868](https://github.com/ClickHouse/ClickHouse/pull/5868) ([alésapine](https://github.com/alesapin)) - Testez les informations de couverture dans chaque demande de validation et de tirage. [\#5896](https://github.com/ClickHouse/ClickHouse/pull/5896) ([alésapine](https://github.com/alesapin)) -- Coopérez avec le désinfectant d’adresse pour soutenir nos allocateurs faits sur commande (`Arena` et `ArenaWithFreeLists`) pour une meilleure mise “use-after-free” erreur. [\#5728](https://github.com/ClickHouse/ClickHouse/pull/5728) ([akuzm](https://github.com/akuzm)) -- Interrupteur à [Implémentation de LLVM libunwind](https://github.com/llvm-mirror/libunwind) pour la gestion des exceptions C++ et pour l’impression des traces de pile [\#4828](https://github.com/ClickHouse/ClickHouse/pull/4828) ([Nikita Lapkov](https://github.com/laplab)) +- Coopérez avec le désinfectant d'adresse pour soutenir nos allocateurs faits sur commande (`Arena` et `ArenaWithFreeLists`) pour une meilleure mise “use-after-free” erreur. [\#5728](https://github.com/ClickHouse/ClickHouse/pull/5728) ([akuzm](https://github.com/akuzm)) +- Interrupteur à [Implémentation de LLVM libunwind](https://github.com/llvm-mirror/libunwind) pour la gestion des exceptions C++ et pour l'impression des traces de pile [\#4828](https://github.com/ClickHouse/ClickHouse/pull/4828) ([Nikita Lapkov](https://github.com/laplab)) - Ajouter deux autres avertissements de-Weverything [\#5923](https://github.com/ClickHouse/ClickHouse/pull/5923) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Permettre de construire ClickHouse avec aseptisant mémoire. [\#3949](https://github.com/ClickHouse/ClickHouse/pull/3949) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Rapport ubsan fixe sur `bitTest` fonction dans fuzz test. [\#5943](https://github.com/ClickHouse/ClickHouse/pull/5943) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Docker: ajout de la possibilité d’initialiser une instance de ClickHouse qui nécessite une authentification. [\#5727](https://github.com/ClickHouse/ClickHouse/pull/5727) ([Korviakov Andrey](https://github.com/shurshun)) +- Docker: ajout de la possibilité d'initialiser une instance de ClickHouse qui nécessite une authentification. [\#5727](https://github.com/ClickHouse/ClickHouse/pull/5727) ([Korviakov Andrey](https://github.com/shurshun)) - Mettre à jour librdkafka vers la version 1.1.0 [\#5872](https://github.com/ClickHouse/ClickHouse/pull/5872) ([Ivan](https://github.com/abyss7)) -- Ajoutez un délai d’attente global pour les tests d’intégration et désactivez certains d’entre eux dans le code des tests. [\#5741](https://github.com/ClickHouse/ClickHouse/pull/5741) ([alésapine](https://github.com/alesapin)) +- Ajoutez un délai d'attente global pour les tests d'intégration et désactivez certains d'entre eux dans le code des tests. [\#5741](https://github.com/ClickHouse/ClickHouse/pull/5741) ([alésapine](https://github.com/alesapin)) - Correction de quelques échecs ThreadSanitizer. [\#5854](https://github.com/ClickHouse/ClickHouse/pull/5854) ([akuzm](https://github.com/akuzm)) -- Le `--no-undefined` option force l’éditeur de liens à vérifier l’existence de tous les noms externes lors de la liaison. Il est très utile de suivre les dépendances réelles entre les bibliothèques en mode de construction fractionnée. [\#5855](https://github.com/ClickHouse/ClickHouse/pull/5855) ([Ivan](https://github.com/abyss7)) +- Le `--no-undefined` option force l'éditeur de liens à vérifier l'existence de tous les noms externes lors de la liaison. Il est très utile de suivre les dépendances réelles entre les bibliothèques en mode de construction fractionnée. [\#5855](https://github.com/ClickHouse/ClickHouse/pull/5855) ([Ivan](https://github.com/abyss7)) - Ajouté test de performance pour [\#5797](https://github.com/ClickHouse/ClickHouse/issues/5797) [\#5914](https://github.com/ClickHouse/ClickHouse/pull/5914) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Correction de la compatibilité avec gcc-7. [\#5840](https://github.com/ClickHouse/ClickHouse/pull/5840) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Ajout du support pour gcc-9. Cela corrige [\#5717](https://github.com/ClickHouse/ClickHouse/issues/5717) [\#5774](https://github.com/ClickHouse/ClickHouse/pull/5774) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’une erreur lorsque libunwind peut être lié de manière incorrecte. [\#5948](https://github.com/ClickHouse/ClickHouse/pull/5948) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction d'une erreur lorsque libunwind peut être lié de manière incorrecte. [\#5948](https://github.com/ClickHouse/ClickHouse/pull/5948) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Correction de quelques avertissements trouvés par PVS-Studio. [\#5921](https://github.com/ClickHouse/ClickHouse/pull/5921) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Ajout du support initial pour `clang-tidy` analyseur statique. [\#5806](https://github.com/ClickHouse/ClickHouse/pull/5806) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Convertir les macros BSD / Linux endian( ‘be64toh’ et ‘htobe64’) aux équivalents Mac OS X [\#5785](https://github.com/ClickHouse/ClickHouse/pull/5785) ([Fu Chen](https://github.com/fredchenbj)) -- Amélioration du guide des tests d’intégration. [\#5796](https://github.com/ClickHouse/ClickHouse/pull/5796) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Amélioration du guide des tests d'intégration. [\#5796](https://github.com/ClickHouse/ClickHouse/pull/5796) ([Vladimir Chebotarev](https://github.com/excitoon)) - Correction de la construction sur macosx + gcc9 [\#5822](https://github.com/ClickHouse/ClickHouse/pull/5822) ([filimonov](https://github.com/filimonov)) -- Correction d’une faute de frappe difficile à repérer: aggreAGte - \> aggregate. [\#5753](https://github.com/ClickHouse/ClickHouse/pull/5753) ([akuzm](https://github.com/akuzm)) +- Correction d'une faute de frappe difficile à repérer: aggreAGte - \> aggregate. [\#5753](https://github.com/ClickHouse/ClickHouse/pull/5753) ([akuzm](https://github.com/akuzm)) - Correction de la construction freebsd [\#5760](https://github.com/ClickHouse/ClickHouse/pull/5760) ([proller](https://github.com/proller)) - Ajouter un lien vers la chaîne Youtube expérimentale au site web [\#5845](https://github.com/ClickHouse/ClickHouse/pull/5845) ([Ivan Blinkov](https://github.com/blinkov)) - CMake: ajouter une option pour les drapeaux de couverture: WITH\_COVERAGE [\#5776](https://github.com/ClickHouse/ClickHouse/pull/5776) ([proller](https://github.com/proller)) - Correction de la taille initiale de certains PODArray en ligne. [\#5787](https://github.com/ClickHouse/ClickHouse/pull/5787) ([akuzm](https://github.com/akuzm)) -- clickhouse-serveur.postinst: correction de la détection du système d’exploitation pour centos 6 [\#5788](https://github.com/ClickHouse/ClickHouse/pull/5788) ([proller](https://github.com/proller)) +- clickhouse-serveur.postinst: correction de la détection du système d'exploitation pour centos 6 [\#5788](https://github.com/ClickHouse/ClickHouse/pull/5788) ([proller](https://github.com/proller)) - Ajout de la génération de paquets Arch linux. [\#5719](https://github.com/ClickHouse/ClickHouse/pull/5719) ([Vladimir Chebotarev](https://github.com/excitoon)) - Diviser commun / config.h par libs (SGBD) [\#5715](https://github.com/ClickHouse/ClickHouse/pull/5715) ([proller](https://github.com/proller)) - Des correctifs pour “Arcadia” créer une plate-forme [\#5795](https://github.com/ClickHouse/ClickHouse/pull/5795) ([proller](https://github.com/proller)) - Correctifs pour la construction non conventionnelle (gcc9, pas de sous-modules) [\#5792](https://github.com/ClickHouse/ClickHouse/pull/5792) ([proller](https://github.com/proller)) -- Exiger un type explicite dans unalignedStore car il a été prouvé qu’il était sujet aux bugs [\#5791](https://github.com/ClickHouse/ClickHouse/pull/5791) ([akuzm](https://github.com/akuzm)) +- Exiger un type explicite dans unalignedStore car il a été prouvé qu'il était sujet aux bugs [\#5791](https://github.com/ClickHouse/ClickHouse/pull/5791) ([akuzm](https://github.com/akuzm)) - Corrige la construction MacOS [\#5830](https://github.com/ClickHouse/ClickHouse/pull/5830) ([filimonov](https://github.com/filimonov)) - Test de Performance concernant la nouvelle fonctionnalité JIT avec un ensemble de données plus grand, comme demandé ici [\#5263](https://github.com/ClickHouse/ClickHouse/issues/5263) [\#5887](https://github.com/ClickHouse/ClickHouse/pull/5887) ([Guillaume Tassery](https://github.com/YiuRULE)) - Exécuter des tests avec État dans le test de stress [12693e568722f11e19859742f56428455501fd2a](https://github.com/ClickHouse/ClickHouse/commit/12693e568722f11e19859742f56428455501fd2a) ([alésapine](https://github.com/alesapin)) @@ -1124,27 +1124,27 @@ toc_title: '2019' #### Nouveauté {#new-feature-7} - Ajouter un nouveau codec de colonne: `T64`. Fait pour les colonnes(U)IntX/EnumX/Data (Time)/DecimalX. Il devrait être bon pour les colonnes avec des valeurs de plage constantes ou petites. Codec lui-même permet agrandir ou réduire le type de données sans re-compression. [\#5557](https://github.com/ClickHouse/ClickHouse/pull/5557) ([Artem Zuikov](https://github.com/4ertus2)) -- Ajouter un moteur de base de données `MySQL` qui permettent d’afficher toutes les tables dans le serveur MySQL distant [\#5599](https://github.com/ClickHouse/ClickHouse/pull/5599) ([L’Hiver Zhang](https://github.com/zhang2014)) -- `bitmapContains` application. C’est 2x plus rapide que `bitmapHasAny` si le second bitmap contient un élément. [\#5535](https://github.com/ClickHouse/ClickHouse/pull/5535) ([Zhichang Yu](https://github.com/yuzhichang)) -- Soutien pour `crc32` fonction (avec un comportement exactement comme dans MySQL ou PHP). Ne l’utilisez pas si vous avez besoin d’une fonction de hachage. [\#5661](https://github.com/ClickHouse/ClickHouse/pull/5661) ([Remen Ivan](https://github.com/BHYCHIK)) -- Mettre `SYSTEM START/STOP DISTRIBUTED SENDS` requêtes pour contrôler les insertions asynchrones dans `Distributed` table. [\#4935](https://github.com/ClickHouse/ClickHouse/pull/4935) ([L’Hiver Zhang](https://github.com/zhang2014)) +- Ajouter un moteur de base de données `MySQL` qui permettent d'afficher toutes les tables dans le serveur MySQL distant [\#5599](https://github.com/ClickHouse/ClickHouse/pull/5599) ([L'Hiver Zhang](https://github.com/zhang2014)) +- `bitmapContains` application. C'est 2x plus rapide que `bitmapHasAny` si le second bitmap contient un élément. [\#5535](https://github.com/ClickHouse/ClickHouse/pull/5535) ([Zhichang Yu](https://github.com/yuzhichang)) +- Soutien pour `crc32` fonction (avec un comportement exactement comme dans MySQL ou PHP). Ne l'utilisez pas si vous avez besoin d'une fonction de hachage. [\#5661](https://github.com/ClickHouse/ClickHouse/pull/5661) ([Remen Ivan](https://github.com/BHYCHIK)) +- Mettre `SYSTEM START/STOP DISTRIBUTED SENDS` requêtes pour contrôler les insertions asynchrones dans `Distributed` table. [\#4935](https://github.com/ClickHouse/ClickHouse/pull/4935) ([L'Hiver Zhang](https://github.com/zhang2014)) #### Bug Fix {#bug-fix-22} -- Ignorer les limites d’exécution des requêtes et la taille maximale des pièces pour les limites de fusion lors de l’exécution des mutations. [\#5659](https://github.com/ClickHouse/ClickHouse/pull/5659) ([Anton Popov](https://github.com/CurtizJ)) -- Correction d’un bug qui peut conduire à la déduplication de blocs normaux (extrêmement rare) et l’insertion de blocs en double (plus souvent). [\#5549](https://github.com/ClickHouse/ClickHouse/pull/5549) ([alésapine](https://github.com/alesapin)) +- Ignorer les limites d'exécution des requêtes et la taille maximale des pièces pour les limites de fusion lors de l'exécution des mutations. [\#5659](https://github.com/ClickHouse/ClickHouse/pull/5659) ([Anton Popov](https://github.com/CurtizJ)) +- Correction d'un bug qui peut conduire à la déduplication de blocs normaux (extrêmement rare) et l'insertion de blocs en double (plus souvent). [\#5549](https://github.com/ClickHouse/ClickHouse/pull/5549) ([alésapine](https://github.com/alesapin)) - Correction de la fonction `arrayEnumerateUniqRanked` pour les arguments avec des tableaux vides [\#5559](https://github.com/ClickHouse/ClickHouse/pull/5559) ([proller](https://github.com/proller)) -- Ne vous abonnez pas aux sujets Kafka sans l’intention d’interroger des messages. [\#5698](https://github.com/ClickHouse/ClickHouse/pull/5698) ([Ivan](https://github.com/abyss7)) -- Faire la mise en `join_use_nulls` n’obtenez aucun effet pour les types qui ne peuvent pas être à L’intérieur de Nullable [\#5700](https://github.com/ClickHouse/ClickHouse/pull/5700) ([Olga Khvostikova](https://github.com/stavrolia)) +- Ne vous abonnez pas aux sujets Kafka sans l'intention d'interroger des messages. [\#5698](https://github.com/ClickHouse/ClickHouse/pull/5698) ([Ivan](https://github.com/abyss7)) +- Faire la mise en `join_use_nulls` n'obtenez aucun effet pour les types qui ne peuvent pas être à L'intérieur de Nullable [\#5700](https://github.com/ClickHouse/ClickHouse/pull/5700) ([Olga Khvostikova](https://github.com/stavrolia)) - Fixe `Incorrect size of index granularity` erreur [\#5720](https://github.com/ClickHouse/ClickHouse/pull/5720) ([coraxster](https://github.com/coraxster)) - Correction du flotteur en décimal convertir le débordement [\#5607](https://github.com/ClickHouse/ClickHouse/pull/5607) ([coraxster](https://github.com/coraxster)) -- Vider le tampon quand `WriteBufferFromHDFS`’s destructeur est appelé. Cela corrige l’écriture dans `HDFS`. [\#5684](https://github.com/ClickHouse/ClickHouse/pull/5684) ([Xindong Peng](https://github.com/eejoin)) +- Vider le tampon quand `WriteBufferFromHDFS`'s destructeur est appelé. Cela corrige l'écriture dans `HDFS`. [\#5684](https://github.com/ClickHouse/ClickHouse/pull/5684) ([Xindong Peng](https://github.com/eejoin)) #### Amélioration {#improvement-7} - Traiter les cellules vides dans `CSV` comme valeurs par défaut lorsque le paramètre `input_format_defaults_for_omitted_fields` est activé. [\#5625](https://github.com/ClickHouse/ClickHouse/pull/5625) ([akuzm](https://github.com/akuzm)) - Chargement Non bloquant des dictionnaires externes. [\#5567](https://github.com/ClickHouse/ClickHouse/pull/5567) ([Vitaly Baranov](https://github.com/vitlibar)) -- Les délais d’attente réseau peuvent être modifiés dynamiquement pour les connexions déjà établies en fonction des paramètres. [\#4558](https://github.com/ClickHouse/ClickHouse/pull/4558) ([Konstantin Podshumok](https://github.com/podshumok)) +- Les délais d'attente réseau peuvent être modifiés dynamiquement pour les connexions déjà établies en fonction des paramètres. [\#4558](https://github.com/ClickHouse/ClickHouse/pull/4558) ([Konstantin Podshumok](https://github.com/podshumok)) - Utiliser “public\_suffix\_list” pour les fonctions `firstSignificantSubdomain`, `cutToFirstSignificantSubdomain`. Il utilise une table de hachage parfaite générée par `gperf` avec une liste générée à partir du fichier: https://publicsuffix.org/list/public\_suffix\_list.dat. (par exemple, nous reconnaissons maintenant le domaine `ac.uk` comme non significatif). [\#5030](https://github.com/ClickHouse/ClickHouse/pull/5030) ([Guillaume Tassery](https://github.com/YiuRULE)) - Adopté `IPv6` type de données dans les tables système; colonnes unified client info dans `system.processes` et `system.query_log` [\#5640](https://github.com/ClickHouse/ClickHouse/pull/5640) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Utilisation de sessions pour les connexions avec le protocole de compatibilité MySQL. \#5476 [\#5646](https://github.com/ClickHouse/ClickHouse/pull/5646) ([Yuriy Baranov](https://github.com/yurriy)) @@ -1154,14 +1154,14 @@ toc_title: '2019' #### Amélioration Des Performances {#performance-improvement-5} -- Ajoutez la possibilité d’écrire la marque finale à la fin des colonnes MergeTree. Il permet d’éviter les lectures inutiles pour les clés qui sont hors de la plage de données de la table. Elle n’est activée que si la granularité d’index adaptatif est utilisée. [\#5624](https://github.com/ClickHouse/ClickHouse/pull/5624) ([alésapine](https://github.com/alesapin)) +- Ajoutez la possibilité d'écrire la marque finale à la fin des colonnes MergeTree. Il permet d'éviter les lectures inutiles pour les clés qui sont hors de la plage de données de la table. Elle n'est activée que si la granularité d'index adaptatif est utilisée. [\#5624](https://github.com/ClickHouse/ClickHouse/pull/5624) ([alésapine](https://github.com/alesapin)) - Amélioration des performances des tables MergeTree sur les systèmes de fichiers très lents en réduisant `stat` syscalls. [\#5648](https://github.com/ClickHouse/ClickHouse/pull/5648) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Correction de la dégradation des performances en lecture à partir des tables MergeTree introduites dans la version 19.6. Corrections \# 5631. [\#5633](https://github.com/ClickHouse/ClickHouse/pull/5633) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -#### Construction / Test / Amélioration De L’Emballage {#buildtestingpackaging-improvement-8} +#### Construction / Test / Amélioration De L'Emballage {#buildtestingpackaging-improvement-8} -- Mettre `TestKeeper` en tant qu’implémentation de l’interface ZooKeeper utilisée pour les tests [\#5643](https://github.com/ClickHouse/ClickHouse/pull/5643) ([alexeï-milovidov](https://github.com/alexey-milovidov)) ([levushkin aleksej](https://github.com/alexey-milovidov)) -- Désormais `.sql` les tests peuvent être exécutés isolés par le serveur, en parallèle, avec une base de données aléatoire. Il permet de les exécuter plus rapidement, d’ajouter de nouveaux tests avec des configurations de serveur personnalisées et de s’assurer que les différents tests ne s’affectent pas les uns les autres. [\#5554](https://github.com/ClickHouse/ClickHouse/pull/5554) ([Ivan](https://github.com/abyss7)) +- Mettre `TestKeeper` en tant qu'implémentation de l'interface ZooKeeper utilisée pour les tests [\#5643](https://github.com/ClickHouse/ClickHouse/pull/5643) ([alexeï-milovidov](https://github.com/alexey-milovidov)) ([levushkin aleksej](https://github.com/alexey-milovidov)) +- Désormais `.sql` les tests peuvent être exécutés isolés par le serveur, en parallèle, avec une base de données aléatoire. Il permet de les exécuter plus rapidement, d'ajouter de nouveaux tests avec des configurations de serveur personnalisées et de s'assurer que les différents tests ne s'affectent pas les uns les autres. [\#5554](https://github.com/ClickHouse/ClickHouse/pull/5554) ([Ivan](https://github.com/abyss7)) - Supprimer `` et `` à partir de tests de performance [\#5672](https://github.com/ClickHouse/ClickHouse/pull/5672) ([Olga Khvostikova](https://github.com/stavrolia)) - Fixe “select\_format” essai de performance pour `Pretty` format [\#5642](https://github.com/ClickHouse/ClickHouse/pull/5642) ([alexeï-milovidov](https://github.com/alexey-milovidov)) @@ -1172,11 +1172,11 @@ toc_title: '2019' #### Bug Fix {#bug-fix-23} - Correction de segfault dans le codec Delta qui affecte les colonnes avec des valeurs inférieures à 32 bits. Le bug a conduit à la corruption de la mémoire aléatoire. [\#5786](https://github.com/ClickHouse/ClickHouse/pull/5786) ([alésapine](https://github.com/alesapin)) -- Correction d’un bug rare dans la vérification de la partie avec la colonne LowCardinality. [\#5832](https://github.com/ClickHouse/ClickHouse/pull/5832) ([alésapine](https://github.com/alesapin)) +- Correction d'un bug rare dans la vérification de la partie avec la colonne LowCardinality. [\#5832](https://github.com/ClickHouse/ClickHouse/pull/5832) ([alésapine](https://github.com/alesapin)) - Correction de segfault dans la fusion TTL avec des colonnes non physiques en bloc. [\#5819](https://github.com/ClickHouse/ClickHouse/pull/5819) ([Anton Popov](https://github.com/CurtizJ)) - Correction du sommeil infini potentiel des requêtes de faible priorité. [\#5842](https://github.com/ClickHouse/ClickHouse/pull/5842) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Correction de la façon dont ClickHouse détermine le fuseau horaire par défaut comme UCT au lieu de UTC. [\#5828](https://github.com/ClickHouse/ClickHouse/pull/5828) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’un bug sur l’exécution distribuée DROP/ALTER/TRUNCATE / OPTIMIZE sur les requêtes de CLUSTER sur la réplique suiveur avant la réplique leader. Maintenant, ils seront exécutés directement sur la réplique leader. [\#5757](https://github.com/ClickHouse/ClickHouse/pull/5757) ([alésapine](https://github.com/alesapin)) +- Correction d'un bug sur l'exécution distribuée DROP/ALTER/TRUNCATE / OPTIMIZE sur les requêtes de CLUSTER sur la réplique suiveur avant la réplique leader. Maintenant, ils seront exécutés directement sur la réplique leader. [\#5757](https://github.com/ClickHouse/ClickHouse/pull/5757) ([alésapine](https://github.com/alesapin)) - Correction de la condition de concurrence, ce qui fait que certaines requêtes peuvent ne pas apparaître instantanément dans query\_log après la requête system FLUSH LOGS. [\#5685](https://github.com/ClickHouse/ClickHouse/pull/5685) ([Anton Popov](https://github.com/CurtizJ)) - Ajout du support manquant pour les arguments constants à `evalMLModel` fonction. [\#5820](https://github.com/ClickHouse/ClickHouse/pull/5820) ([alexeï-milovidov](https://github.com/alexey-milovidov)) @@ -1185,39 +1185,39 @@ toc_title: '2019' #### Nouveauté {#new-feature-8} - Imprimer des informations sur les pièces gelées dans `system.parts` table. [\#5471](https://github.com/ClickHouse/ClickHouse/pull/5471) ([proller](https://github.com/proller)) -- Demander mot de passe du client sur clickhouse-client démarrer sur ats s’il n’est pas dans les arguments [\#5092](https://github.com/ClickHouse/ClickHouse/pull/5092) ([proller](https://github.com/proller)) +- Demander mot de passe du client sur clickhouse-client démarrer sur ats s'il n'est pas dans les arguments [\#5092](https://github.com/ClickHouse/ClickHouse/pull/5092) ([proller](https://github.com/proller)) - Mettre `dictGet` et `dictGetOrDefault` fonctions pour les types Décimaux. [\#5394](https://github.com/ClickHouse/ClickHouse/pull/5394) ([Artem Zuikov](https://github.com/4ertus2)) #### Amélioration {#improvement-8} -- Debian init: ajout d’un délai d’arrêt du service [\#5522](https://github.com/ClickHouse/ClickHouse/pull/5522) ([proller](https://github.com/proller)) +- Debian init: ajout d'un délai d'arrêt du service [\#5522](https://github.com/ClickHouse/ClickHouse/pull/5522) ([proller](https://github.com/proller)) - Ajouter un paramètre interdit par défaut pour créer une table avec des types suspects pour LowCardinality [\#5448](https://github.com/ClickHouse/ClickHouse/pull/5448) ([Olga Khvostikova](https://github.com/stavrolia)) -- Les fonctions de régression renvoient les poids du modèle lorsqu’elles ne sont pas utilisées comme État dans la fonction `evalMLMethod`. [\#5411](https://github.com/ClickHouse/ClickHouse/pull/5411) ([Quid37](https://github.com/Quid37)) +- Les fonctions de régression renvoient les poids du modèle lorsqu'elles ne sont pas utilisées comme État dans la fonction `evalMLMethod`. [\#5411](https://github.com/ClickHouse/ClickHouse/pull/5411) ([Quid37](https://github.com/Quid37)) - Renommer et améliorer les méthodes de régression. [\#5492](https://github.com/ClickHouse/ClickHouse/pull/5492) ([Quid37](https://github.com/Quid37)) - Interfaces plus claires des chercheurs de chaînes. [\#5586](https://github.com/ClickHouse/ClickHouse/pull/5586) ([Danila Kutenin](https://github.com/danlark1)) #### Bug Fix {#bug-fix-24} - Correction de la perte de données potentielle dans Kafka [\#5445](https://github.com/ClickHouse/ClickHouse/pull/5445) ([Ivan](https://github.com/abyss7)) -- Correction d’une boucle infinie potentielle dans `PrettySpace` format lorsqu’il est appelé avec zéro colonnes [\#5560](https://github.com/ClickHouse/ClickHouse/pull/5560) ([Olga Khvostikova](https://github.com/stavrolia)) -- Correction d’un bug de débordement UInt32 dans les modèles linéaires. Autoriser le modèle EVAL ML pour l’argument du modèle non const. [\#5516](https://github.com/ClickHouse/ClickHouse/pull/5516) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- `ALTER TABLE ... DROP INDEX IF EXISTS ...` ne devrait pas soulever une exception si l’index n’existe pas [\#5524](https://github.com/ClickHouse/ClickHouse/pull/5524) ([Gleb Novikov](https://github.com/NanoBjorn)) +- Correction d'une boucle infinie potentielle dans `PrettySpace` format lorsqu'il est appelé avec zéro colonnes [\#5560](https://github.com/ClickHouse/ClickHouse/pull/5560) ([Olga Khvostikova](https://github.com/stavrolia)) +- Correction d'un bug de débordement UInt32 dans les modèles linéaires. Autoriser le modèle EVAL ML pour l'argument du modèle non const. [\#5516](https://github.com/ClickHouse/ClickHouse/pull/5516) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- `ALTER TABLE ... DROP INDEX IF EXISTS ...` ne devrait pas soulever une exception si l'index n'existe pas [\#5524](https://github.com/ClickHouse/ClickHouse/pull/5524) ([Gleb Novikov](https://github.com/NanoBjorn)) - Fixer erreur de segmentation avec `bitmapHasAny` dans la sous-requête scalaire [\#5528](https://github.com/ClickHouse/ClickHouse/pull/5528) ([Zhichang Yu](https://github.com/yuzhichang)) -- Correction d’une erreur lorsque le pool de connexions de réplication ne tente pas de résoudre l’hôte, même lorsque le cache DNS a été supprimé. [\#5534](https://github.com/ClickHouse/ClickHouse/pull/5534) ([alésapine](https://github.com/alesapin)) +- Correction d'une erreur lorsque le pool de connexions de réplication ne tente pas de résoudre l'hôte, même lorsque le cache DNS a été supprimé. [\#5534](https://github.com/ClickHouse/ClickHouse/pull/5534) ([alésapine](https://github.com/alesapin)) - Fixe `ALTER ... MODIFY TTL` sur ReplicatedMergeTree. [\#5539](https://github.com/ClickHouse/ClickHouse/pull/5539) ([Anton Popov](https://github.com/CurtizJ)) -- Correction de L’insertion dans une table distribuée avec une colonne matérialisée [\#5429](https://github.com/ClickHouse/ClickHouse/pull/5429) ([Azat Khuzhin](https://github.com/azat)) -- Correction d’un mauvais alloc lorsque truncate Join storage [\#5437](https://github.com/ClickHouse/ClickHouse/pull/5437) ([TCeason](https://github.com/TCeason)) +- Correction de L'insertion dans une table distribuée avec une colonne matérialisée [\#5429](https://github.com/ClickHouse/ClickHouse/pull/5429) ([Azat Khuzhin](https://github.com/azat)) +- Correction d'un mauvais alloc lorsque truncate Join storage [\#5437](https://github.com/ClickHouse/ClickHouse/pull/5437) ([TCeason](https://github.com/TCeason)) - Dans les versions récentes du paquet tzdata, certains fichiers sont maintenant des liens symboliques. Le mécanisme actuel de détection du fuseau horaire par défaut est cassé et donne des noms erronés pour certains fuseaux horaires. Maintenant, au moins, nous forçons le nom du fuseau horaire au contenu de TZ si fourni. [\#5443](https://github.com/ClickHouse/ClickHouse/pull/5443) ([Ivan](https://github.com/abyss7)) -- Fixer certains cas extrêmement rares avec multivolnitsky searcher lorsque les aiguilles constantes en somme sont au moins 16KB long. L’algorithme a manqué ou écrasé les résultats précédents ce qui peut conduire au résultat incorrect de `multiSearchAny`. [\#5588](https://github.com/ClickHouse/ClickHouse/pull/5588) ([Danila Kutenin](https://github.com/danlark1)) -- Résolvez le problème lorsque les paramètres des requêtes ExternalData ne pouvaient pas utiliser les paramètres ClickHouse. Aussi, pour l’instant, paramètres `date_time_input_format` et `low_cardinality_allow_in_native_format` ne peut pas être utilisé en raison de l’ambiguïté des noms (en données externe, il peut être interprété comme format de tableau et dans la requête, il peut être un paramètre). [\#5455](https://github.com/ClickHouse/ClickHouse/pull/5455) ([Danila Kutenin](https://github.com/danlark1)) -- Correction d’un bug lorsque des pièces ont été supprimées uniquement de FS sans les laisser tomber de Zookeeper. [\#5520](https://github.com/ClickHouse/ClickHouse/pull/5520) ([alésapine](https://github.com/alesapin)) +- Fixer certains cas extrêmement rares avec multivolnitsky searcher lorsque les aiguilles constantes en somme sont au moins 16KB long. L'algorithme a manqué ou écrasé les résultats précédents ce qui peut conduire au résultat incorrect de `multiSearchAny`. [\#5588](https://github.com/ClickHouse/ClickHouse/pull/5588) ([Danila Kutenin](https://github.com/danlark1)) +- Résolvez le problème lorsque les paramètres des requêtes ExternalData ne pouvaient pas utiliser les paramètres ClickHouse. Aussi, pour l'instant, paramètres `date_time_input_format` et `low_cardinality_allow_in_native_format` ne peut pas être utilisé en raison de l'ambiguïté des noms (en données externe, il peut être interprété comme format de tableau et dans la requête, il peut être un paramètre). [\#5455](https://github.com/ClickHouse/ClickHouse/pull/5455) ([Danila Kutenin](https://github.com/danlark1)) +- Correction d'un bug lorsque des pièces ont été supprimées uniquement de FS sans les laisser tomber de Zookeeper. [\#5520](https://github.com/ClickHouse/ClickHouse/pull/5520) ([alésapine](https://github.com/alesapin)) - Supprimer la journalisation de débogage du protocole MySQL [\#5478](https://github.com/ClickHouse/ClickHouse/pull/5478) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Ignorer ZNONODE pendant le traitement de la requête DDL [\#5489](https://github.com/ClickHouse/ClickHouse/pull/5489) ([Azat Khuzhin](https://github.com/azat)) - Fix mix `UNION ALL` type de colonne de résultat. Il y avait des cas avec des données incohérentes et des types de colonnes de colonnes résultantes. [\#5503](https://github.com/ClickHouse/ClickHouse/pull/5503) ([Artem Zuikov](https://github.com/4ertus2)) - Lancer une exception sur des entiers erronés dans `dictGetT` fonctions au lieu de crash. [\#5446](https://github.com/ClickHouse/ClickHouse/pull/5446) ([Artem Zuikov](https://github.com/4ertus2)) - Correction de mauvais element\_count et load\_factor pour le dictionnaire haché dans `system.dictionaries` table. [\#5440](https://github.com/ClickHouse/ClickHouse/pull/5440) ([Azat Khuzhin](https://github.com/azat)) -#### Construction / Test / Amélioration De L’Emballage {#buildtestingpackaging-improvement-9} +#### Construction / Test / Amélioration De L'Emballage {#buildtestingpackaging-improvement-9} - Construction fixe Sans `Brotli` Prise en charge de la compression HTTP (`ENABLE_BROTLI=OFF` cmake variable). [\#5521](https://github.com/ClickHouse/ClickHouse/pull/5521) ([Anton Yuzhaninov](https://github.com/citrin)) - Inclure rugissement.h comme rugissant / rugissant.h [\#5523](https://github.com/ClickHouse/ClickHouse/pull/5523) ([Orivej Desh](https://github.com/orivej)) @@ -1225,8 +1225,8 @@ toc_title: '2019' - Corrigez tous les avertissements lors de la compilation avec gcc-9. Correction de certains problèmes de contrib. Corrigez GCC9 ICE et soumettez-le à bugzilla. [\#5498](https://github.com/ClickHouse/ClickHouse/pull/5498) ([Danila Kutenin](https://github.com/danlark1)) - Liaison fixe avec lld [\#5477](https://github.com/ClickHouse/ClickHouse/pull/5477) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Supprimer les spécialisations dans les dictionnaires [\#5452](https://github.com/ClickHouse/ClickHouse/pull/5452) ([Artem Zuikov](https://github.com/4ertus2)) -- Amélioration des tests de performance pour le formatage et l’analyse des tables pour différents types de fichiers [\#5497](https://github.com/ClickHouse/ClickHouse/pull/5497) ([Olga Khvostikova](https://github.com/stavrolia)) -- Corrections pour l’exécution de test parallèle [\#5506](https://github.com/ClickHouse/ClickHouse/pull/5506) ([proller](https://github.com/proller)) +- Amélioration des tests de performance pour le formatage et l'analyse des tables pour différents types de fichiers [\#5497](https://github.com/ClickHouse/ClickHouse/pull/5497) ([Olga Khvostikova](https://github.com/stavrolia)) +- Corrections pour l'exécution de test parallèle [\#5506](https://github.com/ClickHouse/ClickHouse/pull/5506) ([proller](https://github.com/proller)) - Docker: utiliser les configs de clickhouse-test [\#5531](https://github.com/ClickHouse/ClickHouse/pull/5531) ([proller](https://github.com/proller)) - Correction de la compilation Pour FreeBSD [\#5447](https://github.com/ClickHouse/ClickHouse/pull/5447) ([proller](https://github.com/proller)) - Mise à niveau boost à 1.70 [\#5570](https://github.com/ClickHouse/ClickHouse/pull/5570) ([proller](https://github.com/proller)) @@ -1242,9 +1242,9 @@ toc_title: '2019' - Ajout de fonctions pour travailler avec JSON [\#4686](https://github.com/ClickHouse/ClickHouse/pull/4686) ([hcz](https://github.com/hczhcz)) [\#5124](https://github.com/ClickHouse/ClickHouse/pull/5124). ([Vitaly Baranov](https://github.com/vitlibar)) - Ajouter une fonction basename, avec un comportement similaire à une fonction basename, qui existe dans beaucoup de langues (`os.path.basename` en python, `basename` in PHP, etc…). Work with both an UNIX-like path or a Windows path. [\#5136](https://github.com/ClickHouse/ClickHouse/pull/5136) ([Guillaume Tassery](https://github.com/YiuRULE)) - Ajouter `LIMIT n, m BY` ou `LIMIT m OFFSET n BY` syntaxe pour définir le décalage de N pour la clause LIMIT BY. [\#5138](https://github.com/ClickHouse/ClickHouse/pull/5138) ([Anton Popov](https://github.com/CurtizJ)) -- Ajouté nouveau type de données `SimpleAggregateFunction`, ce qui permet d’avoir des colonnes avec agrégation de lumière dans un `AggregatingMergeTree`. Cela ne peut être utilisé avec des fonctions simples comme `any`, `anyLast`, `sum`, `min`, `max`. [\#4629](https://github.com/ClickHouse/ClickHouse/pull/4629) ([Boris Granveaud](https://github.com/bgranvea)) +- Ajouté nouveau type de données `SimpleAggregateFunction`, ce qui permet d'avoir des colonnes avec agrégation de lumière dans un `AggregatingMergeTree`. Cela ne peut être utilisé avec des fonctions simples comme `any`, `anyLast`, `sum`, `min`, `max`. [\#4629](https://github.com/ClickHouse/ClickHouse/pull/4629) ([Boris Granveaud](https://github.com/bgranvea)) - Ajout du support pour les arguments non constants dans la fonction `ngramDistance` [\#5198](https://github.com/ClickHouse/ClickHouse/pull/5198) ([Danila Kutenin](https://github.com/danlark1)) -- L’ajout de fonctions `skewPop`, `skewSamp`, `kurtPop` et `kurtSamp` pour calculer l’asymétrie de séquence, l’asymétrie de l’échantillon, la kurtose et la kurtose de l’échantillon respectivement. [\#5200](https://github.com/ClickHouse/ClickHouse/pull/5200) ([hcz](https://github.com/hczhcz)) +- L'ajout de fonctions `skewPop`, `skewSamp`, `kurtPop` et `kurtSamp` pour calculer l'asymétrie de séquence, l'asymétrie de l'échantillon, la kurtose et la kurtose de l'échantillon respectivement. [\#5200](https://github.com/ClickHouse/ClickHouse/pull/5200) ([hcz](https://github.com/hczhcz)) - Soutien renommer opération pour `MaterializeView` stockage. [\#5209](https://github.com/ClickHouse/ClickHouse/pull/5209) ([Guillaume Tassery](https://github.com/YiuRULE)) - Serveur Ajouté qui permet de se connecter à ClickHouse en utilisant le client MySQL. [\#4715](https://github.com/ClickHouse/ClickHouse/pull/4715) ([Yuriy Baranov](https://github.com/yurriy)) - Ajouter `toDecimal*OrZero` et `toDecimal*OrNull` fonction. [\#5291](https://github.com/ClickHouse/ClickHouse/pull/5291) ([Artem Zuikov](https://github.com/4ertus2)) @@ -1252,32 +1252,32 @@ toc_title: '2019' - Ajouter `toValidUTF8` function, which replaces all invalid UTF-8 characters by replacement character � (U+FFFD). [\#5322](https://github.com/ClickHouse/ClickHouse/pull/5322) ([Danila Kutenin](https://github.com/danlark1)) - Ajouter `format` fonction. Formatage du motif constant (modèle de format Python simplifié) avec les chaînes listées dans les arguments. [\#5330](https://github.com/ClickHouse/ClickHouse/pull/5330) ([Danila Kutenin](https://github.com/danlark1)) - Ajouter `system.detached_parts` tableau contenant des informations sur les parties détachées de `MergeTree` table. [\#5353](https://github.com/ClickHouse/ClickHouse/pull/5353) ([akuzm](https://github.com/akuzm)) -- Ajouter `ngramSearch` fonction pour calculer la différence non symétrique entre l’aiguille et la botte de foin. [\#5418](https://github.com/ClickHouse/ClickHouse/pull/5418)[\#5422](https://github.com/ClickHouse/ClickHouse/pull/5422) ([Danila Kutenin](https://github.com/danlark1)) -- Mise en œuvre de méthodes d’apprentissage automatique de base (régression linéaire stochastique et régression logistique) à l’aide de l’interface des fonctions agrégées. A différentes stratégies pour mettre à jour les poids du modèle (descente de gradient simple, méthode momentum, méthode Nesterov). Prend également en charge les mini-lots de taille personnalisée. [\#4943](https://github.com/ClickHouse/ClickHouse/pull/4943) ([Quid37](https://github.com/Quid37)) +- Ajouter `ngramSearch` fonction pour calculer la différence non symétrique entre l'aiguille et la botte de foin. [\#5418](https://github.com/ClickHouse/ClickHouse/pull/5418)[\#5422](https://github.com/ClickHouse/ClickHouse/pull/5422) ([Danila Kutenin](https://github.com/danlark1)) +- Mise en œuvre de méthodes d'apprentissage automatique de base (régression linéaire stochastique et régression logistique) à l'aide de l'interface des fonctions agrégées. A différentes stratégies pour mettre à jour les poids du modèle (descente de gradient simple, méthode momentum, méthode Nesterov). Prend également en charge les mini-lots de taille personnalisée. [\#4943](https://github.com/ClickHouse/ClickHouse/pull/4943) ([Quid37](https://github.com/Quid37)) - La mise en œuvre de `geohashEncode` et `geohashDecode` fonction. [\#5003](https://github.com/ClickHouse/ClickHouse/pull/5003) ([Vasily Nemkov](https://github.com/Enmk)) -- Fonction agrégée ajoutée `timeSeriesGroupSum` qui peut regrouper différentes séries de l’échantillon d’horodatage de l’alignement. Il utilisera une interpolation linéaire entre deux échantillons d’horodatage, puis additionnera les séries temporelles ensemble. Fonction agrégée ajoutée `timeSeriesGroupRateSum`, qui calcule le taux de séries chronologiques, puis additionne les taux ensemble. [\#4542](https://github.com/ClickHouse/ClickHouse/pull/4542) ([Yangkuan Liu](https://github.com/LiuYangkuan)) -- L’ajout de fonctions `IPv4CIDRtoIPv4Range` et `IPv6CIDRtoIPv6Range` pour calculer les limites inférieures et supérieures pour une adresse IP dans le sous-réseau à l’aide D’un CIDR. [\#5095](https://github.com/ClickHouse/ClickHouse/pull/5095) ([Guillaume Tassery](https://github.com/YiuRULE)) -- Ajouter un en-tête X-Clickhouse-Summary lorsque nous envoyons une requête en utilisant HTTP avec paramètre activé `send_progress_in_http_headers`. Renvoie les informations habituelles de X-ClickHouse-Progress, avec des informations supplémentaires telles que le nombre de lignes et d’octets insérés dans la requête. [\#5116](https://github.com/ClickHouse/ClickHouse/pull/5116) ([Guillaume Tassery](https://github.com/YiuRULE)) +- Fonction agrégée ajoutée `timeSeriesGroupSum` qui peut regrouper différentes séries de l'échantillon d'horodatage de l'alignement. Il utilisera une interpolation linéaire entre deux échantillons d'horodatage, puis additionnera les séries temporelles ensemble. Fonction agrégée ajoutée `timeSeriesGroupRateSum`, qui calcule le taux de séries chronologiques, puis additionne les taux ensemble. [\#4542](https://github.com/ClickHouse/ClickHouse/pull/4542) ([Yangkuan Liu](https://github.com/LiuYangkuan)) +- L'ajout de fonctions `IPv4CIDRtoIPv4Range` et `IPv6CIDRtoIPv6Range` pour calculer les limites inférieures et supérieures pour une adresse IP dans le sous-réseau à l'aide D'un CIDR. [\#5095](https://github.com/ClickHouse/ClickHouse/pull/5095) ([Guillaume Tassery](https://github.com/YiuRULE)) +- Ajouter un en-tête X-Clickhouse-Summary lorsque nous envoyons une requête en utilisant HTTP avec paramètre activé `send_progress_in_http_headers`. Renvoie les informations habituelles de X-ClickHouse-Progress, avec des informations supplémentaires telles que le nombre de lignes et d'octets insérés dans la requête. [\#5116](https://github.com/ClickHouse/ClickHouse/pull/5116) ([Guillaume Tassery](https://github.com/YiuRULE)) #### Amélioration {#improvements} - Ajouter `max_parts_in_total` paramètre pour la famille de tables MergeTree (par défaut: 100 000) qui empêche la spécification dangereuse de la clé de partition \# 5166. [\#5171](https://github.com/ClickHouse/ClickHouse/pull/5171) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - `clickhouse-obfuscator`: dériver la graine pour les colonnes individuelles en combinant la graine initiale avec le nom de la colonne, pas la position de la colonne. Ceci est destiné à transformer des ensembles de données avec plusieurs tables associées, de sorte que les tables restent joignables après la transformation. [\#5178](https://github.com/ClickHouse/ClickHouse/pull/5178) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- L’ajout de fonctions `JSONExtractRaw`, `JSONExtractKeyAndValues`. Renommé fonctions `jsonExtract` de `JSONExtract`. Quand quelque chose ne va pas, ces fonctions renvoient les valeurs correspondantes, pas `NULL`. Modifié la fonction `JSONExtract`, maintenant, il obtient le type de retour de son dernier paramètre et n’injecte pas nullables. Implémenté repli vers RapidJSON dans le cas où les instructions AVX2 ne sont pas disponibles. Bibliothèque Simdjson mise à jour vers une nouvelle version. [\#5235](https://github.com/ClickHouse/ClickHouse/pull/5235) ([Vitaly Baranov](https://github.com/vitlibar)) +- L'ajout de fonctions `JSONExtractRaw`, `JSONExtractKeyAndValues`. Renommé fonctions `jsonExtract` de `JSONExtract`. Quand quelque chose ne va pas, ces fonctions renvoient les valeurs correspondantes, pas `NULL`. Modifié la fonction `JSONExtract`, maintenant, il obtient le type de retour de son dernier paramètre et n'injecte pas nullables. Implémenté repli vers RapidJSON dans le cas où les instructions AVX2 ne sont pas disponibles. Bibliothèque Simdjson mise à jour vers une nouvelle version. [\#5235](https://github.com/ClickHouse/ClickHouse/pull/5235) ([Vitaly Baranov](https://github.com/vitlibar)) - Maintenant `if` et `multiIf` les fonctions ne dépendent pas de la condition `Nullable`, mais comptez sur les branches pour la compatibilité sql. [\#5238](https://github.com/ClickHouse/ClickHouse/pull/5238) ([Jian Wu](https://github.com/janplus)) -- `In` prédicat génère maintenant `Null` résultat de `Null` d’entrée comme l’ `Equal` fonction. [\#5152](https://github.com/ClickHouse/ClickHouse/pull/5152) ([Jian Wu](https://github.com/janplus)) +- `In` prédicat génère maintenant `Null` résultat de `Null` d'entrée comme l' `Equal` fonction. [\#5152](https://github.com/ClickHouse/ClickHouse/pull/5152) ([Jian Wu](https://github.com/janplus)) - Vérifiez la limite de temps chaque (flush\_interval / poll\_timeout) nombre de lignes de Kafka. Cela permet de casser la lecture de Kafka consumer plus fréquemment et de vérifier les délais pour les flux de niveau supérieur [\#5249](https://github.com/ClickHouse/ClickHouse/pull/5249) ([Ivan](https://github.com/abyss7)) -- Lien rdkafka avec SASL fourni. Il devrait permettre d’utiliser l’authentification SASL SCRAM [\#5253](https://github.com/ClickHouse/ClickHouse/pull/5253) ([Ivan](https://github.com/abyss7)) +- Lien rdkafka avec SASL fourni. Il devrait permettre d'utiliser l'authentification SASL SCRAM [\#5253](https://github.com/ClickHouse/ClickHouse/pull/5253) ([Ivan](https://github.com/abyss7)) - Version par lots de RowRefList pour toutes les jointures. [\#5267](https://github.com/ClickHouse/ClickHouse/pull/5267) ([Artem Zuikov](https://github.com/4ertus2)) -- clickhouse-server: messages d’erreur d’écoute plus informatifs. [\#5268](https://github.com/ClickHouse/ClickHouse/pull/5268) ([proller](https://github.com/proller)) +- clickhouse-server: messages d'erreur d'écoute plus informatifs. [\#5268](https://github.com/ClickHouse/ClickHouse/pull/5268) ([proller](https://github.com/proller)) - Soutien dictionnaires dans clickhouse-copieur pour les fonctions dans `` [\#5270](https://github.com/ClickHouse/ClickHouse/pull/5270) ([proller](https://github.com/proller)) - Ajouter un nouveau paramètre `kafka_commit_every_batch` pour réglementer Kafka engager la Politique. - Il permet de définir le mode de validation: après chaque lot de messages personnels, ou après le bloc entier est écrit dans le stockage. C’est un compromis entre perdre des messages ou les lire deux fois dans certaines situations extrêmes. [\#5308](https://github.com/ClickHouse/ClickHouse/pull/5308) ([Ivan](https://github.com/abyss7)) -- Faire `windowFunnel` supporte d’autres types entiers non signés. [\#5320](https://github.com/ClickHouse/ClickHouse/pull/5320) ([sundyli](https://github.com/sundy-li)) + Il permet de définir le mode de validation: après chaque lot de messages personnels, ou après le bloc entier est écrit dans le stockage. C'est un compromis entre perdre des messages ou les lire deux fois dans certaines situations extrêmes. [\#5308](https://github.com/ClickHouse/ClickHouse/pull/5308) ([Ivan](https://github.com/abyss7)) +- Faire `windowFunnel` supporte d'autres types entiers non signés. [\#5320](https://github.com/ClickHouse/ClickHouse/pull/5320) ([sundyli](https://github.com/sundy-li)) - Autoriser à ombrer la colonne virtuelle `_table` dans le moteur de Fusion. [\#5325](https://github.com/ClickHouse/ClickHouse/pull/5325) ([Ivan](https://github.com/abyss7)) -- Faire `sequenceMatch` les fonctions d’agrégation prennent en charge d’autres types entiers non signés [\#5339](https://github.com/ClickHouse/ClickHouse/pull/5339) ([sundyli](https://github.com/sundy-li)) -- Meilleurs messages d’erreur si l’inadéquation de la somme de contrôle est probablement causée par des défaillances matérielles. [\#5355](https://github.com/ClickHouse/ClickHouse/pull/5355) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Vérifiez que les tables sous-jacentes prennent en charge l’échantillonnage pour `StorageMerge` [\#5366](https://github.com/ClickHouse/ClickHouse/pull/5366) ([Ivan](https://github.com/abyss7)) +- Faire `sequenceMatch` les fonctions d'agrégation prennent en charge d'autres types entiers non signés [\#5339](https://github.com/ClickHouse/ClickHouse/pull/5339) ([sundyli](https://github.com/sundy-li)) +- Meilleurs messages d'erreur si l'inadéquation de la somme de contrôle est probablement causée par des défaillances matérielles. [\#5355](https://github.com/ClickHouse/ClickHouse/pull/5355) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Vérifiez que les tables sous-jacentes prennent en charge l'échantillonnage pour `StorageMerge` [\#5366](https://github.com/ClickHouse/ClickHouse/pull/5366) ([Ivan](https://github.com/abyss7)) - Сlose MySQL connections after their usage in external dictionaries. It is related to issue \#893. [\#5395](https://github.com/ClickHouse/ClickHouse/pull/5395) ([Clément Rodriguez](https://github.com/clemrodriguez)) - Améliorations du protocole de fil MySQL. Changement du nom du format en MySQLWire. Utiliser RAII pour appeler RSA\_free. Désactivation de SSL si le contexte ne peut pas être créé. [\#5419](https://github.com/ClickHouse/ClickHouse/pull/5419) ([Yuriy Baranov](https://github.com/yurriy)) - clickhouse-client: allow to run with unaccessable history file (read-only, no disk space, file is directory, …). [\#5431](https://github.com/ClickHouse/ClickHouse/pull/5431) ([proller](https://github.com/proller)) @@ -1287,9 +1287,9 @@ toc_title: '2019' #### Amélioration Des Performances {#performance-improvements} - Paralléliser le traitement des parties des tables MergeTree non répliquées dans la requête ALTER MODIFY. [\#4639](https://github.com/ClickHouse/ClickHouse/pull/4639) ([Ivan Kush](https://github.com/IvanKush)) -- Optimisations dans l’extraction d’expressions régulières. [\#5193](https://github.com/ClickHouse/ClickHouse/pull/5193) [\#5191](https://github.com/ClickHouse/ClickHouse/pull/5191) ([Danila Kutenin](https://github.com/danlark1)) -- N’ajoutez pas de colonne de clé de jointure droite pour joindre le résultat si elle est utilisée uniquement dans la section join on. [\#5260](https://github.com/ClickHouse/ClickHouse/pull/5260) ([Artem Zuikov](https://github.com/4ertus2)) -- Geler le tampon Kafka après la première réponse vide. Il évite les invocations multiples de `ReadBuffer::next()` pour un résultat vide dans certains flux d’analyse de ligne. [\#5283](https://github.com/ClickHouse/ClickHouse/pull/5283) ([Ivan](https://github.com/abyss7)) +- Optimisations dans l'extraction d'expressions régulières. [\#5193](https://github.com/ClickHouse/ClickHouse/pull/5193) [\#5191](https://github.com/ClickHouse/ClickHouse/pull/5191) ([Danila Kutenin](https://github.com/danlark1)) +- N'ajoutez pas de colonne de clé de jointure droite pour joindre le résultat si elle est utilisée uniquement dans la section join on. [\#5260](https://github.com/ClickHouse/ClickHouse/pull/5260) ([Artem Zuikov](https://github.com/4ertus2)) +- Geler le tampon Kafka après la première réponse vide. Il évite les invocations multiples de `ReadBuffer::next()` pour un résultat vide dans certains flux d'analyse de ligne. [\#5283](https://github.com/ClickHouse/ClickHouse/pull/5283) ([Ivan](https://github.com/abyss7)) - `concat` optimisation de la fonction pour plusieurs arguments. [\#5357](https://github.com/ClickHouse/ClickHouse/pull/5357) ([Danila Kutenin](https://github.com/danlark1)) - Query optimisation. Allow push down IN statement while rewriting commа/cross join into inner one. [\#5396](https://github.com/ClickHouse/ClickHouse/pull/5396) ([Artem Zuikov](https://github.com/4ertus2)) - Mettez à niveau notre implémentation LZ4 avec reference one pour avoir une décompression plus rapide. [\#5070](https://github.com/ClickHouse/ClickHouse/pull/5070) ([Danila Kutenin](https://github.com/danlark1)) @@ -1297,14 +1297,14 @@ toc_title: '2019' #### Corrections De Bugs {#bug-fixes} -- Correction des colonnes push require avec jointure [\#5192](https://github.com/ClickHouse/ClickHouse/pull/5192) ([L’Hiver Zhang](https://github.com/zhang2014)) -- Correction d’un bug, lorsque ClickHouse est exécuté par systemd, la commande `sudo service clickhouse-server forcerestart` ne fonctionne pas comme prévu. [\#5204](https://github.com/ClickHouse/ClickHouse/pull/5204) ([proller](https://github.com/proller)) -- Correction des codes d’erreur http dans DataPartsExchange (le serveur HTTP interserver sur le port 9009 renvoie toujours le code 200, même en cas d’erreurs). [\#5216](https://github.com/ClickHouse/ClickHouse/pull/5216) ([proller](https://github.com/proller)) +- Correction des colonnes push require avec jointure [\#5192](https://github.com/ClickHouse/ClickHouse/pull/5192) ([L'Hiver Zhang](https://github.com/zhang2014)) +- Correction d'un bug, lorsque ClickHouse est exécuté par systemd, la commande `sudo service clickhouse-server forcerestart` ne fonctionne pas comme prévu. [\#5204](https://github.com/ClickHouse/ClickHouse/pull/5204) ([proller](https://github.com/proller)) +- Correction des codes d'erreur http dans DataPartsExchange (le serveur HTTP interserver sur le port 9009 renvoie toujours le code 200, même en cas d'erreurs). [\#5216](https://github.com/ClickHouse/ClickHouse/pull/5216) ([proller](https://github.com/proller)) - Correction de SimpleAggregateFunction pour une chaîne plus longue que MAX\_SMALL\_STRING\_SIZE [\#5311](https://github.com/ClickHouse/ClickHouse/pull/5311) ([Azat Khuzhin](https://github.com/azat)) -- Correction d’une erreur pour `Decimal` de `Nullable(Decimal)` conversion en en. Soutenir D’autres conversions décimales à décimales (y compris différentes échelles). [\#5350](https://github.com/ClickHouse/ClickHouse/pull/5350) ([Artem Zuikov](https://github.com/4ertus2)) +- Correction d'une erreur pour `Decimal` de `Nullable(Decimal)` conversion en en. Soutenir D'autres conversions décimales à décimales (y compris différentes échelles). [\#5350](https://github.com/ClickHouse/ClickHouse/pull/5350) ([Artem Zuikov](https://github.com/4ertus2)) - Correction FPU clobbering dans la bibliothèque simdjson qui conduisent à un mauvais calcul de `uniqHLL` et `uniqCombined` fonction agrégée et fonctions mathématiques telles que `log`. [\#5354](https://github.com/ClickHouse/ClickHouse/pull/5354) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Correction de la gestion des cas const/nonconst mixtes dans les fonctions JSON. [\#5435](https://github.com/ClickHouse/ClickHouse/pull/5435) ([Vitaly Baranov](https://github.com/vitlibar)) -- Fixer `retention` fonction. Maintenant, toutes les conditions qui satisfont dans une rangée de données sont ajoutées à l’état des données. [\#5119](https://github.com/ClickHouse/ClickHouse/pull/5119) ([小路](https://github.com/nicelulu)) +- Fixer `retention` fonction. Maintenant, toutes les conditions qui satisfont dans une rangée de données sont ajoutées à l'état des données. [\#5119](https://github.com/ClickHouse/ClickHouse/pull/5119) ([小路](https://github.com/nicelulu)) - Correction du type de résultat pour `quantileExact` avec des Décimales. [\#5304](https://github.com/ClickHouse/ClickHouse/pull/5304) ([Artem Zuikov](https://github.com/4ertus2)) #### Documentation {#documentation} @@ -1320,7 +1320,7 @@ toc_title: '2019' - Correction de certains rapports de désinfectant qui montrent une utilisation probable après-livraison.[\#5139](https://github.com/ClickHouse/ClickHouse/pull/5139) [\#5143](https://github.com/ClickHouse/ClickHouse/pull/5143) [\#5393](https://github.com/ClickHouse/ClickHouse/pull/5393) ([Ivan](https://github.com/abyss7)) - Déplacez les tests de performance hors de répertoires séparés pour plus de commodité. [\#5158](https://github.com/ClickHouse/ClickHouse/pull/5158) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Correction de tests de performance incorrects. [\#5255](https://github.com/ClickHouse/ClickHouse/pull/5255) ([alésapine](https://github.com/alesapin)) -- Ajout d’un outil pour calculer les sommes de contrôle causées par les retournements de bits pour déboguer les problèmes matériels. [\#5334](https://github.com/ClickHouse/ClickHouse/pull/5334) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Ajout d'un outil pour calculer les sommes de contrôle causées par les retournements de bits pour déboguer les problèmes matériels. [\#5334](https://github.com/ClickHouse/ClickHouse/pull/5334) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Rendre le script runner plus utilisable. [\#5340](https://github.com/ClickHouse/ClickHouse/pull/5340)[\#5360](https://github.com/ClickHouse/ClickHouse/pull/5360) ([filimonov](https://github.com/filimonov)) - Ajouter petite instruction comment écrire des tests de performance. [\#5408](https://github.com/ClickHouse/ClickHouse/pull/5408) ([alésapine](https://github.com/alesapin)) - Ajout de la possibilité de faire des substitutions dans Créer, remplir et supprimer la requête dans les tests de performance [\#5367](https://github.com/ClickHouse/ClickHouse/pull/5367) ([Olga Khvostikova](https://github.com/stavrolia)) @@ -1331,7 +1331,7 @@ toc_title: '2019' #### Bug Fix {#bug-fix-25} -- Correction de la régression des performances dans certaines requêtes avec jointure. [\#5192](https://github.com/ClickHouse/ClickHouse/pull/5192) ([L’Hiver Zhang](https://github.com/zhang2014)) +- Correction de la régression des performances dans certaines requêtes avec jointure. [\#5192](https://github.com/ClickHouse/ClickHouse/pull/5192) ([L'Hiver Zhang](https://github.com/zhang2014)) ### Clickhouse Version 19.7.5.27, 2019-06-09 {#clickhouse-release-19-7-5-27-2019-06-09} @@ -1342,30 +1342,30 @@ toc_title: '2019' #### Corrections De Bugs {#bug-fixes-1} - Fixer erreur de segmentation sur `minmax` INDEX avec valeur nulle. [\#5246](https://github.com/ClickHouse/ClickHouse/pull/5246) ([Nikita Vasilev](https://github.com/nikvas0)) -- Marquez toutes les colonnes d’entrée dans LIMIT BY comme sortie requise. Il fixe ‘Not found column’ erreur dans certaines requêtes distribuées. [\#5407](https://github.com/ClickHouse/ClickHouse/pull/5407) ([Constantin S. Pan](https://github.com/kvap)) +- Marquez toutes les colonnes d'entrée dans LIMIT BY comme sortie requise. Il fixe ‘Not found column’ erreur dans certaines requêtes distribuées. [\#5407](https://github.com/ClickHouse/ClickHouse/pull/5407) ([Constantin S. Pan](https://github.com/kvap)) - Fixer “Column ‘0’ already exists” erreur dans `SELECT .. PREWHERE` sur la colonne avec défaut [\#5397](https://github.com/ClickHouse/ClickHouse/pull/5397) ([proller](https://github.com/proller)) - Fixer `ALTER MODIFY TTL` requête sur `ReplicatedMergeTree`. [\#5539](https://github.com/ClickHouse/ClickHouse/pull/5539/commits) ([Anton Popov](https://github.com/CurtizJ)) -- Ne plantez pas le serveur lorsque les consommateurs Kafka n’ont pas réussi à démarrer. [\#5285](https://github.com/ClickHouse/ClickHouse/pull/5285) ([Ivan](https://github.com/abyss7)) +- Ne plantez pas le serveur lorsque les consommateurs Kafka n'ont pas réussi à démarrer. [\#5285](https://github.com/ClickHouse/ClickHouse/pull/5285) ([Ivan](https://github.com/abyss7)) - Les fonctions Bitmap fixes produisent un mauvais résultat. [\#5359](https://github.com/ClickHouse/ClickHouse/pull/5359) ([Andy Yang](https://github.com/andyyzh)) -- Correction d’element\_count pour le dictionnaire haché (ne pas inclure les doublons) [\#5440](https://github.com/ClickHouse/ClickHouse/pull/5440) ([Azat Khuzhin](https://github.com/azat)) -- Utilisez le contenu de la variable d’environnement TZ comme nom pour le fuseau horaire. Il aide à détecter correctement le fuseau horaire par défaut dans certains cas.[\#5443](https://github.com/ClickHouse/ClickHouse/pull/5443) ([Ivan](https://github.com/abyss7)) -- N’essayez pas de convertir les entiers dans `dictGetT` fonctions, car il ne fonctionne pas correctement. Jetez une exception à la place. [\#5446](https://github.com/ClickHouse/ClickHouse/pull/5446) ([Artem Zuikov](https://github.com/4ertus2)) +- Correction d'element\_count pour le dictionnaire haché (ne pas inclure les doublons) [\#5440](https://github.com/ClickHouse/ClickHouse/pull/5440) ([Azat Khuzhin](https://github.com/azat)) +- Utilisez le contenu de la variable d'environnement TZ comme nom pour le fuseau horaire. Il aide à détecter correctement le fuseau horaire par défaut dans certains cas.[\#5443](https://github.com/ClickHouse/ClickHouse/pull/5443) ([Ivan](https://github.com/abyss7)) +- N'essayez pas de convertir les entiers dans `dictGetT` fonctions, car il ne fonctionne pas correctement. Jetez une exception à la place. [\#5446](https://github.com/ClickHouse/ClickHouse/pull/5446) ([Artem Zuikov](https://github.com/4ertus2)) - Correction des paramètres dans la requête HTTP ExternalData. [\#5455](https://github.com/ClickHouse/ClickHouse/pull/5455) ([Danila Kutenin](https://github.com/danlark1)) -- Correction d’un bug lorsque des pièces ont été supprimées uniquement de FS sans les laisser tomber de Zookeeper. [\#5520](https://github.com/ClickHouse/ClickHouse/pull/5520) ([alésapine](https://github.com/alesapin)) -- Correction d’un défaut de segmentation dans `bitmapHasAny` fonction. [\#5528](https://github.com/ClickHouse/ClickHouse/pull/5528) ([Zhichang Yu](https://github.com/yuzhichang)) -- Correction d’une erreur lorsque le pool de connexions de réplication ne tente pas de résoudre l’hôte, même lorsque le cache DNS a été supprimé. [\#5534](https://github.com/ClickHouse/ClickHouse/pull/5534) ([alésapine](https://github.com/alesapin)) -- Fixe `DROP INDEX IF EXISTS` requête. Maintenant `ALTER TABLE ... DROP INDEX IF EXISTS ...` la requête ne déclenche pas d’exception si l’index fourni n’existe pas. [\#5524](https://github.com/ClickHouse/ClickHouse/pull/5524) ([Gleb Novikov](https://github.com/NanoBjorn)) +- Correction d'un bug lorsque des pièces ont été supprimées uniquement de FS sans les laisser tomber de Zookeeper. [\#5520](https://github.com/ClickHouse/ClickHouse/pull/5520) ([alésapine](https://github.com/alesapin)) +- Correction d'un défaut de segmentation dans `bitmapHasAny` fonction. [\#5528](https://github.com/ClickHouse/ClickHouse/pull/5528) ([Zhichang Yu](https://github.com/yuzhichang)) +- Correction d'une erreur lorsque le pool de connexions de réplication ne tente pas de résoudre l'hôte, même lorsque le cache DNS a été supprimé. [\#5534](https://github.com/ClickHouse/ClickHouse/pull/5534) ([alésapine](https://github.com/alesapin)) +- Fixe `DROP INDEX IF EXISTS` requête. Maintenant `ALTER TABLE ... DROP INDEX IF EXISTS ...` la requête ne déclenche pas d'exception si l'index fourni n'existe pas. [\#5524](https://github.com/ClickHouse/ClickHouse/pull/5524) ([Gleb Novikov](https://github.com/NanoBjorn)) - Correction de la colonne union all supertype. Il y avait des cas avec des données incohérentes et des types de colonnes de colonnes résultantes. [\#5503](https://github.com/ClickHouse/ClickHouse/pull/5503) ([Artem Zuikov](https://github.com/4ertus2)) -- Ignorer ZNONODE pendant le traitement de la requête DDL. Avant si un autre nœud supprime le znode dans la file d’attente des tâches, celui qui +- Ignorer ZNONODE pendant le traitement de la requête DDL. Avant si un autre nœud supprime le znode dans la file d'attente des tâches, celui qui ne pas le traiter, mais déjà obtenir la liste des enfants, mettra fin au thread DDLWorker. [\#5489](https://github.com/ClickHouse/ClickHouse/pull/5489) ([Azat Khuzhin](https://github.com/azat)) -- Correction de L’insertion dans la table Distributed() avec une colonne matérialisée. [\#5429](https://github.com/ClickHouse/ClickHouse/pull/5429) ([Azat Khuzhin](https://github.com/azat)) +- Correction de L'insertion dans la table Distributed() avec une colonne matérialisée. [\#5429](https://github.com/ClickHouse/ClickHouse/pull/5429) ([Azat Khuzhin](https://github.com/azat)) ### Clickhouse Version 19.7.3.9, 2019-05-30 {#clickhouse-release-19-7-3-9-2019-05-30} #### Nouveauté {#new-features-2} -- Permet de limiter la plage d’un paramètre qui peut être spécifiée par l’utilisateur. +- Permet de limiter la plage d'un paramètre qui peut être spécifiée par l'utilisateur. Ces contraintes peuvent être configurées dans le profil des paramètres utilisateur. [\#4931](https://github.com/ClickHouse/ClickHouse/pull/4931) ([Vitaly Baranov](https://github.com/vitlibar)) @@ -1374,8 +1374,8 @@ toc_title: '2019' le comportement est similaire à `groupArray(max_size)(x)` fonction. [\#5026](https://github.com/ClickHouse/ClickHouse/pull/5026) ([Guillaume Tassery](https://github.com/YiuRULE)) -- Pour les formats de fichier D’entrée TSVWithNames/CSVWithNames, l’ordre des colonnes peut maintenant être - déterminé à partir de l’en-tête du fichier. Ceci est contrôlé par +- Pour les formats de fichier D'entrée TSVWithNames/CSVWithNames, l'ordre des colonnes peut maintenant être + déterminé à partir de l'en-tête du fichier. Ceci est contrôlé par `input_format_with_names_use_header` paramètre. [\#5081](https://github.com/ClickHouse/ClickHouse/pull/5081) ([Alexander](https://github.com/Akazz)) @@ -1391,7 +1391,7 @@ toc_title: '2019' - Perte de données sur une charge lourde via KafkaEngine (\#4736) [\#5080](https://github.com/ClickHouse/ClickHouse/pull/5080) ([Ivan](https://github.com/abyss7)) -- Correction d’une condition de concurrence de données très rare qui pourrait se produire lors de l’exécution d’une requête avec UNION impliquant au moins deux sélections du système.colonnes, système.tables système.les pièces, système d’.parts\_tables ou tables de la famille de fusion et effectuer des modifications simultanées des colonnes des tables associées. [\#5189](https://github.com/ClickHouse/ClickHouse/pull/5189) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction d'une condition de concurrence de données très rare qui pourrait se produire lors de l'exécution d'une requête avec UNION impliquant au moins deux sélections du système.colonnes, système.tables système.les pièces, système d'.parts\_tables ou tables de la famille de fusion et effectuer des modifications simultanées des colonnes des tables associées. [\#5189](https://github.com/ClickHouse/ClickHouse/pull/5189) ([alexeï-milovidov](https://github.com/alexey-milovidov)) #### Amélioration Des Performances {#performance-improvements-1} @@ -1434,12 +1434,12 @@ toc_title: '2019' #### Corrections De Bugs {#bug-fixes-3} -- Correction de l’état pushdown pour les requêtes des fonctions de table `mysql` et `odbc` et les moteurs de table correspondants. Cela corrige \#3540 et \# 2384. [\#5313](https://github.com/ClickHouse/ClickHouse/pull/5313) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction de l’impasse dans Zookeeper. [\#5297](https://github.com/ClickHouse/ClickHouse/pull/5297) ([github1youlc](https://github.com/github1youlc)) +- Correction de l'état pushdown pour les requêtes des fonctions de table `mysql` et `odbc` et les moteurs de table correspondants. Cela corrige \#3540 et \# 2384. [\#5313](https://github.com/ClickHouse/ClickHouse/pull/5313) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction de l'impasse dans Zookeeper. [\#5297](https://github.com/ClickHouse/ClickHouse/pull/5297) ([github1youlc](https://github.com/github1youlc)) - Autoriser les décimales entre guillemets en CSV. [\#5284](https://github.com/ClickHouse/ClickHouse/pull/5284) ([Artem Zuikov](https://github.com/4ertus2) - Interdire la conversion de float Inf / NaN en décimales (exception de lancer). [\#5282](https://github.com/ClickHouse/ClickHouse/pull/5282) ([Artem Zuikov](https://github.com/4ertus2)) -- Correction de la course de données dans la requête renommer. [\#5247](https://github.com/ClickHouse/ClickHouse/pull/5247) ([L’Hiver Zhang](https://github.com/zhang2014)) -- Désactiver temporairement LFAlloc. L’utilisation de LFAlloc peut conduire à beaucoup de MAP\_FAILED dans l’allocation de UncompressedCache et, par conséquent, à des plantages de requêtes sur des serveurs chargés en haut. [cfdba93](https://github.com/ClickHouse/ClickHouse/commit/cfdba938ce22f16efeec504f7f90206a515b1280)([Danila Kutenin](https://github.com/danlark1)) +- Correction de la course de données dans la requête renommer. [\#5247](https://github.com/ClickHouse/ClickHouse/pull/5247) ([L'Hiver Zhang](https://github.com/zhang2014)) +- Désactiver temporairement LFAlloc. L'utilisation de LFAlloc peut conduire à beaucoup de MAP\_FAILED dans l'allocation de UncompressedCache et, par conséquent, à des plantages de requêtes sur des serveurs chargés en haut. [cfdba93](https://github.com/ClickHouse/ClickHouse/commit/cfdba938ce22f16efeec504f7f90206a515b1280)([Danila Kutenin](https://github.com/danlark1)) ### Clickhouse Version 19.6.2.11, 2019-05-13 {#clickhouse-release-19-6-2-11-2019-05-13} @@ -1447,21 +1447,21 @@ toc_title: '2019' - Expressions TTL pour les colonnes et les tables. [\#4212](https://github.com/ClickHouse/ClickHouse/pull/4212) ([Anton Popov](https://github.com/CurtizJ)) - Ajout du support pour `brotli` compression pour les réponses HTTP (Accept-Encoding: br) [\#4388](https://github.com/ClickHouse/ClickHouse/pull/4388) ([Mikhail](https://github.com/fandyushin)) -- Ajout d’une fonction nouvelle `isValidUTF8` pour vérifier si un ensemble d’octets est correctement codé en utf-8. [\#4934](https://github.com/ClickHouse/ClickHouse/pull/4934) ([Danila Kutenin](https://github.com/danlark1)) -- Ajouter la nouvelle stratégie d’équilibrage de charge `first_or_random` qui envoie des requêtes au premier hôte spécifié et s’il est inaccessible, envoie des requêtes à des hôtes aléatoires de shard. Utile pour les configurations de topologie de réplication croisée. [\#5012](https://github.com/ClickHouse/ClickHouse/pull/5012) ([nvartolomei](https://github.com/nvartolomei)) +- Ajout d'une fonction nouvelle `isValidUTF8` pour vérifier si un ensemble d'octets est correctement codé en utf-8. [\#4934](https://github.com/ClickHouse/ClickHouse/pull/4934) ([Danila Kutenin](https://github.com/danlark1)) +- Ajouter la nouvelle stratégie d'équilibrage de charge `first_or_random` qui envoie des requêtes au premier hôte spécifié et s'il est inaccessible, envoie des requêtes à des hôtes aléatoires de shard. Utile pour les configurations de topologie de réplication croisée. [\#5012](https://github.com/ClickHouse/ClickHouse/pull/5012) ([nvartolomei](https://github.com/nvartolomei)) #### Caractéristiques Expérimentales {#experimental-features-1} -- Ajouter un paramètre `index_granularity_bytes` (granularité d’index adaptatif) pour la famille de tables MergeTree\*. [\#4826](https://github.com/ClickHouse/ClickHouse/pull/4826) ([alésapine](https://github.com/alesapin)) +- Ajouter un paramètre `index_granularity_bytes` (granularité d'index adaptatif) pour la famille de tables MergeTree\*. [\#4826](https://github.com/ClickHouse/ClickHouse/pull/4826) ([alésapine](https://github.com/alesapin)) #### Amélioration {#improvements-1} - Ajout du support pour les arguments de taille et de longueur non constants et négatifs pour la fonction `substringUTF8`. [\#4989](https://github.com/ClickHouse/ClickHouse/pull/4989) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Désactivez push-down to right table dans LEFT join, left table dans RIGHT join et les deux tables dans full join. Cela corrige les mauvais résultats de jointure dans certains cas. [\#4846](https://github.com/ClickHouse/ClickHouse/pull/4846) ([Ivan](https://github.com/abyss7)) - `clickhouse-copier`: configuration automatique de tâche de téléchargement de `--task-file` option [\#4876](https://github.com/ClickHouse/ClickHouse/pull/4876) ([proller](https://github.com/proller)) -- Ajout d’un gestionnaire de fautes de frappe pour l’usine de stockage et les fonctions de table factory. [\#4891](https://github.com/ClickHouse/ClickHouse/pull/4891) ([Danila Kutenin](https://github.com/danlark1)) +- Ajout d'un gestionnaire de fautes de frappe pour l'usine de stockage et les fonctions de table factory. [\#4891](https://github.com/ClickHouse/ClickHouse/pull/4891) ([Danila Kutenin](https://github.com/danlark1)) - Support des astérisques et des astérisques qualifiés pour plusieurs jointures sans sous-requêtes [\#4898](https://github.com/ClickHouse/ClickHouse/pull/4898) ([Artem Zuikov](https://github.com/4ertus2)) -- Rendre le message d’erreur de colonne manquant plus convivial. [\#4915](https://github.com/ClickHouse/ClickHouse/pull/4915) ([Artem Zuikov](https://github.com/4ertus2)) +- Rendre le message d'erreur de colonne manquant plus convivial. [\#4915](https://github.com/ClickHouse/ClickHouse/pull/4915) ([Artem Zuikov](https://github.com/4ertus2)) #### Amélioration Des Performances {#performance-improvements-2} @@ -1469,22 +1469,22 @@ toc_title: '2019' #### Modifications Incompatibles En Arrière {#backward-incompatible-changes} -- L’en-tête HTTP `Query-Id` a été renommé `X-ClickHouse-Query-Id` pour des raisons de cohérence. [\#4972](https://github.com/ClickHouse/ClickHouse/pull/4972) ([Mikhail](https://github.com/fandyushin)) +- L'en-tête HTTP `Query-Id` a été renommé `X-ClickHouse-Query-Id` pour des raisons de cohérence. [\#4972](https://github.com/ClickHouse/ClickHouse/pull/4972) ([Mikhail](https://github.com/fandyushin)) #### Corrections De Bugs {#bug-fixes-4} - Déréférence du pointeur nul potentiel fixe dans `clickhouse-copier`. [\#4900](https://github.com/ClickHouse/ClickHouse/pull/4900) ([proller](https://github.com/proller)) -- Correction d’une erreur sur la requête avec JOIN + ARRAY JOIN [\#4938](https://github.com/ClickHouse/ClickHouse/pull/4938) ([Artem Zuikov](https://github.com/4ertus2)) -- Fixe suspendu au début du serveur lorsqu’un dictionnaire dépend d’un autre dictionnaire via une base de données avec moteur=Dictionnaire. [\#4962](https://github.com/ClickHouse/ClickHouse/pull/4962) ([Vitaly Baranov](https://github.com/vitlibar)) -- Partially fix distributed\_product\_mode = local. It’s possible to allow columns of local tables in where/having/order by/… via table aliases. Throw exception if table does not have alias. There’s not possible to access to the columns without table aliases yet. [\#4986](https://github.com/ClickHouse/ClickHouse/pull/4986) ([Artem Zuikov](https://github.com/4ertus2)) -- Correction d’un résultat potentiellement erroné pour `SELECT DISTINCT` avec `JOIN` [\#5001](https://github.com/ClickHouse/ClickHouse/pull/5001) ([Artem Zuikov](https://github.com/4ertus2)) -- Correction d’une condition de concurrence de données très rare qui pourrait se produire lors de l’exécution d’une requête avec UNION impliquant au moins deux sélections du système.colonnes, système.tables système.les pièces, système d’.parts\_tables ou tables de la famille de fusion et effectuer des modifications simultanées des colonnes des tables associées. [\#5189](https://github.com/ClickHouse/ClickHouse/pull/5189) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction d'une erreur sur la requête avec JOIN + ARRAY JOIN [\#4938](https://github.com/ClickHouse/ClickHouse/pull/4938) ([Artem Zuikov](https://github.com/4ertus2)) +- Fixe suspendu au début du serveur lorsqu'un dictionnaire dépend d'un autre dictionnaire via une base de données avec moteur=Dictionnaire. [\#4962](https://github.com/ClickHouse/ClickHouse/pull/4962) ([Vitaly Baranov](https://github.com/vitlibar)) +- Partially fix distributed\_product\_mode = local. It's possible to allow columns of local tables in where/having/order by/… via table aliases. Throw exception if table does not have alias. There's not possible to access to the columns without table aliases yet. [\#4986](https://github.com/ClickHouse/ClickHouse/pull/4986) ([Artem Zuikov](https://github.com/4ertus2)) +- Correction d'un résultat potentiellement erroné pour `SELECT DISTINCT` avec `JOIN` [\#5001](https://github.com/ClickHouse/ClickHouse/pull/5001) ([Artem Zuikov](https://github.com/4ertus2)) +- Correction d'une condition de concurrence de données très rare qui pourrait se produire lors de l'exécution d'une requête avec UNION impliquant au moins deux sélections du système.colonnes, système.tables système.les pièces, système d'.parts\_tables ou tables de la famille de fusion et effectuer des modifications simultanées des colonnes des tables associées. [\#5189](https://github.com/ClickHouse/ClickHouse/pull/5189) ([alexeï-milovidov](https://github.com/alexey-milovidov)) #### Construire/Test/Emballage Améliorations {#buildtestingpackaging-improvements-2} -- Correction des échecs de test lors de l’exécution de clickhouse-server sur un hôte différent [\#4713](https://github.com/ClickHouse/ClickHouse/pull/4713) ([Vasily Nemkov](https://github.com/Enmk)) +- Correction des échecs de test lors de l'exécution de clickhouse-server sur un hôte différent [\#4713](https://github.com/ClickHouse/ClickHouse/pull/4713) ([Vasily Nemkov](https://github.com/Enmk)) - clickhouse-test: désactiver les séquences de contrôle de couleur dans un environnement non tty. [\#4937](https://github.com/ClickHouse/ClickHouse/pull/4937) ([alésapine](https://github.com/alesapin)) -- clickhouse-test: Autoriser l’utilisation de toute base de données de test (supprimer `test.` qualification lorsque cela est possible) [\#5008](https://github.com/ClickHouse/ClickHouse/pull/5008) ([proller](https://github.com/proller)) +- clickhouse-test: Autoriser l'utilisation de toute base de données de test (supprimer `test.` qualification lorsque cela est possible) [\#5008](https://github.com/ClickHouse/ClickHouse/pull/5008) ([proller](https://github.com/proller)) - Correction des erreurs ubsan [\#5037](https://github.com/ClickHouse/ClickHouse/pull/5037) ([Vitaly Baranov](https://github.com/vitlibar)) - Yandex LFAlloc a été ajouté à ClickHouse pour allouer des données MarkCache et UncompressedCache de différentes manières pour attraper des segfaults plus fiables [\#4995](https://github.com/ClickHouse/ClickHouse/pull/4995) ([Danila Kutenin](https://github.com/danlark1)) - Python util pour aider avec les rétroportages et les changelogs. [\#4949](https://github.com/ClickHouse/ClickHouse/pull/4949) ([Ivan](https://github.com/abyss7)) @@ -1495,10 +1495,10 @@ toc_title: '2019' #### Corrections De Bugs {#bug-fixes-5} -- Correction d’un crash possible dans les fonctions bitmap\* [\#5220](https://github.com/ClickHouse/ClickHouse/pull/5220) [\#5228](https://github.com/ClickHouse/ClickHouse/pull/5228) ([Andy Yang](https://github.com/andyyzh)) -- Correction d’une condition de concurrence de données très rare qui pourrait se produire lors de l’exécution d’une requête avec UNION impliquant au moins deux sélections du système.colonnes, système.tables système.les pièces, système d’.parts\_tables ou tables de la famille de fusion et effectuer des modifications simultanées des colonnes des tables associées. [\#5189](https://github.com/ClickHouse/ClickHouse/pull/5189) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’une erreur `Set for IN is not created yet in case of using single LowCardinality column in the left part of IN`. Cette erreur s’est produite si la colonne LowCardinality était la partie de la clé primaire. \#5031 [\#5154](https://github.com/ClickHouse/ClickHouse/pull/5154) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Modification de la fonction de rétention: si une ligne satisfait à la fois la première et la nième condition, seule la première condition satisfaite est ajoutée à l’état des données. Maintenant, toutes les conditions qui satisfont dans une rangée de données sont ajoutées à l’état des données. [\#5119](https://github.com/ClickHouse/ClickHouse/pull/5119) ([小路](https://github.com/nicelulu)) +- Correction d'un crash possible dans les fonctions bitmap\* [\#5220](https://github.com/ClickHouse/ClickHouse/pull/5220) [\#5228](https://github.com/ClickHouse/ClickHouse/pull/5228) ([Andy Yang](https://github.com/andyyzh)) +- Correction d'une condition de concurrence de données très rare qui pourrait se produire lors de l'exécution d'une requête avec UNION impliquant au moins deux sélections du système.colonnes, système.tables système.les pièces, système d'.parts\_tables ou tables de la famille de fusion et effectuer des modifications simultanées des colonnes des tables associées. [\#5189](https://github.com/ClickHouse/ClickHouse/pull/5189) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction d'une erreur `Set for IN is not created yet in case of using single LowCardinality column in the left part of IN`. Cette erreur s'est produite si la colonne LowCardinality était la partie de la clé primaire. \#5031 [\#5154](https://github.com/ClickHouse/ClickHouse/pull/5154) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Modification de la fonction de rétention: si une ligne satisfait à la fois la première et la nième condition, seule la première condition satisfaite est ajoutée à l'état des données. Maintenant, toutes les conditions qui satisfont dans une rangée de données sont ajoutées à l'état des données. [\#5119](https://github.com/ClickHouse/ClickHouse/pull/5119) ([小路](https://github.com/nicelulu)) ### Clickhouse Version 19.5.3.8, 2019-04-18 {#clickhouse-release-19-5-3-8-2019-04-18} @@ -1512,93 +1512,93 @@ toc_title: '2019' - [Hyperscan](https://github.com/intel/hyperscan) plusieurs expressions rationnelles a été ajouté (fonctions `multiMatchAny`, `multiMatchAnyIndex`, `multiFuzzyMatchAny`, `multiFuzzyMatchAnyIndex`). [\#4780](https://github.com/ClickHouse/ClickHouse/pull/4780), [\#4841](https://github.com/ClickHouse/ClickHouse/pull/4841) ([Danila Kutenin](https://github.com/danlark1)) - `multiSearchFirstPosition` la fonction a été ajoutée. [\#4780](https://github.com/ClickHouse/ClickHouse/pull/4780) ([Danila Kutenin](https://github.com/danlark1)) -- Implémentez le filtre d’expression prédéfini par ligne pour les tables. [\#4792](https://github.com/ClickHouse/ClickHouse/pull/4792) ([Ivan](https://github.com/abyss7)) -- Un nouveau type d’indices de saut de données basés sur des filtres bloom (peut être utilisé pour `equal`, `in` et `like` fonction). [\#4499](https://github.com/ClickHouse/ClickHouse/pull/4499) ([Nikita Vasilev](https://github.com/nikvas0)) -- Ajouter `ASOF JOIN` ce qui permet d’exécuter des requêtes, qui se joignent à la valeur la plus récente connue. [\#4774](https://github.com/ClickHouse/ClickHouse/pull/4774) [\#4867](https://github.com/ClickHouse/ClickHouse/pull/4867) [\#4863](https://github.com/ClickHouse/ClickHouse/pull/4863) [\#4875](https://github.com/ClickHouse/ClickHouse/pull/4875) ([Martijn Bakker](https://github.com/Gladdy), [Artem Zuikov](https://github.com/4ertus2)) +- Implémentez le filtre d'expression prédéfini par ligne pour les tables. [\#4792](https://github.com/ClickHouse/ClickHouse/pull/4792) ([Ivan](https://github.com/abyss7)) +- Un nouveau type d'indices de saut de données basés sur des filtres bloom (peut être utilisé pour `equal`, `in` et `like` fonction). [\#4499](https://github.com/ClickHouse/ClickHouse/pull/4499) ([Nikita Vasilev](https://github.com/nikvas0)) +- Ajouter `ASOF JOIN` ce qui permet d'exécuter des requêtes, qui se joignent à la valeur la plus récente connue. [\#4774](https://github.com/ClickHouse/ClickHouse/pull/4774) [\#4867](https://github.com/ClickHouse/ClickHouse/pull/4867) [\#4863](https://github.com/ClickHouse/ClickHouse/pull/4863) [\#4875](https://github.com/ClickHouse/ClickHouse/pull/4875) ([Martijn Bakker](https://github.com/Gladdy), [Artem Zuikov](https://github.com/4ertus2)) - Réécrire plusieurs `COMMA JOIN` de `CROSS JOIN`. Puis réécrire `INNER JOIN` si cela est possible. [\#4661](https://github.com/ClickHouse/ClickHouse/pull/4661) ([Artem Zuikov](https://github.com/4ertus2)) #### Amélioration {#improvement-9} - `topK` et `topKWeighted` prend désormais en charge personnalisée `loadFactor` (corrige le problème [\#4252](https://github.com/ClickHouse/ClickHouse/issues/4252)). [\#4634](https://github.com/ClickHouse/ClickHouse/pull/4634) ([Kirill Danshin](https://github.com/kirillDanshin)) -- Permettre d’utiliser `parallel_replicas_count > 1` même pour les tables sans échantillonnage (le paramètre est simplement ignoré pour elles). Dans les versions précédentes, il a été conduit à l’exception. [\#4637](https://github.com/ClickHouse/ClickHouse/pull/4637) ([Alexey Elymanov](https://github.com/digitalist)) +- Permettre d'utiliser `parallel_replicas_count > 1` même pour les tables sans échantillonnage (le paramètre est simplement ignoré pour elles). Dans les versions précédentes, il a été conduit à l'exception. [\#4637](https://github.com/ClickHouse/ClickHouse/pull/4637) ([Alexey Elymanov](https://github.com/digitalist)) - Soutien pour `CREATE OR REPLACE VIEW`. Permettent de créer un afficher ou définir une nouvelle définition dans un seul énoncé. [\#4654](https://github.com/ClickHouse/ClickHouse/pull/4654) ([Boris Granveaud](https://github.com/bgranvea)) - `Buffer` tableau moteur prend désormais en charge `PREWHERE`. [\#4671](https://github.com/ClickHouse/ClickHouse/pull/4671) ([Yangkuan Liu](https://github.com/LiuYangkuan)) - Ajouter la possibilité de démarrer la table répliquée sans métadonnées dans zookeeper dans `readonly` mode. [\#4691](https://github.com/ClickHouse/ClickHouse/pull/4691) ([alésapine](https://github.com/alesapin)) -- Correction du scintillement de la barre de progression dans clickhouse-client. Le problème était le plus perceptible lors de l’utilisation `FORMAT Null` avec des flux de requêtes. [\#4811](https://github.com/ClickHouse/ClickHouse/pull/4811) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Permettent de désactiver les fonctions avec `hyperscan` bibliothèque par utilisateur pour limiter l’utilisation potentiellement excessive et incontrôlée des ressources. [\#4816](https://github.com/ClickHouse/ClickHouse/pull/4816) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction du scintillement de la barre de progression dans clickhouse-client. Le problème était le plus perceptible lors de l'utilisation `FORMAT Null` avec des flux de requêtes. [\#4811](https://github.com/ClickHouse/ClickHouse/pull/4811) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Permettent de désactiver les fonctions avec `hyperscan` bibliothèque par utilisateur pour limiter l'utilisation potentiellement excessive et incontrôlée des ressources. [\#4816](https://github.com/ClickHouse/ClickHouse/pull/4816) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Ajouter le numéro de version se connectant à toutes les erreurs. [\#4824](https://github.com/ClickHouse/ClickHouse/pull/4824) ([proller](https://github.com/proller)) -- Ajout d’une restriction à la `multiMatch` fonctions qui nécessite une taille de chaîne pour s’adapter à `unsigned int`. A également ajouté le nombre d’arguments limite à la `multiSearch` fonction. [\#4834](https://github.com/ClickHouse/ClickHouse/pull/4834) ([Danila Kutenin](https://github.com/danlark1)) -- Amélioration de l’utilisation de l’espace scratch et de la gestion des erreurs dans Hyperscan. [\#4866](https://github.com/ClickHouse/ClickHouse/pull/4866) ([Danila Kutenin](https://github.com/danlark1)) -- Remplir `system.graphite_detentions` à partir d’une table de config `*GraphiteMergeTree` le moteur de tables. [\#4584](https://github.com/ClickHouse/ClickHouse/pull/4584) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) -- Renommer `trigramDistance` la fonction de `ngramDistance` et d’ajouter plus de fonctions avec `CaseInsensitive` et `UTF`. [\#4602](https://github.com/ClickHouse/ClickHouse/pull/4602) ([Danila Kutenin](https://github.com/danlark1)) +- Ajout d'une restriction à la `multiMatch` fonctions qui nécessite une taille de chaîne pour s'adapter à `unsigned int`. A également ajouté le nombre d'arguments limite à la `multiSearch` fonction. [\#4834](https://github.com/ClickHouse/ClickHouse/pull/4834) ([Danila Kutenin](https://github.com/danlark1)) +- Amélioration de l'utilisation de l'espace scratch et de la gestion des erreurs dans Hyperscan. [\#4866](https://github.com/ClickHouse/ClickHouse/pull/4866) ([Danila Kutenin](https://github.com/danlark1)) +- Remplir `system.graphite_detentions` à partir d'une table de config `*GraphiteMergeTree` le moteur de tables. [\#4584](https://github.com/ClickHouse/ClickHouse/pull/4584) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) +- Renommer `trigramDistance` la fonction de `ngramDistance` et d'ajouter plus de fonctions avec `CaseInsensitive` et `UTF`. [\#4602](https://github.com/ClickHouse/ClickHouse/pull/4602) ([Danila Kutenin](https://github.com/danlark1)) - Amélioration du calcul des indices de saut de données. [\#4640](https://github.com/ClickHouse/ClickHouse/pull/4640) ([Nikita Vasilev](https://github.com/nikvas0)) - Garder ordinaire, `DEFAULT`, `MATERIALIZED` et `ALIAS` colonnes dans une seule liste (corrige le problème [\#2867](https://github.com/ClickHouse/ClickHouse/issues/2867)). [\#4707](https://github.com/ClickHouse/ClickHouse/pull/4707) ([Alex Zatelepin](https://github.com/ztlpn)) #### Bug Fix {#bug-fix-26} -- Éviter `std::terminate` en cas d’échec d’allocation de mémoire. Maintenant `std::bad_alloc` exception est levée comme prévu. [\#4665](https://github.com/ClickHouse/ClickHouse/pull/4665) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Corrige la lecture de capnproto à partir du tampon. Parfois, les fichiers N’ont pas été chargés avec succès par HTTP. [\#4674](https://github.com/ClickHouse/ClickHouse/pull/4674) ([Vladislav](https://github.com/smirnov-vs)) -- Correction d’erreur `Unknown log entry type: 0` après `OPTIMIZE TABLE FINAL` requête. [\#4683](https://github.com/ClickHouse/ClickHouse/pull/4683) ([Amos Oiseau](https://github.com/amosbird)) -- Mauvais arguments `hasAny` ou `hasAll` fonctions peut conduire à l’erreur de segmentation. [\#4698](https://github.com/ClickHouse/ClickHouse/pull/4698) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Blocage peut se produire lors de l’exécution `DROP DATABASE dictionary` requête. [\#4701](https://github.com/ClickHouse/ClickHouse/pull/4701) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Éviter `std::terminate` en cas d'échec d'allocation de mémoire. Maintenant `std::bad_alloc` exception est levée comme prévu. [\#4665](https://github.com/ClickHouse/ClickHouse/pull/4665) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Corrige la lecture de capnproto à partir du tampon. Parfois, les fichiers N'ont pas été chargés avec succès par HTTP. [\#4674](https://github.com/ClickHouse/ClickHouse/pull/4674) ([Vladislav](https://github.com/smirnov-vs)) +- Correction d'erreur `Unknown log entry type: 0` après `OPTIMIZE TABLE FINAL` requête. [\#4683](https://github.com/ClickHouse/ClickHouse/pull/4683) ([Amos Oiseau](https://github.com/amosbird)) +- Mauvais arguments `hasAny` ou `hasAll` fonctions peut conduire à l'erreur de segmentation. [\#4698](https://github.com/ClickHouse/ClickHouse/pull/4698) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Blocage peut se produire lors de l'exécution `DROP DATABASE dictionary` requête. [\#4701](https://github.com/ClickHouse/ClickHouse/pull/4701) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Corriger un comportement indéfini dans `median` et `quantile` fonction. [\#4702](https://github.com/ClickHouse/ClickHouse/pull/4702) ([hcz](https://github.com/hczhcz)) - Correction de la détection du niveau de compression lorsque `network_compression_method` en minuscules. Cassé dans v19. 1. [\#4706](https://github.com/ClickHouse/ClickHouse/pull/4706) ([proller](https://github.com/proller)) -- Correction de l’ignorance de `UTC` paramètre (corrige le problème [\#4658](https://github.com/ClickHouse/ClickHouse/issues/4658)). [\#4718](https://github.com/ClickHouse/ClickHouse/pull/4718) ([proller](https://github.com/proller)) +- Correction de l'ignorance de `UTC` paramètre (corrige le problème [\#4658](https://github.com/ClickHouse/ClickHouse/issues/4658)). [\#4718](https://github.com/ClickHouse/ClickHouse/pull/4718) ([proller](https://github.com/proller)) - Fixer `histogram` comportement de la fonction avec `Distributed` table. [\#4741](https://github.com/ClickHouse/ClickHouse/pull/4741) ([olegkv](https://github.com/olegkv)) - Rapport Tsan fixe `destroy of a locked mutex`. [\#4742](https://github.com/ClickHouse/ClickHouse/pull/4742) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Rapport Tsan fixe sur l’arrêt en raison de la condition de concurrence dans l’utilisation des journaux système. Correction de l’utilisation potentielle-après-libre à l’arrêt lorsque part\_log est activé. [\#4758](https://github.com/ClickHouse/ClickHouse/pull/4758) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Fixer revérifier les pièces dans `ReplicatedMergeTreeAlterThread` en cas d’erreur. [\#4772](https://github.com/ClickHouse/ClickHouse/pull/4772) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Les opérations arithmétiques sur les états de fonction d’agrégat intermédiaire ne fonctionnaient pas pour les arguments constants (tels que les résultats de sous-requête). [\#4776](https://github.com/ClickHouse/ClickHouse/pull/4776) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Rapport Tsan fixe sur l'arrêt en raison de la condition de concurrence dans l'utilisation des journaux système. Correction de l'utilisation potentielle-après-libre à l'arrêt lorsque part\_log est activé. [\#4758](https://github.com/ClickHouse/ClickHouse/pull/4758) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Fixer revérifier les pièces dans `ReplicatedMergeTreeAlterThread` en cas d'erreur. [\#4772](https://github.com/ClickHouse/ClickHouse/pull/4772) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Les opérations arithmétiques sur les états de fonction d'agrégat intermédiaire ne fonctionnaient pas pour les arguments constants (tels que les résultats de sous-requête). [\#4776](https://github.com/ClickHouse/ClickHouse/pull/4776) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Toujours renvoyer les noms de colonne dans les métadonnées. Sinon il est impossible de créer une table avec une colonne nommée `index` (le serveur ne redémarre pas en raison de malformé `ATTACH` requête dans les métadonnées). [\#4782](https://github.com/ClickHouse/ClickHouse/pull/4782) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’un crash dans l’ `ALTER ... MODIFY ORDER BY` sur `Distributed` table. [\#4790](https://github.com/ClickHouse/ClickHouse/pull/4790) ([TCeason](https://github.com/TCeason)) -- Fixer erreur de segmentation dans `JOIN ON` avec l’option `enable_optimize_predicate_expression`. [\#4794](https://github.com/ClickHouse/ClickHouse/pull/4794) ([L’Hiver Zhang](https://github.com/zhang2014)) -- Correction d’un bug avec l’ajout d’une ligne étrangère après avoir consommé un message protobuf de Kafka. [\#4808](https://github.com/ClickHouse/ClickHouse/pull/4808) ([Vitaly Baranov](https://github.com/vitlibar)) -- Correction d’un arrêt brutal de `JOIN` sur la colonne non nullable vs nullable. Fixer `NULLs` dans la droite dans `ANY JOIN` + `join_use_nulls`. [\#4815](https://github.com/ClickHouse/ClickHouse/pull/4815) ([Artem Zuikov](https://github.com/4ertus2)) -- Correction d’un défaut de segmentation dans `clickhouse-copier`. [\#4835](https://github.com/ClickHouse/ClickHouse/pull/4835) ([proller](https://github.com/proller)) +- Correction d'un crash dans l' `ALTER ... MODIFY ORDER BY` sur `Distributed` table. [\#4790](https://github.com/ClickHouse/ClickHouse/pull/4790) ([TCeason](https://github.com/TCeason)) +- Fixer erreur de segmentation dans `JOIN ON` avec l'option `enable_optimize_predicate_expression`. [\#4794](https://github.com/ClickHouse/ClickHouse/pull/4794) ([L'Hiver Zhang](https://github.com/zhang2014)) +- Correction d'un bug avec l'ajout d'une ligne étrangère après avoir consommé un message protobuf de Kafka. [\#4808](https://github.com/ClickHouse/ClickHouse/pull/4808) ([Vitaly Baranov](https://github.com/vitlibar)) +- Correction d'un arrêt brutal de `JOIN` sur la colonne non nullable vs nullable. Fixer `NULLs` dans la droite dans `ANY JOIN` + `join_use_nulls`. [\#4815](https://github.com/ClickHouse/ClickHouse/pull/4815) ([Artem Zuikov](https://github.com/4ertus2)) +- Correction d'un défaut de segmentation dans `clickhouse-copier`. [\#4835](https://github.com/ClickHouse/ClickHouse/pull/4835) ([proller](https://github.com/proller)) - Condition de course fixe dans `SELECT` de `system.tables` si la table est renommée ou modifiée simultanément. [\#4836](https://github.com/ClickHouse/ClickHouse/pull/4836) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction de la course de données lors de la récupération d’une partie de données déjà obsolète. [\#4839](https://github.com/ClickHouse/ClickHouse/pull/4839) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction de la course de données lors de la récupération d'une partie de données déjà obsolète. [\#4839](https://github.com/ClickHouse/ClickHouse/pull/4839) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Correction de la course de données rares qui peut se produire pendant `RENAME` table de la famille MergeTree. [\#4844](https://github.com/ClickHouse/ClickHouse/pull/4844) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’un défaut de segmentation en fonction `arrayIntersect`. Une erreur de Segmentation pourrait se produire si la fonction était appelée avec des arguments constants et ordinaires mixtes. [\#4847](https://github.com/ClickHouse/ClickHouse/pull/4847) ([Lixiang Qian](https://github.com/fancyqlx)) +- Correction d'un défaut de segmentation en fonction `arrayIntersect`. Une erreur de Segmentation pourrait se produire si la fonction était appelée avec des arguments constants et ordinaires mixtes. [\#4847](https://github.com/ClickHouse/ClickHouse/pull/4847) ([Lixiang Qian](https://github.com/fancyqlx)) - Correction de la lecture de `Array(LowCardinality)` colonne dans de rares cas où la colonne contenait une longue séquence de tableaux vides. [\#4850](https://github.com/ClickHouse/ClickHouse/pull/4850) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Correction d’un crash dans l’ `FULL/RIGHT JOIN` lorsque nous rejoignons sur nullable vs non nullable. [\#4855](https://github.com/ClickHouse/ClickHouse/pull/4855) ([Artem Zuikov](https://github.com/4ertus2)) +- Correction d'un crash dans l' `FULL/RIGHT JOIN` lorsque nous rejoignons sur nullable vs non nullable. [\#4855](https://github.com/ClickHouse/ClickHouse/pull/4855) ([Artem Zuikov](https://github.com/4ertus2)) - Fixer `No message received` exception lors de la récupération de pièces entre les répliques. [\#4856](https://github.com/ClickHouse/ClickHouse/pull/4856) ([alésapine](https://github.com/alesapin)) - Fixe `arrayIntersect` fonction mauvais résultat en cas de plusieurs valeurs répétées dans un seul tableau. [\#4871](https://github.com/ClickHouse/ClickHouse/pull/4871) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Correction d’une condition de course pendant `ALTER COLUMN` requêtes qui pourraient conduire à un plantage du serveur (corrige le problème [\#3421](https://github.com/ClickHouse/ClickHouse/issues/3421)). [\#4592](https://github.com/ClickHouse/ClickHouse/pull/4592) ([Alex Zatelepin](https://github.com/ztlpn)) -- Correction d’un résultat incorrect dans `FULL/RIGHT JOIN` avec colonne const. [\#4723](https://github.com/ClickHouse/ClickHouse/pull/4723) ([Artem Zuikov](https://github.com/4ertus2)) +- Correction d'une condition de course pendant `ALTER COLUMN` requêtes qui pourraient conduire à un plantage du serveur (corrige le problème [\#3421](https://github.com/ClickHouse/ClickHouse/issues/3421)). [\#4592](https://github.com/ClickHouse/ClickHouse/pull/4592) ([Alex Zatelepin](https://github.com/ztlpn)) +- Correction d'un résultat incorrect dans `FULL/RIGHT JOIN` avec colonne const. [\#4723](https://github.com/ClickHouse/ClickHouse/pull/4723) ([Artem Zuikov](https://github.com/4ertus2)) - Correction des doublons dans `GLOBAL JOIN` avec asterisk. [\#4705](https://github.com/ClickHouse/ClickHouse/pull/4705) ([Artem Zuikov](https://github.com/4ertus2)) -- Paramètre correction de la déduction `ALTER MODIFY` de la colonne `CODEC` lorsque le type de colonne n’est pas spécifié. [\#4883](https://github.com/ClickHouse/ClickHouse/pull/4883) ([alésapine](https://github.com/alesapin)) +- Paramètre correction de la déduction `ALTER MODIFY` de la colonne `CODEC` lorsque le type de colonne n'est pas spécifié. [\#4883](https://github.com/ClickHouse/ClickHouse/pull/4883) ([alésapine](https://github.com/alesapin)) - Fonction `cutQueryStringAndFragment()` et `queryStringAndFragment()` fonctionne maintenant correctement quand `URL` contient un fragment et aucune requête. [\#4894](https://github.com/ClickHouse/ClickHouse/pull/4894) ([Vitaly Baranov](https://github.com/vitlibar)) -- Correction d’un bug rare lors du réglage `min_bytes_to_use_direct_io` est supérieur à zéro, ce qui se produit lorsque le thread doit chercher en arrière dans le fichier de colonne. [\#4897](https://github.com/ClickHouse/ClickHouse/pull/4897) ([alésapine](https://github.com/alesapin)) -- Correction de mauvais types d’arguments pour les fonctions d’agrégation avec `LowCardinality` arguments (corrige le problème [\#4919](https://github.com/ClickHouse/ClickHouse/issues/4919)). [\#4922](https://github.com/ClickHouse/ClickHouse/pull/4922) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Correction d’une qualification de nom incorrecte Dans `GLOBAL JOIN`. [\#4969](https://github.com/ClickHouse/ClickHouse/pull/4969) ([Artem Zuikov](https://github.com/4ertus2)) -- Fonction Fix `toISOWeek` résultat pour l’année 1970. [\#4988](https://github.com/ClickHouse/ClickHouse/pull/4988) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Fixer `DROP`, `TRUNCATE` et `OPTIMIZE` requêtes duplication, lorsqu’elles sont exécutées sur `ON CLUSTER` pour `ReplicatedMergeTree*` les tables de la famille. [\#4991](https://github.com/ClickHouse/ClickHouse/pull/4991) ([alésapine](https://github.com/alesapin)) +- Correction d'un bug rare lors du réglage `min_bytes_to_use_direct_io` est supérieur à zéro, ce qui se produit lorsque le thread doit chercher en arrière dans le fichier de colonne. [\#4897](https://github.com/ClickHouse/ClickHouse/pull/4897) ([alésapine](https://github.com/alesapin)) +- Correction de mauvais types d'arguments pour les fonctions d'agrégation avec `LowCardinality` arguments (corrige le problème [\#4919](https://github.com/ClickHouse/ClickHouse/issues/4919)). [\#4922](https://github.com/ClickHouse/ClickHouse/pull/4922) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Correction d'une qualification de nom incorrecte Dans `GLOBAL JOIN`. [\#4969](https://github.com/ClickHouse/ClickHouse/pull/4969) ([Artem Zuikov](https://github.com/4ertus2)) +- Fonction Fix `toISOWeek` résultat pour l'année 1970. [\#4988](https://github.com/ClickHouse/ClickHouse/pull/4988) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Fixer `DROP`, `TRUNCATE` et `OPTIMIZE` requêtes duplication, lorsqu'elles sont exécutées sur `ON CLUSTER` pour `ReplicatedMergeTree*` les tables de la famille. [\#4991](https://github.com/ClickHouse/ClickHouse/pull/4991) ([alésapine](https://github.com/alesapin)) #### Modification Incompatible En Arrière {#backward-incompatible-change-8} - Renommer le paramètre `insert_sample_with_metadata` définir `input_format_defaults_for_omitted_fields`. [\#4771](https://github.com/ClickHouse/ClickHouse/pull/4771) ([Artem Zuikov](https://github.com/4ertus2)) -- Ajout d’un réglage `max_partitions_per_insert_block` (avec la valeur 100 par défaut). Si le bloc inséré contient un plus grand nombre de partitions, une exception est levée. Mettre à 0 si vous souhaitez supprimer la limite (pas recommandé). [\#4845](https://github.com/ClickHouse/ClickHouse/pull/4845) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Ajout d'un réglage `max_partitions_per_insert_block` (avec la valeur 100 par défaut). Si le bloc inséré contient un plus grand nombre de partitions, une exception est levée. Mettre à 0 si vous souhaitez supprimer la limite (pas recommandé). [\#4845](https://github.com/ClickHouse/ClickHouse/pull/4845) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Les fonctions multi-recherche ont été renommées (`multiPosition` de `multiSearchAllPositions`, `multiSearch` de `multiSearchAny`, `firstMatch` de `multiSearchFirstIndex`). [\#4780](https://github.com/ClickHouse/ClickHouse/pull/4780) ([Danila Kutenin](https://github.com/danlark1)) #### Amélioration Des Performances {#performance-improvement-6} -- Optimisez volnitsky searcher en inlining, donnant environ 5-10% d’amélioration de la recherche pour les requêtes avec de nombreuses aiguilles ou de nombreux bigrams similaires. [\#4862](https://github.com/ClickHouse/ClickHouse/pull/4862) ([Danila Kutenin](https://github.com/danlark1)) -- Correction d’un problème de performance lors du réglage `use_uncompressed_cache` est supérieur à zéro, qui est apparu lorsque toutes les données lues contenues dans le cache. [\#4913](https://github.com/ClickHouse/ClickHouse/pull/4913) ([alésapine](https://github.com/alesapin)) +- Optimisez volnitsky searcher en inlining, donnant environ 5-10% d'amélioration de la recherche pour les requêtes avec de nombreuses aiguilles ou de nombreux bigrams similaires. [\#4862](https://github.com/ClickHouse/ClickHouse/pull/4862) ([Danila Kutenin](https://github.com/danlark1)) +- Correction d'un problème de performance lors du réglage `use_uncompressed_cache` est supérieur à zéro, qui est apparu lorsque toutes les données lues contenues dans le cache. [\#4913](https://github.com/ClickHouse/ClickHouse/pull/4913) ([alésapine](https://github.com/alesapin)) -#### Construction / Test / Amélioration De L’Emballage {#buildtestingpackaging-improvement-10} +#### Construction / Test / Amélioration De L'Emballage {#buildtestingpackaging-improvement-10} -- Durcissement debug build: mappages de mémoire plus granulaires et ASLR; ajouter une protection de mémoire pour le cache de marque et l’index. Cela permet de trouver plus de bugs de piétinement de mémoire dans le cas où ASan et MSan ne peuvent pas le faire. [\#4632](https://github.com/ClickHouse/ClickHouse/pull/4632) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Ajout du support pour les variables cmake `ENABLE_PROTOBUF`, `ENABLE_PARQUET` et `ENABLE_BROTLI` ce qui permet d’activer / désactiver les fonctionnalités ci-dessus (comme nous pouvons le faire pour librdkafka, mysql, etc). [\#4669](https://github.com/ClickHouse/ClickHouse/pull/4669) ([Silviu Caragea](https://github.com/silviucpp)) -- Ajouter la possibilité d’Imprimer la liste des processus et stacktraces de tous les threads si certaines requêtes sont suspendues après l’exécution du test. [\#4675](https://github.com/ClickHouse/ClickHouse/pull/4675) ([alésapine](https://github.com/alesapin)) +- Durcissement debug build: mappages de mémoire plus granulaires et ASLR; ajouter une protection de mémoire pour le cache de marque et l'index. Cela permet de trouver plus de bugs de piétinement de mémoire dans le cas où ASan et MSan ne peuvent pas le faire. [\#4632](https://github.com/ClickHouse/ClickHouse/pull/4632) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Ajout du support pour les variables cmake `ENABLE_PROTOBUF`, `ENABLE_PARQUET` et `ENABLE_BROTLI` ce qui permet d'activer / désactiver les fonctionnalités ci-dessus (comme nous pouvons le faire pour librdkafka, mysql, etc). [\#4669](https://github.com/ClickHouse/ClickHouse/pull/4669) ([Silviu Caragea](https://github.com/silviucpp)) +- Ajouter la possibilité d'Imprimer la liste des processus et stacktraces de tous les threads si certaines requêtes sont suspendues après l'exécution du test. [\#4675](https://github.com/ClickHouse/ClickHouse/pull/4675) ([alésapine](https://github.com/alesapin)) - Ajouter des tentatives sur `Connection loss` erreur dans `clickhouse-test`. [\#4682](https://github.com/ClickHouse/ClickHouse/pull/4682) ([alésapine](https://github.com/alesapin)) - Ajouter freebsd build avec vagrant et construire avec thread sanitizer au script packager. [\#4712](https://github.com/ClickHouse/ClickHouse/pull/4712) [\#4748](https://github.com/ClickHouse/ClickHouse/pull/4748) ([alésapine](https://github.com/alesapin)) -- Maintenant l’Utilisateur a demandé un mot de passe pour l’utilisateur `'default'` lors de l’installation. [\#4725](https://github.com/ClickHouse/ClickHouse/pull/4725) ([proller](https://github.com/proller)) -- Supprimer l’avertissement dans `rdkafka` bibliothèque. [\#4740](https://github.com/ClickHouse/ClickHouse/pull/4740) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Maintenant l'Utilisateur a demandé un mot de passe pour l'utilisateur `'default'` lors de l'installation. [\#4725](https://github.com/ClickHouse/ClickHouse/pull/4725) ([proller](https://github.com/proller)) +- Supprimer l'avertissement dans `rdkafka` bibliothèque. [\#4740](https://github.com/ClickHouse/ClickHouse/pull/4740) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Permettre la capacité de construire sans ssl. [\#4750](https://github.com/ClickHouse/ClickHouse/pull/4750) ([proller](https://github.com/proller)) -- Ajouter un moyen de lancer l’image clickhouse-server à partir d’un utilisateur personnalisé. [\#4753](https://github.com/ClickHouse/ClickHouse/pull/4753) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) +- Ajouter un moyen de lancer l'image clickhouse-server à partir d'un utilisateur personnalisé. [\#4753](https://github.com/ClickHouse/ClickHouse/pull/4753) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) - Mise à niveau contrib boost à 1.69. [\#4793](https://github.com/ClickHouse/ClickHouse/pull/4793) ([proller](https://github.com/proller)) -- Désactiver l’utilisation de `mremap` lorsqu’il est compilé avec thread Sanitizer. Étonnamment, TSan n’intercepte pas `mremap` (bien qu’il ne intercepter `mmap`, `munmap`) qui conduit à des faux positifs. Rapport Tsan fixe dans les tests avec État. [\#4859](https://github.com/ClickHouse/ClickHouse/pull/4859) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Ajouter la vérification de test en utilisant le schéma de format via L’interface HTTP. [\#4864](https://github.com/ClickHouse/ClickHouse/pull/4864) ([Vitaly Baranov](https://github.com/vitlibar)) +- Désactiver l'utilisation de `mremap` lorsqu'il est compilé avec thread Sanitizer. Étonnamment, TSan n'intercepte pas `mremap` (bien qu'il ne intercepter `mmap`, `munmap`) qui conduit à des faux positifs. Rapport Tsan fixe dans les tests avec État. [\#4859](https://github.com/ClickHouse/ClickHouse/pull/4859) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Ajouter la vérification de test en utilisant le schéma de format via L'interface HTTP. [\#4864](https://github.com/ClickHouse/ClickHouse/pull/4864) ([Vitaly Baranov](https://github.com/vitlibar)) ## Clickhouse Version 19.4 {#clickhouse-release-19-4} @@ -1606,38 +1606,38 @@ toc_title: '2019' #### Corrections De Bugs {#bug-fixes-7} -- Éviter `std::terminate` en cas d’échec d’allocation de mémoire. Maintenant `std::bad_alloc` exception est levée comme prévu. [\#4665](https://github.com/ClickHouse/ClickHouse/pull/4665) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Corrige la lecture de capnproto à partir du tampon. Parfois, les fichiers N’ont pas été chargés avec succès par HTTP. [\#4674](https://github.com/ClickHouse/ClickHouse/pull/4674) ([Vladislav](https://github.com/smirnov-vs)) -- Correction d’erreur `Unknown log entry type: 0` après `OPTIMIZE TABLE FINAL` requête. [\#4683](https://github.com/ClickHouse/ClickHouse/pull/4683) ([Amos Oiseau](https://github.com/amosbird)) -- Mauvais arguments `hasAny` ou `hasAll` fonctions peut conduire à l’erreur de segmentation. [\#4698](https://github.com/ClickHouse/ClickHouse/pull/4698) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Blocage peut se produire lors de l’exécution `DROP DATABASE dictionary` requête. [\#4701](https://github.com/ClickHouse/ClickHouse/pull/4701) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Éviter `std::terminate` en cas d'échec d'allocation de mémoire. Maintenant `std::bad_alloc` exception est levée comme prévu. [\#4665](https://github.com/ClickHouse/ClickHouse/pull/4665) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Corrige la lecture de capnproto à partir du tampon. Parfois, les fichiers N'ont pas été chargés avec succès par HTTP. [\#4674](https://github.com/ClickHouse/ClickHouse/pull/4674) ([Vladislav](https://github.com/smirnov-vs)) +- Correction d'erreur `Unknown log entry type: 0` après `OPTIMIZE TABLE FINAL` requête. [\#4683](https://github.com/ClickHouse/ClickHouse/pull/4683) ([Amos Oiseau](https://github.com/amosbird)) +- Mauvais arguments `hasAny` ou `hasAll` fonctions peut conduire à l'erreur de segmentation. [\#4698](https://github.com/ClickHouse/ClickHouse/pull/4698) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Blocage peut se produire lors de l'exécution `DROP DATABASE dictionary` requête. [\#4701](https://github.com/ClickHouse/ClickHouse/pull/4701) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Corriger un comportement indéfini dans `median` et `quantile` fonction. [\#4702](https://github.com/ClickHouse/ClickHouse/pull/4702) ([hcz](https://github.com/hczhcz)) - Correction de la détection du niveau de compression lorsque `network_compression_method` en minuscules. Cassé dans v19. 1. [\#4706](https://github.com/ClickHouse/ClickHouse/pull/4706) ([proller](https://github.com/proller)) -- Correction de l’ignorance de `UTC` paramètre (corrige le problème [\#4658](https://github.com/ClickHouse/ClickHouse/issues/4658)). [\#4718](https://github.com/ClickHouse/ClickHouse/pull/4718) ([proller](https://github.com/proller)) +- Correction de l'ignorance de `UTC` paramètre (corrige le problème [\#4658](https://github.com/ClickHouse/ClickHouse/issues/4658)). [\#4718](https://github.com/ClickHouse/ClickHouse/pull/4718) ([proller](https://github.com/proller)) - Fixer `histogram` comportement de la fonction avec `Distributed` table. [\#4741](https://github.com/ClickHouse/ClickHouse/pull/4741) ([olegkv](https://github.com/olegkv)) - Rapport Tsan fixe `destroy of a locked mutex`. [\#4742](https://github.com/ClickHouse/ClickHouse/pull/4742) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Rapport Tsan fixe sur l’arrêt en raison de la condition de concurrence dans l’utilisation des journaux système. Correction de l’utilisation potentielle-après-libre à l’arrêt lorsque part\_log est activé. [\#4758](https://github.com/ClickHouse/ClickHouse/pull/4758) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Fixer revérifier les pièces dans `ReplicatedMergeTreeAlterThread` en cas d’erreur. [\#4772](https://github.com/ClickHouse/ClickHouse/pull/4772) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Les opérations arithmétiques sur les états de fonction d’agrégat intermédiaire ne fonctionnaient pas pour les arguments constants (tels que les résultats de sous-requête). [\#4776](https://github.com/ClickHouse/ClickHouse/pull/4776) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Rapport Tsan fixe sur l'arrêt en raison de la condition de concurrence dans l'utilisation des journaux système. Correction de l'utilisation potentielle-après-libre à l'arrêt lorsque part\_log est activé. [\#4758](https://github.com/ClickHouse/ClickHouse/pull/4758) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Fixer revérifier les pièces dans `ReplicatedMergeTreeAlterThread` en cas d'erreur. [\#4772](https://github.com/ClickHouse/ClickHouse/pull/4772) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Les opérations arithmétiques sur les états de fonction d'agrégat intermédiaire ne fonctionnaient pas pour les arguments constants (tels que les résultats de sous-requête). [\#4776](https://github.com/ClickHouse/ClickHouse/pull/4776) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Toujours renvoyer les noms de colonne dans les métadonnées. Sinon il est impossible de créer une table avec une colonne nommée `index` (le serveur ne redémarre pas en raison de malformé `ATTACH` requête dans les métadonnées). [\#4782](https://github.com/ClickHouse/ClickHouse/pull/4782) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’un crash dans l’ `ALTER ... MODIFY ORDER BY` sur `Distributed` table. [\#4790](https://github.com/ClickHouse/ClickHouse/pull/4790) ([TCeason](https://github.com/TCeason)) -- Fixer erreur de segmentation dans `JOIN ON` avec l’option `enable_optimize_predicate_expression`. [\#4794](https://github.com/ClickHouse/ClickHouse/pull/4794) ([L’Hiver Zhang](https://github.com/zhang2014)) -- Correction d’un bug avec l’ajout d’une ligne étrangère après avoir consommé un message protobuf de Kafka. [\#4808](https://github.com/ClickHouse/ClickHouse/pull/4808) ([Vitaly Baranov](https://github.com/vitlibar)) -- Correction d’un défaut de segmentation dans `clickhouse-copier`. [\#4835](https://github.com/ClickHouse/ClickHouse/pull/4835) ([proller](https://github.com/proller)) +- Correction d'un crash dans l' `ALTER ... MODIFY ORDER BY` sur `Distributed` table. [\#4790](https://github.com/ClickHouse/ClickHouse/pull/4790) ([TCeason](https://github.com/TCeason)) +- Fixer erreur de segmentation dans `JOIN ON` avec l'option `enable_optimize_predicate_expression`. [\#4794](https://github.com/ClickHouse/ClickHouse/pull/4794) ([L'Hiver Zhang](https://github.com/zhang2014)) +- Correction d'un bug avec l'ajout d'une ligne étrangère après avoir consommé un message protobuf de Kafka. [\#4808](https://github.com/ClickHouse/ClickHouse/pull/4808) ([Vitaly Baranov](https://github.com/vitlibar)) +- Correction d'un défaut de segmentation dans `clickhouse-copier`. [\#4835](https://github.com/ClickHouse/ClickHouse/pull/4835) ([proller](https://github.com/proller)) - Condition de course fixe dans `SELECT` de `system.tables` si la table est renommée ou modifiée simultanément. [\#4836](https://github.com/ClickHouse/ClickHouse/pull/4836) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction de la course de données lors de la récupération d’une partie de données déjà obsolète. [\#4839](https://github.com/ClickHouse/ClickHouse/pull/4839) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction de la course de données lors de la récupération d'une partie de données déjà obsolète. [\#4839](https://github.com/ClickHouse/ClickHouse/pull/4839) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Correction de la course de données rares qui peut se produire pendant `RENAME` table de la famille MergeTree. [\#4844](https://github.com/ClickHouse/ClickHouse/pull/4844) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’un défaut de segmentation en fonction `arrayIntersect`. Une erreur de Segmentation pourrait se produire si la fonction était appelée avec des arguments constants et ordinaires mixtes. [\#4847](https://github.com/ClickHouse/ClickHouse/pull/4847) ([Lixiang Qian](https://github.com/fancyqlx)) +- Correction d'un défaut de segmentation en fonction `arrayIntersect`. Une erreur de Segmentation pourrait se produire si la fonction était appelée avec des arguments constants et ordinaires mixtes. [\#4847](https://github.com/ClickHouse/ClickHouse/pull/4847) ([Lixiang Qian](https://github.com/fancyqlx)) - Correction de la lecture de `Array(LowCardinality)` colonne dans de rares cas où la colonne contenait une longue séquence de tableaux vides. [\#4850](https://github.com/ClickHouse/ClickHouse/pull/4850) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) - Fixer `No message received` exception lors de la récupération de pièces entre les répliques. [\#4856](https://github.com/ClickHouse/ClickHouse/pull/4856) ([alésapine](https://github.com/alesapin)) - Fixe `arrayIntersect` fonction mauvais résultat en cas de plusieurs valeurs répétées dans un seul tableau. [\#4871](https://github.com/ClickHouse/ClickHouse/pull/4871) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Correction d’une condition de course pendant `ALTER COLUMN` requêtes qui pourraient conduire à un plantage du serveur (corrige le problème [\#3421](https://github.com/ClickHouse/ClickHouse/issues/3421)). [\#4592](https://github.com/ClickHouse/ClickHouse/pull/4592) ([Alex Zatelepin](https://github.com/ztlpn)) -- Paramètre correction de la déduction `ALTER MODIFY` de la colonne `CODEC` lorsque le type de colonne n’est pas spécifié. [\#4883](https://github.com/ClickHouse/ClickHouse/pull/4883) ([alésapine](https://github.com/alesapin)) +- Correction d'une condition de course pendant `ALTER COLUMN` requêtes qui pourraient conduire à un plantage du serveur (corrige le problème [\#3421](https://github.com/ClickHouse/ClickHouse/issues/3421)). [\#4592](https://github.com/ClickHouse/ClickHouse/pull/4592) ([Alex Zatelepin](https://github.com/ztlpn)) +- Paramètre correction de la déduction `ALTER MODIFY` de la colonne `CODEC` lorsque le type de colonne n'est pas spécifié. [\#4883](https://github.com/ClickHouse/ClickHouse/pull/4883) ([alésapine](https://github.com/alesapin)) - Fonction `cutQueryStringAndFragment()` et `queryStringAndFragment()` fonctionne maintenant correctement quand `URL` contient un fragment et aucune requête. [\#4894](https://github.com/ClickHouse/ClickHouse/pull/4894) ([Vitaly Baranov](https://github.com/vitlibar)) -- Correction d’un bug rare lors du réglage `min_bytes_to_use_direct_io` est supérieur à zéro, ce qui se produit lorsque le thread doit chercher en arrière dans le fichier de colonne. [\#4897](https://github.com/ClickHouse/ClickHouse/pull/4897) ([alésapine](https://github.com/alesapin)) -- Correction de mauvais types d’arguments pour les fonctions d’agrégation avec `LowCardinality` arguments (corrige le problème [\#4919](https://github.com/ClickHouse/ClickHouse/issues/4919)). [\#4922](https://github.com/ClickHouse/ClickHouse/pull/4922) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Fonction Fix `toISOWeek` résultat pour l’année 1970. [\#4988](https://github.com/ClickHouse/ClickHouse/pull/4988) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Fixer `DROP`, `TRUNCATE` et `OPTIMIZE` requêtes duplication, lorsqu’elles sont exécutées sur `ON CLUSTER` pour `ReplicatedMergeTree*` les tables de la famille. [\#4991](https://github.com/ClickHouse/ClickHouse/pull/4991) ([alésapine](https://github.com/alesapin)) +- Correction d'un bug rare lors du réglage `min_bytes_to_use_direct_io` est supérieur à zéro, ce qui se produit lorsque le thread doit chercher en arrière dans le fichier de colonne. [\#4897](https://github.com/ClickHouse/ClickHouse/pull/4897) ([alésapine](https://github.com/alesapin)) +- Correction de mauvais types d'arguments pour les fonctions d'agrégation avec `LowCardinality` arguments (corrige le problème [\#4919](https://github.com/ClickHouse/ClickHouse/issues/4919)). [\#4922](https://github.com/ClickHouse/ClickHouse/pull/4922) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Fonction Fix `toISOWeek` résultat pour l'année 1970. [\#4988](https://github.com/ClickHouse/ClickHouse/pull/4988) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Fixer `DROP`, `TRUNCATE` et `OPTIMIZE` requêtes duplication, lorsqu'elles sont exécutées sur `ON CLUSTER` pour `ReplicatedMergeTree*` les tables de la famille. [\#4991](https://github.com/ClickHouse/ClickHouse/pull/4991) ([alésapine](https://github.com/alesapin)) #### Amélioration {#improvements-2} @@ -1647,12 +1647,12 @@ toc_title: '2019' #### Corrections De Bugs {#bug-fixes-8} -- Correction d’un crash dans l’ `FULL/RIGHT JOIN` lorsque nous rejoignons sur nullable vs non nullable. [\#4855](https://github.com/ClickHouse/ClickHouse/pull/4855) ([Artem Zuikov](https://github.com/4ertus2)) -- Correction d’un défaut de segmentation dans `clickhouse-copier`. [\#4835](https://github.com/ClickHouse/ClickHouse/pull/4835) ([proller](https://github.com/proller)) +- Correction d'un crash dans l' `FULL/RIGHT JOIN` lorsque nous rejoignons sur nullable vs non nullable. [\#4855](https://github.com/ClickHouse/ClickHouse/pull/4855) ([Artem Zuikov](https://github.com/4ertus2)) +- Correction d'un défaut de segmentation dans `clickhouse-copier`. [\#4835](https://github.com/ClickHouse/ClickHouse/pull/4835) ([proller](https://github.com/proller)) -#### Construction / Test / Amélioration De L’Emballage {#buildtestingpackaging-improvement-11} +#### Construction / Test / Amélioration De L'Emballage {#buildtestingpackaging-improvement-11} -- Ajouter un moyen de lancer l’image clickhouse-server à partir d’un utilisateur personnalisé. [\#4753](https://github.com/ClickHouse/ClickHouse/pull/4753) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) +- Ajouter un moyen de lancer l'image clickhouse-server à partir d'un utilisateur personnalisé. [\#4753](https://github.com/ClickHouse/ClickHouse/pull/4753) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) ### Clickhouse Version 19.4.2.7, 2019-03-30 {#clickhouse-release-19-4-2-7-2019-03-30} @@ -1670,37 +1670,37 @@ toc_title: '2019' #### Nouveauté {#new-features-5} -- Ajout d’un support complet pour `Protobuf` format (entrée et sortie, structures de données imbriquées). [\#4174](https://github.com/ClickHouse/ClickHouse/pull/4174) [\#4493](https://github.com/ClickHouse/ClickHouse/pull/4493) ([Vitaly Baranov](https://github.com/vitlibar)) +- Ajout d'un support complet pour `Protobuf` format (entrée et sortie, structures de données imbriquées). [\#4174](https://github.com/ClickHouse/ClickHouse/pull/4174) [\#4493](https://github.com/ClickHouse/ClickHouse/pull/4493) ([Vitaly Baranov](https://github.com/vitlibar)) - Ajout de fonctions bitmap avec des bitmaps rugissants. [\#4207](https://github.com/ClickHouse/ClickHouse/pull/4207) ([Andy Yang](https://github.com/andyyzh)) [\#4568](https://github.com/ClickHouse/ClickHouse/pull/4568) ([Vitaly Baranov](https://github.com/vitlibar)) - Support du format Parquet. [\#4448](https://github.com/ClickHouse/ClickHouse/pull/4448) ([proller](https://github.com/proller)) - La distance de n-gramme a été ajoutée pour la comparaison de chaîne floue. Il est similaire aux métriques Q-gram en langage R. [\#4466](https://github.com/ClickHouse/ClickHouse/pull/4466) ([Danila Kutenin](https://github.com/danlark1)) -- Combinez des règles pour le cumul de graphite à partir de modèles d’agrégation et de rétention dédiés. [\#4426](https://github.com/ClickHouse/ClickHouse/pull/4426) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) -- Ajouter `max_execution_speed` et `max_execution_speed_bytes` pour limiter l’utilisation des ressources. Ajouter `min_execution_speed_bytes` réglage pour compléter le `min_execution_speed`. [\#4430](https://github.com/ClickHouse/ClickHouse/pull/4430) ([L’Hiver Zhang](https://github.com/zhang2014)) +- Combinez des règles pour le cumul de graphite à partir de modèles d'agrégation et de rétention dédiés. [\#4426](https://github.com/ClickHouse/ClickHouse/pull/4426) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) +- Ajouter `max_execution_speed` et `max_execution_speed_bytes` pour limiter l'utilisation des ressources. Ajouter `min_execution_speed_bytes` réglage pour compléter le `min_execution_speed`. [\#4430](https://github.com/ClickHouse/ClickHouse/pull/4430) ([L'Hiver Zhang](https://github.com/zhang2014)) - Mise en œuvre de la fonction `flatten`. [\#4555](https://github.com/ClickHouse/ClickHouse/pull/4555) [\#4409](https://github.com/ClickHouse/ClickHouse/pull/4409) ([alexeï-milovidov](https://github.com/alexey-milovidov), [kzon](https://github.com/kzon)) -- L’ajout de fonctions `arrayEnumerateDenseRanked` et `arrayEnumerateUniqRanked` (c’est comme `arrayEnumerateUniq` mais permet d’affiner la profondeur du tableau pour regarder à l’intérieur des tableaux multidimensionnels). [\#4475](https://github.com/ClickHouse/ClickHouse/pull/4475) ([proller](https://github.com/proller)) [\#4601](https://github.com/ClickHouse/ClickHouse/pull/4601) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- L'ajout de fonctions `arrayEnumerateDenseRanked` et `arrayEnumerateUniqRanked` (c'est comme `arrayEnumerateUniq` mais permet d'affiner la profondeur du tableau pour regarder à l'intérieur des tableaux multidimensionnels). [\#4475](https://github.com/ClickHouse/ClickHouse/pull/4475) ([proller](https://github.com/proller)) [\#4601](https://github.com/ClickHouse/ClickHouse/pull/4601) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Multiple JOINS with some restrictions: no asterisks, no complex aliases in ON/WHERE/GROUP BY/… [\#4462](https://github.com/ClickHouse/ClickHouse/pull/4462) ([Artem Zuikov](https://github.com/4ertus2)) #### Corrections De Bugs {#bug-fixes-11} - Cette version contient également toutes les corrections de bugs de 19.3 et 19.1. -- Correction d’un bug dans les indices de saut de données: l’ordre des granules après L’insertion était incorrect. [\#4407](https://github.com/ClickHouse/ClickHouse/pull/4407) ([Nikita Vasilev](https://github.com/nikvas0)) -- Fixe `set` indice pour `Nullable` et `LowCardinality` colonne. Avant d’, `set` index avec `Nullable` ou `LowCardinality` colonne a conduit à l’erreur `Data type must be deserialized with multiple streams` lors de la sélection. [\#4594](https://github.com/ClickHouse/ClickHouse/pull/4594) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Correction d'un bug dans les indices de saut de données: l'ordre des granules après L'insertion était incorrect. [\#4407](https://github.com/ClickHouse/ClickHouse/pull/4407) ([Nikita Vasilev](https://github.com/nikvas0)) +- Fixe `set` indice pour `Nullable` et `LowCardinality` colonne. Avant d', `set` index avec `Nullable` ou `LowCardinality` colonne a conduit à l'erreur `Data type must be deserialized with multiple streams` lors de la sélection. [\#4594](https://github.com/ClickHouse/ClickHouse/pull/4594) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) - Définir correctement update\_time sur full `executable` dictionnaire de mise à jour. [\#4551](https://github.com/ClickHouse/ClickHouse/pull/4551) ([Tema Novikov](https://github.com/temoon)) - Correction de la barre de progression cassée dans 19.3. [\#4627](https://github.com/ClickHouse/ClickHouse/pull/4627) ([filimonov](https://github.com/filimonov)) - Correction de valeurs incohérentes de MemoryTracker lorsque la région de mémoire a été réduite, dans certains cas. [\#4619](https://github.com/ClickHouse/ClickHouse/pull/4619) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’un comportement indéfini dans ThreadPool. [\#4612](https://github.com/ClickHouse/ClickHouse/pull/4612) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’un crash très rare avec le message `mutex lock failed: Invalid argument` cela peut se produire lorsqu’une table MergeTree a été supprimée simultanément avec un SELECT. [\#4608](https://github.com/ClickHouse/ClickHouse/pull/4608) ([Alex Zatelepin](https://github.com/ztlpn)) +- Correction d'un comportement indéfini dans ThreadPool. [\#4612](https://github.com/ClickHouse/ClickHouse/pull/4612) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction d'un crash très rare avec le message `mutex lock failed: Invalid argument` cela peut se produire lorsqu'une table MergeTree a été supprimée simultanément avec un SELECT. [\#4608](https://github.com/ClickHouse/ClickHouse/pull/4608) ([Alex Zatelepin](https://github.com/ztlpn)) - Compatibilité du pilote ODBC avec `LowCardinality` type de données. [\#4381](https://github.com/ClickHouse/ClickHouse/pull/4381) ([proller](https://github.com/proller)) - FreeBSD: correction pour `AIOcontextPool: Found io_event with unknown id 0` erreur. [\#4438](https://github.com/ClickHouse/ClickHouse/pull/4438) ([urgordeadbeef](https://github.com/urgordeadbeef)) - `system.part_log` table a été créée, indépendamment de la configuration. [\#4483](https://github.com/ClickHouse/ClickHouse/pull/4483) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Correction du comportement indéfini dans `dictIsIn` fonction pour les dictionnaires de cache. [\#4515](https://github.com/ClickHouse/ClickHouse/pull/4515) ([alésapine](https://github.com/alesapin)) - Fixed a deadlock when a SELECT query locks the same table multiple times (e.g. from different threads or when executing multiple subqueries) and there is a concurrent DDL query. [\#4535](https://github.com/ClickHouse/ClickHouse/pull/4535) ([Alex Zatelepin](https://github.com/ztlpn)) -- Désactiver compile\_expressions par défaut jusqu’à ce que nous obtenions propre `llvm` contrib et peut le tester avec `clang` et `asan`. [\#4579](https://github.com/ClickHouse/ClickHouse/pull/4579) ([alésapine](https://github.com/alesapin)) -- Empêcher `std::terminate` lorsque `invalidate_query` pour `clickhouse` externe dictionnaire source a retourné mauvais jeu de résultats (vide ou plusieurs lignes ou plusieurs colonnes). Correction d’un problème lors de l’ `invalidate_query` a été effectuée toutes les cinq secondes quel que soit le `lifetime`. [\#4583](https://github.com/ClickHouse/ClickHouse/pull/4583) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Éviter l’impasse lorsque le `invalidate_query` pour un dictionnaire avec `clickhouse` source était impliquant `system.dictionaries` table ou `Dictionaries` base de données (cas rare). [\#4599](https://github.com/ClickHouse/ClickHouse/pull/4599) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Désactiver compile\_expressions par défaut jusqu'à ce que nous obtenions propre `llvm` contrib et peut le tester avec `clang` et `asan`. [\#4579](https://github.com/ClickHouse/ClickHouse/pull/4579) ([alésapine](https://github.com/alesapin)) +- Empêcher `std::terminate` lorsque `invalidate_query` pour `clickhouse` externe dictionnaire source a retourné mauvais jeu de résultats (vide ou plusieurs lignes ou plusieurs colonnes). Correction d'un problème lors de l' `invalidate_query` a été effectuée toutes les cinq secondes quel que soit le `lifetime`. [\#4583](https://github.com/ClickHouse/ClickHouse/pull/4583) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Éviter l'impasse lorsque le `invalidate_query` pour un dictionnaire avec `clickhouse` source était impliquant `system.dictionaries` table ou `Dictionaries` base de données (cas rare). [\#4599](https://github.com/ClickHouse/ClickHouse/pull/4599) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Corrections pour CROSS JOIN avec vide où. [\#4598](https://github.com/ClickHouse/ClickHouse/pull/4598) ([Artem Zuikov](https://github.com/4ertus2)) -- Fixe erreur de segmentation en fonction “replicate” lorsque l’argument constant est passé. [\#4603](https://github.com/ClickHouse/ClickHouse/pull/4603) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction de la fonction lambda avec l’optimiseur de prédicats. [\#4408](https://github.com/ClickHouse/ClickHouse/pull/4408) ([L’Hiver Zhang](https://github.com/zhang2014)) +- Fixe erreur de segmentation en fonction “replicate” lorsque l'argument constant est passé. [\#4603](https://github.com/ClickHouse/ClickHouse/pull/4603) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction de la fonction lambda avec l'optimiseur de prédicats. [\#4408](https://github.com/ClickHouse/ClickHouse/pull/4408) ([L'Hiver Zhang](https://github.com/zhang2014)) - Plusieurs jointures plusieurs correctifs. [\#4595](https://github.com/ClickHouse/ClickHouse/pull/4595) ([Artem Zuikov](https://github.com/4ertus2)) #### Amélioration {#improvements-3} @@ -1712,21 +1712,21 @@ toc_title: '2019' #### Amélioration Des Performances {#performance-improvements-3} - Heuristiques améliorées de “move to PREWHERE” optimisation. [\#4405](https://github.com/ClickHouse/ClickHouse/pull/4405) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Utilisez des tables de recherche appropriées qui utilisent L’API de HashTable pour les clés 8 bits et 16 bits. [\#4536](https://github.com/ClickHouse/ClickHouse/pull/4536) ([Amos Oiseau](https://github.com/amosbird)) +- Utilisez des tables de recherche appropriées qui utilisent L'API de HashTable pour les clés 8 bits et 16 bits. [\#4536](https://github.com/ClickHouse/ClickHouse/pull/4536) ([Amos Oiseau](https://github.com/amosbird)) - Amélioration des performances de la comparaison de chaînes. [\#4564](https://github.com/ClickHouse/ClickHouse/pull/4564) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Nettoyage de la file D’attente DDL distribuée dans un thread séparé afin de ne pas ralentir la boucle principale qui traite les tâches DDL distribuées. [\#4502](https://github.com/ClickHouse/ClickHouse/pull/4502) ([Alex Zatelepin](https://github.com/ztlpn)) -- Lorsque `min_bytes_to_use_direct_io` est défini sur 1, Tous les fichiers n’ont pas été ouverts avec le mode O\_DIRECT car la taille des données à lire était parfois sous-estimée par la taille d’un bloc compressé. [\#4526](https://github.com/ClickHouse/ClickHouse/pull/4526) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Nettoyage de la file D'attente DDL distribuée dans un thread séparé afin de ne pas ralentir la boucle principale qui traite les tâches DDL distribuées. [\#4502](https://github.com/ClickHouse/ClickHouse/pull/4502) ([Alex Zatelepin](https://github.com/ztlpn)) +- Lorsque `min_bytes_to_use_direct_io` est défini sur 1, Tous les fichiers n'ont pas été ouverts avec le mode O\_DIRECT car la taille des données à lire était parfois sous-estimée par la taille d'un bloc compressé. [\#4526](https://github.com/ClickHouse/ClickHouse/pull/4526) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -#### Construction / Test / Amélioration De L’Emballage {#buildtestingpackaging-improvement-12} +#### Construction / Test / Amélioration De L'Emballage {#buildtestingpackaging-improvement-12} - Ajout du support pour clang-9 [\#4604](https://github.com/ClickHouse/ClickHouse/pull/4604) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Corrigé de mal `__asm__` instructions (encore une fois) [\#4621](https://github.com/ClickHouse/ClickHouse/pull/4621) ([Konstantin Podshumok](https://github.com/podshumok)) - Ajouter la possibilité de spécifier les paramètres pour `clickhouse-performance-test` à partir de la ligne de commande. [\#4437](https://github.com/ClickHouse/ClickHouse/pull/4437) ([alésapine](https://github.com/alesapin)) -- Ajouter des tests de dictionnaires aux tests d’intégration. [\#4477](https://github.com/ClickHouse/ClickHouse/pull/4477) ([alésapine](https://github.com/alesapin)) +- Ajouter des tests de dictionnaires aux tests d'intégration. [\#4477](https://github.com/ClickHouse/ClickHouse/pull/4477) ([alésapine](https://github.com/alesapin)) - Ajout de requêtes du benchmark sur le site web à des tests de performance automatisés. [\#4496](https://github.com/ClickHouse/ClickHouse/pull/4496) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- `xxhash.h` n’existe pas dans LZ4 externe car il s’agit d’un détail d’implémentation et ses symboles sont `XXH_NAMESPACE` macro. Lorsque lz4 est externe, xxHash doit aussi être externe, et les personnes à charge doivent y accéder. [\#4495](https://github.com/ClickHouse/ClickHouse/pull/4495) ([Orivej Desh](https://github.com/orivej)) -- Correction d’un cas lorsque `quantileTiming` la fonction d’agrégation peut être appelée avec un argument négatif ou à virgule flottante (cela corrige le test fuzz avec un désinfectant de comportement indéfini). [\#4506](https://github.com/ClickHouse/ClickHouse/pull/4506) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’erreur d’orthographe. [\#4531](https://github.com/ClickHouse/ClickHouse/pull/4531) ([sdk2](https://github.com/sdk2)) +- `xxhash.h` n'existe pas dans LZ4 externe car il s'agit d'un détail d'implémentation et ses symboles sont `XXH_NAMESPACE` macro. Lorsque lz4 est externe, xxHash doit aussi être externe, et les personnes à charge doivent y accéder. [\#4495](https://github.com/ClickHouse/ClickHouse/pull/4495) ([Orivej Desh](https://github.com/orivej)) +- Correction d'un cas lorsque `quantileTiming` la fonction d'agrégation peut être appelée avec un argument négatif ou à virgule flottante (cela corrige le test fuzz avec un désinfectant de comportement indéfini). [\#4506](https://github.com/ClickHouse/ClickHouse/pull/4506) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction d'erreur d'orthographe. [\#4531](https://github.com/ClickHouse/ClickHouse/pull/4531) ([sdk2](https://github.com/sdk2)) - Correction de la compilation sur Mac. [\#4371](https://github.com/ClickHouse/ClickHouse/pull/4371) ([Vitaly Baranov](https://github.com/vitlibar)) - Corrections de construction Pour FreeBSD et diverses configurations de construction inhabituelles. [\#4444](https://github.com/ClickHouse/ClickHouse/pull/4444) ([proller](https://github.com/proller)) @@ -1736,19 +1736,19 @@ toc_title: '2019' #### Corrections De Bugs {#bug-fixes-12} -- Correction d’un crash dans l’ `FULL/RIGHT JOIN` lorsque nous rejoignons sur nullable vs non nullable. [\#4855](https://github.com/ClickHouse/ClickHouse/pull/4855) ([Artem Zuikov](https://github.com/4ertus2)) -- Correction d’un défaut de segmentation dans `clickhouse-copier`. [\#4835](https://github.com/ClickHouse/ClickHouse/pull/4835) ([proller](https://github.com/proller)) +- Correction d'un crash dans l' `FULL/RIGHT JOIN` lorsque nous rejoignons sur nullable vs non nullable. [\#4855](https://github.com/ClickHouse/ClickHouse/pull/4855) ([Artem Zuikov](https://github.com/4ertus2)) +- Correction d'un défaut de segmentation dans `clickhouse-copier`. [\#4835](https://github.com/ClickHouse/ClickHouse/pull/4835) ([proller](https://github.com/proller)) - Correction de la lecture de `Array(LowCardinality)` colonne dans de rares cas où la colonne contenait une longue séquence de tableaux vides. [\#4850](https://github.com/ClickHouse/ClickHouse/pull/4850) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -#### Construction / Test / Amélioration De L’Emballage {#buildtestingpackaging-improvement-13} +#### Construction / Test / Amélioration De L'Emballage {#buildtestingpackaging-improvement-13} -- Ajouter un moyen de lancer l’image clickhouse-server à partir d’un utilisateur personnalisé [\#4753](https://github.com/ClickHouse/ClickHouse/pull/4753) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) +- Ajouter un moyen de lancer l'image clickhouse-server à partir d'un utilisateur personnalisé [\#4753](https://github.com/ClickHouse/ClickHouse/pull/4753) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) ### Clickhouse Version 19.3.7, 2019-03-12 {#clickhouse-release-19-3-7-2019-03-12} #### Corrections De Bugs {#bug-fixes-13} -- Correction d’une erreur dans \#3920. Cette erreur se manifeste par une corruption aléatoire du cache (messages `Unknown codec family code`, `Cannot seek through file`) et de segmentation. Cette anomalie est apparue dans la version 19.1 et est présente dans les versions jusqu’à 19.1.10 et 19.3.6. [\#4623](https://github.com/ClickHouse/ClickHouse/pull/4623) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction d'une erreur dans \#3920. Cette erreur se manifeste par une corruption aléatoire du cache (messages `Unknown codec family code`, `Cannot seek through file`) et de segmentation. Cette anomalie est apparue dans la version 19.1 et est présente dans les versions jusqu'à 19.1.10 et 19.3.6. [\#4623](https://github.com/ClickHouse/ClickHouse/pull/4623) ([alexeï-milovidov](https://github.com/alexey-milovidov)) ### Clickhouse Version 19.3.6, 2019-03-02 {#clickhouse-release-19-3-6-2019-03-02} @@ -1756,70 +1756,70 @@ toc_title: '2019' - Quand il y a plus de 1000 threads dans un pool de threads, `std::terminate` peut se produire à la sortie thread. [Azat Khuzhin](https://github.com/azat) [\#4485](https://github.com/ClickHouse/ClickHouse/pull/4485) [\#4505](https://github.com/ClickHouse/ClickHouse/pull/4505) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Maintenant, il est possible de créer `ReplicatedMergeTree*` tables avec commentaires sur les colonnes sans valeurs par défaut et tables avec colonnes codecs Sans commentaires et valeurs par défaut. Corrigez également la comparaison des codecs. [\#4523](https://github.com/ClickHouse/ClickHouse/pull/4523) ([alésapine](https://github.com/alesapin)) -- Correction d’un crash sur la jointure avec un tableau ou un tuple. [\#4552](https://github.com/ClickHouse/ClickHouse/pull/4552) ([Artem Zuikov](https://github.com/4ertus2)) -- Correction d’un crash dans clickhouse-copieur avec le message `ThreadStatus not created`. [\#4540](https://github.com/ClickHouse/ClickHouse/pull/4540) ([Artem Zuikov](https://github.com/4ertus2)) -- Correction du blocage à l’arrêt du serveur si des DDL distribués étaient utilisés. [\#4472](https://github.com/ClickHouse/ClickHouse/pull/4472) ([Alex Zatelepin](https://github.com/ztlpn)) -- Des numéros de colonne incorrects ont été imprimés dans un message d’erreur concernant l’analyse de format de texte pour les colonnes dont le nombre est supérieur à 10. [\#4484](https://github.com/ClickHouse/ClickHouse/pull/4484) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction d'un crash sur la jointure avec un tableau ou un tuple. [\#4552](https://github.com/ClickHouse/ClickHouse/pull/4552) ([Artem Zuikov](https://github.com/4ertus2)) +- Correction d'un crash dans clickhouse-copieur avec le message `ThreadStatus not created`. [\#4540](https://github.com/ClickHouse/ClickHouse/pull/4540) ([Artem Zuikov](https://github.com/4ertus2)) +- Correction du blocage à l'arrêt du serveur si des DDL distribués étaient utilisés. [\#4472](https://github.com/ClickHouse/ClickHouse/pull/4472) ([Alex Zatelepin](https://github.com/ztlpn)) +- Des numéros de colonne incorrects ont été imprimés dans un message d'erreur concernant l'analyse de format de texte pour les colonnes dont le nombre est supérieur à 10. [\#4484](https://github.com/ClickHouse/ClickHouse/pull/4484) ([alexeï-milovidov](https://github.com/alexey-milovidov)) #### Construire/Test/Emballage Améliorations {#buildtestingpackaging-improvements-3} - Correction de la construction avec AVX activé. [\#4527](https://github.com/ClickHouse/ClickHouse/pull/4527) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Activer la comptabilité étendue et la comptabilité D’E / S basée sur une version bien connue au lieu du noyau sous lequel elle est compilée. [\#4541](https://github.com/ClickHouse/ClickHouse/pull/4541) ([nvartolomei](https://github.com/nvartolomei)) -- Permet d’ignorer le paramètre core\_dump.size\_limit, avertissement au lieu de lancer si la limite définie échoue. [\#4473](https://github.com/ClickHouse/ClickHouse/pull/4473) ([proller](https://github.com/proller)) +- Activer la comptabilité étendue et la comptabilité D'E / S basée sur une version bien connue au lieu du noyau sous lequel elle est compilée. [\#4541](https://github.com/ClickHouse/ClickHouse/pull/4541) ([nvartolomei](https://github.com/nvartolomei)) +- Permet d'ignorer le paramètre core\_dump.size\_limit, avertissement au lieu de lancer si la limite définie échoue. [\#4473](https://github.com/ClickHouse/ClickHouse/pull/4473) ([proller](https://github.com/proller)) - Enlevé le `inline` les balises de `void readBinary(...)` dans `Field.cpp`. Également fusionné redondant `namespace DB` bloc. [\#4530](https://github.com/ClickHouse/ClickHouse/pull/4530) ([hcz](https://github.com/hczhcz)) ### Clickhouse Version 19.3.5, 2019-02-21 {#clickhouse-release-19-3-5-2019-02-21} #### Corrections De Bugs {#bug-fixes-15} -- Correction d’un bug avec de grandes requêtes d’insertion http traitement. [\#4454](https://github.com/ClickHouse/ClickHouse/pull/4454) ([alésapine](https://github.com/alesapin)) -- Correction de l’incompatibilité avec les anciennes versions en raison d’une mauvaise implémentation de `send_logs_level` paramètre. [\#4445](https://github.com/ClickHouse/ClickHouse/pull/4445) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction d'un bug avec de grandes requêtes d'insertion http traitement. [\#4454](https://github.com/ClickHouse/ClickHouse/pull/4454) ([alésapine](https://github.com/alesapin)) +- Correction de l'incompatibilité avec les anciennes versions en raison d'une mauvaise implémentation de `send_logs_level` paramètre. [\#4445](https://github.com/ClickHouse/ClickHouse/pull/4445) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Incompatibilité arrière fixe de la fonction de table `remote` introduit avec des commentaires de colonne. [\#4446](https://github.com/ClickHouse/ClickHouse/pull/4446) ([alexeï-milovidov](https://github.com/alexey-milovidov)) ### Clickhouse Version 19.3.4, 2019-02-16 {#clickhouse-release-19-3-4-2019-02-16} #### Amélioration {#improvements-4} -- La taille de l’index de la Table n’est pas prise en compte des limites de mémoire lors de `ATTACH TABLE` requête. Éviter la possibilité qu’un tableau ne peut pas être attachée après avoir été détaché. [\#4396](https://github.com/ClickHouse/ClickHouse/pull/4396) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- La taille de l'index de la Table n'est pas prise en compte des limites de mémoire lors de `ATTACH TABLE` requête. Éviter la possibilité qu'un tableau ne peut pas être attachée après avoir été détaché. [\#4396](https://github.com/ClickHouse/ClickHouse/pull/4396) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Légèrement augmenté la limite sur la chaîne max et la taille du tableau reçu de ZooKeeper. Il permet de continuer à travailler avec une taille accrue de `CLIENT_JVMFLAGS=-Djute.maxbuffer=...` sur la Gardienne. [\#4398](https://github.com/ClickHouse/ClickHouse/pull/4398) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Permettre de réparer la réplique abandonnée même si elle a déjà un grand nombre de nœuds dans sa file d’attente. [\#4399](https://github.com/ClickHouse/ClickHouse/pull/4399) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Permettre de réparer la réplique abandonnée même si elle a déjà un grand nombre de nœuds dans sa file d'attente. [\#4399](https://github.com/ClickHouse/ClickHouse/pull/4399) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Ajouter un argument requis à `SET` index (nombre maximum de lignes stockées). [\#4386](https://github.com/ClickHouse/ClickHouse/pull/4386) ([Nikita Vasilev](https://github.com/nikvas0)) #### Corrections De Bugs {#bug-fixes-16} - Fixe `WITH ROLLUP` résultat pour le groupe par un seul `LowCardinality` clé. [\#4384](https://github.com/ClickHouse/ClickHouse/pull/4384) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Correction d’un bug dans l’index set (laissant tomber un granule s’il contient plus de `max_rows` rangée). [\#4386](https://github.com/ClickHouse/ClickHouse/pull/4386) ([Nikita Vasilev](https://github.com/nikvas0)) +- Correction d'un bug dans l'index set (laissant tomber un granule s'il contient plus de `max_rows` rangée). [\#4386](https://github.com/ClickHouse/ClickHouse/pull/4386) ([Nikita Vasilev](https://github.com/nikvas0)) - Beaucoup de corrections de construction FreeBSD. [\#4397](https://github.com/ClickHouse/ClickHouse/pull/4397) ([proller](https://github.com/proller)) - Correction de la substitution des alias dans les requêtes avec une sous-requête contenant le même alias (problème [\#4110](https://github.com/ClickHouse/ClickHouse/issues/4110)). [\#4351](https://github.com/ClickHouse/ClickHouse/pull/4351) ([Artem Zuikov](https://github.com/4ertus2)) #### Construire/Test/Emballage Améliorations {#buildtestingpackaging-improvements-4} -- Ajouter la possibilité d’exécuter `clickhouse-server` pour les tests sans état dans l’image docker. [\#4347](https://github.com/ClickHouse/ClickHouse/pull/4347) ([Vasily Nemkov](https://github.com/Enmk)) +- Ajouter la possibilité d'exécuter `clickhouse-server` pour les tests sans état dans l'image docker. [\#4347](https://github.com/ClickHouse/ClickHouse/pull/4347) ([Vasily Nemkov](https://github.com/Enmk)) ### Clickhouse Version 19.3.3, 2019-02-13 {#clickhouse-release-19-3-3-2019-02-13} #### Nouveauté {#new-features-6} - Ajouté le `KILL MUTATION` instruction qui permet de supprimer les mutations qui sont pour certaines raisons coincé. Ajouter `latest_failed_part`, `latest_fail_time`, `latest_fail_reason` champs à la `system.mutations` tableau pour faciliter le dépannage. [\#4287](https://github.com/ClickHouse/ClickHouse/pull/4287) ([Alex Zatelepin](https://github.com/ztlpn)) -- Fonction agrégée ajoutée `entropy` qui calcule l’entropie de Shannon. [\#4238](https://github.com/ClickHouse/ClickHouse/pull/4238) ([Quid37](https://github.com/Quid37)) -- Ajout de la possibilité d’envoyer des requêtes `INSERT INTO tbl VALUES (....` au serveur sans fractionnement sur `query` et `data` partie. [\#4301](https://github.com/ClickHouse/ClickHouse/pull/4301) ([alésapine](https://github.com/alesapin)) +- Fonction agrégée ajoutée `entropy` qui calcule l'entropie de Shannon. [\#4238](https://github.com/ClickHouse/ClickHouse/pull/4238) ([Quid37](https://github.com/Quid37)) +- Ajout de la possibilité d'envoyer des requêtes `INSERT INTO tbl VALUES (....` au serveur sans fractionnement sur `query` et `data` partie. [\#4301](https://github.com/ClickHouse/ClickHouse/pull/4301) ([alésapine](https://github.com/alesapin)) - Mise en œuvre générique de `arrayWithConstant` la fonction a été ajoutée. [\#4322](https://github.com/ClickHouse/ClickHouse/pull/4322) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Mettre `NOT BETWEEN` opérateur de comparaison. [\#4228](https://github.com/ClickHouse/ClickHouse/pull/4228) ([Dmitry Naumov](https://github.com/nezed)) - Mettre `sumMapFiltered` afin de pouvoir limiter le nombre de clés pour lesquelles les valeurs seront additionnées par `sumMap`. [\#4129](https://github.com/ClickHouse/ClickHouse/pull/4129) ([Léo Ercolanelli](https://github.com/ercolanelli-leo)) - Ajout du support de `Nullable` types de `mysql` table de fonction. [\#4198](https://github.com/ClickHouse/ClickHouse/pull/4198) ([Emmanuel Donin de Rosière](https://github.com/edonin)) - Prise en charge des expressions constantes arbitraires dans `LIMIT` clause. [\#4246](https://github.com/ClickHouse/ClickHouse/pull/4246) ([k3box](https://github.com/k3box)) -- Ajouter `topKWeighted` fonction d’agrégation qui prend un argument supplémentaire avec un poids (entier non signé). [\#4245](https://github.com/ClickHouse/ClickHouse/pull/4245) ([Andrew Golman](https://github.com/andrewgolman)) -- `StorageJoin` maintenant prend en charge `join_any_take_last_row` paramètre qui permet d’écraser les valeurs existantes de la même clé. [\#3973](https://github.com/ClickHouse/ClickHouse/pull/3973) ([Amos Oiseau](https://github.com/amosbird) +- Ajouter `topKWeighted` fonction d'agrégation qui prend un argument supplémentaire avec un poids (entier non signé). [\#4245](https://github.com/ClickHouse/ClickHouse/pull/4245) ([Andrew Golman](https://github.com/andrewgolman)) +- `StorageJoin` maintenant prend en charge `join_any_take_last_row` paramètre qui permet d'écraser les valeurs existantes de la même clé. [\#3973](https://github.com/ClickHouse/ClickHouse/pull/3973) ([Amos Oiseau](https://github.com/amosbird) - Ajout de la fonction `toStartOfInterval`. [\#4304](https://github.com/ClickHouse/ClickHouse/pull/4304) ([Vitaly Baranov](https://github.com/vitlibar)) - Ajouter `RowBinaryWithNamesAndTypes` format. [\#4200](https://github.com/ClickHouse/ClickHouse/pull/4200) ([Oleg V. Kozlyuk](https://github.com/DarkWanderer)) - Ajouter `IPv4` et `IPv6` types de données. Implémentations plus efficaces de `IPv*` fonction. [\#3669](https://github.com/ClickHouse/ClickHouse/pull/3669) ([Vasily Nemkov](https://github.com/Enmk)) - Ajout de la fonction `toStartOfTenMinutes()`. [\#4298](https://github.com/ClickHouse/ClickHouse/pull/4298) ([Vitaly Baranov](https://github.com/vitlibar)) - Ajouter `Protobuf` le format de sortie. [\#4005](https://github.com/ClickHouse/ClickHouse/pull/4005) [\#4158](https://github.com/ClickHouse/ClickHouse/pull/4158) ([Vitaly Baranov](https://github.com/vitlibar)) -- Ajout du support brotli pour L’interface HTTP pour l’importation de données (INSERTs). [\#4235](https://github.com/ClickHouse/ClickHouse/pull/4235) ([Mikhail](https://github.com/fandyushin)) -- Ajout d’astuces pendant que l’utilisateur fait une faute de frappe dans le nom de la fonction ou tapez dans le client de ligne de commande. [\#4239](https://github.com/ClickHouse/ClickHouse/pull/4239) ([Danila Kutenin](https://github.com/danlark1)) +- Ajout du support brotli pour L'interface HTTP pour l'importation de données (INSERTs). [\#4235](https://github.com/ClickHouse/ClickHouse/pull/4235) ([Mikhail](https://github.com/fandyushin)) +- Ajout d'astuces pendant que l'utilisateur fait une faute de frappe dans le nom de la fonction ou tapez dans le client de ligne de commande. [\#4239](https://github.com/ClickHouse/ClickHouse/pull/4239) ([Danila Kutenin](https://github.com/danlark1)) - Ajouter `Query-Id` de du Serveur HTTP en-tête de Réponse. [\#4231](https://github.com/ClickHouse/ClickHouse/pull/4231) ([Mikhail](https://github.com/fandyushin)) -#### Caractéristiques expérimentales {#experimental-features-2} +#### Caractéristiques Expérimentales {#experimental-features-2} - Ajouter `minmax` et `set` index de saut de données pour la famille de moteurs de table MergeTree. [\#4143](https://github.com/ClickHouse/ClickHouse/pull/4143) ([Nikita Vasilev](https://github.com/nikvas0)) - Conversion ajoutée de `CROSS JOIN` de `INNER JOIN` si cela est possible. [\#4221](https://github.com/ClickHouse/ClickHouse/pull/4221) [\#4266](https://github.com/ClickHouse/ClickHouse/pull/4266) ([Artem Zuikov](https://github.com/4ertus2)) @@ -1828,52 +1828,52 @@ toc_title: '2019' - Fixe `Not found column` pour les colonnes en double dans `JOIN ON` section. [\#4279](https://github.com/ClickHouse/ClickHouse/pull/4279) ([Artem Zuikov](https://github.com/4ertus2)) - Faire `START REPLICATED SENDS` commande démarrer les envois répliqués. [\#4229](https://github.com/ClickHouse/ClickHouse/pull/4229) ([nvartolomei](https://github.com/nvartolomei)) -- Fixe l’exécution des fonctions d’agrégat avec `Array(LowCardinality)` argument. [\#4055](https://github.com/ClickHouse/ClickHouse/pull/4055) ([KochetovNicolai](https://github.com/KochetovNicolai)) +- Fixe l'exécution des fonctions d'agrégat avec `Array(LowCardinality)` argument. [\#4055](https://github.com/ClickHouse/ClickHouse/pull/4055) ([KochetovNicolai](https://github.com/KochetovNicolai)) - Fixe mauvais comportement lors `INSERT ... SELECT ... FROM file(...)` requête et fichier a `CSVWithNames` ou `TSVWIthNames` le format et la première ligne de données est manquant. [\#4297](https://github.com/ClickHouse/ClickHouse/pull/4297) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’un crash sur dictionnaire recharger si le dictionnaire n’est pas disponible. Ce bug est apparu dans 19.1.6. [\#4188](https://github.com/ClickHouse/ClickHouse/pull/4188) ([proller](https://github.com/proller)) +- Correction d'un crash sur dictionnaire recharger si le dictionnaire n'est pas disponible. Ce bug est apparu dans 19.1.6. [\#4188](https://github.com/ClickHouse/ClickHouse/pull/4188) ([proller](https://github.com/proller)) - Fixe `ALL JOIN` avec des doublons dans la table de droite. [\#4184](https://github.com/ClickHouse/ClickHouse/pull/4184) ([Artem Zuikov](https://github.com/4ertus2)) -- Correction d’un défaut de segmentation avec `use_uncompressed_cache=1` et exception avec une mauvaise taille non compressée. Ce bug est apparu dans 19.1.6. [\#4186](https://github.com/ClickHouse/ClickHouse/pull/4186) ([alésapine](https://github.com/alesapin)) +- Correction d'un défaut de segmentation avec `use_uncompressed_cache=1` et exception avec une mauvaise taille non compressée. Ce bug est apparu dans 19.1.6. [\#4186](https://github.com/ClickHouse/ClickHouse/pull/4186) ([alésapine](https://github.com/alesapin)) - Fixe `compile_expressions` bug avec comparaison de grandes dates (plus de int16). [\#4341](https://github.com/ClickHouse/ClickHouse/pull/4341) ([alésapine](https://github.com/alesapin)) - Boucle infinie fixe lors de la sélection de la fonction de table `numbers(0)`. [\#4280](https://github.com/ClickHouse/ClickHouse/pull/4280) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Désactiver temporairement l’optimisation des prédicats pour `ORDER BY`. [\#3890](https://github.com/ClickHouse/ClickHouse/pull/3890) ([L’Hiver Zhang](https://github.com/zhang2014)) -- Fixe `Illegal instruction` erreur lors de l’utilisation des fonctions base64 sur les anciens processeurs. Cette erreur n’a été reproduite que lorsque ClickHouse a été compilé avec gcc-8. [\#4275](https://github.com/ClickHouse/ClickHouse/pull/4275) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Fixe `No message received` erreur lors de l’interaction avec le pilote ODBC PostgreSQL via une connexion TLS. Corrige également segfault lors de l’utilisation du pilote MySQL ODBC. [\#4170](https://github.com/ClickHouse/ClickHouse/pull/4170) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’un résultat incorrect lorsque `Date` et `DateTime` les arguments sont utilisés dans les branches de l’opérateur conditionnel (fonction `if`). Ajouté cas générique pour la fonction `if`. [\#4243](https://github.com/ClickHouse/ClickHouse/pull/4243) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Désactiver temporairement l'optimisation des prédicats pour `ORDER BY`. [\#3890](https://github.com/ClickHouse/ClickHouse/pull/3890) ([L'Hiver Zhang](https://github.com/zhang2014)) +- Fixe `Illegal instruction` erreur lors de l'utilisation des fonctions base64 sur les anciens processeurs. Cette erreur n'a été reproduite que lorsque ClickHouse a été compilé avec gcc-8. [\#4275](https://github.com/ClickHouse/ClickHouse/pull/4275) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Fixe `No message received` erreur lors de l'interaction avec le pilote ODBC PostgreSQL via une connexion TLS. Corrige également segfault lors de l'utilisation du pilote MySQL ODBC. [\#4170](https://github.com/ClickHouse/ClickHouse/pull/4170) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction d'un résultat incorrect lorsque `Date` et `DateTime` les arguments sont utilisés dans les branches de l'opérateur conditionnel (fonction `if`). Ajouté cas générique pour la fonction `if`. [\#4243](https://github.com/ClickHouse/ClickHouse/pull/4243) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Les dictionnaires ClickHouse se chargent maintenant dans `clickhouse` processus. [\#4166](https://github.com/ClickHouse/ClickHouse/pull/4166) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Fixe blocage lorsqu’ `SELECT` à partir d’une table avec `File` moteur a été rejugé après `No such file or directory` erreur. [\#4161](https://github.com/ClickHouse/ClickHouse/pull/4161) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Fixe blocage lorsqu' `SELECT` à partir d'une table avec `File` moteur a été rejugé après `No such file or directory` erreur. [\#4161](https://github.com/ClickHouse/ClickHouse/pull/4161) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Condition de course fixe lors de la sélection de `system.tables` peut donner `table doesn't exist` erreur. [\#4313](https://github.com/ClickHouse/ClickHouse/pull/4313) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - `clickhouse-client` peut segfault à la sortie lors du chargement des données pour les suggestions de ligne de commande si elle a été exécutée en mode interactif. [\#4317](https://github.com/ClickHouse/ClickHouse/pull/4317) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’un bug lors de l’exécution de mutations contenant `IN` les opérateurs produisaient des résultats incorrects. [\#4099](https://github.com/ClickHouse/ClickHouse/pull/4099) ([Alex Zatelepin](https://github.com/ztlpn)) -- Correction d’une erreur: si une base de données avec `Dictionary` moteur, tous les dictionnaires forcés de charger au démarrage du serveur, et s’il y a un dictionnaire avec la source de ClickHouse de localhost, le dictionnaire ne peut pas charger. [\#4255](https://github.com/ClickHouse/ClickHouse/pull/4255) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’une erreur lorsque les journaux système sont tentés de créer à nouveau à l’arrêt du serveur. [\#4254](https://github.com/ClickHouse/ClickHouse/pull/4254) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction d'un bug lors de l'exécution de mutations contenant `IN` les opérateurs produisaient des résultats incorrects. [\#4099](https://github.com/ClickHouse/ClickHouse/pull/4099) ([Alex Zatelepin](https://github.com/ztlpn)) +- Correction d'une erreur: si une base de données avec `Dictionary` moteur, tous les dictionnaires forcés de charger au démarrage du serveur, et s'il y a un dictionnaire avec la source de ClickHouse de localhost, le dictionnaire ne peut pas charger. [\#4255](https://github.com/ClickHouse/ClickHouse/pull/4255) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction d'une erreur lorsque les journaux système sont tentés de créer à nouveau à l'arrêt du serveur. [\#4254](https://github.com/ClickHouse/ClickHouse/pull/4254) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Renvoyer correctement le bon type et gérer correctement les verrous `joinGet` fonction. [\#4153](https://github.com/ClickHouse/ClickHouse/pull/4153) ([Amos Oiseau](https://github.com/amosbird)) - Ajouter `sumMapWithOverflow` fonction. [\#4151](https://github.com/ClickHouse/ClickHouse/pull/4151) ([Léo Ercolanelli](https://github.com/ercolanelli-leo)) - Fixe erreur de segmentation avec `allow_experimental_multiple_joins_emulation`. [52de2c](https://github.com/ClickHouse/ClickHouse/commit/52de2cd927f7b5257dd67e175f0a5560a48840d0) ([Artem Zuikov](https://github.com/4ertus2)) -- Correction d’un bug avec incorrect `Date` et `DateTime` comparaison. [\#4237](https://github.com/ClickHouse/ClickHouse/pull/4237) ([valexey](https://github.com/valexey)) -- Correction d’un test de fuzz sous un désinfectant de comportement indéfini: ajout d’une vérification de type de paramètre pour `quantile*Weighted` la famille de fonctions. [\#4145](https://github.com/ClickHouse/ClickHouse/pull/4145) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’une condition de course rare lors de la suppression d’anciennes pièces de données peuvent échouer avec `File not found` erreur. [\#4378](https://github.com/ClickHouse/ClickHouse/pull/4378) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction du paquet d’installation avec /etc/clickhouse-server/config manquant.XML. [\#4343](https://github.com/ClickHouse/ClickHouse/pull/4343) ([proller](https://github.com/proller)) +- Correction d'un bug avec incorrect `Date` et `DateTime` comparaison. [\#4237](https://github.com/ClickHouse/ClickHouse/pull/4237) ([valexey](https://github.com/valexey)) +- Correction d'un test de fuzz sous un désinfectant de comportement indéfini: ajout d'une vérification de type de paramètre pour `quantile*Weighted` la famille de fonctions. [\#4145](https://github.com/ClickHouse/ClickHouse/pull/4145) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction d'une condition de course rare lors de la suppression d'anciennes pièces de données peuvent échouer avec `File not found` erreur. [\#4378](https://github.com/ClickHouse/ClickHouse/pull/4378) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction du paquet d'installation avec /etc/clickhouse-server/config manquant.XML. [\#4343](https://github.com/ClickHouse/ClickHouse/pull/4343) ([proller](https://github.com/proller)) #### Construire/Test/Emballage Améliorations {#buildtestingpackaging-improvements-5} - Paquet Debian: correct/etc/clickhouse-server / lien prétraité selon config. [\#4205](https://github.com/ClickHouse/ClickHouse/pull/4205) ([proller](https://github.com/proller)) - Divers correctifs de construction Pour FreeBSD. [\#4225](https://github.com/ClickHouse/ClickHouse/pull/4225) ([proller](https://github.com/proller)) - Ajout de la possibilité de créer, remplir et déposer des tables dans perftest. [\#4220](https://github.com/ClickHouse/ClickHouse/pull/4220) ([alésapine](https://github.com/alesapin)) -- Ajout d’un script pour vérifier les doublons comprend. [\#4326](https://github.com/ClickHouse/ClickHouse/pull/4326) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Ajout de la possibilité d’exécuter des requêtes par index dans le test de performance. [\#4264](https://github.com/ClickHouse/ClickHouse/pull/4264) ([alésapine](https://github.com/alesapin)) -- Paquet avec des symboles de débogage est suggéré d’être installé. [\#4274](https://github.com/ClickHouse/ClickHouse/pull/4274) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Ajout d'un script pour vérifier les doublons comprend. [\#4326](https://github.com/ClickHouse/ClickHouse/pull/4326) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Ajout de la possibilité d'exécuter des requêtes par index dans le test de performance. [\#4264](https://github.com/ClickHouse/ClickHouse/pull/4264) ([alésapine](https://github.com/alesapin)) +- Paquet avec des symboles de débogage est suggéré d'être installé. [\#4274](https://github.com/ClickHouse/ClickHouse/pull/4274) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Refactoring de performance-test. Meilleure journalisation et gestion des signaux. [\#4171](https://github.com/ClickHouse/ClickHouse/pull/4171) ([alésapine](https://github.com/alesapin)) - Ajout de documents à Yandex anonymisé.Jeux de données Metrika. [\#4164](https://github.com/ClickHouse/ClickHouse/pull/4164) ([alésapine](https://github.com/alesapin)) - Аdded tool for converting an old month-partitioned part to the custom-partitioned format. [\#4195](https://github.com/ClickHouse/ClickHouse/pull/4195) ([Alex Zatelepin](https://github.com/ztlpn)) - Ajout de documents sur deux ensembles de données dans s3. [\#4144](https://github.com/ClickHouse/ClickHouse/pull/4144) ([alésapine](https://github.com/alesapin)) -- Ajout d’un script qui crée le journal des modifications à partir de la description des requêtes d’extraction. [\#4169](https://github.com/ClickHouse/ClickHouse/pull/4169) [\#4173](https://github.com/ClickHouse/ClickHouse/pull/4173) ([KochetovNicolai](https://github.com/KochetovNicolai)) ([KochetovNicolai](https://github.com/KochetovNicolai)) -- Ajout du module puppet pour Clickhouse. [\#4182](https://github.com/ClickHouse/ClickHouse/pull/4182) ([Maxim Fedotov](https://github.com/MaxFedotov)) -- Ajout de documents pour un groupe de fonctions non documentées. [\#4168](https://github.com/ClickHouse/ClickHouse/pull/4168) ([L’Hiver Zhang](https://github.com/zhang2014)) +- Ajout d'un script qui crée le journal des modifications à partir de la description des requêtes d'extraction. [\#4169](https://github.com/ClickHouse/ClickHouse/pull/4169) [\#4173](https://github.com/ClickHouse/ClickHouse/pull/4173) ([KochetovNicolai](https://github.com/KochetovNicolai)) ([KochetovNicolai](https://github.com/KochetovNicolai)) +- Ajout du module puppet pour ClickHouse. [\#4182](https://github.com/ClickHouse/ClickHouse/pull/4182) ([Maxim Fedotov](https://github.com/MaxFedotov)) +- Ajout de documents pour un groupe de fonctions non documentées. [\#4168](https://github.com/ClickHouse/ClickHouse/pull/4168) ([L'Hiver Zhang](https://github.com/zhang2014)) - ARM construire des correctifs. [\#4210](https://github.com/ClickHouse/ClickHouse/pull/4210)[\#4306](https://github.com/ClickHouse/ClickHouse/pull/4306) [\#4291](https://github.com/ClickHouse/ClickHouse/pull/4291) ([proller](https://github.com/proller)) ([proller](https://github.com/proller)) -- Tests de dictionnaire maintenant capables de s’exécuter à partir de `ctest`. [\#4189](https://github.com/ClickHouse/ClickHouse/pull/4189) ([proller](https://github.com/proller)) +- Tests de dictionnaire maintenant capables de s'exécuter à partir de `ctest`. [\#4189](https://github.com/ClickHouse/ClickHouse/pull/4189) ([proller](https://github.com/proller)) - Maintenant `/etc/ssl` est utilisé comme répertoire par défaut avec les certificats SSL. [\#4167](https://github.com/ClickHouse/ClickHouse/pull/4167) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Ajout de la vérification SSE et AVX instruction au début. [\#4234](https://github.com/ClickHouse/ClickHouse/pull/4234) ([Igr](https://github.com/igron99)) -- Le script d’initialisation attendra le serveur jusqu’au démarrage. [\#4281](https://github.com/ClickHouse/ClickHouse/pull/4281) ([proller](https://github.com/proller)) +- Le script d'initialisation attendra le serveur jusqu'au démarrage. [\#4281](https://github.com/ClickHouse/ClickHouse/pull/4281) ([proller](https://github.com/proller)) #### Modifications Incompatibles En Arrière {#backward-incompatible-changes-1} @@ -1894,20 +1894,20 @@ toc_title: '2019' - Permettre `-C` option du client pour travailler comme `-c` option. [\#4232](https://github.com/ClickHouse/ClickHouse/pull/4232) ([syominsergey](https://github.com/syominsergey)) - Option `--password` utilisé sans valeur nécessite un mot de passe de stdin. [\#4230](https://github.com/ClickHouse/ClickHouse/pull/4230) ([BSD\_Conqueror](https://github.com/bsd-conqueror)) - Ajout de la mise en évidence des métacaractères Non échappés dans les littéraux de chaîne qui contiennent `LIKE` expressions ou expressions rationnelles. [\#4327](https://github.com/ClickHouse/ClickHouse/pull/4327) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Ajout de l’annulation des requêtes HTTP en lecture seule si le socket client disparaît. [\#4213](https://github.com/ClickHouse/ClickHouse/pull/4213) ([nvartolomei](https://github.com/nvartolomei)) +- Ajout de l'annulation des requêtes HTTP en lecture seule si le socket client disparaît. [\#4213](https://github.com/ClickHouse/ClickHouse/pull/4213) ([nvartolomei](https://github.com/nvartolomei)) - Maintenant, le serveur signale la progression pour maintenir les connexions client en vie. [\#4215](https://github.com/ClickHouse/ClickHouse/pull/4215) ([Ivan](https://github.com/abyss7)) - Message légèrement meilleur avec raison pour optimiser la requête avec `optimize_throw_if_noop` paramètre est activé. [\#4294](https://github.com/ClickHouse/ClickHouse/pull/4294) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Ajout du support de `--version` option pour le serveur clickhouse. [\#4251](https://github.com/ClickHouse/ClickHouse/pull/4251) ([Lopatin Konstantin](https://github.com/k-lopatin)) - Ajouter `--help/-h` option pour `clickhouse-server`. [\#4233](https://github.com/ClickHouse/ClickHouse/pull/4233) ([Yuriy Baranov](https://github.com/yurriy)) -- Ajout du support pour les sous-requêtes scalaires avec le résultat de l’état de la fonction d’agrégation. [\#4348](https://github.com/ClickHouse/ClickHouse/pull/4348) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Amélioration du temps d’arrêt du serveur et modifie le temps d’attente. [\#4372](https://github.com/ClickHouse/ClickHouse/pull/4372) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Ajout d’informations sur le paramètre replicated\_can\_become\_leader au système.réplicas et ajouter la journalisation si la réplique ne sera pas essayer de devenir leader. [\#4379](https://github.com/ClickHouse/ClickHouse/pull/4379) ([Alex Zatelepin](https://github.com/ztlpn)) +- Ajout du support pour les sous-requêtes scalaires avec le résultat de l'état de la fonction d'agrégation. [\#4348](https://github.com/ClickHouse/ClickHouse/pull/4348) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Amélioration du temps d'arrêt du serveur et modifie le temps d'attente. [\#4372](https://github.com/ClickHouse/ClickHouse/pull/4372) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Ajout d'informations sur le paramètre replicated\_can\_become\_leader au système.réplicas et ajouter la journalisation si la réplique ne sera pas essayer de devenir leader. [\#4379](https://github.com/ClickHouse/ClickHouse/pull/4379) ([Alex Zatelepin](https://github.com/ztlpn)) ## Clickhouse Version 19.1 {#clickhouse-release-19-1} ### Clickhouse Version 19.1.14, 2019-03-14 {#clickhouse-release-19-1-14-2019-03-14} -- Correction d’une erreur `Column ... queried more than once` cela peut arriver si le réglage `asterisk_left_columns_only` est réglé sur 1 en cas d’utilisation `GLOBAL JOIN` avec `SELECT *` (cas rare). Le problème n’existe pas dans 19.3 et plus récent. [6bac7d8d](https://github.com/ClickHouse/ClickHouse/pull/4692/commits/6bac7d8d11a9b0d6de0b32b53c47eb2f6f8e7062) ([Artem Zuikov](https://github.com/4ertus2)) +- Correction d'une erreur `Column ... queried more than once` cela peut arriver si le réglage `asterisk_left_columns_only` est réglé sur 1 en cas d'utilisation `GLOBAL JOIN` avec `SELECT *` (cas rare). Le problème n'existe pas dans 19.3 et plus récent. [6bac7d8d](https://github.com/ClickHouse/ClickHouse/pull/4692/commits/6bac7d8d11a9b0d6de0b32b53c47eb2f6f8e7062) ([Artem Zuikov](https://github.com/4ertus2)) ### Clickhouse Version 19.1.13, 2019-03-12 {#clickhouse-release-19-1-13-2019-03-12} @@ -1923,14 +1923,14 @@ Cette version contient exactement le même ensemble de patchs 19.3.6. #### Corrections De Bugs {#bug-fixes-18} -- Correction de l’incompatibilité avec les anciennes versions en raison d’une mauvaise implémentation de `send_logs_level` paramètre. [\#4445](https://github.com/ClickHouse/ClickHouse/pull/4445) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction de l'incompatibilité avec les anciennes versions en raison d'une mauvaise implémentation de `send_logs_level` paramètre. [\#4445](https://github.com/ClickHouse/ClickHouse/pull/4445) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Incompatibilité arrière fixe de la fonction de table `remote` introduit avec des commentaires de colonne. [\#4446](https://github.com/ClickHouse/ClickHouse/pull/4446) ([alexeï-milovidov](https://github.com/alexey-milovidov)) ### Clickhouse Version 19.1.8, 2019-02-16 {#clickhouse-release-19-1-8-2019-02-16} #### Corrections De Bugs {#bug-fixes-19} -- Correction du paquet d’installation avec /etc/clickhouse-server/config manquant.XML. [\#4343](https://github.com/ClickHouse/ClickHouse/pull/4343) ([proller](https://github.com/proller)) +- Correction du paquet d'installation avec /etc/clickhouse-server/config manquant.XML. [\#4343](https://github.com/ClickHouse/ClickHouse/pull/4343) ([proller](https://github.com/proller)) ## Clickhouse Version 19.1 {#clickhouse-release-19-1-2} @@ -1939,100 +1939,100 @@ Cette version contient exactement le même ensemble de patchs 19.3.6. #### Corrections De Bugs {#bug-fixes-20} - Renvoyer correctement le bon type et gérer correctement les verrous `joinGet` fonction. [\#4153](https://github.com/ClickHouse/ClickHouse/pull/4153) ([Amos Oiseau](https://github.com/amosbird)) -- Correction d’une erreur lorsque les journaux système sont tentés de créer à nouveau à l’arrêt du serveur. [\#4254](https://github.com/ClickHouse/ClickHouse/pull/4254) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’une erreur: si une base de données avec `Dictionary` moteur, tous les dictionnaires forcés de charger au démarrage du serveur, et s’il y a un dictionnaire avec la source de ClickHouse de localhost, le dictionnaire ne peut pas charger. [\#4255](https://github.com/ClickHouse/ClickHouse/pull/4255) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’un bug lors de l’exécution de mutations contenant `IN` les opérateurs produisaient des résultats incorrects. [\#4099](https://github.com/ClickHouse/ClickHouse/pull/4099) ([Alex Zatelepin](https://github.com/ztlpn)) +- Correction d'une erreur lorsque les journaux système sont tentés de créer à nouveau à l'arrêt du serveur. [\#4254](https://github.com/ClickHouse/ClickHouse/pull/4254) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction d'une erreur: si une base de données avec `Dictionary` moteur, tous les dictionnaires forcés de charger au démarrage du serveur, et s'il y a un dictionnaire avec la source de ClickHouse de localhost, le dictionnaire ne peut pas charger. [\#4255](https://github.com/ClickHouse/ClickHouse/pull/4255) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction d'un bug lors de l'exécution de mutations contenant `IN` les opérateurs produisaient des résultats incorrects. [\#4099](https://github.com/ClickHouse/ClickHouse/pull/4099) ([Alex Zatelepin](https://github.com/ztlpn)) - `clickhouse-client` peut segfault à la sortie lors du chargement des données pour les suggestions de ligne de commande si elle a été exécutée en mode interactif. [\#4317](https://github.com/ClickHouse/ClickHouse/pull/4317) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Condition de course fixe lors de la sélection de `system.tables` peut donner `table doesn't exist` erreur. [\#4313](https://github.com/ClickHouse/ClickHouse/pull/4313) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Fixe blocage lorsqu’ `SELECT` à partir d’une table avec `File` moteur a été rejugé après `No such file or directory` erreur. [\#4161](https://github.com/ClickHouse/ClickHouse/pull/4161) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’un problème: les dictionnaires ClickHouse locaux sont chargés via TCP, mais devraient être chargés dans le processus. [\#4166](https://github.com/ClickHouse/ClickHouse/pull/4166) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Fixe `No message received` erreur lors de l’interaction avec le pilote ODBC PostgreSQL via une connexion TLS. Corrige également segfault lors de l’utilisation du pilote MySQL ODBC. [\#4170](https://github.com/ClickHouse/ClickHouse/pull/4170) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Désactiver temporairement l’optimisation des prédicats pour `ORDER BY`. [\#3890](https://github.com/ClickHouse/ClickHouse/pull/3890) ([L’Hiver Zhang](https://github.com/zhang2014)) +- Fixe blocage lorsqu' `SELECT` à partir d'une table avec `File` moteur a été rejugé après `No such file or directory` erreur. [\#4161](https://github.com/ClickHouse/ClickHouse/pull/4161) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction d'un problème: les dictionnaires ClickHouse locaux sont chargés via TCP, mais devraient être chargés dans le processus. [\#4166](https://github.com/ClickHouse/ClickHouse/pull/4166) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Fixe `No message received` erreur lors de l'interaction avec le pilote ODBC PostgreSQL via une connexion TLS. Corrige également segfault lors de l'utilisation du pilote MySQL ODBC. [\#4170](https://github.com/ClickHouse/ClickHouse/pull/4170) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Désactiver temporairement l'optimisation des prédicats pour `ORDER BY`. [\#3890](https://github.com/ClickHouse/ClickHouse/pull/3890) ([L'Hiver Zhang](https://github.com/zhang2014)) - Boucle infinie fixe lors de la sélection de la fonction de table `numbers(0)`. [\#4280](https://github.com/ClickHouse/ClickHouse/pull/4280) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Fixe `compile_expressions` bug avec comparaison de grandes dates (plus de int16). [\#4341](https://github.com/ClickHouse/ClickHouse/pull/4341) ([alésapine](https://github.com/alesapin)) -- Correction d’un défaut de segmentation avec `uncompressed_cache=1` et exception avec une mauvaise taille non compressée. [\#4186](https://github.com/ClickHouse/ClickHouse/pull/4186) ([alésapine](https://github.com/alesapin)) +- Correction d'un défaut de segmentation avec `uncompressed_cache=1` et exception avec une mauvaise taille non compressée. [\#4186](https://github.com/ClickHouse/ClickHouse/pull/4186) ([alésapine](https://github.com/alesapin)) - Fixe `ALL JOIN` avec des doublons dans la table de droite. [\#4184](https://github.com/ClickHouse/ClickHouse/pull/4184) ([Artem Zuikov](https://github.com/4ertus2)) - Fixe mauvais comportement lors `INSERT ... SELECT ... FROM file(...)` requête et fichier a `CSVWithNames` ou `TSVWIthNames` le format et la première ligne de données est manquant. [\#4297](https://github.com/ClickHouse/ClickHouse/pull/4297) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Fixe l’exécution des fonctions d’agrégat avec `Array(LowCardinality)` argument. [\#4055](https://github.com/ClickHouse/ClickHouse/pull/4055) ([KochetovNicolai](https://github.com/KochetovNicolai)) +- Fixe l'exécution des fonctions d'agrégat avec `Array(LowCardinality)` argument. [\#4055](https://github.com/ClickHouse/ClickHouse/pull/4055) ([KochetovNicolai](https://github.com/KochetovNicolai)) - Paquet Debian: correct/etc/clickhouse-server / lien prétraité selon config. [\#4205](https://github.com/ClickHouse/ClickHouse/pull/4205) ([proller](https://github.com/proller)) -- Correction d’un test de fuzz sous un désinfectant de comportement indéfini: ajout d’une vérification de type de paramètre pour `quantile*Weighted` la famille de fonctions. [\#4145](https://github.com/ClickHouse/ClickHouse/pull/4145) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction d'un test de fuzz sous un désinfectant de comportement indéfini: ajout d'une vérification de type de paramètre pour `quantile*Weighted` la famille de fonctions. [\#4145](https://github.com/ClickHouse/ClickHouse/pull/4145) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Faire `START REPLICATED SENDS` commande démarrer les envois répliqués. [\#4229](https://github.com/ClickHouse/ClickHouse/pull/4229) ([nvartolomei](https://github.com/nvartolomei)) - Fixe `Not found column` pour les colonnes en double dans JOIN ON section. [\#4279](https://github.com/ClickHouse/ClickHouse/pull/4279) ([Artem Zuikov](https://github.com/4ertus2)) - Maintenant `/etc/ssl` est utilisé comme répertoire par défaut avec les certificats SSL. [\#4167](https://github.com/ClickHouse/ClickHouse/pull/4167) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’un crash sur dictionnaire recharger si le dictionnaire n’est pas disponible. [\#4188](https://github.com/ClickHouse/ClickHouse/pull/4188) ([proller](https://github.com/proller)) -- Correction d’un bug avec incorrect `Date` et `DateTime` comparaison. [\#4237](https://github.com/ClickHouse/ClickHouse/pull/4237) ([valexey](https://github.com/valexey)) -- Correction d’un résultat incorrect lorsque `Date` et `DateTime` les arguments sont utilisés dans les branches de l’opérateur conditionnel (fonction `if`). Ajouté cas générique pour la fonction `if`. [\#4243](https://github.com/ClickHouse/ClickHouse/pull/4243) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction d'un crash sur dictionnaire recharger si le dictionnaire n'est pas disponible. [\#4188](https://github.com/ClickHouse/ClickHouse/pull/4188) ([proller](https://github.com/proller)) +- Correction d'un bug avec incorrect `Date` et `DateTime` comparaison. [\#4237](https://github.com/ClickHouse/ClickHouse/pull/4237) ([valexey](https://github.com/valexey)) +- Correction d'un résultat incorrect lorsque `Date` et `DateTime` les arguments sont utilisés dans les branches de l'opérateur conditionnel (fonction `if`). Ajouté cas générique pour la fonction `if`. [\#4243](https://github.com/ClickHouse/ClickHouse/pull/4243) ([alexeï-milovidov](https://github.com/alexey-milovidov)) ### Clickhouse Version 19.1.6, 2019-01-24 {#clickhouse-release-19-1-6-2019-01-24} #### Nouveauté {#new-features-7} -- Codecs de compression personnalisés par colonne pour les tables. [\#3899](https://github.com/ClickHouse/ClickHouse/pull/3899) [\#4111](https://github.com/ClickHouse/ClickHouse/pull/4111) ([alésapine](https://github.com/alesapin), [L’Hiver Zhang](https://github.com/zhang2014), [Anatoli](https://github.com/Sindbag)) +- Codecs de compression personnalisés par colonne pour les tables. [\#3899](https://github.com/ClickHouse/ClickHouse/pull/3899) [\#4111](https://github.com/ClickHouse/ClickHouse/pull/4111) ([alésapine](https://github.com/alesapin), [L'Hiver Zhang](https://github.com/zhang2014), [Anatoli](https://github.com/Sindbag)) - Ajout du codec de compression `Delta`. [\#4052](https://github.com/ClickHouse/ClickHouse/pull/4052) ([alésapine](https://github.com/alesapin)) - Permettre à `ALTER` codecs de compression. [\#4054](https://github.com/ClickHouse/ClickHouse/pull/4054) ([alésapine](https://github.com/alesapin)) -- L’ajout de fonctions `left`, `right`, `trim`, `ltrim`, `rtrim`, `timestampadd`, `timestampsub` pour la compatibilité standard SQL. [\#3826](https://github.com/ClickHouse/ClickHouse/pull/3826) ([Ivan Blinkov](https://github.com/blinkov)) +- L'ajout de fonctions `left`, `right`, `trim`, `ltrim`, `rtrim`, `timestampadd`, `timestampsub` pour la compatibilité standard SQL. [\#3826](https://github.com/ClickHouse/ClickHouse/pull/3826) ([Ivan Blinkov](https://github.com/blinkov)) - Soutien pour écrire dans `HDFS` des tables et des `hdfs` table de fonction. [\#4084](https://github.com/ClickHouse/ClickHouse/pull/4084) ([alésapine](https://github.com/alesapin)) -- L’ajout de fonctions pour rechercher plusieurs chaines de grosse botte de foin: `multiPosition`, `multiSearch` ,`firstMatch` aussi avec `-UTF8`, `-CaseInsensitive`, et `-CaseInsensitiveUTF8` variantes. [\#4053](https://github.com/ClickHouse/ClickHouse/pull/4053) ([Danila Kutenin](https://github.com/danlark1)) +- L'ajout de fonctions pour rechercher plusieurs chaines de grosse botte de foin: `multiPosition`, `multiSearch` ,`firstMatch` aussi avec `-UTF8`, `-CaseInsensitive`, et `-CaseInsensitiveUTF8` variantes. [\#4053](https://github.com/ClickHouse/ClickHouse/pull/4053) ([Danila Kutenin](https://github.com/danlark1)) - Taille des éclats inutilisés si `SELECT` filtres de requête par clé de sharding (réglage `optimize_skip_unused_shards`). [\#3851](https://github.com/ClickHouse/ClickHouse/pull/3851) ([Gleb Kanterov](https://github.com/kanterov), [Ivan](https://github.com/abyss7)) -- Permettre `Kafka` moteur à ignorer certains nombre d’erreurs d’analyse par bloc. [\#4094](https://github.com/ClickHouse/ClickHouse/pull/4094) ([Ivan](https://github.com/abyss7)) +- Permettre `Kafka` moteur à ignorer certains nombre d'erreurs d'analyse par bloc. [\#4094](https://github.com/ClickHouse/ClickHouse/pull/4094) ([Ivan](https://github.com/abyss7)) - Ajout du support pour `CatBoost` évaluation des modèles multiclass. Fonction `modelEvaluate` retourne tuple avec des prédictions brutes par classe pour les modèles multiclasse. `libcatboostmodel.so` devrait être construit avec [\#607](https://github.com/catboost/catboost/pull/607). [\#3959](https://github.com/ClickHouse/ClickHouse/pull/3959) ([KochetovNicolai](https://github.com/KochetovNicolai)) -- L’ajout de fonctions `filesystemAvailable`, `filesystemFree`, `filesystemCapacity`. [\#4097](https://github.com/ClickHouse/ClickHouse/pull/4097) ([Boris Granveaud](https://github.com/bgranvea)) +- L'ajout de fonctions `filesystemAvailable`, `filesystemFree`, `filesystemCapacity`. [\#4097](https://github.com/ClickHouse/ClickHouse/pull/4097) ([Boris Granveaud](https://github.com/bgranvea)) - Ajouté les fonctions de hachage `xxHash64` et `xxHash32`. [\#3905](https://github.com/ClickHouse/ClickHouse/pull/3905) ([filimonov](https://github.com/filimonov)) - Ajouter `gccMurmurHash` fonction de hachage (hachage Murmur aromatisé GCC) qui utilise la même graine de hachage que [gcc](https://github.com/gcc-mirror/gcc/blob/41d6b10e96a1de98e90a7c0378437c3255814b16/libstdc%2B%2B-v3/include/bits/functional_hash.h#L191) [\#4000](https://github.com/ClickHouse/ClickHouse/pull/4000) ([sundyli](https://github.com/sundy-li)) - Ajouté les fonctions de hachage `javaHash`, `hiveHash`. [\#3811](https://github.com/ClickHouse/ClickHouse/pull/3811) ([shangshujie365](https://github.com/shangshujie365)) -- Ajout d’un tableau de fonction `remoteSecure`. Fonction fonctionne comme `remote`, mais il utilise une connexion sécurisée. [\#4088](https://github.com/ClickHouse/ClickHouse/pull/4088) ([proller](https://github.com/proller)) +- Ajout d'un tableau de fonction `remoteSecure`. Fonction fonctionne comme `remote`, mais il utilise une connexion sécurisée. [\#4088](https://github.com/ClickHouse/ClickHouse/pull/4088) ([proller](https://github.com/proller)) -#### Caractéristiques expérimentales {#experimental-features-3} +#### Caractéristiques Expérimentales {#experimental-features-3} - Ajout de plusieurs jointures émulation (`allow_experimental_multiple_joins_emulation` paramètre). [\#3946](https://github.com/ClickHouse/ClickHouse/pull/3946) ([Artem Zuikov](https://github.com/4ertus2)) #### Corrections De Bugs {#bug-fixes-21} - Faire `compiled_expression_cache_size` réglage limité par défaut pour réduire la consommation de mémoire. [\#4041](https://github.com/ClickHouse/ClickHouse/pull/4041) ([alésapine](https://github.com/alesapin)) -- Correction d’un bug qui a conduit à des interruptions dans les threads qui effectuent des modifications de tables répliquées et dans le thread qui met à jour la configuration de ZooKeeper. [\#2947](https://github.com/ClickHouse/ClickHouse/issues/2947) [\#3891](https://github.com/ClickHouse/ClickHouse/issues/3891) [\#3934](https://github.com/ClickHouse/ClickHouse/pull/3934) ([Alex Zatelepin](https://github.com/ztlpn)) -- Correction d’une condition de concurrence lors de l’exécution d’une tâche alter distribuée. La condition de concurrence a conduit à plus d’une réplique essayant d’exécuter la tâche et toutes les répliques sauf une échouant avec une erreur ZooKeeper. [\#3904](https://github.com/ClickHouse/ClickHouse/pull/3904) ([Alex Zatelepin](https://github.com/ztlpn)) -- Correction d’un bug lors de l’ `from_zk` les éléments de configuration n’ont pas été actualisés après l’expiration d’une requête à ZooKeeper. [\#2947](https://github.com/ClickHouse/ClickHouse/issues/2947) [\#3947](https://github.com/ClickHouse/ClickHouse/pull/3947) ([Alex Zatelepin](https://github.com/ztlpn)) -- Correction d’un bug avec un mauvais préfixe pour les masques de sous-réseau IPv4. [\#3945](https://github.com/ClickHouse/ClickHouse/pull/3945) ([alésapine](https://github.com/alesapin)) -- Correction d’un crash (`std::terminate`) dans de rares cas où un nouveau thread ne peut pas être créé en raison de ressources épuisées. [\#3956](https://github.com/ClickHouse/ClickHouse/pull/3956) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’un bug quand dans `remote` exécution de la fonction de table lorsque des restrictions incorrectes ont été utilisées pour in `getStructureOfRemoteTable`. [\#4009](https://github.com/ClickHouse/ClickHouse/pull/4009) ([alésapine](https://github.com/alesapin)) -- Correction d’une fuite de sockets netlink. Ils ont été placés dans un pool où ils n’ont jamais été supprimés et de nouvelles sockets ont été créées au début d’un nouveau thread lorsque toutes les sockets actuelles étaient en cours d’utilisation. [\#4017](https://github.com/ClickHouse/ClickHouse/pull/4017) ([Alex Zatelepin](https://github.com/ztlpn)) -- Correction d’un bug avec fermeture `/proc/self/fd` répertoire plus tôt que tous les fds ont été lus à partir de `/proc` après la bifurcation `odbc-bridge` sous-processus. [\#4120](https://github.com/ClickHouse/ClickHouse/pull/4120) ([alésapine](https://github.com/alesapin)) -- Correction de la chaîne à la conversion monotone UInt en cas d’utilisation de la chaîne dans la clé primaire. [\#3870](https://github.com/ClickHouse/ClickHouse/pull/3870) ([L’Hiver Zhang](https://github.com/zhang2014)) -- Correction d’une erreur dans le calcul de la monotonie de la fonction de conversion entière. [\#3921](https://github.com/ClickHouse/ClickHouse/pull/3921) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Fixe erreur de segmentation dans `arrayEnumerateUniq`, `arrayEnumerateDense` fonctions en cas d’arguments non valides. [\#3909](https://github.com/ClickHouse/ClickHouse/pull/3909) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction d'un bug qui a conduit à des interruptions dans les threads qui effectuent des modifications de tables répliquées et dans le thread qui met à jour la configuration de ZooKeeper. [\#2947](https://github.com/ClickHouse/ClickHouse/issues/2947) [\#3891](https://github.com/ClickHouse/ClickHouse/issues/3891) [\#3934](https://github.com/ClickHouse/ClickHouse/pull/3934) ([Alex Zatelepin](https://github.com/ztlpn)) +- Correction d'une condition de concurrence lors de l'exécution d'une tâche alter distribuée. La condition de concurrence a conduit à plus d'une réplique essayant d'exécuter la tâche et toutes les répliques sauf une échouant avec une erreur ZooKeeper. [\#3904](https://github.com/ClickHouse/ClickHouse/pull/3904) ([Alex Zatelepin](https://github.com/ztlpn)) +- Correction d'un bug lors de l' `from_zk` les éléments de configuration n'ont pas été actualisés après l'expiration d'une requête à ZooKeeper. [\#2947](https://github.com/ClickHouse/ClickHouse/issues/2947) [\#3947](https://github.com/ClickHouse/ClickHouse/pull/3947) ([Alex Zatelepin](https://github.com/ztlpn)) +- Correction d'un bug avec un mauvais préfixe pour les masques de sous-réseau IPv4. [\#3945](https://github.com/ClickHouse/ClickHouse/pull/3945) ([alésapine](https://github.com/alesapin)) +- Correction d'un crash (`std::terminate`) dans de rares cas où un nouveau thread ne peut pas être créé en raison de ressources épuisées. [\#3956](https://github.com/ClickHouse/ClickHouse/pull/3956) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction d'un bug quand dans `remote` exécution de la fonction de table lorsque des restrictions incorrectes ont été utilisées pour in `getStructureOfRemoteTable`. [\#4009](https://github.com/ClickHouse/ClickHouse/pull/4009) ([alésapine](https://github.com/alesapin)) +- Correction d'une fuite de sockets netlink. Ils ont été placés dans un pool où ils n'ont jamais été supprimés et de nouvelles sockets ont été créées au début d'un nouveau thread lorsque toutes les sockets actuelles étaient en cours d'utilisation. [\#4017](https://github.com/ClickHouse/ClickHouse/pull/4017) ([Alex Zatelepin](https://github.com/ztlpn)) +- Correction d'un bug avec fermeture `/proc/self/fd` répertoire plus tôt que tous les fds ont été lus à partir de `/proc` après la bifurcation `odbc-bridge` sous-processus. [\#4120](https://github.com/ClickHouse/ClickHouse/pull/4120) ([alésapine](https://github.com/alesapin)) +- Correction de la chaîne à la conversion monotone UInt en cas d'utilisation de la chaîne dans la clé primaire. [\#3870](https://github.com/ClickHouse/ClickHouse/pull/3870) ([L'Hiver Zhang](https://github.com/zhang2014)) +- Correction d'une erreur dans le calcul de la monotonie de la fonction de conversion entière. [\#3921](https://github.com/ClickHouse/ClickHouse/pull/3921) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Fixe erreur de segmentation dans `arrayEnumerateUniq`, `arrayEnumerateDense` fonctions en cas d'arguments non valides. [\#3909](https://github.com/ClickHouse/ClickHouse/pull/3909) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Corriger UB dans StorageMerge. [\#3910](https://github.com/ClickHouse/ClickHouse/pull/3910) ([Amos Oiseau](https://github.com/amosbird)) - Correction de segfault dans les fonctions `addDays`, `subtractDays`. [\#3913](https://github.com/ClickHouse/ClickHouse/pull/3913) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’une erreur: fonctions `round`, `floor`, `trunc`, `ceil` peut renvoyer un résultat faux lorsqu’il est exécuté sur un argument entier et une grande échelle négative. [\#3914](https://github.com/ClickHouse/ClickHouse/pull/3914) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’un bug induit par ‘kill query sync’ ce qui conduit à une décharge de base. [\#3916](https://github.com/ClickHouse/ClickHouse/pull/3916) ([muVulDeePecker](https://github.com/fancyqlx)) -- Correction d’un bug avec un long délai après la file d’attente de réplication vide. [\#3928](https://github.com/ClickHouse/ClickHouse/pull/3928) [\#3932](https://github.com/ClickHouse/ClickHouse/pull/3932) ([alésapine](https://github.com/alesapin)) -- Correction d’une utilisation excessive de la mémoire en cas d’insertion dans la table avec `LowCardinality` clé primaire. [\#3955](https://github.com/ClickHouse/ClickHouse/pull/3955) ([KochetovNicolai](https://github.com/KochetovNicolai)) +- Correction d'une erreur: fonctions `round`, `floor`, `trunc`, `ceil` peut renvoyer un résultat faux lorsqu'il est exécuté sur un argument entier et une grande échelle négative. [\#3914](https://github.com/ClickHouse/ClickHouse/pull/3914) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction d'un bug induit par ‘kill query sync’ ce qui conduit à une décharge de base. [\#3916](https://github.com/ClickHouse/ClickHouse/pull/3916) ([muVulDeePecker](https://github.com/fancyqlx)) +- Correction d'un bug avec un long délai après la file d'attente de réplication vide. [\#3928](https://github.com/ClickHouse/ClickHouse/pull/3928) [\#3932](https://github.com/ClickHouse/ClickHouse/pull/3932) ([alésapine](https://github.com/alesapin)) +- Correction d'une utilisation excessive de la mémoire en cas d'insertion dans la table avec `LowCardinality` clé primaire. [\#3955](https://github.com/ClickHouse/ClickHouse/pull/3955) ([KochetovNicolai](https://github.com/KochetovNicolai)) - Fixe `LowCardinality` la sérialisation de `Native` format en cas de tableaux vides. [\#3907](https://github.com/ClickHouse/ClickHouse/issues/3907) [\#4011](https://github.com/ClickHouse/ClickHouse/pull/4011) ([KochetovNicolai](https://github.com/KochetovNicolai)) -- Correction d’un résultat incorrect lors de l’utilisation de la colonne numérique distinct by single LowCardinality. [\#3895](https://github.com/ClickHouse/ClickHouse/issues/3895) [\#4012](https://github.com/ClickHouse/ClickHouse/pull/4012) ([KochetovNicolai](https://github.com/KochetovNicolai)) +- Correction d'un résultat incorrect lors de l'utilisation de la colonne numérique distinct by single LowCardinality. [\#3895](https://github.com/ClickHouse/ClickHouse/issues/3895) [\#4012](https://github.com/ClickHouse/ClickHouse/pull/4012) ([KochetovNicolai](https://github.com/KochetovNicolai)) - Agrégation spécialisée fixe avec la clé LowCardinality (dans le cas où `compile` paramètre est activé). [\#3886](https://github.com/ClickHouse/ClickHouse/pull/3886) ([KochetovNicolai](https://github.com/KochetovNicolai)) -- Correction du transfert d’utilisateur et de mot de passe pour les requêtes de tables répliquées. [\#3957](https://github.com/ClickHouse/ClickHouse/pull/3957) ([alésapine](https://github.com/alesapin)) ([小路](https://github.com/nicelulu)) -- Correction d’une condition de course très rare qui peut se produire lors de la liste des tables dans la base de données du dictionnaire lors du rechargement des dictionnaires. [\#3970](https://github.com/ClickHouse/ClickHouse/pull/3970) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’un résultat incorrect lors de L’utilisation avec ROLLUP ou CUBE. [\#3756](https://github.com/ClickHouse/ClickHouse/issues/3756) [\#3837](https://github.com/ClickHouse/ClickHouse/pull/3837) ([Sam Chou](https://github.com/reflection)) -- Alias de colonne fixe pour la requête avec `JOIN ON` syntaxe et tables distribuées. [\#3980](https://github.com/ClickHouse/ClickHouse/pull/3980) ([L’Hiver Zhang](https://github.com/zhang2014)) -- Correction d’une erreur dans la mise en œuvre interne de `quantileTDigest` (trouvé par Artem Vakhrushev). Cette erreur ne se produit jamais dans ClickHouse et n’était pertinente que pour ceux qui utilisent directement clickhouse codebase comme bibliothèque. [\#3935](https://github.com/ClickHouse/ClickHouse/pull/3935) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction du transfert d'utilisateur et de mot de passe pour les requêtes de tables répliquées. [\#3957](https://github.com/ClickHouse/ClickHouse/pull/3957) ([alésapine](https://github.com/alesapin)) ([小路](https://github.com/nicelulu)) +- Correction d'une condition de course très rare qui peut se produire lors de la liste des tables dans la base de données du dictionnaire lors du rechargement des dictionnaires. [\#3970](https://github.com/ClickHouse/ClickHouse/pull/3970) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction d'un résultat incorrect lors de L'utilisation avec ROLLUP ou CUBE. [\#3756](https://github.com/ClickHouse/ClickHouse/issues/3756) [\#3837](https://github.com/ClickHouse/ClickHouse/pull/3837) ([Sam Chou](https://github.com/reflection)) +- Alias de colonne fixe pour la requête avec `JOIN ON` syntaxe et tables distribuées. [\#3980](https://github.com/ClickHouse/ClickHouse/pull/3980) ([L'Hiver Zhang](https://github.com/zhang2014)) +- Correction d'une erreur dans la mise en œuvre interne de `quantileTDigest` (trouvé par Artem Vakhrushev). Cette erreur ne se produit jamais dans ClickHouse et n'était pertinente que pour ceux qui utilisent directement clickhouse codebase comme bibliothèque. [\#3935](https://github.com/ClickHouse/ClickHouse/pull/3935) ([alexeï-milovidov](https://github.com/alexey-milovidov)) #### Amélioration {#improvements-6} - Soutien pour `IF NOT EXISTS` dans `ALTER TABLE ADD COLUMN` les déclarations avec `IF EXISTS` dans `DROP/MODIFY/CLEAR/COMMENT COLUMN`. [\#3900](https://github.com/ClickHouse/ClickHouse/pull/3900) ([Boris Granveaud](https://github.com/bgranvea)) - Fonction `parseDateTimeBestEffort`: prise en charge de formats `DD.MM.YYYY`, `DD.MM.YY`, `DD-MM-YYYY`, `DD-Mon-YYYY`, `DD/Month/YYYY` et similaires. [\#3922](https://github.com/ClickHouse/ClickHouse/pull/3922) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - `CapnProtoInputStream` maintenant soutenir les structures déchiquetées. [\#4063](https://github.com/ClickHouse/ClickHouse/pull/4063) ([Odin Hultgren Van Der Horst](https://github.com/Miniwoffer)) -- Amélioration de la facilité d’utilisation: ajout d’une vérification que le processus du serveur est démarré à partir du propriétaire du répertoire de données. Ne pas autoriser le démarrage du serveur à partir de la racine si les données appartiennent à un utilisateur non root. [\#3785](https://github.com/ClickHouse/ClickHouse/pull/3785) ([sergey-V-galtsev](https://github.com/sergey-v-galtsev)) -- Meilleure logique de vérification des colonnes requises lors de l’analyse des requêtes avec des jointures. [\#3930](https://github.com/ClickHouse/ClickHouse/pull/3930) ([Artem Zuikov](https://github.com/4ertus2)) -- Diminution du nombre de connexions en cas de grand nombre de tables distribuées dans un seul serveur. [\#3726](https://github.com/ClickHouse/ClickHouse/pull/3726) ([L’Hiver Zhang](https://github.com/zhang2014)) +- Amélioration de la facilité d'utilisation: ajout d'une vérification que le processus du serveur est démarré à partir du propriétaire du répertoire de données. Ne pas autoriser le démarrage du serveur à partir de la racine si les données appartiennent à un utilisateur non root. [\#3785](https://github.com/ClickHouse/ClickHouse/pull/3785) ([sergey-V-galtsev](https://github.com/sergey-v-galtsev)) +- Meilleure logique de vérification des colonnes requises lors de l'analyse des requêtes avec des jointures. [\#3930](https://github.com/ClickHouse/ClickHouse/pull/3930) ([Artem Zuikov](https://github.com/4ertus2)) +- Diminution du nombre de connexions en cas de grand nombre de tables distribuées dans un seul serveur. [\#3726](https://github.com/ClickHouse/ClickHouse/pull/3726) ([L'Hiver Zhang](https://github.com/zhang2014)) - Appuyé ligne de totaux pour `WITH TOTALS` requête pour le pilote ODBC. [\#3836](https://github.com/ClickHouse/ClickHouse/pull/3836) ([Maksim Koritckiy](https://github.com/nightweb)) -- Autorisé à utiliser `Enum`s comme entiers à l’intérieur de la fonction if. [\#3875](https://github.com/ClickHouse/ClickHouse/pull/3875) ([Ivan](https://github.com/abyss7)) +- Autorisé à utiliser `Enum`s comme entiers à l'intérieur de la fonction if. [\#3875](https://github.com/ClickHouse/ClickHouse/pull/3875) ([Ivan](https://github.com/abyss7)) - Ajouter `low_cardinality_allow_in_native_format` paramètre. Si désactivé, ne pas utiliser `LowCadrinality` type de `Native` format. [\#3879](https://github.com/ClickHouse/ClickHouse/pull/3879) ([KochetovNicolai](https://github.com/KochetovNicolai)) -- Suppression de certains objets redondants du cache des expressions compilées pour réduire l’utilisation de la mémoire. [\#4042](https://github.com/ClickHouse/ClickHouse/pull/4042) ([alésapine](https://github.com/alesapin)) +- Suppression de certains objets redondants du cache des expressions compilées pour réduire l'utilisation de la mémoire. [\#4042](https://github.com/ClickHouse/ClickHouse/pull/4042) ([alésapine](https://github.com/alesapin)) - Ajouter vérifier que `SET send_logs_level = 'value'` requête accepter la valeur appropriée. [\#3873](https://github.com/ClickHouse/ClickHouse/pull/3873) ([Sabyanin Maxim](https://github.com/s-mx)) -- Vérification de type de données fixe dans les fonctions de conversion de type. [\#3896](https://github.com/ClickHouse/ClickHouse/pull/3896) ([L’Hiver Zhang](https://github.com/zhang2014)) +- Vérification de type de données fixe dans les fonctions de conversion de type. [\#3896](https://github.com/ClickHouse/ClickHouse/pull/3896) ([L'Hiver Zhang](https://github.com/zhang2014)) #### Amélioration Des Performances {#performance-improvements-5} -- Ajouter un paramètre MergeTree `use_minimalistic_part_header_in_zookeeper`. Si cette option est activée, les tables répliquées stockent les métadonnées de partie compacte dans un znode de partie unique. Cela peut réduire considérablement la taille de l’instantané ZooKeeper (surtout si les tables ont beaucoup de colonnes). Notez qu’après avoir activé ce paramètre, vous ne pourrez pas passer à une version qui ne le supporte pas. [\#3960](https://github.com/ClickHouse/ClickHouse/pull/3960) ([Alex Zatelepin](https://github.com/ztlpn)) +- Ajouter un paramètre MergeTree `use_minimalistic_part_header_in_zookeeper`. Si cette option est activée, les tables répliquées stockent les métadonnées de partie compacte dans un znode de partie unique. Cela peut réduire considérablement la taille de l'instantané ZooKeeper (surtout si les tables ont beaucoup de colonnes). Notez qu'après avoir activé ce paramètre, vous ne pourrez pas passer à une version qui ne le supporte pas. [\#3960](https://github.com/ClickHouse/ClickHouse/pull/3960) ([Alex Zatelepin](https://github.com/ztlpn)) - Ajouter une implémentation basée sur DFA pour les fonctions `sequenceMatch` et `sequenceCount` en cas de motif ne contient pas de temps. [\#4004](https://github.com/ClickHouse/ClickHouse/pull/4004) ([Léo Ercolanelli](https://github.com/ercolanelli-leo)) - Amélioration des performances pour la sérialisation des nombres entiers. [\#3968](https://github.com/ClickHouse/ClickHouse/pull/3968) ([Amos Oiseau](https://github.com/amosbird)) - Zéro gauche padding PODArray de sorte que -1 élément est toujours valide et mis à zéro. Il est utilisé pour le calcul sans branche des décalages. [\#3920](https://github.com/ClickHouse/ClickHouse/pull/3920) ([Amos Oiseau](https://github.com/amosbird)) @@ -2040,35 +2040,35 @@ Cette version contient exactement le même ensemble de patchs 19.3.6. #### Modifications Incompatibles En Arrière {#backward-incompatible-changes-2} -- Fonctionnalité non documentée supprimée `ALTER MODIFY PRIMARY KEY` parce qu’il a été remplacé par le `ALTER MODIFY ORDER BY` commande. [\#3887](https://github.com/ClickHouse/ClickHouse/pull/3887) ([Alex Zatelepin](https://github.com/ztlpn)) +- Fonctionnalité non documentée supprimée `ALTER MODIFY PRIMARY KEY` parce qu'il a été remplacé par le `ALTER MODIFY ORDER BY` commande. [\#3887](https://github.com/ClickHouse/ClickHouse/pull/3887) ([Alex Zatelepin](https://github.com/ztlpn)) - Retiré de la fonction `shardByHash`. [\#3833](https://github.com/ClickHouse/ClickHouse/pull/3833) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Interdire l’utilisation de sous-requêtes scalaires avec le résultat de type `AggregateFunction`. [\#3865](https://github.com/ClickHouse/ClickHouse/pull/3865) ([Ivan](https://github.com/abyss7)) +- Interdire l'utilisation de sous-requêtes scalaires avec le résultat de type `AggregateFunction`. [\#3865](https://github.com/ClickHouse/ClickHouse/pull/3865) ([Ivan](https://github.com/abyss7)) #### Construire/Test/Emballage Améliorations {#buildtestingpackaging-improvements-6} - Ajout du support pour PowerPC (`ppc64le`) construire. [\#4132](https://github.com/ClickHouse/ClickHouse/pull/4132) ([Danila Kutenin](https://github.com/danlark1)) - Les tests fonctionnels avec État sont exécutés sur un ensemble de données Public disponible. [\#3969](https://github.com/ClickHouse/ClickHouse/pull/3969) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’une erreur lorsque le serveur ne peut pas démarrer avec le `bash: /usr/bin/clickhouse-extract-from-config: Operation not permitted` message dans Docker ou systemd-nspawn. [\#4136](https://github.com/ClickHouse/ClickHouse/pull/4136) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Mettre `rdkafka` bibliothèque à V1.0. 0-RC5. Utilisé cppkafka au lieu de l’interface c brute. [\#4025](https://github.com/ClickHouse/ClickHouse/pull/4025) ([Ivan](https://github.com/abyss7)) -- Mettre `mariadb-client` bibliothèque. Correction d’un des problèmes trouvés par UBSan. [\#3924](https://github.com/ClickHouse/ClickHouse/pull/3924) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction d'une erreur lorsque le serveur ne peut pas démarrer avec le `bash: /usr/bin/clickhouse-extract-from-config: Operation not permitted` message dans Docker ou systemd-nspawn. [\#4136](https://github.com/ClickHouse/ClickHouse/pull/4136) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Mettre `rdkafka` bibliothèque à V1.0. 0-RC5. Utilisé cppkafka au lieu de l'interface c brute. [\#4025](https://github.com/ClickHouse/ClickHouse/pull/4025) ([Ivan](https://github.com/abyss7)) +- Mettre `mariadb-client` bibliothèque. Correction d'un des problèmes trouvés par UBSan. [\#3924](https://github.com/ClickHouse/ClickHouse/pull/3924) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Quelques corrections pour UBSan construit. [\#3926](https://github.com/ClickHouse/ClickHouse/pull/3926) [\#3021](https://github.com/ClickHouse/ClickHouse/pull/3021) [\#3948](https://github.com/ClickHouse/ClickHouse/pull/3948) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Ajout de tests par validation avec UBSan build. -- Ajout d’exécutions par validation de PVS-Studio static analyzer. +- Ajout d'exécutions par validation de PVS-Studio static analyzer. - Correction de bugs trouvés par PVS-Studio. [\#4013](https://github.com/ClickHouse/ClickHouse/pull/4013) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Correction de problèmes de compatibilité glibc. [\#4100](https://github.com/ClickHouse/ClickHouse/pull/4100) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Déplacez les images Docker vers 18.10 et ajoutez un fichier de compatibilité pour glibc \> = 2.28 [\#3965](https://github.com/ClickHouse/ClickHouse/pull/3965) ([alésapine](https://github.com/alesapin)) -- Ajouter une variable env si l’utilisateur ne veut pas chown répertoires dans l’image Docker du serveur. [\#3967](https://github.com/ClickHouse/ClickHouse/pull/3967) ([alésapine](https://github.com/alesapin)) +- Ajouter une variable env si l'utilisateur ne veut pas chown répertoires dans l'image Docker du serveur. [\#3967](https://github.com/ClickHouse/ClickHouse/pull/3967) ([alésapine](https://github.com/alesapin)) - Activé la plupart des avertissements de `-Weverything` à clang. Permettre `-Wpedantic`. [\#3986](https://github.com/ClickHouse/ClickHouse/pull/3986) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Ajout de quelques avertissements supplémentaires disponibles uniquement dans clang 8. [\#3993](https://github.com/ClickHouse/ClickHouse/pull/3993) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Lien vers `libLLVM` plutôt que de libs LLVM individuels lors de l’utilisation de liens partagés. [\#3989](https://github.com/ClickHouse/ClickHouse/pull/3989) ([Orivej Desh](https://github.com/orivej)) +- Lien vers `libLLVM` plutôt que de libs LLVM individuels lors de l'utilisation de liens partagés. [\#3989](https://github.com/ClickHouse/ClickHouse/pull/3989) ([Orivej Desh](https://github.com/orivej)) - Ajout de variables de désinfection pour les images de test. [\#4072](https://github.com/ClickHouse/ClickHouse/pull/4072) ([alésapine](https://github.com/alesapin)) - `clickhouse-server` le paquet debian recommandera `libcap2-bin` package à utiliser `setcap` outil pour définir les capacités. Cette option est facultative. [\#4093](https://github.com/ClickHouse/ClickHouse/pull/4093) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Amélioration du temps de compilation, fixe comprend. [\#3898](https://github.com/ClickHouse/ClickHouse/pull/3898) ([proller](https://github.com/proller)) - Ajout de tests de performance pour les fonctions de hachage. [\#3918](https://github.com/ClickHouse/ClickHouse/pull/3918) ([filimonov](https://github.com/filimonov)) - Dépendances de bibliothèque cycliques fixes. [\#3958](https://github.com/ClickHouse/ClickHouse/pull/3958) ([proller](https://github.com/proller)) - Amélioration de la compilation avec une faible mémoire disponible. [\#4030](https://github.com/ClickHouse/ClickHouse/pull/4030) ([proller](https://github.com/proller)) -- Ajout d’un script de test pour reproduire la dégradation des performances dans `jemalloc`. [\#4036](https://github.com/ClickHouse/ClickHouse/pull/4036) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction de fautes d’orthographe dans les commentaires et les littéraux de chaîne sous `dbms`. [\#4122](https://github.com/ClickHouse/ClickHouse/pull/4122) ([maiha](https://github.com/maiha)) +- Ajout d'un script de test pour reproduire la dégradation des performances dans `jemalloc`. [\#4036](https://github.com/ClickHouse/ClickHouse/pull/4036) ([alexeï-milovidov](https://github.com/alexey-milovidov)) +- Correction de fautes d'orthographe dans les commentaires et les littéraux de chaîne sous `dbms`. [\#4122](https://github.com/ClickHouse/ClickHouse/pull/4122) ([maiha](https://github.com/maiha)) - Correction de fautes dans les commentaires. [\#4089](https://github.com/ClickHouse/ClickHouse/pull/4089) ([Evgenii Pravda](https://github.com/kvinty)) -## [Changelog pour 2018](https://github.com/ClickHouse/ClickHouse/blob/master/docs/en/changelog/2018.md) {#changelog-for-2018} +## [Changelog pour 2018](./2018.md#clickhouse-release-18-16) {#changelog-for-2018} diff --git a/docs/fr/whats-new/changelog/index.md b/docs/fr/whats-new/changelog/index.md index 98847c8db12..646235770a7 100644 --- a/docs/fr/whats-new/changelog/index.md +++ b/docs/fr/whats-new/changelog/index.md @@ -1,668 +1,9 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_folder_title: Changelog toc_priority: 74 toc_title: '2020' --- -## Clickhouse Version V20. 3 {#clickhouse-release-v20-3} - -### Clickhouse Version V20. 3. 4. 10, 2020-03-20 {#clickhouse-release-v20-3-4-10-2020-03-20} - -#### Bug Fix {#bug-fix} - -- Cette version contient également toutes les corrections de bugs de 20.1.8.41 -- Fixer manquant `rows_before_limit_at_least` pour les requêtes sur http (avec pipeline de processeurs). Cela corrige [\#9730](https://github.com/ClickHouse/ClickHouse/issues/9730). [\#9757](https://github.com/ClickHouse/ClickHouse/pull/9757) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) - -### Clickhouse Version V20. 3. 3. 6, 2020-03-17 {#clickhouse-release-v20-3-3-6-2020-03-17} - -#### Bug Fix {#bug-fix-1} - -- Cette version contient également toutes les corrections de bugs de 20.1.7.38 -- Correction d’un bug dans une réplication qui ne permet pas la réplication de fonctionner si l’Utilisateur a exécuté des mutations sur la version précédente. Cela corrige [\#9645](https://github.com/ClickHouse/ClickHouse/issues/9645). [\#9652](https://github.com/ClickHouse/ClickHouse/pull/9652) ([alésapine](https://github.com/alesapin)). Il rend la version 20.3 rétrocompatible à nouveau. -- Ajouter un paramètre `use_compact_format_in_distributed_parts_names` qui permet d’écrire des fichiers pour `INSERT` les requêtes en `Distributed` tableau avec un format plus compact. Cela corrige [\#9647](https://github.com/ClickHouse/ClickHouse/issues/9647). [\#9653](https://github.com/ClickHouse/ClickHouse/pull/9653) ([alésapine](https://github.com/alesapin)). Il rend la version 20.3 rétrocompatible à nouveau. - -### Clickhouse Version V20. 3. 2. 1, 2020-03-12 {#clickhouse-release-v20-3-2-1-2020-03-12} - -#### Modification Incompatible En Arrière {#backward-incompatible-change} - -- Correction du problème `file name too long` lors de l’envoi de données pour `Distributed` tables pour un grand nombre de répliques. Correction du problème que les informations d’identification de réplique étaient exposées dans le journal du serveur. Le format du nom du répertoire sur le disque a été changé en `[shard{shard_index}[_replica{replica_index}]]`. [\#8911](https://github.com/ClickHouse/ClickHouse/pull/8911) ([Mikhail Korotov](https://github.com/millb)) Après la mise à niveau vers la nouvelle version, vous ne pourrez pas rétrograder sans intervention manuelle, car l’ancienne version du serveur ne reconnaît pas le nouveau format de répertoire. Si vous souhaitez passer, vous devez renommer manuellement les répertoires correspondant à l’ancien format. Cette modification s’applique uniquement si vous avez utilisé asynchrone `INSERT`s `Distributed` table. Dans la version 20.3.3, nous allons introduire un paramètre qui vous permettra d’activer le nouveau format progressivement. -- Modification du format des entrées du journal de réplication pour les commandes de mutation. Vous devez attendre que les anciennes mutations soient traitées avant d’installer la nouvelle version. -- Implémentez un profileur de mémoire simple qui vide stacktraces vers `system.trace_log` chaque n octets au-dessus de la limite d’allocation douce [\#8765](https://github.com/ClickHouse/ClickHouse/pull/8765) ([Ivan](https://github.com/abyss7)) [\#9472](https://github.com/ClickHouse/ClickHouse/pull/9472) ([alexeï-milovidov](https://github.com/alexey-milovidov)) La colonne de `system.trace_log` a été renommé de `timer_type` de `trace_type`. Cela nécessitera des changements dans les outils d’analyse des performances et de traitement flamegraph de tiers. -- Utilisez L’id de thread du système d’exploitation partout au lieu du numéro de thread interne. Cela corrige [\#7477](https://github.com/ClickHouse/ClickHouse/issues/7477) Vieux `clickhouse-client` impossible de recevoir les journaux envoyés par le serveur lorsque le paramètre `send_logs_level` est activé, car les noms et les types des messages de journal structurés ont été modifiés. D’autre part, différentes versions de serveur peuvent envoyer des journaux avec différents types les uns aux autres. Lorsque vous n’utilisez pas l’ `send_logs_level` réglage, vous ne devez pas soin. [\#8954](https://github.com/ClickHouse/ClickHouse/pull/8954) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Supprimer `indexHint` fonction [\#9542](https://github.com/ClickHouse/ClickHouse/pull/9542) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Supprimer `findClusterIndex`, `findClusterValue` fonction. Cela corrige [\#8641](https://github.com/ClickHouse/ClickHouse/issues/8641). Si vous utilisez ces fonctions, envoyez un courriel à `clickhouse-feedback@yandex-team.com` [\#9543](https://github.com/ClickHouse/ClickHouse/pull/9543) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Maintenant, il n’est pas permis de créer des colonnes ou ajouter des colonnes avec `SELECT` sous-requête comme expression par défaut. [\#9481](https://github.com/ClickHouse/ClickHouse/pull/9481) ([alésapine](https://github.com/alesapin)) -- Exiger des alias pour les sous-requêtes dans la JOINTURE. [\#9274](https://github.com/ClickHouse/ClickHouse/pull/9274) ([Artem Zuikov](https://github.com/4ertus2)) -- Améliorer `ALTER MODIFY/ADD` les requêtes de la logique. Maintenant vous ne pouvez pas `ADD` colonne sans type, `MODIFY` l’expression par défaut ne change pas le type de colonne et `MODIFY` type ne perd pas la valeur d’expression par défaut. Fixer [\#8669](https://github.com/ClickHouse/ClickHouse/issues/8669). [\#9227](https://github.com/ClickHouse/ClickHouse/pull/9227) ([alésapine](https://github.com/alesapin)) -- Exiger que le serveur soit redémarré pour appliquer les modifications dans la configuration de journalisation. Il s’agit d’une solution de contournement temporaire pour éviter le bogue où le serveur se connecte à un fichier journal supprimé (voir [\#8696](https://github.com/ClickHouse/ClickHouse/issues/8696)). [\#8707](https://github.com/ClickHouse/ClickHouse/pull/8707) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- Paramètre `experimental_use_processors` est activé par défaut. Ce paramètre active l’utilisation du nouveau pipeline de requêtes. C’est un refactoring interne et nous n’attendons aucun changement visible. Si vous voyez des problèmes, réglez-le sur Retour à zéro. [\#8768](https://github.com/ClickHouse/ClickHouse/pull/8768) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - -#### Nouveauté {#new-feature} - -- Ajouter `Avro` et `AvroConfluent` d’entrée/sortie de formats [\#8571](https://github.com/ClickHouse/ClickHouse/pull/8571) ([Andrew Onyshchuk](https://github.com/oandrew)) [\#8957](https://github.com/ClickHouse/ClickHouse/pull/8957) ([Andrew Onyshchuk](https://github.com/oandrew)) [\#8717](https://github.com/ClickHouse/ClickHouse/pull/8717) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Mises à jour multithread et non bloquantes des clés expirées dans `cache` dictionnaires (avec autorisation facultative pour lire les anciens). [\#8303](https://github.com/ClickHouse/ClickHouse/pull/8303) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- Ajouter une requête `ALTER ... MATERIALIZE TTL`. Il exécute mutation qui force à supprimer les données expirées par TTL et recalcule les méta-informations sur TTL dans toutes les parties. [\#8775](https://github.com/ClickHouse/ClickHouse/pull/8775) ([Anton Popov](https://github.com/CurtizJ)) -- Passez de HashJoin à MergeJoin (sur le disque) si nécessaire [\#9082](https://github.com/ClickHouse/ClickHouse/pull/9082) ([Artem Zuikov](https://github.com/4ertus2)) -- Ajouter `MOVE PARTITION` commande pour `ALTER TABLE` [\#4729](https://github.com/ClickHouse/ClickHouse/issues/4729) [\#6168](https://github.com/ClickHouse/ClickHouse/pull/6168) ([Guillaume Tassery](https://github.com/YiuRULE)) -- Le rechargement de la configuration de stockage du fichier de configuration à la volée. [\#8594](https://github.com/ClickHouse/ClickHouse/pull/8594) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Autorisé à changer `storage_policy` pas moins riche. [\#8107](https://github.com/ClickHouse/ClickHouse/pull/8107) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Ajout du support pour globs / jokers pour le stockage S3 et la fonction de table. [\#8851](https://github.com/ClickHouse/ClickHouse/pull/8851) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Mettre `bitAnd`, `bitOr`, `bitXor`, `bitNot` pour `FixedString(N)` type de données. [\#9091](https://github.com/ClickHouse/ClickHouse/pull/9091) ([Guillaume Tassery](https://github.com/YiuRULE)) -- Ajout de la fonction `bitCount`. Cela corrige [\#8702](https://github.com/ClickHouse/ClickHouse/issues/8702). [\#8708](https://github.com/ClickHouse/ClickHouse/pull/8708) ([alexeï-milovidov](https://github.com/alexey-milovidov)) [\#8749](https://github.com/ClickHouse/ClickHouse/pull/8749) ([ikopylov](https://github.com/ikopylov)) -- Ajouter `generateRandom` fonction de table pour générer des lignes aléatoires avec un schéma donné. Permet de remplir une table de test arbitraire avec des données. [\#8994](https://github.com/ClickHouse/ClickHouse/pull/8994) ([Ilya Yatsishin](https://github.com/qoega)) -- `JSONEachRowFormat`: support cas particulier lorsque les objets enfermés dans un tableau de niveau supérieur. [\#8860](https://github.com/ClickHouse/ClickHouse/pull/8860) ([Kruglov Pavel](https://github.com/Avogar)) -- Il est maintenant possible de créer une colonne avec `DEFAULT` expression qui dépend d’une colonne avec défaut `ALIAS` expression. [\#9489](https://github.com/ClickHouse/ClickHouse/pull/9489) ([alésapine](https://github.com/alesapin)) -- Autoriser à spécifier `--limit` plus que la taille des données source dans `clickhouse-obfuscator`. Les données se répéteront avec différentes graines aléatoires. [\#9155](https://github.com/ClickHouse/ClickHouse/pull/9155) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Ajouter `groupArraySample` fonction (similaire à `groupArray`) avec réservoir algorithme d’échantillonnage. [\#8286](https://github.com/ClickHouse/ClickHouse/pull/8286) ([Amos Oiseau](https://github.com/amosbird)) -- Maintenant, vous pouvez surveiller la taille de la file d’attente de mise à jour dans `cache`/`complex_key_cache` dictionnaires via les métriques du système. [\#9413](https://github.com/ClickHouse/ClickHouse/pull/9413) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- Autoriser L’utilisation de CRLF comme séparateur de ligne au format de sortie CSV avec réglage `output_format_csv_crlf_end_of_line` est réglé sur 1 [\#8934](https://github.com/ClickHouse/ClickHouse/pull/8934) [\#8935](https://github.com/ClickHouse/ClickHouse/pull/8935) [\#8963](https://github.com/ClickHouse/ClickHouse/pull/8963) ([Mikhail Korotov](https://github.com/millb)) -- Mettre en œuvre plus de fonctions de la [H3](https://github.com/uber/h3) API: `h3GetBaseCell`, `h3HexAreaM2`, `h3IndexesAreNeighbors`, `h3ToChildren`, `h3ToString` et `stringToH3` [\#8938](https://github.com/ClickHouse/ClickHouse/pull/8938) ([Nico Mandery](https://github.com/nmandery)) -- Nouveau paramètre introduit: `max_parser_depth` pour contrôler la taille maximale de la pile et permettre de grandes requêtes complexes. Cela corrige [\#6681](https://github.com/ClickHouse/ClickHouse/issues/6681) et [\#7668](https://github.com/ClickHouse/ClickHouse/issues/7668). [\#8647](https://github.com/ClickHouse/ClickHouse/pull/8647) ([Maxim Smirnov](https://github.com/qMBQx8GH)) -- Ajouter un paramètre `force_optimize_skip_unused_shards` réglage sur lancer si le saut d’éclats inutilisés n’est pas possible [\#8805](https://github.com/ClickHouse/ClickHouse/pull/8805) ([Azat Khuzhin](https://github.com/azat)) -- Permet de configurer plusieurs disques / volumes pour stocker des données pour l’envoi `Distributed` moteur [\#8756](https://github.com/ClickHouse/ClickHouse/pull/8756) ([Azat Khuzhin](https://github.com/azat)) -- Politique de stockage de soutien (`` pour le stockage temporaire des données. [\#8750](https://github.com/ClickHouse/ClickHouse/pull/8750) ([Azat Khuzhin](https://github.com/azat)) -- Ajouter `X-ClickHouse-Exception-Code` En-tête HTTP défini si une exception a été levée avant l’envoi de données. Cela met en œuvre [\#4971](https://github.com/ClickHouse/ClickHouse/issues/4971). [\#8786](https://github.com/ClickHouse/ClickHouse/pull/8786) ([Mikhail Korotov](https://github.com/millb)) -- Ajout de la fonction `ifNotFinite`. C’est juste un sucre syntaxique: `ifNotFinite(x, y) = isFinite(x) ? x : y`. [\#8710](https://github.com/ClickHouse/ClickHouse/pull/8710) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Ajouter `last_successful_update_time` colonne en `system.dictionaries` table [\#9394](https://github.com/ClickHouse/ClickHouse/pull/9394) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- Ajouter `blockSerializedSize` fonction (taille sur disque sans compression) [\#8952](https://github.com/ClickHouse/ClickHouse/pull/8952) ([Azat Khuzhin](https://github.com/azat)) -- Ajouter une fonction `moduloOrZero` [\#9358](https://github.com/ClickHouse/ClickHouse/pull/9358) ([hcz](https://github.com/hczhcz)) -- Tables système ajoutées `system.zeros` et `system.zeros_mt` ainsi que les fonctions de conte `zeros()` et `zeros_mt()`. Les Tables (et les fonctions de table) contiennent une seule colonne avec le nom `zero` et le type `UInt8`. Cette colonne contient des zéros. Il est nécessaire à des fins de test comme la méthode la plus rapide pour générer de nombreuses lignes. Cela corrige [\#6604](https://github.com/ClickHouse/ClickHouse/issues/6604) [\#9593](https://github.com/ClickHouse/ClickHouse/pull/9593) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) - -#### Caractéristique Expérimentale {#experimental-feature} - -- Ajouter un nouveau format compact de pièces dans `MergeTree`-table de famille dont toutes les colonnes sont stockées dans un fichier. Il aide à augmenter les performances des inserts petits et fréquents. L’ancien format (un fichier par colonne) s’appelle maintenant wide. Le format de stockage des données est contrôlé par les paramètres `min_bytes_for_wide_part` et `min_rows_for_wide_part`. [\#8290](https://github.com/ClickHouse/ClickHouse/pull/8290) ([Anton Popov](https://github.com/CurtizJ)) -- Prise en charge du stockage S3 pour `Log`, `TinyLog` et `StripeLog` table. [\#8862](https://github.com/ClickHouse/ClickHouse/pull/8862) ([Pavel Kovalenko](https://github.com/Jokser)) - -#### Bug Fix {#bug-fix-2} - -- Correction d’espaces incohérents dans les messages de journal. [\#9322](https://github.com/ClickHouse/ClickHouse/pull/9322) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’un bug dans lequel les tableaux de tuples sans nom ont été aplatis en tant que structures imbriquées lors de la création de la table. [\#8866](https://github.com/ClickHouse/ClickHouse/pull/8866) ([achulkov2](https://github.com/achulkov2)) -- Correction du problème lorsque “Too many open files” l’erreur peut se produire s’il y a trop de fichiers correspondant glob modèle dans `File` table ou `file` table de fonction. Maintenant, les fichiers sont ouverts paresseusement. Cela corrige [\#8857](https://github.com/ClickHouse/ClickHouse/issues/8857) [\#8861](https://github.com/ClickHouse/ClickHouse/pull/8861) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- DROP table temporaire ne supprime plus que la table temporaire. [\#8907](https://github.com/ClickHouse/ClickHouse/pull/8907) ([Vitaly Baranov](https://github.com/vitlibar)) -- Supprimer la partition obsolète lorsque nous éteignons le serveur ou détacher/joindre une table. [\#8602](https://github.com/ClickHouse/ClickHouse/pull/8602) ([Guillaume Tassery](https://github.com/YiuRULE)) -- Pour savoir comment le disque par défaut calcule l’espace libre à partir de `data` répertoire. Correction du problème lorsque la quantité d’espace libre n’est pas calculée correctement si l’ `data` le répertoire est monté sur un appareil séparé (cas rare). Cela corrige [\#7441](https://github.com/ClickHouse/ClickHouse/issues/7441) [\#9257](https://github.com/ClickHouse/ClickHouse/pull/9257) ([Mikhail Korotov](https://github.com/millb)) -- Permettre virgule (croix) joindre avec IN () à l’intérieur. [\#9251](https://github.com/ClickHouse/ClickHouse/pull/9251) ([Artem Zuikov](https://github.com/4ertus2)) -- Permettre de réécrire CROSS to INNER JOIN s’il n’y a pas \[pas\] comme opérateur dans la section WHERE. [\#9229](https://github.com/ClickHouse/ClickHouse/pull/9229) ([Artem Zuikov](https://github.com/4ertus2)) -- Correction d’un résultat incorrect possible après `GROUP BY` avec le paramètre activé `distributed_aggregation_memory_efficient`. Fixer [\#9134](https://github.com/ClickHouse/ClickHouse/issues/9134). [\#9289](https://github.com/ClickHouse/ClickHouse/pull/9289) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Les clés trouvées ont été comptées comme manquées dans les métriques des dictionnaires de cache. [\#9411](https://github.com/ClickHouse/ClickHouse/pull/9411) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- Correction du protocole de réplication incompatibilité introduit dans [\#8598](https://github.com/ClickHouse/ClickHouse/issues/8598). [\#9412](https://github.com/ClickHouse/ClickHouse/pull/9412) ([alésapine](https://github.com/alesapin)) -- Condition de course fixe sur `queue_task_handle` au démarrage de `ReplicatedMergeTree` table. [\#9552](https://github.com/ClickHouse/ClickHouse/pull/9552) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Jeton `NOT` ne fonctionne pas dans `SHOW TABLES NOT LIKE` requête [\#8727](https://github.com/ClickHouse/ClickHouse/issues/8727) [\#8940](https://github.com/ClickHouse/ClickHouse/pull/8940) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Vérification de plage ajoutée à la fonction `h3EdgeLengthM`. Sans cette vérification, un débordement de tampon est possible. [\#8945](https://github.com/ClickHouse/ClickHouse/pull/8945) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’un bug dans les calculs par lots des opérations logiques ternaires sur plusieurs arguments (plus de 10). [\#8718](https://github.com/ClickHouse/ClickHouse/pull/8718) ([Alexander Kazakov](https://github.com/Akazz)) -- Correction d’une erreur D’optimisation de PREWHERE, qui pourrait conduire à des `Inconsistent number of columns got from MergeTreeRangeReader` exception. [\#9024](https://github.com/ClickHouse/ClickHouse/pull/9024) ([Anton Popov](https://github.com/CurtizJ)) -- Fix inattendu `Timeout exceeded while reading from socket` exception, qui se produit aléatoirement sur une connexion sécurisée avant le délai d’expiration réellement dépassé et lorsque query profiler est activé. Également ajouter `connect_timeout_with_failover_secure_ms` paramètres (par défaut 100 ms), qui est similaire à `connect_timeout_with_failover_ms`, mais est utilisé pour les connexions sécurisées (parce que la liaison SSL est plus lente, que la connexion TCP ordinaire) [\#9026](https://github.com/ClickHouse/ClickHouse/pull/9026) ([tavplubix](https://github.com/tavplubix)) -- Correction d’un bug avec la finalisation des mutations, lorsque la mutation peut se bloquer dans l’état avec `parts_to_do=0` et `is_done=0`. [\#9022](https://github.com/ClickHouse/ClickHouse/pull/9022) ([alésapine](https://github.com/alesapin)) -- Utilisez une nouvelle logique de jointure avec `partial_merge_join` paramètre. Il est possible de faire `ANY|ALL|SEMI LEFT` et `ALL INNER` les jointures avec `partial_merge_join=1` maintenant. [\#8932](https://github.com/ClickHouse/ClickHouse/pull/8932) ([Artem Zuikov](https://github.com/4ertus2)) -- Shard pince maintenant les paramètres obtenus de l’initiateur aux constaints de la partition au lieu de lancer une exception. Ce correctif permet d’envoyer des requêtes à un serveur avec un autre contraintes. [\#9447](https://github.com/ClickHouse/ClickHouse/pull/9447) ([Vitaly Baranov](https://github.com/vitlibar)) -- Fixe, problème de gestion de mémoire dans `MergeTreeReadPool`. [\#8791](https://github.com/ClickHouse/ClickHouse/pull/8791) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Fixer `toDecimal*OrNull()` famille de fonctions lorsqu’elle est appelée avec une chaîne `e`. Fixer [\#8312](https://github.com/ClickHouse/ClickHouse/issues/8312) [\#8764](https://github.com/ClickHouse/ClickHouse/pull/8764) ([Artem Zuikov](https://github.com/4ertus2)) -- Assurez-vous que `FORMAT Null` n’envoie pas de données au client. [\#8767](https://github.com/ClickHouse/ClickHouse/pull/8767) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- Correction d’un bug dans cet horodatage `LiveViewBlockInputStream` ne sera pas mis à jour. `LIVE VIEW` est une fonctionnalité expérimentale. [\#8644](https://github.com/ClickHouse/ClickHouse/pull/8644) ([vxider](https://github.com/Vxider)) [\#8625](https://github.com/ClickHouse/ClickHouse/pull/8625) ([vxider](https://github.com/Vxider)) -- Fixe `ALTER MODIFY TTL` mauvais comportement qui n’a pas permis de supprimer les anciennes expressions TTL. [\#8422](https://github.com/ClickHouse/ClickHouse/pull/8422) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Rapport UBSan fixe dans MergeTreeIndexSet. Cela corrige [\#9250](https://github.com/ClickHouse/ClickHouse/issues/9250) [\#9365](https://github.com/ClickHouse/ClickHouse/pull/9365) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction du comportement de `match` et `extract` fonctions lorsque haystack a zéro octets. Le comportement était mauvais quand la botte de foin était constante. Cela corrige [\#9160](https://github.com/ClickHouse/ClickHouse/issues/9160) [\#9163](https://github.com/ClickHouse/ClickHouse/pull/9163) ([alexeï-milovidov](https://github.com/alexey-milovidov)) [\#9345](https://github.com/ClickHouse/ClickHouse/pull/9345) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Évitez de lancer de destructor dans la bibliothèque Apache Avro 3rd-party. [\#9066](https://github.com/ClickHouse/ClickHouse/pull/9066) ([Andrew Onyshchuk](https://github.com/oandrew)) -- Ne commettez pas un lot interrogé à partir de `Kafka` partiellement, car il peut conduire à des trous dans les données. [\#8876](https://github.com/ClickHouse/ClickHouse/pull/8876) ([filimonov](https://github.com/filimonov)) -- Fixer `joinGet` avec les types de retour nullable. https://github.com/ClickHouse/ClickHouse/issues/8919 [\#9014](https://github.com/ClickHouse/ClickHouse/pull/9014) ([Amos Oiseau](https://github.com/amosbird)) -- Correction de l’incompatibilité des données lorsqu’elles sont compressées avec `T64` codec. [\#9016](https://github.com/ClickHouse/ClickHouse/pull/9016) ([Artem Zuikov](https://github.com/4ertus2)) Corriger les ID de type de données dans `T64` codec de compression qui conduit à une mauvaise (de)compression dans les versions affectées. [\#9033](https://github.com/ClickHouse/ClickHouse/pull/9033) ([Artem Zuikov](https://github.com/4ertus2)) -- Ajouter un paramètre `enable_early_constant_folding` et le désactiver dans certains cas, cela conduit à des erreurs. [\#9010](https://github.com/ClickHouse/ClickHouse/pull/9010) ([Artem Zuikov](https://github.com/4ertus2)) -- Fix Pushdown prédicat optimizer avec vue et activer le test [\#9011](https://github.com/ClickHouse/ClickHouse/pull/9011) ([L’Hiver Zhang](https://github.com/zhang2014)) -- Fixer erreur de segmentation dans `Merge` tables, cela peut arriver lors de la lecture de `File` stockage [\#9387](https://github.com/ClickHouse/ClickHouse/pull/9387) ([tavplubix](https://github.com/tavplubix)) -- Ajout d’une vérification de la stratégie de stockage dans `ATTACH PARTITION FROM`, `REPLACE PARTITION`, `MOVE TO TABLE`. Sinon, cela pourrait rendre les données de la partie inaccessibles après le redémarrage et empêcher ClickHouse de démarrer. [\#9383](https://github.com/ClickHouse/ClickHouse/pull/9383) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Fix modifie s’il y a TTL défini pour la table. [\#8800](https://github.com/ClickHouse/ClickHouse/pull/8800) ([Anton Popov](https://github.com/CurtizJ)) -- Correction de la condition de course qui peut se produire lorsque `SYSTEM RELOAD ALL DICTIONARIES` est exécuté pendant que certains dictionnaires sont modifiés / ajoutés / supprimés. [\#8801](https://github.com/ClickHouse/ClickHouse/pull/8801) ([Vitaly Baranov](https://github.com/vitlibar)) -- Dans les versions précédentes `Memory` le moteur de base de données utilise un chemin de données vide, de sorte que les tables sont créées dans `path` directory (e.g. `/var/lib/clickhouse/`), not in data directory of database (e.g. `/var/lib/clickhouse/db_name`). [\#8753](https://github.com/ClickHouse/ClickHouse/pull/8753) ([tavplubix](https://github.com/tavplubix)) -- Correction de messages de journal erronés sur le disque ou la stratégie par défaut manquant. [\#9530](https://github.com/ClickHouse/ClickHouse/pull/9530) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Fix not (has ()) pour l’index bloom\_filter des types de tableau. [\#9407](https://github.com/ClickHouse/ClickHouse/pull/9407) ([achimbab](https://github.com/achimbab)) -- Permettre à première colonne(s) dans un tableau avec `Log` moteur alias [\#9231](https://github.com/ClickHouse/ClickHouse/pull/9231) ([Ivan](https://github.com/abyss7)) -- Fixer l’ordre des plages pendant la lecture d’ `MergeTree` table dans un fil. Cela pourrait conduire à des exceptions `MergeTreeRangeReader` ou mauvais résultats de requête. [\#9050](https://github.com/ClickHouse/ClickHouse/pull/9050) ([Anton Popov](https://github.com/CurtizJ)) -- Faire `reinterpretAsFixedString` retourner `FixedString` plutôt `String`. [\#9052](https://github.com/ClickHouse/ClickHouse/pull/9052) ([Andrew Onyshchuk](https://github.com/oandrew)) -- Évitez les cas extrêmement rares où l’utilisateur peut se tromper message d’erreur (`Success` au lieu d’une description détaillée de l’erreur). [\#9457](https://github.com/ClickHouse/ClickHouse/pull/9457) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Ne pas planter lors de l’utilisation de `Template` format avec modèle de ligne vide. [\#8785](https://github.com/ClickHouse/ClickHouse/pull/8785) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- Les fichiers de métadonnées pour les tables système peuvent être créés au mauvais endroit [\#8653](https://github.com/ClickHouse/ClickHouse/pull/8653) ([tavplubix](https://github.com/tavplubix)) Fixer [\#8581](https://github.com/ClickHouse/ClickHouse/issues/8581). -- Correction de la course de données sur exception\_ptr dans le dictionnaire de cache [\#8303](https://github.com/ClickHouse/ClickHouse/issues/8303). [\#9379](https://github.com/ClickHouse/ClickHouse/pull/9379) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- Ne pas lancer une exception pour la requête `ATTACH TABLE IF NOT EXISTS`. Auparavant, il a été lancé si la table existe déjà, malgré le `IF NOT EXISTS` clause. [\#8967](https://github.com/ClickHouse/ClickHouse/pull/8967) ([Anton Popov](https://github.com/CurtizJ)) -- Correction manquant fermeture paren dans le message d’exception. [\#8811](https://github.com/ClickHouse/ClickHouse/pull/8811) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Éviter de message `Possible deadlock avoided` au démarrage de clickhouse-client en mode interactif. [\#9455](https://github.com/ClickHouse/ClickHouse/pull/9455) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction du problème lorsque le remplissage à la fin de la valeur codée base64 peut être mal formé. Mettre à jour la bibliothèque base64. Cela corrige [\#9491](https://github.com/ClickHouse/ClickHouse/issues/9491), proche [\#9492](https://github.com/ClickHouse/ClickHouse/issues/9492) [\#9500](https://github.com/ClickHouse/ClickHouse/pull/9500) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Empêcher la perte de données dans `Kafka` dans de rares cas, lorsque l’exception se produit après la lecture du suffixe mais avant la validation. Fixer [\#9378](https://github.com/ClickHouse/ClickHouse/issues/9378) [\#9507](https://github.com/ClickHouse/ClickHouse/pull/9507) ([filimonov](https://github.com/filimonov)) -- Correction d’une exception dans `DROP TABLE IF EXISTS` [\#8663](https://github.com/ClickHouse/ClickHouse/pull/8663) ([Nikita Vasilev](https://github.com/nikvas0)) -- Correction de plantage lorsqu’un utilisateur essaie d’ `ALTER MODIFY SETTING` pour Ancien formaté `MergeTree` famille de moteurs de table. [\#9435](https://github.com/ClickHouse/ClickHouse/pull/9435) ([alésapine](https://github.com/alesapin)) -- Prise en charge des numéros UInt64 qui ne correspondent pas à Int64 dans les fonctions liées à JSON. Mettre à jour SIMDJSON à maîtriser. Cela corrige [\#9209](https://github.com/ClickHouse/ClickHouse/issues/9209) [\#9344](https://github.com/ClickHouse/ClickHouse/pull/9344) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Exécution fixe de prédicats inversés lorsque l’index fonctionnel non strictement monotinique est utilisé. [\#9223](https://github.com/ClickHouse/ClickHouse/pull/9223) ([Alexander Kazakov](https://github.com/Akazz)) -- N’essayez pas de plier `IN` constante dans `GROUP BY` [\#8868](https://github.com/ClickHouse/ClickHouse/pull/8868) ([Amos Oiseau](https://github.com/amosbird)) -- Correction d’un bug dans `ALTER DELETE` mutations qui conduit à la corruption d’index. Cela corrige [\#9019](https://github.com/ClickHouse/ClickHouse/issues/9019) et [\#8982](https://github.com/ClickHouse/ClickHouse/issues/8982). En outre fixer des conditions de course extrêmement rares dans `ReplicatedMergeTree` `ALTER` requête. [\#9048](https://github.com/ClickHouse/ClickHouse/pull/9048) ([alésapine](https://github.com/alesapin)) -- Lorsque le réglage `compile_expressions` est activée, vous pouvez obtenir `unexpected column` dans `LLVMExecutableFunction` lorsque nous utilisons `Nullable` type [\#8910](https://github.com/ClickHouse/ClickHouse/pull/8910) ([Guillaume Tassery](https://github.com/YiuRULE)) -- Plusieurs correctifs pour `Kafka` moteur: 1) Correction des doublons qui apparaissaient pendant le rééquilibrage du groupe de consommateurs. 2) Correction rare ‘holes’ apparu lorsque les données ont été interrogées à partir de plusieurs partitions avec un sondage et validées partiellement (maintenant, nous traitons / validons toujours l’ensemble du bloc de messages interrogé). 3) corriger les vidages par taille de bloc (avant que seul le rinçage par Délai d’attente fonctionnait correctement). 4) meilleure procédure d’abonnement (avec rétroaction d’affectation). 5) Faites fonctionner les tests plus rapidement (avec des intervalles et des délais d’attente par défaut). En raison du fait que les données n’étaient pas vidées par la taille du bloc auparavant (comme il se doit selon la documentation), Ce PR peut entraîner une dégradation des performances avec les paramètres par défaut (en raison de vidages plus fréquents et plus petits qui sont moins optimaux). Si vous rencontrez le problème de performance après ce changement - veuillez augmenter `kafka_max_block_size` dans le tableau de la plus grande valeur ( par exemple `CREATE TABLE ...Engine=Kafka ... SETTINGS ... kafka_max_block_size=524288`). Fixer [\#7259](https://github.com/ClickHouse/ClickHouse/issues/7259) [\#8917](https://github.com/ClickHouse/ClickHouse/pull/8917) ([filimonov](https://github.com/filimonov)) -- Fixer `Parameter out of bound` exception dans certaines requêtes après les optimisations PREWHERE. [\#8914](https://github.com/ClickHouse/ClickHouse/pull/8914) ([Baudouin Giard](https://github.com/bgiard)) -- Correction du cas de la consistance mixte des arguments de la fonction `arrayZip`. [\#8705](https://github.com/ClickHouse/ClickHouse/pull/8705) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Lors de l’exécution de `CREATE` requête, plier les expressions constantes dans les arguments du moteur de stockage. Remplacez le nom de base de données vide par la base de données actuelle. Fixer [\#6508](https://github.com/ClickHouse/ClickHouse/issues/6508), [\#3492](https://github.com/ClickHouse/ClickHouse/issues/3492) [\#9262](https://github.com/ClickHouse/ClickHouse/pull/9262) ([tavplubix](https://github.com/tavplubix)) -- Maintenant il n’est pas possible de créer ou d’ajouter des colonnes avec des alias cycliques simples comme `a DEFAULT b, b DEFAULT a`. [\#9603](https://github.com/ClickHouse/ClickHouse/pull/9603) ([alésapine](https://github.com/alesapin)) -- Correction d’un bug avec double mouvement qui peut corrompre la partie originale. Ceci est pertinent si vous utilisez `ALTER TABLE MOVE` [\#8680](https://github.com/ClickHouse/ClickHouse/pull/8680) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Permettre `interval` identifiant pour analyser correctement sans backticks. Correction d’un problème lorsqu’une requête ne peut pas être exécutée même si le `interval` l’identifiant est entouré de backticks ou de guillemets doubles. Cela corrige [\#9124](https://github.com/ClickHouse/ClickHouse/issues/9124). [\#9142](https://github.com/ClickHouse/ClickHouse/pull/9142) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Test de fuzz fixe et comportement incorrect de `bitTestAll`/`bitTestAny` fonction. [\#9143](https://github.com/ClickHouse/ClickHouse/pull/9143) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction plantage possible/mauvais nombre de lignes dans `LIMIT n WITH TIES` quand il y a beaucoup de lignes égales à n’ème ligne. [\#9464](https://github.com/ClickHouse/ClickHouse/pull/9464) ([tavplubix](https://github.com/tavplubix)) -- Correction de mutations avec des parties écrites avec activé `insert_quorum`. [\#9463](https://github.com/ClickHouse/ClickHouse/pull/9463) ([alésapine](https://github.com/alesapin)) -- Correction de la course de données à la destruction de `Poco::HTTPServer`. Cela peut se produire lorsque le serveur est démarré et immédiatement arrêté. [\#9468](https://github.com/ClickHouse/ClickHouse/pull/9468) ([Anton Popov](https://github.com/CurtizJ)) -- Correction d’un bug dans lequel un message d’erreur trompeur a été affiché lors de l’exécution `SHOW CREATE TABLE a_table_that_does_not_exist`. [\#8899](https://github.com/ClickHouse/ClickHouse/pull/8899) ([achulkov2](https://github.com/achulkov2)) -- Fixe `Parameters are out of bound` exception dans de rares cas où nous avons une constante dans le `SELECT` clause quand nous avons un `ORDER BY` et un `LIMIT` clause. [\#8892](https://github.com/ClickHouse/ClickHouse/pull/8892) ([Guillaume Tassery](https://github.com/YiuRULE)) -- Fix finalisation des mutations, quand déjà fait mutation peut avoir le statut `is_done=0`. [\#9217](https://github.com/ClickHouse/ClickHouse/pull/9217) ([alésapine](https://github.com/alesapin)) -- Empêcher l’exécution de `ALTER ADD INDEX` pour les tables MergeTree avec une ancienne syntaxe, car cela ne fonctionne pas. [\#8822](https://github.com/ClickHouse/ClickHouse/pull/8822) ([Mikhail Korotov](https://github.com/millb)) -- Pendant le démarrage du serveur, n’ACCÉDEZ PAS à la table, qui `LIVE VIEW` dépend de, donc le serveur sera en mesure de démarrer. Également supprimer `LIVE VIEW` dépendances lors du détachement `LIVE VIEW`. `LIVE VIEW` est une fonctionnalité expérimentale. [\#8824](https://github.com/ClickHouse/ClickHouse/pull/8824) ([tavplubix](https://github.com/tavplubix)) -- Correction possible segfault dans `MergeTreeRangeReader`, lors de l’exécution `PREWHERE`. [\#9106](https://github.com/ClickHouse/ClickHouse/pull/9106) ([Anton Popov](https://github.com/CurtizJ)) -- Correction d’éventuelles sommes de contrôle non appariées avec la colonne TTL. [\#9451](https://github.com/ClickHouse/ClickHouse/pull/9451) ([Anton Popov](https://github.com/CurtizJ)) -- Correction d’un bug lorsque les pièces n’étaient pas déplacées en arrière-plan par les règles TTL dans le cas où il n’y avait qu’un seul volume. [\#8672](https://github.com/ClickHouse/ClickHouse/pull/8672) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Correction du problème `Method createColumn() is not implemented for data type Set`. Cela corrige [\#7799](https://github.com/ClickHouse/ClickHouse/issues/7799). [\#8674](https://github.com/ClickHouse/ClickHouse/pull/8674) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Maintenant, nous allons essayer de finaliser les mutations plus fréquemment. [\#9427](https://github.com/ClickHouse/ClickHouse/pull/9427) ([alésapine](https://github.com/alesapin)) -- Fixer `intDiv` par moins une constante [\#9351](https://github.com/ClickHouse/ClickHouse/pull/9351) ([hcz](https://github.com/hczhcz)) -- Correction d’une condition de course possible dans `BlockIO`. [\#9356](https://github.com/ClickHouse/ClickHouse/pull/9356) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Correction d’un bug menant à la résiliation du serveur lorsque vous essayez d’utiliser / drop `Kafka` tableau créé avec de mauvais paramètres. [\#9513](https://github.com/ClickHouse/ClickHouse/pull/9513) ([filimonov](https://github.com/filimonov)) -- Ajout d’une solution de contournement si le système d’exploitation renvoie un résultat erroné pour `timer_create` fonction. [\#8837](https://github.com/ClickHouse/ClickHouse/pull/8837) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’une erreur dans l’utilisation de `min_marks_for_seek` paramètre. Correction du message d’erreur lorsqu’il n’y a pas de clé de sharding dans la table distribuée et que nous essayons d’ignorer les fragments inutilisés. [\#8908](https://github.com/ClickHouse/ClickHouse/pull/8908) ([Azat Khuzhin](https://github.com/azat)) - -#### Amélioration {#improvement} - -- Mettre `ALTER MODIFY/DROP` requêtes au-dessus des mutations pour `ReplicatedMergeTree*` les moteurs de la famille. Maintenant `ALTERS` bloque uniquement à l’étape de mise à jour des métadonnées, et ne bloque pas après cela. [\#8701](https://github.com/ClickHouse/ClickHouse/pull/8701) ([alésapine](https://github.com/alesapin)) -- Ajouter la possibilité de réécrire CROSS aux jointures internes avec `WHERE` section contenant des noms Non qialifiés. [\#9512](https://github.com/ClickHouse/ClickHouse/pull/9512) ([Artem Zuikov](https://github.com/4ertus2)) -- Faire `SHOW TABLES` et `SHOW DATABASES` les requêtes prennent en charge le `WHERE` les expressions et les `FROM`/`IN` [\#9076](https://github.com/ClickHouse/ClickHouse/pull/9076) ([sundyli](https://github.com/sundy-li)) -- Ajout d’un paramètre `deduplicate_blocks_in_dependent_materialized_views`. [\#9070](https://github.com/ClickHouse/ClickHouse/pull/9070) ([urykhy](https://github.com/urykhy)) -- Après des changements récents Le client MySQL a commencé à imprimer des chaînes binaires en hexadécimal les rendant ainsi non lisibles ([\#9032](https://github.com/ClickHouse/ClickHouse/issues/9032)). La solution de contournement dans ClickHouse est de marquer les colonnes de chaîne comme UTF-8, ce qui n’est pas toujours le cas, mais généralement le cas. [\#9079](https://github.com/ClickHouse/ClickHouse/pull/9079) ([Yuriy Baranov](https://github.com/yurriy)) -- Ajout du support des clés String et FixedString pour `sumMap` [\#8903](https://github.com/ClickHouse/ClickHouse/pull/8903) ([Baudouin Giard](https://github.com/bgiard)) -- Clés de chaîne de soutien dans les cartes SummingMergeTree [\#8933](https://github.com/ClickHouse/ClickHouse/pull/8933) ([Baudouin Giard](https://github.com/bgiard)) -- Signal terminaison du thread au pool de threads même si le thread a lancé une exception [\#8736](https://github.com/ClickHouse/ClickHouse/pull/8736) ([Ding Xiang Fei](https://github.com/dingxiangfei2009)) -- Permettent de mettre en `query_id` dans `clickhouse-benchmark` [\#9416](https://github.com/ClickHouse/ClickHouse/pull/9416) ([Anton Popov](https://github.com/CurtizJ)) -- N’autorisez pas les expressions étranges `ALTER TABLE ... PARTITION partition` requête. Cela répond à l’ [\#7192](https://github.com/ClickHouse/ClickHouse/issues/7192) [\#8835](https://github.com/ClickHouse/ClickHouse/pull/8835) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Table `system.table_engines` fournit maintenant des informations sur le support des fonctionnalités (comme `supports_ttl` ou `supports_sort_order`). [\#8830](https://github.com/ClickHouse/ClickHouse/pull/8830) ([Max Akhmedov](https://github.com/zlobober)) -- Permettre `system.metric_log` par défaut. Il contiendra des lignes avec des valeurs de ProfileEvents, CurrentMetrics collectées avec “collect\_interval\_milliseconds” intervalle (une seconde par défaut). La table est très petite (généralement par ordre de mégaoctets) et la collecte de ces données par défaut est raisonnable. [\#9225](https://github.com/ClickHouse/ClickHouse/pull/9225) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Initialize query profiler for all threads in a group, e.g. it allows to fully profile insert-queries. Fixes [\#6964](https://github.com/ClickHouse/ClickHouse/issues/6964) [\#8874](https://github.com/ClickHouse/ClickHouse/pull/8874) ([Ivan](https://github.com/abyss7)) -- Maintenant temporaire `LIVE VIEW` est créé par `CREATE LIVE VIEW name WITH TIMEOUT [42] ...` plutôt `CREATE TEMPORARY LIVE VIEW ...` parce que la syntaxe précédente n’était pas conforme à `CREATE TEMPORARY TABLE ...` [\#9131](https://github.com/ClickHouse/ClickHouse/pull/9131) ([tavplubix](https://github.com/tavplubix)) -- Ajouter text\_log.paramètre de configuration de niveau pour limiter les entrées `system.text_log` table [\#8809](https://github.com/ClickHouse/ClickHouse/pull/8809) ([Azat Khuzhin](https://github.com/azat)) -- Permettre de mettre la partie téléchargée sur un disque / volume selon les règles TTL [\#8598](https://github.com/ClickHouse/ClickHouse/pull/8598) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Pour les dictionnaires MySQL externes, autorisez à mutualiser le pool de connexions MySQL pour “share” parmi les dictionnaires. Cette option réduit considérablement le nombre de connexions aux serveurs MySQL. [\#9409](https://github.com/ClickHouse/ClickHouse/pull/9409) ([Clément Rodriguez](https://github.com/clemrodriguez)) -- Afficher le temps d’exécution de la requête le plus proche pour les quantiles dans `clickhouse-benchmark` sortie au lieu de valeurs interpolées. Il est préférable d’afficher les valeurs qui correspondent à l’exécution de certaines requêtes. [\#8712](https://github.com/ClickHouse/ClickHouse/pull/8712) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Possibilité d’ajouter une clé et un horodatage pour le message lors de l’insertion de données dans Kafka. Fixer [\#7198](https://github.com/ClickHouse/ClickHouse/issues/7198) [\#8969](https://github.com/ClickHouse/ClickHouse/pull/8969) ([filimonov](https://github.com/filimonov)) -- Si le serveur est exécuté à partir du terminal, mettez en surbrillance le numéro de thread, l’id de requête et la priorité du journal par couleurs. Ceci permet d’améliorer la lisibilité des messages de journal corrélés pour les développeurs. [\#8961](https://github.com/ClickHouse/ClickHouse/pull/8961) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Meilleur message d’exception lors du chargement des tables pour `Ordinary` la base de données. [\#9527](https://github.com/ClickHouse/ClickHouse/pull/9527) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Mettre `arraySlice` pour les tableaux avec des états de fonction d’agrégat. Cela corrige [\#9388](https://github.com/ClickHouse/ClickHouse/issues/9388) [\#9391](https://github.com/ClickHouse/ClickHouse/pull/9391) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Autoriser les fonctions constantes et les tableaux constants à utiliser sur le côté droit de L’opérateur IN. [\#8813](https://github.com/ClickHouse/ClickHouse/pull/8813) ([Anton Popov](https://github.com/CurtizJ)) -- Si l’exception zookeeper s’est produite lors de la récupération des données du système.les répliques, l’afficher dans une colonne séparée. Cela met en œuvre [\#9137](https://github.com/ClickHouse/ClickHouse/issues/9137) [\#9138](https://github.com/ClickHouse/ClickHouse/pull/9138) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Supprimer atomiquement les parties de données MergeTree sur destroy. [\#8402](https://github.com/ClickHouse/ClickHouse/pull/8402) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Prise en charge de la sécurité au niveau des lignes pour les tables distribuées. [\#8926](https://github.com/ClickHouse/ClickHouse/pull/8926) ([Ivan](https://github.com/abyss7)) -- Now we recognize suffix (like KB, KiB…) in settings values. [\#8072](https://github.com/ClickHouse/ClickHouse/pull/8072) ([Mikhail Korotov](https://github.com/millb)) -- Empêchez la mémoire lors de la construction du résultat d’une jointure importante. [\#8637](https://github.com/ClickHouse/ClickHouse/pull/8637) ([Artem Zuikov](https://github.com/4ertus2)) -- Ajout de noms de clusters aux suggestions en mode interactif dans `clickhouse-client`. [\#8709](https://github.com/ClickHouse/ClickHouse/pull/8709) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Initialize query profiler for all threads in a group, e.g. it allows to fully profile insert-queries [\#8820](https://github.com/ClickHouse/ClickHouse/pull/8820) ([Ivan](https://github.com/abyss7)) -- Ajout de la colonne `exception_code` dans `system.query_log` table. [\#8770](https://github.com/ClickHouse/ClickHouse/pull/8770) ([Mikhail Korotov](https://github.com/millb)) -- Serveur de compatibilité MySQL activé sur le port `9004` par défaut dans le fichier de configuration du serveur. Fixe génération de mot de passe commande dans l’exemple de configuration. [\#8771](https://github.com/ClickHouse/ClickHouse/pull/8771) ([Yuriy Baranov](https://github.com/yurriy)) -- Empêcher l’abandon à l’arrêt si le système de fichiers est en lecture seule. Cela corrige [\#9094](https://github.com/ClickHouse/ClickHouse/issues/9094) [\#9100](https://github.com/ClickHouse/ClickHouse/pull/9100) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Meilleur message d’exception lorsque la longueur est requise dans la requête HTTP POST. [\#9453](https://github.com/ClickHouse/ClickHouse/pull/9453) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Ajouter `_path` et `_file` les colonnes virtuelles à `HDFS` et `File` les moteurs et les `hdfs` et `file` les fonctions de table [\#8489](https://github.com/ClickHouse/ClickHouse/pull/8489) ([Olga Khvostikova](https://github.com/stavrolia)) -- Correction d’erreur `Cannot find column` lors de l’insertion dans `MATERIALIZED VIEW` dans le cas où une nouvelle colonne a été ajoutée à la table interne de la vue. [\#8766](https://github.com/ClickHouse/ClickHouse/pull/8766) [\#8788](https://github.com/ClickHouse/ClickHouse/pull/8788) ([vzakaznikov](https://github.com/vzakaznikov)) [\#8788](https://github.com/ClickHouse/ClickHouse/issues/8788) [\#8806](https://github.com/ClickHouse/ClickHouse/pull/8806) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) [\#8803](https://github.com/ClickHouse/ClickHouse/pull/8803) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Correction de la progression sur le protocole client-serveur natif, en envoyant la progression après la mise à jour finale (comme les journaux). Cela peut être pertinent uniquement pour certains outils tiers qui utilisent le protocole natif. [\#9495](https://github.com/ClickHouse/ClickHouse/pull/9495) ([Azat Khuzhin](https://github.com/azat)) -- Ajouter une métrique système de suivi du nombre de connexions client à L’aide du protocole MySQL ([\#9013](https://github.com/ClickHouse/ClickHouse/issues/9013)). [\#9015](https://github.com/ClickHouse/ClickHouse/pull/9015) ([Eugene Klimov](https://github.com/Slach)) -- A partir de Maintenant, les réponses HTTP auront `X-ClickHouse-Timezone` en-tête défini sur la même valeur de fuseau horaire que `SELECT timezone()` serait-rapport. [\#9493](https://github.com/ClickHouse/ClickHouse/pull/9493) ([Denis Glazachev](https://github.com/traceon)) - -#### Amélioration Des Performances {#performance-improvement} - -- Améliorer les performances de l’analyse de l’indice DANS [\#9261](https://github.com/ClickHouse/ClickHouse/pull/9261) ([Anton Popov](https://github.com/CurtizJ)) -- Code plus simple et plus efficace dans les fonctions logiques + nettoyage de code. Un suivi à [\#8718](https://github.com/ClickHouse/ClickHouse/issues/8718) [\#8728](https://github.com/ClickHouse/ClickHouse/pull/8728) ([Alexander Kazakov](https://github.com/Akazz)) -- Amélioration globale de la performance (de l’ordre de 5%..200% pour les requêtes affectées) en assurant un aliasing encore plus strict avec les fonctionnalités c++20. [\#9304](https://github.com/ClickHouse/ClickHouse/pull/9304) ([Amos Oiseau](https://github.com/amosbird)) -- Aliasing plus strict pour les boucles internes des fonctions de comparaison. [\#9327](https://github.com/ClickHouse/ClickHouse/pull/9327) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Aliasing plus strict pour les boucles internes des fonctions arithmétiques. [\#9325](https://github.com/ClickHouse/ClickHouse/pull/9325) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Une implémentation ~3 fois plus rapide pour ColumnVector::replicate (), via laquelle ColumnConst:: convertToFullColumn () est implémentée. Sera également utile dans les tests lors de la matérialisation des constantes. [\#9293](https://github.com/ClickHouse/ClickHouse/pull/9293) ([Alexander Kazakov](https://github.com/Akazz)) -- Une autre amélioration mineure des performances à `ColumnVector::replicate()` (cela accélère le `materialize` fonction et des fonctions d’ordre supérieur) une amélioration encore plus [\#9293](https://github.com/ClickHouse/ClickHouse/issues/9293) [\#9442](https://github.com/ClickHouse/ClickHouse/pull/9442) ([Alexander Kazakov](https://github.com/Akazz)) -- Amélioration des performances de `stochasticLinearRegression` fonction d’agrégation. Ce patch est fourni par Intel. [\#8652](https://github.com/ClickHouse/ClickHouse/pull/8652) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Améliorer les performances de `reinterpretAsFixedString` fonction. [\#9342](https://github.com/ClickHouse/ClickHouse/pull/9342) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- N’envoyez pas de blocs au client pour `Null` format dans le pipeline de processeurs. [\#8797](https://github.com/ClickHouse/ClickHouse/pull/8797) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) [\#8767](https://github.com/ClickHouse/ClickHouse/pull/8767) ([Alexander Kuzmenkov](https://github.com/akuzm)) - -#### Construction / Test / Amélioration De L’Emballage {#buildtestingpackaging-improvement} - -- La gestion des exceptions fonctionne maintenant correctement sur le sous-système Windows Pour Linux. Tu vois https://github.com/ClickHouse-Extras/libunwind/pull/3 cela corrige [\#6480](https://github.com/ClickHouse/ClickHouse/issues/6480) [\#9564](https://github.com/ClickHouse/ClickHouse/pull/9564) ([sobolevsv](https://github.com/sobolevsv)) -- Remplacer `readline` avec `replxx` interactif, l’édition en ligne en `clickhouse-client` [\#8416](https://github.com/ClickHouse/ClickHouse/pull/8416) ([Ivan](https://github.com/abyss7)) -- Meilleur temps de construction et moins d’instanciations de modèle dans FunctionsComparison. [\#9324](https://github.com/ClickHouse/ClickHouse/pull/9324) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Intégration ajoutée avec `clang-tidy` in CI. Voir aussi [\#6044](https://github.com/ClickHouse/ClickHouse/issues/6044) [\#9566](https://github.com/ClickHouse/ClickHouse/pull/9566) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Maintenant, nous lions ClickHouse dans CI en utilisant `lld` même pour `gcc`. [\#9049](https://github.com/ClickHouse/ClickHouse/pull/9049) ([alésapine](https://github.com/alesapin)) -- Permet de randomiser la planification des threads et d’insérer des problèmes lorsque `THREAD_FUZZER_*` variables d’environnement sont définies. Cela aide les tests. [\#9459](https://github.com/ClickHouse/ClickHouse/pull/9459) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Activer les sockets sécurisés dans les tests sans état [\#9288](https://github.com/ClickHouse/ClickHouse/pull/9288) ([tavplubix](https://github.com/tavplubix)) -- Rendre SPLIT\_SHARED\_LIBRARIES = OFF plus robuste [\#9156](https://github.com/ClickHouse/ClickHouse/pull/9156) ([Azat Khuzhin](https://github.com/azat)) -- Faire “performance\_introspection\_and\_logging” test fiable au serveur aléatoire bloqué. Cela peut se produire dans L’environnement CI. Voir aussi [\#9515](https://github.com/ClickHouse/ClickHouse/issues/9515) [\#9528](https://github.com/ClickHouse/ClickHouse/pull/9528) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Valider XML dans la vérification de style. [\#9550](https://github.com/ClickHouse/ClickHouse/pull/9550) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Condition de course fixe dans l’essai `00738_lock_for_inner_table`. Ce test reposait sur le sommeil. [\#9555](https://github.com/ClickHouse/ClickHouse/pull/9555) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Supprimer les tests de performance de type `once`. Ceci est nécessaire pour exécuter tous les tests de performance en mode de comparaison statistique (plus fiable). [\#9557](https://github.com/ClickHouse/ClickHouse/pull/9557) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Ajout d’un test de performance pour les fonctions arithmétiques. [\#9326](https://github.com/ClickHouse/ClickHouse/pull/9326) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Ajouté test de performance pour `sumMap` et `sumMapWithOverflow` les fonctions d’agrégation. Pour le suivi de la [\#8933](https://github.com/ClickHouse/ClickHouse/issues/8933) [\#8947](https://github.com/ClickHouse/ClickHouse/pull/8947) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Assurez le style des codes D’erreur en vérifiant le style. [\#9370](https://github.com/ClickHouse/ClickHouse/pull/9370) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Ajouter un script pour l’historique des tests. [\#8796](https://github.com/ClickHouse/ClickHouse/pull/8796) ([alésapine](https://github.com/alesapin)) -- Ajouter un avertissement GCC `-Wsuggest-override` pour localiser et réparer tous les endroits où `override` mot-clé doit être utilisé. [\#8760](https://github.com/ClickHouse/ClickHouse/pull/8760) ([kreuzerkrieg](https://github.com/kreuzerkrieg)) -- Ignorer le symbole faible sous Mac OS X car il doit être défini [\#9538](https://github.com/ClickHouse/ClickHouse/pull/9538) ([Utilisateur supprimé](https://github.com/ghost)) -- Normaliser le temps d’exécution de certaines requêtes dans les tests de performance. Ceci est fait en préparation pour exécuter tous les tests de performance en mode comparaison. [\#9565](https://github.com/ClickHouse/ClickHouse/pull/9565) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction de certains tests pour prendre en charge pytest avec des tests de requête [\#9062](https://github.com/ClickHouse/ClickHouse/pull/9062) ([Ivan](https://github.com/abyss7)) -- Activez SSL dans build avec MSan, afin que le serveur n’échoue pas au démarrage lors de l’exécution de tests sans état [\#9531](https://github.com/ClickHouse/ClickHouse/pull/9531) ([tavplubix](https://github.com/tavplubix)) -- Correction de la substitution de base de données dans les résultats des tests [\#9384](https://github.com/ClickHouse/ClickHouse/pull/9384) ([Ilya Yatsishin](https://github.com/qoega)) -- Construire des correctifs pour diverses plates-formes [\#9381](https://github.com/ClickHouse/ClickHouse/pull/9381) ([proller](https://github.com/proller)) [\#8755](https://github.com/ClickHouse/ClickHouse/pull/8755) ([proller](https://github.com/proller)) [\#8631](https://github.com/ClickHouse/ClickHouse/pull/8631) ([proller](https://github.com/proller)) -- Ajout de la section disques à l’image Docker test stateless-with-coverage [\#9213](https://github.com/ClickHouse/ClickHouse/pull/9213) ([Pavel Kovalenko](https://github.com/Jokser)) -- Débarrassez-vous des fichiers in-source-tree lors de la construction avec GRPC [\#9588](https://github.com/ClickHouse/ClickHouse/pull/9588) ([Amos Oiseau](https://github.com/amosbird)) -- Temps de construction légèrement plus rapide en supprimant SessionCleaner du contexte. Rendre le code de SessionCleaner plus simple. [\#9232](https://github.com/ClickHouse/ClickHouse/pull/9232) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Mise à jour de la vérification des requêtes suspendues dans le script clickhouse-test [\#8858](https://github.com/ClickHouse/ClickHouse/pull/8858) ([Alexander Kazakov](https://github.com/Akazz)) -- Suppression de certains fichiers inutiles du référentiel. [\#8843](https://github.com/ClickHouse/ClickHouse/pull/8843) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Changement de type de math perftests de `once` de `loop`. [\#8783](https://github.com/ClickHouse/ClickHouse/pull/8783) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Ajouter une image docker qui permet de créer un rapport HTML interactif du navigateur de code pour notre base de code. [\#8781](https://github.com/ClickHouse/ClickHouse/pull/8781) ([alésapine](https://github.com/alesapin)) Voir [Navigateur De Code Woboq](https://clickhouse.tech/codebrowser/html_report///ClickHouse/src/src/index.html) -- Supprimer certains échecs de test sous MSan. [\#8780](https://github.com/ClickHouse/ClickHouse/pull/8780) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- SpeedUp “exception while insert” test. Ce test expire souvent dans la construction debug-with-coverage. [\#8711](https://github.com/ClickHouse/ClickHouse/pull/8711) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Mettre `libcxx` et `libcxxabi` maîtriser. En préparation à [\#9304](https://github.com/ClickHouse/ClickHouse/issues/9304) [\#9308](https://github.com/ClickHouse/ClickHouse/pull/9308) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction du test flacky `00910_zookeeper_test_alter_compression_codecs`. [\#9525](https://github.com/ClickHouse/ClickHouse/pull/9525) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Nettoyer les drapeaux de l’éditeur de liens dupliqués. Assurez-vous que l’éditeur de liens ne pas rechercher un symbole inattendu. [\#9433](https://github.com/ClickHouse/ClickHouse/pull/9433) ([Amos Oiseau](https://github.com/amosbird)) -- Ajouter `clickhouse-odbc` pilote dans les images de test. Cela permet de tester l’interaction de ClickHouse avec ClickHouse via son propre pilote ODBC. [\#9348](https://github.com/ClickHouse/ClickHouse/pull/9348) ([filimonov](https://github.com/filimonov)) -- Correction de plusieurs bugs dans les tests unitaires. [\#9047](https://github.com/ClickHouse/ClickHouse/pull/9047) ([alésapine](https://github.com/alesapin)) -- Permettre `-Wmissing-include-dirs` Avertissement GCC pour éliminer toutes les inclusions non existantes-principalement à la suite D’erreurs de script CMake [\#8704](https://github.com/ClickHouse/ClickHouse/pull/8704) ([kreuzerkrieg](https://github.com/kreuzerkrieg)) -- Décrivez les raisons si query profiler ne peut pas fonctionner. C’est prévu pour [\#9049](https://github.com/ClickHouse/ClickHouse/issues/9049) [\#9144](https://github.com/ClickHouse/ClickHouse/pull/9144) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Mettre à jour OpenSSL vers le maître en amont. Correction du problème lorsque les connexions TLS peuvent échouer avec le message `OpenSSL SSL_read: error:14094438:SSL routines:ssl3_read_bytes:tlsv1 alert internal error` et `SSL Exception: error:2400006E:random number generator::error retrieving entropy`. Le problème était présent dans la version 20.1. [\#8956](https://github.com/ClickHouse/ClickHouse/pull/8956) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Mettre à jour Dockerfile pour le serveur [\#8893](https://github.com/ClickHouse/ClickHouse/pull/8893) ([Ilya Mazaev](https://github.com/ne-ray)) -- Corrections mineures dans le script build-gcc-from-sources [\#8774](https://github.com/ClickHouse/ClickHouse/pull/8774) ([Michael Nacharov](https://github.com/mnach)) -- Remplacer `numbers` de `zeros` dans perftests où `number` la colonne n’est pas utilisée. Cela conduira à des résultats de test plus propres. [\#9600](https://github.com/ClickHouse/ClickHouse/pull/9600) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Correction d’un problème de débordement de pile lors de l’utilisation de initializer\_list dans les constructeurs de colonnes. [\#9367](https://github.com/ClickHouse/ClickHouse/pull/9367) ([Utilisateur supprimé](https://github.com/ghost)) -- Mise à niveau librdkafka à v1. 3. 0. Activer groupé `rdkafka` et `gsasl` bibliothèques sous Mac OS X. [\#9000](https://github.com/ClickHouse/ClickHouse/pull/9000) ([Andrew Onyshchuk](https://github.com/oandrew)) -- correction de construction sur GCC 9.2.0 [\#9306](https://github.com/ClickHouse/ClickHouse/pull/9306) ([vxider](https://github.com/Vxider)) - -## Clickhouse Version V20. 1 {#clickhouse-release-v20-1} - -### Clickhouse Version V20. 1. 8. 41, 2020-03-20 {#clickhouse-release-v20-1-8-41-2020-03-20} - -#### Bug Fix {#bug-fix-3} - -- Correction possible permanente `Cannot schedule a task` erreur (due à une exception non gérée dans `ParallelAggregatingBlockInputStream::Handler::onFinish/onFinishThread`). Cela corrige [\#6833](https://github.com/ClickHouse/ClickHouse/issues/6833). [\#9154](https://github.com/ClickHouse/ClickHouse/pull/9154) ([Azat Khuzhin](https://github.com/azat)) -- Correction de la consommation excessive de mémoire dans `ALTER` les requêtes (mutations). Cela corrige [\#9533](https://github.com/ClickHouse/ClickHouse/issues/9533) et [\#9670](https://github.com/ClickHouse/ClickHouse/issues/9670). [\#9754](https://github.com/ClickHouse/ClickHouse/pull/9754) ([alésapine](https://github.com/alesapin)) -- Correction d’un bug dans backquoting dans les dictionnaires externes DDL. Cela corrige [\#9619](https://github.com/ClickHouse/ClickHouse/issues/9619). [\#9734](https://github.com/ClickHouse/ClickHouse/pull/9734) ([alésapine](https://github.com/alesapin)) - -### Clickhouse Version V20. 1. 7. 38, 2020-03-18 {#clickhouse-release-v20-1-7-38-2020-03-18} - -#### Bug Fix {#bug-fix-4} - -- Correction de noms de fonctions internes incorrects pour `sumKahan` et `sumWithOverflow`. Je mène à l’exception en utilisant ces fonctions dans les requêtes distantes. [\#9636](https://github.com/ClickHouse/ClickHouse/pull/9636) ([Azat Khuzhin](https://github.com/azat)). Ce problème était dans toutes les versions de ClickHouse. -- Permettre `ALTER ON CLUSTER` de `Distributed` tables avec réplication interne. Cela corrige [\#3268](https://github.com/ClickHouse/ClickHouse/issues/3268). [\#9617](https://github.com/ClickHouse/ClickHouse/pull/9617) ([shinoi2](https://github.com/shinoi2)). Ce problème était dans toutes les versions de ClickHouse. -- Corriger les exceptions possibles `Size of filter doesn't match size of column` et `Invalid number of rows in Chunk` dans `MergeTreeRangeReader`. Ils pouvaient apparaître lors de l’exécution `PREWHERE` dans certains cas. Fixer [\#9132](https://github.com/ClickHouse/ClickHouse/issues/9132). [\#9612](https://github.com/ClickHouse/ClickHouse/pull/9612) ([Anton Popov](https://github.com/CurtizJ)) -- Correction du problème: le fuseau horaire n’a pas été conservé si vous écrivez une expression arithmétique simple comme `time + 1` (contrairement à une expression comme `time + INTERVAL 1 SECOND`). Cela corrige [\#5743](https://github.com/ClickHouse/ClickHouse/issues/5743). [\#9323](https://github.com/ClickHouse/ClickHouse/pull/9323) ([alexeï-milovidov](https://github.com/alexey-milovidov)). Ce problème était dans toutes les versions de ClickHouse. -- Maintenant il n’est pas possible de créer ou d’ajouter des colonnes avec des alias cycliques simples comme `a DEFAULT b, b DEFAULT a`. [\#9603](https://github.com/ClickHouse/ClickHouse/pull/9603) ([alésapine](https://github.com/alesapin)) -- Correction du problème lorsque le remplissage à la fin de la valeur codée base64 peut être mal formé. Mettre à jour la bibliothèque base64. Cela corrige [\#9491](https://github.com/ClickHouse/ClickHouse/issues/9491), proche [\#9492](https://github.com/ClickHouse/ClickHouse/issues/9492) [\#9500](https://github.com/ClickHouse/ClickHouse/pull/9500) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction de la course de données à la destruction de `Poco::HTTPServer`. Cela peut se produire lorsque le serveur est démarré et immédiatement arrêté. [\#9468](https://github.com/ClickHouse/ClickHouse/pull/9468) ([Anton Popov](https://github.com/CurtizJ)) -- Correction plantage possible/mauvais nombre de lignes dans `LIMIT n WITH TIES` quand il y a beaucoup de lignes égales à n’ème ligne. [\#9464](https://github.com/ClickHouse/ClickHouse/pull/9464) ([tavplubix](https://github.com/tavplubix)) -- Correction d’éventuelles sommes de contrôle non appariées avec la colonne TTL. [\#9451](https://github.com/ClickHouse/ClickHouse/pull/9451) ([Anton Popov](https://github.com/CurtizJ)) -- Correction de plantage lorsqu’un utilisateur essaie d’ `ALTER MODIFY SETTING` pour Ancien formaté `MergeTree` famille de moteurs de table. [\#9435](https://github.com/ClickHouse/ClickHouse/pull/9435) ([alésapine](https://github.com/alesapin)) -- Maintenant, nous allons essayer de finaliser les mutations plus fréquemment. [\#9427](https://github.com/ClickHouse/ClickHouse/pull/9427) ([alésapine](https://github.com/alesapin)) -- Correction du protocole de réplication incompatibilité introduit dans [\#8598](https://github.com/ClickHouse/ClickHouse/issues/8598). [\#9412](https://github.com/ClickHouse/ClickHouse/pull/9412) ([alésapine](https://github.com/alesapin)) -- Fix not (has ()) pour l’index bloom\_filter des types de tableau. [\#9407](https://github.com/ClickHouse/ClickHouse/pull/9407) ([achimbab](https://github.com/achimbab)) -- Correction du comportement de `match` et `extract` fonctions lorsque haystack a zéro octets. Le comportement était mauvais quand la botte de foin était constante. Cela corrige [\#9160](https://github.com/ClickHouse/ClickHouse/issues/9160) [\#9163](https://github.com/ClickHouse/ClickHouse/pull/9163) ([alexeï-milovidov](https://github.com/alexey-milovidov)) [\#9345](https://github.com/ClickHouse/ClickHouse/pull/9345) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - -#### Construction / Test / Amélioration De L’Emballage {#buildtestingpackaging-improvement-1} - -- La gestion des exceptions fonctionne maintenant correctement sur le sous-système Windows Pour Linux. Tu vois https://github.com/ClickHouse-Extras/libunwind/pull/3 cela corrige [\#6480](https://github.com/ClickHouse/ClickHouse/issues/6480) [\#9564](https://github.com/ClickHouse/ClickHouse/pull/9564) ([sobolevsv](https://github.com/sobolevsv)) - -### Clickhouse Version V20. 1. 6. 30, 2020-03-05 {#clickhouse-release-v20-1-6-30-2020-03-05} - -#### Bug Fix {#bug-fix-5} - -- Correction de l’incompatibilité des données lorsqu’elles sont compressées avec `T64` codec. - [\#9039](https://github.com/ClickHouse/ClickHouse/pull/9039) [(abyss7)](https://github.com/abyss7) -- Correction de l’ordre des plages lors de la lecture de la table MergeTree dans un thread. Fixer [\#8964](https://github.com/ClickHouse/ClickHouse/issues/8964). - [\#9050](https://github.com/ClickHouse/ClickHouse/pull/9050) [(CurtizJ)](https://github.com/CurtizJ) -- Correction possible segfault dans `MergeTreeRangeReader`, lors de l’exécution `PREWHERE`. Fixer [\#9064](https://github.com/ClickHouse/ClickHouse/issues/9064). - [\#9106](https://github.com/ClickHouse/ClickHouse/pull/9106) [(CurtizJ)](https://github.com/CurtizJ) -- Fixer `reinterpretAsFixedString` retourner `FixedString` plutôt `String`. - [\#9052](https://github.com/ClickHouse/ClickHouse/pull/9052) [(oandrew)](https://github.com/oandrew) -- Fixer `joinGet` avec les types de retour nullable. Fixer [\#8919](https://github.com/ClickHouse/ClickHouse/issues/8919) - [\#9014](https://github.com/ClickHouse/ClickHouse/pull/9014) [(amosbird)](https://github.com/amosbird) -- Correction du test fuzz et du comportement incorrect des fonctions bitTestAll/bitTestAny. - [\#9143](https://github.com/ClickHouse/ClickHouse/pull/9143) [(alexey-milovidov)](https://github.com/alexey-milovidov) -- Corrigez le comportement des fonctions match et extract lorsque haystack a zéro octet. Le comportement était mauvais quand la botte de foin était constante. Fixer [\#9160](https://github.com/ClickHouse/ClickHouse/issues/9160) - [\#9163](https://github.com/ClickHouse/ClickHouse/pull/9163) [(alexey-milovidov)](https://github.com/alexey-milovidov) -- Exécution fixe de prédicats inversés lorsque l’index fonctionnel non strictement monotinique est utilisé. Fixer [\#9034](https://github.com/ClickHouse/ClickHouse/issues/9034) - [\#9223](https://github.com/ClickHouse/ClickHouse/pull/9223) [(Akazz)](https://github.com/Akazz) -- Permettre à réécrire `CROSS` de `INNER JOIN` si il y a `[NOT] LIKE` opérateur `WHERE` section. Fixer [\#9191](https://github.com/ClickHouse/ClickHouse/issues/9191) - [\#9229](https://github.com/ClickHouse/ClickHouse/pull/9229) [(4ertus2)](https://github.com/4ertus2) -- Autoriser la(Les) première (s) colonne (s) dans une table avec Log engine à être un alias. - [\#9231](https://github.com/ClickHouse/ClickHouse/pull/9231) [(abyss7)](https://github.com/abyss7) -- Autoriser la virgule rejoindre `IN()` à l’intérieur. Fixer [\#7314](https://github.com/ClickHouse/ClickHouse/issues/7314). - [\#9251](https://github.com/ClickHouse/ClickHouse/pull/9251) [(4ertus2)](https://github.com/4ertus2) -- Améliorer `ALTER MODIFY/ADD` les requêtes de la logique. Maintenant vous ne pouvez pas `ADD` colonne sans type, `MODIFY` l’expression par défaut ne change pas le type de colonne et `MODIFY` type ne perd pas la valeur d’expression par défaut. Fixer [\#8669](https://github.com/ClickHouse/ClickHouse/issues/8669). - [\#9227](https://github.com/ClickHouse/ClickHouse/pull/9227) [(alesapin)](https://github.com/alesapin) -- Fix finalisation des mutations, quand déjà fait mutation peut avoir le statut is\_done = 0. - [\#9217](https://github.com/ClickHouse/ClickHouse/pull/9217) [(alesapin)](https://github.com/alesapin) -- Soutien “Processors” pipeline pour le système.nombres et système.numbers\_mt. Cela corrige également le bug lorsque `max_execution_time` n’est pas respectée. - [\#7796](https://github.com/ClickHouse/ClickHouse/pull/7796) [(KochetovNicolai)](https://github.com/KochetovNicolai) -- Correction d’un mauvais comptage de `DictCacheKeysRequestedFound` métrique. - [\#9411](https://github.com/ClickHouse/ClickHouse/pull/9411) [(nikitamikhaylov)](https://github.com/nikitamikhaylov) -- Ajout d’une vérification de la stratégie de stockage dans `ATTACH PARTITION FROM`, `REPLACE PARTITION`, `MOVE TO TABLE` ce qui pourrait autrement rendre les données de la partie inaccessibles après le redémarrage et empêcher ClickHouse de démarrer. - [\#9383](https://github.com/ClickHouse/ClickHouse/pull/9383) [(excitoon)](https://github.com/excitoon) -- Rapport UBSan fixe dans `MergeTreeIndexSet`. Cela corrige [\#9250](https://github.com/ClickHouse/ClickHouse/issues/9250) - [\#9365](https://github.com/ClickHouse/ClickHouse/pull/9365) [(alexey-milovidov)](https://github.com/alexey-milovidov) -- Correction possible datarace dans BlockIO. - [\#9356](https://github.com/ClickHouse/ClickHouse/pull/9356) [(KochetovNicolai)](https://github.com/KochetovNicolai) -- Soutien pour `UInt64` nombres qui ne correspondent pas à Int64 dans les fonctions liées à JSON. Mettre `SIMDJSON` maîtriser. Cela corrige [\#9209](https://github.com/ClickHouse/ClickHouse/issues/9209) - [\#9344](https://github.com/ClickHouse/ClickHouse/pull/9344) [(alexey-milovidov)](https://github.com/alexey-milovidov) -- Résoudre le problème lorsque la quantité d’espace libre n’est pas calculée correctement si le répertoire de données est monté sur un appareil séparé. Pour le disque par défaut calculer l’espace libre à partir du sous-répertoire de données. Cela corrige [\#7441](https://github.com/ClickHouse/ClickHouse/issues/7441) - [\#9257](https://github.com/ClickHouse/ClickHouse/pull/9257) [(millb)](https://github.com/millb) -- Corrigez le problème lorsque les connexions TLS peuvent échouer avec le message `OpenSSL SSL_read: error:14094438:SSL routines:ssl3_read_bytes:tlsv1 alert internal error and SSL Exception: error:2400006E:random number generator::error retrieving entropy.` Mettre à jour OpenSSL vers le maître en amont. - [\#8956](https://github.com/ClickHouse/ClickHouse/pull/8956) [(alexey-milovidov)](https://github.com/alexey-milovidov) -- Lors de l’exécution de `CREATE` requête, plier les expressions constantes dans les arguments du moteur de stockage. Remplacez le nom de base de données vide par la base de données actuelle. Fixer [\#6508](https://github.com/ClickHouse/ClickHouse/issues/6508), [\#3492](https://github.com/ClickHouse/ClickHouse/issues/3492). Corrigez également la vérification de l’adresse locale dans ClickHouseDictionarySource. - [\#9262](https://github.com/ClickHouse/ClickHouse/pull/9262) [(tabplubix)](https://github.com/tavplubix) -- Fixer erreur de segmentation dans `StorageMerge`, ce qui peut arriver lors de la lecture de StorageFile. - [\#9387](https://github.com/ClickHouse/ClickHouse/pull/9387) [(tabplubix)](https://github.com/tavplubix) -- Empêcher la perte de données dans `Kafka` dans de rares cas, lorsque l’exception se produit après la lecture du suffixe mais avant la validation. Fixer [\#9378](https://github.com/ClickHouse/ClickHouse/issues/9378). Concerner: [\#7175](https://github.com/ClickHouse/ClickHouse/issues/7175) - [\#9507](https://github.com/ClickHouse/ClickHouse/pull/9507) [(filimonov)](https://github.com/filimonov) -- Correction d’un bug menant à la résiliation du serveur lorsque vous essayez d’utiliser / drop `Kafka` tableau créé avec de mauvais paramètres. Fixer [\#9494](https://github.com/ClickHouse/ClickHouse/issues/9494). Incorporer [\#9507](https://github.com/ClickHouse/ClickHouse/issues/9507). - [\#9513](https://github.com/ClickHouse/ClickHouse/pull/9513) [(filimonov)](https://github.com/filimonov) - -#### Nouveauté {#new-feature-1} - -- Ajouter `deduplicate_blocks_in_dependent_materialized_views` option pour contrôler le comportement des insertions idempotent dans des tables avec des vues matérialisées. Cette nouvelle fonctionnalité a été ajoutée à la version de bugfix par une demande spéciale D’Altinity. - [\#9070](https://github.com/ClickHouse/ClickHouse/pull/9070) [(urykhy)](https://github.com/urykhy) - -### Clickhouse Version V20. 1. 2. 4, 2020-01-22 {#clickhouse-release-v20-1-2-4-2020-01-22} - -#### Modification Incompatible En Arrière {#backward-incompatible-change-1} - -- Effectuer le réglage `merge_tree_uniform_read_distribution` obsolète. Le serveur reconnaît toujours ce paramètre, mais il n’a pas d’effet. [\#8308](https://github.com/ClickHouse/ClickHouse/pull/8308) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Type de retour modifié de la fonction `greatCircleDistance` de `Float32` parce que maintenant, le résultat du calcul est `Float32`. [\#7993](https://github.com/ClickHouse/ClickHouse/pull/7993) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Maintenant, il est prévu que les paramètres de requête sont représentés dans “escaped” format. Par exemple, pour passer de la chaîne `ab` vous devez écrire `a\tb` ou `a\b` et, respectivement,, `a%5Ctb` ou `a%5C%09b` dans L’URL. Ceci est nécessaire pour ajouter la possibilité de passer NULL as `\N`. Cela corrige [\#7488](https://github.com/ClickHouse/ClickHouse/issues/7488). [\#8517](https://github.com/ClickHouse/ClickHouse/pull/8517) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Permettre `use_minimalistic_part_header_in_zookeeper` réglage pour `ReplicatedMergeTree` par défaut. Cela permettra de réduire considérablement la quantité de données stockées dans ZooKeeper. Ce paramètre est pris en charge depuis la version 19.1 et nous l’utilisons déjà en production dans plusieurs services sans aucun problème depuis plus d’une demi-année. Désactivez ce paramètre si vous avez la possibilité de passer à des versions antérieures à 19.1. [\#6850](https://github.com/ClickHouse/ClickHouse/pull/6850) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Les indices de saut de données sont prêts pour la production et activés par défaut. Paramètre `allow_experimental_data_skipping_indices`, `allow_experimental_cross_to_join_conversion` et `allow_experimental_multiple_joins_emulation` sont maintenant obsolètes et ne rien faire. [\#7974](https://github.com/ClickHouse/ClickHouse/pull/7974) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Ajouter de nouveaux `ANY JOIN` logique pour `StorageJoin` compatible avec `JOIN` opération. Pour mettre à niveau sans changement de comportement vous devez ajouter `SETTINGS any_join_distinct_right_table_keys = 1` pour engine Join tables metadata ou recréer ces tables après la mise à niveau. [\#8400](https://github.com/ClickHouse/ClickHouse/pull/8400) ([Artem Zuikov](https://github.com/4ertus2)) -- Exiger que le serveur soit redémarré pour appliquer les modifications dans la configuration de journalisation. Il s’agit d’une solution de contournement temporaire pour éviter le bogue où le serveur se connecte à un fichier journal supprimé (voir [\#8696](https://github.com/ClickHouse/ClickHouse/issues/8696)). [\#8707](https://github.com/ClickHouse/ClickHouse/pull/8707) ([Alexander Kuzmenkov](https://github.com/akuzm)) - -#### Nouveauté {#new-feature-2} - -- Ajout d’informations sur les chemins d’accès `system.merges`. [\#8043](https://github.com/ClickHouse/ClickHouse/pull/8043) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Ajouter la possibilité d’exécuter `SYSTEM RELOAD DICTIONARY` requête en `ON CLUSTER` mode. [\#8288](https://github.com/ClickHouse/ClickHouse/pull/8288) ([Guillaume Tassery](https://github.com/YiuRULE)) -- Ajouter la possibilité d’exécuter `CREATE DICTIONARY` les requêtes en `ON CLUSTER` mode. [\#8163](https://github.com/ClickHouse/ClickHouse/pull/8163) ([alésapine](https://github.com/alesapin)) -- Maintenant, le profil de l’utilisateur dans `users.xml` peut hériter de plusieurs profils. [\#8343](https://github.com/ClickHouse/ClickHouse/pull/8343) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) -- Ajouter `system.stack_trace` table qui permet de regarder les traces de pile de tous les threads du serveur. Ceci est utile pour les développeurs d’introspecter l’état du serveur. Cela corrige [\#7576](https://github.com/ClickHouse/ClickHouse/issues/7576). [\#8344](https://github.com/ClickHouse/ClickHouse/pull/8344) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Ajouter `DateTime64` type de données configurables sous-précision de seconde. [\#7170](https://github.com/ClickHouse/ClickHouse/pull/7170) ([Vasily Nemkov](https://github.com/Enmk)) -- Ajouter une fonction de table `clusterAllReplicas` ce qui permet d’interroger tous les nœuds dans le cluster. [\#8493](https://github.com/ClickHouse/ClickHouse/pull/8493) ([kiran sunkari](https://github.com/kiransunkari)) -- Ajouter une fonction d’agrégat `categoricalInformationValue` qui calcule la valeur d’information d’une fonction discrète. [\#8117](https://github.com/ClickHouse/ClickHouse/pull/8117) ([hcz](https://github.com/hczhcz)) -- Accélérer l’analyse des fichiers de données dans `CSV`, `TSV` et `JSONEachRow` formater en le faisant en parallèle. [\#7780](https://github.com/ClickHouse/ClickHouse/pull/7780) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- Ajouter une fonction `bankerRound` qui effectue l’arrondi. [\#8112](https://github.com/ClickHouse/ClickHouse/pull/8112) ([hcz](https://github.com/hczhcz)) -- Soutenir plus de langues dans le dictionnaire intégré pour les noms de région: ‘ru’, ‘en’, ‘ua’, ‘uk’, ‘by’, ‘kz’, ‘tr’, ‘de’, ‘uz’, ‘lv’, ‘lt’, ‘et’, ‘pt’, ‘he’, ‘vi’. [\#8189](https://github.com/ClickHouse/ClickHouse/pull/8189) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Améliorer l’uniformité de `ANY JOIN` logique. Maintenant `t1 ANY LEFT JOIN t2` égal `t2 ANY RIGHT JOIN t1`. [\#7665](https://github.com/ClickHouse/ClickHouse/pull/7665) ([Artem Zuikov](https://github.com/4ertus2)) -- Ajouter un paramètre `any_join_distinct_right_table_keys` ce qui permet un vieux comportement pour `ANY INNER JOIN`. [\#7665](https://github.com/ClickHouse/ClickHouse/pull/7665) ([Artem Zuikov](https://github.com/4ertus2)) -- Ajouter de nouveaux `SEMI` et `ANTI JOIN`. Vieux `ANY INNER JOIN` comportement maintenant disponible en `SEMI LEFT JOIN`. [\#7665](https://github.com/ClickHouse/ClickHouse/pull/7665) ([Artem Zuikov](https://github.com/4ertus2)) -- Ajouter `Distributed` format pour `File` moteur et `file` fonction de table qui permet de lire à partir `.bin` fichiers générés par des insertions asynchrones dans `Distributed` table. [\#8535](https://github.com/ClickHouse/ClickHouse/pull/8535) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Ajouter un argument de colonne de réinitialisation facultatif pour `runningAccumulate` ce qui permet de réinitialiser les résultats d’agrégation pour chaque nouvelle valeur de clé. [\#8326](https://github.com/ClickHouse/ClickHouse/pull/8326) ([Sergey Kononenko](https://github.com/kononencheg)) -- Ajouter la possibilité d’utiliser ClickHouse comme point de terminaison Prometheus. [\#7900](https://github.com/ClickHouse/ClickHouse/pull/7900) ([vdimir](https://github.com/Vdimir)) -- Ajouter une section `` dans `config.xml` qui restreint les hôtes autorisés pour les moteurs de table distants et les fonctions de table `URL`, `S3`, `HDFS`. [\#7154](https://github.com/ClickHouse/ClickHouse/pull/7154) ([Mikhail Korotov](https://github.com/millb)) -- Ajout de la fonction `greatCircleAngle` qui calcule la distance sur une sphère en degrés. [\#8105](https://github.com/ClickHouse/ClickHouse/pull/8105) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Rayon de la Terre modifié pour être cohérent avec la bibliothèque H3. [\#8105](https://github.com/ClickHouse/ClickHouse/pull/8105) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Ajouter `JSONCompactEachRow` et `JSONCompactEachRowWithNamesAndTypes` les formats d’entrée et de sortie. [\#7841](https://github.com/ClickHouse/ClickHouse/pull/7841) ([Mikhail Korotov](https://github.com/millb)) -- Ajout d’une fonctionnalité pour les moteurs de table liés aux fichiers et les fonctions de table (`File`, `S3`, `URL`, `HDFS`) qui permet de lire et d’écrire `gzip` fichiers basés sur un paramètre de moteur supplémentaire ou une extension de fichier. [\#7840](https://github.com/ClickHouse/ClickHouse/pull/7840) ([Andrey Bodrov](https://github.com/apbodrov)) -- Ajouté le `randomASCII(length)` fonction, générant une chaîne avec un ensemble aléatoire de [ASCII](https://en.wikipedia.org/wiki/ASCII#Printable_characters) caractères imprimables. [\#8401](https://github.com/ClickHouse/ClickHouse/pull/8401) ([Baïonnette](https://github.com/BayoNet)) -- Ajout de la fonction `JSONExtractArrayRaw` qui renvoie un tableau sur des éléments de tableau JSON non analysés de `JSON` chaîne. [\#8081](https://github.com/ClickHouse/ClickHouse/pull/8081) ([Oleg Matrokhin](https://github.com/errx)) -- Ajouter `arrayZip` fonction qui permet de combiner plusieurs tableaux de longueurs égales dans un tableau de n-uplets. [\#8149](https://github.com/ClickHouse/ClickHouse/pull/8149) ([L’Hiver Zhang](https://github.com/zhang2014)) -- Ajouter la possibilité de déplacer des données entre les disques selon configuré `TTL`-expressions pour `*MergeTree` famille de moteurs de table. [\#8140](https://github.com/ClickHouse/ClickHouse/pull/8140) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Ajout d’une nouvelle fonction d’agrégat `avgWeighted` qui permet de calculer la moyenne pondérée. [\#7898](https://github.com/ClickHouse/ClickHouse/pull/7898) ([Andrey Bodrov](https://github.com/apbodrov)) -- Maintenant, l’analyse parallèle est activée par défaut pour `TSV`, `TSKV`, `CSV` et `JSONEachRow` format. [\#7894](https://github.com/ClickHouse/ClickHouse/pull/7894) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- Ajouter plusieurs fonctions de géo `H3` bibliothèque: `h3GetResolution`, `h3EdgeAngle`, `h3EdgeLength`, `h3IsValid` et `h3kRing`. [\#8034](https://github.com/ClickHouse/ClickHouse/pull/8034) ([Konstantin Malanchev](https://github.com/hombit)) -- Ajout du support pour brotli (`br`) compression dans les stockages liés aux fichiers et les fonctions de table. Cela corrige [\#8156](https://github.com/ClickHouse/ClickHouse/issues/8156). [\#8526](https://github.com/ClickHouse/ClickHouse/pull/8526) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Ajouter `groupBit*` fonctions pour l’ `SimpleAggregationFunction` type. [\#8485](https://github.com/ClickHouse/ClickHouse/pull/8485) ([Guillaume Tassery](https://github.com/YiuRULE)) - -#### Bug Fix {#bug-fix-6} - -- Correction du renommage des tables avec `Distributed` moteur. Correction problème [\#7868](https://github.com/ClickHouse/ClickHouse/issues/7868). [\#8306](https://github.com/ClickHouse/ClickHouse/pull/8306) ([tavplubix](https://github.com/tavplubix)) -- Maintenant dictionnaires de soutien `EXPRESSION` pour les attributs dans une chaîne arbitraire en dialecte SQL non-ClickHouse. [\#8098](https://github.com/ClickHouse/ClickHouse/pull/8098) ([alésapine](https://github.com/alesapin)) -- Réparation de `INSERT SELECT FROM mysql(...)` requête. Cela corrige [\#8070](https://github.com/ClickHouse/ClickHouse/issues/8070) et [\#7960](https://github.com/ClickHouse/ClickHouse/issues/7960). [\#8234](https://github.com/ClickHouse/ClickHouse/pull/8234) ([tavplubix](https://github.com/tavplubix)) -- Correction d’erreur “Mismatch column sizes” lors de l’insertion par défaut `Tuple` de `JSONEachRow`. Cela corrige [\#5653](https://github.com/ClickHouse/ClickHouse/issues/5653). [\#8606](https://github.com/ClickHouse/ClickHouse/pull/8606) ([tavplubix](https://github.com/tavplubix)) -- Maintenant, une exception sera levée en cas d’utilisation `WITH TIES` parallèlement `LIMIT BY`. Ajoutez également la possibilité d’utiliser `TOP` avec `LIMIT BY`. Cela corrige [\#7472](https://github.com/ClickHouse/ClickHouse/issues/7472). [\#7637](https://github.com/ClickHouse/ClickHouse/pull/7637) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- Correction de la dépendance unintendent à partir de la nouvelle version de glibc dans `clickhouse-odbc-bridge` binaire. [\#8046](https://github.com/ClickHouse/ClickHouse/pull/8046) ([Amos Oiseau](https://github.com/amosbird)) -- Correction d’un bug dans la fonction de contrôle de `*MergeTree` les moteurs de la famille. Maintenant, il n’échoue pas dans le cas où nous avons une quantité égale de lignes dans le dernier granule et la dernière marque (non finale). [\#8047](https://github.com/ClickHouse/ClickHouse/pull/8047) ([alésapine](https://github.com/alesapin)) -- Fixer l’insertion dans `Enum*` les colonnes après `ALTER` requête, lorsque le type numérique sous-jacent est égal au type spécifié par la table. Cela corrige [\#7836](https://github.com/ClickHouse/ClickHouse/issues/7836). [\#7908](https://github.com/ClickHouse/ClickHouse/pull/7908) ([Anton Popov](https://github.com/CurtizJ)) -- Négatif non constant autorisé “size” argument pour la fonction `substring`. Il n’a pas été autorisé par erreur. Cela corrige [\#4832](https://github.com/ClickHouse/ClickHouse/issues/4832). [\#7703](https://github.com/ClickHouse/ClickHouse/pull/7703) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’un bogue d’analyse lorsque le nombre d’arguments transmis est erroné `(O|J)DBC` tableau moteur. [\#7709](https://github.com/ClickHouse/ClickHouse/pull/7709) ([alésapine](https://github.com/alesapin)) -- Utilisation du nom de commande du processus clickhouse en cours d’exécution lors de l’envoi de journaux à syslog. Dans les versions précédentes, la chaîne vide était utilisée à la place du nom de la commande. [\#8460](https://github.com/ClickHouse/ClickHouse/pull/8460) ([Michael Nacharov](https://github.com/mnach)) -- Correction de la vérification des hôtes autorisés pour `localhost`. Ce PR corrige la solution fournie dans [\#8241](https://github.com/ClickHouse/ClickHouse/pull/8241). [\#8342](https://github.com/ClickHouse/ClickHouse/pull/8342) ([Vitaly Baranov](https://github.com/vitlibar)) -- Correction plantage rare dans `argMin` et `argMax` fonctions pour les arguments de chaîne longue, lorsque result est utilisé dans `runningAccumulate` fonction. Cela corrige [\#8325](https://github.com/ClickHouse/ClickHouse/issues/8325) [\#8341](https://github.com/ClickHouse/ClickHouse/pull/8341) ([dinosaure](https://github.com/769344359)) -- Correction de la surcommission de mémoire pour les tables avec `Buffer` moteur. [\#8345](https://github.com/ClickHouse/ClickHouse/pull/8345) ([Azat Khuzhin](https://github.com/azat)) -- Correction d’un bug potentiel dans les fonctions qui peuvent prendre `NULL` comme l’un des arguments et retourner non-NULL. [\#8196](https://github.com/ClickHouse/ClickHouse/pull/8196) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Meilleurs calculs de métriques dans le pool de threads pour les processus `MergeTree` table des moteurs. [\#8194](https://github.com/ClickHouse/ClickHouse/pull/8194) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Fonction Fix `IN` à l’intérieur de `WHERE` instruction lorsque le filtre de table de niveau ligne est présent. Fixer [\#6687](https://github.com/ClickHouse/ClickHouse/issues/6687) [\#8357](https://github.com/ClickHouse/ClickHouse/pull/8357) ([Ivan](https://github.com/abyss7)) -- Maintenant, une exception est levée si la valeur intégrale n’est pas complètement analysée pour les valeurs des paramètres. [\#7678](https://github.com/ClickHouse/ClickHouse/pull/7678) ([Mikhail Korotov](https://github.com/millb)) -- Fix exception lorsque la fonction est utilisée dans la requête distribuée table avec plus de deux fragments. [\#8164](https://github.com/ClickHouse/ClickHouse/pull/8164) ([小路](https://github.com/nicelulu)) -- Maintenant, bloom filter peut gérer des tableaux de longueur nulle et n’effectue pas de calculs redondants. [\#8242](https://github.com/ClickHouse/ClickHouse/pull/8242) ([achimbab](https://github.com/achimbab)) -- Correction de la vérification si un hôte client est autorisé en faisant correspondre l’hôte client à `host_regexp` spécifié dans `users.xml`. [\#8241](https://github.com/ClickHouse/ClickHouse/pull/8241) ([Vitaly Baranov](https://github.com/vitlibar)) -- Relax colonne ambiguë vérifier qui conduit à des faux positifs dans plusieurs `JOIN ON` section. [\#8385](https://github.com/ClickHouse/ClickHouse/pull/8385) ([Artem Zuikov](https://github.com/4ertus2)) -- Correction possible plantage du serveur (`std::terminate`) lorsque le serveur ne peut pas envoyer ou écrire des données `JSON` ou `XML` format avec les valeurs de `String` type de données (qui nécessitent `UTF-8` validation) ou lors de la compression des données de résultat avec l’algorithme Brotli ou dans certains autres cas rares. Cela corrige [\#7603](https://github.com/ClickHouse/ClickHouse/issues/7603) [\#8384](https://github.com/ClickHouse/ClickHouse/pull/8384) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction de la condition de course dans `StorageDistributedDirectoryMonitor` trouvé par CI. Cela corrige [\#8364](https://github.com/ClickHouse/ClickHouse/issues/8364). [\#8383](https://github.com/ClickHouse/ClickHouse/pull/8383) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Maintenant fond fusionne dans `*MergeTree` la famille des moteurs de table préserve l’ordre de volume de la Politique de stockage avec plus de précision. [\#8549](https://github.com/ClickHouse/ClickHouse/pull/8549) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Maintenant moteur de table `Kafka` fonctionne correctement avec `Native` format. Cela corrige [\#6731](https://github.com/ClickHouse/ClickHouse/issues/6731) [\#7337](https://github.com/ClickHouse/ClickHouse/issues/7337) [\#8003](https://github.com/ClickHouse/ClickHouse/issues/8003). [\#8016](https://github.com/ClickHouse/ClickHouse/pull/8016) ([filimonov](https://github.com/filimonov)) -- Formats fixes avec des en-têtes (comme `CSVWithNames`) qui lançaient une exception sur EOF pour le moteur de table `Kafka`. [\#8016](https://github.com/ClickHouse/ClickHouse/pull/8016) ([filimonov](https://github.com/filimonov)) -- Correction d’un bug avec making set from subquery dans la partie droite de `IN` section. Cela corrige [\#5767](https://github.com/ClickHouse/ClickHouse/issues/5767) et [\#2542](https://github.com/ClickHouse/ClickHouse/issues/2542). [\#7755](https://github.com/ClickHouse/ClickHouse/pull/7755) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- Correction d’un crash possible lors de la lecture à partir du stockage `File`. [\#7756](https://github.com/ClickHouse/ClickHouse/pull/7756) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Correction de la lecture des fichiers en `Parquet` format contenant des colonnes de type `list`. [\#8334](https://github.com/ClickHouse/ClickHouse/pull/8334) ([maxulan](https://github.com/maxulan)) -- Correction d’erreur `Not found column` pour les requêtes distribuées avec `PREWHERE` condition dépendant de la clé d’échantillonnage si `max_parallel_replicas > 1`. [\#7913](https://github.com/ClickHouse/ClickHouse/pull/7913) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Correction d’erreur `Not found column` si la requête utilisée `PREWHERE` dépendant de l’alias de la table et le jeu de résultats était vide en raison de la condition de clé primaire. [\#7911](https://github.com/ClickHouse/ClickHouse/pull/7911) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Type de retour fixe pour les fonctions `rand` et `randConstant` en cas de `Nullable` argument. Maintenant renvoient toujours `UInt32` et jamais `Nullable(UInt32)`. [\#8204](https://github.com/ClickHouse/ClickHouse/pull/8204) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Désactivé prédicat-poussoir vers le bas pour `WITH FILL` expression. Cela corrige [\#7784](https://github.com/ClickHouse/ClickHouse/issues/7784). [\#7789](https://github.com/ClickHouse/ClickHouse/pull/7789) ([L’Hiver Zhang](https://github.com/zhang2014)) -- Fixe incorrect `count()` résultat `SummingMergeTree` lorsque `FINAL` la section est utilisée. [\#3280](https://github.com/ClickHouse/ClickHouse/issues/3280) [\#7786](https://github.com/ClickHouse/ClickHouse/pull/7786) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- Correction d’un résultat incorrect possible pour les fonctions constantes à partir de serveurs distants. C’est arrivé pour les requêtes avec des fonctions comme `version()`, `uptime()`, etc. qui renvoie différentes valeurs constantes pour différents serveurs. Cela corrige [\#7666](https://github.com/ClickHouse/ClickHouse/issues/7666). [\#7689](https://github.com/ClickHouse/ClickHouse/pull/7689) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Correction d’un bug compliqué dans l’optimisation des prédicats push-down qui conduit à de mauvais résultats. Cela résout beaucoup de problèmes sur l’optimisation des prédicats push-down. [\#8503](https://github.com/ClickHouse/ClickHouse/pull/8503) ([L’Hiver Zhang](https://github.com/zhang2014)) -- Correction d’un crash dans l’ `CREATE TABLE .. AS dictionary` requête. [\#8508](https://github.com/ClickHouse/ClickHouse/pull/8508) ([Azat Khuzhin](https://github.com/azat)) -- Plusieurs améliorations grammaire ClickHouse dans `.g4` fichier. [\#8294](https://github.com/ClickHouse/ClickHouse/pull/8294) ([taiyang-li](https://github.com/taiyang-li)) -- Correction d’un bug qui conduit à des plantages dans `JOIN`s avec tables avec moteur `Join`. Cela corrige [\#7556](https://github.com/ClickHouse/ClickHouse/issues/7556) [\#8254](https://github.com/ClickHouse/ClickHouse/issues/8254) [\#7915](https://github.com/ClickHouse/ClickHouse/issues/7915) [\#8100](https://github.com/ClickHouse/ClickHouse/issues/8100). [\#8298](https://github.com/ClickHouse/ClickHouse/pull/8298) ([Artem Zuikov](https://github.com/4ertus2)) -- Corriger les dictionnaires redondants recharger sur `CREATE DATABASE`. [\#7916](https://github.com/ClickHouse/ClickHouse/pull/7916) ([Azat Khuzhin](https://github.com/azat)) -- Limiter le nombre maximum de flux pour lire à partir `StorageFile` et `StorageHDFS`. Corrections https://github.com/ClickHouse/ClickHouse/issues/7650. [\#7981](https://github.com/ClickHouse/ClickHouse/pull/7981) ([alésapine](https://github.com/alesapin)) -- Correction d’un bug dans `ALTER ... MODIFY ... CODEC` requête, lorsque l’utilisateur spécifie à la fois l’expression par défaut et le codec. Fixer [8593](https://github.com/ClickHouse/ClickHouse/issues/8593). [\#8614](https://github.com/ClickHouse/ClickHouse/pull/8614) ([alésapine](https://github.com/alesapin)) -- Correction d’une erreur dans la fusion en arrière-plan des colonnes avec `SimpleAggregateFunction(LowCardinality)` type. [\#8613](https://github.com/ClickHouse/ClickHouse/pull/8613) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Fonction d’enregistrement de type fixe `toDateTime64`. [\#8375](https://github.com/ClickHouse/ClickHouse/pull/8375) ([Vasily Nemkov](https://github.com/Enmk)) -- Maintenant le serveur ne plante pas `LEFT` ou `FULL JOIN` avec et Rejoindre moteur et non pris en charge `join_use_nulls` paramètre. [\#8479](https://github.com/ClickHouse/ClickHouse/pull/8479) ([Artem Zuikov](https://github.com/4ertus2)) -- Maintenant `DROP DICTIONARY IF EXISTS db.dict` la requête ne lance pas d’exception si `db` n’existe pas. [\#8185](https://github.com/ClickHouse/ClickHouse/pull/8185) ([Vitaly Baranov](https://github.com/vitlibar)) -- Correction des plantages possibles dans les fonctions de la table (`file`, `mysql`, `remote`) causés par l’utilisation de la référence à enlever `IStorage` objet. Correction d’une analyse incorrecte des colonnes spécifiées lors de l’insertion dans la fonction de table. [\#7762](https://github.com/ClickHouse/ClickHouse/pull/7762) ([tavplubix](https://github.com/tavplubix)) -- S’assurer du réseau avant de démarrer `clickhouse-server`. Cela corrige [\#7507](https://github.com/ClickHouse/ClickHouse/issues/7507). [\#8570](https://github.com/ClickHouse/ClickHouse/pull/8570) ([Zhichang Yu](https://github.com/yuzhichang)) -- Correction de la gestion des délais d’attente pour les connexions sécurisées, de sorte que les requêtes ne se bloquent pas indéfiniment. Cela corrige [\#8126](https://github.com/ClickHouse/ClickHouse/issues/8126). [\#8128](https://github.com/ClickHouse/ClickHouse/pull/8128) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Fixer `clickhouse-copier`conflit redondant entre les travailleurs concurrents. [\#7816](https://github.com/ClickHouse/ClickHouse/pull/7816) ([Ding Xiang Fei](https://github.com/dingxiangfei2009)) -- Maintenant, les mutations ne sautent pas les parties attachées, même si leur version de mutation était plus grande que la version de mutation actuelle. [\#7812](https://github.com/ClickHouse/ClickHouse/pull/7812) ([Zhichang Yu](https://github.com/yuzhichang)) [\#8250](https://github.com/ClickHouse/ClickHouse/pull/8250) ([alésapine](https://github.com/alesapin)) -- Ignorer les copies redondantes de `*MergeTree` les parties de données après le déplacement vers un autre disque et le redémarrage du serveur. [\#7810](https://github.com/ClickHouse/ClickHouse/pull/7810) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Correction d’un crash dans l’ `FULL JOIN` avec `LowCardinality` dans `JOIN` clé. [\#8252](https://github.com/ClickHouse/ClickHouse/pull/8252) ([Artem Zuikov](https://github.com/4ertus2)) -- Interdit d’utiliser le nom de colonne plus d’une fois dans insert query comme `INSERT INTO tbl (x, y, x)`. Cela corrige [\#5465](https://github.com/ClickHouse/ClickHouse/issues/5465), [\#7681](https://github.com/ClickHouse/ClickHouse/issues/7681). [\#7685](https://github.com/ClickHouse/ClickHouse/pull/7685) ([alésapine](https://github.com/alesapin)) -- Ajout de secours pour la détection du nombre de cœurs de processeur physiques pour les processeurs inconnus (en utilisant le nombre de cœurs de processeur logiques). Cela corrige [\#5239](https://github.com/ClickHouse/ClickHouse/issues/5239). [\#7726](https://github.com/ClickHouse/ClickHouse/pull/7726) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Fixer `There's no column` erreur pour les colonnes matérialisées et alias. [\#8210](https://github.com/ClickHouse/ClickHouse/pull/8210) ([Artem Zuikov](https://github.com/4ertus2)) -- Correction d’un crash sever lorsque `EXISTS` la requête a été utilisé sans `TABLE` ou `DICTIONARY` qualificatif. Tout comme `EXISTS t`. Cela corrige [\#8172](https://github.com/ClickHouse/ClickHouse/issues/8172). Ce bug a été introduit dans la version 19.17. [\#8213](https://github.com/ClickHouse/ClickHouse/pull/8213) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’un bug rare avec erreur `"Sizes of columns doesn't match"` qui pourraient apparaître lors de l’utilisation `SimpleAggregateFunction` colonne. [\#7790](https://github.com/ClickHouse/ClickHouse/pull/7790) ([Boris Granveaud](https://github.com/bgranvea)) -- Correction d’un bug où l’utilisateur avec vide `allow_databases` vous avez accès à toutes les bases de données (et même pour `allow_dictionaries`). [\#7793](https://github.com/ClickHouse/ClickHouse/pull/7793) ([DeifyTheGod](https://github.com/DeifyTheGod)) -- Correction du crash du client lorsque le serveur est déjà déconnecté du client. [\#8071](https://github.com/ClickHouse/ClickHouse/pull/8071) ([Azat Khuzhin](https://github.com/azat)) -- Fixer `ORDER BY` comportement en cas de tri par préfixe de clé primaire et Suffixe de clé non primaire. [\#7759](https://github.com/ClickHouse/ClickHouse/pull/7759) ([Anton Popov](https://github.com/CurtizJ)) -- Vérifiez si la colonne qualifiée est présente dans le tableau. Cela corrige [\#6836](https://github.com/ClickHouse/ClickHouse/issues/6836). [\#7758](https://github.com/ClickHouse/ClickHouse/pull/7758) ([Artem Zuikov](https://github.com/4ertus2)) -- Correction du comportement avec `ALTER MOVE` exécuté immédiatement après la fin de la fusion se déplace superpart De spécifié. Fixer [\#8103](https://github.com/ClickHouse/ClickHouse/issues/8103). [\#8104](https://github.com/ClickHouse/ClickHouse/pull/8104) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Correction d’un crash possible du serveur lors de l’utilisation `UNION` avec un nombre différent de colonnes. Fixer [\#7279](https://github.com/ClickHouse/ClickHouse/issues/7279). [\#7929](https://github.com/ClickHouse/ClickHouse/pull/7929) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Fixer la taille de résultat pour la fonction substring `substr` avec une taille négative. [\#8589](https://github.com/ClickHouse/ClickHouse/pull/8589) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Maintenant le serveur n’exécute pas la mutation partielle dans `MergeTree` s’il n’y a pas assez de threads libres dans le pool d’arrière-plan. [\#8588](https://github.com/ClickHouse/ClickHouse/pull/8588) ([tavplubix](https://github.com/tavplubix)) -- Correction d’une faute de frappe mineure sur le formatage `UNION ALL` AST. [\#7999](https://github.com/ClickHouse/ClickHouse/pull/7999) ([litao91](https://github.com/litao91)) -- Correction des résultats incorrects du filtre bloom pour les nombres négatifs. Cela corrige [\#8317](https://github.com/ClickHouse/ClickHouse/issues/8317). [\#8566](https://github.com/ClickHouse/ClickHouse/pull/8566) ([L’Hiver Zhang](https://github.com/zhang2014)) -- Dépassement de tampon potentiel fixe en décompression. Un utilisateur malveillant peut transmettre des données compressées fabriquées qui provoqueront une lecture après le tampon. Ce problème a été trouvé par Eldar Zaitov de l’équipe de sécurité de L’information Yandex. [\#8404](https://github.com/ClickHouse/ClickHouse/pull/8404) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction d’un résultat incorrect en raison du débordement d’entiers dans `arrayIntersect`. [\#7777](https://github.com/ClickHouse/ClickHouse/pull/7777) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Maintenant `OPTIMIZE TABLE` query n’attendra pas les répliques hors ligne pour effectuer l’opération. [\#8314](https://github.com/ClickHouse/ClickHouse/pull/8314) ([javi santana](https://github.com/javisantana)) -- Fixe `ALTER TTL` analyseur pour `Replicated*MergeTree` table. [\#8318](https://github.com/ClickHouse/ClickHouse/pull/8318) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Correction de la communication entre le serveur et le client, afin que le serveur lise les informations des tables temporaires après l’échec de la requête. [\#8084](https://github.com/ClickHouse/ClickHouse/pull/8084) ([Azat Khuzhin](https://github.com/azat)) -- Fixer `bitmapAnd` erreur de fonction lors de l’intersection d’un bitmap agrégé et d’un bitmap scalaire. [\#8082](https://github.com/ClickHouse/ClickHouse/pull/8082) ([Yue Huang](https://github.com/moon03432)) -- Affiner la définition de `ZXid` selon le Guide du programmeur ZooKeeper qui corrige un bug dans `clickhouse-cluster-copier`. [\#8088](https://github.com/ClickHouse/ClickHouse/pull/8088) ([Ding Xiang Fei](https://github.com/dingxiangfei2009)) -- `odbc` fonction de table respecte maintenant `external_table_functions_use_nulls` paramètre. [\#7506](https://github.com/ClickHouse/ClickHouse/pull/7506) ([Vasily Nemkov](https://github.com/Enmk)) -- Correction d’un bug qui conduisait à une course de données rare. [\#8143](https://github.com/ClickHouse/ClickHouse/pull/8143) ([Alexander Kazakov](https://github.com/Akazz)) -- Maintenant `SYSTEM RELOAD DICTIONARY` recharge complètement un dictionnaire, en ignorant `update_field`. Cela corrige [\#7440](https://github.com/ClickHouse/ClickHouse/issues/7440). [\#8037](https://github.com/ClickHouse/ClickHouse/pull/8037) ([Vitaly Baranov](https://github.com/vitlibar)) -- Ajouter la possibilité de vérifier si le dictionnaire existe dans create query. [\#8032](https://github.com/ClickHouse/ClickHouse/pull/8032) ([alésapine](https://github.com/alesapin)) -- Fixer `Float*` l’analyse en `Values` format. Cela corrige [\#7817](https://github.com/ClickHouse/ClickHouse/issues/7817). [\#7870](https://github.com/ClickHouse/ClickHouse/pull/7870) ([tavplubix](https://github.com/tavplubix)) -- Correction d’un crash lorsque nous ne pouvons pas réserver d’espace dans certaines opérations en arrière-plan de `*MergeTree` famille de moteurs de table. [\#7873](https://github.com/ClickHouse/ClickHouse/pull/7873) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Correction du crash de l’opération de fusion lorsque la table contient `SimpleAggregateFunction(LowCardinality)` colonne. Cela corrige [\#8515](https://github.com/ClickHouse/ClickHouse/issues/8515). [\#8522](https://github.com/ClickHouse/ClickHouse/pull/8522) ([Azat Khuzhin](https://github.com/azat)) -- Restaurez la prise en charge de toutes les locales ICU et ajoutez la possibilité d’appliquer des collations pour les expressions constantes. Ajoutez également le nom de la langue à `system.collations` table. [\#8051](https://github.com/ClickHouse/ClickHouse/pull/8051) ([alésapine](https://github.com/alesapin)) -- Correction d’un bug lorsque les dictionnaires externes avec zéro durée de vie minimale (`LIFETIME(MIN 0 MAX N)`, `LIFETIME(N)`) ne pas mettre à jour en arrière-plan. [\#7983](https://github.com/ClickHouse/ClickHouse/pull/7983) ([alésapine](https://github.com/alesapin)) -- Correction d’un crash lorsque le dictionnaire externe avec la source de ClickHouse a une sous-requête dans la requête. [\#8351](https://github.com/ClickHouse/ClickHouse/pull/8351) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Correction d’une analyse incorrecte de l’extension de fichier dans la table avec le moteur `URL`. Cela corrige [\#8157](https://github.com/ClickHouse/ClickHouse/issues/8157). [\#8419](https://github.com/ClickHouse/ClickHouse/pull/8419) ([Andrey Bodrov](https://github.com/apbodrov)) -- Fixer `CHECK TABLE` requête pour `*MergeTree` les tables sans clé. Fixer [\#7543](https://github.com/ClickHouse/ClickHouse/issues/7543). [\#7979](https://github.com/ClickHouse/ClickHouse/pull/7979) ([alésapine](https://github.com/alesapin)) -- De conversion fixe de `Float64` au type MySQL. [\#8079](https://github.com/ClickHouse/ClickHouse/pull/8079) ([Yuriy Baranov](https://github.com/yurriy)) -- Maintenant, si la table n’a pas été complètement abandonnée en raison d’un plantage du serveur, le serveur va essayer de la restaurer et de la charger. [\#8176](https://github.com/ClickHouse/ClickHouse/pull/8176) ([tavplubix](https://github.com/tavplubix)) -- Correction d’un crash dans la fonction de table `file` lors de l’insertion dans le fichier qui n’existe pas. Maintenant, dans ce cas, le fichier sera créé et insérez seraient traités. [\#8177](https://github.com/ClickHouse/ClickHouse/pull/8177) ([Olga Khvostikova](https://github.com/stavrolia)) -- Correction de l’impasse rare qui peut arriver quand `trace_log` est activé. [\#7838](https://github.com/ClickHouse/ClickHouse/pull/7838) ([filimonov](https://github.com/filimonov)) -- Ajouter la possibilité de travailler avec différents types en outre `Date` dans `RangeHashed` dictionnaire externe créé à partir de la requête DDL. Fixer [7899](https://github.com/ClickHouse/ClickHouse/issues/7899). [\#8275](https://github.com/ClickHouse/ClickHouse/pull/8275) ([alésapine](https://github.com/alesapin)) -- Correction d’un crash lorsque `now64()` est appelé avec un résultat d’une autre fonction. [\#8270](https://github.com/ClickHouse/ClickHouse/pull/8270) ([Vasily Nemkov](https://github.com/Enmk)) -- Correction d’un bug avec la détection de l’adresse IP du client pour les connexions via le protocole de fil mysql. [\#7743](https://github.com/ClickHouse/ClickHouse/pull/7743) ([Dmitry Muzyka](https://github.com/dmitriy-myz)) -- Correction de la gestion du tableau vide dans `arraySplit` fonction. Cela corrige [\#7708](https://github.com/ClickHouse/ClickHouse/issues/7708). [\#7747](https://github.com/ClickHouse/ClickHouse/pull/7747) ([hcz](https://github.com/hczhcz)) -- Correction du problème lorsque `pid-file` d’un autre cours d’exécution `clickhouse-server` peut être supprimée. [\#8487](https://github.com/ClickHouse/ClickHouse/pull/8487) ([Weiqing Xu](https://github.com/weiqxu)) -- Correction du rechargement du dictionnaire s’il a `invalidate_query`, qui a arrêté les mises à jour et une exception sur les tentatives de mise à jour précédentes. [\#8029](https://github.com/ClickHouse/ClickHouse/pull/8029) ([alésapine](https://github.com/alesapin)) -- Correction d’une erreur dans la fonction `arrayReduce` qui peut conduire à “double free” et erreur dans le combinateur de fonction d’agrégat `Resample` que peut provoquer la fuite de mémoire. Fonction agrégée ajoutée `aggThrow`. Cette fonction peut être utilisée à des fins de test. [\#8446](https://github.com/ClickHouse/ClickHouse/pull/8446) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - -#### Amélioration {#improvement-1} - -- Amélioration de la journalisation lorsque vous travaillez avec `S3` tableau moteur. [\#8251](https://github.com/ClickHouse/ClickHouse/pull/8251) ([Grigory Pervakov](https://github.com/GrigoryPervakov)) -- Imprimé message d’aide quand aucun argument n’est passé lors de l’appel `clickhouse-local`. Cela corrige [\#5335](https://github.com/ClickHouse/ClickHouse/issues/5335). [\#8230](https://github.com/ClickHouse/ClickHouse/pull/8230) ([Andrey Nagorny](https://github.com/Melancholic)) -- Ajouter un paramètre `mutations_sync` ce qui permet d’attendre `ALTER UPDATE/DELETE` les requêtes de manière synchrone. [\#8237](https://github.com/ClickHouse/ClickHouse/pull/8237) ([alésapine](https://github.com/alesapin)) -- Autoriser à configurer relative `user_files_path` dans `config.xml` (de la manière similaire à `format_schema_path`). [\#7632](https://github.com/ClickHouse/ClickHouse/pull/7632) ([hcz](https://github.com/hczhcz)) -- Ajouter une exception pour les types illégaux pour les fonctions de conversion avec `-OrZero` postfix. [\#7880](https://github.com/ClickHouse/ClickHouse/pull/7880) ([Andrey Konyaev](https://github.com/akonyaev90)) -- Simplifier le format de l’en-tête de l’envoi des données à un serveur dans une requête distribuée. [\#8044](https://github.com/ClickHouse/ClickHouse/pull/8044) ([Vitaly Baranov](https://github.com/vitlibar)) -- `Live View` refactoring du moteur de table. [\#8519](https://github.com/ClickHouse/ClickHouse/pull/8519) ([vzakaznikov](https://github.com/vzakaznikov)) -- Ajoutez des vérifications supplémentaires pour les dictionnaires externes créés à partir de requêtes DDL. [\#8127](https://github.com/ClickHouse/ClickHouse/pull/8127) ([alésapine](https://github.com/alesapin)) -- Correction d’erreur `Column ... already exists` lors de l’utilisation `FINAL` et `SAMPLE` together, e.g. `select count() from table final sample 1/2`. Fixer [\#5186](https://github.com/ClickHouse/ClickHouse/issues/5186). [\#7907](https://github.com/ClickHouse/ClickHouse/pull/7907) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Table maintenant le premier argument de `joinGet` la fonction peut être tableau identifiant. [\#7707](https://github.com/ClickHouse/ClickHouse/pull/7707) ([Amos Oiseau](https://github.com/amosbird)) -- Autoriser l’utilisation des `MaterializedView` avec les sous-requêtes ci-dessus `Kafka` table. [\#8197](https://github.com/ClickHouse/ClickHouse/pull/8197) ([filimonov](https://github.com/filimonov)) -- Maintenant, l’arrière-plan se déplace entre les disques, exécutez le pool de threads seprate. [\#7670](https://github.com/ClickHouse/ClickHouse/pull/7670) ([Vladimir Chebotarev](https://github.com/excitoon)) -- `SYSTEM RELOAD DICTIONARY` s’exécute maintenant de manière synchrone. [\#8240](https://github.com/ClickHouse/ClickHouse/pull/8240) ([Vitaly Baranov](https://github.com/vitlibar)) -- Les traces de pile affichent désormais des adresses physiques (décalages dans le fichier objet) au lieu des adresses de mémoire virtuelle (où le fichier objet a été chargé). Qui permet l’utilisation de `addr2line` lorsque binaire est indépendant de la position et ASLR est actif. Cela corrige [\#8360](https://github.com/ClickHouse/ClickHouse/issues/8360). [\#8387](https://github.com/ClickHouse/ClickHouse/pull/8387) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Appuyer une nouvelle syntaxe pour la sécurité de niveau ligne filtres: `
…
`. Fixer [\#5779](https://github.com/ClickHouse/ClickHouse/issues/5779). [\#8381](https://github.com/ClickHouse/ClickHouse/pull/8381) ([Ivan](https://github.com/abyss7)) -- Maintenant `cityHash` fonction peut travailler avec `Decimal` et `UUID` type. Fixer [\#5184](https://github.com/ClickHouse/ClickHouse/issues/5184). [\#7693](https://github.com/ClickHouse/ClickHouse/pull/7693) ([Mikhail Korotov](https://github.com/millb)) -- Suppression de la granularité à index fixe (c’était 1024) des journaux système car elle est obsolète après l’implémentation de la granularité adaptative. [\#7698](https://github.com/ClickHouse/ClickHouse/pull/7698) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Serveur de compatibilité MySQL activé lorsque ClickHouse est compilé sans SSL. [\#7852](https://github.com/ClickHouse/ClickHouse/pull/7852) ([Yuriy Baranov](https://github.com/yurriy)) -- Maintenant, les sommes de contrôle du serveur ont distribué des lots, ce qui donne des erreurs plus verbeuses en cas de données corrompues dans le lot. [\#7914](https://github.com/ClickHouse/ClickHouse/pull/7914) ([Azat Khuzhin](https://github.com/azat)) -- Soutien `DROP DATABASE`, `DETACH TABLE`, `DROP TABLE` et `ATTACH TABLE` pour `MySQL` moteur de base de données. [\#8202](https://github.com/ClickHouse/ClickHouse/pull/8202) ([L’Hiver Zhang](https://github.com/zhang2014)) -- Ajouter l’authentification dans la fonction de table S3 et le moteur de table. [\#7623](https://github.com/ClickHouse/ClickHouse/pull/7623) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Ajout de vérifier les pièces supplémentaires de `MergeTree` sur différents disques, afin de ne pas permettre de manquer des parties de données sur des disques indéfinis. [\#8118](https://github.com/ClickHouse/ClickHouse/pull/8118) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Activez la prise en charge SSL pour le client et le serveur Mac. [\#8297](https://github.com/ClickHouse/ClickHouse/pull/8297) ([Ivan](https://github.com/abyss7)) -- Maintenant, ClickHouse peut fonctionner en tant que serveur fédéré MySQL (voir https://dev.mysql.com/doc/refman/5.7/en/federated-create-server.html). [\#7717](https://github.com/ClickHouse/ClickHouse/pull/7717) ([Maxim Fedotov](https://github.com/MaxFedotov)) -- `clickhouse-client` maintenant seulement activer `bracketed-paste` lorsque multiquery est activé et multiline est désactivé. Cette correction (\#7757)\[https://github.com/ClickHouse/ClickHouse/issues/7757\]. [\#7761](https://github.com/ClickHouse/ClickHouse/pull/7761) ([Amos Oiseau](https://github.com/amosbird)) -- Soutien `Array(Decimal)` dans `if` fonction. [\#7721](https://github.com/ClickHouse/ClickHouse/pull/7721) ([Artem Zuikov](https://github.com/4ertus2)) -- Soutien Décimales `arrayDifference`, `arrayCumSum` et `arrayCumSumNegative` fonction. [\#7724](https://github.com/ClickHouse/ClickHouse/pull/7724) ([Artem Zuikov](https://github.com/4ertus2)) -- Ajouter `lifetime` colonne de `system.dictionaries` table. [\#6820](https://github.com/ClickHouse/ClickHouse/issues/6820) [\#7727](https://github.com/ClickHouse/ClickHouse/pull/7727) ([kekekekule](https://github.com/kekekekule)) -- Vérification améliorée des pièces existantes sur différents disques pour `*MergeTree` table des moteurs. Adresse [\#7660](https://github.com/ClickHouse/ClickHouse/issues/7660). [\#8440](https://github.com/ClickHouse/ClickHouse/pull/8440) ([Vladimir Chebotarev](https://github.com/excitoon)) -- L’intégration avec `AWS SDK` pour `S3` interactions qui permet d’utiliser toutes les fonctionnalités S3 hors de la boîte. [\#8011](https://github.com/ClickHouse/ClickHouse/pull/8011) ([Pavel Kovalenko](https://github.com/Jokser)) -- Ajout du support pour les sous-requêtes dans `Live View` table. [\#7792](https://github.com/ClickHouse/ClickHouse/pull/7792) ([vzakaznikov](https://github.com/vzakaznikov)) -- Vérifier à l’aide de `Date` ou `DateTime` colonne de `TTL` des expressions a été supprimé. [\#7920](https://github.com/ClickHouse/ClickHouse/pull/7920) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Informations sur le disque ajouté `system.detached_parts` table. [\#7833](https://github.com/ClickHouse/ClickHouse/pull/7833) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Maintenant paramètres `max_(table|partition)_size_to_drop` peut être modifié sans redémarrage de l’ordinateur. [\#7779](https://github.com/ClickHouse/ClickHouse/pull/7779) ([Grigory Pervakov](https://github.com/GrigoryPervakov)) -- Facilité d’utilisation légèrement meilleure des messages d’erreur. Demander à l’utilisateur de ne pas supprimer les lignes ci-dessous `Stack trace:`. [\#7897](https://github.com/ClickHouse/ClickHouse/pull/7897) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Mieux lire les messages de `Kafka` moteur dans différents formats après [\#7935](https://github.com/ClickHouse/ClickHouse/issues/7935). [\#8035](https://github.com/ClickHouse/ClickHouse/pull/8035) ([Ivan](https://github.com/abyss7)) -- Meilleure compatibilité avec les clients MySQL qui ne prennent pas en charge `sha2_password` greffon auth. [\#8036](https://github.com/ClickHouse/ClickHouse/pull/8036) ([Yuriy Baranov](https://github.com/yurriy)) -- Supporte plus de types de colonnes dans le serveur de compatibilité MySQL. [\#7975](https://github.com/ClickHouse/ClickHouse/pull/7975) ([Yuriy Baranov](https://github.com/yurriy)) -- Mettre `ORDER BY` optimisation pour les `Merge`, `Buffer` et `Materilized View` stockages avec sous-jacent `MergeTree` table. [\#8130](https://github.com/ClickHouse/ClickHouse/pull/8130) ([Anton Popov](https://github.com/CurtizJ)) -- Maintenant, nous utilisons toujours L’implémentation POSIX de `getrandom` pour avoir une meilleure compatibilité avec les anciens noyaux (\< 3.17). [\#7940](https://github.com/ClickHouse/ClickHouse/pull/7940) ([Amos Oiseau](https://github.com/amosbird)) -- Mieux vaut vérifier la destination valide dans une règle de déplacement TTL. [\#8410](https://github.com/ClickHouse/ClickHouse/pull/8410) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Mieux vérifie cassé insérer des lots pour `Distributed` tableau moteur. [\#7933](https://github.com/ClickHouse/ClickHouse/pull/7933) ([Azat Khuzhin](https://github.com/azat)) -- Ajouter une colonne avec un tableau de nom de pièces que les mutations doivent traiter à l’avenir `system.mutations` table. [\#8179](https://github.com/ClickHouse/ClickHouse/pull/8179) ([alésapine](https://github.com/alesapin)) -- Optimisation de tri de fusion parallèle pour les processeurs. [\#8552](https://github.com/ClickHouse/ClickHouse/pull/8552) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Paramètre `mark_cache_min_lifetime` est maintenant obsolète et ne fait rien. Dans les versions précédentes, mark cache peut croître en mémoire supérieure à `mark_cache_size` pour accommoder les données dans `mark_cache_min_lifetime` deuxième. Cela conduisait à la confusion et à une utilisation de la mémoire plus élevée que prévu, ce qui est particulièrement mauvais sur les systèmes à contraintes de mémoire. Si vous constatez une dégradation des performances après l’installation de cette version, vous devez `mark_cache_size`. [\#8484](https://github.com/ClickHouse/ClickHouse/pull/8484) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Préparation à utiliser `tid` partout. Cela est nécessaire pour [\#7477](https://github.com/ClickHouse/ClickHouse/issues/7477). [\#8276](https://github.com/ClickHouse/ClickHouse/pull/8276) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - -#### Amélioration Des Performances {#performance-improvement-1} - -- Optimisations des performances dans le pipeline de processeurs. [\#7988](https://github.com/ClickHouse/ClickHouse/pull/7988) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Mises à jour non bloquantes des clés expirées dans les dictionnaires de cache (avec autorisation de lire les anciennes). [\#8303](https://github.com/ClickHouse/ClickHouse/pull/8303) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- Compiler ClickHouse sans `-fno-omit-frame-pointer` globalement pour épargner un registre de plus. [\#8097](https://github.com/ClickHouse/ClickHouse/pull/8097) ([Amos Oiseau](https://github.com/amosbird)) -- SpeedUp `greatCircleDistance` fonction et ajouter des tests de performance pour elle. [\#7307](https://github.com/ClickHouse/ClickHouse/pull/7307) ([Olga Khvostikova](https://github.com/stavrolia)) -- Amélioration des performances de la fonction `roundDown`. [\#8465](https://github.com/ClickHouse/ClickHouse/pull/8465) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Amélioration des performances de `max`, `min`, `argMin`, `argMax` pour `DateTime64` type de données. [\#8199](https://github.com/ClickHouse/ClickHouse/pull/8199) ([Vasily Nemkov](https://github.com/Enmk)) -- Amélioration des performances de tri sans limite ou avec une grande limite et le tri externe. [\#8545](https://github.com/ClickHouse/ClickHouse/pull/8545) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Amélioration des performances du formatage des nombres à virgule flottante jusqu’à 6 fois. [\#8542](https://github.com/ClickHouse/ClickHouse/pull/8542) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Amélioration des performances de `modulo` fonction. [\#7750](https://github.com/ClickHouse/ClickHouse/pull/7750) ([Amos Oiseau](https://github.com/amosbird)) -- Optimisé `ORDER BY` et la fusion avec une seule clé de colonne. [\#8335](https://github.com/ClickHouse/ClickHouse/pull/8335) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Meilleure mise en œuvre pour `arrayReduce`, `-Array` et `-State` combinators. [\#7710](https://github.com/ClickHouse/ClickHouse/pull/7710) ([Amos Oiseau](https://github.com/amosbird)) -- Maintenant `PREWHERE` doit être optimisé pour être au moins aussi efficace que l’ `WHERE`. [\#7769](https://github.com/ClickHouse/ClickHouse/pull/7769) ([Amos Oiseau](https://github.com/amosbird)) -- Améliorer la façon dont `round` et `roundBankers` manipulation des nombres négatifs. [\#8229](https://github.com/ClickHouse/ClickHouse/pull/8229) ([hcz](https://github.com/hczhcz)) -- Amélioration des performances de décodage `DoubleDelta` et `Gorilla` les codecs par environ de 30 à 40%. Cela corrige [\#7082](https://github.com/ClickHouse/ClickHouse/issues/7082). [\#8019](https://github.com/ClickHouse/ClickHouse/pull/8019) ([Vasily Nemkov](https://github.com/Enmk)) -- Amélioration des performances de `base64` les fonctions connexes. [\#8444](https://github.com/ClickHouse/ClickHouse/pull/8444) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Ajout d’une fonction `geoDistance`. Il est similaire à `greatCircleDistance` mais utilise l’approximation au modèle ellipsoïde WGS-84. Les performances des deux fonctions sont presque les mêmes. [\#8086](https://github.com/ClickHouse/ClickHouse/pull/8086) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Plus rapide `min` et `max` fonctions d’agrégation pour les `Decimal` type de données. [\#8144](https://github.com/ClickHouse/ClickHouse/pull/8144) ([Artem Zuikov](https://github.com/4ertus2)) -- Vectoriser le traitement `arrayReduce`. [\#7608](https://github.com/ClickHouse/ClickHouse/pull/7608) ([Amos Oiseau](https://github.com/amosbird)) -- `if` les chaînes sont maintenant optimisés `multiIf`. [\#8355](https://github.com/ClickHouse/ClickHouse/pull/8355) ([kamalov-ruslan](https://github.com/kamalov-ruslan)) -- Correction de la régression des performances de `Kafka` moteur de table introduit en 19.15. Cela corrige [\#7261](https://github.com/ClickHouse/ClickHouse/issues/7261). [\#7935](https://github.com/ClickHouse/ClickHouse/pull/7935) ([filimonov](https://github.com/filimonov)) -- Retiré “pie” génération de code qui `gcc` de paquets Debian apporte parfois par défaut. [\#8483](https://github.com/ClickHouse/ClickHouse/pull/8483) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Formats de données d’analyse parallèle [\#6553](https://github.com/ClickHouse/ClickHouse/pull/6553) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- Activer l’analyseur optimisé de `Values` avec des expressions par défaut (`input_format_values_deduce_templates_of_expressions=1`). [\#8231](https://github.com/ClickHouse/ClickHouse/pull/8231) ([tavplubix](https://github.com/tavplubix)) - -#### Construction / Test / Amélioration De L’Emballage {#buildtestingpackaging-improvement-2} - -- Construire des correctifs pour `ARM` et en un minimum de mode. [\#8304](https://github.com/ClickHouse/ClickHouse/pull/8304) ([proller](https://github.com/proller)) -- Ajouter le fichier de couverture flush pour `clickhouse-server` lorsque std::atexit n’est pas appelé. Également légèrement amélioré la journalisation dans les tests sans état avec la couverture. [\#8267](https://github.com/ClickHouse/ClickHouse/pull/8267) ([alésapine](https://github.com/alesapin)) -- Mettre à jour la bibliothèque LLVM dans contrib. Évitez D’utiliser LLVM à partir de paquets OS. [\#8258](https://github.com/ClickHouse/ClickHouse/pull/8258) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Faire empaqueté `curl` construire entièrement calme. [\#8232](https://github.com/ClickHouse/ClickHouse/pull/8232) [\#8203](https://github.com/ClickHouse/ClickHouse/pull/8203) ([Pavel Kovalenko](https://github.com/Jokser)) -- Correction de quelques `MemorySanitizer` avertissement. [\#8235](https://github.com/ClickHouse/ClickHouse/pull/8235) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- Utiliser `add_warning` et `no_warning` les macros dans `CMakeLists.txt`. [\#8604](https://github.com/ClickHouse/ClickHouse/pull/8604) ([Ivan](https://github.com/abyss7)) -- Ajout du support de l’objet compatible Minio S3 (https://min.io/) pour de meilleurs tests d’intégration. [\#7863](https://github.com/ClickHouse/ClickHouse/pull/7863) [\#7875](https://github.com/ClickHouse/ClickHouse/pull/7875) ([Pavel Kovalenko](https://github.com/Jokser)) -- Importer `libc` en-têtes à contrib. Il permet de rendre les builds plus cohérents sur différents systèmes (uniquement pour `x86_64-linux-gnu`). [\#5773](https://github.com/ClickHouse/ClickHouse/pull/5773) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Supprimer `-fPIC` à partir de certaines bibliothèques. [\#8464](https://github.com/ClickHouse/ClickHouse/pull/8464) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Propre `CMakeLists.txt` pour le roulage. Tu vois https://github.com/ClickHouse/ClickHouse/pull/8011\#issuecomment-569478910 [\#8459](https://github.com/ClickHouse/ClickHouse/pull/8459) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Avertissements silencieux dans `CapNProto` bibliothèque. [\#8220](https://github.com/ClickHouse/ClickHouse/pull/8220) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Ajouter des tests de performance pour les tables de hachage optimisées par chaîne courte. [\#7679](https://github.com/ClickHouse/ClickHouse/pull/7679) ([Amos Oiseau](https://github.com/amosbird)) -- Maintenant ClickHouse va construire sur `AArch64` même si `MADV_FREE` n’est pas disponible. Cela corrige [\#8027](https://github.com/ClickHouse/ClickHouse/issues/8027). [\#8243](https://github.com/ClickHouse/ClickHouse/pull/8243) ([Amos Oiseau](https://github.com/amosbird)) -- Mettre `zlib-ng` pour résoudre les problèmes de désinfectant de mémoire. [\#7182](https://github.com/ClickHouse/ClickHouse/pull/7182) [\#8206](https://github.com/ClickHouse/ClickHouse/pull/8206) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- Activez la bibliothèque MySQL interne sur un système non Linux, car l’utilisation des paquets du système D’exploitation est très fragile et ne fonctionne généralement pas du tout. Cela corrige [\#5765](https://github.com/ClickHouse/ClickHouse/issues/5765). [\#8426](https://github.com/ClickHouse/ClickHouse/pull/8426) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction de la construction sur certains systèmes après activation `libc++`. Cela annule et remplace [\#8374](https://github.com/ClickHouse/ClickHouse/issues/8374). [\#8380](https://github.com/ClickHouse/ClickHouse/pull/8380) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Faire `Field` méthodes plus de type-sûr pour trouver plus d’erreurs. [\#7386](https://github.com/ClickHouse/ClickHouse/pull/7386) [\#8209](https://github.com/ClickHouse/ClickHouse/pull/8209) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- Ajout de fichiers manquants à la `libc-headers` sous-module. [\#8507](https://github.com/ClickHouse/ClickHouse/pull/8507) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Corrigé de mal `JSON` citation dans la sortie de test de performance. [\#8497](https://github.com/ClickHouse/ClickHouse/pull/8497) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Maintenant, la trace de pile est affichée pour `std::exception` et `Poco::Exception`. Dans les versions précédentes, il était disponible uniquement pour `DB::Exception`. Cela améliore le diagnostic. [\#8501](https://github.com/ClickHouse/ClickHouse/pull/8501) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Le portage `clock_gettime` et `clock_nanosleep` pour les nouvelles versions glibc. [\#8054](https://github.com/ClickHouse/ClickHouse/pull/8054) ([Amos Oiseau](https://github.com/amosbird)) -- Permettre `part_log` dans l’exemple config pour les développeurs. [\#8609](https://github.com/ClickHouse/ClickHouse/pull/8609) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction de la nature asynchrone du rechargement dans `01036_no_superfluous_dict_reload_on_create_database*`. [\#8111](https://github.com/ClickHouse/ClickHouse/pull/8111) ([Azat Khuzhin](https://github.com/azat)) -- Tests de performance codec fixe. [\#8615](https://github.com/ClickHouse/ClickHouse/pull/8615) ([Vasily Nemkov](https://github.com/Enmk)) -- Ajouter des scripts d’installation pour `.tgz` construire et documentation pour eux. [\#8612](https://github.com/ClickHouse/ClickHouse/pull/8612) [\#8591](https://github.com/ClickHouse/ClickHouse/pull/8591) ([alésapine](https://github.com/alesapin)) -- Supprimé Vieux `ZSTD` test (il a été créé en 2016 pour reproduire le bug que la version pré 1.0 de ZSTD a eu). Cela corrige [\#8618](https://github.com/ClickHouse/ClickHouse/issues/8618). [\#8619](https://github.com/ClickHouse/ClickHouse/pull/8619) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction de la construction sur Mac OS Catalina. [\#8600](https://github.com/ClickHouse/ClickHouse/pull/8600) ([meo](https://github.com/meob)) -- Augmentation du nombre de lignes dans les tests de performance du codec pour rendre les résultats visibles. [\#8574](https://github.com/ClickHouse/ClickHouse/pull/8574) ([Vasily Nemkov](https://github.com/Enmk)) -- Dans les versions debug, traiter `LOGICAL_ERROR` exceptions comme Échecs d’assertion, de sorte qu’ils sont plus faciles à remarquer. [\#8475](https://github.com/ClickHouse/ClickHouse/pull/8475) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- Rendre le test de performance lié aux formats plus déterministe. [\#8477](https://github.com/ClickHouse/ClickHouse/pull/8477) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Mettre `lz4` pour corriger un échec MemorySanitizer. [\#8181](https://github.com/ClickHouse/ClickHouse/pull/8181) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- Supprimer un faux positif MemorySanitizer connu dans la gestion des exceptions. [\#8182](https://github.com/ClickHouse/ClickHouse/pull/8182) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- Mettre `gcc` et `g++` à la version 9 dans `build/docker/build.sh` [\#7766](https://github.com/ClickHouse/ClickHouse/pull/7766) ([TLightSky](https://github.com/tlightsky)) -- Ajoutez un cas de test de performance pour tester cela `PREWHERE` est pire que `WHERE`. [\#7768](https://github.com/ClickHouse/ClickHouse/pull/7768) ([Amos Oiseau](https://github.com/amosbird)) -- Progrès vers la fixation d’un test flacky. [\#8621](https://github.com/ClickHouse/ClickHouse/pull/8621) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Évitez le rapport MemorySanitizer pour les données de `libunwind`. [\#8539](https://github.com/ClickHouse/ClickHouse/pull/8539) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Mettre `libc++` la dernière version. [\#8324](https://github.com/ClickHouse/ClickHouse/pull/8324) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Construire la bibliothèque ICU à partir de sources. Cela corrige [\#6460](https://github.com/ClickHouse/ClickHouse/issues/6460). [\#8219](https://github.com/ClickHouse/ClickHouse/pull/8219) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Commutation de `libressl` de `openssl`. ClickHouse devrait prendre en charge TLS 1.3 et SNI après ce changement. Cela corrige [\#8171](https://github.com/ClickHouse/ClickHouse/issues/8171). [\#8218](https://github.com/ClickHouse/ClickHouse/pull/8218) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Rapport UBSan fixe lors de l’utilisation `chacha20_poly1305` de SSL (se produit sur la connexion à https://yandex.ru/). [\#8214](https://github.com/ClickHouse/ClickHouse/pull/8214) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Correction du mode de fichier de mot de passe par défaut pour `.deb` des distributions linux. [\#8075](https://github.com/ClickHouse/ClickHouse/pull/8075) ([proller](https://github.com/proller)) -- Expression améliorée pour obtenir `clickhouse-server` PID dans `clickhouse-test`. [\#8063](https://github.com/ClickHouse/ClickHouse/pull/8063) ([Alexander Kazakov](https://github.com/Akazz)) -- Mise à jour contrib / googletest à v1. 10. 0. [\#8587](https://github.com/ClickHouse/ClickHouse/pull/8587) ([Alexander Burmak](https://github.com/Alex-Burmak)) -- Rapport ThreadSaninitizer fixe dans `base64` bibliothèque. Aussi mis à jour cette bibliothèque à la dernière version, mais cela n’a pas d’importance. Cela corrige [\#8397](https://github.com/ClickHouse/ClickHouse/issues/8397). [\#8403](https://github.com/ClickHouse/ClickHouse/pull/8403) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Fixer `00600_replace_running_query` pour les transformateurs. [\#8272](https://github.com/ClickHouse/ClickHouse/pull/8272) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Supprimer le support pour `tcmalloc` faire `CMakeLists.txt` plus simple. [\#8310](https://github.com/ClickHouse/ClickHouse/pull/8310) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Libérer gcc construit maintenant utiliser `libc++` plutôt `libstdc++`. Récemment `libc++` a été utilisé uniquement avec clang. Cela améliorera la cohérence des configurations de construction et la portabilité. [\#8311](https://github.com/ClickHouse/ClickHouse/pull/8311) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Activer la bibliothèque ICU pour construire avec MemorySanitizer. [\#8222](https://github.com/ClickHouse/ClickHouse/pull/8222) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Supprimer les avertissements de `CapNProto` bibliothèque. [\#8224](https://github.com/ClickHouse/ClickHouse/pull/8224) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Suppression de cas spéciaux de code pour `tcmalloc` parce que c’est plus pris en charge. [\#8225](https://github.com/ClickHouse/ClickHouse/pull/8225) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Dans la tâche de couverture CI, tuez le serveur gracieusement pour lui permettre d’enregistrer le rapport de couverture. Cela corrige les rapports de couverture incomplets que nous avons vus récemment. [\#8142](https://github.com/ClickHouse/ClickHouse/pull/8142) ([alésapine](https://github.com/alesapin)) -- Tests de Performance pour tous les codecs contre `Float64` et `UInt64` valeur. [\#8349](https://github.com/ClickHouse/ClickHouse/pull/8349) ([Vasily Nemkov](https://github.com/Enmk)) -- `termcap` est très obsolète et conduit à divers problèmes (F. G. manquant “up” cap et en écho `^J` au lieu de multi-ligne) . Faveur `terminfo` ou groupés `ncurses`. [\#7737](https://github.com/ClickHouse/ClickHouse/pull/7737) ([Amos Oiseau](https://github.com/amosbird)) -- Fixer `test_storage_s3` test d’intégration. [\#7734](https://github.com/ClickHouse/ClickHouse/pull/7734) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Soutien `StorageFile(, null)` pour insérer un bloc dans un fichier de format donné sans écrire sur le disque. Ceci est requis pour les tests de performance. [\#8455](https://github.com/ClickHouse/ClickHouse/pull/8455) ([Amos Oiseau](https://github.com/amosbird)) -- Argument supplémentaire en `--print-time` aux tests fonctionnels qui imprime le temps d’exécution par test. [\#8001](https://github.com/ClickHouse/ClickHouse/pull/8001) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Ajouté assertions à `KeyCondition` lors de L’évaluation RPN. Cela corrigera l’avertissement de gcc-9. [\#8279](https://github.com/ClickHouse/ClickHouse/pull/8279) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Vider les options cmake dans les builds CI. [\#8273](https://github.com/ClickHouse/ClickHouse/pull/8273) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- Ne générez pas d’informations de débogage pour certaines bibliothèques fat. [\#8271](https://github.com/ClickHouse/ClickHouse/pull/8271) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Faire `log_to_console.xml` connectez-vous toujours à stderr, que ce soit interactif ou non. [\#8395](https://github.com/ClickHouse/ClickHouse/pull/8395) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- Suppression de certaines fonctionnalités inutilisées de `clickhouse-performance-test` outil. [\#8555](https://github.com/ClickHouse/ClickHouse/pull/8555) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Maintenant, nous allons également rechercher `lld-X` avec correspondant `clang-X` version. [\#8092](https://github.com/ClickHouse/ClickHouse/pull/8092) ([alésapine](https://github.com/alesapin)) -- Amélioration de construction de Parquet. [\#8421](https://github.com/ClickHouse/ClickHouse/pull/8421) ([maxulan](https://github.com/maxulan)) -- Plus D’Avertissements GCC [\#8221](https://github.com/ClickHouse/ClickHouse/pull/8221) ([kreuzerkrieg](https://github.com/kreuzerkrieg)) -- Package pour Arch Linux permet maintenant d’exécuter le serveur ClickHouse, et pas seulement le client. [\#8534](https://github.com/ClickHouse/ClickHouse/pull/8534) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Fixer le test avec les processeurs. Corrections de performances minuscules. [\#7672](https://github.com/ClickHouse/ClickHouse/pull/7672) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Mise à jour contrib/protobuf. [\#8256](https://github.com/ClickHouse/ClickHouse/pull/8256) ([Matwey V. Kornilov](https://github.com/matwey)) -- En préparation du passage à c++20 comme une célébration du Nouvel An. “May the C++ force be with ClickHouse.” [\#8447](https://github.com/ClickHouse/ClickHouse/pull/8447) ([Amos Oiseau](https://github.com/amosbird)) - -#### Caractéristique Expérimentale {#experimental-feature-1} - -- Ajouté cadre expérimental `min_bytes_to_use_mmap_io`. Il permet de lire de gros fichiers sans copier les données du noyau vers l’espace utilisateur. Le paramètre est désactivé par défaut. Le seuil recommandé est d’environ 64 Mo, car mmap / munmap est lent. [\#8520](https://github.com/ClickHouse/ClickHouse/pull/8520) ([alexeï-milovidov](https://github.com/alexey-milovidov)) -- Quotas retravaillés dans le cadre du système de contrôle d’accès. Ajouté nouveau tableau `system.quotas` de nouvelles fonctions `currentQuota`, `currentQuotaKey`, nouvelle syntaxe SQL `CREATE QUOTA`, `ALTER QUOTA`, `DROP QUOTA`, `SHOW QUOTA`. [\#7257](https://github.com/ClickHouse/ClickHouse/pull/7257) ([Vitaly Baranov](https://github.com/vitlibar)) -- Autoriser à sauter des paramètres inconnus avec des avertissements au lieu de lancer des exceptions. [\#7653](https://github.com/ClickHouse/ClickHouse/pull/7653) ([Vitaly Baranov](https://github.com/vitlibar)) -- Stratégies de ligne retravaillées dans le cadre du système de contrôle d’accès. Ajouté nouveau tableau `system.row_policies`, nouvelle fonction `currentRowPolicies()`, nouvelle syntaxe SQL `CREATE POLICY`, `ALTER POLICY`, `DROP POLICY`, `SHOW CREATE POLICY`, `SHOW POLICIES`. [\#7808](https://github.com/ClickHouse/ClickHouse/pull/7808) ([Vitaly Baranov](https://github.com/vitlibar)) - -#### Correction De Sécurité {#security-fix} - -- Correction de la possibilité de lire la structure des répertoires dans les tables avec `File` tableau moteur. Cela corrige [\#8536](https://github.com/ClickHouse/ClickHouse/issues/8536). [\#8537](https://github.com/ClickHouse/ClickHouse/pull/8537) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - -## [Changelog pour 2019](https://github.com/ClickHouse/ClickHouse/blob/master/docs/en/changelog/2019.md) {#changelog-for-2019} +{% include "content/changelog.md" %} diff --git a/docs/fr/whats-new/index.md b/docs/fr/whats-new/index.md index 787b260a862..51a77da8ef4 100644 --- a/docs/fr/whats-new/index.md +++ b/docs/fr/whats-new/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 -toc_folder_title: What's New +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: Ce qui est Nouveau toc_priority: 72 --- diff --git a/docs/fr/whats-new/roadmap.md b/docs/fr/whats-new/roadmap.md index 87ee98be5ea..87d64208f67 100644 --- a/docs/fr/whats-new/roadmap.md +++ b/docs/fr/whats-new/roadmap.md @@ -1,19 +1,19 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 74 toc_title: Feuille de route --- -# Feuille De Route {#roadmap} +# Feuille de route {#roadmap} ## Q1 2020 {#q1-2020} -- Contrôle d’accès par rôle +- Contrôle d'accès par rôle ## Q2 2020 {#q2-2020} -- Intégration avec les services d’authentification externes +- Intégration avec les services d'authentification externes - Pools de ressources pour une répartition plus précise de la capacité du cluster entre les utilisateurs {## [Article Original](https://clickhouse.tech/docs/en/roadmap/) ##} diff --git a/docs/fr/whats-new/security-changelog.md b/docs/fr/whats-new/security-changelog.md index 07bce67aa2f..bef992b7a77 100644 --- a/docs/fr/whats-new/security-changelog.md +++ b/docs/fr/whats-new/security-changelog.md @@ -1,63 +1,63 @@ --- machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 76 toc_title: "S\xE9curit\xE9 Changelog" --- -## Correction Dans La Version 19.14.3.3 De ClickHouse, 2019-09-10 {#fixed-in-clickhouse-release-19-14-3-3-2019-09-10} +## Correction dans la version 19.14.3.3 de ClickHouse, 2019-09-10 {#fixed-in-clickhouse-release-19-14-3-3-2019-09-10} ### CVE-2019-15024 {#cve-2019-15024} Аn attacker that has write access to ZooKeeper and who ican run a custom server available from the network where ClickHouse runs, can create a custom-built malicious server that will act as a ClickHouse replica and register it in ZooKeeper. When another replica will fetch data part from the malicious replica, it can force clickhouse-server to write to arbitrary path on filesystem. -Crédits: Eldar Zaitov de L’équipe de sécurité de L’Information Yandex +Crédits: Eldar Zaitov de L'équipe de sécurité de L'Information Yandex ### CVE-2019-16535 {#cve-2019-16535} Аn OOB read, OOB write and integer underflow in decompression algorithms can be used to achieve RCE or DoS via native protocol. -Crédits: Eldar Zaitov de L’équipe de sécurité de L’Information Yandex +Crédits: Eldar Zaitov de L'équipe de sécurité de L'Information Yandex ### CVE-2019-16536 {#cve-2019-16536} Le débordement de pile menant à DoS peut être déclenché par un client authentifié malveillant. -Crédits: Eldar Zaitov de L’équipe de sécurité de L’Information Yandex +Crédits: Eldar Zaitov de L'équipe de sécurité de L'Information Yandex -## Correction De La Version 19.13.6.1 De ClickHouse, 2019-09-20 {#fixed-in-clickhouse-release-19-13-6-1-2019-09-20} +## Correction de la version 19.13.6.1 de ClickHouse, 2019-09-20 {#fixed-in-clickhouse-release-19-13-6-1-2019-09-20} ### CVE-2019-18657 {#cve-2019-18657} -Fonction de Table `url` la vulnérabilité avait-elle permis à l’attaquant d’injecter des en-têtes HTTP arbitraires dans la requête. +Fonction de Table `url` la vulnérabilité avait-elle permis à l'attaquant d'injecter des en-têtes HTTP arbitraires dans la requête. Crédit: [Nikita Tikhomirov](https://github.com/NSTikhomirov) -## Correction Dans La Version ClickHouse 18.12.13, 2018-09-10 {#fixed-in-clickhouse-release-18-12-13-2018-09-10} +## Correction dans la version ClickHouse 18.12.13, 2018-09-10 {#fixed-in-clickhouse-release-18-12-13-2018-09-10} ### CVE-2018-14672 {#cve-2018-14672} -Les fonctions de chargement des modèles CatBoost permettaient de parcourir les chemins et de lire des fichiers arbitraires via des messages d’erreur. +Les fonctions de chargement des modèles CatBoost permettaient de parcourir les chemins et de lire des fichiers arbitraires via des messages d'erreur. -Crédits: Andrey Krasichkov de L’équipe de sécurité de L’Information Yandex +Crédits: Andrey Krasichkov de L'équipe de sécurité de L'Information Yandex -## Correction Dans La Version 18.10.3 De ClickHouse, 2018-08-13 {#fixed-in-clickhouse-release-18-10-3-2018-08-13} +## Correction dans la version 18.10.3 de ClickHouse, 2018-08-13 {#fixed-in-clickhouse-release-18-10-3-2018-08-13} ### CVE-2018-14671 {#cve-2018-14671} -unixODBC a permis de charger des objets partagés arbitraires à partir du système de fichiers, ce qui a conduit à une vulnérabilité D’exécution de Code À Distance. +unixODBC a permis de charger des objets partagés arbitraires à partir du système de fichiers, ce qui a conduit à une vulnérabilité D'exécution de Code À Distance. Crédits: Andrey Krasichkov et Evgeny Sidorov de Yandex Information Security Team -## Correction Dans La Version 1.1.54388 De ClickHouse, 2018-06-28 {#fixed-in-clickhouse-release-1-1-54388-2018-06-28} +## Correction dans la version 1.1.54388 de ClickHouse, 2018-06-28 {#fixed-in-clickhouse-release-1-1-54388-2018-06-28} ### CVE-2018-14668 {#cve-2018-14668} “remote” la fonction de table a permis des symboles arbitraires dans “user”, “password” et “default\_database” champs qui ont conduit à des attaques de falsification de requêtes inter-protocoles. -Crédits: Andrey Krasichkov de L’équipe de sécurité de L’Information Yandex +Crédits: Andrey Krasichkov de L'équipe de sécurité de L'Information Yandex -## Correction Dans La Version 1.1.54390 De ClickHouse, 2018-07-06 {#fixed-in-clickhouse-release-1-1-54390-2018-07-06} +## Correction dans la version 1.1.54390 de ClickHouse, 2018-07-06 {#fixed-in-clickhouse-release-1-1-54390-2018-07-06} ### CVE-2018-14669 {#cve-2018-14669} @@ -65,11 +65,11 @@ Clickhouse client MySQL avait “LOAD DATA LOCAL INFILE” fonctionnalité activ Crédits: Andrey Krasichkov et Evgeny Sidorov de Yandex Information Security Team -## Correction Dans La Version 1.1.54131 De ClickHouse, 2017-01-10 {#fixed-in-clickhouse-release-1-1-54131-2017-01-10} +## Correction dans la version 1.1.54131 de ClickHouse, 2017-01-10 {#fixed-in-clickhouse-release-1-1-54131-2017-01-10} ### CVE-2018-14670 {#cve-2018-14670} -Configuration incorrecte dans le paquet deb pourrait conduire à l’utilisation non autorisée de la base de données. +Configuration incorrecte dans le paquet deb pourrait conduire à l'utilisation non autorisée de la base de données. Crédits: National Cyber Security Centre (NCSC) diff --git a/docs/ja/commercial/cloud.md b/docs/ja/commercial/cloud.md index e98d24bb528..403b34d198c 100644 --- a/docs/ja/commercial/cloud.md +++ b/docs/ja/commercial/cloud.md @@ -1,21 +1,23 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_priority: 1 +toc_title: "\u30AF\u30E9\u30A6\u30C9" --- -# ClickHouseしてクラウドサービスプロバイダ {#clickhouse-cloud-service-providers} +# ClickHouseの雲のサービス提供者 {#clickhouse-cloud-service-providers} !!! info "情報" - の場合において公共クラウド管理clickhouseサービス、お気軽に [プルリクエストを開く](https://github.com/ClickHouse/ClickHouse/edit/master/docs/en/commercial/cloud.md) それを次のリストに追加します。 + Managed ClickHouse serviceを使用してパブリッククラウドを起動した場合は、以下をお気軽にご利用ください [プル要求を開く](https://github.com/ClickHouse/ClickHouse/edit/master/docs/en/commercial/cloud.md) それを次のリストに追加します。 ## Yandexクラウド {#yandex-cloud} -[Yandex管理サービスClickHouse](https://cloud.yandex.com/services/managed-clickhouse?utm_source=referrals&utm_medium=clickhouseofficialsite&utm_campaign=link3) 次の主な機能を提供します: +[ClickHouseのためのYandexの管理サービス](https://cloud.yandex.com/services/managed-clickhouse?utm_source=referrals&utm_medium=clickhouseofficialsite&utm_campaign=link3) 次の主な機能を提供します: -- 完全に管理されたzookeeperサービス [ClickHouse複製](../engines/table-engines/mergetree-family/replication.md) -- 複数ストレージタイプの選択 +- 完全に管理された飼育係サービス [クリックハウス複製](../engines/table-engines/mergetree-family/replication.md) +- 複数の記憶域タイプの選択肢 - 異なる可用性ゾーンのレプリカ - 暗号化と分離 -- 自動保守 +- 自動メンテナンス {## [元の記事](https://clickhouse.tech/docs/en/commercial/cloud/) ##} diff --git a/docs/ja/commercial/index.md b/docs/ja/commercial/index.md index 71bc1afac05..75e13112d9e 100644 --- a/docs/ja/commercial/index.md +++ b/docs/ja/commercial/index.md @@ -1,8 +1,9 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_folder_title: Commercial +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "\u5546\u696D" toc_priority: 70 +toc_title: "\u5546\u696D" --- diff --git a/docs/ja/development/architecture.md b/docs/ja/development/architecture.md index e0f80efdf2b..3b1da1b8552 100644 --- a/docs/ja/development/architecture.md +++ b/docs/ja/development/architecture.md @@ -1,203 +1,203 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 62 -toc_title: "\u30AF\u30EA\u30C3\u30AF\u30CF\u30A6\u30B9\u306E\u6982\u8981" +toc_title: "\u30AF\u30EA\u30C3\u30AF\u30CF\u30A6\u30B9\u5EFA\u7BC9\u306E\u6982\u8981" --- -# クリックハウスの概要 {#overview-of-clickhouse-architecture} +# クリックハウス建築の概要 {#overview-of-clickhouse-architecture} -ClickHouseは真の列指向DBMSです。 データは、列によって、および配列（ベクトルまたは列のチャンク）の実行中に格納されます。 可能な限り、操作は個々の値ではなく、配列にディスパッチされます。 それは呼ばれます “vectorized query execution,” そしてそれは実際のデータ処理の費用を下げるのを助けます。 +ClickHouseは真の列指向のDBMSです。 データは、列によって、および配列（列のベクトルまたはチャンク）の実行中に格納されます。 可能であれば、個々の値ではなく配列に対して演算が送出されます。 それは呼ばれます “vectorized query execution,” そしてそれは実際のデータ処理の費用を下げるのを助けます。 -> この考え方は、新しいものではない。 それはに遡ります `APL` プログラミング言語とその子孫: `A +`, `J`, `K`、と `Q`. 配列プログラミングは科学的データ処理に使用されます。 このアイデアは、リレーショナルデータベースで新しいものでもありません。 `Vectorwise` システム。 +> この考え方は、新しいものではない。 それはにさかのぼります `APL` プログラミング言語とその子孫: `A +`, `J`, `K`,and `Q`. 配列プログラミングは科学データ処理で使用されます。 どちらもこの考え方は関係データベースで新しいものではありません。 `Vectorwise` システム -クエリ処理の高速化には、vectorizedクエリの実行とランタイムコードの生成という二つのアプローチがあります。 後者は、すべての間接指定と動的ディスパッチを削除します。 ずれのアプローチは厳重によります。 ランタイムコードを生成するときにヒューズが多く、業務として活用cpuの実行単位のパイプライン vectorizedクエリを実行できる実用的では一時的ベクトルを明記のことは、キャッシュを読みます。 一時的なデータがl2キャッシュに収まらない場合、これが問題になります。 しかし、ベクトル化されたクエリの実行は、cpuのsimd機能をより簡単に利用します。 a [研究論文](http://15721.courses.cs.cmu.edu/spring2016/papers/p5-sompolski.pdf) 書面による友人達しないことであり、両アプローチ。 ClickHouse用vectorizedクエリを実行して初期支援のためにランタイムコード。 +クエリ処理の高速化には、ベクトル化されたクエリ実行とランタイムコード生成という二つの方法があります。 後者は、すべての間接および動的ディスパッチを削除します。 ずれのアプローチは厳重によります。 ランタイムコード生成は、多くの操作を融合し、CPU実行単位とパイプラインを完全に利用する場合に優れています。 Vectorizedクエリを実行できる実用的では一時的ベクトルを明記のことは、キャッシュを読みます。 一時データがL2キャッシュに収まらない場合、これが問題になります。 しかし、ベクトル化されたクエリの実行は、CPUのSIMD機能をより簡単に利用します。 A [研究論文](http://15721.courses.cs.cmu.edu/spring2016/papers/p5-sompolski.pdf) 書面による友人達しないことであり、両アプローチ。 ClickHouse用vectorizedクエリを実行して初期支援のためにランタイムコード。 ## 列 {#columns} -`IColumn` interfaceは、メモリ内の列（実際には列のチャンク）を表すために使用されます。 このインタフェースのヘルパーの方法の実施のための様々な関係です。 ほとんどすべての操作は不変です：元の列は変更されませんが、新しい変更された列を作成します。 たとえば、 `IColumn :: filter` 法を受け入れフィルタのバイトマスクです。 それはのために使用されます `WHERE` と `HAVING` 関係演算子。 その他の例： `IColumn :: permute` サポートする方法 `ORDER BY`、を `IColumn :: cut` サポートする方法 `LIMIT`. +`IColumn` interfaceは、メモリ内の列（実際には列のチャンク）を表すために使用されます。 このイ 元の列を変更するのではなく、新しい変更された列を作成します。 例えば、 `IColumn :: filter` 法を受け入れフィルタのバイトマスクです。 それはのために使用されます `WHERE` と `HAVING` 関係演算子。 追加の例： `IColumn :: permute` サポートする方法 `ORDER BY` は、 `IColumn :: cut` サポートする方法 `LIMIT`. -様々な `IColumn` 実装 (`ColumnUInt8`, `ColumnString`、というように）列のメモリレイアウトを担当しています。 メモリレイアウトは、通常、連続した配列です。 列の整数型の場合、次のような連続した配列にすぎません `std :: vector`. のために `String` と `Array` すべての配列要素のベクトル、連続して配置されたベクトル、および各配列の先頭にオフセットするためのベクトルです。 また、 `ColumnConst` これはメモリに一つの値だけを格納しますが、列のように見えます。 +各種 `IColumn` 実装 (`ColumnUInt8`, `ColumnString` というように）は、列のメモリレイアウトを担当しています。 メモリレイアウトは、通常、連続した配列です。 整数型の列の場合は、次のように連続した配列にすぎません `std :: vector`. のために `String` と `Array` すべての配列要素に対して一つ、連続して配置され、各配列の先頭へのオフセットに対して二つのベクトルです。 また `ColumnConst` これはメモリに一つの値を格納しますが、列のように見えます。 ## フィールド {#field} -それにもかかわらず、個々の価値を扱うことも可能です。 個々の値を表すには、 `Field` 使用される。 `Field` のちょうど差別された組合である `UInt64`, `Int64`, `Float64`, `String` と `Array`. `IColumn` は、 `operator[]` n番目の値をaとして取得するメソッド `Field` そして `insert` aを追加するメソッド `Field` 列の終わりまで。 これらの方法はあまり効率的ではありません。 `Field` 個々の値を表すオブジェクト。 より効率的な方法があります。 `insertFrom`, `insertRangeFrom`、というように。 +それにもかかわらず、個々の価値を扱うことも可能です。 個々の値を表すために、 `Field` が使用される。 `Field` ただの識別された組合である `UInt64`, `Int64`, `Float64`, `String` と `Array`. `IColumn` は、 `operator[]` n番目の値をaとして取得するメソッド `Field` そして、 `insert` aを追加するメソッド `Field` 列の最後まで。 これらの方法は、一時的な処理を必要とするため、あまり効率的ではありません `Field` 個々の値を表すオブジェクト。 次のようなより効率的な方法があります `insertFrom`, `insertRangeFrom`、というように。 -`Field` テーブルの特定のデータ型に関する十分な情報がありません。 例えば, `UInt8`, `UInt16`, `UInt32`、と `UInt64` すべてとして表されます `UInt64` で `Field`. +`Field` テーブルの特定のデータ型に関する十分な情報がありません。 例えば, `UInt8`, `UInt16`, `UInt32`,and `UInt64` すべてとして表されます `UInt64` で `Field`. ## 漏れやすい抽象化 {#leaky-abstractions} -`IColumn` は方法のための共通の関係変容のデータもあるんですが、そのすべて満たす。 例えば, `ColumnUInt64` 二つの列の合計を計算する方法を持っていない、と `ColumnString` 部分文字列検索を実行するメソッドはありません。 これらの無数のルーチンは、 `IColumn`. +`IColumn` は方法のための共通の関係変容のデータもあるんですが、そのすべて満たす。 例えば, `ColumnUInt64` 二つの列の合計を計算する方法がありません。 `ColumnString` 部分文字列検索を実行するメソッドがありません。 これらの無数のルーチンは `IColumn`. -列のさまざまな関数は、次のように一般的で効率的でない方法で実装できます `IColumn` 抽出する方法 `Field` 値、または特定のデータの内部メモリレイアウトの知識を使用して、特殊な方法で `IColumn` 実装。 これは、関数を特定の `IColumn` 内部表現を直接入力して処理する。 例えば, `ColumnUInt64` は、 `getData` 内部配列への参照を返し、別のルーチンがその配列を直接読み取ったり塗りつぶしたりするメソッドです。 我々は持っている “leaky abstractions” さまざまなルーチンの効率的な専門化を可能にする。 +列のさまざまな関数は、次のようにして、一般的で非効率的な方法で実装できます `IColumn` 抽出する方法 `Field` 値、または特定のデータの内部メモリレイアウトの知識を使用して特殊な方法で `IColumn` 実装。 で実施する鋳造機能を特定 `IColumn` 内部表現を直接入力して処理します。 例えば, `ColumnUInt64` は、 `getData` 内部配列への参照を返すメソッドは、別のルーチンが直接その配列を読み取るか、または塗りつぶします。 我々は持っている “leaky abstractions” さまざまなルーチンの効率的な専門化を可能にする。 ## データ型 {#data_types} -`IDataType` バイナリ形式またはテキスト形式で列または個々の値のチャンクを読み書きするためのシリアル化と逆シリアル化を担当します。 `IDataType` テーブルのデータ型に直接対応します。 たとえば、次のとおりです `DataTypeUInt32`, `DataTypeDateTime`, `DataTypeString` というように。 +`IDataType` 列または個々の値のチャンクをバイナリ形式またはテキスト形式で読み書きするためのものです。 `IDataType` テーブル内のデータ型に直接対応します。 例えば、次のものがあります `DataTypeUInt32`, `DataTypeDateTime`, `DataTypeString` など。 -`IDataType` と `IColumn` 互いにゆるやかに関係しているだけです 異なるデータ型は同じでメモリ内に表すことができます `IColumn` 実装。 例えば, `DataTypeUInt32` と `DataTypeDateTime` どちらも `ColumnUInt32` または `ColumnConstUInt32`. また、同じデータ型を異なるデータ型で表すこともできます `IColumn` 実装。 例えば, `DataTypeUInt8` で表すことができます `ColumnUInt8` または `ColumnConstUInt8`. +`IDataType` と `IColumn` 互いに緩やかに関連しているだけです。 異なるデータ型は、同じメモリで表すことができます `IColumn` 実装。 例えば, `DataTypeUInt32` と `DataTypeDateTime` で表される。 `ColumnUInt32` または `ColumnConstUInt32`. また、同じデータ型で表現することができな `IColumn` 実装。 例えば, `DataTypeUInt8` で表すことができる `ColumnUInt8` または `ColumnConstUInt8`. -`IDataType` 貨物のメタデータを指すものとします。 例えば, `DataTypeUInt8` 何も全く保存しません(vptrを除きます)。 `DataTypeFixedString` 店だけ `N` （固定サイズの文字列のサイズ）。 +`IDataType` 貨物のメタデータを指すものとします。 例えば, `DataTypeUInt8` 何も保存しません(vptrを除きます)。 `DataTypeFixedString` ちょうど店 `N` (固定サイズの文字列のサイズ)。 -`IDataType` はヘルパーの方法のための様々なデータフォーマット たとえば、引用符で値をシリアル化したり、JSONの値をシリアル化したり、XML形式の一部として値をシリアル化したりするメソッドがあります。 データ形式への直接の対応はありません。 たとえば、さまざまなデータ形式 `Pretty` と `TabSeparated` 同じを使用できます `serializeTextEscaped` からヘルパーメソッド `IDataType` インタフェース +`IDataType` はヘルパーの方法のための様々なデータフォーマット たとえば、クォート可能な値をシリアル化したり、JSONの値をシリアル化したり、XML形式の一部として値をシリアル化したりするメソッドがあります。 データ形式への直接の対応はありません。 たとえば、異なるデータ形式 `Pretty` と `TabSeparated` 同じを使用できます `serializeTextEscaped` からのヘルパーメソッド `IDataType` インタフェース ## ブロック {#block} -A `Block` メモリ内のテーブルのサブセット(チャンク)を表すコンテナです。 それはちょうどトリプルのセットです: `(IColumn, IDataType, column name)`. クエリの実行中、データは次の方法で処理されます `Block`我々が持っている場合 `Block`、我々はデータを持っている（で `IColumn` オブジェクト）、そのタイプに関する情報があります `IDataType`）それはその列をどう扱うかを教えてくれるので、列名があります。 テーブルの元の列名か、計算の一時的な結果を得るために割り当てられた人工的な名前のいずれかになります。 +A `Block` メモリ内のテーブルのサブセット（チャンク）を表すコンテナです。 それは単なるトリプルのセットです: `(IColumn, IDataType, column name)`. クエリの実行中、データは次の方法で処理されます `Block`s.私達にaがあれば `Block`,我々はデータを持っている(で `IColumn` そのタイプに関する情報があります `IDataType`）それはその列をどのように扱うかを教えてくれます。 これは、テーブルの元の列名か、一時的な計算結果を取得するために割り当てられた人工的な名前のいずれかです。 -ブロック内の列に対していくつかの関数を計算すると、その結果を持つ別の列がブロックに追加され、関数の引数の列には触れません。 その後、不要な列はブロックから削除できますが、変更はできません。 それは共通の部分式の除去のために便利です。 +ブロック内の列に対して関数を計算するとき、その結果を含む別の列をブロックに追加します。 後で、不要な列はブロックから削除できますが、変更はできません。 共通の部分式を排除するのに便利です。 -ブロックの作成のための各処理チャンクのデータです。 同じタイプの計算では、列名とタイプは異なるブロックで同じままで、列データのみが変更されることに注意してください。 ブロックヘッダーからブロックデータを分割する方が良いのは、小さなブロックサイズでは、shared\_ptrsと列名をコピーするための一時文字列のオーバーヘッドが +ブロックの作成のための各処理チャンクのデータです。 同じタイプの計算では、列名と型は異なるブロックで同じままであり、列データのみが変更されることに注意してください。 ブロックサイズが小さいと、shared\_ptrsと列名をコピーするための一時的な文字列のオーバーヘッドが高くなるため、ブロックヘッダーからブロックデータを分割 ## ブロックの流れ {#block-streams} -ブロッ を使用していま流のブロックからデータを読み込むためのどこかに、データ変換、または書き込みデータをどこかということです。 `IBlockInputStream` は、 `read` 利用可能な間に次のブロックを取得するメソッドです。 `IBlockOutputStream` は、 `write` ブロックをどこかにプッシュする方法。 +ブロックストリームのための加工データです。 を使用していま流のブロックからデータを読み込むためのどこかに、データ変換、または書き込みデータをどこかということです。 `IBlockInputStream` は、 `read` 利用可能な状態で次のブロックを取得するメソッド。 `IBlockOutputStream` は、 `write` どこかにブロックをプッシュする方法。 -ストリー: +ストリームは: 1. テーブルへの読み書き。 のテーブルだけを返しますストリームを読み取りまたは書き込みブロックとなります。 -2. データ形式の実装。 たとえば、データを端末に出力する場合は、 `Pretty` ブロックをプッシュするブロック出力ストリームを作成し、それらをフォーマットします。 -3. データ変換の実行。 たとえば、 `IBlockInputStream` いを作ろうというストリームです。 あなたが作成する `FilterBlockInputStream` ストリームで初期化します。 その後、ブロックを引き出すと `FilterBlockInputStream` で引きブロックからストリーム、フィルタでは、フィルタを返しますブロックします。 クエリの実行パイプラインで表現しました。 +2. データ形式の実装。 たとえば、端末にデータを出力する場合は `Pretty` 書式設定すると、ブロックをプッシュするブロック出力ストリームを作成し、書式設定します。 +3. データ変換の実行。 あなたが持っているとしよう `IBlockInputStream` いを作ろうというストリームです。 作成する `FilterBlockInputStream` ストリームで初期化します その後、ブロックを引っ張るときから `FilterBlockInputStream` で引きブロックからストリーム、フィルタでは、フィルタを返しますブロックします。 クエリの実行パイプラインで表現しました。 -より洗練された変換があります。 たとえば、あなたから引くとき `AggregatingBlockInputStream` ソースからすべてのデータを読み取り、集計してから、集計データのストリームを返します。 別の例: `UnionBlockInputStream` コンストラクタ内の多数の入力ソースと、多数のスレッドを受け入れます。 複数のスレッドを起動し、複数のソースから並行して読み取ります。 +より洗練された変換があります。 例えば、 `AggregatingBlockInputStream` で読み込みのすべてのデータからのフレームワークを利用して、集合体で、それを返しますストリームの集約トへすでに使用されています。 別の例: `UnionBlockInputStream` コンストラクタ内の多くの入力ソースと多数のスレッドを受け入れます。 複数のスレッドを起動し、複数のソースから並列に読み込みます。 -> ブロックストリームは、 “pull” 制御フローへのアプローチ：最初のストリームからブロックをプルすると、ネストされたストリームから必要なブロックがプルされ、実行パイプライン全体 どちらも “pull” また “push” 制御フローは暗黙的であり、複数のクエリの同時実行（多くのパイプラインを一緒にマージする）などのさまざまな機能の実装を制限するため、最適なソ この制限は、コルーチンや、お互いを待つ余分なスレッドを実行するだけで克服することができます。 ある計算単位から別の計算単位にデータを渡すためのロジックを特定すると、制御フローを明示的にすると、より多くの可能性があります。 これを読む [記事](http://journal.stuffwithstuff.com/2013/01/13/iteration-inside-and-out/) より多くの思考のために。 +> ブロックストリームは “pull” 制御フローへのアプローチ：最初のストリームからブロックをプルすると、ネストされたストリームから必要なブロックがプルされ、実行パイプライン全体 どちらも “pull” また “push” 制御フローは暗黙的であり、複数のクエリの同時実行（多くのパイプラインをマージする）などのさまざまな機能の実装を制限するため、最良の解決策で この制限は、コルーチンまたはお互いを待つ余分なスレッドを実行するだけで克服できます。 つまり、ある計算単位から別の計算単位の外部にデータを渡すロジックを見つけると、制御フローを明示的にすると、より多くの可能性があります。 これを読む [記事](http://journal.stuffwithstuff.com/2013/01/13/iteration-inside-and-out/) より多くの思考のため。 -クエリ実行パイプラインは、各ステップで一時的なデータを作成します。 ブロックサイズを十分に小さくして、一時的なデータがcpuキャッシュに収まるようにします。 その仮定では、一時的なデータの書き込みと読み取りは、他の計算と比較してほとんど無料です。 したことを検討してはどうかと思うの代替、ヒューズの多くの業務のパイプラインです。 パイプラインをできるだけ短くし、一時的なデータの多くを削除することができますが、これは利点かもしれませんが、欠点もあります。 たとえば、分割パイプラインを使用すると、中間データのキャッシュ、同時に実行される類似クエリからの中間データの盗み取り、類似クエリのパイプラ +クエリ実行パイプラインでは、各ステップで一時データが作成されます。 一時データがCPUキャッシュに収まるように、ブロックサイズを十分に小さくしておきます。 その前提では、一時データの書き込みと読み込みは、他の計算と比較してほとんど自由です。 これは、パイプライン内の多くの操作を一緒に融合させることです。 パイプラインをできるだけ短くし、一時データの多くを削除することができますが、これは利点ですが、欠点もあります。 たとえば、分割パイプラインを使用すると、中間データのキャッシュ、同時に実行される類似クエリからの中間データの盗み、類似クエリのパイプライン ## 形式 {#formats} -データフォーマットにて実施しブロックわれている。 あります “presentational” 次のように、クライアントへのデータの出力にのみ適した書式を設定します `Pretty` のみを提供する形式 `IBlockOutputStream`. また、次のような入力/出力形式があります `TabSeparated` または `JSONEachRow`. +データフォーマットにて実施しブロックわれている。 そこには “presentational” 次のように、クライアントへのデータ出力にのみ適した形式になります `Pretty` のみを提供する形式 `IBlockOutputStream`. そして入出力フォーマットが、のようなあります `TabSeparated` または `JSONEachRow`. -行ストリームもあります: `IRowInputStream` と `IRowOutputStream`. ブロックではなく、個々の行でデータをプル/プッシュすることができます。 また、行指向のフォーマットの実装を単純化するためにのみ必要です。 その他の機能 `BlockInputStreamFromRowInputStream` と `BlockOutputStreamFromRowOutputStream` 行指向のストリームを通常のブロック指向のストリームに変換できます。 +行ストリームもあります: `IRowInputStream` と `IRowOutputStream`. ブロックではなく、個々の行でデータをプル/プッシュすることができます。 また、行指向形式の実装を簡素化するためにのみ必要です。 ラッパー `BlockInputStreamFromRowInputStream` と `BlockOutputStreamFromRowOutputStream` 行指向のストリームを通常のブロック指向のストリームに変換できます。 ## I/O {#io} -バイト指向の入出力には、以下のものがあります `ReadBuffer` と `WriteBuffer` 抽象クラス。 それらはC++の代わりに使用されます `iostream`すべての成熟したC++プロジェクトは、 `iostream`良い理由のためのs。 +バイト指向の入出力には、次のようなものがあります `ReadBuffer` と `WriteBuffer` 抽象クラス。 それらはC++の代わりに使用されます `iostream`心配しないでください：すべての成熟したC++プロジェクトは、 `iostream`正当な理由のためのs。 -`ReadBuffer` と `WriteBuffer` 単に連続したバッファであり、そのバッファ内の位置を指すカーソルです。 実装にはない独自のメモリにバッファです。 バッファに次のデータを入力するための仮想メソッドがあります `ReadBuffer` または、バッファをどこかにフラッシュする（ `WriteBuffer`). 仮想手法が呼び出されます。 +`ReadBuffer` と `WriteBuffer` 単なる連続したバッファであり、そのバッファ内の位置を指すカーソルです。 実装にはない独自のメモリにバッファです。 以下のデータでバッファを埋める仮想メソッドがあります `ReadBuffer`）またはバッファをどこかにフラッシュする（ `WriteBuffer`). 仮想メソッドはまれに呼び出されます。 -の実装 `ReadBuffer`/`WriteBuffer` 使用ファイルを操作するため、ファイル記述子およびネットワークソケット実施のための圧縮 (`CompressedWriteBuffer` is initialized with another WriteBuffer and performs compression before writing data to it), and for other purposes – the names `ConcatReadBuffer`, `LimitReadBuffer`、と `HashingWriteBuffer` 自分のために話す。 +の実装 `ReadBuffer`/`WriteBuffer` 圧縮を実装するために、ファイルとファイル記述子とネットワークソケッ (`CompressedWriteBuffer` is initialized with another WriteBuffer and performs compression before writing data to it), and for other purposes – the names `ConcatReadBuffer`, `LimitReadBuffer`,and `HashingWriteBuffer` 自分のために話す。 -Read/WriteBuffersはバイトのみを処理します。 からの機能があります `ReadHelpers` と `WriteHelpers` 入力/出力の書式設定に役立つヘッダーファイル。 たとえば、decimal形式で数値を書くヘルパーがあります。 +Read/WriteBuffersはバイトのみを扱います。 からの関数があります `ReadHelpers` と `WriteHelpers` 入力/出力のフォーマットに役立つヘッダファイル。 たとえば、小数の形式で数値を書くヘルパーがあります。 -結果セットを書きたいときに何が起こるかを見てみましょう `JSON` 標準出力にフォーマットします。 あなたは結果セットをフェッチする準備ができています `IBlockInputStream`. あなたが作成する `WriteBufferFromFileDescriptor(STDOUT_FILENO)` stdoutにバイトを書き込む。 あなたが作成する `JSONRowOutputStream`、それで初期化 `WriteBuffer`、行を書き込む `JSON` stdoutに。 あなたが作成する `BlockOutputStreamFromRowOutputStream` その上に、としてそれを表すために `IBlockOutputStream`. その後、呼び出す `copyData` からデータを転送するには `IBlockInputStream` に `IBlockOutputStream`、そしてすべてが動作します。 内部的には, `JSONRowOutputStream` さまざまなJSON区切り文字を書き込み、 `IDataType::serializeTextJSON` への参照を持つメソッド `IColumn` そして、引数として行番号。 その結果, `IDataType::serializeTextJSON` からメソッドを呼び出します `WriteHelpers.h`：例えば, `writeText` 数値型の場合 `writeJSONString` のために `DataTypeString`. +結果セットを書きたいときに何が起こるかを見てみましょう `JSON` 標準出力にフォーマットします。 結果セットを取得する準備ができています `IBlockInputStream`. 作成する `WriteBufferFromFileDescriptor(STDOUT_FILENO)` stdoutにバイトを書き込む。 作成する `JSONRowOutputStream`、それで初期化されます `WriteBuffer`、行を書き込むには `JSON` 標準出力に。 作成する `BlockOutputStreamFromRowOutputStream` その上に、それを次のように表します `IBlockOutputStream`. それから電話する `copyData` データを転送するには `IBlockInputStream` に `IBlockOutputStream` そして、すべてが動作します。 内部的には, `JSONRowOutputStream` さまざまなJSON区切り文字を書き、 `IDataType::serializeTextJSON` を参照するメソッド `IColumn` 引数として行番号を指定します。 その結果, `IDataType::serializeTextJSON` からメソッドを呼び出します `WriteHelpers.h`：例えば, `writeText` 数値型および `writeJSONString` のために `DataTypeString`. ## テーブル {#tables} その `IStorage` インタフェースです。 異なる実装のインタフェースの異なるテーブルエンジンです。 例としては `StorageMergeTree`, `StorageMemory`、というように。 これらのクラスのインスタ -キー `IStorage` 方法は `read` と `write`. また、 `alter`, `rename`, `drop`、というように。 その `read` メソッドは次の引数を受け取ります:テーブルから読み取る列のセット。 `AST` 検討するクエリ、および返すストリームの必要数。 一つまたは複数を返します `IBlockInputStream` クエリの実行中にテーブルエンジン内で完了したデータ処理のステージに関するオブジェクトと情報。 +キー `IStorage` メソッドは `read` と `write`. また、 `alter`, `rename`, `drop`、というように。 その `read` このメソッドは、次の引数を受け入れます。 `AST` 考慮すべきクエリ、および返すストリームの必要な数。 一つまたは複数を返します `IBlockInputStream` クエリの実行中にテーブルエンジン内で完了したデータ処理のステージに関するオブジェクトと情報。 -ほとんどの場合、readメソッドは、それ以降のデータ処理ではなく、テーブルから指定された列を読み取るだけです。 すべてのデータ処理が行われるクエリの通訳や外部の責任 `IStorage`. +ほとんどの場合、readメソッドは、指定された列をテーブルから読み取るだけで、それ以降のデータ処理は行いません。 すべてのデータ処理が行われるクエリの通訳や外部の責任 `IStorage`. しかし、顕著な例外があります: -- ASTクエリは、 `read` 法により処理し、テーブルエンジンを使用できる指の利用と読みの少ないデータを表示します。 -- 時々のテーブルエンジンを処理できるデータそのものである場合でも特定の段階にある。 例えば, `StorageDistributed` クエリをリモートサーバーに送信し、異なるリモートサーバーのデータをマージできるステージにデータを処理するように要求し、その前処理されたデータを返すことが クエリの通訳を仕上げ加工のデータです。 +- ASTクエリは `read` 法により処理し、テーブルエンジンを使用できる指の利用と読みの少ないデータを表示します。 +- 時々のテーブルエンジンを処理できるデータそのものである場合でも特定の段階にある。 例えば, `StorageDistributed` リモートサーバーにクエリを送信し、異なるリモートサーバーからのデータをマージできるステージにデータを処理するように依頼し、その前処理されたデータを返すこ クエリの通訳を仕上げ加工のデータです。 -テーブルの `read` メソッドは、複数の `IBlockInputStream` 並列データ処理を可能にするオブジェクト。 これらの複数のブロックの入力ストリームでテーブルから行なった。 次に、これらのストリームを、独立して計算して作成できるさまざまな変換(式の評価やフィルター処理など)でラップできます。 `UnionBlockInputStream` それらの上に、並行して複数のストリームから読み取ります。 +テーブルの `read` メソッドは、複数の `IBlockInputStream` 並列データ処理を可能にするオブジェクト。 これらの複数のブロックの入力ストリームでテーブルから行なった。 次に、これらのストリームを、独立して計算できるさまざまな変換(式の評価やフィルタリングなど)でラップして、 `UnionBlockInputStream` それらの上に、並列に複数のストリームから読み込みます。 -また、 `TableFunction`s.これらは一時的な関数を返す関数です。 `IStorage` で使用するオブジェクト `FROM` クエリの句。 +また、 `TableFunction`これらは一時的なものを返す関数です `IStorage` で使用するオブジェクト `FROM` クエリの句。 -テーブルエンジンを実装する方法を簡単に知るには、次のような単純なものを見てください `StorageMemory` または `StorageTinyLog`. +テーブルエンジンの実装方法の簡単なアイデアを得るには、次のような単純なものを見てください `StorageMemory` または `StorageTinyLog`. -> の結果として `read` 方法, `IStorage` を返します `QueryProcessingStage` – information about what parts of the query were already calculated inside storage. +> の結果として `read` 方法, `IStorage` ﾂづｩﾂ。 `QueryProcessingStage` – information about what parts of the query were already calculated inside storage. ## パーサー {#parsers} -手書きの再帰的降下パーサーは、クエリを解析します。 例えば, `ParserSelectQuery` クエリのさまざまな部分の基礎となるパーサーを再帰的に呼び出すだけです。 パーサーは `AST`. その `AST` ノードによって表されます。 `IAST`. +手書きの再帰降下パーサーは、クエリを解析します。 例えば, `ParserSelectQuery` クエリのさまざまな部分に対して基になるパーサを再帰的に呼び出すだけです。 パーサーは `AST`. その `AST` ノードによって表されます。 `IAST`. > パーサジェネレータは、使用しない歴史的な理由があります。 ## 通訳者 {#interpreters} -クエリ実行パ `AST`. 以下のような単純な通訳があります `InterpreterExistsQuery` と `InterpreterDropQuery` または、より洗練された `InterpreterSelectQuery`. クエリの実行パイプラインの組み合わせたブロック入力または出力ストリーム. たとえば、次のように解釈されます。 `SELECT` クエリは、 `IBlockInputStream` 結果セットを読み取るには、INSERTクエリの結果は次のようになります。 `IBlockOutputStream` に挿入するためのデータを書き込むには、 `INSERT SELECT` クエリは、 `IBlockInputStream` これは、最初の読み取りで空の結果セットを返しますが、データをコピーします `SELECT` に `INSERT` 同時に。 +インタプリタは、クエリ実行パイプラインの作成を担当します。 `AST`. 以下のような簡単な通訳があります `InterpreterExistsQuery` と `InterpreterDropQuery` またはより洗練された `InterpreterSelectQuery`. クエリの実行パイプラインの組み合わせたブロック入力または出力ストリーム. たとえば、 `SELECT` クエリは `IBlockInputStream` 結果セットを読み取るために、INSERTクエリの結果は次のようになります。 `IBlockOutputStream` に挿入するためのデータを書き込むために、および解釈の結果 `INSERT SELECT` クエリは `IBlockInputStream` これは、最初の読み取り時に空の結果セットを返しますが、データをコピーします `SELECT` に `INSERT` 同時に。 -`InterpreterSelectQuery` 使用 `ExpressionAnalyzer` と `ExpressionActions` クエリ分析と変換のための機械。 これは、ほとんどのルールベースのクエリ最適化が行われる場所です。 `ExpressionAnalyzer` モジュラー変換やクエリを可能にするために、さまざまなクエリ変換と最適化を別々のクラスに抽出する必要があります。 +`InterpreterSelectQuery` 用途 `ExpressionAnalyzer` と `ExpressionActions` クエリ分析と変換のための機械。 ここでは、ほとんどのルールベースのクエリの最適化が行われます。 `ExpressionAnalyzer` モジュラー変換やクエリを可能にするために、さまざまなクエリ変換と最適化を別々のクラスに抽出する必要があります。 -## 機能 {#functions} +## 関数 {#functions} -通常の関数と集約関数があります。 集計関数は、次のセクションを参照してください。 +通常の関数と集計関数があります。 集計関数については、次の節を参照してください。 -Ordinary functions don’t change the number of rows – they work as if they are processing each row independently. In fact, functions are not called for individual rows, but for `Block`ベクトル化されたクエリ実行を実装するためのデータです。 +Ordinary functions don't change the number of rows – they work as if they are processing each row independently. In fact, functions are not called for individual rows, but for `Block`ベクトル化されたクエリ実行を実装するためのデータ。 -いくつかの雑多な機能があります [ブロックサイズ](../sql-reference/functions/other-functions.md#function-blocksize), [rowNumberInBlock](../sql-reference/functions/other-functions.md#function-rownumberinblock)、と [runningAccumulate](../sql-reference/functions/other-functions.md#function-runningaccumulate)、それはブロック処理を悪用し、行の独立性に違反します。 +いくつかのその他の機能があります。 [ブロックサイズ](../sql-reference/functions/other-functions.md#function-blocksize), [rowNumberInBlock](../sql-reference/functions/other-functions.md#function-rownumberinblock),and [runningAccumulate](../sql-reference/functions/other-functions.md#function-runningaccumulate)、それはブロック処理を悪用し、行の独立性に違反します。 -ClickHouseには厳密な型指定があるため、暗黙の型変換はありません。 関数が型の特定の組み合わせをサポートしていない場合は、例外がスローされます。 ものの機能で作業する過負荷のもとに多くの異なる組み合わせます。 たとえば、 `plus` 機能(実装するには `+` 演算子）数値型の任意の組み合わせに対して機能します: `UInt8` + `Float32`, `UInt16` + `Int8`、というように。 また、いくつかの可変個引数関数は、任意の数の引数を受け入れることができます。 `concat` 機能。 +ClickHouseには強い型指定があるため、暗黙的な型変換はありません。 関数が特定の型の組み合わせをサポートしていない場合、例外がスローされます。 ものの機能で作業する過負荷のもとに多くの異なる組み合わせます。 例えば、 `plus` 関数（実装するには `+` 演算子)数値型の任意の組み合わせに対して動作します: `UInt8` + `Float32`, `UInt16` + `Int8`、というように。 また、一部の可変引数関数は、以下のような任意の数の引数を受け入れることができます。 `concat` 機能。 -実施の機能が少し不便での機能を明示的に派遣サポートされているデータの種類と対応 `IColumns`. たとえば、 `plus` 関数は、数値型の各組み合わせのためのC++テンプレートのインスタンス化によって生成されたコードを持っており、定数または非定数左右引数。 +実施の機能が少し不便での機能を明示的に派遣サポートされているデータの種類と対応 `IColumns`. 例えば、 `plus` 関数は、数値型の組み合わせごとにC++テンプレートのインスタンス化によって生成されたコード、および定数または非定数左と右の引数を持っていま -優れた場所をランタイムコード生成を避けるテンプレートコードで膨張. また、fused multiply-addようなfused関数を追加したり、あるループ反復で多重比較を行うこともできます。 +テンプレートコードの膨張を避けるために、実行時コード生成を実装するのに最適な場所です。 また、fused multiply-addのようなfused関数を追加したり、一つのループ反復で多重比較を行うこともできます。 -ベクトル化されたクエリの実行により、関数は短絡されません。 たとえば、次のように書くと `WHERE f(x) AND g(y)`、両側は、行に対しても計算されます。 `f(x)` はゼロです(ただし、 `f(x)` はゼロ定数式である。 しかしの選択率 `f(x)` 条件が高い、との計算 `f(x)` よりもはるかに安いです `g(y)`、それはマルチパス計算を実装する方が良いでしょう。 それは最初に計算します `f(x)`、その後、結果によって列をフィルタリングし、計算 `g(y)` フィルター処理された小さいデータチャンクのみ。 +ベクトル化されたクエリの実行により、関数は短絡されません。 たとえば、 `WHERE f(x) AND g(y)`,両側が計算されます,でも行について,とき `f(x)` がゼロである(場合を除く `f(x)` はゼロ定数式である）。 しかし、の選択性が `f(x)` 条件は高く、計算の `f(x)` よりもはるかに安いです `g(y)`、マルチパス計算を実装する方が良いでしょう。 最初に計算します `f(x)` 次に、結果によって列をフィルター処理し、次に計算します `g(y)` フィルター処理された小さなデータのチャンクのみ。 ## 集計関数 {#aggregate-functions} -集計関数はステートフル関数です。 渡された値をある状態に蓄積し、その状態から結果を得ることができます。 それらはと管理されます `IAggregateFunction` インタフェース 状態はかなり単純なものにすることができます `AggregateFunctionCount` 単なる `UInt64` 値)または非常に複雑な(の状態 `AggregateFunctionUniqCombined` は、線形配列、ハッシュテーブル、および `HyperLogLog` 確率的データ構造）。 +集計関数はステートフル関数です。 渡された値をある状態に蓄積し、その状態から結果を得ることができます。 それらはと管理されます `IAggregateFunction` インタフェース 状態はかなり単純にすることができます（ `AggregateFunctionCount` ただの単一です `UInt64` 値）または非常に複雑な（の状態 `AggregateFunctionUniqCombined` 線形配列、ハッシュテーブル、およびaの組み合わせです `HyperLogLog` 確率データ構造）。 -状態は `Arena` 高カーディナリティを実行しながら複数の状態を処理する（メモリプール） `GROUP BY` クエリ。 たとえば、複雑な集計状態では、追加のメモリ自体を割り当てることができます。 それは、州を創設し、破壊し、所有権と破壊秩序を適切に渡すことにある程度の注意を払う必要があります。 +状態は `Arena` (メモリプール)高カーディナリティの実行中に複数の状態を処理する `GROUP BY` クエリ。 たとえば、複雑な集約状態では、追加のメモリを割り当てることができます。 これは、作成し、状態を破壊し、適切にその所有権と破壊命令を渡すためにいくつかの注意が必要です。 -分散クエリの実行中にネットワーク経由で渡すか、十分なramがないディスクに書き込むために、集約状態をシリアル化および逆シリアル化できます。 それらはテーブルで貯えることができます `DataTypeAggregateFunction` データの増分集計を可能にする。 +集約状態をシリアル化および逆シリアル化して、分散クエリの実行中にネットワーク経由で渡したり、十分なRAMがないディスクに書き込んだりできま それらはあるテーブルで貯えることができます `DataTypeAggregateFunction` データの増分集計を可能にする。 -> 集計関数状態のシリアル化されたデータ形式は、現在バージョン化されていません。 集約状態が一時的にのみ格納されている場合は問題ありません。 しかし、我々は持っている `AggregatingMergeTree` テーブルエンジンが増えた場合の集約、人々に基づき使用されている。 これは、将来の集約関数の直列化形式を変更する際に下位互換性が必要な理由です。 +> 集計関数状態のシリアル化されたデータ形式は、現在バージョン管理されていません。 集約状態が一時的にのみ格納されていればokです。 しかし、我々は持っている `AggregatingMergeTree` テーブルエンジンが増えた場合の集約、人々に基づき使用されている。 これは、将来集計関数のシリアル化形式を変更するときに下位互換性が必要な理由です。 ## サーバ {#server} サーバを実装し複数の複数のインタフェース: -- 外部クライアントのhttpインターフェイス。 -- ネイティブclickhouseクライアント用のtcpインターフェイス、および分散クエリ実行中のサーバー間通信。 +- 外部クライアント用のHTTPインターフェイス。 +- TCPインタフェースのネイティブClickHouseクライアントとクロス-サーバー通信中に分散クエリを実行します。 - インターフェース転送データレプリケーション. -内部的には、コルーチンやファイバーのない原始的なマルチスレッドサーバーです。 サーバーは、単純なクエリの高いレートを処理するが、複雑なクエリの比較的低いレートを処理するように設計されていないので、それらのそれぞれは、分析 +内部的には、コルーチンやファイバーのない原始的なマルチスレッドサーバーです。 サーバーは、単純なクエリの割合が高いのではなく、比較的低い複雑なクエリの割合を処理するように設計されているため、それぞれが分析のために膨大 -サーバーは、 `Context` クエリ実行に必要な環境を持つクラス:利用可能なデータベース、ユーザーおよびアクセス権、設定、クラスター、プロセスリスト、クエリログなどのリスト。 通訳者はこの環境を使用します。 +サーバーは初期化します `Context` クエリ実行に必要な環境を持つクラス:使用可能なデータベース、ユーザーとアクセス権、設定、クラスター、プロセスリスト、クエリログなどのリスト。 通訳者はこの環境を利用します。 -古いクライアントは新しいサーバーと通信でき、新しいクライアントは古いサーバーと通信できます。 しかし、私たちは永遠にそれを維持したくない、と私たちは約一年後に古いバージョンのサポートを削除しています。 +古いクライアントは新しいサーバーと話すことができ、新しいクライアントは古いサーバーと話すことができます。 しかし、我々は永遠にそれを維持したくない、と我々は約一年後に古いバージョンのサポートを削除しています。 -!!! note "メモ" - ほとんどの外部アプリケーションの利用を推奨します。http面でシンプルで使いやすいです。 tcpプロトコルは、内部データ構造により緊密にリンクされています。 そのプロトコルのcライブラリは、実用的ではないclickhouseコードベースのほとんどをリンクする必要があるため、リリースしていません。 +!!! note "注" + ほとんどの外部アプリケーションでは、HTTPインターフェイスを使用することをお勧めします。 TCPプロトコルは、データのブロックを渡すために内部形式を使用し、圧縮されたデータにはカスタムフレーミングを使用します。 まだ公表したCライブラリのためのこのプロトコールすることが必要なことから、リンクのClickHouseコードベース、るのは現実的ではありません。 ## 分散クエリの実行 {#distributed-query-execution} -サーバーにクラスターセットアップがほぼ独立しています。 を作成することができ `Distributed` クラスター内の一つまたはすべてのサーバーの表。 その `Distributed` table does not store data itself – it only provides a “view” クラスターの複数ノード上のすべてのローカルテーブル。 から選択すると `Distributed` クエリを書き換え、ロードバランシング設定に従ってリモートノードを選択し、クエリをクエリに送信します。 その `Distributed` テーブル要求をリモートサーバー処理クエリーだけで最大のペースでの中間結果から異なるサーバできます。 その後、中間結果を受け取り、それらをマージします。 のテーブルを配布してできる限りの仕事へのリモートサーバーを送信しない多くの中間データのネットワーク. +サーバーにクラスターセットアップがほぼ独立しています。 を作成することができます `Distributed` クラスター内のサーバーの表。 その `Distributed` table does not store data itself – it only provides a “view” すべての地方のテーブルに複数のノードのクラスター Aから選択すると `Distributed` テーブルは、そのクエリを書き換え、負荷分散設定に従ってリモートノードを選択し、それらにクエリを送信します。 その `Distributed` テーブル要求をリモートサーバー処理クエリーだけで最大のペースでの中間結果から異なるサーバできます。 その後、中間結果を受信してマージします。 のテーブルを配布してできる限りの仕事へのリモートサーバーを送信しない多くの中間データのネットワーク. -INまたはJOIN節にサブクエリがあり、それぞれがaを使用すると、事態はより複雑になります。 `Distributed` テーブル。 これらのクエリの実行にはさまざまな戦略があります。 +INまたはJOIN句にサブクエリがあり、それぞれが `Distributed` テーブル。 これらのクエリの実行には、さまざまな戦略があります。 -分散クエリ実行用のグローバルクエリプランはありません。 各ノ リモートノードのクエリを送信し、結果をマージします。 しかし、これは、高い基数グループbysまたは結合のための大量の一時的なデータを持つ複雑なクエリでは不可能です。 そのような場合、 “reshuffle” 追加の調整が必要なサーバー間のデータ。 ClickHouseはそのような種類のクエリ実行をサポートしておらず、その上で作業する必要があります。 +分散クエリ実行用のグローバルクエリプランはありません。 各ノードには、ジョブの一部のローカルクエリプランがあります。 リモートノードに対してクエリを送信し、結果をマージします。 しかし、基数の多いグループBYsを持つ複雑なクエリや、結合のための大量の一時データを持つクエリでは、これは不可能です。 そのような場合には、 “reshuffle” 追加の調整が必要なサーバー間のデータ。 ClickHouseはそのようなクエリの実行をサポートしていません。 -## ツリーをマージ {#merge-tree} +## マージツリー {#merge-tree} -`MergeTree` 家族のストレージエンジンを支える指数付けによりその有効なタイプを利用します。 主キーは、列または式の任意のタプルにすることができます。 Aのデータ `MergeTree` テーブルは “parts”. 各パーツは主キーの順序でデータを格納するので、データは主キータプルによって辞書式で順序付けされます。 すべてのテーブル列は別々に格納されます `column.bin` これらの部分のファイル。 ファイルは圧縮ブロックで構成されます。 各ブロックは、平均値のサイズに応じて、通常64KBから1MBの非圧縮データです。 ブロックは、連続して配置された列の値で構成されます。 列の値は各列の順序が同じです（主キーで順序が定義されています）ので、多くの列で反復すると、対応する行の値が取得されます。 +`MergeTree` 主キ 主キーは、列または式の任意のタプルにすることができます。 Aのデータ `MergeTree` テーブルは “parts”. 各パートはデータを主キーの順序で格納するため、データは主キータプルによって辞書順に並べ替えられます。 すべてのテーブル列は別々に格納されます `column.bin` これらの部分のファイル。 ファイルは圧縮ブロックで構成されます。 各ブロックは、平均値のサイズに応じて、通常64KBから1MBの非圧縮データです。 ブロックは、列の値が連続して次々に配置されています。 列の値は各列で同じ順序になるため（主キーによって順序が定義されます）、多くの列で反復処理すると、対応する行の値が取得されます。 -主キー自体は次のとおりです “sparse”. なアドレス毎に単列、ある種のデータです。 独立した `primary.idx` ファイルには、N番目の各行の主キーの値があります。 `index_granularity` (通常、N=8192)。 また、各列について、 `column.mrk` 以下のファイル “marks,” これは、データファイル内の各N番目の行にオフセットされます。 各マークは、圧縮されたブロックの先頭にファイル内のオフセット、およびデータの先頭に圧縮解除されたブロック内のオフセットのペアです。 通常、圧縮ブロックはマークによって整列され、圧縮解除されたブロックのオフセットはゼロです。 データのための `primary.idx` 常にメモリに常駐し、データのための `column.mrk` ファイ +主キー自体は次のとおりです “sparse”. すべての行に対処するのではなく、いくつかの範囲のデータのみを扱います。 別の `primary.idx` fileには、N番目の行ごとに主キーの値があります。 `index_granularity` （通常、N=8192）。 また、各列について、我々は持っている `column.mrk` ファイル “marks,” これは、データファイル内のN番目の行ごとにオフセットされます。 各マークは、ファイル内の圧縮ブロックの先頭までのオフセットと、圧縮解除ブロック内のデータの先頭までのオフセットのペアです。 通常、圧縮されたブロックはマークで整列され、解凍されたブロックのオフセットはゼロです。 データのための `primary.idx` 常にメモリ内に存在し、 `column.mrk` ファイ -我々は一部から何かを読むつもりされているとき `MergeTree`、我々は見て `primary.idx` 要求されたデータを含む可能性のあるデータと範囲を見つけ、次に `column.mrk` これらの範囲の読み取りを開始する場所のオフセットを計算します。 まばらであるため、余分なデータを読み取ることができます。 ClickHouseは、単純なポイントクエリの高負荷には適していません。 `index_granularity` 各キーに対して行を読み取り、圧縮されたブロック全体を各列に対して圧縮解除する必要があります。 インデックスのメモリ消費量が目立つことなく、単一のサーバーごとに数兆行を維持できる必要があるため、インデックスを疎にしました。 また、主キーはスパースであるため、一意ではありません。 テーブルに同じキーを持つ多くの行を持つことができます。 +我々は一部から何かを読むつもりですときに `MergeTree` 私たちは `primary.idx` データと要求されたデータを含む可能性のある範囲を見つけて、次に `column.mrk` これらの範囲の読み取りを開始する場所のデータと計算オフセット。 希薄さのために、余分なデータが読み取られることがあります。 ClickHouseは、単純なポイントクエリの高負荷には適していません。 `index_granularity` キーごとに行を読み取り、圧縮されたブロック全体を列ごとに解凍する必要があります。 私たちは、インデックスの顕著なメモリ消費なしに単一のサーバーごとに数兆行を維持できる必要があるため、インデックスを疎にしました。 また、主キーは疎であるため、一意ではありません。 テーブルに同じキーを持つ多くの行を持つことができます。 -ときあなた `INSERT` データの束に `MergeTree` その束は主キーの順序でソートされ、新しい部分を形成します。 が背景のスレッドを定期的に、選択部分と統合して単一のソート部の部品点数が比較的低い。 それが呼び出される理由です `MergeTree`. もちろん、マージにつながる “write amplification”. すべての部分は不変です：作成され、削除されますが、変更されません。 SELECTが実行されると、テーブル（部品のセット）のスナップショットが保持されます。 マージした後、我々はまた、我々はいくつかのマージされた部分は、おそらく壊れていることを確認した場合、我々は、そのソース部分とそれを置き換えることがで +あなたが `INSERT` データの束に `MergeTree`、その束は主キーの順序でソートされ、新しい部分を形成します。 が背景のスレッドを定期的に、選択部分と統合して単一のソート部の部品点数が比較的低い。 それが呼び出される理由です `MergeTree`. もちろん、マージは “write amplification”. すべてのパーツは不変であり、作成および削除のみが行われますが、変更は行われません。 SELECTが実行されると、テーブルのスナップショット(パーツのセット)が保持されます。 マージ後もしばらくの間、古いパーツを保持して、障害後の回復を容易にするため、マージされたパーツが壊れている可能性があることがわかったら、ソースパーツ -`MergeTree` LSMツリーが含まれていないためではありません “memtable” と “log”: inserted data is written directly to the filesystem. This makes it suitable only to INSERT data in batches, not by individual row and not very frequently – about once per second is ok, but a thousand times a second is not. We did it this way for simplicity’s sake, and because we are already inserting data in batches in our applications. +`MergeTree` LSMツリーではありません。 “memtable” と “log”: inserted data is written directly to the filesystem. This makes it suitable only to INSERT data in batches, not by individual row and not very frequently – about once per second is ok, but a thousand times a second is not. We did it this way for simplicity's sake, and because we are already inserting data in batches in our applications. -> MergeTreeテーブルには、一つの(プライマリ)インデックスしか持たせることができません。 たとえば、データを複数の物理的順序で格納したり、元のデータとともに事前に集計されたデータを使用して表現することもできます。 +> MergeTreeテーブルには、一つの(プライマリ)インデックスしか持つことができません。 たとえば、複数の物理的順序でデータを格納したり、事前に集計されたデータと元のデータを含む表現を許可したりすることもできます。 -バックグラウンドマージ中に追加作業を行っているmergetreeエンジンがあります。 例としては `CollapsingMergeTree` と `AggregatingMergeTree`. この処理として特別支援しました。 なぜなら、ユーザーは通常、バックグラウンドマージが実行される時間と、バックグラウンドマージが実行されるデータを制御できないからです。 `MergeTree` テーブルは、ほとんどの場合、完全にマージされた形式ではなく、複数の部分に格納されます。 +バックグラウンドマージ中に追加の作業を行っているMergeTreeエンジンがあります。 例としては `CollapsingMergeTree` と `AggregatingMergeTree`. この処理として特別支援しました。 なぜなら、ユーザーは通常、バックグラウンドマージが実行される時間とデータを制御することができないからです。 `MergeTree` テーブルは、ほとんどの場合、完全にマージされた形式ではなく、複数の部分に格納されます。 ## 複製 {#replication} -ClickHouseでのレプリケーションは、テーブルごとに構成できます。 きも複製されない一部の複製のテーブルと同じサーバです。 また、テーブルをさまざまな方法でレプリケートすることもできます。 +ClickHouseでのレプリケーションは、テーブルごとに構成できます。 きも複製されない一部の複製のテーブルと同じサーバです。 また、二要素複製のテーブルと三要素複製のテーブルなど、さまざまな方法でテーブルをレプリケートすることもできます。 -レプリケーションは `ReplicatedMergeTree` 貯蔵エンジン。 のパス `ZooKeeper` ストレージエンジンのパラメータとして指定します。 同じパスを持つすべてのテーブル `ZooKeeper` 互いのレプリカになる：彼らは彼らのデータを同期させ、一貫性を維持する。 テーブルを作成または削除するだけで、レプリカを動的に追加および削除できます。 +レプリケーションは `ReplicatedMergeTree` ストレージエンジン のパス `ZooKeeper` ストレージエンジンのパラメータとして指定します。 同じパスを持つすべてのテーブル `ZooKeeper` 互いのレプリカになる：彼らはデータを同期し、一貫性を維持する。 レプリカは、テーブルを作成または削除するだけで動的に追加および削除できます。 -レプリケーション 次のセッションを持つ任意のレプリカにデータを挿入できます `ZooKeeper` データは、他のすべてのレプリカに非同期的に複製されます。 でClickHouseをサポートしていない更新、複製する紛争は無料です。 挿入のクォーラム確認がないため、あるノードが失敗すると、挿入されたばかりのデータが失われる可能性があります。 +複製を使用して非同期マルチマスタースキームです。 次のセッションを持つ任意のレプリカにデータを挿入できます `ZooKeeper` データを複製、その他すべてのレプリカは非同期的に. ClickHouseは更新をサポートしていないため、複製は競合しません。 挿入のクォーラム確認がないため、あるノードに障害が発生すると、挿入されただけのデータが失われる可能性があります。 -メタデータレプリケーションが格納され飼育係. 実行する操作を一覧表示するレプリケーションログがあります。 アクションとしては、get part、merge parts、drop partitionなどがあります。 各レプリカコピー、複製のログをキューにその行動からのキューに挿入します 例えば、挿入時には、 “get the part” actionを作成し、ログイン、レプリカのダウンロードいます。 マージはレプリカ間で調整され、バイトと同じ結果が得られます。 すべての部品を合併した場合と同様にすべてのレプリカ. では達成を補い、一つのレプリカのリーダーとして、レプリカを始めと融合し、書き込みます “merge parts” ログへのアクション。 +複製のメタデータはZooKeeperに格納されます。 実行するアクションを示すレプリケーションログがあります。 アクションは次のとおりです。 各レプリカコピー、複製のログをキューにその行動からのキューに挿入します 例えば、挿入では、 “get the part” actionを作成し、ログイン、レプリカのダウンロードいます。 マージはレプリカ間で調整され、バイトが同一の結果が得られます。 すべての部品を合併した場合と同様にすべてのレプリカ. では達成を補い、一つのレプリカのリーダーとして、レプリカを始めと融合し、書き込みます “merge parts” ログへのアクション。 -クエリではなく、ノード間で転送されるのは圧縮された部分だけです。 合併処理され、各レプリカ多くの場合、自主的に下げるネットワークコストを回避することによるネットワークが増幅。 大合併をさらにネットワークする場合に限り重複製に遅れて波及してきています。 +圧縮された部分のみがノード間で転送され、クエリは転送されません。 合併処理され、各レプリカ多くの場合、自主的に下げるネットワークコストを回避することによるネットワークが増幅。 大合併をさらにネットワークする場合に限り重複製に遅れて波及してきています。 -さらに、各レプリカは、部品とそのチェックサムのセットとしてzookeeperにその状態を保存します。 ローカルファイルシステム上の状態がzookeeperの参照状態から逸脱すると、レプリカは他のレプリカから欠落した部分や破損した部分をダウンロードする ローカルファイルシステムに予期しないデータや破損したデータがある場合、clickhouseはそれを削除せず、別のディレクトリに移動して忘れてしまいます。 +さらに、各レプリカは、その状態をパーツとそのチェックサムのセットとしてZooKeeperに格納します。 ローカルファイルシステム上の状態がZooKeeperの参照状態から乖離すると、レプリカは他のレプリカから欠落している部分と壊れている部分をダウンロード ローカルファイルシステムに予期しないデータや壊れたデータがある場合、ClickHouseはそれを削除しませんが、別のディレクトリに移動して忘れます。 -!!! note "メモ" - クリックハウスクラスタは独立したシャードから成り，各シャードはレプリカから成る。 クラスタは **ない弾性** したがって、新しいシャードを追加した後、データは自動的にシャード間で再調整されません。 代わりに、クラスタの負荷が不均一になるように調整されるはずです。 この実装では、より多くの制御が可能になり、数十のノードなど、比較的小さなクラスタでも問題ありません。 がクラスターの何百人ものノードを用いて、生産-このアプローチが大きな欠点. を実行すべきである"と述べていテーブルエンジンで広がる、クラスターを動的に再現れる可能性がある地域分割のバランスとクラスターの動します。 +!!! note "注" + ClickHouseクラスターは独立したシャードで構成され、各シャードはレプリカで構成されます。 クラスターは **弾性ではない** したがって、新しいシャードを追加した後、データはシャード間で自動的に再調整されません。 代わりに、クラスタ負荷は不均一に調整されることになっています。 この実装はより多くの制御を提供し、数十のノードなどの比較的小さなクラスタでもokです。 しかし、運用環境で使用している数百のノードを持つクラスターでは、このアプローチは重大な欠点になります。 を実行すべきである"と述べていテーブルエンジンで広がる、クラスターを動的に再現れる可能性がある地域分割のバランスとクラスターの動します。 {## [元の記事](https://clickhouse.tech/docs/en/development/architecture/) ##} diff --git a/docs/ja/development/browse-code.md b/docs/ja/development/browse-code.md index d66b14e400f..45eba084f4a 100644 --- a/docs/ja/development/browse-code.md +++ b/docs/ja/development/browse-code.md @@ -1,14 +1,14 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 63 -toc_title: "ClickHouse\u306E\u30BD\u30FC\u30B9\u30B3\u30FC\u30C9\u3092\u53C2\u7167" +toc_title: "\u30BD\u30FC\u30B9\u30B3\u30FC\u30C9\u306E\u53C2\u7167" --- # ClickHouseのソースコードを参照 {#browse-clickhouse-source-code} -を使用することができ **Woboq** オンラインのコードブラウザをご利用 [ここに](https://clickhouse.tech/codebrowser/html_report///ClickHouse/dbms/index.html). このコードナビゲーションや意味のハイライト表示、検索インデックス. コードのスナップショットは随時更新中です。 +以下を使用できます **Woboq** オンラインのコードブラウザをご利用 [ここに](https://clickhouse.tech/codebrowser/html_report/ClickHouse/src/index.html). このコードナビゲーションや意味のハイライト表示、検索インデックス. コードのスナップショットは随時更新中です。 -また、ソースを閲覧することもできます [GitHub](https://github.com/ClickHouse/ClickHouse) いつものように +また、ソースを参照することもできます [GitHub](https://github.com/ClickHouse/ClickHouse) いつものように -使用するideに興味がある場合は、clion、qt creator、vs code、およびkdevelop（注意点あり）をお勧めします。 お気に入りのideを使用できます。 vimとemacsもカウントされます。 +使用するIDEに興味がある場合は、CLion、QT Creator、VS Code、KDevelop（注意点あり）をお勧めします。 任意の好きなIDEを使用できます。 VimとEmacsもカウントされます。 diff --git a/docs/ja/development/build-cross-arm.md b/docs/ja/development/build-cross-arm.md index dda1aab587b..a5fb0595d08 100644 --- a/docs/ja/development/build-cross-arm.md +++ b/docs/ja/development/build-cross-arm.md @@ -1,21 +1,21 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 67 -toc_title: "AARCH64\u7528Linux\u3067ClickHouse\u3092\u30D3\u30EB\u30C9\u3059\u308B\ - \u65B9\u6CD5\uFF08ARM64)" +toc_title: "Aarch64\u7528\u306ELinux\u4E0A\u3067ClickHouse\u3092\u69CB\u7BC9\u3059\ + \u308B\u65B9\u6CD5(ARM64)" --- -# Aarch64(ARM64)アーキテクチャ用のLinuxでClickHouseをビルドする方法 {#how-to-build-clickhouse-on-linux-for-aarch64-arm64-architecture} +# Aarch64(ARM64)アーキテクチャ用のLinux上でClickHouseを構築する方法 {#how-to-build-clickhouse-on-linux-for-aarch64-arm64-architecture} -これは、linuxマシンがあり、それを使ってビルドしたい場合のためのものです `clickhouse` AARCH64CPUアーキテクチャを持つ別のLinuxマシンで実行されるバイナリ。 この目的のために継続的インテグレーションをチェックを実行Linuxサーバー +これは、Linuxマシンを使用してビルドする場合のためのものです `clickhouse` AARCH64CPUアーキテクチャを持つ別のLinuxマシン上で実行されるバイナリ。 この目的のために継続的インテグレーションをチェックを実行Linuxサーバー -AARCH64のクロスビルドは、次の条件に基づいています [ビルド手順](build.md)、最初にそれらに続きなさい。 +AARCH64のクロスビルドは [ビルド命令](build.md) 先について来い -# インストールclang-8 {#install-clang-8} +# Clang-8をインストール {#install-clang-8} -以下の指示に従ってくださいhttps://apt.llvm.org/あなたのubuntuやdebianの設定のために. -たとえば、ubuntu bionicでは、次のコマンドを使用できます: +の指示に従ってくださいhttps://apt.llvm.org/あなたのUbuntuまたはDebianのセットアップ用。 +たとえば、Ubuntu Bionicでは、次のコマンドを使用できます: ``` bash echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic-8 main" | sudo tee /etc/apt/sources.list.d/llvm.list @@ -32,7 +32,7 @@ wget 'https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel tar xJf gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz -C build-aarch64/cmake/toolchain/linux-aarch64 --strip-components=1 ``` -# クリックハウスを構築 {#build-clickhouse} +# ビルドClickHouse {#build-clickhouse} ``` bash cd ClickHouse @@ -41,4 +41,4 @@ CC=clang-8 CXX=clang++-8 cmake . -Bbuild-arm64 -DCMAKE_TOOLCHAIN_FILE=cmake/linu ninja -C build-arm64 ``` -結果のバイナリは、aarch64cpuアーキテクチャを持つlinux上でのみ実行されます。 +結果のバイナリは、Aarch64CPUアーキテクチャを持つLinux上でのみ実行されます。 diff --git a/docs/ja/development/build-cross-osx.md b/docs/ja/development/build-cross-osx.md index 81a94b65dee..5d7dee09827 100644 --- a/docs/ja/development/build-cross-osx.md +++ b/docs/ja/development/build-cross-osx.md @@ -1,20 +1,20 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 66 -toc_title: "Mac OS X\u7528\u306ELinux\u3067ClickHouse\u3092\u69CB\u7BC9\u3059\u308B\ - \u65B9\u6CD5" +toc_title: "Mac OS X\u7528\u306ELinux\u4E0A\u3067ClickHouse\u3092\u69CB\u7BC9\u3059\ + \u308B\u65B9\u6CD5" --- -# Mac OS X用のLinuxでClickHouseを構築する方法 {#how-to-build-clickhouse-on-linux-for-mac-os-x} +# Mac OS X用のLinux上でClickHouseを構築する方法 {#how-to-build-clickhouse-on-linux-for-mac-os-x} -これは、linuxマシンがあり、それを使ってビルドしたい場合のためのものです `clickhouse` OS X上で実行されるバイナリこれは、Linuxサーバー上で実行される継続的な統合チェックを目的としています。 Mac OS XでClickHouseを直接ビルドする場合は、次の手順に進んでください [別の命令](build-osx.md). +これは、Linuxマシンを使用してビルドする場合のためのものです `clickhouse` これは、Linuxサーバー上で実行される継続的な統合チェックを目的としています。 Mac OS X上でClickHouseを直接ビルドする場合は、次の手順に進みます [別の命令](build-osx.md). -Mac OS X用のクロスビルドは、以下に基づいています。 [ビルド手順](build.md)、最初にそれらに続きなさい。 +Mac OS X用のクロスビルドは [ビルド命令](build.md) 先について来い -# インストールclang-8 {#install-clang-8} +# Clang-8をインストール {#install-clang-8} -以下の指示に従ってくださいhttps://apt.llvm.org/あなたのubuntuやdebianの設定のために. +の指示に従ってくださいhttps://apt.llvm.org/あなたのUbuntuまたはDebianのセットアップ用。 例えば、コマンドバイオニックのような: ``` bash @@ -24,7 +24,7 @@ sudo apt-get install clang-8 # クロスコンパイルツールセット {#install-cross-compilation-toolset} -いっしょうにパスを設置 `cctools` として${CCTOOLS} +インストール先のパスを覚えてみましょう `cctools` として${CCTOOLS} ``` bash mkdir ${CCTOOLS} @@ -41,7 +41,7 @@ cd cctools-port/cctools make install ``` -また、macos x sdkを作業ツリーにダウンロードする必要があります。 +また、作業ツリーにmacOS X SDKをダウンロードする必要があります。 ``` bash cd ClickHouse @@ -50,7 +50,7 @@ mkdir -p build-darwin/cmake/toolchain/darwin-x86_64 tar xJf MacOSX10.14.sdk.tar.xz -C build-darwin/cmake/toolchain/darwin-x86_64 --strip-components=1 ``` -# クリックハウスを構築 {#build-clickhouse} +# ビルドClickHouse {#build-clickhouse} ``` bash cd ClickHouse @@ -62,4 +62,4 @@ CC=clang-8 CXX=clang++-8 cmake . -Bbuild-osx -DCMAKE_TOOLCHAIN_FILE=cmake/darwin ninja -C build-osx ``` -結果のバイナリはmach-o実行可能フォーマットを持ち、linuxでは実行できません。 +結果のバイナリはmach-O実行可能フォーマットを持ち、Linux上で実行することはできません。 diff --git a/docs/ja/development/build-osx.md b/docs/ja/development/build-osx.md index b0314495cd0..5d0af7d8f43 100644 --- a/docs/ja/development/build-osx.md +++ b/docs/ja/development/build-osx.md @@ -1,27 +1,27 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 65 toc_title: "Mac OS X\u3067ClickHouse\u3092\u69CB\u7BC9\u3059\u308B\u65B9\u6CD5" --- # Mac OS XでClickHouseを構築する方法 {#how-to-build-clickhouse-on-mac-os-x} -ビルドはmac os x10.15(catalina)で動作するはずです) +ビルドはMac OS X10.15(Catalina) -## ト自作 {#install-homebrew} +## 自作のインストール {#install-homebrew} ``` bash $ /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" ``` -## 必要なコンパイラ、ツール、ライブラ {#install-required-compilers-tools-and-libraries} +## 設置に必要なコンパイラ、ツール、図書館 {#install-required-compilers-tools-and-libraries} ``` bash $ brew install cmake ninja libtool gettext ``` -## レclickhouse源 {#checkout-clickhouse-sources} +## ﾂつｨﾂ姪"ﾂ債ﾂつｹ {#checkout-clickhouse-sources} ``` bash $ git clone --recursive git@github.com:ClickHouse/ClickHouse.git @@ -35,7 +35,7 @@ $ git clone --recursive https://github.com/ClickHouse/ClickHouse.git $ cd ClickHouse ``` -## クリックハウスを構築 {#build-clickhouse} +## ビルドClickHouse {#build-clickhouse} ``` bash $ mkdir build @@ -49,12 +49,12 @@ $ cd .. Clickhouse-serverを実行する場合は、システムのmaxfiles変数を増やしてください。 -!!! info "メモ" +!!! info "注" Sudoを使用する必要があります。 これを行うには、次のファイルを作成します: -/ライブラリ/LaunchDaemons/制限.マックスファイルplist: +/ライブラリ/LaunchDaemons/limit.マックスファイルプリスト: ``` xml @@ -88,6 +88,6 @@ $ sudo chown root:wheel /Library/LaunchDaemons/limit.maxfiles.plist 再起動しろ -それが動作しているかどうかを確認するには、 `ulimit -n` 司令部 +チェックの場合は、利用できる `ulimit -n` コマンド [元の記事](https://clickhouse.tech/docs/en/development/build_osx/) diff --git a/docs/ja/development/build.md b/docs/ja/development/build.md index 036323a4ac0..c1e92efd441 100644 --- a/docs/ja/development/build.md +++ b/docs/ja/development/build.md @@ -1,23 +1,23 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 64 -toc_title: "Linux\u3067ClickHouse\u3092\u69CB\u7BC9\u3059\u308B\u65B9\u6CD5" +toc_title: "Linux\u4E0A\u3067ClickHouse\u3092\u69CB\u7BC9\u3059\u308B\u65B9\u6CD5" --- -# 開発のためのclickhouseを構築する方法 {#how-to-build-clickhouse-for-development} +# 開発のためのClickHouseを構築する方法 {#how-to-build-clickhouse-for-development} -次のチュートリアルはubuntu linuxシステムに基づいています。 -適切な変更により、他のlinuxディストリビューションでも動作するはずです。 -サポートされるプラットフォーム:x86\_64およびaarch64。 power9のサポートは実験的です。 +次のチュートリアルはUbuntu Linuxシステムに基づいています。 +適切な変更により、他のLinuxディストリビューションでも動作するはずです。 +サポートされるプラットフォーム:x86\_64およびAArch64。 Power9のサポートは実験的です。 -## Git、CMake、Pythonと忍者をインストールします。 {#install-git-cmake-python-and-ninja} +## Git、CMake、Pythonと忍者をインストールします {#install-git-cmake-python-and-ninja} ``` bash $ sudo apt-get install git cmake python ninja-build ``` -または古いシステムのcmakeの代わりにcmake3。 +古いシステムではcmakeの代わりにcmake3。 ## GCC9のインストール {#install-gcc-9} @@ -32,18 +32,18 @@ $ sudo apt-get update $ sudo apt-get install gcc-9 g++-9 ``` -### ソースからのイ {#install-from-sources} +### ソースからインスト {#install-from-sources} 見て [utils/ci/build-gcc-from-sources.sh](https://github.com/ClickHouse/ClickHouse/blob/master/utils/ci/build-gcc-from-sources.sh) -## ビルドにはgcc9を使う {#use-gcc-9-for-builds} +## ビルドにGCC9を使用する {#use-gcc-9-for-builds} ``` bash $ export CC=gcc-9 $ export CXX=g++-9 ``` -## レclickhouse源 {#checkout-clickhouse-sources} +## ﾂつｨﾂ姪"ﾂ債ﾂつｹ {#checkout-clickhouse-sources} ``` bash $ git clone --recursive git@github.com:ClickHouse/ClickHouse.git @@ -55,7 +55,7 @@ $ git clone --recursive git@github.com:ClickHouse/ClickHouse.git $ git clone --recursive https://github.com/ClickHouse/ClickHouse.git ``` -## クリックハウスを構築 {#build-clickhouse} +## ビルドClickHouse {#build-clickhouse} ``` bash $ cd ClickHouse @@ -66,21 +66,21 @@ $ ninja $ cd .. ``` -実行可能ファイルを作成するには `ninja clickhouse`. -これは作成します `programs/clickhouse` 実行可能ファイルは、次の場所で使用できます。 `client` または `server` 引数。 +実行可能ファイルを作成するには、 `ninja clickhouse`. +これは作成します `programs/clickhouse` 実行可能ファイル `client` または `server` 引数。 -# 任意のlinux上でclickhouseを構築する方法 {#how-to-build-clickhouse-on-any-linux} +# 任意のLinux上でClickHouseを構築する方法 {#how-to-build-clickhouse-on-any-linux} の構築が必要で以下のコンポーネント: -- Git（ソースのチェックアウトにのみ使用され、ビルドには必要ありません) +- Git（ソースをチェックアウトするためにのみ使用され、ビルドには必要ありません) - CMake3.10以降 - 忍者（推奨）または作る - C++コンパイラ:gcc9またはclang8以降 -- リンカ：lldまたはgold（古典的なgnu ldは動作しません) -- Python（LLVMビルド内でのみ使用され、オプションです) +- リンカ:lldまたはgold(古典的なGNU ldは動作しません) +- Python(LLVMビルド内でのみ使用され、オプションです) -すべてのコンポーネントがインストールされている場合は、上記の手順と同じ方法で構築できます。 +すべてのコンポーネントがインストールされている場合、上記の手順と同じ方法でビルドできます。 Ubuntu Eoanの例: @@ -91,7 +91,7 @@ Ubuntu Eoanの例: cmake ../ClickHouse ninja -OpenSUSE Tumbleweedの例: +OpenSUSEタンブルウィードの例: sudo zypper install git cmake ninja gcc-c++ python lld git clone --recursive https://github.com/ClickHouse/ClickHouse.git @@ -99,7 +99,7 @@ OpenSUSE Tumbleweedの例: cmake ../ClickHouse ninja -Fedoraの生皮のための例: +Fedora Rawhideの例: sudo yum update yum --nogpg install git cmake make gcc-c++ python2 @@ -108,31 +108,31 @@ Fedoraの生皮のための例: cmake ../ClickHouse make -j $(nproc) -# クリックハウスを構築する必要はありません {#you-dont-have-to-build-clickhouse} +# ClickHouseを構築する必要はありません {#you-dont-have-to-build-clickhouse} ClickHouseは、事前に構築されたバイナリとパッケージで利用可能です。 バイナリは移植性があり、任意のLinuxフレーバーで実行できます。 これらのために、安定したprestable-試験スリリースして毎にコミットマスターすべてを引きます。 -から新鮮なビルドを見つけるには `master`、に行く [コミットページ](https://github.com/ClickHouse/ClickHouse/commits/master) 最初の緑色のチェックマークまたはコミット近くの赤い十字をクリックし、 “Details” 右後にリンク “ClickHouse Build Check”. +から新鮮なビルドを見つけるには `master`,に行く [コミットページ](https://github.com/ClickHouse/ClickHouse/commits/master) 最初の緑色のチェックマークまたはコミットの近くにある赤い十字をクリックし、 “Details” 右の後にリンク “ClickHouse Build Check”. -# ClickHouse Debianパッケージをビルドする方法 {#how-to-build-clickhouse-debian-package} +# ClickHouse Debianパッケージのビルド方法 {#how-to-build-clickhouse-debian-package} -## イgitありそう {#install-git-and-pbuilder} +## GitとPbuilderのインストール {#install-git-and-pbuilder} ``` bash $ sudo apt-get update $ sudo apt-get install git python pbuilder debhelper lsb-release fakeroot sudo debian-archive-keyring debian-keyring ``` -## レclickhouse源 {#checkout-clickhouse-sources-1} +## ﾂつｨﾂ姪"ﾂ債ﾂつｹ {#checkout-clickhouse-sources-1} ``` bash $ git clone --recursive --branch master https://github.com/ClickHouse/ClickHouse.git $ cd ClickHouse ``` -## Releaseスクリプトを実行 {#run-release-script} +## 解放スクリプトの実行 {#run-release-script} ``` bash $ ./release diff --git a/docs/ja/development/contrib.md b/docs/ja/development/contrib.md index 5fb0ae8a77e..b8917511e4c 100644 --- a/docs/ja/development/contrib.md +++ b/docs/ja/development/contrib.md @@ -1,43 +1,43 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 70 toc_title: "\u30B5\u30FC\u30C9\u30D1\u30FC\u30C6\u30A3\u88FD\u30E9\u30A4\u30D6\u30E9\ - \u30EA" + \u30EA\u3092\u4F7F\u7528" --- -# サードパーティ製ライブラリ {#third-party-libraries-used} +# サードパーティ製ライブラリを使用 {#third-party-libraries-used} -| ライブラリ | ライセンス | -|------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------| -| base64 | [BSD2-条項ライセンス](https://github.com/aklomp/base64/blob/a27c565d1b6c676beaf297fe503c4518185666f7/LICENSE) | -| ブースト | [Boost Software License1.0](https://github.com/ClickHouse-Extras/boost-extra/blob/6883b40449f378019aec792f9983ce3afc7ff16e/LICENSE_1_0.txt) | -| brotli | [MIT](https://github.com/google/brotli/blob/master/LICENSE) | -| capnproto | [MIT](https://github.com/capnproto/capnproto/blob/master/LICENSE) | -| cctz | [Apacheライセンス2.0](https://github.com/google/cctz/blob/4f9776a310f4952454636363def82c2bf6641d5f/LICENSE.txt) | -| ダブル変換 | [BSD3-条項ライセンス](https://github.com/google/double-conversion/blob/cf2f0f3d547dc73b4612028a155b80536902ba02/LICENSE) | -| FastMemcpy | [MIT](https://github.com/ClickHouse/ClickHouse/blob/master/libs/libmemcpy/impl/LICENSE) | -| googletest | [BSD3-条項ライセンス](https://github.com/google/googletest/blob/master/LICENSE) | -| h3unit description in lists | [Apacheライセンス2.0](https://github.com/uber/h3/blob/master/LICENSE) | -| hyperscan | [BSD3-条項ライセンス](https://github.com/intel/hyperscan/blob/master/LICENSE) | -| libbtrie | [BSD2-条項ライセンス](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/libbtrie/LICENSE) | -| libcxxabi | [BSD + MIT](https://github.com/ClickHouse/ClickHouse/blob/master/libs/libglibc-compatibility/libcxxabi/LICENSE.TXT) | -| libdividename | [Zlibライセンス](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/libdivide/LICENSE.txt) | -| libgsasl | [LGPL v2.1](https://github.com/ClickHouse-Extras/libgsasl/blob/3b8948a4042e34fb00b4fb987535dc9e02e39040/LICENSE) | -| libhdfs3 | [Apacheライセンス2.0](https://github.com/ClickHouse-Extras/libhdfs3/blob/bd6505cbb0c130b0db695305b9a38546fa880e5a/LICENSE.txt) | -| libmetrohash | [Apacheライセンス2.0](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/libmetrohash/LICENSE) | -| libpcg-ランダム | [Apacheライセンス2.0](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/libpcg-random/LICENSE-APACHE.txt) | -| libressl | [OpenSSLライセンス](https://github.com/ClickHouse-Extras/ssl/blob/master/COPYING) | -| librdkafka | [BSD2-条項ライセンス](https://github.com/edenhill/librdkafka/blob/363dcad5a23dc29381cc626620e68ae418b3af19/LICENSE) | -| libwidechar\_widthname | [CC0 1.0ユニバーサル](https://github.com/ClickHouse/ClickHouse/blob/master/libs/libwidechar_width/LICENSE) | -| llvm | [BSD3-条項ライセンス](https://github.com/ClickHouse-Extras/llvm/blob/163def217817c90fb982a6daf384744d8472b92b/llvm/LICENSE.TXT) | -| lz4comment | [BSD2-条項ライセンス](https://github.com/lz4/lz4/blob/c10863b98e1503af90616ae99725ecd120265dfb/LICENSE) | -| mariadb-connector-c | [LGPL v2.1](https://github.com/ClickHouse-Extras/mariadb-connector-c/blob/3.1/COPYING.LIB) | -| murmurhash | [パブリック](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/murmurhash/LICENSE) | -| pdqsort | [Zlibライセンス](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/pdqsort/license.txt) | -| ポコ | [Boostソフトウェアライセンス-Version1.0](https://github.com/ClickHouse-Extras/poco/blob/fe5505e56c27b6ecb0dcbc40c49dc2caf4e9637f/LICENSE) | -| protobuf | [BSD3-条項ライセンス](https://github.com/ClickHouse-Extras/protobuf/blob/12735370922a35f03999afff478e1c6d7aa917a4/LICENSE) | -| re2unit description in lists | [BSD3-条項ライセンス](https://github.com/google/re2/blob/7cf8b88e8f70f97fd4926b56aa87e7f53b2717e0/LICENSE) | -| UnixODBC | [LGPL v2.1](https://github.com/ClickHouse-Extras/UnixODBC/tree/b0ad30f7f6289c12b76f04bfb9d466374bb32168) | -| zlib-ng | [Zlibライセンス](https://github.com/ClickHouse-Extras/zlib-ng/blob/develop/LICENSE.md) | -| zstd | [BSD3-条項ライセンス](https://github.com/facebook/zstd/blob/dev/LICENSE) | +| ライブラリ | ライセンス | +|--------------------|---------------------------------------------------------------------------------------------------------------------------------------------| +| base64 | [BSD2条項ライセンス](https://github.com/aklomp/base64/blob/a27c565d1b6c676beaf297fe503c4518185666f7/LICENSE) | +| ブースト | [Boost Software License1.0](https://github.com/ClickHouse-Extras/boost-extra/blob/6883b40449f378019aec792f9983ce3afc7ff16e/LICENSE_1_0.txt) | +| ブロトリ | [MIT](https://github.com/google/brotli/blob/master/LICENSE) | +| capnproto | [MIT](https://github.com/capnproto/capnproto/blob/master/LICENSE) | +| cctz | [Apacheライセンス2.0](https://github.com/google/cctz/blob/4f9776a310f4952454636363def82c2bf6641d5f/LICENSE.txt) | +| 二重変換 | [BSD3条項ライセンス](https://github.com/google/double-conversion/blob/cf2f0f3d547dc73b4612028a155b80536902ba02/LICENSE) | +| FastMemcpy | [MIT](https://github.com/ClickHouse/ClickHouse/blob/master/libs/libmemcpy/impl/LICENSE) | +| googletest | [BSD3条項ライセンス](https://github.com/google/googletest/blob/master/LICENSE) | +| h3 | [Apacheライセンス2.0](https://github.com/uber/h3/blob/master/LICENSE) | +| hyperscan | [BSD3条項ライセンス](https://github.com/intel/hyperscan/blob/master/LICENSE) | +| libbtrie | [BSD2条項ライセンス](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/libbtrie/LICENSE) | +| libcxxabi | [BSD + MIT](https://github.com/ClickHouse/ClickHouse/blob/master/libs/libglibc-compatibility/libcxxabi/LICENSE.TXT) | +| libdivide | [Zlibライセンス](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/libdivide/LICENSE.txt) | +| libgsasl | [LGPL v2.1](https://github.com/ClickHouse-Extras/libgsasl/blob/3b8948a4042e34fb00b4fb987535dc9e02e39040/LICENSE) | +| libhdfs3 | [Apacheライセンス2.0](https://github.com/ClickHouse-Extras/libhdfs3/blob/bd6505cbb0c130b0db695305b9a38546fa880e5a/LICENSE.txt) | +| libmetrohash | [Apacheライセンス2.0](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/libmetrohash/LICENSE) | +| libpcg-ランダム | [Apacheライセンス2.0](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/libpcg-random/LICENSE-APACHE.txt) | +| libressl | [OpenSSLライセンス](https://github.com/ClickHouse-Extras/ssl/blob/master/COPYING) | +| リブドカフカ | [BSD2条項ライセンス](https://github.com/edenhill/librdkafka/blob/363dcad5a23dc29381cc626620e68ae418b3af19/LICENSE) | +| libwidechar\_width | [CC0 1.0ユニバーサル](https://github.com/ClickHouse/ClickHouse/blob/master/libs/libwidechar_width/LICENSE) | +| llvm | [BSD3条項ライセンス](https://github.com/ClickHouse-Extras/llvm/blob/163def217817c90fb982a6daf384744d8472b92b/llvm/LICENSE.TXT) | +| lz4 | [BSD2条項ライセンス](https://github.com/lz4/lz4/blob/c10863b98e1503af90616ae99725ecd120265dfb/LICENSE) | +| mariadb-コネクタ-c | [LGPL v2.1](https://github.com/ClickHouse-Extras/mariadb-connector-c/blob/3.1/COPYING.LIB) | +| murmurhash | [パブリック](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/murmurhash/LICENSE) | +| pdqsort | [Zlibライセンス](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/pdqsort/license.txt) | +| ポコ | [Boost Software License-バージョン1.0](https://github.com/ClickHouse-Extras/poco/blob/fe5505e56c27b6ecb0dcbc40c49dc2caf4e9637f/LICENSE) | +| プロトブフ | [BSD3条項ライセンス](https://github.com/ClickHouse-Extras/protobuf/blob/12735370922a35f03999afff478e1c6d7aa917a4/LICENSE) | +| re2 | [BSD3条項ライセンス](https://github.com/google/re2/blob/7cf8b88e8f70f97fd4926b56aa87e7f53b2717e0/LICENSE) | +| UnixODBC | [LGPL v2.1](https://github.com/ClickHouse-Extras/UnixODBC/tree/b0ad30f7f6289c12b76f04bfb9d466374bb32168) | +| zlib-ng | [Zlibライセンス](https://github.com/ClickHouse-Extras/zlib-ng/blob/develop/LICENSE.md) | +| zstd | [BSD3条項ライセンス](https://github.com/facebook/zstd/blob/dev/LICENSE) | diff --git a/docs/ja/development/developer-instruction.md b/docs/ja/development/developer-instruction.md index 32850a643b5..d65b25bd98c 100644 --- a/docs/ja/development/developer-instruction.md +++ b/docs/ja/development/developer-instruction.md @@ -1,64 +1,63 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 61 -toc_title: "\u521D\u5FC3\u8005\u306E\u65B9ClickHouse\u958B\u767A\u8005\u306E\u6307\ - \u793A" +toc_title: "\u521D\u5FC3\u8005ClickHouse\u958B\u767A\u8005\u306E\u6307\u793A" --- -ClickHouseの建物は、Linux、FreeBSDおよびMac OS Xでサポートされています。 +ClickHouseのビルドはLinux、FreeBSD、Mac OS Xでサポートされています。 # Windowsを使用する場合 {#if-you-use-windows} -Windowsを使用する場合は、Ubuntuで仮想マシンを作成する必要があります。 するのは、仮想マシンをインストールしてくださいVirtualBox. ダウンロードできますUbuntuのウェブサイト：https://www.ubuntu.com/\#download. を作成してください仮想マシンからダウンロードした画像を保少なくとも4GB RAMめます。 Ubuntuでコマンドライン端末を実行するには、その単語を含むプログラムを探してください “terminal” その名前（gnome端末、konsoleなど）で）またはCtrl+Alt+Tを押すだけです。 +Windowsを使用する場合は、Ubuntuで仮想マシンを作成する必要があります。 するのは、仮想マシンをインストールしてくださいVirtualBox. ダウンロードできますUbuntuのウェブサイト：https://www.ubuntu.com/\#download. を作成してください仮想マシンからダウンロードした画像を保少なくとも4GB RAMめます。 Ubuntuでコマンドライン端末を実行するには、その単語を含むプログラムを見つけてください “terminal” その名前で（gnome-terminal、konsoleなど。）または単にCtrl+Alt+Tを押します。 # 32ビットシステムを使用する場合 {#if-you-use-a-32-bit-system} -ClickHouseは32ビットシステム上で動作または構築することはできません。 きの獲得へのアクセスでは、64ビットのシステムを継続できる。 +ClickHouseできない仕事を32ビットのシステム。 きの獲得へのアクセスでは、64ビットのシステムを継続できる。 # GitHubでのリポジトリの作成 {#creating-a-repository-on-github} -ClickHouseリポジトリでの作業を開始するには、GitHubアカウントが必要です。 +ClickHouse repositoryで作業を開始するには、GitHubアカウントが必要です。 -きょうてつもない場合は、ご登録が必要でhttps://github.com. についてsshキーを押すと、すべき操作を行うというテーマをアップロードしてgithub. それはあなたのパッチを送信するために必要です。 他のsshサーバーと同じsshキーを使用することも可能です。 +おそらく既に持っていますが、そうでない場合は、登録してくださいhttps://github.com.SSHキーがない場合は、それらを生成してGitHubにアップロードする必要があります。 あなたのパッチを送信するために必要です。 他のSSHサーバーで使用するのと同じSSHキーを使用することも可能です。 -ClickHouseリポジトリのフォークを作成します。 それを行うにはクリックしてください “fork” 右上のボタンhttps://github.com/ClickHouse/ClickHouse.それはフォークあなた自身のコピーのClickHouse/ClickHouseにあなたのアカウント。 +ClickHouseリポジトリのフォークを作成します。 それを行うには、 “fork” 右上のボタンhttps://github.com/ClickHouse/ClickHouse.それはあなたのアカウントにClickHouse/ClickHouseの独自のコピーをフォークします。 -開発プロセスは、最初に意図した変更をclickhouseのフォークにコミットし、次に “pull request” これらの変更がメインリポジトリ（ClickHouse/ClickHouse）に受け入れられるために。 +開発プロセスは、最初に意図した変更をClickHouseのフォークにコミットし、次に “pull request” これらの変更をメインリポジトリ（ClickHouse/ClickHouse）に受け入れる。 -Gitリポジトリで作業するには `git`. +作gitリポジトリをインストールしてください `git`. Ubuntuでこれを行うには、コマンドラインターミナルで実行します: sudo apt update sudo apt install git -Gitの使用に関する簡単なマニュアルは、ここにあります：https://services.github.com/on-demand/downloads/github-git-cheat-sheet.pdf。 -Gitの詳細なマニュアルについては、https://git-scm.com/book/en/v2。 +簡単なマニュアルを使用Gitで、できるだけ早く送ってくださhttps://services.github.com/on-demand/downloads/github-git-cheat-sheet.pdf. +詳細なマニュアルGit見https://git-scm.com/book/en/v2. -# 開発マシンへのリポジトリのクローン作成 {#cloning-a-repository-to-your-development-machine} +# 開発マシンへのリポジトリの複製 {#cloning-a-repository-to-your-development-machine} 次に、ソースファイルを作業マシンにダウンロードする必要があります。 これは “to clone a repository” 作業マシン上にリポジトリのローカルコピーを作成するためです。 -コマンドライン端末で: +コマンドラインターミナルで実行: - git clone --recursive git@guthub.com:your_github_username/ClickHouse.git + git clone --recursive git@github.com:your_github_username/ClickHouse.git cd ClickHouse -注:ください、代用 *your\_github\_username* 適切なもので！ +注:、代理して下さい *your\_github\_username* 適切なもので！ -このコマ `ClickHouse` プロジェクトの作業コピーを含む。 +このコマンドディレクトリの作成 `ClickHouse` プロジェクトの作業コピーを含む。 -作業ディレクトリへのパスには、ビルドシステムの実行に問題が生じる可能性があるため、空白が含まれていないことが重要です。 +ビルドシステムの実行に問題が生じる可能性があるため、作業ディレクトリへのパスに空白が含まれていないことが重要です。 -ごclickhouseリポジトリ用 `submodules`. That is what the references to additional repositories are called (i.e. external libraries on which the project depends). It means that when cloning the repository you need to specify the `--recursive` 上記の例のようにフラグ。 場合のリポジトリにてクローニングなsubmodules、ダウンロードを実行する必要がありますの: +ClickHouseリポジトリは以下を使用します `submodules`. That is what the references to additional repositories are called (i.e. external libraries on which the project depends). It means that when cloning the repository you need to specify the `--recursive` 上記の例のようにフラグ。 場合のリポジトリにてクローニングなsubmodules、ダウンロードを実行する必要がありますの: git submodule init git submodule update -ステータスは次のコマンドで確認できます: `git submodule status`. +このコマンドでステータスを確認できます: `git submodule status`. -次のエラーメッセージが表示された場合: +次のエラーメッセージが表示される場合: Permission denied (publickey). fatal: Could not read from remote repository. @@ -66,19 +65,19 @@ Gitの詳細なマニュアルについては、https://git-scm.com/book/en/v2 Please make sure you have the correct access rights and the repository exists. -一般的には、githubに接続するためのsshキーがないことを意味します。 これらのキーは普通あります `~/.ssh`. SSHキーを受け入れるには、GitHub UIの設定セクションにそれらをアップロードする必要があります。 +一般に、GitHubに接続するためのSSHキーがないことを意味します。 これらのキーは、通常、 `~/.ssh`. SSHキーを受け入れるには、GitHub UIの設定セクションにアップロードする必要があります。 またクローンをリポジトリによhttpsプロトコル: git clone https://github.com/ClickHouse/ClickHouse.git -ただし、これにより、変更をサーバーに送信することはできません。 一時的に使用して、後でリポジトリのリモートアドレスを置き換えるsshキーを追加することはできます `git remote` 司令部 +ただし、変更をサーバーに送信することはできません。 にもそのままお使いいただけで一時的にメモリの使用範囲のサイズはSSHキーの後に交換し、リモートアドレスのリポジトリ `git remote` コマンド -元のclickhouseレポのアドレスをローカルリポジトリに追加して、そこから更新を取得することもできます: +元のClickHouseレポのアドレスをローカルリポジトリに追加して、そこから更新を取得することもできます: git remote add upstream git@github.com:ClickHouse/ClickHouse.git -このコマンドを正常に実行すると、メインのclickhouseレポから更新をプルすることができます `git pull upstream master`. +このコマンドを正常に実行すると、次のようにしてメインのClickHouseリポジトリから更新を取得できます `git pull upstream master`. ## サブモジュールの操作 {#working-with-submodules} @@ -94,7 +93,7 @@ Gitでサブモジュールを操作するのは苦痛です。 次のコマン # Two last commands could be merged together git submodule update --init -次のコマンドは、すべてのサブモジュールを初期状態にリセットするのに役立ちます（！警告！ -内部の変更は削除されます): +次のコマンドは、すべてのサブモジュールを初期状態にリセットするのに役立ちます（！ﾂづﾂつｷﾂ。 -内部の変更は削除されます): # Synchronizes submodules' remote URL with .gitmodules git submodule sync --recursive @@ -112,52 +111,52 @@ Gitでサブモジュールを操作するのは苦痛です。 次のコマン # ビルドシステム {#build-system} -ClickHouseはCMakeとNinjaを使用して建物を建てます。 +ClickHouseは、構築のためのCMakeと忍者を使用しています。 -CMake-Ninjaファイル（ビルドタスク）を生成できるメタビルドシステム。 -Ninja-これらのcmake生成タスクを実行するために使用される速度に焦点を当てた小さなビルドシステム。 +CMake-忍者ファイル（ビルドタスク）を生成することができるメタビルドシステム。 +忍者-これらのcmake生成されたタスクを実行するために使用される速度に焦点を当てた小さなビルドシステム。 Ubuntu、DebianまたはMint runにインストールするには `sudo apt install cmake ninja-build`. -セントスでは、redhatは実行します `sudo yum install cmake ninja-build`. +セントスでは、レッドハットラン `sudo yum install cmake ninja-build`. -ArchまたはGentooを使用している場合は、CMakeをインストールする方法を知っているでしょう。 +ArchまたはGentooを使用する場合は、おそらくCMakeのインストール方法を自分で知っています。 -のためのcmakeおよび忍者mac os x初めて自作としてインストールインストールさんによbrew: +のためのCMakeおよび忍者Mac OS X初めて自作としてインストールインストールさんによbrew: /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" brew install cmake ninja -次に、cmakeのバージョンを確認します: `cmake --version`. 場合は以下の3.3v、必要新しいバージョンからのウェブサイト：https://cmake.org/download/. +次に、CMakeのバージョンを確認します: `cmake --version`. 3.3未満の場合は、webサイトから新しいバージョンをインストールしてください。https://cmake.org/download/. -# 省略可能な外部ライブラリ {#optional-external-libraries} +# 任意の外部ライブラリ {#optional-external-libraries} -ClickHouse複数の外部ライブラリのためのビルです。 それらのすべては、サブモジュールにあるソースからClickHouseと一緒に構築されるので、別々にインストールする必要はありません。 リストをチェックすることができ `contrib`. +ClickHouseはビルドに複数の外部ライブラリを使用します。 それらのすべては、サブモジュールにあるソースからClickHouseと一緒に構築されているので、別々にインストールする必要はありません。 リストは次の場所で確認できます `contrib`. # C++コンパイラ {#c-compiler} -バージョン9とclangバージョン8以上から始まるコンパイラgccは、clickhouseの構築に対応しています。 +ClickHouseのビルドには、バージョン9以降のGCCとClangバージョン8以降のコンパイラがサポートされます。 -公式のyandexビルドは、現在、gccを使用しています。 そしてclangは開発のために通常より便利です。 が、当社の継続的インテグレーション（ci)プラットフォームを運チェックのための十数の組み合わせとなります。 +公式のYandexビルドは、わずかに優れたパフォーマンスのマシンコードを生成するため、GCCを使用しています（私たちのベンチマークに応じて最大数パーセントの そしてClangは開発のために通常より便利です。 が、当社の継続的インテグレーション（CI)プラットフォームを運チェックのための十数の組み合わせとなります。 -UbuntuにGCCをインストールするには: `sudo apt install gcc g++` +UBUNTUにGCCをインストールするには: `sudo apt install gcc g++` -Gccのバージョンを確認する: `gcc --version`. それが9以下の場合は、ここで指示に従ってください：https://clickhouse。テック/ドキュメント/en/開発/ビルド/\#インストール-gcc-9. +Gccのバージョンを確認する: `gcc --version`. の場合は下記9その指示に従う。https://clickhouse.tech/docs/en/development/build/\#install-gcc-9. Mac OS XのビルドはClangでのみサポートされています。 ちょうど実行 `brew install llvm` -Clangを使用する場合は、インストールすることもできます `libc++` と `lld` くださるのです。 を使用して `ccache` また、推奨されます。 +Clangを使用する場合は、次のものもインストールできます `libc++` と `lld` あなたがそれが何であるか知っていれば。 を使用して `ccache` また、推奨されます。 # 建築プロセス {#the-building-process} -ClickHouseをビルドする準備ができたら、別のディレクトリを作成することをお勧めします `build` 中 `ClickHouse` それはすべてのビルドの成果物が含まれています: +ClickHouseを構築する準備ができたので、別のディレクトリを作成することをお勧めします `build` 内部 `ClickHouse` それはすべてのビルド人工物が含まれています: mkdir build cd build いくつかの異なるディレクトリ（build\_release、build\_debugなど）を持つことができます。）ビルドの異なるタイプのために。 -内部にいる間 `build` ディレクトリ、CMakeを実行してビルドを設定します。 最初の実行の前に、コンパイラ（この例ではバージョン9gccコンパイラ）を指定する環境変数を定義する必要があります。 +中の間 `build` cmakeを実行してビルドを構成します。 最初の実行の前に、コンパイラ（この例ではバージョン9gccコンパイラ）を指定する環境変数を定義する必要があります。 Linux: @@ -169,92 +168,92 @@ Mac OS X: export CC=clang CXX=clang++ cmake .. -その `CC` variableはCのコンパイラ(Cコンパイラの略)を指定する。 `CXX` 変数は、どのC++コンパイラをビルドに使用するかを指示します。 +その `CC` 変数は、Cのコンパイラを指定します(cコンパイラの略)。 `CXX` variableは、どのC++コンパイラをビルドに使用するかを指示します。 -より速いビルドのために、あなたは `debug` ビルドタイプ-最適化のないビルド。 その供給のために次のパラメータ `-D CMAKE_BUILD_TYPE=Debug`: +より高速なビルドのために、あなたは `debug` ビルドタイプ-最適化のないビルド。 その供給は、以下のパラメータ `-D CMAKE_BUILD_TYPE=Debug`: cmake -D CMAKE_BUILD_TYPE=Debug .. -このコマンドを実行することで、ビルドのタイプを変更できます。 `build` ディレクトリ。 +ビルドのタイプを変更するには、次のコマンドを実行します。 `build` ディレクトリ。 -ビルドするために忍者を実行: +忍者を実行して構築する: ninja clickhouse-server clickhouse-client -この例では、必要なバイナリのみがビルドされます。 +この例では、必要なバイナリのみを構築します。 -すべてのバイナリ（ユーティリティとテス: +必要な場合は、構築すべてのバイナリ(光熱費および試験）を動かして行く必要がある忍者のないパラメータ: ninja -フルビルドでは、メインバイナリを構築するために約30gbの空きディスク容量または15gbが必要です。 +フルの構築が必要約30GBのディスクスペースまたは15GBの主binaries. -ビルドマシンで大量のramが利用可能な場合は、並列に実行されるビルドタスクの数を制限する必要があります `-j` パラメータcomment: +ビルドマシン上で大量のRAMが利用可能な場合は、以下と並行して実行されるビルドタスクの数を制限する必要があります `-j` param: ninja -j 1 clickhouse-server clickhouse-client -RAMが4GBのマシンでは、1をRAM8GBに指定することをお勧めします `-j 2` は推奨。 +4GBのRAMを搭載しているマシンでは、1を8GBのRAMに指定することをお勧めします `-j 2` 推奨されます。 -メッセージが表示された場合: `ninja: error: loading 'build.ninja': No such file or directory` これは、ビルド構成の生成に失敗し、上記のメッセージを調べる必要があることを意味します。 +メッセージが届いたら: `ninja: error: loading 'build.ninja': No such file or directory` これは、ビルド構成の生成が失敗し、上記のメッセージを検査する必要があることを意味します。 ビルドプロセスが正常に開始されると、ビルドの進行状況、つまり処理されたタスクの数とタスクの総数が表示されます。 -ながらメッセージについてprotobufファイルlibhdfs2図書館のような `libprotobuf WARNING` 現れるかもしれない 彼らは何にも影響せず、無視されても安全です。 +ながらメッセージについてprotobufファイルlibhdfs2図書館のような `libprotobuf WARNING` 現れるかも 彼らは何も影響を与えず、無視されても安全です。 -成功を構築するの実行ファイル `ClickHouse//programs/clickhouse`: +ビルドが成功すると、実行可能ファイルを取得します `ClickHouse//programs/clickhouse`: ls -l programs/clickhouse -# ClickHouseの構築された実行可能ファイルを実行する {#running-the-built-executable-of-clickhouse} +# ClickHouseのビルドされた実行可能ファイルの実行 {#running-the-built-executable-of-clickhouse} -のサーバーの現在のユーザーに必要なナビゲート `ClickHouse/programs/server/` (の外にあります `build`)と実行: +現在のユーザーの下でサーバーを実行するには、次の場所に移動します `ClickHouse/programs/server/` （外にあります `build`)と実行: - ../../../build/programs/clickhouse server + ../../build/programs/clickhouse server -この場合、clickhouseは現在のディレクトリにある設定ファイルを使用します。 実行することができ `clickhouse server` からのディレクトリのパスを指定し、設定ファイルとしてコマンドラインパラメータ `--config-file`. +この場合、ClickHouseは現在のディレクトリにある設定ファイルを使用します。 実行できます `clickhouse server` からのディレクトリのパスを指定し、設定ファイルとしてコマンドラインパラメータ `--config-file`. -別のターミナルのclickhouse-clientでclickhouseに接続するには、次のように移動します `ClickHouse/build/programs/` と実行 `clickhouse client`. +別のターミナルでclickhouse-clientを使用してClickHouseに接続するには、次の場所に移動します `ClickHouse/build/programs/` と実行 `clickhouse client`. -あなたが得れば `Connection refused` Mac OS XまたはFreeBSDで、ホストアドレス127.0.0.1を指定してみてください: +あなたが得る場合 `Connection refused` メッセージMac OS XまたはFreeBSDでは、ホストアドレス127.0.0.1を指定してみます: clickhouse client --host 127.0.0.1 -に置き換えることができ生産版clickhouseバイナリインストールされるシステムのカスタム構築clickhouseバイナリー. これを行うには、公式サイトの指示に従ってマシンにclickhouseをインストールします。 次に、以下を実行します: +に置き換えることができ生産版ClickHouseバイナリインストールされるシステムのカスタム構築ClickHouseバイナリー. いるイClickHouse利用するマシンの指示に従って公式サイトから 次に、以下を実行します: sudo service clickhouse-server stop sudo cp ClickHouse/build/programs/clickhouse /usr/bin/ sudo service clickhouse-server start -それに注意 `clickhouse-client`, `clickhouse-server` どsymlinksの共通 `clickhouse` バイナリ +なお `clickhouse-client`, `clickhouse-server` そして、他のものは、一般的に共有される `clickhouse` バイナリ -を運営することも可能ですカスタム構築clickhouseバイナリのコンフィグファイルからのclickhouseパッケージをインストールシステム: +を運営することも可能ですカスタム構築ClickHouseバイナリのコンフィグファイルからのClickHouseパッケージをインストールシステム: sudo service clickhouse-server stop sudo -u clickhouse ClickHouse/build/programs/clickhouse server --config-file /etc/clickhouse-server/config.xml # IDE(統合開発環境) {#ide-integrated-development-environment} -使用するideがわからない場合は、clionを使用することをお勧めします。 clionは商用ソフトウェアですが、30日間の無料試用期間を提供しています。 また、学生のための無料です。 clionは、linuxとmac os xの両方で使用できます。 +使用するIDEがわからない場合は、CLionを使用することをお勧めします。 CLionは商用ソフトウェアですが、30日間の無料試用期間を提供しています。 また、学生のための無料です。 CLionはLinuxとMac OS Xの両方で使用できます。 -KDevelopとQTCreatorは、ClickHouseを開発するためのIDEの他の優れた選択肢です。 KDevelopは非常に便利なIDEとして提供されますが、不安定です。 プロジェクトを開いてしばらくしてからKDevelopがクラッシュした場合は、 “Stop All” プロジェクトのファイルのリストを開くとすぐにボタンを押します。 その後、KDevelopはうまく動作するはずです。 +KDevelopとQTCreatorは、ClickHouseを開発するためのIDEの他の優れた選択肢です。 KDevelopは非常に便利なIDEとして提供されますが、不安定です。 まKDevelopクラッシュが開設すプロジェクト、クリック “Stop All” プロジェクトのファイルのリストを開くとすぐにボタンを押します。 そうした後、KDevelopはうまくいくはずです。 -シンプルなコードエディタとして、sublime textまたはvisual studio code、またはkate（すべてlinuxで利用可能）を使用できます。 +単純なコードエディターとして、Sublime TextまたはVisual Studio Code、またはKate(すべてLinuxで利用可能)を使用できます。 -その場合には、clionが `build` パスはそれ自身で、それはまたそれ自身で選択します `debug` ビルドタイプの場合は、設定のために、あなたがインストールしたものではなく、CLionで定義されたバージョンのCMakeを使用し、最後にCLionが使用します `make` ビルドタスクを実行するには `ninja`. これは通常の動作ですが、混乱を避けるためにそれを念頭に置いてください。 +念のため、CLionが作成することに言及する価値があります `build` 独自のパス、それはまた、独自の選択に `debug` ビルドタイプの場合、構成のために、CLionで定義されているCMakeのバージョンを使用し、インストールされているものではありません。 `make` ビルドタスクを実行するには `ninja`. これは通常の動作ですが、混乱を避けるためにそれを念頭に置いてください。 -# コードの記述 {#writing-code} +# コードの作成 {#writing-code} -ClickHouseの建築の記述はここに見つけることができる:https://clickhouse。技術/ドキュメント/en/開発/アーキテクチャ/ +の説明ClickHouse建築で、できるだけ早く送ってくださhttps://clickhouse.tech/docs/en/開発/アーキテクチャ/ -コードスタイルガイド：https://clickhouse。テック/ドキュメント/en/開発/スタイル/ +コードのスタイルガイド:https://clickhouse.tech/docs/en/開発/スタイル/ -書き込みテスト:https://clickhouse。技術/ドキュメント/en/開発/テスト/ +筆記試験：https://clickhouse.tech/docs/en/development/tests/ -タスクのリスト：https://github.com/clickhouse/clickhouse/blob/master/testsructions/easy\_tasks\_sorted\_en.md +タスクのリスト：https://github.com/ClickHouse/ClickHouse/blob/master/testsructions/easy\_tasks\_sorted\_en.md # テストデータ {#test-data} -開発clickhouseが必要となり載荷実ックスです。 パフォーマンステストでは特に重要です。 yandexからの匿名化されたデータの特別に準備されたセットがあります。メトリカ さらに、3gbの空きディスク容量が必要です。 このデータがないの達成に必要なものの開発事ができます。 +開発ClickHouseが必要となり載荷実ックスです。 パフォーマンステストでは特に重要です。 して特定の匿名化データからのYandex.メトリカ さらに3GBの空きディスク領域が必要です。 このデータは、ほとんどの開発タスクを実行するためには必要ありません。 sudo apt install wget xz-utils @@ -266,6 +265,8 @@ ClickHouseの建築の記述はここに見つけることができる:https://c clickhouse-client + CREATE DATABASE IF NOT EXISTS test + CREATE TABLE test.hits ( WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String, RefererDomain String, Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), UTCEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), RemoteIP UInt32, RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming Int32, DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32, FetchTiming Int32, RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32, LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32, RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String, `ParsedParams.Key1` Array(String), `ParsedParams.Key2` Array(String), `ParsedParams.Key3` Array(String), `ParsedParams.Key4` Array(String), `ParsedParams.Key5` Array(String), `ParsedParams.ValueDouble` Array(Float64), IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree PARTITION BY toYYYYMM(EventDate) SAMPLE BY intHash32(UserID) ORDER BY (CounterID, EventDate, intHash32(UserID), EventTime); CREATE TABLE test.visits ( CounterID UInt32, StartDate Date, Sign Int8, IsNew UInt8, VisitID UInt64, UserID UInt64, StartTime DateTime, Duration UInt32, UTCStartTime DateTime, PageViews Int32, Hits Int32, IsBounce UInt8, Referer String, StartURL String, RefererDomain String, StartURLDomain String, EndURL String, LinkURL String, IsDownload UInt8, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, PlaceID Int32, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), IsYandex UInt8, GoalReachesDepth Int32, GoalReachesURL Int32, GoalReachesAny Int32, SocialSourceNetworkID UInt8, SocialSourcePage String, MobilePhoneModel String, ClientEventTime DateTime, RegionID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RemoteIP UInt32, RemoteIP6 FixedString(16), IPNetworkID UInt32, SilverlightVersion3 UInt32, CodeVersion UInt32, ResolutionWidth UInt16, ResolutionHeight UInt16, UserAgentMajor UInt16, UserAgentMinor UInt16, WindowClientWidth UInt16, WindowClientHeight UInt16, SilverlightVersion2 UInt8, SilverlightVersion4 UInt16, FlashVersion3 UInt16, FlashVersion4 UInt16, ClientTimeZone Int16, OS UInt8, UserAgent UInt8, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, NetMajor UInt8, NetMinor UInt8, MobilePhone UInt8, SilverlightVersion1 UInt8, Age UInt8, Sex UInt8, Income UInt8, JavaEnable UInt8, CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, BrowserLanguage UInt16, BrowserCountry UInt16, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), Params Array(String), `Goals.ID` Array(UInt32), `Goals.Serial` Array(UInt32), `Goals.EventTime` Array(DateTime), `Goals.Price` Array(Int64), `Goals.OrderID` Array(String), `Goals.CurrencyID` Array(UInt32), WatchIDs Array(UInt64), ParamSumPrice Int64, ParamCurrency FixedString(3), ParamCurrencyID UInt16, ClickLogID UInt64, ClickEventID Int32, ClickGoodEvent Int32, ClickEventTime DateTime, ClickPriorityID Int32, ClickPhraseID Int32, ClickPageID Int32, ClickPlaceID Int32, ClickTypeID Int32, ClickResourceID Int32, ClickCost UInt32, ClickClientIP UInt32, ClickDomainID UInt32, ClickURL String, ClickAttempt UInt8, ClickOrderID UInt32, ClickBannerID UInt32, ClickMarketCategoryID UInt32, ClickMarketPP UInt32, ClickMarketCategoryName String, ClickMarketPPName String, ClickAWAPSCampaignName String, ClickPageName String, ClickTargetType UInt16, ClickTargetPhraseID UInt64, ClickContextType UInt8, ClickSelectType Int8, ClickOptions String, ClickGroupBannerID Int32, OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, FirstVisit DateTime, PredLastVisit Date, LastVisit Date, TotalVisits UInt32, `TraficSource.ID` Array(Int8), `TraficSource.SearchEngineID` Array(UInt16), `TraficSource.AdvEngineID` Array(UInt8), `TraficSource.PlaceID` Array(UInt16), `TraficSource.SocialSourceNetworkID` Array(UInt8), `TraficSource.Domain` Array(String), `TraficSource.SearchPhrase` Array(String), `TraficSource.SocialSourcePage` Array(String), Attendance FixedString(16), CLID UInt32, YCLID UInt64, NormalizedRefererHash UInt64, SearchPhraseHash UInt64, RefererDomainHash UInt64, NormalizedStartURLHash UInt64, StartURLDomainHash UInt64, NormalizedEndURLHash UInt64, TopLevelDomain UInt64, URLScheme UInt64, OpenstatServiceNameHash UInt64, OpenstatCampaignIDHash UInt64, OpenstatAdIDHash UInt64, OpenstatSourceIDHash UInt64, UTMSourceHash UInt64, UTMMediumHash UInt64, UTMCampaignHash UInt64, UTMContentHash UInt64, UTMTermHash UInt64, FromHash UInt64, WebVisorEnabled UInt8, WebVisorActivity UInt32, `ParsedParams.Key1` Array(String), `ParsedParams.Key2` Array(String), `ParsedParams.Key3` Array(String), `ParsedParams.Key4` Array(String), `ParsedParams.Key5` Array(String), `ParsedParams.ValueDouble` Array(Float64), `Market.Type` Array(UInt8), `Market.GoalID` Array(UInt32), `Market.OrderID` Array(String), `Market.OrderPrice` Array(Int64), `Market.PP` Array(UInt32), `Market.DirectPlaceID` Array(UInt32), `Market.DirectOrderID` Array(UInt32), `Market.DirectBannerID` Array(UInt32), `Market.GoodID` Array(String), `Market.GoodName` Array(String), `Market.GoodQuantity` Array(Int32), `Market.GoodPrice` Array(Int64), IslandID FixedString(16)) ENGINE = CollapsingMergeTree(Sign) PARTITION BY toYYYYMM(StartDate) SAMPLE BY intHash32(UserID) ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID); @@ -275,12 +276,12 @@ ClickHouseの建築の記述はここに見つけることができる:https://c # プル要求の作成 {#creating-pull-request} -GitHubのUIであなたのフォークリポジトリに移動します。 ブランチで開発している場合は、そのブランチを選択する必要があります。 があるでしょう “Pull request” 画面にあるボタン。 本質的に、これは “create a request for accepting my changes into the main repository”. +GitHubのUIでforkリポジトリに移動します。 ブランチで開発している場合は、そのブランチを選択する必要があります。 があるでしょう “Pull request” 画面上にあるボタン。 本質的に、これは “create a request for accepting my changes into the main repository”. -作業がまだ完了していない場合でも、プル要求を作成できます。 この場合、単語を入れてください “WIP” （進行中の作業）タイトルの冒頭で、後で変更することができます。 これは、利用可能なすべてのテストを実行するだけでなく、変更の協調的なレビューや議論にも役立ちます。 変更内容の簡単な説明を入力することが重要ですが、後でリリースチェンジログの生成に使用されます。 +プル要求は、作業がまだ完了していない場合でも作成できます。 この場合、単語を入れてください “WIP” （進行中の作業）タイトルの先頭に、それは後で変更することができます。 これは、変更の協調的なレビューと議論、および利用可能なすべてのテストの実行に役立ちます。 変更の簡単な説明を提供することが重要です。 -テストは、yandexの従業員がタグであなたのprにラベルを付けるとすぐに開始します “can be tested”. The results of some first checks (e.g. code style) will come in within several minutes. Build check results will arrive within half an hour. And the main set of tests will report itself within an hour. +Yandexの従業員がタグであなたのPRにラベルを付けるとすぐにテストが開始されます “can be tested”. The results of some first checks (e.g. code style) will come in within several minutes. Build check results will arrive within half an hour. And the main set of tests will report itself within an hour. -システムは、プル要求のclickhouseバイナリビルドを個別に準備します。 これらのビルドを取得するには “Details” 次へのリンク “ClickHouse build check” チェックのリストのエントリ。 そこには、ビルドへの直接リンクがあります。あなたも、あなたの生産サーバー上に展開することができClickHouseのdebパッケージ(あなたは恐れていない場合). +システムは、プル要求用にClickHouseバイナリビルドを個別に準備します。 これらのビルドを取得するには “Details” 次のリンク “ClickHouse build check” 小切手のリストのエントリ。 そこには、ビルドへの直接リンクがあります。ClickHouseのdebパッケージは、本番サーバーにも展開できます（恐れがない場合）。 -おそらくいくつかのビルドは最初は失敗します。 これは、gccとclangの両方のビルドをチェックし、既存の警告のほとんどすべてをチェックするためです（常に `-Werror` フラグ)clangで有効になっています。 その同じページでは、すべてのビルドログを見つけることができるので、可能な限りすべての方法でClickHouseをビルドする必要はありません。 +ほとんどの場合、ビルドの一部は最初に失敗します。 これは、gccとclangの両方でビルドをチェックするという事実によるものです。 `-Werror` flag)clangを有効にします。 その同じページで、すべてのビルドログを見つけることができるので、ClickHouseをすべての可能な方法でビルドする必要はありません。 diff --git a/docs/ja/development/index.md b/docs/ja/development/index.md index 10a029893fb..4e14fd1a032 100644 --- a/docs/ja/development/index.md +++ b/docs/ja/development/index.md @@ -1,10 +1,10 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_folder_title: Development +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "\u958B\u767A" toc_hidden: true toc_priority: 58 -toc_title: "\u96A0\u3055\u308C\u305F" +toc_title: "\u96A0\u3057" --- # ClickHouse開発 {#clickhouse-development} diff --git a/docs/ja/development/style.md b/docs/ja/development/style.md index 9980570ed37..ba2d744f6ef 100644 --- a/docs/ja/development/style.md +++ b/docs/ja/development/style.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 68 toc_title: "C++\u30B3\u30FC\u30C9\u306E\u66F8\u304D\u65B9" --- @@ -9,21 +9,21 @@ toc_title: "C++\u30B3\u30FC\u30C9\u306E\u66F8\u304D\u65B9" ## 一般的な推奨事項 {#general-recommendations} -**1.** 以下は要件ではなく推奨事項です。 +**1.** 以下は推奨事項であり、要件ではありません。 -**2.** コードを編集している場合は、既存のコードの書式設定に従うことが理にかなっています。 +**2.** コードを編集する場合は、既存のコードの書式に従うことが理にかなっています。 -**3.** 一貫性のためにコードスタイルが必要です。 一貫性により、コードを読みやすくなり、コードの検索も容易になります。 +**3.** 一貫性のためにコードスタイルが必要です。 一貫性により、コードを読みやすくなり、コードの検索も簡単になります。 **4.** ルールの多くは論理的な理由を持っていない;彼らは確立された慣行によって決定されます。 ## 書式設定 {#formatting} -**1.** 書式設定のほとんどは自動的に行われます `clang-format`. +**1.** 多くのフォーマットは自動的に実行されるのでよ `clang-format`. -**2.** インデントは4スペースです。 タブが四つのスペースを追加するように開発環境を構成します。 +**2.** インデントは4スペースです。 タブにスペースが追加されるように開発環境を構成します。 -**3.** 中括弧を開くと閉じるには、別の行にする必要があります。 +**3.** 中括弧の開始と終了は別の行にする必要があります。 ``` cpp inline void readBoolText(bool & x, ReadBuffer & buf) @@ -34,7 +34,7 @@ inline void readBoolText(bool & x, ReadBuffer & buf) } ``` -**4.** 関数本体全体が単一の場合 `statement`、それは単一ラインに置くことができます。 中括弧の周りにスペースを配置します（行末のスペース以外）。 +**4.** 関数本体全体が単一の場合 `statement`、それは単一行に置くことができます。 中括弧の周りにスペースを配置します（行末のスペースのほかに）。 ``` cpp inline size_t mask() const { return buf_size() - 1; } @@ -51,13 +51,13 @@ void reinsert(const Value & x) memcpy(&buf[place_value], &x, sizeof(x)); ``` -**6.** で `if`, `for`, `while` その他の式では、関数呼び出しではなく、開き括弧の前にスペースが挿入されます。 +**6.** で `if`, `for`, `while` 他の式では、（関数呼び出しとは対照的に）開始括弧の前にスペースが挿入されます。 ``` cpp for (size_t i = 0; i < rows; i += storage.index_granularity) ``` -**7.** 二項演算子の前後にスペースを追加 (`+`, `-`, `*`, `/`, `%`, …) and the ternary operator `?:`. +**7.** 二項演算子の周りにスペースを追加 (`+`, `-`, `*`, `/`, `%`, …) and the ternary operator `?:`. ``` cpp UInt16 year = (s[0] - '0') * 1000 + (s[1] - '0') * 100 + (s[2] - '0') * 10 + (s[3] - '0'); @@ -65,7 +65,7 @@ UInt8 month = (s[5] - '0') * 10 + (s[6] - '0'); UInt8 day = (s[8] - '0') * 10 + (s[9] - '0'); ``` -**8.** ラインフィードが入力された場合は、オペレータを新しい行に置き、その前にインデントを増やします。 +**8.** 改行が入力されている場合は、演算子を新しい行に置き、その前にインデントを増やします。 ``` cpp if (elapsed_ns) @@ -82,17 +82,17 @@ dst.ClickEventID = click.EventID; dst.ClickGoodEvent = click.GoodEvent; ``` -**10.** 演算子の周囲にスペースを使用しない `.`, `->`. +**10.** 演算子の周りにスペースを使用しない `.`, `->`. -必要に応じて、オペレータは次の行にラップすることができます。 この場合、その前のオフセットが増加する。 +必要に応じて、演算子を次の行に折り返すことができます。 この場合、その前のオフセットが増加する。 -**11.** 単項演算子を区切るためにスペースを使用しない (`--`, `++`, `*`, `&`, …) from the argument. +**11.** 単項演算子の区切りにスペースを使用しないでください (`--`, `++`, `*`, `&`, …) from the argument. -**12.** 入れの後に空白、コンマといえるものです。 同じルールはaの中のセミコロンのために行く `for` 式。 +**12.** 入れの後に空白、コンマといえるものです。 同じルールは、内部のセミコロンのために行く `for` 式。 -**13.** を区切るためにスペースを使用しない。 `[]` オペレーター +**13.** 区切りにスペースを使用しないで下さい `[]` オペレーター -**14.** で `template <...>` 式の間にスペースを使用します `template` と `<`;後にスペースなし `<` または前に `>`. +**14.** で `template <...>` 式は、間にスペースを使用します `template` と `<`;後にスペースなし `<` または前に `>`. ``` cpp template @@ -100,7 +100,7 @@ struct AggregatedStatElement {} ``` -**15.** クラスと構造では、 `public`, `private`、と `protected` 同じレベルで `class/struct` コードの残りの部分をインデントします。 +**15.** クラスと構造体では、 `public`, `private`,and `protected` と同じレベルで `class/struct` コードの残りの部分をインデントします。 ``` cpp template @@ -115,7 +115,7 @@ public: **16.** 同じ場合 `namespace` ファイル全体に使用され、他に重要なものはありません。 `namespace`. -**17.** のためのブロック `if`, `for`, `while`、または他の式は、単一の `statement` 中括弧は省略可能です。 場所は `statement` 別の行に、代わりに。 この規則は、ネストされた場合にも有効です `if`, `for`, `while`, … +**17.** のブロックが `if`, `for`, `while`、または他の式は、単一の `statement`、中括弧は省略可能です。 を置く `statement` 別の行で、代わりに。 この規則は、入れ子に対しても有効です `if`, `for`, `while`, … しかし、内側の場合 `statement` 中括弧または `else`、外部ブロックは中括弧で記述する必要があります。 @@ -125,7 +125,7 @@ for (auto & stream : streams) stream.second->finalize(); ``` -**18.** ラインの端にスペースがあってはなりません。 +**18.** 行の終わりにスペースがあってはなりません。 **19.** ソースファイルはUTF-8エンコードです。 @@ -135,11 +135,11 @@ for (auto & stream : streams) << ", " << (timer.elapsed() / chunks_stats.hits) << " μsec/hit."; ``` -**21.** 単一行に複数の式を書き込まないでください。 +**21.** 単一行に複数の式を記述しないでください。 -**22.** 関数内のコードのセクションをグループ化し、複数の空行で区切ります。 +**22.** 関数内のコードのセクションをグループ化し、空行を複数以下で区切ります。 -**23.** 関数、クラスなどを一つまたは二つの空行で区切ります。 +**23.** 関数、クラスなどを空行で区切ります。 **24.** `A const` (値に関連する)型名の前に記述する必要があります。 @@ -151,7 +151,7 @@ const std::string & s char const * pos ``` -**25.** ポインタまたは参照を宣言するとき、 `*` と `&` 記号は両側のスペースで区切る必要があります。 +**25.** ポインタまたは参照を宣言するとき、 `*` と `&` シンボルは、両側のスペースで区切る必要があります。 ``` cpp //correct @@ -161,11 +161,11 @@ const char* pos const char *pos ``` -**26.** テンプレート-タイプを使用する場合は、それらを `using` キーワード(最も単純な場合を除く)。 +**26.** テンプレートタイプを使用するときは、それらを `using` キーワード(最も単純な場合を除く)。 -つまり、テンプレートのパラメータは指定しみ `using` そして、コードで繰り返されていません。 +つまり、テンプレートパラメーターは、 `using` コードでは繰り返されません。 -`using` 関数の内部など、ローカルで宣言できます。 +`using` 関数内など、ローカルで宣言できます。 ``` cpp //correct @@ -175,7 +175,7 @@ FileStreams streams; std::map> streams; ``` -**27.** なを宣言するのに複数の変数の異なる種類の一つです。 +**27.** 一つの文で異なる型の変数を複数宣言しないでください。 ``` cpp //incorrect @@ -191,17 +191,17 @@ std::cerr << (int)c <<; std::endl; std::cerr << static_cast(c) << std::endl; ``` -**29.** 授業や構造体、グループのメンバーは、機能別に各部の可視性です。 +**29.** クラスと構造体では、各可視スコープ内でメンバーと関数を個別にグループ化します。 -**30.** 小さなクラスや構造体の場合、メソッド宣言を実装から分離する必要はありません。 +**30.** 小さなクラスと構造体の場合、メソッド宣言を実装から分離する必要はありません。 同じことが、クラスや構造体の小さなメソッドにも当てはまります。 テンプレート化されたクラスと構造体の場合、メソッド宣言を実装から分離しないでください（そうでない場合は、同じ翻訳単位で定義する必要があ -**31.** 行を140文字で折り返すことができます（80文字ではなく）。 +**31.** 行は140文字で、80文字ではなく折り返すことができます。 -**32.** Postfixが不要な場合は、必ずprefix increment/decrement演算子を使用してください。 +**32.** Postfixが必要ない場合は、常に接頭辞の増分/減分演算子を使用します。 ``` cpp for (Names::const_iterator it = column_names.begin(); it != column_names.end(); ++it) @@ -211,7 +211,7 @@ for (Names::const_iterator it = column_names.begin(); it != column_names.end(); **1.** コードのすべての非自明な部分にコメントを追加してください。 -これは非常に重要です。 書面でのコメントだけを更新したいのですが–このコードに必要な、又は間違っています。 +これは非常に重要です。 書面でのコメントだけを更新したいのですが--このコードに必要な、又は間違っています。 ``` cpp /** Part of piece of memory, that can be used. @@ -221,9 +221,9 @@ for (Names::const_iterator it = column_names.begin(); it != column_names.end(); */ ``` -**2.** コメントは、必要に応じて詳細に記述できます。 +**2.** コメントは必要に応じて詳細に設定できます。 -**3.** コメントを記述するコードの前に配置します。 まれに、コメントは同じ行のコードの後に来ることがあります。 +**3.** 記述するコードの前にコメントを配置します。 まれに、コメントが同じ行のコードの後に来ることがあります。 ``` cpp /** Parses and executes the query. @@ -237,11 +237,11 @@ void executeQuery( ) ``` -**4.** コメントは英語のみで記述する必要があります。 +**4.** コメントは英語のみで書く必要があります。 **5.** を書いていて図書館を含む詳細なコメントで説明を主なヘッダファイルです。 -**6.** 追加情報を提供しないコメントを追加しないでください。 特に放置しないでください空のコメントこのような: +**6.** 追加情報を提供しないコメントは追加しないでください。 特に放置しないでください空のコメントこのような: ``` cpp /* @@ -264,19 +264,19 @@ void executeQuery( */ ``` -この例はリソースから借用されていますhttp://home.tamk.fi/~jaalto/course/coding-style/doc/unmaintainable-code/。 +この例はリソースから借用されていますhttp://home.tamk.fi/~jaalto/course/coding-style/doc/unmainainable-code/. -**7.** ガベージコメント（作成者、作成日）を書いてはいけません。.）各ファイルの先頭に。 +**7.** ごみのコメントを書かないでください（作成者、作成日。.)各ファイルの先頭にある。 -**8.** シングルラインのコメントずつスラッシュ: `///` 複数行のコメントは次の形式で始まります `/**`. これらのコメントは、 “documentation”. +**8.** 単一行のコメントはスラッシュで始まります: `///` 複数行のコメントは `/**`. これらのコメントは、 “documentation”. -注:doxygenを使用して、これらのコメントからドキュメントを生成できます。 しかし、doxygenはideのコードをナビゲートする方が便利なので、一般的には使用されません。 +注:Doxygenを使用すると、これらのコメントからドキュメントを生成できます。 しかし、Ideでコードをナビゲートする方が便利なので、Doxygenは一般的には使用されません。 -**9.** 複数行のコメントは、先頭と末尾に空の行を含めることはできません(複数行のコメントを閉じる行を除く)。 +**9.** 複数行コメントの先頭と末尾に空行を含めることはできません(複数行コメントを閉じる行を除きます)。 -**10.** コメント行コードは、基本的なコメントは、 “documenting” コメント. +**10.** コメント行コードは、基本的なコメントは、 “documenting” コメント。 -**11.** コミットする前に、コードのコメント部分を削除します。 +**11.** 削除のコメントアウトされていパーツのコード深い. **12.** コメントやコードで冒涜を使用しないでください。 @@ -286,7 +286,7 @@ void executeQuery( /// WHAT THE FAIL??? ``` -**14.** 区切り文字の作成にはコメントを使用しません。 +**14.** 使用しないコメントをdelimeters. ``` cpp ///****************************************************** @@ -298,7 +298,7 @@ void executeQuery( /// Why did you do this stuff? ``` -**16.** それが何であるかを記述するブロックの最後にコメントを書く必要はありません。 +**16.** それが何であったかを説明するブロックの最後にコメントを書く必要はありません。 ``` cpp /// for @@ -306,49 +306,49 @@ void executeQuery( ## 名前 {#names} -**1.** 変数とクラスメンバーの名前には、アンダースコア付きの小文字を使用します。 +**1.** 変数とクラスメンバーの名前にアンダースコア付きの小文字を使用します。 ``` cpp size_t max_block_size; ``` -**2.** 関数(メソッド)の名前には、小文字で始まるcamelCaseを使用します。 +**2.** 関数（メソッド）の名前には、小文字で始まるcamelCaseを使用します。 ``` cpp std::string getName() const override { return "Memory"; } ``` -**3.** クラス（構造体）の名前には、大文字で始まるCamelCaseを使用します。 I以外の接頭辞はインターフェイスには使用されません。 +**3.** クラス(構造体)の名前には、大文字で始まるCamelCaseを使用します。 I以外の接頭辞はインターフェイスには使用されません。 ``` cpp class StorageMemory : public IStorage ``` -**4.** `using` クラスと同じように名前が付けられます。 `_t` 最後に。 +**4.** `using` クラスと同じように、または `_t` 最後に。 -**5.** テンプレート型引数の名前:単純なケースでは、 `T`; `T`, `U`; `T1`, `T2`. +**5.** テンプレート型引数の名前:単純な場合は、次のようにします `T`; `T`, `U`; `T1`, `T2`. -より複雑なケースでは、クラス名の規則に従うか、プレフィックスを追加します `T`. +より複雑な場合は、クラス名の規則に従うか、プレフィックスを追加します `T`. ``` cpp template struct AggregatedStatElement ``` -**6.** テンプレート定数引数の名前:変数名の規則に従うか、または `N` 単純なケースでは。 +**6.** テンプレート定数引数の名前:変数名の規則に従うか、または `N` 簡単なケースでは。 ``` cpp template struct ExtractDomain ``` -**7.** 抽象クラス（インターフェイス）の場合は、 `I` 接頭辞。 +**7.** 抽象クラス(インターフェイス)については、 `I` プレフィックス ``` cpp class IBlockInputStream ``` -**8.** ローカルで変数を使用する場合は、短い名前を使用できます。 +**8.** 変数をローカルで使用する場合は、短い名前を使用できます。 それ以外の場合は、意味を説明する名前を使用します。 @@ -356,7 +356,7 @@ class IBlockInputStream bool info_successfully_loaded = false; ``` -**9.** の名前 `define`sおよびグローバル定数は、ALL\_CAPSとアンダースコアを使用します。 +**9.** の名前 `define`sおよびグローバル定数を使用ALL\_CAPSをアンダースコア(\_). ``` cpp #define MAX_SRC_TABLE_NAMES_TO_STORE 1000 @@ -364,16 +364,16 @@ bool info_successfully_loaded = false; **10.** ファイル名は内容と同じスタイルを使用する必要があります。 -ファイルに単一のクラスが含まれている場合は、クラス（camelcase）と同じようにファイルに名前を付けます。 +ファイルに単一のクラスが含まれている場合は、クラス(CamelCase)と同じ方法でファイルに名前を付けます。 -ファイルに単一の関数が含まれている場合は、そのファイルに関数(camelcase)と同じ名前を付けます。 +ファイルに単一の関数が含まれている場合は、関数(camelCase)と同じ方法でファイルに名前を付けます。 **11.** 名前に略語が含まれている場合は、: -- 変数名の場合、省略形は小文字を使用する必要があります `mysql_connection` (ない `mySQL_connection`). -- クラスと関数の名前については、大文字を省略形にしておきます`MySQLConnection` (ない `MySqlConnection`). +- 変数名の場合、省略形は小文字を使用する必要があります `mysql_connection` （ない `mySQL_connection`). +- クラスおよび関数の名前については、省略形に大文字を使用します`MySQLConnection` （ない `MySqlConnection`). -**12.** クラスメンバを初期化するためだけに使用されるコンストラクタ引数は、クラスメンバと同じように名前を付ける必要がありますが、最後にアン +**12.** クラスメンバーを初期化するためだけに使用されるコンストラクター引数には、クラスメンバーと同じ名前を付ける必要がありますが、最後にアンダースコ ``` cpp FileQueueProcessor( @@ -388,15 +388,15 @@ FileQueueProcessor( } ``` -のアンダースコアの接尾辞ければ省略することができ、引数を使用していないのコンストラクタ。 +引数がコンストラクタ本体で使用されていない場合は、アンダースコアの接尾辞を省略できます。 -**13.** ローカル変数とクラスメンバの名前に違いはありません（接頭辞は必要ありません）。 +**13.** ローカル変数とクラスメンバーの名前に違いはありません（接頭辞は必要ありません）。 ``` cpp timer (not m_timer) ``` -**14.** の定数のために `enum`、大文字でキャメルケースを使用します。 ALL\_CAPSも許容されます。 この `enum` ローカルではなく、 `enum class`. +**14.** の定数に対して `enum`、大文字でキャメルケースを使用します。 ALL\_CAPSも許容されます。 もし `enum` は非ローカルである。 `enum class`. ``` cpp enum class CompressionMethod @@ -410,43 +410,43 @@ enum class CompressionMethod not Stroka -**16.** 略語は、よく知られている場合（ウィキペディアや検索エンジンで略語の意味を簡単に見つけることができる場合）には許容されます。 +**16.** 略語は、よく知られている場合（Wikipediaや検索エンジンで略語の意味を簡単に見つけることができる場合）に許容されます。 `AST`, `SQL`. Not `NVDH` (some random letters) -短くされた版が共通の使用なら不完全な単語は受諾可能である。 +短縮版が一般的に使用されている場合、不完全な単語は許容されます。 -コメントの横にフルネームが含まれている場合は、省略形を使用することもできます。 +コメントの横にフルネームが含まれている場合は、略語を使用することもできます。 -**17.** C++ソースコードを持つファイル名には、 `.cpp` 拡張子。 ヘッダーファイルには、 `.h` 拡張子。 +**17.** C++のソースコードを持つファイル名は、 `.cpp` 延長。 ヘッダファイルには `.h` 延長。 ## コードの書き方 {#how-to-write-code} **1.** メモリ管理。 -手動メモリ割り当て解除 (`delete`)ライブラリコードでのみ使用できます。 +手動メモリ解放 (`delete`）ライブラリコードでのみ使用できます。 -ライブラリコードでは、 `delete` 演算子はデストラクターでのみ使用できます。 +ライブラリコードでは、 `delete` operatorはデストラクタでのみ使用できます。 アプリケーショ 例: -- 最も簡単な方法は、スタックにオブジェクトを配置するか、別のクラスのメンバーにすることです。 +- 最も簡単な方法は、スタック上にオブジェクトを配置するか、別のクラスのメンバーにすることです。 - 多数の小さなオブジェクトの場合は、コンテナを使用します。 -- ヒープに存在する少数のオブジェクトの自動割り当て解除の場合は、以下を使用します `shared_ptr/unique_ptr`. +- ヒープ内に存在する少数のオブジェクトの自動割り当て解除には、以下を使用します `shared_ptr/unique_ptr`. **2.** リソース管理。 -使用 `RAII` と上記参照。 +使用 `RAII` 上記を参照してください。 **3.** エラー処理。 -例外を使用します。 ほとんどの場合、例外をスローするだけで、例外をキャッチする必要はありません `RAII`). +例外を使用します。 ほとんどの場合、例外をスローするだけで、それをキャッチする必要はありません（ `RAII`). -オフラインのデータ処理アプリケーションでは、しばしば可能な漁例外をスローしました。 +オフライ ユーザー要求を処理するサーバーでは、通常、接続ハンドラの最上位レベルで例外をキャッチするだけで十分です。 @@ -466,14 +466,14 @@ if (exception) exception->rethrow(); ``` -ない非表示の例外なります。 盲目的にすべての例外をログに記録することはありません。 +ない非表示の例外なります。 すべての例外を盲目的に記録するだけではありません。 ``` cpp //Not correct catch (...) {} ``` -いくつかの例外を無視する必要がある場合は、特定の例外に対してのみ行い、残りを再スローします。 +いくつかの例外を無視する必要がある場合は、特定の例外に対してのみ実行し、残りを再スローします。 ``` cpp catch (const DB::Exception & e) @@ -485,7 +485,7 @@ catch (const DB::Exception & e) } ``` -応答コード付きの関数を使用する場合、または `errno`、常に結果をチェックし、エラーの場合は例外をスローします。 +応答コードまたは関数を使用する場合 `errno`、常に結果をチェックし、エラーの場合は例外をスローします。 ``` cpp if (0 != close(fd)) @@ -502,16 +502,16 @@ if (0 != close(fd)) これは推奨されませんが、許可されています。 -次のオプションを使用: +次のオプションを使用しま: - 関数の作成 (`done()` または `finalize()`）それは例外につながる可能性のあるすべての作業を事前に行います。 その関数が呼び出された場合、後でデストラクタに例外はないはずです。 -- 複雑すぎるタスク（ネットワーク経由でメッセージを送信するなど）は、クラスユーザーが破棄する前に呼び出す必要がある別のメソッドに入れることがで -- デストラクタに例外がある場合は、それを隠すよりもログに記録する方が良いでしょう（ロガーが利用可能な場合）。 -- 簡単な適用では、頼ることは受諾可能です `std::terminate` （以下の場合 `noexcept` デフォルトではC++11）例外を処理する。 +- タスクも複雑で(メッセージを送信するなどのネットワーク)を置くことができます別の方法は、クラスのユーザーを呼び出す前に破壊. +- デストラクタに例外がある場合は、それを非表示にするよりもログに記録する方が良いでしょう（ロガーが利用可能な場合）。 +- 簡単な適用では、頼ることは受諾可能です `std::terminate` （以下の場合 `noexcept` デフォルトではc++11）例外を処理する。 -**6.** 匿名のコードブロック。 +**6.** 匿名コードブロック。 -特定の変数をローカルにするために、単一の関数内に別のコードブロックを作成して、ブロックを終了するときにデストラクタが呼び出されるように +特定の変数をローカルにするために、単一の関数内に別のコードブロックを作成して、ブロックを終了するときにデストラクターが呼び出されるようにす ``` cpp Block block = data.in->read(); @@ -525,27 +525,27 @@ Block block = data.in->read(); ready_any.set(); ``` -**7.** マルチスレッド。 +**7.** マルチスレッド -オフラインのデータ処理プログラム: +オフライ: -- 単一のcpuコアで最高のパフォーマンスを得るようにしてください。 必要に応じてコードを並列化できます。 +- 単一のCPUコアで可能な限り最高のパフォーマンスを得ようとします。 必要に応じて、コードを並列化できます。 -でサーバアプリケーション: +サーバーアプリ: - スレッドプールを使用して要求を処理します。 この時点で、まだたタスクを必要とした管理コスイッチング時の値です。※ Forkは並列化には使用されません。 -**8.** スレッドの同期。 +**8.** 同期スレッド。 -多くの場合、異なるスレッドに異なるメモリセルを使用させることができます（さらに良い：異なるキャッシュライン）。 `joinAll`). +くすることが可能で別のスレッドが別のメモリー細胞により異なるキャッシュ回線)を使用していないスレッドが同期を除く `joinAll`). -同期が必要な場合は、ほとんどの場合、mutexを使用すれば十分です。 `lock_guard`. +同期が必要な場合は、ほとんどの場合、以下の条件でmutexを使用すれば十分です `lock_guard`. -他の例では、システム同期プリミティブを使用します。 使用中の待ち時間を使用しないで下さい。 +他の場合は、システム同期プリミティブを使用します。 Busy waitは使用しないでください。 -原子操作は、最も単純な場合にのみ使用する必要があります。 +原子演算は、最も単純な場合にのみ使用する必要があります。 主な専門分野でない限り、ロックフリーのデータ構造を実装しようとしないでください。 @@ -553,35 +553,35 @@ Forkは並列化には使用されません。 ほとんどの場合、参照を好む。 -**10.** const +**10.** const. 定数参照、定数へのポインタを使用する, `const_iterator`、およびconstメソッド。 -考える `const` デフォルトにしてnonを使用するには-`const` 必要なときだけ。 +考慮する `const` デフォルトで非を使用するには-`const` 必要なときだけ。 -値によって変数を渡すとき、 `const` 通常は意味をなさない。 +変数を値で渡すときは、 `const` 通常は意味がありません。 -**11.** 署名なし +**11.** 無署名 -使用 `unsigned` 必要であれば。 +使用 `unsigned` 必要に応じて。 **12.** 数値型。 -タイプの使用 `UInt8`, `UInt16`, `UInt32`, `UInt64`, `Int8`, `Int16`, `Int32`、と `Int64`、同様に `size_t`, `ssize_t`、と `ptrdiff_t`. +タイプを使用する `UInt8`, `UInt16`, `UInt32`, `UInt64`, `Int8`, `Int16`, `Int32`,and `Int64` だけでなく、 `size_t`, `ssize_t`,and `ptrdiff_t`. これらの型を数値に使用しないでください: `signed/unsigned long`, `long long`, `short`, `signed/unsigned char`, `char`. **13.** 引数を渡す。 -参照によって複雑な値を渡す(以下を含む `std::string`). +参照による複素数値の渡し(含む `std::string`). -関数がヒープで作成されたオブジェクトの所有権を取得する場合は、引数の型を作成します `shared_ptr` または `unique_ptr`. +関数がヒープ内に作成されたオブジェクトの所有権を取得する場合は、引数の型を作成します `shared_ptr` または `unique_ptr`. **14.** 戻り値。 -ほとんどの場合、単に `return`. 書き込まない `[return std::move(res)]{.strike}`. +ほとんどの場合、 `return`. 書かない `[return std::move(res)]{.strike}`. -関数がヒープ上にオブジェクトを割り当て、それを返す場合は、 `shared_ptr` または `unique_ptr`. +関数がオブジェクトをヒープに割り当てて返す場合は、次のようにします `shared_ptr` または `unique_ptr`. まれに、引数を使用して値を返す必要がある場合があります。 この場合、引数は参照でなければなりません。 @@ -601,21 +601,21 @@ public: 別のものを使用する必要はありません `namespace` 適用コードのため。 -小さな図書館にもこれは必要ありません。 +小さな図書館でもこれは必要ありません。 中規模から大規模のライブラリの場合は、すべてを `namespace`. -ライブラリの中で `.h` ファイル、使用できます `namespace detail` アプリケーションコードに必要のない実装の詳細を非表示にする。 +図書館の `.h` ファイル、使用できます `namespace detail` アプリケーションコードに必要ない実装の詳細を非表示にする。 -で `.cpp` ファイル、あなたが使用できる `static` または匿名の名前空間は、記号を非表示にします。 +で `.cpp` を使用することができます `static` またはシンボルを非表示にする匿名の名前空間。 -また、 `namespace` に使用することができ `enum` 対応する名前が外部に落ちないようにするには `namespace` （しかし、それを使用する方が良いです `enum class`). +また、 `namespace` に使用することができます `enum` 対応する名前が外部に落ちないようにするには `namespace` （しかし、それを使用する方が良いです `enum class`). **16.** 遅延初期化。 -初期化に引数が必要な場合は、通常はデフォルトのコンストラクタを記述すべきではありません。 +初期化に引数が必要な場合は、通常は既定のコンストラクタを記述しないでください。 -後で初期化を遅らせる必要がある場合は、無効なオブジェクトを作成する既定のコンストラクターを追加できます。 または、少数のオブジェクトの場合は、次のものを使用できます `shared_ptr/unique_ptr`. +後で初期化を遅らせる必要がある場合は、無効なオブジェクトを作成する既定のコンストラクターを追加できます。 または、少数のオブジェクトの場合は、以下を使用できます `shared_ptr/unique_ptr`. ``` cpp Loader(DB::Connection * connection_, const std::string & query, size_t max_block_size_); @@ -626,19 +626,19 @@ Loader() {} **17.** 仮想関数。 -クラスが多態的な使用を意図していない場合は、関数を仮想にする必要はありません。 これはデストラクタにも当てはまります。 +クラスが多態的な使用を意図していない場合、関数を仮想にする必要はありません。 これはデストラクタにも当てはまります。 -**18.** エンコーディング。 +**18.** エンコーディング -どこでもutf-8を使用します。 使用 `std::string`と`char *`. 使用しない `std::wstring`と`wchar_t`. +どこでもUTF-8を使用します。 使用 `std::string`と`char *`. 使用しない `std::wstring`と`wchar_t`. -**19.** ログ記録。 +**19.** ロギング -コードのどこにでも例を見てください。 +コードのどこでも例を参照してください。 -コミットする前に、無意味なデバッグログとその他のデバッグ出力をすべて削除します。 +コミットする前に、無意味なログやデバッグログ、その他のデバッグ出力をすべて削除します。 -トレースレベルでも、サイクルでのログ記録は避けるべきです。 +トレースレベルでも、サイクルのログインは避けるべきです。 ログには必読でログインです。 @@ -646,31 +646,31 @@ Loader() {} ログメッセージは英語で書く必要があります。 -ログは、システム管理者が理解できることが望ましいです。 +ログは、システム管理者が理解できるようにしてください。 ログに冒涜を使用しないでください。 -ログにutf-8エンコーディングを使用します。 まれに、ログに非ascii文字を使用できます。 +ログでUTF-8エンコーディングを使用します。 まれに、ログに非ASCII文字を使用できます。 **20.** 入出力。 -使用しない `iostreams` アプリケーショ `stringstream`). +使用しない `iostreams` 内部のサイクルにおいて不可欠な存在であのためのアプリケーション性能(い利用 `stringstream`). -を使用 `DB/IO` 代わりに図書館。 +使用する `DB/IO` 代わりに図書館。 **21.** 日付と時刻。 -を見る `DateLUT` ライブラリ。 +を参照。 `DateLUT` 図書館 **22.** 含める。 -常に使用 `#pragma once` 代わりに、警備員を含みます。 +常に使用 `#pragma once` 代わりに警備員を含めます。 -**23.** 使用。 +**23.** を使用して。 -`using namespace` は使用されません。 を使用することができ `using` 特定の何かと。 しかし、クラスや関数の中でローカルにします。 +`using namespace` は使用されません。 以下を使用できます `using` 特定の何かと。 しかし、クラスや関数内でローカルにします。 -**24.** 使用しない `trailing return type` 必要な場合を除き、機能のため。 +**24.** 使用しない `trailing return type` 必要がない限り機能のため。 ``` cpp [auto f() -> void;]{.strike} @@ -687,79 +687,79 @@ std::string s{"Hello"}; auto s = std::string{"Hello"}; ``` -**26.** 仮想関数の場合は、以下を記述します `virtual` 基本クラスでは、次のように記述します `override` 代わりに `virtual` 子孫クラスで。 +**26.** のための仮想関数を書く `virtual` 基本クラスでは、 `override` 代わりに `virtual` 子孫クラスで。 -## C++の未使用機能 {#unused-features-of-c} +## C++の未使用の機能 {#unused-features-of-c} **1.** 仮想継承は使用されません。 **2.** C++03の例外指定子は使用されません。 -## Platform {#platform} +## プラット {#platform} **1.** を書いていますコードの特定の。 それが同じ場合には、クロス-プラットフォームまたは携帯コードが好ましい。 -**2.** 言語：C++17。 +**2.** 言語:C++17. -**3.** コンパイラ: `gcc`. この時点で(December2017)、をコードはコンパイル使用してバージョン7.2。 （コンパイルすることもできます `clang 4`.) +**3.** コンパイラ: `gcc`. 2017年現在、コードはバージョン7.2を使用してコンパイルされている。 （以下を使ってコンパイルできます `clang 4`.) 標準ライブラリが使用されます (`libstdc++` または `libc++`). -**4.**OS：LinuxのUbuntuは、正確なよりも古いではありません。 +**4.**OS：LinuxのUbuntuの、正確よりも古いではありません。 -**5.**コードはx86\_64cpuアーキテクチャ用に書かれている。 +**5.**コードはx86\_64CPUアーキテクチャ用に書かれている。 -CPU命令セットは、当社のサーバー間でサポートされる最小セットです。 現在、SSE4.2です。 +CPU命令セットは、サーバー間でサポートされる最小のセットです。 現在、SSE4.2です。 **6.** 使用 `-Wall -Wextra -Werror` コンパイルフラグ。 -**7.** 静的に接続するのが難しいライブラリを除くすべてのライブラリとの静的リンクを使用します。 `ldd` コマンド）。 +**7.** 静的に接続することが困難なライブラリを除くすべてのライブラリとの静的リンクを使用します。 `ldd` コマンド）。 -**8.** コードは開発され、リリース設定でデバッグされます。 +**8.** コードはリリース設定で開発およびデバッグされます。 ## ツール {#tools} **1.** KDevelopは良いIDEです。 -**2.** デバッグのために、 `gdb`, `valgrind` (`memcheck`), `strace`, `-fsanitize=...`、または `tcmalloc_minimal_debug`. +**2.** デバッグの使用 `gdb`, `valgrind` (`memcheck`), `strace`, `-fsanitize=...`,または `tcmalloc_minimal_debug`. -**3.** プロファイ `Linux Perf`, `valgrind` (`callgrind`)、または `strace -cf`. +**3.** のためのプロファイリングを使用 `Linux Perf`, `valgrind` (`callgrind`）、または `strace -cf`. **4.** ソースはGitにあります。 -**5.** アセンブリ使用 `CMake`. +**5.** アセンブリの使用 `CMake`. **6.** プログラムは `deb` パッケージ。 **7.** ることを約束し、マスターが破ってはいけないの。 -選択したリビジョンのみが実行可能と見なされます。 +選択したリビジョンのみが実行可能とみなされます。 -**8.** コードが部分的にしか準備されていなくても、できるだけ頻繁にコミットを行います。 +**8.** コードが部分的にしか準備できていなくても、できるだけ頻繁にコミットを行います。 -用の支店です。 +この目的のために分岐を使用します。 -あなたのコードが `master` ブランチはまだビルド可能ではない。 `push`. あなたはそれを終了するか、数日以内にそれを削除する必要があります。 +あなたのコードが `master` branchはまだビルド可能ではありません。 `push`. あなたはそれを終了するか、数日以内にそれを削除する必要があります。 -**9.** 些細な変更の場合は、ブランチを使用してサーバーに公開します。 +**9.** 些細な変更ではない場合は、ブランチを使用してサーバーに公開します。 -**10.** 未使用のコードはリポジトリから削除されます。 +**10.** 未使用のコードがリポジトリから削除されます。 -## ライブラリ {#libraries} +## 図書館 {#libraries} **1.** C++14標準ライブラリが使用されています（実験的な拡張が許可されています）。 `boost` と `Poco` フレームワーク -**2.** 必要に応じて、OSパッケージで利用可能な既知のライブラリを使用することができます。 +**2.** 必要に応じて、OSパッケージで利用可能な既知のライブラリを使用できます。 -すでに利用可能な良い解決策がある場合は、別のライブラリをインストールする必要があることを意味していても使用してください。 +すでに利用可能な良い解決策がある場合は、別のライブラリをインストールする必要がある場合でも、それを使用してください。 (が準備をしておいてくださ去の悪い図書館からのコードです。) -**3.** パッケージに必要なものがないか、古いバージョンや間違ったタイプのコンパイルがない場合は、パッケージに含まれていないライブラリをインストール +**3.** パッケージに必要なものがない場合や、古いバージョンや間違った種類のコンパイルがある場合は、パッケージにないライブラリをインストールできます。 -**4.** ライブラリが小さく、独自の複雑なビルドシステムを持たない場合は、ソースファイルを `contrib` フォルダ。 +**4.** ライブラリが小さく、独自の複雑なビルドシステムがない場合は、ソースファイルを `contrib` フォルダ。 **5.** すでに使用されているライブラリが優先されます。 @@ -769,37 +769,37 @@ CPU命令セットは、当社のサーバー間でサポートされる最小 **2.** う最も単純な解決策です。 -**3.** それがどのように機能し、内部ループがどのように機能するかを知るまで、コードを書かないでください。 +**3.** どのように動作し、内部ループがどのように機能するかを知るまで、コードを記述しないでください。 -**4.** 最も単純な場合は、次のようにします `using` クラスや構造体の代わりに。 +**4.** 最も単純なケースでは、 `using` クラスや構造体の代わりに。 -**5.** 可能であれば、コピーコンストラクター、代入演算子、デストラクター(仮想関数を除く、クラスに少なくとも一つの仮想関数が含まれている場合)、コンストラ つまり、コンパイラ生成機能しないでください。 を使用することができ `default`. +**5.** 可能であれば、コピーコンストラクター、代入演算子、デストラクター(クラスに少なくとも一つの仮想関数が含まれている場合は仮想関数を除く)、コンストラク つまり、コンパイラで生成された関数は正しく動作する必要があります。 以下を使用できます `default`. -**6.** コードの単純化が推奨されます。 可能な場合は、コードのサイズを小さくします。 +**6.** コードの簡素化が推奨されます。 可能であれば、コードのサイズを小さくします。 ## その他の推奨事項 {#additional-recommendations} -**1.** 明示的に指定する `std::` からのタイプの場合 `stddef.h` +**1.** 明示的に指定する `std::` からのタイプのため `stddef.h` -は推奨されません。 つまり、我々は書くことをお勧めします `size_t` 代わりに `std::size_t`、それは短いですので。 +推奨されません。 つまり、書くことをお勧めします `size_t` 代わりに `std::size_t` それは短いので。 -それは可能追加する `std::`. +追加することは許容されます `std::`. **2.** 明示的に指定する `std::` 標準Cライブラリの関数の場合 -は推奨されません。 言い換えれば、 `memcpy` 代わりに `std::memcpy`. +推奨されません。 言い換えれば、 `memcpy` 代わりに `std::memcpy`. -その理由は、次のような非標準的な機能があるからです `memmem`. 私達は機会にこれらの機能を使用します。 これらの関数は `namespace std`. +その理由は、次のような非標準的な機能があるからです `memmem`. 私たちは機会にこれらの機能を使用します。 これらの関数は、 `namespace std`. -あなたが書く場合 `std::memcpy` 代わりに `memcpy` どこでも、その後 `memmem` なし `std::` 奇妙に見えます。 +あなたが書く場合 `std::memcpy` 代わりに `memcpy` どこでも、その後、 `memmem` なし `std::` 奇妙に見えます。 -それでも、あなたはまだ `std::` あなたがそれを好むなら。 +それにもかかわらず、 `std::` あなたがそれを好むなら。 -**3.** 同じものが標準C++ライブラリで利用可能な場合、Cの関数を使用する。 +**3.** 標準C++ライブラリで同じ関数が使用できる場合、Cからの関数を使用します。 これは、より効率的であれば許容されます。 -たとえば、以下を使用します `memcpy` 代わりに `std::copy` メモリの大きな塊をコピーするため。 +たとえば、次を使用します `memcpy` 代わりに `std::copy` メモリの大きな塊をコピーするため。 **4.** 複数行の関数の引数。 diff --git a/docs/ja/development/tests.md b/docs/ja/development/tests.md index 9abdc0595cd..0d9fa40cb9b 100644 --- a/docs/ja/development/tests.md +++ b/docs/ja/development/tests.md @@ -1,90 +1,90 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 69 -toc_title: "ClickHouse\u30C6\u30B9\u30C8\u3092\u5B9F\u884C\u3059\u308B\u65B9\u6CD5" +toc_title: "ClickHouse\u30C6\u30B9\u30C8\u306E\u5B9F\u884C\u65B9\u6CD5" --- -# ClickHouse試験 {#clickhouse-testing} +# ClickHouseのテスト {#clickhouse-testing} ## 機能テスト {#functional-tests} -機能テストは最も簡単で使いやすいものです。 clickhouseの機能のほとんどは、機能テストでテストすることができ、彼らはそのようにテストすることができclickhouseコード内のすべての変更のために使用する +機能テストは、最も簡単で使いやすいです。 ClickHouseの機能のほとんどは機能テストでテストすることができ、そのようにテストできるClickHouseコードのすべての変更に使用することが必須です。 -各機能テストは、実行中のclickhouseサーバーに一つまたは複数のクエリを送信し、参照と結果を比較します。 +各機能テストは、実行中のClickHouseサーバーに一つまたは複数のクエリを送信し、結果を参照と比較します。 -テストは `queries` ディレクトリ。 つのサブディレクトリがあります: `stateless` と `stateful`. ステートレステストでは、プリロードされたテストデータを使用せずにクエリを実行します。 ステートフルテストでは、Yandexのテストデータが必要です。メトリカと一般市民には利用できません。 我々は唯一の使用する傾向があります `stateless` テストと新しい追加を避ける `stateful` テスト +テストは `queries` ディレクトリ。 サブディレクトリは二つあります: `stateless` と `stateful`. ステートレステストは、プリロードされたテストデータなしでクエリを実行します。 状態での検査が必要とな予圧試験データからのYandex.Metricaおよび一般に利用できない。 私たちは使用する傾向があります `stateless` テストと新しい追加を避ける `stateful` テストだ -それぞれの試験できるの種類: `.sql` と `.sh`. `.sql` testは、パイプ処理される単純なSQLスクリプトです `clickhouse-client --multiquery --testmode`. `.sh` テストは、単独で実行されるスクリプトです。 +それぞれの試験できるの種類: `.sql` と `.sh`. `.sql` testは、パイプ処理される単純なSQLスクリプトです `clickhouse-client --multiquery --testmode`. `.sh` testは、それ自体で実行されるスクリプトです。 -すべてのテストを実行するには、 `clickhouse-test` ツール。 見て！ `--help` 可能なオプションのリストについて。 できるだけ実行すべての試験または実行のサブセットの試験フィルター部分文字列の試験名: `./clickhouse-test substring`. +すべてのテストを実行するには、 `clickhouse-test` ツール。 見て！ `--help` 可能なオプションのリスト。 できるだけ実行すべての試験または実行のサブセットの試験フィルター部分文字列の試験名: `./clickhouse-test substring`. -機能テストを呼び出す最も簡単な方法は、コピーすることです `clickhouse-client` に `/usr/bin/`、実行 `clickhouse-server` そして、実行 `./clickhouse-test` 独自のディレクトリから。 +機能テストを呼び出す最も簡単な方法は、コピーすることです `clickhouse-client` に `/usr/bin/`,run `clickhouse-server` そして、実行 `./clickhouse-test` 独自のディレクトリから。 -新しいテストを追加するには、 `.sql` または `.sh` ファイル `queries/0_stateless` ディレクトリは、手動でチェックしてから生成 `.reference` 次の方法でファイル: `clickhouse-client -n --testmode < 00000_test.sql > 00000_test.reference` または `./00000_test.sh > ./00000_test.reference`. +新しいテストを追加するには、 `.sql` または `.sh` ファイル `queries/0_stateless` ディレクトリでチェックを手動でその生成 `.reference` 次の方法でファイル: `clickhouse-client -n --testmode < 00000_test.sql > 00000_test.reference` または `./00000_test.sh > ./00000_test.reference`. -テストでは、（create、dropなど）テーブルのみを使用する必要があります `test` テストでは一時テーブルを使用することもできます。 +テストでは、テーブルのみを使用（create、dropなど）する必要があります `test` また、テストでは一時テーブルを使用することもできます。 -機能テストで分散クエリを使用する場合は、次のようにします `remote` テーブル機能との `127.0.0.{1..2}` または、サーバー構成ファイル内の事前定義されたテストクラスターを次のように使用できます `test_shard_localhost`. +機能テストで分散クエリを使用する場合は、以下を利用できます `remote` テーブル関数 `127.0.0.{1..2}` または、サーバー設定ファイルで次のように定義済みのテストクラスタを使用できます `test_shard_localhost`. -いくつかのテストは `zookeeper`, `shard` または `long` 彼らの名前で。 -`zookeeper` ZooKeeperを使用しているテストのためのものです。 `shard` そのテストのためです -サーバーのリッスンが必要 `127.0.0.*`; `distributed` または `global` 同じを持っている -意味は… `long` 少し長く走るテストのためのものです。 あなたはできる -次のテストグループを無効にする `--no-zookeeper`, `--no-shard` と +いくつかのテストには `zookeeper`, `shard` または `long` 彼らの名前で。 +`zookeeper` ZooKeeperを使用しているテスト用です。 `shard` そのテストのためです +サーバーにリッスンが必要 `127.0.0.*`; `distributed` または `global` 同じを持っている +意味だ `long` 少し長く実行されるテストのためのものです。 あなたはできる +disableこれらのグループの試験を使用 `--no-zookeeper`, `--no-shard` と `--no-long` オプション、それぞれ。 ## 既知のバグ {#known-bugs} -機能テストで簡単に再現できるいくつかのバグを知っていれば、準備された機能テストを `tests/queries/bugs` ディレクトリ。 これらのテストはに移動されます `tests/queries/0_stateless` バグが修正されたとき。 +機能テストで簡単に再現できるいくつかのバグがわかっている場合は、準備された機能テストを `tests/queries/bugs` ディレクトリ。 これらのテストは `tests/queries/0_stateless` バグが修正されたとき。 ## 統合テスト {#integration-tests} -統合テストでは、クラスター化された設定でclickhouseをテストし、mysql、postgres、mongodbのような他のサーバーとのclickhouseの相互作用を可能にします。 それらはネットワークの割れ目、包みの低下、等を競争して有用である。 これらの試験する方向に作用しdockerを複数の容器を様々なソフトウェアです。 +統合テストでは、クラスター化された構成でClickHouseをテストし、Mysql、Postgres、MongoDBなどの他のサーバーとClickHouseの相互作用をテストできます。 これらをエミュレートするネットワーク分割、パケットの落下など。 これらの試験する方向に作用しDockerを複数の容器を様々なソフトウェアです。 見る `tests/integration/README.md` これらのテストを実行する方法について。 -ClickHouseとサードパーティドライバの統合はテストされていません。 また、現在、JDBCおよびODBCドライバとの統合テストはありません。 +この統合ClickHouse第三者によるドライバーではない。 また、現在、JDBCおよびODBCドライバとの統合テストはありません。 ## 単体テスト {#unit-tests} -単体テストは、clickhouse全体ではなく、単一の孤立したライブラリまたはクラスをテストする場合に便利です。 テストのビルドを有効または無効にするには `ENABLE_TESTS` CMakeオプション。 単体テスト（およびその他のテストプログラム）は、 `tests` コード全体のサブディレクトリ。 単体テストを実行するには `ninja test`. いくつかのテストは `gtest` しかし、テストの失敗でゼロ以外の終了コードを返すプログラムだけです。 +単体テストは、ClickHouse全体ではなく、単一の孤立したライブラリまたはクラスをテストする場合に便利です。 テストのビルドを有効または無効にするには `ENABLE_TESTS` CMakeオプション。 単体テスト(およびその他のテストプログラム)は `tests` コード全体のサブディレクトリ。 単体テストを実行するには、 `ninja test`. 一部のテストでは `gtest` しかし、いくつかは、テストの失敗でゼロ以外の終了コードを返すプログラムです。 -コードがすでに機能テストでカバーされている場合は、単体テストを行う必要はありません（機能テストは通常ははるかに簡単に使用できます）。 +コードがすでに機能テストでカバーされている場合は、必ずしも単体テストを持つとは限りません（機能テストは通常ははるかに簡単です）。 ## 性能テスト {#performance-tests} -性能試験を測定して比較の一部の縁の一部clickhouse合成ます。 試験は `tests/performance`. 各テストは `.xml` テストケースの説明を含むファイル。 テストは以下で実行されます `clickhouse performance-test` ツール（埋め込まれていること `clickhouse` バイナリ）。 見る `--help` 呼び出しのため。 +パフォーマ テストは `tests/performance`. それぞれの試験に代表される `.xml` テストケースの説明を持つファイル。 テストは以下で実行されます `clickhouse performance-test` ツール(埋め込まれている `clickhouse` バイナリ）。 見る `--help` 呼び出し用。 -各試験の実行はmiltiple索の可能性のある組み合わせのパラメータ)のループ条件のための停止など “maximum execution speed is not changing in three seconds” 測定一部の指標につクエリの性能など “maximum execution speed”). いくつかの試験を含むことができ前提条件に予圧試験データを得る。 +それぞれの試験実行または複数のクエリ(このパラメータの組み合わせ）のループ条件のための停止など “maximum execution speed is not changing in three seconds” 測定一部の指標につクエリの性能など “maximum execution speed”). いくつかの試験を含むことができ前提条件に予圧試験データを得る。 -いくつかのシナリオでclickhouseのパフォーマンスを向上させたい場合や、単純なクエリで改善が見られる場合は、パフォーマンステストを作成することを強 それは常に使用する意味があります `perf top` またはあなたのテスト中に他のperfツール。 +いくつかのシナリオでClickHouseのパフォーマンスを向上させたい場合や、単純なクエリで改善が見られる場合は、パフォーマンステストを作成することを強 いう意味があるのに使用 `perf top` またはあなたのテストの間の他のperf用具。 -## テストツール、スクリプト {#test-tools-and-scripts} +## テストツールとスクリプ {#test-tools-and-scripts} -の一部のプログラム `tests` directoryは準備されたテストではなく、テストツールです。 たとえば、 `Lexer` ツールがあります `dbms/Parsers/tests/lexer` これはstdinのトークン化を行い、結果をstdoutに色付けします。 これらの種類のツールをコード例として、また調査と手動テストに使用できます。 +一部のプログラム `tests` ディレク 例えば、 `Lexer` ツールがあります `src/Parsers/tests/lexer` それはstdinのトークン化を行い、色付けされた結果をstdoutに書き込みます。 これらの種類のツールは、コード例として、また探索と手動テストに使用できます。 -でも一対のファイル `.sh` と `.reference` のツールであるかの定義済みの入力-その後スクリプトの結果と比較することができ `.reference` ファイル。 この種のテストは自動化されていません。 +でも一対のファイル `.sh` と `.reference` いくつかの事前定義された入力でそれを実行するためのツールと一緒に-その後、スクリプトの結果は `.reference` ファイル これらの種類のテストは自動化されていません。 -## Miscellanous試験 {#miscellanous-tests} +## その他のテスト {#miscellaneous-tests} -外部辞書のテストは次の場所にあります `tests/external_dictionaries` そして機械学ばれたモデルのために `tests/external_models`. これらのテストは更新されず、統合テストに転送する必要があります。 +外部辞書のテストは次の場所にあります `tests/external_dictionaries` そして機械学んだモデルのために `tests/external_models`. これらのテストは更新されず、統合テストに転送する必要があります。 -定足数の挿入には個別のテストがあります。 ネットワーク分割、パケットドロップ（clickhouseノード間、clickhouseとzookeeper間、clickhouseサーバーとクライアント間など）など、さまざまな障害ケースをエミュレートします。), `kill -9`, `kill -STOP` と `kill -CONT` 、のように [Jepsen](https://aphyr.com/tags/Jepsen). その後、試験チェックすべての認識を挿入したすべて拒否された挿入しました。 +クォーラム挿入には別のテストがあります。 このテストでは、ネットワーク分割、パケットドロップ（ClickHouseノード間、ClickHouseとZooKeeper間、ClickHouseサーバーとクライアント間など）など、さまざまな障害ケースをエミュレートします。), `kill -9`, `kill -STOP` と `kill -CONT` 例えば [ジェプセン](https://aphyr.com/tags/Jepsen). その後、試験チェックすべての認識を挿入したすべて拒否された挿入しました。 -定足数を緩和試験の筆に別々のチーム前clickhouseしたオープン達した. このチームは、もはやclickhouseで動作しません。 テストはaccidentially javaで書かれました。 これらのことから、決議の定足数テストを書き換え及び移転統合。 +定足数を緩和試験の筆に別々のチーム前ClickHouseしたオープン達した. このチームはClickHouseでは動作しなくなりました。 テストは誤ってJavaで書かれました。 これらのことから、決議の定足数テストを書き換え及び移転統合。 ## 手動テスト {#manual-testing} -新しい機能を開発するときは、手動でテストすることも合理的です。 次の手順で行うことができます: +新しい機能を開発するときは、手動でもテストするのが妥当です。 これを行うには、次の手順を実行します: -ClickHouseをビルドします。 ターミナルからClickHouseを実行します。 `programs/clickhouse-server` そして、それを実行します `./clickhouse-server`. それは構成を使用します (`config.xml`, `users.xml` と内のファイル `config.d` と `users.d` ディレクトリ)から、現在のディレクトリがデフォルトです。 ClickHouseサーバーに接続するには、以下を実行します `programs/clickhouse-client/clickhouse-client`. +ClickHouseを構築します。 ターミナルからClickHouseを実行します。 `programs/clickhouse-server` そして、それを実行します `./clickhouse-server`. それは構成を使用します (`config.xml`, `users.xml` そして内のファイル `config.d` と `users.d` ディレクトリ)から、現在のディレクトリがデフォルトです。 ClickHouseサーバーに接続するには、以下を実行します `programs/clickhouse-client/clickhouse-client`. -これらのclickhouseツール（サーバ、クライアント、などだそうでsymlinks単一のバイナリ名 `clickhouse`. このバイナリは次の場所にあります `programs/clickhouse`. すべてのツ `clickhouse tool` 代わりに `clickhouse-tool`. +これらのclickhouseツール（サーバ、クライアント、などだそうでsymlinks単一のバイナリ名 `clickhouse`. このバイナリは `programs/clickhouse`. すべてのツ `clickhouse tool` 代わりに `clickhouse-tool`. -または、yandexリポジトリからの安定したリリースか、あなた自身のためのパッケージを構築することができます `./release` ClickHouseのソースのルートで. 次に、 `sudo service clickhouse-server start` （またはサーバーを停止するために停止）。 でログを探します `/etc/clickhouse-server/clickhouse-server.log`. +またインストールすることができClickHouseパッケージは安定したリリースからのYandexリポジトリあるいはすることで作ることができるパッケージで `./release` ClickHouseソースルートで. 次に、サーバーを起動します `sudo service clickhouse-server start` (または停止してサーバーを停止します)。 ログを探す `/etc/clickhouse-server/clickhouse-server.log`. -ClickHouseが既にシステムにインストールされている場合は、新しい `clickhouse` バイナリと既存のバイナリを交換: +時ClickHouseでに既にインストールされているシステムを構築できる新しい `clickhouse` 既存のバイナリを置き換えます: ``` bash $ sudo service clickhouse-server stop @@ -92,7 +92,7 @@ $ sudo cp ./clickhouse /usr/bin/ $ sudo service clickhouse-server start ``` -また、システムclickhouse-serverを停止し、同じ設定でターミナルにログインして独自に実行することもできます: +また、システムclickhouse-serverを停止し、同じ構成ではなく端末にログインして独自のものを実行することもできます: ``` bash $ sudo service clickhouse-server stop @@ -105,25 +105,25 @@ Gdbの例: $ sudo -u clickhouse gdb --args /usr/bin/clickhouse server --config-file /etc/clickhouse-server/config.xml ``` -システムクリックハウスサーバーが既に実行されていて、それを停止したくない場合は、ポート番号を変更することができます `config.xml` （またはファイル内でそれらを上書きする `config.d` ディレクトリ)を指定し、適切なデータパスを指定して実行します。 +システムclickhouse-serverがすでに実行されていて、それを停止したくない場合は、次のポート番号を変更できます `config.xml` （または、ファイル内でそれらを上書きする `config.d` ディレクトリ）、適切なデータパスを提供し、それを実行します。 -`clickhouse` バイナリーはほとんどない依存関係の作品を広い範囲のLinuxディストリビューション. サーバー上の変更をすばやく汚れてテストするには、次のようにします `scp` あなたの新鮮な内蔵 `clickhouse` サーバーへのバイナリを作成し、上記の例のように実行します。 +`clickhouse` バイナリーはほとんどない依存関係の作品を広い範囲のLinuxディストリビューション. サーバー上で変更を迅速かつ汚いテストするには、次のことができます `scp` あなたの新鮮な構築 `clickhouse` あなたのサーバーにバイナリし、上記の例のように実行します。 ## テスト環境 {#testing-environment} -安定版としてリリースを公開する前に、テスト環境に展開します。 テスト環境は1/39の部分を処理するクラスタです [Yandexの。Metrica](https://metrica.yandex.com/) データ。 テスト環境をYandexと共有します。メトリカチーム。 ﾂづﾂつｿﾂづｫﾂづｱﾂ鳴ｳﾂ猟ｿﾂづﾂつｷﾂ。 まずデータを処理しなが遅れから、オシロスコープのリアルタイムレプリケーションの継続作業とな問題に見えるYandex.メトリカチーム。 最初のチェックは次の方法で行うことができます: +リリースを安定版として公開する前に、テスト環境に展開します。 テスト環境は1/39の部分を処理する集りです [Yandex.メトリカ](https://metrica.yandex.com/) データ テスト環境をYandexと共有しています。メトリカ-チーム ClickHouseは既存のデータの上にダウンタイムなしで改善される。 私たちは、データがリアルタイムから遅れることなく正常に処理され、複製が動作し続け、Yandexに見える問題はないことを最初に見ています。メトリカ-チーム 最初のチェックは、次の方法で行うことができます: ``` sql SELECT hostName() AS h, any(version()), any(uptime()), max(UTCEventTime), count() FROM remote('example01-01-{1..3}t', merge, hits) WHERE EventDate >= today() - 2 GROUP BY h ORDER BY h; ``` -場合によっては、yandex：market、cloudなどの友人チームのテスト環境にも展開します。 また、開発目的で使用されるハードウェアサーバーもあります。 +市場、クラウドなど：いくつかのケースでは、我々はまた、Yandexの中で私たちの友人チームのテスト環境に展開します また、開発目的で使用されるハードウェアサーバーもあります。 ## 負荷テスト {#load-testing} -テスト環境に展開した後、本番クラスターからのクエリで負荷テストを実行します。 これは手動で行われます。 +後の展開を試験環境を実行負荷テストクエリから生産ます。 これは手動で行われます。 -有効にしていることを確認します `query_log` あなたの生産の集りで。 +有効にしていることを確認します `query_log` 運用クラスター上。 一日以上のクエリログを収集する: @@ -131,122 +131,131 @@ SELECT hostName() AS h, any(version()), any(uptime()), max(UTCEventTime), count( $ clickhouse-client --query="SELECT DISTINCT query FROM system.query_log WHERE event_date = today() AND query LIKE '%ym:%' AND query NOT LIKE '%system.query_log%' AND type = 2 AND is_initial_query" > queries.tsv ``` -これは複雑な例です。 `type = 2` 正常に実行されたクエリをフィルタ処理します。 `query LIKE '%ym:%'` Yandexのから関連するクエリを選択することです。メトリカ `is_initial_query` ClickHouse自体ではなく、クライアントによって開始されたクエリのみを選択することです（分散クエリ処理の一部として）。 +これは複雑な例です。 `type = 2` 正常に実行されたクエリをフィルタ処理します。 `query LIKE '%ym:%'` Yandexから関連するクエリを選択することです。メトリカ `is_initial_query` ClickHouse自体ではなく、クライアントによって開始されたクエリのみを選択することです（分散クエリ処理の一部として）。 -`scp` このログを試験クラスターとして以下の: +`scp` このログをテストクラスタに記録し、次のように実行します: ``` bash $ clickhouse benchmark --concurrency 16 < queries.tsv ``` -（おそらく、あなたはまた、指定したいです `--user`) +（おそらくあなたはまた、 `--user`) -それから夜か週末の間それを残し、取得残りを行きなさい。 +それから夜または週末のためにそれを残し、残りを取る行きなさい。 きることを確認 `clickhouse-server` なクラッシュメモリのフットプリントは有界性なつ品位を傷つける。 -正確なクエリ実行タイミングは記録されず、クエリと環境の変動が大きいため比較されません。 +クエリと環境の変動が大きいため、正確なクエリ実行タイミングは記録されず、比較されません。 ## ビルドテスト {#build-tests} -構築を試験できることを確認の構築においても様々な代替構成されており、外国のシステム。 試験は `ci` ディレクトリ。 彼らはDocker、Vagrantの中のソースからビルドを実行し、時には `qemu-user-static` ドッカー内部。 これらのテストは開発中であり、テスト実行は自動化されていません。 +構築を試験できることを確認の構築においても様々な代替構成されており、外国のシステム。 テストは `ci` ディレクトリ。 Docker、Vagrant、時には以下のようなソースからビルドを実行します `qemu-user-static` ドッカー内部。 これらのテストは開発中であり、テストの実行は自動化されません。 -動機づけ: +動機: -通常、clickhouseビルドの単一のバリアントですべてのテストをリリースして実行します。 しかし、徹底的にテストされていない代替ビルドの変種があります。 例: +通常、ClickHouse buildの単一のバリアントですべてのテストをリリースして実行します。 しかし、徹底的にテストされていない別のビルド変種があります。 例: -- FreeBSD上でのビルド; -- システムパッケージのライブ; -- ライブラリの共有リンク付きビルド; -- AArch64プラットフォーム上に構築; -- PowerPcプラットフォーム上に構築。 +- FreeBSD上でビルド; +- をDebianを対象として図書館システムのパッケージ; +- ライブラリの共有リンクでビルド; +- AArch64プラットフォ; +- PowerPcプラットフォーム上で構築。 -たとえば、構築システムのパッケージが悪い練習ができませんので保証ものに版のパッケージシステムです。 しかし、これは本当にdebianのメンテナに必要です。 このため、少なくともこのビルドの変種をサポートする必要があります。 別の例：共有リンクは一般的なトラブルの原因ですが、一部の愛好家には必要です。 +たとえば、システムパッケージを使用したビルドは悪い習慣です。 しかし、これは本当にDebianメンテナに必要です。 このため、少なくともこのビルドの変種をサポートする必要があります。 別の例：共有リンクは一般的な問題の原因ですが、一部の愛好家にとって必要です。 ができませんので実行した全試験はすべての変異体を構築し、チェックしたい少なくとも上記に記載された各種の構築異な破となりました。 この目的のためにビルドテストを使用します。 ## プロトコル互換性のテスト {#testing-for-protocol-compatibility} -我々はclickhouseのネットワークプロトコルを拡張するとき,我々は、古いclickhouse-クライアントが新しいclickhouse-serverで動作し、新しいclickhouse-clientが古いclickhouse-serverで動作することを手動で +ClickHouse network protocolを拡張すると、古いclickhouse-clientが新しいclickhouse-serverで動作し、新しいclickhouse-clientが古いclickhouse-serverで動作することを手動でテストします（対応するパッケージからバイナリを -## コンパイラからの助け {#help-from-the-compiler} +## コンパイラからのヘルプ {#help-from-the-compiler} -メインクリックハウスコード `dbms` ディレクトリ)は `-Wall -Wextra -Werror` そして、いくつかの追加の有効な警告と。 これらのオプションは有効になっていないためにサードパーティーのライブラリ. +メインクリックハウスコード（にある `dbms` ディレクトリ)は `-Wall -Wextra -Werror` そして、いくつかの追加の有効な警告と。 これらのオプションは有効になっていないためにサードパーティーのライブラリ. Clangにはさらに便利な警告があります。 `-Weverything` デフォルトのビルドに何かを選ぶ。 -プロダクションビルドでは、gccが使用されます（clangよりもやや効率的なコードが生成されます）。 開発のために、clangは通常使用するのがより便利です。 デバッグモードで自分のマシン上に構築することができます（ラップトップのバッテリーを節約するため）が、コンパイラはより多くの警告を生成する `-O3` よりよい制御流れおよび相互プロシージャの分析が原因で。 Clangでビルドするとき, `libc++` の代わりに使用される。 `libstdc++` そして、デバッグモードでビルドするときは、 `libc++` 使用可能にするにはより誤差があります。. +本番ビルドでは、gccが使用されます（clangよりもやや効率的なコードが生成されます）。 開発のために、clangは通常、使用する方が便利です。 あなたは（あなたのラップトップのバッテリーを節約するために）デバッグモードで自分のマシン上で構築することができますが、コンパイラがでより `-O3` よりよい制御フローおよびinter-procedure分析が原因で。 Clangでビルドする場合, `libc++` の代わりに使用されます。 `libstdc++` そして、デバッグモードでビルドするとき、 `libc++` 使用可能にするにはより誤差があります。. -## 消毒剤 {#sanitizers} +## サニタイザー {#sanitizers} -**アドレス消毒剤**. -ASanの下でコミットごとに機能テストと統合テストを実行します。 +**アドレスsanitizer**. +私たちは、コミットごとにASanの下で機能テストと統合テストを実行します。 -**Valgrind(Memcheck)**. -私たちは一晩valgrindの下で機能テストを実行します。 それは複数の時間がかかります。 現在、既知の偽陽性があります `re2` ライブラリ、参照 [この記事](https://research.swtch.com/sparse). +**ヴァルグリンド(曖昧さ回避)**. +私たちは一晩Valgrindの下で機能テストを実行します。 数時間かかります。 現在知られている偽陽性があります `re2` 図書館、参照 [この記事](https://research.swtch.com/sparse). -**未定義の動作消毒剤。** -ASanの下でコミットごとに機能テストと統合テストを実行します。 +**未定義の動作のサニタイザー。** +私たちは、コミットごとにASanの下で機能テストと統合テストを実行します。 -**スレッド消毒剤**. -TSanの下でコミットごとに機能テストを実行します。 TSanの下では、コミットごとに統合テストは実行されません。 +**糸のsanitizer**. +私たちは、コミットごとにTSanの下で機能テストを実行します。 コミットごとにTSanの下で統合テストを実行することはまだありません。 -**メモリ消毒剤**. -現在、我々はまだmsanを使用していません。 +**メモリサニタイザー**. +現在、我々はまだMSanを使用していません。 **デバッグアロケータ。** デバッグバージョン `jemalloc` デバッグビルドに使用されます。 -## Fuzzing {#fuzzing} +## ファジング {#fuzzing} -単純なfuzzテストを使用して、ランダムなsqlクエリを生成し、サーバーが死んでいないことを確認します。 ファジーテストはアドレスサニタイザーで実行されます。 あなたはそれを見つける `00746_sql_fuzzy.pl`. このテストは継続的に実行する必要があります（夜間および長期）。 +ClickHouseファジングは、両方を使用して実装されます [libFuzzer](https://llvm.org/docs/LibFuzzer.html) とランダムSQLクエリ。 +すべてのファズテストは、サニタイザー（アドレスと未定義）で実行する必要があります。 -December2018の時点では、ライブラリコードの孤立したファズテストはまだ使用していません。 +LibFuzzerは、ライブラリコードの分離ファズテストに使用されます。 ファザーはテストコードの一部として実装され “\_fuzzer” 名前の接尾辞。 +Fuzzerの例はで見つけることができます `src/Parsers/tests/lexer_fuzzer.cpp`. LibFuzzer固有の設定、辞書、およびコーパスは次の場所に格納されます `tests/fuzz`. +ご協力をお願いいたし書きファズ試験べての機能を取り扱うユーザー入力します。 + +ファザーはデフォルトではビルドされません。 両方のファザーを構築するには `-DENABLE_FUZZING=1` と `-DENABLE_TESTS=1` 選択は置かれるべきである。 +ファザーのビルド中にJemallocを無効にすることをお勧めします。 ClickHouseファジングを統合するために使用される設定 +Google OSS-Fuzzは次の場所にあります `docker/fuzz`. + +また簡単なファズ試験をランダムなSQLクエリーやことを確認するにはサーバーにな金型を実行します。 +それを見つけることができる `00746_sql_fuzzy.pl`. このテストは、継続的に実行する必要があります（一晩と長い）。 ## セキュリティ監査 {#security-audit} -Yandexのクラウド部門の人々は、セキュリティの観点からClickHouse機能のいくつかの基本的な概要を行います。 +人からのYandexセキュリティチームはいくつかの基本的な概要ClickHouse力からのセキュリティの観点から. -## 静的分析器 {#static-analyzers} +## 静的アナライザ {#static-analyzers} -私たちは走る `PVS-Studio` コミットごとに。 我々は評価した `clang-tidy`, `Coverity`, `cppcheck`, `PVS-Studio`, `tscancode`. あなたは、使用中の使用方法を見つけるでしょう `tests/instructions/` ディレクトリ。 また読むことができます [ロシア語の記事](https://habr.com/company/yandex/blog/342018/). +私たちは走る `PVS-Studio` コミットごと。 私達は評価しました `clang-tidy`, `Coverity`, `cppcheck`, `PVS-Studio`, `tscancode`. 使用のための指示をで見つけます `tests/instructions/` ディレクトリ。 また読むことができます [ロシア語の記事](https://habr.com/company/yandex/blog/342018/). -使用する場合 `CLion` IDEとして、次のいくつかを活用できます `clang-tidy` 箱からの点検。 +を使用する場合 `CLion` IDEとして、いくつかを活用できます `clang-tidy` 箱から出してチェックします。 ## 硬化 {#hardening} -`FORTIFY_SOURCE` デフォルトで使用されます。 それはほとんど役に立たないですが、まれに意味があり、私たちはそれを無効にしません。 +`FORTIFY_SOURCE` デフォルトで使用されます。 それはほとんど役に立たないですが、まれに理にかなっており、それを無効にしません。 ## コードスタイル {#code-style} コードのスタイルのルールを記述 [ここに](https://clickhouse.tech/docs/en/development/style/). -一般的なスタイル違反を確認するには、次のようにします `utils/check-style` スクリプト +チェックのための、共通したスタイル違反、利用できる `utils/check-style` スクリプト -コードの適切なスタイルを強制するには、次のようにします `clang-format`. ファイル `.clang-format` ソースのルートにあります。 主に実際のコードスタイルに対応しています。 しかし、適用することはお勧めしません `clang-format` 既存のファイルには、書式設定が悪化するためです。 を使用することができ `clang-format-diff` clangソースリポジトリにあるツールです。 +コードの適切なスタイルを強制するには、次のようにします `clang-format`. ファイル `.clang-format` ソースルートにあります。 実際のコードスタイルにほとんど対応しています。 しかし、適用することはお勧めしません `clang-format` 既存のファイルへの書式設定が悪化するためです。 以下を使用できます `clang-format-diff` clangソースリポジトリで見つけることができるツール。 -あるいは、 `uncrustify` コードを再フォーマットするツール。 設定は `uncrustify.cfg` ソースのルートで。 での試験によ `clang-format`. +あるいは、 `uncrustify` コードを再フォーマットするツール。 設定は次のとおりです `uncrustify.cfg` ソースルートで。 それはより少なくテストさ `clang-format`. `CLion` 独自のコードをフォーマッタしていると見ることができる調整のためのコードです。 -## メトリカb2bテスト {#metrica-b2b-tests} +## Metrica B2Bテスト {#metrica-b2b-tests} -各clickhouseのリリースは、yandexのメトリカとappmetricaエンジンでテストされています。 クリックハウスのテストおよび安定版は、vm上に配備され、入力データの固定サンプルを処理しているmetricaエンジンの小さなコピーで実行されます。 次に，メトリカエンジンの二つのインスタンスの結果を共に比較した。 +各ClickHouseリリースはYandex MetricaとAppMetricaエンジンでテストされます。 ClickHouseのテスト版と安定版はVmにデプロイされ、入力データの固定サンプルを処理するMetrica engineの小さなコピーで実行されます。 次に，Ｍｅｔｒｉｃａエンジンの二つのインスタンスの結果を比較した。 -これらの試験により自動化されており、別のチームです。 可動部品の数が多いため、テストはほとんどの場合完全に無関係な理由で失敗します。 がこれらの試験は負の値です。 しかしこれらの試験することが明らかとなったが有用である一又は二倍の数百名 +これらの試験により自動化されており、別のチームです。 可動部分の高い数が原因で、テストは把握し非常ににくい完全に無関係な理由によって失敗ほとんどの時間です。 がこれらの試験は負の値です。 しかしこれらの試験することが明らかとなったが有用である一又は二倍の数百名 ## テスト範囲 {#test-coverage} -July2018の時点で、テストカバレッジは追跡されません。 +2018年現在、テストカバーは行っていない。 -## テストの自動化 {#test-automation} +## テスト自動化 {#test-automation} -また試験のyandex内ciと雇用自動化システムの名前 “Sandbox”. +Yandex内部CIとジョブ自動化システムという名前のテストを実行します “Sandbox”. -ビルドジョブとテストは、コミットごとにsandboxで実行されます。 結果のパッケージとテスト結果はgithubに公開され、直接リンクでダウンロードできます。 成果物は永遠に保存されます。 githubでpullリクエストを送信すると、次のようにタグ付けします “can be tested” そして私達のCIシステムはあなたのためのClickHouseのパッケージを造ります（解放、住所のsanitizer、等と、デバッグします）。 +ビルドジョブとテストは、コミットごとにSandboxで実行されます。 結果のパッケージとテスト結果はGitHubに公開され、直接リンクでダウンロードできます。 成果物は永遠に保存されます。 GitHubでプルリクエストを送信すると、次のようにタグ付けします “can be tested” そして私達のCIシステムはあなたのためのClickHouseのパッケージ（住所sanitizerの解放、デバッグ、等）を造ります。 -私たちは、時間と計算能力の限界のためにtravis ciを使用しません。 -ジェンキンスは使わない で使用される前に、現しました嬉しい使用していないjenkins. +時間と計算能力の限界のため、Travis CIは使用しません。 +ジェンキンスは使わない 以前は使用されていましたが、今はJenkinsを使用していません。 [元の記事](https://clickhouse.tech/docs/en/development/tests/) -ベロップメント/テスト/) diff --git a/docs/ja/engines/database-engines/index.md b/docs/ja/engines/database-engines/index.md index f895dae3bd6..1cdd1912b5f 100644 --- a/docs/ja/engines/database-engines/index.md +++ b/docs/ja/engines/database-engines/index.md @@ -1,16 +1,16 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_folder_title: Database Engines +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "\u30C7\u30FC\u30BF\u30D9\u30FC\u30B9" toc_priority: 27 -toc_title: "\u5C0E\u5165" +toc_title: "\u306F\u3058\u3081\u306B" --- # データベース {#database-engines} -データベースエンジンできる仕事です。 +データベースエ -既定では、clickhouseはネイティブのデータベースエンジンを使用します。 [表エンジン](../../engines/table-engines/index.md) と [SQLダイアレクト](../../sql-reference/syntax.md). +既定では、ClickHouseはネイティブのデータベースエンジンを使用します。 [表エンジン](../../engines/table-engines/index.md) そして [SQL方言](../../sql-reference/syntax.md). 次のデータベースエンジンも使用できます: diff --git a/docs/ja/engines/database-engines/lazy.md b/docs/ja/engines/database-engines/lazy.md index 7eb7fe5cc93..504907f6894 100644 --- a/docs/ja/engines/database-engines/lazy.md +++ b/docs/ja/engines/database-engines/lazy.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 31 toc_title: "\u6020\u3051\u8005" --- # 怠け者 {#lazy} -RAMのみのテーブルを保持します `expiration_time_in_seconds` 最後のアクセスの後の秒。 \*ログテーブルでのみ使用できます。 +RAM内のテーブルのみを保持 `expiration_time_in_seconds` 最後のアクセスの秒後。 \*Logテーブルでのみ使用できます。 これは、アクセス間に長い時間間隔がある多くの小さな\*ログテーブルを格納するために最適化されています。 diff --git a/docs/ja/engines/database-engines/mysql.md b/docs/ja/engines/database-engines/mysql.md index f4c6c6bfb3e..efb3893a385 100644 --- a/docs/ja/engines/database-engines/mysql.md +++ b/docs/ja/engines/database-engines/mysql.md @@ -1,15 +1,15 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 30 toc_title: MySQL --- -# Mysql {#mysql} +# MySQL {#mysql} -リモートmysqlサーバー上のデータベースに接続し、 `INSERT` と `SELECT` ClickHouseとMySQLの間でデータを交換するためのクエリ。 +で接続するデータベースのリモートMySQLサーバを実行 `INSERT` と `SELECT` ClickHouseとMySQLの間でデータを交換するためのクエリ。 -その `MySQL` データベースエンジンの翻訳のクエリのMySQLサーバーでの操作を行うことができなど `SHOW TABLES` または `SHOW CREATE TABLE`. +その `MySQL` データベースエンジ `SHOW TABLES` または `SHOW CREATE TABLE`. 次のクエリは実行できません: @@ -21,7 +21,7 @@ toc_title: MySQL ``` sql CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster] -ENGINE = MySQL('host:port', 'database', 'user', 'password') +ENGINE = MySQL('host:port', ['database' | database], 'user', 'password') ``` **エンジン変数** @@ -49,9 +49,9 @@ ENGINE = MySQL('host:port', 'database', 'user', 'password') | DATETIME, TIMESTAMP | [DateTime](../../sql-reference/data-types/datetime.md) | | BINARY | [FixedString](../../sql-reference/data-types/fixedstring.md) | -他のすべてのmysqlデータ型に変換され [文字列](../../sql-reference/data-types/string.md). +他のすべてのMySQLデータ型に変換され [文字列](../../sql-reference/data-types/string.md). -[Nullable](../../sql-reference/data-types/nullable.md) サポートされます。 +[Null可能](../../sql-reference/data-types/nullable.md) サポートされます。 ## 使用例 {#examples-of-use} diff --git a/docs/ja/engines/index.md b/docs/ja/engines/index.md index dcebf255df6..bb8e337adf8 100644 --- a/docs/ja/engines/index.md +++ b/docs/ja/engines/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_folder_title: Engines +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "\u30A8\u30F3\u30B8\u30F3" toc_priority: 25 --- diff --git a/docs/ja/engines/table-engines/index.md b/docs/ja/engines/table-engines/index.md index 79cd4493eb5..d73e9ee32ef 100644 --- a/docs/ja/engines/table-engines/index.md +++ b/docs/ja/engines/table-engines/index.md @@ -1,41 +1,41 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_folder_title: Table Engines +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "\u8868\u30A8\u30F3\u30B8\u30F3" toc_priority: 26 -toc_title: "\u5C0E\u5165" +toc_title: "\u306F\u3058\u3081\u306B" --- # 表エンジン {#table_engines} -表エンジン(表のタイプ: +のテーブルエンジン型式の表を行います。: -- どのようにデータが格納されている場所、それをどこに書き込むか、どこから読み込むか。 -- どのクエリがサポートされ、どのように。 +- データの格納方法と場所、データの書き込み先、およびデータの読み取り先。 +- サポートされているクエリと方法。 - 同時データアクセス。 - インデックスが存在する場合の使用。 -- マルチスレッドリクエストの実行が可能かどうか。 +- マルチスレッド要求の実行が可能かどうか。 - データ複製パラメーター。 ## エンジン家族 {#engine-families} -### Mergetree {#mergetree} +### メルゲツリー {#mergetree} -高負荷仕事のための最も普遍的な、機能テーブルエンジン。 本物件の共有によるこれらのエンジンには迅速にデータを挿入とその後のバックグラウンドデータを処となります。 `MergeTree` 家族のエンジンの支援データレプリケーション（ [複製された\*](mergetree-family/replication.md#replication) バージョンのエンジン)分割、その他の機能で対応していないその他のエンジンです。 +高負荷仕事のための最も普遍的な、機能テーブルエンジン。 本物件の共有によるこれらのエンジンには迅速にデータを挿入とその後のバックグラウンドデータを処となります。 `MergeTree` 家族のエンジンの支援データレプリケーション（ [複製\*](mergetree-family/replication.md#table_engines-replication) バージョンのエンジン)分割、その他の機能で対応していないその他のエンジンです。 家族のエンジン: -- [MergeTree](mergetree-family/mergetree.md#mergetree) -- [ﾂつｨﾂ姪“ﾂつ”ﾂ債ﾂつｹ](mergetree-family/replacingmergetree.md#replacingmergetree) -- [SummingMergeTree](mergetree-family/summingmergetree.md#summingmergetree) -- [ﾂつｨﾂ姪“ﾂつ”ﾂ債ﾂづｭﾂつｹ](mergetree-family/aggregatingmergetree.md#aggregatingmergetree) -- [CollapsingMergeTree](mergetree-family/collapsingmergetree.md#table_engine-collapsingmergetree) -- [VersionedCollapsingMergeTree](mergetree-family/versionedcollapsingmergetree.md#versionedcollapsingmergetree) -- [グラフィットメールグツリー](mergetree-family/graphitemergetree.md#graphitemergetree) +- [メルゲツリー](mergetree-family/mergetree.md#mergetree) +- [置換マージツリー](mergetree-family/replacingmergetree.md#replacingmergetree) +- [サミングマーゲツリー](mergetree-family/summingmergetree.md#summingmergetree) +- [AggregatingMergeTree](mergetree-family/aggregatingmergetree.md#aggregatingmergetree) +- [折りたたみマージツリー](mergetree-family/collapsingmergetree.md#table_engine-collapsingmergetree) +- [バージョニングコラプシングマーゲットリー](mergetree-family/versionedcollapsingmergetree.md#versionedcollapsingmergetree) +- [GraphiteMergeTree](mergetree-family/graphitemergetree.md#graphitemergetree) ### ログ {#log} -軽量 [エンジン](log-family/index.md) 最低の機能性を使って。 多くの小さなテーブル（約1万行まで）をすばやく作成し、後でそれらを全体として読み取る必要がある場合、これらは最も効果的です。 +軽量 [エンジン](log-family/index.md) 最低の機能性を使って。 多くの小さなテーブル（最大約1万行）をすばやく書き込み、後で全体として読み込む必要がある場合に最も効果的です。 家族のエンジン: @@ -49,7 +49,7 @@ toc_title: "\u5C0E\u5165" 家族のエンジン: -- [カフカname](integrations/kafka.md#kafka) +- [カフカ](integrations/kafka.md#kafka) - [MySQL](integrations/mysql.md#mysql) - [ODBC](integrations/odbc.md#table-engine-odbc) - [JDBC](integrations/jdbc.md#table-engine-jdbc) @@ -60,26 +60,26 @@ toc_title: "\u5C0E\u5165" 家族のエンジン: - [分散](special/distributed.md#distributed) -- [MaterializedView](special/materializedview.md#materializedview) +- [マテリアライズドビュー](special/materializedview.md#materializedview) - [辞書](special/dictionary.md#dictionary) -- [マージ](special/merge.md#merge +- \[Merge\](special/merge.md\#merge - [ファイル](special/file.md#file) -- [ヌル](special/null.md#null) +- [Null](special/null.md#null) - [セット](special/set.md#set) - [参加](special/join.md#join) - [URL](special/url.md#table_engines-url) -- [ビュー](special/view.md#table_engines-view) +- [表示](special/view.md#table_engines-view) - [メモリ](special/memory.md#memory) - [バッファ](special/buffer.md#buffer) ## 仮想列 {#table_engines-virtual_columns} -Virtual columnは、エンジンのソースコードで定義されているテーブルエンジンの属性です。 +仮想列は、エンジンのソースコードで定義されている整数テーブルエンジン属性です。 -仮想列を指定しないでください。 `CREATE TABLE` クエリとあなたはそれらを見るこ `SHOW CREATE TABLE` と `DESCRIBE TABLE` クエリ結果。 仮想列も読み取り専用であるため、仮想列にデータを挿入することはできません。 +仮想列を指定するべきではありません。 `CREATE TABLE` あなたはそれらを見ることができません `SHOW CREATE TABLE` と `DESCRIBE TABLE` クエリ結果。 仮想列も読み取り専用であるため、仮想列にデータを挿入することはできません。 -仮想カラムからデータを選択するには、仮想カラムの名前を指定する必要があります。 `SELECT` クエリ。 `SELECT *` 仮想列から値を返しません。 +仮想列からデータを選択するには、仮想列の名前を指定する必要があります。 `SELECT` クエリ。 `SELECT *` 仮想列から値を返しません。 -テーブル仮想列のいずれかと同じ名前の列を持つテーブルを作成すると、仮想列にアクセスできなくなります。 これはお勧めしません。 競合を回避するために、通常、仮想列名にはアンダースコアが付加されます。 +テーブル仮想列のいずれかと同じ名前の列を持つテーブルを作成すると、仮想列にアクセスできなくなります。 これはお勧めしません。 競合を避けるために、仮想列名には通常、アンダースコアが付けられます。 [元の記事](https://clickhouse.tech/docs/en/operations/table_engines/) diff --git a/docs/ja/engines/table-engines/integrations/hdfs.md b/docs/ja/engines/table-engines/integrations/hdfs.md index a81077c4cf5..8b7380d2e7e 100644 --- a/docs/ja/engines/table-engines/integrations/hdfs.md +++ b/docs/ja/engines/table-engines/integrations/hdfs.md @@ -1,29 +1,29 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 36 toc_title: HDFS --- # HDFS {#table_engines-hdfs} -このエンジンは、 [Apache Hadoop](https://en.wikipedia.org/wiki/Apache_Hadoop) データの管理を可能にすることによって生態系 [HDFS](https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/HdfsDesign.html)クリックハウス経由。 このエンジンは同様です +このエンジンは、 [Apache Hadoop](https://en.wikipedia.org/wiki/Apache_Hadoop) 生態系および管理データ [HDFS](https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/HdfsDesign.html)クリックハウス経由。 このエンジンは同様です に [ファイル](../special/file.md#table_engines-file) と [URL](../special/url.md#table_engines-url) エンジンが、Hadoop固有の機能を提供します。 -## 使い方 {#usage} +## 使用法 {#usage} ``` sql ENGINE = HDFS(URI, format) ``` -その `URI` parameterは、HDFSのファイルURI全体です。 +その `URI` パラメータは、HDFS内のファイルURI全体です。 その `format` パラメータを指定するか、ファイルのファイルフォーマット 実行するには -`SELECT` クエリは、形式は、入力のためにサポートされ、実行する必要があります +`SELECT` この形式は、入力と実行のためにサポートされている必要があります `INSERT` queries – for output. The available formats are listed in the -[形式](../../../interfaces/formats.md#formats) セクション。 -のパス部分 `URI` グロブを含む可能性があります。 この場合、テーブルは読み取り専用になります。 +[形式](../../../interfaces/formats.md#formats) セクション +のパス部分 `URI` globsを含むことができます。 この場合、テーブルはreadonlyになります。 -**例えば:** +**例:** **1.** セットアップ `hdfs_engine_table` テーブル: @@ -37,7 +37,7 @@ CREATE TABLE hdfs_engine_table (name String, value UInt32) ENGINE=HDFS('hdfs://h INSERT INTO hdfs_engine_table VALUES ('one', 1), ('two', 2), ('three', 3) ``` -**3.** データのクエリ: +**3.** データの照会: ``` sql SELECT * FROM hdfs_engine_table LIMIT 2 @@ -53,25 +53,25 @@ SELECT * FROM hdfs_engine_table LIMIT 2 ## 実装の詳細 {#implementation-details} - 読み書きできる並列 -- サポートなし: - - `ALTER` と `SELECT...SAMPLE` オペレーション +- 対応していません: + - `ALTER` と `SELECT...SAMPLE` 作戦だ - インデックス。 - - 複製だ + - 複製。 **パス内のグロブ** -複数のパスコンポーネン のための処理中のファイルが存在するマッチのパスのパターンです。 ファイルのリストは、 `SELECT` （ないで `CREATE` 瞬間）。 +複数のパスコンポーネ のための処理中のファイルが存在するマッチのパスのパターンです。 ファイルのリストは `SELECT` （ではない `CREATE` 瞬間）。 - `*` — Substitutes any number of any characters except `/` 空の文字列を含む。 - `?` — Substitutes any single character. - `{some_string,another_string,yet_another_one}` — Substitutes any of strings `'some_string', 'another_string', 'yet_another_one'`. - `{N..M}` — Substitutes any number in range from N to M including both borders. -構造との `{}` に類似していて下さい [リモート](../../../sql-reference/table-functions/remote.md) テーブル機能。 +構造との `{}` に類似しています [リモート](../../../sql-reference/table-functions/remote.md) テーブル関数。 -**例えば** +**例** -1. HDFSに次のUriを持つTSV形式のファイルがいくつかあるとします: +1. HDFS上に次のUriを持つTSV形式のファイルがいくつかあるとします: - ‘hdfs://hdfs1:9000/some\_dir/some\_file\_1’ - ‘hdfs://hdfs1:9000/some\_dir/some\_file\_2’ @@ -101,9 +101,9 @@ CREATE TABLE table_with_asterisk (name String, value UInt32) ENGINE = HDFS('hdfs ``` !!! warning "警告" - ファイルのリストに先行するゼロが付いた数値範囲が含まれている場合は、各桁ごとに中かっこで囲みます。 `?`. + ファイ `?`. -**例えば** +**例** このように作成されたテーブルとファイル名 `file000`, `file001`, … , `file999`: @@ -116,7 +116,7 @@ CREARE TABLE big_table (name String, value UInt32) ENGINE = HDFS('hdfs://hdfs1:9 - `_path` — Path to the file. - `_file` — Name of the file. -**また見なさい** +**も参照。** - [仮想列](../index.md#table_engines-virtual_columns) diff --git a/docs/ja/engines/table-engines/integrations/index.md b/docs/ja/engines/table-engines/integrations/index.md index 8d7196c323a..8aeadb71ecd 100644 --- a/docs/ja/engines/table-engines/integrations/index.md +++ b/docs/ja/engines/table-engines/integrations/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_folder_title: Integrations +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "\u7D71\u5408" toc_priority: 30 --- diff --git a/docs/ja/engines/table-engines/integrations/jdbc.md b/docs/ja/engines/table-engines/integrations/jdbc.md index 07f69b1ed81..ad4520295a0 100644 --- a/docs/ja/engines/table-engines/integrations/jdbc.md +++ b/docs/ja/engines/table-engines/integrations/jdbc.md @@ -1,17 +1,17 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 34 toc_title: JDBC --- # JDBC {#table-engine-jdbc} -ClickHouseが外部データベースに接続できるようにします [JDBC](https://en.wikipedia.org/wiki/Java_Database_Connectivity). +ClickHouseが外部データベースに接続できるようにする [JDBC](https://en.wikipedia.org/wiki/Java_Database_Connectivity). -JDBC接続を実装するには、ClickHouseは別のプログラムを使用します [clickhouse-jdbc橋](https://github.com/alex-krash/clickhouse-jdbc-bridge) うにしてくれました。 +JDBC接続を実装するには、ClickHouseは別のプログラムを使用します [clickhouse-jdbc-bridge](https://github.com/alex-krash/clickhouse-jdbc-bridge) うにしてくれました。 -このエンジンは、 [Nullable](../../../sql-reference/data-types/nullable.md) データ型。 +このエンジンは [Null可能](../../../sql-reference/data-types/nullable.md) データ型。 ## テーブルの作成 {#creating-a-table} @@ -27,7 +27,7 @@ ENGINE = JDBC(dbms_uri, external_database, external_table) - `dbms_uri` — URI of an external DBMS. - 書式: `jdbc:://:/?user=&password=`. + 形式: `jdbc:://:/?user=&password=`. MySQLの例: `jdbc:mysql://localhost:3306/?user=root&password=root`. - `external_database` — Database in an external DBMS. @@ -36,7 +36,7 @@ ENGINE = JDBC(dbms_uri, external_database, external_table) ## 使用例 {#usage-example} -コンソールクライアントと直接接続してmysqlサーバーにテーブルを作成する: +コンソールクライアントに直接接続してMySQL serverでテーブルを作成する: ``` text mysql> CREATE TABLE `test`.`test` ( @@ -59,7 +59,7 @@ mysql> select * from test; 1 row in set (0,00 sec) ``` -ClickHouseサーバーでテーブルを作成し、そこからデータを選択する: +ClickHouse serverでテーブルを作成し、そこからデータを選択する: ``` sql CREATE TABLE jdbc_table @@ -83,7 +83,7 @@ FROM jdbc_table └────────┴──────────────┴───────┴────────────────┘ ``` -## また見なさい {#see-also} +## も参照。 {#see-also} - [JDBCテーブル関数](../../../sql-reference/table-functions/jdbc.md). diff --git a/docs/ja/engines/table-engines/integrations/kafka.md b/docs/ja/engines/table-engines/integrations/kafka.md index 40725a95a6f..6b5d6cc4d8b 100644 --- a/docs/ja/engines/table-engines/integrations/kafka.md +++ b/docs/ja/engines/table-engines/integrations/kafka.md @@ -1,19 +1,19 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 32 -toc_title: "\u30AB\u30D5\u30ABname" +toc_title: "\u30AB\u30D5\u30AB" --- -# カフカname {#kafka} +# カフカ {#kafka} このエンジンは [アパッチ-カフカ](http://kafka.apache.org/). -カフカはあなたを可能にします: +カフカはあなたをできます: - データフローを公開または購読する。 - 整理-フォールトトレラント保管します。 -- プロセスストリームが使用可能になるとき。 +- ストリームが使用可能になったら処理します。 ## テーブルの作成 {#table_engine-kafka-creating-a-table} @@ -32,22 +32,26 @@ SETTINGS [kafka_row_delimiter = 'delimiter_symbol',] [kafka_schema = '',] [kafka_num_consumers = N,] - [kafka_skip_broken_messages = N] + [kafka_max_block_size = 0,] + [kafka_skip_broken_messages = N,] + [kafka_commit_every_batch = 0] ``` 必須パラメータ: - `kafka_broker_list` – A comma-separated list of brokers (for example, `localhost:9092`). - `kafka_topic_list` – A list of Kafka topics. -- `kafka_group_name` – A group of Kafka consumers. Reading margins are tracked for each group separately. If you don’t want messages to be duplicated in the cluster, use the same group name everywhere. -- `kafka_format` – Message format. Uses the same notation as the SQL `FORMAT` 機能、のような `JSONEachRow`. 詳細については、 [形式](../../../interfaces/formats.md) セクション。 +- `kafka_group_name` – A group of Kafka consumers. Reading margins are tracked for each group separately. If you don't want messages to be duplicated in the cluster, use the same group name everywhere. +- `kafka_format` – Message format. Uses the same notation as the SQL `FORMAT` 機能、のような `JSONEachRow`. 詳細については、を参照してください [形式](../../../interfaces/formats.md) セクション 任意変数: - `kafka_row_delimiter` – Delimiter character, which ends the message. -- `kafka_schema` – Parameter that must be used if the format requires a schema definition. For example, [Cap’n Proto](https://capnproto.org/) スキーマファイルへのパスとルートの名前が必要です `schema.capnp:Message` オブジェクト。 +- `kafka_schema` – Parameter that must be used if the format requires a schema definition. For example, [Cap'N Proto](https://capnproto.org/) スキーマファイルへのパスとルートの名前が必要です `schema.capnp:Message` オブジェクト - `kafka_num_consumers` – The number of consumers per table. Default: `1`. 指定しこれからも、多くの消費者の場合、スループットの消費が不足しています。 の総数消費者を超えることはできませんパーティションの数の問題から一つだけの消費者割り当てることができた。 -- `kafka_skip_broken_messages` – Kafka message parser tolerance to schema-incompatible messages per block. Default: `0`. もし `kafka_skip_broken_messages = N` その後、エンジンはスキップ *N* 解析できないKafkaメッセージ(メッセージはデータの行と同じです)。 +- `kafka_max_block_size` -ポーリングの最大バッチサイズ(メッセージ)(デフォルト: `max_block_size`). +- `kafka_skip_broken_messages` – Kafka message parser tolerance to schema-incompatible messages per block. Default: `0`. もし `kafka_skip_broken_messages = N` その後、エンジンがスキップ *N* 解析できないKafkaメッセージ(メッセージはデータの行に等しい)。 +- `kafka_commit_every_batch` コミット毎に消費され、取り扱うバッチの代わりに単一のコミットし、全体をブロック(デフォルト: `0`). 例: @@ -81,7 +85,7 @@ SETTINGS
-テーブルを作成する非推奨の方法 +推奨されていません法テーブルを作成する !!! attention "注意" 用途では使用しないでください方法で新規プロジェクト. 可能であれば、古いプロジェクトを上記の方法に切り替えます。 @@ -95,20 +99,20 @@ Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format ## 説明 {#description} -届いたメッセージは自動的に追跡で、それぞれのメッセージグループでは数えます。 データを取得したい場合は、別のグループ名を持つテーブルのコピーを作成します。 +届いたメッセージは自動的に追跡で、それぞれのメッセージグループでは数えます。 データを二度取得したい場合は、別のグループ名を持つテーブルのコピーを作成します。 -グループは柔軟で、クラスター上で同期されます。 たとえば、クラスター内のテーブルの10のトピックと5つのコピーがある場合、各コピーは2つのトピックを取得します。 コピー数が変更されると、トピックは自動的にコピー全体に再配布されます。 これについての詳細を読むhttp://kafka.apache.org/intro。 +グループは柔軟で、クラスター上で同期されます。 例えば10テーマの5冊のテーブルにクラスターでは、そのコピーを取得し2ます。 コピー数が変更されると、トピックは自動的にコピー間で再配布されます。 もっと読むことでhttp://kafka.apache.org/intro. `SELECT` は特に役立つメッセージを読む(以外のデバッグ)では、それぞれのメッセージでしか読み込むことができます。 ではより実践的な創出の実時間スレッドを実現します。 これを行うには: -1. エンジンを使用してkafkaコンシューマーを作成し、データストリームとみなします。 +1. エンジンを使用してKafkaコンシューマーを作成し、データストリームとみなします。 2. 目的の構造を持つテーブルを作成します。 -3. を実現しュに変換するデータからのエンジンを入れていたとして作成されます。 +3. エンジンからデータを変換し、以前に作成したテーブルに格納するマテリアライズドビューを作成します。 -とき `MATERIALIZED VIEW` 入、エンジンでデータを収集しみいただけます。 これにより、Kafkaからメッセージを継続的に受信し、必要な形式に変換することができます `SELECT`. -このようにして、異なる詳細レベル（grouping-aggregation and without）を持つ複数のテーブルに書き込むことができます。 +ときに `MATERIALIZED VIEW` 入、エンジンでデータを収集しみいただけます。 これにより、kafkaからのメッセージを継続的に受信し、必要な形式に変換することができます `SELECT`. +一つのカフカテーブルは、あなたが好きなだけ多くのマテリアライズドビューを持つことができ、彼らは直接カフカテーブルからデータを読み取るのではな -例えば: +例: ``` sql CREATE TABLE queue ( @@ -130,7 +134,7 @@ Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format SELECT level, sum(total) FROM daily GROUP BY level; ``` -受信したメッ [max\_insert\_block\_size](../../../operations/server-configuration-parameters/settings.md#settings-max_insert_block_size). ブロックが内に形成されなかった場合 [stream\_flush\_interval\_ms](../../../operations/server-configuration-parameters/settings.md) ミリ秒では、データはブロックの完全性に関係なくテーブルにフラッシュされます。 +パフォーマンスを向上させるために、受信したメッセージは [max\_insert\_block\_size](../../../operations/server-configuration-parameters/settings.md#settings-max_insert_block_size). ブロックが内に形成されていない場合 [stream\_flush\_interval\_ms](../../../operations/server-configuration-parameters/settings.md) ミリ秒、データは関係なく、ブロックの完全性のテーブルにフラッシュされます。 リクエストを受けた話題のデータは変更に変換ロジック、切り離しを実現ビュー: @@ -139,11 +143,11 @@ Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format ATTACH TABLE consumer; ``` -ターゲットテーブルを次のように変更する場合 `ALTER` お勧めいたしま字の材質ビューを避ける異なるターゲットテーブルのデータからの眺め。 +を使用してターゲットテーブルを変更する場合 `ALTER` マテリアルビューを無効にすると、ターゲットテーブルとビューのデータとの間の不一致を回避できます。 ## 設定 {#configuration} -GraphiteMergeTreeと同様に、KafkaエンジンはClickHouse configファイルを使用した拡張構成をサポートしています。 使用できる設定キーは二つあります:global (`kafka`）とトピックレベル (`kafka_*`). グローバル構成が最初に適用され、トピックレベル構成が適用されます(存在する場合)。 +GraphiteMergeTreeと同様に、KafkaエンジンはClickHouse設定ファイルを使用した拡張構成をサポートします。 使用できる設定キーは次の二つです。 (`kafka`)とトピックレベル (`kafka_*`). 最初にグローバル構成が適用され、次にトピックレベルの構成が適用されます(存在する場合)。 ``` xml @@ -159,7 +163,7 @@ GraphiteMergeTreeと同様に、KafkaエンジンはClickHouse configファイ ``` -可能な構成オプションの一覧については、 [librdkafka設定リファレンス](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md). アンダースコアを使う (`_`)ClickHouse構成のドットの代わりに。 例えば, `check.crcs=true` されます `true`. +可能な構成オプションのリストについては、 [librdkafka設定リファレンス](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md). アンダースコアを使用する (`_`）ClickHouse設定のドットの代わりに。 例えば, `check.crcs=true` になります `true`. ## 仮想列 {#virtual-columns} @@ -169,7 +173,7 @@ GraphiteMergeTreeと同様に、KafkaエンジンはClickHouse configファイ - `_timestamp` — Timestamp of the message. - `_partition` — Partition of Kafka topic. -**また見なさい** +**も参照。** - [仮想列](../index.md#table_engines-virtual_columns) diff --git a/docs/ja/engines/table-engines/integrations/mysql.md b/docs/ja/engines/table-engines/integrations/mysql.md index fd6351b27a2..6b92c8a36ce 100644 --- a/docs/ja/engines/table-engines/integrations/mysql.md +++ b/docs/ja/engines/table-engines/integrations/mysql.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 33 toc_title: MySQL --- # Mysql {#mysql} -MySQLエンジンでは、次の操作を実行できます `SELECT` リモートMySQLサーバーに格納されているデータを照会します。 +MySQLエンジンでは、次の操作を実行できます `SELECT` リモートMySQLサーバーに格納されているデータに対するクエリ。 ## テーブルの作成 {#creating-a-table} @@ -22,10 +22,10 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] の詳細な説明を参照してください [CREATE TABLE](../../../sql-reference/statements/create.md#create-table-query) クエリ。 -テーブル構造は元のmysqlテーブル構造とは異なる場合があります: +テーブル構造は、元のMySQLテーブル構造と異なる場合があります: -- 列名は元のmysqlテーブルと同じでなければなりませんが、これらの列の一部だけを任意の順序で使用できます。 -- カラムの型は、元のmysqlテーブルの型と異なる場合があります。 クリックハウスは [キャスト](../../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) クリックハウスのデータ型への値。 +- カラム名は元のMySQLテーブルと同じでなければなりませんが、これらのカラムの一部だけを任意の順序で使用できます。 +- 列の型は、元のMySQLテーブルの型とは異なる場合があります。 ClickHouseは [キャスト](../../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) ClickHouseデータ型の値。 **エンジン変数** @@ -39,15 +39,15 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] - `password` — User password. -- `replace_query` — Flag that converts `INSERT INTO` へのクエリ `REPLACE INTO`. もし `replace_query=1` クエリは置換されます。 +- `replace_query` — Flag that converts `INSERT INTO` へのクエリ `REPLACE INTO`. もし `replace_query=1`、クエリが代入されます。 - `on_duplicate_clause` — The `ON DUPLICATE KEY on_duplicate_clause` に追加される式 `INSERT` クエリ。 - 例えば: `INSERT INTO t (c1,c2) VALUES ('a', 2) ON DUPLICATE KEY UPDATE c2 = c2 + 1`、どこ `on_duplicate_clause` は `UPDATE c2 = c2 + 1`. を見る [MySQLの文書](https://dev.mysql.com/doc/refman/8.0/en/insert-on-duplicate.html) これを見つけるには `on_duplicate_clause` あなたはで使用することができ `ON DUPLICATE KEY` 句。 + 例: `INSERT INTO t (c1,c2) VALUES ('a', 2) ON DUPLICATE KEY UPDATE c2 = c2 + 1`,ここで `on_duplicate_clause` は `UPDATE c2 = c2 + 1`. を参照。 [MySQLドキュメント](https://dev.mysql.com/doc/refman/8.0/en/insert-on-duplicate.html) これを見つけるには `on_duplicate_clause` と使用できます `ON DUPLICATE KEY` 句。 - 指定するには `on_duplicate_clause` 合格する必要があります `0` に `replace_query` パラメータ。 あなたが同時に渡す場合 `replace_query = 1` と `on_duplicate_clause`、ClickHouseは例外を生成します。 + 指定するには `on_duplicate_clause` 合格する必要があります `0` に `replace_query` パラメータ。 あなたが同時に合格した場合 `replace_query = 1` と `on_duplicate_clause`,ClickHouseは例外を生成します。 -シンプル `WHERE` 次のような句 `=, !=, >, >=, <, <=` MySQLサーバで実行されます。 +シンプル `WHERE` 次のような句 `=, !=, >, >=, <, <=` MySQLサーバー上で実行されます。 残りの条件と `LIMIT` サンプリング制約は、MySQLへのクエリが終了した後にのみClickHouseで実行されます。 @@ -76,7 +76,7 @@ mysql> select * from test; 1 row in set (0,00 sec) ``` -ClickHouseのテーブル、上記で作成したMySQLテーブルからデータを取得する: +ClickHouseのテーブル、上で作成したMySQLテーブルからデータを取得する: ``` sql CREATE TABLE mysql_table @@ -97,9 +97,9 @@ SELECT * FROM mysql_table └────────────────┴────────┘ ``` -## また見なさい {#see-also} +## も参照。 {#see-also} -- [その ‘mysql’ テーブル機能](../../../sql-reference/table-functions/mysql.md) -- [MySQLを外部辞書のソースとして使用する](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md#dicts-external_dicts_dict_sources-mysql) +- [その ‘mysql’ テーブル関数](../../../sql-reference/table-functions/mysql.md) +- [外部辞書のソースとしてMySQLを使用する](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md#dicts-external_dicts_dict_sources-mysql) [元の記事](https://clickhouse.tech/docs/en/operations/table_engines/mysql/) diff --git a/docs/ja/engines/table-engines/integrations/odbc.md b/docs/ja/engines/table-engines/integrations/odbc.md index 691316daf0f..78709e65e09 100644 --- a/docs/ja/engines/table-engines/integrations/odbc.md +++ b/docs/ja/engines/table-engines/integrations/odbc.md @@ -1,17 +1,17 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 35 toc_title: ODBC --- # ODBC {#table-engine-odbc} -ClickHouseが外部データベースに接続できるようにします [ODBC](https://en.wikipedia.org/wiki/Open_Database_Connectivity). +ClickHouseが外部データベースに接続できるようにする [ODBC](https://en.wikipedia.org/wiki/Open_Database_Connectivity). -ODBC接続を安全に実装するには、ClickHouseは別のプログラムを使用します `clickhouse-odbc-bridge`. ODBCドライバーが直接読み込まれている場合 `clickhouse-server` ドライバの問題でクラッシュのClickHouseサーバーです。 クリックハウスが自動的に起動 `clickhouse-odbc-bridge` それが必要なとき。 ODBCブリッジプログラムは、次のパッケージと同じパッケー `clickhouse-server`. +ODBC接続を安全に実装するために、ClickHouseは別のプログラムを使用します `clickhouse-odbc-bridge`. ODBCドライバーが直接ロードされる場合 `clickhouse-server` ドライバの問題でクラッシュのClickHouseサーバーです。 ClickHouseは自動的に起動します `clickhouse-odbc-bridge` それが必要なとき。 ODBC bridgeプログラムは、 `clickhouse-server`. -このエンジンは、 [Nullable](../../../sql-reference/data-types/nullable.md) データ型。 +このエンジンは [Null可能](../../../sql-reference/data-types/nullable.md) データ型。 ## テーブルの作成 {#creating-a-table} @@ -27,26 +27,26 @@ ENGINE = ODBC(connection_settings, external_database, external_table) の詳細な説明を参照してください [CREATE TABLE](../../../sql-reference/statements/create.md#create-table-query) クエリ。 -のテーブル構造が異なるソースからテーブル構造: +表構造は、ソース表構造とは異なる場合があります: - 列名はソーステーブルと同じにする必要がありますが、これらの列の一部だけを任意の順序で使用できます。 -- 列の型は、ソーステーブルの型と異なる場合があります。 クリックハウスは [キャスト](../../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) クリックハウスのデータ型への値。 +- 列の型は、ソーステーブルの型と異なる場合があります。 ClickHouseは [キャスト](../../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) ClickHouseデータ型の値。 **エンジン変数** -- `connection_settings` — Name of the section with connection settings in the `odbc.ini` ファイル。 +- `connection_settings` — Name of the section with connection settings in the `odbc.ini` ファイル - `external_database` — Name of a database in an external DBMS. - `external_table` — Name of a table in the `external_database`. ## 使用例 {#usage-example} -**ODBC経由でローカルMySQLインストールからデータを取得** +**取得データから地元のMySQLのインストール目盛** -この例は、ubuntu linux18.04およびmysql server5.7で確認されています。 +この例では、Ubuntu Linux18.04およびMySQL server5.7がチェックされています。 UnixODBCとMySQL Connectorがインストールされていることを確認します。 -デフォルトでインストールされた場合、パッケージから),clickhouse開始してユーザー `clickhouse`. したがって、MySQLサーバでこのユーザを作成して設定する必要があります。 +デフォルトでインストールされた場合、パッケージから),ClickHouse開始してユーザー `clickhouse`. したがって、MySQLサーバーでこのユーザーを作成して構成する必要があります。 ``` bash $ sudo mysql @@ -70,7 +70,7 @@ USERNAME = clickhouse PASSWORD = clickhouse ``` -を使用して接続を確認することができ `isql` unixODBCインストールからのユーティリティ。 +接続を確認するには `isql` unixODBCの取付けからの実用性。 ``` bash $ isql -v mysqlconn @@ -124,7 +124,7 @@ SELECT * FROM odbc_t └────────┴────────────────┘ ``` -## また見なさい {#see-also} +## も参照。 {#see-also} - [ODBC外部辞書](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md#dicts-external_dicts_dict_sources-odbc) - [ODBCテーブル関数](../../../sql-reference/table-functions/odbc.md) diff --git a/docs/ja/engines/table-engines/log-family/index.md b/docs/ja/engines/table-engines/log-family/index.md index 330dac5b3af..40f23ead1c7 100644 --- a/docs/ja/engines/table-engines/log-family/index.md +++ b/docs/ja/engines/table-engines/log-family/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_folder_title: Log Family +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "\u30ED\u30B0\u30D5\u30A1\u30DF\u30EA" toc_priority: 29 --- diff --git a/docs/ja/engines/table-engines/log-family/log-family.md b/docs/ja/engines/table-engines/log-family/log-family.md index e3abeda83ee..2606e6ad771 100644 --- a/docs/ja/engines/table-engines/log-family/log-family.md +++ b/docs/ja/engines/table-engines/log-family/log-family.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 31 -toc_title: "\u5C0E\u5165" +toc_title: "\u306F\u3058\u3081\u306B" --- -# 丸太エンジン家族 {#log-engine-family} +# ログエンジン家族 {#log-engine-family} -これらのエンジンは、多くの小さなテーブル（最大約1万行）をすばやく作成し、後で全体として読む必要があるシナリオ用に開発されました。 +これらのエンジンは、多くの小さなテーブル（最大1万行）をすばやく書き込み、後で全体として読み込む必要があるシナリオ用に開発されました。 家族のエンジン: @@ -15,32 +15,32 @@ toc_title: "\u5C0E\u5165" - [ログ](log.md) - [TinyLog](tinylog.md) -## 一般的なプロパティ {#common-properties} +## 共通プロパティ {#common-properties} エンジン: -- ディスク上のデータを格納します。 +- ディスクにデータを格納します。 - 書き込み時にファイルの末尾にデータを追加します。 - 同時データアクセスのサポートロック。 - の間 `INSERT` クエリのテーブルがロックされ、その他の質問を読み込みおよび書き込みデータの両方のテーブルを作成する データ書き込みクエリがない場合は、任意の数のデータ読み取りクエリを同時に実行できます。 + 中 `INSERT` クエリのテーブルがロックされ、その他の質問を読み込みおよび書き込みデータの両方のテーブルを作成する データ書き込みクエリがない場合は、任意の数のデータ読み込みクエリを同時に実行できます。 -- ないサポート [突然変異](../../../sql-reference/statements/alter.md#alter-mutations) オペレーション +- サポートしない [突然変異](../../../sql-reference/statements/alter.md#alter-mutations) 作戦だ -- 索引をサポートしない。 +- 索引をサポートしません。 - これは、 `SELECT` データ範囲のクエリは効率的ではありません。 + つまり `SELECT` データ範囲のクエリは効率的ではありません。 -- データを原子的に書き込まない。 +- 書くわけではありませんデータを原子的に. 取得できるテーブルデータが破損した場合も破れ、書き込み操作は、例えば、異常サーバをシャットダウンしました。 ## 違い {#differences} -その `TinyLog` エンジンは家族の最も簡単で、最も貧しい機能性および最も低い効率を提供する。 その `TinyLog` エンジンをサポートしていない並列データの読み取りによる複数のスレッド）。 それは、並列読み取りをサポートするファミリ内の他のエンジンよりも遅いデータを読み取り、 `Log` エンジンは、各列を別々のファイルに格納するためです。 シンプルな低負荷シナリオで使用します。 +その `TinyLog` エンジンは家族の最も簡単、最も悪い機能性および最も低い効率を提供する。 その `TinyLog` エンジンをサポートしていない並列データの読み取りによる複数のスレッド）。 でデータを読み込む代わりに、各エンジンの家族を支援する並列読みでの使用がほとんど同じになりました記述子としての `Log` エンジンは、各列を別々のファイルに格納するためです。 単純な低負荷のシナリオで使用します。 -その `Log` と `StripeLog` エンジンは平行データ読書を支える。 デー 各スレ その `Log` エンジンは、テーブルの各列に別々のファイルを使用します。 `StripeLog` すべてのデータファイルです。 その結果、 `StripeLog` エンジン用の少ない記述子に、経営システムが、 `Log` エンジンはデータを読むとき高性能を提供する。 +その `Log` と `StripeLog` エンジンの支援並列データです。 読み込み時にデータClickHouse使複数のスレッド）。 各スレッドプロセス別データブロックです。 その `Log` エンジンは、テーブルの各列に個別のファイルを使用します。 `StripeLog` すべてのデータファイルです。 その結果、 `StripeLog` エンジンは、オペレーティングシス `Log` エンジンはデータを読むとき高性能を提供する。 [元の記事](https://clickhouse.tech/docs/en/operations/table_engines/log_family/) diff --git a/docs/ja/engines/table-engines/log-family/log.md b/docs/ja/engines/table-engines/log-family/log.md index c5ad09f0bf3..412b92c5551 100644 --- a/docs/ja/engines/table-engines/log-family/log.md +++ b/docs/ja/engines/table-engines/log-family/log.md @@ -1,16 +1,16 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 33 toc_title: "\u30ED\u30B0" --- # ログ {#log} -エンジンは、ログエンジンの家族に属します。 ログエンジ [丸太エンジン家族](log-family.md) 記事。 +エンジンはログエンジンの系列に属します。 ログエンジンの共通のプロパティとその違いを参照してください [ログエンジン家族](log-family.md) 記事だ -ログとは異なります [TinyLog](tinylog.md) その中での小さなファイル “marks” 列ファイルに常駐します。 これらのマークはすべてのデータブロックに書き込まれ、指定された行数をスキップするためにファイルの読み取りを開始する場所を示すオフセッ この読み取りを可能にする機能がありテーブルデータを複数のスレッド）。 -同時データアクセスの場合、読み取り操作は同時に実行でき、書き込み操作は読み取りをブロックします。 -ログのエンジンなスを作成します。 同様に、執筆する場合はテーブルに失敗したり、テーブルに破からの読み出しでエラーを返します。 ログのエンジンは適切な一時データ書き込み回のテーブルの試験-デモンストレーション。 +ログとは異なります [TinyLog](tinylog.md) その中の小さなファイルの “marks” 列ファイルに存在します。 これらのマークはすべてのデータブロックに書き込まれ、指定された行数をスキップするためにファイルの読み取りを開始する場所を示すオフセット この読み取りを可能にする機能がありテーブルデータを複数のスレッド）。 +同時データアクセスの場合、読み取り操作は同時に実行できますが、書き込み操作は読み取りをブロックします。 +ログエンジ 同様に、テーブルへの書き込みが失敗した場合、テーブルは壊れ、そこから読み込むとエラーが返されます。 ログエンジンは、一時データ、書き込み一度テーブル、およびテストまたはデモの目的に適しています。 [元の記事](https://clickhouse.tech/docs/en/operations/table_engines/log/) diff --git a/docs/ja/engines/table-engines/log-family/stripelog.md b/docs/ja/engines/table-engines/log-family/stripelog.md index ff39a5179b0..d157368824e 100644 --- a/docs/ja/engines/table-engines/log-family/stripelog.md +++ b/docs/ja/engines/table-engines/log-family/stripelog.md @@ -1,15 +1,15 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 32 toc_title: "\u30B9\u30C8\u30EA\u30C3\u30D7\u30ED\u30B0" --- # ストリップログ {#stripelog} -このエン ログエンジ [丸太エンジン家族](log-family.md) 記事。 +このエンジンはログエンジンの系列に属します。 ログエンジンの共通のプロパティとその違いを参照してください [ログエンジン家族](log-family.md) 記事だ -少量のデータ（1万行未満）を含む多数のテーブルを作成する必要がある場合は、このエンジンをシナリオで使用します。 +少量のデータ(1万行未満)で多数のテーブルを記述する必要がある場合に、このエンジンを使用します。 ## テーブルの作成 {#table_engines-stripelog-creating-a-table} @@ -22,22 +22,22 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] ) ENGINE = StripeLog ``` -の詳細な説明を参照してください [CREATE TABLE](../../../sql-reference/statements/create.md#create-table-query) クエリ。 +の詳細な説明を見て下さい [CREATE TABLE](../../../sql-reference/statements/create.md#create-table-query) クエリ。 ## データの書き込み {#table_engines-stripelog-writing-the-data} -その `StripeLog` エンジンの店舗のすべての列を一つのファイルです。 それぞれの `INSERT` クエリ、ClickHouseは、列を一つずつ書き込み、テーブルファイルの最後にデータブロックを追加します。 +その `StripeLog` engineはすべての列を一つのファイルに格納します。 それぞれのため `INSERT` query,ClickHouseは、データブロックをテーブルファイルの最後に追加し、列を一つずつ書き込みます。 -各テーブルclickhouseに書き込み中のファイル: +各テーブルClickHouseに書き込み中のファイル: - `data.bin` — Data file. - `index.mrk` — File with marks. Marks contain offsets for each column of each data block inserted. -その `StripeLog` エンジンはサポートしません `ALTER UPDATE` と `ALTER DELETE` オペレーション +その `StripeLog` エンジンはサポートしません `ALTER UPDATE` と `ALTER DELETE` 作戦だ -## データの読み込み {#table_engines-stripelog-reading-the-data} +## データの読み取り {#table_engines-stripelog-reading-the-data} -ファイルをマークでclickhouseを並列化したデータです。 これは、 `SELECT` クエリは、予期しない順序で行を返します。 を使用 `ORDER BY` 行をソートする句。 +ファイルをマークでClickHouseを並列化したデータです。 これは、 `SELECT` queryは、予測不可能な順序で行を返します。 使用する `ORDER BY` 行をソートする句。 ## 使用例 {#table_engines-stripelog-example-of-use} @@ -60,9 +60,9 @@ INSERT INTO stripe_log_table VALUES (now(),'REGULAR','The first regular message' INSERT INTO stripe_log_table VALUES (now(),'REGULAR','The second regular message'),(now(),'WARNING','The first warning message') ``` -私たちは二つの `INSERT` データブロックを作成するためのクエリ `data.bin` ファイル。 +我々は二つを使用 `INSERT` データブロックを作成するためのクエリ `data.bin` ファイル -ClickHouse利用は、複数のスレッド選択時のデータです。 各スレッドを読み込み、別のデータブロックを返しますよ列として自立で終了します。 結果として、出力の行のブロックの順序は、ほとんどの場合、入力の同じブロックの順序と一致しません。 例えば: +ClickHouse利用は、複数のスレッド選択時のデータです。 各スレッドは、個別のデータブロックを読み取り、終了時に結果の行を個別に返します その結果、出力内の行のブロックの順序は、ほとんどの場合、入力内の同じブロックの順序と一致しません。 例えば: ``` sql SELECT * FROM stripe_log_table diff --git a/docs/ja/engines/table-engines/log-family/tinylog.md b/docs/ja/engines/table-engines/log-family/tinylog.md index 2c9f7a2a338..3cfb05b5597 100644 --- a/docs/ja/engines/table-engines/log-family/tinylog.md +++ b/docs/ja/engines/table-engines/log-family/tinylog.md @@ -1,16 +1,16 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 34 toc_title: TinyLog --- # TinyLog {#tinylog} -エンジンはログエンジンファミリに属します。 見る [丸太エンジン家族](log-family.md) ログエンジンとその違いの一般的な特性のために。 +エンジンはログエンジンファミリに属します。 見る [ログエンジン家族](log-family.md) ログエンジンの共通プロパティとその違い。 -このテーブルエンジンは、通常、write-onceメソッドで使用されます。 たとえば、次のものを使用できます `TinyLog`-小さなバッチで処理される中間データのテーブルを入力します。 多数の小さなテーブルにデータを格納することは非効率的です。 +このテーブルエンジンは、通常、write-onceメソッドで使用されます。 たとえば、次のようにします `TinyLog`-小さなバッチで処理される中間データのテーブルを入力します。 多数の小さなテーブルにデータを格納するのは非効率的です。 -クエリは単一のストリームで実行されます。 言い換えれば、このエンジンは比較的小さなテーブル（約1,000,000行まで）を対象としています。 小さなテーブルがたくさんある場合は、このテーブルエンジンを使用するのが理にかなっています。 [ログ](log.md) エンジン（少数のファイルは開く必要があります）。 +クエリは単一のストリームで実行されます。 言い換えれば、このエンジンは比較的小さなテーブル（最大約1,000,000行）を対象としています。 小さなテーブルが多い場合は、このテーブルエンジンを使用するのが理にかなっています。 [ログ](log.md) エンジン(開く必要の少ないファイル)。 [元の記事](https://clickhouse.tech/docs/en/operations/table_engines/tinylog/) diff --git a/docs/ja/engines/table-engines/mergetree-family/aggregatingmergetree.md b/docs/ja/engines/table-engines/mergetree-family/aggregatingmergetree.md index 4d597526584..d304ec2802d 100644 --- a/docs/ja/engines/table-engines/mergetree-family/aggregatingmergetree.md +++ b/docs/ja/engines/table-engines/mergetree-family/aggregatingmergetree.md @@ -1,20 +1,22 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 35 -toc_title: "\uFF82\u3064\uFF68\uFF82\u59EA\"\uFF82\u3064\"\uFF82\u50B5\uFF82\u3065\ - \uFF6D\uFF82\u3064\uFF79" +toc_title: AggregatingMergeTree --- -# ﾂつｨﾂ姪“ﾂつ”ﾂ債ﾂづｭﾂつｹ {#aggregatingmergetree} +# Aggregatingmergetree {#aggregatingmergetree} -エンジンは [MergeTree](mergetree.md#table_engines-mergetree)、データパーツのマージのロジックを変更する。 ClickHouseは、すべての行を同じ主キー（またはより正確には同じ）で置き換えます [ソートキー](mergetree.md)）集計関数の状態の組み合わせを格納する単一の行（一つのデータ部門内）。 +エンジンはから継承します [メルゲツリー](mergetree.md#table_engines-mergetree)、データ部分のマージのロジックを変更する。 ClickHouseは、すべての行を同じ主キー（またはより正確には同じキー）で置き換えます [ソートキー](mergetree.md)）集計関数の状態の組み合わせを格納する単一の行（一つのデータ部分内）を持つ。 -を使用することができ `AggregatingMergeTree` テーブルが増えた場合のデータ収集、集計を実現します。 +以下を使用できます `AggregatingMergeTree` 集計されたマテリアライズドビューを含む、増分データ集計用の表。 -エンジンプロセスの全てのカラム [AggregateFunction](../../../sql-reference/data-types/aggregatefunction.md) タイプ。 +エンジンは、次の型のすべての列を処理します: -使用するのが適切です `AggregatingMergeTree` 注文によって行数が減る場合。 +- [AggregateFunction](../../../sql-reference/data-types/aggregatefunction.md) +- [SimpleAggregateFunction](../../../sql-reference/data-types/simpleaggregatefunction.md) + +使用することは適切です `AggregatingMergeTree` 注文によって行数を減らす場合。 ## テーブルの作成 {#creating-a-table} @@ -36,11 +38,11 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] **クエリ句** -作成するとき `AggregatingMergeTree` テーブル同じ [句](mergetree.md) 作成するときと同じように、必須です。 `MergeTree` テーブル。 +を作成するとき `AggregatingMergeTree` 同じテーブル [句](mergetree.md) を作成するときのように必要です。 `MergeTree` テーブル。
-テーブルを作成する非推奨の方法 +推奨されていません法テーブルを作成する !!! attention "注意" 可能であれば、古いプロジェクトを上記の方法に切り替えてください。 @@ -54,19 +56,19 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] ) ENGINE [=] AggregatingMergeTree(date-column [, sampling_expression], (primary, key), index_granularity) ``` -すべてのパラメーターの意味は、次のようになります `MergeTree`. +すべてのパラメータは、inと同じ意味を持ちます `MergeTree`.
-## 選択して挿入 {#select-and-insert} +## 選択と挿入 {#select-and-insert} -データを挿入するには [INSERT SELECT](../../../sql-reference/statements/insert-into.md) aggregate-State-functionsを使用したクエリ。 -データを選択するとき `AggregatingMergeTree` テーブル、使用 `GROUP BY` 句とデータを挿入するときと同じ集約関数が、 `-Merge` 接尾辞。 +データを挿入するには、 [INSERT SELECT](../../../sql-reference/statements/insert-into.md) aggregate-State-functionsを使用したクエリ。 +データを選択するとき `AggregatingMergeTree` テーブル、使用 `GROUP BY` データを挿入するときと同じ集計関数ですが、 `-Merge` 接尾辞。 -の結果で `SELECT` クエリ、値の `AggregateFunction` タイプは、すべてのClickHouse出力形式に対して実装固有のバイナリ表現を持ちます。 たとえば、データをダンプする場合, `TabSeparated` フォーマット `SELECT` このダンプは、次のようにロードされます `INSERT` クエリ。 +の結果 `SELECT` クエリ、の値 `AggregateFunction` typeは、すべてのClickHouse出力形式に対して実装固有のバイナリ表現を持ちます。 たとえば、データをダンプする場合, `TabSeparated` フォーマット `SELECT` 次に、このダンプを次のようにロードします `INSERT` クエリ。 ## 集約マテリアライズドビューの例 {#example-of-an-aggregated-materialized-view} -`AggregatingMergeTree` マテリアライズドビュー `test.visits` テーブル: +`AggregatingMergeTree` これは、 `test.visits` テーブル: ``` sql CREATE MATERIALIZED VIEW test.basic @@ -80,13 +82,13 @@ FROM test.visits GROUP BY CounterID, StartDate; ``` -データを挿入する `test.visits` テーブル。 +にデータを挿入する `test.visits` テーブル。 ``` sql INSERT INTO test.visits ... ``` -データは、テーブルとビューの両方に挿入されます `test.basic` それは集約を実行します。 +データはテーブルとビューの両方に挿入されます `test.basic` それは集計を実行します。 集計データを取得するには、次のようなクエリを実行する必要があります `SELECT ... GROUP BY ...` ビューから `test.basic`: diff --git a/docs/ja/engines/table-engines/mergetree-family/collapsingmergetree.md b/docs/ja/engines/table-engines/mergetree-family/collapsingmergetree.md index 487ea73a289..e26172c1fd5 100644 --- a/docs/ja/engines/table-engines/mergetree-family/collapsingmergetree.md +++ b/docs/ja/engines/table-engines/mergetree-family/collapsingmergetree.md @@ -1,15 +1,15 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 36 -toc_title: CollapsingMergeTree +toc_title: "\u6298\u308A\u305F\u305F\u307F\u30DE\u30FC\u30B8\u30C4\u30EA\u30FC" --- -# Collapsingmergetree {#table_engine-collapsingmergetree} +# 折りたたみマージツリー {#table_engine-collapsingmergetree} -エンジンは [MergeTree](mergetree.md) 加算の論理行の崩壊データ部品の統合アルゴリズムです。 +エンジンはから継承します [メルゲツリー](mergetree.md) 加算の論理行の崩壊データ部品の統合アルゴリズムです。 -`CollapsingMergeTree` 並べ替えキー内のすべてのフィールドの場合、行のペアを非同期的に削除(折りたたみ)します (`ORDER BY`)は、特定のフィールドを除いて同等です `Sign` これは `1` と `-1` 値。 ペアのない行は保持されます。 詳細については、 [折りたたみ](#table_engine-collapsingmergetree-collapsing) 文書のセクション。 +`CollapsingMergeTree` 並べ替えキー内のすべてのフィールドの場合、行のペアを非同期に削除(折りたたみ)します (`ORDER BY`）は、特定のフィールドを除いて同等です `Sign` これは `1` と `-1` 値。 ペアのない行は保持されます。 詳細については、 [崩壊](#table_engine-collapsingmergetree-collapsing) 文書のセクション。 エンジンはかなり貯蔵の容積を減らし、効率をの高めるかもしれません `SELECT` 結果としてのクエリ。 @@ -28,21 +28,21 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] [SETTINGS name=value, ...] ``` -説明のクエリパラメータは、 [クエリの説明](../../../sql-reference/statements/create.md). +クエリパラメータの説明については、 [クエリの説明](../../../sql-reference/statements/create.md). **CollapsingMergeTreeパラメータ** -- `sign` — Name of the column with the type of row: `1` は “state” 行, `-1` は “cancel” 行 +- `sign` — Name of the column with the type of row: `1` は “state” 行, `-1` は “cancel” ロウ Column data type — `Int8`. **クエリ句** -作成するとき `CollapsingMergeTree` テーブル、同じ [クエリ句](mergetree.md#table_engine-mergetree-creating-a-table) 作成するときと同じように、必須です。 `MergeTree` テーブル。 +を作成するとき `CollapsingMergeTree` テーブル、同じ [クエリ句](mergetree.md#table_engine-mergetree-creating-a-table) を作成するときのように必要です。 `MergeTree` テーブル。
-テーブルを作成する非推奨の方法 +推奨されていません法テーブルを作成する !!! attention "注意" 可能であれば、古いプロジェクトを上記の方法に切り替えてください。 @@ -56,23 +56,23 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] ) ENGINE [=] CollapsingMergeTree(date-column [, sampling_expression], (primary, key), index_granularity, sign) ``` -すべてのパラメーターを除く `sign` と同じ意味を持つ `MergeTree`. +以下を除くすべてのパラメータ `sign` と同じ意味を持つ `MergeTree`. -- `sign` — Name of the column with the type of row: `1` — “state” 行, `-1` — “cancel” 行 +- `sign` — Name of the column with the type of row: `1` — “state” 行, `-1` — “cancel” ロウ Column Data Type — `Int8`.
-## 折りたたみ {#table_engine-collapsingmergetree-collapsing} +## 崩壊 {#table_engine-collapsingmergetree-collapsing} ### データ {#data} -あるオブジェクトのデータを継続的に変更する必要がある状況を考えてみましょう。 これは、オブジェクトのための一つの行を持っており、任意の変更でそれを更新する論理的に聞こえるが、更新操作は、ストレージ内のデータの書き換え が必要な場合にデータを書き込むには、迅速に更新できませんが、きの変化をオブジェクトの順にしております。 +考える必要がある状況などが保存継続的に変化するデータのオブジェクトです。 オブジェクトの行を一つ持ち、変更時に更新するのは論理的ですが、DBMSではストレージ内のデータを書き換える必要があるため、更新操作はコストがかか が必要な場合にデータを書き込むには、迅速に更新できませんが、きの変化をオブジェクトの順にしております。 -特定の列を使用する `Sign`. もし `Sign = 1` これは、行がオブジェクトの状態であることを意味します。 “state” 行 もし `Sign = -1` これは、同じ属性を持つオブジェクトの状態の取り消しを意味し、それを呼び出しましょう “cancel” 行 +特定の列を使用する `Sign`. もし `Sign = 1` これは、行がオブジェクトの状態であることを意味します。 “state” ロウ もし `Sign = -1` これは、同じ属性を持つオブジェクトの状態の取り消しを意味します。 “cancel” ロウ -例えば、しい計算のページのユーザーかサイトは、長くご愛用いただけると思いがあります。 ある時点で、ユーザーアクティビティの状態で次の行を書きます: +たとえば、ユーザーがあるサイトでチェックしたページ数と、そこにいた時間を計算したいとします。 ある時点で、ユーザーアクティビティの状態で次の行を書きます: ``` text ┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ @@ -80,7 +80,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] └─────────────────────┴───────────┴──────────┴──────┘ ``` -しばらくして、ユーザーアクティビティの変更を登録し、次の二つの行を書き込みます。 +ある時点で、後で我々は、ユーザーの活動の変更を登録し、次の二つの行でそれを書きます。 ``` text ┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ @@ -89,9 +89,9 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] └─────────────────────┴───────────┴──────────┴──────┘ ``` -最初の行は、オブジェクト（user）の前の状態を取り消します。 取り消された状態の並べ替えキーフィールドをコピーします `Sign`. +最初の行は、オブジェクト(ユーザー)の以前の状態を取り消します。 キャンセルされた状態の並べ替えキーフィールドをコピーします。 `Sign`. -次の行には、現在の状態が含まれています。 +次の行には現在の状態が含まれます。 ユーザーアクティビティの最後の状態だけが必要なので、行 @@ -102,42 +102,39 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] └─────────────────────┴───────────┴──────────┴──────┘ ``` -オブジェクトの無効な（古い）状態を崩壊させることを削除することができます。 `CollapsingMergeTree` これは、データ部分のマージ中に行います。 +オブジェクトの無効な（古い）状態を折りたたんで削除することができます。 `CollapsingMergeTree` データパーツのマージ中にこれを行います。 -なぜ私たちは必要2行の各変更のために読む [Algorithm](#table_engine-collapsingmergetree-collapsing-algorithm) 段落。 +なぜ私たちは、読み込まれた各変更のための2行が必要です [アルゴリズ](#table_engine-collapsingmergetree-collapsing-algorithm) 段落。 -**そのようなアプローチ** +**このようなアプローチ** 1. プログラムを書き込み、データ意のオブジェクトをキャンセルはできます。 “Cancel” 文字列はコピーのソートキーの分野において “state” 文字列とその逆 `Sign`. この増加の初期サイズでの保存が可能なデータを書き込む。 -2. 列の長い配列は、書き込みの負荷によるエンジンの効率を低下させます。 より簡単なデータ、より高い効率。 -3. その `SELECT` 結果は、オブジェクトの変更履歴の整合性に強く依存します。 挿入するデータを準備するときは正確です。 たとえば、セッションの深さなどの負でない指標の負の値などです。 +2. 列の長い成長配列は、書き込みの負荷によるエンジンの効率を低下させます。 より簡単なデータ、より高い効率。 +3. その `SELECT` 結果は、オブジェクト変更履歴の一貫性に強く依存します。 挿入のためのデータを準備するときは正確です。 たとえば、セッションの深さなど、負でない指標の負の値など、不整合なデータで予測不可能な結果を得ることができます。 -### Algorithm {#table_engine-collapsingmergetree-collapsing-algorithm} +### アルゴリズ {#table_engine-collapsingmergetree-collapsing-algorithm} -ClickHouseがデータパーツをマージすると、同じソートキーを持つ連続した行の各グループ (`ORDER BY` これは、以下の二つの行に縮小されています。 `Sign = 1` (“state” 行）と別の `Sign = -1` (“cancel” 行）。 言い換えれば、エントリは崩壊する。 +ClickHouseがデータパーツをマージすると、同じ並べ替えキーを持つ連続した行の各グループ (`ORDER BY`）は、以下の二つの行に縮小される。 `Sign = 1` (“state” 行）と別の `Sign = -1` (“cancel” 行）。 つまり、エントリは崩壊します。 -それぞれの結果のデータ部分clickhouse保存: +各り、その結果得られたデータは部分ClickHouse省: -1. 最初の “cancel” そして最後の “state” 行の数が “state” と “cancel” 行は一致し、最後の行はaです “state” 行 +1. 最初の “cancel” そして最後の “state” 行の数が “state” と “cancel” 行は一致し、最後の行は “state” ロウ +2. 最後の “state” より多くがあれば、行 “state” 行より “cancel” 行。 +3. 最初の “cancel” より多くがあれば、行 “cancel” 行より “state” 行。 +4. 他のすべての場合には、行のいずれも。 -2. 最後の “state” 行、より多くのがある場合 “state” 行よりも “cancel” 行。 +また、少なくとも2以上がある場合 “state” 行より “cancel” 行、または少なくとも2以上 “cancel” 次の行 “state” マージは続行されますが、ClickHouseはこの状況を論理エラーとして扱い、サーバーログに記録します。 同じデータが複数回挿入された場合、このエラーが発生する可能性があります。 -3. 最初の “cancel” 行、より多くのがある場合 “cancel” 行よりも “state” 行。 - -4. 他のすべてのケースでは、行はありません。 - -また、少なくとも2以上がある場合 “state” 行よりも “cancel” 行、または少なくとも2以上 “cancel” その後の行 “state” ただし、ClickHouseはこの状況を論理エラーとして扱い、サーバーログに記録します。 このエラーは、同じデータが複数回挿入された場合に発生します。 - -したがって、崩壊は統計の計算結果を変更すべきではありません。 +したがって、崩壊は統計の計算結果を変えてはならない。 変更は徐々に崩壊し、最終的にはほぼすべてのオブジェクトの最後の状態だけが残った。 -その `Sign` マージアルゴリズムは、同じソートキーを持つすべての行が同じ結果のデータ部分にあり、同じ物理サーバー上にあることを保証するものではないため、必須で ClickHouse過程 `SELECT` 複数のスレッドを持つクエリは、結果の行の順序を予測することはできません。 完全に取得する必要がある場合は、集計が必要です “collapsed” からのデータ `CollapsingMergeTree` テーブル。 +その `Sign` マージアルゴリズムでは、同じ並べ替えキーを持つすべての行が同じ結果データ部分にあり、同じ物理サーバー上にあることを保証するものではないため、必 ClickHouseプロセス `SELECT` 複数のスレッドでクエリを実行し、結果の行の順序を予測することはできません。 完全に取得する必要がある場合は、集計が必要です “collapsed” データから `CollapsingMergeTree` テーブル。 -折りたたみを完了するには、次のクエリを記述します `GROUP BY` この符号を考慮する句および集計関数。 たとえば、数量を計算するには、以下を使用します `sum(Sign)` 代わりに `count()`. 何かの合計を計算するには、次のようにします `sum(Sign * x)` 代わりに `sum(x)`、など、とも追加 `HAVING sum(Sign) > 0`. +折りたたみを終了するには、次のクエリを記述します `GROUP BY` 符号を説明する句および集計関数。 たとえば、数量を計算するには、次を使用します `sum(Sign)` 代わりに `count()`. 何かの合計を計算するには、次のようにします `sum(Sign * x)` 代わりに `sum(x)`、というように、また、追加 `HAVING sum(Sign) > 0`. -凝集体 `count`, `sum` と `avg` この方法で計算できます。 合計 `uniq` 算出できる場合にはオブジェクトは、少なくとも一つの状態はまだ崩れていない。 凝集体 `min` と `max` 以下の理由で計算できませんでした `CollapsingMergeTree` 折りたたまれた状態の値の履歴は保存されません。 +集計 `count`, `sum` と `avg` この方法で計算できます。 集計 `uniq` 算出できる場合にはオブジェクトは、少なくとも一つの状態はまだ崩れていない。 集計 `min` と `max` 計算できませんでした。 `CollapsingMergeTree` 折りたたまれた状態の値の履歴は保存されません。 -集計せずにデータを抽出する必要がある場合(たとえば、最新の値が特定の条件に一致する行が存在するかどうかをチェックする場合)は、次のように `FINAL` のための修飾語 `FROM` 句。 このアプローチは大幅に少ない効率的です。 +集計なしでデータを抽出する必要がある場合(たとえば、特定の条件に一致する最新の値を持つ行が存在するかどうかをチェックする場合)には、次の `FINAL` モディファイア `FROM` 句。 このアプローチは、大幅に低効率です。 ## 使用例 {#example-of-use} @@ -175,7 +172,7 @@ INSERT INTO UAct VALUES (4324182021466249494, 5, 146, 1) INSERT INTO UAct VALUES (4324182021466249494, 5, 146, -1),(4324182021466249494, 6, 185, 1) ``` -私たちは二つを使う `INSERT` 二つの異なるデータ部分を作成するクエリ。 また、データの挿入につクエリClickHouseを一つのデータ部分を行いませんが合併します。 +我々は二つを使用します `INSERT` 二つの異なるデータ部分を作成するクエリ。 一つのクエリでデータを挿入すると、ClickHouseは一つのデータ部分を作成し、マージを実行しません。 データの取得: @@ -195,9 +192,9 @@ SELECT * FROM UAct 私たちは何を見て、どこで崩壊していますか？ -二つの `INSERT` クエリーを作成し、2つのデータ部品です。 その `SELECT` クエリは2つのスレッドで実行され、ランダムな行の順序が得られました。 データ部分のマージがまだなかったため、折りたたみは発生しません。 ClickHouseは予測できない未知の瞬間にデータ部分をマージします。 +二つと `INSERT` クエリーを作成し、2つのデータ部品です。 その `SELECT` クエリは2つのスレッドで実行され、行のランダムな順序が得られました。 データパーツのマージがまだないため、折りたたみは発生しませんでした。 ClickHouseは予測できない未知の瞬間にデータ部分をマージします。 -このようにして集計: +したがって、集約が必要です: ``` sql SELECT @@ -215,7 +212,7 @@ HAVING sum(Sign) > 0 └─────────────────────┴───────────┴──────────┘ ``` -集約を必要とせず、強制的に崩壊させたい場合は、以下を使用できます `FINAL` の修飾子 `FROM` 句。 +集約を必要とせず、強制的に崩壊させたい場合は、以下を使用できます `FINAL` 修飾子のための `FROM` 句。 ``` sql SELECT * FROM UAct FINAL @@ -227,7 +224,7 @@ SELECT * FROM UAct FINAL └─────────────────────┴───────────┴──────────┴──────┘ ``` -データを選択するこの方法は非常に非効率的です。 大きなテーブルには使用しないでください。 +データを選択するこの方法は非常に非効率的です。 大きなテーブルには使わないでください。 ## 別のアプローチの例 {#example-of-another-approach} @@ -241,7 +238,7 @@ SELECT * FROM UAct FINAL └─────────────────────┴───────────┴──────────┴──────┘ ``` -アイデアは、マージが唯一のキーフィールドを考慮することです。 そして、 “Cancel” 行符号列を使用せずに合計すると、行の以前のバージョンを等しくする負の値を指定できます。 この方法では、データ型を変更する必要があります `PageViews`,`Duration` UInt8-\>Int16の負の値を格納します。 +アイデアは、マージが考慮にキーフィールドのみを取ることです。 そして、 “Cancel” 行我々は、符号列を使用せずに合計するときに行の以前のバージョンを等しく負の値を指定することができます。 この方法では、データ型を変更する必要があります `PageViews`,`Duration` uint8-\>Int16の負の値を格納する。 ``` sql CREATE TABLE UAct @@ -255,7 +252,7 @@ ENGINE = CollapsingMergeTree(Sign) ORDER BY UserID ``` -アプローチをテストしよう: +ましょう試験のアプローチ: ``` sql insert into UAct values(4324182021466249494, 5, 146, 1); diff --git a/docs/ja/engines/table-engines/mergetree-family/custom-partitioning-key.md b/docs/ja/engines/table-engines/mergetree-family/custom-partitioning-key.md index a4815370dc6..f1058c21c03 100644 --- a/docs/ja/engines/table-engines/mergetree-family/custom-partitioning-key.md +++ b/docs/ja/engines/table-engines/mergetree-family/custom-partitioning-key.md @@ -1,17 +1,17 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 32 toc_title: "\u30AB\u30B9\u30BF\u30E0\u5206\u5272\u30AD\u30FC" --- # カスタム分割キー {#custom-partitioning-key} -パーティション分割は [MergeTree](mergetree.md) ファミリーテーブル [複製された](replication.md) テーブル）。 [マテリアライズ表示](../special/materializedview.md#materializedview) に基づくMergeTreeテーブル支援を分割します。 +パーティション分割は、 [メルゲツリー](mergetree.md) 家族テーブル（含む [複製](replication.md) テーブル）。 [実体化ビュー](../special/materializedview.md#materializedview) に基づくMergeTreeテーブル支援を分割します。 -パーティションが論理的に組み合わせの記録テーブルに指定された評価のポイントになります。 パーティションは、月別、日別、またはイベントタイプ別など、任意の基準によって設定できます。 各パーティションは別に保存される簡単操作のデータです。 アクセス時のデータclickhouseの最小サブセットのパーティションは可能です。 +パーティションは、指定された条件によるテーブル内のレコードの論理的な組合せです。 パーティションは、月別、日別、イベントタイプ別など、任意の条件で設定できます。 各パーティションは別に保存される簡単操作のデータです。 アクセス時のデータClickHouseの最小サブセットのパーティションは可能です。 -パーティションは `PARTITION BY expr` 節とき [テーブルの作成](mergetree.md#table_engine-mergetree-creating-a-table). これはパーティションキーにすることはでき表現からのテーブル列あります。 例えば、指定ョ月の表現を使用 `toYYYYMM(date_column)`: +パーティションは `PARTITION BY expr` 句とき [テーブルの作成](mergetree.md#table_engine-mergetree-creating-a-table). これはパーティションキーにすることはでき表現からのテーブル列あります。 例えば、指定ョ月の表現を使用 `toYYYYMM(date_column)`: ``` sql CREATE TABLE visits @@ -25,7 +25,7 @@ PARTITION BY toYYYYMM(VisitDate) ORDER BY Hour; ``` -これはパーティションキーもできるタプルの表現を [主キー](mergetree.md#primary-keys-and-indexes-in-queries)). 例えば: +パーティションキーは、式のタプルにすることもできます。 [主キー](mergetree.md#primary-keys-and-indexes-in-queries)). 例えば: ``` sql ENGINE = ReplicatedCollapsingMergeTree('/clickhouse/tables/name', 'replica1', Sign) @@ -33,14 +33,14 @@ PARTITION BY (toMonday(StartDate), EventType) ORDER BY (CounterID, StartDate, intHash32(UserID)); ``` -この例では、現在の週に発生したイベントタイプによるパーティション分割を設定します。 +この例では、現在の週に発生したイベントの種類によってパーティション分割を設定します。 -テーブルに新しいデータを挿入すると、このデータは主キーでソートされた別の部分（チャンク）として格納されます。 挿入後10-15分で、同じパーティションのパーツがパート全体にマージされます。 +挿入する際に新しいデータテーブルにこのデータを保存することがで別パーツとして（個）-field-list順にソートその有効なタイプを利用します。 挿入後10-15分で、同じパーティションの部分が部分全体にマージされます。 !!! info "情報" - Mergeは、パーティショニング式と同じ値を持つデータパーツに対してのみ機能します。 これは **なんかを過度に粒状仕切り** （約千パーティション以上）。 それ以外の場合は、 `SELECT` クエリは、ファイルシステムとオープンファイル記述子のファイルの不当に大きな数のために不十分な実行します。 + マージは、パーティション分割式の値が同じデータパーツに対してのみ機能します。 つまり **なんかを過度に粒状仕切り** （千約以上のパーティション）。 それ以外の場合は、 `SELECT` ファイルシステムおよびオープンファイル記述子に不当に多数のファイルがあるため、クエリの実行が不十分です。 -を使用 [システム。パーツ](../../../operations/system-tables.md#system_tables-parts) テーブルのテーブル部品およびパーテッション. たとえば、我々が持っていると仮定しましょう `visits` テーブルを分割する。 のは、実行してみましょう `SELECT` のための問い合わせ `system.parts` テーブル: +使用する [システム部品](../../../operations/system-tables.md#system_tables-parts) 表パーツとパーティションを表示する表。 たとえば、のは、我々が持っていると仮定しましょう `visits` テーブルを分割する。 のは、実行してみましょう `SELECT` のクエリ `system.parts` テーブル: ``` sql SELECT @@ -63,23 +63,23 @@ WHERE table = 'visits' └───────────┴────────────────┴────────┘ ``` -その `partition` 列にはパーティションの名前が含まれます。 この例には二つの区画があります: `201901` と `201902`. この列の値を使用して、パーティション名を指定できます。 [ALTER … PARTITION](#alter_manipulations-with-partitions) クエリ。 +その `partition` 列にはパーティションの名前が含まれます。 あるパーティション例: `201901` と `201902`. この列の値を使用して、パーティション名を指定できます [ALTER … PARTITION](#alter_manipulations-with-partitions) クエリ。 -その `name` カラムの名前を格納して、パーティションのデータ部品です。 この列を使用して、パートの名前を指定することができます。 [ALTER ATTACH PART](#alter_attach-partition) クエリ。 +その `name` カラムの名前を格納して、パーティションのデータ部品です。 この列を使用して、パーツの名前を指定することができます。 [ALTER ATTACH PART](#alter_attach-partition) クエリ。 -最初の部分の名前を分解してみましょう: `201901_1_3_1`: +最初の部分の名前を分解しましょう: `201901_1_3_1`: - `201901` パーティション名です。 - `1` データブロックの最小数です。 - `3` データブロックの最大数です。 -- `1` チャンクレベル(マージツリーの深さ)です。 +- `1` チャンクレベル(形成されるマージツリーの深さ)です。 !!! info "情報" - 古いタイプのテーブルの部分には名前があります: `20190117_20190123_2_2_0` （最小日付-最大日付-最小ブロック番号-最大ブロック番号-レベル）。 + 古いタイプのテーブルの部分には名前があります: `20190117_20190123_2_2_0` (最小日-最大日-最小ブロック番号-最大ブロック番号-レベル)。 -その `active` コラムは部品の状態を示します。 `1` アクティブです; `0` 非アクティブです。 に不活性部品、例えば、ソース部品の残りの後の合併によります。 破損したデータ部分も非アクティブとして示されます。 +その `active` 列は部品の状態を示します。 `1` アクティブです; `0` 非アクティブです。 非アクティブな部分は、たとえば、より大きな部分にマージした後に残るソース部分です。 破損したデータ部分も非アクティブとして示されます。 -この例でわかるように、同じパーティションのいくつかの分離された部分があります（たとえば, `201901_1_3_1` と `201901_1_9_2`). つまり、これらの部分はまだマージされていません。 ClickHouseは、挿入してから約15分後に、データの挿入された部分を定期的にマージします。 さらに、スケジュールされていないマージを実行するには [OPTIMIZE](../../../sql-reference/statements/misc.md#misc_operations-optimize) クエリ。 例えば: +この例でわかるように、同じパーティションにはいくつかの分離された部分があります（たとえば, `201901_1_3_1` と `201901_1_9_2`). つまり、これらの部分はまだマージされていません。 ClickHouseは、データの挿入された部分を定期的にマージし、挿入の約15分後にマージします。 また、スケジュールされていないマージを実行するには [OPTIMIZE](../../../sql-reference/statements/misc.md#misc_operations-optimize) クエリ。 例: ``` sql OPTIMIZE TABLE visits PARTITION 201902; @@ -98,9 +98,9 @@ OPTIMIZE TABLE visits PARTITION 201902; └───────────┴────────────────┴────────┘ ``` -不活性パーツを削除する約10分後の統合. +非アクティブな部分は、マージ後約10分で削除されます。 -部品やパーティションのセットを表示する別の方法は、テーブルのディレクトリに移動することです: `/var/lib/clickhouse/data///`. 例えば: +パーツとパーティションのセットを表示する別の方法は、テーブルのディレクトリに移動します: `/var/lib/clickhouse/data//
/`. 例えば: ``` bash /var/lib/clickhouse/data/default/visits$ ls -l @@ -116,12 +116,12 @@ drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 5 12:09 201902_4_6_1 drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 1 16:48 detached ``` -フォルダ ‘201901\_1\_1\_0’, ‘201901\_1\_7\_1’ というように部品のディレクトリです。 各部に関する対応する分割データが含まれまで一定の月のテーブルこの例では、分割による。 +フォルダ ‘201901\_1\_1\_0’, ‘201901\_1\_7\_1’ そして、部品のディレクトリです。 各部に関する対応する分割データが含まれまで一定の月のテーブルこの例では、分割による。 -その `detached` ディレクト [DETACH](../../../sql-reference/statements/alter.md#alter_detach-partition) クエリ。 破損した部分も削除されるのではなく、このディレクトリに移動されます。 サーバーはサーバーからの部品を使用しません `detached` directory. You can add, delete, or modify the data in this directory at any time – the server will not know about this until you run the [ATTACH](../../../sql-reference/statements/alter.md#alter_attach-partition) クエリ。 +その `detached` ディレクトリに含まれる部品のこともあったかを使って、テーブル [DETACH](../../../sql-reference/statements/alter.md#alter_detach-partition) クエリ。 破損した部分も、削除されるのではなく、このディレクトリに移動されます。 サーバーは、サーバーからの部品を使用しません。 `detached` directory. You can add, delete, or modify the data in this directory at any time – the server will not know about this until you run the [ATTACH](../../../sql-reference/statements/alter.md#alter_attach-partition) クエリ。 -オペレーティングサーバーでは、ファイルシステム上の部品またはそのデータのセットを手動で変更することはできません。 非複製のテーブル、これを実行する事ができます。サーバが停止中でないお勧めします。 レプリケートされたテーブルの場合、パートのセットは変更できません。 +オペレーティングサーバーでは、ファイルシステム上の部品のセットまたはそのデータを手動で変更することはできません。 非複製のテーブル、これを実行する事ができます。サーバが停止中でないお勧めします。 のための複製のテーブルはパーツのセットの変更はできません。 -ClickHouseを使用すると、パーティションを削除したり、テーブル間でコピーしたり、バックアップを作成したりできます。 セクションのすべての操作の一覧を参照してください [パーティションとパーツの操作](../../../sql-reference/statements/alter.md#alter_manipulations-with-partitions). +ClickHouseでは、パーティションの削除、テーブル間のコピー、またはバックアップの作成などの操作を実行できます。 セクションのすべての操作の一覧を参照してください [パーティションとパーツの操作](../../../sql-reference/statements/alter.md#alter_manipulations-with-partitions). [元の記事](https://clickhouse.tech/docs/en/operations/table_engines/custom_partitioning_key/) diff --git a/docs/ja/engines/table-engines/mergetree-family/graphitemergetree.md b/docs/ja/engines/table-engines/mergetree-family/graphitemergetree.md index 35b000fe1b3..4e7522f5f48 100644 --- a/docs/ja/engines/table-engines/mergetree-family/graphitemergetree.md +++ b/docs/ja/engines/table-engines/mergetree-family/graphitemergetree.md @@ -1,18 +1,17 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 38 -toc_title: "\u30B0\u30E9\u30D5\u30A3\u30C3\u30C8\u30E1\u30FC\u30EB\u30B0\u30C4\u30EA\ - \u30FC" +toc_title: GraphiteMergeTree --- -# グラフィットメールグツリー {#graphitemergetree} +# GraphiteMergeTree {#graphitemergetree} -このエン) [黒鉛](http://graphite.readthedocs.io/en/latest/index.html) データ。 GraphiteのデータストアとしてClickHouseを使用したい開発者にとっては役に立つかもしれません。 +このエンジ) [黒鉛](http://graphite.readthedocs.io/en/latest/index.html) データ これは、GraphiteのデータストアとしてClickHouseを使用したい開発者にとって役立つかもしれません。 -ロールアップが必要ない場合は、任意のclickhouseテーブルエンジンを使用してグラファイトデータを保存できますが、ロールアップが必要な場合は使用します `GraphiteMergeTree`. エンジンはストレージの量を減らし、Graphiteからのクエリの効率を高めます。 +を利用できますClickHouseテーブルエンジンの黒鉛のデータが必要ない場rollupが必要な場合は、rollupを使用 `GraphiteMergeTree`. エンジンは貯蔵量を減らし、グラファイトからの照会の効率を高めます。 -エンジンを継承性から [MergeTree](mergetree.md). +エンジンはプロパティを [メルゲツリー](mergetree.md). ## テーブルの作成 {#creating-table} @@ -33,17 +32,17 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] の詳細な説明を参照してください [CREATE TABLE](../../../sql-reference/statements/create.md#create-table-query) クエリ。 -グラファイトデータのテーブルには、次のデータの列が必要です: +グラファイトデータのテーブルには、次のデータの次の列が必要です: -- メトリック名（黒鉛センサ）。 データ型: `String`. +- ミリ規格名（グラファイトセンサ） データ型: `String`. - メトリックを測定する時間。 データ型: `DateTime`. - メトリックの値。 データ型:任意の数値。 -- メトリックのバージョン。 データ型:任意の数値。 +- 指標のバージョン。 データ型:任意の数値。 - ClickHouseは、バージョンが同じであれば、最高のバージョンまたは最後に書かれた行を保存します。 その他の行は、データパーツのマージ中に削除されます。 + ClickHouseは、バージョンが同じ場合は、最も高いバージョンまたは最後に書かれた行を保存します。 他の行は、データパーツのマージ中に削除されます。 これらの列の名前は、ロールアップ構成で設定する必要があります。 @@ -53,11 +52,11 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] **クエリ句** -作成するとき `GraphiteMergeTree` テーブル、同じ [句](mergetree.md#table_engine-mergetree-creating-a-table) 作成するときと同じように、必須です。 `MergeTree` テーブル。 +を作成するとき `GraphiteMergeTree` テーブル、同じ [句](mergetree.md#table_engine-mergetree-creating-a-table) を作成するときのように必要です。 `MergeTree` テーブル。
-テーブルを作成する非推奨の方法 +推奨されていません法テーブルを作成する !!! attention "注意" 可能であれば、古いプロジェクトを上記の方法に切り替えてください。 @@ -74,7 +73,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] ) ENGINE [=] GraphiteMergeTree(date-column [, sampling_expression], (primary, key), index_granularity, config_section) ``` -すべてのパラメーターを除く `config_section` と同じ意味を持つ `MergeTree`. +以下を除くすべてのパラメータ `config_section` と同じ意味を持つ `MergeTree`. - `config_section` — Name of the section in the configuration file, where are the rules of rollup set. @@ -82,7 +81,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] ## ロールアップ構成 {#rollup-configuration} -ロールアップの設定は、次のように定義されます。 [graphite\_rollup](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-graphite) サーバー構成のパラメーター。 パラメータの名前は任意です。 複数の構成を作成し、それらを異なるテーブルに使用できます。 +ロールアップの設定は、 [graphite\_rollup](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-graphite) サーバー構成のパラメータ。 パラメーターの名前は任意です。 複数の構成を作成し、異なるテーブルに使用できます。 ロールアップ構成構造: @@ -122,22 +121,22 @@ default ``` !!! warning "注意" - パタ: + パターンは厳密に注文する必要が: 1. Patterns without `function` or `retention`. 1. Patterns with both `function` and `retention`. 1. Pattern `default`. -行を処理するときに、clickhouseは次のルールをチェックします。 `pattern` セクション。 それぞれの `pattern` （を含む `default`)セクションには `function` 集計のパラメータ, `retention` 変数または両方。 このメトリック名が `regexp`、からのルール `pattern` セクション(またはセクション)が適用されます。 `default` セクションを使用します。 +行を処理するとき、ClickHouseは `pattern` セクション それぞれの `pattern` （含む `default`)セクションには `function` 集計のパラメータ, `retention` 変数または両方。 メトリック名が `regexp`、からのルール `pattern` セクション(またはセクション)が適用されます。 `default` セクションを使用します。 -フィールドの `pattern` と `default` セクション: +のフィールド `pattern` と `default` セクション: - `regexp`– A pattern for the metric name. - `age` – The minimum age of the data in seconds. - `precision`– How precisely to define the age of the data in seconds. Should be a divisor for 86400 (seconds in a day). - `function` – The name of the aggregating function to apply to data whose age falls within the range `[age, age + precision]`. -### 構成例 {#configuration-example} +### 設定例 {#configuration-example} ``` xml diff --git a/docs/ja/engines/table-engines/mergetree-family/index.md b/docs/ja/engines/table-engines/mergetree-family/index.md index b807da4f929..4218d000f8b 100644 --- a/docs/ja/engines/table-engines/mergetree-family/index.md +++ b/docs/ja/engines/table-engines/mergetree-family/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_folder_title: MergeTree Family +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "\u30E1\u30EB\u30B2\u30C4\u30EA\u30FC\u65CF" toc_priority: 28 --- diff --git a/docs/ja/engines/table-engines/mergetree-family/mergetree.md b/docs/ja/engines/table-engines/mergetree-family/mergetree.md index d2415ad184b..55538136a57 100644 --- a/docs/ja/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/ja/engines/table-engines/mergetree-family/mergetree.md @@ -1,36 +1,36 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 30 -toc_title: MergeTree +toc_title: "\u30E1\u30EB\u30B2\u30C4\u30EA\u30FC" --- -# Mergetree {#table_engines-mergetree} +# メルゲツリー {#table_engines-mergetree} -その `MergeTree` この家族 (`*MergeTree`）最も堅牢なクリックハウステーブルエンジンです。 +その `MergeTree` この家族のエンジンそして他のエンジン (`*MergeTree`）は、最も堅牢なClickHouseテーブルエンジンです。 -のエンジン `MergeTree` ファミリは、非常に大量のデータをテーブルに挿入するために設計されています。 のデータが書き込まれ、テーブル部、そのルール適用のための統合のパーツです。 この方法は、挿入時にストレージ内のデータを継続的に書き換えるよりはるかに効率的です。 +のエンジン `MergeTree` ファミリは、非常に大量のデータをテーブルに挿入するために設計されています。 のデータが書き込まれ、テーブル部、そのルール適用のための統合のパーツです。 この方法は効率よく継続的に書き換えのデータストレージ中をサポートしていません。 主な特長: -- 店舗データを整理によりその有効なタイプを利用します。 +- 主キ - これにより、データの検索を高速化する小さなスパース索引を作成できます。 + これを作成できる小型疎指標を見つけるデータを高速に行います。 -- パーティションは、 [分割キー](custom-partitioning-key.md) が指定される。 +- この場合、 [分割キー](custom-partitioning-key.md) が指定される。 ClickHouseは、同じ結果を持つ同じデータに対する一般的な操作よりも効果的なパーティションを持つ特定の操作をサポートします。 ClickHouseも自動的に遮断すると、パーティションデータのパーティショニングキーで指定されたクエリ。 この改善するためのクエリ。 -- データ複製サポート。 +- データ複製のサポート。 - の家族 `ReplicatedMergeTree` 表はデータ複製を提供します。 詳細については、 [データ複製](replication.md). + の家族 `ReplicatedMergeTree` テーブルを提供データレプリケーション. 詳細については、 [データ複製](replication.md). -- データ抜取りサポート。 +- データサンプリングです。 - 必要に応じて、テーブル内のデータサンプリング方法を設定できます。 + 必要に応じて、テーブル内でデータサンプリング方法を設定できます。 !!! info "情報" - その [マージ](../special/merge.md#merge) エンジンはに属しません `*MergeTree` 家族 + その [マージ](../special/merge.md#merge) エンジンはに属しません `*MergeTree` 家族だ ## テーブルの作成 {#table_engine-mergetree-creating-a-table} @@ -51,52 +51,52 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] [SETTINGS name=value, ...] ``` -パラメータの詳細については、 [クエリの説明の作成](../../../sql-reference/statements/create.md). +パラメータの説明については、 [クエリの説明の作成](../../../sql-reference/statements/create.md). -!!! note "メモ" - `INDEX` は、実験的な機能です [データスキップ索引](#table_engine-mergetree-data_skipping-indexes). +!!! note "注" + `INDEX` 実験的な機能です。 [データスキップ索引](#table_engine-mergetree-data_skipping-indexes). ### クエリ句 {#mergetree-query-clauses} -- `ENGINE` — Name and parameters of the engine. `ENGINE = MergeTree()`. その `MergeTree` engineにはパラメータがありません。 +- `ENGINE` — Name and parameters of the engine. `ENGINE = MergeTree()`. その `MergeTree` エンジンにはパラメータがない。 - `PARTITION BY` — The [分割キー](custom-partitioning-key.md). - 月単位でパーティション分割するには、 `toYYYYMM(date_column)` 式、どこ `date_column` 型の日付を持つ列を指定します [日付](../../../sql-reference/data-types/date.md). ここでのパーティション名には、 `"YYYYMM"` フォーマット。 + 月によるパーティション分割の場合は、 `toYYYYMM(date_column)` 式、ここで `date_column` 型の日付を持つ列です [日付](../../../sql-reference/data-types/date.md). ここでのパーティション名は `"YYYYMM"` 形式。 - `ORDER BY` — The sorting key. - 列または任意の式のタプル。 例えば: `ORDER BY (CounterID, EventDate)`. + 列または任意の式のタプル。 例: `ORDER BY (CounterID, EventDate)`. - `PRIMARY KEY` — The primary key if it [ソートキーとは異なります](#choosing-a-primary-key-that-differs-from-the-sorting-key). - デフォルトでは、プライマリキーはソートキー(プライマリキーで指定)と同じです。 `ORDER BY` 句）。 したがって、ほとんどの場合、別の `PRIMARY KEY` 句。 + デフォルトでは、プライマリキーはソートキーと同じです。 `ORDER BY` 節）。 したがって、ほとんどの場合、別の `PRIMARY KEY` 句。 - `SAMPLE BY` — An expression for sampling. - サンプリング式を使用する場合は、主キーに含める必要があります。 例えば: `SAMPLE BY intHash32(UserID) ORDER BY (CounterID, EventDate, intHash32(UserID))`. + サンプリング式を使用する場合は、主キーにそれを含める必要があります。 例: `SAMPLE BY intHash32(UserID) ORDER BY (CounterID, EventDate, intHash32(UserID))`. - `TTL` — A list of rules specifying storage duration of rows and defining logic of automatic parts movement [ディスクとボリューム間](#table_engine-mergetree-multiple-volumes). - 式には次のものが必要です `Date` または `DateTime` 結果としての列。 例えば: + 式には次のものが必要です `Date` または `DateTime` 結果としての列。 例: `TTL date + INTERVAL 1 DAY` - ルールのタイプ `DELETE|TO DISK 'xxx'|TO VOLUME 'xxx'` 式が満たされている場合(現在の時間に達した場合)、その部分を使用して実行するアクションを指定します。 (`TO DISK 'xxx'`)またはボリュームに (`TO VOLUME 'xxx'`). ルールのデフォルトの種類は削除です (`DELETE`). 複数のルールのリストは指定できますが、複数のルールが存在しないはずです `DELETE` ルール。 + ルールのタイプ `DELETE|TO DISK 'xxx'|TO VOLUME 'xxx'` 式が満たされた場合(現在の時刻に達した場合)、パートに対して実行されるアクションを指定します:期限切れの行の削除、パートの移動(パート内のすべての (`TO DISK 'xxx'`）またはボリュームに (`TO VOLUME 'xxx'`). ルールの既定の種類は削除です (`DELETE`). リストに複数のルール指定がありませんよ `DELETE` ルール - 詳細については、 [列とテーブルのttl](#table_engine-mergetree-ttl) + 詳細は、を参照してください [列および表のTTL](#table_engine-mergetree-ttl) - `SETTINGS` — Additional parameters that control the behavior of the `MergeTree`: - - `index_granularity` — Maximum number of data rows between the marks of an index. Default value: 8192. See [データ記憶](#mergetree-data-storage). - - `index_granularity_bytes` — Maximum size of data granules in bytes. Default value: 10Mb. To restrict the granule size only by number of rows, set to 0 (not recommended). See [データ記憶](#mergetree-data-storage). - - `enable_mixed_granularity_parts` — Enables or disables transitioning to control the granule size with the `index_granularity_bytes` 設定。 バージョン19.11以前は、 `index_granularity` 制限の微粒のサイズのための設定。 その `index_granularity_bytes` 設定の改善ClickHouse性能の選定からデータをテーブルの大きな行(数十、数百人のメガバイト). 大きな行を持つテーブルがある場合は、この設定を有効にしてテーブルの効率を向上させることができます `SELECT` クエリ。 - - `use_minimalistic_part_header_in_zookeeper` — Storage method of the data parts headers in ZooKeeper. If `use_minimalistic_part_header_in_zookeeper=1`、その後ZooKeeperは以下のデータを格納します。 詳細については、 [設定の説明](../../../operations/server-configuration-parameters/settings.md#server-settings-use_minimalistic_part_header_in_zookeeper) で “Server configuration parameters”. - - `min_merge_bytes_to_use_direct_io` — The minimum data volume for merge operation that is required for using direct I/O access to the storage disk. When merging data parts, ClickHouse calculates the total storage volume of all the data to be merged. If the volume exceeds `min_merge_bytes_to_use_direct_io` バイトClickHouseを読み込みおよび書き込み、データの保存ディスクの直接のI/Oインターフェース (`O_DIRECT` オプション）。 もし `min_merge_bytes_to_use_direct_io = 0` その後、直接I/Oが無効になります。 デフォルト値: `10 * 1024 * 1024 * 1024` バイト。 + - `index_granularity` — Maximum number of data rows between the marks of an index. Default value: 8192. See [データ保存](#mergetree-data-storage). + - `index_granularity_bytes` — Maximum size of data granules in bytes. Default value: 10Mb. To restrict the granule size only by number of rows, set to 0 (not recommended). See [データ保存](#mergetree-data-storage). + - `enable_mixed_granularity_parts` — Enables or disables transitioning to control the granule size with the `index_granularity_bytes` 設定。 バージョン19.11以前は `index_granularity` 制限の微粒のサイズのための設定。 その `index_granularity_bytes` 設定の改善ClickHouse性能の選定からデータをテーブルの大きな行(数十、数百人のメガバイト). いテーブルの大きな行きのこの設定のテーブルの効率化 `SELECT` クエリ。 + - `use_minimalistic_part_header_in_zookeeper` — Storage method of the data parts headers in ZooKeeper. If `use_minimalistic_part_header_in_zookeeper=1` その飼育係の店が少ない。 詳細については、を参照してください [設定の説明](../../../operations/server-configuration-parameters/settings.md#server-settings-use_minimalistic_part_header_in_zookeeper) で “Server configuration parameters”. + - `min_merge_bytes_to_use_direct_io` — The minimum data volume for merge operation that is required for using direct I/O access to the storage disk. When merging data parts, ClickHouse calculates the total storage volume of all the data to be merged. If the volume exceeds `min_merge_bytes_to_use_direct_io` バイト、ClickHouseは直接入出力インターフェイスを使用して記憶域ディスクにデータを読み、書きます (`O_DIRECT` オプション）。 もし `min_merge_bytes_to_use_direct_io = 0` その後、直接I/Oが無効になります。 デフォルト値: `10 * 1024 * 1024 * 1024` バイト数。 - `merge_with_ttl_timeout` — Minimum delay in seconds before repeating a merge with TTL. Default value: 86400 (1 day). - - `write_final_mark` — Enables or disables writing the final index mark at the end of data part (after the last byte). Default value: 1. Don’t turn it off. + - `write_final_mark` — Enables or disables writing the final index mark at the end of data part (after the last byte). Default value: 1. Don't turn it off. - `merge_max_block_size` — Maximum number of rows in block for merge operations. Default value: 8192. - - `storage_policy` — Storage policy. See [複数ブロックデバイスを使用したデータ保存](#table_engine-mergetree-multiple-volumes). + - `storage_policy` — Storage policy. See [複数のブロックデバイスのためのデータ保存](#table_engine-mergetree-multiple-volumes). **セクション設定例** @@ -104,15 +104,15 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] ENGINE MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity=8192 ``` -この例では、月単位でパーティション分割を設定します。 +この例では、月別にパーティション分割を設定します。 -また、ユーザーidによるハッシュとしてサンプリング用の式を設定します。 これにより、それぞれのテーブルのデータを擬似乱数化することができます `CounterID` と `EventDate`. を定義した場合 [SAMPLE](../../../sql-reference/statements/select.md#select-sample-clause) 句データを選択すると、ClickHouseはユーザーのサブセットに対して均等に擬似乱数データサンプルを返します。 +また、ユーザIDによってハッシュとしてサンプリングする式を設定します。 これにより、各テーブルのデータを擬似乱数化することができます `CounterID` と `EventDate`. を定義すると [SAMPLE](../../../sql-reference/statements/select/sample.md#select-sample-clause) 句データを選択すると、ClickHouseはユーザーのサブセットに対して均等な擬似乱数データサンプルを返します。 -その `index_granularity` 8192がデフォルト値であるため、設定は省略できます。 +その `index_granularity` 8192がデフォルト値であるため、設定を省略できます。
-テーブルを作成する非推奨の方法 +推奨されていません法テーブルを作成する !!! attention "注意" 用途では使用しないでください方法で新規プロジェクト. 可能であれば、古いプロジェクトを上記の方法に切り替えます。 @@ -128,35 +128,35 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] **MergeTree()パラメータ** -- `date-column` — The name of a column of the [日付](../../../sql-reference/data-types/date.md) タイプ。 ClickHouseを自動でパーティション月に基づきます。 パーティション名は `"YYYYMM"` フォーマット。 +- `date-column` — The name of a column of the [日付](../../../sql-reference/data-types/date.md) タイプ。 ClickHouseを自動でパーティション月に基づきます。 パーティション名は `"YYYYMM"` 形式。 - `sampling_expression` — An expression for sampling. - `(primary, key)` — Primary key. Type: [タプル()](../../../sql-reference/data-types/tuple.md) - `index_granularity` — The granularity of an index. The number of data rows between the “marks” インデックスの。 値8192は、ほとんどのタスクに適しています。 -**例えば** +**例** ``` sql MergeTree(EventDate, intHash32(UserID), (CounterID, EventDate, intHash32(UserID)), 8192) ``` -その `MergeTree` エンジンは、メインエンジン構成方法について上記の例と同様に構成される。 +その `MergeTree` エンジンは、メインエンジンの構成方法については、上記の例と同様に構成されています。
-## データ記憶 {#mergetree-data-storage} +## データ保存 {#mergetree-data-storage} テーブルのデータ部品の分別によりその有効なタイプを利用します。 -データがテーブルに挿入されると、別々のデータパーツが作成され、それぞれが主キーで辞書式に並べ替えられます。 たとえば、プライマリキーが次の場合 `(CounterID, Date)` パーツ内のデータは `CounterID`、およびそれぞれの中で `CounterID`、それは順序付けられます `Date`. +データがテーブルに挿入されると、個別のデータパーツが作成され、それぞれが主キーで辞書順に並べ替えられます。 たとえば、主キーが `(CounterID, Date)` パーツ内のデータは次のようにソートされます `CounterID`、およびそれぞれの中 `CounterID`、それは順序付けられます `Date`. -データに属する別のパーティションが分離の異なる部品です。 その背景にclickhouse合併しデータのパーツを効率的に保管します。 パーツに属する別のパーティションがないます。 マージメカニズムは、同じ主キーを持つすべての行が同じデータ部分にあることを保証しません。 +データに属する別のパーティションが分離の異なる部品です。 背景では、ClickHouseはより有効な貯蔵のためのデータ部分を併合する。 異なる区画に属する部分はマージされません。 マージ機構では、同じ主キーを持つすべての行が同じデータ部分に含まれることは保証されません。 -各データ部分は、論理的に顆粒に分割されます。 顆粒は、データを選択するときにclickhouseが読み取る最小の不可分のデータセットです。 clickhouseは行または値を分割しないため、各granule粒には常に整数の行が含まれます。 顆粒の最初の行は、行の主キーの値でマークされます。 各データ、clickhouseを作成しインデックスファイルを格納するのです。 各列について、主キーにあるかどうかにかかわらず、clickhouseには同じマークも格納されます。 これらのマークまたはデータを見つの直列のファイルです。 +各データ部分は論理的にグラニュールに分割されます。 顆粒は、データを選択するときにClickHouseが読み取る最小の不可分データセットです。 ClickHouseは行や値を分割しないため、各グラニュールには常に整数の行が含まれます。 顆粒の最初の行には、行の主キーの値がマークされます。 各データ、ClickHouseを作成しインデックスファイルを格納するのです。 各列について、主キーにあるかどうかにかかわらず、ClickHouseにも同じマークが格納されます。 これらのマークまたはデータを見つの直列のファイルです。 -微粒のサイズはによって制限されます `index_granularity` と `index_granularity_bytes` テーブルエンジンの設定。 微粒の列の数はで置きます `[1, index_granularity]` 行のサイズに応じた範囲。 顆粒のサイズは超えることができます `index_granularity_bytes` 単一行のサイズが設定の値より大きい場合。 この場合、顆粒のサイズは行のサイズに等しくなります。 +微粒のサイズはによって制限されます `index_granularity` と `index_granularity_bytes` テーブルエンジンの設定。 微粒の列の数はで置きます `[1, index_granularity]` 行のサイズに応じて、範囲。 微粒のサイズは超過できます `index_granularity_bytes` 単一行のサイズが設定の値より大きい場合。 この場合、顆粒のサイズは行のサイズに等しい。 -## クエリの主キーとインデックス {#primary-keys-and-indexes-in-queries} +## クエリ内の主キーとインデックス {#primary-keys-and-indexes-in-queries} -を取る `(CounterID, Date)` 例として主キー。 この場合、並べ替えとインデックスは次のように示されます: +を取る `(CounterID, Date)` 例として主キー。 この場合、ソートとインデックスは次のように示すことができます: Whole data: [---------------------------------------------] CounterID: [aaaaaaaaaaaaaaaaaabbbbcdeeeeeeeeeeeeefgggggggghhhhhhhhhiiiiiiiiikllllllll] @@ -165,57 +165,57 @@ MergeTree(EventDate, intHash32(UserID), (CounterID, EventDate, intHash32(UserID) a,1 a,2 a,3 b,3 e,2 e,3 g,1 h,2 i,1 i,3 l,3 Marks numbers: 0 1 2 3 4 5 6 7 8 9 10 -デー: +データクエリが指定する場合: - `CounterID in ('a', 'h')`、サーバーはマークの範囲のデータを読み取ります `[0, 3)` と `[6, 8)`. - `CounterID IN ('a', 'h') AND Date = 3`、サーバーはマークの範囲のデータを読み取ります `[1, 3)` と `[7, 8)`. -- `Date = 3`、サーバーは、マークの範囲内のデータを読み取ります `[1, 10]`. +- `Date = 3`、サーバは、マークの範囲内のデータを読み取ります `[1, 10]`. 上記の例としては常に使用するのがより効果的指標により、フルスキャン！ -に乏指数で追加するデータを読み込みます。 主キーの単一の範囲を読み取るとき `index_granularity * 2` 余分な列の各データブロック読み取ることができます。 +に乏指数で追加するデータを読み込みます。 プライマリキーの単一の範囲を読み取るとき `index_granularity * 2` 余分な列の各データブロック読み取ることができます。 疎指標できる作業は非常に多くのテーブル行において、多くの場合、指数はコンピュータのアプリです。 -ClickHouseは一意の主キーを必要としません。 同じ主キーで複数の行を挿入できます。 +ClickHouseでは一意の主キーは必要ありません。 同じ主キーで複数の行を挿入できます。 ### 主キーの選択 {#selecting-the-primary-key} -主キーの列数は明示的に制限されていません。 データ構造によっては、主キーに多かれ少なかれ列を含めることができます。 この: +主キーの列の数は明示的に制限されません。 データ構造によっては、主キーに多かれ少なかれ含めることができます。 これは: -- インデックスのパフォーマン +- 索引のパフォーマンスを向上させます。 - プライマリキーが `(a, b)` 次に、別の列を追加します `c` 次の条件が満たされるとパフォーマンスが向上します: + 主キーが `(a, b)` 次に、別の列を追加します `c` 以下の条件を満たすと、パフォーマンスが向上します: - - 列に条件があるクエリがあります `c`. - - 長いデータ範囲(数倍長い `index_granularity`)の値が同じである場合 `(a, b)` 一般的です。 言い換えれば、別の列を追加すると、非常に長いデータ範囲をスキップできます。 + - 列に条件を持つクエリがあります `c`. + - 長いデータ範囲(長いデータ範囲より数倍長い `index_granularity`)と同一の値を持つ。 `(a, b)` 一般的です。 つまり、別の列を追加すると、かなり長いデータ範囲をスキップすることができます。 -- データ圧縮を改善する。 +- データ圧縮を改善します。 - ClickHouseは主キーでデータをソートするので、一貫性が高いほど圧縮率が高くなります。 + ClickHouseは主キーによってデータを並べ替えるので、一貫性が高いほど圧縮が良くなります。 -- 追加的なロジックが統合データ部分の [CollapsingMergeTree](collapsingmergetree.md#table_engine-collapsingmergetree) と [SummingMergeTree](summingmergetree.md) エンジン +- 追加的なロジックが統合データ部分の [折りたたみマージツリー](collapsingmergetree.md#table_engine-collapsingmergetree) と [サミングマーゲツリー](summingmergetree.md) エンジンだ - この場合、それは指定することは理にかなって *ソートキー* これは主キーとは異なります。 + この場合、 *ソートキー* これは主キーとは異なります。 -長いprimary keyに悪影響を及ぼす可能性は、挿入性能やメモリ消費が別の列に主キーに影響を与えないclickhouse性能 `SELECT` クエリ。 +長い主キーは挿入のパフォーマンスとメモリ消費に悪影響を及ぼしますが、主キーの追加の列はClickHouseのパフォーマンスには影響しません `SELECT` クエリ。 -### ソートキーとは異なる主キーの選択 {#choosing-a-primary-key-that-differs-from-the-sorting-key} +### 並べ替えキーとは異なる主キーの選択 {#choosing-a-primary-key-that-differs-from-the-sorting-key} -ソートキー（データ部分の行をソートする式）とは異なる主キー（各マークのインデックスファイルに書き込まれる値を持つ式）を指定することができます。 この場合、主キー式タプルは、並べ替えキー式タプルのプレフィックスである必要があります。 +ソートキー(データ部分の行をソートする式)とは異なる主キー(マークごとにインデックスファイルに書き込まれる値を持つ式)を指定することができます。 この場合、主キー式タプルは、並べ替えキー式タプルのプレフィックスでなければなりません。 -この機能は、 [SummingMergeTree](summingmergetree.md) と -[ﾂつｨﾂ姪“ﾂつ”ﾂ債ﾂづｭﾂつｹ](aggregatingmergetree.md) テーブルエンジン。 これらのエンジンを使用する一般的なケースでは、テーブルには二種類の列があります: *寸法* と *対策*. 典型的なクエリは、任意のメジャー列の値を集計します `GROUP BY` そして次元によるろ過。 SummingMergeTreeとAggregatingMergeTreeは、並べ替えキーの同じ値を持つ行を集計するので、すべての次元を追加するのが自然です。 その結果、キー式は長い列のリストで構成され、このリストは新しく追加されたディメンションで頻繁に更新される必要があります。 +この機能は、 [サミングマーゲツリー](summingmergetree.md) と +[AggregatingMergeTree](aggregatingmergetree.md) テーブルエンジン。 共通の場合はご利用になられる場合はエンジンのテーブルは、二種類のカラム: *寸法* と *対策*. 典型的なクエリは、任意のメジャー列の値を集計します `GROUP BY` そして次元によるろ過。 SumingmergetreeとAggregatingMergeTreeは並べ替えキーの値が同じ行を集計するため、すべてのディメンションを追加するのは自然です。 その結果、キー式は列の長いリストで構成され、このリストは頻繁に新しく追加されたディメンションで更新する必要があります。 -この場合、主キーにいくつかの列だけを残して、効率的な範囲スキャンを提供し、残りのディメンション列を並べ替えキータプルに追加することが理に +この場合、効率的な範囲スキャンを提供し、残りのディメンション列を並べ替えキータプルに追加する主キーには数列のみを残すことが理にかなって -[ALTER](../../../sql-reference/statements/alter.md) 新しい列がテーブルとソートキーに同時に追加されると、既存のデータパーツを変更する必要がないため、ソートキーの操作は軽量です。 古いソートキーは新しいソートキーの接頭辞であり、新しく追加された列にデータがないため、データはテーブル変更の時点で古いソートキーと新しいソートキーの両方 +[ALTER](../../../sql-reference/statements/alter.md) 新しい列がテーブルとソートキーに同時に追加されると、既存のデータ部分を変更する必要がないため、ソートキーの軽量な操作です。 古い並べ替えキーは新しい並べ替えキーのプレフィックスであり、新しく追加された列にデータがないため、テーブルの変更の瞬間に、データは新旧の並べ替え ### クエリでの索引とパーティションの使用 {#use-of-indexes-and-partitions-in-queries} -のために `SELECT` ClickHouseは、インデックスを使用できるかどうかを分析します。 インデックスが使用できるのは、 `WHERE/PREWHERE` 句には、等式または不等式の比較演算を表す式（連結要素のいずれかとして、または完全に）があります。 `IN` または `LIKE` 主キーまたはパーティショニングキーに含まれる列または式、またはこれらの列の特定の部分反復関数、またはこれらの式の論理関係に固定プレフィッ +のために `SELECT` クClickHouseを分析するかどうかの指標を使用できます。 インデックスは、次の場合に使用できます `WHERE/PREWHERE` 句には、等価比較演算または不等比較演算を表す式（連結要素のいずれかとして、または完全に）があります。 `IN` または `LIKE` 主キーまたはパーティショニングキーにある列または式、またはこれらの列の特定の部分的に反復機能、またはこれらの式の論理的関係に固定プレフィッ -したがって、主キーの一つまたは複数の範囲でクエリをすばやく実行することができます。 この例では、特定のトラッキングタグ、特定のタグおよび日付範囲、特定のタグおよび日付、日付範囲を持つ複数のタグなどに対して実行すると、クエ +したがって、主キーの一つまたは多くの範囲でクエリを迅速に実行することができます。 この例では、特定の追跡タグ、特定のタグと日付範囲、特定のタグと日付、日付範囲を持つ複数のタグなどに対してクエリを実行すると、クエリが高速 次のように構成されたエンジンを見てみましょう: @@ -239,31 +239,31 @@ ClickHouseの主キー指標のトリムで不正なデータを毎月パーテ SELECT count() FROM table WHERE CounterID = 34 OR URL LIKE '%upyachka%' ``` -確認clickhouseできるとの利用時の走行クエリに対して、使用の設定 [force\_index\_by\_date](../../../operations/settings/settings.md#settings-force_index_by_date) と [force\_primary\_key](../../../operations/settings/settings.md). +クエリの実行時にClickHouseがインデックスを使用できるかどうかを確認するには、設定を使用します [force\_index\_by\_date](../../../operations/settings/settings.md#settings-force_index_by_date) と [force\_primary\_key](../../../operations/settings/settings.md). -の分割による月で読み込みのみこれらのデータブロックを含むからスピーチへのマークの範囲内で適切に取扱います。 この場合、データブロックには多くの日付（月全体まで）のデータが含まれることがあります。 ブロック内では、データは主キーによってソートされます。 このため、主キープレフィックスを指定しない日付条件のみを持つクエリを使用すると、単一の日付よりも多くのデータが読み取られます。 +の分割による月で読み込みのみこれらのデータブロックを含むからスピーチへのマークの範囲内で適切に取扱います。 この場合、データブロックには多くの日付(月全体まで)のデータが含まれる場合があります。 ブロック内では、データは主キーによってソートされます。 このため、主キープレフィックスを指定しない日付条件のみのクエリを使用すると、単一の日付よりも多くのデータが読み取られます。 -### 部分的に単調な主キーに対するインデックスの使用 {#use-of-index-for-partially-monotonic-primary-keys} +### 部分的に単調な主キーのインデックスの使用 {#use-of-index-for-partially-monotonic-primary-keys} -たとえば、月の日数を考えてみましょう。 彼らは形成する [単調系列](https://en.wikipedia.org/wiki/Monotonic_function) 一ヶ月のために、しかし、より長期間単調ではありません。 これは部分的に単調なシーケンスです。 ユーザーが部分的に単調な主キーを持つテーブルを作成する場合、ClickHouseは通常どおりスパースインデックスを作成します。 ユーザーがこの種類のテーブルからデータを選択すると、ClickHouseはクエリ条件を分析します。 ユーザーは、インデックスの二つのマークの間のデータを取得したいと、これらのマークの両方が一ヶ月以内に落ちる場合、それはクエリとインデックスマーク +例えば、月の日を考えてみましょう。 それらはaを形作る [単調系列](https://en.wikipedia.org/wiki/Monotonic_function) 一ヶ月のために、しかし、より長期間単調ではありません。 これは部分的に単調なシーケンスです。 ユーザーが部分的に単調な主キーでテーブルを作成する場合、ClickHouseは通常どおりスパースインデックスを作成します。 ユーザーがこの種のテーブルからデータを選択すると、ClickHouseはクエリ条件を分析します。 これは、クエリのパラメータとインデックスマークの間の距離を計算できるためです。 -クエリパラメーターの範囲内の主キーの値が単調順序を表さない場合、clickhouseはインデックスを使用できません。 この場合、clickhouseはフルスキャン方式を使用します。 +クエリパラメータ範囲内の主キーの値が単調なシーケンスを表していない場合、ClickHouseはインデックスを使用できません。 この場合、ClickHouseはフルスキャン方法を使用します。 -ClickHouseは、月シーケンスの日数だけでなく、部分的に単調なシーケンスを表すプライマリキーについても、このロジックを使用します。 +ClickHouseは、このロジックを月の日数シーケンスだけでなく、部分的に単調なシーケンスを表す主キーにも使用します。 ### データスキップインデックス(実験) {#table_engine-mergetree-data_skipping-indexes} -インデックス宣言は、次の列セクションにあります `CREATE` クエリ。 +インデックス宣言は、 `CREATE` クエリ。 ``` sql INDEX index_name expr TYPE type(...) GRANULARITY granularity_value ``` -からのテーブルの場合 `*MergeTree` 家族データの飛び指標を指定できます。 +からのテーブルのため `*MergeTree` 家族データの飛び指標を指定できます。 -これらのインデックスは、ブロックの指定された式に関する情報を集約します。 `granularity_value` 微粒（微粒のサイズはを使用して指定されます `index_granularity` テーブルエンジンの設定）。 次に、これらの集約は `SELECT` ディスクから読み取るデータの量を減らすためのクエリ `where` クエリは満たされません。 +これらのインデックスは、指定された式に関する情報をブロックに集約します。 `granularity_value` 微粒（微粒のサイズはを使用して指定されます `index_granularity` テーブルエンジンでの設定）。 次に、これらの集計は `SELECT` クエリを削減量のデータから読み込むディスクにより操大きなブロックのデータを `where` クエリが満たされません。 -**例えば** +**例** ``` sql CREATE TABLE table_name @@ -278,26 +278,26 @@ CREATE TABLE table_name ... ``` -この例のインデックスをclickhouseで使用すると、次のクエリでディスクから読み取るデータの量を減らすことができます: +この例のインデックスをClickHouseで使用すると、次のクエリでディスクから読み取るデータ量を減らすことができます: ``` sql SELECT count() FROM table WHERE s < 'z' SELECT count() FROM table WHERE u64 * i32 == 10 AND u64 * length(s) >= 1234 ``` -#### 使用可能なインデックスの種類 {#available-types-of-indices} +#### 使用可能なインデックスタイプ {#available-types-of-indices} - `minmax` - 指定された式の極値を格納します(式が指定されている場合 `tuple` そして、それは各要素のための極端をの貯えます `tuple`)を使用して保存情報の飛びブロックのようなデータは、その有効なタイプを利用します。 + 指定された式の極値を格納します(式が `tuple` の各要素の極値を格納します。 `tuple`)を使用して保存情報の飛びブロックのようなデータは、その有効なタイプを利用します。 - `set(max_rows)` - 指定された式の一意の値を格納します。 `max_rows` 行, `max_rows=0` 意味 “no limits”). この値を使用して、 `WHERE` 式はデータブロックでは充足可能ではありません。 + 指定された式の一意の値を格納します。 `max_rows` 行, `max_rows=0` つまり “no limits”). 値を使用して、 `WHERE` 式はデータのブロックでは満足できません。 - `ngrambf_v1(n, size_of_bloom_filter_in_bytes, number_of_hash_functions, random_seed)` - 店a [Bloom filter](https://en.wikipedia.org/wiki/Bloom_filter) これには、データブロックのすべてのngramsが含まれます。 文字列でのみ動作します。 の最適化に使用することができます `equals`, `like` と `in` 式。 + ストアa [Bloomフィルタ](https://en.wikipedia.org/wiki/Bloom_filter) これによりngramsからブロックのデータです。 文字列でのみ動作します。 の最適化に使用することができます `equals`, `like` と `in` 式。 - `n` — ngram size, - `size_of_bloom_filter_in_bytes` — Bloom filter size in bytes (you can use large values here, for example, 256 or 512, because it can be compressed well). @@ -306,15 +306,15 @@ SELECT count() FROM table WHERE u64 * i32 == 10 AND u64 * length(s) >= 1234 - `tokenbf_v1(size_of_bloom_filter_in_bytes, number_of_hash_functions, random_seed)` - 同じように `ngrambf_v1` しかし、ngramsの代わりにトークンを格納します。 トークンは、英数字以外の文字で区切られた順序です。 + と同じ `ngrambf_v1` しかし、ngramsの代わりにトークンを格納します。 トークンは英数字以外の文字で区切られた配列です。 -- `bloom_filter([false_positive])` — Stores a [Bloom filter](https://en.wikipedia.org/wiki/Bloom_filter) 指定された列の場合。 +- `bloom_filter([false_positive])` — Stores a [Bloomフィルタ](https://en.wikipedia.org/wiki/Bloom_filter) 指定された列の場合。 - 任意 `false_positive` パラメーターは、フィルターから偽陽性の応答を受信する確率です。 可能な値:(0,1)。 デフォルト値:0.025. + 任意 `false_positive` パラメータは、フィルタから偽陽性応答を受信する確率です。 可能な値:(0,1)。 既定値は0.025です。 - 対応データ型: `Int*`, `UInt*`, `Float*`, `Enum`, `Date`, `DateTime`, `String`, `FixedString`, `Array`, `LowCardinality`, `Nullable`. + 対応するデータ型: `Int*`, `UInt*`, `Float*`, `Enum`, `Date`, `DateTime`, `String`, `FixedString`, `Array`, `LowCardinality`, `Nullable`. - 次の関数はそれを使用できます: [等しい](../../../sql-reference/functions/comparison-functions.md), [notEquals](../../../sql-reference/functions/comparison-functions.md), [で](../../../sql-reference/functions/in-functions.md), [notIn](../../../sql-reference/functions/in-functions.md), [持っている](../../../sql-reference/functions/array-functions.md). + 以下の関数が使用できます: [等しい](../../../sql-reference/functions/comparison-functions.md), [notEquals](../../../sql-reference/functions/comparison-functions.md), [で](../../../sql-reference/functions/in-functions.md), [ノティン](../../../sql-reference/functions/in-functions.md), [は](../../../sql-reference/functions/array-functions.md). @@ -326,40 +326,40 @@ INDEX sample_index3 (lower(str), str) TYPE ngrambf_v1(3, 256, 2, 0) GRANULARITY #### 機能サポート {#functions-support} -の条件 `WHERE` clauseには、列で操作する関数の呼び出しが含まれます。 列がインデックスの一部である場合、ClickHouseは関数の実行時にこのインデックスを使用しようとします。 ClickHouse支援の異なるサブセットの機能を使用。 +の条件 `WHERE` 句には、列で動作する関数の呼び出しが含まれます。 列がインデックスの一部である場合、ClickHouseは関数の実行時にこのインデックスを使用しようとします。 ClickHouse支援の異なるサブセットの機能を使用。 -その `set` indexは、すべての関数で使用できます。 他のインデックスの関数サブセットを以下の表に示します。 +その `set` 索引はすべての機能と使用することができる。 その他のインデックスの関数サブセットを以下の表に示します。 -| 関数(演算子)/インデックス | 主キー | minmax | ngrambf\_v1 | tokenbf\_v1 | bloom\_filter | -|-------------------------------------------------------------------------------------------------------------|--------|--------|-------------|-------------|---------------| -| [equals(=,==))](../../../sql-reference/functions/comparison-functions.md#function-equals) | ✔ | ✔ | ✔ | ✔ | ✔ | -| [notEquals(!=, \<\>)](../../../sql-reference/functions/comparison-functions.md#function-notequals) | ✔ | ✔ | ✔ | ✔ | ✔ | -| [のように](../../../sql-reference/functions/string-search-functions.md#function-like) | ✔ | ✔ | ✔ | ✗ | ✗ | -| [notLike](../../../sql-reference/functions/string-search-functions.md#function-notlike) | ✔ | ✔ | ✔ | ✗ | ✗ | -| [startsWith](../../../sql-reference/functions/string-functions.md#startswith) | ✔ | ✔ | ✔ | ✔ | ✗ | -| [エンドスウィス](../../../sql-reference/functions/string-functions.md#endswith) | ✗ | ✗ | ✔ | ✔ | ✗ | -| [マルチセアチャンネル](../../../sql-reference/functions/string-search-functions.md#function-multisearchany) | ✗ | ✗ | ✔ | ✗ | ✗ | -| [で](../../../sql-reference/functions/in-functions.md#in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ | -| [notIn](../../../sql-reference/functions/in-functions.md#in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ | -| [less(\<)](../../../sql-reference/functions/comparison-functions.md#function-less) | ✔ | ✔ | ✗ | ✗ | ✗ | -| [グレーター(\>)](../../../sql-reference/functions/comparison-functions.md#function-greater) | ✔ | ✔ | ✗ | ✗ | ✗ | -| [lessOrEquals(\<=)](../../../sql-reference/functions/comparison-functions.md#function-lessorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | -| [greaterOrEquals(\>=)](../../../sql-reference/functions/comparison-functions.md#function-greaterorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | -| [空](../../../sql-reference/functions/array-functions.md#function-empty) | ✔ | ✔ | ✗ | ✗ | ✗ | -| [notEmpty](../../../sql-reference/functions/array-functions.md#function-notempty) | ✔ | ✔ | ✗ | ✗ | ✗ | -| ハストケンcity in germany | ✗ | ✗ | ✗ | ✔ | ✗ | +| 関数(演算子)/インデックス | 主キー | minmax | ngrambf\_v1 | tokenbf\_v1 | bloom\_filter name | +|-----------------------------------------------------------------------------------------------------------|--------|--------|-------------|-------------|--------------------| +| [等しい(=,==)](../../../sql-reference/functions/comparison-functions.md#function-equals) | ✔ | ✔ | ✔ | ✔ | ✔ | +| [notEquals(!=, \<\>)](../../../sql-reference/functions/comparison-functions.md#function-notequals) | ✔ | ✔ | ✔ | ✔ | ✔ | +| [のように](../../../sql-reference/functions/string-search-functions.md#function-like) | ✔ | ✔ | ✔ | ✗ | ✗ | +| [notLike](../../../sql-reference/functions/string-search-functions.md#function-notlike) | ✔ | ✔ | ✔ | ✗ | ✗ | +| [スタート](../../../sql-reference/functions/string-functions.md#startswith) | ✔ | ✔ | ✔ | ✔ | ✗ | +| [endsWith](../../../sql-reference/functions/string-functions.md#endswith) | ✗ | ✗ | ✔ | ✔ | ✗ | +| [マルチサーチ](../../../sql-reference/functions/string-search-functions.md#function-multisearchany) | ✗ | ✗ | ✔ | ✗ | ✗ | +| [で](../../../sql-reference/functions/in-functions.md#in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ | +| [ノティン](../../../sql-reference/functions/in-functions.md#in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ | +| [less(\<)](../../../sql-reference/functions/comparison-functions.md#function-less) | ✔ | ✔ | ✗ | ✗ | ✗ | +| [グレーター(\>)](../../../sql-reference/functions/comparison-functions.md#function-greater) | ✔ | ✔ | ✗ | ✗ | ✗ | +| [lessOrEquals(\<=)](../../../sql-reference/functions/comparison-functions.md#function-lessorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | +| [greaterOrEquals(\>=)](../../../sql-reference/functions/comparison-functions.md#function-greaterorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | +| [空](../../../sql-reference/functions/array-functions.md#function-empty) | ✔ | ✔ | ✗ | ✗ | ✗ | +| [ノーテンプティ](../../../sql-reference/functions/array-functions.md#function-notempty) | ✔ | ✔ | ✗ | ✗ | ✗ | +| ヘイストケン | ✗ | ✗ | ✗ | ✔ | ✗ | -Ngramサイズより小さい定数引数を持つ関数は、 `ngrambf_v1` クエリの最適化のため。 +Ngramサイズよりも小さい定数引数を持つ関数は、次のように使用できません `ngrambf_v1` クエリ最適化のため。 -ブルでは偽陽性一致すので、 `ngrambf_v1`, `tokenbf_v1`、と `bloom_filter` インデックスは、関数の結果がfalseであると予想されるクエリの最適化には使用できません。: +ブルでは偽陽性一致すので、 `ngrambf_v1`, `tokenbf_v1`,and `bloom_filter` インデックスは、関数の結果がfalseになると予想されるクエリの最適化には使用できません。: -- 最適化することができる: +- 最適化できます: - `s LIKE '%test%'` - `NOT s NOT LIKE '%test%'` - `s = 1` - `NOT s != 1` - `startsWith(s, 'test')` -- 最適化できません: +- 最適化できない: - `NOT s LIKE '%test%'` - `s NOT LIKE '%test%'` - `NOT s = 1` @@ -368,37 +368,37 @@ Ngramサイズより小さい定数引数を持つ関数は、 `ngrambf_v1` ク ## 同時データアクセス {#concurrent-data-access} -同時テーブルアクセスでは、マルチバージョンを使用します。 つまり、テーブルが同時に読み取られて更新されると、クエリ時に現在のパーツのセットからデータが読み取られます。 長いロックはありません。 挿入は読み取り操作の方法では得られません。 +同時テーブルアクセスには、マルチバージョン管理を使用します。 つまり、テーブルの読み取りと更新が同時に行われると、クエリの時点で現在の部分のセットからデータが読み取られます。 長いロックはありません。 挿入は読み取り操作の邪魔になりません。 テーブルからの読み取りは自動的に並列化されます。 -## 列とテーブルのttl {#table_engine-mergetree-ttl} +## 列および表のTTL {#table_engine-mergetree-ttl} -値の存続期間を決定します。 +値の有効期間を決定します。 -その `TTL` 句は、テーブル全体と個々の列ごとに設定することができます。 テーブルレベルのTTLで指定した論理の自動移動のデータディスクの間とします。 +その `TTL` 句は、テーブル全体および個々の列ごとに設定できます。 テーブルレベルのTTLで指定した論理の自動移動のデータディスクの間とします。 -式は評価する必要があります [日付](../../../sql-reference/data-types/date.md) または [DateTime](../../../sql-reference/data-types/datetime.md) データ型。 +表現を行い、評価しなければならなす [日付](../../../sql-reference/data-types/date.md) または [DateTime](../../../sql-reference/data-types/datetime.md) データ型。 -例えば: +例: ``` sql TTL time_column TTL time_column + interval ``` -定義する `interval`、使用 [時間間隔](../../../sql-reference/operators.md#operators-datetime) 演算子。 +定義するには `interval`,use [時間間隔](../../../sql-reference/operators/index.md#operators-datetime) 演算子。 ``` sql TTL date_time + INTERVAL 1 MONTH TTL date_time + INTERVAL 15 HOUR ``` -### 列ttl {#mergetree-column-ttl} +### 列TTL {#mergetree-column-ttl} -列の値が期限切れになると、clickhouseは列のデータ型の既定値に置き換えます。 すべてのカラム値のデータ部分を切clickhouse削除するこのカラムからのデータにファイルシステム. +列の値が期限切れになると、ClickHouseは列のデータ型の既定値に置き換えます。 データ部分のすべての列の値が期限切れになった場合、ClickHouseはファイルシステム内のデータ部分からこの列を削除します。 -その `TTL` キー列には句を使用できません。 +その `TTL` 句はキー列には使用できません。 例: @@ -417,7 +417,7 @@ PARTITION BY toYYYYMM(d) ORDER BY d; ``` -既存のテーブルの列にttlを追加する +既存のテーブルの列へのTTLの追加 ``` sql ALTER TABLE example_table @@ -425,7 +425,7 @@ ALTER TABLE example_table c String TTL d + INTERVAL 1 DAY; ``` -列のttlの変更 +列のTTLを変更する ``` sql ALTER TABLE example_table @@ -433,19 +433,19 @@ ALTER TABLE example_table c String TTL d + INTERVAL 1 MONTH; ``` -### テーブルttl {#mergetree-table-ttl} +### テーブルTTL {#mergetree-table-ttl} -テーブルでの表現の除去に終了しました列、複数の表現を自動で部品の移動と [ディスク](#table_engine-mergetree-multiple-volumes). 時テーブルの行の有効期間ClickHouseをすべて削除して対応さい。 部品移動フィーチャの場合、部品のすべての行が移動式の基準を満たしている必要があります。 +テーブルでの表現の除去に終了しました列、複数の表現を自動で部品の移動と [ディスクまた](#table_engine-mergetree-multiple-volumes). 表の行が期限切れになると、ClickHouseは対応するすべての行を削除します。 パーツ移動フィーチャの場合、パーツのすべての行が移動式基準を満たす必要があります。 ``` sql TTL expr [DELETE|TO DISK 'aaa'|TO VOLUME 'bbb'], ... ``` -TTLルールのタイプは、各TTL式に従います。 これは、式が満たされると実行されるアクションに影響します（現在の時間に達します): +TTL規則のタイプは、各TTL式に従うことができます。 これは、式が満たされた後（現在の時刻に達する）に実行されるアクションに影響します): - `DELETE` 削除行を終了しました(デフォルトアクション); -- `TO DISK 'aaa'` -ディスクに部品を移動 `aaa`; -- `TO VOLUME 'bbb'` -ディスクに部品を移動 `bbb`. +- `TO DISK 'aaa'` -ディスクへの部分の移動 `aaa`; +- `TO VOLUME 'bbb'` -ディスクへの部分の移動 `bbb`. 例: @@ -465,7 +465,7 @@ TTL d + INTERVAL 1 MONTH [DELETE], d + INTERVAL 2 WEEK TO DISK 'bbb'; ``` -テーブルのttlの変更 +テーブルのTTLの変更 ``` sql ALTER TABLE example_table @@ -474,34 +474,34 @@ ALTER TABLE example_table **データの削除** -データ切れのttlを取り除きclickhouse合併しデータの部品です。 +期限切れのTTLのデータはClickHouseがデータ部分を結合するとき取除かれる。 -時clickhouseるデータの期間は終了しましたので、行offスケジュール内スケジュールする必要がありません。 このようなマージの頻度を制御するには、次のように設定します `merge_with_ttl_timeout`. 値が低すぎる場合は、多くのリソースを消費する可能性のある多くのオフスケジュールマージを実行します。 +時ClickHouseるデータの期間は終了しましたので、行offスケジュール内スケジュールする必要がありません。 このようなマージの頻度を制御するには、次のように設定できます `merge_with_ttl_timeout`. 値が小さすぎると、大量のリソースを消費する可能性のある、スケジュール外のマージが多数実行されます。 -あなたが実行する場合 `SELECT` 期限切れのデータを取得できます。 それを避けるために、を使用 [OPTIMIZE](../../../sql-reference/statements/misc.md#misc_operations-optimize) 前にクエリ `SELECT`. +を実行する場合 `SELECT` クエリと合併はありませんが、できれば終了しました。 それを避けるには、 [OPTIMIZE](../../../sql-reference/statements/misc.md#misc_operations-optimize) クエリの前に `SELECT`. -## 複数ブロックデバイスを使用したデータ保存 {#table_engine-mergetree-multiple-volumes} +## 複数のブロックデバイスのためのデータ保存 {#table_engine-mergetree-multiple-volumes} -### 導入 {#introduction} +### はじめに {#introduction} -`MergeTree` 家族のテーブルエンジンでデータを複数のブロックデバイス たとえば、特定のテーブルのデータが暗黙的に分割されている場合に便利です “hot” と “cold”. 最新のデータは定期的に要求されますが、必要な領域はわずかです。 それどころか、fat-tailed履歴データはまれに要求される。 複数のディスクが使用可能な場合は、 “hot” データは高速ディスク(たとえば、NVMe Ssdまたはメモリ内)にあります。 “cold” データ-比較的遅いもの（例えば、HDD）。 +`MergeTree` 家族のテーブルエンジンでデータを複数のブロックデバイス たとえば、特定のテーブルのデータが暗黙的に次のように分割される場合に便利です “hot” と “cold”. 最新のデータは定期的に要求されますが、わずかなスペースしか必要ありません。 逆に、fat-tailed履歴データはほとんど要求されません。 複数のディスクが利用できれば、 “hot” データは高速ディスク(例えば、NVMe Ssdまたはメモリ内)に配置されることがあります。 “cold” データ-比較的遅いもの（例えば、HDD）。 -データ部分は最低の移動可能な単位のためのです `MergeTree`-エンジンテーブル。 ある部分に属するデータは、あるディスクに格納されます。 データ部分は背景のディスクの間で（ユーザーの設定に従って）、またによって動かすことができます [ALTER](../../../sql-reference/statements/alter.md#alter_move-partition) クエリ。 +データ部分は最低の移動可能な単位のためのです `MergeTree`-エンジンテーブル。 ある部分に属するデータは、一つのディスクに保存されます。 データパーツは、バックグラウンドで(ユーザー設定に従って)ディスク間で移動することができます。 [ALTER](../../../sql-reference/statements/alter.md#alter_move-partition) クエリ。 -### 条件 {#terms} +### 用語 {#terms} - Disk — Block device mounted to the filesystem. - Default disk — Disk that stores the path specified in the [パス](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-path) サーバー設定。 - Volume — Ordered set of equal disks (similar to [JBOD](https://en.wikipedia.org/wiki/Non-RAID_drive_architectures)). - Storage policy — Set of volumes and the rules for moving data between them. -の名称を記載することから、システムテーブル, [システム。ストレージ\_policies](../../../operations/system-tables.md#system_tables-storage_policies) と [システム。ディスク](../../../operations/system-tables.md#system_tables-disks). テーブ `storage_policy` の設定 `MergeTree`-エンジン家族のテーブル。 +の名称を記載することから、システムテーブル, [システムストレージポリシー](../../../operations/system-tables.md#system_tables-storage_policies) と [システムディスク](../../../operations/system-tables.md#system_tables-disks). 適用の設定を保存政策のためのテーブルを使用 `storage_policy` 設定の `MergeTree`-エンジン家族のテーブル。 ### 設定 {#table_engine-mergetree-multiple-volumes_configure} -ディスク、ボリューム、およびストレージポリシーは、 `` メインファイルのいずれかのタグ `config.xml` または、 `config.d` ディレクトリ。 +ディスク、ボリューム、およびストレージポリシーは、 `` メインファイルにタグを付ける `config.xml` または、 `config.d` ディレクトリ。 -構成の構造: +構成構造: ``` xml @@ -533,7 +533,7 @@ ALTER TABLE example_table ディスク定義の順序は重要ではありません。 -ストレージポリシ: +保管方針の設定のマークアップ: ``` xml @@ -567,10 +567,10 @@ ALTER TABLE example_table - `policy_name_N` — Policy name. Policy names must be unique. - `volume_name_N` — Volume name. Volume names must be unique. - `disk` — a disk within a volume. -- `max_data_part_size_bytes` — the maximum size of a part that can be stored on any of the volume’s disks. +- `max_data_part_size_bytes` — the maximum size of a part that can be stored on any of the volume's disks. - `move_factor` — when the amount of available space gets lower than this factor, data automatically start to move on the next volume if any (by default, 0.1). -Cofigurationの例: +構成の例: ``` xml @@ -602,12 +602,12 @@ Cofigurationの例: ``` -与えられた例では、 `hdd_in_order` ポリシーの実装 [ラウンドロビン](https://en.wikipedia.org/wiki/Round-robin_scheduling) アプローチ。 したがって、このポリシ (`single`)データパーツは、すべてのディスクに循環順序で格納されます。 こうした政策れぞれの知見について学ぶとともに有が複数ある場合は同様のディスク搭載のシステムがRAIDな設定を行います。 個々のディスクドライブはそれぞれ信頼できないため、複製係数が3以上になるように補正する必要があることに注意してください。 +与えられた例では、 `hdd_in_order` ポリシーは、 [ラウンドロビン](https://en.wikipedia.org/wiki/Round-robin_scheduling) アプローチ このようにこの方針を定義しみ量 (`single`）、データ部分は円形の順序ですべてのディスクに格納されています。 こうした政策れぞれの知見について学ぶとともに有が複数ある場合は同様のディスク搭載のシステムがRAIDな設定を行います。 個々のディスクドライブは信頼できないため、複製係数が3以上で補う必要がある場合があります。 -システムで使用可能なディスクの種類が異なる場合, `moving_from_ssd_to_hdd` ポリシーは代わりに使用できます。 ボリューム `hot` SSDディスクで構成されています (`fast_ssd`このボリュームに格納できるパーツの最大サイズは1GBです。 サイズが1GBより大きいすべての部品はで直接貯えられます `cold` HDDディスクを含むボリューム `disk1`. -また、一度ディスク `fast_ssd` 80%以上によって満たされて得ます、データはに移ります `disk1` 背景プロセ +システムで使用可能なディスクの種類が異なる場合, `moving_from_ssd_to_hdd` 代わりにポリシーを使用できます。 ボリューム `hot` SSDディスクで構成されています (`fast_ssd`)、このボリュームに格納できるパーツの最大サイズは1GBです。 1GBより大きいサイズのすべての部品はで直接貯えられます `cold` HDDディスクを含むボリューム `disk1`. +また、一度ディスク `fast_ssd` 80%以上によって満たされて、データはに移ります得ます `disk1` 背景プロセスによって。 -ストレージポリシー内のボリューム列挙の順序は重要です。 ボリュームがオーバーフィルされると、データは次のものに移動されます。 ディスク列挙の順序は、データが順番に格納されるため、重要です。 +ストレージポリシー内のボリューム列挙の順序は重要です。 ボリュームが満杯になると、データは次のボリュームに移動されます。 データは順番に格納されるため、ディスク列挙の順序も重要です。 作成時にテーブルは、適用の設定を保存方針で: @@ -623,32 +623,32 @@ PARTITION BY toYYYYMM(EventDate) SETTINGS storage_policy = 'moving_from_ssd_to_hdd' ``` -その `default` ストレージポリシーは、ボリュームを一つだけ使用することを意味します。 ``. テーブルを作成すると、そのストレージポリシーは変更できません。 +その `default` ストレージポリシーは、一つのボリュームのみを使用することを意味します。 ``. テーブルを作成すると、そのストレージポリシーは変更できません。 ### 詳細 {#details} の場合 `MergeTree` テーブル、データがあるディスクには、異なる方法: - 挿入の結果として (`INSERT` クエリ）。 -- バックグラウンドマージ時 [突然変異](../../../sql-reference/statements/alter.md#alter-mutations). -- 別のレプリカか -- 仕切りの凍結の結果として [ALTER TABLE … FREEZE PARTITION](../../../sql-reference/statements/alter.md#alter_freeze-partition). +- バックグラウンドマージ中 [突然変異](../../../sql-reference/statements/alter.md#alter-mutations). +- 別のレプリカからダウンロ +- パーティションの凍結の結果として [ALTER TABLE … FREEZE PARTITION](../../../sql-reference/statements/alter.md#alter_freeze-partition). すべてのこれらの場合を除き、突然変異とパーティションの凍結は、一部が保存され、大量のディスクに保存政策: -1. パートを格納するのに十分なディスク領域を持つ最初のボリューム(定義の順序で) (`unreserved_space > current_part_size`)特定のサイズの部品を格納することができます (`max_data_part_size_bytes > current_part_size`)が選択されます。 -2. このボリューム内では、前のデータチャンクを格納するために使用されたディスクに続くディスクが選択され、パーツサイズよりも空き領域が多くなり (`unreserved_space - keep_free_space_bytes > current_part_size`). +1. 部品を格納するのに十分なディスク領域を持つ最初のボリューム(定義順) (`unreserved_space > current_part_size`）と指定されたサイズの部分を格納することができます (`max_data_part_size_bytes > current_part_size`）が選ばれる。 +2. このボリューム内では、データの前のチャンクを格納するために使用され、パートサイズよりも空き領域が多いディスクが選択されます (`unreserved_space - keep_free_space_bytes > current_part_size`). フードの下で、突然変異および仕切りの凍結は利用します [ハードリンク](https://en.wikipedia.org/wiki/Hard_link). ハードリンクとディスクには対応していないため、この場合、パーツの保管と同じディスクの初期ます。 -バックグラウンドでは、空き領域の量に基づいて部品がボリューム間で移動されます (`move_factor` パラメータ)順序に従って、設定ファイルでボリュームが宣言されます。 -データは最後のものから最初のものに転送されません。 システムテーブルを使用できる [システム。part\_log](../../../operations/system-tables.md#system_tables-part-log) (フィールド `type = MOVE_PART`） [システム。パーツ](../../../operations/system-tables.md#system_tables-parts) (フィールド `path` と `disk`）背景の動きを監視する。 また、詳細な情報はサーバーログに記載されています。 +バックグラウンドでは、部品は空き領域の量に基づいてボリューム間で移動されます (`move_factor` パラメータ)ボリュームが設定ファイルで宣言されている順序に従って。 +データは、最後のデータから最初のデータに転送されることはありません。 システムテーブル [システムpart\_log](../../../operations/system-tables.md#system_tables-part-log) （フィールド `type = MOVE_PART`)と [システム部品](../../../operations/system-tables.md#system_tables-parts) （フィールド `path` と `disk`）背景の動きを監視します。 また、詳細情報はサーバーログに記載されています。 -ユーザーの力で移動中の一部またはパーティションから量別のクエリ [ALTER TABLE … MOVE PART\|PARTITION … TO VOLUME\|DISK …](../../../sql-reference/statements/alter.md#alter_move-partition)、バックグラウンド操作のすべての制限が考慮されます。 クエリは単独で移動を開始し、バックグラウンド操作が完了するのを待機しません。 十分な空き領域がない場合、または必要な条件のいずれかが満たされない場合、ユーザーはエラーメッセージが表示されます。 +ユーザーの力で移動中の一部またはパーティションから量別のクエリ [ALTER TABLE … MOVE PART\|PARTITION … TO VOLUME\|DISK …](../../../sql-reference/statements/alter.md#alter_move-partition) バックグラウンド操作のすべての制限が考慮されます。 クエリは単独で移動を開始し、バックグラウンド操作が完了するまで待機しません。 十分な空き領域がない場合、または必要な条件のいずれかが満たされていない場合、ユーザーはエラーメッセージを表示します。 -データの移動はデータの複製を妨げません。 そのため、異なる保管方針を指定することができ、同じテーブルの異なるレプリカ. +データの移動は、データの複製を妨げません。 そのため、異なる保管方針を指定することができ、同じテーブルの異なるレプリカ. バックグラウンドマージと突然変異の完了後、古い部分は一定時間後にのみ削除されます (`old_parts_lifetime`). -この間、他のボリュームやディスクに移動されることはありません。 したがって、部品が最終的に取り外されるまで、それらはまだ占有ディスクスペースの評価のために考慮される。 +この間、他のボリュームまたはディスクには移動されません。 したがって、部品が最終的に除去されるまで、それらは占有されたディスク領域の評価のために考慮される。 [元の記事](https://clickhouse.tech/docs/ru/operations/table_engines/mergetree/) diff --git a/docs/ja/engines/table-engines/mergetree-family/replacingmergetree.md b/docs/ja/engines/table-engines/mergetree-family/replacingmergetree.md index 37df9077744..c3df9559415 100644 --- a/docs/ja/engines/table-engines/mergetree-family/replacingmergetree.md +++ b/docs/ja/engines/table-engines/mergetree-family/replacingmergetree.md @@ -1,18 +1,17 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 33 -toc_title: "\uFF82\u3064\uFF68\uFF82\u59EA\"\uFF82\u3064\"\uFF82\u50B5\uFF82\u3064\ - \uFF79" +toc_title: "\u7F6E\u63DB\u30DE\u30FC\u30B8\u30C4\u30EA\u30FC" --- -# ﾂつｨﾂ姪“ﾂつ”ﾂ債ﾂつｹ {#replacingmergetree} +# 置換マージツリー {#replacingmergetree} -エンジンは [MergeTree](mergetree.md#table_engines-mergetree) それは、同じ主キー値（またはより正確には同じ値）を持つ重複エントリを削除するという点で [ソートキー](mergetree.md) 値）。 +エンジンは [メルゲツリー](mergetree.md#table_engines-mergetree) 同じ主キー値を持つ重複したエントリを削除するという点で、より正確には同じです [ソートキー](mergetree.md) 値）。 -データ重複除去は、マージ中にのみ行われます。 マージは未知の時間にバックグラウンドで行われるため、計画することはできません。 一部のデータは未処理のままです。 スケジュールされていないマージを実行するには `OPTIMIZE` クエリは、それを使用してカウントされません。 `OPTIMIZE` クエリは大量のデータを読み書きします。 +データ重複除外は、マージ中にのみ発生します。 マージは未知の時間にバックグラウンドで発生するため、計画することはできません。 一部のデータは未処理のままになる場合があります。 スケジュールされていないマージを実行するには `OPTIMIZE` クエリは、それを使用してカウントされませんので、 `OPTIMIZE` クエリは大量のデータを読み書きします。 -したがって, `ReplacingMergeTree` に適した清算出重複データを背景に保存するための空間が保証するものではありませんが重複している。 +従って, `ReplacingMergeTree` に適した清算出重複データを背景に保存するための空間が保証するものではありませんが重複している。 ## テーブルの作成 {#creating-a-table} @@ -34,20 +33,20 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] **ReplacingMergeTreeパラメータ** -- `ver` — column with version. Type `UInt*`, `Date` または `DateTime`. 省略可能なパラメータ。 +- `ver` — column with version. Type `UInt*`, `Date` または `DateTime`. 任意パラメータ。 - マージ時, `ReplacingMergeTree` 同じ主キーを持つすべての行から一つだけを残します: + マージ時, `ReplacingMergeTree` 同じ主キーを持つすべての行から、一つだけを残します: - - 選択の最後の場合 `ver` 設定されていません。 - - 最大バージョンでは、 `ver` 指定します。 + - 選択範囲の最後にある場合 `ver` 設定されていません。 + - 最大バージョンでは、次の場合 `ver` 指定。 **クエリ句** -作成するとき `ReplacingMergeTree` テーブル同じ [句](mergetree.md) 作成するときと同じように、必須です。 `MergeTree` テーブル。 +を作成するとき `ReplacingMergeTree` 同じテーブル [句](mergetree.md) を作成するときのように必要です。 `MergeTree` テーブル。
-テーブルを作成する非推奨の方法 +推奨されていません法テーブルを作成する !!! attention "注意" 可能であれば、古いプロジェクトを上記の方法に切り替えてください。 @@ -61,9 +60,9 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] ) ENGINE [=] ReplacingMergeTree(date-column [, sampling_expression], (primary, key), index_granularity, [ver]) ``` -すべてのパラメーターを除く `ver` と同じ意味を持つ `MergeTree`. +以下を除くすべてのパラメータ `ver` と同じ意味を持つ `MergeTree`. -- `ver` -バージョンの列。 省略可能なパラメータ。 説明は上記のテキストを参照してください。 +- `ver` -バージョンの列。 任意パラメータ。 説明は、上記のテキストを参照してください。
diff --git a/docs/ja/engines/table-engines/mergetree-family/replication.md b/docs/ja/engines/table-engines/mergetree-family/replication.md index 6e2c6e354f1..bcc5de5e364 100644 --- a/docs/ja/engines/table-engines/mergetree-family/replication.md +++ b/docs/ja/engines/table-engines/mergetree-family/replication.md @@ -1,42 +1,42 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 31 toc_title: "\u30C7\u30FC\u30BF\u8907\u88FD" --- # データ複製 {#table_engines-replication} -複製がサポートされる唯一のためのテーブルのmergetree家族: +複製がサポートされる唯一のためのテーブルのMergeTree家族: -- レプリケートされたmergetree -- ﾂつｨﾂ姪“ﾂつ”ﾂ債ﾂづｭﾂつｹﾂ-faq -- ﾂつｨﾂ姪“ﾂつ”ﾂ債ﾂづｭﾂつｹﾂ-faq -- ﾂつｨﾂ姪“ﾂつ”ﾂ債ﾂづｭﾂつｹﾂ-faq -- レプリケートされたcollapsingmergetree +- 複製マージツリー +- 複製されたサミングマージツリー +- レプリケートリプレースマージツリー +- 複製された集合マージツリー +- レプリケートコラプシングマージツリー - ReplicatedVersionedCollapsingMergetree -- ReplicatedGraphiteMergeTree +- レプリケートグラフィティマージツリー -複製は、サーバー全体ではなく、個々のテーブルのレベルで機能します。 サーバーでの店舗も複製、非複製のテーブルでも同時に行います。 +複製の作品のレベルを個別のテーブルではなく、全体のサーバーです。 サーバーでの店舗も複製、非複製のテーブルでも同時に行います。 -複製はシャーディングに依存しません。 各シャードには独自の独立した複製があります。 +複製はシャーディングに依存しません。 各シャードには、独自の独立した複製があります。 -圧縮データのための `INSERT` と `ALTER` クエリを複製(詳細については、ドキュメンテーションに [ALTER](../../../sql-reference/statements/alter.md#query_language_queries_alter)). +圧縮されたデータ `INSERT` と `ALTER` クエリを複製(詳細については、ドキュメンテーションに [ALTER](../../../sql-reference/statements/alter.md#query_language_queries_alter)). `CREATE`, `DROP`, `ATTACH`, `DETACH` と `RENAME` クエリは単一サーバーで実行され、レプリケートされません: -- その `CREATE TABLE` queryは、クエリが実行されるサーバー上に新しい複製可能テーブルを作成します。 このテーブルが既にあるその他のサーバーを加え新たなレプリカ. -- その `DROP TABLE` クエリは、クエリが実行されているサーバー上のレプリカを削除します。 -- その `RENAME` queryは、いずれかのレプリカでテーブルの名前を変更します。 つまり、複製のテーブルでの異なる名称の異なるレプリカ. +- その `CREATE TABLE` クエリは、クエリが実行されるサーバー上に新しい複製可能テーブルを作成します。 このテーブルが既にあるその他のサーバーを加え新たなレプリカ. +- その `DROP TABLE` クエリは、クエリが実行されるサーバー上にあるレプリカを削除します。 +- その `RENAME` queryは、レプリカのいずれかのテーブルの名前を変更します。 つまり、複製のテーブルでの異なる名称の異なるレプリカ. -ClickHouse用 [アパッチの飼育係](https://zookeeper.apache.org) レプリカのメタ情報を格納するため。 使用ZooKeeperバージョン3.4.5以降。 +ClickHouseの使用 [アパッチの飼育係](https://zookeeper.apache.org) レプリカのメタ情報を格納するため。 ZooKeeperバージョン3.4.5以降を使用します。 -レプリケーションを使用するには、 [zookeeper](../../../operations/server-configuration-parameters/settings.md#server-settings_zookeeper) サーバー構成セクション. +レプリケーションを使用するには、 [飼育係](../../../operations/server-configuration-parameters/settings.md#server-settings_zookeeper) サーバー構成セクション。 !!! attention "注意" - セキュリ クリックハウスは `digest` [ACLスキーム](https://zookeeper.apache.org/doc/current/zookeeperProgrammers.html#sc_ZooKeeperAccessControl) ZooKeeperのセキュリティサブシステムの + セキュリティ設定を無視しないでください クリックハウスは `digest` [ACLスキーム](https://zookeeper.apache.org/doc/current/zookeeperProgrammers.html#sc_ZooKeeperAccessControl) ZooKeeperセキュリティサブシステムの。 -ZooKeeperクラスタのアドレス設定例: +ZooKeeperクラスタのアドレスを設定する例: ``` xml @@ -55,40 +55,40 @@ ZooKeeperクラスタのアドレス設定例: ``` -既存のzookeeperクラスターを指定すると、システムは独自のデータ用のディレクトリを使用します（レプリケート可能なテーブルを作成するときにディレクトリ +既存のZooKeeperクラスタを指定すると、システムはそのクラスタ上のディレクトリを独自のデータとして使用します(複製可能なテーブルを作成するときに -ZooKeeperが設定ファイルに設定されていない場合は、複製されたテーブルを作成することはできず、既存の複製されたテーブルは読み取り専用になります。 +場合飼育係な設定コンフィグファイルを創り上げられないんで再現しテーブル、および既存の複製のテーブル読み取り専用になります。 -ZooKeeperは使用されません `SELECT` レプリケーショ `SELECT` との質問を行ってい非再現します。 分散レプリケートテーブルを照会する場合、ClickHouseの動作は設定によって制御されます [max\_replica\_delay\_for\_distributed\_queries](../../../operations/settings/settings.md#settings-max_replica_delay_for_distributed_queries) と [fallback\_to\_stale\_replicas\_for\_distributed\_queries](../../../operations/settings/settings.md#settings-fallback_to_stale_replicas_for_distributed_queries). +飼育係はで使用されていません `SELECT` レプリケーションのパフォーマン `SELECT` また、クエリは非レプリケートテーブルの場合と同様に高速に実行されます。 時の照会に配布再現し、テーブルClickHouse行動制御の設定 [max\_replica\_delay\_for\_distributed\_queries](../../../operations/settings/settings.md#settings-max_replica_delay_for_distributed_queries) と [フォールバック\_to\_stale\_replicas\_for\_distributed\_queries](../../../operations/settings/settings.md#settings-fallback_to_stale_replicas_for_distributed_queries). -それぞれの `INSERT` クエリー、契約時に応募を追加飼育係を務取引等 (より正確には、これはデータの挿入された各ブロックに対するものです。 `max_insert_block_size = 1048576` 行。）これは、 `INSERT` と比較して再現します。 しかし、推奨事項に従ってデータを複数のバッチで挿入する場合 `INSERT` 毎秒、それは問題を作成しない。 全体のClickHouseクラスターの使用のための調整一飼育係のクラスタでは、合計数百 `INSERTs` 秒あたり。 データ挿入のスループット(秒あたりの行数)は、レプリケートされていないデータと同じくらい高くなります。 +それぞれのため `INSERT` クエリー、契約時に応募を追加飼育係を務取引等 （より正確には、これは挿入されたデータの各ブロックに対するものです。 `max_insert_block_size = 1048576` 行。)これは、 `INSERT` 非レプリケートテーブルと比較します。 しかし、推奨事項に従ってデータを複数のバッチで挿入する場合 `INSERT` 毎秒、それは問題を作成しません。 一つのZooKeeperクラスターを調整するために使用されるClickHouseクラスター全体の合計は数百です `INSERTs` 毎秒 データ挿入のスループット(秒あたりの行数)は、レプリケートされていないデータの場合と同じくらい高くなります。 -のための非常に大きなクラスターで異なるクラスター飼育係の異なる破片. しかし、これはyandexで必要なことは証明されていません。metricaクラスター(約300台のサーバー)。 +非常に大きなクラスタの場合、異なるシャードに異なるZooKeeperクラスタを使用できます。 しかし、これはYandexの上で必要な証明されていません。メトリカクラスタ（約300台）。 -複製は非同期、マルチます。 `INSERT` クエリ（と同様 `ALTER`）利用可能な任意のサーバーに送信することができます。 クエリが実行されているサーバーにデータが挿入され、そのデータが他のサーバーにコピーされます。 非同期であるため、最近挿入されたデータが他のレプリカに何らかの遅延で表示されます。 レプリカの一部が使用できない場合、データは使用できるようになった時点で書き込まれます。 レプリカが使用可能な場合、待機時間は、圧縮されたデータのブロックをネットワーク経由で転送するのにかかる時間です。 +複製は非同期でマルチマスターです。 `INSERT` クエリ(および `ALTER`）利用可能な任意のサーバに送信することができます。 データは、クエリが実行されるサーバーに挿入され、他のサーバーにコピーされます。 ですので非同期であるため、最近では挿入されデータが表示され、その他のレプリカとの待ち時間をゼロにすることに レプリカの一部が使用できない場合、データは使用可能になったときに書き込まれます。 レプリカが使用可能な場合、待機時間は、圧縮されたデータのブロックをネットワーク経由で転送するのにかかる時間です。 -既定では、挿入クエリは、単一のレプリカからのデータの書き込みの確認を待機します。 データが正常に単一のレプリカに書き込まれ、このレプリカを持つサーバーが存在しなくなると、格納されたデータは失われます。 複数のレプリカからデー `insert_quorum` オプション。 +既定では、挿入クエリは、単一のレプリカからのデータの書き込みの確認を待機します。 データが得られない場合には成功した記述につのレプリカのサーバーのこのレプリカが消滅し、保存したデータは失われます。 複数のレプリカからのデータ書き込みの確認の取得を有効にするには、 `insert_quorum` オプション -データの各ブロックは原子的に書き込まれます。 挿入クエリは、以下のブロックに分割されます `max_insert_block_size = 1048576` 行。 言い換えれば、 `INSERT` クエリは、それが原子的に作られ、1048576未満の行を持っています。 +データの各ブロックは原子的に書き込まれます。 挿入クエリは、次のブロックに分割されます `max_insert_block_size = 1048576` 行。 言い換えれば、 `INSERT` クエリには1048576行未満があり、アトミックに作成されます。 -データブロックは重複除外されます。 同じデータブロック（同じ順序で同じ行を含む同じサイズのデータブロック）の複数書き込みの場合、ブロックは一度だけ書き込まれます。 この理由は、クライアントアプリケーションがデータがdbに書き込まれたかどうかを知らない場合のネットワーク障害の場合です。 `INSERT` クエリーするだけで簡単に繰り返します。 どのレプリカ挿入が同じデータで送信されたかは関係ありません。 `INSERTs` べき等である。 重複排除圧縮パラメータの制御 [merge\_tree](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-merge_tree) サーバー設定。 +データブロックは重複排除されます。 同じデータブロック（同じ順序で同じ行を含む同じサイズのデータブロック）の複数の書き込みの場合、ブロックは一度だけ書き込まれます。 この理由は、クライアントアプリケーションがデータがDBに書き込まれたかどうかを知らないときにネットワーク障害が発生した場合です。 `INSERT` クエリーするだけで簡単に繰り返します。 どのレプリカ挿入が同一のデータで送信されたかは関係ありません。 `INSERTs` 冪等である。 重複排除圧縮パラメータの制御 [merge\_tree](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-merge_tree) サーバー設定。 -レプリケーショ さらなるデータ変換（マージ）は、すべてのレプリカで同じ方法で調整され、実行されます。 これにより、ネットワークの使用を最小限に抑えることができます。 (複製の主な目的は、異なるデータセンター内のデータを複製することです。) +複製時に、元のデータの挿入には転送されます。 さらなるデータ変換(マージ)は、すべてのレプリカで同じ方法で調整され、実行されます。 つまり、レプリカが異なるデータセンターに存在する場合、レプリケーションは適切に機能します。 レプリケーションの主な目的は、異なるデータセンターでデータを複製することです。) -同じデータの任意の数のレプリカを持つことができます。 yandexの。metricaは、本番環境で二重の複製を使用します。 各サーバーはraid-5またはraid-6を使用し、場合によってはraid-10を使用します。 これは比較的信頼性が高く便利な解決策です。 +同じデータの任意の数のレプリカを持つことができます。 Yandex.Metricaは本番環境で二重複製を使用します。 各サーバーはRAID-5またはRAID-6を使用し、場合によってはRAID-10を使用します。 これは比較的信頼性が高く便利な解決策です。 -システムは、レプリカ上のデータ同期性を監視し、障害発生後に回復することができます。 フェールオーバーは、自動(データのわずかな差異の場合)または半自動(データが大きく異なる場合、構成エラーを示す可能性があります)です。 +システムは、レプリカのデータ同期を監視し、障害発生後に回復することができます。 フェールオーバーは、自動(データのわずかな違いの場合)または半自動(データの異なりが多すぎる場合、構成エラーを示す可能性があります)です。 ## 複製テーブルの作成 {#creating-replicated-tables} その `Replicated` テーブルエンジン名に接頭辞が追加されます。 例えば:`ReplicatedMergeTree`. -**複製\*マージツリーパラメータ** +**複製\*MergeTreeパラメータ** - `zoo_path` — The path to the table in ZooKeeper. - `replica_name` — The replica name in ZooKeeper. -例えば: +例: ``` sql CREATE TABLE table_name @@ -104,7 +104,7 @@ SAMPLE BY intHash32(UserID)
-非推奨構文の例 +非推奨の構文の例 ``` sql CREATE TABLE table_name @@ -117,7 +117,7 @@ CREATE TABLE table_name
-その例としては、これらのパラメータを含むことができ換巻きていただけるボディーです。 置換された価値はから取られます ‘macros’ 設定ファイルのセクション。 例えば: +その例としては、これらのパラメータを含むことができ換巻きていただけるボディーです。 置き換えられた値は、 ‘macros’ 設定ファイルのセクション。 例: ``` xml @@ -130,40 +130,40 @@ CREATE TABLE table_name の表の飼育係るべきで機能していませんが将来的には再現します。 テーブルの異なる資料は異なる。 この場合、パスは次の部分で構成されます: -`/clickhouse/tables/` は共通の接頭辞です。 の使用をお勧めしまうことです。 +`/clickhouse/tables/` 共通の接頭辞です。 の使用をお勧めしまうことです。 `{layer}-{shard}` シャード識別子です。 この例では、Yandexので、二つの部分で構成されています。Metricaクラスターの使用インターネット上のファイル転送sharding. ほとんどのタスクでは、{shard}置換だけを残すことができます。 -`table_name` ZooKeeperのテーブルのノードの名前です。 テーブル名と同じにすることをお勧めします。 テーブル名とは対照的に、名前の変更クエリの後に変更されないため、明示的に定義されています。 +`table_name` ZooKeeper内のテーブルのノード名です。 テーブル名と同じにすることをお勧めします。 これは、テーブル名とは対照的に、名前変更クエリの後に変更されないため、明示的に定義されています。 *HINT*：データベース名を追加することができます `table_name` 同様に。 例えば `db_name.table_name` -レプリカ名は同じテーブルの別のレプリカを識別します。 この例のように、このサーバー名を使用できます。 名前は各シャード内で一意である必要があります。 +のレプリカの名前を識別のレプリカと同じ。 これには、例のようにサーバー名を使用できます。 名前は、各シャード内で一意である必要があります。 -置換を使用する代わりに、パラメーターを明示的に定義できます。 これは、テストや小さなクラスターの構成に便利です。 ただし、分散ddlクエリは使用できません (`ON CLUSTER`)この場合。 +置換を使用する代わりに、パラメーターを明示的に定義できます。 これは、テストや小さなクラスターの構成に便利です。 ただし、分散DDLクエリは使用できません (`ON CLUSTER`）この場合。 -組み合わせによる方がクラスターの使用をお勧めいたしま換その可能性を低減するにはエラーになります。 +大規模なクラスターで作業する場合は、エラーの可能性を減らすため、置換を使用することをお勧めします。 -実行する `CREATE TABLE` 各レプリカに対するクエリ。 このクエ +実行 `CREATE TABLE` 各レプリカのクエリ。 このクエリを複製テーブルを追加し、新たなレプリカは、既存します。 -テーブルに他のレプリカのデータがすでに含まれている後に新しいレプリカを追加すると、データはクエリの実行後に他のレプリカから新しいレプリ つまり、新しいレプリカは他のレプリカと同期します。 +テーブルに他のレプリカのデータがすでに含まれている後に新しいレプリカを追加すると、クエリの実行後にデータが他のレプリカから新しいレプリカ つまり、新しいレプリカは他のレプリカと同期します。 -レプリカを削除するには `DROP TABLE`. However, only one replica is deleted – the one that resides on the server where you run the query. +レプリカを削除するには、 `DROP TABLE`. However, only one replica is deleted – the one that resides on the server where you run the query. -## 障害後の復旧 {#recovery-after-failures} +## 障害後の回復 {#recovery-after-failures} -場合飼育係が不可の場合、サーバは、複製のテーブルスイッチ読み取り専用モードになります。 システムは定期的にzookeeperに接続しようとします。 +場合飼育係が不可の場合、サーバは、複製のテーブルスイッチ読み取り専用モードになります。 システムは定期的にZooKeeperへの接続を試みます。 -ZooKeeperが使用中に利用できない場合 `INSERT`、またはZooKeeperとやり取りするとエラーが発生し、例外がスローされます。 +飼育係が中に使用できない場合 `INSERT`、またはZooKeeperとの対話時にエラーが発生すると、例外がスローされます。 -ZooKeeperに接続した後、システムはローカルファイルシステムのデータセットが期待されるデータセットと一致するかどうかをチェックします（ZooKeeperはこの情報 小さな不整合がある場合、システムはデータをレプリカと同期することで解決します。 +ZooKeeperに接続すると、ローカルファイルシステム内のデータセットが予想されるデータセットと一致するかどうかがチェックされます(ZooKeeperはこの情報を格納し がある場合は軽微な不整合の解消による同期データのレプリカ. -システムが壊れたデータ部分（ファイルのサイズが間違っている）または認識されない部分（ファイルシステムに書き込まれたがzookeeperに記録されていな `detached` サブディレクトリ(削除されません)。 他の部分がコピーからのレプリカ. +システムが壊れたデータ部分(ファイルのサイズが間違っている)または認識できない部分(ファイルシステムに書き込まれているが、ZooKeeperに記録されて `detached` サブディレクトリ(削除されません)。 他の部分がコピーからのレプリカ. -ClickHouseは大量のデータを自動的に削除するなどの破壊的な操作を実行しません。 +ClickHouseは、大量のデータを自動的に削除するなどの破壊的な操作は実行しません。 -サーバーが起動（またはzookeeperとの新しいセッションを確立）すると、すべてのファイルの量とサイズのみをチェックします。 ファイルサイズが一致しているが、バイトが途中で変更されている場合、これはすぐには検出されません。 `SELECT` クエリ。 クエリは、一致しないチェックサムまたは圧縮ブロックのサイズに関する例外をスローします。 この場合、データパーツは検証キューに追加され、必要に応じてレプリカからコピーされます。 +サーバが起動時に表示されます(または新たに設立し、セッションとの飼育係）でのみチェックの量やサイズのすべてのファイルです。 ファイルサイズは一致するが、途中でバイトが変更された場合、これはすぐには検出されず、データを読み取ろうとしたときにのみ検出されます。 `SELECT` クエリ。 クエリが例外をスローしつつ、非マッチングのチェックサムはサイズに圧縮されたブロックです。 この場合、データパーツは検証キューに追加され、必要に応じてレプリカからコピーされます。 -データのローカルセットが予想されるセットと大きく異なる場合は、安全機構がトリガーされます。 サーバーはこれをログに入力し、起動を拒否します。 この理由は、シャード上のレプリカが別のシャード上のレプリカのように誤って構成された場合など、このケースが構成エラーを示している可能性がある しかし、しきい値をこの機構の設定かなり低く、こうした状況が起こる中で、失敗を回復しました。 この場合、データは半自動的に復元されます。 “pushing a button”. +場合には地元のデータセットが異なりすぎとられ、安全機構を起動します。 サーバーはこれをログに入力し、起動を拒否します。 この理由は、シャード上のレプリカが別のシャード上のレプリカのように誤って構成された場合など、このケースが構成エラーを示す可能性があるためです。 しかし、しきい値をこの機構の設定かなり低く、こうした状況が起こる中で、失敗を回復しました。 この場合、データは半自動的に復元されます。 “pushing a button”. 回復を開始するには、ノードを作成します `/path_to_table/replica_name/flags/force_restore_data` で飼育係とコンテンツ、またはコマンドを実行し復元すべての複製のテーブル: @@ -171,48 +171,48 @@ ClickHouseは大量のデータを自動的に削除するなどの破壊的な sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data ``` -次に、サーバーを再起動します。 開始時に、サーバーはこれらのフラグを削除し、回復を開始します。 +次に、サーバーを再起動します。 起動時に、サーバーはこれらのフラグを削除し、回復を開始します。 -## 完全なデータの損失後の回復 {#recovery-after-complete-data-loss} +## 完全なデータ損失の後の回復 {#recovery-after-complete-data-loss} すべてのデータやメタデータ消えたらサーバには、次の手順に従ってください復興: -1. サーバーにclickhouseをインストール. シャード識別子とレプリカを含むコンフィグファイルで置換を正しく定義します。 -2. サーバー上で手動で複製する必要のある複雑でないテーブルがある場合は、ディレクトリ内のレプリカからデータをコピーします `/var/lib/clickhouse/data/db_name/table_name/`). -3. にあるテーブル定義のコピー `/var/lib/clickhouse/metadata/` レプリカから。 テーブル定義でシャードまたはレプリカ識別子が明示的に定義されている場合は、このレプリカに対応するように修正します。 （あるいは、サーバーを起動してすべての `ATTACH TABLE` にあったはずのクエリ。のsqlファイル `/var/lib/clickhouse/metadata/`.) -4. 回復を開始するには、zookeeperノードを作成します `/path_to_table/replica_name/flags/force_restore_data` 他のコンテンツ、またはコマンドを実行し復元すべての複製のテーブル: `sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data` +1. ClickHouseをサーバにインストールします。 シャード識別子とレプリカを使用する場合は、設定ファイルで置換を正しく定義します。 +2. サーバー上で手動で複製する必要がある未複製のテーブルがある場合は、レプリカからデータをコピーします(ディレクトリ内)。 `/var/lib/clickhouse/data/db_name/table_name/`). +3. テーブル定義のコピー `/var/lib/clickhouse/metadata/` レプリカから。 シャード識別子またはレプリカ識別子がテーブル定義で明示的に定義されている場合は、このレプリカに対応するように修正します。 (または、サーバーを起動し、すべての `ATTACH TABLE` にあったはずのクエリ.sqlファイル `/var/lib/clickhouse/metadata/`.) +4. 回復を開始するには、ZooKeeperノードを作成します `/path_to_table/replica_name/flags/force_restore_data` コマンドを実行してレプリケートされたテーブルをすべて復元します: `sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data` -その後、サーバーを起動します（既に実行されている場合は再起動します）。 デー +次に、サーバーを起動します（すでに実行されている場合は再起動します）。 データダウンロードからのレプリカ. -代替の回復オプションは削除に関する情報は失われたレプリカから飼育係 (`/path_to_table/replica_name`)、レプリカを再度作成します。 “[複製テーブルの作成](#creating-replicated-tables)”. +代替の回復オプションは削除に関する情報は失われたレプリカから飼育係 (`/path_to_table/replica_name`)で説明したように、レプリカを再度作成します “[複製テーブルの作成](#creating-replicated-tables)”. -リカバリ中のネットワーク帯域幅に制限はありません。 一度に多くのレプリカを復元する場合は、この点に留意してください。 +復旧時のネットワーク帯域幅に制限はありません。 一度に多数のレプリカを復元する場合は、この点に注意してください。 -## MergetreeからReplicatedmergetreeへの変換 {#converting-from-mergetree-to-replicatedmergetree} +## MergeTreeからReplicatedMergeTreeへの変換 {#converting-from-mergetree-to-replicatedmergetree} -我々はこの用語を使用する： `MergeTree` のすべてのテーブルエンジンを参照するには `MergeTree family`、の場合と同じ `ReplicatedMergeTree`. +我々はこの用語を使用する `MergeTree` すべてのテーブルエンジンを参照するには `MergeTree family` の場合と同じです。 `ReplicatedMergeTree`. -あなたが持っていた場合 `MergeTree` したテーブルを手動で再現でき換で再現します。 すでに大量のデータを収集している場合は、これを行う必要があります `MergeTree` これで、レプリケーションを有効にします。 +あなたが持っていた場合 `MergeTree` 手動で複製されたテーブルは、複製されたテーブルに変換することができます。 すでに大量のデータを収集している場合は、これを行う必要があるかもしれません。 `MergeTree` レプリケーションを有効にします。 -さまざまなレプリカでデータが異なる場合は、最初に同期するか、レプリカ以外のすべてのデータを削除します。 +さまざまなレプリカでデータが異なる場合は、最初に同期するか、このデータを除くすべてのレプリカで削除します。 -既存のmergetreeテーブルの名前を変更し、 `ReplicatedMergeTree` 古い名前のテーブル。 -古いテーブルからデータを移動する `detached` サブディレクトリ内のディレクトリを新しいテーブルデータ (`/var/lib/clickhouse/data/db_name/table_name/`). -その後、実行 `ALTER TABLE ATTACH PARTITION` 作業セットにこれらのデータ部分を追加するレプリカのいずれか。 +既存のMergeTreeテーブルの名前を変更し、次に `ReplicatedMergeTree` 古い名前のテーブル。 +データを古いテーブルから `detached` サブディレクトリ内のディレクトリを新しいテーブルデータ (`/var/lib/clickhouse/data/db_name/table_name/`). +その後、実行 `ALTER TABLE ATTACH PARTITION` これらのデータパーツを作業セットに追加するレプリカのいずれかで。 -## ReplicatedmergetreeからMergetreeへの変換 {#converting-from-replicatedmergetree-to-mergetree} +## ReplicatedMergeTreeからMergeTreeへの変換 {#converting-from-replicatedmergetree-to-mergetree} -別の名前のmergetreeテーブルを作成します。 ディレクトリからすべてのデータを `ReplicatedMergeTree` テーブルデータを新しいテーブルのデータディレクトリです。 次に、 `ReplicatedMergeTree` テーブルとサーバーを再起動します。 +別の名前のMergeTreeテーブルを作成します。 ディレクトリからすべてのデータを移動します。 `ReplicatedMergeTree` テーブルデータを新しいテーブルのデータディレクトリです。 その後、削除 `ReplicatedMergeTree` サーバーを表にして再起動します。 -あなたが取り除きたいなら `ReplicatedMergeTree` サーバーを起動せずにテーブル: +あなたが取り除きたい場合は、 `ReplicatedMergeTree` サーバーを起動しないテーブル: -- 対応するものを削除する `.sql` メタデータディレク (`/var/lib/clickhouse/metadata/`). -- ZooKeeperの対応するパスを削除します (`/path_to_table/replica_name`). +- 対応するものを削除する `.sql` ファイルのメタデータディレクトリ (`/var/lib/clickhouse/metadata/`). +- ZooKeeperで対応するパスを削除します (`/path_to_table/replica_name`). -この後、サーバーを起動し、 `MergeTree` テーブル、そのディレクトリにデータを移動し、サーバーを再起動します。 +この後、サーバーを起動し、 `MergeTree` データをそのディレクトリに移動し、サーバーを再起動します。 -## Zookeeperクラスター内のメタデータが失われたり破損した場合の回復 {#recovery-when-metadata-in-the-zookeeper-cluster-is-lost-or-damaged} +## Zookeeperクラスター内のメタデータが紛失または破損した場合の復旧 {#recovery-when-metadata-in-the-zookeeper-cluster-is-lost-or-damaged} -ZooKeeper内のデータが失われたり破損したりした場合は、上記のように単純なテーブルに移動してデータを保存することができます。 +ZooKeeperのデータが紛失または破損している場合は、上記のように再生されていないテーブルに移動することでデータを保存できます。 [元の記事](https://clickhouse.tech/docs/en/operations/table_engines/replication/) diff --git a/docs/ja/engines/table-engines/mergetree-family/summingmergetree.md b/docs/ja/engines/table-engines/mergetree-family/summingmergetree.md index 678f83a6502..356b10037d2 100644 --- a/docs/ja/engines/table-engines/mergetree-family/summingmergetree.md +++ b/docs/ja/engines/table-engines/mergetree-family/summingmergetree.md @@ -1,15 +1,15 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 34 -toc_title: SummingMergeTree +toc_title: "\u30B5\u30DF\u30F3\u30B0\u30DE\u30FC\u30B2\u30C4\u30EA\u30FC" --- -# Summingmergetree {#summingmergetree} +# サミングマーゲツリー {#summingmergetree} -エンジンは [MergeTree](mergetree.md#table_engines-mergetree). 違いは、データ部分をマージするとき `SummingMergeTree` テーブルClickHouseは、すべての行を同じ主キー（またはより正確には同じ）で置き換えます [ソートキー](mergetree.md))数値データ型を持つ列の集計値を含む行。 並べ替えキーが単一のキー値が多数の行に対応するように構成されている場合、これによりストレージボリュームが大幅に削減され、データ選択がスピードア +エンジンはから継承します [メルゲツリー](mergetree.md#table_engines-mergetree). 違いは、データ部分をマージするときに `SummingMergeTree` テーブルClickHouseは、すべての行を同じ主キー（またはより正確には同じキー）で置き換えます [ソートキー](mergetree.md)）数値データ型の列の集計値を含む行。 並べ替えキーが単一のキー値が多数の行に対応するように構成されている場合、ストレージ容量が大幅に削減され、データ選択が高速化されます。 -私たちは使用するエンジンと一緒に `MergeTree`. 完全なデータを格納する `MergeTree` テーブル、および使用 `SummingMergeTree` レポートを準備するときなど、集計データを保存する場合。 このようなアプローチは、誤って構成された主キーのために貴重なデー +エンジンを一緒に使用することをお勧めします `MergeTree`. 完全なデータを格納する `MergeTree` テーブルおよび使用 `SummingMergeTree` たとえば、レポートの準備時など、集計データを格納する場合。 このようなアプローチを防ぎまらな貴重なデータにより正しく構成しその有効なタイプを利用します。 ## テーブルの作成 {#creating-a-table} @@ -28,20 +28,20 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] 説明リクエストパラメータの参照 [要求の説明](../../../sql-reference/statements/create.md). -**SummingMergeTreeのパラメータ** +**Sumingmergetreeのパラメータ** -- `columns` -値が要約される列の名前を持つタプル。 省略可能なパラメータ。 - 列は数値型である必要があり、主キーに含めることはできません。 +- `columns` -値が集計される列の名前を持つタプル。 任意パラメータ。 + 列は数値型である必要があり、主キーには含まれていない必要があります。 - もし `columns` 指定されていない場合、ClickHouseは、プライマリキーに含まれていない数値データ型を持つすべての列の値を集計します。 + もし `columns` 指定されていないClickHouseは、主キーにない数値データ型を持つすべての列の値を集計します。 **クエリ句** -作成するとき `SummingMergeTree` テーブル同じ [句](mergetree.md) 作成するときと同じように、必須です。 `MergeTree` テーブル。 +を作成するとき `SummingMergeTree` 同じテーブル [句](mergetree.md) を作成するときのように必要です。 `MergeTree` テーブル。
-テーブルを作成する非推奨の方法 +推奨されていません法テーブルを作成する !!! attention "注意" 可能であれば、古いプロジェクトを上記の方法に切り替えてください。 @@ -55,7 +55,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] ) ENGINE [=] SummingMergeTree(date-column [, sampling_expression], (primary, key), index_granularity, [columns]) ``` -すべてのパラメーターを除く `columns` と同じ意味を持つ `MergeTree`. +以下を除くすべてのパラメータ `columns` と同じ意味を持つ `MergeTree`. - `columns` — tuple with names of columns values of which will be summarized. Optional parameter. For a description, see the text above. @@ -63,7 +63,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] ## 使用例 {#usage-example} -次の表を考えてみます: +次の表を考えます: ``` sql CREATE TABLE summtt @@ -75,13 +75,13 @@ ENGINE = SummingMergeTree() ORDER BY key ``` -それにデータを挿入する: +データを挿入する: ``` sql INSERT INTO summtt Values(1,1),(1,2),(2,1) ``` -ClickHouseは完全ではないすべての行を合計してもよい ([以下を参照](#data-processing)）ので、我々は集計関数を使用します `sum` と `GROUP BY` クエリ内の句。 +ClickHouseは完全ではないすべての行を合計可能性があります ([以下を参照](#data-processing)）ので、集計関数を使用します `sum` と `GROUP BY` クエリ内の句。 ``` sql SELECT key, sum(value) FROM summtt GROUP BY key @@ -96,9 +96,9 @@ SELECT key, sum(value) FROM summtt GROUP BY key ## データ処理 {#data-processing} -データがテーブルに挿入されると、そのまま保存されます。 これは、同じプライマリキーを持つ行が合計され、結果のデータ部分ごとに行が置き換えられたときです。 +データがテーブルに挿入されると、そのまま保存されます。 ClickHouseは、データの挿入された部分を定期的にマージし、これは、同じ主キーを持つ行が合計され、結果として得られるデータの各部分に対して行に置き換えられる -ClickHouse can merge the data parts so that different resulting parts of data cat consist rows with the same primary key, i.e. the summation will be incomplete. Therefore (`SELECT`)集計関数 [合計()](../../../sql-reference/aggregate-functions/reference.md#agg_function-sum) と `GROUP BY` 上記の例で説明したように、クエリで句を使用する必要があります。 +ClickHouse can merge the data parts so that different resulting parts of data cat consist rows with the same primary key, i.e. the summation will be incomplete. Therefore (`SELECT`)集計関数 [和()](../../../sql-reference/aggregate-functions/reference.md#agg_function-sum) と `GROUP BY` 上記の例で説明したように、クエリで句を使用する必要があります。 ### 合計の共通ルール {#common-rules-for-summation} @@ -106,13 +106,13 @@ ClickHouse can merge the data parts so that different resulting parts of data ca 合計のすべての列の値が0の場合、行は削除されます。 -列が主キーに含まれておらず、まとめられていない場合は、既存の値から任意の値が選択されます。 +Columnが主キーになく、集計されていない場合は、既存の値から任意の値が選択されます。 主キーの列の値は集計されません。 ### Aggregatefunction列の合計 {#the-summation-in-the-aggregatefunction-columns} -列の場合 [AggregateFunctionタイプ](../../../sql-reference/data-types/aggregatefunction.md) クリックハウスは [ﾂつｨﾂ姪“ﾂつ”ﾂ債ﾂづｭﾂつｹ](aggregatingmergetree.md) 機能に従って集約するエンジン。 +の列に対して [AggregateFunctionタイプ](../../../sql-reference/data-types/aggregatefunction.md) ClickHouseとして振る舞うと考えられてい [AggregatingMergeTree](aggregatingmergetree.md) 機能に従って集計するエンジン。 ### 入れ子構造 {#nested-structures} @@ -120,10 +120,10 @@ ClickHouse can merge the data parts so that different resulting parts of data ca 入れ子になったテーブルの名前が `Map` また、以下の条件を満たす少なくとも二つの列が含まれています: -- 最初の列は数値です `(*Int*, Date, DateTime)` または文字列 `(String, FixedString)`、それを呼びましょう `key`, -- 他の列は算術演算です `(*Int*, Float32/64)`、それを呼びましょう `(values...)`, +- 最初の列は数値です `(*Int*, Date, DateTime)` または文字列 `(String, FixedString)` それを呼びましょう `key`, +- 他の列は算術演算です `(*Int*, Float32/64)` それを呼びましょう `(values...)`, -次に、このネストされたテーブルは、 `key => (values...)` 行をマージすると、二つのデータセットの要素は次のようにマージされます `key` 対応する `(values...)`. +次に、この入れ子になったテーブルは `key => (values...)` その行をマージするとき、二つのデータセットの要素は `key` 対応するの合計と `(values...)`. 例: @@ -134,8 +134,8 @@ ClickHouse can merge the data parts so that different resulting parts of data ca [(1, 100), (2, 150)] + [(1, -100)] -> [(2, 150)] ``` -データを要求するときは、 [sumMap(キー,値)](../../../sql-reference/aggregate-functions/reference.md) の集約のための関数 `Map`. +データを要求するときは、 [sumMap(キー、値)](../../../sql-reference/aggregate-functions/reference.md) の集合のための関数 `Map`. -入れ子になったデータ構造の場合、合計の列のタプルに列を指定する必要はありません。 +入れ子になったデータ構造の場合、合計のために列のタプルにその列を指定する必要はありません。 [元の記事](https://clickhouse.tech/docs/en/operations/table_engines/summingmergetree/) diff --git a/docs/ja/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md b/docs/ja/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md index 5371fbffdf4..2d323629c4e 100644 --- a/docs/ja/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md +++ b/docs/ja/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md @@ -1,20 +1,21 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 37 -toc_title: VersionedCollapsingMergeTree +toc_title: "\u30D0\u30FC\u30B8\u30E7\u30CB\u30F3\u30B0\u30B3\u30E9\u30D7\u30B7\u30F3\ + \u30B0\u30DE\u30FC\u30B2\u30C3\u30C8\u30EA\u30FC" --- -# Versionedcollapsingmergetree {#versionedcollapsingmergetree} +# バージョニングコラプシングマーゲットリー {#versionedcollapsingmergetree} このエンジン: - では迅速書き込みオブジェクトとは常に変化しています。 -- バックグラウン これを大幅に削減量に保管します。 +- 削除古いオブジェクト状態の背景になります。 これにより、保管量が大幅に削減されます。 -セクションを見る [折りたたみ](#table_engines_versionedcollapsingmergetree) 詳細については。 +セクションを参照 [崩壊](#table_engines_versionedcollapsingmergetree) 詳細については. -エンジンは [MergeTree](mergetree.md#table_engines-mergetree) 追加した論理崩壊行のアルゴリズムのための統合データ部品です。 `VersionedCollapsingMergeTree` と同じ目的を果たす [CollapsingMergeTree](collapsingmergetree.md) が異なる崩壊のアルゴリズムを挿入し、データを任意の順番で複数のスレッド）。 特に、 `Version` 列は、間違った順序で挿入されていても、行を適切に折りたたむのに役立ちます。 対照的に, `CollapsingMergeTree` 厳密に連続した挿入のみを許可します。 +エンジンはから継承します [メルゲツリー](mergetree.md#table_engines-mergetree) 追加した論理崩壊行のアルゴリズムのための統合データ部品です。 `VersionedCollapsingMergeTree` と同じ目的を果たしています [折りたたみマージツリー](collapsingmergetree.md) が異なる崩壊のアルゴリズムを挿入し、データを任意の順番で複数のスレッド）。 特に、 `Version` 列は、間違った順序で挿入されても、行を適切に折りたたむのに役立ちます。 対照的に, `CollapsingMergeTree` 厳密に連続した挿入のみを許可します。 ## テーブルの作成 {#creating-a-table} @@ -31,7 +32,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] [SETTINGS name=value, ...] ``` -説明のクエリパラメータの [クエリの説明](../../../sql-reference/statements/create.md). +クエリパラメータの説明については、 [クエリの説明](../../../sql-reference/statements/create.md). **エンジン変数** @@ -39,21 +40,21 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] VersionedCollapsingMergeTree(sign, version) ``` -- `sign` — Name of the column with the type of row: `1` は “state” 行, `-1` は “cancel” 行 +- `sign` — Name of the column with the type of row: `1` は “state” 行, `-1` は “cancel” ロウ - 列データ型は次のようになります `Int8`. + 列のデータ型は次のとおりです `Int8`. - `version` — Name of the column with the version of the object state. - 列データ型は次のようになります `UInt*`. + 列のデータ型は次のとおりです `UInt*`. **クエリ句** -作成するとき `VersionedCollapsingMergeTree` テーブル、同じ [句](mergetree.md) を作成するときに必要です。 `MergeTree` テーブル。 +を作成するとき `VersionedCollapsingMergeTree` テーブル、同じ [句](mergetree.md) を作成するときに必要です。 `MergeTree` テーブル。
-テーブルを作成する非推奨の方法 +推奨されていません法テーブルを作成する !!! attention "注意" 用途では使用しないでください方法で新規プロジェクト. 可能であれば、古いプロジェクトを上記の方法に切り替えます。 @@ -64,30 +65,30 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], ... -) ENGINE [=] VersionedCollapsingMergeTree(date-column [, sampling_expression], (primary, key), index_granularity, sign, version) +) ENGINE [=] VersionedCollapsingMergeTree(date-column [, samp#table_engines_versionedcollapsingmergetreeling_expression], (primary, key), index_granularity, sign, version) ``` 以下を除くすべてのパラメータ `sign` と `version` と同じ意味を持つ `MergeTree`. -- `sign` — Name of the column with the type of row: `1` は “state” 行, `-1` は “cancel” 行 +- `sign` — Name of the column with the type of row: `1` は “state” 行, `-1` は “cancel” ロウ Column Data Type — `Int8`. - `version` — Name of the column with the version of the object state. - 列データ型は次のようになります `UInt*`. + 列のデータ型は次のとおりです `UInt*`.
-## 折りたたみ {#table_engines_versionedcollapsingmergetree} +## 崩壊 {#table_engines_versionedcollapsingmergetree} ### データ {#data} -あるオブジェクトのデータを継続的に変更する必要がある状況を考えてみましょう。 オブジェクトに対して一つの行を持ち、変更があるときはいつでも行を更新するのが妥当です。 ただし、dbmsでは、ストレージ内のデータの書き換えが必要なため、更新操作は高価で低速です。 データをすばやく書き込む必要がある場合は更新できませんが、オブジェクトに変更を順番に書き込むことができます。 +うな状況を考える必要がある場合の保存継続的に変化するデータのオブジェクトです。 オブジェクトに対して一つの行を持ち、変更があるたびにその行を更新するのが妥当です。 ただし、DBMSではストレージ内のデータを書き換える必要があるため、更新操作はコストがかかり、時間がかかります。 更新はできませんが必要な場合は書き込みデータはすでに書き込み、変更オブジェクトの順にしております。 -を使用 `Sign` 行を書き込むときの列。 もし `Sign = 1` これは、行がオブジェクトの状態であることを意味します（それを呼び出しましょう “state” 行）。 もし `Sign = -1` これは、同じ属性を持つオブジェクトの状態の取り消しを示します（ “cancel” 行）。 また、 `Version` 別の番号を持つオブジェクトの各状態を識別する必要がある列。 +使用する `Sign` 行を書き込むときの列。 もし `Sign = 1` これは、行がオブジェクトの状態であることを意味します（ “state” 行）。 もし `Sign = -1` これは、同じ属性を持つオブジェクトの状態の取り消しを示します（ “cancel” 行）。 また、 `Version` オブジェクトの各状態を個別の番号で識別する必要があります。 -たとえば、ユーザーがいくつかのサイトで訪問したページの数と、そこにいた期間を計算したいとします。 ある時点で、ユーザーアクティビティの状態で次の行を書きます: +たとえば、ユーザーがいくつのサイトにアクセスしたのか、どのくらいの時間があったのかを計算します。 ある時点で、次の行をユーザーアクティビティの状態で記述します: ``` text ┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┬─Version─┐ @@ -95,7 +96,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] └─────────────────────┴───────────┴──────────┴──────┴─────────┘ ``` -ある時点で、ユーザーアクティビティの変更を登録し、次の二つの行を書き込みます。 +後のある時点で、ユーザーアクティビティの変更を登録し、次の二つの行でそれを書きます。 ``` text ┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┬─Version─┐ @@ -104,9 +105,9 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] └─────────────────────┴───────────┴──────────┴──────┴─────────┘ ``` -最初の行は、オブジェクト（user）の前の状態を取り消します。 でコピーのすべての分野を中止状態を除く `Sign`. +最初の行は、オブジェクト(ユーザー)の以前の状態を取り消します。 キャンセルされた状態のすべてのフィールドをコピーします。 `Sign`. -次の行には、現在の状態が含まれています。 +次の行には現在の状態が含まれます。 ユーザーアクティビティの最後の状態だけが必要なので、行 @@ -117,31 +118,31 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] └─────────────────────┴───────────┴──────────┴──────┴─────────┘ ``` -オブジェクトの無効な（古い）状態を削除して、削除することができます。 `VersionedCollapsingMergeTree` これは、データ部分をマージ中に行います。 +オブジェクトの無効な（古い）状態を折りたたんで、削除することができます。 `VersionedCollapsingMergeTree` このことを融合させたデータの部品です。 -そんなしが必要であるが、行の変更を参照 [Algorithm](#table_engines-versionedcollapsingmergetree-algorithm). +変更ごとに二つの行が必要な理由については、以下を参照してください [アルゴリズ](#table_engines-versionedcollapsingmergetree-algorithm). **使用上の注意** -1. プログラムを書き込み、データ意のオブジェクトを解除します。 その “cancel” 文字列は、 “state” 反対の文字列 `Sign`. この増加の初期サイズでの保存が可能なデータを書き込む。 -2. 列の長い配列は、書き込みの負荷のためにエンジンの効率を低下させます。 データがより簡単になればなるほど、効率は向上します。 -3. `SELECT` 結果は、オブジェクト変更の履歴の一貫性に強く依存します。 挿入するデータを準備するときは正確です。 セッションの深さなどの非負の指標の負の値など、一貫性のないデータで予測不可能な結果を得ることができます。 +1. プログラムを書き込み、データ意のオブジェクトを解除します。 その “cancel” のコピーでなければなりません。 “state” 反対の文字列 `Sign`. この増加の初期サイズでの保存が可能なデータを書き込む。 +2. 列の長い成長配列は、書き込みの負荷のためにエンジンの効率を低下させる。 より簡単なデータ、より良い効率。 +3. `SELECT` 結果は、オブジェクト変更の履歴の一貫性に強く依存します。 挿入のためのデータを準備するときは正確です。 セッションの深さなど、負でない指標の負の値など、不整合なデータで予測不可能な結果を得ることができます。 -### Algorithm {#table_engines-versionedcollapsingmergetree-algorithm} +### アルゴリズ {#table_engines-versionedcollapsingmergetree-algorithm} -ClickHouseは、データパーツをマージするときに、同じ主キーとバージョンが異なる行の各ペアを削除します `Sign`. 行の順序は関係ありません。 +ClickHouseは、データパーツをマージするときに、同じ主キーとバージョンと異なる行の各ペアを削除します `Sign`. 行の順序は重要ではありません。 -ClickHouseがデータを挿入すると、主キーで行を並べ替えます。 この `Version` 列は主キーにはなく、ClickHouseはそれを主キーに暗黙的に最後のフィールドとして追加し、それを順序付けに使用します。 +ClickHouseはデータを挿入するとき、主キーによって行を並べ替えます。 もし `Version` 列が主キーにない場合、ClickHouseはそれを最後のフィールドとして暗黙的に主キーに追加し、順序付けに使用します。 ## データの選択 {#selecting-data} -ClickHouseは、同じ主キーを持つすべての行が同じ結果のデータ部分にあるか、同じ物理サーバー上にあることを保証するものではありません。 これは、データの書き込みとそれに続くデータ部分のマージの両方に当てはまります。 さらに、ClickHouseプロセス `SELECT` 複数のスレッドを持つクエリは、結果の行の順序を予測することはできません。 これは、完全に取得する必要がある場合に集約が必要であることを意味します “collapsed” からのデータ `VersionedCollapsingMergeTree` テーブル。 +ClickHouseは、同じ主キーを持つすべての行が同じ結果のデータ部分または同じ物理サーバー上にあることを保証するものではありません。 これは、データの書き込みとその後のデータ部分のマージの両方に当てはまります。 さらに、ClickHouseプロセス `SELECT` 複数のスレッドを使用してクエリを実行し、結果の行の順序を予測することはできません。 これは、完全に取得する必要がある場合に集約が必要であることを意味します “collapsed” aからのデータ `VersionedCollapsingMergeTree` テーブル。 -折りたたみを完了するには、次のようにクエリを記述します `GROUP BY` この符号を考慮する句および集計関数。 たとえば、数量を計算するには、以下を使用します `sum(Sign)` 代わりに `count()`. 何かの合計を計算するには、次のようにします `sum(Sign * x)` 代わりに `sum(x)`、と追加 `HAVING sum(Sign) > 0`. +折りたたみを終了するには、 `GROUP BY` 符号を説明する句および集計関数。 たとえば、数量を計算するには、次を使用します `sum(Sign)` 代わりに `count()`. 何かの合計を計算するには、次のようにします `sum(Sign * x)` 代わりに `sum(x)` を追加します。 `HAVING sum(Sign) > 0`. -凝集体 `count`, `sum` と `avg` この方法で計算できます。 合計 `uniq` オブジェクトに折りたたまれていない状態がある場合に計算できます。 凝集体 `min` と `max` 計算できないのは `VersionedCollapsingMergeTree` 折りたたまれた状態の値の履歴は保存されません。 +集計 `count`, `sum` と `avg` この方法で計算できます。 集計 `uniq` オブジェクトが少なくとも一つの非折りたたみ状態を持つ場合に計算できます。 集計 `min` と `max` 計算できないのは `VersionedCollapsingMergeTree` 折りたたまれた状態の値の履歴は保存されません。 -データを抽出する必要がある場合 “collapsing” な集計（例えば、確認列が存在する最新の値に一致条件)を使用できます `FINAL` のための修飾語 `FROM` 句。 このアプローチは非効率で利用すべきではありませんの大きます。 +データを抽出する必要がある場合は “collapsing” な集計（例えば、確認列が存在する最新の値に一致条件)を使用できます `FINAL` モディファイア `FROM` 句。 このアプローチは非効率で利用すべきではありませんの大きます。 ## 使用例 {#example-of-use} @@ -180,7 +181,7 @@ INSERT INTO UAct VALUES (4324182021466249494, 5, 146, 1, 1) INSERT INTO UAct VALUES (4324182021466249494, 5, 146, -1, 1),(4324182021466249494, 6, 185, 1, 2) ``` -私たちは二つを使う `INSERT` 二つの異なるデータ部分を作成するクエリ。 単一のクエリでデータを挿入すると、ClickHouseは単一のデータ部分を作成し、マージは実行しません。 +我々は二つを使用します `INSERT` 二つの異なるデータ部分を作成するクエリ。 単一のクエリでデータを挿入すると、ClickHouseは一つのデータパーツを作成し、マージを実行することはありません。 データの取得: @@ -198,9 +199,9 @@ SELECT * FROM UAct └─────────────────────┴───────────┴──────────┴──────┴─────────┘ ``` -ここでは何が見え、折りたたまれた部分はどこにありますか？ -二つのデータパーツを二つ作成しました `INSERT` クエリ。 その `SELECT` クエリは二つのスレッドで実行され、結果は行のランダムな順序です。 -デー clickhouseは、予測できない未知の時点でデータパーツをマージします。 +私たちはここで何を見て、崩壊した部分はどこにありますか？ +二つのデータパーツを作成しました `INSERT` クエリ。 その `SELECT` クエリは二つのスレッドで実行され、結果は行のランダムな順序です。 +崩壊の発生はありませんので、データのパーツがなされております。 ClickHouseは、予測できない未知の時点でデータパーツをマージします。 これが集約が必要な理由です: @@ -221,7 +222,7 @@ HAVING sum(Sign) > 0 └─────────────────────┴───────────┴──────────┴─────────┘ ``` -集約を必要とせず、強制的に崩壊させたい場合は、 `FINAL` のための修飾語 `FROM` 句。 +集約を必要とせず、強制的に折りたたみたいなら、以下を使うことができます `FINAL` モディファイア `FROM` 句。 ``` sql SELECT * FROM UAct FINAL diff --git a/docs/ja/engines/table-engines/special/buffer.md b/docs/ja/engines/table-engines/special/buffer.md index bc97d9ab3d7..c3ef6bd1157 100644 --- a/docs/ja/engines/table-engines/special/buffer.md +++ b/docs/ja/engines/table-engines/special/buffer.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 45 toc_title: "\u30D0\u30C3\u30D5\u30A1" --- # バッファ {#buffer} -バッファのデータを書き込ram、定期的にフラッシングで別表に示す。 読み取り操作中に、バッファと他のテーブルからデータが同時に読み取られます。 +バッファのデータを書き込RAM、定期的にフラッシングで別表に示す。 読み取り動作中、データはバッファと他のテーブルから同時に読み取られます。 ``` sql Buffer(database, table, num_layers, min_time, max_time, min_rows, max_rows, min_bytes, max_bytes) @@ -17,55 +17,55 @@ Buffer(database, table, num_layers, min_time, max_time, min_rows, max_rows, min_ - `database` – Database name. Instead of the database name, you can use a constant expression that returns a string. - `table` – Table to flush data to. -- `num_layers` – Parallelism layer. Physically, the table will be represented as `num_layers` 独立した緩衝の。 推奨値:16。 -- `min_time`, `max_time`, `min_rows`, `max_rows`, `min_bytes`、と `max_bytes` – Conditions for flushing data from the buffer. +- `num_layers` – Parallelism layer. Physically, the table will be represented as `num_layers` 独立した緩衝の。 推奨値:16. +- `min_time`, `max_time`, `min_rows`, `max_rows`, `min_bytes`,and `max_bytes` – Conditions for flushing data from the buffer. -データはバッファーからフラッシュされ、すべてのファイルが `min*` 条件または少なくとも一つ `max*` 条件が満たされる。 +データはバッファからフラッシュされ、宛先テーブルに書き込まれます。 `min*` 条件または少なくとも一つ `max*` 条件が満たされる。 - `min_time`, `max_time` – Condition for the time in seconds from the moment of the first write to the buffer. - `min_rows`, `max_rows` – Condition for the number of rows in the buffer. - `min_bytes`, `max_bytes` – Condition for the number of bytes in the buffer. -書き込み操作中に、データはaに挿入されます。 `num_layers` ランダムバッファの数。 または、挿入するデータ部分が十分大きい場合（ `max_rows` または `max_bytes`）、バッファを省略して、宛先テーブルに直接書き込まれます。 +書き込み操作の間、データはaに挿入されます `num_layers` ランダムバッファの数。 または、挿入するデータ部分が十分に大きい（より大きい）場合 `max_rows` または `max_bytes`）、それはバッファを省略して、宛先テーブルに直接書き込まれます。 -データを洗い流すための条件はのそれぞれのために別に計算されます `num_layers` バッファ。 たとえば、次の場合 `num_layers = 16` と `max_bytes = 100000000`、最大RAM消費は1.6GBです。 +データをフラッシュするための条件は、 `num_layers` バッファ たとえば、 `num_layers = 16` と `max_bytes = 100000000`、最大RAM消費量は1.6GBです。 -例えば: +例: ``` sql CREATE TABLE merge.hits_buffer AS merge.hits ENGINE = Buffer(merge, hits, 16, 10, 100, 10000, 1000000, 10000000, 100000000) ``` -作成する ‘merge.hits\_buffer’ 同じ構造のテーブル ‘merge.hits’ そして緩衝エンジンを使用して。 このテーブルに書き込むとき、データはRAMにバッファリングされ、後で ‘merge.hits’ テーブル。 16バッファが作成されます。 100秒が経過した場合、または百万行が書き込まれた場合、または100MBのデータが書き込まれた場合、または同時に10秒が経過し、10,000行と10MBのデータが たとえば、一つの行だけが書き込まれた場合、100秒後には何があってもフラッシュされます。 しかし、多くの行が書き込まれた場合、データはすぐにフラッシュされます。 +作成 ‘merge.hits\_buffer’ と同じ構造を持つテーブル ‘merge.hits’ そして、バッファエンジンを使用。 このテーブルに書き込むとき、データはRAMにバッファされ、後で ‘merge.hits’ テーブル。 16のバッファが作成されます。 それぞれのデータは、100秒が経過した場合、または百万行が書き込まれた場合、または100MBのデータが書き込まれた場合、または10秒が経過して10,000行と10 たとえば、ただ一つの行が書き込まれている場合、100秒後には何があってもフラッシュされます。 しかし、多くの行が書き込まれている場合、データは早くフラッシュされます。 -サーバが停止し、ドテーブルやテーブル取り外し、バッファのデータも出力先にフラッシュされる。 +DROP TABLEまたはDETACH TABLEを使用してサーバーを停止すると、バッファーデータも宛先テーブルにフラッシュされます。 -データベース名およびテーブル名には、一重引quotationで空の文字列を設定できます。 これは、宛先テーブルがないことを示します。 この場合、データのフラッシュ条件に達すると、バッファは単純にクリアされます。 これは、データのウィンドウをメモリに保持するのに便利です。 +データベース名とテーブル名には、単一引quotationで空の文字列を設定できます。 これは、宛先テーブルがないことを示します。 この場合、データフラッシュ条件に達すると、バッファは単純にクリアされます。 これは、データのウィンドウをメモリに保持する場合に便利です。 -バッファテーブルから読み取るとき、データはバッファと宛先テーブルの両方から処理されます（存在する場合）。 -このバッファーとなる指数です。 つまり、データのバッファが読み取れる大きなバッファ. （下位テーブルのデータの場合、サポートするインデックスが使用されます。) +バッファテーブルから読み取るとき、データはバッファと宛先テーブルの両方から処理されます(存在する場合)。 +バッファテーブルはインデックスをサポートしません。 つまり、バッファ内のデータは完全にスキャンされます。 (下位テーブルのデータについては、サポートするインデックスが使用されます。) -バッファテーブルの列のセットが下位テーブルの列のセットと一致しない場合、両方のテーブルに存在する列のサブセットが挿入されます。 +バッファテーブル内の列のセットが下位テーブル内の列のセットと一致しない場合、両方のテーブルに存在する列のサブセットが挿入されます。 -バッファテーブルと下位テーブルのいずれかの列に型が一致しない場合、サーバーログにエラーメッセージが入力され、バッファがクリアされます。 +バッファテーブル内のいずれかの列と下位テーブルで型が一致しない場合、サーバログにエラーメッセージが入力され、バッファがクリアされます。 バッファがフラッシュされたときに下位テーブルが存在しない場合も同じことが起こります。 -下位テーブルとバッファテーブルに対してalterを実行する必要がある場合は、まずバッファテーブルを削除し、その下位テーブルに対してalterを実行してから +下位テーブルとバッファーテーブルに対してALTERを実行する必要がある場合は、まずバッファーテーブルを削除し、下位テーブルに対してALTERを実行してから、バッ サーバーが異常に再起動されると、バッファー内のデータは失われます。 -バッファテーブルでは、finalとsampleが正しく動作しません。 これらの条件は宛先テーブルに渡されますが、バッファー内のデータの処理には使用されません。 これらの特徴が必要のみを使用することをお勧めしまうバッファのテーブルを書き込み、読みの行先表に示す。 +最終サンプルな正常に動作しないためのバッファです。 これらの条件は宛先テーブルに渡されますが、バッファー内のデータの処理には使用されません。 これらの特徴が必要のみを使用することをお勧めしまうバッファのテーブルを書き込み、読みの行先表に示す。 -を追加する場合にデータをバッファのバッファがロックされています。 これにより、読み取り操作がテーブルから同時に実行されると遅延が発生します。 +を追加する場合にデータをバッファのバッファがロックされています。 これにより、テーブルから読み取り操作が同時に実行される場合に遅延が発生します。 -バッファテーブルに挿入されるデータは、異なる順序で異なるブロックで従属テーブルに格納される可能性があります。 このため、バッファテーブルは、collapsingmergetreeへの書き込みには正しく使用することが困難です。 問題を回避するには、次のように設定します ‘num\_layers’ 1になります。 +バッファテーブルに挿入されるデータは、下位テーブル内で異なる順序と異なるブロックになる場合があります。 このため、CollapsingMergeTreeに正しく書き込むためにバッファテーブルを使用することは困難です。 問題を回避するには、以下を設定できます ‘num\_layers’ に1. -の場合は先テーブルがそのままに再現され、期待の特徴を再現でテーブルが失われた書き込みバッファへの表に示す。 データパーツの行とサイズの順序をランダムに変更すると、データ重複排除が機能しなくなります。 ‘exactly once’ 書く再現します。 +の場合は先テーブルがそのままに再現され、期待の特徴を再現でテーブルが失われた書き込みバッファへの表に示す。 行の順序とデータ部分のサイズがランダムに変更されると、データ重複排除が動作を終了します。 ‘exactly once’ 書く再現します。 -これらの欠点のために、まれにバッファテーブルの使用を推奨することができます。 +これらの欠点があるため、まれにしかバッファテーブルの使用を推奨できません。 -バッファテーブルは、ある時間単位で多数のサーバーから大量の挿入が受信され、挿入前にデータをバッファリングできない場合に使用されます。 +バッファテーブルは、単位時間にわたって多数のサーバーから多数の挿入を受信し、挿入の前にデータをバッファリングできない場合に使用されます。 -バッファテーブルの場合でも、一度に一行ずつデータを挿入するのは意味がないことに注意してください。 これにより、毎秒数千行の速度しか生成されませんが、より大きなデータブロックを挿入すると、毎秒百万行を生成することができます（セクションを “Performance”). +バッファテーブルであっても、一度にデータ行を挿入することは意味がありません。 これにより、毎秒数千行の速度が生成されますが、より大きなデータブロックを挿入すると、毎秒百万行を超える速度が生成されます（セクションを参照 “Performance”). [元の記事](https://clickhouse.tech/docs/en/operations/table_engines/buffer/) diff --git a/docs/ja/engines/table-engines/special/dictionary.md b/docs/ja/engines/table-engines/special/dictionary.md index d55390afde3..9de7c7b78db 100644 --- a/docs/ja/engines/table-engines/special/dictionary.md +++ b/docs/ja/engines/table-engines/special/dictionary.md @@ -1,15 +1,15 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 35 toc_title: "\u8F9E\u66F8" --- # 辞書 {#dictionary} -その `Dictionary` エンジンは表示します [辞書](../../../sql-reference/dictionaries/external-dictionaries/external-dicts.md) クリックハウス表としてのデータ。 +その `Dictionary` エンジンは表示します [辞書](../../../sql-reference/dictionaries/external-dictionaries/external-dicts.md) ClickHouseテーブルとしてのデータ。 -例として、次の辞書を考えてみましょう `products` 次の構成では: +例として、の辞書を考えてみましょう `products` 以下の構成で: ``` xml @@ -42,7 +42,7 @@ toc_title: "\u8F9E\u66F8" ``` -辞書データのクエリ: +辞書データの照会: ``` sql SELECT @@ -64,9 +64,9 @@ WHERE name = 'products' └──────────┴──────┴────────┴─────────────────┴─────────────────┴─────────────────┴───────────────┴─────────────────┘ ``` -を使用することができ [dictGet\*](../../../sql-reference/functions/ext-dict-functions.md#ext_dict_functions) この形式の辞書データを取得する関数です。 +を使用することができます [dictGet\*](../../../sql-reference/functions/ext-dict-functions.md#ext_dict_functions) この形式の辞書データを取得する関数。 -このビューがない便だが、rawデータ、または行う場合には、 `JOIN` オペレーション これらのケースでは、以下を使用できます。 `Dictionary` テーブル内のディクショナリデータを表示するエンジン。 +このビューは、生データを取得する必要がある場合や、 `JOIN` 作戦だ これらのケースでは、 `Dictionary` ディクショナリデータをテーブルに表示するエンジン。 構文: @@ -82,7 +82,7 @@ create table products (product_id UInt64, title String) Engine = Dictionary(prod Ok -テーブルに何があるかを見てみましょう。 +テーブルにあるものを見てみなさい。 ``` sql select * from products limit 1; diff --git a/docs/ja/engines/table-engines/special/distributed.md b/docs/ja/engines/table-engines/special/distributed.md index 98deb01a16c..429a34ce022 100644 --- a/docs/ja/engines/table-engines/special/distributed.md +++ b/docs/ja/engines/table-engines/special/distributed.md @@ -1,45 +1,45 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 33 toc_title: "\u5206\u6563" --- # 分散 {#distributed} -**分散エンジンを持つテーブルは、自身がデータを保存しません** しかし、複数のサーバー上の分散クエリ処理を許可します。 -読書は自動的に平行である。 読み取り中に、リモートサーバー上のテーブルインデックスが使用されます。 +**分散エンジンを備えたテーブルは、データ自体を格納しません** しかし、複数のサーバーで分散クエリ処理を許可します。 +読み取りは自動的に並列化されます。 読み取り中に、リモートサーバー上のテーブルインデックスが存在する場合に使用されます。 -の分散型エンジンを受け付けパラメータ: +分散エンジ: -- サーバーの設定ファイル内のクラスター名 +- サーバーの設定ファイル内のクラスタ名 - リモートデータベースの名前 - リモートテーブルの名前 -- シャーディングキー +- （オプション)shardingキー -- （オプションで）ポリシー名は、非同期送信のための一時ファイルを格納するために使用される +- (必要に応じて)ポリシー名は、非同期送信の一時ファイルを格納するために使用されます - また見なさい: + も参照。: - `insert_distributed_sync` 設定 - - [MergeTree](../mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes) 例のため + - [メルゲツリー](../mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes) 例については -例えば: +例: ``` sql Distributed(logs, default, hits[, sharding_key[, policy_name]]) ``` -データはすべてのサーバから読み込まれます。 ‘logs’ デフォルトのクラスター。ヒットテーブルに位置毎にサーバのクラスター -データは読み取られるだけでなく、リモートサーバーで部分的に処理されます（これが可能な限り）。 -たとえば、group byを使用するクエリの場合、データはリモートサーバー上で集計され、集計関数の中間状態がリクエスターサーバーに送信されます。 その後、データはさらに集約されます。 +すべてのサーバからデータが読み取られます。 ‘logs’ デフォルトからのクラスター。ヒットテーブルに位置毎にサーバのクラスター +データは読み取りだけでなく、リモートサーバーで部分的に処理されます（可能な限り）。 +たとえば、GROUP BYを使用したクエリの場合、リモートサーバーでデータが集計され、集計関数の中間状態が要求元サーバーに送信されます。 その後、データはさらに集計されます。 -データベース名の代わりに、文字列を返す定数式を使用できます。 たとえば、次のようになります。 +データベース名の代わりに、文字列を返す定数式を使用できます。 たとえば、currentDatabase()です。 -logs – The cluster name in the server’s config file. +logs – The cluster name in the server's config file. クラスターがセットのようなこ: @@ -77,75 +77,75 @@ logs – The cluster name in the server’s config file. ``` -ここでは、クラスターは名前で定義されます ‘logs’ それぞれ二つのレプリカを含む二つのシャードで構成されています。 -シャードは、データの異なる部分を含むサーバーを参照します（すべてのデータを読み取るには、すべてのシャードにアクセスする必要があります）。 +ここで、クラスターは名前で定義されます ‘logs’ これは二つの破片で構成され、それぞれに二つの複製が含まれています。 +シャードは、データの異なる部分を含むサーバーを指します(すべてのデータを読み取るには、すべてのシャードにアクセスする必要があります)。 レプリカはサーバーを複製しています（すべてのデータを読み取るために、レプリカのいずれかのデータにアクセスできます）。 -クラ +クラスターの名前などを含めないでくださいませんでした。 -パラメータ `host`, `port`、およびオプション `user`, `password`, `secure`, `compression` サーバーごとに指定します: -- `host` – The address of the remote server. You can use either the domain or the IPv4 or IPv6 address. If you specify the domain, the server makes a DNS request when it starts, and the result is stored as long as the server is running. If the DNS request fails, the server doesn’t start. If you change the DNS record, restart the server. -- `port` – The TCP port for messenger activity (‘tcp\_port’ 設定では、通常9000）に設定します。 それをhttp\_portと混同しないでください。 +パラメータ `host`, `port`、および必要に応じて `user`, `password`, `secure`, `compression` サーバーごとに指定されます: +- `host` – The address of the remote server. You can use either the domain or the IPv4 or IPv6 address. If you specify the domain, the server makes a DNS request when it starts, and the result is stored as long as the server is running. If the DNS request fails, the server doesn't start. If you change the DNS record, restart the server. +- `port` – The TCP port for messenger activity (‘tcp\_port’ 設定では、通常9000に設定されています）。 Http\_portと混同しないでください。 - `user` – Name of the user for connecting to a remote server. Default value: default. This user must have access to connect to the specified server. Access is configured in the users.xml file. For more information, see the section [アクセス権](../../../operations/access-rights.md). - `password` – The password for connecting to a remote server (not masked). Default value: empty string. -- `secure` -接続にsslを使用します。 `port` = 9440. サーバーがリッスンする `9440` と正しい証明書。 +- `secure` -接続にsslを使用します。 `port` = 9440. サーバーがリッスンする `9440` 正しい証明書を持っています。 - `compression` -データ圧縮を使用します。 デフォルト値:true。 When specifying replicas, one of the available replicas will be selected for each of the shards when reading. You can configure the algorithm for load balancing (the preference for which replica to access) – see the [load\_balancing](../../../operations/settings/settings.md#settings-load_balancing) 設定。 -サーバーとの接続が確立されていない場合は、短いタイムアウトで接続しようとします。 接続に失敗すると、すべてのレプリカに対して次のレプリカが選択されます。 すべてのレプリカに対して接続の試行が失敗した場合、その試行は同じ方法で何度も繰り返されます。 -リモートサーバーは接続を受け入れる可能性がありますが、動作しない可能性があります。 +サーバーとの接続が確立されていない場合は、短いタイムアウトで接続しようとします。 接続に失敗した場合は、すべてのレプリカに対して次のレプリカが選択されます。 すべてのレプリカで接続試行が失敗した場合、その試行は同じ方法で何度か繰り返されます。 +リモートサーバーは接続を受け入れるかもしれませんが、動作しないか、または不十分に動作する可能性があります。 -シャードのいずれかを指定できます(この場合、クエリ処理は分散ではなくリモートと呼ばれる必要があります)、または任意の数のシャードまで指定でき 各シャードでは、レプリカのいずれかから任意の数に指定することができます。 シャードごとに異なる数のレプリカを指定できます。 +いずれかのシャードのみを指定することができます(この場合、クエリ処理は分散ではなくリモートと呼ばれる必要があります)。 各シャードでは、いずれかから任意の数のレプリカを指定できます。 シャードごとに異なる数のレプリカを指定できます。 -構成では、任意の数のクラスターを指定できます。 +設定では、必要な数のクラスターを指定できます。 -クラスタを表示するには、以下を使用します ‘system.clusters’ テーブル。 +クラスターを表示するには、 ‘system.clusters’ テーブル。 -の分散型エンジン能にすることで、社会とクラスターのように現地サーバーです。 ただし、クラスターの構成はサーバー設定ファイルに書き込む必要があります(クラスターのすべてのサーバーではさらに優れています)。 +の分散型エンジン能にすることで、社会とクラスターのように現地サーバーです。 サーバー設定ファイルにその設定を書き込む必要があります(すべてのクラスターのサーバーでさらに優れています)。 -The Distributed engine requires writing clusters to the config file. Clusters from the config file are updated on the fly, without restarting the server. If you need to send a query to an unknown set of shards and replicas each time, you don’t need to create a Distributed table – use the ‘remote’ 代わりにテーブル関数。 セクションを見る [テーブル関数](../../../sql-reference/table-functions/index.md). +The Distributed engine requires writing clusters to the config file. Clusters from the config file are updated on the fly, without restarting the server. If you need to send a query to an unknown set of shards and replicas each time, you don't need to create a Distributed table – use the ‘remote’ 代わりにテーブル関数。 セクションを参照 [テーブル関数](../../../sql-reference/table-functions/index.md). クラスターにデータを書き込む方法は二つあります: -まず、どのサーバーにどのデータを書き込むかを定義し、各シャードで直接書き込みを実行できます。 つまり、分散テーブルのテーブルにinsertを実行します “looks at”. これは、主題領域の要件のために自明ではないシャーディングスキームを使用できるため、最も柔軟なソリューションです。 これも最適なソリューションからデータを書き込むことができるの異なる資料が完全に独立。 +まず、どのサーバーにどのデータを書き込むかを定義し、各シャードで直接書き込みを実行できます。 つまり、分散テーブルのテーブルに挿入を実行します “looks at”. これは、対象領域の要件のために自明ではない可能性のあるシャーディングスキームを使用できるので、最も柔軟なソリューションです。 データは完全に独立して異なるシャードに書き込むことができるので、これも最適な解決策です。 -次に、分散テーブルでinsertを実行できます。 この場合、テーブルは挿入されたデータをサーバー自体に分散します。 分散テーブルに書き込むには、シャーディングキーセット（最後のパラメータ）が必要です。 さらに、単一のシャードしかない場合、書き込み操作はシャーディングキーを指定せずに動作します。 +次に、分散テーブルでINSERTを実行できます。 この場合、テーブルは挿入されたデータをサーバー自体に分散します。 分散テーブルに書き込むには、シャーディングキーセット(最後のパラメーター)が必要です。 さらに、シャードがひとつしかない場合、この場合は何も意味しないため、シャーディングキーを指定せずに書き込み操作が機能します。 -各シャードは設定ファイルで定義された重みを持つことができます。 デフォルトでは、重みは一つに等しいです。 データは、シャードウェイトに比例した量でシャード全体に分散されます。 たとえば、二つのシャードがあり、最初のものが9の重みを持ち、第二のものが10の重みを持つ場合、最初のシャードは9/19の行に送られ、第二のものは10/19 +各シャードには、設定ファイルで定義された重みを設定できます。 デフォルトでは、重みは重みと等しくなります。 データは、シャードの重みに比例する量でシャードに分散されます。 たとえば、シャードが二つあり、最初のシャードが9の重みを持ち、次のシャードが10の重みを持つ場合、最初のシャードは行の9/19部分に送信され、次のシャー -各破片は持つことができます ‘internal\_replication’ 設定ファイルで定義されたパラメータ。 +各シャードは、 ‘internal\_replication’ 設定ファイルで定義されたパラメータ。 -このパラメータが設定されている場合 ‘true’ 書き込み操作は、最初の正常なレプリカを選択し、それにデータを書き込みます。 分散テーブルの場合は、この代替を使用します “looks at” 複製されたテーブル。 言い換えれば、データが書き込まれるテーブルがそれ自体を複製する場合です。 +このパラメータが ‘true’ 書き込み操作は、最初の正常なレプリカを選択し、それにデータを書き込みます。 分散テーブルの場合は、この代替を使用します “looks at” 複製されたテーブル。 言い換えれば、データが書き込まれるテーブルがそれ自体を複製する場合。 -に設定されている場合 ‘false’ データはすべてのレプリカに書き込まれます。 本質的に、これは、分散テーブルがデータ自体を複製することを意味します。 レプリカの整合性はチェックされず、時間の経過とともにわずかに異なるデータが含まれるためです。 +に設定されている場合 ‘false’ データはすべてのレプリカに書き込まれます。 本質的に、これは分散テーブルがデータ自体を複製することを意味します。 これは、レプリケートされたテーブルを使用するよりも悪いことです。 -データの行が送信されるシャードを選択するには、シャーディング式が分析され、残りの部分がシャードの合計ウェイトで除算されます。 行は、残りの半分の間隔に対応するシャードに送られます ‘prev\_weight’ に ‘prev\_weights + weight’、どこ ‘prev\_weights’ 最小の数を持つシャードの合計重量です。 ‘weight’ このシャードの重さです。 たとえば、二つのシャードがあり、最初のシャードの重みが9で、二番目のシャードの重みが10である場合、行は\[0,9)の範囲から残りのシャードの最初のシャー +データの行が送信されるシャードを選択するために、シャーディング式が分析され、残りの部分がシャードの総重量で除算されます。 この行は、残りの半分の間隔に対応するシャードに送信されます。 ‘prev\_weight’ に ‘prev\_weights + weight’,ここで ‘prev\_weights’ は、最小の数を持つ破片の総重量です。 ‘weight’ この破片の重量です。 たとえば、シャードが二つあり、最初のシャードの重みが9で、次のシャードの重みが10である場合、行は範囲\[0,9)の残りのシャードの最初のシャードに送信され、 -シャーディング式には、整数を返す定数とテーブル列からの任意の式を指定できます。 たとえば、次の式を使用できます ‘rand()’ データのランダムな分布の場合、または ‘UserID’ ユーザーのIDを分割する残りの部分で配布する場合（単一のユーザーのデータは単一のシャードに存在し、ユーザーの実行と参加が簡単になります）。 列のいずれかが十分に均等に分散されていない場合は、ハッシュ関数でラップすることができます：intHash64（UserID）。 +シャーディング式には、整数を返す定数およびテーブル列の任意の式を使用できます。 たとえば、次の式を使用できます ‘rand()’ データのランダム分布の場合、または ‘UserID’ ユーザーのIDを分割することからの残りの部分による分配のために（単一のユーザーのデータは単一のシャード上に存在し、ユーザーによる実行と結合を簡素化する）。 いずれかの列が十分に均等に分散されていない場合は、ハッシュ関数intHash64(UserID)でラップできます。 -簡単なリマインダからの限定シshardingんを常に適しています。 中規模および大量のデータ（数十のサーバー）では機能しますが、非常に大量のデータ（数百のサーバー以上）では機能しません。 後者の場合はshardingスキームに必要なのではなく、エントリに配布します。 +簡単なリマインダからの限定シshardingんを常に適しています。 これは、データの中規模および大規模なボリューム（サーバーの数十）ではなく、データの非常に大規模なボリューム（サーバーの数百以上）のために動作します。 後者の場合、分散テーブルのエントリを使用するのではなく、サブジェクト領域で必要なシャーディングスキームを使用します。 -SELECT queries are sent to all the shards and work regardless of how data is distributed across the shards (they can be distributed completely randomly). When you add a new shard, you don’t have to transfer the old data to it. You can write new data with a heavier weight – the data will be distributed slightly unevenly, but queries will work correctly and efficiently. +SELECT queries are sent to all the shards and work regardless of how data is distributed across the shards (they can be distributed completely randomly). When you add a new shard, you don't have to transfer the old data to it. You can write new data with a heavier weight – the data will be distributed slightly unevenly, but queries will work correctly and efficiently. -次の場合、シャーディングスキームについて心配する必要があります: +次の場合は、シャーディングスキームについて心配する必要があります: -- 特定のキーによるデータの結合(inまたはjoin)が必要なクエリが使用されます。 このキーによってデータがシャードされている場合は、グローバルinまたはグローバル結合の代わりにローカルinまたはjoinを使用できます。 -- 多数のサーバー（数百またはそれ以上）が使用され、多数の小さなクエリ（個々のクライアントのクエリ-ウェブサイト、広告主、またはパートナー）が使用されます。 小さなクエリがクラスタ全体に影響を与えないようにするには、単一のクライアントのデータを単一のシャードに配置することが理にかなっていま また、我々はyandexの中でやったように。metricaでは、biレベルのシャーディングを設定できます。 “layers” レイヤーが複数のシャードで構成されている場合。 単一のクライアントのデータは単一のレイヤーに配置されますが、必要に応じてシャードをレイヤーに追加することができ、データはその中にランダムに配 分散テーブルはレイヤごとに作成され、グローバルクエリ用に単一の共有分散テーブルが作成されます。 +- 特定のキーによるデータの結合(INまたはJOIN)を必要とするクエリが使用されます。 このキーによってデータがシャードされる場合は、GLOBAL INまたはGLOBAL JOINの代わりにlocal INまたはJOINを使用できます。 +- 多数のサーバーが、多数の小さなクエリ（個々のクライアント-ウェブサイト、広告主、またはパートナーのクエリ）で使用されます（数百以上）。 小さなクエリがクラスタ全体に影響を与えないようにするには、単一のシャード上の単一のクライアントのデータを検索することが理にかなってい また、我々はYandexのでやったように。Metricaでは、biレベルのシャーディングを設定できます：クラスタ全体を次のように分割します “layers” ここで、レイヤーは複数のシャードで構成されます。 単一のクライアントのデータは単一のレイヤー上にありますが、必要に応じてシャードをレイヤーに追加することができ、データはランダムに分散されます。 分散テーブルはレイヤごとに作成され、グローバルクエリ用に単一の共有分散テーブルが作成されます。 -データは非同期に書き込まれます。 テーブルに挿入すると、データブロックはローカルファイルシステムに書き込まれます。 データはできるだけ早くバックグラウンドでリモートサーバーに送信されます。 データを送信する期間は、以下によって管理されます。 [distributed\_directory\_monitor\_sleep\_time\_ms](../../../operations/settings/settings.md#distributed_directory_monitor_sleep_time_ms) と [distributed\_directory\_monitor\_max\_sleep\_time\_ms](../../../operations/settings/settings.md#distributed_directory_monitor_max_sleep_time_ms) 設定。 その `Distributed` エンジンを送信し、各ファイルを挿入したデータが別々にまでを一括送信ファイルの [distributed\_directory\_monitor\_batch\_inserts](../../../operations/settings/settings.md#distributed_directory_monitor_batch_inserts) 設定。 この設定の改善にクラスターの性能をより一層の活用地域のサーバやネットワーク資源です。 を確認しておきましょうか否かのデータが正常に送信されるチェックリストファイル(データまたは間に-をはさんだ)はテーブルディレクトリ: `/var/lib/clickhouse/data/database/table/`. +データは非同期に書き込まれます。 テーブルに挿入すると、データブロックはローカルファイルシステムに書き込まれます。 データはできるだけ早くバックグラウンドでリモートサーバーに送信されます。 データを送信するための期間は、 [distributed\_directory\_monitor\_sleep\_time\_ms](../../../operations/settings/settings.md#distributed_directory_monitor_sleep_time_ms) と [distributed\_directory\_monitor\_max\_sleep\_time\_ms](../../../operations/settings/settings.md#distributed_directory_monitor_max_sleep_time_ms) 設定。 その `Distributed` エンジンは、挿入されたデータを含む各ファイルを別々に送信しますが、 [distributed\_directory\_monitor\_batch\_inserts](../../../operations/settings/settings.md#distributed_directory_monitor_batch_inserts) 設定。 この設定の改善にクラスターの性能をより一層の活用地域のサーバやネットワーク資源です。 を確認しておきましょうか否かのデータが正常に送信されるチェックリストファイル(データまたは間に-をはさんだ)はテーブルディレクトリ: `/var/lib/clickhouse/data/database/table/`. -分散テーブルへの挿入後にサーバーが存在しなくなった場合、または大まかな再起動(デバイス障害など)が発生した場合は、挿入されたデータが失われる可 破損したデータ部分がテーブルディレクトリで検出された場合、そのデータ部分は、 ‘broken’ サブディレクトリと、もはや使用。 +分散テーブルへの挿入後にサーバーが存在しなくなった場合、またはデバイスの障害後などに大まかな再起動が行われた場合は、挿入されたデータが失われ テーブルディレクトリで破損したデータ部分が検出されると、その部分は ‘broken’ 使用されなくなりました。 -Max\_parallel\_replicasオプションを有効にすると、単一のシャード内のすべてのレプリカでクエリ処理が並列化されます。 詳細については、以下を参照してください [max\_parallel\_replicas](../../../operations/settings/settings.md#settings-max_parallel_replicas). +Max\_parallel\_replicasオプションを有効にすると、単一のシャード内のすべてのレプリカでクエリ処理が並列化されます。 詳細については [max\_parallel\_replicas](../../../operations/settings/settings.md#settings-max_parallel_replicas). ## 仮想列 {#virtual-columns} - `_shard_num` — Contains the `shard_num` （から `system.clusters`). タイプ: [UInt32](../../../sql-reference/data-types/int-uint.md). -!!! note "メモ" - それ以来 [`remote`](../../../sql-reference/table-functions/remote.md)/`cluster` テーブル機能の内部を一時のインスタンスと同じ分散型エンジン, `_shard_num` あまりにもそこに利用可能です。 +!!! note "注" + 以来 [`remote`](../../../sql-reference/table-functions/remote.md)/`cluster` 表関数は内部的に同じ分散エンジンの一時インスタンスを作成します, `_shard_num` あまりにもそこに利用可能です。 -**また見なさい** +**も参照。** - [仮想列](index.md#table_engines-virtual_columns) diff --git a/docs/ja/engines/table-engines/special/external-data.md b/docs/ja/engines/table-engines/special/external-data.md index 18188960259..afa727b6ebb 100644 --- a/docs/ja/engines/table-engines/special/external-data.md +++ b/docs/ja/engines/table-engines/special/external-data.md @@ -1,39 +1,39 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 34 toc_title: "\u5916\u90E8\u30C7\u30FC\u30BF" --- -# クエリ処理のための外部データ {#external-data-for-query-processing} +# クエリ処理の外部データ {#external-data-for-query-processing} -ClickHouseでは、クエリの処理に必要なデータとSELECTクエリをサーバーに送信できます。 このデータは一時テーブルに格納されます(セクションを参照 “Temporary tables” また、クエリで使用することもできます(たとえば、IN演算子など)。 +ClickHouseでは、クエリの処理に必要なデータをSELECTクエリと共にサーバーに送信できます。 このデータは一時テーブルに格納されます(セクションを参照 “Temporary tables”）とクエリで使用できます（たとえば、in演算子）。 -たとえば、重要なユーザー識別子を持つテキストファイルがある場合は、このリストのフィルタリングを使用するクエリと一緒にサーバーにアップロードで +たとえば、重要なユーザー識別子を持つテキストファイルがある場合は、このリストによるフィルタリングを使用するクエリと共にサーバーにアップロードで -大量の外部データを含む複数のクエリを実行する必要がある場合は、この機能を使用しないでください。 事前にデータをdbにアップロードする方が良いです。 +大量の外部データを含む複数のクエリを実行する必要がある場合は、この機能を使用しないでください。 事前にDBにデータをアップロードする方が良いです。 -外部データは、コマンドラインクライアント(非対話モード)またはhttpインターフェイスを使用してアップロードできます。 +外部データは、コマンドラインクライアント(非対話モード)またはHTTPインターフェイスを使用してアップロードできます。 -コマンドラインクライアントでは、次の形式でparametersセクションを指定できます +コマンドラインクライアントでは、parametersセクションを次の形式で指定できます ``` bash --external --file=... [--name=...] [--format=...] [--types=...|--structure=...] ``` -送信されるテーブルの数については、このような複数のセクションがあります。 +して複数の部分のように、このテーブルが送信されます。 **–external** – Marks the beginning of a clause. **–file** – Path to the file with the table dump, or -, which refers to stdin. -Stdinから取得できるのは、単一のテーブルだけです。 +Stdinから取得できるのは単一のテーブルのみです。 -次のパラメーターは省略可能です: **–name**– Name of the table. If omitted, \_data is used. +次のパラメータは省略可能です: **–name**– Name of the table. If omitted, \_data is used. **–format** – Data format in the file. If omitted, TabSeparated is used. -次のパラメーターのいずれかが必要です:**–types** – A list of comma-separated column types. For example: `UInt64,String`. The columns will be named \_1, \_2, … -**–structure**– The table structure in the format`UserID UInt64`, `URL String`. 列の名前と型を定義します。 +次のいずれかのパラメータが必要です:**–types** – A list of comma-separated column types. For example: `UInt64,String`. The columns will be named \_1, \_2, … +**–structure**– The table structure in the format`UserID UInt64`, `URL String`. 列名と型を定義します。 -で指定されたファイル ‘file’ で指定された形式によって解析されます。 ‘format’ で指定されたデータ型を使用する。 ‘types’ または ‘structure’. テーブルはサーバーにアップロードされ、そこにアクセスできるようになります。 ‘name’. +で指定されたファイル ‘file’ に指定された形式で解析されます。 ‘format’ で指定されたデータ型を使用します ‘types’ または ‘structure’. のテーブルがアップロードサーバへのアクセスが一時テーブルの名前 ‘name’. 例: @@ -48,9 +48,9 @@ $ cat /etc/passwd | sed 's/:/\t/g' | clickhouse-client --query="SELECT shell, co /bin/sync 1 ``` -HTTPインターフェイスを使用する場合、外部データはmultipart/form-data形式で渡されます。 各テーブルは別のファイルとして送信されます。 テーブル名はファイル名から取得されます。 その ‘query\_string’ パラメータが渡されます ‘name\_format’, ‘name\_types’、と ‘name\_structure’、どこ ‘name’ これらのパラメーターが対応するテーブルの名前です。 パラメータの意味は、コマンドラインクライアントを使用する場合と同じです。 +HTTPインターフェイスを使用する場合、外部データはmultipart/form-data形式で渡されます。 各テーブルは別々のファイルとして送信されます。 テーブル名は、ファイル名から取得されます。 その ‘query\_string’ パラメータが渡されます ‘name\_format’, ‘name\_types’,and ‘name\_structure’,ここで ‘name’ これらのパラメーターが対応するテーブルの名前を指定します。 パラメーターの意味は、コマンドラインクライアントを使用する場合と同じです。 -例えば: +例: ``` bash $ cat /etc/passwd | sed 's/:/\t/g' > passwd.tsv @@ -63,6 +63,6 @@ $ curl -F 'passwd=@passwd.tsv;' 'http://localhost:8123/?query=SELECT+shell,+coun /bin/sync 1 ``` -分散クエリ処理の場合、一時テーブルはすべてのリモートサーバーに送信されます。 +分散クエリ処理では、一時テーブルがすべてのリモートサーバーに送信されます。 [元の記事](https://clickhouse.tech/docs/en/operations/table_engines/external_data/) diff --git a/docs/ja/engines/table-engines/special/file.md b/docs/ja/engines/table-engines/special/file.md index e42ff99501c..8427c369c2a 100644 --- a/docs/ja/engines/table-engines/special/file.md +++ b/docs/ja/engines/table-engines/special/file.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 37 toc_title: "\u30D5\u30A1\u30A4\u30EB" --- @@ -8,35 +8,35 @@ toc_title: "\u30D5\u30A1\u30A4\u30EB" # ファイル {#table_engines-file} ファイルにテーブルエンジンのデータをファイルを使ったり、 [ファイル -形式](../../../interfaces/formats.md#formats) （TabSeparated、Nativeなど）。). +形式](../../../interfaces/formats.md#formats) (TabSeparated、Nativeなど). 使用例: -- データからの輸出clickhouseるファイルです。 +- ClickHouseからファイルにデータエクスポート。 - ある形式から別の形式にデータを変換します。 -- ディスク上のファイルを編集して、clickhouseのデータを更新する。 +- データ更新にClickHouse経由で編集ファイルディスク。 -## Clickhouseサーバーでの使用状況 {#usage-in-clickhouse-server} +## ClickHouseサーバーでの使用状況 {#usage-in-clickhouse-server} ``` sql File(Format) ``` その `Format` パラメータを指定するか、ファイルのファイルフォーマット 実行するには -`SELECT` クエリは、形式は、入力のためにサポートされ、実行する必要があります +`SELECT` この形式は、入力と実行のためにサポートされている必要があります `INSERT` queries – for output. The available formats are listed in the -[形式](../../../interfaces/formats.md#formats) セクション。 +[形式](../../../interfaces/formats.md#formats) セクション -クリックハウ`File`. で定義されたフォルダを使用します [パス](../../../operations/server-configuration-parameters/settings.md) サーバー構成での設定。 +ClickHouseはファイルシステムのパスを`File`. で定義されたフォルダを使用します [パス](../../../operations/server-configuration-parameters/settings.md) サーバー構成の設定。 -テーブルを作成するとき `File(Format)` で空のサブディレクトリとフォルダにまとめた。 データがそのテーブルに書き込まれると、 `data.Format` サブディレクト +を使用して表を作成する場合 `File(Format)` で空のサブディレクトリとフォルダにまとめた。 データがそのテーブルに書き込まれると、 `data.Format` そのサブディレ -このサブフォルダとファイルをserver filesystemに手動で作成してから [ATTACH](../../../sql-reference/statements/misc.md) でテーブルの情報をマッチングの名前でデータベースバックエンドからファイルです。 +お手動で作成このサブフォルダやファイルサーバのファイルシステムとし [ATTACH](../../../sql-reference/statements/misc.md) 一致する名前で情報を表すので、そのファイルからデータを照会することができます。 !!! warning "警告" - ClickHouseはそのようなファイルの外部変更を追跡しないため、この機能には注意してください。 ClickHouseを介して同時に書き込みを行い、ClickHouseの外部に書き込みを行った結果は未定義です。 + ClickHouseはそのようなファイルへの外部の変更を追跡しないので、この機能に注意してください。 ClickHouseとClickHouseの外部での同時書き込みの結果は未定義です。 -**例えば:** +**例:** **1.** セットアップ `file_engine_table` テーブル: @@ -44,9 +44,9 @@ File(Format) CREATE TABLE file_engine_table (name String, value UInt32) ENGINE=File(TabSeparated) ``` -デフォルトでclickhouseフォルダを作成します `/var/lib/clickhouse/data/default/file_engine_table`. +デフォルトでClickHouseフォルダを作成します `/var/lib/clickhouse/data/default/file_engine_table`. -**2.** 手動で作成する `/var/lib/clickhouse/data/default/file_engine_table/data.TabSeparated` を含む: +**2.** 手動で作成 `/var/lib/clickhouse/data/default/file_engine_table/data.TabSeparated` 含む: ``` bash $ cat data.TabSeparated @@ -54,7 +54,7 @@ one 1 two 2 ``` -**3.** データのクエリ: +**3.** データの照会: ``` sql SELECT * FROM file_engine_table @@ -67,21 +67,21 @@ SELECT * FROM file_engine_table └──────┴───────┘ ``` -## Clickhouseでの使用-ローカル {#usage-in-clickhouse-local} +## ClickHouseでの使用-ローカル {#usage-in-clickhouse-local} -で [ﾂつ"ﾂづ按つｵﾂ！](../../../operations/utilities/clickhouse-local.md) ファイルエンジ `Format`. デフォルトの入力/出力ストリームは、数値または人間が読める名前を使用して指定できます `0` または `stdin`, `1` または `stdout`. -**例えば:** +で [ﾂつｨﾂ姪"ﾂ債ﾂつｹ](../../../operations/utilities/clickhouse-local.md) ファイルエンジンは、以下に加えて `Format`. デフォルトの入出力ストリームは、次のような数値または人間が読める名前で指定できます `0` または `stdin`, `1` または `stdout`. +**例:** ``` bash $ echo -e "1,2\n3,4" | clickhouse-local -q "CREATE TABLE table (a Int64, b Int64) ENGINE = File(CSV, stdin); SELECT a, b FROM table; DROP TABLE table" ``` -## 実装の詳細 {#details-of-implementation} +## 実施内容 {#details-of-implementation} -- 複数 `SELECT` クエリは同時に実行できますが、 `INSERT` クエリはお互いを待ちます。 -- 新しいファイルの作成に対応 `INSERT` クエリ。 +- 複数 `SELECT` クエリは同時に実行できますが `INSERT` クエリは互いに待機します。 +- 新しいファイルを作成する `INSERT` クエリ。 - ファイルが存在する場合, `INSERT` それに新しい値を追加します。 -- サポートなし: +- 対応していません: - `ALTER` - `SELECT ... SAMPLE` - 指数 diff --git a/docs/ja/engines/table-engines/special/generate.md b/docs/ja/engines/table-engines/special/generate.md index 7a23ecac598..58005ed2717 100644 --- a/docs/ja/engines/table-engines/special/generate.md +++ b/docs/ja/engines/table-engines/special/generate.md @@ -1,33 +1,33 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 46 toc_title: GenerateRandom --- # Generaterandom {#table_engines-generate} -のgeneraterandomテーブルエンジンの生産ランダムなデータが与えられたテーブルのスキーマ. +のGenerateRandomテーブルエンジンの生産ランダムなデータが与えられたテーブルのスキーマ. 使用例: -- 再現可能な大きいテーブルに住むテストの使用。 +- 再現可能で大きいテーブルを移入するテストの使用。 - ファジングテストのランダム入力を生成します。 -## Clickhouseサーバーでの使用状況 {#usage-in-clickhouse-server} +## ClickHouseサーバーでの使用状況 {#usage-in-clickhouse-server} ``` sql ENGINE = GenerateRandom(random_seed, max_string_length, max_array_length) ``` -その `max_array_length` と `max_string_length` すべ +その `max_array_length` と `max_string_length` パラメータを指定し最大限の長さのすべて 生成されたデータに対応する配列の列と文字列。 -テーブル生成エンジンは `SELECT` クエリ。 +Generate table engineのサポートのみ `SELECT` クエリ。 -対応して [データタイプ](../../../sql-reference/data-types/index.md) これは、以下を除いてテーブルに格納できます `LowCardinality` と `AggregateFunction`. +それはすべて [データ型](../../../sql-reference/data-types/index.md) を除いてテーブルに格納することができます `LowCardinality` と `AggregateFunction`. -**例えば:** +**例:** **1.** セットアップ `generate_engine_table` テーブル: @@ -35,7 +35,7 @@ ENGINE = GenerateRandom(random_seed, max_string_length, max_array_length) CREATE TABLE generate_engine_table (name String, value UInt32) ENGINE = GenerateRandom(1, 5, 3) ``` -**2.** データのクエリ: +**2.** データの照会: ``` sql SELECT * FROM generate_engine_table LIMIT 3 @@ -49,9 +49,9 @@ SELECT * FROM generate_engine_table LIMIT 3 └──────┴────────────┘ ``` -## 実装の詳細 {#details-of-implementation} +## 実施内容 {#details-of-implementation} -- サポートなし: +- 対応していません: - `ALTER` - `SELECT ... SAMPLE` - `INSERT` diff --git a/docs/ja/engines/table-engines/special/index.md b/docs/ja/engines/table-engines/special/index.md index 301ca7f0005..ae6bf09b8e7 100644 --- a/docs/ja/engines/table-engines/special/index.md +++ b/docs/ja/engines/table-engines/special/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_folder_title: Special +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "\u30B9\u30DA\u30B7\u30E3\u30EB" toc_priority: 31 --- diff --git a/docs/ja/engines/table-engines/special/join.md b/docs/ja/engines/table-engines/special/join.md index 866ddd2a959..4a3cc351e51 100644 --- a/docs/ja/engines/table-engines/special/join.md +++ b/docs/ja/engines/table-engines/special/join.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 40 toc_title: "\u53C2\u52A0" --- # 参加 {#join} -Inを使用するための準備済みデータ構造 [JOIN](../../../sql-reference/statements/select.md#select-join) オペレーション +で使用するための準備されたデータ構造 [JOIN](../../../sql-reference/statements/select/join.md#select-join) 作戦だ ## テーブルの作成 {#creating-a-table} @@ -19,19 +19,19 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] ) ENGINE = Join(join_strictness, join_type, k1[, k2, ...]) ``` -の詳細な説明を参照してください [CREATE TABLE](../../../sql-reference/statements/create.md#create-table-query) クエリ。 +の詳細な説明を見て下さい [CREATE TABLE](../../../sql-reference/statements/create.md#create-table-query) クエリ。 **エンジン変数** -- `join_strictness` – [厳密に結合する](../../../sql-reference/statements/select.md#select-join-strictness). -- `join_type` – [結合タイプ](../../../sql-reference/statements/select.md#select-join-types). -- `k1[, k2, ...]` – Key columns from the `USING` その句 `JOIN` 操作はで行われる。 +- `join_strictness` – [厳密に結合する](../../../sql-reference/statements/select/join.md#select-join-strictness). +- `join_type` – [結合タイプ](../../../sql-reference/statements/select/join.md#select-join-types). +- `k1[, k2, ...]` – Key columns from the `USING` 句は、 `JOIN` 操作はでなされる。 -入力 `join_strictness` と `join_type` 引用符なしのパラメーター。, `Join(ANY, LEFT, col1)`. 彼らは `JOIN` テーブルが使用される操作。 パラメータが一致しない場合、ClickHouseは例外をスローせず、誤ったデータを返すことがあります。 +入力 `join_strictness` と `join_type` 引用符のないパラメータ, `Join(ANY, LEFT, col1)`. も一致しなければならない `JOIN` テーブルが使用される操作。 パラメーターが一致しない場合、ClickHouseは例外をスローせず、誤ったデータを返す可能性があります。 ## テーブルの使用法 {#table-usage} -### 例えば {#example} +### 例 {#example} 左側のテーブルの作成: @@ -67,7 +67,7 @@ SELECT * FROM id_val ANY LEFT JOIN id_val_join USING (id) SETTINGS join_use_null └────┴─────┴─────────────────┘ ``` -代わりとして、データを取り出すことができます `Join` 結合キー値を指定するテーブル: +別の方法として、 `Join` 結合キー値を指定するテーブル: ``` sql SELECT joinGet('id_val_join', 'val', toUInt32(1)) @@ -81,12 +81,12 @@ SELECT joinGet('id_val_join', 'val', toUInt32(1)) ### データの選択と挿入 {#selecting-and-inserting-data} -を使用することができ `INSERT` データを追加するクエリ `Join`-エンジンテーブル。 テーブルが作成された場合 `ANY` 厳密さ、重複キーのデータは無視されます。 と `ALL` 厳密さは、すべての行が追加されます。 +以下を使用できます `INSERT` にデータを追加するクエリ `Join`-エンジンテーブル。 テーブルが作成された場合 `ANY` 厳密さ、重複キーのデータは無視されます。 と `ALL` 厳密さは、すべての行が追加されます。 -実行することはできません `SELECT` テーブルから直接クエリします。 代わりに、次のいずれかの方法を使用します: +を実行できません。 `SELECT` テーブルから直接クエリ。 代わりに、次のいずれかの方法を使用します: -- テーブルをaの右側に置きます `JOIN` 句。 -- コールを [joinGet](../../../sql-reference/functions/other-functions.md#joinget) この関数を使用すると、テーブルからデータをディクショナリと同じ方法で抽出できます。 +- Aの右側にテーブルを置いて下さい `JOIN` 句。 +- コール [joinGet](../../../sql-reference/functions/other-functions.md#joinget) 辞書と同じ方法でテーブルからデータを抽出できる関数。 ### 制限事項と設定 {#join-limitations-and-settings} @@ -98,14 +98,14 @@ SELECT joinGet('id_val_join', 'val', toUInt32(1)) - [join\_overflow\_mode](../../../operations/settings/query-complexity.md#settings-join_overflow_mode) - [join\_any\_take\_last\_row](../../../operations/settings/settings.md#settings-join_any_take_last_row) -その `Join`-エンジンテーブルは使用できません `GLOBAL JOIN` オペレーション +その `Join`-エンジンテーブルは使用できません `GLOBAL JOIN` 作戦だ -その `Join`-エンジンは、使用 [join\_use\_nulls](../../../operations/settings/settings.md#join_use_nulls) の設定 `CREATE TABLE` 声明。 と [SELECT](../../../sql-reference/statements/select.md) クエリは、使用を可能に `join_use_nulls` あまりにも。 あなたが持って異なる `join_use_nulls` 設定は、テーブルを結合エラーを得ることができます。 それは結合の種類に依存します。 使用するとき [joinGet](../../../sql-reference/functions/other-functions.md#joinget) 機能、同じを使用しなければなりません `join_use_nulls` の設定 `CRATE TABLE` と `SELECT` 文。 +その `Join`-エンジンは使用を許可する [join\_use\_nulls](../../../operations/settings/settings.md#join_use_nulls) の設定 `CREATE TABLE` 声明。 と [SELECT](../../../sql-reference/statements/select/index.md) クエリを使用できる `join_use_nulls` あまりにも。 あなたが違う場合 `join_use_nulls` 設定することができるエラー入社。 それは結合の種類に依存します。 使用するとき [joinGet](../../../sql-reference/functions/other-functions.md#joinget) 関数、あなたは同じを使用する必要があります `join_use_nulls` 設定 `CRATE TABLE` と `SELECT` 声明。 -## データ記憶 {#data-storage} +## データ保存 {#data-storage} `Join` テーブルデータは常にRAMにあります。 を挿入する際、列表ClickHouseに書き込みデータブロックのディレクトリのディスクできるように復元され、サーバが再起動してしまいます。 -場合はサーバが再起動誤り、データブロックのディスクがいます。 この場合、破損したデータを含むファイルを手動で削除する必要があります。 +場合はサーバが再起動誤り、データブロックのディスクがいます。 この場合、破損したデータを含むファイルを手動で削除する必要がある場合があります。 [元の記事](https://clickhouse.tech/docs/en/operations/table_engines/join/) diff --git a/docs/ja/engines/table-engines/special/materializedview.md b/docs/ja/engines/table-engines/special/materializedview.md index 1e52922cae6..2935e0492ce 100644 --- a/docs/ja/engines/table-engines/special/materializedview.md +++ b/docs/ja/engines/table-engines/special/materializedview.md @@ -1,12 +1,12 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 43 -toc_title: MaterializedView +toc_title: "\u30DE\u30C6\u30EA\u30A2\u30E9\u30A4\u30BA\u30C9\u30D3\u30E5\u30FC" --- -# Materializedview {#materializedview} +# マテリアライズドビュー {#materializedview} -マテリアライズドビューの実装に使用されます(詳細については、 [CREATE TABLE](../../../sql-reference/statements/create.md)). データを格納するために、ビューの作成時に指定された別のエンジンを使用します。 テーブルから読み取るときは、このエンジンを使用します。 +マテリアライズドビューの実装に使用されます(詳細については、 [CREATE TABLE](../../../sql-reference/statements/create.md)). データを格納するために、ビューの作成時に指定された別のエンジンを使用します。 読み込み時にテーブルから、使用してこのエンジンです。 [元の記事](https://clickhouse.tech/docs/en/operations/table_engines/materializedview/) diff --git a/docs/ja/engines/table-engines/special/memory.md b/docs/ja/engines/table-engines/special/memory.md index 06ae102558a..c2b50553023 100644 --- a/docs/ja/engines/table-engines/special/memory.md +++ b/docs/ja/engines/table-engines/special/memory.md @@ -1,19 +1,19 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 44 toc_title: "\u30E1\u30E2\u30EA" --- # メモリ {#memory} -メモリエンジンは、データを非圧縮形式でramに格納します。 データは、読み取り時に受信されるのとまったく同じ形式で格納されます。 言い換えれば、この表からの読み取りは完全に無料です。 +メモリエンジンは、非圧縮形式でRAMにデータを格納します。 データは、読み取り時に受信されるのとまったく同じ形式で格納されます。 言い換えれば、この表からの読書は完全に無料です。 同時データアクセスは同期されます。 ロックは短く、読み取り操作と書き込み操作は互いにブロックしません。 索引はサポートされません。 読み取りは並列化されます。 -ディスクからの読み取り、デンプレッシング、データの逆シリアル化がないため、単純なクエリで最大の生産性(10gb/秒以上)に達します。 （多くの場合、mergetreeエンジンの生産性はほぼ同じ高さにあることに注意してください。) +単純なクエリでは、ディスクからの読み取り、データの解凍、または逆シリアル化が行われないため、最大生産性(10GB/秒以上)に達します。 （多くの場合、MergeTreeエンジンの生産性はほぼ同じくらい高いことに注意してください。) サーバーを再起動すると、テーブルからデータが消え、テーブルが空になります。 -通常、このテーブルエンジンの使用は正当化されません。 ただし、テストや、比較的少数の行で最大速度が必要なタスク(およそ100,000,000まで)に使用できます。 +通常、このテーブルエンジンの使用は正当化されません。 ただし、テストや、比較的少数の行(最大約100,000,000)で最大速度が必要なタスクに使用できます。 -メモリーのエンジンを使用するシステムの一時テーブルの外部クエリデータの項をご参照ください “External data for processing a query”グローバルでの実装のために(セクションを参照 “IN operators”). +メモリーのエンジンを使用するシステムの一時テーブルの外部クエリデータの項をご参照ください “External data for processing a query”グローバルでの実装については、セクションを参照 “IN operators”). [元の記事](https://clickhouse.tech/docs/en/operations/table_engines/memory/) diff --git a/docs/ja/engines/table-engines/special/merge.md b/docs/ja/engines/table-engines/special/merge.md index c88ec22cf3b..1d2ff7ce794 100644 --- a/docs/ja/engines/table-engines/special/merge.md +++ b/docs/ja/engines/table-engines/special/merge.md @@ -1,37 +1,37 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 36 toc_title: "\u30DE\u30FC\u30B8" --- # マージ {#merge} -その `Merge` エンジン（と混同しないように `MergeTree`）データ自体を格納しませんが、同時に他のテーブルの任意の数からの読み取りを可能にします。 -読書は自動的に平行である。 表への書き込みはサポートされません。 読み取り時には、実際に読み取られているテーブルのインデックスが存在する場合に使用されます。 -その `Merge` テーブルのデータベース名と正規表現です。 +その `Merge` エンジン（と混同しないように `MergeTree`)を格納していないデータそのものができるから、他のテーブルを同時に +読み取りは自動的に並列化されます。 表への書き込みはサポートされません。 読み取り時には、実際に読み取られているテーブルのインデックスが存在する場合に使用されます。 +その `Merge` データベース名とテーブルの正規表現です。 -例えば: +例: ``` sql Merge(hits, '^WatchLog') ``` -データはテーブルから読み込まれます。 `hits` 正規表現に一致する名前を持つデータベース ‘`^WatchLog`’. +データはのテーブルから読まれます `hits` 正規表現に一致する名前を持つデータベース ‘`^WatchLog`’. データベース名の代わりに、文字列を返す定数式を使用できます。 例えば, `currentDatabase()`. -Regular expressions — [re2unit description in lists](https://github.com/google/re2) (PCREのサブセットをサポート)、大文字と小文字を区別します。 -正規表現のエスケープシンボルに関する注意事項を参照してください。 “match” セクション。 +Regular expressions — [re2](https://github.com/google/re2) (PCREのサブセットをサポート)、大文字と小文字を区別します。 +正規表現におけるシンボルのエスケープに関する注意 “match” セクション -読み込むテーブルを選択するとき、 `Merge` 正規表現と一致していても、テーブル自体は選択されません。 これはループを避けるためです。 -それは可能に作成二つ `Merge` お互いのデータを無限に読み取ろうとするテーブルですが、これは良い考えではありません。 +読み取るテーブルを選択するとき、 `Merge` 正規表現と一致しても、テーブル自体は選択されません。 これはループを避けるためです。 +二つを作成することが可能です `Merge` 無限にお互いのデータを読み込もうとするテーブルですが、これは良い考えではありません。 -使用する典型的な方法 `Merge` エンジンは多数を使用のためです `TinyLog` 単一のテーブルと同様にテーブル。 +使用する典型的な方法 `Merge` エンジンは多数を使用のためです `TinyLog` 単一のテーブルを持つかのようにテーブル。 例2: -古いテーブル（watchlog\_old）があり、データを新しいテーブル（watchlog\_new）に移動せずにパーティション分割を変更することにしたとしましょう。 +古いテーブル（WatchLog\_old）があり、データを新しいテーブル（WatchLog\_new）に移動せずにパーティション分割を変更することにしたとしましょう。 ``` sql CREATE TABLE WatchLog_old(date Date, UserId Int64, EventType String, Cnt UInt64) @@ -61,9 +61,9 @@ FROM WatchLog - `_table` — Contains the name of the table from which data was read. Type: [文字列](../../../sql-reference/data-types/string.md). - 定数条件を設定することができます `_table` で `WHERE/PREWHERE` 句(例えば, `WHERE _table='xyz'`). この場合、読み取り操作はそのテーブルに対してのみ実行されます。 `_table` は満足しているので、 `_table` 列は索引として機能します。 + 定数条件は次のように設定できます `_table` で `WHERE/PREWHERE` 句(例えば, `WHERE _table='xyz'`). この場合、読み取り操作は、条件がオンのテーブルに対してのみ実行されます `_table` は満足しているので、 `_table` 列は索引として機能します。 -**また見なさい** +**も参照。** - [仮想列](index.md#table_engines-virtual_columns) diff --git a/docs/ja/engines/table-engines/special/null.md b/docs/ja/engines/table-engines/special/null.md index 24af505190d..588500cdcde 100644 --- a/docs/ja/engines/table-engines/special/null.md +++ b/docs/ja/engines/table-engines/special/null.md @@ -1,14 +1,14 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 38 -toc_title: "\u30CC\u30EB" +toc_title: 'Null' --- -# ヌル {#null} +# Null {#null} Nullテーブルに書き込む場合、データは無視されます。 Nullテーブルから読み取る場合、応答は空です。 -ただし、nullテーブルにマテリアライズドビューを作成できます。 したがって、テーブルに書き込まれたデータはビューに表示されます。 +ただし、Nullテーブルにマテリアライズドビューを作成できます。 したがって、テーブルに書き込まれたデータはビュー内で終了します。 [元の記事](https://clickhouse.tech/docs/en/operations/table_engines/null/) diff --git a/docs/ja/engines/table-engines/special/set.md b/docs/ja/engines/table-engines/special/set.md index 6099fca639e..2b6cd0a2a9e 100644 --- a/docs/ja/engines/table-engines/special/set.md +++ b/docs/ja/engines/table-engines/special/set.md @@ -1,19 +1,19 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 39 toc_title: "\u30BB\u30C3\u30C8" --- # セット {#set} -常にramにあるデータセット。 これは、in演算子の右側で使用するためのものです（セクションを参照 “IN operators”). +常にRAM内にあるデータ-セット。 これは、in演算子の右側での使用を意図しています（セクションを参照してください “IN operators”). -INSERTを使用すると、テーブルにデータを挿入できます。 新しい要素がデータセットに追加され、重複が無視されます。 -しかし、表からselectを実行することはできません。 データを取得する唯一の方法は、in演算子の右半分でデータを使用することです。 +INSERTを使用すると、テーブルにデータを挿入できます。 新しい要素がデータセットに追加され、重複は無視されます。 +ができない行から選択します。 データを取得する唯一の方法は、in演算子の右半分でデータを使用することです。 -データは常にramにあります。 insertの場合、挿入されたデータのブロックもディスク上のテーブルのディレクトリに書き込まれます。 サーバーを起動すると、このデータはramにロードされます。 つまり、再起動後もデータはそのまま残ります。 +データは常にRAMにあります。 INSERTでは、挿入されたデータのブロックもディスク上のテーブルのディレクトリに書き込まれます。 サーバーを起動すると、このデータはRAMにロードされます。 つまり、再起動後、データは所定の位置に残ります。 -ラサーバを再起動し、ブロックのデータのディスクが失われることも想定されます。 後者の場合、破損したデータを含むファイルを手動で削除する必要があります。 +ラサーバを再起動し、ブロックのデータのディスクが失われることも想定されます。 後者の場合、破損したデータを含むファイルを手動で削除する必要がある場合があります。 [元の記事](https://clickhouse.tech/docs/en/operations/table_engines/set/) diff --git a/docs/ja/engines/table-engines/special/url.md b/docs/ja/engines/table-engines/special/url.md index 3c5d1ea8448..ded439caa4c 100644 --- a/docs/ja/engines/table-engines/special/url.md +++ b/docs/ja/engines/table-engines/special/url.md @@ -1,34 +1,34 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 41 toc_title: URL --- -# URL(URL,フォーマット) {#table_engines-url} +# URL(URL,形式) {#table_engines-url} -リモートhttp/httpsサーバー上のデータを管理します。 このエンジンは同様です -に [ファイル](file.md) エンジン。 +リモートHTTP/HTTPSサーバー上のデータを管理します。 このエンジンは同様です +に [ファイル](file.md) エンジン -## Clickhouseサーバーでのエンジンの使用 {#using-the-engine-in-the-clickhouse-server} +## ClickHouseサーバーでのエンジンの使用 {#using-the-engine-in-the-clickhouse-server} その `format` ClickHouseが使用できるものでなければなりません -`SELECT` クエリと、必要に応じて、 `INSERTs`. サポートさ +`SELECT` クエリと、必要に応じて、 `INSERTs`. サポートされている形式の完全な一覧については、 [形式](../../../interfaces/formats.md#formats). -その `URL` 統一リソースロケータの構造に準拠する必要があります。 指定されたURLはサーバーを指す必要があります -HTTPまたはHTTPSを使用します。 これには何も必要ありません -サーバーからの応答を取得するための追加のヘッダー。 +その `URL` Uniform Resource Locatorの構造に準拠している必要があります。 指定したURLはサーバーを指す必要があります +HTTPまたはHTTPSを使用します。 これは必要ありません +サーバーからの応答を取得するための追加ヘッダー。 -`INSERT` と `SELECT` 質問への `POST` と `GET` リクエスト, -それぞれ。 処理のため `POST` ご要望遠隔のサーバをサポートする必要があ +`INSERT` と `SELECT` 質問への `POST` と `GET` 要求, +それぞれ。 処理のため `POST` 要求は、リモートサーバーが [チャンク転送エンコード](https://en.wikipedia.org/wiki/Chunked_transfer_encoding). -HTTP GETリダイレクトホップの最大数を制限するには、次のコマンドを使用します [max\_http\_get\_redirects](../../../operations/settings/settings.md#setting-max_http_get_redirects) 設定。 +HTTP GETリダイレクトホップの最大数を制限するには [max\_http\_get\_redirects](../../../operations/settings/settings.md#setting-max_http_get_redirects) 設定。 -**例えば:** +**例:** -**1.** 作成する `url_engine_table` サーバー上の表 : +**1.** 作成 `url_engine_table` サーバー上のテーブル : ``` sql CREATE TABLE url_engine_table (word String, value UInt64) @@ -71,12 +71,12 @@ SELECT * FROM url_engine_table └───────┴───────┘ ``` -## 実装の詳細 {#details-of-implementation} +## 実施内容 {#details-of-implementation} - 読み書きできる並列 -- サポートなし: - - `ALTER` と `SELECT...SAMPLE` オペレーション +- 対応していません: + - `ALTER` と `SELECT...SAMPLE` 作戦だ - インデックス。 - - 複製だ + - 複製。 [元の記事](https://clickhouse.tech/docs/en/operations/table_engines/url/) diff --git a/docs/ja/engines/table-engines/special/view.md b/docs/ja/engines/table-engines/special/view.md index cee57bd86e1..270246bedb9 100644 --- a/docs/ja/engines/table-engines/special/view.md +++ b/docs/ja/engines/table-engines/special/view.md @@ -1,12 +1,12 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 42 -toc_title: "\u30D3\u30E5\u30FC" +toc_title: "\u8868\u793A" --- -# ビュー {#table_engines-view} +# 表示 {#table_engines-view} -ビューを実装するために使用されます(詳細については、 `CREATE VIEW query`). で格納していないデータだけで店舗を指定された `SELECT` クエリ。 テーブルから読み取るときは、このクエリを実行します（クエリから不要な列をすべて削除します）。 +ビューの実装に使用されます(詳細については、 `CREATE VIEW query`). これはデータを格納せず、指定されたデータのみを格納します `SELECT` クエリ。 テーブルから読み取るときに、このクエリが実行されます（クエリから不要な列がすべて削除されます）。 [元の記事](https://clickhouse.tech/docs/en/operations/table_engines/view/) diff --git a/docs/ja/faq/general.md b/docs/ja/faq/general.md index e188f1ed41c..260826160d2 100644 --- a/docs/ja/faq/general.md +++ b/docs/ja/faq/general.md @@ -1,25 +1,25 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 78 toc_title: "\u4E00\u822C\u7684\u306A\u8CEA\u554F" --- # 一般的な質問 {#general-questions} -## なぜmapreduceのようなものを使わないのですか？ {#why-not-use-something-like-mapreduce} +## なぜMapReduceのようなものを使わないのですか？ {#why-not-use-something-like-mapreduce} -MapReduceのようなシステムは、reduce操作が分散ソートに基づいている分散計算システムとして参照することができます。 このクラ [Apache Hadoop](http://hadoop.apache.org). Yandexのは、その社内ソリューション、YTを使用しています。 +MapReduceのようなシステムは、reduce操作が分散ソートに基づいている分散コンピューティングシステムと呼ぶことができます。 このクラ [Apache Hadoop](http://hadoop.apache.org). Yandexは社内ソリューションYTを使用しています。 -これらのシス つまり、webインターフェイスのバックエンドとして使用することはできません。 これらのシステムなに役立つリアルタイムデータの更新をした。 分散ソートは、操作の結果とすべての中間結果（存在する場合）が単一のサーバーのramにある場合、reduce操作を実行する最善の方法ではありません。 このような場合、ハッシュテーブルはreduce操作を実行するのに最適な方法です。 map-reduceタスクを最適化する一般的なアプローチは、ramのハッシュテーブルを使用した事前集約(部分削減)です。 ユーザーはこの最適化を手動で実行します。 分散ソートは、単純なmap-reduceタスクを実行するときのパフォーマンス低下の主な原因の一つです。 +これらのシステムなの適切なオンラインのクエリによる高の待ち時間をゼロにすることに つまり、webインターフェイスのバックエンドとして使用することはできません。 これらのシステムなに役立つリアルタイムデータの更新をした。 分散ソートは、操作の結果とすべての中間結果（存在する場合）が単一のサーバーのRAMにある場合、reduce操作を実行する最良の方法ではありません。 このような場合、ハッシュテーブルはreduce操作を実行する最適な方法です。 Map-reduceタスクを最適化するための一般的なアプローチは、RAM内のハッシュテーブルを使用した事前集約（部分削減）です。 ユーザーはこの最適化を手動で実行します。 分散ソートは、単純なmap-reduceタスクを実行するときのパフォーマンスの低下の主な原因の一つです。 -ほとんどのmapreduce実装では、クラスター上で任意のコードを実行できます。 しかし、宣言的なクエリ言語は、実験を迅速に実行するためにolapに適しています。 たとえば、hadoopにはhiveとpigがあります。 また、sparkのためのcloudera impalaまたはshark（旧式）、spark sql、presto、およびapache drillも検討してください。 このようなタスクを実行するときのパフォーマンスは、特殊なシステムに比べて非常に低いですが、比較的待ち時間が長いため、これらのシステムを +ほとんどのMapReduce実装では、クラスター上で任意のコードを実行できます。 しかし、宣言型クエリ言語は、実験を迅速に実行するためにOLAPに適しています。 たとえば、HadoopにはHiveとPigがあります。 また、SparkのCloudera ImpalaやShark(旧式)、Spark SQL、Presto、Apache Drillについても検討してください。 このようなタスクを実行するときのパフォーマンスは、特殊なシステムに比べて非常に最適ではありませんが、比較的遅延が高いため、これらのシステ -## ORACLEをODBC経由で使用するときにエンコードに問題がある場合はどうなりますか？ {#oracle-odbc-encodings} +## になっているのでしょうか？しているのでエンコーディング利用の場合OracleをODBC? {#oracle-odbc-encodings} -外部辞書のソースとしてodbcドライバーを使用してoracleを使用する場合は、正しい値を設定する必要があります。 `NLS_LANG` の環境変数 `/etc/default/clickhouse`. 詳細については、 [Oracle NLS\_LANG FAQ](https://www.oracle.com/technetwork/products/globalization/nls-lang-099431.html). +外部ディクショナリのソースとしてODBCドライバを介してOracleを使用する場合は、外部ディクショナリの正しい値を設定する必要が `NLS_LANG` 環境変数in `/etc/default/clickhouse`. 詳細については、を参照してください [Oracle NLS\_LANG FAQ](https://www.oracle.com/technetwork/products/globalization/nls-lang-099431.html). -**例えば** +**例** ``` sql NLS_LANG=RUSSIAN_RUSSIA.UTF8 @@ -29,7 +29,7 @@ NLS_LANG=RUSSIAN_RUSSIA.UTF8 ### INTO OUTFILE句の使用 {#using-into-outfile-clause} -追加 [INTO OUTFILE](../query_language/select/#into-outfile-clause) クエリへの句。 +追加 [INTO OUTFILE](../sql-reference/statements/select/into-outfile.md#into-outfile-clause) クエリの句。 例えば: @@ -37,7 +37,7 @@ NLS_LANG=RUSSIAN_RUSSIA.UTF8 SELECT * FROM table INTO OUTFILE 'file' ``` -デフォルトでは、clickhouseは [タブ区切り](../interfaces/formats.md#tabseparated) 出力データの形式。 を選択する [データ形式](../interfaces/formats.md)、を使用 [フォーマット句](../query_language/select/#format-clause). +デフォルトでは、ClickHouseは [TabSeparated](../interfaces/formats.md#tabseparated) 出力データの形式。 選択するには [データ形式](../interfaces/formats.md) は、 [FORMAT句](../sql-reference/statements/select/format.md#format-clause). 例えば: @@ -49,12 +49,12 @@ SELECT * FROM table INTO OUTFILE 'file' FORMAT CSV 見る [ファイル](../engines/table-engines/special/file.md). -### コマンドラインのリダイ {#using-command-line-redirection} +### コマンドラインリダイレクト {#using-command-line-redirection} ``` sql $ clickhouse-client --query "SELECT * from table" --format FormatName > result.txt ``` -見る [クリックハウス-顧客](../interfaces/cli.md). +見る [clickhouse-クライアント](../interfaces/cli.md). {## [元の記事](https://clickhouse.tech/docs/en/faq/general/) ##} diff --git a/docs/ja/faq/index.md b/docs/ja/faq/index.md index d0338c572e4..a44dbb31e89 100644 --- a/docs/ja/faq/index.md +++ b/docs/ja/faq/index.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_folder_title: F.A.Q. toc_priority: 76 --- diff --git a/docs/ja/getting-started/example-datasets/amplab-benchmark.md b/docs/ja/getting-started/example-datasets/amplab-benchmark.md index 2bf079238ba..06e318f3442 100644 --- a/docs/ja/getting-started/example-datasets/amplab-benchmark.md +++ b/docs/ja/getting-started/example-datasets/amplab-benchmark.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 17 toc_title: "AMPLab Big Data\u30D9\u30F3\u30C1\u30DE\u30FC\u30AF" --- # AMPLab Big Dataベンチマーク {#amplab-big-data-benchmark} -見るhttps://amplab.cs.berkeley.edu/benchmark/ +参照https://amplab.cs.berkeley.edu/benchmark/ で無料アカウントにサインアップhttps://aws.amazon.com.それはクレジットカード、電子メール、および電話番号が必要です。 新しいアクセスキーを取得するhttps://console.aws.amazon.com/iam/home?nc2=h\_m\_sc\#security\_credential @@ -26,7 +26,7 @@ $ s3cmd sync s3://big-data-benchmark/pavlo/text-deflate/5nodes/ . $ cd .. ``` -次のclickhouseクエリを実行します: +次のClickHouseクエリを実行します: ``` sql CREATE TABLE rankings_tiny @@ -101,7 +101,7 @@ $ for i in 5nodes/rankings/*.deflate; do echo $i; zlib-flate -uncompress < $i | $ for i in 5nodes/uservisits/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO uservisits_5nodes_on_single FORMAT CSV"; done ``` -デー: +クエリデータの取得サンプル: ``` sql SELECT pageURL, pageRank FROM rankings_1node WHERE pageRank > 1000 diff --git a/docs/ja/getting-started/example-datasets/criteo.md b/docs/ja/getting-started/example-datasets/criteo.md index 63aee0bf6d9..bd45935c4e3 100644 --- a/docs/ja/getting-started/example-datasets/criteo.md +++ b/docs/ja/getting-started/example-datasets/criteo.md @@ -1,21 +1,22 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 19 -toc_title: "Criteo\u304B\u3089\u306ETerabyte\u30AF\u30EA\u30C3\u30AF\u30ED\u30B0" +toc_title: "Criteo\u304B\u3089\u30C6\u30E9\u30D0\u30A4\u30C8\u306E\u30AF\u30EA\u30C3\ + \u30AF\u30ED\u30B0" --- # Criteoからのクリックログのテラバイト {#terabyte-of-click-logs-from-criteo} データのダウンロードhttp://labs.criteo.com/downloads/download-terabyte-click-logs/ -ログをインポートするテーブルを作成する: +ログをインポートするテーブルを作成します: ``` sql CREATE TABLE criteo_log (date Date, clicked UInt8, int1 Int32, int2 Int32, int3 Int32, int4 Int32, int5 Int32, int6 Int32, int7 Int32, int8 Int32, int9 Int32, int10 Int32, int11 Int32, int12 Int32, int13 Int32, cat1 String, cat2 String, cat3 String, cat4 String, cat5 String, cat6 String, cat7 String, cat8 String, cat9 String, cat10 String, cat11 String, cat12 String, cat13 String, cat14 String, cat15 String, cat16 String, cat17 String, cat18 String, cat19 String, cat20 String, cat21 String, cat22 String, cat23 String, cat24 String, cat25 String, cat26 String) ENGINE = Log ``` -ダウンロードデータ: +データダウンロード: ``` bash $ for i in {00..23}; do echo $i; zcat datasets/criteo/day_${i#0}.gz | sed -r 's/^/2000-01-'${i/00/24}'\t/' | clickhouse-client --host=example-perftest01j --query="INSERT INTO criteo_log FORMAT TabSeparated"; done @@ -70,7 +71,7 @@ CREATE TABLE criteo ) ENGINE = MergeTree(date, intHash32(icat1), (date, intHash32(icat1)), 8192) ``` -生のログからデータを変換し、第二のテーブルにそれを置きます: +生のログからデータを変換し、第二のテーブルに入れます: ``` sql INSERT INTO criteo SELECT date, clicked, int1, int2, int3, int4, int5, int6, int7, int8, int9, int10, int11, int12, int13, reinterpretAsUInt32(unhex(cat1)) AS icat1, reinterpretAsUInt32(unhex(cat2)) AS icat2, reinterpretAsUInt32(unhex(cat3)) AS icat3, reinterpretAsUInt32(unhex(cat4)) AS icat4, reinterpretAsUInt32(unhex(cat5)) AS icat5, reinterpretAsUInt32(unhex(cat6)) AS icat6, reinterpretAsUInt32(unhex(cat7)) AS icat7, reinterpretAsUInt32(unhex(cat8)) AS icat8, reinterpretAsUInt32(unhex(cat9)) AS icat9, reinterpretAsUInt32(unhex(cat10)) AS icat10, reinterpretAsUInt32(unhex(cat11)) AS icat11, reinterpretAsUInt32(unhex(cat12)) AS icat12, reinterpretAsUInt32(unhex(cat13)) AS icat13, reinterpretAsUInt32(unhex(cat14)) AS icat14, reinterpretAsUInt32(unhex(cat15)) AS icat15, reinterpretAsUInt32(unhex(cat16)) AS icat16, reinterpretAsUInt32(unhex(cat17)) AS icat17, reinterpretAsUInt32(unhex(cat18)) AS icat18, reinterpretAsUInt32(unhex(cat19)) AS icat19, reinterpretAsUInt32(unhex(cat20)) AS icat20, reinterpretAsUInt32(unhex(cat21)) AS icat21, reinterpretAsUInt32(unhex(cat22)) AS icat22, reinterpretAsUInt32(unhex(cat23)) AS icat23, reinterpretAsUInt32(unhex(cat24)) AS icat24, reinterpretAsUInt32(unhex(cat25)) AS icat25, reinterpretAsUInt32(unhex(cat26)) AS icat26 FROM criteo_log; diff --git a/docs/ja/getting-started/example-datasets/index.md b/docs/ja/getting-started/example-datasets/index.md index f4bb5e33ad5..53a8bc2f5f8 100644 --- a/docs/ja/getting-started/example-datasets/index.md +++ b/docs/ja/getting-started/example-datasets/index.md @@ -1,22 +1,22 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_folder_title: Example Datasets +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "\u30C7\u30FC\u30BF\u30BB\u30C3\u30C8\u4F8B" toc_priority: 12 -toc_title: "\u5C0E\u5165" +toc_title: "\u306F\u3058\u3081\u306B" --- # データセット例 {#example-datasets} -この方法について説明し得る例データセットおよび輸入しclickhouse. +この方法について説明し得る例データセットおよび輸入しClickHouse. 一部のデータセットの例ではクエリーもございます。 -- [匿名のyandexの。metricaデータセット](metrica.md) -- [スタースキーマのベンチマーク](star-schema.md) -- [WikiStat](wikistat.md) +- [匿名Yandexの。メトリカデータセット](metrica.md) +- [Star Schemaベンチマーク](star-schema.md) +- [ウィキスタット](wikistat.md) - [Criteoからのクリックログのテラバイト](criteo.md) - [AMPLab Big Dataベンチマーク](amplab-benchmark.md) -- [ニューヨーク](nyc-taxi.md) +- [ニューヨークタクシ](nyc-taxi.md) - [オンタイム](ontime.md) [元の記事](https://clickhouse.tech/docs/en/getting_started/example_datasets) diff --git a/docs/ja/getting-started/example-datasets/metrica.md b/docs/ja/getting-started/example-datasets/metrica.md index a53831afc89..897331ea97a 100644 --- a/docs/ja/getting-started/example-datasets/metrica.md +++ b/docs/ja/getting-started/example-datasets/metrica.md @@ -1,15 +1,15 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_priority: 21 -toc_title: "Yandex\u306E\u3002Metrica\u30C7\u30FC\u30BF" +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_priority: 14 +toc_title: "Yandex.\u30E1\u30C8\u30EA\u30AB\u30C7\u30FC\u30BF" --- -# 匿名のyandexの。metricaデータ {#anonymized-yandex-metrica-data} +# 匿名Yandexの。メトリカデータ {#anonymized-yandex-metrica-data} -データセットのテーブルを含む匿名化されたデーター (`hits_v1`)および訪問 (`visits_v1`）Yandexの。メトリカ につなげていくかを学びますYandex.Metrica in [クリックハウスの歴史](../../introduction/history.md) セクション。 +データセット (`hits_v1`）と訪問 (`visits_v1` Yandexの）。メトリカ につなげていくかを学びますYandex.メトリカ [クリックハウスの歴史](../../introduction/history.md) セクション -どちらのテーブルも圧縮形式でダウンロードできます `tsv.xz` ファイルに対して割. それに加えて、の拡張バージョン `hits` 100万行を含むテーブルは、TSVとして利用可能ですhttps://clickhouse-datasets.s3.yandex.net/hits/tsv/hits\_100m\_obfuscated\_v1.tsv.xz そして準備された仕切りとしてhttps://clickhouse-datasets.s3.yandex.net/hits/partitions/hits\_100m\_obfuscated\_v1.tar.xz。 +のデータセットのテーブル、つぶやきの本文は、以下のクエリをダウンロードした圧縮 `tsv.xz` ファイルに対して割. それに加えて、の拡張バージョン `hits` テーブルを含む100万行はTSVでhttps://clickhouse-datasets.s3.yandex.net/hits/tsv/hits\_100m\_obfuscated\_v1.tsv.xz としての準備が大切でhttps://clickhouse-datasets.s3.yandex.net/hits/partitions/hits\_100m\_obfuscated\_v1.tar.xz. ## 取得のテーブルから調の間仕切り {#obtaining-tables-from-prepared-partitions} @@ -23,7 +23,7 @@ sudo service clickhouse-server restart clickhouse-client --query "SELECT COUNT(*) FROM datasets.hits_v1" ``` -ダウンロードと読み込み: +ダウンロード、輸入訪問: ``` bash curl -O https://clickhouse-datasets.s3.yandex.net/visits/partitions/visits_v1.tar @@ -33,9 +33,9 @@ sudo service clickhouse-server restart clickhouse-client --query "SELECT COUNT(*) FROM datasets.visits_v1" ``` -## 圧縮されたtsvファイルからの表の取得 {#obtaining-tables-from-compressed-tsv-file} +## 圧縮TSVファイルからのテーブルの取得 {#obtaining-tables-from-compressed-tsv-file} -圧縮されたtsvファ: +圧縮TSVファイルか: ``` bash curl https://clickhouse-datasets.s3.yandex.net/hits/tsv/hits_v1.tsv.xz | unxz --threads=`nproc` > hits_v1.tsv @@ -49,7 +49,7 @@ clickhouse-client --query "OPTIMIZE TABLE datasets.hits_v1 FINAL" clickhouse-client --query "SELECT COUNT(*) FROM datasets.hits_v1" ``` -圧縮tsv-fileからの訪問のダウンロードとインポート: +ダウンロード、輸入からの訪問圧縮tsvファイル: ``` bash curl https://clickhouse-datasets.s3.yandex.net/visits/tsv/visits_v1.tsv.xz | unxz --threads=`nproc` > visits_v1.tsv @@ -63,8 +63,8 @@ clickhouse-client --query "OPTIMIZE TABLE datasets.visits_v1 FINAL" clickhouse-client --query "SELECT COUNT(*) FROM datasets.visits_v1" ``` -## クエリ例 {#example-queries} +## クエリの例 {#example-queries} -[ClickHouseチュートリアル](../../getting-started/tutorial.md) は、Yandexのに基づいています。Metricaデータの推奨使うことができるようにこのデータやトランザクションデータだけを通してチュートリアルです。 +[ClickHouseチュートリアル](../../getting-started/tutorial.md) Yandexに基づいています。Metricaデータの推奨使うことができるようにこのデータやトランザクションデータだけを通してチュートリアルです。 -これらのテーブルへのクエリの追加の例は、 [ステートフルテスト](https://github.com/ClickHouse/ClickHouse/tree/master/tests/queries/1_stateful) ClickHouseの（彼らは名前が付けられています `test.hists` と `test.visits` そこ）。 +追加の質問をこれらのテーブルで見られるもので、 [ステートフルテスト](https://github.com/ClickHouse/ClickHouse/tree/master/tests/queries/1_stateful) クリックハウ `test.hists` と `test.visits` ）がある。 diff --git a/docs/ja/getting-started/example-datasets/nyc-taxi.md b/docs/ja/getting-started/example-datasets/nyc-taxi.md index 340af79c9b3..c2000422f56 100644 --- a/docs/ja/getting-started/example-datasets/nyc-taxi.md +++ b/docs/ja/getting-started/example-datasets/nyc-taxi.md @@ -1,23 +1,23 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 16 -toc_title: "\u30CB\u30E5\u30FC\u30E8\u30FC\u30AF" +toc_title: "\u30CB\u30E5\u30FC\u30E8\u30FC\u30AF\u30BF\u30AF\u30B7" --- -# ニューヨーク {#new-york-taxi-data} +# ニューヨークタクシ {#new-york-taxi-data} このデータセットは二つの方法で取得できます: - 生データからインポート - ダウンロード調の間仕切り -## 生データをインポートする方法 {#how-to-import-the-raw-data} +## 生データのインポート方法 {#how-to-import-the-raw-data} -見るhttps://github.com/toddwschneider/nyc-taxi-data とhttp://tech.marksblogg.com/billion-nyc-taxi-rides-redshift.html データセットの説明とダウンロードの手順について。 +参照https://github.com/toddwschneider/nyc-taxi-data そしてhttp://tech.marksblogg.com/billion-nyc-taxi-rides-redshift.html データセットの説明とダウンロード手順。 -ダウンロードは、csvファイル内の非圧縮データの約227ギガバイトになります。 ダウンロードは約時間以上、1gbit接続方法(並列からのダウンロードs3.amazonaws.com 復少なくとも半分の1ギガビチャンネル） -一部のファイルがダウンロードす。 ファイルサイズを確認し、疑わしいものを再ダウンロードします。 +ダウンロードすると、CSVファイルで約227GBの非圧縮データが生成されます。 ダウンロードには約1Gbitの接続時間がかかります（からの並列ダウンロードs3.amazonaws.com 1Gbitチャネルの少なくとも半分を回復する）。 +一部のファイルがダウンロードす。 チェックのファイルサイズを再ダウンロードのように見えた。 一部のファイルが含無効にされています。 次のように修正できます: @@ -28,11 +28,11 @@ mv data/yellow_tripdata_2010-02.csv_ data/yellow_tripdata_2010-02.csv mv data/yellow_tripdata_2010-03.csv_ data/yellow_tripdata_2010-03.csv ``` -その後、postgresqlでデータを前処理する必要があります。 これにより、ポリゴン内のポイントの選択が作成され（マップ上のポイントとニューヨーク市の自治区とのマッチング）、結合を使用してすべてのデータが これを行うには、postgisサポートでpostgresqlをインストールする必要があります。 +その後、PostgreSQLでデータを前処理する必要があります。 これにより、ポリゴン内のポイントの選択が作成され（マップ上のポイントとニューヨーク市の自治区を一致させるため）、結合を使用してすべてのデータを これを行うには、PostgisサポートでPostgreSQLをインストールする必要があります。 -実行中に注意してください `initialize_database.sh` すべてのテーブルが正しく作成されたことを手動で再確認します。 +の際はご注意ください走行 `initialize_database.sh` すべてのテーブルが正しく作成されたことを手動で再確認します。 -毎月のpostgresqlデータの処理には約20-30分かかり、合計で約48時間かかります。 +PostgreSQLで毎月のデータを処理するのに約20-30分、合計で約48時間かかります。 ダウンロードした行数は、次のように確認できます: @@ -45,11 +45,11 @@ $ time psql nyc-taxi-data -c "SELECT count(*) FROM trips;" real 7m9.164s ``` -（これは、一連のブログ記事でMark Litwintschikによって報告された11億行をわずかに超えています。) +（これは、一連のブログ記事でMark Litwintschikによって報告された11億行よりもわずかに多い。) PostgreSQLのデータは370GBのスペースを使用します。 -PostgreSQLからデータをエクスポート: +PostgreSQLからデータのエクスポート: ``` sql COPY @@ -121,10 +121,10 @@ COPY ) TO '/opt/milovidov/nyc-taxi-data/trips.tsv'; ``` -データスナップショットは、毎秒約50mbの速度で作成されます。 スナップショットの作成中に、postgresqlはディスクから約28mb/秒の速度で読み取ります。 -これには約5時間かかります。 結果のtsvファイルは590612904969バイトです。 +データスナップショットは、毎秒約50MBの速度で作成されます。 スナップショットの作成中、PostgreSQLは毎秒28MB程度の速度でディスクから読み取ります。 +これには約5時間かかります。 結果のTSVファイルは590612904969バイトです。 -ClickHouseでテンポラリテーブルを作成する: +ClickHouseで一時テーブルを作成する: ``` sql CREATE TABLE trips @@ -183,7 +183,7 @@ dropoff_puma Nullable(String) ) ENGINE = Log; ``` -これは、フィールドをより正確なデータ型に変換し、可能であればnullを排除するために必要です。 +フィールドをより正確なデータ型に変換し、可能であればNullを除去するために必要です。 ``` bash $ time clickhouse-client --query="INSERT INTO trips FORMAT TabSeparated" < trips.tsv @@ -191,15 +191,15 @@ $ time clickhouse-client --query="INSERT INTO trips FORMAT TabSeparated" < trips real 75m56.214s ``` -データは112-140mb/秒の速度で読み取られます。 +データは112-140Mb/秒の速度で読み取られます。 データの読み込みログタイプテーブルに一流した76分です。 -この表のデータは、142gbを使用します。 +この表のデータは142GBを使用します。 -（Postgresから直接データをインポートすることも可能です `COPY ... TO PROGRAM`.) +（Postgresから直接データをインポートすることもできます `COPY ... TO PROGRAM`.) Unfortunately, all the fields associated with the weather (precipitation…average\_wind\_speed) were filled with NULL. Because of this, we will remove them from the final data set. -まず、単一のサーバー上にテーブルを作成します。 その後、テーブルを配布します。 +まず、単一のサーバー上にテーブルを作成します。 後でテーブルを配布します。 サマリーテーブルの作成と設定: @@ -265,10 +265,10 @@ toUInt16(ifNull(dropoff_puma, '0')) AS dropoff_puma FROM trips ``` -これは3030秒で約428,000行/秒の速度で行われます。 -それをより速くロードするには、次のように表を作成します `Log` エンジンの代わりに `MergeTree`. この場合、ダウンロードは200秒より速く動作します。 +これには3030秒の速度で約428,000行/秒かかります。 +それをより速くロードするには、テーブルを作成します。 `Log` 代わりにエンジン `MergeTree`. この場合、ダウンロードは200秒よりも速く動作します。 -この表では、126gbのディスク領域を使用します。 +テーブルには126GBのディスク領域が使用されます。 ``` sql SELECT formatReadableSize(sum(bytes)) FROM system.parts WHERE table = 'trips_mergetree' AND active @@ -280,7 +280,7 @@ SELECT formatReadableSize(sum(bytes)) FROM system.parts WHERE table = 'trips_mer └────────────────────────────────┘ ``` -とりわけ、mergetreeでoptimizeクエリを実行することができます。 でも必要ないからさま、おめでとうございますどのくらい実装されているか。 +とりわけ、MERGETREEでOPTIMIZEクエリを実行できます。 でも必要ないからさま、おめでとうございますどのくらい実装されているか。 ## ダウンロード調の間仕切り {#download-of-prepared-partitions} @@ -295,7 +295,7 @@ $ clickhouse-client --query "select count(*) from datasets.trips_mergetree" !!! info "情報" 以下で説明するクエリを実行する場合は、完全なテーブル名を使用する必要があります, `datasets.trips_mergetree`. -## 単一サーバー上の結果 {#results-on-single-server} +## 単一サーバーでの結果 {#results-on-single-server} Q1: @@ -319,7 +319,7 @@ Q3: SELECT passenger_count, toYear(pickup_date) AS year, count(*) FROM trips_mergetree GROUP BY passenger_count, year ``` -2.104秒。 +2.104秒 Q4: @@ -332,11 +332,11 @@ ORDER BY year, count(*) DESC 3.593秒 -次のサーバーが使用されました: +以下のサーバーが使用されました: -二つのインテル(r)xeon(r)cpu e5-2650v2@2.60ghzの、16物理カーネル合計、128gib ram、8×6tb hd上のハードウェアraid-5 +二つのインテル(R)Xeon(R)CPU E5-2650v2@2.60GHzの,16物理カーネルの合計,128GiBのRAM,ハードウェアRAID上の8x6TBのHD-5 -実行時間は、三つの実行の最高です。 だから、クエリからデータを読み込むためのファイルシステムます。 データは読み出され、各実行で処理されます。 +実行時間は、三つの実行の中で最高です。 だから、クエリからデータを読み込むためのファイルシステムます。 データは各実行で読み出され、処理されます。 三つのサーバー上のテーブルの作成: @@ -346,7 +346,7 @@ ORDER BY year, count(*) DESC CREATE TABLE default.trips_mergetree_third ( trip_id UInt32, vendor_id Enum8('1' = 1, '2' = 2, 'CMT' = 3, 'VTS' = 4, 'DDS' = 5, 'B02512' = 10, 'B02598' = 11, 'B02617' = 12, 'B02682' = 13, 'B02764' = 14), pickup_date Date, pickup_datetime DateTime, dropoff_date Date, dropoff_datetime DateTime, store_and_fwd_flag UInt8, rate_code_id UInt8, pickup_longitude Float64, pickup_latitude Float64, dropoff_longitude Float64, dropoff_latitude Float64, passenger_count UInt8, trip_distance Float64, fare_amount Float32, extra Float32, mta_tax Float32, tip_amount Float32, tolls_amount Float32, ehail_fee Float32, improvement_surcharge Float32, total_amount Float32, payment_type_ Enum8('UNK' = 0, 'CSH' = 1, 'CRE' = 2, 'NOC' = 3, 'DIS' = 4), trip_type UInt8, pickup FixedString(25), dropoff FixedString(25), cab_type Enum8('yellow' = 1, 'green' = 2, 'uber' = 3), pickup_nyct2010_gid UInt8, pickup_ctlabel Float32, pickup_borocode UInt8, pickup_boroname Enum8('' = 0, 'Manhattan' = 1, 'Bronx' = 2, 'Brooklyn' = 3, 'Queens' = 4, 'Staten Island' = 5), pickup_ct2010 FixedString(6), pickup_boroct2010 FixedString(7), pickup_cdeligibil Enum8(' ' = 0, 'E' = 1, 'I' = 2), pickup_ntacode FixedString(4), pickup_ntaname Enum16('' = 0, 'Airport' = 1, 'Allerton-Pelham Gardens' = 2, 'Annadale-Huguenot-Prince\'s Bay-Eltingville' = 3, 'Arden Heights' = 4, 'Astoria' = 5, 'Auburndale' = 6, 'Baisley Park' = 7, 'Bath Beach' = 8, 'Battery Park City-Lower Manhattan' = 9, 'Bay Ridge' = 10, 'Bayside-Bayside Hills' = 11, 'Bedford' = 12, 'Bedford Park-Fordham North' = 13, 'Bellerose' = 14, 'Belmont' = 15, 'Bensonhurst East' = 16, 'Bensonhurst West' = 17, 'Borough Park' = 18, 'Breezy Point-Belle Harbor-Rockaway Park-Broad Channel' = 19, 'Briarwood-Jamaica Hills' = 20, 'Brighton Beach' = 21, 'Bronxdale' = 22, 'Brooklyn Heights-Cobble Hill' = 23, 'Brownsville' = 24, 'Bushwick North' = 25, 'Bushwick South' = 26, 'Cambria Heights' = 27, 'Canarsie' = 28, 'Carroll Gardens-Columbia Street-Red Hook' = 29, 'Central Harlem North-Polo Grounds' = 30, 'Central Harlem South' = 31, 'Charleston-Richmond Valley-Tottenville' = 32, 'Chinatown' = 33, 'Claremont-Bathgate' = 34, 'Clinton' = 35, 'Clinton Hill' = 36, 'Co-op City' = 37, 'College Point' = 38, 'Corona' = 39, 'Crotona Park East' = 40, 'Crown Heights North' = 41, 'Crown Heights South' = 42, 'Cypress Hills-City Line' = 43, 'DUMBO-Vinegar Hill-Downtown Brooklyn-Boerum Hill' = 44, 'Douglas Manor-Douglaston-Little Neck' = 45, 'Dyker Heights' = 46, 'East Concourse-Concourse Village' = 47, 'East Elmhurst' = 48, 'East Flatbush-Farragut' = 49, 'East Flushing' = 50, 'East Harlem North' = 51, 'East Harlem South' = 52, 'East New York' = 53, 'East New York (Pennsylvania Ave)' = 54, 'East Tremont' = 55, 'East Village' = 56, 'East Williamsburg' = 57, 'Eastchester-Edenwald-Baychester' = 58, 'Elmhurst' = 59, 'Elmhurst-Maspeth' = 60, 'Erasmus' = 61, 'Far Rockaway-Bayswater' = 62, 'Flatbush' = 63, 'Flatlands' = 64, 'Flushing' = 65, 'Fordham South' = 66, 'Forest Hills' = 67, 'Fort Greene' = 68, 'Fresh Meadows-Utopia' = 69, 'Ft. Totten-Bay Terrace-Clearview' = 70, 'Georgetown-Marine Park-Bergen Beach-Mill Basin' = 71, 'Glen Oaks-Floral Park-New Hyde Park' = 72, 'Glendale' = 73, 'Gramercy' = 74, 'Grasmere-Arrochar-Ft. Wadsworth' = 75, 'Gravesend' = 76, 'Great Kills' = 77, 'Greenpoint' = 78, 'Grymes Hill-Clifton-Fox Hills' = 79, 'Hamilton Heights' = 80, 'Hammels-Arverne-Edgemere' = 81, 'Highbridge' = 82, 'Hollis' = 83, 'Homecrest' = 84, 'Hudson Yards-Chelsea-Flatiron-Union Square' = 85, 'Hunters Point-Sunnyside-West Maspeth' = 86, 'Hunts Point' = 87, 'Jackson Heights' = 88, 'Jamaica' = 89, 'Jamaica Estates-Holliswood' = 90, 'Kensington-Ocean Parkway' = 91, 'Kew Gardens' = 92, 'Kew Gardens Hills' = 93, 'Kingsbridge Heights' = 94, 'Laurelton' = 95, 'Lenox Hill-Roosevelt Island' = 96, 'Lincoln Square' = 97, 'Lindenwood-Howard Beach' = 98, 'Longwood' = 99, 'Lower East Side' = 100, 'Madison' = 101, 'Manhattanville' = 102, 'Marble Hill-Inwood' = 103, 'Mariner\'s Harbor-Arlington-Port Ivory-Graniteville' = 104, 'Maspeth' = 105, 'Melrose South-Mott Haven North' = 106, 'Middle Village' = 107, 'Midtown-Midtown South' = 108, 'Midwood' = 109, 'Morningside Heights' = 110, 'Morrisania-Melrose' = 111, 'Mott Haven-Port Morris' = 112, 'Mount Hope' = 113, 'Murray Hill' = 114, 'Murray Hill-Kips Bay' = 115, 'New Brighton-Silver Lake' = 116, 'New Dorp-Midland Beach' = 117, 'New Springville-Bloomfield-Travis' = 118, 'North Corona' = 119, 'North Riverdale-Fieldston-Riverdale' = 120, 'North Side-South Side' = 121, 'Norwood' = 122, 'Oakland Gardens' = 123, 'Oakwood-Oakwood Beach' = 124, 'Ocean Hill' = 125, 'Ocean Parkway South' = 126, 'Old Astoria' = 127, 'Old Town-Dongan Hills-South Beach' = 128, 'Ozone Park' = 129, 'Park Slope-Gowanus' = 130, 'Parkchester' = 131, 'Pelham Bay-Country Club-City Island' = 132, 'Pelham Parkway' = 133, 'Pomonok-Flushing Heights-Hillcrest' = 134, 'Port Richmond' = 135, 'Prospect Heights' = 136, 'Prospect Lefferts Gardens-Wingate' = 137, 'Queens Village' = 138, 'Queensboro Hill' = 139, 'Queensbridge-Ravenswood-Long Island City' = 140, 'Rego Park' = 141, 'Richmond Hill' = 142, 'Ridgewood' = 143, 'Rikers Island' = 144, 'Rosedale' = 145, 'Rossville-Woodrow' = 146, 'Rugby-Remsen Village' = 147, 'Schuylerville-Throgs Neck-Edgewater Park' = 148, 'Seagate-Coney Island' = 149, 'Sheepshead Bay-Gerritsen Beach-Manhattan Beach' = 150, 'SoHo-TriBeCa-Civic Center-Little Italy' = 151, 'Soundview-Bruckner' = 152, 'Soundview-Castle Hill-Clason Point-Harding Park' = 153, 'South Jamaica' = 154, 'South Ozone Park' = 155, 'Springfield Gardens North' = 156, 'Springfield Gardens South-Brookville' = 157, 'Spuyten Duyvil-Kingsbridge' = 158, 'St. Albans' = 159, 'Stapleton-Rosebank' = 160, 'Starrett City' = 161, 'Steinway' = 162, 'Stuyvesant Heights' = 163, 'Stuyvesant Town-Cooper Village' = 164, 'Sunset Park East' = 165, 'Sunset Park West' = 166, 'Todt Hill-Emerson Hill-Heartland Village-Lighthouse Hill' = 167, 'Turtle Bay-East Midtown' = 168, 'University Heights-Morris Heights' = 169, 'Upper East Side-Carnegie Hill' = 170, 'Upper West Side' = 171, 'Van Cortlandt Village' = 172, 'Van Nest-Morris Park-Westchester Square' = 173, 'Washington Heights North' = 174, 'Washington Heights South' = 175, 'West Brighton' = 176, 'West Concourse' = 177, 'West Farms-Bronx River' = 178, 'West New Brighton-New Brighton-St. George' = 179, 'West Village' = 180, 'Westchester-Unionport' = 181, 'Westerleigh' = 182, 'Whitestone' = 183, 'Williamsbridge-Olinville' = 184, 'Williamsburg' = 185, 'Windsor Terrace' = 186, 'Woodhaven' = 187, 'Woodlawn-Wakefield' = 188, 'Woodside' = 189, 'Yorkville' = 190, 'park-cemetery-etc-Bronx' = 191, 'park-cemetery-etc-Brooklyn' = 192, 'park-cemetery-etc-Manhattan' = 193, 'park-cemetery-etc-Queens' = 194, 'park-cemetery-etc-Staten Island' = 195), pickup_puma UInt16, dropoff_nyct2010_gid UInt8, dropoff_ctlabel Float32, dropoff_borocode UInt8, dropoff_boroname Enum8('' = 0, 'Manhattan' = 1, 'Bronx' = 2, 'Brooklyn' = 3, 'Queens' = 4, 'Staten Island' = 5), dropoff_ct2010 FixedString(6), dropoff_boroct2010 FixedString(7), dropoff_cdeligibil Enum8(' ' = 0, 'E' = 1, 'I' = 2), dropoff_ntacode FixedString(4), dropoff_ntaname Enum16('' = 0, 'Airport' = 1, 'Allerton-Pelham Gardens' = 2, 'Annadale-Huguenot-Prince\'s Bay-Eltingville' = 3, 'Arden Heights' = 4, 'Astoria' = 5, 'Auburndale' = 6, 'Baisley Park' = 7, 'Bath Beach' = 8, 'Battery Park City-Lower Manhattan' = 9, 'Bay Ridge' = 10, 'Bayside-Bayside Hills' = 11, 'Bedford' = 12, 'Bedford Park-Fordham North' = 13, 'Bellerose' = 14, 'Belmont' = 15, 'Bensonhurst East' = 16, 'Bensonhurst West' = 17, 'Borough Park' = 18, 'Breezy Point-Belle Harbor-Rockaway Park-Broad Channel' = 19, 'Briarwood-Jamaica Hills' = 20, 'Brighton Beach' = 21, 'Bronxdale' = 22, 'Brooklyn Heights-Cobble Hill' = 23, 'Brownsville' = 24, 'Bushwick North' = 25, 'Bushwick South' = 26, 'Cambria Heights' = 27, 'Canarsie' = 28, 'Carroll Gardens-Columbia Street-Red Hook' = 29, 'Central Harlem North-Polo Grounds' = 30, 'Central Harlem South' = 31, 'Charleston-Richmond Valley-Tottenville' = 32, 'Chinatown' = 33, 'Claremont-Bathgate' = 34, 'Clinton' = 35, 'Clinton Hill' = 36, 'Co-op City' = 37, 'College Point' = 38, 'Corona' = 39, 'Crotona Park East' = 40, 'Crown Heights North' = 41, 'Crown Heights South' = 42, 'Cypress Hills-City Line' = 43, 'DUMBO-Vinegar Hill-Downtown Brooklyn-Boerum Hill' = 44, 'Douglas Manor-Douglaston-Little Neck' = 45, 'Dyker Heights' = 46, 'East Concourse-Concourse Village' = 47, 'East Elmhurst' = 48, 'East Flatbush-Farragut' = 49, 'East Flushing' = 50, 'East Harlem North' = 51, 'East Harlem South' = 52, 'East New York' = 53, 'East New York (Pennsylvania Ave)' = 54, 'East Tremont' = 55, 'East Village' = 56, 'East Williamsburg' = 57, 'Eastchester-Edenwald-Baychester' = 58, 'Elmhurst' = 59, 'Elmhurst-Maspeth' = 60, 'Erasmus' = 61, 'Far Rockaway-Bayswater' = 62, 'Flatbush' = 63, 'Flatlands' = 64, 'Flushing' = 65, 'Fordham South' = 66, 'Forest Hills' = 67, 'Fort Greene' = 68, 'Fresh Meadows-Utopia' = 69, 'Ft. Totten-Bay Terrace-Clearview' = 70, 'Georgetown-Marine Park-Bergen Beach-Mill Basin' = 71, 'Glen Oaks-Floral Park-New Hyde Park' = 72, 'Glendale' = 73, 'Gramercy' = 74, 'Grasmere-Arrochar-Ft. Wadsworth' = 75, 'Gravesend' = 76, 'Great Kills' = 77, 'Greenpoint' = 78, 'Grymes Hill-Clifton-Fox Hills' = 79, 'Hamilton Heights' = 80, 'Hammels-Arverne-Edgemere' = 81, 'Highbridge' = 82, 'Hollis' = 83, 'Homecrest' = 84, 'Hudson Yards-Chelsea-Flatiron-Union Square' = 85, 'Hunters Point-Sunnyside-West Maspeth' = 86, 'Hunts Point' = 87, 'Jackson Heights' = 88, 'Jamaica' = 89, 'Jamaica Estates-Holliswood' = 90, 'Kensington-Ocean Parkway' = 91, 'Kew Gardens' = 92, 'Kew Gardens Hills' = 93, 'Kingsbridge Heights' = 94, 'Laurelton' = 95, 'Lenox Hill-Roosevelt Island' = 96, 'Lincoln Square' = 97, 'Lindenwood-Howard Beach' = 98, 'Longwood' = 99, 'Lower East Side' = 100, 'Madison' = 101, 'Manhattanville' = 102, 'Marble Hill-Inwood' = 103, 'Mariner\'s Harbor-Arlington-Port Ivory-Graniteville' = 104, 'Maspeth' = 105, 'Melrose South-Mott Haven North' = 106, 'Middle Village' = 107, 'Midtown-Midtown South' = 108, 'Midwood' = 109, 'Morningside Heights' = 110, 'Morrisania-Melrose' = 111, 'Mott Haven-Port Morris' = 112, 'Mount Hope' = 113, 'Murray Hill' = 114, 'Murray Hill-Kips Bay' = 115, 'New Brighton-Silver Lake' = 116, 'New Dorp-Midland Beach' = 117, 'New Springville-Bloomfield-Travis' = 118, 'North Corona' = 119, 'North Riverdale-Fieldston-Riverdale' = 120, 'North Side-South Side' = 121, 'Norwood' = 122, 'Oakland Gardens' = 123, 'Oakwood-Oakwood Beach' = 124, 'Ocean Hill' = 125, 'Ocean Parkway South' = 126, 'Old Astoria' = 127, 'Old Town-Dongan Hills-South Beach' = 128, 'Ozone Park' = 129, 'Park Slope-Gowanus' = 130, 'Parkchester' = 131, 'Pelham Bay-Country Club-City Island' = 132, 'Pelham Parkway' = 133, 'Pomonok-Flushing Heights-Hillcrest' = 134, 'Port Richmond' = 135, 'Prospect Heights' = 136, 'Prospect Lefferts Gardens-Wingate' = 137, 'Queens Village' = 138, 'Queensboro Hill' = 139, 'Queensbridge-Ravenswood-Long Island City' = 140, 'Rego Park' = 141, 'Richmond Hill' = 142, 'Ridgewood' = 143, 'Rikers Island' = 144, 'Rosedale' = 145, 'Rossville-Woodrow' = 146, 'Rugby-Remsen Village' = 147, 'Schuylerville-Throgs Neck-Edgewater Park' = 148, 'Seagate-Coney Island' = 149, 'Sheepshead Bay-Gerritsen Beach-Manhattan Beach' = 150, 'SoHo-TriBeCa-Civic Center-Little Italy' = 151, 'Soundview-Bruckner' = 152, 'Soundview-Castle Hill-Clason Point-Harding Park' = 153, 'South Jamaica' = 154, 'South Ozone Park' = 155, 'Springfield Gardens North' = 156, 'Springfield Gardens South-Brookville' = 157, 'Spuyten Duyvil-Kingsbridge' = 158, 'St. Albans' = 159, 'Stapleton-Rosebank' = 160, 'Starrett City' = 161, 'Steinway' = 162, 'Stuyvesant Heights' = 163, 'Stuyvesant Town-Cooper Village' = 164, 'Sunset Park East' = 165, 'Sunset Park West' = 166, 'Todt Hill-Emerson Hill-Heartland Village-Lighthouse Hill' = 167, 'Turtle Bay-East Midtown' = 168, 'University Heights-Morris Heights' = 169, 'Upper East Side-Carnegie Hill' = 170, 'Upper West Side' = 171, 'Van Cortlandt Village' = 172, 'Van Nest-Morris Park-Westchester Square' = 173, 'Washington Heights North' = 174, 'Washington Heights South' = 175, 'West Brighton' = 176, 'West Concourse' = 177, 'West Farms-Bronx River' = 178, 'West New Brighton-New Brighton-St. George' = 179, 'West Village' = 180, 'Westchester-Unionport' = 181, 'Westerleigh' = 182, 'Whitestone' = 183, 'Williamsbridge-Olinville' = 184, 'Williamsburg' = 185, 'Windsor Terrace' = 186, 'Woodhaven' = 187, 'Woodlawn-Wakefield' = 188, 'Woodside' = 189, 'Yorkville' = 190, 'park-cemetery-etc-Bronx' = 191, 'park-cemetery-etc-Brooklyn' = 192, 'park-cemetery-etc-Manhattan' = 193, 'park-cemetery-etc-Queens' = 194, 'park-cemetery-etc-Staten Island' = 195), dropoff_puma UInt16) ENGINE = MergeTree(pickup_date, pickup_datetime, 8192) ``` -ソースサーバー上: +移行元サーバー上: ``` sql CREATE TABLE trips_mergetree_x3 AS trips_mergetree_third ENGINE = Distributed(perftest, default, trips_mergetree_third, rand()) @@ -360,7 +360,7 @@ INSERT INTO trips_mergetree_x3 SELECT * FROM trips_mergetree これには2454秒かかります。 -三つのサーバーで: +三つのサーバー上で: Q1:0.212秒。 Q2:0.438秒。 @@ -369,22 +369,22 @@ Q4:1.241秒。 詳細ここでは、以降のクエリはスケール直線. -我々はまた、140サーバーのクラスタからの結果を持っている: +また、140台のサーバーのクラスタからの結果も得られます: Q1:0.028秒。 Q2:0.043秒。 Q3:0.051秒。 Q4:0.072秒。 -この場合、クエリの処理時間を設定したすべてのネットワークの待ち時間をゼロにすることに -またクエリをクライアントに位置すyandexデータセンター、フィンランドスキーロシアで、約20msの待ち時間をゼロにすることに +この場合、クエリ処理時間は、とりわけネットワーク遅延によって決定されます。 +またクエリをクライアントに位置すYandexデータセンター、フィンランドスキーロシアで、約20msの待ち時間をゼロにすることに ## 概要 {#summary} -| サーバー | Q1 | Q2 | Q3 | Q4 | -|----------|-------|-------|-------|-------| -| 1 | 0.490 | 1.224 | 2.104 | 3.593 | -| 3 | 0.212 | 0.438 | 0.733 | 1.241 | -| 140 | 0.028 | 0.043 | 0.051 | 0.072 | +| サーバ | Q1 | Q2 | Q3 | Q4 | +|--------|-------|-------|-------|-------| +| 1 | 0.490 | 1.224 | 2.104 | 3.593 | +| 3 | 0.212 | 0.438 | 0.733 | 1.241 | +| 140 | 0.028 | 0.043 | 0.051 | 0.072 | [元の記事](https://clickhouse.tech/docs/en/getting_started/example_datasets/nyc_taxi/) diff --git a/docs/ja/getting-started/example-datasets/ontime.md b/docs/ja/getting-started/example-datasets/ontime.md index 0f7d18d9458..94207b80446 100644 --- a/docs/ja/getting-started/example-datasets/ontime.md +++ b/docs/ja/getting-started/example-datasets/ontime.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 15 toc_title: "\u30AA\u30F3\u30BF\u30A4\u30E0" --- @@ -14,7 +14,7 @@ toc_title: "\u30AA\u30F3\u30BF\u30A4\u30E0" ## 生データからインポート {#import-from-raw-data} -データ: +デー: ``` bash for s in `seq 1987 2018` @@ -180,7 +180,7 @@ FROM ); ``` -Q1. 2000年から2008年までの一日あたりの便数 +Q1. 2000年から2008年までの一日あたりのフライト数 ``` sql SELECT DayOfWeek, count(*) AS c @@ -190,7 +190,7 @@ GROUP BY DayOfWeek ORDER BY c DESC; ``` -Q2. 10分以上遅れたフライトの数は、2000年から2008年まで、曜日ごとにグループ化されています +Q2。 10分以上遅延したフライトの数は、2000年から2008年の曜日でグループ化されています ``` sql SELECT DayOfWeek, count(*) AS c @@ -200,7 +200,7 @@ GROUP BY DayOfWeek ORDER BY c DESC; ``` -Q3. 2000-2008のための空港による遅延数 +Q3. 2000年から2008年の空港による遅延の数 ``` sql SELECT Origin, count(*) AS c @@ -211,7 +211,7 @@ ORDER BY c DESC LIMIT 10; ``` -Q4 2007年のキャリアによる遅延の数 +Q4 2007年のキャリア別の遅延の数 ``` sql SELECT Carrier, count(*) @@ -221,7 +221,7 @@ GROUP BY Carrier ORDER BY count(*) DESC; ``` -Q5 2007年のキャリアによる遅延の割合 +Q5 2007年のキャリア別遅延の割合 ``` sql SELECT Carrier, c, c2, c*100/c2 as c3 @@ -257,7 +257,7 @@ GROUP BY Carrier ORDER BY c3 DESC ``` -Q6 年のより広い範囲のための以前の要求,2000-2008 +Q6 年のより広い範囲のための前の要求、2000-2008 ``` sql SELECT Carrier, c, c2, c*100/c2 as c3 @@ -293,7 +293,7 @@ GROUP BY Carrier ORDER BY c3 DESC; ``` -Q7 年間で10分以上遅れたフライトの割合 +Q7 年ごとに10分以上遅延したフライトの割合 ``` sql SELECT Year, c1/c2 @@ -326,7 +326,7 @@ GROUP BY Year ORDER BY Year; ``` -Q8 様々な年の範囲のための直接接続された都市の数によって最も人気のある目的地 +Q8 さまざまな年の範囲のための直接接続された都市の数によって最も人気のある目的地 ``` sql SELECT DestCityName, uniqExact(OriginCityName) AS u @@ -400,7 +400,7 @@ ORDER BY c DESC LIMIT 10; ``` -この性能試験はvadim tkachenkoによって作成されました。 見る: +この性能試験はVadim Tkachenkoによって作成されました。 見る: - https://www.percona.com/blog/2009/10/02/analyzing-air-traffic-performance-with-infobright-and-monetdb/ - https://www.percona.com/blog/2009/10/26/air-traffic-queries-in-luciddb/ diff --git a/docs/ja/getting-started/example-datasets/star-schema.md b/docs/ja/getting-started/example-datasets/star-schema.md index 0cf23d94871..1fee12fc0e3 100644 --- a/docs/ja/getting-started/example-datasets/star-schema.md +++ b/docs/ja/getting-started/example-datasets/star-schema.md @@ -1,12 +1,11 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 20 -toc_title: "\u30B9\u30BF\u30FC\u30B9\u30AD\u30FC\u30DE\u306E\u30D9\u30F3\u30C1\u30DE\ - \u30FC\u30AF" +toc_title: "Star Schema\u30D9\u30F3\u30C1\u30DE\u30FC\u30AF" --- -# スタースキーマのベンチマーク {#star-schema-benchmark} +# Star Schemaベンチマーク {#star-schema-benchmark} Dbgenのコンパイル: @@ -19,7 +18,7 @@ $ make データの生成: !!! warning "注意" - と `-s 100` dbgenは600万行（67GB）を生成しますが、 `-s 1000` それは6億行を生成します（これには多くの時間がかかります) + と `-s 100` dbgenは600万行（67GB）を生成します。 `-s 1000` それは6億行を生成します（これは多くの時間がかかります) ``` bash $ ./dbgen -s 1000 -T c @@ -103,7 +102,7 @@ $ clickhouse-client --query "INSERT INTO supplier FORMAT CSV" < supplier.tbl $ clickhouse-client --query "INSERT INTO lineorder FORMAT CSV" < lineorder.tbl ``` -変換 “star schema” 非正規化するには “flat schema”: +変換 “star schema” 非正規化に “flat schema”: ``` sql SET max_memory_usage = 20000000000; diff --git a/docs/ja/getting-started/example-datasets/wikistat.md b/docs/ja/getting-started/example-datasets/wikistat.md index 8b92c8cff1b..4c4e0e1b14a 100644 --- a/docs/ja/getting-started/example-datasets/wikistat.md +++ b/docs/ja/getting-started/example-datasets/wikistat.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 18 -toc_title: WikiStat +toc_title: "\u30A6\u30A3\u30AD\u30B9\u30BF\u30C3\u30C8" --- -# WikiStat {#wikistat} +# ウィキスタット {#wikistat} 参照:http://dumps.wikimedia.org/other/pagecounts-raw/ diff --git a/docs/ja/getting-started/index.md b/docs/ja/getting-started/index.md index cf1cd049f89..53b6347d7de 100644 --- a/docs/ja/getting-started/index.md +++ b/docs/ja/getting-started/index.md @@ -1,15 +1,15 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_folder_title: Getting Started +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "\u306F\u3058\u3081\u306B" toc_hidden: true toc_priority: 8 -toc_title: "\u96A0\u3055\u308C\u305F" +toc_title: "\u96A0\u3057" --- # はじめに {#getting-started} -あなたがclickhouseに慣れていないし、そのパフォーマンスのハンズオン感を取得したい場合は、まず、あなたが通過する必要があります [インストール処理](install.md). その後、あなたは: +あなたがClickHouseに新しく、そのパフォーマンスの実践的な感覚を取得したい場合は、まず第一に、あなたが通過する必要があります [インストール処理](install.md). その後できます: - [詳細なチュートリアルを通過](tutorial.md) - [データセット例の実験](example-datasets/ontime.md) diff --git a/docs/ja/getting-started/install.md b/docs/ja/getting-started/install.md index c9662254bf2..9710fdae6a7 100644 --- a/docs/ja/getting-started/install.md +++ b/docs/ja/getting-started/install.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 11 toc_title: "\u8A2D\u7F6E" --- @@ -9,42 +9,40 @@ toc_title: "\u8A2D\u7F6E" ## システム要件 {#system-requirements} -ClickHouseは、x86\_64、AArch64、またはPowerPC64LE CPUアーキテクチャを使用して、Linux、FreeBSD、またはMac OS X上で実行できます。 +ClickHouseは、x86\_64、AArch64、またはPowerPC64LE CPUアーキテクチャを持つLinux、FreeBSD、またはMac OS X上で実行できます。 -公式の事前構築されたバイナリは、通常、x86\_64用にコンパイルされ、sse4.2命令セットを利用するため、特に明記されていない限り、それをサポートす 現在のcpuがsse4.2をサポートしているかどう: +公式の事前ビルドされたバイナリは、通常、x86\_64用にコンパイルされ、SSE4.2命令セットを利用するため、特に明記されていない限り、それをサポートす このコマンドをチェックが現在のCPU支援のためのSSE4.2: ``` bash $ grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not supported" ``` -走clickhouseにプロセッサーをサポートしていないsse4.2てaarch64はpowerpc64le建築き [ソースからのClickHouseのビルド](#from-sources) 適切な構成の調節を使って。 +走ClickHouseにプロセッサーをサポートしていないSSE4.2てAArch64はPowerPC64LE建築き [ソースからClickHouseを構築](#from-sources) 適切な構成調節を使って。 -## 使用可能な設置方法 {#available-installation-options} +## 利用できる設置選択 {#available-installation-options} ### DEBパッケージから {#install-from-deb-packages} -公式の事前コンパイルを使用することをお勧めします `deb` DebianやUbuntuのためのパッケージ. - -そこにこれらのコマンド置パッケージ: +公式の事前コンパイルを使用することをお勧めします `deb` DebianまたはUbuntu用のパッケージ。 走りこれらのコマンド置パッケージ: ``` bash {% include 'install/deb.sh' %} ``` -でもダウンロードとインストールパッケージを手動で下からもアクセスできます。https://repo.clickhouse.tech/deb/stable/main/. +最新のバージョンを使用する場合は、置き換えます `stable` と `testing` (これはテスト環境に推奨されます)。 -最新のバージョンを使用する場合は、以下を置き換えます `stable` と `testing` (これはテスト環境に推奨されます)。 +でもダウンロードとインストールパッケージ動 [ここに](https://repo.clickhouse.tech/deb/stable/main/). #### パッケージ {#packages} - `clickhouse-common-static` — Installs ClickHouse compiled binary files. - `clickhouse-server` — Creates a symbolic link for `clickhouse-server` とをインストールしデフォルトのサーバーの設定をします。 -- `clickhouse-client` — Creates a symbolic link for `clickhouse-client` そして他の顧客関係した用具。 および設置お客様の設定ファイルです。 +- `clickhouse-client` — Creates a symbolic link for `clickhouse-client` そして他の顧客関連の用具。 および設置お客様の設定ファイルです。 - `clickhouse-common-static-dbg` — Installs ClickHouse compiled binary files with debug info. ### RPMパッケージから {#from-rpm-packages} -公式の事前コンパイルを使用することをお勧めします `rpm` パッケージCentOS,RedHat、その他のrpmベLinuxディストリビューション. +公式の事前コンパイルを使用することをお勧めします `rpm` CentOS、RedHat、その他すべてのrpmベースのLinuxディストリビューション用のパッケージ。 まず、公式リポジトリを追加する必要があります: @@ -54,7 +52,7 @@ sudo rpm --import https://repo.clickhouse.tech/CLICKHOUSE-KEY.GPG sudo yum-config-manager --add-repo https://repo.clickhouse.tech/rpm/stable/x86_64 ``` -最新のバージョンを使用する場合は、以下を置き換えます `stable` と `testing` (これはテスト環境に推奨されます)。 その `prestable` タグも時々利用可能です。 +最新のバージョンを使用する場合は、置き換えます `stable` と `testing` (これはテスト環境に推奨されます)。 その `prestable` タグは時々あまりにも利用可能です。 そこにこれらのコマンド置パッケージ: @@ -62,13 +60,13 @@ sudo yum-config-manager --add-repo https://repo.clickhouse.tech/rpm/stable/x86_6 sudo yum install clickhouse-server clickhouse-client ``` -でもダウンロードとインストールパッケージを手動で下からもアクセスできます。https://repo.クリックハウス。テック/rpm/安定した/x86\_64. +でもダウンロードとインストールパッケージ動 [ここに](https://repo.clickhouse.tech/rpm/stable/x86_64). ### Tgzアーカイブから {#from-tgz-archives} -公式の事前コンパイルを使用することをお勧めします `tgz` すべてのLinuxディストリビュ `deb` または `rpm` パッケージは不可能です。 +公式の事前コンパイルを使用することをお勧めします `tgz` のインストール `deb` または `rpm` パッケージはできません。 -必要なバージョンは `curl` または `wget` リポジトリからhttps://repo.yandex.ru/clickhouse/tgz/。 +必要なバージョンは次のとおりです `curl` または `wget` リポジトリからhttps://repo.yandex.ru/clickhouse/tgz/. その後、アーカイブをダウンロードは開梱と設置と設置のためのイントロダクションです。 最新バージョンの例: ``` bash @@ -92,27 +90,27 @@ tar -xzvf clickhouse-client-$LATEST_VERSION.tgz sudo clickhouse-client-$LATEST_VERSION/install/doinst.sh ``` -運用環境では、最新のものを使用することをお勧めします `stable`-バージョン。 その番号はGitHubページにありますhttps://github.com/ClickHouse/ClickHouse/tags 後置を使って `-stable`. +本番環境では、最新のものを使用することをお勧めします `stable`-バージョン。 き、その番号をGitHubのページhttps://github.com/ClickHouse/ClickHouse/tags とpostfix `-stable`. -### ドッカーの画像から {#from-docker-image} +### Dockerイメージから {#from-docker-image} -Dockerの中でClickHouseを実行するには、次のガイドに従います [Docker拠点](https://hub.docker.com/r/yandex/clickhouse-server/). これらの画像は、公式を使用 `deb` 中のパッケージ。 +Docker内でClickHouseを実行するには、次のガイドに従います [Dockerハブ](https://hub.docker.com/r/yandex/clickhouse-server/). このように映像公 `deb` 中のパッケージ。 ### ソースから {#from-sources} ClickHouseを手動でコンパイルするには、以下の手順に従います [Linux](../development/build.md) または [Mac OS X](../development/build-osx.md). -できるコンパイルパッケージはインストールしていたプログラムを使用もインストールせずにパッケージ。 またビルを手動で無数の組み合わせで自分だけのsse4.2に必要構築のためのaarch64定する必要はありません。 +できるコンパイルパッケージはインストールしていたプログラムを使用もインストールせずにパッケージ。 またビルを手動で無数の組み合わせで自分だけのSSE4.2に必要構築のためのAArch64定する必要はありません。 Client: programs/clickhouse-client Server: programs/clickhouse-server -データとメタデータフォルダを作成する必要があります `chown` 目的のユーザーのためのそれら。 それらのパスは、server config（src/programs/server/config）で変更できます。xml)、デフォルトでは次のとおりです: +データフォルダとメタデータフォルダを作成し、 `chown` 目的のユーザーのためのそれら。 それらのパスは、サーバー設定（src/programs/server/config）で変更することができます。xml)、デフォルトでは: /opt/clickhouse/data/default/ /opt/clickhouse/metadata/default/ -Gentooでは、次のものを使用できます `emerge clickhouse` ソースからClickHouseをインストールする。 +Gentooでは、以下を使用することができます `emerge clickhouse` ソースからClickHouseをインストールする。 ## 起動 {#launch} @@ -122,7 +120,7 @@ Gentooでは、次のものを使用できます `emerge clickhouse` ソース $ sudo service clickhouse-server start ``` -あなたが持っていない場合 `service` コマンド、実行 +あなたが持っていない場合 `service` コマンドとして実行 ``` bash $ sudo /etc/init.d/clickhouse-server start @@ -130,7 +128,7 @@ $ sudo /etc/init.d/clickhouse-server start のログを参照してください `/var/log/clickhouse-server/` ディレクトリ。 -サーバーが起動しない場合は、ファイル内の設定を確認してください `/etc/clickhouse-server/config.xml`. +サーバーが起動しない場合は、ファイル内の構成を確認してください `/etc/clickhouse-server/config.xml`. または手動で開始のサーバーからのコンソール: @@ -139,24 +137,24 @@ $ clickhouse-server --config-file=/etc/clickhouse-server/config.xml ``` この場合、ログはコンソールに印刷され、開発中に便利です。 -設定ファイルがカレントディレクトリにある場合は、以下を指定する必要はありません。 `--config-file` パラメータ。 デフォルトでは、 `./config.xml`. +設定ファイルがカレントディレクトリにある場合は、 `--config-file` パラメータ。 デフォルトでは、 `./config.xml`. -ClickHouse対応アクセス制限を設定します。 彼らはに位置しています `users.xml` ファイル(次へ `config.xml`). -デフォルトでは、アクセスはどこからでも可能です。 `default` ユーザー、パスワードなし。 見る `user/default/networks`. -詳細については、以下を参照してください [“Configuration Files”](../operations/configuration-files.md). +ClickHouse対応アクセス制限を設定します。 彼らはに位置しています `users.xml` ファイル(隣のファイル `config.xml`). +デフォルトでは、 `default` ユーザー、パスワードなし。 見る `user/default/networks`. +詳細については [“Configuration Files”](../operations/configuration-files.md). -サーバーの起動後、コマンドラインクライアントを使用してサーバーに接続できます: +Serverを起動した後、コマンドラインクライアントを使用してserverに接続できます: ``` bash $ clickhouse-client ``` -デフォルトでは、 `localhost:9000` ユーザーに代わって `default` パスワードなし。 また、以下を使用してリモートサーバーに接続することもできます `--host` 引数。 +デフォルトでは、 `localhost:9000` ユーザーに代わって `default` パスワードなし。 また、リモートサーバに接続するために使用することもできます `--host` 引数。 -端末はutf-8エンコードを使用する必要があります。 -詳細については、以下を参照してください [“Command-line client”](../interfaces/cli.md). +端末はUTF-8エンコードを使用する必要があります。 +詳細については [“Command-line client”](../interfaces/cli.md). -例えば: +例: ``` bash $ ./clickhouse-client @@ -177,8 +175,8 @@ SELECT 1 :) ``` -**おめでとう、システムの作品！** +**生、おめでとうございます、システムを作ります！！** -継続実験をダウンロードでき、試験データセットやじ [tutorial](https://clickhouse.tech/tutorial.html). +継続実験をダウンロードでき、試験データセットやじ [チュートリ](https://clickhouse.tech/tutorial.html). [元の記事](https://clickhouse.tech/docs/en/getting_started/install/) diff --git a/docs/ja/getting-started/playground.md b/docs/ja/getting-started/playground.md index d6fbc3e3b75..84469a2aa1d 100644 --- a/docs/ja/getting-started/playground.md +++ b/docs/ja/getting-started/playground.md @@ -1,16 +1,16 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 14 toc_title: "\u904A\u3073\u5834" --- -# ClickHouseの遊び場 {#clickhouse-playground} +# ClickHouseの運動場 {#clickhouse-playground} -[ClickHouseの遊び場](https://play.clickhouse.tech?file=welcome) サーバーやクラスターを設定せずに、クエリを即座に実行することで、ClickHouseを試すことができます。 -複数の例ではデータセットの遊び場などのサンプルのクエリを表すclickhouse特徴です。 +[ClickHouseの運動場](https://play.clickhouse.tech?file=welcome) "いちばん、人を考える実験ClickHouseによる走行クエリを瞬時にな設定をサーバまたはクラスター +複数の例ではデータセットの遊び場などのサンプルのクエリを表すClickHouse特徴です。 -クエリは読み取り専用ユーザーとして実行されます。 では一部制限: +クエリは読み取り専用ユーザーとして実行されます。 いくつかの制限を意味します: - DDLクエリは許可されません - 挿入クエリは許可されません @@ -22,26 +22,26 @@ toc_title: "\u904A\u3073\u5834" - [`max_execution_time=60000`](../operations/settings/query_complexity/#max-execution-time) ClickHouseの運動場はm2の経験を与える。小さい -[管理サービスclickhouse](https://cloud.yandex.com/services/managed-clickhouse) -インスタンス [Yandexの。クラウド](https://cloud.yandex.com/). -詳細については、 [クラウドプロバイダー](../commercial/cloud.md). +[ClickHouseの管理サービス](https://cloud.yandex.com/services/managed-clickhouse) +インスタンス [Yandex.クラウド](https://cloud.yandex.com/). +詳細について [クラウドプロバイダー](../commercial/cloud.md). -ﾂつｨﾂ姪“ﾂつ”ﾂ債ﾂつｹ [HTTP API](../interfaces/http.md). -コミュニケーションが円滑にバックエンドがありclickhouseクラスターになサーバーサイド願います。 -ClickHouse HTTPS評価項目としても利用可能ですの一部が遊べない日々が続いていました。 +ClickHouse遊びwebインタフェースによって、要求によClickHouse [HTTP API](../interfaces/http.md). +コミュニケーションが円滑にバックエンドがありClickHouseクラスターになサーバーサイド願います。 +ClickHouse HTTPSエンドポイントは、遊び場の一部としても利用できます。 -できるクエリーの遊び場をhttpお客様は、例えば [カール](https://curl.haxx.se) または [wget](https://www.gnu.org/software/wget/)、または以下を使用して接続を設定する [JDBC](../interfaces/jdbc.md) または [ODBC](../interfaces/odbc.md) ドライバー -情報ソフトウェア製品を支えるclickhouse可能 [ここに](../interfaces/index.md). +任意のHTTPクライアントを使用してplaygroundにクエリを実行できます。 [カール](https://curl.haxx.se) または [wget](https://www.gnu.org/software/wget/) または、次を使用して接続を設定します [JDBC](../interfaces/jdbc.md) または [ODBC](../interfaces/odbc.md) ドライバー +ClickHouseをサポートするソフトウェア製品の詳細については、 [ここに](../interfaces/index.md). -| パラメータ | 値 | -|:-----------|:----------------------------------------------| -| エンドポイ | https://play-api。クリックハウス。テック:8443 | -| ユーザ | `playground` | -| パスワード | `clickhouse` | +| パラメータ | 値 | +|:-----------|:-----------------------------------------| +| 端点 | https://play-api.クリックハウス技術:8443 | +| ユーザ | `playground` | +| パスワード | `clickhouse` | このエンドポイントには安全な接続が必要です。 -例えば: +例: ``` bash curl "https://play-api.clickhouse.tech:8443/?query=SELECT+'Play+ClickHouse!';&user=playground&password=clickhouse&database=datasets" diff --git a/docs/ja/getting-started/tutorial.md b/docs/ja/getting-started/tutorial.md index 5e5e0e4e4ba..e4219bdb674 100644 --- a/docs/ja/getting-started/tutorial.md +++ b/docs/ja/getting-started/tutorial.md @@ -1,21 +1,21 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 12 -toc_title: Tutorial +toc_title: "\u30C1\u30E5\u30FC\u30C8\u30EA" --- # ClickHouseチュートリアル {#clickhouse-tutorial} ## このチュートリアルから何を期待する？ {#what-to-expect-from-this-tutorial} -このチュートリアルでは、クリックハウスクラスタを設定する方法について説明します。 それは小さい、しかし耐障害性および拡張可能である。 次に、例のデータセットのいずれかを使用してデータを入力し、いくつかのデモクエリを実行します。 +このチュートリアルでは、単純なClickHouseクラスターを設定する方法について説明します。 それは小さいが、耐障害性とスケーラブルになります。 次に、例のデータセットのいずれかを使用してデータを入力し、いくつかのデモクエリを実行します。 -## 単一ノード設定 {#single-node-setup} +## 単一ノードの設定 {#single-node-setup} -分散環境の複雑さを延期するには、まず、単一のサーバーまたは仮想マシンにclickhouseを展開します。 clickhouseは通常からインストール [deb](install.md#install-from-deb-packages) または [rpm](install.md#from-rpm-packages) パッケージがあります [代替案](install.md#from-docker-image) の営業システムな支援します。 +分散環境の複雑さを先送りするために、単一のサーバーまたは仮想マシンにClickHouseを展開することから始めます。 ClickHouseは通常、 [deb](install.md#install-from-deb-packages) または [rpm](install.md#from-rpm-packages) パッケージが、あります [代替案](install.md#from-docker-image) の営業システムな支援します。 -たとえば、次の項目を選択しました `deb` パッケージと実行: +たとえば、次のように選択します `deb` パッケージと実行: ``` bash {% include 'install/deb.sh' %} @@ -23,13 +23,13 @@ toc_title: Tutorial インストールされたパッケージには何がありますか: -- `clickhouse-client` パッケージ内容: [クリックハウス-顧客](../interfaces/cli.md) ケイClickHouseコンソールです。 +- `clickhouse-client` パッケージ [clickhouse-クライアント](../interfaces/cli.md) 適用、相互ClickHouseコンソール顧客。 - `clickhouse-common` パッケージが含まれてClickHouse実行可能ファイルです。 - `clickhouse-server` パッケージを含む設定ファイルを実行ClickHouseしています。 -サーバ設定ファイルを置 `/etc/clickhouse-server/`. さらに進む前に、 `` 要素の `config.xml`. パスはデータストレージの場所を決定するので、ディスク容量の大きいボリュームに配置する必要があります。 `/var/lib/clickhouse/`. 設定を調整したい場合は、直接編集するのは便利ではありません `config.xml` ファイルは、将来のパッケージ更新で書き直される可能性があります。 のオーバーライドは、config要素の作成 [config内のファイル。dディレクトリ](../operations/configuration-files.md) として役立つ “patches” 設定する。xmlだ +サーバー設定ファイルは `/etc/clickhouse-server/`. 更に行く前に、気づいて下さい `` の要素 `config.xml`. Pathはデータストレージの場所を決定するため、ディスク容量の大きいボリューム上に配置する必要があります。 `/var/lib/clickhouse/`. 設定を調整したい場合は、直接編集するのは便利ではありません `config.xml` ファイルで、このように書き換え、将来のパッケージです。 のオーバーライドは、config要素の作成 [config内のファイル。dディレクトリ](../operations/configuration-files.md) として役立つ “patches” 設定する。xml。 -あなたが気づいたように, `clickhouse-server` パッケージのイ 更新後も自動的に再起動されることはありません。 サーバーを起動する方法は、initシステムによって異なります。: +お気づきの通り, `clickhouse-server` が開始後、自動的にパッケージ設置できます。 更新後も自動的に再起動されることはありません。 サーバーの起動方法はinitシステムによって異なります。: ``` bash sudo service clickhouse-server start @@ -41,13 +41,14 @@ sudo service clickhouse-server start sudo /etc/init.d/clickhouse-server start ``` -サーバーログのデフォルトの場所は、 `/var/log/clickhouse-server/`. サーバーはクライアント接続を処理する準備ができています。 `Ready for connections` メッセージ +サーバーログの既定の場所 `/var/log/clickhouse-server/`. サーバーは、クライアント接続を処理する準備ができています。 `Ready for connections` メッセージ -一度 `clickhouse-server` 稼働している、我々は使用することができます `clickhouse-client` サーバーに接続し、次のようなテストクエリを実行するには `SELECT "Hello, world!";`. +一度 `clickhouse-server` 稼働中であれば `clickhouse-client` サーバーに接続し、次のようなテストクエリを実行するには `SELECT "Hello, world!";`.
-索のヒントclickhouse-クライアント +Clickhouse-クライアントのクイックヒント + 対話モード: ``` bash @@ -55,7 +56,7 @@ clickhouse-client clickhouse-client --host=... --port=... --user=... --password=... ``` -複数行のクエリを有効にする: +複数行クエリを有効にする: ``` bash clickhouse-client -m @@ -70,7 +71,7 @@ echo 'SELECT 1' | clickhouse-client clickhouse-client <<< 'SELECT 1' ``` -指定した形式のファイルからデータを挿入する: +指定した形式でファイルからデータを挿入する: ``` bash clickhouse-client --query='INSERT INTO table VALUES' < data.txt @@ -79,9 +80,9 @@ clickhouse-client --query='INSERT INTO table FORMAT TabSeparated' < data.tsv
-## インポートサンプル {#import-sample-dataset} +## インポート {#import-sample-dataset} -今回は入社clickhouseサーバーサンプルデータです。 このチュートリアルでは、yandexの匿名化されたデータを使用します。metricaは、オープンソースになる前にclickhouseを運用方法で実行する最初のサービスです（詳細は [履歴セクション](../introduction/history.md)). あります [Yandexをインポートする複数の方法。Metricaデータセット](example-datasets/metrica.md)、そしてチュートリアルのために、我々は最も現実的なもので行くよ。 +今度は、ClickHouseサーバーにサンプルデータを入力します。 このチュートリアルでは、Yandexの匿名化されたデータを使用します。Metrica、それがオープンソースになる前に、本番の方法でClickHouseを実行する最初のサービス（その詳細については [歴史セクション](../introduction/history.md)). そこには [Yandexのをインポートする複数の方法。メトリカデータセット](example-datasets/metrica.md) そして、チュートリアルのために、私たちは最も現実的なものに行きます。 ### 表データのダウンロードと抽出 {#download-and-extract-table-data} @@ -90,28 +91,28 @@ curl https://clickhouse-datasets.s3.yandex.net/hits/tsv/hits_v1.tsv.xz | unxz -- curl https://clickhouse-datasets.s3.yandex.net/visits/tsv/visits_v1.tsv.xz | unxz --threads=`nproc` > visits_v1.tsv ``` -抽出されたファイルのサイズは約10gbです。 +抽出されたファイルのサイズは約10GBです。 ### テーブルの作成 {#create-tables} -ほとんどのデータベース管理システムclickhouse論理的にグテーブル “databases”. そこには `default` データベースが、我々は名前の新しいものを作成します `tutorial`: +ほとんどのデータベース管理システムClickHouse論理的にグテーブル “databases”. そこには `default` データベースが、我々は名前の新しいものを作成します `tutorial`: ``` bash clickhouse-client --query "CREATE DATABASE IF NOT EXISTS tutorial" ``` -テーブルを作成するための構文は、データベースに比べて複雑です [参照](../sql-reference/statements/create.md). 一般的に `CREATE TABLE` 声明を設定するつもの: +テーブルを作成するための構文は、データベースに比べて複雑です(参照 [参照](../sql-reference/statements/create.md). 一般に `CREATE TABLE` 声明を設定するつもの: 1. 作成するテーブルの名前。 2. Table schema, i.e. list of columns and their [データ型](../sql-reference/data-types/index.md). -3. [表エンジン](../engines/table-engines/index.md) これは、このテーブルへのクエリが物理的にどのように実行されるかに関するすべての詳細を決定します。 +3. [表エンジン](../engines/table-engines/index.md) このテーブルへのクエリが物理的に実行される方法に関するすべての詳細を決定します。 -Yandexの。Metricaはweb分析サービスであり、サンプルデータセットはその完全な機能をカバーしていません。: +Yandex.Metricaはweb分析サービスであり、サンプルデータセットは完全な機能をカバーしていないため、作成するテーブルは二つしかありません: - `hits` とができるテーブルの各行動によるすべてのユーザーはすべてのwebサイトのサービスです。 - `visits` はテーブルを含む組み立て済みセッションの代わりに個別に行動します。 -これらのテーブルの実際のcreate tableクエリを見て、実行しましょう: +これらのテーブルの実際のcreate tableクエリを見て実行しましょう: ``` sql CREATE TABLE tutorial.hits_v1 @@ -454,22 +455,22 @@ SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192 ``` -ドできるようになりました。方のクエリのインタラクティブモードの `clickhouse-client` （事前にクエリを指定せずに端末で起動するだけです）またはいくつか試してみてください [代わりとなるインターフェース](../interfaces/index.md) あなたが望むなら。 +ドできるようになりました。方のクエリのインタラクティブモードの `clickhouse-client` （事前にクエリを指定せずに端末で起動するだけです）またはいくつか試してみてください [代替インターフェ](../interfaces/index.md) 望むなら -ご覧の通り, `hits_v1` 使用します [基本的なMergeTreeエンジン](../engines/table-engines/mergetree-family/mergetree.md)、ながら `visits_v1` 使用します [折りたたみ](../engines/table-engines/mergetree-family/collapsingmergetree.md) バリアント。 +ご覧のとおり, `hits_v1` を使用して [基本的なMergeTreeエンジン](../engines/table-engines/mergetree-family/mergetree.md) は、 `visits_v1` を使用して [崩壊](../engines/table-engines/mergetree-family/collapsingmergetree.md) バリアント ### デー {#import-data} -ClickHouseへのデータのインポートは、 [INSERT INTO](../sql-reference/statements/insert-into.md) 他の多くのSQLデータベースのような照会。 ただし、データは通常、次のいずれかで提供されます。 [対応するシリアル化形式](../interfaces/formats.md) 代わりに `VALUES` 句(これもサポートされています)。 +ClickHouseへのデータのインポートは [INSERT INTO](../sql-reference/statements/insert-into.md) 他の多くのSQLデータベースと同様のクエリ。 ただし、データは通常、次のいずれかで提供されます [対応するシリアル化形式](../interfaces/formats.md) 代わりに `VALUES` 句(これもサポートされています)。 -以前にダウンロードしたファ: +ファイルをダウンロード前にタブ区切り形式では下記のように輸入したいお客様のコンソール: ``` bash clickhouse-client --query "INSERT INTO tutorial.hits_v1 FORMAT TSV" --max_insert_block_size=100000 < hits_v1.tsv clickhouse-client --query "INSERT INTO tutorial.visits_v1 FORMAT TSV" --max_insert_block_size=100000 < visits_v1.tsv ``` -ClickHouseには多くのものがあります [調整する設定](../operations/settings/index.md) そして、コンソールクライアントでそれらを指定する一つの方法は、引数を使用することです `--max_insert_block_size`. どのような設定が利用可能であるか、それらが何を意味するのか、そしてデフォルトが何であるかを理解する最も簡単な方法は、 `system.settings` テーブル: +ClickHouseは多くの [調整する設定](../operations/settings/index.md) そして、コンソールクライアントでそれらを指定する一つの方法は、引数を介してです。 `--max_insert_block_size`. どのような設定が利用可能か、それらが何を意味し、デフォルトが何であるかを把握する最も簡単な方法は、 `system.settings` テーブル: ``` sql SELECT name, value, changed, description @@ -480,23 +481,23 @@ FORMAT TSV max_insert_block_size 1048576 0 "The maximum block size for insertion, if we control the creation of blocks for insertion." ``` -必要に応じ [OPTIMIZE](../query_language/misc/#misc_operations-optimize) インポート後のテーブル。 MergeTree-familyのエンジンで構成されたテーブルは、データストレージを最適化するために、常にバックグラウンドでデータ部分のマージを行います（または少なくとも これらのクエリのテーブルエンジンな保管の最適化現在の代わりについては後日、: +必要に応じ [OPTIMIZE](../sql-reference/statements/misc.md#misc_operations-optimize) インポート後のテーブル。 MergeTree-familyのエンジンで構成されているテーブルは、常にバックグラウンドでデータ部分のマージを行い、データストレージを最適化します（または少なくとも理にか これらのクエリのテーブルエンジンな保管の最適化現在の代わりについては後日、: ``` bash clickhouse-client --query "OPTIMIZE TABLE tutorial.hits_v1 FINAL" clickhouse-client --query "OPTIMIZE TABLE tutorial.visits_v1 FINAL" ``` -したがって、テーブルが一貫して新しいデータを受け取る場合は、そのままにして、マージをバックグラウンドで実行する方がよいでしょう。 +これらのクエリはI/OとCPUを大量に消費する操作を開始するため、テーブルが一貫して新しいデータを受信する場合は、そのままにして、マージをバックグ -テーブルインポートが成功したかどうかを確認できます: +今までチェックできる場合、テーブルの輸入に成功した: ``` bash clickhouse-client --query "SELECT COUNT(*) FROM tutorial.hits_v1" clickhouse-client --query "SELECT COUNT(*) FROM tutorial.visits_v1" ``` -## クエリ例 {#example-queries} +## クエリの例 {#example-queries} ``` sql SELECT @@ -518,18 +519,18 @@ FROM tutorial.visits_v1 WHERE (CounterID = 912887) AND (toYYYYMM(StartDate) = 201403) AND (domain(StartURL) = 'yandex.ru') ``` -## クラスターの展開 {#cluster-deployment} +## クラスター展開 {#cluster-deployment} -ClickHouseの集りは同種の集りである。 セットアップの手順: +ClickHouseクラスターは均質なクラスターです。 設定手順: -1. イclickhouseサーバーのすべての機械のクラスター -2. 構成ファイルでのクラスタ構成のセットアップ +1. イClickHouseサーバーのすべての機械のクラスター +2. 構成ファイルでのクラスター設定の設定 3. 各インスタ -4. 作成する [分散テーブル](../engines/table-engines/special/distributed.md) +4. 作成 [分散テーブル](../engines/table-engines/special/distributed.md) -[分散テーブル](../engines/table-engines/special/distributed.md) 実際には “view” 地元のテーブルのClickHouse。 SELECTクエリから分散型のテーブル実行が持つリソースを活用したすべてのクラスターの破片. を指定しますconfigs複数のクラスターを作成した複数のテーブルのビューを提供する別のクラスター +[分散テーブル](../engines/table-engines/special/distributed.md) 実際には “view” ClickHouseクラスタのローカルテーブルに。 SELECTクエリから分散型のテーブル実行が持つリソースを活用したすべてのクラスターの破片. を指定しますconfigs複数のクラスターを作成した複数のテーブルのビューを提供する別のクラスター -ﾂつｨﾂ姪“ﾂつ”ﾂ債ﾂづｭﾂつｹﾂづ債つｱﾂつｿﾂづｧ: +クラスターの設定例: ``` xml @@ -556,7 +557,7 @@ ClickHouseの集りは同種の集りである。 セットアップの手順: ``` -さらなる実証しましょう新しい地域のテーブルと同じ `CREATE TABLE` 私たちが使用したクエリ `hits_v1`、しかし、異なるテーブル名: +さらなる実証しましょう新しい地域のテーブルと同じ `CREATE TABLE` 私たちが使用したクエリ `hits_v1` しかし、異なるテーブル名: ``` sql CREATE TABLE tutorial.hits_local (...) ENGINE = MergeTree() ... @@ -569,24 +570,24 @@ CREATE TABLE tutorial.hits_all AS tutorial.hits_local ENGINE = Distributed(perftest_3shards_1replicas, tutorial, hits_local, rand()); ``` -一般的な方法は、クラスターのすべてのマシンで同様の分散テーブルを作成することです。 クラスターの任意のマシンで分散クエリを実行できます。 また、特定のselectクエリを使用して一時分散テーブルを作成する代替オプションもあります [リモート](../sql-reference/table-functions/remote.md) テーブル機能。 +一般的な方法は、クラスターのすべてのマシンで同様の分散テーブルを作成することです。 これは、クラスタの任意のマシン上で分散クエリを実行できます。 また、特定のSELECTクエリに対して一時的な分散テーブルを作成する別のオプションもあります [リモート](../sql-reference/table-functions/remote.md) テーブル関数。 -逃げよう [INSERT SELECT](../sql-reference/statements/insert-into.md) 分散テーブルに分散テーブルを複数のサーバーに分散させます。 +走ろう [INSERT SELECT](../sql-reference/statements/insert-into.md) 分散テーブルにテーブルを複数のサーバーに分散させます。 ``` sql INSERT INTO tutorial.hits_all SELECT * FROM tutorial.hits_v1; ``` -!!! warning "気づく" - このアプローチは適しませんのshardingの大きます。 別のツールがあります [クリックハウスコピー機](../operations/utilities/clickhouse-copier.md) できるre-ザ-シャーを任意の大きます。 +!!! warning "通知" + この方法は、大規模なテーブルのシャーディングには適していません。 別のツールがあります [クリックハウス-複写機](../operations/utilities/clickhouse-copier.md) 任意の大きなテーブルを再シャードできます。 -予想されるように、計算量の多いクエリは、3つのサーバーを使用する場合にn倍高速に実行されます。 +予想されるように、計算上重いクエリは、3つのサーバーの代わりに使用するとn倍高速に実行されます。 この場合、3つのシャードを持つクラスターを使用し、それぞれに単一のレプリカが含まれています。 -運用環境で復元性を提供するには、各シャードに、複数のアベイラビリティーゾーンまたはデータセンター(または少なくともラック)の間に2～3個のレプリカ clickhouseでは、レプリカの数に制限はありません。 +運用環境でレジリエンスを提供するには、各シャードに複数のアベイラビリティーゾーンまたはデータセンター(または少なくともラック)間に2-3個のレプリ ClickHouseでは、レプリカの数に制限はありません。 -レプリカを含むシャードのクラスタの設定例: +レプリカを含むシャードのクラスターの設定例: ``` xml @@ -610,12 +611,12 @@ INSERT INTO tutorial.hits_all SELECT * FROM tutorial.hits_v1; ``` -ネイティブ複製を有効にする [ZooKeeper](http://zookeeper.apache.org/) は必須です。 ClickHouseは、すべてのレプリカでデータの整合性を管理し、障害後に自動的に復元手順を実行します。 ZooKeeperクラスターを別々のサーバーに展開することをお勧めします（ClickHouseを含む他のプロセスは実行されていません）。 +ネイティブ複製を有効にする [飼育係](http://zookeeper.apache.org/) 必須です。 ClickHouseのデータの整合性はすべてのレプリカと回復手続き後の不動します。 別のサーバー(ClickHouseを含む他のプロセスが実行されていない場所)にZooKeeperクラスターを展開することをお勧めします。 -!!! note "メモ" - いくつかの簡単なケースでは、アプリケーションコードからすべてのレプリカにデータを書き込むことでデータを複製できます。 このアプローチは **ない** 推奨、この場合、ClickHouseはすべてのレプリカでデータの整合性を保証することはできません。 従ってそれはあなたの適用の責任になります。 +!!! note "注" + 単純なケースでは、アプリケーションコードからすべてのレプリカにデータを書き込むことでデータを複製できます。 このアプローチは **ない** この場合、ClickHouseはすべてのレプリカでデータの一貫性を保証できません。 従ってそれはあなたの適用の責任になる。 -ZooKeeperの場所は設定ファイルで指定します: +ZooKeeperの場所は設定ファイルで指定されます: ``` xml @@ -634,7 +635,7 @@ ZooKeeperの場所は設定ファイルで指定します: ``` -また、テーブル作成時に使用される各シャードとレプリカを識別するマクロを設定する必要があります: +また、テーブル作成時に使用する各シャードとレプリカを識別するためのマクロを設定する必要があります: ``` xml @@ -643,7 +644,7 @@ ZooKeeperの場所は設定ファイルで指定します: ``` -がない場合にレプリカの瞬間に複製表を作成し、新しい最初のレプリカスのインスタンスが作成. がある場合でライブレプリカを新たなレプリカのクローンからデータを設定しています。 最初にすべての複製テーブルを作成し、それにデータを挿入するオプションがあります。 別のオプションを作れるレプリカを追加しその他の長期データを挿入出来ます。 +がない場合にレプリカの瞬間に複製表を作成し、新しい最初のレプリカスのインスタンスが作成. がある場合でライブレプリカを新たなレプリカのクローンからデータを設定しています。 するオプションを複製のテーブル、データを挿入します。 別のオプションを作れるレプリカを追加しその他の長期データを挿入出来ます。 ``` sql CREATE TABLE tutorial.hits_replica (...) @@ -654,12 +655,12 @@ ENGINE = ReplcatedMergeTree( ... ``` -ここでは、 [レプリケートされたmergetree](../engines/table-engines/mergetree-family/replication.md) テーブルエンジン。 パラメータを指定飼育係のパスを含むザ-シャープ識別子のことです。 +ここでは、 [複製マージツリー](../engines/table-engines/mergetree-family/replication.md) テーブルエンジン。 パラメータでは、シャードとレプリカ識別子を含むZooKeeper pathを指定します。 ``` sql INSERT INTO tutorial.hits_replica SELECT * FROM tutorial.hits_local; ``` -複製はマルチマスターモードで動作します。 データは任意のレプリカにロードすることができ、システムは他のインスタンスと自動的に同期します。 複製は非同期で一定の瞬間にも、すべてのレプリカを含む場合があり、最近に挿入されます。 少なくとも一つのレプリカのようにするためのデータで測定す その同期データの修理整合性が活躍できます。 このアプローチでは、最近挿入されたデータの損失の可能性が低いことができます。 +複製はマルチマスターモードで動作します。 データは任意のレプリカにロードでき、システムはそれを他のインスタンスと自動的に同期します。 複製は非同期で一定の瞬間にも、すべてのレプリカを含む場合があり、最近に挿入されます。 少なくとも一つのレプリカのようにするためのデータで測定す 他の人は、データを同期し、再びアクティブになると一貫性を修復します。 この方法では、最近挿入されたデータが失われる可能性が低いことに注意してください。 [元の記事](https://clickhouse.tech/docs/en/getting_started/tutorial/) diff --git a/docs/ja/guides/apply-catboost-model.md b/docs/ja/guides/apply-catboost-model.md index 584feb390c7..e3adb19c6c7 100644 --- a/docs/ja/guides/apply-catboost-model.md +++ b/docs/ja/guides/apply-catboost-model.md @@ -1,41 +1,41 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 41 toc_title: "CatBoost\u30E2\u30C7\u30EB\u306E\u9069\u7528" --- # ClickHouseでのCatboostモデルの適用 {#applying-catboost-model-in-clickhouse} -[CatBoost](https://catboost.ai) では、このオープンソースの勾配向上の図書館が開発した [Yandex](https://yandex.com/company/) 機械学習のために。 +[CatBoost](https://catboost.ai) で開発された無料でオープンソースの勾配昇圧ライブラリです [Yandex](https://yandex.com/company/) 機械学習のために。 -この命令では、sqlからモデル推論を実行して、事前に学習したモデルをclickhouseに適用する方法を学習します。 +この手順では、SQLからモデル推論を実行して、ClickHouseで事前に訓練されたモデルを適用する方法を学習します。 ClickHouseでCatBoostモデルを適用するには: 1. [テーブルの作成](#create-table). -2. [データをテーブルに挿入する](#insert-data-to-table). -3. [ClickhouseにCatBoostを統合](#integrate-catboost-into-clickhouse) （任意ステップ）。 +2. [テーブルにデータを挿入します](#insert-data-to-table). +3. [ClickHouseにCatBoostを統合する](#integrate-catboost-into-clickhouse) （任意ステップ）。 4. [SQLからモデル推論を実行する](#run-model-inference). -CatBoostモデルのトレーニングの詳細については、 [訓練用モデル](https://catboost.ai/docs/features/training.html#training). +CatBoostモデルのトレーニングの詳細については、 [モデルの学習と適用](https://catboost.ai/docs/features/training.html#training). ## 前提条件 {#prerequisites} -あなたが持っていない場合 [Docker](https://docs.docker.com/install/) しかし、インストールしてください。 +あなたが持っていない場合 [ドッカー](https://docs.docker.com/install/) まだ、それを取付けなさい。 -!!! note "メモ" - [Docker](https://www.docker.com) であるソフトウェアプラットフォームを作成することができる容器を隔離するCatBoostとClickHouse設置からシステム。 +!!! note "注" + [ドッカー](https://www.docker.com) CatBoostとClickHouseのインストールをシステムの残りの部分から分離するコンテナを作成できるソフトウェアプラットフォームです。 CatBoostモデルを適用する前に: -**1.** を引く [Dockerイメージ](https://hub.docker.com/r/yandex/tutorial-catboost-clickhouse) レジストリから: +**1.** プル [ドッカー画像](https://hub.docker.com/r/yandex/tutorial-catboost-clickhouse) レジストリから: ``` bash $ docker pull yandex/tutorial-catboost-clickhouse ``` -このdocker画像を含むものを実行する必要がありますcatboostとclickhouse:コードでは、ランタイム時において、図書館、環境変数の設定ファイルです。 +このDockerイメージには、CatBoostとClickHouseを実行するために必要なコード、ランタイム、ライブラリ、環境変数、設定ファイルがすべて含まれています。 **2.** Dockerイメージが正常にプルされたことを確認します: @@ -45,7 +45,7 @@ REPOSITORY TAG IMAGE ID CR yandex/tutorial-catboost-clickhouse latest 622e4d17945b 22 hours ago 1.37GB ``` -**3.** この画像に基づいてDockerコンテナを起動します: +**3.** 起Dockerコンテナに基づくこのイメージ: ``` bash $ docker run -it -p 8888:8888 yandex/tutorial-catboost-clickhouse @@ -53,18 +53,18 @@ $ docker run -it -p 8888:8888 yandex/tutorial-catboost-clickhouse ## 1. テーブルの作成 {#create-table} -トレーニングサンプルのclickhouseテーブルを作成するには: +トレーニングサンプルのClickHouseテーブルを作成するには: -**1.** 開始ClickHouseコンソールがクライアントのインタラクティブモード: +**1.** 対話モードでClickHouse consoleクライアントを起動する: ``` bash $ clickhouse client ``` -!!! note "メモ" - ClickHouseサーバーはすでにDockerコンテナ内で実行されています。 +!!! note "注" + ClickHouseサーバーはDockerコンテナ内で既に実行されています。 -**2.** テーブルを作成しのコマンドを使用して: +**2.** コマンドを使用して表を作成します: ``` sql :) CREATE TABLE amazon_train @@ -84,13 +84,13 @@ $ clickhouse client ENGINE = MergeTree ORDER BY date ``` -**3.** ClickHouseコンソールクライアン: +**3.** ClickHouse consoleクライアントからの終了: ``` sql :) exit ``` -## 2. データをテーブルに挿入する {#insert-data-to-table} +## 2. テーブルにデータを挿入します {#insert-data-to-table} データを挿入するには: @@ -100,13 +100,13 @@ ENGINE = MergeTree ORDER BY date $ clickhouse client --host 127.0.0.1 --query 'INSERT INTO amazon_train FORMAT CSVWithNames' < ~/amazon/train.csv ``` -**2.** 開始ClickHouseコンソールがクライアントのインタラクティブモード: +**2.** 対話モードでClickHouse consoleクライアントを起動する: ``` bash $ clickhouse client ``` -**3.** データがアップロードされている: +**3.** データがアップロードされたことを確認: ``` sql :) SELECT count() FROM amazon_train @@ -119,20 +119,20 @@ FROM amazon_train +-------+ ``` -## 3. ClickhouseにCatBoostを統合 {#integrate-catboost-into-clickhouse} +## 3. ClickHouseにCatBoostを統合する {#integrate-catboost-into-clickhouse} -!!! note "メモ" - **省略可能なステップ。** のDocker画像を含むものを実行する必要がありますCatBoostとClickHouse. +!!! note "注" + **任意ステップ。** Dockerイメージには、CatBoostとClickHouseを実行するために必要なすべてが含まれています。 -CatBoostをClickHouseに統合するには: +ClickhouseにCatBoostを統合するには: **1.** 評価ライブラリを構築します。 -CatBoostモデルを評価する最速の方法はcompileです `libcatboostmodel.` ライブラリ。 に関する詳細については、図書館を参照 [CatBoost書](https://catboost.ai/docs/concepts/c-plus-plus-api_dynamic-c-pluplus-wrapper.html). +CatBoostモデルを評価する最速の方法はコンパイルです `libcatboostmodel.` 図書館 に関する詳細については、図書館を参照 [CatBoostドキュメント](https://catboost.ai/docs/concepts/c-plus-plus-api_dynamic-c-pluplus-wrapper.html). -**2.** 新しいディレクトリを任意の場所に、任意の名前で作成します。, `data` 作成したライブラリをその中に入れます。 Dockerイメージにはすでにライブ `data/libcatboostmodel.so`. +**2.** 新しいディレクトリを任意の場所に作成し、任意の名前で作成します。, `data` 作成したライブラリをその中に入れます。 のDocker画像がすでに含まれている図書館 `data/libcatboostmodel.so`. -**3.** Configモデルの新しいディレクトリを任意の場所に、任意の名前で作成します。, `models`. +**3.** Config modelの新しいディレクトリを任意の場所に、任意の名前で作成します。, `models`. **4.** 任意の名前のモデル構成ファイルを作成します。, `models/amazon_model.xml`. @@ -163,7 +163,7 @@ CatBoostモデルを評価する最速の方法はcompileです `libcatboostmode ## 4. SQLからモデル推論を実行する {#run-model-inference} -試験モデルのclickhouseト `$ clickhouse client`. +試験モデルのClickHouseト `$ clickhouse client`. モデルが動作していることを確認しましょう: @@ -184,10 +184,10 @@ FROM amazon_train LIMIT 10 ``` -!!! note "メモ" - 機能 [モデル値](../sql-reference/functions/other-functions.md#function-modelevaluate) マルチクラスモデルのクラスごとの生の予測を持つタプルを返します。 +!!! note "注" + 関数 [モデル評価](../sql-reference/functions/other-functions.md#function-modelevaluate) マルチクラスモデルのクラスごとの生の予測を持つタプルを返します。 -確率を予測してみましょう: +のは、確率を予測してみましょう: ``` sql :) SELECT @@ -207,10 +207,10 @@ FROM amazon_train LIMIT 10 ``` -!!! note "メモ" +!!! note "注" 詳細について [exp()](../sql-reference/functions/math-functions.md) 機能。 -サンプルのloglossを計算してみましょう: +サンプルのLogLossを計算しましょう: ``` sql :) SELECT -avg(tg * log(prob) + (1 - tg) * log(1 - prob)) AS logloss @@ -233,7 +233,7 @@ FROM ) ``` -!!! note "メモ" - 詳細について [平均()](../sql-reference/aggregate-functions/reference.md#agg_function-avg) と [ログ()](../sql-reference/functions/math-functions.md) 機能。 +!!! note "注" + 詳細について [avg()](../sql-reference/aggregate-functions/reference.md#agg_function-avg) と [ログ()](../sql-reference/functions/math-functions.md) 機能。 [元の記事](https://clickhouse.tech/docs/en/guides/apply_catboost_model/) diff --git a/docs/ja/guides/index.md b/docs/ja/guides/index.md index 84eb460b56c..585fe7fa983 100644 --- a/docs/ja/guides/index.md +++ b/docs/ja/guides/index.md @@ -1,16 +1,16 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_folder_title: Guides +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "\u30AC\u30A4\u30C9" toc_priority: 38 toc_title: "\u6982\u8981" --- -# クリックハウス {#clickhouse-guides} +# ClickHouseガイド {#clickhouse-guides} -ClickHouseを使用してさまざまなタスクを解決するのに役立つ詳細な手順のリスト: +ClickHouseを使用してさまざまなタスクを解決するのに役立つ詳細なステップバイステップの手順の一覧: -- [ストリートビューを簡単にクラスタの設定](../getting-started/tutorial.md) +- [簡単なクラスター設定のチュートリアル](../getting-started/tutorial.md) - [ClickHouseでのCatBoostモデルの適用](apply-catboost-model.md) [元の記事](https://clickhouse.tech/docs/en/guides/) diff --git a/docs/ja/interfaces/cli.md b/docs/ja/interfaces/cli.md index 1cccc83b47f..0c10ee902f5 100644 --- a/docs/ja/interfaces/cli.md +++ b/docs/ja/interfaces/cli.md @@ -1,15 +1,15 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 17 -toc_title: "\u30B3\u30DE\u30F3\u30C9\u30E9\u30A4\u30F3" +toc_title: "\u30B3\u30DE\u30F3\u30C9\u884C\u30AF\u30E9\u30A4\u30A2\u30F3\u30C8" --- -# コマンドライン {#command-line-client} +# コマンド行クライアント {#command-line-client} -ClickHouseスネイティブコマンドラインのクライアント: `clickhouse-client`. クライアン 詳細については、 [設定](#interfaces_cli_configuration). +ClickHouseスネイティブコマンドラインのクライアント: `clickhouse-client`. クライア 詳細については、 [設定](#interfaces_cli_configuration). -[設置](../getting-started/index.md) それから `clickhouse-client` パッケージとコマンドでそれを実行 `clickhouse-client`. +[インスト](../getting-started/index.md) それから `clickhouse-client` パッケージ化し、コマンドで実行します `clickhouse-client`. ``` bash $ clickhouse-client @@ -20,13 +20,13 @@ Connected to ClickHouse server version 19.17.1 revision 54428. :) ``` -異なるクライアントとサーババージョンと互換性が、一部機能が利用できない古いクライアント. サーバーアプリと同じバージョンのクライアン 古いバージョンのクライアントを使用しようとすると、サーバー, `clickhouse-client` メッセージを表示する: +異なるクライアントとサーババージョンと互換性が、一部機能が利用できない古いクライアント. お使いになることをお勧めしま同じバージョンのクライアントのサーバーアプリです。 古いバージョンのクライアントを使用しようとすると、サーバー, `clickhouse-client` メッセージを表示する: ClickHouse client version is older than ClickHouse server. It may lack support for new features. -## 使い方 {#cli_usage} +## 使用法 {#cli_usage} -このクラ バッチモードを使用するには、 ‘query’ パラメータ、または送信データに ‘stdin’ （それはそれを確認します ‘stdin’ は端末ではありません)、またはその両方。 HTTPインターフェイスと同様に、 ‘query’ パラメータとデータの送信先 ‘stdin’ リクエストはリクエストの連結である。 ‘query’ パラメータ、ラインフィード、および ‘stdin’. これは、大規模な挿入クエリに便利です。 +クライアントは、対話型および非対話型(バッチ)モードで使用できます。 バッチモードを使用するには、 ‘query’ 変数は、またはデータをに送ります ‘stdin’ （これは、 ‘stdin’ は端末ではない）、またはその両方。 HTTPインターフェイスと同様に、 ‘query’ 変数およびデータをに送ります ‘stdin’ は、リクエストの連結である。 ‘query’ パラメータ、改行、およびデータ ‘stdin’. これは、大規模なINSERTクエリに便利です。 クライアントを使用してデータを挿入する例: @@ -41,38 +41,38 @@ _EOF $ cat file.csv | clickhouse-client --database=test --query="INSERT INTO test FORMAT CSV"; ``` -バッチモードでは、デフォルトのデータ形式はtabseparatedです。 形式は、クエリのformat句で設定できます。 +バッチモードでは、デフォルトのデータ形式はTabSeparatedです。 クエリのFORMAT句で書式を設定できます。 -既定では、単一のクエリのみをバッチモードで処理できます。 aから複数のクエリを作成するには “script,” を使用 `--multiquery` パラメータ。 これはINSERT以外のすべてのクエリで機能します。 クエリ結果は、追加の区切り文字なしで連続して出力されます。 同様に、多数のクエリを処理するには、以下を実行します ‘clickhouse-client’ 各クエリについて。 それは起動に数十ミリ秒かかることに注意してください ‘clickhouse-client’ プログラム。 +既定では、バッチモードでのみ単一のクエリを処理できます。 複数のクエリを作成するには “script,” 使用する `--multiquery` パラメータ。 このため、すべてのクエリを除をサポートしていません。 クエリ結果は、追加の区切り文字なしで連続して出力されます。 同様に、多数のクエリを処理するには、次のように実行できます ‘clickhouse-client’ 各クエリに対して。 起動には数十ミリ秒かかることに注意してください。 ‘clickhouse-client’ プログラム -対話モードでは、クエリを入力できるコマンドラインが表示されます。 +対話モードでは、クエリを入力できるコマンドラインを取得します。 -もし ‘multiline’ クエリを実行するには、Enterキーを押します。 クエリの最後にセミコロンは必要ありません。 複数行のクエリを入力するには、円記号を入力します `\` ラインフィードの前に。 Enterキーを押すと、クエリの次の行を入力するように求められます。 +もし ‘multiline’ 指定されていない(デフォルト):クエリを実行するには、Enterキーを押します。 クエリの最後にセミコロンは必要ありません。 複数行のクエリを入力するには、円記号を入力します `\` ラインフィードの前に。 Enterキーを押すと、クエリの次の行を入力するように求められます。 -複数行が指定されている場合:クエリを実行するには、クエリをセミコロンで終了し、enterキーを押します。 入力された行の最後にセミコロンが省略された場合は、クエリの次の行を入力するように求められます。 +複数行が指定されている場合:クエリを実行するには、セミコロンで終了し、Enterキーを押します。 入力した行の最後にセミコロンが省略されている場合は、クエリの次の行を入力するよう求められます。 -単一のクエリのみが実行されるので、セミコロンの後のすべてが無視されます。 +単一のクエリのみが実行されるため、セミコロンの後のすべてが無視されます。 -指定できます `\G` 代わりに、またはセミコロンの後に。 これは縦書式を示します。 この形式では、各値は別々の行に印刷されます。 この珍しい機能は、MySQL CLIとの互換性のために追加されました。 +以下を指定できます `\G` セミコロンの代わりに、またはその後。 これは、垂直形式を示します。 この形式では、各値は別々の行に印刷されます。 この異常な機能は、MySQL CLIとの互換性のために追加されました。 -コマンドラインは以下に基づきます ‘replxx’ （に似て ‘readline’). つまり、使い慣れたキーボードショートカットを使用し、履歴を保持します。 歴史はに書かれています `~/.clickhouse-client-history`. +のコマンドラインに基づく ‘replxx’ （類似 ‘readline’). つまり、使い慣れたキーボードショートカットを使用し、履歴を保持します。 歴史はに書かれています `~/.clickhouse-client-history`. -デフォルトでは、使用される形式はprettycompactです。 クエリのformat句で形式を変更するか、次のように指定できます `\G` クエリの最後に、次のコマンドを使用します `--format` または `--vertical` コマンドラインの引数、またはクライアント構成ファイルの使用。 +既定では、使用される形式はPrettyCompactです。 クエリのFORMAT句で書式を変更するか、次のように指定することができます `\G` クエリの最後に、 `--format` または `--vertical` コマンドラインでの引数、またはクライアント構成ファイルの使用。 -クライアントを終了するには、ctrl+d(またはctrl+c)を押すか、クエリの代わりに次のいずれかを入力します: “exit”, “quit”, “logout”, “exit;”, “quit;”, “logout;”, “q”, “Q”, “:q” +クライアントを終了するには、Ctrl+D(またはCtrl+C)を押すか、クエリの代わりに次のいずれかを入力します: “exit”, “quit”, “logout”, “exit;”, “quit;”, “logout;”, “q”, “Q”, “:q” が処理クエリー、クライアントを示し: -1. （デフォルトでは）毎秒せいぜい10回updatedされている進捗、。 クイッククエリの場合、進行状況が表示される時間がないことがあります。 -2. デバッグのための、解析後の書式付きクエリ。 +1. (デフォルトでは)毎秒10回を超えないように更新されます。 クイッククエリの場合、進行状況を表示する時間がない場合があります。 +2. 解析後、デバッグ用に書式設定されたクエリ。 3. 指定された形式の結果。 -4. 結果の行数、経過時間、およびクエリ処理の平均速度。 +4. 結果の行数、経過した時間、およびクエリ処理の平均速度。 -ただし、サーバーが要求を中止するのを少し待つ必要があります。 特定の段階でクエリをキャンセルすることはできません。 待たずにctrl+cをもう一度押すと、クライアントは終了します。 +ただし、サーバーが要求を中止するのを少し待つ必要があります。 特定の段階でクエリをキャンセルすることはできません。 もう一度待ってCtrl+Cを押さないと、クライアントは終了します。 -コマン 詳細については、以下を参照してください “External data for query processing”. +コマンドライ 詳細については “External data for query processing”. -### クエリパラメータ {#cli-queries-with-parameters} +### パラメー {#cli-queries-with-parameters} を作成でき、クエリパラメータおよびパスの値からのお知らクライアントアプリケーション. これを避けるフォーマットのクエリが特定の動的価値観にクライアント側で行われます。 例えば: @@ -80,18 +80,18 @@ $ cat file.csv | clickhouse-client --database=test --query="INSERT INTO test FOR $ clickhouse-client --param_parName="[1, 2]" -q "SELECT * FROM table WHERE a = {parName:Array(UInt16)}" ``` -#### クエリの構文 {#cli-queries-with-parameters-syntax} +#### クエリ構文 {#cli-queries-with-parameters-syntax} -通常どおりにクエリを書式設定し、次の形式でアプリパラメーターからクエリに渡す値を中かっこで囲みます: +通常どおりにクエリを書式設定し、アプリパラメータからクエリに渡す値を次の形式の中かっこで囲みます: ``` sql {:} ``` - `name` — Placeholder identifier. In the console client it should be used in app parameters as `--param_ = value`. -- `data type` — [データ型](../sql-reference/data-types/index.md) アプリのパラメータ値の。 たとえば、次のようなデータ構造 `(integer, ('string', integer))` を持つことができ `Tuple(UInt8, Tuple(String, UInt8))` デー [整数](../sql-reference/data-types/int-uint.md) タイプ）。 +- `data type` — [データ型](../sql-reference/data-types/index.md) アプリのパラメータ値の たとえば、次のようなデータ構造 `(integer, ('string', integer))` を持つことができ `Tuple(UInt8, Tuple(String, UInt8))` データ型(別のデータ型も使用できます [整数](../sql-reference/data-types/int-uint.md) タイプ）。 -#### 例えば {#example} +#### 例 {#example} ``` bash $ clickhouse-client --param_tuple_in_tuple="(10, ('dt', 10))" -q "SELECT * FROM table WHERE val = {tuple_in_tuple:Tuple(UInt8, Tuple(String, UInt8))}" @@ -99,19 +99,19 @@ $ clickhouse-client --param_tuple_in_tuple="(10, ('dt', 10))" -q "SELECT * FROM ## 設定 {#interfaces_cli_configuration} -パラメータを渡すには `clickhouse-client` (すべてのパラメータの既定値がある): +パラメータを渡すには `clickhouse-client` (すべてのパラメータには既定値があります): - コマンドラインから コマンドラインオプションは、構成ファイルの既定値と設定を上書きします。 -- 構成ファイル。 +- 設定ファイル。 - 構成ファイルの設定は、デフォルト値を上書きします。 + 構成ファイルの設定は、既定値を上書きします。 -### コマンドラインオプ {#command-line-options} +### コマンドライ {#command-line-options} -- `--host, -h` -– The server name, ‘localhost’ デフォルトでは。 名前またはIPv4アドレスまたはIPv6アドレスを使用できます。 +- `--host, -h` -– The server name, ‘localhost’ デフォルトでは。 名前またはIPv4またはIPv6アドレスを使用できます。 - `--port` – The port to connect to. Default value: 9000. Note that the HTTP interface and the native interface use different ports. - `--user, -u` – The username. Default value: default. - `--password` – The password. Default value: empty string. @@ -120,18 +120,18 @@ $ clickhouse-client --param_tuple_in_tuple="(10, ('dt', 10))" -q "SELECT * FROM - `--multiline, -m` – If specified, allow multiline queries (do not send the query on Enter). - `--multiquery, -n` – If specified, allow processing multiple queries separated by semicolons. - `--format, -f` – Use the specified default format to output the result. -- `--vertical, -E` – If specified, use the Vertical format by default to output the result. This is the same as ‘–format=Vertical’. この形式では、各値は別の行に印刷されます。 -- `--time, -t` – If specified, print the query execution time to ‘stderr’ 非対話モードでは。 +- `--vertical, -E` – If specified, use the Vertical format by default to output the result. This is the same as ‘–format=Vertical’. この形式では、各値は別々の行に印刷されます。 +- `--time, -t` – If specified, print the query execution time to ‘stderr’ 非対話型モードで。 - `--stacktrace` – If specified, also print the stack trace if an exception occurs. - `--config-file` – The name of the configuration file. - `--secure` – If specified, will connect to server over secure connection. -- `--param_` — Value for a [クエリパラメータ](#cli-queries-with-parameters). +- `--param_` — Value for a [パラメー](#cli-queries-with-parameters). ### 設定ファイル {#configuration_files} `clickhouse-client` 次の最初の既存のファイルを使用します: -- で定義される `--config-file` パラメータ。 +- で定義される。 `--config-file` パラメータ。 - `./clickhouse-client.xml` - `~/.clickhouse-client/config.xml` - `/etc/clickhouse-client/config.xml` diff --git a/docs/ja/interfaces/cpp.md b/docs/ja/interfaces/cpp.md index fec0fb72690..caec195e85d 100644 --- a/docs/ja/interfaces/cpp.md +++ b/docs/ja/interfaces/cpp.md @@ -1,12 +1,12 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 24 -toc_title: "C++\u30AF\u30E9\u30A4\u30A2\u30F3\u30C8" +toc_title: "C++\u30AF\u30E9\u30A4\u30A2\u30F3\u30C8\u30E9\u30A4" --- -# C++クライアント {#c-client-library} +# C++クライアントライ {#c-client-library} -ﾂつｨﾂ姪"ﾂ債ﾂつｹ [クリックハウス-cpp](https://github.com/ClickHouse/clickhouse-cpp) リポジトリ +のREADMEを参照してください [クリックハウス-cpp](https://github.com/ClickHouse/clickhouse-cpp) リポジトリ。 [元の記事](https://clickhouse.tech/docs/en/interfaces/cpp/) diff --git a/docs/ja/interfaces/formats.md b/docs/ja/interfaces/formats.md index 2513fa7279f..2b4d3b06424 100644 --- a/docs/ja/interfaces/formats.md +++ b/docs/ja/interfaces/formats.md @@ -1,63 +1,64 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 21 -toc_title: "\u5165\u529B\u304A\u3088\u3073\u51FA\u529B\u5F62\u5F0F" +toc_title: "\u5165\u529B\u304A\u3088\u3073\u51FA\u529B\u30D5\u30A9\u30FC\u30DE\u30C3\ + \u30C8" --- -# 入力データと出力データの形式 {#formats} +# 入出力データのフォーマット {#formats} -ClickHouse受け入れと返信データなフレームワークです。 入力でサポートされている形式を使用して、指定されたデータを解析できます。 `INSERT`s、実行する `SELECT`ファイル、URL、またはHDFSなどのファイルバックアップテーブルから、または外部辞書を読み取ります。 出力用にサポートされている形式を使用して、 -の結果 `SELECT`、および実行する `INSERT`ファイルによって支持される表へのs。 +ClickHouse受け入れと返信データなフレームワークです。 入力用にサポートされている形式を使用して、指定されたデータを解析できます `INSERT`s、実行する `SELECT`ファイル、URL、HDFSなどのファイルバックアップテーブルから、または外部辞書を読み取ることができます。 出力でサポートされているフォーマットを使用して、 +aの結果 `SELECT`、および実行する `INSERT`ファイルバックアップテーブルにs。 のサポートされるフォーマットは: -| 書式 | 入力 | 出力 | -|---------------------------------------------------------------------|------|------| -| [タブ区切り](#tabseparated) | ✔ | ✔ | -| [TabSeparatedRaw](#tabseparatedraw) | ✗ | ✔ | -| [Tabseparatedwithnamesname](#tabseparatedwithnames) | ✔ | ✔ | -| [Tabseparatedwithnamesandtypesname](#tabseparatedwithnamesandtypes) | ✔ | ✔ | -| [テンプレ](#format-template) | ✔ | ✔ | -| [TemplateIgnoreSpaces](#templateignorespaces) | ✔ | ✗ | -| [CSV](#csv) | ✔ | ✔ | -| [Csvwithnamesname](#csvwithnames) | ✔ | ✔ | -| [CustomSeparated](#format-customseparated) | ✔ | ✔ | -| [値](#data-format-values) | ✔ | ✔ | -| [垂直](#vertical) | ✗ | ✔ | -| [VerticalRaw](#verticalraw) | ✗ | ✔ | -| [JSON](#json) | ✗ | ✔ | -| [JSONCompact](#jsoncompact) | ✗ | ✔ | -| [JSONEachRow](#jsoneachrow) | ✔ | ✔ | -| [TSKV](#tskv) | ✔ | ✔ | -| [可愛い](#pretty) | ✗ | ✔ | -| [PrettyCompact](#prettycompact) | ✗ | ✔ | -| [PrettyCompactMonoBlock](#prettycompactmonoblock) | ✗ | ✔ | -| [PrettyNoEscapes](#prettynoescapes) | ✗ | ✔ | -| [PrettySpace](#prettyspace) | ✗ | ✔ | -| [Protobuf](#protobuf) | ✔ | ✔ | -| [アブロ](#data-format-avro) | ✔ | ✔ | -| [AvroConfluent](#data-format-avro-confluent) | ✔ | ✗ | -| [Parquet張り](#data-format-parquet) | ✔ | ✔ | -| [ORC](#data-format-orc) | ✔ | ✗ | -| [RowBinary](#rowbinary) | ✔ | ✔ | -| [RowBinaryWithNamesAndTypes](#rowbinarywithnamesandtypes) | ✔ | ✔ | -| [ネイティブ](#native) | ✔ | ✔ | -| [ヌル](#null) | ✗ | ✔ | -| [XML](#xml) | ✗ | ✔ | -| [CapnProto](#capnproto) | ✔ | ✗ | +| 形式 | 入力 | 出力 | +|-----------------------------------------------------------------|------|------| +| [TabSeparated](#tabseparated) | ✔ | ✔ | +| [TabSeparatedRaw](#tabseparatedraw) | ✗ | ✔ | +| [TabSeparatedWithNames](#tabseparatedwithnames) | ✔ | ✔ | +| [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes) | ✔ | ✔ | +| [テンプレー](#format-template) | ✔ | ✔ | +| [TemplateIgnoreSpaces](#templateignorespaces) | ✔ | ✗ | +| [CSV](#csv) | ✔ | ✔ | +| [CSVWithNames](#csvwithnames) | ✔ | ✔ | +| [カスタム区切り](#format-customseparated) | ✔ | ✔ | +| [値](#data-format-values) | ✔ | ✔ | +| [垂直](#vertical) | ✗ | ✔ | +| [VerticalRaw](#verticalraw) | ✗ | ✔ | +| [JSON](#json) | ✗ | ✔ | +| [JSONCompact](#jsoncompact) | ✗ | ✔ | +| [JSONEachRow](#jsoneachrow) | ✔ | ✔ | +| [TSKV](#tskv) | ✔ | ✔ | +| [プリティ](#pretty) | ✗ | ✔ | +| [プリティコンパクト](#prettycompact) | ✗ | ✔ | +| [プリティコンパクトモノブロック](#prettycompactmonoblock) | ✗ | ✔ | +| [プリティノスケープ](#prettynoescapes) | ✗ | ✔ | +| [PrettySpace](#prettyspace) | ✗ | ✔ | +| [プロトブフ](#protobuf) | ✔ | ✔ | +| [アヴロ](#data-format-avro) | ✔ | ✔ | +| [アブロコンフルエント](#data-format-avro-confluent) | ✔ | ✗ | +| [寄木細工](#data-format-parquet) | ✔ | ✔ | +| [ORC](#data-format-orc) | ✔ | ✗ | +| [ローバイナリ](#rowbinary) | ✔ | ✔ | +| [RowBinaryWithNamesAndTypes](#rowbinarywithnamesandtypes) | ✔ | ✔ | +| [ネイティブ](#native) | ✔ | ✔ | +| [Null](#null) | ✗ | ✔ | +| [XML](#xml) | ✗ | ✔ | +| [CapnProto](#capnproto) | ✔ | ✗ | -ClickHouseの設定で、一部のフォーマット処理パラメータを制御できます。 詳細については、 [設定](../operations/settings/settings.md) セクション。 +ClickHouse設定を使用して、いくつかの形式処理パラメータを制御できます。 詳細については、 [設定](../operations/settings/settings.md) セクション -## タブ区切り {#tabseparated} +## TabSeparated {#tabseparated} -TabSeparated形式では、データは行によって書き込まれます。 各行にはタブで区切られた値が含まれます。 各値の後には、行の最後の値を除くタブが続き、その後に改行が続きます。 厳密にUnixの改行はどこでも想定されます。 最後の行には、最後に改行が含まれている必要があります。 値は、引用符を囲まずにテキスト形式で書き込まれ、特殊文字はエスケープされます。 +TabSeparated形式では、データは行ごとに書き込まれます。 各行にはタブで区切られた値が含まれます。 各値の後にはタブが続きますが、行の最後の値の後には改行が続きます。 厳正なUnixラインフィードと仮定します。 最後の行には、最後に改行も含める必要があります。 値は、引用符を囲まずにテキスト形式で書き込まれ、特殊文字はエスケープされます。 -この形式は、名前の下でも利用できます `TSV`. +この形式は、名前でも使用できます `TSV`. -その `TabSeparated` 形式は便利な加工データをカスタムプログラムやイントロダクションです。 デフォルトでは、HTTPインターフェイスとコマンドラインクライアントのバッチモードで使用されます。 この形式は、異なるDbms間でデータを転送することもできます。 たとえば、MySQLからダンプを取得してClickHouseにアップロードすることも、その逆にすることもできます。 +その `TabSeparated` 形式は便利な加工データをカスタムプログラムやイントロダクションです。 これは、HTTPインターフェイスおよびコマンドラインクライアントのバッチモードで既定で使用されます。 この形式は、異なるDbms間でデータを転送することもできます。 たとえば、MySQLからダンプを取得してClickHouseにアップロードすることも、その逆も可能です。 -その `TabSeparated` formatでは、合計値（合計と共に使用する場合）と極端な値（次の場合）の出力をサポートします ‘extremes’ 1)に設定します。 このような場合、メインデータの後に合計値と極値が出力されます。 主な結果、合計値、および極値は、空の行で区切られます。 例えば: +その `TabSeparated` 書式では、合計値(合計と共に使用する場合)と極値(場合)の出力がサポートされます ‘extremes’ 1)に設定される。 このような場合、合計値と極値がメインデータの後に出力されます。 主な結果、合計値、および極値は、空の行で区切られます。 例: ``` sql SELECT EventDate, count() AS c FROM test.hits GROUP BY EventDate WITH TOTALS ORDER BY EventDate FORMAT TabSeparated`` @@ -80,20 +81,20 @@ SELECT EventDate, count() AS c FROM test.hits GROUP BY EventDate WITH TOTALS ORD ### データの書式設定 {#data-formatting} -整数は小数で書かれています。 数字には、 “+” 最初の文字(解析時は無視され、書式設定時には記録されません)。 負でない数値には、負の符号を含めることはできません。 読み込むときには、空の文字列をゼロとして解析するか、（符号付きの型の場合）マイナス記号だけをゼロとして含む文字列を解析することができま 対応するデータ型に収まらない数値は、エラーメッセージなしで別の数値として解析できます。 +整数は小数点形式で書かれています。 番号を含むことができ追加 “+” 先頭の文字(解析時は無視され、書式設定時は記録されません)。 負でない数値には負符号を含めることはできません。 読み取り時には、空の文字列をゼロとして解析するか、（符号付き型の場合）マイナス記号だけで構成される文字列をゼロとして解析することができ 対応するデータ型に適合しない数値は、エラーメッセージなしで別の数値として解析される可能性があります。 -浮動小数点数は小数で書かれています。 ドットは小数点として使用されます。 指数作に対応してい ‘inf’, ‘+inf’, ‘-inf’、と ‘nan’. 浮動小数点数のエントリは、小数点で開始または終了することができます。 -書式設定時に、浮動小数点数の精度が失われることがあります。 +浮動小数点数は小数点形式で記述されます。 ドットは小数点区切り記号として使用されます。 指数エントリがサポートされています。 ‘inf’, ‘+inf’, ‘-inf’,and ‘nan’. 浮動小数点数のエントリは、小数点で始まったり終わったりすることがあります。 +書式設定中に、浮動小数点数の精度が失われる可能性があります。 解析中に、最も近いマシン表現可能な番号を読み取ることは厳密には必要ありません。 -日付はyyyy-mm-dd形式で書かれ、同じ形式で解析されますが、任意の文字を区切り文字として使用します。 -時刻を含む日付は、次の形式で書き込まれます `YYYY-MM-DD hh:mm:ss` 同じ形式で解析されますが、区切り文字として任意の文字が使用されます。 -これはすべて、クライアントまたはサーバーの起動時にシステムタイムゾーンで発生します(データの形式に応じて異なります)。 時刻を含む日付の場合、夏時間は指定されません。 したがって、ダンプが夏時間の間に時間がある場合、ダンプはデータと明確に一致しません。 -読み取り操作中に、時間を含む不適切な日付と日付は、エラーメッセージなしで自然なオーバーフローまたはnull日付と時刻として解析できます。 +日付はYYYY-MM-DD形式で記述され、同じ形式で解析されますが、区切り文字として任意の文字が使用されます。 +時刻付きの日付は、次の形式で記述されます `YYYY-MM-DD hh:mm:ss` 同じ形式で解析されますが、区切り文字として任意の文字で解析されます。 +これはすべて、クライアントまたはサーバーが起動したときのシステムタイムゾーンで発生します(データの形式に応じて)。 時刻のある日付の場合、夏時間は指定されません。 したがって、夏時間の間にダンプに時間がある場合、ダンプはデータと明確に一致しないため、解析は二回のいずれかを選択します。 +読み取り操作中に、エラーメッセージなしで、自然なオーバーフローまたはnullの日付と時刻として、不適切な日付と時刻を解析することができます。 -例外として、時刻を含む日付の解析は、unixタイムスタンプ形式でもサポートされています(正確に10桁の数字で構成されている場合)。 結果はタイムゾーンに依存しません。 フォーマットyyyy-mm-dd hh:mm:ssとnnnnnnnnnは自動的に区別されます。 +例外として、正確に10桁の数字で構成されている場合は、時刻を含む日付の解析もUnixタイムスタンプ形式でサポートされます。 結果はタイムゾーンに依存しません。 YYYY-MM-DD hh:mm:ssとNNNNNNNNNNの形式は自動的に区別されます。 -文字列はバックスラッシュでエスケープされた特殊文字で出力されます。 以下のエスケープシーケンスを使用出力: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\'`, `\\`. 解析にも対応し配列 `\a`, `\v`、と `\xHH` (hexエスケープシーケンス)および `\c` シーケンス、ここで `c` は任意の文字です（これらのシーケンスは `c`). このように、データを読み込む形式に対応し、改行して書き込み可能で `\n` または `\`、または改行として。 たとえば、文字列 `Hello world` スペースではなく単語間の改行を使用すると、次のいずれかのバリエーションで解析できます: +文字列はバックスラッシュでエスケープされた特殊文字で出力されます。 以下のエスケープシーケンスを使用出力: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\'`, `\\`. 解析にも対応し配列 `\a`, `\v`,and `\xHH` (hexエスケープシーケンス)と `\c` シーケンス `c` は任意の文字です(これらのシーケンスは `c`). このように、データを読み込む形式に対応し、改行して書き込み可能で `\n` または `\`、または改行として。 たとえば、文字列 `Hello world` スペースの代わりに単語の間の改行を使用すると、次のいずれかのバリエーションで解析できます: ``` text Hello\nworld @@ -102,17 +103,17 @@ Hello\ world ``` -これは、mysqlがタブで区切られたダンプを書き込むときに使用するためです。 +の変異体でサポートしてMySQLで書く時にタブ区切り捨て場. -TabSeparated形式でデータを渡すときにエスケープする必要がある文字の最小セット：tab、改行（LF）、およびバックスラッシュ。 +TabSeparated形式でデータを渡すときにエスケープする必要がある最小文字セット:tab、改行(LF)、およびバックスラッシュ。 -小さなシンボルのみがエスケープされます。 あなたの端末が出力で台無しにする文字列値に簡単につまずくことができます。 +のみの小さなセットの記号は自動的にエスケープされます。 あなたの端末が出力で台無しにする文字列値に簡単に遭遇することができます。 -配列は、角かっこで囲まれたコンマ区切りの値のリストとして記述されます。 配列内の数値項目は通常どおりに書式設定されます。 `Date` と `DateTime` 型は一重引quotesで書き込まれます。 文字列は、上記と同じエスケープ規則で一重引quotesで書き込まれます。 +配列は、角かっこで囲まれたコンマ区切りの値のリストとして記述されます。 配列内のNumber項目は、通常どおりに書式設定されます。 `Date` と `DateTime` 型は単一引quotesで記述されます。 文字列は、上記と同じエスケープ規則で一重引quotesで記述されます。 -[NULL](../sql-reference/syntax.md) フォーマットとして `\N`. +[NULL](../sql-reference/syntax.md) として書式設定される `\N`. -の各要素 [ネスト](../sql-reference/data-types/nested-data-structures/nested.md) 構造体は配列として表されます。 +の各要素 [入れ子](../sql-reference/data-types/nested-data-structures/nested.md) 構造体は配列として表されます。 例えば: @@ -142,75 +143,75 @@ SELECT * FROM nestedt FORMAT TSV ## TabSeparatedRaw {#tabseparatedraw} -とは異なります `TabSeparated` エスケープせずに行が書き込まれるという形式です。 -この形式は、クエリ結果を出力する場合にのみ適切ですが、解析(テーブルに挿入するデータの取得)には適していません。 +とは異なる `TabSeparated` 行がエスケープせずに書き込まれる形式。 +この形式は、クエリ結果の出力にのみ適していますが、解析(テーブルに挿入するデータの取得)には適していません。 -この形式は、名前の下でも利用できます `TSVRaw`. +この形式は、名前でも使用できます `TSVRaw`. -## Tabseparatedwithnamesname {#tabseparatedwithnames} +## TabSeparatedWithNames {#tabseparatedwithnames} -とは異なり `TabSeparated` 列名が最初の行に書き込まれる形式。 -解析中、最初の行は完全に無視されます。 列名を使用して、列の位置を特定したり、列の正確性を確認したりすることはできません。 -（ヘッダー行の解析のサポートは、将来追加される可能性があります。) +とは異なる `TabSeparated` 列名が最初の行に書き込まれる形式。 +解析中、最初の行は完全に無視されます。 列名を使用して位置を特定したり、列の正しさを確認したりすることはできません。 +（ヘッダー行の解析のサポートは、将来的に追加される可能性があります。) -この形式は、名前の下でも利用できます `TSVWithNames`. +この形式は、名前でも使用できます `TSVWithNames`. -## Tabseparatedwithnamesandtypesname {#tabseparatedwithnamesandtypes} +## TabSeparatedWithNamesAndTypes {#tabseparatedwithnamesandtypes} -とは異なり `TabSeparated` 列名が最初の行に書き込まれ、列タイプが次の行に書き込まれるという形式です。 -解析時には、最初と二番目の行は完全に無視されます。 +とは異なる `TabSeparated` 列名は最初の行に書き込まれ、列タイプは二番目の行に書き込まれます。 +解析中、第一行と第二行は完全に無視されます。 -この形式は、名前の下でも利用できます `TSVWithNamesAndTypes`. +この形式は、名前でも使用できます `TSVWithNamesAndTypes`. -## テンプレ {#format-template} +## テンプレー {#format-template} このフォーマットで指定するカスタムフォーマット文字列とプレースホルダーのための値を指定して逃げます。 -それは設定を使用します `format_template_resultset`, `format_template_row`, `format_template_rows_between_delimiter` and some settings of other formats (e.g. `output_format_json_quote_64bit_integers` 使用する場合 `JSON` エスケープ,さらに見る) +設定を使用します `format_template_resultset`, `format_template_row`, `format_template_rows_between_delimiter` and some settings of other formats (e.g. `output_format_json_quote_64bit_integers` 使用する場合 `JSON` エスケープ、さらに参照) -設定 `format_template_row` 次の構文の行の書式文字列を含むファイルへのパスを指定します: +設定 `format_template_row` 次の構文で行の書式指定文字列を含むファイルへのパスを指定します: `delimiter_1${column_1:serializeAs_1}delimiter_2${column_2:serializeAs_2} ... delimiter_N`, -どこに `delimiter_i` 値間の区切り文字です (`$` シンボルは `$$`), +どこに `delimiter_i` 値の区切り文字です (`$` シンボルでき逃してい `$$`), `column_i` 値が選択または挿入される列の名前またはインデックスを指定します(空の場合、列はスキップされます), -`serializeAs_i` 列の値のエスケープ規則です。 以下の脱出ルールに対応: +`serializeAs_i` 列値のエスケープ規則です。 以下の脱出ルールに対応: -- `CSV`, `JSON`, `XML` （同じ名前の形式と同様に) +- `CSV`, `JSON`, `XML` （同じ名前の形式と同様) - `Escaped` （同様に `TSV`) - `Quoted` （同様に `Values`) - `Raw` （エスケープせずに、同様に `TSVRaw`) -- `None` (エスケープルールはありません。) +- `None` （エスケープルールなし、詳細を参照) -エスケープルールが省略された場合は、 `None` 使用されます。 `XML` と `Raw` 出力にのみ適しています。 +エスケープ規則が省略された場合、 `None` 使用されます。 `XML` と `Raw` 出力のためにだけ適しています。 -したがって、次の書式文字列については: +したがって、次の書式文字列の場合: `Search phrase: ${SearchPhrase:Quoted}, count: ${c:Escaped}, ad price: $$${price:JSON};` -の値 `SearchPhrase`, `c` と `price` としてエスケープされる列 `Quoted`, `Escaped` と `JSON` (選択のために)印刷されるか、または(挿入のために)その間期待されます `Search phrase:`, `, count:`, `, ad price: $` と `;` それぞれ区切り文字。 例えば: +の値 `SearchPhrase`, `c` と `price` 列は、次のようにエスケープ `Quoted`, `Escaped` と `JSON` その間に（選択のために）印刷されるか、または（挿入のために）期待されます `Search phrase:`, `, count:`, `, ad price: $` と `;` 区切り文字。 例えば: `Search phrase: 'bathroom interior design', count: 2166, ad price: $3;` -その `format_template_rows_between_delimiter` 最後の行を除くすべての行の後に印刷される（または期待される）行間の区切り文字を指定します (`\n` デフォルトでは) +その `format_template_rows_between_delimiter` これは、最後の行を除くすべての行の後に印刷されます（または期待されます）。 (`\n` 既定では) -設定 `format_template_resultset` resultsetの書式文字列を含むファイルへのパスを指定します。 Resultsetの書式文字列は、行の書式文字列と同じ構文を持ち、接頭辞、接尾辞、およびいくつかの追加情報を出力する方法を指定できます。 で以下のプレースホルダの代わりにカラム名: +設定 `format_template_resultset` このパスには、resultsetの書式指定文字列が含まれます。 Resultsetの書式指定文字列は、行の書式指定文字列と同じ構文を持ち、接頭辞、接尾辞、および追加情報を出力する方法を指定できます。 列名の代わりに次のプレースホルダが含まれます: -- `data` データのある行ですか `format_template_row` フォーマット `format_template_rows_between_delimiter`. このプレースホルダーの最初のプレースホルダー形式の文字列になります。 -- `totals` 合計値が入っている行です `format_template_row` 形式(合計と共に使用する場合) -- `min` 最小値を持つ行です `format_template_row` フォーマット(極値が1に設定されている場合) -- `max` は、最大値を持つ行です `format_template_row` フォーマット(極値が1に設定されている場合) +- `data` はデータを含む行です `format_template_row` で区切られた形式 `format_template_rows_between_delimiter`. このプレースホルダーの最初のプレースホルダー形式の文字列になります。 +- `totals` は、合計値を持つ行です。 `format_template_row` 書式(合計で使用する場合) +- `min` は最小値を持つ行です。 `format_template_row` 書式(極値が1に設定されている場合) +- `max` は最大値を持つ行です。 `format_template_row` 書式(極値が1に設定されている場合) - `rows` 出力行の合計数です -- `rows_before_limit` そこにあったであろう行の最小数は制限なしです。 出力の場合のみを含むクエリを制限します。 クエリにGROUP BYが含まれている場合、rows\_before\_limit\_at\_leastは、制限なしで存在していた正確な行数です。 +- `rows_before_limit` そこにあったであろう行の最小数は制限なしです。 出力の場合のみを含むクエリを制限します。 クエリにGROUP BYが含まれている場合、rows\_before\_limit\_at\_leastは制限なしで行われていた行の正確な数です。 - `time` リクエストの実行時間を秒単位で指定します -- `rows_read` 読み取られた行の数です -- `bytes_read` 読み込まれたバイト数(圧縮されていないバイト数)を指定します +- `rows_read` 読み込まれた行数です +- `bytes_read` (圧縮されていない)読み込まれたバイト数です -プレースホルダ `data`, `totals`, `min` と `max` 必要な脱出ルールの指定（または `None` 明示的に指定する必要があります)。 残りのプレースホ -この `format_template_resultset` 設定は空の文字列です, `${data}` デフォルト値として使用されます。 -Insertクエリ形式では、いくつかの列またはいくつかのフィールドをスキップすることができます。 +プレースホルダ `data`, `totals`, `min` と `max` エスケープルールを指定してはいけません `None` 明示的に指定する必要があります)。 残りのプレースホルダはあるの脱出ルールを指定します。 +もし `format_template_resultset` 設定は空の文字列です, `${data}` デフォルト値として使用されます。 +Insertクエリ形式では、接頭辞または接尾辞の場合は、一部の列または一部のフィールドをスキップできます（例を参照）。 -選択例: +例を選択: ``` sql SELECT SearchPhrase, count() AS c FROM test.hits GROUP BY SearchPhrase ORDER BY c DESC LIMIT 5 FORMAT Template SETTINGS @@ -289,14 +290,14 @@ Some header\n${data}\nTotal rows: ${:CSV}\n Page views: ${PageViews:CSV}, User id: ${UserID:CSV}, Useless field: ${:CSV}, Duration: ${Duration:CSV}, Sign: ${Sign:CSV} ``` -`PageViews`, `UserID`, `Duration` と `Sign` 内部のプレースホルダーは、テーブル内の列の名前です。 その後の値 `Useless field` 行とその後 `\nTotal rows:` サフィックスでは無視されます。 +`PageViews`, `UserID`, `Duration` と `Sign` 内部のプレースホルダーは、テーブル内の列の名前です。 後の値 `Useless field` 行とその後 `\nTotal rows:` in suffixは無視されます。 すべての区切り文字の入力データを厳密に等しい区切り文字で指定されたフォーマット文字列です。 ## TemplateIgnoreSpaces {#templateignorespaces} この形式は入力にのみ適しています。 -に似て `Template` ただし、入力ストリームの区切り文字と値の間の空白文字はスキップします。 ただし、書式指定文字列に空白文字が含まれている場合は、これらの文字が入力ストリームに必要になります。 空のプレースホルダも指定できます (`${}` または `${:None}`)いくつかの区切り文字を別々の部分に分割して、それらの間の空白を無視する。 などのプレースホルダを使用させていただきますの飛び空白文字です。 -それは読むことが可能です `JSON` 列の値がすべての行で同じ順序を持つ場合、この形式を使用します。 たとえば、次のリクエストは、formatの出力例からデータを挿入するために使用できます [JSON](#json): +に類似した `Template` しかし、入力ストリーム内の区切り文字と値の間の空白文字をスキップします。 ただし、書式指定文字列に空白文字が含まれている場合、これらの文字は入力ストリーム内で使用されます。 でも指定空のプレースホルダー (`${}` または `${:None}`）区切り文字を別々の部分に分割して、それらの間のスペースを無視します。 などのプレースホルダを使用させていただきますの飛び空白文字です。 +それは読むことが可能です `JSON` 列の値がすべての行で同じ順序を持つ場合は、この形式を使用します。 たとえば、次のリクエストは、formatの出力例からデータを挿入するために使用できます [JSON](#json): ``` sql INSERT INTO table_name FORMAT TemplateIgnoreSpaces SETTINGS @@ -317,7 +318,7 @@ format_template_resultset = '/some/path/resultset.format', format_template_row = ## TSKV {#tskv} -TabSeparatedに似ていますが、name=value形式で値を出力します。 名前はTabSeparated形式と同じようにエスケープされ、=記号もエスケープされます。 +TabSeparatedに似ていますが、name=value形式で値を出力します。 名前はTabSeparated形式と同じ方法でエスケープされ、=記号もエスケープされます。 ``` text SearchPhrase= count()=8267016 @@ -332,7 +333,7 @@ SearchPhrase=curtain designs count()=1064 SearchPhrase=baku count()=1000 ``` -[NULL](../sql-reference/syntax.md) フォーマットとして `\N`. +[NULL](../sql-reference/syntax.md) として書式設定される `\N`. ``` sql SELECT * FROM t_null FORMAT TSKV @@ -342,46 +343,46 @@ SELECT * FROM t_null FORMAT TSKV x=1 y=\N ``` -多数の小さな列がある場合、この形式は無効であり、一般的にそれを使用する理由はありません。 それにもかかわらず、それは効率の面でjsoneachrowよりも悪くありません。 +小さな列が多数ある場合、この形式は無効であり、一般的に使用する理由はありません。 それにもかかわらず、それはJSONEachRowよりも悪くありません効率の面で。 Both data output and parsing are supported in this format. For parsing, any order is supported for the values of different columns. It is acceptable for some values to be omitted – they are treated as equal to their default values. In this case, zeros and blank rows are used as default values. Complex values that could be specified in the table are not supported as defaults. -解析により、追加フィールドの存在が許可されます `tskv` 等号または値なし。 この項目は無視されます。 +構文解析できるの存在は、追加のフィールド `tskv` 等号または値なし。 この項目は無視されます。 ## CSV {#csv} -コンマ区切りの値の形式 ([RFC](https://tools.ietf.org/html/rfc4180)). +カンマ区切りの値の形式 ([RFC](https://tools.ietf.org/html/rfc4180)). -書式設定の場合、行は二重引用符で囲まれます。 文字列内の二重引用符は、行内の二つの二重引用符として出力されます。 文字をエスケープする他の規則はありません。 日付と日時は二重引用符で囲みます。 数字は引用符なしで出力されます。 値は区切り文字で区切られます。 `,` デフォルトでは。 区切り文字は設定で定義されます [format\_csv\_delimiter](../operations/settings/settings.md#settings-format_csv_delimiter). 行は、Unixの改行(LF)を使用して区切られます。 まず、配列をTabSeparated形式のように文字列にシリアル化し、結果の文字列を二重引用符でCSVに出力します。 CSV形式の組は、別々の列としてシリアル化されます（つまり、組内の入れ子は失われます）。 +書式設定の場合、行は二重引用符で囲まれます。 文字列内の二重引用符は、行の二重引用符として出力されます。 文字をエスケープするルールは他にありません。 Dateとdate-timeは二重引用符で囲みます。 数値は引用符なしで出力されます。 値は区切り文字で区切られます。 `,` デフォルトでは。 区切り文字は設定で定義されています [format\_csv\_delimiter](../operations/settings/settings.md#settings-format_csv_delimiter). 行は、Unixラインフィード(LF)を使用して区切られます。 最初に、配列はTabSeparated形式のように文字列にシリアル化され、結果の文字列は二重引用符でCSVに出力されます。 CSV形式のタプルは、個別の列としてシリアル化されます(つまり、タプル内の入れ子は失われます)。 ``` bash $ clickhouse-client --format_csv_delimiter="|" --query="INSERT INTO test.csv FORMAT CSV" < data.csv ``` -\*デフォルトでは、区切り文字は `,`. を見る [format\_csv\_delimiter](../operations/settings/settings.md#settings-format_csv_delimiter) より多くの情報のための設定。 +\*デフォルトでは、区切り文字は `,`. を参照。 [format\_csv\_delimiter](../operations/settings/settings.md#settings-format_csv_delimiter) より多くの情報のための設定。 -解析時には、すべての値を引用符で囲んで解析することができます。 二重引用符と一重引quotesの両方がサポートされます。 行は、引用符なしで配置することもできます。 この場合、それらは区切り文字または改行（crまたはlf）まで解析されます。 rfcに違反して、引用符なしで行を解析するとき、先頭と末尾のスペースとタブは無視されます。 改行には、unix(lf)、windows(cr lf)、およびmac os classic(cr lf)タイプがすべてサポートされています。 +解析時には、すべての値を引用符の有無にかかわらず解析できます。 二重引用符と単一引quotesの両方がサポートされています。 行は引用符なしで配置することもできます。 この場合、区切り文字または改行（CRまたはLF）まで解析されます。 RFCに違反して、引用符なしで行を解析すると、先頭と末尾のスペースとタブは無視されます。 ラインフィードでは、Unix(LF)、Windows(CR LF)、およびMac OS Classic(CR LF)タイプがすべてサポートされています。 空の引用符で囲まれていない入力値は、それぞれの列のデフォルト値に置き換えられます。 [input\_format\_defaults\_for\_omitted\_fields](../operations/settings/settings.md#session_settings-input_format_defaults_for_omitted_fields) -は有効です。 +有効です。 -`NULL` フォーマットとして `\N` または `NULL` または、引用符で囲まれていない空の文字列(“設定”を参照 [input\_format\_csv\_unquoted\_null\_literal\_as\_null](../operations/settings/settings.md#settings-input_format_csv_unquoted_null_literal_as_null) と [input\_format\_defaults\_for\_omitted\_fields](../operations/settings/settings.md#session_settings-input_format_defaults_for_omitted_fields)). +`NULL` として書式設定される `\N` または `NULL` または空の引用符で囲まれていない文字列(設定を参照 [input\_format\_csv\_unquoted\_null\_literal\_as\_null](../operations/settings/settings.md#settings-input_format_csv_unquoted_null_literal_as_null) と [input\_format\_defaults\_for\_omitted\_fields](../operations/settings/settings.md#session_settings-input_format_defaults_for_omitted_fields)). -CSV形式は、totalsとextremesの出力を次のようにサポートします `TabSeparated`. +CSV形式では、合計と極値の出力は次のようにサポートされます `TabSeparated`. -## Csvwithnamesname {#csvwithnames} +## CSVWithNames {#csvwithnames} また、次のようなヘッダー行も出力します `TabSeparatedWithNames`. -## CustomSeparated {#format-customseparated} +## カスタム区切り {#format-customseparated} -に似て [テンプレ](#format-template) ですが、版画を読み込みまたは全てのカラムを使用脱出ルールからの設定 `format_custom_escaping_rule` 設定からの区切り文字 `format_custom_field_delimiter`, `format_custom_row_before_delimiter`, `format_custom_row_after_delimiter`, `format_custom_row_between_delimiter`, `format_custom_result_before_delimiter` と `format_custom_result_after_delimiter`、書式文字列からではありません。 -また、 `CustomSeparatedIgnoreSpaces` フォーマット `TemplateIgnoreSpaces`. +に類似した [テンプレー](#format-template) ですが、版画を読み込みまたは全てのカラムを使用脱出ルールからの設定 `format_custom_escaping_rule` 設定から区切り文字 `format_custom_field_delimiter`, `format_custom_row_before_delimiter`, `format_custom_row_after_delimiter`, `format_custom_row_between_delimiter`, `format_custom_result_before_delimiter` と `format_custom_result_after_delimiter` 書式指定文字列からではありません。 +また `CustomSeparatedIgnoreSpaces` 形式は次のようになります `TemplateIgnoreSpaces`. ## JSON {#json} -JSON形式でデータを出力します。 データテーブルのほかに、列名と型、およびいくつかの追加情報(出力行の合計数、および制限がない場合に出力される可能性のある行の数)も出力します。 例えば: +JSON形式でデータを出力します。 データテーブルのほかに、列名と型、および出力行の合計数、制限がない場合に出力される可能性のある行の数などの追加情報も出力します。 例: ``` sql SELECT SearchPhrase, count() AS c FROM test.hits GROUP BY SearchPhrase WITH TOTALS ORDER BY c DESC LIMIT 5 FORMAT JSON @@ -451,28 +452,28 @@ SELECT SearchPhrase, count() AS c FROM test.hits GROUP BY SearchPhrase WITH TOTA } ``` -JSONはJavaScriptと互換性があります。 これを確実にするために、一部の文字は追加でエスケープされます。 `/` としてエスケープ `\/`;代替改行 `U+2028` と `U+2029` いくつかのブラウザを破る、としてエスケープ `\uXXXX`. バックスペース、フォームフィード、ラインフィード、キャリッジリターン、および水平タブがエスケープされます `\b`, `\f`, `\n`, `\r`, `\t` 00-1F範囲の残りのバイトと同様に、 `\uXXXX` sequences. Invalid UTF-8 sequences are changed to the replacement character � so the output text will consist of valid UTF-8 sequences. For compatibility with JavaScript, Int64 and UInt64 integers are enclosed in double-quotes by default. To remove the quotes, you can set the configuration parameter [output\_format\_json\_quote\_64bit\_integers](../operations/settings/settings.md#session_settings-output_format_json_quote_64bit_integers) に0. +JSONはJavaScriptと互換性があります。 これを確実にするために、一部の文字は追加でエスケープされます。 `/` としてエスケープ `\/`;代替の改行 `U+2028` と `U+2029` ブラウザによってはエスケープされます `\uXXXX`. ASCII制御文字はエスケープされます。 `\b`, `\f`, `\n`, `\r`, `\t` を使用して、00-1F範囲の残りのバイトと同様に `\uXXXX` sequences. Invalid UTF-8 sequences are changed to the replacement character � so the output text will consist of valid UTF-8 sequences. For compatibility with JavaScript, Int64 and UInt64 integers are enclosed in double-quotes by default. To remove the quotes, you can set the configuration parameter [output\_format\_json\_quote\_64bit\_integers](../operations/settings/settings.md#session_settings-output_format_json_quote_64bit_integers) 0にする。 `rows` – The total number of output rows. `rows_before_limit_at_least` そこにある行の最小数は制限なしであったでしょう。 出力の場合のみを含むクエリを制限します。 -クエリにgroup byが含まれている場合、rows\_before\_limit\_at\_leastは、制限なしで存在していた正確な行数です。 +クエリにGROUP BYが含まれている場合、rows\_before\_limit\_at\_leastは制限なしで行われていた行の正確な数です。 `totals` – Total values (when using WITH TOTALS). `extremes` – Extreme values (when extremes are set to 1). -この形式は、クエリ結果を出力する場合にのみ適切ですが、解析(テーブルに挿入するデータの取得)には適していません。 +この形式は、クエリ結果の出力にのみ適していますが、解析(テーブルに挿入するデータの取得)には適していません。 -ClickHouse支援 [NULL](../sql-reference/syntax.md) として表示されます `null` JSON出力で。 +ClickHouseサポート [NULL](../sql-reference/syntax.md) として表示されます。 `null` JSON出力で。 -また、 [JSONEachRow](#jsoneachrow) フォーマット。 +も参照。 [JSONEachRow](#jsoneachrow) 形式。 ## JSONCompact {#jsoncompact} -JSONとは異なり、データ行はオブジェクトではなく配列内に出力されます。 +JSONとは異なるのは、データ行がオブジェクトではなく配列で出力される点だけです。 -例えば: +例: ``` json { @@ -511,12 +512,12 @@ JSONとは異なり、データ行はオブジェクトではなく配列内に } ``` -この形式は、クエリ結果を出力する場合にのみ適切ですが、解析(テーブルに挿入するデータの取得)には適していません。 -また、 `JSONEachRow` フォーマット。 +この形式は、クエリ結果の出力にのみ適していますが、解析(テーブルに挿入するデータの取得)には適していません。 +も参照。 `JSONEachRow` 形式。 ## JSONEachRow {#jsoneachrow} -この形式を使用する場合、clickhouseは行を区切られた改行で区切られたjsonオブジェクトとして出力しますが、データ全体が有効なjsonではありません。 +この形式を使用する場合、ClickHouseは行を区切り、改行区切りのJSONオブジェクトとして出力しますが、データ全体は有効なJSONではありません。 ``` json {"SearchPhrase":"curtain designs","count()":"1064"} @@ -524,7 +525,7 @@ JSONとは異なり、データ行はオブジェクトではなく配列内に {"SearchPhrase":"","count()":"8267016"} ``` -データを挿入するときは、各行に別々のjsonオブジェクトを指定する必要があります。 +データを挿入するときは、各行に個別のJSONオブジェクトを指定する必要があります。 ### データの挿入 {#inserting-data} @@ -532,20 +533,20 @@ JSONとは異なり、データ行はオブジェクトではなく配列内に INSERT INTO UserActivity FORMAT JSONEachRow {"PageViews":5, "UserID":"4324182021466249494", "Duration":146,"Sign":-1} {"UserID":"4324182021466249494","PageViews":6,"Duration":185,"Sign":1} ``` -クリックハウスは: +クリックハウス: -- オブジェクト内のキーと値のペアの順序。 -- いくつかの値を省略する。 +- オブジェクト内のキーと値のペアの任意の順序。 +- いくつかの値を省略します。 ClickHouseを無視した空間要素には、カンマの後にオブジェクト。 すべてのオブジェクトを一行で渡すことができます。 改行で区切る必要はありません。 **省略された値の処理** -ClickHouseは、省略された値を対応するデフォルト値に置き換えます [データ型](../sql-reference/data-types/index.md). +ClickHouseは省略された値を対応するデフォルト値に置き換えます [データ型](../sql-reference/data-types/index.md). -もし `DEFAULT expr` は、ClickHouseはに応じて異なる置換規則を使用して、指定されています [input\_format\_defaults\_for\_omitted\_fields](../operations/settings/settings.md#session_settings-input_format_defaults_for_omitted_fields) 設定。 +もし `DEFAULT expr` に応じて異なる置換規則を使用します。 [input\_format\_defaults\_for\_omitted\_fields](../operations/settings/settings.md#session_settings-input_format_defaults_for_omitted_fields) 設定。 -次の表を考えてみます: +次の表を考えます: ``` sql CREATE TABLE IF NOT EXISTS example_table @@ -555,15 +556,15 @@ CREATE TABLE IF NOT EXISTS example_table ) ENGINE = Memory; ``` -- もし `input_format_defaults_for_omitted_fields = 0` のデフォルト値を返します。 `x` と `a` 等しい `0` のデフォルト値として `UInt32` データ型)。 -- もし `input_format_defaults_for_omitted_fields = 1` のデフォルト値を返します。 `x` 等しい `0` しかし、デフォルト値は `a` 等しい `x * 2`. +- もし `input_format_defaults_for_omitted_fields = 0` のデフォルト値 `x` と `a` 等しい `0` のデフォルト値として `UInt32` データ型）。 +- もし `input_format_defaults_for_omitted_fields = 1` のデフォルト値 `x` 等しい `0` しかし、デフォルト値は `a` 等しい `x * 2`. !!! note "警告" - データを挿入するとき `insert_sample_with_metadata = 1`、ClickHouseは、より多くの計算リソースを消費します。 `insert_sample_with_metadata = 0`. + データを挿入するとき `insert_sample_with_metadata = 1`,ClickHouseは、挿入と比較して、より多くの計算リソースを消費します `insert_sample_with_metadata = 0`. ### データの選択 {#selecting-data} -考慮する `UserActivity` 例として表: +を考える `UserActivity` 例としての表: ``` text ┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ @@ -572,35 +573,35 @@ CREATE TABLE IF NOT EXISTS example_table └─────────────────────┴───────────┴──────────┴──────┘ ``` -クエリ `SELECT * FROM UserActivity FORMAT JSONEachRow` を返します: +クエリ `SELECT * FROM UserActivity FORMAT JSONEachRow` ﾂづｩﾂ。: ``` text {"UserID":"4324182021466249494","PageViews":5,"Duration":146,"Sign":-1} {"UserID":"4324182021466249494","PageViews":6,"Duration":185,"Sign":1} ``` -とは異なり [JSON](#json) 形式は、無効なUTF-8シーケンスの置換はありません。 値は、forと同じ方法でエスケープされます `JSON`. +とは異なり [JSON](#json) 無効なUTF-8シーケンスの置換はありません。 値は、次の場合と同じ方法でエスケープされます `JSON`. -!!! note "メモ" - 任意のバイトセットを文字列に出力することができます。 を使用 `JSONEachRow` テーブル内のデータをJSON形式にすることができると確信している場合は、情報を失うことなく書式設定します。 +!!! note "注" + 任意のバイトセットを文字列に出力できます。 使用する `JSONEachRow` テーブル内のデータが情報を失うことなくJSONとしてフォーマットできることが確実な場合は、format。 -### 入れ子構造の使用法 {#jsoneachrow-nested} +### 入れ子構造の使用 {#jsoneachrow-nested} -あなたがテーブルを持っている場合 [ネスト](../sql-reference/data-types/nested-data-structures/nested.md) データ型の列には、同じ構造でJSONデータを挿入することができます。 この機能を有効にするには [input\_format\_import\_nested\_json](../operations/settings/settings.md#settings-input_format_import_nested_json) 設定。 +あなたがテーブルを持っている場合 [入れ子](../sql-reference/data-types/nested-data-structures/nested.md) データ型の列には、同じ構造でJSONデータを挿入することができます。 この機能を有効にするには [input\_format\_import\_nested\_json](../operations/settings/settings.md#settings-input_format_import_nested_json) 設定。 -たとえば、次の表を考えてみます: +たとえば、次の表を考えてみましょう: ``` sql CREATE TABLE json_each_row_nested (n Nested (s String, i Int32) ) ENGINE = Memory ``` -あなたが見ることができるように `Nested` データ型の説明、ClickHouseは、入れ子構造の各コンポーネントを個別の列として扱います (`n.s` と `n.i` 私達のテーブルのため）。 次の方法でデータを挿入できます: +で見ることができるように `Nested` ClickHouseは、入れ子構造の各コンポーネントを個別の列として扱います (`n.s` と `n.i` 私達のテーブルのため）。 次の方法でデータを挿入できます: ``` sql INSERT INTO json_each_row_nested FORMAT JSONEachRow {"n.s": ["abc", "def"], "n.i": [1, 23]} ``` -データを階層jsonオブジェクトとして挿入するには、 [input\_format\_import\_nested\_json=1](../operations/settings/settings.md#settings-input_format_import_nested_json). +挿入データとしての階層JSONオブジェクト [input\_format\_import\_nested\_json=1](../operations/settings/settings.md#settings-input_format_import_nested_json). ``` json { @@ -611,7 +612,7 @@ INSERT INTO json_each_row_nested FORMAT JSONEachRow {"n.s": ["abc", "def"], "n.i } ``` -この設定がない場合、clickhouseは例外をスローします。 +この設定がない場合、ClickHouseは例外をスローします。 ``` sql SELECT name, value FROM system.settings WHERE name = 'input_format_import_nested_json' @@ -645,24 +646,24 @@ SELECT * FROM json_each_row_nested ## ネイティブ {#native} -最も効率的な形式。 データ書き込みおよび読み込みをブロックのバイナリ形式です。 各ブロックについて、行数、列数、列名と型、およびこのブロック内の列の一部が次々に記録されます。 つまり、この形式は次のとおりです “columnar” – it doesn’t convert columns to rows. This is the format used in the native interface for interaction between servers, for using the command-line client, and for C++ clients. +最も効率的な形式。 データ書き込みおよび読み込みをブロックのバイナリ形式です。 ブロックごとに、このブロック内の行数、列数、列名と型、および列の一部が次々と記録されます。 つまり、この形式は “columnar” – it doesn't convert columns to rows. This is the format used in the native interface for interaction between servers, for using the command-line client, and for C++ clients. -この形式を使用すると、clickhouse dbmsでのみ読み取ることができるダンプをすばやく生成できます。 この形式を自分で操作するのは意味がありません。 +この形式を使用すると、ClickHouse DBMSでのみ読み取ることができるダンプをすばやく生成できます。 この形式で自分で作業するのは理にかなっていません。 -## ヌル {#null} +## Null {#null} -何も出力されません。 ただし、クエリが処理され、コマンドラインクライアントを使用すると、データがクライアントに送信されます。 パフォーマンステストを含むテストに使用されます。 +何も出力されません。 ただし、クエリは処理され、コマンドラインクライアントを使用する場合、データはクライアントに送信されます。 この使用のための試験、性能試験をします。 明らかに、この形式は出力にのみ適しており、解析には適していません。 -## 可愛い {#pretty} +## プリティ {#pretty} -出力データとしてのunicodeトテーブルも用ansi-エスケープシーケンス設定色の端子です。 -テーブルの完全なグリッドが描画され、各行は端末内の二行を占めています。 -各結果ブロックは、別のテーブルとして出力されます。 これは、結果をバッファリングせずにブロックを出力できるようにするために必要です（すべての値の可視幅を事前に計算するためにバッファリ +出力データとしてのUnicodeトテーブルも用ANSI-エスケープシーケンス設定色の端子です。 +テーブルの完全なグリッドが描画され、各行は、端末内の二つの行を占めています。 +各結果ブロックは、個別のテーブルとして出力されます。 これは、結果をバッファリングせずにブロックを出力できるようにするために必要です（すべての値の可視幅を事前に計算するためにはバッファ [NULL](../sql-reference/syntax.md) として出力されます `ᴺᵁᴸᴸ`. -例(以下に示す [PrettyCompact](#prettycompact) 書式): +例（以下に示す [プリティコンパクト](#prettycompact) 形式): ``` sql SELECT * FROM t_null @@ -674,7 +675,7 @@ SELECT * FROM t_null └───┴──────┘ ``` -行はpretty\*形式でエスケープされません。 例はのために示されています [PrettyCompact](#prettycompact) 書式: +行はきれいな\*形式ではエスケープされません。 例はのために示されています [プリティコンパクト](#prettycompact) 形式: ``` sql SELECT 'String with \'quotes\' and \t character' AS Escaping_test @@ -686,10 +687,10 @@ SELECT 'String with \'quotes\' and \t character' AS Escaping_test └──────────────────────────────────────┘ ``` -ターミナルへのデータのダンプを避けるために、最初の10,000行だけが出力されます。 行数が10,000以上の場合、メッセージは次のようになります “Showed first 10 000” 印刷されます。 -この形式は、クエリ結果を出力する場合にのみ適切ですが、解析(テーブルに挿入するデータの取得)には適していません。 +端末に大量のデータをダンプするのを避けるために、最初の10,000行だけが印刷されます。 行数が10,000以上の場合、メッセージ “Showed first 10 000” 印刷されます。 +この形式は、クエリ結果の出力にのみ適していますが、解析(テーブルに挿入するデータの取得)には適していません。 -かの形式に対応出力の合計値（利用の場合との合計)は、極端な場合 ‘extremes’ 1)に設定します。 このような場合、合計値と極値がメインデータの後に別のテーブルで出力されます。 例(以下に示す [PrettyCompact](#prettycompact) 書式): +Pretty形式は、合計値(合計と共に使用する場合)および極値(場合)の出力をサポートします ‘extremes’ 1)に設定される。 このような場合、合計値と極値は、メインデータの後に個別のテーブルで出力されます。 例（以下に示す [プリティコンパクト](#prettycompact) 形式): ``` sql SELECT EventDate, count() AS c FROM test.hits GROUP BY EventDate WITH TOTALS ORDER BY EventDate FORMAT PrettyCompact @@ -718,57 +719,57 @@ Extremes: └────────────┴─────────┘ ``` -## PrettyCompact {#prettycompact} +## プリティコンパクト {#prettycompact} -とは異なります [可愛い](#pretty) グリッドが行間に描画され、結果がよりコンパクトになるという点です。 -この形式は、対話モードのコマンドラインクライアントでは既定で使用されます。 +とは異なる [プリティ](#pretty) グリッドが行の間に描画され、結果がよりコンパクトになるという点で。 +この形式は、対話モードのコマンドラインクライアントで既定で使用されます。 -## PrettyCompactMonoBlock {#prettycompactmonoblock} +## プリティコンパクトモノブロック {#prettycompactmonoblock} -とは異なります [PrettyCompact](#prettycompact) その中で最大10,000行がバッファリングされ、ブロックではなく単一のテーブルとして出力されます。 +とは異なる [プリティコンパクト](#prettycompact) 最大10,000行がバッファリングされ、ブロックではなく単一のテーブルとして出力されます。 -## PrettyNoEscapes {#prettynoescapes} +## プリティノスケープ {#prettynoescapes} -Ansi-escapeシーケンスが使用されていない点がPrettyと異なります。 これは、ブラウザでこの形式を表示するためだけでなく、 ‘watch’ コマンドラインユーティ +Ansiエスケープシーケンスが使用されない点でPrettyとは異なります。 これは、ブラウザでこの形式を表示するために必要です。 ‘watch’ コマンドライン -例えば: +例: ``` bash $ watch -n1 "clickhouse-client --query='SELECT event, value FROM system.events FORMAT PrettyCompactNoEscapes'" ``` -ブラウザに表示するためにhttpインターフェイスを使用できます。 +HTTPインターフェイスを使用して、ブラウザーで表示できます。 -### Prettompactnoescapes {#prettycompactnoescapes} +### プリティコンパクトノスケープ {#prettycompactnoescapes} 前の設定と同じです。 -### PrettySpaceNoEscapes {#prettyspacenoescapes} +### プリティスパセノスケープス {#prettyspacenoescapes} 前の設定と同じです。 ## PrettySpace {#prettyspace} -とは異なります [PrettyCompact](#prettycompact) その空白（空白文字）では、グリッドの代わりに使用されます。 +とは異なる [プリティコンパクト](#prettycompact) その中で、グリッドの代わりに空白（空白文字）が使用されます。 -## RowBinary {#rowbinary} +## ローバイナリ {#rowbinary} -バイナリ形式の行ごとにデータを書式設定および解析します。 行と値は、区切り文字なしで連続して一覧表示されます。 -この形式は、行ベースであるため、ネイティブ形式よりも効率的ではありません。 +バイナリ形式で行ごとにデータを書式設定および解析します。 行と値は、区切り文字なしで連続して表示されます。 +この形式は行ベースであるため、ネイティブ形式よりも効率的ではありません。 -整数は固定長のリトルエンディアン表現を使用します。 たとえば、uint64は8バイトを使用します。 +整数は固定長のリトルエンディアン表現を使用します。 たとえば、UInt64は8バイトを使用します。 DateTimeは、Unixタイムスタンプを値として含むUInt32として表されます。 -日付はuint16オブジェクトとして表され、このオブジェクトには1970-01-01からの日数が値として含まれます。 -Stringは、varintの長さ(符号なし)として表されます [LEB128](https://en.wikipedia.org/wiki/LEB128)その後に文字列のバイトが続きます。 -FixedStringは、単純にバイトのシーケンスとして表されます。 +Dateは、値として1970-01-01以降の日数を含むUInt16オブジェクトとして表されます。 +文字列はvarintの長さ(符号なし)で表されます [LEB128](https://en.wikipedia.org/wiki/LEB128))の後に文字列のバイトが続きます。 +FixedStringは、単にバイトのシーケンスとして表されます。 -配列は、varintの長さ(符号なし)として表されます [LEB128](https://en.wikipedia.org/wiki/LEB128)）、配列の連続した要素が続きます。 +配列はvarintの長さとして表されます(符号なし [LEB128](https://en.wikipedia.org/wiki/LEB128))の後に、配列の連続する要素が続きます。 -のために [NULL](../sql-reference/syntax.md#null-literal) 支援、追加のバイトを含む1または0が追加される前に各 [Nullable](../sql-reference/data-types/nullable.md) 値。 1の場合、値は次のようになります `NULL` このバイトは別の値として解釈されます。 0の場合、バイトの後の値はそうではありません `NULL`. +のために [NULL](../sql-reference/syntax.md#null-literal) 1または0を含む追加のバイトがそれぞれの前に追加されます [Null可能](../sql-reference/data-types/nullable.md) 値。 1の場合、値は `NULL` このバイトは別の値として解釈されます。 0の場合、バイトの後の値は `NULL`. ## RowBinaryWithNamesAndTypes {#rowbinarywithnamesandtypes} -に似て [RowBinary](#rowbinary)、しかし、追加されたヘッダと: +に類似した [ローバイナリ](#rowbinary),しかし、追加されたヘッダ: - [LEB128](https://en.wikipedia.org/wiki/LEB128)-エンコードされた列数(N) - N `String`s列名の指定 @@ -776,21 +777,21 @@ FixedStringは、単純にバイトのシーケンスとして表されます。 ## 値 {#data-format-values} -版画毎に行ットに固定して使用します。 行はコンマで区切られます。 最後の行の後にコンマはありません。 角かっこ内の値もコンマで区切られます。 数字は引用符なしの小数点形式で出力されます。 配列は角かっこで囲まれて出力されます。 文字列、日付、および時刻を含む日付が引用符で囲まれて出力されます。 ルールのエスケープと解析は、 [タブ区切り](#tabseparated) フォーマット。 書式設定時には、余分なスペースは挿入されませんが、解析時には、それらは許可され、スキップされます（配列値内のスペースは許可されません）。 [NULL](../sql-reference/syntax.md) として表されます `NULL`. +すべての行をかっこで表示します。 行はカンマで区切られます。 最後の行の後にコンマはありません。 角かっこ内の値もコンマ区切りです。 数値は、引用符なしで小数点形式で出力されます。 配列は角かっこで出力されます。 文字列、日付、および時刻付きの日付は引用符で出力されます。 ルールのエスケープと解析は [TabSeparated](#tabseparated) 形式。 フォーマット中に余分なスペースは挿入されませんが、解析中には許可され、スキップされます（配列値内のスペースは許可されません）。 [NULL](../sql-reference/syntax.md) と表される。 `NULL`. The minimum set of characters that you need to escape when passing data in Values ​​format: single quotes and backslashes. -これは、以下で使用される形式です `INSERT INTO t VALUES ...` ただし、クエリ結果の書式設定にも使用できます。 +これはで使用される形式です `INSERT INTO t VALUES ...` ただし、クエリ結果の書式設定にも使用できます。 -また見なさい: [input\_format\_values\_interpret\_expressions](../operations/settings/settings.md#settings-input_format_values_interpret_expressions) と [input\_format\_values\_deduce\_templates\_of\_expressions](../operations/settings/settings.md#settings-input_format_values_deduce_templates_of_expressions) 設定。 +も参照。: [input\_format\_values\_interpret\_expressions](../operations/settings/settings.md#settings-input_format_values_interpret_expressions) と [input\_format\_values\_deduce\_templates\_of\_expressions](../operations/settings/settings.md#settings-input_format_values_deduce_templates_of_expressions) 設定。 ## 垂直 {#vertical} -各値を、指定された列名とは別の行に出力します。 このフォーマットは、各行が多数の列で構成されている場合に、単一または少数の行だけを印刷する場合に便利です。 +列名を指定して、それぞれの値を別々の行に出力します。 この形式は、各行が多数の列で構成されている場合、一つまたは数つの行だけを印刷するのに便利です。 [NULL](../sql-reference/syntax.md) として出力されます `ᴺᵁᴸᴸ`. -例えば: +例: ``` sql SELECT * FROM t_null FORMAT Vertical @@ -816,15 +817,15 @@ test: string with 'quotes' and with some special characters ``` -この形式は、クエリ結果を出力する場合にのみ適切ですが、解析(テーブルに挿入するデータの取得)には適していません。 +この形式は、クエリ結果の出力にのみ適していますが、解析(テーブルに挿入するデータの取得)には適していません。 ## VerticalRaw {#verticalraw} -に似て [垂直](#vertical) しかし、無効にエスケープすると。 この形式は、クエリ結果の出力にのみ適しており、解析（データの受信とテーブルへの挿入）には適していません。 +に類似した [垂直](#vertical) しかし、エスケープ無効で。 この形式は、クエリ結果の出力にのみ適しており、解析（データの受信とテーブルへの挿入）には適していません。 ## XML {#xml} -XML形式は出力にのみ適しており、解析には適していません。 例えば: +XML形式は出力にのみ適しており、解析には適していません。 例: ``` xml @@ -888,24 +889,24 @@ XML形式は出力にのみ適しており、解析には適していません ``` -列名に許容可能な形式がない場合は、 ‘field’ 要素名として使用されます。 一般に、XML構造はJSON構造に従います。 +列名に許容可能な形式がない場合は、次のようにします ‘field’ 要素名として使用されます。 一般に、XML構造はJSON構造に従います。 Just as for JSON, invalid UTF-8 sequences are changed to the replacement character � so the output text will consist of valid UTF-8 sequences. 文字列値では、文字 `<` と `&` としてエスケープ `<` と `&`. -配列は出力されます `HelloWorld...`、およびタプルとして `HelloWorld...`. +配列は次のように出力される `HelloWorld...`、およびタプルとして `HelloWorld...`. ## CapnProto {#capnproto} -Cap’n Protoは、プロトコルバッファやThriftに似たバイナリメッセージ形式ですが、JSONやMessagePackには似ていません。 +Cap'n Protoは、プロトコルバッファや倹約に似たバイナリメッセージ形式ですが、JSONやMessagePackのようなものではありません。 -Cap’n Protoメッセージは厳密に型付けされており、自己記述型ではありません。 スキーマはその場で適用され、クエリごとにキャッシュされます。 +Cap'n Protoメッセージは厳密に型指定され、自己記述ではないため、外部スキーマ記述が必要です。 スキーマはその場で適用され、クエリごとにキャッシュされます。 ``` bash $ cat capnproto_messages.bin | clickhouse-client --query "INSERT INTO test.hits FORMAT CapnProto SETTINGS format_schema='schema:Message'" ``` -どこに `schema.capnp` このように見える: +どこに `schema.capnp` このように見えます: ``` capnp struct Message { @@ -914,16 +915,16 @@ struct Message { } ``` -逆シリアル化は効果的であり、通常はシステムの負荷を増加させません。 +逆シリアル化は効果的であり、通常はシステム負荷を増加させません。 -また見なさい [書式スキーマ](#formatschema). +も参照。 [スキーマの書式](#formatschema). -## Protobuf {#protobuf} +## プロトブフ {#protobuf} -Protobufは-です [プロトコル](https://developers.google.com/protocol-buffers/) フォーマット。 +Protobufは-です [プロトコル](https://developers.google.com/protocol-buffers/) 形式。 この形式には、外部形式スキーマが必要です。 このスキーマをキャッシュ間のクエリ. -クリックハウスは、 `proto2` と `proto3` 構文。 繰り返し/省略可能/必須項目がサポートされます。 +ClickHouseは両方をサポート `proto2` と `proto3` 構文。 繰り返/オプションに必要な分野に対応しています。 使用例: @@ -935,7 +936,7 @@ SELECT * FROM test.table FORMAT Protobuf SETTINGS format_schema = 'schemafile:Me cat protobuf_messages.bin | clickhouse-client --query "INSERT INTO test.table FORMAT Protobuf SETTINGS format_schema='schemafile:MessageType'" ``` -ここで、ファイル `schemafile.proto` このように見える: +ここで、ファイル `schemafile.proto` このように見えます: ``` capnp syntax = "proto3"; @@ -948,11 +949,11 @@ message MessageType { }; ``` -の対応関係はテーブル列-分野のプロトコルバッファのメッセージタイプclickhouseを比較しつけられた名前が使われている。 -この比較では、大文字と小文字は区別されません `_` （アンダースコア）と `.` （ドット）は等しいとみなされます。 -列の型とプロトコルバッファのメッセージのフィールドが異なる場合、必要な変換が適用されます。 +の対応関係はテーブル列-分野のプロトコルバッファのメッセージタイプClickHouseを比較しつけられた名前が使われている。 +この比較では、大文字と小文字は区別されません。 `_` (アンダースコア)と `.` （ドット）は等しいと見なされます。 +列の型とプロトコルバッファのメッセージのフィールドが異なる場合は、必要な変換が適用されます。 -ネストしたメッセージに対応します。 たとえば、フィールドの場合 `z` 次のメッセージタイプ +ネストしたメッセージに対応します。 たとえば、フィールドの場合 `z` 次のメッセージの種類 ``` capnp message MessageType { @@ -966,10 +967,10 @@ message MessageType { }; ``` -ClickHouseは、名前の付いた列を検索しようとします `x.y.z` （または `x_y_z` または `X.y_Z` など）。 -ネストしたメッセージを入力と出力 [入れ子のデータ構造](../sql-reference/data-types/nested-data-structures/nested.md). +ClickHouseは、名前のある列を検索しようとします `x.y.z` （または `x_y_z` または `X.y_Z` というように）。 +入れ子になったメッセージは、 [入れ子データ構造](../sql-reference/data-types/nested-data-structures/nested.md). -このようなprotobufスキーマで定義されたデフォルト値 +次のようにprotobufスキーマで定義されたデフォルト値 ``` capnp syntax = "proto2"; @@ -979,58 +980,58 @@ message MessageType { } ``` -適用されない。 [表のデフォルト](../sql-reference/statements/create.md#create-default-values) それらの代わりに使用されます。 +は適用されない。 [テーブルの既定値](../sql-reference/statements/create.md#create-default-values) それらの代わりに使用されます。 -クリックハウスの入力および出力のprotobufメッセージ `length-delimited` フォーマット。 -これは、すべてのメッセージがその長さを [varint](https://developers.google.com/protocol-buffers/docs/encoding#varints). -また見なさい [一般的な言語で長さ区切りのprotobufメッセージを読み書きする方法](https://cwiki.apache.org/confluence/display/GEODE/Delimiting+Protobuf+Messages). +ClickHouseの入力および出力protobufのメッセージ `length-delimited` 形式。 +これは、すべてのメッセージがその長さを書き込まれる前に [varint](https://developers.google.com/protocol-buffers/docs/encoding#varints). +も参照。 [一般的な言語で長さ区切りのprotobufメッセージを読み書きする方法](https://cwiki.apache.org/confluence/display/GEODE/Delimiting+Protobuf+Messages). -## アブロ {#data-format-avro} +## アヴロ {#data-format-avro} -[Apache Avro](http://avro.apache.org/) は、列指向データを直列化の枠組みに発展してApache Hadoopのプロジェクト. +[Apache Avro](http://avro.apache.org/) ApacheのHadoopプロジェクト内で開発された行指向のデータ直列化フレームワークです。 -ClickHouseアブロ形式の読み書きの支援 [Avroデータファイル](http://avro.apache.org/docs/current/spec.html#Object+Container+Files). +ClickHouse Avro形式は読み書きをサポートします [Avroデータファイル](http://avro.apache.org/docs/current/spec.html#Object+Container+Files). -### 一致するデータ型 {#data_types-matching} +### データ型の一致 {#data_types-matching} -下の表に、サポートされているデータの種類とどのように試合clickhouse [データ型](../sql-reference/data-types/index.md) で `INSERT` と `SELECT` クエリ。 +次の表に、サポートされているデータ型とClickHouseとの一致を示します [データ型](../sql-reference/data-types/index.md) で `INSERT` と `SELECT` クエリ。 -| Avroデータ型 `INSERT` | ClickHouseデータタイプ | Avroデータ型 `SELECT` | +| Avroデータ型 `INSERT` | ClickHouseデータ型 | Avroデータ型 `SELECT` | |---------------------------------------------|-------------------------------------------------------------------------------------------------------------------|------------------------------| | `boolean`, `int`, `long`, `float`, `double` | [Int(8/16/32)](../sql-reference/data-types/int-uint.md), [UInt(8/16/32)](../sql-reference/data-types/int-uint.md) | `int` | | `boolean`, `int`, `long`, `float`, `double` | [Int64](../sql-reference/data-types/int-uint.md), [UInt64](../sql-reference/data-types/int-uint.md) | `long` | | `boolean`, `int`, `long`, `float`, `double` | [Float32](../sql-reference/data-types/float.md) | `float` | | `boolean`, `int`, `long`, `float`, `double` | [Float64](../sql-reference/data-types/float.md) | `double` | | `bytes`, `string`, `fixed`, `enum` | [文字列](../sql-reference/data-types/string.md) | `bytes` | -| `bytes`, `string`, `fixed` | [FixedString(N)](../sql-reference/data-types/fixedstring.md) | `fixed(N)` | -| `enum` | [Enum(8/16)](../sql-reference/data-types/enum.md) | `enum` | -| `array(T)` | [配列(t)](../sql-reference/data-types/array.md) | `array(T)` | +| `bytes`, `string`, `fixed` | [固定文字列(N)](../sql-reference/data-types/fixedstring.md) | `fixed(N)` | +| `enum` | [Enum（8月16日)](../sql-reference/data-types/enum.md) | `enum` | +| `array(T)` | [配列(T)](../sql-reference/data-types/array.md) | `array(T)` | | `union(null, T)`, `union(T, null)` | [Nullable(T)](../sql-reference/data-types/date.md) | `union(null, T)` | -| `null` | [Nullable(何もなし)](../sql-reference/data-types/special-data-types/nothing.md) | `null` | +| `null` | [Nullable(Nothing)](../sql-reference/data-types/special-data-types/nothing.md) | `null` | | `int (date)` \* | [日付](../sql-reference/data-types/date.md) | `int (date)` \* | | `long (timestamp-millis)` \* | [DateTime64(3)](../sql-reference/data-types/datetime.md) | `long (timestamp-millis)` \* | | `long (timestamp-micros)` \* | [DateTime64(6)](../sql-reference/data-types/datetime.md) | `long (timestamp-micros)` \* | -\* [Avro論理型](http://avro.apache.org/docs/current/spec.html#Logical+Types) +\* [Avro論理タイプ](http://avro.apache.org/docs/current/spec.html#Logical+Types) -未サポートのavroデータ型: `record` (非ルート), `map` +未サポートのAvroデータ型: `record` （非ルート), `map` -サポートされないavro論理データ型: `uuid`, `time-millis`, `time-micros`, `duration` +サポートされないAvro論理データ型: `uuid`, `time-millis`, `time-micros`, `duration` ### データの挿入 {#inserting-data-1} -AvroファイルのデータをClickHouseテーブルに挿入するには: +AvroファイルからClickHouseテーブルにデータを挿入するには: ``` bash $ cat file.avro | clickhouse-client --query="INSERT INTO {some_table} FORMAT Avro" ``` -入力avroファ `record` タイプ。 +入力Avroファ `record` タイプ。 -AvroスキーマClickHouseのテーブル列とフィールド間の対応を検索するには、その名前を比較します。 この比較では、大文字と小文字が区別されます。 +の対応関係はテーブル列分野のアブロスキーマClickHouseを比較しつけられた名前が使われている。 この比較では、大文字と小文字が区別されます。 未使用の項目はスキップされます。 -データの種類clickhouseテーブルの列ができ、対応する分野においてアブロのデータを挿入します。 データを挿入するとき、clickhouseは上記の表に従ってデータ型を解釈します [キャスト](../query_language/functions/type_conversion_functions/#type_conversion_function-cast) 対応する列タイプのデータ。 +データの種類ClickHouseテーブルの列ができ、対応する分野においてアブロのデータを挿入します。 データを挿入するとき、ClickHouseは上記の表に従ってデータ型を解釈し、次に [キャスト](../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) 対応する列タイプのデータ。 ### データの選択 {#selecting-data-1} @@ -1042,28 +1043,28 @@ $ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Avro" > file.avro 列名は必須です: -- 始める `[A-Za-z_]` +- で始まる `[A-Za-z_]` - その後のみ `[A-Za-z0-9_]` -出力avroファイル圧縮およびsync間隔はと形成することができます [output\_format\_avro\_codec](../operations/settings/settings.md#settings-output_format_avro_codec) と [output\_format\_avro\_sync\_interval](../operations/settings/settings.md#settings-output_format_avro_sync_interval) それぞれ。 +出力Avroファイルの圧縮および同期間隔は以下で設定できます [output\_format\_avro\_codec](../operations/settings/settings.md#settings-output_format_avro_codec) と [output\_format\_avro\_sync\_interval](../operations/settings/settings.md#settings-output_format_avro_sync_interval) それぞれ。 -## AvroConfluent {#data-format-avro-confluent} +## アブロコンフルエント {#data-format-avro-confluent} -AvroConfluent支援復号単一のオブジェクトアブロのメッセージを使用する [カフカname](https://kafka.apache.org/) と [Confluentスキーマレジストリ](https://docs.confluent.io/current/schema-registry/index.html). +AvroConfluent支援復号単一のオブジェクトアブロのメッセージを使用する [カフカ](https://kafka.apache.org/) と [Confluentスキーマレジストリ](https://docs.confluent.io/current/schema-registry/index.html). -各avroメッセージには、スキーマレジストリを使用して実際のスキーマに解決できるスキーマidが埋め込まれます。 +各Avroメッセージには、スキーマレジストリを使用して実際のスキーマに解決できるスキーマidが埋め込まれます。 スキーマがキャッシュ一度に解決されます。 -スキーマレジスト [format\_avro\_schema\_registry\_url](../operations/settings/settings.md#settings-format_avro_schema_registry_url) +スキーマのレジストリのURLは設定され [format\_avro\_schema\_registry\_url](../operations/settings/settings.md#settings-format_avro_schema_registry_url) -### 一致するデータ型 {#data_types-matching-1} +### データ型の一致 {#data_types-matching-1} -と同じ [アブロ](#data-format-avro) +同じ [アヴロ](#data-format-avro) -### 使い方 {#usage} +### 使用法 {#usage} -使用できるスキーマ解決をすばやく検証するには [kafkacat](https://github.com/edenhill/kafkacat) と [ﾂつ"ﾂづ按つｵﾂ！](../operations/utilities/clickhouse-local.md): +迅速な検証スキーマ分解能を使用でき [kafkacat](https://github.com/edenhill/kafkacat) と [ﾂつｨﾂ姪"ﾂ債ﾂつｹ](../operations/utilities/clickhouse-local.md): ``` bash $ kafkacat -b kafka-broker -C -t topic1 -o beginning -f '%s' -c 3 | clickhouse-local --input-format AvroConfluent --format_avro_schema_registry_url 'http://schema-registry' -S "field1 Int64, field2 String" -q 'select * from table' @@ -1072,7 +1073,7 @@ $ kafkacat -b kafka-broker -C -t topic1 -o beginning -f '%s' -c 3 | clickhouse- 3 c ``` -使用するには `AvroConfluent` と [カフカname](../engines/table-engines/integrations/kafka.md): +使用するには `AvroConfluent` と [カフカ](../engines/table-engines/integrations/kafka.md): ``` sql CREATE TABLE topic1_stream @@ -1093,17 +1094,17 @@ SELECT * FROM topic1_stream; ``` !!! note "警告" - 設定 `format_avro_schema_registry_url` で構成する必要があります `users.xml` 再起動後にその価値を維持する。 + 設定 `format_avro_schema_registry_url` で構成する必要があります `users.xml` 再起動後にその値を維持する。 -## Parquet張り {#data-format-parquet} +## 寄木細工 {#data-format-parquet} -[Apacheの寄木細工](http://parquet.apache.org/) Hadoopエコシステムでは柱状のストレージ形式が普及しています。 ClickHouse支援を読み取りと書き込みの操作のためにこの形式です。 +[アパッチの寄木細工](http://parquet.apache.org/) Hadoopエコシステムに広く普及している柱状ストレージ形式です。 ClickHouse支援を読み取りと書き込みの操作のためにこの形式です。 -### 一致するデータ型 {#data_types-matching-2} +### データ型の一致 {#data_types-matching-2} -下の表に、サポートされているデータの種類とどのように試合clickhouse [データ型](../sql-reference/data-types/index.md) で `INSERT` と `SELECT` クエリ。 +次の表に、サポートされているデータ型とClickHouseとの一致を示します [データ型](../sql-reference/data-types/index.md) で `INSERT` と `SELECT` クエリ。 -| Parquetデータ型 (`INSERT`) | ClickHouseデータタイプ | Parquetデータ型 (`SELECT`) | +| Parquetデータ型 (`INSERT`) | ClickHouseデータ型 | Parquetデータ型 (`SELECT`) | |----------------------------|-----------------------------------------------------------|----------------------------| | `UINT8`, `BOOL` | [UInt8](../sql-reference/data-types/int-uint.md) | `UINT8` | | `INT8` | [Int8](../sql-reference/data-types/int-uint.md) | `INT8` | @@ -1119,39 +1120,39 @@ SELECT * FROM topic1_stream; | `DATE64`, `TIMESTAMP` | [DateTime](../sql-reference/data-types/datetime.md) | `UINT32` | | `STRING`, `BINARY` | [文字列](../sql-reference/data-types/string.md) | `STRING` | | — | [FixedString](../sql-reference/data-types/fixedstring.md) | `STRING` | -| `DECIMAL` | [小数](../sql-reference/data-types/decimal.md) | `DECIMAL` | +| `DECIMAL` | [小数点](../sql-reference/data-types/decimal.md) | `DECIMAL` | -ClickHouseは構成可能の精密をの支えます `Decimal` タイプ。 その `INSERT` クエリは寄木細工を扱います `DECIMAL` クリックハウスとして入力 `Decimal128` タイプ。 +ClickHouseは構成可能の精密を支えます `Decimal` タイプ。 その `INSERT` クエリは、寄木細工を扱います `DECIMAL` ClickHouseとして入力します `Decimal128` タイプ。 -Parquetデータ型: `DATE32`, `TIME32`, `FIXED_SIZE_BINARY`, `JSON`, `UUID`, `ENUM`. +対応していな寄木細工のデータ種類: `DATE32`, `TIME32`, `FIXED_SIZE_BINARY`, `JSON`, `UUID`, `ENUM`. -データの種類clickhouseテーブルの列ができ、対応する分野の寄木細工のデータを挿入します。 データを挿入するとき、clickhouseは上記の表に従ってデータ型を解釈します [キャスト](../query_language/functions/type_conversion_functions/#type_conversion_function-cast) ClickHouseテーブル列に設定されているデータ型へのデータ。 +データの種類ClickHouseテーブルの列ができ、対応する分野の寄木細工のデータを挿入します。 データを挿入するとき、ClickHouseは上記の表に従ってデータ型を解釈し、次に [キャスト](../query_language/functions/type_conversion_functions/#type_conversion_function-cast) ClickHouseテーブル列に設定されているそのデータ型のデータ。 ### データの挿入と選択 {#inserting-and-selecting-data} -次のコマンドを使用して、ファイルのparquetデータをclickhouseテーブルに挿入できます: +次のコマンドで、ファイルからパーケットデータをClickHouseテーブルに挿入できます: ``` bash $ cat {filename} | clickhouse-client --query="INSERT INTO {some_table} FORMAT Parquet" ``` -次のコマンドを実行すると、clickhouseテーブルからデータを選択し、parquet形式のファイルに保存することができます: +ClickHouseテーブルからデータを選択し、次のコマンドでParquet形式でファイルに保存できます: ``` bash $ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Parquet" > {some_file.pq} ``` -Hadoopとデータを交換するには、次のようにします [HDFSテーブルエンジン](../engines/table-engines/integrations/hdfs.md). +Hadoopとデータを交換するには、以下を使用できます [HDFSテーブルエンジン](../engines/table-engines/integrations/hdfs.md). ## ORC {#data-format-orc} -[Apache ORC](https://orc.apache.org/) Hadoopエコシステムでは柱状のストレージ形式が普及しています。 この形式のデータのみをClickHouseに挿入できます。 +[Apache ORC](https://orc.apache.org/) Hadoopエコシステムに広く普及している柱状ストレージ形式です。 この形式のデータはClickHouseにのみ挿入できます。 -### 一致するデータ型 {#data_types-matching-3} +### データ型の一致 {#data_types-matching-3} -下の表に、サポートされているデータの種類とどのように試合clickhouse [データ型](../sql-reference/data-types/index.md) で `INSERT` クエリ。 +次の表に、サポートされているデータ型とClickHouseとの一致を示します [データ型](../sql-reference/data-types/index.md) で `INSERT` クエリ。 -| ORCデータ型 (`INSERT`) | ClickHouseデータタイプ | +| ORCデータ型 (`INSERT`) | ClickHouseデータ型 | |------------------------|-----------------------------------------------------| | `UINT8`, `BOOL` | [UInt8](../sql-reference/data-types/int-uint.md) | | `INT8` | [Int8](../sql-reference/data-types/int-uint.md) | @@ -1166,47 +1167,47 @@ Hadoopとデータを交換するには、次のようにします [HDFSテー | `DATE32` | [日付](../sql-reference/data-types/date.md) | | `DATE64`, `TIMESTAMP` | [DateTime](../sql-reference/data-types/datetime.md) | | `STRING`, `BINARY` | [文字列](../sql-reference/data-types/string.md) | -| `DECIMAL` | [小数](../sql-reference/data-types/decimal.md) | +| `DECIMAL` | [小数点](../sql-reference/data-types/decimal.md) | -ClickHouseはの構成可能の精密を支えます `Decimal` タイプ。 その `INSERT` クエリはORCを処理します `DECIMAL` クリックハウスとして入力 `Decimal128` タイプ。 +ClickHouseはの構成可能の精密を支えます `Decimal` タイプ。 その `INSERT` クエリはORCを扱います `DECIMAL` ClickHouseとして入力します `Decimal128` タイプ。 -サポートされていな: `DATE32`, `TIME32`, `FIXED_SIZE_BINARY`, `JSON`, `UUID`, `ENUM`. +未サポートのORCデータ型: `DATE32`, `TIME32`, `FIXED_SIZE_BINARY`, `JSON`, `UUID`, `ENUM`. -ClickHouseテーブル列のデータ型は、対応するORCデータフィールドと一致する必要はありません。 データを挿入するとき、ClickHouseは上記の表に従ってデータ型を解釈します [キャスト](../query_language/functions/type_conversion_functions/#type_conversion_function-cast) ClickHouseテーブル列のデータ型セットに対するデータ。 +ClickHouseテーブル列のデータ型は、対応するORCデータフィールドと一致する必要はありません。 データを挿入するとき、ClickHouseは上記の表に従ってデータ型を解釈し、次に [キャスト](../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) ClickHouseテーブル列のデータ型セットへのデータ。 ### データの挿入 {#inserting-data-2} -次のコマンドを実行すると、ファイルのorcデータをclickhouseテーブルに挿入できます: +次のコマンドで、ファイルからOrcデータをClickHouseテーブルに挿入できます: ``` bash $ cat filename.orc | clickhouse-client --query="INSERT INTO some_table FORMAT ORC" ``` -Hadoopとデータを交換するには、次のようにします [HDFSテーブルエンジン](../engines/table-engines/integrations/hdfs.md). +Hadoopとデータを交換するには、以下を使用できます [HDFSテーブルエンジン](../engines/table-engines/integrations/hdfs.md). -## 書式スキーマ {#formatschema} +## スキーマの書式 {#formatschema} -ファイルのファイル名を含む形式スキーマを設定し、設定 `format_schema`. -いずれかの形式を使用する場合は、この設定を行う必要があります `Cap'n Proto` と `Protobuf`. -フォーマット-スキーマは、このファイル内のファイル名とメッセージ-タイプ名の組み合わせで、コロンで区切られます, +形式スキーマを含むファイル名は、この設定によって設定されます `format_schema`. +いずれかの形式を使用する場合は、この設定を設定する必要があります `Cap'n Proto` と `Protobuf`. +形式スキーマは、ファイル名とこのファイル内のメッセージ型の名前の組み合わせで、コロンで区切られます, e.g. `schemafile.proto:MessageType`. -ファイルにフォーマットの標準拡張子がある場合(例えば, `.proto` のために `Protobuf`), -これは省略することができ、この場合、形式スキーマは次のようになります `schemafile:MessageType`. +ファイルが形式の標準拡張子を持っている場合（たとえば, `.proto` のために `Protobuf`), +この場合、形式スキーマは次のようになります `schemafile:MessageType`. -データを入力するか、または出力すれば [お客様](../interfaces/cli.md) で [対話モード](../interfaces/cli.md#cli_usage)、フォーマットスキーマで指定されたファイル名 -クライアン -でクライアントを使用する場合 [バッチモード](../interfaces/cli.md#cli_usage) は、パスのスキーマ“相対的”に指定する必要があります。 +を介してデータを入力または出力する場合 [クライアン](../interfaces/cli.md) で [対話モード](../interfaces/cli.md#cli_usage),形式スキーマで指定されたファイル名 +を含むことができ、絶対パス名は相対パスを現在のディレクトリのクライアント +クライアントを使用する場合 [バッチモード](../interfaces/cli.md#cli_usage) は、パスのスキーマ"相対的"に指定する必要があります。 -データを入力するか、または出力すれば [HTTPインター](../interfaces/http.md) フォーマットスキーマで指定したファイル名 -に指定されたディレクトリにあるはずです。 [format\_schema\_path](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-format_schema_path) +を介してデータを入力または出力する場合 [HTTPインターフェ](../interfaces/http.md) 形式スキーマで指定されたファイル名 +指定されたディレクトリにあるはずです。 [format\_schema\_path](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-format_schema_path) サーバー構成で。 -## エラーのスキップ {#skippingerrors} +## スキップエラー {#skippingerrors} -以下のようないくつかの形式 `CSV`, `TabSeparated`, `TSKV`, `JSONEachRow`, `Template`, `CustomSeparated` と `Protobuf` キ壊れた列が構文解析エラーが発生したときの解析から初めます。 見る [input\_format\_allow\_errors\_num](../operations/settings/settings.md#settings-input_format_allow_errors_num) と +次のような形式があります `CSV`, `TabSeparated`, `TSKV`, `JSONEachRow`, `Template`, `CustomSeparated` と `Protobuf` 解析エラーが発生した場合に壊れた行をスキップし、次の行の先頭から解析を続行できます。 見る [input\_format\_allow\_errors\_num](../operations/settings/settings.md#settings-input_format_allow_errors_num) と [input\_format\_allow\_errors\_ratio](../operations/settings/settings.md#settings-input_format_allow_errors_ratio) 設定。 制限: --解析エラーの場合 `JSONEachRow` 新しい行（またはEOF）まですべてのデータをスキップするので、行は次のように区切られます `\n` エラーを正確にカウントする。 -- `Template` と `CustomSeparated` 最後の列の後にdelimiterを使用し、次の行の先頭を見つけるために行間の区切り文字を使用するので、エラーをスキップすると、少なくとも一方が空でない +-解析エラーの場合 `JSONEachRow` 新しい行(またはEOF)まですべてのデータをスキップします。 `\n` エラーを正しく数える。 +- `Template` と `CustomSeparated` 最後の列の後にdelimiterを使用し、行の間にdelimiterを使用すると、次の行の先頭を見つけることができます。 [元の記事](https://clickhouse.tech/docs/en/interfaces/formats/) diff --git a/docs/ja/interfaces/http.md b/docs/ja/interfaces/http.md index 71673d5f196..c76b1ba0827 100644 --- a/docs/ja/interfaces/http.md +++ b/docs/ja/interfaces/http.md @@ -1,36 +1,36 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 19 -toc_title: "HTTP\u30A4\u30F3\u30BF\u30FC" +toc_title: "HTTP\u30A4\u30F3\u30BF\u30FC\u30D5\u30A7" --- -# HTTPインター {#http-interface} +# HTTPインターフェ {#http-interface} -ﾂづﾂつｿﾂづｫﾂづｱﾂ鳴ｳﾂ猟ｿﾂづﾂづﾂつｫﾂづ慊つｷﾂ。 私たちは、javaやperl、シェルスクリプトから作業するためにそれを使用します。 他の部門では、perl、python、goからhttpインターフェイスが使用されています。 httpのインタフェースが限られにより、ネイティブインタフェースでより良い対応しています。 +HTTPイトのご利用ClickHouseにプラットフォームからゆるプログラミング言語です。 JavaやPerl、シェルスクリプトからの作業に使用します。 他の部門では、HttpインターフェイスはPerl、Python、Goから使用されます。 HTTPイ -デフォルトでは、clickhouse-serverはポート8123でhttpをリッスンします（これは設定で変更できます）。 +デフォルトでは、clickhouse-serverはポート8123でHTTPをリッスンします（これは設定で変更できます）。 -パラメータを指定せずにget/requestを実行すると、200個の応答コードと、以下で定義されている文字列が返されます [http\_server\_default\_response](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-http_server_default_response) デフォルト値 “Ok.” （最後にラインフィード付き) +パラメータなしでGET/requestを行うと、200の応答コードとで定義された文字列が返されます [http\_server\_default\_response](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-http_server_default_response) デフォルト値 “Ok.” （最後に改行があります) ``` bash $ curl 'http://localhost:8123/' Ok. ``` -ヘルスチェックスクリプトでget/ping要求を使用します。 このハンドラは常に “Ok.” （最後に改行を入れて）。 バージョン18.12.13から利用可能。 +ヘルスチェックスクリプトでGET/ping要求を使用します。 このハンドラは常に “Ok.” （最後にラインフィード付き）。 バージョン18.12.13から利用可能。 ``` bash $ curl 'http://localhost:8123/ping' Ok. ``` -リクエストをurlとして送信する ‘query’ パラメータ、または投稿として。 または、クエリの先頭を ‘query’ パラメータ、およびポストの残りの部分（これが必要な理由を後で説明します）。 URLのサイズは16KBに制限されているため、大きなクエリを送信するときはこの点に注意してください。 +リクエストをURLとして送信する ‘query’ パラメータ、またはポストとして。 または、クエリの先頭を ‘query’ パラメータ、およびポストの残りの部分（これが必要な理由を後で説明します）。 URLのサイズは16KBに制限されているため、大規模なクエリを送信する場合はこの点に注意してください。 -成功すると、200応答コードとその結果が応答本文に表示されます。 -エラーが発生すると、応答本文に500応答コードとエラーの説明テキストが表示されます。 +成功すると、200の応答コードとその結果が応答本文に表示されます。 +エラーが発生すると、応答本文に500応答コードとエラー説明テキストが表示されます。 -GETメソッドを使用する場合, ‘readonly’ 設定されています。 つまり、データを変更するクエリでは、POSTメソッドのみを使用できます。 クエリ自体は、POST本体またはURLパラメータのいずれかで送信できます。 +GETメソッドを使用する場合, ‘readonly’ 設定されています。 つまり、データを変更するクエリでは、POSTメソッドのみを使用できます。 クエリ自体は、POST本文またはURLパラメータのいずれかで送信できます。 例: @@ -53,8 +53,8 @@ X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","writ 1 ``` -あなたが見ることができるように、curlはurlエスケープする必要があります。 -Wgetはそれ自体をすべてエスケープしますが、keep-aliveとTransfer-Encoding:chunkedを使用するとHTTP1.1よりもうまく動作しないため、使用することはお勧めしません。 +ご覧の通り、カールはやや不便ですがそのスペースするURLが自動的にエスケープされます。 +Wgetはすべてそのものをエスケープしますが、keep-aliveとTransfer-Encoding:chunkedを使用する場合、HTTP1.1よりもうまく動作しないため、使用することはお勧めしません。 ``` bash $ echo 'SELECT 1' | curl 'http://localhost:8123/' --data-binary @- @@ -77,8 +77,8 @@ ECT 1 , expected One of: SHOW TABLES, SHOW DATABASES, SELECT, INSERT, CREATE, ATTACH, RENAME, DROP, DETACH, USE, SET, OPTIMIZE., e.what() = DB::Exception ``` -デフォルトでは、データはtabseparated形式で返されます(詳細については、 “Formats” セクション）。 -他の形式を要求するには、クエリのformat句を使用します。 +デフォルトでは、データはTabSeparated形式で返されます(詳細については、 “Formats” セクション）。 +他の形式を要求するには、クエリのFORMAT句を使用します。 ``` bash $ echo 'SELECT 1 FORMAT Pretty' | curl 'http://localhost:8123/?' --data-binary @- @@ -89,7 +89,7 @@ $ echo 'SELECT 1 FORMAT Pretty' | curl 'http://localhost:8123/?' --data-binary @ └───┘ ``` -データを送信するpostメソッドは、insertクエリに必要です。 この場合、urlパラメータにクエリの先頭を書き込み、postを使用して挿入するデータを渡すことができます。 挿入するデータは、たとえば、mysqlからタブで区切られたダンプです。 このようにして、insertクエリはmysqlからのload data local infileを置き換えます。 +データを送信するPOSTメソッドは、INSERTクエリに必要です。 この場合、URLパラメーターにクエリの先頭を記述し、POSTを使用して挿入するデータを渡すことができます。 挿入するデータは、たとえば、MySQLからのタブ区切りのダンプです。 このようにして、INSERTクエリはMYSQLからのLOAD DATA LOCAL INFILEを置き換えます。 例:テーブルの作成: @@ -97,7 +97,7 @@ $ echo 'SELECT 1 FORMAT Pretty' | curl 'http://localhost:8123/?' --data-binary @ $ echo 'CREATE TABLE t (a UInt8) ENGINE = Memory' | curl 'http://localhost:8123/' --data-binary @- ``` -データ挿入用の使い慣れたinsertクエリの使用: +データ挿入のための使い慣れたINSERTクエリの使用: ``` bash $ echo 'INSERT INTO t VALUES (1),(2),(3)' | curl 'http://localhost:8123/' --data-binary @- @@ -109,7 +109,7 @@ $ echo 'INSERT INTO t VALUES (1),(2),(3)' | curl 'http://localhost:8123/' --data $ echo '(4),(5),(6)' | curl 'http://localhost:8123/?query=INSERT%20INTO%20t%20VALUES' --data-binary @- ``` -任意のデータ形式を指定できます。 その ‘Values’ formatは、INSERTをt値に書き込むときに使用されるものと同じです: +任意のデータ形式を指定できます。 その ‘Values’ 書式は、INSERTをt値に書き込むときに使用される書式と同じです: ``` bash $ echo '(7),(8),(9)' | curl 'http://localhost:8123/?query=INSERT%20INTO%20t%20FORMAT%20Values' --data-binary @- @@ -121,7 +121,7 @@ $ echo '(7),(8),(9)' | curl 'http://localhost:8123/?query=INSERT%20INTO%20t%20FO $ echo -ne '10\n11\n12\n' | curl 'http://localhost:8123/?query=INSERT%20INTO%20t%20FORMAT%20TabSeparated' --data-binary @- ``` -テーブルの内容を読む。 データは、並列クエリ処理によりランダムな順序で出力されます: +テーブルの内容を読む。 並列クエリ処理により、データがランダムに出力されます: ``` bash $ curl 'http://localhost:8123/?query=SELECT%20a%20FROM%20t' @@ -145,14 +145,14 @@ $ curl 'http://localhost:8123/?query=SELECT%20a%20FROM%20t' $ echo 'DROP TABLE t' | curl 'http://localhost:8123/' --data-binary @- ``` -データテーブルを返さない要求が成功すると、空のレスポンスボディが返されます。 +データテーブルを返さない正常な要求の場合、空の応答本文が返されます。 -データを送信するときには、内部のclickhouse圧縮形式を使用できます。 圧縮されたデータには標準以外の形式があり、特別な形式を使用する必要があります `clickhouse-compressor` それを使用するプログラム（それは `clickhouse-client` パッケージ）。 データ挿入の効率を高めるために、サーバー側のチェックサム検証を無効にするには [http\_native\_compression\_disable\_checksumming\_on\_decompress](../operations/settings/settings.md#settings-http_native_compression_disable_checksumming_on_decompress) 設定。 +データを送信するときは、内部ClickHouse圧縮形式を使用できます。 圧縮されたデータは非標準形式であり、特別な形式を使用する必要があります `clickhouse-compressor` それで動作するようにプログラム（それは `clickhouse-client` パッケージ）。 データ挿入の効率を高めるために、以下を使用してサーバー側のチェックサム検証を無効にできます [http\_native\_compression\_disable\_checksumming\_on\_decompress](../operations/settings/settings.md#settings-http_native_compression_disable_checksumming_on_decompress) 設定。 -指定した場合 `compress=1` URLでは、サーバーは送信するデータを圧縮します。 -指定した場合 `decompress=1` URLでは、サーバーは渡したデータと同じデータを解凍します。 `POST` 方法。 +指定した場合 `compress=1` URLでは、サーバーが送信するデータを圧縮します。 +指定した場合 `decompress=1` このURLでは、サーバーは渡すデータと同じデータを解凍します。 `POST` 方法。 -また、使用することを選択 [HTTP圧縮](https://en.wikipedia.org/wiki/HTTP_compression). 圧縮を送信するには `POST` 要求、要求ヘッダーを追加します `Content-Encoding: compression_method`. ClickHouseが応答を圧縮するには、次のように追加する必要があります `Accept-Encoding: compression_method`. ClickHouse支援 `gzip`, `br`、と `deflate` [圧縮方法](https://en.wikipedia.org/wiki/HTTP_compression#Content-Encoding_tokens). HTTP圧縮を有効にするには、ClickHouseを使用する必要があります [enable\_http\_compression](../operations/settings/settings.md#settings-enable_http_compression) 設定。 データ圧縮レベルを設定することができます [http\_zlib\_compression\_level](#settings-http_zlib_compression_level) すべての圧縮方法の設定。 +また、使用することもできます [HTTP圧縮](https://en.wikipedia.org/wiki/HTTP_compression). 圧縮を送信するには `POST` リクエストヘッダーを追加します `Content-Encoding: compression_method`. ClickHouseが応答を圧縮するには、次のように追加する必要があります `Accept-Encoding: compression_method`. ClickHouseサポート `gzip`, `br`,and `deflate` [圧縮方法](https://en.wikipedia.org/wiki/HTTP_compression#Content-Encoding_tokens). HTTP圧縮を有効にするには、ClickHouseを使用する必要があります [enable\_http\_compression](../operations/settings/settings.md#settings-enable_http_compression) 設定。 データ圧縮レベルは、 [http\_zlib\_compression\_level](#settings-http_zlib_compression_level) すべての圧縮方法の設定。 利用することができ削減ネットワーク通信の送受信には大量のデータをダンプすると直ちに圧縮されます。 @@ -166,10 +166,10 @@ $ curl -vsS "http://localhost:8123/?enable_http_compression=1" -d 'SELECT number $ echo "SELECT 1" | gzip -c | curl -sS --data-binary @- -H 'Content-Encoding: gzip' 'http://localhost:8123/' ``` -!!! note "メモ" - 一部のhttpクライアントは、デフォルトでサーバーからデータを解凍します `gzip` と `deflate` 圧縮設定を正しく使用していても、圧縮解除されたデータが得られることがあります。 +!!! note "注" + あるHTTPお客様が圧縮解除データからサーバによるデフォルト（ `gzip` と `deflate`）圧縮設定を正しく使用しても、圧縮解除されたデータが得られることがあります。 -を使用することができ ‘database’ URLパラメータの指定はデフォルトのデータベースです。 +を使用することができます ‘database’ 既定のデータベースを指定するURLパラメータ。 ``` bash $ echo 'SELECT number FROM numbers LIMIT 10' | curl 'http://localhost:8123/?database=system' --data-binary @- @@ -185,11 +185,11 @@ $ echo 'SELECT number FROM numbers LIMIT 10' | curl 'http://localhost:8123/?data 9 ``` -既定では、サーバー設定に登録されているデータベースが既定のデータベースとして使用されます。 既定では、これはデータベースと呼ばれます ‘default’. あるいは、必ず指定のデータベースをドットの前にテーブルの名前です。 +既定では、サーバー設定に登録されているデータベースが既定のデータベースとして使用されます。 デフォルトでは、このデータベース ‘default’. または、テーブル名の前にドットを使用してデータベースを常に指定できます。 ユーザー名とパスワードは、次のいずれかの方法で指定できます: -1. HTTP基本認証を使用する。 例えば: +1. HTTP基本認証の使用。 例: @@ -197,7 +197,7 @@ $ echo 'SELECT number FROM numbers LIMIT 10' | curl 'http://localhost:8123/?data $ echo 'SELECT 1' | curl 'http://user:password@localhost:8123/' -d @- ``` -1. で ‘user’ と ‘password’ URLパラメーター。 例えば: +1. で ‘user’ と ‘password’ URLパラメータ。 例: @@ -205,7 +205,7 @@ $ echo 'SELECT 1' | curl 'http://user:password@localhost:8123/' -d @- $ echo 'SELECT 1' | curl 'http://localhost:8123/?user=user&password=password' -d @- ``` -1. を使用して ‘X-ClickHouse-User’ と ‘X-ClickHouse-Key’ ヘッダー。 例えば: +1. を使用して ‘X-ClickHouse-User’ と ‘X-ClickHouse-Key’ ヘッダー。 例: @@ -213,10 +213,10 @@ $ echo 'SELECT 1' | curl 'http://localhost:8123/?user=user&password=password' -d $ echo 'SELECT 1' | curl -H 'X-ClickHouse-User: user' -H 'X-ClickHouse-Key: password' 'http://localhost:8123/' -d @- ``` -ユーザー名が指定されていない場合は、 `default` 名前が使用されます。 パスワードを指定しない場合は、空のパスワードが使用されます。 -にお使いいただけますurlパラメータで指定した設定処理の単一クエリーまたは全体をプロファイルを設定します。 例：http：//localhost：8123/？プロファイル=ウェブ&max\_rows\_to\_read=1000000000&クエリ=選択+1 +ユーザ名が指定されていない場合、 `default` 名前が使用されます。 パスワードを指定しない場合は、空のパスワードが使用されます。 +にお使いいただけますURLパラメータで指定した設定処理の単一クエリーまたは全体をプロファイルを設定します。 例:http://localhost:8123/?profile=web&max\_rows\_to\_read=1000000000&query=SELECT+1 -詳細については、 [設定](../operations/settings/index.md) セクション。 +詳細については、を参照してください [設定](../operations/settings/index.md) セクション ``` bash $ echo 'SELECT number FROM system.numbers LIMIT 10' | curl 'http://localhost:8123/?' --data-binary @- @@ -232,11 +232,11 @@ $ echo 'SELECT number FROM system.numbers LIMIT 10' | curl 'http://localhost:812 9 ``` -その他のパラ “SET”. +のための情報その他のパラメータの項をご参照ください “SET”. -同様に、httpプロトコルでclickhouseセッションを使用できます。 これを行うには、以下を追加する必要があります `session_id` 要求のパラメーターを取得します。 セッションIDとして任意の文字列を使用できます。 デフォルトでは、セッションは60秒の非アクティブの後に終了します。 このタイムアウトを変更するには、 `default_session_timeout` サーバー構成での設定、または `session_timeout` 要求のパラメーターを取得します。 セッショ `session_check=1` パラメータ。 単一のセッション内で実行できるのは、一度にひとつのクエリのみです。 +同様に、HttpプロトコルでClickHouseセッションを使用できます。 これを行うには、 `session_id` 要求のパラメータを取得します。 セッションIDとして任意の文字列を使用できます。 既定では、セッションは非アクティブの60秒後に終了します。 このタイムアウトを変更するには、 `default_session_timeout` サーバー構成で設定するか、または `session_timeout` 要求のパラメータを取得します。 セッションステータスを確認するには、 `session_check=1` パラメータ。 単一のセッション内で一度に一つのクエリだけを実行できます。 -クエリの進行状況に関する情報を受け取ることができます `X-ClickHouse-Progress` 応答ヘッダー。 これを行うには、 [send\_progress\_in\_http\_headers](../operations/settings/settings.md#settings-send_progress_in_http_headers). ヘッダーシーケンスの例: +クエリの進行状況に関する情報を受け取ることができます `X-ClickHouse-Progress` 応答ヘッダー。 これを行うには、有効にします [send\_progress\_in\_http\_headers](../operations/settings/settings.md#settings-send_progress_in_http_headers). ヘッダシーケンスの例: ``` text X-ClickHouse-Progress: {"read_rows":"2752512","read_bytes":"240570816","total_rows_to_read":"8880128"} @@ -244,7 +244,7 @@ X-ClickHouse-Progress: {"read_rows":"5439488","read_bytes":"482285394","total_ro X-ClickHouse-Progress: {"read_rows":"8783786","read_bytes":"819092887","total_rows_to_read":"8880128"} ``` -可能なヘッダフィールド: +可能なヘッダ項目: - `read_rows` — Number of rows read. - `read_bytes` — Volume of data read in bytes. @@ -252,40 +252,40 @@ X-ClickHouse-Progress: {"read_rows":"8783786","read_bytes":"819092887","total_ro - `written_rows` — Number of rows written. - `written_bytes` — Volume of data written in bytes. -HTTP接続が失われた場合、実行中の要求は自動的に停止しません。 解析とデータフォーマットはサーバー側で実行され、ネットワークを使用することは効果がありません。 -任意 ‘query\_id’ パラメータは、クエリID(任意の文字列)として渡すことができます。 詳細については、以下を参照してください “Settings, replace\_running\_query”. +HTTP接続が失われても、要求の実行は自動的に停止しません。 解析とデータの書式設定はサーバー側で実行され、ネットワークを使用すると無効になる可能性があります。 +任意 ‘query\_id’ パラメータは、クエリID(任意の文字列)として渡すことができます。 詳細については “Settings, replace\_running\_query”. -任意 ‘quota\_key’ パラメータとして渡すことができ、クォーターキー(切文字列). 詳細については、以下を参照してください “Quotas”. +任意 ‘quota\_key’ パラメータとして渡すことができ、クォーターキー(切文字列). 詳細については “Quotas”. -HTTPのインターフェースにより通外部データ（外部テンポラリテーブル)照会. 詳細については、以下を参照してください “External data for query processing”. +HTTPイ 詳細については “External data for query processing”. ## 応答バッファリング {#response-buffering} サーバー側で応答バッファリングを有効にできます。 その `buffer_size` と `wait_end_of_query` URLパラメータを提供しています。 -`buffer_size` サーバーメモリ内のバッファーに結果内のバイト数を決定します。 結果本体がこのしきい値より大きい場合、バッファはHTTPチャネルに書き込まれ、残りのデータはHTTPチャネルに直接送信されます。 +`buffer_size` サーバーメモリ内のバッファーに格納する結果のバイト数を決定します。 結果の本文がこのしきい値より大きい場合、バッファはHTTPチャネルに書き込まれ、残りのデータはHTTPチャネルに直接送信されます。 -応答全体を確実にバッファリングするには、以下を設定します `wait_end_of_query=1`. この場合、メモリに格納されていないデータは、一時サーバーファイルにバッファーされます。 +保全の対応はバッファ処理されますが、設定 `wait_end_of_query=1`. この場合、メモリに格納されていないデータは一時サーバーファイルにバッファされます。 -例えば: +例: ``` bash $ curl -sS 'http://localhost:8123/?max_result_bytes=4000000&buffer_size=3000000&wait_end_of_query=1' -d 'SELECT toUInt8(number) FROM system.numbers LIMIT 9000000 FORMAT RowBinary' ``` -使用バッファリングなどでクエリの処理エラーが発生した後は、応答コード、httpヘッダが送信されます。 この状況では、エラーメッセージが応答本文の最後に書き込まれ、クライアント側では、解析の段階でのみエラーを検出できます。 +使用バッファリングなどでクエリの処理エラーが発生した後は、応答コード、HTTPヘッダが送信されます。 この状況では、応答本文の最後にエラーメッセージが書き込まれ、クライアント側では、エラーは解析段階でのみ検出できます。 -### クエリパラメータ {#cli-queries-with-parameters} +### パラメー {#cli-queries-with-parameters} -を作成でき、クエリパラメータおよびパスの値から、対応するhttpリクエストパラメータ. 詳細については、 [CLIのパラメータを持つクエリ](cli.md#cli-queries-with-parameters). +パラメータを使用してクエリを作成し、対応するHTTP要求パラメータから値を渡すことができます。 詳細については、 [CLIのパラメータを使用した照会](cli.md#cli-queries-with-parameters). -### 例えば {#example} +### 例 {#example} ``` bash $ curl -sS "
?param_id=2¶m_phrase=test" -d "SELECT * FROM table WHERE int_column = {id:UInt8} and string_column = {phrase:String}" ``` -## 所定のhttpス {#predefined_http_interface} +## 所定のHTTPス {#predefined_http_interface} ClickHouse支特定のクエリはHTTPインターフェース。 たとえば、次のように表にデータを書き込むことができます: @@ -293,46 +293,51 @@ ClickHouse支特定のクエリはHTTPインターフェース。 たとえば $ echo '(4),(5),(6)' | curl 'http://localhost:8123/?query=INSERT%20INTO%20t%20VALUES' --data-binary @- ``` -ClickHouseは定義済みのHTTPインターフェイスもサポートしています。 [プロメテウス輸出](https://github.com/percona-lab/clickhouse_exporter). +ClickHouseにも対応した所定のHTTPインターフェースができありがとの統合に第三者のリーディングプロジェクト [プロメテウス輸出](https://github.com/percona-lab/clickhouse_exporter). -例えば: +例: -- まず、このセクションをサーバー構成ファイルに追加します: +- まず、このセクションをサーバー設定ファイルに追加します: ``` xml - - /metrics - GET - + + /predefined_query + POST,GET + + predefined_query_handler SELECT * FROM system.metrics LIMIT 5 FORMAT Template SETTINGS format_template_resultset = 'prometheus_template_output_format_resultset', format_template_row = 'prometheus_template_output_format_row', format_template_rows_between_delimiter = '\n' - - + + + ... + ... ``` -- Prometheus形式のデータのurlを直接要求できるようになりました: +- Prometheus形式のデータのurlを直接リクエストできるようになりました: ``` bash -curl -vvv 'http://localhost:8123/metrics' +$ curl -v 'http://localhost:8123/predefined_query' * Trying ::1... * Connected to localhost (::1) port 8123 (#0) -> GET /metrics HTTP/1.1 +> GET /predefined_query HTTP/1.1 > Host: localhost:8123 > User-Agent: curl/7.47.0 > Accept: */* > < HTTP/1.1 200 OK -< Date: Wed, 27 Nov 2019 08:54:25 GMT +< Date: Tue, 28 Apr 2020 08:52:56 GMT < Connection: Keep-Alive < Content-Type: text/plain; charset=UTF-8 -< X-ClickHouse-Server-Display-Name: i-tl62qd0o +< X-ClickHouse-Server-Display-Name: i-mloy5trc < Transfer-Encoding: chunked -< X-ClickHouse-Query-Id: f39235f6-6ed7-488c-ae07-c7ceafb960f6 +< X-ClickHouse-Query-Id: 96fe0052-01e6-43ce-b12a-6b7370de6e8a +< X-ClickHouse-Format: Template +< X-ClickHouse-Timezone: Asia/Shanghai < Keep-Alive: timeout=3 < X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} < @@ -356,117 +361,62 @@ curl -vvv 'http://localhost:8123/metrics' # TYPE "ReplicatedSend" counter "ReplicatedSend" 0 +* Connection #0 to host localhost left intact + + * Connection #0 to host localhost left intact ``` -この例からわかるように、 `` 設定で設定されています。xmlファイル、ClickHouseは、定義済みのタイプで受信したHTTP要求と一致します `` 一致が成功した場合、ClickHouseは対応する事前定義されたクエリを実行します。 +この例からわかるように、 `` 設定で構成されています。xmlファイルと `` を含むことができ多くの `s`. ClickHouseは、受信したHTTP要求を事前定義されたタイプに一致させます `` 最初に一致したハンドラが実行されます。 一致が成功すると、ClickHouseは対応する事前定義されたクエリを実行します。 -さて `` 設定できます ``, ``, ``, `` と `` . +> さて `` 構成できます ``, ``, ``,``: +> `` HTTP要求のメソッド部分の照合を担当します。 `` 十分に定義にの合致します [方法](https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods) HTTPプロトコルで。 これはオプションの構成です。 構成ファイルで定義されていない場合、HTTP要求のメソッド部分と一致しません。 +> +> `` HTTPリクエストのurl部分の照合を担当します。 それはと互換性があります [RE2](https://github.com/google/re2)の正規表現。 これはオプションの構成です。 構成ファイルで定義されていない場合、HTTP要求のurl部分と一致しません。 +> +> `` HTTPリクエストのヘッダー部分の照合を担当します。 RE2の正規表現と互換性があります。 これはオプションの構成です。 構成ファイルで定義されていない場合、HTTP要求のヘッダー部分と一致しません。 +> +> `` 主要な処理の部品を含んでいます。 さて `` 構成できます ``, ``, ``, ``, ``, ``. +> \> `` 現在サポート: **predefined\_query\_handler**, **dynamic\_query\_handler**, **静的**. +> \> +> \> `` -predefined\_query\_handler型で使用し、ハンドラが呼び出されたときにクエリを実行します。 +> \> +> \> `` -dynamic\_query\_handler型で使用すると、それに対応する値を抽出して実行します。 `` HTTP要求パラメータの値。 +> \> +> \> `` -静的タイプ、応答ステータスコードで使用します。 +> \> +> \> `` -静的なタイプ、応答との使用 [コンテンツタイプ](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type). +> \> +> \> `` 使用静止型で、応答が送信（発信）したコンテンツをクライアントでご使用になる場合、接頭辞 ‘file://’ または ‘config://’、クライアントに送信するファイルまたは構成から内容を検索します。 -## root\_handler {#root_handler} - -`` ルートパス要求の指定された内容を返します。 特定の戻りコンテンツは、 `http_server_default_response` 設定で。xmlだ 指定しない場合は、戻り値 **わかった** - -`http_server_default_response` 定義されておらず、HttpリクエストがClickHouseに送信されます。 結果は次のとおりです: - -``` xml - - - -``` - - $ curl 'http://localhost:8123' - Ok. - -`http_server_default_response` 定義され、HttpリクエストがClickHouseに送信されます。 結果は次のとおりです: - -``` xml -
]]> - - - - -``` - - $ curl 'http://localhost:8123' -
% - -## ping\_handler {#ping_handler} - -`` 現在のClickHouseサーバーの健康を調査するのに使用することができます。 がClickHouse HTTPサーバが正常にアクセスClickHouseを通じて `` 戻ります **わかった**. - -例えば: - -``` xml - - /ping - -``` - -``` bash -$ curl 'http://localhost:8123/ping' -Ok. -``` - -## replicas\_status\_handler {#replicas_status_handler} - -`` レプリカノードの状態を検出して返すために使用されます **わかった** レプリカノードに遅延がない場合。 遅延がある場合、リターン特定の遅延。 の値 `` カスタマイズ対応。 指定しない場合 ``、ClickHouseデフォルト設定 `` は **/replicas\_status**. - -例えば: - -``` xml - - /replicas_status - -``` - -遅延なしケース: - -``` bash -$ curl 'http://localhost:8123/replicas_status' -Ok. -``` - -遅延ケース: - -``` bash -$ curl 'http://localhost:8123/replicas_status' -db.stats: Absolute delay: 22. Relative delay: 22. -``` +次に、異なる設定方法を示します ``. ## predefined\_query\_handler {#predefined_query_handler} -設定できます ``, ``, `` と `` で ``. +`` 設定とquery\_params値の設定をサポートします。 設定できます `` のタイプで ``. -`` は、HTTPリクエストのメソッド部分のマッチングを担当します。 `` 定義にの十分に合致します [方法](https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods) HTTPプロトコルで。 これはオプションの設定です。 構成ファイルで定義されていない場合は、HTTP要求のメソッド部分と一致しません - -`` HTTPリクエストのurl部分を照合する責任があります。 それはと互換性があります [RE2](https://github.com/google/re2)’の正規表現。 これはオプションの設定です。 構成ファイルで定義されていない場合は、HTTP要求のurl部分と一致しません - -`` HTTPリクエストのヘッダ部分に一致させる責任があります。 これはRE2の正規表現と互換性があります。 これはオプションの設定です。 構成ファイルで定義されていない場合は、HTTP要求のヘッダー部分と一致しません - -`` 値は、以下の定義済みのクエリです `` これは、HTTP要求が一致し、クエリの結果が返されたときにClickHouseによって実行されます。 これは必須の設定です。 - -`` 設定とquery\_params値の設定をサポートしています。 +`` 値は以下の定義済みクエリです `` これは、Http要求が一致し、クエリの結果が返されたときにClickHouseによって実行されます。 これは必須構成です。 次の例では、次の値を定義します `max_threads` と `max_alter_threads` 設定、そしてクエリのテーブルから設定設定します。 -例えば: +例: ``` xml - - + + + [^/]+)(/(?P[^/]+))?]]> GET TEST_HEADER_VALUE [^/]+)(/(?P[^/]+))?]]> - [^/]+)(/(?P[^/]+))?]]> - + + predefined_query_handler SELECT value FROM system.settings WHERE name = {name_1:String} SELECT name, value FROM system.settings WHERE name = {name_2:String} - - - + + + ``` ``` bash @@ -475,37 +425,193 @@ $ curl -H 'XXX:TEST_HEADER_VALUE' -H 'PARAMS_XXX:max_threads' 'http://localhost: max_alter_threads 2 ``` -!!! note "メモ" - 一つに ``、ワン `` のみをサポート `` 挿入タイプの。 +!!! note "注意" + 一つで `` 一つだけをサポート `` 挿入タイプ。 ## dynamic\_query\_handler {#dynamic_query_handler} -`` より `` 増加した `` . +で ``、クエリは、HTTP要求のparamの形式で書かれています。 違いは、 ``、クエリは設定ファイルに書き込まれます。 設定できます `` で ``. -ClickHouse抽出し実行値に対応する `` HTTPリクエストのurlの値。 -ClickHouseのデフォルト設定 `` は `/query` . これはオプションの設定です。 設定ファイルに定義がない場合、paramは渡されません。 +クリックハウスは、 `` HTTP要求のurlの値。 のデフォルト値 `` は `/query` . これはオプションの構成です。 設定ファイルに定義がない場合、paramは渡されません。 -この機能を試すために、この例では、max\_threadsとmax\_alter\_threadsの値を定義し、設定が正常に設定されたかどうかを照会します。 -違いは、 ``、クエリは、設定ファイルに書き込まれます。 しかし、 ``、クエリは、HTTP要求のparamの形で書かれています。 +この機能を試すために、この例ではmax\_threadsとmax\_alter\_threadsの値を定義し、設定が正常に設定されたかどうかを照会します。 -例えば: +例: ``` xml - - - - TEST_HEADER_VALUE_DYNAMIC - [^/]+)(/(?P[^/]+))?]]> - + + + + TEST_HEADER_VALUE_DYNAMIC + + dynamic_query_handler query_param - - + + + ``` ``` bash -$ curl -H 'XXX:TEST_HEADER_VALUE_DYNAMIC' -H 'PARAMS_XXX:max_threads' 'http://localhost:8123/?query_param=SELECT%20value%20FROM%20system.settings%20where%20name%20=%20%7Bname_1:String%7D%20OR%20name%20=%20%7Bname_2:String%7D&max_threads=1&max_alter_threads=2¶m_name_2=max_alter_threads' -1 -2 +$ curl -H 'XXX:TEST_HEADER_VALUE_DYNAMIC' 'http://localhost:8123/own?max_threads=1&max_alter_threads=2¶m_name_1=max_threads¶m_name_2=max_alter_threads&query_param=SELECT%20name,value%20FROM%20system.settings%20where%20name%20=%20%7Bname_1:String%7D%20OR%20name%20=%20%7Bname_2:String%7D' +max_threads 1 +max_alter_threads 2 +``` + +## 静的 {#static} + +`` 戻れる [content\_type](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type), [状態](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status) そしてresponse\_content。 response\_contentは、指定された内容を返すことができます + +例: + +メッセージを返す。 + +``` xml + + + GET + xxx + /hi + + static + 402 + text/html; charset=UTF-8 + Say Hi! + + + +``` + +``` bash +$ curl -vv -H 'XXX:xxx' 'http://localhost:8123/hi' +* Trying ::1... +* Connected to localhost (::1) port 8123 (#0) +> GET /hi HTTP/1.1 +> Host: localhost:8123 +> User-Agent: curl/7.47.0 +> Accept: */* +> XXX:xxx +> +< HTTP/1.1 402 Payment Required +< Date: Wed, 29 Apr 2020 03:51:26 GMT +< Connection: Keep-Alive +< Content-Type: text/html; charset=UTF-8 +< Transfer-Encoding: chunked +< Keep-Alive: timeout=3 +< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} +< +* Connection #0 to host localhost left intact +Say Hi!% +``` + +のコンテンツから構成送信します。 + +``` xml +
]]> + + + + GET + xxx + /get_config_static_handler + + static + config://get_config_static_handler + + + +``` + +``` bash +$ curl -v -H 'XXX:xxx' 'http://localhost:8123/get_config_static_handler' +* Trying ::1... +* Connected to localhost (::1) port 8123 (#0) +> GET /get_config_static_handler HTTP/1.1 +> Host: localhost:8123 +> User-Agent: curl/7.47.0 +> Accept: */* +> XXX:xxx +> +< HTTP/1.1 200 OK +< Date: Wed, 29 Apr 2020 04:01:24 GMT +< Connection: Keep-Alive +< Content-Type: text/plain; charset=UTF-8 +< Transfer-Encoding: chunked +< Keep-Alive: timeout=3 +< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} +< +* Connection #0 to host localhost left intact +
% +``` + +クライアントに送信するファイルから内容を検索します。 + +``` xml + + + GET + xxx + /get_absolute_path_static_handler + + static + text/html; charset=UTF-8 + file:///absolute_path_file.html + + + + GET + xxx + /get_relative_path_static_handler + + static + text/html; charset=UTF-8 + file://./relative_path_file.html + + + +``` + +``` bash +$ user_files_path='/var/lib/clickhouse/user_files' +$ sudo echo "Relative Path File" > $user_files_path/relative_path_file.html +$ sudo echo "Absolute Path File" > $user_files_path/absolute_path_file.html +$ curl -vv -H 'XXX:xxx' 'http://localhost:8123/get_absolute_path_static_handler' +* Trying ::1... +* Connected to localhost (::1) port 8123 (#0) +> GET /get_absolute_path_static_handler HTTP/1.1 +> Host: localhost:8123 +> User-Agent: curl/7.47.0 +> Accept: */* +> XXX:xxx +> +< HTTP/1.1 200 OK +< Date: Wed, 29 Apr 2020 04:18:16 GMT +< Connection: Keep-Alive +< Content-Type: text/html; charset=UTF-8 +< Transfer-Encoding: chunked +< Keep-Alive: timeout=3 +< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} +< +Absolute Path File +* Connection #0 to host localhost left intact +$ curl -vv -H 'XXX:xxx' 'http://localhost:8123/get_relative_path_static_handler' +* Trying ::1... +* Connected to localhost (::1) port 8123 (#0) +> GET /get_relative_path_static_handler HTTP/1.1 +> Host: localhost:8123 +> User-Agent: curl/7.47.0 +> Accept: */* +> XXX:xxx +> +< HTTP/1.1 200 OK +< Date: Wed, 29 Apr 2020 04:18:31 GMT +< Connection: Keep-Alive +< Content-Type: text/html; charset=UTF-8 +< Transfer-Encoding: chunked +< Keep-Alive: timeout=3 +< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} +< +Relative Path File +* Connection #0 to host localhost left intact ``` [元の記事](https://clickhouse.tech/docs/en/interfaces/http_interface/) diff --git a/docs/ja/interfaces/index.md b/docs/ja/interfaces/index.md index 0a40f3610be..bf93b4f8f66 100644 --- a/docs/ja/interfaces/index.md +++ b/docs/ja/interfaces/index.md @@ -1,29 +1,29 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_folder_title: Interfaces +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "\u30A4\u30F3\u30BF\u30FC" toc_priority: 14 -toc_title: "\u5C0E\u5165" +toc_title: "\u306F\u3058\u3081\u306B" --- -# 界面 {#interfaces} +# インター {#interfaces} -ﾂつｨﾂ姪“ﾂつ”ﾂ債ﾂづｭﾂつｹﾂづ債つｱﾂつｿﾂづｧﾂつｩﾂづｧ): +ClickHouseについての知識とネットワークインターフェイス（両方できる任意に包まれたTLSのための追加のセキュリティ): -- [HTTP](http.md)、文書化され、直接使いやすいです。 -- [ネイティブtcp](tcp.md) オーバーヘッドが少ない。 +- [HTTP](http.md)、文書化され、直接使用するのは簡単です。 +- [ネイティブTCP](tcp.md) より少ない間接費がある。 -るケースがほとんどでの使用をお勧めの適切なツールや図書館での交流と直結。 公式にサポートしよyandexは次のとお: +るケースがほとんどでの使用をお勧めの適切なツールや図書館での交流と直結。 公式にサポートしよYandexは次のとお: -- [コマンドライン](cli.md) +- [コマンド行クライアント](cli.md) - [JDBCドライバ](jdbc.md) - [ODBCドライバ](odbc.md) -- [C++クライアント](cpp.md) +- [C++クライアントライ](cpp.md) -また、clickhouseで作業するための幅広いサードパーティ製ライブラリもあります: +ClickHouseを操作するための幅広いサードパーティのライブラリもあります: - [クライアント](third-party/client-libraries.md) - [統合](third-party/integrations.md) -- [ビジュアル](third-party/gui.md) +- [視界面](third-party/gui.md) [元の記事](https://clickhouse.tech/docs/en/interfaces/) diff --git a/docs/ja/interfaces/jdbc.md b/docs/ja/interfaces/jdbc.md index 8ad05c955f6..6dc8f36c58d 100644 --- a/docs/ja/interfaces/jdbc.md +++ b/docs/ja/interfaces/jdbc.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 22 toc_title: "JDBC\u30C9\u30E9\u30A4\u30D0" --- @@ -9,7 +9,7 @@ toc_title: "JDBC\u30C9\u30E9\u30A4\u30D0" - **[公式ドライバー](https://github.com/ClickHouse/clickhouse-jdbc)** - サードパーティドライバ: - - [ﾂつｨﾂ姪“ﾂつ”ﾂ債ﾂつｹ](https://github.com/housepower/ClickHouse-Native-JDBC) + - [ClickHouse-Native-JDBC](https://github.com/housepower/ClickHouse-Native-JDBC) - [clickhouse4j](https://github.com/blynkkk/clickhouse4j) [元の記事](https://clickhouse.tech/docs/en/interfaces/jdbc/) diff --git a/docs/ja/interfaces/mysql.md b/docs/ja/interfaces/mysql.md index 701f3994132..49b20339f3e 100644 --- a/docs/ja/interfaces/mysql.md +++ b/docs/ja/interfaces/mysql.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 20 -toc_title: "MySQL\u30A4\u30F3" +toc_title: "MySQL\u30A4\u30F3\u30BF" --- -# MySQLイン {#mysql-interface} +# MySQLインタ {#mysql-interface} -ﾂつｨﾂ姪“ﾂつ”ﾂ債ﾂづｭﾂつｹ これは次の方法で有効にできます [mysql\_portgenericname](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-mysql_port) 設定ファイルでの設定: +ClickHouseはMySQL wire protocolをサポートしています。 で有効にすることができる [mysql\_port](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-mysql_port) 設定ファイルでの設定: ``` xml 9004 @@ -37,8 +37,8 @@ Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. mysql> ``` -すべてのmysqlクライアントとの互換性のために、ユーザーパスワードを [ダブルSHA1](../operations/settings/settings-users.md#password_double_sha1_hex) 構成ファイルで。 -ユー [SHA256](../operations/settings/settings-users.md#password_sha256_hex) いくつかのクライアントは認証できません（mysqljsと古いバージョンのコマンドラインツールmysql）。 +との互換性を維持するため、すべてのMySQLのお客様におすすめで指定ユーザのパスワード [ダブルSHA1](../operations/settings/settings-users.md#password_double_sha1_hex) 設定ファイル。 +場合は、ユーザのパスワードが指定 [SHA256](../operations/settings/settings-users.md#password_sha256_hex) 一部のクライアントは認証できません（mysqljsおよび古いバージョンのコマンドラインツールmysql）。 制限: diff --git a/docs/ja/interfaces/odbc.md b/docs/ja/interfaces/odbc.md index d316f760536..a40047eb04f 100644 --- a/docs/ja/interfaces/odbc.md +++ b/docs/ja/interfaces/odbc.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 23 toc_title: "ODBC\u30C9\u30E9\u30A4\u30D0" --- diff --git a/docs/ja/interfaces/tcp.md b/docs/ja/interfaces/tcp.md index d006d0da27a..6f7df838be6 100644 --- a/docs/ja/interfaces/tcp.md +++ b/docs/ja/interfaces/tcp.md @@ -1,13 +1,12 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 18 -toc_title: "\u30CD\u30A4\u30C6\u30A3\u30D6\u30A4\u30F3\u30BF\u30D5\u30A7\u30FC\u30B9\ - (TCP)" +toc_title: "\u30CD\u30A4\u30C6\u30A3\u30D6\u30A4)" --- -# ネイティブインタフェース(tcp) {#native-interface-tcp} +# ネイティブイ) {#native-interface-tcp} -ネイティブプロトコルは [コマンドライン](cli.md)、分散クエリ処理中のサーバー間通信のために、また、他のC++プログラムで。 残念ながら、ネイティブのClickHouseプロトコルには正式な仕様はまだありませんが、ClickHouseのソースコードから逆操作することができます（開始 [この辺りは](https://github.com/ClickHouse/ClickHouse/tree/master/dbms/Client) TCPトラフィックを傍受および分析することによって。 +ネイティブプロトコルは [コマンド行クライアント](cli.md)、分散クエリ処理中のサーバー間通信、および他のC++プログラムでも使用できます。 残念ながら、ネイティブClickHouseプロトコルは形式的仕様記述を利用できるバーからClickHouseソースコードを開始 [この辺り](https://github.com/ClickHouse/ClickHouse/tree/master/src/Client) や傍受し、分析すTCPます。 [元の記事](https://clickhouse.tech/docs/en/interfaces/tcp/) diff --git a/docs/ja/interfaces/third-party/client-libraries.md b/docs/ja/interfaces/third-party/client-libraries.md index b561e504f7c..4a68c25281b 100644 --- a/docs/ja/interfaces/third-party/client-libraries.md +++ b/docs/ja/interfaces/third-party/client-libraries.md @@ -1,59 +1,60 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 26 toc_title: "\u30AF\u30E9\u30A4\u30A2\u30F3\u30C8" --- -# お客様の図書館からのサードパーティー {#client-libraries-from-third-party-developers} +# サードパーテ {#client-libraries-from-third-party-developers} !!! warning "免責事項" - Yandexは **ない** を維持する図書館が以下の各号に記載されていない為にその質を評価する。 + Yandexのは **ない** 以下のライブラリを維持し、その品質を保証するための広範なテストを行っていません。 - Python - - [infi.clickhouse\_orm](https://github.com/Infinidat/infi.clickhouse_orm) - - [クリックハウスドライバ](https://github.com/mymarilyn/clickhouse-driver) - - [クリックハウス-顧客](https://github.com/yurial/clickhouse-client) + - [インフィclickhouse\_orm](https://github.com/Infinidat/infi.clickhouse_orm) + - [clickhouse-ドライバ](https://github.com/mymarilyn/clickhouse-driver) + - [clickhouse-クライアント](https://github.com/yurial/clickhouse-client) - [aiochclient](https://github.com/maximdanilchenko/aiochclient) - PHP - [smi2/phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse) - [8bitov/clickhouse-php-クライアント](https://packagist.org/packages/8bitov/clickhouse-php-client) - - [bozerkins/clickhouse-クライアント](https://packagist.org/packages/bozerkins/clickhouse-client) + - [ﾂつｨﾂ姪"ﾂつ"ﾂ債ﾂづｭﾂつｹ](https://packagist.org/packages/bozerkins/clickhouse-client) - [simpod/clickhouse-クライアント](https://packagist.org/packages/simpod/clickhouse-client) - - [seva-コード/php-クリック-ハウス-クライアント](https://packagist.org/packages/seva-code/php-click-house-client) + - [seva-code/php-click-house-client](https://packagist.org/packages/seva-code/php-click-house-client) - [SeasClick C++クライアント](https://github.com/SeasX/SeasClick) - 行け - [クリックハウス](https://github.com/kshvakov/clickhouse/) - [ゴークリックハウス](https://github.com/roistat/go-clickhouse) - - [mailrugo-clickhouse](https://github.com/mailru/go-clickhouse) - - [golang-clickhouse](https://github.com/leprosus/golang-clickhouse) + - [メールルーゴ-クリックハウス](https://github.com/mailru/go-clickhouse) + - [ゴラン-クリックハウス](https://github.com/leprosus/golang-clickhouse) - NodeJs - - [clickhouse(NodeJs)](https://github.com/TimonKK/clickhouse) - - [ノードクリックハウス](https://github.com/apla/node-clickhouse) + - [クリックハウス(曖昧さ回避)](https://github.com/TimonKK/clickhouse) + - [ノード-クリックハウス](https://github.com/apla/node-clickhouse) - Perl - - [perl-DBD-ClickHouse](https://github.com/elcamlost/perl-DBD-ClickHouse) - - [HTTP-ClickHouse](https://metacpan.org/release/HTTP-ClickHouse) - - [AnyEvent-ClickHouse](https://metacpan.org/release/AnyEvent-ClickHouse) + - [perl-DBD-クリックハウス](https://github.com/elcamlost/perl-DBD-ClickHouse) + - [HTTP-クリックハウス](https://metacpan.org/release/HTTP-ClickHouse) + - [AnyEvent-クリックハウス](https://metacpan.org/release/AnyEvent-ClickHouse) - Ruby - - [クリックハウス(ruby)](https://github.com/shlima/click_house) - - [clickhouse-activerecord](https://github.com/PNixx/clickhouse-activerecord) + - [クリックハウス(Ruby)](https://github.com/shlima/click_house) + - [クリックハウス-activerecord](https://github.com/PNixx/clickhouse-activerecord) - R - [クリックハウス-r](https://github.com/hannesmuehleisen/clickhouse-r) - - [Rクリックハウス](https://github.com/IMSMWU/RClickhouse) + - [RClickHouse](https://github.com/IMSMWU/RClickHouse) - Java - [clickhouse-クライアント-java](https://github.com/VirtusAI/clickhouse-client-java) - - [クリックハウス-顧客](https://github.com/Ecwid/clickhouse-client) + - [clickhouse-クライアント](https://github.com/Ecwid/clickhouse-client) - Scala - [clickhouse-scala-クライアント](https://github.com/crobox/clickhouse-scala-client) -- Kotlin +- コトリン - [AORM](https://github.com/TanVD/AORM) - C\# - - [クリックハウス。ado](https://github.com/killwort/ClickHouse-Net) - - [クリックハウス。お客様](https://github.com/DarkWanderer/ClickHouse.Client) + - [クリックハウスAdo](https://github.com/killwort/ClickHouse-Net) + - [クリックハウスクライアン](https://github.com/DarkWanderer/ClickHouse.Client) - [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net) - エリクサー - - [clickhousex](https://github.com/appodeal/clickhousex/) + - [クリックハウス](https://github.com/appodeal/clickhousex/) + - [柱](https://github.com/sofakingworld/pillar) - Nim - - [nim-clickhouse](https://github.com/leonardoce/nim-clickhouse) + - [ニム-クリックハウス](https://github.com/leonardoce/nim-clickhouse) [元の記事](https://clickhouse.tech/docs/en/interfaces/third-party/client_libraries/) diff --git a/docs/ja/interfaces/third-party/gui.md b/docs/ja/interfaces/third-party/gui.md index 46deefb7919..13b178984d3 100644 --- a/docs/ja/interfaces/third-party/gui.md +++ b/docs/ja/interfaces/third-party/gui.md @@ -1,53 +1,53 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 28 -toc_title: "\u30D3\u30B8\u30E5\u30A2\u30EB" +toc_title: "\u8996\u754C\u9762" --- -# サードパー {#visual-interfaces-from-third-party-developers} +# サードパーテ {#visual-interfaces-from-third-party-developers} ## オープンソース {#open-source} ### Tabix {#tabix} -のclickhouseのための網インターフェイス [Tabix](https://github.com/tabixio/tabix) プロジェクト。 +のClickHouseのためのWebインターフェイス [Tabix](https://github.com/tabixio/tabix) プロジェクト 特徴: -- 追加のソフトウェアをインストールする必要なく、ブラウザから直接clickhouseで動作します。 -- 構文強調表示のクエリエディター。 +- 追加のソフトウェアをインストールする必要なしに、ブラウザから直接ClickHouseで動作します。 +- 構文の強調表示とクエリエディタ。 - コマンドの自動補完。 -- クエリ実行のグラフ分析のためのツール。 +- ツールのためのグラフィカルに分析クエリを実行します。 - 配色オプション。 [Tabixドキュメント](https://tabix.io/doc/). -### HouseOps {#houseops} +### ハウスオップ {#houseops} -[HouseOps](https://github.com/HouseOps/HouseOps) OSX、Linux、Windows用のUI/IDEです。 +[ハウスオップ](https://github.com/HouseOps/HouseOps) OSX、Linux、Windows用のUI/IDEです。 特徴: -- 構文の強調表示を使用したクエリビルダー。 テーブルまたはjsonビューで応答を表示します。 +- 構文の強調表示とクエリビルダー。 テーブルまたはJSONビューで応答を表示します。 - CSVまたはJSONとしてエクスポートクエリ結果。 - 説明付きのプロセスのリスト。 書き込みモード。 停止する能力 (`KILL`）プロセス。 -- データベースグラフ すべてのテーブルとその列に追加情報を表示します。 +- データベー すべてのテーブルとその列に追加情報が表示されます。 - 列サイズのクイックビュー。 - サーバー構成。 -次の機能は、開発のために計画されています: +以下の機能の開発が計画されています: - データベース管理。 - ユーザー管理。 -- リアルタイムデータ分析。 -- クラスタ監視。 -- クラスター管理。 -- 監視レプリケートおよびkafkaテーブル。 +- リアルタイムデータ解析。 +- クラスター監視。 +- クラスタ管理。 +- 複製されたテーブルとカフカテーブルの監視。 ### 灯台 {#lighthouse} -[灯台](https://github.com/VKCOM/lighthouse) ClickHouseのための軽量のwebインターフェイスです。 +[灯台](https://github.com/VKCOM/lighthouse) ClickHouseのための軽量なwebインターフェイスです。 特徴: @@ -55,97 +55,101 @@ toc_title: "\u30D3\u30B8\u30E5\u30A2\u30EB" - テーブルのプレビューとフィルタです。 - 読み取り専用クエリの実行。 -### Redash {#redash} +### レダッシュ {#redash} -[Redash](https://github.com/getredash/redash) めるためのプラットフォームのデータを可視化する。 +[レダッシュ](https://github.com/getredash/redash) めるためのプラットフォームのデータを可視化する。 -サポート、多数のデータソースを含むclickhouse,redash参加できる結果のクエリからデータソースへの最終データセットである。 +サポート、多数のデータソースを含むClickHouse,Redash参加できる結果のクエリからデータソースへの最終データセットである。 特徴: - クエリの強力なエディタ。 -- データベ -- 視覚化ツールを使用すると、さまざまな形式のデータを表現できます。 +- エクスプローラ +- さまざまな形でデータを表現できる視覚化ツール。 -### デービーバーname {#dbeaver} +### DBeaver {#dbeaver} -[デービーバーname](https://dbeaver.io/) -ユニバーサルデスクトップのデータベースのクライアントClickHouseます。 +[DBeaver](https://dbeaver.io/) -ユニバーサルデスクトップのデータベースのクライアントClickHouseます。 特徴: -- 構文ハイライトと自動補完によるクエリ開発。 +- 構文の強調表示と自動補完によるクエリ開発。 - テーブルリフィルとメタデータを検索する - 表データプレビュー。 -- フルテキスト検索。 +- 全文検索。 -### クリックハウス-cli {#clickhouse-cli} +### clickhouse-cli {#clickhouse-cli} -[クリックハウス-cli](https://github.com/hatarist/clickhouse-cli) Python3で書かれたClickHouseの代替コマンドラインクライアントです。 +[clickhouse-cli](https://github.com/hatarist/clickhouse-cli) Python3で書かれたClickHouseの代替コマンドラインクライアントです。 特徴: - 自動補完。 - クエリとデータ出力の構文強調表示。 -- データ出力のためのポケベルサポート。 -- カスタムpostgresqlのようなコマンド。 +- データ出力のためのポケットベルサポート。 +- カスタムPostgreSQLのようなコマンド。 -### clickhouse-flamegraph {#clickhouse-flamegraph} +### クリックハウス-フラメグラフ {#clickhouse-flamegraph} -[clickhouse-flamegraph](https://github.com/Slach/clickhouse-flamegraph) 視覚化する専門にされた用具はある `system.trace_log` として [flamegraph](http://www.brendangregg.com/flamegraphs.html). +[クリックハウス-フラメグラフ](https://github.com/Slach/clickhouse-flamegraph) このツールは `system.trace_log` として [フラメグラフ](http://www.brendangregg.com/flamegraphs.html). + +### クリックハウス-プランタム {#clickhouse-plantuml} + +[チックハウス-プランタム](https://pypi.org/project/clickhouse-plantuml/) 生成するスクリプトです [プランタム](https://plantuml.com/) テーブルのスキームの図。 ## 商業 {#commercial} -### データグリップ {#datagrip} +### DataGrip {#datagrip} -[データグリップ](https://www.jetbrains.com/datagrip/) JetbrainsのデータベースIDEで、ClickHouse専用サポートがあります。 PyCharm、IntelliJ IDEA、GoLand、PhpStormなどの他のIntelliJベースのツールにも組み込まれています。 +[DataGrip](https://www.jetbrains.com/datagrip/) ClickHouse専用のサポートを持つJetBrainsのデータベースIDEです。 PyCharm、IntelliJ IDEA、GoLand、PhpStormなど、他のIntelliJベースのツールにも埋め込まれています。 特徴: - 非常に高速なコード補完。 - ClickHouse構文の強調表示。 -- ClickHouse固有の機能のサポート。 +- ネストされた列、テーブルエンジンなど、ClickHouse固有の機能のサポート。 - データエディタ。 - リファクタリング。 - 検索とナビゲーション。 -### YandexのDataLens {#yandex-datalens} +### Yandexデータレンズ {#yandex-datalens} -[YandexのDataLens](https://cloud.yandex.ru/services/datalens) データの可視化と分析のサービスです。 +[Yandexデータレンズ](https://cloud.yandex.ru/services/datalens) データの可視化と分析のサービスです。 特徴: - シンプルな棒グラフから複雑なダッシュボードまで、幅広い視覚化が可能です。 -- ダッシュボードは一般公開されます。 +- ダッシュボードを公開できます。 - ClickHouseを含む複数のデータソースのサポート。 -- ClickHouseに基づく具体化されたデータのための貯蔵。 +- ClickHouseに基づく実体化されたデータのストレージ。 -データレンスは [無料で利用可能](https://cloud.yandex.com/docs/datalens/pricing) 商業使用のための低負荷プロジェクトのため。 +データレンズは [自由のために利用できる](https://cloud.yandex.com/docs/datalens/pricing) 商業使用の低負荷プロジェクトのため。 -- [DataLens書](https://cloud.yandex.com/docs/datalens/). -- [Tutorial](https://cloud.yandex.com/docs/solutions/datalens/data-from-ch-visualization) ClickHouseデータベースからデータを視覚化する。 +- [DataLensドキュメント](https://cloud.yandex.com/docs/datalens/). +- [チュートリ](https://cloud.yandex.com/docs/solutions/datalens/data-from-ch-visualization) 上の可視化するデータからClickHouseデータベースです。 ### Holisticsソフトウェア {#holistics-software} -[ホリスティクス](https://www.holistics.io/) フルスタックのデータプラットフォームは、ビジネスインツールです。 +[ホリスティック](https://www.holistics.io/) フルスタックのデータプラットフォームは、ビジネスインツールです。 特徴: -- 自動メール、slackやグーグルシートのスケジュール。 -- SQLエディタと可視化、バージョン管理の自動完了し、再利用可能なクエリー部品、ダイナミックフィルター. -- Iframe経由のレポートとダッシュボードの埋め込み分析。 -- データ準備およびetl機能。 +- 自動メール、Slackやグーグルシートのスケジュール。 +- Visualizations、バージョン管理、自動補完、再利用可能なクエリコンポーネントと動的フィルタとSQLエディタ。 +- Iframeによるレポートとダッシュボードの組み込み分析。 +- データ準備とETL機能。 - SQLデータモデリング支援のためのリレーショナルマッピングのデータです。 -### 見物人 {#looker} +### ルッカー {#looker} -[見物人](https://looker.com) ClickHouseを含む50以上のデータベース方言をサポートするdata platform and business intelligenceツールです。 LookerはSaaSプラットフォームとして利用でき、セルフホスト型です。 ユーザーが利用できLookerる場合は、vpnクライアントの直接探索、データの構築の可視化とダッシュボード、スケジュール、識農場管理について学んでいます。 Lookerのツールを埋め込むためのチャプターでは、これらの機能の他のアプリケーション、およびAPI +[ルッカー](https://looker.com) はデータプラットフォームは、ビジネスインツールをサポート50+データベースの方言を含むClickHouse. LookerはSaaSプラットフォームとして利用可能で、セルフホスト型です。 ユーザーが利用できLookerる場合は、vpnクライアントの直接探索、データの構築の可視化とダッシュボード、スケジュール、識農場管理について学んでいます。 Lookerのツールを埋め込むためのチャプターでは、これらの機能の他のアプリケーション、およびAPI 統合データを、他のアプリケーション 特徴: -- 簡単-アジャイル開発をlookml、言語に対応したキュレーション - [データモデル](https://looker.com/platform/data-modeling) レポート作成者とエンドユーザーをサポートする。 -- Lookerのを経由して強力なワークフローの統合 [データ操作](https://looker.com/platform/actions). +- LookMLを使った簡単でアジャイルな開発 + [データモデル化](https://looker.com/platform/data-modeling) レポート作成者とエンドユーザーをサポートする。 +- 見物人による強力なワークフローの統合 [データ操作](https://looker.com/platform/actions). [LookerでClickHouseを設定する方法。](https://docs.looker.com/setup-and-management/database-config/clickhouse) diff --git a/docs/ja/interfaces/third-party/index.md b/docs/ja/interfaces/third-party/index.md index 5b94c79ad35..1956e6caf8f 100644 --- a/docs/ja/interfaces/third-party/index.md +++ b/docs/ja/interfaces/third-party/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_folder_title: Third-Party +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "\u30B5\u30FC\u30C9\u30D1\u30FC\u30C6\u30A3" toc_priority: 24 --- diff --git a/docs/ja/interfaces/third-party/integrations.md b/docs/ja/interfaces/third-party/integrations.md index dbd0054fab3..59876cd63a0 100644 --- a/docs/ja/interfaces/third-party/integrations.md +++ b/docs/ja/interfaces/third-party/integrations.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 27 toc_title: "\u7D71\u5408" --- @@ -8,88 +8,91 @@ toc_title: "\u7D71\u5408" # サードパーティ開発者からの統合ライブラリ {#integration-libraries-from-third-party-developers} !!! warning "免責事項" - Yandexは **ない** 以下に示すツールとライブラリを維持し、その品質を保証するための広範なテストを行っていません。 + Yandexのは **ない** 以下のツールとライブラリを維持し、その品質を確保するための広範なテストを行っていません。 ## インフラ製品 {#infrastructure-products} - リレーショナルデータベース管理システム - [MySQL](https://www.mysql.com) - - [Proxysqlcomment](https://github.com/sysown/proxysql/wiki/ClickHouse-Support) - - [clickhouse-mysql-data-reader](https://github.com/Altinity/clickhouse-mysql-data-reader) - - [horgh-replicator](https://github.com/larsnovikov/horgh-replicator) + - [ProxySQL](https://github.com/sysown/proxysql/wiki/ClickHouse-Support) + - [clickhouse-mysql-データリーダー](https://github.com/Altinity/clickhouse-mysql-data-reader) + - [horgh-レプリケーター](https://github.com/larsnovikov/horgh-replicator) - [PostgreSQL](https://www.postgresql.org) - [clickhousedb\_fdw](https://github.com/Percona-Lab/clickhousedb_fdw) - - [infi.clickhouse\_fdw](https://github.com/Infinidat/infi.clickhouse_fdw) (用途 [infi.clickhouse\_orm](https://github.com/Infinidat/infi.clickhouse_orm)) + - [インフィclickhouse\_fdw](https://github.com/Infinidat/infi.clickhouse_fdw) （用途 [インフィclickhouse\_orm](https://github.com/Infinidat/infi.clickhouse_orm)) - [pg2ch](https://github.com/mkabilov/pg2ch) - [clickhouse\_fdw](https://github.com/adjust/clickhouse_fdw) - [MSSQL](https://en.wikipedia.org/wiki/Microsoft_SQL_Server) - [ClickHouseMigrator](https://github.com/zlzforever/ClickHouseMigrator) - メッセージキュ - - [カフカname](https://kafka.apache.org) - - [clickhouse\_sinker](https://github.com/housepower/clickhouse_sinker) (用途 [クライアントへ](https://github.com/kshvakov/clickhouse/)) -- オブジェクト保存 + - [カフカ](https://kafka.apache.org) + - [clickhouse\_sinker](https://github.com/housepower/clickhouse_sinker) （用途 [Goクライアント](https://github.com/ClickHouse/clickhouse-go/)) +- ストリーム処理 + - [フリンク](https://flink.apache.org) + - [フリンク-クリックハウス-シンク](https://github.com/ivi-ru/flink-clickhouse-sink) +- オブジェクト保管 - [S3](https://en.wikipedia.org/wiki/Amazon_S3) - [clickhouse-バックアップ](https://github.com/AlexAkulov/clickhouse-backup) - 容器の協奏 - [Kubernetes](https://kubernetes.io) - - [クリックハウス-演算子](https://github.com/Altinity/clickhouse-operator) + - [clickhouse-演算子](https://github.com/Altinity/clickhouse-operator) - 構成管理 - - [人形](https://puppet.com) - - [innogames/clickhouse](https://forge.puppet.com/innogames/clickhouse) - - [mfedotov/clickhouse](https://forge.puppet.com/mfedotov/clickhouse) + - [パペット](https://puppet.com) + - [イノゲームズ/クリックハウス](https://forge.puppet.com/innogames/clickhouse) + - [mfedotov/クリックハウス](https://forge.puppet.com/mfedotov/clickhouse) - 監視 - [黒鉛](https://graphiteapp.org) - [グラファウス](https://github.com/yandex/graphouse) - [カーボンクリックハウス](https://github.com/lomik/carbon-clickhouse) + - - [黒鉛-clickhouse](https://github.com/lomik/graphite-clickhouse) - - [graphite-ch-optimizer](https://github.com/innogames/graphite-ch-optimizer) -staledパーティションを最適化します。 [\*グラフィットマージツリー](../../engines/table-engines/mergetree-family/graphitemergetree.md#graphitemergetree) からのルールの場合 [ロールアップ構成](../../engines/table-engines/mergetree-family/graphitemergetree.md#rollup-configuration) 適用できます + - [グラファイト-クリック](https://github.com/lomik/graphite-clickhouse) + - [黒鉛-ch-オプティマイザー](https://github.com/innogames/graphite-ch-optimizer) -staled仕切りを最大限に活用する [\*GraphiteMergeTree](../../engines/table-engines/mergetree-family/graphitemergetree.md#graphitemergetree) からのルールの場合 [ロールアップ構成](../../engines/table-engines/mergetree-family/graphitemergetree.md#rollup-configuration) 応用できます - [グラファナ](https://grafana.com/) - [クリックハウス-グラファナ](https://github.com/Vertamedia/clickhouse-grafana) - [プロメテウス](https://prometheus.io/) - [clickhouse\_exporter](https://github.com/f1yegor/clickhouse_exporter) - [プロムハウス](https://github.com/Percona-Lab/PromHouse) - - [clickhouse\_exporter](https://github.com/hot-wifi/clickhouse_exporter) (用途 [クライアントへ](https://github.com/kshvakov/clickhouse/)) - - [Nagios](https://www.nagios.org/) + - [clickhouse\_exporter](https://github.com/hot-wifi/clickhouse_exporter) （用途 [Goクライアント](https://github.com/kshvakov/clickhouse/)) + - [ナギオス](https://www.nagios.org/) - [check\_clickhouse](https://github.com/exogroup/check_clickhouse/) - [check\_clickhouse.py](https://github.com/innogames/igmonplugins/blob/master/src/check_clickhouse.py) - [Zabbix](https://www.zabbix.com) - [clickhouse-zabbix-テンプレート](https://github.com/Altinity/clickhouse-zabbix-template) - - [Sematextgenericname](https://sematext.com/) - - [クリックハウス統合](https://github.com/sematext/sematext-agent-integrations/tree/master/clickhouse) -- ログ記録 + - [Sematext](https://sematext.com/) + - [clickhouseの統合](https://github.com/sematext/sematext-agent-integrations/tree/master/clickhouse) +- ロギング - [rsyslog](https://www.rsyslog.com/) - - [omclickhouse](https://www.rsyslog.com/doc/master/configuration/modules/omclickhouse.html) - - [fluentd](https://www.fluentd.org) + - [オムクリックハウス](https://www.rsyslog.com/doc/master/configuration/modules/omclickhouse.html) + - [フルエント](https://www.fluentd.org) - [ログハウス](https://github.com/flant/loghouse) （のために [Kubernetes](https://kubernetes.io)) - [logagent](https://www.sematext.com/logagent) - - [logagent出力-プラグイン-clickhouse](https://sematext.com/docs/logagent/output-plugin-clickhouse/) -- ジオ + - [logagent output-plugin-clickhouse](https://sematext.com/docs/logagent/output-plugin-clickhouse/) +- Geo - [MaxMind](https://dev.maxmind.com/geoip/) - [クリックハウス-maxmind-geoip](https://github.com/AlexeyKupershtokh/clickhouse-maxmind-geoip) -## プログラミング言語の生態系 {#programming-language-ecosystems} +## プログラミ {#programming-language-ecosystems} - Python - [SQLAlchemy](https://www.sqlalchemy.org) - - [sqlalchemy-clickhouse](https://github.com/cloudflare/sqlalchemy-clickhouse) (用途 [infi.clickhouse\_orm](https://github.com/Infinidat/infi.clickhouse_orm)) + - [sqlalchemy-クリックハウス](https://github.com/cloudflare/sqlalchemy-clickhouse) （用途 [インフィclickhouse\_orm](https://github.com/Infinidat/infi.clickhouse_orm)) - [パンダ](https://pandas.pydata.org) - [パンダハウス](https://github.com/kszucs/pandahouse) - PHP - - [Doctrine](https://www.doctrine-project.org/) - - [dbal-clickhouse](https://packagist.org/packages/friendsofdoctrine/dbal-clickhouse) + - [教義](https://www.doctrine-project.org/) + - [dbal-クリックハウス](https://packagist.org/packages/friendsofdoctrine/dbal-clickhouse) - R - [dplyr](https://db.rstudio.com/dplyr/) - - [Rクリックハウス](https://github.com/IMSMWU/RClickhouse) (用途 [クリックハウス-cpp](https://github.com/artpaul/clickhouse-cpp)) + - [RClickHouse](https://github.com/IMSMWU/RClickHouse) （用途 [クリックハウス-cpp](https://github.com/artpaul/clickhouse-cpp)) - Java - [Hadoop](http://hadoop.apache.org) - - [クリックハウス-hdfs-ローダー](https://github.com/jaykelin/clickhouse-hdfs-loader) (用途 [JDBC](../../sql-reference/table-functions/jdbc.md)) + - [clickhouse-hdfs-loader](https://github.com/jaykelin/clickhouse-hdfs-loader) （用途 [JDBC](../../sql-reference/table-functions/jdbc.md)) - Scala - - [Akka](https://akka.io) + - [アッカ](https://akka.io) - [clickhouse-scala-クライアント](https://github.com/crobox/clickhouse-scala-client) - C\# - [ADO.NET](https://docs.microsoft.com/en-us/dotnet/framework/data/adonet/ado-net-overview) - - [クリックハウス。ado](https://github.com/killwort/ClickHouse-Net) - - [クリックハウス。お客様](https://github.com/DarkWanderer/ClickHouse.Client) + - [クリックハウスAdo](https://github.com/killwort/ClickHouse-Net) + - [クリックハウスクライアン](https://github.com/DarkWanderer/ClickHouse.Client) - [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net) - [ClickHouse.Net.Migrations](https://github.com/ilyabreev/ClickHouse.Net.Migrations) - エリクサー diff --git a/docs/ja/interfaces/third-party/proxy.md b/docs/ja/interfaces/third-party/proxy.md index 7305a8e7f22..e817b743787 100644 --- a/docs/ja/interfaces/third-party/proxy.md +++ b/docs/ja/interfaces/third-party/proxy.md @@ -1,39 +1,39 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 29 toc_title: "\u30D7\u30ED\u30AD\u30B7" --- -# サードパーティ開発者のプロキシサーバー {#proxy-servers-from-third-party-developers} +# サードパーテ {#proxy-servers-from-third-party-developers} ## chproxy {#chproxy} -[chproxy](https://github.com/Vertamedia/chproxy)、ClickHouseデータベース用のHTTPプロキシとロードバランサです。 +[chproxy](https://github.com/Vertamedia/chproxy),は、ClickHouseデータベース用のHTTPプロキシとロードバランサです。 特徴: -- ユーザー単位のルーティ -- 柔軟な制限。 -- 自動ssl証明書の更新。 +- ユーザーごとのルーティ +- 適用範囲が広い限界。 +- SSL証明書の自動renewal。 -Goで実装されています。 +Goで実装。 -## KittenHouse {#kittenhouse} +## キッテンハウス {#kittenhouse} -[KittenHouse](https://github.com/VKCOM/kittenhouse) することが重要である現地代理人とClickHouse、アプリケーションサーバの場合でもバッファに挿入データとお客様側です。 +[キッテンハウス](https://github.com/VKCOM/kittenhouse) することが重要である現地代理人とClickHouse、アプリケーションサーバの場合でもバッファに挿入データとお客様側です。 特徴: - メモリ内およびディスク上のデータバッファリング。 - テーブル単位のルーティング。 -- 負荷分散とヘルスチェック。 +- 負荷分散と正常性チェック。 -Goで実装されています。 +Goで実装。 ## クリックハウス-バルク {#clickhouse-bulk} -[クリックハウス-バルク](https://github.com/nikepan/clickhouse-bulk) 簡単なClickHouseの挿入物のコレクターはある。 +[クリックハウス-バルク](https://github.com/nikepan/clickhouse-bulk) 単純なClickHouse挿入コレクターです。 特徴: @@ -41,6 +41,6 @@ Goで実装されています。 - 複数のリモートサーバー。 - 基本認証。 -Goで実装されています。 +Goで実装。 [元の記事](https://clickhouse.tech/docs/en/interfaces/third-party/proxy/) diff --git a/docs/ja/introduction/adopters.md b/docs/ja/introduction/adopters.md index ffc938c9eb9..084b5034a62 100644 --- a/docs/ja/introduction/adopters.md +++ b/docs/ja/introduction/adopters.md @@ -1,82 +1,86 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 8 -toc_title: "\u30A2\u30FC\u30EA\u30FC\u30A2\u30C0\u30D7\u30BF\u30FC" +toc_title: "\u30A2\u30C0\u30D7\u30BF\u30FC" --- -# ClickHouseアーリーアダプター {#clickhouse-adopters} +# ﾂつｨﾂ姪"ﾂ債ﾂつｹ {#clickhouse-adopters} !!! warning "免責事項" - 以下のリストを使用している企業のclickhouseとその成功を組み立てから共に、これとは異なる場合がござ流を実現しました。 あなたの会社でclickhouseを採用するという話を共有していただければ幸いです [リストに追加します](https://github.com/ClickHouse/ClickHouse/edit/master/docs/en/introduction/adopters.md) しかし、そうすることによってNDAの問題がないことを確認してください。 他の会社からの出版物との更新を提供することはまた有用である。 + 以下のリストを使用している企業のClickHouseとその成功を組み立てから共に、これとは異なる場合がござ流を実現しました。 あなたの会社でClickHouseを採用するという話を共有していただければ幸いです [リストに追加する](https://github.com/ClickHouse/ClickHouse/edit/master/docs/en/introduction/adopters.md) しかし、そうすることでNDAの問題がないことを確認してください。 他の会社からの出版物との更新を提供することはまた有用である。 -| 会社 | 産業 | Usecase | 集りのサイズ | (Un)圧縮データサイズ\* | 参照 | -|-----------------------------------------------------------------------|------------------------------|----------------------|-----------------------------------------------|--------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [2gis](https://2gis.ru) | マップ | 監視 | — | — | [トークでロシア、月2019](https://youtu.be/58sPkXfq6nw) | -| [Aloha Browser](https://alohabrowser.com/) | モバイルアプリ | ブラウザバックエンド | — | — | [ロシア語のスライド、2019](https://github.com/yandex/clickhouse-presentations/blob/master/meetup22/aloha.pdf) | -| [アマデウス](https://amadeus.com/) | 旅行 | Analytics | — | — | [プレスリリース、april2018](https://www.altinity.com/blog/2018/4/5/amadeus-technologies-launches-investment-and-insights-tool-based-on-machine-learning-and-strategy-algorithms) | -| [Appsflyer](https://www.appsflyer.com) | モバイル分析 | 主な製品 | — | — | [トークでロシア、月2019](https://www.youtube.com/watch?v=M3wbRlcpBbY) | -| [ArenaData](https://arenadata.tech/) | データ基盤 | 主な製品 | — | — | [スライドでロシア、december2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup38/indexes.pdf) | -| [Badoo](https://badoo.com) | デート | Timeseries | — | — | [スライドでロシア、december2019](https://presentations.clickhouse.tech/meetup38/forecast.pdf) | -| [ベノク](https://www.benocs.com/) | ネットワークテレメトリと分析 | 主な製品 | — | — | [英語でのスライド,十月2017](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup9/lpm.pdf) | -| [ブルームバーグ](https://www.bloomberg.com/) | 金融、メディア | 監視 | 102サーバ | — | [スライド,もっと2018](https://www.slideshare.net/Altinity/http-analytics-for-6m-requests-per-second-using-clickhouse-by-alexander-bocharov) | -| [Bloxy](https://bloxy.info) | ブロック鎖 | Analytics | — | — | [ロシア語のスライド、august2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/4_bloxy.pptx) | -| `Dataliance/UltraPower` | テレコム | Analytics | — | — | [スライドで中国、月2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup12/telecom.pdf) | -| [CARTO](https://carto.com/) | ビジネス情報 | ジオ分析 | — | — | [クリックハウスによる地理空間処理](https://carto.com/blog/geospatial-processing-with-clickhouse/) | -| [CERN](http://public.web.cern.ch/public/) | 研究 | 実験 | — | — | [プレスリリース、april2012](https://www.yandex.com/company/press_center/press_releases/2012/2012-04-10/) | -| [Cisco](http://cisco.com/) | ネットワーク | トラフィック分析 | — | — | [雷トーク、月2019](https://youtu.be/-hI1vDR2oPY?t=5057) | -| [シタデル証券](https://www.citadelsecurities.com/) | 金融 | — | — | — | [貢献、月2019](https://github.com/ClickHouse/ClickHouse/pull/4774) | -| [シティモービルcity in germany](https://city-mobil.ru) | タクシー | Analytics | — | — | [ロシアのブログ記事、march2020](https://habr.com/en/company/citymobil/blog/490660/) | -| [コンテンツスクエア](https://contentsquare.com) | ウェブ分析 | 主な製品 | — | — | [フランス語でのブログ記事,十一月2018](http://souslecapot.net/2018/11/21/patrick-chatain-vp-engineering-chez-contentsquare-penser-davantage-amelioration-continue-que-revolution-constante/) | -| [Cloudflare](https://cloudflare.com) | CDN | トラフィック分析 | 36サーバ | — | [ブログ記事,もっと2017](https://blog.cloudflare.com/how-cloudflare-analyzes-1m-dns-queries-per-second/), [ブログ記事,行進2018](https://blog.cloudflare.com/http-analytics-for-6m-requests-per-second-using-clickhouse/) | -| [コルネ](https://coru.net/) | Analytics | 主な製品 | — | — | [英語のスライド、april2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup21/predictive_models.pdf) | -| [CraiditX 氪信](https://creditx.com) | ファイナンスai | 分析 | — | — | [スライドで英語、月2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/udf.pptx) | -| [認定マーク/storetail](https://www.criteo.com/) | 小売り | 主な製品 | — | — | [英語でのスライド,十月2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/3_storetail.pptx) | -| [ドイツ銀行](https://db.com) | 金融 | BI分析 | — | — | [スライドで英語、月2019](https://bigdatadays.ru/wp-content/uploads/2019/10/D2-H3-3_Yakunin-Goihburg.pdf) | -| [ディーバ-e](https://www.diva-e.com) | デジタルコンサル | 主な製品 | — | — | [英語でのスライド、2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup29/ClickHouse-MeetUp-Unusual-Applications-sd-2019-09-17.pdf) | -| [Exness](https://www.exness.com) | 取引 | 指標、ロギング | — | — | [ロシア語で話す、2019](https://youtu.be/_rpU-TvSfZ8?t=3215) | -| [ジュニエ](https://geniee.co.jp) | 広告ネットワーク | 主な製品 | — | — | [日本語でのブログ記事,七月2017](https://tech.geniee.co.jp/entry/2017/07/20/160100) | -| [HUYA](https://www.huya.com/) | ビデオ流出 | Analytics | — | — | [スライドで中国、月2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/7.%20ClickHouse万亿数据分析实践%20李本旺(sundy-li)%20虎牙.pdf) | -| [Idealista](https://www.idealista.com) | 不動産 | Analytics | — | — | [英語でのブログ記事,四月2019](https://clickhouse.yandex/blog/en/clickhouse-meetup-in-madrid-on-april-2-2019) | -| [インフォビスタ](https://www.infovista.com/) | ネットワーク | Analytics | — | — | [スライドで英語、月2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup30/infovista.pdf) | -| [InnoGames](https://www.innogames.com) | ゲーム | 指標、ロギング | — | — | [スライドでロシア、2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/graphite_and_clickHouse.pdf) | -| [インテグロスcolor](https://integros.com) | Platformビデオサービス | Analytics | — | — | [ロシア語のスライド、2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | -| [Kodiakデータ](https://www.kodiakdata.com/) | 雲 | 主な製品 | — | — | [Engishでスライド,四月2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup13/kodiak_data.pdf) | -| [Konturgenericname](https://kontur.ru) | ソフトウェア開発 | 指標 | — | — | [トークでロシア、月2018](https://www.youtube.com/watch?v=U4u4Bd0FtrY) | -| [LifeStreet](https://lifestreet.com/) | 広告ネットワーク | 主な製品 | 75サーバー(レプリカ3) | 5.27PiB | [ブログ記事でロシア、月2017](https://habr.com/en/post/322620/) | -| [Mail.ru クラウドソリューション](https://mcs.mail.ru/) | クラウドサービス | 主な製品 | — | — | [ロシア語でのclickhouseインスタンスの実行](https://mcs.mail.ru/help/db-create/clickhouse#) | -| [MessageBird](https://www.messagebird.com) | 電気通信 | 統計 | — | — | [スライドで英語、月2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup20/messagebird.pdf) | -| [MGID](https://www.mgid.com/) | 広告ネットワーク | ウェブ分析 | — | — | [ロシア語での分析dbms clickhouseの実装経験](http://gs-studio.com/news-about-it/32777----clickhouse---c) | -| [OneAPM](https://www.oneapm.com/) | モニタリングとデータ解析 | 主な製品 | — | — | [スライドで中国、月2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/8.%20clickhouse在OneAPM的应用%20杜龙.pdf) | -| [Pragma革新](http://www.pragma-innovation.fr/) | テレメトリとビッグデータ分析 | 主な製品 | — | — | [英語でのスライド,十月2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/4_pragma_innovation.pdf) | -| [QINGCLOUD](https://www.qingcloud.com/) | クラウドサービス | 主な製品 | — | — | [スライドで中国、月2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/4.%20Cloud%20%2B%20TSDB%20for%20ClickHouse%20张健%20QingCloud.pdf) | -| [Qratorgenericname](https://qrator.net) | DDoS保護 | 主な製品 | — | — | [ブログ記事,行進2019](https://blog.qrator.net/en/clickhouse-ddos-mitigation_37/) | -| [北京パーセント情報技術有限公司、株式会社](https://www.percent.cn/) | Analytics | 主な製品 | — | — | [スライドで中国、月2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/4.%20ClickHouse万亿数据双中心的设计与实践%20.pdf) | -| [ランブラー](https://rambler.ru) | インターネットサービス | Analytics | — | — | [ロシア語で話す、april2018](https://medium.com/@ramblertop/разработка-api-clickhouse-для-рамблер-топ-100-f4c7e56f3141) | -| [Tencent](https://www.tencent.com) | メール | ログ記録 | — | — | [トークで中国、月2019](https://youtu.be/T-iVQRuw-QY?t=5050) | -| [交通星](https://trafficstars.com/) | 広告ネットワーク | — | — | — | [ロシア語のスライド,may2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup15/lightning/ninja.pdf) | -| [S7エアラインズ](https://www.s7.ru) | 航空会社 | 指標、ロギング | — | — | [ロシア語で話す、2019](https://www.youtube.com/watch?v=nwG68klRpPg&t=15s) | -| [セムルシ](https://www.semrush.com/) | 販売 | 主な製品 | — | — | [ロシア語のスライド、august2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/5_semrush.pdf) | -| [サイリウム社](https://www.scireum.de/) | eコマース | 主な製品 | — | — | [ドイツ語で話す、february2020](https://www.youtube.com/watch?v=7QWAn5RbyR4) | -| [セントリー](https://sentry.io/) | ソフトウェア開発者 | 製品のバックエンド | — | — | [英語でのブログ記事,もっと2019](https://blog.sentry.io/2019/05/16/introducing-snuba-sentrys-new-search-infrastructure) | -| [SGK](http://www.sgk.gov.tr/wps/portal/sgk/tr) | 政府の社会保障 | Analytics | — | — | [スライドで英語、月2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup35/ClickHouse%20Meetup-Ramazan%20POLAT.pdf) | -| [seo.do](https://seo.do/) | Analytics | 主な製品 | — | — | [スライドで英語、月2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup35/CH%20Presentation-%20Metehan%20Çetinkaya.pdf) | -| [シーナ](http://english.sina.com/index.html) | ニュース | — | — | — | [スライドで中国、月2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/6.%20ClickHouse最佳实践%20高鹏_新浪.pdf) | -| [SMI2](https://smi2.ru/) | ニュース | Analytics | — | — | [ブログ記事でロシア、月2017](https://habr.com/ru/company/smi2/blog/314558/) | -| [Splunk](https://www.splunk.com/) | ビジネス分析 | 主な製品 | — | — | [スライドで英語、月2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup12/splunk.pdf) | -| [Spotify](https://www.spotify.com) | 音楽 | 実験 | — | — | [スライド,七月2018](https://www.slideshare.net/glebus/using-clickhouse-for-experimentation-104247173) | -| [Tencent](https://www.tencent.com) | ビッグデータ | データ処理 | — | — | [スライドで中国、月2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/5.%20ClickHouse大数据集群应用_李俊飞腾讯网媒事业部.pdf) | -| [Uber](https://www.uber.com) | タクシー | ログ記録 | — | — | [スライド,二月2020](https://presentations.clickhouse.tech/meetup40/uber.pdf) | -| [VKontakte](https://vk.com) | 社会的ネットワーク | 統計、ロギング | — | — | [ロシア語のスライド、august2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/3_vk.pdf) | -| [Wisebits](https://wisebits.com/) | ITソリューション | Analytics | — | — | [ロシア語のスライド、2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | -| [Xiaoxinの技術。](https://www.xiaoheiban.cn/) | 教育 | 共通の目的 | — | — | [スライドで英語、月2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/sync-clickhouse-with-mysql-mongodb.pptx) | -| [Ximalaya](https://www.ximalaya.com/) | オーディオ共有 | OLAP | — | — | [スライドで英語、月2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/ximalaya.pdf) | -| [Yandexクラウド](https://cloud.yandex.ru/services/managed-clickhouse) | パブリック | 主な製品 | — | — | [トークでロシア、december2019](https://www.youtube.com/watch?v=pgnak9e_E0o) | -| [YandexのDataLens](https://cloud.yandex.ru/services/datalens) | ビジネス情報 | 主な製品 | — | — | [スライドでロシア、december2019](https://presentations.clickhouse.tech/meetup38/datalens.pdf) | -| [Yandexの市場](https://market.yandex.ru/) | eコマース | 指標、ロギング | — | — | [ロシア語で話す、january2019](https://youtu.be/_l1qP0DyBcA?t=478) | -| [Yandex Metrica](https://metrica.yandex.com) | ウェブ分析 | 主な製品 | 360サーバを一つのクラスター1862サーバーの一部 | 66.41PiB/5.68PiB | [スライド,二月2020](https://presentations.clickhouse.tech/meetup40/introduction/#13) | -| [ЦВТ](https://htc-cs.ru/) | ソフトウェア開発 | 指標、ロギング | — | — | [ブログ記事,月2019,ロシア語で](https://vc.ru/dev/62715-kak-my-stroili-monitoring-na-prometheus-clickhouse-i-elk) | -| [МКБ](https://mkb.ru/) | 銀行 | Webシステムの監視 | — | — | [スライドでロシア、2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/mkb.pdf) | -| [金数据](https://jinshuju.net) | BI分析 | 主な製品 | — | — | [スライドで中国、月2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/3.%20金数据数据架构调整方案Public.pdf) | +| 会社 | 産業 | Usecase | 集りのサイズ | (Un)圧縮データサイズ\* | 参照 | +|------------------------------------------------------------------------------------------------|------------------------------|----------------------|---------------------------------------------------|--------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 2gis | 地図 | 監視 | — | — | [2019年ロシア語](https://youtu.be/58sPkXfq6nw) | +| Aloha Browser | モバイルアプリ | ブラウザバックエンド | — | — | [ロシア語でのスライド,月2019](https://github.com/yandex/clickhouse-presentations/blob/master/meetup22/aloha.pdf) | +| アマデウス | 旅行 | 分析 | — | — | [プレスリリース,April2018](https://www.altinity.com/blog/2018/4/5/amadeus-technologies-launches-investment-and-insights-tool-based-on-machine-learning-and-strategy-algorithms) | +| Appsflyer | モバイル分析 | 主な製品 | — | — | [2019年ロシア語](https://www.youtube.com/watch?v=M3wbRlcpBbY) | +| ArenaData | データ基盤 | 主な製品 | — | — | [2019年ロシア](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup38/indexes.pdf) | +| バドゥ | デート | 時系列 | — | — | [2019年ロシア](https://presentations.clickhouse.tech/meetup38/forecast.pdf) | +| ベノク | ネットワークテレメトリと分析 | 主な製品 | — | — | [スライド英語,October2017](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup9/lpm.pdf) | +| ブルームバーグ | 金融、メディア | 監視 | 102サーバー | — | [スライド2018](https://www.slideshare.net/Altinity/http-analytics-for-6m-requests-per-second-using-clickhouse-by-alexander-bocharov) | +| Bloxy | Blockchain | 分析 | — | — | [ロシア語でのスライド,八月2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/4_bloxy.pptx) | +| 中国電信のためのDataliance | テレコム | 分析 | — | — | [スライド中国語,January2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup12/telecom.pdf) | +| CARTO | ビジネス情報 | ジオ分析 | — | — | [ClickHouseによる地理空間処理](https://carto.com/blog/geospatial-processing-with-clickhouse/) | +| CERN | 研究 | 実験 | — | — | [プレスリリース,四月2012](https://www.yandex.com/company/press_center/press_releases/2012/2012-04-10/) | +| Cisco | ネットワーク | トラフィック分析 | — | — | [ライトニングトーク2019](https://youtu.be/-hI1vDR2oPY?t=5057) | +| Citadel Securities | 金融 | — | — | — | [2019年の貢献](https://github.com/ClickHouse/ClickHouse/pull/4774) | +| シティモービル | タクシー | 分析 | — | — | [ロシア語でのブログ投稿,月2020](https://habr.com/en/company/citymobil/blog/490660/) | +| ContentSquare | ウェブ分析 | 主な製品 | — | — | [フランス語でのブログ投稿,November2018](http://souslecapot.net/2018/11/21/patrick-chatain-vp-engineering-chez-contentsquare-penser-davantage-amelioration-continue-que-revolution-constante/) | +| Cloudflare | CDN | トラフィック分析 | 36台のサーバー | — | [ブログ投稿,月2017](https://blog.cloudflare.com/how-cloudflare-analyzes-1m-dns-queries-per-second/), [ブログ投稿,月2018](https://blog.cloudflare.com/http-analytics-for-6m-requests-per-second-using-clickhouse/) | +| コルネット | 分析 | 主な製品 | — | — | [2019年英語スライド](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup21/predictive_models.pdf) | +| CraiditX 氪信 | ファイナンスAI | 分析 | — | — | [2019年のスライド](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/udf.pptx) | +| Criteo | 小売り | 主な製品 | — | — | [スライド英語,October2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/3_storetail.pptx) | +| Deutsche Bank | 金融 | BI分析 | — | — | [2019年のスライド](https://bigdatadays.ru/wp-content/uploads/2019/10/D2-H3-3_Yakunin-Goihburg.pdf) | +| 歌姫-e | デジタルコンサル | 主な製品 | — | — | [2019年英語でのスライド](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup29/ClickHouse-MeetUp-Unusual-Applications-sd-2019-09-17.pdf) | +| Exness | 取引 | 指標、ロギング | — | — | [ロシア語で話す,May2019](https://youtu.be/_rpU-TvSfZ8?t=3215) | +| 魔神 | 広告ネットワーク | 主な製品 | — | — | [ブログ投稿日本語,July2017](https://tech.geniee.co.jp/entry/2017/07/20/160100) | +| HUYA | ビデオストリーミング | 分析 | — | — | [中国語でのスライド,October2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/7.%20ClickHouse万亿数据分析实践%20李本旺(sundy-li)%20虎牙.pdf) | +| イデアリスタ | 不動産 | 分析 | — | — | [ブログ投稿英語,April2019](https://clickhouse.yandex/blog/en/clickhouse-meetup-in-madrid-on-april-2-2019) | +| インフォビスタ | ネット | 分析 | — | — | [2019年のスライド](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup30/infovista.pdf) | +| InnoGames | ゲーム | 指標、ロギング | — | — | [2019年ロシア](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/graphite_and_clickHouse.pdf) | +| インテグロス | Platformビデオサービス | 分析 | — | — | [ロシア語でのスライド,月2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | +| Kodiakデータ | 雲 | 主な製品 | — | — | [2018年エンギッシュのスライド](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup13/kodiak_data.pdf) | +| コントゥール | ソフトウェア開発 | メトリック | — | — | [2018年ロシア語](https://www.youtube.com/watch?v=U4u4Bd0FtrY) | +| LifeStreet | 広告ネットワーク | 主な製品 | 75サーバー(3つのレプリカ) | 5.27PiB | [ロシア語でのブログ投稿,二月2017](https://habr.com/en/post/322620/) | +| Mail.ru クラウドソリューション | クラウド | 主な製品 | — | — | [ロシア語の記事](https://mcs.mail.ru/help/db-create/clickhouse#) | +| MessageBird | 通信 | 統計 | — | — | [2018年のスライド](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup20/messagebird.pdf) | +| MGID | 広告ネットワーク | ウェブ分析 | — | — | [ロシア語でのブログ投稿,April2020](http://gs-studio.com/news-about-it/32777----clickhouse---c) | +| OneAPM | 監視およびデータ分析 | 主な製品 | — | — | [中国語でのスライド,October2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/8.%20clickhouse在OneAPM的应用%20杜龙.pdf) | +| Pragma Innovation | テレメトリとビッグデータ分析 | 主な製品 | — | — | [スライド英語,October2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/4_pragma_innovation.pdf) | +| QINGCLOUD | クラウド | 主な製品 | — | — | [中国語でのスライド,October2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/4.%20Cloud%20%2B%20TSDB%20for%20ClickHouse%20张健%20QingCloud.pdf) | +| Qrator | DDoS保護 | 主な製品 | — | — | [2019年のブログ記事](https://blog.qrator.net/en/clickhouse-ddos-mitigation_37/) | +| Percent 百分点 | 分析 | 主な製品 | — | — | [2019年のスライド](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/4.%20ClickHouse万亿数据双中心的设计与实践%20.pdf) | +| ランブラー | インターネット | 分析 | — | — | [ロシア語での講演,April2018](https://medium.com/@ramblertop/разработка-api-clickhouse-для-рамблер-топ-100-f4c7e56f3141) | +| テンセント | メッセージ | ロギング | — | — | [2019年、中国語でのトーク](https://youtu.be/T-iVQRuw-QY?t=5050) | +| Traffic Stars | 広告ネットワーク | — | — | — | [ロシア語でのスライド,月2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup15/lightning/ninja.pdf) | +| S7 Airlines | 航空会社 | 指標、ロギング | — | — | [2019年ロシア語](https://www.youtube.com/watch?v=nwG68klRpPg&t=15s) | +| セムルシュ | 販売 | 主な製品 | — | — | [ロシア語でのスライド,八月2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/5_semrush.pdf) | +| scireum GmbH | eコマース | 主な製品 | — | — | [2020年ドイツ語での講演](https://www.youtube.com/watch?v=7QWAn5RbyR4) | +| セントリー | ソフトウェア開発者 | 製品のバックエンド | — | — | [ブログ投稿英語,May2019](https://blog.sentry.io/2019/05/16/introducing-snuba-sentrys-new-search-infrastructure) | +| SGK | 政府の社会保障 | 分析 | — | — | [2019年のスライド](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup35/ClickHouse%20Meetup-Ramazan%20POLAT.pdf) | +| seo.do | 分析 | 主な製品 | — | — | [2019年のスライド](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup35/CH%20Presentation-%20Metehan%20Çetinkaya.pdf) | +| シーナ | ニュース | — | — | — | [中国語でのスライド,October2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/6.%20ClickHouse最佳实践%20高鹏_新浪.pdf) | +| SMI2 | ニュース | 分析 | — | — | [ロシア語でのブログ投稿,November2017](https://habr.com/ru/company/smi2/blog/314558/) | +| Splunk | ビジネス分析 | 主な製品 | — | — | [2018年のスライド](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup12/splunk.pdf) | +| Spotify | 音楽 | 実験 | — | — | [スライド2018](https://www.slideshare.net/glebus/using-clickhouse-for-experimentation-104247173) | +| テンセント | ビッグデータ | データ処理 | — | — | [中国語でのスライド,October2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/5.%20ClickHouse大数据集群应用_李俊飞腾讯网媒事业部.pdf) | +| Uber | タクシー | ロギング | — | — | [スライド2020](https://presentations.clickhouse.tech/meetup40/uber.pdf) | +| VKontakte | ソーシャルネ | 統計、ロギング | — | — | [ロシア語でのスライド,八月2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/3_vk.pdf) | +| Wisebits | ITソリューショ | 分析 | — | — | [ロシア語でのスライド,月2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | +| Xiaoxin Tech | 教育 | 共通の目的 | — | — | [2019年のスライド](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/sync-clickhouse-with-mysql-mongodb.pptx) | +| シマラヤ | オーディオ共有 | OLAP | — | — | [2019年のスライド](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/ximalaya.pdf) | +| Yandex Cloud | パブリック | 主な製品 | — | — | [2019年ロシア語](https://www.youtube.com/watch?v=pgnak9e_E0o) | +| Yandex DataLens | ビジネス情報 | 主な製品 | — | — | [2019年ロシア](https://presentations.clickhouse.tech/meetup38/datalens.pdf) | +| Yandex Market | eコマース | 指標、ロギング | — | — | [2019年ロシア語](https://youtu.be/_l1qP0DyBcA?t=478) | +| Yandex Metrica | ウェブ分析 | 主な製品 | 一つのクラスタに360台、一つの部門に1862台のサーバ | 66.41PiB/5.68PiB | [スライド2020](https://presentations.clickhouse.tech/meetup40/introduction/#13) | +| ЦВТ | ソフトウェア開発 | 指標、ロギング | — | — | [ブログ投稿,月2019,ロシア語で](https://vc.ru/dev/62715-kak-my-stroili-monitoring-na-prometheus-clickhouse-i-elk) | +| МКБ | 銀行 | Webシステム監視 | — | — | [2019年ロシア](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/mkb.pdf) | +| Jinshuju 金数据 | BI分析 | 主な製品 | — | — | [2019年の中国](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/3.%20金数据数据架构调整方案Public.pdf) | +| インスタナ | APM | 主な製品 | — | — | [Twitter投稿](https://twitter.com/mieldonkers/status/1248884119158882304) | +| Wargaming | ゲーム | | — | — | [インタビュ](https://habr.com/en/post/496954/) | +| Crazypanda | ゲーム | | — | — | ClickHouse meetupのライブセッション | +| FunCorp | ゲーム | | — | — | [記事](https://www.altinity.com/blog/migrating-from-redshift-to-clickhouse) | [元の記事](https://clickhouse.tech/docs/en/introduction/adopters/) diff --git a/docs/ja/introduction/index.md b/docs/ja/introduction/index.md index 317489d277b..c1ae23b18a8 100644 --- a/docs/ja/introduction/index.md +++ b/docs/ja/introduction/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_folder_title: Introduction +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "\u306F\u3058\u3081\u306B" toc_priority: 1 --- diff --git a/docs/ja/operations/access-rights.md b/docs/ja/operations/access-rights.md index 1c384f14f5d..e02e7529ecb 100644 --- a/docs/ja/operations/access-rights.md +++ b/docs/ja/operations/access-rights.md @@ -1,113 +1,144 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 48 -toc_title: "\u30A2\u30AF\u30BB\u30B9\u6A29" +toc_title: "\u30A2\u30AF\u30BB\u30B9\u5236\u5FA1\u3068\u30A2\u30AB\u30A6\u30F3\u30C8\ + \u7BA1\u7406" --- -# アクセス権 {#access-rights} +# アクセス制御とアカウント管理 {#access-control} -ユーザーとアクセス権は、ユーザー設定で設定されます。 これは通常 `users.xml`. +ClickHouseはに基づくアクセス制御管理を支えます [RBAC](https://en.wikipedia.org/wiki/Role-based_access_control) アプローチ -ユーザーは `users` セクション。 ここでの断片です `users.xml` ファイル: +ClickHouseアクセス事業体: +- [ユーザー](#user-account-management) +- [役割](#role-management) +- [行ポリシー](#row-policy-management) +- [設定プロファイル](#settings-profiles-management) +- [クォータ](#quotas-management) -``` xml - - - - - - +- サーバ [設定ファイル](configuration-files.md) `users.xml` と `config.xml`. - - +!!! note "警告" + 両方の構成方法で同じaccessエンティティを同時に管理することはできません。 - - default +## 使用法 {#access-control-usage} - - default - +既定では、ClickHouseサーバーはユーザーアカウントを提供します `default` SQL駆動型のアクセス制御とアカウント管理を使用することはできませんが、すべての権限と権限を持っています。 その `default` ユーザーアカウントは、クライアントからのログイン時や分散クエリなど、ユーザー名が定義されていない場合に使用されます。 分散クエリ処理のデフォルトのユーザーアカウントをお使いの場合、設定のサーバまたはクラスターを指定していないの [ユーザとパスワード](../engines/table-engines/special/distributed.md) プロパティ。 - - - - - web - default - - test - - - test - - - -``` +ClickHouseの使用を開始したばかりの場合は、次のシナリオを使用できます: -する宣言からユーザー: `default`と`web`. 私達は加えました `web` ユーザー別途。 +1. [有効にする](#enabling-access-control) のためのSQL駆動型アクセス制御およびアカウント管理 `default` ユーザー。 +2. の下でログイン `default` ユーザーアカウントを作成し、必要なすべてのユーザー 管理者アカウントの作成を忘れないでください (`GRANT ALL ON *.* WITH GRANT OPTION TO admin_user_account`). +3. [権限の制限](settings/permissions-for-queries.md#permissions_for_queries) のために `default` SQL駆動型のアクセス制御とアカウント管理をユーザーと無効にします。 -その `default` ユーザは、ユーザ名が渡されない場合に選択されます。 その `default` userは、サーバーまたはクラスターの構成が指定されていない場合は、分散クエリ処理にも使用されます。 `user` と `password` (上のセクションを参照 [分散](../engines/table-engines/special/distributed.md) エンジン）。 +### 現在のソリューションの特性 {#access-control-properties} -The user that is used for exchanging information between servers combined in a cluster must not have substantial restrictions or quotas – otherwise, distributed queries will fail. +- データベースとテーブルが存在しない場合でも、権限を付与できます。 +- テーブルが削除された場合、このテーブルに対応するすべての権限は取り消されません。 したがって、後で同じ名前で新しいテーブルが作成されると、すべての特権が再び実際になります。 削除されたテーブルに対応する権限を取り消すには、次のように実行する必要があります。 `REVOKE ALL PRIVILEGES ON db.table FROM ALL` クエリ。 +- 特権の有効期間の設定はありません。 -パスワードは、クリアテキスト(非推奨)またはsha-256で指定します。 ハッシュは塩漬けじゃない この点に関し、すべきではないと考えるこれらのパスワードとして提供すことに対する潜在的悪意のある攻撃であった。 むしろ、従業員からの保護のために必要です。 +## ユーザー {#user-account-management} -アクセスを許可するネットワークのリストが指定されています。 この例では、両方のユーザーのネットワークの一覧が別のファイルから読み込まれます (`/etc/metrika.xml`)を含む `networks` 置換。 ここにそれの断片があります: +ユーザーアカウントは、ClickHouseで誰かを承認できるアクセスエンティティです。 ユーザーアカウ: -``` xml - - ... - - ::/64 - 203.0.113.0/24 - 2001:DB8::/32 - ... - - -``` +- 識別情報。 +- [特権](../sql-reference/statements/grant.md#grant-privileges) これは、ユーザーが実行できるクエリの範囲を定義します。 +- ClickHouseサーバーへの接続が許可されているホスト。 +- 付与された役割と既定の役割。 +- ユーザーのログイン時にデフォルトで適用される制約を含む設定。 +- 割り当ての設定を行います。 -このネットワークのリストを直接 `users.xml` またはファイル内の `users.d` ディレクトリ(詳細については、セクションを参照 “[設定ファイル](configuration-files.md#configuration_files)”). +ユーザーアカウントに対する権限は、 [GRANT](../sql-reference/statements/grant.md) クエリまたは割り当て [役割](#role-management). ユーザーから特権を取り消すために、ClickHouseは [REVOKE](../sql-reference/statements/revoke.md) クエリ。 ユーザーの権限を一覧表示するには、 - [SHOW GRANTS](../sql-reference/statements/show.md#show-grants-statement) 声明。 -コンフィグを含むコメントする方法を説明するオープンアクセスいたしました。 +管理クエリ: -生産の使用のために、指定して下さいただ `ip` 要素（IPアドレスとそのマスク）、 `host` と `hoost_regexp` が原因別の待ち時間をゼロにすることに +- [CREATE USER](../sql-reference/statements/create.md#create-user-statement) +- [ALTER USER](../sql-reference/statements/alter.md#alter-user-statement) +- [DROP USER](../sql-reference/statements/misc.md#drop-user-statement) +- [SHOW CREATE USER](../sql-reference/statements/show.md#show-create-user-statement) -次のユーザー設定プロファイルが指定の項を参照 “[設定プロファイル](settings/settings-profiles.md)”. 既定のプロファイルを指定できます, `default'`. プロファイルの名前は任意です。 異なるユーザーに同じプロファイルを指定できます。 最も重要なことが書ける設定プロフィール `readonly=1` 読み取り専用アクセスを保証します。 次に、使用するクォータを指定します(セクションを参照 “[クォータ](quotas.md#quotas)”). 既定のクォータを指定できます: `default`. It is set in the config by default to only count resource usage, without restricting it. The quota can have any name. You can specify the same quota for different users – in this case, resource usage is calculated for each user individually. +### 設定の適用 {#access-control-settings-applying} -オプションで `` 部を指定することもできますリストのデータベースのユーザーがアクセスできる デフォルトでは、すべてのデータベースのユーザーです。 を指定することができ `default` データベース この場合、ユーザーはデフォルトでデータベースにアクセスできます。 +設定は、さまざまな方法で設定できます。 ユーザーログイン時に、異なるアクセスエンティティで設定が設定されている場合、この設定の値および制約は、以下の優先順位によって適用されます(): -オプションで `` また、ユーザーがアクセスできる辞書のリストを指定することもできます。 デフォルトでは、すべての辞書はユーザーが使用できます。 +1. ユーザーアカウント設定。 +2. ユーザーアカウントの既定のロールの設定。 一部のロールで設定が設定されている場合、設定の適用順序は未定義です。 +3. ユーザーまたはその既定のロールに割り当てられた設定プロファイルの設定。 いくつかのプロファイルで設定が設定されている場合、設定適用の順序は未定義です。 +4. すべてのサーバーにデフォルトまたは [標準プロファイル](server-configuration-parameters/settings.md#default-profile). -へのアクセス `system` データベースは常に可（このデータベースを使用して処理クエリ). +## 役割 {#role-management} -ユーザーの一覧を取得してデータベースやテーブルを用いてこれら `SHOW` クエリやシステムテーブルの場合でも、アクセス、個人データベースなのです。 +Roleは、ユーザーアカウントに付与できるaccessエンティティのコンテナです。 -データベースアクセスは、 [読み取り専用](settings/permissions-for-queries.md#settings_readonly) 設定。 できな助成金の全アクセスをデータベース `readonly` 別のものへのアクセス。 +ロール: + +- [特権](../sql-reference/statements/grant.md#grant-privileges) +- 設定と制約 +- 付与されたロールのリスト + +管理クエリ: + +- [CREATE ROLE](../sql-reference/statements/create.md#create-role-statement) +- [ALTER ROLE](../sql-reference/statements/alter.md#alter-role-statement) +- [DROP ROLE](../sql-reference/statements/misc.md#drop-role-statement) +- [SET ROLE](../sql-reference/statements/misc.md#set-role-statement) +- [SET DEFAULT ROLE](../sql-reference/statements/misc.md#set-default-role-statement) +- [SHOW CREATE ROLE](../sql-reference/statements/show.md#show-create-role-statement) + +ロールに対する権限は、 [GRANT](../sql-reference/statements/grant.md) クエリ。 ロールClickHouseから特権を取り消すには [REVOKE](../sql-reference/statements/revoke.md) クエリ。 + +## 行ポリシー {#row-policy-management} + +行ポリシーは、ユーザーまたはロールで使用できる行または行を定義するフィルターです。 行政政策を含むィのための特定のテーブルリストの役割および/またはユーザーはこの行政政策です。 + +管理クエリ: + +- [CREATE ROW POLICY](../sql-reference/statements/create.md#create-row-policy-statement) +- [ALTER ROW POLICY](../sql-reference/statements/alter.md#alter-row-policy-statement) +- [DROP ROW POLICY](../sql-reference/statements/misc.md#drop-row-policy-statement) +- [SHOW CREATE ROW POLICY](../sql-reference/statements/show.md#show-create-row-policy-statement) + +## 設定プロファイル {#settings-profiles-management} + +設定プロファイルは [設定](settings/index.md). 設定プロファイルには、設定と制約、およびこのクォータが適用されるロールやユーザーのリストが含まれます。 + +管理クエリ: + +- [CREATE SETTINGS PROFILE](../sql-reference/statements/create.md#create-settings-profile-statement) +- [ALTER SETTINGS PROFILE](../sql-reference/statements/alter.md#alter-settings-profile-statement) +- [DROP SETTINGS PROFILE](../sql-reference/statements/misc.md#drop-settings-profile-statement) +- [SHOW CREATE SETTINGS PROFILE](../sql-reference/statements/show.md#show-create-settings-profile-statement) + +## クォータ {#quotas-management} + +クォータ制限資源利用に 見る [クォータ](quotas.md). + +定員の制限のために一部の時間、リストの役割および/またはユーザーはこの数量に達した場合。 + +管理クエリ: + +- [CREATE QUOTA](../sql-reference/statements/create.md#create-quota-statement) +- [ALTER QUOTA](../sql-reference/statements/alter.md#alter-quota-statement) +- [DROP QUOTA](../sql-reference/statements/misc.md#drop-quota-statement) +- [SHOW CREATE QUOTA](../sql-reference/statements/show.md#show-create-quota-statement) + +## SQL駆動型アクセス制御とアカウント管理の有効化 {#enabling-access-control} + +- 設定ディレクトリ構成を保管します。 + + ClickHouseは、アクセスエンティティ設定を [access\_control\_path](server-configuration-parameters/settings.md#access_control_path) サーバー構成パラメータ。 + +- SQL駆動型のアクセス制御とアカウント管理を有効にします。 + + デフォルトのSQL型のアクセス制御及び特別口座の口座管理オのすべてのユーザー ユーザーを設定する必要があります。 `users.xml` に1を割り当てます。 [access\_management](settings/settings-users.md#access_management-user-setting) 設定。 [元の記事](https://clickhouse.tech/docs/en/operations/access_rights/) diff --git a/docs/ja/operations/backup.md b/docs/ja/operations/backup.md index 799b377f3f3..994271371a4 100644 --- a/docs/ja/operations/backup.md +++ b/docs/ja/operations/backup.md @@ -1,40 +1,40 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 49 -toc_title: "\u30C7\u30FC\u30BF\u30D0\u30C3\u30AF" +toc_title: "\u30C7\u30FC\u30BF\u30D0\u30C3\u30AF\u30A2" --- -# データバック {#data-backup} +# データバックア {#data-backup} -しばらく [複製](../engines/table-engines/mergetree-family/replication.md) provides protection from hardware failures, it does not protect against human errors: accidental deletion of data, deletion of the wrong table or a table on the wrong cluster, and software bugs that result in incorrect data processing or data corruption. In many cases mistakes like these will affect all replicas. ClickHouse has built-in safeguards to prevent some types of mistakes — for example, by default [50Gbを超えるデータを含むMergeTreeのようなエンジンでテーブルを削除することはできません](https://github.com/ClickHouse/ClickHouse/blob/v18.14.18-stable/programs/server/config.xml#L322-L330). しかし、これらの保障措置がカバーしないすべてのケースで回避. +ながら [複製](../engines/table-engines/mergetree-family/replication.md) provides protection from hardware failures, it does not protect against human errors: accidental deletion of data, deletion of the wrong table or a table on the wrong cluster, and software bugs that result in incorrect data processing or data corruption. In many cases mistakes like these will affect all replicas. ClickHouse has built-in safeguards to prevent some types of mistakes — for example, by default [50Gbを超えるデータを含むMergeTreeのようなエンジンでは、テーブルを削除することはできません](https://github.com/ClickHouse/ClickHouse/blob/v18.14.18-stable/programs/server/config.xml#L322-L330). しかし、これらの保障措置がカバーしないすべてのケースで回避. -人為的なミスを効果的に軽減するには、データのバックアップと復元に関する戦略を慎重に準備する必要があります **事前に**. +ヒューマンエラーを効果的に軽減するには、データのバックアップと復元のための戦略を慎重に準備する必要があります **事前に**. -クリックハウスのバックアップとリストアのためのユニバーサルソリューションはありません。 gigabyteのデータで動作するものは、数十ペタバイトでは動作しません。 自分の長所と短所を持つ様々な可能なアプローチがありますが、これについては以下で説明します。 それは彼らの様々な欠点を補うために一つの代わりに、いくつかのアプローチを使用することをお勧めします。 +それぞれの会社には、利用可能なリソースとビジネス要件が異なるため、あらゆる状況に合ったClickHouseバックアップと復元のための普遍的なソリューショ 何がデータのギガバイトのために働く可能性が高い数十ペタバイトのために動作しません。 自分の長所と短所を持つ様々な可能なアプローチがありますが、これは以下で説明します。 で使うようにするといいでしょうくつかの方法でそれを補うためにさまの様々な問題があった。 -!!! note "メモ" - あなたが何かをバックアップし、それを復元しようとしたことがない場合、チャンスはあなたが実際にそれを必要とするときに正常に動作しない だから、どのようなバックアップアプローチを選択し、同様に復元プロセスを自動化することを確認し、定期的に予備のclickhouseクラスタでそれを練習。 +!!! note "注" + あなたが何かをバックアップし、それを復元しようとしたことがない場合、チャンスは、あなたが実際にそれを必要とするときに復元が正常に動作 したがって、どのようなバックアップアプローチを選択しても、復元プロセスも自動化し、予備のClickHouseクラスターで定期的に練習してください。 -## ソースデータの他の場所への複製 {#duplicating-source-data-somewhere-else} +## ソースデータを他の場所に複製する {#duplicating-source-data-somewhere-else} -多くの場合、clickhouseに取り込まれたデータは、次のような何らかの永続キューを介して配信されます [アパッチ-カフカ](https://kafka.apache.org). この場合、ClickHouseに書き込まれている間に同じデータストリームを読み込み、それをどこかのコールドストレージに保存する追加のサブスクライバセットを構 これは、オブジェクトストアまたは次のような分散ファイルシステムです [HDFS](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HdfsDesign.html). +多くの場合、ClickHouseに取り込まれたデータは、次のような何らかの永続キューを介して配信されます [アパッチ-カフカ](https://kafka.apache.org). この場合、ClickHouseに書き込まれている間に同じデータストリームを読み取り、どこかのコールドストレージに保存する追加のサブスクライバーセットを構成するこ ほとんどの企業はすでにいくつかのデフォルト推奨コールドストレージを持っています。 [HDFS](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HdfsDesign.html). -## ファイル {#filesystem-snapshots} +## ファイルシステ {#filesystem-snapshots} -一部地域のファイルシステムをスナップショット機能(例えば, [ZFS](https://en.wikipedia.org/wiki/ZFS)ものではないのでいただける住ます。 可能な解決策は、この種のファイルシステムを使用して追加のレプリカを作成し、それらを [分散](../engines/table-engines/special/distributed.md) 使用されるテーブル `SELECT` クエリ。 スナップショットなどを複製するものでなければならないのクエリがデータを変更する. ボーナスパーツとして、これらのレプリカが特別なハードウェア構成によりディスクに付属のサーバ、コスト効果的です。 +一部地域のファイルシステムをスナップショット機能(例えば, [ZFS](https://en.wikipedia.org/wiki/ZFS)しかし、ライブクエリを提供するための最良の選択ではないかもしれません。 可能な解決策は、この種のファイルシステムで追加のレプリカを作成し、それらを [分散](../engines/table-engines/special/distributed.md) 以下の目的で使用されるテーブル `SELECT` クエリ。 スナップショットなどを複製するものでなければならないのクエリがデータを変更する. ボーナスパーツとして、これらのレプリカが特別なハードウェア構成によりディスクに付属のサーバ、コスト効果的です。 -## クリックハウスコピー機 {#clickhouse-copier} +## クリックハウス-複写機 {#clickhouse-copier} -[クリックハウスコピー機](utilities/clickhouse-copier.md) 最初にペタバイトサイズのテーブルを再シャードするために作成された汎用性の高いツールです。 でも使用できるバックアップと復元を目的なので確実にコピーしたデータのClickHouseテーブルとクラスター +[クリックハウス-複写機](utilities/clickhouse-copier.md) ペタバイトサイズのテーブルを再シャードするために最初に作成された多目的な用具はある。 ClickHouseテーブルとクラスター間でデータを確実にコピーするため、バックアップと復元の目的でも使用できます。 -データの小さい容積のため、簡単の `INSERT INTO ... SELECT ...` リモートテーブルが作業しています。 +データの小さなボリュームのために、単純な `INSERT INTO ... SELECT ...` リモートテーブルが作業しています。 ## 部品による操作 {#manipulations-with-parts} -ClickHouseは使用を可能にします `ALTER TABLE ... FREEZE PARTITION ...` クエリをコピーのテーブル割. これは、ハードリンクを使用して実装されます `/var/lib/clickhouse/shadow/` フォルダで、通常は消費するエディスクスペースのための古いデータです。 作成されたファイルのコピーはClickHouseサーバーによって処理されないので、そこに残すことができます：追加の外部システムを必要としない単純なバックア このため、リモートで別の場所にコピーしてからローカルコピーを削除する方がよいでしょう。 分散ファイルシステムとオブジェクトストアはまだこのための良いオプションですが、十分な大きさの容量を持つ通常の添付ファイルサーバは、同 [rsync](https://en.wikipedia.org/wiki/Rsync)). +ClickHouseは、 `ALTER TABLE ... FREEZE PARTITION ...` クエリをコピーのテーブル割. これはハードリンクを使って実装される。 `/var/lib/clickhouse/shadow/` それは通常、古いデータのための余分なディスク領域を消費しません。 作成されたファイルのコピーはClickHouse serverによって処理されないので、そこに残すことができます：追加の外部システムを必要としない簡単なバックアップ このため、リモートで別の場所にコピーしてからローカルコピーを削除する方が良いでしょう。 分散ファイルシステムとオブジェクトストアはまだこのための良いオプションですが、十分な容量を持つ通常の添付ファイルサーバも同様に動作 [rsync](https://en.wikipedia.org/wiki/Rsync)). -パーティション操作に関連するクエリの詳細については、 [変更文書](../sql-reference/statements/alter.md#alter_manipulations-with-partitions). +パーティション操作に関連するクエリの詳細については、 [文書の変更](../sql-reference/statements/alter.md#alter_manipulations-with-partitions). 第三者ツールを自動化するこのアプローチ: [clickhouse-バックアップ](https://github.com/AlexAkulov/clickhouse-backup). diff --git a/docs/ja/operations/configuration-files.md b/docs/ja/operations/configuration-files.md index c7a4b5ed8fb..9457c89e3d9 100644 --- a/docs/ja/operations/configuration-files.md +++ b/docs/ja/operations/configuration-files.md @@ -1,34 +1,34 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 50 toc_title: "\u8A2D\u5B9A\u30D5\u30A1\u30A4\u30EB" --- # 設定ファイル {#configuration_files} -ClickHouseは複数ファイル構成管理をサポートしています。 主サーバ設定ファイルで指定することがで `/etc/clickhouse-server/config.xml`. その他のファイルは `/etc/clickhouse-server/config.d` ディレクトリ。 +ClickHouseは複数のファイル構成管理をサポートします。 主サーバ設定ファイルで指定することがで `/etc/clickhouse-server/config.xml`. その他のファイルは `/etc/clickhouse-server/config.d` ディレクトリ。 -!!! note "メモ" - すべての設定ファイルはxml形式である必要があります。 また、通常は同じルート要素を持つ必要があります ``. +!!! note "注" + すべての構成ファイルはXML形式である必要があります。 また、通常は同じルート要素を持つ必要があります ``. -メイン構成ファイルで指定された一部の設定は、他の構成ファイルで上書きできます。 その `replace` または `remove` 属性は、これらの構成ファイルの要素に対して指定できます。 +メイン構成ファイルで指定された一部の設定は、他の構成ファイルで上書きできます。 その `replace` または `remove` これらの構成ファイルの要素に属性を指定できます。 -どちらも指定されていない場合は、重複する子の値を置き換えて、要素の内容を再帰的に結合します。 +どちらも指定されていない場合は、要素の内容を再帰的に結合し、重複する子の値を置き換えます。 -もし `replace` 指定された要素全体を指定された要素で置き換えます。 +もし `replace` 指定された要素全体を指定された要素に置き換えます。 -もし `remove` 指定した要素を削除します。 +もし `remove` 指定されると、要素を削除します。 -設定はまた定義できます “substitutions”. 要素が持っている場合 `incl` 属性は、ファイルから対応する置換は、値として使用されます。 デフォルトでは、置換を含むファイルへのパスは、 `/etc/metrika.xml`. これはで変えることができます [include\_from](server-configuration-parameters/settings.md#server_configuration_parameters-include_from) サーバー設定の要素。 置換値は、以下で指定されます `/yandex/substitution_name` このファイルの要素。 で指定された置換の場合 `incl` 存在しない、それはログに記録されます。 防ClickHouseからログイン失、置換、指定し `optional="true"` 属性(たとえば、 [マクロ](server-configuration-parameters/settings.md)). +この設定はまた、 “substitutions”. 要素が `incl` 属性は、ファイルからの対応する置換が値として使用されます。 デフォルトでは、ファイルへのパスとの置換を行う `/etc/metrika.xml`. これはで変えることができます [include\_from](server-configuration-parameters/settings.md#server_configuration_parameters-include_from) サーバー設定の要素。 置換値は、次のように指定されます `/yandex/substitution_name` このファイル内の要素。 で指定された置換の場合 `incl` 存在しない場合は、ログに記録されます。 ClickHouseが不足している置換をログに記録しないようにするには、 `optional="true"` 属性(たとえば、 [マクロ](server-configuration-parameters/settings.md)). -置換はまた、飼育係から行うことができます。 これを行うには、属性を指定します `from_zk = "/path/to/node"`. 要素の値は、次の場所にあるノードの内容に置き換えられます `/path/to/node` 飼育係で また、Xmlサブツリー全体をZooKeeperノードに置くこともでき、ソース要素に完全に挿入されます。 +置換はZooKeeperからも実行できます。 これを行うには、属性を指定します `from_zk = "/path/to/node"`. 要素の値は、ノードの内容に置き換えられます。 `/path/to/node` 飼育係で。 また、ZooKeeperノードにXMLサブツリー全体を配置することもできます。 -その `config.xml` ファイルを指定することで別のconfigユーザー設定、プロファイルに割り振ります。 この設定への相対パスは `users_config` 要素。 デフォルトでは、 `users.xml`. もし `users_config` ユーザー設定、プロファイル、およびクォータは、次の場所で直接指定されます `config.xml`. +その `config.xml` ファイルを指定することで別のconfigユーザー設定、プロファイルに割り振ります。 この設定への相対パスは `users_config` 要素。 既定では、次のようになります `users.xml`. もし `users_config` ユーザー設定、プロファイル、およびクォータは、 `config.xml`. -ユーザー構成は、次のような別々のファイルに分割できます `config.xml` と `config.d/`. -ディレクトリ名は、 `users_config` 設定なし `.xml` 後置は `.d`. -ディレク `users.d` デフォルトでは、 `users_config` デフォルトは `users.xml`. +ユーザー設定は、次のような別々のファイルに分割できます `config.xml` と `config.d/`. +ディレク `users_config` 設定なし `.xml` と連結された後置 `.d`. +Directory `users.d` デフォルトでは `users_config` デフォルトは `users.xml`. たとえば、次のようにユーザーごとに別々の設定ファイルを作成できます: ``` bash @@ -50,8 +50,8 @@ $ cat /etc/clickhouse-server/users.d/alice.xml ``` -各設定ファイルについて、サーバーも生成します `file-preprocessed.xml` 起動時のファイル。 これらのファイルには、完了した置換と上書きがすべて含まれており、情報の使用を目的としています。 ZooKeeperの置換が設定ファイルで使用されたが、ZooKeeperがサーバーの起動時に利用できない場合、サーバーは前処理されたファイルから設定をロードします。 +各設定ファイルでは、サーバともある `file-preprocessed.xml` 起動時のファイル。 これらのファイルには、完了したすべての置換と上書きが含まれており、情報提供を目的としています。 設定ファイルでZooKeeperの置換が使用されていても、サーバーの起動時にZooKeeperが使用できない場合、サーバーは前処理されたファイルから設定をロードします。 -サーバーは、設定ファイルの変更を追跡するだけでなく、置換と上書きを実行するときに使用されたファイルとzookeeperノード、およびその場でユーザーとクラスタ つまり、サーバーを再起動することなく、クラスター、ユーザー、およびその設定を変更できます。 +サーバーは、設定ファイルの変更、置換および上書きの実行時に使用されたファイルおよびZooKeeperノードを追跡し、ユーザーおよびクラスターの設定をその場で再ロード つまり、サーバーを再起動せずにクラスター、ユーザー、およびその設定を変更できます。 [元の記事](https://clickhouse.tech/docs/en/operations/configuration_files/) diff --git a/docs/ja/operations/index.md b/docs/ja/operations/index.md index dc634fdcc8e..3fc7d869800 100644 --- a/docs/ja/operations/index.md +++ b/docs/ja/operations/index.md @@ -1,9 +1,9 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_folder_title: Operations +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "\u64CD\u4F5C" toc_priority: 41 -toc_title: "\u5C0E\u5165" +toc_title: "\u306F\u3058\u3081\u306B" --- # 操作 {#operations} @@ -12,17 +12,17 @@ ClickHouse操作マニュアルの以下の主要部: - [要件](requirements.md) - [監視](monitoring.md) -- [トラブル](troubleshooting.md) -- [使用の推奨事項](tips.md) +- [トラブルシ](troubleshooting.md) +- [使用法の推奨事項](tips.md) - [更新手順](update.md) - [アクセス権](access-rights.md) -- [データバック](backup.md) +- [データバックア](backup.md) - [設定ファイル](configuration-files.md) - [クォータ](quotas.md) - [システム表](system-tables.md) -- [サーバ設定パラメータ](server-configuration-parameters/index.md) +- [サーバ構成パラメータ](server-configuration-parameters/index.md) - [ClickHouseでハードウェアをテストする方法](performance-test.md) - [設定](settings/index.md) - [ユーティリ](utilities/index.md) -[元の記事](https://clickhouse.tech/docs/en/operations/) +{## [元の記事](https://clickhouse.tech/docs/en/operations/) ##} diff --git a/docs/ja/operations/monitoring.md b/docs/ja/operations/monitoring.md index 99e63505a42..0d2ecd7f5b4 100644 --- a/docs/ja/operations/monitoring.md +++ b/docs/ja/operations/monitoring.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 45 toc_title: "\u76E3\u8996" --- @@ -10,35 +10,37 @@ toc_title: "\u76E3\u8996" 監視できます: - ハードウェアリソースの利用。 -- クリックハウスサーバー指標。 +- ClickHouseサーバー指標。 ## リソース使用率 {#resource-utilization} ClickHouseは、ハードウェアリソースの状態を単独で監視しません。 -それは強く推奨するセットアップ監視のための: +監視をセットアップすることを強く推奨します: - プロセッサの負荷と温度。 - を使用することができ [dmesg](https://en.wikipedia.org/wiki/Dmesg), [ターボスタット](https://www.linux.org/docs/man8/turbostat.html) または他の楽器。 + 以下を使用できます [dmesg](https://en.wikipedia.org/wiki/Dmesg), [ターボスタット](https://www.linux.org/docs/man8/turbostat.html) または他の楽器。 -- ストレージシステムの利用,ramとネットワーク. +- ストレージシステム、RAM、ネットワークの利用。 -## Clickhouseサーバーメトリクス {#clickhouse-server-metrics} +## ClickHouseサーバー指標 {#clickhouse-server-metrics} -ClickHouseサーバーは自己状態の監視のための器械を埋め込んだ。 +ClickHouse serverには、自己状態の監視のための計測器が組み込まれています。 -追跡サーバのイベントサーバーを利用ます。 を見る [ロガー](server-configuration-parameters/settings.md#server_configuration_parameters-logger) 設定ファイルのセクション。 +追跡サーバのイベントサーバーを利用ます。 を参照。 [ロガー](server-configuration-parameters/settings.md#server_configuration_parameters-logger) 設定ファイルのセクション。 -ClickHouseの収集: +クリックハウス収集: - 異なるメトリクスのサーバがどのように利用計算資源です。 - クエリ処理に関する一般的な統計。 -メトリックは次の場所にあります [システム。指標](../operations/system-tables.md#system_tables-metrics), [システム。イベント](../operations/system-tables.md#system_tables-events)、と [システム。asynchronous\_metrics](../operations/system-tables.md#system_tables-asynchronous_metrics) テーブル。 +メトリックは、次のとおりです。 [システムメトリック](../operations/system-tables.md#system_tables-metrics), [システムイベント](../operations/system-tables.md#system_tables-events),and [システムasynchronous\_metrics](../operations/system-tables.md#system_tables-asynchronous_metrics) テーブル -を設定することができclickhouse輸出の指標に [黒鉛](https://github.com/graphite-project). を見る [グラファイト部](server-configuration-parameters/settings.md#server_configuration_parameters-graphite) クリックハウスサーバー設定ファイルで。 を設定する前に輸出のメトリックトに設定する必要があります黒鉛は以下の公式 [ガイド](https://graphite.readthedocs.io/en/latest/install.html). +を設定することができClickHouse輸出の指標に [黒鉛](https://github.com/graphite-project). を参照。 [グラファイト部](server-configuration-parameters/settings.md#server_configuration_parameters-graphite) ClickHouseサーバー設定ファイル内。 指標のエクスポートを設定する前に、公式に従ってGraphiteを設定する必要があります [ガイド](https://graphite.readthedocs.io/en/latest/install.html). -さらに、http apiを使用してサーバーの可用性を監視できます。 を送信 `HTTP GET` への要求 `/ping`. サーバーが使用可能な場合は、次のように応答します `200 OK`. +を設定することができClickHouse輸出の指標に [プロメテウス](https://prometheus.io). を参照。 [プロメテウス節](server-configuration-parameters/settings.md#server_configuration_parameters-prometheus) ClickHouseサーバー設定ファイル内。 メトリックのエクスポートを設定する前に、公式に従ってPrometheusを設定してください [ガイド](https://prometheus.io/docs/prometheus/latest/installation/). -クラスター構成内のサーバーを監視するには、以下を設定します。 [max\_replica\_delay\_for\_distributed\_queries](settings/settings.md#settings-max_replica_delay_for_distributed_queries) HTTPリソースのパラメーターと使用 `/replicas_status`. を要求する `/replicas_status` を返します `200 OK` レプリカが使用可能で、他のレプリカの後ろに遅延されていない場合。 レプリカが遅延している場合は、 `503 HTTP_SERVICE_UNAVAILABLE` ギャップについての情報と。 +さらに、HTTP APIを使用してサーバーの可用性を監視できます。 送信 `HTTP GET` リクエスト先 `/ping`. サーバーが利用可能な場合は、次のように応答します `200 OK`. + +監視サーバーにクラスター構成設定してください [max\_replica\_delay\_for\_distributed\_queries](settings/settings.md#settings-max_replica_delay_for_distributed_queries) パラメータとHTTPリソースの使用 `/replicas_status`. への要求 `/replicas_status` ﾂづｩﾂ。 `200 OK` レプリカが使用可能で、他のレプリカより遅れていない場合。 レプリカが遅延すると、次のようになります `503 HTTP_SERVICE_UNAVAILABLE` ギャップについての情報と。 diff --git a/docs/ja/operations/optimizing-performance/index.md b/docs/ja/operations/optimizing-performance/index.md index 13dc51db028..19e7466940c 100644 --- a/docs/ja/operations/optimizing-performance/index.md +++ b/docs/ja/operations/optimizing-performance/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_folder_title: Optimizing Performance +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "\u6027\u80FD\u306E\u6700\u9069\u5316" toc_priority: 52 --- diff --git a/docs/ja/operations/optimizing-performance/sampling-query-profiler.md b/docs/ja/operations/optimizing-performance/sampling-query-profiler.md index 2610237cbd0..e88b62ce22c 100644 --- a/docs/ja/operations/optimizing-performance/sampling-query-profiler.md +++ b/docs/ja/operations/optimizing-performance/sampling-query-profiler.md @@ -1,47 +1,47 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 54 toc_title: "Query\u30D7\u30ED\u30D5\u30A1\u30A4\u30EA\u30F3\u30B0" --- -# クエリプ {#sampling-query-profiler} +# サンプリングクロファイラ {#sampling-query-profiler} -ClickHouse運転サンプリングプロファイラでの分析クエリを実行します。 使用プロファイラでソースコードのルーチンを用いた最中に頻繁にクエリを実行します。 CPU時間とアイドル時間を含む壁時計の時間をトレースできます。 +ClickHouse運転サンプリングプロファイラでの分析クエリを実行します。 Profilerを使用すると、クエリの実行中に最も頻繁に使用されるソースコードルーチンを検索できます。 CPU時間とアイドル時間を含む壁時計時間をトレースできます。 -プロファイラを使用する: +Profilerを使用するには: -- この [trace\_log](../server-configuration-parameters/settings.md#server_configuration_parameters-trace_log) サーバー設定のセクション。 +- セットアップ [trace\_log](../server-configuration-parameters/settings.md#server_configuration_parameters-trace_log) サーバー構成のセクション。 - このセクションでは、 [trace\_log](../../operations/system-tables.md#system_tables-trace_log) プロファイラーの機能の結果を含むシステムテーブル。 デフォルトで設定されています。 このデータをこのテーブルのみ有効なオペレーティングシステムサーバーです。 後、サーバを再起動ClickHouseないクリーンのテーブルに格納された仮想メモリアドレスが無効になります。 + このセクションでは、 [trace\_log](../../operations/system-tables.md#system_tables-trace_log) プロファイラ機能の結果を含むシステムテーブル。 これは既定で構成されています。 この表のデータは、実行中のサーバーに対してのみ有効です。 後、サーバを再起動ClickHouseないクリーンのテーブルに格納された仮想メモリアドレスが無効になります。 -- この [query\_profiler\_cpu\_time\_period\_ns](../settings/settings.md#query_profiler_cpu_time_period_ns) または [query\_profiler\_real\_time\_period\_ns](../settings/settings.md#query_profiler_real_time_period_ns) 設定。 両方の設定を同時に使用できます。 +- セットアップ [query\_profiler\_cpu\_time\_period\_ns](../settings/settings.md#query_profiler_cpu_time_period_ns) または [query\_profiler\_real\_time\_period\_ns](../settings/settings.md#query_profiler_real_time_period_ns) 設定。 両方の設定を同時に使用できます。 - これらの設定を許可する設定プロファイラータイマー. これらはセッション設定であるため、サーバー全体、個々のユーザーまたはユーザープロファイル、対話型セッション、および個々のクエリごとに異なるサンプリング + これらの設定を許可する設定プロファイラータイマー. これらはセッション設定であるため、サーバー全体、個々のユーザーまたはユーザープロファイル、対話式セッション、および個々のクエリごとに異なるサンプリング周波数 -デフォルトのサンプリング周波数はサンプルや、cpu、リアルタイマーが有効になっています。 この周波数により収集に関する情報を十分にclickhouse。 同時に、この頻度で作業しても、プロファイラーはclickhouseサーバーのパフォーマンスに影響しません。 が必要な場合にプロファイル毎に個別のクエリを利用するようにして高サンプリング周波数です。 +デフォルトのサンプリング周波数はサンプルや、CPU、リアルタイマーが有効になっています。 この頻度により、ClickHouse clusterに関する十分な情報を収集できます。 同時に、この頻度で作業すると、profilerはClickHouse serverのパフォーマンスには影響しません。 が必要な場合にプロファイル毎に個別のクエリを利用するようにして高サンプリング周波数です。 -分析するため `trace_log` システム表: +分析するには `trace_log` システム表: -- インストール `clickhouse-common-static-dbg` パッケージ。 見る [DEBパッケージからのイ](../../getting-started/install.md#install-from-deb-packages). +- インストール `clickhouse-common-static-dbg` パッケージ。 見る [DEBパッケージから](../../getting-started/install.md#install-from-deb-packages). -- によってイントロスペクション機能を許可する [allow\_introspection\_functions](../settings/settings.md#settings-allow_introspection_functions) 設定。 +- によるイントロスペクション関数を許可する。 [allow\_introspection\_functions](../settings/settings.md#settings-allow_introspection_functions) 設定。 - セキュ + セキュリティ上の理由から、introspection関数は既定で無効になっています。 -- を使用 `addressToLine`, `addressToSymbol` と `demangle` [イントロスペクション関数](../../sql-reference/functions/introspection.md) ClickHouseコードで関数名とその位置を取得する。 いくつかのクエリのプロファイルを取得するには、 `trace_log` テーブル。 個々の関数またはスタックトレース全体でデータを集計できます。 +- 使用する `addressToLine`, `addressToSymbol` と `demangle` [内観関数](../../sql-reference/functions/introspection.md) ClickHouseコードで関数名とその位置を取得するには。 いくつかのクエリのプロファイルを取得するには、 `trace_log` テーブル。 個々の関数またはスタックトレース全体でデータを集計できます。 -あなたが視覚化する必要がある場合 `trace_log` 情報、試してみる [flamegraph](../../interfaces/third-party/gui/#clickhouse-flamegraph) と [speedscope](https://github.com/laplab/clickhouse-speedscope). +視覚化する必要がある場合 `trace_log` 情報、試して [フラメグラフ](../../interfaces/third-party/gui/#clickhouse-flamegraph) と [スピードスコープ](https://github.com/laplab/clickhouse-speedscope). -## 例えば {#example} +## 例 {#example} この例では、: -- フィルタ `trace_log` クエリ識別子と現在の日付によるデータ。 +- フィルタ処理 `trace_log` クエリ識別子と現在の日付によるデータ。 - スタックトレースによる集計。 -- イントロスペクション関数を使用して、我々のレポートを取得します: +- イントロスペクション関数を使用して、我々はのレポートを取得します: - シンボルおよび対応するソースコード関数の名前。 - これらの関数のソースコードの場所。 diff --git a/docs/ja/operations/performance-test.md b/docs/ja/operations/performance-test.md index f4ecddc217b..ff3a0192b49 100644 --- a/docs/ja/operations/performance-test.md +++ b/docs/ja/operations/performance-test.md @@ -1,21 +1,21 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 54 toc_title: "\u30CF\u30FC\u30C9\u30A6\u30A7\u30A2\u8A66\u9A13" --- # ClickHouseでハードウェアをテストする方法 {#how-to-test-your-hardware-with-clickhouse} -この命令を実行できますが基本的なclickhouse性能試験はサーバーなしでの設置clickhouseパッケージ。 +この命令を実行できますが基本的なClickHouse性能試験はサーバーなしでの設置ClickHouseパッケージ。 1. に行く “commits” ページ:https://github.com/ClickHouse/ClickHouse/commits/master -2. 最初の緑色のチェックマークまたは緑色の赤い十字をクリックします “ClickHouse Build Check” とをクリック “Details” リンク近く “ClickHouse Build Check”. +2. 最初の緑色のチェックマークまたは緑色の赤十字をクリックします “ClickHouse Build Check” をクリックして “Details” 近くのリンク “ClickHouse Build Check”. いくつかのコミットにはそのようなリンクはありません。 この場合、このリンクを持つ最も近いコミットを選択します。 3. リンクをコピーする “clickhouse” amd64またはaarch64のバイナリ。 -4. サーバーにsshを実行し、wgetでダウンロードします: +4. サーバーにsshし、wgetでダウンロードします: @@ -36,7 +36,7 @@ toc_title: "\u30CF\u30FC\u30C9\u30A6\u30A7\u30A2\u8A66\u9A13" wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/programs/server/config.d/path.xml -O config.d/path.xml wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/programs/server/config.d/log_to_console.xml -O config.d/log_to_console.xml -1. Benchmarkファイル: +1. ダウンロードファイルのベンチマーク: @@ -44,7 +44,7 @@ toc_title: "\u30CF\u30FC\u30C9\u30A6\u30A7\u30A2\u8A66\u9A13" chmod a+x benchmark-new.sh wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/benchmark/clickhouse/queries.sql -1. ダウンロード試験データによると [Yandexの。Metricaデータセット](../getting-started/example-datasets/metrica.md) 指示 (“hits” 100万行を含むテーブル）。 +1. に従うダウンロードテストデータ [Yandex.メトリカデータセット](../getting-started/example-datasets/metrica.md) 命令 (“hits” 100万行を含むテーブル）。 @@ -58,14 +58,14 @@ toc_title: "\u30CF\u30FC\u30C9\u30A6\u30A7\u30A2\u8A66\u9A13" ./clickhouse server -1. データを確認する：別のターミナルのサーバーへのssh +1. データを確認する：別の端末のサーバーへのssh ./clickhouse client --query "SELECT count() FROM hits_100m_obfuscated" 100000000 -1. 編集するbenchmark-new.sh,変更 “clickhouse-client” に “./clickhouse client” と追加 “–max\_memory\_usage 100000000000” パラメータ。 +1. 編集benchmark-new.sh,変更 `clickhouse-client` に `./clickhouse client` と追加 `–-max_memory_usage 100000000000` パラメータ。 @@ -77,6 +77,6 @@ toc_title: "\u30CF\u30FC\u30C9\u30A6\u30A7\u30A2\u8A66\u9A13" ./benchmark-new.sh hits_100m_obfuscated -1. ハードウェア構成に関する番号と情報を以下に送信しますclickhouse-feedback@yandex-team.com +1. ハードウェア構成に関する番号と情報を次の宛先に送信しますclickhouse-feedback@yandex-team.com -すべての結果はここに掲載されています：https://clickhouse。ﾂつｨﾂ姪“ﾂつ”ﾂ債ﾂづｭﾂつｹhtml +すべての結果をこちらに発表します:https://clickhouse.技術/基準/ハードウェア/ diff --git a/docs/ja/operations/quotas.md b/docs/ja/operations/quotas.md index bec5f1ebda0..bca708cd041 100644 --- a/docs/ja/operations/quotas.md +++ b/docs/ja/operations/quotas.md @@ -1,20 +1,20 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 51 toc_title: "\u30AF\u30A9\u30FC\u30BF" --- # クォータ {#quotas} -クォータを使用すると、一定期間にわたってリソースの使用を制限したり、単にリソースの使用を追跡したりできます。 -クォータは、ユーザー設定で設定されます。 これは通常 ‘users.xml’. +クォータを使用すると、一定期間のリソース使用量を制限したり、リソースの使用を追跡したりできます。 +クォータはユーザー設定で設定されます。 ‘users.xml’. -システムには、単一のクエリの複雑さを制限する機能もあります。 セクションを見る “Restrictions on query complexity”). +このシステムには、単一のクエリの複雑さを制限する機能もあります。 セクションを参照 “Restrictions on query complexity”). クエリの複雑さの制限とは対照的に、クォータ: -- 単一のクエリを制限するのではなく、ある期間にわたって実行できるクエリのセットに制限を設定します。 +- 単一のクエリを制限するのではなく、一定期間にわたって実行できるクエリのセットに制限を設定します。 - 口座のために費やすべてのリモートサーバーのための分散クエリ処となります。 のセクションを見てみましょう ‘users.xml’ クォータを定義するファイル。 @@ -39,8 +39,8 @@ toc_title: "\u30AF\u30A9\u30FC\u30BF" ``` -デフォルトでは、クォータだけでトラック資源の消費のそれぞれの時間を狭く限定することなく、利用 -各間隔で計算されたリソース消費は、各要求の後にサーバーログに出力されます。 +既定では、クォータは、使用量を制限することなく、各時間のリソース消費量を追跡します。 +各間隔ごとに計算されたリソース消費量は、各要求の後にサーバーログに出力されます。 ``` xml @@ -68,9 +68,9 @@ toc_title: "\u30AF\u30A9\u30FC\u30BF" ``` -のための ‘statbox’ クォータ、制限は、毎時および24時間（86,400秒）ごとに設定されます。 時間間隔は、実装定義の固定moment間から開始してカウントされます。 言い換えれば、24時間の間隔は必ずしも真夜中に始まるとは限りません。 +のために ‘statbox’ クォータ、制限は、時間ごとおよび24時間ごと(86,400秒)に設定されます。 時間間隔は、実装定義の固定モーメントから開始してカウントされます。 つまり、24時間の間隔は必ずしも深夜に開始されるわけではありません。 -間隔が終了すると、収集された値はすべてクリアされます。 次の時間には、クォータの計算が最初からやり直されます。 +間隔が終了すると、収集された値はすべてクリアされます。 次の時間は、クォータの計算がやり直されます。 制限できる金額は次のとおりです: @@ -78,15 +78,15 @@ toc_title: "\u30AF\u30A9\u30FC\u30BF" `errors` – The number of queries that threw an exception. -`result_rows` – The total number of rows given as the result. +`result_rows` – The total number of rows given as a result. -`read_rows` – The total number of source rows read from tables for running the query, on all remote servers. +`read_rows` – The total number of source rows read from tables for running the query on all remote servers. `execution_time` – The total query execution time, in seconds (wall time). -制限を少なくとも一回の間隔で超えた場合は、制限を超過したテキスト、間隔、および新しい間隔の開始時(クエリを再度送信できる場合)についての +制限を超えた場合は、どの制限を超えたか、どの間隔を超えたか、および新しい間隔が開始されたとき(クエリを再度送信できるとき)に関するテキス -クォータは以下を使用できます “quota key” 複数のキーのリソースを個別に報告する機能。 これの例は次のとおりです: +クォータは、 “quota key” 複数のキーのリソースを個別に報告する機能。 これの例を次に示します: ``` xml @@ -97,15 +97,15 @@ toc_title: "\u30AF\u30A9\u30FC\u30BF" so the quota will be counted separately for each username. Using keys makes sense only if quota_key is transmitted by the program, not by a user. - You can also write so the IP address is used as the quota key. + You can also write , so the IP address is used as the quota key. (But keep in mind that users can change the IPv6 address fairly easily.) --> ``` -クォータはユーザーに割り当てられます。 ‘users’ 設定のセクション。 セクションを見る “Access rights”. +クォータは ‘users’ 設定のセクション。 セクションを参照 “Access rights”. -分散クエリ処理では、累積金額がリクエスタサーバに格納されます。 ここでは、ユーザーが別のサーバーの定員がありま “start over”. +分散クエリ処理の場合、累積金額は要求元サーバーに格納されます。 ここでは、ユーザーが別のサーバーの定員がありま “start over”. サーバーを再起動すると、クォータがリセットされます。 diff --git a/docs/ja/operations/requirements.md b/docs/ja/operations/requirements.md index ac7c35e2dea..f4998bda005 100644 --- a/docs/ja/operations/requirements.md +++ b/docs/ja/operations/requirements.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 44 toc_title: "\u8981\u4EF6" --- @@ -9,53 +9,53 @@ toc_title: "\u8981\u4EF6" ## CPU {#cpu} -Prebuilt debパッケージからインストールする場合は、X86\_64アーキテクチャを持つCPUを使用し、SSE4.2命令をサポートします。 Sse4.2をサポートしていない、またはAArch64またはPowerPC64LEアーキテクチャを持つプロセッサでClickHouseを実行するには、ソースからClickHouseをビルドする必要があり +ビルド済みのdebパッケージからインストールするには、X86\_64アーキテクチャのCPUを使用し、SSE4.2命令をサポートします。 走ClickHouseプロセッサをサポートしていないSSE4.2てAArch64はPowerPC64LE建築、協力して進めることが必要でありClickHouseから。 -ClickHouseは、並列処理を実装し、利用可能なすべてのハードウェアリソースを使用します。 プロセッサを選択するときは、コア数が多い構成では、コア数が少ない構成よりもクロックレートが低く、クロックレートが高い構成では、クリックハウ 例えば、16MHzの2600コアは、8MHzの3600コアよりも好ましい。 +ClickHouseを実装した並列データの処理-利用のすべてのハードウェア資料を備えています。 プロセッサを選択するときは、ClickHouseが多数のコアを持つ構成でより効率的に動作するが、コアが少なくクロックレートが高い構成ではatよりも低いク たとえば、16 2600MHzのコアは、8 3600MHzのコアよりも好ましいです。 -の使用 **ターボブースト** と **ハイパースレッド** 技術が推奨されます。 典型的な負荷でパフォーマンスが大幅に向上します。 +使用することをお勧めします **ターボブースト** と **ハイパースレッド** テクノロジー 大幅なパフォーマンス向上を図は、典型的な負荷も大きくなっていました。 ## RAM {#ram} -些細なクエリを実行するには、最小4gbのramを使用することをお勧めします。 のclickhouseサーバーへアクセスできる走りはるかに小さなramが要求されるメモリ処理ます。 +些細でないクエリを実行するには、最小4GBのRAMを使用することをお勧めします。 ClickHouseサーバーは、はるかに少ない量のRAMで実行できますが、クエリを処理するためにメモリが必要です。 -必要なramの容量は次のとおりです: +必要なRAM容量は次のとおりです: - クエリの複雑さ。 - クエリで処理されるデータの量。 -計算に必要な量のram、推定値のサイズを一時的にデータのための [GROUP BY](../sql-reference/statements/select.md#select-group-by-clause), [DISTINCT](../sql-reference/statements/select.md#select-distinct), [JOIN](../sql-reference/statements/select.md#select-join) そしてあなたが使用する他の操作。 +RAMの必要量を計算するには、次のような一時データのサイズを推定する必要があります [GROUP BY](../sql-reference/statements/select/group-by.md#select-group-by-clause), [DISTINCT](../sql-reference/statements/select/distinct.md#select-distinct), [JOIN](../sql-reference/statements/select/join.md#select-join) そしてあなたが使用する他の操作。 -ClickHouseは、一時的なデータに外部メモリを使用できます。 見る [外部メモリによるグループ化](../sql-reference/statements/select.md#select-group-by-in-external-memory) 詳細については。 +ClickHouseは一時データに外部メモリを使用できます。 見る [外部メモリのGROUP BY](../sql-reference/statements/select/group-by.md#select-group-by-in-external-memory) 詳細については. ## Swapファイル {#swap-file} -運用環境用のスワップファイルを無効にします。 +運用環境のスワップファイルを無効にします。 -## 格納サブシステム {#storage-subsystem} +## Storageサブシステム {#storage-subsystem} -ClickHouseをインストールするには2GBの空きディスク容量が必要です。 +ClickHouseをインストールするには、2GBの空きディスク領域が必要です。 -データに必要なストレージ容量は、個別に計算する必要があります。 評価には: +データに必要なストレージ容量は、別々に計算する必要があります。 評価には: - データ量の推定。 - データのサンプルを取得し、そこから行の平均サイズを取得できます。 次に、値に格納する予定の行の数を掛けます。 + データのサンプルを取得し、そこから行の平均サイズを取得できます。 次に、値に格納する予定の行数を掛けます。 - データ圧縮係数。 - データ圧縮係数を推定するには、データのサンプルをclickhouseにロードし、データの実際のサイズと格納されているテーブルのサイズを比較します。 たとえば、通常、クリックストリームデータは6-10倍圧縮されます。 + データ圧縮係数を推定するには、データのサンプルをClickHouseに読み込み、データの実際のサイズと格納されているテーブルのサイズを比較します。 たとえば、clickstreamデータは通常6-10回圧縮されます。 -保存するデータの最終ボリュームを計算するには、推定データボリュームに圧縮係数を適用します。 複数のレプリカにデータを格納する場合は、推定ボリュームにレプリカの数を掛けます。 +格納するデータの最終ボリュームを計算するには、推定データボリュームに圧縮係数を適用します。 複数のレプリカにデータを格納する場合は、推定ボリュームにレプリカの数を掛けます。 -## ネットワーク {#network} +## ネット {#network} -可能であれば、10g以上のネットワークを使用してください。 +可能であれば、10G以上のネットワークを使用してください。 -ネットワーク帯域幅は、大量の中間データを使用して分散クエリを処理する場合に重要です。 また、ネットワーク速度に影響する複製プロセス。 +大量の中間データを含む分散クエリを処理するには、ネットワーク帯域幅が重要です。 また、ネットワーク速度に影響する複製プロセス。 ## ソフト {#software} -ClickHouseが開発されたLinuxの家族システムです。 推奨されるLinuxの配布はUbuntuです。 その `tzdata` パッケージを設置する必要がある。 +ClickHouseの開発を中心に、Linuxの家族システムです。 推奨されるLinuxディストリビュ その `tzdata` パッケ ClickHouse働きかけることができ、その他業務システム。 の詳細を参照してください [はじめに](../getting-started/index.md) ドキュメントのセクション。 diff --git a/docs/ja/operations/server-configuration-parameters/index.md b/docs/ja/operations/server-configuration-parameters/index.md index 1b16e96c288..f8e63412a1c 100644 --- a/docs/ja/operations/server-configuration-parameters/index.md +++ b/docs/ja/operations/server-configuration-parameters/index.md @@ -1,19 +1,19 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_folder_title: Server Configuration Parameters +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "\u30B5\u30FC\u30D0\u69CB\u6210\u30D1\u30E9\u30E1\u30FC\u30BF" toc_priority: 54 -toc_title: "\u5C0E\u5165" +toc_title: "\u306F\u3058\u3081\u306B" --- -# サーバ設定パラメータ {#server-settings} +# サーバ構成パラメータ {#server-settings} ここでの記述が含まれてサーバーの設定を変更することはできないのセッションor検索。 これらの設定は `config.xml` ClickHouseサーバー上のファイル。 -その他の設定については、 “[設定](../settings/index.md#settings)” セクション。 +その他の設定については、 “[設定](../settings/index.md#session-settings-intro)” セクション -設定を勉強する前に、 [設定ファイル](../configuration-files.md#configuration_files) の使用に注意してください。 `incl` と `optional` 属性)。 +設定を勉強する前に、 [設定ファイル](../configuration-files.md#configuration_files) セクションと置換の使用に注意してください( `incl` と `optional` 属性）。 [元の記事](https://clickhouse.tech/docs/en/operations/server_configuration_parameters/) diff --git a/docs/ja/operations/server-configuration-parameters/settings.md b/docs/ja/operations/server-configuration-parameters/settings.md index b89883bab62..2b460e8aca6 100644 --- a/docs/ja/operations/server-configuration-parameters/settings.md +++ b/docs/ja/operations/server-configuration-parameters/settings.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 57 toc_title: "\u30B5\u30FC\u30D0\u30FC\u8A2D\u5B9A" --- @@ -9,13 +9,13 @@ toc_title: "\u30B5\u30FC\u30D0\u30FC\u8A2D\u5B9A" ## builtin\_dictionaries\_reload\_interval {#builtin-dictionaries-reload-interval} -組み込みの辞書を再ロードする前の秒単位の間隔。 +組み込み辞書を再ロードするまでの秒単位の間隔。 -クリックハウスは、内蔵の辞書ごとにx秒をリロードします。 これにより、辞書の編集が可能になります “on the fly” サーバーを再起動せずに。 +ClickHouseはx秒ごとに組み込みの辞書を再読み込みします。 これにより、辞書の編集が可能になります “on the fly” サーバーを再起動せずに。 -デフォルト値:3600. +デフォルト値は3600です。 -**例えば** +**例** ``` xml 3600 @@ -23,10 +23,10 @@ toc_title: "\u30B5\u30FC\u30D0\u30FC\u8A2D\u5B9A" ## 圧縮 {#server-settings-compression} -以下のためのデータ圧縮設定 [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md)-エンジンテーブル。 +データ圧縮の設定 [メルゲツリー](../../engines/table-engines/mergetree-family/mergetree.md)-エンジンテーブル。 !!! warning "警告" - ClickHouseを使用し始めたばかりの場合は使用しないでください。 + ClickHouseの使用を開始したばかりの場合は、使用しないでください。 構成テンプレート: @@ -47,16 +47,16 @@ toc_title: "\u30B5\u30FC\u30D0\u30FC\u8A2D\u5B9A" - `min_part_size_ratio` – The ratio of the data part size to the table size. - `method` – Compression method. Acceptable values: `lz4` または `zstd`. -複数を設定できます `` セクション。 +複数の設定が可能です `` セクション -条件が満たされたときの動作: +条件を満たした場合のアクション: -- データパーツが条件セットと一致する場合、clickhouseは指定された圧縮方法を使用します。 -- データパートが複数の条件セットと一致する場合、clickhouseは最初に一致した条件セットを使用します。 +- データ部分が条件セットに一致する場合、ClickHouseは指定された圧縮方法を使用します。 +- データパーツが複数の条件セットに一致する場合、ClickHouseは最初に一致した条件セットを使用します。 -デー `lz4` 圧縮。 +データパーツの条件が満たされていない場合、ClickHouseは `lz4` 圧縮。 -**例えば** +**例** ``` xml @@ -74,7 +74,7 @@ toc_title: "\u30B5\u30FC\u30D0\u30FC\u8A2D\u5B9A" データベースのリストを取得するには、 [SHOW DATABASES](../../sql-reference/statements/show.md#show-databases) クエリ。 -**例えば** +**例** ``` xml default @@ -84,9 +84,9 @@ toc_title: "\u30B5\u30FC\u30D0\u30FC\u8A2D\u5B9A" 既定の設定プロファイル。 -設定プロファイルはパラ `user_config`. +設定プロファイル内のファイルで指定されたパラメータ `user_config`. -**例えば** +**例** ``` xml default @@ -94,16 +94,16 @@ toc_title: "\u30B5\u30FC\u30D0\u30FC\u8A2D\u5B9A" ## dictionaries\_config {#server_configuration_parameters-dictionaries_config} -外部ディクショナリの設定ファイルへのパス。 +外部辞書の設定ファイルへのパス。 パス: -- サーバー設定ファイルに対する絶対パスまたは相対パスを指定します。 +- 絶対パスまたはサーバー設定ファイルに対する相対パスを指定します。 - のパスを含むことができワイルドカード\*や?. -また見なさい “[外部辞書](../../sql-reference/dictionaries/external-dictionaries/external-dicts.md)”. +も参照。 “[外部辞書](../../sql-reference/dictionaries/external-dictionaries/external-dicts.md)”. -**例えば** +**例** ``` xml *_dictionary.xml @@ -111,15 +111,15 @@ toc_title: "\u30B5\u30FC\u30D0\u30FC\u8A2D\u5B9A" ## dictionaries\_lazy\_load {#server_configuration_parameters-dictionaries_lazy_load} -辞書の遅延ロード。 +辞書の遅延読み込み。 -もし `true` その後、各辞書は最初の使用時に作成されます。 辞書の作成に失敗した場合、辞書を使用していた関数は例外をスローします。 +もし `true` その後、各辞書は、最初の使用時に作成されます。 辞書の作成に失敗した場合、辞書を使用していた関数は例外をスローします。 -もし `false` すべての辞書は、サーバーの起動時に作成され、エラーが発生した場合、サーバーはシャットダウンされます。 +もし `false` すべての辞書は、サーバーの起動時に作成され、エラーがある場合は、サーバーがシャットダウンされます。 -デフォルトは `true`. +既定値は次のとおりです `true`. -**例えば** +**例** ``` xml true @@ -127,9 +127,9 @@ toc_title: "\u30B5\u30FC\u30D0\u30FC\u8A2D\u5B9A" ## format\_schema\_path {#server_configuration_parameters-format_schema_path} -入力データのスキームを持つディレクトリへのパス。 [CapnProto](../../interfaces/formats.md#capnproto) フォーマット。 +入力データのスキーマを含むディレクトリへのパス。 [CapnProto](../../interfaces/formats.md#capnproto) 形式。 -**例えば** +**例** ``` xml @@ -147,14 +147,14 @@ toc_title: "\u30B5\u30FC\u30D0\u30FC\u8A2D\u5B9A" - interval – The interval for sending, in seconds. - timeout – The timeout for sending data, in seconds. - root\_path – Prefix for keys. -- metrics – Sending data from the [システム。指標](../../operations/system-tables.md#system_tables-metrics) テーブル。 -- events – Sending deltas data accumulated for the time period from the [システム。イベント](../../operations/system-tables.md#system_tables-events) テーブル。 -- events\_cumulative – Sending cumulative data from the [システム。イベント](../../operations/system-tables.md#system_tables-events) テーブル。 -- asynchronous\_metrics – Sending data from the [システム。asynchronous\_metrics](../../operations/system-tables.md#system_tables-asynchronous_metrics) テーブル。 +- metrics – Sending data from the [システムメトリック](../../operations/system-tables.md#system_tables-metrics) テーブル。 +- events – Sending deltas data accumulated for the time period from the [システムイベント](../../operations/system-tables.md#system_tables-events) テーブル。 +- events\_cumulative – Sending cumulative data from the [システムイベント](../../operations/system-tables.md#system_tables-events) テーブル。 +- asynchronous\_metrics – Sending data from the [システムasynchronous\_metrics](../../operations/system-tables.md#system_tables-asynchronous_metrics) テーブル。 -複数を設定できます `` 句。 たとえば、異なる間隔で異なるデータを送信するためにこれを使用できます。 +複数の設定が可能です `` 句。 たとえば、異なる間隔で異なるデータを送信するためにこれを使用できます。 -**例えば** +**例** ``` xml @@ -172,11 +172,11 @@ toc_title: "\u30B5\u30FC\u30D0\u30FC\u8A2D\u5B9A" ## graphite\_rollup {#server_configuration_parameters-graphite-rollup} -グラファイトの間引きデータの設定。 +グラファイトのデータを薄くする設定。 -詳細については、 [グラフィットメールグツリー](../../engines/table-engines/mergetree-family/graphitemergetree.md). +詳細は、を参照してください [GraphiteMergeTree](../../engines/table-engines/mergetree-family/graphitemergetree.md). -**例えば** +**例** ``` xml @@ -202,11 +202,11 @@ toc_title: "\u30B5\u30FC\u30D0\u30FC\u8A2D\u5B9A" HTTP経由でサーバーに接続するためのポート。 -もし `https_port` が指定される。, [openSSL](#server_configuration_parameters-openssl) 構成する必要があります。 +もし `https_port` 指定される, [openSSL](#server_configuration_parameters-openssl) 設定する必要があります。 -もし `http_port` が指定されている場合、OpenSSL設定が設定されていても、その設定は無視される。 +もし `http_port` が指定されている場合、OpenSSL設定が設定されていても無視されます。 -**例えば** +**例** ``` xml 0000 @@ -215,9 +215,9 @@ HTTP経由でサーバーに接続するためのポート。 ## http\_server\_default\_response {#server_configuration_parameters-http_server_default_response} ClickHouse HTTP(s)サーバーにアクセスするときにデフォルトで表示されるページ。 -デフォルト値は “Ok.” （最後にラインフィード付き) +既定値は次のとおりです “Ok.” （最後に改行があります) -**例えば** +**例** 開く `https://tabix.io/` アクセス時 `http://localhost: http_port`. @@ -229,11 +229,11 @@ ClickHouse HTTP(s)サーバーにアクセスするときにデフォルトで ## include\_from {#server_configuration_parameters-include_from} -置換を伴うファイルへのパス。 +置換されたファイルへのパス。 -詳細については、以下を参照してください “[設定ファイル](../configuration-files.md#configuration_files)”. +詳細については “[設定ファイル](../configuration-files.md#configuration_files)”. -**例えば** +**例** ``` xml /etc/metrica.xml @@ -241,9 +241,9 @@ ClickHouse HTTP(s)サーバーにアクセスするときにデフォルトで ## interserver\_http\_port {#interserver-http-port} -ClickHouseサーバ間でデータを交換するためのポート。 +ClickHouseサーバー間でデータを交換するポート。 -**例えば** +**例** ``` xml 9009 @@ -253,11 +253,11 @@ ClickHouseサーバ間でデータを交換するためのポート。 このサーバーへのアクセスに他のサーバーが使用できるホスト名。 -省略された場合、それは同じ方法で定義されます `hostname-f` 司令部 +省略された場合、これは `hostname-f` コマンド -特定のネットワー +特定のネッ -**例えば** +**例** ``` xml example.yandex.ru @@ -265,15 +265,15 @@ ClickHouseサーバ間でデータを交換するためのポート。 ## interserver\_http\_credentials {#server-settings-interserver-http-credentials} -認証に使用するユーザー名とパスワード [複製](../../engines/table-engines/mergetree-family/replication.md) レプリケートされた\*エンジン。 これらの資格情報は、レプリカ間の通信にのみ使用され、ClickHouseクライアントの資格情報とは無関係です。 サーバーにあるチェックにこれらの資格の接続にはレプリカと同じ資格を接続する場合はその他のレプリカ. なので、これらの資格を設定する同じすべてのレプリカ、クラスター -デフォルトでは、認証は使用されません。 +認証時に使用されるユーザー名とパスワード [複製](../../engines/table-engines/mergetree-family/replication.md) 複製された\*エンジンで。 これらの資格情報は、レプリカ間の通信にのみ使用され、ClickHouseクライアントの資格情報とは無関係です。 サーバーにあるチェックにこれらの資格の接続にはレプリカと同じ資格を接続する場合はその他のレプリカ. なので、これらの資格を設定する同じすべてのレプリカ、クラスター +既定では、認証は使用されません。 -このセクショ: +このセクションでは以下のパラメータ: - `user` — username. - `password` — password. -**例えば** +**例** ``` xml @@ -284,9 +284,9 @@ ClickHouseサーバ間でデータを交換するためのポート。 ## keep\_alive\_timeout {#keep-alive-timeout} -接続を閉じる前に、clickhouseが着信要求を待機する秒数。 デフォルトは3秒です。 +ClickHouseが接続を閉じる前に受信要求を待機する秒数。 既定値は3秒です。 -**例えば** +**例** ``` xml 3 @@ -294,7 +294,7 @@ ClickHouseサーバ間でデータを交換するためのポート。 ## listen\_host {#server_configuration_parameters-listen_host} -要求元のホストの制限。 したい場合はサーバーの回答をしているが、それらを指定し `::`. +要求元のホストに対する制限。 したい場合はサーバーの回答をしているが、それらを指定し `::`. 例: @@ -305,17 +305,17 @@ ClickHouseサーバ間でデータを交換するためのポート。 ## ロガー {#server_configuration_parameters-logger} -ログの設定。 +ログ設定。 キー: - level – Logging level. Acceptable values: `trace`, `debug`, `information`, `warning`, `error`. - log – The log file. Contains all the entries according to `level`. - errorlog – Error log file. -- size – Size of the file. Applies to `log`と`errorlog`. ファイルが届くと `size`、ClickHouseのアーカイブと名前を変更し、その場所に新しいログファイルを作成します。 +- size – Size of the file. Applies to `log`と`errorlog`. ファイルが到達すると `size`、ClickHouseはアーカイブし、それの名前を変更し、その場所に新しいログファイルを作成します。 - count – The number of archived log files that ClickHouse stores. -**例えば** +**例** ``` xml @@ -327,7 +327,7 @@ ClickHouseサーバ間でデータを交換するためのポート。 ``` -Syslogへの書き込みもサポートされています。 設定例: +Syslogへの書き込みもサポートされています。 設定の例: ``` xml @@ -346,19 +346,19 @@ Syslogへの書き込みもサポートされています。 設定例: - use\_syslog — Required setting if you want to write to the syslog. - address — The host\[:port\] of syslogd. If omitted, the local daemon is used. - hostname — Optional. The name of the host that logs are sent from. -- facility — [Syslog機能キーワード](https://en.wikipedia.org/wiki/Syslog#Facility) 大文字で “LOG\_” 接頭辞: (`LOG_USER`, `LOG_DAEMON`, `LOG_LOCAL3`、というように)。 - デフォルト値: `LOG_USER` もし `address` が指定される。, `LOG_DAEMON otherwise.` +- facility — [Syslog機能キーワード](https://en.wikipedia.org/wiki/Syslog#Facility) 大文字では “LOG\_” 接頭辞: (`LOG_USER`, `LOG_DAEMON`, `LOG_LOCAL3`、というように）。 + デフォルト値: `LOG_USER` もし `address` 指定される, `LOG_DAEMON otherwise.` - format – Message format. Possible values: `bsd` と `syslog.` ## マクロ {#macros} -パラメータの置換のために再現します。 +複製されたテーブルのパラメーター置換。 ければ省略することができ複製のテーブルは使用しておりません。 -詳細については、以下を参照してください “[複製テーブルの作成](../../engines/table-engines/mergetree-family/replication.md)”. +詳細については “[複製テーブルの作成](../../engines/table-engines/mergetree-family/replication.md)”. -**例えば** +**例** ``` xml @@ -366,11 +366,11 @@ Syslogへの書き込みもサポートされています。 設定例: ## mark\_cache\_size {#server-mark-cache-size} -約サイズ(バイトのキャッシュのマークの使用によりテーブルエンジンの [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) 家族 +テーブルエンジンが使用するマークのキャッシュのおおよそのサイズ(バイト単位) [メルゲツリー](../../engines/table-engines/mergetree-family/mergetree.md) 家族だ -キャッシュの共有のサーバーメモリが割り当てられます。 キャッシュサイズは5368709120以上である必要があります。 +キャッシュの共有のサーバーメモリが割り当てられます。 キャッシュサイズは、少なくとも5368709120である必要があります。 -**例えば** +**例** ``` xml 5368709120 @@ -378,9 +378,9 @@ Syslogへの書き込みもサポートされています。 設定例: ## max\_concurrent\_queries {#max-concurrent-queries} -同時に処理された要求の最大数。 +同時に処理される要求の最大数。 -**例えば** +**例** ``` xml 100 @@ -390,7 +390,7 @@ Syslogへの書き込みもサポートされています。 設定例: 受信接続の最大数。 -**例えば** +**例** ``` xml 4096 @@ -400,11 +400,11 @@ Syslogへの書き込みもサポートされています。 設定例: 開いているファイルの最大数。 -デフォルトでは: `maximum`. +既定では: `maximum`. -Mac OS Xでこのオプションを使用することをお勧めします。 `getrlimit()` 関数は不正な値を返します。 +このオプションをMac OS Xで使用することをお勧めします。 `getrlimit()` 関数は、誤った値を返します。 -**例えば** +**例** ``` xml 262144 @@ -414,15 +414,15 @@ Mac OS Xでこのオプションを使用することをお勧めします。 `g テーブルの削除に関する制限。 -のサイズ [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) テーブルを超え `max_table_size_to_drop` （バイト単位）、ドロップクエリを使用して削除することはできません。 +のサイズが [メルゲツリー](../../engines/table-engines/mergetree-family/mergetree.md) テーブル超過 `max_table_size_to_drop` （バイト単位）、ドロップクエリを使用して削除することはできません。 -それでもclickhouseサーバーを再起動せずにテーブルを削除する必要がある場合は、 `/flags/force_drop_table` ドロップクエリを実行します。 +ClickHouseサーバーを再起動せずにテーブルを削除する必要がある場合は、 `/flags/force_drop_table` DROPクエリをファイルして実行します。 -デフォルト値:50gb. +デフォルト値:50GB。 -値0は、制限なしにすべてのテーブルを削除できることを意味します。 +値0は、制限なしですべてのテーブルを削除できることを意味します。 -**例えば** +**例** ``` xml 0 @@ -430,11 +430,11 @@ Mac OS Xでこのオプションを使用することをお勧めします。 `g ## merge\_tree {#server_configuration_parameters-merge_tree} -のテーブルのための微調整 [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md). +テーブルの微調整 [メルゲツリー](../../engines/table-engines/mergetree-family/mergetree.md). -詳細については、“mergetreesettings”を参照してください。hヘッダファイル。 +詳細については、MergeTreeSettingsを参照してください。hヘッダファイル。 -**例えば** +**例** ``` xml @@ -446,14 +446,14 @@ Mac OS Xでこのオプションを使用することをお勧めします。 `g SSLクライアント/サーバー構成。 -SSLのサポートは、 `libpoco` ライブラリ。 ユーザーインターフェイスはファイルに記述 [SSLManager.h](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h) +SSLのサポートは以下によって提供されます `libpoco` 図書館 ユーザーインターフェイスはファイルに記述 [SSLManager.h](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h) サーバー/クライアント設定のキー: - privateKeyFile – The path to the file with the secret key of the PEM certificate. The file may contain a key and certificate at the same time. - certificateFile – The path to the client/server certificate file in PEM format. You can omit it if `privateKeyFile` 証明書が含まれています。 - caConfig – The path to the file or directory that contains trusted root certificates. -- verificationMode – The method for checking the node’s certificates. Details are in the description of the [文脈](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/Context.h) クラス。 可能な値: `none`, `relaxed`, `strict`, `once`. +- verificationMode – The method for checking the node's certificates. Details are in the description of the [文脈](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/Context.h) クラス 可能な値: `none`, `relaxed`, `strict`, `once`. - verificationDepth – The maximum length of the verification chain. Verification will fail if the certificate chain length exceeds the set value. - loadDefaultCAFile – Indicates that built-in CA certificates for OpenSSL will be used. Acceptable values: `true`, `false`. \| - cipherList – Supported OpenSSL encryptions. For example: `ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH`. @@ -465,7 +465,7 @@ SSLのサポートは、 `libpoco` ライブラリ。 ユーザーインター - requireTLSv1 – Require a TLSv1 connection. Acceptable values: `true`, `false`. - requireTLSv1\_1 – Require a TLSv1.1 connection. Acceptable values: `true`, `false`. - requireTLSv1 – Require a TLSv1.2 connection. Acceptable values: `true`, `false`. -- fips – Activates OpenSSL FIPS mode. Supported if the library’s OpenSSL version supports FIPS. +- fips – Activates OpenSSL FIPS mode. Supported if the library's OpenSSL version supports FIPS. - privateKeyPassphraseHandler – Class (PrivateKeyPassphraseHandler subclass) that requests the passphrase for accessing the private key. For example: ``, `KeyFileHandler`, `test`, ``. - invalidCertificateHandler – Class (a subclass of CertificateHandler) for verifying invalid certificates. For example: ` ConsoleCertificateHandler ` . - disableProtocols – Protocols that are not allowed to use. @@ -503,9 +503,9 @@ SSLのサポートは、 `libpoco` ライブラリ。 ユーザーインター ## part\_log {#server_configuration_parameters-part-log} -関連付けられているログイベント [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md). たとえば、データの追加やマージなどです。 利用できるログを統合アルゴリズムと比較しています。 マージプロセスを視覚化できます。 +関連付けられたイベントのログ記録 [メルゲツリー](../../engines/table-engines/mergetree-family/mergetree.md). たとえば、データの追加やマージなどです。 利用できるログを統合アルゴリズムと比較しています。 マージプロセスを視覚化できます。 -クエリはログに記録されます [システム。part\_log](../../operations/system-tables.md#system_tables-part-log) テーブル、別のファイルではありません。 このテーブルの名前を設定することができます `table` パラメータ(下記参照)。 +クエリは [システムpart\_log](../../operations/system-tables.md#system_tables-part-log) 別のファイルではなく、テーブル。 このテーブルの名前は、 `table` パラメータ(下記参照)。 以下のパラメータの設定ロギング: @@ -514,7 +514,7 @@ SSLのサポートは、 `libpoco` ライブラリ。 ユーザーインター - `partition_by` – Sets a [カスタム分割キー](../../engines/table-engines/mergetree-family/custom-partitioning-key.md). - `flush_interval_milliseconds` – Interval for flushing data from the buffer in memory to the table. -**例えば** +**例** ``` xml @@ -529,31 +529,55 @@ SSLのサポートは、 `libpoco` ライブラリ。 ユーザーインター データを含むディレクトリへのパス。 -!!! note "メモ" +!!! note "注" 末尾のスラッシュは必須です。 -**例えば** +**例** ``` xml /var/lib/clickhouse/ ``` -## クエリーログ {#server_configuration_parameters-query-log} +## プロメテウス {#server_configuration_parameters-prometheus} -で受信したロギングクエリの設定 [log\_queries=1](../settings/settings.md) 設定。 +スクレイピングの指標データの公開 [プロメテウス](https://prometheus.io). -クエリはログに記録されます [システム。クエリーログ](../../operations/system-tables.md#system_tables-query_log) テーブル、別のファイルではありません。 テーブルの名前を変更することができます `table` パラメータ(下記参照)。 +設定: + +- `endpoint` – HTTP endpoint for scraping metrics by prometheus server. Start from ‘/’. +- `port` – Port for `endpoint`. +- `metrics` – Flag that sets to expose metrics from the [システムメトリック](../system-tables.md#system_tables-metrics) テーブル。 +- `events` – Flag that sets to expose metrics from the [システムイベント](../system-tables.md#system_tables-events) テーブル。 +- `asynchronous_metrics` – Flag that sets to expose current metrics values from the [システムasynchronous\_metrics](../system-tables.md#system_tables-asynchronous_metrics) テーブル。 + +**例** + +``` xml + + /metrics + 8001 + true + true + true + +``` + +## query\_log {#server_configuration_parameters-query-log} + +クエリをログに記録するための設定 [log\_queries=1](../settings/settings.md) 設定。 + +クエリは [システムquery\_log](../../operations/system-tables.md#system_tables-query_log) 別のファイルではなく、テーブル。 テーブルの名前を変更することができます。 `table` パラメータ(下記参照)。 以下のパラメータの設定ロギング: - `database` – Name of the database. - `table` – Name of the system table the queries will be logged in. -- `partition_by` – Sets a [カスタム分割キー](../../engines/table-engines/mergetree-family/custom-partitioning-key.md) テーブルのため。 +- `partition_by` – Sets a [カスタム分割キー](../../engines/table-engines/mergetree-family/custom-partitioning-key.md) テーブルのために。 - `flush_interval_milliseconds` – Interval for flushing data from the buffer in memory to the table. -テーブルが存在しない場合、clickhouseはそれを作成します。 clickhouseサーバーが更新されたときにクエリログの構造が変更された場合、古い構造を持つテーブルの名前が変更され、新しいテーブルが自動的に作成されます。 +テーブルが存在しない場合、ClickHouseはそれを作成します。 ClickHouseサーバーが更新されたときにクエリログの構造が変更された場合、古い構造のテーブルの名前が変更され、新しいテーブルが自動的に作成されます。 -**例えば** +**例** ``` xml @@ -566,9 +590,9 @@ SSLのサポートは、 `libpoco` ライブラリ。 ユーザーインター ## query\_thread\_log {#server_configuration_parameters-query-thread-log} -受信したクエリのスレッドをログに記録する設定 [log\_query\_threads=1](../settings/settings.md#settings-log-query-threads) 設定。 +クエリのスレッドをログに記録するための設定 [log\_query\_threads=1](../settings/settings.md#settings-log-query-threads) 設定。 -クエリはログに記録されます [システム。query\_thread\_log](../../operations/system-tables.md#system_tables-query-thread-log) テーブル、別のファイルではありません。 テーブルの名前を変更することができます `table` パラメータ(下記参照)。 +クエリは [システムquery\_thread\_log](../../operations/system-tables.md#system_tables-query-thread-log) 別のファイルではなく、テーブル。 テーブルの名前を変更することができます。 `table` パラメータ(下記参照)。 以下のパラメータの設定ロギング: @@ -577,9 +601,9 @@ SSLのサポートは、 `libpoco` ライブラリ。 ユーザーインター - `partition_by` – Sets a [カスタム分割キー](../../engines/table-engines/mergetree-family/custom-partitioning-key.md) システムテーブルの場合。 - `flush_interval_milliseconds` – Interval for flushing data from the buffer in memory to the table. -テーブルが存在しない場合、clickhouseはそれを作成します。 clickhouseサーバーの更新時にクエリスレッドログの構造が変更された場合、古い構造を持つテーブルの名前が変更され、新しいテーブルが自動的に作成されます。 +テーブルが存在しない場合、ClickHouseはそれを作成します。 ClickHouseサーバーの更新時にクエリスレッドログの構造が変更された場合、古い構造のテーブルの名前が変更され、新しいテーブルが自動的に作成されます。 -**例えば** +**例** ``` xml @@ -592,7 +616,7 @@ SSLのサポートは、 `libpoco` ライブラリ。 ユーザーインター ## trace\_log {#server_configuration_parameters-trace_log} -のための設定 [trace\_log](../../operations/system-tables.md#system_tables-trace_log) システムテーブル操作。 +の設定 [trace\_log](../../operations/system-tables.md#system_tables-trace_log) システムテーブル操作。 パラメータ: @@ -601,7 +625,7 @@ SSLのサポートは、 `libpoco` ライブラリ。 ユーザーインター - `partition_by` — [カスタム分割キー](../../engines/table-engines/mergetree-family/custom-partitioning-key.md) システムテーブルの場合。 - `flush_interval_milliseconds` — Interval for flushing data from the buffer in memory to the table. -既定のサーバー設定ファイル `config.xml` 次の設定セクションを含みます: +既定のサーバー構成ファイル `config.xml` 次の設定セクションがあります: ``` xml @@ -614,12 +638,12 @@ SSLのサポートは、 `libpoco` ライブラリ。 ユーザーインター ## query\_masking\_rules {#query-masking-rules} -サーバーログに保存する前に、クエリとすべてのログメッセージに適用される、regexpベースのルール, -`system.query_log`, `system.text_log`, `system.processes` テーブル、およびクライアントに送信されたログ。 これにより -SQLクエリからの機密データ漏えい（名前、電子メール、個人など） +サーバーログに格納する前に、すべてのログメッセージだけでなく、クエリにも適用されます, +`system.query_log`, `system.text_log`, `system.processes` クライアントに送信されたログ。 これにより +SQLクエリからの機密データ漏洩(名前、電子メール、個人など ログへの識別子またはクレジットカード番号）。 -**例えば** +**例** ``` xml @@ -633,29 +657,29 @@ SQLクエリからの機密データ漏えい（名前、電子メール、個 設定フィールド: - `name` -ルールの名前(オプション) -- `regexp` -RE2互換の正規表現(必須) -- `replace` -機密データのための置換文字列（デフォルトではオプション-sixアスタリスク) +- `regexp` -RE2互換の正規表現（必須) +- `replace` -機密データの置換文字列(オプション、デフォルトでは-六つのアスタリスク) -マスキングルールは、クエリ全体に適用されます（不正な形式の非解析可能なクエリからの機密データの漏洩を防ぐため）。 +マスキングルールは、クエリ全体に適用されます(不正な形式/解析不可能なクエリから機密データが漏れるのを防ぐため)。 -`system.events` テーブルに反対がある `QueryMaskingRulesMatch` クエリマスキングルールの総数が一致する。 +`system.events` テーブルカウンター `QueryMaskingRulesMatch` して全体のマスキングルール。 -分散クエリの場合、各サーバーを個別に構成する必要があります。 -ノードはマスクせずに保存されます。 +分散クエリの場合は、各サーバーを個別に構成する必要があります。 +ノードはマスクなしで保存されます。 -## リモートサーバー {#server-settings-remote-servers} +## remote\_servers {#server-settings-remote-servers} -によって使用されるクラスターの構成 [分散](../../engines/table-engines/special/distributed.md) テーブルエンジンと `cluster` テーブル機能。 +によって使用されるクラスタの構成 [分散](../../engines/table-engines/special/distributed.md) テーブルエンジンと `cluster` テーブル関数。 -**例えば** +**例** ``` xml ``` -の値について `incl` 属性、セクションを参照 “[設定ファイル](../configuration-files.md#configuration_files)”. +の値に対して `incl` 属性は、節を参照してください “[設定ファイル](../configuration-files.md#configuration_files)”. -**また見なさい** +**も参照。** - [skip\_unavailable\_shards](../settings/settings.md#settings-skip_unavailable_shards) @@ -665,19 +689,19 @@ SQLクエリからの機密データ漏えい（名前、電子メール、個 UTCタイムゾーンまたは地理的位置(たとえば、Africa/Abidjan)のIANA識別子として指定します。 -タイムゾーンは、datetimeフィールドがテキスト形式（画面またはファイルに印刷される）に出力される場合、および文字列からdatetimeを取得する場合に、文字列とdatetime さらに、タイムゾーンは、入力パラメータでタイムゾーンを受信しなかった場合、時刻と日付を扱う関数で使用されます。 +タイムゾーンは、DateTimeフィールドをテキスト形式(画面またはファイルに出力)に出力するとき、および文字列からDateTimeを取得するときに、文字列とDateTime形式の間 また、タイムゾーンは、入力パラメーターでタイムゾーンを受信しなかった場合、時刻と日付を扱う関数で使用されます。 -**例えば** +**例** ``` xml Europe/Moscow ``` -## tcp\_portgenericname {#server_configuration_parameters-tcp_port} +## tcp\_port {#server_configuration_parameters-tcp_port} TCPプロトコル経由でクライアントと通信するポート。 -**例えば** +**例** ``` xml 9000 @@ -685,7 +709,7 @@ TCPプロトコル経由でクライアントと通信するポート。 ## tcp\_port\_secure {#server_configuration_parameters-tcp_port_secure} -クライアン それを使用する [OpenSSL](#server_configuration_parameters-openssl) 設定。 +クライアントとの安全な通信用のTCPポート。 それを使用して [OpenSSL](#server_configuration_parameters-openssl) 設定。 **可能な値** @@ -697,15 +721,15 @@ TCPプロトコル経由でクライアントと通信するポート。 9440 ``` -## mysql\_portgenericname {#server_configuration_parameters-mysql_port} +## mysql\_port {#server_configuration_parameters-mysql_port} -ポートと通信すmysqlプロトコルです。 +MySQLプロトコ **可能な値** 正の整数。 -例えば +例 ``` xml 9004 @@ -715,10 +739,10 @@ TCPプロトコル経由でクライアントと通信するポート。 大規模なクエリを処理するための一時データへのパス。 -!!! note "メモ" +!!! note "注" 末尾のスラッシュは必須です。 -**例えば** +**例** ``` xml /var/lib/clickhouse/tmp/ @@ -726,24 +750,24 @@ TCPプロトコル経由でクライアントと通信するポート。 ## tmp\_policy {#server-settings-tmp-policy} -からのポリシー [`storage_configuration`](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes) 一時ファイルを保存する。 +ポリシーから [`storage_configuration`](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes) 一時ファイルを格納する。 設定されていない場合 [`tmp_path`](#server-settings-tmp_path) それ以外の場合は無視されます。 -!!! note "メモ" - - `move_factor` は無視されます -- `keep_free_space_bytes` は無視されます -- `max_data_part_size_bytes` は無視されます --そのポリシーには正確に一つのボリュームが必要です +!!! note "注" + - `move_factor` 無視される +- `keep_free_space_bytes` 無視される +- `max_data_part_size_bytes` 無視される +なければならない同一数量の政策 ## uncompressed\_cache\_size {#server-settings-uncompressed_cache_size} -テーブルエンジンによって使用される非圧縮データのキャッシュサイズ(バイト単位)。 [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md). +テーブルエンジンが使用する非圧縮データのキャッシュサイズ(バイト単位) [メルゲツリー](../../engines/table-engines/mergetree-family/mergetree.md). -サーバーの共有キャッシュがあります。 メモ このオプ [use\_uncompressed\_cache](../settings/settings.md#setting-use_uncompressed_cache) は有効です。 +サーバーの共有キャッシュが一つあります。 メモリが割り当てられます。 キャッシュが使用されるのは [use\_uncompressed\_cache](../settings/settings.md#setting-use_uncompressed_cache) 有効です。 -圧縮されていないキャッシュは、個々のケースで非常に短いクエリに有利です。 +非圧縮キャッシュは、個々のケースで非常に短いクエリで有利です。 -**例えば** +**例** ``` xml 8589934592 @@ -751,9 +775,9 @@ TCPプロトコル経由でクライアントと通信するポート。 ## user\_files\_path {#server_configuration_parameters-user_files_path} -ユー テーブル関数で使用されます [ファイル()](../../sql-reference/table-functions/file.md). +ユ テーブル関数で使用されます [ファイル()](../../sql-reference/table-functions/file.md). -**例えば** +**例** ``` xml /var/lib/clickhouse/user_files/ @@ -761,26 +785,26 @@ TCPプロトコル経由でクライアントと通信するポート。 ## users\_config {#users-config} -以下を含むファイルへのパス: +以下のファイルへのパス: - ユーザー構成。 - アクセス権。 - 設定プロファイル。 -- クォータの設定。 +- クォータ設定。 -**例えば** +**例** ``` xml users.xml ``` -## zookeeper {#server-settings_zookeeper} +## 飼育係 {#server-settings_zookeeper} -ClickHouseとの対話を許可する設定が含まれています [ZooKeeper](http://zookeeper.apache.org/) クラスター +ClickHouseと対話できるようにする設定が含まれています。 [飼育係](http://zookeeper.apache.org/) クラスター。 ClickHouse用飼育係の保存メタデータのレプリカの使用時に再現します。 場合は複製のテーブルを使用していないので、このパラメータを省略することができます。 -このセクショ: +このセクションでは以下のパラメータ: - `node` — ZooKeeper endpoint. You can set multiple endpoints. @@ -798,7 +822,7 @@ ClickHouse用飼育係の保存メタデータのレプリカの使用時に再 The `index` attribute specifies the node order when trying to connect to the ZooKeeper cluster. - `session_timeout` — Maximum timeout for the client session in milliseconds. -- `root` — The [znode](http://zookeeper.apache.org/doc/r3.5.5/zookeeperOver.html#Nodes+and+ephemeral+nodes) これは、ClickHouseサーバーで使用されるznodesのルートとして使用されます。 任意です。 +- `root` — The [znode](http://zookeeper.apache.org/doc/r3.5.5/zookeeperOver.html#Nodes+and+ephemeral+nodes) これはClickHouseサーバーで使用されるznodeのルートとして使用されます。 任意。 - `identity` — User and password, that can be required by ZooKeeper to give access to requested znodes. Optional. **設定例** @@ -822,34 +846,34 @@ ClickHouse用飼育係の保存メタデータのレプリカの使用時に再 ``` -**また見なさい** +**も参照。** - [複製](../../engines/table-engines/mergetree-family/replication.md) -- [ZooKeeperプログラマーズガイド](http://zookeeper.apache.org/doc/current/zookeeperProgrammers.html) +- [ZooKeeperプログラマガイド](http://zookeeper.apache.org/doc/current/zookeeperProgrammers.html) ## use\_minimalistic\_part\_header\_in\_zookeeper {#server-settings-use_minimalistic_part_header_in_zookeeper} -ZooKeeperのデータパートヘッダーの保存方法。 +ZooKeeperのデータ部分ヘッダーの格納方法。 -この設定は、 `MergeTree` 家族 指定できます: +この設定は、 `MergeTree` 家族だ 指定できます: -- グローバルに [merge\_tree](#server_configuration_parameters-merge_tree) のセクション `config.xml` ファイル。 +- グローバルに [merge\_tree](#server_configuration_parameters-merge_tree) のセクション `config.xml` ファイル - ClickHouseは、サーバー上のすべてのテーブルの設定を使用します。 設定はいつでも変更できます。 既存の表は、設定が変更されたときの動作を変更します。 + ClickHouseは、サーバー上のすべてのテーブルの設定を使用します。 設定はいつでも変更できます。 既存のテーブルは、設定が変更されると動作を変更します。 - 各テーブルのため。 - テーブルを作成するときは、対応する [エンジンの設定](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table). この設定を持つ既存のテーブルの動作は、グローバル設定が変更されても変更されません。 + テーブルを作成するときは、対応する [エンジン設定](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table). この設定を持つ既存のテーブルの動作は、グローバル設定が変更されても変更されません。 **可能な値** - 0 — Functionality is turned off. - 1 — Functionality is turned on. -もし `use_minimalistic_part_header_in_zookeeper = 1`、その後 [複製された](../../engines/table-engines/mergetree-family/replication.md) テーブルのヘッダのデータ部品のコンパクトを `znode`. の場合はテーブルを含む多く、この保管方法を大幅に低減量のデータが保存されて飼育係. +もし `use_minimalistic_part_header_in_zookeeper = 1` その後 [複製](../../engines/table-engines/mergetree-family/replication.md) テーブルは、単一のデータパーツのヘッダーをコンパクトに格納します `znode`. テーブルに多数の列が含まれている場合、この格納方法はZookeeperに格納されるデータの量を大幅に削減します。 !!! attention "注意" - 適用後 `use_minimalistic_part_header_in_zookeeper = 1` ClickHouseサーバーをこの設定をサポートしないバージョンにダウングレードすることはできません。 するとアップグレード時に注意ClickHouseサーバーにクラスター なアップの全てのサーバーです。 テスト環境で、またはクラスターのほんの数台のサーバーで、新しいバージョンのClickHouseをテストする方が安全です。 + 申請後 `use_minimalistic_part_header_in_zookeeper = 1` ClickHouseサーバーをこの設定をサポートしないバージョンにダウングレードすることはできません。 するとアップグレード時に注意ClickHouseサーバーにクラスター なアップの全てのサーバーです。 ClickHouseの新しいバージョンをテストするには、テスト環境またはクラスターの少数のサーバーでテストする方が安全です。 Data part headers already stored with this setting can't be restored to their previous (non-compact) representation. @@ -857,16 +881,26 @@ ZooKeeperのデータパートヘッダーの保存方法。 ## disable\_internal\_dns\_cache {#server-settings-disable-internal-dns-cache} -内部dnsキャッシュを無効にします。 システムの作動のclickhouseのために推薦される -Kubernetesのような頻繁に変更の下部組織を使って。 +内部DNSキャッシュを無効にします。 システムの作動のClickHouseのために推薦される +頻繁に変化するインフラなどのKubernetes. **デフォルト値:** 0. ## dns\_cache\_update\_period {#server-settings-dns-cache-update-period} -ClickHouse内部DNSキャッシュに保存されているIPアドレスの更新期間(秒単位)。 +ClickHouse内部DNSキャッシュに格納されているIPアドレスの更新期間(秒単位)。 更新は、別のシステムスレッドで非同期に実行されます。 **デフォルト値**: 15. +## access\_control\_path {#access_control_path} + +パフォルダがClickHouseサーバー店舗ユーザーの役割構成で作成したSQLコマンド. + +デフォルト値: `/var/lib/clickhouse/access/`. + +**も参照。** + +- [アクセス制御とアカウント管理](../access-rights.md#access-control) + [元の記事](https://clickhouse.tech/docs/en/operations/server_configuration_parameters/settings/) diff --git a/docs/ja/operations/settings/constraints-on-settings.md b/docs/ja/operations/settings/constraints-on-settings.md index db917fe96e8..da7371564b4 100644 --- a/docs/ja/operations/settings/constraints-on-settings.md +++ b/docs/ja/operations/settings/constraints-on-settings.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 62 toc_title: "\u8A2D\u5B9A\u306E\u5236\u7D04" --- # 設定の制約 {#constraints-on-settings} -設定に関する制約は、次のように定義できます。 `profiles` のセクション `user.xml` 設定ファイルおよびユーザーが設定の一部を変更することを禁止します。 `SET` クエリ。 +設定に関する制約は、以下で定義することができます。 `profiles` のセクション `user.xml` 設定ファイルとユーザーが設定の一部を変更することを禁止します。 `SET` クエリ。 制約は次のように定義されます: ``` xml @@ -33,9 +33,9 @@ toc_title: "\u8A2D\u5B9A\u306E\u5236\u7D04" ``` ユーザーが制約に違反しようとすると、例外がスローされ、設定は変更されません。 -サポートされている制約は次の三種類です: `min`, `max`, `readonly`. その `min` と `max` 制約は数値設定の上限と下限を指定し、組み合わせて使用できます。 その `readonly` 制約を指定すると、ユーザーは変更できませんので、対応する設定です。 +サポートされている制約は以下の通りです: `min`, `max`, `readonly`. その `min` と `max` 制約は、数値設定の上限と下限を指定し、組み合わせて使用できます。 その `readonly` 制約を指定すると、ユーザーは変更できませんので、対応する設定です。 -**例えば:** さあ `users.xml` 行を含む: +**例:** さあ `users.xml` 行を含む: ``` xml @@ -70,6 +70,6 @@ Code: 452, e.displayText() = DB::Exception: Setting max_memory_usage should not Code: 452, e.displayText() = DB::Exception: Setting force_index_by_date should not be changed. ``` -**メモ:** その `default` プロファイルには特別な処理があります。 `default` プロファイルはデフォルトの制約になるため、すべてのユーザーを明示的に上書きするまで制限します。 +**注:** その `default` プロファイルには特別な処理があります。 `default` プロのデフォルトの制約、その制限するすべてのユーザーまでの彼らのメソッドを明示的にこれらのユーザー [元の記事](https://clickhouse.tech/docs/en/operations/settings/constraints_on_settings/) diff --git a/docs/ja/operations/settings/index.md b/docs/ja/operations/settings/index.md index 9eae8f24799..f64c339ef9e 100644 --- a/docs/ja/operations/settings/index.md +++ b/docs/ja/operations/settings/index.md @@ -1,32 +1,33 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_folder_title: Settings +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "\u8A2D\u5B9A" toc_priority: 55 -toc_title: "\u5C0E\u5165" +toc_title: "\u306F\u3058\u3081\u306B" --- -# 設定 {#settings} +# 設定 {#session-settings-intro} -以下に説明するすべての設定を行うには、複数の方法があります。 -設定はレイヤーで構成されるので、後続の各レイヤーは以前の設定を再定義します。 +複数あるというものですすべての設定は、このセクションで説明する文書 -優先順位の順に設定を構成する方法: +設定はレイヤーで構成されるため、後続の各レイヤーは以前の設定を再定義します。 + +優先順位の順に設定する方法: - の設定 `users.xml` サーバー構成ファイル。 - 要素内に設定する ``. + 要素に設定 ``. -- セッションの設定。 +- セッション設定。 - 送信 `SET setting=value` 対話モードでのClickHouseコンソールクライアントから。 - 同様に、httpプロトコルでclickhouseセッションを使用できます。 これを行うには、以下を指定する必要があります。 `session_id` HTTPパラメータ。 + 送信 `SET setting=value` 対話モードでClickHouseコンソールクライアントから。 + 同様に、HttpプロトコルでClickHouseセッションを使用できます。 これを行うには、以下を指定する必要があります `session_id` HTTPパラメータ。 -- クエリの設定。 +- クエリ設定。 - - を開始する場合にclickhouseコンソールがクライアントを非インタラクティブモードの設定を起動パラメータ `--setting=value`. - - HTTP APIを使用する場合は、CGIパラメーターを渡します (`URL?setting_1=value&setting_2=value...`). + - ClickHouse consoleクライアントを非対話モードで起動するときは、startupパラメータを設定します `--setting=value`. + - HTTP APIを使用する場合は、CGIパラメータを渡します (`URL?setting_1=value&setting_2=value...`). -このセクションでは、server configファイルでのみ行うことができる設定については説明しません。 +サーバー設定ファイルでのみ行うことができる設定は、このセクションでは説明しません。 [元の記事](https://clickhouse.tech/docs/en/operations/settings/) diff --git a/docs/ja/operations/settings/permissions-for-queries.md b/docs/ja/operations/settings/permissions-for-queries.md index c97ff0f88be..c299337d62f 100644 --- a/docs/ja/operations/settings/permissions-for-queries.md +++ b/docs/ja/operations/settings/permissions-for-queries.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 58 toc_title: "\u30AF\u30A8\u30EA\u306E\u6A29\u9650" --- # クエリの権限 {#permissions_for_queries} -問合せclickhouse大きく分けて複数の種類: +問合せClickHouse大きく分けて複数の種類: 1. データを読み込むためのクエリー: `SELECT`, `SHOW`, `DESCRIBE`, `EXISTS`. 2. データクエリの記述: `INSERT`, `OPTIMIZE`. @@ -15,7 +15,7 @@ toc_title: "\u30AF\u30A8\u30EA\u306E\u6A29\u9650" 4. [DDL](https://en.wikipedia.org/wiki/Data_definition_language) クエリ: `CREATE`, `ALTER`, `RENAME`, `ATTACH`, `DETACH`, `DROP` `TRUNCATE`. 5. `KILL QUERY`. -次の設定では、クエリの種類に応じてユーザー権限を調整します: +次の設定では、クエリの種類によってユーザー権限を調整します: - [読み取り専用](#settings_readonly) — Restricts permissions for all types of queries except DDL queries. - [allow\_ddl](#settings_allow_ddl) — Restricts permissions for DDL queries. @@ -24,9 +24,9 @@ toc_title: "\u30AF\u30A8\u30EA\u306E\u6A29\u9650" ## 読み取り専用 {#settings_readonly} -データの読み取り、データの書き込み、設定の変更の権限を制限します。 +制限のアクセス権読書データの書き込みデータや設定の変更ます。 -クエリを型に分割する方法を参照してください [上](#permissions_for_queries). +クエリを型に分割する方法を参照してください [上記](#permissions_for_queries). 可能な値: @@ -34,12 +34,12 @@ toc_title: "\u30AF\u30A8\u30EA\u306E\u6A29\u9650" - 1 — Only read data queries are allowed. - 2 — Read data and change settings queries are allowed. -設定後 `readonly = 1`、ユーザーは変更できません `readonly` と `allow_ddl` 現在のセッションの設定。 +設定後 `readonly = 1` ユーザーは変更できません `readonly` と `allow_ddl` 現在のセッションの設定。 -を使用する場合 `GET` の方法 [HTTPインター](../../interfaces/http.md), `readonly = 1` 自動的に設定されます。 データを変更するには、 `POST` 方法。 +を使用する場合 `GET` の方法 [HTTPインターフェ](../../interfaces/http.md), `readonly = 1` 自動的に設定されます。 データを変更するには、 `POST` 方法。 -設定 `readonly = 1` ユーザーによるすべての設定の変更を禁止します。 ユーザーを禁止する方法があります -特定の設定のみを変更するから、詳細については [設定の制約](constraints-on-settings.md). +設定 `readonly = 1` ユーザーがすべての設定を変更するのを禁止します。 ユーザーを禁止する方法があります +からの変更は、特定の設定の詳細を見る [設定の制約](constraints-on-settings.md). デフォルト値:0 @@ -47,14 +47,14 @@ toc_title: "\u30AF\u30A8\u30EA\u306E\u6A29\u9650" 許可または拒否 [DDL](https://en.wikipedia.org/wiki/Data_definition_language) クエリ。 -クエリを型に分割する方法を参照してください [上](#permissions_for_queries). +クエリを型に分割する方法を参照してください [上記](#permissions_for_queries). 可能な値: - 0 — DDL queries are not allowed. - 1 — DDL queries are allowed. -あなたは実行できません `SET allow_ddl = 1` もし `allow_ddl = 0` 現在のセッションの場合。 +実行できない `SET allow_ddl = 1` もし `allow_ddl = 0` 現在のセッションの場合。 デフォルト値:1 diff --git a/docs/ja/operations/settings/query-complexity.md b/docs/ja/operations/settings/query-complexity.md index 2ef666ae5f3..45b7a09f17f 100644 --- a/docs/ja/operations/settings/query-complexity.md +++ b/docs/ja/operations/settings/query-complexity.md @@ -1,107 +1,107 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 59 toc_title: "\u30AF\u30A8\u30EA\u306E\u8907\u96D1\u3055\u306E\u5236\u9650" --- # クエリの複雑さの制限 {#restrictions-on-query-complexity} -クエリの複雑さの制限は、設定の一部です。 +クエリの複雑さに関する制限は、設定の一部です。 これらをより安全な実行のユーザーインターフェースです。 -ほぼすべての制限が適用されます `SELECT`. 分散クエリ処理では、各サーバーに個別に制限が適用されます。 +ほとんどすべての制限が適用されます `SELECT`. 分散クエリ処理では、各サーバーに個別に制限が適用されます。 -ClickHouseは、各行ではなく、データパーツの制限をチェックします。 これは、データ部分のサイズで制限の値を超えることができることを意味します。 +ClickHouseは、各行ではなく、データ部分の制限をチェックします。 これは、データ部分のサイズで制限の値を超えることができることを意味します。 の制限 “maximum amount of something” 値0を取ることができます。 “unrestricted”. -ほとんどの制限には、 ‘overflow\_mode’ 設定、制限を超えたときに何をすべきかを意味します。 -それは二つの値のいずれか: `throw` または `break`. 集計の制限(group\_by\_overflow\_mode)にも値があります `any`. +ほとんどの制限には ‘overflow\_mode’ 設定、制限を超えたときに何をすべきかを意味します。 +での値: `throw` または `break`. 集計の制限(group\_by\_overflow\_mode)にも値があります `any`. `throw` – Throw an exception (default). `break` – Stop executing the query and return the partial result, as if the source data ran out. -`any (only for group_by_overflow_mode)` – Continuing aggregation for the keys that got into the set, but don’t add new keys to the set. +`any (only for group_by_overflow_mode)` – Continuing aggregation for the keys that got into the set, but don't add new keys to the set. ## max\_memory\_usage {#settings_max_memory_usage} -単一のサーバーでクエリを実行するために使用するramの最大量。 +単一サーバーでクエリを実行するために使用するRAMの最大量。 -デフォルトの設定ファイルでは、最大値は10gbです。 +既定の構成ファイルでは、最大10GBです。 -この設定では、使用可能なメモリの量やマシン上のメモリの総量は考慮されません。 -この制限は、単一のサーバー内の単一のクエリに適用されます。 -を使用することができ `SHOW PROCESSLIST` 各クエリの現在のメモリ消費量を表示します。 -さらに、各クエリに対してピークのメモリ消費が追跡され、ログに書き込まれます。 +この設定では、使用可能なメモリの容量やマシン上のメモリの合計容量は考慮されません。 +この制限は、単一サーバー内の単一のクエリに適用されます。 +以下を使用できます `SHOW PROCESSLIST` 各クエリの現在のメモリ消費量を確認します。 +さらに、ピークメモリ消費は各クエリに対して追跡され、ログに書き込まれます。 -特定の集計関数の状態に対するメモリ使用量は監視されません。 +特定の集計関数の状態については、メモリ使用量は監視されません。 -集計関数の状態に対するメモリ使用量は完全には追跡されません `min`, `max`, `any`, `anyLast`, `argMin`, `argMax` から `String` と `Array` 引数。 +集計関数の状態に対してメモリ使用量が完全に追跡されません `min`, `max`, `any`, `anyLast`, `argMin`, `argMax` から `String` と `Array` 引数。 メモリ消費もパラメータによって制限されます `max_memory_usage_for_user` と `max_memory_usage_for_all_queries`. ## max\_memory\_usage\_for\_user {#max-memory-usage-for-user} -単一のサーバー上でユーザーのクエリを実行するために使用するramの最大量。 +単一サーバー上でユーザーのクエリを実行するために使用するRAMの最大量。 -デフォルト値は [設定。h](https://github.com/ClickHouse/ClickHouse/blob/master/dbms/Core/Settings.h#L288). デフォルトでは、金額は制限されません (`max_memory_usage_for_user = 0`). +デフォルト値は [設定。h](https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/Settings.h#L288). デフォルトでは、金額は制限されません (`max_memory_usage_for_user = 0`). の説明も参照してください [max\_memory\_usage](#settings_max_memory_usage). ## max\_memory\_usage\_for\_all\_queries {#max-memory-usage-for-all-queries} -単一のサーバー上ですべてのクエリを実行するために使用するramの最大量。 +単一サーバー上ですべてのクエリを実行するために使用するRAMの最大量。 -デフォルト値は [設定。h](https://github.com/ClickHouse/ClickHouse/blob/master/dbms/Core/Settings.h#L289). デフォルトでは、金額は制限されません (`max_memory_usage_for_all_queries = 0`). +デフォルト値は [設定。h](https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/Settings.h#L289). デフォルトでは、金額は制限されません (`max_memory_usage_for_all_queries = 0`). の説明も参照してください [max\_memory\_usage](#settings_max_memory_usage). ## max\_rows\_to\_read {#max-rows-to-read} -各行ではなく、各ブロックで次の制限を確認できます。 つまり、制限は少し壊れる可能性があります。 -複数のスレッドでクエリを実行する場合、次の制限が各スレッドに個別に適用されます。 +次の制限は、各ブロック（各行ではなく）で確認できます。 つまり、制限は少し壊れる可能性があります。 +複数のスレッドでクエリを実行する場合、次の制限は各スレッドに個別に適用されます。 クエリの実行時にテーブルから読み取ることができる最大行数。 ## max\_bytes\_to\_read {#max-bytes-to-read} -クエリの実行時にテーブルから読み取ることができる最大バイト数(圧縮されていないデータ)。 +クエリの実行時にテーブルから読み取ることができる最大バイト数(非圧縮データ)。 ## read\_overflow\_mode {#read-overflow-mode} -データの読み取り量がいずれかの制限を超えた場合の対処方法: ‘throw’ または ‘break’. デフォルトでは、投げる。 +読み込まれるデータ量がいずれかの制限を超えた場合の対処方法: ‘throw’ または ‘break’. デフォルトでは、throw。 ## max\_rows\_to\_group\_by {#settings-max-rows-to-group-by} -集約から受け取った一意のキーの最大数。 この設定では、集計時のメモリ消費量を制限できます。 +集計から受け取った一意のキーの最大数。 この設定を使用すると、集計時のメモリ消費量を制限できます。 ## group\_by\_overflow\_mode {#group-by-overflow-mode} -集計の一意のキーの数が制限を超えた場合の対処方法: ‘throw’, ‘break’、または ‘any’. デフォルトでは、投げる。 -を使用して ‘any’ valueを使用すると、GROUP BYの近似を実行できます。 この近似の品質は、データの統計的性質に依存します。 +集計の一意キーの数が制限を超えた場合の対処方法: ‘throw’, ‘break’,または ‘any’. デフォルトでは、throw。 +を使用して ‘any’ valueでは、GROUP BYの近似を実行できます。 この近似の品質は、データの統計的性質に依存します。 ## max\_bytes\_before\_external\_group\_by {#settings-max_bytes_before_external_group_by} -の実行を有効または無効にします。 `GROUP BY` 外部メモリ内の句。 見る [外部メモリによるグループ化](../../sql-reference/statements/select.md#select-group-by-in-external-memory). +の実行を有効または無効にします。 `GROUP BY` 外部メモリ内の句。 見る [外部メモリのGROUP BY](../../sql-reference/statements/select/group-by.md#select-group-by-in-external-memory). 可能な値: -- シングルで使用できるramの最大ボリューム(バイト単位)。 [GROUP BY](../../sql-reference/statements/select.md#select-group-by-clause) オペレーション -- 0 — `GROUP BY` 外部メモリで無効。 +- シングルで使用できるRAMの最大ボリューム(バイト単位) [GROUP BY](../../sql-reference/statements/select/group-by.md#select-group-by-clause) 作戦だ +- 0 — `GROUP BY` 外部メモリでは無効です。 -デフォルト値:0. +デフォルト値は0です。 ## max\_rows\_to\_sort {#max-rows-to-sort} -並べ替え前の行の最大数。 これにより、ソート時のメモリ消費量を制限できます。 +並べ替え前の最大行数。 これにより、ソート時のメモリ消費を制限できます。 ## max\_bytes\_to\_sort {#max-bytes-to-sort} -ソート前の最大バイト数。 +並べ替え前の最大バイト数。 ## sort\_overflow\_mode {#sort-overflow-mode} -ソート前に受け取った行の数がいずれかの制限を超えた場合の対処方法: ‘throw’ または ‘break’. デフォルトでは、投げる。 +ソート前に受信した行数がいずれかの制限を超えた場合の対処方法: ‘throw’ または ‘break’. デフォルトでは、throw。 ## max\_result\_rows {#setting-max_result_rows} @@ -113,11 +113,11 @@ ClickHouseは、各行ではなく、データパーツの制限をチェック ## result\_overflow\_mode {#result-overflow-mode} -結果のボリュームがいずれかの制限を超えた場合の対処方法: ‘throw’ または ‘break’. デフォルトでは、投げる。 +結果の量が制限のいずれかを超えた場合の対処方法: ‘throw’ または ‘break’. デフォルトでは、throw。 -を使用して ‘break’ LIMITを使用するのと似ています。 `Break` ブロックレベルでのみ実行を中断します。 これは、返される行の量が [max\_result\_rows](#setting-max_result_rows)、の倍数 [max\_block\_size](settings.md#setting-max_block_size) そして依存します [max\_threads](settings.md#settings-max_threads). +を使用して ‘break’ LIMITの使用に似ています。 `Break` ブロックレベルでのみ実行を中断します。 これは、返される行の量が [max\_result\_rows](#setting-max_result_rows) の倍数 [max\_block\_size](settings.md#setting-max_block_size) そして依存する [max\_threads](settings.md#settings-max_threads). -例えば: +例: ``` sql SET max_threads = 3, max_block_size = 3333; @@ -136,79 +136,79 @@ FORMAT Null; ## max\_execution\_time {#max-execution-time} -クエリの最大実行時間(秒)。 -このとき、ソート段階のいずれか、または集計関数のマージおよびファイナライズ時にはチェックされません。 +クエリの最大実行時間を秒単位で指定します。 +現時点では、ソートステージのいずれか、または集計関数のマージおよびファイナライズ時にはチェックされません。 ## timeout\_overflow\_mode {#timeout-overflow-mode} -クエリがより長く実行される場合の対処方法 ‘max\_execution\_time’: ‘throw’ または ‘break’. デフォルトでは、投げる。 +クエリが実行される時間よりも長い場合の対処方法 ‘max\_execution\_time’: ‘throw’ または ‘break’. デフォルトでは、throw。 ## min\_execution\_speed {#min-execution-speed} -毎秒行の最小の実行速度。 チェックすべてのデータブロックの場合 ‘timeout\_before\_checking\_execution\_speed’ 期限が切れる 実行速度が遅い場合は、例外がスローされます。 +毎秒行単位の最小実行速度。 すべてのデータブロックで ‘timeout\_before\_checking\_execution\_speed’ 有効期限が切れます。 実行速度が低い場合は、例外がスローされます。 ## min\_execution\_speed\_bytes {#min-execution-speed-bytes} -実行バイト/秒の最小数。 チェックすべてのデータブロックの場合 ‘timeout\_before\_checking\_execution\_speed’ 期限が切れる 実行速度が遅い場合は、例外がスローされます。 +秒あたりの最小実行バイト数。 すべてのデータブロックで ‘timeout\_before\_checking\_execution\_speed’ 有効期限が切れます。 実行速度が低い場合は、例外がスローされます。 ## max\_execution\_speed {#max-execution-speed} -秒あたりの実行行の最大数。 チェックすべてのデータブロックの場合 ‘timeout\_before\_checking\_execution\_speed’ 期限が切れる 実行速度が速い場合、実行速度が低下します。 +毎秒の実行行の最大数。 すべてのデータブロックで ‘timeout\_before\_checking\_execution\_speed’ 有効期限が切れます。 実行速度が高い場合は、実行速度が低下します。 ## max\_execution\_speed\_bytes {#max-execution-speed-bytes} -実行バイト/秒の最大数。 チェックすべてのデータブロックの場合 ‘timeout\_before\_checking\_execution\_speed’ 期限が切れる 実行速度が速い場合、実行速度が低下します。 +毎秒の実行バイト数の最大値。 すべてのデータブロックで ‘timeout\_before\_checking\_execution\_speed’ 有効期限が切れます。 実行速度が高い場合は、実行速度が低下します。 ## timeout\_before\_checking\_execution\_speed {#timeout-before-checking-execution-speed} -実行速度が遅すぎないことをチェックする ‘min\_execution\_speed’指定された時間が経過した後、秒で）。 +実行速度が遅すぎないことをチェックします ‘min\_execution\_speed’)、指定された時間が秒単位で経過した後。 ## max\_columns\_to\_read {#max-columns-to-read} -単一のクエリでテーブルから読み取ることができる列の最大数。 クエリでより多くの列を読み取る必要がある場合は、例外がスローされます。 +単一のクエリ内のテーブルから読み取ることができる列の最大数。 クエリでより多くの列を読み取る必要がある場合は、例外がスローされます。 ## max\_temporary\_columns {#max-temporary-columns} -定数の列を含む、クエリを実行するときにramに同時に保持する必要がある一時的な列の最大数。 これよりも一時的な列が多い場合は、例外がスローされます。 +定数列を含む、クエリを実行するときに同時にRAMに保持する必要がある一時列の最大数。 これよりも多くの一時列がある場合、例外がスローされます。 ## max\_temporary\_non\_const\_columns {#max-temporary-non-const-columns} -同じものとして ‘max\_temporary\_columns’ しかし、一定の列を数えずに。 -定数の列は、クエリを実行するときにかなり頻繁に形成されますが、計算リソースはほぼゼロになります。 +同じことと ‘max\_temporary\_columns’ しかし、定数列を数えずに。 +定数列は、クエリを実行するときにかなり頻繁に形成されますが、計算リソースはほぼゼロです。 ## max\_subquery\_depth {#max-subquery-depth} -サブクエリの最大ネスト深度。 サブクエリが深い場合は、例外がスローされます。 デフォルトでは、100。 +サブクエリの最大ネスト深さ。 サブクエリが深い場合は、例外がスローされます。 既定では100です。 ## max\_pipeline\_depth {#max-pipeline-depth} -パイプラインの最大深さ。 クエリ処理中に各データブロックが通過する変換の数に対応します。 単一のサーバーの制限内で数えられます。 パイプラインの深さが大きい場合は、例外がスローされます。 デフォルトでは、1000。 +最大パイプライン深さ。 クエリ処理中に各データブロックが処理する変換の数に対応します。 単一サーバーの範囲内でカウントされます。 パイプラインの深さが大きい場合は、例外がスローされます。 既定では、1000です。 ## max\_ast\_depth {#max-ast-depth} クエリ構文ツリーの最大ネスト深さ。 超過すると、例外がスローされます。 -現時点では、解析中にチェックされず、クエリを解析した後でのみチェックされます。 つまり、構文解析中に深すぎる構文木を作成することはできますが、クエリは失敗します。 デフォルトでは、1000。 +現時点では、解析中にはチェックされず、クエリの解析後にのみチェックされます。 つまり、解析中に深すぎる構文ツリーを作成することができますが、クエリは失敗します。 既定では、1000です。 ## max\_ast\_elements {#max-ast-elements} クエリ構文ツリー内の要素の最大数。 超過すると、例外がスローされます。 -以前の設定と同じように、クエリを解析した後にのみチェックされます。 デフォルトでは、50,000。 +前の設定と同じように、クエリを解析した後にのみチェックされます。 既定では、50,000です。 ## max\_rows\_in\_set {#max-rows-in-set} -サブクエリから作成されたin句のデータ-セットの最大行数。 +サブクエリから作成されたIN句内のデータ-セットの最大行数。 -## max\_bytes\_inset {#max-bytes-in-set} +## max\_bytes\_in\_set {#max-bytes-in-set} -サブクエリから作成されたin句のセットによって使用される最大バイト数(圧縮されていないデータ)。 +サブクエリから作成されたIN句のセットで使用される最大バイト数(非圧縮データ)。 ## set\_overflow\_mode {#set-overflow-mode} -データの量がいずれかの制限を超えた場合の対処方法: ‘throw’ または ‘break’. デフォルトでは、投げる。 +データ量がいずれかの制限を超えた場合の対処方法: ‘throw’ または ‘break’. デフォルトでは、throw。 ## max\_rows\_in\_distinct {#max-rows-in-distinct} -DISTINCTを使用する場合の異なる行の最大数。 +DISTINCTを使用する場合の最大行数。 ## max\_bytes\_in\_distinct {#max-bytes-in-distinct} @@ -216,57 +216,57 @@ DISTINCTを使用するときにハッシュテーブルで使用される最大 ## distinct\_overflow\_mode {#distinct-overflow-mode} -データの量がいずれかの制限を超えた場合の対処方法: ‘throw’ または ‘break’. デフォルトでは、投げる。 +データ量がいずれかの制限を超えた場合の対処方法: ‘throw’ または ‘break’. デフォルトでは、throw。 -## max\_rows\_tokenトランスファー {#max-rows-to-transfer} +## max\_rows\_to\_transfer {#max-rows-to-transfer} -リモートサーバーに渡すか、global inを使用するときに一時テーブルに保存できる行の最大数。 +グローバルINを使用するときに、リモートサーバーに渡すか、一時テーブルに保存できる最大行数。 ## max\_bytes\_to\_transfer {#max-bytes-to-transfer} -リモートサーバーに渡すか、global inを使用するときに一時テーブルに保存できる最大バイト数(圧縮されていないデータ)。 +グローバルINを使用するときに、リモートサーバーに渡すか、一時テーブルに保存できる最大バイト数(非圧縮データ)。 ## transfer\_overflow\_mode {#transfer-overflow-mode} -データの量がいずれかの制限を超えた場合の対処方法: ‘throw’ または ‘break’. デフォルトでは、投げる。 +データ量がいずれかの制限を超えた場合の対処方法: ‘throw’ または ‘break’. デフォルトでは、throw。 ## max\_rows\_in\_join {#settings-max_rows_in_join} -テーブルを結合するときに使用されるハッシュテーブルの行数を制限します。 +テーブルを結合するときに使用されるハッシュテーブル内の行数を制限します。 -この設定は以下に適用されます [SELECT … JOIN](../../sql-reference/statements/select.md#select-join) 業務の [参加](../../engines/table-engines/special/join.md) テーブルエンジン。 +この設定は以下に適用されます [SELECT … JOIN](../../sql-reference/statements/select/join.md#select-join) 操作および [参加](../../engines/table-engines/special/join.md) テーブルエンジン。 -クエリに複数の結合が含まれている場合、clickhouseは中間結果ごとにこの設定をチェックします。 +クエリに複数の結合が含まれている場合、ClickHouseはこの設定で中間結果をすべてチェックします。 -ClickHouseは、制限に達したときにさまざまなアクションを実行できます。 を使用 [join\_overflow\_mode](#settings-join_overflow_mode) アクションを選択する設定。 +ClickHouseは、制限に達したときにさまざまなアクションを続行できます。 使用する [join\_overflow\_mode](#settings-join_overflow_mode) アクションを選択する設定。 可能な値: - 正の整数。 - 0 — Unlimited number of rows. -デフォルト値:0. +デフォルト値は0です。 ## max\_bytes\_in\_join {#settings-max_bytes_in_join} 制限サイズをバイトのハッシュテーブルが参加す。 -この設定は以下に適用されます [SELECT … JOIN](../../sql-reference/statements/select.md#select-join) 操作と [結合テーブルエンジン](../../engines/table-engines/special/join.md). +この設定は以下に適用されます [SELECT … JOIN](../../sql-reference/statements/select/join.md#select-join) 操作および [結合テーブルエンジン](../../engines/table-engines/special/join.md). -クエリに結合が含まれている場合、clickhouseは中間結果ごとにこの設定をチェックします。 +クエリに結合が含まれている場合、ClickHouseは中間結果ごとにこの設定をチェックします。 -ClickHouseは、制限に達したときにさまざまなアクションを実行できます。 使用 [join\_overflow\_mode](#settings-join_overflow_mode) アクションを選択するための設定。 +ClickHouseは、制限に達したときにさまざまなアクションを続行できます。 使用 [join\_overflow\_mode](#settings-join_overflow_mode) アクションを選択する設定。 可能な値: - 正の整数。 - 0 — Memory control is disabled. -デフォルト値:0. +デフォルト値は0です。 ## join\_overflow\_mode {#settings-join_overflow_mode} -次の結合制限のいずれかに達したときにclickhouseが実行するアクションを定義します: +次のいずれかの結合制限に達したときにClickHouseが実行するアクションを定義します: - [max\_bytes\_in\_join](#settings-max_bytes_in_join) - [max\_rows\_in\_join](#settings-max_rows_in_join) @@ -274,27 +274,27 @@ ClickHouseは、制限に達したときにさまざまなアクションを実 可能な値: - `THROW` — ClickHouse throws an exception and breaks operation. -- `BREAK` — ClickHouse breaks operation and doesn’t throw an exception. +- `BREAK` — ClickHouse breaks operation and doesn't throw an exception. デフォルト値: `THROW`. -**また見なさい** +**も参照。** -- [JOIN句](../../sql-reference/statements/select.md#select-join) +- [JOIN句](../../sql-reference/statements/select/join.md#select-join) - [結合テーブルエンジン](../../engines/table-engines/special/join.md) ## max\_partitions\_per\_insert\_block {#max-partitions-per-insert-block} -単一の挿入ブロック内のパーティションの最大数を制限します。 +単一挿入ブロック内のパーティションの最大数を制限します。 - 正の整数。 - 0 — Unlimited number of partitions. -デフォルト値:100。 +デフォルト値は100です。 **詳細** -を挿入する際、データclickhouse計算パーティションの数に挿入されます。 パーティションの数が `max_partitions_per_insert_block`、ClickHouseは、次のテキストで例外をスローします: +を挿入する際、データClickHouse計算パーティションの数に挿入されます。 パーティションの数が `max_partitions_per_insert_block`,ClickHouseは、次のテキストで例外をスローします: > “Too many partitions for single INSERT block (more than” +toString(max\_parts)+ “). The limit is controlled by ‘max\_partitions\_per\_insert\_block’ setting. A large number of partitions is a common misconception. It will lead to severe negative performance impact, including slow server startup, slow INSERT queries and slow SELECT queries. Recommended total number of partitions for a table is under 1000..10000. Please note, that partitioning is not intended to speed up SELECT queries (ORDER BY key is sufficient to make range queries fast). Partitions are intended for data manipulation (DROP PARTITION, etc).” diff --git a/docs/ja/operations/settings/settings-profiles.md b/docs/ja/operations/settings/settings-profiles.md index d9d2d1ff114..09f10a231cf 100644 --- a/docs/ja/operations/settings/settings-profiles.md +++ b/docs/ja/operations/settings/settings-profiles.md @@ -1,16 +1,24 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 61 toc_title: "\u8A2D\u5B9A\u30D7\u30ED\u30D5\u30A1\u30A4\u30EB" --- # 設定プロファイル {#settings-profiles} -設定プロファイルは、同じ名前でグループ化された設定の集合です。 各clickhouseユーザプロファイル. +設定プロファイルは、同じ名前でグループ化された設定の集合です。 + +!!! note "情報" + ClickHouseはまた支えます [SQL駆動型ワークフロー](../access-rights.md#access-control) 設定プロファイルを管理する。 お勧めいたします。 + +プロファイルのどれでも持つ事ができます。 プロファイルのどれでも持つ事ができます。 異なるユーザーに同じプロファイルを指定できます。 最も重要なことが書ける設定プロフィール `readonly=1` 読み取り専用アクセスを保証します。 + +設定プロファイルは相互に継承できます。 継承を使用するには、一つまたは複数を指定します `profile` プロファイルにリストされている他の設定の前の設定。 ある設定が異なるプロファイルで定義されている場合は、定義された最新の設定が使用されます。 + プロファイル内のすべての設定を適用するには、 `profile` 設定。 -例えば: +例: インストール `web` プロフィール @@ -18,9 +26,9 @@ toc_title: "\u8A2D\u5B9A\u30D7\u30ED\u30D5\u30A1\u30A4\u30EB" SET profile = 'web' ``` -設定プロファイルは、user configファイルで宣言されます。 これは通常 `users.xml`. +設定プロファイルで宣言されたユーザのconfigファイルです。 これは通常です `users.xml`. -例えば: +例: ``` xml @@ -64,8 +72,10 @@ SET profile = 'web' ``` -この例では、: `default` と `web`. その `default` プロファイルには特別な目的があります。 つまり、 `default` オプションの設定デフォルトを設定します。 その `web` プロファイルは通常のプロファイルです。 `SET` HTTPクエリでURLパラメーターを照会または使用する。 +この例では、: `default` と `web`. -設定プロファイルは相互に継承できます。 継承を使用するには、一つまたは複数を指定します `profile` プロファイルにリストされている他の設定の前に設定します。 ある設定が異なるプロファイルで定義されている場合は、最新の設定が使用されます。 +その `default` プロファイルには特別な目的があります。 つまり、 `default` オプションの設定デフォルトを設定します。 + +その `web` プロファイルは通常のプロファイルです。 `SET` クエリまたはHTTPクエリでURLパラメータを使用する。 [元の記事](https://clickhouse.tech/docs/en/operations/settings/settings_profiles/) diff --git a/docs/ja/operations/settings/settings-users.md b/docs/ja/operations/settings/settings-users.md index c0e4473eecf..7b6fca249d7 100644 --- a/docs/ja/operations/settings/settings-users.md +++ b/docs/ja/operations/settings/settings-users.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 63 toc_title: "\u30E6\u30FC\u30B6\u30FC\u8A2D\u5B9A" --- @@ -9,6 +9,9 @@ toc_title: "\u30E6\u30FC\u30B6\u30FC\u8A2D\u5B9A" その `users` のセクション `user.xml` 設定ファイルにユーザを設定します。 +!!! note "情報" + ClickHouseはまた支えます [SQL駆動型ワークフロー](../access-rights.md#access-control) ユーザーを管理するため。 お勧めいたします。 + の構造 `users` セクション: ``` xml @@ -19,6 +22,8 @@ toc_title: "\u30E6\u30FC\u30B6\u30FC\u8A2D\u5B9A" + 0|1 + @@ -40,15 +45,15 @@ toc_title: "\u30E6\u30FC\u30B6\u30FC\u8A2D\u5B9A" ### user\_name/パスワード {#user-namepassword} -パスワードは、平文またはsha256（hex形式）で指定できます。 +パスワードは、平文またはSHA256(hex形式)で指定できます。 -- 平文でパスワードを割り当てるには (**推奨しない**）、それを置く `password` 要素。 +- 平文でパスワードを割り当てるには (**推奨されない**）、それをaに置きます `password` 要素。 例えば, `qwerty`. パスワードは空白のままにできます。 -- SHA256ハッシュを使用してパスワードを割り当てるには、 `password_sha256_hex` 要素。 +- SHA256ハッシュを使用してパスワードを割り当てるには、パスワードを `password_sha256_hex` 要素。 例えば, `65e84be33532fb784c48129675f9eff3a682b27168c0ea744b2cf58ee02337c5`. @@ -56,11 +61,11 @@ toc_title: "\u30E6\u30FC\u30B6\u30FC\u8A2D\u5B9A" PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha256sum | tr -d '-' - 結果の最初の行はパスワードです。 第二の行は、対応するsha256ハッシュです。 + 結果の最初の行はパスワードです。 第二の行は、対応するSHA256ハッシュです。 -- MySQLクライアントとの互換性のために、passwordはダブルSHA1ハッシュで指定できます。 それを置く `password_double_sha1_hex` 要素。 +- MySQLクライアントとの互換性のために、パスワードは二重SHA1ハッシュで指定できます。 それを置く `password_double_sha1_hex` 要素。 例えば, `08b4a0f1de6ad37da17359e592c8d74788a83eb0`. @@ -68,13 +73,24 @@ toc_title: "\u30E6\u30FC\u30B6\u30FC\u8A2D\u5B9A" PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha1sum | tr -d '-' | xxd -r -p | sha1sum | tr -d '-' - 結果の最初の行はパスワードです。 第二の行は、対応するダブルsha1ハッシュです。 + 結果の最初の行はパスワードです。 第二の行は、対応する二重SHA1ハッシュです。 -### user\_name/networks {#user-namenetworks} +### access\_management {#access_management-user-setting} -ユーザーがclickhouseサーバーに接続できるネットワークのリスト。 +この設定では、SQLドリブンの使用を無効にできます [アクセス制御とアカウント管理](../access-rights.md#access-control) ユーザーのために。 -リストの各要素には、次のいずれかの形式があります: +可能な値: + +- 0 — Disabled. +- 1 — Enabled. + +デフォルト値は0です。 + +### user\_name/ネットワーク {#user-namenetworks} + +ユーザーがClickHouseサーバーに接続できるネットワークのリスト。 + +リストの各要素には、次のいずれかの形式を使用できます: - `` — IP address or network mask. @@ -82,17 +98,17 @@ toc_title: "\u30E6\u30FC\u30B6\u30FC\u8A2D\u5B9A" - `` — Hostname. - 例えば: `example01.host.ru`. + 例: `example01.host.ru`. - アクセスを確認するには、dnsクエリが実行され、返されたすべてのipアドレスがピアアドレスと比較されます。 + チェックアクセス、DNS問い合わせを行い、すべて返されたIPアドレスと比べてのピアがアドレスです。 - `` — Regular expression for hostnames. - 例えば, `^example\d\d-\d\d-\d\.host\.ru$` + 例, `^example\d\d-\d\d-\d\.host\.ru$` - アクセスを確認するには、 [DNS PTRクエリ](https://en.wikipedia.org/wiki/Reverse_DNS_lookup) ピアアドレスに対して実行され、指定された正規表現が適用されます。 次に、PTRクエリの結果に対して別のDNSクエリが実行され、すべての受信アドレスがピアアドレスと比較されます。 Regexpは$で終わることを強くお勧めします。 + アクセスを確認するには、 [DNS PTRクエリ](https://en.wikipedia.org/wiki/Reverse_DNS_lookup) ピアアドレスに対して実行され、指定された正規表現が適用されます。 次に、PTRクエリの結果に対して別のDNSクエリが実行され、受信したすべてのアドレスがピアアドレスと比較されます。 正規表現は$で終わることを強くお勧めします。 -すべての結果のdnsの要求をキャッシュまでのサーバが再起動してしまいます。 +すべての結果のDNSの要求をキャッシュまでのサーバが再起動してしまいます。 **例** @@ -114,22 +130,22 @@ toc_title: "\u30E6\u30FC\u30B6\u30FC\u8A2D\u5B9A" ### user\_name/プロファイル {#user-nameprofile} -を割り当てることができる設定プロファイルをユーザーです。 設定プロファイルはの別のセクションで設定されます `users.xml` ファイル。 詳細については、 [設定のプロファイル](settings-profiles.md). +を割り当てることができる設定プロファイルをユーザーです。 設定プロファイルは、 `users.xml` ファイル 詳細については、 [設定のプロファイル](settings-profiles.md). -### ユーザー名/クォータ {#user-namequota} +### user\_name/クォータ {#user-namequota} -クォータを使用すると、一定期間にわたってリソース使用量を追跡または制限できます。 クォータは、 `quotas` -のセクション `users.xml` 構成ファイル。 +クォータを使用すると、一定期間のリソース使用量を追跡または制限できます。 クォータは `quotas` +のセクション `users.xml` 設定ファイル。 ユーザにクォータセットを割り当てることができます。 クォータ設定の詳細については、以下を参照してください [クォータ](../quotas.md#quotas). ### user\_name/データベース {#user-namedatabases} -このセクションでは、clickhouseによって返される行を制限することができます `SELECT` 現在のユーザーが行うクエリは、基本的な行レベルのセキュリティを実装します。 +このセクションでは、ClickHouseによって返される行を以下の目的で制限することができます `SELECT` クエリーによる、現在のユーザが実施基本列レベルです。 -**例えば** +**例** -以下の構成力がユーザー `user1` の行だけを見ることができます `table1` の結果として `SELECT` クエリ、ここでの値 `id` フィールドは1000です。 +以下の構成力がユーザー `user1` の行だけを見ることができます `table1` の結果として `SELECT` クエリの値は、次のとおりです。 `id` フィールドは1000です。 ``` xml @@ -143,6 +159,6 @@ toc_title: "\u30E6\u30FC\u30B6\u30FC\u8A2D\u5B9A" ``` -その `filter` 結果として得られる任意の式を指定できます [UInt8](../../sql-reference/data-types/int-uint.md)-タイプ値。 通常、比較演算子と論理演算子が含まれます。 からの行 `database_name.table1` このユーザーに対して0のフィルター結果は返されません。 フィルタリングは `PREWHERE` 操作および無効化 `WHERE→PREWHERE` 最適化。 +その `filter` 任意の式にすることができます。 [UInt8](../../sql-reference/data-types/int-uint.md)-タイプ値。 通常、比較演算子と論理演算子が含まれています。 からの行 `database_name.table1` る結果をフィルターを0においても返却いたしませんこのユーザーです。 このフィルタリングは `PREWHERE` 操作と無効 `WHERE→PREWHERE` 最適化。 [元の記事](https://clickhouse.tech/docs/en/operations/settings/settings_users/) diff --git a/docs/ja/operations/settings/settings.md b/docs/ja/operations/settings/settings.md index a369a85f10e..be97c0934b7 100644 --- a/docs/ja/operations/settings/settings.md +++ b/docs/ja/operations/settings/settings.md @@ -1,15 +1,13 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_priority: 60 -toc_title: "\u8A2D\u5B9A" +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd --- # 設定 {#settings} ## distributed\_product\_mode {#distributed-product-mode} -の動作を変更します。 [分散サブクエリ](../../sql-reference/statements/select.md). +の動作を変更します [分散サブクエリ](../../sql-reference/operators/in.md). ClickHouse applies this setting when the query contains the product of distributed tables, i.e. when the query for a distributed table contains a non-GLOBAL subquery for the distributed table. @@ -24,14 +22,14 @@ ClickHouse applies this setting when the query contains the product of distribut - `deny` — Default value. Prohibits using these types of subqueries (returns the “Double-distributed in/JOIN subqueries is denied” 例外）。 - `local` — Replaces the database and table in the subquery with local ones for the destination server (shard), leaving the normal `IN`/`JOIN.` -- `global` — Replaces the `IN`/`JOIN` クエリと `GLOBAL IN`/`GLOBAL JOIN.` +- `global` — Replaces the `IN`/`JOIN` クエリ `GLOBAL IN`/`GLOBAL JOIN.` - `allow` — Allows the use of these types of subqueries. ## enable\_optimize\_predicate\_expression {#enable-optimize-predicate-expression} 述語プッシュダウンをオンにする `SELECT` クエリ。 -プレディケートプッシュダウ場合を大幅に削減ネットワーク通信のために配布します。 +述語プッシュダウ 可能な値: @@ -40,7 +38,7 @@ ClickHouse applies this setting when the query contains the product of distribut デフォルト値:1。 -使い方 +使用法 次のクエリを検討します: @@ -49,41 +47,41 @@ ClickHouse applies this setting when the query contains the product of distribut もし `enable_optimize_predicate_expression = 1` ClickHouseが適用されるため、これらのクエリの実行時間は等しくなります `WHERE` それを処理するときにサブクエリに。 -もし `enable_optimize_predicate_expression = 0` その後、第二のクエリの実行時間がはるかに長いです。 `WHERE` サブクエリが終了した後、すべてのデータに句が適用されます。 +もし `enable_optimize_predicate_expression = 0` 次に、第二のクエリの実行時間ははるかに長くなります。 `WHERE` 句は、サブクエリの終了後にすべてのデータに適用されます。 -## fallback\_to\_stale\_replicas\_for\_distributed\_queries {#settings-fallback_to_stale_replicas_for_distributed_queries} +## フォールバック\_to\_stale\_replicas\_for\_distributed\_queries {#settings-fallback_to_stale_replicas_for_distributed_queries} -更新されたデータが利用できない場合、クエリを古いレプリカに強制的に適用します。 見る [複製](../../engines/table-engines/mergetree-family/replication.md). +更新されたデータが使用できない場合は、クエリを古いレプリカに強制します。 見る [複製](../../engines/table-engines/mergetree-family/replication.md). ClickHouseは、テーブルの古いレプリカから最も関連性の高いものを選択します。 -実行するときに使用 `SELECT` レプリケートされたテーブルを指す分散テーブルから +実行時使用 `SELECT` 複製されたテーブルを指す分散テーブルから。 -デフォルトでは、1(有効)。 +デフォルトでは、1(有効)です。 ## force\_index\_by\_date {#settings-force_index_by_date} インデックスを日付で使用できない場合は、クエリの実行を無効にします。 -MergeTreeファミリーのテーブルで動作します。 +MergeTreeファミリ内のテーブルで動作します。 -もし `force_index_by_date=1` ClickHouseは、データ範囲の制限に使用できる日付キー条件がクエリにあるかどうかをチェックします。 適切な条件がない場合は、例外がスローされます。 ただし、読み取るデータの量が条件によって減少するかどうかはチェックされません。 たとえば、条件 `Date != ' 2000-01-01 '` テーブル内のすべてのデータに一致する場合でも許容されます（つまり、クエリを実行するにはフルスキャンが必要です）。 MergeTreeテーブルのデータ範囲の詳細については、次を参照してください [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md). +もし `force_index_by_date=1`,ClickHouseは、クエリにデータ範囲を制限するために使用できる日付キー条件があるかどうかをチェックします。 適切な条件がない場合は、例外をスローします。 ただし、条件によって読み取るデータ量が減少するかどうかはチェックされません。 たとえば、次の条件 `Date != ' 2000-01-01 '` テーブル内のすべてのデータと一致する場合でも許容されます（つまり、クエリを実行するにはフルスキャンが必要です）。 MergeTreeテーブル内のデータ範囲の詳細については、以下を参照してください [メルゲツリー](../../engines/table-engines/mergetree-family/mergetree.md). ## force\_primary\_key {#force-primary-key} -オクエリの実行が指数付けにより、主キーはできません。 +主キーによる索引付けが不可能な場合は、クエリの実行を無効にします。 -MergeTreeファミリーのテーブルで動作します。 +MergeTreeファミリ内のテーブルで動作します。 -もし `force_primary_key=1` ClickHouseは、データ範囲の制限に使用できる主キー条件がクエリにあるかどうかを確認します。 適切な条件がない場合は、例外がスローされます。 ただし、読み取るデータの量が条件によって減少するかどうかはチェックされません。 MergeTreeテーブルのデータ範囲の詳細については、 [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md). +もし `force_primary_key=1`,ClickHouseは、クエリにデータ範囲を制限するために使用できる主キー条件があるかどうかを確認します。 適切な条件がない場合は、例外をスローします。 ただし、条件によって読み取るデータ量が減少するかどうかはチェックされません。 MergeTreeテーブルのデータ範囲の詳細については、以下を参照してください [メルゲツリー](../../engines/table-engines/mergetree-family/mergetree.md). ## format\_schema {#format-schema} -このパラメーターは、次のようなスキーマ定義を必要とする形式を使用する場合に便利です [Cap’n Proto](https://capnproto.org/) または [Protobuf](https://developers.google.com/protocol-buffers/). 値は形式によって異なります。 +このパラメーターは、次のようなスキーマ定義を必要とする形式を使用している場合に便利です [Cap'N Proto](https://capnproto.org/) または [プロトブフ](https://developers.google.com/protocol-buffers/). 値は形式によって異なります。 ## fsync\_metadata {#fsync-metadata} -有効または無効 [fsync](http://pubs.opengroup.org/onlinepubs/9699919799/functions/fsync.html) 書くとき `.sql` ファイル 既定で有効になっています。 +有効または無効にします [fsync](http://pubs.opengroup.org/onlinepubs/9699919799/functions/fsync.html) 書くとき `.sql` ファイル 既定で有効になっています。 ことは、あってはならないことで無効にすることもできれば、サーバは、数百万の小さなテーブルが続々と生まれてくると破壊されました。 @@ -91,104 +89,104 @@ MergeTreeファミリーのテーブルで動作します。 HTTP要求に対する応答のデータ圧縮を有効または無効にします。 -詳細については、 [HTTPインタフェースの説明](../../interfaces/http.md). +詳細については、 [HTTP](../../interfaces/http.md). 可能な値: - 0 — Disabled. - 1 — Enabled. -デフォルト値:0. +デフォルト値は0です。 ## http\_zlib\_compression\_level {#settings-http_zlib_compression_level} -HTTP要求に対する応答のデータ圧縮レベルを次の場合に設定します [enable\_http\_compression=1](#settings-enable_http_compression). +HTTP要求に対する応答のデータ圧縮のレベルを次の場合に設定します [enable\_http\_compression=1](#settings-enable_http_compression). -可能な値：1から9までの数字。 +可能な値:1から9までの数値。 -デフォルト値:3. +デフォルト値は3です。 ## http\_native\_compression\_disable\_checksumming\_on\_decompress {#settings-http_native_compression_disable_checksumming_on_decompress} -クライアントからのhttp postデータの解凍時にチェックサム検証を有効または無効にします。 にのみ使用clickhouseネイティブの圧縮フォーマット（使用されません `gzip` または `deflate`). +クライア ClickHouseネイティブ圧縮フォーマットにのみ使用されます( `gzip` または `deflate`). -詳細については、 [HTTPインタフェースの説明](../../interfaces/http.md). +詳細については、 [HTTP](../../interfaces/http.md). 可能な値: - 0 — Disabled. - 1 — Enabled. -デフォルト値:0. +デフォルト値は0です。 ## send\_progress\_in\_http\_headers {#settings-send_progress_in_http_headers} -有効または無効 `X-ClickHouse-Progress` HTTP応答ヘッダー `clickhouse-server` 応答。 +有効または無効にします `X-ClickHouse-Progress` HTTP応答ヘッダー `clickhouse-server` 応答。 -詳細については、 [HTTPインタフェースの説明](../../interfaces/http.md). +詳細については、 [HTTP](../../interfaces/http.md). 可能な値: - 0 — Disabled. - 1 — Enabled. -デフォルト値:0. +デフォルト値は0です。 ## max\_http\_get\_redirects {#setting-max_http_get_redirects} -HTTP GETリダイレクトホップの最大数を制限する [URL](../../engines/table-engines/special/url.md)-エンジンテーブル。 この設定は、両方のタイプのテーブルに適用されます。 [CREATE TABLE](../../query_language/create/#create-table-query) クエリとによって [url](../../sql-reference/table-functions/url.md) テーブル機能。 +HTTP GETリダイレクトホップの最大数を制限します。 [URL](../../engines/table-engines/special/url.md)-エンジンテーブル。 この設定は、両方のタイプのテーブルに適用されます。 [CREATE TABLE](../../sql-reference/statements/create.md#create-table-query) クエリとによって [url](../../sql-reference/table-functions/url.md) テーブル関数。 可能な値: -- 任意の正の整数のホップ数。 +- 任意の正の整数ホップ数。 - 0 — No hops allowed. -デフォルト値:0. +デフォルト値は0です。 ## input\_format\_allow\_errors\_num {#settings-input_format_allow_errors_num} -テキスト形式（csv、tsvなど）から読み取るときに許容されるエラーの最大数を設定します。). +テキスト形式（CSV、TSVなど）から読み取るときに許容されるエラーの最大数を設定します。). -デフォルト値は0です。 +既定値は0です。 -常にペアそれと `input_format_allow_errors_ratio`. +常にそれをペアにします `input_format_allow_errors_ratio`. -行の読み取り中にエラーが発生したが、エラーカウンタがまだ小さい場合 `input_format_allow_errors_num`、ClickHouseは、行を無視して、次のいずれかに移動します。 +行の読み取り中にエラーが発生したが、エラーカウンタがそれより小さい場合 `input_format_allow_errors_num`,ClickHouseは行を無視し、次の行に移動します。 -両方の場合 `input_format_allow_errors_num` と `input_format_allow_errors_ratio` 超過すると、ClickHouseは例外をスローします。 +両方の場合 `input_format_allow_errors_num` と `input_format_allow_errors_ratio` を超えた場合、ClickHouseは例外をスローします。 ## input\_format\_allow\_errors\_ratio {#settings-input_format_allow_errors_ratio} -テキスト形式（csv、tsvなど）から読み取るときに許可されるエラーの最大パーセントを設定します。). -エラーの割合は、0～1の間の浮動小数点数として設定されます。 +テキスト形式（CSV、TSVなど）から読み取るときに許容されるエラーの最大割合を設定します。). +エラーの割合は、0から1の間の浮動小数点数として設定されます。 -デフォルト値は0です。 +既定値は0です。 -常にペアそれと `input_format_allow_errors_num`. +常にそれをペアにします `input_format_allow_errors_num`. -行の読み取り中にエラーが発生したが、エラーカウンタがまだ小さい場合 `input_format_allow_errors_ratio`、ClickHouseは、行を無視して、次のいずれかに移動します。 +行の読み取り中にエラーが発生したが、エラーカウンタがそれより小さい場合 `input_format_allow_errors_ratio`,ClickHouseは行を無視し、次の行に移動します。 -両方の場合 `input_format_allow_errors_num` と `input_format_allow_errors_ratio` 超過すると、ClickHouseは例外をスローします。 +両方の場合 `input_format_allow_errors_num` と `input_format_allow_errors_ratio` を超えた場合、ClickHouseは例外をスローします。 ## input\_format\_values\_interpret\_expressions {#settings-input_format_values_interpret_expressions} -を有効または無効にしのsqlのパーサの場合の高速ストリームのパーサで構文解析のデータです。 この設定は、 [値](../../interfaces/formats.md#data-format-values) データ挿入時のフォーマット。 構文の解析の詳細については、以下を参照してください [構文](../../sql-reference/syntax.md) セクション。 +を有効または無効にしのSQLのパーサの場合の高速ストリームのパーサで構文解析のデータです。 この設定は、 [値](../../interfaces/formats.md#data-format-values) データ挿入時の書式。 構文解析の詳細については、以下を参照してください [構文](../../sql-reference/syntax.md) セクション 可能な値: - 0 — Disabled. - この場合、提供しなければなりません形式のデータです。 を見る [形式](../../interfaces/formats.md) セクション。 + この場合、提供しなければなりません形式のデータです。 を参照。 [形式](../../interfaces/formats.md) セクション - 1 — Enabled. - この場合、sql式を値として使用できますが、データの挿入はこの方法ではるかに遅くなります。 書式設定されたデータのみを挿入する場合、clickhouseは設定値が0であるかのように動作します。 + この場合、SQL式を値として使用できますが、データの挿入はこの方法ではるかに遅くなります。 書式設定されたデータのみを挿入すると、ClickHouseは設定値が0であるかのように動作します。 デフォルト値:1。 使用例 -を挿入 [DateTime](../../sql-reference/data-types/datetime.md) 異なる設定で値を入力します。 +挿入 [DateTime](../../sql-reference/data-types/datetime.md) 異なる設定で値を入力します。 ``` sql SET input_format_values_interpret_expressions = 0; @@ -209,7 +207,7 @@ INSERT INTO datetime_t VALUES (now()) Ok. ``` -最後のクエリは次のクエリと同じです: +最後のクエリは次のものと同じです: ``` sql SET input_format_values_interpret_expressions = 0; @@ -222,21 +220,28 @@ Ok. ## input\_format\_values\_deduce\_templates\_of\_expressions {#settings-input_format_values_deduce_templates_of_expressions} -Sql式のテンプレート控除を有効または無効にします。 [値](../../interfaces/formats.md#data-format-values) フォーマット。 これにより、式の解析と解釈が可能になります。 `Values` 連続する行の式が同じ構造を持つ場合、はるかに高速です。 ClickHouseは、式のテンプレートを推測し、このテンプレートを使用して次の行を解析し、正常に解析された行のバッチで式を評価しようとします。 次のクエリの場合: +SQL式のテンプレート控除を有効または無効にします。 [値](../../interfaces/formats.md#data-format-values) 形式。 この解析と解釈表現 `Values` 連続する行の式が同じ構造を持つ場合、はるかに高速です。 ClickHouseは式のテンプレートを推論し、このテンプレートを使用して次の行を解析し、正常に解析された行のバッチで式を評価しようとします。 + +可能な値: + +- 0 — Disabled. +- 1 — Enabled. + +デフォルト値:1。 + +次のクエリの場合: ``` sql INSERT INTO test VALUES (lower('Hello')), (lower('world')), (lower('INSERT')), (upper('Values')), ... ``` -- もし `input_format_values_interpret_expressions=1` と `format_values_deduce_templates_of_expressions=0` 式は行ごとに別々に解釈されます（これは、多数の行では非常に遅いです) -- もし `input_format_values_interpret_expressions=0` と `format_values_deduce_templates_of_expressions=1` 最初、二番目、および三番目の行の式は、テンプレートを使用して解析されます `lower(String)` 一緒に解釈されると、式はforth行が別のテンプレートで解析されます (`upper(String)`) -- もし `input_format_values_interpret_expressions=1` と `format_values_deduce_templates_of_expressions=1` -前の場合と同じですが、テンプレートを推論することができない場合は、式を別々に解釈することもできます。 - -既定で有効になっています。 +- もし `input_format_values_interpret_expressions=1` と `format_values_deduce_templates_of_expressions=0`,式は各行ごとに別々に解釈されます（これは多数の行では非常に遅いです）。 +- もし `input_format_values_interpret_expressions=0` と `format_values_deduce_templates_of_expressions=1`、第一、第二および第三の行の式は、テンプレートを使用して解析されます `lower(String)` そして一緒に解釈されると、forth行の式は別のテンプレートで解析されます (`upper(String)`). +- もし `input_format_values_interpret_expressions=1` と `format_values_deduce_templates_of_expressions=1`、前の場合と同じですが、テンプレートを推論できない場合は、式を別々に解釈することもできます。 ## input\_format\_values\_accurate\_types\_of\_literals {#settings-input-format-values-accurate-types-of-literals} -この設定は次の場合にのみ使用されます `input_format_values_deduce_templates_of_expressions = 1`. いくつかの列の式は同じ構造を持ちますが、異なる型の数値リテラルを含んでいます +この設定は次の場合にのみ使用されます `input_format_values_deduce_templates_of_expressions = 1`. いくつかの列の式は同じ構造を持ちますが、異なる型の数値リテラルが含まれています。 ``` sql (..., abs(0), ...), -- UInt64 literal @@ -244,16 +249,24 @@ INSERT INTO test VALUES (lower('Hello')), (lower('world')), (lower('INSERT')), ( (..., abs(-1), ...), -- Int64 literal ``` -この設定を有効にすると、clickhouseは実際のリテラルの型をチェックし、対応する型の式テンプレートを使用します。 場合によっては、式の評価が大幅に遅くなることがあります。 `Values`. -When disabled, ClickHouse may use more general type for some literals (e.g. `Float64` または `Int64` 代わりに `UInt64` のために `42`が、その原因となりオーバーフローおよび精度の問題です。 -既定で有効になっています。 +可能な値: + +- 0 — Disabled. + + In this case, ClickHouse may use a more general type for some literals (e.g., `Float64` または `Int64` 代わりに `UInt64` のために `42`が、その原因となりオーバーフローおよび精度の問題です。 + +- 1 — Enabled. + + この場合、ClickHouseはリテラルの実際の型をチェックし、対応する型の式テンプレートを使用します。 場合によっては、 `Values`. + +デフォルト値:1。 ## input\_format\_defaults\_for\_omitted\_fields {#session_settings-input_format_defaults_for_omitted_fields} -実行するとき `INSERT` クエリでは、省略された入力列の値をそれぞれの列の既定値に置き換えます。 このオプションは [JSONEachRow](../../interfaces/formats.md#jsoneachrow), [CSV](../../interfaces/formats.md#csv) と [タブ区切り](../../interfaces/formats.md#tabseparated) フォーマット。 +実行するとき `INSERT` 省略された入力列の値を、それぞれの列のデフォルト値に置き換えます。 このオプションは [JSONEachRow](../../interfaces/formats.md#jsoneachrow), [CSV](../../interfaces/formats.md#csv) と [TabSeparated](../../interfaces/formats.md#tabseparated) フォーマット。 -!!! note "メモ" - このオプショ それはサーバーの付加的な計算資源を消費し、性能を減らすことができる。 +!!! note "注" + このオプショ 消費量で追加的に計算資源をサーバーを低減できる。 可能な値: @@ -264,25 +277,25 @@ When disabled, ClickHouse may use more general type for some literals (e.g. `Fl ## input\_format\_tsv\_empty\_as\_default {#settings-input-format-tsv-empty-as-default} -有効にすると、tsvの空の入力フィールドを既定値に置き換えます。 複雑な既定の式の場合 `input_format_defaults_for_omitted_fields` 有効にする必要があります。 +有効にすると、TSVの空の入力フィールドを既定値に置き換えます。 複雑な既定の式の場合 `input_format_defaults_for_omitted_fields` 有効にする必要があります。 -デフォルトでは無効です。 +既定では無効です。 ## input\_format\_null\_as\_default {#settings-input-format-null-as-default} -入力デー `NULL` しかし、対応する列のデータ型は `Nullable(T)` (テキスト入力形式の場合)。 +を有効または無効にし用のデフォルト値が入力データを含む `NULL` しかし、notの対応する列のデータ型 `Nullable(T)` （テキスト入力形式の場合）。 ## input\_format\_skip\_unknown\_fields {#settings-input-format-skip-unknown-fields} -追加データの挿入のスキップを有効または無効にします。 +追加データのスキップ挿入を有効または無効にします。 -書き込みデータclickhouseが例外をスローした場合入力データを含むカラム特別な権限は必要ありません使用します。 スキップが有効になっている場合、clickhouseは余分なデータを挿入せず、例外もスローしません。 +書き込みデータClickHouseが例外をスローした場合入力データを含むカラム特別な権限は必要ありません使用します。 スキップが有効な場合、ClickHouseは余分なデータを挿入せず、例外をスローしません。 -対応フォーマット: +対応形式: - [JSONEachRow](../../interfaces/formats.md#jsoneachrow) -- [Csvwithnamesname](../../interfaces/formats.md#csvwithnames) -- [Tabseparatedwithnamesname](../../interfaces/formats.md#tabseparatedwithnames) +- [CSVWithNames](../../interfaces/formats.md#csvwithnames) +- [TabSeparatedWithNames](../../interfaces/formats.md#tabseparatedwithnames) - [TSKV](../../interfaces/formats.md#tskv) 可能な値: @@ -290,13 +303,13 @@ When disabled, ClickHouse may use more general type for some literals (e.g. `Fl - 0 — Disabled. - 1 — Enabled. -デフォルト値:0. +デフォルト値は0です。 ## input\_format\_import\_nested\_json {#settings-input_format_import_nested_json} -を有効または無効にし、挿入のjsonデータをネストしたオブジェクト。 +を有効または無効にし、挿入のJSONデータをネストしたオブジェクト。 -対応フォーマット: +対応形式: - [JSONEachRow](../../interfaces/formats.md#jsoneachrow) @@ -305,22 +318,22 @@ When disabled, ClickHouse may use more general type for some literals (e.g. `Fl - 0 — Disabled. - 1 — Enabled. -デフォルト値:0. +デフォルト値は0です。 -また見なさい: +も参照。: -- [入れ子構造の使用法](../../interfaces/formats.md#jsoneachrow-nested) と `JSONEachRow` フォーマット。 +- [入れ子構造の使用](../../interfaces/formats.md#jsoneachrow-nested) と `JSONEachRow` 形式。 ## input\_format\_with\_names\_use\_header {#settings-input-format-with-names-use-header} -データ挿入時に列の順序の確認を有効または無効にします。 +データ挿入時の列順序の確認を有効または無効にします。 -挿入パフォーマンスを向上させるには、入力データの列の順序がターゲットテーブルと同じであることが確実な場合は、このチェックを無効にすることを +挿入のパフォーマンスを向上させるために、入力データの列の順序がターゲットテーブルと同じであることが確実な場合は、このチェックを無効にするこ -対応フォーマット: +対応形式: -- [Csvwithnamesname](../../interfaces/formats.md#csvwithnames) -- [Tabseparatedwithnamesname](../../interfaces/formats.md#tabseparatedwithnames) +- [CSVWithNames](../../interfaces/formats.md#csvwithnames) +- [TabSeparatedWithNames](../../interfaces/formats.md#tabseparatedwithnames) 可能な値: @@ -333,28 +346,28 @@ When disabled, ClickHouse may use more general type for some literals (e.g. `Fl 日付と時刻のテキスト表現のパーサーを選択できます。 -この設定は、以下には適用されません [日付と時刻の関数](../../sql-reference/functions/date-time-functions.md). +この設定は次の場合には適用されません [日付と時刻の関数](../../sql-reference/functions/date-time-functions.md). 可能な値: - `'best_effort'` — Enables extended parsing. - ClickHouseは基本を解析することができます `YYYY-MM-DD HH:MM:SS` 形式とすべて [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601) 日付と時刻の形式。 例えば, `'2018-06-08T01:02:03.000Z'`. + ClickHouseは基本を解析できます `YYYY-MM-DD HH:MM:SS` 書式とすべて [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601) 日付と時刻の形式。 例えば, `'2018-06-08T01:02:03.000Z'`. - `'basic'` — Use basic parser. - ClickHouseは基本のみを解析できます `YYYY-MM-DD HH:MM:SS` フォーマット。 例えば, `'2019-08-20 10:18:56'`. + ClickHouseは基本のみを解析できます `YYYY-MM-DD HH:MM:SS` 形式。 例えば, `'2019-08-20 10:18:56'`. デフォルト値: `'basic'`. -また見なさい: +も参照。: - [DateTimeデータ型。](../../sql-reference/data-types/datetime.md) - [日付と時刻を操作するための関数。](../../sql-reference/functions/date-time-functions.md) ## join\_default\_strictness {#settings-join_default_strictness} -デフォルトの厳密さを [結合句](../../sql-reference/statements/select.md#select-join). +デフォルトの厳密さを [結合句](../../sql-reference/statements/select/join.md#select-join). 可能な値: @@ -367,104 +380,104 @@ When disabled, ClickHouse may use more general type for some literals (e.g. `Fl ## join\_any\_take\_last\_row {#settings-join_any_take_last_row} -Join操作の動作を次のもので変更する `ANY` 厳密さ +結合操作の動作を `ANY` 厳密さ。 !!! warning "注意" - この設定は、 `JOIN` との操作 [参加](../../engines/table-engines/special/join.md) エンジンテーブル。 + この設定は `JOIN` との操作 [参加](../../engines/table-engines/special/join.md) エンジンテーブル。 可能な値: - 0 — If the right table has more than one matching row, only the first one found is joined. - 1 — If the right table has more than one matching row, only the last one found is joined. -デフォルト値:0. +デフォルト値は0です。 -また見なさい: +も参照。: -- [JOIN句](../../sql-reference/statements/select.md#select-join) +- [JOIN句](../../sql-reference/statements/select/join.md#select-join) - [結合テーブルエンジン](../../engines/table-engines/special/join.md) - [join\_default\_strictness](#settings-join_default_strictness) ## join\_use\_nulls {#join_use_nulls} -のタイプを設定します。 [JOIN](../../sql-reference/statements/select.md) 行動。 際融合のテーブル、空細胞が表示される場合があります。 ClickHouseは、この設定に基づいて異なる塗りつぶします。 +のタイプを設定します。 [JOIN](../../sql-reference/statements/select/join.md) 行動。 際融合のテーブル、空細胞が表示される場合があります。 ClickHouseは、この設定に基づいて異なる塗りつぶします。 可能な値: - 0 — The empty cells are filled with the default value of the corresponding field type. -- 1 — `JOIN` 標準SQLと同じように動作します。 対応するフィールドの型は次のように変換されます [Nullable](../../sql-reference/data-types/nullable.md#data_type-nullable) 空のセルは [NULL](../../sql-reference/syntax.md). +- 1 — `JOIN` 標準SQLと同じように動作します。 対応するフィールドの型は次のように変換されます [Null可能](../../sql-reference/data-types/nullable.md#data_type-nullable) 空のセルは [NULL](../../sql-reference/syntax.md). -デフォルト値:0. +デフォルト値は0です。 ## max\_block\_size {#setting-max_block_size} -ClickHouseでは、データはブロック（列部分のセット）によって処理されます。 単一のブロックの内部処理サイクルは十分に効率的ですが、各ブロックに顕著な支出があります。 その `max_block_size` 設定は、テーブルからロードするブロックのサイズ（行数）の推奨値です。 ブロックサイズが小さすぎないようにして、各ブロックの支出はまだ目立つが、最初のブロックが迅速に処理された後に完了する制限付きのクエ 目標は、複数のスレッドで多数の列を抽出するときに大量のメモリを消費しないようにし、少なくともいくつかのキャッシュの局所性を維持する +ClickHouseでは、データはブロック（列部分のセット）によって処理されます。 単一ブロックの内部処理サイクルは十分に効率的ですが、各ブロックには顕著な支出があります。 その `max_block_size` 設定は、テーブルからロードするブロックのサイズ（行数）の推奨事項です。 ブロックサイズは小さすぎるので、各ブロックの支出はまだ目立ちますが、最初のブロックの後に完了したLIMITのクエリがすばやく処理されるよう 目標は、複数のスレッドで多数の列を抽出するときにメモリを消費するのを避け、少なくともいくつかのキャッシュの局所性を保持することです。 -デフォルト値:65,536. +デフォルト値は65,536です。 -ブロックのサイズ `max_block_size` ていないから読み込まれます。 少ないデータを取得する必要があることが明らかであれば、小さいブロックが処理されます。 +ブロックのサイズ `max_block_size` ていないから読み込まれます。 場合ことは明らかであることなくデータを取得され、さらに小型のブロックを処理します。 ## preferred\_block\_size\_bytes {#preferred-block-size-bytes} -同じ目的のために使用される `max_block_size` しかし、ブロック内の行数に適応させることによって、推奨されるブロックサイズをバイト単位で設定します。 -ただし、ブロックサイズは `max_block_size` 行。 -デフォルト:1,000,000。 mergetreeエンジンから読み取る場合にのみ機能します。 +と同じ目的のために使用される `max_block_size` しかし、ブロック内の行数に適応させることによって、推奨されるブロックサイズをバイト単位で設定します。 +ただし、ブロックサイズは次のようになります `max_block_size` 行。 +既定では、1,000,000です。 MergeTreeエンジンから読み取るときにのみ動作します。 ## merge\_tree\_min\_rows\_for\_concurrent\_read {#setting-merge-tree-min-rows-for-concurrent-read} -Aのファイルから読み込まれる行の数 [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) テーブルを超え `merge_tree_min_rows_for_concurrent_read` その後ClickHouseしようとして行な兼職の状況からの読み出しこのファイルに複数のスレッド）。 +のファイルから読み込まれる行数が [メルゲツリー](../../engines/table-engines/mergetree-family/mergetree.md) テーブル超過 `merge_tree_min_rows_for_concurrent_read` その後ClickHouseしようとして行な兼職の状況からの読み出しこのファイルに複数のスレッド）。 可能な値: - 任意の正の整数。 -デフォルト値:163840. +デフォルト値は163840です。 ## merge\_tree\_min\_bytes\_for\_concurrent\_read {#setting-merge-tree-min-bytes-for-concurrent-read} -ファイルから読み込むバイト数 [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md)-エンジンテーブル超え `merge_tree_min_bytes_for_concurrent_read` そのClickHouseを同時に読みこのファイルから複数のスレッド）。 +のファイルから読み取るバイト数が [メルゲツリー](../../engines/table-engines/mergetree-family/mergetree.md)-エンジン表超過 `merge_tree_min_bytes_for_concurrent_read` 次に、ClickHouseはこのファイルから複数のスレッドで同時に読み取ろうとします。 可能な値: - 任意の正の整数。 -デフォルト値:251658240. +デフォルト値は251658240です。 ## merge\_tree\_min\_rows\_for\_seek {#setting-merge-tree-min-rows-for-seek} -ファイル内で読み込まれる二つのデータブロック間の距離がより小さい場合 `merge_tree_min_rows_for_seek` その後、ClickHouseはファイルをシークしませんが、データを順次読み取ります。 +一つのファイルに読み込まれる二つのデータブロック間の距離が `merge_tree_min_rows_for_seek` その後、ClickHouseはファイルをシークせず、データを順番に読み込みます。 可能な値: - 任意の正の整数。 -デフォルト値:0. +デフォルト値は0です。 ## merge\_tree\_min\_bytes\_for\_seek {#setting-merge-tree-min-bytes-for-seek} -ファイル内で読み込まれる二つのデータブロック間の距離がより小さい場合 `merge_tree_min_bytes_for_seek` その後、ClickHouseは両方のブロックを含むファイルの範囲を順次読み取り、余分なシークを避けます。 +一つのファイルに読み込まれる二つのデータブロック間の距離が `merge_tree_min_bytes_for_seek` その後、ClickHouseは両方のブロックを含むファイルの範囲を順番に読み込むため、余分なシークを避けます。 可能な値: - 任意の正の整数。 -デフォルト値:0. +デフォルト値は0です。 -## merge\_tree\_coarse\_index\_granularitycomment {#setting-merge-tree-coarse-index-granularity} +## merge\_tree\_coarse\_index\_granularity {#setting-merge-tree-coarse-index-granularity} -する場合のデータclickhouseチェックのデータにファイルです。 まclickhouseが必要なキーの一部の範囲、とりわけこの範囲を `merge_tree_coarse_index_granularity` 必要なキーを再帰的に検索します。 +する場合のデータClickHouseチェックのデータにファイルです。 必要なキーがある範囲にあることをClickHouseが検出すると、この範囲を次のように分割します `merge_tree_coarse_index_granularity` 必要なキーを再帰的に検索します。 可能な値: - 任意の正の偶数の整数。 -デフォルト値:8. +デフォルト値:8。 ## merge\_tree\_max\_rows\_to\_use\_cache {#setting-merge-tree-max-rows-to-use-cache} -ClickHouseはより多くを読むべきであれば `merge_tree_max_rows_to_use_cache` あるクエリの行では、圧縮されていないブロックのキャッシュは使用されません。 +ClickHouseがより多くを読むべきであれば `merge_tree_max_rows_to_use_cache` あるクエリでは、非圧縮ブロックのキャッシュを使用しません。 -のキャッシュされた、圧縮解除されたブロックの店舗データを抽出したためます。 clickhouseこのキャッシュの高速化対応小の繰り返します。 この設定は、大量のデータを読み取るクエリによってキャッシュが破棄されるのを防ぎます。 その [uncompressed\_cache\_size](../server-configuration-parameters/settings.md#server-settings-uncompressed_cache_size) サーバー設定は、非圧縮ブロックのキャッシュのサイズを定義します。 +のキャッシュされた、圧縮解除されたブロックの店舗データを抽出したためます。 ClickHouseこのキャッシュの高速化対応小の繰り返します。 この設定は、大量のデータを読み取るクエリによってキャッシュが破損するのを防ぎます。 その [uncompressed\_cache\_size](../server-configuration-parameters/settings.md#server-settings-uncompressed_cache_size) サーバー設定は、非圧縮ブロックのキャッシュのサイズを定義します。 可能な値: @@ -474,48 +487,66 @@ Default value: 128 ✕ 8192. ## merge\_tree\_max\_bytes\_to\_use\_cache {#setting-merge-tree-max-bytes-to-use-cache} -ClickHouseはより多くを読むべきであれば `merge_tree_max_bytes_to_use_cache` バイトあるクエリでは、圧縮されていないブロックのキャッシュは使用されません。 +ClickHouseがより多くを読むべきであれば `merge_tree_max_bytes_to_use_cache` 一つのクエリでは、非圧縮ブロックのキャッシュを使用しません。 -のキャッシュされた、圧縮解除されたブロックの店舗データを抽出したためます。 clickhouseこのキャッシュの高速化対応小の繰り返します。 この設定は、大量のデータを読み取るクエリによってキャッシュが破棄されるのを防ぎます。 その [uncompressed\_cache\_size](../server-configuration-parameters/settings.md#server-settings-uncompressed_cache_size) サーバー設定は、非圧縮ブロックのキャッシュのサイズを定義します。 +のキャッシュされた、圧縮解除されたブロックの店舗データを抽出したためます。 ClickHouseこのキャッシュの高速化対応小の繰り返します。 この設定は、大量のデータを読み取るクエリによってキャッシュが破損するのを防ぎます。 その [uncompressed\_cache\_size](../server-configuration-parameters/settings.md#server-settings-uncompressed_cache_size) サーバー設定は、非圧縮ブロックのキャッシュのサイズを定義します。 可能な値: - 任意の正の整数。 -デフォルト値:2013265920. +既定値:2013265920。 ## min\_bytes\_to\_use\_direct\_io {#settings-min-bytes-to-use-direct-io} -記憶域ディスクへの直接i/oアクセスを使用するために必要な最小データ量。 +ストレージディスクへの直接I/Oアクセスを使用するために必要な最小データ量。 -ClickHouseこの設定からデータを読み込むときます。 読み取られるすべてのデータの合計ストレージボリュームが `min_bytes_to_use_direct_io` ディスクからデータを読み取ります。 `O_DIRECT` オプション。 +ClickHouseこの設定からデータを読み込むときます。 読み取るすべてのデータの合計ストレージ容量が超過した場合 `min_bytes_to_use_direct_io` その後、ClickHouseはストレージディスクからデータを読み取ります。 `O_DIRECT` オプション 可能な値: - 0 — Direct I/O is disabled. - 正の整数。 -デフォルト値:0. +デフォルト値は0です。 ## log\_queries {#settings-log-queries} クエリログの設定。 -この設定でclickhouseに送信されたクエリは、次のルールに従ってログに記録されます。 [クエリーログ](../server-configuration-parameters/settings.md#server_configuration_parameters-query-log) サーバー構成パラメータ。 +この設定でClickHouseに送信されたクエリは、 [query\_log](../server-configuration-parameters/settings.md#server_configuration_parameters-query-log) サーバー構成パラメータ。 -例えば: +例: ``` text log_queries=1 ``` +## log\_queries\_min\_type {#settings-log-queries-min-type} + +`query_log` ログに記録する最小タイプ。 + +可能な値: +- `QUERY_START` (`=1`) +- `QUERY_FINISH` (`=2`) +- `EXCEPTION_BEFORE_START` (`=3`) +- `EXCEPTION_WHILE_PROCESSING` (`=4`) + +デフォルト値: `QUERY_START`. + +どのエンティリーが行くかを制限するために使用できます `query_log` んだ興味深いだけ誤差を利用することができ `EXCEPTION_WHILE_PROCESSING`: + +``` text +log_queries_min_type='EXCEPTION_WHILE_PROCESSING' +``` + ## log\_query\_threads {#settings-log-query-threads} クエリスレッドログの設定。 -この設定でclickhouseによって実行されたクエリのスレッドは、以下のルールに従ってログに記録されます [query\_thread\_log](../server-configuration-parameters/settings.md#server_configuration_parameters-query-thread-log) サーバー構成パラメータ。 +この設定でClickHouseによって実行されたクエリのスレッドは、 [query\_thread\_log](../server-configuration-parameters/settings.md#server_configuration_parameters-query-thread-log) サーバー構成パラメータ。 -例えば: +例: ``` text log_query_threads=1 @@ -525,38 +556,60 @@ log_query_threads=1 テーブルに挿入するために形成するブロックのサイズ。 この設定は、サーバーがブロックを形成する場合にのみ適用されます。 -たとえば、httpインターフェイスを介した挿入の場合、サーバーはデータ形式を解析し、指定されたサイズのブロックを形成します。 -しかし、clickhouse-clientを使用すると、クライアントはデータ自体を解析し、 ‘max\_insert\_block\_size’ サーバー上の設定は、挿入されたブロックのサイズには影響しません。 -この設定は、select後に形成されるのと同じブロックを使用してデータが挿入されるため、insert selectを使用する場合にも目的がありません。 +たとえば、HTTPインターフェイスを介した挿入の場合、サーバーはデータ形式を解析し、指定されたサイズのブロックを形成します。 +しかし、clickhouse-clientを使用すると、クライアントはデータ自体を解析し、 ‘max\_insert\_block\_size’ サーバーでの設定は、挿入されたブロックのサイズには影響しません。 +データはSELECT後に形成されるのと同じブロックを使用して挿入されるため、INSERT SELECTを使用するときにもこの設定には目的がありません。 -デフォルト値:1,048,576. +既定値は1,048,576です。 -デフォルトは、 `max_block_size`. この理由は、特定のテーブルエンジン (`*MergeTree`）挿入された各ブロックのディスク上にデータ部分を形成する。 同様に, `*MergeTree` テーブルデータを並べ替え時の挿入やるのに十分な大きさのブロックサイズを選別データにアプリです。 +デフォルトは、 `max_block_size`. この理由は、特定のテーブルエンジンが原因です (`*MergeTree`）挿入されたブロックごとにディスク上にデータ部分を形成します。 同様に, `*MergeTree` テーブルデータを並べ替え時の挿入やるのに十分な大きさのブロックサイズを選別データにアプリです。 + +## min\_insert\_block\_size\_rows {#min-insert-block-size-rows} + +テーブルに挿入できるブロック内の最小行数を設定します。 `INSERT` クエリ。 小さめサイズのブロックをつぶし入ります。 + +可能な値: + +- 正の整数。 +- 0 — Squashing disabled. + +デフォルト値は1048576です。 + +## min\_insert\_block\_size\_bytes {#min-insert-block-size-bytes} + +テーブルに挿入できるブロック内の最小バイト数を設定します。 `INSERT` クエリ。 小さめサイズのブロックをつぶし入ります。 + +可能な値: + +- 正の整数。 +- 0 — Squashing disabled. + +デフォルト値:268435456。 ## max\_replica\_delay\_for\_distributed\_queries {#settings-max_replica_delay_for_distributed_queries} 分散クエリの遅延レプリカを無効にします。 見る [複製](../../engines/table-engines/mergetree-family/replication.md). -時間を秒単位で設定します。 レプリカが設定値よりも遅れている場合、このレプリカは使用されません。 +時間を秒単位で設定します。 レプリカが設定値より遅れている場合、このレプリカは使用されません。 -デフォルト値:300. +デフォルト値は300です。 -実行するときに使用 `SELECT` レプリケートされたテーブルを指す分散テーブルから +実行時使用 `SELECT` 複製されたテーブルを指す分散テーブルから。 ## max\_threads {#settings-max_threads} -最大の問合せ処理のスレッドを除き、スレッドの取得のためのデータからリモートサーバーの ‘max\_distributed\_connections’ パラメータ）。 +リモートサーバーからデータを取得するためのスレッドを除く、クエリ処理スレッドの最大数 ‘max\_distributed\_connections’ 変数）。 このパラメータに適用されるスレッドは、それらのスレッドが同じ段階での問合せ処理パイプライン。 -たとえば、テーブルから読み取るときに、関数を使用して式を評価できる場合は、whereとfilterを使用し、少なくともusingを使用してgroup byを並列に事前集計しま ‘max\_threads’ その後、スレッドの数 ‘max\_threads’ 使用されます。 +たとえば、テーブルから読み込むときに、関数で式を評価することができる場合は、少なくともを使用してWHEREとGROUP BYの事前集計を並列に使用してフィル ‘max\_threads’ その後、スレッドの数 ‘max\_threads’ 使用されます。 -デフォルト値:物理cpuコアの数。 +デフォルト値:物理CPUコアの数。 -通常、一度にサーバーで実行されるselectクエリが少ない場合は、このパラメーターを実際のプロセッサコア数より少し小さい値に設定します。 +サーバーで同時に実行されるSELECTクエリが通常より少ない場合は、このパラメーターを実際のプロセッサコア数よりわずかに小さい値に設定します。 -制限があるためすぐに完了するクエリの場合は、以下を設定できます ‘max\_threads’. たとえば、必要な数のエントリがすべてのブロックにあり、max\_threads=8の場合、8つのブロックが取得されますが、読み込むだけで十分です。 +制限のために迅速に完了するクエリの場合は、低い値を設定できます ‘max\_threads’. たとえば、必要な数のエントリがすべてのブロックにあり、max\_threads=8の場合、8つのブロックが取得されます。 -小さい `max_threads` 値は、消費されるメモリ量が少ない。 +小さいほど `max_threads` 値は、より少ないメモリが消費されます。 ## max\_insert\_threads {#settings-max-insert-threads} @@ -564,52 +617,52 @@ log_query_threads=1 可能な値: -- 0 (or 1) — `INSERT SELECT` 並列実行なし。 +- 0 (or 1) — `INSERT SELECT` 並列実行はありません。 - 正の整数。 1より大きい。 -デフォルト値:0. +デフォルト値は0です。 -並列 `INSERT SELECT` のみ有効です。 `SELECT` パートは並列に実行されます。 [max\_threads](#settings-max_threads) 設定。 -値を大きくすると、メモリ使用量が増加します。 +平行 `INSERT SELECT` のみ効果があります。 `SELECT` パーツは並列で実行されます。 [max\_threads](#settings-max_threads) 設定。 +値を大きくすると、メモリ使用量が増えます。 ## max\_compress\_block\_size {#max-compress-block-size} -テーブルへの書き込み用に圧縮する前の非圧縮データのブロックの最大サイズ。 デフォルトでは、1,048,576(1mib)。 サイズが縮小されると、圧縮率が大幅に低下し、キャッシュの局所性のために圧縮および圧縮解凍速度がわずかに増加し、メモリ消費が減少する。 通常、この設定を変更する理由はありません。 +テーブルに書き込むための圧縮前の非圧縮データのブロックの最大サイズ。 既定では、1,048,576(1MiB)です。 サイズを小さくすると、圧縮率が大幅に低下し、キャッシュの局所性のために圧縮と解凍の速度がわずかに増加し、メモリ消費が減少します。 通常、この設定を変更する理由はありません。 -圧縮のためのブロック（バイトからなるメモリの塊）とクエリ処理のためのブロック（テーブルからの行のセット）を混同しないでください。 +圧縮のためのブロック（バイトで構成されるメモリの塊）とクエリ処理のためのブロック（テーブルからの行のセット）を混同しないでください。 ## min\_compress\_block\_size {#min-compress-block-size} -のために [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md)"テーブル。 削減のため、遅延が処理クエリーのブロックの圧縮を書くとき、次のマークがそのサイズは少なくとも ‘min\_compress\_block\_size’. デフォルトでは、65,536。 +のために [メルゲツリー](../../engines/table-engines/mergetree-family/mergetree.md)"テーブル。 削減のため、遅延が処理クエリーのブロックの圧縮を書くとき、次のマークがそのサイズは少なくとも ‘min\_compress\_block\_size’. 既定では、65,536です。 -圧縮されていないデータが以下の場合、ブロックの実際のサイズ ‘max\_compress\_block\_size’、この値よりも小さく、一つのマークのためのデータの量よりも小さくありません。 +圧縮されていないデータが以下の場合、ブロックの実際のサイズ ‘max\_compress\_block\_size’ は、この値以上であり、一つのマークのデータ量以上である。 -例を見よう。 それを仮定する ‘index\_granularity’ テーブルの作成中に8192に設定されました。 +例を見てみましょう。 仮定すると ‘index\_granularity’ テーブル作成時に8192に設定されました。 -私たちはuint32型の列（値あたり4バイト）を書いています。 8192行を書き込む場合、合計は32kbのデータになります。 min\_compress\_block\_size=65,536以降、圧縮ブロックは二つのマークごとに形成されます。 +UInt32型の列（値あたり4バイト）を書いています。 8192行を書き込むと、合計は32KBのデータになります。 Min\_compress\_block\_size=65,536ので、圧縮されたブロックはすべてのマークに対して形成されます。 -私たちは、文字列型（値あたり60バイトの平均サイズ）でurl列を書いています。 8192行を書き込む場合、平均はデータの500kbよりわずかに小さくなります。 これは65,536以上であるため、各マークに圧縮ブロックが形成されます。 この場合、ディスクからシングルマークの範囲でデータを読み取るとき、余分なデータは解凍されません。 +文字列タイプ（値あたり60バイトの平均サイズ）のURL列を作成しています。 8192行を書き込むと、平均は500KBのデータよりわずかに小さくなります。 これは65,536以上であるため、各マークに圧縮されたブロックが形成されます。 この場合、単一のマークの範囲でディスクからデータを読み取るとき、余分なデータは解凍されません。 通常、この設定を変更する理由はありません。 ## max\_query\_size {#settings-max_query_size} -SQLパーサーで解析するためにRAMに取り込むことができるクエリの最大部分。 -INSERTクエリには、この制限に含まれていない別のストリームパーサー（O(1)RAMを消費する）によって処理されるINSERTのデータも含まれています。 +SQLパーサーを使用して解析するためにRAMに取り込むことができるクエリの最大部分。 +INSERTクエリには、別のストリームパーサー(O(1)RAMを消費する)によって処理されるINSERTのデータも含まれていますが、この制限には含まれていません。 -デフォルト値:256kib. +デフォルト値:256KiB。 ## interactive\_delay {#interactive-delay} 区間マイクロ秒単位で確認を行うための要求実行中止となり、送信を行います。 -デフォルト値:100,000(キャンセルのチェックを行い、進捗を毎秒十回送信します)。 +デフォルト値:100,000(キャンセルをチェックし、進行状況を秒単位で送信します)。 ## connect\_timeout,receive\_timeout,send\_timeout {#connect-timeout-receive-timeout-send-timeout} クライアントとの通信に使用されるソケットの秒単位のタイムアウト。 -デフォルト値:10、300、300。 +デフォルト値:10,300,300。 ## cancel\_http\_readonly\_queries\_on\_client\_close {#cancel-http-readonly-queries-on-client-close} @@ -619,71 +672,71 @@ Cancels HTTP read-only queries (e.g. SELECT) when a client closes the connectio ## poll\_interval {#poll-interval} -指定された秒数の待機ループでロックします。 +指定された秒数の待機ループをロックします。 -デフォルト値:10. +デフォルト値は10です。 ## max\_distributed\_connections {#max-distributed-connections} -単一の分散テーブルへの単一クエリの分散処理のためのリモートサーバーとの同時接続の最大数。 クラスター内のサーバー数以上の値を設定することをお勧めします。 +単一の分散テーブルへの単一のクエリの分散処理のためのリモートサーバーとの同時接続の最大数。 クラスター内のサーバー数以上の値を設定することをお勧めします。 デフォルト値:1024。 -次のパラメータは、分散テーブルを作成するとき（およびサーバーを起動するとき）にのみ使用されるため、実行時に変更する必要はありません。 +次のパラメーターは、分散テーブルを作成するとき(およびサーバーを起動するとき)にのみ使用されるため、実行時に変更する理由はありません。 ## distributed\_connections\_pool\_size {#distributed-connections-pool-size} -単一の分散テーブルへのすべてのクエリの分散処理のためのリモートサーバーとの同時接続の最大数。 クラスター内のサーバー数以上の値を設定することをお勧めします。 +単一の分散テーブルに対するすべてのクエリの分散処理のためのリモートサーバーとの同時接続の最大数。 クラスター内のサーバー数以上の値を設定することをお勧めします。 デフォルト値:1024。 ## connect\_timeout\_with\_failover\_ms {#connect-timeout-with-failover-ms} -分散テーブルエンジンのリモートサーバーに接続するためのタイムアウト(ミリ秒)。 ‘shard’ と ‘replica’ セクションはクラスター定義で使用されます。 -失敗した場合は、さまざまなレプリカへの接続を何度か試行します。 +分散テーブルエンジンのリモートサーバーに接続するためのタイムアウト時間(ミリ秒)。 ‘shard’ と ‘replica’ セクションは、クラスタ定義で使用されます。 +失敗した場合は、さまざまなレプリカへの接続が試行されます。 -デフォルト値:50。 +デフォルト値は50です。 ## connections\_with\_failover\_max\_tries {#connections-with-failover-max-tries} -分散テーブルエンジンの各レプリカでの接続試行の最大数。 +分散テーブルエンジンの各レプリカとの接続試行の最大数。 -デフォルト値:3. +デフォルト値は3です。 ## 極端な {#extremes} -極値(クエリ結果の列の最小値と最大値)をカウントするかどうか。 0または1を受け入れます。 デフォルトでは、0(無効)。 -詳細については、以下を参照してください “Extreme values”. +極端な値(クエリ結果の列の最小値と最大値)を数えるかどうか。 0または1を受け入れます。 既定では、0(無効)です。 +詳細については “Extreme values”. ## use\_uncompressed\_cache {#setting-use_uncompressed_cache} -非圧縮ブロックのキャッシュを使用するかどうか。 0または1を受け入れます。 デフォルトでは、0(無効)。 -圧縮されていないキャッシュ(mergetreeファミリーのテーブルのみ)を使用すると、多数の短いクエリを処理するときに待ち時間が大幅に短縮され、スループット この設定を有効にユーザーに送信頻繁に短います。 また、に注意を払う [uncompressed\_cache\_size](../server-configuration-parameters/settings.md#server-settings-uncompressed_cache_size) configuration parameter (only set in the config file) – the size of uncompressed cache blocks. By default, it is 8 GiB. The uncompressed cache is filled in as needed and the least-used data is automatically deleted. +非圧縮ブロックのキャッシュを使用するかどうか。 0または1を受け入れます。 既定では、0(無効)です。 +非圧縮キャッシュ(MergeTreeファミリ内のテーブルのみ)を使用すると、多数の短いクエリを処理する場合に、待ち時間を大幅に削減してスループットを向上させ この設定を有効にユーザーに送信頻繁に短います。 またに注意を払って下さい [uncompressed\_cache\_size](../server-configuration-parameters/settings.md#server-settings-uncompressed_cache_size) configuration parameter (only set in the config file) – the size of uncompressed cache blocks. By default, it is 8 GiB. The uncompressed cache is filled in as needed and the least-used data is automatically deleted. -少なくとも大量のデータ（百万行以上）を読み取るクエリの場合、圧縮されていないキャッシュは自動的に無効になり、本当に小さなクエリの容量を節 これは保つことができることを意味する ‘use\_uncompressed\_cache’ 常に1に設定します。 +少なくとも幾分大きな量のデータ（百万行以上）を読み取るクエリの場合、圧縮されていないキャッシュは自動的に無効になり、本当に小さなクエリの これは保つことができることを意味する ‘use\_uncompressed\_cache’ 設定は常に1に設定します。 ## replace\_running\_query {#replace-running-query} HTTPインターフェイスを使用する場合、 ‘query\_id’ 変数は渡すことができます。 これは、クエリ識別子として機能する任意の文字列です。 -同じユーザーからのクエリが同じ場合 ‘query\_id’ この時点で既に存在している場合、その動作は ‘replace\_running\_query’ パラメータ。 +同じユーザーからのクエリが同じ場合 ‘query\_id’ この時点で既に存在しているので、動作は ‘replace\_running\_query’ パラメータ。 -`0` (default) – Throw an exception (don’t allow the query to run if a query with the same ‘query\_id’ すでに実行されている）。 +`0` (default) – Throw an exception (don't allow the query to run if a query with the same ‘query\_id’ すでに実行されています）。 `1` – Cancel the old query and start running the new one. -Yandexの。Metricaこのパラメータセットが1の実施のための提案のための分割ます。 次の文字を入力した後、古いクエリがまだ完了していない場合は、キャンセルする必要があります。 +Yandex.Metricaこのパラメータセットが1の実施のための提案のための分割ます。 次の文字を入力した後、古いクエリがまだ終了していない場合は、キャンセルする必要があります。 ## stream\_flush\_interval\_ms {#stream-flush-interval-ms} 作品のテーブルストリーミングの場合はタイムアウトした場合、またはスレッドを生成す [max\_insert\_block\_size](#settings-max_insert_block_size) 行。 -デフォルト値は7500です。 +既定値は7500です。 -値が小さいほど、データはテーブルにフラッシュされる頻度が高くなります。 値が低すぎると、パフォーマンスが低下します。 +値が小さいほど、データがテーブルにフラッシュされる頻度が高くなります。 値を小さく設定すると、パフォーマンスが低下します。 ## load\_balancing {#settings-load_balancing} -分散クエリ処理に使用するレプリカ選択のアルゴリズムを指定します。 +分散クエリ処理に使用されるレプリカ選択のアルゴリズムを指定します。 ClickHouse対応し、以下のようなアルゴリズムの選択のレプリカ: @@ -698,8 +751,8 @@ ClickHouse対応し、以下のようなアルゴリズムの選択のレプリ load_balancing = random ``` -エラーの数は、各レプリカに対して数えられます。 クエリは、エラーが最も少なくレプリカに送信され、エラーがいくつかある場合は、そのうちの誰にも送信されます。 -レプリカに異なるデータがある場合は、異なるデータも取得します。 +エラーの数は、レプリカごとにカウントされます。 クエリは、エラーが最も少ないレプリカに送信されます。 +レプリカのデータが異なる場合は、異なるデータも取得されます。 ### 最寄りのホスト名 {#load_balancing-nearest_hostname} @@ -707,13 +760,13 @@ load_balancing = random load_balancing = nearest_hostname ``` -The number of errors is counted for each replica. Every 5 minutes, the number of errors is integrally divided by 2. Thus, the number of errors is calculated for a recent time with exponential smoothing. If there is one replica with a minimal number of errors (i.e. errors occurred recently on the other replicas), the query is sent to it. If there are multiple replicas with the same minimal number of errors, the query is sent to the replica with a hostname that is most similar to the server’s hostname in the config file (for the number of different characters in identical positions, up to the minimum length of both hostnames). +The number of errors is counted for each replica. Every 5 minutes, the number of errors is integrally divided by 2. Thus, the number of errors is calculated for a recent time with exponential smoothing. If there is one replica with a minimal number of errors (i.e. errors occurred recently on the other replicas), the query is sent to it. If there are multiple replicas with the same minimal number of errors, the query is sent to the replica with a hostname that is most similar to the server's hostname in the config file (for the number of different characters in identical positions, up to the minimum length of both hostnames). -たとえば、example01-01-1とexample01-01-2.yandex.ru example01-01-1とexample01-02-2は二つの場所で異なりますが、一つの位置では異なります。 -この方法はプリミティブに思えるかもしれませんが、ネットワークトポロジに関する外部データを必要とせず、ipv6アドレスでは複雑なipアドレスを +例えば、example01-01-1やexample01-01-2.yandex.ru 一つの場所で異なるが、example01-01-1とexample01-02-2は二つの場所で異なる。 +この方法は原始的に見えるかもしれませんが、ネットワークトポロジに関する外部データを必要とせず、Ipアドレスを比較することもありません。 -したがって、同等の複製がある場合は、名前によって最も近い複製が優先されます。 -また、同じサーバーにクエリを送信するときに、障害がなければ、分散クエリも同じサーバーに移動すると想定することもできます。 したがって、レプリカに異なるデータが配置されていても、クエリはほとんど同じ結果を返します。 +したがって、同等のレプリカがある場合は、名前で最も近いレプリカが優先されます。 +また、同じサーバーにクエリを送信するときに、障害がない場合、分散クエリも同じサーバーに移動すると仮定することもできます。 したがって、レプリカに異なるデータが配置されても、クエリはほとんど同じ結果を返します。 ### 順番に {#load_balancing-in_order} @@ -721,8 +774,8 @@ The number of errors is counted for each replica. Every 5 minutes, the number of load_balancing = in_order ``` -エラーの数が同じレプリカは、構成で指定されているのと同じ順序でアクセスされます。 -この方法は適切なが分かっている場合は、正確にレプリカが好ましい。 +同じ数のエラーを持つレプリカは、構成で指定されている順序と同じ順序でアクセスされます。 +この方法は、適切なレプリカを正確に把握している場合に適しています。 ### 最初またはランダム {#load_balancing-first_or_random} @@ -730,9 +783,9 @@ load_balancing = in_order load_balancing = first_or_random ``` -このアルゴリズムは、セット内の最初のレプリカを選択します。 この効果のクロス-複製をトポロジーのセットアップ、ものなどが可能です。- +このアルゴリズムは、セット内の最初のレプリカを選択します。 クロスレプリケーショントポロジの設定では有効ですが、他の構成では役に立ちません。 -その `first_or_random` アルゴリズムはの問題を解決します `in_order` アルゴリズムだ と `in_order` あるレプリカがダウンした場合、次のレプリカは二重の負荷を受け、残りのレプリカは通常のトラフィック量を処理します。 を使用する場合 `first_or_random` アルゴリズムは、負荷が均等にまだ利用可能なレプリカ間で分散されています。 +その `first_or_random` アルゴリズムはの問題を解決します `in_order` アルゴリズム と `in_order` あるレプリカがダウンした場合、残りのレプリカは通常のトラフィック量を処理しますが、次のレプリカは二重負荷を受けます。 を使用する場合 `first_or_random` アルゴリズムでは、負荷はまだ利用可能なレプリカ間で均等に分散されます。 ## prefer\_localhost\_replica {#settings-prefer-localhost-replica} @@ -750,39 +803,39 @@ load_balancing = first_or_random ## totals\_mode {#totals-mode} -HAVINGが存在する場合の合計を計算する方法と、max\_rows\_to\_group\_byとgroup\_by\_overflow\_mode= ‘any’ 存在する。 -セクションを見る “WITH TOTALS modifier”. +有するときの合計の計算方法、およびmax\_rows\_to\_group\_byおよびgroup\_by\_overflow\_mode= ‘any’ 存在する。 +セクションを参照 “WITH TOTALS modifier”. ## totals\_auto\_threshold {#totals-auto-threshold} のしきい値 `totals_mode = 'auto'`. -セクションを見る “WITH TOTALS modifier”. +セクションを参照 “WITH TOTALS modifier”. ## max\_parallel\_replicas {#settings-max_parallel_replicas} -クエリを実行するときの各シャードのレプリカの最大数。 -一貫性（同じデータ分割の異なる部分を取得する）の場合、このオプションはサンプリングキーが設定されている場合にのみ機能します。 -レプリカの遅延は制御されません。 +クエリ実行時の各シャードのレプリカの最大数。 +のための一貫性を異なる部分に同じデータを分割)、このオプションにしているときだけサンプリングキーを設定します。 +レプリカラグは制御されません。 ## コンパイル {#compile} -を編集ます。 デフォルトでは、0(無効)。 +を編集ます。 既定では、0(無効)です。 コンパイルは、クエリ処理パイプラインの一部にのみ使用されます。 -この部分のパイプラインのためのクエリを実行するアによる展開の短サイクルinlining集計機能。 複数の単純な集計関数を使用するクエリでは、パフォーマンスの最大向上(まれに、最大で四倍高速)が見られます。 通常、パフォーマンスの向上はわずかです。 まれに、クエリの実行が遅くなることがあります。 +この部分のパイプラインのためのクエリを実行するアによる展開の短サイクルinlining集計機能。 複数の単純な集計関数を使用するクエリでは、最大のパフォーマンスの向上が見られます。 通常、性能は軽微であります。 非常に珍しい例で遅くなクエリを実行します。 ## min\_count\_to\_compile {#min-count-to-compile} -コンパイルを実行する前にコンパイルされたコードのチャンクを使用する回数。 デフォルトでは、3。 +り方を潜在的に利用コチャンクのコードの実行前に作成する。 デフォルトでは3. For testing, the value can be set to 0: compilation runs synchronously and the query waits for the end of the compilation process before continuing execution. For all other cases, use values ​​starting with 1. Compilation normally takes about 5-10 seconds. -値が1以上の場合、コンパイルは別のスレッドで非同期に行われます。 結果は、現在実行中のクエリを含め、準備が整ったらすぐに使用されます。 +値が1以上の場合、コンパイルは別のスレッドで非同期に実行されます。 結果は、現在実行中のクエリを含め、準備が整うとすぐに使用されます。 -コンパイルされたコードは、クエリで使用される集計関数とgroup by句のキーのタイプの組み合わせごとに必要です。 -The results of the compilation are saved in the build directory in the form of .so files. There is no restriction on the number of compilation results since they don’t use very much space. Old results will be used after server restarts, except in the case of a server upgrade – in this case, the old results are deleted. +コンパイルされたコードは、クエリで使用される集計関数とGROUP BY句内のキーの種類のそれぞれの異なる組み合わせに必要です。 +The results of the compilation are saved in the build directory in the form of .so files. There is no restriction on the number of compilation results since they don't use very much space. Old results will be used after server restarts, except in the case of a server upgrade – in this case, the old results are deleted. ## output\_format\_json\_quote\_64bit\_integers {#session_settings-output_format_json_quote_64bit_integers} -値がtrueの場合、json\*int64およびuint64形式を使用するときに整数が引用符で囲まれます（ほとんどのjavascript実装との互換性のため）。 +値がtrueの場合、json\*Int64およびUInt64形式（ほとんどのJavaScript実装との互換性のため）を使用するときに整数が引用符で表示されます。 ## format\_csv\_delimiter {#settings-format_csv_delimiter} @@ -790,15 +843,15 @@ CSVデータの区切り文字として解釈される文字。 デフォルト ## input\_format\_csv\_unquoted\_null\_literal\_as\_null {#settings-input_format_csv_unquoted_null_literal_as_null} -FOR CSV入力形式では、引用符なしの解析を有効または無効にします `NULL` リテラルとして(シノニム `\N`). +CSV入力形式の場合、引用符なしの解析を有効または無効にします `NULL` リテラルとして(のシノニム `\N`). ## output\_format\_csv\_crlf\_end\_of\_line {#settings-output-format-csv-crlf-end-of-line} -UNIXスタイル（LF）の代わりにCSVでDOS/Windowsスタイルの行区切り（CRLF）を使用します。 +CSVでは、UNIXスタイル（LF）の代わりにDOS/Windowsスタイルの行区切り記号（CRLF）を使用します。 ## output\_format\_tsv\_crlf\_end\_of\_line {#settings-output-format-tsv-crlf-end-of-line} -UNIXスタイル（LF）の代わりにTSVでDOC/Windowsスタイルの行区切り（CRLF）を使用します。 +TSVでは、UNIXスタイル(LF)の代わりにDOC/Windowsスタイルの行区切り記号(CRLF)を使用します。 ## insert\_quorum {#settings-insert_quorum} @@ -807,60 +860,60 @@ UNIXスタイル（LF）の代わりにTSVでDOC/Windowsスタイルの行区切 - もし `insert_quorum < 2` クォーラム書き込みは無効です。 - もし `insert_quorum >= 2` クォーラム書き込みが有効になります。 -デフォルト値:0. +デフォルト値は0です。 定足数書き込み -`INSERT` 承ClickHouse管理を正しく書き込みデータの `insert_quorum` の間のレプリカの `insert_quorum_timeout`. 何らかの理由で書き込みが成功したレプリカの数に達しない場合 `insert_quorum` を書くのは失敗したとClickHouseを削除するに挿入したブロックからすべてのレプリカがデータがすでに記されています。 +`INSERT` 承ClickHouse管理を正しく書き込みデータの `insert_quorum` の間のレプリカの `insert_quorum_timeout`. 何らかの理由で書き込みが成功したレプリカの数が `insert_quorum` を書くのは失敗したとClickHouseを削除するに挿入したブロックからすべてのレプリカがデータがすでに記されています。 クォーラム内のすべてのレプリカは一貫性があります。 `INSERT` クエリ。 その `INSERT` シーケンスは線形化されます。 -から書き込まれたデータを読み取る場合 `insert_quorum`、を使用することができ [select\_sequential\_consistency](#settings-select_sequential_consistency) オプション。 +から書き込まれたデータを読み取ると `insert_quorum` を使用することができます [select\_sequential\_consistency](#settings-select_sequential_consistency) オプション ClickHouseは例外を生成します -- クエリの時点で利用可能なレプリカの数がクエリの時点でのレプリカの数より少ない場合 `insert_quorum`. -- 前のブロックがまだ挿入されていないときにデータを書き込もうとします。 `insert_quorum` レプリカの。 こうした状況が発生した場合、ユーザーしようとしを行う `INSERT` 前のものの前に `insert_quorum` 完了です。 +- クエリの時点で使用可能なレプリカの数が `insert_quorum`. +- 前のブロックがまだ挿入されていないときにデータを書き込もうとすると `insert_quorum` レプリカの。 この状況は、ユーザーが `INSERT` 前のものの前に `insert_quorum` 完了です。 -また見なさい: +も参照。: - [insert\_quorum\_timeout](#settings-insert_quorum_timeout) - [select\_sequential\_consistency](#settings-select_sequential_consistency) ## insert\_quorum\_timeout {#settings-insert_quorum_timeout} -書き込み数が定員タイムアウトを秒で指定します。 タイムアウトが経過し、まだ書き込みが行われていない場合、clickhouseは例外を生成し、クライアントは同じブロックまたは他のレプリカに同じブロック +書き込み数が定員タイムアウトを秒で指定します。 タイムアウトが経過し、まだ書き込みが行われていない場合、ClickHouseは例外を生成し、クライアントは同じまたは他のレプリカに同じブロックを書き込む -デフォルト値:60秒。 +デフォルト値は60秒です。 -また見なさい: +も参照。: - [insert\_quorum](#settings-insert_quorum) - [select\_sequential\_consistency](#settings-select_sequential_consistency) ## select\_sequential\_consistency {#settings-select_sequential_consistency} -シーケンシャル-コンシステ `SELECT` クエリ: +の順次整合性を有効または無効にします `SELECT` クエリ: 可能な値: - 0 — Disabled. - 1 — Enabled. -デフォルト値:0. +デフォルト値は0です。 -使い方 +使用法 -シーケンシャル一貫性が有効になっている場合、clickhouseはクライアントが `SELECT` 以前のすべてのデータを含むレプリカに対してのみ照会する `INSERT` 以下で実行されるクエリ `insert_quorum`. クライアントが部分レプリカを参照している場合、ClickHouseは例外を生成します。 SELECTクエリには、レプリカのクォーラムにまだ書き込まれていないデータは含まれません。 +順次整合性が有効になっている場合、ClickHouseはクライアントが `SELECT` 以前のすべてのデータを含むレプリカのみを照会します `INSERT` 以下で実行されるクエリ `insert_quorum`. クライアントが部分レプリカを参照している場合、ClickHouseは例外を生成します。 SELECTクエリには、レプリカのクォーラムにまだ書き込まれていないデータは含まれません。 -また見なさい: +も参照。: - [insert\_quorum](#settings-insert_quorum) - [insert\_quorum\_timeout](#settings-insert_quorum_timeout) ## insert\_deduplicate {#settings-insert-deduplicate} -ブロックの重複排除を有効または無効にします。 `INSERT` (複製された\*テーブルの場合)。 +重複除外のブロックを有効または無効にします。 `INSERT` (複製された\*テーブルの場合)。 可能な値: @@ -869,9 +922,9 @@ ClickHouseは例外を生成します デフォルト値:1。 -デフォルトでは、レプリケートされたテーブルに `INSERT` ステートメントは重複排除されます(\[データレプリケーション\](../engines/table\_engines/mergetree\_family/replication.md)を参照)。 +デフォルトでは、ブロックは `INSERT` ステートメントは重複排除されます [データ複製](../../engines/table-engines/mergetree-family/replication.md)). -## 重複除外\_blocks\_in\_dependent\_materialized\_views {#settings-deduplicate-blocks-in-dependent-materialized-views} +## deduplicate\_blocks\_in\_dependent\_materialized\_views {#settings-deduplicate-blocks-in-dependent-materialized-views} を有効または無効にし、重複排除圧縮をチェックを実現し意見を受け取るデータから複製\*ます。 @@ -880,70 +933,70 @@ ClickHouseは例外を生成します 0 — Disabled. 1 — Enabled. -デフォルト値:0. +デフォルト値は0です。 -使い方 +使用法 -デフォルトで、重複排除圧縮を行わないための顕在化が行われは上流のソース。 -ソーステーブルの重複排除により挿入されたブロックがスキップされた場合、マテリアライズドビューには挿入されません。 この動作は、マテリアライズドビューに高度に集計されたデータを挿入できるようにするために存在します。 -同時に、この動作 “breaks” `INSERT` 冪等性 もし `INSERT` メインテーブルに成功したと `INSERT` into a materialized view failed (e.g. because of communication failure with Zookeeper) a client will get an error and can retry the operation. However, the materialized view won’t receive the second insert because it will be discarded by deduplication in the main (source) table. The setting `deduplicate_blocks_in_dependent_materialized_views` この動作を変更できます。 再試行の際、マテリアライズドビューは繰り返しインサートを受け取り、重複排除チェックを単独で実行します, -ソーステーブルのチェック結果を無視すると、最初の失敗のために失われた行が挿入されます。 +デフォルトでは、重複除外はマテリアライズドビューでは実行されませんが、ソーステーブルの上流で実行されます。 +ソーステーブルの重複除外のために挿入されたブロックがスキップされた場合、添付されたマテリアライズドビューには挿入されません。 この動作は、マテリアライズドビューの集計後に挿入されたブロックが同じで、ソーステーブルへの異なる挿入から派生した場合に、高度に集計されたデータ +同時に、この動作 “breaks” `INSERT` べき等性。 もし `INSERT` メインテーブルに成功したと `INSERT` into a materialized view failed (e.g. because of communication failure with Zookeeper) a client will get an error and can retry the operation. However, the materialized view won't receive the second insert because it will be discarded by deduplication in the main (source) table. The setting `deduplicate_blocks_in_dependent_materialized_views` この動作を変更できます。 再試行すると、マテリアライズドビューは繰り返し挿入を受け取り、重複除外チェックを単独で実行します, +ソーステーブルのチェック結果を無視し、最初の失敗のために失われた行を挿入します。 ## max\_network\_bytes {#settings-max-network-bytes} -クエリの実行時にネットワークを介して受信または送信されるデータ量をバイト単位で制限します。 この設定は、個々のクエリごとに適用されます。 +クエリの実行時にネットワーク経由で受信または送信されるデータ量(バイト単位)を制限します。 この設定は、個々のクエリごとに適用されます。 可能な値: - 正の整数。 - 0 — Data volume control is disabled. -デフォルト値:0. +デフォルト値は0です。 ## max\_network\_bandwidth {#settings-max-network-bandwidth} -ネットワーク上でのデータ交換の速度をバイト/秒で制限します。 この設定は、各クエリに適用されます。 +ネットワーク上でのデータ交換の速度を毎秒バイト単位で制限します。 この設定はすべての照会に適用されます。 可能な値: - 正の整数。 - 0 — Bandwidth control is disabled. -デフォルト値:0. +デフォルト値は0です。 ## max\_network\_bandwidth\_for\_user {#settings-max-network-bandwidth-for-user} -ネットワーク上でのデータ交換の速度をバイト/秒で制限します。 この設定は、単一ユーザーが実行するすべての同時実行クエリに適用されます。 +ネットワーク上でのデータ交換の速度を毎秒バイト単位で制限します。 この設定は、単一ユーザーが同時に実行するすべてのクエリに適用されます。 可能な値: - 正の整数。 - 0 — Control of the data speed is disabled. -デフォルト値:0. +デフォルト値は0です。 ## max\_network\_bandwidth\_for\_all\_users {#settings-max-network-bandwidth-for-all-users} -ネットワーク上でデータが交換される速度をバイト/秒で制限します。 この設定が適用されるのはすべての同時走行に関するお問い合わせます。 +ネットワーク経由でデータが交換される速度を毎秒バイト単位で制限します。 この設定が適用されるのはすべての同時走行に関するお問い合わせます。 可能な値: - 正の整数。 - 0 — Control of the data speed is disabled. -デフォルト値:0. +デフォルト値は0です。 ## count\_distinct\_implementation {#settings-count_distinct_implementation} -どちらを指定するか `uniq*` 機能は実行するのに使用されるべきです [COUNT(DISTINCT …)](../../sql-reference/aggregate-functions/reference.md#agg_function-count) 建設。 +これは、 `uniq*` を実行するために使用する必要があります。 [COUNT(DISTINCT …)](../../sql-reference/aggregate-functions/reference.md#agg_function-count) 建設。 可能な値: - [uniq](../../sql-reference/aggregate-functions/reference.md#agg_function-uniq) - [uniqCombined](../../sql-reference/aggregate-functions/reference.md#agg_function-uniqcombined) - [uniqCombined64](../../sql-reference/aggregate-functions/reference.md#agg_function-uniqcombined64) -- [unihll12](../../sql-reference/aggregate-functions/reference.md#agg_function-uniqhll12) -- [ユニキャック](../../sql-reference/aggregate-functions/reference.md#agg_function-uniqexact) +- [uniqHLL12](../../sql-reference/aggregate-functions/reference.md#agg_function-uniqhll12) +- [uniqExact](../../sql-reference/aggregate-functions/reference.md#agg_function-uniqexact) デフォルト値: `uniqExact`. @@ -955,31 +1008,31 @@ ClickHouseは例外を生成します - ClickHouseは何らかの理由でレプリカに接続できません。 - レプリカに接続するとき、clickhouseはいくつかの試行を実行します。 すべてこれらの試みが失敗し、レプリカとはできます。 + レプリカに接続すると、ClickHouseはいくつかの試行を実行します。 すべてこれらの試みが失敗し、レプリカとはできます。 -- レプリカはdnsで解決できません。 +- レプリカはDNSで解決できません。 - レプリカのホスト名をdnsで解決できない場合は、次のような状況を示すことができます: + レプリカのホスト名をDNS経由で解決できない場合は、次の状況を示すことがあります: - - レプリカのホストにdnsレコードがない。 たとえば、動的dnsを使用するシステムで発生する可能性があります, [Kubernetes](https://kubernetes.io) ダウンタイム中にノードが解けなくなる可能性があり、これはエラーではありません。 + - レプリカのホストにDNSレコードがない。 これは、動的DNSを持つシステムで発生する可能性があります。, [Kubernetes](https://kubernetes.io) ここで、ノードはダウンタイム中に解決できず、これはエラーではありません。 - - 構成エラー。 clickhouse設定ファイルが含まれて間違ったホスト名. + - 設定エラー。 ClickHouse設定ファイルが含まれて間違ったホスト名. 可能な値: - 1 — skipping enabled. - シャードが使用できない場合、clickhouseは部分的なデータに基づいて結果を返し、ノードの可用性の問題は報告しません。 + シャードが使用できない場合、ClickHouseは部分的なデータに基づいて結果を返し、ノードの可用性の問題は報告しません。 - 0 — skipping disabled. - シャードが使用できない場合、clickhouseは例外をスローします。 + シャードが使用できない場合、ClickHouseは例外をスローします。 -デフォルト値:0. +デフォルト値は0です。 ## optimize\_skip\_unused\_shards {#settings-optimize_skip_unused_shards} -PREWHERE/WHEREでシャーディングキー条件を持つSELECTクエリの未使用シャードのスキップを有効または無効にします(データがシャーディングキーによって分散されてい +PREWHERE/WHEREにシャーディングキー条件があるSELECTクエリの未使用のシャードのスキップを有効または無効にします(データがシャーディングキーによって配布される デフォルト値:0 @@ -990,133 +1043,133 @@ PREWHERE/WHEREでシャーディングキー条件を持つSELECTクエリの未 可能な値: - 0-無効(スローしない) -- 1-テーブ +- 1-除クエリの実行の場合のみ表はshardingキー - 2-を無効にクエリの実行に関わらずshardingキー定義のテーブル デフォルト値:0 ## force\_optimize\_skip\_unused\_shards\_no\_nested {#settings-force_optimize_skip_unused_shards_no_nested} -リセット [`optimize_skip_unused_shards`](#settings-force_optimize_skip_unused_shards) 入れ子のため `Distributed` テーブル +リセット [`optimize_skip_unused_shards`](#settings-force_optimize_skip_unused_shards) 入れ子の場合 `Distributed` テーブル 可能な値: - 1 — Enabled. - 0 — Disabled. -デフォルト値:0. +デフォルト値は0です。 ## optimize\_throw\_if\_noop {#setting-optimize_throw_if_noop} -例外のスローを有効または無効にします。 [OPTIMIZE](../../sql-reference/statements/misc.md#misc_operations-optimize) クエリはマージを実行しませんでした。 +例外のスローを有効または無効にします。 [OPTIMIZE](../../sql-reference/statements/misc.md#misc_operations-optimize) クエリがマージを実行しませんでした。 -デフォルトでは, `OPTIMIZE` 何もしなかった場合でも正常に戻ります。 この設定では、これらの状況を区別し、例外メッセージの理由を取得できます。 +既定では, `OPTIMIZE` 何もしなくても正常に戻ります。 この設定を使用すると、これらの状況を区別し、例外メッセージで理由を取得できます。 可能な値: - 1 — Throwing an exception is enabled. - 0 — Throwing an exception is disabled. -デフォルト値:0. +デフォルト値は0です。 -## distributed\_replica\_error\_half\_life {#settings-distributed_replica_error_half_life} +## ディストリビューター {#settings-distributed_replica_error_half_life} - タイプ:秒 - デフォルト値:60秒 -分散テーブルのエラーをゼロにする速度を制御します。 レプリカがしばらく使用できず、5つのエラーが蓄積され、distributed\_replica\_error\_half\_lifeが1秒に設定されている場合、レプリカは最後のエラーから3秒後に通常の状態 +分散テーブルのエラーをゼロにする速度を制御します。 レプリカがしばらく使用できず、5つのエラーが蓄積され、distributed\_replica\_error\_half\_lifeが1秒に設定されている場合、レプリカは最後のエラーの3秒後に正常と見なされま -また見なさい: +も参照。: - [分散テーブルエンジン](../../engines/table-engines/special/distributed.md) -- [distributed\_replica\_error\_cap](#settings-distributed_replica_error_cap) +- [ディストリビューター](#settings-distributed_replica_error_cap) -## distributed\_replica\_error\_cap {#settings-distributed_replica_error_cap} +## ディストリビューター {#settings-distributed_replica_error_cap} -- タイプ:unsigned int +- 型:unsigned int - デフォルト値:1000 -各レプリカのエラー数がこの値に制限されるため、単一のレプリカによるエラーの蓄積が妨げられます。 +各レプリカのエラ -また見なさい: +も参照。: - [分散テーブルエンジン](../../engines/table-engines/special/distributed.md) -- [distributed\_replica\_error\_half\_life](#settings-distributed_replica_error_half_life) +- [ディストリビューター](#settings-distributed_replica_error_half_life) ## distributed\_directory\_monitor\_sleep\_time\_ms {#distributed_directory_monitor_sleep_time_ms} -のための基礎間隔 [分散](../../engines/table-engines/special/distributed.md) データを送信する表エンジン。 実際の間隔は、エラーが発生した場合に指数関数的に増加します。 +の基本区間 [分散](../../engines/table-engines/special/distributed.md) データを送信する表エンジン。 実際の間隔は、エラーが発生した場合に指数関数的に増加します。 可能な値: -- ミリ秒の正の整数の数。 +- ミリ秒の正の整数。 -デフォルト値:100ミリ秒。 +既定値は100ミリ秒です。 ## distributed\_directory\_monitor\_max\_sleep\_time\_ms {#distributed_directory_monitor_max_sleep_time_ms} -のための最大間隔 [分散](../../engines/table-engines/special/distributed.md) データを送信する表エンジン。 インターバルセットの指数関数的な成長を制限する [distributed\_directory\_monitor\_sleep\_time\_ms](#distributed_directory_monitor_sleep_time_ms) 設定。 +の最大間隔 [分散](../../engines/table-engines/special/distributed.md) データを送信する表エンジン。 の区間の指数関数的成長を制限する。 [distributed\_directory\_monitor\_sleep\_time\_ms](#distributed_directory_monitor_sleep_time_ms) 設定。 可能な値: -- ミリ秒の正の整数の数。 +- ミリ秒の正の整数。 デフォルト値:30000ミリ秒(30秒)。 ## distributed\_directory\_monitor\_batch\_inserts {#distributed_directory_monitor_batch_inserts} -挿入されたデータのバッチでの送信を有効/無効にします。 +挿入されたデータのバッチ送信を有効または無効にします。 -バッチ送信が有効になっている場合は、 [分散](../../engines/table-engines/special/distributed.md) テーブルエンジンをお送り複数のファイルの挿入データを移動するようになっていますの代わりに送信します。 一括送信の改善にクラスターの性能をより活用してサーバやネットワーク資源です。 +バッチ送信が有効になっている場合、 [分散](../../engines/table-engines/special/distributed.md) テーブルエンジンをお送り複数のファイルの挿入データを移動するようになっていますの代わりに送信します。 一括送信の改善にクラスターの性能をより活用してサーバやネットワーク資源です。 可能な値: - 1 — Enabled. - 0 — Disabled. -デフォルト値:0. +デフォルト値は0です。 ## os\_thread\_priority {#setting-os-thread-priority} -優先度を設定します ([ニース](https://en.wikipedia.org/wiki/Nice_(Unix)))クエリを実行するスレッドの場合。 OSスケジューラは、使用可能な各CPUコアで実行する次のスレッドを選択するときにこの優先度を考慮します。 +優先度を設定します ([ニース](https://en.wikipedia.org/wiki/Nice_(Unix)))クエリを実行するスレッドの場合。 OSスケジューラは、使用可能な各CPUコアで実行する次のスレッドを選択する際に、この優先順位を考慮します。 !!! warning "警告" - この設定を使用するには、以下を設定する必要があります。 `CAP_SYS_NICE` 機能。 その `clickhouse-server` パッケージ設定します。 一部の仮想環境では、以下の設定を行うことができません。 `CAP_SYS_NICE` 機能。 この場合, `clickhouse-server` 開始時にそれについてのメッセージを表示します。 + この設定を使用するには、 `CAP_SYS_NICE` 能力。 その `clickhouse-server` パッケ いくつかの仮想環境では、 `CAP_SYS_NICE` 能力。 この場合, `clickhouse-server` 開始時にそれに関するメッセージを表示します。 可能な値: -- 範囲内で値を設定できます `[-20, 19]`. +- 範囲の値を設定できます `[-20, 19]`. -低値が高い優先されます。 低いの糸 `nice` 優先度の値は、高い値を持つスレッドよりも頻繁に実行されます。 長時間実行される非インタラクティブクエリでは、短いインタラクティブクエリが到着したときにリソースをすばやく放棄することができるため、 +値が低いほど優先度が高くなります。 低いの糸 `nice` 優先度の値は、高い値のスレッドよりも頻繁に実行されます。 長時間実行される非対話型クエリでは、短い対話型クエリが到着したときにリソースをすばやく放棄できるため、高い値が望ましいです。 -デフォルト値:0. +デフォルト値は0です。 ## query\_profiler\_real\_time\_period\_ns {#query_profiler_real_time_period_ns} -の実際のクロックタイマーの期間を設定します。 [クエリプロファイラ](../../operations/optimizing-performance/sampling-query-profiler.md). リアルクロックタイマーカウント壁時計の時間。 +の実クロックタイマーの期間を設定します。 [クエリプロファイラ](../../operations/optimizing-performance/sampling-query-profiler.md). リアルクロックタイマーカウント壁掛時計。 可能な値: -- ナノ秒単位の正の整数。 +- ナノ秒単位の正の整数です。 推奨値: - 10000000 (100 times a second) nanoseconds and less for single queries. - 1000000000 (once a second) for cluster-wide profiling. -- タイマーを消すための0。 +- タイマーをオフにする場合は0。 タイプ: [UInt64](../../sql-reference/data-types/int-uint.md). デフォルト値:1000000000ナノ秒(秒)。 -また見なさい: +も参照。: - システム表 [trace\_log](../../operations/system-tables.md#system_tables-trace_log) ## query\_profiler\_cpu\_time\_period\_ns {#query_profiler_cpu_time_period_ns} -のcpuクロックタイマーの期間を設定します。 [クエリプロファイラ](../../operations/optimizing-performance/sampling-query-profiler.md). このタ +のCPUクロックタイマーの期間を設定します。 [クエリプロファイラ](../../operations/optimizing-performance/sampling-query-profiler.md). このタイマーカウントのみのCPU時間。 可能な値: @@ -1127,49 +1180,49 @@ PREWHERE/WHEREでシャーディングキー条件を持つSELECTクエリの未 - 10000000 (100 times a second) nanoseconds and more for single queries. - 1000000000 (once a second) for cluster-wide profiling. -- タイマーを消すための0。 +- タイマーをオフにする場合は0。 タイプ: [UInt64](../../sql-reference/data-types/int-uint.md). デフォルト値:1000000000ナノ秒。 -また見なさい: +も参照。: - システム表 [trace\_log](../../operations/system-tables.md#system_tables-trace_log) ## allow\_introspection\_functions {#settings-allow_introspection_functions} -ディスエーブルの有効 [introspections関数](../../sql-reference/functions/introspection.md) のためのクエリープロファイリング. +ディスエーブルの有効 [イントロスペクション関数](../../sql-reference/functions/introspection.md) クエリプロファイル用。 可能な値: - 1 — Introspection functions enabled. - 0 — Introspection functions disabled. -デフォルト値:0. +デフォルト値は0です。 -**また見なさい** +**も参照。** -- [クエリプ](../optimizing-performance/sampling-query-profiler.md) +- [サンプリングクロファイラ](../optimizing-performance/sampling-query-profiler.md) - システム表 [trace\_log](../../operations/system-tables.md#system_tables-trace_log) -## input\_format\_parallel\_parallel\_paralsing {#input-format-parallel-parsing} +## input\_format\_parallel\_parsing {#input-format-parallel-parsing} - タイプ:bool -- デフォルト値:true +- 既定値:True -データ形式の並行解析を有効にします。 tsv、tksv、csvおよびjsoneachrow形式でのみサポートされています。 +データ形式の順序保持並列解析を有効にします。 TSV、TKSV、CSVおよびJSONEachRow形式でのみサポートされます。 -## min\_chunk\_bytes\_for\_parallel\_parall\_paral {#min-chunk-bytes-for-parallel-parsing} +## min\_chunk\_bytes\_for\_parallel\_parsing {#min-chunk-bytes-for-parallel-parsing} -- タイプ:unsigned int -- デフォルト値:1mib +- 型:unsigned int +- デフォルト値:1MiB -各スレッドが並列に解析する最小チャンクサイズ(バイト単位)。 +各スレッドが並列に解析する最小チャンクサイズをバイト単位で表します。 ## output\_format\_avro\_codec {#settings-output_format_avro_codec} -出力avroファイルに使用する圧縮コーデックを設定します。 +出力Avroファイルに使用する圧縮コーデックを設定します。 タイプ:文字列 @@ -1177,26 +1230,36 @@ PREWHERE/WHEREでシャーディングキー条件を持つSELECTクエリの未 - `null` — No compression - `deflate` — Compress with Deflate (zlib) -- `snappy` — Compress with [てきぱき](https://google.github.io/snappy/) +- `snappy` — Compress with [スナッピー](https://google.github.io/snappy/) -デフォルト値: `snappy` (利用可能な場合)または `deflate`. +デフォルト値: `snappy` （利用可能な場合）または `deflate`. ## output\_format\_avro\_sync\_interval {#settings-output_format_avro_sync_interval} -出力avroファイルの同期マーカー間の最小データサイズ(バイト単位)を設定します。 +出力Avroファイルの同期マーカー間の最小データサイズ(バイト単位)を設定します。 -タイプ:unsigned int +型:unsigned int -使用可能な値:32(32バイト)-1073741824(1gib) +使用可能な値:32(32バイト)-1073741824(1GiB) -デフォルト値:32768(32kib) +デフォルト値:32768(32KiB) ## format\_avro\_schema\_registry\_url {#settings-format_avro_schema_registry_url} -使用するスキーマレジストリurlを設定します [AvroConfluent](../../interfaces/formats.md#data-format-avro-confluent) 書式 +使用するConfluentスキーマレジストリURLを設定します [アブロコンフルエント](../../interfaces/formats.md#data-format-avro-confluent) 形式 -タイプ:url +タイプ:URL -デフォルト値:空 +既定値:空 + +## background\_pool\_size {#background_pool_size} + +セットのスレッド数を行う背景事業のテーブルエンジン（例えば、合併に [MergeTreeエンジン](../../engines/table-engines/mergetree-family/index.md) テーブル）。 この設定はClickHouseサーバーの起動時に適用され、ユーザーセッションでは変更できません。 この設定を調整することで、CPUとディスクの負荷を管理します。 小さなプールサイズを以下のCPUやディスクの資源が背景のプロセスの事前の遅れが影響をクエリす。 + +可能な値: + +- 任意の正の整数。 + +デフォルト値は16です。 [元の記事](https://clickhouse.tech/docs/en/operations/settings/settings/) diff --git a/docs/ja/operations/system-tables.md b/docs/ja/operations/system-tables.md index b42292cbed5..6e073f93cce 100644 --- a/docs/ja/operations/system-tables.md +++ b/docs/ja/operations/system-tables.md @@ -1,28 +1,28 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 52 toc_title: "\u30B7\u30B9\u30C6\u30E0\u8868" --- # システム表 {#system-tables} -システムテーブルは、システムの機能の一部を実装し、システムの動作に関する情報へのアクセスを提供するために使用されます。 -システムテーブルは削除できません（ただし、デタッチを実行できます）。 -システムテーブ サーバーは起動時にすべてのシステムテーブルを作成します。 +システムテーブルは、システムの機能の一部を実装したり、システムの動作に関する情報へのアクセスを提供するために使用されます。 +システムテーブルを削除することはできません（ただし、DETACHを実行できます）。 +システムテーブルのないファイルデータのハードディスクまたはファイルとメタデータを指すものとします。 サーバーは起動時にすべてのシステムテーブルを作成します。 システムテーブルは読み取り専用です。 -彼らはに位置しています ‘system’ データベース +彼らはに位置しています ‘system’ データベース。 -## システム。asynchronous\_metrics {#system_tables-asynchronous_metrics} +## システムasynchronous\_metrics {#system_tables-asynchronous_metrics} -バックグラウンドで定期的に計算される指標が含まれます。 例えば、使用中のramの量。 +バックグラウンドで定期的に計算される指標が含まれます。 例えば、使用中のRAMの量。 列: - `metric` ([文字列](../sql-reference/data-types/string.md)) — Metric name. - `value` ([Float64](../sql-reference/data-types/float.md)) — Metric value. -**例えば** +**例** ``` sql SELECT * FROM system.asynchronous_metrics LIMIT 10 @@ -43,14 +43,14 @@ SELECT * FROM system.asynchronous_metrics LIMIT 10 └─────────────────────────────────────────┴────────────┘ ``` -**また見なさい** +**も参照。** - [監視](monitoring.md) — Base concepts of ClickHouse monitoring. -- [システム。指標](#system_tables-metrics) — Contains instantly calculated metrics. -- [システム。イベント](#system_tables-events) — Contains a number of events that have occurred. -- [システム。metric\_log](#system_tables-metric_log) — Contains a history of metrics values from tables `system.metrics` и `system.events`. +- [システムメトリック](#system_tables-metrics) — Contains instantly calculated metrics. +- [システムイベント](#system_tables-events) — Contains a number of events that have occurred. +- [システムmetric\_log](#system_tables-metric_log) — Contains a history of metrics values from tables `system.metrics` и `system.events`. -## システム。クラスター {#system-clusters} +## システムクラスタ {#system-clusters} についての情報が含まれてクラスターのコンフィグファイルをサーバーです。 @@ -65,21 +65,21 @@ SELECT * FROM system.asynchronous_metrics LIMIT 10 - `port` (UInt16) — The port to use for connecting to the server. - `user` (String) — The name of the user for connecting to the server. - `errors_count` (UInt32)-このホストがレプリカに到達できなかった回数。 -- `estimated_recovery_time` (UInt32)-レプリカエラーカウントがゼロになるまで残された秒数で、正常に戻ったと見なされます。 +- `estimated_recovery_time` (UInt32)-レプリカエラーカウントがゼロになり、正常に戻るまでの秒数。 -ご注意ください `errors_count` クエリごとにクラスターに一度updatedされますが、 `estimated_recovery_time` オンデマンドで再計算されます。 したがって、ゼロ以外の場合があります `errors_count` とゼロ `estimated_recovery_time`、その次のクエリはゼロ `errors_count` また、エラーがないかのようにreplicaを使用してみてください。 +ご注意ください `errors_count` クラスタに対するクエリごとに一度updatedされますが `estimated_recovery_time` オンデマンドで再計算されます。 だから、ゼロ以外の場合があるかもしれません `errors_count` とゼロ `estimated_recovery_time` 次のクエリはゼロになります `errors_count` エラーがないかのようにreplicaを使用してみてください。 -**また見なさい** +**も参照。** - [分散テーブルエンジン](../engines/table-engines/special/distributed.md) - [distributed\_replica\_error\_cap設定](settings/settings.md#settings-distributed_replica_error_cap) - [distributed\_replica\_error\_half\_life設定](settings/settings.md#settings-distributed_replica_error_half_life) -## システム。列 {#system-columns} +## システム列 {#system-columns} -すべてのテーブルの列に関する情報を含みます。 +すべてのテーブルの列に関する情報を格納します。 -このテーブルを使用して、次のような情報を取得できます。 [DESCRIBE TABLE](../sql-reference/statements/misc.md#misc-describe-table) 一度に複数のテーブルのクエリ。 +このテーブルを使用すると、次のような情報を取得できます [DESCRIBE TABLE](../sql-reference/statements/misc.md#misc-describe-table) クエリが、一度に複数のテーブルのために。 その `system.columns` テーブルを含む以下のカラムのカラムタイプはブラケット): @@ -87,7 +87,7 @@ SELECT * FROM system.asynchronous_metrics LIMIT 10 - `table` (String) — Table name. - `name` (String) — Column name. - `type` (String) — Column type. -- `default_kind` (String) — Expression type (`DEFAULT`, `MATERIALIZED`, `ALIAS`)デフォルト値の場合、または空の文字列が定義されていない場合。 +- `default_kind` (String) — Expression type (`DEFAULT`, `MATERIALIZED`, `ALIAS` デフォルト値の場合は)、定義されていない場合は空の文字列。 - `default_expression` (String) — Expression for the default value, or an empty string if it is not defined. - `data_compressed_bytes` (UInt64) — The size of compressed data, in bytes. - `data_uncompressed_bytes` (UInt64) — The size of decompressed data, in bytes. @@ -98,15 +98,15 @@ SELECT * FROM system.asynchronous_metrics LIMIT 10 - `is_in_primary_key` (UInt8) — Flag that indicates whether the column is in the primary key expression. - `is_in_sampling_key` (UInt8) — Flag that indicates whether the column is in the sampling key expression. -## システム。貢献者 {#system-contributors} +## システム貢献者 {#system-contributors} -を含むに関する情報提供者が保持しています。 ランダムな順序ですべてのconstributors。 順序は、クエリの実行時にランダムです。 +を含むに関する情報提供者が保持しています。 ランダムな順序ですべてのconstributors。 順序は、クエリ実行時にランダムです。 列: - `name` (String) — Contributor (author) name from git log. -**例えば** +**例** ``` sql SELECT * FROM system.contributors LIMIT 10 @@ -127,7 +127,7 @@ SELECT * FROM system.contributors LIMIT 10 └──────────────────┘ ``` -テーブルで自分自身を見つけるには、クエリを使用します: +テーブル内で自分自身を知るには、クエリを使用します: ``` sql SELECT * FROM system.contributors WHERE name='Olga Khvostikova' @@ -139,41 +139,81 @@ SELECT * FROM system.contributors WHERE name='Olga Khvostikova' └──────────────────┘ ``` -## システム。データ {#system-databases} +## システムデータ {#system-databases} このテーブルを含む単一の文字列カラムと呼ばれ ‘name’ – the name of a database. 各データベースのサーバーについて知っていて対応するエントリの表に示す。 -このシステム表は実行のために使用されます `SHOW DATABASES` クエリ。 +このシステムテーブルは、 `SHOW DATABASES` クエリ。 -## システム。detached\_parts {#system_tables-detached_parts} +## システムdetached\_parts {#system_tables-detached_parts} -についての情報が含まれて外部 [MergeTree](../engines/table-engines/mergetree-family/mergetree.md) テーブル。 その `reason` カラムを指定理由の一部でした。 ユーザーがデタッチしたパーツの場合、理由は空です。 このような部品は [ALTER TABLE ATTACH PARTITION\|PART](../query_language/query_language/alter/#alter_attach-partition) 司令部 の内容その他のカラムを参照 [システム。パーツ](#system_tables-parts). パート名が無効な場合、一部の列の値は次のようになります `NULL`. このような部分は、 [ALTER TABLE DROP DETACHED PART](../query_language/query_language/alter/#alter_drop-detached). +についての情報が含まれて外部 [メルゲツリー](../engines/table-engines/mergetree-family/mergetree.md) テーブル その `reason` column部品が切り離された理由を指定します。 ユーザーが取り外した部品の場合、その理由は空です。 このような部品は、 [ALTER TABLE ATTACH PARTITION\|PART](../sql-reference/statements/alter.md#alter_attach-partition) コマンド その他の列の説明については、 [システム部品](#system_tables-parts). パーツ名が無効な場合、一部のカラムの値は次のようになります `NULL`. このような部分は、以下で削除できます [ALTER TABLE DROP DETACHED PART](../sql-reference/statements/alter.md#alter_drop-detached). -## システム。辞書 {#system-dictionaries} +## システム辞書 {#system_tables-dictionaries} -外部辞書に関する情報が含まれます。 +についての情報が含まれて [外部辞書](../sql-reference/dictionaries/external-dictionaries/external-dicts.md). 列: -- `name` (String) — Dictionary name. -- `type` (String) — Dictionary type: Flat, Hashed, Cache. -- `origin` (String) — Path to the configuration file that describes the dictionary. -- `attribute.names` (Array(String)) — Array of attribute names provided by the dictionary. -- `attribute.types` (Array(String)) — Corresponding array of attribute types that are provided by the dictionary. -- `has_hierarchy` (UInt8) — Whether the dictionary is hierarchical. -- `bytes_allocated` (UInt64) — The amount of RAM the dictionary uses. -- `hit_rate` (Float64) — For cache dictionaries, the percentage of uses for which the value was in the cache. -- `element_count` (UInt64) — The number of items stored in the dictionary. -- `load_factor` (Float64) — The percentage filled in the dictionary (for a hashed dictionary, the percentage filled in the hash table). -- `creation_time` (DateTime) — The time when the dictionary was created or last successfully reloaded. -- `last_exception` (String) — Text of the error that occurs when creating or reloading the dictionary if the dictionary couldn’t be created. -- `source` (String) — Text describing the data source for the dictionary. +- `database` ([文字列](../sql-reference/data-types/string.md)) — Name of the database containing the dictionary created by DDL query. Empty string for other dictionaries. +- `name` ([文字列](../sql-reference/data-types/string.md)) — [辞書名](../sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md). +- `status` ([Enum8](../sql-reference/data-types/enum.md)) — Dictionary status. Possible values: + - `NOT_LOADED` — Dictionary was not loaded because it was not used. + - `LOADED` — Dictionary loaded successfully. + - `FAILED` — Unable to load the dictionary as a result of an error. + - `LOADING` — Dictionary is loading now. + - `LOADED_AND_RELOADING` — Dictionary is loaded successfully, and is being reloaded right now (frequent reasons: [SYSTEM RELOAD DICTIONARY](../sql-reference/statements/system.md#query_language-system-reload-dictionary) クエリ、タイムアウト、辞書の設定が変更されました）。 + - `FAILED_AND_RELOADING` — Could not load the dictionary as a result of an error and is loading now. +- `origin` ([文字列](../sql-reference/data-types/string.md)) — Path to the configuration file that describes the dictionary. +- `type` ([文字列](../sql-reference/data-types/string.md)) — Type of a dictionary allocation. [メモリへの辞書の格納](../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md). +- `key` — [キータイプ](../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md#ext_dict_structure-key):数値キー ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) or Сomposite key ([文字列](../sql-reference/data-types/string.md)) — form “(type 1, type 2, …, type n)”. +- `attribute.names` ([配列](../sql-reference/data-types/array.md)([文字列](../sql-reference/data-types/string.md))) — Array of [属性名](../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md#ext_dict_structure-attributes) 辞書によって提供されます。 +- `attribute.types` ([配列](../sql-reference/data-types/array.md)([文字列](../sql-reference/data-types/string.md))) — Corresponding array of [属性タイプ](../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md#ext_dict_structure-attributes) それは辞書によって提供されます。 +- `bytes_allocated` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Amount of RAM allocated for the dictionary. +- `query_count` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Number of queries since the dictionary was loaded or since the last successful reboot. +- `hit_rate` ([Float64](../sql-reference/data-types/float.md)) — For cache dictionaries, the percentage of uses for which the value was in the cache. +- `element_count` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Number of items stored in the dictionary. +- `load_factor` ([Float64](../sql-reference/data-types/float.md)) — Percentage filled in the dictionary (for a hashed dictionary, the percentage filled in the hash table). +- `source` ([文字列](../sql-reference/data-types/string.md)) — Text describing the [データソース](../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md) 辞書のために。 +- `lifetime_min` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Minimum [生涯](../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md) その後、ClickHouseは辞書をリロードしようとします（もし `invalidate_query` それが変更された場合にのみ、設定されています）。 秒単位で設定します。 +- `lifetime_max` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Maximum [生涯](../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md) その後、ClickHouseは辞書をリロードしようとします（もし `invalidate_query` それが変更された場合にのみ、設定されています）。 秒単位で設定します。 +- `loading_start_time` ([DateTime](../sql-reference/data-types/datetime.md)) — Start time for loading the dictionary. +- `last_successful_update_time` ([DateTime](../sql-reference/data-types/datetime.md)) — End time for loading or updating the dictionary. Helps to monitor some troubles with external sources and investigate causes. +- `loading_duration` ([Float32](../sql-reference/data-types/float.md)) — Duration of a dictionary loading. +- `last_exception` ([文字列](../sql-reference/data-types/string.md)) — Text of the error that occurs when creating or reloading the dictionary if the dictionary couldn't be created. -辞書によって使用されるメモリの量は、それに格納されているアイテムの数に比例しないことに注意してください。 くフラットおよびキャッシュされた辞書のすべてのメモリー細胞により、予告なしにご指定済みを問わずどのように辞書を実現する +**例** -## システム。イベント {#system_tables-events} +辞書を設定します。 -システムで発生したイベントの数に関する情報が含まれています。 たとえば、テーブルでは、次のように多くの `SELECT` ClickHouseサーバーの起動後にクエリが処理されました。 +``` sql +CREATE DICTIONARY dictdb.dict +( + `key` Int64 DEFAULT -1, + `value_default` String DEFAULT 'world', + `value_expression` String DEFAULT 'xxx' EXPRESSION 'toString(127 * 172)' +) +PRIMARY KEY key +SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'dicttbl' DB 'dictdb')) +LIFETIME(MIN 0 MAX 1) +LAYOUT(FLAT()) +``` + +辞書が読み込まれていることを確認します。 + +``` sql +SELECT * FROM system.dictionaries +``` + +``` text +┌─database─┬─name─┬─status─┬─origin──────┬─type─┬─key────┬─attribute.names──────────────────────┬─attribute.types─────┬─bytes_allocated─┬─query_count─┬─hit_rate─┬─element_count─┬───────────load_factor─┬─source─────────────────────┬─lifetime_min─┬─lifetime_max─┬──loading_start_time─┌──last_successful_update_time─┬──────loading_duration─┬─last_exception─┐ +│ dictdb │ dict │ LOADED │ dictdb.dict │ Flat │ UInt64 │ ['value_default','value_expression'] │ ['String','String'] │ 74032 │ 0 │ 1 │ 1 │ 0.0004887585532746823 │ ClickHouse: dictdb.dicttbl │ 0 │ 1 │ 2020-03-04 04:17:34 │ 2020-03-04 04:30:34 │ 0.002 │ │ +└──────────┴──────┴────────┴─────────────┴──────┴────────┴──────────────────────────────────────┴─────────────────────┴─────────────────┴─────────────┴──────────┴───────────────┴───────────────────────┴────────────────────────────┴──────────────┴──────────────┴─────────────────────┴──────────────────────────────┘───────────────────────┴────────────────┘ +``` + +## システムイベント {#system_tables-events} + +システムで発生したイベントの数に関する情報が含まれます。 例えば、テーブルでどのように多くの `SELECT` ClickHouseサーバーが起動してからクエリが処理されました。 列: @@ -181,7 +221,7 @@ SELECT * FROM system.contributors WHERE name='Olga Khvostikova' - `value` ([UInt64](../sql-reference/data-types/int-uint.md)) — Number of events occurred. - `description` ([文字列](../sql-reference/data-types/string.md)) — Event description. -**例えば** +**例** ``` sql SELECT * FROM system.events LIMIT 5 @@ -197,41 +237,41 @@ SELECT * FROM system.events LIMIT 5 └───────────────────────────────────────┴───────┴────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ ``` -**また見なさい** +**も参照。** -- [システム。asynchronous\_metrics](#system_tables-asynchronous_metrics) — Contains periodically calculated metrics. -- [システム。指標](#system_tables-metrics) — Contains instantly calculated metrics. -- [システム。metric\_log](#system_tables-metric_log) — Contains a history of metrics values from tables `system.metrics` и `system.events`. +- [システムasynchronous\_metrics](#system_tables-asynchronous_metrics) — Contains periodically calculated metrics. +- [システムメトリック](#system_tables-metrics) — Contains instantly calculated metrics. +- [システムmetric\_log](#system_tables-metric_log) — Contains a history of metrics values from tables `system.metrics` и `system.events`. - [監視](monitoring.md) — Base concepts of ClickHouse monitoring. -## システム。機能 {#system-functions} +## システム関数 {#system-functions} -通常の関数と集計関数に関する情報が含まれます。 +標準関数と集計関数に関する情報が含まれます。 列: - `name`(`String`) – The name of the function. - `is_aggregate`(`UInt8`) — Whether the function is aggregate. -## システム。graphite\_retentions {#system-graphite-retentions} +## システムgraphite\_retentions {#system-graphite-retentions} -についての情報が含まれてパラメータ [graphite\_rollup](server-configuration-parameters/settings.md#server_configuration_parameters-graphite) これは、次の表で使用されます [\*グラフィットマージツリー](../engines/table-engines/mergetree-family/graphitemergetree.md) エンジン +パラメ [graphite\_rollup](server-configuration-parameters/settings.md#server_configuration_parameters-graphite) テーブルで使用される [\*GraphiteMergeTree](../engines/table-engines/mergetree-family/graphitemergetree.md) エンジンだ 列: - `config_name` (文字列) - `graphite_rollup` パラメータ名。 - `regexp` (String)-メトリック名のパターン。 - `function` (String)-集計関数の名前。 -- `age` (UInt64)-データの最小経過時間(秒)。 -- `precision` （UInt64）-どのように正確に秒単位でデータの年齢を定義します。 -- `priority` (UInt16)-パターンの優先順位。 -- `is_default` (UInt8)-パターンがデフォルトであるかどうか。 -- `Tables.database` (Array(String))-使用するデータベーステーブルの名前の配列 `config_name` パラメータ。 -- `Tables.table` (Array(String))-使用するテーブル名の配列 `config_name` パラメータ。 +- `age` (UInt64)-データの最小年齢を秒単位で表します。 +- `precision` (UInt64)-データの年齢を秒単位で正確に定義する方法。 +- `priority` (UInt16)-パターンの優先度。 +- `is_default` (UInt8)-パターンがデフォルトかどうか。 +- `Tables.database` (Array(String))-データベーステーブルの名前の配列。 `config_name` パラメータ。 +- `Tables.table` (Array(String))-テーブル名の配列 `config_name` パラメータ。 -## システム。マージ {#system-merges} +## システムマージ {#system-merges} -MergeTreeファミリーのテーブルのマージおよび現在処理中のパーツの変更に関する情報を格納します。 +MergeTreeファミリー内のテーブルで現在処理中のマージおよびパートの変異に関する情報が含まれます。 列: @@ -241,7 +281,7 @@ MergeTreeファミリーのテーブルのマージおよび現在処理中の - `progress` (Float64) — The percentage of completed work from 0 to 1. - `num_parts` (UInt64) — The number of pieces to be merged. - `result_part_name` (String) — The name of the part that will be formed as the result of merging. -- `is_mutation` (UInt8)-1このプロセスが部分突然変異の場合。 +- `is_mutation` (UInt8)-1このプロセスが部分突然変異である場合。 - `total_size_bytes_compressed` (UInt64) — The total size of the compressed data in the merged chunks. - `total_size_marks` (UInt64) — The total number of marks in the merged parts. - `bytes_read_uncompressed` (UInt64) — Number of bytes read, uncompressed. @@ -249,9 +289,9 @@ MergeTreeファミリーのテーブルのマージおよび現在処理中の - `bytes_written_uncompressed` (UInt64) — Number of bytes written, uncompressed. - `rows_written` (UInt64) — Number of rows written. -## システム。指標 {#system_tables-metrics} +## システムメトリック {#system_tables-metrics} -瞬時に計算されるか、現在の値を持つことができる指標が含まれています。 たとえば、同時に処理されたクエリの数や現在のレプリカの遅延などです。 このテーブルは常に最新です。 +即座に計算できるメトリック、または現在の値が含まれます。 たとえば、同時に処理されたクエリの数や現在のレプリカ遅延などです。 このテーブルは常に最新です。 列: @@ -259,9 +299,9 @@ MergeTreeファミリーのテーブルのマージおよび現在処理中の - `value` ([Int64](../sql-reference/data-types/int-uint.md)) — Metric value. - `description` ([文字列](../sql-reference/data-types/string.md)) — Metric description. -サポートされている指標のリストを以下に示します [dbms/Common/CurrentMetrics。cpp](https://github.com/ClickHouse/ClickHouse/blob/master/dbms/Common/CurrentMetrics.cpp) ClickHouseのソースファイル。 +サポートされている指標のリストは、次のとおりです [src/Common/CurrentMetrics。cpp](https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/CurrentMetrics.cpp) ClickHouseのソースファイル。 -**例えば** +**例** ``` sql SELECT * FROM system.metrics LIMIT 10 @@ -282,17 +322,17 @@ SELECT * FROM system.metrics LIMIT 10 └────────────────────────────┴───────┴──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ ``` -**また見なさい** +**も参照。** -- [システム。asynchronous\_metrics](#system_tables-asynchronous_metrics) — Contains periodically calculated metrics. -- [システム。イベント](#system_tables-events) — Contains a number of events that occurred. -- [システム。metric\_log](#system_tables-metric_log) — Contains a history of metrics values from tables `system.metrics` и `system.events`. +- [システムasynchronous\_metrics](#system_tables-asynchronous_metrics) — Contains periodically calculated metrics. +- [システムイベント](#system_tables-events) — Contains a number of events that occurred. +- [システムmetric\_log](#system_tables-metric_log) — Contains a history of metrics values from tables `system.metrics` и `system.events`. - [監視](monitoring.md) — Base concepts of ClickHouse monitoring. -## システム。metric\_log {#system_tables-metric_log} +## システムmetric\_log {#system_tables-metric_log} -表からのメトリック値の履歴を含む `system.metrics` と `system.events`、定期的にディスクにフラッシュ。 -メトリック履歴の収集をオンにするには `system.metric_log`,作成 `/etc/clickhouse-server/config.d/metric_log.xml` 以下の内容で: +を含む履歴メトリクスの値からテーブル `system.metrics` と `system.events`、定期的にディスクにフラッシュ。 +メトリック履歴の収集を有効にするには `system.metric_log`,作成 `/etc/clickhouse-server/config.d/metric_log.xml` 次の内容を使って: ``` xml @@ -305,7 +345,7 @@ SELECT * FROM system.metrics LIMIT 10 ``` -**例えば** +**例** ``` sql SELECT * FROM system.metric_log LIMIT 1 FORMAT Vertical; @@ -338,33 +378,33 @@ CurrentMetric_ReplicatedChecks: 0 ... ``` -**また見なさい** +**も参照。** -- [システム。asynchronous\_metrics](#system_tables-asynchronous_metrics) — Contains periodically calculated metrics. -- [システム。イベント](#system_tables-events) — Contains a number of events that occurred. -- [システム。指標](#system_tables-metrics) — Contains instantly calculated metrics. +- [システムasynchronous\_metrics](#system_tables-asynchronous_metrics) — Contains periodically calculated metrics. +- [システムイベント](#system_tables-events) — Contains a number of events that occurred. +- [システムメトリック](#system_tables-metrics) — Contains instantly calculated metrics. - [監視](monitoring.md) — Base concepts of ClickHouse monitoring. -## システム。数字 {#system-numbers} +## システム数字 {#system-numbers} -このテーブルを一uint64カラム名 ‘number’ ゼロから始まるほぼすべての自然数が含まれています。 -このテーブルをテストに使用するか、ブルートフォース検索を実行する必要がある場合に使用できます。 +このテーブルを一UInt64カラム名 ‘number’ ゼロから始まるほぼすべての自然数が含まれています。 +このテーブルは、テストのため、またはブルートフォース検索を行う必要がある場合に使用できます。 この表からの読み取りは並列化されません。 -## システム。numbers\_mt {#system-numbers-mt} +## システムnumbers\_mt {#system-numbers-mt} -同じように ‘system.numbers’ しかし、読み込みは平行です。 数字は任意の順序で返すことができます。 -使用試験までを実施。 +と同じ ‘system.numbers’ しかし、読み取りは並列処理されます。 番号は任意の順序で返すことができます。 +テストに使用されます。 -## システム。ワン {#system-one} +## システムワン {#system-one} -このテーブルには、単一行と単一行が含まれます ‘dummy’ 値を含むUInt8列0. -このテーブルは、selectクエリがfrom句を指定しない場合に使用されます。 -これは、他のdbmsで見つかったデュアルテーブルに似ています。 +このテーブルには、単一の行が含まれています。 ‘dummy’ UInt8値を含む列0。 +このテーブルは、SELECTクエリでFROM句が指定されていない場合に使用されます。 +これは、他のDbmsにあるデュアルテーブルに似ています。 -## システム。パーツ {#system_tables-parts} +## システム部品 {#system_tables-parts} -の部分に関する情報が含まれます [MergeTree](../engines/table-engines/mergetree-family/mergetree.md) テーブル。 +についての情報が含まれて部品の [メルゲツリー](../engines/table-engines/mergetree-family/mergetree.md) テーブル 各行は、一つのデータ部分を記述します。 @@ -379,9 +419,9 @@ CurrentMetric_ReplicatedChecks: 0 - `name` (`String`) – Name of the data part. -- `active` (`UInt8`) – Flag that indicates whether the data part is active. If a data part is active, it’s used in a table. Otherwise, it’s deleted. Inactive data parts remain after merging. +- `active` (`UInt8`) – Flag that indicates whether the data part is active. If a data part is active, it's used in a table. Otherwise, it's deleted. Inactive data parts remain after merging. -- `marks` (`UInt64`) – The number of marks. To get the approximate number of rows in a data part, multiply `marks` インデックスの粒度（通常は8192）（このヒントは適応的な粒度では機能しません）。 +- `marks` (`UInt64`) – The number of marks. To get the approximate number of rows in a data part, multiply `marks` インデックスの粒度（通常は8192）で指定します（このヒントは適応的な粒度では機能しません）。 - `rows` (`UInt64`) – The number of rows. @@ -421,7 +461,7 @@ CurrentMetric_ReplicatedChecks: 0 - `primary_key_bytes_in_memory_allocated` (`UInt64`) – The amount of memory (in bytes) reserved for primary key values. -- `is_frozen` (`UInt8`) – Flag that shows that a partition data backup exists. 1, the backup exists. 0, the backup doesn’t exist. For more details, see [FREEZE PARTITION](../sql-reference/statements/alter.md#alter_freeze-partition) +- `is_frozen` (`UInt8`) – Flag that shows that a partition data backup exists. 1, the backup exists. 0, the backup doesn't exist. For more details, see [FREEZE PARTITION](../sql-reference/statements/alter.md#alter_freeze-partition) - `database` (`String`) – Name of the database. @@ -433,21 +473,21 @@ CurrentMetric_ReplicatedChecks: 0 - `disk` (`String`) – Name of a disk that stores the data part. -- `hash_of_all_files` (`String`) – [サイファシー128](../sql-reference/functions/hash-functions.md#hash_functions-siphash128) 圧縮されたファイルの。 +- `hash_of_all_files` (`String`) – [sipHash128](../sql-reference/functions/hash-functions.md#hash_functions-siphash128) 圧縮されたファイルの。 -- `hash_of_uncompressed_files` (`String`) – [サイファシー128](../sql-reference/functions/hash-functions.md#hash_functions-siphash128) 非圧縮ファイル(マーク付きファイル、インデックスファイルなど)). +- `hash_of_uncompressed_files` (`String`) – [sipHash128](../sql-reference/functions/hash-functions.md#hash_functions-siphash128) 非圧縮ファイル（マーク付きファイル、インデックスファイルなど。). -- `uncompressed_hash_of_compressed_files` (`String`) – [サイファシー128](../sql-reference/functions/hash-functions.md#hash_functions-siphash128) 圧縮されていないかのように圧縮されたファイル内のデータ。 +- `uncompressed_hash_of_compressed_files` (`String`) – [sipHash128](../sql-reference/functions/hash-functions.md#hash_functions-siphash128) それらが圧縮されていないかのように圧縮されたファイル内のデータの。 - `bytes` (`UInt64`) – Alias for `bytes_on_disk`. - `marks_size` (`UInt64`) – Alias for `marks_bytes`. -## システム。part\_log {#system_tables-part-log} +## システムpart\_log {#system_tables-part-log} -その `system.part_log` テーブルが作成されるのは、 [part\_log](server-configuration-parameters/settings.md#server_configuration_parameters-part-log) サーバー設定を指定します。 +その `system.part_log` テーブルが作成されるのは、 [part\_log](server-configuration-parameters/settings.md#server_configuration_parameters-part-log) サーバ設定を指定します。 -このテーブルには、 [データ部品](../engines/table-engines/mergetree-family/custom-partitioning-key.md) で [MergeTree](../engines/table-engines/mergetree-family/mergetree.md) データの追加やマージなどのファミリテーブル。 +このテーブルについての情報が含まれてイベントが発生した [データパーツ](../engines/table-engines/mergetree-family/custom-partitioning-key.md) で [メルゲツリー](../engines/table-engines/mergetree-family/mergetree.md) データの追加やマージなどのファミリテーブル。 その `system.part_log` テーブルを含む以下のカラム: @@ -476,34 +516,34 @@ CurrentMetric_ReplicatedChecks: 0 その `system.part_log` テーブルは、最初にデータを挿入した後に作成されます。 `MergeTree` テーブル。 -## システム。プロセス {#system_tables-processes} +## システムプロセス {#system_tables-processes} -このシステム表は実行のために使用されます `SHOW PROCESSLIST` クエリ。 +このシステムテーブルは、 `SHOW PROCESSLIST` クエリ。 列: -- `user` (String) – The user who made the query. Keep in mind that for distributed processing, queries are sent to remote servers under the `default` ユーザー。 このフィールドには、特定のクエリのユーザー名が含まれています。 -- `address` (String) – The IP address the request was made from. The same for distributed processing. To track where a distributed query was originally made from, look at `system.processes` クエリリクエスターサーバーです。 +- `user` (String) – The user who made the query. Keep in mind that for distributed processing, queries are sent to remote servers under the `default` ユーザー。 の分野のユーザー名で特定のクエリは、クエリはこのクエリも開始しています。 +- `address` (String) – The IP address the request was made from. The same for distributed processing. To track where a distributed query was originally made from, look at `system.processes` クエリ要求サーバー上。 - `elapsed` (Float64) – The time in seconds since request execution started. - `rows_read` (UInt64) – The number of rows read from the table. For distributed processing, on the requestor server, this is the total for all remote servers. - `bytes_read` (UInt64) – The number of uncompressed bytes read from the table. For distributed processing, on the requestor server, this is the total for all remote servers. - `total_rows_approx` (UInt64) – The approximation of the total number of rows that should be read. For distributed processing, on the requestor server, this is the total for all remote servers. It can be updated during request processing, when new sources to process become known. - `memory_usage` (UInt64) – Amount of RAM the request uses. It might not include some types of dedicated memory. See the [max\_memory\_usage](../operations/settings/query-complexity.md#settings_max_memory_usage) 設定。 -- `query` (String) – The query text. For `INSERT`、それは挿入するデータが含まれていません。 +- `query` (String) – The query text. For `INSERT`,挿入するデータは含まれません。 - `query_id` (String) – Query ID, if defined. -## システム。text\_log {#system-tables-text-log} +## システムtext\_log {#system-tables-text-log} -ログエントリを含む。 この表に行くロギングのレベルはで限られます `text_log.level` サーバー設定。 +を含むログイン作品の応募がありました。 ログレベルがこのテーブルで限定 `text_log.level` サーバー設定。 列: -- `event_date` (`Date`)-エントリの日付。 +- `event_date` (`Date`）-エントリの日付。 - `event_time` (`DateTime`）-エントリの時間。 - `microseconds` (`UInt32`)-エントリのマイクロ秒。 - `thread_name` (String) — Name of the thread from which the logging was done. - `thread_id` (UInt64) — OS thread ID. -- `level` (`Enum8`)-エントリーレベル。 +- `level` (`Enum8`）-エントリーレベル。 - `'Fatal' = 1` - `'Critical' = 2` - `'Error' = 3` @@ -514,26 +554,26 @@ CurrentMetric_ReplicatedChecks: 0 - `'Trace' = 8` - `query_id` (`String`)-クエリのID。 - `logger_name` (`LowCardinality(String)`) - Name of the logger (i.e. `DDLWorker`) -- `message` (`String`)-メッセージそのもの。 -- `revision` (`UInt32`)-ClickHouseリビジョン. +- `message` (`String`）-メッセージ自体。 +- `revision` (`UInt32`）-ClickHouseリビジョン。 - `source_file` (`LowCardinality(String)`)-ロギングが行われたソースファイル。 -- `source_line` (`UInt64`)-ロギングが行われたソースライン。 +- `source_line` (`UInt64`)-ロギングが行われたソース行。 -## システム。クエリーログ {#system_tables-query_log} +## システムquery\_log {#system_tables-query_log} -クエリの実行に関する情報が含まれます。 各クエリについて、処理の開始時間、処理時間、エラーメッセージおよびその他の情報を確認できます。 +クエリの実行に関する情報が含まれます。 クエリごとに、処理開始時間、処理時間、エラーメッセージおよびその他の情報を確認できます。 -!!! note "メモ" +!!! note "注" テーブルには以下の入力データは含まれません `INSERT` クエリ。 -ClickHouseはこのテーブルを作成します。 [クエリーログ](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) サーバパラメータを指定します。 このパラメーターは、ロギング間隔やクエリがログに記録されるテーブルの名前などのロギングルールを設定します。 +ClickHouseはこのテーブルを作成します。 [query\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) serverパラメータを指定します。 このパラメーターは、クエリがログインするテーブルのログ間隔や名前などのログルールを設定します。 -するクエリのロギングの設定を [log\_queries](settings/settings.md#settings-log-queries) 1へのパラメータ。 詳細については、 [設定](settings/settings.md) セクション。 +クエリロギングを有効にするには、 [log\_queries](settings/settings.md#settings-log-queries) パラメータは1。 詳細については、 [設定](settings/settings.md) セクション -その `system.query_log` テーブルレジスタ二種類のクエリ: +その `system.query_log` テーブルレジスタの種類は問合せ: 1. クライアントによって直接実行された初期クエリ。 -2. 他のクエリによって開始された子クエリ(分散クエリ実行用)。 これらのタイプのクエリについては、親クエリに関する情報が表示されます。 `initial_*` 列。 +2. 他のクエリによって開始された子クエリ(分散クエリ実行用)。 これらのタイプのクエリについては、親クエリに関する情報が `initial_*` 列。 列: @@ -548,8 +588,8 @@ ClickHouseはこのテーブルを作成します。 [クエリーログ](server - `query_duration_ms` (UInt64) — Duration of query execution. - `read_rows` (UInt64) — Number of read rows. - `read_bytes` (UInt64) — Number of read bytes. -- `written_rows` (UInt64) — For `INSERT` クエリ、書き込まれた行の数。 他のクエリの場合、列の値は0です。 -- `written_bytes` (UInt64) — For `INSERT` クエリ、書き込まれたバイト数。 他のクエリの場合、列の値は0です。 +- `written_rows` (UInt64) — For `INSERT` クエリ、書き込まれた行の数。 その他のクエリの場合、列の値は0です。 +- `written_bytes` (UInt64) — For `INSERT` クエリ、書き込まれたバイト数。 その他のクエリの場合、列の値は0です。 - `result_rows` (UInt64) — Number of rows in the result. - `result_bytes` (UInt64) — Number of bytes in the result. - `memory_usage` (UInt64) — Memory consumption by the query. @@ -570,48 +610,48 @@ ClickHouseはこのテーブルを作成します。 [クエリーログ](server - `interface` (UInt8) — Interface that the query was initiated from. Possible values: - 1 — TCP. - 2 — HTTP. -- `os_user` (String) — OS’s username who runs [クリックハウス-顧客](../interfaces/cli.md). -- `client_hostname` (String) — Hostname of the client machine where the [クリックハウス-顧客](../interfaces/cli.md) または他のTCPクライアントが実行されます。 -- `client_name` (String) — The [クリックハウス-顧客](../interfaces/cli.md) または別のTCPクライアント名。 -- `client_revision` (UInt32) — Revision of the [クリックハウス-顧客](../interfaces/cli.md) または別のTCPクライアント。 -- `client_version_major` (UInt32) — Major version of the [クリックハウス-顧客](../interfaces/cli.md) または別のTCPクライアント。 -- `client_version_minor` (UInt32) — Minor version of the [クリックハウス-顧客](../interfaces/cli.md) または別のTCPクライアント。 -- `client_version_patch` (UInt32) — Patch component of the [クリックハウス-顧客](../interfaces/cli.md) または別のTCPクライアン +- `os_user` (String) — OS's username who runs [clickhouse-クライアント](../interfaces/cli.md). +- `client_hostname` (String) — Hostname of the client machine where the [clickhouse-クライアント](../interfaces/cli.md) または他のTCPクライアントが実行されます。 +- `client_name` (String) — The [clickhouse-クライアント](../interfaces/cli.md) または別のTCPクライアント名。 +- `client_revision` (UInt32) — Revision of the [clickhouse-クライアント](../interfaces/cli.md) または別のTCPクライアント。 +- `client_version_major` (UInt32) — Major version of the [clickhouse-クライアント](../interfaces/cli.md) または別のTCPクライアント。 +- `client_version_minor` (UInt32) — Minor version of the [clickhouse-クライアント](../interfaces/cli.md) または別のTCPクライアント。 +- `client_version_patch` (UInt32) — Patch component of the [clickhouse-クライアント](../interfaces/cli.md) 別のTCPクライアントバージョン。 - `http_method` (UInt8) — HTTP method that initiated the query. Possible values: - 0 — The query was launched from the TCP interface. - 1 — `GET` 方法を用いた。 - 2 — `POST` 方法を用いた。 - `http_user_agent` (String) — The `UserAgent` HTTP要求で渡されるヘッダー。 -- `quota_key` (String) — The “quota key” で指定される [クォータ](quotas.md) 設定（参照 `keyed`). +- `quota_key` (String) — The “quota key” で指定される。 [クォータ](quotas.md) 設定(参照 `keyed`). - `revision` (UInt32) — ClickHouse revision. - `thread_numbers` (Array(UInt32)) — Number of threads that are participating in query execution. -- `ProfileEvents.Names` (Array(String)) — Counters that measure different metrics. The description of them could be found in the table [システム。イベント](#system_tables-events) -- `ProfileEvents.Values` (Array(UInt64)) — Values of metrics that are listed in the `ProfileEvents.Names` コラム -- `Settings.Names` (Array(String)) — Names of settings that were changed when the client ran the query. To enable logging changes to settings, set the `log_query_settings` 1へのパラメータ。 -- `Settings.Values` (Array(String)) — Values of settings that are listed in the `Settings.Names` コラム +- `ProfileEvents.Names` (Array(String)) — Counters that measure different metrics. The description of them could be found in the table [システムイベント](#system_tables-events) +- `ProfileEvents.Values` (Array(UInt64)) — Values of metrics that are listed in the `ProfileEvents.Names` 列。 +- `Settings.Names` (Array(String)) — Names of settings that were changed when the client ran the query. To enable logging changes to settings, set the `log_query_settings` パラメータは1。 +- `Settings.Values` (Array(String)) — Values of settings that are listed in the `Settings.Names` 列。 -それぞれのクエリでは、次の行が作成されます。 `query_log` クエリのステータスに応じたテーブル: +それぞれのクエリでは、一つまたは二つの行が `query_log` クエリのステータスに応じて、テーブル: 1. クエリの実行が成功すると、タイプ1とタイプ2のイベントが作成されます。 `type` 列）。 -2. クエリ処理中にエラーが発生した場合は、タイプ1とタイプ4のイベントが作成されます。 +2. クエリ処理中にエラーが発生した場合は、タイプ1と4のイベントが作成されます。 3. クエリを起動する前にエラーが発生した場合は、タイプ3の単一のイベントが作成されます。 -デフォルトでは、7.5秒間隔でログがテーブルに追加されます。 この間隔を設定することができます [クエリーログ](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) サーバー設定（参照してください `flush_interval_milliseconds` パラメータ）。 フラッシュを強制的にログからのメモリバッファ、テーブルを使用 `SYSTEM FLUSH LOGS` クエリ。 +既定では、ログは7.5秒間隔でテーブルに追加されます。 この間隔は [query\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) サーバ設定(参照 `flush_interval_milliseconds` 変数）。 ログをメモリバッファからテーブルに強制的にフラッシュするには、 `SYSTEM FLUSH LOGS` クエリ。 -テーブルが手動で削除されると、その場で自動的に作成されます。 以前のログはすべて削除されることに注意してください。 +テーブルを手動で削除すると、その場で自動的に作成されます。 以前のログはすべて削除されます。 -!!! note "メモ" - ログの保存期間は無制限です。 ログはテーブルから自動的に削除されません。 古いログの削除を自分で整理する必要があります。 +!!! note "注" + ログの保存期間は無制限です。 ログはテーブルから自動的には削除されません。 古いログの削除を自分で整理する必要があります。 -任意のパーティション分割キーを指定できます。 `system.query_log` のテーブル [クエリーログ](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) サーバー設定（参照してください `partition_by` パラメータ）。 +パーティショニングキーを指定できます。 `system.query_log` のテーブル [query\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) サーバ設定(参照 `partition_by` 変数）。 -## システム。query\_thread\_log {#system_tables-query-thread-log} +## システムquery\_thread\_log {#system_tables-query-thread-log} のテーブルについての情報が含まれてそれぞれの検索キーワード実行スレッド. -ClickHouseはこのテーブルを作成します。 [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-thread-log) サーバパラメータを指定します。 このパラメーターは、ロギング間隔やクエリがログに記録されるテーブルの名前などのロギングルールを設定します。 +ClickHouseはこのテーブルを作成します。 [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-thread-log) serverパラメータを指定します。 このパラメーターは、クエリがログインするテーブルのログ間隔や名前などのログルールを設定します。 -するクエリのロギングの設定を [log\_query\_threads](settings/settings.md#settings-log-query-threads) 1へのパラメータ。 詳細については、 [設定](settings/settings.md) セクション。 +クエリロギングを有効にするには、 [log\_query\_threads](settings/settings.md#settings-log-query-threads) パラメータは1。 詳細については、 [設定](settings/settings.md) セクション 列: @@ -621,8 +661,8 @@ ClickHouseはこのテーブルを作成します。 [query\_thread\_log](server - `query_duration_ms` (UInt64) — Duration of query execution. - `read_rows` (UInt64) — Number of read rows. - `read_bytes` (UInt64) — Number of read bytes. -- `written_rows` (UInt64) — For `INSERT` クエリ、書き込まれた行の数。 他のクエリの場合、列の値は0です。 -- `written_bytes` (UInt64) — For `INSERT` クエリ、書き込まれたバイト数。 他のクエリの場合、列の値は0です。 +- `written_rows` (UInt64) — For `INSERT` クエリ、書き込まれた行の数。 その他のクエリの場合、列の値は0です。 +- `written_bytes` (UInt64) — For `INSERT` クエリ、書き込まれたバイト数。 その他のクエリの場合、列の値は0です。 - `memory_usage` (Int64) — The difference between the amount of allocated and freed memory in context of this thread. - `peak_memory_usage` (Int64) — The maximum difference between the amount of allocated and freed memory in context of this thread. - `thread_name` (String) — Name of the thread. @@ -644,62 +684,64 @@ ClickHouseはこのテーブルを作成します。 [query\_thread\_log](server - `interface` (UInt8) — Interface that the query was initiated from. Possible values: - 1 — TCP. - 2 — HTTP. -- `os_user` (String) — OS’s username who runs [クリックハウス-顧客](../interfaces/cli.md). -- `client_hostname` (String) — Hostname of the client machine where the [クリックハウス-顧客](../interfaces/cli.md) または他のTCPクライアントが実行されます。 -- `client_name` (String) — The [クリックハウス-顧客](../interfaces/cli.md) または別のTCPクライアント名。 -- `client_revision` (UInt32) — Revision of the [クリックハウス-顧客](../interfaces/cli.md) または別のTCPクライアント。 -- `client_version_major` (UInt32) — Major version of the [クリックハウス-顧客](../interfaces/cli.md) または別のTCPクライアント。 -- `client_version_minor` (UInt32) — Minor version of the [クリックハウス-顧客](../interfaces/cli.md) または別のTCPクライアント。 -- `client_version_patch` (UInt32) — Patch component of the [クリックハウス-顧客](../interfaces/cli.md) または別のTCPクライアン +- `os_user` (String) — OS's username who runs [clickhouse-クライアント](../interfaces/cli.md). +- `client_hostname` (String) — Hostname of the client machine where the [clickhouse-クライアント](../interfaces/cli.md) または他のTCPクライアントが実行されます。 +- `client_name` (String) — The [clickhouse-クライアント](../interfaces/cli.md) または別のTCPクライアント名。 +- `client_revision` (UInt32) — Revision of the [clickhouse-クライアント](../interfaces/cli.md) または別のTCPクライアント。 +- `client_version_major` (UInt32) — Major version of the [clickhouse-クライアント](../interfaces/cli.md) または別のTCPクライアント。 +- `client_version_minor` (UInt32) — Minor version of the [clickhouse-クライアント](../interfaces/cli.md) または別のTCPクライアント。 +- `client_version_patch` (UInt32) — Patch component of the [clickhouse-クライアント](../interfaces/cli.md) 別のTCPクライアントバージョン。 - `http_method` (UInt8) — HTTP method that initiated the query. Possible values: - 0 — The query was launched from the TCP interface. - 1 — `GET` 方法を用いた。 - 2 — `POST` 方法を用いた。 - `http_user_agent` (String) — The `UserAgent` HTTP要求で渡されるヘッダー。 -- `quota_key` (String) — The “quota key” で指定される [クォータ](quotas.md) 設定（参照 `keyed`). +- `quota_key` (String) — The “quota key” で指定される。 [クォータ](quotas.md) 設定(参照 `keyed`). - `revision` (UInt32) — ClickHouse revision. -- `ProfileEvents.Names` (Array(String)) — Counters that measure different metrics for this thread. The description of them could be found in the table [システム。イベント](#system_tables-events) -- `ProfileEvents.Values` (Array(UInt64)) — Values of metrics for this thread that are listed in the `ProfileEvents.Names` コラム +- `ProfileEvents.Names` (Array(String)) — Counters that measure different metrics for this thread. The description of them could be found in the table [システムイベント](#system_tables-events) +- `ProfileEvents.Values` (Array(UInt64)) — Values of metrics for this thread that are listed in the `ProfileEvents.Names` 列。 -デフォルトでは、7.5秒間隔でログがテーブルに追加されます。 この間隔を設定することができます [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-thread-log) サーバー設定（参照してください `flush_interval_milliseconds` パラメータ）。 フラッシュを強制的にログからのメモリバッファ、テーブルを使用 `SYSTEM FLUSH LOGS` クエリ。 +既定では、ログは7.5秒間隔でテーブルに追加されます。 この間隔は [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-thread-log) サーバ設定(参照 `flush_interval_milliseconds` 変数）。 ログをメモリバッファからテーブルに強制的にフラッシュするには、 `SYSTEM FLUSH LOGS` クエリ。 -テーブルが手動で削除されると、その場で自動的に作成されます。 以前のログはすべて削除されることに注意してください。 +テーブルを手動で削除すると、その場で自動的に作成されます。 以前のログはすべて削除されます。 -!!! note "メモ" - ログの保存期間は無制限です。 ログはテーブルから自動的に削除されません。 古いログの削除を自分で整理する必要があります。 +!!! note "注" + ログの保存期間は無制限です。 ログはテーブルから自動的には削除されません。 古いログの削除を自分で整理する必要があります。 -任意のパーティション分割キーを指定できます。 `system.query_thread_log` のテーブル [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-thread-log) サーバー設定（参照してください `partition_by` パラメータ）。 +パーティショニングキーを指定できます。 `system.query_thread_log` のテーブル [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-thread-log) サーバ設定(参照 `partition_by` 変数）。 -## システム。trace\_log {#system_tables-trace_log} +## システムtrace\_log {#system_tables-trace_log} -サンプリングクエリプ +を含むスタックトレースの収集、サンプリングクロファイラ. -ClickHouseはこのテーブルを作成します [trace\_log](server-configuration-parameters/settings.md#server_configuration_parameters-trace_log) サーバの設定が設定されます。 また、 [query\_profiler\_real\_time\_period\_ns](settings/settings.md#query_profiler_real_time_period_ns) と [query\_profiler\_cpu\_time\_period\_ns](settings/settings.md#query_profiler_cpu_time_period_ns) 設定は設定する必要があります。 +ClickHouseはこのテーブルを作成します。 [trace\_log](server-configuration-parameters/settings.md#server_configuration_parameters-trace_log) サーバの設定が設定されます。 また、 [query\_profiler\_real\_time\_period\_ns](settings/settings.md#query_profiler_real_time_period_ns) と [query\_profiler\_cpu\_time\_period\_ns](settings/settings.md#query_profiler_cpu_time_period_ns) 設定は設定する必要があります。 -ログを分析するには、以下を使用します `addressToLine`, `addressToSymbol` と `demangle` イントロスペクション関数。 +ログを分析するには、 `addressToLine`, `addressToSymbol` と `demangle` イントロスペクション関数。 列: -- `event_date`([日付](../sql-reference/data-types/date.md)) — Date of sampling moment. +- `event_date` ([日付](../sql-reference/data-types/date.md)) — Date of sampling moment. -- `event_time`([DateTime](../sql-reference/data-types/datetime.md)) — Timestamp of sampling moment. +- `event_time` ([DateTime](../sql-reference/data-types/datetime.md)) — Timestamp of the sampling moment. -- `revision`([UInt32](../sql-reference/data-types/int-uint.md)) — ClickHouse server build revision. +- `timestamp_ns` ([UInt64](../sql-reference/data-types/int-uint.md)) — Timestamp of the sampling moment in nanoseconds. - サーバーに接続する場合 `clickhouse-client`、あなたは次のような文字列が表示されます `Connected to ClickHouse server version 19.18.1 revision 54429.`. このフィールドには、 `revision`、しかしない `version` サーバーの。 +- `revision` ([UInt32](../sql-reference/data-types/int-uint.md)) — ClickHouse server build revision. -- `timer_type`([Enum8](../sql-reference/data-types/enum.md)) — Timer type: + サーバーに接続するとき `clickhouse-client` のような文字列を参照してください `Connected to ClickHouse server version 19.18.1 revision 54429.`. このフィールドには `revision` ではなく、 `version` サーバーの。 - - `Real` 壁時計の時刻を表します。 +- `timer_type` ([Enum8](../sql-reference/data-types/enum.md)) — Timer type: + + - `Real` 壁時計の時間を表します。 - `CPU` CPU時間を表します。 -- `thread_number`([UInt32](../sql-reference/data-types/int-uint.md)) — Thread identifier. +- `thread_number` ([UInt32](../sql-reference/data-types/int-uint.md)) — Thread identifier. -- `query_id`([文字列](../sql-reference/data-types/string.md)) — Query identifier that can be used to get details about a query that was running from the [クエリーログ](#system_tables-query_log) システムテーブル。 +- `query_id` ([文字列](../sql-reference/data-types/string.md)) — Query identifier that can be used to get details about a query that was running from the [query\_log](#system_tables-query_log) システムテーブル。 -- `trace`([配列(uint64)](../sql-reference/data-types/array.md)) — Stack trace at the moment of sampling. Each element is a virtual memory address inside ClickHouse server process. +- `trace` ([配列(UInt64)](../sql-reference/data-types/array.md)) — Stack trace at the moment of sampling. Each element is a virtual memory address inside ClickHouse server process. -**例えば** +**例** ``` sql SELECT * FROM system.trace_log LIMIT 1 \G @@ -717,12 +759,12 @@ query_id: acc4d61f-5bd1-4a3e-bc91-2180be37c915 trace: [94222141367858,94222152240175,94222152325351,94222152329944,94222152330796,94222151449980,94222144088167,94222151682763,94222144088167,94222151682763,94222144088167,94222144058283,94222144059248,94222091840750,94222091842302,94222091831228,94222189631488,140509950166747,140509942945935] ``` -## システム。レプリカ {#system_tables-replicas} +## システムレプリカ {#system_tables-replicas} 情報および状況を再現しテーブル在住の地元のサーバーです。 -このテーブルは、監視に使用できます。 のテーブルが含まれて行毎に再現\*ます。 +このテーブルは監視に使用することができる。 のテーブルが含まれて行毎に再現\*ます。 -例えば: +例: ``` sql SELECT * @@ -771,39 +813,39 @@ active_replicas: 2 - `database` (`String`)-データベース名 - `table` (`String`)-テーブル名 - `engine` (`String`)-テーブルエンジン名 -- `is_leader` (`UInt8`)-レプリカがリーダーであるかどうか。 - リーダーは一度にひとつのレプリカのみです。 リーダーは実行するバックグラウンドマージの選択を担当します。 - 書き込みは、リーダーであるかどうかにかかわらず、利用可能であり、zkにセッションを持つ任意のレプリカに対して実行できます。 -- `can_become_leader` (`UInt8`)-レプリカをリーダーとして選出できるかどうか。 +- `is_leader` (`UInt8`）-レプリカがリーダーであるかどうか。 + 一度に一つのレプリカだけがリーダーになれます。 リーダーは、実行するバックグラウンドマージを選択します。 + 書き込みは、リーダーであるかどうかに関係なく、使用可能でZKにセッションがある任意のレプリカに対して実行できます。 +- `can_become_leader` (`UInt8`）-レプリカがリーダーとして選出できるかどうか。 - `is_readonly` (`UInt8`)-レプリカが読み取り専用モードであるかどうか。 - このモードは、zookeeperでセッションを再初期化するときに不明なエラーが発生した場合、およびzookeeperでのセッション再初期化中に、zookeeperとのセクションが設定さ -- `is_session_expired` (`UInt8`)-ZooKeeperとのセッションが終了しました。 基本的には `is_readonly`. + このモードは、設定にZooKeeperのセクションがない場合、zookeeperでセッションを再初期化するとき、およびZooKeeperでセッションを再初期化するときに不明なエラーが発生 +- `is_session_expired` (`UInt8`）-ZooKeeperとのセッションが終了しました。 基本的には `is_readonly`. - `future_parts` (`UInt32`)-まだ行われていない挿入またはマージの結果として表示されるデータパーツの数。 -- `parts_to_check` (`UInt32`)-検証のためのキュー内のデータパーツの数。 破損している可能性があるという疑いがある場合、部品は検証キューに入れられます。 -- `zookeeper_path` (`String`)-ZooKeeperのテーブルデータへのパス。 -- `replica_name` (`String`）-飼育係のレプリカ名。 異なるレプリカと同じテーブルの異名をとります。 -- `replica_path` (`String`）-飼育係のレプリカデータへのパス。 連結と同じです ‘zookeeper\_path/replicas/replica\_path’. -- `columns_version` (`Int32`)-テーブル構造のバージョン番号。 ALTERが実行された回数を示します。 場合にレプリカは異なるバージョンで一部のレプリカさんのすべての変更はまだない。 -- `queue_size` (`UInt32`)-実行待ち操作のキューのサイズ。 業務などのブロックを挿入し、データ統合し、行動します。 それは通常 `future_parts`. -- `inserts_in_queue` (`UInt32`)-作成する必要があるデータブロックの挿入数。 挿入は、通常、かなり迅速に複製されます。 この数が大きい場合は、何かが間違っていることを意味します。 -- `merges_in_queue` (`UInt32`)-作成されるのを待機しているマージの数。 マージが時間がかかることがあるので、この値は長い間ゼロより大きくなることがあります。 +- `parts_to_check` (`UInt32`)-検証のためのキュー内のデータ部分の数。 破損の疑いがある場合は、部品を検証キューに入れます。 +- `zookeeper_path` (`String`）-ZooKeeperのテーブルデータへのパス。 +- `replica_name` (`String`）-飼育係のレプリカ名。 同じテーブルの異なるレプリカの名前は異なります。 +- `replica_path` (`String`）-ZooKeeperのレプリカデータへのパス。 連結と同じ ‘zookeeper\_path/replicas/replica\_path’. +- `columns_version` (`Int32`)-テーブル構造のバージョン番号。 変更が実行された回数を示します。 場合にレプリカは異なるバージョンで一部のレプリカさんのすべての変更はまだない。 +- `queue_size` (`UInt32`)-実行待ちの操作のキューのサイズ。 操作には、データのブロックの挿入、マージ、その他の特定の操作が含まれます。 それは通常と一致します `future_parts`. +- `inserts_in_queue` (`UInt32`)-必要なデータブロックの挿入数。 挿入は通常、かなり迅速に複製されます。 この数が大きい場合は、何かが間違っていることを意味します。 +- `merges_in_queue` (`UInt32`)-行われるのを待っているマージの数。 時にはマージが長いので、この値は長い間ゼロより大きくなることがあります。 - `part_mutations_in_queue` (`UInt32`）-作られるのを待っている突然変異の数。 -- `queue_oldest_time` (`DateTime`)-If `queue_size` 0より大きい場合、最も古い操作がキューに追加された日時が表示されます。 -- `inserts_oldest_time` (`DateTime`)-見る `queue_oldest_time` -- `merges_oldest_time` (`DateTime`)-見る `queue_oldest_time` -- `part_mutations_oldest_time` (`DateTime`)-見る `queue_oldest_time` +- `queue_oldest_time` (`DateTime`)-If `queue_size` 0より大きい場合は、最も古い操作がいつキューに追加されたかを示します。 +- `inserts_oldest_time` (`DateTime`）-参照 `queue_oldest_time` +- `merges_oldest_time` (`DateTime`）-参照 `queue_oldest_time` +- `part_mutations_oldest_time` (`DateTime`）-参照 `queue_oldest_time` -次の4列は、zkとのアクティブなセッションがある場合にのみ、ゼロ以外の値を持ちます。 +次の4つの列は、zkとのアクティブなセッションがある場合にのみゼロ以外の値を持ちます。 -- `log_max_index` (`UInt64`）-一般的な活動のログの最大エントリ番号。 -- `log_pointer` (`UInt64`)-レプリカが実行キューにコピーした一般的なアクティビティのログの最大エントリ番号。 もし `log_pointer` はるかに小さいよりも `log_max_index`、何かが間違っている。 -- `last_queue_update` (`DateTime`）-キューが前回updatedされたとき。 -- `absolute_delay` (`UInt64`）-どのように大きな遅れ秒で現在のレプリカがあります。 +- `log_max_index` (`UInt64`）-一般的な活動のログ内の最大エントリ番号。 +- `log_pointer` (`UInt64`)-レプリカがその実行キューにコピーした一般的なアクティビティのログ内の最大エントリ番号を加えたもの。 もし `log_pointer` はるかに小さい `log_max_index` 何かおかしい +- `last_queue_update` (`DateTime`)-キューが前回updatedされたとき。 +- `absolute_delay` (`UInt64`）-現在のレプリカが持っている秒単位の大きな遅れ。 - `total_replicas` (`UInt8`)-このテーブルの既知のレプリカの総数。 -- `active_replicas` (`UInt8`）-ZooKeeperでセッションを持つこのテーブルのレプリカの数（つまり、機能するレプリカの数）。 +- `active_replicas` (`UInt8`)-ZooKeeperにセッションがあるこのテーブルのレプリカの数(つまり、機能しているレプリカの数)。 -希望される場合は、すべての列は、テーブルが少しゆっくりと、くつかの読み込みから飼育係したがって行います。 -最後の4列（log\_max\_index、log\_pointer、total\_replicas、active\_replicas）を要求しないと、テーブルはすぐに動作します。 +すべての列を要求すると、ZooKeeperからのいくつかの読み取りが行ごとに行われるため、テーブルは少し遅く動作する可能性があります。 +最後の4つの列（log\_max\_index、log\_pointer、total\_replicas、active\_replicas）を要求しないと、テーブルはすぐに機能します。 たとえば、次のようにすべてが正常に動作していることを確認できます: @@ -837,32 +879,61 @@ WHERE OR active_replicas < total_replicas ``` -このクエリが何も返さない場合は、すべてが正常であることを意味します。 +このクエリが何も返さない場合は、すべて正常であることを意味します。 -## システム。設定 {#system-settings} +## システム設定 {#system-tables-system-settings} -現在使用中の設定に関する情報が含まれます。 -つまり、システムからの読み取りに使用しているクエリを実行するために使用されます。設定テーブル. +現在のユ 列: -- `name` (String) — Setting name. -- `value` (String) — Setting value. -- `description` (String) — Setting description. -- `type` (String) — Setting type (implementation specific string value). -- `changed` (UInt8) — Whether the setting was explicitly defined in the config or explicitly changed. -- `min` (Nullable(String)) — Get minimum allowed value (if any is set via [制約](settings/constraints-on-settings.md#constraints-on-settings)). -- `max` (Nullable(String)) — Get maximum allowed value (if any is set via [制約](settings/constraints-on-settings.md#constraints-on-settings)). -- `readonly` (UInt8) — Can user change this setting (for more info, look into [制約](settings/constraints-on-settings.md#constraints-on-settings)). +- `name` ([文字列](../sql-reference/data-types/string.md)) — Setting name. +- `value` ([文字列](../sql-reference/data-types/string.md)) — Setting value. +- `changed` ([UInt8](../sql-reference/data-types/int-uint.md#uint-ranges)) — Shows whether a setting is changed from its default value. +- `description` ([文字列](../sql-reference/data-types/string.md)) — Short setting description. +- `min` ([Null可能](../sql-reference/data-types/nullable.md)([文字列](../sql-reference/data-types/string.md))) — Minimum value of the setting, if any is set via [制約](settings/constraints-on-settings.md#constraints-on-settings). 最小値が設定されていない場合は、以下を含みます [NULL](../sql-reference/syntax.md#null-literal). +- `max` ([Null可能](../sql-reference/data-types/nullable.md)([文字列](../sql-reference/data-types/string.md))) — Maximum value of the setting, if any is set via [制約](settings/constraints-on-settings.md#constraints-on-settings). 最大値が設定されていない場合は、以下を含みます [NULL](../sql-reference/syntax.md#null-literal). +- `readonly` ([UInt8](../sql-reference/data-types/int-uint.md#uint-ranges)) — Shows whether the current user can change the setting: + - `0` — Current user can change the setting. + - `1` — Current user can't change the setting. -例えば: +**例** + +次の例は、名前に含まれる設定に関する情報を取得する方法を示しています `min_i`. ``` sql -SELECT name, value +SELECT * FROM system.settings -WHERE changed +WHERE name LIKE '%min_i%' ``` +``` text +┌─name────────────────────────────────────────┬─value─────┬─changed─┬─description───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┬─min──┬─max──┬─readonly─┐ +│ min_insert_block_size_rows │ 1048576 │ 0 │ Squash blocks passed to INSERT query to specified size in rows, if blocks are not big enough. │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ 0 │ +│ min_insert_block_size_bytes │ 268435456 │ 0 │ Squash blocks passed to INSERT query to specified size in bytes, if blocks are not big enough. │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ 0 │ +│ read_backoff_min_interval_between_events_ms │ 1000 │ 0 │ Settings to reduce the number of threads in case of slow reads. Do not pay attention to the event, if the previous one has passed less than a certain amount of time. │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ 0 │ +└─────────────────────────────────────────────┴───────────┴─────────┴───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┴──────┴──────┴──────────┘ +``` + +の使用 `WHERE changed` たとえば、チェックしたいときに便利です: + +- 構成ファイルの設定が正しく読み込まれ、使用されているかどうか。 +- 現在のセッションで変更された設定。 + + + +``` sql +SELECT * FROM system.settings WHERE changed AND name='load_balancing' +``` + +**も参照。** + +- [設定](settings/index.md#session-settings-intro) +- [クエリの権限](settings/permissions-for-queries.md#settings_readonly) +- [設定の制約](settings/constraints-on-settings.md) + +## システムtable\_engines {#system.table_engines} + ``` text ┌─name───────────────────┬─value───────┐ │ max_threads │ 8 │ @@ -872,9 +943,9 @@ WHERE changed └────────────────────────┴─────────────┘ ``` -## システム。merge\_tree\_settings {#system-merge_tree_settings} +## システムmerge\_tree\_settings {#system-merge_tree_settings} -についての情報が含まれて設定 `MergeTree` テーブル。 +の設定に関する情報が含まれます `MergeTree` テーブル 列: @@ -884,7 +955,7 @@ WHERE changed - `type` (String) — Setting type (implementation specific string value). - `changed` (UInt8) — Whether the setting was explicitly defined in the config or explicitly changed. -## システム。table\_engines {#system-table-engines} +## システムtable\_engines {#system-table-engines} を含むの記述のテーブルエンジンをサポートサーバーとその特徴を支援す。 @@ -898,7 +969,7 @@ WHERE changed - `supports_replication` (UInt8) — Flag that indicates if table engine supports [データ複製](../engines/table-engines/mergetree-family/replication.md). - `supports_duduplication` (UInt8) — Flag that indicates if table engine supports data deduplication. -例えば: +例: ``` sql SELECT * @@ -914,13 +985,13 @@ WHERE name in ('Kafka', 'MergeTree', 'ReplicatedCollapsingMergeTree') └───────────────────────────────┴───────────────────┴───────────────────────────┴─────────────────────┴──────────────┴──────────────────────┴────────────────────────┘ ``` -**また見なさい** +**も参照。** -- マージツリーファミリー [クエリ句](../engines/table-engines/mergetree-family/mergetree.md#mergetree-query-clauses) -- カフカname [設定](../engines/table-engines/integrations/kafka.md#table_engine-kafka-creating-a-table) +- メルゲツリー族 [クエリ句](../engines/table-engines/mergetree-family/mergetree.md#mergetree-query-clauses) +- カフカ [設定](../engines/table-engines/integrations/kafka.md#table_engine-kafka-creating-a-table) - 参加 [設定](../engines/table-engines/special/join.md#join-limitations-and-settings) -## システム。テーブル {#system-tables} +## システムテーブル {#system-tables} を含むメタデータは各テーブルサーバーに知っています。 デタッチされたテーブルは `system.tables`. @@ -934,15 +1005,15 @@ WHERE name in ('Kafka', 'MergeTree', 'ReplicatedCollapsingMergeTree') - `is_temporary` (UInt8)-テーブルが一時的かどうかを示すフラグ。 -- `data_path` （文字列）-ファイルシステム内のテーブルデータへのパス。 +- `data_path` (String)-ファイルシステム内のテーブルデータへのパス。 - `metadata_path` (String)-ファイルシステム内のテーブルメタデータへのパス。 -- `metadata_modification_time` (DateTime)-テーブルメタデータの最新の変更の時刻。 +- `metadata_modification_time` (DateTime)-テーブルメタデータの最新の変更時刻。 - `dependencies_database` (Array(String))-データベースの依存関係。 -- `dependencies_table` (Array(String))-テーブルの依存関係 ([MaterializedView](../engines/table-engines/special/materializedview.md) 現在のテーブルに基づくテーブル)。 +- `dependencies_table` (Array(String))-テーブルの依存関係 ([マテリアライズドビュー](../engines/table-engines/special/materializedview.md) 現在のテーブルに基づくテーブル）。 - `create_table_query` (String)-テーブルの作成に使用されたクエリ。 @@ -950,34 +1021,34 @@ WHERE name in ('Kafka', 'MergeTree', 'ReplicatedCollapsingMergeTree') - `partition_key` (String)-テーブルで指定されたパーティションキー式。 -- `sorting_key` (String)-テーブルで指定された並べ替えキー式。 +- `sorting_key` (String)-テーブルで指定されたソートキー式。 - `primary_key` (String)-テーブルで指定された主キー式。 - `sampling_key` (String)-テーブルで指定されたサンプリングキー式。 -- `storage_policy` (文字列)-ストレージポリシー: +- `storage_policy` (String)-ストレージポリシー: - - [MergeTree](../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes) + - [メルゲツリー](../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes) - [分散](../engines/table-engines/special/distributed.md#distributed) -- `total_rows` (Nullable(UInt64))-テーブル内の正確な行数をすばやく決定できる場合は、行の総数。 `Null` （含むunderying `Buffer` テーブル）。 +- `total_rows` (Nullable(UInt64))-テーブル内の正確な行数をすばやく決定できる場合は、行の合計数です。 `Null` （アンダーイングを含む `Buffer` 表）。 -- `total_bytes` (Nullable(UInt64))-ストレージ上のテーブルの正確なバイト数を迅速に決定できる場合は、合計バイト数。 `Null` (**しない** を含みます)。 +- `total_bytes` (Nullable(UInt64))-ストレージ上のテーブルの正確なバイト数を迅速に決定できる場合は、合計バイト数。 `Null` (**ない** 基になるストレージを含む)。 - If the table stores data on disk, returns used space on disk (i.e. compressed). - - テーブルがメモリにデータを格納する場合,メモリ内の使用バイトの近似数を返します. + - テーブルがメモリにデータを格納している場合は、メモリ内の使用バイト数の概算を返します。 -その `system.tables` テーブルは `SHOW TABLES` クエリの実装。 +その `system.tables` テーブルはで使用されます `SHOW TABLES` クエリの実装。 -## システム。zookeeper {#system-zookeeper} +## システム飼育係 {#system-zookeeper} ZooKeeperが設定されていない場合、テーブルは存在しません。 できるデータを読み込んで飼育係クラスタで定義され、config. -クエリには次のものが必要です ‘path’ WHERE句の等価条件です。 これは、データを取得したい子供のためのZooKeeperのパスです。 +クエリには ‘path’ WHERE句の等価条件。 これは、データを取得する子供のためのZooKeeperのパスです。 クエリ `SELECT * FROM system.zookeeper WHERE path = '/clickhouse'` すべての子のデータを出力します。 `/clickhouse` ノード すべてのルートノードのデータを出力するには、path= ‘/’. -で指定されたパスの場合 ‘path’ 存在しない場合、例外がスローされます。 +指定されたパスの場合 ‘path’ 存在しない場合、例外がスローされます。 列: @@ -996,7 +1067,7 @@ ZooKeeperが設定されていない場合、テーブルは存在しません - `aversion` (Int32) — Number of changes to the ACL. - `ephemeralOwner` (Int64) — For ephemeral nodes, the ID of the session that owns this node. -例えば: +例: ``` sql SELECT * @@ -1041,35 +1112,35 @@ pzxid: 987021252247 path: /clickhouse/tables/01-08/visits/replicas ``` -## システム。突然変異 {#system_tables-mutations} +## システム突然変異 {#system_tables-mutations} -のテーブルについての情報が含まれて [突然変異](../sql-reference/statements/alter.md#alter-mutations) マージツリーテーブルとその進捗状況の。 各突然変異コマンドは、単一の行で表されます。 テーブルには次の列があります: +のテーブルについての情報が含まれて [突然変異](../sql-reference/statements/alter.md#alter-mutations) マージツリーテーブルとその進捗状況。 各突然変異コマンドは単一の行で表されます。 テーブルには次の列があります: **データ**, **テーブル** -突然変異が適用されたデータベースとテーブルの名前。 -**mutation\_id** -変異のID。 のための複製のテーブルこのIdに対応すznode名の `/mutations/` ZooKeeperのディレクトリ。 複雑でないテーブルの場合、Idはテーブルのデータディレクトリ内のファイル名に対応します。 +**mutation\_id** -突然変異のID。 レプリケートされたテーブルの場合、これらのIdは `/mutations/` 飼育係のディレクトリ。 未複製テーブルの場合、Idはテーブルのデータディレクトリ内のファイル名に対応します。 -**コマンド** -突然変異コマンド文字列（後のクエリの一部 `ALTER TABLE [db.]table`). +**コマンド** -突然変異コマンド文字列（後のクエリの部分 `ALTER TABLE [db.]table`). **create\_time** -この突然変異コマンドが実行のために提出されたとき。 -**ブロック番号。partition\_id**, **ブロック番号。番号** -入れ子になった列。 つまり、パーティションIDと、そのパーティションの変更によって取得されたブロック番号より小さい数のブロックを含むパーティションのみが変更さ 非複製のテーブル、ブロック番号の全ての仕切りがひとつのシーケンスです。 こないということを意味している変異体再現し、テーブルの列として展開しているのが記録するとともにシングルブロック番号の取得による突然変異が原因です。 +**ブロック番号partition\_id**, **ブロック番号番号** -入れ子になった列。 パーティションIDと、その変異によって取得されたブロック番号(各パーティションでは、そのパーティション内の変異によって取得されたブロック番号 非複製のテーブル、ブロック番号の全ての仕切りがひとつのシーケンスです。 こないということを意味している変異体再現し、テーブルの列として展開しているのが記録するとともにシングルブロック番号の取得による突然変異が原因です。 -**parts\_to\_do** -突然変異が終了するために突然変異する必要があるデータ部分の数。 +**parts\_to\_do** -突然変異が完了するために突然変異する必要があるデータ部分の数。 -**is\_done** -変異は終わったのか？ たとえそうであっても `parts_to_do = 0` レプリケートされたテーブルの変更は、変更する必要のある新しいデータ部分を作成する実行時間の長いINSERTのためにまだ行われていない可能性があり +**is\_done** -突然変異は？ たとえ `parts_to_do = 0` 変更する必要がある新しいデータパーツを作成する長時間実行されるINSERTのために、複製されたテーブルの突然変異がまだ行われていない可能性があり -一部の部分の変更に問題があった場合、次の列には追加情報が含まれています: +一部のパーツの変更に問題がある場合は、次の列に追加情報が含まれています: -**latest\_failed\_part** -突然変異できなかった最新の部分の名前。 +**latest\_failed\_part** -変異することができなかった最新の部分の名前。 -**latest\_fail\_time** -最も最近の部分変異失敗の時間。 +**latest\_fail\_time** -最も最近の部分突然変異の失敗の時間。 -**latest\_fail\_reason** -最も最近の部分の突然変異の失敗を引き起こした例外メッセージ。 +**latest\_fail\_reason** -最新の部品突然変異の失敗を引き起こした例外メッセージ。 -## システム。ディスク {#system_tables-disks} +## システムディスク {#system_tables-disks} -についての情報が含まれてディスクの定義に [サーバー構成](../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes_configure). +ディス [サーバー構成](../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes_configure). 列: @@ -1079,9 +1150,9 @@ path: /clickhouse/tables/01-08/visits/replicas - `total_space` ([UInt64](../sql-reference/data-types/int-uint.md)) — Disk volume in bytes. - `keep_free_space` ([UInt64](../sql-reference/data-types/int-uint.md)) — Amount of disk space that should stay free on disk in bytes. Defined in the `keep_free_space_bytes` ディスク構成のパラメータ。 -## システム。ストレージ\_policies {#system_tables-storage_policies} +## システムストレージポリシー {#system_tables-storage_policies} -についての情報が含まれて保管方針の量を定義する [サーバー構成](../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes_configure). +ストレージポリシ [サーバー構成](../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes_configure). 列: @@ -1092,6 +1163,6 @@ path: /clickhouse/tables/01-08/visits/replicas - `max_data_part_size` ([UInt64](../sql-reference/data-types/int-uint.md)) — Maximum size of a data part that can be stored on volume disks (0 — no limit). - `move_factor` ([Float64](../sql-reference/data-types/float.md)) — Ratio of free disk space. When the ratio exceeds the value of configuration parameter, ClickHouse start to move data to the next volume in order. -ストレージポリシーに複数のボリュームが含まれている場合、各ボリュームの情報はテーブルの個々の行に格納されます。 +ストレージポリシーに複数のボリュームが含まれている場合は、各ボリュームの情報がテーブルの個々の行に格納されます。 [元の記事](https://clickhouse.tech/docs/en/operations/system_tables/) diff --git a/docs/ja/operations/tips.md b/docs/ja/operations/tips.md index e57d94b9b10..630d62730fd 100644 --- a/docs/ja/operations/tips.md +++ b/docs/ja/operations/tips.md @@ -1,15 +1,15 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 58 -toc_title: "\u4F7F\u7528\u306E\u63A8\u5968\u4E8B\u9805" +toc_title: "\u4F7F\u7528\u6CD5\u306E\u63A8\u5968\u4E8B\u9805" --- -# 使用の推奨事項 {#usage-recommendations} +# 使用法の推奨事項 {#usage-recommendations} -## CPUスケールガバナー {#cpu-scaling-governor} +## CPUのスケール知事 {#cpu-scaling-governor} -常に使用する `performance` スケーリング知事。 その `on-demand` スケーリング知事は、常に高い需要とはるかに悪い作品。 +常に `performance` スケーリング知事。 その `on-demand` スケーリング総裁の作品もないと常に高いです。 ``` bash $ echo 'performance' | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor @@ -17,36 +17,36 @@ $ echo 'performance' | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_gov ## CPUの制限 {#cpu-limitations} -プロセッサでの過熱を防止します。 使用 `dmesg` 過熱によりCPUのクロックレートが制限されているかどうかを確認します。 -の制限を設定することもできます外部のデータセンターです。 を使用することができ `turbostat` 負荷の下でそれを監視する。 +プロセッサでの過熱を防止します。 使用 `dmesg` cpuのクロックレートが過熱により制限されているかどうかを確認します。 +制限は、データセンターレベルで外部で設定することもできます。 以下を使用できます `turbostat` 負荷の下でそれを監視する。 ## RAM {#ram} -少量のデータ（最大-200gb圧縮）の場合は、データ量と同じくらいのメモリを使用するのが最善です。 -大量のデータと対話型（オンライン）クエリを処理する場合は、ホットデータサブセットがページのキャッシュに収まるように、妥当な量のram（128gb以上）を使 -でもデータ量の50tbサーバ用のもの128gb ramを大幅に向上するクエリの性能に比べて64gbにサンプルがあります。 +少量のデータ（最大200GB圧縮）の場合は、データ量と同じくらいのメモリを使用することをお勧めします。 +大量のデータの場合、および対話型(オンライン)クエリを処理する場合は、ホットデータサブセットがページのキャッシュに収まるように、妥当な量のRAM(128GB +でもデータ量の50TBサーバ用のもの128GB RAMを大幅に向上するクエリの性能に比べて64GBにサンプルがあります。 -Overcommitを無効にしないでください。 を値 `cat /proc/sys/vm/overcommit_memory` 0または1である必要があります。 走れ。 +オーバーコミットを無効にしません。 値 `cat /proc/sys/vm/overcommit_memory` 0または1にする必要があります。 走れ。 ``` bash $ echo 0 | sudo tee /proc/sys/vm/overcommit_memory ``` -## ヒュージページ {#huge-pages} +## 巨大なページ {#huge-pages} -常に透明な巨大ページを無効にします。 これはメモリアロケータに干渉し、パフォーマンスが大幅に低下します。 +常に透明な巨大ページを無効にします。 これはメモリアロケータと干渉し、パフォーマンスが大幅に低下します。 ``` bash $ echo 'never' | sudo tee /sys/kernel/mm/transparent_hugepage/enabled ``` -使用 `perf top` メモリ管理のためにカーネルに費やされた時間を監視する。 -永続的なヒュージページも割り当てる必要はありません。 +使用 `perf top` メモリ管理のためにカーネルで費やされた時間を監視する。 +永続的な巨大なページも割り当てる必要はありません。 -## 格納サブシステム {#storage-subsystem} +## Storageサブシステム {#storage-subsystem} -予算でssdを使用できる場合は、ssdを使用してください。 -そうでない場合は、hddを使用します。 sata hdd7200rpmが実行されます。 +予算でSSDを使用できる場合は、SSDを使用してください。 +そうでない場合は、HDDを使用します。 SATA Hdd7200RPMが行います。 優先のサーバー地域のハードディスク上に小さな複数のサーバーが付属ディスクが増す。 ものの保存アーカイブでクエリー、棚します。 @@ -54,29 +54,29 @@ $ echo 'never' | sudo tee /sys/kernel/mm/transparent_hugepage/enabled ## RAID {#raid} HDDを使用する場合は、RAID-10、RAID-5、RAID-6またはRAID-50を組み合わせることができます。 -Linuxでは、ソフトウェアRAIDが優れている（と `mdadm`). LVMの使用はお勧めしません。 -RAID-10を作成するときは、以下を選択します。 `far` レイアウト。 -予算が許せば、raid-10を選択します。 +Linuxでは、ソフトウェアRAIDが優れています（ `mdadm`). LVMの使用はお勧めしません。 +RAID-10を作成するときは、 `far` レイアウト。 +予算が許せば、RAID-10を選択します。 -4台以上のディスクがある場合は、raid-6(優先)またはraid-50を使用します(raid-5の代わりに使用します)。 -RAID-5、RAID-6またはRAID-50を使用する場合、デフォルト値は通常最良の選択ではないので、常にstripe\_cache\_sizeを増加させます。 +4つ以上のディスクがある場合は、RAID-6(優先)またはRAID-50を使用してください。 +RAID-5、RAID-6、またはRAID-50を使用する場合は、常にstripe\_cache\_sizeを増やしてください。 ``` bash $ echo 4096 | sudo tee /sys/block/md2/md/stripe_cache_size ``` -次の式を使用して、デバイスの数とブロックサイズから正確な数を計算します: `2 * num_devices * chunk_size_in_bytes / 4096`. +式を使用して、デバイスの数とブロックサイズから正確な数を計算します: `2 * num_devices * chunk_size_in_bytes / 4096`. -1024kbのブロックサイズは、すべてのraid構成で十分です。 +すべてのRAID構成では、1024KBのブロックサイズで十分です。 ないセットのブロックサイズでは多すぎます。 -SSDにRAID-0を使用できます。 -に関わらずraidの利用、使用複製のためのデータです。 +SSDではRAID-0を使用できます。 +に関わらずRAIDの利用、使用複製のためのデータです。 -長いキューでncqを有効にします。 hddの場合はcfqスケジューラを選択し、ssdの場合はnoopを選択します。 減らしてはいけない ‘readahead’ 設定。 -のためのハードディスク（hdd）を、書き込みます。 +長いキューでNCQを有効にします。 HDDの場合はCFQスケジューラを選択し、SSDの場合はnoopを選択します。 減らさないで下さい ‘readahead’ 設定。 +HDDの場合、ライトキャッシュを有効にします。 -## ファイル {#file-system} +## ファイルシス {#file-system} Ext4は最も信頼性の高いオプションです。 マウントオプションの設定 `noatime, nobarrier`. XFSも適していますが、ClickHouseで徹底的にテストされていません。 @@ -84,34 +84,34 @@ XFSも適していますが、ClickHouseで徹底的にテストされていま ## Linuxカーネル {#linux-kernel} -古いlinuxカーネルを使用しないでください。 +古いLinuxカーネルを使用しないでください。 -## ネットワーク {#network} +## ネット {#network} -IPv6を使用している場合は、ルートキャッシュのサイズを大きくします。 -3.2より前のlinuxカーネルでは、ipv6の実装に多くの問題がありました。 +IPv6を使用している場合は、ルートキャッシュのサイズを増やします。 +3.2より前のLinuxカーネルには、IPv6実装に関して多くの問題がありました。 -可能な場合は、少なくとも10gbのネットワークを使用します。 1gbも動作しますが、数十テラバイトのデータを含むレプリカにパッチを適用したり、大量の中間データを含む分散クエリを処理する場合は、さらに悪 +可能であれば、10GB以上のネットワークを使用してください。 1Gbも動作しますが、数十テラバイトのデータを使用してレプリカにパッチを適用する場合や、大量の中間データを使用して分散クエリを処理する場合 -## ZooKeeper {#zookeeper} +## 飼育係 {#zookeeper} -おそらく既にzookeeperを他の目的で使用しているでしょう。 それがまだ過負荷になっていない場合は、zookeeperと同じインストールを使用できます。 +おそらく既に他の目的のためにZooKeeperを使用しています。 まだ過負荷になっていない場合は、ZooKeeperと同じインストールを使用できます。 -It’s best to use a fresh version of ZooKeeper – 3.4.9 or later. The version in stable Linux distributions may be outdated. +It's best to use a fresh version of ZooKeeper – 3.4.9 or later. The version in stable Linux distributions may be outdated. -異なるzookeeperクラスタ間でデータを転送するために手動で記述されたスクリプトを使用することはありません。 決して使用 “zkcopy” 同じ理由でユーティリティ：https://github.com/ksprojects/zkcopy/issues/15 +異なるZooKeeperクラスター間でデータを転送するには、手動で書かれたスクリプトを使用しないでください。 決して使用しない “zkcopy” 同じ理由でユーティリティ：https://github.com/ksprojects/zkcopy/issues/15 -既存のzookeeperクラスターを二つに分割したい場合、正しい方法はレプリカの数を増やし、それを二つの独立したクラスターとして再構成することです。 +既存のZooKeeperクラスタを二つに分割する場合、正しい方法は、そのレプリカの数を増やし、それを二つの独立したクラスタとして再構成することです。 -ClickHouseと同じサーバーでZooKeeperを実行しないでください。 で飼育係が非常に敏感なために時間遅れとClickHouseを利用することも可能で利用可能なすべてシステム資源です。 +ClickHouseと同じサーバー上でZooKeeperを実行しないでください。 で飼育係が非常に敏感なために時間遅れとClickHouseを利用することも可能で利用可能なすべてシステム資源です。 -デフォルトの設定では、zookeeperは時限爆弾です: +デフォルト設定では、飼育係は時限爆弾です: -> ZooKeeperサーバーは、デフォルト設定（autopurgeを参照）を使用するときに古いスナップショットやログからファイルを削除することはありません。 +> ZooKeeperサーバーは、デフォルト設定を使用するときに古いスナップショットとログからファイルを削除しません(autopurgeを参照)。 -この爆弾は取り除かれなければならない +この爆弾は取り除かなければならない -以下のzookeeper（3.5.1）設定はyandexで使用されています。月のメトリカの生産環境20,2017: +以下の飼育係（3.5.1）設定はYandexで使用されています。月のようMetricaの生産環境20,2017: 動物園cfg: diff --git a/docs/ja/operations/troubleshooting.md b/docs/ja/operations/troubleshooting.md index b4169a746b6..989e30eb5f2 100644 --- a/docs/ja/operations/troubleshooting.md +++ b/docs/ja/operations/troubleshooting.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 46 -toc_title: "\u30C8\u30E9\u30D6\u30EB" +toc_title: "\u30C8\u30E9\u30D6\u30EB\u30B7" --- -# トラブル {#troubleshooting} +# トラブルシ {#troubleshooting} - [設置](#troubleshooting-installation-errors) - [サーバーへの接続](#troubleshooting-accepts-no-connections) @@ -14,10 +14,10 @@ toc_title: "\u30C8\u30E9\u30D6\u30EB" ## 設置 {#troubleshooting-installation-errors} -### Apt-getでClickhouseリポジトリからDebパッケージを取得できません {#you-cannot-get-deb-packages-from-clickhouse-repository-with-apt-get} +### Apt-getではClickHouseリポジトリからDebパッケージを取得できません {#you-cannot-get-deb-packages-from-clickhouse-repository-with-apt-get} -- ファイア -- できない場合はアクセスリポジトリのために、何らかの理由でダウンロードパッケージに記載のとおり [はじめに](../getting-started/index.md) を使用して手動でインストールします。 `sudo dpkg -i ` 司令部 また、必要になります `tzdata` パッケージ。 +- ファイアウォールの設定 +- できない場合はアクセスリポジトリのために、何らかの理由でダウンロードパッケージに記載のとおり [はじめに](../getting-started/index.md) 記事とを使用して手動でインストール `sudo dpkg -i ` コマンド また、必要になります `tzdata` パッケージ。 ## サーバーへの接続 {#troubleshooting-accepts-no-connections} @@ -26,9 +26,9 @@ toc_title: "\u30C8\u30E9\u30D6\u30EB" - サーバーが実行されていません。 - 想定外または誤った設定パラメータ。 -### サーバーが実行中でない {#server-is-not-running} +### サーバの実行中に {#server-is-not-running} -**サーバーがrunnnigかどうかチェック** +**サーバーがrunnnigかどうかを確認する** コマンド: @@ -44,7 +44,7 @@ $ sudo service clickhouse-server start **ログの確認** -のメインログ `clickhouse-server` にある `/var/log/clickhouse-server/clickhouse-server.log` デフォルトでは。 +のメインログ `clickhouse-server` である `/var/log/clickhouse-server/clickhouse-server.log` デフォルトでは。 サーバーが正常に起動した場合は、文字列が表示されます: @@ -63,7 +63,7 @@ $ sudo service clickhouse-server start Application: starting up. ``` -次のインスタンスを起動しようとすると、 `clickhouse-server` サーバーには、次のログが表示されます: +あなたが第二のインスタンスを起動しようとすると `clickhouse-server` サーバーには、次のログが表示されます: ``` text 2019.01.11 15:25:11.151730 [ 1 ] {} : Starting ClickHouse 19.1.0 with revision 54413 @@ -81,13 +81,13 @@ Revision: 54413 **システムを参照。dログ** -有用な情報が見つからない場合は `clickhouse-server` ログがないか、ログがない場合は、次のように表示できます `system.d` コマンドを使用したログ: +で有用な情報が見つからない場合 `clickhouse-server` ログまたはログがない場合は、表示できます `system.d` コマンドを使用したログ: ``` bash $ sudo journalctl -u clickhouse-server ``` -**インタラクティブモードでclickhouse-serverを起動** +**Clickhouse-serverを対話モードで起動する** ``` bash $ sudo -u clickhouse /usr/bin/clickhouse-server --config-file /etc/clickhouse-server/config.xml @@ -95,19 +95,19 @@ $ sudo -u clickhouse /usr/bin/clickhouse-server --config-file /etc/clickhouse-se このコマ このモードでは `clickhouse-server` 版画のすべてのイベントメッセージです。 -### 構成変数 {#configuration-parameters} +### 構成パラメータ {#configuration-parameters} チェック: -- Dockerの設定。 +- ドッカーの設定。 - IPv6ネットワークのDockerでClickHouseを実行する場合は、次のことを確認してください `network=host` 設定されています。 + DockerでClickhouseをIPv6ネットワークで実行する場合は、次のことを確認してください `network=host` 設定されています。 - エンドポイント設定。 - チェック [listen\_host](server-configuration-parameters/settings.md#server_configuration_parameters-listen_host) と [tcp\_portgenericname](server-configuration-parameters/settings.md#server_configuration_parameters-tcp_port) 設定。 + チェック [listen\_host](server-configuration-parameters/settings.md#server_configuration_parameters-listen_host) と [tcp\_port](server-configuration-parameters/settings.md#server_configuration_parameters-tcp_port) 設定。 - ClickHouseサーバーを受け入れlocalhostの接続のみによるデフォルトです。 + ClickHouse serverはデフォルトでのみlocalhost接続を受け入れます。 - HTTPプロトコル設定。 @@ -118,9 +118,9 @@ $ sudo -u clickhouse /usr/bin/clickhouse-server --config-file /etc/clickhouse-se チェック: - その [tcp\_port\_secure](server-configuration-parameters/settings.md#server_configuration_parameters-tcp_port_secure) 設定。 - - の設定 [SSL sertificates](server-configuration-parameters/settings.md#server_configuration_parameters-openssl). + - の設定 [SSLセルティクス](server-configuration-parameters/settings.md#server_configuration_parameters-openssl). - 適切なパラメータを接続 たとえば、以下を使用します `port_secure` 変数との `clickhouse_client`. + 適切なパラメータを接続 たとえば、 `port_secure` 変数との `clickhouse_client`. - ユーザー設定。 @@ -128,19 +128,19 @@ $ sudo -u clickhouse /usr/bin/clickhouse-server --config-file /etc/clickhouse-se ## クエリ処理 {#troubleshooting-does-not-process-queries} -ClickHouseがクエリを処理できない場合は、クライアントにエラーの説明を送信します。 で `clickhouse-client` コンソールにエラーの説明が表示されます。 HTTPインターフェイスを使用している場合、ClickHouseはレスポンス本文にエラーの説明を送信します。 例えば: +ClickHouseは、クエリを処理できない場合は、クライアントにエラーの説明を送信します。 で `clickhouse-client` コンソールにエラーの説明が表示されます。 HTTPインターフェイスを使用している場合、ClickHouseは応答本文にエラーの説明を送信します。 例えば: ``` bash $ curl 'http://localhost:8123/' --data-binary "SELECT a" Code: 47, e.displayText() = DB::Exception: Unknown identifier: a. Note that there are no tables (FROM clause) in your query, context: required_names: 'a' source_tables: table_aliases: private_aliases: column_aliases: public_columns: 'a' masked_columns: array_join_columns: source_columns: , e.what() = DB::Exception ``` -あなたが始めるなら `clickhouse-client` と `stack-trace` パラメータ、ClickHouseは、エラーの説明とサーバースタックトレースを返します。 +あなたが開始した場合 `clickhouse-client` と `stack-trace` パラメータClickHouseは、エラーの説明を含むサーバースタックトレースを返します。 -あるいは、メッセージが壊れて接続します。 この場合、クエリを繰り返すことができます。 クエリを実行するたびに接続が切断された場合は、サーバーログにエラーがないか確認します。 +あるいは、メッセージが壊れて接続します。 この場合、クエリを繰り返すことができます。 クエリを実行するたびに接続が切断された場合は、サーバーログでエラーを確認します。 ## クエリ処理の効率 {#troubleshooting-too-slow} -だがclickhouseでもゆっくりが必要にプロファイルをサーバーに負荷をかける資源とネットワークのためのご質問. +ClickHouseの動作が遅すぎる場合は、クエリのサーバーリソースとネットワークの負荷をプロファイルする必要があります。 -で利用できますclickhouse-ベンチマークユーティリティプます。 これは、毎秒処理されるクエリの数、毎秒処理される行の数、およびクエリの処理時間の百分位数を示します。 +Clickhouse-benchmarkユーティリテ 毎秒処理されたクエリの数、毎秒処理された行数、およびクエリ処理時間の百分位数が表示されます。 diff --git a/docs/ja/operations/update.md b/docs/ja/operations/update.md index e68ed4b4500..8eff7d6969a 100644 --- a/docs/ja/operations/update.md +++ b/docs/ja/operations/update.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 47 -toc_title: "\u30AF\u30EA\u30C3\u30AF\u30CF\u30A6\u30B9\u66F4\u65B0" +toc_title: "\u30AF\u30EA\u30C3\u30AF\u30CF\u30A6\u30B9\u306E\u66F4\u65B0" --- -# クリックハウス更新 {#clickhouse-update} +# クリックハウスの更新 {#clickhouse-update} -まclickhouse設置されたからdebパッケージ、以下のコマンドを実行し、サーバー: +ClickHouseがdebパッケージからインストールさ: ``` bash $ sudo apt-get update @@ -15,6 +15,6 @@ $ sudo apt-get install clickhouse-client clickhouse-server $ sudo service clickhouse-server restart ``` -推奨されるdebパッケージ以外のものを使用してclickhouseをインストールした場合は、適切な更新方法を使用します。 +推奨されるdebパッケージ以外のものを使用してClickHouseをインストールした場合は、適切な更新方法を使用します。 -ClickHouseは分散updateをサポートしていません。 この操作は、個別のサーバーごとに連続して実行する必要があります。 クラスター上のすべてのサーバーを同時に更新しないでください。 +ClickHouseは分散updateをサポートしていません。 操作は、個別のサーバーごとに連続して実行する必要があります。 クラスター上のすべてのサーバーを同時に更新しないでください。 diff --git a/docs/ja/operations/utilities/clickhouse-benchmark.md b/docs/ja/operations/utilities/clickhouse-benchmark.md index eeb90b0a7b5..1a099b78ea0 100644 --- a/docs/ja/operations/utilities/clickhouse-benchmark.md +++ b/docs/ja/operations/utilities/clickhouse-benchmark.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 61 toc_title: "clickhouse-\u30D9\u30F3\u30C1\u30DE\u30FC\u30AF" --- # clickhouse-ベンチマーク {#clickhouse-benchmark} -ClickHouseサーバーに接続し、指定されたクエリを繰り返し送信します。 +ClickHouseサーバーに接続し、指定したクエリを繰り返し送信します。 構文: @@ -28,7 +28,7 @@ SELECT * FROM system.numbers LIMIT 10000000 SELECT 1 ``` -次に、このファイルを次の標準入力に渡します `clickhouse-benchmark`. +次に、このファイルを標準入力に渡します。 `clickhouse-benchmark`. ``` bash clickhouse-benchmark [keys] < queries_file @@ -38,27 +38,27 @@ clickhouse-benchmark [keys] < queries_file - `-c N`, `--concurrency=N` — Number of queries that `clickhouse-benchmark` 同時に送信します。 デフォルト値:1。 - `-d N`, `--delay=N` — Interval in seconds between intermediate reports (set 0 to disable reports). Default value: 1. -- `-h WORD`, `--host=WORD` — Server host. Default value: `localhost`. のための [比較モード](#clickhouse-benchmark-comparison-mode) 複数を使用できます `-h` 鍵を -- `-p N`, `--port=N` — Server port. Default value: 9000. For the [比較モード](#clickhouse-benchmark-comparison-mode) 複数を使用できます `-p` 鍵を +- `-h WORD`, `--host=WORD` — Server host. Default value: `localhost`. のために [比較モード](#clickhouse-benchmark-comparison-mode) 複数を使用できます `-h` 鍵だ +- `-p N`, `--port=N` — Server port. Default value: 9000. For the [比較モード](#clickhouse-benchmark-comparison-mode) 複数を使用できます `-p` 鍵だ - `-i N`, `--iterations=N` — Total number of queries. Default value: 0. - `-r`, `--randomize` — Random order of queries execution if there is more then one input query. - `-s`, `--secure` — Using TLS connection. -- `-t N`, `--timelimit=N` — Time limit in seconds. `clickhouse-benchmark` 指定された制限時間に達すると、クエリの送信を停止します。 デフォルト値:0(制限時間は無効)。 -- `--confidence=N` — Level of confidence for T-test. Possible values: 0 (80%), 1 (90%), 2 (95%), 3 (98%), 4 (99%), 5 (99.5%). Default value: 5. In the [比較モード](#clickhouse-benchmark-comparison-mode) `clickhouse-benchmark` を実行します。 [独立した二sample学生のt-テスト](https://en.wikipedia.org/wiki/Student%27s_t-test#Independent_two-sample_t-test) 二つの分布が選択した信頼度と異ならないかどうかをテストします。 +- `-t N`, `--timelimit=N` — Time limit in seconds. `clickhouse-benchmark` 指定された時間制限に達すると、クエリの送信を停止します。 デフォルト値:0(制限時間無効)。 +- `--confidence=N` — Level of confidence for T-test. Possible values: 0 (80%), 1 (90%), 2 (95%), 3 (98%), 4 (99%), 5 (99.5%). Default value: 5. In the [比較モード](#clickhouse-benchmark-comparison-mode) `clickhouse-benchmark` を実行します。 [独立した二つのサンプル学生のtテスト](https://en.wikipedia.org/wiki/Student%27s_t-test#Independent_two-sample_t-test) これらの分布が選択された信頼度と異ならないかどうかを判定します。 - `--cumulative` — Printing cumulative data instead of data per interval. - `--database=DATABASE_NAME` — ClickHouse database name. Default value: `default`. -- `--json=FILEPATH` — JSON output. When the key is set, `clickhouse-benchmark` 指定したJSONファイルにレポートを出力します。 +- `--json=FILEPATH` — JSON output. When the key is set, `clickhouse-benchmark` 指定されたJSONファイルにレポートを出力します。 - `--user=USERNAME` — ClickHouse user name. Default value: `default`. - `--password=PSWD` — ClickHouse user password. Default value: empty string. -- `--stacktrace` — Stack traces output. When the key is set, `clickhouse-bencmark` 例外スタックトレースを出力します。 +- `--stacktrace` — Stack traces output. When the key is set, `clickhouse-bencmark` 出力スタックトレースの例外をスローしました。 - `--stage=WORD` — Query processing stage at server. ClickHouse stops query processing and returns answer to `clickhouse-benchmark` 指定された段階で。 可能な値: `complete`, `fetch_columns`, `with_mergeable_state`. デフォルト値: `complete`. - `--help` — Shows the help message. -あなたがいくつかを適用したい場合 [設定](../../operations/settings/index.md) クエリの場合は、キーとして渡します `--= SETTING_VALUE`. 例えば, `--max_memory_usage=1048576`. +あなたはいくつかを適用したい場合 [設定](../../operations/settings/index.md) クエリの場合は、キーとして渡します `--= SETTING_VALUE`. 例えば, `--max_memory_usage=1048576`. ## 出力 {#clickhouse-benchmark-output} -デフォルトでは, `clickhouse-benchmark` それぞれのレポート `--delay` 間隔。 +既定では, `clickhouse-benchmark` 各レポート `--delay` インターバル レポートの例: @@ -83,29 +83,29 @@ localhost:9000, queries 10, QPS: 6.772, RPS: 67904487.440, MiB/s: 518.070, resul 99.990% 0.150 sec. ``` -このレポートでは、: +このレポートでは: -- その中のクエリの数 `Queries executed:` フィールド。 +- クエリの数 `Queries executed:` フィールド -- ステータスストリングを含む(順): +- ステータス文字列を含む(順): - ClickHouseサーバーのエンドポイント。 - 処理されたクエリの数。 - - QPS:QPS:クエリサーバーは、指定された期間に毎秒何回実行されたか `--delay` 引数。 - - RPS:指定された期間にサーバが毎秒読み込んだ行数 `--delay` 引数。 - - MiB/s:指定された期間中に毎秒読み取られるmebibytesサーバーの数 `--delay` 引数。 - - result RPS:サーバーによって指定された期間における秒あたりのクエリの結果に配置された行数 `--delay` 引数。 - - どのように多くのmebibytesは秒あたりのクエリの結果にサーバーによって配置されます。 `--delay` 引数。 + - QPS:QPS:で指定された期間に毎秒実行されるクエリサーバーの数 `--delay` 引数。 + - RPS:サーバーが指定された期間に毎秒読み取った行数 `--delay` 引数。 + - MiB/s:指定された期間に毎秒読み取られるmebibytesサーバーの数 `--delay` 引数。 + - 結果RPS:サーバーによって指定された期間内にクエリの結果に対して毎秒どのくらいの行が配置されますか `--delay` 引数。 + - クエリの結果に対してサーバによって指定された期間内に秒あたりのメビバイト数 `--delay` 引数。 - クエリの実行時間の百分位数。 ## 比較モード {#clickhouse-benchmark-comparison-mode} -`clickhouse-benchmark` ﾂつｨﾂ姪“ﾂつ”ﾂ債ﾂづｭﾂつｹ +`clickhouse-benchmark` ﾂつｨﾂ姪"ﾂつ"ﾂ債ﾂづｭﾂつｹ -利用の比較モードを指定し端のサーバーによるペア `--host`, `--port` 鍵を キーは、最初の引数リスト内の位置によって一致します `--host` は最初のものと一致します `--port` というように。 `clickhouse-benchmark` 両方のサーバーへの接続を確立し、クエリを送信します。 各クエリは、ランダムに選択されたサーバー宛。 結果は、サーバーごとに個別に表示されます。 +比較モードを使用するには、両方のサーバーのエンドポイントを `--host`, `--port` 鍵だ 引数リスト内の位置によって一致するキーは、最初の `--host` 最初のものと一致します `--port` など。 `clickhouse-benchmark` 両方のサーバーへの接続を確立し、クエリを送信します。 ランダムに選択されたサーバー宛の各クエリ。 結果は、各サーバーごとに個別に表示されます。 -## 例えば {#clickhouse-benchmark-example} +## 例 {#clickhouse-benchmark-example} ``` bash $ echo "SELECT * FROM system.numbers LIMIT 10000000 OFFSET 10000000" | clickhouse-benchmark -i 10 diff --git a/docs/ja/operations/utilities/clickhouse-copier.md b/docs/ja/operations/utilities/clickhouse-copier.md index bb230a03ca8..43e96647775 100644 --- a/docs/ja/operations/utilities/clickhouse-copier.md +++ b/docs/ja/operations/utilities/clickhouse-copier.md @@ -1,19 +1,19 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 59 -toc_title: "\u30AF\u30EA\u30C3\u30AF\u30CF\u30A6\u30B9\u30B3\u30D4\u30FC\u6A5F" +toc_title: "\u30AF\u30EA\u30C3\u30AF\u30CF\u30A6\u30B9-\u8907\u5199\u6A5F" --- -# クリックハウスコピー機 {#clickhouse-copier} +# クリックハウス-複写機 {#clickhouse-copier} コピーデータからのテーブルを一つクラスターテーブルの他の同クラスター 複数実行できます `clickhouse-copier` インスタンスの異なるサーバーを行う仕事です。 ZooKeeperはプロセスの同期に使用されます。 -始まることの後, `clickhouse-copier`: +開始後, `clickhouse-copier`: -- ZooKeeperに接続して受信する: +- ZooKeeperに接続し、受信します: - ジョブのコピー。 - コピージョブの状態。 @@ -24,11 +24,11 @@ toc_title: "\u30AF\u30EA\u30C3\u30AF\u30CF\u30A6\u30B9\u30B3\u30D4\u30FC\u6A5F" `clickhouse-copier` ZooKeeperの変更を追跡し、その場でそれらを適用します。 -ネットワークトラフィッ `clickhouse-copier` ソースデータがある同じサーバー上。 +ネットワークトラフィ `clickhouse-copier` ソースデータが配置されている同じサーバー上。 -## ランニングclickhouse-コピー機 {#running-clickhouse-copier} +## クリックハウスコピー機の実行 {#running-clickhouse-copier} -このユーティ: +このユーテ: ``` bash $ clickhouse-copier copier --daemon --config zookeeper.xml --task-path /task/path --base-dir /path/to/dir @@ -38,12 +38,12 @@ $ clickhouse-copier copier --daemon --config zookeeper.xml --task-path /task/pat - `daemon` — Starts `clickhouse-copier` デーモンモードで。 - `config` — The path to the `zookeeper.xml` ZooKeeperへの接続のためのパラメータを持つファイル。 -- `task-path` — The path to the ZooKeeper node. This node is used for syncing `clickhouse-copier` プロセスと格納タスク。 タスクは `$task-path/description`. +- `task-path` — The path to the ZooKeeper node. This node is used for syncing `clickhouse-copier` プロセスとタスクの保存。 タスクは `$task-path/description`. - `task-file` — Optional path to file with task configuration for initial upload to ZooKeeper. -- `task-upload-force` — Force upload `task-file` ノードが既に存在する場合でも。 -- `base-dir` — The path to logs and auxiliary files. When it starts, `clickhouse-copier` 作成 `clickhouse-copier_YYYYMMHHSS_` サブディレクトリ `$base-dir`. このパラメーターを省略すると、ディレクトリーは次の場所に作成されます `clickhouse-copier` 発売されました。 +- `task-upload-force` — Force upload `task-file` ノードが既に存在していても。 +- `base-dir` — The path to logs and auxiliary files. When it starts, `clickhouse-copier` 作成 `clickhouse-copier_YYYYMMHHSS_` サブディレクトリ `$base-dir`. このパラメーターを省略すると、ディレクトリは次の場所に作成されます `clickhouse-copier` 発売された。 -## 飼育係のフォーマット。xml {#format-of-zookeeper-xml} +## 飼育係の形式。xml {#format-of-zookeeper-xml} ``` xml @@ -62,7 +62,7 @@ $ clickhouse-copier copier --daemon --config zookeeper.xml --task-path /task/pat ``` -## コピータスクの設定 {#configuration-of-copying-tasks} +## コピータスクの構成 {#configuration-of-copying-tasks} ``` xml @@ -171,6 +171,6 @@ $ clickhouse-copier copier --daemon --config zookeeper.xml --task-path /task/pat ``` -`clickhouse-copier` の変更を追跡します。 `/task/path/description` そして、その場でそれらを適用します。 たとえば、次の値を変更すると `max_workers`、タスクを実行しているプロセスの数も変更されます。 +`clickhouse-copier` の変更を追跡します `/task/path/description` そしてその場でそれらを適用します。 たとえば、次の値を変更すると `max_workers`、タスクを実行しているプロセスの数も変更されます。 [元の記事](https://clickhouse.tech/docs/en/operations/utils/clickhouse-copier/) diff --git a/docs/ja/operations/utilities/clickhouse-local.md b/docs/ja/operations/utilities/clickhouse-local.md index 5f6bfc1c399..5f665bde19f 100644 --- a/docs/ja/operations/utilities/clickhouse-local.md +++ b/docs/ja/operations/utilities/clickhouse-local.md @@ -1,24 +1,24 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 60 -toc_title: "\uFF82\u3064\"\uFF82\u3065\u6309\u3064\uFF75\uFF82\uFF01" +toc_title: "\uFF82\u3064\uFF68\uFF82\u59EA\"\uFF82\u50B5\uFF82\u3064\uFF79" --- -# ﾂつ"ﾂづ按つｵﾂ！ {#clickhouse-local} +# ﾂつｨﾂ姪"ﾂ債ﾂつｹ {#clickhouse-local} -その `clickhouse-local` プログラムは、展開し、ClickHouseサーバーを構成することなく、ローカルファイルに高速処理を実行できます。 +その `clickhouse-local` プログラムで行う高速処理が地元のファイルを展開に合わせて構成されていますClickHouseサーバーです。 -データを受け入れを表すテーブル、クエリを利用して [クリックハウスsql方言](../../sql-reference/index.md). +データを受け入れを表すテーブル、クエリを利用して [クリックハウスSQL方言](../../sql-reference/index.md). -`clickhouse-local` ClickHouse serverと同じコアを使用するため、ほとんどの機能と同じフォーマットとテーブルエンジンをサポートします。 +`clickhouse-local` ClickHouse serverと同じコアを使用するため、ほとんどの機能と同じフォーマットとテーブルエンジンのセットをサポートします。 -デフォルトでは `clickhouse-local` 同じホスト上のデータにアクセスすることはできませんが、 `--config-file` 引数。 +既定では `clickhouse-local` なアクセスデータを同じホストにも対応して搭載サーバの設定を使用 `--config-file` 引数。 !!! warning "警告" - るおそれがあります。負荷生産サーバの設定に `clickhouse-local` 人為的なミスの場合、データが破損する可能性があるためです。 + 運用サーバーの構成をロードすることはお勧めしません `clickhouse-local` ヒューマンエラーの場合にデータが破損する可能性があるため。 -## 使い方 {#usage} +## 使用法 {#usage} 基本的な使用法: @@ -31,16 +31,16 @@ $ clickhouse-local --structure "table_structure" --input-format "format_of_incom - `-S`, `--structure` — table structure for input data. - `-if`, `--input-format` — input format, `TSV` デフォルトでは。 - `-f`, `--file` — path to data, `stdin` デフォルトでは。 -- `-q` `--query` — queries to execute with `;` デリメートルとして +- `-q` `--query` — queries to execute with `;` デリメーターとして。 - `-N`, `--table` — table name where to put output data, `table` デフォルトでは。 - `-of`, `--format`, `--output-format` — output format, `TSV` デフォルトでは。 - `--stacktrace` — whether to dump debug output in case of exception. - `--verbose` — more details on query execution. -- `-s` — disables `stderr` ログ記録。 +- `-s` — disables `stderr` ロギング - `--config-file` — path to configuration file in same format as for ClickHouse server, by default the configuration empty. - `--help` — arguments references for `clickhouse-local`. -また、より一般的に使用される各clickhouse構成変数の引数があります `--config-file`. +また、ClickHouse設定変数ごとに引数があります。 `--config-file`. ## 例 {#examples} @@ -51,7 +51,7 @@ Read 2 rows, 32.00 B in 0.000 sec., 5182 rows/sec., 80.97 KiB/sec. 3 4 ``` -前の例は次のようになります: +前の例と同じです: ``` bash $ echo -e "1,2\n3,4" | clickhouse-local -q "CREATE TABLE table (a Int64, b Int64) ENGINE = File(CSV, stdin); SELECT a, b FROM table; DROP TABLE table" @@ -60,7 +60,7 @@ Read 2 rows, 32.00 B in 0.000 sec., 4987 rows/sec., 77.93 KiB/sec. 3 4 ``` -次に、各unixユーザーのメモリユーザーを出力します: +次に、各Unixユーザのメモリユーザを出力しましょう: ``` bash $ ps aux | tail -n +2 | awk '{ printf("%s\t%s\n", $1, $4) }' | clickhouse-local -S "user String, mem Float64" -q "SELECT user, round(sum(mem), 2) as memTotal FROM table GROUP BY user ORDER BY memTotal DESC FORMAT Pretty" diff --git a/docs/ja/operations/utilities/index.md b/docs/ja/operations/utilities/index.md index 89b111da578..8dd37072b50 100644 --- a/docs/ja/operations/utilities/index.md +++ b/docs/ja/operations/utilities/index.md @@ -1,15 +1,15 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_folder_title: Utilities +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "\u30E6\u30FC\u30C6\u30A3\u30EA" toc_priority: 56 toc_title: "\u6982\u8981" --- -# ClickHouse明 {#clickhouse-utility} +# ClickHouseユーティリティ {#clickhouse-utility} -- [ﾂつ"ﾂづ按つｵﾂ！](clickhouse-local.md) — Allows running SQL queries on data without stopping the ClickHouse server, similar to how `awk` これを行います。 -- [クリックハウスコピー機](clickhouse-copier.md) — Copies (and reshards) data from one cluster to another cluster. +- [ﾂつｨﾂ姪"ﾂ債ﾂつｹ](clickhouse-local.md) — Allows running SQL queries on data without stopping the ClickHouse server, similar to how `awk` これを行います。 +- [クリックハウス-複写機](clickhouse-copier.md) — Copies (and reshards) data from one cluster to another cluster. - [clickhouse-ベンチマーク](clickhouse-benchmark.md) — Loads server with the custom queries and settings. [元の記事](https://clickhouse.tech/docs/en/operations/utils/) diff --git a/docs/ja/sql-reference/aggregate-functions/combinators.md b/docs/ja/sql-reference/aggregate-functions/combinators.md index 0a23d7c16c1..c5121f5259a 100644 --- a/docs/ja/sql-reference/aggregate-functions/combinators.md +++ b/docs/ja/sql-reference/aggregate-functions/combinators.md @@ -1,85 +1,162 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 37 -toc_title: "\u96C6\u8A08\u95A2\u6570\u306E\u30B3\u30F3\u30D3\u30CD\u30FC\u30BF" +toc_title: "\u30B3\u30F3\u30D3\u30CD\u30FC\u30BF" --- -# 集計関数のコンビネータ {#aggregate_functions_combinators} +# 集計関数コンビネータ {#aggregate_functions_combinators} -集計関数の名前には、それに接尾辞を付けることができます。 これにより、集計関数の動作方法が変更されます。 +集計関数の名前には、接尾辞を付加することができます。 これにより、集計関数の動作方法が変更されます。 ## -もし {#agg-functions-combinator-if} The suffix -If can be appended to the name of any aggregate function. In this case, the aggregate function accepts an extra argument – a condition (Uint8 type). The aggregate function processes only the rows that trigger the condition. If the condition was not triggered even once, it returns a default value (usually zeros or empty strings). -例: `sumIf(column, cond)`, `countIf(cond)`, `avgIf(x, cond)`, `quantilesTimingIf(level1, level2)(x, cond)`, `argMinIf(arg, val, cond)` というように。 +例: `sumIf(column, cond)`, `countIf(cond)`, `avgIf(x, cond)`, `quantilesTimingIf(level1, level2)(x, cond)`, `argMinIf(arg, val, cond)` など。 -条件付集計関数を使用すると、サブクエリを使用せずに複数の条件の集計を一度に計算できます。 `JOIN`例えば、Yandexの中。Metrica、条件付き集約関数は、セグメント比較機能を実装するために使用されます。 +条件集計関数を使用すると、サブクエリとサブクエリを使用せずに、複数の条件の集計を一度に計算できます。 `JOIN`例えば、Yandexの中。Metricaは、条件付き集計関数を使用してセグメント比較機能を実装します。 ## -配列 {#agg-functions-combinator-array} --arrayサフィックスは、任意の集計関数に追加できます。 この場合、aggregate関数は次の引数を取ります ‘Array(T)’ 代わりにタイプ（配列） ‘T’ 型引数。 集計関数が複数の引数を受け入れる場合、これは同じ長さの配列でなければなりません。 配列を処理する場合、aggregate関数は、すべての配列要素にわたって元の集計関数と同様に機能します。 +-Arrayサフィックスは、任意の集計関数に追加できます。 この場合、集計関数は ‘Array(T)’ 型(配列)の代わりに ‘T’ 引数を入力します。 集計関数が複数の引数を受け入れる場合、これは同じ長さの配列でなければなりません。 配列を処理する場合、集計関数はすべての配列要素にわたって元の集計関数と同様に機能します。 -例1: `sumArray(arr)` -すべてのすべての要素を合計します ‘arr’ 配列だ この例では、より簡単に書かれている可能性があります: `sum(arraySum(arr))`. +例1: `sumArray(arr)` -すべてのすべての要素を合計します ‘arr’ 配列。 この例では、より簡単に書くことができます: `sum(arraySum(arr))`. -例2: `uniqArray(arr)` – Counts the number of unique elements in all ‘arr’ 配列だ これは簡単な方法で行うことができます: `uniq(arrayJoin(arr))` しかし、それは常に追加することはできません ‘arrayJoin’ クエリに。 +例2: `uniqArray(arr)` – Counts the number of unique elements in all ‘arr’ 配列。 これは簡単な方法で行うことができます: `uniq(arrayJoin(arr))` しかし、常に追加することはできません ‘arrayJoin’ クエリに。 --Ifと-配列を組み合わせることができます。 しかし, ‘Array’ 第一だから、その ‘If’. 例: `uniqArrayIf(arr, cond)`, `quantilesTimingArrayIf(level1, level2)(arr, cond)`. この順序のために、 ‘cond’ 引数は配列ではありません。 +-Ifと-Arrayを組み合わせることができます。 しかし, ‘Array’ 最初に来なければならない ‘If’. 例: `uniqArrayIf(arr, cond)`, `quantilesTimingArrayIf(level1, level2)(arr, cond)`. この順序が原因で、 ‘cond’ 引数は配列ではありません。 ## -状態 {#agg-functions-combinator-state} -このコンビネーターを適用すると、集計関数は結果の値を返しません(たとえば、このコンビネーターの一意の値の数など)。 [uniq](reference.md#agg_function-uniq) の中間状態である。 `uniq`、これは一意の値の数を計算するためのハッシュテーブルです）。 これは `AggregateFunction(...)` これをさらなる処理に使用したり、テーブルに格納して後で集計を完了することができます。 +このコンビネータを適用すると、集計関数は結果の値（例えば、コンビネータの一意の値の数など）を返しません。 [uniq](reference.md#agg_function-uniq) の中間状態である。 `uniq`、これは一意の値の数を計算するためのハッシュテーブルです）。 これは `AggregateFunction(...)` 利用できるため、さらなる処理や保存のテーブルに仕上げを集計します。 これらの国は、利用: -- [ﾂつｨﾂ姪“ﾂつ”ﾂ債ﾂづｭﾂつｹ](../../engines/table-engines/mergetree-family/aggregatingmergetree.md) テーブルエンジン。 +- [AggregatingMergeTree](../../engines/table-engines/mergetree-family/aggregatingmergetree.md) テーブルエンジン。 - [finalizeAggregation](../../sql-reference/functions/other-functions.md#function-finalizeaggregation) 機能。 - [runningAccumulate](../../sql-reference/functions/other-functions.md#function-runningaccumulate) 機能。 -- [-マージ](#aggregate_functions_combinators-merge) コンビネータ -- [-MergeState](#aggregate_functions_combinators-mergestate) コンビネータ +- [-マージ](#aggregate_functions_combinators-merge) コンビネーター +- [-MergeState](#aggregate_functions_combinators-mergestate) コンビネーター ## -マージ {#aggregate_functions_combinators-merge} -このコンビネーターを適用すると、aggregate関数は中間の集約状態を引数として受け取り、状態を結合して集計を終了し、結果の値を返します。 +このコンビネータを適用すると、aggregate関数は中間集計状態を引数として受け取り、状態を結合して集計を終了し、結果の値を返します。 ## -MergeState {#aggregate_functions_combinators-mergestate} --mergeコンビネータと同じ方法で中間の集約状態をマージします。 しかし、結果の値を返すのではなく、-stateコンビネータに似た中間の集約状態を返します。 +-Mergeコンビネータと同じ方法で中間集計状態をマージします。 ただし、結果の値を返すのではなく、-Stateコンビネータに似た中間集計状態を返します。 ## -ForEach {#agg-functions-combinator-foreach} -テーブルの集計関数を、対応する配列項目を集約して結果の配列を返す配列の集計関数に変換します。 例えば, `sumForEach` 配列の場合 `[1, 2]`, `[3, 4, 5]`と`[6, 7]`結果を返します `[10, 13, 5]` 対応する配列項目を一緒に追加した後。 +テーブルの集計関数を、対応する配列項目を集計し、結果の配列を返す配列の集計関数に変換します。 例えば, `sumForEach` 配列の場合 `[1, 2]`, `[3, 4, 5]`と`[6, 7]`結果を返します `[10, 13, 5]` 対応する配列項目を一緒に追加した後。 ## -オルデフォルト {#agg-functions-combinator-ordefault} -集約する値が何もない場合は、集計関数の戻り値のデフォルト値を設定します。 +集計関数の動作を変更します。 + +集計関数に入力値がない場合、このコンビネータを使用すると、戻り値のデータ型のデフォルト値が返されます。 空の入力データを取ることができる集計関数に適用されます。 + +`-OrDefault` 他の組合せ器と使用することができる。 + +**構文** + +``` sql +OrDefault(x) +``` + +**パラメータ** + +- `x` — Aggregate function parameters. + +**戻り値** + +集計するものがない場合は、集計関数の戻り値の型の既定値を返します。 + +型は、使用される集計関数に依存します。 + +**例** + +クエリ: ``` sql SELECT avg(number), avgOrDefault(number) FROM numbers(0) ``` +結果: + ``` text ┌─avg(number)─┬─avgOrDefault(number)─┐ │ nan │ 0 │ └─────────────┴──────────────────────┘ ``` -## -オルヌル {#agg-functions-combinator-ornull} +また `-OrDefault` 他のコンビネータと併用できます。 集計関数が空の入力を受け入れない場合に便利です。 -塗りつぶし `null` 集計するものがない場合。 戻り列はnull可能になります。 +クエリ: ``` sql -SELECT avg(number), avgOrNull(number) FROM numbers(0) +SELECT avgOrDefaultIf(x, x > 10) +FROM +( + SELECT toDecimal32(1.23, 2) AS x +) ``` +結果: + ``` text -┌─avg(number)─┬─avgOrNull(number)─┐ -│ nan │ ᴺᵁᴸᴸ │ -└─────────────┴───────────────────┘ +┌─avgOrDefaultIf(x, greater(x, 10))─┐ +│ 0.00 │ +└───────────────────────────────────┘ ``` --OrDefaultと-OrNullは他のコンビネータと組み合わせることができます。 これは、集計関数が空の入力を受け入れない場合に便利です。 +## -オルヌル {#agg-functions-combinator-ornull} + +集計関数の動作を変更します。 + +このコンビネータは、集計関数の結果を [Null可能](../data-types/nullable.md) データ型。 集計関数に計算する値がない場合は、次の値が返されます [NULL](../syntax.md#null-literal). + +`-OrNull` 他の組合せ器と使用することができる。 + +**構文** + +``` sql +OrNull(x) +``` + +**パラメータ** + +- `x` — Aggregate function parameters. + +**戻り値** + +- 集計関数の結果は、次のように変換されます。 `Nullable` データ型。 +- `NULL`、集約するものがない場合。 + +タイプ: `Nullable(aggregate function return type)`. + +**例** + +追加 `-orNull` 集計関数の最後に。 + +クエリ: + +``` sql +SELECT sumOrNull(number), toTypeName(sumOrNull(number)) FROM numbers(10) WHERE number > 10 +``` + +結果: + +``` text +┌─sumOrNull(number)─┬─toTypeName(sumOrNull(number))─┐ +│ ᴺᵁᴸᴸ │ Nullable(UInt64) │ +└───────────────────┴───────────────────────────────┘ +``` + +また `-OrNull` 他のコンビネータと併用できます。 集計関数が空の入力を受け入れない場合に便利です。 + +クエリ: ``` sql SELECT avgOrNullIf(x, x > 10) @@ -89,15 +166,17 @@ FROM ) ``` +結果: + ``` text ┌─avgOrNullIf(x, greater(x, 10))─┐ │ ᴺᵁᴸᴸ │ └────────────────────────────────┘ ``` -## -リサンプル {#agg-functions-combinator-resample} +## -リサンプリング {#agg-functions-combinator-resample} -データをグループに分割し、それらのグループのデータを個別に集計できます。 グループは、ある列の値を間隔に分割することによって作成されます。 +データをグループに分割し、それらのグループ内のデータを個別に集計できます。 グループは、ある列の値を間隔に分割することによって作成されます。 ``` sql Resample(start, end, step)(, resampling_key) @@ -106,18 +185,18 @@ FROM **パラメータ** - `start` — Starting value of the whole required interval for `resampling_key` 値。 -- `stop` — Ending value of the whole required interval for `resampling_key` 値。 全体の間隔は含まれていません `stop` 値 `[start, stop)`. -- `step` — Step for separating the whole interval into subintervals. The `aggFunction` 実行されるそれぞれのsubintervals。 +- `stop` — Ending value of the whole required interval for `resampling_key` 値。 全体の区間は含まれていません `stop` 値 `[start, stop)`. +- `step` — Step for separating the whole interval into subintervals. The `aggFunction` これらの部分区間のそれぞれに対して独立に実行されます。 - `resampling_key` — Column whose values are used for separating data into intervals. -- `aggFunction_params` — `aggFunction` パラメータ。 +- `aggFunction_params` — `aggFunction` 変数。 **戻り値** - の配列 `aggFunction` 各サブインターバルの結果。 -**例えば** +**例** -考慮する `people` テーブルのデータ: +を考える `people` 次のデータを含むテーブル: ``` text ┌─name───┬─age─┬─wage─┐ @@ -130,9 +209,9 @@ FROM └────────┴─────┴──────┘ ``` -のは、その年齢の間隔にある人の名前を取得してみましょう `[30,60)` と `[60,75)`. 私たちは年齢の整数表現を使用しているので、私たちはで年齢を取得します `[30, 59]` と `[60,74]` 間隔。 +のは、その年齢の間隔にある人の名前を取得してみましょう `[30,60)` と `[60,75)`. 年齢に整数表現を使用するので、年齢を取得します `[30, 59]` と `[60,74]` 間隔。 -配列内の名前を集約するには、次のものを使用します [グルーパー](reference.md#agg_function-grouparray) 集計関数。 それは一つの議論を取る。 私たちの場合、それは `name` コラム その `groupArrayResample` 関数は `age` 年齢別に名前を集計する列。 必要な間隔を定義するために、 `30, 75, 30` への引数 `groupArrayResample` 機能。 +配列内の名前を集計するには、 [グルーパレイ](reference.md#agg_function-grouparray) 集計関数。 それは一つの引数を取ります。 私たちの場合、それは `name` 列。 その `groupArrayResample` 関数は、 `age` 年齢別に名前を集計する列。 必要な間隔を定義するために、 `30, 75, 30` の引数 `groupArrayResample` 機能。 ``` sql SELECT groupArrayResample(30, 75, 30)(name, age) FROM people @@ -144,11 +223,11 @@ SELECT groupArrayResample(30, 75, 30)(name, age) FROM people └───────────────────────────────────────────────┘ ``` -結果を考慮する。 +結果を考えてみましょう。 -`Jonh` 彼は若すぎるので、サンプルの外です。 他の人は、指定された年齢区間に従って配布されます。 +`Jonh` 彼は若すぎるので、サンプルの外にあります。 他の人は、指定された年齢間隔に従って分配されます。 -プラグインのインス数の合計人数とその平均賃金には、指定された年齢の間隔とします。 +今度は、指定された年齢間隔での人々の総数とその平均賃金を数えましょう。 ``` sql SELECT diff --git a/docs/ja/sql-reference/aggregate-functions/index.md b/docs/ja/sql-reference/aggregate-functions/index.md index 09234931f1f..bf1e47103ba 100644 --- a/docs/ja/sql-reference/aggregate-functions/index.md +++ b/docs/ja/sql-reference/aggregate-functions/index.md @@ -1,27 +1,27 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_folder_title: Aggregate Functions +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "\u96C6\u8A08\u95A2\u6570" toc_priority: 33 -toc_title: "\u5C0E\u5165" +toc_title: "\u306F\u3058\u3081\u306B" --- # 集計関数 {#aggregate-functions} -集計関数は、 [通常の](http://www.sql-tutorial.com/sql-aggregate-functions-sql-tutorial) 方法として期待されデータベースの専門家です。 +集計関数は、 [標準](http://www.sql-tutorial.com/sql-aggregate-functions-sql-tutorial) データベースの専門家が予想通りの方法。 ClickHouseはまた支えます: -- [パラメトリックに集計機能](parametric-functions.md#aggregate_functions_parametric) 列に加えて他のパラメータを受け入れる。 -- [Combinators](combinators.md#aggregate_functions_combinators)、集計関数の動作を変更します。 +- [パラメトリック集計関数](parametric-functions.md#aggregate_functions_parametric) 列に加えて他のパラメータを受け入れます。 +- [コンビネータ](combinators.md#aggregate_functions_combinators)、集計関数の動作を変更します。 -## NULLの場合の処理 {#null-processing} +## ヌル処理 {#null-processing} 集計中、すべて `NULL`sはスキップされます。 **例:** -この表を考慮する: +次の表を考えます: ``` text ┌─x─┬────y─┐ @@ -43,9 +43,9 @@ SELECT sum(y) FROM t_null_big │ 7 │ └────────┘ -その `sum` 関数の解釈 `NULL` として `0`. 特に、これは、関数がすべての値がある選択の入力を受け取った場合 `NULL` その後、結果は次のようになります `0`、ない `NULL`. +その `sum` 関数は解釈します `NULL` として `0`. 特に、これは、関数がすべての値が次のような選択範囲の入力を受け取った場合 `NULL` 結果は次のようになります `0` ない `NULL`. -今すぐ使用できます `groupArray` から配列を作成する関数 `y` 列: +今すぐ使用することができます `groupArray` から配列を作成する関数 `y` 列: ``` sql SELECT groupArray(y) FROM t_null_big @@ -57,6 +57,6 @@ SELECT groupArray(y) FROM t_null_big └───────────────┘ ``` -`groupArray` 含まれていません `NULL` 結果の配列です。 +`groupArray` 含まない `NULL` 結果の配列で。 [元の記事](https://clickhouse.tech/docs/en/query_language/agg_functions/) diff --git a/docs/ja/sql-reference/aggregate-functions/parametric-functions.md b/docs/ja/sql-reference/aggregate-functions/parametric-functions.md index 359fd8975a2..22d9dbe95ea 100644 --- a/docs/ja/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/ja/sql-reference/aggregate-functions/parametric-functions.md @@ -1,15 +1,15 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 38 -toc_title: "\u30D1\u30E9\u30E1\u30C8\u30EA\u30C3\u30AF\u96C6\u8A08\u95A2\u6570" +toc_title: "\u30D1\u30E9\u30E1\u30C8\u30EA\u30C3\u30AF" --- # パラメトリック集計関数 {#aggregate_functions_parametric} Some aggregate functions can accept not only argument columns (used for compression), but a set of parameters – constants for initialization. The syntax is two pairs of brackets instead of one. The first is for parameters, and the second is for arguments. -## ヒストグラム {#histogram} +## ヒス {#histogram} 適応ヒストグラムを計算します。 正確な結果を保証するものではありません。 @@ -17,12 +17,12 @@ Some aggregate functions can accept not only argument columns (used for compress histogram(number_of_bins)(values) ``` -関数は以下を使用します [ストリーミングの並列決定木アルゴリズム](http://jmlr.org/papers/volume11/ben-haim10a/ben-haim10a.pdf). ヒストグラムビンの境界は、新しいデータが関数に入ると調整されます。 一般的なケースでは、ビンの幅は等しくありません。 +関数は以下を使用します [ストリーミン](http://jmlr.org/papers/volume11/ben-haim10a/ben-haim10a.pdf). ヒストグラムビンの境界は、新しいデータが関数に入ると調整されます。 一般的なケースでは、ビンの幅は等しくありません。 **パラメータ** `number_of_bins` — Upper limit for the number of bins in the histogram. The function automatically calculates the number of bins. It tries to reach the specified number of bins, but if it fails, it uses fewer bins. -`values` — [式](../syntax.md#syntax-expressions) その結果、入力値が得られます。 +`values` — [式](../syntax.md#syntax-expressions) 結果として入力値が生成されます。 **戻り値** @@ -36,7 +36,7 @@ histogram(number_of_bins)(values) - `upper` — Upper bound of the bin. - `height` — Calculated height of the bin. -**例えば** +**例** ``` sql SELECT histogram(5)(number + 1) @@ -53,7 +53,7 @@ FROM ( └─────────────────────────────────────────────────────────────────────────┘ ``` -ヒストグラムを視覚化することができます [バー](../../sql-reference/functions/other-functions.md#function-bar) たとえば、関数: +ヒストグラムを視覚化するには [バー](../../sql-reference/functions/other-functions.md#function-bar) 関数、例えば: ``` sql WITH histogram(5)(rand() % 100) AS hist @@ -89,7 +89,7 @@ sequenceMatch(pattern)(timestamp, cond1, cond2, ...) ``` !!! warning "警告" - 同じ秒で発生するイベントは、結果に影響を与える未定義の順序でシーケンス内に置くことができます。 + 同じ秒で発生するイベントは、結果に影響を与える未定義の順序でシーケンス内に存在する可能性があります。 **パラメータ** @@ -97,27 +97,27 @@ sequenceMatch(pattern)(timestamp, cond1, cond2, ...) - `timestamp` — Column considered to contain time data. Typical data types are `Date` と `DateTime`. も利用できますの対応 [UInt](../../sql-reference/data-types/int-uint.md) データ型。 -- `cond1`, `cond2` — Conditions that describe the chain of events. Data type: `UInt8`. 最大32個の条件引数を渡すことができます。 この関数は、これらの条件で説明されているイベントのみを考慮します。 シーケンスに条件に記述されていないデータが含まれている場合、関数はそれらをスキップします。 +- `cond1`, `cond2` — Conditions that describe the chain of events. Data type: `UInt8`. 最大32個の条件引数を渡すことができます。 この関数は、これらの条件で説明されているイベントのみを考慮します。 シーケンスに条件で記述されていないデータが含まれている場合、関数はそれらをスキップします。 **戻り値** -- パターンが一致すれば、1。 -- 0、パターンが一致しない場合。 +- パターンが一致する場合は、1。 +- パターンが一致しない場合は0。 タイプ: `UInt8`. **パターン構文** -- `(?N)` — Matches the condition argument at position `N`. 条件には、 `[1, 32]` 範囲。 例えば, `(?1)` に渡された引数にマッチします。 `cond1` パラメータ。 +- `(?N)` — Matches the condition argument at position `N`. 条件は、 `[1, 32]` 範囲 例えば, `(?1)` に渡された引数と一致します。 `cond1` パラメータ。 -- `.*` — Matches any number of events. You don’t need conditional arguments to match this element of the pattern. +- `.*` — Matches any number of events. You don't need conditional arguments to match this element of the pattern. -- `(?t operator value)` — Sets the time in seconds that should separate two events. For example, pattern `(?1)(?t>1800)(?2)` 互いに1800秒を超えて発生するイベントに一致します。 任意の数は、当社が定めるインターネットを築くことです。 を使用することができ `>=`, `>`, `<`, `<=` 演算子。 +- `(?t operator value)` — Sets the time in seconds that should separate two events. For example, pattern `(?1)(?t>1800)(?2)` お互いから1800秒以上発生するイベントと一致します。 これらのイベントの間に任意の数のイベントを配置できます。 を使用することができます `>=`, `>`, `<`, `<=` 演算子。 **例** -のデータを考慮して下さい `t` テーブル: +のデータを考慮する `t` テーブル: ``` text ┌─time─┬─number─┐ @@ -139,7 +139,7 @@ SELECT sequenceMatch('(?1)(?2)')(time, number = 1, number = 2) FROM t └───────────────────────────────────────────────────────────────────────┘ ``` -この関数は、番号2が番号1に続くイベントチェーンを見つけました。 数字はイベントとして記述されていないため、3番をスキップしました。 この例で与えられたイベントチェーンを検索するときにこの番号を考慮に入れたい場合は、その条件を作成する必要があります。 +関数は、番号2が番号1に続くイベントチェーンを見つけました。 番号はイベントとして記述されていないため、それらの間の番号3をスキップしました。 例で与えられたイベントチェーンを検索するときにこの番号を考慮に入れたい場合は、それに対する条件を作成する必要があります。 ``` sql SELECT sequenceMatch('(?1)(?2)')(time, number = 1, number = 2, number = 3) FROM t @@ -151,7 +151,7 @@ SELECT sequenceMatch('(?1)(?2)')(time, number = 1, number = 2, number = 3) FROM └──────────────────────────────────────────────────────────────────────────────────────────┘ ``` -この場合、関数は、3番のイベントが1と2の間で発生したため、パターンに一致するイベントチェーンを見つけることができませんでした。 同じケースで4の条件をチェックした場合、シーケンスはパターンに一致します。 +この場合、番号3のイベントが1と2の間で発生したため、関数はパターンに一致するイベントチェーンを見つけることができませんでした。 同じケースで4番の条件をチェックした場合、シーケンスはパターンと一致します。 ``` sql SELECT sequenceMatch('(?1)(?2)')(time, number = 1, number = 2, number = 4) FROM t @@ -163,16 +163,16 @@ SELECT sequenceMatch('(?1)(?2)')(time, number = 1, number = 2, number = 4) FROM └──────────────────────────────────────────────────────────────────────────────────────────┘ ``` -**また見なさい** +**も参照。** -- [sequenceCount](#function-sequencecount) +- [シーケンスカウント](#function-sequencecount) ## sequenceCount(pattern)(time, cond1, cond2, …) {#function-sequencecount} -パターンに一致するイベントチェーンの数を数えます。 この関数は、重複しないイベントチェーンを検索します。 現在のチェーンが一致した後、次のチェーンの検索を開始します。 +パターンに一致したイベントチェーンの数を数えます。 この関数は、重複しないイベントチェーンを検索します。 現在のチェーンが一致した後、次のチェーンの検索を開始します。 !!! warning "警告" - 同じ秒で発生するイベントは、結果に影響を与える未定義の順序でシーケンス内に置くことができます。 + 同じ秒で発生するイベントは、結果に影響を与える未定義の順序でシーケンス内に存在する可能性があります。 ``` sql sequenceCount(pattern)(timestamp, cond1, cond2, ...) @@ -184,7 +184,7 @@ sequenceCount(pattern)(timestamp, cond1, cond2, ...) - `timestamp` — Column considered to contain time data. Typical data types are `Date` と `DateTime`. も利用できますの対応 [UInt](../../sql-reference/data-types/int-uint.md) データ型。 -- `cond1`, `cond2` — Conditions that describe the chain of events. Data type: `UInt8`. 最大32個の条件引数を渡すことができます。 この関数は、これらの条件で説明されているイベントのみを考慮します。 シーケンスに条件に記述されていないデータが含まれている場合、関数はそれらをスキップします。 +- `cond1`, `cond2` — Conditions that describe the chain of events. Data type: `UInt8`. 最大32個の条件引数を渡すことができます。 この関数は、これらの条件で説明されているイベントのみを考慮します。 シーケンスに条件で記述されていないデータが含まれている場合、関数はそれらをスキップします。 **戻り値** @@ -192,9 +192,9 @@ sequenceCount(pattern)(timestamp, cond1, cond2, ...) タイプ: `UInt64`. -**例えば** +**例** -のデータを考慮して下さい `t` テーブル: +のデータを考慮する `t` テーブル: ``` text ┌─time─┬─number─┐ @@ -207,7 +207,7 @@ sequenceCount(pattern)(timestamp, cond1, cond2, ...) └──────┴────────┘ ``` -数2は、それらの間の他の数字の任意の量と数1の後に発生した回数をカウント: +数2が数1の後に何回発生するかを数えます。: ``` sql SELECT sequenceCount('(?1).*(?2)')(time, number = 1, number = 2) FROM t @@ -219,21 +219,21 @@ SELECT sequenceCount('(?1).*(?2)')(time, number = 1, number = 2) FROM t └─────────────────────────────────────────────────────────────────────────┘ ``` -**また見なさい** +**も参照。** -- [sequenceMatch](#function-sequencematch) +- [シーケンスマッチ](#function-sequencematch) -## windowfunnelcomment {#windowfunnel} +## ウィンドウファンネル {#windowfunnel} -スライドタイムウィンドウでイベントチェーンを検索し、チェーンから発生したイベントの最大数を計算します。 +スライドタイムウィンドウ内のイベントチェーンを検索し、チェーンから発生したイベントの最大数を計算します。 -関数はアルゴリズムに従って動作します: +機能の動作に応じてアルゴリズム: -- この関数は、チェーン内の最初の条件をトリガーするデータを検索し、イベントカウンターを1に設定します。 これは、スライドウィンドウが始まる瞬間です。 +- この関数は、チェーン内の最初の条件をトリガーし、イベントカウンターを1に設定するデータを検索します。 これはスライディングウィンドウが始まる瞬間です。 -- だから、チェーンが順次内のウインドウのカウンタを増加されます。 イベントのシーケンスが中断された場合、カウンターは増分されません。 +- だから、チェーンが順次内のウインドウのカウンタを増加されます。 シーケンスのイベントに障害が発生、カウンターな増加されます。 -- データにさまざまな完了点で複数のイベントチェーンがある場合、関数は最長チェーンのサイズのみを出力します。 +- さまざまな完了ポイントでデータに複数のイベントチェーンがある場合、関数は最も長いチェーンのサイズのみを出力します。 **構文** @@ -244,28 +244,28 @@ windowFunnel(window, [mode])(timestamp, cond1, cond2, ..., condN) **パラメータ** - `window` — Length of the sliding window in seconds. -- `mode` -省略可能な引数です。 - - `'strict'` -とき `'strict'` windowFunnel()は、一意の値に対してのみ条件を適用します。 -- `timestamp` — Name of the column containing the timestamp. Data types supported: [日付](../../sql-reference/data-types/date.md), [DateTime](../../sql-reference/data-types/datetime.md#data_type-datetime) その他の符号なし整数型(timestampがサポートしているにもかかわらず `UInt64` 値はInt64最大値を超えることはできません.2^63-1）。 +- `mode` -これはオプションの引数です。 + - `'strict'` -とき `'strict'` 設定されている場合、windowFunnel()は一意の値に対してのみ条件を適用します。 +- `timestamp` — Name of the column containing the timestamp. Data types supported: [日付](../../sql-reference/data-types/date.md), [DateTime](../../sql-reference/data-types/datetime.md#data_type-datetime) その他の符号なし整数型（timestampがサポートしているにもかかわらず `UInt64` 値はInt64最大値を超えることはできません（2^63-1）。 - `cond` — Conditions or data describing the chain of events. [UInt8](../../sql-reference/data-types/int-uint.md). **戻り値** -スライディングタイムウィンドウ内のチェーンからの連続トリガー条件の最大数。 +スライディングタイムウィンドウ内のチェーンからの連続したトリガ条件の最大数。 選択内のすべてのチェーンが分析されます。 タイプ: `Integer`. -**例えば** +**例** -ユーザーが電話を選択してオンラインストアで二度購入するのに十分な期間が設定されているかどうかを判断します。 +ユーザーが電話を選択してオンラインストアで二度購入するのに十分な時間があるかどうかを判断します。 次の一連のイベントを設定します: -1. ユーザーがストアのアカウントにログインした場合 (`eventID = 1003`). -2. ユーザーは電話を検索します (`eventID = 1007, product = 'phone'`). +1. ユーザーがストアのアカウントにログインした (`eventID = 1003`). +2. ユーザーが電話を検索する (`eventID = 1007, product = 'phone'`). 3. ユーザーが注文した (`eventID = 1009`). -4. ユーザーが再び注文した (`eventID = 1010`). +4. ユーザーが再び注文しました (`eventID = 1010`). 入力テーブル: @@ -284,7 +284,7 @@ windowFunnel(window, [mode])(timestamp, cond1, cond2, ..., condN) └────────────┴─────────┴─────────────────────┴─────────┴─────────┘ ``` -ユーザーの距離を調べる `user_id` を介して得ることができるチェーンで期間で月-月の2019。 +ユーザーの距離を調べる `user_id` 2019年にチェーンを抜けることができた。 クエリ: @@ -315,10 +315,10 @@ ORDER BY level ASC ## 保持 {#retention} -関数は引数として1から32までの条件のセットを受け取ります。 `UInt8` るかどうかを示す一定の条件を満ためのイベントです。 -任意の条件を引数として指定することができます。 [WHERE](../../sql-reference/statements/select.md#select-where)). +この関数は、1から32までの型の引数の条件のセットを引数として受け取ります `UInt8` るかどうかを示す一定の条件を満ためのイベントです。 +任意の条件を引数として指定することができます [WHERE](../../sql-reference/statements/select/where.md#select-where)). -第一と第二が真であれば第二の結果は真であり、第一と第二が真であれば第三の結果は真である。 +最初の条件を除く条件は、ペアで適用されます：最初と第二が真であれば第二の結果は真になり、最初とfirdが真であれば第三の結果は真になります。 **構文** @@ -335,13 +335,13 @@ retention(cond1, cond2, ..., cond32); 1または0の配列。 - 1 — condition was met for the event. -- 0 — condition wasn’t met for the event. +- 0 — condition wasn't met for the event. タイプ: `UInt8`. -**例えば** +**例** -の計算の例を考えてみましょう `retention` サイトトラフィックを決定する機能。 +のは、計算の例を考えてみましょう `retention` サイトトラフィックを決定する機能。 **1.** Сreate a table to illustrate an example. @@ -402,7 +402,7 @@ SELECT * FROM retention_test └────────────┴─────┘ ``` -**2.** グループのユーザーによるユニークID `uid` を使用して `retention` 機能。 +**2.** ユーザを一意のIDでグループ化 `uid` を使用して `retention` 機能。 クエリ: @@ -468,22 +468,22 @@ FROM どこに: -- `r1`-2020-01-01の間にサイトを訪問したユニーク訪問者の数 `cond1` 条件）。 -- `r2`-2020-01-01から2020-01-02までの特定の期間にサイトを訪問したユニーク訪問者の数 (`cond1` と `cond2` 条件）。 -- `r3`-2020-01-01から2020-01-03までの特定の期間にサイトを訪問したユニーク訪問者の数 (`cond1` と `cond3` 条件）。 +- `r1`-2020-01-01中にサイトを訪問したユニークな訪問者の数（ `cond1` 条件）。 +- `r2`-2020-01-01から2020-01-02の間の特定の期間にサイトを訪問したユニーク訪問者の数 (`cond1` と `cond2` 条件）。 +- `r3`-2020-01-01から2020-01-03の間の特定の期間にサイトを訪問したユニーク訪問者の数 (`cond1` と `cond3` 条件）。 ## uniqUpTo(N)(x) {#uniquptonx} Calculates the number of different argument values ​​if it is less than or equal to N. If the number of different argument values is greater than N, it returns N + 1. -小さいnsの使用のために推薦される、10まで。 nの最大値は100です。 +小さいNsの使用のために推薦される、10まで。 Nの最大値は100です。 -集計関数の状態については、1+n\*に等しいメモリの量をバイトの一つの値のサイズを使用しています。 +集計関数の状態については、1+N\*バイトの値のサイズに等しいメモリ量を使用します。 文字列の場合、8バイトの非暗号化ハッシュを格納します。 つまり、計算は文字列に対して近似されます。 この関数は、いくつかの引数でも機能します。 -大きなn値が使用され、一意の値の数がnよりわずかに少ない場合を除いて、できるだけ速く動作します。 +大きなN値が使用され、一意の値の数がNよりわずかに小さい場合を除いて、できるだけ高速に動作します。 使用例: @@ -494,6 +494,6 @@ Solution: Write in the GROUP BY query SearchPhrase HAVING uniqUpTo(4)(UserID) >= [元の記事](https://clickhouse.tech/docs/en/query_language/agg_functions/parametric_functions/) -## sumMapFiltered(keys\_to\_keep)(キー、値) {#summapfilteredkeys-to-keepkeys-values} +## sumMapFiltered(keys\_to\_keep)(キー,値) {#summapfilteredkeys-to-keepkeys-values} -同じ動作として [sumMap](reference.md#agg_functions-summap) キーの配列がパラメータとして渡されることを除いて。 これは、キーの高い基数を扱うときに特に便利です。 +と同じ動作 [サマップ](reference.md#agg_functions-summap) ただし、キーの配列はパラメータとして渡されます。 これは、キーの基数が高い場合に特に便利です。 diff --git a/docs/ja/sql-reference/aggregate-functions/reference.md b/docs/ja/sql-reference/aggregate-functions/reference.md index 5fecb1facd9..3a4266e7a02 100644 --- a/docs/ja/sql-reference/aggregate-functions/reference.md +++ b/docs/ja/sql-reference/aggregate-functions/reference.md @@ -1,15 +1,15 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 36 toc_title: "\u53C2\u7167" --- -# 関数リファレンス {#function-reference} +# 集計関数の参照 {#aggregate-functions-reference} ## カウント {#agg_function-count} -行数またはnull以外の値をカウントします。 +行数またはnot-NULL値をカウントします。 ClickHouseは以下の構文をサポートしています `count`: - `count(expr)` または `COUNT(DISTINCT expr)`. @@ -19,21 +19,21 @@ ClickHouseは以下の構文をサポートしています `count`: 機能は取ることができます: -- ゼロ変数。 +- ゼロパラメータ。 - ワン [式](../syntax.md#syntax-expressions). **戻り値** -- 関数がパラメータなしで呼び出されると、行数がカウントされます。 -- この [式](../syntax.md#syntax-expressions) が渡されると、この関数は、この式がnullではなく返された回数をカウントします。 式がaを返す場合 [Nullable](../../sql-reference/data-types/nullable.md)-タイプ値、そして結果の `count` 滞在しない `Nullable`. 式が返された場合、関数は0を返します `NULL` すべての行について。 +- 関数がパラメータなしで呼び出された場合、行の数がカウントされます。 +- もし [式](../syntax.md#syntax-expressions) 渡されると、関数はこの式がnot nullを返した回数をカウントします。 式がaを返す場合 [Null可能](../../sql-reference/data-types/nullable.md)-型の値、その後の結果 `count` 滞在しない `Nullable`. 式が返された場合、関数は0を返します `NULL` すべての行について。 -どちらの場合も、戻り値の型は次のようになります [UInt64](../../sql-reference/data-types/int-uint.md). +どちらの場合も、戻り値の型は次のとおりです [UInt64](../../sql-reference/data-types/int-uint.md). **詳細** -クリックハウスは `COUNT(DISTINCT ...)` 構文。 この構成の動作は、 [count\_distinct\_implementation](../../operations/settings/settings.md#settings-count_distinct_implementation) 設定。 それはどれをの定義します [uniq\*](#agg_function-uniq) 関数は、操作を実行するために使用されます。 デフォルトは [ユニキャック](#agg_function-uniqexact) 機能。 +クリックハウスは `COUNT(DISTINCT ...)` 構文。 この構造の動作は、 [count\_distinct\_implementation](../../operations/settings/settings.md#settings-count_distinct_implementation) 設定。 それはのどれを定義します [uniq\*](#agg_function-uniq) 関数は、操作を実行するために使用されます。 デフォルトは [uniqExact](#agg_function-uniqexact) 機能。 -その `SELECT count() FROM table` テーブル内のエントリの数が別々に格納されていないため、クエリは最適化されません。 テーブルから小さな列を選択し、その中の値の数を数えます。 +その `SELECT count() FROM table` テーブル内のエントリの数が別々に格納されないため、クエリは最適化されません。 テーブルから小さな列を選択し、その中の値の数をカウントします。 **例** @@ -71,21 +71,21 @@ SELECT count(DISTINCT num) FROM t └────────────────┘ ``` -この例では、 `count(DISTINCT num)` によって実行される。 `uniqExact` に従う機能 `count_distinct_implementation` 設定値。 +この例では、 `count(DISTINCT num)` によって実行されます。 `uniqExact` に従う機能 `count_distinct_implementation` 設定値。 ## 任意(x) {#agg_function-any} 最初に検出された値を選択します。 -クエリは、毎回異なる順序で実行することができるため、この関数の結果は不確定です。 -確定的な結果を得るには、 ‘min’ または ‘max’ 関数の代わりに ‘any’. +クエリは毎回異なる順序でも任意の順序で実行できるため、この関数の結果は不確定です。 +決定的な結果を得るには、 ‘min’ または ‘max’ 関数の代わりに ‘any’. -場合によっては、実行順序に頼ることができます。 これは、order byを使用するサブクエリからのselectの場合に適用されます。 +場合によっては、実行の順序に依存することができます。 これは、SELECTがORDER BYを使用するサブクエリから来ている場合に適用されます。 -とき `SELECT` クエリには `GROUP BY` 句または少なくとも一つの集計関数、ClickHouse（MySQLとは対照的に）内のすべての式ということが必要です `SELECT`, `HAVING`、と `ORDER BY` 句は、キーまたは集計関数から計算されます。 つまり、テーブルから選択された各列は、キーまたは集計関数内で使用する必要があります。 MySQLのような動作を得るには、他の列を `any` 集計関数。 +ときa `SELECT` クエリには `GROUP BY` クラスタ内のすべての式は、クラスタ内のすべての式を必要とします。 `SELECT`, `HAVING`,and `ORDER BY` 句は、キーまたは集計関数から計算されます。 つまり、テーブルから選択された各列は、キーまたは集計関数内で使用する必要があります。 MySQLのような動作を得るには、他の列を `any` 集計関数。 ## anyHeavy(x) {#anyheavyx} -頻繁に発生する値を選択します。 [ヘビーヒッターズ](http://www.cs.umd.edu/~samir/498/karp.pdf) アルゴリズムだ 各クエリの実行スレッドのケースの半分を超える値がある場合は、この値が返されます。 通常、結果は非決定的です。 +を使用して頻繁に発生する値を選択します。 [ヘビーヒッター](http://www.cs.umd.edu/~samir/498/karp.pdf) アルゴリズム 各クエリの実行スレッドでケースの半分を超える値がある場合、この値が返されます。 通常、結果は非決定的です。 ``` sql anyHeavy(column) @@ -95,9 +95,9 @@ anyHeavy(column) - `column` – The column name. -**例えば** +**例** -を取る [オンタイム](../../getting-started/example-datasets/ontime.md) データセットと選択頻繁に発生する値で `AirlineID` コラム +を取る [オンタイム](../../getting-started/example-datasets/ontime.md) データ-セットはの頻繁に発生する値を選び、 `AirlineID` 列。 ``` sql SELECT anyHeavy(AirlineID) AS res @@ -110,14 +110,14 @@ FROM ontime └───────┘ ``` -## anyllast(x) {#anylastx} +## anyLast(x) {#anylastx} 最後に検出された値を選択します。 -結果は、次の場合と同様に不確定です `any` 機能。 +結果は同じように不確定です。 `any` 機能。 -## groupBitAnd {#groupbitand} +## グループビタンド {#groupbitand} -ビットごとの適用 `AND` 一連の数字のために。 +ビット単位で適用 `AND` 数のシリーズのために。 ``` sql groupBitAnd(expr) @@ -131,7 +131,7 @@ groupBitAnd(expr) の値 `UInt*` タイプ。 -**例えば** +**例** テストデータ: @@ -149,7 +149,7 @@ binary decimal SELECT groupBitAnd(num) FROM t ``` -どこに `num` テストデータの列です。 +どこに `num` テストデータを含む列です。 結果: @@ -160,7 +160,7 @@ binary decimal ## groupBitOr {#groupbitor} -ビットごとの適用 `OR` 一連の数字のために。 +ビット単位で適用 `OR` 数のシリーズのために。 ``` sql groupBitOr(expr) @@ -174,7 +174,7 @@ groupBitOr(expr) の値 `UInt*` タイプ。 -**例えば** +**例** テストデータ: @@ -192,7 +192,7 @@ binary decimal SELECT groupBitOr(num) FROM t ``` -どこに `num` テストデータの列です。 +どこに `num` テストデータを含む列です。 結果: @@ -203,7 +203,7 @@ binary decimal ## groupBitXor {#groupbitxor} -ビットごとの適用 `XOR` 一連の数字のために。 +ビット単位で適用 `XOR` 数のシリーズのために。 ``` sql groupBitXor(expr) @@ -217,7 +217,7 @@ groupBitXor(expr) の値 `UInt*` タイプ。 -**例えば** +**例** テストデータ: @@ -235,7 +235,7 @@ binary decimal SELECT groupBitXor(num) FROM t ``` -どこに `num` テストデータの列です。 +どこに `num` テストデータを含む列です。 結果: @@ -246,7 +246,7 @@ binary decimal ## groupBitmap {#groupbitmap} -符号なし整数列からのビットマップ計算または集計計算を行い、uint64型のカーディナリティを返します。 [ビットマップ](../../sql-reference/functions/bitmap-functions.md). +符号なし整数列からのビットマップまたは集計計算、uint64型の基数を返します。 [ビットマップ](../../sql-reference/functions/bitmap-functions.md). ``` sql groupBitmap(expr) @@ -260,7 +260,7 @@ groupBitmap(expr) の値 `UInt64` タイプ。 -**例えば** +**例** テストデータ: @@ -285,19 +285,19 @@ num 3 ``` -## 最小(x) {#agg_function-min} +## min(x) {#agg_function-min} 最小値を計算します。 -## 最大(x) {#agg_function-max} +## max(x) {#agg_function-max} 最大値を計算します。 -## argMin(arg,val) {#agg-function-argmin} +## アルグミン(arg,val) {#agg-function-argmin} -を計算し ‘arg’ 最小値の値 ‘val’ 値。 いくつかの異なる値がある場合 ‘arg’ のための最小値 ‘val’ これらの値のうち、最初に検出された値が出力されます。 +計算します ‘arg’ 最小値の値 ‘val’ 値。 のいくつかの異なる値がある場合 ‘arg’ の最小値に対して ‘val’、検出された最初のこれらの値が出力されます。 -**例えば:** +**例:** ``` text ┌─user─────┬─salary─┐ @@ -317,28 +317,29 @@ SELECT argMin(user, salary) FROM salary └──────────────────────┘ ``` -## argMax(arg,val) {#agg-function-argmax} +## アルグマックス(arg,val) {#agg-function-argmax} -を計算し ‘arg’ 最大値の値 ‘val’ 値。 いくつかの異なる値がある場合 ‘arg’ の最大値 ‘val’ これらの値のうち、最初に検出された値が出力されます。 +計算します ‘arg’ 最大値 ‘val’ 値。 のいくつかの異なる値がある場合 ‘arg’ の最大値の場合 ‘val’、検出された最初のこれらの値が出力されます。 ## sum(x) {#agg_function-sum} 合計を計算します。 -数字のためにのみ動作します。 +数字のみで動作します。 ## sumWithOverflow(x) {#sumwithoverflowx} -入力パラメーターの結果と同じデータ型を使用して、数値の合計を計算します。 合計がこのデータ型の最大値を超えると、関数はエラーを返します。 +結果には、入力パラメーターと同じデータ型を使用して、数値の合計を計算します。 合計がこのデータ型の最大値を超えると、関数はエラーを返します。 -数字のためにのみ動作します。 +数字のみで動作します。 -## sumMap(キー,値) {#agg_functions-summap} +## sumMap(キー,値),sumMap(タプル(キー,値))) {#agg_functions-summap} -合計 ‘value’ 配列に指定されたキーに応じて ‘key’ 配列だ -の要素の数 ‘key’ と ‘value’ 合計される行ごとに同じでなければなりません。 +合計 ‘value’ に指定されたキーに従って配列 ‘key’ 配列 +キーと値の配列のタプルを渡すことは、キーと値の二つの配列を渡すことと同義です。 +要素の数 ‘key’ と ‘value’ 合計される行ごとに同じである必要があります。 Returns a tuple of two arrays: keys in sorted order, and values ​​summed for the corresponding keys. -例えば: +例: ``` sql CREATE TABLE sum_map( @@ -347,30 +348,33 @@ CREATE TABLE sum_map( statusMap Nested( status UInt16, requests UInt64 - ) + ), + statusMapTuple Tuple(Array(Int32), Array(Int32)) ) ENGINE = Log; INSERT INTO sum_map VALUES - ('2000-01-01', '2000-01-01 00:00:00', [1, 2, 3], [10, 10, 10]), - ('2000-01-01', '2000-01-01 00:00:00', [3, 4, 5], [10, 10, 10]), - ('2000-01-01', '2000-01-01 00:01:00', [4, 5, 6], [10, 10, 10]), - ('2000-01-01', '2000-01-01 00:01:00', [6, 7, 8], [10, 10, 10]); + ('2000-01-01', '2000-01-01 00:00:00', [1, 2, 3], [10, 10, 10], ([1, 2, 3], [10, 10, 10])), + ('2000-01-01', '2000-01-01 00:00:00', [3, 4, 5], [10, 10, 10], ([3, 4, 5], [10, 10, 10])), + ('2000-01-01', '2000-01-01 00:01:00', [4, 5, 6], [10, 10, 10], ([4, 5, 6], [10, 10, 10])), + ('2000-01-01', '2000-01-01 00:01:00', [6, 7, 8], [10, 10, 10], ([6, 7, 8], [10, 10, 10])); + SELECT timeslot, - sumMap(statusMap.status, statusMap.requests) + sumMap(statusMap.status, statusMap.requests), + sumMap(statusMapTuple) FROM sum_map GROUP BY timeslot ``` ``` text -┌────────────timeslot─┬─sumMap(statusMap.status, statusMap.requests)─┐ -│ 2000-01-01 00:00:00 │ ([1,2,3,4,5],[10,10,20,10,10]) │ -│ 2000-01-01 00:01:00 │ ([4,5,6,7,8],[10,10,20,10,10]) │ -└─────────────────────┴──────────────────────────────────────────────┘ +┌────────────timeslot─┬─sumMap(statusMap.status, statusMap.requests)─┬─sumMap(statusMapTuple)─────────┐ +│ 2000-01-01 00:00:00 │ ([1,2,3,4,5],[10,10,20,10,10]) │ ([1,2,3,4,5],[10,10,20,10,10]) │ +│ 2000-01-01 00:01:00 │ ([4,5,6,7,8],[10,10,20,10,10]) │ ([4,5,6,7,8],[10,10,20,10,10]) │ +└─────────────────────┴──────────────────────────────────────────────┴────────────────────────────────┘ ``` ## skewPop {#skewpop} -を計算します [歪み](https://en.wikipedia.org/wiki/Skewness) シーケンスの。 +を計算する [歪み](https://en.wikipedia.org/wiki/Skewness) シーケンスの。 ``` sql skewPop(expr) @@ -384,17 +388,17 @@ skewPop(expr) The skewness of the given distribution. Type — [Float64](../../sql-reference/data-types/float.md) -**例えば** +**例** ``` sql SELECT skewPop(value) FROM series_with_value_column ``` -## 串焼き {#skewsamp} +## skewSamp {#skewsamp} -を計算します [サンプルの歪度](https://en.wikipedia.org/wiki/Skewness) シーケンスの。 +を計算する [サンプル歪度](https://en.wikipedia.org/wiki/Skewness) シーケンスの。 -これは、渡された値がそのサンプルを形成する場合、確率変数の歪度の不偏推定値を表します。 +渡された値がサンプルを形成する場合、確率変数の歪度の不偏推定値を表します。 ``` sql skewSamp(expr) @@ -406,17 +410,17 @@ skewSamp(expr) **戻り値** -The skewness of the given distribution. Type — [Float64](../../sql-reference/data-types/float.md). もし `n <= 1` (`n` はサンプルのサイズです）、関数は次の値を返します `nan`. +The skewness of the given distribution. Type — [Float64](../../sql-reference/data-types/float.md). もし `n <= 1` (`n` はサンプルのサイズです）、その後、関数が返します `nan`. -**例えば** +**例** ``` sql SELECT skewSamp(value) FROM series_with_value_column ``` -## kurtPop {#kurtpop} +## クルトポップ {#kurtpop} -を計算します [尖度](https://en.wikipedia.org/wiki/Kurtosis) シーケンスの。 +を計算する [尖度](https://en.wikipedia.org/wiki/Kurtosis) シーケンスの。 ``` sql kurtPop(expr) @@ -430,17 +434,17 @@ kurtPop(expr) The kurtosis of the given distribution. Type — [Float64](../../sql-reference/data-types/float.md) -**例えば** +**例** ``` sql SELECT kurtPop(value) FROM series_with_value_column ``` -## kurtSamp {#kurtsamp} +## クルツァンプ {#kurtsamp} -を計算します [サンプル尖度](https://en.wikipedia.org/wiki/Kurtosis) のシーケンスです。 +を計算する [尖度のサンプル](https://en.wikipedia.org/wiki/Kurtosis) シーケンスの。 -これは、渡された値がサンプルを形成する場合、確率変数の尖度の不偏推定値を表します。 +渡された値がその標本を形成する場合、確率変数の尖度の不偏推定値を表します。 ``` sql kurtSamp(expr) @@ -452,9 +456,9 @@ kurtSamp(expr) **戻り値** -The kurtosis of the given distribution. Type — [Float64](../../sql-reference/data-types/float.md). もし `n <= 1` (`n` はサンプルのサイズです）、関数は次の値を返します `nan`. +The kurtosis of the given distribution. Type — [Float64](../../sql-reference/data-types/float.md). もし `n <= 1` (`n` はサンプルのサイズです）、関数は `nan`. -**例えば** +**例** ``` sql SELECT kurtSamp(value) FROM series_with_value_column @@ -465,15 +469,15 @@ SELECT kurtSamp(value) FROM series_with_value_column `timeSeriesGroupSum` 総異なる時系列のサンプルのタイムスタンプなアライメントを実施します。 これは、二つのサンプルタイムスタンプ間の線形補間を使用して、一緒に時系列を合計します。 -- `uid` タイムシリーズの一意のidです, `UInt64`. +- `uid` 時系列は一意のidですか, `UInt64`. - `timestamp` ミリ秒またはマイクロ秒をサポートするためにInt64型です。 -- `value` メトリックです。 +- `value` は指標です。 -この関数は、以下のタプルの配列を返します `(timestamp, aggregated_value)` のペアになっています。 +この関数は、次のような組の配列を返します `(timestamp, aggregated_value)` ペア。 -この機能を使用する前に確認 `timestamp` は昇順です。 +この関数を使用する前に、必ず `timestamp` 昇順です。 -例えば: +例: ``` text ┌─uid─┬─timestamp─┬─value─┐ @@ -512,22 +516,63 @@ FROM ( [(2,0.2),(3,0.9),(7,2.1),(8,2.4),(12,3.6),(17,5.1),(18,5.4),(24,7.2),(25,2.5)] ``` -## timeSeriesGroupRateSum(uid,ts,val) {#agg-function-timeseriesgroupratesum} +## タイムセリエスグロプラテスム(uid,ts,val) {#agg-function-timeseriesgroupratesum} -同様にtimeseriesgroupratesum、timeseriesgroupratesumは、時系列のレートを計算し、その後、一緒にレートを合計します。 -また、この関数を使用する前にタイムスタンプが昇順になるはずです。 +同様に `timeSeriesGroupSum`, `timeSeriesGroupRateSum` 時系列のレートを計算し、レートを合計します。 +また、timestampはこの関数を使用する前に上昇順にする必要があります。 -この関数を使用すると、上記の結果は次のようになります: +のデータにこの関数を適用します。 `timeSeriesGroupSum` 例では、次の結果が得られます: ``` text [(2,0),(3,0.1),(7,0.3),(8,0.3),(12,0.3),(17,0.3),(18,0.3),(24,0.3),(25,0.1)] ``` -## 平均(x) {#agg_function-avg} +## avg(x) {#agg_function-avg} 平均を計算します。 -数字のためにのみ動作します。 -結果は常にfloat64です。 +数字のみで動作します。 +結果は常にFloat64です。 + +## avgWeighted {#avgweighted} + +計算します [加重算術平均](https://en.wikipedia.org/wiki/Weighted_arithmetic_mean). + +**構文** + +``` sql +avgWeighted(x, weight) +``` + +**パラメータ** + +- `x` — Values. [整数](../data-types/int-uint.md) または [浮動小数点数](../data-types/float.md). +- `weight` — Weights of the values. [整数](../data-types/int-uint.md) または [浮動小数点数](../data-types/float.md). + +タイプの `x` と `weight` 同じである必要があります。 + +**戻り値** + +- 加重平均。 +- `NaN`. すべての重みが0に等しい場合。 + +タイプ: [Float64](../data-types/float.md). + +**例** + +クエリ: + +``` sql +SELECT avgWeighted(x, w) +FROM values('x Int8, w Int8', (4, 1), (1, 0), (10, 2)) +``` + +結果: + +``` text +┌─avgWeighted(x, weight)─┐ +│ 8 │ +└────────────────────────┘ +``` ## uniq {#agg_function-uniq} @@ -539,7 +584,7 @@ uniq(x[, ...]) **パラメータ** -この関数は、可変個のパラメータを受け取ります。 変数は `Tuple`, `Array`, `Date`, `DateTime`, `String`、または数値型。 +この関数は、可変数のパラメータを取ります。 変数はあります `Tuple`, `Array`, `Date`, `DateTime`, `String` または数値型。 **戻り値** @@ -547,24 +592,24 @@ uniq(x[, ...]) **実装の詳細** -機能: +関数: -- 集計内のすべてのパラメータのハッシュを計算し、それを計算に使用します。 +- 集計内のすべてのパラメーターのハッシュを計算し、それを計算に使用します。 -- を使用して適応サンプリングアルゴリズムです。 計算状態の場合、関数は65536までの要素ハッシュ値のサンプルを使用します。 +- を使用して適応サンプリングアルゴリズムです。 計算状態では、関数は65536までの要素ハッシュ値のサンプルを使用します。 This algorithm is very accurate and very efficient on the CPU. When the query contains several of these functions, using `uniq` is almost as fast as using other aggregate functions. -- 結果を確定的に提供します（クエリ処理の順序に依存しません）。 +- 結果を確定的に提供します(クエリ処理順序に依存しません)。 使用をお勧めしますこの機能はほとんど全てのシナリオ. -**また見なさい** +**も参照。** - [uniqCombined](#agg_function-uniqcombined) - [uniqCombined64](#agg_function-uniqcombined64) -- [unihll12](#agg_function-uniqhll12) -- [ユニキャック](#agg_function-uniqexact) +- [uniqHLL12](#agg_function-uniqhll12) +- [uniqExact](#agg_function-uniqexact) ## uniqCombined {#agg_function-uniqcombined} @@ -574,53 +619,53 @@ uniq(x[, ...]) uniqCombined(HLL_precision)(x[, ...]) ``` -その `uniqCombined` 関数は、異なる値の数を計算するのに適しています。 +その `uniqCombined` 関数は、異なる値の数を計算するための良い選択です。 **パラメータ** -この関数は、可変個のパラメータを受け取ります。 変数は `Tuple`, `Array`, `Date`, `DateTime`, `String`、または数値型。 +この関数は、可変数のパラメータを取ります。 変数はあります `Tuple`, `Array`, `Date`, `DateTime`, `String` または数値型。 -`HLL_precision` は、2のセル数の底の対数です [ハイパーログ](https://en.wikipedia.org/wiki/HyperLogLog). オプションで、次のように関数を使用できます `uniqCombined(x[, ...])`. のデフォルト値 `HLL_precision` は17で、これは効果的に96KiBのスペース（2^17セル、6ビットそれぞれ）です。 +`HLL_precision` は、セルの数の底2対数です。 [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog). オプションで、次のように関数を使用できます `uniqCombined(x[, ...])`. のデフォルト値 `HLL_precision` は17であり、これは実質的に96KiBの空間（2^17セル、各6ビット）である。 **戻り値** -- を番号 [UInt64](../../sql-reference/data-types/int-uint.md)-タイプ番号。 +- 番号 [UInt64](../../sql-reference/data-types/int-uint.md)-タイプ番号。 **実装の詳細** -機能: +関数: -- ハッシュを計算します(64ビットのハッシュ `String` それ以外の場合は32ビット）は、集計内のすべてのパラメータに対して、それを計算に使用します。 +- ハッシュ(64ビットのハッシュ `String` それ以外の場合は32ビット）集計内のすべてのパラメータについて、それを計算に使用します。 -- 配列、ハッシュテーブル、およびhyperloglogとエラー修正テーブルの組み合わせを使用します。 +- 配列、ハッシュテーブル、HyperLogLogと誤り訂正表の組み合わせを使用します。 For a small number of distinct elements, an array is used. When the set size is larger, a hash table is used. For a larger number of elements, HyperLogLog is used, which will occupy a fixed amount of memory. -- 結果を確定的に提供します（クエリ処理の順序に依存しません）。 +- 結果を確定的に提供します(クエリ処理順序に依存しません)。 -!!! note "メモ" - それは32ビットハッシュを使用しているので-`String` タイプすると、結果はカーディナリティのエラーが非常に大きくなります `UINT_MAX` （エラーは数十億の異なる値の後にすぐに発生します）、この場合は次のようにしてください [uniqCombined64](#agg_function-uniqcombined64) +!!! note "注" + 非32ビットハッシュを使用するので-`String` タイプは、結果よりも有意に大きい基数のための非常に高い誤差を有する `UINT_MAX` （エラーは数十億の異なる値の後にすぐに発生します）、この場合は以下を使用する必要があります [uniqCombined64](#agg_function-uniqcombined64) -に比べて [uniq](#agg_function-uniq) 機能、を `uniqCombined`: +と比較される [uniq](#agg_function-uniq) 関数は、 `uniqCombined`: -- 数回少ないメモリを消費します。 +- 数倍少ないメモリを消費します。 - 数倍高い精度で計算します。 -- 通常は若干低い性能を持っています。 一部のシナリオでは, `uniqCombined` より良い実行できる `uniq` たとえば、ネットワークを介して多数の集約状態を送信する分散クエリを使用します。 +- 通常はやや性能が低い。 シナリオによっては, `uniqCombined` ではどのように絡んでいるのかを調べ `uniq` たとえば、ネットワーク経由で多数の集約状態を送信する分散クエリです。 -**また見なさい** +**も参照。** - [uniq](#agg_function-uniq) - [uniqCombined64](#agg_function-uniqcombined64) -- [unihll12](#agg_function-uniqhll12) -- [ユニキャック](#agg_function-uniqexact) +- [uniqHLL12](#agg_function-uniqhll12) +- [uniqExact](#agg_function-uniqexact) ## uniqCombined64 {#agg_function-uniqcombined64} -と同じ [uniqCombined](#agg_function-uniqcombined) ただし、すべてのデータ型に64ビットハッシュを使用します。 +同じ [uniqCombined](#agg_function-uniqcombined) しかし、すべてのデータ型に64ビットハッシュを使用します。 -## unihll12 {#agg_function-uniqhll12} +## uniqHLL12 {#agg_function-uniqhll12} -を使用して、異なる引数値のおおよその数を計算します [ハイパーログ](https://en.wikipedia.org/wiki/HyperLogLog) アルゴリズムだ +異なる引数値のおおよその数を計算します。 [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog) アルゴリズム ``` sql uniqHLL12(x[, ...]) @@ -628,7 +673,7 @@ uniqHLL12(x[, ...]) **パラメータ** -この関数は、可変個のパラメータを受け取ります。 変数は `Tuple`, `Array`, `Date`, `DateTime`, `String`、または数値型。 +この関数は、可変数のパラメータを取ります。 変数はあります `Tuple`, `Array`, `Date`, `DateTime`, `String` または数値型。 **戻り値** @@ -636,25 +681,25 @@ uniqHLL12(x[, ...]) **実装の詳細** -機能: +関数: -- 集計内のすべてのパラメータのハッシュを計算し、それを計算に使用します。 +- 集計内のすべてのパラメーターのハッシュを計算し、それを計算に使用します。 - HyperLogLogアルゴリズムを使用して、異なる引数値の数を近似します。 212 5-bit cells are used. The size of the state is slightly more than 2.5 KB. The result is not very accurate (up to ~10% error) for small data sets (<10K elements). However, the result is fairly accurate for high-cardinality data sets (10K-100M), with a maximum error of ~1.6%. Starting from 100M, the estimation error increases, and the function will return very inaccurate results for data sets with extremely high cardinality (1B+ elements). -- 確定的な結果を提供します（クエリ処理の順序に依存しません）。 +- 決定的な結果を提供します(クエリ処理順序に依存しません)。 -この機能を使用することはお勧めしません。 ほとんどの場合、 [uniq](#agg_function-uniq) または [uniqCombined](#agg_function-uniqcombined) 機能。 +この機能の使用はお勧めしません。 ほとんどの場合、 [uniq](#agg_function-uniq) または [uniqCombined](#agg_function-uniqcombined) 機能。 -**また見なさい** +**も参照。** - [uniq](#agg_function-uniq) - [uniqCombined](#agg_function-uniqcombined) -- [ユニキャック](#agg_function-uniqexact) +- [uniqExact](#agg_function-uniqexact) -## ユニキャック {#agg_function-uniqexact} +## uniqExact {#agg_function-uniqexact} 異なる引数値の正確な数を計算します。 @@ -662,65 +707,139 @@ uniqHLL12(x[, ...]) uniqExact(x[, ...]) ``` -を使用 `uniqExact` 機能あなたは絶対に正確な結果が必要な場合。 それ以外の場合は、 [uniq](#agg_function-uniq) 機能。 +使用する `uniqExact` あなたが絶対に正確な結果が必要な場合は、関数。 それ以外の場合は、 [uniq](#agg_function-uniq) 機能。 -その `uniqExact` 機能の使用ます。 `uniq`、状態のサイズは、異なる値の数が増加するにつれて無制限の成長を有するからである。 +その `uniqExact` 関数はより多くのメモリを使用 `uniq`、状態の大きさは、異なる値の数が増加するにつれて無限の成長を有するからである。 **パラメータ** -この関数は、可変個のパラメータを受け取ります。 変数は `Tuple`, `Array`, `Date`, `DateTime`, `String`、または数値型。 +この関数は、可変数のパラメータを取ります。 変数はあります `Tuple`, `Array`, `Date`, `DateTime`, `String` または数値型。 -**また見なさい** +**も参照。** - [uniq](#agg_function-uniq) - [uniqCombined](#agg_function-uniqcombined) -- [unihll12](#agg_function-uniqhll12) +- [uniqHLL12](#agg_function-uniqhll12) ## groupArray(x),groupArray(max\_size)(x) {#agg_function-grouparray} -引数の値の配列を作成します。 +引数値の配列を作成します。 値は、任意の（不確定な）順序で配列に追加できます。 -第二のバージョン（ `max_size` パラメータ）結果の配列のサイズを次のように制限します `max_size` 要素。 -例えば, `groupArray (1) (x)` に相当します `[any (x)]`. +第二のバージョン（と `max_size` パラメータ)結果の配列のサイズを以下に制限します `max_size` 要素。 +例えば, `groupArray (1) (x)` に等しい。 `[any (x)]`. -場合によっては、実行の順序に依拠することもできます。 これは、次の場合に適用されます `SELECT` 使用するサブクエリーから取得されます `ORDER BY`. +場合によっては、実行順序に依存することもあります。 これは、次の場合に適用されます `SELECT` を使用するサブクエリから来ています `ORDER BY`. -## groupArrayInsertAt(値、位置) {#grouparrayinsertatvalue-position} +## グループパラインセルタット {#grouparrayinsertat} -指定した位置の配列に値を挿入します。 +指定された位置に配列に値を挿入します。 -!!! note "メモ" - この関数はゼロベースの位置を使用します。 +**構文** -Accepts the value and position as input. If several values ​​are inserted into the same position, any of them might end up in the resulting array (the first one will be used in the case of single-threaded execution). If no value is inserted into a position, the position is assigned the default value. +``` sql +groupArrayInsertAt(default_x, size)(x, pos); +``` -任意変数: +あるクエリで複数の値が同じ位置に挿入された場合、関数は次のように動作します: -- 空の位置に置き換えるためのデフォルト値。 -- 結果の配列の長さ。 これにより、すべての集約キーで同じサイズの配列を受け取ることができます。 このパラメーターを使用する場合は、既定値を指定する必要があります。 +- クエリが単一のスレッドで実行される場合、挿入された最初の値が使用されます。 +- クエリが複数のスレッドで実行される場合、結果の値は挿入された値のいずれかになります。 -## グルーパーレイモビングサムcity in new mexico usa {#agg_function-grouparraymovingsum} +**パラメータ** -入力値の移動和を計算します。 +- `x` — Value to be inserted. [式](../syntax.md#syntax-expressions) のいずれかになります [対応するデータ型](../../sql-reference/data-types/index.md). +- `pos` — Position at which the specified element `x` 挿入されるべきです。 配列のインデックス番号はゼロから始まります。 [UInt32](../../sql-reference/data-types/int-uint.md#uint-ranges). +- `default_x`— Default value for substituting in empty positions. Optional parameter. [式](../syntax.md#syntax-expressions) その結果、データ型は `x` パラメータ。 もし `default_x` は定義されていない。 [デフォルト値](../../sql-reference/statements/create.md#create-default-values) 使用されます。 +- `size`— Length of the resulting array. Optional parameter. When using this parameter, the default value `default_x` 指定する必要があります。 [UInt32](../../sql-reference/data-types/int-uint.md#uint-ranges). + +**戻り値** + +- 値が挿入された配列。 + +タイプ: [配列](../../sql-reference/data-types/array.md#data-type-array). + +**例** + +クエリ: + +``` sql +SELECT groupArrayInsertAt(toString(number), number * 2) FROM numbers(5); +``` + +結果: + +``` text +┌─groupArrayInsertAt(toString(number), multiply(number, 2))─┐ +│ ['0','','1','','2','','3','','4'] │ +└───────────────────────────────────────────────────────────┘ +``` + +クエリ: + +``` sql +SELECT groupArrayInsertAt('-')(toString(number), number * 2) FROM numbers(5); +``` + +結果: + +``` text +┌─groupArrayInsertAt('-')(toString(number), multiply(number, 2))─┐ +│ ['0','-','1','-','2','-','3','-','4'] │ +└────────────────────────────────────────────────────────────────┘ +``` + +クエリ: + +``` sql +SELECT groupArrayInsertAt('-', 5)(toString(number), number * 2) FROM numbers(5); +``` + +結果: + +``` text +┌─groupArrayInsertAt('-', 5)(toString(number), multiply(number, 2))─┐ +│ ['0','-','1','-','2'] │ +└───────────────────────────────────────────────────────────────────┘ +``` + +一つの位置に要素のマルチスレッド挿入。 + +クエリ: + +``` sql +SELECT groupArrayInsertAt(number, 0) FROM numbers_mt(10) SETTINGS max_block_size = 1; +``` + +このクエリの結果として、あなたはランダムな整数を取得します `[0,9]` 範囲 例えば: + +``` text +┌─groupArrayInsertAt(number, 0)─┐ +│ [7] │ +└───────────────────────────────┘ +``` + +## グループパレイモビングスム {#agg_function-grouparraymovingsum} + +入力値の移動合計を計算します。 ``` sql groupArrayMovingSum(numbers_for_summing) groupArrayMovingSum(window_size)(numbers_for_summing) ``` -この機能できるウィンドウサイズとしてのパラメータとします。 指定しない場合、この関数は、列の行数と同じウィンドウサイズをとります。 +この機能できるウィンドウサイズとしてのパラメータとします。 指定されていない場合、関数は列の行数に等しいウィンドウサイズを取ります。 **パラメータ** -- `numbers_for_summing` — [式](../syntax.md#syntax-expressions) その結果、数値データ型の値が返されます。 +- `numbers_for_summing` — [式](../syntax.md#syntax-expressions) 数値データ型の値になります。 - `window_size` — Size of the calculation window. **戻り値** -- 入力データと同じサイズおよびタイプの配列。 +- 入力データと同じサイズと型の配列。 -**例えば** +**例** サンプルテーブル: @@ -773,7 +892,7 @@ FROM t └────────────┴─────────────────────────────────┴────────────────────────┘ ``` -## groupparraymovingavg {#agg_function-grouparraymovingavg} +## groupArrayMovingAvg {#agg_function-grouparraymovingavg} 入力値の移動平均を計算します。 @@ -782,20 +901,20 @@ groupArrayMovingAvg(numbers_for_summing) groupArrayMovingAvg(window_size)(numbers_for_summing) ``` -この機能できるウィンドウサイズとしてのパラメータとします。 指定しない場合、この関数は、列の行数と同じウィンドウサイズをとります。 +この機能できるウィンドウサイズとしてのパラメータとします。 指定されていない場合、関数は列の行数に等しいウィンドウサイズを取ります。 **パラメータ** -- `numbers_for_summing` — [式](../syntax.md#syntax-expressions) その結果、数値データ型の値が返されます。 +- `numbers_for_summing` — [式](../syntax.md#syntax-expressions) 数値データ型の値になります。 - `window_size` — Size of the calculation window. **戻り値** -- 入力データと同じサイズおよびタイプの配列。 +- 入力データと同じサイズと型の配列。 -この関数は [ゼロに向かって丸め](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero). 結果のデータ型の小数点以下の桁を切り捨てます。 +この関数は [ゼロへの丸め](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero). 結果のデータ型の小数点以下の桁数を切り捨てます。 -**例えば** +**例** サンプルテーブル `b`: @@ -850,18 +969,18 @@ FROM t ## groupUniqArray(x),groupUniqArray(max\_size)(x) {#groupuniqarrayx-groupuniqarraymax-sizex} -異なる引数値から配列を作成します。 メモリ消費量は、 `uniqExact` 機能。 +異なる引数値から配列を作成します。 メモリ消費量は `uniqExact` 機能。 -第二のバージョン（ `max_size` パラメータ）結果の配列のサイズを次のように制限します `max_size` 要素。 -例えば, `groupUniqArray(1)(x)` に相当します `[any(x)]`. +第二のバージョン（と `max_size` パラメータ)結果の配列のサイズを以下に制限します `max_size` 要素。 +例えば, `groupUniqArray(1)(x)` に等しい。 `[any(x)]`. -## 分位値 {#quantile} +## 分位数 {#quantile} -近似値を計算します [分位値](https://en.wikipedia.org/wiki/Quantile) 数値データシーケンス。 +近似を計算します [分位数](https://en.wikipedia.org/wiki/Quantile) 数値データシーケンスの。 -この関数が適用されます [貯蔵所の見本抽出](https://en.wikipedia.org/wiki/Reservoir_sampling) 8192までの貯蔵所のサイズおよび見本抽出のための乱数発電機を使って。 結果は非決定的です。 正確な分位値を取得するには、以下を使用します [quantileExact](#quantileexact) 機能。 +この関数が適用されます [貯蔵所の見本抽出](https://en.wikipedia.org/wiki/Reservoir_sampling) 8192までの貯蔵所のサイズおよび見本抽出のための乱数発生器を使って。 結果は非決定的です。 正確な分位値を取得するには、次の式を使用します [quantileExact](#quantileexact) 機能。 -複数を使用する場合 `quantile*` クエリの異なるレベルを持つ関数は、内部状態が結合されていません（つまり、クエリはそれほど効率的ではありません）。 この場合は、 [分位数](#quantiles) 機能。 +複数を使用する場合 `quantile*` クエリ内の異なるレベルを持つ関数では、内部状態は結合されません(つまり、クエリの動作ができるほど効率的ではありません)。 この場合、 [分位数](#quantiles) 機能。 **構文** @@ -869,24 +988,24 @@ FROM t quantile(level)(expr) ``` -エイリアス: `median`. +別名: `median`. **パラメータ** -- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` の範囲内の値 `[0.01, 0.99]`. デフォルト値:0.5. で `level=0.5` 機能は計算する [中央値](https://en.wikipedia.org/wiki/Median). +- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` の範囲の値 `[0.01, 0.99]`. デフォルト値は0.5です。 で `level=0.5` この関数は [中央値](https://en.wikipedia.org/wiki/Median). - `expr` — Expression over the column values resulting in numeric [データ型](../../sql-reference/data-types/index.md#data_types), [日付](../../sql-reference/data-types/date.md) または [DateTime](../../sql-reference/data-types/datetime.md). **戻り値** -- 指定したレベルの概算値。 +- 指定されたレベルの近似分位値。 タイプ: -- [Float64](../../sql-reference/data-types/float.md) 数値データ型の入力。 +- [Float64](../../sql-reference/data-types/float.md) 数値データ型入力の場合。 - [日付](../../sql-reference/data-types/date.md) 入力値が `Date` タイプ。 - [DateTime](../../sql-reference/data-types/datetime.md) 入力値が `DateTime` タイプ。 -**例えば** +**例** 入力テーブル: @@ -913,18 +1032,18 @@ SELECT quantile(val) FROM t └───────────────┘ ``` -**また見なさい** +**も参照。** - [中央値](#median) - [分位数](#quantiles) -## quantiedeterministic {#quantiledeterministic} +## quantileDeterministic {#quantiledeterministic} -近似値を計算します [分位値](https://en.wikipedia.org/wiki/Quantile) 数値データシーケンス。 +近似を計算します [分位数](https://en.wikipedia.org/wiki/Quantile) 数値データシーケンスの。 -この関数が適用されます [貯蔵所の見本抽出](https://en.wikipedia.org/wiki/Reservoir_sampling) 8192までの貯蔵所のサイズおよび見本抽出の決定論のアルゴリズムを使って。 結果は決定的です。 正確な分位値を取得するには、以下を使用します [quantileExact](#quantileexact) 機能。 +この関数が適用されます [貯蔵所の見本抽出](https://en.wikipedia.org/wiki/Reservoir_sampling) 貯蔵所のサイズ8192までおよび見本抽出の決定論的なアルゴリズムを使って。 結果は決定的です。 正確な分位値を取得するには、次の式を使用します [quantileExact](#quantileexact) 機能。 -複数を使用する場合 `quantile*` クエリの異なるレベルを持つ関数は、内部状態が結合されていません（つまり、クエリはそれほど効率的ではありません）。 この場合は、 [分位数](#quantiles) 機能。 +複数を使用する場合 `quantile*` クエリ内の異なるレベルを持つ関数では、内部状態は結合されません(つまり、クエリの動作ができるほど効率的ではありません)。 この場合、 [分位数](#quantiles) 機能。 **構文** @@ -932,25 +1051,25 @@ SELECT quantile(val) FROM t quantileDeterministic(level)(expr, determinator) ``` -エイリアス: `medianDeterministic`. +別名: `medianDeterministic`. **パラメータ** -- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` の範囲内の値 `[0.01, 0.99]`. デフォルト値:0.5. で `level=0.5` 機能は計算する [中央値](https://en.wikipedia.org/wiki/Median). +- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` の範囲の値 `[0.01, 0.99]`. デフォルト値は0.5です。 で `level=0.5` この関数は [中央値](https://en.wikipedia.org/wiki/Median). - `expr` — Expression over the column values resulting in numeric [データ型](../../sql-reference/data-types/index.md#data_types), [日付](../../sql-reference/data-types/date.md) または [DateTime](../../sql-reference/data-types/datetime.md). - `determinator` — Number whose hash is used instead of a random number generator in the reservoir sampling algorithm to make the result of sampling deterministic. As a determinator you can use any deterministic positive number, for example, a user id or an event id. If the same determinator value occures too often, the function works incorrectly. **戻り値** -- 指定したレベルの概算値。 +- 指定されたレベルの近似分位値。 タイプ: -- [Float64](../../sql-reference/data-types/float.md) 数値データ型の入力。 +- [Float64](../../sql-reference/data-types/float.md) 数値データ型入力の場合。 - [日付](../../sql-reference/data-types/date.md) 入力値が `Date` タイプ。 - [DateTime](../../sql-reference/data-types/datetime.md) 入力値が `DateTime` タイプ。 -**例えば** +**例** 入力テーブル: @@ -977,18 +1096,18 @@ SELECT quantileDeterministic(val, 1) FROM t └───────────────────────────────┘ ``` -**また見なさい** +**も参照。** - [中央値](#median) - [分位数](#quantiles) ## quantileExact {#quantileexact} -正確に計算する [分位値](https://en.wikipedia.org/wiki/Quantile) 数値データシーケンス。 +正確に計算する [分位数](https://en.wikipedia.org/wiki/Quantile) 数値データシーケンスの。 -To get exact value, all the passed values ​​are combined into an array, which is then partially sorted. Therefore, the function consumes `O(n)` メモリ、どこ `n` 渡された値の数です。 しかし、少数の値の場合、関数は非常に効果的です。 +To get exact value, all the passed values ​​are combined into an array, which is then partially sorted. Therefore, the function consumes `O(n)` メモリ、どこ `n` 渡された値の数です。 しかし、少数の値の場合、この関数は非常に効果的です。 -複数を使用する場合 `quantile*` クエリの異なるレベルを持つ関数は、内部状態が結合されていません（つまり、クエリはそれほど効率的ではありません）。 この場合は、 [分位数](#quantiles) 機能。 +複数を使用する場合 `quantile*` クエリ内の異なるレベルを持つ関数では、内部状態は結合されません(つまり、クエリの動作ができるほど効率的ではありません)。 この場合、 [分位数](#quantiles) 機能。 **構文** @@ -996,11 +1115,11 @@ To get exact value, all the passed values ​​are combined into an array, whic quantileExact(level)(expr) ``` -エイリアス: `medianExact`. +別名: `medianExact`. **パラメータ** -- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` の範囲内の値 `[0.01, 0.99]`. デフォルト値:0.5. で `level=0.5` 機能は計算する [中央値](https://en.wikipedia.org/wiki/Median). +- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` の範囲の値 `[0.01, 0.99]`. デフォルト値は0.5です。 で `level=0.5` この関数は [中央値](https://en.wikipedia.org/wiki/Median). - `expr` — Expression over the column values resulting in numeric [データ型](../../sql-reference/data-types/index.md#data_types), [日付](../../sql-reference/data-types/date.md) または [DateTime](../../sql-reference/data-types/datetime.md). **戻り値** @@ -1009,11 +1128,11 @@ quantileExact(level)(expr) タイプ: -- [Float64](../../sql-reference/data-types/float.md) 数値データ型の入力。 +- [Float64](../../sql-reference/data-types/float.md) 数値データ型入力の場合。 - [日付](../../sql-reference/data-types/date.md) 入力値が `Date` タイプ。 - [DateTime](../../sql-reference/data-types/datetime.md) 入力値が `DateTime` タイプ。 -**例えば** +**例** クエリ: @@ -1029,18 +1148,18 @@ SELECT quantileExact(number) FROM numbers(10) └───────────────────────┘ ``` -**また見なさい** +**も参照。** - [中央値](#median) - [分位数](#quantiles) ## quantileExactWeighted {#quantileexactweighted} -正確に計算する [分位値](https://en.wikipedia.org/wiki/Quantile) 各要素の重みを考慮した数値データシーケンス。 +正確に計算する [分位数](https://en.wikipedia.org/wiki/Quantile) 各要素の重みを考慮して、数値データシーケンスの重みを指定します。 To get exact value, all the passed values ​​are combined into an array, which is then partially sorted. Each value is counted with its weight, as if it is present `weight` times. A hash table is used in the algorithm. Because of this, if the passed values ​​are frequently repeated, the function consumes less RAM than [quantileExact](#quantileexact). この関数は、次の代わりに使用できます `quantileExact` そして、重み1を指定します。 -複数を使用する場合 `quantile*` クエリの異なるレベルを持つ関数は、内部状態が結合されていません（つまり、クエリはそれほど効率的ではありません）。 この場合は、 [分位数](#quantiles) 機能。 +複数を使用する場合 `quantile*` クエリ内の異なるレベルを持つ関数では、内部状態は結合されません(つまり、クエリの動作ができるほど効率的ではありません)。 この場合、 [分位数](#quantiles) 機能。 **構文** @@ -1048,11 +1167,11 @@ To get exact value, all the passed values ​​are combined into an array, whic quantileExactWeighted(level)(expr, weight) ``` -エイリアス: `medianExactWeighted`. +別名: `medianExactWeighted`. **パラメータ** -- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` の範囲内の値 `[0.01, 0.99]`. デフォルト値:0.5. で `level=0.5` 機能は計算する [中央値](https://en.wikipedia.org/wiki/Median). +- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` の範囲の値 `[0.01, 0.99]`. デフォルト値は0.5です。 で `level=0.5` この関数は [中央値](https://en.wikipedia.org/wiki/Median). - `expr` — Expression over the column values resulting in numeric [データ型](../../sql-reference/data-types/index.md#data_types), [日付](../../sql-reference/data-types/date.md) または [DateTime](../../sql-reference/data-types/datetime.md). - `weight` — Column with weights of sequence members. Weight is a number of value occurrences. @@ -1062,11 +1181,11 @@ quantileExactWeighted(level)(expr, weight) タイプ: -- [Float64](../../sql-reference/data-types/float.md) 数値データ型の入力。 +- [Float64](../../sql-reference/data-types/float.md) 数値データ型入力の場合。 - [日付](../../sql-reference/data-types/date.md) 入力値が `Date` タイプ。 - [DateTime](../../sql-reference/data-types/datetime.md) 入力値が `DateTime` タイプ。 -**例えば** +**例** 入力テーブル: @@ -1093,18 +1212,18 @@ SELECT quantileExactWeighted(n, val) FROM t └───────────────────────────────┘ ``` -**また見なさい** +**も参照。** - [中央値](#median) - [分位数](#quantiles) ## クオンタイミング {#quantiletiming} -決定された精度では、 [分位値](https://en.wikipedia.org/wiki/Quantile) 数値データシーケンス。 +決定された精度では、 [分位数](https://en.wikipedia.org/wiki/Quantile) 数値データシーケンスの。 -結果は決定的です（クエリ処理の順序に依存しません）。 この機能を最適化と配列における分布のような積載ウェブページではバックエンド対応。 +結果は決定的です（クエリ処理順序に依存しません）。 この機能を最適化と配列における分布のような積載ウェブページではバックエンド対応。 -複数を使用する場合 `quantile*` クエリの異なるレベルを持つ関数は、内部状態が結合されていません（つまり、クエリはそれほど効率的ではありません）。 この場合は、 [分位数](#quantiles) 機能。 +複数を使用する場合 `quantile*` クエリ内の異なるレベルを持つ関数では、内部状態は結合されません(つまり、クエリの動作ができるほど効率的ではありません)。 この場合、 [分位数](#quantiles) 機能。 **構文** @@ -1112,13 +1231,13 @@ SELECT quantileExactWeighted(n, val) FROM t quantileTiming(level)(expr) ``` -エイリアス: `medianTiming`. +別名: `medianTiming`. **パラメータ** -- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` の範囲内の値 `[0.01, 0.99]`. デフォルト値:0.5. で `level=0.5` 機能は計算する [中央値](https://en.wikipedia.org/wiki/Median). +- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` の範囲の値 `[0.01, 0.99]`. デフォルト値は0.5です。 で `level=0.5` この関数は [中央値](https://en.wikipedia.org/wiki/Median). -- `expr` — [式](../syntax.md#syntax-expressions) aを返す列の値を超える [フロート\*](../../sql-reference/data-types/float.md)-タイプ番号。 +- `expr` — [式](../syntax.md#syntax-expressions) aを返す列の値に対して [フロート\*](../../sql-reference/data-types/float.md)-タイプ番号。 - If negative values are passed to the function, the behavior is undefined. - If the value is greater than 30,000 (a page loading time of more than 30 seconds), it is assumed to be 30,000. @@ -1128,12 +1247,12 @@ quantileTiming(level)(expr) 計算は次の場合に正確です: - 値の総数は5670を超えません。 -- 値の総数は5670を超えていますが、ページの読み込み時間は1024ms未満です。 +- 値の総数は5670を超えますが、ページの読み込み時間は1024ms未満です。 それ以外の場合、計算の結果は16msの最も近い倍数に丸められます。 -!!! note "メモ" - ページの読み込み時間の分位数を計算するために、この関数はより効果的で正確です [分位値](#quantile). +!!! note "注" + ページの読み込み時間の分位数を計算するために、この機能はより有効、正確です [分位数](#quantile). **戻り値** @@ -1141,10 +1260,10 @@ quantileTiming(level)(expr) タイプ: `Float32`. -!!! note "メモ" - 関数に値が渡されない場合（以下を使用する場合 `quantileTimingIf`), [ナン](../../sql-reference/data-types/float.md#data_type-float-nan-inf) 返されます。 この目的は、これらのケースをゼロになるケースと区別することです。 見る [ORDER BY句](../statements/select.md#select-order-by) ソートに関する注意事項 `NaN` 値。 +!!! note "注" + 関数に値が渡されない場合（使用する場合 `quantileTimingIf`), [ナン](../../sql-reference/data-types/float.md#data_type-float-nan-inf) 返されます。 この目的は、これらのケースをゼロになるケースと区別することです。 見る [ORDER BY句](../statements/select/order-by.md#select-order-by) ソートに関する注意事項 `NaN` 値。 -**例えば** +**例** 入力テーブル: @@ -1176,18 +1295,18 @@ SELECT quantileTiming(response_time) FROM t └───────────────────────────────┘ ``` -**また見なさい** +**も参照。** - [中央値](#median) - [分位数](#quantiles) -## quantitimingweighted {#quantiletimingweighted} +## quantitetimingweighted {#quantiletimingweighted} -決定された精度では、 [分位値](https://en.wikipedia.org/wiki/Quantile) 各シーケンスメンバの重みに応じた数値データシーケンス。 +決定された精度では、 [分位数](https://en.wikipedia.org/wiki/Quantile) 各シーケンスメンバの重みに応じた数値データシーケンスの。 -結果は決定的です（クエリ処理の順序に依存しません）。 この機能を最適化と配列における分布のような積載ウェブページではバックエンド対応。 +結果は決定的です（クエリ処理順序に依存しません）。 この機能を最適化と配列における分布のような積載ウェブページではバックエンド対応。 -複数を使用する場合 `quantile*` クエリの異なるレベルを持つ関数は、内部状態が結合されていません（つまり、クエリはそれほど効率的ではありません）。 この場合は、 [分位数](#quantiles) 機能。 +複数を使用する場合 `quantile*` クエリ内の異なるレベルを持つ関数では、内部状態は結合されません(つまり、クエリの動作ができるほど効率的ではありません)。 この場合、 [分位数](#quantiles) 機能。 **構文** @@ -1195,13 +1314,13 @@ SELECT quantileTiming(response_time) FROM t quantileTimingWeighted(level)(expr, weight) ``` -エイリアス: `medianTimingWeighted`. +別名: `medianTimingWeighted`. **パラメータ** -- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` の範囲内の値 `[0.01, 0.99]`. デフォルト値:0.5. で `level=0.5` 機能は計算する [中央値](https://en.wikipedia.org/wiki/Median). +- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` の範囲の値 `[0.01, 0.99]`. デフォルト値は0.5です。 で `level=0.5` この関数は [中央値](https://en.wikipedia.org/wiki/Median). -- `expr` — [式](../syntax.md#syntax-expressions) aを返す列の値を超える [フロート\*](../../sql-reference/data-types/float.md)-タイプ番号。 +- `expr` — [式](../syntax.md#syntax-expressions) aを返す列の値に対して [フロート\*](../../sql-reference/data-types/float.md)-タイプ番号。 - If negative values are passed to the function, the behavior is undefined. - If the value is greater than 30,000 (a page loading time of more than 30 seconds), it is assumed to be 30,000. @@ -1213,12 +1332,12 @@ quantileTimingWeighted(level)(expr, weight) 計算は次の場合に正確です: - 値の総数は5670を超えません。 -- 値の総数は5670を超えていますが、ページの読み込み時間は1024ms未満です。 +- 値の総数は5670を超えますが、ページの読み込み時間は1024ms未満です。 それ以外の場合、計算の結果は16msの最も近い倍数に丸められます。 -!!! note "メモ" - ページの読み込み時間の分位数を計算するために、この関数はより効果的で正確です [分位値](#quantile). +!!! note "注" + ページの読み込み時間の分位数を計算するために、この機能はより有効、正確です [分位数](#quantile). **戻り値** @@ -1226,10 +1345,10 @@ quantileTimingWeighted(level)(expr, weight) タイプ: `Float32`. -!!! note "メモ" - 関数に値が渡されない場合（以下を使用する場合 `quantileTimingIf`), [ナン](../../sql-reference/data-types/float.md#data_type-float-nan-inf) 返されます。 この目的は、これらのケースをゼロになるケースと区別することです。 見る [ORDER BY句](../statements/select.md#select-order-by) ソートに関する注意事項 `NaN` 値。 +!!! note "注" + 関数に値が渡されない場合（使用する場合 `quantileTimingIf`), [ナン](../../sql-reference/data-types/float.md#data_type-float-nan-inf) 返されます。 この目的は、これらのケースをゼロになるケースと区別することです。 見る [ORDER BY句](../statements/select/order-by.md#select-order-by) ソートに関する注意事項 `NaN` 値。 -**例えば** +**例** 入力テーブル: @@ -1258,20 +1377,20 @@ SELECT quantileTimingWeighted(response_time, weight) FROM t └───────────────────────────────────────────────┘ ``` -**また見なさい** +**も参照。** - [中央値](#median) - [分位数](#quantiles) -## quantiletdigestcomment {#quantiletdigest} +## quantileTDigest {#quantiletdigest} -近似値を計算します [分位値](https://en.wikipedia.org/wiki/Quantile) を使用する数値データシーケンスの [t-ダイジェスト](https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf) アルゴリズムだ +近似を計算します [分位数](https://en.wikipedia.org/wiki/Quantile) を用いた数値データシーケンスの [t-ダイジェスト](https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf) アルゴリズム -最大誤差は1%です。 メモリ消費量は `log(n)`、どこ `n` 値の数です。 結果は、クエリの実行順序によって異なり、非決定的です。 +最大誤差は1%です。 メモリ消費は `log(n)`,ここで `n` 値の数です。 結果は、クエリの実行順序に依存し、非決定的です。 -機能の性能は性能より低いですの [分位値](#quantile) または [クオンタイミング](#quantiletiming). 状態サイズと精度の比に関しては、この関数はよりもはるかに優れています `quantile`. +機能の性能はの性能より低いです [分位数](#quantile) または [クオンタイミング](#quantiletiming). 状態サイズと精度の比に関しては、この関数は `quantile`. -複数を使用する場合 `quantile*` クエリの異なるレベルを持つ関数は、内部状態が結合されていません（つまり、クエリはそれほど効率的ではありません）。 この場合は、 [分位数](#quantiles) 機能。 +複数を使用する場合 `quantile*` クエリ内の異なるレベルを持つ関数では、内部状態は結合されません(つまり、クエリの動作ができるほど効率的ではありません)。 この場合、 [分位数](#quantiles) 機能。 **構文** @@ -1279,24 +1398,24 @@ SELECT quantileTimingWeighted(response_time, weight) FROM t quantileTDigest(level)(expr) ``` -エイリアス: `medianTDigest`. +別名: `medianTDigest`. **パラメータ** -- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` の範囲内の値 `[0.01, 0.99]`. デフォルト値:0.5. で `level=0.5` 機能は計算する [中央値](https://en.wikipedia.org/wiki/Median). +- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` の範囲の値 `[0.01, 0.99]`. デフォルト値は0.5です。 で `level=0.5` この関数は [中央値](https://en.wikipedia.org/wiki/Median). - `expr` — Expression over the column values resulting in numeric [データ型](../../sql-reference/data-types/index.md#data_types), [日付](../../sql-reference/data-types/date.md) または [DateTime](../../sql-reference/data-types/datetime.md). **戻り値** -- 指定したレベルの概算値。 +- 指定されたレベルの近似分位値。 タイプ: -- [Float64](../../sql-reference/data-types/float.md) 数値データ型の入力。 +- [Float64](../../sql-reference/data-types/float.md) 数値データ型入力の場合。 - [日付](../../sql-reference/data-types/date.md) 入力値が `Date` タイプ。 - [DateTime](../../sql-reference/data-types/datetime.md) 入力値が `DateTime` タイプ。 -**例えば** +**例** クエリ: @@ -1312,20 +1431,20 @@ SELECT quantileTDigest(number) FROM numbers(10) └─────────────────────────┘ ``` -**また見なさい** +**も参照。** - [中央値](#median) - [分位数](#quantiles) ## quantileTDigestWeighted {#quantiletdigestweighted} -近似値を計算します [分位値](https://en.wikipedia.org/wiki/Quantile) を使用する数値データシーケンスの [t-ダイジェスト](https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf) アルゴリズムだ この関数は、各シーケンスメンバーの重みを考慮に入れます。 最大誤差は1%です。 メモリ消費量は `log(n)`、どこ `n` 値の数です。 +近似を計算します [分位数](https://en.wikipedia.org/wiki/Quantile) を用いた数値データシーケンスの [t-ダイジェスト](https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf) アルゴリズム この関数は、各シーケンスメンバーの重みを考慮に入れます。 最大誤差は1%です。 メモリ消費は `log(n)`,ここで `n` 値の数です。 -機能の性能は性能より低いですの [分位値](#quantile) または [クオンタイミング](#quantiletiming). 状態サイズと精度の比に関しては、この関数はよりもはるかに優れています `quantile`. +機能の性能はの性能より低いです [分位数](#quantile) または [クオンタイミング](#quantiletiming). 状態サイズと精度の比に関しては、この関数は `quantile`. -結果は、クエリの実行順序によって異なり、非決定的です。 +結果は、クエリの実行順序に依存し、非決定的です。 -複数を使用する場合 `quantile*` クエリの異なるレベルを持つ関数は、内部状態が結合されていません（つまり、クエリはそれほど効率的ではありません）。 この場合は、 [分位数](#quantiles) 機能。 +複数を使用する場合 `quantile*` クエリ内の異なるレベルを持つ関数では、内部状態は結合されません(つまり、クエリの動作ができるほど効率的ではありません)。 この場合、 [分位数](#quantiles) 機能。 **構文** @@ -1333,25 +1452,25 @@ SELECT quantileTDigest(number) FROM numbers(10) quantileTDigest(level)(expr) ``` -エイリアス: `medianTDigest`. +別名: `medianTDigest`. **パラメータ** -- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` の範囲内の値 `[0.01, 0.99]`. デフォルト値:0.5. で `level=0.5` 機能は計算する [中央値](https://en.wikipedia.org/wiki/Median). +- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` の範囲の値 `[0.01, 0.99]`. デフォルト値は0.5です。 で `level=0.5` この関数は [中央値](https://en.wikipedia.org/wiki/Median). - `expr` — Expression over the column values resulting in numeric [データ型](../../sql-reference/data-types/index.md#data_types), [日付](../../sql-reference/data-types/date.md) または [DateTime](../../sql-reference/data-types/datetime.md). - `weight` — Column with weights of sequence elements. Weight is a number of value occurrences. **戻り値** -- 指定したレベルの概算値。 +- 指定されたレベルの近似分位値。 タイプ: -- [Float64](../../sql-reference/data-types/float.md) 数値データ型の入力。 +- [Float64](../../sql-reference/data-types/float.md) 数値データ型入力の場合。 - [日付](../../sql-reference/data-types/date.md) 入力値が `Date` タイプ。 - [DateTime](../../sql-reference/data-types/datetime.md) 入力値が `DateTime` タイプ。 -**例えば** +**例** クエリ: @@ -1367,7 +1486,7 @@ SELECT quantileTDigestWeighted(number, 1) FROM numbers(10) └────────────────────────────────────┘ ``` -**また見なさい** +**も参照。** - [中央値](#median) - [分位数](#quantiles) @@ -1376,18 +1495,18 @@ SELECT quantileTDigestWeighted(number, 1) FROM numbers(10) その `median*` 関数は、対応する関数のエイリアスです `quantile*` 機能。 数値データサンプルの中央値を計算します。 -機能: +関数: -- `median` — Alias for [分位値](#quantile). -- `medianDeterministic` — Alias for [quantiedeterministic](#quantiledeterministic). +- `median` — Alias for [分位数](#quantile). +- `medianDeterministic` — Alias for [quantileDeterministic](#quantiledeterministic). - `medianExact` — Alias for [quantileExact](#quantileexact). - `medianExactWeighted` — Alias for [quantileExactWeighted](#quantileexactweighted). - `medianTiming` — Alias for [クオンタイミング](#quantiletiming). -- `medianTimingWeighted` — Alias for [quantitimingweighted](#quantiletimingweighted). -- `medianTDigest` — Alias for [quantiletdigestcomment](#quantiletdigest). +- `medianTimingWeighted` — Alias for [quantitetimingweighted](#quantiletimingweighted). +- `medianTDigest` — Alias for [quantileTDigest](#quantiletdigest). - `medianTDigestWeighted` — Alias for [quantileTDigestWeighted](#quantiletdigestweighted). -**例えば** +**例** 入力テーブル: @@ -1416,41 +1535,53 @@ SELECT medianDeterministic(val, 1) FROM t ## quantiles(level1, level2, …)(x) {#quantiles} -すべての分位数関数には、対応する分位数関数もあります: `quantiles`, `quantilesDeterministic`, `quantilesTiming`, `quantilesTimingWeighted`, `quantilesExact`, `quantilesExactWeighted`, `quantilesTDigest`. これらの関数は、あるパスでリストされたレベルのすべての分位数を計算し、結果の値の配列を返します。 +すべての分位関数にも対応する分位関数があります: `quantiles`, `quantilesDeterministic`, `quantilesTiming`, `quantilesTimingWeighted`, `quantilesExact`, `quantilesExactWeighted`, `quantilesTDigest`. これらの関数は、リストされたレベルのすべての分位数を一つのパスで計算し、結果の値の配列を返します。 ## varSamp(x) {#varsampx} -金額を計算します `Σ((x - x̅)^2) / (n - 1)`、どこ `n` サンプルサイズは `x̅`の平均値です `x`. +金額を計算します `Σ((x - x̅)^2) / (n - 1)`,ここで `n` はサンプルサイズであり、 `x̅`の平均値です `x`. -これは、渡された値がそのサンプルを形成する場合、確率変数の分散の不偏推定値を表します。 +渡された値がその標本を形成する場合、確率変数の分散の不偏推定値を表します。 -を返します `Float64`. とき `n <= 1`、戻り値 `+∞`. +ﾂづｩﾂ。 `Float64`. とき `n <= 1`,戻り値 `+∞`. + +!!! note "注" + この機能の利用を数値的に不安定なアルゴリズムです。 必要とすれば [数値安定性](https://en.wikipedia.org/wiki/Numerical_stability) 計算では、 `varSampStable` 機能。 それは遅く動作しますが、計算誤差は低くなります。 ## varPop(x) {#varpopx} -金額を計算します `Σ((x - x̅)^2) / n`、どこ `n` サンプルサイズは `x̅`の平均値です `x`. +金額を計算します `Σ((x - x̅)^2) / n`,ここで `n` はサンプルサイズであり、 `x̅`の平均値です `x`. -つまり、値のセットの分散。 を返します `Float64`. +言い換えれば、値の集合に対する分散。 ﾂづｩﾂ。 `Float64`. + +!!! note "注" + この機能の利用を数値的に不安定なアルゴリズムです。 必要とすれば [数値安定性](https://en.wikipedia.org/wiki/Numerical_stability) 計算では、 `varPopStable` 機能。 それは遅く動作しますが、計算誤差は低くなります。 ## stddevSamp(x) {#stddevsampx} -結果はの平方根に等しい `varSamp(x)`. +結果はの平方根に等しくなります `varSamp(x)`. + +!!! note "注" + この機能の利用を数値的に不安定なアルゴリズムです。 必要とすれば [数値安定性](https://en.wikipedia.org/wiki/Numerical_stability) 計算では、 `stddevSampStable` 機能。 それは遅く動作しますが、計算誤差は低くなります。 ## stddevPop(x) {#stddevpopx} -結果はの平方根に等しい `varPop(x)`. +結果はの平方根に等しくなります `varPop(x)`. + +!!! note "注" + この機能の利用を数値的に不安定なアルゴリズムです。 必要とすれば [数値安定性](https://en.wikipedia.org/wiki/Numerical_stability) 計算では、 `stddevPopStable` 機能。 それは遅く動作しますが、計算誤差は低くなります。 ## topK(N)(x) {#topknx} -指定された列のほぼ最も頻繁に使用される値の配列を返します。 結果の配列は、値のおおよその頻度の降順でソートされます（値そのものではありません）。 +指定された列の中で最も頻度の高い値の配列を返します。 結果の配列は、値の近似頻度の降順でソートされます（値自体ではありません）。 -実装する [ろ過されたスペース節約](http://www.l2f.inesc-id.pt/~fmmb/wiki/uploads/Work/misnis.ref0a.pdf) からのreduce-and-combineアルゴリズムに基づいてTopKを分析するアルゴリズム [パラレル省スペース](https://arxiv.org/pdf/1401.0702.pdf). +を実装する。 [ろ過されたスペース節約](http://www.l2f.inesc-id.pt/~fmmb/wiki/uploads/Work/misnis.ref0a.pdf) TopKを解析するためのアルゴリズム。 [並列スペース節約](https://arxiv.org/pdf/1401.0702.pdf). ``` sql topK(N)(column) ``` -この関数は保証された結果を提供しません。 特定の状況では、エラーが発生し、最も頻度の高い値ではない頻繁な値が返されることがあります。 +この関数は保証された結果を提供しません。 特定の状況では、エラーが発生し、最も頻繁な値ではない頻繁な値が返されることがあります。 私達は使用を推薦します `N < 10` 価値;性能は大きいと減ります `N` 値。 の最大値 `N = 65536`. @@ -1458,15 +1589,15 @@ topK(N)(column) - ‘N’ 返す要素の数です。 -パラメーターを省略すると、既定値10が使用されます。 +このパラメーターを省略すると、既定値10が使用されます。 **引数** -- ’ x ’ – The value to calculate frequency. +- ' x ' – The value to calculate frequency. -**例えば** +**例** -を取る [オンタイム](../../getting-started/example-datasets/ontime.md) データセットを選択し、最も頻繁に発生する三つの値を選択します。 `AirlineID` コラム +を取る [オンタイム](../../getting-started/example-datasets/ontime.md) データセットで最も頻繁に発生する三つの値を選択します。 `AirlineID` 列。 ``` sql SELECT topK(3)(AirlineID) AS res @@ -1479,9 +1610,9 @@ FROM ontime └─────────────────────┘ ``` -## トップクイット {#topkweighted} +## トップウェイト {#topkweighted} -に似て `topK` しかし、整数型の一つの追加の引数を取ります - `weight`. あらゆる価値は説明されます `weight` 頻度計算のための時間。 +に類似した `topK` しかし、整数型の追加引数を取ります - `weight`. すべての値が考慮されます `weight` 周波数計算のための時間。 **構文** @@ -1500,9 +1631,9 @@ topKWeighted(N)(x, weight) **戻り値** -最大およその重みの合計を持つ値の配列を返します。 +おおよその重みの合計が最大の値の配列を返します。 -**例えば** +**例** クエリ: @@ -1518,33 +1649,42 @@ SELECT topKWeighted(10)(number, number) FROM numbers(1000) └───────────────────────────────────────────┘ ``` -## covarSamp(x,y) {#covarsampx-y} +## コバルサンプ(x,y) {#covarsampx-y} の値を計算します `Σ((x - x̅)(y - y̅)) / (n - 1)`. Float64を返します。 とき `n <= 1`, returns +∞. -## covarPop(x,y) {#covarpopx-y} +!!! note "注" + この機能の利用を数値的に不安定なアルゴリズムです。 必要とすれば [数値安定性](https://en.wikipedia.org/wiki/Numerical_stability) 計算では、 `covarSampStable` 機能。 それは遅く動作しますが、計算誤差は低くなります。 + +## コバルポップ(x,y) {#covarpopx-y} の値を計算します `Σ((x - x̅)(y - y̅)) / n`. +!!! note "注" + この機能の利用を数値的に不安定なアルゴリズムです。 必要とすれば [数値安定性](https://en.wikipedia.org/wiki/Numerical_stability) 計算では、 `covarPopStable` 機能。 それは遅く動作しますが、計算誤差は低くなります。 + ## corr(x,y) {#corrx-y} ピアソン相関係数を計算します: `Σ((x - x̅)(y - y̅)) / sqrt(Σ((x - x̅)^2) * Σ((y - y̅)^2))`. +!!! note "注" + この機能の利用を数値的に不安定なアルゴリズムです。 必要とすれば [数値安定性](https://en.wikipedia.org/wiki/Numerical_stability) 計算では、 `corrStable` 機能。 それは遅く動作しますが、計算誤差は低くなります。 + ## categoricalInformationValue {#categoricalinformationvalue} -の値を計算します `(P(tag = 1) - P(tag = 0))(log(P(tag = 1)) - log(P(tag = 0)))` 各カテゴリの。 +の値を計算します `(P(tag = 1) - P(tag = 0))(log(P(tag = 1)) - log(P(tag = 0)))` カテゴリごとに。 ``` sql categoricalInformationValue(category1, category2, ..., tag) ``` -結果は、離散(カテゴリカル)フィーチャがどのようにして `[category1, category2, ...]` の値を予測する学習モデルに貢献する `tag`. +結果は、離散(カテゴリカル)フィーチャの方法を示します `[category1, category2, ...]` の値を予測する学習モデルに貢献する `tag`. -## simplelearregression {#simplelinearregression} +## simpleLinearRegression {#simplelinearregression} -単純な(一次元的な)線形回帰を実行します。 +単純(一次元)線形回帰を実行します。 ``` sql simpleLinearRegression(x, y) @@ -1583,31 +1723,31 @@ SELECT arrayReduce('simpleLinearRegression', [0, 1, 2, 3], [3, 4, 5, 6]) ## stochasticLinearRegression {#agg_functions-stochasticlinearregression} -この関数は、確率的線形回帰を実装します。 それは率、l2正則化係数、ミニバッチサイズを学ぶための注文変数を支え、重量を更新するための少数の方法を有する ([アダム](https://en.wikipedia.org/wiki/Stochastic_gradient_descent#Adam) （デフォルトで使用), [シンプルSGD](https://en.wikipedia.org/wiki/Stochastic_gradient_descent), [勢い](https://en.wikipedia.org/wiki/Stochastic_gradient_descent#Momentum), [ネステロフ](https://mipt.ru/upload/medialibrary/d7e/41-91.pdf)). +この関数は、確率的線形回帰を実装します。 学習率、L2正則化係数、ミニバッチサイズのカスタムパラメータをサポートし、重みを更新する方法はほとんどありません ([アダム](https://en.wikipedia.org/wiki/Stochastic_gradient_descent#Adam) （デフォルトで使用), [シンプルSGD](https://en.wikipedia.org/wiki/Stochastic_gradient_descent), [勢い](https://en.wikipedia.org/wiki/Stochastic_gradient_descent#Momentum), [ネステロフ](https://mipt.ru/upload/medialibrary/d7e/41-91.pdf)). ### パラメータ {#agg_functions-stochasticlinearregression-parameters} -が4カスタマイズ可能パラメータ。 それらは関数に順番に渡されますが、すべての四つのデフォルト値を渡す必要はありませんが、良いモデルにはいくつかのパラメータ調整が必要で +が4カスタマイズ可能パラメータ。 デフォルト値が使用されますが、良いモデルではパラメータの調整が必要です。 ``` text stochasticLinearRegression(1.0, 1.0, 10, 'SGD') ``` -1. `learning rate` 勾配降下ステップが実行されるときのステップ長の係数です。 大きすぎる学習率の原因となり無限の量のモデルです。 デフォルトは `0.00001`. -2. `l2 regularization coefficient` これは過食防止に役立つ可能性があります。 デフォルトは `0.1`. -3. `mini-batch size` グラデーションディセントのステップを実行するために、グラデーションを計算して合計する要素の数を設定します。 純粋な確率降下は一つの要素を使用しますが、小さなバッチ（約10の要素）を持つことで勾配ステップがより安定します。 デフォルトは `15`. -4. `method for updating weights`、彼らは: `Adam` （デフォルトでは), `SGD`, `Momentum`, `Nesterov`. `Momentum` と `Nesterov` もう少し計算とメモリが必要ですが、確率勾配法の収束と安定性の点で有用です。 +1. `learning rate` 勾配降下ステップを実行したときのステップ長の係数です。 大きすぎる学習率の原因となり無限の量のモデルです。 デフォルトは `0.00001`. +2. `l2 regularization coefficient` 過適合を防ぐのに役立つかもしれません。 デフォルトは `0.1`. +3. `mini-batch size` グラデーションが計算され、グラデーション降下のステップを実行するために合計される要素の数を設定します。 純粋な確率降下は一つの要素を使用しますが、小さなバッチ（約10要素）を持つと勾配ステップがより安定します。 デフォルトは `15`. +4. `method for updating weights`、彼らは: `Adam` （デフォルトでは), `SGD`, `Momentum`, `Nesterov`. `Momentum` と `Nesterov` もう少し多くの計算とメモリを必要とするが、収束の速度と確率勾配法の安定性の点で有用であることが起こる。 -### 使い方 {#agg_functions-stochasticlinearregression-usage} +### 使用法 {#agg_functions-stochasticlinearregression-usage} -`stochasticLinearRegression` モデルのフィッティングと新しいデータの予測です。 モデルを適合させ、後で使用するためにその状態を保存するために、 `-State` 基本的に状態（モデルの重みなど）を保存するcombinator。 -予測するには、関数を使用します [evalMLMethod](../functions/machine-learning-functions.md#machine_learning_methods-evalmlmethod) これは、状態を予測する機能と同様に引数として取ります。 +`stochasticLinearRegression` モデルの近似と新しいデータの予測です。 モデルを適合させ、後で使用するためにその状態を保存するために、 `-State` 基本的に状態（モデルの重みなど）を保存するcombinator。 +予測するには関数を使います [evalMLMethod](../functions/machine-learning-functions.md#machine_learning_methods-evalmlmethod) は、状態を引数として取り、予測する機能も備えています。 -**1.** 継手 +**1.** フィット -このようなクエリを使用できます。 +このようなクエリが使用され得る。 ``` sql CREATE TABLE IF NOT EXISTS train_data @@ -1622,12 +1762,12 @@ stochasticLinearRegressionState(0.1, 0.0, 5, 'SGD')(target, param1, param2) AS state FROM train_data; ``` -ここでは、データを挿入する必要もあります `train_data` テーブル。 パラメータの数は固定されていません。 `linearRegressionState`. 彼らはすべての必要数値です。 -ターゲット値（予測することを学びたい）を持つ列が最初の引数として挿入されることに注意してください。 +ここでは、データを挿入する必要もあります `train_data` テーブル。 パラメータの数は固定されておらず、渡された引数の数にのみ依存します `linearRegressionState`. すべて数値でなければなりません。 +ターゲット値を持つ列（予測することを学びたい）が最初の引数として挿入されることに注意してください。 **2.** 予測 -状態をテーブルに保存した後、予測に複数回使用したり、他の状態とマージして新しいモデルを作成したりすることもできます。 +テーブルに状態を保存した後、予測に複数回使用したり、他の状態とマージして新しいより良いモデルを作成したりすることもできます。 ``` sql WITH (SELECT state FROM your_model) AS model SELECT @@ -1638,28 +1778,28 @@ evalMLMethod(model, param1, param2) FROM test_data `test_data` のようなテーブルです `train_data` が含まれないことがあります。 -### 備考 {#agg_functions-stochasticlinearregression-notes} +### ノート {#agg_functions-stochasticlinearregression-notes} 1. 統合モデルにはユーザーの作成などのクエリ: `sql SELECT state1 + state2 FROM your_models` - どこに `your_models` テーブルの両方のモデルです。 このクエリはnewを返します `AggregateFunctionState` オブジェクト。 + どこに `your_models` テーブルの両方のモデルです。 このクエリはnewを返します `AggregateFunctionState` オブジェクト -2. ユーザーのフェッチのウエイトを作成したモデルとして独自の目的で保存しないモデルについていない場合 `-State` combinatorが使用されます。 +2. ユーザーは、作成されたモデルのウェイトを独自の目的で取得することができます。 `-State` combinatorが使用されます。 `sql SELECT stochasticLinearRegression(0.01)(target, param1, param2) FROM train_data` - そのようなクエリはモデルに適合し、その重みを返します-最初はモデルのパラメータに対応する重みです。 したがって、上記の例では、クエリは3つの値を持つ列を返します。 + このようなクエリはモデルに適合し、その重みを返します-最初はモデルのパラメータに対応する重みであり、最後はバイアスです。 したがって、上記の例では、クエリは3つの値を持つ列を返します。 -**また見なさい** +**も参照。** - [stochasticLogisticRegression](#agg_functions-stochasticlogisticregression) -- [線形およびロジスティック回帰の違い](https://stackoverflow.com/questions/12146914/what-is-the-difference-between-linear-regression-and-logistic-regression) +- [線形回帰とロジスティック回帰の違い](https://stackoverflow.com/questions/12146914/what-is-the-difference-between-linear-regression-and-logistic-regression) ## stochasticLogisticRegression {#agg_functions-stochasticlogisticregression} -この関数は、確率論的ロジスティック回帰を実装します。 これは、バイナリ分類問題に使用することができ、stochasticlinearregressionと同じカスタムパラメータをサポートし、同じ方法で動作します。 +この関数は、確率的ロジスティック回帰を実装します。 これは、バイナリ分類問題に使用することができ、stochasticLinearRegressionと同じカスタムパラメータをサポートし、同じように動作します。 ### パラメータ {#agg_functions-stochasticlogisticregression-parameters} -パラメーターは、stochasticlinearregressionとまったく同じです: +パラメーターはstochasticLinearRegressionとまったく同じです: `learning rate`, `l2 regularization coefficient`, `mini-batch size`, `method for updating weights`. 詳細については、 [パラメータ](#agg_functions-stochasticlinearregression-parameters). @@ -1667,7 +1807,7 @@ evalMLMethod(model, param1, param2) FROM test_data stochasticLogisticRegression(1.0, 1.0, 10, 'SGD') ``` -1. 継手 +1. フィット @@ -1700,14 +1840,14 @@ stochasticLogisticRegression(1.0, 1.0, 10, 'SGD') `test_data` is a table like `train_data` but may not contain target value. -**また見なさい** +**も参照。** - [stochasticLinearRegression](#agg_functions-stochasticlinearregression) -- [線形およびロジスティック回帰の違い。](https://stackoverflow.com/questions/12146914/what-is-the-difference-between-linear-regression-and-logistic-regression) +- [線形回帰とロジスティック回帰の違い。](https://stackoverflow.com/questions/12146914/what-is-the-difference-between-linear-regression-and-logistic-regression) -## groupbitmapandgenericname {#groupbitmapand} +## groupBitmapAnd {#groupbitmapand} -ビットマップ列のandを計算し、型uint64のカーディナリティを返します。 [ビットマップ](../../sql-reference/functions/bitmap-functions.md). +ビットマップ列のANDを計算し、uint64型の基数を返します。 [ビットマップ](../../sql-reference/functions/bitmap-functions.md). ``` sql groupBitmapAnd(expr) @@ -1721,7 +1861,7 @@ groupBitmapAnd(expr) の値 `UInt64` タイプ。 -**例えば** +**例** ``` sql DROP TABLE IF EXISTS bitmap_column_expr_test2; @@ -1750,7 +1890,7 @@ SELECT arraySort(bitmapToArray(groupBitmapAndState(z))) FROM bitmap_column_expr_ ## groupBitmapOr {#groupbitmapor} -ビットマップ列のorを計算し、型uint64のカーディナリティを返します。 [ビットマップ](../../sql-reference/functions/bitmap-functions.md). これは `groupBitmapMerge`. +ビットマップ列のORを計算し、uint64型の基数を返します。 [ビットマップ](../../sql-reference/functions/bitmap-functions.md). これは `groupBitmapMerge`. ``` sql groupBitmapOr(expr) @@ -1764,7 +1904,7 @@ groupBitmapOr(expr) の値 `UInt64` タイプ。 -**例えば** +**例** ``` sql DROP TABLE IF EXISTS bitmap_column_expr_test2; @@ -1793,7 +1933,7 @@ SELECT arraySort(bitmapToArray(groupBitmapOrState(z))) FROM bitmap_column_expr_t ## groupBitmapXor {#groupbitmapxor} -ビットマップ列のxorを計算し、uint64型のカーディナリティを返します。 [ビットマップ](../../sql-reference/functions/bitmap-functions.md). +ビットマップ列のXORを計算し、uint64型の基数を返します。 [ビットマップ](../../sql-reference/functions/bitmap-functions.md). ``` sql groupBitmapOr(expr) @@ -1807,7 +1947,7 @@ groupBitmapOr(expr) の値 `UInt64` タイプ。 -**例えば** +**例** ``` sql DROP TABLE IF EXISTS bitmap_column_expr_test2; diff --git a/docs/ja/sql-reference/ansi.md b/docs/ja/sql-reference/ansi.md deleted file mode 120000 index 3cf6bffed67..00000000000 --- a/docs/ja/sql-reference/ansi.md +++ /dev/null @@ -1 +0,0 @@ -../../en/sql-reference/ansi.md \ No newline at end of file diff --git a/docs/ja/sql-reference/ansi.md b/docs/ja/sql-reference/ansi.md new file mode 100644 index 00000000000..246036125b2 --- /dev/null +++ b/docs/ja/sql-reference/ansi.md @@ -0,0 +1,180 @@ +--- +machine_translated: true +machine_translated_rev: ad252bbb4f7e2899c448eb42ecc39ff195c8faa1 +toc_priority: 40 +toc_title: "ANSI\u306E\u4E92\u63DB\u6027" +--- + +# ClickHouse SQLの方言のANSI SQLの互換性 {#ansi-sql-compatibility-of-clickhouse-sql-dialect} + +!!! note "注" + この記事では、表38に依存しています, “Feature taxonomy and definition for mandatory features”, Annex F of ISO/IEC CD 9075-2:2013. + +## 行動の違い {#differences-in-behaviour} + +次の表は、ClickHouseでクエリ機能が機能しますが、ANSI SQLで指定されているように動作しないケースを示しています。 + +| Feature ID | 機能名 | 違い | +|------------|----------------------------------|---------------------------------------------------------------------------------| +| E011 | 数値データ型 | 数値リテラルと期間として解釈される近似 (`Float64`）の代わりに正確な (`Decimal`) | +| E051-05 | 選択した項目の名前を変更できます | アイテムの名前変更は、選択結果だけよりも可視性の範囲が広くなります | +| E141-01 | NULLでない制約 | `NOT NULL` は黙示のためのテーブル列によるデフォルト | +| E011-04 | 算術演算子 | ClickHouse溢れの代わりに次の算術演算の結果のデータ型に基づくカスタムルール | + +## 機能の状態 {#feature-status} + +| Feature ID | 機能名 | 状態 | コメント | +|------------|---------------------------------------------------------------------------------------------------|----------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **E011** | **数値データ型** | **部分的**{.text-warning} | | +| E011-01 | 整数およびSMALLINTデータ型 | はい。{.text-success} | | +| E011-02 | 実数、倍精度および浮動小数点データ型データ型 | 部分的{.text-warning} | `FLOAT()`, `REAL` と `DOUBLE PRECISION` 対応していません | +| E011-03 | DECIMALおよびNUMERICデータ型 | 部分的{.text-warning} | のみ `DECIMAL(p,s)` サポートされています。 `NUMERIC` | +| E011-04 | 算術演算子 | はい。{.text-success} | | +| E011-05 | 数値比較 | はい。{.text-success} | | +| E011-06 | 数値データ型間の暗黙的なキャスト | いいえ。{.text-danger} | ANSI SQLできる任意の暗黙的な数値型の間のキャストがClickHouseに依存しての機能を有する複数の過負荷の代わりに暗黙的なキャスト | +| **E021** | **文字列タイプ** | **部分的**{.text-warning} | | +| E021-01 | 文字データ型 | いいえ。{.text-danger} | | +| E021-02 | 文字変化型データ型 | いいえ。{.text-danger} | `String` 動作同様に、長さの制限内 | +| E021-03 | 文字リテラル | 部分的{.text-warning} | 連続したリテラルと文字セットの自動連結はサポートされません | +| E021-04 | CHARACTER\_LENGTH関数 | 部分的{.text-warning} | いいえ。 `USING` 句 | +| E021-05 | OCTET\_LENGTH関数 | いいえ。{.text-danger} | `LENGTH` 同様に動作します | +| E021-06 | SUBSTRING | 部分的{.text-warning} | サポートなし `SIMILAR` と `ESCAPE` 句、ない `SUBSTRING_REGEX` バリアント | +| E021-07 | 文字の連結 | 部分的{.text-warning} | いいえ。 `COLLATE` 句 | +| E021-08 | 上部および下の機能 | はい。{.text-success} | | +| E021-09 | トリム機能 | はい。{.text-success} | | +| E021-10 | 固定長および可変長文字ストリング型間の暗黙的なキャスト | いいえ。{.text-danger} | ANSI SQLできる任意の暗黙の間のキャスト文字列の種類がClickHouseに依存しての機能を有する複数の過負荷の代わりに暗黙的なキャスト | +| E021-11 | 位置関数 | 部分的{.text-warning} | サポートなし `IN` と `USING` 句、ない `POSITION_REGEX` バリアント | +| E021-12 | 文字の比較 | はい。{.text-success} | | +| **E031** | **識別子** | **部分的**{.text-warning} | | +| E031-01 | 区切り識別子 | 部分的{.text-warning} | Unicodeリテラルの支援は限られ | +| E031-02 | 小文字の識別子 | はい。{.text-success} | | +| E031-03 | 末尾のアンダースコア | はい。{.text-success} | | +| **E051** | **基本的なクエリ仕様** | **部分的**{.text-warning} | | +| E051-01 | SELECT DISTINCT | はい。{.text-success} | | +| E051-02 | GROUP BY句 | はい。{.text-success} | | +| E051-04 | グループによる列を含むことができない `
MODIFY SETTING = `. [\#6366](https://github.com/ClickHouse/ClickHouse/pull/6366) [\#6669](https://github.com/ClickHouse/ClickHouse/pull/6669) [\#6685](https://github.com/ClickHouse/ClickHouse/pull/6685) ([alesapin](https://github.com/alesapin)) -- 取り外した部品の取り外しをサポート。 構文: `ALTER TABLE DROP DETACHED PART ''`. [\#6158](https://github.com/ClickHouse/ClickHouse/pull/6158) ([tavplubix](https://github.com/tavplubix)) -- テーブルの制約。 挿入時にチェックされるテーブル定義に制約を追加することができます。 [\#5273](https://github.com/ClickHouse/ClickHouse/pull/5273) ([Gleb Novikov](https://github.com/NanoBjorn)) [\#6652](https://github.com/ClickHouse/ClickHouse/pull/6652) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 支援のためのカスケード型を実現します。 [\#6324](https://github.com/ClickHouse/ClickHouse/pull/6324) ([アモスの鳥](https://github.com/amosbird)) -- On queryプロファイラのデフォルトでサンプル毎にクエリの実行スレッドだ。 [\#6283](https://github.com/ClickHouse/ClickHouse/pull/6283) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- `WITH FILL` 修飾子のための `ORDER BY`. （の続き [\#5069](https://github.com/ClickHouse/ClickHouse/issues/5069)) [\#6610](https://github.com/ClickHouse/ClickHouse/pull/6610) ([アントン-ポポフ](https://github.com/CurtizJ)) +- `WITH TIES` 修飾子のための `LIMIT`. （の続き [\#5069](https://github.com/ClickHouse/ClickHouse/issues/5069)) [\#6610](https://github.com/ClickHouse/ClickHouse/pull/6610) ([アントン-ポポフ](https://github.com/CurtizJ)) +- 引用符なしの解析 `NULL` NULLとしてリテラル(設定の場合 `format_csv_unquoted_null_literal_as_null=1`). このフィールドのデータ型がnull許容でない場合（設定の場合）、nullフィールドを既定値で初期化します `input_format_null_as_default=1`). [\#5990](https://github.com/ClickHouse/ClickHouse/issues/5990) [\#6055](https://github.com/ClickHouse/ClickHouse/pull/6055) ([tavplubix](https://github.com/tavplubix)) +- 表関数のパスでのワイルドカードのサポート `file` と `hdfs`. パスにワイルドカードが含まれる場合、テーブルはreadonlyになります。 使用例: `select * from hdfs('hdfs://hdfs1:9000/some_dir/another_dir/*/file{0..9}{0..9}')` と `select * from file('some_dir/{some_file,another_file,yet_another}.tsv', 'TSV', 'value UInt32')`. [\#6092](https://github.com/ClickHouse/ClickHouse/pull/6092) ([Olga Khvostikova](https://github.com/stavrolia)) +- 新しい `system.metric_log` の値を格納するテーブル `system.events` と `system.metrics` 指定された時間間隔で。 [\#6363](https://github.com/ClickHouse/ClickHouse/issues/6363) [\#6467](https://github.com/ClickHouse/ClickHouse/pull/6467) ([ニキータ-ミハイロフ](https://github.com/nikitamikhaylov)) [\#6530](https://github.com/ClickHouse/ClickHouse/pull/6530) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- ClickHouseテキストログの書き込みを許可する `system.text_log` テーブル。 [\#6037](https://github.com/ClickHouse/ClickHouse/issues/6037) [\#6103](https://github.com/ClickHouse/ClickHouse/pull/6103) ([ニキータ-ミハイロフ](https://github.com/nikitamikhaylov)) [\#6164](https://github.com/ClickHouse/ClickHouse/pull/6164) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- ショー民間のシンボルスタックトレース（ターを通じて構文解析シンボルテーブルのELFファイル). 追加情報ファイルと行番号をスタックトレースの場合のデバッグ情報があります。 高速シンボル名のルックアップインデックスの記号が存在します。 内観のための新しいSQL関数を追加しました: `demangle` と `addressToLine`. 名前変更された関数 `symbolizeAddress` に `addressToSymbol` 一貫性のために。 関数 `addressToSymbol` パフォーマンス上の理由から壊れた名前を返し、適用する必要があります `demangle`. 設定を追加 `allow_introspection_functions` デフォルトではオフになっています。 [\#6201](https://github.com/ClickHouse/ClickHouse/pull/6201) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- テーブル関数 `values` (名前は大文字と小文字を区別しません)。 ることができるから読み込む `VALUES` で提案されたリスト [\#5984](https://github.com/ClickHouse/ClickHouse/issues/5984). 例: `SELECT * FROM VALUES('a UInt64, s String', (1, 'one'), (2, 'two'), (3, 'three'))`. [\#6217](https://github.com/ClickHouse/ClickHouse/issues/6217). [\#6209](https://github.com/ClickHouse/ClickHouse/pull/6209) ([dimarub2000](https://github.com/dimarub2000)) +- ストレージ設定を変更する機能を追加しました。 構文: `ALTER TABLE
MODIFY SETTING = `. [\#6366](https://github.com/ClickHouse/ClickHouse/pull/6366) [\#6669](https://github.com/ClickHouse/ClickHouse/pull/6669) [\#6685](https://github.com/ClickHouse/ClickHouse/pull/6685) ([アレサピン](https://github.com/alesapin)) +- 孤立した部品の取除くためのサポート。 構文: `ALTER TABLE DROP DETACHED PART ''`. [\#6158](https://github.com/ClickHouse/ClickHouse/pull/6158) ([tavplubix](https://github.com/tavplubix)) +- テーブル制約。 挿入時にチェックされるテーブル定義に制約を追加することができます。 [\#5273](https://github.com/ClickHouse/ClickHouse/pull/5273) ([グレブ-ノビコフ](https://github.com/NanoBjorn)) [\#6652](https://github.com/ClickHouse/ClickHouse/pull/6652) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- カスケードマテリアライズドビューのSuppport。 [\#6324](https://github.com/ClickHouse/ClickHouse/pull/6324) ([アモス鳥](https://github.com/amosbird)) +- On queryプロファイラのデフォルトでサンプル毎にクエリの実行スレッドだ。 [\#6283](https://github.com/ClickHouse/ClickHouse/pull/6283) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) - 入力形式 `ORC`. [\#6454](https://github.com/ClickHouse/ClickHouse/pull/6454) [\#6703](https://github.com/ClickHouse/ClickHouse/pull/6703) ([akonyaev90](https://github.com/akonyaev90)) -- 二つの新機能を追加しました: `sigmoid` と `tanh` （これは機械学習アプリケーションに便利です）。 [\#6254](https://github.com/ClickHouse/ClickHouse/pull/6254) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 機能 `hasToken(haystack, token)`, `hasTokenCaseInsensitive(haystack, token)` 指定されたトークンがhaystackにあるかどうかを確認する。 トークンは、二つの非英数字ASCII文字（またはhaystackの境界）の間の最大長の部分文字列です。 トークンを入力する必要がある定数文字列になります。 支tokenbf\_v1指数の専門性を高めます。 [\#6596](https://github.com/ClickHouse/ClickHouse/pull/6596), [\#6662](https://github.com/ClickHouse/ClickHouse/pull/6662) ([Vasily Nemkov](https://github.com/Enmk)) -- 新しい機能 `neighbor(value, offset[, default_value])`. データブロック内の列内の前/次の値に到達することができます。 [\#5925](https://github.com/ClickHouse/ClickHouse/pull/5925) ([Alex Krash](https://github.com/alex-krash)) [6685365ab8c5b74f9650492c88a012596eb1b0c6](https://github.com/ClickHouse/ClickHouse/commit/6685365ab8c5b74f9650492c88a012596eb1b0c6) [341e2e4587a18065c2da1ca888c73389f48ce36c](https://github.com/ClickHouse/ClickHouse/commit/341e2e4587a18065c2da1ca888c73389f48ce36c) [Alexey Milovidov](https://github.com/alexey-milovidov) -- 関数の作成 `currentUser()`、承認されたユーザのログインを返す。 エイリアスを追加 `user()` MySQLとの互換性のために。 [\#6470](https://github.com/ClickHouse/ClickHouse/pull/6470) ([Alex Krash](https://github.com/alex-krash)) -- 新しい集計関数 `quantilesExactInclusive` と `quantilesExactExclusive` これはで提案されました [\#5885](https://github.com/ClickHouse/ClickHouse/issues/5885). [\#6477](https://github.com/ClickHouse/ClickHouse/pull/6477) ([ディマルブ2000](https://github.com/dimarub2000)) -- 機能 `bitmapRange(bitmap, range_begin, range_end)` これは、指定された範囲の新しいセットを返します（ `range_end`). [\#6314](https://github.com/ClickHouse/ClickHouse/pull/6314) ([Zhichang Yu](https://github.com/yuzhichang)) -- 機能 `geohashesInBox(longitude_min, latitude_min, longitude_max, latitude_max, precision)` 提供された区域をカバーするgeohash箱の精密長い一連の配列を作成するかどれが。 [\#6127](https://github.com/ClickHouse/ClickHouse/pull/6127) ([Vasily Nemkov](https://github.com/Enmk)) -- INSERTクエリのサポートを実装する `Kafka` テーブル。 [\#6012](https://github.com/ClickHouse/ClickHouse/pull/6012) ([イワン](https://github.com/abyss7)) -- のサポートを追加 `_partition` と `_timestamp` カフカエンジンへの仮想列。 [\#6400](https://github.com/ClickHouse/ClickHouse/pull/6400) ([イワン](https://github.com/abyss7)) +- 二つの新機能を追加しました: `sigmoid` と `tanh` （機械学習アプリケーションに役立ちます）。 [\#6254](https://github.com/ClickHouse/ClickHouse/pull/6254) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 関数 `hasToken(haystack, token)`, `hasTokenCaseInsensitive(haystack, token)` 指定されたトークンがhaystackにあるかどうかを確認する。 トークンは、英数字でないASCII文字（またはhaystackの境界）の間の最大長の部分文字列です。 Tokenは定数文字列である必要があります。 Tokenbf\_v1インデックス特殊化でサポートされます。 [\#6596](https://github.com/ClickHouse/ClickHouse/pull/6596), [\#6662](https://github.com/ClickHouse/ClickHouse/pull/6662) ([ヴァシーリー-ネムコフ](https://github.com/Enmk)) +- 新しい機能 `neighbor(value, offset[, default_value])`. データのブロック内の列内の前/次の値に到達することができます。 [\#5925](https://github.com/ClickHouse/ClickHouse/pull/5925) ([Alex Krash](https://github.com/alex-krash)) [6685365ab8c5b74f9650492c88a012596eb1b0c6](https://github.com/ClickHouse/ClickHouse/commit/6685365ab8c5b74f9650492c88a012596eb1b0c6) [341e2e4587a18065c2da1ca888c73389f48ce36c](https://github.com/ClickHouse/ClickHouse/commit/341e2e4587a18065c2da1ca888c73389f48ce36c) [アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov) +- 関数の作成 `currentUser()`、許可されたユーザのログインを返す。 エイリアスを追加 `user()` MySQLとの互換性のため。 [\#6470](https://github.com/ClickHouse/ClickHouse/pull/6470) ([Alex Krash](https://github.com/alex-krash)) +- 新しい集計関数 `quantilesExactInclusive` と `quantilesExactExclusive` で提案された [\#5885](https://github.com/ClickHouse/ClickHouse/issues/5885). [\#6477](https://github.com/ClickHouse/ClickHouse/pull/6477) ([dimarub2000](https://github.com/dimarub2000)) +- 関数 `bitmapRange(bitmap, range_begin, range_end)` 指定された範囲を持つ新しいセットを返します。 `range_end`). [\#6314](https://github.com/ClickHouse/ClickHouse/pull/6314) ([Zhichang Yu](https://github.com/yuzhichang)) +- 関数 `geohashesInBox(longitude_min, latitude_min, longitude_max, latitude_max, precision)` これは、提供された領域をカバーするgeohashボックスの精度の長い文字列の配列を作成します。 [\#6127](https://github.com/ClickHouse/ClickHouse/pull/6127) ([ヴァシーリー-ネムコフ](https://github.com/Enmk)) +- 挿入クエリのサポートを実装する `Kafka` テーブル [\#6012](https://github.com/ClickHouse/ClickHouse/pull/6012) ([イワン](https://github.com/abyss7)) +- 追加されたサポート `_partition` と `_timestamp` カフカエンジンへの仮想列。 [\#6400](https://github.com/ClickHouse/ClickHouse/pull/6400) ([イワン](https://github.com/abyss7)) - から機密データを削除する可能性 `query_log`、サーバーログ、regexpベースのルールを持つプロセスリスト。 [\#5710](https://github.com/ClickHouse/ClickHouse/pull/5710) ([フィリモノフ](https://github.com/filimonov)) -#### 実験的特徴 {#experimental-feature-2} +#### 実験的な特徴 {#experimental-feature-2} -- 入力および出力データ形式 `Template`. これは、入力と出力のカスタム書式文字列を指定することができます。 [\#4354](https://github.com/ClickHouse/ClickHouse/issues/4354) [\#6727](https://github.com/ClickHouse/ClickHouse/pull/6727) ([tavplubix](https://github.com/tavplubix)) -- の実装 `LIVE VIEW` 最初に提案されたテーブル [\#2898](https://github.com/ClickHouse/ClickHouse/pull/2898)、準備される [\#3925](https://github.com/ClickHouse/ClickHouse/issues/3925)、その後で更新 [\#5541](https://github.com/ClickHouse/ClickHouse/issues/5541). 見る [\#5541](https://github.com/ClickHouse/ClickHouse/issues/5541) 詳細な説明のため。 [\#5541](https://github.com/ClickHouse/ClickHouse/issues/5541) ([vzakaznikov](https://github.com/vzakaznikov)) [\#6425](https://github.com/ClickHouse/ClickHouse/pull/6425) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) [\#6656](https://github.com/ClickHouse/ClickHouse/pull/6656) ([vzakaznikov](https://github.com/vzakaznikov)）ことに注意 `LIVE VIEW` 特集は削除される可能性があり、来できます。 +- 入出力データ形式 `Template`. これは、入力と出力のカスタム書式文字列を指定することができます。 [\#4354](https://github.com/ClickHouse/ClickHouse/issues/4354) [\#6727](https://github.com/ClickHouse/ClickHouse/pull/6727) ([tavplubix](https://github.com/tavplubix)) +- の実装 `LIVE VIEW` 最初に提案されたテーブル [\#2898](https://github.com/ClickHouse/ClickHouse/pull/2898)、で調製 [\#3925](https://github.com/ClickHouse/ClickHouse/issues/3925) で更新されます。 [\#5541](https://github.com/ClickHouse/ClickHouse/issues/5541). 見る [\#5541](https://github.com/ClickHouse/ClickHouse/issues/5541) 詳細な説明のため。 [\#5541](https://github.com/ClickHouse/ClickHouse/issues/5541) ([ヴザカズニコフ](https://github.com/vzakaznikov)) [\#6425](https://github.com/ClickHouse/ClickHouse/pull/6425) ([ニコライ-コチェトフ](https://github.com/KochetovNicolai)) [\#6656](https://github.com/ClickHouse/ClickHouse/pull/6656) ([ヴザカズニコフ](https://github.com/vzakaznikov)）なお `LIVE VIEW` 特集は削除される可能性があり、来できます。 #### バグ修正 {#bug-fix-8} -- このリリースも含む全てのバグ修正から19.13と19.11. -- 固定の区分断層のテーブルはスキップ指標および垂直統合などが挙げられる。 [\#6723](https://github.com/ClickHouse/ClickHouse/pull/6723) ([alesapin](https://github.com/alesapin)) -- 非自明な列のデフォルトで列ごとのttlを修正しました。 以前は、force ttl mergeの場合は次のようになりました `OPTIMIZE ... FINAL` クエリー、終了しました値に置き換えられたタイプのデフォルトの代わりにユーザが指定した列のデフォルトする [\#6796](https://github.com/ClickHouse/ClickHouse/pull/6796) ([アントン-ポポフ](https://github.com/CurtizJ)) -- 通常のサーバーの再起動時にkafkaメッセージの重複の問題を修正。 [\#6597](https://github.com/ClickHouse/ClickHouse/pull/6597) ([イワン](https://github.com/abyss7)) -- カフカメッセージを読むときに固定無限ループ。 それ以外の場合は、いくつかのシナリオで無期限に一時停止することがあります。 [\#6354](https://github.com/ClickHouse/ClickHouse/pull/6354) ([イワン](https://github.com/abyss7)) -- 修正 `Key expression contains comparison between inconvertible types` での例外 `bitmapContains` 機能。 [\#6136](https://github.com/ClickHouse/ClickHouse/issues/6136) [\#6146](https://github.com/ClickHouse/ClickHouse/issues/6146) [\#6156](https://github.com/ClickHouse/ClickHouse/pull/6156) ([ディマルブ2000](https://github.com/dimarub2000)) -- 有効にしてsegfaultを修正 `optimize_skip_unused_shards` シャーディングキーがない [\#6384](https://github.com/ClickHouse/ClickHouse/pull/6384) ([アントン-ポポフ](https://github.com/CurtizJ)) -- メモリの破損につながる可能性があり、突然変異で間違ったコードを修正. アドレスの読み取りによる固定segfault `0x14c0` それは同時に起こったかもしれない `DROP TABLE` と `SELECT` から `system.parts` または `system.parts_columns`. 突然変異クエリの準備の競合状態を修正しました。 によるデッドロックを修正 `OPTIMIZE` レプリケートされたテーブルと同時変更操作のような変更。 [\#6514](https://github.com/ClickHouse/ClickHouse/pull/6514) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- MySQLインターフェイスで削除された余分な冗長ログ [\#6389](https://github.com/ClickHouse/ClickHouse/pull/6389) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- からブール値の設定を解析する機能を返します ‘true’ と ‘false’ 設定ファイルで。 [\#6278](https://github.com/ClickHouse/ClickHouse/pull/6278) ([alesapin](https://github.com/alesapin)) -- でクラッシュを修正 `quantile` と `median` 関数オーバー `Nullable(Decimal128)`. [\#6378](https://github.com/ClickHouse/ClickHouse/pull/6378) ([Artem Zuikov](https://github.com/4ertus2)) -- によって返された可能な不完全な結果を修正 `SELECT` クエリと `WHERE` 主キーの条件には、Float型への変換が含まれていました。 それは単調性の誤ったチェックによって引き起こされた `toFloat` 機能。 [\#6248](https://github.com/ClickHouse/ClickHouse/issues/6248) [\#6374](https://github.com/ClickHouse/ClickHouse/pull/6374) ([ディマルブ2000](https://github.com/dimarub2000)) -- チェック `max_expanded_ast_elements` 突然変異のための設定。 後の明確な突然変異 `TRUNCATE TABLE`. [\#6205](https://github.com/ClickHouse/ClickHouse/pull/6205) ([冬張](https://github.com/zhang2014)) -- と共に使用するときにキー列の結合結果を修正 `join_use_nulls`. 列の既定値の代わりにNullをアタッチします。 [\#6249](https://github.com/ClickHouse/ClickHouse/pull/6249) ([Artem Zuikov](https://github.com/4ertus2)) -- 垂直マージと変更とスキップインデックスの修正。 修正のための `Bad size of marks file` 例外だ [\#6594](https://github.com/ClickHouse/ClickHouse/issues/6594) [\#6713](https://github.com/ClickHouse/ClickHouse/pull/6713) ([alesapin](https://github.com/alesapin)) -- でレアクラッシュを修正 `ALTER MODIFY COLUMN` マージ/変更された部分のいずれかが空（0行）のときに垂直マージ) [\#6746](https://github.com/ClickHouse/ClickHouse/issues/6746) [\#6780](https://github.com/ClickHouse/ClickHouse/pull/6780) ([alesapin](https://github.com/alesapin)) -- の変換のバグを修正しました `LowCardinality` タイプ `AggregateFunctionFactory`. この修正 [\#6257](https://github.com/ClickHouse/ClickHouse/issues/6257). [\#6281](https://github.com/ClickHouse/ClickHouse/pull/6281) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- このリリースには、19.13と19.11のすべてのバグ修正も含まれています。 +- 固定の区分断層のテーブルはスキップ指標および垂直統合などが挙げられる。 [\#6723](https://github.com/ClickHouse/ClickHouse/pull/6723) ([アレサピン](https://github.com/alesapin)) +- 非自明な列のデフォルトで列ごとのTTLを修正しました。 以前は強制TTLマージの場合 `OPTIMIZE ... FINAL` クエリー、終了しました値に置き換えられたタイプのデフォルトの代わりにユーザが指定した列のデフォルトする [\#6796](https://github.com/ClickHouse/ClickHouse/pull/6796) ([アントン-ポポフ](https://github.com/CurtizJ)) +- 固定カフカメッセージの複製問題が通常のサーバを再起動します。 [\#6597](https://github.com/ClickHouse/ClickHouse/pull/6597) ([イワン](https://github.com/abyss7)) +- カフカメッセージを読み取るときに固定無限ループ。 そうしないと、一部のシナリオで無期限に一時停止されることがあります。 [\#6354](https://github.com/ClickHouse/ClickHouse/pull/6354) ([イワン](https://github.com/abyss7)) +- 修正 `Key expression contains comparison between inconvertible types` 例外 `bitmapContains` 機能。 [\#6136](https://github.com/ClickHouse/ClickHouse/issues/6136) [\#6146](https://github.com/ClickHouse/ClickHouse/issues/6146) [\#6156](https://github.com/ClickHouse/ClickHouse/pull/6156) ([dimarub2000](https://github.com/dimarub2000)) +- 有効でsegfaultを修正 `optimize_skip_unused_shards` シャーディングキーがない [\#6384](https://github.com/ClickHouse/ClickHouse/pull/6384) ([アントン-ポポフ](https://github.com/CurtizJ)) +- 固定の間違ったコードに変異のありメモリが破損す アドレスの読み取りで固定segfault `0x14c0` それは同時に起こった可能性があります `DROP TABLE` と `SELECT` から `system.parts` または `system.parts_columns`. 突然変異クエリの準備における競合状態を修正しました。 によるデッドロックを修正 `OPTIMIZE` 変更のようなレプリケートされたテーブルと同時変更操作の。 [\#6514](https://github.com/ClickHouse/ClickHouse/pull/6514) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- MySQLインターフェイスで余分な冗長ロギングを削除 [\#6389](https://github.com/ClickHouse/ClickHouse/pull/6389) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- ブール値の設定を解析する機能を返します ‘true’ と ‘false’ 設定ファイル内。 [\#6278](https://github.com/ClickHouse/ClickHouse/pull/6278) ([アレサピン](https://github.com/alesapin)) +- でクラッシュを修正 `quantile` と `median` 関数オーバー `Nullable(Decimal128)`. [\#6378](https://github.com/ClickHouse/ClickHouse/pull/6378) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- によって返される不完全な結果を修正 `SELECT` クエリ `WHERE` 主キーの条件に浮動小数点型への変換が含まれていました。 これは単調性の誤ったチェックによって引き起こされた `toFloat` 機能。 [\#6248](https://github.com/ClickHouse/ClickHouse/issues/6248) [\#6374](https://github.com/ClickHouse/ClickHouse/pull/6374) ([dimarub2000](https://github.com/dimarub2000)) +- チェック `max_expanded_ast_elements` 突然変異の設定。 後の明確な突然変異 `TRUNCATE TABLE`. [\#6205](https://github.com/ClickHouse/ClickHouse/pull/6205) ([冬張](https://github.com/zhang2014)) +- キー列の結合結果を修正 `join_use_nulls`. 列の既定値の代わりにNullを添付します。 [\#6249](https://github.com/ClickHouse/ClickHouse/pull/6249) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- 垂直マージと変更とスキップインデックスの修正。 のための修正 `Bad size of marks file` 例外だ [\#6594](https://github.com/ClickHouse/ClickHouse/issues/6594) [\#6713](https://github.com/ClickHouse/ClickHouse/pull/6713) ([アレサピン](https://github.com/alesapin)) +- で珍しいクラッシュを修正 `ALTER MODIFY COLUMN` マージされた/変更された部分のいずれかが空の場合（0行）、垂直マージ) [\#6746](https://github.com/ClickHouse/ClickHouse/issues/6746) [\#6780](https://github.com/ClickHouse/ClickHouse/pull/6780) ([アレサピン](https://github.com/alesapin)) +- の変換のバグを修正しました `LowCardinality` タイプ `AggregateFunctionFactory`. この修正 [\#6257](https://github.com/ClickHouse/ClickHouse/issues/6257). [\#6281](https://github.com/ClickHouse/ClickHouse/pull/6281) ([ニコライ-コチェトフ](https://github.com/KochetovNicolai)) - 間違った動作と可能なsegfaultsを修正 `topK` と `topKWeighted` 集計関数。 [\#6404](https://github.com/ClickHouse/ClickHouse/pull/6404) ([アントン-ポポフ](https://github.com/CurtizJ)) -- 周りの固定安全でないコード `getIdentifier` 機能。 [\#6401](https://github.com/ClickHouse/ClickHouse/issues/6401) [\#6409](https://github.com/ClickHouse/ClickHouse/pull/6409) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 問題を修正しましたmysqlのワイヤーロを使用すると接続するclickhouse形mysqlクライアント). によって引き起こされる `PacketPayloadWriteBuffer`. [\#6212](https://github.com/ClickHouse/ClickHouse/pull/6212) ([ユーリーバラノフ](https://github.com/yurriy)) -- 固定メモリリーク `bitmapSubsetInRange` 機能。 [\#6819](https://github.com/ClickHouse/ClickHouse/pull/6819) ([Zhichang Yu](https://github.com/yuzhichang)) -- 粒度の変更後に突然変異が実行されたときに稀なバグを修正しました。 [\#6816](https://github.com/ClickHouse/ClickHouse/pull/6816) ([alesapin](https://github.com/alesapin)) -- きprotobufメッセージの全ての分野でのデフォルトです。 [\#6132](https://github.com/ClickHouse/ClickHouse/pull/6132) ([Vitaly Baranov](https://github.com/vitlibar)) -- バグを解決するには `nullIf` 私達がaを送る場合の機能 `NULL` 第二引数の引数。 [\#6446](https://github.com/ClickHouse/ClickHouse/pull/6446) ([ギヨームタッセリー](https://github.com/YiuRULE)) -- 文字列フィールドを持つ複雑なキーキャッシュ辞書で間違ったメモリ割り当て/割り当て解除を使用して、まれなバグを修正しました（メモリリークのよ バグは、文字列サイズが八（8、16、32、等）から始まる二つの累乗だったときに再現します。 [\#6447](https://github.com/ClickHouse/ClickHouse/pull/6447) ([alesapin](https://github.com/alesapin)) -- 例外を引き起こした小さな配列に固定ゴリラエンコード `Cannot write after end of buffer`. [\#6398](https://github.com/ClickHouse/ClickHouse/issues/6398) [\#6444](https://github.com/ClickHouse/ClickHouse/pull/6444) ([Vasily Nemkov](https://github.com/Enmk)) -- Nullableではない型をJOINsで使用できるようにする `join_use_nulls` 有効。 [\#6705](https://github.com/ClickHouse/ClickHouse/pull/6705) ([Artem Zuikov](https://github.com/4ertus2)) -- 無効にする `Poco::AbstractConfiguration` クエリ内での置換 `clickhouse-client`. [\#6706](https://github.com/ClickHouse/ClickHouse/pull/6706) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- デッドロックを回避 `REPLACE PARTITION`. [\#6677](https://github.com/ClickHouse/ClickHouse/pull/6677) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- を使用して `arrayReduce` 定数引数の場合、segfaultにつながる可能性があります。 [\#6242](https://github.com/ClickHouse/ClickHouse/issues/6242) [\#6326](https://github.com/ClickHouse/ClickHouse/pull/6326) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- レプリカが後に復元された場合に表示される不整合な部分を修正 `DROP PARTITION`. [\#6522](https://github.com/ClickHouse/ClickHouse/issues/6522) [\#6523](https://github.com/ClickHouse/ClickHouse/pull/6523) ([tavplubix](https://github.com/tavplubix)) -- 固定こつ `JSONExtractRaw` 機能。 [\#6195](https://github.com/ClickHouse/ClickHouse/issues/6195) [\#6198](https://github.com/ClickHouse/ClickHouse/pull/6198) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 修正のバグと誤ったスキップ指数の直列化を行い、その凝集と適応粒度. [\#6594](https://github.com/ClickHouse/ClickHouse/issues/6594). [\#6748](https://github.com/ClickHouse/ClickHouse/pull/6748) ([alesapin](https://github.com/alesapin)) -- 修正 `WITH ROLLUP` と `WITH CUBE` の修飾子 `GROUP BY` 二レベルの集計。 [\#6225](https://github.com/ClickHouse/ClickHouse/pull/6225) ([アントン-ポポフ](https://github.com/CurtizJ)) -- のを修正した。筆二次指標マーク適応型粒度. [\#6126](https://github.com/ClickHouse/ClickHouse/pull/6126) ([alesapin](https://github.com/alesapin)) -- サーバーの起動中に初期化の順序を修正します。 それ以来 `StorageMergeTree::background_task_handle` で初期化される。 `startup()` その `MergeTreeBlockOutputStream::write()` 初期化の前に使用しようとするかもしれません。 すぐチェックインの場合は初期化されます。 [\#6080](https://github.com/ClickHouse/ClickHouse/pull/6080) ([イワン](https://github.com/abyss7)) -- エラーで完了した前の読み取り操作からデータバッファーをクリアします。 [\#6026](https://github.com/ClickHouse/ClickHouse/pull/6026) ([ニコライ](https://github.com/bopohaa)) -- 固定バを適応的粒度を新規作成時のレプリカのための複製\*mergetreeます。 [\#6394](https://github.com/ClickHouse/ClickHouse/issues/6394) [\#6452](https://github.com/ClickHouse/ClickHouse/pull/6452) ([alesapin](https://github.com/alesapin)) -- 例外が発生した場合のサーバーの起動時に可能なクラッシュを修正しました `libunwind` 初期化されていないアクセス時の例外時 `ThreadStatus` 構造。 [\#6456](https://github.com/ClickHouse/ClickHouse/pull/6456) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- でクラッシュを修正 `yandexConsistentHash` 機能。 ファズテストによって発見。 [\#6304](https://github.com/ClickHouse/ClickHouse/issues/6304) [\#6305](https://github.com/ClickHouse/ClickHouse/pull/6305) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 固定の可能性を掛けのクエリがサーバが過負荷状態で運転され、グローバルスレッドプールが近い。 これは、分散クエリが各シャードに接続ごとにスレッドを割り当てるため、多数のシャード(数百)を持つクラスターで発生する可能性が高くなります。 たとえば、330個のシャードのクラスターが30個の同時分散クエリを処理している場合、この問題は再現されます。 この問題に影響するすべてのバージョンから19.2. [\#6301](https://github.com/ClickHouse/ClickHouse/pull/6301) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 固定ロジックの `arrayEnumerateUniqRanked` 機能。 [\#6423](https://github.com/ClickHouse/ClickHouse/pull/6423) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- シンボルテーブルのデコード時にsegfaultを修正。 [\#6603](https://github.com/ClickHouse/ClickHouse/pull/6603) ([アモスの鳥](https://github.com/amosbird)) -- のキャストで修正された無関係な例外 `LowCardinality(Nullable)` to not-Nullable column in case if it doesn’t contain Nulls (e.g. in query like `SELECT CAST(CAST('Hello' AS LowCardinality(Nullable(String))) AS String)`. [\#6094](https://github.com/ClickHouse/ClickHouse/issues/6094) [\#6119](https://github.com/ClickHouse/ClickHouse/pull/6119) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- で説明の余分な引用を削除しました `system.settings` テーブル。 [\#6696](https://github.com/ClickHouse/ClickHouse/issues/6696) [\#6699](https://github.com/ClickHouse/ClickHouse/pull/6699) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- デッドロックの回避 `TRUNCATE` 複製されたテーブルの。 [\#6695](https://github.com/ClickHouse/ClickHouse/pull/6695) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- 安全でないコードを修正 `getIdentifier` 機能。 [\#6401](https://github.com/ClickHouse/ClickHouse/issues/6401) [\#6409](https://github.com/ClickHouse/ClickHouse/pull/6409) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 問題を修正しましたMySQLのワイヤーロを使用すると接続するClickHouse形MySQLクライアント). によるヒープバッファオーバーフロー `PacketPayloadWriteBuffer`. [\#6212](https://github.com/ClickHouse/ClickHouse/pull/6212) ([ユーリー-バラノフ](https://github.com/yurriy)) +- 修正されたメモリリーク `bitmapSubsetInRange` 機能。 [\#6819](https://github.com/ClickHouse/ClickHouse/pull/6819) ([Zhichang Yu](https://github.com/yuzhichang)) +- 粒度の変更後に突然変異が実行される珍しいバグを修正しました。 [\#6816](https://github.com/ClickHouse/ClickHouse/pull/6816) ([アレサピン](https://github.com/alesapin)) +- きprotobufメッセージの全ての分野でのデフォルトです。 [\#6132](https://github.com/ClickHouse/ClickHouse/pull/6132) ([ヴィタリー-バラノフ](https://github.com/vitlibar)) +- バグを解決する `nullIf` 私達がaを送る場合の機能 `NULL` 第二の引数の引数。 [\#6446](https://github.com/ClickHouse/ClickHouse/pull/6446) ([ギヨーム-タッセリー](https://github.com/YiuRULE)) +- 無限のメモリ消費（メモリリークのように見える）につながる文字列フィールドを持つ複雑なキーキャッシュ辞書で間違ったメモリ割り当て/割り当て解除 バグは、文字列のサイズが八から始まる二の累乗（8、16、32など）だったときに再現します。 [\#6447](https://github.com/ClickHouse/ClickHouse/pull/6447) ([アレサピン](https://github.com/alesapin)) +- 固定Gorillaエンコーディングの小型配列により例外 `Cannot write after end of buffer`. [\#6398](https://github.com/ClickHouse/ClickHouse/issues/6398) [\#6444](https://github.com/ClickHouse/ClickHouse/pull/6444) ([ヴァシーリー-ネムコフ](https://github.com/Enmk)) +- 結合でnull許容できない型を使用できるようにする `join_use_nulls` 有効。 [\#6705](https://github.com/ClickHouse/ClickHouse/pull/6705) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- 無効にする `Poco::AbstractConfiguration` クエリ内の置換 `clickhouse-client`. [\#6706](https://github.com/ClickHouse/ClickHouse/pull/6706) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- でデッドロックを回避 `REPLACE PARTITION`. [\#6677](https://github.com/ClickHouse/ClickHouse/pull/6677) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- を使用して `arrayReduce` 定数引数の場合、segfaultにつながる可能性があります。 [\#6242](https://github.com/ClickHouse/ClickHouse/issues/6242) [\#6326](https://github.com/ClickHouse/ClickHouse/pull/6326) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 後にレプリカが復元された場合に表示される矛盾した部分を修正 `DROP PARTITION`. [\#6522](https://github.com/ClickHouse/ClickHouse/issues/6522) [\#6523](https://github.com/ClickHouse/ClickHouse/pull/6523) ([tavplubix](https://github.com/tavplubix)) +- 固定ハングイン `JSONExtractRaw` 機能。 [\#6195](https://github.com/ClickHouse/ClickHouse/issues/6195) [\#6198](https://github.com/ClickHouse/ClickHouse/pull/6198) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 修正のバグと誤ったスキップ指数の直列化を行い、その凝集と適応粒度. [\#6594](https://github.com/ClickHouse/ClickHouse/issues/6594). [\#6748](https://github.com/ClickHouse/ClickHouse/pull/6748) ([アレサピン](https://github.com/alesapin)) +- 修正 `WITH ROLLUP` と `WITH CUBE` の修飾子 `GROUP BY` 二レベルの集計を使って。 [\#6225](https://github.com/ClickHouse/ClickHouse/pull/6225) ([アントン-ポポフ](https://github.com/CurtizJ)) +- 適応粒度で二次インデックスマークを書くとバグを修正しました。 [\#6126](https://github.com/ClickHouse/ClickHouse/pull/6126) ([アレサピン](https://github.com/alesapin)) +- サーバーの起動時に初期化の順序を修正しました。 以来 `StorageMergeTree::background_task_handle` で初期化される。 `startup()` その `MergeTreeBlockOutputStream::write()` 初期化の前に使用しようとする可能性があります。 すぐチェックインの場合は初期化されます。 [\#6080](https://github.com/ClickHouse/ClickHouse/pull/6080) ([イワン](https://github.com/abyss7)) +- エラーで完了した前の読み取り操作からデータバッファをクリアします。 [\#6026](https://github.com/ClickHouse/ClickHouse/pull/6026) ([ニコライ](https://github.com/bopohaa)) +- 固定バを適応的粒度を新規作成時のレプリカのための複製\*MergeTreeます。 [\#6394](https://github.com/ClickHouse/ClickHouse/issues/6394) [\#6452](https://github.com/ClickHouse/ClickHouse/pull/6452) ([アレサピン](https://github.com/alesapin)) +- 例外が発生した場合にサーバーの起動時にクラッシュする可能性を修正 `libunwind` 未初期化へのアクセス時の例外中 `ThreadStatus` 構造。 [\#6456](https://github.com/ClickHouse/ClickHouse/pull/6456) ([ニキータ-ミハイロフ](https://github.com/nikitamikhaylov)) +- でクラッシュを修正 `yandexConsistentHash` 機能。 ファズテストによって発見。 [\#6304](https://github.com/ClickHouse/ClickHouse/issues/6304) [\#6305](https://github.com/ClickHouse/ClickHouse/pull/6305) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 固定の可能性を掛けのクエリがサーバが過負荷状態で運転され、グローバルスレッドプールが近い。 これは、分散クエリが各シャードに接続ごとにスレッドを割り当てるため、多数のシャード(数百)を持つクラスターで発生する可能性が高くなります。 例えば、この問題が再現場クラスターの330の破片が処理を同時30分布します。 この問題は19.2以降のすべてのバージョンに影響します。 [\#6301](https://github.com/ClickHouse/ClickHouse/pull/6301) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- の固定ロジック `arrayEnumerateUniqRanked` 機能。 [\#6423](https://github.com/ClickHouse/ClickHouse/pull/6423) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 固定segfaultデコードする際のシンボル。 [\#6603](https://github.com/ClickHouse/ClickHouse/pull/6603) ([アモス鳥](https://github.com/amosbird)) +- キャストの無関係な例外を修正しました `LowCardinality(Nullable)` to not-Nullable column in case if it doesn't contain Nulls (e.g. in query like `SELECT CAST(CAST('Hello' AS LowCardinality(Nullable(String))) AS String)`. [\#6094](https://github.com/ClickHouse/ClickHouse/issues/6094) [\#6119](https://github.com/ClickHouse/ClickHouse/pull/6119) ([ニコライ-コチェトフ](https://github.com/KochetovNicolai)) +- の説明の余分な引用を削除しました `system.settings` テーブル。 [\#6696](https://github.com/ClickHouse/ClickHouse/issues/6696) [\#6699](https://github.com/ClickHouse/ClickHouse/pull/6699) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- で可能なデッドロックを回避 `TRUNCATE` 複製されたテーブルの。 [\#6695](https://github.com/ClickHouse/ClickHouse/pull/6695) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) - ソートキーの順に読み取りを修正しました。 [\#6189](https://github.com/ClickHouse/ClickHouse/pull/6189) ([アントン-ポポフ](https://github.com/CurtizJ)) -- 修正 `ALTER TABLE ... UPDATE` とテーブルのクエリ `enable_mixed_granularity_parts=1`. [\#6543](https://github.com/ClickHouse/ClickHouse/pull/6543) ([alesapin](https://github.com/alesapin)) -- によって開かバグを修正 [\#4405](https://github.com/ClickHouse/ClickHouse/pull/4405) （19.4.0以来）。 列をクエリしない場合は、MergeTreeテーブルを使用して分散テーブルへのクエリを再現します (`SELECT 1`). [\#6236](https://github.com/ClickHouse/ClickHouse/pull/6236) ([alesapin](https://github.com/alesapin)) -- 固定オーバーフローの整数部署名-タイプを符号なしタイプです。 この動作は、cまたはc++言語（整数昇格ルール）とまったく同じで、驚くかもしれません。 大きな符号付き数を大きな符号なし数に分割する場合、またはその逆の場合にはオーバーフローが可能であることに注意してください（ただし、その場合 の問題が全てのサーバーのバージョン [\#6214](https://github.com/ClickHouse/ClickHouse/issues/6214) [\#6233](https://github.com/ClickHouse/ClickHouse/pull/6233) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- スロットリング時の最大スリープ時間を制限する `max_execution_speed` または `max_execution_speed_bytes` 設定されています。 固定偽のようなエラー `Estimated query execution time (inf seconds) is too long`. [\#5547](https://github.com/ClickHouse/ClickHouse/issues/5547) [\#6232](https://github.com/ClickHouse/ClickHouse/pull/6232) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 使用に関する問題を修正 `MATERIALIZED` の列とエイリアス `MaterializedView`. [\#448](https://github.com/ClickHouse/ClickHouse/issues/448) [\#3484](https://github.com/ClickHouse/ClickHouse/issues/3484) [\#3450](https://github.com/ClickHouse/ClickHouse/issues/3450) [\#2878](https://github.com/ClickHouse/ClickHouse/issues/2878) [\#2285](https://github.com/ClickHouse/ClickHouse/issues/2285) [\#3796](https://github.com/ClickHouse/ClickHouse/pull/3796) ([アモスの鳥](https://github.com/amosbird)) [\#6316](https://github.com/ClickHouse/ClickHouse/pull/6316) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 修正 `FormatFactory` プロセッサとして実装されていない入力ストリームの動作。 [\#6495](https://github.com/ClickHouse/ClickHouse/pull/6495) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 修正されたタイプミス。 [\#6631](https://github.com/ClickHouse/ClickHouse/pull/6631) ([Alex Ryndin](https://github.com/alexryndin)) -- エラーメッセージのタイプミス（is-\>are）。 [\#6839](https://github.com/ClickHouse/ClickHouse/pull/6839) ([Denis Zhuravlev](https://github.com/den-crane)) -- 固定誤差を解析カラムのリストから文字列の場合タイプが含まれるコンマ(この問題に関連する `File`, `URL`, `HDFS` ストレージ) [\#6217](https://github.com/ClickHouse/ClickHouse/issues/6217). [\#6209](https://github.com/ClickHouse/ClickHouse/pull/6209) ([ディマルブ2000](https://github.com/dimarub2000)) +- 修正 `ALTER TABLE ... UPDATE` テーブルのクエリ `enable_mixed_granularity_parts=1`. [\#6543](https://github.com/ClickHouse/ClickHouse/pull/6543) ([アレサピン](https://github.com/alesapin)) +- で開かれたバグを修正 [\#4405](https://github.com/ClickHouse/ClickHouse/pull/4405) （19.4.0以降）。 列を照会しないときに、MergeTreeテーブル上の分散テーブルへのクエリを再現します (`SELECT 1`). [\#6236](https://github.com/ClickHouse/ClickHouse/pull/6236) ([アレサピン](https://github.com/alesapin)) +- 符号付き型から符号なし型への整数除算でのオーバーフローを修正。 この動作は、cまたはC++言語（整数昇格規則）とまったく同じで、驚くべきことです。 大きな符号付き数値を大きな符号なし数値に分割する場合、またはその逆に分割する場合は、オーバーフローが可能であることに注意してください（ただし、 の問題が全てのサーバーのバージョン [\#6214](https://github.com/ClickHouse/ClickHouse/issues/6214) [\#6233](https://github.com/ClickHouse/ClickHouse/pull/6233) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- スロットルの最大スリープ時間を制限する場合 `max_execution_speed` または `max_execution_speed_bytes` 設定されています。 固定falseのようなエラー `Estimated query execution time (inf seconds) is too long`. [\#5547](https://github.com/ClickHouse/ClickHouse/issues/5547) [\#6232](https://github.com/ClickHouse/ClickHouse/pull/6232) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 使用に関する問題を修正 `MATERIALIZED` 列とエイリアス `MaterializedView`. [\#448](https://github.com/ClickHouse/ClickHouse/issues/448) [\#3484](https://github.com/ClickHouse/ClickHouse/issues/3484) [\#3450](https://github.com/ClickHouse/ClickHouse/issues/3450) [\#2878](https://github.com/ClickHouse/ClickHouse/issues/2878) [\#2285](https://github.com/ClickHouse/ClickHouse/issues/2285) [\#3796](https://github.com/ClickHouse/ClickHouse/pull/3796) ([アモス鳥](https://github.com/amosbird)) [\#6316](https://github.com/ClickHouse/ClickHouse/pull/6316) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 修正 `FormatFactory` プロセッサとして実装されていない入力ストリームの動作。 [\#6495](https://github.com/ClickHouse/ClickHouse/pull/6495) ([ニコライ-コチェトフ](https://github.com/KochetovNicolai)) +- タイプミスを修正。 [\#6631](https://github.com/ClickHouse/ClickHouse/pull/6631) ([アレックス-リンディン](https://github.com/alexryndin)) +- エラーメッセージの入力ミス(is-\>are)。 [\#6839](https://github.com/ClickHouse/ClickHouse/pull/6839) ([デニス-ジュラヴレフ](https://github.com/den-crane)) +- Typeにカンマが含まれている場合、stringからの列リストの解析中にエラーが修正されました（この問題は `File`, `URL`, `HDFS` ストレージ) [\#6217](https://github.com/ClickHouse/ClickHouse/issues/6217). [\#6209](https://github.com/ClickHouse/ClickHouse/pull/6209) ([dimarub2000](https://github.com/dimarub2000)) #### セキュリティ修正 {#security-fix} - このリリースも含む全てのバグのセキュリティ修正をか19.13と19.11. -- SQLパーサーのスタックオーバーフローによりサーバーがクラッシュする可能性がある問題を修正 固定の可能性スタックオーバーフローに統合、配布し、テーブルが現実の景色の件本件は、行レベルのセキュリティなサブクエリ. [\#6433](https://github.com/ClickHouse/ClickHouse/pull/6433) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- 固定の可能性に作製したクエリがサーバのクラッシュによるスタックオーバーフローアプリケーションパーサです。 固定の可能性スタックオーバーフローに統合、配布し、テーブルが現実の景色の件本件は、行レベルのセキュリティなサブクエリ. [\#6433](https://github.com/ClickHouse/ClickHouse/pull/6433) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) #### 改善 {#improvement-3} -- のための三元論理の正しい実装 `AND/OR`. [\#6048](https://github.com/ClickHouse/ClickHouse/pull/6048) ([Alexander Kazakov](https://github.com/Akazz)) -- これで、ttlの有効期限が切れた値と行が削除されます。 `OPTIMIZE ... FINAL` query from old parts without TTL infos or with outdated TTL infos, e.g. after `ALTER ... MODIFY TTL` クエリ。 追加されたクエリ `SYSTEM STOP/START TTL MERGES` 可に/を割り当てを合併TTLおよびフィルター終了しました値をすべてが合併。 [\#6274](https://github.com/ClickHouse/ClickHouse/pull/6274) ([アントン-ポポフ](https://github.com/CurtizJ)) -- クライアントのclickhouse履歴ファイルの場所を変更する可能性 `CLICKHOUSE_HISTORY_FILE` env [\#6840](https://github.com/ClickHouse/ClickHouse/pull/6840) ([フィリモノフ](https://github.com/filimonov)) -- 削除 `dry_run` フラグから `InterpreterSelectQuery`. … [\#6375](https://github.com/ClickHouse/ClickHouse/pull/6375) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- サポート `ASOF JOIN` と `ON` セクション。 [\#6211](https://github.com/ClickHouse/ClickHouse/pull/6211) ([Artem Zuikov](https://github.com/4ertus2)) -- 突然変異および複製のためのskip索引のよりよいサポート。 のサポート `MATERIALIZE/CLEAR INDEX ... IN PARTITION` クエリ。 `UPDATE x = x` 列を使用するすべてのインデックスの再計算 `x`. [\#5053](https://github.com/ClickHouse/ClickHouse/pull/5053) ([Nikita Vasilev](https://github.com/nikvas0)) -- 許可する `ATTACH` ライブビュー(たとえば、サーバーの起動時など) `allow_experimental_live_view` 設定。 [\#6754](https://github.com/ClickHouse/ClickHouse/pull/6754) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- のためのスタックトレースに集まるエリプロファイラに含まれているものも含むスタックフレームが発生するqueryプロファイラです。 [\#6250](https://github.com/ClickHouse/ClickHouse/pull/6250) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Nowテーブル関数 `values`, `file`, `url`, `hdfs` ALIAS列をサポートしている。 [\#6255](https://github.com/ClickHouse/ClickHouse/pull/6255) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 次の場合に例外をスローする `config.d` ファイルには、対応するルート要素が設定ファイルとして存在しません。 [\#6123](https://github.com/ClickHouse/ClickHouse/pull/6123) ([ディマルブ2000](https://github.com/dimarub2000)) -- 例外メッセージの余分な情報を印刷する `no space left on device`. [\#6182](https://github.com/ClickHouse/ClickHouse/issues/6182), [\#6252](https://github.com/ClickHouse/ClickHouse/issues/6252) [\#6352](https://github.com/ClickHouse/ClickHouse/pull/6352) ([tavplubix](https://github.com/tavplubix)) -- Aの破片を決定するとき `Distributed` 読み取りクエリによってカバーされるテーブル(for `optimize_skip_unused_shards` =1)ClickHouseは両方から条件をチェックします `prewhere` と `where` select文の句。 [\#6521](https://github.com/ClickHouse/ClickHouse/pull/6521) ([Alexander Kazakov](https://github.com/Akazz)) -- 有効 `SIMDJSON` AVX2のないしかしSSE4.2およびPCLMULの命令セットの機械のため。 [\#6285](https://github.com/ClickHouse/ClickHouse/issues/6285) [\#6320](https://github.com/ClickHouse/ClickHouse/pull/6320) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- ClickHouseでファイルシステムな `O_DIRECT` 追加のチューニングなしでサポート（ZFSとBtrFSなど）。 [\#4449](https://github.com/ClickHouse/ClickHouse/issues/4449) [\#6730](https://github.com/ClickHouse/ClickHouse/pull/6730) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 支援を押しであったが，最終的にサブクエリです。 [\#6120](https://github.com/ClickHouse/ClickHouse/pull/6120) ([Tcheason](https://github.com/TCeason)) [\#6162](https://github.com/ClickHouse/ClickHouse/pull/6162) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- より良い `JOIN ON` キーの抽出 [\#6131](https://github.com/ClickHouse/ClickHouse/pull/6131) ([Artem Zuikov](https://github.com/4ertus2)) -- Upated `SIMDJSON`. [\#6285](https://github.com/ClickHouse/ClickHouse/issues/6285). [\#6306](https://github.com/ClickHouse/ClickHouse/pull/6306) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 最小の列の選択を最適化する `SELECT count()` クエリ。 [\#6344](https://github.com/ClickHouse/ClickHouse/pull/6344) ([アモスの鳥](https://github.com/amosbird)) -- 追加 `strict` パラメータin `windowFunnel()`. とき `strict` は、 `windowFunnel()` 一意の値にのみ条件を適用します。 [\#6548](https://github.com/ClickHouse/ClickHouse/pull/6548) ([achimbabcomment](https://github.com/achimbab)) -- より安全なインタフェース `mysqlxx::Pool`. [\#6150](https://github.com/ClickHouse/ClickHouse/pull/6150) ([avasiliev](https://github.com/avasiliev)) -- オプション行サイズ `--help` オプションに対応した端末のサイズです。 [\#6590](https://github.com/ClickHouse/ClickHouse/pull/6590) ([ディマルブ2000](https://github.com/dimarub2000)) -- 無効にする “read in order” キーなしの集約の最適化。 [\#6599](https://github.com/ClickHouse/ClickHouse/pull/6599) ([アントン-ポポフ](https://github.com/CurtizJ)) -- のhttpステータスコード `INCORRECT_DATA` と `TYPE_MISMATCH` エラーコードをデフォルトから変更 `500 Internal Server Error` に `400 Bad Request`. [\#6271](https://github.com/ClickHouse/ClickHouse/pull/6271) ([Alexander Rodin](https://github.com/a-rodin)) -- 結合オブジェクトの移動元 `ExpressionAction` に `AnalyzedJoin`. `ExpressionAnalyzer` と `ExpressionAction` 知らない `Join` もはやクラス。 その論理は `AnalyzedJoin` フェイス [\#6801](https://github.com/ClickHouse/ClickHouse/pull/6801) ([Artem Zuikov](https://github.com/4ertus2)) -- 固定可能な行き詰まりの分散クエリーの資料はlocalhostでのクエリを送ネットワーク経由で接続します。 [\#6759](https://github.com/ClickHouse/ClickHouse/pull/6759) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 複数のテーブルの意味を変更 `RENAME` 可能なデッドロックを避けるため。 [\#6757](https://github.com/ClickHouse/ClickHouse/issues/6757). [\#6756](https://github.com/ClickHouse/ClickHouse/pull/6756) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 書き換えmysqlサーバーの互換性防止への負荷フルパケットペイロードに。 各接続のメモリ消費量の減少 `2 * DBMS_DEFAULT_BUFFER_SIZE` (読み取り/書き込みバッファ)。 [\#5811](https://github.com/ClickHouse/ClickHouse/pull/5811) ([ユーリーバラノフ](https://github.com/yurriy)) -- クエリのセマンティクスについて何も知る必要のないast alias interpreting logicをパーサーから外します。 [\#6108](https://github.com/ClickHouse/ClickHouse/pull/6108) ([Artem Zuikov](https://github.com/4ertus2)) -- もう少し安全な構文解析 `NamesAndTypesList`. [\#6408](https://github.com/ClickHouse/ClickHouse/issues/6408). [\#6410](https://github.com/ClickHouse/ClickHouse/pull/6410) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- `clickhouse-copier`:使用を許可する `where_condition` 設定から `partition_key` エイリアスクエリのためのチェック分配の存在についても同様とすでに使用されただけでデータを読み込むクエリ). [\#6577](https://github.com/ClickHouse/ClickHouse/pull/6577) ([proller](https://github.com/proller)) -- 追加オプションのメッセージ引数 `throwIf`. ([\#5772](https://github.com/ClickHouse/ClickHouse/issues/5772)) [\#6329](https://github.com/ClickHouse/ClickHouse/pull/6329) ([Vdimir](https://github.com/Vdimir)) -- クライアントでも挿入データの送信中にサーバー例外が発生しました。 [\#5891](https://github.com/ClickHouse/ClickHouse/issues/5891) [\#6711](https://github.com/ClickHouse/ClickHouse/pull/6711) ([ディマルブ2000](https://github.com/dimarub2000)) -- メトリックを追加 `DistributedFilesToInsert` その総数のファイルをファイルシステムを送信リモートサーバーに配布します。 数はすべての破片を合計します。 [\#6600](https://github.com/ClickHouse/ClickHouse/pull/6600) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- ほとんどの結合を準備ロジックから移動する `ExpressionAction/ExpressionAnalyzer` に `AnalyzedJoin`. [\#6785](https://github.com/ClickHouse/ClickHouse/pull/6785) ([Artem Zuikov](https://github.com/4ertus2)) -- TSanを修正 [警告](https://clickhouse-test-reports.s3.yandex.net/6399/c1c1d1daa98e199e620766f1bd06a5921050a00d/functional_stateful_tests_(thread).html) ‘lock-order-inversion’. [\#6740](https://github.com/ClickHouse/ClickHouse/pull/6740) ([Vasily Nemkov](https://github.com/Enmk)) -- Linuxの機能の欠如に関するより良い情報メッセージ。 致命的なエラーのログ記録 “fatal” レベル、それはそれが簡単で見つけることになります `system.text_log`. [\#6441](https://github.com/ClickHouse/ClickHouse/pull/6441) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- ディスクへの一時データのダンプを有効にして、使用中のメモリ使用量を制限する場合 `GROUP BY`, `ORDER BY` でかチェックのディスクスペース。 修正は、新しい設定を追加します `min_free_disk_space`、ときに空きディスク領域それ小さいし、しきい値は、クエリが停止し、スローされます `ErrorCodes::NOT_ENOUGH_SPACE`. [\#6678](https://github.com/ClickHouse/ClickHouse/pull/6678) ([Weiqing Xu](https://github.com/weiqxu)) [\#6691](https://github.com/ClickHouse/ClickHouse/pull/6691) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- スレッドによる再帰的なrwlockの削除。 スレッドはクエリ間で再利用されるため、意味がありません。 `SELECT` クエリがロックを取得するスレッド、ロックから別のスレッドの出口から。 同時に、最初のスレッドは次の方法で再利用できます `DROP` クエリ。 これはfalseにつながります “Attempt to acquire exclusive lock recursively” メッセージ [\#6771](https://github.com/ClickHouse/ClickHouse/pull/6771) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 分割 `ExpressionAnalyzer.appendJoin()`. 場所を準備する `ExpressionAnalyzer` のために `MergeJoin`. [\#6524](https://github.com/ClickHouse/ClickHouse/pull/6524) ([Artem Zuikov](https://github.com/4ertus2)) -- 追加 `mysql_native_password` MySQLの互換性サーバーへの認証プラグイン。 [\#6194](https://github.com/ClickHouse/ClickHouse/pull/6194) ([ユーリーバラノフ](https://github.com/yurriy)) -- より少ない数の `clock_gettime` のデバッグ/リリース間のABIの互換性を修正しました `Allocator` （取るに足りない問題）。 [\#6197](https://github.com/ClickHouse/ClickHouse/pull/6197) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 移動 `collectUsedColumns` から `ExpressionAnalyzer` に `SyntaxAnalyzer`. `SyntaxAnalyzer` 作る `required_source_columns` 今自体。 [\#6416](https://github.com/ClickHouse/ClickHouse/pull/6416) ([Artem Zuikov](https://github.com/4ertus2)) -- 設定を追加 `joined_subquery_requires_alias` サブセレクトおよびテーブル関数のエイリアスを要求するには `FROM` that more than one table is present (i.e. queries with JOINs). [\#6733](https://github.com/ClickHouse/ClickHouse/pull/6733) ([Artem Zuikov](https://github.com/4ertus2)) -- 抽出 `GetAggregatesVisitor` クラスから `ExpressionAnalyzer`. [\#6458](https://github.com/ClickHouse/ClickHouse/pull/6458) ([Artem Zuikov](https://github.com/4ertus2)) -- `system.query_log`:データタイプの変更 `type` コラムへの `Enum`. [\#6265](https://github.com/ClickHouse/ClickHouse/pull/6265) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- 静的リンクの `sha256_password` 認証プラグイン。 [\#6512](https://github.com/ClickHouse/ClickHouse/pull/6512) ([ユーリーバラノフ](https://github.com/yurriy)) -- 設定の余分な依存関係を避ける `compile` 働くため。 以前のバージョンでは `cannot open crti.o`, `unable to find library -lc` など。 [\#6309](https://github.com/ClickHouse/ClickHouse/pull/6309) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 悪意のあるレプリカから来る可能性のある入力のより多くの検証。 [\#6303](https://github.com/ClickHouse/ClickHouse/pull/6303) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- さて `clickhouse-obfuscator` ファイルは `clickhouse-client` パッケージ。 以前のバージョンでは、 `clickhouse obfuscator` （空白を含む）。 [\#5816](https://github.com/ClickHouse/ClickHouse/issues/5816) [\#6609](https://github.com/ClickHouse/ClickHouse/pull/6609) ([ディマルブ2000](https://github.com/dimarub2000)) -- 固定行き詰まりが少なくとも二つのクエリの読み取り少なくとも二つのテーブルに異なる秩序や他のクエリを実行するddl操作の一つです。 固定も非常に珍しいデッドロックします。 [\#6764](https://github.com/ClickHouse/ClickHouse/pull/6764) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 追加 `os_thread_ids` コラムへの `system.processes` と `system.query_log` のためのデバッグ可能です。 [\#6763](https://github.com/ClickHouse/ClickHouse/pull/6763) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 次の場合に発生するphp mysqlnd拡張バグの回避策 `sha256_password` デフォルトの認証プラグインとして使用されます。 [\#6031](https://github.com/ClickHouse/ClickHouse/issues/6031)). [\#6113](https://github.com/ClickHouse/ClickHouse/pull/6113) ([ユーリーバラノフ](https://github.com/yurriy)) -- Nullability列が変更された不要な場所を削除します。 [\#6693](https://github.com/ClickHouse/ClickHouse/pull/6693) ([Artem Zuikov](https://github.com/4ertus2)) -- 設定のデフォルト値 `queue_max_wait_ms` 現在の値（五秒）は意味をなさないので、ゼロに。 この設定を使用している場合は、まれな状況があります。 追加された設定 `replace_running_query_max_wait_ms`, `kafka_max_wait_ms` と `connection_pool_max_wait_ms` 曖昧さ回避のために。 [\#6692](https://github.com/ClickHouse/ClickHouse/pull/6692) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 抽出 `SelectQueryExpressionAnalyzer` から `ExpressionAnalyzer`. 選択されていないクエリの最後のクエリを保持します。 [\#6499](https://github.com/ClickHouse/ClickHouse/pull/6499) ([Artem Zuikov](https://github.com/4ertus2)) -- 重複する入力および出力形式を削除しました。 [\#6239](https://github.com/ClickHouse/ClickHouse/pull/6239) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- ユーザーの上書きを許可する `poll_interval` と `idle_connection_timeout` 接続時の設定。 [\#6230](https://github.com/ClickHouse/ClickHouse/pull/6230) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- `MergeTree` 現在、追加オプション `ttl_only_drop_parts` （デフォルトでは無効）パーツの部分的な枝刈りを避けるため、パーツ内のすべての行が期限切れになったときに完全に削除されます。 [\#6191](https://github.com/ClickHouse/ClickHouse/pull/6191) ([Catalonia\_comarques.Kgm](https://github.com/svladykin)) -- セットインデックス関数の型チェック。 関数が間違った型を持つ場合は例外をスローします。 これはubsanでファズテストを修正します。 [\#6511](https://github.com/ClickHouse/ClickHouse/pull/6511) ([Nikita Vasilev](https://github.com/nikvas0)) +- のための三項論理の正しい実装 `AND/OR`. [\#6048](https://github.com/ClickHouse/ClickHouse/pull/6048) ([Alexander Kazakov](https://github.com/Akazz)) +- 有効期限が切れたTTLの値と行は、次のように削除されます `OPTIMIZE ... FINAL` query from old parts without TTL infos or with outdated TTL infos, e.g. after `ALTER ... MODIFY TTL` クエリ。 クエリの追加 `SYSTEM STOP/START TTL MERGES` 可に/を割り当てを合併TTLおよびフィルター終了しました値をすべてが合併。 [\#6274](https://github.com/ClickHouse/ClickHouse/pull/6274) ([アントン-ポポフ](https://github.com/CurtizJ)) +- 使用してクライアントのClickHouse履歴ファイルの場所を変更する可能性 `CLICKHOUSE_HISTORY_FILE` env. [\#6840](https://github.com/ClickHouse/ClickHouse/pull/6840) ([フィリモノフ](https://github.com/filimonov)) +- 削除 `dry_run` からの旗 `InterpreterSelectQuery`. … [\#6375](https://github.com/ClickHouse/ClickHouse/pull/6375) ([ニコライ-コチェトフ](https://github.com/KochetovNicolai)) +- サポート `ASOF JOIN` と `ON` セクション [\#6211](https://github.com/ClickHouse/ClickHouse/pull/6211) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- よりよい支援をスキップ指数の突然変異およびます。 のサポート `MATERIALIZE/CLEAR INDEX ... IN PARTITION` クエリ。 `UPDATE x = x` recalculatesすべての指標を使用するカラム `x`. [\#5053](https://github.com/ClickHouse/ClickHouse/pull/5053) ([ニキータ-ヴァシレフ](https://github.com/nikvas0)) +- 許可する `ATTACH` ライブビュー(たとえば、サーバー起動時)に関係なく `allow_experimental_live_view` 設定。 [\#6754](https://github.com/ClickHouse/ClickHouse/pull/6754) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- のためのスタックトレースに集まるエリプロファイラに含まれているものも含むスタックフレームが発生するqueryプロファイラです。 [\#6250](https://github.com/ClickHouse/ClickHouse/pull/6250) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 今すぐテーブル関数 `values`, `file`, `url`, `hdfs` 別名の列をサポートしています。 [\#6255](https://github.com/ClickHouse/ClickHouse/pull/6255) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 例外をスローする場合 `config.d` fileには、設定ファイルとして対応するルート要素がありません。 [\#6123](https://github.com/ClickHouse/ClickHouse/pull/6123) ([dimarub2000](https://github.com/dimarub2000)) +- のための例外メッセージに余分な情報を印刷 `no space left on device`. [\#6182](https://github.com/ClickHouse/ClickHouse/issues/6182), [\#6252](https://github.com/ClickHouse/ClickHouse/issues/6252) [\#6352](https://github.com/ClickHouse/ClickHouse/pull/6352) ([tavplubix](https://github.com/tavplubix)) +- のシャードを決定するとき `Distributed` 読み取りクエリでカバーされるテーブル `optimize_skip_unused_shards` =1)ClickHouseは今両方からの条件を点検します `prewhere` と `where` select文の句。 [\#6521](https://github.com/ClickHouse/ClickHouse/pull/6521) ([Alexander Kazakov](https://github.com/Akazz)) +- 有効 `SIMDJSON` AVX2のないしかしSSE4.2およびPCLMULの指示セットが付いている機械のため。 [\#6285](https://github.com/ClickHouse/ClickHouse/issues/6285) [\#6320](https://github.com/ClickHouse/ClickHouse/pull/6320) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- ClickHouseはファイルシステムで `O_DIRECT` 追加のチューニングなしのサポート(ZFSやBtrFSなど)。 [\#4449](https://github.com/ClickHouse/ClickHouse/issues/4449) [\#6730](https://github.com/ClickHouse/ClickHouse/pull/6730) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 最終サブクエリのプッシュダウン述語をサポート。 [\#6120](https://github.com/ClickHouse/ClickHouse/pull/6120) ([Tシーズン](https://github.com/TCeason)) [\#6162](https://github.com/ClickHouse/ClickHouse/pull/6162) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- より良い `JOIN ON` キー抽出 [\#6131](https://github.com/ClickHouse/ClickHouse/pull/6131) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- Upated `SIMDJSON`. [\#6285](https://github.com/ClickHouse/ClickHouse/issues/6285). [\#6306](https://github.com/ClickHouse/ClickHouse/pull/6306) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 最小の列の選択を最適化する `SELECT count()` クエリ。 [\#6344](https://github.com/ClickHouse/ClickHouse/pull/6344) ([アモス鳥](https://github.com/amosbird)) +- 追加 `strict` パラメータin `windowFunnel()`. ときに `strict` は、 `windowFunnel()` 一意の値にのみ条件を適用します。 [\#6548](https://github.com/ClickHouse/ClickHouse/pull/6548) ([アチンバ州](https://github.com/achimbab)) +- より安全なインタフェース `mysqlxx::Pool`. [\#6150](https://github.com/ClickHouse/ClickHouse/pull/6150) ([アヴァシリエフ](https://github.com/avasiliev)) +- オプションの行サイズ `--help` オプションに対応した端末のサイズです。 [\#6590](https://github.com/ClickHouse/ClickHouse/pull/6590) ([dimarub2000](https://github.com/dimarub2000)) +- 無効にする “read in order” キーなしの集計の最適化。 [\#6599](https://github.com/ClickHouse/ClickHouse/pull/6599) ([アントン-ポポフ](https://github.com/CurtizJ)) +- HTTPステータスコード `INCORRECT_DATA` と `TYPE_MISMATCH` エラーコードをデフォルトから変更 `500 Internal Server Error` に `400 Bad Request`. [\#6271](https://github.com/ClickHouse/ClickHouse/pull/6271) ([アレクサンドロダン](https://github.com/a-rodin)) +- 結合オブジェクトの移動 `ExpressionAction` に `AnalyzedJoin`. `ExpressionAnalyzer` と `ExpressionAction` 知らない `Join` もう授業だ その論理は `AnalyzedJoin` アイフェイス [\#6801](https://github.com/ClickHouse/ClickHouse/pull/6801) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- 固定可能な行き詰まりの分散クエリーの資料はlocalhostでのクエリを送ネットワーク経由で接続します。 [\#6759](https://github.com/ClickHouse/ClickHouse/pull/6759) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 複数のテーブルの意味を変更 `RENAME` 可能なデッドロックを避けるため。 [\#6757](https://github.com/ClickHouse/ClickHouse/issues/6757). [\#6756](https://github.com/ClickHouse/ClickHouse/pull/6756) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 書き換えMySQLサーバーの互換性防止への負荷フルパケットペイロードに。 各接続のメモリ消費を約 `2 * DBMS_DEFAULT_BUFFER_SIZE` (読み取り/書き込みバッファ)。 [\#5811](https://github.com/ClickHouse/ClickHouse/pull/5811) ([ユーリー-バラノフ](https://github.com/yurriy)) +- クエリセマンティクスについて何も知らなくてもパーサーからAST別名の解釈ロジックを移動します。 [\#6108](https://github.com/ClickHouse/ClickHouse/pull/6108) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- もう少し安全な解析 `NamesAndTypesList`. [\#6408](https://github.com/ClickHouse/ClickHouse/issues/6408). [\#6410](https://github.com/ClickHouse/ClickHouse/pull/6410) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- `clickhouse-copier`:使用を許可 `where_condition` 設定から `partition_key` エイリアスクエリのためのチェック分配の存在についても同様とすでに使用されただけでデータを読み込むクエリ). [\#6577](https://github.com/ClickHouse/ClickHouse/pull/6577) ([プロラー](https://github.com/proller)) +- オプションのメッセージ引数を `throwIf`. ([\#5772](https://github.com/ClickHouse/ClickHouse/issues/5772)) [\#6329](https://github.com/ClickHouse/ClickHouse/pull/6329) ([Vdimir](https://github.com/Vdimir)) +- 挿入データの送信中にサーバー例外が発生し、クライアントでも処理されます。 [\#5891](https://github.com/ClickHouse/ClickHouse/issues/5891) [\#6711](https://github.com/ClickHouse/ClickHouse/pull/6711) ([dimarub2000](https://github.com/dimarub2000)) +- 指標を追加 `DistributedFilesToInsert` その総数のファイルをファイルシステムを送信リモートサーバーに配布します。 この数は、すべてのシャードで合計されます。 [\#6600](https://github.com/ClickHouse/ClickHouse/pull/6600) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- ほとんどの結合の移動準備ロジックから `ExpressionAction/ExpressionAnalyzer` に `AnalyzedJoin`. [\#6785](https://github.com/ClickHouse/ClickHouse/pull/6785) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- ツァンを修正 [警告](https://clickhouse-test-reports.s3.yandex.net/6399/c1c1d1daa98e199e620766f1bd06a5921050a00d/functional_stateful_tests_(thread).html) ‘lock-order-inversion’. [\#6740](https://github.com/ClickHouse/ClickHouse/pull/6740) ([ヴァシーリー-ネムコフ](https://github.com/Enmk)) +- Linuxの機能の欠如に関するより良い情報メッセージ。 致命的なエラーのログ “fatal” レベルとして迎えることを見 `system.text_log`. [\#6441](https://github.com/ClickHouse/ClickHouse/pull/6441) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- ディスクへの一時データのダンプを有効にして、メモリ使用量を制限する場合 `GROUP BY`, `ORDER BY` ディスクの空き容量をチェックしませんでした。 修正は、新しい設定を追加します `min_free_disk_space`、空きディスク領域は、それがしきい値より小さい場合、クエリが停止し、スローされます `ErrorCodes::NOT_ENOUGH_SPACE`. [\#6678](https://github.com/ClickHouse/ClickHouse/pull/6678) ([Weiqing Xu](https://github.com/weiqxu)) [\#6691](https://github.com/ClickHouse/ClickHouse/pull/6691) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- スレッドによる再帰rwlockを削除しました。 スレッドはクエリ間で再利用されるため、意味がありません。 `SELECT` クエリがロックを取得するスレッド、ロックから別のスレッドの出口から。 同時に、最初のスレッドは次のように再利用できます `DROP` クエリ。 これはfalseにつながります “Attempt to acquire exclusive lock recursively” メッセージ [\#6771](https://github.com/ClickHouse/ClickHouse/pull/6771) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 分割 `ExpressionAnalyzer.appendJoin()`. 場所を準備する `ExpressionAnalyzer` のために `MergeJoin`. [\#6524](https://github.com/ClickHouse/ClickHouse/pull/6524) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- 追加 `mysql_native_password` MySQL互換サーバーへの認証プラグイン。 [\#6194](https://github.com/ClickHouse/ClickHouse/pull/6194) ([ユーリー-バラノフ](https://github.com/yurriy)) +- より少ない数の `clock_gettime` デバッグ/リリース間のABIの互換性を修正しました。 `Allocator` （重要でない問題）。 [\#6197](https://github.com/ClickHouse/ClickHouse/pull/6197) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 移動 `collectUsedColumns` から `ExpressionAnalyzer` に `SyntaxAnalyzer`. `SyntaxAnalyzer` 作る `required_source_columns` 今自体。 [\#6416](https://github.com/ClickHouse/ClickHouse/pull/6416) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- 設定を追加 `joined_subquery_requires_alias` サブセレクトおよびテーブル関数にエイリアスを要求するには `FROM` that more than one table is present (i.e. queries with JOINs). [\#6733](https://github.com/ClickHouse/ClickHouse/pull/6733) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- 抽出 `GetAggregatesVisitor` クラスから `ExpressionAnalyzer`. [\#6458](https://github.com/ClickHouse/ClickHouse/pull/6458) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- `system.query_log`:データ型の変更 `type` 列へ `Enum`. [\#6265](https://github.com/ClickHouse/ClickHouse/pull/6265) ([ニキータ-ミハイロフ](https://github.com/nikitamikhaylov)) +- の静的リンク `sha256_password` 認証プラグイン。 [\#6512](https://github.com/ClickHouse/ClickHouse/pull/6512) ([ユーリー-バラノフ](https://github.com/yurriy)) +- 設定の余分な依存関係を避ける `compile` 仕事に 以前のバージョンでは、 `cannot open crti.o`, `unable to find library -lc` 等。 [\#6309](https://github.com/ClickHouse/ClickHouse/pull/6309) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 悪意のあるレプリカから来る可能性のある入力のより多くの検証。 [\#6303](https://github.com/ClickHouse/ClickHouse/pull/6303) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- さて `clickhouse-obfuscator` ファイルは `clickhouse-client` パッケージ。 以前のバージョンでは `clickhouse obfuscator` （空白付き）。 [\#5816](https://github.com/ClickHouse/ClickHouse/issues/5816) [\#6609](https://github.com/ClickHouse/ClickHouse/pull/6609) ([dimarub2000](https://github.com/dimarub2000)) +- 私たちは、異なる順序で少なくとも二つのテーブルを読み取り、いずれかのテーブルにDDL操作を実行する別のクエリを少なくとも二つのクエリを持って 非常に稀なデッドロックを修正しました。 [\#6764](https://github.com/ClickHouse/ClickHouse/pull/6764) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 追加 `os_thread_ids` 列へ `system.processes` と `system.query_log` より良いデバッグの可能性。 [\#6763](https://github.com/ClickHouse/ClickHouse/pull/6763) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 以下の場合に発生するPHP mysqlnd拡張バグの回避策 `sha256_password` デフォルトの認証プラグインとして使用されます [\#6031](https://github.com/ClickHouse/ClickHouse/issues/6031)). [\#6113](https://github.com/ClickHouse/ClickHouse/pull/6113) ([ユーリー-バラノフ](https://github.com/yurriy)) +- Null許容列が変更された不要な場所を削除します。 [\#6693](https://github.com/ClickHouse/ClickHouse/pull/6693) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- デフォルト値の設定 `queue_max_wait_ms` 現在の値（五秒）は意味がありませんので、ゼロにします。 この設定が使用されることはまれです。 設定の追加 `replace_running_query_max_wait_ms`, `kafka_max_wait_ms` と `connection_pool_max_wait_ms` 曖昧さの解消のために。 [\#6692](https://github.com/ClickHouse/ClickHouse/pull/6692) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 抽出 `SelectQueryExpressionAnalyzer` から `ExpressionAnalyzer`. 非選択クエリの最後のクエリを保持します。 [\#6499](https://github.com/ClickHouse/ClickHouse/pull/6499) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- 入力形式と出力形式の重複を削除しました。 [\#6239](https://github.com/ClickHouse/ClickHouse/pull/6239) ([ニコライ-コチェトフ](https://github.com/KochetovNicolai)) +- ユーザーに上書きを許可する `poll_interval` と `idle_connection_timeout` 接続時の設定。 [\#6230](https://github.com/ClickHouse/ClickHouse/pull/6230) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- `MergeTree` 現在、追加オプション `ttl_only_drop_parts` （デフォルトでは無効）部品の部分的な剪定を避けるため、部品内のすべての行が期限切れになったときに完全に削除されます。 [\#6191](https://github.com/ClickHouse/ClickHouse/pull/6191) ([セルジ-ウラジキン](https://github.com/svladykin)) +- 型は、set index関数をチェックします。 関数の型が間違っている場合は例外をスローします。 これは、UBSanでファズテストを修正します。 [\#6511](https://github.com/ClickHouse/ClickHouse/pull/6511) ([ニキータ-ヴァシレフ](https://github.com/nikvas0)) #### 性能向上 {#performance-improvement-2} -- クエリを最適化する `ORDER BY expressions` 句、どこ `expressions` プレフィックスとソートキーが一致している `MergeTree` テーブル。 この最適化は `optimize_read_in_order` 設定。 [\#6054](https://github.com/ClickHouse/ClickHouse/pull/6054) [\#6629](https://github.com/ClickHouse/ClickHouse/pull/6629) ([アントン-ポポフ](https://github.com/CurtizJ)) -- 使用に応じることは、複数のスレッドの中で部品の搭載となります。 [\#6372](https://github.com/ClickHouse/ClickHouse/issues/6372) [\#6074](https://github.com/ClickHouse/ClickHouse/issues/6074) [\#6438](https://github.com/ClickHouse/ClickHouse/pull/6438) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 集計関数の状態を更新するバッチバリアントを実装。 で与えられる実装になっていると性能です。 [\#6435](https://github.com/ClickHouse/ClickHouse/pull/6435) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- を使用して `FastOps` 関数のライブラリ `exp`, `log`, `sigmoid`, `tanh`. FastOpsはマイケルParakhin（YandexのCTO）からの高速ベクトル数学ライブラリです。 改善された性能の `exp` と `log` 機能6回以上。 を機能 `exp` と `log` から `Float32` 引数戻ります `Float32` （以前のバージョンでは、常に戻ります `Float64`). さて `exp(nan)` 戻る可能性がある `inf`. 結果の `exp` と `log` 関数は、真の答えに最も近いマシン表現可能な番号ではないかもしれません。 [\#6254](https://github.com/ClickHouse/ClickHouse/pull/6254) ([alexey-milovidov](https://github.com/alexey-milovidov))働くfastopsを作るダニラKuteninの変形を使用して [\#6317](https://github.com/ClickHouse/ClickHouse/pull/6317) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- の連続したキーの最適化を無効にする `UInt8/16`. [\#6298](https://github.com/ClickHouse/ClickHouse/pull/6298) [\#6701](https://github.com/ClickHouse/ClickHouse/pull/6701) ([azerbaijan.kgm](https://github.com/akuzm)) -- 改善された性能の `simdjson` の動的割り当てを取り除くことによって `ParsedJson::Iterator`. [\#6479](https://github.com/ClickHouse/ClickHouse/pull/6479) ([Vitaly Baranov](https://github.com/vitlibar)) -- メモリを割り当てるときの事前フォールトページ `mmap()`. [\#6667](https://github.com/ClickHouse/ClickHouse/pull/6667) ([azerbaijan.kgm](https://github.com/akuzm)) -- 固定性能のバグを修正 `Decimal` 比較。 [\#6380](https://github.com/ClickHouse/ClickHouse/pull/6380) ([Artem Zuikov](https://github.com/4ertus2)) +- クエリの最適化 `ORDER BY expressions` 句,where `expressions` ソートキーとプレフィックスが一致している `MergeTree` テーブル この最適化は `optimize_read_in_order` 設定。 [\#6054](https://github.com/ClickHouse/ClickHouse/pull/6054) [\#6629](https://github.com/ClickHouse/ClickHouse/pull/6629) ([アントン-ポポフ](https://github.com/CurtizJ)) +- 使用に応じることは、複数のスレッドの中で部品の搭載となります。 [\#6372](https://github.com/ClickHouse/ClickHouse/issues/6372) [\#6074](https://github.com/ClickHouse/ClickHouse/issues/6074) [\#6438](https://github.com/ClickHouse/ClickHouse/pull/6438) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 実施バッチ変異体の更新に集計機能です。 で与えられる実装になっていると性能です。 [\#6435](https://github.com/ClickHouse/ClickHouse/pull/6435) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- を使用して `FastOps` 関数用ライブラリ `exp`, `log`, `sigmoid`, `tanh`. FastOpsはマイケルParakhin（YandexのCTO）から高速ベクトル数学ライブラリです。 の改善された性能 `exp` と `log` 機能6回以上。 機能 `exp` と `log` から `Float32` 引数が返されます `Float32` （以前のバージョンでは、常に戻ります `Float64`). さて `exp(nan)` 返すように `inf`. の結果 `exp` と `log` 関数は、真の答えに最も近いマシン表現可能な数ではないかもしれません。 [\#6254](https://github.com/ClickHouse/ClickHouse/pull/6254) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)）Fastopsを働かせるDanila Kuteninの変形を使用して [\#6317](https://github.com/ClickHouse/ClickHouse/pull/6317) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 連続キーの最適化を無効にする `UInt8/16`. [\#6298](https://github.com/ClickHouse/ClickHouse/pull/6298) [\#6701](https://github.com/ClickHouse/ClickHouse/pull/6701) ([akuzm](https://github.com/akuzm)) +- の改善された性能 `simdjson` 図書館によくダイナミックな割り当てる `ParsedJson::Iterator`. [\#6479](https://github.com/ClickHouse/ClickHouse/pull/6479) ([ヴィタリー-バラノフ](https://github.com/vitlibar)) +- メモリを割り当てるときの障害前のページ `mmap()`. [\#6667](https://github.com/ClickHouse/ClickHouse/pull/6667) ([akuzm](https://github.com/akuzm)) +- パフォーマンスバグを修正 `Decimal` 比較。 [\#6380](https://github.com/ClickHouse/ClickHouse/pull/6380) ([アルテム-ズイコフ](https://github.com/4ertus2)) -#### ビルド/テスト/パッケージの改善 {#buildtestingpackaging-improvement-4} +#### 造り/テスト/包装の改善 {#buildtestingpackaging-improvement-4} -- コンパイラ(ランタイムテンプレートのインスタンス化)を削除します。 [\#6646](https://github.com/ClickHouse/ClickHouse/pull/6646) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 追加の性能試験への性能の低下gcc-9により孤立した。 [\#6302](https://github.com/ClickHouse/ClickHouse/pull/6302) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- テーブル機能を追加 `numbers_mt` のマルチスレッドバージョンです。 `numbers`. 更新性能試験のハッシュ機能 [\#6554](https://github.com/ClickHouse/ClickHouse/pull/6554) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 比較モード `clickhouse-benchmark` [\#6220](https://github.com/ClickHouse/ClickHouse/issues/6220) [\#6343](https://github.com/ClickHouse/ClickHouse/pull/6343) ([ディマルブ2000](https://github.com/dimarub2000)) -- スタックトレース印刷のための最善の努力。 また、追加 `SIGPROF` 実行中のスレッドのスタックトレースを出力するデバッグ信号として。 [\#6529](https://github.com/ClickHouse/ClickHouse/pull/6529) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 独自のファイル、パート10のすべての機能。 [\#6321](https://github.com/ClickHouse/ClickHouse/pull/6321) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 倍精度の定数を削除 `TABLE_IS_READ_ONLY`. [\#6566](https://github.com/ClickHouse/ClickHouse/pull/6566) ([フィリモノフ](https://github.com/filimonov)) -- のための書式設定の変更 `StringHashMap` PR [\#5417](https://github.com/ClickHouse/ClickHouse/issues/5417). [\#6700](https://github.com/ClickHouse/ClickHouse/pull/6700) ([azerbaijan.kgm](https://github.com/akuzm)) -- Join作成のためのより良いサブクエリ `ExpressionAnalyzer`. [\#6824](https://github.com/ClickHouse/ClickHouse/pull/6824) ([Artem Zuikov](https://github.com/4ertus2)) -- 冗長な条件（pvs studioによって検出された）を削除します。 [\#6775](https://github.com/ClickHouse/ClickHouse/pull/6775) ([azerbaijan.kgm](https://github.com/akuzm)) -- 別々のハッシュテーブルインタフェース `ReverseIndex`. [\#6672](https://github.com/ClickHouse/ClickHouse/pull/6672) ([azerbaijan.kgm](https://github.com/akuzm)) -- 設定のリファクタリング。 [\#6689](https://github.com/ClickHouse/ClickHouse/pull/6689) ([alesapin](https://github.com/alesapin)) -- コメントの追加 `set` インデックス関数。 [\#6319](https://github.com/ClickHouse/ClickHouse/pull/6319) ([Nikita Vasilev](https://github.com/nikvas0)) -- 増oomスコアデバッグ版プログラムを利用しています。. [\#6152](https://github.com/ClickHouse/ClickHouse/pull/6152) ([azerbaijan.kgm](https://github.com/akuzm)) +- 削除コンパイラ(ランタイムテンプレートインスタンス化でいきます。 [\#6646](https://github.com/ClickHouse/ClickHouse/pull/6646) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 追加の性能試験への性能の低下gcc-9により孤立した。 [\#6302](https://github.com/ClickHouse/ClickHouse/pull/6302) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- テーブル関数を追加 `numbers_mt` のマルチスレッドバージョンです。 `numbers`. 更新性能試験のハッシュ機能 [\#6554](https://github.com/ClickHouse/ClickHouse/pull/6554) ([ニコライ-コチェトフ](https://github.com/KochetovNicolai)) +- 比較モード `clickhouse-benchmark` [\#6220](https://github.com/ClickHouse/ClickHouse/issues/6220) [\#6343](https://github.com/ClickHouse/ClickHouse/pull/6343) ([dimarub2000](https://github.com/dimarub2000)) +- スタックトレースの印刷に最適です。 また、追加 `SIGPROF` 実行中のスレッドのスタックトレースを出力するデバッグ信号として。 [\#6529](https://github.com/ClickHouse/ClickHouse/pull/6529) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 独自のファイル内のすべての関数、パート10。 [\#6321](https://github.com/ClickHouse/ClickHouse/pull/6321) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 倍増定数を削除 `TABLE_IS_READ_ONLY`. [\#6566](https://github.com/ClickHouse/ClickHouse/pull/6566) ([フィリモノフ](https://github.com/filimonov)) +- 書式設定の変更 `StringHashMap` PR [\#5417](https://github.com/ClickHouse/ClickHouse/issues/5417). [\#6700](https://github.com/ClickHouse/ClickHouse/pull/6700) ([akuzm](https://github.com/akuzm)) +- 結合作成のためのより良いサブクエリ `ExpressionAnalyzer`. [\#6824](https://github.com/ClickHouse/ClickHouse/pull/6824) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- (PVS Studioによって見つかった)冗長な条件を削除します。 [\#6775](https://github.com/ClickHouse/ClickHouse/pull/6775) ([akuzm](https://github.com/akuzm)) +- 別々のハッシュテーブルインタフェース `ReverseIndex`. [\#6672](https://github.com/ClickHouse/ClickHouse/pull/6672) ([akuzm](https://github.com/akuzm)) +- 設定のリファクタリング。 [\#6689](https://github.com/ClickHouse/ClickHouse/pull/6689) ([アレサピン](https://github.com/alesapin)) +- コメントを追加 `set` インデックス関数。 [\#6319](https://github.com/ClickHouse/ClickHouse/pull/6319) ([ニキータ-ヴァシレフ](https://github.com/nikvas0)) +- LinuxのデバッグバージョンでOOMスコアを上げます。 [\#6152](https://github.com/ClickHouse/ClickHouse/pull/6152) ([akuzm](https://github.com/akuzm)) - HDFS HAはデバッグビルドで動作します。 [\#6650](https://github.com/ClickHouse/ClickHouse/pull/6650) ([Weiqing Xu](https://github.com/weiqxu)) -- にテストを追加しました `transform_query_for_external_database`. [\#6388](https://github.com/ClickHouse/ClickHouse/pull/6388) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Kafkaテーブルのマテリアライズドビューのテストを追加します。 [\#6509](https://github.com/ClickHouse/ClickHouse/pull/6509) ([イワン](https://github.com/abyss7)) -- よりよい造りの機構を作りなさい。 [\#6500](https://github.com/ClickHouse/ClickHouse/pull/6500) ([イワン](https://github.com/abyss7)) -- 固定 `test_external_dictionaries` 非rootユーザーの下で実行された場合の統合。 [\#6507](https://github.com/ClickHouse/ClickHouse/pull/6507) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 書き込まれたパケットの合計サイズが `DBMS_DEFAULT_BUFFER_SIZE`. [\#6204](https://github.com/ClickHouse/ClickHouse/pull/6204) ([ユーリーバラノフ](https://github.com/yurriy)) -- のテストを追加しました `RENAME` テーブルの競合状態 [\#6752](https://github.com/ClickHouse/ClickHouse/pull/6752) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 設定のデータ競争をの避けて下さい `KILL QUERY`. [\#6753](https://github.com/ClickHouse/ClickHouse/pull/6753) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- キャッ [\#6755](https://github.com/ClickHouse/ClickHouse/pull/6755) ([Vitaly Baranov](https://github.com/vitlibar)) -- Mac OSでELFオブジェクトファイルの解析を無効にする。 [\#6578](https://github.com/ClickHouse/ClickHouse/pull/6578) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 変更ログジェネレータを改善しようとします。 [\#6327](https://github.com/ClickHouse/ClickHouse/pull/6327) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 追加 `-Wshadow` GCCに切り替えます。 [\#6325](https://github.com/ClickHouse/ClickHouse/pull/6325) ([kreuzerkrieg](https://github.com/kreuzerkrieg)) -- の廃止されたコードを削除 `mimalloc` ます。 [\#6715](https://github.com/ClickHouse/ClickHouse/pull/6715) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- `zlib-ng` x86機能を決定し、この情報をグローバル変数に保存します。 これは、異なるスレッドによって同時に行うことができるdefalteInit呼び出しで行われます。 を避けるマルチスレッドに書き込み、図書館で起動します。 [\#6141](https://github.com/ClickHouse/ClickHouse/pull/6141) ([azerbaijan.kgm](https://github.com/akuzm)) -- In結合で修正されたバグの回帰テスト [\#5192](https://github.com/ClickHouse/ClickHouse/issues/5192). [\#6147](https://github.com/ClickHouse/ClickHouse/pull/6147) ([Bakhtiyor Ruziev](https://github.com/theruziev)) -- 固定msanレポート。 [\#6144](https://github.com/ClickHouse/ClickHouse/pull/6144) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 修正フラッピングttlテスト。 [\#6782](https://github.com/ClickHouse/ClickHouse/pull/6782) ([アントン-ポポフ](https://github.com/CurtizJ)) -- 固定偽データレースで `MergeTreeDataPart::is_frozen` フィールド。 [\#6583](https://github.com/ClickHouse/ClickHouse/pull/6583) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- ファズテストでタイムアウトを修正しました。 以前のバージョ `SELECT * FROM numbers_mt(gccMurmurHash(''))`. [\#6582](https://github.com/ClickHouse/ClickHouse/pull/6582) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- デバッグチェックを追加 `static_cast` 列の。 [\#6581](https://github.com/ClickHouse/ClickHouse/pull/6581) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 公式のrpmパッケージでのoracle linuxのサポート。 [\#6356](https://github.com/ClickHouse/ClickHouse/issues/6356) [\#6585](https://github.com/ClickHouse/ClickHouse/pull/6585) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- からの変更json perftests `once` に `loop` タイプ。 [\#6536](https://github.com/ClickHouse/ClickHouse/pull/6536) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- `odbc-bridge.cpp` 定義 `main()` したがって、それは `clickhouse-lib`. [\#6538](https://github.com/ClickHouse/ClickHouse/pull/6538) ([Orivej Desh](https://github.com/orivej)) -- クラッシュのテスト `FULL|RIGHT JOIN` 右側のテーブルのキーにヌルがあります。 [\#6362](https://github.com/ClickHouse/ClickHouse/pull/6362) ([Artem Zuikov](https://github.com/4ertus2)) -- 念のためにエイリアスの拡張の制限のためのテストを追加しました。 [\#6442](https://github.com/ClickHouse/ClickHouse/pull/6442) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- から切り替え `boost::filesystem` に `std::filesystem` 適切な場合。 [\#6253](https://github.com/ClickHouse/ClickHouse/pull/6253) [\#6385](https://github.com/ClickHouse/ClickHouse/pull/6385) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 追加rpmパッケージです。 [\#6251](https://github.com/ClickHouse/ClickHouse/pull/6251) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 固定のテストを追加する `Unknown identifier` での例外 `IN` セクション。 [\#6708](https://github.com/ClickHouse/ClickHouse/pull/6708) ([Artem Zuikov](https://github.com/4ertus2)) -- 簡略化 `shared_ptr_helper` 人々はそれを理解困難に直面しているので。 [\#6675](https://github.com/ClickHouse/ClickHouse/pull/6675) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 固定gorillaとdoubledeltaコーデックのパフォーマンステストを追加しました。 [\#6179](https://github.com/ClickHouse/ClickHouse/pull/6179) ([Vasily Nemkov](https://github.com/Enmk)) -- 統合テストの分割 `test_dictionaries` 4つの別々のテストに。 [\#6776](https://github.com/ClickHouse/ClickHouse/pull/6776) ([Vitaly Baranov](https://github.com/vitlibar)) -- PVS-Studioの警告を修正する `PipelineExecutor`. [\#6777](https://github.com/ClickHouse/ClickHouse/pull/6777) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 使用を許可する `library` ASanの辞書ソース。 [\#6482](https://github.com/ClickHouse/ClickHouse/pull/6482) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Prのリストからchangelogを生成するオプションを追加しました。 [\#6350](https://github.com/ClickHouse/ClickHouse/pull/6350) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- をロック `TinyLog` 読む場合の貯蔵。 [\#6226](https://github.com/ClickHouse/ClickHouse/pull/6226) ([azerbaijan.kgm](https://github.com/akuzm)) -- チェックを破symlinks ci. [\#6634](https://github.com/ClickHouse/ClickHouse/pull/6634) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- の増加のタイムアウト “stack overflow” デバッグビルドでは長い時間がかかるため、テストします。 [\#6637](https://github.com/ClickHouse/ClickHouse/pull/6637) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 二重の空白のチェックを追加しました。 [\#6643](https://github.com/ClickHouse/ClickHouse/pull/6643) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 修正 `new/delete` メモリ追跡ときビルドで消毒。 追跡は明確ではありません。 テストでのメモリ制限の例外を防止するだけです。 [\#6450](https://github.com/ClickHouse/ClickHouse/pull/6450) ([Artem Zuikov](https://github.com/4ertus2)) +- にテストを追加しました `transform_query_for_external_database`. [\#6388](https://github.com/ClickHouse/ClickHouse/pull/6388) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- カフカ表の複数のマテリアライズドビューのテストを追加します。 [\#6509](https://github.com/ClickHouse/ClickHouse/pull/6509) ([イワン](https://github.com/abyss7)) +- より良いビルドスキームを作る。 [\#6500](https://github.com/ClickHouse/ClickHouse/pull/6500) ([イワン](https://github.com/abyss7)) +- 固定 `test_external_dictionaries` 非rootユーザーの下で実行された場合の統合。 [\#6507](https://github.com/ClickHouse/ClickHouse/pull/6507) ([ニコライ-コチェトフ](https://github.com/KochetovNicolai)) +- このバグは、書き込まれたパケットの合計サイズが `DBMS_DEFAULT_BUFFER_SIZE`. [\#6204](https://github.com/ClickHouse/ClickHouse/pull/6204) ([ユーリー-バラノフ](https://github.com/yurriy)) +- のテストを追加しました `RENAME` テーブル競合状態 [\#6752](https://github.com/ClickHouse/ClickHouse/pull/6752) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 設定のデータ競争を避けなさい `KILL QUERY`. [\#6753](https://github.com/ClickHouse/ClickHouse/pull/6753) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- キャッシュ辞書によるエラー処理の統合テストを追加します。 [\#6755](https://github.com/ClickHouse/ClickHouse/pull/6755) ([ヴィタリー-バラノフ](https://github.com/vitlibar)) +- それは意味がないので、Mac OS上でELFオブジェクトファイルの解析を無効にします。 [\#6578](https://github.com/ClickHouse/ClickHouse/pull/6578) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- を試みることchangelog発生装置。 [\#6327](https://github.com/ClickHouse/ClickHouse/pull/6327) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 追加 `-Wshadow` GCCに切り替えます。 [\#6325](https://github.com/ClickHouse/ClickHouse/pull/6325) ([クロイツェルクリーク](https://github.com/kreuzerkrieg)) +- 削除された古いコード `mimalloc` サポート。 [\#6715](https://github.com/ClickHouse/ClickHouse/pull/6715) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- `zlib-ng` x86の機能を決定し、この情報をグローバル変数に保存します。 これは、同時に異なるスレッドによって行われてもよいdefalteInit呼び出しで行われます。 を避けるマルチスレッドに書き込み、図書館で起動します。 [\#6141](https://github.com/ClickHouse/ClickHouse/pull/6141) ([akuzm](https://github.com/akuzm)) +- 結合で修正されたバグの回帰テスト [\#5192](https://github.com/ClickHouse/ClickHouse/issues/5192). [\#6147](https://github.com/ClickHouse/ClickHouse/pull/6147) ([バフティヤール-ルジエフ](https://github.com/theruziev)) +- MSanレポートを修正しました。 [\#6144](https://github.com/ClickHouse/ClickHouse/pull/6144) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- TTLテストをフラッピング修正。 [\#6782](https://github.com/ClickHouse/ClickHouse/pull/6782) ([アントン-ポポフ](https://github.com/CurtizJ)) +- で修正された偽のデータレース `MergeTreeDataPart::is_frozen` フィールド [\#6583](https://github.com/ClickHouse/ClickHouse/pull/6583) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- ファズテストのタイムアウトを修正しました。 旧バージョンで簡単にfalse電話を切るクエリ `SELECT * FROM numbers_mt(gccMurmurHash(''))`. [\#6582](https://github.com/ClickHouse/ClickHouse/pull/6582) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- デバッグチェックを追加 `static_cast` 列の。 [\#6581](https://github.com/ClickHouse/ClickHouse/pull/6581) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 公式RPMパッケージでのOracle Linuxのサポート。 [\#6356](https://github.com/ClickHouse/ClickHouse/issues/6356) [\#6585](https://github.com/ClickHouse/ClickHouse/pull/6585) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- Jsonパーフテストを `once` に `loop` タイプ。 [\#6536](https://github.com/ClickHouse/ClickHouse/pull/6536) ([ニコライ-コチェトフ](https://github.com/KochetovNicolai)) +- `odbc-bridge.cpp` 定義 `main()` したがって、それは `clickhouse-lib`. [\#6538](https://github.com/ClickHouse/ClickHouse/pull/6538) ([オリヴェイ-デシュ](https://github.com/orivej)) +- クラッシュのテスト `FULL|RIGHT JOIN` 右のテーブルのキーにnullがあります。 [\#6362](https://github.com/ClickHouse/ClickHouse/pull/6362) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- 念のためにエイリアスの拡張の制限のテストを追加しました。 [\#6442](https://github.com/ClickHouse/ClickHouse/pull/6442) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- から切り替え `boost::filesystem` に `std::filesystem` 必要に応じて。 [\#6253](https://github.com/ClickHouse/ClickHouse/pull/6253) [\#6385](https://github.com/ClickHouse/ClickHouse/pull/6385) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 追加RPMパッケージです。 [\#6251](https://github.com/ClickHouse/ClickHouse/pull/6251) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 固定のテストを追加する `Unknown identifier` 例外 `IN` セクション [\#6708](https://github.com/ClickHouse/ClickHouse/pull/6708) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- 簡素化 `shared_ptr_helper` が直面する人びとの苦難を理解します。 [\#6675](https://github.com/ClickHouse/ClickHouse/pull/6675) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 固定GorillaとDoubleDeltaコーデックのパフォーマンステストを追加しました。 [\#6179](https://github.com/ClickHouse/ClickHouse/pull/6179) ([ヴァシーリー-ネムコフ](https://github.com/Enmk)) +- 統合テストの分割 `test_dictionaries` 4つの別々のテストに。 [\#6776](https://github.com/ClickHouse/ClickHouse/pull/6776) ([ヴィタリー-バラノフ](https://github.com/vitlibar)) +- PVS-Studioの警告を修正 `PipelineExecutor`. [\#6777](https://github.com/ClickHouse/ClickHouse/pull/6777) ([ニコライ-コチェトフ](https://github.com/KochetovNicolai)) +- 使用を許可する `library` ASanと辞書ソース。 [\#6482](https://github.com/ClickHouse/ClickHouse/pull/6482) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- PRsのリストから変更履歴を生成するオプションを追加しました。 [\#6350](https://github.com/ClickHouse/ClickHouse/pull/6350) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- ロック `TinyLog` 読む場合の貯蔵。 [\#6226](https://github.com/ClickHouse/ClickHouse/pull/6226) ([akuzm](https://github.com/akuzm)) +- チェックを破symlinks CI. [\#6634](https://github.com/ClickHouse/ClickHouse/pull/6634) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- タイムアウトの増加 “stack overflow” 試験で時間がかかる場合にはデバッグ。 [\#6637](https://github.com/ClickHouse/ClickHouse/pull/6637) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 二重の空白のチェックを追加しました。 [\#6643](https://github.com/ClickHouse/ClickHouse/pull/6643) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 修正 `new/delete` サニタイザーでビルドするときのメモリ追跡。 追跡は明確ではない。 テストでのメモリ制限の例外のみを防ぎます。 [\#6450](https://github.com/ClickHouse/ClickHouse/pull/6450) ([アルテム-ズイコフ](https://github.com/4ertus2)) - リンク中に未定義のシンボルのチェックを有効にします。 [\#6453](https://github.com/ClickHouse/ClickHouse/pull/6453) ([イワン](https://github.com/abyss7)) -- 再構築を避ける `hyperscan` 毎日です。 [\#6307](https://github.com/ClickHouse/ClickHouse/pull/6307) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 固定ubsanレポートで `ProtobufWriter`. [\#6163](https://github.com/ClickHouse/ClickHouse/pull/6163) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 互換性がないため、クエリプロファイラーをサニタイザーで使用することはできません。 [\#6769](https://github.com/ClickHouse/ClickHouse/pull/6769) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 追加試験のためのリロード辞書の後に失敗するタイマー. [\#6114](https://github.com/ClickHouse/ClickHouse/pull/6114) ([Vitaly Baranov](https://github.com/vitlibar)) -- 矛盾を修正する `PipelineExecutor::prepareProcessor` 引数の型。 [\#6494](https://github.com/ClickHouse/ClickHouse/pull/6494) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 追加の試験のための悪いuriです。 [\#6493](https://github.com/ClickHouse/ClickHouse/pull/6493) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- より多くのチェックを追加 `CAST` 機能。 こいつの間にか．ファジィテストです。 [\#6346](https://github.com/ClickHouse/ClickHouse/pull/6346) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 追加 `gcc-9` サポートへの `docker/builder` ローカルで画像を構築するコンテナ。 [\#6333](https://github.com/ClickHouse/ClickHouse/pull/6333) ([Gleb Novikov](https://github.com/NanoBjorn)) -- 主キーのテスト `LowCardinality(String)`. [\#5044](https://github.com/ClickHouse/ClickHouse/issues/5044) [\#6219](https://github.com/ClickHouse/ClickHouse/pull/6219) ([ディマルブ2000](https://github.com/dimarub2000)) -- 固定試験の影響を受けゆっくりとしたスタックトレースの印刷もできます。 [\#6315](https://github.com/ClickHouse/ClickHouse/pull/6315) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- クラッシュのテストケースを追加する `groupUniqArray` 固定で [\#6029](https://github.com/ClickHouse/ClickHouse/pull/6029). [\#4402](https://github.com/ClickHouse/ClickHouse/issues/4402) [\#6129](https://github.com/ClickHouse/ClickHouse/pull/6129) ([azerbaijan.kgm](https://github.com/akuzm)) -- 固定インデックス突然変異テスト。 [\#6645](https://github.com/ClickHouse/ClickHouse/pull/6645) ([Nikita Vasilev](https://github.com/nikvas0)) -- パフォーマンステス [\#6427](https://github.com/ClickHouse/ClickHouse/pull/6427) ([azerbaijan.kgm](https://github.com/akuzm)) -- マテリアライズドビューは、疑わしい低基数タイプに関する設定に関係なく、低基数タイプで作成できるようになりました。 [\#6428](https://github.com/ClickHouse/ClickHouse/pull/6428) ([Olga Khvostikova](https://github.com/stavrolia)) -- 更新されたテスト `send_logs_level` 設定。 [\#6207](https://github.com/ClickHouse/ClickHouse/pull/6207) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Gcc-8.2でビルドを修正しました。 [\#6196](https://github.com/ClickHouse/ClickHouse/pull/6196) ([Max Akhmedov](https://github.com/zlobober)) +- 再構築を避ける `hyperscan` 毎日 [\#6307](https://github.com/ClickHouse/ClickHouse/pull/6307) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 固定されたUBSanのレポート `ProtobufWriter`. [\#6163](https://github.com/ClickHouse/ClickHouse/pull/6163) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- クエリプロファイラは互換性がないため、サニタイザでの使用を許可しないでください。 [\#6769](https://github.com/ClickHouse/ClickHouse/pull/6769) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- タイマーで失敗した後に辞書を再ロードするテストを追加します。 [\#6114](https://github.com/ClickHouse/ClickHouse/pull/6114) ([ヴィタリー-バラノフ](https://github.com/vitlibar)) +- で矛盾を修正 `PipelineExecutor::prepareProcessor` 引数の型。 [\#6494](https://github.com/ClickHouse/ClickHouse/pull/6494) ([ニコライ-コチェトフ](https://github.com/KochetovNicolai)) +- 不正なUriのテストを追加しました。 [\#6493](https://github.com/ClickHouse/ClickHouse/pull/6493) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- より多くのチェックを追加 `CAST` 機能。 こいつの間にか．ファジィテストです。 [\#6346](https://github.com/ClickHouse/ClickHouse/pull/6346) ([ニコライ-コチェトフ](https://github.com/KochetovNicolai)) +- 追加 `gcc-9` サポートへの `docker/builder` 画像をローカルに作成するコンテナ。 [\#6333](https://github.com/ClickHouse/ClickHouse/pull/6333) ([グレブ-ノビコフ](https://github.com/NanoBjorn)) +- 主キーのテスト `LowCardinality(String)`. [\#5044](https://github.com/ClickHouse/ClickHouse/issues/5044) [\#6219](https://github.com/ClickHouse/ClickHouse/pull/6219) ([dimarub2000](https://github.com/dimarub2000)) +- 遅いスタックトレース印刷の影響を修正しました。 [\#6315](https://github.com/ClickHouse/ClickHouse/pull/6315) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- クラッシュのテストケースを追加する `groupUniqArray` で固定 [\#6029](https://github.com/ClickHouse/ClickHouse/pull/6029). [\#4402](https://github.com/ClickHouse/ClickHouse/issues/4402) [\#6129](https://github.com/ClickHouse/ClickHouse/pull/6129) ([akuzm](https://github.com/akuzm)) +- 固定指標変異テスト。 [\#6645](https://github.com/ClickHouse/ClickHouse/pull/6645) ([ニキータ-ヴァシレフ](https://github.com/nikvas0)) +- パフォーマンステス [\#6427](https://github.com/ClickHouse/ClickHouse/pull/6427) ([akuzm](https://github.com/akuzm)) +- マテリアライズドビューは、疑わしい低い基数タイプに関する設定に関係なく、低い基数タイプで作成できるようになりました。 [\#6428](https://github.com/ClickHouse/ClickHouse/pull/6428) ([Olga Khvostikova](https://github.com/stavrolia)) +- 更新されたテスト `send_logs_level` 設定。 [\#6207](https://github.com/ClickHouse/ClickHouse/pull/6207) ([ニコライ-コチェトフ](https://github.com/KochetovNicolai)) +- Gcc-8.2の下でビルドを修正しました。 [\#6196](https://github.com/ClickHouse/ClickHouse/pull/6196) ([マックス-アクメドフ](https://github.com/zlobober)) - 内部libc++でビルドを修正しました。 [\#6724](https://github.com/ClickHouse/ClickHouse/pull/6724) ([イワン](https://github.com/abyss7)) -- 共有ビルドを修正する `rdkafka` ライブラリ [\#6101](https://github.com/ClickHouse/ClickHouse/pull/6101) ([イワン](https://github.com/abyss7)) -- Mac OSビルドの修正（不完全）。 [\#6390](https://github.com/ClickHouse/ClickHouse/pull/6390) ([alexey-milovidov](https://github.com/alexey-milovidov)) [\#6429](https://github.com/ClickHouse/ClickHouse/pull/6429) ([alex-zaitsev](https://github.com/alex-zaitsev)) -- 修正 “splitted” ビルド。 [\#6618](https://github.com/ClickHouse/ClickHouse/pull/6618) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- その他のビルドの修正: [\#6186](https://github.com/ClickHouse/ClickHouse/pull/6186) ([アモスの鳥](https://github.com/amosbird)) [\#6486](https://github.com/ClickHouse/ClickHouse/pull/6486) [\#6348](https://github.com/ClickHouse/ClickHouse/pull/6348) ([vxider](https://github.com/Vxider)) [\#6744](https://github.com/ClickHouse/ClickHouse/pull/6744) ([イワン](https://github.com/abyss7)) [\#6016](https://github.com/ClickHouse/ClickHouse/pull/6016) [\#6421](https://github.com/ClickHouse/ClickHouse/pull/6421) [\#6491](https://github.com/ClickHouse/ClickHouse/pull/6491) ([proller](https://github.com/proller)) +- 共有ビルドを修正 `rdkafka` ライブラリ [\#6101](https://github.com/ClickHouse/ClickHouse/pull/6101) ([イワン](https://github.com/abyss7)) +- Mac OSのビルド（不完全）のための修正。 [\#6390](https://github.com/ClickHouse/ClickHouse/pull/6390) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) [\#6429](https://github.com/ClickHouse/ClickHouse/pull/6429) ([アレックス-ザイツェフ](https://github.com/alex-zaitsev)) +- 修正 “splitted” ビルド [\#6618](https://github.com/ClickHouse/ClickHouse/pull/6618) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- その他のビルド修正: [\#6186](https://github.com/ClickHouse/ClickHouse/pull/6186) ([アモス鳥](https://github.com/amosbird)) [\#6486](https://github.com/ClickHouse/ClickHouse/pull/6486) [\#6348](https://github.com/ClickHouse/ClickHouse/pull/6348) ([vxider](https://github.com/Vxider)) [\#6744](https://github.com/ClickHouse/ClickHouse/pull/6744) ([イワン](https://github.com/abyss7)) [\#6016](https://github.com/ClickHouse/ClickHouse/pull/6016) [\#6421](https://github.com/ClickHouse/ClickHouse/pull/6421) [\#6491](https://github.com/ClickHouse/ClickHouse/pull/6491) ([プロラー](https://github.com/proller)) #### 下位互換性のない変更 {#backward-incompatible-change-3} -- 削除が使用されることが少なテーブル機能 `catBoostPool` および貯蔵 `CatBoostPool`. このテーブル機能を使用したら、電子メールをに書いて下さい `clickhouse-feedback@yandex-team.com`. CatBoost統合は引き続きサポートされることに注意してください。 [\#6279](https://github.com/ClickHouse/ClickHouse/pull/6279) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 無効にする `ANY RIGHT JOIN` と `ANY FULL JOIN` デフォルトでは。 セット `any_join_distinct_right_table_keys` それらを有効にする設定。 [\#5126](https://github.com/ClickHouse/ClickHouse/issues/5126) [\#6351](https://github.com/ClickHouse/ClickHouse/pull/6351) ([Artem Zuikov](https://github.com/4ertus2)) +- めったに使用されない表関数を削除 `catBoostPool` そして貯蔵 `CatBoostPool`. を使った場合、このテーブル機能、メールを書いてください `clickhouse-feedback@yandex-team.com`. CatBoost統合は引き続きサポートされることに注意してください。 [\#6279](https://github.com/ClickHouse/ClickHouse/pull/6279) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 無効にする `ANY RIGHT JOIN` と `ANY FULL JOIN` デフォルトでは。 セット `any_join_distinct_right_table_keys` それらを有効にする設定。 [\#5126](https://github.com/ClickHouse/ClickHouse/issues/5126) [\#6351](https://github.com/ClickHouse/ClickHouse/pull/6351) ([アルテム-ズイコフ](https://github.com/4ertus2)) -## クリックハウスリリース19.13 {#clickhouse-release-19-13} +## ClickHouseリリース19.13 {#clickhouse-release-19-13} -### クリックハウスリリース19.13.6.51,2019-10-02 {#clickhouse-release-19-13-6-51-2019-10-02} +### ClickHouseリリース19.13.6.51,2019-10-02 {#clickhouse-release-19-13-6-51-2019-10-02} #### バグ修正 {#bug-fix-9} - このリリースも含む全てのバグ修正から19.11.12.69. -### ClickHouseリリース19.13.5.44、2019-09-20 {#clickhouse-release-19-13-5-44-2019-09-20} +### ClickHouseリリース19.13.5.44,2019-09-20 {#clickhouse-release-19-13-5-44-2019-09-20} #### バグ修正 {#bug-fix-10} - このリリースには、19.14.6.12のすべてのバグ修正も含まれています。 -- 実行中のテーブルの一貫性のない状態を修正 `DROP` クエリーのための複製テーブルが飼育係アクセスすることはできません。 [\#6045](https://github.com/ClickHouse/ClickHouse/issues/6045) [\#6413](https://github.com/ClickHouse/ClickHouse/pull/6413) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- StorageMergeでのデータレースの修正 [\#6717](https://github.com/ClickHouse/ClickHouse/pull/6717) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- ソケットから無限のrecvにつながるクエリプロファイラで導入されたバグを修正。 [\#6386](https://github.com/ClickHouse/ClickHouse/pull/6386) ([alesapin](https://github.com/alesapin)) -- 実行中の過度のcpu使用率を修正 `JSONExtractRaw` ブール値に対する関数です。 [\#6208](https://github.com/ClickHouse/ClickHouse/pull/6208) ([Vitaly Baranov](https://github.com/vitlibar)) -- マテリアライズドビュ [\#6415](https://github.com/ClickHouse/ClickHouse/pull/6415) ([イワン](https://github.com/abyss7)) -- テーブル機能 `url` この脆弱性により、攻撃者が要求に任意のHTTPヘッダーを挿入することができました。 この問題は、 [Nikita Tikhomirov](https://github.com/NSTikhomirov). [\#6466](https://github.com/ClickHouse/ClickHouse/pull/6466) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 役に立たない修正 `AST` セットの索引のチェックイン。 [\#6510](https://github.com/ClickHouse/ClickHouse/issues/6510) [\#6651](https://github.com/ClickHouse/ClickHouse/pull/6651) ([Nikita Vasilev](https://github.com/nikvas0)) -- の固定解析 `AggregateFunction` クエリに埋め込まれた値。 [\#6575](https://github.com/ClickHouse/ClickHouse/issues/6575) [\#6773](https://github.com/ClickHouse/ClickHouse/pull/6773) ([Zhichang Yu](https://github.com/yuzhichang)) -- 固定間違った動作の `trim` 機能ファミリ。 [\#6647](https://github.com/ClickHouse/ClickHouse/pull/6647) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- 実行中にテーブルの一貫性のない状態を修正しました `DROP` クエリーのための複製テーブルが飼育係アクセスすることはできません。 [\#6045](https://github.com/ClickHouse/ClickHouse/issues/6045) [\#6413](https://github.com/ClickHouse/ClickHouse/pull/6413) ([ニキータ-ミハイロフ](https://github.com/nikitamikhaylov)) +- StorageMergeでのデータレースの修正 [\#6717](https://github.com/ClickHouse/ClickHouse/pull/6717) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- ソケットから無限のrecvにつながるクエリプロファイラのバグを修正しました。 [\#6386](https://github.com/ClickHouse/ClickHouse/pull/6386) ([アレサピン](https://github.com/alesapin)) +- 実行中に過剰なCPU使用率を修正 `JSONExtractRaw` ブール値に対する関数。 [\#6208](https://github.com/ClickHouse/ClickHouse/pull/6208) ([ヴィタリー-バラノフ](https://github.com/vitlibar)) +- 修正の回帰が進を実現します。 [\#6415](https://github.com/ClickHouse/ClickHouse/pull/6415) ([イワン](https://github.com/abyss7)) +- テーブル関数 `url` この脆弱性により、攻撃者は要求に任意のHTTPヘッダーを挿入することができました。 この問題は [ニキータ-チホミロフ](https://github.com/NSTikhomirov). [\#6466](https://github.com/ClickHouse/ClickHouse/pull/6466) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 役に立たない修正 `AST` Set indexをチェックインします。 [\#6510](https://github.com/ClickHouse/ClickHouse/issues/6510) [\#6651](https://github.com/ClickHouse/ClickHouse/pull/6651) ([ニキータ-ヴァシレフ](https://github.com/nikvas0)) +- の修正された解析 `AggregateFunction` クエリに埋め込まれた値。 [\#6575](https://github.com/ClickHouse/ClickHouse/issues/6575) [\#6773](https://github.com/ClickHouse/ClickHouse/pull/6773) ([Zhichang Yu](https://github.com/yuzhichang)) +- 修正された間違った動作 `trim` 機能ファミリー。 [\#6647](https://github.com/ClickHouse/ClickHouse/pull/6647) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) -### ClickHouseリリース19.13.4.32、2019-09-10 {#clickhouse-release-19-13-4-32-2019-09-10} +### ClickHouseリリース19.13.4.32,2019-09-10 {#clickhouse-release-19-13-4-32-2019-09-10} #### バグ修正 {#bug-fix-11} - このリリースには、19.11.9.52と19.11.10.54のすべてのバグセキュリティ修正も含まれています。 -- 固定データレースで `system.parts` テーブルと `ALTER` クエリ。 [\#6245](https://github.com/ClickHouse/ClickHouse/issues/6245) [\#6513](https://github.com/ClickHouse/ClickHouse/pull/6513) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- ストリームのヘッダーの不一致を修正したのは、sampleとprewhereで空の分散テーブルからの読み取りの場合でした。 [\#6167](https://github.com/ClickHouse/ClickHouse/issues/6167) ([Lixiang Qian](https://github.com/fancyqlx)) [\#6823](https://github.com/ClickHouse/ClickHouse/pull/6823) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 使用して固定クラッシュ `IN` タプルを含むサブクエリを含む句。 [\#6125](https://github.com/ClickHouse/ClickHouse/issues/6125) [\#6550](https://github.com/ClickHouse/ClickHouse/pull/6550) ([tavplubix](https://github.com/tavplubix)) -- 同じ列名のケースを修正 `GLOBAL JOIN ON` セクション。 [\#6181](https://github.com/ClickHouse/ClickHouse/pull/6181) ([Artem Zuikov](https://github.com/4ertus2)) -- 固定したときにクラッシュする場合が鋳造型 `Decimal` それをサポートしていません。 代わりに例外をスロー。 [\#6297](https://github.com/ClickHouse/ClickHouse/pull/6297) ([Artem Zuikov](https://github.com/4ertus2)) -- で固定クラッシュ `extractAll()` 機能。 [\#6644](https://github.com/ClickHouse/ClickHouse/pull/6644) ([Artem Zuikov](https://github.com/4ertus2)) -- 以下のためのクエリ変換 `MySQL`, `ODBC`, `JDBC` テーブル関数は現在、 `SELECT WHERE` 複数のクエリ `AND` 式。 [\#6381](https://github.com/ClickHouse/ClickHouse/issues/6381) [\#6676](https://github.com/ClickHouse/ClickHouse/pull/6676) ([ディマルブ2000](https://github.com/dimarub2000)) -- MySQL8の統合のための追加された以前の宣言チェック。 [\#6569](https://github.com/ClickHouse/ClickHouse/pull/6569) ([ラファエルdavid tinoco](https://github.com/rafaeldtinoco)) +- で固定データレース `system.parts` テーブルと `ALTER` クエリ。 [\#6245](https://github.com/ClickHouse/ClickHouse/issues/6245) [\#6513](https://github.com/ClickHouse/ClickHouse/pull/6513) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- Sampleとprewhereで空の分散テーブルから読み込む場合に、ストリーム内のヘッダーの不一致を修正しました。 [\#6167](https://github.com/ClickHouse/ClickHouse/issues/6167) ([Lixiang銭](https://github.com/fancyqlx)) [\#6823](https://github.com/ClickHouse/ClickHouse/pull/6823) ([ニコライ-コチェトフ](https://github.com/KochetovNicolai)) +- 使用時のクラッシュを修正 `IN` タプルを持つサブクエリを持つ句。 [\#6125](https://github.com/ClickHouse/ClickHouse/issues/6125) [\#6550](https://github.com/ClickHouse/ClickHouse/pull/6550) ([tavplubix](https://github.com/tavplubix)) +- 同じ列名の大文字と小文字を修正 `GLOBAL JOIN ON` セクション [\#6181](https://github.com/ClickHouse/ClickHouse/pull/6181) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- 固定したときにクラッシュする場合が鋳造型 `Decimal` それをサポートしていません。 代わりに例外を投げます。 [\#6297](https://github.com/ClickHouse/ClickHouse/pull/6297) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- で固定クラッシュ `extractAll()` 機能。 [\#6644](https://github.com/ClickHouse/ClickHouse/pull/6644) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- のクエリ変換 `MySQL`, `ODBC`, `JDBC` 表関数が正しく動作するようになった `SELECT WHERE` 複数のクエリ `AND` 式。 [\#6381](https://github.com/ClickHouse/ClickHouse/issues/6381) [\#6676](https://github.com/ClickHouse/ClickHouse/pull/6676) ([dimarub2000](https://github.com/dimarub2000)) +- MySQL8統合のための以前の宣言チェックを追加しました。 [\#6569](https://github.com/ClickHouse/ClickHouse/pull/6569) ([ラファエル-ダヴィド-ティノコ](https://github.com/rafaeldtinoco)) #### セキュリティ修正 {#security-fix-1} -- 修二の脆弱性がコーデックに減圧相（悪意のあるユーザーが可能で圧縮データにつながるバッファオーバーフローの減圧). [\#6670](https://github.com/ClickHouse/ClickHouse/pull/6670) ([Artem Zuikov](https://github.com/4ertus2)) +- 修二の脆弱性がコーデックに減圧相（悪意のあるユーザーが可能で圧縮データにつながるバッファオーバーフローの減圧). [\#6670](https://github.com/ClickHouse/ClickHouse/pull/6670) ([アルテム-ズイコフ](https://github.com/4ertus2)) -### ClickHouseリリース19.13.3.26、2019-08-22 {#clickhouse-release-19-13-3-26-2019-08-22} +### ClickHouse Release19.13.3.26,2019-08-22 {#clickhouse-release-19-13-3-26-2019-08-22} #### バグ修正 {#bug-fix-12} -- 修正 `ALTER TABLE ... UPDATE` とテーブルのクエリ `enable_mixed_granularity_parts=1`. [\#6543](https://github.com/ClickHouse/ClickHouse/pull/6543) ([alesapin](https://github.com/alesapin)) +- 修正 `ALTER TABLE ... UPDATE` テーブルのクエリ `enable_mixed_granularity_parts=1`. [\#6543](https://github.com/ClickHouse/ClickHouse/pull/6543) ([アレサピン](https://github.com/alesapin)) - タプルを持つサブクエリでin句を使用するときにnpeを修正しました。 [\#6125](https://github.com/ClickHouse/ClickHouse/issues/6125) [\#6550](https://github.com/ClickHouse/ClickHouse/pull/6550) ([tavplubix](https://github.com/tavplubix)) -- 固定問題の場合はュレプリカになり、生存していてデータ部分が撤去されることによります。 [\#6522](https://github.com/ClickHouse/ClickHouse/issues/6522) [\#6523](https://github.com/ClickHouse/ClickHouse/pull/6523) ([tavplubix](https://github.com/tavplubix)) -- CSVを解析する問題を修正しました [\#6426](https://github.com/ClickHouse/ClickHouse/issues/6426) [\#6559](https://github.com/ClickHouse/ClickHouse/pull/6559) ([tavplubix](https://github.com/tavplubix)) -- システム内の固定データレース.パーツテーブルと変更クエリ。 この修正 [\#6245](https://github.com/ClickHouse/ClickHouse/issues/6245). [\#6513](https://github.com/ClickHouse/ClickHouse/pull/6513) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- メモリの破損につながる可能性があり、突然変異で間違ったコードを修正. アドレスの読み取りによる固定segfault `0x14c0` それは同時に起こったかもしれない `DROP TABLE` と `SELECT` から `system.parts` または `system.parts_columns`. 突然変異クエリの準備の競合状態を修正しました。 によるデッドロックを修正 `OPTIMIZE` レプリケートされたテーブルと同時変更操作のような変更。 [\#6514](https://github.com/ClickHouse/ClickHouse/pull/6514) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 後に固定可能なデータ損失 `ALTER DELETE` 索引をスキップしてテーブルを照会します。 [\#6224](https://github.com/ClickHouse/ClickHouse/issues/6224) [\#6282](https://github.com/ClickHouse/ClickHouse/pull/6282) ([Nikita Vasilev](https://github.com/nikvas0)) +- 古いレプリカがアライブ状態になっても、DROP PARTITIONによって削除されたデータパーツが残っている可能性がある問題を修正しました。 [\#6522](https://github.com/ClickHouse/ClickHouse/issues/6522) [\#6523](https://github.com/ClickHouse/ClickHouse/pull/6523) ([tavplubix](https://github.com/tavplubix)) +- CSVの解析に関する問題を修正 [\#6426](https://github.com/ClickHouse/ClickHouse/issues/6426) [\#6559](https://github.com/ClickHouse/ClickHouse/pull/6559) ([tavplubix](https://github.com/tavplubix)) +- システム内の固定データレース。パーツテーブルとALTERクエリ。 この修正 [\#6245](https://github.com/ClickHouse/ClickHouse/issues/6245). [\#6513](https://github.com/ClickHouse/ClickHouse/pull/6513) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 固定の間違ったコードに変異のありメモリが破損す アドレスの読み取りで固定segfault `0x14c0` それは同時に起こった可能性があります `DROP TABLE` と `SELECT` から `system.parts` または `system.parts_columns`. 突然変異クエリの準備における競合状態を修正しました。 によるデッドロックを修正 `OPTIMIZE` 変更のようなレプリケートされたテーブルと同時変更操作の。 [\#6514](https://github.com/ClickHouse/ClickHouse/pull/6514) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 固定可能なデータ損失の後 `ALTER DELETE` クエリーテーブルとキース。 [\#6224](https://github.com/ClickHouse/ClickHouse/issues/6224) [\#6282](https://github.com/ClickHouse/ClickHouse/pull/6282) ([ニキータ-ヴァシレフ](https://github.com/nikvas0)) #### セキュリティ修正 {#security-fix-2} -- 攻撃者は、zookeeperへの書き込みアクセス権を持っており、clickhouseの実行ネットワークから利用できるカスタムサーバーを実行することができる場合,それはclickhouseのレ きものレプリカまでデータを取得すから悪意のあるレプリカで力clickhouse-サーバへの書き込みを任意のパスにファイルシステム. eldar zaitov、yandexの情報セキュリティチームによって発見された。 [\#6247](https://github.com/ClickHouse/ClickHouse/pull/6247) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- 攻撃者がZooKeeperへの書き込みアクセス権を持ち、ClickHouseが実行されるネットワークから利用可能なカスタムサーバーを実行できる場合、ClickHouseのレプリカとして機能す きものレプリカまでデータを取得すから悪意のあるレプリカで力clickhouse-サーバへの書き込みを任意のパスにファイルシステム. Yandexの情報セキュリティチームEldar Zaitovによって発見されました。 [\#6247](https://github.com/ClickHouse/ClickHouse/pull/6247) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) -### ClickHouseリリース19.13.2.19、2019-08-14 {#clickhouse-release-19-13-2-19-2019-08-14} +### ClickHouseリリース19.13.2.19,2019-08-14 {#clickhouse-release-19-13-2-19-2019-08-14} -#### 新しい機能 {#new-feature-5} +#### 新機能 {#new-feature-5} -- サンプリングプロファイラーに照会です。 [例えば](https://gist.github.com/alexey-milovidov/92758583dd41c24c360fdb8d6a4da194). [\#4247](https://github.com/ClickHouse/ClickHouse/issues/4247) ([laplab](https://github.com/laplab)) [\#6124](https://github.com/ClickHouse/ClickHouse/pull/6124) ([alexey-milovidov](https://github.com/alexey-milovidov)) [\#6250](https://github.com/ClickHouse/ClickHouse/pull/6250) [\#6283](https://github.com/ClickHouse/ClickHouse/pull/6283) [\#6386](https://github.com/ClickHouse/ClickHouse/pull/6386) -- 列のリストを指定できるようにする `COLUMNS('regexp')` より洗練された変種のように動作する表現 `*` アスタリスク [\#5951](https://github.com/ClickHouse/ClickHouse/pull/5951) ([mfridental](https://github.com/mfridental)), ([alexey-milovidov](https://github.com/alexey-milovidov)) -- `CREATE TABLE AS table_function()` 可能になりました [\#6057](https://github.com/ClickHouse/ClickHouse/pull/6057) ([ディマルブ2000](https://github.com/dimarub2000)) -- デフォルトでは、確率的勾配降下のためのadamオプティマイザが `stochasticLinearRegression()` と `stochasticLogisticRegression()` 集計機能を示すためのもので、良質なほとんど調整することがあります。 [\#6000](https://github.com/ClickHouse/ClickHouse/pull/6000) ([Quid37](https://github.com/Quid37)) -- Added functions for working with the сustom week number [\#5212](https://github.com/ClickHouse/ClickHouse/pull/5212) ([アンディヤング](https://github.com/andyyzh)) -- `RENAME` 問合せで出てきますが、すべての倉庫. [\#5953](https://github.com/ClickHouse/ClickHouse/pull/5953) ([イワン](https://github.com/abyss7)) -- 現在お客様の受信ログからサーバに要求レベルに設定 `send_logs_level` サーバー設定で指定されたログレベルにかかわらず。 [\#5964](https://github.com/ClickHouse/ClickHouse/pull/5964) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +- サンプリングプロファイラーに照会です。 [例](https://gist.github.com/alexey-milovidov/92758583dd41c24c360fdb8d6a4da194). [\#4247](https://github.com/ClickHouse/ClickHouse/issues/4247) ([laplab](https://github.com/laplab)) [\#6124](https://github.com/ClickHouse/ClickHouse/pull/6124) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) [\#6250](https://github.com/ClickHouse/ClickHouse/pull/6250) [\#6283](https://github.com/ClickHouse/ClickHouse/pull/6283) [\#6386](https://github.com/ClickHouse/ClickHouse/pull/6386) +- 列のリストを指定できるようにする `COLUMNS('regexp')` のより洗練された変種のように動作する式 `*` アスタリスク [\#5951](https://github.com/ClickHouse/ClickHouse/pull/5951) ([ムフリデンタル](https://github.com/mfridental)), ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- `CREATE TABLE AS table_function()` 今では可能です [\#6057](https://github.com/ClickHouse/ClickHouse/pull/6057) ([dimarub2000](https://github.com/dimarub2000)) +- 確率勾配降下のAdamオプティマイザは、デフォルトでは `stochasticLinearRegression()` と `stochasticLogisticRegression()` 集計機能を示すためのもので、良質なほとんど調整することがあります。 [\#6000](https://github.com/ClickHouse/ClickHouse/pull/6000) ([Quid37](https://github.com/Quid37)) +- Added functions for working with the сustom week number [\#5212](https://github.com/ClickHouse/ClickHouse/pull/5212) ([アンディ-ヤン](https://github.com/andyyzh)) +- `RENAME` クエリはすべてのストレージで動作します。 [\#5953](https://github.com/ClickHouse/ClickHouse/pull/5953) ([イワン](https://github.com/abyss7)) +- 現在お客様の受信ログからサーバに要求レベルに設定 `send_logs_level` サーバー設定で指定されたログレベルに関係なく。 [\#5964](https://github.com/ClickHouse/ClickHouse/pull/5964) ([ニキータ-ミハイロフ](https://github.com/nikitamikhaylov)) #### 下位互換性のない変更 {#backward-incompatible-change-4} -- を設定 `input_format_defaults_for_omitted_fields` デフォルトでは有効です。 分散テーブルの挿入では、この設定をクラスタで同じにする必要があります（更新をロールする前に設定する必要があります）。 省略されたフィールドの複雑な既定の式の計算を有効にします `JSONEachRow` と `CSV*` フォーマット。 この挙動があるが無視できる性能の差です。 [\#6043](https://github.com/ClickHouse/ClickHouse/pull/6043) ([Artem Zuikov](https://github.com/4ertus2)), [\#5625](https://github.com/ClickHouse/ClickHouse/pull/5625) ([azerbaijan.kgm](https://github.com/akuzm)) +- 設定 `input_format_defaults_for_omitted_fields` 既定では有効です。 挿入学の領域において研究が行われてテーブルがこの設定は同じクラスター(設定する必要がありますので前に転じた。 この計算の複雑なデフォルト表現のための省略分野 `JSONEachRow` と `CSV*` フォーマット。 この挙動があるが無視できる性能の差です。 [\#6043](https://github.com/ClickHouse/ClickHouse/pull/6043) ([アルテム-ズイコフ](https://github.com/4ertus2)), [\#5625](https://github.com/ClickHouse/ClickHouse/pull/5625) ([akuzm](https://github.com/akuzm)) -#### 実験の特徴 {#experimental-features} +#### 実験的な特徴 {#experimental-features} -- 新規クエリ処理パイプライン。 使用 `experimental_use_processors=1` それを有効にするオプション。 あなた自身の悩みのための使用。 [\#4914](https://github.com/ClickHouse/ClickHouse/pull/4914) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- 新規クエリ処理パイプライン。 使用 `experimental_use_processors=1` それを有効にするオプション。 あなた自身の悩みのための使用。 [\#4914](https://github.com/ClickHouse/ClickHouse/pull/4914) ([ニコライ-コチェトフ](https://github.com/KochetovNicolai)) #### バグ修正 {#bug-fix-13} -- Kafkaの統合は、このバージョンで修正されました。 -- 固定 `DoubleDelta` の符号化 `Int64` 大きいのため `DoubleDelta` 値、改善 `DoubleDelta` ランダムデータのエンコード `Int32`. [\#5998](https://github.com/ClickHouse/ClickHouse/pull/5998) ([Vasily Nemkov](https://github.com/Enmk)) -- の固定過大評価 `max_rows_to_read` 設定の場合 `merge_tree_uniform_read_distribution` は0に設定されます。 [\#6019](https://github.com/ClickHouse/ClickHouse/pull/6019) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- カフカの統合は、このバージョンで修正されました。 +- 固定 `DoubleDelta` のエンコード `Int64` 大きいのため `DoubleDelta` 値、改善 `DoubleDelta` ランダムデータのエンコード `Int32`. [\#5998](https://github.com/ClickHouse/ClickHouse/pull/5998) ([ヴァシーリー-ネムコフ](https://github.com/Enmk)) +- の固定過大評価 `max_rows_to_read` 設定の場合 `merge_tree_uniform_read_distribution` 0に設定されます。 [\#6019](https://github.com/ClickHouse/ClickHouse/pull/6019) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) #### 改善 {#improvement-4} -- 次の場合に例外をスローする `config.d` ファイルを持っていないので対応するルート要素としての設定ファイル [\#6123](https://github.com/ClickHouse/ClickHouse/pull/6123) ([ディマルブ2000](https://github.com/dimarub2000)) +- 例外をスローする場合 `config.d` fileには、設定ファイルとして対応するルート要素がありません [\#6123](https://github.com/ClickHouse/ClickHouse/pull/6123) ([dimarub2000](https://github.com/dimarub2000)) #### 性能向上 {#performance-improvement-3} -- 最適化 `count()`. 今では（可能な場合）最小の列を使用しています。 [\#6028](https://github.com/ClickHouse/ClickHouse/pull/6028) ([アモスの鳥](https://github.com/amosbird)) +- 最適化 `count()`. これで、最小の列が使用されます（可能であれば）。 [\#6028](https://github.com/ClickHouse/ClickHouse/pull/6028) ([アモス鳥](https://github.com/amosbird)) -#### ビルド/テスト/パッケージの改善 {#buildtestingpackaging-improvement-5} +#### 造り/テスト/包装の改善 {#buildtestingpackaging-improvement-5} -- パフォーマンステス [\#5899](https://github.com/ClickHouse/ClickHouse/pull/5899) ([azerbaijan.kgm](https://github.com/akuzm)) -- 外部でビルドを修正する `libcxx` [\#6010](https://github.com/ClickHouse/ClickHouse/pull/6010) ([イワン](https://github.com/abyss7)) -- 共有ビルドを修正する `rdkafka` ライブラリ [\#6101](https://github.com/ClickHouse/ClickHouse/pull/6101) ([イワン](https://github.com/abyss7)) +- パフォーマンステ [\#5899](https://github.com/ClickHouse/ClickHouse/pull/5899) ([akuzm](https://github.com/akuzm)) +- 外部でビルドを修正 `libcxx` [\#6010](https://github.com/ClickHouse/ClickHouse/pull/6010) ([イワン](https://github.com/abyss7)) +- 共有ビルドを修正 `rdkafka` ライブラリ [\#6101](https://github.com/ClickHouse/ClickHouse/pull/6101) ([イワン](https://github.com/abyss7)) ## ClickHouseリリース19.11 {#clickhouse-release-19-11} @@ -889,443 +889,443 @@ toc_title: '2019' #### バグ修正 {#bug-fix-14} -- 固定珍しいクラッシュ `ALTER MODIFY COLUMN` そして、マージ/変更された部分のいずれかが空（0行）のときに垂直マージ。 [\#6780](https://github.com/ClickHouse/ClickHouse/pull/6780) ([alesapin](https://github.com/alesapin)) +- で固定レアクラッシュ `ALTER MODIFY COLUMN` マージ/変更された部分のいずれかが空の場合（0行）に垂直マージします。 [\#6780](https://github.com/ClickHouse/ClickHouse/pull/6780) ([アレサピン](https://github.com/alesapin)) - の手動update `SIMDJSON`. これにより、偽のjson診断メッセージでstderrファイルが氾濫する可能性が修正されます。 [\#7548](https://github.com/ClickHouse/ClickHouse/pull/7548) ([Alexander Kazakov](https://github.com/Akazz)) -- とのバグを修正 `mrk` 突然変異のファイル拡張子 ([alesapin](https://github.com/alesapin)) +- とのバグを修正 `mrk` mutationsのファイル拡張子 ([アレサピン](https://github.com/alesapin)) -### ClickHouseリリース19.11.12.69、2019-10-02 {#clickhouse-release-19-11-12-69-2019-10-02} +### ClickHouse Release19.11.12.69,2019-10-02 {#clickhouse-release-19-11-12-69-2019-10-02} #### バグ修正 {#bug-fix-15} -- 固定性能の劣化指標分析複雑なテンキーの大きます。 この修正 [\#6924](https://github.com/ClickHouse/ClickHouse/issues/6924). [\#7075](https://github.com/ClickHouse/ClickHouse/pull/7075) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- を避ける希少sigsegvを送信するデータテーブル分散型エンジン (`Failed to send batch: file with index XXXXX is absent`). [\#7032](https://github.com/ClickHouse/ClickHouse/pull/7032) ([Azat Khuzhin](https://github.com/azat)) -- 修正 `Unknown identifier` 複数の結合を持つ。 この修正 [\#5254](https://github.com/ClickHouse/ClickHouse/issues/5254). [\#7022](https://github.com/ClickHouse/ClickHouse/pull/7022) ([Artem Zuikov](https://github.com/4ertus2)) +- 固定性能の劣化指標分析複雑なテンキーの大きます。 この修正 [\#6924](https://github.com/ClickHouse/ClickHouse/issues/6924). [\#7075](https://github.com/ClickHouse/ClickHouse/pull/7075) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 分散エンジンでテーブルにデータを送信する際に、まれなSIGSEGVを回避 (`Failed to send batch: file with index XXXXX is absent`). [\#7032](https://github.com/ClickHouse/ClickHouse/pull/7032) ([Azat Khuzhin](https://github.com/azat)) +- 修正 `Unknown identifier` 複数の結合を使用します。 この修正 [\#5254](https://github.com/ClickHouse/ClickHouse/issues/5254). [\#7022](https://github.com/ClickHouse/ClickHouse/pull/7022) ([アルテム-ズイコフ](https://github.com/4ertus2)) -### ClickHouseリリース19.11.11.57,2019-09-13 {#clickhouse-release-19-11-11-57-2019-09-13} +### ClickHouse Release19.11.11.57,2019-09-13 {#clickhouse-release-19-11-11-57-2019-09-13} -- カフカ空のトピックから選択する際にsegfaultsを引き起こす論理的なエラーを修正。 [\#6902](https://github.com/ClickHouse/ClickHouse/issues/6902) [\#6909](https://github.com/ClickHouse/ClickHouse/pull/6909) ([イワン](https://github.com/abyss7)) -- 関数の修正 `АrrayEnumerateUniqRanked` paramsに空の配列があります。 [\#6928](https://github.com/ClickHouse/ClickHouse/pull/6928) ([proller](https://github.com/proller)) +- カフカ空のトピックから選択するとsegfaultsの原因となる論理エラーを修正しました。 [\#6902](https://github.com/ClickHouse/ClickHouse/issues/6902) [\#6909](https://github.com/ClickHouse/ClickHouse/pull/6909) ([イワン](https://github.com/abyss7)) +- 機能のための苦境 `АrrayEnumerateUniqRanked` paramsに空の配列があります。 [\#6928](https://github.com/ClickHouse/ClickHouse/pull/6928) ([プロラー](https://github.com/proller)) -### ClickHouseリリース19.11.10.54,2019-09-10 {#clickhouse-release-19-11-10-54-2019-09-10} +### ClickHouse Release19.11.10.54,2019-09-10 {#clickhouse-release-19-11-10-54-2019-09-10} #### バグ修正 {#bug-fix-16} -- Kafkaメッセージのオフセットを手動で保存すると、すべてのパーティションに対して一度にコミットできます。 潜在的な重複を修正 “one consumer - many partitions” シナリオだ [\#6872](https://github.com/ClickHouse/ClickHouse/pull/6872) ([イワン](https://github.com/abyss7)) +- い店舗-オフセットカフカメッセージを手動できることを確約できることすべてを一度にすべての仕切り. 潜在的な重複を修正 “one consumer - many partitions” シナリオ [\#6872](https://github.com/ClickHouse/ClickHouse/pull/6872) ([イワン](https://github.com/abyss7)) -### ClickHouseリリース19.11.9.52,2019-09-6 {#clickhouse-release-19-11-9-52-2019-09-6} +### ClickHouse Release19.11.9.52,2019-09-6 {#clickhouse-release-19-11-9-52-2019-09-6} -- キャッ [\#6737](https://github.com/ClickHouse/ClickHouse/pull/6737) ([Vitaly Baranov](https://github.com/vitlibar)) -- 機能のバグを修正 `arrayEnumerateUniqRanked`. [\#6779](https://github.com/ClickHouse/ClickHouse/pull/6779) ([proller](https://github.com/proller)) -- 修正 `JSONExtract` 関数を抽出しながら `Tuple` JSONから。 [\#6718](https://github.com/ClickHouse/ClickHouse/pull/6718) ([Vitaly Baranov](https://github.com/vitlibar)) -- 後に固定可能なデータ損失 `ALTER DELETE` 索引をスキップしてテーブルを照会します。 [\#6224](https://github.com/ClickHouse/ClickHouse/issues/6224) [\#6282](https://github.com/ClickHouse/ClickHouse/pull/6282) ([Nikita Vasilev](https://github.com/nikvas0)) -- 固定性能テスト。 [\#6392](https://github.com/ClickHouse/ClickHouse/pull/6392) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 寄木細工：ブール値列の読み取りを修正。 [\#6579](https://github.com/ClickHouse/ClickHouse/pull/6579) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 固定間違った動作の `nullIf` 定数引数の関数。 [\#6518](https://github.com/ClickHouse/ClickHouse/pull/6518) ([ギヨームタッセリー](https://github.com/YiuRULE)) [\#6580](https://github.com/ClickHouse/ClickHouse/pull/6580) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 通常のサーバーの再起動時にkafkaメッセージの重複の問題を修正。 [\#6597](https://github.com/ClickHouse/ClickHouse/pull/6597) ([イワン](https://github.com/abyss7)) -- ときに長い問題を修正しました `ALTER UPDATE` または `ALTER DELETE` 通常のマージが実行されない場合があります。 利用可能な十分な空きスレッドがない場合、突然変異の実行を防ぎます。 [\#6502](https://github.com/ClickHouse/ClickHouse/issues/6502) [\#6617](https://github.com/ClickHouse/ClickHouse/pull/6617) ([tavplubix](https://github.com/tavplubix)) -- 処理によるエラーの修正 “timezone” サーバー構成ファイルで。 [\#6709](https://github.com/ClickHouse/ClickHouse/pull/6709) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- カフカのテストを修正。 [\#6805](https://github.com/ClickHouse/ClickHouse/pull/6805) ([イワン](https://github.com/abyss7)) +- 改善についてはキャッシュを生成する事ができます。 [\#6737](https://github.com/ClickHouse/ClickHouse/pull/6737) ([ヴィタリー-バラノフ](https://github.com/vitlibar)) +- 関数のバグを修正しました `arrayEnumerateUniqRanked`. [\#6779](https://github.com/ClickHouse/ClickHouse/pull/6779) ([プロラー](https://github.com/proller)) +- 修正 `JSONExtract` を抽出しながら `Tuple` JSONから。 [\#6718](https://github.com/ClickHouse/ClickHouse/pull/6718) ([ヴィタリー-バラノフ](https://github.com/vitlibar)) +- 固定可能なデータ損失の後 `ALTER DELETE` クエリーテーブルとキース。 [\#6224](https://github.com/ClickHouse/ClickHouse/issues/6224) [\#6282](https://github.com/ClickHouse/ClickHouse/pull/6282) ([ニキータ-ヴァシレフ](https://github.com/nikvas0)) +- 固定性能テスト。 [\#6392](https://github.com/ClickHouse/ClickHouse/pull/6392) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 寄木細工：ブール値の列を読んで修正します。 [\#6579](https://github.com/ClickHouse/ClickHouse/pull/6579) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 修正された間違った動作 `nullIf` 定数引数の関数。 [\#6518](https://github.com/ClickHouse/ClickHouse/pull/6518) ([ギヨーム-タッセリー](https://github.com/YiuRULE)) [\#6580](https://github.com/ClickHouse/ClickHouse/pull/6580) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 固定カフカメッセージの複製問題が通常のサーバを再起動します。 [\#6597](https://github.com/ClickHouse/ClickHouse/pull/6597) ([イワン](https://github.com/abyss7)) +- 長いときに問題を修正しました `ALTER UPDATE` または `ALTER DELETE` 通常のマージを実行できない場合があります。 利用可能な十分な空きスレッドがない場合、突然変異が実行されないようにします。 [\#6502](https://github.com/ClickHouse/ClickHouse/issues/6502) [\#6617](https://github.com/ClickHouse/ClickHouse/pull/6617) ([tavplubix](https://github.com/tavplubix)) +- 処理中のエラーを修正 “timezone” サーバー設定ファイル内。 [\#6709](https://github.com/ClickHouse/ClickHouse/pull/6709) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- カフカのテストを修正します。 [\#6805](https://github.com/ClickHouse/ClickHouse/pull/6805) ([イワン](https://github.com/abyss7)) #### セキュリティ修正 {#security-fix-3} -- 攻撃者がzookeeperへの書き込みアクセス権を持ち、clickhouseが実行されるネットワークから利用可能なカスタムサーバーを実行できる場合、それはclickhouseレプリカとして きものレプリカまでデータを取得すから悪意のあるレプリカで力clickhouse-サーバへの書き込みを任意のパスにファイルシステム. eldar zaitov、yandexの情報セキュリティチームによって発見された。 [\#6247](https://github.com/ClickHouse/ClickHouse/pull/6247) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- 攻撃者がZooKeeperへの書き込みアクセス権を持ち、ClickHouseが実行されているネットワークから利用可能なカスタムサーバーを実行できる場合、ClickHouseのレプリカとして機能 きものレプリカまでデータを取得すから悪意のあるレプリカで力clickhouse-サーバへの書き込みを任意のパスにファイルシステム. Yandexの情報セキュリティチームEldar Zaitovによって発見されました。 [\#6247](https://github.com/ClickHouse/ClickHouse/pull/6247) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) -### ClickHouseリリース19.11.8.46,2019-08-22 {#clickhouse-release-19-11-8-46-2019-08-22} +### ClickHouse Release19.11.8.46,2019-08-22 {#clickhouse-release-19-11-8-46-2019-08-22} #### バグ修正 {#bug-fix-17} -- 修正 `ALTER TABLE ... UPDATE` とテーブルのクエリ `enable_mixed_granularity_parts=1`. [\#6543](https://github.com/ClickHouse/ClickHouse/pull/6543) ([alesapin](https://github.com/alesapin)) +- 修正 `ALTER TABLE ... UPDATE` テーブルのクエリ `enable_mixed_granularity_parts=1`. [\#6543](https://github.com/ClickHouse/ClickHouse/pull/6543) ([アレサピン](https://github.com/alesapin)) - タプルを持つサブクエリでin句を使用するときにnpeを修正しました。 [\#6125](https://github.com/ClickHouse/ClickHouse/issues/6125) [\#6550](https://github.com/ClickHouse/ClickHouse/pull/6550) ([tavplubix](https://github.com/tavplubix)) -- 固定問題の場合はュレプリカになり、生存していてデータ部分が撤去されることによります。 [\#6522](https://github.com/ClickHouse/ClickHouse/issues/6522) [\#6523](https://github.com/ClickHouse/ClickHouse/pull/6523) ([tavplubix](https://github.com/tavplubix)) -- CSVを解析する問題を修正しました [\#6426](https://github.com/ClickHouse/ClickHouse/issues/6426) [\#6559](https://github.com/ClickHouse/ClickHouse/pull/6559) ([tavplubix](https://github.com/tavplubix)) -- システム内の固定データレース.パーツテーブルと変更クエリ。 この修正 [\#6245](https://github.com/ClickHouse/ClickHouse/issues/6245). [\#6513](https://github.com/ClickHouse/ClickHouse/pull/6513) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- メモリの破損につながる可能性があり、突然変異で間違ったコードを修正. アドレスの読み取りによる固定segfault `0x14c0` それは同時に起こったかもしれない `DROP TABLE` と `SELECT` から `system.parts` または `system.parts_columns`. 突然変異クエリの準備の競合状態を修正しました。 によるデッドロックを修正 `OPTIMIZE` レプリケートされたテーブルと同時変更操作のような変更。 [\#6514](https://github.com/ClickHouse/ClickHouse/pull/6514) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- 古いレプリカがアライブ状態になっても、DROP PARTITIONによって削除されたデータパーツが残っている可能性がある問題を修正しました。 [\#6522](https://github.com/ClickHouse/ClickHouse/issues/6522) [\#6523](https://github.com/ClickHouse/ClickHouse/pull/6523) ([tavplubix](https://github.com/tavplubix)) +- CSVの解析に関する問題を修正 [\#6426](https://github.com/ClickHouse/ClickHouse/issues/6426) [\#6559](https://github.com/ClickHouse/ClickHouse/pull/6559) ([tavplubix](https://github.com/tavplubix)) +- システム内の固定データレース。パーツテーブルとALTERクエリ。 この修正 [\#6245](https://github.com/ClickHouse/ClickHouse/issues/6245). [\#6513](https://github.com/ClickHouse/ClickHouse/pull/6513) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 固定の間違ったコードに変異のありメモリが破損す アドレスの読み取りで固定segfault `0x14c0` それは同時に起こった可能性があります `DROP TABLE` と `SELECT` から `system.parts` または `system.parts_columns`. 突然変異クエリの準備における競合状態を修正しました。 によるデッドロックを修正 `OPTIMIZE` 変更のようなレプリケートされたテーブルと同時変更操作の。 [\#6514](https://github.com/ClickHouse/ClickHouse/pull/6514) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) -### ClickHouseリリース19.11.7.40,2019-08-14 {#clickhouse-release-19-11-7-40-2019-08-14} +### ClickHouse Release19.11.7.40,2019-08-14 {#clickhouse-release-19-11-7-40-2019-08-14} #### バグ修正 {#bug-fix-18} -- Kafkaの統合は、このバージョンで修正されました。 -- 使用しているときにsegfaultを修正 `arrayReduce` 定数の引数の場合。 [\#6326](https://github.com/ClickHouse/ClickHouse/pull/6326) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 固定 `toFloat()` 単調性 [\#6374](https://github.com/ClickHouse/ClickHouse/pull/6374) ([ディマルブ2000](https://github.com/dimarub2000)) -- 有効にしてsegfaultを修正 `optimize_skip_unused_shards` シャーディングキーがない [\#6384](https://github.com/ClickHouse/ClickHouse/pull/6384) ([CurtizJ](https://github.com/CurtizJ)) -- 固定ロジックの `arrayEnumerateUniqRanked` 機能。 [\#6423](https://github.com/ClickHouse/ClickHouse/pull/6423) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- MySQLハンドラから余分な冗長ロギングを削除しました。 [\#6389](https://github.com/ClickHouse/ClickHouse/pull/6389) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 間違った動作と可能なsegfaultsを修正 `topK` と `topKWeighted` 集計関数。 [\#6404](https://github.com/ClickHouse/ClickHouse/pull/6404) ([CurtizJ](https://github.com/CurtizJ)) -- 仮想列を公開しないでください `system.columns` テーブル。 これは、下位互換性のために必要です。 [\#6406](https://github.com/ClickHouse/ClickHouse/pull/6406) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 複雑なキーキャッシュ辞書の文字列フィールドのメモリ割り当てのバグを修正。 [\#6447](https://github.com/ClickHouse/ClickHouse/pull/6447) ([alesapin](https://github.com/alesapin)) -- Bug Fixとを可能に適応粒度の作成時に新たなレプリカのために `Replicated*MergeTree` テーブル。 [\#6452](https://github.com/ClickHouse/ClickHouse/pull/6452) ([alesapin](https://github.com/alesapin)) +- カフカの統合は、このバージョンで修正されました。 +- 使用するときにsegfaultを修正 `arrayReduce` 定数引数の場合。 [\#6326](https://github.com/ClickHouse/ClickHouse/pull/6326) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 固定 `toFloat()` 単調性。 [\#6374](https://github.com/ClickHouse/ClickHouse/pull/6374) ([dimarub2000](https://github.com/dimarub2000)) +- 有効でsegfaultを修正 `optimize_skip_unused_shards` シャーディングキーがない [\#6384](https://github.com/ClickHouse/ClickHouse/pull/6384) ([クルティジ](https://github.com/CurtizJ)) +- の固定ロジック `arrayEnumerateUniqRanked` 機能。 [\#6423](https://github.com/ClickHouse/ClickHouse/pull/6423) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- MySQLハンドラから余分な冗長ロギングを削除しました。 [\#6389](https://github.com/ClickHouse/ClickHouse/pull/6389) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 間違った動作と可能なsegfaultsを修正 `topK` と `topKWeighted` 集計関数。 [\#6404](https://github.com/ClickHouse/ClickHouse/pull/6404) ([クルティジ](https://github.com/CurtizJ)) +- 仮想列を公開しない `system.columns` テーブル。 これは下位互換性のために必要です。 [\#6406](https://github.com/ClickHouse/ClickHouse/pull/6406) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 修正のバグとメモリの割り当てのための文字列の分野での複合キーキャッシュを辞書で調べました。 [\#6447](https://github.com/ClickHouse/ClickHouse/pull/6447) ([アレサピン](https://github.com/alesapin)) +- 新しいレプリカを作成するときに適応粒度を有効にするバグを修正 `Replicated*MergeTree` テーブル。 [\#6452](https://github.com/ClickHouse/ClickHouse/pull/6452) ([アレサピン](https://github.com/alesapin)) - 修正の無限ループ読み込み時にカフカメッセージ [\#6354](https://github.com/ClickHouse/ClickHouse/pull/6354) ([abyss7](https://github.com/abyss7)) -- 固定の可能性に作製したクエリがサーバのクラッシュによるスタックオーバーフローアプリケーションのパーサの可能性スタックオーバーフロー `Merge` と `Distributed` テーブル [\#6433](https://github.com/ClickHouse/ClickHouse/pull/6433) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 固定gorillaエンコードエラーの小型dnaの塩基配列を決定した。 [\#6444](https://github.com/ClickHouse/ClickHouse/pull/6444) ([Enmk](https://github.com/Enmk)) +- Sqlパーサーでのスタックオーバーフローによるサーバークラッシュとスタックオーバーフロ `Merge` と `Distributed` テーブル [\#6433](https://github.com/ClickHouse/ClickHouse/pull/6433) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 固定Gorillaエンコードエラーの小型dnaの塩基配列を決定した。 [\#6444](https://github.com/ClickHouse/ClickHouse/pull/6444) ([Enmk](https://github.com/Enmk)) #### 改善 {#improvement-5} -- ユーザーの上書きを許可する `poll_interval` と `idle_connection_timeout` 接続時の設定。 [\#6230](https://github.com/ClickHouse/ClickHouse/pull/6230) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- ユーザーに上書きを許可する `poll_interval` と `idle_connection_timeout` 接続時の設定。 [\#6230](https://github.com/ClickHouse/ClickHouse/pull/6230) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) -### ClickHouseリリース19.11.5.28、2019-08-05 {#clickhouse-release-19-11-5-28-2019-08-05} +### ClickHouse Release19.11.5.28,2019-08-05 {#clickhouse-release-19-11-5-28-2019-08-05} #### バグ修正 {#bug-fix-19} -- 固定の可能性を掛けクエリの場合はサーバが過負荷状態で運転されています。 [\#6301](https://github.com/ClickHouse/ClickHouse/pull/6301) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 固定fpeにyandexconsistenthashます。 この修正 [\#6304](https://github.com/ClickHouse/ClickHouse/issues/6304). [\#6126](https://github.com/ClickHouse/ClickHouse/pull/6126) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- の変換のバグを修正しました `LowCardinality` タイプ `AggregateFunctionFactory`. この修正 [\#6257](https://github.com/ClickHouse/ClickHouse/issues/6257). [\#6281](https://github.com/ClickHouse/ClickHouse/pull/6281) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 修正の解析 `bool` からの設定 `true` と `false` 構成ファイル内の文字列。 [\#6278](https://github.com/ClickHouse/ClickHouse/pull/6278) ([alesapin](https://github.com/alesapin)) -- クエリの互換性のないストリ `Distributed` テーブルオーバ `MergeTree` テーブルの一部 `WHERE` に移動します `PREWHERE`. [\#6236](https://github.com/ClickHouse/ClickHouse/pull/6236) ([alesapin](https://github.com/alesapin)) -- 固定オーバーフローの整数部署名-タイプを符号なしタイプです。 この修正 [\#6214](https://github.com/ClickHouse/ClickHouse/issues/6214). [\#6233](https://github.com/ClickHouse/ClickHouse/pull/6233) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- 固定の可能性を掛けクエリの場合はサーバが過負荷状態で運転されています。 [\#6301](https://github.com/ClickHouse/ClickHouse/pull/6301) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- YandexConsistentHash関数のFPEを修正しました。 この修正 [\#6304](https://github.com/ClickHouse/ClickHouse/issues/6304). [\#6126](https://github.com/ClickHouse/ClickHouse/pull/6126) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- の変換のバグを修正しました `LowCardinality` タイプ `AggregateFunctionFactory`. この修正 [\#6257](https://github.com/ClickHouse/ClickHouse/issues/6257). [\#6281](https://github.com/ClickHouse/ClickHouse/pull/6281) ([ニコライ-コチェトフ](https://github.com/KochetovNicolai)) +- の解析を修正 `bool` からの設定 `true` と `false` 設定ファイル内の文字列。 [\#6278](https://github.com/ClickHouse/ClickHouse/pull/6278) ([アレサピン](https://github.com/alesapin)) +- 固定珍しいバグと互換性のないストリームヘッダをクエリー `Distributed` テーブル `MergeTree` テーブルの一部 `WHERE` 移動先 `PREWHERE`. [\#6236](https://github.com/ClickHouse/ClickHouse/pull/6236) ([アレサピン](https://github.com/alesapin)) +- 符号付き型から符号なし型への整数除算でのオーバーフローを修正。 この修正 [\#6214](https://github.com/ClickHouse/ClickHouse/issues/6214). [\#6233](https://github.com/ClickHouse/ClickHouse/pull/6233) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) #### 下位互換性のない変更 {#backward-incompatible-change-5} - `Kafka` まだ壊れてる -### ClickHouseリリース19.11.4.24,2019-08-01 {#clickhouse-release-19-11-4-24-2019-08-01} +### ClickHouse Release19.11.4.24,2019-08-01 {#clickhouse-release-19-11-4-24-2019-08-01} #### バグ修正 {#bug-fix-20} -- のを修正した。筆二次指標マーク適応型粒度. [\#6126](https://github.com/ClickHouse/ClickHouse/pull/6126) ([alesapin](https://github.com/alesapin)) -- 修正 `WITH ROLLUP` と `WITH CUBE` の修飾子 `GROUP BY` 二レベルの集計。 [\#6225](https://github.com/ClickHouse/ClickHouse/pull/6225) ([アントン-ポポフ](https://github.com/CurtizJ)) -- 固定こつ `JSONExtractRaw` 機能。 固定 [\#6195](https://github.com/ClickHouse/ClickHouse/issues/6195) [\#6198](https://github.com/ClickHouse/ClickHouse/pull/6198) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- ExternalLoader::reloadOutdated()のセグフォルトを修正しました。 [\#6082](https://github.com/ClickHouse/ClickHouse/pull/6082) ([Vitaly Baranov](https://github.com/vitlibar)) -- 固定の場合はサーバが切れることがあり聞くソケットがセットのリスクマネジメントの継続け残ります。 ﾂつｨﾂ姪“ﾂつ”ﾂ債ﾂづｭﾂつｹﾂづ債つｱﾂつｿﾂづｧ サーバーがエラーを返す場合があります `bad_function_call` 残りのクエリの場合。 [\#6231](https://github.com/ClickHouse/ClickHouse/pull/6231) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- ODBC、MySQL、ClickHouseとHTTP経由で外部辞書の初期ロードのための更新フィールド上の固定役に立たないと間違った条件。 この修正 [\#6069](https://github.com/ClickHouse/ClickHouse/issues/6069) [\#6083](https://github.com/ClickHouse/ClickHouse/pull/6083) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- のキャストで修正された無関係な例外 `LowCardinality(Nullable)` to not-Nullable column in case if it doesn’t contain Nulls (e.g. in query like `SELECT CAST(CAST('Hello' AS LowCardinality(Nullable(String))) AS String)`. [\#6094](https://github.com/ClickHouse/ClickHouse/issues/6094) [\#6119](https://github.com/ClickHouse/ClickHouse/pull/6119) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 非決定性の結果を修正する “uniq” 極端なまれなケースでの集計関数。 バグはすべてのClickHouseバージョンに存在していました。 [\#6058](https://github.com/ClickHouse/ClickHouse/pull/6058) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Segfault私たちは少し高すぎる設定CIDRに機能 `IPv6CIDRToRange`. [\#6068](https://github.com/ClickHouse/ClickHouse/pull/6068) ([ギヨームタッセリー](https://github.com/YiuRULE)) -- 固定小さなメモリリークがサーバに捨てる多くの例外から多くの異なるコンテキストを共有します。 [\#6144](https://github.com/ClickHouse/ClickHouse/pull/6144) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- 適応粒度で二次インデックスマークを書くとバグを修正しました。 [\#6126](https://github.com/ClickHouse/ClickHouse/pull/6126) ([アレサピン](https://github.com/alesapin)) +- 修正 `WITH ROLLUP` と `WITH CUBE` の修飾子 `GROUP BY` 二レベルの集計を使って。 [\#6225](https://github.com/ClickHouse/ClickHouse/pull/6225) ([アントン-ポポフ](https://github.com/CurtizJ)) +- 固定ハングイン `JSONExtractRaw` 機能。 固定 [\#6195](https://github.com/ClickHouse/ClickHouse/issues/6195) [\#6198](https://github.com/ClickHouse/ClickHouse/pull/6198) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- ExternalLoader::reloadOutdated()でsegfaultを修正しました。 [\#6082](https://github.com/ClickHouse/ClickHouse/pull/6082) ([ヴィタリー-バラノフ](https://github.com/vitlibar)) +- 固定の場合はサーバが切れることがあり聞くソケットがセットのリスクマネジメントの継続け残ります。 ﾂつｻﾂつ､ﾂつ"ﾂつ､ﾂつｱﾂづﾂづﾂつｷﾂ。 サーバーがエラーを返すことがあります `bad_function_call` 残りのクエリの場合。 [\#6231](https://github.com/ClickHouse/ClickHouse/pull/6231) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- ODBC、MySQL、ClickHouseおよびHTTP経由で外部辞書の初期読み込みのための更新フィールドに役に立たないと間違った条件を修正しました。 この修正 [\#6069](https://github.com/ClickHouse/ClickHouse/issues/6069) [\#6083](https://github.com/ClickHouse/ClickHouse/pull/6083) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- キャストの無関係な例外を修正しました `LowCardinality(Nullable)` to not-Nullable column in case if it doesn't contain Nulls (e.g. in query like `SELECT CAST(CAST('Hello' AS LowCardinality(Nullable(String))) AS String)`. [\#6094](https://github.com/ClickHouse/ClickHouse/issues/6094) [\#6119](https://github.com/ClickHouse/ClickHouse/pull/6119) ([ニコライ-コチェトフ](https://github.com/KochetovNicolai)) +- の非決定的な結果を修正 “uniq” 極端なまれなケースでの集計関数。 このバグはすべてのClickHouseバージョンに存在しました。 [\#6058](https://github.com/ClickHouse/ClickHouse/pull/6058) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 私たちは関数に少し高すぎるcidrを設定するとSegfault `IPv6CIDRToRange`. [\#6068](https://github.com/ClickHouse/ClickHouse/pull/6068) ([ギヨーム-タッセリー](https://github.com/YiuRULE)) +- 固定小さなメモリリークがサーバに捨てる多くの例外から多くの異なるコンテキストを共有します。 [\#6144](https://github.com/ClickHouse/ClickHouse/pull/6144) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) - 固定の状態で消費者も一時停止前の契約とな再開します。 [\#6075](https://github.com/ClickHouse/ClickHouse/pull/6075) ([イワン](https://github.com/abyss7) 注このカフカを砕このバージョン。 -- エラーで完了した前の読み取り操作からkafkaデータバッファーをクリアします [\#6026](https://github.com/ClickHouse/ClickHouse/pull/6026) ([ニコライ](https://github.com/bopohaa) 注このカフカを砕このバージョン。 -- それ以来 `StorageMergeTree::background_task_handle` で初期化される。 `startup()` その `MergeTreeBlockOutputStream::write()` 初期化の前に使用しようとするかもしれません。 すぐチェックインの場合は初期化されます。 [\#6080](https://github.com/ClickHouse/ClickHouse/pull/6080) ([イワン](https://github.com/abyss7)) +- エラーで完了した前回の読み取り操作からKafkaデータバッファをクリアする [\#6026](https://github.com/ClickHouse/ClickHouse/pull/6026) ([ニコライ](https://github.com/bopohaa) 注このカフカを砕このバージョン。 +- 以来 `StorageMergeTree::background_task_handle` で初期化される。 `startup()` その `MergeTreeBlockOutputStream::write()` 初期化の前に使用しようとする可能性があります。 すぐチェックインの場合は初期化されます。 [\#6080](https://github.com/ClickHouse/ClickHouse/pull/6080) ([イワン](https://github.com/abyss7)) -#### ビルド/テスト/パッケージの改善 {#buildtestingpackaging-improvement-6} +#### 造り/テスト/包装の改善 {#buildtestingpackaging-improvement-6} -- 追加された公式 `rpm` パッケージ。 [\#5740](https://github.com/ClickHouse/ClickHouse/pull/5740) ([proller](https://github.com/proller)) ([alesapin](https://github.com/alesapin)) -- ビルドする機能を追加する `.rpm` と `.tgz` パッケージと `packager` スクリプト [\#5769](https://github.com/ClickHouse/ClickHouse/pull/5769) ([alesapin](https://github.com/alesapin)) -- 以下のための修正 “Arcadia” ビルドシステム。 [\#6223](https://github.com/ClickHouse/ClickHouse/pull/6223) ([proller](https://github.com/proller)) +- 追加された公式 `rpm` パッケージ。 [\#5740](https://github.com/ClickHouse/ClickHouse/pull/5740) ([プロラー](https://github.com/proller)) ([アレサピン](https://github.com/alesapin)) +- ビルド機能の追加 `.rpm` と `.tgz` パッケージ `packager` スクリプト [\#5769](https://github.com/ClickHouse/ClickHouse/pull/5769) ([アレサピン](https://github.com/alesapin)) +- の修正 “Arcadia” ビルドシステム。 [\#6223](https://github.com/ClickHouse/ClickHouse/pull/6223) ([プロラー](https://github.com/proller)) #### 下位互換性のない変更 {#backward-incompatible-change-6} - `Kafka` このバージョンでは壊れている。 -### ClickHouseリリース19.11.3.11,2019-07-18 {#clickhouse-release-19-11-3-11-2019-07-18} +### ClickHouse Release19.11.3.11,2019-07-18 {#clickhouse-release-19-11-3-11-2019-07-18} -#### 新しい機能 {#new-feature-6} +#### 新機能 {#new-feature-6} -- 準備文のサポートが追加されました。 [\#5331](https://github.com/ClickHouse/ClickHouse/pull/5331/) ([Alexander](https://github.com/sanych73)) [\#5630](https://github.com/ClickHouse/ClickHouse/pull/5630) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- `DoubleDelta` と `Gorilla` 列コーデック [\#5600](https://github.com/ClickHouse/ClickHouse/pull/5600) ([Vasily Nemkov](https://github.com/Enmk)) -- 追加 `os_thread_priority` を制御することを可能にする設定 “nice” OSが動的スケジューリング優先順位を調整するために使用するクエリ処理スレッドの値。 それは必要です `CAP_SYS_NICE` 動作する機能。 これは [\#5858](https://github.com/ClickHouse/ClickHouse/issues/5858) [\#5909](https://github.com/ClickHouse/ClickHouse/pull/5909) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- るためのサポートを追加しました用意します。 [\#5331](https://github.com/ClickHouse/ClickHouse/pull/5331/) ([Alexander](https://github.com/sanych73)) [\#5630](https://github.com/ClickHouse/ClickHouse/pull/5630) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- `DoubleDelta` と `Gorilla` 列コーデック [\#5600](https://github.com/ClickHouse/ClickHouse/pull/5600) ([ヴァシーリー-ネムコフ](https://github.com/Enmk)) +- 追加 `os_thread_priority` 制御することを割り当てる設定 “nice” 動的スケジューリング優先度を調整するためにOSが使用するクエリ処理スレッドの値。 それは必要です `CAP_SYS_NICE` 働く機能。 これは [\#5858](https://github.com/ClickHouse/ClickHouse/issues/5858) [\#5909](https://github.com/ClickHouse/ClickHouse/pull/5909) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) - 実装 `_topic`, `_offset`, `_key` カフカエンジンの列 [\#5382](https://github.com/ClickHouse/ClickHouse/pull/5382) ([イワン](https://github.com/abyss7) 注このカフカを砕このバージョン。 -- 集計関数コンビネータを追加 `-Resample` [\#5590](https://github.com/ClickHouse/ClickHouse/pull/5590) ([hcz](https://github.com/hczhcz)) -- 集計関数 `groupArrayMovingSum(win_size)(x)` と `groupArrayMovingAvg(win_size)(x)`、ウィンドウサイズの制限の有無にかかわらず、移動の合計/平均を計算します。 [\#5595](https://github.com/ClickHouse/ClickHouse/pull/5595) ([inv2004](https://github.com/inv2004)) +- 集計関数コンビネータの追加 `-Resample` [\#5590](https://github.com/ClickHouse/ClickHouse/pull/5590) ([hcz](https://github.com/hczhcz)) +- 集計関数 `groupArrayMovingSum(win_size)(x)` と `groupArrayMovingAvg(win_size)(x)` ウィンドウサイズの制限の有無にかかわらず、移動合計/平均を計算します。 [\#5595](https://github.com/ClickHouse/ClickHouse/pull/5595) ([inv2004](https://github.com/inv2004)) - Synonimを追加 `arrayFlatten` \<-\> `flatten` [\#5764](https://github.com/ClickHouse/ClickHouse/pull/5764) ([hcz](https://github.com/hczhcz)) -- インターゲートh3機能 `geoToH3` ユーバーから. [\#4724](https://github.com/ClickHouse/ClickHouse/pull/4724) ([Remen Ivan](https://github.com/BHYCHIK)) [\#5805](https://github.com/ClickHouse/ClickHouse/pull/5805) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- インターゲートH3機能 `geoToH3` ユーバーから。 [\#4724](https://github.com/ClickHouse/ClickHouse/pull/4724) ([レメン-イヴァン](https://github.com/BHYCHIK)) [\#5805](https://github.com/ClickHouse/ClickHouse/pull/5805) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) #### バグ修正 {#bug-fix-21} -- 非同期updateでdnsキャッシュを実装します。 個別のスレッドで解決すべてのホストを更新dnsキャッシュが期間(設定 `dns_cache_update_period`). ホストのipが頻繁に変更されるときに役立ちます。 [\#5857](https://github.com/ClickHouse/ClickHouse/pull/5857) ([アントン-ポポフ](https://github.com/CurtizJ)) -- Segfaultを修正する `Delta` 32ビットサイズ未満の値を持つ列に影響を与えるコーデック。 バグはランダムメモリの破損につながった。 [\#5786](https://github.com/ClickHouse/ClickHouse/pull/5786) ([alesapin](https://github.com/alesapin)) -- ブロック内の非物理列とttlマージでsegfaultを修正しました。 [\#5819](https://github.com/ClickHouse/ClickHouse/pull/5819) ([アントン-ポポフ](https://github.com/CurtizJ)) -- との部分のチェックでまれなバグを修正 `LowCardinality` コラム 以前は `checkDataPart` 常に `LowCardinality` コラム [\#5832](https://github.com/ClickHouse/ClickHouse/pull/5832) ([alesapin](https://github.com/alesapin)) -- 回避掛けに接続した場合、サーバスレッドプールを行います。 それはからの接続のために重要です `remote` 長い接続タイムアウトがある場合、テーブル関数またはレプリカなしのシャードへの接続。 この修正 [\#5878](https://github.com/ClickHouse/ClickHouse/issues/5878) [\#5881](https://github.com/ClickHouse/ClickHouse/pull/5881) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 定数引数のサポート `evalMLModel` 機能。 この修正 [\#5817](https://github.com/ClickHouse/ClickHouse/issues/5817) [\#5820](https://github.com/ClickHouse/ClickHouse/pull/5820) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 固定に問題がclickhouse判断したデフォルトのタイムゾーンとして `UCT` 代わりに `UTC`. この修正 [\#5804](https://github.com/ClickHouse/ClickHouse/issues/5804). [\#5828](https://github.com/ClickHouse/ClickHouse/pull/5828) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 固定バッファアンダーフローで `visitParamExtractRaw`. この修正 [\#5901](https://github.com/ClickHouse/ClickHouse/issues/5901) [\#5902](https://github.com/ClickHouse/ClickHouse/pull/5902) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 今すぐ配布 `DROP/ALTER/TRUNCATE/OPTIMIZE ON CLUSTER` クエリはリーダーレプリカで直接実行されます。 [\#5757](https://github.com/ClickHouse/ClickHouse/pull/5757) ([alesapin](https://github.com/alesapin)) -- 修正 `coalesce` のために `ColumnConst` と `ColumnNullable` +関連する変更。 [\#5755](https://github.com/ClickHouse/ClickHouse/pull/5755) ([Artem Zuikov](https://github.com/4ertus2)) -- 修正する `ReadBufferFromKafkaConsumer` なで読む新しいメッセージ `commit()` たとえそれが以前に失速したとしても [\#5852](https://github.com/ClickHouse/ClickHouse/pull/5852) ([イワン](https://github.com/abyss7)) -- 修正 `FULL` と `RIGHT` 結合時の結合結果 `Nullable` 右のテーブルのキー。 [\#5859](https://github.com/ClickHouse/ClickHouse/pull/5859) ([Artem Zuikov](https://github.com/4ertus2)) -- 優先度の低いクエリの無限スリープの可能性のある修正。 [\#5842](https://github.com/ClickHouse/ClickHouse/pull/5842) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- これにより、いくつかのクエリがquery\_logの後に表示されないことがあります `SYSTEM FLUSH LOGS` クエリ。 [\#5456](https://github.com/ClickHouse/ClickHouse/issues/5456) [\#5685](https://github.com/ClickHouse/ClickHouse/pull/5685) ([アントン-ポポフ](https://github.com/CurtizJ)) -- 固定 `heap-use-after-free` 阿讃注意ClusterCopierによる腕時計を利用するようにしてすでに削除され複写機のオブジェクトです。 [\#5871](https://github.com/ClickHouse/ClickHouse/pull/5871) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 固定間違った `StringRef` の実装によって返されるポインタ `IColumn::deserializeAndInsertFromArena`. このバグは単体テストのみに影響しました。 [\#5973](https://github.com/ClickHouse/ClickHouse/pull/5973) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 同じ名前の列をマスクするソースおよび中間配列の結合列を防ぎます。 [\#5941](https://github.com/ClickHouse/ClickHouse/pull/5941) ([Artem Zuikov](https://github.com/4ertus2)) -- MySQLスタイル識別子の引用とMySQLエンジンへの挿入と選択クエリを修正しました。 [\#5704](https://github.com/ClickHouse/ClickHouse/pull/5704) ([冬張](https://github.com/zhang2014)) -- さて `CHECK TABLE` queryは、MergeTreeエンジンファミリで動作します。 各パート（またはsimplierエンジンの場合はファイル）のチェックステータスとメッセージが返されます。 また、壊れた部分のフェッチのバグを修正しました。 [\#5865](https://github.com/ClickHouse/ClickHouse/pull/5865) ([alesapin](https://github.com/alesapin)) -- SPLIT\_SHARED\_LIBRARIESランタイムを修正 [\#5793](https://github.com/ClickHouse/ClickHouse/pull/5793) ([Danila Kutenin](https://github.com/danlark1)) -- 固定タイムゾーンの初期化 `/etc/localtime` シンボリックリン `../usr/share/zoneinfo/Europe/Moscow` [\#5922](https://github.com/ClickHouse/ClickHouse/pull/5922) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- clickhouse-コピー機：シャットダウン時に使用-after freeを修正 [\#5752](https://github.com/ClickHouse/ClickHouse/pull/5752) ([proller](https://github.com/proller)) -- 更新 `simdjson`. ゼロバイトを持ついくつかの無効なJSONsが正常に解析する問題を修正しました。 [\#5938](https://github.com/ClickHouse/ClickHouse/pull/5938) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- 非同期updateでDNSキャッシュを実装します。 個別のスレッドで解決すべてのホストを更新DNSキャッシュが期間(設定 `dns_cache_update_period`). ホストのipが頻繁に変化するときに役立ちます。 [\#5857](https://github.com/ClickHouse/ClickHouse/pull/5857) ([アントン-ポポフ](https://github.com/CurtizJ)) +- でsegfaultを修正 `Delta` 32ビットサイズ未満の値を持つ列に影響を与えるコーデック。 バグを修正ledランダムメモリが破損す [\#5786](https://github.com/ClickHouse/ClickHouse/pull/5786) ([アレサピン](https://github.com/alesapin)) +- TTLマージでsegfaultを修正ブロック内の非物理列と。 [\#5819](https://github.com/ClickHouse/ClickHouse/pull/5819) ([アントン-ポポフ](https://github.com/CurtizJ)) +- パーツのチェックで珍しいバグを修正 `LowCardinality` 列。 以前は `checkDataPart` 常に失敗します `LowCardinality` 列。 [\#5832](https://github.com/ClickHouse/ClickHouse/pull/5832) ([アレサピン](https://github.com/alesapin)) +- 回避掛けに接続した場合、サーバスレッドプールを行います。 それはからの接続にとって重要です `remote` 長い接続タイムアウトがある場合、レプリカのないシャードへの表関数または接続。 この修正 [\#5878](https://github.com/ClickHouse/ClickHouse/issues/5878) [\#5881](https://github.com/ClickHouse/ClickHouse/pull/5881) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 定数引数のサポート `evalMLModel` 機能。 この修正 [\#5817](https://github.com/ClickHouse/ClickHouse/issues/5817) [\#5820](https://github.com/ClickHouse/ClickHouse/pull/5820) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- ClickHouseがデフォルトのタイムゾーンを `UCT` 代わりに `UTC`. この修正 [\#5804](https://github.com/ClickHouse/ClickHouse/issues/5804). [\#5828](https://github.com/ClickHouse/ClickHouse/pull/5828) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 固定バッファアンダーフロー `visitParamExtractRaw`. この修正 [\#5901](https://github.com/ClickHouse/ClickHouse/issues/5901) [\#5902](https://github.com/ClickHouse/ClickHouse/pull/5902) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 現在配布中 `DROP/ALTER/TRUNCATE/OPTIMIZE ON CLUSTER` クエリはリーダーレプリカで直接実行されます。 [\#5757](https://github.com/ClickHouse/ClickHouse/pull/5757) ([アレサピン](https://github.com/alesapin)) +- 修正 `coalesce` のために `ColumnConst` と `ColumnNullable` +関連する変更。 [\#5755](https://github.com/ClickHouse/ClickHouse/pull/5755) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- を修正 `ReadBufferFromKafkaConsumer` 新しいメッセージを読み続けるように `commit()` 前に失速したとしても [\#5852](https://github.com/ClickHouse/ClickHouse/pull/5852) ([イワン](https://github.com/abyss7)) +- 修正 `FULL` と `RIGHT` 参加時の結果参加 `Nullable` 右のテーブルのキー。 [\#5859](https://github.com/ClickHouse/ClickHouse/pull/5859) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- 優先度の低いクエリの無限のスリープの可能性のある修正。 [\#5842](https://github.com/ClickHouse/ClickHouse/pull/5842) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- いくつかのクエリがquery\_logに表示されない原因となる競合状態を修正しました `SYSTEM FLUSH LOGS` クエリ。 [\#5456](https://github.com/ClickHouse/ClickHouse/issues/5456) [\#5685](https://github.com/ClickHouse/ClickHouse/pull/5685) ([アントン-ポポフ](https://github.com/CurtizJ)) +- 固定 `heap-use-after-free` 既に削除された複写機オブジェクトを使用しようとする時計によって引き起こされるClusterCopierのASan警告。 [\#5871](https://github.com/ClickHouse/ClickHouse/pull/5871) ([ニコライ-コチェトフ](https://github.com/KochetovNicolai)) +- 間違って修正 `StringRef` 一部の実装によって返されるポインタ `IColumn::deserializeAndInsertFromArena`. このバグは単体テストのみに影響しました。 [\#5973](https://github.com/ClickHouse/ClickHouse/pull/5973) ([ニコライ-コチェトフ](https://github.com/KochetovNicolai)) +- 同じ名前の列をマスクするソースおよび中間配列結合列を防止します。 [\#5941](https://github.com/ClickHouse/ClickHouse/pull/5941) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- Mysqlスタイルの識別子を引用してMySQLエンジンにクエリを挿入し、選択を修正しました。 [\#5704](https://github.com/ClickHouse/ClickHouse/pull/5704) ([冬張](https://github.com/zhang2014)) +- さて `CHECK TABLE` クエリはMergeTreeエンジンファミリで動作できます。 を返しますチェック状態とメッセージの場合の各部分(またはファイルの場合simplierエンジン） また、壊れた部分のフェッチのバグを修正しました。 [\#5865](https://github.com/ClickHouse/ClickHouse/pull/5865) ([アレサピン](https://github.com/alesapin)) +- SPLIT\_SHARED\_LIBRARIESランタイムを修正 [\#5793](https://github.com/ClickHouse/ClickHouse/pull/5793) ([ダニラ-クテニン](https://github.com/danlark1)) +- 固定時のタイムゾーンの初期化 `/etc/localtime` のような相対的なシンボリックリンク `../usr/share/zoneinfo/Europe/Moscow` [\#5922](https://github.com/ClickHouse/ClickHouse/pull/5922) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- clickhouse-複写機：シャットダウン時に無料で使用した後に修正 [\#5752](https://github.com/ClickHouse/ClickHouse/pull/5752) ([プロラー](https://github.com/proller)) +- 更新 `simdjson`. ゼロバイトのいくつかの無効なJSONsが正常に解析する問題を修正しました。 [\#5938](https://github.com/ClickHouse/ClickHouse/pull/5938) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) - SystemLogsのシャットダウンを修正 [\#5802](https://github.com/ClickHouse/ClickHouse/pull/5802) ([アントン-ポポフ](https://github.com/CurtizJ)) -- Invalidate\_queryの条件が辞書に依存しているときにハングする問題を修正しました。 [\#6011](https://github.com/ClickHouse/ClickHouse/pull/6011) ([Vitaly Baranov](https://github.com/vitlibar)) +- Invalidate\_queryの条件が辞書に依存する場合のハングを修正しました。 [\#6011](https://github.com/ClickHouse/ClickHouse/pull/6011) ([ヴィタリー-バラノフ](https://github.com/vitlibar)) #### 改善 {#improvement-6} -- クラスター構成で解決できないアドレスを許可します。 彼らは利用できないとみなされ、すべての接続試行で解決しようとします。 これはkubernetesに特に便利です。 この修正 [\#5714](https://github.com/ClickHouse/ClickHouse/issues/5714) [\#5924](https://github.com/ClickHouse/ClickHouse/pull/5924) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- アイドル状態のtcp接続を閉じる(デフォルトでは一時間待ち)。 これは、すべてのサーバーが他のすべてのサーバーへの接続プールを保持する可能性があり、ピーククエリの同時実行の後に接続が停止するためです。 この修正 [\#5879](https://github.com/ClickHouse/ClickHouse/issues/5879) [\#5880](https://github.com/ClickHouse/ClickHouse/pull/5880) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- より良い品質の `topK` 機能。 新しい要素の重みが大きい場合、最後の要素を削除するようにSavingSpace設定の動作を変更しました。 [\#5833](https://github.com/ClickHouse/ClickHouse/issues/5833) [\#5850](https://github.com/ClickHouse/ClickHouse/pull/5850) ([ギヨームタッセリー](https://github.com/YiuRULE)) -- URLの機能と作業領域は今では不完全なUrlなスキーム [\#5725](https://github.com/ClickHouse/ClickHouse/pull/5725) ([alesapin](https://github.com/alesapin)) -- に追加されたチェックサム `system.parts_columns` テーブル。 [\#5874](https://github.com/ClickHouse/ClickHouse/pull/5874) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- 追加 `Enum` synonimとしてのデータ型 `Enum8` または `Enum16`. [\#5886](https://github.com/ClickHouse/ClickHouse/pull/5886) ([ディマルブ2000](https://github.com/dimarub2000)) -- フルビット転置変異体のための `T64` コーデック。 より良い圧縮につながる可能性がある `zstd`. [\#5742](https://github.com/ClickHouse/ClickHouse/pull/5742) ([Artem Zuikov](https://github.com/4ertus2)) -- 条件に `startsWith` 機能は主キーを使用することができます。 この修正 [\#5310](https://github.com/ClickHouse/ClickHouse/issues/5310) と [\#5882](https://github.com/ClickHouse/ClickHouse/issues/5882) [\#5919](https://github.com/ClickHouse/ClickHouse/pull/5919) ([ディマルブ2000](https://github.com/dimarub2000)) +- クラスター構成で解決不可能なアドレスを許可します。 これらは使用できないと見なされ、接続の試行ごとに解決が試みられます。 これはKubernetesにとって特に便利です。 この修正 [\#5714](https://github.com/ClickHouse/ClickHouse/issues/5714) [\#5924](https://github.com/ClickHouse/ClickHouse/pull/5924) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- アイドル状態のTCP接続を閉じます(既定では一時間のタイムアウト)。 これは、すべてのサーバーに複数の分散テーブルがある大規模なクラスターでは、すべてのサーバーが他のすべてのサーバーとの接続プールを保持でき、クエリの同時実 この修正 [\#5879](https://github.com/ClickHouse/ClickHouse/issues/5879) [\#5880](https://github.com/ClickHouse/ClickHouse/pull/5880) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- より良い品質の `topK` 機能。 新しい要素の重みが大きい場合、最後の要素を削除するようにSavingSpace setの動作を変更しました。 [\#5833](https://github.com/ClickHouse/ClickHouse/issues/5833) [\#5850](https://github.com/ClickHouse/ClickHouse/pull/5850) ([ギヨーム-タッセリー](https://github.com/YiuRULE)) +- URLの機能と作業領域は今では不完全なUrlなスキーム [\#5725](https://github.com/ClickHouse/ClickHouse/pull/5725) ([アレサピン](https://github.com/alesapin)) +- に追加されたチェックサム `system.parts_columns` テーブル。 [\#5874](https://github.com/ClickHouse/ClickHouse/pull/5874) ([ニキータ-ミハイロフ](https://github.com/nikitamikhaylov)) +- 追加 `Enum` のためのsynonimとしてのデータ型 `Enum8` または `Enum16`. [\#5886](https://github.com/ClickHouse/ClickHouse/pull/5886) ([dimarub2000](https://github.com/dimarub2000)) +- のための完全なビット転置の変形 `T64` コーデック。 がより良い圧縮 `zstd`. [\#5742](https://github.com/ClickHouse/ClickHouse/pull/5742) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- 条件 `startsWith` 機能するまでに使用その有効なタイプを利用します。 この修正 [\#5310](https://github.com/ClickHouse/ClickHouse/issues/5310) と [\#5882](https://github.com/ClickHouse/ClickHouse/issues/5882) [\#5919](https://github.com/ClickHouse/ClickHouse/pull/5919) ([dimarub2000](https://github.com/dimarub2000)) - 使用を許可する `clickhouse-copier` クロス-複製クラスタトポロジーを許可する空のデータベースの名前です。 [\#5745](https://github.com/ClickHouse/ClickHouse/pull/5745) ([nvartolomei](https://github.com/nvartolomei)) -- 使用 `UTC` システム上のデフォルトのタイムゾーンとして `tzdata` (e.g. bare Docker container). Before this patch, error message `Could not determine local time zone` 印刷され、サーバまたはクライアン [\#5827](https://github.com/ClickHouse/ClickHouse/pull/5827) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 関数の浮動小数点引数のサポートを返しました `quantileTiming` 下位互換性のため。 [\#5911](https://github.com/ClickHouse/ClickHouse/pull/5911) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- エラーメッセージの列がないテーブルを表示します。 [\#5768](https://github.com/ClickHouse/ClickHouse/pull/5768) ([イワン](https://github.com/abyss7)) -- さまざまなユーザーが同じquery\_idで実行クエリを許可しない [\#5430](https://github.com/ClickHouse/ClickHouse/pull/5430) ([proller](https://github.com/proller)) -- より強固なコードの送信メトリクスをグラファイトを表してい それは長い倍数の間に働きます `RENAME TABLE` オペレーション [\#5875](https://github.com/ClickHouse/ClickHouse/pull/5875) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- よりエラーメッセージが表示されますのでthreadpoolできない予定、タスクを実行します。 この修正 [\#5305](https://github.com/ClickHouse/ClickHouse/issues/5305) [\#5801](https://github.com/ClickHouse/ClickHouse/pull/5801) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- NgramSearchを反転させると、より直感的になります [\#5807](https://github.com/ClickHouse/ClickHouse/pull/5807) ([Danila Kutenin](https://github.com/danlark1)) +- 使用 `UTC` システムのデフォルトのタイムゾーンとして `tzdata` (e.g. bare Docker container). Before this patch, error message `Could not determine local time zone` サーバーまたはクライアントが起動を拒否しました。 [\#5827](https://github.com/ClickHouse/ClickHouse/pull/5827) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 関数内の浮動小数点引数のサポートを返しました `quantileTiming` 下位互換性のため。 [\#5911](https://github.com/ClickHouse/ClickHouse/pull/5911) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- エラーメッセージに列がないテーブルを表示します。 [\#5768](https://github.com/ClickHouse/ClickHouse/pull/5768) ([イワン](https://github.com/abyss7)) +- さまざまなユーザーが同じquery\_idでクエリを実行できない [\#5430](https://github.com/ClickHouse/ClickHouse/pull/5430) ([プロラー](https://github.com/proller)) +- より強固なコードの送信メトリクスをグラファイトを表してい それは長い倍数の間に働きます `RENAME TABLE` 作戦だ [\#5875](https://github.com/ClickHouse/ClickHouse/pull/5875) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- よりエラーメッセージが表示されますのでThreadPoolできない予定、タスクを実行します。 この修正 [\#5305](https://github.com/ClickHouse/ClickHouse/issues/5305) [\#5801](https://github.com/ClickHouse/ClickHouse/pull/5801) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- より直感的にngramsearchを反転する [\#5807](https://github.com/ClickHouse/ClickHouse/pull/5807) ([ダニラ-クテニン](https://github.com/danlark1)) - HDFS engine builderでのユーザー解析の追加 [\#5946](https://github.com/ClickHouse/ClickHouse/pull/5946) ([akonyaev90](https://github.com/akonyaev90)) -- 更新のデフォルト値 `max_ast_elements parameter` [\#5933](https://github.com/ClickHouse/ClickHouse/pull/5933) ([Artem Konovalov](https://github.com/izebit)) -- 廃止された設定の概念を追加しました。 廃止された設定 `allow_experimental_low_cardinality_type` 効果なしで使用することができます。 [0f15c01c6802f7ce1a1494c846be8c98944cd](https://github.com/ClickHouse/ClickHouse/commit/0f15c01c6802f7ce1a1494c12c846be8c98944cd) [Alexey Milovidov](https://github.com/alexey-milovidov) +- デフォルト値の更新 `max_ast_elements parameter` [\#5933](https://github.com/ClickHouse/ClickHouse/pull/5933) ([アルテム-コノヴァロフ](https://github.com/izebit)) +- 廃止された設定の概念を追加しました。 廃止された設定 `allow_experimental_low_cardinality_type` 効果無しで使用することができます。 [0f15c01c6802f7ce1a1494c12c846be8c98944cd](https://github.com/ClickHouse/ClickHouse/commit/0f15c01c6802f7ce1a1494c12c846be8c98944cd) [アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov) #### 性能向上 {#performance-improvement-4} -- 増加数の河川から選択するとmergeテーブルにより均一に分布す。 追加された設定 `max_streams_multiplier_for_merge_tables`. この修正 [\#5797](https://github.com/ClickHouse/ClickHouse/issues/5797) [\#5915](https://github.com/ClickHouse/ClickHouse/pull/5915) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- 増加数の河川から選択するとMergeテーブルにより均一に分布す。 設定を追加 `max_streams_multiplier_for_merge_tables`. この修正 [\#5797](https://github.com/ClickHouse/ClickHouse/issues/5797) [\#5915](https://github.com/ClickHouse/ClickHouse/pull/5915) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) -#### ビルド/テスト/パッケージの改善 {#buildtestingpackaging-improvement-7} +#### 造り/テスト/包装の改善 {#buildtestingpackaging-improvement-7} -- Clickhouseの異なるバージョンとクライアン [\#5868](https://github.com/ClickHouse/ClickHouse/pull/5868) ([alesapin](https://github.com/alesapin)) -- テスト対象の情報を毎にコミットを引きます。 [\#5896](https://github.com/ClickHouse/ClickHouse/pull/5896) ([alesapin](https://github.com/alesapin)) -- カスタムアロケータをサポートするaddress sanitizerと連携 (`Arena` と `ArenaWithFreeLists`)より良いデバッグのための “use-after-free” エラー。 [\#5728](https://github.com/ClickHouse/ClickHouse/pull/5728) ([azerbaijan.kgm](https://github.com/akuzm)) -- に切り替える [LLVM libunwindの実装](https://github.com/llvm-mirror/libunwind) C++例外処理およびスタックトレース印刷用 [\#4828](https://github.com/ClickHouse/ClickHouse/pull/4828) ([Nikita Lapkov](https://github.com/laplab)) -- さらに二つの警告を追加-weverything [\#5923](https://github.com/ClickHouse/ClickHouse/pull/5923) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- メモリ消毒剤とclickhouseを構築することができます。 [\#3949](https://github.com/ClickHouse/ClickHouse/pull/3949) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 固定ubsanレポートについて `bitTest` ファズテストの機能。 [\#5943](https://github.com/ClickHouse/ClickHouse/pull/5943) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Docker：認証を必要とするClickHouseインスタンスを初期化する可能性を追加しました。 [\#5727](https://github.com/ClickHouse/ClickHouse/pull/5727) ([Korviakov Andrey](https://github.com/shurshun)) -- バージョン1.1.0にlibrdkafkaを更新 [\#5872](https://github.com/ClickHouse/ClickHouse/pull/5872) ([イワン](https://github.com/abyss7)) -- 追加グローバルタイムアウトのための統合の試験を無効にし試験ます。 [\#5741](https://github.com/ClickHouse/ClickHouse/pull/5741) ([alesapin](https://github.com/alesapin)) -- いくつかのthreadsanitizerの障害を修正します。 [\#5854](https://github.com/ClickHouse/ClickHouse/pull/5854) ([azerbaijan.kgm](https://github.com/akuzm)) +- Clickhouseの異なるバージョンとのクライアントサーバーの相互作用の下位互換性テストを追加します。 [\#5868](https://github.com/ClickHouse/ClickHouse/pull/5868) ([アレサピン](https://github.com/alesapin)) +- テスト対象の情報を毎にコミットを引きます。 [\#5896](https://github.com/ClickHouse/ClickHouse/pull/5896) ([アレサピン](https://github.com/alesapin)) +- アドレスsanitizerと協力してカスタムアロケータをサポート (`Arena` と `ArenaWithFreeLists`）のよりよいデバッグのため “use-after-free” エラー [\#5728](https://github.com/ClickHouse/ClickHouse/pull/5728) ([akuzm](https://github.com/akuzm)) +- に切り替える [LLVM libunwindの実装](https://github.com/llvm-mirror/libunwind) C++例外処理およびスタックトレース印刷の場合 [\#4828](https://github.com/ClickHouse/ClickHouse/pull/4828) ([ニキータ-ラプコフ](https://github.com/laplab)) +- からさらに二つの警告を追加-Weverything [\#5923](https://github.com/ClickHouse/ClickHouse/pull/5923) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- メモリサニタイザーでClickHouseを構築することができます。 [\#3949](https://github.com/ClickHouse/ClickHouse/pull/3949) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- についての固定ubsanレポート `bitTest` fuzzテストの機能。 [\#5943](https://github.com/ClickHouse/ClickHouse/pull/5943) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- Docker：認証を必要とするClickHouseインスタンスを初期化する可能性を追加しました。 [\#5727](https://github.com/ClickHouse/ClickHouse/pull/5727) ([コルビャコフ-アンドレイ](https://github.com/shurshun)) +- Librdkafkaをバージョン1.1.0に更新する [\#5872](https://github.com/ClickHouse/ClickHouse/pull/5872) ([イワン](https://github.com/abyss7)) +- 追加グローバルタイムアウトのための統合の試験を無効にし試験ます。 [\#5741](https://github.com/ClickHouse/ClickHouse/pull/5741) ([アレサピン](https://github.com/alesapin)) +- いくつかのThreadSanitizerエラーを修正しました。 [\#5854](https://github.com/ClickHouse/ClickHouse/pull/5854) ([akuzm](https://github.com/akuzm)) - その `--no-undefined` オプション力、リンカーをチェックすべての外部の名の存在をリンク 分割ビルドモードでライブラリ間の実際の依存関係を追跡することは非常に便利です。 [\#5855](https://github.com/ClickHouse/ClickHouse/pull/5855) ([イワン](https://github.com/abyss7)) -- のための追加された性能試験 [\#5797](https://github.com/ClickHouse/ClickHouse/issues/5797) [\#5914](https://github.com/ClickHouse/ClickHouse/pull/5914) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Gcc-7との互換性を修正しました。 [\#5840](https://github.com/ClickHouse/ClickHouse/pull/5840) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Gcc-9のサポートが追加されました。 この修正 [\#5717](https://github.com/ClickHouse/ClickHouse/issues/5717) [\#5774](https://github.com/ClickHouse/ClickHouse/pull/5774) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Libunwindが正しくリンクできない場合のエラーを修正。 [\#5948](https://github.com/ClickHouse/ClickHouse/pull/5948) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- PVS-Studioによって検出されたいくつかの警告を修正しました。 [\#5921](https://github.com/ClickHouse/ClickHouse/pull/5921) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 追加された初期サポート `clang-tidy` 静的な検光子。 [\#5806](https://github.com/ClickHouse/ClickHouse/pull/5806) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- BSD/Linuxエンディアンマクロを変換する( ‘be64toh’ と ‘htobe64’)Mac OS Xに相当するもの [\#5785](https://github.com/ClickHouse/ClickHouse/pull/5785) ([フーチェン](https://github.com/fredchenbj)) -- 統合テストガイドの改善。 [\#5796](https://github.com/ClickHouse/ClickHouse/pull/5796) ([Vladimir Chebotarev](https://github.com/excitoon)) +- のための追加された性能試験 [\#5797](https://github.com/ClickHouse/ClickHouse/issues/5797) [\#5914](https://github.com/ClickHouse/ClickHouse/pull/5914) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- Gcc-7との互換性を修正しました。 [\#5840](https://github.com/ClickHouse/ClickHouse/pull/5840) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- Gcc-9のサポートが追加されました。 この修正 [\#5717](https://github.com/ClickHouse/ClickHouse/issues/5717) [\#5774](https://github.com/ClickHouse/ClickHouse/pull/5774) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- Libunwindが正しくリンクされていない場合のエラーを修正。 [\#5948](https://github.com/ClickHouse/ClickHouse/pull/5948) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- PVS-Studioによって発見されたいくつかの警告を修正しました。 [\#5921](https://github.com/ClickHouse/ClickHouse/pull/5921) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 追加された初期サポート `clang-tidy` 静的な検光子。 [\#5806](https://github.com/ClickHouse/ClickHouse/pull/5806) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- BSD/Linuxエンディアンマクロの変換( ‘be64toh’ と ‘htobe64’）に相当する。 [\#5785](https://github.com/ClickHouse/ClickHouse/pull/5785) ([フー-チェン](https://github.com/fredchenbj)) +- 改善された統合テストガイド。 [\#5796](https://github.com/ClickHouse/ClickHouse/pull/5796) ([ウラジーミル-チェボタレフ](https://github.com/excitoon)) - Macosx+gcc9でのビルドの修正 [\#5822](https://github.com/ClickHouse/ClickHouse/pull/5822) ([フィリモノフ](https://github.com/filimonov)) -- 難しいタイプミスを修正：aggreagte-\>aggregate。 [\#5753](https://github.com/ClickHouse/ClickHouse/pull/5753) ([azerbaijan.kgm](https://github.com/akuzm)) -- Freebsdビルドの修正 [\#5760](https://github.com/ClickHouse/ClickHouse/pull/5760) ([proller](https://github.com/proller)) -- 追加リンク実験youtubeチャンネルサイト [\#5845](https://github.com/ClickHouse/ClickHouse/pull/5845) ([Ivan Blinkov](https://github.com/blinkov)) -- CMake:カバレッジフラグのオプションを追加:WITH\_COVERAGE [\#5776](https://github.com/ClickHouse/ClickHouse/pull/5776) ([proller](https://github.com/proller)) -- いくつかのインラインpodarrayの初期サイズを修正。 [\#5787](https://github.com/ClickHouse/ClickHouse/pull/5787) ([azerbaijan.kgm](https://github.com/akuzm)) -- clickhouse-サーバー.postinst：centos6のos検出を修正 [\#5788](https://github.com/ClickHouse/ClickHouse/pull/5788) ([proller](https://github.com/proller)) -- 追加されたアーチlinuxパッケージ生成。 [\#5719](https://github.com/ClickHouse/ClickHouse/pull/5719) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 共通/設定を分割します。h by libs(dbms) [\#5715](https://github.com/ClickHouse/ClickHouse/pull/5715) ([proller](https://github.com/proller)) -- 以下のための修正 “Arcadia” ムの構築 [\#5795](https://github.com/ClickHouse/ClickHouse/pull/5795) ([proller](https://github.com/proller)) -- 型にはまらないビルドの修正(gcc9、サブモジュールなし) [\#5792](https://github.com/ClickHouse/ClickHouse/pull/5792) ([proller](https://github.com/proller)) -- バグが発生しやすいことが証明されているため、unalignedstoreで明示的な型を必要とします [\#5791](https://github.com/ClickHouse/ClickHouse/pull/5791) ([azerbaijan.kgm](https://github.com/akuzm)) -- Macosのビルドを修正 [\#5830](https://github.com/ClickHouse/ClickHouse/pull/5830) ([フィリモノフ](https://github.com/filimonov)) -- ここで要求されたより大きなデータセットを持つ新しいjit機能に関する性能試験 [\#5263](https://github.com/ClickHouse/ClickHouse/issues/5263) [\#5887](https://github.com/ClickHouse/ClickHouse/pull/5887) ([ギヨームタッセリー](https://github.com/YiuRULE)) -- 走行状態での試験はストレステスト [12693e568722f11e19859742f56428455501fd2a](https://github.com/ClickHouse/ClickHouse/commit/12693e568722f11e19859742f56428455501fd2a) ([alesapin](https://github.com/alesapin)) +- AggreAGte-\>aggregateという難しいタイプミスを修正しました。 [\#5753](https://github.com/ClickHouse/ClickHouse/pull/5753) ([akuzm](https://github.com/akuzm)) +- Freebsdのビルドを修正 [\#5760](https://github.com/ClickHouse/ClickHouse/pull/5760) ([プロラー](https://github.com/proller)) +- 追加リンク実験YouTubeチャンネルサイト [\#5845](https://github.com/ClickHouse/ClickHouse/pull/5845) ([イヴァン-ブリンコフ](https://github.com/blinkov)) +- CMake:カバレッジフラグのオプションを追加:WITH\_COVERAGE [\#5776](https://github.com/ClickHouse/ClickHouse/pull/5776) ([プロラー](https://github.com/proller)) +- インラインPODArrayの初期サイズを修正しました。 [\#5787](https://github.com/ClickHouse/ClickHouse/pull/5787) ([akuzm](https://github.com/akuzm)) +- clickhouse-サーバー.postinst:centos6のos検出を修正 [\#5788](https://github.com/ClickHouse/ClickHouse/pull/5788) ([プロラー](https://github.com/proller)) +- Arch linuxパッケージの生成を追加。 [\#5719](https://github.com/ClickHouse/ClickHouse/pull/5719) ([ウラジーミル-チェボタレフ](https://github.com/excitoon)) +- 共通/設定を分割します。h by libs(dbms) [\#5715](https://github.com/ClickHouse/ClickHouse/pull/5715) ([プロラー](https://github.com/proller)) +- の修正 “Arcadia” ムの構築 [\#5795](https://github.com/ClickHouse/ClickHouse/pull/5795) ([プロラー](https://github.com/proller)) +- 型破りなビルドの修正(gcc9、サブモジュールなし) [\#5792](https://github.com/ClickHouse/ClickHouse/pull/5792) ([プロラー](https://github.com/proller)) +- バグが発生しやすいことが判明したため、unalignedstoreで明示的な型を必要とする [\#5791](https://github.com/ClickHouse/ClickHouse/pull/5791) ([akuzm](https://github.com/akuzm)) +- MacOSのビルドを修正 [\#5830](https://github.com/ClickHouse/ClickHouse/pull/5830) ([フィリモノフ](https://github.com/filimonov)) +- より大きなデータセットを持つ新しいJIT機能に関する性能テスト [\#5263](https://github.com/ClickHouse/ClickHouse/issues/5263) [\#5887](https://github.com/ClickHouse/ClickHouse/pull/5887) ([ギヨーム-タッセリー](https://github.com/YiuRULE)) +- Stress testでのステートフルテストの実行 [12693e568722f11e19859742f56428455501fd2a](https://github.com/ClickHouse/ClickHouse/commit/12693e568722f11e19859742f56428455501fd2a) ([アレサピン](https://github.com/alesapin)) #### 下位互換性のない変更 {#backward-incompatible-change-7} - `Kafka` このバージョンでは壊れている。 -- 有効 `adaptive_index_granularity` =新しいのためのデフォルトで10MB `MergeTree` テーブル。 バージョン19.11以降で新しいMergeTreeテーブルを作成した場合、19.6より前のバージョンへのダウングレードは不可能になります。 [\#5628](https://github.com/ClickHouse/ClickHouse/pull/5628) ([alesapin](https://github.com/alesapin)) -- Yandexのによって使用された廃止された文書化されていない埋め込まれた辞書を削除。メトリカ を機能 `OSIn`, `SEIn`, `OSToRoot`, `SEToRoot`, `OSHierarchy`, `SEHierarchy` もはや利用できません。 これらの機能を使用している場合は、電子メールをclickhouse-feedback@yandex-team.com。注：最後の瞬間に我々はしばらくの間、これらの機能を維持することを決めました。 [\#5780](https://github.com/ClickHouse/ClickHouse/pull/5780) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- 有効にする `adaptive_index_granularity` =新しいのためのデフォルトで10MB `MergeTree` テーブル バージョン19.11+で新しいMergeTreeテーブルを作成した場合、19.6より前のバージョンへのダウングレードは不可能になります。 [\#5628](https://github.com/ClickHouse/ClickHouse/pull/5628) ([アレサピン](https://github.com/alesapin)) +- Yandexのによって使用された古い文書化されていない埋め込み辞書を削除しました。メトリカ 機能 `OSIn`, `SEIn`, `OSToRoot`, `SEToRoot`, `OSHierarchy`, `SEHierarchy` 利用できなくなりました。 ご利用の場合これらの機能は、メールclickhouse-feedback@yandex-team.com. 注:最後の瞬間にしてくれました。 [\#5780](https://github.com/ClickHouse/ClickHouse/pull/5780) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) ## ClickHouseリリース19.10 {#clickhouse-release-19-10} -### クリックハウスリリース19.10.1.5,2019-07-12 {#clickhouse-release-19-10-1-5-2019-07-12} +### ClickHouseリリース19.10.1.5,2019-07-12 {#clickhouse-release-19-10-1-5-2019-07-12} -#### 新しい機能 {#new-feature-7} +#### 新機能 {#new-feature-7} -- 新しい列コーデックを追加: `T64`. (U)IntX/EnumX/Data(Time)/DecimalX列用に作成されます。 定数値または小さい範囲値を持つ列に適しているはずです。 コーデック自体は、拡大または再圧縮せずにデータ型を縮小できます。 [\#5557](https://github.com/ClickHouse/ClickHouse/pull/5557) ([Artem Zuikov](https://github.com/4ertus2)) +- 新しい列コーデックの追加: `T64`. (U)IntX/EnumX/Data(Time)/DecimalX列用に作成されました。 これは、一定または小さな範囲の値を持つ列に適しているはずです。 コーデック自体は、再圧縮せずにデータ型を拡大または縮小できます。 [\#5557](https://github.com/ClickHouse/ClickHouse/pull/5557) ([アルテム-ズイコフ](https://github.com/4ertus2)) - データベースエンジ `MySQL` できる全てのテーブルをリモートMySQLサーバー [\#5599](https://github.com/ClickHouse/ClickHouse/pull/5599) ([冬張](https://github.com/zhang2014)) -- `bitmapContains` 実装。 それは2xより速いです `bitmapHasAny` 第二のビットマップが一つの要素を含む場合。 [\#5535](https://github.com/ClickHouse/ClickHouse/pull/5535) ([Zhichang Yu](https://github.com/yuzhichang)) -- のサポート `crc32` 関数（MySQLやPHPとまったく同じ動作）。 ハッシュ関数が必要な場合は使用しないでください。 [\#5661](https://github.com/ClickHouse/ClickHouse/pull/5661) ([Remen Ivan](https://github.com/BHYCHIK)) -- 実装 `SYSTEM START/STOP DISTRIBUTED SENDS` 非同期挿入を制御するクエリ `Distributed` テーブル。 [\#4935](https://github.com/ClickHouse/ClickHouse/pull/4935) ([冬張](https://github.com/zhang2014)) +- `bitmapContains` 実装。 それはより速い2xです `bitmapHasAny` の場合、二つのビットマップ一要素となります。 [\#5535](https://github.com/ClickHouse/ClickHouse/pull/5535) ([Zhichang Yu](https://github.com/yuzhichang)) +- のサポート `crc32` 関数（MySQLやPHPと同じように動作します）。 ハッシュ関数が必要な場合は使用しないでください。 [\#5661](https://github.com/ClickHouse/ClickHouse/pull/5661) ([レメン-イヴァン](https://github.com/BHYCHIK)) +- 実装 `SYSTEM START/STOP DISTRIBUTED SENDS` 非同期挿入を制御するクエリ `Distributed` テーブル [\#4935](https://github.com/ClickHouse/ClickHouse/pull/4935) ([冬張](https://github.com/zhang2014)) #### バグ修正 {#bug-fix-22} -- マージ制限のクエリ実行制限および最大パーツサイズを無視する。 [\#5659](https://github.com/ClickHouse/ClickHouse/pull/5659) ([アントン-ポポフ](https://github.com/CurtizJ)) -- 通常のブロックの重複排除（非常にまれ）と重複ブロックの挿入（より頻繁に）につながる可能性のあるバグを修正しました。 [\#5549](https://github.com/ClickHouse/ClickHouse/pull/5549) ([alesapin](https://github.com/alesapin)) -- 機能の修正 `arrayEnumerateUniqRanked` 空の配列を持つ引数の場合 [\#5559](https://github.com/ClickHouse/ClickHouse/pull/5559) ([proller](https://github.com/proller)) +- 無視するクエリの実行を制限およびmaxのパーツのサイズの合併に制限が実行中の突然変異. [\#5659](https://github.com/ClickHouse/ClickHouse/pull/5659) ([アントン-ポポフ](https://github.com/CurtizJ)) +- 通常のブロックの重複除外（非常にまれ）と重複ブロックの挿入（より頻繁に）につながる可能性のあるバグを修正しました。 [\#5549](https://github.com/ClickHouse/ClickHouse/pull/5549) ([アレサピン](https://github.com/alesapin)) +- 機能の修正 `arrayEnumerateUniqRanked` 空の配列を持つ引数の場合 [\#5559](https://github.com/ClickHouse/ClickHouse/pull/5559) ([プロラー](https://github.com/proller)) - な購読カフカ題なく意思をポーリングメッセージ. [\#5698](https://github.com/ClickHouse/ClickHouse/pull/5698) ([イワン](https://github.com/abyss7)) -- 設定を行う `join_use_nulls` Nullable内にできない型に対しては何の効果も得られません [\#5700](https://github.com/ClickHouse/ClickHouse/pull/5700) ([Olga Khvostikova](https://github.com/stavrolia)) -- 固定 `Incorrect size of index granularity` エラー [\#5720](https://github.com/ClickHouse/ClickHouse/pull/5720) ([コラクススター](https://github.com/coraxster)) -- 小数変換オーバーフローを修正 [\#5607](https://github.com/ClickHouse/ClickHouse/pull/5607) ([コラクススター](https://github.com/coraxster)) -- フラッシュバッファの場合 `WriteBufferFromHDFS`のデストラクタが呼び出されます。 これにより、 `HDFS`. [\#5684](https://github.com/ClickHouse/ClickHouse/pull/5684) ([Xindong Peng](https://github.com/eejoin)) +- 設定を行う `join_use_nulls` Nullable内に入ることができない型に対しては効果を得ない [\#5700](https://github.com/ClickHouse/ClickHouse/pull/5700) ([Olga Khvostikova](https://github.com/stavrolia)) +- 固定 `Incorrect size of index granularity` エラー [\#5720](https://github.com/ClickHouse/ClickHouse/pull/5720) ([コラクスター](https://github.com/coraxster)) +- 小数点変換オーバーフローに浮動小数点を修正 [\#5607](https://github.com/ClickHouse/ClickHouse/pull/5607) ([コラクスター](https://github.com/coraxster)) +- フラッシュバッファ `WriteBufferFromHDFS`デストラクタと呼ばれる。 これは書き込みを修正する `HDFS`. [\#5684](https://github.com/ClickHouse/ClickHouse/pull/5684) ([新東鵬](https://github.com/eejoin)) #### 改善 {#improvement-7} -- 空のセルを `CSV` としてデフォルト値を設定 `input_format_defaults_for_omitted_fields` は有効です。 [\#5625](https://github.com/ClickHouse/ClickHouse/pull/5625) ([azerbaijan.kgm](https://github.com/akuzm)) -- 外部辞書の非ブロッキングロード。 [\#5567](https://github.com/ClickHouse/ClickHouse/pull/5567) ([Vitaly Baranov](https://github.com/vitlibar)) -- ネットワークタイムアウトできるダイナミックな変化のための既存の接続に従って設定します。 [\#4558](https://github.com/ClickHouse/ClickHouse/pull/4558) ([Konstantin Podshumok](https://github.com/podshumok)) -- を使用して “public\_suffix\_list” 機能のため `firstSignificantSubdomain`, `cutToFirstSignificantSubdomain`. これは、 `gperf` ファイルから生成されたリスト：https://publicsuffix.org/list/public\_suffix\_list.dat（例えば、今我々はドメインを認識する `ac.uk` 有意ではない）。 [\#5030](https://github.com/ClickHouse/ClickHouse/pull/5030) ([ギヨームタッセリー](https://github.com/YiuRULE)) -- 採用 `IPv6` システムテーブルのデータ型。 `system.processes` と `system.query_log` [\#5640](https://github.com/ClickHouse/ClickHouse/pull/5640) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- MySQL互換性プロトコルとの接続にセッションを使用する。 \#5476 [\#5646](https://github.com/ClickHouse/ClickHouse/pull/5646) ([ユーリーバラノフ](https://github.com/yurriy)) -- サポートより `ALTER` クエリ `ON CLUSTER`. [\#5593](https://github.com/ClickHouse/ClickHouse/pull/5593) [\#5613](https://github.com/ClickHouse/ClickHouse/pull/5613) ([sundyli](https://github.com/sundy-li)) -- サポート `` のセクション `clickhouse-local` 設定ファイル。 [\#5540](https://github.com/ClickHouse/ClickHouse/pull/5540) ([proller](https://github.com/proller)) -- クエリの実行を許可する `remote` テーブル機能 `clickhouse-local` [\#5627](https://github.com/ClickHouse/ClickHouse/pull/5627) ([proller](https://github.com/proller)) +- 空のセルを扱う `CSV` 設定時のデフォルト値として `input_format_defaults_for_omitted_fields` 有効です。 [\#5625](https://github.com/ClickHouse/ClickHouse/pull/5625) ([akuzm](https://github.com/akuzm)) +- 外部辞書の非ブロック読み込み。 [\#5567](https://github.com/ClickHouse/ClickHouse/pull/5567) ([ヴィタリー-バラノフ](https://github.com/vitlibar)) +- ネットワークタイムアウトできるダイナミックな変化のための既存の接続に従って設定します。 [\#4558](https://github.com/ClickHouse/ClickHouse/pull/4558) ([コンスタンチン-ポドシュモク](https://github.com/podshumok)) +- を使用して “public\_suffix\_list” 関数の場合 `firstSignificantSubdomain`, `cutToFirstSignificantSubdomain`. で用いたハッシュテーブルの生成 `gperf` ファイルから生成されたリスト：https://publicsuffix.org/list/public\_suffix\_list.dat.(たとえば、ドメインを認識します `ac.uk` 非重要として）。 [\#5030](https://github.com/ClickHouse/ClickHouse/pull/5030) ([ギヨーム-タッセリー](https://github.com/YiuRULE)) +- 採用 `IPv6` システムテーブルのデータ型。 `system.processes` と `system.query_log` [\#5640](https://github.com/ClickHouse/ClickHouse/pull/5640) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- MySQL互換プロトコルとの接続にセッションを使用する。 \#5476 [\#5646](https://github.com/ClickHouse/ClickHouse/pull/5646) ([ユーリー-バラノフ](https://github.com/yurriy)) +- よりサポート `ALTER` クエリ `ON CLUSTER`. [\#5593](https://github.com/ClickHouse/ClickHouse/pull/5593) [\#5613](https://github.com/ClickHouse/ClickHouse/pull/5613) ([スンディリ](https://github.com/sundy-li)) +- サポート `` セクション `clickhouse-local` 設定ファイル。 [\#5540](https://github.com/ClickHouse/ClickHouse/pull/5540) ([プロラー](https://github.com/proller)) +- クエリの実行を許可する `remote` テーブル関数 `clickhouse-local` [\#5627](https://github.com/ClickHouse/ClickHouse/pull/5627) ([プロラー](https://github.com/proller)) #### 性能向上 {#performance-improvement-5} -- MergeTree列の最後に最後のマークを書き込む可能性を追加します。 これにより、テーブルデータ範囲外のキーの無駄な読み込みを回避できます。 適応インデックスの粒度が使用されている場合にのみ有効になります。 [\#5624](https://github.com/ClickHouse/ClickHouse/pull/5624) ([alesapin](https://github.com/alesapin)) -- 非常に遅いファイルシステム上のmergetreeテーブルのパフォーマンスの向上 `stat` シスコール [\#5648](https://github.com/ClickHouse/ClickHouse/pull/5648) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 固定性能の劣化からの読み出しmergetreeテーブルで導入されたバージョン19.6. 修正\#5631. [\#5633](https://github.com/ClickHouse/ClickHouse/pull/5633) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- MergeTree列の最後に最後のマークを書き込む可能性を追加します。 これを避ける無駄を読み込みのためのキーのテーブルデータです。 適応インデックスの粒度が使用されている場合にのみ有効になります。 [\#5624](https://github.com/ClickHouse/ClickHouse/pull/5624) ([アレサピン](https://github.com/alesapin)) +- 非常に遅いファイルシステムでのMergeTreeテーブルのパフォーマンスの向上 `stat` シスコール [\#5648](https://github.com/ClickHouse/ClickHouse/pull/5648) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 固定性能の劣化からの読み出しMergeTreeテーブルで導入されたバージョン19.6. 修正\#5631. [\#5633](https://github.com/ClickHouse/ClickHouse/pull/5633) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) -#### ビルド/テスト/パッケージの改善 {#buildtestingpackaging-improvement-8} +#### 造り/テスト/包装の改善 {#buildtestingpackaging-improvement-8} -- 実装 `TestKeeper` テストに使用されるZooKeeperインタフェースの実装として [\#5643](https://github.com/ClickHouse/ClickHouse/pull/5643) ([alexey-milovidov](https://github.com/alexey-milovidov)) ([levushkin aleksej](https://github.com/alexey-milovidov)) -- これからは `.sql` 試験走行ができるによって切り離されたサーバを並列には、ランダムなデータベースです。 それらをより速く実行し、カスタムサーバー構成で新しいテストを追加し、異なるテストが互いに影響しないことを確認します。 [\#5554](https://github.com/ClickHouse/ClickHouse/pull/5554) ([イワン](https://github.com/abyss7)) +- 実装 `TestKeeper` テストに使用されるZooKeeperインタフェースの実装として [\#5643](https://github.com/ClickHouse/ClickHouse/pull/5643) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) ([レブシュキン-アレクセイ](https://github.com/alexey-milovidov)) +- これからは `.sql` 試験走行ができるによって切り離されたサーバを並列には、ランダムなデータベースです。 ることができると考えて、追加試験とカスタムサーバー構成は、必ず異なる試験に影響しません。 [\#5554](https://github.com/ClickHouse/ClickHouse/pull/5554) ([イワン](https://github.com/abyss7)) - 削除 `` と `` 性能テストから [\#5672](https://github.com/ClickHouse/ClickHouse/pull/5672) ([Olga Khvostikova](https://github.com/stavrolia)) -- 固定 “select\_format” 性能試験のための `Pretty` 形式 [\#5642](https://github.com/ClickHouse/ClickHouse/pull/5642) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- 固定 “select\_format” 性能試験のための `Pretty` 形式 [\#5642](https://github.com/ClickHouse/ClickHouse/pull/5642) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) -## クリックハウスリリース19.9 {#clickhouse-release-19-9} +## ClickHouseリリース19.9 {#clickhouse-release-19-9} -### クリックハウスリリース19.9.3.31,2019-07-05 {#clickhouse-release-19-9-3-31-2019-07-05} +### ClickHouseリリース19.9.3.31,2019-07-05 {#clickhouse-release-19-9-3-31-2019-07-05} #### バグ修正 {#bug-fix-23} -- 32ビットサイズ未満の値を持つ列に影響を与えるデルタコーデックでsegfaultを修正しました。 バグはランダムメモリの破損につながった。 [\#5786](https://github.com/ClickHouse/ClickHouse/pull/5786) ([alesapin](https://github.com/alesapin)) -- LowCardinalityのコラムと部分の点検でまれな虫を修理して下さい。 [\#5832](https://github.com/ClickHouse/ClickHouse/pull/5832) ([alesapin](https://github.com/alesapin)) -- ブロック内の非物理列とttlマージでsegfaultを修正しました。 [\#5819](https://github.com/ClickHouse/ClickHouse/pull/5819) ([アントン-ポポフ](https://github.com/CurtizJ)) -- 低優先度のクエリの潜在的な無限の睡眠を修正しました。 [\#5842](https://github.com/ClickHouse/ClickHouse/pull/5842) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 固定方法clickhouse判断したデフォルトのタイムゾーンとしてuctの代わりにutcです。 [\#5828](https://github.com/ClickHouse/ClickHouse/pull/5828) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- のを修正した。約の実行の分散drop/alter/quick/最適化クラスターに関するお問い合わせフォロワレプリカの前にリーダーレプリカ. 今、彼らはリーダーのレプリカに直接実行されます。 [\#5757](https://github.com/ClickHouse/ClickHouse/pull/5757) ([alesapin](https://github.com/alesapin)) +- 32ビットサイズ未満の値を持つ列に影響を与えるデルタコーデックでsegfaultを修正しました。 バグを修正ledランダムメモリが破損す [\#5786](https://github.com/ClickHouse/ClickHouse/pull/5786) ([アレサピン](https://github.com/alesapin)) +- 低カーディナリティカラムでパーツをチェックする際のまれなバグを修正。 [\#5832](https://github.com/ClickHouse/ClickHouse/pull/5832) ([アレサピン](https://github.com/alesapin)) +- TTLマージでsegfaultを修正ブロック内の非物理列と。 [\#5819](https://github.com/ClickHouse/ClickHouse/pull/5819) ([アントン-ポポフ](https://github.com/CurtizJ)) +- 優先度の低いクエリの潜在的な無限のスリープを修正しました。 [\#5842](https://github.com/ClickHouse/ClickHouse/pull/5842) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- ClickHouseがUTCではなくUCTとして既定のタイムゾーンを決定する方法を修正しました。 [\#5828](https://github.com/ClickHouse/ClickHouse/pull/5828) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- のを修正した。約の実行の分散DROP/ALTER/QUICK/最適化クラスターに関するお問い合わせフォロワレプリカの前にリーダーレプリカ. これで、リーダーレプリカで直接実行されます。 [\#5757](https://github.com/ClickHouse/ClickHouse/pull/5757) ([アレサピン](https://github.com/alesapin)) - 固定レースの条件を回避することが可能となり、一部のクエリーのような画面が、表示されないでquery\_logぐにシステムのフラッシュログを返します。 [\#5685](https://github.com/ClickHouse/ClickHouse/pull/5685) ([アントン-ポポフ](https://github.com/CurtizJ)) -- 定数引数のサポートがないことを追加 `evalMLModel` 機能。 [\#5820](https://github.com/ClickHouse/ClickHouse/pull/5820) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- 追加の欠落支援のための定数の引数 `evalMLModel` 機能。 [\#5820](https://github.com/ClickHouse/ClickHouse/pull/5820) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) -### クリックハウスリリース19.9.2.4,2019-06-24 {#clickhouse-release-19-9-2-4-2019-06-24} +### ClickHouse Release19.9.2.4,2019-06-24 {#clickhouse-release-19-9-2-4-2019-06-24} -#### 新しい機能 {#new-feature-8} +#### 新機能 {#new-feature-8} -- 凍結する部品についての印刷物情報 `system.parts` テーブル。 [\#5471](https://github.com/ClickHouse/ClickHouse/pull/5471) ([proller](https://github.com/proller)) -- 引数に設定されていない場合は、ttyでclickhouse-client startにクライアントパスワードを尋ねる [\#5092](https://github.com/ClickHouse/ClickHouse/pull/5092) ([proller](https://github.com/proller)) -- 実装 `dictGet` と `dictGetOrDefault` 小数タイプの関数。 [\#5394](https://github.com/ClickHouse/ClickHouse/pull/5394) ([Artem Zuikov](https://github.com/4ertus2)) +- 冷凍部品に関する情報を印刷する `system.parts` テーブル。 [\#5471](https://github.com/ClickHouse/ClickHouse/pull/5471) ([プロラー](https://github.com/proller)) +- お客様のパスワードをclickhouse-クライアント開始tty名が設定されていない場合には引数 [\#5092](https://github.com/ClickHouse/ClickHouse/pull/5092) ([プロラー](https://github.com/proller)) +- 実装 `dictGet` と `dictGetOrDefault` Decimal型の関数。 [\#5394](https://github.com/ClickHouse/ClickHouse/pull/5394) ([アルテム-ズイコフ](https://github.com/4ertus2)) #### 改善 {#improvement-8} -- Debian init:サービス停止タイムアウトの追加 [\#5522](https://github.com/ClickHouse/ClickHouse/pull/5522) ([proller](https://github.com/proller)) -- 疑わしいタイプのテーブルを作成するには、デフォルトで禁止されている設定を追加します [\#5448](https://github.com/ClickHouse/ClickHouse/pull/5448) ([Olga Khvostikova](https://github.com/stavrolia)) +- Debian init:サービス停止タイムアウトの追加 [\#5522](https://github.com/ClickHouse/ClickHouse/pull/5522) ([プロラー](https://github.com/proller)) +- 追加の設定により禁止されるデフォルトテーブルの作成と怪しいタイプLowCardinality [\#5448](https://github.com/ClickHouse/ClickHouse/pull/5448) ([Olga Khvostikova](https://github.com/stavrolia)) - 回帰機能を返却時の重量モデルとして用いられていない状態で機能 `evalMLMethod`. [\#5411](https://github.com/ClickHouse/ClickHouse/pull/5411) ([Quid37](https://github.com/Quid37)) -- 回帰方法の名前を変更して改善します。 [\#5492](https://github.com/ClickHouse/ClickHouse/pull/5492) ([Quid37](https://github.com/Quid37)) -- 明のインタフェースを文字列が揃. [\#5586](https://github.com/ClickHouse/ClickHouse/pull/5586) ([Danila Kutenin](https://github.com/danlark1)) +- 回帰方法の名前を変更し、改善します。 [\#5492](https://github.com/ClickHouse/ClickHouse/pull/5492) ([Quid37](https://github.com/Quid37)) +- 明のインタフェースを文字列が揃. [\#5586](https://github.com/ClickHouse/ClickHouse/pull/5586) ([ダニラ-クテニン](https://github.com/danlark1)) #### バグ修正 {#bug-fix-24} -- カフカの潜在的なデータ損失を修正 [\#5445](https://github.com/ClickHouse/ClickHouse/pull/5445) ([イワン](https://github.com/abyss7)) -- 潜在的な無限ループを修正 `PrettySpace` ゼロ列で呼び出されたときの形式 [\#5560](https://github.com/ClickHouse/ClickHouse/pull/5560) ([Olga Khvostikova](https://github.com/stavrolia)) -- 線形モデルのuint32オーバーフローバグを修正。 非constモデル引数のeval mlモデルを許可します。 [\#5516](https://github.com/ClickHouse/ClickHouse/pull/5516) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- `ALTER TABLE ... DROP INDEX IF EXISTS ...` なる例外を提供される場合指数が存在しない [\#5524](https://github.com/ClickHouse/ClickHouse/pull/5524) ([Gleb Novikov](https://github.com/NanoBjorn)) -- セグメントフォールトを修正 `bitmapHasAny` スカラーサブクエリ [\#5528](https://github.com/ClickHouse/ClickHouse/pull/5528) ([Zhichang Yu](https://github.com/yuzhichang)) -- 固定の場合にはエラー複製を接続プールなリトライを解決するホストでも、dnsキャッシュした。 [\#5534](https://github.com/ClickHouse/ClickHouse/pull/5534) ([alesapin](https://github.com/alesapin)) -- 固定 `ALTER ... MODIFY TTL` レプリケートされたマーゲットリーで [\#5539](https://github.com/ClickHouse/ClickHouse/pull/5539) ([アントン-ポポフ](https://github.com/CurtizJ)) -- マテリアライズ列を使用して分散テーブルに挿入を修正 [\#5429](https://github.com/ClickHouse/ClickHouse/pull/5429) ([Azat Khuzhin](https://github.com/azat)) -- Truncate Joinストレージの割り当ての問題を修正 [\#5437](https://github.com/ClickHouse/ClickHouse/pull/5437) ([Tcheason](https://github.com/TCeason)) -- に最近のバージョンのパッケージtzdata一部のファイルsymlinksます。 現在の仕組みを検出するデフォルトのタイムゾーンの故障が考えられ、間違った名前に一部のタイムゾーン. 少なくとも、提供されていれば、タイムゾーン名をtzの内容に強制します。 [\#5443](https://github.com/ClickHouse/ClickHouse/pull/5443) ([イワン](https://github.com/abyss7)) -- 一定の針が合計で少なくとも16kbの長さである場合、multivolnitsky searcherでいくつかの非常にまれなケースを修正します。 このアルゴリズムは、以前の結果を見逃したり上書きしたりして、誤った結果につながります `multiSearchAny`. [\#5588](https://github.com/ClickHouse/ClickHouse/pull/5588) ([Danila Kutenin](https://github.com/danlark1)) -- ExternalData要求の設定でClickHouseの設定を使用できない場合の問題を修正しました。 また、今のところ、設定 `date_time_input_format` と `low_cardinality_allow_in_native_format` 名前のあいまいさのために使用することはできません（外部データではテーブル形式と解釈でき、クエリでは設定にすることができます）。 [\#5455](https://github.com/ClickHouse/ClickHouse/pull/5455) ([Danila Kutenin](https://github.com/danlark1)) -- 部品は飼育係からそれらを落とすことなく、fsからのみ削除されたバグを修正。 [\#5520](https://github.com/ClickHouse/ClickHouse/pull/5520) ([alesapin](https://github.com/alesapin)) -- 削除デバッグログインからmysqlプロトコル [\#5478](https://github.com/ClickHouse/ClickHouse/pull/5478) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- カフカで潜在的なデータ損失を修正 [\#5445](https://github.com/ClickHouse/ClickHouse/pull/5445) ([イワン](https://github.com/abyss7)) +- で潜在的な無限ループを修正 `PrettySpace` 列がゼロで呼び出された場合の書式 [\#5560](https://github.com/ClickHouse/ClickHouse/pull/5560) ([Olga Khvostikova](https://github.com/stavrolia)) +- 線形モデルの固定UInt32オーバーフローバグ。 非constモデル引数のeval MLモデルを許可します。 [\#5516](https://github.com/ClickHouse/ClickHouse/pull/5516) ([ニコライ-コチェトフ](https://github.com/KochetovNicolai)) +- `ALTER TABLE ... DROP INDEX IF EXISTS ...` なる例外を提供される場合指数が存在しない [\#5524](https://github.com/ClickHouse/ClickHouse/pull/5524) ([グレブ-ノビコフ](https://github.com/NanoBjorn)) +- でsegfaultを修正 `bitmapHasAny` スカラーサブクエリで [\#5528](https://github.com/ClickHouse/ClickHouse/pull/5528) ([Zhichang Yu](https://github.com/yuzhichang)) +- 固定の場合にはエラー複製を接続プールなリトライを解決するホストでも、DNSキャッシュした。 [\#5534](https://github.com/ClickHouse/ClickHouse/pull/5534) ([アレサピン](https://github.com/alesapin)) +- 固定 `ALTER ... MODIFY TTL` 複製されたマージツリーです [\#5539](https://github.com/ClickHouse/ClickHouse/pull/5539) ([アントン-ポポフ](https://github.com/CurtizJ)) +- マテリアライズされた列で分散テーブルに挿入を修正 [\#5429](https://github.com/ClickHouse/ClickHouse/pull/5429) ([Azat Khuzhin](https://github.com/azat)) +- Fix悪allocが切り詰め加入の保管 [\#5437](https://github.com/ClickHouse/ClickHouse/pull/5437) ([Tシーズン](https://github.com/TCeason)) +- に最近のバージョンのパッケージtzdata一部のファイルsymlinksます。 現在の仕組みを検出するデフォルトのタイムゾーンの故障が考えられ、間違った名前に一部のタイムゾーン. これで、少なくともタイムゾーン名を指定した場合は、TZの内容に強制します。 [\#5443](https://github.com/ClickHouse/ClickHouse/pull/5443) ([イワン](https://github.com/abyss7)) +- 合計で一定の針が少なくとも16KBの長さである場合、MultiVolnitskyサーチャーでいくつかの非常にまれなケースを修正します。 このアルゴリズムは、以前の結果を見逃したり上書きしたりすることがあります。 `multiSearchAny`. [\#5588](https://github.com/ClickHouse/ClickHouse/pull/5588) ([ダニラ-クテニン](https://github.com/danlark1)) +- ExternalData要求の設定でClickHouse設定を使用できなかった場合の問題を修正しました。 また、今のところ、設定 `date_time_input_format` と `low_cardinality_allow_in_native_format` 名前のあいまいさのために使用することはできません（外部データではテーブル形式として解釈でき、クエリでは設定にすることができます）。 [\#5455](https://github.com/ClickHouse/ClickHouse/pull/5455) ([ダニラ-クテニン](https://github.com/danlark1)) +- パーツがZOOKEEPERから落とさずにFSからのみ取り除かれたバグを修正しました。 [\#5520](https://github.com/ClickHouse/ClickHouse/pull/5520) ([アレサピン](https://github.com/alesapin)) +- MySQLプロトコルか [\#5478](https://github.com/ClickHouse/ClickHouse/pull/5478) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) - DDLクエリ処理中にZNONODEをスキップ [\#5489](https://github.com/ClickHouse/ClickHouse/pull/5489) ([Azat Khuzhin](https://github.com/azat)) -- 修正ミックス `UNION ALL` 結果列の種類。 結果の列のデータ型および列の型が一致しない場合がありました。 [\#5503](https://github.com/ClickHouse/ClickHouse/pull/5503) ([Artem Zuikov](https://github.com/4ertus2)) -- 間違った整数の例外をスローする `dictGetT` 機能の代わりにクラッシュ。 [\#5446](https://github.com/ClickHouse/ClickHouse/pull/5446) ([Artem Zuikov](https://github.com/4ertus2)) -- ハッシュ化された辞書のための間違ったelement\_countとload\_factorを修正 `system.dictionaries` テーブル。 [\#5440](https://github.com/ClickHouse/ClickHouse/pull/5440) ([Azat Khuzhin](https://github.com/azat)) +- ミックスを修正 `UNION ALL` 結果列タイプ。 があった場合に矛盾したデータとカラムの種類によ列あります。 [\#5503](https://github.com/ClickHouse/ClickHouse/pull/5503) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- 間違った整数に対して例外をスローする `dictGetT` クラッシュの代わりに関数。 [\#5446](https://github.com/ClickHouse/ClickHouse/pull/5446) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- ハッシュ化されたディクショナリのelement\_countとload\_factorが間違っているのを修正 `system.dictionaries` テーブル。 [\#5440](https://github.com/ClickHouse/ClickHouse/pull/5440) ([Azat Khuzhin](https://github.com/azat)) -#### ビルド/テスト/パッケージの改善 {#buildtestingpackaging-improvement-9} +#### 造り/テスト/包装の改善 {#buildtestingpackaging-improvement-9} -- 固定ビルドなし `Brotli` HTTP圧縮のサポート (`ENABLE_BROTLI=OFF` cmake変数)。 [\#5521](https://github.com/ClickHouse/ClickHouse/pull/5521) ([Anton Yuzhaninov](https://github.com/citrin)) -- Roaringを含める。轟音/轟音としてh。h [\#5523](https://github.com/ClickHouse/ClickHouse/pull/5523) ([Orivej Desh](https://github.com/orivej)) -- ハイパースキャンでgcc9の警告を修正(\#lineディレクティブは悪です!) [\#5546](https://github.com/ClickHouse/ClickHouse/pull/5546) ([Danila Kutenin](https://github.com/danlark1)) -- Gcc-9でコンパイルする際のすべての警告を修正。 いくつかのcontribの問題を修正しました。 Gcc9氷を修正しbugzillaに提出. [\#5498](https://github.com/ClickHouse/ClickHouse/pull/5498) ([Danila Kutenin](https://github.com/danlark1)) -- Lldとのリンクを修正 [\#5477](https://github.com/ClickHouse/ClickHouse/pull/5477) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 辞書で使用されていない特殊化を削除する [\#5452](https://github.com/ClickHouse/ClickHouse/pull/5452) ([Artem Zuikov](https://github.com/4ertus2)) +- 固定ビルドなし `Brotli` HTTP圧縮サポート (`ENABLE_BROTLI=OFF` cmake変数）。 [\#5521](https://github.com/ClickHouse/ClickHouse/pull/5521) ([アントン-ユジャニノフ](https://github.com/citrin)) +- 轟音を含める。轟音/轟音としてh。h [\#5523](https://github.com/ClickHouse/ClickHouse/pull/5523) ([オリヴェイ-デシュ](https://github.com/orivej)) +- Hyperscanでgcc9の警告を修正しました（＃ラインディレクティブは悪です！) [\#5546](https://github.com/ClickHouse/ClickHouse/pull/5546) ([ダニラ-クテニン](https://github.com/danlark1)) +- 全ての場合の警告表gcc-9. いくつかのcontribの問題を修正します。 Gcc9ICEを修正し、bugzillaに提出してください。 [\#5498](https://github.com/ClickHouse/ClickHouse/pull/5498) ([ダニラ-クテニン](https://github.com/danlark1)) +- Lldとのリンクを修正 [\#5477](https://github.com/ClickHouse/ClickHouse/pull/5477) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 辞書で使用されていない特殊化を削除する [\#5452](https://github.com/ClickHouse/ClickHouse/pull/5452) ([アルテム-ズイコフ](https://github.com/4ertus2)) - 向上性能試験のためのフォーマットと構文解析表の異なる種類のファイル [\#5497](https://github.com/ClickHouse/ClickHouse/pull/5497) ([Olga Khvostikova](https://github.com/stavrolia)) -- 並列テスト実行のための修正 [\#5506](https://github.com/ClickHouse/ClickHouse/pull/5506) ([proller](https://github.com/proller)) -- Docker:clickhouse-testからconfigsを使う [\#5531](https://github.com/ClickHouse/ClickHouse/pull/5531) ([proller](https://github.com/proller)) -- FreeBSD用のコンパイルを修正 [\#5447](https://github.com/ClickHouse/ClickHouse/pull/5447) ([proller](https://github.com/proller)) -- Boostを1.70にアップグレード [\#5570](https://github.com/ClickHouse/ClickHouse/pull/5570) ([proller](https://github.com/proller)) -- サブモジュールとしてビルドclickhouseを修正 [\#5574](https://github.com/ClickHouse/ClickHouse/pull/5574) ([proller](https://github.com/proller)) -- JSONExtractパフォーマンステストの改善 [\#5444](https://github.com/ClickHouse/ClickHouse/pull/5444) ([Vitaly Baranov](https://github.com/vitlibar)) +- 並列テスト実行の修正 [\#5506](https://github.com/ClickHouse/ClickHouse/pull/5506) ([プロラー](https://github.com/proller)) +- Docker:clickhouse-testからconfigsを使用する [\#5531](https://github.com/ClickHouse/ClickHouse/pull/5531) ([プロラー](https://github.com/proller)) +- FreeBSD用のコンパイルを修正 [\#5447](https://github.com/ClickHouse/ClickHouse/pull/5447) ([プロラー](https://github.com/proller)) +- 更にブースト1.70 [\#5570](https://github.com/ClickHouse/ClickHouse/pull/5570) ([プロラー](https://github.com/proller)) +- サブモジュールとしてビルドclickhouseを修正 [\#5574](https://github.com/ClickHouse/ClickHouse/pull/5574) ([プロラー](https://github.com/proller)) +- JSONExtractパフォーマンステストの改善 [\#5444](https://github.com/ClickHouse/ClickHouse/pull/5444) ([ヴィタリー-バラノフ](https://github.com/vitlibar)) ## ClickHouseリリース19.8 {#clickhouse-release-19-8} -### ClickHouseリリース19.8.3.8、2019-06-11 {#clickhouse-release-19-8-3-8-2019-06-11} +### ClickHouse Release19.8.3.8,2019-06-11 {#clickhouse-release-19-8-3-8-2019-06-11} #### 新しい機能 {#new-features} -- JSONで動作する機能を追加しました [\#4686](https://github.com/ClickHouse/ClickHouse/pull/4686) ([hcz](https://github.com/hczhcz)) [\#5124](https://github.com/ClickHouse/ClickHouse/pull/5124). ([Vitaly Baranov](https://github.com/vitlibar)) -- 多くの言語に存在するbasename関数と同様の動作を持つ関数basenameを追加します (`os.path.basename` pythonでは, `basename` in PHP, etc…). Work with both an UNIX-like path or a Windows path. [\#5136](https://github.com/ClickHouse/ClickHouse/pull/5136) ([ギヨームタッセリー](https://github.com/YiuRULE)) +- JSONで動作する関数を追加しました [\#4686](https://github.com/ClickHouse/ClickHouse/pull/4686) ([hcz](https://github.com/hczhcz)) [\#5124](https://github.com/ClickHouse/ClickHouse/pull/5124). ([ヴィタリー-バラノフ](https://github.com/vitlibar)) +- 多くの言語に存在するbasename関数と同様の動作を持つ関数basenameを追加します (`os.path.basename` pythonでは, `basename` in PHP, etc…). Work with both an UNIX-like path or a Windows path. [\#5136](https://github.com/ClickHouse/ClickHouse/pull/5136) ([ギヨーム-タッセリー](https://github.com/YiuRULE)) - 追加 `LIMIT n, m BY` または `LIMIT m OFFSET n BY` LIMIT BY句にnのオフセットを設定する構文。 [\#5138](https://github.com/ClickHouse/ClickHouse/pull/5138) ([アントン-ポポフ](https://github.com/CurtizJ)) -- 追加された新しいデータ型 `SimpleAggregateFunction` を持つことができます。 `AggregatingMergeTree`. これは、次のような単純な関数でのみ使用できます `any`, `anyLast`, `sum`, `min`, `max`. [\#4629](https://github.com/ClickHouse/ClickHouse/pull/4629) ([Boris Granveaud](https://github.com/bgranvea)) -- 関数の非定数引数のサポートが追加されました `ngramDistance` [\#5198](https://github.com/ClickHouse/ClickHouse/pull/5198) ([Danila Kutenin](https://github.com/danlark1)) -- 機能追加 `skewPop`, `skewSamp`, `kurtPop` と `kurtSamp` シーケンスの歪度、標本の歪度、尖度、標本の尖度をそれぞれ計算します。 [\#5200](https://github.com/ClickHouse/ClickHouse/pull/5200) ([hcz](https://github.com/hczhcz)) -- 支援の名前変更操作のための `MaterializeView` ストレージ。 [\#5209](https://github.com/ClickHouse/ClickHouse/pull/5209) ([ギヨームタッセリー](https://github.com/YiuRULE)) -- 追加のサーバで接続するclickhouse mysqlを使用してクライアント [\#4715](https://github.com/ClickHouse/ClickHouse/pull/4715) ([ユーリーバラノフ](https://github.com/yurriy)) -- 追加 `toDecimal*OrZero` と `toDecimal*OrNull` 機能。 [\#5291](https://github.com/ClickHouse/ClickHouse/pull/5291) ([Artem Zuikov](https://github.com/4ertus2)) -- 機能のサポート十進のタイプ: `quantile`, `quantiles`, `median`, `quantileExactWeighted`, `quantilesExactWeighted`、medianExactWeighted。 [\#5304](https://github.com/ClickHouse/ClickHouse/pull/5304) ([Artem Zuikov](https://github.com/4ertus2)) -- 追加 `toValidUTF8` function, which replaces all invalid UTF-8 characters by replacement character � (U+FFFD). [\#5322](https://github.com/ClickHouse/ClickHouse/pull/5322) ([Danila Kutenin](https://github.com/danlark1)) -- 追加 `format` 機能。 引数にリストされた文字列を含む定数パターン(簡略化されたPython形式のパターン)の書式設定。 [\#5330](https://github.com/ClickHouse/ClickHouse/pull/5330) ([Danila Kutenin](https://github.com/danlark1)) -- 追加 `system.detached_parts` テーブルの情報を含む外部 `MergeTree` テーブル。 [\#5353](https://github.com/ClickHouse/ClickHouse/pull/5353) ([azerbaijan.kgm](https://github.com/akuzm)) -- 追加 `ngramSearch` 針と干し草の山の間の非対称差を計算する関数。 [\#5418](https://github.com/ClickHouse/ClickHouse/pull/5418)[\#5422](https://github.com/ClickHouse/ClickHouse/pull/5422) ([Danila Kutenin](https://github.com/danlark1)) -- 集約関数インタフェースを使用して、基本的な機械学習方法（確率線形回帰とロジスティック回帰）の実装。 モデルの重みを更新するための戦略が異なります（単純勾配降下、運動量法、nesterov法）。 また注文のサイズのミニバッチを支える。 [\#4943](https://github.com/ClickHouse/ClickHouse/pull/4943) ([Quid37](https://github.com/Quid37)) -- の実装 `geohashEncode` と `geohashDecode` 機能。 [\#5003](https://github.com/ClickHouse/ClickHouse/pull/5003) ([Vasily Nemkov](https://github.com/Enmk)) -- 集計関数の追加 `timeSeriesGroupSum` る累積の異なる時系列のサンプルのタイムスタンプなアライメントを実施します。 これは、二つのサンプルタイムスタンプ間の線形補間を使用して、一緒に時系列を合計します。 集計関数の追加 `timeSeriesGroupRateSum` これは、時系列のレートを計算し、その後一緒にレートを合計します。 [\#4542](https://github.com/ClickHouse/ClickHouse/pull/4542) ([楊関劉](https://github.com/LiuYangkuan)) -- 機能追加 `IPv4CIDRtoIPv4Range` と `IPv6CIDRtoIPv6Range` CIDRを使用してサブネット内のIPの下限と上限を計算する。 [\#5095](https://github.com/ClickHouse/ClickHouse/pull/5095) ([ギヨームタッセリー](https://github.com/YiuRULE)) -- 有効な設定でhttpを使用してクエリを送信するときにx-clickhouse-summaryヘッダーを追加します `send_progress_in_http_headers`. X-ClickHouse-Progressの通常の情報を返し、クエリに挿入された行数やバイト数などの追加情報を返します。 [\#5116](https://github.com/ClickHouse/ClickHouse/pull/5116) ([ギヨームタッセリー](https://github.com/YiuRULE)) +- 新しいデータ型を追加 `SimpleAggregateFunction` これにより、軽い集計を持つ列を `AggregatingMergeTree`. これは、次のような単純な関数でのみ使用できます `any`, `anyLast`, `sum`, `min`, `max`. [\#4629](https://github.com/ClickHouse/ClickHouse/pull/4629) ([ボリス-グランヴォー](https://github.com/bgranvea)) +- 関数内の非定数引数のサポートが追加されました `ngramDistance` [\#5198](https://github.com/ClickHouse/ClickHouse/pull/5198) ([ダニラ-クテニン](https://github.com/danlark1)) +- 追加された機能 `skewPop`, `skewSamp`, `kurtPop` と `kurtSamp` シーケンスの歪度、サンプルの歪度、尖度、およびサンプルの尖度をそれぞれ計算します。 [\#5200](https://github.com/ClickHouse/ClickHouse/pull/5200) ([hcz](https://github.com/hczhcz)) +- サポートの名前変更の操作 `MaterializeView` ストレージ。 [\#5209](https://github.com/ClickHouse/ClickHouse/pull/5209) ([ギヨーム-タッセリー](https://github.com/YiuRULE)) +- MySQL clientを使用してClickHouseに接続できるサーバーを追加しました。 [\#4715](https://github.com/ClickHouse/ClickHouse/pull/4715) ([ユーリー-バラノフ](https://github.com/yurriy)) +- 追加 `toDecimal*OrZero` と `toDecimal*OrNull` 機能。 [\#5291](https://github.com/ClickHouse/ClickHouse/pull/5291) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- 関数の小数点型のサポート: `quantile`, `quantiles`, `median`, `quantileExactWeighted`, `quantilesExactWeighted`,medianExactWeighted. [\#5304](https://github.com/ClickHouse/ClickHouse/pull/5304) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- 追加 `toValidUTF8` function, which replaces all invalid UTF-8 characters by replacement character � (U+FFFD). [\#5322](https://github.com/ClickHouse/ClickHouse/pull/5322) ([ダニラ-クテニン](https://github.com/danlark1)) +- 追加 `format` 機能。 引数にリストされた文字列で定数パターン（簡略化されたPython形式パターン）を書式設定します。 [\#5330](https://github.com/ClickHouse/ClickHouse/pull/5330) ([ダニラ-クテニン](https://github.com/danlark1)) +- 追加 `system.detached_parts` の分離された部分に関する情報を含む表 `MergeTree` テーブル [\#5353](https://github.com/ClickHouse/ClickHouse/pull/5353) ([akuzm](https://github.com/akuzm)) +- 追加 `ngramSearch` 針と干し草の山の非対称差を計算する関数。 [\#5418](https://github.com/ClickHouse/ClickHouse/pull/5418)[\#5422](https://github.com/ClickHouse/ClickHouse/pull/5422) ([ダニラ-クテニン](https://github.com/danlark1)) +- 集計関数インタフェースを用いた基本的な機械学習法（確率的線形回帰とロジスティック回帰）の実装。 モデルの重みを更新するための異なる戦略（単純勾配降下法、運動量の方法、ネステロフ法）を持っています。 にも対応-ミニバッチのカスタムサイズです。 [\#4943](https://github.com/ClickHouse/ClickHouse/pull/4943) ([Quid37](https://github.com/Quid37)) +- の実装 `geohashEncode` と `geohashDecode` 機能。 [\#5003](https://github.com/ClickHouse/ClickHouse/pull/5003) ([ヴァシーリー-ネムコフ](https://github.com/Enmk)) +- 集計関数を追加 `timeSeriesGroupSum` る累積の異なる時系列のサンプルのタイムスタンプなアライメントを実施します。 これは、二つのサンプルタイムスタンプ間の線形補間を使用して、一緒に時系列を合計します。 集計関数を追加 `timeSeriesGroupRateSum`、時系列のレートを計算し、その後、一緒にレートを合計します。 [\#4542](https://github.com/ClickHouse/ClickHouse/pull/4542) ([ヤンクアン-リュウ](https://github.com/LiuYangkuan)) +- 追加された機能 `IPv4CIDRtoIPv4Range` と `IPv6CIDRtoIPv6Range` CIDRを使用して、サブネット内のIPの下限と上限を計算します。 [\#5095](https://github.com/ClickHouse/ClickHouse/pull/5095) ([ギヨーム-タッセリー](https://github.com/YiuRULE)) +- 有効な設定でHTTPを使用してクエリを送信するときに、X-ClickHouse-Summaryヘッダーを追加します `send_progress_in_http_headers`. X-ClickHouse-Progressの通常の情報と、クエリに挿入された行数やバイト数などの追加情報を返します。 [\#5116](https://github.com/ClickHouse/ClickHouse/pull/5116) ([ギヨーム-タッセリー](https://github.com/YiuRULE)) #### 改善 {#improvements} -- 追加 `max_parts_in_total` パーティションキー\#5166の安全でない指定を防ぐテーブルのMergeTreeファミリーの設定(デフォルト:100 000)。 [\#5171](https://github.com/ClickHouse/ClickHouse/pull/5171) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- `clickhouse-obfuscator`:derive種のため、個々のカラムを組み合わせ種の利用が重要であると考えられとカラム名、カラム位置にします。 ことを目的として変換するデータセットに複数の関連するテーブル、テーブルはJOINableに設定します。 [\#5178](https://github.com/ClickHouse/ClickHouse/pull/5178) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 機能追加 `JSONExtractRaw`, `JSONExtractKeyAndValues`. 関数の名前を変更 `jsonExtract` に `JSONExtract`. 何かがうまくいかない場合、これらの関数は対応する値を返します。 `NULL`. 変更された機能 `JSONExtract` 今度は、最後のパラメータから戻り値の型を取得し、nullableを注入しません。 AVX2命令が利用できない場合にRapidJSONにフォールバックを実装しました。 新しいバージョンに更新Simdjsonライブラリ。 [\#5235](https://github.com/ClickHouse/ClickHouse/pull/5235) ([Vitaly Baranov](https://github.com/vitlibar)) -- さて `if` と `multiIf` 機能は条件に頼りません `Nullable` しかし、sqlの互換性のためにブランチに依存しています。 [\#5238](https://github.com/ClickHouse/ClickHouse/pull/5238) ([建呉](https://github.com/janplus)) -- `In` 述語が生成されます `Null` 結果から `Null` のような入力 `Equal` 機能。 [\#5152](https://github.com/ClickHouse/ClickHouse/pull/5152) ([建呉](https://github.com/janplus)) -- Kafkaからの行数ごとの時間制限（flush\_interval/poll\_timeout）をチェックします。 この読みからのカフカの消費者をより頻繁にチェックの時間制限のトップレベルの流れ [\#5249](https://github.com/ClickHouse/ClickHouse/pull/5249) ([イワン](https://github.com/abyss7)) -- バンドルsaslとリンクrdkafka. sasl scram認証を使用できるようにする必要があります [\#5253](https://github.com/ClickHouse/ClickHouse/pull/5253) ([イワン](https://github.com/abyss7)) -- すべてのジョインのrowreflistのバッチバージョン。 [\#5267](https://github.com/ClickHouse/ClickHouse/pull/5267) ([Artem Zuikov](https://github.com/4ertus2)) -- clickhouse-サーバ:より有益なエラーメッセージを聞きます. [\#5268](https://github.com/ClickHouse/ClickHouse/pull/5268) ([proller](https://github.com/proller)) -- Clickhouseのサポート辞書-機能のための複写機 `` [\#5270](https://github.com/ClickHouse/ClickHouse/pull/5270) ([proller](https://github.com/proller)) -- 新しい設定を追加 `kafka_commit_every_batch` カフカの政策を規制する。 - ることができる設定のコミットモード：バッチのメッセージの取り扱い、後のブロック全体に書きます。 このトレードオフの関係を失うメッセージやみ表示することを目的としていま倍もいる。 [\#5308](https://github.com/ClickHouse/ClickHouse/pull/5308) ([イワン](https://github.com/abyss7)) -- 作る `windowFunnel` 他の符号なし整数型をサポート。 [\#5320](https://github.com/ClickHouse/ClickHouse/pull/5320) ([sundyli](https://github.com/sundy-li)) -- 仮想列をシャドウする `_table` マージエンジンで。 [\#5325](https://github.com/ClickHouse/ClickHouse/pull/5325) ([イワン](https://github.com/abyss7)) -- 作る `sequenceMatch` 集計関数は、他の符号なし整数型をサポート [\#5339](https://github.com/ClickHouse/ClickHouse/pull/5339) ([sundyli](https://github.com/sundy-li)) -- より良いエラーメッセージの場合はチェックサムミスマッチは一によるものと考えられるハードウェアです。 [\#5355](https://github.com/ClickHouse/ClickHouse/pull/5355) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- 追加 `max_parts_in_total` パーティションキー\#5166の安全でない指定を防ぐMergeTreeファミリのテーブル(デフォルト:100 000)の設定。 [\#5171](https://github.com/ClickHouse/ClickHouse/pull/5171) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- `clickhouse-obfuscator`：初期シードと列の位置ではなく列名を組み合わせて、個々の列のシードを導出します。 ことを目的として変換するデータセットに複数の関連するテーブル、テーブルはJOINableに設定します。 [\#5178](https://github.com/ClickHouse/ClickHouse/pull/5178) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 追加された機能 `JSONExtractRaw`, `JSONExtractKeyAndValues`. 名前変更された関数 `jsonExtract` に `JSONExtract`. 何かがうまくいかないとき、これらの関数は対応する値を返します。 `NULL`. 変更された関数 `JSONExtract` これで、最後のパラメータから戻り値の型を取得し、nullablesを注入しません。 AVX2命令が利用できない場合にRapidJSONへのフォールバックを実装しました。 Simdjsonライブラリを新しいバージョンに更新しました。 [\#5235](https://github.com/ClickHouse/ClickHouse/pull/5235) ([ヴィタリー-バラノフ](https://github.com/vitlibar)) +- さて `if` と `multiIf` 関数は条件に依存しません `Nullable` しかし、sqlの互換性のためにブランチに依存しています。 [\#5238](https://github.com/ClickHouse/ClickHouse/pull/5238) ([ジャン-ウー](https://github.com/janplus)) +- `In` 述語が生成されます `Null` 結果から `Null` のような入力 `Equal` 機能。 [\#5152](https://github.com/ClickHouse/ClickHouse/pull/5152) ([ジャン-ウー](https://github.com/janplus)) +- カフカからの行数(flush\_interval/poll\_timeout)ごとに制限時間をチェックします。 これにより、カフカの消費者からの読み取りをより頻繁に中断し、トップレベルのストリームの時間制限を確認することができます [\#5249](https://github.com/ClickHouse/ClickHouse/pull/5249) ([イワン](https://github.com/abyss7)) +- バンドルされたSASLとrdkafkaをリンク. SASLスクラム認証の使用を許可する必要があります [\#5253](https://github.com/ClickHouse/ClickHouse/pull/5253) ([イワン](https://github.com/abyss7)) +- すべての結合のRowRefListのバッチバージョン。 [\#5267](https://github.com/ClickHouse/ClickHouse/pull/5267) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- clickhouse-server:より有益なlistenエラーメッセージ。 [\#5268](https://github.com/ClickHouse/ClickHouse/pull/5268) ([プロラー](https://github.com/proller)) +- 機能のためのclickhouse-copierのサポート辞書 `` [\#5270](https://github.com/ClickHouse/ClickHouse/pull/5270) ([プロラー](https://github.com/proller)) +- 新しい設定を追加 `kafka_commit_every_batch` カフカコミット政策を規制する。 + メッセージのすべてのバッチが処理された後、またはブロック全体がストレージに書き込まれた後です。 このトレードオフの関係を失うメッセージやみ表示することを目的としていま倍もいる。 [\#5308](https://github.com/ClickHouse/ClickHouse/pull/5308) ([イワン](https://github.com/abyss7)) +- 作る `windowFunnel` 他の符号なし整数型をサポートする。 [\#5320](https://github.com/ClickHouse/ClickHouse/pull/5320) ([スンディリ](https://github.com/sundy-li)) +- 仮想列のシャドウを許可する `_table` マージエンジンで。 [\#5325](https://github.com/ClickHouse/ClickHouse/pull/5325) ([イワン](https://github.com/abyss7)) +- 作る `sequenceMatch` 集計機能の支援その他の符号なし整数型 [\#5339](https://github.com/ClickHouse/ClickHouse/pull/5339) ([スンディリ](https://github.com/sundy-li)) +- より良いエラーメッセージの場合はチェックサムミスマッチは一によるものと考えられるハードウェアです。 [\#5355](https://github.com/ClickHouse/ClickHouse/pull/5355) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) - チェックすると配下のテーブル支援のためのサンプリング `StorageMerge` [\#5366](https://github.com/ClickHouse/ClickHouse/pull/5366) ([イワン](https://github.com/abyss7)) - Сlose MySQL connections after their usage in external dictionaries. It is related to issue \#893. [\#5395](https://github.com/ClickHouse/ClickHouse/pull/5395) ([Clément Rodriguez](https://github.com/clemrodriguez)) -- MySQLワイヤプロトコルの改善。 フォーマットの名前をMySQLWireに変更しました。 RAIIをRSA\_freeを呼び出すために使用します。 コンテキス [\#5419](https://github.com/ClickHouse/ClickHouse/pull/5419) ([ユーリーバラノフ](https://github.com/yurriy)) -- clickhouse-client: allow to run with unaccessable history file (read-only, no disk space, file is directory, …). [\#5431](https://github.com/ClickHouse/ClickHouse/pull/5431) ([proller](https://github.com/proller)) -- 分散テーブルへの非同期挿入のクエリ設定を考慮します。 [\#4936](https://github.com/ClickHouse/ClickHouse/pull/4936) ([Tcheason](https://github.com/TCeason)) -- 関数の名前を変更 `leastSqr` に `simpleLinearRegression`, `LinearRegression` に `linearRegression`, `LogisticRegression` に `logisticRegression`. [\#5391](https://github.com/ClickHouse/ClickHouse/pull/5391) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- MySQLワイヤプロトコルの改善。 形式の名前をMySQLWireに変更しました。 RAIIを使ってRSA\_freeを呼び出す。 コンテキ [\#5419](https://github.com/ClickHouse/ClickHouse/pull/5419) ([ユーリー-バラノフ](https://github.com/yurriy)) +- clickhouse-client: allow to run with unaccessable history file (read-only, no disk space, file is directory, …). [\#5431](https://github.com/ClickHouse/ClickHouse/pull/5431) ([プロラー](https://github.com/proller)) +- 分散テーブルへの非同期挿入のクエリ設定を尊重します。 [\#4936](https://github.com/ClickHouse/ClickHouse/pull/4936) ([Tシーズン](https://github.com/TCeason)) +- 名前変更された関数 `leastSqr` に `simpleLinearRegression`, `LinearRegression` に `linearRegression`, `LogisticRegression` に `logisticRegression`. [\#5391](https://github.com/ClickHouse/ClickHouse/pull/5391) ([ニコライ-コチェトフ](https://github.com/KochetovNicolai)) -#### 性能の改善 {#performance-improvements} +#### 性能の向上 {#performance-improvements} -- ALTER MODIFYクエリの非レプリケートされたMergeTreeテーブルの一部の並列処理。 [\#4639](https://github.com/ClickHouse/ClickHouse/pull/4639) ([Ivan Kush](https://github.com/IvanKush)) -- 正規表現の抽出における最適化。 [\#5193](https://github.com/ClickHouse/ClickHouse/pull/5193) [\#5191](https://github.com/ClickHouse/ClickHouse/pull/5191) ([Danila Kutenin](https://github.com/danlark1)) -- 結果を結合するために右結合キー列を追加しないでください。 [\#5260](https://github.com/ClickHouse/ClickHouse/pull/5260) ([Artem Zuikov](https://github.com/4ertus2)) -- 最初の空の応答の後にkafkaバッファをフリーズします。 それは多数のinvokationsをの避けます `ReadBuffer::next()` いくつかの行解析ストリームの空の結果。 [\#5283](https://github.com/ClickHouse/ClickHouse/pull/5283) ([イワン](https://github.com/abyss7)) -- `concat` 複数の引数の関数の最適化。 [\#5357](https://github.com/ClickHouse/ClickHouse/pull/5357) ([Danila Kutenin](https://github.com/danlark1)) -- Query optimisation. Allow push down IN statement while rewriting commа/cross join into inner one. [\#5396](https://github.com/ClickHouse/ClickHouse/pull/5396) ([Artem Zuikov](https://github.com/4ertus2)) -- より速い解凍を持つように、lz4実装を参照してアップグレードします。 [\#5070](https://github.com/ClickHouse/ClickHouse/pull/5070) ([Danila Kutenin](https://github.com/danlark1)) -- MSD基数ソート（kxsortに基づく）、および部分ソートを実装しました。 [\#5129](https://github.com/ClickHouse/ClickHouse/pull/5129) ([Evgenii Pravda](https://github.com/kvinty)) +- ALTER MODIFY queryで、複製されていないMergeTreeテーブルの一部の処理を並列化します。 [\#4639](https://github.com/ClickHouse/ClickHouse/pull/4639) ([イヴァン-クシュ](https://github.com/IvanKush)) +- 正規表現の抽出における最適化。 [\#5193](https://github.com/ClickHouse/ClickHouse/pull/5193) [\#5191](https://github.com/ClickHouse/ClickHouse/pull/5191) ([ダニラ-クテニン](https://github.com/danlark1)) +- Join onセクションでのみ使用されている場合は、右結合キー列をjoin resultに追加しないでください。 [\#5260](https://github.com/ClickHouse/ClickHouse/pull/5260) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- 最初の空の応答の後、Kafkaバッファをフリーズします。 それは多数の呼出しをの避けます `ReadBuffer::next()` いくつかの行解析ストリームの空の結果の場合。 [\#5283](https://github.com/ClickHouse/ClickHouse/pull/5283) ([イワン](https://github.com/abyss7)) +- `concat` 複数の引数に対する関数の最適化。 [\#5357](https://github.com/ClickHouse/ClickHouse/pull/5357) ([ダニラ-クテニン](https://github.com/danlark1)) +- Query optimisation. Allow push down IN statement while rewriting commа/cross join into inner one. [\#5396](https://github.com/ClickHouse/ClickHouse/pull/5396) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- LZ4の実装をリファレンスワンでアップグレードして、解凍を高速化します。 [\#5070](https://github.com/ClickHouse/ClickHouse/pull/5070) ([ダニラ-クテニン](https://github.com/danlark1)) +- MSD基数ソート(kxsortに基づく)、および部分ソートを実装しました。 [\#5129](https://github.com/ClickHouse/ClickHouse/pull/5129) ([エフゲニー-プラウダ](https://github.com/kvinty)) #### バグ修正 {#bug-fixes} -- プッシュが結合で列を必要と修正 [\#5192](https://github.com/ClickHouse/ClickHouse/pull/5192) ([冬張](https://github.com/zhang2014)) -- バグを修正,clickhouseはsystemdによって実行されると,コマンド `sudo service clickhouse-server forcerestart` 期待通りに動作しませんでした。 [\#5204](https://github.com/ClickHouse/ClickHouse/pull/5204) ([proller](https://github.com/proller)) -- （9009ポート上のサーバー間のhttpサーバーは常に偶数エラーで、コード200を返しました）datapartsexchangeのhttpエラーコードを修正します。 [\#5216](https://github.com/ClickHouse/ClickHouse/pull/5216) ([proller](https://github.com/proller)) -- MAX\_SMALL\_STRING\_SIZEより長い文字列のSimpleAggregateFunctionを修正しました [\#5311](https://github.com/ClickHouse/ClickHouse/pull/5311) ([Azat Khuzhin](https://github.com/azat)) -- のためのエラーを修正 `Decimal` に `Nullable(Decimal)` 変換で。 支援その他数を小数点の変換を含む異なる）を採用。 [\#5350](https://github.com/ClickHouse/ClickHouse/pull/5350) ([Artem Zuikov](https://github.com/4ertus2)) -- 間違った計算につながるsimdjsonライブラリ内の固定fpu clobbering `uniqHLL` と `uniqCombined` 次のような集計関数と数学関数 `log`. [\#5354](https://github.com/ClickHouse/ClickHouse/pull/5354) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- JSON関数でのconst/nonconstの混合ケースの処理を修正しました。 [\#5435](https://github.com/ClickHouse/ClickHouse/pull/5435) ([Vitaly Baranov](https://github.com/vitlibar)) -- 修正 `retention` 機能。 これで、データの行を満たすすべての条件がデータの状態に追加されます。 [\#5119](https://github.com/ClickHouse/ClickHouse/pull/5119) ([小路](https://github.com/nicelulu)) -- 結果のタイプを修正する `quantileExact` 小数で。 [\#5304](https://github.com/ClickHouse/ClickHouse/pull/5304) ([Artem Zuikov](https://github.com/4ertus2)) +- 結合でプッシュが必要な列を修正 [\#5192](https://github.com/ClickHouse/ClickHouse/pull/5192) ([冬張](https://github.com/zhang2014)) +- ClickHouseがsystemdによって実行されているときに、コマンドを修正しました `sudo service clickhouse-server forcerestart` 期待通りに動作していませんでした。 [\#5204](https://github.com/ClickHouse/ClickHouse/pull/5204) ([プロラー](https://github.com/proller)) +- DataPartsExchangeのhttpエラーコードを修正しました（9009ポートのinterserver httpサーバーは常にエラーであってもコード200を返しました）。 [\#5216](https://github.com/ClickHouse/ClickHouse/pull/5216) ([プロラー](https://github.com/proller)) +- MAX\_SMALL\_STRING\_SIZEより長い文字列のSimpleAggregateFunctionを修正 [\#5311](https://github.com/ClickHouse/ClickHouse/pull/5311) ([Azat Khuzhin](https://github.com/azat)) +- エラーを修正 `Decimal` に `Nullable(Decimal)` inでの変換。 （異なるスケールを含む）小数点変換への他の小数をサポート。 [\#5350](https://github.com/ClickHouse/ClickHouse/pull/5350) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- 誤った計算につながるsimdjsonライブラリ内の固定FPU clobbering `uniqHLL` と `uniqCombined` 集計関数と数学関数のような `log`. [\#5354](https://github.com/ClickHouse/ClickHouse/pull/5354) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- JSON関数でのconst/nonconstの混合ケースの処理を修正しました。 [\#5435](https://github.com/ClickHouse/ClickHouse/pull/5435) ([ヴィタリー-バラノフ](https://github.com/vitlibar)) +- 修正 `retention` 機能。 これで、データ行で満たすすべての条件がデータ状態に追加されます。 [\#5119](https://github.com/ClickHouse/ClickHouse/pull/5119) ([小路](https://github.com/nicelulu)) +- 結果の種類を修正 `quantileExact` 小数で。 [\#5304](https://github.com/ClickHouse/ClickHouse/pull/5304) ([アルテム-ズイコフ](https://github.com/4ertus2)) #### 文書 {#documentation} -- 以下のための文書を翻訳 `CollapsingMergeTree` 中国語に。 [\#5168](https://github.com/ClickHouse/ClickHouse/pull/5168) ([张风啸](https://github.com/AlexZFX)) -- テーブルエンジンに関する文書を中国語に翻訳します。 +- ドキュメントの翻訳 `CollapsingMergeTree` 中国語に。 [\#5168](https://github.com/ClickHouse/ClickHouse/pull/5168) ([张风啸](https://github.com/AlexZFX)) +- 翻訳一部のアンダーグラウンドテーブルエンジン。 [\#5134](https://github.com/ClickHouse/ClickHouse/pull/5134) [\#5328](https://github.com/ClickHouse/ClickHouse/pull/5328) ([決してリー](https://github.com/neverlee)) #### ビルド/テスト/パッケージの改善 {#buildtestingpackaging-improvements} -- ﾂづﾂつ､ﾂつｼﾂつｲﾂ猟.用ﾂつｭﾂつｾﾂつｳﾂつ"ﾂ。[\#5139](https://github.com/ClickHouse/ClickHouse/pull/5139) [\#5143](https://github.com/ClickHouse/ClickHouse/pull/5143) [\#5393](https://github.com/ClickHouse/ClickHouse/pull/5393) ([イワン](https://github.com/abyss7)) -- 移動性能試験の個別のディレクトリが便利です。 [\#5158](https://github.com/ClickHouse/ClickHouse/pull/5158) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- パフォーマンステストの修正 [\#5255](https://github.com/ClickHouse/ClickHouse/pull/5255) ([alesapin](https://github.com/alesapin)) -- 追加ツールをチェックサムを計算によるビット切り替えデバッグハードウェアます。 [\#5334](https://github.com/ClickHouse/ClickHouse/pull/5334) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- ありそうな使用後の無料を示すいくつかのサニタイザーレポートを修正します。[\#5139](https://github.com/ClickHouse/ClickHouse/pull/5139) [\#5143](https://github.com/ClickHouse/ClickHouse/pull/5143) [\#5393](https://github.com/ClickHouse/ClickHouse/pull/5393) ([イワン](https://github.com/abyss7)) +- 移動性能試験の個別のディレクトリが便利です。 [\#5158](https://github.com/ClickHouse/ClickHouse/pull/5158) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 修正誤性能試験までを実施。 [\#5255](https://github.com/ClickHouse/ClickHouse/pull/5255) ([アレサピン](https://github.com/alesapin)) +- 追加ツールをチェックサムを計算によるビット切り替えデバッグハードウェアます。 [\#5334](https://github.com/ClickHouse/ClickHouse/pull/5334) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) - くランナーのスクリプトをより使用できます。 [\#5340](https://github.com/ClickHouse/ClickHouse/pull/5340)[\#5360](https://github.com/ClickHouse/ClickHouse/pull/5360) ([フィリモノフ](https://github.com/filimonov)) -- パフォーマンステス [\#5408](https://github.com/ClickHouse/ClickHouse/pull/5408) ([alesapin](https://github.com/alesapin)) -- パフォーマンステストでのクエリの作成、入力、および削除での置換を行う機能を追加 [\#5367](https://github.com/ClickHouse/ClickHouse/pull/5367) ([Olga Khvostikova](https://github.com/stavrolia)) +- 追加小さな命令をどう書き込み能力を発揮する。 [\#5408](https://github.com/ClickHouse/ClickHouse/pull/5408) ([アレサピン](https://github.com/alesapin)) +- パフォーマンステストでのクエリの作成、入力、削除で置換を行う機能を追加 [\#5367](https://github.com/ClickHouse/ClickHouse/pull/5367) ([Olga Khvostikova](https://github.com/stavrolia)) -## クリックハウスリリース19.7 {#clickhouse-release-19-7} +## ClickHouseリリース19.7 {#clickhouse-release-19-7} ### ClickHouseリリース19.7.5.29,2019-07-05 {#clickhouse-release-19-7-5-29-2019-07-05} @@ -1333,77 +1333,77 @@ toc_title: '2019' - 固定能の回帰一部のクエリを処理するクラウドの場合。 [\#5192](https://github.com/ClickHouse/ClickHouse/pull/5192) ([冬張](https://github.com/zhang2014)) -### クリックハウスリリース19.7.5.27,2019-06-09 {#clickhouse-release-19-7-5-27-2019-06-09} +### ClickHouseリリース19.7.5.27,2019-06-09 {#clickhouse-release-19-7-5-27-2019-06-09} #### 新しい機能 {#new-features-1} -- ビットマップ関連の機能を追加 `bitmapHasAny` と `bitmapHasAll` に類似した `hasAny` と `hasAll` 配列の関数。 [\#5279](https://github.com/ClickHouse/ClickHouse/pull/5279) ([Catalonia\_comarques.Kgm](https://github.com/svladykin)) +- ビットマップ関連の機能を追加 `bitmapHasAny` と `bitmapHasAll` 類似する `hasAny` と `hasAll` 配列の関数。 [\#5279](https://github.com/ClickHouse/ClickHouse/pull/5279) ([セルジ-ウラジキン](https://github.com/svladykin)) #### バグ修正 {#bug-fixes-1} -- Segfaultをオンにする `minmax` Null値を持つインデックス。 [\#5246](https://github.com/ClickHouse/ClickHouse/pull/5246) ([Nikita Vasilev](https://github.com/nikvas0)) -- LIMITのすべての入力列を必要な出力としてマークします。 それは修正します ‘Not found column’ いくつかの分散クエリのエラー。 [\#5407](https://github.com/ClickHouse/ClickHouse/pull/5407) ([Constantin S.Pan](https://github.com/kvap)) -- 修正 “Column ‘0’ already exists” エラーで `SELECT .. PREWHERE` デフォルトの列 [\#5397](https://github.com/ClickHouse/ClickHouse/pull/5397) ([proller](https://github.com/proller)) -- 修正 `ALTER MODIFY TTL` 上のクエリ `ReplicatedMergeTree`. [\#5539](https://github.com/ClickHouse/ClickHouse/pull/5539/commits) ([アントン-ポポフ](https://github.com/CurtizJ)) -- Kafkaの消費者が起動に失敗したときにサーバをクラッシュさせないでください。 [\#5285](https://github.com/ClickHouse/ClickHouse/pull/5285) ([イワン](https://github.com/abyss7)) -- 固定ビットマップ機能を誤った結果です。 [\#5359](https://github.com/ClickHouse/ClickHouse/pull/5359) ([アンディヤング](https://github.com/andyyzh)) -- ハッシュ化された辞書のための修正element\_count（重複を含めないでください) [\#5440](https://github.com/ClickHouse/ClickHouse/pull/5440) ([Azat Khuzhin](https://github.com/azat)) -- タイムゾーンの名前として環境変数tzの内容を使用します。 すでに正しく検出デフォルトのタイムゾーンもあります。[\#5443](https://github.com/ClickHouse/ClickHouse/pull/5443) ([イワン](https://github.com/abyss7)) -- 整数を変換しようとしないでください `dictGetT` それが正しく動作しないため、機能。 代わりに例外をスローします。 [\#5446](https://github.com/ClickHouse/ClickHouse/pull/5446) ([Artem Zuikov](https://github.com/4ertus2)) +- Segfaultをオンに修正 `minmax` Null値のインデックス。 [\#5246](https://github.com/ClickHouse/ClickHouse/pull/5246) ([ニキータ-ヴァシレフ](https://github.com/nikvas0)) +- すべての入力列を必要な出力に応じてLIMITにマークします。 それは修正します ‘Not found column’ 一部の分散クエリでエラー。 [\#5407](https://github.com/ClickHouse/ClickHouse/pull/5407) ([コンスタンティン-S-パン](https://github.com/kvap)) +- 修正 “Column ‘0’ already exists” エラー `SELECT .. PREWHERE` デフォルトの列 [\#5397](https://github.com/ClickHouse/ClickHouse/pull/5397) ([プロラー](https://github.com/proller)) +- 修正 `ALTER MODIFY TTL` クエリオン `ReplicatedMergeTree`. [\#5539](https://github.com/ClickHouse/ClickHouse/pull/5539/commits) ([アントン-ポポフ](https://github.com/CurtizJ)) +- なクラッシュサーバの場合カフカの消費者が開始されます。 [\#5285](https://github.com/ClickHouse/ClickHouse/pull/5285) ([イワン](https://github.com/abyss7)) +- 固定ビットマップ機能を誤った結果です。 [\#5359](https://github.com/ClickHouse/ClickHouse/pull/5359) ([アンディ-ヤン](https://github.com/andyyzh)) +- ハッシュ化された辞書のelement\_countを修正（重複を含まない) [\#5440](https://github.com/ClickHouse/ClickHouse/pull/5440) ([Azat Khuzhin](https://github.com/azat)) +- 環境変数TZの内容をタイムゾーンの名前として使用します。 すでに正しく検出デフォルトのタイムゾーンもあります。[\#5443](https://github.com/ClickHouse/ClickHouse/pull/5443) ([イワン](https://github.com/abyss7)) +- 整数を変換しようとしないでください `dictGetT` それは正しく動作しないため、機能します。 代わりに例外をスローします。 [\#5446](https://github.com/ClickHouse/ClickHouse/pull/5446) ([アルテム-ズイコフ](https://github.com/4ertus2)) - ExternalData HTTP要求の設定を修正しました。 [\#5455](https://github.com/ClickHouse/ClickHouse/pull/5455) ([ダニラ クテニン](https://github.com/danlark1)) -- 部品は飼育係からそれらを落とすことなく、fsからのみ削除されたバグを修正。 [\#5520](https://github.com/ClickHouse/ClickHouse/pull/5520) ([alesapin](https://github.com/alesapin)) -- セグメンテーショ `bitmapHasAny` 機能。 [\#5528](https://github.com/ClickHouse/ClickHouse/pull/5528) ([Zhichang Yu](https://github.com/yuzhichang)) -- 固定の場合にはエラー複製を接続プールなリトライを解決するホストでも、dnsキャッシュした。 [\#5534](https://github.com/ClickHouse/ClickHouse/pull/5534) ([alesapin](https://github.com/alesapin)) -- 固定 `DROP INDEX IF EXISTS` クエリ。 さて `ALTER TABLE ... DROP INDEX IF EXISTS ...` 指定されたインデックスが存在しない場合、queryは例外を発生させません。 [\#5524](https://github.com/ClickHouse/ClickHouse/pull/5524) ([Gleb Novikov](https://github.com/NanoBjorn)) -- Union all supertype列を修正しました。 結果の列のデータ型および列の型が一致しない場合がありました。 [\#5503](https://github.com/ClickHouse/ClickHouse/pull/5503) ([Artem Zuikov](https://github.com/4ertus2)) -- DDLクエリの処理中にZNONODEをスキップします。 別のノードがタスクキューのznodeを削除する前に、 - それを処理しませんでしたが、すでに子のリストを取得していますが、DDLWorkerスレッドを終了します。 [\#5489](https://github.com/ClickHouse/ClickHouse/pull/5489) ([Azat Khuzhin](https://github.com/azat)) -- マテリアライズドカラムを使用して分散（）テーブルに挿入を修正。 [\#5429](https://github.com/ClickHouse/ClickHouse/pull/5429) ([Azat Khuzhin](https://github.com/azat)) +- パーツがZOOKEEPERから落とさずにFSからのみ取り除かれたバグを修正しました。 [\#5520](https://github.com/ClickHouse/ClickHouse/pull/5520) ([アレサピン](https://github.com/alesapin)) +- 固定の区分断層に `bitmapHasAny` 機能。 [\#5528](https://github.com/ClickHouse/ClickHouse/pull/5528) ([Zhichang Yu](https://github.com/yuzhichang)) +- 固定の場合にはエラー複製を接続プールなリトライを解決するホストでも、DNSキャッシュした。 [\#5534](https://github.com/ClickHouse/ClickHouse/pull/5534) ([アレサピン](https://github.com/alesapin)) +- 固定 `DROP INDEX IF EXISTS` クエリ。 さて `ALTER TABLE ... DROP INDEX IF EXISTS ...` 指定されたインデックスが存在しない場合、queryは例外を発生させません。 [\#5524](https://github.com/ClickHouse/ClickHouse/pull/5524) ([グレブ-ノビコフ](https://github.com/NanoBjorn)) +- 修正unionすべてのスーパータイプ列。 があった場合に矛盾したデータとカラムの種類によ列あります。 [\#5503](https://github.com/ClickHouse/ClickHouse/pull/5503) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- DDLクエリ処理中にZNONODEをスキップします。 前に別のノードがタスクキュー内のznodeを削除すると、 + それを処理しませんでしたが、すでに子のリストを取得し、DDLWorkerスレッドを終了します。 [\#5489](https://github.com/ClickHouse/ClickHouse/pull/5489) ([Azat Khuzhin](https://github.com/azat)) +- 固定挿入配布()テーブルを実現します。 [\#5429](https://github.com/ClickHouse/ClickHouse/pull/5429) ([Azat Khuzhin](https://github.com/azat)) -### ClickHouseリリース19.7.3.9,2019-05-30 {#clickhouse-release-19-7-3-9-2019-05-30} +### ClickHouse Release19.7.3.9,2019-05-30 {#clickhouse-release-19-7-3-9-2019-05-30} #### 新しい機能 {#new-features-2} - ユーザーが指定できる設定の範囲を制限することができます。 これらの制約は、ユーザー設定プロファイルで設定できます。 [\#4931](https://github.com/ClickHouse/ClickHouse/pull/4931) ([ヴィタリ - Baranov](https://github.com/vitlibar)) + バラノフ](https://github.com/vitlibar)) - 関数の第二のバージョンを追加します `groupUniqArray` 任意を使って `max_size` 結果の配列のサイズを制限するパラメーター。 この 動作は次のようになります `groupArray(max_size)(x)` 機能。 - [\#5026](https://github.com/ClickHouse/ClickHouse/pull/5026) ([Guillaume - Tassery](https://github.com/YiuRULE)) + [\#5026](https://github.com/ClickHouse/ClickHouse/pull/5026) ([ギヨーム + タッサリー](https://github.com/YiuRULE)) - TSVWithNames/CSVWithNames入力ファイル形式の場合、列の順序は次のようになります - ファイルヘッダーから決定。 これは、 + ファイ これはによって制御される `input_format_with_names_use_header` パラメータ。 [\#5081](https://github.com/ClickHouse/ClickHouse/pull/5081) ([Alexander](https://github.com/Akazz)) #### バグ修正 {#bug-fixes-2} -- マージ中にuncompressed\_cache+joinでクラッシュ(\#5197) +- マージ中にuncompressed\_cache+JOINでクラッシュする(\#5197) [\#5133](https://github.com/ClickHouse/ClickHouse/pull/5133) ([ダニラ クテニン](https://github.com/danlark1)) -- Segmentation faultにclickhouse-クライアントがクエリーのシステムです。 \#5066 +- システムテーブルへのclickhouse-clientクエリのセグメンテーション障害。 \#5066 [\#5127](https://github.com/ClickHouse/ClickHouse/pull/5127) ([イワン](https://github.com/abyss7)) -- KafkaEngine経由で重い負荷のデータ損失（＃4736) +- KafkaEngineによる重負荷時のデータ損失(\#4736) [\#5080](https://github.com/ClickHouse/ClickHouse/pull/5080) ([イワン](https://github.com/abyss7)) -- 固定非常に珍しいデータ競合状態が起こるのを実行する際にクエリとeuのすべての関少なくとも二つから選択します。列、システム。テーブル、システム。部品、システム。マージファミリのparts\_tablesまたはテーブルと、関連するテーブルの列の変更を同時に実行する。 [\#5189](https://github.com/ClickHouse/ClickHouse/pull/5189) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- UNION ALLを使用してクエリを実行するときに発生する可能性のある非常にまれなデータ競合状態を修正しました。列,システム.テーブル、システム。部品、システム。parts\_tablesまたはマージファミリのテーブルと、関連するテーブルの列の変更を同時に実行します。 [\#5189](https://github.com/ClickHouse/ClickHouse/pull/5189) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) -#### 性能の改善 {#performance-improvements-1} +#### 性能の向上 {#performance-improvements-1} -- の単一の数値列によるソートに基数ソートを使用します `ORDER BY` なし +- 単一の数値列によるソートに基数ソートを使用する `ORDER BY` なし `LIMIT`. [\#5106](https://github.com/ClickHouse/ClickHouse/pull/5106), [\#4439](https://github.com/ClickHouse/ClickHouse/pull/4439) - ([Evgenii Pravda](https://github.com/kvinty), - [alexey-milovidov](https://github.com/alexey-milovidov)) + ([エフゲニー-プラウダ](https://github.com/kvinty), + [アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) #### 文書 {#documentation-1} -- 翻訳書類の一部のテーブルエンジン。 +- 一部のテーブルエンジンの文書を中国語に翻訳します。 [\#5107](https://github.com/ClickHouse/ClickHouse/pull/5107), [\#5094](https://github.com/ClickHouse/ClickHouse/pull/5094), [\#5087](https://github.com/ClickHouse/ClickHouse/pull/5087) @@ -1415,660 +1415,660 @@ toc_title: '2019' - UTF-8文字を正しく印刷する `clickhouse-test`. [\#5084](https://github.com/ClickHouse/ClickHouse/pull/5084) - ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 追加コマンドラインパラメータclickhouse-クライアントに常に負荷の提案 - データ。 [\#5102](https://github.com/ClickHouse/ClickHouse/pull/5102) - ([alexey-milovidov](https://github.com/alexey-milovidov)) -- いくつかのpvs-studioの警告を解決します。 + ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 常に提案をロードするclickhouse-clientのコマンドラインパラメータを追加 + データ [\#5102](https://github.com/ClickHouse/ClickHouse/pull/5102) + ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- PVS-Studioの警告の一部を解決します。 [\#5082](https://github.com/ClickHouse/ClickHouse/pull/5082) - ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 更新lz4 [\#5040](https://github.com/ClickHouse/ClickHouse/pull/5040) ([ダニラ + ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- LZ4の更新 [\#5040](https://github.com/ClickHouse/ClickHouse/pull/5040) ([ダニラ クテニン](https://github.com/danlark1)) -- 今後のプル要求＃5030の要件を構築するためにgperfを追加します。 +- 今後のプル要求\#5030の要件を構築するためにgperfを追加します。 [\#5110](https://github.com/ClickHouse/ClickHouse/pull/5110) - ([proller](https://github.com/proller)) + ([プロラー](https://github.com/proller)) ## ClickHouseリリース19.6 {#clickhouse-release-19-6} -### クリックハウスリリース19.6.3.18,2019-06-13 {#clickhouse-release-19-6-3-18-2019-06-13} +### ClickHouse Release19.6.3.18,2019-06-13 {#clickhouse-release-19-6-3-18-2019-06-13} #### バグ修正 {#bug-fixes-3} -- テーブル関数からのクエリの条件プッシュダウンで修正 `mysql` と `odbc` と対応するテーブルエンジン。 これは＃3540と＃2384を修正します。 [\#5313](https://github.com/ClickHouse/ClickHouse/pull/5313) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Zookeeperのデッドロックを修正します。 [\#5297](https://github.com/ClickHouse/ClickHouse/pull/5297) ([github1youlcname](https://github.com/github1youlc)) -- CSVで引用小数を許可します。 [\#5284](https://github.com/ClickHouse/ClickHouse/pull/5284) ([Artem Zuikov](https://github.com/4ertus2) -- Float Inf/NaNからDecimalsへの変換を禁止します(例外をスローします)。 [\#5282](https://github.com/ClickHouse/ClickHouse/pull/5282) ([Artem Zuikov](https://github.com/4ertus2)) -- リネームクエリでデータレースを修正。 [\#5247](https://github.com/ClickHouse/ClickHouse/pull/5247) ([冬張](https://github.com/zhang2014)) -- 一時的にlfallocを無効にします。 lfallocの使用は、uncompressedcacheを割り当てる際に多くのmap\_failedにつながり、その結果、高負荷のサーバーでのクエリのクラッシュにつながる可能性があります。 [cfdba93comment](https://github.com/ClickHouse/ClickHouse/commit/cfdba938ce22f16efeec504f7f90206a515b1280)([Danila Kutenin](https://github.com/danlark1)) +- テーブル関数からのクエリの条件プッシュダウンで修正 `mysql` と `odbc` と対応するテーブルエンジン。 これは＃3540と＃2384を修正します。 [\#5313](https://github.com/ClickHouse/ClickHouse/pull/5313) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- Zookeeperのデッドロックを修正しました。 [\#5297](https://github.com/ClickHouse/ClickHouse/pull/5297) ([ギスブ1世](https://github.com/github1youlc)) +- CSVで引用符で囲まれた小数を許可します。 [\#5284](https://github.com/ClickHouse/ClickHouse/pull/5284) ([アルテム-ズイコフ](https://github.com/4ertus2) +- Float Inf/NaNから小数への変換を禁止します（例外をスローします）。 [\#5282](https://github.com/ClickHouse/ClickHouse/pull/5282) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- 名前変更クエリのデータ競合を修正しました。 [\#5247](https://github.com/ClickHouse/ClickHouse/pull/5247) ([冬張](https://github.com/zhang2014)) +- LFAllocを一時的に無効にします。 LFAllocを使用すると、UncompressedCacheを割り当てる際に多くのMAP\_FAILEDが発生し、結果として負荷の高いサーバーでのクエリのクラッシュにつながる可能性があります。 [cfdba93](https://github.com/ClickHouse/ClickHouse/commit/cfdba938ce22f16efeec504f7f90206a515b1280)([ダニラ-クテニン](https://github.com/danlark1)) -### ClickHouseリリース19.6.2.11、2019-05-13 {#clickhouse-release-19-6-2-11-2019-05-13} +### ClickHouse Release19.6.2.11,2019-05-13 {#clickhouse-release-19-6-2-11-2019-05-13} #### 新しい機能 {#new-features-3} -- 列およびテーブルのttl式。 [\#4212](https://github.com/ClickHouse/ClickHouse/pull/4212) ([アントン-ポポフ](https://github.com/CurtizJ)) -- のサポートを追加 `brotli` HTTPレスポンスの圧縮(Accept-Encoding:br) [\#4388](https://github.com/ClickHouse/ClickHouse/pull/4388) ([ミハイル](https://github.com/fandyushin)) -- 新しい機能を追加 `isValidUTF8` バイトのセットが正しくutf-8エンコードされているかどう [\#4934](https://github.com/ClickHouse/ClickHouse/pull/4934) ([Danila Kutenin](https://github.com/danlark1)) -- 新しい負荷分散ポリシーの追加 `first_or_random` 送信されるクエリを最初に指定されたホストの場合は利用できな送信をクエリーダ主催のチャームのボー. クロスレプリケーショ [\#5012](https://github.com/ClickHouse/ClickHouse/pull/5012) ([nvartolomei](https://github.com/nvartolomei)) +- 列および表のTTL式。 [\#4212](https://github.com/ClickHouse/ClickHouse/pull/4212) ([アントン-ポポフ](https://github.com/CurtizJ)) +- 追加されたサポート `brotli` HTTPレスポンスの圧縮(Accept-Encoding:br) [\#4388](https://github.com/ClickHouse/ClickHouse/pull/4388) ([ミハイル](https://github.com/fandyushin)) +- 新しい機能を追加 `isValidUTF8` 確認を行うための設定のバイトが正常にutf-8エンコードされます。 [\#4934](https://github.com/ClickHouse/ClickHouse/pull/4934) ([ダニラ-クテニン](https://github.com/danlark1)) +- 新しい負荷分散ポリシーの追加 `first_or_random` 送信されるクエリを最初に指定されたホストの場合は利用できな送信をクエリーダ主催のチャームのボー. クロスレプリケーショントポロジ設定に便利です。 [\#5012](https://github.com/ClickHouse/ClickHouse/pull/5012) ([nvartolomei](https://github.com/nvartolomei)) -#### 実験の特徴 {#experimental-features-1} +#### 実験的な特徴 {#experimental-features-1} -- 設定を追加 `index_granularity_bytes` (アダプティブインデックス粒度)MergeTree\*テーブルファミリの場合。 [\#4826](https://github.com/ClickHouse/ClickHouse/pull/4826) ([alesapin](https://github.com/alesapin)) +- 設定を追加 `index_granularity_bytes` MergeTree\*テーブルファミリの(適応インデックス粒度)。 [\#4826](https://github.com/ClickHouse/ClickHouse/pull/4826) ([アレサピン](https://github.com/alesapin)) #### 改善 {#improvements-1} -- 関数の非定数および負のサイズと長さの引数のサポートが追加されました `substringUTF8`. [\#4989](https://github.com/ClickHouse/ClickHouse/pull/4989) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 左結合ではプッシュダウンから右テーブル、右結合では左テーブル、フル結合では両方のテーブルを無効にします。 このおかしくなる問題を修正に入実績もあります。 [\#4846](https://github.com/ClickHouse/ClickHouse/pull/4846) ([イワン](https://github.com/abyss7)) -- `clickhouse-copier`:タスク設定の自動アップロード `--task-file` オプション [\#4876](https://github.com/ClickHouse/ClickHouse/pull/4876) ([proller](https://github.com/proller)) -- 追加の誤字ハンドラに保存工場とテーブル機能の工場です。 [\#4891](https://github.com/ClickHouse/ClickHouse/pull/4891) ([Danila Kutenin](https://github.com/danlark1)) -- サブクエリなしで複数のジョインのアスタリスクと修飾アスタリスクをサポート [\#4898](https://github.com/ClickHouse/ClickHouse/pull/4898) ([Artem Zuikov](https://github.com/4ertus2)) -- く不カラムのエラーメッセージよりユーザーにも優しい。 [\#4915](https://github.com/ClickHouse/ClickHouse/pull/4915) ([Artem Zuikov](https://github.com/4ertus2)) +- 関数の非定数引数と負のサイズと長さの引数のサポートが追加されました `substringUTF8`. [\#4989](https://github.com/ClickHouse/ClickHouse/pull/4989) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 左結合の右表へのプッシュダウン、右結合の左表へのプッシュダウン、および完全結合の両方の表を無効にします。 これにより、誤った結合結果が修正されます。 [\#4846](https://github.com/ClickHouse/ClickHouse/pull/4846) ([イワン](https://github.com/abyss7)) +- `clickhouse-copier`:からの自動アップロードタスク設定 `--task-file` オプション [\#4876](https://github.com/ClickHouse/ClickHouse/pull/4876) ([プロラー](https://github.com/proller)) +- 追加の誤字ハンドラに保存工場とテーブル機能の工場です。 [\#4891](https://github.com/ClickHouse/ClickHouse/pull/4891) ([ダニラ-クテニン](https://github.com/danlark1)) +- サブクエリなしで複数の結合のためのアスタリスクと修飾アスタリスクをサポート [\#4898](https://github.com/ClickHouse/ClickHouse/pull/4898) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- く不カラムのエラーメッセージよりユーザーにも優しい。 [\#4915](https://github.com/ClickHouse/ClickHouse/pull/4915) ([アルテム-ズイコフ](https://github.com/4ertus2)) -#### 性能の改善 {#performance-improvements-2} +#### 性能の向上 {#performance-improvements-2} -- ASOF結合の大幅な高速化 [\#4924](https://github.com/ClickHouse/ClickHouse/pull/4924) ([Martijn Bakker](https://github.com/Gladdy)) +- ASOF結合の大幅な高速化 [\#4924](https://github.com/ClickHouse/ClickHouse/pull/4924) ([マルティン-バッカー](https://github.com/Gladdy)) #### 下位互換性のない変更 {#backward-incompatible-changes} -- HTTPヘッダ `Query-Id` に改名された `X-ClickHouse-Query-Id` 一貫性のために。 [\#4972](https://github.com/ClickHouse/ClickHouse/pull/4972) ([ミハイル](https://github.com/fandyushin)) +- HTTPヘッダ `Query-Id` に改名されました `X-ClickHouse-Query-Id` 一貫性のために。 [\#4972](https://github.com/ClickHouse/ClickHouse/pull/4972) ([ミハイル](https://github.com/fandyushin)) #### バグ修正 {#bug-fixes-4} -- 固定された潜在的なnullポインタの逆参照 `clickhouse-copier`. [\#4900](https://github.com/ClickHouse/ClickHouse/pull/4900) ([proller](https://github.com/proller)) -- JOIN+ARRAY JOINによるクエリのエラーを修正 [\#4938](https://github.com/ClickHouse/ClickHouse/pull/4938) ([Artem Zuikov](https://github.com/4ertus2)) -- 固定掛けの開始にサーバーが辞書により他の辞書を介してデータベースエンジン=辞書で調べました。 [\#4962](https://github.com/ClickHouse/ClickHouse/pull/4962) ([Vitaly Baranov](https://github.com/vitlibar)) -- Partially fix distributed\_product\_mode = local. It’s possible to allow columns of local tables in where/having/order by/… via table aliases. Throw exception if table does not have alias. There’s not possible to access to the columns without table aliases yet. [\#4986](https://github.com/ClickHouse/ClickHouse/pull/4986) ([Artem Zuikov](https://github.com/4ertus2)) -- のための潜在的に間違った結果を修正 `SELECT DISTINCT` と `JOIN` [\#5001](https://github.com/ClickHouse/ClickHouse/pull/5001) ([Artem Zuikov](https://github.com/4ertus2)) -- 固定非常に珍しいデータ競合状態が起こるのを実行する際にクエリとeuのすべての関少なくとも二つから選択します。列、システム。テーブル、システム。部品、システム。マージファミリのparts\_tablesまたはテーブルと、関連するテーブルの列の変更を同時に実行する。 [\#5189](https://github.com/ClickHouse/ClickHouse/pull/5189) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- 固定された潜在的なヌルポインタの逆参照 `clickhouse-copier`. [\#4900](https://github.com/ClickHouse/ClickHouse/pull/4900) ([プロラー](https://github.com/proller)) +- 結合+配列結合でクエリのエラーを修正しました [\#4938](https://github.com/ClickHouse/ClickHouse/pull/4938) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- 固定掛けの開始にサーバーが辞書により他の辞書を介してデータベースエンジン=辞書で調べました。 [\#4962](https://github.com/ClickHouse/ClickHouse/pull/4962) ([ヴィタリー-バラノフ](https://github.com/vitlibar)) +- Partially fix distributed\_product\_mode = local. It's possible to allow columns of local tables in where/having/order by/… via table aliases. Throw exception if table does not have alias. There's not possible to access to the columns without table aliases yet. [\#4986](https://github.com/ClickHouse/ClickHouse/pull/4986) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- 潜在的に間違った結果を修正 `SELECT DISTINCT` と `JOIN` [\#5001](https://github.com/ClickHouse/ClickHouse/pull/5001) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- UNION ALLを使用してクエリを実行するときに発生する可能性のある非常にまれなデータ競合状態を修正しました。列,システム.テーブル、システム。部品、システム。parts\_tablesまたはマージファミリのテーブルと、関連するテーブルの列の変更を同時に実行します。 [\#5189](https://github.com/ClickHouse/ClickHouse/pull/5189) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) #### ビルド/テスト/パッケージの改善 {#buildtestingpackaging-improvements-2} -- 別のホストでclickhouse-serverを実行しているときのテストの失敗を修正 [\#4713](https://github.com/ClickHouse/ClickHouse/pull/4713) ([Vasily Nemkov](https://github.com/Enmk)) -- clickhouse-テスト：非tty環境でカラーコントロールシーケンスを無効にします。 [\#4937](https://github.com/ClickHouse/ClickHouse/pull/4937) ([alesapin](https://github.com/alesapin)) -- clickhouse-test:任意のテストデータベースの使用を許可する(削除する `test.` それが可能な資格) [\#5008](https://github.com/ClickHouse/ClickHouse/pull/5008) ([proller](https://github.com/proller)) -- Ubsanエラーの修正 [\#5037](https://github.com/ClickHouse/ClickHouse/pull/5037) ([Vitaly Baranov](https://github.com/vitlibar)) -- Yandex LFAllocがClickHouseに追加され、MarkCacheとUncompressedCacheデータをさまざまな方法で割り当てて、より信頼性の高いsegfaultをキャッチしました [\#4995](https://github.com/ClickHouse/ClickHouse/pull/4995) ([Danila Kutenin](https://github.com/danlark1)) -- バックポートとチェンジログを支援するためのpython util。 [\#4949](https://github.com/ClickHouse/ClickHouse/pull/4949) ([イワン](https://github.com/abyss7)) +- 別のホストでclickhouse-serverを実行しているときのテストエラーを修正 [\#4713](https://github.com/ClickHouse/ClickHouse/pull/4713) ([ヴァシーリー-ネムコフ](https://github.com/Enmk)) +- clickhouse-test:非tty環境でカラーコントロールシーケンスを無効にします。 [\#4937](https://github.com/ClickHouse/ClickHouse/pull/4937) ([アレサピン](https://github.com/alesapin)) +- clickhouse-test:任意のテストデータベースの使用を許可(削除 `test.` それが可能な資格) [\#5008](https://github.com/ClickHouse/ClickHouse/pull/5008) ([プロラー](https://github.com/proller)) +- Ubsanのエラーを修正 [\#5037](https://github.com/ClickHouse/ClickHouse/pull/5037) ([ヴィタリー-バラノフ](https://github.com/vitlibar)) +- Yandex LFAllocがClickHouseに追加され、markcacheとUncompressedCacheのデータをさまざまな方法で割り当て、segfaultsをより信頼性の高いものにすることができました [\#4995](https://github.com/ClickHouse/ClickHouse/pull/4995) ([ダニラ-クテニン](https://github.com/danlark1)) +- バックポートとチェンジログを支援するPython util。 [\#4949](https://github.com/ClickHouse/ClickHouse/pull/4949) ([イワン](https://github.com/abyss7)) ## ClickHouseリリース19.5 {#clickhouse-release-19-5} -### ClickHouseリリース19.5.4.22、2019-05-13 {#clickhouse-release-19-5-4-22-2019-05-13} +### ClickHouse Release19.5.4.22,2019-05-13 {#clickhouse-release-19-5-4-22-2019-05-13} #### バグ修正 {#bug-fixes-5} -- ビットマップ\*機能のクラッシュを修正 [\#5220](https://github.com/ClickHouse/ClickHouse/pull/5220) [\#5228](https://github.com/ClickHouse/ClickHouse/pull/5228) ([アンディヤング](https://github.com/andyyzh)) -- 固定非常に珍しいデータ競合状態が起こるのを実行する際にクエリとeuのすべての関少なくとも二つから選択します。列、システム。テーブル、システム。部品、システム。マージファミリのparts\_tablesまたはテーブルと、関連するテーブルの列の変更を同時に実行する。 [\#5189](https://github.com/ClickHouse/ClickHouse/pull/5189) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 固定エラー `Set for IN is not created yet in case of using single LowCardinality column in the left part of IN`. このエラーは、LowCardinality列が主キーの一部であった場合に発生しました。 \#5031 [\#5154](https://github.com/ClickHouse/ClickHouse/pull/5154) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 保持機能の変更：行が最初の条件とn番目の条件の両方を満たす場合、最初の満足条件のみがデータ状態に追加されます。 これで、データの行を満たすすべての条件がデータの状態に追加されます。 [\#5119](https://github.com/ClickHouse/ClickHouse/pull/5119) ([小路](https://github.com/nicelulu)) +- ビットマップ\*関数でのクラッシュを修正 [\#5220](https://github.com/ClickHouse/ClickHouse/pull/5220) [\#5228](https://github.com/ClickHouse/ClickHouse/pull/5228) ([アンディ-ヤン](https://github.com/andyyzh)) +- UNION ALLを使用してクエリを実行するときに発生する可能性のある非常にまれなデータ競合状態を修正しました。列,システム.テーブル、システム。部品、システム。parts\_tablesまたはマージファミリのテーブルと、関連するテーブルの列の変更を同時に実行します。 [\#5189](https://github.com/ClickHouse/ClickHouse/pull/5189) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 修正されたエラー `Set for IN is not created yet in case of using single LowCardinality column in the left part of IN`. このエラーは、LowCardinality列が主キーの一部である場合に発生しました。 \#5031 [\#5154](https://github.com/ClickHouse/ClickHouse/pull/5154) ([ニコライ-コチェトフ](https://github.com/KochetovNicolai)) +- 保持関数の変更：行が最初の条件とN番目の条件の両方を満たす場合、最初に満たされた条件のみがデータ状態に追加されます。 これで、データ行で満たすすべての条件がデータ状態に追加されます。 [\#5119](https://github.com/ClickHouse/ClickHouse/pull/5119) ([小路](https://github.com/nicelulu)) -### ClickHouseリリース19.5.3.8,2019-04-18 {#clickhouse-release-19-5-3-8-2019-04-18} +### ClickHouse Release19.5.3.8,2019-04-18 {#clickhouse-release-19-5-3-8-2019-04-18} #### バグ修正 {#bug-fixes-6} -- 設定の固定タイプ `max_partitions_per_insert_block` ブール値からUInt64へ。 [\#5028](https://github.com/ClickHouse/ClickHouse/pull/5028) ([Mohammad Hostsein Sekhavat](https://github.com/mhsekhavat)) +- 固定タイプの設定 `max_partitions_per_insert_block` ブール値からUInt64まで。 [\#5028](https://github.com/ClickHouse/ClickHouse/pull/5028) ([モハマド-ホセイン-セハヴァト](https://github.com/mhsekhavat)) -### ClickHouseリリース19.5.2.6,2019-04-15 {#clickhouse-release-19-5-2-6-2019-04-15} +### ClickHouse Release19.5.2.6,2019-04-15 {#clickhouse-release-19-5-2-6-2019-04-15} #### 新しい機能 {#new-features-4} -- [Hyperscan](https://github.com/intel/hyperscan) 複数の正規表現マッチングが追加されました(関数 `multiMatchAny`, `multiMatchAnyIndex`, `multiFuzzyMatchAny`, `multiFuzzyMatchAnyIndex`). [\#4780](https://github.com/ClickHouse/ClickHouse/pull/4780), [\#4841](https://github.com/ClickHouse/ClickHouse/pull/4841) ([Danila Kutenin](https://github.com/danlark1)) -- `multiSearchFirstPosition` 機能が追加されました。 [\#4780](https://github.com/ClickHouse/ClickHouse/pull/4780) ([Danila Kutenin](https://github.com/danlark1)) +- [Hyperscan](https://github.com/intel/hyperscan) 複数の正規表現マッチングが追加されました(関数 `multiMatchAny`, `multiMatchAnyIndex`, `multiFuzzyMatchAny`, `multiFuzzyMatchAnyIndex`). [\#4780](https://github.com/ClickHouse/ClickHouse/pull/4780), [\#4841](https://github.com/ClickHouse/ClickHouse/pull/4841) ([ダニラ-クテニン](https://github.com/danlark1)) +- `multiSearchFirstPosition` 機能を追加しました。 [\#4780](https://github.com/ClickHouse/ClickHouse/pull/4780) ([ダニラ-クテニン](https://github.com/danlark1)) - の実施を所定の表現フィルター配列です。 [\#4792](https://github.com/ClickHouse/ClickHouse/pull/4792) ([イワン](https://github.com/abyss7)) -- 新しいタイプのデータを飛び指標に基づくブル(使用可能 `equal`, `in` と `like` 機能）。 [\#4499](https://github.com/ClickHouse/ClickHouse/pull/4499) ([Nikita Vasilev](https://github.com/nikvas0)) -- 追加 `ASOF JOIN` これにより、既知の最新の値に結合するクエリを実行できます。 [\#4774](https://github.com/ClickHouse/ClickHouse/pull/4774) [\#4867](https://github.com/ClickHouse/ClickHouse/pull/4867) [\#4863](https://github.com/ClickHouse/ClickHouse/pull/4863) [\#4875](https://github.com/ClickHouse/ClickHouse/pull/4875) ([Martijn Bakker](https://github.com/Gladdy), [Artem Zuikov](https://github.com/4ertus2)) -- 複数の書き換え `COMMA JOIN` に `CROSS JOIN`. 次にそれらを書き換える `INNER JOIN` 可能であれば。 [\#4661](https://github.com/ClickHouse/ClickHouse/pull/4661) ([Artem Zuikov](https://github.com/4ertus2)) +- 新しいタイプのデータを飛び指標に基づくブル(使用可能 `equal`, `in` と `like` 機能）。 [\#4499](https://github.com/ClickHouse/ClickHouse/pull/4499) ([ニキータ-ヴァシレフ](https://github.com/nikvas0)) +- 追加 `ASOF JOIN` これにより、既知の最新の値に結合するクエリを実行できます。 [\#4774](https://github.com/ClickHouse/ClickHouse/pull/4774) [\#4867](https://github.com/ClickHouse/ClickHouse/pull/4867) [\#4863](https://github.com/ClickHouse/ClickHouse/pull/4863) [\#4875](https://github.com/ClickHouse/ClickHouse/pull/4875) ([マルティン-バッカー](https://github.com/Gladdy), [アルテム-ズイコフ](https://github.com/4ertus2)) +- 複数の書き換え `COMMA JOIN` に `CROSS JOIN`. その書き換えて `INNER JOIN` 可能であれば。 [\#4661](https://github.com/ClickHouse/ClickHouse/pull/4661) ([アルテム-ズイコフ](https://github.com/4ertus2)) #### 改善 {#improvement-9} -- `topK` と `topKWeighted` 今サポートカスタム `loadFactor` （修正の問題 [\#4252](https://github.com/ClickHouse/ClickHouse/issues/4252)). [\#4634](https://github.com/ClickHouse/ClickHouse/pull/4634) ([キリル丹心](https://github.com/kirillDanshin)) -- 使用を許可する `parallel_replicas_count > 1` サンプリングされていないテーブルの場合でも（設定は単に無視されます）。 以前のバージョンでは、例外が発生しました。 [\#4637](https://github.com/ClickHouse/ClickHouse/pull/4637) ([Alexey Elymanov](https://github.com/digitalist)) -- のサポート `CREATE OR REPLACE VIEW`. ビューの作成または単一のステートメントでの新しい定義の設定を許可します。 [\#4654](https://github.com/ClickHouse/ClickHouse/pull/4654) ([Boris Granveaud](https://github.com/bgranvea)) -- `Buffer` テーブルエンジン `PREWHERE`. [\#4671](https://github.com/ClickHouse/ClickHouse/pull/4671) ([楊関劉](https://github.com/LiuYangkuan)) -- Zookeeperのメタデータなしで複製テーブルを開始する機能を追加 `readonly` モード。 [\#4691](https://github.com/ClickHouse/ClickHouse/pull/4691) ([alesapin](https://github.com/alesapin)) -- Clickhouse-clientのプログレスバーのフリッカーを修正しました。 この問題は、 `FORMAT Null` ストリーミングクエ [\#4811](https://github.com/ClickHouse/ClickHouse/pull/4811) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 機能を無効にするには `hyperscan` ユーザーごとにライブラリを使用して、過度かつ無制限のリソース使用量を制限します。 [\#4816](https://github.com/ClickHouse/ClickHouse/pull/4816) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 追加バージョン番号でログインしてすべてのエラー. [\#4824](https://github.com/ClickHouse/ClickHouse/pull/4824) ([proller](https://github.com/proller)) -- に制限を追加しました `multiMatch` 文字列サイズを必要とする関数 `unsigned int`. また、引数の数の制限を追加しました。 `multiSearch` 機能。 [\#4834](https://github.com/ClickHouse/ClickHouse/pull/4834) ([Danila Kutenin](https://github.com/danlark1)) -- 改善の利用をゼロスペースおよびエラー処理にhyperscan. [\#4866](https://github.com/ClickHouse/ClickHouse/pull/4866) ([Danila Kutenin](https://github.com/danlark1)) -- 塗りつぶし `system.graphite_detentions` テーブルの設定から `*GraphiteMergeTree` エンジンテーブル。 [\#4584](https://github.com/ClickHouse/ClickHouse/pull/4584) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) -- 名前変更 `trigramDistance` 機能への `ngramDistance` と追加より機能で `CaseInsensitive` と `UTF`. [\#4602](https://github.com/ClickHouse/ClickHouse/pull/4602) ([Danila Kutenin](https://github.com/danlark1)) -- データスキップインデックス計算の改善。 [\#4640](https://github.com/ClickHouse/ClickHouse/pull/4640) ([Nikita Vasilev](https://github.com/nikvas0)) -- 普通に保つ, `DEFAULT`, `MATERIALIZED` と `ALIAS` 単一のリストの列（修正の問題 [\#2867](https://github.com/ClickHouse/ClickHouse/issues/2867)). [\#4707](https://github.com/ClickHouse/ClickHouse/pull/4707) ([Alex Zatelepin](https://github.com/ztlpn)) +- `topK` と `topKWeighted` 今すぐカスタム `loadFactor` （修正の問題 [\#4252](https://github.com/ClickHouse/ClickHouse/issues/4252)). [\#4634](https://github.com/ClickHouse/ClickHouse/pull/4634) ([キリルダンシン](https://github.com/kirillDanshin)) +- 使用を許可する `parallel_replicas_count > 1` サンプリングのないテーブルでも（設定は単に無視されます）。 以前のバージョンでは例外につながっていました。 [\#4637](https://github.com/ClickHouse/ClickHouse/pull/4637) ([アレクセイ-エリマノフ](https://github.com/digitalist)) +- のサポート `CREATE OR REPLACE VIEW`. 単一のステートメントでビューを作成したり、新しい定義を設定したりできます。 [\#4654](https://github.com/ClickHouse/ClickHouse/pull/4654) ([ボリス-グランヴォー](https://github.com/bgranvea)) +- `Buffer` テーブルエンジンに対応しま `PREWHERE`. [\#4671](https://github.com/ClickHouse/ClickHouse/pull/4671) ([ヤンクアン-リュウ](https://github.com/LiuYangkuan)) +- 追加能力を開始複製テーブルなしデータを飼育係に `readonly` モード [\#4691](https://github.com/ClickHouse/ClickHouse/pull/4691) ([アレサピン](https://github.com/alesapin)) +- Clickhouse-clientのプログレスバーのちらつきを修正しました。 この問題は、使用するときに最も顕著でした `FORMAT Null` ストリーミン [\#4811](https://github.com/ClickHouse/ClickHouse/pull/4811) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 関数を無効にすることを許可する `hyperscan` 図書館にユーザー毎に制限潜在的過剰および再資源利用に [\#4816](https://github.com/ClickHouse/ClickHouse/pull/4816) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 追加バージョン番号でログインしてすべてのエラー. [\#4824](https://github.com/ClickHouse/ClickHouse/pull/4824) ([プロラー](https://github.com/proller)) +- に制限を追加しました `multiMatch` 文字列に収まるように文字列サイズを必要とする関数 `unsigned int`. また、引数の数の制限を追加しました。 `multiSearch` 機能。 [\#4834](https://github.com/ClickHouse/ClickHouse/pull/4834) ([ダニラ-クテニン](https://github.com/danlark1)) +- Hyperscanでのスクラッチ領域とエラー処理の使用が改善されました。 [\#4866](https://github.com/ClickHouse/ClickHouse/pull/4866) ([ダニラ-クテニン](https://github.com/danlark1)) +- 塗りつぶし `system.graphite_detentions` のテーブル設定から `*GraphiteMergeTree` エンジンテーブル。 [\#4584](https://github.com/ClickHouse/ClickHouse/pull/4584) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) +- 名前変更 `trigramDistance` への関数 `ngramDistance` とより多くの機能を追加 `CaseInsensitive` と `UTF`. [\#4602](https://github.com/ClickHouse/ClickHouse/pull/4602) ([ダニラ-クテニン](https://github.com/danlark1)) +- の改善データを飛び指標を計算します。 [\#4640](https://github.com/ClickHouse/ClickHouse/pull/4640) ([ニキータ-ヴァシレフ](https://github.com/nikvas0)) +- 普通に保つ, `DEFAULT`, `MATERIALIZED` と `ALIAS` 単一のリスト内の列（修正の問題 [\#2867](https://github.com/ClickHouse/ClickHouse/issues/2867)). [\#4707](https://github.com/ClickHouse/ClickHouse/pull/4707) ([アレックス-ザテレピン](https://github.com/ztlpn)) #### バグ修正 {#bug-fix-26} -- 避ける `std::terminate` メモリ割り当てに失敗した場合。 さて `std::bad_alloc` 期待どおりに例外がスローされます。 [\#4665](https://github.com/ClickHouse/ClickHouse/pull/4665) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- バッファからcapnprotoの読み取りを修正します。 時にファイルなロードに成功するhttp. [\#4674](https://github.com/ClickHouse/ClickHouse/pull/4674) ([ウラジスラフ](https://github.com/smirnov-vs)) -- エラーの修正 `Unknown log entry type: 0` 後に `OPTIMIZE TABLE FINAL` クエリ。 [\#4683](https://github.com/ClickHouse/ClickHouse/pull/4683) ([アモスの鳥](https://github.com/amosbird)) -- 間違った引数へ `hasAny` または `hasAll` 関数はsegfaultにつながる可能性があります。 [\#4698](https://github.com/ClickHouse/ClickHouse/pull/4698) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 行き詰まりが発生する恐れがあるとしながら実行 `DROP DATABASE dictionary` クエリ。 [\#4701](https://github.com/ClickHouse/ClickHouse/pull/4701) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 未定義の動作を修正する `median` と `quantile` 機能。 [\#4702](https://github.com/ClickHouse/ClickHouse/pull/4702) ([hcz](https://github.com/hczhcz)) -- ときに圧縮レベル検出を修正 `network_compression_method` 小文字で。 V19.1で壊れた。 [\#4706](https://github.com/ClickHouse/ClickHouse/pull/4706) ([proller](https://github.com/proller)) -- の固定無知 `UTC` 設定（修正の問題 [\#4658](https://github.com/ClickHouse/ClickHouse/issues/4658)). [\#4718](https://github.com/ClickHouse/ClickHouse/pull/4718) ([proller](https://github.com/proller)) -- 修正 `histogram` 関数の振る舞い `Distributed` テーブル。 [\#4741](https://github.com/ClickHouse/ClickHouse/pull/4741) ([olegkv](https://github.com/olegkv)) -- 固定tsanレポート `destroy of a locked mutex`. [\#4742](https://github.com/ClickHouse/ClickHouse/pull/4742) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- システムログ使用時の競合状態によるシャットダウン時のtsanレポートを修正。 part\_logが有効になっているときにシャットダウン時に固定された潜在的な使用後無料。 [\#4758](https://github.com/ClickHouse/ClickHouse/pull/4758) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- パーツの再チェックを修正 `ReplicatedMergeTreeAlterThread` エラーの場合。 [\#4772](https://github.com/ClickHouse/ClickHouse/pull/4772) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 中間集計関数状態に対する算術演算は、定数引数(サブクエリ結果など)に対して機能していませんでした。 [\#4776](https://github.com/ClickHouse/ClickHouse/pull/4776) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 常にメタデータの列名を引用します。 それ以外の場合は、列という名前の表を作成することは不可能です `index` （サーバーは不正な形式のために再起動しません `ATTACH` メタデータ内のクエリ)。 [\#4782](https://github.com/ClickHouse/ClickHouse/pull/4782) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- でクラッシュを修正 `ALTER ... MODIFY ORDER BY` に `Distributed` テーブル。 [\#4790](https://github.com/ClickHouse/ClickHouse/pull/4790) ([Tcheason](https://github.com/TCeason)) -- Segfaultを修正する `JOIN ON` 有効にした場合 `enable_optimize_predicate_expression`. [\#4794](https://github.com/ClickHouse/ClickHouse/pull/4794) ([冬張](https://github.com/zhang2014)) -- カフカからprotobufメッセージを消費した後、余分な行を追加するとバグを修正しました。 [\#4808](https://github.com/ClickHouse/ClickHouse/pull/4808) ([Vitaly Baranov](https://github.com/vitlibar)) -- のクラッシュを修正 `JOIN` nullable列とnullable列ではありません。 修正 `NULLs` 右キーで `ANY JOIN` + `join_use_nulls`. [\#4815](https://github.com/ClickHouse/ClickHouse/pull/4815) ([Artem Zuikov](https://github.com/4ertus2)) -- セグメンテーショ `clickhouse-copier`. [\#4835](https://github.com/ClickHouse/ClickHouse/pull/4835) ([proller](https://github.com/proller)) -- 固定競合状態で `SELECT` から `system.tables` テーブルが同時に名前変更または変更された場合。 [\#4836](https://github.com/ClickHouse/ClickHouse/pull/4836) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 既に廃止されたデータ部分をフェッチする際のデータレースを修正。 [\#4839](https://github.com/ClickHouse/ClickHouse/pull/4839) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 中に発生することができます `RENAME` MergeTree家族のテーブル。 [\#4844](https://github.com/ClickHouse/ClickHouse/pull/4844) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 機能の固定細分化の欠陥 `arrayIntersect`. Segmentation faultう場合は関数と呼ばれたとの混合の定数、通常の引数になります。 [\#4847](https://github.com/ClickHouse/ClickHouse/pull/4847) ([Lixiang Qian](https://github.com/fancyqlx)) -- 固定読み取りから `Array(LowCardinality)` 列に空の配列の長いシーケンスが含まれている場合はまれです。 [\#4850](https://github.com/ClickHouse/ClickHouse/pull/4850) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- でクラッシュを修正 `FULL/RIGHT JOIN` 私たちはnullable対nullableではないに参加するとき. [\#4855](https://github.com/ClickHouse/ClickHouse/pull/4855) ([Artem Zuikov](https://github.com/4ertus2)) -- 修正 `No message received` レプリカ間のパーツの取得中の例外。 [\#4856](https://github.com/ClickHouse/ClickHouse/pull/4856) ([alesapin](https://github.com/alesapin)) -- 固定 `arrayIntersect` 単一の配列のいくつかの繰り返しの値の場合に機能間違った結果。 [\#4871](https://github.com/ClickHouse/ClickHouse/pull/4871) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 同時実行中の競合状態の修正 `ALTER COLUMN` クエリを生み出しうるサーバストレスとソーシャル-サポート問題の修正 [\#3421](https://github.com/ClickHouse/ClickHouse/issues/3421)). [\#4592](https://github.com/ClickHouse/ClickHouse/pull/4592) ([Alex Zatelepin](https://github.com/ztlpn)) -- 誤った結果を修正 `FULL/RIGHT JOIN` constの列を持つ。 [\#4723](https://github.com/ClickHouse/ClickHouse/pull/4723) ([Artem Zuikov](https://github.com/4ertus2)) -- 重複を修正する `GLOBAL JOIN` アスタリスク付き。 [\#4705](https://github.com/ClickHouse/ClickHouse/pull/4705) ([Artem Zuikov](https://github.com/4ertus2)) -- 修正パラメータ控除で `ALTER MODIFY` 列の `CODEC` 列の型が指定されていない場合。 [\#4883](https://github.com/ClickHouse/ClickHouse/pull/4883) ([alesapin](https://github.com/alesapin)) -- 機能 `cutQueryStringAndFragment()` と `queryStringAndFragment()` 今すぐ正しく動作します `URL` くれないのを返します。 [\#4894](https://github.com/ClickHouse/ClickHouse/pull/4894) ([Vitaly Baranov](https://github.com/vitlibar)) -- 設定時にまれなバグを修正 `min_bytes_to_use_direct_io` これは、スレッドが列ファイル内で逆方向にシークする必要があるときに発生します。 [\#4897](https://github.com/ClickHouse/ClickHouse/pull/4897) ([alesapin](https://github.com/alesapin)) -- 集計関数の誤った引数の型を修正する `LowCardinality` 引数(修正の問題 [\#4919](https://github.com/ClickHouse/ClickHouse/issues/4919)). [\#4922](https://github.com/ClickHouse/ClickHouse/pull/4922) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 間違った名前の修飾を修正 `GLOBAL JOIN`. [\#4969](https://github.com/ClickHouse/ClickHouse/pull/4969) ([Artem Zuikov](https://github.com/4ertus2)) -- 修正機能 `toISOWeek` 1970年の結果。 [\#4988](https://github.com/ClickHouse/ClickHouse/pull/4988) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 修正 `DROP`, `TRUNCATE` と `OPTIMIZE` 実行されたときのクエリの重複 `ON CLUSTER` のために `ReplicatedMergeTree*` テーブルの家族。 [\#4991](https://github.com/ClickHouse/ClickHouse/pull/4991) ([alesapin](https://github.com/alesapin)) +- 避ける `std::terminate` メモリ割り当てに失敗した場合。 さて `std::bad_alloc` 例外は期待どおりにスローされます。 [\#4665](https://github.com/ClickHouse/ClickHouse/pull/4665) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- バッファからの読み取りを修正capnproto。 時にファイルなロードに成功するHTTP. [\#4674](https://github.com/ClickHouse/ClickHouse/pull/4674) ([ウラジスラフ](https://github.com/smirnov-vs)) +- 修正エラー `Unknown log entry type: 0` 後に `OPTIMIZE TABLE FINAL` クエリ。 [\#4683](https://github.com/ClickHouse/ClickHouse/pull/4683) ([アモス鳥](https://github.com/amosbird)) +- 間違った引数に `hasAny` または `hasAll` 関数はsegfaultにつながる可能性があります。 [\#4698](https://github.com/ClickHouse/ClickHouse/pull/4698) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 実行中にデッドロックが発生する `DROP DATABASE dictionary` クエリ。 [\#4701](https://github.com/ClickHouse/ClickHouse/pull/4701) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- で未定義の動作を修正 `median` と `quantile` 機能。 [\#4702](https://github.com/ClickHouse/ClickHouse/pull/4702) ([hcz](https://github.com/hczhcz)) +- 圧縮レベル検出時の修正 `network_compression_method` 小文字で。 V19.1で壊れた。 [\#4706](https://github.com/ClickHouse/ClickHouse/pull/4706) ([プロラー](https://github.com/proller)) +- の固定された無知 `UTC` 設定（修正の問題 [\#4658](https://github.com/ClickHouse/ClickHouse/issues/4658)). [\#4718](https://github.com/ClickHouse/ClickHouse/pull/4718) ([プロラー](https://github.com/proller)) +- 修正 `histogram` 関数の動作 `Distributed` テーブル [\#4741](https://github.com/ClickHouse/ClickHouse/pull/4741) ([olegkv](https://github.com/olegkv)) +- 固定tsanレポート `destroy of a locked mutex`. [\#4742](https://github.com/ClickHouse/ClickHouse/pull/4742) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 固定賛育の報告書の停止によるレースの条件のシステムのログ利用 固定の潜在的利用に停止時part\_logが有効になります。 [\#4758](https://github.com/ClickHouse/ClickHouse/pull/4758) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 修正再チェック部品 `ReplicatedMergeTreeAlterThread` エラーの場合。 [\#4772](https://github.com/ClickHouse/ClickHouse/pull/4772) ([ニコライ-コチェトフ](https://github.com/KochetovNicolai)) +- 中間集計関数状態に対する算術演算が、定数引数(サブクエリ結果など)に対して機能していませんでした。 [\#4776](https://github.com/ClickHouse/ClickHouse/pull/4776) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 常にbackquoteカラム名は、メタデータを指すものとします。 それ以外の場合は、列名のテーブルを作成することは不可能です `index` (不正な形式のためサーバーは再起動されません `ATTACH` メタデータ内のクエリ）。 [\#4782](https://github.com/ClickHouse/ClickHouse/pull/4782) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- でクラッシュを修正 `ALTER ... MODIFY ORDER BY` に `Distributed` テーブル。 [\#4790](https://github.com/ClickHouse/ClickHouse/pull/4790) ([Tシーズン](https://github.com/TCeason)) +- でsegfaultを修正 `JOIN ON` 有効にした場合 `enable_optimize_predicate_expression`. [\#4794](https://github.com/ClickHouse/ClickHouse/pull/4794) ([冬張](https://github.com/zhang2014)) +- カフカからprotobufメッセージを消費した後に余分な行を追加するバグを修正しました。 [\#4808](https://github.com/ClickHouse/ClickHouse/pull/4808) ([ヴィタリー-バラノフ](https://github.com/vitlibar)) +- のクラッシュを修正 `JOIN` null許容ではない列とnull許容の列です。 修正 `NULLs` in右キー in `ANY JOIN` + `join_use_nulls`. [\#4815](https://github.com/ClickHouse/ClickHouse/pull/4815) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- 固定の区分断層に `clickhouse-copier`. [\#4835](https://github.com/ClickHouse/ClickHouse/pull/4835) ([プロラー](https://github.com/proller)) +- で固定された競合状態 `SELECT` から `system.tables` テーブルの名前が変更または同時に変更された場合。 [\#4836](https://github.com/ClickHouse/ClickHouse/pull/4836) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 固定データが取得データの一部は既に互換性のために残されています。 [\#4839](https://github.com/ClickHouse/ClickHouse/pull/4839) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 固定珍しいデータレースができ起こすこと `RENAME` MergeTree家族のテーブル。 [\#4844](https://github.com/ClickHouse/ClickHouse/pull/4844) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 機能の固定分割の欠陥 `arrayIntersect`. Segmentation faultう場合は関数と呼ばれたとの混合の定数、通常の引数になります。 [\#4847](https://github.com/ClickHouse/ClickHouse/pull/4847) ([Lixiang銭](https://github.com/fancyqlx)) +- からの固定読み取り `Array(LowCardinality)` 列に空の配列の長いシーケンスが含まれている場合はまれです。 [\#4850](https://github.com/ClickHouse/ClickHouse/pull/4850) ([ニコライ-コチェトフ](https://github.com/KochetovNicolai)) +- でクラッシュを修正 `FULL/RIGHT JOIN` nullable対nullableに参加するとき。 [\#4855](https://github.com/ClickHouse/ClickHouse/pull/4855) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- 修正 `No message received` レプリカ間のパーツの取得中は例外です。 [\#4856](https://github.com/ClickHouse/ClickHouse/pull/4856) ([アレサピン](https://github.com/alesapin)) +- 固定 `arrayIntersect` 単一の配列内のいくつかの繰り返し値の場合、関数が間違った結果。 [\#4871](https://github.com/ClickHouse/ClickHouse/pull/4871) ([ニコライ-コチェトフ](https://github.com/KochetovNicolai)) +- 同時実行の間に競合状態を修正する `ALTER COLUMN` サーバークラッシュにつながる可能性のあるクエリ(問題の修正 [\#3421](https://github.com/ClickHouse/ClickHouse/issues/3421)). [\#4592](https://github.com/ClickHouse/ClickHouse/pull/4592) ([アレックス-ザテレピン](https://github.com/ztlpn)) +- 誤った結果を修正 `FULL/RIGHT JOIN` const列を持つ。 [\#4723](https://github.com/ClickHouse/ClickHouse/pull/4723) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- で重複を修正 `GLOBAL JOIN` アスタリスク付き。 [\#4705](https://github.com/ClickHouse/ClickHouse/pull/4705) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- でパラメータ控除を修正 `ALTER MODIFY` 列の `CODEC` 列タイプが指定されていない場合。 [\#4883](https://github.com/ClickHouse/ClickHouse/pull/4883) ([アレサピン](https://github.com/alesapin)) +- 関数 `cutQueryStringAndFragment()` と `queryStringAndFragment()` 今正しく動作するとき `URL` フラグメントとクエリを含みません。 [\#4894](https://github.com/ClickHouse/ClickHouse/pull/4894) ([ヴィタリー-バラノフ](https://github.com/vitlibar)) +- 設定時に珍しいバグを修正 `min_bytes_to_use_direct_io` これは、スレッドが列ファイル内で後方にシークする必要があるときに発生します。 [\#4897](https://github.com/ClickHouse/ClickHouse/pull/4897) ([アレサピン](https://github.com/alesapin)) +- 集計関数の誤った引数型を修正 `LowCardinality` 引数（修正の問題 [\#4919](https://github.com/ClickHouse/ClickHouse/issues/4919)). [\#4922](https://github.com/ClickHouse/ClickHouse/pull/4922) ([ニコライ-コチェトフ](https://github.com/KochetovNicolai)) +- 間違った名前の修飾を修正 `GLOBAL JOIN`. [\#4969](https://github.com/ClickHouse/ClickHouse/pull/4969) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- 修正機能 `toISOWeek` 1970年の結果。 [\#4988](https://github.com/ClickHouse/ClickHouse/pull/4988) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 修正 `DROP`, `TRUNCATE` と `OPTIMIZE` クエリの重複 `ON CLUSTER` のために `ReplicatedMergeTree*` テーブル家族. [\#4991](https://github.com/ClickHouse/ClickHouse/pull/4991) ([アレサピン](https://github.com/alesapin)) #### 下位互換性のない変更 {#backward-incompatible-change-8} -- 名前変更の設定 `insert_sample_with_metadata` 設定する `input_format_defaults_for_omitted_fields`. [\#4771](https://github.com/ClickHouse/ClickHouse/pull/4771) ([Artem Zuikov](https://github.com/4ertus2)) -- 追加された設定 `max_partitions_per_insert_block` （デフォルトでは値100）。 場合に挿入したブロックを含むより多くのパーティション例外がスローされます。 制限を削除する場合は0に設定します（推奨されません）。 [\#4845](https://github.com/ClickHouse/ClickHouse/pull/4845) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- マルチサーチ機能の名称を変更 (`multiPosition` に `multiSearchAllPositions`, `multiSearch` に `multiSearchAny`, `firstMatch` に `multiSearchFirstIndex`). [\#4780](https://github.com/ClickHouse/ClickHouse/pull/4780) ([Danila Kutenin](https://github.com/danlark1)) +- 名前変更の設定 `insert_sample_with_metadata` 設定へ `input_format_defaults_for_omitted_fields`. [\#4771](https://github.com/ClickHouse/ClickHouse/pull/4771) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- 設定を追加 `max_partitions_per_insert_block` （デフォルトでは値100）。 場合に挿入したブロックを含むより多くのパーティション例外がスローされます。 制限を削除する場合は、0に設定します（推奨しません）。 [\#4845](https://github.com/ClickHouse/ClickHouse/pull/4845) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 複数検索機能の名前が変更されました (`multiPosition` に `multiSearchAllPositions`, `multiSearch` に `multiSearchAny`, `firstMatch` に `multiSearchFirstIndex`). [\#4780](https://github.com/ClickHouse/ClickHouse/pull/4780) ([ダニラ-クテニン](https://github.com/danlark1)) #### 性能向上 {#performance-improvement-6} -- 多くの針または多くの同様のbigramsとのクエリのために約5-10％の検索改善を与え、インライン化することによってvolnitsky検索を最適化します。 [\#4862](https://github.com/ClickHouse/ClickHouse/pull/4862) ([Danila Kutenin](https://github.com/danlark1)) -- 設定時のパフォーマンス問題を修正 `use_uncompressed_cache` がゼロより大き登場したときのすべてのデータに含まれる。 [\#4913](https://github.com/ClickHouse/ClickHouse/pull/4913) ([alesapin](https://github.com/alesapin)) +- 多くの針または多くの類似bigramsとのクエリのための約5-10％の検索改善を与え、インラインによってVolnitsky検索を最適化します。 [\#4862](https://github.com/ClickHouse/ClickHouse/pull/4862) ([ダニラ-クテニン](https://github.com/danlark1)) +- 設定時のパフォーマンス問題を修正 `use_uncompressed_cache` がゼロより大き登場したときのすべてのデータに含まれる。 [\#4913](https://github.com/ClickHouse/ClickHouse/pull/4913) ([アレサピン](https://github.com/alesapin)) -#### ビルド/テスト/パッケージの改善 {#buildtestingpackaging-improvement-10} +#### 造り/テスト/包装の改善 {#buildtestingpackaging-improvement-10} -- マークキャッシュとインデックスのメモリ保護を追加します。 これによりメモリの揃い踏みのバグの場合には豆やmsanできます。 [\#4632](https://github.com/ClickHouse/ClickHouse/pull/4632) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Cmake変数のサポートの追加 `ENABLE_PROTOBUF`, `ENABLE_PARQUET` と `ENABLE_BROTLI` これにより、上記の機能を有効/無効にすることができます（librdkafka、mysqlなどでも同じことができます）。 [\#4669](https://github.com/ClickHouse/ClickHouse/pull/4669) ([Silviu Caragea](https://github.com/silviucpp)) -- 追加ダイレクトに刷版を出力するプロセス一覧表示およびstacktracesのすべてのスレッドの場合一部のクエリで吊るされているだけなので後の試験です。 [\#4675](https://github.com/ClickHouse/ClickHouse/pull/4675) ([alesapin](https://github.com/alesapin)) -- 再試行の追加 `Connection loss` エラーで `clickhouse-test`. [\#4682](https://github.com/ClickHouse/ClickHouse/pull/4682) ([alesapin](https://github.com/alesapin)) -- Freebsdのビルドとスレッドのサニタイザを使ったビルドをパッケージャスクリプトに追加します。 [\#4712](https://github.com/ClickHouse/ClickHouse/pull/4712) [\#4748](https://github.com/ClickHouse/ClickHouse/pull/4748) ([alesapin](https://github.com/alesapin)) -- 現在ユーザーのためのパスワードユーザー `'default'` 取付けの間。 [\#4725](https://github.com/ClickHouse/ClickHouse/pull/4725) ([proller](https://github.com/proller)) -- の警告を抑制する `rdkafka` ライブラリ。 [\#4740](https://github.com/ClickHouse/ClickHouse/pull/4740) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Sslなしで構築する能力を許可します。 [\#4750](https://github.com/ClickHouse/ClickHouse/pull/4750) ([proller](https://github.com/proller)) +- デバッグビルドの強化：より詳細なメモリマッピングとASLR。 これによりメモリの揃い踏みのバグの場合には豆やMSanできます。 [\#4632](https://github.com/ClickHouse/ClickHouse/pull/4632) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- Cmake変数のサポートの追加 `ENABLE_PROTOBUF`, `ENABLE_PARQUET` と `ENABLE_BROTLI` これにより、上記の機能を有効/無効にすることができます（librdkafka、mysqlなどの場合と同じです）。 [\#4669](https://github.com/ClickHouse/ClickHouse/pull/4669) ([シルヴィウ-カラゲア](https://github.com/silviucpp)) +- 追加ダイレクトに刷版を出力するプロセス一覧表示およびstacktracesのすべてのスレッドの場合一部のクエリで吊るされているだけなので後の試験です。 [\#4675](https://github.com/ClickHouse/ClickHouse/pull/4675) ([アレサピン](https://github.com/alesapin)) +- 再試行の追加 `Connection loss` エラー `clickhouse-test`. [\#4682](https://github.com/ClickHouse/ClickHouse/pull/4682) ([アレサピン](https://github.com/alesapin)) +- Freebsd build with vagrantをパッケージャスクリプトに追加し、スレッドサニタイザでビルドします。 [\#4712](https://github.com/ClickHouse/ClickHouse/pull/4712) [\#4748](https://github.com/ClickHouse/ClickHouse/pull/4748) ([アレサピン](https://github.com/alesapin)) +- 現在ユーザーのためのパスワードユーザー `'default'` 取付けの間。 [\#4725](https://github.com/ClickHouse/ClickHouse/pull/4725) ([プロラー](https://github.com/proller)) +- 警告を抑制する `rdkafka` 図書館 [\#4740](https://github.com/ClickHouse/ClickHouse/pull/4740) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- Sslなしでビルドできるようにする。 [\#4750](https://github.com/ClickHouse/ClickHouse/pull/4750) ([プロラー](https://github.com/proller)) - カスタムユーザーからclickhouse-serverイメージを起動する方法を追加します。 [\#4753](https://github.com/ClickHouse/ClickHouse/pull/4753) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) -- Contrib boostを1.69にアップグレードします。 [\#4793](https://github.com/ClickHouse/ClickHouse/pull/4793) ([proller](https://github.com/proller)) -- 使用を無効にする `mremap` きめにスレッドに指消毒剤. 驚いたことに、TSanは傍受しません `mremap` （それは傍受しますが `mmap`, `munmap`)それは偽陽性につながります. ステートフルテストで修正TSanレポート。 [\#4859](https://github.com/ClickHouse/ClickHouse/pull/4859) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 追加試験にチェックを使用形式スキーマによhttpインターフェース。 [\#4864](https://github.com/ClickHouse/ClickHouse/pull/4864) ([Vitaly Baranov](https://github.com/vitlibar)) +- Contrib boostを1.69にアップグレードします。 [\#4793](https://github.com/ClickHouse/ClickHouse/pull/4793) ([プロラー](https://github.com/proller)) +- 使用を無効にする `mremap` きめにスレッドに指消毒剤. 驚いたことに、TSanは傍受しません `mremap` （それは傍受しますが `mmap`, `munmap`）それは偽陽性につながる。 ステートフルテストでのTSanレポートの修正 [\#4859](https://github.com/ClickHouse/ClickHouse/pull/4859) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 追加試験にチェックを使用形式スキーマによHTTPインターフェース。 [\#4864](https://github.com/ClickHouse/ClickHouse/pull/4864) ([ヴィタリー-バラノフ](https://github.com/vitlibar)) -## クリックハウスリリース19.4 {#clickhouse-release-19-4} +## ClickHouseリリース19.4 {#clickhouse-release-19-4} ### ClickHouseリリース19.4.4.33,2019-04-17 {#clickhouse-release-19-4-4-33-2019-04-17} #### バグ修正 {#bug-fixes-7} -- 避ける `std::terminate` メモリ割り当てに失敗した場合。 さて `std::bad_alloc` 期待どおりに例外がスローされます。 [\#4665](https://github.com/ClickHouse/ClickHouse/pull/4665) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- バッファからcapnprotoの読み取りを修正します。 時にファイルなロードに成功するhttp. [\#4674](https://github.com/ClickHouse/ClickHouse/pull/4674) ([ウラジスラフ](https://github.com/smirnov-vs)) -- エラーの修正 `Unknown log entry type: 0` 後に `OPTIMIZE TABLE FINAL` クエリ。 [\#4683](https://github.com/ClickHouse/ClickHouse/pull/4683) ([アモスの鳥](https://github.com/amosbird)) -- 間違った引数へ `hasAny` または `hasAll` 関数はsegfaultにつながる可能性があります。 [\#4698](https://github.com/ClickHouse/ClickHouse/pull/4698) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 行き詰まりが発生する恐れがあるとしながら実行 `DROP DATABASE dictionary` クエリ。 [\#4701](https://github.com/ClickHouse/ClickHouse/pull/4701) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 未定義の動作を修正する `median` と `quantile` 機能。 [\#4702](https://github.com/ClickHouse/ClickHouse/pull/4702) ([hcz](https://github.com/hczhcz)) -- ときに圧縮レベル検出を修正 `network_compression_method` 小文字で。 V19.1で壊れた。 [\#4706](https://github.com/ClickHouse/ClickHouse/pull/4706) ([proller](https://github.com/proller)) -- の固定無知 `UTC` 設定（修正の問題 [\#4658](https://github.com/ClickHouse/ClickHouse/issues/4658)). [\#4718](https://github.com/ClickHouse/ClickHouse/pull/4718) ([proller](https://github.com/proller)) -- 修正 `histogram` 関数の振る舞い `Distributed` テーブル。 [\#4741](https://github.com/ClickHouse/ClickHouse/pull/4741) ([olegkv](https://github.com/olegkv)) -- 固定tsanレポート `destroy of a locked mutex`. [\#4742](https://github.com/ClickHouse/ClickHouse/pull/4742) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- システムログ使用時の競合状態によるシャットダウン時のtsanレポートを修正。 part\_logが有効になっているときにシャットダウン時に固定された潜在的な使用後無料。 [\#4758](https://github.com/ClickHouse/ClickHouse/pull/4758) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- パーツの再チェックを修正 `ReplicatedMergeTreeAlterThread` エラーの場合。 [\#4772](https://github.com/ClickHouse/ClickHouse/pull/4772) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 中間集計関数状態に対する算術演算は、定数引数(サブクエリ結果など)に対して機能していませんでした。 [\#4776](https://github.com/ClickHouse/ClickHouse/pull/4776) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 常にメタデータの列名を引用します。 それ以外の場合は、列という名前の表を作成することは不可能です `index` （サーバーは不正な形式のために再起動しません `ATTACH` メタデータ内のクエリ)。 [\#4782](https://github.com/ClickHouse/ClickHouse/pull/4782) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- でクラッシュを修正 `ALTER ... MODIFY ORDER BY` に `Distributed` テーブル。 [\#4790](https://github.com/ClickHouse/ClickHouse/pull/4790) ([Tcheason](https://github.com/TCeason)) -- Segfaultを修正する `JOIN ON` 有効にした場合 `enable_optimize_predicate_expression`. [\#4794](https://github.com/ClickHouse/ClickHouse/pull/4794) ([冬張](https://github.com/zhang2014)) -- カフカからprotobufメッセージを消費した後、余分な行を追加するとバグを修正しました。 [\#4808](https://github.com/ClickHouse/ClickHouse/pull/4808) ([Vitaly Baranov](https://github.com/vitlibar)) -- セグメンテーショ `clickhouse-copier`. [\#4835](https://github.com/ClickHouse/ClickHouse/pull/4835) ([proller](https://github.com/proller)) -- 固定競合状態で `SELECT` から `system.tables` テーブルが同時に名前変更または変更された場合。 [\#4836](https://github.com/ClickHouse/ClickHouse/pull/4836) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 既に廃止されたデータ部分をフェッチする際のデータレースを修正。 [\#4839](https://github.com/ClickHouse/ClickHouse/pull/4839) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 中に発生することができます `RENAME` MergeTree家族のテーブル。 [\#4844](https://github.com/ClickHouse/ClickHouse/pull/4844) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 機能の固定細分化の欠陥 `arrayIntersect`. Segmentation faultう場合は関数と呼ばれたとの混合の定数、通常の引数になります。 [\#4847](https://github.com/ClickHouse/ClickHouse/pull/4847) ([Lixiang Qian](https://github.com/fancyqlx)) -- 固定読み取りから `Array(LowCardinality)` 列に空の配列の長いシーケンスが含まれている場合はまれです。 [\#4850](https://github.com/ClickHouse/ClickHouse/pull/4850) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 修正 `No message received` レプリカ間のパーツの取得中の例外。 [\#4856](https://github.com/ClickHouse/ClickHouse/pull/4856) ([alesapin](https://github.com/alesapin)) -- 固定 `arrayIntersect` 単一の配列のいくつかの繰り返しの値の場合に機能間違った結果。 [\#4871](https://github.com/ClickHouse/ClickHouse/pull/4871) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 同時実行中の競合状態の修正 `ALTER COLUMN` クエリを生み出しうるサーバストレスとソーシャル-サポート問題の修正 [\#3421](https://github.com/ClickHouse/ClickHouse/issues/3421)). [\#4592](https://github.com/ClickHouse/ClickHouse/pull/4592) ([Alex Zatelepin](https://github.com/ztlpn)) -- 修正パラメータ控除で `ALTER MODIFY` 列の `CODEC` 列の型が指定されていない場合。 [\#4883](https://github.com/ClickHouse/ClickHouse/pull/4883) ([alesapin](https://github.com/alesapin)) -- 機能 `cutQueryStringAndFragment()` と `queryStringAndFragment()` 今すぐ正しく動作します `URL` くれないのを返します。 [\#4894](https://github.com/ClickHouse/ClickHouse/pull/4894) ([Vitaly Baranov](https://github.com/vitlibar)) -- 設定時にまれなバグを修正 `min_bytes_to_use_direct_io` これは、スレッドが列ファイル内で逆方向にシークする必要があるときに発生します。 [\#4897](https://github.com/ClickHouse/ClickHouse/pull/4897) ([alesapin](https://github.com/alesapin)) -- 集計関数の誤った引数の型を修正する `LowCardinality` 引数(修正の問題 [\#4919](https://github.com/ClickHouse/ClickHouse/issues/4919)). [\#4922](https://github.com/ClickHouse/ClickHouse/pull/4922) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 修正機能 `toISOWeek` 1970年の結果。 [\#4988](https://github.com/ClickHouse/ClickHouse/pull/4988) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 修正 `DROP`, `TRUNCATE` と `OPTIMIZE` 実行されたときのクエリの重複 `ON CLUSTER` のために `ReplicatedMergeTree*` テーブルの家族。 [\#4991](https://github.com/ClickHouse/ClickHouse/pull/4991) ([alesapin](https://github.com/alesapin)) +- 避ける `std::terminate` メモリ割り当てに失敗した場合。 さて `std::bad_alloc` 例外は期待どおりにスローされます。 [\#4665](https://github.com/ClickHouse/ClickHouse/pull/4665) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- バッファからの読み取りを修正capnproto。 時にファイルなロードに成功するHTTP. [\#4674](https://github.com/ClickHouse/ClickHouse/pull/4674) ([ウラジスラフ](https://github.com/smirnov-vs)) +- 修正エラー `Unknown log entry type: 0` 後に `OPTIMIZE TABLE FINAL` クエリ。 [\#4683](https://github.com/ClickHouse/ClickHouse/pull/4683) ([アモス鳥](https://github.com/amosbird)) +- 間違った引数に `hasAny` または `hasAll` 関数はsegfaultにつながる可能性があります。 [\#4698](https://github.com/ClickHouse/ClickHouse/pull/4698) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 実行中にデッドロックが発生する `DROP DATABASE dictionary` クエリ。 [\#4701](https://github.com/ClickHouse/ClickHouse/pull/4701) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- で未定義の動作を修正 `median` と `quantile` 機能。 [\#4702](https://github.com/ClickHouse/ClickHouse/pull/4702) ([hcz](https://github.com/hczhcz)) +- 圧縮レベル検出時の修正 `network_compression_method` 小文字で。 V19.1で壊れた。 [\#4706](https://github.com/ClickHouse/ClickHouse/pull/4706) ([プロラー](https://github.com/proller)) +- の固定された無知 `UTC` 設定（修正の問題 [\#4658](https://github.com/ClickHouse/ClickHouse/issues/4658)). [\#4718](https://github.com/ClickHouse/ClickHouse/pull/4718) ([プロラー](https://github.com/proller)) +- 修正 `histogram` 関数の動作 `Distributed` テーブル [\#4741](https://github.com/ClickHouse/ClickHouse/pull/4741) ([olegkv](https://github.com/olegkv)) +- 固定tsanレポート `destroy of a locked mutex`. [\#4742](https://github.com/ClickHouse/ClickHouse/pull/4742) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 固定賛育の報告書の停止によるレースの条件のシステムのログ利用 固定の潜在的利用に停止時part\_logが有効になります。 [\#4758](https://github.com/ClickHouse/ClickHouse/pull/4758) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 修正再チェック部品 `ReplicatedMergeTreeAlterThread` エラーの場合。 [\#4772](https://github.com/ClickHouse/ClickHouse/pull/4772) ([ニコライ-コチェトフ](https://github.com/KochetovNicolai)) +- 中間集計関数状態に対する算術演算が、定数引数(サブクエリ結果など)に対して機能していませんでした。 [\#4776](https://github.com/ClickHouse/ClickHouse/pull/4776) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 常にbackquoteカラム名は、メタデータを指すものとします。 それ以外の場合は、列名のテーブルを作成することは不可能です `index` (不正な形式のためサーバーは再起動されません `ATTACH` メタデータ内のクエリ）。 [\#4782](https://github.com/ClickHouse/ClickHouse/pull/4782) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- でクラッシュを修正 `ALTER ... MODIFY ORDER BY` に `Distributed` テーブル。 [\#4790](https://github.com/ClickHouse/ClickHouse/pull/4790) ([Tシーズン](https://github.com/TCeason)) +- でsegfaultを修正 `JOIN ON` 有効にした場合 `enable_optimize_predicate_expression`. [\#4794](https://github.com/ClickHouse/ClickHouse/pull/4794) ([冬張](https://github.com/zhang2014)) +- カフカからprotobufメッセージを消費した後に余分な行を追加するバグを修正しました。 [\#4808](https://github.com/ClickHouse/ClickHouse/pull/4808) ([ヴィタリー-バラノフ](https://github.com/vitlibar)) +- 固定の区分断層に `clickhouse-copier`. [\#4835](https://github.com/ClickHouse/ClickHouse/pull/4835) ([プロラー](https://github.com/proller)) +- で固定された競合状態 `SELECT` から `system.tables` テーブルの名前が変更または同時に変更された場合。 [\#4836](https://github.com/ClickHouse/ClickHouse/pull/4836) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 固定データが取得データの一部は既に互換性のために残されています。 [\#4839](https://github.com/ClickHouse/ClickHouse/pull/4839) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 固定珍しいデータレースができ起こすこと `RENAME` MergeTree家族のテーブル。 [\#4844](https://github.com/ClickHouse/ClickHouse/pull/4844) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 機能の固定分割の欠陥 `arrayIntersect`. Segmentation faultう場合は関数と呼ばれたとの混合の定数、通常の引数になります。 [\#4847](https://github.com/ClickHouse/ClickHouse/pull/4847) ([Lixiang銭](https://github.com/fancyqlx)) +- からの固定読み取り `Array(LowCardinality)` 列に空の配列の長いシーケンスが含まれている場合はまれです。 [\#4850](https://github.com/ClickHouse/ClickHouse/pull/4850) ([ニコライ-コチェトフ](https://github.com/KochetovNicolai)) +- 修正 `No message received` レプリカ間のパーツの取得中は例外です。 [\#4856](https://github.com/ClickHouse/ClickHouse/pull/4856) ([アレサピン](https://github.com/alesapin)) +- 固定 `arrayIntersect` 単一の配列内のいくつかの繰り返し値の場合、関数が間違った結果。 [\#4871](https://github.com/ClickHouse/ClickHouse/pull/4871) ([ニコライ-コチェトフ](https://github.com/KochetovNicolai)) +- 同時実行の間に競合状態を修正する `ALTER COLUMN` サーバークラッシュにつながる可能性のあるクエリ(問題の修正 [\#3421](https://github.com/ClickHouse/ClickHouse/issues/3421)). [\#4592](https://github.com/ClickHouse/ClickHouse/pull/4592) ([アレックス-ザテレピン](https://github.com/ztlpn)) +- でパラメータ控除を修正 `ALTER MODIFY` 列の `CODEC` 列タイプが指定されていない場合。 [\#4883](https://github.com/ClickHouse/ClickHouse/pull/4883) ([アレサピン](https://github.com/alesapin)) +- 関数 `cutQueryStringAndFragment()` と `queryStringAndFragment()` 今正しく動作するとき `URL` フラグメントとクエリを含みません。 [\#4894](https://github.com/ClickHouse/ClickHouse/pull/4894) ([ヴィタリー-バラノフ](https://github.com/vitlibar)) +- 設定時に珍しいバグを修正 `min_bytes_to_use_direct_io` これは、スレッドが列ファイル内で後方にシークする必要があるときに発生します。 [\#4897](https://github.com/ClickHouse/ClickHouse/pull/4897) ([アレサピン](https://github.com/alesapin)) +- 集計関数の誤った引数型を修正 `LowCardinality` 引数（修正の問題 [\#4919](https://github.com/ClickHouse/ClickHouse/issues/4919)). [\#4922](https://github.com/ClickHouse/ClickHouse/pull/4922) ([ニコライ-コチェトフ](https://github.com/KochetovNicolai)) +- 修正機能 `toISOWeek` 1970年の結果。 [\#4988](https://github.com/ClickHouse/ClickHouse/pull/4988) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 修正 `DROP`, `TRUNCATE` と `OPTIMIZE` クエリの重複 `ON CLUSTER` のために `ReplicatedMergeTree*` テーブル家族. [\#4991](https://github.com/ClickHouse/ClickHouse/pull/4991) ([アレサピン](https://github.com/alesapin)) #### 改善 {#improvements-2} -- 普通に保つ, `DEFAULT`, `MATERIALIZED` と `ALIAS` 単一のリストの列（修正の問題 [\#2867](https://github.com/ClickHouse/ClickHouse/issues/2867)). [\#4707](https://github.com/ClickHouse/ClickHouse/pull/4707) ([Alex Zatelepin](https://github.com/ztlpn)) +- 普通に保つ, `DEFAULT`, `MATERIALIZED` と `ALIAS` 単一のリスト内の列（修正の問題 [\#2867](https://github.com/ClickHouse/ClickHouse/issues/2867)). [\#4707](https://github.com/ClickHouse/ClickHouse/pull/4707) ([アレックス-ザテレピン](https://github.com/ztlpn)) ### ClickHouseリリース19.4.3.11,2019-04-02 {#clickhouse-release-19-4-3-11-2019-04-02} #### バグ修正 {#bug-fixes-8} -- でクラッシュを修正 `FULL/RIGHT JOIN` 私たちはnullable対nullableではないに参加するとき. [\#4855](https://github.com/ClickHouse/ClickHouse/pull/4855) ([Artem Zuikov](https://github.com/4ertus2)) -- セグメンテーショ `clickhouse-copier`. [\#4835](https://github.com/ClickHouse/ClickHouse/pull/4835) ([proller](https://github.com/proller)) +- でクラッシュを修正 `FULL/RIGHT JOIN` nullable対nullableに参加するとき。 [\#4855](https://github.com/ClickHouse/ClickHouse/pull/4855) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- 固定の区分断層に `clickhouse-copier`. [\#4835](https://github.com/ClickHouse/ClickHouse/pull/4835) ([プロラー](https://github.com/proller)) -#### ビルド/テスト/パッケージの改善 {#buildtestingpackaging-improvement-11} +#### 造り/テスト/包装の改善 {#buildtestingpackaging-improvement-11} - カスタムユーザーからclickhouse-serverイメージを起動する方法を追加します。 [\#4753](https://github.com/ClickHouse/ClickHouse/pull/4753) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) -### クリックハウスリリース19.4.2.7,2019-03-30 {#clickhouse-release-19-4-2-7-2019-03-30} +### ClickHouseリリース19.4.2.7,2019-03-30 {#clickhouse-release-19-4-2-7-2019-03-30} #### バグ修正 {#bug-fixes-9} -- 固定読み取りから `Array(LowCardinality)` 列に空の配列の長いシーケンスが含まれている場合はまれです。 [\#4850](https://github.com/ClickHouse/ClickHouse/pull/4850) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- からの固定読み取り `Array(LowCardinality)` 列に空の配列の長いシーケンスが含まれている場合はまれです。 [\#4850](https://github.com/ClickHouse/ClickHouse/pull/4850) ([ニコライ-コチェトフ](https://github.com/KochetovNicolai)) -### クリックハウスリリース19.4.1.3,2019-03-19 {#clickhouse-release-19-4-1-3-2019-03-19} +### ClickHouse Release19.4.1.3,2019-03-19 {#clickhouse-release-19-4-1-3-2019-03-19} #### バグ修正 {#bug-fixes-10} -- 両方を含む固定リモートクエリ `LIMIT BY` と `LIMIT`. 以前は、 `LIMIT BY` と `LIMIT` リモートクエリに使用された, `LIMIT` 前に起こる可能性が `LIMIT BY` るもの濾過します。 [\#4708](https://github.com/ClickHouse/ClickHouse/pull/4708) ([Constantin S.Pan](https://github.com/kvap)) +- 両方を含む固定リモートクエリ `LIMIT BY` と `LIMIT`. 以前は、 `LIMIT BY` と `LIMIT` リモートクエリに使用された, `LIMIT` 前に起こる可能性が `LIMIT BY`、あまりにもろ過結果につながった。 [\#4708](https://github.com/ClickHouse/ClickHouse/pull/4708) ([コンスタンティン-S-パン](https://github.com/kvap)) ### ClickHouseリリース19.4.0.49,2019-03-09 {#clickhouse-release-19-4-0-49-2019-03-09} #### 新しい機能 {#new-features-5} -- 追加の完全なサポート `Protobuf` フォーマット(入出力、ネストされたデータ構造)。 [\#4174](https://github.com/ClickHouse/ClickHouse/pull/4174) [\#4493](https://github.com/ClickHouse/ClickHouse/pull/4493) ([Vitaly Baranov](https://github.com/vitlibar)) -- ビットマップを追加しました。 [\#4207](https://github.com/ClickHouse/ClickHouse/pull/4207) ([アンディヤング](https://github.com/andyyzh)) [\#4568](https://github.com/ClickHouse/ClickHouse/pull/4568) ([Vitaly Baranov](https://github.com/vitlibar)) -- 寄木細工の形式のサポート。 [\#4448](https://github.com/ClickHouse/ClickHouse/pull/4448) ([proller](https://github.com/proller)) -- ファジィ文字列の比較のために、n-gram距離が追加されました。 これは、r言語のq-gramメトリックに似ています。 [\#4466](https://github.com/ClickHouse/ClickHouse/pull/4466) ([Danila Kutenin](https://github.com/danlark1)) +- 追加されたフルサポート `Protobuf` フォーマット(入力と出力、入れ子になったデータ構造)。 [\#4174](https://github.com/ClickHouse/ClickHouse/pull/4174) [\#4493](https://github.com/ClickHouse/ClickHouse/pull/4493) ([ヴィタリー-バラノフ](https://github.com/vitlibar)) +- 轟音ビットマップとビットマップ関数を追加しました。 [\#4207](https://github.com/ClickHouse/ClickHouse/pull/4207) ([アンディ-ヤン](https://github.com/andyyzh)) [\#4568](https://github.com/ClickHouse/ClickHouse/pull/4568) ([ヴィタリー-バラノフ](https://github.com/vitlibar)) +- 寄木細工の形式のサポート。 [\#4448](https://github.com/ClickHouse/ClickHouse/pull/4448) ([プロラー](https://github.com/proller)) +- ファジィ文字列の比較のためにN-gram距離を追加しました。 これは、R言語のq-gramメトリックに似ています。 [\#4466](https://github.com/ClickHouse/ClickHouse/pull/4466) ([ダニラ-クテニン](https://github.com/danlark1)) - 結合ルールのための黒鉛rollupから専用の凝集-保持。 [\#4426](https://github.com/ClickHouse/ClickHouse/pull/4426) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) -- 追加 `max_execution_speed` と `max_execution_speed_bytes` リソースの使用を制限する。 追加 `min_execution_speed_bytes` 補完する設定 `min_execution_speed`. [\#4430](https://github.com/ClickHouse/ClickHouse/pull/4430) ([冬張](https://github.com/zhang2014)) -- 機能実装 `flatten`. [\#4555](https://github.com/ClickHouse/ClickHouse/pull/4555) [\#4409](https://github.com/ClickHouse/ClickHouse/pull/4409) ([alexey-milovidov](https://github.com/alexey-milovidov), [kzon](https://github.com/kzon)) -- 機能追加 `arrayEnumerateDenseRanked` と `arrayEnumerateUniqRanked` （それはのようなものだ `arrayEnumerateUniq` しかし、多次元配列の内部を調べるために配列の深さを微調整することができます）。 [\#4475](https://github.com/ClickHouse/ClickHouse/pull/4475) ([proller](https://github.com/proller)) [\#4601](https://github.com/ClickHouse/ClickHouse/pull/4601) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Multiple JOINS with some restrictions: no asterisks, no complex aliases in ON/WHERE/GROUP BY/… [\#4462](https://github.com/ClickHouse/ClickHouse/pull/4462) ([Artem Zuikov](https://github.com/4ertus2)) +- 追加 `max_execution_speed` と `max_execution_speed_bytes` リソース使用量を制限する。 追加 `min_execution_speed_bytes` を補完するように設定する `min_execution_speed`. [\#4430](https://github.com/ClickHouse/ClickHouse/pull/4430) ([冬張](https://github.com/zhang2014)) +- 実装関数 `flatten`. [\#4555](https://github.com/ClickHouse/ClickHouse/pull/4555) [\#4409](https://github.com/ClickHouse/ClickHouse/pull/4409) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov), [kzon](https://github.com/kzon)) +- 追加された機能 `arrayEnumerateDenseRanked` と `arrayEnumerateUniqRanked` （それはのようです `arrayEnumerateUniq` しかし、多次元配列の内部を見るために配列の深さを微調整することができます）。 [\#4475](https://github.com/ClickHouse/ClickHouse/pull/4475) ([プロラー](https://github.com/proller)) [\#4601](https://github.com/ClickHouse/ClickHouse/pull/4601) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- Multiple JOINS with some restrictions: no asterisks, no complex aliases in ON/WHERE/GROUP BY/… [\#4462](https://github.com/ClickHouse/ClickHouse/pull/4462) ([アルテム-ズイコフ](https://github.com/4ertus2)) #### バグ修正 {#bug-fixes-11} -- このリリースも含む全てのバグ修正から19.3 19.1. -- データスキップインデックスのバグを修正：挿入後の顆粒の順序が間違っていた。 [\#4407](https://github.com/ClickHouse/ClickHouse/pull/4407) ([Nikita Vasilev](https://github.com/nikvas0)) -- 固定 `set` インデックス `Nullable` と `LowCardinality` 列。 その前に, `set` 索引とともに `Nullable` または `LowCardinality` 列ledにエラー `Data type must be deserialized with multiple streams` 選択中。 [\#4594](https://github.com/ClickHouse/ClickHouse/pull/4594) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 完全にupdate\_timeを正しく設定する `executable` 辞書の更新。 [\#4551](https://github.com/ClickHouse/ClickHouse/pull/4551) ([Tema Novikov](https://github.com/temoon)) +- このリリースには、19.3と19.1のすべてのバグ修正も含まれています。 +- インデックスをスキップするデータのバグを修正しました。 [\#4407](https://github.com/ClickHouse/ClickHouse/pull/4407) ([ニキータ-ヴァシレフ](https://github.com/nikvas0)) +- 固定 `set` インデックス `Nullable` と `LowCardinality` 列。 その前に, `set` インデックス `Nullable` または `LowCardinality` 列がエラーにつながった `Data type must be deserialized with multiple streams` 選択中。 [\#4594](https://github.com/ClickHouse/ClickHouse/pull/4594) ([ニコライ-コチェトフ](https://github.com/KochetovNicolai)) +- 完全にupdate\_timeを正しく設定 `executable` 辞書の更新。 [\#4551](https://github.com/ClickHouse/ClickHouse/pull/4551) ([テマ-ノヴィコフ](https://github.com/temoon)) - 19.3で壊れたプログレスバーを修正。 [\#4627](https://github.com/ClickHouse/ClickHouse/pull/4627) ([フィリモノフ](https://github.com/filimonov)) -- 特定のケースで、メモリ領域をシュリンクしたときのmemorytrackerの値の不一致を修正しました。 [\#4619](https://github.com/ClickHouse/ClickHouse/pull/4619) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- ThreadPoolでの未定義の動作を修正しました。 [\#4612](https://github.com/ClickHouse/ClickHouse/pull/4612) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- メッセージで非常にまれなクラッシュを修正 `mutex lock failed: Invalid argument` これは、MergeTreeテーブルがSELECTと同時に削除されたときに発生する可能性があります。 [\#4608](https://github.com/ClickHouse/ClickHouse/pull/4608) ([Alex Zatelepin](https://github.com/ztlpn)) -- ODBCドライバとの互換性 `LowCardinality` データ型。 [\#4381](https://github.com/ClickHouse/ClickHouse/pull/4381) ([proller](https://github.com/proller)) -- FreeBSD:Fixup for `AIOcontextPool: Found io_event with unknown id 0` エラー。 [\#4438](https://github.com/ClickHouse/ClickHouse/pull/4438) ([urgordeadbeef](https://github.com/urgordeadbeef)) -- `system.part_log` テーブルは構成に関係なく作成されました。 [\#4483](https://github.com/ClickHouse/ClickHouse/pull/4483) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 未定義の動作を修正する `dictIsIn` キャッシュ辞書の関数。 [\#4515](https://github.com/ClickHouse/ClickHouse/pull/4515) ([alesapin](https://github.com/alesapin)) -- Fixed a deadlock when a SELECT query locks the same table multiple times (e.g. from different threads or when executing multiple subqueries) and there is a concurrent DDL query. [\#4535](https://github.com/ClickHouse/ClickHouse/pull/4535) ([Alex Zatelepin](https://github.com/ztlpn)) -- デフォルトでcompile\_expressionsを無効にします。 `llvm` contribはそれをとのテストし、 `clang` と `asan`. [\#4579](https://github.com/ClickHouse/ClickHouse/pull/4579) ([alesapin](https://github.com/alesapin)) -- 防ぐ `std::terminate` とき `invalidate_query` のために `clickhouse` 外部辞書ソースが間違った結果セット（空または複数の行または複数の列）を返しました。 ときに問題を修正しました `invalidate_query` にかかわらず、五秒ごとに実行されました `lifetime`. [\#4583](https://github.com/ClickHouse/ClickHouse/pull/4583) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- ときにデッドロックを避ける `invalidate_query` 辞書のために `clickhouse` ソースが関与していた `system.dictionaries` テーブルまたは `Dictionaries` データベース（まれなケース）。 [\#4599](https://github.com/ClickHouse/ClickHouse/pull/4599) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- クロスのための修正は、空のwhereに参加します。 [\#4598](https://github.com/ClickHouse/ClickHouse/pull/4598) ([Artem Zuikov](https://github.com/4ertus2)) -- 機能の固定segfault “replicate” 定数引数が渡されるとき。 [\#4603](https://github.com/ClickHouse/ClickHouse/pull/4603) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 固定ラムダ機能と述語オプティマイザ. [\#4408](https://github.com/ClickHouse/ClickHouse/pull/4408) ([冬張](https://github.com/zhang2014)) -- 複数のフィックスを結合します。 [\#4595](https://github.com/ClickHouse/ClickHouse/pull/4595) ([Artem Zuikov](https://github.com/4ertus2)) +- 特定のケースでは、メモリ領域が縮小されたときにMemoryTrackerの矛盾した値を修正しました。 [\#4619](https://github.com/ClickHouse/ClickHouse/pull/4619) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- ThreadPoolの未定義の動作を修正しました。 [\#4612](https://github.com/ClickHouse/ClickHouse/pull/4612) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- メッセージで非常にまれなクラッシュを修正 `mutex lock failed: Invalid argument` これは、MergeTreeテーブルがSELECTと同時に削除された場合に発生する可能性があります。 [\#4608](https://github.com/ClickHouse/ClickHouse/pull/4608) ([アレックス-ザテレピン](https://github.com/ztlpn)) +- ODBCドライバとの互換性 `LowCardinality` データ型。 [\#4381](https://github.com/ClickHouse/ClickHouse/pull/4381) ([プロラー](https://github.com/proller)) +- FreeBSD:フィックスアップ `AIOcontextPool: Found io_event with unknown id 0` エラー [\#4438](https://github.com/ClickHouse/ClickHouse/pull/4438) ([ウルゴルドビーフ](https://github.com/urgordeadbeef)) +- `system.part_log` 設定に関係なくテーブルが作成されました。 [\#4483](https://github.com/ClickHouse/ClickHouse/pull/4483) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 未定義の動作を修正 `dictIsIn` キャッシュ辞書の機能。 [\#4515](https://github.com/ClickHouse/ClickHouse/pull/4515) ([アレサピン](https://github.com/alesapin)) +- Fixed a deadlock when a SELECT query locks the same table multiple times (e.g. from different threads or when executing multiple subqueries) and there is a concurrent DDL query. [\#4535](https://github.com/ClickHouse/ClickHouse/pull/4535) ([アレックス-ザテレピン](https://github.com/ztlpn)) +- デフォルトでcompile\_expressionsを無効にします。 `llvm` contribはそれをとのテストし、 `clang` と `asan`. [\#4579](https://github.com/ClickHouse/ClickHouse/pull/4579) ([アレサピン](https://github.com/alesapin)) +- 防ぐ `std::terminate` とき `invalidate_query` のために `clickhouse` 外部ディクショナリソースから誤った結果セット(空または複数の行または複数の列)が返されました。 問題を修正しました。 `invalidate_query` に関係なく、五秒ごとに行われました `lifetime`. [\#4583](https://github.com/ClickHouse/ClickHouse/pull/4583) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- デッドロックを避けなさい時 `invalidate_query` 辞書の場合 `clickhouse` ソースが関与していた `system.dictionaries` テーブルまたは `Dictionaries` データベース（まれなケース）。 [\#4599](https://github.com/ClickHouse/ClickHouse/pull/4599) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 空の場所でクロス結合の修正。 [\#4598](https://github.com/ClickHouse/ClickHouse/pull/4598) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- 関数内の固定segfault “replicate” 定数引数が渡されたとき。 [\#4603](https://github.com/ClickHouse/ClickHouse/pull/4603) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 述語オプティマイザでラムダ関数を修正。 [\#4408](https://github.com/ClickHouse/ClickHouse/pull/4408) ([冬張](https://github.com/zhang2014)) +- 複数の結合複数の修正。 [\#4595](https://github.com/ClickHouse/ClickHouse/pull/4595) ([アルテム-ズイコフ](https://github.com/4ertus2)) #### 改善 {#improvements-3} -- 支援のエイリアスに参加でき課右テーブル列あります。 [\#4412](https://github.com/ClickHouse/ClickHouse/pull/4412) ([Artem Zuikov](https://github.com/4ertus2)) -- 複数の結合の結果は、サブセレクトで使用される正しい結果名が必要です。 平置き換えエイリアスとソース名ます。 [\#4474](https://github.com/ClickHouse/ClickHouse/pull/4474) ([Artem Zuikov](https://github.com/4ertus2)) -- 結合文のプッシュダウンロジックを改善します。 [\#4387](https://github.com/ClickHouse/ClickHouse/pull/4387) ([イワン](https://github.com/abyss7)) +- 右のテーブルの列のセクションの結合のサポートエイリアス。 [\#4412](https://github.com/ClickHouse/ClickHouse/pull/4412) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- 複数の結合の結果は、サブセレクトで使用する正しい結果名が必要です。 平置き換えエイリアスとソース名ます。 [\#4474](https://github.com/ClickHouse/ClickHouse/pull/4474) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- 改善プッシュダウンの論理のために参加します。 [\#4387](https://github.com/ClickHouse/ClickHouse/pull/4387) ([イワン](https://github.com/abyss7)) -#### 性能の改善 {#performance-improvements-3} +#### 性能の向上 {#performance-improvements-3} -- 改善されたヒューリスティック “move to PREWHERE” 最適化。 [\#4405](https://github.com/ClickHouse/ClickHouse/pull/4405) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 8ビットおよび16ビットのキーにhashtableのapiを使用する適切なルックアップテーブルを使用します。 [\#4536](https://github.com/ClickHouse/ClickHouse/pull/4536) ([アモスの鳥](https://github.com/amosbird)) -- 文字列比較のパフォーマンスの向上。 [\#4564](https://github.com/ClickHouse/ClickHouse/pull/4564) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 分散ddlタスクを処理するメインループが遅くならないように、別のスレッドで分散ddlキューをクリーンアップします。 [\#4502](https://github.com/ClickHouse/ClickHouse/pull/4502) ([Alex Zatelepin](https://github.com/ztlpn)) -- とき `min_bytes_to_use_direct_io` 読み込むデータサイズが圧縮されたブロックのサイズによって過小評価されることがあるため、すべてのファイルがO\_DIRECTモードで開かれたわけでは [\#4526](https://github.com/ClickHouse/ClickHouse/pull/4526) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- のヒューリスティックの改善 “move to PREWHERE” 最適化。 [\#4405](https://github.com/ClickHouse/ClickHouse/pull/4405) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 8ビットキーと16ビットキーにHashTableのAPIを使用する適切なルックアップテーブルを使用します。 [\#4536](https://github.com/ClickHouse/ClickHouse/pull/4536) ([アモス鳥](https://github.com/amosbird)) +- 文字列比較のパフォーマンスの向上。 [\#4564](https://github.com/ClickHouse/ClickHouse/pull/4564) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 分散DDLタスクを処理するメインループが遅くならないように、分散DDLキューを別のスレッドでクリーンアップします。 [\#4502](https://github.com/ClickHouse/ClickHouse/pull/4502) ([アレックス-ザテレピン](https://github.com/ztlpn)) +- とき `min_bytes_to_use_direct_io` 読み込むデータサイズが圧縮されたブロックのサイズによって過小評価されることがあるため、すべてのファイルがO\_DIRECTモードで開かれたわけではあ [\#4526](https://github.com/ClickHouse/ClickHouse/pull/4526) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) -#### ビルド/テスト/パッケージの改善 {#buildtestingpackaging-improvement-12} +#### 造り/テスト/包装の改善 {#buildtestingpackaging-improvement-12} -- Clang-9のサポートを追加 [\#4604](https://github.com/ClickHouse/ClickHouse/pull/4604) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 間違った修正 `__asm__` 指示(再び) [\#4621](https://github.com/ClickHouse/ClickHouse/pull/4621) ([Konstantin Podshumok](https://github.com/podshumok)) -- 設定を指定する機能を追加する `clickhouse-performance-test` コマンドラインから。 [\#4437](https://github.com/ClickHouse/ClickHouse/pull/4437) ([alesapin](https://github.com/alesapin)) -- 統合テストに辞書テストを追加します。 [\#4477](https://github.com/ClickHouse/ClickHouse/pull/4477) ([alesapin](https://github.com/alesapin)) -- 追加のクエリからのベンチマークのサイトを自動化性能試験までを実施。 [\#4496](https://github.com/ClickHouse/ClickHouse/pull/4496) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- `xxhash.h` それは実装の詳細であり、そのシンボルは次の名前空間であるため、外部lz4には存在しません `XXH_NAMESPACE` マクロ Lz4が外部の場合、xxHashも外部になければならず、扶養家族はそれにリンクする必要があります。 [\#4495](https://github.com/ClickHouse/ClickHouse/pull/4495) ([Orivej Desh](https://github.com/orivej)) -- 次の場合にケースを修正 `quantileTiming` 集約関数は、負の引数または浮動小数点引数で呼び出すことができます（これは、未定義の動作消滅器でfuzzテストを修正します）。 [\#4506](https://github.com/ClickHouse/ClickHouse/pull/4506) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- スペルエラー訂正。 [\#4531](https://github.com/ClickHouse/ClickHouse/pull/4531) ([sdk2unit description in lists](https://github.com/sdk2)) -- Macでのコンパイルの修正。 [\#4371](https://github.com/ClickHouse/ClickHouse/pull/4371) ([Vitaly Baranov](https://github.com/vitlibar)) -- FreeBSDおよび様々な異常なビルド設定のためのビルドの修正。 [\#4444](https://github.com/ClickHouse/ClickHouse/pull/4444) ([proller](https://github.com/proller)) +- Clang-9のサポートが追加されました [\#4604](https://github.com/ClickHouse/ClickHouse/pull/4604) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 間違った修正 `__asm__` 指示（再び) [\#4621](https://github.com/ClickHouse/ClickHouse/pull/4621) ([コンスタンチン-ポドシュモク](https://github.com/podshumok)) +- の設定を指定する機能を追加 `clickhouse-performance-test` コマンドラインから。 [\#4437](https://github.com/ClickHouse/ClickHouse/pull/4437) ([アレサピン](https://github.com/alesapin)) +- 統合テストに辞書テストを追加します。 [\#4477](https://github.com/ClickHouse/ClickHouse/pull/4477) ([アレサピン](https://github.com/alesapin)) +- 追加のクエリからのベンチマークのサイトを自動化性能試験までを実施。 [\#4496](https://github.com/ClickHouse/ClickHouse/pull/4496) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- `xxhash.h` これは実装の詳細であり、そのシンボルは名前空間であるため、外部lz4には存在しません `XXH_NAMESPACE` マクロ Lz4が外部の場合、xxHashも外部でなければならず、依存者はそれにリンクする必要があります。 [\#4495](https://github.com/ClickHouse/ClickHouse/pull/4495) ([オリヴェイ-デシュ](https://github.com/orivej)) +- ときにケースを修正 `quantileTiming` 集計関数は、負または浮動小数点引数で呼び出すことができます（これは未定義の振る舞いsanitizerでファズテストを修正します）。 [\#4506](https://github.com/ClickHouse/ClickHouse/pull/4506) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- スペル誤り訂正。 [\#4531](https://github.com/ClickHouse/ClickHouse/pull/4531) ([sdk2](https://github.com/sdk2)) +- Mac上でコンパイルを修正。 [\#4371](https://github.com/ClickHouse/ClickHouse/pull/4371) ([ヴィタリー-バラノフ](https://github.com/vitlibar)) +- FreeBSDおよび様々な異常なビルド構成のビルド修正。 [\#4444](https://github.com/ClickHouse/ClickHouse/pull/4444) ([プロラー](https://github.com/proller)) -## クリックハウスリリース19.3 {#clickhouse-release-19-3} +## ClickHouseリリース19.3 {#clickhouse-release-19-3} ### ClickHouseリリース19.3.9.1,2019-04-02 {#clickhouse-release-19-3-9-1-2019-04-02} #### バグ修正 {#bug-fixes-12} -- でクラッシュを修正 `FULL/RIGHT JOIN` 私たちはnullable対nullableではないに参加するとき. [\#4855](https://github.com/ClickHouse/ClickHouse/pull/4855) ([Artem Zuikov](https://github.com/4ertus2)) -- セグメンテーショ `clickhouse-copier`. [\#4835](https://github.com/ClickHouse/ClickHouse/pull/4835) ([proller](https://github.com/proller)) -- 固定読み取りから `Array(LowCardinality)` 列に空の配列の長いシーケンスが含まれている場合はまれです。 [\#4850](https://github.com/ClickHouse/ClickHouse/pull/4850) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- でクラッシュを修正 `FULL/RIGHT JOIN` nullable対nullableに参加するとき。 [\#4855](https://github.com/ClickHouse/ClickHouse/pull/4855) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- 固定の区分断層に `clickhouse-copier`. [\#4835](https://github.com/ClickHouse/ClickHouse/pull/4835) ([プロラー](https://github.com/proller)) +- からの固定読み取り `Array(LowCardinality)` 列に空の配列の長いシーケンスが含まれている場合はまれです。 [\#4850](https://github.com/ClickHouse/ClickHouse/pull/4850) ([ニコライ-コチェトフ](https://github.com/KochetovNicolai)) -#### ビルド/テスト/パッケージの改善 {#buildtestingpackaging-improvement-13} +#### 造り/テスト/包装の改善 {#buildtestingpackaging-improvement-13} - カスタムユーザーからclickhouse-serverイメージを起動する方法を追加する [\#4753](https://github.com/ClickHouse/ClickHouse/pull/4753) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) -### ClickHouseリリース19.3.7,2019-03-12 {#clickhouse-release-19-3-7-2019-03-12} +### ClickHouse Release19.3.7,2019-03-12 {#clickhouse-release-19-3-7-2019-03-12} #### バグ修正 {#bug-fixes-13} -- ＃3920で修正されたエラー。 このエラ `Unknown codec family code`, `Cannot seek through file` とsegfaults。 このバグはバージョン19.1で最初に登場し、19.1.10および19.3.6までのバージョンに存在します。 [\#4623](https://github.com/ClickHouse/ClickHouse/pull/4623) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- ＃3920のエラーを修正しました。 このエラ `Unknown codec family code`, `Cannot seek through file`）とsegfaults。 このバグはバージョン19.1で最初に登場し、19.1.10および19.3.6までのバージョンに存在します。 [\#4623](https://github.com/ClickHouse/ClickHouse/pull/4623) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) -### ClickHouseリリース19.3.6,2019-03-02 {#clickhouse-release-19-3-6-2019-03-02} +### ClickHouse Release19.3.6,2019-03-02 {#clickhouse-release-19-3-6-2019-03-02} #### バグ修正 {#bug-fixes-14} -- スレッドプールに1000を超えるスレッドがある場合, `std::terminate` が起こるためのスレッド終了します。 [Azat Khuzhin](https://github.com/azat) [\#4485](https://github.com/ClickHouse/ClickHouse/pull/4485) [\#4505](https://github.com/ClickHouse/ClickHouse/pull/4505) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 今では作成することが可能です `ReplicatedMergeTree*` コメントとデフォルトのない列のコメントを持つ表。 また、コーデックの比較を修正。 [\#4523](https://github.com/ClickHouse/ClickHouse/pull/4523) ([alesapin](https://github.com/alesapin)) -- 配列またはタプルとの結合にクラッシュを修正しました。 [\#4552](https://github.com/ClickHouse/ClickHouse/pull/4552) ([Artem Zuikov](https://github.com/4ertus2)) -- メッセージとclickhouse-コピー機で固定クラッシュ `ThreadStatus not created`. [\#4540](https://github.com/ClickHouse/ClickHouse/pull/4540) ([Artem Zuikov](https://github.com/4ertus2)) -- 固定電話を切るサーバー停止の場合は分散ddlsを使用した。 [\#4472](https://github.com/ClickHouse/ClickHouse/pull/4472) ([Alex Zatelepin](https://github.com/ztlpn)) -- 10より大きい列のテキスト形式の解析に関するエラーメッセージで、誤った列番号が出力されました。 [\#4484](https://github.com/ClickHouse/ClickHouse/pull/4484) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- スレッドプールに1000を超えるスレッドがある場合, `std::terminate` が起こるためのスレッド終了します。 [Azat Khuzhin](https://github.com/azat) [\#4485](https://github.com/ClickHouse/ClickHouse/pull/4485) [\#4505](https://github.com/ClickHouse/ClickHouse/pull/4505) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- これで作成することができます `ReplicatedMergeTree*` テーブルのコメントするカラムなデフォルトテーブルとカラムコーデックなコメントがデフォルトする また、コーデックの比較を修正。 [\#4523](https://github.com/ClickHouse/ClickHouse/pull/4523) ([アレサピン](https://github.com/alesapin)) +- 固定時にクラッシュするバグに参加す配列またはタプル. [\#4552](https://github.com/ClickHouse/ClickHouse/pull/4552) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- メッセージとclickhouse-コピー機で固定クラッシュ `ThreadStatus not created`. [\#4540](https://github.com/ClickHouse/ClickHouse/pull/4540) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- 固定電話を切るサーバー停止の場合は分散DDLsを使用した。 [\#4472](https://github.com/ClickHouse/ClickHouse/pull/4472) ([アレックス-ザテレピン](https://github.com/ztlpn)) +- 正しくない列番号が10より大きい列のテキスト形式の解析に関するエラーメッセージに表示されました。 [\#4484](https://github.com/ClickHouse/ClickHouse/pull/4484) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) #### ビルド/テスト/パッケージの改善 {#buildtestingpackaging-improvements-3} -- AVXを有効にした固定ビルド。 [\#4527](https://github.com/ClickHouse/ClickHouse/pull/4527) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- コンパイルされるカーネルの代わりに、既知のバージョンに基づいて拡張会計とio会計を有効にします。 [\#4541](https://github.com/ClickHouse/ClickHouse/pull/4541) ([nvartolomei](https://github.com/nvartolomei)) -- Core\_dumpの設定をスキップできるようにします。size\_limit、制限セットが失敗した場合はスローの代わりに警告します。 [\#4473](https://github.com/ClickHouse/ClickHouse/pull/4473) ([proller](https://github.com/proller)) -- 削除された `inline` タグの `void readBinary(...)` で `Field.cpp`. また、冗長マージ `namespace DB` ブロック。 [\#4530](https://github.com/ClickHouse/ClickHouse/pull/4530) ([hcz](https://github.com/hczhcz)) +- AVXを有効にした固定ビルド。 [\#4527](https://github.com/ClickHouse/ClickHouse/pull/4527) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- コンパイルされているカーネルではなく、既知のバージョンに基づいて拡張会計とIO会計を有効にします。 [\#4541](https://github.com/ClickHouse/ClickHouse/pull/4541) ([nvartolomei](https://github.com/nvartolomei)) +- Core\_dumpの設定をスキップできるようにする。size\_limit、リミットセットが失敗した場合はスローの代わりに警告。 [\#4473](https://github.com/ClickHouse/ClickHouse/pull/4473) ([プロラー](https://github.com/proller)) +- 削除された `inline` のタグ `void readBinary(...)` で `Field.cpp`. また、マージ冗長 `namespace DB` ブロック [\#4530](https://github.com/ClickHouse/ClickHouse/pull/4530) ([hcz](https://github.com/hczhcz)) -### クリックハウスリリース19.3.5,2019-02-21 {#clickhouse-release-19-3-5-2019-02-21} +### ClickHouse Release19.3.5,2019-02-21 {#clickhouse-release-19-3-5-2019-02-21} #### バグ修正 {#bug-fixes-15} -- 大規模なhttp挿入クエリ処理のバグを修正しました。 [\#4454](https://github.com/ClickHouse/ClickHouse/pull/4454) ([alesapin](https://github.com/alesapin)) -- の間違った実装による古いバージョンとの固定後方の非互換性 `send_logs_level` 設定。 [\#4445](https://github.com/ClickHouse/ClickHouse/pull/4445) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- テーブル機能の後方互換性を修正しました `remote` 列のコメントと共に導入。 [\#4446](https://github.com/ClickHouse/ClickHouse/pull/4446) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- 大規模なhttp挿入クエリ処理のバグを修正しました。 [\#4454](https://github.com/ClickHouse/ClickHouse/pull/4454) ([アレサピン](https://github.com/alesapin)) +- 間違った実装のために古いバージョンとの下位互換性を修正しました `send_logs_level` 設定。 [\#4445](https://github.com/ClickHouse/ClickHouse/pull/4445) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- テーブル関数の下位互換性を修正 `remote` 列のコメントで導入。 [\#4446](https://github.com/ClickHouse/ClickHouse/pull/4446) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) -### ClickHouseリリース19.3.4,2019-02-16 {#clickhouse-release-19-3-4-2019-02-16} +### ClickHouse Release19.3.4,2019-02-16 {#clickhouse-release-19-3-4-2019-02-16} #### 改善 {#improvements-4} -- テーブル-インデックスをさせていただく事があり占めのメモリの制限を行うと `ATTACH TABLE` クエリ。 デタッチされた後にテーブルを添付できない可能性を回避しました。 [\#4396](https://github.com/ClickHouse/ClickHouse/pull/4396) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- ZooKeeperから受け取った最大文字列と配列サイズの上限をわずかに上げました。 でも引き続き増加しのサイズ `CLIENT_JVMFLAGS=-Djute.maxbuffer=...` 飼育係に。 [\#4398](https://github.com/ClickHouse/ClickHouse/pull/4398) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- すでにキューに多数のノードがある場合でも、放棄されたレプリカを修復できます。 [\#4399](https://github.com/ClickHouse/ClickHouse/pull/4399) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 必要な引数を一つ追加する `SET` インデックス(最大保存行数)。 [\#4386](https://github.com/ClickHouse/ClickHouse/pull/4386) ([Nikita Vasilev](https://github.com/nikvas0)) +- テーブル-インデックスをさせていただく事があり占めのメモリの制限を行うと `ATTACH TABLE` クエリ。 デタッチ後にテーブルをアタッチできない可能性を回避しました。 [\#4396](https://github.com/ClickHouse/ClickHouse/pull/4396) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- ZooKeeperから受け取った最大文字列と配列サイズの制限を少し上げました。 でも引き続き増加しのサイズ `CLIENT_JVMFLAGS=-Djute.maxbuffer=...` 飼育係に。 [\#4398](https://github.com/ClickHouse/ClickHouse/pull/4398) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- すると修理を断念レプリカでも既に膨大な数のノードがそのキューに挿入します [\#4399](https://github.com/ClickHouse/ClickHouse/pull/4399) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 必要な引数を一つ追加する `SET` インデックス(格納されている最大行数)。 [\#4386](https://github.com/ClickHouse/ClickHouse/pull/4386) ([ニキータ-ヴァシレフ](https://github.com/nikvas0)) #### バグ修正 {#bug-fixes-16} -- 固定 `WITH ROLLUP` 単一のグループの結果 `LowCardinality` キー。 [\#4384](https://github.com/ClickHouse/ClickHouse/pull/4384) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 固定バグの設定指数を落と顆粒が含まれている場合以 `max_rows` 行）。 [\#4386](https://github.com/ClickHouse/ClickHouse/pull/4386) ([Nikita Vasilev](https://github.com/nikvas0)) -- 多くのfreebsdビルドの修正。 [\#4397](https://github.com/ClickHouse/ClickHouse/pull/4397) ([proller](https://github.com/proller)) -- 固定エイリアス置換にクエリサブクエリを含む同じエイリアス（発行 [\#4110](https://github.com/ClickHouse/ClickHouse/issues/4110)). [\#4351](https://github.com/ClickHouse/ClickHouse/pull/4351) ([Artem Zuikov](https://github.com/4ertus2)) +- 固定 `WITH ROLLUP` 単一によるグループの結果 `LowCardinality` キー [\#4384](https://github.com/ClickHouse/ClickHouse/pull/4384) ([ニコライ-コチェトフ](https://github.com/KochetovNicolai)) +- 固定バグの設定指数を落と顆粒が含まれている場合以 `max_rows` 行）。 [\#4386](https://github.com/ClickHouse/ClickHouse/pull/4386) ([ニキータ-ヴァシレフ](https://github.com/nikvas0)) +- 多くのFreeBSDビルド修正。 [\#4397](https://github.com/ClickHouse/ClickHouse/pull/4397) ([プロラー](https://github.com/proller)) +- 固定エイリアス置換にクエリサブクエリを含む同じエイリアス（発行 [\#4110](https://github.com/ClickHouse/ClickHouse/issues/4110)). [\#4351](https://github.com/ClickHouse/ClickHouse/pull/4351) ([アルテム-ズイコフ](https://github.com/4ertus2)) #### ビルド/テスト/パッケージの改善 {#buildtestingpackaging-improvements-4} -- 実行する機能を追加 `clickhouse-server` dockerイメージのステートレステストの場合。 [\#4347](https://github.com/ClickHouse/ClickHouse/pull/4347) ([Vasily Nemkov](https://github.com/Enmk)) +- 実行する機能を追加する `clickhouse-server` docker imageのステートレステストの場合。 [\#4347](https://github.com/ClickHouse/ClickHouse/pull/4347) ([ヴァシーリー-ネムコフ](https://github.com/Enmk)) -### クリックハウスリリース19.3.3,2019-02-13 {#clickhouse-release-19-3-3-2019-02-13} +### ClickHouse Release19.3.3,2019-02-13 {#clickhouse-release-19-3-3-2019-02-13} #### 新しい機能 {#new-features-6} -- を追加しました `KILL MUTATION` いくつかの理由である突然変異を除去することを可能にする声明。 追加 `latest_failed_part`, `latest_fail_time`, `latest_fail_reason` フィールドに `system.mutations` テーブルやtroubleshooting. [\#4287](https://github.com/ClickHouse/ClickHouse/pull/4287) ([Alex Zatelepin](https://github.com/ztlpn)) -- 集計関数の追加 `entropy` シャノンのエントロピーを計算します [\#4238](https://github.com/ClickHouse/ClickHouse/pull/4238) ([Quid37](https://github.com/Quid37)) -- クエリを送信する機能を追加 `INSERT INTO tbl VALUES (....` 分割せずにサーバーに `query` と `data` パーツだ [\#4301](https://github.com/ClickHouse/ClickHouse/pull/4301) ([alesapin](https://github.com/alesapin)) -- の一般的な実装 `arrayWithConstant` 機能が追加されました。 [\#4322](https://github.com/ClickHouse/ClickHouse/pull/4322) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 実装 `NOT BETWEEN` 比較演算子です。 [\#4228](https://github.com/ClickHouse/ClickHouse/pull/4228) ([ドミトリー-ナウモフ](https://github.com/nezed)) +- 追加された `KILL MUTATION` 何らかの理由で立ち往生している突然変異を除去することを可能にする文。 追加 `latest_failed_part`, `latest_fail_time`, `latest_fail_reason` のフィールド `system.mutations` テーブルやtroubleshooting. [\#4287](https://github.com/ClickHouse/ClickHouse/pull/4287) ([アレックス-ザテレピン](https://github.com/ztlpn)) +- 集計関数を追加 `entropy` シャノンエントロピーを計算します [\#4238](https://github.com/ClickHouse/ClickHouse/pull/4238) ([Quid37](https://github.com/Quid37)) +- クエリを送信する機能を追加 `INSERT INTO tbl VALUES (....` 分割せずにサーバーに `query` と `data` 部品だ [\#4301](https://github.com/ClickHouse/ClickHouse/pull/4301) ([アレサピン](https://github.com/alesapin)) +- の汎用実装 `arrayWithConstant` 機能を追加しました。 [\#4322](https://github.com/ClickHouse/ClickHouse/pull/4322) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 実装 `NOT BETWEEN` 比較演算子。 [\#4228](https://github.com/ClickHouse/ClickHouse/pull/4228) ([ドミトリー-ナウモフ](https://github.com/nezed)) - 実装 `sumMapFiltered` 値が合計されるキーの数を制限できるようにするには `sumMap`. [\#4129](https://github.com/ClickHouse/ClickHouse/pull/4129) ([Léo Ercolanelli](https://github.com/ercolanelli-leo)) -- のサポートを追加 `Nullable` タイプ `mysql` テーブル機能。 [\#4198](https://github.com/ClickHouse/ClickHouse/pull/4198) ([Emmanuel Donin de Rosière](https://github.com/edonin)) -- 任意の定数式のサポート `LIMIT` 句。 [\#4246](https://github.com/ClickHouse/ClickHouse/pull/4246) ([k3box](https://github.com/k3box)) -- 追加 `topKWeighted` (符号なし整数)重みを持つ追加の引数を取る集約関数。 [\#4245](https://github.com/ClickHouse/ClickHouse/pull/4245) ([アンドリュー golman](https://github.com/andrewgolman)) -- `StorageJoin` 今サポート `join_any_take_last_row` 同じキーの既存の値を上書きできるようにする設定。 [\#3973](https://github.com/ClickHouse/ClickHouse/pull/3973) ([アモスの鳥](https://github.com/amosbird) -- 機能追加 `toStartOfInterval`. [\#4304](https://github.com/ClickHouse/ClickHouse/pull/4304) ([Vitaly Baranov](https://github.com/vitlibar)) -- 追加 `RowBinaryWithNamesAndTypes` フォーマット。 [\#4200](https://github.com/ClickHouse/ClickHouse/pull/4200) ([Oleg V.Kozlyuk](https://github.com/DarkWanderer)) -- 追加 `IPv4` と `IPv6` データ型。 より効果的な実装 `IPv*` 機能。 [\#3669](https://github.com/ClickHouse/ClickHouse/pull/3669) ([Vasily Nemkov](https://github.com/Enmk)) -- 機能追加 `toStartOfTenMinutes()`. [\#4298](https://github.com/ClickHouse/ClickHouse/pull/4298) ([Vitaly Baranov](https://github.com/vitlibar)) -- 追加 `Protobuf` 出力形式。 [\#4005](https://github.com/ClickHouse/ClickHouse/pull/4005) [\#4158](https://github.com/ClickHouse/ClickHouse/pull/4158) ([Vitaly Baranov](https://github.com/vitlibar)) -- 追加brotli支援のためのhttpインタフェースデータインポート(挿入します). [\#4235](https://github.com/ClickHouse/ClickHouse/pull/4235) ([ミハイル](https://github.com/fandyushin)) -- ユーザーが関数名にタイプミスをしたり、コマンドラインクライアン [\#4239](https://github.com/ClickHouse/ClickHouse/pull/4239) ([Danila Kutenin](https://github.com/danlark1)) +- の追加されたサポート `Nullable` タイプ `mysql` テーブル関数。 [\#4198](https://github.com/ClickHouse/ClickHouse/pull/4198) ([Emmanuel Donin de Rosière](https://github.com/edonin)) +- の任意の定数式のサポート `LIMIT` 句。 [\#4246](https://github.com/ClickHouse/ClickHouse/pull/4246) ([k3box](https://github.com/k3box)) +- 追加 `topKWeighted` (符号なし整数)重みを持つ追加の引数を取る集計関数。 [\#4245](https://github.com/ClickHouse/ClickHouse/pull/4245) ([アンドリュ](https://github.com/andrewgolman)) +- `StorageJoin` 今すぐサポート `join_any_take_last_row` 同じキーの既存の値を上書きできるようにする設定。 [\#3973](https://github.com/ClickHouse/ClickHouse/pull/3973) ([アモス鳥](https://github.com/amosbird) +- 追加された機能 `toStartOfInterval`. [\#4304](https://github.com/ClickHouse/ClickHouse/pull/4304) ([ヴィタリー-バラノフ](https://github.com/vitlibar)) +- 追加 `RowBinaryWithNamesAndTypes` 形式。 [\#4200](https://github.com/ClickHouse/ClickHouse/pull/4200) ([オレグ-コズリュク](https://github.com/DarkWanderer)) +- 追加 `IPv4` と `IPv6` データ型。 より効果的な実装 `IPv*` 機能。 [\#3669](https://github.com/ClickHouse/ClickHouse/pull/3669) ([ヴァシーリー-ネムコフ](https://github.com/Enmk)) +- 追加された機能 `toStartOfTenMinutes()`. [\#4298](https://github.com/ClickHouse/ClickHouse/pull/4298) ([ヴィタリー-バラノフ](https://github.com/vitlibar)) +- 追加 `Protobuf` 出力形式。 [\#4005](https://github.com/ClickHouse/ClickHouse/pull/4005) [\#4158](https://github.com/ClickHouse/ClickHouse/pull/4158) ([ヴィタリー-バラノフ](https://github.com/vitlibar)) +- 追加brotli支援のためのHTTPインタフェースデータインポート(挿入します). [\#4235](https://github.com/ClickHouse/ClickHouse/pull/4235) ([ミハイル](https://github.com/fandyushin)) +- 追加ヒントがユーザーを商品につけられたタ関数の名称又は種コマンドラインです。 [\#4239](https://github.com/ClickHouse/ClickHouse/pull/4239) ([ダニラ-クテニン](https://github.com/danlark1)) - 追加 `Query-Id` サーバーのHTTP応答ヘッダーへ。 [\#4231](https://github.com/ClickHouse/ClickHouse/pull/4231) ([ミハイル](https://github.com/fandyushin)) -#### 実験の特徴 {#experimental-features-2} +#### 実験的な特徴 {#experimental-features-2} -- 追加 `minmax` と `set` データ飛指標MergeTreeテーブルエンジンです。 [\#4143](https://github.com/ClickHouse/ClickHouse/pull/4143) ([Nikita Vasilev](https://github.com/nikvas0)) -- の追加された変換 `CROSS JOIN` に `INNER JOIN` 可能であれば。 [\#4221](https://github.com/ClickHouse/ClickHouse/pull/4221) [\#4266](https://github.com/ClickHouse/ClickHouse/pull/4266) ([Artem Zuikov](https://github.com/4ertus2)) +- 追加 `minmax` と `set` データ飛指標MergeTreeテーブルエンジンです。 [\#4143](https://github.com/ClickHouse/ClickHouse/pull/4143) ([ニキータ-ヴァシレフ](https://github.com/nikvas0)) +- の変換を追加しました `CROSS JOIN` に `INNER JOIN` 可能であれば。 [\#4221](https://github.com/ClickHouse/ClickHouse/pull/4221) [\#4266](https://github.com/ClickHouse/ClickHouse/pull/4266) ([アルテム-ズイコフ](https://github.com/4ertus2)) #### バグ修正 {#bug-fixes-17} -- 固定 `Not found column` 重複する列の場合 `JOIN ON` セクション。 [\#4279](https://github.com/ClickHouse/ClickHouse/pull/4279) ([Artem Zuikov](https://github.com/4ertus2)) -- 作る `START REPLICATED SENDS` コマンド開始レプリケート送信。 [\#4229](https://github.com/ClickHouse/ClickHouse/pull/4229) ([nvartolomei](https://github.com/nvartolomei)) -- 固定集計関数の実行 `Array(LowCardinality)` 引数。 [\#4055](https://github.com/ClickHouse/ClickHouse/pull/4055) ([KochetovNicolai](https://github.com/KochetovNicolai)) -- 修正された間違った行動 `INSERT ... SELECT ... FROM file(...)` クエリとファイルは `CSVWithNames` または `TSVWIthNames` フォーマットと最初のデータ行がありません。 [\#4297](https://github.com/ClickHouse/ClickHouse/pull/4297) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 辞書が利用できない場合辞書リロードに固定クラッシュ。 このバグは19.1.6で登場しました。 [\#4188](https://github.com/ClickHouse/ClickHouse/pull/4188) ([proller](https://github.com/proller)) -- 固定 `ALL JOIN` 右のテーブルに重複しています。 [\#4184](https://github.com/ClickHouse/ClickHouse/pull/4184) ([Artem Zuikov](https://github.com/4ertus2)) -- 固定細分化の欠陥との `use_uncompressed_cache=1` そして、間違った非圧縮サイズの例外。 このバグは19.1.6で登場しました。 [\#4186](https://github.com/ClickHouse/ClickHouse/pull/4186) ([alesapin](https://github.com/alesapin)) -- 固定 `compile_expressions` 大きな（int16以上の）日付の比較を伴うバグ。 [\#4341](https://github.com/ClickHouse/ClickHouse/pull/4341) ([alesapin](https://github.com/alesapin)) -- テーブル関数から選択する固定無限ループ `numbers(0)`. [\#4280](https://github.com/ClickHouse/ClickHouse/pull/4280) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- 固定 `Not found column` 重複する列の場合 `JOIN ON` セクション [\#4279](https://github.com/ClickHouse/ClickHouse/pull/4279) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- 作る `START REPLICATED SENDS` コマ [\#4229](https://github.com/ClickHouse/ClickHouse/pull/4229) ([nvartolomei](https://github.com/nvartolomei)) +- 固定集計関数の実行 `Array(LowCardinality)` 引数。 [\#4055](https://github.com/ClickHouse/ClickHouse/pull/4055) ([コチェトヴニコライ](https://github.com/KochetovNicolai)) +- 間違った動作を修正しました `INSERT ... SELECT ... FROM file(...)` クエリとファイルは `CSVWithNames` または `TSVWIthNames` フォーマットと最初のデータ行がありません。 [\#4297](https://github.com/ClickHouse/ClickHouse/pull/4297) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 固定クラッシュ辞書の再読み込みの場合の辞書をございません。 このバグは19.1.6に登場しました。 [\#4188](https://github.com/ClickHouse/ClickHouse/pull/4188) ([プロラー](https://github.com/proller)) +- 固定 `ALL JOIN` 右の表に重複があります。 [\#4184](https://github.com/ClickHouse/ClickHouse/pull/4184) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- 固定分割の欠陥との `use_uncompressed_cache=1` そして、間違った非圧縮サイズの例外。 このバグは19.1.6に登場しました。 [\#4186](https://github.com/ClickHouse/ClickHouse/pull/4186) ([アレサピン](https://github.com/alesapin)) +- 固定 `compile_expressions` 大きな（int16以上の）日付の比較に関するバグ。 [\#4341](https://github.com/ClickHouse/ClickHouse/pull/4341) ([アレサピン](https://github.com/alesapin)) +- テーブル関数から選択するときの無限ループを修正 `numbers(0)`. [\#4280](https://github.com/ClickHouse/ClickHouse/pull/4280) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) - 述語の最適化を一時的に無効にする `ORDER BY`. [\#3890](https://github.com/ClickHouse/ClickHouse/pull/3890) ([冬張](https://github.com/zhang2014)) -- 固定 `Illegal instruction` 古いCpuでbase64関数を使用するときにエラーが発生しました。 このエラーは、ClickHouseがgcc-8でコンパイルされた場合にのみ再現されています。 [\#4275](https://github.com/ClickHouse/ClickHouse/pull/4275) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 固定 `No message received` エラーが発生との交流PostgreSQL ODBCドライバーを通してTLS接続します。 MySQLのODBCドライバを使用する場合にも、segfaultを修正します。 [\#4170](https://github.com/ClickHouse/ClickHouse/pull/4170) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 固定間違った結果 `Date` と `DateTime` 引数は、条件付き演算子（関数）の分岐で使用されます `if`). 機能のための追加された汎用ケース `if`. [\#4243](https://github.com/ClickHouse/ClickHouse/pull/4243) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- クリックハウス辞書は今内ロード `clickhouse` プロセス。 [\#4166](https://github.com/ClickHouse/ClickHouse/pull/4166) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 固定デッドロック時 `SELECT` テーブルから `File` エンジンは後に再試行されました `No such file or directory` エラー。 [\#4161](https://github.com/ClickHouse/ClickHouse/pull/4161) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 固定レース条件から選択すると `system.tables` を与える `table doesn't exist` エラー。 [\#4313](https://github.com/ClickHouse/ClickHouse/pull/4313) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- `clickhouse-client` でsegfault出口がデータを読み込むためのコマンドラインの提案いたインタラクティブモードになります。 [\#4317](https://github.com/ClickHouse/ClickHouse/pull/4317) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 変異の実行が含むバグを修正しました `IN` 演算子は、誤った結果を生成していた。 [\#4099](https://github.com/ClickHouse/ClickHouse/pull/4099) ([Alex Zatelepin](https://github.com/ztlpn)) -- 固定エラー：データベースがある場合 `Dictionary` エンジン、サーバーの起動時に強制的にロードされるすべての辞書、およびlocalhostからのClickHouseソースを持つ辞書がある場合、辞書はロードできません。 [\#4255](https://github.com/ClickHouse/ClickHouse/pull/4255) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 固定エラー時のシステムログのようにして作成時サーバをシャットダウンしました。 [\#4254](https://github.com/ClickHouse/ClickHouse/pull/4254) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 正しい型を正しく返し、適切にロックを処理する `joinGet` 機能。 [\#4153](https://github.com/ClickHouse/ClickHouse/pull/4153) ([アモスの鳥](https://github.com/amosbird)) +- 固定 `Illegal instruction` 古いCpuでbase64関数を使用するときにエラーが発生しました。 このエラーは、ClickHouseがgcc-8でコンパイルされたときにのみ再現されます。 [\#4275](https://github.com/ClickHouse/ClickHouse/pull/4275) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 固定 `No message received` TLS接続を介してPostgreSQL ODBCドライバと対話するときのエラー。 MySQL ODBCドライバを使用するときにもsegfaultを修正します。 [\#4170](https://github.com/ClickHouse/ClickHouse/pull/4170) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 修正された誤った結果とき `Date` と `DateTime` 引数は、条件演算子（関数）の分岐で使用されます `if`). 関数の一般的なケースを追加しました `if`. [\#4243](https://github.com/ClickHouse/ClickHouse/pull/4243) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- ClickHouse辞書は今内読み込みます `clickhouse` プロセス。 [\#4166](https://github.com/ClickHouse/ClickHouse/pull/4166) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 固定デッドロック時 `SELECT` テーブルから `File` エンジンは後に再試行されました `No such file or directory` エラー [\#4161](https://github.com/ClickHouse/ClickHouse/pull/4161) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- から選択したときに修正された競合状態 `system.tables` を与える `table doesn't exist` エラー [\#4313](https://github.com/ClickHouse/ClickHouse/pull/4313) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- `clickhouse-client` 対話モードで実行された場合、コマンドライン候補のデータをロードしているときに終了時にsegfaultできます。 [\#4317](https://github.com/ClickHouse/ClickHouse/pull/4317) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- バグを修正しました。 `IN` 演算子が誤った結果を生成していた。 [\#4099](https://github.com/ClickHouse/ClickHouse/pull/4099) ([アレックス-ザテレピン](https://github.com/ztlpn)) +- 修正されたエラー：データベースがある場合 `Dictionary` localhostからClickHouseソースを持つ辞書がある場合、辞書はロードできません。 [\#4255](https://github.com/ClickHouse/ClickHouse/pull/4255) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 固定エラー時のシステムログのようにして作成時サーバをシャットダウンしました。 [\#4254](https://github.com/ClickHouse/ClickHouse/pull/4254) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 正しく正しい型を返し、ロックを適切に処理します `joinGet` 機能。 [\#4153](https://github.com/ClickHouse/ClickHouse/pull/4153) ([アモス鳥](https://github.com/amosbird)) - 追加 `sumMapWithOverflow` 機能。 [\#4151](https://github.com/ClickHouse/ClickHouse/pull/4151) ([Léo Ercolanelli](https://github.com/ercolanelli-leo)) -- 固定segfaultと `allow_experimental_multiple_joins_emulation`. [52de2c](https://github.com/ClickHouse/ClickHouse/commit/52de2cd927f7b5257dd67e175f0a5560a48840d0) ([Artem Zuikov](https://github.com/4ertus2)) -- 間違ったとのバグを修正 `Date` と `DateTime` 比較。 [\#4237](https://github.com/ClickHouse/ClickHouse/pull/4237) ([valexey](https://github.com/valexey)) -- 未定義の動作の下で固定ファズテストサニタイザ：追加されたパラメータの型チェック `quantile*Weighted` 機能の系列。 [\#4145](https://github.com/ClickHouse/ClickHouse/pull/4145) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 古いデータパーツの削除が失敗することがある稀な競合状態を修正しました `File not found` エラー。 [\#4378](https://github.com/ClickHouse/ClickHouse/pull/4378) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- /etc/clickhouse-server/configが見つからないインストールパッケージを修正xmlだ [\#4343](https://github.com/ClickHouse/ClickHouse/pull/4343) ([proller](https://github.com/proller)) +- で固定segfault `allow_experimental_multiple_joins_emulation`. [52de2c](https://github.com/ClickHouse/ClickHouse/commit/52de2cd927f7b5257dd67e175f0a5560a48840d0) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- 不正なバグを修正しました `Date` と `DateTime` 比較。 [\#4237](https://github.com/ClickHouse/ClickHouse/pull/4237) ([ヴァレクセイ](https://github.com/valexey)) +- 未定義の動作サニタイザの下で固定ファズテスト：追加されたパラメータ型のチェック `quantile*Weighted` 機能の系列。 [\#4145](https://github.com/ClickHouse/ClickHouse/pull/4145) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 古いデータパーツの削除が失敗する可能性がある場合に、まれな競合状態を修正 `File not found` エラー [\#4378](https://github.com/ClickHouse/ClickHouse/pull/4378) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- /Etc/clickhouse-server/configが欠落しているパッケージをインストールする修正。xml。 [\#4343](https://github.com/ClickHouse/ClickHouse/pull/4343) ([プロラー](https://github.com/proller)) #### ビルド/テスト/パッケージの改善 {#buildtestingpackaging-improvements-5} -- Debianパッケージ：設定に従って/etc/clickhouse-server/preprocessedリンクを修正します。 [\#4205](https://github.com/ClickHouse/ClickHouse/pull/4205) ([proller](https://github.com/proller)) -- FreeBSDのための様々なビルドの修正. [\#4225](https://github.com/ClickHouse/ClickHouse/pull/4225) ([proller](https://github.com/proller)) -- Perftestでテーブルを作成、入力、削除する機能を追加しました。 [\#4220](https://github.com/ClickHouse/ClickHouse/pull/4220) ([alesapin](https://github.com/alesapin)) -- 重複をチェックするスクリプトを追加しました。 [\#4326](https://github.com/ClickHouse/ClickHouse/pull/4326) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- パフォーマンステス [\#4264](https://github.com/ClickHouse/ClickHouse/pull/4264) ([alesapin](https://github.com/alesapin)) -- パッケージはデバッグシンボルとを示唆を設置することができます。 [\#4274](https://github.com/ClickHouse/ClickHouse/pull/4274) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- パフォーマンステストのリファクタリング。 より良いロギングと信号の処理。 [\#4171](https://github.com/ClickHouse/ClickHouse/pull/4171) ([alesapin](https://github.com/alesapin)) -- 匿名のyandexにドキュメントを追加しました。メトリカのデータセット。 [\#4164](https://github.com/ClickHouse/ClickHouse/pull/4164) ([alesapin](https://github.com/alesapin)) -- Аdded tool for converting an old month-partitioned part to the custom-partitioned format. [\#4195](https://github.com/ClickHouse/ClickHouse/pull/4195) ([Alex Zatelepin](https://github.com/ztlpn)) -- 追加docsつのデータセットにs3. [\#4144](https://github.com/ClickHouse/ClickHouse/pull/4144) ([alesapin](https://github.com/alesapin)) -- プル要求の説明からchangelogを作成するスクリプトを追加しました。 [\#4169](https://github.com/ClickHouse/ClickHouse/pull/4169) [\#4173](https://github.com/ClickHouse/ClickHouse/pull/4173) ([KochetovNicolai](https://github.com/KochetovNicolai)) ([KochetovNicolai](https://github.com/KochetovNicolai)) -- Clickhouseの人形モジュールを追加しました。 [\#4182](https://github.com/ClickHouse/ClickHouse/pull/4182) ([Maxim Fedotov](https://github.com/MaxFedotov)) +- Debianパッケージ:設定に従って/etc/clickhouse-server/前処理されたリンクを修正します。 [\#4205](https://github.com/ClickHouse/ClickHouse/pull/4205) ([プロラー](https://github.com/proller)) +- FreeBSDのための様々なビルド修正。 [\#4225](https://github.com/ClickHouse/ClickHouse/pull/4225) ([プロラー](https://github.com/proller)) +- Perftestでテーブルを作成、記入、ドロップする機能を追加しました。 [\#4220](https://github.com/ClickHouse/ClickHouse/pull/4220) ([アレサピン](https://github.com/alesapin)) +- 追加スクリプトチェックを複製します。 [\#4326](https://github.com/ClickHouse/ClickHouse/pull/4326) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- パフォーマ [\#4264](https://github.com/ClickHouse/ClickHouse/pull/4264) ([アレサピン](https://github.com/alesapin)) +- パッケージはデバッグシンボルとを示唆を設置することができます。 [\#4274](https://github.com/ClickHouse/ClickHouse/pull/4274) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- パフォーマンステストのリファクタリング。 より良いロギングと信号処理。 [\#4171](https://github.com/ClickHouse/ClickHouse/pull/4171) ([アレサピン](https://github.com/alesapin)) +- 匿名Yandexのにドキュメントを追加しました。メトリカ-データセット [\#4164](https://github.com/ClickHouse/ClickHouse/pull/4164) ([アレサピン](https://github.com/alesapin)) +- Аdded tool for converting an old month-partitioned part to the custom-partitioned format. [\#4195](https://github.com/ClickHouse/ClickHouse/pull/4195) ([アレックス-ザテレピン](https://github.com/ztlpn)) +- 追加docsつのデータセットにs3. [\#4144](https://github.com/ClickHouse/ClickHouse/pull/4144) ([アレサピン](https://github.com/alesapin)) +- プル要求の説明から変更履歴を作成するスクリプトを追加しました。 [\#4169](https://github.com/ClickHouse/ClickHouse/pull/4169) [\#4173](https://github.com/ClickHouse/ClickHouse/pull/4173) ([コチェトヴニコライ](https://github.com/KochetovNicolai)) ([コチェトヴニコライ](https://github.com/KochetovNicolai)) +- 追加人形のモジュールClickHouse. [\#4182](https://github.com/ClickHouse/ClickHouse/pull/4182) ([マキシム-フェドトフ](https://github.com/MaxFedotov)) - 文書化されていない関数のグループのドキュメントを追加しました。 [\#4168](https://github.com/ClickHouse/ClickHouse/pull/4168) ([冬張](https://github.com/zhang2014)) -- ARMビルドの修正。 [\#4210](https://github.com/ClickHouse/ClickHouse/pull/4210)[\#4306](https://github.com/ClickHouse/ClickHouse/pull/4306) [\#4291](https://github.com/ClickHouse/ClickHouse/pull/4291) ([proller](https://github.com/proller)) ([proller](https://github.com/proller)) -- 辞書テストを実行できるようになりました `ctest`. [\#4189](https://github.com/ClickHouse/ClickHouse/pull/4189) ([proller](https://github.com/proller)) -- さて `/etc/ssl` はデフォルトとして使用されると、ディレクトリにSSL証明書 [\#4167](https://github.com/ClickHouse/ClickHouse/pull/4167) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 開始時にsseおよびavx命令の確認を追加しました。 [\#4234](https://github.com/ClickHouse/ClickHouse/pull/4234) ([Igr](https://github.com/igron99)) -- Initスクリプトは開始までサーバを待機します。 [\#4281](https://github.com/ClickHouse/ClickHouse/pull/4281) ([proller](https://github.com/proller)) +- ARMビルドの修正。 [\#4210](https://github.com/ClickHouse/ClickHouse/pull/4210)[\#4306](https://github.com/ClickHouse/ClickHouse/pull/4306) [\#4291](https://github.com/ClickHouse/ClickHouse/pull/4291) ([プロラー](https://github.com/proller)) ([プロラー](https://github.com/proller)) +- 辞書テストを実行できるようになりました `ctest`. [\#4189](https://github.com/ClickHouse/ClickHouse/pull/4189) ([プロラー](https://github.com/proller)) +- さて `/etc/ssl` SSL証明書の既定のディレクトリとして使用されます。 [\#4167](https://github.com/ClickHouse/ClickHouse/pull/4167) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 起動時にSSEとAVX命令のチェックを追加しました。 [\#4234](https://github.com/ClickHouse/ClickHouse/pull/4234) ([Igr](https://github.com/igron99)) +- Initスクリプトが待つサーバーで開始されます。 [\#4281](https://github.com/ClickHouse/ClickHouse/pull/4281) ([プロラー](https://github.com/proller)) #### 下位互換性のない変更 {#backward-incompatible-changes-1} -- 削除 `allow_experimental_low_cardinality_type` 設定。 `LowCardinality` デー [\#4323](https://github.com/ClickHouse/ClickHouse/pull/4323) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 削減マークのキャッシュされた、圧縮解除されたキャッシュサイズに従ってメインメニューを開きます。 [\#4240](https://github.com/ClickHouse/ClickHouse/pull/4240) ([ロパチンコンスタンチン](https://github.com/k-lopatin) -- キーワードを追加 `INDEX` で `CREATE TABLE` クエリ。 名前のある列 `index` バッククォートまたは二重引用符で囲む必要があります: `` `index` ``. [\#4143](https://github.com/ClickHouse/ClickHouse/pull/4143) ([Nikita Vasilev](https://github.com/nikvas0)) -- `sumMap` を推進する結果の型の代わりにオーバーフロー. 古いの `sumMap` 動作は、以下を使用して取得できます `sumMapWithOverflow` 機能。 [\#4151](https://github.com/ClickHouse/ClickHouse/pull/4151) ([Léo Ercolanelli](https://github.com/ercolanelli-leo)) +- 削除 `allow_experimental_low_cardinality_type` 設定。 `LowCardinality` データ型は運用準備が整いました。 [\#4323](https://github.com/ClickHouse/ClickHouse/pull/4323) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 削減マークのキャッシュされた、圧縮解除されたキャッシュサイズに従ってメインメニューを開きます。 [\#4240](https://github.com/ClickHouse/ClickHouse/pull/4240) ([ロパチン-コンスタンチン](https://github.com/k-lopatin) +- キーワードを追加 `INDEX` で `CREATE TABLE` クエリ。 名前のある列 `index` バッククォートまたは二重引用符で囲む必要があります: `` `index` ``. [\#4143](https://github.com/ClickHouse/ClickHouse/pull/4143) ([ニキータ-ヴァシレフ](https://github.com/nikvas0)) +- `sumMap` を推進する結果の型の代わりにオーバーフロー. 古い `sumMap` 動作は次のようにして取得できます `sumMapWithOverflow` 機能。 [\#4151](https://github.com/ClickHouse/ClickHouse/pull/4151) ([Léo Ercolanelli](https://github.com/ercolanelli-leo)) -#### 性能の改善 {#performance-improvements-4} +#### 性能の向上 {#performance-improvements-4} -- `std::sort` に置き換え `pdqsort` なしのクエリの場合 `LIMIT`. [\#4236](https://github.com/ClickHouse/ClickHouse/pull/4236) ([Evgenii Pravda](https://github.com/kvinty)) -- 現在サーバーの再利用にスレッドからグローバルスレッドプールがあります。 この影響性能の一部のコーナー。 [\#4150](https://github.com/ClickHouse/ClickHouse/pull/4150) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- `std::sort` に置き換え `pdqsort` なしのクエリの場合 `LIMIT`. [\#4236](https://github.com/ClickHouse/ClickHouse/pull/4236) ([エフゲニー-プラウダ](https://github.com/kvinty)) +- 現在サーバーの再利用にスレッドからグローバルスレッドプールがあります。 この影響性能の一部のコーナー。 [\#4150](https://github.com/ClickHouse/ClickHouse/pull/4150) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) #### 改善 {#improvements-5} -- FreeBSDのAIOサポートを実装しました。 [\#4305](https://github.com/ClickHouse/ClickHouse/pull/4305) ([urgordeadbeef](https://github.com/urgordeadbeef)) -- `SELECT * FROM a JOIN b USING a, b` 今すぐ戻る `a` と `b` 左側のテーブルからの列のみ。 [\#4141](https://github.com/ClickHouse/ClickHouse/pull/4141) ([Artem Zuikov](https://github.com/4ertus2)) -- 許可 `-C` オプションクライアントとして `-c` オプション。 [\#4232](https://github.com/ClickHouse/ClickHouse/pull/4232) ([syominsergey](https://github.com/syominsergey)) +- FreeBSDのAIOサポートを実装しました。 [\#4305](https://github.com/ClickHouse/ClickHouse/pull/4305) ([ウルゴルドビーフ](https://github.com/urgordeadbeef)) +- `SELECT * FROM a JOIN b USING a, b` 今すぐ戻る `a` と `b` 左側のテーブルからの列のみ。 [\#4141](https://github.com/ClickHouse/ClickHouse/pull/4141) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- 許可 `-C` として働く顧客の選択 `-c` オプション [\#4232](https://github.com/ClickHouse/ClickHouse/pull/4232) ([ショミンセルゲイ](https://github.com/syominsergey)) - Nowオプション `--password` 使用せずに値を必要とパスワードからstdin. [\#4230](https://github.com/ClickHouse/ClickHouse/pull/4230) ([BSD\_Conqueror](https://github.com/bsd-conqueror)) -- エスケープされていないメタ文字を含む文字列リテラルで強調表示するようにした `LIKE` 式または正規表現。 [\#4327](https://github.com/ClickHouse/ClickHouse/pull/4327) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- クライアントソケッ [\#4213](https://github.com/ClickHouse/ClickHouse/pull/4213) ([nvartolomei](https://github.com/nvartolomei)) +- エスケープされていないメタ文字を含む文字列リテラルの強調表示を追加しました `LIKE` 式または正規表現。 [\#4327](https://github.com/ClickHouse/ClickHouse/pull/4327) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- クライア [\#4213](https://github.com/ClickHouse/ClickHouse/pull/4213) ([nvartolomei](https://github.com/nvartolomei)) - 現在サーバーの進捗報告書くクライアント接続の待機を開始。 [\#4215](https://github.com/ClickHouse/ClickHouse/pull/4215) ([イワン](https://github.com/abyss7)) -- 最適化クエリの理由がわずかに良いメッセージ `optimize_throw_if_noop` 設定は有効です。 [\#4294](https://github.com/ClickHouse/ClickHouse/pull/4294) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- のサポートを追加 `--version` clickhouseサーバーのための選択。 [\#4251](https://github.com/ClickHouse/ClickHouse/pull/4251) ([ロパチンコンスタンチン](https://github.com/k-lopatin)) -- 追加 `--help/-h` オプションへ `clickhouse-server`. [\#4233](https://github.com/ClickHouse/ClickHouse/pull/4233) ([ユーリーバラノフ](https://github.com/yurriy)) -- るためのサポートを追加しましたスカラサブクエリと集計関数の状態ます。 [\#4348](https://github.com/ClickHouse/ClickHouse/pull/4348) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 向上サーバー停止時間の変更を待ってます。 [\#4372](https://github.com/ClickHouse/ClickHouse/pull/4372) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Replicated\_can\_become\_leader設定に関する情報をsystemに追加しました。レトロギングの場合、レプリカなくなります。 [\#4379](https://github.com/ClickHouse/ClickHouse/pull/4379) ([Alex Zatelepin](https://github.com/ztlpn)) +- クエリを最適化する理由を持つやや良いメッセージ `optimize_throw_if_noop` 設定を有効にします。 [\#4294](https://github.com/ClickHouse/ClickHouse/pull/4294) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- の追加されたサポート `--version` clickhouseサーバーのオプション。 [\#4251](https://github.com/ClickHouse/ClickHouse/pull/4251) ([ロパチン-コンスタンチン](https://github.com/k-lopatin)) +- 追加 `--help/-h` オプション `clickhouse-server`. [\#4233](https://github.com/ClickHouse/ClickHouse/pull/4233) ([ユーリー-バラノフ](https://github.com/yurriy)) +- るためのサポートを追加しましたスカラサブクエリと集計関数の状態ます。 [\#4348](https://github.com/ClickHouse/ClickHouse/pull/4348) ([ニコライ-コチェトフ](https://github.com/KochetovNicolai)) +- 向上サーバー停止時間の変更を待ってます。 [\#4372](https://github.com/ClickHouse/ClickHouse/pull/4372) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- Replicated\_can\_become\_leader設定に関する情報をシステムに追加しました。レプリカがリーダーになろうとしない場合は、レプリカとログの追加。 [\#4379](https://github.com/ClickHouse/ClickHouse/pull/4379) ([アレックス-ザテレピン](https://github.com/ztlpn)) -## クリックハウスリリース19.1 {#clickhouse-release-19-1} +## ClickHouseリリース19.1 {#clickhouse-release-19-1} -### ClickHouseリリース19.1.14、2019-03-14 {#clickhouse-release-19-1-14-2019-03-14} +### ClickHouse Release19.1.14,2019-03-14 {#clickhouse-release-19-1-14-2019-03-14} -- 固定エラー `Column ... queried more than once` それは起こるかもしれません設定 `asterisk_left_columns_only` 使用する場合は1に設定します `GLOBAL JOIN` と `SELECT *` （まれなケース）。 この問題は19.3以降には存在しません。 [6bac7d8d](https://github.com/ClickHouse/ClickHouse/pull/4692/commits/6bac7d8d11a9b0d6de0b32b53c47eb2f6f8e7062) ([Artem Zuikov](https://github.com/4ertus2)) +- 修正されたエラー `Column ... queried more than once` これは、 `asterisk_left_columns_only` を使用する場合は1に設定されます `GLOBAL JOIN` と `SELECT *` （まれなケース）。 この問題は19.3以降には存在しません。 [6bac7d8d](https://github.com/ClickHouse/ClickHouse/pull/4692/commits/6bac7d8d11a9b0d6de0b32b53c47eb2f6f8e7062) ([アルテム-ズイコフ](https://github.com/4ertus2)) -### クリックハウスリリース19.1.13,2019-03-12 {#clickhouse-release-19-1-13-2019-03-12} +### ClickHouse Release19.1.13,2019-03-12 {#clickhouse-release-19-1-13-2019-03-12} -このリリースには、19.3.7とまったく同じパッチが含まれています。 +このリリ -### ClickHouseリリース19.1.10、2019-03-03 {#clickhouse-release-19-1-10-2019-03-03} +### ClickHouseリリース19.1.10,2019-03-03 {#clickhouse-release-19-1-10-2019-03-03} -このリリースには、19.3.6とまったく同じパッチが含まれています。 +このリリ -## クリックハウスリリース19.1 {#clickhouse-release-19-1-1} +## ClickHouseリリース19.1 {#clickhouse-release-19-1-1} -### ClickHouseリリース19.1.9,2019-02-21 {#clickhouse-release-19-1-9-2019-02-21} +### ClickHouse Release19.1.9,2019-02-21 {#clickhouse-release-19-1-9-2019-02-21} #### バグ修正 {#bug-fixes-18} -- の間違った実装による古いバージョンとの固定後方の非互換性 `send_logs_level` 設定。 [\#4445](https://github.com/ClickHouse/ClickHouse/pull/4445) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- テーブル機能の後方互換性を修正しました `remote` 列のコメントと共に導入。 [\#4446](https://github.com/ClickHouse/ClickHouse/pull/4446) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- 間違った実装のために古いバージョンとの下位互換性を修正しました `send_logs_level` 設定。 [\#4445](https://github.com/ClickHouse/ClickHouse/pull/4445) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- テーブル関数の下位互換性を修正 `remote` 列のコメントで導入。 [\#4446](https://github.com/ClickHouse/ClickHouse/pull/4446) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) -### クリックハウスリリース19.1.8,2019-02-16 {#clickhouse-release-19-1-8-2019-02-16} +### ClickHouse Release19.1.8,2019-02-16 {#clickhouse-release-19-1-8-2019-02-16} #### バグ修正 {#bug-fixes-19} -- /etc/clickhouse-server/configが見つからないインストールパッケージを修正xmlだ [\#4343](https://github.com/ClickHouse/ClickHouse/pull/4343) ([proller](https://github.com/proller)) +- /Etc/clickhouse-server/configが欠落しているパッケージをインストールする修正。xml。 [\#4343](https://github.com/ClickHouse/ClickHouse/pull/4343) ([プロラー](https://github.com/proller)) -## クリックハウスリリース19.1 {#clickhouse-release-19-1-2} +## ClickHouseリリース19.1 {#clickhouse-release-19-1-2} -### クリックハウスリリース19.1.7,2019-02-15 {#clickhouse-release-19-1-7-2019-02-15} +### ClickHouse Release19.1.7,2019-02-15 {#clickhouse-release-19-1-7-2019-02-15} #### バグ修正 {#bug-fixes-20} -- 正しい型を正しく返し、適切にロックを処理する `joinGet` 機能。 [\#4153](https://github.com/ClickHouse/ClickHouse/pull/4153) ([アモスの鳥](https://github.com/amosbird)) -- 固定エラー時のシステムログのようにして作成時サーバをシャットダウンしました。 [\#4254](https://github.com/ClickHouse/ClickHouse/pull/4254) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 固定エラー：データベースがある場合 `Dictionary` エンジン、サーバーの起動時に強制的にロードされるすべての辞書、およびlocalhostからのClickHouseソースを持つ辞書がある場合、辞書はロードできません。 [\#4255](https://github.com/ClickHouse/ClickHouse/pull/4255) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 変異の実行が含むバグを修正しました `IN` 演算子は、誤った結果を生成していた。 [\#4099](https://github.com/ClickHouse/ClickHouse/pull/4099) ([Alex Zatelepin](https://github.com/ztlpn)) -- `clickhouse-client` でsegfault出口がデータを読み込むためのコマンドラインの提案いたインタラクティブモードになります。 [\#4317](https://github.com/ClickHouse/ClickHouse/pull/4317) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 固定レース条件から選択すると `system.tables` を与える `table doesn't exist` エラー。 [\#4313](https://github.com/ClickHouse/ClickHouse/pull/4313) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 固定デッドロック時 `SELECT` テーブルから `File` エンジンは後に再試行されました `No such file or directory` エラー。 [\#4161](https://github.com/ClickHouse/ClickHouse/pull/4161) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 固定問題:地方clickhouse辞書読み込まれtcpが負荷以内です。 [\#4166](https://github.com/ClickHouse/ClickHouse/pull/4166) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 固定 `No message received` エラーが発生との交流PostgreSQL ODBCドライバーを通してTLS接続します。 MySQLのODBCドライバを使用する場合にも、segfaultを修正します。 [\#4170](https://github.com/ClickHouse/ClickHouse/pull/4170) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- 正しく正しい型を返し、ロックを適切に処理します `joinGet` 機能。 [\#4153](https://github.com/ClickHouse/ClickHouse/pull/4153) ([アモス鳥](https://github.com/amosbird)) +- 固定エラー時のシステムログのようにして作成時サーバをシャットダウンしました。 [\#4254](https://github.com/ClickHouse/ClickHouse/pull/4254) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 修正されたエラー：データベースがある場合 `Dictionary` localhostからClickHouseソースを持つ辞書がある場合、辞書はロードできません。 [\#4255](https://github.com/ClickHouse/ClickHouse/pull/4255) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- バグを修正しました。 `IN` 演算子が誤った結果を生成していた。 [\#4099](https://github.com/ClickHouse/ClickHouse/pull/4099) ([アレックス-ザテレピン](https://github.com/ztlpn)) +- `clickhouse-client` 対話モードで実行された場合、コマンドライン候補のデータをロードしているときに終了時にsegfaultできます。 [\#4317](https://github.com/ClickHouse/ClickHouse/pull/4317) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- から選択したときに修正された競合状態 `system.tables` を与える `table doesn't exist` エラー [\#4313](https://github.com/ClickHouse/ClickHouse/pull/4313) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 固定デッドロック時 `SELECT` テーブルから `File` エンジンは後に再試行されました `No such file or directory` エラー [\#4161](https://github.com/ClickHouse/ClickHouse/pull/4161) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 固定問題:地方ClickHouse辞書読み込まれTCPが負荷以内です。 [\#4166](https://github.com/ClickHouse/ClickHouse/pull/4166) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 固定 `No message received` TLS接続を介してPostgreSQL ODBCドライバと対話するときのエラー。 MySQL ODBCドライバを使用するときにもsegfaultを修正します。 [\#4170](https://github.com/ClickHouse/ClickHouse/pull/4170) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) - 述語の最適化を一時的に無効にする `ORDER BY`. [\#3890](https://github.com/ClickHouse/ClickHouse/pull/3890) ([冬張](https://github.com/zhang2014)) -- テーブル関数から選択する固定無限ループ `numbers(0)`. [\#4280](https://github.com/ClickHouse/ClickHouse/pull/4280) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 固定 `compile_expressions` 大きな（int16以上の）日付の比較を伴うバグ。 [\#4341](https://github.com/ClickHouse/ClickHouse/pull/4341) ([alesapin](https://github.com/alesapin)) -- 固定細分化の欠陥との `uncompressed_cache=1` そして、間違った非圧縮サイズの例外。 [\#4186](https://github.com/ClickHouse/ClickHouse/pull/4186) ([alesapin](https://github.com/alesapin)) -- 固定 `ALL JOIN` 右のテーブルに重複しています。 [\#4184](https://github.com/ClickHouse/ClickHouse/pull/4184) ([Artem Zuikov](https://github.com/4ertus2)) -- 修正された間違った行動 `INSERT ... SELECT ... FROM file(...)` クエリとファイルは `CSVWithNames` または `TSVWIthNames` フォーマットと最初のデータ行がありません。 [\#4297](https://github.com/ClickHouse/ClickHouse/pull/4297) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 固定集計関数の実行 `Array(LowCardinality)` 引数。 [\#4055](https://github.com/ClickHouse/ClickHouse/pull/4055) ([KochetovNicolai](https://github.com/KochetovNicolai)) -- Debianパッケージ：設定に従って/etc/clickhouse-server/preprocessedリンクを修正します。 [\#4205](https://github.com/ClickHouse/ClickHouse/pull/4205) ([proller](https://github.com/proller)) -- 未定義の動作の下で固定ファズテストサニタイザ：追加されたパラメータの型チェック `quantile*Weighted` 機能の系列。 [\#4145](https://github.com/ClickHouse/ClickHouse/pull/4145) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 作る `START REPLICATED SENDS` コマンド開始レプリケート送信。 [\#4229](https://github.com/ClickHouse/ClickHouse/pull/4229) ([nvartolomei](https://github.com/nvartolomei)) -- 固定 `Not found column` セクションの結合で重複する列の場合。 [\#4279](https://github.com/ClickHouse/ClickHouse/pull/4279) ([Artem Zuikov](https://github.com/4ertus2)) -- さて `/etc/ssl` はデフォルトとして使用されると、ディレクトリにSSL証明書 [\#4167](https://github.com/ClickHouse/ClickHouse/pull/4167) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 辞書が利用できない場合辞書リロードに固定クラッシュ。 [\#4188](https://github.com/ClickHouse/ClickHouse/pull/4188) ([proller](https://github.com/proller)) -- 間違ったとのバグを修正 `Date` と `DateTime` 比較。 [\#4237](https://github.com/ClickHouse/ClickHouse/pull/4237) ([valexey](https://github.com/valexey)) -- 固定間違った結果 `Date` と `DateTime` 引数は、条件付き演算子（関数）の分岐で使用されます `if`). 機能のための追加された汎用ケース `if`. [\#4243](https://github.com/ClickHouse/ClickHouse/pull/4243) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- テーブル関数から選択するときの無限ループを修正 `numbers(0)`. [\#4280](https://github.com/ClickHouse/ClickHouse/pull/4280) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 固定 `compile_expressions` 大きな（int16以上の）日付の比較に関するバグ。 [\#4341](https://github.com/ClickHouse/ClickHouse/pull/4341) ([アレサピン](https://github.com/alesapin)) +- 固定分割の欠陥との `uncompressed_cache=1` そして、間違った非圧縮サイズの例外。 [\#4186](https://github.com/ClickHouse/ClickHouse/pull/4186) ([アレサピン](https://github.com/alesapin)) +- 固定 `ALL JOIN` 右の表に重複があります。 [\#4184](https://github.com/ClickHouse/ClickHouse/pull/4184) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- 間違った動作を修正しました `INSERT ... SELECT ... FROM file(...)` クエリとファイルは `CSVWithNames` または `TSVWIthNames` フォーマットと最初のデータ行がありません。 [\#4297](https://github.com/ClickHouse/ClickHouse/pull/4297) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 固定集計関数の実行 `Array(LowCardinality)` 引数。 [\#4055](https://github.com/ClickHouse/ClickHouse/pull/4055) ([コチェトヴニコライ](https://github.com/KochetovNicolai)) +- Debianパッケージ:設定に従って/etc/clickhouse-server/前処理されたリンクを修正します。 [\#4205](https://github.com/ClickHouse/ClickHouse/pull/4205) ([プロラー](https://github.com/proller)) +- 未定義の動作サニタイザの下で固定ファズテスト：追加されたパラメータ型のチェック `quantile*Weighted` 機能の系列。 [\#4145](https://github.com/ClickHouse/ClickHouse/pull/4145) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 作る `START REPLICATED SENDS` コマ [\#4229](https://github.com/ClickHouse/ClickHouse/pull/4229) ([nvartolomei](https://github.com/nvartolomei)) +- 固定 `Not found column` セクションの結合で重複する列の場合。 [\#4279](https://github.com/ClickHouse/ClickHouse/pull/4279) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- さて `/etc/ssl` SSL証明書の既定のディレクトリとして使用されます。 [\#4167](https://github.com/ClickHouse/ClickHouse/pull/4167) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 固定クラッシュ辞書の再読み込みの場合の辞書をございません。 [\#4188](https://github.com/ClickHouse/ClickHouse/pull/4188) ([プロラー](https://github.com/proller)) +- 不正なバグを修正しました `Date` と `DateTime` 比較。 [\#4237](https://github.com/ClickHouse/ClickHouse/pull/4237) ([ヴァレクセイ](https://github.com/valexey)) +- 修正された誤った結果とき `Date` と `DateTime` 引数は、条件演算子（関数）の分岐で使用されます `if`). 関数の一般的なケースを追加しました `if`. [\#4243](https://github.com/ClickHouse/ClickHouse/pull/4243) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) -### ClickHouseリリース19.1.6,2019-01-24 {#clickhouse-release-19-1-6-2019-01-24} +### ClickHouse Release19.1.6,2019-01-24 {#clickhouse-release-19-1-6-2019-01-24} #### 新しい機能 {#new-features-7} -- 表の列ごとのカスタム圧縮コーデック。 [\#3899](https://github.com/ClickHouse/ClickHouse/pull/3899) [\#4111](https://github.com/ClickHouse/ClickHouse/pull/4111) ([alesapin](https://github.com/alesapin), [冬張](https://github.com/zhang2014), [アナトリー](https://github.com/Sindbag)) -- 圧縮コーデックを追加 `Delta`. [\#4052](https://github.com/ClickHouse/ClickHouse/pull/4052) ([alesapin](https://github.com/alesapin)) -- 許可する `ALTER` 圧縮コーデック。 [\#4054](https://github.com/ClickHouse/ClickHouse/pull/4054) ([alesapin](https://github.com/alesapin)) -- 機能追加 `left`, `right`, `trim`, `ltrim`, `rtrim`, `timestampadd`, `timestampsub` SQL標準の互換性のために。 [\#3826](https://github.com/ClickHouse/ClickHouse/pull/3826) ([Ivan Blinkov](https://github.com/blinkov)) -- 書き込みのサポート `HDFS` テーブルと `hdfs` テーブル機能。 [\#4084](https://github.com/ClickHouse/ClickHouse/pull/4084) ([alesapin](https://github.com/alesapin)) -- 大きな干し草の山から複数の定数文字列を検索する機能を追加しました: `multiPosition`, `multiSearch` ,`firstMatch` また `-UTF8`, `-CaseInsensitive`、と `-CaseInsensitiveUTF8` バリアント。 [\#4053](https://github.com/ClickHouse/ClickHouse/pull/4053) ([Danila Kutenin](https://github.com/danlark1)) -- 未使用の破片の切り取ること `SELECT` シャーディングキーによるクエリフィ `optimize_skip_unused_shards`). [\#3851](https://github.com/ClickHouse/ClickHouse/pull/3851) ([Gleb Kanterov](https://github.com/kanterov), [イワン](https://github.com/abyss7)) -- 許可 `Kafka` エンジンを無視するいくつかの構文解析誤りのブロックです。 [\#4094](https://github.com/ClickHouse/ClickHouse/pull/4094) ([イワン](https://github.com/abyss7)) -- のサポートを追加 `CatBoost` マルチクラスモデルの評価。 機能 `modelEvaluate` マルチクラスモデルのクラスごとの生の予測を持つタプルを返します。 `libcatboostmodel.so` で構築する必要があります [\#607](https://github.com/catboost/catboost/pull/607). [\#3959](https://github.com/ClickHouse/ClickHouse/pull/3959) ([KochetovNicolai](https://github.com/KochetovNicolai)) -- 機能追加 `filesystemAvailable`, `filesystemFree`, `filesystemCapacity`. [\#4097](https://github.com/ClickHouse/ClickHouse/pull/4097) ([Boris Granveaud](https://github.com/bgranvea)) -- ハッシュ機能を追加 `xxHash64` と `xxHash32`. [\#3905](https://github.com/ClickHouse/ClickHouse/pull/3905) ([フィリモノフ](https://github.com/filimonov)) -- 追加 `gccMurmurHash` 同じハッシュシードを使用するハッシュ関数(GCC風味のつぶやきハッシュ) [gcc](https://github.com/gcc-mirror/gcc/blob/41d6b10e96a1de98e90a7c0378437c3255814b16/libstdc%2B%2B-v3/include/bits/functional_hash.h#L191) [\#4000](https://github.com/ClickHouse/ClickHouse/pull/4000) ([sundyli](https://github.com/sundy-li)) -- ハッシュ機能を追加 `javaHash`, `hiveHash`. [\#3811](https://github.com/ClickHouse/ClickHouse/pull/3811) ([shangshujie365](https://github.com/shangshujie365)) -- テーブル機能を追加 `remoteSecure`. 機能として動作 `remote` しかし、安全な接続を使用しています。 [\#4088](https://github.com/ClickHouse/ClickHouse/pull/4088) ([proller](https://github.com/proller)) +- 表の列ごとのカスタム圧縮コーデック。 [\#3899](https://github.com/ClickHouse/ClickHouse/pull/3899) [\#4111](https://github.com/ClickHouse/ClickHouse/pull/4111) ([アレサピン](https://github.com/alesapin), [冬張](https://github.com/zhang2014), [アナトリー](https://github.com/Sindbag)) +- 圧縮コーデックを追加 `Delta`. [\#4052](https://github.com/ClickHouse/ClickHouse/pull/4052) ([アレサピン](https://github.com/alesapin)) +- 許可する `ALTER` 圧縮コーデック。 [\#4054](https://github.com/ClickHouse/ClickHouse/pull/4054) ([アレサピン](https://github.com/alesapin)) +- 追加された機能 `left`, `right`, `trim`, `ltrim`, `rtrim`, `timestampadd`, `timestampsub` SQL標準の互換性のため。 [\#3826](https://github.com/ClickHouse/ClickHouse/pull/3826) ([イヴァン-ブリンコフ](https://github.com/blinkov)) +- 書き込みのサポート `HDFS` テーブルと `hdfs` テーブル関数。 [\#4084](https://github.com/ClickHouse/ClickHouse/pull/4084) ([アレサピン](https://github.com/alesapin)) +- Big haystackから複数の定数文字列を検索する機能を追加しました: `multiPosition`, `multiSearch` ,`firstMatch` また `-UTF8`, `-CaseInsensitive`,and `-CaseInsensitiveUTF8` 変種。 [\#4053](https://github.com/ClickHouse/ClickHouse/pull/4053) ([ダニラ-クテニン](https://github.com/danlark1)) +- 未使用の破片の剪定 `SELECT` シャーディングキー(設定 `optimize_skip_unused_shards`). [\#3851](https://github.com/ClickHouse/ClickHouse/pull/3851) ([グレブ-カンテロフ](https://github.com/kanterov), [イワン](https://github.com/abyss7)) +- 許可 `Kafka` ブロックごとにいくつかの解析エラーを無視するエンジン。 [\#4094](https://github.com/ClickHouse/ClickHouse/pull/4094) ([イワン](https://github.com/abyss7)) +- 追加されたサポート `CatBoost` マルチクラスモデルの評価。 関数 `modelEvaluate` マルチクラスモデルのクラスごとの生の予測を持つタプルを返します。 `libcatboostmodel.so` で構築する必要があります [\#607](https://github.com/catboost/catboost/pull/607). [\#3959](https://github.com/ClickHouse/ClickHouse/pull/3959) ([コチェトヴニコライ](https://github.com/KochetovNicolai)) +- 追加された機能 `filesystemAvailable`, `filesystemFree`, `filesystemCapacity`. [\#4097](https://github.com/ClickHouse/ClickHouse/pull/4097) ([ボリス-グランヴォー](https://github.com/bgranvea)) +- ハッシュ機能の追加 `xxHash64` と `xxHash32`. [\#3905](https://github.com/ClickHouse/ClickHouse/pull/3905) ([フィリモノフ](https://github.com/filimonov)) +- 追加 `gccMurmurHash` 同じハッシュシードを使用するハッシュ関数(GCC flavoured Murmur hash) [gcc](https://github.com/gcc-mirror/gcc/blob/41d6b10e96a1de98e90a7c0378437c3255814b16/libstdc%2B%2B-v3/include/bits/functional_hash.h#L191) [\#4000](https://github.com/ClickHouse/ClickHouse/pull/4000) ([スンディリ](https://github.com/sundy-li)) +- ハッシュ機能の追加 `javaHash`, `hiveHash`. [\#3811](https://github.com/ClickHouse/ClickHouse/pull/3811) ([shangshujie365](https://github.com/shangshujie365)) +- テーブル関数を追加 `remoteSecure`. 機能として `remote` しかし、安全な接続を使用します。 [\#4088](https://github.com/ClickHouse/ClickHouse/pull/4088) ([プロラー](https://github.com/proller)) -#### 実験の特徴 {#experimental-features-3} +#### 実験的な特徴 {#experimental-features-3} -- 複数の結合エミュレーションを追加 (`allow_experimental_multiple_joins_emulation` 設定）。 [\#3946](https://github.com/ClickHouse/ClickHouse/pull/3946) ([Artem Zuikov](https://github.com/4ertus2)) +- 複数の結合エミュレーションを追加 (`allow_experimental_multiple_joins_emulation` 設定）。 [\#3946](https://github.com/ClickHouse/ClickHouse/pull/3946) ([アルテム-ズイコフ](https://github.com/4ertus2)) #### バグ修正 {#bug-fixes-21} -- 作る `compiled_expression_cache_size` 設定により限定のデフォルトの低メモリを消費する。 [\#4041](https://github.com/ClickHouse/ClickHouse/pull/4041) ([alesapin](https://github.com/alesapin)) -- レプリケートされたテーブルの変更を実行するスレッドと、zookeeperから設定を更新するスレッドのバグを修正しました。 [\#2947](https://github.com/ClickHouse/ClickHouse/issues/2947) [\#3891](https://github.com/ClickHouse/ClickHouse/issues/3891) [\#3934](https://github.com/ClickHouse/ClickHouse/pull/3934) ([Alex Zatelepin](https://github.com/ztlpn)) -- 分散alterタスクを実行する際の競合状態を修正しました。 レース条件以上のレプリカを実行しようとしたところ、すべてのレプリカのものを除く失敗との飼育係エラーになります。 [\#3904](https://github.com/ClickHouse/ClickHouse/pull/3904) ([Alex Zatelepin](https://github.com/ztlpn)) -- 次の場合にバグを修正する `from_zk` config要素はないのにリフレッシュした後、求め飼育係わる。 [\#2947](https://github.com/ClickHouse/ClickHouse/issues/2947) [\#3947](https://github.com/ClickHouse/ClickHouse/pull/3947) ([Alex Zatelepin](https://github.com/ztlpn)) -- IPv4サブネッ [\#3945](https://github.com/ClickHouse/ClickHouse/pull/3945) ([alesapin](https://github.com/alesapin)) -- 固定クラッシュ (`std::terminate`）まれに、リソースが枯渇したために新しいスレッドを作成できない場合。 [\#3956](https://github.com/ClickHouse/ClickHouse/pull/3956) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- ときにバグを修正 `remote` 間違った制限がinに使用されたときのテーブル関数の実行 `getStructureOfRemoteTable`. [\#4009](https://github.com/ClickHouse/ClickHouse/pull/4009) ([alesapin](https://github.com/alesapin)) -- Netlinkソケットのリークを修正します。 これらのソケットはプール内に置かれ、削除されることはなく、現在のすべてのソケットが使用されているときに、新しいソケットが新しいスレッド [\#4017](https://github.com/ClickHouse/ClickHouse/pull/4017) ([Alex Zatelepin](https://github.com/ztlpn)) -- 閉じるとバグを修正 `/proc/self/fd` すべてのfdsが読み込まれたディレクトリ `/proc` フォーク後 `odbc-bridge` サブプロセス。 [\#4120](https://github.com/ClickHouse/ClickHouse/pull/4120) ([alesapin](https://github.com/alesapin)) -- 主キーの使用文字列の場合にはuint単調変換するための固定文字列。 [\#3870](https://github.com/ClickHouse/ClickHouse/pull/3870) ([冬張](https://github.com/zhang2014)) -- 整数変換関数の単調性の計算におけるエラーを修正しました。 [\#3921](https://github.com/ClickHouse/ClickHouse/pull/3921) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 固定segfaultで `arrayEnumerateUniq`, `arrayEnumerateDense` いくつかの無効な引数の場合の関数。 [\#3909](https://github.com/ClickHouse/ClickHouse/pull/3909) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- StorageMergeでUBを修正. [\#3910](https://github.com/ClickHouse/ClickHouse/pull/3910) ([アモスの鳥](https://github.com/amosbird)) -- 機能の固定segfault `addDays`, `subtractDays`. [\#3913](https://github.com/ClickHouse/ClickHouse/pull/3913) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 固定エラー：機能 `round`, `floor`, `trunc`, `ceil` を返すことが偽の結果が実行される整数の引数と大きな負のです。 [\#3914](https://github.com/ClickHouse/ClickHouse/pull/3914) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- によって誘導されるバグを修正 ‘kill query sync’ これはコアダンプにつながります。 [\#3916](https://github.com/ClickHouse/ClickHouse/pull/3916) ([muVulDeePecker](https://github.com/fancyqlx)) -- のを修正した。長の遅延の後に空の複製します。 [\#3928](https://github.com/ClickHouse/ClickHouse/pull/3928) [\#3932](https://github.com/ClickHouse/ClickHouse/pull/3932) ([alesapin](https://github.com/alesapin)) -- テーブルに挿入する場合の過度のメモリ使用量を修正しました `LowCardinality` 主キー。 [\#3955](https://github.com/ClickHouse/ClickHouse/pull/3955) ([KochetovNicolai](https://github.com/KochetovNicolai)) -- 固定 `LowCardinality` のためのシリアル化 `Native` 空の配列の場合の形式。 [\#3907](https://github.com/ClickHouse/ClickHouse/issues/3907) [\#4011](https://github.com/ClickHouse/ClickHouse/pull/4011) ([KochetovNicolai](https://github.com/KochetovNicolai)) -- 単一のlowcardinality数値列によってdistinctを使用している間、不正な結果を修正しました。 [\#3895](https://github.com/ClickHouse/ClickHouse/issues/3895) [\#4012](https://github.com/ClickHouse/ClickHouse/pull/4012) ([KochetovNicolai](https://github.com/KochetovNicolai)) -- 低カーディナリティキーを使用した特殊な集計を修正しました(以下の場合 `compile` 設定が有効になっています)。 [\#3886](https://github.com/ClickHouse/ClickHouse/pull/3886) ([KochetovNicolai](https://github.com/KochetovNicolai)) -- 固定ユーザとパスワードを転送のための複製のテーブルのクエリ. [\#3957](https://github.com/ClickHouse/ClickHouse/pull/3957) ([alesapin](https://github.com/alesapin)) ([小路](https://github.com/nicelulu)) -- 固定非常に珍しい競合状態とされるようにすることが一覧表の辞書データベースをリロードを生成する事ができます。 [\#3970](https://github.com/ClickHouse/ClickHouse/pull/3970) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- ロールアップまたはcubeで使用されたときの不正な結果を修正しました。 [\#3756](https://github.com/ClickHouse/ClickHouse/issues/3756) [\#3837](https://github.com/ClickHouse/ClickHouse/pull/3837) ([サム-チョウ](https://github.com/reflection)) -- クエリの固定列エイリアス `JOIN ON` 構文と分散テーブル。 [\#3980](https://github.com/ClickHouse/ClickHouse/pull/3980) ([冬張](https://github.com/zhang2014)) -- の内部実装における固定エラー `quantileTDigest` （アルテムVakhrushevによって発見）。 このエラーはClickHouseでは決して起こらず、ClickHouseコードベースをライブラリとして直接使用する人にのみ関連していました。 [\#3935](https://github.com/ClickHouse/ClickHouse/pull/3935) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- 作る `compiled_expression_cache_size` 設定により限定のデフォルトの低メモリを消費する。 [\#4041](https://github.com/ClickHouse/ClickHouse/pull/4041) ([アレサピン](https://github.com/alesapin)) +- レプリケートされたテーブルの変更を実行するスレッドと、ZooKeeperから設定を更新するスレッドでハングアップするバグを修正しました。 [\#2947](https://github.com/ClickHouse/ClickHouse/issues/2947) [\#3891](https://github.com/ClickHouse/ClickHouse/issues/3891) [\#3934](https://github.com/ClickHouse/ClickHouse/pull/3934) ([アレックス-ザテレピン](https://github.com/ztlpn)) +- 分散型変更タスクを実行するときの競合状態を修正しました。 競合状態により、複数のレプリカがタスクを実行しようとし、ZooKeeperエラーで失敗したものを除くすべてのレプリカが発生しました。 [\#3904](https://github.com/ClickHouse/ClickHouse/pull/3904) ([アレックス-ザテレピン](https://github.com/ztlpn)) +- ときにバグを修正 `from_zk` zookeeperへの要求がタイムアウトした後、config要素が更新されませんでした。 [\#2947](https://github.com/ClickHouse/ClickHouse/issues/2947) [\#3947](https://github.com/ClickHouse/ClickHouse/pull/3947) ([アレックス-ザテレピン](https://github.com/ztlpn)) +- 修正のバグが間違っているとの接頭辞IPv4サブネットマスクを商品化しました。 [\#3945](https://github.com/ClickHouse/ClickHouse/pull/3945) ([アレサピン](https://github.com/alesapin)) +- 固定クラッシュ (`std::terminate`)まれに、リソースの消耗により新しいスレッドが作成できない場合があります。 [\#3956](https://github.com/ClickHouse/ClickHouse/pull/3956) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- ときにバグを修正 `remote` inに対して間違った制限が使用された場合の表関数の実行 `getStructureOfRemoteTable`. [\#4009](https://github.com/ClickHouse/ClickHouse/pull/4009) ([アレサピン](https://github.com/alesapin)) +- Netlinkソケットの漏れを修正しました。 これらは削除されなかったプールに配置され、現在のすべてのソケットが使用されていたときに新しいスレッドの開始時に新しいソケットが作成 [\#4017](https://github.com/ClickHouse/ClickHouse/pull/4017) ([アレックス-ザテレピン](https://github.com/ztlpn)) +- 閉じるとバグを修正 `/proc/self/fd` すべてのfdsが読み取られた前のディレクトリ `/proc` フォーク後 `odbc-bridge` サブプロセス [\#4120](https://github.com/ClickHouse/ClickHouse/pull/4120) ([アレサピン](https://github.com/alesapin)) +- 主キーの使用文字列の場合にuint単調変換に文字列を修正しました。 [\#3870](https://github.com/ClickHouse/ClickHouse/pull/3870) ([冬張](https://github.com/zhang2014)) +- 整数変換関数の単調性の計算エラーを修正しました。 [\#3921](https://github.com/ClickHouse/ClickHouse/pull/3921) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- で固定segfault `arrayEnumerateUniq`, `arrayEnumerateDense` いくつかの無効な引数の場合の関数。 [\#3909](https://github.com/ClickHouse/ClickHouse/pull/3909) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- StorageMergeでubを修正しました。 [\#3910](https://github.com/ClickHouse/ClickHouse/pull/3910) ([アモス鳥](https://github.com/amosbird)) +- 関数内の固定segfault `addDays`, `subtractDays`. [\#3913](https://github.com/ClickHouse/ClickHouse/pull/3913) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 固定エラー：関数 `round`, `floor`, `trunc`, `ceil` 整数引数と大きな負のスケールで実行すると、偽の結果を返す可能性があります。 [\#3914](https://github.com/ClickHouse/ClickHouse/pull/3914) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- によって誘発されるバグを修正 ‘kill query sync’ これはコアダンプにつながります。 [\#3916](https://github.com/ClickHouse/ClickHouse/pull/3916) ([マブルディーペッカー](https://github.com/fancyqlx)) +- 空の複製キューの後に長い遅延のバグを修正しました。 [\#3928](https://github.com/ClickHouse/ClickHouse/pull/3928) [\#3932](https://github.com/ClickHouse/ClickHouse/pull/3932) ([アレサピン](https://github.com/alesapin)) +- テーブルに挿入する場合の過剰なメモリ使用量を修正しました `LowCardinality` 主キー。 [\#3955](https://github.com/ClickHouse/ClickHouse/pull/3955) ([コチェトヴニコライ](https://github.com/KochetovNicolai)) +- 固定 `LowCardinality` のシリアル化 `Native` 空の配列の場合の書式。 [\#3907](https://github.com/ClickHouse/ClickHouse/issues/3907) [\#4011](https://github.com/ClickHouse/ClickHouse/pull/4011) ([コチェトヴニコライ](https://github.com/KochetovNicolai)) +- に修正結果が異なる単一LowCardinality数値カラムです。 [\#3895](https://github.com/ClickHouse/ClickHouse/issues/3895) [\#4012](https://github.com/ClickHouse/ClickHouse/pull/4012) ([コチェトヴニコライ](https://github.com/KochetovNicolai)) +- 低カーディナリティキーを持つ特殊化された集計を修正しました `compile` 設定が有効になっています）。 [\#3886](https://github.com/ClickHouse/ClickHouse/pull/3886) ([コチェトヴニコライ](https://github.com/KochetovNicolai)) +- 固定ユーザとパスワードを転送のための複製のテーブルのクエリ. [\#3957](https://github.com/ClickHouse/ClickHouse/pull/3957) ([アレサピン](https://github.com/alesapin)) ([小路](https://github.com/nicelulu)) +- 固定非常に珍しい競合状態とされるようにすることが一覧表の辞書データベースをリロードを生成する事ができます。 [\#3970](https://github.com/ClickHouse/ClickHouse/pull/3970) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- に修正が発生した場合の結果を使用したとROLLUPはュールをご用意しております。 [\#3756](https://github.com/ClickHouse/ClickHouse/issues/3756) [\#3837](https://github.com/ClickHouse/ClickHouse/pull/3837) ([サム-チョウ](https://github.com/reflection)) +- クエリの列エイリアスを修正しました。 `JOIN ON` 構文と分散テーブル。 [\#3980](https://github.com/ClickHouse/ClickHouse/pull/3980) ([冬張](https://github.com/zhang2014)) +- 内部実装のエラーを修正しました。 `quantileTDigest` （Artem Vakhrushevによって発見された）。 このエラーはClickHouseでは決して発生せず、ClickHouseコードベースをライブラリとして直接使用する人にのみ関連していました。 [\#3935](https://github.com/ClickHouse/ClickHouse/pull/3935) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) #### 改善 {#improvements-6} -- のサポート `IF NOT EXISTS` で `ALTER TABLE ADD COLUMN` と一緒に文 `IF EXISTS` で `DROP/MODIFY/CLEAR/COMMENT COLUMN`. [\#3900](https://github.com/ClickHouse/ClickHouse/pull/3900) ([Boris Granveaud](https://github.com/bgranvea)) -- 機能 `parseDateTimeBestEffort`:形式のサポート `DD.MM.YYYY`, `DD.MM.YY`, `DD-MM-YYYY`, `DD-Mon-YYYY`, `DD/Month/YYYY` と似ています。 [\#3922](https://github.com/ClickHouse/ClickHouse/pull/3922) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- `CapnProtoInputStream` 今サポートギザギザの構造。 [\#4063](https://github.com/ClickHouse/ClickHouse/pull/4063) ([Odin Hultgren Van Der Horst](https://github.com/Miniwoffer)) -- ユーザビリティ向上に追加チェックがサーバプロセスからのデータディレクトリはオーナーを想定しています。 できない開始のサーバーからのルートデータが得られない場合には所属非rootユーザーです。 [\#3785](https://github.com/ClickHouse/ClickHouse/pull/3785) ([sergey-v-galtsev](https://github.com/sergey-v-galtsev)) -- 結合によるクエリの分析中に必要な列をチェックするロジックの改善。 [\#3930](https://github.com/ClickHouse/ClickHouse/pull/3930) ([Artem Zuikov](https://github.com/4ertus2)) -- 単一のサーバーに多数の分散テーブルがある場合の接続数を減らしました。 [\#3726](https://github.com/ClickHouse/ClickHouse/pull/3726) ([冬張](https://github.com/zhang2014)) -- サポートされている合計行 `WITH TOTALS` ODBCドライバのクエリ。 [\#3836](https://github.com/ClickHouse/ClickHouse/pull/3836) ([Maksim Koritckiy](https://github.com/nightweb)) -- 使用を許可する `Enum`関数の中の整数としてのs。 [\#3875](https://github.com/ClickHouse/ClickHouse/pull/3875) ([イワン](https://github.com/abyss7)) -- 追加 `low_cardinality_allow_in_native_format` 設定。 無効の場合は、使用しないでください `LowCadrinality` タイプイン `Native` フォーマット。 [\#3879](https://github.com/ClickHouse/ClickHouse/pull/3879) ([KochetovNicolai](https://github.com/KochetovNicolai)) -- 削除の冗長化物からの集計表現のキャッシュの低メモリ使用量 [\#4042](https://github.com/ClickHouse/ClickHouse/pull/4042) ([alesapin](https://github.com/alesapin)) -- チェックを追加する `SET send_logs_level = 'value'` クエリーを受け適切な値です。 [\#3873](https://github.com/ClickHouse/ClickHouse/pull/3873) ([Sabyanin Maxim](https://github.com/s-mx)) -- タイプ変換関数で固定されたデータ型のチェック。 [\#3896](https://github.com/ClickHouse/ClickHouse/pull/3896) ([冬張](https://github.com/zhang2014)) +- のサポート `IF NOT EXISTS` で `ALTER TABLE ADD COLUMN` と一緒に文 `IF EXISTS` で `DROP/MODIFY/CLEAR/COMMENT COLUMN`. [\#3900](https://github.com/ClickHouse/ClickHouse/pull/3900) ([ボリス-グランヴォー](https://github.com/bgranvea)) +- 関数 `parseDateTimeBestEffort`：形式のサポート `DD.MM.YYYY`, `DD.MM.YY`, `DD-MM-YYYY`, `DD-Mon-YYYY`, `DD/Month/YYYY` と同様。 [\#3922](https://github.com/ClickHouse/ClickHouse/pull/3922) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- `CapnProtoInputStream` 今ギザギザの構造をサポート。 [\#4063](https://github.com/ClickHouse/ClickHouse/pull/4063) ([稼動に伴Hultgren Van Der Horst](https://github.com/Miniwoffer)) +- ユーザビリティ向上に追加チェックがサーバプロセスからのデータディレクトリはオーナーを想定しています。 できない開始のサーバーからのルートデータが得られない場合には所属非rootユーザーです。 [\#3785](https://github.com/ClickHouse/ClickHouse/pull/3785) ([セルゲイ-v-ガルツェフ](https://github.com/sergey-v-galtsev)) +- より良いロジックの確認に必要なカラムの中での分析クエリを処理するクラウドの場合が仲間入り。 [\#3930](https://github.com/ClickHouse/ClickHouse/pull/3930) ([アルテム-ズイコフ](https://github.com/4ertus2)) +- 単一のサーバーに多数の分散テーブルがある場合の接続数を減少させました。 [\#3726](https://github.com/ClickHouse/ClickHouse/pull/3726) ([冬張](https://github.com/zhang2014)) +- サポートされている合計行 `WITH TOTALS` ODBCドライバのクエリ。 [\#3836](https://github.com/ClickHouse/ClickHouse/pull/3836) ([マクシム-コリツキー](https://github.com/nightweb)) +- 使用を許可 `Enum`if関数内の整数としてのs。 [\#3875](https://github.com/ClickHouse/ClickHouse/pull/3875) ([イワン](https://github.com/abyss7)) +- 追加 `low_cardinality_allow_in_native_format` 設定。 無効の場合は、使用しないでください `LowCadrinality` 入力 `Native` 形式。 [\#3879](https://github.com/ClickHouse/ClickHouse/pull/3879) ([コチェトヴニコライ](https://github.com/KochetovNicolai)) +- 削除の冗長化物からの集計表現のキャッシュの低メモリ使用量 [\#4042](https://github.com/ClickHouse/ClickHouse/pull/4042) ([アレサピン](https://github.com/alesapin)) +- それをチェック追加 `SET send_logs_level = 'value'` クエリーを受け適切な値です。 [\#3873](https://github.com/ClickHouse/ClickHouse/pull/3873) ([サビヤニン-マキシム](https://github.com/s-mx)) +- 型変換関数の固定データ型チェック。 [\#3896](https://github.com/ClickHouse/ClickHouse/pull/3896) ([冬張](https://github.com/zhang2014)) -#### 性能の改善 {#performance-improvements-5} +#### 性能の向上 {#performance-improvements-5} -- マージツリー設定の追加 `use_minimalistic_part_header_in_zookeeper`. 有効になっている場合、複製のテーブル店舗のコンパクト部分のメタデータの一部znode. これは著しく低下するので、飼育係スナップショットサイズ（場合には、あらゆるテーブルのカラム). この設定を有効にすると、それをサポートしていないバージョンにダウングレードすることはできません。 [\#3960](https://github.com/ClickHouse/ClickHouse/pull/3960) ([Alex Zatelepin](https://github.com/ztlpn)) -- Dfaベースの関数の実装を追加します。 `sequenceMatch` と `sequenceCount` patternに時間が含まれていない場合。 [\#4004](https://github.com/ClickHouse/ClickHouse/pull/4004) ([Léo Ercolanelli](https://github.com/ercolanelli-leo)) -- 整数のシリアル化のパフォーマンスの向上。 [\#3968](https://github.com/ClickHouse/ClickHouse/pull/3968) ([アモスの鳥](https://github.com/amosbird)) -- Zero left padding PODArrayので、-1要素は常に有効でゼロになります。 これは、オフセットの分岐のない計算に使用されます。 [\#3920](https://github.com/ClickHouse/ClickHouse/pull/3920) ([アモスの鳥](https://github.com/amosbird)) -- 元に戻す `jemalloc` パフォーマン [\#4018](https://github.com/ClickHouse/ClickHouse/pull/4018) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- MergeTree設定を追加する `use_minimalistic_part_header_in_zookeeper`. 有効になっている場合、複製のテーブル店舗のコンパクト部分のメタデータの一部znode. これは著しく低下するので、飼育係スナップショットサイズ（場合には、あらゆるテーブルのカラム). この設定を有効にすると、サポートされていないバージョンにダウングレードすることはできません。 [\#3960](https://github.com/ClickHouse/ClickHouse/pull/3960) ([アレックス-ザテレピン](https://github.com/ztlpn)) +- 関数のDFAベースの実装を追加する `sequenceMatch` と `sequenceCount` パターンに時間が含まれていない場合。 [\#4004](https://github.com/ClickHouse/ClickHouse/pull/4004) ([Léo Ercolanelli](https://github.com/ercolanelli-leo)) +- 整数シリアル化のパフォーマンス向上。 [\#3968](https://github.com/ClickHouse/ClickHouse/pull/3968) ([アモス鳥](https://github.com/amosbird)) +- ゼロ左パディングPODArrayように-1要素は常に有効とゼロになります。 これは、オフセットの分岐のない計算に使用されます。 [\#3920](https://github.com/ClickHouse/ClickHouse/pull/3920) ([アモス鳥](https://github.com/amosbird)) +- 元に戻した `jemalloc` 性能低下につながるバージョン。 [\#4018](https://github.com/ClickHouse/ClickHouse/pull/4018) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) #### 下位互換性のない変更 {#backward-incompatible-changes-2} -- 文書化されていない機能を削除 `ALTER MODIFY PRIMARY KEY` それがによって取って代わられたので `ALTER MODIFY ORDER BY` 司令部 [\#3887](https://github.com/ClickHouse/ClickHouse/pull/3887) ([Alex Zatelepin](https://github.com/ztlpn)) -- 削除機能 `shardByHash`. [\#3833](https://github.com/ClickHouse/ClickHouse/pull/3833) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- タイプの結果を持つスカラーサブクエリの使用を禁止する `AggregateFunction`. [\#3865](https://github.com/ClickHouse/ClickHouse/pull/3865) ([イワン](https://github.com/abyss7)) +- 文書化されていない機能を削除 `ALTER MODIFY PRIMARY KEY` それはによって取って代わられたので `ALTER MODIFY ORDER BY` コマンド [\#3887](https://github.com/ClickHouse/ClickHouse/pull/3887) ([アレックス-ザテレピン](https://github.com/ztlpn)) +- 削除された機能 `shardByHash`. [\#3833](https://github.com/ClickHouse/ClickHouse/pull/3833) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- Result of typeでのスカラーサブクエリの使用を禁止する `AggregateFunction`. [\#3865](https://github.com/ClickHouse/ClickHouse/pull/3865) ([イワン](https://github.com/abyss7)) #### ビルド/テスト/パッケージの改善 {#buildtestingpackaging-improvements-6} -- PowerPCのサポートを追加 (`ppc64le`)ビルド. [\#4132](https://github.com/ClickHouse/ClickHouse/pull/4132) ([Danila Kutenin](https://github.com/danlark1)) -- ステートフル機能試験を実般に利用可能データセットである。 [\#3969](https://github.com/ClickHouse/ClickHouse/pull/3969) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- サーバーが起動できない場合のエラーを修正しました `bash: /usr/bin/clickhouse-extract-from-config: Operation not permitted` Dockerまたはsystemd-nspawn内のメッセージ。 [\#4136](https://github.com/ClickHouse/ClickHouse/pull/4136) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 更新 `rdkafka` v1.0.0-RC5へのライブラリ。 生のCインターフェイスの代わりにcppkafkaを使用します。 [\#4025](https://github.com/ClickHouse/ClickHouse/pull/4025) ([イワン](https://github.com/abyss7)) -- 更新 `mariadb-client` ライブラリ。 UBSanで見つかった問題のいずれかを修正しました。 [\#3924](https://github.com/ClickHouse/ClickHouse/pull/3924) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- UBSanビルドのいくつかの修正。 [\#3926](https://github.com/ClickHouse/ClickHouse/pull/3926) [\#3021](https://github.com/ClickHouse/ClickHouse/pull/3021) [\#3948](https://github.com/ClickHouse/ClickHouse/pull/3948) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- PowerPCのサポートを追加 (`ppc64le`）ビルド。 [\#4132](https://github.com/ClickHouse/ClickHouse/pull/4132) ([ダニラ-クテニン](https://github.com/danlark1)) +- ステートフル機能テストは、public available datasetで実行されます。 [\#3969](https://github.com/ClickHouse/ClickHouse/pull/3969) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 固定エラー時にサーバーはできませんの `bash: /usr/bin/clickhouse-extract-from-config: Operation not permitted` Dockerまたはsystemd-nspawn内のメッセージ。 [\#4136](https://github.com/ClickHouse/ClickHouse/pull/4136) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 更新 `rdkafka` v1.0.0-RC5へのライブラリ。 生のCインタフェースの代わりにcppkafkaを使用しました。 [\#4025](https://github.com/ClickHouse/ClickHouse/pull/4025) ([イワン](https://github.com/abyss7)) +- 更新 `mariadb-client` 図書館 ウブサンによって発見された問題の一つを修正しました。 [\#3924](https://github.com/ClickHouse/ClickHouse/pull/3924) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- UBSanビルドのいくつかの修正。 [\#3926](https://github.com/ClickHouse/ClickHouse/pull/3926) [\#3021](https://github.com/ClickHouse/ClickHouse/pull/3021) [\#3948](https://github.com/ClickHouse/ClickHouse/pull/3948) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) - UBSanビルドによるテストのコミットごとの実行を追加しました。 - PVS-Studio static analyzerのコミットごとの実行を追加しました。 -- PVS-Studioによって発見されたバグを修正しました。 [\#4013](https://github.com/ClickHouse/ClickHouse/pull/4013) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 固定glibc互換性の問題。 [\#4100](https://github.com/ClickHouse/ClickHouse/pull/4100) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Dockerイメージを18.10に移動し、glibc\>=2.28の互換性ファイルを追加します [\#3965](https://github.com/ClickHouse/ClickHouse/pull/3965) ([alesapin](https://github.com/alesapin)) -- 追加環境変数の場合はユーザーを行わないchownディレクトリをサーバー dockerイメージです。 [\#3967](https://github.com/ClickHouse/ClickHouse/pull/3967) ([alesapin](https://github.com/alesapin)) -- からの警告のほとんどを有効に `-Weverything` クラングで。 有効 `-Wpedantic`. [\#3986](https://github.com/ClickHouse/ClickHouse/pull/3986) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 唯一のクラングで利用可能ないくつかのより多くの警告を追加しました8. [\#3993](https://github.com/ClickHouse/ClickHouse/pull/3993) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- リンク先 `libLLVM` 共有リンクを使用する場合は、個々のLLVMライブラリではなく。 [\#3989](https://github.com/ClickHouse/ClickHouse/pull/3989) ([Orivej Desh](https://github.com/orivej)) -- テスト画像のための追加された消毒剤の変数。 [\#4072](https://github.com/ClickHouse/ClickHouse/pull/4072) ([alesapin](https://github.com/alesapin)) -- `clickhouse-server` debianパッケージは `libcap2-bin` 使用するパッケージ `setcap` 機能を設定するためのツール。 これは任意です。 [\#4093](https://github.com/ClickHouse/ClickHouse/pull/4093) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 改善されたコンパイル時間、固定includesむ。 [\#3898](https://github.com/ClickHouse/ClickHouse/pull/3898) ([proller](https://github.com/proller)) -- ハッシュ関数のパフォーマンステス [\#3918](https://github.com/ClickHouse/ClickHouse/pull/3918) ([フィリモノフ](https://github.com/filimonov)) -- 固定巡回ライブラリ依存。 [\#3958](https://github.com/ClickHouse/ClickHouse/pull/3958) ([proller](https://github.com/proller)) -- 低利用可能なメモリとコンパイルの改善。 [\#4030](https://github.com/ClickHouse/ClickHouse/pull/4030) ([proller](https://github.com/proller)) -- 追加試験スクリプトの再現性能の劣化 `jemalloc`. [\#4036](https://github.com/ClickHouse/ClickHouse/pull/4036) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 下のコメントや文字列リテラルのスペルミスを修正 `dbms`. [\#4122](https://github.com/ClickHouse/ClickHouse/pull/4122) ([マイハー](https://github.com/maiha)) -- コメントの誤字を修正しました。 [\#4089](https://github.com/ClickHouse/ClickHouse/pull/4089) ([Evgenii Pravda](https://github.com/kvinty)) +- PVS-Studioで見つかったバグを修正しました。 [\#4013](https://github.com/ClickHouse/ClickHouse/pull/4013) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- Glibcの互換性の問題を修正しました。 [\#4100](https://github.com/ClickHouse/ClickHouse/pull/4100) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- Dockerイメージを18.10に移動し、glibc\>=2.28の互換性ファイルを追加します [\#3965](https://github.com/ClickHouse/ClickHouse/pull/3965) ([アレサピン](https://github.com/alesapin)) +- 追加環境変数の場合はユーザーを行わないchownディレクトリをサーバー Dockerイメージです。 [\#3967](https://github.com/ClickHouse/ClickHouse/pull/3967) ([アレサピン](https://github.com/alesapin)) +- からの警告のほとんどを有効に `-Weverything` クラング 有効 `-Wpedantic`. [\#3986](https://github.com/ClickHouse/ClickHouse/pull/3986) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- Clang8でのみ使用できるいくつかの警告を追加しました。 [\#3993](https://github.com/ClickHouse/ClickHouse/pull/3993) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- リンク先 `libLLVM` 共有リンクを使用する場合、個々のLLVM libに対してではなく。 [\#3989](https://github.com/ClickHouse/ClickHouse/pull/3989) ([オリヴェイ-デシュ](https://github.com/orivej)) +- テスト画像のサニタイザー変数を追加しました。 [\#4072](https://github.com/ClickHouse/ClickHouse/pull/4072) ([アレサピン](https://github.com/alesapin)) +- `clickhouse-server` debianパッケージは推奨します `libcap2-bin` 使用するパッケージ `setcap` 機能を設定するためのツール。 これは任意です。 [\#4093](https://github.com/ClickHouse/ClickHouse/pull/4093) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 改善作成時に固定。 [\#3898](https://github.com/ClickHouse/ClickHouse/pull/3898) ([プロラー](https://github.com/proller)) +- ハッシュ関数の性能テストを追加しました。 [\#3918](https://github.com/ClickHouse/ClickHouse/pull/3918) ([フィリモノフ](https://github.com/filimonov)) +- 周期ライブラリの依存関係を修正しました。 [\#3958](https://github.com/ClickHouse/ClickHouse/pull/3958) ([プロラー](https://github.com/proller)) +- 利用可能なメモリが少ないコンパイルの改善。 [\#4030](https://github.com/ClickHouse/ClickHouse/pull/4030) ([プロラー](https://github.com/proller)) +- パフォーマン `jemalloc`. [\#4036](https://github.com/ClickHouse/ClickHouse/pull/4036) ([アレクセイ-ミロヴィドフ](https://github.com/alexey-milovidov)) +- 下のコメントと文字列リテラルのスペルミスを修正しました `dbms`. [\#4122](https://github.com/ClickHouse/ClickHouse/pull/4122) ([舞羽（まいは](https://github.com/maiha)) +- コメントの修正タイプミス。 [\#4089](https://github.com/ClickHouse/ClickHouse/pull/4089) ([エフゲニー-プラウダ](https://github.com/kvinty)) -## [2018年の変更履歴](https://github.com/ClickHouse/ClickHouse/blob/master/docs/en/changelog/2018.md) {#changelog-for-2018} +## [2018年の変更履歴](./2018.md#clickhouse-release-18-16) {#changelog-for-2018} diff --git a/docs/ja/whats-new/changelog/index.md b/docs/ja/whats-new/changelog/index.md index 71a322d31ba..ac12f2d7e4f 100644 --- a/docs/ja/whats-new/changelog/index.md +++ b/docs/ja/whats-new/changelog/index.md @@ -1,668 +1,9 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_folder_title: Changelog +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "\u5909\u66F4\u5C65\u6B74" toc_priority: 74 toc_title: '2020' --- -## ClickHouseリリースv20.3 {#clickhouse-release-v20-3} - -### ClickHouseリリリースv20.3.4.10,2020-03-20 {#clickhouse-release-v20-3-4-10-2020-03-20} - -#### バグ修正 {#bug-fix} - -- このリリースも含む全てのバグ修正から20.1.8.41 -- 不足している修正 `rows_before_limit_at_least` プロセッサパイプラインを使用したhttpクエリの場合。 この修正 [\#9730](https://github.com/ClickHouse/ClickHouse/issues/9730). [\#9757](https://github.com/ClickHouse/ClickHouse/pull/9757) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) - -### ClickHouseリリリースv20.3.3.6,2020-03-17 {#clickhouse-release-v20-3-3-6-2020-03-17} - -#### バグ修正 {#bug-fix-1} - -- このリリースも含む全てのバグ修正から20.1.7.38 -- ユーザーが以前のバージョンで突然変異を実行した場合、複製が機能しないレプリケーションのバグを修正しました。 この修正 [\#9645](https://github.com/ClickHouse/ClickHouse/issues/9645). [\#9652](https://github.com/ClickHouse/ClickHouse/pull/9652) ([alesapin](https://github.com/alesapin)). こ版20.3後方互換。 -- 設定を追加 `use_compact_format_in_distributed_parts_names` これにより、 `INSERT` へのクエリ `Distributed` よりコンパクトな形式のテーブル。 この修正 [\#9647](https://github.com/ClickHouse/ClickHouse/issues/9647). [\#9653](https://github.com/ClickHouse/ClickHouse/pull/9653) ([alesapin](https://github.com/alesapin)). こ版20.3後方互換。 - -### ClickHouseリリリースv20.3.2.1,2020-03-12 {#clickhouse-release-v20-3-2-1-2020-03-12} - -#### 下位互換性のない変更 {#backward-incompatible-change} - -- 問題を修正しました `file name too long` データを送信するとき `Distributed` 多数のレプリカのテーブル。 レプリカの資格情報がサーバーログに表示される問題を修正しました。 ディスク上のディレクトリ名の形式が `[shard{shard_index}[_replica{replica_index}]]`. [\#8911](https://github.com/ClickHouse/ClickHouse/pull/8911) ([Mikhail Korotov](https://github.com/millb)）新しいバージョンにアップグレードした後、古いサーバーのバージョンが新しいディレクトリ形式を認識しないため、手動の介入なしにダウングレードするこ ダウングレードする場合は、対応するディレクトリの名前を手動で古い形式に変更する必要があります。 この変更は、非同期を使用した場合にのみ関連します `INSERT`にs `Distributed` テーブル。 バージョン20.3.3では、新しいフォーマットを徐々に有効にするための設定を紹介します。 -- 変更コマンドのレプリケーションログエントリの形式を変更。 新しいバージョンをイ -- Stacktracesをダンプするシンプルなメモリプロファイラを実装する `system.trace_log` 毎N文字以上のソフト配分を制限 [\#8765](https://github.com/ClickHouse/ClickHouse/pull/8765) ([イワン](https://github.com/abyss7)) [\#9472](https://github.com/ClickHouse/ClickHouse/pull/9472) ([alexey-milovidov](https://github.com/alexey-milovidov)）の列 `system.trace_log` から改名されました `timer_type` に `trace_type`. この変更が必要な第三者機関の性能解析およびflamegraph処理ツールです。 -- 内部スレッド番号の代わりにosスレッドidを使用します。 この修正 [\#7477](https://github.com/ClickHouse/ClickHouse/issues/7477) 古い `clickhouse-client` サーバーから送信されるログを受信できない `send_logs_level` これは、構造化ログメッセージの名前と種類が変更されたためです。 一方、異なるサーバーバージョンでは、異なるタイプのログを相互に送信できます。 あなたが使用しないとき `send_logs_level` 設定、あなたは気にしないでください。 [\#8954](https://github.com/ClickHouse/ClickHouse/pull/8954) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 削除 `indexHint` 機能 [\#9542](https://github.com/ClickHouse/ClickHouse/pull/9542) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 削除 `findClusterIndex`, `findClusterValue` 機能。 この修正 [\#8641](https://github.com/ClickHouse/ClickHouse/issues/8641). これらの機能を使用していた場合は、メールを送信します `clickhouse-feedback@yandex-team.com` [\#9543](https://github.com/ClickHouse/ClickHouse/pull/9543) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- これで、列を作成したり、列を追加したりすることはできません `SELECT` 既定の式としてサブクエリ。 [\#9481](https://github.com/ClickHouse/ClickHouse/pull/9481) ([alesapin](https://github.com/alesapin)) -- JOIN内のサブクエリのエイリアスが必要です。 [\#9274](https://github.com/ClickHouse/ClickHouse/pull/9274) ([Artem Zuikov](https://github.com/4ertus2)) -- 改善された `ALTER MODIFY/ADD` クエリロジック。 今はできません `ADD` タイプのない列, `MODIFY` デフォルトの式では、列の型は変更されません。 `MODIFY` 型は既定の式の値を緩めません。 修正 [\#8669](https://github.com/ClickHouse/ClickHouse/issues/8669). [\#9227](https://github.com/ClickHouse/ClickHouse/pull/9227) ([alesapin](https://github.com/alesapin)) -- ログ設定の変更を適用するには、サーバーを再起動する必要があります。 これは、サーバーが削除されたログファイルにログを記録するバグを回避するための一時的な回避策です。 [\#8696](https://github.com/ClickHouse/ClickHouse/issues/8696)). [\#8707](https://github.com/ClickHouse/ClickHouse/pull/8707) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- を設定 `experimental_use_processors` デフォルトでは有効です。 この設定をご利用の新しいクエリのパイプライン これは内部リファクタリングであり、目に見える変更は期待していません。 問題が表示される場合は、ゼロをバックアップするように設定します。 [\#8768](https://github.com/ClickHouse/ClickHouse/pull/8768) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -#### 新しい機能 {#new-feature} - -- 追加 `Avro` と `AvroConfluent` 入力/出力形式 [\#8571](https://github.com/ClickHouse/ClickHouse/pull/8571) ([Andrew Onyshchuk](https://github.com/oandrew)) [\#8957](https://github.com/ClickHouse/ClickHouse/pull/8957) ([Andrew Onyshchuk](https://github.com/oandrew)) [\#8717](https://github.com/ClickHouse/ClickHouse/pull/8717) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 期限切れのキーのマルチスレッドおよび非ブロック更新 `cache` 辞書（古いものを読むための任意の許可を持つ）。 [\#8303](https://github.com/ClickHouse/ClickHouse/pull/8303) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- クエリの追加 `ALTER ... MATERIALIZE TTL`. TTLによって期限切れのデータを強制的に削除し、すべての部分でTTLに関するメタ情報を再計算する突然変異を実行します。 [\#8775](https://github.com/ClickHouse/ClickHouse/pull/8775) ([アントン-ポポフ](https://github.com/CurtizJ)) -- 必要に応じて、hashjoinからmergejoin（ディスク上）に切り替えます [\#9082](https://github.com/ClickHouse/ClickHouse/pull/9082) ([Artem Zuikov](https://github.com/4ertus2)) -- 追加 `MOVE PARTITION` コマンド `ALTER TABLE` [\#4729](https://github.com/ClickHouse/ClickHouse/issues/4729) [\#6168](https://github.com/ClickHouse/ClickHouse/pull/6168) ([ギヨームタッセリー](https://github.com/YiuRULE)) -- 設定ファイルからストレージ設定をリロードする。 [\#8594](https://github.com/ClickHouse/ClickHouse/pull/8594) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 変更できる `storage_policy` あまり豊かではないものに。 [\#8107](https://github.com/ClickHouse/ClickHouse/pull/8107) ([Vladimir Chebotarev](https://github.com/excitoon)) -- S3ストレージとテーブル機能のglobs/wildcardsのサポートを追加しました。 [\#8851](https://github.com/ClickHouse/ClickHouse/pull/8851) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 実装 `bitAnd`, `bitOr`, `bitXor`, `bitNot` のために `FixedString(N)` データ型。 [\#9091](https://github.com/ClickHouse/ClickHouse/pull/9091) ([ギヨームタッセリー](https://github.com/YiuRULE)) -- 機能追加 `bitCount`. この修正 [\#8702](https://github.com/ClickHouse/ClickHouse/issues/8702). [\#8708](https://github.com/ClickHouse/ClickHouse/pull/8708) ([alexey-milovidov](https://github.com/alexey-milovidov)) [\#8749](https://github.com/ClickHouse/ClickHouse/pull/8749) ([ikopylov](https://github.com/ikopylov)) -- 追加 `generateRandom` テーブル機能をランダム行に指定されたschema. 任意のテストテーブルにデータを設定できます。 [\#8994](https://github.com/ClickHouse/ClickHouse/pull/8994) ([イリヤ-ヤツィシン](https://github.com/qoega)) -- `JSONEachRowFormat` 支援特別の場合オブジェ囲まれたトップレベルの配列になります。 [\#8860](https://github.com/ClickHouse/ClickHouse/pull/8860) ([Kruglov Pavel](https://github.com/Avogar)) -- これで、列を作成することができます `DEFAULT` デフォルトの列に依存する式 `ALIAS` 式。 [\#9489](https://github.com/ClickHouse/ClickHouse/pull/9489) ([alesapin](https://github.com/alesapin)) -- 指定できるようにする `--limit` ソースデータサイズよりも `clickhouse-obfuscator`. データは異なるランダムシードで繰り返されます。 [\#9155](https://github.com/ClickHouse/ClickHouse/pull/9155) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 追加 `groupArraySample` 機能（に類似した `groupArray` とreserviorサンプリングアルゴリズムです。 [\#8286](https://github.com/ClickHouse/ClickHouse/pull/8286) ([アモスの鳥](https://github.com/amosbird)) -- これで、更新キューのサイズを監視することができます `cache`/`complex_key_cache` システム指標による辞書。 [\#9413](https://github.com/ClickHouse/ClickHouse/pull/9413) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- CSV出力形式の行区切りとしてCRLFを使用できるようにする `output_format_csv_crlf_end_of_line` は1に設定されます [\#8934](https://github.com/ClickHouse/ClickHouse/pull/8934) [\#8935](https://github.com/ClickHouse/ClickHouse/pull/8935) [\#8963](https://github.com/ClickHouse/ClickHouse/pull/8963) ([Mikhail Korotov](https://github.com/millb)) -- より多くの機能を実装する [H3](https://github.com/uber/h3) API: `h3GetBaseCell`, `h3HexAreaM2`, `h3IndexesAreNeighbors`, `h3ToChildren`, `h3ToString` と `stringToH3` [\#8938](https://github.com/ClickHouse/ClickHouse/pull/8938) ([ニコ-マンデリー](https://github.com/nmandery)) -- 新しい設定を導入: `max_parser_depth` 最大スタックサイズを制御し、大規模な複雑なクエリを許可する。 この修正 [\#6681](https://github.com/ClickHouse/ClickHouse/issues/6681) と [\#7668](https://github.com/ClickHouse/ClickHouse/issues/7668). [\#8647](https://github.com/ClickHouse/ClickHouse/pull/8647) ([Maxim Smirnov](https://github.com/qMBQx8GH)) -- 設定を追加する `force_optimize_skip_unused_shards` 未使用のシャードをスキップできない場合にスローする設定 [\#8805](https://github.com/ClickHouse/ClickHouse/pull/8805) ([Azat Khuzhin](https://github.com/azat)) -- この設定は複数のディスク/量のデータを格納するための送付 `Distributed` エンジン [\#8756](https://github.com/ClickHouse/ClickHouse/pull/8756) ([Azat Khuzhin](https://github.com/azat)) -- 支援の保管方針 (``)一時的なデータを貯えるため。 [\#8750](https://github.com/ClickHouse/ClickHouse/pull/8750) ([Azat Khuzhin](https://github.com/azat)) -- 追加 `X-ClickHouse-Exception-Code` データを送信する前に例外がスローされた場合に設定されるHTTPヘッダー。 これは [\#4971](https://github.com/ClickHouse/ClickHouse/issues/4971). [\#8786](https://github.com/ClickHouse/ClickHouse/pull/8786) ([Mikhail Korotov](https://github.com/millb)) -- 機能追加 `ifNotFinite`. それは単なる統語的な砂糖です: `ifNotFinite(x, y) = isFinite(x) ? x : y`. [\#8710](https://github.com/ClickHouse/ClickHouse/pull/8710) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 追加 `last_successful_update_time` コラムの `system.dictionaries` テーブル [\#9394](https://github.com/ClickHouse/ClickHouse/pull/9394) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- 追加 `blockSerializedSize` 機能（圧縮なしのディスク上のサイズ) [\#8952](https://github.com/ClickHouse/ClickHouse/pull/8952) ([Azat Khuzhin](https://github.com/azat)) -- 機能を追加 `moduloOrZero` [\#9358](https://github.com/ClickHouse/ClickHouse/pull/9358) ([hcz](https://github.com/hczhcz)) -- 追加されたシステム表 `system.zeros` と `system.zeros_mt` だけでなく、物語機能 `zeros()` と `zeros_mt()`. テーブル（テーブル機能を含む単一カラム名 `zero` とタイプ `UInt8`. この列にはゼロがあります。 これは、多くの行を生成する最速の方法としてテスト目的に必要です。 この修正 [\#6604](https://github.com/ClickHouse/ClickHouse/issues/6604) [\#9593](https://github.com/ClickHouse/ClickHouse/pull/9593) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) - -#### 実験的特徴 {#experimental-feature} - -- 部品の新しいコンパクトな形式を追加 `MergeTree`-すべての列が一つのファイルに格納されている家族のテーブル。 それは小さく、頻繁な挿入物の性能を高めるのを助ける。 古いフォーマット(列ごとに一つのファイル)がwideと呼ばれます。 データ格納形式は設定によって制御 `min_bytes_for_wide_part` と `min_rows_for_wide_part`. [\#8290](https://github.com/ClickHouse/ClickHouse/pull/8290) ([アントン-ポポフ](https://github.com/CurtizJ)) -- S3ストレージのサポート `Log`, `TinyLog` と `StripeLog` テーブル。 [\#8862](https://github.com/ClickHouse/ClickHouse/pull/8862) ([Pavel Kovalenko](https://github.com/Jokser)) - -#### バグ修正 {#bug-fix-2} - -- ログメッセージの不整合な空白を修正しました。 [\#9322](https://github.com/ClickHouse/ClickHouse/pull/9322) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- テーブル作成時に、名前のないタプルの配列がネストされた構造としてフラット化されたバグを修正。 [\#8866](https://github.com/ClickHouse/ClickHouse/pull/8866) ([achulkov2comment](https://github.com/achulkov2)) -- ときに問題を修正しました “Too many open files” エラーが発生する恐れがあると多数の場合はファイルのマッチングglobパターン `File` テーブルまたは `file` テーブル機能。 今すぐファイルが遅延開かれます。 この修正 [\#8857](https://github.com/ClickHouse/ClickHouse/issues/8857) [\#8861](https://github.com/ClickHouse/ClickHouse/pull/8861) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- DROP TEMPORARY TABLEは現在、一時テーブルのみを削除します。 [\#8907](https://github.com/ClickHouse/ClickHouse/pull/8907) ([Vitaly Baranov](https://github.com/vitlibar)) -- 削除した旧式の仕切りした時停止のサーバーは取り外し、添付を表示します。 [\#8602](https://github.com/ClickHouse/ClickHouse/pull/8602) ([ギヨームタッセリー](https://github.com/YiuRULE)) -- のためにどのようにデフォルトのディスクを算定し、自由空間から `data` サブディレクトリ。 空き容量が正しく計算されない場合の問題を修正しました。 `data` ディレクト この修正 [\#7441](https://github.com/ClickHouse/ClickHouse/issues/7441) [\#9257](https://github.com/ClickHouse/ClickHouse/pull/9257) ([Mikhail Korotov](https://github.com/millb)) -- カンマ（クロス）は、内部の（）に参加することができます。 [\#9251](https://github.com/ClickHouse/ClickHouse/pull/9251) ([Artem Zuikov](https://github.com/4ertus2)) -- WHERE節に演算子のようなものがある場合は、INNER JOINにクロスを書き換えることができます。 [\#9229](https://github.com/ClickHouse/ClickHouse/pull/9229) ([Artem Zuikov](https://github.com/4ertus2)) -- 後に可能な誤った結果を修正 `GROUP BY` 有効に設定 `distributed_aggregation_memory_efficient`. 修正 [\#9134](https://github.com/ClickHouse/ClickHouse/issues/9134). [\#9289](https://github.com/ClickHouse/ClickHouse/pull/9289) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 見つかりキーのカウントとして見るメトリクスのキャッシュを生成する事ができます。 [\#9411](https://github.com/ClickHouse/ClickHouse/pull/9411) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- レプリケーションプロト [\#8598](https://github.com/ClickHouse/ClickHouse/issues/8598). [\#9412](https://github.com/ClickHouse/ClickHouse/pull/9412) ([alesapin](https://github.com/alesapin)) -- 上の固定レース条件 `queue_task_handle` の起動時に `ReplicatedMergeTree` テーブル。 [\#9552](https://github.com/ClickHouse/ClickHouse/pull/9552) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- その他の通貨 `NOT` で動作しませんでした `SHOW TABLES NOT LIKE` クエリ [\#8727](https://github.com/ClickHouse/ClickHouse/issues/8727) [\#8940](https://github.com/ClickHouse/ClickHouse/pull/8940) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 機能に範囲チェックを追加しました `h3EdgeLengthM`. このチェッ [\#8945](https://github.com/ClickHouse/ClickHouse/pull/8945) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 複数の引数（10以上）の三元論理演算のバッチ計算のバグを修正しました。 [\#8718](https://github.com/ClickHouse/ClickHouse/pull/8718) ([Alexander Kazakov](https://github.com/Akazz)) -- Prewhere最適化のエラーを修正しました。 `Inconsistent number of columns got from MergeTreeRangeReader` 例外だ [\#9024](https://github.com/ClickHouse/ClickHouse/pull/9024) ([アントン-ポポフ](https://github.com/CurtizJ)) -- 予期しない修正 `Timeout exceeded while reading from socket` 例外は、ランダムに起きにセキュア接続前にタイムアウト実を超えた場queryプロファイラが有効になります。 また、追加 `connect_timeout_with_failover_secure_ms` 設定(デフォルトは100ミリ秒)です。 `connect_timeout_with_failover_ms` ただし、セキュアな接続に使用されます(SSLハンドシェイクが通常のTCP接続よりも遅いため) [\#9026](https://github.com/ClickHouse/ClickHouse/pull/9026) ([tavplubix](https://github.com/tavplubix)) -- バグを修正しました。 `parts_to_do=0` と `is_done=0`. [\#9022](https://github.com/ClickHouse/ClickHouse/pull/9022) ([alesapin](https://github.com/alesapin)) -- 新しい任意の結合ロジックを使用する `partial_merge_join` 設定。 それは作ることが可能です `ANY|ALL|SEMI LEFT` と `ALL INNER` との結合 `partial_merge_join=1` 今だ [\#8932](https://github.com/ClickHouse/ClickHouse/pull/8932) ([Artem Zuikov](https://github.com/4ertus2)) -- シャードは、例外をスローするのではなく、イニシエータから取得した設定をシャードのconstaintsにクランプします。 この修正では、別の制約を持つシャードにクエリを送信できます。 [\#9447](https://github.com/ClickHouse/ClickHouse/pull/9447) ([Vitaly Baranov](https://github.com/vitlibar)) -- 固定メモリ管理の問題 `MergeTreeReadPool`. [\#8791](https://github.com/ClickHouse/ClickHouse/pull/8791) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 修正 `toDecimal*OrNull()` 文字列で呼び出されたときの関数群 `e`. 修正 [\#8312](https://github.com/ClickHouse/ClickHouse/issues/8312) [\#8764](https://github.com/ClickHouse/ClickHouse/pull/8764) ([Artem Zuikov](https://github.com/4ertus2)) -- う `FORMAT Null` クライアントにデータを送信しません。 [\#8767](https://github.com/ClickHouse/ClickHouse/pull/8767) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- そのタイムスタンプを修正 `LiveViewBlockInputStream` 更新されません。 `LIVE VIEW` 実験的特徴です [\#8644](https://github.com/ClickHouse/ClickHouse/pull/8644) ([vxider](https://github.com/Vxider)) [\#8625](https://github.com/ClickHouse/ClickHouse/pull/8625) ([vxider](https://github.com/Vxider)) -- 固定 `ALTER MODIFY TTL` 古いTTL式を削除することを許さなかった誤った動作。 [\#8422](https://github.com/ClickHouse/ClickHouse/pull/8422) ([Vladimir Chebotarev](https://github.com/excitoon)) -- MergeTreeIndexSet内のUBSanレポートを修正しました。 この修正 [\#9250](https://github.com/ClickHouse/ClickHouse/issues/9250) [\#9365](https://github.com/ClickHouse/ClickHouse/pull/9365) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- の動作を修正しました `match` と `extract` haystackにゼロバイトがある場合の関数。 Haystackが一定の場合、その動作は間違っていました。 この修正 [\#9160](https://github.com/ClickHouse/ClickHouse/issues/9160) [\#9163](https://github.com/ClickHouse/ClickHouse/pull/9163) ([alexey-milovidov](https://github.com/alexey-milovidov)) [\#9345](https://github.com/ClickHouse/ClickHouse/pull/9345) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Apache Avro3rd-partyライブラリのデストラクターから投げを避けます。 [\#9066](https://github.com/ClickHouse/ClickHouse/pull/9066) ([Andrew Onyshchuk](https://github.com/oandrew)) -- ポーリングされたバッチをコミットしない `Kafka` 部分的には、データの穴につながる可能性があります。 [\#8876](https://github.com/ClickHouse/ClickHouse/pull/8876) ([フィリモノフ](https://github.com/filimonov)) -- 修正 `joinGet` null可能な戻り値の型を指定します。 https://github.com/ClickHouse/ClickHouse/issues/8919 [\#9014](https://github.com/ClickHouse/ClickHouse/pull/9014) ([アモスの鳥](https://github.com/amosbird)) -- 圧縮時にデータの非互換性を修正する `T64` コーデック。 [\#9016](https://github.com/ClickHouse/ClickHouse/pull/9016) ([Artem Zuikov](https://github.com/4ertus2))データ型idの修正 `T64` 影響を受けるバージョンで間違った（de）圧縮につながる圧縮コーデック。 [\#9033](https://github.com/ClickHouse/ClickHouse/pull/9033) ([Artem Zuikov](https://github.com/4ertus2)) -- 設定を追加 `enable_early_constant_folding` 無効にするのである。 [\#9010](https://github.com/ClickHouse/ClickHouse/pull/9010) ([Artem Zuikov](https://github.com/4ertus2)) -- 修正プッシュダウ述語オプティマイザとビューの試験 [\#9011](https://github.com/ClickHouse/ClickHouse/pull/9011) ([冬張](https://github.com/zhang2014)) -- Segfaultを修正する `Merge` から読み取るときに発生する可能性があります `File` ストレージ [\#9387](https://github.com/ClickHouse/ClickHouse/pull/9387) ([tavplubix](https://github.com/tavplubix)) -- ストレージポリシーのチェックを追加 `ATTACH PARTITION FROM`, `REPLACE PARTITION`, `MOVE TO TABLE`. そうでない場合はこのデータの一部になり、再起動後の防止ClickHouse。 [\#9383](https://github.com/ClickHouse/ClickHouse/pull/9383) ([Vladimir Chebotarev](https://github.com/excitoon)) -- テーブルにttlが設定されている場合に変更を修正します。 [\#8800](https://github.com/ClickHouse/ClickHouse/pull/8800) ([アントン-ポポフ](https://github.com/CurtizJ)) -- ときに発生する可能性が競合状態を修正 `SYSTEM RELOAD ALL DICTIONARIES` いくつかの辞書が変更/追加/削除されている間に実行されます。 [\#8801](https://github.com/ClickHouse/ClickHouse/pull/8801) ([Vitaly Baranov](https://github.com/vitlibar)) -- 以前のバージョンでは `Memory` データベースエンジ `path` directory (e.g. `/var/lib/clickhouse/`), not in data directory of database (e.g. `/var/lib/clickhouse/db_name`). [\#8753](https://github.com/ClickHouse/ClickHouse/pull/8753) ([tavplubix](https://github.com/tavplubix)) -- デフォル [\#9530](https://github.com/ClickHouse/ClickHouse/pull/9530) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 配列型のbloom\_filterインデックスのnot(has())を修正しました。 [\#9407](https://github.com/ClickHouse/ClickHouse/pull/9407) ([achimbabcomment](https://github.com/achimbab)) -- テーブルの最初の列を許可する `Log` エンジンは別名である [\#9231](https://github.com/ClickHouse/ClickHouse/pull/9231) ([イワン](https://github.com/abyss7)) -- から読み込み中の範囲の順序を修正 `MergeTree` 一つのスレッドのテーブル。 それは例外につながる可能性があります `MergeTreeRangeReader` または間違ったクエリ結果。 [\#9050](https://github.com/ClickHouse/ClickHouse/pull/9050) ([アントン-ポポフ](https://github.com/CurtizJ)) -- 作る `reinterpretAsFixedString` 戻るには `FixedString` 代わりに `String`. [\#9052](https://github.com/ClickHouse/ClickHouse/pull/9052) ([Andrew Onyshchuk](https://github.com/oandrew)) -- を避ける非常に珍しい場合には、ユーザーで間違ったエラーメッセージ (`Success` 詳細なエラーの説明の代わりに）。 [\#9457](https://github.com/ClickHouse/ClickHouse/pull/9457) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 使用するとき衝突しないで下さい `Template` 空の行テンプレートを使用した形式。 [\#8785](https://github.com/ClickHouse/ClickHouse/pull/8785) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- システムテーブルのメタデータファイ [\#8653](https://github.com/ClickHouse/ClickHouse/pull/8653) ([tavplubix](https://github.com/tavplubix))修正 [\#8581](https://github.com/ClickHouse/ClickHouse/issues/8581). -- キャッシュ辞書でexception\_ptrのデータレースを修正 [\#8303](https://github.com/ClickHouse/ClickHouse/issues/8303). [\#9379](https://github.com/ClickHouse/ClickHouse/pull/9379) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- クエリの例外をスローしない `ATTACH TABLE IF NOT EXISTS`. 以前は、テーブルが既に存在する場合にスローされました。 `IF NOT EXISTS` 句。 [\#8967](https://github.com/ClickHouse/ClickHouse/pull/8967) ([アントン-ポポフ](https://github.com/CurtizJ)) -- 例外メッセージで行方不明の閉じる括弧を修正しました。 [\#8811](https://github.com/ClickHouse/ClickHouse/pull/8811) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- メッセージの回避 `Possible deadlock avoided` インタラクティブモードでのclickhouse-クライアントの起動時に。 [\#9455](https://github.com/ClickHouse/ClickHouse/pull/9455) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Base64でエンコードされた値の末尾にパディングが不正な形式になる場合がある問題を修正しました。 更新base64ライブラリ。 この修正 [\#9491](https://github.com/ClickHouse/ClickHouse/issues/9491)、閉じます [\#9492](https://github.com/ClickHouse/ClickHouse/issues/9492) [\#9500](https://github.com/ClickHouse/ClickHouse/pull/9500) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- データを失うのを防ぐ `Kafka` まれに、接尾辞を読んだ後でコミットする前に例外が発生した場合。 修正 [\#9378](https://github.com/ClickHouse/ClickHouse/issues/9378) [\#9507](https://github.com/ClickHouse/ClickHouse/pull/9507) ([フィリモノフ](https://github.com/filimonov)) -- 固定例外で `DROP TABLE IF EXISTS` [\#8663](https://github.com/ClickHouse/ClickHouse/pull/8663) ([Nikita Vasilev](https://github.com/nikvas0)) -- ユーザーが `ALTER MODIFY SETTING` 古いformatedのため `MergeTree` テーブルエンジン家族。 [\#9435](https://github.com/ClickHouse/ClickHouse/pull/9435) ([alesapin](https://github.com/alesapin)) -- JSON関連の関数でInt64に収まらないUInt64の数値のサポート。 SIMDJSONをmasterに更新します。 この修正 [\#9209](https://github.com/ClickHouse/ClickHouse/issues/9209) [\#9344](https://github.com/ClickHouse/ClickHouse/pull/9344) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 非厳密に単調な関数索引が使用されている場合の逆述語の実行を修正しました。 [\#9223](https://github.com/ClickHouse/ClickHouse/pull/9223) ([Alexander Kazakov](https://github.com/Akazz)) -- 折ることを試みてはいけない `IN` 定数の `GROUP BY` [\#8868](https://github.com/ClickHouse/ClickHouse/pull/8868) ([アモスの鳥](https://github.com/amosbird)) -- バグを修正 `ALTER DELETE` インデックスの破損につながる変異。 この修正 [\#9019](https://github.com/ClickHouse/ClickHouse/issues/9019) と [\#8982](https://github.com/ClickHouse/ClickHouse/issues/8982). さらに、非常にまれな競合状態を修正 `ReplicatedMergeTree` `ALTER` クエリ。 [\#9048](https://github.com/ClickHouse/ClickHouse/pull/9048) ([alesapin](https://github.com/alesapin)) -- ときは設定 `compile_expressions` が有効になっている場合は、 `unexpected column` で `LLVMExecutableFunction` 私達が使用する時 `Nullable` タイプ [\#8910](https://github.com/ClickHouse/ClickHouse/pull/8910) ([ギヨームタッセリー](https://github.com/YiuRULE)) -- 以下のための複数の修正 `Kafka` エンジン：1）消費者グループのリバランス中に表示された重複を修正します。 2)修正レア ‘holes’ 登場時のデータをポーリングから割と世論調査および為の一部（現在の私たちは常にプロセス/コミット全体のポーリングブロックメッセージ). 3)固定フラッシュによるブロックサイズ(前のみにフラッシングによるタイムアウトした作業と同様に扱う。 4)より契約手続(入力フィードバック. 5)テストをより速く動作させる(デフォルトの間隔とタイムアウト)。 データは以前はブロックサイズでフラッシュされていなかったため（ドキュメントによると）、PRはデフォルト設定でパフォーマンスが低下する可能性 その変更後にパフォーマンスの問題が発生した場合-増加してください `kafka_max_block_size` より大きな値へのテーブル（例えば `CREATE TABLE ...Engine=Kafka ... SETTINGS ... kafka_max_block_size=524288`). 修正 [\#7259](https://github.com/ClickHouse/ClickHouse/issues/7259) [\#8917](https://github.com/ClickHouse/ClickHouse/pull/8917) ([フィリモノフ](https://github.com/filimonov)) -- 修正 `Parameter out of bound` PREWHERE最適化の後のいくつかのクエリの例外。 [\#8914](https://github.com/ClickHouse/ClickHouse/pull/8914) ([Baudouin Giard](https://github.com/bgiard)) -- 関数の引数の混合constの場合を修正しました `arrayZip`. [\#8705](https://github.com/ClickHouse/ClickHouse/pull/8705) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 実行時期 `CREATE` クエリー、倍定表現のストレージエンジンの引数です。 空のデータベース名を現在のデータベ 修正 [\#6508](https://github.com/ClickHouse/ClickHouse/issues/6508), [\#3492](https://github.com/ClickHouse/ClickHouse/issues/3492) [\#9262](https://github.com/ClickHouse/ClickHouse/pull/9262) ([tavplubix](https://github.com/tavplubix)) -- 次のような単純な循環エイリアスを持つ列を作成または追加することはできません `a DEFAULT b, b DEFAULT a`. [\#9603](https://github.com/ClickHouse/ClickHouse/pull/9603) ([alesapin](https://github.com/alesapin)) -- 元の部分が破損する可能性があるダブル移動のバグを修正しました。 これは、 `ALTER TABLE MOVE` [\#8680](https://github.com/ClickHouse/ClickHouse/pull/8680) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 許可 `interval` バッククォートなしで正しく解析する識別子。 場合でも、クエリを実行できない問題を修正しました。 `interval` 識別子は、バッククォートまたは二重引用符で囲まれています。 この修正 [\#9124](https://github.com/ClickHouse/ClickHouse/issues/9124). [\#9142](https://github.com/ClickHouse/ClickHouse/pull/9142) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 固定ファズテストとの不正な動作 `bitTestAll`/`bitTestAny` 機能。 [\#9143](https://github.com/ClickHouse/ClickHouse/pull/9143) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 可能なクラッシュ/間違った行数を修正 `LIMIT n WITH TIES` n行目に等しい行がたくさんあるとき。 [\#9464](https://github.com/ClickHouse/ClickHouse/pull/9464) ([tavplubix](https://github.com/tavplubix)) -- Enabledで書かれたパーツによる修正 `insert_quorum`. [\#9463](https://github.com/ClickHouse/ClickHouse/pull/9463) ([alesapin](https://github.com/alesapin)) -- の破壊でデータレースを修正 `Poco::HTTPServer`. どこの場合のサーバを起動直ちに停止しております。 [\#9468](https://github.com/ClickHouse/ClickHouse/pull/9468) ([アントン-ポポフ](https://github.com/CurtizJ)) -- 実行中に誤解を招くエラーメッセージが表示されたバグを修正 `SHOW CREATE TABLE a_table_that_does_not_exist`. [\#8899](https://github.com/ClickHouse/ClickHouse/pull/8899) ([achulkov2comment](https://github.com/achulkov2)) -- 固定 `Parameters are out of bound` 私たちが定数を持っているいくつかのまれなケースでは例外 `SELECT` 私たちが持っているときの句 `ORDER BY` と `LIMIT` 句。 [\#8892](https://github.com/ClickHouse/ClickHouse/pull/8892) ([ギヨームタッセリー](https://github.com/YiuRULE)) -- 既に完了した突然変異がステータスを持つことができるとき `is_done=0`. [\#9217](https://github.com/ClickHouse/ClickHouse/pull/9217) ([alesapin](https://github.com/alesapin)) -- 実行を防ぐ `ALTER ADD INDEX` 古い構文のMergeTreeテーブルでは、動作しないためです。 [\#8822](https://github.com/ClickHouse/ClickHouse/pull/8822) ([Mikhail Korotov](https://github.com/millb)) -- サーバーの起動中にテーブルにアクセスしない。 `LIVE VIEW` 依存するので、サーバーは起動できます。 また、削除 `LIVE VIEW` デタッチ時の依存関係 `LIVE VIEW`. `LIVE VIEW` 実験的特徴です [\#8824](https://github.com/ClickHouse/ClickHouse/pull/8824) ([tavplubix](https://github.com/tavplubix)) -- で可能なsegfaultを修正 `MergeTreeRangeReader`,実行中 `PREWHERE`. [\#9106](https://github.com/ClickHouse/ClickHouse/pull/9106) ([アントン-ポポフ](https://github.com/CurtizJ)) -- 列ttlsによるチェックサムの不一致を修正しました。 [\#9451](https://github.com/ClickHouse/ClickHouse/pull/9451) ([アントン-ポポフ](https://github.com/CurtizJ)) -- ボリュームが一つしかない場合にttlルールによってパーツがバックグラウンドで移動されないバグを修正しました。 [\#8672](https://github.com/ClickHouse/ClickHouse/pull/8672) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 問題を修正しました `Method createColumn() is not implemented for data type Set`. この修正 [\#7799](https://github.com/ClickHouse/ClickHouse/issues/7799). [\#8674](https://github.com/ClickHouse/ClickHouse/pull/8674) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 今度は、突然変異をより頻繁に確定しようとします。 [\#9427](https://github.com/ClickHouse/ClickHouse/pull/9427) ([alesapin](https://github.com/alesapin)) -- 修正 `intDiv` マイナス一つの定数による [\#9351](https://github.com/ClickHouse/ClickHouse/pull/9351) ([hcz](https://github.com/hczhcz)) -- 可能な競合状態を修正 `BlockIO`. [\#9356](https://github.com/ClickHouse/ClickHouse/pull/9356) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- のを修正した。先サーバを終了しようとした場合に使用/drop `Kafka` テーブル作成されたパラメータ。 [\#9513](https://github.com/ClickHouse/ClickHouse/pull/9513) ([フィリモノフ](https://github.com/filimonov)) -- OSが間違った結果を返す場合の回避策を追加 `timer_create` 機能。 [\#8837](https://github.com/ClickHouse/ClickHouse/pull/8837) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- の使用で修正されたエラー `min_marks_for_seek` パラメータ。 固定のエラーメッセージがない場合shardingキーテーブルの配布に努めのスキップ未使用の破片. [\#8908](https://github.com/ClickHouse/ClickHouse/pull/8908) ([Azat Khuzhin](https://github.com/azat)) - -#### 改善 {#improvement} - -- 実装 `ALTER MODIFY/DROP` 以下のための突然変異の上にクエリ `ReplicatedMergeTree*` エンジンファミリー さて `ALTERS` メタデータ更新ステージでのみブロックし、その後はブロックしません。 [\#8701](https://github.com/ClickHouse/ClickHouse/pull/8701) ([alesapin](https://github.com/alesapin)) -- 内部結合にcrossを書き換える機能を追加する `WHERE` シリアル化されていない名前のセクション。 [\#9512](https://github.com/ClickHouse/ClickHouse/pull/9512) ([Artem Zuikov](https://github.com/4ertus2)) -- 作る `SHOW TABLES` と `SHOW DATABASES` クエリは、 `WHERE` 式と `FROM`/`IN` [\#9076](https://github.com/ClickHouse/ClickHouse/pull/9076) ([sundyli](https://github.com/sundy-li)) -- 設定を追加しました `deduplicate_blocks_in_dependent_materialized_views`. [\#9070](https://github.com/ClickHouse/ClickHouse/pull/9070) ([urykhy](https://github.com/urykhy)) -- 最近の変更後、mysqlクライアントはバイナリ文字列をhexで印刷し始め、読みにくくなりました ([\#9032](https://github.com/ClickHouse/ClickHouse/issues/9032)). ClickHouseの回避策は、文字列の列をUTF-8としてマークすることです。 [\#9079](https://github.com/ClickHouse/ClickHouse/pull/9079) ([ユーリーバラノフ](https://github.com/yurriy)) -- 文字列とfixedstringキーのサポートを追加する `sumMap` [\#8903](https://github.com/ClickHouse/ClickHouse/pull/8903) ([Baudouin Giard](https://github.com/bgiard)) -- SummingMergeTreeマップでの文字列キーのサポート [\#8933](https://github.com/ClickHouse/ClickHouse/pull/8933) ([Baudouin Giard](https://github.com/bgiard)) -- スレッドが例外をスローした場合でも、スレッドプールへのスレッドの信号終端 [\#8736](https://github.com/ClickHouse/ClickHouse/pull/8736) ([丁象飛](https://github.com/dingxiangfei2009)) -- 設定を許可する `query_id` で `clickhouse-benchmark` [\#9416](https://github.com/ClickHouse/ClickHouse/pull/9416) ([アントン-ポポフ](https://github.com/CurtizJ)) -- 奇妙な表現を許可しない `ALTER TABLE ... PARTITION partition` クエリ。 このアドレス [\#7192](https://github.com/ClickHouse/ClickHouse/issues/7192) [\#8835](https://github.com/ClickHouse/ClickHouse/pull/8835) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- テーブル `system.table_engines` 機能のサポートに関する情報を提供します `supports_ttl` または `supports_sort_order`). [\#8830](https://github.com/ClickHouse/ClickHouse/pull/8830) ([Max Akhmedov](https://github.com/zlobober)) -- 有効 `system.metric_log` デフォルトでは。 これには、ProfileEventsの値を持つ行が含まれます。 “collect\_interval\_milliseconds” 間隔(デフォルトでは秒)。 テーブルは非常に小さく（通常はメガバイトの順で）、デフォルトでこのデータを収集することは妥当です。 [\#9225](https://github.com/ClickHouse/ClickHouse/pull/9225) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Initialize query profiler for all threads in a group, e.g. it allows to fully profile insert-queries. Fixes [\#6964](https://github.com/ClickHouse/ClickHouse/issues/6964) [\#8874](https://github.com/ClickHouse/ClickHouse/pull/8874) ([イワン](https://github.com/abyss7)) -- 今すぐ一時的 `LIVE VIEW` によって作成されます `CREATE LIVE VIEW name WITH TIMEOUT [42] ...` 代わりに `CREATE TEMPORARY LIVE VIEW ...` 前の構文は、次の構文と一致していなかったためです `CREATE TEMPORARY TABLE ...` [\#9131](https://github.com/ClickHouse/ClickHouse/pull/9131) ([tavplubix](https://github.com/tavplubix)) -- Text\_logを追加します。行くエントリを制限するレベル構成パラメータ `system.text_log` テーブル [\#8809](https://github.com/ClickHouse/ClickHouse/pull/8809) ([Azat Khuzhin](https://github.com/azat)) -- を入れてダウンロード部にディスク/量によるttlル [\#8598](https://github.com/ClickHouse/ClickHouse/pull/8598) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 外部mysqlディクショナリの場合、mysql接続プールをmutualizeすることができます “share” それらの間で辞書。 このオプションは、MySQLサーバーへの接続数を大幅に削減します。 [\#9409](https://github.com/ClickHouse/ClickHouse/pull/9409) ([Clément Rodriguez](https://github.com/clemrodriguez)) -- 最も近いクエリの実行時間内の変位値を表示する `clickhouse-benchmark` 内挿された値の代わりに出力します。 いくつかのクエリの実行時間に対応する値を表示する方がよいでしょう。 [\#8712](https://github.com/ClickHouse/ClickHouse/pull/8712) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Kafkaにデータを挿入するときにメッセージのキーとタイムスタンプを追加する可能性。 修正 [\#7198](https://github.com/ClickHouse/ClickHouse/issues/7198) [\#8969](https://github.com/ClickHouse/ClickHouse/pull/8969) ([フィリモノフ](https://github.com/filimonov)) -- 場合はサーバはターミナルから、ハイライトのスレッド号、クエリをidでログインを優先する色をします。 ここは改善の可読性の相関のログメッセージのステータスです。 [\#8961](https://github.com/ClickHouse/ClickHouse/pull/8961) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- より良い例外のメッセージ読み込み中にテーブル `Ordinary` データベース [\#9527](https://github.com/ClickHouse/ClickHouse/pull/9527) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 実装 `arraySlice` 集約関数の状態を持つ配列の場合。 この修正 [\#9388](https://github.com/ClickHouse/ClickHouse/issues/9388) [\#9391](https://github.com/ClickHouse/ClickHouse/pull/9391) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- IN演算子の右側で定数関数と定数配列を使用できます。 [\#8813](https://github.com/ClickHouse/ClickHouse/pull/8813) ([アントン-ポポフ](https://github.com/CurtizJ)) -- システムのデータを取得している間にzookeeperの例外が発生した場合。レプリカは、別の列に表示します。 これは [\#9137](https://github.com/ClickHouse/ClickHouse/issues/9137) [\#9138](https://github.com/ClickHouse/ClickHouse/pull/9138) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Destroy上のMergeTreeデータ部分を原子的に削除します。 [\#8402](https://github.com/ClickHouse/ClickHouse/pull/8402) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 支援を行レベルのセキュリティ配布します。 [\#8926](https://github.com/ClickHouse/ClickHouse/pull/8926) ([イワン](https://github.com/abyss7)) -- Now we recognize suffix (like KB, KiB…) in settings values. [\#8072](https://github.com/ClickHouse/ClickHouse/pull/8072) ([Mikhail Korotov](https://github.com/millb)) -- 大きな結合の結果を構築しながらメモリ不足を防ぎます。 [\#8637](https://github.com/ClickHouse/ClickHouse/pull/8637) ([Artem Zuikov](https://github.com/4ertus2)) -- インタラクティブモードでの提案にクラスタの名前を追加 `clickhouse-client`. [\#8709](https://github.com/ClickHouse/ClickHouse/pull/8709) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Initialize query profiler for all threads in a group, e.g. it allows to fully profile insert-queries [\#8820](https://github.com/ClickHouse/ClickHouse/pull/8820) ([イワン](https://github.com/abyss7)) -- 追加された列 `exception_code` で `system.query_log` テーブル。 [\#8770](https://github.com/ClickHouse/ClickHouse/pull/8770) ([Mikhail Korotov](https://github.com/millb)) -- ポート上の有効mysql互換性サーバ `9004` デフォルトのサーバー設定ファイル。 設定の例の固定パスワード生成コマンド。 [\#8771](https://github.com/ClickHouse/ClickHouse/pull/8771) ([ユーリーバラノフ](https://github.com/yurriy)) -- 防止に停止した場合のファイルシステムが読み取り専用になります。 この修正 [\#9094](https://github.com/ClickHouse/ClickHouse/issues/9094) [\#9100](https://github.com/ClickHouse/ClickHouse/pull/9100) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- HTTP POSTクエリで長さが必要な場合は、より良い例外メッセージ。 [\#9453](https://github.com/ClickHouse/ClickHouse/pull/9453) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 追加 `_path` と `_file` 仮想列へ `HDFS` と `File` エンジンと `hdfs` と `file` テーブル関数 [\#8489](https://github.com/ClickHouse/ClickHouse/pull/8489) ([Olga Khvostikova](https://github.com/stavrolia)) -- エラーの修正 `Cannot find column` 挿入している間 `MATERIALIZED VIEW` 新しい列がビューの内部テーブルに追加された場合。 [\#8766](https://github.com/ClickHouse/ClickHouse/pull/8766) [\#8788](https://github.com/ClickHouse/ClickHouse/pull/8788) ([vzakaznikov](https://github.com/vzakaznikov)) [\#8788](https://github.com/ClickHouse/ClickHouse/issues/8788) [\#8806](https://github.com/ClickHouse/ClickHouse/pull/8806) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) [\#8803](https://github.com/ClickHouse/ClickHouse/pull/8803) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- ネイティブクライアントサーバープロトコルを介して、最終更新後の送信の進行状況を修正(ログなど)。 この問題にのみ一部の第三者ツールを使用するネイティブプロトコルです。 [\#9495](https://github.com/ClickHouse/ClickHouse/pull/9495) ([Azat Khuzhin](https://github.com/azat)) -- 追加システムの指標を追跡する多数のクライアント接続の待機を開始mysqlを使用してプロトコル ([\#9013](https://github.com/ClickHouse/ClickHouse/issues/9013)). [\#9015](https://github.com/ClickHouse/ClickHouse/pull/9015) ([Eugene Klimov](https://github.com/Slach)) -- これからは、httpレスポンスには `X-ClickHouse-Timezone` 同じタイムゾーン値に設定されたヘッダ `SELECT timezone()` 報告する [\#9493](https://github.com/ClickHouse/ClickHouse/pull/9493) ([Denis Glazachev](https://github.com/traceon)) - -#### 性能向上 {#performance-improvement} - -- INとの指標の分析のパフォーマンスを向上させる [\#9261](https://github.com/ClickHouse/ClickHouse/pull/9261) ([アントン-ポポフ](https://github.com/CurtizJ)) -- 論理関数+コードのクリーンアップで、よりシンプルで効率的なコード。 フォローアップへ [\#8718](https://github.com/ClickHouse/ClickHouse/issues/8718) [\#8728](https://github.com/ClickHouse/ClickHouse/pull/8728) ([Alexander Kazakov](https://github.com/Akazz)) -- 全体的なパフォーマンスの向上（5％の範囲で。.200％の影響のクエリをもっと厳しいエイリアシングとc++20特徴です。 [\#9304](https://github.com/ClickHouse/ClickHouse/pull/9304) ([アモスの鳥](https://github.com/amosbird)) -- 比較関数の内部ループのためのより厳密なエイリアシング。 [\#9327](https://github.com/ClickHouse/ClickHouse/pull/9327) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 算術関数の内部ループのより厳密なエイリアシング。 [\#9325](https://github.com/ClickHouse/ClickHouse/pull/9325) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- ColumnVector::replicate()の実装は、ColumnConst::convertToFullColumn()が実装されています。 また、定数を具体化する際のテストにも役立ちます。 [\#9293](https://github.com/ClickHouse/ClickHouse/pull/9293) ([Alexander Kazakov](https://github.com/Akazz)) -- 別のマイナーな性能向上へ `ColumnVector::replicate()` （これは `materialize` 機能および高位機能）へのそれ以上の改善 [\#9293](https://github.com/ClickHouse/ClickHouse/issues/9293) [\#9442](https://github.com/ClickHouse/ClickHouse/pull/9442) ([Alexander Kazakov](https://github.com/Akazz)) -- 改善された性能の `stochasticLinearRegression` 集計関数。 このパッチはIntelによって提供されます。 [\#8652](https://github.com/ClickHouse/ClickHouse/pull/8652) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 性能をの改善して下さい `reinterpretAsFixedString` 機能。 [\#9342](https://github.com/ClickHouse/ClickHouse/pull/9342) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- ブロックをクライアントに送信しない `Null` フォーマットプロセッサのパイプライン [\#8797](https://github.com/ClickHouse/ClickHouse/pull/8797) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) [\#8767](https://github.com/ClickHouse/ClickHouse/pull/8767) ([Alexander Kuzmenkov](https://github.com/akuzm)) - -#### ビルド/テスト/パッケージの改善 {#buildtestingpackaging-improvement} - -- 例外処理は現在、linux用のwindowsサブシステム上で正しく動作します。 見るhttps://github.com/clickhouse-extras/libunwind/pull/3 この修正 [\#6480](https://github.com/ClickHouse/ClickHouse/issues/6480) [\#9564](https://github.com/ClickHouse/ClickHouse/pull/9564) ([sobolevsv](https://github.com/sobolevsv)) -- 置換 `readline` と `replxx` インタラクティブライン編集 `clickhouse-client` [\#8416](https://github.com/ClickHouse/ClickHouse/pull/8416) ([イワン](https://github.com/abyss7)) -- FunctionsComparisonでより良いビルド時間と少ないテンプレートインスタンス化。 [\#9324](https://github.com/ClickHouse/ClickHouse/pull/9324) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- との統合を追加しました `clang-tidy` CIで。 また見なさい [\#6044](https://github.com/ClickHouse/ClickHouse/issues/6044) [\#9566](https://github.com/ClickHouse/ClickHouse/pull/9566) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 今、私たちはリンクを使用して、ciでclickhouse `lld` のために `gcc`. [\#9049](https://github.com/ClickHouse/ClickHouse/pull/9049) ([alesapin](https://github.com/alesapin)) -- するとランダムスレッドのスケジューリングに挿入しな障害の場合 `THREAD_FUZZER_*` 環境変数が設定されます。 これはテストを助ける。 [\#9459](https://github.com/ClickHouse/ClickHouse/pull/9459) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- ステートレステストでsecure socketsを有効にす [\#9288](https://github.com/ClickHouse/ClickHouse/pull/9288) ([tavplubix](https://github.com/tavplubix)) -- SPLIT\_SHARED\_LIBRARIES=OFFをより堅牢にする [\#9156](https://github.com/ClickHouse/ClickHouse/pull/9156) ([Azat Khuzhin](https://github.com/azat)) -- 作る “performance\_introspection\_and\_logging” 試験信頼性の高いランダムにサーバーの付かない。 これはCI環境で発生する可能性があります。 また見なさい [\#9515](https://github.com/ClickHouse/ClickHouse/issues/9515) [\#9528](https://github.com/ClickHouse/ClickHouse/pull/9528) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- スタイルチェックでxmlを検証する。 [\#9550](https://github.com/ClickHouse/ClickHouse/pull/9550) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- テストの競合状態を修正 `00738_lock_for_inner_table`. このテストは睡眠に頼った。 [\#9555](https://github.com/ClickHouse/ClickHouse/pull/9555) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 除去性能試験の種類 `once`. このに必要なすべての性能試験の統計比較モード(信頼性の高い). [\#9557](https://github.com/ClickHouse/ClickHouse/pull/9557) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 算術関数のパフォーマンステストを追加。 [\#9326](https://github.com/ClickHouse/ClickHouse/pull/9326) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- のための追加された性能試験 `sumMap` と `sumMapWithOverflow` 集計関数。 フォローアップのための [\#8933](https://github.com/ClickHouse/ClickHouse/issues/8933) [\#8947](https://github.com/ClickHouse/ClickHouse/pull/8947) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 確保のスタイルerrorcodesスタイルにチェック。 [\#9370](https://github.com/ClickHouse/ClickHouse/pull/9370) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 追加スクリプトのための試験。 [\#8796](https://github.com/ClickHouse/ClickHouse/pull/8796) ([alesapin](https://github.com/alesapin)) -- GCC警告を追加する `-Wsuggest-override` すべての場所を見つけて修正するには `override` キーワー [\#8760](https://github.com/ClickHouse/ClickHouse/pull/8760) ([kreuzerkrieg](https://github.com/kreuzerkrieg)) -- Mac OS Xの下で弱い記号を無視するのは、定義する必要があるためです [\#9538](https://github.com/ClickHouse/ClickHouse/pull/9538) ([削除されたユーザ](https://github.com/ghost)) -- パフォーマンステストでの一部のクエリの実行時間の正規化。 この準備の性能試験との比較モードになります。 [\#9565](https://github.com/ClickHouse/ClickHouse/pull/9565) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- クエリテストでpytestをサポートするテストを修正 [\#9062](https://github.com/ClickHouse/ClickHouse/pull/9062) ([イワン](https://github.com/abyss7)) -- をsslの構築とmsan、サーバーな起動時に走行時の状態試験 [\#9531](https://github.com/ClickHouse/ClickHouse/pull/9531) ([tavplubix](https://github.com/tavplubix)) -- テスト結果でのデータベース置換の修正 [\#9384](https://github.com/ClickHouse/ClickHouse/pull/9384) ([イリヤ-ヤツィシン](https://github.com/qoega)) -- の構築に対する修正その他ー [\#9381](https://github.com/ClickHouse/ClickHouse/pull/9381) ([proller](https://github.com/proller)) [\#8755](https://github.com/ClickHouse/ClickHouse/pull/8755) ([proller](https://github.com/proller)) [\#8631](https://github.com/ClickHouse/ClickHouse/pull/8631) ([proller](https://github.com/proller)) -- 追加ディスク部無国籍-と-カバレッジ-テストdocker画像 [\#9213](https://github.com/ClickHouse/ClickHouse/pull/9213) ([Pavel Kovalenko](https://github.com/Jokser)) -- GRPCでビルドするときに、ソースツリー内のファイルを取り除く [\#9588](https://github.com/ClickHouse/ClickHouse/pull/9588) ([アモスの鳥](https://github.com/amosbird)) -- 少し早く構築時間を取り除いsessioncleanerからのコンテキスト sessioncleanerのコードをよりシンプルにする。 [\#9232](https://github.com/ClickHouse/ClickHouse/pull/9232) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Clickhouseテストスクリプトでハングクエリのチェックを更新 [\#8858](https://github.com/ClickHouse/ClickHouse/pull/8858) ([Alexander Kazakov](https://github.com/Akazz)) -- リポジトリか [\#8843](https://github.com/ClickHouse/ClickHouse/pull/8843) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- から数学perftestsの変更タイプ `once` に `loop`. [\#8783](https://github.com/ClickHouse/ClickHouse/pull/8783) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 追加docker画像を構築ィコードのブラウザのhtmlレポート当社のコードベース. [\#8781](https://github.com/ClickHouse/ClickHouse/pull/8781) ([alesapin](https://github.com/alesapin))見る [Woboqコードブラウザ](https://clickhouse.tech/codebrowser/html_report///ClickHouse/dbms/src/index.html) -- MSanの下でいくつかのテストの失敗を抑制. [\#8780](https://github.com/ClickHouse/ClickHouse/pull/8780) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- スピードアップ “exception while insert” テスト。 このテス [\#8711](https://github.com/ClickHouse/ClickHouse/pull/8711) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 更新 `libcxx` と `libcxxabi` マスターに。 準備のために [\#9304](https://github.com/ClickHouse/ClickHouse/issues/9304) [\#9308](https://github.com/ClickHouse/ClickHouse/pull/9308) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- フラッキーテストの修正 `00910_zookeeper_test_alter_compression_codecs`. [\#9525](https://github.com/ClickHouse/ClickHouse/pull/9525) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 清掃は複製リンカのフラグがあります。 リンカーが予期しないシンボルを検索しないことを確認します。 [\#9433](https://github.com/ClickHouse/ClickHouse/pull/9433) ([アモスの鳥](https://github.com/amosbird)) -- 追加 `clickhouse-odbc` テスト画像にドライバ。 これは、独自のODBCドライバを経由してClickHouseとClickHouseの相互作用をテストすることができます。 [\#9348](https://github.com/ClickHouse/ClickHouse/pull/9348) ([フィリモノフ](https://github.com/filimonov)) -- 単体テストでいくつかのバグを修正。 [\#9047](https://github.com/ClickHouse/ClickHouse/pull/9047) ([alesapin](https://github.com/alesapin)) -- 有効 `-Wmissing-include-dirs` CMakeスクリプトエラーの結果として、すべての既存のインクルードを排除するGCC警告 [\#8704](https://github.com/ClickHouse/ClickHouse/pull/8704) ([kreuzerkrieg](https://github.com/kreuzerkrieg)) -- クエリプ これは [\#9049](https://github.com/ClickHouse/ClickHouse/issues/9049) [\#9144](https://github.com/ClickHouse/ClickHouse/pull/9144) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Opensslを上流のマスターに更新します。 メッセージでTLS接続が失敗する問題を修正しました `OpenSSL SSL_read: error:14094438:SSL routines:ssl3_read_bytes:tlsv1 alert internal error` と `SSL Exception: error:2400006E:random number generator::error retrieving entropy`. この問題はバージョン20.1に存在していました。 [\#8956](https://github.com/ClickHouse/ClickHouse/pull/8956) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- サーバーのdockerfileの更新 [\#8893](https://github.com/ClickHouse/ClickHouse/pull/8893) ([Ilya Mazaev](https://github.com/ne-ray)) -- ビルド-gcc-from-sourcesスクリプトのマイナーな修正 [\#8774](https://github.com/ClickHouse/ClickHouse/pull/8774) ([Michael Nacharov](https://github.com/mnach)) -- 置換 `numbers` に `zeros` どこperftestsで `number` 列は使用されません。 これはよりきれいなテスト結果につながります。 [\#9600](https://github.com/ClickHouse/ClickHouse/pull/9600) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 固定しスタックオーバーフローされる際に問題が起きた場合、利用initializer\_list列コンストラクタ. [\#9367](https://github.com/ClickHouse/ClickHouse/pull/9367) ([削除されたユーザ](https://github.com/ghost)) -- Libdkafkaをv1.3.0にアップグレードします。 バンドル有効 `rdkafka` と `gsasl` Mac OS X上のライブラリ [\#9000](https://github.com/ClickHouse/ClickHouse/pull/9000) ([Andrew Onyshchuk](https://github.com/oandrew)) -- GCC9.2.0でのビルド修正 [\#9306](https://github.com/ClickHouse/ClickHouse/pull/9306) ([vxider](https://github.com/Vxider)) - -## ClickHouseリリースv20.1 {#clickhouse-release-v20-1} - -### ClickHouseリリリースv20.1.8.41,2020-03-20 {#clickhouse-release-v20-1-8-41-2020-03-20} - -#### バグ修正 {#bug-fix-3} - -- 可能永久修正 `Cannot schedule a task` エラー(ハンドルされていない例外が原因で `ParallelAggregatingBlockInputStream::Handler::onFinish/onFinishThread`). この修正 [\#6833](https://github.com/ClickHouse/ClickHouse/issues/6833). [\#9154](https://github.com/ClickHouse/ClickHouse/pull/9154) ([Azat Khuzhin](https://github.com/azat)) -- 過度のメモリ消費を修正 `ALTER` クエリ（突然変異）。 この修正 [\#9533](https://github.com/ClickHouse/ClickHouse/issues/9533) と [\#9670](https://github.com/ClickHouse/ClickHouse/issues/9670). [\#9754](https://github.com/ClickHouse/ClickHouse/pull/9754) ([alesapin](https://github.com/alesapin)) -- 外部辞書のddlにバッククォートのバグを修正しました。 この修正 [\#9619](https://github.com/ClickHouse/ClickHouse/issues/9619). [\#9734](https://github.com/ClickHouse/ClickHouse/pull/9734) ([alesapin](https://github.com/alesapin)) - -### ClickHouseリリリースv20.1.7.38,2020-03-18 {#clickhouse-release-v20-1-7-38-2020-03-18} - -#### バグ修正 {#bug-fix-4} - -- 固定誤った内部関数名のための `sumKahan` と `sumWithOverflow`. 先頭に立って例外がこの機能をリモートます。 [\#9636](https://github.com/ClickHouse/ClickHouse/pull/9636) ([Azat Khuzhin](https://github.com/azat)). この問題はすべてClickHouseのリリースにありました。 -- 許可 `ALTER ON CLUSTER` の `Distributed` 内部レプリケーショ この修正 [\#3268](https://github.com/ClickHouse/ClickHouse/issues/3268). [\#9617](https://github.com/ClickHouse/ClickHouse/pull/9617) ([品生2](https://github.com/shinoi2)). この問題はすべてClickHouseのリリースにありました。 -- 可能な例外を修正 `Size of filter doesn't match size of column` と `Invalid number of rows in Chunk` で `MergeTreeRangeReader`. 実行中に表示される可能性があります `PREWHERE` いくつかのケースでは。 修正 [\#9132](https://github.com/ClickHouse/ClickHouse/issues/9132). [\#9612](https://github.com/ClickHouse/ClickHouse/pull/9612) ([アントン-ポポフ](https://github.com/CurtizJ)) -- 次のような単純な算術式を書くと、タイムゾーンが保持されないという問題を修正しました `time + 1` （次のような表現とは対照的に `time + INTERVAL 1 SECOND`). この修正 [\#5743](https://github.com/ClickHouse/ClickHouse/issues/5743). [\#9323](https://github.com/ClickHouse/ClickHouse/pull/9323) ([alexey-milovidov](https://github.com/alexey-milovidov)). この問題はすべてClickHouseのリリースにありました。 -- 次のような単純な循環エイリアスを持つ列を作成または追加することはできません `a DEFAULT b, b DEFAULT a`. [\#9603](https://github.com/ClickHouse/ClickHouse/pull/9603) ([alesapin](https://github.com/alesapin)) -- Base64でエンコードされた値の末尾にパディングが不正な形式になる場合がある問題を修正しました。 更新base64ライブラリ。 この修正 [\#9491](https://github.com/ClickHouse/ClickHouse/issues/9491)、閉じます [\#9492](https://github.com/ClickHouse/ClickHouse/issues/9492) [\#9500](https://github.com/ClickHouse/ClickHouse/pull/9500) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- の破壊でデータレースを修正 `Poco::HTTPServer`. どこの場合のサーバを起動直ちに停止しております。 [\#9468](https://github.com/ClickHouse/ClickHouse/pull/9468) ([アントン-ポポフ](https://github.com/CurtizJ)) -- 可能なクラッシュ/間違った行数を修正 `LIMIT n WITH TIES` n行目に等しい行がたくさんあるとき。 [\#9464](https://github.com/ClickHouse/ClickHouse/pull/9464) ([tavplubix](https://github.com/tavplubix)) -- 列ttlsによるチェックサムの不一致を修正しました。 [\#9451](https://github.com/ClickHouse/ClickHouse/pull/9451) ([アントン-ポポフ](https://github.com/CurtizJ)) -- ユーザーが `ALTER MODIFY SETTING` 古いformatedのため `MergeTree` テーブルエンジン家族。 [\#9435](https://github.com/ClickHouse/ClickHouse/pull/9435) ([alesapin](https://github.com/alesapin)) -- 今度は、突然変異をより頻繁に確定しようとします。 [\#9427](https://github.com/ClickHouse/ClickHouse/pull/9427) ([alesapin](https://github.com/alesapin)) -- レプリケーションプロト [\#8598](https://github.com/ClickHouse/ClickHouse/issues/8598). [\#9412](https://github.com/ClickHouse/ClickHouse/pull/9412) ([alesapin](https://github.com/alesapin)) -- 配列型のbloom\_filterインデックスのnot(has())を修正しました。 [\#9407](https://github.com/ClickHouse/ClickHouse/pull/9407) ([achimbabcomment](https://github.com/achimbab)) -- の動作を修正しました `match` と `extract` haystackにゼロバイトがある場合の関数。 Haystackが一定の場合、その動作は間違っていました。 この修正 [\#9160](https://github.com/ClickHouse/ClickHouse/issues/9160) [\#9163](https://github.com/ClickHouse/ClickHouse/pull/9163) ([alexey-milovidov](https://github.com/alexey-milovidov)) [\#9345](https://github.com/ClickHouse/ClickHouse/pull/9345) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -#### ビルド/テスト/パッケージの改善 {#buildtestingpackaging-improvement-1} - -- 例外処理は現在、linux用のwindowsサブシステム上で正しく動作します。 見るhttps://github.com/clickhouse-extras/libunwind/pull/3 この修正 [\#6480](https://github.com/ClickHouse/ClickHouse/issues/6480) [\#9564](https://github.com/ClickHouse/ClickHouse/pull/9564) ([sobolevsv](https://github.com/sobolevsv)) - -### ClickHouseリリリースv20.1.6.30,2020-03-05 {#clickhouse-release-v20-1-6-30-2020-03-05} - -#### バグ修正 {#bug-fix-5} - -- 圧縮時にデータの非互換性を修正する `T64` コーデック。 - [\#9039](https://github.com/ClickHouse/ClickHouse/pull/9039) [(abyss7)](https://github.com/abyss7) -- 一つのスレッドでmergetreeテーブルから読み込み中の範囲の順序を修正しました。 修正 [\#8964](https://github.com/ClickHouse/ClickHouse/issues/8964). - [\#9050](https://github.com/ClickHouse/ClickHouse/pull/9050) [(CurtizJ)](https://github.com/CurtizJ) -- で可能なsegfaultを修正 `MergeTreeRangeReader`,実行中 `PREWHERE`. 修正 [\#9064](https://github.com/ClickHouse/ClickHouse/issues/9064). - [\#9106](https://github.com/ClickHouse/ClickHouse/pull/9106) [(CurtizJ)](https://github.com/CurtizJ) -- 修正 `reinterpretAsFixedString` 戻るには `FixedString` 代わりに `String`. - [\#9052](https://github.com/ClickHouse/ClickHouse/pull/9052) [（oandrew)](https://github.com/oandrew) -- 修正 `joinGet` null可能な戻り値の型を指定します。 修正 [\#8919](https://github.com/ClickHouse/ClickHouse/issues/8919) - [\#9014](https://github.com/ClickHouse/ClickHouse/pull/9014) [（アモスバード)](https://github.com/amosbird) -- ファズテストとbittestall/bittestany関数の不正な動作を修正しました。 - [\#9143](https://github.com/ClickHouse/ClickHouse/pull/9143) [（アレクセイ-ミロビドフ)](https://github.com/alexey-milovidov) -- Haystackにゼロバイトがある場合、match関数とextract関数の動作を修正しました。 Haystackが一定の場合、その動作は間違っていました。 修正 [\#9160](https://github.com/ClickHouse/ClickHouse/issues/9160) - [\#9163](https://github.com/ClickHouse/ClickHouse/pull/9163) [（アレクセイ-ミロビドフ)](https://github.com/alexey-milovidov) -- 非厳密に単調な関数索引が使用されている場合の逆述語の実行を修正しました。 修正 [\#9034](https://github.com/ClickHouse/ClickHouse/issues/9034) - [\#9223](https://github.com/ClickHouse/ClickHouse/pull/9223) [（Akazz)](https://github.com/Akazz) -- 書き換えを許可する `CROSS` に `INNER JOIN` もしあれば `[NOT] LIKE` 演算子in `WHERE` セクション。 修正 [\#9191](https://github.com/ClickHouse/ClickHouse/issues/9191) - [\#9229](https://github.com/ClickHouse/ClickHouse/pull/9229) [(4tus2)](https://github.com/4ertus2) -- ログエンジンを持つテーブルの最初の列をエイリアスにする。 - [\#9231](https://github.com/ClickHouse/ClickHouse/pull/9231) [(abyss7)](https://github.com/abyss7) -- カンマの結合を許可する `IN()` 中に 修正 [\#7314](https://github.com/ClickHouse/ClickHouse/issues/7314). - [\#9251](https://github.com/ClickHouse/ClickHouse/pull/9251) [(4tus2)](https://github.com/4ertus2) -- 改善する `ALTER MODIFY/ADD` クエリロジック。 今はできません `ADD` タイプのない列, `MODIFY` デフォルトの式では、列の型は変更されません。 `MODIFY` 型は既定の式の値を緩めません。 修正 [\#8669](https://github.com/ClickHouse/ClickHouse/issues/8669). - [\#9227](https://github.com/ClickHouse/ClickHouse/pull/9227) [(alesapin)](https://github.com/alesapin) -- 既に行われた変異は、ステータスis\_done=0を持つことができたときに、突然変異の終了を修正。 - [\#9217](https://github.com/ClickHouse/ClickHouse/pull/9217) [(alesapin)](https://github.com/alesapin) -- サポート “Processors” システムのため数字とシステム.numbers\_mt. これはまたバグを修正します `max_execution_time` 尊重されていません。 - [\#7796](https://github.com/ClickHouse/ClickHouse/pull/7796) [（KochetovNicolai)](https://github.com/KochetovNicolai) -- の間違ったカウントを修正 `DictCacheKeysRequestedFound` メトリック。 - [\#9411](https://github.com/ClickHouse/ClickHouse/pull/9411) [（nikitamikhaylov)](https://github.com/nikitamikhaylov) -- ストレージポリシーのチェックを追加 `ATTACH PARTITION FROM`, `REPLACE PARTITION`, `MOVE TO TABLE` 合がデータの一部になり、再起動後の防止ClickHouse。 - [\#9383](https://github.com/ClickHouse/ClickHouse/pull/9383) [（エキシーン)](https://github.com/excitoon) -- 固定ubsanレポートで `MergeTreeIndexSet`. この修正 [\#9250](https://github.com/ClickHouse/ClickHouse/issues/9250) - [\#9365](https://github.com/ClickHouse/ClickHouse/pull/9365) [（アレクセイ-ミロビドフ)](https://github.com/alexey-milovidov) -- BlockIOで可能なdataraceを修正. - [\#9356](https://github.com/ClickHouse/ClickHouse/pull/9356) [（KochetovNicolai)](https://github.com/KochetovNicolai) -- のサポート `UInt64` JSON関連の関数のInt64に収まらない数値。 更新 `SIMDJSON` マスターに。 この修正 [\#9209](https://github.com/ClickHouse/ClickHouse/issues/9209) - [\#9344](https://github.com/ClickHouse/ClickHouse/pull/9344) [（アレクセイ-ミロビドフ)](https://github.com/alexey-milovidov) -- 具合を修正しましたが、金額のフリースペースが正しく計算されませんが、データディレクトリに取り付けには別の装置です。 デフォルトのディスクの計算には無料のスペースからデータのサブディレクトリの. この修正 [\#7441](https://github.com/ClickHouse/ClickHouse/issues/7441) - [\#9257](https://github.com/ClickHouse/ClickHouse/pull/9257) [（ミルブ)](https://github.com/millb) -- メッセージでtls接続が失敗する場合の問題を修正しました `OpenSSL SSL_read: error:14094438:SSL routines:ssl3_read_bytes:tlsv1 alert internal error and SSL Exception: error:2400006E:random number generator::error retrieving entropy.` Opensslを上流のマスターに更新します。 - [\#8956](https://github.com/ClickHouse/ClickHouse/pull/8956) [（アレクセイ-ミロビドフ)](https://github.com/alexey-milovidov) -- 実行時期 `CREATE` クエリー、倍定表現のストレージエンジンの引数です。 空のデータベース名を現在のデータベ 修正 [\#6508](https://github.com/ClickHouse/ClickHouse/issues/6508), [\#3492](https://github.com/ClickHouse/ClickHouse/issues/3492). また、ClickHouseDictionarySourceのローカルアドレスのチェックを修正しました。 - [\#9262](https://github.com/ClickHouse/ClickHouse/pull/9262) [(tabplubix)](https://github.com/tavplubix) -- Segfaultを修正する `StorageMerge` これは、StorageFileから読み込むときに発生します。 - [\#9387](https://github.com/ClickHouse/ClickHouse/pull/9387) [(tabplubix)](https://github.com/tavplubix) -- データを失うのを防ぐ `Kafka` まれに、接尾辞を読んだ後でコミットする前に例外が発生した場合。 修正 [\#9378](https://github.com/ClickHouse/ClickHouse/issues/9378). 関連: [\#7175](https://github.com/ClickHouse/ClickHouse/issues/7175) - [\#9507](https://github.com/ClickHouse/ClickHouse/pull/9507) [（フィリモノフ)](https://github.com/filimonov) -- のを修正した。先サーバを終了しようとした場合に使用/drop `Kafka` テーブル作成されたパラメータ。 修正 [\#9494](https://github.com/ClickHouse/ClickHouse/issues/9494). 組み込み [\#9507](https://github.com/ClickHouse/ClickHouse/issues/9507). - [\#9513](https://github.com/ClickHouse/ClickHouse/pull/9513) [（フィリモノフ)](https://github.com/filimonov) - -#### 新しい機能 {#new-feature-1} - -- 追加 `deduplicate_blocks_in_dependent_materialized_views` マテリアライズドビューを持つテーブルへの冪等挿入の動作を制御するオプション。 この新機能は、Altinityからの特別な要求によってbugfixリリースに追加されました。 - [\#9070](https://github.com/ClickHouse/ClickHouse/pull/9070) [（urykhy)](https://github.com/urykhy) - -### ClickHouseリリースv20.1.2.4,2020-01-22 {#clickhouse-release-v20-1-2-4-2020-01-22} - -#### 下位互換性のない変更 {#backward-incompatible-change-1} - -- 設定を行う `merge_tree_uniform_read_distribution` 廃止されました。 サーバーはこの設定を認識しますが、効果はありません。 [\#8308](https://github.com/ClickHouse/ClickHouse/pull/8308) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 関数の戻り値の型を変更しました `greatCircleDistance` に `Float32` なぜなら今計算の結果は `Float32`. [\#7993](https://github.com/ClickHouse/ClickHouse/pull/7993) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- これで、クエリパラメータが “escaped” フォーマット。 たとえば、文字列を渡すには `ab` あなたは `a\tb` または `a\b` とそれぞれ, `a%5Ctb` または `a%5C%09b` URLで。 これは、NULLを渡す可能性を追加するために必要です `\N`. この修正 [\#7488](https://github.com/ClickHouse/ClickHouse/issues/7488). [\#8517](https://github.com/ClickHouse/ClickHouse/pull/8517) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 有効 `use_minimalistic_part_header_in_zookeeper` のための設定 `ReplicatedMergeTree` デフォルトでは。 このことを大幅に削減量のデータが保存されて飼育係. この設定はバージョン19.1以降でサポートされており、半年以上問題なく複数のサービスで本番環境で使用されています。 19.1より古いバージョンにダウングレードできる場合は、この設定を無効にします。 [\#6850](https://github.com/ClickHouse/ClickHouse/pull/6850) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- データの飛び設計生産準備、デフォルトで有効です. 設定 `allow_experimental_data_skipping_indices`, `allow_experimental_cross_to_join_conversion` と `allow_experimental_multiple_joins_emulation` 今は時代遅れであり、何もしません。 [\#7974](https://github.com/ClickHouse/ClickHouse/pull/7974) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 新規追加 `ANY JOIN` のための論理 `StorageJoin` と一貫した `JOIN` オペレーション 動作を変更せずにアップグレードするには、 `SETTINGS any_join_distinct_right_table_keys = 1` エンジンにテーブルを追加のメタデータを再現これらのテーブル後のアップグレードを開始します。 [\#8400](https://github.com/ClickHouse/ClickHouse/pull/8400) ([Artem Zuikov](https://github.com/4ertus2)) -- ログ設定の変更を適用するには、サーバーを再起動する必要があります。 これは、サーバーが削除されたログファイルにログを記録するバグを回避するための一時的な回避策です。 [\#8696](https://github.com/ClickHouse/ClickHouse/issues/8696)). [\#8707](https://github.com/ClickHouse/ClickHouse/pull/8707) ([Alexander Kuzmenkov](https://github.com/akuzm)) - -#### 新しい機能 {#new-feature-2} - -- パーツパスに関する情報を追加 `system.merges`. [\#8043](https://github.com/ClickHouse/ClickHouse/pull/8043) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 実行する機能を追加 `SYSTEM RELOAD DICTIONARY` でクエリ `ON CLUSTER` モード。 [\#8288](https://github.com/ClickHouse/ClickHouse/pull/8288) ([ギヨームタッセリー](https://github.com/YiuRULE)) -- 実行する機能を追加 `CREATE DICTIONARY` でのクエリ `ON CLUSTER` モード。 [\#8163](https://github.com/ClickHouse/ClickHouse/pull/8163) ([alesapin](https://github.com/alesapin)) -- 今、ユーザーのプロフィール `users.xml` 継承した複数のデータ。 [\#8343](https://github.com/ClickHouse/ClickHouse/pull/8343) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) -- 追加 `system.stack_trace` テーブルで眺めるスタックトレースのすべてのサーバスレッド）。 これは、開発者がサーバーの状態をイントロスペクトするのに便利です。 この修正 [\#7576](https://github.com/ClickHouse/ClickHouse/issues/7576). [\#8344](https://github.com/ClickHouse/ClickHouse/pull/8344) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 追加 `DateTime64` 設定可能な秒未満の精度を持つデータ型。 [\#7170](https://github.com/ClickHouse/ClickHouse/pull/7170) ([Vasily Nemkov](https://github.com/Enmk)) -- テーブル関数の追加 `clusterAllReplicas` ることのできるクエリのすべてのノードのクラスター [\#8493](https://github.com/ClickHouse/ClickHouse/pull/8493) ([キラン-スンカリ](https://github.com/kiransunkari)) -- 集計関数の追加 `categoricalInformationValue` これは、離散フィーチャの情報値を計算します。 [\#8117](https://github.com/ClickHouse/ClickHouse/pull/8117) ([hcz](https://github.com/hczhcz)) -- データファイルの解析を高速化 `CSV`, `TSV` と `JSONEachRow` それを並行して行うことによって書式。 [\#7780](https://github.com/ClickHouse/ClickHouse/pull/7780) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- 機能を追加 `bankerRound` これは、銀行の丸めを実行します。 [\#8112](https://github.com/ClickHouse/ClickHouse/pull/8112) ([hcz](https://github.com/hczhcz)) -- 地域名の埋め込み辞書でより多くの言語をサポート: ‘ru’, ‘en’, ‘ua’, ‘uk’, ‘by’, ‘kz’, ‘tr’, ‘de’, ‘uz’, ‘lv’, ‘lt’, ‘et’, ‘pt’, ‘he’, ‘vi’. [\#8189](https://github.com/ClickHouse/ClickHouse/pull/8189) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- の一貫性の改善 `ANY JOIN` ロジック。 さて `t1 ANY LEFT JOIN t2` 等しい `t2 ANY RIGHT JOIN t1`. [\#7665](https://github.com/ClickHouse/ClickHouse/pull/7665) ([Artem Zuikov](https://github.com/4ertus2)) -- 設定を追加 `any_join_distinct_right_table_keys` これは古い動作を可能にします `ANY INNER JOIN`. [\#7665](https://github.com/ClickHouse/ClickHouse/pull/7665) ([Artem Zuikov](https://github.com/4ertus2)) -- 新規追加 `SEMI` と `ANTI JOIN`. 古い `ANY INNER JOIN` 行動として `SEMI LEFT JOIN`. [\#7665](https://github.com/ClickHouse/ClickHouse/pull/7665) ([Artem Zuikov](https://github.com/4ertus2)) -- 追加 `Distributed` の形式 `File` エンジンと `file` から読むことを可能にするテーブル機能 `.bin` によって生成されたファイル `Distributed` テーブル。 [\#8535](https://github.com/ClickHouse/ClickHouse/pull/8535) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- オプションのreset column引数を追加する `runningAccumulate` これにより、各新しいキー値の集計結果をリセットできます。 [\#8326](https://github.com/ClickHouse/ClickHouse/pull/8326) ([Sergey Kononenko](https://github.com/kononencheg)) -- PrometheusエンドポイントとしてClickHouseを使用する機能を追加します。 [\#7900](https://github.com/ClickHouse/ClickHouse/pull/7900) ([vdimir](https://github.com/Vdimir)) -- セクションを追加 `` で `config.xml` の制約が許されたアイテムのリモートテーブルエンジンとテーブル機能 `URL`, `S3`, `HDFS`. [\#7154](https://github.com/ClickHouse/ClickHouse/pull/7154) ([Mikhail Korotov](https://github.com/millb)) -- 機能追加 `greatCircleAngle` これは度で球の距離を計算します。 [\#8105](https://github.com/ClickHouse/ClickHouse/pull/8105) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 地球の半径をh3ライブラリと一致するように変更しました。 [\#8105](https://github.com/ClickHouse/ClickHouse/pull/8105) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 追加 `JSONCompactEachRow` と `JSONCompactEachRowWithNamesAndTypes` 入力と出力の形式。 [\#7841](https://github.com/ClickHouse/ClickHouse/pull/7841) ([Mikhail Korotov](https://github.com/millb)) -- ファイル関連のテーブルエンジンとテーブル関数の機能を追加 (`File`, `S3`, `URL`, `HDFS`）読み書きすることができます `gzip` ファイルに基づく追加のエンジンのパラメータまたはファイル拡張子. [\#7840](https://github.com/ClickHouse/ClickHouse/pull/7840) ([Andrey Bodrov](https://github.com/apbodrov)) -- を追加しました `randomASCII(length)` のランダムなセットを持つ文字列を生成する関数 [ASCII](https://en.wikipedia.org/wiki/ASCII#Printable_characters) 印刷可能な文字。 [\#8401](https://github.com/ClickHouse/ClickHouse/pull/8401) ([バヨネット](https://github.com/BayoNet)) -- 機能追加 `JSONExtractArrayRaw` これは、解析されていないjson配列要素の配列を返します `JSON` 文字列。 [\#8081](https://github.com/ClickHouse/ClickHouse/pull/8081) ([Oleg Matrokhin](https://github.com/errx)) -- 追加 `arrayZip` 等しい長さの複数の配列をタプルの一つの配列に結合することを可能にする関数。 [\#8149](https://github.com/ClickHouse/ClickHouse/pull/8149) ([冬張](https://github.com/zhang2014)) -- 設定に従ってディスク間でデータを移動する機能を追加 `TTL`-のための式 `*MergeTree` テーブルエンジン家族。 [\#8140](https://github.com/ClickHouse/ClickHouse/pull/8140) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 新しい集計関数を追加 `avgWeighted` 加重平均を計算することができます。 [\#7898](https://github.com/ClickHouse/ClickHouse/pull/7898) ([Andrey Bodrov](https://github.com/apbodrov)) -- デフォルトでは、並列解析が有効になりました `TSV`, `TSKV`, `CSV` と `JSONEachRow` フォーマット。 [\#7894](https://github.com/ClickHouse/ClickHouse/pull/7894) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- からいくつかの地理機能を追加 `H3` ライブラリ: `h3GetResolution`, `h3EdgeAngle`, `h3EdgeLength`, `h3IsValid` と `h3kRing`. [\#8034](https://github.com/ClickHouse/ClickHouse/pull/8034) ([コンスタンチン-マランチェフ](https://github.com/hombit)) -- Brotliのサポートを追加しました (`br` ファイル関連のストレージとテーブルの機能で）圧縮。 この修正 [\#8156](https://github.com/ClickHouse/ClickHouse/issues/8156). [\#8526](https://github.com/ClickHouse/ClickHouse/pull/8526) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 追加 `groupBit*` のための機能 `SimpleAggregationFunction` タイプ。 [\#8485](https://github.com/ClickHouse/ClickHouse/pull/8485) ([ギヨームタッセリー](https://github.com/YiuRULE)) - -#### バグ修正 {#bug-fix-6} - -- とテーブルの名前の変更を修正 `Distributed` エンジン。 修正の問題 [\#7868](https://github.com/ClickHouse/ClickHouse/issues/7868). [\#8306](https://github.com/ClickHouse/ClickHouse/pull/8306) ([tavplubix](https://github.com/tavplubix)) -- 今辞書サポート `EXPRESSION` 非ClickHouse SQLダイアレクト内の任意の文字列の属性の場合。 [\#8098](https://github.com/ClickHouse/ClickHouse/pull/8098) ([alesapin](https://github.com/alesapin)) -- 壊れた修正 `INSERT SELECT FROM mysql(...)` クエリ。 この修正 [\#8070](https://github.com/ClickHouse/ClickHouse/issues/8070) と [\#7960](https://github.com/ClickHouse/ClickHouse/issues/7960). [\#8234](https://github.com/ClickHouse/ClickHouse/pull/8234) ([tavplubix](https://github.com/tavplubix)) -- エラーの修正 “Mismatch column sizes” デフォルトを挿入する `Tuple` から `JSONEachRow`. この修正 [\#5653](https://github.com/ClickHouse/ClickHouse/issues/5653). [\#8606](https://github.com/ClickHouse/ClickHouse/pull/8606) ([tavplubix](https://github.com/tavplubix)) -- これで、usingの場合に例外がスローされます `WITH TIES` 一緒に `LIMIT BY`. また、使用する機能を追加 `TOP` と `LIMIT BY`. この修正 [\#7472](https://github.com/ClickHouse/ClickHouse/issues/7472). [\#7637](https://github.com/ClickHouse/ClickHouse/pull/7637) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- 新鮮なglibcバージョンからの意図しない依存関係を修正 `clickhouse-odbc-bridge` バイナリ [\#8046](https://github.com/ClickHouse/ClickHouse/pull/8046) ([アモスの鳥](https://github.com/amosbird)) -- のチェック機能のバグを修正 `*MergeTree` エンジンファミリー 最後の顆粒と最後のマーク（最終ではない）に等しい量の行がある場合、今度は失敗しません。 [\#8047](https://github.com/ClickHouse/ClickHouse/pull/8047) ([alesapin](https://github.com/alesapin)) -- に挿入を修正 `Enum*` 後の列 `ALTER` 基になる数値型がテーブル指定された型と等しい場合のクエリです。 この修正 [\#7836](https://github.com/ClickHouse/ClickHouse/issues/7836). [\#7908](https://github.com/ClickHouse/ClickHouse/pull/7908) ([アントン-ポポフ](https://github.com/CurtizJ)) -- 許可される非定数負 “size” 関数の引数 `substring`. それは誤って許可されませんでした。 この修正 [\#4832](https://github.com/ClickHouse/ClickHouse/issues/4832). [\#7703](https://github.com/ClickHouse/ClickHouse/pull/7703) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 引数の数が間違って渡されたときにバグを解析する修正 `(O|J)DBC` テーブルエンジン。 [\#7709](https://github.com/ClickHouse/ClickHouse/pull/7709) ([alesapin](https://github.com/alesapin)) -- Syslogにログを送信するときに、実行中のclickhouseプロセスのコマンド名を使用します。 以前のバージョ [\#8460](https://github.com/ClickHouse/ClickHouse/pull/8460) ([Michael Nacharov](https://github.com/mnach)) -- 許可されたホストの修正チェック `localhost`. このPRでは、 [\#8241](https://github.com/ClickHouse/ClickHouse/pull/8241). [\#8342](https://github.com/ClickHouse/ClickHouse/pull/8342) ([Vitaly Baranov](https://github.com/vitlibar)) -- でレアクラッシュを修正 `argMin` と `argMax` resultが使用されているときの長い文字列引数の関数 `runningAccumulate` 機能。 この修正 [\#8325](https://github.com/ClickHouse/ClickHouse/issues/8325) [\#8341](https://github.com/ClickHouse/ClickHouse/pull/8341) ([恐竜](https://github.com/769344359)) -- とテーブルのメモリオーバーコミットを修正 `Buffer` エンジン。 [\#8345](https://github.com/ClickHouse/ClickHouse/pull/8345) ([Azat Khuzhin](https://github.com/azat)) -- 取ることができる機能の潜在的なバグを修正 `NULL` 引数の一つとして、非NULLを返します。 [\#8196](https://github.com/ClickHouse/ClickHouse/pull/8196) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- より良いメトリクス計算のスレッドプールを対象としたバックグラウンドプロセス `MergeTree` テーブルエンジン。 [\#8194](https://github.com/ClickHouse/ClickHouse/pull/8194) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 修正機能 `IN` 中 `WHERE` 決が行レベルテーブルフィルターがあります。 修正 [\#6687](https://github.com/ClickHouse/ClickHouse/issues/6687) [\#8357](https://github.com/ClickHouse/ClickHouse/pull/8357) ([イワン](https://github.com/abyss7)) -- これで、設定値の整数値が完全に解析されない場合、例外がスローされます。 [\#7678](https://github.com/ClickHouse/ClickHouse/pull/7678) ([Mikhail Korotov](https://github.com/millb)) -- 複数のローカルシャードを持つ分散テーブルへのクエリで集計関数を使用すると例外が修正されました。 [\#8164](https://github.com/ClickHouse/ClickHouse/pull/8164) ([小路](https://github.com/nicelulu)) -- Bloom filterは長さゼロの配列を扱うことができ、冗長な計算を実行しません。 [\#8242](https://github.com/ClickHouse/ClickHouse/pull/8242) ([achimbabcomment](https://github.com/achimbab)) -- クライアン `host_regexp` で指定される `users.xml`. [\#8241](https://github.com/ClickHouse/ClickHouse/pull/8241) ([Vitaly Baranov](https://github.com/vitlibar)) -- 複数の偽陽性につながるあいまいな列チェックを緩和する `JOIN ON` セクション。 [\#8385](https://github.com/ClickHouse/ClickHouse/pull/8385) ([Artem Zuikov](https://github.com/4ertus2)) -- 固定可能なサーバのクラッシュ (`std::terminate`)サーバーがデータを送信または書き込みできない場合 `JSON` または `XML` の値を持つ形式 `String` データ型(必要なデータ型 `UTF-8` 検証）またはBrotliアルゴリズムまたは他のまれなケースで結果データを圧縮するとき。 この修正 [\#7603](https://github.com/ClickHouse/ClickHouse/issues/7603) [\#8384](https://github.com/ClickHouse/ClickHouse/pull/8384) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 競合状態の修正 `StorageDistributedDirectoryMonitor` CIによって発見。 この修正 [\#8364](https://github.com/ClickHouse/ClickHouse/issues/8364). [\#8383](https://github.com/ClickHouse/ClickHouse/pull/8383) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 今、背景には、マージ `*MergeTree` テーブルエンジンの家族の保存-保存政策に大量注文しております。 [\#8549](https://github.com/ClickHouse/ClickHouse/pull/8549) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Nowテーブルエンジン `Kafka` で適切に動作します `Native` フォーマット。 この修正 [\#6731](https://github.com/ClickHouse/ClickHouse/issues/6731) [\#7337](https://github.com/ClickHouse/ClickHouse/issues/7337) [\#8003](https://github.com/ClickHouse/ClickHouse/issues/8003). [\#8016](https://github.com/ClickHouse/ClickHouse/pull/8016) ([フィリモノフ](https://github.com/filimonov)) -- ヘッダーを持つ固定形式（のような `CSVWithNames` テーブルエンジンのEOFについて例外を投げていた `Kafka`. [\#8016](https://github.com/ClickHouse/ClickHouse/pull/8016) ([フィリモノフ](https://github.com/filimonov)) -- の右側の部分にサブクエリからセットを作るとバグを修正しました `IN` セクション。 この修正 [\#5767](https://github.com/ClickHouse/ClickHouse/issues/5767) と [\#2542](https://github.com/ClickHouse/ClickHouse/issues/2542). [\#7755](https://github.com/ClickHouse/ClickHouse/pull/7755) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- ストレージか `File`. [\#7756](https://github.com/ClickHouse/ClickHouse/pull/7756) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- でファイルの固定読み取り `Parquet` 型の列を含む形式 `list`. [\#8334](https://github.com/ClickHouse/ClickHouse/pull/8334) ([マクスラン](https://github.com/maxulan)) -- エラーの修正 `Not found column` 分散クエリの場合 `PREWHERE` サンプリングキーに依存する条件 `max_parallel_replicas > 1`. [\#7913](https://github.com/ClickHouse/ClickHouse/pull/7913) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- エラーの修正 `Not found column` クエリを使用した場合 `PREWHERE` テーブルのエイリアスに依存し、主キー条件のために結果セットは空でした。 [\#7911](https://github.com/ClickHouse/ClickHouse/pull/7911) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 関数の戻り値の型を修正 `rand` と `randConstant` の場合 `Nullable` 引数。 Now関数は常に戻ります `UInt32` そして決して `Nullable(UInt32)`. [\#8204](https://github.com/ClickHouse/ClickHouse/pull/8204) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 無効述語プッシュダウンのための `WITH FILL` 式。 この修正 [\#7784](https://github.com/ClickHouse/ClickHouse/issues/7784). [\#7789](https://github.com/ClickHouse/ClickHouse/pull/7789) ([冬張](https://github.com/zhang2014)) -- 固定間違った `count()` 結果のための `SummingMergeTree` とき `FINAL` セクションを使用します。 [\#3280](https://github.com/ClickHouse/ClickHouse/issues/3280) [\#7786](https://github.com/ClickHouse/ClickHouse/pull/7786) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- 固定可能な誤った結果常に機能することができます。 そのためのクエリ機能 `version()`, `uptime()`、等。 サーバーごとに異なる定数値を返します。 この修正 [\#7666](https://github.com/ClickHouse/ClickHouse/issues/7666). [\#7689](https://github.com/ClickHouse/ClickHouse/pull/7689) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 間違った結果につながるプッシュダウン述語の最適化の複雑なバグを修正します。 これにより、プッシュダウン述語の最適化に多くの問題が修正されます。 [\#8503](https://github.com/ClickHouse/ClickHouse/pull/8503) ([冬張](https://github.com/zhang2014)) -- でクラッシュを修正 `CREATE TABLE .. AS dictionary` クエリ。 [\#8508](https://github.com/ClickHouse/ClickHouse/pull/8508) ([Azat Khuzhin](https://github.com/azat)) -- いくつかの改善clickhouse文法で `.g4` ファイル。 [\#8294](https://github.com/ClickHouse/ClickHouse/pull/8294) ([タイヤン-リ](https://github.com/taiyang-li)) -- でクラッシュにつながるバグを修正 `JOIN`エンジン付きテーブル付きs `Join`. この修正 [\#7556](https://github.com/ClickHouse/ClickHouse/issues/7556) [\#8254](https://github.com/ClickHouse/ClickHouse/issues/8254) [\#7915](https://github.com/ClickHouse/ClickHouse/issues/7915) [\#8100](https://github.com/ClickHouse/ClickHouse/issues/8100). [\#8298](https://github.com/ClickHouse/ClickHouse/pull/8298) ([Artem Zuikov](https://github.com/4ertus2)) -- 冗長辞書のリロードを修正 `CREATE DATABASE`. [\#7916](https://github.com/ClickHouse/ClickHouse/pull/7916) ([Azat Khuzhin](https://github.com/azat)) -- 読み込み元のストリームの最大数を制限する `StorageFile` と `StorageHDFS`. 修正https://github.com/ClickHouse/ClickHouse/issues/7650。 [\#7981](https://github.com/ClickHouse/ClickHouse/pull/7981) ([alesapin](https://github.com/alesapin)) -- バグを修正 `ALTER ... MODIFY ... CODEC` クエリがユーザーの両方を指定しデフォルトの表現-コーデック. 修正 [8593](https://github.com/ClickHouse/ClickHouse/issues/8593). [\#8614](https://github.com/ClickHouse/ClickHouse/pull/8614) ([alesapin](https://github.com/alesapin)) -- 列のバックグラウンドマージでエラーを修正 `SimpleAggregateFunction(LowCardinality)` タイプ。 [\#8613](https://github.com/ClickHouse/ClickHouse/pull/8613) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 固定型チェックイン機能 `toDateTime64`. [\#8375](https://github.com/ClickHouse/ClickHouse/pull/8375) ([Vasily Nemkov](https://github.com/Enmk)) -- 今、サーバーがクラッシュしない `LEFT` または `FULL JOIN` と参加エンジンと非サポート `join_use_nulls` 設定。 [\#8479](https://github.com/ClickHouse/ClickHouse/pull/8479) ([Artem Zuikov](https://github.com/4ertus2)) -- さて `DROP DICTIONARY IF EXISTS db.dict` クエリが例外をスローしない場合 `db` 存在しない [\#8185](https://github.com/ClickHouse/ClickHouse/pull/8185) ([Vitaly Baranov](https://github.com/vitlibar)) -- テーブル関数のクラッシュの修正 (`file`, `mysql`, `remote`）削除への参照の使用によって引き起こされる `IStorage` オブジェクト。 テーブル関数への挿入時に指定された列の不正な解析を修正しました。 [\#7762](https://github.com/ClickHouse/ClickHouse/pull/7762) ([tavplubix](https://github.com/tavplubix)) -- をネットワークとなる前に `clickhouse-server`. この修正 [\#7507](https://github.com/ClickHouse/ClickHouse/issues/7507). [\#8570](https://github.com/ClickHouse/ClickHouse/pull/8570) ([Zhichang Yu](https://github.com/yuzhichang)) -- 安全な接続のためのタイムアウト処理を修正しました。 この修正 [\#8126](https://github.com/ClickHouse/ClickHouse/issues/8126). [\#8128](https://github.com/ClickHouse/ClickHouse/pull/8128) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 修正 `clickhouse-copier`’並行作業者間の冗長な競合。 [\#7816](https://github.com/ClickHouse/ClickHouse/pull/7816) ([丁象飛](https://github.com/dingxiangfei2009)) -- たとえその変異バージョンが現在の変異バージョ [\#7812](https://github.com/ClickHouse/ClickHouse/pull/7812) ([Zhichang Yu](https://github.com/yuzhichang)) [\#8250](https://github.com/ClickHouse/ClickHouse/pull/8250) ([alesapin](https://github.com/alesapin)) -- の冗長コピーを無視する。 `*MergeTree` 別のディスクに移動してサーバーを再起動した後のデータ部分。 [\#7810](https://github.com/ClickHouse/ClickHouse/pull/7810) ([Vladimir Chebotarev](https://github.com/excitoon)) -- でクラッシュを修正 `FULL JOIN` と `LowCardinality` で `JOIN` キー。 [\#8252](https://github.com/ClickHouse/ClickHouse/pull/8252) ([Artem Zuikov](https://github.com/4ertus2)) -- Insertクエリで列名を複数回使用することは禁じられています `INSERT INTO tbl (x, y, x)`. この修正 [\#5465](https://github.com/ClickHouse/ClickHouse/issues/5465), [\#7681](https://github.com/ClickHouse/ClickHouse/issues/7681). [\#7685](https://github.com/ClickHouse/ClickHouse/pull/7685) ([alesapin](https://github.com/alesapin)) -- 検出のためのフォールバックを追加しました（論理cpuコアの数を使用して）未知のcpuのための物理cpuコアの数。 この修正 [\#5239](https://github.com/ClickHouse/ClickHouse/issues/5239). [\#7726](https://github.com/ClickHouse/ClickHouse/pull/7726) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 修正 `There's no column` 誤差を実現し、エイリアス列あります。 [\#8210](https://github.com/ClickHouse/ClickHouse/pull/8210) ([Artem Zuikov](https://github.com/4ertus2)) -- 固定断つクラッシュ時 `EXISTS` クエリが使用されなかった `TABLE` または `DICTIONARY` 修飾子。 ただのような `EXISTS t`. この修正 [\#8172](https://github.com/ClickHouse/ClickHouse/issues/8172). このバグはバージョン19.17で導入されました。 [\#8213](https://github.com/ClickHouse/ClickHouse/pull/8213) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- エラーでまれなバグを修正 `"Sizes of columns doesn't match"` これは、 `SimpleAggregateFunction` コラム [\#7790](https://github.com/ClickHouse/ClickHouse/pull/7790) ([Boris Granveaud](https://github.com/bgranvea)) -- 空のユーザーバグを修正 `allow_databases` すべてのデータベース（同じ `allow_dictionaries`). [\#7793](https://github.com/ClickHouse/ClickHouse/pull/7793) ([DeifyTheGod](https://github.com/DeifyTheGod)) -- 固定顧客のクラッシュがサーバーで接続しています。 [\#8071](https://github.com/ClickHouse/ClickHouse/pull/8071) ([Azat Khuzhin](https://github.com/azat)) -- 修正 `ORDER BY` 主キー接頭辞と非主キー接尾辞によるソートの場合の動作。 [\#7759](https://github.com/ClickHouse/ClickHouse/pull/7759) ([アントン-ポポフ](https://github.com/CurtizJ)) -- テーブルに修飾列が存在するかどうかを確認します。 この修正 [\#6836](https://github.com/ClickHouse/ClickHouse/issues/6836). [\#7758](https://github.com/ClickHouse/ClickHouse/pull/7758) ([Artem Zuikov](https://github.com/4ertus2)) -- 固定された動作と `ALTER MOVE` merge finishが指定したスーパーパーを移動した直後に実行されます。 修正 [\#8103](https://github.com/ClickHouse/ClickHouse/issues/8103). [\#8104](https://github.com/ClickHouse/ClickHouse/pull/8104) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 使用中のサーバーのクラッシュを修正 `UNION` 異なる数の列を持つ。 修正 [\#7279](https://github.com/ClickHouse/ClickHouse/issues/7279). [\#7929](https://github.com/ClickHouse/ClickHouse/pull/7929) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 固定サイズの結果、部分文字列のための機能 `substr` 負のサイズ。 [\#8589](https://github.com/ClickHouse/ClickHouse/pull/8589) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 今、サーバーは `MergeTree` が足りないときは無料のスレッドの背景プールがあります。 [\#8588](https://github.com/ClickHouse/ClickHouse/pull/8588) ([tavplubix](https://github.com/tavplubix)) -- 書式設定にマイナータイプミスを修正 `UNION ALL` AST. [\#7999](https://github.com/ClickHouse/ClickHouse/pull/7999) ([litao91](https://github.com/litao91)) -- 固定間違ったブルームフィルタの負の数の結果。 この修正 [\#8317](https://github.com/ClickHouse/ClickHouse/issues/8317). [\#8566](https://github.com/ClickHouse/ClickHouse/pull/8566) ([冬張](https://github.com/zhang2014)) -- Decompressでバッファオーバーフローの可能性を修正 悪意のあるユーザーで製作した圧縮データが読み後のバッファです。 この問題は、Yandexの情報セキュリティチームのEldar Zaitovによって発見されました。 [\#8404](https://github.com/ClickHouse/ClickHouse/pull/8404) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 整数のオーバーフローのために誤った結果を修正 `arrayIntersect`. [\#7777](https://github.com/ClickHouse/ClickHouse/pull/7777) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- さて `OPTIMIZE TABLE` クエリを待ちませんがオフラインのレプリカを行います。 [\#8314](https://github.com/ClickHouse/ClickHouse/pull/8314) ([javi santana](https://github.com/javisantana)) -- 固定 `ALTER TTL` のためのパーサ `Replicated*MergeTree` テーブル。 [\#8318](https://github.com/ClickHouse/ClickHouse/pull/8318) ([Vladimir Chebotarev](https://github.com/excitoon)) -- サーバとクライアン [\#8084](https://github.com/ClickHouse/ClickHouse/pull/8084) ([Azat Khuzhin](https://github.com/azat)) -- 修正 `bitmapAnd` 機能エラーが交差に集約ビットマップおよびスカラービットマップ. [\#8082](https://github.com/ClickHouse/ClickHouse/pull/8082) ([越黄](https://github.com/moon03432)) -- 定義をの精製して下さい `ZXid` バグを修正するZooKeeperプログラマーズガイドによると `clickhouse-cluster-copier`. [\#8088](https://github.com/ClickHouse/ClickHouse/pull/8088) ([丁象飛](https://github.com/dingxiangfei2009)) -- `odbc` テーブル関数は今 `external_table_functions_use_nulls` 設定。 [\#7506](https://github.com/ClickHouse/ClickHouse/pull/7506) ([Vasily Nemkov](https://github.com/Enmk)) -- 稀なデータレースにつながるバグを修正しました。 [\#8143](https://github.com/ClickHouse/ClickHouse/pull/8143) ([Alexander Kazakov](https://github.com/Akazz)) -- さて `SYSTEM RELOAD DICTIONARY` 辞書を完全にリロードし、無視します `update_field`. この修正 [\#7440](https://github.com/ClickHouse/ClickHouse/issues/7440). [\#8037](https://github.com/ClickHouse/ClickHouse/pull/8037) ([Vitaly Baranov](https://github.com/vitlibar)) -- Create queryに辞書が存在するかどうかを確認する機能を追加します。 [\#8032](https://github.com/ClickHouse/ClickHouse/pull/8032) ([alesapin](https://github.com/alesapin)) -- 修正 `Float*` 解析 `Values` フォーマット。 この修正 [\#7817](https://github.com/ClickHouse/ClickHouse/issues/7817). [\#7870](https://github.com/ClickHouse/ClickHouse/pull/7870) ([tavplubix](https://github.com/tavplubix)) -- 私たちはいくつかのバックグ `*MergeTree` テーブルエンジン家族。 [\#7873](https://github.com/ClickHouse/ClickHouse/pull/7873) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 固定クラッシュの統合運用がテーブル `SimpleAggregateFunction(LowCardinality)` コラム この修正 [\#8515](https://github.com/ClickHouse/ClickHouse/issues/8515). [\#8522](https://github.com/ClickHouse/ClickHouse/pull/8522) ([Azat Khuzhin](https://github.com/azat)) -- すべてのicuロケールのサポートを復元し、定数式の照合順序を適用する機能を追加します。 また、言語名を追加する `system.collations` テーブル。 [\#8051](https://github.com/ClickHouse/ClickHouse/pull/8051) ([alesapin](https://github.com/alesapin)) -- ゼロ最小限の寿命を持つ外部辞書バグを修正しました (`LIFETIME(MIN 0 MAX N)`, `LIFETIME(N)` バックグラウンドで更新しない。 [\#7983](https://github.com/ClickHouse/ClickHouse/pull/7983) ([alesapin](https://github.com/alesapin)) -- 固定したときにクラッシュする場合が外部辞書でclickhouseソースがサブクエリに返します。 [\#8351](https://github.com/ClickHouse/ClickHouse/pull/8351) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- エンジンでテーブル内のファイル拡張子の誤った解析を修正 `URL`. この修正 [\#8157](https://github.com/ClickHouse/ClickHouse/issues/8157). [\#8419](https://github.com/ClickHouse/ClickHouse/pull/8419) ([Andrey Bodrov](https://github.com/apbodrov)) -- 修正 `CHECK TABLE` のためのクエリ `*MergeTree` キーのないテーブル。 修正 [\#7543](https://github.com/ClickHouse/ClickHouse/issues/7543). [\#7979](https://github.com/ClickHouse/ClickHouse/pull/7979) ([alesapin](https://github.com/alesapin)) -- 固定変換の `Float64` MySQLのタイプに。 [\#8079](https://github.com/ClickHouse/ClickHouse/pull/8079) ([ユーリーバラノフ](https://github.com/yurriy)) -- 今ればいけない完全に落ちてしまったがサーバのクラッシュ、サーバーへの復元とその負荷ます。 [\#8176](https://github.com/ClickHouse/ClickHouse/pull/8176) ([tavplubix](https://github.com/tavplubix)) -- テーブル機能のクラッシュを修正 `file` 存在しないファイルに挿入している間。 この場合、ファイルが作成され、insertが処理されます。 [\#8177](https://github.com/ClickHouse/ClickHouse/pull/8177) ([Olga Khvostikova](https://github.com/stavrolia)) -- ときに発生する可能性がまれなデッドロックを修正 `trace_log` 有効になっています。 [\#7838](https://github.com/ClickHouse/ClickHouse/pull/7838) ([フィリモノフ](https://github.com/filimonov)) -- ほかに異なるタイプで動作する機能を追加 `Date` で `RangeHashed` DDLクエリから作成された外部ディクショナリ。 修正 [7899](https://github.com/ClickHouse/ClickHouse/issues/7899). [\#8275](https://github.com/ClickHouse/ClickHouse/pull/8275) ([alesapin](https://github.com/alesapin)) -- ときの修正クラッシュ `now64()` 別の関数の結果で呼び出されます。 [\#8270](https://github.com/ClickHouse/ClickHouse/pull/8270) ([Vasily Nemkov](https://github.com/Enmk)) -- 固定バグ検出クライアントip接続を通じてmysqlワイヤプロトコルです。 [\#7743](https://github.com/ClickHouse/ClickHouse/pull/7743) ([Dmitry Muzyka](https://github.com/dmitriy-myz)) -- 空の配列の処理を修正 `arraySplit` 機能。 この修正 [\#7708](https://github.com/ClickHouse/ClickHouse/issues/7708). [\#7747](https://github.com/ClickHouse/ClickHouse/pull/7747) ([hcz](https://github.com/hczhcz)) -- ときに問題を修正しました `pid-file` 別のランニングの `clickhouse-server` 削除される可能性があります。 [\#8487](https://github.com/ClickHouse/ClickHouse/pull/8487) ([Weiqing Xu](https://github.com/weiqxu)) -- それが持っている場合 `invalidate_query` これは、以前の更新試行時に更新といくつかの例外を停止しました。 [\#8029](https://github.com/ClickHouse/ClickHouse/pull/8029) ([alesapin](https://github.com/alesapin)) -- 関数のエラーを修正しました `arrayReduce` それはにつながる可能性 “double free” そして、集計関数combinatorのエラー `Resample` それはメモリリークの原因となります。 集計関数の追加 `aggThrow`. この関数は、テスト目的で使用できます。 [\#8446](https://github.com/ClickHouse/ClickHouse/pull/8446) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -#### 改善 {#improvement-1} - -- での作業時に改善されたロギング `S3` テーブルエンジン。 [\#8251](https://github.com/ClickHouse/ClickHouse/pull/8251) ([Grigory Pervakov](https://github.com/GrigoryPervakov)) -- 印刷ヘルプメッセージがない場合引数が渡された通話の場合 `clickhouse-local`. この修正 [\#5335](https://github.com/ClickHouse/ClickHouse/issues/5335). [\#8230](https://github.com/ClickHouse/ClickHouse/pull/8230) ([Andrey Nagorny](https://github.com/Melancholic)) -- 設定を追加 `mutations_sync` 待つことができます `ALTER UPDATE/DELETE` 同期クエリ。 [\#8237](https://github.com/ClickHouse/ClickHouse/pull/8237) ([alesapin](https://github.com/alesapin)) -- 相対セットアップを許可する `user_files_path` で `config.xml` （同様の方法で `format_schema_path`). [\#7632](https://github.com/ClickHouse/ClickHouse/pull/7632) ([hcz](https://github.com/hczhcz)) -- 変換関数の不正な型の例外を追加するには `-OrZero` 後置。 [\#7880](https://github.com/ClickHouse/ClickHouse/pull/7880) ([Andrey Konyaev](https://github.com/akonyaev90)) -- 分散クエリでシャードに送信するデータのヘッダーの形式を簡素化します。 [\#8044](https://github.com/ClickHouse/ClickHouse/pull/8044) ([Vitaly Baranov](https://github.com/vitlibar)) -- `Live View` テーブルエンジンリファクタリング。 [\#8519](https://github.com/ClickHouse/ClickHouse/pull/8519) ([vzakaznikov](https://github.com/vzakaznikov)) -- DDLクエリから作成された外部ディクショナリのチェックを追加します。 [\#8127](https://github.com/ClickHouse/ClickHouse/pull/8127) ([alesapin](https://github.com/alesapin)) -- エラーの修正 `Column ... already exists` 使用している間 `FINAL` と `SAMPLE` together, e.g. `select count() from table final sample 1/2`. 修正 [\#5186](https://github.com/ClickHouse/ClickHouse/issues/5186). [\#7907](https://github.com/ClickHouse/ClickHouse/pull/7907) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 今の最初の引数を表 `joinGet` 関数はテーブル識別子にすることができます。 [\#7707](https://github.com/ClickHouse/ClickHouse/pull/7707) ([アモスの鳥](https://github.com/amosbird)) -- 使用を許可する `MaterializedView` 上記のサブクエリを使用する `Kafka` テーブル。 [\#8197](https://github.com/ClickHouse/ClickHouse/pull/8197) ([フィリモノフ](https://github.com/filimonov)) -- これで、ディスク間の背景移動がseprateスレッドプールを実行します。 [\#7670](https://github.com/ClickHouse/ClickHouse/pull/7670) ([Vladimir Chebotarev](https://github.com/excitoon)) -- `SYSTEM RELOAD DICTIONARY` 今同期的に実行されます。 [\#8240](https://github.com/ClickHouse/ClickHouse/pull/8240) ([Vitaly Baranov](https://github.com/vitlibar)) -- スタックトレース表示の物理アドレス(オフセットオブジェクトファイルの代わりに仮想メモリのアドレスのオブジェクトファイルが読み込まれ). それは使用をの可能にします `addr2line` binaryが独立した位置でASLRがアクティブな場合。 この修正 [\#8360](https://github.com/ClickHouse/ClickHouse/issues/8360). [\#8387](https://github.com/ClickHouse/ClickHouse/pull/8387) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 行レベルのセキュリ: `
…
`. 修正 [\#5779](https://github.com/ClickHouse/ClickHouse/issues/5779). [\#8381](https://github.com/ClickHouse/ClickHouse/pull/8381) ([イワン](https://github.com/abyss7)) -- さて `cityHash` 機能で動作することができ `Decimal` と `UUID` タイプ。 修正 [\#5184](https://github.com/ClickHouse/ClickHouse/issues/5184). [\#7693](https://github.com/ClickHouse/ClickHouse/pull/7693) ([Mikhail Korotov](https://github.com/millb)) -- アダプティブ粒度の実装後に廃止されたため、システムログから固定インデックス粒度（1024）が削除されました。 [\#7698](https://github.com/ClickHouse/ClickHouse/pull/7698) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 有効なmysqlサーバーの互換性がclickhouseはめずにボタンを使用します。 [\#7852](https://github.com/ClickHouse/ClickHouse/pull/7852) ([ユーリーバラノフ](https://github.com/yurriy)) -- これにより、バッチ内のデータが破損した場合の詳細なエラーが発生します。 [\#7914](https://github.com/ClickHouse/ClickHouse/pull/7914) ([Azat Khuzhin](https://github.com/azat)) -- サポート `DROP DATABASE`, `DETACH TABLE`, `DROP TABLE` と `ATTACH TABLE` のために `MySQL` データベースエンジ [\#8202](https://github.com/ClickHouse/ClickHouse/pull/8202) ([冬張](https://github.com/zhang2014)) -- S3テーブル機能とテーブルエンジンに認証を追加します。 [\#7623](https://github.com/ClickHouse/ClickHouse/pull/7623) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 追加されたチェックの余分な部品 `MergeTree` 異なるディスクでは、未定義のディスクでデータ部分を見逃さないようにするためです。 [\#8118](https://github.com/ClickHouse/ClickHouse/pull/8118) ([Vladimir Chebotarev](https://github.com/excitoon)) -- を、sslをサポートのためにmacをクライアントとサーバーです。 [\#8297](https://github.com/ClickHouse/ClickHouse/pull/8297) ([イワン](https://github.com/abyss7)) -- 今clickhouseできる作品としてmysql連携サーバを参照https://dev.mysql.com/doc/refman/5.7/en/federated-create-server.html). [\#7717](https://github.com/ClickHouse/ClickHouse/pull/7717) ([Maxim Fedotov](https://github.com/MaxFedotov)) -- `clickhouse-client` 今だけ有効にする `bracketed-paste` マルチクエリがオンで、マルチラインがオフの場合。 この修正(\#7757)\[https://github.com/ClickHouse/ClickHouse/issues/7757\]。 [\#7761](https://github.com/ClickHouse/ClickHouse/pull/7761) ([アモスの鳥](https://github.com/amosbird)) -- サポート `Array(Decimal)` で `if` 機能。 [\#7721](https://github.com/ClickHouse/ClickHouse/pull/7721) ([Artem Zuikov](https://github.com/4ertus2)) -- サポート小数で `arrayDifference`, `arrayCumSum` と `arrayCumSumNegative` 機能。 [\#7724](https://github.com/ClickHouse/ClickHouse/pull/7724) ([Artem Zuikov](https://github.com/4ertus2)) -- 追加 `lifetime` コラムへの `system.dictionaries` テーブル。 [\#6820](https://github.com/ClickHouse/ClickHouse/issues/6820) [\#7727](https://github.com/ClickHouse/ClickHouse/pull/7727) ([kekekekule](https://github.com/kekekekule)) -- 改良されたチェックインのための既存の部品の異なるハードディスク `*MergeTree` テーブルエンジン。 アドレス [\#7660](https://github.com/ClickHouse/ClickHouse/issues/7660). [\#8440](https://github.com/ClickHouse/ClickHouse/pull/8440) ([Vladimir Chebotarev](https://github.com/excitoon)) -- との統合 `AWS SDK` のために `S3` 箱から出してすべてのS3の機能を使用することができます相互作用。 [\#8011](https://github.com/ClickHouse/ClickHouse/pull/8011) ([Pavel Kovalenko](https://github.com/Jokser)) -- サブクエリのサポートが追加されました `Live View` テーブル。 [\#7792](https://github.com/ClickHouse/ClickHouse/pull/7792) ([vzakaznikov](https://github.com/vzakaznikov)) -- 使用のための点検 `Date` または `DateTime` からの列 `TTL` 式は削除されました。 [\#7920](https://github.com/ClickHouse/ClickHouse/pull/7920) ([Vladimir Chebotarev](https://github.com/excitoon)) -- ディスクに関する情報が追加された `system.detached_parts` テーブル。 [\#7833](https://github.com/ClickHouse/ClickHouse/pull/7833) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 今すぐ設定 `max_(table|partition)_size_to_drop` 再起動せずに変更することができます。 [\#7779](https://github.com/ClickHouse/ClickHouse/pull/7779) ([Grigory Pervakov](https://github.com/GrigoryPervakov)) -- エラーメッ ユーザーに以下の行を削除しないように依頼する `Stack trace:`. [\#7897](https://github.com/ClickHouse/ClickHouse/pull/7897) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- より良いメッセージを読むら `Kafka` 後の様々な形式のエンジン [\#7935](https://github.com/ClickHouse/ClickHouse/issues/7935). [\#8035](https://github.com/ClickHouse/ClickHouse/pull/8035) ([イワン](https://github.com/abyss7)) -- サポートしていないmysqlクライアントとの互換性の向上 `sha2_password` authプラグイン。 [\#8036](https://github.com/ClickHouse/ClickHouse/pull/8036) ([ユーリーバラノフ](https://github.com/yurriy)) -- 支援の列タイプのmysqlサーバーの互換性. [\#7975](https://github.com/ClickHouse/ClickHouse/pull/7975) ([ユーリーバラノフ](https://github.com/yurriy)) -- 実装 `ORDER BY` 最適化のための `Merge`, `Buffer` と `Materilized View` 下になるとストレージ `MergeTree` テーブル。 [\#8130](https://github.com/ClickHouse/ClickHouse/pull/8130) ([アントン-ポポフ](https://github.com/CurtizJ)) -- 今、私たちは常にposixの実装を使用します `getrandom` 古いカーネル(\<3.17)との互換性を改善する。 [\#7940](https://github.com/ClickHouse/ClickHouse/pull/7940) ([アモスの鳥](https://github.com/amosbird)) -- 移動ttlルールの有効な宛先をよりよくチェックします。 [\#8410](https://github.com/ClickHouse/ClickHouse/pull/8410) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 壊れた挿入のバッチのためのよりよい点検 `Distributed` テーブルエンジン。 [\#7933](https://github.com/ClickHouse/ClickHouse/pull/7933) ([Azat Khuzhin](https://github.com/azat)) -- 将来突然変異が処理されなければならない部品名の配列を持つ列を追加する `system.mutations` テーブル。 [\#8179](https://github.com/ClickHouse/ClickHouse/pull/8179) ([alesapin](https://github.com/alesapin)) -- 並列マージプロセッサのソート最適化。 [\#8552](https://github.com/ClickHouse/ClickHouse/pull/8552) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 設定 `mark_cache_min_lifetime` 今は時代遅れで、何もしません。 以前のバージョンでは、マークキャッシュはメモリ内で `mark_cache_size` 内のデータを収容するために、 `mark_cache_min_lifetime` 秒。 それは、メモリ制約のあるシステムでは特に悪いことです。 このリリースをインストールした後にパフォーマンスが低下する場合は、 `mark_cache_size`. [\#8484](https://github.com/ClickHouse/ClickHouse/pull/8484) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 使用する準備 `tid` どこにでも これは次の場合に必要です [\#7477](https://github.com/ClickHouse/ClickHouse/issues/7477). [\#8276](https://github.com/ClickHouse/ClickHouse/pull/8276) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -#### 性能向上 {#performance-improvement-1} - -- 性能の最適化、プロセッサのパイプライン [\#7988](https://github.com/ClickHouse/ClickHouse/pull/7988) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 非ブロック更新の有効期限が切れたキーキャッシュの辞書（許可を読古い。 [\#8303](https://github.com/ClickHouse/ClickHouse/pull/8303) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- クリック `-fno-omit-frame-pointer` 世界的に余裕一するものとする。 [\#8097](https://github.com/ClickHouse/ClickHouse/pull/8097) ([アモスの鳥](https://github.com/amosbird)) -- スピードアップ `greatCircleDistance` それのための性能試験を機能し、加えて下さい。 [\#7307](https://github.com/ClickHouse/ClickHouse/pull/7307) ([Olga Khvostikova](https://github.com/stavrolia)) -- 機能のパフォーマンスの向上 `roundDown`. [\#8465](https://github.com/ClickHouse/ClickHouse/pull/8465) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 改善された性能の `max`, `min`, `argMin`, `argMax` のために `DateTime64` データ型。 [\#8199](https://github.com/ClickHouse/ClickHouse/pull/8199) ([Vasily Nemkov](https://github.com/Enmk)) -- 大きい限界および外的な分類の限界のないまたは分類の改善された性能。 [\#8545](https://github.com/ClickHouse/ClickHouse/pull/8545) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 6回までの浮動小数点数の書式設定のパフォーマンスが向上しました。 [\#8542](https://github.com/ClickHouse/ClickHouse/pull/8542) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 改善された性能の `modulo` 機能。 [\#7750](https://github.com/ClickHouse/ClickHouse/pull/7750) ([アモスの鳥](https://github.com/amosbird)) -- 最適化 `ORDER BY` 単一の列キーとのマージ。 [\#8335](https://github.com/ClickHouse/ClickHouse/pull/8335) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- より良い実装 `arrayReduce`, `-Array` と `-State` コンビネーター [\#7710](https://github.com/ClickHouse/ClickHouse/pull/7710) ([アモスの鳥](https://github.com/amosbird)) -- さて `PREWHERE` 少なくとも次のように最適化する必要があります `WHERE`. [\#7769](https://github.com/ClickHouse/ClickHouse/pull/7769) ([アモスの鳥](https://github.com/amosbird)) -- 方法を改善する `round` と `roundBankers` 負の数を扱う。 [\#8229](https://github.com/ClickHouse/ClickHouse/pull/8229) ([hcz](https://github.com/hczhcz)) -- デコード性能の向上 `DoubleDelta` と `Gorilla` 大体30-40％のコーデック。 この修正 [\#7082](https://github.com/ClickHouse/ClickHouse/issues/7082). [\#8019](https://github.com/ClickHouse/ClickHouse/pull/8019) ([Vasily Nemkov](https://github.com/Enmk)) -- 改善された性能の `base64` 関連機能。 [\#8444](https://github.com/ClickHouse/ClickHouse/pull/8444) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 機能を追加しました `geoDistance`. それは類似していますに `greatCircleDistance` しかし、WGS-84楕円体モデルに近似を使用します。 両方の機能のパフォーマンスは同じに近いです。 [\#8086](https://github.com/ClickHouse/ClickHouse/pull/8086) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- より速く `min` と `max` 以下のための集計関数 `Decimal` データ型。 [\#8144](https://github.com/ClickHouse/ClickHouse/pull/8144) ([Artem Zuikov](https://github.com/4ertus2)) -- ベクトル化処理 `arrayReduce`. [\#7608](https://github.com/ClickHouse/ClickHouse/pull/7608) ([アモスの鳥](https://github.com/amosbird)) -- `if` 鎖は今最大限に活用されます `multiIf`. [\#8355](https://github.com/ClickHouse/ClickHouse/pull/8355) ([kamalov-ruslan](https://github.com/kamalov-ruslan)) -- パフォーマンスの回帰の修正 `Kafka` 19.15で導入されたテーブルエンジン。 この修正 [\#7261](https://github.com/ClickHouse/ClickHouse/issues/7261). [\#7935](https://github.com/ClickHouse/ClickHouse/pull/7935) ([フィリモノフ](https://github.com/filimonov)) -- 削除 “pie” コード生成 `gcc` からDebianパッケージの時となります。 [\#8483](https://github.com/ClickHouse/ClickHouse/pull/8483) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- データ形式の並列解析 [\#6553](https://github.com/ClickHouse/ClickHouse/pull/6553) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- 最適化されたパーサーの有効化 `Values` デフォルトでの式の使用 (`input_format_values_deduce_templates_of_expressions=1`). [\#8231](https://github.com/ClickHouse/ClickHouse/pull/8231) ([tavplubix](https://github.com/tavplubix)) - -#### ビルド/テスト/パッケージの改善 {#buildtestingpackaging-improvement-2} - -- ビルドの修正 `ARM` そして、最小限のモードで。 [\#8304](https://github.com/ClickHouse/ClickHouse/pull/8304) ([proller](https://github.com/proller)) -- 追加取材ファイルのフラッシュ用 `clickhouse-server` std::atexitが呼び出されないとき。 も若干の改善にログイン状態試験。 [\#8267](https://github.com/ClickHouse/ClickHouse/pull/8267) ([alesapin](https://github.com/alesapin)) -- ContribのLLVMライブラリを更新します。 OSパッケージからのLLVMの使用を避けます。 [\#8258](https://github.com/ClickHouse/ClickHouse/pull/8258) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- バンドルする `curl` 完全に静かなビルド。 [\#8232](https://github.com/ClickHouse/ClickHouse/pull/8232) [\#8203](https://github.com/ClickHouse/ClickHouse/pull/8203) ([Pavel Kovalenko](https://github.com/Jokser)) -- いくつかを修正 `MemorySanitizer` 警告。 [\#8235](https://github.com/ClickHouse/ClickHouse/pull/8235) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- 使用 `add_warning` と `no_warning` マクロ `CMakeLists.txt`. [\#8604](https://github.com/ClickHouse/ClickHouse/pull/8604) ([イワン](https://github.com/abyss7)) -- Minio S3互換オブジェクトのサポートを追加(https://min.io/）より良い統合テストのために。 [\#7863](https://github.com/ClickHouse/ClickHouse/pull/7863) [\#7875](https://github.com/ClickHouse/ClickHouse/pull/7875) ([Pavel Kovalenko](https://github.com/Jokser)) -- インポート `libc` contribへのヘッダー。 ることができる作をより一貫性のあるさまざまなシステムのみ `x86_64-linux-gnu`). [\#5773](https://github.com/ClickHouse/ClickHouse/pull/5773) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 削除 `-fPIC` いくつかの図書館から。 [\#8464](https://github.com/ClickHouse/ClickHouse/pull/8464) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- クリーン `CMakeLists.txt` カールのため。 見るhttps://github.com/ClickHouse/ClickHouse/pull/8011\#issuecomment-569478910 [\#8459](https://github.com/ClickHouse/ClickHouse/pull/8459) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- サイレント警告 `CapNProto` ライブラリ。 [\#8220](https://github.com/ClickHouse/ClickHouse/pull/8220) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 追加の性能試験のため短い文字列を最適化ハッシュテーブル [\#7679](https://github.com/ClickHouse/ClickHouse/pull/7679) ([アモスの鳥](https://github.com/amosbird)) -- 今clickhouseは上に構築されます `AArch64` たとえ `MADV_FREE` は利用できません。 この修正 [\#8027](https://github.com/ClickHouse/ClickHouse/issues/8027). [\#8243](https://github.com/ClickHouse/ClickHouse/pull/8243) ([アモスの鳥](https://github.com/amosbird)) -- 更新 `zlib-ng` メモリ消毒の問題を修正するには. [\#7182](https://github.com/ClickHouse/ClickHouse/pull/7182) [\#8206](https://github.com/ClickHouse/ClickHouse/pull/8206) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- OSパッケージの使用は非常に脆弱で、通常はまったく動作しないため、Linux以外のシステムで内部MySQLライブラリを有効にします。 この修正 [\#5765](https://github.com/ClickHouse/ClickHouse/issues/5765). [\#8426](https://github.com/ClickHouse/ClickHouse/pull/8426) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 固定構築システムによっては後に可能 `libc++`. これは [\#8374](https://github.com/ClickHouse/ClickHouse/issues/8374). [\#8380](https://github.com/ClickHouse/ClickHouse/pull/8380) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 作る `Field` 方法によりtype-安全なものがあります。。 [\#7386](https://github.com/ClickHouse/ClickHouse/pull/7386) [\#8209](https://github.com/ClickHouse/ClickHouse/pull/8209) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- に不足しているファイルを追加 `libc-headers` サブモジュール [\#8507](https://github.com/ClickHouse/ClickHouse/pull/8507) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 間違った修正 `JSON` パフォーマンステスト出力の引用。 [\#8497](https://github.com/ClickHouse/ClickHouse/pull/8497) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- スタックトレースが表示されます `std::exception` と `Poco::Exception`. 以前のバージョンでは、 `DB::Exception`. これは診断を改善します。 [\#8501](https://github.com/ClickHouse/ClickHouse/pull/8501) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 移植 `clock_gettime` と `clock_nanosleep` 新しいglibc版のため。 [\#8054](https://github.com/ClickHouse/ClickHouse/pull/8054) ([アモスの鳥](https://github.com/amosbird)) -- 有効 `part_log` 開発者のための例の設定で。 [\#8609](https://github.com/ClickHouse/ClickHouse/pull/8609) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- リロードの非同期の性質を修正 `01036_no_superfluous_dict_reload_on_create_database*`. [\#8111](https://github.com/ClickHouse/ClickHouse/pull/8111) ([Azat Khuzhin](https://github.com/azat)) -- 固定コーデック性能テスト。 [\#8615](https://github.com/ClickHouse/ClickHouse/pull/8615) ([Vasily Nemkov](https://github.com/Enmk)) -- インストールスクリプト `.tgz` それらのビルドとドキュメント。 [\#8612](https://github.com/ClickHouse/ClickHouse/pull/8612) [\#8591](https://github.com/ClickHouse/ClickHouse/pull/8591) ([alesapin](https://github.com/alesapin)) -- 古いものを削除 `ZSTD` テスト（2016年に作成され、ZSTDの1.0バージョンが持っていたバグを再現しました）。 この修正 [\#8618](https://github.com/ClickHouse/ClickHouse/issues/8618). [\#8619](https://github.com/ClickHouse/ClickHouse/pull/8619) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Mac OSカタリナ上の固定ビルド。 [\#8600](https://github.com/ClickHouse/ClickHouse/pull/8600) ([meo](https://github.com/meob)) -- 増加数行のコーデックの性能試験を果たしますのでご連絡ください [\#8574](https://github.com/ClickHouse/ClickHouse/pull/8574) ([Vasily Nemkov](https://github.com/Enmk)) -- デバッグビルドでは、 `LOGICAL_ERROR` アサーションの失敗としての例外は、気付きやすくなります。 [\#8475](https://github.com/ClickHouse/ClickHouse/pull/8475) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- 形式関連のパフォーマンステストをより確定的にします。 [\#8477](https://github.com/ClickHouse/ClickHouse/pull/8477) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 更新 `lz4` メモリを修正するには市民の失敗。 [\#8181](https://github.com/ClickHouse/ClickHouse/pull/8181) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- 例外処理で既知のmemorysanitizer false positiveを抑制します。 [\#8182](https://github.com/ClickHouse/ClickHouse/pull/8182) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- 更新 `gcc` と `g++` バージョン9へ `build/docker/build.sh` [\#7766](https://github.com/ClickHouse/ClickHouse/pull/7766) ([TLightSky](https://github.com/tlightsky)) -- 追加の性能試験場合試験 `PREWHERE` より悪いです `WHERE`. [\#7768](https://github.com/ClickHouse/ClickHouse/pull/7768) ([アモスの鳥](https://github.com/amosbird)) -- ﾂつｨﾂ姪“ﾂつ”ﾂ債ﾂづｭﾂつｹ [\#8621](https://github.com/ClickHouse/ClickHouse/pull/8621) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- からのデータのmemorysanitizerレポートを避けます `libunwind`. [\#8539](https://github.com/ClickHouse/ClickHouse/pull/8539) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 更新 `libc++` 最新バージョンへ。 [\#8324](https://github.com/ClickHouse/ClickHouse/pull/8324) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- ソースからのビルドicuライブラリ。 この修正 [\#6460](https://github.com/ClickHouse/ClickHouse/issues/6460). [\#8219](https://github.com/ClickHouse/ClickHouse/pull/8219) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- から切り替え `libressl` に `openssl`. ClickHouseは、この変更後にTLS1.3とSNIをサポートする必要があります。 この修正 [\#8171](https://github.com/ClickHouse/ClickHouse/issues/8171). [\#8218](https://github.com/ClickHouse/ClickHouse/pull/8218) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 固定ubsanレポートを使用する場合 `chacha20_poly1305` SSLから（接続時に発生するhttps://yandex.ru/）。 [\#8214](https://github.com/ClickHouse/ClickHouse/pull/8214) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- デフォルトのパスワードファイ `.deb` linuxディストリビュート。 [\#8075](https://github.com/ClickHouse/ClickHouse/pull/8075) ([proller](https://github.com/proller)) -- 取得のための改善された表現 `clickhouse-server` PID `clickhouse-test`. [\#8063](https://github.com/ClickHouse/ClickHouse/pull/8063) ([Alexander Kazakov](https://github.com/Akazz)) -- Contrib/googletestをv1.10.0に更新しました。 [\#8587](https://github.com/ClickHouse/ClickHouse/pull/8587) ([Alexander Burmak](https://github.com/Alex-Burmak)) -- 。固定スレッドサニアイナイザレポートで `base64` ライブラリ。 また、このライブラリを最新バージョンに更新しましたが、問題はありません。 この修正 [\#8397](https://github.com/ClickHouse/ClickHouse/issues/8397). [\#8403](https://github.com/ClickHouse/ClickHouse/pull/8403) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 修正 `00600_replace_running_query` プロセッサの場合。 [\#8272](https://github.com/ClickHouse/ClickHouse/pull/8272) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- サポートの削除 `tcmalloc` 作るため `CMakeLists.txt` もっと簡単に [\#8310](https://github.com/ClickHouse/ClickHouse/pull/8310) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- リリースgccは今使用ビルド `libc++` 代わりに `libstdc++`. 最近 `libc++` clangでのみ使用されました。 これにより、ビルド構成の一貫性と移植性が向上します。 [\#8311](https://github.com/ClickHouse/ClickHouse/pull/8311) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- MemorySanitizerでビルドするためのICUライブラリを有効にします。 [\#8222](https://github.com/ClickHouse/ClickHouse/pull/8222) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 抑制する警告から `CapNProto` ライブラリ。 [\#8224](https://github.com/ClickHouse/ClickHouse/pull/8224) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- のためのコードの特別なケースを削除 `tcmalloc` サポートされなくなったからです [\#8225](https://github.com/ClickHouse/ClickHouse/pull/8225) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- CIカバレッジタスクでは、カバレッジレポートを保存できるようにサーバーを正常に終了します。 これは、我々が最近見てきた不完全な報道レポートを修正します。 [\#8142](https://github.com/ClickHouse/ClickHouse/pull/8142) ([alesapin](https://github.com/alesapin)) -- すべてのコーデックのパフォーマ `Float64` と `UInt64` 値。 [\#8349](https://github.com/ClickHouse/ClickHouse/pull/8349) ([Vasily Nemkov](https://github.com/Enmk)) -- `termcap` 非常に非難され、さまざまな問題につながる（f.g.missing “up” 帽子およびエコー `^J` マルチラインの代わりに）。 お願い `terminfo` またはバンドル `ncurses`. [\#7737](https://github.com/ClickHouse/ClickHouse/pull/7737) ([アモスの鳥](https://github.com/amosbird)) -- 修正 `test_storage_s3` 統合テスト。 [\#7734](https://github.com/ClickHouse/ClickHouse/pull/7734) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- サポート `StorageFile(, null)` 挿入をブロックを所定のフォーマットでなくファイルを実際に書き込みます。 これは必要な性能試験までを実施。 [\#8455](https://github.com/ClickHouse/ClickHouse/pull/8455) ([アモスの鳥](https://github.com/amosbird)) -- 追加された引数 `--print-time` テストごとに実行時間を出力する機能テスト。 [\#8001](https://github.com/ClickHouse/ClickHouse/pull/8001) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- にアサートを追加しました `KeyCondition` RPNを評価しながら。 これにより、gcc-9からの警告が修正されます。 [\#8279](https://github.com/ClickHouse/ClickHouse/pull/8279) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- ダンプcmakeのオプションci構造を作成する環境が整いました [\#8273](https://github.com/ClickHouse/ClickHouse/pull/8273) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- 一部のfatライブラリのデバッグ情報を生成しません。 [\#8271](https://github.com/ClickHouse/ClickHouse/pull/8271) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 作る `log_to_console.xml` それは対話的であるかどうかにかかわらず、常にstderrにログインします。 [\#8395](https://github.com/ClickHouse/ClickHouse/pull/8395) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- 除去も未使用の特徴から `clickhouse-performance-test` ツール。 [\#8555](https://github.com/ClickHouse/ClickHouse/pull/8555) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- 今我々はまた検索する `lld-X` 対応を使って `clang-X` バージョン。 [\#8092](https://github.com/ClickHouse/ClickHouse/pull/8092) ([alesapin](https://github.com/alesapin)) -- 寄木細工のビルドの改善。 [\#8421](https://github.com/ClickHouse/ClickHouse/pull/8421) ([マクスラン](https://github.com/maxulan)) -- より多くのgccの警告 [\#8221](https://github.com/ClickHouse/ClickHouse/pull/8221) ([kreuzerkrieg](https://github.com/kreuzerkrieg)) -- パッケージアーチlinuxすることはできなくなるようで走clickhouseサーバーになります。 [\#8534](https://github.com/ClickHouse/ClickHouse/pull/8534) ([Vladimir Chebotarev](https://github.com/excitoon)) -- プロセッサでテストを修正。 小さな性能の修正。 [\#7672](https://github.com/ClickHouse/ClickHouse/pull/7672) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 更新contrib/protobuf. [\#8256](https://github.com/ClickHouse/ClickHouse/pull/8256) ([Matwey V.Kornilov](https://github.com/matwey)) -- 新年のお祝いとしてc++20への切り替えの準備で。 “May the C++ force be with ClickHouse.” [\#8447](https://github.com/ClickHouse/ClickHouse/pull/8447) ([アモスの鳥](https://github.com/amosbird)) - -#### 実験的特徴 {#experimental-feature-1} - -- 実験的な設定を追加しました `min_bytes_to_use_mmap_io`. ることができるreadビッグファイルのコピーをせずにデータをカーネルを使うこと. の設定が無効になってデフォルトです。 推奨しきい値は、mmap/munmapが遅いため、約64MBです。 [\#8520](https://github.com/ClickHouse/ClickHouse/pull/8520) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- アクセス制御システムの一部としてのクォータの変更。 新しいテーブルを追加 `system.quotas`、新しい機能 `currentQuota`, `currentQuotaKey`、新しいSQL構文 `CREATE QUOTA`, `ALTER QUOTA`, `DROP QUOTA`, `SHOW QUOTA`. [\#7257](https://github.com/ClickHouse/ClickHouse/pull/7257) ([Vitaly Baranov](https://github.com/vitlibar)) -- を飛び未設定警告の代わりに投げることができます。 [\#7653](https://github.com/ClickHouse/ClickHouse/pull/7653) ([Vitaly Baranov](https://github.com/vitlibar)) -- アクセス制御システムの一部としての行ポリシーの変更。 新しいテーブルを追加 `system.row_policies`、新しい機能 `currentRowPolicies()`、新しいSQL構文 `CREATE POLICY`, `ALTER POLICY`, `DROP POLICY`, `SHOW CREATE POLICY`, `SHOW POLICIES`. [\#7808](https://github.com/ClickHouse/ClickHouse/pull/7808) ([Vitaly Baranov](https://github.com/vitlibar)) - -#### セキュリティ修正 {#security-fix} - -- テーブル内のディレクトリ構造を読み取る可能性を修正 `File` テーブルエンジン。 この修正 [\#8536](https://github.com/ClickHouse/ClickHouse/issues/8536). [\#8537](https://github.com/ClickHouse/ClickHouse/pull/8537) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -## [2019年の変更履歴](https://github.com/ClickHouse/ClickHouse/blob/master/docs/en/changelog/2019.md) {#changelog-for-2019} +{% include "content/changelog.md" %} diff --git a/docs/ja/whats-new/index.md b/docs/ja/whats-new/index.md index ac27b70b8bd..f6775abee3c 100644 --- a/docs/ja/whats-new/index.md +++ b/docs/ja/whats-new/index.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 -toc_folder_title: What's New +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "\u65B0\u7740\u60C5\u5831" toc_priority: 72 --- diff --git a/docs/ja/whats-new/roadmap.md b/docs/ja/whats-new/roadmap.md index e64d35b1df3..468d4599b8f 100644 --- a/docs/ja/whats-new/roadmap.md +++ b/docs/ja/whats-new/roadmap.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 74 -toc_title: "\u30ED\u30FC\u30C9\u30DE\u30C3\u30D7" +toc_title: "\u30ED\u30FC\u30C9" --- -# ロードマップ {#roadmap} +# ロード {#roadmap} ## Q1 2020 {#q1-2020} diff --git a/docs/ja/whats-new/security-changelog.md b/docs/ja/whats-new/security-changelog.md index 6e0bd77e2f9..d56b465acfa 100644 --- a/docs/ja/whats-new/security-changelog.md +++ b/docs/ja/whats-new/security-changelog.md @@ -1,76 +1,76 @@ --- machine_translated: true -machine_translated_rev: d734a8e46ddd7465886ba4133bff743c55190626 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 76 -toc_title: "\u30BB\u30AD\u30E5\u30EA\u30C6\u30A3\u306E\u5909\u66F4\u5C65\u6B74" +toc_title: "\u30BB\u30AD\u30E5\u30EA\u30C6\u30A3\u5909\u66F4\u5C65\u6B74" --- -## ClickHouseリリース19.14.3.3で修正、2019-09-10 {#fixed-in-clickhouse-release-19-14-3-3-2019-09-10} +## ClickHouseリリース19.14.3.3,2019-09-10で修正 {#fixed-in-clickhouse-release-19-14-3-3-2019-09-10} ### CVE-2019-15024 {#cve-2019-15024} Аn attacker that has write access to ZooKeeper and who ican run a custom server available from the network where ClickHouse runs, can create a custom-built malicious server that will act as a ClickHouse replica and register it in ZooKeeper. When another replica will fetch data part from the malicious replica, it can force clickhouse-server to write to arbitrary path on filesystem. -クレジット：yandex情報セキュリティチームのeldar zaitov +クレジット：Yandexの情報セキュリティチームのEldar Zaitov ### CVE-2019-16535 {#cve-2019-16535} Аn OOB read, OOB write and integer underflow in decompression algorithms can be used to achieve RCE or DoS via native protocol. -クレジット：yandex情報セキュリティチームのeldar zaitov +クレジット：Yandexの情報セキュリティチームのEldar Zaitov ### CVE-2019-16536 {#cve-2019-16536} -スタックオーバーフローへのdosによっても実行させることができ、悪意のある認証クライアント +スタックオーバーフローへのDoSによっても実行させることができ、悪意のある認証クライアント -クレジット：yandex情報セキュリティチームのeldar zaitov +クレジット：Yandexの情報セキュリティチームのEldar Zaitov -## クリックハウスリリース19.13.6.1で修正、2019-09-20 {#fixed-in-clickhouse-release-19-13-6-1-2019-09-20} +## ClickHouseリリース19.13.6.1,2019-09-20で修正 {#fixed-in-clickhouse-release-19-13-6-1-2019-09-20} ### CVE-2019-18657 {#cve-2019-18657} -テーブル機能 `url` この脆弱性により、攻撃者が要求に任意のHTTPヘッダーを挿入することができました。 +テーブル関数 `url` この脆弱性により、攻撃者は要求に任意のHTTPヘッダーを挿入することができました。 -クレジット: [Nikita Tikhomirov](https://github.com/NSTikhomirov) +クレジット: [ニキータ-チホミロフ](https://github.com/NSTikhomirov) -## クリックハウスリリース18.12.13で修正、2018-09-10 {#fixed-in-clickhouse-release-18-12-13-2018-09-10} +## ClickHouseリリース18.12.13、2018-09-10で修正されました {#fixed-in-clickhouse-release-18-12-13-2018-09-10} ### CVE-2018-14672 {#cve-2018-14672} -機能搭載catboostモデルの可路のフォーカストラバーサル読書任意のファイルをエラーメッセージが返されます。 +機能搭載CatBoostモデルの可路のフォーカストラバーサル読書任意のファイルをエラーメッセージが返されます。 -クレジット：yandexの情報セキュリティチームのandrey krasichkov +クレジット：Yandexの情報セキュリティチームのアンドレイKrasichkov -## クリックハウスリリース18.10.3、2018-08-13で修正 {#fixed-in-clickhouse-release-18-10-3-2018-08-13} +## ClickHouseリリース18.10.3、2018-08-13で修正されました {#fixed-in-clickhouse-release-18-10-3-2018-08-13} ### CVE-2018-14671 {#cve-2018-14671} unixODBC許容荷重任意の共有オブジェクトからのファイルシステムにおけるリモートでコードが実行の脆弱性が存在します。 -クレジット：yandexの情報セキュリティチームのandrey krasichkovとevgeny sidorov +クレジット：Yandex情報セキュリティチームのAndrey KrasichkovとEvgeny Sidorov -## クリックハウスリリース1.1.54388で修正、2018-06-28 {#fixed-in-clickhouse-release-1-1-54388-2018-06-28} +## ClickHouseリリース1.1.54388、2018-06-28で修正 {#fixed-in-clickhouse-release-1-1-54388-2018-06-28} ### CVE-2018-14668 {#cve-2018-14668} -“remote” テーブル関数は、任意のシンボルを “user”, “password” と “default\_database” クロスプロトコル要求偽造攻撃につながったフィールド。 +“remote” テーブル関数で任意のシンボルを許可 “user”, “password” と “default\_database” 分野を横断プロトコルの要求偽造攻撃であった。 -クレジット：yandexの情報セキュリティチームのandrey krasichkov +クレジット：Yandexの情報セキュリティチームのアンドレイKrasichkov -## クリックハウスリリース1.1.54390、2018-07-06で修正 {#fixed-in-clickhouse-release-1-1-54390-2018-07-06} +## ClickHouseリリース1.1.54390、2018-07-06で修正 {#fixed-in-clickhouse-release-1-1-54390-2018-07-06} ### CVE-2018-14669 {#cve-2018-14669} -ClickHouse MySQLクライアントは “LOAD DATA LOCAL INFILE” 接続されたClickHouseサーバから任意のファイルを読み取る悪意のあるMySQLデータベー +ClickHouse MySQLクライアントがあった “LOAD DATA LOCAL INFILE” 機能が有効のとき、悪意のあるMySQLデータベース読む任意のファイルからの接続ClickHouseサーバーです。 -クレジット：yandexの情報セキュリティチームのandrey krasichkovとevgeny sidorov +クレジット：Yandex情報セキュリティチームのAndrey KrasichkovとEvgeny Sidorov -## クリックハウスリリース1.1.54131、2017-01-10で修正されました {#fixed-in-clickhouse-release-1-1-54131-2017-01-10} +## ClickHouseリリース1.1.54131、2017-01-10で修正 {#fixed-in-clickhouse-release-1-1-54131-2017-01-10} ### CVE-2018-14670 {#cve-2018-14670} -Debパッケージの不適切な構成は、データベースの不正使用につながる可能性があります。 +Debパッケージ内の不適切な構成は、データベースの不正使用につながる可能性があります。 -クレジット:英国の国家サイバーセキュリティセンター(ncsc) +クレジット：英国の国立サイバーセキュリティセンター（NCSC) {## [元の記事](https://clickhouse.tech/docs/en/security_changelog/) ##} diff --git a/docs/ru/faq/general.md b/docs/ru/faq/general.md index de1e34601f0..4862a069b95 100644 --- a/docs/ru/faq/general.md +++ b/docs/ru/faq/general.md @@ -25,7 +25,7 @@ NLS_LANG=RUSSIAN_RUSSIA.UTF8 ### Секция INTO OUTFILE {#sektsiia-into-outfile} -Добавьте секцию [INTO OUTFILE](../sql-reference/statements/select.md#into-outfile-clause) к своему запросу. +Добавьте секцию [INTO OUTFILE](../sql-reference/statements/select/into-outfile.md#into-outfile-clause) к своему запросу. Например: @@ -33,7 +33,7 @@ NLS_LANG=RUSSIAN_RUSSIA.UTF8 SELECT * FROM table INTO OUTFILE 'file' ``` -По умолчанию, для выдачи данных ClickHouse использует формат [TabSeparated](../interfaces/formats.md#tabseparated). Чтобы выбрать [формат данных](../interfaces/formats.md), используйте [секцию FORMAT](../sql-reference/statements/select.md#format-clause). +По умолчанию, для выдачи данных ClickHouse использует формат [TabSeparated](../interfaces/formats.md#tabseparated). Чтобы выбрать [формат данных](../interfaces/formats.md), используйте [секцию FORMAT](../sql-reference/statements/select/format.md#format-clause). Например: diff --git a/docs/ru/operations/requirements.md b/docs/ru/operations/requirements.md index 7fecbfe4e8e..459ccf4de25 100644 --- a/docs/ru/operations/requirements.md +++ b/docs/ru/operations/requirements.md @@ -17,9 +17,9 @@ ClickHouse реализует параллельную обработку дан - Сложности запросов. - Объёма данных, обрабатываемых в запросах. -Для расчета объёма RAM необходимо оценить размер промежуточных данных для операций [GROUP BY](../sql-reference/statements/select.md#select-group-by-clause), [DISTINCT](../sql-reference/statements/select.md#select-distinct), [JOIN](../sql-reference/statements/select.md#select-join) а также других операций, которыми вы пользуетесь. +Для расчета объёма RAM необходимо оценить размер промежуточных данных для операций [GROUP BY](../sql-reference/statements/select/group-by.md#select-group-by-clause), [DISTINCT](../sql-reference/statements/select/distinct.md#select-distinct), [JOIN](../sql-reference/statements/select/join.md#select-join) а также других операций, которыми вы пользуетесь. -ClickHouse может использовать внешнюю память для промежуточных данных. Подробнее смотрите в разделе [GROUP BY во внешней памяти](../sql-reference/statements/select.md#select-group-by-in-external-memory). +ClickHouse может использовать внешнюю память для промежуточных данных. Подробнее смотрите в разделе [GROUP BY во внешней памяти](../sql-reference/statements/select/group-by.md#select-group-by-in-external-memory). ## Файл подкачки {#fail-podkachki} diff --git a/docs/ru/operations/settings/query-complexity.md b/docs/ru/operations/settings/query-complexity.md index cb9a379b3bc..651f597c4d2 100644 --- a/docs/ru/operations/settings/query-complexity.md +++ b/docs/ru/operations/settings/query-complexity.md @@ -76,11 +76,11 @@ ## max\_bytes\_before\_external\_group\_by {#settings-max_bytes_before_external_group_by} -Включает или отключает выполнение секций `GROUP BY` во внешней памяти. Смотрите [GROUP BY во внешней памяти](../../sql-reference/statements/select.md#select-group-by-in-external-memory). +Включает или отключает выполнение секций `GROUP BY` во внешней памяти. Смотрите [GROUP BY во внешней памяти](../../sql-reference/statements/select/group-by.md#select-group-by-in-external-memory). Возможные значения: -- Максимальный объём RAM (в байтах), который может использовать отдельная операция [GROUP BY](../../sql-reference/statements/select.md#select-group-by-clause). +- Максимальный объём RAM (в байтах), который может использовать отдельная операция [GROUP BY](../../sql-reference/statements/select/group-by.md#select-group-by-clause). - 0 — `GROUP BY` во внешней памяти отключен. Значение по умолчанию — 0. @@ -228,7 +228,7 @@ FORMAT Null; Ограничивает количество строк в хэш-таблице, используемой при соединении таблиц. -Параметр применяется к операциям [SELECT… JOIN](../../sql-reference/statements/select.md#select-join) и к движку таблиц [Join](../../engines/table-engines/special/join.md). +Параметр применяется к операциям [SELECT… JOIN](../../sql-reference/statements/select/join.md#select-join) и к движку таблиц [Join](../../engines/table-engines/special/join.md). Если запрос содержит несколько `JOIN`, то ClickHouse проверяет значение настройки для каждого промежуточного результата. @@ -245,7 +245,7 @@ FORMAT Null; Ограничивает размер (в байтах) хэш-таблицы, используемой при объединении таблиц. -Параметр применяется к операциям [SELECT… JOIN](../../sql-reference/statements/select.md#select-join) и к движку таблиц [Join](../../engines/table-engines/special/join.md). +Параметр применяется к операциям [SELECT… JOIN](../../sql-reference/statements/select/join.md#select-join) и к движку таблиц [Join](../../engines/table-engines/special/join.md). Если запрос содержит несколько `JOIN`, то ClickHouse проверяет значение настройки для каждого промежуточного результата. @@ -274,7 +274,7 @@ FORMAT Null; **Смотрите также** -- [Секция JOIN](../../sql-reference/statements/select.md#select-join) +- [Секция JOIN](../../sql-reference/statements/select/join.md#select-join) - [Движоy таблиц Join](../../engines/table-engines/special/join.md) ## max\_partitions\_per\_insert\_block {#max-partitions-per-insert-block} diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 3a97f9e7cf4..16d840fe4b1 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -7,7 +7,7 @@ toc_title: Settings ## distributed\_product\_mode {#distributed-product-mode} -Изменяет поведение [распределенных подзапросов](../../sql-reference/statements/select.md). +Изменяет поведение [распределенных подзапросов](../../sql-reference/operators/index.md). ClickHouse применяет настройку в тех случаях, когда запрос содержит произведение распределённых таблиц, т.е. когда запрос к распределенной таблице содержит не-GLOBAL подзапрос к также распределенной таблице. @@ -367,7 +367,7 @@ INSERT INTO test VALUES (lower('Hello')), (lower('world')), (lower('INSERT')), ( ## join\_default\_strictness {#settings-join_default_strictness} -Устанавливает строгость по умолчанию для [JOIN](../../sql-reference/statements/select.md#select-join). +Устанавливает строгость по умолчанию для [JOIN](../../sql-reference/statements/select/join.md#select-join). Возможные значения @@ -393,13 +393,13 @@ INSERT INTO test VALUES (lower('Hello')), (lower('world')), (lower('INSERT')), ( См. также: -- [Секция JOIN](../../sql-reference/statements/select.md#select-join) +- [Секция JOIN](../../sql-reference/statements/select/join.md#select-join) - [Движок таблиц Join](../../engines/table-engines/special/join.md) - [join\_default\_strictness](#settings-join_default_strictness) ## join\_use\_nulls {#join_use_nulls} -Устанавливает тип поведения [JOIN](../../sql-reference/statements/select.md). При объединении таблиц могут появиться пустые ячейки. ClickHouse заполняет их по-разному в зависимости от настроек. +Устанавливает тип поведения [JOIN](../../sql-reference/statements/select/join.md). При объединении таблиц могут появиться пустые ячейки. ClickHouse заполняет их по-разному в зависимости от настроек. Возможные значения diff --git a/docs/ru/sql-reference/aggregate-functions/parametric-functions.md b/docs/ru/sql-reference/aggregate-functions/parametric-functions.md index b980ed199a8..7cc0bdef581 100644 --- a/docs/ru/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/ru/sql-reference/aggregate-functions/parametric-functions.md @@ -299,7 +299,7 @@ ORDER BY level ASC Аналитическая функция, которая показывает, насколько выдерживаются те или иные условия, например, удержание динамики/уровня [посещаемости сайта](https://yandex.ru/support/partner2/statistics/metrika-visitors-statistics.html?lang=ru). -Функция принимает набор (от 1 до 32) логических условий, как в [WHERE](../../sql-reference/statements/select.md#select-where), и применяет их к заданному набору данных. +Функция принимает набор (от 1 до 32) логических условий, как в [WHERE](../../sql-reference/statements/select/where.md#select-where), и применяет их к заданному набору данных. Условия, кроме первого, применяются попарно: результат второго будет истинным, если истинно первое и второе, третьего - если истинно первое и третье и т. д. diff --git a/docs/ru/sql-reference/aggregate-functions/reference.md b/docs/ru/sql-reference/aggregate-functions/reference.md index 8de1018026c..3705ac91180 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference.md +++ b/docs/ru/sql-reference/aggregate-functions/reference.md @@ -1249,7 +1249,7 @@ quantileTiming(level)(expr) Тип: `Float32`. !!! note "Примечания" - Если в функцию `quantileTimingIf` не передать значений, то вернётся [NaN](../../sql-reference/aggregate-functions/reference.md#data_type-float-nan-inf). Это необходимо для отделения подобных случаев от случаев, когда результат 0. Подробности про сортировку `NaN` cмотрите в разделе [Секция ORDER BY](../../sql-reference/statements/select.md#select-order-by). + Если в функцию `quantileTimingIf` не передать значений, то вернётся [NaN](../../sql-reference/aggregate-functions/reference.md#data_type-float-nan-inf). Это необходимо для отделения подобных случаев от случаев, когда результат 0. Подробности про сортировку `NaN` cмотрите в разделе [Секция ORDER BY](../../sql-reference/statements/select/order-by.md#select-order-by). **Пример** @@ -1334,7 +1334,7 @@ quantileTimingWeighted(level)(expr, weight) Тип: `Float32`. !!! note "Примечания" - Если в функцию `quantileTimingIf` не передать значений, то вернётся [NaN](../../sql-reference/aggregate-functions/reference.md#data_type-float-nan-inf). Это необходимо для отделения подобных случаев от случаев, когда результат 0. Подробности про сортировку `NaN` cмотрите в разделе [Секция ORDER BY](../../sql-reference/statements/select.md#select-order-by). + Если в функцию `quantileTimingIf` не передать значений, то вернётся [NaN](../../sql-reference/aggregate-functions/reference.md#data_type-float-nan-inf). Это необходимо для отделения подобных случаев от случаев, когда результат 0. Подробности про сортировку `NaN` cмотрите в разделе [Секция ORDER BY](../../sql-reference/statements/select/order-by.md#select-order-by). **Пример** diff --git a/docs/ru/sql-reference/functions/conditional-functions.md b/docs/ru/sql-reference/functions/conditional-functions.md index 78bdab826d9..7efb6f7bfc5 100644 --- a/docs/ru/sql-reference/functions/conditional-functions.md +++ b/docs/ru/sql-reference/functions/conditional-functions.md @@ -70,7 +70,7 @@ SELECT if(0, plus(2, 2), plus(2, 6)) ## multiIf {#multiif} -Позволяет более компактно записать оператор [CASE](../operators.md#operator_case) в запросе. +Позволяет более компактно записать оператор [CASE](../operators/index.md#operator_case) в запросе. multiIf(cond_1, then_1, cond_2, then_2...else) diff --git a/docs/ru/sql-reference/functions/in-functions.md b/docs/ru/sql-reference/functions/in-functions.md index 4153eb1aac0..679fcbccc21 100644 --- a/docs/ru/sql-reference/functions/in-functions.md +++ b/docs/ru/sql-reference/functions/in-functions.md @@ -2,7 +2,7 @@ ## in, notIn, globalIn, globalNotIn {#in-functions} -Смотрите раздел [Операторы IN](../statements/select.md#select-in-operators). +Смотрите раздел [Операторы IN](../operators/in.md#select-in-operators). ## tuple(x, y, …), оператор (x, y, …) {#tuplex-y-operator-x-y} diff --git a/docs/ru/sql-reference/index.md b/docs/ru/sql-reference/index.md index 3dff768cb7e..a13e3774b86 100644 --- a/docs/ru/sql-reference/index.md +++ b/docs/ru/sql-reference/index.md @@ -7,7 +7,7 @@ toc_title: hidden # Справка по SQL {#spravka-po-sql} -- [SELECT](statements/select.md) +- [SELECT](statements/select/index.md) - [INSERT INTO](statements/insert-into.md) - [CREATE](statements/create.md) - [ALTER](statements/alter.md) diff --git a/docs/ru/sql-reference/operators/in.md b/docs/ru/sql-reference/operators/in.md new file mode 120000 index 00000000000..3a2feda2f61 --- /dev/null +++ b/docs/ru/sql-reference/operators/in.md @@ -0,0 +1 @@ +../../../en/sql-reference/operators/in.md \ No newline at end of file diff --git a/docs/ru/sql-reference/operators.md b/docs/ru/sql-reference/operators/index.md similarity index 88% rename from docs/ru/sql-reference/operators.md rename to docs/ru/sql-reference/operators/index.md index 85fa861fec5..ef4d1ae048e 100644 --- a/docs/ru/sql-reference/operators.md +++ b/docs/ru/sql-reference/operators/index.md @@ -55,7 +55,7 @@ ## Операторы для работы с множествами {#operatory-dlia-raboty-s-mnozhestvami} -*Смотрите раздел [Операторы IN](../sql-reference/statements/select.md#select-in-operators).* +*Смотрите раздел [Операторы IN](../../sql-reference/operators/in.md#select-in-operators).* `a IN ...` - функция `in(a, b)` @@ -86,7 +86,7 @@ EXTRACT(part FROM date); Эти значения могут быть указаны также в нижнем регистре (`day`, `month`). -В параметре `date` указывается исходная дата. Поддерживаются типы [Date](../sql-reference/data-types/date.md) и [DateTime](../sql-reference/data-types/datetime.md). +В параметре `date` указывается исходная дата. Поддерживаются типы [Date](../../sql-reference/data-types/date.md) и [DateTime](../../sql-reference/data-types/datetime.md). Примеры: @@ -133,7 +133,7 @@ FROM test.Orders; ### INTERVAL {#operator-interval} -Создаёт значение типа [Interval](../sql-reference/operators.md) которое должно использоваться в арифметических операциях со значениями типов [Date](../sql-reference/operators.md) и [DateTime](../sql-reference/operators.md). +Создаёт значение типа [Interval](../../sql-reference/operators/index.md) которое должно использоваться в арифметических операциях со значениями типов [Date](../../sql-reference/operators/index.md) и [DateTime](../../sql-reference/operators/index.md). Типы интервалов: - `SECOND` @@ -162,8 +162,8 @@ SELECT now() AS current_date_time, current_date_time + INTERVAL 4 DAY + INTERVAL **Смотрите также** -- Тип данных [Interval](../sql-reference/operators.md) -- Функции преобразования типов [toInterval](../sql-reference/operators.md#function-tointerval) +- Тип данных [Interval](../../sql-reference/operators/index.md) +- Функции преобразования типов [toInterval](../../sql-reference/operators/index.md#function-tointerval) ## Оператор логического отрицания {#operator-logicheskogo-otritsaniia} @@ -183,7 +183,7 @@ SELECT now() AS current_date_time, current_date_time + INTERVAL 4 DAY + INTERVAL Примечание: -Условный оператор сначала вычисляет значения b и c, затем проверяет выполнение условия a, и только после этого возвращает соответствующее значение. Если в качестве b или с выступает функция [arrayJoin()](../sql-reference/operators.md#functions_arrayjoin), то размножение каждой строки произойдет вне зависимости от условия а. +Условный оператор сначала вычисляет значения b и c, затем проверяет выполнение условия a, и только после этого возвращает соответствующее значение. Если в качестве b или с выступает функция [arrayJoin()](../../sql-reference/operators/index.md#functions_arrayjoin), то размножение каждой строки произойдет вне зависимости от условия а. ## Условное выражение {#operator_case} @@ -232,7 +232,7 @@ ClickHouse поддерживает операторы `IS NULL` и `IS NOT NULL ### IS NULL {#operator-is-null} -- Для значений типа [Nullable](../sql-reference/operators.md) оператор `IS NULL` возвращает: +- Для значений типа [Nullable](../../sql-reference/operators/index.md) оператор `IS NULL` возвращает: - `1`, если значение — `NULL`. - `0` в обратном случае. - Для прочих значений оператор `IS NULL` всегда возвращает `0`. @@ -251,7 +251,7 @@ SELECT x+100 FROM t_null WHERE y IS NULL ### IS NOT NULL {#is-not-null} -- Для значений типа [Nullable](../sql-reference/operators.md) оператор `IS NOT NULL` возвращает: +- Для значений типа [Nullable](../../sql-reference/operators/index.md) оператор `IS NOT NULL` возвращает: - `0`, если значение — `NULL`. - `1`, в обратном случае. - Для прочих значений оператор `IS NOT NULL` всегда возвращает `1`. diff --git a/docs/ru/sql-reference/statements/grant.md b/docs/ru/sql-reference/statements/grant.md new file mode 120000 index 00000000000..f2acbe125b4 --- /dev/null +++ b/docs/ru/sql-reference/statements/grant.md @@ -0,0 +1 @@ +../../../en/sql-reference/statements/grant.md \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/revoke.md b/docs/ru/sql-reference/statements/revoke.md new file mode 120000 index 00000000000..4321fdb14a7 --- /dev/null +++ b/docs/ru/sql-reference/statements/revoke.md @@ -0,0 +1 @@ +../../../en/sql-reference/statements/revoke.md \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/select.md b/docs/ru/sql-reference/statements/select.md deleted file mode 100644 index 57aaaad87da..00000000000 --- a/docs/ru/sql-reference/statements/select.md +++ /dev/null @@ -1,1406 +0,0 @@ ---- -toc_priority: 33 -toc_title: SELECT ---- - -# Синтаксис запросов SELECT {#sintaksis-zaprosov-select} - -`SELECT` осуществляет выборку данных. - -``` sql -[WITH expr_list|(subquery)] -SELECT [DISTINCT] expr_list -[FROM [db.]table | (subquery) | table_function] [FINAL] -[SAMPLE sample_coeff] -[ARRAY JOIN ...] -[GLOBAL] [ANY|ALL] [INNER|LEFT|RIGHT|FULL|CROSS] [OUTER] JOIN (subquery)|table USING columns_list -[PREWHERE expr] -[WHERE expr] -[GROUP BY expr_list] [WITH TOTALS] -[HAVING expr] -[ORDER BY expr_list] -[LIMIT [offset_value, ]n BY columns] -[LIMIT [n, ]m] -[UNION ALL ...] -[INTO OUTFILE filename] -[FORMAT format] -``` - -Все секции, кроме списка выражений сразу после SELECT, являются необязательными. -Ниже секции будут описаны в порядке, почти соответствующем конвейеру выполнения запроса. - -Если в запросе отсутствуют секции `DISTINCT`, `GROUP BY`, `ORDER BY`, подзапросы в `IN` и `JOIN`, то запрос будет обработан полностью потоково, с использованием O(1) количества оперативки. -Иначе запрос может съесть много оперативки, если не указаны подходящие ограничения `max_memory_usage`, `max_rows_to_group_by`, `max_rows_to_sort`, `max_rows_in_distinct`, `max_bytes_in_distinct`, `max_rows_in_set`, `max_bytes_in_set`, `max_rows_in_join`, `max_bytes_in_join`, `max_bytes_before_external_sort`, `max_bytes_before_external_group_by`. Подробнее смотрите в разделе «Настройки». Присутствует возможность использовать внешнюю сортировку (с сохранением временных данных на диск) и внешнюю агрегацию. `Merge join` в системе нет. - -### Секция WITH {#sektsiia-with} - -Данная секция представляет собой [CTE](https://ru.wikipedia.org/wiki/Иерархические_и_рекурсивные_запросы_в_SQL), с рядом ограничений: -1. Рекурсивные запросы не поддерживаются -2. Если в качестве выражения используется подзапрос, то результат должен содержать ровно одну строку -3. Результаты выражений нельзя переиспользовать во вложенных запросах -В дальнейшем, результаты выражений можно использовать в секции SELECT. - -Пример 1: Использование константного выражения как «переменной» - -``` sql -WITH '2019-08-01 15:23:00' as ts_upper_bound -SELECT * -FROM hits -WHERE - EventDate = toDate(ts_upper_bound) AND - EventTime <= ts_upper_bound -``` - -Пример 2: Выкидывание выражения sum(bytes) из списка колонок в SELECT - -``` sql -WITH sum(bytes) as s -SELECT - formatReadableSize(s), - table -FROM system.parts -GROUP BY table -ORDER BY s -``` - -Пример 3: Использование результатов скалярного подзапроса - -``` sql -/* запрос покажет TOP 10 самых больших таблиц */ -WITH - ( - SELECT sum(bytes) - FROM system.parts - WHERE active - ) AS total_disk_usage -SELECT - (sum(bytes) / total_disk_usage) * 100 AS table_disk_usage, - table -FROM system.parts -GROUP BY table -ORDER BY table_disk_usage DESC -LIMIT 10 -``` - -Пример 4: Переиспользование выражения -В настоящий момент, переиспользование выражения из секции WITH внутри подзапроса возможно только через дублирование. - -``` sql -WITH ['hello'] AS hello -SELECT - hello, - * -FROM -( - WITH ['hello'] AS hello - SELECT hello -) -``` - -``` text -┌─hello─────┬─hello─────┐ -│ ['hello'] │ ['hello'] │ -└───────────┴───────────┘ -``` - -### Секция FROM {#select-from} - -Если секция FROM отсутствует, то данные будут читаться из таблицы `system.one`. -Таблица `system.one` содержит ровно одну строку (то есть, эта таблица выполняет такую же роль, как таблица DUAL, которую можно найти в других СУБД). - -Cекция `FROM` определяет источник данных: - -- Таблица -- Подзапрос -- [Табличная функция](../../sql-reference/statements/select.md) - -Также могут присутствовать `ARRAY JOIN` и обычный `JOIN` (смотрите ниже). - -Вместо таблицы, может быть указан подзапрос `SELECT` в скобках. -В отличие от стандартного SQL, после подзапроса не обязательно указывать его синоним. - -Для выполнения запроса, из соответствующей таблицы, вынимаются все столбцы, перечисленные в запросе. Из подзапросов выкидываются столбцы, не нужные для внешнего запроса. -Если в запросе не перечислено ни одного столбца (например, `SELECT count() FROM t`), то из таблицы всё равно вынимается один какой-нибудь столбец (предпочитается самый маленький), для того, чтобы можно было посчитать количество строк. - -Модификатор `FINAL` может быть использован в запросе `SELECT` из таблиц семейства [MergeTree](../../engines/table-engines/mergetree-family/index.md). При указании `FINAL`, данные будут выбираться полностью «домерженными». Стоит учитывать, что использование `FINAL` приводит к чтению также столбцов, относящихся к первичному ключу. Также, запрос будет выполняться в один поток, и при выполнении запроса будет выполняться слияние данных. Это приводит к тому, что при использовании `FINAL`, запрос выполняется медленнее. В большинстве случаев, следует избегать использования `FINAL`. -Модификатор `FINAL` может быть использован для всех таблиц семейства `MergeTree`, которые производят преобразования данных в процессе фоновых слияний (кроме GraphiteMergeTree). - -#### FINAL Modifier {#select-from-final} - -Применим при выборке данных из таблиц с движками таблиц семейства [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md)), кроме `GraphiteMergeTree`. Если в запросе используется `FINAL`, то ClickHouse полностью мёржит данные перед выдачей результата, таким образом выполняя все преобразования данных, которые производятся движком таблиц при мёржах. - -Также поддержан для движков: - -- [Replicated](../../engines/table-engines/mergetree-family/replication.md)-версий `MergeTree`. -- [View](../../engines/table-engines/special/view.md), [Buffer](../../engines/table-engines/special/buffer.md), [Distributed](../../engines/table-engines/special/distributed.md), и [MaterializedView](../../engines/table-engines/special/materializedview.md), которые работают поверх других движков, если они созданы для таблиц с движками семейства `MergeTree`. - -Запросы, использующие `FINAL` исполняются медленнее аналогичных запросов без `FINAL`, поскольку: - -- Запрос исполняется в один поток и данные мёржатся в процессе выполнения. -- Запросы с модификатором `FINAL` дополнительно к столбцам, указанным в запросе, читают столбцы первичного ключа. - -По возможности не используйте модификатор `FINAL`. - -### Секция SAMPLE {#select-sample-clause} - -Секция `SAMPLE` позволяет выполнять запросы приближённо. Например, чтобы посчитать статистику по всем визитам, можно обработать 1/10 всех визитов и результат домножить на 10. - -Сэмплирование имеет смысл, когда: - -1. Точность результата не важна, например, для оценочных расчетов. -2. Возможности аппаратной части не позволяют соответствовать строгим критериям. Например, время ответа должно быть \<100 мс. При этом точность расчета имеет более низкий приоритет. -3. Точность результата участвует в бизнес-модели сервиса. Например, пользователи с бесплатной подпиской на сервис могут получать отчеты с меньшей точностью, чем пользователи с премиум подпиской. - -!!! note "Внимание" - Не стоит использовать сэмплирование в тех задачах, где важна точность расчетов. Например, при работе с финансовыми отчетами. - -Свойства сэмплирования: - -- Сэмплирование работает детерминированно. При многократном выполнении одного и того же запроса `SELECT .. SAMPLE`, результат всегда будет одинаковым. -- Сэмплирование поддерживает консистентность для разных таблиц. Имеется в виду, что для таблиц с одним и тем же ключом сэмплирования, подмножество данных в выборках будет одинаковым (выборки при этом должны быть сформированы для одинаковой доли данных). Например, выборка по идентификаторам посетителей выберет из разных таблиц строки с одинаковым подмножеством всех возможных идентификаторов. Это свойство позволяет использовать выборки в подзапросах в секции [IN](#select-in-operators), а также объединять выборки с помощью [JOIN](#select-join). -- Сэмплирование позволяет читать меньше данных с диска. Обратите внимание, для этого необходимо корректно указать ключ сэмплирования. Подробнее см. в разделе [Создание таблицы MergeTree](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table). - -Сэмплирование поддерживается только таблицами семейства [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table) и только в том случае, если для таблиц был указан ключ сэмплирования (выражение, на основе которого должна производиться выборка). Подробнее см. в разделе [Создание таблиц MergeTree](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table). - -Выражение `SAMPLE` в запросе можно задать следующими способами: - -| Способ задания SAMPLE | Описание | -|-----------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `SAMPLE k` | Здесь `k` – это дробное число в интервале от 0 до 1.
Запрос будет выполнен по `k` доле данных. Например, если указано `SAMPLE 1/10`, то запрос будет выполнен для выборки из 1/10 данных. [Подробнее](#select-sample-k) | -| `SAMPLE n` | Здесь `n` – это достаточно большое целое число.
Запрос будет выполнен для выборки, состоящей из не менее чем `n` строк. Например, если указано `SAMPLE 10000000`, то запрос будет выполнен для не менее чем 10,000,000 строк. [Подробнее](#select-sample-n) | -| `SAMPLE k OFFSET m` | Здесь `k` и `m` – числа от 0 до 1.
Запрос будет выполнен по `k` доле данных. При этом выборка будет сформирована со смещением на `m` долю. [Подробнее](#select-sample-offset) | - -#### SAMPLE k {#select-sample-k} - -Здесь `k` – число в интервале от 0 до 1. Поддерживается как дробная, так и десятичная форма записи. Например, `SAMPLE 1/2` или `SAMPLE 0.5`. - -Если задано выражение `SAMPLE k`, запрос будет выполнен для `k` доли данных. Рассмотрим пример: - -``` sql -SELECT - Title, - count() * 10 AS PageViews -FROM hits_distributed -SAMPLE 0.1 -WHERE - CounterID = 34 -GROUP BY Title -ORDER BY PageViews DESC LIMIT 1000 -``` - -В этом примере запрос выполняется по выборке из 0.1 (10%) данных. Значения агрегатных функций не корректируются автоматически, поэтому чтобы получить приближённый результат, значение `count()` нужно вручную умножить на 10. - -Выборка с указанием относительного коэффициента является «согласованной»: для таблиц с одним и тем же ключом сэмплирования, выборка с одинаковой относительной долей всегда будет составлять одно и то же подмножество данных. То есть выборка из разных таблиц, на разных серверах, в разное время, формируется одинаковым образом. - -#### SAMPLE n {#select-sample-n} - -Здесь `n` – это достаточно большое целое число. Например, `SAMPLE 10000000`. - -Если задано выражение `SAMPLE n`, запрос будет выполнен для выборки из не менее `n` строк (но не значительно больше этого значения). Например, если задать `SAMPLE 10000000`, в выборку попадут не менее 10,000,000 строк. - -!!! note "Примечание" - Следует иметь в виду, что `n` должно быть достаточно большим числом. Так как минимальной единицей данных для чтения является одна гранула (её размер задаётся настройкой `index_granularity` для таблицы), имеет смысл создавать выборки, размер которых существенно превосходит размер гранулы. - -При выполнении `SAMPLE n` коэффициент сэмплирования заранее неизвестен (то есть нет информации о том, относительно какого количества данных будет сформирована выборка). Чтобы узнать коэффициент сэмплирования, используйте столбец `_sample_factor`. - -Виртуальный столбец `_sample_factor` автоматически создается в тех таблицах, для которых задано выражение `SAMPLE BY` (подробнее см. в разделе [Создание таблицы MergeTree](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table)). В столбце содержится коэффициент сэмплирования для таблицы – он рассчитывается динамически по мере добавления данных в таблицу. Ниже приведены примеры использования столбца `_sample_factor`. - -Предположим, у нас есть таблица, в которой ведется статистика посещений сайта. Пример ниже показывает, как рассчитать суммарное число просмотров: - -``` sql -SELECT sum(PageViews * _sample_factor) -FROM visits -SAMPLE 10000000 -``` - -Следующий пример показывает, как посчитать общее число визитов: - -``` sql -SELECT sum(_sample_factor) -FROM visits -SAMPLE 10000000 -``` - -В примере ниже рассчитывается среднее время на сайте. Обратите внимание, при расчете средних значений, умножать результат на коэффициент сэмплирования не нужно. - -``` sql -SELECT avg(Duration) -FROM visits -SAMPLE 10000000 -``` - -#### SAMPLE k OFFSET m {#select-sample-offset} - -Здесь `k` и `m` – числа в интервале от 0 до 1. Например, `SAMPLE 0.1 OFFSET 0.5`. Поддерживается как дробная, так и десятичная форма записи. - -При задании `SAMPLE k OFFSET m`, выборка будет сформирована из `k` доли данных со смещением на долю `m`. Примеры приведены ниже. - -**Пример 1** - -``` sql -SAMPLE 1/10 -``` - -В этом примере выборка будет сформирована по 1/10 доле всех данных: - -`[++------------------]` - -**Пример 2** - -``` sql -SAMPLE 1/10 OFFSET 1/2 -``` - -Здесь выборка, которая состоит из 1/10 доли данных, взята из второй половины данных. - -`[----------++--------]` - -### Секция ARRAY JOIN {#select-array-join-clause} - -Позволяет выполнить `JOIN` с массивом или вложенной структурой данных. Смысл похож на функцию [arrayJoin](../../sql-reference/statements/select.md#functions_arrayjoin), но функциональность более широкая. - -``` sql -SELECT -FROM -[LEFT] ARRAY JOIN -[WHERE|PREWHERE ] -... -``` - -В запросе может быть указано не более одной секции `ARRAY JOIN`. - -При использовании `ARRAY JOIN`, порядок выполнения запроса оптимизируется. Несмотря на то что секция `ARRAY JOIN` всегда указывается перед выражением `WHERE / PREWHERE`, преобразование `JOIN` может быть выполнено как до выполнения выражения `WHERE / PREWHERE` (если результат необходим в этом выражении), так и после (чтобы уменьшить объём расчетов). Порядок обработки контролируется оптимизатором запросов. - -Секция `ARRAY JOIN` поддерживает следующие формы записи: - -- `ARRAY JOIN` — в этом случае результат `JOIN` не будет содержать пустые массивы; -- `LEFT ARRAY JOIN` — пустые массивы попадут в результат выполнения `JOIN`. В качестве значения для пустых массивов устанавливается значение по умолчанию. Обычно это 0, пустая строка или NULL, в зависимости от типа элементов массива. - -Рассмотрим примеры использования `ARRAY JOIN` и `LEFT ARRAY JOIN`. Для начала создадим таблицу, содержащую столбец с типом [Array](../../sql-reference/statements/select.md), и добавим в него значение: - -``` sql -CREATE TABLE arrays_test -( - s String, - arr Array(UInt8) -) ENGINE = Memory; - -INSERT INTO arrays_test -VALUES ('Hello', [1,2]), ('World', [3,4,5]), ('Goodbye', []); -``` - -``` text -┌─s───────────┬─arr─────┐ -│ Hello │ [1,2] │ -│ World │ [3,4,5] │ -│ Goodbye │ [] │ -└─────────────┴─────────┘ -``` - -В примере ниже используется `ARRAY JOIN`: - -``` sql -SELECT s, arr -FROM arrays_test -ARRAY JOIN arr; -``` - -``` text -┌─s─────┬─arr─┐ -│ Hello │ 1 │ -│ Hello │ 2 │ -│ World │ 3 │ -│ World │ 4 │ -│ World │ 5 │ -└───────┴─────┘ -``` - -Следующий пример использует `LEFT ARRAY JOIN`: - -``` sql -SELECT s, arr -FROM arrays_test -LEFT ARRAY JOIN arr; -``` - -``` text -┌─s───────────┬─arr─┐ -│ Hello │ 1 │ -│ Hello │ 2 │ -│ World │ 3 │ -│ World │ 4 │ -│ World │ 5 │ -│ Goodbye │ 0 │ -└─────────────┴─────┘ -``` - -#### Использование алиасов {#ispolzovanie-aliasov} - -Для массива в секции `ARRAY JOIN` может быть указан алиас. В этом случае, элемент массива будет доступен под этим алиасом, а сам массив — под исходным именем. Пример: - -``` sql -SELECT s, arr, a -FROM arrays_test -ARRAY JOIN arr AS a; -``` - -``` text -┌─s─────┬─arr─────┬─a─┐ -│ Hello │ [1,2] │ 1 │ -│ Hello │ [1,2] │ 2 │ -│ World │ [3,4,5] │ 3 │ -│ World │ [3,4,5] │ 4 │ -│ World │ [3,4,5] │ 5 │ -└───────┴─────────┴───┘ -``` - -Используя алиасы, можно выполнять `JOIN` с внешними массивами: - -``` sql -SELECT s, arr_external -FROM arrays_test -ARRAY JOIN [1, 2, 3] AS arr_external; -``` - -``` text -┌─s───────────┬─arr_external─┐ -│ Hello │ 1 │ -│ Hello │ 2 │ -│ Hello │ 3 │ -│ World │ 1 │ -│ World │ 2 │ -│ World │ 3 │ -│ Goodbye │ 1 │ -│ Goodbye │ 2 │ -│ Goodbye │ 3 │ -└─────────────┴──────────────┘ -``` - -В секции `ARRAY JOIN` можно указать через запятую сразу несколько массивов. В этом случае, `JOIN` делается с ними одновременно (прямая сумма, а не прямое произведение). Обратите внимание, массивы должны быть одинаковых размеров. Примеры: - -``` sql -SELECT s, arr, a, num, mapped -FROM arrays_test -ARRAY JOIN arr AS a, arrayEnumerate(arr) AS num, arrayMap(x -> x + 1, arr) AS mapped; -``` - -``` text -┌─s─────┬─arr─────┬─a─┬─num─┬─mapped─┐ -│ Hello │ [1,2] │ 1 │ 1 │ 2 │ -│ Hello │ [1,2] │ 2 │ 2 │ 3 │ -│ World │ [3,4,5] │ 3 │ 1 │ 4 │ -│ World │ [3,4,5] │ 4 │ 2 │ 5 │ -│ World │ [3,4,5] │ 5 │ 3 │ 6 │ -└───────┴─────────┴───┴─────┴────────┘ -``` - -В примере ниже используется функция [arrayEnumerate](../../sql-reference/statements/select.md#array_functions-arrayenumerate): - -``` sql -SELECT s, arr, a, num, arrayEnumerate(arr) -FROM arrays_test -ARRAY JOIN arr AS a, arrayEnumerate(arr) AS num; -``` - -``` text -┌─s─────┬─arr─────┬─a─┬─num─┬─arrayEnumerate(arr)─┐ -│ Hello │ [1,2] │ 1 │ 1 │ [1,2] │ -│ Hello │ [1,2] │ 2 │ 2 │ [1,2] │ -│ World │ [3,4,5] │ 3 │ 1 │ [1,2,3] │ -│ World │ [3,4,5] │ 4 │ 2 │ [1,2,3] │ -│ World │ [3,4,5] │ 5 │ 3 │ [1,2,3] │ -└───────┴─────────┴───┴─────┴─────────────────────┘ -``` - -#### ARRAY JOIN с вложенными структурами данных {#array-join-s-vlozhennymi-strukturami-dannykh} - -`ARRAY JOIN` также работает с [вложенными структурами данных](../../sql-reference/statements/select.md). Пример: - -``` sql -CREATE TABLE nested_test -( - s String, - nest Nested( - x UInt8, - y UInt32) -) ENGINE = Memory; - -INSERT INTO nested_test -VALUES ('Hello', [1,2], [10,20]), ('World', [3,4,5], [30,40,50]), ('Goodbye', [], []); -``` - -``` text -┌─s───────┬─nest.x──┬─nest.y─────┐ -│ Hello │ [1,2] │ [10,20] │ -│ World │ [3,4,5] │ [30,40,50] │ -│ Goodbye │ [] │ [] │ -└─────────┴─────────┴────────────┘ -``` - -``` sql -SELECT s, `nest.x`, `nest.y` -FROM nested_test -ARRAY JOIN nest; -``` - -``` text -┌─s─────┬─nest.x─┬─nest.y─┐ -│ Hello │ 1 │ 10 │ -│ Hello │ 2 │ 20 │ -│ World │ 3 │ 30 │ -│ World │ 4 │ 40 │ -│ World │ 5 │ 50 │ -└───────┴────────┴────────┘ -``` - -При указании имени вложенной структуры данных в `ARRAY JOIN`, смысл такой же, как `ARRAY JOIN` со всеми элементами-массивами, из которых она состоит. Пример: - -``` sql -SELECT s, `nest.x`, `nest.y` -FROM nested_test -ARRAY JOIN `nest.x`, `nest.y`; -``` - -``` text -┌─s─────┬─nest.x─┬─nest.y─┐ -│ Hello │ 1 │ 10 │ -│ Hello │ 2 │ 20 │ -│ World │ 3 │ 30 │ -│ World │ 4 │ 40 │ -│ World │ 5 │ 50 │ -└───────┴────────┴────────┘ -``` - -Такой вариант тоже имеет смысл: - -``` sql -SELECT s, `nest.x`, `nest.y` -FROM nested_test -ARRAY JOIN `nest.x`; -``` - -``` text -┌─s─────┬─nest.x─┬─nest.y─────┐ -│ Hello │ 1 │ [10,20] │ -│ Hello │ 2 │ [10,20] │ -│ World │ 3 │ [30,40,50] │ -│ World │ 4 │ [30,40,50] │ -│ World │ 5 │ [30,40,50] │ -└───────┴────────┴────────────┘ -``` - -Алиас для вложенной структуры данных можно использовать, чтобы выбрать как результат `JOIN`-а, так и исходный массив. Пример: - -``` sql -SELECT s, `n.x`, `n.y`, `nest.x`, `nest.y` -FROM nested_test -ARRAY JOIN nest AS n; -``` - -``` text -┌─s─────┬─n.x─┬─n.y─┬─nest.x──┬─nest.y─────┐ -│ Hello │ 1 │ 10 │ [1,2] │ [10,20] │ -│ Hello │ 2 │ 20 │ [1,2] │ [10,20] │ -│ World │ 3 │ 30 │ [3,4,5] │ [30,40,50] │ -│ World │ 4 │ 40 │ [3,4,5] │ [30,40,50] │ -│ World │ 5 │ 50 │ [3,4,5] │ [30,40,50] │ -└───────┴─────┴─────┴─────────┴────────────┘ -``` - -Пример использования функции [arrayEnumerate](../../sql-reference/statements/select.md#array_functions-arrayenumerate): - -``` sql -SELECT s, `n.x`, `n.y`, `nest.x`, `nest.y`, num -FROM nested_test -ARRAY JOIN nest AS n, arrayEnumerate(`nest.x`) AS num; -``` - -``` text -┌─s─────┬─n.x─┬─n.y─┬─nest.x──┬─nest.y─────┬─num─┐ -│ Hello │ 1 │ 10 │ [1,2] │ [10,20] │ 1 │ -│ Hello │ 2 │ 20 │ [1,2] │ [10,20] │ 2 │ -│ World │ 3 │ 30 │ [3,4,5] │ [30,40,50] │ 1 │ -│ World │ 4 │ 40 │ [3,4,5] │ [30,40,50] │ 2 │ -│ World │ 5 │ 50 │ [3,4,5] │ [30,40,50] │ 3 │ -└───────┴─────┴─────┴─────────┴────────────┴─────┘ -``` - -### Секция JOIN {#select-join} - -Соединяет данные в привычном для [SQL JOIN](https://en.wikipedia.org/wiki/Join_(SQL)) смысле. - -!!! info "Примечание" - Не связана с функциональностью [ARRAY JOIN](#select-array-join-clause). - -``` sql -SELECT -FROM -[GLOBAL] [ANY|ALL] [INNER|LEFT|RIGHT|FULL|CROSS] [OUTER] JOIN -(ON )|(USING ) ... -``` - -Вместо `` и `` можно указать имена таблиц. Это эквивалентно подзапросу `SELECT * FROM table`, за исключением особого случая таблицы с движком [Join](../../sql-reference/statements/select.md) – массива, подготовленного для присоединения. - -#### Поддерживаемые типы `JOIN` {#select-join-types} - -- `INNER JOIN` (or `JOIN`) -- `LEFT JOIN` (or `LEFT OUTER JOIN`) -- `RIGHT JOIN` (or `RIGHT OUTER JOIN`) -- `FULL JOIN` (or `FULL OUTER JOIN`) -- `CROSS JOIN` (or `,` ) - -Смотрите описание стандартного [SQL JOIN](https://en.wikipedia.org/wiki/Join_(SQL)). - -#### Множественный JOIN {#mnozhestvennyi-join} - -При выполнении запросов, ClickHouse перезаписывает множественный `JOIN` как комбинацию двух-табличных объединений и обрабатывает их последовательно. Например, если необходимо объединить четыре таблицы, ClickHouse объединяет первую и вторую таблицы, затем соединяет результат с третьей, а затем с четвертой. - -Если запрос содержит секцию `WHERE`, ClickHouse пытается пробросить фильтры из этой секции в промежуточный `JOIN`. Если он не может пробросить фильтр в каждый промежуточный `JOIN`, ClickHouse применяет фильтры после того, как все `JOIN` будут выполнены. - -Для создания запросов мы рекомендуем использовать синтаксис `JOIN ON` или `JOIN USING`. Например: - -``` sql -SELECT * FROM t1 JOIN t2 ON t1.a = t2.a JOIN t3 ON t1.a = t3.a -``` - -В секции `FROM` вы можете использовать разделенные запятыми списки таблиц для объединения. Например: - -``` sql -SELECT * FROM t1, t2, t3 WHERE t1.a = t2.a AND t1.a = t3.a -``` - -Не смешивайте синтаксисы. - -ClickHouse не поддерживает синтаксис с запятыми напрямую и мы не рекомендуем его использовать. Алгоритм пытается переписать запрос с помощью секций `CROSS JOIN` и `INNER JOIN` и затем продолжает его выполнение. При переписывании запроса, ClickHouse пытается оптимизировать производительность и потребление памяти. По умолчанию, ClickHouse трактует запятые как `INNER JOIN` и конвертирует их в `CROSS JOIN` когда не может гарантировать, что `INNER JOIN` возвращает запрошенные данные. - -#### Строгость {#select-join-strictness} - -- `ALL` — если правая таблица содержит несколько подходящих строк, то ClickHouse выполняет их [декартово произведение](https://ru.wikipedia.org/wiki/Прямое_произведение). Это стандартное поведение `JOIN` в SQL. -- `ANY` — если в правой таблице несколько соответствующих строк, то присоединяется только первая найденная. Если в правой таблице есть только одна подходящая строка, то результаты `ANY` и `ALL` совпадают. -- `ASOF` — для объединения последовательностей с нечётким совпадением. `ASOF JOIN` описан ниже по тексту. - -**Использование ASOF JOIN** - -`ASOF JOIN` применим в том случае, когда необходимо объединять записи, которые не имеют точного совпадения. - -Таблицы для `ASOF JOIN` должны иметь столбец с отсортированной последовательностью. Этот столбец не может быть единственным в таблице и должен быть одного из типов: `UInt32`, `UInt64`, `Float32`, `Float64`, `Date` и `DateTime`. - -Синтаксис `ASOF JOIN ... ON`: - -``` sql -SELECT expressions_list -FROM table_1 -ASOF LEFT JOIN table_2 -ON equi_cond AND closest_match_cond -``` - -Можно использовать произвольное количество условий равенства и одно условие на ближайшее совпадение. Например, `SELECT count() FROM table_1 ASOF LEFT JOIN table_2 ON table_1.a == table_2.b AND table_2.t <= table_1.t`. - -Условия, поддержанные для проверки на ближайшее совпадение: `>`, `>=`, `<`, `<=`. - -Синтаксис `ASOF JOIN ... USING`: - -``` sql -SELECT expressions_list -FROM table_1 -ASOF JOIN table_2 -USING (equi_column1, ... equi_columnN, asof_column) -``` - -Для слияния по равенству `ASOF JOIN` использует `equi_columnX`, а для слияния по ближайшему совпадению использует `asof_column` с условием `table_1.asof_column >= table_2.asof_column`. Столбец `asof_column` должен быть последним в секции `USING`. - -Например, рассмотрим следующие таблицы: - - table_1 table_2 - event | ev_time | user_id event | ev_time | user_id - ----------|---------|---------- ----------|---------|---------- - ... ... - event_1_1 | 12:00 | 42 event_2_1 | 11:59 | 42 - ... event_2_2 | 12:30 | 42 - event_1_2 | 13:00 | 42 event_2_3 | 13:00 | 42 - ... ... - -`ASOF JOIN` принимает метку времени пользовательского события из `table_1` и находит такое событие в `table_2` метка времени которого наиболее близка к метке времени события из `table_1` в соответствии с условием на ближайшее совпадение. При этом столбец `user_id` используется для объединения по равенству, а столбец `ev_time` для объединения по ближайшему совпадению. В нашем примере `event_1_1` может быть объединено с `event_2_1`, `event_1_2` может быть объединено с `event_2_3`, а `event_2_2` не объединяется. - -!!! note "Примечание" - `ASOF JOIN` не поддержан для движка таблиц [Join](../../sql-reference/statements/select.md). - -Чтобы задать значение строгости по умолчанию, используйте сессионный параметр [join\_default\_strictness](../../operations/settings/settings.md#settings-join_default_strictness). - -#### GLOBAL JOIN {#global-join} - -При использовании обычного `JOIN` , запрос отправляется на удалённые серверы. На каждом из них выполняются подзапросы для формирования «правой» таблицы, и с этой таблицей выполняется соединение. То есть, «правая» таблица формируется на каждом сервере отдельно. - -При использовании `GLOBAL ... JOIN`, сначала сервер-инициатор запроса запускает подзапрос для вычисления правой таблицы. Эта временная таблица передаётся на каждый удалённый сервер, и на них выполняются запросы с использованием переданных временных данных. - -Будьте аккуратны при использовании `GLOBAL` . За дополнительной информацией обращайтесь в раздел [Распределенные подзапросы](#select-distributed-subqueries). - -**Советы по использованию** - -Из подзапроса удаляются все столбцы, ненужные для `JOIN`. - -При запуске `JOIN`, отсутствует оптимизация порядка выполнения по отношению к другим стадиям запроса. Соединение (поиск в «правой» таблице) выполняется до фильтрации в `WHERE` и до агрегации. Чтобы явно задать порядок вычислений, рекомендуется выполнять `JOIN` подзапроса с подзапросом. - -Пример: - -``` sql -SELECT - CounterID, - hits, - visits -FROM -( - SELECT - CounterID, - count() AS hits - FROM test.hits - GROUP BY CounterID -) ANY LEFT JOIN -( - SELECT - CounterID, - sum(Sign) AS visits - FROM test.visits - GROUP BY CounterID -) USING CounterID -ORDER BY hits DESC -LIMIT 10 -``` - -``` text -┌─CounterID─┬───hits─┬─visits─┐ -│ 1143050 │ 523264 │ 13665 │ -│ 731962 │ 475698 │ 102716 │ -│ 722545 │ 337212 │ 108187 │ -│ 722889 │ 252197 │ 10547 │ -│ 2237260 │ 196036 │ 9522 │ -│ 23057320 │ 147211 │ 7689 │ -│ 722818 │ 90109 │ 17847 │ -│ 48221 │ 85379 │ 4652 │ -│ 19762435 │ 77807 │ 7026 │ -│ 722884 │ 77492 │ 11056 │ -└───────────┴────────┴────────┘ -``` - -У подзапросов нет возможности задать имена и нет возможности их использовать для того, чтобы сослаться на столбец из конкретного подзапроса. -Требуется, чтобы столбцы, указанные в `USING`, назывались одинаково в обоих подзапросах, а остальные столбцы - по-разному. Изменить имена столбцов в подзапросах можно с помощью синонимов (в примере используются синонимы hits и visits). - -В секции `USING` указывается один или несколько столбцов для соединения, что обозначает условие на равенство этих столбцов. Список столбцов задаётся без скобок. Более сложные условия соединения не поддерживаются. - -«Правая» таблица (результат подзапроса) располагается в оперативной памяти. Если её не хватает, вы не сможете выполнить `JOIN`. - -Каждый раз для выполнения запроса с одинаковым `JOIN`, подзапрос выполняется заново — результат не кэшируется. Это можно избежать, используя специальный движок таблиц [Join](../../engines/table-engines/special/join.md), представляющий собой подготовленное множество для соединения, которое всегда находится в оперативке. - -В некоторых случаях более эффективно использовать `IN` вместо `JOIN`. -Среди разных типов `JOIN`, наиболее эффективен `ANY LEFT JOIN`, следующий по эффективности `ANY INNER JOIN`. Наименее эффективны `ALL LEFT JOIN` и `ALL INNER JOIN`. - -Если `JOIN` необходим для соединения с таблицами измерений (dimension tables - сравнительно небольшие таблицы, которые содержат свойства измерений - например, имена для рекламных кампаний), то использование `JOIN` может быть не очень удобным из-за громоздкости синтаксиса, а также из-за того, что правая таблица читается заново при каждом запросе. Специально для таких случаев существует функциональность «Внешние словари», которую следует использовать вместо `JOIN`. Дополнительные сведения смотрите в разделе [Внешние словари](../../sql-reference/statements/select.md). - -**Ограничения по памяти** - -ClickHouse использует алгоритм [hash join](https://en.wikipedia.org/wiki/Hash_join). ClickHouse принимает `` и создает для него хэш-таблицу в RAM. Чтобы ограничить потребление памяти операцией `JOIN`, используйте следующие параметры: - -- [max\_rows\_in\_join](../../operations/settings/query-complexity.md#settings-max_rows_in_join) — ограничивает количество строк в хэш-таблице. -- [max\_bytes\_in\_join](../../operations/settings/query-complexity.md#settings-max_bytes_in_join) — ограничивает размер хэш-таблицы. - -По достижении любого из этих ограничений, ClickHouse действует в соответствии с настройкой [join\_overflow\_mode](../../operations/settings/query-complexity.md#settings-join_overflow_mode). - -#### Обработка пустых ячеек и NULL {#obrabotka-pustykh-iacheek-i-null} - -При слиянии таблиц могут появляться пустые ячейки. То, каким образом ClickHouse заполняет эти ячейки, определяется настройкой [join\_use\_nulls](../../operations/settings/settings.md#join_use_nulls). - -Если ключами `JOIN` выступают поля типа [Nullable](../../sql-reference/statements/select.md), то строки, где хотя бы один из ключей имеет значение [NULL](../../sql-reference/syntax.md#null-literal), не соединяются. - -#### Ограничения синтаксиса {#ogranicheniia-sintaksisa} - -Для множественных секций `JOIN` в одном запросе `SELECT`: - -- Получение всех столбцов через `*` возможно только при объединении таблиц, но не подзапросов. -- Секция `PREWHERE` недоступна. - -Для секций `ON`, `WHERE` и `GROUP BY`: - -- Нельзя использовать произвольные выражения в секциях `ON`, `WHERE`, и `GROUP BY`, однако можно определить выражение в секции `SELECT` и затем использовать его через алиас в других секциях. - -### Секция WHERE {#select-where} - -Позволяет задать выражение, которое ClickHouse использует для фильтрации данных перед всеми другими действиями в запросе кроме выражений, содержащихся в секции [PREWHERE](#select-prewhere). Обычно, это выражение с логическими операторами. - -Результат выражения должен иметь тип `UInt8`. - -ClickHouse использует в выражении индексы, если это позволяет [движок таблицы](../../sql-reference/statements/select.md). - -Если в секции необходимо проверить [NULL](../../sql-reference/syntax.md#null-literal), то используйте операторы [IS NULL](../operators.md#operator-is-null) и [IS NOT NULL](../operators.md#is-not-null), а также соответствующие функции `isNull` и `isNotNull`. В противном случае выражение будет считаться всегда не выполненным. - -Пример проверки на `NULL`: - -``` sql -SELECT * FROM t_null WHERE y IS NULL -``` - -``` text -┌─x─┬────y─┐ -│ 1 │ ᴺᵁᴸᴸ │ -└───┴──────┘ -``` - -### Секция PREWHERE {#select-prewhere} - -Имеет такой же смысл, как и секция [WHERE](#select-where). Отличие состоит в том, какие данные читаются из таблицы. -При использовании `PREWHERE`, из таблицы сначала читаются только столбцы, необходимые для выполнения `PREWHERE`. Затем читаются остальные столбцы, нужные для выполнения запроса, но из них только те блоки, в которых выражение в `PREWHERE` истинное. - -`PREWHERE` имеет смысл использовать, если есть условия фильтрации, которые использует меньшинство столбцов из тех, что есть в запросе, но достаточно сильно фильтрует данные. Таким образом, сокращается количество читаемых данных. - -Например, полезно писать `PREWHERE` для запросов, которые вынимают много столбцов, но в которых фильтрация производится лишь по нескольким столбцам. - -`PREWHERE` поддерживается только таблицами семейства `*MergeTree`. - -В запросе могут быть одновременно указаны секции `PREWHERE` и `WHERE`. В этом случае, `PREWHERE` идёт перед `WHERE`. - -Если настройка `optimize_move_to_prewhere` выставлена в `1`, то при отсутствии `PREWHERE`, система будет автоматически переносить части выражений из `WHERE` в `PREWHERE` согласно некоторой эвристике. - -### Секция GROUP BY {#select-group-by-clause} - -Это одна из наиболее важных частей СУБД. - -Секция GROUP BY, если есть, должна содержать список выражений. Каждое выражение далее будем называть «ключом». -При этом, все выражения в секциях SELECT, HAVING, ORDER BY, должны вычисляться из ключей или из агрегатных функций. То есть, каждый выбираемый из таблицы столбец, должен использоваться либо в ключах, либо внутри агрегатных функций. - -Если запрос содержит столбцы таблицы только внутри агрегатных функций, то секция GROUP BY может не указываться, и подразумевается агрегация по пустому набору ключей. - -Пример: - -``` sql -SELECT - count(), - median(FetchTiming > 60 ? 60 : FetchTiming), - count() - sum(Refresh) -FROM hits -``` - -Но, в отличие от стандартного SQL, если в таблице нет строк (вообще нет или после фильтрации с помощью WHERE), в качестве результата возвращается пустой результат, а не результат из одной строки, содержащий «начальные» значения агрегатных функций. - -В отличие от MySQL (и в соответствии со стандартом SQL), вы не можете получить какое-нибудь значение некоторого столбца, не входящего в ключ или агрегатную функцию (за исключением константных выражений). Для обхода этого вы можете воспользоваться агрегатной функцией any (получить первое попавшееся значение) или min/max. - -Пример: - -``` sql -SELECT - domainWithoutWWW(URL) AS domain, - count(), - any(Title) AS title -- getting the first occurred page header for each domain. -FROM hits -GROUP BY domain -``` - -GROUP BY вычисляет для каждого встретившегося различного значения ключей, набор значений агрегатных функций. - -Не поддерживается GROUP BY по столбцам-массивам. - -Не поддерживается указание констант в качестве аргументов агрегатных функций. Пример: sum(1). Вместо этого, вы можете избавиться от констант. Пример: `count()`. - -#### Обработка NULL {#obrabotka-null} - -При группировке, ClickHouse рассматривает [NULL](../syntax.md#null-literal) как значение, причём `NULL=NULL`. - -Рассмотрим, что это значит на примере. - -Пусть есть таблица: - -``` text -┌─x─┬────y─┐ -│ 1 │ 2 │ -│ 2 │ ᴺᵁᴸᴸ │ -│ 3 │ 2 │ -│ 3 │ 3 │ -│ 3 │ ᴺᵁᴸᴸ │ -└───┴──────┘ -``` - -В результате запроса `SELECT sum(x), y FROM t_null_big GROUP BY y` мы получим: - -``` text -┌─sum(x)─┬────y─┐ -│ 4 │ 2 │ -│ 3 │ 3 │ -│ 5 │ ᴺᵁᴸᴸ │ -└────────┴──────┘ -``` - -Видно, что `GROUP BY` для `У = NULL` просуммировал `x`, как будто `NULL` — это значение. - -Если в `GROUP BY` передать несколько ключей, то в результате мы получим все комбинации выборки, как если бы `NULL` был конкретным значением. - -#### Модификатор WITH TOTALS {#modifikator-with-totals} - -Если указан модификатор WITH TOTALS, то будет посчитана ещё одна строчка, в которой в столбцах-ключах будут содержаться значения по умолчанию (нули, пустые строки), а в столбцах агрегатных функций - значения, посчитанные по всем строкам («тотальные» значения). - -Эта дополнительная строчка выводится в форматах JSON\*, TabSeparated\*, Pretty\* отдельно от остальных строчек. В остальных форматах эта строчка не выводится. - -В форматах JSON\* строчка выводится отдельным полем totals. В форматах TabSeparated\* строчка выводится после основного результата, и перед ней (после остальных данных) вставляется пустая строка. В форматах Pretty\* строчка выводится отдельной табличкой после основного результата. - -`WITH TOTALS` может выполняться по-разному при наличии HAVING. Поведение зависит от настройки totals\_mode. -По умолчанию `totals_mode = 'before_having'`. В этом случае totals считается по всем строчкам, включая непрошедших через HAVING и max\_rows\_to\_group\_by. - -Остальные варианты учитывают в totals только строчки, прошедшие через HAVING, и имеют разное поведение при наличии настройки `max_rows_to_group_by` и `group_by_overflow_mode = 'any'`. - -`after_having_exclusive` - не учитывать строчки, не прошедшие `max_rows_to_group_by`. То есть в totals попадёт меньше или столько же строчек, чем если бы `max_rows_to_group_by` не было. - -`after_having_inclusive` - учитывать в totals все строчки, не прошедшие max\_rows\_to\_group\_by. То есть в totals попадёт больше или столько же строчек, чем если бы `max_rows_to_group_by` не было. - -`after_having_auto` - считать долю строчек, прошедших через HAVING. Если она больше некоторого значения (по умолчанию - 50%), то включить все строчки, не прошедшие max\_rows\_to\_group\_by в totals, иначе - не включить. - -`totals_auto_threshold` - по умолчанию 0.5. Коэффициент для работы `after_having_auto`. - -Если `max_rows_to_group_by` и `group_by_overflow_mode = 'any'` не используются, то все варианты вида `after_having` не отличаются, и вы можете использовать любой из них, например, `after_having_auto`. - -Вы можете использовать WITH TOTALS в подзапросах, включая подзапросы в секции JOIN (в этом случае соответствующие тотальные значения будут соединены). - -#### GROUP BY во внешней памяти {#select-group-by-in-external-memory} - -Можно включить сброс временных данных на диск, чтобы ограничить потребление оперативной памяти при выполнении `GROUP BY`. -Настройка [max\_bytes\_before\_external\_group\_by](../../operations/settings/settings.md#settings-max_bytes_before_external_group_by) определяет пороговое значение потребления RAM, по достижении которого временные данные `GROUP BY` сбрасываются в файловую систему. Если равно 0 (по умолчанию) - значит выключено. - -При использовании `max_bytes_before_external_group_by`, рекомендуем выставить `max_memory_usage` приблизительно в два раза больше. Это следует сделать, потому что агрегация выполняется в две стадии: чтение и формирование промежуточных данных (1) и слияние промежуточных данных (2). Сброс данных на файловую систему может производиться только на стадии 1. Если сброса временных данных не было, то на стадии 2 может потребляться до такого же объёма памяти, как на стадии 1. - -Например, если [max\_memory\_usage](../../operations/settings/settings.md#settings_max_memory_usage) было выставлено в 10000000000, и вы хотите использовать внешнюю агрегацию, то имеет смысл выставить `max_bytes_before_external_group_by` в 10000000000, а max\_memory\_usage в 20000000000. При срабатывании внешней агрегации (если был хотя бы один сброс временных данных в файловую систему) максимальное потребление оперативки будет лишь чуть-чуть больше `max_bytes_before_external_group_by`. - -При распределённой обработке запроса внешняя агрегация производится на удалённых серверах. Для того чтобы на сервере-инициаторе запроса использовалось немного оперативки, нужно выставить настройку `distributed_aggregation_memory_efficient` в 1. - -При слиянии данных, сброшенных на диск, а также при слиянии результатов с удалённых серверов, при включенной настройке `distributed_aggregation_memory_efficient`, потребляется до `1/256 * the_number_of_threads` количество потоков от общего объёма оперативки. - -При включенной внешней агрегации, если данных было меньше `max_bytes_before_external_group_by` (то есть сброса данных не было), то запрос работает так же быстро, как без внешней агрегации. Если же какие-то временные данные были сброшены, то время выполнения будет в несколько раз больше (примерно в три раза). - -Если есть `ORDER BY` с `LIMIT` после `GROUP BY`, то объём потребляемой RAM будет зависеть от объёма данных в `LIMIT`, а не во всей таблице. Однако, если `ORDER BY` используется без `LIMIT`, не забудьте включить внешнюю сортировку (`max_bytes_before_external_sort`). - -### Секция LIMIT BY {#sektsiia-limit-by} - -Запрос с секцией `LIMIT n BY expressions` выбирает первые `n` строк для каждого отличного значения `expressions`. Ключ `LIMIT BY` может содержать любое количество [выражений](../syntax.md#syntax-expressions). - -ClickHouse поддерживает следующий синтаксис: - -- `LIMIT [offset_value, ]n BY expressions` -- `LIMIT n OFFSET offset_value BY expressions` - -Во время обработки запроса, ClickHouse выбирает данные, упорядоченные по ключу сортировки. Ключ сортировки задаётся явно в секции [ORDER BY](#select-order-by) или неявно в свойствах движка таблицы. Затем ClickHouse применяет `LIMIT n BY expressions` и возвращает первые `n` для каждой отличной комбинации `expressions`. Если указан `OFFSET`, то для каждого блока данных, который принадлежит отдельной комбинации `expressions`, ClickHouse отступает `offset_value` строк от начала блока и возвращает не более `n`. Если `offset_value` больше, чем количество строк в блоке данных, ClickHouse не возвращает ни одной строки. - -`LIMIT BY` не связана с секцией `LIMIT`. Их можно использовать в одном запросе. - -**Примеры** - -Образец таблицы: - -``` sql -CREATE TABLE limit_by(id Int, val Int) ENGINE = Memory; -INSERT INTO limit_by values(1, 10), (1, 11), (1, 12), (2, 20), (2, 21); -``` - -Запросы: - -``` sql -SELECT * FROM limit_by ORDER BY id, val LIMIT 2 BY id -``` - -``` text -┌─id─┬─val─┐ -│ 1 │ 10 │ -│ 1 │ 11 │ -│ 2 │ 20 │ -│ 2 │ 21 │ -└────┴─────┘ -``` - -``` sql -SELECT * FROM limit_by ORDER BY id, val LIMIT 1, 2 BY id -``` - -``` text -┌─id─┬─val─┐ -│ 1 │ 11 │ -│ 1 │ 12 │ -│ 2 │ 21 │ -└────┴─────┘ -``` - -Запрос `SELECT * FROM limit_by ORDER BY id, val LIMIT 2 OFFSET 1 BY id` возвращает такой же результат. - -Следующий запрос выбирает топ 5 рефереров для каждой пары `domain, device_type`, но не более 100 строк (`LIMIT n BY + LIMIT`). - -``` sql -SELECT - domainWithoutWWW(URL) AS domain, - domainWithoutWWW(REFERRER_URL) AS referrer, - device_type, - count() cnt -FROM hits -GROUP BY domain, referrer, device_type -ORDER BY cnt DESC -LIMIT 5 BY domain, device_type -LIMIT 100 -``` - -Запрос выберет топ 5 рефереров для каждой пары `domain, device_type`, но не более 100 строк (`LIMIT n BY + LIMIT`). - -`LIMIT n BY` работает с [NULL](../syntax.md#null-literal) как если бы это было конкретное значение. Т.е. в результате запроса пользователь получит все комбинации полей, указанных в `BY`. - -### Секция HAVING {#sektsiia-having} - -Позволяет отфильтровать результат, полученный после GROUP BY, аналогично секции WHERE. -WHERE и HAVING отличаются тем, что WHERE выполняется до агрегации (GROUP BY), а HAVING - после. -Если агрегации не производится, то HAVING использовать нельзя. - -### Секция ORDER BY {#select-order-by} - -Секция ORDER BY содержит список выражений, к каждому из которых также может быть приписано DESC или ASC (направление сортировки). Если ничего не приписано - это аналогично приписыванию ASC. ASC - сортировка по возрастанию, DESC - сортировка по убыванию. Обозначение направления сортировки действует на одно выражение, а не на весь список. Пример: `ORDER BY Visits DESC, SearchPhrase` - -Для сортировки по значениям типа String есть возможность указать collation (сравнение). Пример: `ORDER BY SearchPhrase COLLATE 'tr'` - для сортировки по поисковой фразе, по возрастанию, с учётом турецкого алфавита, регистронезависимо, при допущении, что строки в кодировке UTF-8. COLLATE может быть указан или не указан для каждого выражения в ORDER BY независимо. Если есть ASC или DESC, то COLLATE указывается после них. При использовании COLLATE сортировка всегда регистронезависима. - -Рекомендуется использовать COLLATE только для окончательной сортировки небольшого количества строк, так как производительность сортировки с указанием COLLATE меньше, чем обычной сортировки по байтам. - -Строки, для которых список выражений, по которым производится сортировка, принимает одинаковые значения, выводятся в произвольном порядке, который может быть также недетерминированным (каждый раз разным). -Если секция ORDER BY отсутствует, то, аналогично, порядок, в котором идут строки, не определён, и может быть недетерминированным. - -Порядок сортировки `NaN` и `NULL`: - -- С модификатором `NULLS FIRST` — Сначала `NULL`, затем `NaN`, затем остальные значения. -- С модификатором `NULLS LAST` — Сначала значения, затем `NaN`, затем `NULL`. -- По умолчанию — Как с модификатором `NULLS LAST`. - -Пример: - -Для таблицы - -``` text -┌─x─┬────y─┐ -│ 1 │ ᴺᵁᴸᴸ │ -│ 2 │ 2 │ -│ 1 │ nan │ -│ 2 │ 2 │ -│ 3 │ 4 │ -│ 5 │ 6 │ -│ 6 │ nan │ -│ 7 │ ᴺᵁᴸᴸ │ -│ 6 │ 7 │ -│ 8 │ 9 │ -└───┴──────┘ -``` - -Выполним запрос `SELECT * FROM t_null_nan ORDER BY y NULLS FIRST`, получим: - -``` text -┌─x─┬────y─┐ -│ 1 │ ᴺᵁᴸᴸ │ -│ 7 │ ᴺᵁᴸᴸ │ -│ 1 │ nan │ -│ 6 │ nan │ -│ 2 │ 2 │ -│ 2 │ 2 │ -│ 3 │ 4 │ -│ 5 │ 6 │ -│ 6 │ 7 │ -│ 8 │ 9 │ -└───┴──────┘ -``` - -Если кроме ORDER BY указан также не слишком большой LIMIT, то расходуется меньше оперативки. Иначе расходуется количество памяти, пропорциональное количеству данных для сортировки. При распределённой обработке запроса, если отсутствует GROUP BY, сортировка частично делается на удалённых серверах, а на сервере-инициаторе запроса производится слияние результатов. Таким образом, при распределённой сортировке, может сортироваться объём данных, превышающий размер памяти на одном сервере. - -Существует возможность выполнять сортировку во внешней памяти (с созданием временных файлов на диске), если оперативной памяти не хватает. Для этого предназначена настройка `max_bytes_before_external_sort`. Если она выставлена в 0 (по умолчанию), то внешняя сортировка выключена. Если она включена, то при достижении объёмом данных для сортировки указанного количества байт, накопленные данные будут отсортированы и сброшены во временный файл. После того, как все данные будут прочитаны, будет произведено слияние всех сортированных файлов и выдача результата. Файлы записываются в директорию /var/lib/clickhouse/tmp/ (по умолчанию, может быть изменено с помощью параметра tmp\_path) в конфиге. - -На выполнение запроса может расходоваться больше памяти, чем max\_bytes\_before\_external\_sort. Поэтому, значение этой настройки должно быть существенно меньше, чем max\_memory\_usage. Для примера, если на вашем сервере 128 GB оперативки, и вам нужно выполнить один запрос, то выставите max\_memory\_usage в 100 GB, а max\_bytes\_before\_external\_sort в 80 GB. - -Внешняя сортировка работает существенно менее эффективно, чем сортировка в оперативке. - -### Секция SELECT {#select-select} - -[Выражения](../syntax.md#syntax-expressions) указанные в секции `SELECT` анализируются после завершения всех вычислений из секций, описанных выше. Вернее, анализируются выражения, стоящие над агрегатными функциями, если есть агрегатные функции. -Сами агрегатные функции и то, что под ними, вычисляются при агрегации (`GROUP BY`). Эти выражения работают так, как будто применяются к отдельным строкам результата. - -Если в результат необходимо включить все столбцы, используйте символ звёздочка (`*`). Например, `SELECT * FROM ...`. - -Чтобы включить в результат несколько столбцов, выбрав их имена с помощью регулярных выражений [re2](https://en.wikipedia.org/wiki/RE2_(software)), используйте выражение `COLUMNS`. - -``` sql -COLUMNS('regexp') -``` - -Например, рассмотрим таблицу: - -``` sql -CREATE TABLE default.col_names (aa Int8, ab Int8, bc Int8) ENGINE = TinyLog -``` - -Следующий запрос выбирает данные из всех столбцов, содержащих в имени символ `a`. - -``` sql -SELECT COLUMNS('a') FROM col_names -``` - -``` text -┌─aa─┬─ab─┐ -│ 1 │ 1 │ -└────┴────┘ -``` - -Выбранные стоблцы возвращаются не в алфавитном порядке. - -В запросе можно использовать несколько выражений `COLUMNS`, а также вызывать над ними функции. - -Например: - -``` sql -SELECT COLUMNS('a'), COLUMNS('c'), toTypeName(COLUMNS('c')) FROM col_names -``` - -``` text -┌─aa─┬─ab─┬─bc─┬─toTypeName(bc)─┐ -│ 1 │ 1 │ 1 │ Int8 │ -└────┴────┴────┴────────────────┘ -``` - -Каждый столбец, возвращённый выражением `COLUMNS`, передаётся в функцию отдельным аргументом. Также можно передавать и другие аргументы, если функция их поддерживаем. Аккуратно используйте функции. Если функция не поддерживает переданное количество аргументов, то ClickHouse генерирует исключение. - -Например: - -``` sql -SELECT COLUMNS('a') + COLUMNS('c') FROM col_names -``` - -``` text -Received exception from server (version 19.14.1): -Code: 42. DB::Exception: Received from localhost:9000. DB::Exception: Number of arguments for function plus doesn't match: passed 3, should be 2. -``` - -В этом примере, `COLUMNS('a')` возвращает два столбца: `aa` и `ab`. `COLUMNS('c')` возвращает столбец `bc`. Оператор `+` не работает с тремя аргументами, поэтому ClickHouse генерирует исключение с соответствущим сообщением. - -Столбцы, которые возвращаются выражением `COLUMNS` могут быть разных типов. Если `COLUMNS` не возвращает ни одного столбца и это единственное выражение в запросе `SELECT`, то ClickHouse генерирует исключение. - -### Секция DISTINCT {#select-distinct} - -Если указано `DISTINCT`, то из всех множеств полностью совпадающих строк результата, будет оставляться только одна строка. -Результат выполнения будет таким же, как если указано `GROUP BY` по всем указанным полям в `SELECT` и не указаны агрегатные функции. Но имеется несколько отличий от `GROUP BY`: - -- `DISTINCT` может применяться совместно с `GROUP BY`; -- при отсутствии `ORDER BY` и наличии `LIMIT`, запрос прекратит выполнение сразу после того, как будет прочитано необходимое количество различных строк - в этом случае использование DISTINCT существенно более оптимально; -- блоки данных будут выдаваться по мере их обработки, не дожидаясь выполнения всего запроса. - -`DISTINCT` не поддерживается, если в `SELECT` присутствует хотя бы один столбец типа массив. - -`DISTINCT` работает с [NULL](../syntax.md#null-literal) как если бы `NULL` был конкретным значением, причём `NULL=NULL`. Т.е. в результате `DISTINCT` разные комбинации с `NULL` встретятся только по одному разу. - -ClickHouse поддерживает использование в одном запросе секций `DISTINCT` и `ORDER BY` для разных столбцов. Секция `DISTINCT` исполняется перед секцией `ORDER BY`. - -Таблица для примера: - -``` text -┌─a─┬─b─┐ -│ 2 │ 1 │ -│ 1 │ 2 │ -│ 3 │ 3 │ -│ 2 │ 4 │ -└───┴───┘ -``` - -При выборке данных запросом `SELECT DISTINCT a FROM t1 ORDER BY b ASC`, мы получаем следующий результат: - -``` text -┌─a─┐ -│ 2 │ -│ 1 │ -│ 3 │ -└───┘ -``` - -Если изменить направление сортировки `SELECT DISTINCT a FROM t1 ORDER BY b DESC`, то результат получается следующий: - -``` text -┌─a─┐ -│ 3 │ -│ 1 │ -│ 2 │ -└───┘ -``` - -Строка `2, 4` была удалена перед сортировкой. - -Учитывайте эту особенность реализации при программировании запросов. - -### Секция LIMIT {#sektsiia-limit} - -`LIMIT m` позволяет выбрать из результата первые `m` строк. - -`LIMIT n, m` позволяет выбрать из результата первые `m` строк после пропуска первых `n` строк. Синтаксис `LIMIT m OFFSET n` также поддерживается. - -`n` и `m` должны быть неотрицательными целыми числами. - -При отсутствии секции `ORDER BY`, однозначно сортирующей результат, результат может быть произвольным и может являться недетерминированным. - -### Секция UNION ALL {#sektsiia-union-all} - -Произвольное количество запросов может быть объединено с помощью `UNION ALL`. Пример: - -``` sql -SELECT CounterID, 1 AS table, toInt64(count()) AS c - FROM test.hits - GROUP BY CounterID - -UNION ALL - -SELECT CounterID, 2 AS table, sum(Sign) AS c - FROM test.visits - GROUP BY CounterID - HAVING c > 0 -``` - -Поддерживается только `UNION ALL`. Обычный `UNION` (`UNION DISTINCT`) не поддерживается. Если вам нужен `UNION DISTINCT`, то вы можете написать `SELECT DISTINCT` из подзапроса, содержащего `UNION ALL`. - -Запросы - части `UNION ALL` могут выполняться параллельно, и их результаты могут возвращаться вперемешку. - -Структура результатов (количество и типы столбцов) у запросов должна совпадать. Но имена столбцов могут отличаться. В этом случае, имена столбцов для общего результата будут взяты из первого запроса. При объединении выполняется приведение типов. Например, если в двух объединяемых запросах одно и тоже поле имеет типы не-`Nullable` и `Nullable` от совместимого типа, то в результате `UNION ALL` получим поле типа `Nullable`. - -Запросы - части `UNION ALL` нельзя заключить в скобки. `ORDER BY` и `LIMIT` применяются к отдельным запросам, а не к общему результату. Если вам нужно применить какое-либо преобразование к общему результату, то вы можете разместить все запросы с `UNION ALL` в подзапросе в секции `FROM`. - -### Секция INTO OUTFILE {#into-outfile-clause} - -При указании `INTO OUTFILE filename` (где filename - строковый литерал), результат запроса будет сохранён в файл filename. -В отличие от MySQL, файл создаётся на стороне клиента. Если файл с таким именем уже существует, это приведёт к ошибке. -Функциональность доступна в клиенте командной строки и clickhouse-local (попытка выполнить запрос с INTO OUTFILE через HTTP интерфейс приведёт к ошибке). - -Формат вывода по умолчанию - TabSeparated, как и в не интерактивном режиме клиента командной строки. - -### Секция FORMAT {#format-clause} - -При указании FORMAT format вы можете получить данные в любом указанном формате. -Это может использоваться для удобства или для создания дампов. -Подробнее смотрите раздел «Форматы». -Если секция FORMAT отсутствует, то используется формат по умолчанию, который зависит от используемого интерфейса для доступа к БД и от настроек. Для HTTP интерфейса, а также для клиента командной строки, используемого в batch-режиме, по умолчанию используется формат TabSeparated. Для клиента командной строки, используемого в интерактивном режиме, по умолчанию используется формат PrettyCompact (прикольные таблички, компактные). - -При использовании клиента командной строки данные на клиент передаются во внутреннем эффективном формате. При этом клиент самостоятельно интерпретирует секцию FORMAT запроса и форматирует данные на своей стороне (снимая нагрузку на сеть и сервер). - -### Операторы IN {#select-in-operators} - -Операторы `IN`, `NOT IN`, `GLOBAL IN`, `GLOBAL NOT IN` рассматриваются отдельно, так как их функциональность достаточно богатая. - -В качестве левой части оператора, может присутствовать как один столбец, так и кортеж. - -Примеры: - -``` sql -SELECT UserID IN (123, 456) FROM ... -SELECT (CounterID, UserID) IN ((34, 123), (101500, 456)) FROM ... -``` - -Если слева стоит один столбец, входящий в индекс, а справа - множество констант, то при выполнении запроса, система воспользуется индексом. - -Не перечисляйте слишком большое количество значений (миллионы) явно. Если множество большое - лучше загрузить его во временную таблицу (например, смотрите раздел «Внешние данные для обработки запроса»), и затем воспользоваться подзапросом. - -В качестве правой части оператора может быть множество константных выражений, множество кортежей с константными выражениями (показано в примерах выше), а также имя таблицы или подзапрос SELECT в скобках. - -Если в качестве правой части оператора указано имя таблицы (например, `UserID IN users`), то это эквивалентно подзапросу `UserID IN (SELECT * FROM users)`. Это используется при работе с внешними данными, отправляемым вместе с запросом. Например, вместе с запросом может быть отправлено множество идентификаторов посетителей, загруженное во временную таблицу users, по которому следует выполнить фильтрацию. - -Если в качестве правой части оператора, указано имя таблицы, имеющий движок Set (подготовленное множество, постоянно находящееся в оперативке), то множество не будет создаваться заново при каждом запросе. - -В подзапросе может быть указано более одного столбца для фильтрации кортежей. -Пример: - -``` sql -SELECT (CounterID, UserID) IN (SELECT CounterID, UserID FROM ...) FROM ... -``` - -Типы столбцов слева и справа оператора IN, должны совпадать. - -Оператор IN и подзапрос могут встречаться в любой части запроса, в том числе в агрегатных и лямбда функциях. -Пример: - -``` sql -SELECT - EventDate, - avg(UserID IN - ( - SELECT UserID - FROM test.hits - WHERE EventDate = toDate('2014-03-17') - )) AS ratio -FROM test.hits -GROUP BY EventDate -ORDER BY EventDate ASC -``` - -``` text -┌──EventDate─┬────ratio─┐ -│ 2014-03-17 │ 1 │ -│ 2014-03-18 │ 0.807696 │ -│ 2014-03-19 │ 0.755406 │ -│ 2014-03-20 │ 0.723218 │ -│ 2014-03-21 │ 0.697021 │ -│ 2014-03-22 │ 0.647851 │ -│ 2014-03-23 │ 0.648416 │ -└────────────┴──────────┘ -``` - -за каждый день после 17 марта считаем долю хитов, сделанных посетителями, которые заходили на сайт 17 марта. -Подзапрос в секции IN на одном сервере всегда выполняется только один раз. Зависимых подзапросов не существует. - -#### Обработка NULL {#obrabotka-null-1} - -При обработке запроса оператор IN будет считать, что результат операции с [NULL](../syntax.md#null-literal) всегда равен `0`, независимо от того, находится `NULL` в правой или левой части оператора. Значения `NULL` не входят ни в какое множество, не соответствуют друг другу и не могут сравниваться. - -Рассмотрим для примера таблицу `t_null`: - -``` text -┌─x─┬────y─┐ -│ 1 │ ᴺᵁᴸᴸ │ -│ 2 │ 3 │ -└───┴──────┘ -``` - -При выполнении запроса `SELECT x FROM t_null WHERE y IN (NULL,3)` получим следующий результат: - -``` text -┌─x─┐ -│ 2 │ -└───┘ -``` - -Видно, что строка, в которой `y = NULL`, выброшена из результатов запроса. Это произошло потому, что ClickHouse не может решить входит ли `NULL` в множество `(NULL,3)`, возвращает результат операции `0`, а `SELECT` выбрасывает эту строку из финальной выдачи. - -``` sql -SELECT y IN (NULL, 3) -FROM t_null -``` - -``` text -┌─in(y, tuple(NULL, 3))─┐ -│ 0 │ -│ 1 │ -└───────────────────────┘ -``` - -#### Распределённые подзапросы {#select-distributed-subqueries} - -Существует два варианта IN-ов с подзапросами (аналогично для JOIN-ов): обычный `IN` / `JOIN` и `GLOBAL IN` / `GLOBAL JOIN`. Они отличаются способом выполнения при распределённой обработке запроса. - -!!! attention "Attention" - Помните, что алгоритмы, описанные ниже, могут работать иначе в зависимости от [настройки](../../operations/settings/settings.md) `distributed_product_mode`. - -При использовании обычного IN-а, запрос отправляется на удалённые серверы, и на каждом из них выполняются подзапросы в секциях `IN` / `JOIN`. - -При использовании `GLOBAL IN` / `GLOBAL JOIN-а`, сначала выполняются все подзапросы для `GLOBAL IN` / `GLOBAL JOIN-ов`, и результаты складываются во временные таблицы. Затем эти временные таблицы передаются на каждый удалённый сервер, и на них выполняются запросы, с использованием этих переданных временных данных. - -Если запрос не распределённый, используйте обычный `IN` / `JOIN`. - -Следует быть внимательным при использовании подзапросов в секции `IN` / `JOIN` в случае распределённой обработки запроса. - -Рассмотрим это на примерах. Пусть на каждом сервере кластера есть обычная таблица **local\_table**. Пусть также есть таблица **distributed\_table** типа **Distributed**, которая смотрит на все серверы кластера. - -При запросе к распределённой таблице **distributed\_table**, запрос будет отправлен на все удалённые серверы, и на них будет выполнен с использованием таблицы **local\_table**. - -Например, запрос - -``` sql -SELECT uniq(UserID) FROM distributed_table -``` - -будет отправлен на все удалённые серверы в виде - -``` sql -SELECT uniq(UserID) FROM local_table -``` - -, выполнен параллельно на каждом из них до стадии, позволяющей объединить промежуточные результаты; затем промежуточные результаты вернутся на сервер-инициатор запроса, будут на нём объединены, и финальный результат будет отправлен клиенту. - -Теперь рассмотрим запрос с IN-ом: - -``` sql -SELECT uniq(UserID) FROM distributed_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM local_table WHERE CounterID = 34) -``` - -- расчёт пересечения аудиторий двух сайтов. - -Этот запрос будет отправлен на все удалённые серверы в виде - -``` sql -SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM local_table WHERE CounterID = 34) -``` - -То есть, множество в секции IN будет собрано на каждом сервере независимо, только по тем данным, которые есть локально на каждом из серверов. - -Это будет работать правильно и оптимально, если вы предусмотрели такой случай, и раскладываете данные по серверам кластера таким образом, чтобы данные одного UserID-а лежали только на одном сервере. В таком случае все необходимые данные будут присутствовать на каждом сервере локально. В противном случае результат будет посчитан неточно. Назовём этот вариант запроса «локальный IN». - -Чтобы исправить работу запроса, когда данные размазаны по серверам кластера произвольным образом, можно было бы указать **distributed\_table** внутри подзапроса. Запрос будет выглядеть так: - -``` sql -SELECT uniq(UserID) FROM distributed_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM distributed_table WHERE CounterID = 34) -``` - -Этот запрос будет отправлен на все удалённые серверы в виде - -``` sql -SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM distributed_table WHERE CounterID = 34) -``` - -На каждом удалённом сервере начнёт выполняться подзапрос. Так как в подзапросе используется распределённая таблица, то подзапрос будет, на каждом удалённом сервере, снова отправлен на каждый удалённый сервер, в виде - -``` sql -SELECT UserID FROM local_table WHERE CounterID = 34 -``` - -Например, если у вас кластер из 100 серверов, то выполнение всего запроса потребует 10 000 элементарных запросов, что, как правило, является неприемлемым. - -В таких случаях всегда следует использовать GLOBAL IN вместо IN. Рассмотрим его работу для запроса - -``` sql -SELECT uniq(UserID) FROM distributed_table WHERE CounterID = 101500 AND UserID GLOBAL IN (SELECT UserID FROM distributed_table WHERE CounterID = 34) -``` - -На сервере-инициаторе запроса будет выполнен подзапрос - -``` sql -SELECT UserID FROM distributed_table WHERE CounterID = 34 -``` - -, и результат будет сложен во временную таблицу в оперативке. Затем запрос будет отправлен на каждый удалённый сервер в виде - -``` sql -SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID GLOBAL IN _data1 -``` - -, и вместе с запросом, на каждый удалённый сервер будет отправлена временная таблица `_data1` (имя временной таблицы - implementation defined). - -Это гораздо более оптимально, чем при использовании обычного IN. Но при этом, следует помнить о нескольких вещах: - -1. При создании временной таблицы данные не уникализируются. Чтобы уменьшить объём передаваемых по сети данных, укажите в подзапросе DISTINCT (для обычного IN-а этого делать не нужно). -2. Временная таблица будет передана на все удалённые серверы. Передача не учитывает топологию сети. Например, если 10 удалённых серверов расположены в удалённом относительно сервера-инициатора запроса дата-центре, то по каналу в удалённый дата-центр данные будет переданы 10 раз. Старайтесь не использовать большие множества при использовании GLOBAL IN. -3. При передаче данных на удалённые серверы не настраивается ограничение использования сетевой полосы. Вы можете перегрузить сеть. -4. Старайтесь распределять данные по серверам так, чтобы в GLOBAL IN-ах не было частой необходимости. -5. Если в GLOBAL IN есть частая необходимость, то спланируйте размещение кластера ClickHouse таким образом, чтобы в каждом дата-центре была хотя бы одна реплика каждого шарда, и среди них была быстрая сеть - чтобы запрос целиком можно было бы выполнить, передавая данные в пределах одного дата-центра. - -В секции `GLOBAL IN` также имеет смысл указывать локальную таблицу - в случае, если эта локальная таблица есть только на сервере-инициаторе запроса, и вы хотите воспользоваться данными из неё на удалённых серверах. - -### Экстремальные значения {#ekstremalnye-znacheniia} - -Вы можете получить в дополнение к результату также минимальные и максимальные значения по столбцам результата. Для этого выставите настройку **extremes** в 1. Минимумы и максимумы считаются для числовых типов, дат, дат-с-временем. Для остальных столбцов будут выведены значения по умолчанию. - -Вычисляются дополнительные две строчки - минимумы и максимумы, соответственно. Эти две дополнительные строки выводятся в [форматах](../../interfaces/formats.md) `JSON*`, `TabSeparated*`, и `Pretty*` отдельно от остальных строчек. В остальных форматах они не выводится. - -Во форматах `JSON*`, экстремальные значения выводятся отдельным полем ‘extremes’. В форматах `TabSeparated*`, строка выводится после основного результата и после ‘totals’ если есть. Перед ней (после остальных данных) вставляется пустая строка. В форматах `Pretty*`, строка выводится отдельной таблицей после основного результата и после `totals` если есть. - -Экстремальные значения вычисляются для строк перед `LIMIT`, но после `LIMIT BY`. Однако при использовании `LIMIT offset, size`, строки перед `offset` включаются в `extremes`. В потоковых запросах, в результате может учитываться также небольшое количество строчек, прошедших `LIMIT`. - -### Замечания {#zamechaniia} - -В секциях `GROUP BY`, `ORDER BY`, в отличие от диалекта MySQL, и в соответствии со стандартным SQL, не поддерживаются позиционные аргументы. -Например, если вы напишите `GROUP BY 1, 2` - то это будет воспринято, как группировка по константам (то есть, агрегация всех строк в одну). - -Вы можете использовать синонимы (алиасы `AS`) в любом месте запроса. - -В любом месте запроса, вместо выражения, может стоять звёздочка. При анализе запроса звёздочка раскрывается в список всех столбцов таблицы (за исключением `MATERIALIZED` и `ALIAS` столбцов). Есть лишь немного случаев, когда оправдано использовать звёздочку: - -- при создании дампа таблицы; -- для таблиц, содержащих всего несколько столбцов - например, системных таблиц; -- для получения информации о том, какие столбцы есть в таблице; в этом случае, укажите `LIMIT 1`. Но лучше используйте запрос `DESC TABLE`; -- при наличии сильной фильтрации по небольшому количеству столбцов с помощью `PREWHERE`; -- в подзапросах (так как из подзапросов выкидываются столбцы, не нужные для внешнего запроса). - -В других случаях использование звёздочки является издевательством над системой, так как вместо преимуществ столбцовой СУБД вы получаете недостатки. То есть использовать звёздочку не рекомендуется. - -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/select/) diff --git a/docs/ru/sql-reference/statements/select/array-join.md b/docs/ru/sql-reference/statements/select/array-join.md new file mode 100644 index 00000000000..f8f11ba1b17 --- /dev/null +++ b/docs/ru/sql-reference/statements/select/array-join.md @@ -0,0 +1,277 @@ +# Секция ARRAY JOIN {#select-array-join-clause} + +Типовая операция для таблиц, содержащих столбец-массив — произвести новую таблицу, которая будет иметь столбец с каждым отдельным элементом массивов из изначального столбца, в то время как значения других столбцов дублируются. Это основной сценарий использования секции `ARRAY JOIN`. + +Название этой секции происходит от того, что эту операцию можно рассматривать как исполняющий `JOIN` с массивом или вложенной структурой данных. Цель использования похожа на функцию [arrayJoin](../../functions/array-join.md#functions_arrayjoin), но функциональность секции шире. + +Синтаксис: + +``` sql +SELECT +FROM +[LEFT] ARRAY JOIN +[WHERE|PREWHERE ] +... +``` + +Вы можете указать только одну секцию `ARRAY JOIN` в `SELECT` запросе. + +Поддерживаемые виды `ARRAY JOIN` перечислены ниже: + +- `ARRAY JOIN` - В базовом случае пустые массивы не включаются в результат. +- `LEFT ARRAY JOIN` - Результат содержит строки с пустыми массивами. Значение для пустого массива устанавливается равным значению по умолчанию для типа элемента массива (обычно 0, пустая строка или NULL). + +## Базовые примеры ARRAY JOIN {#basic-array-join-examples} + +Приведенные ниже примеры демонстрируют использование `ARRAY JOIN` и `LEFT ARRAY JOIN`. Cоздадим таблицу с колонкой типа данных [Array](../../../sql-reference/data-types/array.md) и вставим в него значения: + +``` sql +CREATE TABLE arrays_test +( + s String, + arr Array(UInt8) +) ENGINE = Memory; + +INSERT INTO arrays_test +VALUES ('Hello', [1,2]), ('World', [3,4,5]), ('Goodbye', []); +``` + +``` text +┌─s───────────┬─arr─────┐ +│ Hello │ [1,2] │ +│ World │ [3,4,5] │ +│ Goodbye │ [] │ +└─────────────┴─────────┘ +``` + +В приведенном ниже примере используется секция `ARRAY JOIN`: + +``` sql +SELECT s, arr +FROM arrays_test +ARRAY JOIN arr; +``` + +``` text +┌─s─────┬─arr─┐ +│ Hello │ 1 │ +│ Hello │ 2 │ +│ World │ 3 │ +│ World │ 4 │ +│ World │ 5 │ +└───────┴─────┘ +``` + +В следующем примере используется `LEFT ARRAY JOIN`: + +``` sql +SELECT s, arr +FROM arrays_test +LEFT ARRAY JOIN arr; +``` + +``` text +┌─s───────────┬─arr─┐ +│ Hello │ 1 │ +│ Hello │ 2 │ +│ World │ 3 │ +│ World │ 4 │ +│ World │ 5 │ +│ Goodbye │ 0 │ +└─────────────┴─────┘ +``` + +## Использование алиасов {#using-aliases} + +В секции `ARRAY JOIN` может быть указан алиас для массива. В этом случае элемент массива доступен по этому алиас, а сам массив доступен по исходному имени. Пример: + +``` sql +SELECT s, arr, a +FROM arrays_test +ARRAY JOIN arr AS a; +``` + +``` text +┌─s─────┬─arr─────┬─a─┐ +│ Hello │ [1,2] │ 1 │ +│ Hello │ [1,2] │ 2 │ +│ World │ [3,4,5] │ 3 │ +│ World │ [3,4,5] │ 4 │ +│ World │ [3,4,5] │ 5 │ +└───────┴─────────┴───┘ +``` + +Используя псевдонимы, вы можете выполнять `ARRAY JOIN` с внешними массивами. Например: + +``` sql +SELECT s, arr_external +FROM arrays_test +ARRAY JOIN [1, 2, 3] AS arr_external; +``` + +``` text +┌─s───────────┬─arr_external─┐ +│ Hello │ 1 │ +│ Hello │ 2 │ +│ Hello │ 3 │ +│ World │ 1 │ +│ World │ 2 │ +│ World │ 3 │ +│ Goodbye │ 1 │ +│ Goodbye │ 2 │ +│ Goodbye │ 3 │ +└─────────────┴──────────────┘ +``` + +Несколько массивов могут быть перечислены через запятую в секции `ARRAY JOIN`. В этом случае, `JOIN` выполняется с ними одновременно (прямая сумма, а не декартово произведение). Обратите внимание, что все массивы должны иметь одинаковый размер. Пример: + +``` sql +SELECT s, arr, a, num, mapped +FROM arrays_test +ARRAY JOIN arr AS a, arrayEnumerate(arr) AS num, arrayMap(x -> x + 1, arr) AS mapped; +``` + +``` text +┌─s─────┬─arr─────┬─a─┬─num─┬─mapped─┐ +│ Hello │ [1,2] │ 1 │ 1 │ 2 │ +│ Hello │ [1,2] │ 2 │ 2 │ 3 │ +│ World │ [3,4,5] │ 3 │ 1 │ 4 │ +│ World │ [3,4,5] │ 4 │ 2 │ 5 │ +│ World │ [3,4,5] │ 5 │ 3 │ 6 │ +└───────┴─────────┴───┴─────┴────────┘ +``` + +В приведенном ниже примере используется функция [arrayEnumerate](../../../sql-reference/functions/array-functions.md#array_functions-arrayenumerate): + +``` sql +SELECT s, arr, a, num, arrayEnumerate(arr) +FROM arrays_test +ARRAY JOIN arr AS a, arrayEnumerate(arr) AS num; +``` + +``` text +┌─s─────┬─arr─────┬─a─┬─num─┬─arrayEnumerate(arr)─┐ +│ Hello │ [1,2] │ 1 │ 1 │ [1,2] │ +│ Hello │ [1,2] │ 2 │ 2 │ [1,2] │ +│ World │ [3,4,5] │ 3 │ 1 │ [1,2,3] │ +│ World │ [3,4,5] │ 4 │ 2 │ [1,2,3] │ +│ World │ [3,4,5] │ 5 │ 3 │ [1,2,3] │ +└───────┴─────────┴───┴─────┴─────────────────────┘ +``` + +## ARRAY JOIN со вложенной структурой данных {#array-join-with-nested-data-structure} + +`ARRAY JOIN` также работает с [вложенными структурами данных](../../../sql-reference/data-types/nested-data-structures/nested.md): + +``` sql +CREATE TABLE nested_test +( + s String, + nest Nested( + x UInt8, + y UInt32) +) ENGINE = Memory; + +INSERT INTO nested_test +VALUES ('Hello', [1,2], [10,20]), ('World', [3,4,5], [30,40,50]), ('Goodbye', [], []); +``` + +``` text +┌─s───────┬─nest.x──┬─nest.y─────┐ +│ Hello │ [1,2] │ [10,20] │ +│ World │ [3,4,5] │ [30,40,50] │ +│ Goodbye │ [] │ [] │ +└─────────┴─────────┴────────────┘ +``` + +``` sql +SELECT s, `nest.x`, `nest.y` +FROM nested_test +ARRAY JOIN nest; +``` + +``` text +┌─s─────┬─nest.x─┬─nest.y─┐ +│ Hello │ 1 │ 10 │ +│ Hello │ 2 │ 20 │ +│ World │ 3 │ 30 │ +│ World │ 4 │ 40 │ +│ World │ 5 │ 50 │ +└───────┴────────┴────────┘ +``` + +При указании имен вложенных структур данных в `ARRAY JOIN`, секция работает так же, как и `ARRAY JOIN` со всеми элементами массива, из которых он состоит. Примеры приведены ниже: + +``` sql +SELECT s, `nest.x`, `nest.y` +FROM nested_test +ARRAY JOIN `nest.x`, `nest.y`; +``` + +``` text +┌─s─────┬─nest.x─┬─nest.y─┐ +│ Hello │ 1 │ 10 │ +│ Hello │ 2 │ 20 │ +│ World │ 3 │ 30 │ +│ World │ 4 │ 40 │ +│ World │ 5 │ 50 │ +└───────┴────────┴────────┘ +``` + +Такая вариация также возможна: + +``` sql +SELECT s, `nest.x`, `nest.y` +FROM nested_test +ARRAY JOIN `nest.x`; +``` + +``` text +┌─s─────┬─nest.x─┬─nest.y─────┐ +│ Hello │ 1 │ [10,20] │ +│ Hello │ 2 │ [10,20] │ +│ World │ 3 │ [30,40,50] │ +│ World │ 4 │ [30,40,50] │ +│ World │ 5 │ [30,40,50] │ +└───────┴────────┴────────────┘ +``` + +Алиас для вложенной структуры данных можно использовать, чтобы выбрать как результат JOIN-а, так и исходный массив. Пример: + +``` sql +SELECT s, `n.x`, `n.y`, `nest.x`, `nest.y` +FROM nested_test +ARRAY JOIN nest AS n; +``` + +``` text +┌─s─────┬─n.x─┬─n.y─┬─nest.x──┬─nest.y─────┐ +│ Hello │ 1 │ 10 │ [1,2] │ [10,20] │ +│ Hello │ 2 │ 20 │ [1,2] │ [10,20] │ +│ World │ 3 │ 30 │ [3,4,5] │ [30,40,50] │ +│ World │ 4 │ 40 │ [3,4,5] │ [30,40,50] │ +│ World │ 5 │ 50 │ [3,4,5] │ [30,40,50] │ +└───────┴─────┴─────┴─────────┴────────────┘ +``` + +Пример использования функции [arrayEnumerate](../../../sql-reference/functions/array-functions.md#array_functions-arrayenumerate): + +``` sql +SELECT s, `n.x`, `n.y`, `nest.x`, `nest.y`, num +FROM nested_test +ARRAY JOIN nest AS n, arrayEnumerate(`nest.x`) AS num; +``` + +``` text +┌─s─────┬─n.x─┬─n.y─┬─nest.x──┬─nest.y─────┬─num─┐ +│ Hello │ 1 │ 10 │ [1,2] │ [10,20] │ 1 │ +│ Hello │ 2 │ 20 │ [1,2] │ [10,20] │ 2 │ +│ World │ 3 │ 30 │ [3,4,5] │ [30,40,50] │ 1 │ +│ World │ 4 │ 40 │ [3,4,5] │ [30,40,50] │ 2 │ +│ World │ 5 │ 50 │ [3,4,5] │ [30,40,50] │ 3 │ +└───────┴─────┴─────┴─────────┴────────────┴─────┘ +``` + +## Детали реализации {#implementation-details} + +Хотя секция `ARRAY JOIN` всегда должна быть указана перед [WHERE](where.md)/[PREWHERE](prewhere.md), технически они могут быть выполнены в любом порядке, если только результат `ARRAY JOIN` используется для фильтрации. Порядок обработки контролируется оптимизатором запросов. diff --git a/docs/ru/sql-reference/statements/select/distinct.md b/docs/ru/sql-reference/statements/select/distinct.md new file mode 100644 index 00000000000..62e2e25b7e5 --- /dev/null +++ b/docs/ru/sql-reference/statements/select/distinct.md @@ -0,0 +1,58 @@ +# Секция DISTINCT {#select-distinct} + +Если указан `SELECT DISTINCT`, то в результате запроса останутся только уникальные строки. Таким образом, из всех наборов полностью совпадающих строк в результате останется только одна строка. + +## Обработк NULL {#null-processing} + +`DISTINCT` работает с [NULL](../../syntax.md#null-literal) как-будто `NULL` — обычное значение и `NULL==NULL`. Другими словами, в результате `DISTINCT`, различные комбинации с `NULL` встретятся только один раз. Это отличается от обработки `NULL` в большинстве других контекстов. + +## Альтернативы {#alternatives} + +Такой же результат можно получить, применив секцию [GROUP BY](group-by.md) для того же набора значений, которые указан в секции `SELECT`, без использования каких-либо агрегатных функций. Но есть от `GROUP BY` несколько отличий: + +- `DISTINCT` может применяться вместе с `GROUP BY`. +- Когда секция [ORDER BY](order-by.md) опущена, а секция [LIMIT](limit.md) присутствует, запрос прекращает выполнение сразу после считывания необходимого количества различных строк. +- Блоки данных выводятся по мере их обработки, не дожидаясь завершения выполнения всего запроса. + +## Ограничения {#limitations} + +`DISTINCT` не поддерживается, если `SELECT` имеет по крайней мере один столбец-массив. + +## Примеры {#examples} + +ClickHouse поддерживает использование секций `DISTINCT` и `ORDER BY` для разных столбцов в одном запросе. Секция `DISTINCT` выполняется до секции `ORDER BY`. + +Таблица для примера: + +``` text +┌─a─┬─b─┐ +│ 2 │ 1 │ +│ 1 │ 2 │ +│ 3 │ 3 │ +│ 2 │ 4 │ +└───┴───┘ +``` + +При выборе данных с помощью `SELECT DISTINCT a FROM t1 ORDER BY b ASC`, мы получаем следующий результат: + +``` text +┌─a─┐ +│ 2 │ +│ 1 │ +│ 3 │ +└───┘ +``` + +Если мы изменим направление сортировки `SELECT DISTINCT a FROM t1 ORDER BY b DESC`, мы получаем следующий результат: + +``` text +┌─a─┐ +│ 3 │ +│ 1 │ +│ 2 │ +└───┘ +``` + +Ряд `2, 4` был разрезан перед сортировкой. + +Учитывайте эту специфику при разработке запросов. diff --git a/docs/ru/sql-reference/statements/select/format.md b/docs/ru/sql-reference/statements/select/format.md new file mode 100644 index 00000000000..dad0ef0d62c --- /dev/null +++ b/docs/ru/sql-reference/statements/select/format.md @@ -0,0 +1,13 @@ +# Секция FORMAT {#format-clause} + +ClickHouse поддерживает широкий спектр [форматов сериализации](../../../interfaces/formats.md) это может быть использовано, в частности, для результатов запросов. Существует несколько способов выбора формата для `SELECT`, один из них заключается в том, чтобы указать `FORMAT format` в конце запроса, чтобы получить результирующие данные в любом конкретном формате. + +Определенный формат может использоваться для удобства, интеграции с другими системами или для повышения производительности. + +## Формат по умолчанию {#default-format} + +Если `FORMAT` предложение опущено, используется формат по умолчанию, который зависит как от настроек, так и от интерфейса, используемого для доступа к серверу ClickHouse. Для [HTTP интерфейса](../../../interfaces/http.md) и [клиента командной строки](../../../interfaces/cli.md) в пакетном режиме, формат по умолчанию — `TabSeparated`. Для клиента командной строки в интерактивном режиме по умолчанию используется формат `PrettyCompact` (он производит компактные человекочитаемые таблицы). + +## Детали реализации {#implementation-details} + +При использовании клиента командной строки данные всегда передаются по сети во внутреннем эффективном формате (`Native`). Клиент самостоятельно интерпретирует `FORMAT` предложение запроса и форматирует сами данные (тем самым освобождая сеть и сервер от дополнительной нагрузки). diff --git a/docs/ru/sql-reference/statements/select/from.md b/docs/ru/sql-reference/statements/select/from.md new file mode 100644 index 00000000000..ac0ab1dcd3f --- /dev/null +++ b/docs/ru/sql-reference/statements/select/from.md @@ -0,0 +1,40 @@ +# Секция FROM {#select-from} + +В секции `FROM` указывается источник, из которого будут читаться данные: + +- [Таблица](../../../engines/table-engines/index.md) +- [Подзапрос](index.md) {## TODO: better link ##} +- [Табличная функция](../../table-functions/index.md#table-functions) + +Секция [JOIN](join.md) и [ARRAY JOIN](array-join.md) могут быть использованы для расширения функциональных возможностей секции `FROM`. + +Подзапрос — дополнительный `SELECT` запрос, который может быть указан в круглых скобках внутри секции `FROM`. + +Секция `FROM` может содержать несколько источников данных, указанных через запятую, что эквивалентно выполнению [CROSS JOIN](join.md) на них. + +## Модификатор FINAL {#select-from-final} + +Если в запросе используется модификатор `FINAL`, то ClickHouse полностью мёржит данные перед выдачей результата, таким образом выполняя все преобразования данных, которые производятся движком таблиц при мёржах. + +Он применим при выборе данных из таблиц, использующих [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md)- семейство движков (кроме `GraphiteMergeTree`). Также поддерживается для: + +- [Replicated](../../../engines/table-engines/mergetree-family/replication.md) варианты исполнения `MergeTree` движков. +- [View](../../../engines/table-engines/special/view.md), [Buffer](../../../engines/table-engines/special/buffer.md), [Distributed](../../../engines/table-engines/special/distributed.md), и [MaterializedView](../../../engines/table-engines/special/materializedview.md), которые работают поверх других движков, если они созданы для таблиц с движками семейства `MergeTree`. + +### Недостатки {#drawbacks} + +Запросы, которые используют `FINAL` выполняются не так быстро, как аналогичные запросы без него, потому что: + +- Запрос выполняется в одном потоке, и данные мёржатся во время выполнения запроса. +- Запросы с модификатором `FINAL` читают столбцы первичного ключа в дополнение к столбцам, используемым в запросе. + +**В большинстве случаев избегайте использования `FINAL`.** Общий подход заключается в использовании агрегирующих запросов, которые предполагают, что фоновые процессы движков семейства `MergeTree` ещё не случились (например, сами отбрасывают дубликаты). {## TODO: examples ##} + +## Детали реализации {#implementation-details} + +Если секция `FROM` опущена, данные будут считываться из таблицы `system.one`. +Таблица `system.one` содержит ровно одну строку. + +Для выполнения запроса, из соответствующей таблицы, вынимаются все столбцы, перечисленные в запросе. Из подзапросов выкидываются столбцы, не нужные для внешнего запроса. + +Если в запросе не перечислено ни одного столбца (например, `SELECT count() FROM t)`, то из таблицы всё равно вынимается один какой-нибудь столбец (предпочитается самый маленький), для того, чтобы можно было посчитать количество строк. diff --git a/docs/ru/sql-reference/statements/select/group-by.md b/docs/ru/sql-reference/statements/select/group-by.md new file mode 100644 index 00000000000..f68a8547f13 --- /dev/null +++ b/docs/ru/sql-reference/statements/select/group-by.md @@ -0,0 +1,119 @@ +# Секция GROUP BY {#select-group-by-clause} + +Секция `GROUP BY` переключает `SELECT` запрос в режим агрегации, который работает следующим образом: + +- Секция `GROUP BY` содержит список выражений (или одно выражение, которое считается списком длины один). Этот список действует как «ключ группировки», в то время как каждое отдельное выражение будет называться «ключевым выражением». +- Все выражения в секциях [SELECT](index.md), [HAVING](having.md), и [ORDER BY](order-by.md) статьи **должны** быть вычисленными на основе ключевых выражений **или** на [агрегатных функций](../../../sql-reference/aggregate-functions/index.md) над неключевыми выражениями (включая столбцы). Другими словами, каждый столбец, выбранный из таблицы, должен использоваться либо в ключевом выражении, либо внутри агрегатной функции, но не в обоих. +- В результате агрегирования `SELECT` запрос будет содержать столько строк, сколько было уникальных значений ключа группировки в исходной таблице. Обычно агрегация значительно уменьшает количество строк, часто на порядки, но не обязательно: количество строк остается неизменным, если все исходные значения ключа группировки ценности были различны. + +!!! note "Примечание" + Есть ещё один способ запустить агрегацию по таблице. Если запрос содержит столбцы исходной таблицы только внутри агрегатных функций, то `GROUP BY` секцию можно опустить, и предполагается агрегирование по пустому набору ключей. Такие запросы всегда возвращают ровно одну строку. + +## Обработка NULL {#null-processing} + +При агрегации ClickHouse интерпретирует [NULL](../../syntax.md#null-literal) как обычное значение, то есть `NULL==NULL`. Это отличается от обработки `NULL` в большинстве других контекстов. + +Предположим, что у вас есть эта таблица: + +``` text +┌─x─┬────y─┐ +│ 1 │ 2 │ +│ 2 │ ᴺᵁᴸᴸ │ +│ 3 │ 2 │ +│ 3 │ 3 │ +│ 3 │ ᴺᵁᴸᴸ │ +└───┴──────┘ +``` + +Запрос `SELECT sum(x), y FROM t_null_big GROUP BY y` выведет: + +``` text +┌─sum(x)─┬────y─┐ +│ 4 │ 2 │ +│ 3 │ 3 │ +│ 5 │ ᴺᵁᴸᴸ │ +└────────┴──────┘ +``` + +Видно, что `GROUP BY` для `У = NULL` просуммировал `x`, как будто `NULL` — это значение. + +Если в `GROUP BY` передать несколько ключей, то в результате мы получим все комбинации выборки, как если бы `NULL` был конкретным значением. + +## Модификатор WITH TOTALS {#with-totals-modifier} + +Если указан модификатор `WITH TOTALS`, то будет посчитана ещё одна строчка, в которой в столбцах-ключах будут содержаться значения по умолчанию (нули, пустые строки), а в столбцах агрегатных функций - значения, посчитанные по всем строкам («тотальные» значения). + +Этот дополнительный ряд выводится только в форматах `JSON*`, `TabSeparated*`, и `Pretty*`, отдельно от других строк: + +- В `JSON*` форматах, эта строка выводится как отдельное поле ‘totals’. +- В `TabSeparated*` форматах, строка идет после основного результата, через дополнительную пустую строку (после остальных данных). +- В `Pretty*` форматах, строка выводится в виде отдельной таблицы после основного результата. +- В других форматах она не доступна. + +При использовании секции [HAVING](having.md) поведение `WITH TOTALS` контролируется настройкой `totals_mode`. + +### Настройка обработки итогов {#configuring-totals-processing} + +По умолчанию `totals_mode = 'before_having'`. В этом случае totals считается по всем строчкам, включая непрошедших через HAVING и max\_rows\_to\_group\_by. + +Остальные варианты учитывают в totals только строчки, прошедшие через HAVING, и имеют разное поведение при наличии настройки `max_rows_to_group_by` и `group_by_overflow_mode = 'any'`. + +`after_having_exclusive` - не учитывать строчки, не прошедшие `max_rows_to_group_by`. То есть в totals попадёт меньше или столько же строчек, чем если бы `max_rows_to_group_by` не было. + +`after_having_inclusive` - учитывать в totals все строчки, не прошедшие max\_rows\_to\_group\_by. То есть в totals попадёт больше или столько же строчек, чем если бы `max_rows_to_group_by` не было. + +`after_having_auto` - считать долю строчек, прошедших через HAVING. Если она больше некоторого значения (по умолчанию - 50%), то включить все строчки, не прошедшие max\_rows\_to\_group\_by в totals, иначе - не включить. + +`totals_auto_threshold` - по умолчанию 0.5. Коэффициент для работы `after_having_auto`. + +Если `max_rows_to_group_by` и `group_by_overflow_mode = 'any'` не используются, то все варианты вида `after_having` не отличаются, и вы можете использовать любой из них, например, `after_having_auto`. + +Вы можете использовать `WITH TOTALS` в подзапросах, включая подзапросы в секции [JOIN](join.md) (в этом случае соответствующие тотальные значения будут соединены). + +## Примеры {#examples} + +Пример: + +``` sql +SELECT + count(), + median(FetchTiming > 60 ? 60 : FetchTiming), + count() - sum(Refresh) +FROM hits +``` + +Но, в отличие от стандартного SQL, если в таблице нет строк (вообще нет или после фильтрации с помощью WHERE), в качестве результата возвращается пустой результат, а не результат из одной строки, содержащий «начальные» значения агрегатных функций. + +В отличие от MySQL (и в соответствии со стандартом SQL), вы не можете получить какое-нибудь значение некоторого столбца, не входящего в ключ или агрегатную функцию (за исключением константных выражений). Для обхода этого вы можете воспользоваться агрегатной функцией any (получить первое попавшееся значение) или min/max. + +Пример: + +``` sql +SELECT + domainWithoutWWW(URL) AS domain, + count(), + any(Title) AS title -- getting the first occurred page header for each domain. +FROM hits +GROUP BY domain +``` + +GROUP BY вычисляет для каждого встретившегося различного значения ключей, набор значений агрегатных функций. + +Не поддерживается GROUP BY по столбцам-массивам. + +Не поддерживается указание констант в качестве аргументов агрегатных функций. Пример: `sum(1)`. Вместо этого, вы можете избавиться от констант. Пример: `count()`. + +## Детали реализации {#implementation-details} + +Агрегация является одной из наиболее важных возможностей столбцовых СУБД, и поэтому её реализация является одной из наиболее сильно оптимизированных частей ClickHouse. По умолчанию агрегирование выполняется в памяти с помощью хэш-таблицы. Она имеет более 40 специализаций, которые выбираются автоматически в зависимости от типов данных ключа группировки. + +### Группировка во внешней памяти {#select-group-by-in-external-memory} + +Можно включить сброс временных данных на диск, чтобы ограничить потребление оперативной памяти при выполнении `GROUP BY`. +Настройка [max\_bytes\_before\_external\_group\_by](../../../operations/settings/settings.md#settings-max_bytes_before_external_group_by) определяет пороговое значение потребления RAM, по достижении которого временные данные `GROUP BY` сбрасываются в файловую систему. Если равно 0 (по умолчанию) - значит выключено. + +При использовании `max_bytes_before_external_group_by`, рекомендуем выставить `max_memory_usage` приблизительно в два раза больше. Это следует сделать, потому что агрегация выполняется в две стадии: чтение и формирование промежуточных данных (1) и слияние промежуточных данных (2). Сброс данных на файловую систему может производиться только на стадии 1. Если сброса временных данных не было, то на стадии 2 может потребляться до такого же объёма памяти, как на стадии 1. + +Например, если [max\_memory\_usage](../../../operations/settings/settings.md#settings_max_memory_usage) было выставлено в 10000000000, и вы хотите использовать внешнюю агрегацию, то имеет смысл выставить `max_bytes_before_external_group_by` в 10000000000, а `max_memory_usage` в 20000000000. При срабатывании внешней агрегации (если был хотя бы один сброс временных данных в файловую систему) максимальное потребление оперативки будет лишь чуть-чуть больше `max_bytes_before_external_group_by`. + +При распределённой обработке запроса внешняя агрегация производится на удалённых серверах. Для того чтобы на сервере-инициаторе запроса использовалось немного оперативки, нужно выставить настройку `distributed_aggregation_memory_efficient` в 1. diff --git a/docs/ru/sql-reference/statements/select/having.md b/docs/ru/sql-reference/statements/select/having.md new file mode 100644 index 00000000000..83f58c5566f --- /dev/null +++ b/docs/ru/sql-reference/statements/select/having.md @@ -0,0 +1,9 @@ +# Секция HAVING {#having-clause} + +Позволяет фильтровать результаты агрегации, полученные с помощью [GROUP BY](group-by.md). Разница с [WHERE](where.md) в том, что `WHERE` выполняется перед агрегацией, в то время как `HAVING` выполняется после него. + +Из секции `HAVING` можно ссылаться на результаты агреации из секции `SELECT` по их алиасу. Также секция `HAVING` может фильтровать по результатам дополнительных агрегатов, которые не возвращаются в результатах запроса. + +## Ограничения {#limitations} + +`HAVING` нельзя использовать, если агрегация не выполняется. Вместо этого можно использовать `WHERE`. diff --git a/docs/ru/sql-reference/statements/select/index.md b/docs/ru/sql-reference/statements/select/index.md new file mode 100644 index 00000000000..fb7c130983b --- /dev/null +++ b/docs/ru/sql-reference/statements/select/index.md @@ -0,0 +1,162 @@ +--- +toc_priority: 33 +toc_title: SELECT +--- + +# Синтаксис запросов SELECT {#select-queries-syntax} + +`SELECT` выполняет получение данных. + +``` sql +[WITH expr_list|(subquery)] +SELECT [DISTINCT] expr_list +[FROM [db.]table | (subquery) | table_function] [FINAL] +[SAMPLE sample_coeff] +[ARRAY JOIN ...] +[GLOBAL] [ANY|ALL] [INNER|LEFT|RIGHT|FULL|CROSS] [OUTER] JOIN (subquery)|table USING columns_list +[PREWHERE expr] +[WHERE expr] +[GROUP BY expr_list] [WITH TOTALS] +[HAVING expr] +[ORDER BY expr_list] +[LIMIT [offset_value, ]n BY columns] +[LIMIT [n, ]m] +[UNION ALL ...] +[INTO OUTFILE filename] +[FORMAT format] +``` + +Все секции являются необязательными, за исключением списка выражений сразу после `SELECT`, о котором более подробно будет рассказано [ниже](#select-clause). + +Особенности каждой необязательной секции рассматриваются в отдельных разделах, которые перечислены в том же порядке, в каком они выполняются: + +- [Секция WITH](with.md) +- [Секция FROM](from.md) +- [Секция SAMPLE](sample.md) +- [Секция JOIN](join.md) +- [Секция PREWHERE](prewhere.md) +- [Секция WHERE](where.md) +- [Секция GROUP BY](group-by.md) +- [Секция LIMIT BY](limit-by.md) +- [Секция HAVING](having.md) +- [Секция SELECT](#select-clause) +- [Секция DISTINCT](distinct.md) +- [Секция LIMIT](limit.md) +- [Секция UNION ALL](union-all.md) +- [Секция INTO OUTFILE](into-outfile.md) +- [Секция FORMAT](format.md) + +## Секция SELECT {#select-clause} + +[Выражения](../../syntax.md#syntax-expressions) указанные в секции `SELECT` анализируются после завершения всех вычислений из секций, описанных выше. Вернее, анализируются выражения, стоящие над агрегатными функциями, если есть агрегатные функции. +Сами агрегатные функции и то, что под ними, вычисляются при агрегации (`GROUP BY`). Эти выражения работают так, как будто применяются к отдельным строкам результата. + +Если в результат необходимо включить все столбцы, используйте символ звёздочка (`*`). Например, `SELECT * FROM ...`. + +Чтобы включить в результат несколько столбцов, выбрав их имена с помощью регулярных выражений [re2](https://en.wikipedia.org/wiki/RE2_(software)), используйте выражение `COLUMNS`. + +``` sql +COLUMNS('regexp') +``` + +Например, рассмотрим таблицу: + +``` sql +CREATE TABLE default.col_names (aa Int8, ab Int8, bc Int8) ENGINE = TinyLog +``` + +Следующий запрос выбирает данные из всех столбцов, содержащих в имени символ `a`. + +``` sql +SELECT COLUMNS('a') FROM col_names +``` + +``` text +┌─aa─┬─ab─┐ +│ 1 │ 1 │ +└────┴────┘ +``` + +Выбранные стоблцы возвращаются не в алфавитном порядке. + +В запросе можно использовать несколько выражений `COLUMNS`, а также вызывать над ними функции. + +Например: + +``` sql +SELECT COLUMNS('a'), COLUMNS('c'), toTypeName(COLUMNS('c')) FROM col_names +``` + +``` text +┌─aa─┬─ab─┬─bc─┬─toTypeName(bc)─┐ +│ 1 │ 1 │ 1 │ Int8 │ +└────┴────┴────┴────────────────┘ +``` + +Каждый столбец, возвращённый выражением `COLUMNS`, передаётся в функцию отдельным аргументом. Также можно передавать и другие аргументы, если функция их поддерживаем. Аккуратно используйте функции. Если функция не поддерживает переданное количество аргументов, то ClickHouse генерирует исключение. + +Например: + +``` sql +SELECT COLUMNS('a') + COLUMNS('c') FROM col_names +``` + +``` text +Received exception from server (version 19.14.1): +Code: 42. DB::Exception: Received from localhost:9000. DB::Exception: Number of arguments for function plus doesn't match: passed 3, should be 2. +``` + +В этом примере, `COLUMNS('a')` возвращает два столбца: `aa` и `ab`. `COLUMNS('c')` возвращает столбец `bc`. Оператор `+` не работает с тремя аргументами, поэтому ClickHouse генерирует исключение с соответствущим сообщением. + +Столбцы, которые возвращаются выражением `COLUMNS` могут быть разных типов. Если `COLUMNS` не возвращает ни одного столбца и это единственное выражение в запросе `SELECT`, то ClickHouse генерирует исключение. + +### Звёздочка {#asterisk} + +В любом месте запроса, вместо выражения, может стоять звёздочка. При анализе запроса звёздочка раскрывается в список всех столбцов таблицы (за исключением `MATERIALIZED` и `ALIAS` столбцов). Есть лишь немного случаев, когда оправдано использовать звёздочку: + +- при создании дампа таблицы; +- для таблиц, содержащих всего несколько столбцов - например, системных таблиц; +- для получения информации о том, какие столбцы есть в таблице; в этом случае, укажите `LIMIT 1`. Но лучше используйте запрос `DESC TABLE`; +- при наличии сильной фильтрации по небольшому количеству столбцов с помощью `PREWHERE`; +- в подзапросах (так как из подзапросов выкидываются столбцы, не нужные для внешнего запроса). + +В других случаях использование звёздочки является издевательством над системой, так как вместо преимуществ столбцовой СУБД вы получаете недостатки. То есть использовать звёздочку не рекомендуется. + +### Экстремальные значения {#extreme-values} + +Вы можете получить в дополнение к результату также минимальные и максимальные значения по столбцам результата. Для этого выставите настройку **extremes** в 1. Минимумы и максимумы считаются для числовых типов, дат, дат-с-временем. Для остальных столбцов будут выведены значения по умолчанию. + +Вычисляются дополнительные две строчки - минимумы и максимумы, соответственно. Эти две дополнительные строки выводятся в [форматах](../../../interfaces/formats.md) `JSON*`, `TabSeparated*`, и `Pretty*` отдельно от остальных строчек. В остальных форматах они не выводится. + +Во форматах `JSON*`, экстремальные значения выводятся отдельным полем ‘extremes’. В форматах `TabSeparated*`, строка выводится после основного результата и после ‘totals’ если есть. Перед ней (после остальных данных) вставляется пустая строка. В форматах `Pretty*`, строка выводится отдельной таблицей после основного результата и после `totals` если есть. + +Экстремальные значения вычисляются для строк перед `LIMIT`, но после `LIMIT BY`. Однако при использовании `LIMIT offset, size`, строки перед `offset` включаются в `extremes`. В потоковых запросах, в результате может учитываться также небольшое количество строчек, прошедших `LIMIT`. + +### Замечания {#notes} + +Вы можете использовать синонимы (алиасы `AS`) в любом месте запроса. + +В секциях `GROUP BY`, `ORDER BY`, в отличие от диалекта MySQL, и в соответствии со стандартным SQL, не поддерживаются позиционные аргументы. +Например, если вы напишите `GROUP BY 1, 2` - то это будет воспринято, как группировка по константам (то есть, агрегация всех строк в одну). + + +## Детали реализации {#implementation-details} + +Если в запросе отсутствуют секции `DISTINCT`, `GROUP BY`, `ORDER BY`, подзапросы в `IN` и `JOIN`, то запрос будет обработан полностью потоково, с использованием O(1) количества оперативки. +Иначе запрос может съесть много оперативки, если не указаны подходящие ограничения: + +- `max_memory_usage` +- `max_rows_to_group_by` +- `max_rows_to_sort` +- `max_rows_in_distinct` +- `max_bytes_in_distinct` +- `max_rows_in_set` +- `max_bytes_in_set` +- `max_rows_in_join` +- `max_bytes_in_join` +- `max_bytes_before_external_sort` +- `max_bytes_before_external_group_by` + +Подробнее смотрите в разделе «Настройки». Присутствует возможность использовать внешнюю сортировку (с сохранением временных данных на диск) и внешнюю агрегацию. + +{## [Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/statements/select/) ##} diff --git a/docs/ru/sql-reference/statements/select/into-outfile.md b/docs/ru/sql-reference/statements/select/into-outfile.md new file mode 100644 index 00000000000..8f0126068d1 --- /dev/null +++ b/docs/ru/sql-reference/statements/select/into-outfile.md @@ -0,0 +1,9 @@ +# Секция INTO OUTFILE {#into-outfile-clause} + +Чтобы перенаправить вывод `SELECT` запроса в указанный файл на стороне клиента, добавьте к нему секцию `INTO OUTFILE filename` (где filenam — строковый литерал). + +## Детали реализации {#implementation-details} + +- Эта функция доступна только в следующих интерфейсах: [клиент командной строки](../../../interfaces/cli.md) и [clickhouse-local](../../../operations/utilities/clickhouse-local.md). Таким образом, запрос, отправленный через [HTTP интерфейс](../../../interfaces/http.md) вернет ошибку. +- Запрос завершится ошибкой, если файл с тем же именем уже существует. +- По умолчанию используется [выходной формат](../../../interfaces/formats.md) `TabSeparated` (как в пакетном режиме клиента командной строки). diff --git a/docs/ru/sql-reference/statements/select/join.md b/docs/ru/sql-reference/statements/select/join.md new file mode 100644 index 00000000000..60f391d888b --- /dev/null +++ b/docs/ru/sql-reference/statements/select/join.md @@ -0,0 +1,187 @@ +# Секция JOIN {#select-join} + +Join создаёт новую таблицу путем объединения столбцов из одной или нескольких таблиц с использованием общих для каждой из них значений. Это обычная операция в базах данных с поддержкой SQL, которая соответствует join из [реляционной алгебры](https://en.wikipedia.org/wiki/Relational_algebra#Joins_and_join-like_operators). Частный случай соединения одной таблицы часто называют «self-join». + +Синтаксис: + +``` sql +SELECT +FROM +[GLOBAL] [ANY|ALL|ASOF] [INNER|LEFT|RIGHT|FULL|CROSS] [OUTER|SEMI|ANTI] JOIN +(ON )|(USING ) ... +``` + +Выражения из секции `ON` и столбцы из секции `USING` называется «ключами соединения». Если не указано иное, при присоединение создаётся [Декартово произведение](https://en.wikipedia.org/wiki/Cartesian_product) из строк с совпадающими значениями ключей соединения, что может привести к получению результатов с гораздо большим количеством строк, чем исходные таблицы. + +## Поддерживаемые типы соединения {#select-join-types} + +Весе типы из стандартого [SQL JOIN](https://en.wikipedia.org/wiki/Join_(SQL)) поддерживаются: + +- `INNER JOIN`, возвращаются только совпадающие строки. +- `LEFT OUTER JOIN`, не совпадающие строки из левой таблицы возвращаются в дополнение к совпадающим строкам. +- `RIGHT OUTER JOIN`, не совпадающие строки из правой таблицы возвращаются в дополнение к совпадающим строкам. +- `FULL OUTER JOIN`, не совпадающие строки из обеих таблиц возвращаются в дополнение к совпадающим строкам. +- `CROSS JOIN`, производит декартово произведение таблиц целиком, ключи соединения не указываются. + +Без указания типа `JOIN` подразумевается `INNER`. Ключевое слово `OUTER` можно опускать. Альтернативным синтаксисом для `CROSS JOIN` является ли указание нескольких таблиц, разделённых запятыми, в [секции FROM](from.md). + +Дополнительные типы соединений, доступные в ClickHouse: + +- `LEFT SEMI JOIN` и `RIGHT SEMI JOIN`, белый список по ключам соединения, не производит декартово произведение. +- `LEFT ANTI JOIN` и `RIGHT ANTI JOIN`, черный список по ключам соединения, не производит декартово произведение. + +## Строгость {#select-join-strictness} + +Изменяет способ сопоставления по ключам соединения: + +- `ALL` — стандартное поведение `JOIN` в SQL, как описано выше. По умолчанию. +- `ANY` — Частично (для противоположных сторон `LEFT` и `RIGHT`) или полностью (для `INNER` и `FULL`) отключает декартово произведение для стандартых видов `JOIN`. +- `ASOF` — Для соединения последовательностей по нечеткому совпадению. Использование `ASOF JOIN` описано ниже. + +!!! note "Примечание" + Значение строгости по умолчанию может быть переопределено с помощью настройки [join\_default\_strictness](../../../operations/settings/settings.md#settings-join_default_strictness). + +### Использование ASOF JOIN {#asof-join-usage} + +`ASOF JOIN` применим в том случае, когда необходимо объединять записи, которые не имеют точного совпадения. + +Таблицы для `ASOF JOIN` должны иметь столбец с отсортированной последовательностью. Этот столбец не может быть единственным в таблице и должен быть одного из типов: `UInt32`, `UInt64`, `Float32`, `Float64`, `Date` и `DateTime`. + +Синтаксис `ASOF JOIN ... ON`: + +``` sql +SELECT expressions_list +FROM table_1 +ASOF LEFT JOIN table_2 +ON equi_cond AND closest_match_cond +``` + +Можно использовать произвольное количество условий равенства и одно условие на ближайшее совпадение. Например, `SELECT count() FROM table_1 ASOF LEFT JOIN table_2 ON table_1.a == table_2.b AND table_2.t <= table_1.t`. + +Условия, поддержанные для проверки на ближайшее совпадение: `>`, `>=`, `<`, `<=`. + +Синтаксис `ASOF JOIN ... USING`: + +``` sql +SELECT expressions_list +FROM table_1 +ASOF JOIN table_2 +USING (equi_column1, ... equi_columnN, asof_column) +``` + +Для слияния по равенству `ASOF JOIN` использует `equi_columnX`, а для слияния по ближайшему совпадению использует `asof_column` с условием `table_1.asof_column >= table_2.asof_column`. Столбец `asof_column` должен быть последним в секции `USING`. + +Например, рассмотрим следующие таблицы: + + table_1 table_2 + event | ev_time | user_id event | ev_time | user_id + ----------|---------|---------- ----------|---------|---------- + ... ... + event_1_1 | 12:00 | 42 event_2_1 | 11:59 | 42 + ... event_2_2 | 12:30 | 42 + event_1_2 | 13:00 | 42 event_2_3 | 13:00 | 42 + ... ... + +`ASOF JOIN` принимает метку времени пользовательского события из `table_1` и находит такое событие в `table_2` метка времени которого наиболее близка к метке времени события из `table_1` в соответствии с условием на ближайшее совпадение. При этом столбец `user_id` используется для объединения по равенству, а столбец `ev_time` для объединения по ближайшему совпадению. В нашем примере `event_1_1` может быть объединено с `event_2_1`, `event_1_2` может быть объединено с `event_2_3`, а `event_2_2` не объединяется. + +!!! note "Примечание" + `ASOF JOIN` не поддержан для движка таблиц [Join](../../../engines/table-engines/special/join.md). + +Чтобы задать значение строгости по умолчанию, используйте сессионный параметр [join\_default\_strictness](../../../operations/settings/settings.md#settings-join_default_strictness). + +#### Распределённый join {#global-join} + +Есть два пути для выполнения соединения с участием распределённых таблиц: + +- При использовании обычного `JOIN` , запрос отправляется на удалённые серверы. На каждом из них выполняются подзапросы для формирования «правой» таблицы, и с этой таблицей выполняется соединение. То есть, «правая» таблица формируется на каждом сервере отдельно. +- При использовании `GLOBAL ... JOIN`, сначала сервер-инициатор запроса запускает подзапрос для вычисления правой таблицы. Эта временная таблица передаётся на каждый удалённый сервер, и на них выполняются запросы с использованием переданных временных данных. + +Будьте аккуратны при использовании `GLOBAL`. За дополнительной информацией обращайтесь в раздел [Распределенные подзапросы](#select-distributed-subqueries). + +## Рекомендации по использованию {#usage-recommendations} + +### Обработка пустых ячеек и NULL {#processing-of-empty-or-null-cells} + +При соединении таблиц могут появляться пустые ячейки. Настройка [join\_use\_nulls](../../../operations/settings/settings.md#join_use_nulls) определяет, как ClickHouse заполняет эти ячейки. + +Если ключами `JOIN` выступают поля типа [Nullable](../../../sql-reference/data-types/nullable.md), то строки, где хотя бы один из ключей имеет значение [NULL](../../../sql-reference/syntax.md#null-literal), не соединяются. + +### Синтаксис {#syntax} + +Требуется, чтобы столбцы, указанные в `USING`, назывались одинаково в обоих подзапросах, а остальные столбцы - по-разному. Изменить имена столбцов в подзапросах можно с помощью синонимов. + +В секции `USING` указывается один или несколько столбцов для соединения, что обозначает условие на равенство этих столбцов. Список столбцов задаётся без скобок. Более сложные условия соединения не поддерживаются. + +### Ограничения cинтаксиса {#syntax-limitations} + +Для множественных секций `JOIN` в одном запросе `SELECT`: + +- Получение всех столбцов через `*` возможно только при объединении таблиц, но не подзапросов. +- Секция `PREWHERE` недоступна. + +Для секций `ON`, `WHERE` и `GROUP BY`: + +- Нельзя использовать произвольные выражения в секциях `ON`, `WHERE`, и `GROUP BY`, однако можно определить выражение в секции `SELECT` и затем использовать его через алиас в других секциях. + +### Производительность {#performance} + +При запуске `JOIN`, отсутствует оптимизация порядка выполнения по отношению к другим стадиям запроса. Соединение (поиск в «правой» таблице) выполняется до фильтрации в `WHERE` и до агрегации. Чтобы явно задать порядок вычислений, рекомендуется выполнять `JOIN` подзапроса с подзапросом. + +Каждый раз для выполнения запроса с одинаковым `JOIN`, подзапрос выполняется заново — результат не кэшируется. Это можно избежать, используя специальный движок таблиц [Join](../../../engines/table-engines/special/join.md), представляющий собой подготовленное множество для соединения, которое всегда находится в оперативке. + +В некоторых случаях это более эффективно использовать [IN](../../operators/in.md) вместо `JOIN`. + +Если `JOIN` необходим для соединения с таблицами измерений (dimension tables - сравнительно небольшие таблицы, которые содержат свойства измерений - например, имена для рекламных кампаний), то использование `JOIN` может быть не очень удобным из-за громоздкости синтаксиса, а также из-за того, что правая таблица читается заново при каждом запросе. Специально для таких случаев существует функциональность «Внешние словари», которую следует использовать вместо `JOIN`. Дополнительные сведения смотрите в разделе «Внешние словари». + + +### Ограничения по памяти {#memory-limitations} + +По умолчанию ClickHouse использует алгоритм [hash join](https://en.wikipedia.org/wiki/Hash_join). ClickHouse берет `` и создает для него хэш-таблицу в оперативной памяти. После некоторого порога потребления памяти ClickHouse переходит к алгоритму merge join. + +- [max\_rows\_in\_join](../../../operations/settings/query-complexity.md#settings-max_rows_in_join) — ограничивает количество строк в хэш-таблице. +- [max\_bytes\_in\_join](../../../operations/settings/query-complexity.md#settings-max_bytes_in_join) — ограничивает размер хэш-таблицы. + +По достижении любого из этих ограничений, ClickHouse действует в соответствии с настройкой [join\_overflow\_mode](../../../operations/settings/query-complexity.md#settings-join_overflow_mode). + +## Примеры {#examples} + +Пример: + +``` sql +SELECT + CounterID, + hits, + visits +FROM +( + SELECT + CounterID, + count() AS hits + FROM test.hits + GROUP BY CounterID +) ANY LEFT JOIN +( + SELECT + CounterID, + sum(Sign) AS visits + FROM test.visits + GROUP BY CounterID +) USING CounterID +ORDER BY hits DESC +LIMIT 10 +``` + +``` text +┌─CounterID─┬───hits─┬─visits─┐ +│ 1143050 │ 523264 │ 13665 │ +│ 731962 │ 475698 │ 102716 │ +│ 722545 │ 337212 │ 108187 │ +│ 722889 │ 252197 │ 10547 │ +│ 2237260 │ 196036 │ 9522 │ +│ 23057320 │ 147211 │ 7689 │ +│ 722818 │ 90109 │ 17847 │ +│ 48221 │ 85379 │ 4652 │ +│ 19762435 │ 77807 │ 7026 │ +│ 722884 │ 77492 │ 11056 │ +└───────────┴────────┴────────┘ +``` diff --git a/docs/ru/sql-reference/statements/select/limit-by.md b/docs/ru/sql-reference/statements/select/limit-by.md new file mode 100644 index 00000000000..ea5d467ae4f --- /dev/null +++ b/docs/ru/sql-reference/statements/select/limit-by.md @@ -0,0 +1,69 @@ +# Секция LIMIT BY {#limit-by-clause} + +Запрос с секцией `LIMIT n BY expressions` выбирает первые `n` строк для каждого отличного значения `expressions`. Ключ `LIMIT BY` может содержать любое количество [выражений](../../syntax.md#syntax-expressions). + +ClickHouse поддерживает следующий синтаксис: + +- `LIMIT [offset_value, ]n BY expressions` +- `LIMIT n OFFSET offset_value BY expressions` + +Во время обработки запроса, ClickHouse выбирает данные, упорядоченные по ключу сортировки. Ключ сортировки задаётся явно в секции [ORDER BY](order-by.md#select-order-by) или неявно в свойствах движка таблицы. Затем ClickHouse применяет `LIMIT n BY expressions` и возвращает первые `n` для каждой отличной комбинации `expressions`. Если указан `OFFSET`, то для каждого блока данных, который принадлежит отдельной комбинации `expressions`, ClickHouse отступает `offset_value` строк от начала блока и возвращает не более `n`. Если `offset_value` больше, чем количество строк в блоке данных, ClickHouse не возвращает ни одной строки. + +`LIMIT BY` не связана с секцией `LIMIT`. Их можно использовать в одном запросе. + +## Примеры + +Образец таблицы: + +``` sql +CREATE TABLE limit_by(id Int, val Int) ENGINE = Memory; +INSERT INTO limit_by values(1, 10), (1, 11), (1, 12), (2, 20), (2, 21); +``` + +Запросы: + +``` sql +SELECT * FROM limit_by ORDER BY id, val LIMIT 2 BY id +``` + +``` text +┌─id─┬─val─┐ +│ 1 │ 10 │ +│ 1 │ 11 │ +│ 2 │ 20 │ +│ 2 │ 21 │ +└────┴─────┘ +``` + +``` sql +SELECT * FROM limit_by ORDER BY id, val LIMIT 1, 2 BY id +``` + +``` text +┌─id─┬─val─┐ +│ 1 │ 11 │ +│ 1 │ 12 │ +│ 2 │ 21 │ +└────┴─────┘ +``` + +Запрос `SELECT * FROM limit_by ORDER BY id, val LIMIT 2 OFFSET 1 BY id` возвращает такой же результат. + +Следующий запрос выбирает топ 5 рефереров для каждой пары `domain, device_type`, но не более 100 строк (`LIMIT n BY + LIMIT`). + +``` sql +SELECT + domainWithoutWWW(URL) AS domain, + domainWithoutWWW(REFERRER_URL) AS referrer, + device_type, + count() cnt +FROM hits +GROUP BY domain, referrer, device_type +ORDER BY cnt DESC +LIMIT 5 BY domain, device_type +LIMIT 100 +``` + +Запрос выберет топ 5 рефереров для каждой пары `domain, device_type`, но не более 100 строк (`LIMIT n BY + LIMIT`). + +`LIMIT n BY` работает с [NULL](../../syntax.md#null-literal) как если бы это было конкретное значение. Т.е. в результате запроса пользователь получит все комбинации полей, указанных в `BY`. diff --git a/docs/ru/sql-reference/statements/select/limit.md b/docs/ru/sql-reference/statements/select/limit.md new file mode 100644 index 00000000000..5c68c22d548 --- /dev/null +++ b/docs/ru/sql-reference/statements/select/limit.md @@ -0,0 +1,9 @@ +# Секция LIMIT {#limit-clause} + +`LIMIT m` позволяет выбрать из результата первые `m` строк. + +`LIMIT n, m` позволяет выбрать из результата первые `m` строк после пропуска первых `n` строк. Синтаксис `LIMIT m OFFSET n` также поддерживается. + +`n` и `m` должны быть неотрицательными целыми числами. + +При отсутствии секции [ORDER BY](order-by.md), однозначно сортирующей результат, результат может быть произвольным и может являться недетерминированным. diff --git a/docs/ru/sql-reference/statements/select/order-by.md b/docs/ru/sql-reference/statements/select/order-by.md new file mode 100644 index 00000000000..83b704cec8f --- /dev/null +++ b/docs/ru/sql-reference/statements/select/order-by.md @@ -0,0 +1,68 @@ +# Секция ORDER BY {#select-order-by} + +Секция `ORDER BY` содержит список выражений, к каждому из которых также может быть приписано `DESC` или `ASC` (направление сортировки). Если ничего не приписано - это аналогично приписыванию `ASC`. `ASC` - сортировка по возрастанию, `DESC` - сортировка по убыванию. Обозначение направления сортировки действует на одно выражение, а не на весь список. Пример: `ORDER BY Visits DESC, SearchPhrase` + +Строки, для которых список выражений, по которым производится сортировка, принимает одинаковые значения, выводятся в произвольном порядке, который может быть также недетерминированным (каждый раз разным). +Если секция ORDER BY отсутствует, то, аналогично, порядок, в котором идут строки, не определён, и может быть недетерминированным. + +## Сортировка специальных значений {#sorting-of-special-values} + +Существует два подхода к участию `NaN` и `NULL` в порядке сортировки: + +- По умолчанию или с модификатором `NULLS LAST`: сначала остальные значения, затем `NaN`, затем `NULL`. +- С модификатором `NULLS FIRST`: сначала `NULL`, затем `NaN`, затем другие значения. + +### Пример {#example} + +Для таблицы + +``` text +┌─x─┬────y─┐ +│ 1 │ ᴺᵁᴸᴸ │ +│ 2 │ 2 │ +│ 1 │ nan │ +│ 2 │ 2 │ +│ 3 │ 4 │ +│ 5 │ 6 │ +│ 6 │ nan │ +│ 7 │ ᴺᵁᴸᴸ │ +│ 6 │ 7 │ +│ 8 │ 9 │ +└───┴──────┘ +``` + +Выполнение запроса `SELECT * FROM t_null_nan ORDER BY y NULLS FIRST` получить: + +``` text +┌─x─┬────y─┐ +│ 1 │ ᴺᵁᴸᴸ │ +│ 7 │ ᴺᵁᴸᴸ │ +│ 1 │ nan │ +│ 6 │ nan │ +│ 2 │ 2 │ +│ 2 │ 2 │ +│ 3 │ 4 │ +│ 5 │ 6 │ +│ 6 │ 7 │ +│ 8 │ 9 │ +└───┴──────┘ +``` + +При сортировке чисел с плавающей запятой NaNs отделяются от других значений. Независимо от порядка сортировки, NaNs приходят в конце. Другими словами, при восходящей сортировке они помещаются так, как будто они больше всех остальных чисел, а при нисходящей сортировке они помещаются так, как будто они меньше остальных. + +## Поддержка collation {#collation-support} + +Для сортировки по значениям типа String есть возможность указать collation (сравнение). Пример: `ORDER BY SearchPhrase COLLATE 'tr'` - для сортировки по поисковой фразе, по возрастанию, с учётом турецкого алфавита, регистронезависимо, при допущении, что строки в кодировке UTF-8. `COLLATE` может быть указан или не указан для каждого выражения в ORDER BY независимо. Если есть `ASC` или `DESC`, то `COLLATE` указывается после них. При использовании `COLLATE` сортировка всегда регистронезависима. + +Рекомендуется использовать `COLLATE` только для окончательной сортировки небольшого количества строк, так как производительность сортировки с указанием `COLLATE` меньше, чем обычной сортировки по байтам. + +## Деталь реализации {#implementation-details} + +Если кроме `ORDER BY` указан также не слишком большой [LIMIT](limit.md), то расходуется меньше оперативки. Иначе расходуется количество памяти, пропорциональное количеству данных для сортировки. При распределённой обработке запроса, если отсутствует [GROUP BY](group-by.md), сортировка частично делается на удалённых серверах, а на сервере-инициаторе запроса производится слияние результатов. Таким образом, при распределённой сортировке, может сортироваться объём данных, превышающий размер памяти на одном сервере. + +Существует возможность выполнять сортировку во внешней памяти (с созданием временных файлов на диске), если оперативной памяти не хватает. Для этого предназначена настройка `max_bytes_before_external_sort`. Если она выставлена в 0 (по умолчанию), то внешняя сортировка выключена. Если она включена, то при достижении объёмом данных для сортировки указанного количества байт, накопленные данные будут отсортированы и сброшены во временный файл. После того, как все данные будут прочитаны, будет произведено слияние всех сортированных файлов и выдача результата. Файлы записываются в директорию `/var/lib/clickhouse/tmp/` (по умолчанию, может быть изменено с помощью параметра `tmp_path`) в конфиге. + +На выполнение запроса может расходоваться больше памяти, чем `max_bytes_before_external_sort`. Поэтому, значение этой настройки должно быть существенно меньше, чем `max_memory_usage`. Для примера, если на вашем сервере 128 GB оперативки, и вам нужно выполнить один запрос, то выставите `max_memory_usage` в 100 GB, а `max_bytes_before_external_sort` в 80 GB. + +Внешняя сортировка работает существенно менее эффективно, чем сортировка в оперативке. + diff --git a/docs/ru/sql-reference/statements/select/prewhere.md b/docs/ru/sql-reference/statements/select/prewhere.md new file mode 100644 index 00000000000..d83b7ca3b18 --- /dev/null +++ b/docs/ru/sql-reference/statements/select/prewhere.md @@ -0,0 +1,17 @@ +# Секции PREWHERE {#prewhere-clause} + +Prewhere — это оптимизация для более эффективного применения фильтрации. Она включена по умолчанию, даже если секция `PREWHERE` явно не указана. В этом случае работает автоматическое перемещение части выражения из [WHERE](where.md) до стадии prewhere. Роль секции `PREWHERE` только для управления этой оптимизацией, если вы думаете, что знаете, как сделать перемещение условия лучше, чем это происходит по умолчанию. + +При оптимизации prewhere сначала читываются только те столбцы, которые необходимы для выполнения выражения prewhere. Затем читаются другие столбцы, необходимые для выполнения остальной части запроса, но только те блоки, в которых находится выражение prewhere «верно» по крайней мере для некоторых рядов. Если есть много блоков, где выражение prewhere «ложно» для всех строк и для выражения prewhere требуется меньше столбцов, чем для других частей запроса, это часто позволяет считывать гораздо меньше данных с диска для выполнения запроса. + +## Управление prewhere вручную {#controlling-prewhere-manually} + +`PREWHERE` имеет смысл использовать, если есть условия фильтрации, которые использует меньшинство столбцов из тех, что есть в запросе, но достаточно сильно фильтрует данные. Таким образом, сокращается количество читаемых данных. + +В запрос может быть одновременно указано и `PREWHERE` и `WHERE`. В этом случае, `PREWHERE` предшествует `WHERE`. + +Если значение параметра `optimize_move_to_prewhere` равно 0, эвристика по автоматическому перемещнию части выражений из `WHERE` к `PREWHERE` отключается. + +## Ограничения {#limitations} + +`PREWHERE` поддерживается только табличными движками из семейства `*MergeTree`. diff --git a/docs/ru/sql-reference/statements/select/sample.md b/docs/ru/sql-reference/statements/select/sample.md new file mode 100644 index 00000000000..ca6b49c9ad6 --- /dev/null +++ b/docs/ru/sql-reference/statements/select/sample.md @@ -0,0 +1,113 @@ +# Секция SAMPLE {#select-sample-clause} + +Секция `SAMPLE` позволяет выполнять запросы приближённо. Например, чтобы посчитать статистику по всем визитам, можно обработать 1/10 всех визитов и результат домножить на 10. + +Сэмплирование имеет смысл, когда: + +1. Точность результата не важна, например, для оценочных расчетов. +2. Возможности аппаратной части не позволяют соответствовать строгим критериям. Например, время ответа должно быть \<100 мс. При этом точность расчета имеет более низкий приоритет. +3. Точность результата участвует в бизнес-модели сервиса. Например, пользователи с бесплатной подпиской на сервис могут получать отчеты с меньшей точностью, чем пользователи с премиум подпиской. + +!!! note "Внимание" + Не стоит использовать сэмплирование в тех задачах, где важна точность расчетов. Например, при работе с финансовыми отчетами. + +Свойства сэмплирования: + +- Сэмплирование работает детерминированно. При многократном выполнении одного и того же запроса `SELECT .. SAMPLE`, результат всегда будет одинаковым. +- Сэмплирование поддерживает консистентность для разных таблиц. Имеется в виду, что для таблиц с одним и тем же ключом сэмплирования, подмножество данных в выборках будет одинаковым (выборки при этом должны быть сформированы для одинаковой доли данных). Например, выборка по идентификаторам посетителей выберет из разных таблиц строки с одинаковым подмножеством всех возможных идентификаторов. Это свойство позволяет использовать выборки в подзапросах в секции [IN](../../operators/in.md#select-in-operators), а также объединять выборки с помощью [JOIN](join.md#select-join). +- Сэмплирование позволяет читать меньше данных с диска. Обратите внимание, для этого необходимо корректно указать ключ сэмплирования. Подробнее см. в разделе [Создание таблицы MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table). + +Сэмплирование поддерживается только таблицами семейства [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table) и только в том случае, если для таблиц был указан ключ сэмплирования (выражение, на основе которого должна производиться выборка). Подробнее см. в разделе [Создание таблиц MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table). + +Выражение `SAMPLE` в запросе можно задать следующими способами: + +| Способ задания SAMPLE | Описание | +|-----------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `SAMPLE k` | Здесь `k` – это дробное число в интервале от 0 до 1.
Запрос будет выполнен по `k` доле данных. Например, если указано `SAMPLE 1/10`, то запрос будет выполнен для выборки из 1/10 данных. [Подробнее](#select-sample-k) | +| `SAMPLE n` | Здесь `n` – это достаточно большое целое число.
Запрос будет выполнен для выборки, состоящей из не менее чем `n` строк. Например, если указано `SAMPLE 10000000`, то запрос будет выполнен для не менее чем 10,000,000 строк. [Подробнее](#select-sample-n) | +| `SAMPLE k OFFSET m` | Здесь `k` и `m` – числа от 0 до 1.
Запрос будет выполнен по `k` доле данных. При этом выборка будет сформирована со смещением на `m` долю. [Подробнее](#select-sample-offset) | + +## SAMPLE k {#select-sample-k} + +Здесь `k` – число в интервале от 0 до 1. Поддерживается как дробная, так и десятичная форма записи. Например, `SAMPLE 1/2` или `SAMPLE 0.5`. + +Если задано выражение `SAMPLE k`, запрос будет выполнен для `k` доли данных. Рассмотрим пример: + +``` sql +SELECT + Title, + count() * 10 AS PageViews +FROM hits_distributed +SAMPLE 0.1 +WHERE + CounterID = 34 +GROUP BY Title +ORDER BY PageViews DESC LIMIT 1000 +``` + +В этом примере запрос выполняется по выборке из 0.1 (10%) данных. Значения агрегатных функций не корректируются автоматически, поэтому чтобы получить приближённый результат, значение `count()` нужно вручную умножить на 10. + +Выборка с указанием относительного коэффициента является «согласованной»: для таблиц с одним и тем же ключом сэмплирования, выборка с одинаковой относительной долей всегда будет составлять одно и то же подмножество данных. То есть выборка из разных таблиц, на разных серверах, в разное время, формируется одинаковым образом. + +## SAMPLE n {#select-sample-n} + +Здесь `n` – это достаточно большое целое число. Например, `SAMPLE 10000000`. + +Если задано выражение `SAMPLE n`, запрос будет выполнен для выборки из не менее `n` строк (но не значительно больше этого значения). Например, если задать `SAMPLE 10000000`, в выборку попадут не менее 10,000,000 строк. + +!!! note "Примечание" + Следует иметь в виду, что `n` должно быть достаточно большим числом. Так как минимальной единицей данных для чтения является одна гранула (её размер задаётся настройкой `index_granularity` для таблицы), имеет смысл создавать выборки, размер которых существенно превосходит размер гранулы. + +При выполнении `SAMPLE n` коэффициент сэмплирования заранее неизвестен (то есть нет информации о том, относительно какого количества данных будет сформирована выборка). Чтобы узнать коэффициент сэмплирования, используйте столбец `_sample_factor`. + +Виртуальный столбец `_sample_factor` автоматически создается в тех таблицах, для которых задано выражение `SAMPLE BY` (подробнее см. в разделе [Создание таблицы MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table)). В столбце содержится коэффициент сэмплирования для таблицы – он рассчитывается динамически по мере добавления данных в таблицу. Ниже приведены примеры использования столбца `_sample_factor`. + +Предположим, у нас есть таблица, в которой ведется статистика посещений сайта. Пример ниже показывает, как рассчитать суммарное число просмотров: + +``` sql +SELECT sum(PageViews * _sample_factor) +FROM visits +SAMPLE 10000000 +``` + +Следующий пример показывает, как посчитать общее число визитов: + +``` sql +SELECT sum(_sample_factor) +FROM visits +SAMPLE 10000000 +``` + +В примере ниже рассчитывается среднее время на сайте. Обратите внимание, при расчете средних значений, умножать результат на коэффициент сэмплирования не нужно. + +``` sql +SELECT avg(Duration) +FROM visits +SAMPLE 10000000 +``` + +## SAMPLE k OFFSET m {#select-sample-offset} + +Здесь `k` и `m` – числа в интервале от 0 до 1. Например, `SAMPLE 0.1 OFFSET 0.5`. Поддерживается как дробная, так и десятичная форма записи. + +При задании `SAMPLE k OFFSET m`, выборка будет сформирована из `k` доли данных со смещением на долю `m`. Примеры приведены ниже. + +**Пример 1** + +``` sql +SAMPLE 1/10 +``` + +В этом примере выборка будет сформирована по 1/10 доле всех данных: + +`[++------------------]` + +**Пример 2** + +``` sql +SAMPLE 1/10 OFFSET 1/2 +``` + +Здесь выборка, которая состоит из 1/10 доли данных, взята из второй половины данных. + +`[----------++--------]` diff --git a/docs/ru/sql-reference/statements/select/union-all.md b/docs/ru/sql-reference/statements/select/union-all.md new file mode 100644 index 00000000000..bffd667fa1d --- /dev/null +++ b/docs/ru/sql-reference/statements/select/union-all.md @@ -0,0 +1,30 @@ +# Секция UNION ALL {#union-all-clause} + +Вы можете использовать `UNION ALL` чтобы объединить любое количество `SELECT` запросы путем расширения их результатов. Пример: + +``` sql +SELECT CounterID, 1 AS table, toInt64(count()) AS c + FROM test.hits + GROUP BY CounterID + +UNION ALL + +SELECT CounterID, 2 AS table, sum(Sign) AS c + FROM test.visits + GROUP BY CounterID + HAVING c > 0 +``` + +Результирующие столбцы сопоставляются по их индексу (порядку внутри `SELECT`). Если имена столбцов не совпадают, то имена для конечного результата берутся из первого запроса. + +При объединении выполняет приведение типов. Например, если два запроса имеют одно и то же поле с не-`Nullable` и `Nullable` совместимыми типами, полученные в результате `UNION ALL` данные будут иметь `Nullable` тип. + +Запросы, которые являются частью `UNION ALL` не могут быть заключен в круглые скобки. [ORDER BY](order-by.md) и [LIMIT](limit.md) применяются к отдельным запросам, а не к конечному результату. Если вам нужно применить преобразование к конечному результату, вы можете разместить все объединенные с помощью `UNION ALL` запросы в подзапрос в секции [FROM](from.md). + +## Ограничения {#limitations} + +Поддерживается только `UNION ALL`. Обычный `UNION` (`UNION DISTINCT`) не поддерживается. Если вам это нужно `UNION DISTINCT`, вы можете написать `SELECT DISTINCT` из подзапроса, содержащего `UNION ALL`. + +## Детали реализации {#implementation-details} + +Запросы, которые являются частью `UNION ALL` выполняются параллельно, и их результаты могут быть смешаны вместе. diff --git a/docs/ru/sql-reference/statements/select/where.md b/docs/ru/sql-reference/statements/select/where.md new file mode 100644 index 00000000000..e38d36e6fa8 --- /dev/null +++ b/docs/ru/sql-reference/statements/select/where.md @@ -0,0 +1,25 @@ +# Предложение WHERE {#select-where} + +Позволяет задать выражение, которое ClickHouse использует для фильтрации данных перед всеми другими действиями в запросе кроме выражений, содержащихся в секции [PREWHERE](prewhere.md#select-prewhere). Обычно, это выражение с логическими операторами. + +Результат выражения должен иметь тип `UInt8`. + +ClickHouse использует в выражении индексы, если это позволяет [движок таблицы](../../../engines/table-engines/index.md). + +Если в секции необходимо проверить [NULL](../../../sql-reference/syntax.md#null-literal), то используйте операторы [IS NULL](../../operators/index.md#operator-is-null) и [IS NOT NULL](../../operators/index.md#is-not-null), а также соответствующие функции `isNull` и `isNotNull`. В противном случае выражение будет считаться всегда не выполненным. + +Пример проверки на `NULL`: + +``` sql +SELECT * FROM t_null WHERE y IS NULL +``` + +``` text +┌─x─┬────y─┐ +│ 1 │ ᴺᵁᴸᴸ │ +└───┴──────┘ +``` + +!!! note "Примечание" + Существует оптимизация фильтрации под названием [prewhere](prewhere.md). + diff --git a/docs/ru/sql-reference/statements/select/with.md b/docs/ru/sql-reference/statements/select/with.md new file mode 100644 index 00000000000..a5be733866f --- /dev/null +++ b/docs/ru/sql-reference/statements/select/with.md @@ -0,0 +1,76 @@ +# Секция WITH {#sektsiia-with} + +Данная секция представляет собой [Common Table Expressions](https://ru.wikipedia.org/wiki/Иерархические_и_рекурсивные_запросы_в_SQL), то есть позволяет использовать результаты выражений из секции `WITH` в остальной части `SELECT` запроса. + +### Ограничения + +1. Рекурсивные запросы не поддерживаются +2. Если в качестве выражения используется подзапрос, то результат должен содержать ровно одну строку +3. Результаты выражений нельзя переиспользовать во вложенных запросах +В дальнейшем, результаты выражений можно использовать в секции SELECT. + +### Примеры + +**Пример 1:** Использование константного выражения как «переменной» + +``` sql +WITH '2019-08-01 15:23:00' as ts_upper_bound +SELECT * +FROM hits +WHERE + EventDate = toDate(ts_upper_bound) AND + EventTime <= ts_upper_bound +``` + +**Пример 2:** Выкидывание выражения sum(bytes) из списка колонок в SELECT + +``` sql +WITH sum(bytes) as s +SELECT + formatReadableSize(s), + table +FROM system.parts +GROUP BY table +ORDER BY s +``` + +**Пример 3:** Использование результатов скалярного подзапроса + +``` sql +/* запрос покажет TOP 10 самых больших таблиц */ +WITH + ( + SELECT sum(bytes) + FROM system.parts + WHERE active + ) AS total_disk_usage +SELECT + (sum(bytes) / total_disk_usage) * 100 AS table_disk_usage, + table +FROM system.parts +GROUP BY table +ORDER BY table_disk_usage DESC +LIMIT 10 +``` + +**Пример 4:** Переиспользование выражения + +В настоящий момент, переиспользование выражения из секции WITH внутри подзапроса возможно только через дублирование. + +``` sql +WITH ['hello'] AS hello +SELECT + hello, + * +FROM +( + WITH ['hello'] AS hello + SELECT hello +) +``` + +``` text +┌─hello─────┬─hello─────┐ +│ ['hello'] │ ['hello'] │ +└───────────┴───────────┘ +``` diff --git a/docs/ru/sql-reference/syntax.md b/docs/ru/sql-reference/syntax.md index 57101111603..24ab2be8a16 100644 --- a/docs/ru/sql-reference/syntax.md +++ b/docs/ru/sql-reference/syntax.md @@ -95,7 +95,7 @@ INSERT INTO t VALUES (1, 'Hello, world'), (2, 'abc'), (3, 'def') При обработке `NULL` есть множество особенностей. Например, если хотя бы один из аргументов операции сравнения — `NULL`, то результатом такой операции тоже будет `NULL`. Этим же свойством обладают операции умножения, сложения и пр. Подробнее читайте в документации на каждую операцию. -В запросах можно проверить `NULL` с помощью операторов [IS NULL](operators.md#operator-is-null) и [IS NOT NULL](operators.md), а также соответствующих функций `isNull` и `isNotNull`. +В запросах можно проверить `NULL` с помощью операторов [IS NULL](operators/index.md#operator-is-null) и [IS NOT NULL](operators/index.md), а также соответствующих функций `isNull` и `isNotNull`. ## Функции {#funktsii} diff --git a/docs/ru/sql-reference/table-functions/index.md b/docs/ru/sql-reference/table-functions/index.md index 79b247eaaf6..28f437a814a 100644 --- a/docs/ru/sql-reference/table-functions/index.md +++ b/docs/ru/sql-reference/table-functions/index.md @@ -11,7 +11,7 @@ toc_title: "\u0412\u0432\u0435\u0434\u0435\u043D\u0438\u0435" Табличные функции можно использовать в: -- Секции [FROM](../statements/select.md#select-from) запроса `SELECT`. +- Секции [FROM](../statements/select/from.md#select-from) запроса `SELECT`. Это способ создания временной таблицы, которая доступна только в текущем запросе. diff --git a/docs/ru/whats-new/changelog/index.md b/docs/ru/whats-new/changelog/index.md index 4b49085dfc3..c13441a8bd5 100644 --- a/docs/ru/whats-new/changelog/index.md +++ b/docs/ru/whats-new/changelog/index.md @@ -4,663 +4,4 @@ toc_priority: 74 toc_title: '2020' --- -## ClickHouse Release V20.3 {#clickhouse-release-v20-3} - -### ClickHouse Release V20.3.4.10, 2020-03-20 {#clickhouse-release-v20-3-4-10-2020-03-20} - -#### Bug Fix {#bug-fix} - -- This release also contains all bug fixes from 20.1.8.41 -- Fix missing `rows_before_limit_at_least` for queries over http (with processors pipeline). This fixes [\#9730](https://github.com/ClickHouse/ClickHouse/issues/9730). [\#9757](https://github.com/ClickHouse/ClickHouse/pull/9757) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) - -### ClickHouse Release V20.3.3.6, 2020-03-17 {#clickhouse-release-v20-3-3-6-2020-03-17} - -#### Bug Fix {#bug-fix-1} - -- This release also contains all bug fixes from 20.1.7.38 -- Fix bug in a replication that doesn’t allow replication to work if the user has executed mutations on the previous version. This fixes [\#9645](https://github.com/ClickHouse/ClickHouse/issues/9645). [\#9652](https://github.com/ClickHouse/ClickHouse/pull/9652) ([alesapin](https://github.com/alesapin)). It makes version 20.3 backward compatible again. -- Add setting `use_compact_format_in_distributed_parts_names` which allows to write files for `INSERT` queries into `Distributed` table with more compact format. This fixes [\#9647](https://github.com/ClickHouse/ClickHouse/issues/9647). [\#9653](https://github.com/ClickHouse/ClickHouse/pull/9653) ([alesapin](https://github.com/alesapin)). It makes version 20.3 backward compatible again. - -### ClickHouse Release V20.3.2.1, 2020-03-12 {#clickhouse-release-v20-3-2-1-2020-03-12} - -#### Backward Incompatible Change {#backward-incompatible-change} - -- Fixed the issue `file name too long` when sending data for `Distributed` tables for a large number of replicas. Fixed the issue that replica credentials were exposed in the server log. The format of directory name on disk was changed to `[shard{shard_index}[_replica{replica_index}]]`. [\#8911](https://github.com/ClickHouse/ClickHouse/pull/8911) ([Mikhail Korotov](https://github.com/millb)) After you upgrade to the new version, you will not be able to downgrade without manual intervention, because old server version does not recognize the new directory format. If you want to downgrade, you have to manually rename the corresponding directories to the old format. This change is relevant only if you have used asynchronous `INSERT`s to `Distributed` tables. In the version 20.3.3 we will introduce a setting that will allow you to enable the new format gradually. -- Changed the format of replication log entries for mutation commands. You have to wait for old mutations to process before installing the new version. -- Implement simple memory profiler that dumps stacktraces to `system.trace_log` every N bytes over soft allocation limit [\#8765](https://github.com/ClickHouse/ClickHouse/pull/8765) ([Ivan](https://github.com/abyss7)) [\#9472](https://github.com/ClickHouse/ClickHouse/pull/9472) ([alexey-milovidov](https://github.com/alexey-milovidov)) The column of `system.trace_log` was renamed from `timer_type` to `trace_type`. This will require changes in third-party performance analysis and flamegraph processing tools. -- Use OS thread id everywhere instead of internal thread number. This fixes [\#7477](https://github.com/ClickHouse/ClickHouse/issues/7477) Old `clickhouse-client` cannot receive logs that are send from the server when the setting `send_logs_level` is enabled, because the names and types of the structured log messages were changed. On the other hand, different server versions can send logs with different types to each other. When you don’t use the `send_logs_level` setting, you should not care. [\#8954](https://github.com/ClickHouse/ClickHouse/pull/8954) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Remove `indexHint` function [\#9542](https://github.com/ClickHouse/ClickHouse/pull/9542) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Remove `findClusterIndex`, `findClusterValue` functions. This fixes [\#8641](https://github.com/ClickHouse/ClickHouse/issues/8641). If you were using these functions, send an email to `clickhouse-feedback@yandex-team.com` [\#9543](https://github.com/ClickHouse/ClickHouse/pull/9543) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Now it’s not allowed to create columns or add columns with `SELECT` subquery as default expression. [\#9481](https://github.com/ClickHouse/ClickHouse/pull/9481) ([alesapin](https://github.com/alesapin)) -- Require aliases for subqueries in JOIN. [\#9274](https://github.com/ClickHouse/ClickHouse/pull/9274) ([Artem Zuikov](https://github.com/4ertus2)) -- Improved `ALTER MODIFY/ADD` queries logic. Now you cannot `ADD` column without type, `MODIFY` default expression doesn’t change type of column and `MODIFY` type doesn’t loose default expression value. Fixes [\#8669](https://github.com/ClickHouse/ClickHouse/issues/8669). [\#9227](https://github.com/ClickHouse/ClickHouse/pull/9227) ([alesapin](https://github.com/alesapin)) -- Require server to be restarted to apply the changes in logging configuration. This is a temporary workaround to avoid the bug where the server logs to a deleted log file (see [\#8696](https://github.com/ClickHouse/ClickHouse/issues/8696)). [\#8707](https://github.com/ClickHouse/ClickHouse/pull/8707) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- The setting `experimental_use_processors` is enabled by default. This setting enables usage of the new query pipeline. This is internal refactoring and we expect no visible changes. If you will see any issues, set it to back zero. [\#8768](https://github.com/ClickHouse/ClickHouse/pull/8768) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -#### New Feature {#new-feature} - -- Add `Avro` and `AvroConfluent` input/output formats [\#8571](https://github.com/ClickHouse/ClickHouse/pull/8571) ([Andrew Onyshchuk](https://github.com/oandrew)) [\#8957](https://github.com/ClickHouse/ClickHouse/pull/8957) ([Andrew Onyshchuk](https://github.com/oandrew)) [\#8717](https://github.com/ClickHouse/ClickHouse/pull/8717) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Multi-threaded and non-blocking updates of expired keys in `cache` dictionaries (with optional permission to read old ones). [\#8303](https://github.com/ClickHouse/ClickHouse/pull/8303) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- Add query `ALTER ... MATERIALIZE TTL`. It runs mutation that forces to remove expired data by TTL and recalculates meta-information about TTL in all parts. [\#8775](https://github.com/ClickHouse/ClickHouse/pull/8775) ([Anton Popov](https://github.com/CurtizJ)) -- Switch from HashJoin to MergeJoin (on disk) if needed [\#9082](https://github.com/ClickHouse/ClickHouse/pull/9082) ([Artem Zuikov](https://github.com/4ertus2)) -- Added `MOVE PARTITION` command for `ALTER TABLE` [\#4729](https://github.com/ClickHouse/ClickHouse/issues/4729) [\#6168](https://github.com/ClickHouse/ClickHouse/pull/6168) ([Guillaume Tassery](https://github.com/YiuRULE)) -- Reloading storage configuration from configuration file on the fly. [\#8594](https://github.com/ClickHouse/ClickHouse/pull/8594) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Allowed to change `storage_policy` to not less rich one. [\#8107](https://github.com/ClickHouse/ClickHouse/pull/8107) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Added support for globs/wildcards for S3 storage and table function. [\#8851](https://github.com/ClickHouse/ClickHouse/pull/8851) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Implement `bitAnd`, `bitOr`, `bitXor`, `bitNot` for `FixedString(N)` datatype. [\#9091](https://github.com/ClickHouse/ClickHouse/pull/9091) ([Guillaume Tassery](https://github.com/YiuRULE)) -- Added function `bitCount`. This fixes [\#8702](https://github.com/ClickHouse/ClickHouse/issues/8702). [\#8708](https://github.com/ClickHouse/ClickHouse/pull/8708) ([alexey-milovidov](https://github.com/alexey-milovidov)) [\#8749](https://github.com/ClickHouse/ClickHouse/pull/8749) ([ikopylov](https://github.com/ikopylov)) -- Add `generateRandom` table function to generate random rows with given schema. Allows to populate arbitrary test table with data. [\#8994](https://github.com/ClickHouse/ClickHouse/pull/8994) ([Ilya Yatsishin](https://github.com/qoega)) -- `JSONEachRowFormat`: support special case when objects enclosed in top-level array. [\#8860](https://github.com/ClickHouse/ClickHouse/pull/8860) ([Kruglov Pavel](https://github.com/Avogar)) -- Now it’s possible to create a column with `DEFAULT` expression which depends on a column with default `ALIAS` expression. [\#9489](https://github.com/ClickHouse/ClickHouse/pull/9489) ([alesapin](https://github.com/alesapin)) -- Allow to specify `--limit` more than the source data size in `clickhouse-obfuscator`. The data will repeat itself with different random seed. [\#9155](https://github.com/ClickHouse/ClickHouse/pull/9155) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Added `groupArraySample` function (similar to `groupArray`) with reservior sampling algorithm. [\#8286](https://github.com/ClickHouse/ClickHouse/pull/8286) ([Amos Bird](https://github.com/amosbird)) -- Now you can monitor the size of update queue in `cache`/`complex_key_cache` dictionaries via system metrics. [\#9413](https://github.com/ClickHouse/ClickHouse/pull/9413) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- Allow to use CRLF as a line separator in CSV output format with setting `output_format_csv_crlf_end_of_line` is set to 1 [\#8934](https://github.com/ClickHouse/ClickHouse/pull/8934) [\#8935](https://github.com/ClickHouse/ClickHouse/pull/8935) [\#8963](https://github.com/ClickHouse/ClickHouse/pull/8963) ([Mikhail Korotov](https://github.com/millb)) -- Implement more functions of the [H3](https://github.com/uber/h3) API: `h3GetBaseCell`, `h3HexAreaM2`, `h3IndexesAreNeighbors`, `h3ToChildren`, `h3ToString` and `stringToH3` [\#8938](https://github.com/ClickHouse/ClickHouse/pull/8938) ([Nico Mandery](https://github.com/nmandery)) -- New setting introduced: `max_parser_depth` to control maximum stack size and allow large complex queries. This fixes [\#6681](https://github.com/ClickHouse/ClickHouse/issues/6681) and [\#7668](https://github.com/ClickHouse/ClickHouse/issues/7668). [\#8647](https://github.com/ClickHouse/ClickHouse/pull/8647) ([Maxim Smirnov](https://github.com/qMBQx8GH)) -- Add a setting `force_optimize_skip_unused_shards` setting to throw if skipping of unused shards is not possible [\#8805](https://github.com/ClickHouse/ClickHouse/pull/8805) ([Azat Khuzhin](https://github.com/azat)) -- Allow to configure multiple disks/volumes for storing data for send in `Distributed` engine [\#8756](https://github.com/ClickHouse/ClickHouse/pull/8756) ([Azat Khuzhin](https://github.com/azat)) -- Support storage policy (``) for storing temporary data. [\#8750](https://github.com/ClickHouse/ClickHouse/pull/8750) ([Azat Khuzhin](https://github.com/azat)) -- Added `X-ClickHouse-Exception-Code` HTTP header that is set if exception was thrown before sending data. This implements [\#4971](https://github.com/ClickHouse/ClickHouse/issues/4971). [\#8786](https://github.com/ClickHouse/ClickHouse/pull/8786) ([Mikhail Korotov](https://github.com/millb)) -- Added function `ifNotFinite`. It is just a syntactic sugar: `ifNotFinite(x, y) = isFinite(x) ? x : y`. [\#8710](https://github.com/ClickHouse/ClickHouse/pull/8710) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Added `last_successful_update_time` column in `system.dictionaries` table [\#9394](https://github.com/ClickHouse/ClickHouse/pull/9394) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- Add `blockSerializedSize` function (size on disk without compression) [\#8952](https://github.com/ClickHouse/ClickHouse/pull/8952) ([Azat Khuzhin](https://github.com/azat)) -- Add function `moduloOrZero` [\#9358](https://github.com/ClickHouse/ClickHouse/pull/9358) ([hcz](https://github.com/hczhcz)) -- Added system tables `system.zeros` and `system.zeros_mt` as well as tale functions `zeros()` and `zeros_mt()`. Tables (and table functions) contain single column with name `zero` and type `UInt8`. This column contains zeros. It is needed for test purposes as the fastest method to generate many rows. This fixes [\#6604](https://github.com/ClickHouse/ClickHouse/issues/6604) [\#9593](https://github.com/ClickHouse/ClickHouse/pull/9593) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) - -#### Experimental Feature {#experimental-feature} - -- Add new compact format of parts in `MergeTree`-family tables in which all columns are stored in one file. It helps to increase performance of small and frequent inserts. The old format (one file per column) is now called wide. Data storing format is controlled by settings `min_bytes_for_wide_part` and `min_rows_for_wide_part`. [\#8290](https://github.com/ClickHouse/ClickHouse/pull/8290) ([Anton Popov](https://github.com/CurtizJ)) -- Support for S3 storage for `Log`, `TinyLog` and `StripeLog` tables. [\#8862](https://github.com/ClickHouse/ClickHouse/pull/8862) ([Pavel Kovalenko](https://github.com/Jokser)) - -#### Bug Fix {#bug-fix-2} - -- Fixed inconsistent whitespaces in log messages. [\#9322](https://github.com/ClickHouse/ClickHouse/pull/9322) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix bug in which arrays of unnamed tuples were flattened as Nested structures on table creation. [\#8866](https://github.com/ClickHouse/ClickHouse/pull/8866) ([achulkov2](https://github.com/achulkov2)) -- Fixed the issue when “Too many open files” error may happen if there are too many files matching glob pattern in `File` table or `file` table function. Now files are opened lazily. This fixes [\#8857](https://github.com/ClickHouse/ClickHouse/issues/8857) [\#8861](https://github.com/ClickHouse/ClickHouse/pull/8861) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- DROP TEMPORARY TABLE now drops only temporary table. [\#8907](https://github.com/ClickHouse/ClickHouse/pull/8907) ([Vitaly Baranov](https://github.com/vitlibar)) -- Remove outdated partition when we shutdown the server or DETACH/ATTACH a table. [\#8602](https://github.com/ClickHouse/ClickHouse/pull/8602) ([Guillaume Tassery](https://github.com/YiuRULE)) -- For how the default disk calculates the free space from `data` subdirectory. Fixed the issue when the amount of free space is not calculated correctly if the `data` directory is mounted to a separate device (rare case). This fixes [\#7441](https://github.com/ClickHouse/ClickHouse/issues/7441) [\#9257](https://github.com/ClickHouse/ClickHouse/pull/9257) ([Mikhail Korotov](https://github.com/millb)) -- Allow comma (cross) join with IN () inside. [\#9251](https://github.com/ClickHouse/ClickHouse/pull/9251) ([Artem Zuikov](https://github.com/4ertus2)) -- Allow to rewrite CROSS to INNER JOIN if there’s \[NOT\] LIKE operator in WHERE section. [\#9229](https://github.com/ClickHouse/ClickHouse/pull/9229) ([Artem Zuikov](https://github.com/4ertus2)) -- Fix possible incorrect result after `GROUP BY` with enabled setting `distributed_aggregation_memory_efficient`. Fixes [\#9134](https://github.com/ClickHouse/ClickHouse/issues/9134). [\#9289](https://github.com/ClickHouse/ClickHouse/pull/9289) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Found keys were counted as missed in metrics of cache dictionaries. [\#9411](https://github.com/ClickHouse/ClickHouse/pull/9411) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- Fix replication protocol incompatibility introduced in [\#8598](https://github.com/ClickHouse/ClickHouse/issues/8598). [\#9412](https://github.com/ClickHouse/ClickHouse/pull/9412) ([alesapin](https://github.com/alesapin)) -- Fixed race condition on `queue_task_handle` at the startup of `ReplicatedMergeTree` tables. [\#9552](https://github.com/ClickHouse/ClickHouse/pull/9552) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- The token `NOT` didn’t work in `SHOW TABLES NOT LIKE` query [\#8727](https://github.com/ClickHouse/ClickHouse/issues/8727) [\#8940](https://github.com/ClickHouse/ClickHouse/pull/8940) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Added range check to function `h3EdgeLengthM`. Without this check, buffer overflow is possible. [\#8945](https://github.com/ClickHouse/ClickHouse/pull/8945) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed up a bug in batched calculations of ternary logical OPs on multiple arguments (more than 10). [\#8718](https://github.com/ClickHouse/ClickHouse/pull/8718) ([Alexander Kazakov](https://github.com/Akazz)) -- Fix error of PREWHERE optimization, which could lead to segfaults or `Inconsistent number of columns got from MergeTreeRangeReader` exception. [\#9024](https://github.com/ClickHouse/ClickHouse/pull/9024) ([Anton Popov](https://github.com/CurtizJ)) -- Fix unexpected `Timeout exceeded while reading from socket` exception, which randomly happens on secure connection before timeout actually exceeded and when query profiler is enabled. Also add `connect_timeout_with_failover_secure_ms` settings (default 100ms), which is similar to `connect_timeout_with_failover_ms`, but is used for secure connections (because SSL handshake is slower, than ordinary TCP connection) [\#9026](https://github.com/ClickHouse/ClickHouse/pull/9026) ([tavplubix](https://github.com/tavplubix)) -- Fix bug with mutations finalization, when mutation may hang in state with `parts_to_do=0` and `is_done=0`. [\#9022](https://github.com/ClickHouse/ClickHouse/pull/9022) ([alesapin](https://github.com/alesapin)) -- Use new ANY JOIN logic with `partial_merge_join` setting. It’s possible to make `ANY|ALL|SEMI LEFT` and `ALL INNER` joins with `partial_merge_join=1` now. [\#8932](https://github.com/ClickHouse/ClickHouse/pull/8932) ([Artem Zuikov](https://github.com/4ertus2)) -- Shard now clamps the settings got from the initiator to the shard’s constaints instead of throwing an exception. This fix allows to send queries to a shard with another constraints. [\#9447](https://github.com/ClickHouse/ClickHouse/pull/9447) ([Vitaly Baranov](https://github.com/vitlibar)) -- Fixed memory management problem in `MergeTreeReadPool`. [\#8791](https://github.com/ClickHouse/ClickHouse/pull/8791) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Fix `toDecimal*OrNull()` functions family when called with string `e`. Fixes [\#8312](https://github.com/ClickHouse/ClickHouse/issues/8312) [\#8764](https://github.com/ClickHouse/ClickHouse/pull/8764) ([Artem Zuikov](https://github.com/4ertus2)) -- Make sure that `FORMAT Null` sends no data to the client. [\#8767](https://github.com/ClickHouse/ClickHouse/pull/8767) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- Fix bug that timestamp in `LiveViewBlockInputStream` will not updated. `LIVE VIEW` is an experimental feature. [\#8644](https://github.com/ClickHouse/ClickHouse/pull/8644) ([vxider](https://github.com/Vxider)) [\#8625](https://github.com/ClickHouse/ClickHouse/pull/8625) ([vxider](https://github.com/Vxider)) -- Fixed `ALTER MODIFY TTL` wrong behavior which did not allow to delete old TTL expressions. [\#8422](https://github.com/ClickHouse/ClickHouse/pull/8422) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Fixed UBSan report in MergeTreeIndexSet. This fixes [\#9250](https://github.com/ClickHouse/ClickHouse/issues/9250) [\#9365](https://github.com/ClickHouse/ClickHouse/pull/9365) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed the behaviour of `match` and `extract` functions when haystack has zero bytes. The behaviour was wrong when haystack was constant. This fixes [\#9160](https://github.com/ClickHouse/ClickHouse/issues/9160) [\#9163](https://github.com/ClickHouse/ClickHouse/pull/9163) ([alexey-milovidov](https://github.com/alexey-milovidov)) [\#9345](https://github.com/ClickHouse/ClickHouse/pull/9345) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Avoid throwing from destructor in Apache Avro 3rd-party library. [\#9066](https://github.com/ClickHouse/ClickHouse/pull/9066) ([Andrew Onyshchuk](https://github.com/oandrew)) -- Don’t commit a batch polled from `Kafka` partially as it can lead to holes in data. [\#8876](https://github.com/ClickHouse/ClickHouse/pull/8876) ([filimonov](https://github.com/filimonov)) -- Fix `joinGet` with nullable return types. https://github.com/ClickHouse/ClickHouse/issues/8919 [\#9014](https://github.com/ClickHouse/ClickHouse/pull/9014) ([Amos Bird](https://github.com/amosbird)) -- Fix data incompatibility when compressed with `T64` codec. [\#9016](https://github.com/ClickHouse/ClickHouse/pull/9016) ([Artem Zuikov](https://github.com/4ertus2)) Fix data type ids in `T64` compression codec that leads to wrong (de)compression in affected versions. [\#9033](https://github.com/ClickHouse/ClickHouse/pull/9033) ([Artem Zuikov](https://github.com/4ertus2)) -- Add setting `enable_early_constant_folding` and disable it in some cases that leads to errors. [\#9010](https://github.com/ClickHouse/ClickHouse/pull/9010) ([Artem Zuikov](https://github.com/4ertus2)) -- Fix pushdown predicate optimizer with VIEW and enable the test [\#9011](https://github.com/ClickHouse/ClickHouse/pull/9011) ([Winter Zhang](https://github.com/zhang2014)) -- Fix segfault in `Merge` tables, that can happen when reading from `File` storages [\#9387](https://github.com/ClickHouse/ClickHouse/pull/9387) ([tavplubix](https://github.com/tavplubix)) -- Added a check for storage policy in `ATTACH PARTITION FROM`, `REPLACE PARTITION`, `MOVE TO TABLE`. Otherwise it could make data of part inaccessible after restart and prevent ClickHouse to start. [\#9383](https://github.com/ClickHouse/ClickHouse/pull/9383) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Fix alters if there is TTL set for table. [\#8800](https://github.com/ClickHouse/ClickHouse/pull/8800) ([Anton Popov](https://github.com/CurtizJ)) -- Fix race condition that can happen when `SYSTEM RELOAD ALL DICTIONARIES` is executed while some dictionary is being modified/added/removed. [\#8801](https://github.com/ClickHouse/ClickHouse/pull/8801) ([Vitaly Baranov](https://github.com/vitlibar)) -- In previous versions `Memory` database engine use empty data path, so tables are created in `path` directory (e.g. `/var/lib/clickhouse/`), not in data directory of database (e.g. `/var/lib/clickhouse/db_name`). [\#8753](https://github.com/ClickHouse/ClickHouse/pull/8753) ([tavplubix](https://github.com/tavplubix)) -- Fixed wrong log messages about missing default disk or policy. [\#9530](https://github.com/ClickHouse/ClickHouse/pull/9530) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Fix not(has()) for the bloom\_filter index of array types. [\#9407](https://github.com/ClickHouse/ClickHouse/pull/9407) ([achimbab](https://github.com/achimbab)) -- Allow first column(s) in a table with `Log` engine be an alias [\#9231](https://github.com/ClickHouse/ClickHouse/pull/9231) ([Ivan](https://github.com/abyss7)) -- Fix order of ranges while reading from `MergeTree` table in one thread. It could lead to exceptions from `MergeTreeRangeReader` or wrong query results. [\#9050](https://github.com/ClickHouse/ClickHouse/pull/9050) ([Anton Popov](https://github.com/CurtizJ)) -- Make `reinterpretAsFixedString` to return `FixedString` instead of `String`. [\#9052](https://github.com/ClickHouse/ClickHouse/pull/9052) ([Andrew Onyshchuk](https://github.com/oandrew)) -- Avoid extremely rare cases when the user can get wrong error message (`Success` instead of detailed error description). [\#9457](https://github.com/ClickHouse/ClickHouse/pull/9457) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Do not crash when using `Template` format with empty row template. [\#8785](https://github.com/ClickHouse/ClickHouse/pull/8785) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- Metadata files for system tables could be created in wrong place [\#8653](https://github.com/ClickHouse/ClickHouse/pull/8653) ([tavplubix](https://github.com/tavplubix)) Fixes [\#8581](https://github.com/ClickHouse/ClickHouse/issues/8581). -- Fix data race on exception\_ptr in cache dictionary [\#8303](https://github.com/ClickHouse/ClickHouse/issues/8303). [\#9379](https://github.com/ClickHouse/ClickHouse/pull/9379) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- Do not throw an exception for query `ATTACH TABLE IF NOT EXISTS`. Previously it was thrown if table already exists, despite the `IF NOT EXISTS` clause. [\#8967](https://github.com/ClickHouse/ClickHouse/pull/8967) ([Anton Popov](https://github.com/CurtizJ)) -- Fixed missing closing paren in exception message. [\#8811](https://github.com/ClickHouse/ClickHouse/pull/8811) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Avoid message `Possible deadlock avoided` at the startup of clickhouse-client in interactive mode. [\#9455](https://github.com/ClickHouse/ClickHouse/pull/9455) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed the issue when padding at the end of base64 encoded value can be malformed. Update base64 library. This fixes [\#9491](https://github.com/ClickHouse/ClickHouse/issues/9491), closes [\#9492](https://github.com/ClickHouse/ClickHouse/issues/9492) [\#9500](https://github.com/ClickHouse/ClickHouse/pull/9500) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Prevent losing data in `Kafka` in rare cases when exception happens after reading suffix but before commit. Fixes [\#9378](https://github.com/ClickHouse/ClickHouse/issues/9378) [\#9507](https://github.com/ClickHouse/ClickHouse/pull/9507) ([filimonov](https://github.com/filimonov)) -- Fixed exception in `DROP TABLE IF EXISTS` [\#8663](https://github.com/ClickHouse/ClickHouse/pull/8663) ([Nikita Vasilev](https://github.com/nikvas0)) -- Fix crash when a user tries to `ALTER MODIFY SETTING` for old-formated `MergeTree` table engines family. [\#9435](https://github.com/ClickHouse/ClickHouse/pull/9435) ([alesapin](https://github.com/alesapin)) -- Support for UInt64 numbers that don’t fit in Int64 in JSON-related functions. Update SIMDJSON to master. This fixes [\#9209](https://github.com/ClickHouse/ClickHouse/issues/9209) [\#9344](https://github.com/ClickHouse/ClickHouse/pull/9344) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed execution of inversed predicates when non-strictly monotinic functional index is used. [\#9223](https://github.com/ClickHouse/ClickHouse/pull/9223) ([Alexander Kazakov](https://github.com/Akazz)) -- Don’t try to fold `IN` constant in `GROUP BY` [\#8868](https://github.com/ClickHouse/ClickHouse/pull/8868) ([Amos Bird](https://github.com/amosbird)) -- Fix bug in `ALTER DELETE` mutations which leads to index corruption. This fixes [\#9019](https://github.com/ClickHouse/ClickHouse/issues/9019) and [\#8982](https://github.com/ClickHouse/ClickHouse/issues/8982). Additionally fix extremely rare race conditions in `ReplicatedMergeTree` `ALTER` queries. [\#9048](https://github.com/ClickHouse/ClickHouse/pull/9048) ([alesapin](https://github.com/alesapin)) -- When the setting `compile_expressions` is enabled, you can get `unexpected column` in `LLVMExecutableFunction` when we use `Nullable` type [\#8910](https://github.com/ClickHouse/ClickHouse/pull/8910) ([Guillaume Tassery](https://github.com/YiuRULE)) -- Multiple fixes for `Kafka` engine: 1) fix duplicates that were appearing during consumer group rebalance. 2) Fix rare ‘holes’ appeared when data were polled from several partitions with one poll and committed partially (now we always process / commit the whole polled block of messages). 3) Fix flushes by block size (before that only flushing by timeout was working properly). 4) better subscription procedure (with assignment feedback). 5) Make tests work faster (with default intervals and timeouts). Due to the fact that data was not flushed by block size before (as it should according to documentation), that PR may lead to some performance degradation with default settings (due to more often & tinier flushes which are less optimal). If you encounter the performance issue after that change - please increase `kafka_max_block_size` in the table to the bigger value ( for example `CREATE TABLE ...Engine=Kafka ... SETTINGS ... kafka_max_block_size=524288`). Fixes [\#7259](https://github.com/ClickHouse/ClickHouse/issues/7259) [\#8917](https://github.com/ClickHouse/ClickHouse/pull/8917) ([filimonov](https://github.com/filimonov)) -- Fix `Parameter out of bound` exception in some queries after PREWHERE optimizations. [\#8914](https://github.com/ClickHouse/ClickHouse/pull/8914) ([Baudouin Giard](https://github.com/bgiard)) -- Fixed the case of mixed-constness of arguments of function `arrayZip`. [\#8705](https://github.com/ClickHouse/ClickHouse/pull/8705) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- When executing `CREATE` query, fold constant expressions in storage engine arguments. Replace empty database name with current database. Fixes [\#6508](https://github.com/ClickHouse/ClickHouse/issues/6508), [\#3492](https://github.com/ClickHouse/ClickHouse/issues/3492) [\#9262](https://github.com/ClickHouse/ClickHouse/pull/9262) ([tavplubix](https://github.com/tavplubix)) -- Now it’s not possible to create or add columns with simple cyclic aliases like `a DEFAULT b, b DEFAULT a`. [\#9603](https://github.com/ClickHouse/ClickHouse/pull/9603) ([alesapin](https://github.com/alesapin)) -- Fixed a bug with double move which may corrupt original part. This is relevant if you use `ALTER TABLE MOVE` [\#8680](https://github.com/ClickHouse/ClickHouse/pull/8680) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Allow `interval` identifier to correctly parse without backticks. Fixed issue when a query cannot be executed even if the `interval` identifier is enclosed in backticks or double quotes. This fixes [\#9124](https://github.com/ClickHouse/ClickHouse/issues/9124). [\#9142](https://github.com/ClickHouse/ClickHouse/pull/9142) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed fuzz test and incorrect behaviour of `bitTestAll`/`bitTestAny` functions. [\#9143](https://github.com/ClickHouse/ClickHouse/pull/9143) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix possible crash/wrong number of rows in `LIMIT n WITH TIES` when there are a lot of rows equal to n’th row. [\#9464](https://github.com/ClickHouse/ClickHouse/pull/9464) ([tavplubix](https://github.com/tavplubix)) -- Fix mutations with parts written with enabled `insert_quorum`. [\#9463](https://github.com/ClickHouse/ClickHouse/pull/9463) ([alesapin](https://github.com/alesapin)) -- Fix data race at destruction of `Poco::HTTPServer`. It could happen when server is started and immediately shut down. [\#9468](https://github.com/ClickHouse/ClickHouse/pull/9468) ([Anton Popov](https://github.com/CurtizJ)) -- Fix bug in which a misleading error message was shown when running `SHOW CREATE TABLE a_table_that_does_not_exist`. [\#8899](https://github.com/ClickHouse/ClickHouse/pull/8899) ([achulkov2](https://github.com/achulkov2)) -- Fixed `Parameters are out of bound` exception in some rare cases when we have a constant in the `SELECT` clause when we have an `ORDER BY` and a `LIMIT` clause. [\#8892](https://github.com/ClickHouse/ClickHouse/pull/8892) ([Guillaume Tassery](https://github.com/YiuRULE)) -- Fix mutations finalization, when already done mutation can have status `is_done=0`. [\#9217](https://github.com/ClickHouse/ClickHouse/pull/9217) ([alesapin](https://github.com/alesapin)) -- Prevent from executing `ALTER ADD INDEX` for MergeTree tables with old syntax, because it doesn’t work. [\#8822](https://github.com/ClickHouse/ClickHouse/pull/8822) ([Mikhail Korotov](https://github.com/millb)) -- During server startup do not access table, which `LIVE VIEW` depends on, so server will be able to start. Also remove `LIVE VIEW` dependencies when detaching `LIVE VIEW`. `LIVE VIEW` is an experimental feature. [\#8824](https://github.com/ClickHouse/ClickHouse/pull/8824) ([tavplubix](https://github.com/tavplubix)) -- Fix possible segfault in `MergeTreeRangeReader`, while executing `PREWHERE`. [\#9106](https://github.com/ClickHouse/ClickHouse/pull/9106) ([Anton Popov](https://github.com/CurtizJ)) -- Fix possible mismatched checksums with column TTLs. [\#9451](https://github.com/ClickHouse/ClickHouse/pull/9451) ([Anton Popov](https://github.com/CurtizJ)) -- Fixed a bug when parts were not being moved in background by TTL rules in case when there is only one volume. [\#8672](https://github.com/ClickHouse/ClickHouse/pull/8672) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Fixed the issue `Method createColumn() is not implemented for data type Set`. This fixes [\#7799](https://github.com/ClickHouse/ClickHouse/issues/7799). [\#8674](https://github.com/ClickHouse/ClickHouse/pull/8674) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Now we will try finalize mutations more frequently. [\#9427](https://github.com/ClickHouse/ClickHouse/pull/9427) ([alesapin](https://github.com/alesapin)) -- Fix `intDiv` by minus one constant [\#9351](https://github.com/ClickHouse/ClickHouse/pull/9351) ([hcz](https://github.com/hczhcz)) -- Fix possible race condition in `BlockIO`. [\#9356](https://github.com/ClickHouse/ClickHouse/pull/9356) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Fix bug leading to server termination when trying to use / drop `Kafka` table created with wrong parameters. [\#9513](https://github.com/ClickHouse/ClickHouse/pull/9513) ([filimonov](https://github.com/filimonov)) -- Added workaround if OS returns wrong result for `timer_create` function. [\#8837](https://github.com/ClickHouse/ClickHouse/pull/8837) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed error in usage of `min_marks_for_seek` parameter. Fixed the error message when there is no sharding key in Distributed table and we try to skip unused shards. [\#8908](https://github.com/ClickHouse/ClickHouse/pull/8908) ([Azat Khuzhin](https://github.com/azat)) - -#### Improvement {#improvement} - -- Implement `ALTER MODIFY/DROP` queries on top of mutations for `ReplicatedMergeTree*` engines family. Now `ALTERS` blocks only at the metadata update stage, and don’t block after that. [\#8701](https://github.com/ClickHouse/ClickHouse/pull/8701) ([alesapin](https://github.com/alesapin)) -- Add ability to rewrite CROSS to INNER JOINs with `WHERE` section containing unqialified names. [\#9512](https://github.com/ClickHouse/ClickHouse/pull/9512) ([Artem Zuikov](https://github.com/4ertus2)) -- Make `SHOW TABLES` and `SHOW DATABASES` queries support the `WHERE` expressions and `FROM`/`IN` [\#9076](https://github.com/ClickHouse/ClickHouse/pull/9076) ([sundyli](https://github.com/sundy-li)) -- Added a setting `deduplicate_blocks_in_dependent_materialized_views`. [\#9070](https://github.com/ClickHouse/ClickHouse/pull/9070) ([urykhy](https://github.com/urykhy)) -- After recent changes MySQL client started to print binary strings in hex thereby making them not readable ([\#9032](https://github.com/ClickHouse/ClickHouse/issues/9032)). The workaround in ClickHouse is to mark string columns as UTF-8, which is not always, but usually the case. [\#9079](https://github.com/ClickHouse/ClickHouse/pull/9079) ([Yuriy Baranov](https://github.com/yurriy)) -- Add support of String and FixedString keys for `sumMap` [\#8903](https://github.com/ClickHouse/ClickHouse/pull/8903) ([Baudouin Giard](https://github.com/bgiard)) -- Support string keys in SummingMergeTree maps [\#8933](https://github.com/ClickHouse/ClickHouse/pull/8933) ([Baudouin Giard](https://github.com/bgiard)) -- Signal termination of thread to the thread pool even if the thread has thrown exception [\#8736](https://github.com/ClickHouse/ClickHouse/pull/8736) ([Ding Xiang Fei](https://github.com/dingxiangfei2009)) -- Allow to set `query_id` in `clickhouse-benchmark` [\#9416](https://github.com/ClickHouse/ClickHouse/pull/9416) ([Anton Popov](https://github.com/CurtizJ)) -- Don’t allow strange expressions in `ALTER TABLE ... PARTITION partition` query. This addresses [\#7192](https://github.com/ClickHouse/ClickHouse/issues/7192) [\#8835](https://github.com/ClickHouse/ClickHouse/pull/8835) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- The table `system.table_engines` now provides information about feature support (like `supports_ttl` or `supports_sort_order`). [\#8830](https://github.com/ClickHouse/ClickHouse/pull/8830) ([Max Akhmedov](https://github.com/zlobober)) -- Enable `system.metric_log` by default. It will contain rows with values of ProfileEvents, CurrentMetrics collected with “collect\_interval\_milliseconds” interval (one second by default). The table is very small (usually in order of megabytes) and collecting this data by default is reasonable. [\#9225](https://github.com/ClickHouse/ClickHouse/pull/9225) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Initialize query profiler for all threads in a group, e.g. it allows to fully profile insert-queries. Fixes [\#6964](https://github.com/ClickHouse/ClickHouse/issues/6964) [\#8874](https://github.com/ClickHouse/ClickHouse/pull/8874) ([Ivan](https://github.com/abyss7)) -- Now temporary `LIVE VIEW` is created by `CREATE LIVE VIEW name WITH TIMEOUT [42] ...` instead of `CREATE TEMPORARY LIVE VIEW ...`, because the previous syntax was not consistent with `CREATE TEMPORARY TABLE ...` [\#9131](https://github.com/ClickHouse/ClickHouse/pull/9131) ([tavplubix](https://github.com/tavplubix)) -- Add text\_log.level configuration parameter to limit entries that goes to `system.text_log` table [\#8809](https://github.com/ClickHouse/ClickHouse/pull/8809) ([Azat Khuzhin](https://github.com/azat)) -- Allow to put downloaded part to a disks/volumes according to TTL rules [\#8598](https://github.com/ClickHouse/ClickHouse/pull/8598) ([Vladimir Chebotarev](https://github.com/excitoon)) -- For external MySQL dictionaries, allow to mutualize MySQL connection pool to “share” them among dictionaries. This option significantly reduces the number of connections to MySQL servers. [\#9409](https://github.com/ClickHouse/ClickHouse/pull/9409) ([Clément Rodriguez](https://github.com/clemrodriguez)) -- Show nearest query execution time for quantiles in `clickhouse-benchmark` output instead of interpolated values. It’s better to show values that correspond to the execution time of some queries. [\#8712](https://github.com/ClickHouse/ClickHouse/pull/8712) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Possibility to add key & timestamp for the message when inserting data to Kafka. Fixes [\#7198](https://github.com/ClickHouse/ClickHouse/issues/7198) [\#8969](https://github.com/ClickHouse/ClickHouse/pull/8969) ([filimonov](https://github.com/filimonov)) -- If server is run from terminal, highlight thread number, query id and log priority by colors. This is for improved readability of correlated log messages for developers. [\#8961](https://github.com/ClickHouse/ClickHouse/pull/8961) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Better exception message while loading tables for `Ordinary` database. [\#9527](https://github.com/ClickHouse/ClickHouse/pull/9527) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Implement `arraySlice` for arrays with aggregate function states. This fixes [\#9388](https://github.com/ClickHouse/ClickHouse/issues/9388) [\#9391](https://github.com/ClickHouse/ClickHouse/pull/9391) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Allow constant functions and constant arrays to be used on the right side of IN operator. [\#8813](https://github.com/ClickHouse/ClickHouse/pull/8813) ([Anton Popov](https://github.com/CurtizJ)) -- If zookeeper exception has happened while fetching data for system.replicas, display it in a separate column. This implements [\#9137](https://github.com/ClickHouse/ClickHouse/issues/9137) [\#9138](https://github.com/ClickHouse/ClickHouse/pull/9138) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Atomically remove MergeTree data parts on destroy. [\#8402](https://github.com/ClickHouse/ClickHouse/pull/8402) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Support row-level security for Distributed tables. [\#8926](https://github.com/ClickHouse/ClickHouse/pull/8926) ([Ivan](https://github.com/abyss7)) -- Now we recognize suffix (like KB, KiB…) in settings values. [\#8072](https://github.com/ClickHouse/ClickHouse/pull/8072) ([Mikhail Korotov](https://github.com/millb)) -- Prevent out of memory while constructing result of a large JOIN. [\#8637](https://github.com/ClickHouse/ClickHouse/pull/8637) ([Artem Zuikov](https://github.com/4ertus2)) -- Added names of clusters to suggestions in interactive mode in `clickhouse-client`. [\#8709](https://github.com/ClickHouse/ClickHouse/pull/8709) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Initialize query profiler for all threads in a group, e.g. it allows to fully profile insert-queries [\#8820](https://github.com/ClickHouse/ClickHouse/pull/8820) ([Ivan](https://github.com/abyss7)) -- Added column `exception_code` in `system.query_log` table. [\#8770](https://github.com/ClickHouse/ClickHouse/pull/8770) ([Mikhail Korotov](https://github.com/millb)) -- Enabled MySQL compatibility server on port `9004` in the default server configuration file. Fixed password generation command in the example in configuration. [\#8771](https://github.com/ClickHouse/ClickHouse/pull/8771) ([Yuriy Baranov](https://github.com/yurriy)) -- Prevent abort on shutdown if the filesystem is readonly. This fixes [\#9094](https://github.com/ClickHouse/ClickHouse/issues/9094) [\#9100](https://github.com/ClickHouse/ClickHouse/pull/9100) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Better exception message when length is required in HTTP POST query. [\#9453](https://github.com/ClickHouse/ClickHouse/pull/9453) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Add `_path` and `_file` virtual columns to `HDFS` and `File` engines and `hdfs` and `file` table functions [\#8489](https://github.com/ClickHouse/ClickHouse/pull/8489) ([Olga Khvostikova](https://github.com/stavrolia)) -- Fix error `Cannot find column` while inserting into `MATERIALIZED VIEW` in case if new column was added to view’s internal table. [\#8766](https://github.com/ClickHouse/ClickHouse/pull/8766) [\#8788](https://github.com/ClickHouse/ClickHouse/pull/8788) ([vzakaznikov](https://github.com/vzakaznikov)) [\#8788](https://github.com/ClickHouse/ClickHouse/issues/8788) [\#8806](https://github.com/ClickHouse/ClickHouse/pull/8806) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) [\#8803](https://github.com/ClickHouse/ClickHouse/pull/8803) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Fix progress over native client-server protocol, by send progress after final update (like logs). This may be relevant only to some third-party tools that are using native protocol. [\#9495](https://github.com/ClickHouse/ClickHouse/pull/9495) ([Azat Khuzhin](https://github.com/azat)) -- Add a system metric tracking the number of client connections using MySQL protocol ([\#9013](https://github.com/ClickHouse/ClickHouse/issues/9013)). [\#9015](https://github.com/ClickHouse/ClickHouse/pull/9015) ([Eugene Klimov](https://github.com/Slach)) -- From now on, HTTP responses will have `X-ClickHouse-Timezone` header set to the same timezone value that `SELECT timezone()` would report. [\#9493](https://github.com/ClickHouse/ClickHouse/pull/9493) ([Denis Glazachev](https://github.com/traceon)) - -#### Performance Improvement {#performance-improvement} - -- Improve performance of analysing index with IN [\#9261](https://github.com/ClickHouse/ClickHouse/pull/9261) ([Anton Popov](https://github.com/CurtizJ)) -- Simpler and more efficient code in Logical Functions + code cleanups. A followup to [\#8718](https://github.com/ClickHouse/ClickHouse/issues/8718) [\#8728](https://github.com/ClickHouse/ClickHouse/pull/8728) ([Alexander Kazakov](https://github.com/Akazz)) -- Overall performance improvement (in range of 5%..200% for affected queries) by ensuring even more strict aliasing with C++20 features. [\#9304](https://github.com/ClickHouse/ClickHouse/pull/9304) ([Amos Bird](https://github.com/amosbird)) -- More strict aliasing for inner loops of comparison functions. [\#9327](https://github.com/ClickHouse/ClickHouse/pull/9327) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- More strict aliasing for inner loops of arithmetic functions. [\#9325](https://github.com/ClickHouse/ClickHouse/pull/9325) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- A ~3 times faster implementation for ColumnVector::replicate(), via which ColumnConst::convertToFullColumn() is implemented. Also will be useful in tests when materializing constants. [\#9293](https://github.com/ClickHouse/ClickHouse/pull/9293) ([Alexander Kazakov](https://github.com/Akazz)) -- Another minor performance improvement to `ColumnVector::replicate()` (this speeds up the `materialize` function and higher order functions) an even further improvement to [\#9293](https://github.com/ClickHouse/ClickHouse/issues/9293) [\#9442](https://github.com/ClickHouse/ClickHouse/pull/9442) ([Alexander Kazakov](https://github.com/Akazz)) -- Improved performance of `stochasticLinearRegression` aggregate function. This patch is contributed by Intel. [\#8652](https://github.com/ClickHouse/ClickHouse/pull/8652) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Improve performance of `reinterpretAsFixedString` function. [\#9342](https://github.com/ClickHouse/ClickHouse/pull/9342) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Do not send blocks to client for `Null` format in processors pipeline. [\#8797](https://github.com/ClickHouse/ClickHouse/pull/8797) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) [\#8767](https://github.com/ClickHouse/ClickHouse/pull/8767) ([Alexander Kuzmenkov](https://github.com/akuzm)) - -#### Build/Testing/Packaging Improvement {#buildtestingpackaging-improvement} - -- Exception handling now works correctly on Windows Subsystem for Linux. See https://github.com/ClickHouse-Extras/libunwind/pull/3 This fixes [\#6480](https://github.com/ClickHouse/ClickHouse/issues/6480) [\#9564](https://github.com/ClickHouse/ClickHouse/pull/9564) ([sobolevsv](https://github.com/sobolevsv)) -- Replace `readline` with `replxx` for interactive line editing in `clickhouse-client` [\#8416](https://github.com/ClickHouse/ClickHouse/pull/8416) ([Ivan](https://github.com/abyss7)) -- Better build time and less template instantiations in FunctionsComparison. [\#9324](https://github.com/ClickHouse/ClickHouse/pull/9324) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Added integration with `clang-tidy` in CI. See also [\#6044](https://github.com/ClickHouse/ClickHouse/issues/6044) [\#9566](https://github.com/ClickHouse/ClickHouse/pull/9566) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Now we link ClickHouse in CI using `lld` even for `gcc`. [\#9049](https://github.com/ClickHouse/ClickHouse/pull/9049) ([alesapin](https://github.com/alesapin)) -- Allow to randomize thread scheduling and insert glitches when `THREAD_FUZZER_*` environment variables are set. This helps testing. [\#9459](https://github.com/ClickHouse/ClickHouse/pull/9459) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Enable secure sockets in stateless tests [\#9288](https://github.com/ClickHouse/ClickHouse/pull/9288) ([tavplubix](https://github.com/tavplubix)) -- Make SPLIT\_SHARED\_LIBRARIES=OFF more robust [\#9156](https://github.com/ClickHouse/ClickHouse/pull/9156) ([Azat Khuzhin](https://github.com/azat)) -- Make “performance\_introspection\_and\_logging” test reliable to random server stuck. This may happen in CI environment. See also [\#9515](https://github.com/ClickHouse/ClickHouse/issues/9515) [\#9528](https://github.com/ClickHouse/ClickHouse/pull/9528) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Validate XML in style check. [\#9550](https://github.com/ClickHouse/ClickHouse/pull/9550) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed race condition in test `00738_lock_for_inner_table`. This test relied on sleep. [\#9555](https://github.com/ClickHouse/ClickHouse/pull/9555) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Remove performance tests of type `once`. This is needed to run all performance tests in statistical comparison mode (more reliable). [\#9557](https://github.com/ClickHouse/ClickHouse/pull/9557) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Added performance test for arithmetic functions. [\#9326](https://github.com/ClickHouse/ClickHouse/pull/9326) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Added performance test for `sumMap` and `sumMapWithOverflow` aggregate functions. Follow-up for [\#8933](https://github.com/ClickHouse/ClickHouse/issues/8933) [\#8947](https://github.com/ClickHouse/ClickHouse/pull/8947) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Ensure style of ErrorCodes by style check. [\#9370](https://github.com/ClickHouse/ClickHouse/pull/9370) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Add script for tests history. [\#8796](https://github.com/ClickHouse/ClickHouse/pull/8796) ([alesapin](https://github.com/alesapin)) -- Add GCC warning `-Wsuggest-override` to locate and fix all places where `override` keyword must be used. [\#8760](https://github.com/ClickHouse/ClickHouse/pull/8760) ([kreuzerkrieg](https://github.com/kreuzerkrieg)) -- Ignore weak symbol under Mac OS X because it must be defined [\#9538](https://github.com/ClickHouse/ClickHouse/pull/9538) ([Deleted user](https://github.com/ghost)) -- Normalize running time of some queries in performance tests. This is done in preparation to run all the performance tests in comparison mode. [\#9565](https://github.com/ClickHouse/ClickHouse/pull/9565) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix some tests to support pytest with query tests [\#9062](https://github.com/ClickHouse/ClickHouse/pull/9062) ([Ivan](https://github.com/abyss7)) -- Enable SSL in build with MSan, so server will not fail at startup when running stateless tests [\#9531](https://github.com/ClickHouse/ClickHouse/pull/9531) ([tavplubix](https://github.com/tavplubix)) -- Fix database substitution in test results [\#9384](https://github.com/ClickHouse/ClickHouse/pull/9384) ([Ilya Yatsishin](https://github.com/qoega)) -- Build fixes for miscellaneous platforms [\#9381](https://github.com/ClickHouse/ClickHouse/pull/9381) ([proller](https://github.com/proller)) [\#8755](https://github.com/ClickHouse/ClickHouse/pull/8755) ([proller](https://github.com/proller)) [\#8631](https://github.com/ClickHouse/ClickHouse/pull/8631) ([proller](https://github.com/proller)) -- Added disks section to stateless-with-coverage test docker image [\#9213](https://github.com/ClickHouse/ClickHouse/pull/9213) ([Pavel Kovalenko](https://github.com/Jokser)) -- Get rid of in-source-tree files when building with GRPC [\#9588](https://github.com/ClickHouse/ClickHouse/pull/9588) ([Amos Bird](https://github.com/amosbird)) -- Slightly faster build time by removing SessionCleaner from Context. Make the code of SessionCleaner more simple. [\#9232](https://github.com/ClickHouse/ClickHouse/pull/9232) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Updated checking for hung queries in clickhouse-test script [\#8858](https://github.com/ClickHouse/ClickHouse/pull/8858) ([Alexander Kazakov](https://github.com/Akazz)) -- Removed some useless files from repository. [\#8843](https://github.com/ClickHouse/ClickHouse/pull/8843) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Changed type of math perftests from `once` to `loop`. [\#8783](https://github.com/ClickHouse/ClickHouse/pull/8783) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Add docker image which allows to build interactive code browser HTML report for our codebase. [\#8781](https://github.com/ClickHouse/ClickHouse/pull/8781) ([alesapin](https://github.com/alesapin)) See [Woboq Code Browser](https://clickhouse.tech/codebrowser/html_report///ClickHouse/dbms/src/index.html) -- Suppress some test failures under MSan. [\#8780](https://github.com/ClickHouse/ClickHouse/pull/8780) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- Speedup “exception while insert” test. This test often time out in debug-with-coverage build. [\#8711](https://github.com/ClickHouse/ClickHouse/pull/8711) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Updated `libcxx` and `libcxxabi` to master. In preparation to [\#9304](https://github.com/ClickHouse/ClickHouse/issues/9304) [\#9308](https://github.com/ClickHouse/ClickHouse/pull/9308) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix flacky test `00910_zookeeper_test_alter_compression_codecs`. [\#9525](https://github.com/ClickHouse/ClickHouse/pull/9525) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Clean up duplicated linker flags. Make sure the linker won’t look up an unexpected symbol. [\#9433](https://github.com/ClickHouse/ClickHouse/pull/9433) ([Amos Bird](https://github.com/amosbird)) -- Add `clickhouse-odbc` driver into test images. This allows to test interaction of ClickHouse with ClickHouse via its own ODBC driver. [\#9348](https://github.com/ClickHouse/ClickHouse/pull/9348) ([filimonov](https://github.com/filimonov)) -- Fix several bugs in unit tests. [\#9047](https://github.com/ClickHouse/ClickHouse/pull/9047) ([alesapin](https://github.com/alesapin)) -- Enable `-Wmissing-include-dirs` GCC warning to eliminate all non-existing includes - mostly as a result of CMake scripting errors [\#8704](https://github.com/ClickHouse/ClickHouse/pull/8704) ([kreuzerkrieg](https://github.com/kreuzerkrieg)) -- Describe reasons if query profiler cannot work. This is intended for [\#9049](https://github.com/ClickHouse/ClickHouse/issues/9049) [\#9144](https://github.com/ClickHouse/ClickHouse/pull/9144) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Update OpenSSL to upstream master. Fixed the issue when TLS connections may fail with the message `OpenSSL SSL_read: error:14094438:SSL routines:ssl3_read_bytes:tlsv1 alert internal error` and `SSL Exception: error:2400006E:random number generator::error retrieving entropy`. The issue was present in version 20.1. [\#8956](https://github.com/ClickHouse/ClickHouse/pull/8956) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Update Dockerfile for server [\#8893](https://github.com/ClickHouse/ClickHouse/pull/8893) ([Ilya Mazaev](https://github.com/ne-ray)) -- Minor fixes in build-gcc-from-sources script [\#8774](https://github.com/ClickHouse/ClickHouse/pull/8774) ([Michael Nacharov](https://github.com/mnach)) -- Replace `numbers` to `zeros` in perftests where `number` column is not used. This will lead to more clean test results. [\#9600](https://github.com/ClickHouse/ClickHouse/pull/9600) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Fix stack overflow issue when using initializer\_list in Column constructors. [\#9367](https://github.com/ClickHouse/ClickHouse/pull/9367) ([Deleted user](https://github.com/ghost)) -- Upgrade librdkafka to v1.3.0. Enable bundled `rdkafka` and `gsasl` libraries on Mac OS X. [\#9000](https://github.com/ClickHouse/ClickHouse/pull/9000) ([Andrew Onyshchuk](https://github.com/oandrew)) -- build fix on GCC 9.2.0 [\#9306](https://github.com/ClickHouse/ClickHouse/pull/9306) ([vxider](https://github.com/Vxider)) - -## ClickHouse Release V20.1 {#clickhouse-release-v20-1} - -### ClickHouse Release V20.1.8.41, 2020-03-20 {#clickhouse-release-v20-1-8-41-2020-03-20} - -#### Bug Fix {#bug-fix-3} - -- Fix possible permanent `Cannot schedule a task` error (due to unhandled exception in `ParallelAggregatingBlockInputStream::Handler::onFinish/onFinishThread`). This fixes [\#6833](https://github.com/ClickHouse/ClickHouse/issues/6833). [\#9154](https://github.com/ClickHouse/ClickHouse/pull/9154) ([Azat Khuzhin](https://github.com/azat)) -- Fix excessive memory consumption in `ALTER` queries (mutations). This fixes [\#9533](https://github.com/ClickHouse/ClickHouse/issues/9533) and [\#9670](https://github.com/ClickHouse/ClickHouse/issues/9670). [\#9754](https://github.com/ClickHouse/ClickHouse/pull/9754) ([alesapin](https://github.com/alesapin)) -- Fix bug in backquoting in external dictionaries DDL. This fixes [\#9619](https://github.com/ClickHouse/ClickHouse/issues/9619). [\#9734](https://github.com/ClickHouse/ClickHouse/pull/9734) ([alesapin](https://github.com/alesapin)) - -### ClickHouse Release V20.1.7.38, 2020-03-18 {#clickhouse-release-v20-1-7-38-2020-03-18} - -#### Bug Fix {#bug-fix-4} - -- Fixed incorrect internal function names for `sumKahan` and `sumWithOverflow`. I lead to exception while using this functions in remote queries. [\#9636](https://github.com/ClickHouse/ClickHouse/pull/9636) ([Azat Khuzhin](https://github.com/azat)). This issue was in all ClickHouse releases. -- Allow `ALTER ON CLUSTER` of `Distributed` tables with internal replication. This fixes [\#3268](https://github.com/ClickHouse/ClickHouse/issues/3268). [\#9617](https://github.com/ClickHouse/ClickHouse/pull/9617) ([shinoi2](https://github.com/shinoi2)). This issue was in all ClickHouse releases. -- Fix possible exceptions `Size of filter doesn't match size of column` and `Invalid number of rows in Chunk` in `MergeTreeRangeReader`. They could appear while executing `PREWHERE` in some cases. Fixes [\#9132](https://github.com/ClickHouse/ClickHouse/issues/9132). [\#9612](https://github.com/ClickHouse/ClickHouse/pull/9612) ([Anton Popov](https://github.com/CurtizJ)) -- Fixed the issue: timezone was not preserved if you write a simple arithmetic expression like `time + 1` (in contrast to an expression like `time + INTERVAL 1 SECOND`). This fixes [\#5743](https://github.com/ClickHouse/ClickHouse/issues/5743). [\#9323](https://github.com/ClickHouse/ClickHouse/pull/9323) ([alexey-milovidov](https://github.com/alexey-milovidov)). This issue was in all ClickHouse releases. -- Now it’s not possible to create or add columns with simple cyclic aliases like `a DEFAULT b, b DEFAULT a`. [\#9603](https://github.com/ClickHouse/ClickHouse/pull/9603) ([alesapin](https://github.com/alesapin)) -- Fixed the issue when padding at the end of base64 encoded value can be malformed. Update base64 library. This fixes [\#9491](https://github.com/ClickHouse/ClickHouse/issues/9491), closes [\#9492](https://github.com/ClickHouse/ClickHouse/issues/9492) [\#9500](https://github.com/ClickHouse/ClickHouse/pull/9500) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix data race at destruction of `Poco::HTTPServer`. It could happen when server is started and immediately shut down. [\#9468](https://github.com/ClickHouse/ClickHouse/pull/9468) ([Anton Popov](https://github.com/CurtizJ)) -- Fix possible crash/wrong number of rows in `LIMIT n WITH TIES` when there are a lot of rows equal to n’th row. [\#9464](https://github.com/ClickHouse/ClickHouse/pull/9464) ([tavplubix](https://github.com/tavplubix)) -- Fix possible mismatched checksums with column TTLs. [\#9451](https://github.com/ClickHouse/ClickHouse/pull/9451) ([Anton Popov](https://github.com/CurtizJ)) -- Fix crash when a user tries to `ALTER MODIFY SETTING` for old-formated `MergeTree` table engines family. [\#9435](https://github.com/ClickHouse/ClickHouse/pull/9435) ([alesapin](https://github.com/alesapin)) -- Now we will try finalize mutations more frequently. [\#9427](https://github.com/ClickHouse/ClickHouse/pull/9427) ([alesapin](https://github.com/alesapin)) -- Fix replication protocol incompatibility introduced in [\#8598](https://github.com/ClickHouse/ClickHouse/issues/8598). [\#9412](https://github.com/ClickHouse/ClickHouse/pull/9412) ([alesapin](https://github.com/alesapin)) -- Fix not(has()) for the bloom\_filter index of array types. [\#9407](https://github.com/ClickHouse/ClickHouse/pull/9407) ([achimbab](https://github.com/achimbab)) -- Fixed the behaviour of `match` and `extract` functions when haystack has zero bytes. The behaviour was wrong when haystack was constant. This fixes [\#9160](https://github.com/ClickHouse/ClickHouse/issues/9160) [\#9163](https://github.com/ClickHouse/ClickHouse/pull/9163) ([alexey-milovidov](https://github.com/alexey-milovidov)) [\#9345](https://github.com/ClickHouse/ClickHouse/pull/9345) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -#### Build/Testing/Packaging Improvement {#buildtestingpackaging-improvement-1} - -- Exception handling now works correctly on Windows Subsystem for Linux. See https://github.com/ClickHouse-Extras/libunwind/pull/3 This fixes [\#6480](https://github.com/ClickHouse/ClickHouse/issues/6480) [\#9564](https://github.com/ClickHouse/ClickHouse/pull/9564) ([sobolevsv](https://github.com/sobolevsv)) - -### ClickHouse Release V20.1.6.30, 2020-03-05 {#clickhouse-release-v20-1-6-30-2020-03-05} - -#### Bug Fix {#bug-fix-5} - -- Fix data incompatibility when compressed with `T64` codec. - [\#9039](https://github.com/ClickHouse/ClickHouse/pull/9039) [(abyss7)](https://github.com/abyss7) -- Fix order of ranges while reading from MergeTree table in one thread. Fixes [\#8964](https://github.com/ClickHouse/ClickHouse/issues/8964). - [\#9050](https://github.com/ClickHouse/ClickHouse/pull/9050) [(CurtizJ)](https://github.com/CurtizJ) -- Fix possible segfault in `MergeTreeRangeReader`, while executing `PREWHERE`. Fixes [\#9064](https://github.com/ClickHouse/ClickHouse/issues/9064). - [\#9106](https://github.com/ClickHouse/ClickHouse/pull/9106) [(CurtizJ)](https://github.com/CurtizJ) -- Fix `reinterpretAsFixedString` to return `FixedString` instead of `String`. - [\#9052](https://github.com/ClickHouse/ClickHouse/pull/9052) [(oandrew)](https://github.com/oandrew) -- Fix `joinGet` with nullable return types. Fixes [\#8919](https://github.com/ClickHouse/ClickHouse/issues/8919) - [\#9014](https://github.com/ClickHouse/ClickHouse/pull/9014) [(amosbird)](https://github.com/amosbird) -- Fix fuzz test and incorrect behaviour of bitTestAll/bitTestAny functions. - [\#9143](https://github.com/ClickHouse/ClickHouse/pull/9143) [(alexey-milovidov)](https://github.com/alexey-milovidov) -- Fix the behaviour of match and extract functions when haystack has zero bytes. The behaviour was wrong when haystack was constant. Fixes [\#9160](https://github.com/ClickHouse/ClickHouse/issues/9160) - [\#9163](https://github.com/ClickHouse/ClickHouse/pull/9163) [(alexey-milovidov)](https://github.com/alexey-milovidov) -- Fixed execution of inversed predicates when non-strictly monotinic functional index is used. Fixes [\#9034](https://github.com/ClickHouse/ClickHouse/issues/9034) - [\#9223](https://github.com/ClickHouse/ClickHouse/pull/9223) [(Akazz)](https://github.com/Akazz) -- Allow to rewrite `CROSS` to `INNER JOIN` if there’s `[NOT] LIKE` operator in `WHERE` section. Fixes [\#9191](https://github.com/ClickHouse/ClickHouse/issues/9191) - [\#9229](https://github.com/ClickHouse/ClickHouse/pull/9229) [(4ertus2)](https://github.com/4ertus2) -- Allow first column(s) in a table with Log engine be an alias. - [\#9231](https://github.com/ClickHouse/ClickHouse/pull/9231) [(abyss7)](https://github.com/abyss7) -- Allow comma join with `IN()` inside. Fixes [\#7314](https://github.com/ClickHouse/ClickHouse/issues/7314). - [\#9251](https://github.com/ClickHouse/ClickHouse/pull/9251) [(4ertus2)](https://github.com/4ertus2) -- Improve `ALTER MODIFY/ADD` queries logic. Now you cannot `ADD` column without type, `MODIFY` default expression doesn’t change type of column and `MODIFY` type doesn’t loose default expression value. Fixes [\#8669](https://github.com/ClickHouse/ClickHouse/issues/8669). - [\#9227](https://github.com/ClickHouse/ClickHouse/pull/9227) [(alesapin)](https://github.com/alesapin) -- Fix mutations finalization, when already done mutation can have status is\_done=0. - [\#9217](https://github.com/ClickHouse/ClickHouse/pull/9217) [(alesapin)](https://github.com/alesapin) -- Support “Processors” pipeline for system.numbers and system.numbers\_mt. This also fixes the bug when `max_execution_time` is not respected. - [\#7796](https://github.com/ClickHouse/ClickHouse/pull/7796) [(KochetovNicolai)](https://github.com/KochetovNicolai) -- Fix wrong counting of `DictCacheKeysRequestedFound` metric. - [\#9411](https://github.com/ClickHouse/ClickHouse/pull/9411) [(nikitamikhaylov)](https://github.com/nikitamikhaylov) -- Added a check for storage policy in `ATTACH PARTITION FROM`, `REPLACE PARTITION`, `MOVE TO TABLE` which otherwise could make data of part inaccessible after restart and prevent ClickHouse to start. - [\#9383](https://github.com/ClickHouse/ClickHouse/pull/9383) [(excitoon)](https://github.com/excitoon) -- Fixed UBSan report in `MergeTreeIndexSet`. This fixes [\#9250](https://github.com/ClickHouse/ClickHouse/issues/9250) - [\#9365](https://github.com/ClickHouse/ClickHouse/pull/9365) [(alexey-milovidov)](https://github.com/alexey-milovidov) -- Fix possible datarace in BlockIO. - [\#9356](https://github.com/ClickHouse/ClickHouse/pull/9356) [(KochetovNicolai)](https://github.com/KochetovNicolai) -- Support for `UInt64` numbers that don’t fit in Int64 in JSON-related functions. Update `SIMDJSON` to master. This fixes [\#9209](https://github.com/ClickHouse/ClickHouse/issues/9209) - [\#9344](https://github.com/ClickHouse/ClickHouse/pull/9344) [(alexey-milovidov)](https://github.com/alexey-milovidov) -- Fix the issue when the amount of free space is not calculated correctly if the data directory is mounted to a separate device. For default disk calculate the free space from data subdirectory. This fixes [\#7441](https://github.com/ClickHouse/ClickHouse/issues/7441) - [\#9257](https://github.com/ClickHouse/ClickHouse/pull/9257) [(millb)](https://github.com/millb) -- Fix the issue when TLS connections may fail with the message `OpenSSL SSL_read: error:14094438:SSL routines:ssl3_read_bytes:tlsv1 alert internal error and SSL Exception: error:2400006E:random number generator::error retrieving entropy.` Update OpenSSL to upstream master. - [\#8956](https://github.com/ClickHouse/ClickHouse/pull/8956) [(alexey-milovidov)](https://github.com/alexey-milovidov) -- When executing `CREATE` query, fold constant expressions in storage engine arguments. Replace empty database name with current database. Fixes [\#6508](https://github.com/ClickHouse/ClickHouse/issues/6508), [\#3492](https://github.com/ClickHouse/ClickHouse/issues/3492). Also fix check for local address in ClickHouseDictionarySource. - [\#9262](https://github.com/ClickHouse/ClickHouse/pull/9262) [(tabplubix)](https://github.com/tavplubix) -- Fix segfault in `StorageMerge`, which can happen when reading from StorageFile. - [\#9387](https://github.com/ClickHouse/ClickHouse/pull/9387) [(tabplubix)](https://github.com/tavplubix) -- Prevent losing data in `Kafka` in rare cases when exception happens after reading suffix but before commit. Fixes [\#9378](https://github.com/ClickHouse/ClickHouse/issues/9378). Related: [\#7175](https://github.com/ClickHouse/ClickHouse/issues/7175) - [\#9507](https://github.com/ClickHouse/ClickHouse/pull/9507) [(filimonov)](https://github.com/filimonov) -- Fix bug leading to server termination when trying to use / drop `Kafka` table created with wrong parameters. Fixes [\#9494](https://github.com/ClickHouse/ClickHouse/issues/9494). Incorporates [\#9507](https://github.com/ClickHouse/ClickHouse/issues/9507). - [\#9513](https://github.com/ClickHouse/ClickHouse/pull/9513) [(filimonov)](https://github.com/filimonov) - -#### New Feature {#new-feature-1} - -- Add `deduplicate_blocks_in_dependent_materialized_views` option to control the behaviour of idempotent inserts into tables with materialized views. This new feature was added to the bugfix release by a special request from Altinity. - [\#9070](https://github.com/ClickHouse/ClickHouse/pull/9070) [(urykhy)](https://github.com/urykhy) - -### ClickHouse Release V20.1.2.4, 2020-01-22 {#clickhouse-release-v20-1-2-4-2020-01-22} - -#### Backward Incompatible Change {#backward-incompatible-change-1} - -- Make the setting `merge_tree_uniform_read_distribution` obsolete. The server still recognizes this setting but it has no effect. [\#8308](https://github.com/ClickHouse/ClickHouse/pull/8308) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Changed return type of the function `greatCircleDistance` to `Float32` because now the result of calculation is `Float32`. [\#7993](https://github.com/ClickHouse/ClickHouse/pull/7993) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Now it’s expected that query parameters are represented in “escaped” format. For example, to pass string `ab` you have to write `a\tb` or `a\b` and respectively, `a%5Ctb` or `a%5C%09b` in URL. This is needed to add the possibility to pass NULL as `\N`. This fixes [\#7488](https://github.com/ClickHouse/ClickHouse/issues/7488). [\#8517](https://github.com/ClickHouse/ClickHouse/pull/8517) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Enable `use_minimalistic_part_header_in_zookeeper` setting for `ReplicatedMergeTree` by default. This will significantly reduce amount of data stored in ZooKeeper. This setting is supported since version 19.1 and we already use it in production in multiple services without any issues for more than half a year. Disable this setting if you have a chance to downgrade to versions older than 19.1. [\#6850](https://github.com/ClickHouse/ClickHouse/pull/6850) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Data skipping indices are production ready and enabled by default. The settings `allow_experimental_data_skipping_indices`, `allow_experimental_cross_to_join_conversion` and `allow_experimental_multiple_joins_emulation` are now obsolete and do nothing. [\#7974](https://github.com/ClickHouse/ClickHouse/pull/7974) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Add new `ANY JOIN` logic for `StorageJoin` consistent with `JOIN` operation. To upgrade without changes in behaviour you need add `SETTINGS any_join_distinct_right_table_keys = 1` to Engine Join tables metadata or recreate these tables after upgrade. [\#8400](https://github.com/ClickHouse/ClickHouse/pull/8400) ([Artem Zuikov](https://github.com/4ertus2)) -- Require server to be restarted to apply the changes in logging configuration. This is a temporary workaround to avoid the bug where the server logs to a deleted log file (see [\#8696](https://github.com/ClickHouse/ClickHouse/issues/8696)). [\#8707](https://github.com/ClickHouse/ClickHouse/pull/8707) ([Alexander Kuzmenkov](https://github.com/akuzm)) - -#### New Feature {#new-feature-2} - -- Added information about part paths to `system.merges`. [\#8043](https://github.com/ClickHouse/ClickHouse/pull/8043) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Add ability to execute `SYSTEM RELOAD DICTIONARY` query in `ON CLUSTER` mode. [\#8288](https://github.com/ClickHouse/ClickHouse/pull/8288) ([Guillaume Tassery](https://github.com/YiuRULE)) -- Add ability to execute `CREATE DICTIONARY` queries in `ON CLUSTER` mode. [\#8163](https://github.com/ClickHouse/ClickHouse/pull/8163) ([alesapin](https://github.com/alesapin)) -- Now user’s profile in `users.xml` can inherit multiple profiles. [\#8343](https://github.com/ClickHouse/ClickHouse/pull/8343) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) -- Added `system.stack_trace` table that allows to look at stack traces of all server threads. This is useful for developers to introspect server state. This fixes [\#7576](https://github.com/ClickHouse/ClickHouse/issues/7576). [\#8344](https://github.com/ClickHouse/ClickHouse/pull/8344) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Add `DateTime64` datatype with configurable sub-second precision. [\#7170](https://github.com/ClickHouse/ClickHouse/pull/7170) ([Vasily Nemkov](https://github.com/Enmk)) -- Add table function `clusterAllReplicas` which allows to query all the nodes in the cluster. [\#8493](https://github.com/ClickHouse/ClickHouse/pull/8493) ([kiran sunkari](https://github.com/kiransunkari)) -- Add aggregate function `categoricalInformationValue` which calculates the information value of a discrete feature. [\#8117](https://github.com/ClickHouse/ClickHouse/pull/8117) ([hcz](https://github.com/hczhcz)) -- Speed up parsing of data files in `CSV`, `TSV` and `JSONEachRow` format by doing it in parallel. [\#7780](https://github.com/ClickHouse/ClickHouse/pull/7780) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- Add function `bankerRound` which performs banker’s rounding. [\#8112](https://github.com/ClickHouse/ClickHouse/pull/8112) ([hcz](https://github.com/hczhcz)) -- Support more languages in embedded dictionary for region names: ‘ru’, ‘en’, ‘ua’, ‘uk’, ‘by’, ‘kz’, ‘tr’, ‘de’, ‘uz’, ‘lv’, ‘lt’, ‘et’, ‘pt’, ‘he’, ‘vi’. [\#8189](https://github.com/ClickHouse/ClickHouse/pull/8189) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Improvements in consistency of `ANY JOIN` logic. Now `t1 ANY LEFT JOIN t2` equals `t2 ANY RIGHT JOIN t1`. [\#7665](https://github.com/ClickHouse/ClickHouse/pull/7665) ([Artem Zuikov](https://github.com/4ertus2)) -- Add setting `any_join_distinct_right_table_keys` which enables old behaviour for `ANY INNER JOIN`. [\#7665](https://github.com/ClickHouse/ClickHouse/pull/7665) ([Artem Zuikov](https://github.com/4ertus2)) -- Add new `SEMI` and `ANTI JOIN`. Old `ANY INNER JOIN` behaviour now available as `SEMI LEFT JOIN`. [\#7665](https://github.com/ClickHouse/ClickHouse/pull/7665) ([Artem Zuikov](https://github.com/4ertus2)) -- Added `Distributed` format for `File` engine and `file` table function which allows to read from `.bin` files generated by asynchronous inserts into `Distributed` table. [\#8535](https://github.com/ClickHouse/ClickHouse/pull/8535) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Add optional reset column argument for `runningAccumulate` which allows to reset aggregation results for each new key value. [\#8326](https://github.com/ClickHouse/ClickHouse/pull/8326) ([Sergey Kononenko](https://github.com/kononencheg)) -- Add ability to use ClickHouse as Prometheus endpoint. [\#7900](https://github.com/ClickHouse/ClickHouse/pull/7900) ([vdimir](https://github.com/Vdimir)) -- Add section `` in `config.xml` which restricts allowed hosts for remote table engines and table functions `URL`, `S3`, `HDFS`. [\#7154](https://github.com/ClickHouse/ClickHouse/pull/7154) ([Mikhail Korotov](https://github.com/millb)) -- Added function `greatCircleAngle` which calculates the distance on a sphere in degrees. [\#8105](https://github.com/ClickHouse/ClickHouse/pull/8105) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Changed Earth radius to be consistent with H3 library. [\#8105](https://github.com/ClickHouse/ClickHouse/pull/8105) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Added `JSONCompactEachRow` and `JSONCompactEachRowWithNamesAndTypes` formats for input and output. [\#7841](https://github.com/ClickHouse/ClickHouse/pull/7841) ([Mikhail Korotov](https://github.com/millb)) -- Added feature for file-related table engines and table functions (`File`, `S3`, `URL`, `HDFS`) which allows to read and write `gzip` files based on additional engine parameter or file extension. [\#7840](https://github.com/ClickHouse/ClickHouse/pull/7840) ([Andrey Bodrov](https://github.com/apbodrov)) -- Added the `randomASCII(length)` function, generating a string with a random set of [ASCII](https://en.wikipedia.org/wiki/ASCII#Printable_characters) printable characters. [\#8401](https://github.com/ClickHouse/ClickHouse/pull/8401) ([BayoNet](https://github.com/BayoNet)) -- Added function `JSONExtractArrayRaw` which returns an array on unparsed json array elements from `JSON` string. [\#8081](https://github.com/ClickHouse/ClickHouse/pull/8081) ([Oleg Matrokhin](https://github.com/errx)) -- Add `arrayZip` function which allows to combine multiple arrays of equal lengths into one array of tuples. [\#8149](https://github.com/ClickHouse/ClickHouse/pull/8149) ([Winter Zhang](https://github.com/zhang2014)) -- Add ability to move data between disks according to configured `TTL`-expressions for `*MergeTree` table engines family. [\#8140](https://github.com/ClickHouse/ClickHouse/pull/8140) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Added new aggregate function `avgWeighted` which allows to calculate weighted average. [\#7898](https://github.com/ClickHouse/ClickHouse/pull/7898) ([Andrey Bodrov](https://github.com/apbodrov)) -- Now parallel parsing is enabled by default for `TSV`, `TSKV`, `CSV` and `JSONEachRow` formats. [\#7894](https://github.com/ClickHouse/ClickHouse/pull/7894) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- Add several geo functions from `H3` library: `h3GetResolution`, `h3EdgeAngle`, `h3EdgeLength`, `h3IsValid` and `h3kRing`. [\#8034](https://github.com/ClickHouse/ClickHouse/pull/8034) ([Konstantin Malanchev](https://github.com/hombit)) -- Added support for brotli (`br`) compression in file-related storages and table functions. This fixes [\#8156](https://github.com/ClickHouse/ClickHouse/issues/8156). [\#8526](https://github.com/ClickHouse/ClickHouse/pull/8526) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Add `groupBit*` functions for the `SimpleAggregationFunction` type. [\#8485](https://github.com/ClickHouse/ClickHouse/pull/8485) ([Guillaume Tassery](https://github.com/YiuRULE)) - -#### Bug Fix {#bug-fix-6} - -- Fix rename of tables with `Distributed` engine. Fixes issue [\#7868](https://github.com/ClickHouse/ClickHouse/issues/7868). [\#8306](https://github.com/ClickHouse/ClickHouse/pull/8306) ([tavplubix](https://github.com/tavplubix)) -- Now dictionaries support `EXPRESSION` for attributes in arbitrary string in non-ClickHouse SQL dialect. [\#8098](https://github.com/ClickHouse/ClickHouse/pull/8098) ([alesapin](https://github.com/alesapin)) -- Fix broken `INSERT SELECT FROM mysql(...)` query. This fixes [\#8070](https://github.com/ClickHouse/ClickHouse/issues/8070) and [\#7960](https://github.com/ClickHouse/ClickHouse/issues/7960). [\#8234](https://github.com/ClickHouse/ClickHouse/pull/8234) ([tavplubix](https://github.com/tavplubix)) -- Fix error “Mismatch column sizes” when inserting default `Tuple` from `JSONEachRow`. This fixes [\#5653](https://github.com/ClickHouse/ClickHouse/issues/5653). [\#8606](https://github.com/ClickHouse/ClickHouse/pull/8606) ([tavplubix](https://github.com/tavplubix)) -- Now an exception will be thrown in case of using `WITH TIES` alongside `LIMIT BY`. Also add ability to use `TOP` with `LIMIT BY`. This fixes [\#7472](https://github.com/ClickHouse/ClickHouse/issues/7472). [\#7637](https://github.com/ClickHouse/ClickHouse/pull/7637) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- Fix unintendent dependency from fresh glibc version in `clickhouse-odbc-bridge` binary. [\#8046](https://github.com/ClickHouse/ClickHouse/pull/8046) ([Amos Bird](https://github.com/amosbird)) -- Fix bug in check function of `*MergeTree` engines family. Now it doesn’t fail in case when we have equal amount of rows in last granule and last mark (non-final). [\#8047](https://github.com/ClickHouse/ClickHouse/pull/8047) ([alesapin](https://github.com/alesapin)) -- Fix insert into `Enum*` columns after `ALTER` query, when underlying numeric type is equal to table specified type. This fixes [\#7836](https://github.com/ClickHouse/ClickHouse/issues/7836). [\#7908](https://github.com/ClickHouse/ClickHouse/pull/7908) ([Anton Popov](https://github.com/CurtizJ)) -- Allowed non-constant negative “size” argument for function `substring`. It was not allowed by mistake. This fixes [\#4832](https://github.com/ClickHouse/ClickHouse/issues/4832). [\#7703](https://github.com/ClickHouse/ClickHouse/pull/7703) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix parsing bug when wrong number of arguments passed to `(O|J)DBC` table engine. [\#7709](https://github.com/ClickHouse/ClickHouse/pull/7709) ([alesapin](https://github.com/alesapin)) -- Using command name of the running clickhouse process when sending logs to syslog. In previous versions, empty string was used instead of command name. [\#8460](https://github.com/ClickHouse/ClickHouse/pull/8460) ([Michael Nacharov](https://github.com/mnach)) -- Fix check of allowed hosts for `localhost`. This PR fixes the solution provided in [\#8241](https://github.com/ClickHouse/ClickHouse/pull/8241). [\#8342](https://github.com/ClickHouse/ClickHouse/pull/8342) ([Vitaly Baranov](https://github.com/vitlibar)) -- Fix rare crash in `argMin` and `argMax` functions for long string arguments, when result is used in `runningAccumulate` function. This fixes [\#8325](https://github.com/ClickHouse/ClickHouse/issues/8325) [\#8341](https://github.com/ClickHouse/ClickHouse/pull/8341) ([dinosaur](https://github.com/769344359)) -- Fix memory overcommit for tables with `Buffer` engine. [\#8345](https://github.com/ClickHouse/ClickHouse/pull/8345) ([Azat Khuzhin](https://github.com/azat)) -- Fixed potential bug in functions that can take `NULL` as one of the arguments and return non-NULL. [\#8196](https://github.com/ClickHouse/ClickHouse/pull/8196) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Better metrics calculations in thread pool for background processes for `MergeTree` table engines. [\#8194](https://github.com/ClickHouse/ClickHouse/pull/8194) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Fix function `IN` inside `WHERE` statement when row-level table filter is present. Fixes [\#6687](https://github.com/ClickHouse/ClickHouse/issues/6687) [\#8357](https://github.com/ClickHouse/ClickHouse/pull/8357) ([Ivan](https://github.com/abyss7)) -- Now an exception is thrown if the integral value is not parsed completely for settings values. [\#7678](https://github.com/ClickHouse/ClickHouse/pull/7678) ([Mikhail Korotov](https://github.com/millb)) -- Fix exception when aggregate function is used in query to distributed table with more than two local shards. [\#8164](https://github.com/ClickHouse/ClickHouse/pull/8164) ([小路](https://github.com/nicelulu)) -- Now bloom filter can handle zero length arrays and doesn’t perform redundant calculations. [\#8242](https://github.com/ClickHouse/ClickHouse/pull/8242) ([achimbab](https://github.com/achimbab)) -- Fixed checking if a client host is allowed by matching the client host to `host_regexp` specified in `users.xml`. [\#8241](https://github.com/ClickHouse/ClickHouse/pull/8241) ([Vitaly Baranov](https://github.com/vitlibar)) -- Relax ambiguous column check that leads to false positives in multiple `JOIN ON` section. [\#8385](https://github.com/ClickHouse/ClickHouse/pull/8385) ([Artem Zuikov](https://github.com/4ertus2)) -- Fixed possible server crash (`std::terminate`) when the server cannot send or write data in `JSON` or `XML` format with values of `String` data type (that require `UTF-8` validation) or when compressing result data with Brotli algorithm or in some other rare cases. This fixes [\#7603](https://github.com/ClickHouse/ClickHouse/issues/7603) [\#8384](https://github.com/ClickHouse/ClickHouse/pull/8384) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix race condition in `StorageDistributedDirectoryMonitor` found by CI. This fixes [\#8364](https://github.com/ClickHouse/ClickHouse/issues/8364). [\#8383](https://github.com/ClickHouse/ClickHouse/pull/8383) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Now background merges in `*MergeTree` table engines family preserve storage policy volume order more accurately. [\#8549](https://github.com/ClickHouse/ClickHouse/pull/8549) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Now table engine `Kafka` works properly with `Native` format. This fixes [\#6731](https://github.com/ClickHouse/ClickHouse/issues/6731) [\#7337](https://github.com/ClickHouse/ClickHouse/issues/7337) [\#8003](https://github.com/ClickHouse/ClickHouse/issues/8003). [\#8016](https://github.com/ClickHouse/ClickHouse/pull/8016) ([filimonov](https://github.com/filimonov)) -- Fixed formats with headers (like `CSVWithNames`) which were throwing exception about EOF for table engine `Kafka`. [\#8016](https://github.com/ClickHouse/ClickHouse/pull/8016) ([filimonov](https://github.com/filimonov)) -- Fixed a bug with making set from subquery in right part of `IN` section. This fixes [\#5767](https://github.com/ClickHouse/ClickHouse/issues/5767) and [\#2542](https://github.com/ClickHouse/ClickHouse/issues/2542). [\#7755](https://github.com/ClickHouse/ClickHouse/pull/7755) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- Fix possible crash while reading from storage `File`. [\#7756](https://github.com/ClickHouse/ClickHouse/pull/7756) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Fixed reading of the files in `Parquet` format containing columns of type `list`. [\#8334](https://github.com/ClickHouse/ClickHouse/pull/8334) ([maxulan](https://github.com/maxulan)) -- Fix error `Not found column` for distributed queries with `PREWHERE` condition dependent on sampling key if `max_parallel_replicas > 1`. [\#7913](https://github.com/ClickHouse/ClickHouse/pull/7913) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Fix error `Not found column` if query used `PREWHERE` dependent on table’s alias and the result set was empty because of primary key condition. [\#7911](https://github.com/ClickHouse/ClickHouse/pull/7911) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Fixed return type for functions `rand` and `randConstant` in case of `Nullable` argument. Now functions always return `UInt32` and never `Nullable(UInt32)`. [\#8204](https://github.com/ClickHouse/ClickHouse/pull/8204) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Disabled predicate push-down for `WITH FILL` expression. This fixes [\#7784](https://github.com/ClickHouse/ClickHouse/issues/7784). [\#7789](https://github.com/ClickHouse/ClickHouse/pull/7789) ([Winter Zhang](https://github.com/zhang2014)) -- Fixed incorrect `count()` result for `SummingMergeTree` when `FINAL` section is used. [\#3280](https://github.com/ClickHouse/ClickHouse/issues/3280) [\#7786](https://github.com/ClickHouse/ClickHouse/pull/7786) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- Fix possible incorrect result for constant functions from remote servers. It happened for queries with functions like `version()`, `uptime()`, etc. which returns different constant values for different servers. This fixes [\#7666](https://github.com/ClickHouse/ClickHouse/issues/7666). [\#7689](https://github.com/ClickHouse/ClickHouse/pull/7689) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Fix complicated bug in push-down predicate optimization which leads to wrong results. This fixes a lot of issues on push-down predicate optimization. [\#8503](https://github.com/ClickHouse/ClickHouse/pull/8503) ([Winter Zhang](https://github.com/zhang2014)) -- Fix crash in `CREATE TABLE .. AS dictionary` query. [\#8508](https://github.com/ClickHouse/ClickHouse/pull/8508) ([Azat Khuzhin](https://github.com/azat)) -- Several improvements ClickHouse grammar in `.g4` file. [\#8294](https://github.com/ClickHouse/ClickHouse/pull/8294) ([taiyang-li](https://github.com/taiyang-li)) -- Fix bug that leads to crashes in `JOIN`s with tables with engine `Join`. This fixes [\#7556](https://github.com/ClickHouse/ClickHouse/issues/7556) [\#8254](https://github.com/ClickHouse/ClickHouse/issues/8254) [\#7915](https://github.com/ClickHouse/ClickHouse/issues/7915) [\#8100](https://github.com/ClickHouse/ClickHouse/issues/8100). [\#8298](https://github.com/ClickHouse/ClickHouse/pull/8298) ([Artem Zuikov](https://github.com/4ertus2)) -- Fix redundant dictionaries reload on `CREATE DATABASE`. [\#7916](https://github.com/ClickHouse/ClickHouse/pull/7916) ([Azat Khuzhin](https://github.com/azat)) -- Limit maximum number of streams for read from `StorageFile` and `StorageHDFS`. Fixes https://github.com/ClickHouse/ClickHouse/issues/7650. [\#7981](https://github.com/ClickHouse/ClickHouse/pull/7981) ([alesapin](https://github.com/alesapin)) -- Fix bug in `ALTER ... MODIFY ... CODEC` query, when user specify both default expression and codec. Fixes [8593](https://github.com/ClickHouse/ClickHouse/issues/8593). [\#8614](https://github.com/ClickHouse/ClickHouse/pull/8614) ([alesapin](https://github.com/alesapin)) -- Fix error in background merge of columns with `SimpleAggregateFunction(LowCardinality)` type. [\#8613](https://github.com/ClickHouse/ClickHouse/pull/8613) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Fixed type check in function `toDateTime64`. [\#8375](https://github.com/ClickHouse/ClickHouse/pull/8375) ([Vasily Nemkov](https://github.com/Enmk)) -- Now server do not crash on `LEFT` or `FULL JOIN` with and Join engine and unsupported `join_use_nulls` settings. [\#8479](https://github.com/ClickHouse/ClickHouse/pull/8479) ([Artem Zuikov](https://github.com/4ertus2)) -- Now `DROP DICTIONARY IF EXISTS db.dict` query doesn’t throw exception if `db` doesn’t exist. [\#8185](https://github.com/ClickHouse/ClickHouse/pull/8185) ([Vitaly Baranov](https://github.com/vitlibar)) -- Fix possible crashes in table functions (`file`, `mysql`, `remote`) caused by usage of reference to removed `IStorage` object. Fix incorrect parsing of columns specified at insertion into table function. [\#7762](https://github.com/ClickHouse/ClickHouse/pull/7762) ([tavplubix](https://github.com/tavplubix)) -- Ensure network be up before starting `clickhouse-server`. This fixes [\#7507](https://github.com/ClickHouse/ClickHouse/issues/7507). [\#8570](https://github.com/ClickHouse/ClickHouse/pull/8570) ([Zhichang Yu](https://github.com/yuzhichang)) -- Fix timeouts handling for secure connections, so queries doesn’t hang indefenitely. This fixes [\#8126](https://github.com/ClickHouse/ClickHouse/issues/8126). [\#8128](https://github.com/ClickHouse/ClickHouse/pull/8128) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix `clickhouse-copier`’s redundant contention between concurrent workers. [\#7816](https://github.com/ClickHouse/ClickHouse/pull/7816) ([Ding Xiang Fei](https://github.com/dingxiangfei2009)) -- Now mutations doesn’t skip attached parts, even if their mutation version were larger than current mutation version. [\#7812](https://github.com/ClickHouse/ClickHouse/pull/7812) ([Zhichang Yu](https://github.com/yuzhichang)) [\#8250](https://github.com/ClickHouse/ClickHouse/pull/8250) ([alesapin](https://github.com/alesapin)) -- Ignore redundant copies of `*MergeTree` data parts after move to another disk and server restart. [\#7810](https://github.com/ClickHouse/ClickHouse/pull/7810) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Fix crash in `FULL JOIN` with `LowCardinality` in `JOIN` key. [\#8252](https://github.com/ClickHouse/ClickHouse/pull/8252) ([Artem Zuikov](https://github.com/4ertus2)) -- Forbidden to use column name more than once in insert query like `INSERT INTO tbl (x, y, x)`. This fixes [\#5465](https://github.com/ClickHouse/ClickHouse/issues/5465), [\#7681](https://github.com/ClickHouse/ClickHouse/issues/7681). [\#7685](https://github.com/ClickHouse/ClickHouse/pull/7685) ([alesapin](https://github.com/alesapin)) -- Added fallback for detection the number of physical CPU cores for unknown CPUs (using the number of logical CPU cores). This fixes [\#5239](https://github.com/ClickHouse/ClickHouse/issues/5239). [\#7726](https://github.com/ClickHouse/ClickHouse/pull/7726) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix `There's no column` error for materialized and alias columns. [\#8210](https://github.com/ClickHouse/ClickHouse/pull/8210) ([Artem Zuikov](https://github.com/4ertus2)) -- Fixed sever crash when `EXISTS` query was used without `TABLE` or `DICTIONARY` qualifier. Just like `EXISTS t`. This fixes [\#8172](https://github.com/ClickHouse/ClickHouse/issues/8172). This bug was introduced in version 19.17. [\#8213](https://github.com/ClickHouse/ClickHouse/pull/8213) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix rare bug with error `"Sizes of columns doesn't match"` that might appear when using `SimpleAggregateFunction` column. [\#7790](https://github.com/ClickHouse/ClickHouse/pull/7790) ([Boris Granveaud](https://github.com/bgranvea)) -- Fix bug where user with empty `allow_databases` got access to all databases (and same for `allow_dictionaries`). [\#7793](https://github.com/ClickHouse/ClickHouse/pull/7793) ([DeifyTheGod](https://github.com/DeifyTheGod)) -- Fix client crash when server already disconnected from client. [\#8071](https://github.com/ClickHouse/ClickHouse/pull/8071) ([Azat Khuzhin](https://github.com/azat)) -- Fix `ORDER BY` behaviour in case of sorting by primary key prefix and non primary key suffix. [\#7759](https://github.com/ClickHouse/ClickHouse/pull/7759) ([Anton Popov](https://github.com/CurtizJ)) -- Check if qualified column present in the table. This fixes [\#6836](https://github.com/ClickHouse/ClickHouse/issues/6836). [\#7758](https://github.com/ClickHouse/ClickHouse/pull/7758) ([Artem Zuikov](https://github.com/4ertus2)) -- Fixed behavior with `ALTER MOVE` ran immediately after merge finish moves superpart of specified. Fixes [\#8103](https://github.com/ClickHouse/ClickHouse/issues/8103). [\#8104](https://github.com/ClickHouse/ClickHouse/pull/8104) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Fix possible server crash while using `UNION` with different number of columns. Fixes [\#7279](https://github.com/ClickHouse/ClickHouse/issues/7279). [\#7929](https://github.com/ClickHouse/ClickHouse/pull/7929) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Fix size of result substring for function `substr` with negative size. [\#8589](https://github.com/ClickHouse/ClickHouse/pull/8589) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Now server does not execute part mutation in `MergeTree` if there are not enough free threads in background pool. [\#8588](https://github.com/ClickHouse/ClickHouse/pull/8588) ([tavplubix](https://github.com/tavplubix)) -- Fix a minor typo on formatting `UNION ALL` AST. [\#7999](https://github.com/ClickHouse/ClickHouse/pull/7999) ([litao91](https://github.com/litao91)) -- Fixed incorrect bloom filter results for negative numbers. This fixes [\#8317](https://github.com/ClickHouse/ClickHouse/issues/8317). [\#8566](https://github.com/ClickHouse/ClickHouse/pull/8566) ([Winter Zhang](https://github.com/zhang2014)) -- Fixed potential buffer overflow in decompress. Malicious user can pass fabricated compressed data that will cause read after buffer. This issue was found by Eldar Zaitov from Yandex information security team. [\#8404](https://github.com/ClickHouse/ClickHouse/pull/8404) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix incorrect result because of integers overflow in `arrayIntersect`. [\#7777](https://github.com/ClickHouse/ClickHouse/pull/7777) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Now `OPTIMIZE TABLE` query will not wait for offline replicas to perform the operation. [\#8314](https://github.com/ClickHouse/ClickHouse/pull/8314) ([javi santana](https://github.com/javisantana)) -- Fixed `ALTER TTL` parser for `Replicated*MergeTree` tables. [\#8318](https://github.com/ClickHouse/ClickHouse/pull/8318) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Fix communication between server and client, so server read temporary tables info after query failure. [\#8084](https://github.com/ClickHouse/ClickHouse/pull/8084) ([Azat Khuzhin](https://github.com/azat)) -- Fix `bitmapAnd` function error when intersecting an aggregated bitmap and a scalar bitmap. [\#8082](https://github.com/ClickHouse/ClickHouse/pull/8082) ([Yue Huang](https://github.com/moon03432)) -- Refine the definition of `ZXid` according to the ZooKeeper Programmer’s Guide which fixes bug in `clickhouse-cluster-copier`. [\#8088](https://github.com/ClickHouse/ClickHouse/pull/8088) ([Ding Xiang Fei](https://github.com/dingxiangfei2009)) -- `odbc` table function now respects `external_table_functions_use_nulls` setting. [\#7506](https://github.com/ClickHouse/ClickHouse/pull/7506) ([Vasily Nemkov](https://github.com/Enmk)) -- Fixed bug that lead to a rare data race. [\#8143](https://github.com/ClickHouse/ClickHouse/pull/8143) ([Alexander Kazakov](https://github.com/Akazz)) -- Now `SYSTEM RELOAD DICTIONARY` reloads a dictionary completely, ignoring `update_field`. This fixes [\#7440](https://github.com/ClickHouse/ClickHouse/issues/7440). [\#8037](https://github.com/ClickHouse/ClickHouse/pull/8037) ([Vitaly Baranov](https://github.com/vitlibar)) -- Add ability to check if dictionary exists in create query. [\#8032](https://github.com/ClickHouse/ClickHouse/pull/8032) ([alesapin](https://github.com/alesapin)) -- Fix `Float*` parsing in `Values` format. This fixes [\#7817](https://github.com/ClickHouse/ClickHouse/issues/7817). [\#7870](https://github.com/ClickHouse/ClickHouse/pull/7870) ([tavplubix](https://github.com/tavplubix)) -- Fix crash when we cannot reserve space in some background operations of `*MergeTree` table engines family. [\#7873](https://github.com/ClickHouse/ClickHouse/pull/7873) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Fix crash of merge operation when table contains `SimpleAggregateFunction(LowCardinality)` column. This fixes [\#8515](https://github.com/ClickHouse/ClickHouse/issues/8515). [\#8522](https://github.com/ClickHouse/ClickHouse/pull/8522) ([Azat Khuzhin](https://github.com/azat)) -- Restore support of all ICU locales and add the ability to apply collations for constant expressions. Also add language name to `system.collations` table. [\#8051](https://github.com/ClickHouse/ClickHouse/pull/8051) ([alesapin](https://github.com/alesapin)) -- Fix bug when external dictionaries with zero minimal lifetime (`LIFETIME(MIN 0 MAX N)`, `LIFETIME(N)`) don’t update in background. [\#7983](https://github.com/ClickHouse/ClickHouse/pull/7983) ([alesapin](https://github.com/alesapin)) -- Fix crash when external dictionary with ClickHouse source has subquery in query. [\#8351](https://github.com/ClickHouse/ClickHouse/pull/8351) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Fix incorrect parsing of file extension in table with engine `URL`. This fixes [\#8157](https://github.com/ClickHouse/ClickHouse/issues/8157). [\#8419](https://github.com/ClickHouse/ClickHouse/pull/8419) ([Andrey Bodrov](https://github.com/apbodrov)) -- Fix `CHECK TABLE` query for `*MergeTree` tables without key. Fixes [\#7543](https://github.com/ClickHouse/ClickHouse/issues/7543). [\#7979](https://github.com/ClickHouse/ClickHouse/pull/7979) ([alesapin](https://github.com/alesapin)) -- Fixed conversion of `Float64` to MySQL type. [\#8079](https://github.com/ClickHouse/ClickHouse/pull/8079) ([Yuriy Baranov](https://github.com/yurriy)) -- Now if table was not completely dropped because of server crash, server will try to restore and load it. [\#8176](https://github.com/ClickHouse/ClickHouse/pull/8176) ([tavplubix](https://github.com/tavplubix)) -- Fixed crash in table function `file` while inserting into file that doesn’t exist. Now in this case file would be created and then insert would be processed. [\#8177](https://github.com/ClickHouse/ClickHouse/pull/8177) ([Olga Khvostikova](https://github.com/stavrolia)) -- Fix rare deadlock which can happen when `trace_log` is in enabled. [\#7838](https://github.com/ClickHouse/ClickHouse/pull/7838) ([filimonov](https://github.com/filimonov)) -- Add ability to work with different types besides `Date` in `RangeHashed` external dictionary created from DDL query. Fixes [7899](https://github.com/ClickHouse/ClickHouse/issues/7899). [\#8275](https://github.com/ClickHouse/ClickHouse/pull/8275) ([alesapin](https://github.com/alesapin)) -- Fixes crash when `now64()` is called with result of another function. [\#8270](https://github.com/ClickHouse/ClickHouse/pull/8270) ([Vasily Nemkov](https://github.com/Enmk)) -- Fixed bug with detecting client IP for connections through mysql wire protocol. [\#7743](https://github.com/ClickHouse/ClickHouse/pull/7743) ([Dmitry Muzyka](https://github.com/dmitriy-myz)) -- Fix empty array handling in `arraySplit` function. This fixes [\#7708](https://github.com/ClickHouse/ClickHouse/issues/7708). [\#7747](https://github.com/ClickHouse/ClickHouse/pull/7747) ([hcz](https://github.com/hczhcz)) -- Fixed the issue when `pid-file` of another running `clickhouse-server` may be deleted. [\#8487](https://github.com/ClickHouse/ClickHouse/pull/8487) ([Weiqing Xu](https://github.com/weiqxu)) -- Fix dictionary reload if it has `invalidate_query`, which stopped updates and some exception on previous update tries. [\#8029](https://github.com/ClickHouse/ClickHouse/pull/8029) ([alesapin](https://github.com/alesapin)) -- Fixed error in function `arrayReduce` that may lead to “double free” and error in aggregate function combinator `Resample` that may lead to memory leak. Added aggregate function `aggThrow`. This function can be used for testing purposes. [\#8446](https://github.com/ClickHouse/ClickHouse/pull/8446) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -#### Improvement {#improvement-1} - -- Improved logging when working with `S3` table engine. [\#8251](https://github.com/ClickHouse/ClickHouse/pull/8251) ([Grigory Pervakov](https://github.com/GrigoryPervakov)) -- Printed help message when no arguments are passed when calling `clickhouse-local`. This fixes [\#5335](https://github.com/ClickHouse/ClickHouse/issues/5335). [\#8230](https://github.com/ClickHouse/ClickHouse/pull/8230) ([Andrey Nagorny](https://github.com/Melancholic)) -- Add setting `mutations_sync` which allows to wait `ALTER UPDATE/DELETE` queries synchronously. [\#8237](https://github.com/ClickHouse/ClickHouse/pull/8237) ([alesapin](https://github.com/alesapin)) -- Allow to set up relative `user_files_path` in `config.xml` (in the way similar to `format_schema_path`). [\#7632](https://github.com/ClickHouse/ClickHouse/pull/7632) ([hcz](https://github.com/hczhcz)) -- Add exception for illegal types for conversion functions with `-OrZero` postfix. [\#7880](https://github.com/ClickHouse/ClickHouse/pull/7880) ([Andrey Konyaev](https://github.com/akonyaev90)) -- Simplify format of the header of data sending to a shard in a distributed query. [\#8044](https://github.com/ClickHouse/ClickHouse/pull/8044) ([Vitaly Baranov](https://github.com/vitlibar)) -- `Live View` table engine refactoring. [\#8519](https://github.com/ClickHouse/ClickHouse/pull/8519) ([vzakaznikov](https://github.com/vzakaznikov)) -- Add additional checks for external dictionaries created from DDL-queries. [\#8127](https://github.com/ClickHouse/ClickHouse/pull/8127) ([alesapin](https://github.com/alesapin)) -- Fix error `Column ... already exists` while using `FINAL` and `SAMPLE` together, e.g. `select count() from table final sample 1/2`. Fixes [\#5186](https://github.com/ClickHouse/ClickHouse/issues/5186). [\#7907](https://github.com/ClickHouse/ClickHouse/pull/7907) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Now table the first argument of `joinGet` function can be table indentifier. [\#7707](https://github.com/ClickHouse/ClickHouse/pull/7707) ([Amos Bird](https://github.com/amosbird)) -- Allow using `MaterializedView` with subqueries above `Kafka` tables. [\#8197](https://github.com/ClickHouse/ClickHouse/pull/8197) ([filimonov](https://github.com/filimonov)) -- Now background moves between disks run it the seprate thread pool. [\#7670](https://github.com/ClickHouse/ClickHouse/pull/7670) ([Vladimir Chebotarev](https://github.com/excitoon)) -- `SYSTEM RELOAD DICTIONARY` now executes synchronously. [\#8240](https://github.com/ClickHouse/ClickHouse/pull/8240) ([Vitaly Baranov](https://github.com/vitlibar)) -- Stack traces now display physical addresses (offsets in object file) instead of virtual memory addresses (where the object file was loaded). That allows the use of `addr2line` when binary is position independent and ASLR is active. This fixes [\#8360](https://github.com/ClickHouse/ClickHouse/issues/8360). [\#8387](https://github.com/ClickHouse/ClickHouse/pull/8387) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Support new syntax for row-level security filters: `…
`. Fixes [\#5779](https://github.com/ClickHouse/ClickHouse/issues/5779). [\#8381](https://github.com/ClickHouse/ClickHouse/pull/8381) ([Ivan](https://github.com/abyss7)) -- Now `cityHash` function can work with `Decimal` and `UUID` types. Fixes [\#5184](https://github.com/ClickHouse/ClickHouse/issues/5184). [\#7693](https://github.com/ClickHouse/ClickHouse/pull/7693) ([Mikhail Korotov](https://github.com/millb)) -- Removed fixed index granularity (it was 1024) from system logs because it’s obsolete after implementation of adaptive granularity. [\#7698](https://github.com/ClickHouse/ClickHouse/pull/7698) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Enabled MySQL compatibility server when ClickHouse is compiled without SSL. [\#7852](https://github.com/ClickHouse/ClickHouse/pull/7852) ([Yuriy Baranov](https://github.com/yurriy)) -- Now server checksums distributed batches, which gives more verbose errors in case of corrupted data in batch. [\#7914](https://github.com/ClickHouse/ClickHouse/pull/7914) ([Azat Khuzhin](https://github.com/azat)) -- Support `DROP DATABASE`, `DETACH TABLE`, `DROP TABLE` and `ATTACH TABLE` for `MySQL` database engine. [\#8202](https://github.com/ClickHouse/ClickHouse/pull/8202) ([Winter Zhang](https://github.com/zhang2014)) -- Add authentication in S3 table function and table engine. [\#7623](https://github.com/ClickHouse/ClickHouse/pull/7623) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Added check for extra parts of `MergeTree` at different disks, in order to not allow to miss data parts at undefined disks. [\#8118](https://github.com/ClickHouse/ClickHouse/pull/8118) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Enable SSL support for Mac client and server. [\#8297](https://github.com/ClickHouse/ClickHouse/pull/8297) ([Ivan](https://github.com/abyss7)) -- Now ClickHouse can work as MySQL federated server (see https://dev.mysql.com/doc/refman/5.7/en/federated-create-server.html). [\#7717](https://github.com/ClickHouse/ClickHouse/pull/7717) ([Maxim Fedotov](https://github.com/MaxFedotov)) -- `clickhouse-client` now only enable `bracketed-paste` when multiquery is on and multiline is off. This fixes (\#7757)\[https://github.com/ClickHouse/ClickHouse/issues/7757\]. [\#7761](https://github.com/ClickHouse/ClickHouse/pull/7761) ([Amos Bird](https://github.com/amosbird)) -- Support `Array(Decimal)` in `if` function. [\#7721](https://github.com/ClickHouse/ClickHouse/pull/7721) ([Artem Zuikov](https://github.com/4ertus2)) -- Support Decimals in `arrayDifference`, `arrayCumSum` and `arrayCumSumNegative` functions. [\#7724](https://github.com/ClickHouse/ClickHouse/pull/7724) ([Artem Zuikov](https://github.com/4ertus2)) -- Added `lifetime` column to `system.dictionaries` table. [\#6820](https://github.com/ClickHouse/ClickHouse/issues/6820) [\#7727](https://github.com/ClickHouse/ClickHouse/pull/7727) ([kekekekule](https://github.com/kekekekule)) -- Improved check for existing parts on different disks for `*MergeTree` table engines. Addresses [\#7660](https://github.com/ClickHouse/ClickHouse/issues/7660). [\#8440](https://github.com/ClickHouse/ClickHouse/pull/8440) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Integration with `AWS SDK` for `S3` interactions which allows to use all S3 features out of the box. [\#8011](https://github.com/ClickHouse/ClickHouse/pull/8011) ([Pavel Kovalenko](https://github.com/Jokser)) -- Added support for subqueries in `Live View` tables. [\#7792](https://github.com/ClickHouse/ClickHouse/pull/7792) ([vzakaznikov](https://github.com/vzakaznikov)) -- Check for using `Date` or `DateTime` column from `TTL` expressions was removed. [\#7920](https://github.com/ClickHouse/ClickHouse/pull/7920) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Information about disk was added to `system.detached_parts` table. [\#7833](https://github.com/ClickHouse/ClickHouse/pull/7833) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Now settings `max_(table|partition)_size_to_drop` can be changed without a restart. [\#7779](https://github.com/ClickHouse/ClickHouse/pull/7779) ([Grigory Pervakov](https://github.com/GrigoryPervakov)) -- Slightly better usability of error messages. Ask user not to remove the lines below `Stack trace:`. [\#7897](https://github.com/ClickHouse/ClickHouse/pull/7897) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Better reading messages from `Kafka` engine in various formats after [\#7935](https://github.com/ClickHouse/ClickHouse/issues/7935). [\#8035](https://github.com/ClickHouse/ClickHouse/pull/8035) ([Ivan](https://github.com/abyss7)) -- Better compatibility with MySQL clients which don’t support `sha2_password` auth plugin. [\#8036](https://github.com/ClickHouse/ClickHouse/pull/8036) ([Yuriy Baranov](https://github.com/yurriy)) -- Support more column types in MySQL compatibility server. [\#7975](https://github.com/ClickHouse/ClickHouse/pull/7975) ([Yuriy Baranov](https://github.com/yurriy)) -- Implement `ORDER BY` optimization for `Merge`, `Buffer` and `Materilized View` storages with underlying `MergeTree` tables. [\#8130](https://github.com/ClickHouse/ClickHouse/pull/8130) ([Anton Popov](https://github.com/CurtizJ)) -- Now we always use POSIX implementation of `getrandom` to have better compatibility with old kernels (\< 3.17). [\#7940](https://github.com/ClickHouse/ClickHouse/pull/7940) ([Amos Bird](https://github.com/amosbird)) -- Better check for valid destination in a move TTL rule. [\#8410](https://github.com/ClickHouse/ClickHouse/pull/8410) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Better checks for broken insert batches for `Distributed` table engine. [\#7933](https://github.com/ClickHouse/ClickHouse/pull/7933) ([Azat Khuzhin](https://github.com/azat)) -- Add column with array of parts name which mutations must process in future to `system.mutations` table. [\#8179](https://github.com/ClickHouse/ClickHouse/pull/8179) ([alesapin](https://github.com/alesapin)) -- Parallel merge sort optimization for processors. [\#8552](https://github.com/ClickHouse/ClickHouse/pull/8552) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- The settings `mark_cache_min_lifetime` is now obsolete and does nothing. In previous versions, mark cache can grow in memory larger than `mark_cache_size` to accomodate data within `mark_cache_min_lifetime` seconds. That was leading to confusion and higher memory usage than expected, that is especially bad on memory constrained systems. If you will see performance degradation after installing this release, you should increase the `mark_cache_size`. [\#8484](https://github.com/ClickHouse/ClickHouse/pull/8484) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Preparation to use `tid` everywhere. This is needed for [\#7477](https://github.com/ClickHouse/ClickHouse/issues/7477). [\#8276](https://github.com/ClickHouse/ClickHouse/pull/8276) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -#### Performance Improvement {#performance-improvement-1} - -- Performance optimizations in processors pipeline. [\#7988](https://github.com/ClickHouse/ClickHouse/pull/7988) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Non-blocking updates of expired keys in cache dictionaries (with permission to read old ones). [\#8303](https://github.com/ClickHouse/ClickHouse/pull/8303) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- Compile ClickHouse without `-fno-omit-frame-pointer` globally to spare one more register. [\#8097](https://github.com/ClickHouse/ClickHouse/pull/8097) ([Amos Bird](https://github.com/amosbird)) -- Speedup `greatCircleDistance` function and add performance tests for it. [\#7307](https://github.com/ClickHouse/ClickHouse/pull/7307) ([Olga Khvostikova](https://github.com/stavrolia)) -- Improved performance of function `roundDown`. [\#8465](https://github.com/ClickHouse/ClickHouse/pull/8465) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Improved performance of `max`, `min`, `argMin`, `argMax` for `DateTime64` data type. [\#8199](https://github.com/ClickHouse/ClickHouse/pull/8199) ([Vasily Nemkov](https://github.com/Enmk)) -- Improved performance of sorting without a limit or with big limit and external sorting. [\#8545](https://github.com/ClickHouse/ClickHouse/pull/8545) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Improved performance of formatting floating point numbers up to 6 times. [\#8542](https://github.com/ClickHouse/ClickHouse/pull/8542) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Improved performance of `modulo` function. [\#7750](https://github.com/ClickHouse/ClickHouse/pull/7750) ([Amos Bird](https://github.com/amosbird)) -- Optimized `ORDER BY` and merging with single column key. [\#8335](https://github.com/ClickHouse/ClickHouse/pull/8335) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Better implementation for `arrayReduce`, `-Array` and `-State` combinators. [\#7710](https://github.com/ClickHouse/ClickHouse/pull/7710) ([Amos Bird](https://github.com/amosbird)) -- Now `PREWHERE` should be optimized to be at least as efficient as `WHERE`. [\#7769](https://github.com/ClickHouse/ClickHouse/pull/7769) ([Amos Bird](https://github.com/amosbird)) -- Improve the way `round` and `roundBankers` handling negative numbers. [\#8229](https://github.com/ClickHouse/ClickHouse/pull/8229) ([hcz](https://github.com/hczhcz)) -- Improved decoding performance of `DoubleDelta` and `Gorilla` codecs by roughly 30-40%. This fixes [\#7082](https://github.com/ClickHouse/ClickHouse/issues/7082). [\#8019](https://github.com/ClickHouse/ClickHouse/pull/8019) ([Vasily Nemkov](https://github.com/Enmk)) -- Improved performance of `base64` related functions. [\#8444](https://github.com/ClickHouse/ClickHouse/pull/8444) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Added a function `geoDistance`. It is similar to `greatCircleDistance` but uses approximation to WGS-84 ellipsoid model. The performance of both functions are near the same. [\#8086](https://github.com/ClickHouse/ClickHouse/pull/8086) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Faster `min` and `max` aggregation functions for `Decimal` data type. [\#8144](https://github.com/ClickHouse/ClickHouse/pull/8144) ([Artem Zuikov](https://github.com/4ertus2)) -- Vectorize processing `arrayReduce`. [\#7608](https://github.com/ClickHouse/ClickHouse/pull/7608) ([Amos Bird](https://github.com/amosbird)) -- `if` chains are now optimized as `multiIf`. [\#8355](https://github.com/ClickHouse/ClickHouse/pull/8355) ([kamalov-ruslan](https://github.com/kamalov-ruslan)) -- Fix performance regression of `Kafka` table engine introduced in 19.15. This fixes [\#7261](https://github.com/ClickHouse/ClickHouse/issues/7261). [\#7935](https://github.com/ClickHouse/ClickHouse/pull/7935) ([filimonov](https://github.com/filimonov)) -- Removed “pie” code generation that `gcc` from Debian packages occasionally brings by default. [\#8483](https://github.com/ClickHouse/ClickHouse/pull/8483) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Parallel parsing data formats [\#6553](https://github.com/ClickHouse/ClickHouse/pull/6553) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- Enable optimized parser of `Values` with expressions by default (`input_format_values_deduce_templates_of_expressions=1`). [\#8231](https://github.com/ClickHouse/ClickHouse/pull/8231) ([tavplubix](https://github.com/tavplubix)) - -#### Build/Testing/Packaging Improvement {#buildtestingpackaging-improvement-2} - -- Build fixes for `ARM` and in minimal mode. [\#8304](https://github.com/ClickHouse/ClickHouse/pull/8304) ([proller](https://github.com/proller)) -- Add coverage file flush for `clickhouse-server` when std::atexit is not called. Also slightly improved logging in stateless tests with coverage. [\#8267](https://github.com/ClickHouse/ClickHouse/pull/8267) ([alesapin](https://github.com/alesapin)) -- Update LLVM library in contrib. Avoid using LLVM from OS packages. [\#8258](https://github.com/ClickHouse/ClickHouse/pull/8258) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Make bundled `curl` build fully quiet. [\#8232](https://github.com/ClickHouse/ClickHouse/pull/8232) [\#8203](https://github.com/ClickHouse/ClickHouse/pull/8203) ([Pavel Kovalenko](https://github.com/Jokser)) -- Fix some `MemorySanitizer` warnings. [\#8235](https://github.com/ClickHouse/ClickHouse/pull/8235) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- Use `add_warning` and `no_warning` macros in `CMakeLists.txt`. [\#8604](https://github.com/ClickHouse/ClickHouse/pull/8604) ([Ivan](https://github.com/abyss7)) -- Add support of Minio S3 Compatible object (https://min.io/) for better integration tests. [\#7863](https://github.com/ClickHouse/ClickHouse/pull/7863) [\#7875](https://github.com/ClickHouse/ClickHouse/pull/7875) ([Pavel Kovalenko](https://github.com/Jokser)) -- Imported `libc` headers to contrib. It allows to make builds more consistent across various systems (only for `x86_64-linux-gnu`). [\#5773](https://github.com/ClickHouse/ClickHouse/pull/5773) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Remove `-fPIC` from some libraries. [\#8464](https://github.com/ClickHouse/ClickHouse/pull/8464) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Clean `CMakeLists.txt` for curl. See https://github.com/ClickHouse/ClickHouse/pull/8011\#issuecomment-569478910 [\#8459](https://github.com/ClickHouse/ClickHouse/pull/8459) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Silent warnings in `CapNProto` library. [\#8220](https://github.com/ClickHouse/ClickHouse/pull/8220) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Add performance tests for short string optimized hash tables. [\#7679](https://github.com/ClickHouse/ClickHouse/pull/7679) ([Amos Bird](https://github.com/amosbird)) -- Now ClickHouse will build on `AArch64` even if `MADV_FREE` is not available. This fixes [\#8027](https://github.com/ClickHouse/ClickHouse/issues/8027). [\#8243](https://github.com/ClickHouse/ClickHouse/pull/8243) ([Amos Bird](https://github.com/amosbird)) -- Update `zlib-ng` to fix memory sanitizer problems. [\#7182](https://github.com/ClickHouse/ClickHouse/pull/7182) [\#8206](https://github.com/ClickHouse/ClickHouse/pull/8206) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- Enable internal MySQL library on non-Linux system, because usage of OS packages is very fragile and usually doesn’t work at all. This fixes [\#5765](https://github.com/ClickHouse/ClickHouse/issues/5765). [\#8426](https://github.com/ClickHouse/ClickHouse/pull/8426) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed build on some systems after enabling `libc++`. This supersedes [\#8374](https://github.com/ClickHouse/ClickHouse/issues/8374). [\#8380](https://github.com/ClickHouse/ClickHouse/pull/8380) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Make `Field` methods more type-safe to find more errors. [\#7386](https://github.com/ClickHouse/ClickHouse/pull/7386) [\#8209](https://github.com/ClickHouse/ClickHouse/pull/8209) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- Added missing files to the `libc-headers` submodule. [\#8507](https://github.com/ClickHouse/ClickHouse/pull/8507) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix wrong `JSON` quoting in performance test output. [\#8497](https://github.com/ClickHouse/ClickHouse/pull/8497) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Now stack trace is displayed for `std::exception` and `Poco::Exception`. In previous versions it was available only for `DB::Exception`. This improves diagnostics. [\#8501](https://github.com/ClickHouse/ClickHouse/pull/8501) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Porting `clock_gettime` and `clock_nanosleep` for fresh glibc versions. [\#8054](https://github.com/ClickHouse/ClickHouse/pull/8054) ([Amos Bird](https://github.com/amosbird)) -- Enable `part_log` in example config for developers. [\#8609](https://github.com/ClickHouse/ClickHouse/pull/8609) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix async nature of reload in `01036_no_superfluous_dict_reload_on_create_database*`. [\#8111](https://github.com/ClickHouse/ClickHouse/pull/8111) ([Azat Khuzhin](https://github.com/azat)) -- Fixed codec performance tests. [\#8615](https://github.com/ClickHouse/ClickHouse/pull/8615) ([Vasily Nemkov](https://github.com/Enmk)) -- Add install scripts for `.tgz` build and documentation for them. [\#8612](https://github.com/ClickHouse/ClickHouse/pull/8612) [\#8591](https://github.com/ClickHouse/ClickHouse/pull/8591) ([alesapin](https://github.com/alesapin)) -- Removed old `ZSTD` test (it was created in year 2016 to reproduce the bug that pre 1.0 version of ZSTD has had). This fixes [\#8618](https://github.com/ClickHouse/ClickHouse/issues/8618). [\#8619](https://github.com/ClickHouse/ClickHouse/pull/8619) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed build on Mac OS Catalina. [\#8600](https://github.com/ClickHouse/ClickHouse/pull/8600) ([meo](https://github.com/meob)) -- Increased number of rows in codec performance tests to make results noticeable. [\#8574](https://github.com/ClickHouse/ClickHouse/pull/8574) ([Vasily Nemkov](https://github.com/Enmk)) -- In debug builds, treat `LOGICAL_ERROR` exceptions as assertion failures, so that they are easier to notice. [\#8475](https://github.com/ClickHouse/ClickHouse/pull/8475) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- Make formats-related performance test more deterministic. [\#8477](https://github.com/ClickHouse/ClickHouse/pull/8477) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Update `lz4` to fix a MemorySanitizer failure. [\#8181](https://github.com/ClickHouse/ClickHouse/pull/8181) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- Suppress a known MemorySanitizer false positive in exception handling. [\#8182](https://github.com/ClickHouse/ClickHouse/pull/8182) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- Update `gcc` and `g++` to version 9 in `build/docker/build.sh` [\#7766](https://github.com/ClickHouse/ClickHouse/pull/7766) ([TLightSky](https://github.com/tlightsky)) -- Add performance test case to test that `PREWHERE` is worse than `WHERE`. [\#7768](https://github.com/ClickHouse/ClickHouse/pull/7768) ([Amos Bird](https://github.com/amosbird)) -- Progress towards fixing one flacky test. [\#8621](https://github.com/ClickHouse/ClickHouse/pull/8621) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Avoid MemorySanitizer report for data from `libunwind`. [\#8539](https://github.com/ClickHouse/ClickHouse/pull/8539) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Updated `libc++` to the latest version. [\#8324](https://github.com/ClickHouse/ClickHouse/pull/8324) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Build ICU library from sources. This fixes [\#6460](https://github.com/ClickHouse/ClickHouse/issues/6460). [\#8219](https://github.com/ClickHouse/ClickHouse/pull/8219) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Switched from `libressl` to `openssl`. ClickHouse should support TLS 1.3 and SNI after this change. This fixes [\#8171](https://github.com/ClickHouse/ClickHouse/issues/8171). [\#8218](https://github.com/ClickHouse/ClickHouse/pull/8218) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed UBSan report when using `chacha20_poly1305` from SSL (happens on connect to https://yandex.ru/). [\#8214](https://github.com/ClickHouse/ClickHouse/pull/8214) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix mode of default password file for `.deb` linux distros. [\#8075](https://github.com/ClickHouse/ClickHouse/pull/8075) ([proller](https://github.com/proller)) -- Improved expression for getting `clickhouse-server` PID in `clickhouse-test`. [\#8063](https://github.com/ClickHouse/ClickHouse/pull/8063) ([Alexander Kazakov](https://github.com/Akazz)) -- Updated contrib/googletest to v1.10.0. [\#8587](https://github.com/ClickHouse/ClickHouse/pull/8587) ([Alexander Burmak](https://github.com/Alex-Burmak)) -- Fixed ThreadSaninitizer report in `base64` library. Also updated this library to the latest version, but it doesn’t matter. This fixes [\#8397](https://github.com/ClickHouse/ClickHouse/issues/8397). [\#8403](https://github.com/ClickHouse/ClickHouse/pull/8403) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix `00600_replace_running_query` for processors. [\#8272](https://github.com/ClickHouse/ClickHouse/pull/8272) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Remove support for `tcmalloc` to make `CMakeLists.txt` simpler. [\#8310](https://github.com/ClickHouse/ClickHouse/pull/8310) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Release gcc builds now use `libc++` instead of `libstdc++`. Recently `libc++` was used only with clang. This will improve consistency of build configurations and portability. [\#8311](https://github.com/ClickHouse/ClickHouse/pull/8311) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Enable ICU library for build with MemorySanitizer. [\#8222](https://github.com/ClickHouse/ClickHouse/pull/8222) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Suppress warnings from `CapNProto` library. [\#8224](https://github.com/ClickHouse/ClickHouse/pull/8224) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Removed special cases of code for `tcmalloc`, because it’s no longer supported. [\#8225](https://github.com/ClickHouse/ClickHouse/pull/8225) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- In CI coverage task, kill the server gracefully to allow it to save the coverage report. This fixes incomplete coverage reports we’ve been seeing lately. [\#8142](https://github.com/ClickHouse/ClickHouse/pull/8142) ([alesapin](https://github.com/alesapin)) -- Performance tests for all codecs against `Float64` and `UInt64` values. [\#8349](https://github.com/ClickHouse/ClickHouse/pull/8349) ([Vasily Nemkov](https://github.com/Enmk)) -- `termcap` is very much deprecated and lead to various problems (f.g. missing “up” cap and echoing `^J` instead of multi line) . Favor `terminfo` or bundled `ncurses`. [\#7737](https://github.com/ClickHouse/ClickHouse/pull/7737) ([Amos Bird](https://github.com/amosbird)) -- Fix `test_storage_s3` integration test. [\#7734](https://github.com/ClickHouse/ClickHouse/pull/7734) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Support `StorageFile(, null)` to insert block into given format file without actually write to disk. This is required for performance tests. [\#8455](https://github.com/ClickHouse/ClickHouse/pull/8455) ([Amos Bird](https://github.com/amosbird)) -- Added argument `--print-time` to functional tests which prints execution time per test. [\#8001](https://github.com/ClickHouse/ClickHouse/pull/8001) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Added asserts to `KeyCondition` while evaluating RPN. This will fix warning from gcc-9. [\#8279](https://github.com/ClickHouse/ClickHouse/pull/8279) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Dump cmake options in CI builds. [\#8273](https://github.com/ClickHouse/ClickHouse/pull/8273) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- Don’t generate debug info for some fat libraries. [\#8271](https://github.com/ClickHouse/ClickHouse/pull/8271) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Make `log_to_console.xml` always log to stderr, regardless of is it interactive or not. [\#8395](https://github.com/ClickHouse/ClickHouse/pull/8395) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- Removed some unused features from `clickhouse-performance-test` tool. [\#8555](https://github.com/ClickHouse/ClickHouse/pull/8555) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Now we will also search for `lld-X` with corresponding `clang-X` version. [\#8092](https://github.com/ClickHouse/ClickHouse/pull/8092) ([alesapin](https://github.com/alesapin)) -- Parquet build improvement. [\#8421](https://github.com/ClickHouse/ClickHouse/pull/8421) ([maxulan](https://github.com/maxulan)) -- More GCC warnings [\#8221](https://github.com/ClickHouse/ClickHouse/pull/8221) ([kreuzerkrieg](https://github.com/kreuzerkrieg)) -- Package for Arch Linux now allows to run ClickHouse server, and not only client. [\#8534](https://github.com/ClickHouse/ClickHouse/pull/8534) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Fix test with processors. Tiny performance fixes. [\#7672](https://github.com/ClickHouse/ClickHouse/pull/7672) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Update contrib/protobuf. [\#8256](https://github.com/ClickHouse/ClickHouse/pull/8256) ([Matwey V. Kornilov](https://github.com/matwey)) -- In preparation of switching to c++20 as a new year celebration. “May the C++ force be with ClickHouse.” [\#8447](https://github.com/ClickHouse/ClickHouse/pull/8447) ([Amos Bird](https://github.com/amosbird)) - -#### Experimental Feature {#experimental-feature-1} - -- Added experimental setting `min_bytes_to_use_mmap_io`. It allows to read big files without copying data from kernel to userspace. The setting is disabled by default. Recommended threshold is about 64 MB, because mmap/munmap is slow. [\#8520](https://github.com/ClickHouse/ClickHouse/pull/8520) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Reworked quotas as a part of access control system. Added new table `system.quotas`, new functions `currentQuota`, `currentQuotaKey`, new SQL syntax `CREATE QUOTA`, `ALTER QUOTA`, `DROP QUOTA`, `SHOW QUOTA`. [\#7257](https://github.com/ClickHouse/ClickHouse/pull/7257) ([Vitaly Baranov](https://github.com/vitlibar)) -- Allow skipping unknown settings with warnings instead of throwing exceptions. [\#7653](https://github.com/ClickHouse/ClickHouse/pull/7653) ([Vitaly Baranov](https://github.com/vitlibar)) -- Reworked row policies as a part of access control system. Added new table `system.row_policies`, new function `currentRowPolicies()`, new SQL syntax `CREATE POLICY`, `ALTER POLICY`, `DROP POLICY`, `SHOW CREATE POLICY`, `SHOW POLICIES`. [\#7808](https://github.com/ClickHouse/ClickHouse/pull/7808) ([Vitaly Baranov](https://github.com/vitlibar)) - -#### Security Fix {#security-fix} - -- Fixed the possibility of reading directories structure in tables with `File` table engine. This fixes [\#8536](https://github.com/ClickHouse/ClickHouse/issues/8536). [\#8537](https://github.com/ClickHouse/ClickHouse/pull/8537) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -## [Changelog for 2019](https://github.com/ClickHouse/ClickHouse/blob/master/docs/en/changelog/2019.md) {#changelog-for-2019} +{% include "content/changelog.md" %} diff --git a/docs/tools/make_links.sh b/docs/tools/make_links.sh index 2a27990f445..743d4eebf16 100755 --- a/docs/tools/make_links.sh +++ b/docs/tools/make_links.sh @@ -2,23 +2,21 @@ # Fixes missing documentation in other languages # by putting relative symbolic links to the original doc file. -# This is to be run from root of language directory, like "docs/en". + +BASE_DIR=$(dirname $(readlink -f $0)) function do_make_links() { + set -x langs=(en es zh fr ru ja tr fa) src_file="$1" for lang in "${langs[@]}" do - # replacing "/./" with / - dst_file="../${lang}${src_file}" - dst_file="${dst_file/\/\.\//\/}" - dst_file="${dst_file/${lang}\./${lang}}" - + dst_file="${src_file/\/en\///${lang}/}" mkdir -p $(dirname "${dst_file}") ln -sr "${src_file}" "${dst_file}" 2>/dev/null done } export -f do_make_links -find . -iname '*.md' -exec /bin/bash -c 'do_make_links "{}"' \; +find "${BASE_DIR}/../en" -iname '*.md' -exec /bin/bash -c 'do_make_links "{}"' \; diff --git a/docs/tools/test.py b/docs/tools/test.py index 10e83357a67..63b84885d9f 100755 --- a/docs/tools/test.py +++ b/docs/tools/test.py @@ -93,7 +93,8 @@ def test_single_page(input_path, lang): if links_to_nowhere: logging.warning(f'Found {links_to_nowhere} links to nowhere in {lang}') - sys.exit(1) + if lang == 'en': # TODO: check all languages again + sys.exit(1) if len(anchor_points) <= 10: logging.error('Html parsing is probably broken') diff --git a/docs/tools/translate/translate.py b/docs/tools/translate/translate.py index 759e5b849d2..6486a8cbcc7 100755 --- a/docs/tools/translate/translate.py +++ b/docs/tools/translate/translate.py @@ -2,6 +2,7 @@ import os import random +import re import sys import time import urllib.parse @@ -15,11 +16,12 @@ import typograph_ru translator = googletrans.Translator() default_target_language = os.environ.get('TARGET_LANGUAGE', 'ru') +curly_braces_re = re.compile('({[^}]+})') is_yandex = os.environ.get('YANDEX') is not None -def translate(text, target_language=None): +def translate_impl(text, target_language=None): target_language = target_language or default_target_language if target_language == 'en': return text @@ -46,6 +48,16 @@ def translate(text, target_language=None): return translator.translate(text, target_language).text +def translate(text, target_language=None): + result = [] + for part in re.split(curly_braces_re, text): + if part.startswith('{') and part.endswith('}'): + result.append(part) + else: + result.append(translate_impl(part, target_language=target_language)) + return ''.join(result) + + def translate_toc(root, lang): global is_yandex is_yandex = True diff --git a/docs/tools/translate/update-all-machine-translated.sh b/docs/tools/translate/update-all-machine-translated.sh new file mode 100755 index 00000000000..fae2aae787f --- /dev/null +++ b/docs/tools/translate/update-all-machine-translated.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +BASE_DIR=$(dirname $(readlink -f $0)) + +function translate() { + set -x + LANGUAGE=$1 + DOCS_ROOT="${BASE_DIR}/../../" + REV="$(git rev-parse HEAD)" + for FILENAME in $(find "${DOCS_ROOT}${LANGUAGE}" -name "*.md" -type f) + do + HAS_MT_TAG=$(grep -c "machine_translated: true" "${FILENAME}") + IS_UP_TO_DATE=$(grep -c "machine_translated_rev: \"${REV}\"" "${FILENAME}") + if [ "${HAS_MT_TAG}" -eq "1" ] && [ "${IS_UP_TO_DATE}" -eq "0" ] + then + set -e + EN_FILENAME=${FILENAME/\/${LANGUAGE}\///en/} + rm "${FILENAME}" || true + cp "${EN_FILENAME}" "${FILENAME}" + DEBUG=1 SLEEP=1 ${BASE_DIR}/replace-with-translation.sh ${LANGUAGE} "${FILENAME}" + set +e + fi + done +} +export BASE_DIR +export -f translate +parallel translate ::: es fr zh ja fa tr diff --git a/docs/tr/commercial/cloud.md b/docs/tr/commercial/cloud.md index 1d5ae7b3368..42ad0db7c26 100644 --- a/docs/tr/commercial/cloud.md +++ b/docs/tr/commercial/cloud.md @@ -1,6 +1,8 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_priority: 1 +toc_title: Bulut --- # ClickHouse Bulut Servis Sağlayıcıları {#clickhouse-cloud-service-providers} diff --git a/docs/tr/commercial/index.md b/docs/tr/commercial/index.md index 104a3427146..4f01c90970c 100644 --- a/docs/tr/commercial/index.md +++ b/docs/tr/commercial/index.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_folder_title: Ticari toc_priority: 70 toc_title: Ticari diff --git a/docs/tr/development/architecture.md b/docs/tr/development/architecture.md index 619778f3bd4..37b5060e91e 100644 --- a/docs/tr/development/architecture.md +++ b/docs/tr/development/architecture.md @@ -1,17 +1,17 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 62 toc_title: "ClickHouse mimarisine genel bak\u0131\u015F" --- -# ClickHouse Mimarisine Genel bakış {#overview-of-clickhouse-architecture} +# ClickHouse mimarisine genel bakış {#overview-of-clickhouse-architecture} ClickHouse gerçek bir sütun yönelimli DBMS olduğunu. Veriler sütunlar tarafından ve dizilerin yürütülmesi sırasında (vektörler veya sütun parçaları) saklanır. Mümkün olduğunda, işlemler tek tek değerler yerine dizilere gönderilir. Buna denir “vectorized query execution,” ve gerçek veri işleme maliyetini düşürmeye yardımcı olur. > Bu fikir yeni bir şey değil. Bu kadar uzanır `APL` programlama dili ve Tor andunları: `A +`, `J`, `K`, ve `Q`. Dizi programlama bilimsel veri işlemede kullanılır. Bu fikir ilişkisel veritabanlarında yeni bir şey değildir: örneğin, `Vectorwise` sistem. -Sorgu işlemeyi hızlandırmak için iki farklı yaklaşım vardır: vektörize sorgu yürütme ve çalışma zamanı kodu oluşturma. İkincisi, tüm Yönlendirme ve dinamik gönderimi kaldırır. Bu yaklaşımların hiçbiri diğerinden kesinlikle daha iyi değildir. Çalışma zamanı kodu üretimi, birçok işlemi birleştirdiğinde daha iyi olabilir, böylece CPU yürütme birimlerini ve boru hattını tam olarak kullanır. Vectorized sorgu yürütme daha az pratik olabilir, çünkü önbelleğe yazılması ve geri okunması gereken geçici vektörler içerir. Geçici veri L2 önbelleğinde uymuyorsa, bu bir sorun haline gelir. Ancak vektörize sorgu yürütme, CPU’nun SIMD yeteneklerini daha kolay kullanır. Bir [araştırma öd paperevi](http://15721.courses.cs.cmu.edu/spring2016/papers/p5-sompolski.pdf) arkadaşlarımız tarafından yazıldı, her iki yaklaşımı birleştirmenin daha iyi olduğunu gösteriyor. ClickHouse vectorized sorgu yürütme kullanır ve çalışma zamanı kodu üretimi için başlangıç desteği sınırlıdır. +Sorgu işlemeyi hızlandırmak için iki farklı yaklaşım vardır: vektörize sorgu yürütme ve çalışma zamanı kodu oluşturma. İkincisi, tüm Yönlendirme ve dinamik gönderimi kaldırır. Bu yaklaşımların hiçbiri diğerinden kesinlikle daha iyi değildir. Çalışma zamanı kodu üretimi, birçok işlemi birleştirdiğinde daha iyi olabilir, böylece CPU yürütme birimlerini ve boru hattını tam olarak kullanır. Vectorized sorgu yürütme daha az pratik olabilir, çünkü önbelleğe yazılması ve geri okunması gereken geçici vektörler içerir. Geçici veri L2 önbelleğinde uymuyorsa, bu bir sorun haline gelir. Ancak vektörize sorgu yürütme, CPU'nun SIMD yeteneklerini daha kolay kullanır. Bir [araştırma öd paperevi](http://15721.courses.cs.cmu.edu/spring2016/papers/p5-sompolski.pdf) arkadaşlarımız tarafından yazıldı, her iki yaklaşımı birleştirmenin daha iyi olduğunu gösteriyor. ClickHouse vectorized sorgu yürütme kullanır ve çalışma zamanı kodu üretimi için başlangıç desteği sınırlıdır. ## Sütun {#columns} @@ -21,7 +21,7 @@ Sorgu işlemeyi hızlandırmak için iki farklı yaklaşım vardır: vektörize ## Alan {#field} -Bununla birlikte, bireysel değerlerle de çalışmak mümkündür. Bireysel bir değeri temsil etmek için, `Field` kullanılır. `Field` sadece ayrımcılığa uğramış bir birlik mi `UInt64`, `Int64`, `Float64`, `String` ve `Array`. `IColumn` has the `operator[]` n - inci değerini bir olarak alma yöntemi `Field` ve… `insert` bir ekleme yöntemi `Field` bir sütunun sonuna. Bu yöntemler çok verimli değildir, çünkü geçici olarak uğraşmayı gerektirirler `Field` tek bir değeri temsil eden nesneler. Daha etkili yöntemleri vardır, mesela: `insertFrom`, `insertRangeFrom` ve bu yüzden. +Bununla birlikte, bireysel değerlerle de çalışmak mümkündür. Bireysel bir değeri temsil etmek için, `Field` kullanılır. `Field` sadece ayrımcılığa uğramış bir birlik mi `UInt64`, `Int64`, `Float64`, `String` ve `Array`. `IColumn` has the `operator[]` n - inci değerini bir olarak alma yöntemi `Field` ve... `insert` bir ekleme yöntemi `Field` bir sütunun sonuna. Bu yöntemler çok verimli değildir, çünkü geçici olarak uğraşmayı gerektirirler `Field` tek bir değeri temsil eden nesneler. Daha etkili yöntemleri vardır, mesela: `insertFrom`, `insertRangeFrom` ve bu yüzden. `Field` bir tablo için belirli bir veri türü hakkında yeterli bilgiye sahip değildir. Mesela, `UInt8`, `UInt16`, `UInt32`, ve `UInt64` hepsi olarak temsil edilir `UInt64` in a `Field`. @@ -41,7 +41,7 @@ Sütunlar üzerinde çeşitli işlevler kullanarak genel, verimli olmayan bir ş `IDataType` çeşitli veri formatları için yardımcı yöntemlere sahiptir. Örnekler, Olası Alıntı ile bir değeri serileştirmek, json için bir değeri serileştirmek ve XML formatının bir parçası olarak bir değeri serileştirmek için kullanılan yöntemlerdir. Veri formatlarına doğrudan yazışma yoktur. Örneğin, farklı veri biçimleri `Pretty` ve `TabSeparated` aynı kullanabilirsiniz `serializeTextEscaped` hel methodper yöntemi `IDataType` Arabirim. -## Blok {#block} +## Engel {#block} A `Block` bellekteki bir tablonun bir alt kümesini (yığın) temsil eden bir kapsayıcıdır. Bu sadece üçlü bir dizi: `(IColumn, IDataType, column name)`. Sorgu yürütme sırasında veri tarafından işlenir `Block`s. Eğer bir `Block`(bu yaptığımız verileri `IColumn` nesne), biz onun türü hakkında bilgi var (içinde `IDataType`) bu bize bu sütunla nasıl başa çıkacağımızı söyler ve sütun adına sahibiz. Tablodan orijinal sütun adı veya hesaplamaların geçici sonuçlarını almak için atanan bazı yapay ad olabilir. @@ -120,11 +120,11 @@ Sorgu yürütme kanalının oluşturulmasından tercümanlar sorumludur. `AST`. Sıradan fonksiyonlar ve toplam fonksiyonlar vardır. Toplama işlevleri için bir sonraki bölüme bakın. -Ordinary functions don’t change the number of rows – they work as if they are processing each row independently. In fact, functions are not called for individual rows, but for `Block`’s vectorized sorgu yürütme uygulamak için veri. +Ordinary functions don't change the number of rows – they work as if they are processing each row independently. In fact, functions are not called for individual rows, but for `Block`'s vectorized sorgu yürütme uygulamak için veri. Gibi bazı çeşitli fonksiyonlar vardır [blockSize](../sql-reference/functions/other-functions.md#function-blocksize), [rowNumberİnBlock](../sql-reference/functions/other-functions.md#function-rownumberinblock), ve [runningAccumulate](../sql-reference/functions/other-functions.md#function-runningaccumulate), blok işlemeyi istismar eden ve satırların bağımsızlığını ihlal eden. -Clickhouse’un güçlü yazımı var, bu yüzden örtük tür dönüşümü yok. Bir işlev belirli bir tür kombinasyonunu desteklemiyorsa, bir istisna atar. Ancak, birçok farklı tür kombinasyonu için işlevler çalışabilir (aşırı yüklenebilir). Örneğin, `plus` fonksiyonu (uygulamak için `+` operatör) sayısal türlerin herhangi bir kombinasyonu için çalışır: `UInt8` + `Float32`, `UInt16` + `Int8` ve bu yüzden. Ayrıca, bazı variadic işlevleri gibi bağımsız değişkenlerin herhangi bir sayıda kabul edebilir `concat` işlev. +Clickhouse'un güçlü yazımı var, bu yüzden örtük tür dönüşümü yok. Bir işlev belirli bir tür kombinasyonunu desteklemiyorsa, bir istisna atar. Ancak, birçok farklı tür kombinasyonu için işlevler çalışabilir (aşırı yüklenebilir). Örneğin, `plus` fonksiyonu (uygulamak için `+` operatör) sayısal türlerin herhangi bir kombinasyonu için çalışır: `UInt8` + `Float32`, `UInt16` + `Int8` ve bu yüzden. Ayrıca, bazı variadic işlevleri gibi bağımsız değişkenlerin herhangi bir sayıda kabul edebilir `concat` İşlev. Bir işlev açıkça desteklenen veri türlerini gönderir ve desteklenen çünkü bir işlev uygulamak biraz rahatsız edici olabilir `IColumns`. Örneğin, `plus` işlev, sayısal türlerin ve sabit veya sabit olmayan sol ve sağ bağımsız değişkenlerin her birleşimi için bir C++ şablonunun örneklendirilmesiyle oluşturulan koda sahiptir. @@ -171,13 +171,13 @@ Dağıtılmış sorgu yürütme için genel bir sorgu planı yoktur. Her düğü `MergeTree` birincil anahtarla dizin oluşturmayı destekleyen bir depolama altyapısı ailesidir. Birincil anahtar, isteğe bağlı bir sütun veya ifade kümesi olabilir. Veri `MergeTree` tablo saklanır “parts”. Her bölüm verileri birincil anahtar sırasına göre saklar, böylece veriler birincil anahtar tuple tarafından lexicographically sıralanır. Tüm tablo sütunları ayrı olarak saklanır `column.bin` bu kısımlardaki dosyalar. Dosyalar sıkıştırılmış bloklardan oluşur. Her blok, ortalama değer boyutuna bağlı olarak genellikle 64 KB ila 1 MB sıkıştırılmamış veridir. Bloklar, birbiri ardına bitişik olarak yerleştirilmiş sütun değerlerinden oluşur. Sütun değerleri her sütun için aynı sıradadır (birincil anahtar siparişi tanımlar), bu nedenle birçok sütun tarafından yineleme yaptığınızda, karşılık gelen satırlar için değerler alırsınız. -Birincil anahtarın kendisi “sparse”. Her satır Adres yok ama verilerin sadece biraz değişir. Ayıran `primary.idx` dosya, n’nin çağrıldığı her N-inci satır için birincil anahtarın değerine sahiptir `index_granularity` (genellikle, n = 8192). Ayrıca, her sütun için, biz var `column.mrk` dosyaları ile “marks,” veri dosyasındaki her N-inci satıra ofset olan. Her işaret bir çifttir: dosyadaki ofset sıkıştırılmış bloğun başlangıcına ve sıkıştırılmış bloktaki ofset verilerin başlangıcına. Genellikle, sıkıştırılmış bloklar işaretlerle hizalanır ve sıkıştırılmış bloktaki ofset sıfırdır. İçin veri `primary.idx` her zaman bellekte bulunur ve veri `column.mrk` dosyalar önbelleğe alınır. +Birincil anahtarın kendisi “sparse”. Her satır Adres yok ama verilerin sadece biraz değişir. Ayıran `primary.idx` dosya, n'nin çağrıldığı her N-inci satır için birincil anahtarın değerine sahiptir `index_granularity` (genellikle, n = 8192). Ayrıca, her sütun için, biz var `column.mrk` dosyaları ile “marks,” veri dosyasındaki her N-inci satıra ofset olan. Her işaret bir çifttir: dosyadaki ofset sıkıştırılmış bloğun başlangıcına ve sıkıştırılmış bloktaki ofset verilerin başlangıcına. Genellikle, sıkıştırılmış bloklar işaretlerle hizalanır ve sıkıştırılmış bloktaki ofset sıfırdır. İçin veri `primary.idx` her zaman bellekte bulunur ve veri `column.mrk` dosyalar önbelleğe alınır. Bir kısm aından bir şey okuy readacağımız zaman `MergeTree` bak biz `primary.idx` veri ve istenen verileri içerebilecek aralıkları bulun, ardından `column.mrk` veri ve bu aralıkları okumaya başlamak için nerede için uzaklıklar hesaplayın. Çünkü seyrek, fazla veri okunabilir. ClickHouse, basit nokta sorgularının yüksek bir yükü için uygun değildir, çünkü tüm Aralık `index_granularity` her anahtar için satırlar okunmalı ve her sütun için sıkıştırılmış bloğun tamamı sıkıştırılmalıdır. Dizin için fark edilebilir bellek tüketimi olmadan tek bir sunucu başına trilyonlarca satır tutabilmemiz gerektiğinden dizini seyrek yaptık. Ayrıca, birincil anahtar seyrek olduğundan, benzersiz değildir: ekleme zamanında tablodaki anahtarın varlığını denetleyemez. Bir tabloda aynı anahtara sahip birçok satır olabilir. Ne zaman sen `INSERT` içine veri bir demet `MergeTree`, bu grup birincil anahtar sırasına göre sıralanır ve yeni bir bölüm oluşturur. Bazı parçaları periyodik olarak seçen ve parça sayısını nispeten düşük tutmak için bunları tek bir sıralanmış parçaya birleştiren arka plan iş parçacıkları vardır. Bu yüzden denir `MergeTree`. Tabii ki, birleştirme yol açar “write amplification”. Tüm parçalar değişmez: sadece oluşturulur ve silinir, ancak değiştirilmez. SELECT yürütüldüğünde, tablonun bir anlık görüntüsünü (bir parça kümesi) tutar. Birleştirildikten sonra, arızadan sonra iyileşmeyi kolaylaştırmak için eski parçaları bir süre tutuyoruz, bu nedenle birleştirilmiş bir parçanın muhtemelen kırıldığını görürsek, kaynak parçalarıyla değiştirebiliriz. -`MergeTree` içermediği için bir lsm ağacı değildir “memtable” ve “log”: inserted data is written directly to the filesystem. This makes it suitable only to INSERT data in batches, not by individual row and not very frequently – about once per second is ok, but a thousand times a second is not. We did it this way for simplicity’s sake, and because we are already inserting data in batches in our applications. +`MergeTree` içermediği için bir lsm ağacı değildir “memtable” ve “log”: inserted data is written directly to the filesystem. This makes it suitable only to INSERT data in batches, not by individual row and not very frequently – about once per second is ok, but a thousand times a second is not. We did it this way for simplicity's sake, and because we are already inserting data in batches in our applications. > MergeTree tabloları yalnızca bir (birincil) dizine sahip olabilir: herhangi bir ikincil dizin yoktur. Bir mantıksal tablo altında birden fazla fiziksel gösterime izin vermek, örneğin verileri birden fazla fiziksel sırayla depolamak veya hatta orijinal verilerle birlikte önceden toplanmış verilerle gösterimlere izin vermek güzel olurdu. diff --git a/docs/tr/development/browse-code.md b/docs/tr/development/browse-code.md index 58c6f8fac63..a4b6a90020e 100644 --- a/docs/tr/development/browse-code.md +++ b/docs/tr/development/browse-code.md @@ -1,14 +1,14 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 63 -toc_title: "ClickHouse Kaynak Koduna G\xF6z At\u0131n" +toc_title: "Kaynak Koduna G\xF6zat" --- # ClickHouse Kaynak Koduna Göz Atın {#browse-clickhouse-source-code} -Kullanabilirsiniz **Woboq** online kod tarayıcı mevcut [burada](https://clickhouse.tech/codebrowser/html_report///ClickHouse/src/index.html). Bu kod navigasyon ve semantik vurgulama, arama ve indeksleme sağlar. Kod anlık görüntüsü günlük olarak güncellenir. +Kullanabilirsiniz **Woboq** online kod tarayıcı mevcut [burada](https://clickhouse.tech/codebrowser/html_report/ClickHouse/src/index.html). Bu kod navigasyon ve semantik vurgulama, arama ve indeksleme sağlar. Kod anlık görüntüsü günlük olarak güncellenir. Ayrıca, kaynaklara göz atabilirsiniz [GitHub](https://github.com/ClickHouse/ClickHouse) herzamanki. -IDE’NİN ne kullanacağı ile ilgileniyorsanız, CLion, QT Creator, vs Code ve KDevelop (uyarılar ile) öneririz. Herhangi bir favori IDE kullanabilirsiniz. Vim ve Emacs da sayılır. +IDE'NİN ne kullanacağı ile ilgileniyorsanız, CLion, QT Creator, vs Code ve KDevelop (uyarılar ile) öneririz. Herhangi bir favori IDE kullanabilirsiniz. Vim ve Emacs da sayılır. diff --git a/docs/tr/development/build-cross-arm.md b/docs/tr/development/build-cross-arm.md index 7801d88c265..f0a283bfd98 100644 --- a/docs/tr/development/build-cross-arm.md +++ b/docs/tr/development/build-cross-arm.md @@ -1,20 +1,20 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 67 toc_title: "AARCH64 (ARM64) i\xE7in Linux'ta ClickHouse nas\u0131l olu\u015Fturulur)" --- -# AARCH64 (ARM64) Mimarisi için Linux’ta ClickHouse nasıl oluşturulur {#how-to-build-clickhouse-on-linux-for-aarch64-arm64-architecture} +# AARCH64 (ARM64) mimarisi için Linux'ta ClickHouse nasıl oluşturulur {#how-to-build-clickhouse-on-linux-for-aarch64-arm64-architecture} Bu, Linux makineniz olduğunda ve onu oluşturmak için kullanmak istediğinizde geçerlidir `clickhouse` AARCH64 CPU mimarisi ile başka bir Linux makinede çalışacak ikili. Bu, Linux sunucularında çalışan sürekli entegrasyon kontrolleri için tasarlanmıştır. AARCH64 için çapraz yapı, [Inşa talimatları](build.md) önce onları takip et. -# Clang-8’i Yükle {#install-clang-8} +# Clang-8'i Yükle {#install-clang-8} Yönergeleri izleyin https://apt.llvm.org / Ubuntu veya Debian kurulumunuz için. -Örneğin, Ubuntu Bionic’te aşağıdaki komutları kullanabilirsiniz: +Örneğin, Ubuntu Bionic'te aşağıdaki komutları kullanabilirsiniz: ``` bash echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic-8 main" | sudo tee /etc/apt/sources.list.d/llvm.list @@ -40,4 +40,4 @@ CC=clang-8 CXX=clang++-8 cmake . -Bbuild-arm64 -DCMAKE_TOOLCHAIN_FILE=cmake/linu ninja -C build-arm64 ``` -Ortaya çıkan ikili, yalnızca AARCH64 CPU mimarisi ile Linux’ta çalışacaktır. +Ortaya çıkan ikili, yalnızca AARCH64 CPU mimarisi ile Linux'ta çalışacaktır. diff --git a/docs/tr/development/build-cross-osx.md b/docs/tr/development/build-cross-osx.md index 0e56b0aeb78..1c9e28746c1 100644 --- a/docs/tr/development/build-cross-osx.md +++ b/docs/tr/development/build-cross-osx.md @@ -1,17 +1,17 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 66 toc_title: "Mac OS X i\xE7in Linux'ta ClickHouse nas\u0131l olu\u015Fturulur" --- -# Mac OS X için Linux’ta ClickHouse nasıl oluşturulur {#how-to-build-clickhouse-on-linux-for-mac-os-x} +# Mac OS X için Linux'ta ClickHouse nasıl oluşturulur {#how-to-build-clickhouse-on-linux-for-mac-os-x} -Bu, Linux makineniz olduğunda ve onu oluşturmak için kullanmak istediğinizde geçerlidir `clickhouse` OS X üzerinde çalışacak ikili. bu, Linux sunucularında çalışan sürekli entegrasyon kontrolleri için tasarlanmıştır. Clickhouse’u doğrudan Mac OS X’te oluşturmak istiyorsanız, devam edin [başka bir talimat](build-osx.md). +Bu, Linux makineniz olduğunda ve onu oluşturmak için kullanmak istediğinizde geçerlidir `clickhouse` OS X üzerinde çalışacak ikili. bu, Linux sunucularında çalışan sürekli entegrasyon kontrolleri için tasarlanmıştır. Clickhouse'u doğrudan Mac OS X'te oluşturmak istiyorsanız, devam edin [başka bir talimat](build-osx.md). Mac OS X için çapraz yapı, [Inşa talimatları](build.md) önce onları takip et. -# Clang-8’i Yükle {#install-clang-8} +# Clang-8'i Yükle {#install-clang-8} Yönergeleri izleyin https://apt.llvm.org / Ubuntu veya Debian kurulumunuz için. Örneğin biyonik için komutlar gibidir: @@ -40,7 +40,7 @@ cd cctools-port/cctools make install ``` -Ayrıca, MacOS X SDK’YI çalışma ağacına indirmemiz gerekiyor. +Ayrıca, MacOS X SDK'YI çalışma ağacına indirmemiz gerekiyor. ``` bash cd ClickHouse diff --git a/docs/tr/development/build-osx.md b/docs/tr/development/build-osx.md index f2ea9ac6e46..f8cecc46c3c 100644 --- a/docs/tr/development/build-osx.md +++ b/docs/tr/development/build-osx.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 65 toc_title: "Mac OS X \xFCzerinde ClickHouse nas\u0131l olu\u015Fturulur" --- @@ -15,7 +15,7 @@ Build Mac OS X 10.15 (Catalina) üzerinde çalışmalıdır) $ /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" ``` -## Gerekli Derleyicileri, araçları Ve kitaplıkları yükleyin {#install-required-compilers-tools-and-libraries} +## Gerekli derleyicileri, araçları ve kitaplıkları yükleyin {#install-required-compilers-tools-and-libraries} ``` bash $ brew install cmake ninja libtool gettext diff --git a/docs/tr/development/build.md b/docs/tr/development/build.md index 8d29e647dd6..18ef2cd66ae 100644 --- a/docs/tr/development/build.md +++ b/docs/tr/development/build.md @@ -1,17 +1,17 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 64 toc_title: "Linux \xFCzerinde ClickHouse nas\u0131l olu\u015Fturulur" --- -# Geliştirme için ClickHouse nasıl inşa Edilir {#how-to-build-clickhouse-for-development} +# Geliştirme için ClickHouse nasıl inşa edilir {#how-to-build-clickhouse-for-development} Aşağıdaki öğretici Ubuntu Linux sistemine dayanmaktadır. Uygun değişikliklerle, başka herhangi bir Linux dağıtımı üzerinde de çalışması gerekir. Desteklenen platformlar: x86\_64 ve AArch64. Power9 için destek deneyseldir. -## Git, Cmake, Python Ve Ninja’yı yükleyin {#install-git-cmake-python-and-ninja} +## Git, Cmake, Python ve Ninja'yı yükleyin {#install-git-cmake-python-and-ninja} ``` bash $ sudo apt-get install git cmake python ninja-build @@ -19,11 +19,11 @@ $ sudo apt-get install git cmake python ninja-build Veya eski sistemlerde cmake yerine cmake3. -## Gcc 9’u yükle {#install-gcc-9} +## Gcc 9'u yükle {#install-gcc-9} Bunu yapmak için çeşitli yollar vardır. -### Bir PPA Paketinden yükleme {#install-from-a-ppa-package} +### Bir PPA paketinden yükleme {#install-from-a-ppa-package} ``` bash $ sudo apt-get install software-properties-common @@ -69,7 +69,7 @@ $ cd .. Bir yürütülebilir dosya oluşturmak için çalıştırın `ninja clickhouse`. Bu yaratacak `programs/clickhouse` ile kullanılabilecek çalıştırılabilir `client` veya `server` değişkenler. -# Herhangi Bir Linux üzerinde ClickHouse nasıl oluşturulur {#how-to-build-clickhouse-on-any-linux} +# Herhangi bir Linux üzerinde ClickHouse nasıl oluşturulur {#how-to-build-clickhouse-on-any-linux} Yapı aşağıdaki bileşenleri gerektirir: @@ -108,7 +108,7 @@ Fedora Rawhide için örnek: cmake ../ClickHouse make -j $(nproc) -# ClickHouse inşa Etmek Zorunda değilsiniz {#you-dont-have-to-build-clickhouse} +# ClickHouse inşa etmek zorunda değilsiniz {#you-dont-have-to-build-clickhouse} ClickHouse önceden oluşturulmuş ikili ve paketlerde mevcuttur. İkili dosyalar taşınabilir ve herhangi bir Linux lezzet üzerinde çalıştırılabilir. @@ -116,9 +116,9 @@ Onlar sürece her Master taahhüt ve her çekme isteği için kararlı, prestabl En taze yapıyı bulmak için `master`, go to [taahhüt sayfası](https://github.com/ClickHouse/ClickHouse/commits/master), commit yakınındaki ilk yeşil onay işaretini veya kırmızı çarpı işaretini tıklayın ve “Details” hemen sonra bağlantı “ClickHouse Build Check”. -# ClickHouse Debian Paketi nasıl oluşturulur {#how-to-build-clickhouse-debian-package} +# ClickHouse Debian paketi nasıl oluşturulur {#how-to-build-clickhouse-debian-package} -## Git Ve Pbuilder’ı yükleyin {#install-git-and-pbuilder} +## Git ve Pbuilder'ı yükleyin {#install-git-and-pbuilder} ``` bash $ sudo apt-get update diff --git a/docs/tr/development/contrib.md b/docs/tr/development/contrib.md index 7f09d9bc49b..de254f992dc 100644 --- a/docs/tr/development/contrib.md +++ b/docs/tr/development/contrib.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 70 toc_title: "Kullan\u0131lan \xDC\xE7\xFCnc\xFC Taraf K\xFCt\xFCphaneleri" --- diff --git a/docs/tr/development/developer-instruction.md b/docs/tr/development/developer-instruction.md index 8e5f83e00a4..a65c6666288 100644 --- a/docs/tr/development/developer-instruction.md +++ b/docs/tr/development/developer-instruction.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 61 toc_title: "Acemi ClickHouse Geli\u015Ftirici Talimat" --- @@ -9,17 +9,17 @@ ClickHouse binası Linux, FreeBSD ve Mac OS X üzerinde desteklenmektedir. # Windows Kullanıyorsanız {#if-you-use-windows} -Windows kullanıyorsanız, Ubuntu ile bir sanal makine oluşturmanız gerekir. Bir sanal makine ile çalışmaya başlamak için VirtualBox yükleyin. UB :unt :u’yu web sitesinden indirebilirsiniz: https://www.ubuntu.com/\#download. lütfen indirilen görüntüden bir sanal makine oluşturun (bunun için en az 4GB RAM ayırmalısınız). Ubuntu’da bir komut satırı terminali çalıştırmak için lütfen kelimeyi içeren bir program bulun “terminal” adına (gnome-terminal, konsole vb.)) veya sadece Ctrl+Alt+T tuşlarına basın. +Windows kullanıyorsanız, Ubuntu ile bir sanal makine oluşturmanız gerekir. Bir sanal makine ile çalışmaya başlamak için VirtualBox yükleyin. UB :unt :u'yu web sitesinden indirebilirsiniz: https://www.ubuntu.com/\#download. lütfen indirilen görüntüden bir sanal makine oluşturun (bunun için en az 4GB RAM ayırmalısınız). Ubuntu'da bir komut satırı terminali çalıştırmak için lütfen kelimeyi içeren bir program bulun “terminal” adına (gnome-terminal, konsole vb.)) veya sadece Ctrl+Alt+T tuşlarına basın. # 32 bit sistem kullanıyorsanız {#if-you-use-a-32-bit-system} ClickHouse çalışamaz veya 32-bit bir sistem üzerinde oluşturun. 64-bit bir sisteme erişim kazanmanız gerekir ve okumaya devam edebilirsiniz. -# Github’da Bir Depo oluşturma {#creating-a-repository-on-github} +# Github'da bir depo oluşturma {#creating-a-repository-on-github} ClickHouse repository ile çalışmaya başlamak için bir GitHub hesabına ihtiyacınız olacaktır. -Muhtemelen zaten bir tane var, ama yapmazsanız, lütfen kayıt olun https://github.com. SSH anahtarlarınız yoksa, bunları üretmeli ve daha sonra Github’a yüklemelisiniz. Bu yamalar üzerinden göndermek için gereklidir. Diğer SSH sunucularıyla kullandığınız aynı SSH anahtarlarını kullanmak da mümkündür - muhtemelen zaten bunlara sahipsiniz. +Muhtemelen zaten bir tane var, ama yapmazsanız, lütfen kayıt olun https://github.com. SSH anahtarlarınız yoksa, bunları üretmeli ve daha sonra Github'a yüklemelisiniz. Bu yamalar üzerinden göndermek için gereklidir. Diğer SSH sunucularıyla kullandığınız aynı SSH anahtarlarını kullanmak da mümkündür - muhtemelen zaten bunlara sahipsiniz. ClickHouse deposunun bir çatalı oluşturun. Bunu yapmak için lütfen tıklayın “fork” sağ üst köşedeki düğme https://github.com/ClickHouse/ClickHouse. bu hesabınıza ClickHouse / ClickHouse kendi kopyasını çatal olacaktır. @@ -27,7 +27,7 @@ Geliştirme süreci ilk ClickHouse sizin çatal içine amaçlanan değişiklikle Git depoları ile çalışmak için lütfen yükleyin `git`. -Bunu Ubuntu’da yapmak için komut satırı terminalinde çalışırsınız: +Bunu Ubuntu'da yapmak için komut satırı terminalinde çalışırsınız: sudo apt update sudo apt install git @@ -35,13 +35,13 @@ Bunu Ubuntu’da yapmak için komut satırı terminalinde çalışırsınız: Git kullanımı ile ilgili kısa bir el kitabı burada bulunabilir: https://services.github.com/on-demand/downloads/github-git-cheat-sheet.pdf. Git ile ilgili ayrıntılı bir el kitabı için bkz. https://git-scm.com/book/en/v2. -# Geliştirme Makinenize Bir Depo Klonlama {#cloning-a-repository-to-your-development-machine} +# Geliştirme Makinenize bir depo klonlama {#cloning-a-repository-to-your-development-machine} Ardından, kaynak dosyaları çalışma makinenize indirmeniz gerekir. Bu denir “to clone a repository” çünkü çalışma makinenizde deponun yerel bir kopyasını oluşturur. Komut satırında terminal Çalıştır: - git clone --recursive git@guthub.com:your_github_username/ClickHouse.git + git clone --recursive git@github.com:your_github_username/ClickHouse.git cd ClickHouse Not: lütfen, yerine *your\_github\_username* uygun olanı ile! @@ -65,7 +65,7 @@ Aşağıdaki hata iletisini alırsanız: Please make sure you have the correct access rights and the repository exists. -Genellikle Github’a bağlanmak için SSH anahtarlarının eksik olduğu anlamına gelir. Bu anahtarlar normalde `~/.ssh`. SSH anahtarlarının kabul edilmesi için bunları GitHub kullanıcı arayüzünün ayarlar bölümüne yüklemeniz gerekir. +Genellikle Github'a bağlanmak için SSH anahtarlarının eksik olduğu anlamına gelir. Bu anahtarlar normalde `~/.ssh`. SSH anahtarlarının kabul edilmesi için bunları GitHub kullanıcı arayüzünün ayarlar bölümüne yüklemeniz gerekir. Depoyu https protokolü aracılığıyla da klonlayabilirsiniz: @@ -73,7 +73,7 @@ Depoyu https protokolü aracılığıyla da klonlayabilirsiniz: Ancak bu, değişikliklerinizi sunucuya göndermenize izin vermez. Yine de geçici olarak kullanabilir ve SSH anahtarlarını daha sonra deponun uzak adresini değiştirerek ekleyebilirsiniz `git remote` komut. -Oradan güncellemeleri çekmek için orijinal ClickHouse repo’nun adresini yerel deponuza da ekleyebilirsiniz: +Oradan güncellemeleri çekmek için orijinal ClickHouse repo'nun adresini yerel deponuza da ekleyebilirsiniz: git remote add upstream git@github.com:ClickHouse/ClickHouse.git @@ -81,7 +81,7 @@ Başarıyla bu komutu çalıştırdıktan sonra çalıştırarak ana ClickHouse ## Alt modüllerle çalışma {#working-with-submodules} -Git’teki alt modüllerle çalışmak acı verici olabilir. Sonraki komutlar onu yönetmeye yardımcı olacaktır: +Git'teki alt modüllerle çalışmak acı verici olabilir. Sonraki komutlar onu yönetmeye yardımcı olacaktır: # ! each command accepts --recursive # Update remote URLs for submodules. Barely rare case @@ -116,18 +116,18 @@ ClickHouse bina için Cmake ve Ninja kullanır. Cmake-ninja dosyaları (yapı görevleri) üretebilir bir meta-yapı sistemi. Ninja-bu cmake oluşturulan görevleri yürütmek için kullanılan hıza odaklanarak daha küçük bir yapı sistemi. -Ubuntu, Debian veya Mint run’a yüklemek için `sudo apt install cmake ninja-build`. +Ubuntu, Debian veya Mint run'a yüklemek için `sudo apt install cmake ninja-build`. -Centos’ta, RedHat koşusu `sudo yum install cmake ninja-build`. +Centos'ta, RedHat koşusu `sudo yum install cmake ninja-build`. -Arch veya Gentoo kullanıyorsanız, muhtemelen cmake’i nasıl kuracağınızı kendiniz biliyorsunuz. +Arch veya Gentoo kullanıyorsanız, muhtemelen cmake'i nasıl kuracağınızı kendiniz biliyorsunuz. Mac OS X üzerinde cmake ve Ninja yüklemek için ilk homebrew yüklemek ve daha sonra demlemek yoluyla her şeyi yüklemek: /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" brew install cmake ninja -Ardından, cmake sürümünü kontrol edin: `cmake --version`. 3.3’ün altındaysa, web sitesinden daha yeni bir sürüm yüklemelisiniz: https://cmake.org/download/. +Ardından, cmake sürümünü kontrol edin: `cmake --version`. 3.3'ün altındaysa, web sitesinden daha yeni bir sürüm yüklemelisiniz: https://cmake.org/download/. # İsteğe Bağlı Harici Kütüphaneler {#optional-external-libraries} @@ -137,11 +137,11 @@ ClickHouse, bina için birkaç dış kütüphane kullanır. Alt modüllerde bulu Derleyiciler gcc sürüm 9 ve Clang sürüm 8 veya üzeri başlayarak ClickHouse bina için desteklenmektedir. -Resmi Yandex şu anda GCC’Yİ kullanıyor çünkü biraz daha iyi performansa sahip makine kodu üretiyor (kriterlerimize göre yüzde birkaçına kadar bir fark yaratıyor). Ve Clang genellikle geliştirme için daha uygundur. Yine de, sürekli entegrasyon (CI) platformumuz yaklaşık bir düzine yapı kombinasyonunu denetler. +Resmi Yandex şu anda GCC'Yİ kullanıyor çünkü biraz daha iyi performansa sahip makine kodu üretiyor (kriterlerimize göre yüzde birkaçına kadar bir fark yaratıyor). Ve Clang genellikle geliştirme için daha uygundur. Yine de, sürekli entegrasyon (CI) platformumuz yaklaşık bir düzine yapı kombinasyonunu denetler. Ubuntu run GCC yüklemek için: `sudo apt install gcc g++` -Gcc sürümünü kontrol edin: `gcc --version`. 9’un altındaysa, buradaki talimatları izleyin: https://clickhouse.tech / docs/TR/development / build / \#ınstall-gcc-9. +Gcc sürümünü kontrol edin: `gcc --version`. 9'un altındaysa, buradaki talimatları izleyin: https://clickhouse.tech / docs/TR/development / build / \#ınstall-gcc-9. Mac OS X build sadece Clang için desteklenir. Sadece koş `brew install llvm` @@ -182,7 +182,7 @@ Bu komutu çalıştırarak yapı türünü değiştirebilirsiniz. `build` dizin. Bu örnekte yalnızca gerekli ikili dosyalar oluşturulacaktır. -Tüm ikili dosyaları (Yardımcı Programlar ve testler) oluşturmanız gerekiyorsa, ninja’yı parametre olmadan çalıştırmalısınız: +Tüm ikili dosyaları (Yardımcı Programlar ve testler) oluşturmanız gerekiyorsa, ninja'yı parametre olmadan çalıştırmalısınız: ninja @@ -192,7 +192,7 @@ Yapı makinesinde büyük miktarda RAM mevcut olduğunda, paralel olarak çalı ninja -j 1 clickhouse-server clickhouse-client -4GB RAM’Lİ makinelerde, 8GB RAM için 1 belirtmeniz önerilir `-j 2` tavsiye edilir. +4GB RAM'Lİ makinelerde, 8GB RAM için 1 belirtmeniz önerilir `-j 2` tavsiye edilir. Mesajı alırsanız: `ninja: error: loading 'build.ninja': No such file or directory` bu, bir yapı yapılandırması oluşturmanın başarısız olduğu ve yukarıdaki mesajı incelemeniz gerektiği anlamına gelir. @@ -204,21 +204,21 @@ Başarılı bir yapı üzerine yürütülebilir bir dosya alırsınız `ClickHou ls -l programs/clickhouse -# Clickhouse’un yerleşik yürütülebilir dosyasını çalıştırma {#running-the-built-executable-of-clickhouse} +# Clickhouse'un yerleşik yürütülebilir dosyasını çalıştırma {#running-the-built-executable-of-clickhouse} Sunucuyu geçerli kullanıcı altında çalıştırmak için aşağıdakilere gitmeniz gerekir `ClickHouse/programs/server/` (dışında bulunan `build`) ve koş: - ../../../build/programs/clickhouse server + ../../build/programs/clickhouse server Bu durumda, ClickHouse geçerli dizinde bulunan yapılandırma dosyalarını kullanır. Koş youabilirsiniz `clickhouse server` komut satırı parametresi olarak bir yapılandırma dosyasının yolunu belirten herhangi bir dizinden `--config-file`. -Başka bir terminalde clickhouse-client ile Clickhouse’a bağlanmak için `ClickHouse/build/programs/` ve koş `clickhouse client`. +Başka bir terminalde clickhouse-client ile Clickhouse'a bağlanmak için `ClickHouse/build/programs/` ve koş `clickhouse client`. -Eğer alırsanız `Connection refused` Mac OS X veya Freebsd’de mesaj, ana bilgisayar adresi 127.0.0.1 belirtmeyi deneyin: +Eğer alırsanız `Connection refused` Mac OS X veya Freebsd'de mesaj, ana bilgisayar adresi 127.0.0.1 belirtmeyi deneyin: clickhouse client --host 127.0.0.1 -Sisteminizde yüklü olan ClickHouse binary’nin üretim sürümünü özel olarak oluşturulmuş ClickHouse binaryinizle değiştirebilirsiniz. Bunu yapmak için resmi web sitesinden talimatları izleyerek Makinenize ClickHouse yükleyin. Ardından, aşağıdakileri çalıştırın: +Sisteminizde yüklü olan ClickHouse binary'nin üretim sürümünü özel olarak oluşturulmuş ClickHouse binaryinizle değiştirebilirsiniz. Bunu yapmak için resmi web sitesinden talimatları izleyerek Makinenize ClickHouse yükleyin. Ardından, aşağıdakileri çalıştırın: sudo service clickhouse-server stop sudo cp ClickHouse/build/programs/clickhouse /usr/bin/ @@ -235,11 +235,11 @@ Ayrıca sisteminizde yüklü ClickHouse paketinden yapılandırma dosyası ile Hangi IDE kullanmak bilmiyorsanız, clion kullanmanızı öneririz. CLion ticari bir yazılımdır, ancak 30 günlük ücretsiz deneme süresi sunar. Öğrenciler için de ücretsizdir. CLion Linux ve Mac OS X hem de kullanılabilir. -KDevelop ve QTCreator, ClickHouse geliştirmek için bir IDE’NİN diğer harika alternatifleridir. KDevelop kararsız olmasına rağmen çok kullanışlı bir IDE olarak geliyor. KDevelop projeyi açtıktan sonra bir süre sonra çökerse, tıklamanız gerekir “Stop All” proje dosyalarının listesini açar açmaz düğme. Bunu yaptıktan sonra KDevelop ile çalışmak iyi olmalıdır. +KDevelop ve QTCreator, ClickHouse geliştirmek için bir IDE'NİN diğer harika alternatifleridir. KDevelop kararsız olmasına rağmen çok kullanışlı bir IDE olarak geliyor. KDevelop projeyi açtıktan sonra bir süre sonra çökerse, tıklamanız gerekir “Stop All” proje dosyalarının listesini açar açmaz düğme. Bunu yaptıktan sonra KDevelop ile çalışmak iyi olmalıdır. -Basit kod editörleri olarak, Yüce metin veya Visual Studio kodunu veya Kate’i (hepsi Linux’ta kullanılabilir) kullanabilirsiniz. +Basit kod editörleri olarak, Yüce metin veya Visual Studio kodunu veya Kate'i (hepsi Linux'ta kullanılabilir) kullanabilirsiniz. -Her ihtimale karşı, Clion’un yarattığını belirtmek gerekir `build` kendi başına yol, aynı zamanda kendi seçtikleri `debug` yapı türü için, yapılandırma için Clion’da tanımlanan ve sizin tarafınızdan yüklenmeyen bir cmake sürümünü kullanır ve son olarak CLion kullanacaktır `make` yerine yapı görevlerini çalıştırmak için `ninja`. Bu normal bir davranıştır, sadece karışıklığı önlemek için bunu aklınızda bulundurun. +Her ihtimale karşı, Clion'un yarattığını belirtmek gerekir `build` kendi başına yol, aynı zamanda kendi seçtikleri `debug` yapı türü için, yapılandırma için Clion'da tanımlanan ve sizin tarafınızdan yüklenmeyen bir cmake sürümünü kullanır ve son olarak CLion kullanacaktır `make` yerine yapı görevlerini çalıştırmak için `ninja`. Bu normal bir davranıştır, sadece karışıklığı önlemek için bunu aklınızda bulundurun. # Kod Yazma {#writing-code} @@ -253,7 +253,7 @@ Görevlerin listesi: https://github.com/ClickHouse/ClickHouse/blob/master/testsr # Test Verileri {#test-data} -Clickhouse’un geliştirilmesi genellikle gerçekçi veri kümelerinin yüklenmesini gerektirir. Performans testi için özellikle önemlidir. Yandex’ten özel olarak hazırlanmış anonim veri setimiz var.Metrica. Ayrıca bazı 3GB boş disk alanı gerektirir. Bu verilerin geliştirme görevlerinin çoğunu gerçekleştirmek için gerekli olmadığını unutmayın. +Clickhouse'un geliştirilmesi genellikle gerçekçi veri kümelerinin yüklenmesini gerektirir. Performans testi için özellikle önemlidir. Yandex'ten özel olarak hazırlanmış anonim veri setimiz var.Metrica. Ayrıca bazı 3GB boş disk alanı gerektirir. Bu verilerin geliştirme görevlerinin çoğunu gerçekleştirmek için gerekli olmadığını unutmayın. sudo apt install wget xz-utils @@ -265,6 +265,8 @@ Clickhouse’un geliştirilmesi genellikle gerçekçi veri kümelerinin yüklenm clickhouse-client + CREATE DATABASE IF NOT EXISTS test + CREATE TABLE test.hits ( WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String, RefererDomain String, Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), UTCEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), RemoteIP UInt32, RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming Int32, DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32, FetchTiming Int32, RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32, LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32, RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String, `ParsedParams.Key1` Array(String), `ParsedParams.Key2` Array(String), `ParsedParams.Key3` Array(String), `ParsedParams.Key4` Array(String), `ParsedParams.Key5` Array(String), `ParsedParams.ValueDouble` Array(Float64), IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree PARTITION BY toYYYYMM(EventDate) SAMPLE BY intHash32(UserID) ORDER BY (CounterID, EventDate, intHash32(UserID), EventTime); CREATE TABLE test.visits ( CounterID UInt32, StartDate Date, Sign Int8, IsNew UInt8, VisitID UInt64, UserID UInt64, StartTime DateTime, Duration UInt32, UTCStartTime DateTime, PageViews Int32, Hits Int32, IsBounce UInt8, Referer String, StartURL String, RefererDomain String, StartURLDomain String, EndURL String, LinkURL String, IsDownload UInt8, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, PlaceID Int32, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), IsYandex UInt8, GoalReachesDepth Int32, GoalReachesURL Int32, GoalReachesAny Int32, SocialSourceNetworkID UInt8, SocialSourcePage String, MobilePhoneModel String, ClientEventTime DateTime, RegionID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RemoteIP UInt32, RemoteIP6 FixedString(16), IPNetworkID UInt32, SilverlightVersion3 UInt32, CodeVersion UInt32, ResolutionWidth UInt16, ResolutionHeight UInt16, UserAgentMajor UInt16, UserAgentMinor UInt16, WindowClientWidth UInt16, WindowClientHeight UInt16, SilverlightVersion2 UInt8, SilverlightVersion4 UInt16, FlashVersion3 UInt16, FlashVersion4 UInt16, ClientTimeZone Int16, OS UInt8, UserAgent UInt8, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, NetMajor UInt8, NetMinor UInt8, MobilePhone UInt8, SilverlightVersion1 UInt8, Age UInt8, Sex UInt8, Income UInt8, JavaEnable UInt8, CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, BrowserLanguage UInt16, BrowserCountry UInt16, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), Params Array(String), `Goals.ID` Array(UInt32), `Goals.Serial` Array(UInt32), `Goals.EventTime` Array(DateTime), `Goals.Price` Array(Int64), `Goals.OrderID` Array(String), `Goals.CurrencyID` Array(UInt32), WatchIDs Array(UInt64), ParamSumPrice Int64, ParamCurrency FixedString(3), ParamCurrencyID UInt16, ClickLogID UInt64, ClickEventID Int32, ClickGoodEvent Int32, ClickEventTime DateTime, ClickPriorityID Int32, ClickPhraseID Int32, ClickPageID Int32, ClickPlaceID Int32, ClickTypeID Int32, ClickResourceID Int32, ClickCost UInt32, ClickClientIP UInt32, ClickDomainID UInt32, ClickURL String, ClickAttempt UInt8, ClickOrderID UInt32, ClickBannerID UInt32, ClickMarketCategoryID UInt32, ClickMarketPP UInt32, ClickMarketCategoryName String, ClickMarketPPName String, ClickAWAPSCampaignName String, ClickPageName String, ClickTargetType UInt16, ClickTargetPhraseID UInt64, ClickContextType UInt8, ClickSelectType Int8, ClickOptions String, ClickGroupBannerID Int32, OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, FirstVisit DateTime, PredLastVisit Date, LastVisit Date, TotalVisits UInt32, `TraficSource.ID` Array(Int8), `TraficSource.SearchEngineID` Array(UInt16), `TraficSource.AdvEngineID` Array(UInt8), `TraficSource.PlaceID` Array(UInt16), `TraficSource.SocialSourceNetworkID` Array(UInt8), `TraficSource.Domain` Array(String), `TraficSource.SearchPhrase` Array(String), `TraficSource.SocialSourcePage` Array(String), Attendance FixedString(16), CLID UInt32, YCLID UInt64, NormalizedRefererHash UInt64, SearchPhraseHash UInt64, RefererDomainHash UInt64, NormalizedStartURLHash UInt64, StartURLDomainHash UInt64, NormalizedEndURLHash UInt64, TopLevelDomain UInt64, URLScheme UInt64, OpenstatServiceNameHash UInt64, OpenstatCampaignIDHash UInt64, OpenstatAdIDHash UInt64, OpenstatSourceIDHash UInt64, UTMSourceHash UInt64, UTMMediumHash UInt64, UTMCampaignHash UInt64, UTMContentHash UInt64, UTMTermHash UInt64, FromHash UInt64, WebVisorEnabled UInt8, WebVisorActivity UInt32, `ParsedParams.Key1` Array(String), `ParsedParams.Key2` Array(String), `ParsedParams.Key3` Array(String), `ParsedParams.Key4` Array(String), `ParsedParams.Key5` Array(String), `ParsedParams.ValueDouble` Array(Float64), `Market.Type` Array(UInt8), `Market.GoalID` Array(UInt32), `Market.OrderID` Array(String), `Market.OrderPrice` Array(Int64), `Market.PP` Array(UInt32), `Market.DirectPlaceID` Array(UInt32), `Market.DirectOrderID` Array(UInt32), `Market.DirectBannerID` Array(UInt32), `Market.GoodID` Array(String), `Market.GoodName` Array(String), `Market.GoodQuantity` Array(Int32), `Market.GoodPrice` Array(Int64), IslandID FixedString(16)) ENGINE = CollapsingMergeTree(Sign) PARTITION BY toYYYYMM(StartDate) SAMPLE BY intHash32(UserID) ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID); @@ -274,11 +276,11 @@ Clickhouse’un geliştirilmesi genellikle gerçekçi veri kümelerinin yüklenm # Çekme İsteği Oluşturma {#creating-pull-request} -Github’un kullanıcı arayüzünde çatal deposuna gidin. Bir dalda gelişiyorsanız, o Dalı seçmeniz gerekir. Bir olacak “Pull request” ekranda bulunan düğme. Özünde, bu demektir “create a request for accepting my changes into the main repository”. +Github'un kullanıcı arayüzünde çatal deposuna gidin. Bir dalda gelişiyorsanız, o Dalı seçmeniz gerekir. Bir olacak “Pull request” ekranda bulunan düğme. Özünde, bu demektir “create a request for accepting my changes into the main repository”. Çalışma henüz tamamlanmamış olsa bile bir çekme isteği oluşturulabilir. Bu durumda lütfen kelimeyi koyun “WIP” (devam eden çalışma) başlığın başında, daha sonra değiştirilebilir. Bu, kooperatif Gözden geçirme ve değişikliklerin tartışılması ve mevcut tüm testlerin çalıştırılması için kullanışlıdır. Değişikliklerinizin kısa bir açıklamasını sağlamanız önemlidir, daha sonra sürüm değişiklikleri oluşturmak için kullanılacaktır. -Yandex çalışanları PR’NİZİ bir etiketle etiketlediğinde testler başlayacaktır “can be tested”. The results of some first checks (e.g. code style) will come in within several minutes. Build check results will arrive within half an hour. And the main set of tests will report itself within an hour. +Yandex çalışanları PR'NİZİ bir etiketle etiketlediğinde testler başlayacaktır “can be tested”. The results of some first checks (e.g. code style) will come in within several minutes. Build check results will arrive within half an hour. And the main set of tests will report itself within an hour. Sistem, çekme isteğiniz için ayrı ayrı ClickHouse ikili yapıları hazırlayacaktır. Bu yapıları almak için tıklayın “Details” yanındaki bağlantı “ClickHouse build check” çekler listesinde giriş. Orada inşa doğrudan bağlantılar bulacaksınız .eğer üretim sunucularında bile dağıtabilirsiniz ClickHouse DEB paketleri (eğer hiçbir korku varsa). diff --git a/docs/tr/development/index.md b/docs/tr/development/index.md index fdd4c0c0805..ef972914d46 100644 --- a/docs/tr/development/index.md +++ b/docs/tr/development/index.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_folder_title: "Geli\u015Fme" toc_hidden: true toc_priority: 58 diff --git a/docs/tr/development/style.md b/docs/tr/development/style.md index fae00422d18..3e8f5960d64 100644 --- a/docs/tr/development/style.md +++ b/docs/tr/development/style.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 68 toc_title: "C++ kodu nas\u0131l yaz\u0131l\u0131r" --- -# C++ Kodu nasıl yazılır {#how-to-write-c-code} +# C++ kodu nasıl yazılır {#how-to-write-c-code} ## Genel Öneriler {#general-recommendations} @@ -88,7 +88,7 @@ Gerekirse, operatör bir sonraki satıra sarılabilir. Bu durumda, önündeki of **11.** Tekli operatörleri ayırmak için boşluk kullanmayın (`--`, `++`, `*`, `&`, …) from the argument. -**12.** Virgülden sonra bir boşluk koyun, ancak ondan önce değil. Aynı kural, bir içindeki noktalı virgül için de geçerlidir `for` ifade. +**12.** Virgülden sonra bir boşluk koyun, ancak ondan önce değil. Aynı kural, bir içindeki noktalı virgül için de geçerlidir `for` İfade. **13.** Ayırmak için boşluk kullanmayın `[]` operatör. @@ -270,7 +270,7 @@ void executeQuery( **8.** Tek satırlı yorumlar üç eğik çizgi ile başlar: `///` ve çok satırlı yorumlar ile başlar `/**`. Bu yorumlar dikkate alınır “documentation”. -Not: bu yorumlardan belgeler oluşturmak için Doxygen kullanabilirsiniz. Ancak DOXYGEN genellikle kullanılmaz, çünkü IDE’DEKİ kodda gezinmek daha uygundur. +Not: bu yorumlardan belgeler oluşturmak için Doxygen kullanabilirsiniz. Ancak DOXYGEN genellikle kullanılmaz, çünkü IDE'DEKİ kodda gezinmek daha uygundur. **9.** Çok satırlı açıklamaların başında ve sonunda (çok satırlı bir açıklamayı kapatan satır hariç) boş satırları olmamalıdır. @@ -396,7 +396,7 @@ Bağımsız değişken yapıcı gövdesinde kullanılmazsa, alt çizgi soneki at timer (not m_timer) ``` -**14.** Bir de SAB theitler için `enum`, büyük harfle CamelCase kullanın. ALL\_CAPS da kabul edilebilir. Eğer… `enum` yerel olmayan, bir `enum class`. +**14.** Bir de SAB theitler için `enum`, büyük harfle CamelCase kullanın. ALL\_CAPS da kabul edilebilir. Eğer... `enum` yerel olmayan, bir `enum class`. ``` cpp enum class CompressionMethod @@ -410,7 +410,7 @@ enum class CompressionMethod not Stroka -**16.** Kısaltmalar iyi biliniyorsa kabul edilebilir (kısaltmanın anlamını Wikipedia’da veya bir arama motorunda kolayca bulabilirsiniz). +**16.** Kısaltmalar iyi biliniyorsa kabul edilebilir (kısaltmanın anlamını Wikipedia'da veya bir arama motorunda kolayca bulabilirsiniz). `AST`, `SQL`. @@ -656,13 +656,13 @@ Günlüğünde UTF-8 kodlamasını kullanın. Nadir durumlarda, günlüğünde A Kullanmayın `iostreams` uygulama performansı için kritik olan iç döngülerde (ve asla kullanmayın `stringstream`). -Kullan… `DB/IO` kütüphane yerine. +Kullan... `DB/IO` kütüphane yerine. **21.** Tarih ve zaman. Görmek `DateLUT` kitaplık. -**22.** içermek. +**22.** İçermek. Her zaman kullanın `#pragma once` korumaları dahil etmek yerine. @@ -689,7 +689,7 @@ auto s = std::string{"Hello"}; **26.** Sanal işlevler için yaz `virtual` temel sınıfta, ama yaz `override` yerine `virtual` soyundan gelen sınıflarda. -## C++ ’ ın kullanılmayan özellikleri {#unused-features-of-c} +## C++ ' ın kullanılmayan özellikleri {#unused-features-of-c} **1.** Sanal devralma kullanılmaz. @@ -727,7 +727,7 @@ CPU komut seti, sunucularımız arasında desteklenen minimum kümedir. Şu anda **3.** Profilleme için kullanın `Linux Perf`, `valgrind` (`callgrind`), veya `strace -cf`. -**4.** Kaynaklar Git’te. +**4.** Kaynaklar Git'te. **5.** Montaj kullanımları `CMake`. @@ -795,7 +795,7 @@ Yazar yousan `std::memcpy` yerine `memcpy` her yerde, o zaman `memmem` olarak `s Yine de, hala kullanabilirsiniz `std::` eğer tercih ederseniz edin. -**3.** Aynı olanlar standart C++ kütüphanesinde mevcut olduğunda C’den işlevleri kullanma. +**3.** Aynı olanlar standart C++ kütüphanesinde mevcut olduğunda C'den işlevleri kullanma. Daha verimli ise bu kabul edilebilir. diff --git a/docs/tr/development/tests.md b/docs/tr/development/tests.md index 0ebfebe0316..b2b9d83fe7a 100644 --- a/docs/tr/development/tests.md +++ b/docs/tr/development/tests.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 69 toc_title: "ClickHouse testleri nas\u0131l \xE7al\u0131\u015Ft\u0131r\u0131l\u0131\ r" @@ -14,7 +14,7 @@ Fonksiyonel testler en basit ve kullanımı kolay olanlardır. ClickHouse özell Her işlevsel test, çalışan ClickHouse sunucusuna bir veya birden çok sorgu gönderir ve sonucu referansla karşılaştırır. -Testler bulunur `queries` dizin. İki alt dizin var: `stateless` ve `stateful`. Durumsuz testler, önceden yüklenmiş test verileri olmadan sorguları çalıştırır - genellikle testin kendisinde anında küçük sentetik veri kümeleri oluştururlar. Durum bilgisi testleri, Yandex’ten önceden yüklenmiş test verileri gerektirir.Metrica ve halka açık değil. Biz sadece kullanmak eğilimindedir `stateless` testler ve yeni eklemekten kaçının `stateful` testler. +Testler bulunur `queries` dizin. İki alt dizin var: `stateless` ve `stateful`. Durumsuz testler, önceden yüklenmiş test verileri olmadan sorguları çalıştırır - genellikle testin kendisinde anında küçük sentetik veri kümeleri oluştururlar. Durum bilgisi testleri, Yandex'ten önceden yüklenmiş test verileri gerektirir.Metrica ve halka açık değil. Biz sadece kullanmak eğilimindedir `stateless` testler ve yeni eklemekten kaçının `stateful` testler. Her test iki tipten biri olabilir: `.sql` ve `.sh`. `.sql` test için borulu basit SQL komut dosyasıdır `clickhouse-client --multiquery --testmode`. `.sh` test kendisi tarafından çalıştırılan bir komut dosyasıdır. @@ -41,15 +41,15 @@ Fonksiyonel testlerle kolayca çoğaltılabilen bazı hatalar biliyorsak, hazır ## Entegrasyon Testleri {#integration-tests} -Entegrasyon testleri, kümelenmiş konfigürasyonda Clickhouse’u ve MySQL, Postgres, MongoDB gibi diğer sunucularla ClickHouse etkileşimini test etmeyi sağlar. Ağ bölmelerini, paket damlalarını vb. taklit etmek için kullanışlıdırlar. Bu testler Docker altında çalıştırılır ve çeşitli yazılımlarla birden fazla konteyner oluşturur. +Entegrasyon testleri, kümelenmiş konfigürasyonda Clickhouse'u ve MySQL, Postgres, MongoDB gibi diğer sunucularla ClickHouse etkileşimini test etmeyi sağlar. Ağ bölmelerini, paket damlalarını vb. taklit etmek için kullanışlıdırlar. Bu testler Docker altında çalıştırılır ve çeşitli yazılımlarla birden fazla konteyner oluşturur. Görmek `tests/integration/README.md` bu testlerin nasıl çalıştırılacağı hakkında. -Clickhouse’un üçüncü taraf sürücülerle entegrasyonunun sınanmadığını unutmayın. Ayrıca şu anda JDBC ve ODBC sürücülerimizle entegrasyon testlerimiz yok. +Clickhouse'un üçüncü taraf sürücülerle entegrasyonunun sınanmadığını unutmayın. Ayrıca şu anda JDBC ve ODBC sürücülerimizle entegrasyon testlerimiz yok. ## Ünite Testleri {#unit-tests} -Birim testleri, Clickhouse’u bir bütün olarak değil, tek bir yalıtılmış kitaplık veya sınıfı test etmek istediğinizde kullanışlıdır. Etkinleştirebilir veya devre dışı bırakma ile testlerin yapı `ENABLE_TESTS` Cmake seçeneği. Birim testleri (ve diğer test programları) bulunur `tests` kodun alt dizinleri. Birim testlerini çalıştırmak için şunları yazın `ninja test`. Bazı testler kullanın `gtest`, ancak bazıları test başarısızlığında sıfır olmayan çıkış kodunu döndüren programlardır. +Birim testleri, Clickhouse'u bir bütün olarak değil, tek bir yalıtılmış kitaplık veya sınıfı test etmek istediğinizde kullanışlıdır. Etkinleştirebilir veya devre dışı bırakma ile testlerin yapı `ENABLE_TESTS` Cmake seçeneği. Birim testleri (ve diğer test programları) bulunur `tests` kodun alt dizinleri. Birim testlerini çalıştırmak için şunları yazın `ninja test`. Bazı testler kullanın `gtest`, ancak bazıları test başarısızlığında sıfır olmayan çıkış kodunu döndüren programlardır. Kodun zaten işlevsel testler tarafından kapsanması durumunda birim testlerine sahip olmak zorunlu değildir (ve işlevsel testler genellikle kullanımı çok daha basittir). @@ -59,11 +59,11 @@ Performans testleri ölçmek ve sentetik sorguları ClickHouse bazı izole kısm Her test, durdurma için bazı koşullarla (örneğin, bir döngüde bir veya birden fazla sorgu (muhtemelen parametre kombinasyonlarıyla) çalıştırır “maximum execution speed is not changing in three seconds”) ve sorgu performansı ile ilgili bazı metrikleri ölçün (örneğin “maximum execution speed”). Bazı testler önceden yüklenmiş test veri kümesinde Önkoşullar içerebilir. -Bazı senaryoda Clickhouse’un performansını artırmak istiyorsanız ve basit sorgularda iyileştirmeler gözlemlenebiliyorsa, bir performans testi yazmanız önerilir. Her zaman kullanmak mantıklı `perf top` testleriniz sırasında veya diğer perf araçları. +Bazı senaryoda Clickhouse'un performansını artırmak istiyorsanız ve basit sorgularda iyileştirmeler gözlemlenebiliyorsa, bir performans testi yazmanız önerilir. Her zaman kullanmak mantıklı `perf top` testleriniz sırasında veya diğer perf araçları. -## Test araçları Ve Komut dosyaları {#test-tools-and-scripts} +## Test araçları ve komut dosyaları {#test-tools-and-scripts} -Bazı programlar `tests` dizin testleri hazırlanmış değil, ancak test araçlarıdır. Örneğin, için `Lexer` bir araç var `src/Parsers/tests/lexer` bu sadece stdin’in tokenizasyonunu yapar ve renklendirilmiş sonucu stdout’a yazar. Bu tür araçları kod örnekleri olarak ve keşif ve manuel test için kullanabilirsiniz. +Bazı programlar `tests` dizin testleri hazırlanmış değil, ancak test araçlarıdır. Örneğin, için `Lexer` bir araç var `src/Parsers/tests/lexer` bu sadece stdin'in tokenizasyonunu yapar ve renklendirilmiş sonucu stdout'a yazar. Bu tür araçları kod örnekleri olarak ve keşif ve manuel test için kullanabilirsiniz. Ayrıca Çift Dosya yerleştirebilirsiniz `.sh` ve `.reference` aracı ile birlikte bazı önceden tanımlanmış giriş üzerinde çalıştırmak için-daha sonra komut sonucu karşılaştırılabilir `.reference` Dosya. Bu tür testler otomatik değildir. @@ -79,7 +79,7 @@ Clickhouse açık kaynaklı önce çekirdek testi ayrı ekip tarafından yazılm Yeni bir özellik geliştirdiğinizde, el ile de test etmek mantıklıdır. Bunu aşağıdaki adımlarla yapabilirsiniz: -ClickHouse Oluşturun. Terminalden Clickhouse’u çalıştırın: dizini değiştir `programs/clickhouse-server` ve ile çalıştırın `./clickhouse-server`. Bu yapılandırma kullanacak (`config.xml`, `users.xml` ve içindeki dosyalar `config.d` ve `users.d` dizinler) geçerli dizinden varsayılan olarak. ClickHouse sunucusuna bağlanmak için, çalıştırın `programs/clickhouse-client/clickhouse-client`. +ClickHouse Oluşturun. Terminalden Clickhouse'u çalıştırın: dizini değiştir `programs/clickhouse-server` ve ile çalıştırın `./clickhouse-server`. Bu yapılandırma kullanacak (`config.xml`, `users.xml` ve içindeki dosyalar `config.d` ve `users.d` dizinler) geçerli dizinden varsayılan olarak. ClickHouse sunucusuna bağlanmak için, çalıştırın `programs/clickhouse-client/clickhouse-client`. Tüm clickhouse araçlarının (sunucu, istemci, vb.) sadece tek bir ikili için symlinks olduğunu unutmayın `clickhouse`. Bu ikili bulabilirsiniz `programs/clickhouse`. Tüm araçlar olarak da çağrılabilir `clickhouse tool` yerine `clickhouse-tool`. @@ -118,7 +118,7 @@ Kararlı olarak yayınlamadan önce test ortamında dağıtın. Test ortamı, 1/ SELECT hostName() AS h, any(version()), any(uptime()), max(UTCEventTime), count() FROM remote('example01-01-{1..3}t', merge, hits) WHERE EventDate >= today() - 2 GROUP BY h ORDER BY h; ``` -Bazı durumlarda yandex’teki arkadaş ekiplerimizin test ortamına da dağıtım yapıyoruz: Pazar, Bulut, vb. Ayrıca geliştirme amacıyla kullanılan bazı donanım sunucularımız var. +Bazı durumlarda yandex'teki arkadaş ekiplerimizin test ortamına da dağıtım yapıyoruz: Pazar, Bulut, vb. Ayrıca geliştirme amacıyla kullanılan bazı donanım sunucularımız var. ## Yük Testi {#load-testing} @@ -132,7 +132,7 @@ Bir gün veya daha fazla sorgu günlüğü toplayın: $ clickhouse-client --query="SELECT DISTINCT query FROM system.query_log WHERE event_date = today() AND query LIKE '%ym:%' AND query NOT LIKE '%system.query_log%' AND type = 2 AND is_initial_query" > queries.tsv ``` -Bu şekilde karmaşık bir örnektir. `type = 2` başarıyla yürütülen sorguları süzer. `query LIKE '%ym:%'` yandex’ten ilgili sorguları seçmektir.Metrica. `is_initial_query` yalnızca istemci tarafından başlatılan sorguları seçmektir, Clickhouse’un kendisi tarafından değil (dağıtılmış sorgu işlemenin parçaları olarak). +Bu şekilde karmaşık bir örnektir. `type = 2` başarıyla yürütülen sorguları süzer. `query LIKE '%ym:%'` yandex'ten ilgili sorguları seçmektir.Metrica. `is_initial_query` yalnızca istemci tarafından başlatılan sorguları seçmektir, Clickhouse'un kendisi tarafından değil (dağıtılmış sorgu işlemenin parçaları olarak). `scp` bu test kümenize günlük ve aşağıdaki gibi çalıştırın: @@ -150,7 +150,7 @@ Kesin sorgu yürütme zamanlamaları kaydedilmez ve sorguların ve ortamın yük ## Yapı Testleri {#build-tests} -Yapı testleri, yapının çeşitli alternatif konfigürasyonlarda ve bazı yabancı sistemlerde bozulmadığını kontrol etmeyi sağlar. Testler bulunur `ci` dizin. Docker, Vagrant ve bazen de `qemu-user-static` Docker’ın içinde. Bu testler geliştirme aşamasındadır ve test çalıştırmaları otomatik değildir. +Yapı testleri, yapının çeşitli alternatif konfigürasyonlarda ve bazı yabancı sistemlerde bozulmadığını kontrol etmeyi sağlar. Testler bulunur `ci` dizin. Docker, Vagrant ve bazen de `qemu-user-static` Docker'ın içinde. Bu testler geliştirme aşamasındadır ve test çalıştırmaları otomatik değildir. Motivasyon: @@ -166,7 +166,7 @@ Normalde tüm testleri ClickHouse yapısının tek bir varyantında serbest bır Tüm yapı varyantlarında tüm testleri çalıştıramasak da, en azından çeşitli yapı varyantlarının bozulmadığını kontrol etmek istiyoruz. Bu amaçla yapı testlerini kullanıyoruz. -## Protokol uyumluluğu Testi {#testing-for-protocol-compatibility} +## Protokol uyumluluğu testi {#testing-for-protocol-compatibility} ClickHouse ağ protokolünü genişlettiğimizde, eski clickhouse istemcisinin yeni clickhouse sunucusu ile çalıştığını ve yeni clickhouse istemcisinin eski clickhouse sunucusu ile çalıştığını (sadece ilgili paketlerden ikili dosyaları çalıştırarak) manuel olarak test ediyoruz. @@ -176,7 +176,7 @@ Ana ClickHouse kodu (bu `dbms` dizin) ile inşa edilmiştir `-Wall -Wextra -Werr Clang daha yararlı uyarılar vardır-Sen ile onları arayabilirsiniz `-Weverything` ve varsayılan oluşturmak için bir şey seçin. -Üretim yapıları için gcc kullanılır (hala clang’dan biraz daha verimli kod üretir). Geliştirme için, clang genellikle kullanımı daha uygundur. Hata ayıklama modu ile kendi makinenizde inşa edebilirsiniz (dizüstü bilgisayarınızın pilinden tasarruf etmek için), ancak derleyicinin daha fazla uyarı üretebileceğini lütfen unutmayın `-O3` daha iyi kontrol akışı ve prosedürler arası analiz nedeniyle. Clang ile inşa ederken, `libc++` yerine kullanılır `libstdc++` ve hata ayıklama modu ile oluştururken, hata ayıklama sürümü `libc++` çalışma zamanında daha fazla hata yakalamak için izin verir kullanılır. +Üretim yapıları için gcc kullanılır (hala clang'dan biraz daha verimli kod üretir). Geliştirme için, clang genellikle kullanımı daha uygundur. Hata ayıklama modu ile kendi makinenizde inşa edebilirsiniz (dizüstü bilgisayarınızın pilinden tasarruf etmek için), ancak derleyicinin daha fazla uyarı üretebileceğini lütfen unutmayın `-O3` daha iyi kontrol akışı ve prosedürler arası analiz nedeniyle. Clang ile inşa ederken, `libc++` yerine kullanılır `libstdc++` ve hata ayıklama modu ile oluştururken, hata ayıklama sürümü `libc++` çalışma zamanında daha fazla hata yakalamak için izin verir kullanılır. ## Dezenfektanlar {#sanitizers} @@ -200,13 +200,23 @@ Hata ayıklama sürümü `jemalloc` hata ayıklama oluşturmak için kullanılı ## Fuzzing {#fuzzing} -Rastgele SQL sorguları oluşturmak ve sunucunun ölmediğini kontrol etmek için basit fuzz testi kullanıyoruz. Fuzz testi Adres dezenfektanı ile yapılır. İçinde bulabilirsiniz `00746_sql_fuzzy.pl`. Bu test sürekli olarak (gece ve daha uzun) çalıştırılmalıdır. +ClickHouse fuzzing hem kullanılarak uygulanmaktadır [libFuzzer](https://llvm.org/docs/LibFuzzer.html) ve rastgele SQL sorguları. +Tüm fuzz testleri sanitizers (Adres ve tanımsız) ile yapılmalıdır. -Aralık 2018 itibariyle, hala kütüphane kodunun izole fuzz testini kullanmıyoruz. +LibFuzzer kütüphane kodu izole fuzz testi için kullanılır. Fuzzers test kodunun bir parçası olarak uygulanır ve “\_fuzzer” adı postfixes. +Fuzzer örneği bulunabilir `src/Parsers/tests/lexer_fuzzer.cpp`. LibFuzzer özgü yapılandırmalar, sözlükler ve corpus saklanır `tests/fuzz`. +Kullanıcı girişini işleyen her işlevsellik için fuzz testleri yazmanızı öneririz. + +Fuzzers varsayılan olarak oluşturulmaz. Hem fuzzers inşa etmek `-DENABLE_FUZZING=1` ve `-DENABLE_TESTS=1` seçenekler ayarlanmalıdır. +Fuzzers oluştururken Jemalloc'u devre dışı bırakmanızı öneririz. ClickHouse fuzzing'i entegre etmek için kullanılan yapılandırma +Google OSS-Fuzz bulunabilir `docker/fuzz`. + +Ayrıca rastgele SQL sorguları oluşturmak ve sunucunun bunları çalıştırarak ölmediğini kontrol etmek için basit fuzz testi kullanıyoruz. +İçinde bulabilirsiniz `00746_sql_fuzzy.pl`. Bu test sürekli olarak (gece ve daha uzun) çalıştırılmalıdır. ## Güvenlik Denetimi {#security-audit} -Yandex Bulut departmanından insanlar, güvenlik açısından ClickHouse yeteneklerine bazı temel genel bakışlar yaparlar. +Yandex Güvenlik ekibinden insanlar güvenlik açısından ClickHouse yetenekleri bazı temel bakış yapmak. ## Statik Analizörler {#static-analyzers} @@ -230,9 +240,9 @@ Alternatif olarak deneyebilirsiniz `uncrustify` kodunuzu yeniden biçimlendirmek `CLion` kod stilimiz için ayarlanması gereken kendi kod biçimlendiricisine sahiptir. -## Metrica B2B Testleri {#metrica-b2b-tests} +## Metrica B2B testleri {#metrica-b2b-tests} -Her ClickHouse sürümü Yandex Metrica ve AppMetrica motorları ile test edilir. Clickhouse’un Test ve kararlı sürümleri Vm’lerde dağıtılır ve Giriş verilerinin sabit örneğini işleyen Metrica motorunun küçük bir kopyasıyla çalışır. Daha sonra Metrica motorunun iki örneğinin sonuçları birlikte karşılaştırılır. +Her ClickHouse sürümü Yandex Metrica ve AppMetrica motorları ile test edilir. Clickhouse'un Test ve kararlı sürümleri Vm'lerde dağıtılır ve Giriş verilerinin sabit örneğini işleyen Metrica motorunun küçük bir kopyasıyla çalışır. Daha sonra Metrica motorunun iki örneğinin sonuçları birlikte karşılaştırılır. Bu testler ayrı ekip tarafından otomatikleştirilir. Yüksek sayıda hareketli parça nedeniyle, testler çoğu zaman tamamen ilgisiz nedenlerle başarısız olur, bu da anlaşılması çok zordur. Büyük olasılıkla bu testlerin bizim için negatif değeri var. Bununla birlikte, bu testlerin yüzlerce kişiden yaklaşık bir veya iki kez yararlı olduğu kanıtlanmıştır. @@ -244,9 +254,9 @@ Temmuz 2018 itibariyle test kapsamını takip etmiyoruz. Yandex dahili CI ve iş otomasyon sistemi ile testler yapıyoruz “Sandbox”. -Yapı işleri ve testler, taahhüt bazında sanal alanda çalıştırılır. Ortaya çıkan paketler ve test sonuçları Github’da yayınlanır ve doğrudan bağlantılar tarafından indirilebilir. Eserler sonsuza dek saklanır. Eğer GitHub bir çekme isteği gönderdiğinizde, biz olarak etiketlemek “can be tested” ve bizim CI sistemi sizin için ClickHouse paketleri (yayın, hata ayıklama, Adres dezenfektanı ile, vb) inşa edecek. +Yapı işleri ve testler, taahhüt bazında sanal alanda çalıştırılır. Ortaya çıkan paketler ve test sonuçları Github'da yayınlanır ve doğrudan bağlantılar tarafından indirilebilir. Eserler sonsuza dek saklanır. Eğer GitHub bir çekme isteği gönderdiğinizde, biz olarak etiketlemek “can be tested” ve bizim CI sistemi sizin için ClickHouse paketleri (yayın, hata ayıklama, Adres dezenfektanı ile, vb) inşa edecek. Travis CI, zaman ve hesaplama gücü sınırı nedeniyle kullanmıyoruz. -Jenkins’i kullanmayız. Daha önce kullanıldı ve şimdi Jenkins kullanmadığımız için mutluyuz. +Jenkins'i kullanmayız. Daha önce kullanıldı ve şimdi Jenkins kullanmadığımız için mutluyuz. [Orijinal makale](https://clickhouse.tech/docs/en/development/tests/) diff --git a/docs/tr/engines/database-engines/index.md b/docs/tr/engines/database-engines/index.md index f3fcd3e8d10..e15ae0f3365 100644 --- a/docs/tr/engines/database-engines/index.md +++ b/docs/tr/engines/database-engines/index.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_folder_title: "Veritaban\u0131 Motorlar\u0131" toc_priority: 27 toc_title: "Giri\u015F" diff --git a/docs/tr/engines/database-engines/lazy.md b/docs/tr/engines/database-engines/lazy.md index 8b2a4f716bd..41226c320de 100644 --- a/docs/tr/engines/database-engines/lazy.md +++ b/docs/tr/engines/database-engines/lazy.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 31 toc_title: Tembel --- # Tembel {#lazy} -Tabloları yalnızca RAM’de tutar `expiration_time_in_seconds` son erişimden saniyeler sonra. Sadece \* Log tabloları ile kullanılabilir. +Tabloları yalnızca RAM'de tutar `expiration_time_in_seconds` son erişimden saniyeler sonra. Sadece \* Log tabloları ile kullanılabilir. Erişimler arasında uzun bir zaman aralığı olan birçok küçük \* günlük tablosunu saklamak için optimize edilmiştir. diff --git a/docs/tr/engines/database-engines/mysql.md b/docs/tr/engines/database-engines/mysql.md index f5e20528d67..2120add59aa 100644 --- a/docs/tr/engines/database-engines/mysql.md +++ b/docs/tr/engines/database-engines/mysql.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 30 toc_title: MySQL --- @@ -53,7 +53,7 @@ Diğer tüm MySQL veri türleri dönüştürülür [Dize](../../sql-reference/da [Nullable](../../sql-reference/data-types/nullable.md) desteklenir. -## Kullanım Örnekleri {#examples-of-use} +## Kullanım örnekleri {#examples-of-use} MySQL tablo: @@ -79,7 +79,7 @@ mysql> select * from mysql_table; 1 row in set (0,00 sec) ``` -Clickhouse’daki veritabanı, MySQL sunucusu ile veri alışverişi: +Clickhouse'daki veritabanı, MySQL sunucusu ile veri alışverişi: ``` sql CREATE DATABASE mysql_db ENGINE = MySQL('localhost:3306', 'test', 'my_user', 'user_password') diff --git a/docs/tr/engines/index.md b/docs/tr/engines/index.md index 48004afa1c8..221dc87bdfb 100644 --- a/docs/tr/engines/index.md +++ b/docs/tr/engines/index.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_folder_title: Motorlar toc_priority: 25 --- diff --git a/docs/tr/engines/table-engines/index.md b/docs/tr/engines/table-engines/index.md index 66623170123..01a447dbc6c 100644 --- a/docs/tr/engines/table-engines/index.md +++ b/docs/tr/engines/table-engines/index.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_folder_title: "Masa Motorlar\u0131" toc_priority: 26 toc_title: "Giri\u015F" @@ -21,7 +21,7 @@ Tablo motoru (tablo türü) belirler: ### MergeTree {#mergetree} -Yüksek yük görevleri için en evrensel ve fonksiyonel masa motorları. Bu motorlar tarafından paylaşılan özellik, sonraki arka plan veri işleme ile hızlı veri ekleme ’ dir. `MergeTree` aile motorları destek veri çoğaltma (ile [Çoğaltıyordu\*](mergetree-family/replication.md#replication) sürümleri), bölümleme ve diğer özellikler diğer motorlarda desteklenmez. +Yüksek yük görevleri için en evrensel ve fonksiyonel masa motorları. Bu motorlar tarafından paylaşılan özellik, sonraki arka plan veri işleme ile hızlı veri ekleme ' dir. `MergeTree` aile motorları destek veri çoğaltma (ile [Çoğaltıyordu\*](mergetree-family/replication.md#table_engines-replication) sürümleri), bölümleme ve diğer özellikler diğer motorlarda desteklenmez. Ailede motorlar: @@ -62,14 +62,14 @@ Ailede motorlar: - [Dağılı](special/distributed.md#distributed) - [MaterializedView](special/materializedview.md#materializedview) - [Sözlük](special/dictionary.md#dictionary) -- [Birleştirmek](special/merge.md#merge +- \[Mer \]ge\] (spec /ial / mer \#ge. md\#mer \#ge - [Dosya](special/file.md#file) - [Boş](special/null.md#null) - [Koymak](special/set.md#set) - [Katmak](special/join.md#join) - [URL](special/url.md#table_engines-url) - [Görünüm](special/view.md#table_engines-view) -- [Bellek](special/memory.md#memory) +- [Hafıza](special/memory.md#memory) - [Arabellek](special/buffer.md#buffer) ## Sanal Sütunlar {#table_engines-virtual_columns} diff --git a/docs/tr/engines/table-engines/integrations/hdfs.md b/docs/tr/engines/table-engines/integrations/hdfs.md index af538a9bc90..a69d717f0e4 100644 --- a/docs/tr/engines/table-engines/integrations/hdfs.md +++ b/docs/tr/engines/table-engines/integrations/hdfs.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 36 toc_title: HDFS --- @@ -16,7 +16,7 @@ to the [Dosya](../special/file.md#table_engines-file) ve [URL](../special/url.md ENGINE = HDFS(URI, format) ``` -Bu `URI` parametre, HDFS’DEKİ tüm dosya URI’SIDIR. +Bu `URI` parametre, HDFS'DEKİ tüm dosya URI'SIDIR. Bu `format` parametre kullanılabilir dosya biçimlerinden birini belirtir. Gerçekleştirmek `SELECT` sorgular, biçim giriş için desteklenmeli ve gerçekleştirmek için `INSERT` queries – for output. The available formats are listed in the @@ -71,7 +71,7 @@ Birden çok yol bileşenleri globs olabilir. İşlenmek için dosya var olmalı **Örnek** -1. HDFS’DE aşağıdaki Urı’lerle TSV formatında birkaç dosyamız olduğunu varsayalım: +1. HDFS'DE aşağıdaki Urı'lerle TSV formatında birkaç dosyamız olduğunu varsayalım: - ‘hdfs://hdfs1:9000/some\_dir/some\_file\_1’ - ‘hdfs://hdfs1:9000/some\_dir/some\_file\_2’ diff --git a/docs/tr/engines/table-engines/integrations/index.md b/docs/tr/engines/table-engines/integrations/index.md index 608fc900e62..937532c247d 100644 --- a/docs/tr/engines/table-engines/integrations/index.md +++ b/docs/tr/engines/table-engines/integrations/index.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_folder_title: Entegrasyonlar toc_priority: 30 --- diff --git a/docs/tr/engines/table-engines/integrations/jdbc.md b/docs/tr/engines/table-engines/integrations/jdbc.md index acc7660e24f..76deb938047 100644 --- a/docs/tr/engines/table-engines/integrations/jdbc.md +++ b/docs/tr/engines/table-engines/integrations/jdbc.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 34 toc_title: JDBC --- @@ -59,7 +59,7 @@ mysql> select * from test; 1 row in set (0,00 sec) ``` -ClickHouse Server’da bir tablo oluşturma ve ondan veri seçme: +ClickHouse Server'da bir tablo oluşturma ve ondan veri seçme: ``` sql CREATE TABLE jdbc_table diff --git a/docs/tr/engines/table-engines/integrations/kafka.md b/docs/tr/engines/table-engines/integrations/kafka.md index 7845c1609b0..d74711b21e4 100644 --- a/docs/tr/engines/table-engines/integrations/kafka.md +++ b/docs/tr/engines/table-engines/integrations/kafka.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 32 toc_title: Kafka --- @@ -32,22 +32,26 @@ SETTINGS [kafka_row_delimiter = 'delimiter_symbol',] [kafka_schema = '',] [kafka_num_consumers = N,] - [kafka_skip_broken_messages = N] + [kafka_max_block_size = 0,] + [kafka_skip_broken_messages = N,] + [kafka_commit_every_batch = 0] ``` Gerekli parametreler: - `kafka_broker_list` – A comma-separated list of brokers (for example, `localhost:9092`). - `kafka_topic_list` – A list of Kafka topics. -- `kafka_group_name` – A group of Kafka consumers. Reading margins are tracked for each group separately. If you don’t want messages to be duplicated in the cluster, use the same group name everywhere. +- `kafka_group_name` – A group of Kafka consumers. Reading margins are tracked for each group separately. If you don't want messages to be duplicated in the cluster, use the same group name everywhere. - `kafka_format` – Message format. Uses the same notation as the SQL `FORMAT` fonksiyon gibi `JSONEachRow`. Daha fazla bilgi için, bkz: [Biçimliler](../../../interfaces/formats.md) bölme. İsteğe bağlı parametreler: - `kafka_row_delimiter` – Delimiter character, which ends the message. -- `kafka_schema` – Parameter that must be used if the format requires a schema definition. For example, [Cap’n Proto](https://capnproto.org/) şema dosyasının yolunu ve kök adını gerektirir `schema.capnp:Message` nesne. +- `kafka_schema` – Parameter that must be used if the format requires a schema definition. For example, [Cap'n Proto](https://capnproto.org/) şema dosyasının yolunu ve kök adını gerektirir `schema.capnp:Message` nesne. - `kafka_num_consumers` – The number of consumers per table. Default: `1`. Bir tüketicinin verimi yetersizse daha fazla tüketici belirtin. Bölüm başına yalnızca bir tüketici atanabileceğinden, toplam tüketici sayısı konudaki bölüm sayısını geçmemelidir. +- `kafka_max_block_size` - Anket için maksimum toplu iş boyutu (mesajlarda) (varsayılan: `max_block_size`). - `kafka_skip_broken_messages` – Kafka message parser tolerance to schema-incompatible messages per block. Default: `0`. Eğer `kafka_skip_broken_messages = N` sonra motor atlar *N* Ayrıştırılamayan Kafka iletileri (bir ileti bir veri satırına eşittir). +- `kafka_commit_every_batch` - Bütün bir blok yazdıktan sonra tek bir taahhüt yerine tüketilen ve işlenen her toplu işlemi gerçekleştirin (varsayılan: `0`). Örnekler: @@ -105,7 +109,7 @@ Gruplar esnek ve kümede senkronize edilir. Örneğin, bir kümede 10 konu ve bi 2. İstenen yapıya sahip bir tablo oluşturun. 3. Verileri motordan dönüştüren ve daha önce oluşturulmuş bir tabloya koyan materyalleştirilmiş bir görünüm oluşturun. -Ne zaman `MATERIALIZED VIEW` motora katılır, arka planda veri toplamaya başlar. Bu, kafka’dan sürekli olarak mesaj almanızı ve bunları kullanarak gerekli biçime dönüştürmenizi sağlar `SELECT`. +Ne zaman `MATERIALIZED VIEW` motora katılır, arka planda veri toplamaya başlar. Bu, kafka'dan sürekli olarak mesaj almanızı ve bunları kullanarak gerekli biçime dönüştürmenizi sağlar `SELECT`. Bir kafka tablosu istediğiniz kadar materialized görüşe sahip olabilir, kafka tablosundan doğrudan veri okumazlar, ancak yeni kayıtlar (bloklar halinde) alırlar, bu şekilde farklı ayrıntı seviyesine sahip birkaç tabloya yazabilirsiniz (gruplama-toplama ve olmadan). Örnek: diff --git a/docs/tr/engines/table-engines/integrations/mysql.md b/docs/tr/engines/table-engines/integrations/mysql.md index 27b6ee1029f..6f490a26b48 100644 --- a/docs/tr/engines/table-engines/integrations/mysql.md +++ b/docs/tr/engines/table-engines/integrations/mysql.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 33 toc_title: MySQL --- @@ -49,7 +49,7 @@ Tablo yapısı orijinal MySQL tablo yapısından farklı olabilir: Basit `WHERE` gibi maddeler `=, !=, >, >=, <, <=` MySQL sunucusunda yürütülür. -Geri kalan şartlar ve `LIMIT` örnekleme kısıtlaması, yalnızca MySQL sorgusu bittikten sonra Clickhouse’da yürütülür. +Geri kalan şartlar ve `LIMIT` örnekleme kısıtlaması, yalnızca MySQL sorgusu bittikten sonra Clickhouse'da yürütülür. ## Kullanım Örneği {#usage-example} @@ -76,7 +76,7 @@ mysql> select * from test; 1 row in set (0,00 sec) ``` -Clickhouse’daki tablo, yukarıda oluşturulan MySQL tablosundan veri alma: +Clickhouse'daki tablo, yukarıda oluşturulan MySQL tablosundan veri alma: ``` sql CREATE TABLE mysql_table diff --git a/docs/tr/engines/table-engines/integrations/odbc.md b/docs/tr/engines/table-engines/integrations/odbc.md index 5b56b90a1e4..1c3208f0e5d 100644 --- a/docs/tr/engines/table-engines/integrations/odbc.md +++ b/docs/tr/engines/table-engines/integrations/odbc.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 35 toc_title: ODBC --- @@ -103,7 +103,7 @@ mysql> select * from test; 1 row in set (0,00 sec) ``` -Clickhouse’daki tablo, MySQL tablosundan veri alma: +Clickhouse'daki tablo, MySQL tablosundan veri alma: ``` sql CREATE TABLE odbc_t diff --git a/docs/tr/engines/table-engines/log-family/index.md b/docs/tr/engines/table-engines/log-family/index.md index 062087a5874..39bc7c21f51 100644 --- a/docs/tr/engines/table-engines/log-family/index.md +++ b/docs/tr/engines/table-engines/log-family/index.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_folder_title: "G\xFCnl\xFCk Aile" toc_priority: 29 --- diff --git a/docs/tr/engines/table-engines/log-family/log-family.md b/docs/tr/engines/table-engines/log-family/log-family.md index 664a7e4ab22..074f9363e9f 100644 --- a/docs/tr/engines/table-engines/log-family/log-family.md +++ b/docs/tr/engines/table-engines/log-family/log-family.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 31 toc_title: "Giri\u015F" --- diff --git a/docs/tr/engines/table-engines/log-family/log.md b/docs/tr/engines/table-engines/log-family/log.md index 6c23df468d6..cdebf76515e 100644 --- a/docs/tr/engines/table-engines/log-family/log.md +++ b/docs/tr/engines/table-engines/log-family/log.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 33 toc_title: "G\xFCnl\xFCk" --- # Günlük {#log} -Motor günlük motorları ailesine aittir. Günlük motorlarının ortak özelliklerini ve farklılıklarını görün [Log Engine Ailesi](log-family.md) makale. +Motor günlük motorları ailesine aittir. Günlük motorlarının ortak özelliklerini ve farklılıklarını görün [Log Engine Ailesi](log-family.md) Makale. Log differsar differsit fromma [TinyLog](tinylog.md) bu küçük bir dosyada “marks” sütun dosyaları ile bulunur. Bu işaretler her veri bloğuna yazılır ve belirtilen satır sayısını atlamak için dosyayı okumaya nereden başlayacağınızı gösteren uzaklıklar içerir. Bu, tablo verilerini birden çok iş parçacığında okumayı mümkün kılar. Eşzamanlı veri erişimi için, okuma işlemleri aynı anda gerçekleştirilebilirken, yazma işlemleri okur ve birbirlerini engeller. diff --git a/docs/tr/engines/table-engines/log-family/stripelog.md b/docs/tr/engines/table-engines/log-family/stripelog.md index 518c057e0b2..038c6c28569 100644 --- a/docs/tr/engines/table-engines/log-family/stripelog.md +++ b/docs/tr/engines/table-engines/log-family/stripelog.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 32 toc_title: StripeLog --- # Stripelog {#stripelog} -Bu motor günlük motor ailesine aittir. Günlük motorlarının ortak özelliklerini ve farklılıklarını görün [Log Engine Ailesi](log-family.md) makale. +Bu motor günlük motor ailesine aittir. Günlük motorlarının ortak özelliklerini ve farklılıklarını görün [Log Engine Ailesi](log-family.md) Makale. Az miktarda veri içeren (1 milyondan az satır) birçok tablo yazmanız gerektiğinde, bu altyapıyı senaryolarda kullanın. @@ -24,7 +24,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] Ayrıntılı açıklamasına bakın [CREATE TABLE](../../../sql-reference/statements/create.md#create-table-query) sorgu. -## Veri Yazma {#table_engines-stripelog-writing-the-data} +## Veri yazma {#table_engines-stripelog-writing-the-data} Bu `StripeLog` motor tüm sütunları tek bir dosyada saklar. Her biri için `INSERT` sorgu, ClickHouse veri bloğunu bir tablo dosyasının sonuna ekler, sütunları tek tek yazar. @@ -35,11 +35,11 @@ Her tablo için ClickHouse dosyaları yazar: Bu `StripeLog` motor desteklemiyor `ALTER UPDATE` ve `ALTER DELETE` harekat. -## Verileri Okuma {#table_engines-stripelog-reading-the-data} +## Verileri okuma {#table_engines-stripelog-reading-the-data} -İşaretli dosya, Clickhouse’un verilerin okunmasını paralelleştirmesine izin verir. Bu demektir `SELECT` sorgu satırları öngörülemeyen bir sırayla döndürür. Kullan… `ORDER BY` satırları sıralamak için yan tümce. +İşaretli dosya, Clickhouse'un verilerin okunmasını paralelleştirmesine izin verir. Bu demektir `SELECT` sorgu satırları öngörülemeyen bir sırayla döndürür. Kullan... `ORDER BY` satırları sıralamak için yan tümce. -## Kullanım Örneği {#table_engines-stripelog-example-of-use} +## Kullanım örneği {#table_engines-stripelog-example-of-use} Tablo oluşturma: diff --git a/docs/tr/engines/table-engines/log-family/tinylog.md b/docs/tr/engines/table-engines/log-family/tinylog.md index ea6032abbac..6975e6e127c 100644 --- a/docs/tr/engines/table-engines/log-family/tinylog.md +++ b/docs/tr/engines/table-engines/log-family/tinylog.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 34 toc_title: TinyLog --- diff --git a/docs/tr/engines/table-engines/mergetree-family/aggregatingmergetree.md b/docs/tr/engines/table-engines/mergetree-family/aggregatingmergetree.md index f824a7226fe..f2e94f7f707 100644 --- a/docs/tr/engines/table-engines/mergetree-family/aggregatingmergetree.md +++ b/docs/tr/engines/table-engines/mergetree-family/aggregatingmergetree.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 35 toc_title: AggregatingMergeTree --- @@ -11,7 +11,10 @@ Motor devralır [MergeTree](mergetree.md#table_engines-mergetree), veri parçala Kullanabilirsiniz `AggregatingMergeTree` artımlı veri toplama, toplanan materialized görünümleri de dahil olmak üzere tablolar. -Motor tüm sütunları ile işler [AggregateFunction](../../../sql-reference/data-types/aggregatefunction.md) tür. +Motor, tüm sütunları aşağıdaki türlerle işler: + +- [AggregateFunction](../../../sql-reference/data-types/aggregatefunction.md) +- [SimpleAggregateFunction](../../../sql-reference/data-types/simpleaggregatefunction.md) Kullanmak uygundur `AggregatingMergeTree` siparişlere göre satır sayısını azaltırsa. @@ -56,14 +59,14 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] Tüm parametreler, aşağıdaki gibi aynı anlama sahiptir `MergeTree`.
-## Seç Ve Ekle {#select-and-insert} +## Seç ve Ekle {#select-and-insert} Veri eklemek için şunları kullanın [INSERT SELECT](../../../sql-reference/statements/insert-into.md) agrega-Devlet-fonksiyonları ile sorgu. Veri seçerken `AggregatingMergeTree` tablo kullanın `GROUP BY` yan tümce ve veri eklerken aynı toplama işlevleri, ancak kullanarak `-Merge` sonek. Sonuç inlarında `SELECT` sorgu, değerleri `AggregateFunction` türü, Tüm ClickHouse çıktı biçimleri için uygulamaya özgü ikili gösterime sahiptir. Örneğin, veri dökümü, `TabSeparated` ile format `SELECT` sorgu daha sonra bu dökümü kullanarak geri yüklenebilir `INSERT` sorgu. -## Toplu Bir Somutlaştırılmış Görünüm örneği {#example-of-an-aggregated-materialized-view} +## Toplu bir Somutlaştırılmış Görünüm örneği {#example-of-an-aggregated-materialized-view} `AggregatingMergeTree` saatler hayata görünüm `test.visits` Tablo: diff --git a/docs/tr/engines/table-engines/mergetree-family/collapsingmergetree.md b/docs/tr/engines/table-engines/mergetree-family/collapsingmergetree.md index 75e643d1f89..a1d5d6738ef 100644 --- a/docs/tr/engines/table-engines/mergetree-family/collapsingmergetree.md +++ b/docs/tr/engines/table-engines/mergetree-family/collapsingmergetree.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 36 toc_title: CollapsingMergeTree --- @@ -119,11 +119,8 @@ ClickHouse veri parçalarını birleştirdiğinde, her ardışık satır grubu a Elde edilen her veri parçası için ClickHouse kaydeder: 1. Birincilik “cancel” ve son “state” satır sayısı ise “state” ve “cancel” satırlar eşleşir ve son satır bir “state” satır. - 2. Son “state” satır, daha varsa “state” satırlar daha “cancel” satırlar. - 3. Birincilik “cancel” satır, daha varsa “cancel” satırlar daha “state” satırlar. - 4. Diğer tüm durumlarda satırların hiçbiri. Ayrıca en az 2 tane daha olduğunda “state” satırlar daha “cancel” satırlar veya en az 2 tane daha “cancel” r rowsows th thenen “state” satırlar, birleştirme devam eder, ancak ClickHouse bu durumu mantıksal bir hata olarak değerlendirir ve sunucu günlüğüne kaydeder. Aynı veriler birden çok kez eklendiğinde, bu hata oluşabilir. @@ -139,7 +136,7 @@ Toplanan `count`, `sum` ve `avg` bu şekilde hesaplanmış olabilir. Toplanan `u Toplama olmadan veri ayıklamanız gerekiyorsa (örneğin, en yeni değerleri belirli koşullarla eşleşen satırların mevcut olup olmadığını kontrol etmek için) `FINAL` değiştirici için `FROM` yan. Bu yaklaşım önemli ölçüde daha az etkilidir. -## Kullanım Örneği {#example-of-use} +## Kullanım örneği {#example-of-use} Örnek veriler: @@ -229,7 +226,7 @@ SELECT * FROM UAct FINAL Verileri seçmenin bu yolu çok verimsizdir. Büyük masalar için kullanmayın. -## Başka Bir Yaklaşım Örneği {#example-of-another-approach} +## Başka bir yaklaşım örneği {#example-of-another-approach} Örnek veriler: diff --git a/docs/tr/engines/table-engines/mergetree-family/custom-partitioning-key.md b/docs/tr/engines/table-engines/mergetree-family/custom-partitioning-key.md index 89437b5dc8d..9dba2ac627e 100644 --- a/docs/tr/engines/table-engines/mergetree-family/custom-partitioning-key.md +++ b/docs/tr/engines/table-engines/mergetree-family/custom-partitioning-key.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 32 toc_title: "\xD6zel B\xF6l\xFCmleme Anahtar\u0131" --- @@ -40,7 +40,7 @@ Bir tabloya yeni veri eklerken, bu veriler birincil anahtara göre sıralanmış !!! info "Bilgin" Birleştirme yalnızca bölümleme ifadesi için aynı değere sahip veri parçaları için çalışır. Bu demektir **aşırı granüler bölümler yapmamalısınız** (yaklaşık binden fazla bölüm). Aksi takdirde, `SELECT` sorgu, dosya sistemindeki ve açık dosya tanımlayıcılarındaki makul olmayan sayıda dosya nedeniyle yetersiz performans gösterir. -Kullan… [sistem.parçalar](../../../operations/system-tables.md#system_tables-parts) tablo tablo parçaları ve bölümleri görüntülemek için. Örneğin, bir var varsayalım `visits` aya göre bölümleme ile tablo. Hadi gerçekleştirelim `SELECT` sorgu için `system.parts` Tablo: +Kullan... [sistem.parçalar](../../../operations/system-tables.md#system_tables-parts) tablo tablo parçaları ve bölümleri görüntülemek için. Örneğin, bir var varsayalım `visits` aya göre bölümleme ile tablo. Hadi gerçekleştirelim `SELECT` sorgu için `system.parts` Tablo: ``` sql SELECT diff --git a/docs/tr/engines/table-engines/mergetree-family/graphitemergetree.md b/docs/tr/engines/table-engines/mergetree-family/graphitemergetree.md index dde6efac225..3fa7ed529af 100644 --- a/docs/tr/engines/table-engines/mergetree-family/graphitemergetree.md +++ b/docs/tr/engines/table-engines/mergetree-family/graphitemergetree.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 38 toc_title: "Graph\u0131temergetree" --- # Graphıtemergetree {#graphitemergetree} -Bu motor inceltme ve toplama/ortalama (toplaması) için tasarlanmıştır) [Grafit](http://graphite.readthedocs.io/en/latest/index.html) veriler. Clickhouse’u Grafit için bir veri deposu olarak kullanmak isteyen geliştiriciler için yararlı olabilir. +Bu motor inceltme ve toplama/ortalama (toplaması) için tasarlanmıştır) [Grafit](http://graphite.readthedocs.io/en/latest/index.html) veriler. Clickhouse'u Grafit için bir veri deposu olarak kullanmak isteyen geliştiriciler için yararlı olabilir. Toplamaya ihtiyacınız yoksa Grafit verilerini depolamak için herhangi bir ClickHouse tablo motorunu kullanabilirsiniz, ancak bir toplamaya ihtiyacınız varsa `GraphiteMergeTree`. Motor, depolama hacmini azaltır ve grafitten gelen sorguların verimliliğini arttırır. diff --git a/docs/tr/engines/table-engines/mergetree-family/index.md b/docs/tr/engines/table-engines/mergetree-family/index.md index e722564f4dd..912eecc10f7 100644 --- a/docs/tr/engines/table-engines/mergetree-family/index.md +++ b/docs/tr/engines/table-engines/mergetree-family/index.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_folder_title: MergeTree Ailesi toc_priority: 28 --- diff --git a/docs/tr/engines/table-engines/mergetree-family/mergetree.md b/docs/tr/engines/table-engines/mergetree-family/mergetree.md index 6aca2720f74..4065b5fdcbd 100644 --- a/docs/tr/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/tr/engines/table-engines/mergetree-family/mergetree.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 30 toc_title: MergeTree --- @@ -68,7 +68,7 @@ Parametrelerin açıklaması için bkz. [Sorgu açıklaması oluştur](../../../ Sütun veya keyfi ifadeler bir tuple. Örnek: `ORDER BY (CounterID, EventDate)`. -- `PRIMARY KEY` — The primary key if it [sıralama anahtarından farklıdır](mergetree.md). +- `PRIMARY KEY` — The primary key if it [sıralama anahtarından farklıdır](#choosing-a-primary-key-that-differs-from-the-sorting-key). Varsayılan olarak, birincil anahtar sıralama anahtarıyla aynıdır (bu anahtar tarafından belirtilir). `ORDER BY` yan). Bu nedenle çoğu durumda ayrı bir belirtmek gereksizdir `PRIMARY KEY` yan. @@ -94,7 +94,7 @@ Parametrelerin açıklaması için bkz. [Sorgu açıklaması oluştur](../../../ - `min_merge_bytes_to_use_direct_io` — The minimum data volume for merge operation that is required for using direct I/O access to the storage disk. When merging data parts, ClickHouse calculates the total storage volume of all the data to be merged. If the volume exceeds `min_merge_bytes_to_use_direct_io` bayt, ClickHouse okur ve doğrudan I/O arabirimi kullanarak depolama diskine veri yazar (`O_DIRECT` seçenek). Eğer `min_merge_bytes_to_use_direct_io = 0`, sonra doğrudan g / Ç devre dışı bırakılır. Varsayılan değer: `10 * 1024 * 1024 * 1024` baytlar. - `merge_with_ttl_timeout` — Minimum delay in seconds before repeating a merge with TTL. Default value: 86400 (1 day). - - `write_final_mark` — Enables or disables writing the final index mark at the end of data part (after the last byte). Default value: 1. Don’t turn it off. + - `write_final_mark` — Enables or disables writing the final index mark at the end of data part (after the last byte). Default value: 1. Don't turn it off. - `merge_max_block_size` — Maximum number of rows in block for merge operations. Default value: 8192. - `storage_policy` — Storage policy. See [Veri depolama için birden fazla blok cihazı kullanma](#table_engine-mergetree-multiple-volumes). @@ -106,7 +106,7 @@ ENGINE MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDa Örnekte, aylara göre bölümleme ayarladık. -Biz de kullanıcı kimliği ile karma olarak örnekleme için bir ifade ayarlayın. Bu, her biri için tablodaki verileri pseudorandomize etmenizi sağlar `CounterID` ve `EventDate`. Tanım yoularsanız bir [SAMPLE](../../../sql-reference/statements/select.md#select-sample-clause) yan tümcesi verileri seçerken, ClickHouse kullanıcıların bir alt kümesi için eşit pseudorandom veri örneği döndürür. +Biz de kullanıcı kimliği ile karma olarak örnekleme için bir ifade ayarlayın. Bu, her biri için tablodaki verileri pseudorandomize etmenizi sağlar `CounterID` ve `EventDate`. Tanım yoularsanız bir [SAMPLE](../../../sql-reference/statements/select/sample.md#select-sample-clause) yan tümcesi verileri seçerken, ClickHouse kullanıcıların bir alt kümesi için eşit pseudorandom veri örneği döndürür. Bu `index_granularity` 8192 varsayılan değer olduğundan ayarı atlanabilir. @@ -150,11 +150,11 @@ Veri bir tabloya eklendiğinde, ayrı veri parçaları oluşturulur ve bunların Farklı bölümlere ait veriler farklı parçalara ayrılır. Arka planda, ClickHouse daha verimli depolama için veri parçalarını birleştirir. Farklı bölümlere ait parçalar birleştirilmez. Birleştirme mekanizması, aynı birincil anahtara sahip tüm satırların aynı veri bölümünde olacağını garanti etmez. -Her veri parçası mantıksal olarak granüllere ayrılmıştır. Bir granül, Clickhouse’un veri seçerken okuduğu en küçük bölünmez veri kümesidir. ClickHouse satırları veya değerleri bölmez, bu nedenle her granül her zaman bir tamsayı satır içerir. Bir granülün ilk satırı, satır için birincil anahtarın değeri ile işaretlenir. Her veri bölümü için ClickHouse işaretleri depolayan bir dizin dosyası oluşturur. Her sütun için, birincil anahtarda olsun ya da olmasın, ClickHouse aynı işaretleri de saklar. Bu işaretler, verileri doğrudan sütun dosyalarında bulmanızı sağlar. +Her veri parçası mantıksal olarak granüllere ayrılmıştır. Bir granül, Clickhouse'un veri seçerken okuduğu en küçük bölünmez veri kümesidir. ClickHouse satırları veya değerleri bölmez, bu nedenle her granül her zaman bir tamsayı satır içerir. Bir granülün ilk satırı, satır için birincil anahtarın değeri ile işaretlenir. Her veri bölümü için ClickHouse işaretleri depolayan bir dizin dosyası oluşturur. Her sütun için, birincil anahtarda olsun ya da olmasın, ClickHouse aynı işaretleri de saklar. Bu işaretler, verileri doğrudan sütun dosyalarında bulmanızı sağlar. Granül boyutu ile sınırlıdır `index_granularity` ve `index_granularity_bytes` tablo motorunun ayarları. Bir granüldeki satır sayısı `[1, index_granularity]` Aralık, satırların boyutuna bağlı olarak. Bir granülün boyutu aşabilir `index_granularity_bytes` tek bir satırın boyutu ayarın değerinden büyükse. Bu durumda, granülün boyutu satırın boyutuna eşittir. -## Sorgularda Birincil Anahtarlar Ve Dizinler {#primary-keys-and-indexes-in-queries} +## Sorgularda birincil anahtarlar ve dizinler {#primary-keys-and-indexes-in-queries} Tak thee the `(CounterID, Date)` örnek olarak birincil anahtar. Bu durumda, sıralama ve dizin aşağıdaki gibi gösterilebilir: @@ -175,11 +175,11 @@ Yukarıdaki örnekler, her zaman bir dizin tam taramadan daha etkili olduğunu g Seyrek bir dizin, ekstra verilerin okunmasına izin verir. Birincil anahtarın tek bir aralığını okurken, `index_granularity * 2` her veri bloğundaki ekstra satırlar okunabilir. -Seyrek dizinler, çok sayıda tablo satırı ile çalışmanıza izin verir, çünkü çoğu durumda, bu tür dizinler bilgisayarın RAM’İNE sığar. +Seyrek dizinler, çok sayıda tablo satırı ile çalışmanıza izin verir, çünkü çoğu durumda, bu tür dizinler bilgisayarın RAM'İNE sığar. ClickHouse benzersiz bir birincil anahtar gerektirmez. Aynı birincil anahtar ile birden çok satır ekleyebilirsiniz. -### Birincil Anahtar seçme {#selecting-the-primary-key} +### Birincil anahtar seçme {#selecting-the-primary-key} Birincil anahtardaki sütun sayısı açıkça sınırlı değildir. Veri yapısına bağlı olarak, birincil anahtara daha fazla veya daha az sütun ekleyebilirsiniz. Bu Mayıs: @@ -200,18 +200,18 @@ Birincil anahtardaki sütun sayısı açıkça sınırlı değildir. Veri yapıs Uzun bir birincil anahtar, ekleme performansını ve bellek tüketimini olumsuz yönde etkiler, ancak birincil anahtardaki ek sütunlar, ClickHouse performansını etkilemez `SELECT` sorgular. -### Sıralama anahtarından farklı Bir Birincil Anahtar seçme {#choosing-a-primary-key-that-differs-from-the-sorting-key} +### Sıralama anahtarından farklı bir birincil anahtar seçme {#choosing-a-primary-key-that-differs-from-the-sorting-key} -Sıralama anahtarından (veri bölümlerindeki satırları sıralamak için bir ifade) farklı bir birincil anahtar (her işaret için dizin dosyasında yazılan değerlere sahip bir ifade) belirtmek mümkündür. Bu durumda, birincil anahtar ifadesi tuple, sıralama anahtarı ifadesi tuple’ın bir öneki olmalıdır. +Sıralama anahtarından (veri bölümlerindeki satırları sıralamak için bir ifade) farklı bir birincil anahtar (her işaret için dizin dosyasında yazılan değerlere sahip bir ifade) belirtmek mümkündür. Bu durumda, birincil anahtar ifadesi tuple, sıralama anahtarı ifadesi tuple'ın bir öneki olmalıdır. Bu özellik kullanırken yararlıdır [SummingMergeTree](summingmergetree.md) ve -[AggregatingMergeTree](aggregatingmergetree.md) masa motorları. Bu motorları kullanırken yaygın bir durumda, tablonun iki tür sütunu vardır: *boyutlular* ve *ölçümler*. Tipik sorgular, rasgele ölçü sütunlarının değerlerini toplar `GROUP BY` ve boyutlara göre filtreleme. Çünkü SummingMergeTree ve AggregatingMergeTree sıralama anahtarının aynı değere sahip satırları toplamak, tüm boyutları eklemek doğaldır. Sonuç olarak, anahtar ifadesi uzun bir sütun listesinden oluşur ve bu liste yeni eklenen boyutlarla sık sık güncelleştirilmelidir. +[AggregatingMergeTree](aggregatingmergetree.md) masa motorları. Bu motorları kullanırken yaygın bir durumda, tablonun iki tür sütunu vardır: *boyutlar* ve *ölçümler*. Tipik sorgular, rasgele ölçü sütunlarının değerlerini toplar `GROUP BY` ve boyutlara göre filtreleme. Çünkü SummingMergeTree ve AggregatingMergeTree sıralama anahtarının aynı değere sahip satırları toplamak, tüm boyutları eklemek doğaldır. Sonuç olarak, anahtar ifadesi uzun bir sütun listesinden oluşur ve bu liste yeni eklenen boyutlarla sık sık güncelleştirilmelidir. Bu durumda, birincil anahtarda verimli Aralık taramaları sağlayacak ve kalan boyut sütunlarını sıralama anahtarı kümesine ekleyecek yalnızca birkaç sütun bırakmak mantıklıdır. [ALTER](../../../sql-reference/statements/alter.md) yeni bir sütun aynı anda tabloya ve sıralama anahtarı eklendiğinde, varolan veri parçaları değiştirilmesi gerekmez, çünkü sıralama anahtarının hafif bir işlemdir. Eski sıralama anahtarı yeni sıralama anahtarının bir öneki olduğundan ve yeni eklenen sütunda veri olmadığından, veriler tablo değişikliği anında hem eski hem de yeni sıralama anahtarlarına göre sıralanır. -### Sorgularda Dizin Ve bölümlerin kullanımı {#use-of-indexes-and-partitions-in-queries} +### Sorgularda dizin ve bölümlerin kullanımı {#use-of-indexes-and-partitions-in-queries} İçin `SELECT` sorgular, ClickHouse bir dizin kullanılabilir olup olmadığını analiz eder. Eğer bir dizin kullanılabilir `WHERE/PREWHERE` yan tümce, bir eşitlik veya eşitsizlik karşılaştırma işlemini temsil eden bir ifadeye (bağlantı öğelerinden biri olarak veya tamamen) sahiptir veya varsa `IN` veya `LIKE` sütun veya birincil anahtar veya bölümleme anahtar veya bu sütunların belirli kısmen tekrarlayan işlevleri veya bu ifadelerin mantıksal ilişkileri olan ifadeler üzerinde sabit bir önek ile. @@ -239,11 +239,11 @@ Aşağıdaki örnekte, dizin kullanılamaz. SELECT count() FROM table WHERE CounterID = 34 OR URL LIKE '%upyachka%' ``` -Clickhouse’un bir sorgu çalıştırırken dizini kullanıp kullanamayacağını kontrol etmek için ayarları kullanın [force\_index\_by\_date](../../../operations/settings/settings.md#settings-force_index_by_date) ve [force\_primary\_key](../../../operations/settings/settings.md). +Clickhouse'un bir sorgu çalıştırırken dizini kullanıp kullanamayacağını kontrol etmek için ayarları kullanın [force\_index\_by\_date](../../../operations/settings/settings.md#settings-force_index_by_date) ve [force\_primary\_key](../../../operations/settings/settings.md). Aylara göre bölümleme anahtarı, yalnızca uygun aralıktaki tarihleri içeren veri bloklarını okumanıza izin verir. Bu durumda, veri bloğu birçok tarih için veri içerebilir (bir aya kadar). Bir blok içinde veriler, ilk sütun olarak tarihi içermeyen birincil anahtara göre sıralanır. Bu nedenle, birincil anahtar önekini belirtmeyen yalnızca bir tarih koşulu ile bir sorgu kullanarak tek bir tarih için okunacak daha fazla veri neden olur. -### Kısmen Monotonik Birincil Anahtarlar için Endeks kullanımı {#use-of-index-for-partially-monotonic-primary-keys} +### Kısmen monotonik birincil anahtarlar için Endeks kullanımı {#use-of-index-for-partially-monotonic-primary-keys} Örneğin, Ayın günlerini düşünün. Onlar formu bir [monotonik dizisi](https://en.wikipedia.org/wiki/Monotonic_function) bir ay boyunca, ancak daha uzun süreler için monotonik değil. Bu kısmen monotonik bir dizidir. Bir kullanıcı kısmen monoton birincil anahtar ile tablo oluşturursa, ClickHouse her zamanki gibi seyrek bir dizin oluşturur. Bir kullanıcı bu tür bir tablodan veri seçtiğinde, ClickHouse sorgu koşullarını analiz eder. Kullanıcı, dizinin iki işareti arasında veri almak isterse ve bu işaretlerin her ikisi de bir ay içinde düşerse, ClickHouse bu özel durumda dizini kullanabilir, çünkü sorgu parametreleri ile dizin işaretleri arasındaki mesafeyi hesaplayabilir. @@ -251,7 +251,7 @@ Sorgu parametresi aralığındaki birincil anahtarın değerleri monotonik bir s ClickHouse bu mantığı yalnızca ay dizilerinin günleri için değil, kısmen monotonik bir diziyi temsil eden herhangi bir birincil anahtar için kullanır. -### Veri Atlama Indeksleri (deneysel) {#table_engine-mergetree-data_skipping-indexes} +### Veri atlama indeksleri (deneysel) {#table_engine-mergetree-data_skipping-indexes} Dizin bildirimi sütunlar bölümünde `CREATE` sorgu. @@ -285,7 +285,7 @@ SELECT count() FROM table WHERE s < 'z' SELECT count() FROM table WHERE u64 * i32 == 10 AND u64 * length(s) >= 1234 ``` -#### Mevcut Endeks Türleri {#available-types-of-indices} +#### Mevcut Endeks türleri {#available-types-of-indices} - `minmax` @@ -293,7 +293,7 @@ SELECT count() FROM table WHERE u64 * i32 == 10 AND u64 * length(s) >= 1234 - `set(max_rows)` - Belirtilen ifadenin benzersiz değerlerini depolar (en fazla `max_rows` satırlar, `max_rows=0` anlama “no limits”). Kontrol etmek için değerleri kullanır `WHERE` ifade, bir veri bloğu üzerinde tatmin edilemez değildir. + Belirtilen ifadenin benzersiz değerlerini depolar (en fazla `max_rows` satırlar, `max_rows=0` Anlamına geliyor “no limits”). Kontrol etmek için değerleri kullanır `WHERE` ifade, bir veri bloğu üzerinde tatmin edilemez değildir. - `ngrambf_v1(n, size_of_bloom_filter_in_bytes, number_of_hash_functions, random_seed)` @@ -372,7 +372,7 @@ Eşzamanlı tablo erişimi için çoklu sürüm kullanıyoruz. Başka bir deyiş Bir tablodan okuma otomatik olarak paralelleştirilir. -## Sütunlar Ve Tablolar için TTL {#table_engine-mergetree-ttl} +## Sütunlar ve tablolar için TTL {#table_engine-mergetree-ttl} Değerlerin ömrünü belirler. @@ -387,7 +387,7 @@ TTL time_column TTL time_column + interval ``` -Tanımlamak `interval`, kullanma [zaman aralığı](../../../sql-reference/operators.md#operators-datetime) operatörler. +Tanımlamak `interval`, kullanma [zaman aralığı](../../../sql-reference/operators/index.md#operators-datetime) operatörler. ``` sql TTL date_time + INTERVAL 1 MONTH @@ -480,11 +480,11 @@ ClickHouse, verilerin süresi dolduğunu gördüğünde, zamanlama dışı bir b Gerçekleştir theirseniz `SELECT` birleştirme arasında sorgu, süresi dolmuş veri alabilirsiniz. Bunu önlemek için, [OPTIMIZE](../../../sql-reference/statements/misc.md#misc_operations-optimize) önce sorgu `SELECT`. -## Veri Depolama İçin Birden Fazla Blok Cihazı Kullanma {#table_engine-mergetree-multiple-volumes} +## Veri depolama için birden fazla blok cihazı kullanma {#table_engine-mergetree-multiple-volumes} ### Giriş {#introduction} -`MergeTree` aile tablo motorları birden fazla blok cihazlarda veri saklayabilirsiniz. Örneğin, belirli bir tablonun verileri örtük olarak bölündüğünde yararlı olabilir “hot” ve “cold”. En son veriler düzenli olarak talep edilir, ancak yalnızca az miktarda alan gerektirir. Aksine, yağ kuyruklu tarihsel veriler nadiren talep edilir. Birkaç disk varsa, “hot” veriler hızlı disklerde (örneğin, NVMe SSD’ler veya bellekte) bulunabilir; “cold” veri-nispeten yavaş olanlar (örneğin, HDD). +`MergeTree` aile tablo motorları birden fazla blok cihazlarda veri saklayabilirsiniz. Örneğin, belirli bir tablonun verileri örtük olarak bölündüğünde yararlı olabilir “hot” ve “cold”. En son veriler düzenli olarak talep edilir, ancak yalnızca az miktarda alan gerektirir. Aksine, yağ kuyruklu tarihsel veriler nadiren talep edilir. Birkaç disk varsa, “hot” veriler hızlı disklerde (örneğin, NVMe SSD'ler veya bellekte) bulunabilir; “cold” veri-nispeten yavaş olanlar (örneğin, HDD). Veri kısmı için minimum hareketli birimdir `MergeTree`- motor masaları. Bir parçaya ait veriler bir diskte saklanır. Veri parçaları arka planda diskler arasında (kullanıcı ayarlarına göre) ve aynı zamanda [ALTER](../../../sql-reference/statements/alter.md#alter_move-partition) sorgular. @@ -567,7 +567,7 @@ Etiketler: - `policy_name_N` — Policy name. Policy names must be unique. - `volume_name_N` — Volume name. Volume names must be unique. - `disk` — a disk within a volume. -- `max_data_part_size_bytes` — the maximum size of a part that can be stored on any of the volume’s disks. +- `max_data_part_size_bytes` — the maximum size of a part that can be stored on any of the volume's disks. - `move_factor` — when the amount of available space gets lower than this factor, data automatically start to move on the next volume if any (by default, 0.1). Cofiguration örnekleri: @@ -602,10 +602,10 @@ Cofiguration örnekleri: ``` -Verilen örnekte, `hdd_in_order` politika uygular [Ro -und-robin](https://en.wikipedia.org/wiki/Round-robin_scheduling) yaklaşma. Böylece bu politika yalnızca bir birim tanımlar (`single`), veri parçaları tüm disklerinde dairesel sırayla saklanır. Bu tür bir politika, sisteme birkaç benzer disk takılıysa, ancak RAID yapılandırılmamışsa oldukça yararlı olabilir. Her bir disk sürücüsünün güvenilir olmadığını ve bunu 3 veya daha fazla çoğaltma faktörü ile telafi etmek isteyebileceğinizi unutmayın. +Verilen örnekte, `hdd_in_order` politika uygular [Ro -und-robin](https://en.wikipedia.org/wiki/Round-robin_scheduling) yaklaşmak. Böylece bu politika yalnızca bir birim tanımlar (`single`), veri parçaları tüm disklerinde dairesel sırayla saklanır. Bu tür bir politika, sisteme birkaç benzer disk takılıysa, ancak RAID yapılandırılmamışsa oldukça yararlı olabilir. Her bir disk sürücüsünün güvenilir olmadığını ve bunu 3 veya daha fazla çoğaltma faktörü ile telafi etmek isteyebileceğinizi unutmayın. Sistemde farklı türde diskler varsa, `moving_from_ssd_to_hdd` politika yerine kullanılabilir. Birim `hot` bir SSD disk oluşur (`fast_ssd`) ve bu birimde saklanabilecek bir parçanın maksimum boyutu 1GB. Tüm parçaları ile boyutu daha büyük 1 GB üzerinde doğrudan saklanır `cold` bir HDD diski içeren birim `disk1`. -Ayrıca, bir kez disk `fast_ssd` 80’den fazla % tarafından doldurulur, veri transfer edilecektir `disk1` bir arka plan işlemi ile. +Ayrıca, bir kez disk `fast_ssd` 80'den fazla % tarafından doldurulur, veri transfer edilecektir `disk1` bir arka plan işlemi ile. Depolama ilkesi içindeki birim numaralandırma sırası önemlidir. Bir birim aşırı doldurulduktan sonra, veriler bir sonrakine taşınır. Disk numaralandırma sırası da önemlidir, çünkü veriler sırayla depolanır. diff --git a/docs/tr/engines/table-engines/mergetree-family/replacingmergetree.md b/docs/tr/engines/table-engines/mergetree-family/replacingmergetree.md index a1ea0fc0d1b..a24c84e9a16 100644 --- a/docs/tr/engines/table-engines/mergetree-family/replacingmergetree.md +++ b/docs/tr/engines/table-engines/mergetree-family/replacingmergetree.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 33 toc_title: ReplacingMergeTree --- diff --git a/docs/tr/engines/table-engines/mergetree-family/replication.md b/docs/tr/engines/table-engines/mergetree-family/replication.md index 68a130a78c6..108ddf42d7f 100644 --- a/docs/tr/engines/table-engines/mergetree-family/replication.md +++ b/docs/tr/engines/table-engines/mergetree-family/replication.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 31 toc_title: "Veri \xC7o\u011Faltma" --- @@ -63,7 +63,7 @@ ZooKeeper kullanılmaz `SELECT` çoğaltma performansını etkilemez çünkü so Her biri için `INSERT` sorgu, yaklaşık on girişleri zookeeper birkaç işlemler aracılığıyla eklenir. (Daha kesin olmak gerekirse, bu eklenen her veri bloğu içindir; bir ekleme sorgusu her bir blok veya bir blok içerir `max_insert_block_size = 1048576` satırlar.) Bu, biraz daha uzun gecikmelere yol açar `INSERT` çoğaltılmamış tablolarla karşılaştırıldığında. Ancak, birden fazla olmayan gruplar halinde veri eklemek için önerileri izlerseniz `INSERT` saniyede, herhangi bir sorun yaratmaz. Bir ZooKeeper kümesini koordine etmek için kullanılan tüm ClickHouse kümesinin toplam birkaç yüzü vardır `INSERTs` saniyede. Veri eklerindeki verim (saniyede satır sayısı), çoğaltılmamış veriler için olduğu kadar yüksektir. -Çok büyük kümeler için, farklı kırıklar için farklı ZooKeeper kümelerini kullanabilirsiniz. Ancak, bu Yandex’de gerekli değildir.Metrica küme (yaklaşık 300 sunucu). +Çok büyük kümeler için, farklı kırıklar için farklı ZooKeeper kümelerini kullanabilirsiniz. Ancak, bu Yandex'de gerekli değildir.Metrica küme (yaklaşık 300 sunucu). Çoğaltma zaman uyumsuz ve çok ana. `INSERT` sorgular (yanı sıra `ALTER`) mevcut herhangi bir sunucuya gönderilebilir. Veri sorgu çalıştırıldığı sunucuda eklenir ve sonra diğer sunuculara kopyalanır. Zaman uyumsuz olduğundan, son eklenen veriler bazı gecikme ile diğer yinelemeler görünür. Yinelemelerin bir kısmı mevcut değilse, veriler kullanılabilir olduklarında yazılır. Bir çoğaltma varsa, gecikme, sıkıştırılmış veri bloğunu ağ üzerinden aktarmak için gereken süredir. @@ -71,7 +71,7 @@ Varsayılan olarak, bir INSERT sorgusu yalnızca bir yinelemeden veri yazma onay Her veri bloğu atomik olarak yazılır. Ekle sorgusu kadar bloklara ayrılmıştır `max_insert_block_size = 1048576` satırlar. Diğer bir deyişle, `INSERT` sorgu 1048576 satırdan daha az, atomik olarak yapılır. -Veri blokları tekilleştirilmiştir. Aynı veri bloğunun (aynı sırayla aynı satırları içeren aynı boyuttaki veri blokları) birden fazla yazımı için, blok yalnızca bir kez yazılır. Bunun nedeni, istemci uygulaması verilerin DB’YE yazılıp yazılmadığını bilmediğinde ağ arızaları durumunda, `INSERT` sorgu sadece tekrar edilebilir. Hangi çoğaltma eklerinin aynı verilerle gönderildiği önemli değildir. `INSERTs` idempotent vardır. Tekilleştirme parametreleri tarafından kontrol edilir [merge\_tree](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-merge_tree) sunucu ayarları. +Veri blokları tekilleştirilmiştir. Aynı veri bloğunun (aynı sırayla aynı satırları içeren aynı boyuttaki veri blokları) birden fazla yazımı için, blok yalnızca bir kez yazılır. Bunun nedeni, istemci uygulaması verilerin DB'YE yazılıp yazılmadığını bilmediğinde ağ arızaları durumunda, `INSERT` sorgu sadece tekrar edilebilir. Hangi çoğaltma eklerinin aynı verilerle gönderildiği önemli değildir. `INSERTs` idempotent vardır. Tekilleştirme parametreleri tarafından kontrol edilir [merge\_tree](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-merge_tree) sunucu ayarları. Çoğaltma sırasında, yalnızca eklenecek kaynak veriler ağ üzerinden aktarılır. Daha fazla veri dönüşümü (birleştirme), tüm kopyalarda aynı şekilde koordine edilir ve gerçekleştirilir. Bu, ağ kullanımını en aza indirir; bu, çoğaltmaların farklı veri merkezlerinde bulunduğu zaman çoğaltmanın iyi çalıştığı anlamına gelir. (Farklı veri merkezlerinde çoğaltmanın çoğaltmanın ana hedefi olduğunu unutmayın .) @@ -132,10 +132,10 @@ Bu durumda, yol aşağıdaki parçalardan oluşur: `/clickhouse/tables/` ortak önek. Tam olarak bunu kullanmanızı öneririz. -`{layer}-{shard}` shard tanımlayıcısıdır. Bu örnekte Yandex’den beri iki bölümden oluşmaktadır.Metrica küme iki seviyeli sharding kullanır. Çoğu görev için, yalnızca shard tanımlayıcısına genişletilecek olan {shard} ikamesini bırakabilirsiniz. +`{layer}-{shard}` shard tanımlayıcısıdır. Bu örnekte Yandex'den beri iki bölümden oluşmaktadır.Metrica küme iki seviyeli sharding kullanır. Çoğu görev için, yalnızca shard tanımlayıcısına genişletilecek olan {shard} ikamesini bırakabilirsiniz. `table_name` ZooKeeper tablo için düğüm adıdır. Tablo adı ile aynı yapmak için iyi bir fikirdir. Açıkça tanımlanır, çünkü tablo adının aksine, bir yeniden adlandırma sorgusundan sonra değişmez. -*HINT*: önüne bir veritabanı adı ekleyebilirsiniz `table_name` yanında. E. g. `db_name.table_name` +*HINT*: önüne bir veritabanı adı ekleyebilirsiniz `table_name` ayrıca. E. g. `db_name.table_name` Çoğaltma adı, aynı tablonun farklı yinelemelerini tanımlar. Örnekte olduğu gibi bunun için sunucu adını kullanabilirsiniz. Adın sadece her parça içinde benzersiz olması gerekir. @@ -157,9 +157,9 @@ ZooKeeper sırasında kullanılamıyorsa bir `INSERT`, veya ZooKeeper ile etkile ZooKeeper bağlandıktan sonra, sistem yerel dosya sistemindeki veri kümesinin beklenen veri kümesiyle eşleşip eşleşmediğini kontrol eder (ZooKeeper bu bilgileri saklar). Küçük tutarsızlıklar varsa, sistem verileri kopyalarla senkronize ederek bunları çözer. -Sistem bozuk veri parçalarını (yanlış dosya boyutu ile) veya tanınmayan parçaları (dosya sistemine yazılmış ancak Zookeeper’da kaydedilmemiş parçalar) tespit ederse, bunları `detached` alt dizin (silinmez). Eksik parçalar kopyalardan kopyalanır. +Sistem bozuk veri parçalarını (yanlış dosya boyutu ile) veya tanınmayan parçaları (dosya sistemine yazılmış ancak Zookeeper'da kaydedilmemiş parçalar) tespit ederse, bunları `detached` alt dizin (silinmez). Eksik parçalar kopyalardan kopyalanır. -Clickhouse’un büyük miktarda veriyi otomatik olarak silme gibi yıkıcı eylemler gerçekleştirmediğini unutmayın. +Clickhouse'un büyük miktarda veriyi otomatik olarak silme gibi yıkıcı eylemler gerçekleştirmediğini unutmayın. Sunucu başlatıldığında (veya ZooKeeper ile yeni bir oturum kurduğunda), yalnızca tüm dosyaların miktarını ve boyutlarını kontrol eder. Dosya boyutları eşleşirse, ancak bayt ortasında bir yerde değiştirilmişse, bu hemen algılanmaz, ancak yalnızca bir dosya için verileri okumaya çalışırken algılanmaz. `SELECT` sorgu. Sorgu, eşleşen olmayan bir sağlama toplamı veya sıkıştırılmış bir bloğun boyutu hakkında bir özel durum atar. Bu durumda, veri parçaları doğrulama kuyruğuna eklenir ve gerekirse kopyalardan kopyalanır. @@ -177,7 +177,7 @@ Sunucuyu yeniden başlatın. Başlangıçta, sunucu bu bayrakları siler ve kurt Tüm veriler ve meta veriler sunuculardan birinden kaybolduysa, kurtarma için şu adımları izleyin: -1. Clickhouse’u sunucuya yükleyin. Bunları kullanırsanız, shard tanımlayıcısı ve yinelemeleri içeren yapılandırma dosyasında doğru değiştirmelerin tanımlayın. +1. Clickhouse'u sunucuya yükleyin. Bunları kullanırsanız, shard tanımlayıcısı ve yinelemeleri içeren yapılandırma dosyasında doğru değiştirmelerin tanımlayın. 2. Sunucularda el ile çoğaltılması gereken yinelenmemiş tablolar varsa, verilerini bir kopyadan kopyalayın (dizinde `/var/lib/clickhouse/data/db_name/table_name/`). 3. Bulunan tablo tanım copylarını kopyala `/var/lib/clickhouse/metadata/` bir kopyadan. Tablo tanımlarında bir parça veya çoğaltma tanımlayıcısı açıkça tanımlanmışsa, bu kopyaya karşılık gelecek şekilde düzeltin. (Alternatif olarak, sunucuyu başlatın ve tüm `ATTACH TABLE` içinde olması gereken sorgular .sql dosyaları `/var/lib/clickhouse/metadata/`.) 4. Kurtarma işlemini başlatmak için ZooKeeper düğümünü oluşturun `/path_to_table/replica_name/flags/force_restore_data` herhangi bir içerikle veya tüm çoğaltılmış tabloları geri yüklemek için komutu çalıştırın: `sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data` @@ -188,7 +188,7 @@ Alternatif bir kurtarma seçeneği zookeeper kayıp yineleme hakkında bilgi sil Kurtarma sırasında ağ bant genişliği üzerinde herhangi bir kısıtlama yoktur. Aynı anda birçok yinelemeyi geri yüklüyorsanız bunu aklınızda bulundurun. -## Mergetree’den Replicatedmergetree’ye Dönüştürme {#converting-from-mergetree-to-replicatedmergetree} +## Mergetree'den Replicatedmergetree'ye dönüştürme {#converting-from-mergetree-to-replicatedmergetree} Terimi kullanıyoruz `MergeTree` tüm tablo motorlarına başvurmak için `MergeTree family` için aynı `ReplicatedMergeTree`. @@ -200,7 +200,7 @@ Varolan MergeTree tablosunu yeniden adlandırın, sonra bir `ReplicatedMergeTree Eski tablodan veri taşıma `detached` yeni tablo verileri ile dizin içindeki alt dizin (`/var/lib/clickhouse/data/db_name/table_name/`). Sonra koş `ALTER TABLE ATTACH PARTITION` bu veri parçalarını çalışma kümesine eklemek için yinelemelerden birinde. -## Replicatedmergetree’den Mergetree’ye Dönüştürme {#converting-from-replicatedmergetree-to-mergetree} +## Replicatedmergetree'den Mergetree'ye dönüştürme {#converting-from-replicatedmergetree-to-mergetree} Farklı bir adla bir MergeTree tablosu oluşturun. İle dizinden tüm verileri taşıyın `ReplicatedMergeTree` yeni tablonun veri dizinine tablo verileri. Sonra Sil `ReplicatedMergeTree` tablo ve sunucuyu yeniden başlatın. @@ -211,7 +211,7 @@ Eğer bir kurtulmak istiyorsanız `ReplicatedMergeTree` sunucu başlatmadan tabl Bundan sonra, sunucuyu başlatabilir, bir `MergeTree` tablo, verileri kendi dizinine taşıyın ve sonra sunucuyu yeniden başlatın. -## Zookeeper kümesindeki Meta Veriler kaybolduğunda Veya Zarar gördüğünde Kurtarma {#recovery-when-metadata-in-the-zookeeper-cluster-is-lost-or-damaged} +## Zookeeper kümesindeki meta veriler kaybolduğunda veya zarar gördüğünde kurtarma {#recovery-when-metadata-in-the-zookeeper-cluster-is-lost-or-damaged} ZooKeeper içindeki veriler kaybolduysa veya hasar gördüyse, verileri yukarıda açıklandığı gibi yinelenmemiş bir tabloya taşıyarak kaydedebilirsiniz. diff --git a/docs/tr/engines/table-engines/mergetree-family/summingmergetree.md b/docs/tr/engines/table-engines/mergetree-family/summingmergetree.md index d937ca5371f..3092339658c 100644 --- a/docs/tr/engines/table-engines/mergetree-family/summingmergetree.md +++ b/docs/tr/engines/table-engines/mergetree-family/summingmergetree.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 34 toc_title: SummingMergeTree --- @@ -100,7 +100,7 @@ Veriler bir tabloya eklendiğinde, bunlar olduğu gibi kaydedilir. ClickHouse, v ClickHouse can merge the data parts so that different resulting parts of data cat consist rows with the same primary key, i.e. the summation will be incomplete. Therefore (`SELECT`) bir toplama fonksiyonu [toplam()](../../../sql-reference/aggregate-functions/reference.md#agg_function-sum) ve `GROUP BY` yukarıdaki örnekte açıklandığı gibi yan tümcesi bir sorguda kullanılmalıdır. -### Toplama İçin Ortak Kurallar {#common-rules-for-summation} +### Toplama için ortak kurallar {#common-rules-for-summation} Sayısal veri türüne sahip sütunlardaki değerler özetlenir. Sütun kümesi parametre tarafından tanımlanır `columns`. @@ -110,7 +110,7 @@ Sütun birincil anahtarda değilse ve özetlenmezse, mevcut olanlardan rasgele b Değerler, birincil anahtardaki sütunlar için özetlenmez. -### Aggregatefunction Sütunlarındaki Toplama {#the-summation-in-the-aggregatefunction-columns} +### Aggregatefunction Sütunlarındaki toplama {#the-summation-in-the-aggregatefunction-columns} Sütunlar için [AggregateFunction türü](../../../sql-reference/data-types/aggregatefunction.md) ClickHouse olarak davranır [AggregatingMergeTree](aggregatingmergetree.md) işleve göre motor toplama. diff --git a/docs/tr/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md b/docs/tr/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md index 53da2f2e185..966b6b1a7d4 100644 --- a/docs/tr/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md +++ b/docs/tr/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 37 toc_title: VersionedCollapsingMergeTree --- @@ -64,7 +64,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], ... -) ENGINE [=] VersionedCollapsingMergeTree(date-column [, sampling_expression], (primary, key), index_granularity, sign, version) +) ENGINE [=] VersionedCollapsingMergeTree(date-column [, samp#table_engines_versionedcollapsingmergetreeling_expression], (primary, key), index_granularity, sign, version) ``` Dışındaki tüm parametreler `sign` ve `version` içinde olduğu gibi aynı anlama sahip `MergeTree`. @@ -85,7 +85,7 @@ Dışındaki tüm parametreler `sign` ve `version` içinde olduğu gibi aynı an Bazı nesneler için sürekli değişen verileri kaydetmeniz gereken bir durumu düşünün. Bir nesne için bir satıra sahip olmak ve değişiklikler olduğunda satırı güncellemek mantıklıdır. Ancak, depolama alanındaki verileri yeniden yazmayı gerektirdiğinden, güncelleştirme işlemi bir DBMS için pahalı ve yavaştır. Verileri hızlı bir şekilde yazmanız gerekiyorsa güncelleştirme kabul edilemez, ancak değişiklikleri bir nesneye sırayla aşağıdaki gibi yazabilirsiniz. -Kullan… `Sign` satır yazarken sütun. Eğer `Sign = 1` bu, satırın bir nesnenin durumu olduğu anlamına gelir (diyelim “state” satır). Eğer `Sign = -1` aynı özelliklere sahip bir nesnenin durumunun iptal edildiğini gösterir (buna “cancel” satır). Ayrıca kullanın `Version` bir nesnenin her durumunu ayrı bir sayı ile tanımlaması gereken sütun. +Kullan... `Sign` satır yazarken sütun. Eğer `Sign = 1` bu, satırın bir nesnenin durumu olduğu anlamına gelir (diyelim “state” satır). Eğer `Sign = -1` aynı özelliklere sahip bir nesnenin durumunun iptal edildiğini gösterir (buna “cancel” satır). Ayrıca kullanın `Version` bir nesnenin her durumunu ayrı bir sayı ile tanımlaması gereken sütun. Örneğin, kullanıcıların bazı sitede kaç sayfa ziyaret ettiğini ve ne kadar süre orada olduklarını hesaplamak istiyoruz. Bir noktada, kullanıcı etkinliği durumu ile aşağıdaki satırı yazıyoruz: @@ -131,7 +131,7 @@ Her değişiklik için neden iki satıra ihtiyacımız olduğunu bulmak için bk ClickHouse veri parçalarını birleştirdiğinde, aynı birincil anahtar ve sürüm ve farklı olan her satır çiftini siler `Sign`. Satırların sırası önemli değil. -ClickHouse veri eklediğinde, satırları birincil anahtarla sipariş eder. Eğer… `Version` sütun birincil anahtarda değil, ClickHouse onu birincil anahtara örtük olarak son alan olarak ekler ve sipariş vermek için kullanır. +ClickHouse veri eklediğinde, satırları birincil anahtarla sipariş eder. Eğer... `Version` sütun birincil anahtarda değil, ClickHouse onu birincil anahtara örtük olarak son alan olarak ekler ve sipariş vermek için kullanır. ## Veri Seçme {#selecting-data} @@ -143,7 +143,7 @@ Toplanan `count`, `sum` ve `avg` bu şekilde hesaplanabilir. Toplanan `uniq` bir İle verileri ayıklamak gerekiyorsa “collapsing” ancak toplama olmadan (örneğin, en yeni değerleri belirli koşullarla eşleşen satırların mevcut olup olmadığını kontrol etmek için) `FINAL` değiştirici için `FROM` yan. Bu yaklaşım verimsizdir ve büyük tablolarla kullanılmamalıdır. -## Kullanım Örneği {#example-of-use} +## Kullanım örneği {#example-of-use} Örnek veriler: diff --git a/docs/tr/engines/table-engines/special/buffer.md b/docs/tr/engines/table-engines/special/buffer.md index f15e2a5407a..1770738db00 100644 --- a/docs/tr/engines/table-engines/special/buffer.md +++ b/docs/tr/engines/table-engines/special/buffer.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 45 toc_title: Arabellek --- # Arabellek {#buffer} -RAM’de yazmak için verileri tamponlar, periyodik olarak başka bir tabloya temizler. Okuma işlemi sırasında veri arabellekten ve diğer tablodan aynı anda okunur. +RAM'de yazmak için verileri tamponlar, periyodik olarak başka bir tabloya temizler. Okuma işlemi sırasında veri arabellekten ve diğer tablodan aynı anda okunur. ``` sql Buffer(database, table, num_layers, min_time, max_time, min_rows, max_rows, min_bytes, max_bytes) @@ -28,7 +28,7 @@ Veri arabellekten temizlendi ve hedef tabloya yazılır eğer tüm `min*` koşul Yazma işlemi sırasında veri bir `num_layers` rastgele tampon sayısı. Veya, eklenecek veri kısmı yeterince büyükse (daha büyük `max_rows` veya `max_bytes`), arabelleği atlayarak doğrudan hedef tabloya yazılır. -Verilerin yıkanması için koşullar, her biri için ayrı ayrı hesaplanır. `num_layers` arabellekler. Örneğin, `num_layers = 16` ve `max_bytes = 100000000`, maksimum RAM tüketimi 1,6 GB’DİR. +Verilerin yıkanması için koşullar, her biri için ayrı ayrı hesaplanır. `num_layers` arabellekler. Örneğin, `num_layers = 16` ve `max_bytes = 100000000`, maksimum RAM tüketimi 1,6 GB'DİR. Örnek: @@ -36,7 +36,7 @@ Verilerin yıkanması için koşullar, her biri için ayrı ayrı hesaplanır. ` CREATE TABLE merge.hits_buffer AS merge.hits ENGINE = Buffer(merge, hits, 16, 10, 100, 10000, 1000000, 10000000, 100000000) ``` -Oluşturma Bir ‘merge.hits\_buffer’ ile aynı yapıya sahip tablo ‘merge.hits’ ve Tampon motorunu kullanarak. Bu tabloya yazarken, veriler RAM’de arabelleğe alınır ve daha sonra ‘merge.hits’ Tablo. 16 tamponlar oluşturulur. 100 saniye geçti veya bir milyon satır yazılmış veya 100 MB veri yazılmıştır; ya da aynı anda 10 saniye geçti ve 10.000 satır ve 10 MB veri yazılmıştır, bunların her veri temizlendi. Örneğin, sadece bir satır yazılmışsa, 100 saniye sonra ne olursa olsun, yıkanacaktır. Ancak, birçok satır yazılmışsa, veriler daha erken temizlenecektir. +Oluşturma Bir ‘merge.hits\_buffer’ ile aynı yapıya sahip tablo ‘merge.hits’ ve Tampon motorunu kullanarak. Bu tabloya yazarken, veriler RAM'de arabelleğe alınır ve daha sonra ‘merge.hits’ Tablo. 16 tamponlar oluşturulur. 100 saniye geçti veya bir milyon satır yazılmış veya 100 MB veri yazılmıştır; ya da aynı anda 10 saniye geçti ve 10.000 satır ve 10 MB veri yazılmıştır, bunların her veri temizlendi. Örneğin, sadece bir satır yazılmışsa, 100 saniye sonra ne olursa olsun, yıkanacaktır. Ancak, birçok satır yazılmışsa, veriler daha erken temizlenecektir. Sunucu DROP TABLE veya DETACH TABLE ile durdurulduğunda, arabellek verileri de hedef tabloya temizlendi. @@ -58,7 +58,7 @@ Son ve örnek arabellek tabloları için düzgün çalışmıyor. Bu koşullar h Bir arabelleğe veri eklerken, arabelleklerden biri kilitlenir. Bir okuma işlemi aynı anda tablodan gerçekleştiriliyor, bu gecikmelere neden olur. -Bir arabellek tablosuna eklenen veriler, alt tabloda farklı bir sırada ve farklı bloklarda sonuçlanabilir. Bu nedenle, bir arabellek tablo CollapsingMergeTree doğru yazmak için kullanmak zordur. Sorunları önlemek için şunları ayarlayabilirsiniz ‘num\_layers’ 1’e. +Bir arabellek tablosuna eklenen veriler, alt tabloda farklı bir sırada ve farklı bloklarda sonuçlanabilir. Bu nedenle, bir arabellek tablo CollapsingMergeTree doğru yazmak için kullanmak zordur. Sorunları önlemek için şunları ayarlayabilirsiniz ‘num\_layers’ 1'e. Hedef tablo yinelenirse, bir arabellek tablosuna yazarken yinelenmiş tabloların bazı beklenen özellikleri kaybolur. Satır ve veri parçaları boyutlarda sipariş için rasgele değişiklikler veri çoğaltma güvenilir olması mümkün olmadığını ifade eden çalışma, kapanmasına neden ‘exactly once’ çoğaltılan tablolara yazın. diff --git a/docs/tr/engines/table-engines/special/dictionary.md b/docs/tr/engines/table-engines/special/dictionary.md index be3d5b5a72d..987bcf8e1c1 100644 --- a/docs/tr/engines/table-engines/special/dictionary.md +++ b/docs/tr/engines/table-engines/special/dictionary.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 35 toc_title: "S\xF6zl\xFCk" --- @@ -66,7 +66,7 @@ WHERE name = 'products' Kullanabilirsiniz [dictGet\*](../../../sql-reference/functions/ext-dict-functions.md#ext_dict_functions) sözlük verilerini bu formatta almak için işlev. -Bu görünüm, ham veri almanız gerektiğinde veya bir `JOIN` işleyiş. Bu durumlar için şunları kullanabilirsiniz `Dictionary` bir tabloda sözlük verilerini görüntüleyen motor. +Bu görünüm, ham veri almanız gerektiğinde veya bir `JOIN` operasyon. Bu durumlar için şunları kullanabilirsiniz `Dictionary` bir tabloda sözlük verilerini görüntüleyen motor. Sözdizimi: diff --git a/docs/tr/engines/table-engines/special/distributed.md b/docs/tr/engines/table-engines/special/distributed.md index 0c6843cc495..d4ea832194b 100644 --- a/docs/tr/engines/table-engines/special/distributed.md +++ b/docs/tr/engines/table-engines/special/distributed.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 33 toc_title: "Da\u011F\u0131l\u0131" --- @@ -39,7 +39,7 @@ Veriler yalnızca okunmakla kalmaz, aynı zamanda uzak sunucularda kısmen işle Veritabanı adı yerine, bir dize döndüren sabit bir ifade kullanabilirsiniz. Örneğin: currentDatabase (). -logs – The cluster name in the server’s config file. +logs – The cluster name in the server's config file. Kümeler şöyle ayarlanır: @@ -84,7 +84,7 @@ Yinelemeler sunucuları çoğaltılıyor (tüm verileri okumak için, yinelemele Küme adları nokta içermemelidir. Parametre `host`, `port` ve isteğe bağlı olarak `user`, `password`, `secure`, `compression` her sunucu için belirtilir: -- `host` – The address of the remote server. You can use either the domain or the IPv4 or IPv6 address. If you specify the domain, the server makes a DNS request when it starts, and the result is stored as long as the server is running. If the DNS request fails, the server doesn’t start. If you change the DNS record, restart the server. +- `host` – The address of the remote server. You can use either the domain or the IPv4 or IPv6 address. If you specify the domain, the server makes a DNS request when it starts, and the result is stored as long as the server is running. If the DNS request fails, the server doesn't start. If you change the DNS record, restart the server. - `port` – The TCP port for messenger activity (‘tcp\_port’ yapılandırmada, genellikle 9000 olarak ayarlanır). Http\_port ile karıştırmayın. - `user` – Name of the user for connecting to a remote server. Default value: default. This user must have access to connect to the specified server. Access is configured in the users.xml file. For more information, see the section [Erişim hakları](../../../operations/access-rights.md). - `password` – The password for connecting to a remote server (not masked). Default value: empty string. @@ -103,7 +103,7 @@ Kümelerinizi görüntülemek için ‘system.clusters’ Tablo. Dağıtılmış motor, yerel bir sunucu gibi bir küme ile çalışmaya izin verir. Ancak, küme uzatılamaz: yapılandırmasını sunucu yapılandırma dosyasına yazmanız gerekir (tüm kümenin sunucuları için daha da iyisi). -The Distributed engine requires writing clusters to the config file. Clusters from the config file are updated on the fly, without restarting the server. If you need to send a query to an unknown set of shards and replicas each time, you don’t need to create a Distributed table – use the ‘remote’ bunun yerine tablo işlevi. Bölümüne bakınız [Tablo fonksiyonları](../../../sql-reference/table-functions/index.md). +The Distributed engine requires writing clusters to the config file. Clusters from the config file are updated on the fly, without restarting the server. If you need to send a query to an unknown set of shards and replicas each time, you don't need to create a Distributed table – use the ‘remote’ bunun yerine tablo işlevi. Bölümüne bakınız [Tablo fonksiyonları](../../../sql-reference/table-functions/index.md). Bir kümeye veri yazmak için iki yöntem vardır: @@ -111,7 +111,7 @@ Bir kümeye veri yazmak için iki yöntem vardır: İkinci olarak, dağıtılmış bir tabloda ekleme gerçekleştirebilirsiniz. Bu durumda, tablo eklenen verileri sunucuların kendisine dağıtacaktır. Dağıtılmış bir tabloya yazmak için, bir sharding anahtar kümesi (son parametre) olmalıdır. Ek olarak, yalnızca bir parça varsa, yazma işlemi sharding anahtarını belirtmeden çalışır, çünkü bu durumda hiçbir şey ifade etmez. -Her parça yapılandırma dosyasında tanımlanan bir ağırlığa sahip olabilir. Varsayılan olarak, ağırlık bir eşittir. Veriler, parça ağırlığı ile orantılı miktarda parçalara dağıtılır. Örneğin, iki parça varsa ve birincisi 9’luk bir ağırlığa sahipse, ikincisi 10’luk bir ağırlığa sahipse, ilk satırların 9 / 19 parçası gönderilir ve ikincisi 10 / 19 gönderilir. +Her parça yapılandırma dosyasında tanımlanan bir ağırlığa sahip olabilir. Varsayılan olarak, ağırlık bir eşittir. Veriler, parça ağırlığı ile orantılı miktarda parçalara dağıtılır. Örneğin, iki parça varsa ve birincisi 9'luk bir ağırlığa sahipse, ikincisi 10'luk bir ağırlığa sahipse, ilk satırların 9 / 19 parçası gönderilir ve ikincisi 10 / 19 gönderilir. Her shard olabilir ‘internal\_replication’ yapılandırma dosyasında tanımlanan parametre. @@ -119,18 +119,18 @@ Bu parametre şu şekilde ayarlanırsa ‘true’, yazma işlemi ilk sağlıklı Olarak ayarlan ifmışsa ‘false’ (varsayılan), veriler tüm kopyalara yazılır. Özünde, bu, dağıtılmış tablonun verilerin kendisini çoğalttığı anlamına gelir. Bu, çoğaltılmış tabloları kullanmaktan daha kötüdür, çünkü kopyaların tutarlılığı denetlenmez ve zamanla biraz farklı veriler içerirler. -Bir veri satırının gönderildiği parçayı seçmek için, parçalama ifadesi analiz edilir ve kalan kısmı, parçaların toplam ağırlığına bölünmesinden alınır. Satır, kalanların yarı aralığına karşılık gelen parçaya gönderilir. ‘prev\_weight’ -e doğru ‘prev\_weights + weight’, nere ‘prev\_weights’ en küçük sayıya sahip parçaların toplam ağırlığı ve ‘weight’ bu parçanın ağırlığı. Örneğin, iki parça varsa ve birincisi 9’luk bir ağırlığa sahipse, ikincisi 10’luk bir ağırlığa sahipse, satır \[0, 9) aralığından kalanlar için ilk parçaya ve ikincisine \[9, 19) aralığından kalanlar için gönderilecektir. +Bir veri satırının gönderildiği parçayı seçmek için, parçalama ifadesi analiz edilir ve kalan kısmı, parçaların toplam ağırlığına bölünmesinden alınır. Satır, kalanların yarı aralığına karşılık gelen parçaya gönderilir. ‘prev\_weight’ -e doğru ‘prev\_weights + weight’, nere ‘prev\_weights’ en küçük sayıya sahip parçaların toplam ağırlığı ve ‘weight’ bu parçanın ağırlığı. Örneğin, iki parça varsa ve birincisi 9'luk bir ağırlığa sahipse, ikincisi 10'luk bir ağırlığa sahipse, satır \[0, 9) aralığından kalanlar için ilk parçaya ve ikincisine \[9, 19) aralığından kalanlar için gönderilecektir. Sharding ifadesi, bir tamsayı döndüren sabitler ve tablo sütunlarından herhangi bir ifade olabilir. Örneğin, ifadeyi kullanabilirsiniz ‘rand()’ verilerin rastgele dağılımı için veya ‘UserID’ kullanıcının kimliğinin bölünmesinden kalanın dağıtımı için (daha sonra tek bir kullanıcının verileri, kullanıcılar tarafından çalışmayı ve katılmayı basitleştiren tek bir parçada bulunur). Sütunlardan biri yeterince eşit olarak dağıtılmazsa, onu bir karma işleve sarabilirsiniz: ınthash64(Userıd). -Bölüm’den basit bir hatırlatma, sharding için sınırlı bir çözümdür ve her zaman uygun değildir. Orta ve büyük hacimlerde veri (düzinelerce sunucu) için çalışır, ancak çok büyük hacimlerde veri (yüzlerce sunucu veya daha fazla) için değildir. İkinci durumda, dağıtılmış tablolarda girdileri kullanmak yerine konu alanı tarafından gerekli olan sharding şemasını kullanın. +Bölüm'den basit bir hatırlatma, sharding için sınırlı bir çözümdür ve her zaman uygun değildir. Orta ve büyük hacimlerde veri (düzinelerce sunucu) için çalışır, ancak çok büyük hacimlerde veri (yüzlerce sunucu veya daha fazla) için değildir. İkinci durumda, dağıtılmış tablolarda girdileri kullanmak yerine konu alanı tarafından gerekli olan sharding şemasını kullanın. -SELECT queries are sent to all the shards and work regardless of how data is distributed across the shards (they can be distributed completely randomly). When you add a new shard, you don’t have to transfer the old data to it. You can write new data with a heavier weight – the data will be distributed slightly unevenly, but queries will work correctly and efficiently. +SELECT queries are sent to all the shards and work regardless of how data is distributed across the shards (they can be distributed completely randomly). When you add a new shard, you don't have to transfer the old data to it. You can write new data with a heavier weight – the data will be distributed slightly unevenly, but queries will work correctly and efficiently. Aşağıdaki durumlarda sharding şeması hakkında endişelenmelisiniz: - Belirli bir anahtar tarafından veri (veya birleştirme) birleştirme gerektiren sorgular kullanılır. Veriler bu anahtar tarafından parçalanırsa, GLOBAL IN veya GLOBAL JOİN yerine local IN veya JOİN kullanabilirsiniz, bu da çok daha etkilidir. -- Çok sayıda küçük Sorgu ile çok sayıda sunucu (yüzlerce veya daha fazla) kullanılır (bireysel müşterilerin sorguları - web siteleri, reklamverenler veya ortaklar). Küçük sorguların tüm kümeyi etkilememesi için, tek bir istemci için tek bir parça üzerinde veri bulmak mantıklıdır. Alternatif olarak, Yandex’te yaptığımız gibi.Metrica, iki seviyeli sharding kurabilirsiniz: tüm kümeyi bölün “layers”, bir katmanın birden fazla parçadan oluşabileceği yer. Tek bir istemci için veriler tek bir katmanda bulunur, ancak kırıklar gerektiğinde bir katmana eklenebilir ve veriler rastgele dağıtılır. Her katman için dağıtılmış tablolar oluşturulur ve genel sorgular için tek bir paylaşılan dağıtılmış tablo oluşturulur. +- Çok sayıda küçük Sorgu ile çok sayıda sunucu (yüzlerce veya daha fazla) kullanılır (bireysel müşterilerin sorguları - web siteleri, reklamverenler veya ortaklar). Küçük sorguların tüm kümeyi etkilememesi için, tek bir istemci için tek bir parça üzerinde veri bulmak mantıklıdır. Alternatif olarak, Yandex'te yaptığımız gibi.Metrica, iki seviyeli sharding kurabilirsiniz: tüm kümeyi bölün “layers”, bir katmanın birden fazla parçadan oluşabileceği yer. Tek bir istemci için veriler tek bir katmanda bulunur, ancak kırıklar gerektiğinde bir katmana eklenebilir ve veriler rastgele dağıtılır. Her katman için dağıtılmış tablolar oluşturulur ve genel sorgular için tek bir paylaşılan dağıtılmış tablo oluşturulur. Veriler zaman uyumsuz olarak yazılır. Tabloya eklendiğinde, veri bloğu sadece yerel dosya sistemine yazılır. Veriler en kısa sürede arka planda uzak sunuculara gönderilir. Veri gönderme süresi tarafından yönetilir [distributed\_directory\_monitor\_sleep\_time\_ms](../../../operations/settings/settings.md#distributed_directory_monitor_sleep_time_ms) ve [distributed\_directory\_monitor\_max\_sleep\_time\_ms](../../../operations/settings/settings.md#distributed_directory_monitor_max_sleep_time_ms) ayarlar. Bu `Distributed` motor ayrı ayrı eklenen verilerle her dosyayı gönderir, ancak toplu dosya gönderme etkinleştirebilirsiniz [distributed\_directory\_monitor\_batch\_ınserts](../../../operations/settings/settings.md#distributed_directory_monitor_batch_inserts) ayar. Bu ayar, yerel sunucu ve ağ kaynaklarını daha iyi kullanarak küme performansını artırır. Tablo dizinindeki dosyaların listesini (gönderilmeyi bekleyen veriler) kontrol ederek verilerin başarıyla gönderilip gönderilmediğini kontrol etmelisiniz: `/var/lib/clickhouse/data/database/table/`. diff --git a/docs/tr/engines/table-engines/special/external-data.md b/docs/tr/engines/table-engines/special/external-data.md index f9cbad62112..3dc7863c1b3 100644 --- a/docs/tr/engines/table-engines/special/external-data.md +++ b/docs/tr/engines/table-engines/special/external-data.md @@ -1,17 +1,17 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 34 toc_title: "D\u0131\u015F veri" --- -# Sorgu işleme için Harici Veriler {#external-data-for-query-processing} +# Sorgu işleme için harici veriler {#external-data-for-query-processing} ClickHouse bir sunucu bir SELECT sorgusu ile birlikte bir sorgu işlemek için gerekli olan verileri gönderme sağlar. Bu veriler geçici bir tabloya konur (bölüme bakın “Temporary tables”) ve sorguda kullanılabilir (örneğin, işleçlerde). Örneğin, önemli kullanıcı tanımlayıcılarına sahip bir metin dosyanız varsa, bu listeyi süzme kullanan bir sorgu ile birlikte sunucuya yükleyebilirsiniz. -Büyük hacimli dış verilerle birden fazla sorgu çalıştırmanız gerekiyorsa, bu özelliği kullanmayın. Verileri vaktinden önce DB’YE yüklemek daha iyidir. +Büyük hacimli dış verilerle birden fazla sorgu çalıştırmanız gerekiyorsa, bu özelliği kullanmayın. Verileri vaktinden önce DB'YE yüklemek daha iyidir. Harici veriler komut satırı istemcisi (etkileşimli olmayan modda) veya HTTP arabirimi kullanılarak yüklenebilir. @@ -25,7 +25,7 @@ Komut satırı istemcisinde, formatta bir parametreler bölümü belirtebilirsin **–external** – Marks the beginning of a clause. **–file** – Path to the file with the table dump, or -, which refers to stdin. -Stdın’den yalnızca tek bir tablo alınabilir. +Stdın'den yalnızca tek bir tablo alınabilir. Aşağıdaki parametreler isteğe bağlıdır: **–name**– Name of the table. If omitted, \_data is used. **–format** – Data format in the file. If omitted, TabSeparated is used. diff --git a/docs/tr/engines/table-engines/special/file.md b/docs/tr/engines/table-engines/special/file.md index 812d69bb3c3..4d3d99c2cf3 100644 --- a/docs/tr/engines/table-engines/special/file.md +++ b/docs/tr/engines/table-engines/special/file.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 37 toc_title: Dosya --- @@ -12,11 +12,11 @@ biçimliler](../../../interfaces/formats.md#formats) (TabSeparated, yerli, vb.). Kullanım örnekleri: -- Clickhouse’dan dosyaya veri aktarımı. +- Clickhouse'dan dosyaya veri aktarımı. - Verileri bir biçimden diğerine dönüştürün. - Bir diskte bir dosya düzenleme yoluyla ClickHouse veri güncelleme. -## ClickHouse Sunucusunda Kullanım {#usage-in-clickhouse-server} +## ClickHouse sunucusunda kullanım {#usage-in-clickhouse-server} ``` sql File(Format) @@ -67,7 +67,7 @@ SELECT * FROM file_engine_table └──────┴───────┘ ``` -## Clickhouse’da kullanım-yerel {#usage-in-clickhouse-local} +## Clickhouse'da kullanım-yerel {#usage-in-clickhouse-local} İçinde [clickhouse-yerel](../../../operations/utilities/clickhouse-local.md) Dosya motoru ek olarak dosya yolunu kabul eder `Format`. Varsayılan giriş / çıkış akışları gibi sayısal veya insan tarafından okunabilir isimler kullanılarak belirtilebilir `0` veya `stdin`, `1` veya `stdout`. **Örnek:** diff --git a/docs/tr/engines/table-engines/special/generate.md b/docs/tr/engines/table-engines/special/generate.md index 30f0b6a0734..4147fd00647 100644 --- a/docs/tr/engines/table-engines/special/generate.md +++ b/docs/tr/engines/table-engines/special/generate.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 46 toc_title: GenerateRandom --- @@ -14,7 +14,7 @@ Kullanım örnekleri: - Tekrarlanabilir büyük tabloyu doldurmak için testte kullanın. - Fuzzing testleri için rastgele girdi oluşturun. -## ClickHouse Sunucusunda Kullanım {#usage-in-clickhouse-server} +## ClickHouse sunucusunda kullanım {#usage-in-clickhouse-server} ``` sql ENGINE = GenerateRandom(random_seed, max_string_length, max_array_length) diff --git a/docs/tr/engines/table-engines/special/index.md b/docs/tr/engines/table-engines/special/index.md index 2e754a86bc8..641135eb1ce 100644 --- a/docs/tr/engines/table-engines/special/index.md +++ b/docs/tr/engines/table-engines/special/index.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_folder_title: "\xD6zellikli" toc_priority: 31 --- diff --git a/docs/tr/engines/table-engines/special/join.md b/docs/tr/engines/table-engines/special/join.md index 966a0b84d0c..bc9182d9823 100644 --- a/docs/tr/engines/table-engines/special/join.md +++ b/docs/tr/engines/table-engines/special/join.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 40 toc_title: Katmak --- # Katmak {#join} -Kullanılmak üzere hazırlanmış veri yapısı [JOIN](../../../sql-reference/statements/select.md#select-join) harekat. +Kullanılmak üzere hazırlanmış veri yapısı [JOIN](../../../sql-reference/statements/select/join.md#select-join) harekat. ## Tablo oluşturma {#creating-a-table} @@ -23,8 +23,8 @@ Ayrıntılı açıklamasına bakın [CREATE TABLE](../../../sql-reference/statem **Motor Parametreleri** -- `join_strictness` – [Katılık katılın](../../../sql-reference/statements/select.md#select-join-strictness). -- `join_type` – [Birleştirme türü](../../../sql-reference/statements/select.md#select-join-types). +- `join_strictness` – [Katılık katılın](../../../sql-reference/statements/select/join.md#select-join-strictness). +- `join_type` – [Birleştirme türü](../../../sql-reference/statements/select/join.md#select-join-types). - `k1[, k2, ...]` – Key columns from the `USING` fık thera: `JOIN` işlemi yapılmamaktadır. Girmek `join_strictness` ve `join_type` tırnak işaretleri olmadan parametreler, örneğin, `Join(ANY, LEFT, col1)`. Onlar eşleşmelidir `JOIN` tablo için kullanılacak işlem. Parametreler eşleşmezse, ClickHouse bir istisna atmaz ve yanlış veri döndürebilir. @@ -79,16 +79,16 @@ SELECT joinGet('id_val_join', 'val', toUInt32(1)) └────────────────────────────────────────────┘ ``` -### Veri seçme Ve Ekleme {#selecting-and-inserting-data} +### Veri seçme ve ekleme {#selecting-and-inserting-data} -Kullanabilirsiniz `INSERT` veri eklemek için sorgular `Join`- motor masaları. Tablo ile oluşturulmuş ise `ANY` katılık, yinelenen anahtarlar için veriler göz ardı edilir. İle… `ALL` katılık, tüm satırlar eklenir. +Kullanabilirsiniz `INSERT` veri eklemek için sorgular `Join`- motor masaları. Tablo ile oluşturulmuş ise `ANY` katılık, yinelenen anahtarlar için veriler göz ardı edilir. İle... `ALL` katılık, tüm satırlar eklenir. Gerçekleştir aemezsiniz `SELECT` doğrudan tablodan sorgulayın. Bunun yerine, aşağıdaki yöntemlerden birini kullanın: - Tabloyu sağ tarafa yerleştirin. `JOIN` yan. - Call the [joinGet](../../../sql-reference/functions/other-functions.md#joinget) tablodan bir sözlükten aynı şekilde veri ayıklamanızı sağlayan işlev. -### Sınırlamalar Ve Ayarlar {#join-limitations-and-settings} +### Sınırlamalar ve Ayarlar {#join-limitations-and-settings} Bir tablo oluştururken aşağıdaki ayarlar uygulanır: @@ -100,11 +100,11 @@ Bir tablo oluştururken aşağıdaki ayarlar uygulanır: Bu `Join`- motor tabloları kullanılamaz `GLOBAL JOIN` harekat. -Bu `Join`- motor kullanımına izin verir [join\_use\_nulls](../../../operations/settings/settings.md#join_use_nulls) ayarı `CREATE TABLE` deyim. Ve [SELECT](../../../sql-reference/statements/select.md) sorgu kullanımına izin verir `join_use_nulls` çok. Eğer farklı varsa `join_use_nulls` ayarlar, tablo birleştirme bir hata alabilirsiniz. Bu katılmak türüne bağlıdır. Kullandığınızda [joinGet](../../../sql-reference/functions/other-functions.md#joinget) fonksiyonu, aynı kullanmak zorunda `join_use_nulls` ayarı `CRATE TABLE` ve `SELECT` deyimler. +Bu `Join`- motor kullanımına izin verir [join\_use\_nulls](../../../operations/settings/settings.md#join_use_nulls) ayarı `CREATE TABLE` deyim. Ve [SELECT](../../../sql-reference/statements/select/index.md) sorgu kullanımına izin verir `join_use_nulls` çok. Eğer farklı varsa `join_use_nulls` ayarlar, tablo birleştirme bir hata alabilirsiniz. Bu katılmak türüne bağlıdır. Kullandığınızda [joinGet](../../../sql-reference/functions/other-functions.md#joinget) fonksiyonu, aynı kullanmak zorunda `join_use_nulls` ayarı `CRATE TABLE` ve `SELECT` deyimler. ## Veri Depolama {#data-storage} -`Join` tablo verileri her zaman RAM’de bulunur. Bir tabloya satır eklerken, sunucu yeniden başlatıldığında geri yüklenebilir, böylece ClickHouse disk üzerindeki dizine veri bloklarını yazar. +`Join` tablo verileri her zaman RAM'de bulunur. Bir tabloya satır eklerken, sunucu yeniden başlatıldığında geri yüklenebilir, böylece ClickHouse disk üzerindeki dizine veri bloklarını yazar. Sunucu yanlış yeniden başlatılırsa, diskteki veri bloğu kaybolabilir veya zarar görebilir. Bu durumda, dosyayı hasarlı verilerle el ile silmeniz gerekebilir. diff --git a/docs/tr/engines/table-engines/special/materializedview.md b/docs/tr/engines/table-engines/special/materializedview.md index 5182395e21c..485b5ae7d14 100644 --- a/docs/tr/engines/table-engines/special/materializedview.md +++ b/docs/tr/engines/table-engines/special/materializedview.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 43 toc_title: MaterializedView --- diff --git a/docs/tr/engines/table-engines/special/memory.md b/docs/tr/engines/table-engines/special/memory.md index d11a52affc1..3c88cade274 100644 --- a/docs/tr/engines/table-engines/special/memory.md +++ b/docs/tr/engines/table-engines/special/memory.md @@ -1,18 +1,18 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 44 -toc_title: Bellek +toc_title: "Haf\u0131za" --- -# Bellek {#memory} +# Hafıza {#memory} Bellek altyapısı verileri RAM, sıkıştırılmamış biçimde depolar. Veri okunduğunda alınan tam olarak aynı biçimde saklanır. Başka bir deyişle, bu tablodan okuma tamamen ücretsizdir. Eşzamanlı veri erişimi senkronize edilir. Kilitler kısa: okuma ve yazma işlemleri birbirini engellemez. Dizinler desteklenmiyor. Okuma paralelleştirilmiştir. -Basit sorgularda maksimum üretkenliğe (10 GB/sn’den fazla) ulaşılır, çünkü diskten okuma, açma veya veri serisini kaldırma yoktur. (Birçok durumda MergeTree motorunun verimliliğinin neredeyse yüksek olduğunu unutmamalıyız.) +Basit sorgularda maksimum üretkenliğe (10 GB/sn'den fazla) ulaşılır, çünkü diskten okuma, açma veya veri serisini kaldırma yoktur. (Birçok durumda MergeTree motorunun verimliliğinin neredeyse yüksek olduğunu unutmamalıyız.) Bir sunucu yeniden başlatılırken, veri tablodan kaybolur ve tablo boş olur. -Normalde, bu tablo motorunu kullanmak haklı değildir. Bununla birlikte, testler ve nispeten az sayıda satırda (yaklaşık 100.000.000’a kadar) maksimum hızın gerekli olduğu görevler için kullanılabilir. +Normalde, bu tablo motorunu kullanmak haklı değildir. Bununla birlikte, testler ve nispeten az sayıda satırda (yaklaşık 100.000.000'a kadar) maksimum hızın gerekli olduğu görevler için kullanılabilir. Bellek motoru, harici sorgu verilerine sahip geçici tablolar için sistem tarafından kullanılır (bkz. “External data for processing a query”) ve GLOBAL In uygulanması için (bkz. “IN operators”). diff --git a/docs/tr/engines/table-engines/special/merge.md b/docs/tr/engines/table-engines/special/merge.md index 811dd7f1770..a707954c13f 100644 --- a/docs/tr/engines/table-engines/special/merge.md +++ b/docs/tr/engines/table-engines/special/merge.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 36 toc_title: "Birle\u015Ftirmek" --- diff --git a/docs/tr/engines/table-engines/special/null.md b/docs/tr/engines/table-engines/special/null.md index 19d518f415f..da7e38eddf0 100644 --- a/docs/tr/engines/table-engines/special/null.md +++ b/docs/tr/engines/table-engines/special/null.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 38 toc_title: "Bo\u015F" --- diff --git a/docs/tr/engines/table-engines/special/set.md b/docs/tr/engines/table-engines/special/set.md index 65c88beb359..151f98e5c54 100644 --- a/docs/tr/engines/table-engines/special/set.md +++ b/docs/tr/engines/table-engines/special/set.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 39 toc_title: Koymak --- @@ -12,7 +12,7 @@ Her zaman RAM olan bir veri kümesi. In operatörünün sağ tarafında kullanı Tabloya veri eklemek için INSERT kullanabilirsiniz. Veri kümesine yeni öğeler eklenirken, yinelenenler göz ardı edilir. Ancak tablodan seçim yapamazsınız. Verileri almak için tek yol, IN operatörünün sağ yarısında kullanmaktır. -Veri her zaman RAM yer almaktadır. INSERT için, eklenen veri blokları da diskteki tabloların dizinine yazılır. Sunucuyu başlatırken, bu veriler RAM’e yüklenir. Başka bir deyişle, yeniden başlattıktan sonra veriler yerinde kalır. +Veri her zaman RAM yer almaktadır. INSERT için, eklenen veri blokları da diskteki tabloların dizinine yazılır. Sunucuyu başlatırken, bu veriler RAM'e yüklenir. Başka bir deyişle, yeniden başlattıktan sonra veriler yerinde kalır. Kaba bir sunucu yeniden başlatma için diskteki veri bloğu kaybolabilir veya zarar görebilir. İkinci durumda, dosyayı hasarlı verilerle el ile silmeniz gerekebilir. diff --git a/docs/tr/engines/table-engines/special/url.md b/docs/tr/engines/table-engines/special/url.md index 3534f27b74b..2632f1e5046 100644 --- a/docs/tr/engines/table-engines/special/url.md +++ b/docs/tr/engines/table-engines/special/url.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 41 toc_title: URL --- @@ -10,9 +10,9 @@ toc_title: URL Uzak bir HTTP/HTTPS sunucusundaki verileri yönetir. Bu motor benzer to the [Dosya](file.md) motor. -## ClickHouse Sunucusunda Motoru Kullanma {#using-the-engine-in-the-clickhouse-server} +## ClickHouse sunucusunda motoru kullanma {#using-the-engine-in-the-clickhouse-server} -Bu `format` Clickhouse’un kullanabileceği bir tane olmalı +Bu `format` Clickhouse'un kullanabileceği bir tane olmalı `SELECT` sorgular ve gerekirse `INSERTs`. Desteklenen formatların tam listesi için bkz. [Biçimliler](../../../interfaces/formats.md#formats). diff --git a/docs/tr/engines/table-engines/special/view.md b/docs/tr/engines/table-engines/special/view.md index e3b46a7b926..294477824a3 100644 --- a/docs/tr/engines/table-engines/special/view.md +++ b/docs/tr/engines/table-engines/special/view.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 42 toc_title: "G\xF6r\xFCn\xFCm" --- diff --git a/docs/tr/faq/general.md b/docs/tr/faq/general.md index e49471bfb9b..f17baee24e9 100644 --- a/docs/tr/faq/general.md +++ b/docs/tr/faq/general.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 78 toc_title: Genel Sorular --- @@ -9,13 +9,13 @@ toc_title: Genel Sorular ## Neden MapReduce Gibi Bir Şey Kullanmıyorsun? {#why-not-use-something-like-mapreduce} -Mapreduce gibi sistemlere, azaltma işleminin dağıtılmış sıralamaya dayandığı dağıtılmış bilgi işlem sistemleri olarak başvurabiliriz. Bu sınıftaki en yaygın açık kaynak çözümü [Apache Hadoop](http://hadoop.apache.org). Yandex, şirket içi çözümünü, YT’Yİ kullanıyor. +Mapreduce gibi sistemlere, azaltma işleminin dağıtılmış sıralamaya dayandığı dağıtılmış bilgi işlem sistemleri olarak başvurabiliriz. Bu sınıftaki en yaygın açık kaynak çözümü [Apache Hadoop](http://hadoop.apache.org). Yandex, şirket içi çözümünü, YT'Yİ kullanıyor. -Bu sistemler, yüksek gecikme süreleri nedeniyle çevrimiçi sorgular için uygun değildir. Başka bir deyişle, bir web arayüzü için arka uç olarak kullanılamazlar. Bu tür sistemler gerçek zamanlı veri güncellemeleri için yararlı değildir. Dağıtılmış sıralama, işlemin sonucu ve tüm ara sonuçlar (varsa) tek bir sunucunun RAM’İNDE bulunuyorsa, genellikle çevrimiçi sorgular için geçerli olan işlemleri azaltmanın en iyi yolu değildir. Böyle bir durumda, bir karma tablo azaltma işlemlerini gerçekleştirmek için en uygun yoldur. Harita azaltma görevlerini optimize etmek için ortak bir yaklaşım, RAM’de bir karma tablo kullanarak ön toplama (kısmi azaltma) ’ dir. Kullanıcı bu optimizasyonu manuel olarak gerçekleştirir. Dağıtılmış sıralama, basit harita azaltma görevlerini çalıştırırken düşük performansın ana nedenlerinden biridir. +Bu sistemler, yüksek gecikme süreleri nedeniyle çevrimiçi sorgular için uygun değildir. Başka bir deyişle, bir web arayüzü için arka uç olarak kullanılamazlar. Bu tür sistemler gerçek zamanlı veri güncellemeleri için yararlı değildir. Dağıtılmış sıralama, işlemin sonucu ve tüm ara sonuçlar (varsa) tek bir sunucunun RAM'İNDE bulunuyorsa, genellikle çevrimiçi sorgular için geçerli olan işlemleri azaltmanın en iyi yolu değildir. Böyle bir durumda, bir karma tablo azaltma işlemlerini gerçekleştirmek için en uygun yoldur. Harita azaltma görevlerini optimize etmek için ortak bir yaklaşım, RAM'de bir karma tablo kullanarak ön toplama (kısmi azaltma) ' dir. Kullanıcı bu optimizasyonu manuel olarak gerçekleştirir. Dağıtılmış sıralama, basit harita azaltma görevlerini çalıştırırken düşük performansın ana nedenlerinden biridir. -Çoğu MapReduce uygulaması, bir kümede rasgele kod çalıştırmanıza izin verir. Ancak bildirimsel bir sorgu dili, deneyleri hızlı bir şekilde çalıştırmak için OLAP için daha uygundur. Örneğin, Hadoop kovanı ve domuz vardır. Ayrıca Spark için Cloudera Impala veya Shark’ı (modası geçmiş) ve Spark SQL, Presto ve Apache Drill’i de düşünün. Bu tür görevleri çalıştırırken performans, özel sistemlere kıyasla oldukça düşük bir seviyededir, ancak nispeten yüksek gecikme, bu sistemleri bir web arayüzü için arka uç olarak kullanmayı gerçekçi kılmaktadır. +Çoğu MapReduce uygulaması, bir kümede rasgele kod çalıştırmanıza izin verir. Ancak bildirimsel bir sorgu dili, deneyleri hızlı bir şekilde çalıştırmak için OLAP için daha uygundur. Örneğin, Hadoop kovanı ve domuz vardır. Ayrıca Spark için Cloudera Impala veya Shark'ı (modası geçmiş) ve Spark SQL, Presto ve Apache Drill'i de düşünün. Bu tür görevleri çalıştırırken performans, özel sistemlere kıyasla oldukça düşük bir seviyededir, ancak nispeten yüksek gecikme, bu sistemleri bir web arayüzü için arka uç olarak kullanmayı gerçekçi kılmaktadır. -## Oracle aracılığıyla ODBC kullanırken Kodlamalarla Ilgili Bir Sorunum Varsa Ne Olur? {#oracle-odbc-encodings} +## Oracle aracılığıyla ODBC kullanırken Kodlamalarla ilgili bir sorunum varsa ne olur? {#oracle-odbc-encodings} Oracle ODBC sürücüsü aracılığıyla dış sözlükler kaynağı olarak kullanırsanız, doğru değeri ayarlamanız gerekir. `NLS_LANG` ortam değişkeni `/etc/default/clickhouse`. Daha fazla bilgi için, bkz: [Oracle NLS\_LANG SSS](https://www.oracle.com/technetwork/products/globalization/nls-lang-099431.html). @@ -25,11 +25,11 @@ Oracle ODBC sürücüsü aracılığıyla dış sözlükler kaynağı olarak kul NLS_LANG=RUSSIAN_RUSSIA.UTF8 ``` -## Clickhouse’dan Bir Dosyaya Verileri nasıl dışa aktarırım? {#how-to-export-to-file} +## Clickhouse'dan bir dosyaya verileri nasıl dışa aktarırım? {#how-to-export-to-file} -### INTO OUTFİLE Yan tümcesini Kullanma {#using-into-outfile-clause} +### INTO OUTFİLE yan tümcesini kullanma {#using-into-outfile-clause} -Add an [INTO OUTFILE](../sql-reference/statements/select.md#into-outfile-clause) sorgunuza yan tümce. +Add an [INTO OUTFILE](../sql-reference/statements/select/into-outfile.md#into-outfile-clause) sorgunuza yan tümce. Mesela: @@ -37,7 +37,7 @@ Mesela: SELECT * FROM table INTO OUTFILE 'file' ``` -Varsayılan olarak, ClickHouse kullanır [TabSeparated](../interfaces/formats.md#tabseparated) çıktı verileri için Biçim. Seçmek için [Veri formatı](../interfaces/formats.md), use the [FORMAT CLA clauseuse](../sql-reference/statements/select.md#format-clause). +Varsayılan olarak, ClickHouse kullanır [TabSeparated](../interfaces/formats.md#tabseparated) çıktı verileri için Biçim. Seçmek için [Veri formatı](../interfaces/formats.md), use the [FORMAT CLA clauseuse](../sql-reference/statements/select/format.md#format-clause). Mesela: @@ -45,7 +45,7 @@ Mesela: SELECT * FROM table INTO OUTFILE 'file' FORMAT CSV ``` -### Dosya altyapısı Tablosu Kullanma {#using-a-file-engine-table} +### Dosya altyapısı tablosu kullanma {#using-a-file-engine-table} Görmek [Dosya](../engines/table-engines/special/file.md). diff --git a/docs/tr/faq/index.md b/docs/tr/faq/index.md index 591011fb66d..a44dbb31e89 100644 --- a/docs/tr/faq/index.md +++ b/docs/tr/faq/index.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_folder_title: F.A.Q. toc_priority: 76 --- diff --git a/docs/tr/getting-started/example-datasets/amplab-benchmark.md b/docs/tr/getting-started/example-datasets/amplab-benchmark.md index e6f95df68b8..affeb465c84 100644 --- a/docs/tr/getting-started/example-datasets/amplab-benchmark.md +++ b/docs/tr/getting-started/example-datasets/amplab-benchmark.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 17 toc_title: "AMPLab B\xFCy\xFCk Veri Benchmark" --- diff --git a/docs/tr/getting-started/example-datasets/criteo.md b/docs/tr/getting-started/example-datasets/criteo.md index f9fc2975e34..3351cf3e7fa 100644 --- a/docs/tr/getting-started/example-datasets/criteo.md +++ b/docs/tr/getting-started/example-datasets/criteo.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 19 toc_title: "Criteo'dan Terabyte t\u0131klama g\xFCnl\xFCkleri" --- -# Criteo’dan tıklama günlüklerinin Terabayt {#terabyte-of-click-logs-from-criteo} +# Criteo'dan tıklama günlüklerinin terabayt {#terabyte-of-click-logs-from-criteo} Verileri indirin http://labs.criteo.com/downloads/download-terabyte-click-logs/ diff --git a/docs/tr/getting-started/example-datasets/index.md b/docs/tr/getting-started/example-datasets/index.md index 950b7dc0d7f..b31b0faa6fd 100644 --- a/docs/tr/getting-started/example-datasets/index.md +++ b/docs/tr/getting-started/example-datasets/index.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_folder_title: "\xD6rnek Veri K\xFCmeleri" toc_priority: 12 toc_title: "Giri\u015F" @@ -8,13 +8,13 @@ toc_title: "Giri\u015F" # Örnek Veri Kümeleri {#example-datasets} -Bu bölümde, örnek veri kümelerinin nasıl elde edileceği ve bunları Clickhouse’a nasıl içe aktarılacağı açıklanmaktadır. +Bu bölümde, örnek veri kümelerinin nasıl elde edileceği ve bunları Clickhouse'a nasıl içe aktarılacağı açıklanmaktadır. Bazı veri kümeleri için örnek sorgular da mevcuttur. - [Anonim Yandex.Metrica Veri Kümesi](metrica.md) - [Yıldız Şema Ben Benchmarkch Benchmarkmark](star-schema.md) - [WikiStat](wikistat.md) -- [Criteo’dan tıklama günlüklerinin terabayt](criteo.md) +- [Criteo'dan tıklama günlüklerinin terabayt](criteo.md) - [AMPLab Büyük Veri Benchmark](amplab-benchmark.md) - [New York Taksi Verileri](nyc-taxi.md) - [OnTime](ontime.md) diff --git a/docs/tr/getting-started/example-datasets/metrica.md b/docs/tr/getting-started/example-datasets/metrica.md index f9af62d7e0c..22f96f6761a 100644 --- a/docs/tr/getting-started/example-datasets/metrica.md +++ b/docs/tr/getting-started/example-datasets/metrica.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 -toc_priority: 21 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_priority: 14 toc_title: "\xDCye.Metrica Verileri" --- @@ -11,7 +11,7 @@ Veri kümesi, isabetlerle ilgili anonimleştirilmiş verileri içeren iki tablod Veri kümesi iki tablodan oluşur, bunlardan biri sıkıştırılmış olarak indirilebilir `tsv.xz` dosya veya hazırlanmış bölümler olarak. Buna ek olarak, genişletilmiş bir sürümü `hits` 100 milyon satır içeren tablo TSV olarak mevcuttur https://clickhouse-datasets.s3.yandex.net/hits/tsv/hits\_100m\_obfuscated\_v1.tsv.xz ve hazırlanan bölümler olarak https://clickhouse-datasets.s3.yandex.net/hits/partitions/hits\_100m\_obfuscated\_v1.tar.xz. -## Hazırlanan bölümlerden Tablolar Elde Etme {#obtaining-tables-from-prepared-partitions} +## Hazırlanan bölümlerden tablolar elde etme {#obtaining-tables-from-prepared-partitions} İndirme ve ithalat tablo hits: @@ -33,7 +33,7 @@ sudo service clickhouse-server restart clickhouse-client --query "SELECT COUNT(*) FROM datasets.visits_v1" ``` -## Sıkıştırılmış TSV dosyasından Tablo Alma {#obtaining-tables-from-compressed-tsv-file} +## Sıkıştırılmış TSV dosyasından Tablo alma {#obtaining-tables-from-compressed-tsv-file} Sıkıştırılmış TSV dosyasından indir ve İçe Aktar: diff --git a/docs/tr/getting-started/example-datasets/nyc-taxi.md b/docs/tr/getting-started/example-datasets/nyc-taxi.md index bf3d62d51a6..70569d47542 100644 --- a/docs/tr/getting-started/example-datasets/nyc-taxi.md +++ b/docs/tr/getting-started/example-datasets/nyc-taxi.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 16 toc_title: New York Taksi Verileri --- @@ -12,7 +12,7 @@ Bu veri kümesi iki şekilde elde edilebilir: - ham verilerden içe aktarma - hazırlanan bölüm downloadlerin indir downloadilmesi -## Ham Veri nasıl alınır {#how-to-import-the-raw-data} +## Ham veri nasıl alınır {#how-to-import-the-raw-data} Bkz. https://github.com/toddwschneider/nyc-taxi-data ve http://tech.marksblogg.com/billion-nyc-taxi-rides-redshift.html bir veri kümesinin açıklaması ve indirme talimatları için. @@ -28,11 +28,11 @@ mv data/yellow_tripdata_2010-02.csv_ data/yellow_tripdata_2010-02.csv mv data/yellow_tripdata_2010-03.csv_ data/yellow_tripdata_2010-03.csv ``` -Daha sonra veriler Postgresql’de önceden işlenmelidir. Bu, çokgenlerdeki noktaların seçimlerini oluşturacaktır (Haritadaki noktaları New York şehrinin ilçeleriyle eşleştirmek için) ve tüm verileri bir birleştirme kullanarak tek bir denormalize düz tabloda birleştirecektir. Bunu yapmak için Postgresql’i Postgıs desteği ile yüklemeniz gerekir. +Daha sonra veriler Postgresql'de önceden işlenmelidir. Bu, çokgenlerdeki noktaların seçimlerini oluşturacaktır (Haritadaki noktaları New York şehrinin ilçeleriyle eşleştirmek için) ve tüm verileri bir birleştirme kullanarak tek bir denormalize düz tabloda birleştirecektir. Bunu yapmak için Postgresql'i Postgıs desteği ile yüklemeniz gerekir. Çalışırken dikkatli olun `initialize_database.sh` ve tüm tabloların doğru şekilde oluşturulduğunu manuel olarak yeniden kontrol edin. -Postgresql’deki her ayın verilerini işlemek yaklaşık 20-30 dakika sürer, toplam yaklaşık 48 saat sürer. +Postgresql'deki her ayın verilerini işlemek yaklaşık 20-30 dakika sürer, toplam yaklaşık 48 saat sürer. İndirilen satır sayısını aşağıdaki gibi kontrol edebilirsiniz: @@ -47,7 +47,7 @@ real 7m9.164s (Bu, Mark Litwintschik tarafından bir dizi blog gönderisinde bildirilen 1.1 milyar satırdan biraz daha fazladır .) -Postgresql’deki veriler 370 GB alan kullanıyor. +Postgresql'deki veriler 370 GB alan kullanıyor. PostgreSQL veri verme: @@ -124,7 +124,7 @@ COPY Veri anlık görüntüsü saniyede yaklaşık 50 MB hızında oluşturulur. Anlık görüntü oluştururken, PostgreSQL diskten saniyede yaklaşık 28 MB hızında okur. Bu yaklaşık 5 saat sürer. Elde edilen TSV dosyası 590612904969 bayttır. -Clickhouse’da geçici bir tablo oluşturma: +Clickhouse'da geçici bir tablo oluşturma: ``` sql CREATE TABLE trips @@ -195,7 +195,7 @@ Veri 112-140 Mb/saniye hızında okunur. Bir akışta bir günlük türü tablosuna veri yükleme 76 dakika sürdü. Bu tablodaki veriler 142 GB kullanır. -(Verileri doğrudan Postgres’ten içe aktarmak da mümkündür `COPY ... TO PROGRAM`.) +(Verileri doğrudan Postgres'ten içe aktarmak da mümkündür `COPY ... TO PROGRAM`.) Unfortunately, all the fields associated with the weather (precipitation…average\_wind\_speed) were filled with NULL. Because of this, we will remove them from the final data set. @@ -282,7 +282,7 @@ SELECT formatReadableSize(sum(bytes)) FROM system.parts WHERE table = 'trips_mer Diğer şeylerin yanı sıra, MERGETREE üzerinde en iyi duruma getirme sorgusunu çalıştırabilirsiniz. Ama her şey onsuz iyi olacak çünkü gerekli değildir. -## Hazırlanan Bölüm Downloadlerin Indir Downloadilmesi {#download-of-prepared-partitions} +## Hazırlanan Bölüm downloadlerin indir downloadilmesi {#download-of-prepared-partitions} ``` bash $ curl -O https://clickhouse-datasets.s3.yandex.net/trips_mergetree/partitions/trips_mergetree.tar @@ -295,7 +295,7 @@ $ clickhouse-client --query "select count(*) from datasets.trips_mergetree" !!! info "Bilgin" Aşağıda açıklanan sorguları çalıştıracaksanız, tam tablo adını kullanmanız gerekir, `datasets.trips_mergetree`. -## Tek Server Ile Ilgili sonuçlar {#results-on-single-server} +## Tek Server ile ilgili sonuçlar {#results-on-single-server} Q1: @@ -377,7 +377,7 @@ Q3: 0.051 sn. Q4: 0.072 sn. Bu durumda, sorgu işleme süresi her şeyden önce ağ gecikmesi ile belirlenir. -Finlandiya’daki bir Yandex veri merkezinde bulunan ve Rusya’daki bir kümede bulunan ve yaklaşık 20 ms gecikme süresi ekleyen bir istemci kullanarak sorgular çalıştırdık. +Finlandiya'daki bir Yandex veri merkezinde bulunan ve Rusya'daki bir kümede bulunan ve yaklaşık 20 ms gecikme süresi ekleyen bir istemci kullanarak sorgular çalıştırdık. ## Özet {#summary} diff --git a/docs/tr/getting-started/example-datasets/ontime.md b/docs/tr/getting-started/example-datasets/ontime.md index ff489ee7e5c..f1d477dbc6e 100644 --- a/docs/tr/getting-started/example-datasets/ontime.md +++ b/docs/tr/getting-started/example-datasets/ontime.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 15 toc_title: OnTime --- @@ -12,7 +12,7 @@ Bu veri kümesi iki şekilde elde edilebilir: - ham verilerden içe aktarma - hazırlanan bölüm downloadlerin indir downloadilmesi -## Ham Verilerden İçe Aktarma {#import-from-raw-data} +## Ham verilerden içe aktarma {#import-from-raw-data} Veri indirme: @@ -153,7 +153,7 @@ Veri yükleme: $ for i in *.zip; do echo $i; unzip -cq $i '*.csv' | sed 's/\.00//g' | clickhouse-client --host=example-perftest01j --query="INSERT INTO ontime FORMAT CSVWithNames"; done ``` -## Hazırlanan Bölüm Downloadlerin Indir Downloadilmesi {#download-of-prepared-partitions} +## Hazırlanan Bölüm downloadlerin indir downloadilmesi {#download-of-prepared-partitions} ``` bash $ curl -O https://clickhouse-datasets.s3.yandex.net/ontime/partitions/ontime.tar diff --git a/docs/tr/getting-started/example-datasets/star-schema.md b/docs/tr/getting-started/example-datasets/star-schema.md index b2de52ff5d6..a82bcd569f9 100644 --- a/docs/tr/getting-started/example-datasets/star-schema.md +++ b/docs/tr/getting-started/example-datasets/star-schema.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 20 toc_title: "Y\u0131ld\u0131z \u015Eema Ben Benchmarkch Benchmarkmark" --- @@ -28,7 +28,7 @@ $ ./dbgen -s 1000 -T s $ ./dbgen -s 1000 -T d ``` -Clickhouse’da tablolar oluşturma: +Clickhouse'da tablolar oluşturma: ``` sql CREATE TABLE customer diff --git a/docs/tr/getting-started/example-datasets/wikistat.md b/docs/tr/getting-started/example-datasets/wikistat.md index 0fc24dd5bb1..f171502afe7 100644 --- a/docs/tr/getting-started/example-datasets/wikistat.md +++ b/docs/tr/getting-started/example-datasets/wikistat.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 18 toc_title: WikiStat --- diff --git a/docs/tr/getting-started/index.md b/docs/tr/getting-started/index.md index c03665813ad..7e2f649040b 100644 --- a/docs/tr/getting-started/index.md +++ b/docs/tr/getting-started/index.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_folder_title: "Ba\u015Flarken" toc_hidden: true toc_priority: 8 diff --git a/docs/tr/getting-started/install.md b/docs/tr/getting-started/install.md index 2d02cb0b1b1..3bf319430bd 100644 --- a/docs/tr/getting-started/install.md +++ b/docs/tr/getting-started/install.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 11 toc_title: Kurulum --- @@ -11,36 +11,27 @@ toc_title: Kurulum ClickHouse, x86\_64, AArch64 veya PowerPC64LE CPU mimarisine sahip herhangi bir Linux, FreeBSD veya Mac OS X üzerinde çalışabilir. -Resmi önceden oluşturulmuş ikili dosyalar genellikle x86\_64 ve kaldıraç sse 4.2 komut seti için derlenir, bu nedenle destekleyen CPU’nun aksi belirtilmedikçe ek bir sistem gereksinimi haline gelir. Geçerli CPU’nun sse 4.2 desteği olup olmadığını kontrol etmek için komut: +Resmi önceden oluşturulmuş ikili dosyalar genellikle x86\_64 ve kaldıraç sse 4.2 komut seti için derlenir, bu nedenle destekleyen CPU'nun aksi belirtilmedikçe ek bir sistem gereksinimi haline gelir. Geçerli CPU'nun sse 4.2 desteği olup olmadığını kontrol etmek için komut: ``` bash $ grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not supported" ``` -SSE 4.2’yi desteklemeyen veya AArch64 veya PowerPC64LE mimarisine sahip işlemcilerde Clickhouse’u çalıştırmak için şunları yapmalısınız [kaynaklardan ClickHouse oluşturun](#from-sources) uygun yapılandırma ayarlamaları ile. +SSE 4.2'yi desteklemeyen veya AArch64 veya PowerPC64LE mimarisine sahip işlemcilerde Clickhouse'u çalıştırmak için şunları yapmalısınız [kaynaklardan ClickHouse oluşturun](#from-sources) uygun yapılandırma ayarlamaları ile. ## Mevcut Kurulum Seçenekleri {#available-installation-options} -### DEB Paket Fromlerinden {#install-from-deb-packages} +### DEB paket fromlerinden {#install-from-deb-packages} -Resmi önceden derlenmiş kullanılması tavsiye edilir `deb` Debian veya Ubuntu için paketler. +Resmi önceden derlenmiş kullanılması tavsiye edilir `deb` Debian veya Ubuntu için paketler. Paketleri yüklemek için bu komutları çalıştırın: -Resmi paketleri yüklemek için Yandex deposunu ekleyin `/etc/apt/sources.list` veya ayrı bir `/etc/apt/sources.list.d/clickhouse.list` Dosya: - - deb https://repo.clickhouse.tech/deb/stable/ main/ +``` bash +{% include 'install/deb.sh' %} +``` En son sürümü kullanmak istiyorsanız, değiştirin `stable` ile `testing` (bu, test ortamlarınız için önerilir). -Sonra paketleri yüklemek için bu komutları çalıştırın: - -``` bash -sudo apt-get install apt-transport-https ca-certificates dirmngr # optional -sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E0C56BD4 # optional -sudo apt-get update -sudo apt-get install clickhouse-client clickhouse-server -``` - -Paketleri buradan manuel olarak indirebilir ve kurabilirsiniz: https://repo.yandex.ru/clickhouse/deb/stable/main/. +Ayrıca paketleri manuel olarak indirebilir ve yükleyebilirsiniz [burada](https://repo.clickhouse.tech/deb/stable/main/). #### Paketler {#packages} @@ -49,7 +40,7 @@ Paketleri buradan manuel olarak indirebilir ve kurabilirsiniz: https://repo.yand - `clickhouse-client` — Creates a symbolic link for `clickhouse-client` ve diğer istemci ile ilgili araçlar. ve istemci yapılandırma dosyalarını yükler. - `clickhouse-common-static-dbg` — Installs ClickHouse compiled binary files with debug info. -### RPM Paket Fromlerinden {#from-rpm-packages} +### RPM paket fromlerinden {#from-rpm-packages} Resmi önceden derlenmiş kullanılması tavsiye edilir `rpm` CentOS, RedHat ve diğer tüm rpm tabanlı Linux dağıtımları için paketler. @@ -69,7 +60,7 @@ Sonra paketleri yüklemek için bu komutları çalıştırın: sudo yum install clickhouse-server clickhouse-client ``` -Paketleri buradan manuel olarak indirebilir ve kurabilirsiniz: https://repo.clickhouse.teknoloji / rpm / kararlı / x86\_64. +Ayrıca paketleri manuel olarak indirebilir ve yükleyebilirsiniz [burada](https://repo.clickhouse.tech/rpm/stable/x86_64). ### Tgz Arşivlerinden {#from-tgz-archives} @@ -107,9 +98,9 @@ Docker içinde ClickHouse çalıştırmak için kılavuzu izleyin [Docker Hub](h ### Kaynaklardan {#from-sources} -Clickhouse’u el ile derlemek için aşağıdaki talimatları izleyin [Linux](../development/build.md) veya [Mac OS X](../development/build-osx.md). +Clickhouse'u el ile derlemek için aşağıdaki talimatları izleyin [Linux](../development/build.md) veya [Mac OS X](../development/build-osx.md). -Paketleri derleyebilir ve yükleyebilir veya paketleri yüklemeden programları kullanabilirsiniz. Ayrıca elle inşa ederek SSE 4.2 gereksinimini devre dışı bırakabilir veya AArch64 CPU’lar için oluşturabilirsiniz. +Paketleri derleyebilir ve yükleyebilir veya paketleri yüklemeden programları kullanabilirsiniz. Ayrıca elle inşa ederek SSE 4.2 gereksinimini devre dışı bırakabilir veya AArch64 CPU'lar için oluşturabilirsiniz. Client: programs/clickhouse-client Server: programs/clickhouse-server @@ -119,9 +110,9 @@ Bir veri ve meta veri klasörleri oluşturmanız gerekir ve `chown` onları iste /opt/clickhouse/data/default/ /opt/clickhouse/metadata/default/ -Gentoo üzerinde, sadece kullanabilirsiniz `emerge clickhouse` Clickhouse’u kaynaklardan yüklemek için. +Gentoo üzerinde, sadece kullanabilirsiniz `emerge clickhouse` Clickhouse'u kaynaklardan yüklemek için. -## Başlamak {#launch} +## Başlatmak {#launch} Sunucuyu bir daemon olarak başlatmak için çalıştırın: @@ -158,7 +149,7 @@ Sunucuyu başlattıktan sonra, ona bağlanmak için komut satırı istemcisini k $ clickhouse-client ``` -Varsayılan olarak, bağlanır `localhost:9000` kullanıcı adına `default` şifre olmadan. Kullanarak uzak bir sunucuya bağlanmak için de kullanılabilir `--host` değişken. +Varsayılan olarak, bağlanır `localhost:9000` kullanıcı adına `default` şifre olmadan. Kullanarak uzak bir sunucuya bağlanmak için de kullanılabilir `--host` tartışma. Terminal UTF-8 kodlamasını kullanmalıdır. Daha fazla bilgi için bölüme bakın [“Command-line client”](../interfaces/cli.md). diff --git a/docs/tr/getting-started/playground.md b/docs/tr/getting-started/playground.md index f1eabbb5e86..5d23ac34b71 100644 --- a/docs/tr/getting-started/playground.md +++ b/docs/tr/getting-started/playground.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 14 toc_title: "Bah\xE7e" --- @@ -31,7 +31,7 @@ Bahçesi arka uç herhangi bir ek sunucu tarafı uygulaması olmadan sadece bir ClickHouse HTTPS bitiş noktası da oyun alanının bir parçası olarak kullanılabilir. Herhangi bir HTTP istemcisi kullanarak oyun alanına sorgu yapabilirsiniz, örneğin [kıvrılma](https://curl.haxx.se) veya [wget](https://www.gnu.org/software/wget/), veya kullanarak bir bağlantı kurmak [JDBC](../interfaces/jdbc.md) veya [ODBC](../interfaces/odbc.md) sürücüler. -Clickhouse’u destekleyen yazılım ürünleri hakkında daha fazla bilgi mevcuttur [burada](../interfaces/index.md). +Clickhouse'u destekleyen yazılım ürünleri hakkında daha fazla bilgi mevcuttur [burada](../interfaces/index.md). | Parametre | Değer | |:----------|:----------------------------------------| diff --git a/docs/tr/getting-started/tutorial.md b/docs/tr/getting-started/tutorial.md index 2c65f29268b..5cca914fb35 100644 --- a/docs/tr/getting-started/tutorial.md +++ b/docs/tr/getting-started/tutorial.md @@ -1,30 +1,24 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 12 toc_title: "\xD6\u011Fretici" --- # ClickHouse Eğitimi {#clickhouse-tutorial} -## Bu Öğreticiden Ne Beklenir? {#what-to-expect-from-this-tutorial} +## Bu Öğreticiden ne beklenir? {#what-to-expect-from-this-tutorial} Bu öğreticiden geçerek, basit bir ClickHouse kümesinin nasıl kurulacağını öğreneceksiniz. Küçük ama hataya dayanıklı ve ölçeklenebilir olacak. Ardından, verilerle doldurmak ve bazı demo sorguları yürütmek için örnek veri kümelerinden birini kullanacağız. ## Tek Düğüm Kurulumu {#single-node-setup} -Dağıtılmış bir ortamın karmaşıklığını ertelemek için, Clickhouse’u tek bir sunucu veya sanal makinede dağıtmaya başlayacağız. ClickHouse genellikle [deb](index.md#install-from-deb-packages) veya [rpm](index.md#from-rpm-packages) paketler, ama var [alternatifler](index.md#from-docker-image) onları desteklemeyen işletim sistemleri için. +Dağıtılmış bir ortamın karmaşıklığını ertelemek için, Clickhouse'u tek bir sunucu veya sanal makinede dağıtmaya başlayacağız. ClickHouse genellikle [deb](install.md#install-from-deb-packages) veya [rpm](install.md#from-rpm-packages) paketler, ama var [alternatifler](install.md#from-docker-image) onları desteklemeyen işletim sistemleri için. Örneğin, seçtiğiniz `deb` paketler ve yürütülen: ``` bash -sudo apt-get install apt-transport-https ca-certificates dirmngr -sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E0C56BD4 - -echo "deb https://repo.clickhouse.tech/deb/stable/ main/" | sudo tee /etc/apt/sources.list.d/clickhouse.list -sudo apt-get update - -sudo apt-get install -y clickhouse-server clickhouse-client +{% include 'install/deb.sh' %} ``` Yüklü olan paketlerde ne var: @@ -54,6 +48,7 @@ Bir kez `clickhouse-server` yukarı ve çalışıyor, biz kullanabilirsiniz `cli
Clickhouse-client için hızlı ipuçları + İnteraktif mod: ``` bash @@ -87,9 +82,9 @@ clickhouse-client --query='INSERT INTO table FORMAT TabSeparated' < data.tsv ## Örnek Veri Kümesini İçe Aktar {#import-sample-dataset} -Şimdi ClickHouse sunucumuzu bazı örnek verilerle doldurmanın zamanı geldi. Bu eğitimde, yandex’in anonim verilerini kullanacağız.Metrica, açık kaynak olmadan önce Clickhouse’u üretim yolunda çalıştıran ilk hizmet (daha fazlası [tarih bölümü](../introduction/history.md)). Var [Yandex’i içe aktarmanın birden fazla yolu.Metrica veri kümesi](example-datasets/metrica.md) ve öğretici uğruna, en gerçekçi olanı ile gideceğiz. +Şimdi ClickHouse sunucumuzu bazı örnek verilerle doldurmanın zamanı geldi. Bu eğitimde, yandex'in anonim verilerini kullanacağız.Metrica, açık kaynak olmadan önce Clickhouse'u üretim yolunda çalıştıran ilk hizmet (daha fazlası [tarih bölümü](../introduction/history.md)). Var [Yandex'i içe aktarmanın birden fazla yolu.Metrica veri kümesi](example-datasets/metrica.md) ve öğretici uğruna, en gerçekçi olanı ile gideceğiz. -### Tablo Verilerini Indirin Ve ayıklayın {#download-and-extract-table-data} +### Tablo verilerini indirin ve ayıklayın {#download-and-extract-table-data} ``` bash curl https://clickhouse-datasets.s3.yandex.net/hits/tsv/hits_v1.tsv.xz | unxz --threads=`nproc` > hits_v1.tsv @@ -466,7 +461,7 @@ Gördüğümüz gibi, `hits_v1` kullanır [temel MergeTree motoru](../engines/ta ### Verileri İçe Aktar {#import-data} -Clickhouse’a veri aktarımı yapılır [INSERT INTO](../sql-reference/statements/insert-into.md) diğer birçok SQL veritabanlarında olduğu gibi sorgu. Bununla birlikte, veriler genellikle [desteklenen seri hale getirme biçimleri](../interfaces/formats.md) yerine `VALUES` fıkra clausesı (ayrıca desteklenmektedir). +Clickhouse'a veri aktarımı yapılır [INSERT INTO](../sql-reference/statements/insert-into.md) diğer birçok SQL veritabanlarında olduğu gibi sorgu. Bununla birlikte, veriler genellikle [desteklenen seri hale getirme biçimleri](../interfaces/formats.md) yerine `VALUES` fıkra clausesı (ayrıca desteklenmektedir). Onları almak için ne kadar daha önce indirdiğimiz dosyaları sekme ayrılmış biçimde, yani burada konsol istemci ile : @@ -486,7 +481,7 @@ FORMAT TSV max_insert_block_size 1048576 0 "The maximum block size for insertion, if we control the creation of blocks for insertion." ``` -İsteğe bağlı olarak şunları yapabilirsiniz [OPTIMIZE](../sql-reference/statements/misc.md#misc_operations-optimize) ithalattan sonra tablolar. MergeTree-family’den bir motorla yapılandırılmış tablolar, veri depolamayı en iyi duruma getirmek (veya en azından mantıklı olup olmadığını kontrol etmek) için her zaman arka planda veri parçalarının birleştirilmesini sağlar. Bu sorgular, tablo motorunu bir süre sonra yerine şu anda depolama optimizasyonu yapmaya zorlar: +İsteğe bağlı olarak şunları yapabilirsiniz [OPTIMIZE](../sql-reference/statements/misc.md#misc_operations-optimize) ithalattan sonra tablolar. MergeTree-family'den bir motorla yapılandırılmış tablolar, veri depolamayı en iyi duruma getirmek (veya en azından mantıklı olup olmadığını kontrol etmek) için her zaman arka planda veri parçalarının birleştirilmesini sağlar. Bu sorgular, tablo motorunu bir süre sonra yerine şu anda depolama optimizasyonu yapmaya zorlar: ``` bash clickhouse-client --query "OPTIMIZE TABLE tutorial.hits_v1 FINAL" @@ -528,7 +523,7 @@ WHERE (CounterID = 912887) AND (toYYYYMM(StartDate) = 201403) AND (domain(StartU ClickHouse kümesi homojen bir kümedir. Kurulum adımları: -1. Kümenin tüm makinelerine ClickHouse Server’ı yükleyin +1. Kümenin tüm makinelerine ClickHouse Server'ı yükleyin 2. Yapılandırma dosyalarında küme yapılandırmalarını ayarlama 3. Her örnekte yerel tablolar oluşturun 4. Create a [Dağıtılmış tablo](../engines/table-engines/special/distributed.md) diff --git a/docs/tr/guides/apply-catboost-model.md b/docs/tr/guides/apply-catboost-model.md index c60b35287e7..2a22c36ab32 100644 --- a/docs/tr/guides/apply-catboost-model.md +++ b/docs/tr/guides/apply-catboost-model.md @@ -1,22 +1,22 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 41 toc_title: CatBoost Modellerini Uygulamak --- -# Clickhouse’da Bir Catboost Modeli Uygulamak {#applying-catboost-model-in-clickhouse} +# Clickhouse'da bir Catboost modeli uygulamak {#applying-catboost-model-in-clickhouse} [CatBoost](https://catboost.ai) geliştirilen ücretsiz ve açık kaynak kodlu bir GRA anddi libraryent kütüphan aesidir. [Yandex](https://yandex.com/company/) makine öğrenimi için. -Bu Talimatla, Sql’den model çıkarımı çalıştırarak Clickhouse’da önceden eğitilmiş modelleri uygulamayı öğreneceksiniz. +Bu Talimatla, Sql'den model çıkarımı çalıştırarak Clickhouse'da önceden eğitilmiş modelleri uygulamayı öğreneceksiniz. -Clickhouse’da bir CatBoost modeli uygulamak için: +Clickhouse'da bir CatBoost modeli uygulamak için: 1. [Tablo oluşturma](#create-table). 2. [Verileri tabloya ekleme](#insert-data-to-table). -3. [Catboost’u Clickhouse’a entegre edin](#integrate-catboost-into-clickhouse) (İsteğe bağlı adım). -4. [SQL’DEN Model Çıkarımını çalıştırın](#run-model-inference). +3. [Catboost'u Clickhouse'a entegre edin](#integrate-catboost-into-clickhouse) (İsteğe bağlı adım). +4. [SQL'DEN Model Çıkarımını çalıştırın](#run-model-inference). Eğitim CatBoost modelleri hakkında daha fazla bilgi için bkz [Eğitim ve uygulama modelleri](https://catboost.ai/docs/features/training.html#training). @@ -119,12 +119,12 @@ FROM amazon_train +-------+ ``` -## 3. Catboost’u Clickhouse’a entegre edin {#integrate-catboost-into-clickhouse} +## 3. Catboost'u Clickhouse'a entegre edin {#integrate-catboost-into-clickhouse} !!! note "Not" **İsteğe bağlı adım.** Docker görüntü catboost ve ClickHouse çalıştırmak için gereken her şeyi içerir. -Catboost’u Clickhouse’a entegre etmek için: +Catboost'u Clickhouse'a entegre etmek için: **1.** Değerlendirme kitaplığı oluşturun. @@ -161,7 +161,7 @@ Bir CatBoost modelini değerlendirmenin en hızlı yolu derlemedir `libcatboostm /home/catboost/models/*_model.xml ``` -## 4. SQL’DEN Model Çıkarımını çalıştırın {#run-model-inference} +## 4. SQL'DEN Model Çıkarımını çalıştırın {#run-model-inference} Test modeli için ClickHouse istemcisini çalıştırın `$ clickhouse client`. @@ -208,7 +208,7 @@ LIMIT 10 ``` !!! note "Not" - Hakkında daha fazla bilgi [exp()](../sql-reference/functions/math-functions.md) işlev. + Hakkında daha fazla bilgi [exp()](../sql-reference/functions/math-functions.md) İşlev. Örnek üzerinde LogLoss hesaplayalım: diff --git a/docs/tr/guides/index.md b/docs/tr/guides/index.md index 44e3910c026..194fd02712b 100644 --- a/docs/tr/guides/index.md +++ b/docs/tr/guides/index.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_folder_title: "K\u0131lavuzlar" toc_priority: 38 toc_title: "Genel bak\u0131\u015F" @@ -11,6 +11,6 @@ toc_title: "Genel bak\u0131\u015F" ClickHouse kullanarak çeşitli görevleri çözmeye yardımcı olan ayrıntılı adım adım talimatların listesi: - [Basit küme kurulumu eğitimi](../getting-started/tutorial.md) -- [Clickhouse’da bir CatBoost modeli uygulamak](apply-catboost-model.md) +- [Clickhouse'da bir CatBoost modeli uygulamak](apply-catboost-model.md) [Orijinal makale](https://clickhouse.tech/docs/en/guides/) diff --git a/docs/tr/interfaces/cli.md b/docs/tr/interfaces/cli.md index 4a28610e602..de62b3003af 100644 --- a/docs/tr/interfaces/cli.md +++ b/docs/tr/interfaces/cli.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 17 toc_title: "Komut Sat\u0131r\u0131 \u0130stemcisi" --- -# Komut satırı Istemcisi {#command-line-client} +# Komut satırı istemcisi {#command-line-client} ClickHouse yerel bir komut satırı istemcisi sağlar: `clickhouse-client`. İstemci komut satırı seçeneklerini ve yapılandırma dosyalarını destekler. Daha fazla bilgi için, bkz. [Yapılandırma](#interfaces_cli_configuration). @@ -43,7 +43,7 @@ $ cat file.csv | clickhouse-client --database=test --query="INSERT INTO test FOR Toplu iş modunda, varsayılan veri biçimi TabSeparated. Sorgunun biçim yan tümcesinde biçimi ayarlayabilirsiniz. -Varsayılan olarak, yalnızca tek bir sorguyu toplu iş modunda işleyebilirsiniz. Birden çok sorgu yapmak için bir “script,” kullan… `--multiquery` parametre. Bu, INSERT dışındaki tüm sorgular için çalışır. Sorgu sonuçları, ek ayırıcılar olmadan ardışık olarak çıktılanır. Benzer şekilde, çok sayıda sorgu işlemek için, çalıştırabilirsiniz ‘clickhouse-client’ her sorgu için. Başlatmak için onlarca milisaniye sürebilir unutmayın ‘clickhouse-client’ program. +Varsayılan olarak, yalnızca tek bir sorguyu toplu iş modunda işleyebilirsiniz. Birden çok sorgu yapmak için bir “script,” kullan... `--multiquery` parametre. Bu, INSERT dışındaki tüm sorgular için çalışır. Sorgu sonuçları, ek ayırıcılar olmadan ardışık olarak çıktılanır. Benzer şekilde, çok sayıda sorgu işlemek için, çalıştırabilirsiniz ‘clickhouse-client’ her sorgu için. Başlatmak için onlarca milisaniye sürebilir unutmayın ‘clickhouse-client’ program. Etkileşimli modda, sorguları girebileceğiniz bir komut satırı alırsınız. @@ -72,7 +72,7 @@ Ctrl + C tuşlarına basarak uzun bir sorguyu iptal edebilirsiniz. ancak, sunucu Komut satırı istemcisi, sorgulamak için dış verileri (dış geçici tablolar) geçirmenize izin verir. Daha fazla bilgi için bölüme bakın “External data for query processing”. -### Parametrelerle Sorgular {#cli-queries-with-parameters} +### Parametrelerle sorgular {#cli-queries-with-parameters} Parametrelerle bir sorgu oluşturabilir ve istemci uygulamasından onlara değerler aktarabilirsiniz. Bu, istemci tarafında belirli dinamik değerlerle biçimlendirme sorgusunu önlemeye izin verir. Mesela: diff --git a/docs/tr/interfaces/cpp.md b/docs/tr/interfaces/cpp.md index 9ebf93286ff..f5c2227aba6 100644 --- a/docs/tr/interfaces/cpp.md +++ b/docs/tr/interfaces/cpp.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 24 toc_title: "C++ \u0130stemci Kitapl\u0131\u011F\u0131" --- diff --git a/docs/tr/interfaces/formats.md b/docs/tr/interfaces/formats.md index cda944e9014..e031782d356 100644 --- a/docs/tr/interfaces/formats.md +++ b/docs/tr/interfaces/formats.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 21 toc_title: "Giri\u015F ve \xE7\u0131k\u0131\u015F bi\xE7imleri" --- -# Giriş Ve çıkış Verileri için biçimler {#formats} +# Giriş ve çıkış verileri için biçimler {#formats} ClickHouse kabul ve çeşitli biçimlerde veri dönebilirsiniz. Giriş için desteklenen bir biçim, sağlanan verileri ayrıştırmak için kullanılabilir `INSERT`s, gerçekleştirmek için `SELECT`s dosya, URL veya HDFS gibi bir dosya destekli tablodan veya harici bir sözlük okumak için. Çıktı için desteklenen bir biçim düzenlemek için kullanılabilir sonuçları bir `SELECT` ve gerçekleştirmek için `INSERT`s dosya destekli bir tabloya. @@ -42,7 +42,7 @@ Desteklenen formatlar şunlardır: | [ORC](#data-format-orc) | ✔ | ✗ | | [RowBinary](#rowbinary) | ✔ | ✔ | | [Rowbinarywithnames ve türleri](#rowbinarywithnamesandtypes) | ✔ | ✔ | -| [Yerel](#native) | ✔ | ✔ | +| [Yerli](#native) | ✔ | ✔ | | [Boş](#null) | ✗ | ✔ | | [XML](#xml) | ✗ | ✔ | | [CapnProto](#capnproto) | ✔ | ✗ | @@ -55,7 +55,7 @@ Sekmede ayrı format, veri satır ile yazılır. Her satır sekmelerle ayrılmı Bu biçim adı altında da kullanılabilir `TSV`. -Bu `TabSeparated` format, özel programlar ve komut dosyaları kullanarak verileri işlemek için uygundur. Varsayılan olarak HTTP arabiriminde ve komut satırı istemcisinin toplu iş modunda kullanılır. Bu format aynı zamanda farklı Dbms’ler arasında veri aktarımı sağlar. Örneğin, Mysql’den bir dökümü alabilir ve Clickhouse’a yükleyebilirsiniz veya tam tersi. +Bu `TabSeparated` format, özel programlar ve komut dosyaları kullanarak verileri işlemek için uygundur. Varsayılan olarak HTTP arabiriminde ve komut satırı istemcisinin toplu iş modunda kullanılır. Bu format aynı zamanda farklı Dbms'ler arasında veri aktarımı sağlar. Örneğin, Mysql'den bir dökümü alabilir ve Clickhouse'a yükleyebilirsiniz veya tam tersi. Bu `TabSeparated` biçim, toplam değerleri (TOPLAMLARLA birlikte kullanıldığında) ve aşırı değerleri (ne zaman ‘extremes’ 1 olarak ayarlanır). Bu durumlarda, toplam değerler ve aşırılıklar ana veriden sonra çıkar. Ana sonuç, toplam değerler ve aşırılıklar birbirinden boş bir çizgi ile ayrılır. Örnek: @@ -207,7 +207,7 @@ Ayar `format_template_resultset` resultset için bir biçim dizesi içeren dosya - `bytes_read` bayt sayısı (sıkıştırılmamış) okundu mu Tutucu `data`, `totals`, `min` ve `max` kaç rulema kuralı belirtilm (em (elidir (veya `None` açıkça belirtilen) olmalıdır. Kalan yer tutucuları belirtilen kaçan herhangi bir kural olabilir. -Eğer… `format_template_resultset` ayar boş bir dizedir, `${data}` varsayılan değer olarak kullanılır. +Eğer... `format_template_resultset` ayar boş bir dizedir, `${data}` varsayılan değer olarak kullanılır. Insert sorguları biçimi için önek veya sonek varsa bazı sütunları veya bazı alanları atlamaya izin verir (örneğe bakın). Örnek seç: @@ -342,7 +342,7 @@ SELECT * FROM t_null FORMAT TSKV x=1 y=\N ``` -Çok sayıda küçük sütun olduğunda, bu biçim etkisizdir ve genellikle kullanmak için hiçbir neden yoktur. Bununla birlikte, verimlilik açısından Jsoneachrow’dan daha kötü değildir. +Çok sayıda küçük sütun olduğunda, bu biçim etkisizdir ve genellikle kullanmak için hiçbir neden yoktur. Bununla birlikte, verimlilik açısından Jsoneachrow'dan daha kötü değildir. Both data output and parsing are supported in this format. For parsing, any order is supported for the values of different columns. It is acceptable for some values to be omitted – they are treated as equal to their default values. In this case, zeros and blank rows are used as default values. Complex values that could be specified in the table are not supported as defaults. @@ -352,7 +352,7 @@ Ayrıştırma, ek alanın varlığına izin verir `tskv` eşit işareti veya bir Virgülle ayrılmış değerler biçimi ([RFC](https://tools.ietf.org/html/rfc4180)). -Biçimlendirme yaparken, satırlar çift tırnak içine alınır. Bir dizenin içindeki çift alıntı, bir satırda iki çift tırnak olarak çıktılanır. Karakterlerden kaçmak için başka kural yoktur. Tarih ve Tarih-Saat çift tırnak içine alınır. Sayılar tırnak işaretleri olmadan çıktı. Değerler, bir sınırlayıcı karakterle ayrılır; `,` varsayılan olarak. Sınırlayıcı karakteri ayarında tanımlanır [format\_csv\_delimiter](../operations/settings/settings.md#settings-format_csv_delimiter). Satırlar Unıx satır besleme (LF) kullanılarak ayrılır. Diziler CSV’DE aşağıdaki gibi serileştirilir: ilk olarak, dizi TabSeparated biçiminde olduğu gibi bir dizeye serileştirilir ve daha sonra ortaya çıkan dize çift tırnak içinde CSV’YE çıkarılır. CSV biçimindeki Tuples ayrı sütunlar olarak serileştirilir(yani, tuple’daki yuvalanmaları kaybolur). +Biçimlendirme yaparken, satırlar çift tırnak içine alınır. Bir dizenin içindeki çift alıntı, bir satırda iki çift tırnak olarak çıktılanır. Karakterlerden kaçmak için başka kural yoktur. Tarih ve Tarih-Saat çift tırnak içine alınır. Sayılar tırnak işaretleri olmadan çıktı. Değerler, bir sınırlayıcı karakterle ayrılır; `,` varsayılan olarak. Sınırlayıcı karakteri ayarında tanımlanır [format\_csv\_delimiter](../operations/settings/settings.md#settings-format_csv_delimiter). Satırlar Unıx satır besleme (LF) kullanılarak ayrılır. Diziler CSV'DE aşağıdaki gibi serileştirilir: ilk olarak, dizi TabSeparated biçiminde olduğu gibi bir dizeye serileştirilir ve daha sonra ortaya çıkan dize çift tırnak içinde CSV'YE çıkarılır. CSV biçimindeki Tuples ayrı sütunlar olarak serileştirilir(yani, tuple'daki yuvalanmaları kaybolur). ``` bash $ clickhouse-client --format_csv_delimiter="|" --query="INSERT INTO test.csv FORMAT CSV" < data.csv @@ -360,7 +360,7 @@ $ clickhouse-client --format_csv_delimiter="|" --query="INSERT INTO test.csv FOR \* Varsayılan olarak, sınırlayıcı `,`. Görmek [format\_csv\_delimiter](../operations/settings/settings.md#settings-format_csv_delimiter) daha fazla bilgi için ayarlama. -Ayrıştırma yaparken, tüm değerler tırnak işaretleri ile veya tırnak işaretleri olmadan ayrıştırılabilir. Hem çift hem de tek tırnak desteklenmektedir. Satırlar tırnak işaretleri olmadan da düzenlenebilir. Bu durumda, sınırlayıcı karaktere veya satır beslemesine (CR veya LF) ayrıştırılır. RFC’Yİ ihlal ederken, satırları tırnak işaretleri olmadan ayrıştırırken, önde gelen ve sondaki boşluklar ve sekmeler göz ardı edilir. Hat beslemesi için Unix (LF), Windows (CR LF) ve Mac OS Classic (CR LF) türleri desteklenir. +Ayrıştırma yaparken, tüm değerler tırnak işaretleri ile veya tırnak işaretleri olmadan ayrıştırılabilir. Hem çift hem de tek tırnak desteklenmektedir. Satırlar tırnak işaretleri olmadan da düzenlenebilir. Bu durumda, sınırlayıcı karaktere veya satır beslemesine (CR veya LF) ayrıştırılır. RFC'Yİ ihlal ederken, satırları tırnak işaretleri olmadan ayrıştırırken, önde gelen ve sondaki boşluklar ve sekmeler göz ardı edilir. Hat beslemesi için Unix (LF), Windows (CR LF) ve Mac OS Classic (CR LF) türleri desteklenir. Boş unquoted giriş değerleri, ilgili sütunlar için varsayılan değerlerle değiştirilir [ınput\_format\_defaults\_for\_omitted\_fields](../operations/settings/settings.md#session_settings-input_format_defaults_for_omitted_fields) @@ -451,7 +451,7 @@ SELECT SearchPhrase, count() AS c FROM test.hits GROUP BY SearchPhrase WITH TOTA } ``` -Json JavaScript ile uyumludur. Bunu sağlamak için, bazı karakterler ek olarak kaçar: eğik çizgi `/` olarak kaç İsar `\/`; alternatif Satır sonları `U+2028` ve `U+2029`, hangi bazı tarayıcılar kırmak, olarak kaçtı `\uXXXX`. ASCII denetim karakterleri kaçtı: backspace, form besleme, satır besleme, satır başı ve yatay sekme ile değiştirilir `\b`, `\f`, `\n`, `\r`, `\t` , 00-1f aralığında kalan baytların yanı sıra `\uXXXX` sequences. Invalid UTF-8 sequences are changed to the replacement character � so the output text will consist of valid UTF-8 sequences. For compatibility with JavaScript, Int64 and UInt64 integers are enclosed in double-quotes by default. To remove the quotes, you can set the configuration parameter [output\_format\_json\_quote\_64bit\_integers](../operations/settings/settings.md#session_settings-output_format_json_quote_64bit_integers) 0’a. +Json JavaScript ile uyumludur. Bunu sağlamak için, bazı karakterler ek olarak kaçar: eğik çizgi `/` olarak kaç İsar `\/`; alternatif Satır sonları `U+2028` ve `U+2029`, hangi bazı tarayıcılar kırmak, olarak kaçtı `\uXXXX`. ASCII denetim karakterleri kaçtı: backspace, form besleme, satır besleme, satır başı ve yatay sekme ile değiştirilir `\b`, `\f`, `\n`, `\r`, `\t` , 00-1f aralığında kalan baytların yanı sıra `\uXXXX` sequences. Invalid UTF-8 sequences are changed to the replacement character � so the output text will consist of valid UTF-8 sequences. For compatibility with JavaScript, Int64 and UInt64 integers are enclosed in double-quotes by default. To remove the quotes, you can set the configuration parameter [output\_format\_json\_quote\_64bit\_integers](../operations/settings/settings.md#session_settings-output_format_json_quote_64bit_integers) 0'a. `rows` – The total number of output rows. @@ -470,7 +470,7 @@ Ayrıca bakınız [JSONEachRow](#jsoneachrow) biçimli. ## JSONCompact {#jsoncompact} -Yalnızca veri satırlarında json’dan farklıdır, nesnelerde değil, dizilerde çıktıdır. +Yalnızca veri satırlarında json'dan farklıdır, nesnelerde değil, dizilerde çıktıdır. Örnek: @@ -582,7 +582,7 @@ Sorgu `SELECT * FROM UserActivity FORMAT JSONEachRow` dönüşler: Aksine [JSON](#json) biçimi, geçersiz UTF-8 dizilerinin hiçbir ikame yoktur. Değerleri için olduğu gibi aynı şekilde kaçtı `JSON`. !!! note "Not" - Herhangi bir bayt kümesi dizelerde çıktı olabilir. Kullan… `JSONEachRow` tablodaki verilerin herhangi bir bilgi kaybetmeden JSON olarak biçimlendirilebileceğinden eminseniz biçimlendirin. + Herhangi bir bayt kümesi dizelerde çıktı olabilir. Kullan... `JSONEachRow` tablodaki verilerin herhangi bir bilgi kaybetmeden JSON olarak biçimlendirilebileceğinden eminseniz biçimlendirin. ### İç içe yapıların kullanımı {#jsoneachrow-nested} @@ -643,9 +643,9 @@ SELECT * FROM json_each_row_nested └───────────────┴────────┘ ``` -## Yerel {#native} +## Yerli {#native} -En verimli biçim. Veriler ikili formatta bloklar tarafından yazılır ve okunur. Her blok için satır sayısı, sütun sayısı, sütun adları ve türleri ve bu bloktaki sütunların parçaları birbiri ardına kaydedilir. Başka bir deyişle, bu format “columnar” – it doesn’t convert columns to rows. This is the format used in the native interface for interaction between servers, for using the command-line client, and for C++ clients. +En verimli biçim. Veriler ikili formatta bloklar tarafından yazılır ve okunur. Her blok için satır sayısı, sütun sayısı, sütun adları ve türleri ve bu bloktaki sütunların parçaları birbiri ardına kaydedilir. Başka bir deyişle, bu format “columnar” – it doesn't convert columns to rows. This is the format used in the native interface for interaction between servers, for using the command-line client, and for C++ clients. Bu biçimi, yalnızca ClickHouse DBMS tarafından okunabilen dökümleri hızlı bir şekilde oluşturmak için kullanabilirsiniz. Bu formatla kendiniz çalışmak mantıklı değil. @@ -686,7 +686,7 @@ SELECT 'String with \'quotes\' and \t character' AS Escaping_test └──────────────────────────────────────┘ ``` -Terminale çok fazla veri boşaltmaktan kaçınmak için yalnızca ilk 10.000 satır yazdırılır. Satır sayısı 10.000’den büyük veya eşitse, ileti “Showed first 10 000” bas .ılmıştır. +Terminale çok fazla veri boşaltmaktan kaçınmak için yalnızca ilk 10.000 satır yazdırılır. Satır sayısı 10.000'den büyük veya eşitse, ileti “Showed first 10 000” bas .ılmıştır. Bu biçim yalnızca bir sorgu sonucu çıktısı için uygundur, ancak ayrıştırma için değil (bir tabloya eklemek için veri alma). Güzel biçim, toplam değerleri (TOPLAMLARLA birlikte kullanıldığında) ve aşırılıkları (ne zaman ‘extremes’ 1 olarak ayarlanır). Bu durumlarda, toplam değerler ve aşırı değerler ana veriden sonra ayrı tablolarda çıktılanır. Örnek (gösterilen [PrettyCompact](#prettycompact) biçimli): @@ -766,7 +766,7 @@ Dizi varint uzunluğu (imzasız) olarak temsil edilir [LEB128](https://en.wikipe İçin [NULL](../sql-reference/syntax.md#null-literal) destek, 1 veya 0 içeren ek bir bayt her önce eklenir [Nullable](../sql-reference/data-types/nullable.md) değer. 1 ise, o zaman değer `NULL` ve bu bayt ayrı bir değer olarak yorumlanır. 0 ise, bayttan sonraki değer değil `NULL`. -## Rowbinarywithnames Ve türleri {#rowbinarywithnamesandtypes} +## Rowbinarywithnames ve türleri {#rowbinarywithnamesandtypes} Benzer [RowBinary](#rowbinary), ancak eklenen Başlık ile: @@ -897,9 +897,9 @@ Diziler olarak çıktı `HelloWorld...` ## CapnProto {#capnproto} -Cap’n Proto, Protokol Tamponlarına ve tasarrufuna benzer, ancak JSON veya MessagePack gibi olmayan bir ikili mesaj biçimidir. +Cap'n Proto, Protokol Tamponlarına ve tasarrufuna benzer, ancak JSON veya MessagePack gibi olmayan bir ikili mesaj biçimidir. -Cap’n Proto mesajları kesinlikle yazılır ve kendi kendini tanımlamaz, yani harici bir şema açıklamasına ihtiyaç duyarlar. Şema anında uygulanır ve her sorgu için önbelleğe alınır. +Cap'n Proto mesajları kesinlikle yazılır ve kendi kendini tanımlamaz, yani harici bir şema açıklamasına ihtiyaç duyarlar. Şema anında uygulanır ve her sorgu için önbelleğe alınır. ``` bash $ cat capnproto_messages.bin | clickhouse-client --query "INSERT INTO test.hits FORMAT CapnProto SETTINGS format_schema='schema:Message'" @@ -948,7 +948,7 @@ message MessageType { }; ``` -İletişim kuralı arabellekleri’ ileti türü Tablo sütunları ve alanları arasındaki yazışmaları bulmak için clickhouse adlarını karşılaştırır. +İletişim kuralı arabellekleri' ileti türü Tablo sütunları ve alanları arasındaki yazışmaları bulmak için clickhouse adlarını karşılaştırır. Bu karşılaştırma büyük / küçük harf duyarsız ve karakterler `_` (alt çizgi) ve `.` (nokta) eşit olarak kabul edilir. Bir sütun türleri ve protokol arabellekleri ileti alanı farklıysa, gerekli dönüştürme uygulanır. @@ -987,13 +987,13 @@ Ayrıca bakınız [popüler dillerde uzunlukla ayrılmış protobuf mesajları n ## Avro {#data-format-avro} -[Apache Avro](http://avro.apache.org/) Apache’nin Hadoop projesi kapsamında geliştirilen satır odaklı veri serileştirme çerçevesidir. +[Apache Avro](http://avro.apache.org/) Apache'nin Hadoop projesi kapsamında geliştirilen satır odaklı veri serileştirme çerçevesidir. ClickHouse Avro biçimi okuma ve yazma destekler [Avro veri dosyaları](http://avro.apache.org/docs/current/spec.html#Object+Container+Files). ### Veri Türleri Eşleştirme {#data_types-matching} -Aşağıdaki tablo, desteklenen veri türlerini ve Clickhouse’la nasıl eşleştiğini gösterir [veri türleri](../sql-reference/data-types/index.md) içinde `INSERT` ve `SELECT` sorgular. +Aşağıdaki tablo, desteklenen veri türlerini ve Clickhouse'la nasıl eşleştiğini gösterir [veri türleri](../sql-reference/data-types/index.md) içinde `INSERT` ve `SELECT` sorgular. | Avro veri türü `INSERT` | ClickHouse veri türü | Avro veri türü `SELECT` | |---------------------------------------------|-------------------------------------------------------------------------------------------------------------------|------------------------------| @@ -1055,7 +1055,7 @@ Her Avro iletisi, şema Kayıt defterinin yardımıyla gerçek şemaya çözüle Şemalar çözüldükten sonra önbelleğe alınır. -Şema kayıt defteri URL’si ile yapılandırılır [format\_avro\_schema\_registry\_url](../operations/settings/settings.md#settings-format_avro_schema_registry_url) +Şema kayıt defteri URL'si ile yapılandırılır [format\_avro\_schema\_registry\_url](../operations/settings/settings.md#settings-format_avro_schema_registry_url) ### Veri Türleri Eşleştirme {#data_types-matching-1} @@ -1101,7 +1101,7 @@ SELECT * FROM topic1_stream; ### Veri Türleri Eşleştirme {#data_types-matching-2} -Aşağıdaki tablo, desteklenen veri türlerini ve Clickhouse’la nasıl eşleştiğini gösterir [veri türleri](../sql-reference/data-types/index.md) içinde `INSERT` ve `SELECT` sorgular. +Aşağıdaki tablo, desteklenen veri türlerini ve Clickhouse'la nasıl eşleştiğini gösterir [veri türleri](../sql-reference/data-types/index.md) içinde `INSERT` ve `SELECT` sorgular. | Parke veri türü (`INSERT`) | ClickHouse veri türü | Parke veri türü (`SELECT`) | |----------------------------|-----------------------------------------------------------|----------------------------| @@ -1127,7 +1127,7 @@ Desteklen datameyen veri türleri: `DATE32`, `TIME32`, `FIXED_SIZE_BINARY`, `JSO ClickHouse tablo sütunlarının veri türleri, eklenen parke verilerinin ilgili alanlarından farklı olabilir. Veri eklerken, ClickHouse veri türlerini yukarıdaki tabloya göre yorumlar ve sonra [döküm](../query_language/functions/type_conversion_functions/#type_conversion_function-cast) ClickHouse tablo sütunu için ayarlanmış olan bu veri türüne ait veriler. -### Veri Ekleme Ve seçme {#inserting-and-selecting-data} +### Veri ekleme ve seçme {#inserting-and-selecting-data} Bir dosyadan parke verilerini ClickHouse tablosuna aşağıdaki komutla ekleyebilirsiniz: @@ -1145,11 +1145,11 @@ Hadoop ile veri alışverişi yapmak için şunları kullanabilirsiniz [HDFS tab ## ORC {#data-format-orc} -[Apache ORCC](https://orc.apache.org/) hadoop ekosisteminde yaygın bir sütunlu depolama biçimidir. Bu formatta yalnızca Clickhouse’a veri ekleyebilirsiniz. +[Apache ORCC](https://orc.apache.org/) hadoop ekosisteminde yaygın bir sütunlu depolama biçimidir. Bu formatta yalnızca Clickhouse'a veri ekleyebilirsiniz. ### Veri Türleri Eşleştirme {#data_types-matching-3} -Aşağıdaki tablo, desteklenen veri türlerini ve Clickhouse’la nasıl eşleştiğini gösterir [veri türleri](../sql-reference/data-types/index.md) içinde `INSERT` sorgular. +Aşağıdaki tablo, desteklenen veri türlerini ve Clickhouse'la nasıl eşleştiğini gösterir [veri türleri](../sql-reference/data-types/index.md) içinde `INSERT` sorgular. | Orc veri türü (`INSERT`) | ClickHouse veri türü | |--------------------------|-----------------------------------------------------| diff --git a/docs/tr/interfaces/http.md b/docs/tr/interfaces/http.md index a5ba5183743..2b92dd0ed9b 100644 --- a/docs/tr/interfaces/http.md +++ b/docs/tr/interfaces/http.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 19 toc_title: "HTTP aray\xFCz\xFC" --- # HTTP arayüzü {#http-interface} -HTTP arayüzü, herhangi bir programlama dilinden herhangi bir platformda Clickhouse’u kullanmanızı sağlar. Java ve Perl’den ve kabuk komut dosyalarından çalışmak için kullanıyoruz. Diğer bölümlerde, HTTP arayüzü Perl, Python ve Go’dan kullanılır. HTTP arabirimi yerel arabirimden daha sınırlıdır, ancak daha iyi uyumluluğa sahiptir. +HTTP arayüzü, herhangi bir programlama dilinden herhangi bir platformda Clickhouse'u kullanmanızı sağlar. Java ve Perl'den ve kabuk komut dosyalarından çalışmak için kullanıyoruz. Diğer bölümlerde, HTTP arayüzü Perl, Python ve Go'dan kullanılır. HTTP arabirimi yerel arabirimden daha sınırlıdır, ancak daha iyi uyumluluğa sahiptir. Varsayılan olarak, clickhouse-server, 8123 numaralı bağlantı noktasında HTTP dinler (bu, yapılandırmada değiştirilebilir). @@ -25,7 +25,7 @@ $ curl 'http://localhost:8123/ping' Ok. ``` -İsteği URL olarak gönder ‘query’ parametre veya bir POST olarak. Veya sorgunun başlangıcını gönder ‘query’ parametre ve postadaki geri kalanı (bunun neden gerekli olduğunu daha sonra açıklayacağız). URL’nin boyutu 16 KB ile sınırlıdır, bu nedenle büyük sorgular gönderirken bunu aklınızda bulundurun. +İsteği URL olarak gönder ‘query’ parametre veya bir POST olarak. Veya sorgunun başlangıcını gönder ‘query’ parametre ve postadaki geri kalanı (bunun neden gerekli olduğunu daha sonra açıklayacağız). URL'nin boyutu 16 KB ile sınırlıdır, bu nedenle büyük sorgular gönderirken bunu aklınızda bulundurun. Başarılı olursa, 200 yanıt Kodu ve yanıt gövdesinde sonucu alırsınız. Bir hata oluşursa, 500 yanıt Kodu ve yanıt gövdesinde bir hata açıklaması metni alırsınız. @@ -53,7 +53,7 @@ X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","writ 1 ``` -Gördüğünüz gibi, curl, boşlukların URL’den kaçması gerektiği konusunda biraz rahatsız edici. +Gördüğünüz gibi, curl, boşlukların URL'den kaçması gerektiği konusunda biraz rahatsız edici. Her ne kadar wget her şeyden kaçsa da, onu kullanmanızı önermiyoruz çünkü keep-alive ve Transfer-Encoding: chunked kullanırken HTTP 1.1 üzerinde iyi çalışmıyor. ``` bash @@ -89,7 +89,7 @@ $ echo 'SELECT 1 FORMAT Pretty' | curl 'http://localhost:8123/?' --data-binary @ └───┘ ``` -Ekleme sorguları için veri iletmenin POST yöntemi gereklidir. Bu durumda, URL parametresinde sorgunun başlangıcını yazabilir ve eklemek için verileri iletmek için POST’u kullanabilirsiniz. Eklenecek veriler, örneğin Mysql’den sekmeyle ayrılmış bir döküm olabilir. Bu şekilde, INSERT sorgusu MYSQL’DEN load DATA LOCAL INFİLE’IN yerini alır. +Ekleme sorguları için veri iletmenin POST yöntemi gereklidir. Bu durumda, URL parametresinde sorgunun başlangıcını yazabilir ve eklemek için verileri iletmek için POST'u kullanabilirsiniz. Eklenecek veriler, örneğin Mysql'den sekmeyle ayrılmış bir döküm olabilir. Bu şekilde, INSERT sorgusu MYSQL'DEN load DATA LOCAL INFİLE'IN yerini alır. Örnekler: tablo oluşturma: @@ -149,10 +149,10 @@ Veri tablosu döndürmeyen başarılı istekler için boş bir yanıt gövdesi d Veri iletirken dahili ClickHouse sıkıştırma formatını kullanabilirsiniz. Sıkıştırılmış veriler standart olmayan bir biçime sahiptir ve özel `clickhouse-compressor` onunla çalışmak için program (bu ile yüklü `clickhouse-client` paket). Veri ekleme verimliliğini artırmak için, sunucu tarafı sağlama toplamı doğrulamasını kullanarak devre dışı bırakabilirsiniz. [http\_native\_compression\_disable\_checksumming\_on\_decompress](../operations/settings/settings.md#settings-http_native_compression_disable_checksumming_on_decompress) ayar. -Belirt ift ifiyseniz `compress=1` URL’de, sunucu size gönderdiği verileri sıkıştırır. -Belirt ift ifiyseniz `decompress=1` URL’de, sunucu içinde geçirdiğiniz aynı verileri açar. `POST` yöntem. +Belirt ift ifiyseniz `compress=1` URL'de, sunucu size gönderdiği verileri sıkıştırır. +Belirt ift ifiyseniz `decompress=1` URL'de, sunucu içinde geçirdiğiniz aynı verileri açar. `POST` yöntem. -Ayrıca kullanmayı seçebilirsiniz [HTTP sıkıştırma](https://en.wikipedia.org/wiki/HTTP_compression). Sıkıştırılmış bir göndermek için `POST` istek, istek başlığını Ekle `Content-Encoding: compression_method`. Clickhouse’un yanıtı sıkıştırması için şunları eklemelisiniz `Accept-Encoding: compression_method`. ClickHouse destekler `gzip`, `br`, ve `deflate` [sıkıştırma yöntemleri](https://en.wikipedia.org/wiki/HTTP_compression#Content-Encoding_tokens). HTTP sıkıştırmasını etkinleştirmek için Clickhouse’u kullanmanız gerekir [enable\_http\_compression](../operations/settings/settings.md#settings-enable_http_compression) ayar. Veri sıkıştırma düzeyini [http\_zlib\_compression\_level](#settings-http_zlib_compression_level) tüm sıkıştırma yöntemleri için ayarlama. +Ayrıca kullanmayı seçebilirsiniz [HTTP sıkıştırma](https://en.wikipedia.org/wiki/HTTP_compression). Sıkıştırılmış bir göndermek için `POST` istek, istek başlığını Ekle `Content-Encoding: compression_method`. Clickhouse'un yanıtı sıkıştırması için şunları eklemelisiniz `Accept-Encoding: compression_method`. ClickHouse destekler `gzip`, `br`, ve `deflate` [sıkıştırma yöntemleri](https://en.wikipedia.org/wiki/HTTP_compression#Content-Encoding_tokens). HTTP sıkıştırmasını etkinleştirmek için Clickhouse'u kullanmanız gerekir [enable\_http\_compression](../operations/settings/settings.md#settings-enable_http_compression) ayar. Veri sıkıştırma düzeyini [http\_zlib\_compression\_level](#settings-http_zlib_compression_level) tüm sıkıştırma yöntemleri için ayarlama. Bunu, büyük miktarda veri iletirken ağ trafiğini azaltmak veya hemen sıkıştırılmış dökümler oluşturmak için kullanabilirsiniz. @@ -275,7 +275,7 @@ $ curl -sS 'http://localhost:8123/?max_result_bytes=4000000&buffer_size=3000000& Yanıt Kodu ve HTTP üstbilgileri istemciye gönderildikten sonra bir sorgu işleme hatası oluştu durumları önlemek için arabelleğe alma kullanın. Bu durumda, yanıt gövdesinin sonunda bir hata iletisi yazılır ve istemci tarafında hata yalnızca ayrıştırma aşamasında algılanabilir. -### Parametrelerle Sorgular {#cli-queries-with-parameters} +### Parametrelerle sorgular {#cli-queries-with-parameters} Parametrelerle bir sorgu oluşturabilir ve karşılık gelen HTTP istek parametrelerinden onlar için değerler geçirebilirsiniz. Daha fazla bilgi için, bkz. [CLI için parametrelerle sorgular](cli.md#cli-queries-with-parameters). @@ -285,7 +285,7 @@ Parametrelerle bir sorgu oluşturabilir ve karşılık gelen HTTP istek parametr $ curl -sS "
?param_id=2¶m_phrase=test" -d "SELECT * FROM table WHERE int_column = {id:UInt8} and string_column = {phrase:String}" ``` -## Önceden tanımlanmış HTTP Arabirimi {#predefined_http_interface} +## Önceden tanımlanmış HTTP arabirimi {#predefined_http_interface} ClickHouse HTTP arabirimi üzerinden belirli sorguları destekler. Örneğin, bir tabloya aşağıdaki gibi veri yazabilirsiniz: @@ -303,13 +303,16 @@ ClickHouse ayrıca gibi üçüncü parti araçları ile daha kolay entegrasyon y ``` xml - - /metrics - GET - + + /predefined_query + POST,GET + + predefined_query_handler SELECT * FROM system.metrics LIMIT 5 FORMAT Template SETTINGS format_template_resultset = 'prometheus_template_output_format_resultset', format_template_row = 'prometheus_template_output_format_row', format_template_rows_between_delimiter = '\n' - - + + + ... + ... ``` @@ -318,21 +321,23 @@ ClickHouse ayrıca gibi üçüncü parti araçları ile daha kolay entegrasyon y ``` bash -curl -vvv 'http://localhost:8123/metrics' +$ curl -v 'http://localhost:8123/predefined_query' * Trying ::1... * Connected to localhost (::1) port 8123 (#0) -> GET /metrics HTTP/1.1 +> GET /predefined_query HTTP/1.1 > Host: localhost:8123 > User-Agent: curl/7.47.0 > Accept: */* > < HTTP/1.1 200 OK -< Date: Wed, 27 Nov 2019 08:54:25 GMT +< Date: Tue, 28 Apr 2020 08:52:56 GMT < Connection: Keep-Alive < Content-Type: text/plain; charset=UTF-8 -< X-ClickHouse-Server-Display-Name: i-tl62qd0o +< X-ClickHouse-Server-Display-Name: i-mloy5trc < Transfer-Encoding: chunked -< X-ClickHouse-Query-Id: f39235f6-6ed7-488c-ae07-c7ceafb960f6 +< X-ClickHouse-Query-Id: 96fe0052-01e6-43ce-b12a-6b7370de6e8a +< X-ClickHouse-Format: Template +< X-ClickHouse-Timezone: Asia/Shanghai < Keep-Alive: timeout=3 < X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} < @@ -356,117 +361,62 @@ curl -vvv 'http://localhost:8123/metrics' # TYPE "ReplicatedSend" counter "ReplicatedSend" 0 +* Connection #0 to host localhost left intact + + * Connection #0 to host localhost left intact ``` -Örnekten görebileceğiniz gibi, Eğer `` yapılandırmada yapılandırılır.XML dosyası, ClickHouse önceden tanımlanmış türüne alınan HTTP istekleri eşleşecek `` Maç başarılı olursa, ClickHouse ilgili önceden tanımlanmış sorgu yürütecektir. +Örnekten görebileceğiniz gibi, Eğer `` yapılandırmada yapılandırılır.xml dosyası ve `` birçok içerebilir `s`. ClickHouse, önceden tanımlanmış türe alınan HTTP istekleriyle eşleşir `` ve ilk eşleşen işleyiciyi çalıştırır. Ardından, Maç başarılı olursa, ClickHouse ilgili önceden tanımlanmış sorguyu yürütecektir. -Şimdi `` Yapılandır configureılabilir ``, ``, ``, `` ve `` . +> Şimdi `` Yapılandır configureılabilir ``, ``, ``,``: +> `` HTTP isteğinin yöntem bölümünü eşleştirmekten sorumludur. `` tam tanımına uygundur [yöntem](https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods) HTTP protokolünde. İsteğe bağlı bir yapılandırmadır. Yapılandırma dosyasında tanımlanmamışsa, HTTP isteğinin yöntem kısmıyla eşleşmez. +> +> `` HTTP isteğinin url bölümünü eşleştirmekten sorumludur. İle uyumludur [RE2](https://github.com/google/re2)'In düzenli ifadeleri. İsteğe bağlı bir yapılandırmadır. Yapılandırma dosyasında tanımlanmamışsa, HTTP isteğinin url kısmıyla eşleşmez. +> +> `` HTTP isteğinin başlık kısmını eşleştirmekten sorumludur. Bu re2 düzenli ifadeler ile uyumludur. İsteğe bağlı bir yapılandırmadır. Yapılandırma dosyasında tanımlanmamışsa, HTTP isteğinin üstbilgi bölümü eşleşmiyor. +> +> `` ana işleme bölümünü içerir. Şimdi `` Yapılandır configureılabilir ``, ``, ``, ``, ``, ``. +> \> `` şu anda üç tip destekler: **predefined\_query\_handler**, **dynamic\_query\_handler**, **sabit**. +> \> +> \> `` - işleyici çağrıldığında predefined\_query\_handler türü ile kullanın, sorgu yürütür. +> \> +> \> `` - dynamic\_query\_handler tipi ile kullanın, özler ve karşılık gelen değeri yürütür `` HTTP isteği params değeri. +> \> +> \> `` - statik tip, yanıt durum kodu ile kullanın. +> \> +> \> `` - statik tip, tepki ile kullanın [içerik türü](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type). +> \> +> \> `` - önek kullanırken, istemciye gönderilen statik tip, yanıt içeriği ile kullanın ‘file://’ veya ‘config://’, dosyadan içeriği bulun veya yapılandırma istemciye gönderin. -## root\_handler {#root_handler} - -`` kök yolu isteği için belirtilen içeriği döndürür. Belirli dönüş içeriği tarafından yapılandırılır `http_server_default_response` config.xml. belirtilmemişse, iade **Tamam.** - -`http_server_default_response` tanımlanmadı ve Clickhouse’a bir HTTP isteği gönderildi. Sonuç aşağıdaki gibidir: - -``` xml - - - -``` - - $ curl 'http://localhost:8123' - Ok. - -`http_server_default_response` tanımlanır ve Clickhouse’a bir HTTP isteği gönderilir. Sonuç aşağıdaki gibidir: - -``` xml -
]]> - - - - -``` - - $ curl 'http://localhost:8123' -
% - -## ping\_handler {#ping_handler} - -`` geçerli ClickHouse sunucusunun durumunu araştırmak için kullanılabilir. ClickHouse HTTP Sunucusu normal olduğunda, Clickhouse’a erişme `` dön willecektir **Tamam.**. - -Örnek: - -``` xml - - /ping - -``` - -``` bash -$ curl 'http://localhost:8123/ping' -Ok. -``` - -## replicas\_status\_handler {#replicas_status_handler} - -`` çoğaltma düğümünün durumunu algılamak ve geri dönmek için kullanılır **Tamam.** çoğaltma düğümünde gecikme yoksa. Bir gecikme varsa, belirli bir gecikmeyi iade edin. Değeri `` özelleştirme destekler. Belirt specifymezseniz ``, ClickHouse varsayılan ayarı `` oluyor **/ replicas\_status**. - -Örnek: - -``` xml - - /replicas_status - -``` - -Hiçbir gecikme durumda: - -``` bash -$ curl 'http://localhost:8123/replicas_status' -Ok. -``` - -Gecikmeli dava: - -``` bash -$ curl 'http://localhost:8123/replicas_status' -db.stats: Absolute delay: 22. Relative delay: 22. -``` +Sonraki farklı yapılandırma yöntemleri ``. ## predefined\_query\_handler {#predefined_query_handler} -Yapılandırabilirsiniz ``, ``, `` ve `` içinde ``. +`` ayar ayarları ve query\_params değerlerini destekler. Yapılandırabilirsiniz `` tip ininde ``. -`` HTTP isteğinin yöntem bölümünü eşleştirmekten sorumludur. `` tam tanımına uygundur [yöntem](https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods) HTTP protokolünde. İsteğe bağlı bir yapılandırmadır. Yapılandırma dosyasında tanımlanmamışsa, HTTP isteğinin yöntem kısmıyla eşleşmez - -`` HTTP isteğinin url bölümünü eşleştirmekten sorumludur. İle uyumludur [RE2](https://github.com/google/re2)’In düzenli ifadeleri. İsteğe bağlı bir yapılandırmadır. Yapılandırma dosyasında tanımlanmamışsa, HTTP isteğinin url kısmıyla eşleşmez - -`` HTTP isteğinin başlık kısmını eşleştirmekten sorumludur. Bu re2 düzenli ifadeler ile uyumludur. İsteğe bağlı bir yapılandırmadır. Yapılandırma dosyasında tanımlanmamışsa, HTTP isteğinin başlık kısmıyla eşleşmez - -`` değer, önceden tanımlanmış bir sorgudur ``, bir HTTP isteği eşleştirildiğinde ve sorgunun sonucu döndürüldüğünde ClickHouse tarafından yürütülür. Bu bir zorunluluktur yapılandırma. - -`` ayar ayarları ve query\_params değerlerini destekler. +`` değer, önceden tanımlanmış bir sorgudur ``, bir HTTP isteği eşleştirildiğinde ve sorgunun sonucu döndürüldüğünde ClickHouse tarafından yürütülür. Bu bir zorunluluktur yapılandırma. Aşağıdaki örnek değerleri tanımlar `max_threads` ve `max_alter_threads` ayarlar, ardından bu ayarların başarıyla ayarlanıp ayarlanmadığını kontrol etmek için sistem tablosunu sorgular. Örnek: ``` xml - - + + + [^/]+)(/(?P[^/]+))?]]> GET TEST_HEADER_VALUE [^/]+)(/(?P[^/]+))?]]> - [^/]+)(/(?P[^/]+))?]]> - + + predefined_query_handler SELECT value FROM system.settings WHERE name = {name_1:String} SELECT name, value FROM system.settings WHERE name = {name_2:String} - - - + + + ``` ``` bash @@ -475,37 +425,193 @@ $ curl -H 'XXX:TEST_HEADER_VALUE' -H 'PARAMS_XXX:max_threads' 'http://localhost: max_alter_threads 2 ``` -!!! note "Not" - Birinde ``, biri `` sadece birini destekler `` bir ekleme türü. +!!! note "Dikkat" + Birinde `` sadece birini destekler `` bir ekleme türü. ## dynamic\_query\_handler {#dynamic_query_handler} -`` göre `` artmak `` . +İçinde ``, sorgu HTTP isteğinin param şeklinde yazılır. Fark şu ki ``, sorgu yapılandırma dosyasında yazılır. Yapılandırabilirsiniz `` içinde ``. -ClickHouse ayıklar ve karşılık gelen değeri yürütür `` HTTP isteğinin url’sindeki değer. -ClickHouse varsayılan ayarı `` oluyor `/query` . İsteğe bağlı bir yapılandırmadır. Yapılandırma dosyasında tanım yoksa, param iletilmez. +ClickHouse ayıklar ve karşılık gelen değeri yürütür `` HTTP isteğinin url'sindeki değer. Varsayılan değer `` oluyor `/query` . İsteğe bağlı bir yapılandırmadır. Yapılandırma dosyasında tanım yoksa, param iletilmez. Bu işlevselliği denemek için örnek max\_threads ve max\_alter\_threads değerlerini tanımlar ve ayarların başarıyla ayarlanıp ayarlanmadığını sorgular. -Fark şu ki ``, sorgu yapılandırma dosyasında yazılır. Ama içinde ``, sorgu HTTP isteğinin param şeklinde yazılır. Örnek: ``` xml - - - - TEST_HEADER_VALUE_DYNAMIC - [^/]+)(/(?P[^/]+))?]]> - + + + + TEST_HEADER_VALUE_DYNAMIC + + dynamic_query_handler query_param - - + + + ``` ``` bash -$ curl -H 'XXX:TEST_HEADER_VALUE_DYNAMIC' -H 'PARAMS_XXX:max_threads' 'http://localhost:8123/?query_param=SELECT%20value%20FROM%20system.settings%20where%20name%20=%20%7Bname_1:String%7D%20OR%20name%20=%20%7Bname_2:String%7D&max_threads=1&max_alter_threads=2¶m_name_2=max_alter_threads' -1 -2 +$ curl -H 'XXX:TEST_HEADER_VALUE_DYNAMIC' 'http://localhost:8123/own?max_threads=1&max_alter_threads=2¶m_name_1=max_threads¶m_name_2=max_alter_threads&query_param=SELECT%20name,value%20FROM%20system.settings%20where%20name%20=%20%7Bname_1:String%7D%20OR%20name%20=%20%7Bname_2:String%7D' +max_threads 1 +max_alter_threads 2 +``` + +## sabit {#static} + +`` dön canebilir [content\_type](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type), [durum](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status) ve response\_content. response\_content belirtilen içeriği döndürebilir + +Örnek: + +Mesajı dönmektedir. + +``` xml + + + GET + xxx + /hi + + static + 402 + text/html; charset=UTF-8 + Say Hi! + + + +``` + +``` bash +$ curl -vv -H 'XXX:xxx' 'http://localhost:8123/hi' +* Trying ::1... +* Connected to localhost (::1) port 8123 (#0) +> GET /hi HTTP/1.1 +> Host: localhost:8123 +> User-Agent: curl/7.47.0 +> Accept: */* +> XXX:xxx +> +< HTTP/1.1 402 Payment Required +< Date: Wed, 29 Apr 2020 03:51:26 GMT +< Connection: Keep-Alive +< Content-Type: text/html; charset=UTF-8 +< Transfer-Encoding: chunked +< Keep-Alive: timeout=3 +< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} +< +* Connection #0 to host localhost left intact +Say Hi!% +``` + +Yapılandırmadan içeriği bulun istemciye gönder. + +``` xml +
]]> + + + + GET + xxx + /get_config_static_handler + + static + config://get_config_static_handler + + + +``` + +``` bash +$ curl -v -H 'XXX:xxx' 'http://localhost:8123/get_config_static_handler' +* Trying ::1... +* Connected to localhost (::1) port 8123 (#0) +> GET /get_config_static_handler HTTP/1.1 +> Host: localhost:8123 +> User-Agent: curl/7.47.0 +> Accept: */* +> XXX:xxx +> +< HTTP/1.1 200 OK +< Date: Wed, 29 Apr 2020 04:01:24 GMT +< Connection: Keep-Alive +< Content-Type: text/plain; charset=UTF-8 +< Transfer-Encoding: chunked +< Keep-Alive: timeout=3 +< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} +< +* Connection #0 to host localhost left intact +
% +``` + +İstemciye gönder dosyadan içeriği bulun. + +``` xml + + + GET + xxx + /get_absolute_path_static_handler + + static + text/html; charset=UTF-8 + file:///absolute_path_file.html + + + + GET + xxx + /get_relative_path_static_handler + + static + text/html; charset=UTF-8 + file://./relative_path_file.html + + + +``` + +``` bash +$ user_files_path='/var/lib/clickhouse/user_files' +$ sudo echo "Relative Path File" > $user_files_path/relative_path_file.html +$ sudo echo "Absolute Path File" > $user_files_path/absolute_path_file.html +$ curl -vv -H 'XXX:xxx' 'http://localhost:8123/get_absolute_path_static_handler' +* Trying ::1... +* Connected to localhost (::1) port 8123 (#0) +> GET /get_absolute_path_static_handler HTTP/1.1 +> Host: localhost:8123 +> User-Agent: curl/7.47.0 +> Accept: */* +> XXX:xxx +> +< HTTP/1.1 200 OK +< Date: Wed, 29 Apr 2020 04:18:16 GMT +< Connection: Keep-Alive +< Content-Type: text/html; charset=UTF-8 +< Transfer-Encoding: chunked +< Keep-Alive: timeout=3 +< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} +< +Absolute Path File +* Connection #0 to host localhost left intact +$ curl -vv -H 'XXX:xxx' 'http://localhost:8123/get_relative_path_static_handler' +* Trying ::1... +* Connected to localhost (::1) port 8123 (#0) +> GET /get_relative_path_static_handler HTTP/1.1 +> Host: localhost:8123 +> User-Agent: curl/7.47.0 +> Accept: */* +> XXX:xxx +> +< HTTP/1.1 200 OK +< Date: Wed, 29 Apr 2020 04:18:31 GMT +< Connection: Keep-Alive +< Content-Type: text/html; charset=UTF-8 +< Transfer-Encoding: chunked +< Keep-Alive: timeout=3 +< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} +< +Relative Path File +* Connection #0 to host localhost left intact ``` [Orijinal makale](https://clickhouse.tech/docs/en/interfaces/http_interface/) diff --git a/docs/tr/interfaces/index.md b/docs/tr/interfaces/index.md index d80ea8d8f57..4453135adfb 100644 --- a/docs/tr/interfaces/index.md +++ b/docs/tr/interfaces/index.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_folder_title: Arabirimler toc_priority: 14 toc_title: "Giri\u015F" @@ -8,7 +8,7 @@ toc_title: "Giri\u015F" # Arabirimler {#interfaces} -ClickHouse iki ağ arabirimi sağlar (Her ikisi de isteğe bağlı olarak ek güvenlik için TLS’YE sarılabilir): +ClickHouse iki ağ arabirimi sağlar (Her ikisi de isteğe bağlı olarak ek güvenlik için TLS'YE sarılabilir): - [HTTP](http.md), belgelenmiş ve doğrudan kullanımı kolay olan. - [Yerel TCP](tcp.md) daha az yükü olan. diff --git a/docs/tr/interfaces/jdbc.md b/docs/tr/interfaces/jdbc.md index a7e69550c5a..b96cecf5116 100644 --- a/docs/tr/interfaces/jdbc.md +++ b/docs/tr/interfaces/jdbc.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 22 toc_title: "JDBC s\xFCr\xFCc\xFCs\xFC" --- diff --git a/docs/tr/interfaces/mysql.md b/docs/tr/interfaces/mysql.md index 97658803589..72df467d0ad 100644 --- a/docs/tr/interfaces/mysql.md +++ b/docs/tr/interfaces/mysql.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 20 toc_title: "MySQL Aray\xFCz\xFC" --- diff --git a/docs/tr/interfaces/odbc.md b/docs/tr/interfaces/odbc.md index 3cc0cc35700..0605a2a6aaa 100644 --- a/docs/tr/interfaces/odbc.md +++ b/docs/tr/interfaces/odbc.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 23 toc_title: "ODBC s\xFCr\xFCc\xFCs\xFC" --- diff --git a/docs/tr/interfaces/tcp.md b/docs/tr/interfaces/tcp.md index b5382ae8ec3..6fbe2d34223 100644 --- a/docs/tr/interfaces/tcp.md +++ b/docs/tr/interfaces/tcp.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 18 toc_title: Yerel arabirim (TCP) --- -# Yerel Arabirim (TCP) {#native-interface-tcp} +# Yerel arabirim (TCP) {#native-interface-tcp} Yerel protokol kullanılır [komut satırı istemcisi](cli.md), dağıtılmış sorgu işleme sırasında ve ayrıca diğer C++ programlarında sunucular arası iletişim için. Ne yazık ki, yerel ClickHouse protokolü henüz resmi bir spesifikasyona sahip değildir, ancak ClickHouse kaynak kodundan ters mühendislik yapılabilir (başlangıç [bu civarda](https://github.com/ClickHouse/ClickHouse/tree/master/src/Client)) ve / veya TCP trafiğini ele alarak ve analiz ederek. diff --git a/docs/tr/interfaces/third-party/client-libraries.md b/docs/tr/interfaces/third-party/client-libraries.md index a676cec9885..b9940b557b5 100644 --- a/docs/tr/interfaces/third-party/client-libraries.md +++ b/docs/tr/interfaces/third-party/client-libraries.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 26 toc_title: "\u0130stemci Kitapl\u0131klar\u0131" --- -# Üçüncü Taraf geliştiricilerin Istemci kitaplıkları {#client-libraries-from-third-party-developers} +# Üçüncü taraf geliştiricilerin istemci kitaplıkları {#client-libraries-from-third-party-developers} !!! warning "Uyarı" Yandex yapar **değil** Aşağıda listelenen kütüphaneleri koruyun ve kalitelerini sağlamak için kapsamlı bir test yapmadınız. @@ -53,6 +53,7 @@ toc_title: "\u0130stemci Kitapl\u0131klar\u0131" - [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net) - İksir - [clickhousex](https://github.com/appodeal/clickhousex/) + - [sütun](https://github.com/sofakingworld/pillar) - Nim - [nim-clickhouse](https://github.com/leonardoce/nim-clickhouse) diff --git a/docs/tr/interfaces/third-party/gui.md b/docs/tr/interfaces/third-party/gui.md index 91bd1e399de..d67478fe2b4 100644 --- a/docs/tr/interfaces/third-party/gui.md +++ b/docs/tr/interfaces/third-party/gui.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 28 toc_title: "G\xF6rsel Aray\xFCzler" --- -# Üçüncü Taraf geliştiricilerin görsel arayüzleri {#visual-interfaces-from-third-party-developers} +# Üçüncü taraf geliştiricilerin görsel arayüzleri {#visual-interfaces-from-third-party-developers} ## Açık Kaynak {#open-source} @@ -93,6 +93,10 @@ ClickHouse dahil olmak üzere birden fazla veri kaynağı için destekler, Redas [clickhouse-flamegraph](https://github.com/Slach/clickhouse-flamegraph) görselleştirmek için özel bir araçtır `system.trace_log` olarak [flamegraph](http://www.brendangregg.com/flamegraphs.html). +### clickhouse-plantuml {#clickhouse-plantuml} + +[cickhouse-plantuml](https://pypi.org/project/clickhouse-plantuml/) oluşturmak için bir komut dosyası mı [PlantUML](https://plantuml.com/) tablo şemalarının diyagramı. + ## Ticari {#commercial} ### Datriagrpip {#datagrip} @@ -103,7 +107,7 @@ ClickHouse dahil olmak üzere birden fazla veri kaynağı için destekler, Redas - Çok hızlı kod tamamlama. - ClickHouse sözdizimi vurgulama. -- Clickhouse’a özgü özellikler için destek, örneğin iç içe geçmiş sütunlar, tablo motorları. +- Clickhouse'a özgü özellikler için destek, örneğin iç içe geçmiş sütunlar, tablo motorları. - Veri Editörü. - Refactorings. - Arama ve navigasyon. diff --git a/docs/tr/interfaces/third-party/index.md b/docs/tr/interfaces/third-party/index.md index d8332c00c26..c9c66fc1c64 100644 --- a/docs/tr/interfaces/third-party/index.md +++ b/docs/tr/interfaces/third-party/index.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_folder_title: "\xDC\xE7\xFCnc\xFC Taraf" toc_priority: 24 --- diff --git a/docs/tr/interfaces/third-party/integrations.md b/docs/tr/interfaces/third-party/integrations.md index a076a5e2558..91f0151fa8a 100644 --- a/docs/tr/interfaces/third-party/integrations.md +++ b/docs/tr/interfaces/third-party/integrations.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 27 toc_title: Entegrasyonlar --- -# Üçüncü Taraf geliştiricilerin Entegrasyon kütüphaneleri {#integration-libraries-from-third-party-developers} +# Üçüncü taraf geliştiricilerin entegrasyon kütüphaneleri {#integration-libraries-from-third-party-developers} !!! warning "Uyarı" Yandex yapar **değil** Aşağıda listelenen araçları ve kütüphaneleri koruyun ve kalitelerini sağlamak için kapsamlı bir test yapmadınız. @@ -26,7 +26,10 @@ toc_title: Entegrasyonlar - [ClickHouseMigrator](https://github.com/zlzforever/ClickHouseMigrator) - Mesaj kuyrukları - [Kafka](https://kafka.apache.org) - - [clickhouse\_sinker](https://github.com/housepower/clickhouse_sinker) (kullanma [Go client](https://github.com/kshvakov/clickhouse/)) + - [clickhouse\_sinker](https://github.com/housepower/clickhouse_sinker) (kullanma [Go client](https://github.com/ClickHouse/clickhouse-go/)) +- Akış işleme + - [Flink](https://flink.apache.org) + - [flink-clickhouse-lavabo](https://github.com/ivi-ru/flink-clickhouse-sink) - Nesne depoları - [S3](https://en.wikipedia.org/wiki/Amazon_S3) - [clickhouse-yedekleme](https://github.com/AlexAkulov/clickhouse-backup) @@ -75,7 +78,7 @@ toc_title: Entegrasyonlar - [Pandalar](https://pandas.pydata.org) - [pandahouse](https://github.com/kszucs/pandahouse) - PHP - - [Doctrine](https://www.doctrine-project.org/) + - [Doktrin](https://www.doctrine-project.org/) - [dbal-clickhouse](https://packagist.org/packages/friendsofdoctrine/dbal-clickhouse) - R - [dplyr](https://db.rstudio.com/dplyr/) diff --git a/docs/tr/interfaces/third-party/proxy.md b/docs/tr/interfaces/third-party/proxy.md index e25ccf50ca2..c326b432f7c 100644 --- a/docs/tr/interfaces/third-party/proxy.md +++ b/docs/tr/interfaces/third-party/proxy.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 29 toc_title: Vekiller --- -# Üçüncü Taraf geliştiricilerin Proxy sunucuları {#proxy-servers-from-third-party-developers} +# Üçüncü taraf geliştiricilerin Proxy sunucuları {#proxy-servers-from-third-party-developers} ## chproxy {#chproxy} diff --git a/docs/tr/introduction/adopters.md b/docs/tr/introduction/adopters.md index d2509ff4256..444902e0b96 100644 --- a/docs/tr/introduction/adopters.md +++ b/docs/tr/introduction/adopters.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 8 toc_title: Benimseyenler --- @@ -8,76 +8,79 @@ toc_title: Benimseyenler # ClickHouse Benimseyenler {#clickhouse-adopters} !!! warning "Uyarı" - ClickHouse ve onların Başarı Öyküleri kullanarak şirketlerin aşağıdaki liste kamu kaynaklarından monte edilir, böylece mevcut gerçeklikten farklı olabilir. Şirketinizde Clickhouse’u benimseme hikayesini paylaşırsanız memnun oluruz ve [listeye Ekle](https://github.com/ClickHouse/ClickHouse/edit/master/docs/en/introduction/adopters.md), ancak lütfen bunu yaparak herhangi bir NDA sorun yaşamayacağınızdan emin olun. Diğer şirketlerden gelen yayınlarla güncellemeler sağlamak da yararlıdır. + ClickHouse ve onların Başarı Öyküleri kullanarak şirketlerin aşağıdaki liste kamu kaynaklarından monte edilir, böylece mevcut gerçeklikten farklı olabilir. Şirketinizde Clickhouse'u benimseme hikayesini paylaşırsanız memnun oluruz ve [listeye Ekle](https://github.com/ClickHouse/ClickHouse/edit/master/docs/en/introduction/adopters.md), ancak lütfen bunu yaparak herhangi bir NDA sorun yaşamayacağınızdan emin olun. Diğer şirketlerden gelen yayınlarla güncellemeler sağlamak da yararlıdır. -| Şirket | Sektör | Usecase | Küme Boyutu | (Un) Sıkıştırılmış Veri Boyutu\* | Başvurma | -|----------------------------------------------------------------------------------------------------------|---------------------------------|-----------------------------|------------------------------------------------|------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [2gıs](https://2gis.ru) | Haritalar | İzleme | — | — | [Rusça konuşun, Temmuz 2019](https://youtu.be/58sPkXfq6nw) | -| [Aloha Tarayıcı](https://alohabrowser.com/) | Mobil Uygulama | Tarayıcı arka ucu | — | — | [Rusça slaytlar, Mayıs 2019](https://github.com/yandex/clickhouse-presentations/blob/master/meetup22/aloha.pdf) | -| [Amadeus](https://amadeus.com/) | Seyahat | Analiz | — | — | [Basın Bülteni, Nisan 2018](https://www.altinity.com/blog/2018/4/5/amadeus-technologies-launches-investment-and-insights-tool-based-on-machine-learning-and-strategy-algorithms) | -| [Appsflyer](https://www.appsflyer.com) | Mobil analitik | Ana ürün | — | — | [Rusça konuşun, Temmuz 2019](https://www.youtube.com/watch?v=M3wbRlcpBbY) | -| [ArenaData](https://arenadata.tech/) | Veri Platformu | Ana ürün | — | — | [Rusça slaytlar, Aralık 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup38/indexes.pdf) | -| [Badoo](https://badoo.com) | Çıkma | Timeseries | — | — | [Rusça slaytlar, Aralık 2019](https://presentations.clickhouse.tech/meetup38/forecast.pdf) | -| [Benocs](https://www.benocs.com/) | Ağ telemetri ve analitik | Ana Ürün | — | — | [İngilizce slaytlar, Ekim 2017](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup9/lpm.pdf) | -| [Bloomberg](https://www.bloomberg.com/) | Finans, Medya | İzleme | 102 sunucular | — | [Slaytlar, Mayıs 2018](https://www.slideshare.net/Altinity/http-analytics-for-6m-requests-per-second-using-clickhouse-by-alexander-bocharov) | -| [Bloxy](https://bloxy.info) | Blockchain | Analiz | — | — | [Rusça slaytlar, Ağustos 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/4_bloxy.pptx) | -| `Dataliance/UltraPower` | Telekom | Analiz | — | — | [Çince slaytlar, Ocak 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup12/telecom.pdf) | -| [CARTO](https://carto.com/) | İş Zek Businessası | G geoeo analyt analyticsics | — | — | [ClickHouse ile coğrafi işleme](https://carto.com/blog/geospatial-processing-with-clickhouse/) | -| [CERN](http://public.web.cern.ch/public/) | Araştırma | Deney | — | — | [Basın bülteni, Nisan 2012](https://www.yandex.com/company/press_center/press_releases/2012/2012-04-10/) | -| [Cisco](http://cisco.com/) | Ağ | Trafik analizi | — | — | [Yıldırım konuşması, Ekim 2019](https://youtu.be/-hI1vDR2oPY?t=5057) | -| [Kale Menkul Değerler](https://www.citadelsecurities.com/) | Finansman | — | — | — | [Katkı, Mart 2019](https://github.com/ClickHouse/ClickHouse/pull/4774) | -| [Citymobil](https://city-mobil.ru) | Taksicilik | Analiz | — | — | [Rusça blog yazısı, Mart 2020](https://habr.com/en/company/citymobil/blog/490660/) | -| [ContentSquare](https://contentsquare.com) | Web analyt webics | Ana ürün | — | — | [Fransızca Blog yazısı, Kasım 2018](http://souslecapot.net/2018/11/21/patrick-chatain-vp-engineering-chez-contentsquare-penser-davantage-amelioration-continue-que-revolution-constante/) | -| [Cloudflare](https://cloudflare.com) | CDN | Trafik analizi | 36 sunucu | — | [Blog yazısı, Mayıs 2017](https://blog.cloudflare.com/how-cloudflare-analyzes-1m-dns-queries-per-second/), [Blog yazısı, Mart 2018](https://blog.cloudflare.com/http-analytics-for-6m-requests-per-second-using-clickhouse/) | -| [Corunet](https://coru.net/) | Analiz | Ana ürün | — | — | [İngilizce slaytlar, Nisan 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup21/predictive_models.pdf) | -| [CraiditX 氪信](https://creditx.com) | Finans AI | Analiz | — | — | [İngilizce slaytlar, Kasım 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/udf.pptx) | -| [Criteo/Storetail](https://www.criteo.com/) | Perakendecilik | Ana ürün | — | — | [İngilizce slaytlar, Ekim 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/3_storetail.pptx) | -| [Deut Banksche Bank](https://db.com) | Finansman | Bİ analitik | — | — | [İngilizce slaytlar, Ekim 2019](https://bigdatadays.ru/wp-content/uploads/2019/10/D2-H3-3_Yakunin-Goihburg.pdf) | -| [Diva-e](https://www.diva-e.com) | Dijital danışmanlık | Ana Ürün | — | — | [İngilizce slaytlar, Eylül 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup29/ClickHouse-MeetUp-Unusual-Applications-sd-2019-09-17.pdf) | -| [Exness](https://www.exness.com) | Ticaret | Metrikler, Günlük Kaydı | — | — | [Rusça konuşun, Mayıs 2019](https://youtu.be/_rpU-TvSfZ8?t=3215) | -| [Geniee](https://geniee.co.jp) | Reklam Ağı | Ana ürün | — | — | [Japonca Blog yazısı, Temmuz 2017](https://tech.geniee.co.jp/entry/2017/07/20/160100) | -| [HUYA](https://www.huya.com/) | Video Akışı | Analiz | — | — | [Çince slaytlar, Ekim 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/7.%20ClickHouse万亿数据分析实践%20李本旺(sundy-li)%20虎牙.pdf) | -| [Idealista](https://www.idealista.com) | Emlak | Analiz | — | — | [İngilizce Blog yazısı, Nisan 2019](https://clickhouse.yandex/blog/en/clickhouse-meetup-in-madrid-on-april-2-2019) | -| [Infovista](https://www.infovista.com/) | Ağlar | Analiz | — | — | [İngilizce slaytlar, Ekim 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup30/infovista.pdf) | -| [Innogames](https://www.innogames.com) | Oyun | Metrikler, Günlük Kaydı | — | — | [Rusça slaytlar, Eylül 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/graphite_and_clickHouse.pdf) | -| [Integros](https://integros.com) | Video hizmetleri platformu | Analiz | — | — | [Rusça slaytlar, Mayıs 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | -| [Kodiak Verileri](https://www.kodiakdata.com/) | Bulutlar | Ana ürün | — | — | [Engish slaytlar, Nisan 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup13/kodiak_data.pdf) | -| [Kontur](https://kontur.ru) | Yazılım Geliştirme | Metrik | — | — | [Rusça konuşma, Kasım 2018](https://www.youtube.com/watch?v=U4u4Bd0FtrY) | -| [LifeStreet](https://lifestreet.com/) | Reklam Ağı | Ana ürün | 75 sunucu (3 kopya) | 5.27 Pıb | [Rusça Blog yazısı, Şubat 2017](https://habr.com/en/post/322620/) | -| [Mail.ru Bulut Çözümleri](https://mcs.mail.ru/) | Bulut hizmetleri | Ana ürün | — | — | [ClickHouse örneğini Rusça olarak çalıştırma](https://mcs.mail.ru/help/db-create/clickhouse#) | -| [MessageBird](https://www.messagebird.com) | Telekomünikasyonlar | İstatistik | — | — | [İngilizce slaytlar, Kasım 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup20/messagebird.pdf) | -| [MGID](https://www.mgid.com/) | Reklam Ağı | Web-analyt -ics | — | — | [Analitik DBMS ClickHouse uygulama deneyimimiz, Rusça](http://gs-studio.com/news-about-it/32777----clickhouse---c) | -| [OneAPM](https://www.oneapm.com/) | İzleme ve veri analizi | Ana ürün | — | — | [Çince slaytlar, Ekim 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/8.%20clickhouse在OneAPM的应用%20杜龙.pdf) | -| [Pragma Yenilik](http://www.pragma-innovation.fr/) | Telemetri ve büyük veri analizi | Ana ürün | — | — | [İngilizce slaytlar, Ekim 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/4_pragma_innovation.pdf) | -| [QINGCLOUD](https://www.qingcloud.com/) | Bulut hizmetleri | Ana ürün | — | — | [Çince slaytlar, Ekim 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/4.%20Cloud%20%2B%20TSDB%20for%20ClickHouse%20张健%20QingCloud.pdf) | -| [Qrator](https://qrator.net) | DDoS koruması | Ana ürün | — | — | [Blog Yazısı, Mart 2019](https://blog.qrator.net/en/clickhouse-ddos-mitigation_37/) | -| [Beijing per PERCENTC İnformationent Information Technology Co., Ltd. Ltd.Şti.](https://www.percent.cn/) | Analiz | Ana Ürün | — | — | [Çince slaytlar, Haziran 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/4.%20ClickHouse万亿数据双中心的设计与实践%20.pdf) | -| [Serseri](https://rambler.ru) | İnternet Hizmetleri | Analiz | — | — | [Rusça konuşma, Nisan 2018](https://medium.com/@ramblertop/разработка-api-clickhouse-для-рамблер-топ-100-f4c7e56f3141) | -| [Tencent](https://www.tencent.com) | Mesaj | Günlük | — | — | [Çince konuşun, Kasım 2019](https://youtu.be/T-iVQRuw-QY?t=5050) | -| [Trafik Yıldız Starsları](https://trafficstars.com/) | Reklam Ağı | — | — | — | [Rusça slaytlar, Mayıs 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup15/lightning/ninja.pdf) | -| [S7 Havayolları](https://www.s7.ru) | Havayolular | Metrikler, Günlük Kaydı | — | — | [Rusça konuş, Mart 2019](https://www.youtube.com/watch?v=nwG68klRpPg&t=15s) | -| [SEMrush](https://www.semrush.com/) | Pazarlamacı | Ana ürün | — | — | [Rusça slaytlar, Ağustos 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/5_semrush.pdf) | -| [scireum GmbH](https://www.scireum.de/) | e-ticaret | Ana ürün | — | — | [Almanca konuşma, Şubat 2020](https://www.youtube.com/watch?v=7QWAn5RbyR4) | -| [Nöbet](https://sentry.io/) | Yazılımcı | Ürün için arka uç | — | — | [İngilizce Blog yazısı, Mayıs 2019](https://blog.sentry.io/2019/05/16/introducing-snuba-sentrys-new-search-infrastructure) | -| [SGK](http://www.sgk.gov.tr/wps/portal/sgk/tr) | Devlet Sosyal Güvenlik | Analiz | — | — | [İngilizce slaytlar, Kasım 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup35/ClickHouse%20Meetup-Ramazan%20POLAT.pdf) | -| [seo.do](https://seo.do/) | Analiz | Ana ürün | — | — | [İngilizce slaytlar, Kasım 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup35/CH%20Presentation-%20Metehan%20Çetinkaya.pdf) | -| [Sina](http://english.sina.com/index.html) | Haberci | — | — | — | [Çince slaytlar, Ekim 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/6.%20ClickHouse最佳实践%20高鹏_新浪.pdf) | -| [SMI2](https://smi2.ru/) | Haberci | Analiz | — | — | [Rusça blog yazısı, Kasım 2017](https://habr.com/ru/company/smi2/blog/314558/) | -| [Splunk](https://www.splunk.com/) | İş Analitiği | Ana ürün | — | — | [İngilizce slaytlar, Ocak 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup12/splunk.pdf) | -| [Spotify](https://www.spotify.com) | Müzik | Deney | — | — | [Slaytlar, Temmuz 2018](https://www.slideshare.net/glebus/using-clickhouse-for-experimentation-104247173) | -| [Tencent](https://www.tencent.com) | Büyük Veri | Veri işleme | — | — | [Çince slaytlar, Ekim 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/5.%20ClickHouse大数据集群应用_李俊飞腾讯网媒事业部.pdf) | -| [Uber](https://www.uber.com) | Taksicilik | Günlük | — | — | [Slay ,tlar, Şubat 20 202020](https://presentations.clickhouse.tech/meetup40/uber.pdf) | -| [VKontakte](https://vk.com) | Sosyal Ağ | İstatistik, Günlük | — | — | [Rusça slaytlar, Ağustos 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/3_vk.pdf) | -| [Wisebits](https://wisebits.com/) | BT çözümleri | Analiz | — | — | [Rusça slaytlar, Mayıs 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | -| [Xiaoxin Tech.](https://www.xiaoheiban.cn/) | Eğitici | Ortak amaç | — | — | [İngilizce slaytlar, Kasım 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/sync-clickhouse-with-mysql-mongodb.pptx) | -| [Ximalaya](https://www.ximalaya.com/) | Ses paylaşımı | OLAP | — | — | [İngilizce slaytlar, Kasım 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/ximalaya.pdf) | -| [Yandex Bulut](https://cloud.yandex.ru/services/managed-clickhouse) | Genel Bulut | Ana ürün | — | — | [Rusça konuşun, Aralık 2019](https://www.youtube.com/watch?v=pgnak9e_E0o) | -| [Yandex DataLens](https://cloud.yandex.ru/services/datalens) | İş Zek Businessası | Ana ürün | — | — | [Rusça slaytlar, Aralık 2019](https://presentations.clickhouse.tech/meetup38/datalens.pdf) | -| [Yandex Pazarı](https://market.yandex.ru/) | e-ticaret | Metrikler, Günlük Kaydı | — | — | [Rusça konuşma, Ocak 2019](https://youtu.be/_l1qP0DyBcA?t=478) | -| [Yandex Metrica](https://metrica.yandex.com) | Web analyt webics | Ana ürün | Bir kümede 360 sunucu, bir bölümde 1862 sunucu | 66.41 Pıb / 5.68 Pıb | [Slay ,tlar, Şubat 20 202020](https://presentations.clickhouse.tech/meetup40/introduction/#13) | -| [ЦВТ](https://htc-cs.ru/) | Yazılım Geliştirme | Metrikler, Günlük Kaydı | — | — | [Blog yazısı, Mart 2019, Rusça](https://vc.ru/dev/62715-kak-my-stroili-monitoring-na-prometheus-clickhouse-i-elk) | -| [МКБ](https://mkb.ru/) | Banka | Web-sistem izleme | — | — | [Rusça slaytlar, Eylül 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/mkb.pdf) | -| [金数据](https://jinshuju.net) | Bİ analitik | Ana ürün | — | — | [Çince slaytlar, Ekim 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/3.%20金数据数据架构调整方案Public.pdf) | -| [Instana](https://www.instana.com) | APM Platformu | Ana ürün | — | — | [Twitter mesaj](https://twitter.com/mieldonkers/status/1248884119158882304) | +| Şirket | Sektör | Usecase | Küme Boyutu | (Un) Sıkıştırılmış Veri Boyutu\* | Başvurma | +|------------------------------------------------------------------------------------------------|---------------------------------|-----------------------------|------------------------------------------------|------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 2gıs | Haritalar | İzleme | — | — | [Rusça konuşun, Temmuz 2019](https://youtu.be/58sPkXfq6nw) | +| Aloha Browser | Mobil Uygulama | Tarayıcı arka ucu | — | — | [Rusça slaytlar, Mayıs 2019](https://github.com/yandex/clickhouse-presentations/blob/master/meetup22/aloha.pdf) | +| Amadeus | Seyahat | Analiz | — | — | [Basın Bülteni, Nisan 2018](https://www.altinity.com/blog/2018/4/5/amadeus-technologies-launches-investment-and-insights-tool-based-on-machine-learning-and-strategy-algorithms) | +| Appsflyer | Mobil analitik | Ana ürün | — | — | [Rusça konuşun, Temmuz 2019](https://www.youtube.com/watch?v=M3wbRlcpBbY) | +| ArenaData | Veri Platformu | Ana ürün | — | — | [Rusça slaytlar, Aralık 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup38/indexes.pdf) | +| Badoo | Çıkma | Timeseries | — | — | [Rusça slaytlar, Aralık 2019](https://presentations.clickhouse.tech/meetup38/forecast.pdf) | +| Benocs | Ağ telemetri ve analitik | Ana Ürün | — | — | [İngilizce slaytlar, Ekim 2017](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup9/lpm.pdf) | +| Bloomberg | Finans, Medya | İzleme | 102 sunucular | — | [Slaytlar, Mayıs 2018](https://www.slideshare.net/Altinity/http-analytics-for-6m-requests-per-second-using-clickhouse-by-alexander-bocharov) | +| Bloxy | Blockchain | Analiz | — | — | [Rusça slaytlar, Ağustos 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/4_bloxy.pptx) | +| Çin Telekom için Dataliance | Telekom | Analiz | — | — | [Çince slaytlar, Ocak 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup12/telecom.pdf) | +| CARTO | İş Zek Businessası | G geoeo analyt analyticsics | — | — | [ClickHouse ile coğrafi işleme](https://carto.com/blog/geospatial-processing-with-clickhouse/) | +| CERN | Araştırma | Deney | — | — | [Basın bülteni, Nisan 2012](https://www.yandex.com/company/press_center/press_releases/2012/2012-04-10/) | +| Cisco | Ağ | Trafik analizi | — | — | [Yıldırım konuşması, Ekim 2019](https://youtu.be/-hI1vDR2oPY?t=5057) | +| Citadel Securities | Finansman | — | — | — | [Katkı, Mart 2019](https://github.com/ClickHouse/ClickHouse/pull/4774) | +| Citymobil | Taksicilik | Analiz | — | — | [Rusça blog yazısı, Mart 2020](https://habr.com/en/company/citymobil/blog/490660/) | +| ContentSquare | Web analyt webics | Ana ürün | — | — | [Fransızca Blog yazısı, Kasım 2018](http://souslecapot.net/2018/11/21/patrick-chatain-vp-engineering-chez-contentsquare-penser-davantage-amelioration-continue-que-revolution-constante/) | +| Cloudflare | CDN | Trafik analizi | 36 sunucu | — | [Blog yazısı, Mayıs 2017](https://blog.cloudflare.com/how-cloudflare-analyzes-1m-dns-queries-per-second/), [Blog yazısı, Mart 2018](https://blog.cloudflare.com/http-analytics-for-6m-requests-per-second-using-clickhouse/) | +| Corunet | Analiz | Ana ürün | — | — | [İngilizce slaytlar, Nisan 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup21/predictive_models.pdf) | +| CraiditX 氪信 | Finans AI | Analiz | — | — | [İngilizce slaytlar, Kasım 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/udf.pptx) | +| Criteo | Perakendecilik | Ana ürün | — | — | [İngilizce slaytlar, Ekim 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/3_storetail.pptx) | +| Deutsche Bank | Finansman | Bİ analitik | — | — | [İngilizce slaytlar, Ekim 2019](https://bigdatadays.ru/wp-content/uploads/2019/10/D2-H3-3_Yakunin-Goihburg.pdf) | +| Diva-e | Dijital danışmanlık | Ana Ürün | — | — | [İngilizce slaytlar, Eylül 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup29/ClickHouse-MeetUp-Unusual-Applications-sd-2019-09-17.pdf) | +| Exness | Ticaret | Metrikler, Günlük Kaydı | — | — | [Rusça konuşun, Mayıs 2019](https://youtu.be/_rpU-TvSfZ8?t=3215) | +| Geniee | Reklam Ağı | Ana ürün | — | — | [Japonca Blog yazısı, Temmuz 2017](https://tech.geniee.co.jp/entry/2017/07/20/160100) | +| HUYA | Video Akışı | Analiz | — | — | [Çince slaytlar, Ekim 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/7.%20ClickHouse万亿数据分析实践%20李本旺(sundy-li)%20虎牙.pdf) | +| Idealista | Emlak | Analiz | — | — | [İngilizce Blog yazısı, Nisan 2019](https://clickhouse.yandex/blog/en/clickhouse-meetup-in-madrid-on-april-2-2019) | +| Infovista | Ağlar | Analiz | — | — | [İngilizce slaytlar, Ekim 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup30/infovista.pdf) | +| Innogames | Oyun | Metrikler, Günlük Kaydı | — | — | [Rusça slaytlar, Eylül 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/graphite_and_clickHouse.pdf) | +| Integros | Video hizmetleri platformu | Analiz | — | — | [Rusça slaytlar, Mayıs 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | +| Kodiak Verileri | Bulutlar | Ana ürün | — | — | [Engish slaytlar, Nisan 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup13/kodiak_data.pdf) | +| Kontur | Yazılım Geliştirme | Metrik | — | — | [Rusça konuşma, Kasım 2018](https://www.youtube.com/watch?v=U4u4Bd0FtrY) | +| LifeStreet | Reklam Ağı | Ana ürün | 75 sunucu (3 kopya) | 5.27 Pıb | [Rusça Blog yazısı, Şubat 2017](https://habr.com/en/post/322620/) | +| Mail.ru Bulut Çözümleri | Bulut hizmetleri | Ana ürün | — | — | [Rusça makale](https://mcs.mail.ru/help/db-create/clickhouse#) | +| MessageBird | Telekomünikasyonlar | İstatistik | — | — | [İngilizce slaytlar, Kasım 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup20/messagebird.pdf) | +| MGID | Reklam Ağı | Web-analyt -ics | — | — | [Rusça blog yazısı, Nisan 2020](http://gs-studio.com/news-about-it/32777----clickhouse---c) | +| OneAPM | İzleme ve veri analizi | Ana ürün | — | — | [Çince slaytlar, Ekim 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/8.%20clickhouse在OneAPM的应用%20杜龙.pdf) | +| Pragma Innovation | Telemetri ve büyük veri analizi | Ana ürün | — | — | [İngilizce slaytlar, Ekim 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/4_pragma_innovation.pdf) | +| QINGCLOUD | Bulut hizmetleri | Ana ürün | — | — | [Çince slaytlar, Ekim 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/4.%20Cloud%20%2B%20TSDB%20for%20ClickHouse%20张健%20QingCloud.pdf) | +| Qrator | DDoS koruması | Ana ürün | — | — | [Blog Yazısı, Mart 2019](https://blog.qrator.net/en/clickhouse-ddos-mitigation_37/) | +| Percent 百分点 | Analiz | Ana Ürün | — | — | [Çince slaytlar, Haziran 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/4.%20ClickHouse万亿数据双中心的设计与实践%20.pdf) | +| Serseri | İnternet Hizmetleri | Analiz | — | — | [Rusça konuşma, Nisan 2018](https://medium.com/@ramblertop/разработка-api-clickhouse-для-рамблер-топ-100-f4c7e56f3141) | +| Tencent | Mesaj | Günlük | — | — | [Çince konuşun, Kasım 2019](https://youtu.be/T-iVQRuw-QY?t=5050) | +| Traffic Stars | Reklam Ağı | — | — | — | [Rusça slaytlar, Mayıs 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup15/lightning/ninja.pdf) | +| S7 Airlines | Havayolular | Metrikler, Günlük Kaydı | — | — | [Rusça konuş, Mart 2019](https://www.youtube.com/watch?v=nwG68klRpPg&t=15s) | +| SEMrush | Pazarlama | Ana ürün | — | — | [Rusça slaytlar, Ağustos 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/5_semrush.pdf) | +| scireum GmbH | e-ticaret | Ana ürün | — | — | [Almanca konuşma, Şubat 2020](https://www.youtube.com/watch?v=7QWAn5RbyR4) | +| Nöbet | Yazılımcı | Ürün için arka uç | — | — | [İngilizce Blog yazısı, Mayıs 2019](https://blog.sentry.io/2019/05/16/introducing-snuba-sentrys-new-search-infrastructure) | +| SGK | Devlet Sosyal Güvenlik | Analiz | — | — | [İngilizce slaytlar, Kasım 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup35/ClickHouse%20Meetup-Ramazan%20POLAT.pdf) | +| seo.do | Analiz | Ana ürün | — | — | [İngilizce slaytlar, Kasım 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup35/CH%20Presentation-%20Metehan%20Çetinkaya.pdf) | +| Sina | Haberci | — | — | — | [Çince slaytlar, Ekim 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/6.%20ClickHouse最佳实践%20高鹏_新浪.pdf) | +| SMI2 | Haberci | Analiz | — | — | [Rusça blog yazısı, Kasım 2017](https://habr.com/ru/company/smi2/blog/314558/) | +| Splunk | İş Analitiği | Ana ürün | — | — | [İngilizce slaytlar, Ocak 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup12/splunk.pdf) | +| Spotify | Müzik | Deney | — | — | [Slaytlar, Temmuz 2018](https://www.slideshare.net/glebus/using-clickhouse-for-experimentation-104247173) | +| Tencent | Büyük Veri | Veri işleme | — | — | [Çince slaytlar, Ekim 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/5.%20ClickHouse大数据集群应用_李俊飞腾讯网媒事业部.pdf) | +| Uber | Taksicilik | Günlük | — | — | [Slay ,tlar, Şubat 20 202020](https://presentations.clickhouse.tech/meetup40/uber.pdf) | +| VKontakte | Sosyal Ağ | İstatistik, Günlük | — | — | [Rusça slaytlar, Ağustos 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/3_vk.pdf) | +| Wisebits | BT çözümleri | Analiz | — | — | [Rusça slaytlar, Mayıs 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | +| Xiaoxin Tech | Eğitici | Ortak amaç | — | — | [İngilizce slaytlar, Kasım 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/sync-clickhouse-with-mysql-mongodb.pptx) | +| Ximalaya | Ses paylaşımı | OLAP | — | — | [İngilizce slaytlar, Kasım 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/ximalaya.pdf) | +| Yandex Cloud | Genel Bulut | Ana ürün | — | — | [Rusça konuşun, Aralık 2019](https://www.youtube.com/watch?v=pgnak9e_E0o) | +| Yandex DataLens | İş Zek Businessası | Ana ürün | — | — | [Rusça slaytlar, Aralık 2019](https://presentations.clickhouse.tech/meetup38/datalens.pdf) | +| Yandex Market | e-ticaret | Metrikler, Günlük Kaydı | — | — | [Rusça konuşma, Ocak 2019](https://youtu.be/_l1qP0DyBcA?t=478) | +| Yandex Metrica | Web analyt webics | Ana ürün | Bir kümede 360 sunucu, bir bölümde 1862 sunucu | 66.41 Pıb / 5.68 Pıb | [Slay ,tlar, Şubat 20 202020](https://presentations.clickhouse.tech/meetup40/introduction/#13) | +| ЦВТ | Yazılım Geliştirme | Metrikler, Günlük Kaydı | — | — | [Blog yazısı, Mart 2019, Rusça](https://vc.ru/dev/62715-kak-my-stroili-monitoring-na-prometheus-clickhouse-i-elk) | +| МКБ | Banka | Web-sistem izleme | — | — | [Rusça slaytlar, Eylül 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/mkb.pdf) | +| Jinshuju 金数据 | Bİ analitik | Ana ürün | — | — | [Çince slaytlar, Ekim 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/3.%20金数据数据架构调整方案Public.pdf) | +| Instana | APM Platformu | Ana ürün | — | — | [Twitter mesaj](https://twitter.com/mieldonkers/status/1248884119158882304) | +| Wargaming | Oyun | | — | — | [Röportaj](https://habr.com/en/post/496954/) | +| Crazypanda | Oyun | | — | — | ClickHouse meetup canlı oturum | +| FunCorp | Oyun | | — | — | [Makale](https://www.altinity.com/blog/migrating-from-redshift-to-clickhouse) | [Orijinal makale](https://clickhouse.tech/docs/en/introduction/adopters/) diff --git a/docs/tr/introduction/distinctive-features.md b/docs/tr/introduction/distinctive-features.md index 996a182b676..be7fd6ff160 100644 --- a/docs/tr/introduction/distinctive-features.md +++ b/docs/tr/introduction/distinctive-features.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 4 toc_title: "Ay\u0131rt Edici \xD6zellikler" --- -# Clickhouse’un ayırt Edici özellikleri {#distinctive-features-of-clickhouse} +# Clickhouse'un ayırt edici özellikleri {#distinctive-features-of-clickhouse} ## Doğru sütun yönelimli DBMS {#true-column-oriented-dbms} @@ -13,7 +13,7 @@ Bir gerçek sütun yönelimli DBMS, hiçbir ek veri değerleri ile depolanır. D Farklı sütunların değerlerini ayrı ayrı depolayabilen, ancak diğer senaryolar için optimizasyonları nedeniyle analitik sorguları etkili bir şekilde işleyemeyen sistemler olduğu için dikkat çekicidir. Örnekler HBase, BigTable, Cassandra ve HyperTable. Bu sistemlerde, saniyede yüz bin satır civarında verim elde edersiniz, ancak saniyede yüz milyonlarca satır olmaz. -Clickhouse’un tek bir veritabanı değil, bir veritabanı yönetim sistemi olduğunu da belirtmek gerekir. ClickHouse, çalışma zamanında tablolar ve veritabanları oluşturmak, veri yüklemek ve sunucuyu yeniden yapılandırmadan ve yeniden başlatmadan sorguları çalıştırmaya izin verir. +Clickhouse'un tek bir veritabanı değil, bir veritabanı yönetim sistemi olduğunu da belirtmek gerekir. ClickHouse, çalışma zamanında tablolar ve veritabanları oluşturmak, veri yüklemek ve sunucuyu yeniden yapılandırmadan ve yeniden başlatmadan sorguları çalıştırmaya izin verir. ## Veri Sıkıştırma {#data-compression} @@ -21,20 +21,20 @@ Bazı sütun yönelimli DBMSs (InfiniDB CE ve MonetDB) veri sıkıştırma kulla ## Verilerin Disk Depolama {#disk-storage-of-data} -Verileri fiziksel olarak birincil anahtara göre sıralamak, belirli değerleri veya değer aralıkları için düşük gecikme süresi ile birkaç düzine milisaniyeden daha az veri ayıklamayı mümkün kılar. Bazı sütun yönelimli Dbms’ler (SAP HANA ve Google PowerDrill gibi) yalnızca RAM’de çalışabilir. Bu yaklaşım, gerçek zamanlı analiz için gerekenden daha büyük bir donanım bütçesinin tahsisini teşvik eder. ClickHouse düzenli sabit diskler üzerinde çalışmak üzere tasarlanmıştır, bu da GB veri depolama başına maliyetin düşük olduğu anlamına gelir, ancak varsa SSD ve ek RAM de tamamen kullanılır. +Verileri fiziksel olarak birincil anahtara göre sıralamak, belirli değerleri veya değer aralıkları için düşük gecikme süresi ile birkaç düzine milisaniyeden daha az veri ayıklamayı mümkün kılar. Bazı sütun yönelimli Dbms'ler (SAP HANA ve Google PowerDrill gibi) yalnızca RAM'de çalışabilir. Bu yaklaşım, gerçek zamanlı analiz için gerekenden daha büyük bir donanım bütçesinin tahsisini teşvik eder. ClickHouse düzenli sabit diskler üzerinde çalışmak üzere tasarlanmıştır, bu da GB veri depolama başına maliyetin düşük olduğu anlamına gelir, ancak varsa SSD ve ek RAM de tamamen kullanılır. -## Birden Fazla çekirdekte Paralel işleme {#parallel-processing-on-multiple-cores} +## Birden fazla çekirdekte paralel işleme {#parallel-processing-on-multiple-cores} Büyük sorgular, geçerli sunucuda bulunan tüm gerekli kaynakları alarak doğal olarak paralelleştirilir. -## Birden çok Sunucuda dağıtılmış işleme {#distributed-processing-on-multiple-servers} +## Birden çok sunucuda dağıtılmış işleme {#distributed-processing-on-multiple-servers} -Yukarıda belirtilen sütunlu Dbms’lerin neredeyse hiçbiri dağıtılmış sorgu işleme desteğine sahip değildir. -Clickhouse’da, veriler farklı parçalarda bulunabilir. Her parça, hata toleransı için kullanılan bir grup kopya olabilir. Tüm kırıklar, kullanıcı için şeffaf olarak paralel bir sorgu çalıştırmak için kullanılır. +Yukarıda belirtilen sütunlu Dbms'lerin neredeyse hiçbiri dağıtılmış sorgu işleme desteğine sahip değildir. +Clickhouse'da, veriler farklı parçalarda bulunabilir. Her parça, hata toleransı için kullanılan bir grup kopya olabilir. Tüm kırıklar, kullanıcı için şeffaf olarak paralel bir sorgu çalıştırmak için kullanılır. ## SQL desteği {#sql-support} -ClickHouse, çoğu durumda SQL standardına özdeş olan sql’i temel alan bildirime dayalı bir sorgu dilini destekler. +ClickHouse, çoğu durumda SQL standardına özdeş olan sql'i temel alan bildirime dayalı bir sorgu dilini destekler. Desteklenen sorgular arasında GROUP BY, ORDER BY, from, ın ve JOIN yan tümceleri ve skaler alt sorgular bulunur. Bağımlı alt sorgular ve pencere işlevleri desteklenmez. @@ -42,7 +42,7 @@ Bağımlı alt sorgular ve pencere işlevleri desteklenmez. Veriler yalnızca sütunlar tarafından saklanmakla kalmaz, aynı zamanda yüksek CPU verimliliği elde etmeyi sağlayan vektörler (sütunların parçaları) tarafından işlenir. -## Gerçek zamanlı Veri güncellemeleri {#real-time-data-updates} +## Gerçek zamanlı veri güncellemeleri {#real-time-data-updates} ClickHouse, birincil anahtarlı tabloları destekler. Birincil anahtar aralığındaki sorguları hızlı bir şekilde gerçekleştirmek için, veriler birleştirme ağacını kullanarak aşamalı olarak sıralanır. Bu nedenle, veriler sürekli olarak tabloya eklenebilir. Yeni veri Yutulduğunda hiçbir kilit alınır. @@ -50,11 +50,11 @@ ClickHouse, birincil anahtarlı tabloları destekler. Birincil anahtar aralığ Birincil anahtara göre fiziksel olarak sıralanmış bir veriye sahip olmak, belirli değerleri veya değer aralıkları için düşük gecikme süresi ile birkaç düzine milisaniyeden daha az veri çıkarmayı mümkün kılar. -## Çevrimiçi Sorgular için Uygundur {#suitable-for-online-queries} +## Çevrimiçi sorgular için uygundur {#suitable-for-online-queries} Düşük gecikme süresi, kullanıcı arayüzü sayfası yüklenirken, sorguların gecikmeden ve önceden bir cevap hazırlamaya çalışmadan işlenebileceği anlamına gelir. Başka bir deyişle, çevrimiçi. -## Yaklaşık Hesaplamalar için Destek {#support-for-approximated-calculations} +## Yaklaşık hesaplamalar için destek {#support-for-approximated-calculations} ClickHouse performans için doğruluk ticaret için çeşitli yollar sağlar: @@ -62,16 +62,16 @@ ClickHouse performans için doğruluk ticaret için çeşitli yollar sağlar: 2. Verilerin bir bölümünü (örnek) temel alan bir sorguyu çalıştırmak ve yaklaşık bir sonuç almak. Bu durumda, diskten orantılı olarak daha az veri alınır. 3. Tüm anahtarlar yerine, sınırlı sayıda rastgele anahtar için bir toplama çalıştırma. Verilerde anahtar dağıtımı için belirli koşullar altında, bu daha az kaynak kullanırken makul derecede doğru bir sonuç sağlar. -## Veri çoğaltma Ve Veri Bütünlüğü desteği {#data-replication-and-data-integrity-support} +## Veri çoğaltma ve Veri Bütünlüğü desteği {#data-replication-and-data-integrity-support} ClickHouse zaman uyumsuz çoklu ana çoğaltma kullanır. Kullanılabilir herhangi bir yineleme için yazıldıktan sonra kalan tüm yinelemeler arka planda kendi kopyasını almak. Sistem, farklı yinelemelerde aynı verileri korur. Çoğu arızadan sonra kurtarma, karmaşık durumlarda otomatik olarak veya yarı otomatik olarak gerçekleştirilir. Daha fazla bilgi için bölüme bakın [Veri çoğaltma](../engines/table-engines/mergetree-family/replication.md). -## Dezavantajları Olarak Kabul Edilebilir özellikler {#clickhouse-features-that-can-be-considered-disadvantages} +## Dezavantajları olarak kabul edilebilir özellikler {#clickhouse-features-that-can-be-considered-disadvantages} 1. Tam teşekküllü işlemler yok. 2. Yüksek oranda ve düşük gecikme ile zaten eklenen verileri değiştirme veya silme yeteneği eksikliği. Verileri temizlemek veya değiştirmek için toplu silme ve güncellemeler vardır, örneğin Aşağıdakilere uymak için [GDPR](https://gdpr-info.eu). -3. Seyrek dizin, Clickhouse’u anahtarlarıyla tek satırları almak için nokta sorguları için çok uygun değildir. +3. Seyrek dizin, Clickhouse'u anahtarlarıyla tek satırları almak için nokta sorguları için çok uygun değildir. [Orijinal makale](https://clickhouse.tech/docs/en/introduction/distinctive_features/) diff --git a/docs/tr/introduction/history.md b/docs/tr/introduction/history.md index 2f32929fe90..8d4e2cc3ed2 100644 --- a/docs/tr/introduction/history.md +++ b/docs/tr/introduction/history.md @@ -1,26 +1,26 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 7 -toc_title: "Tarih\xE7e" +toc_title: Tarih --- # ClickHouse Geçmişi {#clickhouse-history} -ClickHouse güç başlangıçta geliştirilmiştir [Üye.Metrica](https://metrica.yandex.com/), [dünyanın en büyük ikinci web analiz platformu](http://w3techs.com/technologies/overview/traffic_analysis/all) ve bu sistemin temel bileşeni olmaya devam ediyor. Veritabanında 13 trilyondan fazla kayıt ve günlük 20 milyardan fazla etkinlik ile ClickHouse, doğrudan toplanmamış verilerden anında özel raporlar oluşturmanıza olanak tanır. Bu makale Kısaca Clickhouse’un gelişiminin ilk aşamalarında hedeflerini kapsamaktadır. +ClickHouse güç başlangıçta geliştirilmiştir [Üye.Metrica](https://metrica.yandex.com/), [dünyanın en büyük ikinci web analiz platformu](http://w3techs.com/technologies/overview/traffic_analysis/all) ve bu sistemin temel bileşeni olmaya devam ediyor. Veritabanında 13 trilyondan fazla kayıt ve günlük 20 milyardan fazla etkinlik ile ClickHouse, doğrudan toplanmamış verilerden anında özel raporlar oluşturmanıza olanak tanır. Bu makale Kısaca Clickhouse'un gelişiminin ilk aşamalarında hedeflerini kapsamaktadır. Üye.Metrica kullanıcı tarafından tanımlanan keyfi kesimleri ile, hit ve oturumları dayalı anında özelleştirilmiş raporlar oluşturur. Bunu sık sık yapmak, benzersiz kullanıcı sayısı gibi karmaşık agregalar oluşturmayı gerektirir. Bir rapor oluşturmak için yeni veriler gerçek zamanlı olarak gelir. Nisan 2014 itibariyle, Yandex.Metrica, günlük olarak yaklaşık 12 milyar olayı (sayfa görüntüleme ve tıklama) izliyordu. Tüm bu olaylar özel raporlar oluşturmak için saklanmalıdır. Tek bir sorgu, birkaç yüz milisaniye içinde milyonlarca satırı veya sadece birkaç saniye içinde yüz milyonlarca satırı taramayı gerektirebilir. -## Yandex kullanımı.Metrica Ve diğer Yandex Hizmetleri {#usage-in-yandex-metrica-and-other-yandex-services} +## Yandex kullanımı.Metrica ve diğer Yandex Hizmetleri {#usage-in-yandex-metrica-and-other-yandex-services} -ClickHouse, Yandex’te birden fazla amaca hizmet eder.Metrica. -Ana görevi, toplanmamış verileri kullanarak çevrimiçi modda raporlar oluşturmaktır. Veritabanında 20.3 trilyon satırdan fazla depolayan 374 sunucu kümesi kullanır. Sıkıştırılmış verilerin hacmi, yinelenenleri ve kopyaları hesaba katmadan yaklaşık 2 PB’DİR. Sıkıştırılmamış verilerin hacmi (TSV formatında) Yaklaşık 17 PB olacaktır. +ClickHouse, Yandex'te birden fazla amaca hizmet eder.Metrica. +Ana görevi, toplanmamış verileri kullanarak çevrimiçi modda raporlar oluşturmaktır. Veritabanında 20.3 trilyon satırdan fazla depolayan 374 sunucu kümesi kullanır. Sıkıştırılmış verilerin hacmi, yinelenenleri ve kopyaları hesaba katmadan yaklaşık 2 PB'DİR. Sıkıştırılmamış verilerin hacmi (TSV formatında) Yaklaşık 17 PB olacaktır. ClickHouse ayrıca aşağıdaki süreçlerde önemli bir rol oynar: -- Yandex’den oturum tekrarı için veri saklama.Metrica. +- Yandex'den oturum tekrarı için veri saklama.Metrica. - Ara veri işleme. - Analitik ile küresel raporlar oluşturma. - Yandex hata ayıklama için sorguları çalıştırma.Metrica motoru. @@ -28,7 +28,7 @@ ClickHouse ayrıca aşağıdaki süreçlerde önemli bir rol oynar: Günümüzde, diğer Yandex hizmetlerinde ve bölümlerinde birden fazla düzine ClickHouse kurulumu bulunmaktadır: arama dikey, e-ticaret, reklam, iş analitiği, mobil geliştirme, kişisel hizmetler ve diğerleri. -## Toplanmış Ve toplanmamış Veriler {#aggregated-and-non-aggregated-data} +## Toplanmış ve toplanmamış veriler {#aggregated-and-non-aggregated-data} İstatistikleri etkili bir şekilde hesaplamak için, veri hacmini azalttığından verileri toplamanız gerektiğine dair yaygın bir görüş vardır. @@ -38,7 +38,7 @@ Ancak veri toplama birçok sınırlama ile birlikte gelir: - Kullanıcı özel raporlar yapamaz. - Çok sayıda farklı anahtar üzerinde toplanırken, veri hacmi zorlukla azaltılır, bu nedenle toplama işe yaramaz. - Çok sayıda rapor için çok fazla toplama varyasyonu vardır (kombinatoryal patlama). -- Anahtarları yüksek önemlilik (URL’ler gibi) ile toplarken, veri hacmi çok fazla azaltılmaz (iki kattan daha az). +- Anahtarları yüksek önemlilik (URL'ler gibi) ile toplarken, veri hacmi çok fazla azaltılmaz (iki kattan daha az). - Bu nedenle, toplama ile veri hacmi küçültmek yerine büyüyebilir. - Kullanıcılar onlar için oluşturduğumuz tüm raporları görüntülemez. Bu hesaplamaların büyük bir kısmı işe yaramaz. - Verilerin mantıksal bütünlüğü, çeşitli toplamalar için ihlal edilebilir. @@ -48,8 +48,8 @@ Hiçbir şeyi toplamazsak ve toplanmamış verilerle çalışırsak, bu hesaplam Bununla birlikte, toplama ile, çalışmanın önemli bir kısmı çevrimdışı olarak alınır ve nispeten sakin bir şekilde tamamlanır. Buna karşılık, çevrimiçi hesaplamalar, kullanıcı sonucu beklediğinden mümkün olduğunca hızlı hesaplamayı gerektirir. Üye.Metrica, raporların çoğunluğu için kullanılan Metrage adı verilen verileri toplamak için özel bir sisteme sahiptir. -2009’dan itibaren Yandex.Metrica, daha önce Rapor Oluşturucusu için kullanılan OLAPServer adlı toplanmamış veriler için özel bir OLAP veritabanı da kullandı. -OLAPServer, toplanmamış veriler için iyi çalıştı, ancak tüm raporlar için istenildiği gibi kullanılmasına izin vermeyen birçok kısıtlamaya sahipti. Bunlar, veri türleri için destek eksikliği (yalnızca sayılar) ve verileri gerçek zamanlı olarak aşamalı olarak güncelleyememe (yalnızca verileri günlük olarak yeniden yazarak yapılabilir) içeriyordu. OLAPServer bir DBMS değil, özel bir dB’dir. +2009'dan itibaren Yandex.Metrica, daha önce Rapor Oluşturucusu için kullanılan OLAPServer adlı toplanmamış veriler için özel bir OLAP veritabanı da kullandı. +OLAPServer, toplanmamış veriler için iyi çalıştı, ancak tüm raporlar için istenildiği gibi kullanılmasına izin vermeyen birçok kısıtlamaya sahipti. Bunlar, veri türleri için destek eksikliği (yalnızca sayılar) ve verileri gerçek zamanlı olarak aşamalı olarak güncelleyememe (yalnızca verileri günlük olarak yeniden yazarak yapılabilir) içeriyordu. OLAPServer bir DBMS değil, özel bir dB'dir. ClickHouse için ilk hedef OLAPServer sınırlamaları kaldırmak ve tüm raporlar için toplanmamış verilerle çalışma sorunu çözmek oldu, ama yıllar içinde, analitik görevler geniş bir yelpazede için uygun bir genel amaçlı veritabanı yönetim sistemi haline gelmiştir. diff --git a/docs/tr/introduction/index.md b/docs/tr/introduction/index.md index 9691671aefa..3296e8bcaf0 100644 --- a/docs/tr/introduction/index.md +++ b/docs/tr/introduction/index.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_folder_title: "Giri\u015F" toc_priority: 1 --- diff --git a/docs/tr/introduction/performance.md b/docs/tr/introduction/performance.md index ac0804f1854..ddbd4436748 100644 --- a/docs/tr/introduction/performance.md +++ b/docs/tr/introduction/performance.md @@ -1,19 +1,19 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 6 toc_title: Performans --- # Performans {#performance} -Yandex’deki dahili test sonuçlarına göre, ClickHouse, test için mevcut olan sınıfının sistemleri arasında karşılaştırılabilir işletim senaryoları için en iyi performansı (hem uzun sorgular için en yüksek verim hem de kısa sorgularda en düşük gecikme süresi) gösterir. Test sonuçlarını bir [ayrı sayfa](https://clickhouse.tech/benchmark/dbms/). +Yandex'deki dahili test sonuçlarına göre, ClickHouse, test için mevcut olan sınıfının sistemleri arasında karşılaştırılabilir işletim senaryoları için en iyi performansı (hem uzun sorgular için en yüksek verim hem de kısa sorgularda en düşük gecikme süresi) gösterir. Test sonuçlarını bir [ayrı sayfa](https://clickhouse.tech/benchmark/dbms/). Çok sayıda bağımsız kriterler benzer sonuçlara geldi. Bir internet araması kullanarak bulmak zor değildir veya görebilirsiniz [ilgili bağlantı ourlardan oluşan küçük koleksiyon collectionumuz](https://clickhouse.tech/#independent-benchmarks). -## Tek Bir büyük Sorgu için çıktı {#throughput-for-a-single-large-query} +## Tek bir büyük sorgu için çıktı {#throughput-for-a-single-large-query} -Verim, saniyede satır veya saniyede megabayt olarak ölçülebilir. Veriler sayfa önbelleğine yerleştirilirse, çok karmaşık olmayan bir sorgu, modern donanım üzerinde tek bir sunucuda yaklaşık 2-10 GB/s sıkıştırılmamış veri hızında işlenir (en basit durumlar için, hız 30 GB/s’ye ulaşabilir). Veri sayfa önbelleğine yerleştirilmezse, hız disk alt sistemine ve veri sıkıştırma hızına bağlıdır. Örneğin, disk alt sistemi 400 MB/s veri okuma izin verir ve veri sıkıştırma hızı 3 ise, hız 1.2 GB / s civarında olması bekleniyor. saniyede satır hızı elde etmek için hızı saniyede bayt cinsinden sorguda kullanılan sütunların toplam boyutuna bölün. Örneğin, 10 bayt sütun ayıklanırsa, hızın saniyede yaklaşık 100-200 milyon satır olması beklenir. +Verim, saniyede satır veya saniyede megabayt olarak ölçülebilir. Veriler sayfa önbelleğine yerleştirilirse, çok karmaşık olmayan bir sorgu, modern donanım üzerinde tek bir sunucuda yaklaşık 2-10 GB/s sıkıştırılmamış veri hızında işlenir (en basit durumlar için, hız 30 GB/s'ye ulaşabilir). Veri sayfa önbelleğine yerleştirilmezse, hız disk alt sistemine ve veri sıkıştırma hızına bağlıdır. Örneğin, disk alt sistemi 400 MB/s veri okuma izin verir ve veri sıkıştırma hızı 3 ise, hız 1.2 GB / s civarında olması bekleniyor. saniyede satır hızı elde etmek için hızı saniyede bayt cinsinden sorguda kullanılan sütunların toplam boyutuna bölün. Örneğin, 10 bayt sütun ayıklanırsa, hızın saniyede yaklaşık 100-200 milyon satır olması beklenir. İşlem hızı, dağıtılmış işlem için neredeyse doğrusal olarak artar, ancak yalnızca toplama veya sıralamadan kaynaklanan satır sayısı çok büyük değilse. @@ -21,7 +21,7 @@ Verim, saniyede satır veya saniyede megabayt olarak ölçülebilir. Veriler say Bir sorgu birincil anahtar kullanır ve çok fazla sütun ve satır (yüzbinlerce) işlemek için seçmez, veri sayfa önbelleğine yerleştirilirse, 50 milisaniyeden daha az gecikme süresi (en iyi durumda milisaniye tek basamak) bekleyebilirsiniz. Aksi takdirde, gecikme çoğunlukla arama sayısı tarafından hakimdir. Aşırı yüklenmemiş bir sistem için dönen disk sürücüleri kullanırsanız, gecikme bu formülle tahmin edilebilir: `seek time (10 ms) * count of columns queried * count of data parts`. -## Büyük Miktarda kısa Sorgu işlerken Verim {#throughput-when-processing-a-large-quantity-of-short-queries} +## Büyük miktarda kısa sorgu işlerken verim {#throughput-when-processing-a-large-quantity-of-short-queries} Aynı koşullar altında, ClickHouse tek bir sunucuda saniyede birkaç yüz sorgu işleyebilir (en iyi durumda birkaç bine kadar). Bu senaryo analitik DBMSs için tipik olmadığından, saniyede en fazla 100 sorgu beklemenizi öneririz. diff --git a/docs/tr/operations/access-rights.md b/docs/tr/operations/access-rights.md index 91a1581591e..87c7bf5a373 100644 --- a/docs/tr/operations/access-rights.md +++ b/docs/tr/operations/access-rights.md @@ -1,113 +1,143 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 48 -toc_title: "Eri\u015Fim Haklar\u0131" +toc_title: "Eri\u015Fim Kontrol\xFC ve hesap y\xF6netimi" --- -# Erişim Hakları {#access-rights} +# Erişim Kontrolü ve hesap yönetimi {#access-control} -Kullanıcılar ve erişim hakları kullanıcı yapılandırmasında ayarlanır. Bu genellikle `users.xml`. +ClickHouse dayalı erişim kontrol yönetimini destekler [RBAC](https://en.wikipedia.org/wiki/Role-based_access_control) yaklaşmak. -Kullanıcılar kaydedilir `users` bölme. İşte bir parçası `users.xml` Dosya: +ClickHouse erişim varlıkları: +- [Kullanıcı hesabı](#user-account-management) +- [Rol](#role-management) +- [Satır Politikası](#row-policy-management) +- [Ayarlar Profili](#settings-profiles-management) +- [Kota](#quotas-management) -``` xml - - - - - - +- Hizmetçi [yapılandırma dosyaları](configuration-files.md) `users.xml` ve `config.xml`. - - +!!! note "Uyarıcı" + Aynı erişim varlığını her iki yapılandırma yöntemiyle aynı anda yönetemezsiniz. - - default +## Kullanma {#access-control-usage} - - default - +Varsayılan olarak, ClickHouse sunucusu kullanıcı hesabını sağlar `default` SQL güdümlü erişim denetimi ve hesap yönetimi kullanılarak izin verilmez, ancak tüm haklara ve izinlere sahiptir. Bu `default` kullanıcı hesabı, kullanıcı adı tanımlanmadığında, örneğin istemciden girişte veya dağıtılmış sorgularda kullanılır. Dağıtılmış sorgu işleme varsayılan kullanıcı hesabı kullanılır, sunucu veya küme yapılandırması belirtmezse [kullanıcı ve şifre](../engines/table-engines/special/distributed.md) özellikler. - - - - - web - default - - test - - - test - - - -``` +Sadece ClickHouse kullanmaya başlarsanız, aşağıdaki senaryoyu kullanabilirsiniz: -İki kullanıcıdan bir bildirim görebilirsiniz: `default`ve`web`. Ek weledik `web` kullanıcı ayrı ayrı. +1. [Etkinleştirmek](#enabling-access-control) SQL-driven erişim kontrolü ve hesap yönetimi için `default` kullanan. +2. Log underin un underder the `default` kullanıcı hesabı ve gerekli tüm kullanıcıları oluşturun. Yönetici hesabı oluşturmayı unutmayın (`GRANT ALL ON *.* WITH GRANT OPTION TO admin_user_account`). +3. [İzinleri kısıtla](settings/permissions-for-queries.md#permissions_for_queries) için `default` kullanıcı ve devre dışı SQL odaklı erişim kontrolü ve bunun için hesap yönetimi. -Bu `default` kullanıcı adı geçilmez durumlarda kullanıcı seçilir. Bu `default` kullanıcı, sunucu veya kümenin yapılandırması, sunucu veya kümenin yapılandırılmasını belirtmezse, dağıtılmış sorgu işleme için de kullanılır. `user` ve `password` (on bölümüne bakın [Dağılı](../engines/table-engines/special/distributed.md) motor). +### Mevcut çözümün özellikleri {#access-control-properties} -The user that is used for exchanging information between servers combined in a cluster must not have substantial restrictions or quotas – otherwise, distributed queries will fail. +- Olmasa bile veritabanları ve tablolar için izinler verebilirsiniz. +- Bir tablo silindiyse, bu tabloya karşılık gelen tüm ayrıcalıklar iptal edilmez. Bu nedenle, daha sonra aynı ada sahip yeni bir tablo oluşturulursa, tüm ayrıcalıklar tekrar gerçek olur. Silinen tabloya karşılık gelen ayrıcalıkları iptal etmek için, örneğin, `REVOKE ALL PRIVILEGES ON db.table FROM ALL` sorgu. +- Ayrıcalıklar için ömür boyu ayarları yoktur. -Parola, açık metin (önerilmez) veya SHA-256’da belirtilir. Haşhaş tuzlu değil. Bu bağlamda, bu şifreleri potansiyel kötü amaçlı saldırılara karşı güvenlik sağlamak olarak düşünmemelisiniz. Aksine, çalışanlardan korunmak için gereklidir. +## Kullanıcı hesabı {#user-account-management} -Erişime izin verilen ağların listesi belirtilir. Bu örnekte, her iki kullanıcı için ağ listesi ayrı bir dosyadan yüklenir (`/etc/metrika.xml`) içeren `networks` ikame. İşte bunun bir parçası: +Bir kullanıcı hesabı, Clickhouse'da birisini yetkilendirmeye izin veren bir erişim varlığıdır. Bir kullanıcı hesabı içerir: -``` xml - - ... - - ::/64 - 203.0.113.0/24 - 2001:DB8::/32 - ... - - -``` +- Kimlik bilgileri. +- [Ayrıcalıklar](../sql-reference/statements/grant.md#grant-privileges) kullanıcı gerçekleştirebilir sorgular kapsamı tanımlayın. +- ClickHouse sunucusuna bağlantının izin verildiği ana bilgisayarlar. +- Verilen ve varsayılan roller. +- Kullanıcının girişinde varsayılan olarak geçerli olan kısıtlamaları olan ayarlar. +- Atanan ayarlar profilleri. -Bu ağ listesini doğrudan tanımlayabilirsiniz `users.xml` veya bir dosyada `users.d` dizin (daha fazla bilgi için bölüme bakın “[Yapılandırma dosyaları](configuration-files.md#configuration_files)”). +Bir kullanıcı hesabına ayrıcalıklar tarafından verilebilir [GRANT](../sql-reference/statements/grant.md) sorgu veya atama ile [roller](#role-management). Bir kullanıcının ayrıcalıklarını iptal etmek için, ClickHouse [REVOKE](../sql-reference/statements/revoke.md) sorgu. Bir kullanıcının ayrıcalıklarını listelemek için - [SHOW GRANTS](../sql-reference/statements/show.md#show-grants-statement) deyim. -Yapılandırma, her yerden erişimin nasıl açılacağını açıklayan yorumları içerir. +Yönetim sorguları: -Üretimde kullanım için, sadece belirtin `ip` elemanları (IP adresleri ve maskeleri), kullanıl ,dığından beri `host` ve `hoost_regexp` ekstra gecikmeye neden olabilir. +- [CREATE USER](../sql-reference/statements/create.md#create-user-statement) +- [ALTER USER](../sql-reference/statements/alter.md#alter-user-statement) +- [DROP USER](../sql-reference/statements/misc.md#drop-user-statement) +- [SHOW CREATE USER](../sql-reference/statements/show.md#show-create-user-statement) -Daha sonra kullanıcı ayarları profili belirtilir (bölüme bakın “[Ayarlar profilleri](settings/settings-profiles.md)”. Varsayılan profili belirtebilirsiniz, `default'`. Profilin herhangi bir adı olabilir. Farklı kullanıcılar için aynı profili belirtebilirsiniz. Ayarlar profilinde yazabileceğiniz en önemli şey `readonly=1` sağlar okumak-sadece erişim. Ardından kullanılacak kotayı belirtin (bölüme bakın “[Kotalar](quotas.md#quotas)”). Varsayılan kotayı belirtebilirsiniz: `default`. It is set in the config by default to only count resource usage, without restricting it. The quota can have any name. You can specify the same quota for different users – in this case, resource usage is calculated for each user individually. +### Uygulama Ayarları {#access-control-settings-applying} -İsteğe bağlı `` bölümünde, kullanıcının erişebileceği veritabanlarının bir listesini de belirtebilirsiniz. Varsayılan olarak, tüm veritabanları kullanıcı tarafından kullanılabilir. Belirtebilirsiniz `default` veritabanı. Bu durumda, kullanıcı varsayılan olarak veritabanına erişim alır. +Ayarlar farklı şekillerde ayarlanabilir: bir kullanıcı hesabı için, verilen roller ve ayarlar profillerinde. Bir kullanıcı girişinde, farklı erişim varlıklarında bir ayar ayarlanırsa, bu ayarın değeri ve kısıtlamaları aşağıdaki öncelikler tarafından uygulanır (yüksekten düşüğe): -İsteğe bağlı `` bölümünde, kullanıcının erişebileceği sözlüklerin bir listesini de belirtebilirsiniz. Varsayılan olarak, tüm sözlükler kullanıcı tarafından kullanılabilir. +1. Kullanıcı hesabı ayarı. +2. Kullanıcı hesabının varsayılan rollerinin ayarları. Bazı rollerde bir ayar ayarlanmışsa, uygulama ayarının sırası tanımsızdır. +3. Bir kullanıcıya veya varsayılan rollerine atanan ayarlar profilleri'ndeki ayarlar. Bazı profillerde bir ayar ayarlanmışsa, uygulama ayarı sırası tanımsızdır. +4. Varsayılan olarak tüm sunucuya uygulanan ayarlar veya [varsayılan profil](server-configuration-parameters/settings.md#default-profile). -Erişim `system` veritabanı her zaman izin verilir (bu veritabanı sorguları işlemek için kullanıldığından). +## Rol {#role-management} -Kullanıcı kullanarak onları tüm veritabanları ve tabloların bir listesini alabilirsiniz `SHOW` tek tek veritabanlarına erişime izin verilmese bile, sorgular veya sistem tabloları. +Rol, bir kullanıcı hesabına verilebilecek erişim varlıkları için bir kapsayıcıdır. -Veritabanı erişimi ile ilgili değildir [readonly](settings/permissions-for-queries.md#settings_readonly) ayar. Bir veritabanına tam erişim izni veremezsiniz ve `readonly` başka birine erişim. +Rol içerir: + +- [Ayrıcalıklar](../sql-reference/statements/grant.md#grant-privileges) +- Ayarlar ve kısıtlamalar +- Verilen roller listesi + +Yönetim sorguları: + +- [CREATE ROLE](../sql-reference/statements/create.md#create-role-statement) +- [ALTER ROLE](../sql-reference/statements/alter.md#alter-role-statement) +- [DROP ROLE](../sql-reference/statements/misc.md#drop-role-statement) +- [SET ROLE](../sql-reference/statements/misc.md#set-role-statement) +- [SET DEFAULT ROLE](../sql-reference/statements/misc.md#set-default-role-statement) +- [SHOW CREATE ROLE](../sql-reference/statements/show.md#show-create-role-statement) + +Bir rol için ayrıcalıklar tarafından verilebilir [GRANT](../sql-reference/statements/grant.md) sorgu. Bir rolden ayrıcalıkları iptal etmek için ClickHouse şunları sağlar: [REVOKE](../sql-reference/statements/revoke.md) sorgu. + +## Satır Politikası {#row-policy-management} + +Satır ilkesi, bir kullanıcı veya rol için hangi veya satırların kullanılabilir olduğunu tanımlayan bir filtredir. Satır ilkesi, belirli bir tablo ve bu satır ilkesini kullanması gereken rollerin ve/veya kullanıcıların listesi için filtreler içerir. + +Yönetim sorguları: + +- [CREATE ROW POLICY](../sql-reference/statements/create.md#create-row-policy-statement) +- [ALTER ROW POLICY](../sql-reference/statements/alter.md#alter-row-policy-statement) +- [DROP ROW POLICY](../sql-reference/statements/misc.md#drop-row-policy-statement) +- [SHOW CREATE ROW POLICY](../sql-reference/statements/show.md#show-create-row-policy-statement) + +## Ayarlar Profili {#settings-profiles-management} + +Ayarlar profili bir koleksiyon [ayarlar](settings/index.md). Ayarlar profili, ayarları ve kısıtlamaları ve bu kotanın uygulandığı rollerin ve/veya kullanıcıların listesini içerir. + +Yönetim sorguları: + +- [CREATE SETTINGS PROFILE](../sql-reference/statements/create.md#create-settings-profile-statement) +- [ALTER SETTINGS PROFILE](../sql-reference/statements/alter.md#alter-settings-profile-statement) +- [DROP SETTINGS PROFILE](../sql-reference/statements/misc.md#drop-settings-profile-statement) +- [SHOW CREATE SETTINGS PROFILE](../sql-reference/statements/show.md#show-create-settings-profile-statement) + +## Kota {#quotas-management} + +Kota kaynak kullanımını sınırlar. Görmek [Kotalar](quotas.md). + +Kota, bazı süreler için bir dizi sınır ve bu kotayı kullanması gereken rollerin ve/veya kullanıcıların listesini içerir. + +Yönetim sorguları: + +- [CREATE QUOTA](../sql-reference/statements/create.md#create-quota-statement) +- [ALTER QUOTA](../sql-reference/statements/alter.md#alter-quota-statement) +- [DROP QUOTA](../sql-reference/statements/misc.md#drop-quota-statement) +- [SHOW CREATE QUOTA](../sql-reference/statements/show.md#show-create-quota-statement) + +## SQL tabanlı erişim denetimi ve hesap yönetimini etkinleştirme {#enabling-access-control} + +- Yapılandırmaları depolama için bir dizin Kur. + + ClickHouse, erişim varlık yapılandırmalarını, [access\_control\_path](server-configuration-parameters/settings.md#access_control_path) sunucu yapılandırma parametresi. + +- En az bir kullanıcı hesabı için SQL tabanlı erişim denetimi ve hesap yönetimini etkinleştirin. + + Varsayılan olarak SQL güdümlü erişim denetimi ve hesap yönetimi, tüm kullanıcılar için açık. En az bir kullanıcı yapılandırmanız gerekir `users.xml` yapılandırma dosyası ve atama 1 [access\_management](settings/settings-users.md#access_management-user-setting) ayar. [Orijinal makale](https://clickhouse.tech/docs/en/operations/access_rights/) diff --git a/docs/tr/operations/backup.md b/docs/tr/operations/backup.md index 22cc6211341..23544539dac 100644 --- a/docs/tr/operations/backup.md +++ b/docs/tr/operations/backup.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 49 toc_title: Veri Yedekleme --- # Veri Yedekleme {#data-backup} -Karşın [çoğalma](../engines/table-engines/mergetree-family/replication.md) provides protection from hardware failures, it does not protect against human errors: accidental deletion of data, deletion of the wrong table or a table on the wrong cluster, and software bugs that result in incorrect data processing or data corruption. In many cases mistakes like these will affect all replicas. ClickHouse has built-in safeguards to prevent some types of mistakes — for example, by default [50 GB’den fazla veri içeren MergeTree benzeri bir motorla tabloları bırakamazsınız](https://github.com/ClickHouse/ClickHouse/blob/v18.14.18-stable/programs/server/config.xml#L322-L330). Ancak, bu önlemler olası tüm davaları kapsamaz ve atlatılabilir. +Karşın [çoğalma](../engines/table-engines/mergetree-family/replication.md) provides protection from hardware failures, it does not protect against human errors: accidental deletion of data, deletion of the wrong table or a table on the wrong cluster, and software bugs that result in incorrect data processing or data corruption. In many cases mistakes like these will affect all replicas. ClickHouse has built-in safeguards to prevent some types of mistakes — for example, by default [50 GB'den fazla veri içeren MergeTree benzeri bir motorla tabloları bırakamazsınız](https://github.com/ClickHouse/ClickHouse/blob/v18.14.18-stable/programs/server/config.xml#L322-L330). Ancak, bu önlemler olası tüm davaları kapsamaz ve atlatılabilir. Olası insan hatalarını etkili bir şekilde azaltmak için, verilerinizi yedeklemek ve geri yüklemek için dikkatli bir şekilde bir strateji hazırlamanız gerekir **önceden**. @@ -18,7 +18,7 @@ Her şirketin farklı kaynakları ve iş gereksinimleri vardır, bu nedenle her ## Kaynak Verileri Başka Bir Yerde Çoğaltma {#duplicating-source-data-somewhere-else} -Genellikle Clickhouse’a alınan veriler, aşağıdaki gibi bir tür kalıcı sıra yoluyla teslim edilir [Apache Kafka](https://kafka.apache.org). Bu durumda, Clickhouse’a yazılırken aynı veri akışını okuyacak ve bir yerde soğuk depoda depolayacak ek bir abone kümesi yapılandırmak mümkündür. Çoğu şirket zaten bir nesne deposu veya dağıtılmış bir dosya sistemi gibi olabilecek bazı varsayılan önerilen soğuk depolamaya sahiptir [HDFS](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HdfsDesign.html). +Genellikle Clickhouse'a alınan veriler, aşağıdaki gibi bir tür kalıcı sıra yoluyla teslim edilir [Apache Kafka](https://kafka.apache.org). Bu durumda, Clickhouse'a yazılırken aynı veri akışını okuyacak ve bir yerde soğuk depoda depolayacak ek bir abone kümesi yapılandırmak mümkündür. Çoğu şirket zaten bir nesne deposu veya dağıtılmış bir dosya sistemi gibi olabilecek bazı varsayılan önerilen soğuk depolamaya sahiptir [HDFS](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HdfsDesign.html). ## Dosya Sistemi Anlık Görüntüleri {#filesystem-snapshots} @@ -30,7 +30,7 @@ Bazı yerel dosya sistemleri anlık görüntü işlevselliği sağlar (örneğin Daha küçük veri hacimleri için, basit bir `INSERT INTO ... SELECT ...` uzak tablolara da çalışabilir. -## Parçalar Ile manipülasyonlar {#manipulations-with-parts} +## Parçalar ile manipülasyonlar {#manipulations-with-parts} ClickHouse kullanarak sağlar `ALTER TABLE ... FREEZE PARTITION ...` tablo bölümleri yerel bir kopyasını oluşturmak için sorgu. Bu hardlinks kullanarak uygulanır `/var/lib/clickhouse/shadow/` klasör, bu yüzden genellikle eski veriler için ekstra disk alanı tüketmez. Oluşturulan dosyaların kopyaları ClickHouse server tarafından işlenmez, bu yüzden onları orada bırakabilirsiniz: herhangi bir ek harici sistem gerektirmeyen basit bir yedeklemeniz olacak, ancak yine de donanım sorunlarına eğilimli olacaktır. Bu nedenle, bunları uzaktan başka bir konuma kopyalamak ve ardından yerel kopyaları kaldırmak daha iyidir. Dağıtılmış dosya sistemleri ve nesne depoları bunun için hala iyi bir seçenektir, ancak yeterince büyük kapasiteye sahip normal ekli dosya sunucuları da işe yarayabilir (bu durumda aktarım ağ dosya sistemi veya belki de [rsync](https://en.wikipedia.org/wiki/Rsync)). diff --git a/docs/tr/operations/configuration-files.md b/docs/tr/operations/configuration-files.md index 82378d0cab7..59f268981a3 100644 --- a/docs/tr/operations/configuration-files.md +++ b/docs/tr/operations/configuration-files.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 50 toc_title: "Yap\u0131land\u0131rma Dosyalar\u0131" --- @@ -20,7 +20,7 @@ Eğer `replace` belirtilen, tüm öğeyi belirtilen ile değiştirir. Eğer `remove` belirt .ilirse, öğeyi siler. -Yapılandırma ayrıca tanımlayabilir “substitutions”. Bir öğe varsa `incl` öznitelik, dosyadan karşılık gelen ikame değeri olarak kullanılacaktır. Varsayılan olarak, değiştirmeler ile dosyanın yolu `/etc/metrika.xml`. Bu değiştirilebilir [include\_from](server-configuration-parameters/settings.md#server_configuration_parameters-include_from) sunucu yapılandırmasında öğe. İkame değerleri belirtilen `/yandex/substitution_name` bu dosyadaki öğeler. Belirtilen bir ika Ame halinde `incl` yok, günlüğe kaydedilir. Clickhouse’un eksik değiştirmelerin günlüğe kaydedilmesini önlemek için `optional="true"` öznitelik (örneğin, ayarlar [makrolar](server-configuration-parameters/settings.md)). +Yapılandırma ayrıca tanımlayabilir “substitutions”. Bir öğe varsa `incl` öznitelik, dosyadan karşılık gelen ikame değeri olarak kullanılacaktır. Varsayılan olarak, değiştirmeler ile dosyanın yolu `/etc/metrika.xml`. Bu değiştirilebilir [include\_from](server-configuration-parameters/settings.md#server_configuration_parameters-include_from) sunucu yapılandırmasında öğe. İkame değerleri belirtilen `/yandex/substitution_name` bu dosyadaki öğeler. Belirtilen bir ika Ame halinde `incl` yok, günlüğe kaydedilir. Clickhouse'un eksik değiştirmelerin günlüğe kaydedilmesini önlemek için `optional="true"` öznitelik (örneğin, ayarlar [makrolar](server-configuration-parameters/settings.md)). İkame da ZooKeeper yapılabilir. Bunu yapmak için özniteliği belirtin `from_zk = "/path/to/node"`. Eleman değeri, düğümün içeriği ile değiştirilir `/path/to/node` ZooKeeper. Ayrıca ZooKeeper düğümünde bir XML alt ağacının tamamını koyabilirsiniz ve kaynak öğeye tamamen eklenecektir. diff --git a/docs/tr/operations/index.md b/docs/tr/operations/index.md index 042529cc62a..d3639c42bf1 100644 --- a/docs/tr/operations/index.md +++ b/docs/tr/operations/index.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_folder_title: Harekat toc_priority: 41 toc_title: "Giri\u015F" @@ -25,4 +25,4 @@ ClickHouse işlemleri kılavuzu aşağıdaki ana bölümlerden oluşur: - [Ayarlar](settings/index.md) - [Programlar](utilities/index.md) -[Orijinal makale](https://clickhouse.tech/docs/en/operations/) +{## [Orijinal makale](https://clickhouse.tech/docs/en/operations/) ##} diff --git a/docs/tr/operations/monitoring.md b/docs/tr/operations/monitoring.md index 811face9de7..bed1b9985ef 100644 --- a/docs/tr/operations/monitoring.md +++ b/docs/tr/operations/monitoring.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 45 toc_title: "\u0130zleme" --- @@ -37,9 +37,9 @@ ClickHouse toplar: Metrikleri şu adreste bulabilirsiniz: [sistem.metrik](../operations/system-tables.md#system_tables-metrics), [sistem.etkinlik](../operations/system-tables.md#system_tables-events), ve [sistem.asynchronous\_metrics](../operations/system-tables.md#system_tables-asynchronous_metrics) Tablolar. -Clickhouse’u metrikleri dışa aktaracak şekilde yapılandırabilirsiniz [Grafit](https://github.com/graphite-project). Görmek [Graf sectionit bölümü](server-configuration-parameters/settings.md#server_configuration_parameters-graphite) ClickHouse sunucu yapılandırma dosyasında. Metriklerin dışa aktarımını yapılandırmadan önce, grafit’i resmi olarak takip ederek ayarlamanız gerekir [kılavuz](https://graphite.readthedocs.io/en/latest/install.html). +Clickhouse'u metrikleri dışa aktaracak şekilde yapılandırabilirsiniz [Grafit](https://github.com/graphite-project). Görmek [Graf sectionit bölümü](server-configuration-parameters/settings.md#server_configuration_parameters-graphite) ClickHouse sunucu yapılandırma dosyasında. Metriklerin dışa aktarımını yapılandırmadan önce, grafit'i resmi olarak takip ederek ayarlamanız gerekir [kılavuz](https://graphite.readthedocs.io/en/latest/install.html). -Clickhouse’u metrikleri dışa aktaracak şekilde yapılandırabilirsiniz [Prometheus](https://prometheus.io). Görmek [Prometheus bölümü](server-configuration-parameters/settings.md#server_configuration_parameters-prometheus) ClickHouse sunucu yapılandırma dosyasında. Metriklerin dışa aktarılmasını yapılandırmadan önce, prometheus’u yetkililerini takip ederek ayarlamanız gerekir [kılavuz](https://prometheus.io/docs/prometheus/latest/installation/). +Clickhouse'u metrikleri dışa aktaracak şekilde yapılandırabilirsiniz [Prometheus](https://prometheus.io). Görmek [Prometheus bölümü](server-configuration-parameters/settings.md#server_configuration_parameters-prometheus) ClickHouse sunucu yapılandırma dosyasında. Metriklerin dışa aktarılmasını yapılandırmadan önce, prometheus'u yetkililerini takip ederek ayarlamanız gerekir [kılavuz](https://prometheus.io/docs/prometheus/latest/installation/). Ayrıca, http API aracılığıyla sunucu kullanılabilirliğini izleyebilirsiniz. Sen sendd the `HTTP GET` istek için `/ping`. Sunucu mevcutsa, yanıt verir `200 OK`. diff --git a/docs/tr/operations/optimizing-performance/index.md b/docs/tr/operations/optimizing-performance/index.md index 9d39c082a98..4457d125c1f 100644 --- a/docs/tr/operations/optimizing-performance/index.md +++ b/docs/tr/operations/optimizing-performance/index.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_folder_title: "Performans\u0131 Optimize Etme" toc_priority: 52 --- diff --git a/docs/tr/operations/optimizing-performance/sampling-query-profiler.md b/docs/tr/operations/optimizing-performance/sampling-query-profiler.md index 8ffbd665b0a..e73151397e9 100644 --- a/docs/tr/operations/optimizing-performance/sampling-query-profiler.md +++ b/docs/tr/operations/optimizing-performance/sampling-query-profiler.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 54 toc_title: "Sorgu Profili Olu\u015Fturma" --- # Örnekleme Sorgusu Profiler {#sampling-query-profiler} -ClickHouse, sorgu yürütülmesini analiz etmeyi sağlayan örnekleme profiler’i çalıştırır. Profiler kullanarak sorgu yürütme sırasında en sık kullanılan kaynak kodu yordamları bulabilirsiniz. Boşta kalma süresi de dahil olmak üzere harcanan CPU zamanını ve duvar saati zamanını izleyebilirsiniz. +ClickHouse, sorgu yürütülmesini analiz etmeyi sağlayan örnekleme profiler'i çalıştırır. Profiler kullanarak sorgu yürütme sırasında en sık kullanılan kaynak kodu yordamları bulabilirsiniz. Boşta kalma süresi de dahil olmak üzere harcanan CPU zamanını ve duvar saati zamanını izleyebilirsiniz. Profiler kullanmak için: @@ -29,7 +29,7 @@ Analiz etmek `trace_log` sistem tablosu: Güvenlik nedenleriyle, iç gözlem işlevleri varsayılan olarak devre dışı bırakılır. -- Kullan… `addressToLine`, `addressToSymbol` ve `demangle` [iç gözlem fonksiyonları](../../sql-reference/functions/introspection.md) ClickHouse kodu işlev adları ve konumlarını almak için. Bazı sorgu için bir profil almak için, `trace_log` Tablo. Bireysel fonksiyonları bütün yığın izleri ya da veri toplama yapabilirsiniz. +- Kullan... `addressToLine`, `addressToSymbol` ve `demangle` [iç gözlem fonksiyonları](../../sql-reference/functions/introspection.md) ClickHouse kodu işlev adları ve konumlarını almak için. Bazı sorgu için bir profil almak için, `trace_log` Tablo. Bireysel fonksiyonları bütün yığın izleri ya da veri toplama yapabilirsiniz. Görselleştirmeniz gerekiyorsa `trace_log` bilgi, deneyin [flamegraph](../../interfaces/third-party/gui/#clickhouse-flamegraph) ve [speedscope](https://github.com/laplab/clickhouse-speedscope). diff --git a/docs/tr/operations/performance-test.md b/docs/tr/operations/performance-test.md index 85b03f4e4d8..498f3851861 100644 --- a/docs/tr/operations/performance-test.md +++ b/docs/tr/operations/performance-test.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 54 toc_title: "Donan\u0131m Test" --- -# Donanımınızı ClickHouse Ile Test Etme {#how-to-test-your-hardware-with-clickhouse} +# Donanımınızı ClickHouse ile Test etme {#how-to-test-your-hardware-with-clickhouse} Bu talimat ile ClickHouse paketlerinin kurulumu olmadan herhangi bir sunucuda temel ClickHouse performans testi çalıştırabilirsiniz. @@ -79,4 +79,4 @@ Bu talimat ile ClickHouse paketlerinin kurulumu olmadan herhangi bir sunucuda te 1. Donanım yapılandırmanız hakkındaki numaraları ve bilgileri şu adrese gönderin clickhouse-feedback@yandex-team.com -Tüm sonuçlar burada yayınlanmaktadır: https://clickhouse.teknoloji / benchmark\_hardware.html +Tüm sonuçlar burada yayınlanmaktadır: https://clickhouse.teknoloji / ben benchmarkch /mark / donanım/ diff --git a/docs/tr/operations/quotas.md b/docs/tr/operations/quotas.md index e6b1de2b7f8..b646e47e50c 100644 --- a/docs/tr/operations/quotas.md +++ b/docs/tr/operations/quotas.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 51 toc_title: Kotalar --- diff --git a/docs/tr/operations/requirements.md b/docs/tr/operations/requirements.md index 8d1245557d3..c45fc93e068 100644 --- a/docs/tr/operations/requirements.md +++ b/docs/tr/operations/requirements.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 44 toc_title: Gereksinimler --- @@ -9,9 +9,9 @@ toc_title: Gereksinimler ## CPU {#cpu} -Önceden oluşturulmuş deb paketlerinden kurulum için, x86\_64 mimarisine sahip bir CPU kullanın ve sse 4.2 yönergelerini destekleyin. Clickhouse’u SSE 4.2’yi desteklemeyen veya AArch64 veya PowerPC64LE mimarisine sahip işlemcilerle çalıştırmak için, kaynaklardan Clickhouse’u oluşturmanız gerekir. +Önceden oluşturulmuş deb paketlerinden kurulum için, x86\_64 mimarisine sahip bir CPU kullanın ve sse 4.2 yönergelerini destekleyin. Clickhouse'u SSE 4.2'yi desteklemeyen veya AArch64 veya PowerPC64LE mimarisine sahip işlemcilerle çalıştırmak için, kaynaklardan Clickhouse'u oluşturmanız gerekir. -ClickHouse paralel veri işleme uygular ve mevcut tüm donanım kaynaklarını kullanır. Bir işlemci seçerken, Clickhouse’un çok sayıda çekirdeğe sahip konfigürasyonlarda daha verimli çalıştığını, ancak daha az çekirdeğe ve daha yüksek bir saat hızına sahip konfigürasyonlardan daha düşük bir saat hızına sahip olduğunu göz önünde bulundurun. Örneğin, 2600 MHz’lik 16 çekirdek, 3600 MHz’lik 8 çekirdeğe tercih edilir. +ClickHouse paralel veri işleme uygular ve mevcut tüm donanım kaynaklarını kullanır. Bir işlemci seçerken, Clickhouse'un çok sayıda çekirdeğe sahip konfigürasyonlarda daha verimli çalıştığını, ancak daha az çekirdeğe ve daha yüksek bir saat hızına sahip konfigürasyonlardan daha düşük bir saat hızına sahip olduğunu göz önünde bulundurun. Örneğin, 2600 MHz'lik 16 çekirdek, 3600 MHz'lik 8 çekirdeğe tercih edilir. Kullanılması tavsiye edilir **Turbo Bo Boostost** ve **hyper-thre -ading** teknolojiler. Tipik bir iş yükü ile performansı önemli ölçüde artırır. @@ -24,9 +24,9 @@ Gerekli RAM hacmi Aşağıdakilere bağlıdır: - Sorguların karmaşıklığı. - Sorgularda işlenen veri miktarı. -Gerekli RAM hacmini hesaplamak için, aşağıdakiler için geçici verilerin boyutunu tahmin etmelisiniz [GROUP BY](../sql-reference/statements/select.md#select-group-by-clause), [DISTINCT](../sql-reference/statements/select.md#select-distinct), [JOIN](../sql-reference/statements/select.md#select-join) ve kullandığınız diğer işlemler. +Gerekli RAM hacmini hesaplamak için, aşağıdakiler için geçici verilerin boyutunu tahmin etmelisiniz [GROUP BY](../sql-reference/statements/select/group-by.md#select-group-by-clause), [DISTINCT](../sql-reference/statements/select/distinct.md#select-distinct), [JOIN](../sql-reference/statements/select/join.md#select-join) ve kullandığınız diğer işlemler. -ClickHouse geçici veriler için harici bellek kullanabilirsiniz. Görmek [Harici bellekte grupla](../sql-reference/statements/select.md#select-group-by-in-external-memory) ayrıntılar için. +ClickHouse geçici veriler için harici bellek kullanabilirsiniz. Görmek [Harici bellekte grupla](../sql-reference/statements/select/group-by.md#select-group-by-in-external-memory) ayrıntılar için. ## Takas Dosyası {#swap-file} @@ -34,7 +34,7 @@ ClickHouse geçici veriler için harici bellek kullanabilirsiniz. Görmek [Haric ## Depolama Alt Sistemi {#storage-subsystem} -Clickhouse’u yüklemek için 2GB Boş disk alanına sahip olmanız gerekir. +Clickhouse'u yüklemek için 2GB Boş disk alanına sahip olmanız gerekir. Verileriniz için gereken depolama hacmi ayrı ayrı hesaplanmalıdır. Değerlendirme şunları içermelidir: @@ -44,7 +44,7 @@ Verileriniz için gereken depolama hacmi ayrı ayrı hesaplanmalıdır. Değerle - Veri sıkıştırma katsayısı. - Veri sıkıştırma katsayısını tahmin etmek için, verilerinizin bir örneğini Clickhouse’a yükleyin ve verilerin gerçek boyutunu depolanan tablonun boyutuyla karşılaştırın. Örneğin, clickstream verileri genellikle 6-10 kez sıkıştırılır. + Veri sıkıştırma katsayısını tahmin etmek için, verilerinizin bir örneğini Clickhouse'a yükleyin ve verilerin gerçek boyutunu depolanan tablonun boyutuyla karşılaştırın. Örneğin, clickstream verileri genellikle 6-10 kez sıkıştırılır. Saklanacak verilerin son hacmini hesaplamak için, sıkıştırma katsayısını tahmini veri hacmine uygulayın. Verileri birkaç yinelemede depolamayı planlıyorsanız, tahmini birimi yinelemelerin sayısıyla çarpın. @@ -56,6 +56,6 @@ Ağ bant genişliği, büyük miktarda Ara veriyle dağıtılmış sorguları i ## Yazılım {#software} -ClickHouse öncelikle Linux işletim sistemleri ailesi için geliştirilmiştir. Önerilen Linux dağıtımı Ubuntu’dur. Bu `tzdata` paket sisteme kurulmalıdır. +ClickHouse öncelikle Linux işletim sistemleri ailesi için geliştirilmiştir. Önerilen Linux dağıtımı Ubuntu'dur. Bu `tzdata` paket sisteme kurulmalıdır. ClickHouse diğer işletim sistemi ailelerinde de çalışabilir. Ayrıntıları görün [Başlarken](../getting-started/index.md) belgelerin bölümü. diff --git a/docs/tr/operations/server-configuration-parameters/index.md b/docs/tr/operations/server-configuration-parameters/index.md index 5b0da1ee7eb..0531896bf39 100644 --- a/docs/tr/operations/server-configuration-parameters/index.md +++ b/docs/tr/operations/server-configuration-parameters/index.md @@ -1,18 +1,18 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_folder_title: "Sunucu Yap\u0131land\u0131rma Parametreleri" toc_priority: 54 toc_title: "Giri\u015F" --- -# Sunucu yapılandırma Parametreleri {#server-settings} +# Sunucu Yapılandırma Parametreleri {#server-settings} Bu bölüm, oturum veya sorgu düzeyinde değiştirilemeyen sunucu ayarlarının açıklamalarını içerir. Bu ayarlar saklanır `config.xml` ClickHouse sunucusunda dosya. -Diğer ayarlar aşağıda açıklanmıştır “[Ayarlar](../settings/index.md#settings)” bölme. +Diğer ayarlar aşağıda açıklanmıştır “[Ayarlar](../settings/index.md#session-settings-intro)” bölme. Ayarları incelemeden önce, [Yapılandırma dosyaları](../configuration-files.md#configuration_files) bölüm ve ikame kullanımı (not `incl` ve `optional` öznitelik). diff --git a/docs/tr/operations/server-configuration-parameters/settings.md b/docs/tr/operations/server-configuration-parameters/settings.md index 1bb70e49198..a944261de8d 100644 --- a/docs/tr/operations/server-configuration-parameters/settings.md +++ b/docs/tr/operations/server-configuration-parameters/settings.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 57 toc_title: "Sunucu Ayarlar\u0131" --- @@ -453,7 +453,7 @@ Sunucu/istemci ayarları için tuşlar: - privateKeyFile – The path to the file with the secret key of the PEM certificate. The file may contain a key and certificate at the same time. - certificateFile – The path to the client/server certificate file in PEM format. You can omit it if `privateKeyFile` sertifika içerir. - caConfig – The path to the file or directory that contains trusted root certificates. -- verificationMode – The method for checking the node’s certificates. Details are in the description of the [Bağlama](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/Context.h) sınıf. Olası değerler: `none`, `relaxed`, `strict`, `once`. +- verificationMode – The method for checking the node's certificates. Details are in the description of the [Bağlam](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/Context.h) sınıf. Olası değerler: `none`, `relaxed`, `strict`, `once`. - verificationDepth – The maximum length of the verification chain. Verification will fail if the certificate chain length exceeds the set value. - loadDefaultCAFile – Indicates that built-in CA certificates for OpenSSL will be used. Acceptable values: `true`, `false`. \| - cipherList – Supported OpenSSL encryptions. For example: `ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH`. @@ -465,7 +465,7 @@ Sunucu/istemci ayarları için tuşlar: - requireTLSv1 – Require a TLSv1 connection. Acceptable values: `true`, `false`. - requireTLSv1\_1 – Require a TLSv1.1 connection. Acceptable values: `true`, `false`. - requireTLSv1 – Require a TLSv1.2 connection. Acceptable values: `true`, `false`. -- fips – Activates OpenSSL FIPS mode. Supported if the library’s OpenSSL version supports FIPS. +- fips – Activates OpenSSL FIPS mode. Supported if the library's OpenSSL version supports FIPS. - privateKeyPassphraseHandler – Class (PrivateKeyPassphraseHandler subclass) that requests the passphrase for accessing the private key. For example: ``, `KeyFileHandler`, `test`, ``. - invalidCertificateHandler – Class (a subclass of CertificateHandler) for verifying invalid certificates. For example: ` ConsoleCertificateHandler ` . - disableProtocols – Protocols that are not allowed to use. @@ -689,7 +689,7 @@ Sunucunun saat dilimi. UTC saat dilimi veya coğrafi konum (örneğin, Afrika / Abidjan) için bir IANA tanımlayıcısı olarak belirtilir. -Saat dilimi, datetime alanları metin biçimine (ekranda veya dosyada yazdırıldığında) çıktığında ve datetime’ı bir dizeden alırken dize ve DateTime biçimleri arasındaki dönüşümler için gereklidir. Ayrıca, saat dilimi, giriş parametrelerinde saat dilimini almadıkları takdirde saat ve tarih ile çalışan işlevlerde kullanılır. +Saat dilimi, datetime alanları metin biçimine (ekranda veya dosyada yazdırıldığında) çıktığında ve datetime'ı bir dizeden alırken dize ve DateTime biçimleri arasındaki dönüşümler için gereklidir. Ayrıca, saat dilimi, giriş parametrelerinde saat dilimini almadıkları takdirde saat ve tarih ile çalışan işlevlerde kullanılır. **Örnek** @@ -870,10 +870,10 @@ Bu ayar yalnızca `MergeTree` aile. Belirt specifiedilebilir: - 0 — Functionality is turned off. - 1 — Functionality is turned on. -Eğer `use_minimalistic_part_header_in_zookeeper = 1`, sonraları [çoğaltıyordu](../../engines/table-engines/mergetree-family/replication.md) tablolar, veri parçalarının başlıklarını tek bir `znode`. Tablo çok sayıda sütun içeriyorsa, bu depolama yöntemi Zookeeper’da depolanan verilerin hacmini önemli ölçüde azaltır. +Eğer `use_minimalistic_part_header_in_zookeeper = 1`, sonraları [çoğaltıyordu](../../engines/table-engines/mergetree-family/replication.md) tablolar, veri parçalarının başlıklarını tek bir `znode`. Tablo çok sayıda sütun içeriyorsa, bu depolama yöntemi Zookeeper'da depolanan verilerin hacmini önemli ölçüde azaltır. !!! attention "Dikkat" - Uyguladıktan sonra `use_minimalistic_part_header_in_zookeeper = 1`, ClickHouse sunucusunu bu ayarı desteklemeyen bir sürüme düşüremezsiniz. Bir kümedeki sunucularda ClickHouse yükseltirken dikkatli olun. Tüm sunucuları bir kerede yükseltmeyin. Clickhouse’un yeni sürümlerini bir test ortamında veya bir kümenin yalnızca birkaç sunucusunda test etmek daha güvenlidir. + Uyguladıktan sonra `use_minimalistic_part_header_in_zookeeper = 1`, ClickHouse sunucusunu bu ayarı desteklemeyen bir sürüme düşüremezsiniz. Bir kümedeki sunucularda ClickHouse yükseltirken dikkatli olun. Tüm sunucuları bir kerede yükseltmeyin. Clickhouse'un yeni sürümlerini bir test ortamında veya bir kümenin yalnızca birkaç sunucusunda test etmek daha güvenlidir. Data part headers already stored with this setting can't be restored to their previous (non-compact) representation. @@ -893,4 +893,14 @@ Güncelleştirme, ayrı bir sistem iş parçacığında zaman uyumsuz olarak ger **Varsayılan değer**: 15. +## access\_control\_path {#access_control_path} + +ClickHouse sunucusunun SQL komutları tarafından oluşturulan kullanıcı ve rol yapılandırmalarını depoladığı bir klasörün yolu. + +Varsayılan değer: `/var/lib/clickhouse/access/`. + +**Ayrıca bakınız** + +- [Erişim Kontrolü ve hesap yönetimi](../access-rights.md#access-control) + [Orijinal makale](https://clickhouse.tech/docs/en/operations/server_configuration_parameters/settings/) diff --git a/docs/tr/operations/settings/constraints-on-settings.md b/docs/tr/operations/settings/constraints-on-settings.md index a9319c2df69..d1e8aa58005 100644 --- a/docs/tr/operations/settings/constraints-on-settings.md +++ b/docs/tr/operations/settings/constraints-on-settings.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 62 toc_title: "Ayarlardaki k\u0131s\u0131tlamalar" --- diff --git a/docs/tr/operations/settings/index.md b/docs/tr/operations/settings/index.md index ed9d7f49151..181a33d7fb3 100644 --- a/docs/tr/operations/settings/index.md +++ b/docs/tr/operations/settings/index.md @@ -1,14 +1,15 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_folder_title: Ayarlar toc_priority: 55 toc_title: "Giri\u015F" --- -# Ayarlar {#settings} +# Ayarlar {#session-settings-intro} + +Belgelerin bu bölümünde açıklanan tüm ayarları yapmanın birden çok yolu vardır. -Aşağıda açıklanan tüm ayarları yapmanın birden çok yolu vardır. Ayarlar katmanlar halinde yapılandırılır, böylece sonraki her katman önceki ayarları yeniden tanımlar. Öncelik sırasına göre ayarları yapılandırma yolları: @@ -25,7 +26,7 @@ Ayarlar katmanlar halinde yapılandırılır, böylece sonraki her katman öncek - Sorgu ayarları. - ClickHouse konsol istemcisini etkileşimli olmayan modda başlatırken, başlangıç parametresini ayarlayın `--setting=value`. - - HTTP API’sini kullanırken, CGI parametrelerini geçirin (`URL?setting_1=value&setting_2=value...`). + - HTTP API'sini kullanırken, CGI parametrelerini geçirin (`URL?setting_1=value&setting_2=value...`). Yalnızca sunucu yapılandırma dosyasında yapılabilecek ayarlar bu bölümde yer almaz. diff --git a/docs/tr/operations/settings/permissions-for-queries.md b/docs/tr/operations/settings/permissions-for-queries.md index d4b023f5fb3..499d704230f 100644 --- a/docs/tr/operations/settings/permissions-for-queries.md +++ b/docs/tr/operations/settings/permissions-for-queries.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 58 toc_title: "Sorgular i\xE7in izinler" --- -# Sorgular için Izinler {#permissions_for_queries} +# Sorgular için izinler {#permissions_for_queries} -Clickhouse’daki sorgular birkaç türe ayrılabilir: +Clickhouse'daki sorgular birkaç türe ayrılabilir: 1. Veri sorgularını oku: `SELECT`, `SHOW`, `DESCRIBE`, `EXISTS`. 2. Veri sorgularını yaz: `INSERT`, `OPTIMIZE`. diff --git a/docs/tr/operations/settings/query-complexity.md b/docs/tr/operations/settings/query-complexity.md index 248f7884f60..c1880fc2ce7 100644 --- a/docs/tr/operations/settings/query-complexity.md +++ b/docs/tr/operations/settings/query-complexity.md @@ -1,12 +1,12 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 59 toc_title: "Sorgu karma\u015F\u0131kl\u0131\u011F\u0131 \xFCzerindeki k\u0131s\u0131\ tlamalar" --- -# Sorgu Karmaşıklığı Üzerindeki Kısıtlamalar {#restrictions-on-query-complexity} +# Sorgu karmaşıklığı üzerindeki kısıtlamalar {#restrictions-on-query-complexity} Sorgu karmaşıklığı üzerindeki kısıtlamalar ayarların bir parçasıdır. Kullanıcı arabiriminden daha güvenli yürütme sağlamak için kullanılırlar. @@ -22,13 +22,13 @@ ClickHouse, her satır için değil, veri bölümleri için kısıtlamaları den `break` – Stop executing the query and return the partial result, as if the source data ran out. -`any (only for group_by_overflow_mode)` – Continuing aggregation for the keys that got into the set, but don’t add new keys to the set. +`any (only for group_by_overflow_mode)` – Continuing aggregation for the keys that got into the set, but don't add new keys to the set. ## max\_memory\_usage {#settings_max_memory_usage} Tek bir sunucuda bir sorgu çalıştırmak için kullanılacak en fazla RAM miktarı. -Varsayılan yapılandırma dosyasında maksimum 10 GB’DİR. +Varsayılan yapılandırma dosyasında maksimum 10 GB'DİR. Bu ayar, kullanılabilir belleğin hacmini veya makinedeki toplam bellek hacmini dikkate almaz. Kısıtlama, tek bir sunucu içindeki tek bir sorgu için geçerlidir. @@ -79,15 +79,15 @@ Toplama alınan benzersiz anahtarların maksimum sayısı. Bu ayar, toplama sır ## group\_by\_overflow\_mode {#group-by-overflow-mode} Toplama için benzersiz anahtarların sayısı sınırı aştığında ne yapmalı: ‘throw’, ‘break’, veya ‘any’. Varsayılan olarak, atın. -Kullanarak ‘any’ değer, GROUP BY’NİN bir yaklaşımını çalıştırmanızı sağlar. Bu yaklaşımın kalitesi, verilerin istatistiksel niteliğine bağlıdır. +Kullanarak ‘any’ değer, GROUP BY'NİN bir yaklaşımını çalıştırmanızı sağlar. Bu yaklaşımın kalitesi, verilerin istatistiksel niteliğine bağlıdır. ## max\_bytes\_before\_external\_group\_by {#settings-max_bytes_before_external_group_by} -Çalıştırmayı etkinleştirir veya devre dışı bırakır `GROUP BY` harici bellekte yan tümceleri. Görmek [Harici bellekte grupla](../../sql-reference/statements/select.md#select-group-by-in-external-memory). +Çalıştırmayı etkinleştirir veya devre dışı bırakır `GROUP BY` harici bellekte yan tümceleri. Görmek [Harici bellekte grupla](../../sql-reference/statements/select/group-by.md#select-group-by-in-external-memory). Olası değerler: -- Tek tarafından kullanılabilecek maksimum RAM hacmi (bayt cinsinden) [GROUP BY](../../sql-reference/statements/select.md#select-group-by-clause) işleyiş. +- Tek tarafından kullanılabilecek maksimum RAM hacmi (bayt cinsinden) [GROUP BY](../../sql-reference/statements/select/group-by.md#select-group-by-clause) operasyon. - 0 — `GROUP BY` harici bellekte devre dışı. Varsayılan değer: 0. @@ -170,7 +170,7 @@ Tek bir sorguda bir tablodan okunabilen sütun sayısı. Bir sorgu daha fazla sa ## max\_temporary\_columns {#max-temporary-columns} -Sabit sütunlar da dahil olmak üzere bir sorgu çalıştırırken aynı anda RAM’de tutulması gereken geçici sütun sayısı. Bundan daha fazla geçici sütun varsa, bir istisna atar. +Sabit sütunlar da dahil olmak üzere bir sorgu çalıştırırken aynı anda RAM'de tutulması gereken geçici sütun sayısı. Bundan daha fazla geçici sütun varsa, bir istisna atar. ## max\_temporary\_non\_const\_columns {#max-temporary-non-const-columns} @@ -235,11 +235,11 @@ Veri miktarı sınırlardan birini aştığında ne yapmalı: ‘throw’ veya Tabloları birleştirirken kullanılan karma tablodaki satır sayısını sınırlar. -Bu ayarlar aşağıdakiler için geçerlidir [SELECT … JOIN](../../sql-reference/statements/select.md#select-join) işlemleri ve [Katmak](../../engines/table-engines/special/join.md) masa motoru. +Bu ayarlar aşağıdakiler için geçerlidir [SELECT … JOIN](../../sql-reference/statements/select/join.md#select-join) işlemleri ve [Katmak](../../engines/table-engines/special/join.md) masa motoru. Bir sorgu birden çok birleşim içeriyorsa, ClickHouse her Ara sonuç için bu ayarı denetler. -Limit ulaşıldığında ClickHouse farklı eylemlerle devam edebilirsiniz. Kullan… [join\_overflow\_mode](#settings-join_overflow_mode) eylemi seçmek için ayarlama. +Limit ulaşıldığında ClickHouse farklı eylemlerle devam edebilirsiniz. Kullan... [join\_overflow\_mode](#settings-join_overflow_mode) eylemi seçmek için ayarlama. Olası değerler: @@ -252,7 +252,7 @@ Varsayılan değer: 0. Tabloları birleştirirken kullanılan karma tablonun bayt cinsinden boyutunu sınırlar. -Bu ayarlar aşağıdakiler için geçerlidir [SELECT … JOIN](../../sql-reference/statements/select.md#select-join) işlemleri ve [Jo tablein table engine](../../engines/table-engines/special/join.md). +Bu ayarlar aşağıdakiler için geçerlidir [SELECT … JOIN](../../sql-reference/statements/select/join.md#select-join) işlemleri ve [Jo tablein table engine](../../engines/table-engines/special/join.md). Sorgu birleşimler içeriyorsa, ClickHouse her Ara sonuç için bu ayarı denetler. @@ -275,13 +275,13 @@ Tanımlar katılın aşağıdaki sınırlar her zaman eylem ClickHouse gerçekle Olası değerler: - `THROW` — ClickHouse throws an exception and breaks operation. -- `BREAK` — ClickHouse breaks operation and doesn’t throw an exception. +- `BREAK` — ClickHouse breaks operation and doesn't throw an exception. Varsayılan değer: `THROW`. **Ayrıca Bakınız** -- [Jo](../../sql-reference/statements/select.md#select-join) +- [Jo](../../sql-reference/statements/select/join.md#select-join) - [Jo tablein table engine](../../engines/table-engines/special/join.md) ## max\_partitions\_per\_ınsert\_block {#max-partitions-per-insert-block} diff --git a/docs/tr/operations/settings/settings-profiles.md b/docs/tr/operations/settings/settings-profiles.md index 318276ab6c8..d5fa58384d7 100644 --- a/docs/tr/operations/settings/settings-profiles.md +++ b/docs/tr/operations/settings/settings-profiles.md @@ -1,13 +1,21 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 61 toc_title: Ayarlar Profilleri --- # Ayarlar Profilleri {#settings-profiles} -Ayarlar profili, aynı ad altında gruplandırılmış ayarlar topluluğudur. Her ClickHouse kullanıcısının bir profili vardır. +Ayarlar profili, aynı ad altında gruplandırılmış ayarlar topluluğudur. + +!!! note "Bilgi" + ClickHouse da destekler [SQL tabanlı iş akışı](../access-rights.md#access-control) ayarları profilleri yönetmek için. Bunu kullanmanızı öneririz. + +Bir profilin herhangi bir adı olabilir. Profilin herhangi bir adı olabilir. Farklı kullanıcılar için aynı profili belirtebilirsiniz. Ayarlar profilinde yazabileceğiniz en önemli şey `readonly=1` sağlar okumak-sadece erişim. + +Ayarlar profilleri birbirinden miras alabilir. Kalıtım kullanmak için, bir veya birden fazla belirtiniz `profile` ayarlar profilde listelenen diğer ayarlardan önce. Farklı profillerde bir ayar tanımlandığında, en son tanımlı kullanılır. + Bir profildeki tüm ayarları uygulamak için `profile` ayar. Örnek: @@ -64,8 +72,10 @@ Ayarlar profilleri kullanıcı yapılandırma dosyasında bildirilir. Bu genelli ``` -Örnek iki profili belirtir: `default` ve `web`. Bu `default` profilin özel bir amacı vardır: her zaman mevcut olmalı ve sunucuyu başlatırken uygulanır. Diğer bir deyişle, `default` profil varsayılan ayarları içerir. Bu `web` profil kullanılarak ayarlanabilir düzenli bir profil `SET` sorgu veya bir HTTP sorgusunda bir URL parametresi kullanma. +Örnek iki profili belirtir: `default` ve `web`. -Ayarlar profilleri birbirinden miras alabilir. Kalıtım kullanmak için, bir veya birden fazla belirtiniz `profile` ayarlar profilde listelenen diğer ayarlardan önce. Farklı profillerde bir ayar tanımlandığında, en son tanımlı kullanılır. +Bu `default` profilin özel bir amacı vardır: her zaman mevcut olmalı ve sunucuyu başlatırken uygulanır. Diğer bir deyişle, `default` profil varsayılan ayarları içerir. + +Bu `web` profil kullanılarak ayarlanabilir düzenli bir profil `SET` sorgu veya bir HTTP sorgusunda bir URL parametresi kullanma. [Orijinal makale](https://clickhouse.tech/docs/en/operations/settings/settings_profiles/) diff --git a/docs/tr/operations/settings/settings-users.md b/docs/tr/operations/settings/settings-users.md index 43fb14f4013..eead71c0a77 100644 --- a/docs/tr/operations/settings/settings-users.md +++ b/docs/tr/operations/settings/settings-users.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 63 toc_title: "Kullan\u0131c\u0131 Ayarlar\u0131" --- @@ -9,6 +9,9 @@ toc_title: "Kullan\u0131c\u0131 Ayarlar\u0131" Bu `users` bu bölüm `user.xml` yapılandırma dosyası kullanıcı ayarlarını içerir. +!!! note "Bilgi" + ClickHouse da destekler [SQL tabanlı iş akışı](../access-rights.md#access-control) kullanıcıları yönetmek için. Bunu kullanmanızı öneririz. + Bu yapı `users` bölme: ``` xml @@ -19,6 +22,8 @@ Bu yapı `users` bölme: + 0|1 + @@ -70,6 +75,17 @@ Bu yapı `users` bölme: Sonucun ilk satırı şifredir. İkinci satır karşılık gelen çift SHA1 karmasıdır. +### access\_management {#access_management-user-setting} + +Bu ayar, SQL-driven kullanarak devre dışı bırakır sağlar [erişim kontrolü ve hesap yönetimi](../access-rights.md#access-control) kullanıcı için. + +Olası değerler: + +- 0 — Disabled. +- 1 — Enabled. + +Varsayılan değer: 0. + ### kullanıcı\_adı / ağlar {#user-namenetworks} Kullanıcının ClickHouse sunucusuna bağlanabileceği ağların listesi. @@ -90,7 +106,7 @@ Listenin her öğesi aşağıdaki formlardan birine sahip olabilir: Örnek, `^example\d\d-\d\d-\d\.host\.ru$` - Erişimi kontrol etmek için, bir [DNS ptr sorgusu](https://en.wikipedia.org/wiki/Reverse_DNS_lookup) eş adresi için gerçekleştirilir ve sonra belirtilen regexp uygulanır. Daha sonra PTR sorgusunun sonuçları için başka bir DNS sorgusu gerçekleştirilir ve alınan tüm adresler eş adresine karşılaştırılır. Regexp’nin $ile bitmesini şiddetle tavsiye ederiz. + Erişimi kontrol etmek için, bir [DNS ptr sorgusu](https://en.wikipedia.org/wiki/Reverse_DNS_lookup) eş adresi için gerçekleştirilir ve sonra belirtilen regexp uygulanır. Daha sonra PTR sorgusunun sonuçları için başka bir DNS sorgusu gerçekleştirilir ve alınan tüm adresler eş adresine karşılaştırılır. Regexp'nin $ile bitmesini şiddetle tavsiye ederiz. Sunucu yeniden başlatılıncaya kadar DNS isteklerinin tüm sonuçları önbelleğe alınır. @@ -105,7 +121,7 @@ Herhangi bir ağdan kullanıcı için erişimi açmak için şunları belirtin: !!! warning "Uyarıcı" Düzgün yapılandırılmış bir güvenlik duvarınız yoksa veya sunucu doğrudan internete bağlı değilse, herhangi bir ağdan erişimi açmak güvensizdir. -Erişimi yalnızca localhost’tan açmak için şunları belirtin: +Erişimi yalnızca localhost'tan açmak için şunları belirtin: ``` xml ::1 @@ -129,7 +145,7 @@ Bu bölümde, ClickHouse tarafından döndürülen satırları sınırlayabilirs **Örnek** -Aşağıdaki yapılandırma bu kullanıcıyı zorlar `user1` sadece satırları görebilirsiniz `table1` sonucu olarak `SELECT` sorgular, burada değeri `id` alan 1000’dir. +Aşağıdaki yapılandırma bu kullanıcıyı zorlar `user1` sadece satırları görebilirsiniz `table1` sonucu olarak `SELECT` sorgular, burada değeri `id` alan 1000'dir. ``` xml diff --git a/docs/tr/operations/settings/settings.md b/docs/tr/operations/settings/settings.md index 8e094174454..342c35caab2 100644 --- a/docs/tr/operations/settings/settings.md +++ b/docs/tr/operations/settings/settings.md @@ -1,15 +1,13 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 -toc_priority: 60 -toc_title: Ayarlar +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd --- # Ayarlar {#settings} ## distributed\_product\_mode {#distributed-product-mode} -Davranışını değiştirir [dağıtılmış alt sorgular](../../sql-reference/statements/select.md). +Davranışını değiştirir [dağıtılmış alt sorgular](../../sql-reference/operators/in.md). ClickHouse applies this setting when the query contains the product of distributed tables, i.e. when the query for a distributed table contains a non-GLOBAL subquery for the distributed table. @@ -18,7 +16,7 @@ Kısıtlama: - Yalnızca ın ve JOIN alt sorguları için uygulanır. - Yalnızca FROM bölümü birden fazla parça içeren dağıtılmış bir tablo kullanıyorsa. - Alt sorgu birden fazla parça içeren dağıtılmış bir tablo ile ilgiliyse. -- Bir tablo için kullanılmaz-değerli [uzak](../../sql-reference/table-functions/remote.md) işlev. +- Bir tablo için kullanılmaz-değerli [uzak](../../sql-reference/table-functions/remote.md) İşlev. Olası değerler: @@ -79,7 +77,7 @@ Eğer `force_primary_key=1`, ClickHouse, sorgunun veri aralıklarını kısıtla ## format\_schema {#format-schema} -Bu parametre, aşağıdaki gibi bir şema tanımı gerektiren biçimler kullanırken kullanışlıdır [Cap’n Proto](https://capnproto.org/) veya [Protobuf](https://developers.google.com/protocol-buffers/). Değer biçime bağlıdır. +Bu parametre, aşağıdaki gibi bir şema tanımı gerektiren biçimler kullanırken kullanışlıdır [Cap'n Proto](https://capnproto.org/) veya [Protobuf](https://developers.google.com/protocol-buffers/). Değer biçime bağlıdır. ## fsync\_metadata {#fsync-metadata} @@ -104,7 +102,7 @@ Varsayılan değer: 0. Eğer bir HTTP isteğine yanıt veri sıkıştırma düzeyini ayarlar [enable\_http\_compression = 1](#settings-enable_http_compression). -Olası değerler: 1’den 9’a kadar olan sayılar. +Olası değerler: 1'den 9'a kadar olan sayılar. Varsayılan değer: 3. @@ -149,7 +147,7 @@ Varsayılan değer: 0. Metin biçimlerinden (CSV, TSV, vb.) okurken kabul edilebilir hataların maksimum sayısını ayarlar.). -Varsayılan değer 0’dır. +Varsayılan değer 0'dır. Her zaman ile eşleştirmek `input_format_allow_errors_ratio`. @@ -162,7 +160,7 @@ Eğer her ikisi de `input_format_allow_errors_num` ve `input_format_allow_errors Metin biçimlerinden (CSV, TSV, vb.) okurken izin verilen maksimum hata yüzdesini ayarlar.). Hataların yüzdesi 0 ile 1 arasında kayan nokta sayısı olarak ayarlanır. -Varsayılan değer 0’dır. +Varsayılan değer 0'dır. Her zaman ile eşleştirmek `input_format_allow_errors_num`. @@ -279,7 +277,7 @@ Varsayılan değer: 1. ## ınput\_format\_tsv\_empty\_as\_default {#settings-input-format-tsv-empty-as-default} -Etkinleştirildiğinde, TSV’DEKİ boş giriş alanlarını varsayılan değerlerle değiştirin. Karmaşık varsayılan ifadeler için `input_format_defaults_for_omitted_fields` de etkin olmalıdır. +Etkinleştirildiğinde, TSV'DEKİ boş giriş alanlarını varsayılan değerlerle değiştirin. Karmaşık varsayılan ifadeler için `input_format_defaults_for_omitted_fields` de etkin olmalıdır. Varsayılan olarak devre dışı. @@ -324,7 +322,7 @@ Varsayılan değer: 0. Ayrıca bakınız: -- [İç içe yapıların kullanımı](../../interfaces/formats.md#jsoneachrow-nested) ile… `JSONEachRow` biçimli. +- [İç içe yapıların kullanımı](../../interfaces/formats.md#jsoneachrow-nested) ile... `JSONEachRow` biçimli. ## ınput\_format\_with\_names\_use\_header {#settings-input-format-with-names-use-header} @@ -369,11 +367,11 @@ Ayrıca bakınız: ## join\_default\_strictness {#settings-join_default_strictness} -Ayarlar varsayılan strictness için [Maddeleri KATILIN](../../sql-reference/statements/select.md#select-join). +Ayarlar varsayılan strictness için [Maddeleri KATILIN ](../../sql-reference/statements/select/join.md#select-join). Olası değerler: -- `ALL` — If the right table has several matching rows, ClickHouse creates a [Kartezyen ürün](https://en.wikipedia.org/wiki/Cartesian_product) eşleşen satırlardan. Bu normaldir `JOIN` standart SQL’DEN davranış. +- `ALL` — If the right table has several matching rows, ClickHouse creates a [Kartezyen ürün](https://en.wikipedia.org/wiki/Cartesian_product) eşleşen satırlardan. Bu normaldir `JOIN` standart SQL'DEN davranış. - `ANY` — If the right table has several matching rows, only the first one found is joined. If the right table has only one matching row, the results of `ANY` ve `ALL` aynı. - `ASOF` — For joining sequences with an uncertain match. - `Empty string` — If `ALL` veya `ANY` sorguda belirtilmezse, ClickHouse bir özel durum atar. @@ -396,13 +394,13 @@ Varsayılan değer: 0. Ayrıca bakınız: -- [Jo](../../sql-reference/statements/select.md#select-join) +- [Jo](../../sql-reference/statements/select/join.md#select-join) - [Jo tablein table engine](../../engines/table-engines/special/join.md) - [join\_default\_strictness](#settings-join_default_strictness) ## join\_use\_nulls {#join_use_nulls} -Türünü ayarlar [JOIN](../../sql-reference/statements/select.md) davranış. Tabloları birleştirirken boş hücreler görünebilir. ClickHouse bu ayara göre onları farklı şekilde doldurur. +Türünü ayarlar [JOIN](../../sql-reference/statements/select/join.md) davranış. Tabloları birleştirirken boş hücreler görünebilir. ClickHouse bu ayara göre onları farklı şekilde doldurur. Olası değerler: @@ -413,7 +411,7 @@ Varsayılan değer: 0. ## max\_block\_size {#setting-max_block_size} -Clickhouse’da, veriler bloklarla (sütun parçaları kümeleri) işlenir. Tek bir blok için dahili işlem döngüleri yeterince verimlidir, ancak her blokta gözle görülür harcamalar vardır. Bu `max_block_size` ayar, blokun boyutunun (satır sayımında) tablolardan yükleneceği bir öneridir. Blok boyutu çok küçük olmamalı, böylece her bloktaki harcamalar hala fark edilebilir, ancak çok büyük olmamalı, böylece ilk blok hızla işlendikten sonra tamamlanan limitli sorgu çok büyük olmamalıdır. Amaç, birden çok iş parçacığında çok sayıda sütun ayıklarken çok fazla bellek tüketmekten kaçınmak ve en azından bazı önbellek konumlarını korumaktır. +Clickhouse'da, veriler bloklarla (sütun parçaları kümeleri) işlenir. Tek bir blok için dahili işlem döngüleri yeterince verimlidir, ancak her blokta gözle görülür harcamalar vardır. Bu `max_block_size` ayar, blokun boyutunun (satır sayımında) tablolardan yükleneceği bir öneridir. Blok boyutu çok küçük olmamalı, böylece her bloktaki harcamalar hala fark edilebilir, ancak çok büyük olmamalı, böylece ilk blok hızla işlendikten sonra tamamlanan limitli sorgu çok büyük olmamalıdır. Amaç, birden çok iş parçacığında çok sayıda sütun ayıklarken çok fazla bellek tüketmekten kaçınmak ve en azından bazı önbellek konumlarını korumaktır. Varsayılan değer: 65,536. @@ -516,7 +514,7 @@ Varsayılan değer: 0. Sorgu günlüğü ayarlama. -Bu kurulum ile Clickhouse’a gönderilen sorgular, [query\_log](../server-configuration-parameters/settings.md#server_configuration_parameters-query-log) sunucu yapılandırma parametresi. +Bu kurulum ile Clickhouse'a gönderilen sorgular, [query\_log](../server-configuration-parameters/settings.md#server_configuration_parameters-query-log) sunucu yapılandırma parametresi. Örnek: @@ -564,7 +562,29 @@ Veri SELECT sonra oluşturulan aynı blokları kullanarak eklendiğinden, INSERT Varsayılan değer: 1.048,576. -Varsayılan biraz daha fazla `max_block_size`. Bunun nedeni, bazı tablo motorlarının (`*MergeTree`) oldukça büyük bir varlık olan eklenen her blok için diskte bir veri parçası oluşturun. Benzer bir şekilde, `*MergeTree` tablolar ekleme sırasında verileri sıralar ve yeterince büyük bir blok boyutu RAM’de daha fazla veriyi sıralamaya izin verir. +Varsayılan biraz daha fazla `max_block_size`. Bunun nedeni, bazı tablo motorlarının (`*MergeTree`) oldukça büyük bir varlık olan eklenen her blok için diskte bir veri parçası oluşturun. Benzer bir şekilde, `*MergeTree` tablolar ekleme sırasında verileri sıralar ve yeterince büyük bir blok boyutu RAM'de daha fazla veriyi sıralamaya izin verir. + +## min\_insert\_block\_size\_rows {#min-insert-block-size-rows} + +Bir tabloya eklenebilen blok içindeki minimum satır sayısını ayarlar. `INSERT` sorgu. Daha küçük boyutlu bloklar daha büyük olanlara ezilir. + +Olası değerler: + +- Pozitif tamsayı. +- 0 — Squashing disabled. + +Varsayılan değer: 1048576. + +## min\_insert\_block\_size\_bytes {#min-insert-block-size-bytes} + +Bir tabloya eklenebilen blok içindeki minimum bayt sayısını ayarlar. `INSERT` sorgu. Daha küçük boyutlu bloklar daha büyük olanlara ezilir. + +Olası değerler: + +- Pozitif tamsayı. +- 0 — Squashing disabled. + +Varsayılan değer: 268435456. ## max\_replica\_delay\_for\_distributed\_queries {#settings-max_replica_delay_for_distributed_queries} @@ -587,7 +607,7 @@ Varsayılan değer: fiziksel CPU çekirdeği sayısı. Bir kerede bir sunucuda normal olarak birden az SELECT sorgusu çalıştırılırsa, bu parametreyi gerçek işlemci çekirdeği sayısından biraz daha küçük bir değere ayarlayın. -Bir sınır nedeniyle hızlı bir şekilde tamamlanan sorgular için, daha düşük bir ‘max\_threads’. Örneğin, gerekli sayıda giriş her blokta ve max\_threads = 8’de bulunuyorsa, sadece bir tane okumak için yeterli olsa da, 8 blok alınır. +Bir sınır nedeniyle hızlı bir şekilde tamamlanan sorgular için, daha düşük bir ‘max\_threads’. Örneğin, gerekli sayıda giriş her blokta ve max\_threads = 8'de bulunuyorsa, sadece bir tane okumak için yeterli olsa da, 8 blok alınır. Daha küçük `max_threads` değer, daha az bellek tüketilir. @@ -598,7 +618,7 @@ Daha küçük `max_threads` değer, daha az bellek tüketilir. Olası değerler: - 0 (or 1) — `INSERT SELECT` paralel infaz yok. -- Pozitif tamsayı. 1’den büyük. +- Pozitif tamsayı. 1'den büyük. Varsayılan değer: 0. @@ -621,13 +641,13 @@ Bir örneğe bakalım. Varsaymak ‘index\_granularity’ tablo oluşturma sıra Bir uint32 tipi sütun yazıyoruz (değer başına 4 bayt). 8192 satır yazarken, toplam 32 KB veri olacaktır. Min\_compress\_block\_size = 65.536 olduğundan, her iki işaret için sıkıştırılmış bir blok oluşturulacaktır. -Dize türüne sahip bir URL sütunu yazıyoruz (değer başına ortalama 60 bayt boyutu). 8192 satır yazarken, ortalama 500 KB veri biraz daha az olacaktır. Bu 65,536’dan fazla olduğu için, her işaret için sıkıştırılmış bir blok oluşturulacaktır. Bu durumda, diskteki verileri tek bir işaret aralığında okurken, ekstra veriler sıkıştırılmaz. +Dize türüne sahip bir URL sütunu yazıyoruz (değer başına ortalama 60 bayt boyutu). 8192 satır yazarken, ortalama 500 KB veri biraz daha az olacaktır. Bu 65,536'dan fazla olduğu için, her işaret için sıkıştırılmış bir blok oluşturulacaktır. Bu durumda, diskteki verileri tek bir işaret aralığında okurken, ekstra veriler sıkıştırılmaz. Bu ayarı değiştirmek için genellikle herhangi bir neden yoktur. ## max\_query\_size {#settings-max_query_size} -SQL ayrıştırıcısı ile ayrıştırmak için RAM’e alınabilecek bir sorgunun en büyük kısmı. +SQL ayrıştırıcısı ile ayrıştırmak için RAM'e alınabilecek bir sorgunun en büyük kısmı. INSERT sorgusu, bu kısıtlamaya dahil olmayan ayrı bir akış ayrıştırıcısı (o(1) RAM tüketir) tarafından işlenen INSERT için veri de içerir. Varsayılan değer: 256 KiB. @@ -700,7 +720,7 @@ En azından biraz büyük bir veri hacmi (bir milyon satır veya daha fazla) oku HTTP arayüzünü kullanırken, ‘query\_id’ parametre geçirilebilir. Bu, sorgu tanımlayıcısı olarak hizmet veren herhangi bir dizedir. Aynı kullanıcıdan aynı sorgu varsa ‘query\_id’ zaten şu anda var, davranış bağlıdır ‘replace\_running\_query’ parametre. -`0` (default) – Throw an exception (don’t allow the query to run if a query with the same ‘query\_id’ zaten çalışan) var. +`0` (default) – Throw an exception (don't allow the query to run if a query with the same ‘query\_id’ zaten çalışan) var. `1` – Cancel the old query and start running the new one. @@ -710,7 +730,7 @@ Aynı kullanıcıdan aynı sorgu varsa ‘query\_id’ zaten şu anda var, davra Bir zaman aşımı durumunda akışlı tablolar için çalışır veya bir iş parçacığı oluşturduğunda [max\_ınsert\_block\_size](#settings-max_insert_block_size) satırlar. -Varsayılan değer 7500’dür. +Varsayılan değer 7500'dür. Küçük değer, daha sık veri tablosuna temizlendi. Değeri çok düşük ayarlamak, düşük performansa yol açar. @@ -725,7 +745,7 @@ ClickHouse kopyaları seçme aşağıdaki algoritmaları destekler: - [Sıralı](#load_balancing-in_order) - [İlk veya rastgele](#load_balancing-first_or_random) -### Rastgele (varsayılan Olarak) {#load_balancing-random} +### Rastgele (varsayılan olarak) {#load_balancing-random} ``` sql load_balancing = random @@ -740,7 +760,7 @@ Dezavantajları: sunucu yakınlık hesaba değil; kopyaları farklı veri varsa, load_balancing = nearest_hostname ``` -The number of errors is counted for each replica. Every 5 minutes, the number of errors is integrally divided by 2. Thus, the number of errors is calculated for a recent time with exponential smoothing. If there is one replica with a minimal number of errors (i.e. errors occurred recently on the other replicas), the query is sent to it. If there are multiple replicas with the same minimal number of errors, the query is sent to the replica with a hostname that is most similar to the server’s hostname in the config file (for the number of different characters in identical positions, up to the minimum length of both hostnames). +The number of errors is counted for each replica. Every 5 minutes, the number of errors is integrally divided by 2. Thus, the number of errors is calculated for a recent time with exponential smoothing. If there is one replica with a minimal number of errors (i.e. errors occurred recently on the other replicas), the query is sent to it. If there are multiple replicas with the same minimal number of errors, the query is sent to the replica with a hostname that is most similar to the server's hostname in the config file (for the number of different characters in identical positions, up to the minimum length of both hostnames). Örneğin, example01-01-1 ve example01-01-2.yandex.ru bir pozisyonda farklıdır, örneği01-01-1 ve örneği01-02-2 iki yerde farklılık gösterir. Bu yöntem ilkel görünebilir, ancak ağ topolojisi hakkında harici veri gerektirmez ve IPv6 adreslerimiz için karmaşık olan IP adreslerini karşılaştırmaz. @@ -757,7 +777,7 @@ load_balancing = in_order Yapılandırmada belirtilen hataları aynı sayıda yinelemeler aynı sırayla erişilir. Bu yöntem, tam olarak hangi kopyanın tercih edildiğini bildiğinizde uygundur. -### İlk Veya Rastgele {#load_balancing-first_or_random} +### İlk veya rastgele {#load_balancing-first_or_random} ``` sql load_balancing = first_or_random @@ -811,7 +831,7 @@ For testing, the value can be set to 0: compilation runs synchronously and the q Değer 1 veya daha fazla ise, derleme zaman uyumsuz olarak ayrı bir iş parçacığında oluşur. Sonuç, şu anda çalışmakta olan sorgular da dahil olmak üzere hazır olduğu anda kullanılacaktır. Derlenmiş kod, sorguda kullanılan toplama işlevlerinin her farklı birleşimi ve GROUP BY yan tümcesindeki anahtarların türü için gereklidir. -The results of the compilation are saved in the build directory in the form of .so files. There is no restriction on the number of compilation results since they don’t use very much space. Old results will be used after server restarts, except in the case of a server upgrade – in this case, the old results are deleted. +The results of the compilation are saved in the build directory in the form of .so files. There is no restriction on the number of compilation results since they don't use very much space. Old results will be used after server restarts, except in the case of a server upgrade – in this case, the old results are deleted. ## output\_format\_json\_quote\_64bit\_integers {#session_settings-output_format_json_quote_64bit_integers} @@ -827,11 +847,11 @@ CSV giriş biçimi sağlar veya unquoted ayrıştırma devre dışı bırakır i ## output\_format\_csv\_crlf\_end\_of\_line {#settings-output-format-csv-crlf-end-of-line} -Unix stili (LF) yerine CSV’DE DOS/Windows stili çizgi ayırıcı (CRLF) kullanın. +Unix stili (LF) yerine CSV'DE DOS/Windows stili çizgi ayırıcı (CRLF) kullanın. ## output\_format\_tsv\_crlf\_end\_of\_line {#settings-output-format-tsv-crlf-end-of-line} -Unıx stili (LF) yerine TSV’DE DOC/Windows stili çizgi ayırıcı (CRLF) kullanın. +Unıx stili (LF) yerine TSV'DE DOC/Windows stili çizgi ayırıcı (CRLF) kullanın. ## insert\_quorum {#settings-insert_quorum} @@ -902,7 +922,7 @@ Olası değerler: Varsayılan değer: 1. -Varsayılan olarak, çoğaltılmış tablolara eklenen bloklar `INSERT` deyim tekilleştirilir (bkz. \[Data Replication\] (../engines/table\_engines/mergetree\_family/replication.md). +Varsayılan olarak, çoğaltılmış tablolara eklenen bloklar `INSERT` açıklama tekilleştirilmiştir (bkz [Veri Çoğaltma](../../engines/table-engines/mergetree-family/replication.md)). ## deduplicate\_blocks\_ın\_dependent\_materialized\_views {#settings-deduplicate-blocks-in-dependent-materialized-views} @@ -919,7 +939,7 @@ Kullanma Varsayılan olarak, tekilleştirme materialized görünümler için gerçekleştirilmez, ancak kaynak tabloda, Yukarı akış yapılır. Eklenen bir blok, kaynak tablodaki tekilleştirme nedeniyle atlanırsa, ekli materialized görünümlerine ekleme olmaz. Bu davranış, eklenen blokların materialized görünüm toplamasından sonra aynı olduğu, ancak kaynak tabloya farklı eklerden türetildiği durumlar için, yüksek oranda toplanmış verilerin materialized görünümlere eklenmesini sağlamak için vardır. -Aynı zamanda, bu davranış “breaks” `INSERT` idempotency. Eğer bir `INSERT` ana tabloya başarılı oldu ve `INSERT` into a materialized view failed (e.g. because of communication failure with Zookeeper) a client will get an error and can retry the operation. However, the materialized view won’t receive the second insert because it will be discarded by deduplication in the main (source) table. The setting `deduplicate_blocks_in_dependent_materialized_views` bu davranışı değiştirmeye izin verir. Yeniden denemede, somutlaştırılmış bir görünüm tekrar ekleme işlemini alacak ve tekilleştirme kontrolünü kendi başına gerçekleştirecektir, +Aynı zamanda, bu davranış “breaks” `INSERT` idempotency. Eğer bir `INSERT` ana tabloya başarılı oldu ve `INSERT` into a materialized view failed (e.g. because of communication failure with Zookeeper) a client will get an error and can retry the operation. However, the materialized view won't receive the second insert because it will be discarded by deduplication in the main (source) table. The setting `deduplicate_blocks_in_dependent_materialized_views` bu davranışı değiştirmeye izin verir. Yeniden denemede, somutlaştırılmış bir görünüm tekrar ekleme işlemini alacak ve tekilleştirme kontrolünü kendi başına gerçekleştirecektir, kaynak tablo için onay sonucunu yoksayar ve ilk hata nedeniyle kaybedilen satırları ekler. ## max\_network\_bytes {#settings-max-network-bytes} @@ -968,7 +988,7 @@ Varsayılan değer: 0. ## count\_distinct\_implementation {#settings-count_distinct_implementation} -Aşağıdakilerden hang theisinin `uniq*` işlevleri gerçekleştirmek için kullanılmalıdır [COUNT(DISTINCT …)](../../sql-reference/aggregate-functions/reference.md#agg_function-count) yapma. +Aşağıdakilerden hang theisinin `uniq*` işlevleri gerçekleştirmek için kullanılmalıdır [COUNT(DISTINCT …)](../../sql-reference/aggregate-functions/reference.md#agg_function-count) yapı. Olası değerler: @@ -994,7 +1014,7 @@ Tüm kopyaları kullanılamıyorsa, Shard kullanılamaz olarak kabul edilir. Aş Çoğaltmanın ana bilgisayar adı DNS aracılığıyla çözümlenemezse, aşağıdaki durumları gösterebilir: - - Çoğaltma ana bilgisayar DNS kaydı yok. Dinamik DNS’YE sahip sistemlerde oluşabilir, örneğin, [Kubernetes](https://kubernetes.io), burada düğümler kesinti sırasında çözülmez olabilir ve bu bir hata değildir. + - Çoğaltma ana bilgisayar DNS kaydı yok. Dinamik DNS'YE sahip sistemlerde oluşabilir, örneğin, [Kubernetes](https://kubernetes.io), burada düğümler kesinti sırasında çözülmez olabilir ve bu bir hata değildir. - Yapılandırma hatası. ClickHouse yapılandırma dosyası yanlış bir ana bilgisayar adı içerir. @@ -1232,4 +1252,14 @@ Type: URL Varsayılan değer: boş +## background\_pool\_size {#background_pool_size} + +Tablo altyapılarında arka plan işlemlerini gerçekleştiren iş parçacıklarının sayısını ayarlar (örneğin, [MergeTree motoru](../../engines/table-engines/mergetree-family/index.md) Tablolar). Bu ayar ClickHouse sunucu başlangıcında uygulanır ve bir kullanıcı oturumunda değiştirilemez. Bu ayarı ayarlayarak, CPU ve disk yükünü yönetirsiniz. Daha küçük havuz boyutu daha az CPU ve disk kaynağı kullanır, ancak arka plan işlemleri daha yavaş ilerler ve bu da sorgu performansını etkileyebilir. + +Olası değerler: + +- Herhangi bir pozitif tamsayı. + +Varsayılan değer: 16. + [Orijinal makale](https://clickhouse.tech/docs/en/operations/settings/settings/) diff --git a/docs/tr/operations/system-tables.md b/docs/tr/operations/system-tables.md index 2bb0bcea68a..6a98f15aded 100644 --- a/docs/tr/operations/system-tables.md +++ b/docs/tr/operations/system-tables.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 52 toc_title: "Sistem Tablolar\u0131" --- @@ -180,7 +180,7 @@ Sütun: - `loading_start_time` ([DateTime](../sql-reference/data-types/datetime.md)) — Start time for loading the dictionary. - `last_successful_update_time` ([DateTime](../sql-reference/data-types/datetime.md)) — End time for loading or updating the dictionary. Helps to monitor some troubles with external sources and investigate causes. - `loading_duration` ([Float32](../sql-reference/data-types/float.md)) — Duration of a dictionary loading. -- `last_exception` ([Dize](../sql-reference/data-types/string.md)) — Text of the error that occurs when creating or reloading the dictionary if the dictionary couldn’t be created. +- `last_exception` ([Dize](../sql-reference/data-types/string.md)) — Text of the error that occurs when creating or reloading the dictionary if the dictionary couldn't be created. **Örnek** @@ -400,7 +400,7 @@ Testler için kullanılır. Bu tablo, tek bir satır içeren tek bir satır içerir ‘dummy’ 0 değerini içeren uint8 sütunu. SELECT sorgusu FROM yan tümcesi belirtmezse, bu tablo kullanılır. -Bu, diğer Dbms’lerde bulunan ikili tabloya benzer. +Bu, diğer Dbms'lerde bulunan ikili tabloya benzer. ## sistem.parçalar {#system_tables-parts} @@ -419,7 +419,7 @@ Sütun: - `name` (`String`) – Name of the data part. -- `active` (`UInt8`) – Flag that indicates whether the data part is active. If a data part is active, it’s used in a table. Otherwise, it’s deleted. Inactive data parts remain after merging. +- `active` (`UInt8`) – Flag that indicates whether the data part is active. If a data part is active, it's used in a table. Otherwise, it's deleted. Inactive data parts remain after merging. - `marks` (`UInt64`) – The number of marks. To get the approximate number of rows in a data part, multiply `marks` dizin ayrıntısına göre (genellikle 8192) (bu ipucu uyarlanabilir ayrıntı için çalışmaz). @@ -461,7 +461,7 @@ Sütun: - `primary_key_bytes_in_memory_allocated` (`UInt64`) – The amount of memory (in bytes) reserved for primary key values. -- `is_frozen` (`UInt8`) – Flag that shows that a partition data backup exists. 1, the backup exists. 0, the backup doesn’t exist. For more details, see [FREEZE PARTITION](../sql-reference/statements/alter.md#alter_freeze-partition) +- `is_frozen` (`UInt8`) – Flag that shows that a partition data backup exists. 1, the backup exists. 0, the backup doesn't exist. For more details, see [FREEZE PARTITION](../sql-reference/statements/alter.md#alter_freeze-partition) - `database` (`String`) – Name of the database. @@ -588,8 +588,8 @@ Sütun: - `query_duration_ms` (UInt64) — Duration of query execution. - `read_rows` (UInt64) — Number of read rows. - `read_bytes` (UInt64) — Number of read bytes. -- `written_rows` (UInt64) — For `INSERT` sorgular, yazılı satır sayısı. Diğer sorgular için sütun değeri 0’dır. -- `written_bytes` (UInt64) — For `INSERT` sorgular, yazılı bayt sayısı. Diğer sorgular için sütun değeri 0’dır. +- `written_rows` (UInt64) — For `INSERT` sorgular, yazılı satır sayısı. Diğer sorgular için sütun değeri 0'dır. +- `written_bytes` (UInt64) — For `INSERT` sorgular, yazılı bayt sayısı. Diğer sorgular için sütun değeri 0'dır. - `result_rows` (UInt64) — Number of rows in the result. - `result_bytes` (UInt64) — Number of bytes in the result. - `memory_usage` (UInt64) — Memory consumption by the query. @@ -610,7 +610,7 @@ Sütun: - `interface` (UInt8) — Interface that the query was initiated from. Possible values: - 1 — TCP. - 2 — HTTP. -- `os_user` (String) — OS’s username who runs [clickhouse-müşteri](../interfaces/cli.md). +- `os_user` (String) — OS's username who runs [clickhouse-müşteri](../interfaces/cli.md). - `client_hostname` (String) — Hostname of the client machine where the [clickhouse-müşteri](../interfaces/cli.md) veya başka bir TCP istemcisi çalıştırılır. - `client_name` (String) — The [clickhouse-müşteri](../interfaces/cli.md) veya başka bir TCP istemci adı. - `client_revision` (UInt32) — Revision of the [clickhouse-müşteri](../interfaces/cli.md) veya başka bir TCP istemcisi. @@ -661,8 +661,8 @@ Sütun: - `query_duration_ms` (UInt64) — Duration of query execution. - `read_rows` (UInt64) — Number of read rows. - `read_bytes` (UInt64) — Number of read bytes. -- `written_rows` (UInt64) — For `INSERT` sorgular, yazılı satır sayısı. Diğer sorgular için sütun değeri 0’dır. -- `written_bytes` (UInt64) — For `INSERT` sorgular, yazılı bayt sayısı. Diğer sorgular için sütun değeri 0’dır. +- `written_rows` (UInt64) — For `INSERT` sorgular, yazılı satır sayısı. Diğer sorgular için sütun değeri 0'dır. +- `written_bytes` (UInt64) — For `INSERT` sorgular, yazılı bayt sayısı. Diğer sorgular için sütun değeri 0'dır. - `memory_usage` (Int64) — The difference between the amount of allocated and freed memory in context of this thread. - `peak_memory_usage` (Int64) — The maximum difference between the amount of allocated and freed memory in context of this thread. - `thread_name` (String) — Name of the thread. @@ -684,7 +684,7 @@ Sütun: - `interface` (UInt8) — Interface that the query was initiated from. Possible values: - 1 — TCP. - 2 — HTTP. -- `os_user` (String) — OS’s username who runs [clickhouse-müşteri](../interfaces/cli.md). +- `os_user` (String) — OS's username who runs [clickhouse-müşteri](../interfaces/cli.md). - `client_hostname` (String) — Hostname of the client machine where the [clickhouse-müşteri](../interfaces/cli.md) veya başka bir TCP istemcisi çalıştırılır. - `client_name` (String) — The [clickhouse-müşteri](../interfaces/cli.md) veya başka bir TCP istemci adı. - `client_revision` (UInt32) — Revision of the [clickhouse-müşteri](../interfaces/cli.md) veya başka bir TCP istemcisi. @@ -720,24 +720,26 @@ Günlükleri analiz etmek için `addressToLine`, `addressToSymbol` ve `demangle` Sütun: -- `event_date`([Tarihli](../sql-reference/data-types/date.md)) — Date of sampling moment. +- `event_date` ([Tarihli](../sql-reference/data-types/date.md)) — Date of sampling moment. -- `event_time`([DateTime](../sql-reference/data-types/datetime.md)) — Timestamp of sampling moment. +- `event_time` ([DateTime](../sql-reference/data-types/datetime.md)) — Timestamp of the sampling moment. -- `revision`([Uİnt32](../sql-reference/data-types/int-uint.md)) — ClickHouse server build revision. +- `timestamp_ns` ([Uİnt64](../sql-reference/data-types/int-uint.md)) — Timestamp of the sampling moment in nanoseconds. + +- `revision` ([Uİnt32](../sql-reference/data-types/int-uint.md)) — ClickHouse server build revision. Tarafından sunucuya Bağlan byırken `clickhouse-client`, benzer diz theg seeeyi görüyorsunuz `Connected to ClickHouse server version 19.18.1 revision 54429.`. Bu alan şunları içerir `revision` ama `version` bir sunucunun. -- `timer_type`([Enum8](../sql-reference/data-types/enum.md)) — Timer type: +- `timer_type` ([Enum8](../sql-reference/data-types/enum.md)) — Timer type: - `Real` duvar saati zamanını temsil eder. - `CPU` CPU süresini temsil eder. -- `thread_number`([Uİnt32](../sql-reference/data-types/int-uint.md)) — Thread identifier. +- `thread_number` ([Uİnt32](../sql-reference/data-types/int-uint.md)) — Thread identifier. -- `query_id`([Dize](../sql-reference/data-types/string.md)) — Query identifier that can be used to get details about a query that was running from the [query\_log](#system_tables-query_log) sistem tablosu. +- `query_id` ([Dize](../sql-reference/data-types/string.md)) — Query identifier that can be used to get details about a query that was running from the [query\_log](#system_tables-query_log) sistem tablosu. -- `trace`([Dizi (Uİnt64)](../sql-reference/data-types/array.md)) — Stack trace at the moment of sampling. Each element is a virtual memory address inside ClickHouse server process. +- `trace` ([Dizi (Uİnt64)](../sql-reference/data-types/array.md)) — Stack trace at the moment of sampling. Each element is a virtual memory address inside ClickHouse server process. **Örnek** @@ -816,7 +818,7 @@ Sütun: Yazma kullanılabilir ve bir oturum ZK, bir lider olup olmadığına bakılmaksızın olan herhangi bir yineleme için gerçekleştirilebilir unutmayın. - `can_become_leader` (`UInt8`)- Rep .lik leaderanın lider olarak seçil .ip seçil .emeyeceği. - `is_readonly` (`UInt8`)- Yinelemenin salt okunur modda olup olmadığı. - Yapılandırmanın ZooKeeper ile bölümleri yoksa, zookeeper’daki oturumları yeniden başlatırken ve Zookeeper’daki oturum yeniden başlatılırken bilinmeyen bir hata oluşmuşsa bu mod açılır. + Yapılandırmanın ZooKeeper ile bölümleri yoksa, zookeeper'daki oturumları yeniden başlatırken ve Zookeeper'daki oturum yeniden başlatılırken bilinmeyen bir hata oluşmuşsa bu mod açılır. - `is_session_expired` (`UInt8`)- ZooKeeper ile oturum süresi doldu. Temelde aynı `is_readonly`. - `future_parts` (`UInt32`)- Henüz yapılmamış ekler veya birleştirmelerin sonucu olarak görünecek veri parçalarının sayısı. - `parts_to_check` (`UInt32`)- Doğrulama için kuyruktaki veri parçalarının sayısı. Hasar görebileceğinden şüphe varsa, bir parça doğrulama kuyruğuna konur. @@ -893,7 +895,7 @@ Sütun: - `max` ([Nullable](../sql-reference/data-types/nullable.md)([Dize](../sql-reference/data-types/string.md))) — Maximum value of the setting, if any is set via [kısıtlamalar](settings/constraints-on-settings.md#constraints-on-settings). Ayarın maksimum değeri yoksa, şunları içerir [NULL](../sql-reference/syntax.md#null-literal). - `readonly` ([Uİnt8](../sql-reference/data-types/int-uint.md#uint-ranges)) — Shows whether the current user can change the setting: - `0` — Current user can change the setting. - - `1` — Current user can’t change the setting. + - `1` — Current user can't change the setting. **Örnek** @@ -926,7 +928,7 @@ SELECT * FROM system.settings WHERE changed AND name='load_balancing' **Ayrıca bakınız** -- [Ayarlar](settings/index.md#settings) +- [Ayarlar](settings/index.md#session-settings-intro) - [Sorgular için izinler](settings/permissions-for-queries.md#settings_readonly) - [Ayarlardaki kısıtlamalar](settings/constraints-on-settings.md) diff --git a/docs/tr/operations/tips.md b/docs/tr/operations/tips.md index 34d1efd44e5..50e78dc772f 100644 --- a/docs/tr/operations/tips.md +++ b/docs/tr/operations/tips.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 58 toc_title: "Kullan\u0131m \xD6nerileri" --- @@ -17,14 +17,14 @@ $ echo 'performance' | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_gov ## CPU sınırlamaları {#cpu-limitations} -İşlemciler aşırı ısınabilir. Kullanmak `dmesg` aşırı ısınma nedeniyle CPU’nun saat hızının sınırlı olup olmadığını görmek için. +İşlemciler aşırı ısınabilir. Kullanmak `dmesg` aşırı ısınma nedeniyle CPU'nun saat hızının sınırlı olup olmadığını görmek için. Kısıtlama, veri merkezi düzeyinde harici olarak da ayarlanabilir. Kullanabilirsiniz `turbostat` bir yük altında izlemek için. ## RAM {#ram} -Küçük miktarlarda veri için (~200 GB’a kadar sıkıştırılmış), veri hacmi kadar bellek kullanmak en iyisidir. +Küçük miktarlarda veri için (~200 GB'a kadar sıkıştırılmış), veri hacmi kadar bellek kullanmak en iyisidir. Büyük miktarda veri için ve etkileşimli (çevrimiçi) sorguları işlerken, sıcak veri alt kümesi sayfaların önbelleğine sığacak şekilde makul miktarda RAM (128 GB veya daha fazla) kullanmalısınız. -Sunucu başına ~50 TB veri hacimleri için bile, 128 GB RAM kullanmak, 64 GB’ye kıyasla sorgu performansını önemli ölçüde artırır. +Sunucu başına ~50 TB veri hacimleri için bile, 128 GB RAM kullanmak, 64 GB'ye kıyasla sorgu performansını önemli ölçüde artırır. Overcommit devre dışı bırakmayın. Değer `cat /proc/sys/vm/overcommit_memory` 0 veya 1 olmalıdır. Koşmak @@ -46,19 +46,19 @@ Kalıcı büyük sayfaların da tahsis edilmesine gerek yoktur. ## Depolama Alt Sistemi {#storage-subsystem} Bütçeniz SSD kullanmanıza izin veriyorsa, SSD kullanın. -Eğer değilse, sabit disk kullanın. SATA HDD’ler 7200 RPM yapacak. +Eğer değilse, sabit disk kullanın. SATA HDD'ler 7200 RPM yapacak. Bağlı disk raflarına sahip daha az sayıda sunucu üzerinde yerel sabit disklere sahip birçok sunucuyu tercih edin. Ancak nadir sorguları olan arşivleri saklamak için raflar çalışacaktır. ## RAID {#raid} -HDD kullanırken, RAID-10, RAID-5, RAID-6 veya RAID-50’yi birleştirebilirsiniz. -Linux için, yazılım RAID daha iyidir (ile `mdadm`). LVM’Yİ kullanmanızı önermiyoruz. +HDD kullanırken, RAID-10, RAID-5, RAID-6 veya RAID-50'yi birleştirebilirsiniz. +Linux için, yazılım RAID daha iyidir (ile `mdadm`). LVM'Yİ kullanmanızı önermiyoruz. RAID-10 oluştururken, `far` düzen. -Bütçeniz izin veriyorsa, RAID-10’u seçin. +Bütçeniz izin veriyorsa, RAID-10'u seçin. -4’ten fazla diskiniz varsa, RAID-5 yerine RAID-6 (tercih edilen) veya RAID-50 kullanın. +4'ten fazla diskiniz varsa, RAID-5 yerine RAID-6 (tercih edilen) veya RAID-50 kullanın. RAID-5, RAID-6 veya RAID-50 kullanırken, varsayılan değer genellikle en iyi seçenek olmadığından daima stripe\_cache\_size değerini artırın. ``` bash @@ -70,10 +70,10 @@ Formülü kullanarak cihaz sayısından ve blok boyutundan tam sayıyı hesaplay Tüm RAID yapılandırmaları için 1024 KB blok boyutu yeterlidir. Blok boyutunu asla çok küçük veya çok büyük ayarlamayın. -SSD’DE RAID-0 kullanabilirsiniz. +SSD'DE RAID-0 kullanabilirsiniz. RAID kullanımı ne olursa olsun, her zaman veri güvenliği için çoğaltma kullanın. -Uzun bir kuyruk ile NCQ etkinleştirin. HDD için CFQ zamanlayıcısını seçin ve SSD için noop’u seçin. Azalt themayın ‘readahead’ ayar. +Uzun bir kuyruk ile NCQ etkinleştirin. HDD için CFQ zamanlayıcısını seçin ve SSD için noop'u seçin. Azalt themayın ‘readahead’ ayar. HDD için yazma önbelleğini etkinleştirin. ## Dosya Sistemi {#file-system} @@ -97,7 +97,7 @@ Mümkünse en az 10 GB ağ kullanın. 1 Gb de çalışacak, ancak onlarca teraba Muhtemelen zaten başka amaçlar için ZooKeeper kullanıyor. Zaten aşırı değilse, aynı ZooKeeper kurulumunu kullanabilirsiniz. -It’s best to use a fresh version of ZooKeeper – 3.4.9 or later. The version in stable Linux distributions may be outdated. +It's best to use a fresh version of ZooKeeper – 3.4.9 or later. The version in stable Linux distributions may be outdated. Sonuç sıralı düğümler için yanlış olacağından, farklı ZooKeeper kümeleri arasında veri aktarmak için el ile yazılmış komut dosyalarını asla kullanmamalısınız. Asla kullanmayın “zkcopy” aynı nedenle yardımcı program: https://github.com/ksprojects/zkcopy/issues/15 @@ -111,7 +111,7 @@ Varsayılan ayarlarla, ZooKeeper bir saatli bomba: Bu bomba etkisiz hale getirilmeli. -Aşağıdaki ZooKeeper (3.5.1) yapılandırması Yandex’te kullanılmaktadır.20 Mayıs 2017 tarihi itibariyle Metrica üretim ortamı: +Aşağıdaki ZooKeeper (3.5.1) yapılandırması Yandex'te kullanılmaktadır.20 Mayıs 2017 tarihi itibariyle Metrica üretim ortamı: hayvanat bahçesi.cfg: diff --git a/docs/tr/operations/troubleshooting.md b/docs/tr/operations/troubleshooting.md index 0c998e727ef..fb547342b2f 100644 --- a/docs/tr/operations/troubleshooting.md +++ b/docs/tr/operations/troubleshooting.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 46 toc_title: "Ar\u0131za" --- @@ -14,7 +14,7 @@ toc_title: "Ar\u0131za" ## Kurulum {#troubleshooting-installation-errors} -### Apt-get Ile ClickHouse Deposundan Deb Paketleri alınamıyor {#you-cannot-get-deb-packages-from-clickhouse-repository-with-apt-get} +### Apt-get ile ClickHouse deposundan Deb paketleri alınamıyor {#you-cannot-get-deb-packages-from-clickhouse-repository-with-apt-get} - Güvenlik Duvarı ayarlarını kontrol edin. - Depoya herhangi bir nedenle erişemiyorsanız, paketleri aşağıda açıklandığı gibi indirin [Başlarken](../getting-started/index.md) makale ve bunları kullanarak manuel olarak yükleyin `sudo dpkg -i ` komut. Ayrıca ihtiyacınız olacak `tzdata` paket. @@ -87,7 +87,7 @@ Eğer herhangi bir yararlı bilgi bulamazsanız `clickhouse-server` günlükler $ sudo journalctl -u clickhouse-server ``` -**Clickhouse-Server’ı etkileşimli modda Başlat** +**Clickhouse-Server'ı etkileşimli modda Başlat** ``` bash $ sudo -u clickhouse /usr/bin/clickhouse-server --config-file /etc/clickhouse-server/config.xml @@ -101,7 +101,7 @@ Kontrol: - Docker ayarları. - Bir IPv6 ağında Docker’da ClickHouse çalıştırırsanız, `network=host` ayar .lanmıştır. + Bir IPv6 ağında Docker'da ClickHouse çalıştırırsanız, `network=host` ayar .lanmıştır. - Bitiş noktası ayarları. @@ -135,12 +135,12 @@ $ curl 'http://localhost:8123/' --data-binary "SELECT a" Code: 47, e.displayText() = DB::Exception: Unknown identifier: a. Note that there are no tables (FROM clause) in your query, context: required_names: 'a' source_tables: table_aliases: private_aliases: column_aliases: public_columns: 'a' masked_columns: array_join_columns: source_columns: , e.what() = DB::Exception ``` -Eğer başlarsanız `clickhouse-client` ile… `stack-trace` parametre, ClickHouse bir hata açıklaması ile sunucu yığın izleme döndürür. +Eğer başlarsanız `clickhouse-client` ile... `stack-trace` parametre, ClickHouse bir hata açıklaması ile sunucu yığın izleme döndürür. Bozuk bir bağlantı hakkında bir mesaj görebilirsiniz. Bu durumda, sorguyu tekrarlayabilirsiniz. Sorguyu her gerçekleştirdiğinizde bağlantı kesilirse, sunucu günlüklerini hatalar için denetleyin. ## Sorgu işleme verimliliği {#troubleshooting-too-slow} -Clickhouse’un çok yavaş çalıştığını görürseniz, sorgularınız için sunucu kaynakları ve ağdaki yükü profillemeniz gerekir. +Clickhouse'un çok yavaş çalıştığını görürseniz, sorgularınız için sunucu kaynakları ve ağdaki yükü profillemeniz gerekir. Profil sorguları için clickhouse-benchmark yardımcı programını kullanabilirsiniz. Saniyede işlenen sorgu sayısını, saniyede işlenen satır sayısını ve sorgu işleme sürelerinin yüzdelerini gösterir. diff --git a/docs/tr/operations/update.md b/docs/tr/operations/update.md index 1529729321e..08168cf4f64 100644 --- a/docs/tr/operations/update.md +++ b/docs/tr/operations/update.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 47 toc_title: "ClickHouse G\xFCncelleme" --- diff --git a/docs/tr/operations/utilities/clickhouse-benchmark.md b/docs/tr/operations/utilities/clickhouse-benchmark.md index 68fcc437aa2..52c4d5c23ed 100644 --- a/docs/tr/operations/utilities/clickhouse-benchmark.md +++ b/docs/tr/operations/utilities/clickhouse-benchmark.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 61 toc_title: clickhouse-benchmark --- @@ -91,11 +91,11 @@ Raporda bulabilirsiniz: - ClickHouse sunucusunun bitiş noktası. - İşlenen sorgu sayısı. - - QPS: qps: kaç sorgu sunucusu saniyede belirtilen bir süre boyunca gerçekleştirilen `--delay` değişken. - - RPS: kaç satır sunucu saniyede belirtilen bir süre boyunca okuma `--delay` değişken. - - MıB / s: kaç mebibytes sunucu saniyede belirtilen bir süre boyunca okuma `--delay` değişken. - - sonuç RPS: sunucu tarafından belirtilen bir süre boyunca saniyede bir sorgunun sonucuna kaç satır yerleştirilir `--delay` değişken. - - sonuç MıB / s. kaç mebibytes sunucu tarafından belirtilen bir dönemde saniyede bir sorgu sonucu yerleştirilir `--delay` değişken. + - QPS: qps: kaç sorgu sunucusu saniyede belirtilen bir süre boyunca gerçekleştirilen `--delay` tartışma. + - RPS: kaç satır sunucu saniyede belirtilen bir süre boyunca okuma `--delay` tartışma. + - MıB / s: kaç mebibytes sunucu saniyede belirtilen bir süre boyunca okuma `--delay` tartışma. + - sonuç RPS: sunucu tarafından belirtilen bir süre boyunca saniyede bir sorgunun sonucuna kaç satır yerleştirilir `--delay` tartışma. + - sonuç MıB / s. kaç mebibytes sunucu tarafından belirtilen bir dönemde saniyede bir sorgu sonucu yerleştirilir `--delay` tartışma. - Sorgu yürütme süresi yüzdelik. diff --git a/docs/tr/operations/utilities/clickhouse-copier.md b/docs/tr/operations/utilities/clickhouse-copier.md index 932519086c1..04b93c41fd6 100644 --- a/docs/tr/operations/utilities/clickhouse-copier.md +++ b/docs/tr/operations/utilities/clickhouse-copier.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 59 toc_title: clickhouse-fotokopi makinesi --- diff --git a/docs/tr/operations/utilities/clickhouse-local.md b/docs/tr/operations/utilities/clickhouse-local.md index 32d70455388..d39a642b830 100644 --- a/docs/tr/operations/utilities/clickhouse-local.md +++ b/docs/tr/operations/utilities/clickhouse-local.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 60 toc_title: clickhouse-yerel --- @@ -13,7 +13,7 @@ Tabloları temsil eden verileri kabul eder ve bunları kullanarak sorgular [Clic `clickhouse-local` ClickHouse server ile aynı çekirdeği kullanır, bu nedenle özelliklerin çoğunu ve aynı format ve tablo motorlarını destekler. -Varsayılan olarak `clickhouse-local` aynı ana bilgisayarda verilere erişimi yok, ancak kullanarak yükleme sunucu yapılandırmasını destekler `--config-file` değişken. +Varsayılan olarak `clickhouse-local` aynı ana bilgisayarda verilere erişimi yok, ancak kullanarak yükleme sunucu yapılandırmasını destekler `--config-file` tartışma. !!! warning "Uyarıcı" İçine üretim sunucusu yapılandırmasını yüklemek için tavsiye edilmez `clickhouse-local` çünkü insan hatası durumunda veriler zarar görebilir. diff --git a/docs/tr/operations/utilities/index.md b/docs/tr/operations/utilities/index.md index 17d3849cc6a..cdd99610911 100644 --- a/docs/tr/operations/utilities/index.md +++ b/docs/tr/operations/utilities/index.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_folder_title: Programlar toc_priority: 56 toc_title: "Genel bak\u0131\u015F" diff --git a/docs/tr/sql-reference/aggregate-functions/combinators.md b/docs/tr/sql-reference/aggregate-functions/combinators.md index c8db7bcb127..a085ef6595e 100644 --- a/docs/tr/sql-reference/aggregate-functions/combinators.md +++ b/docs/tr/sql-reference/aggregate-functions/combinators.md @@ -1,8 +1,8 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 37 -toc_title: "Toplama fonksiyonu birle\u015Ftiriciler" +toc_title: "Birle\u015Ftiriciler" --- # Toplama Fonksiyonu Birleştiriciler {#aggregate_functions_combinators} @@ -34,8 +34,8 @@ Bu birleştiriciyi uygularsanız, toplama işlevi elde edilen değeri döndürme Bu durumlarla çalışmak için şunları kullanın: - [AggregatingMergeTree](../../engines/table-engines/mergetree-family/aggregatingmergetree.md) masa motoru. -- [finalizeAggregation](../../sql-reference/functions/other-functions.md#function-finalizeaggregation) işlev. -- [runningAccumulate](../../sql-reference/functions/other-functions.md#function-runningaccumulate) işlev. +- [finalizeAggregation](../../sql-reference/functions/other-functions.md#function-finalizeaggregation) İşlev. +- [runningAccumulate](../../sql-reference/functions/other-functions.md#function-runningaccumulate) İşlev. - [-Birleştirmek](#aggregate_functions_combinators-merge) birleştirici. - [- MergeState](#aggregate_functions_combinators-mergestate) birleştirici. @@ -45,7 +45,7 @@ Bu birleştiriciyi uygularsanız, toplama işlevi Ara toplama durumunu bağıms ## - MergeState {#aggregate_functions_combinators-mergestate} -Ara toplama durumlarını-birleştirme Birleştiricisi ile aynı şekilde birleştirir. Bununla birlikte, elde edilen değeri döndürmez, ancak-State combinator’a benzer bir ara toplama durumu döndürür. +Ara toplama durumlarını-birleştirme Birleştiricisi ile aynı şekilde birleştirir. Bununla birlikte, elde edilen değeri döndürmez, ancak-State combinator'a benzer bir ara toplama durumu döndürür. ## - ForEach {#agg-functions-combinator-foreach} @@ -53,33 +53,110 @@ Tablolar için bir toplama işlevi, karşılık gelen dizi öğelerini toplayan ## - OrDefault {#agg-functions-combinator-ordefault} -Toplamak için hiçbir şey yoksa, toplama işlevinin dönüş türünün Varsayılan değerini doldurur. +Toplama işlevinin davranışını değiştirir. + +Bir toplama işlevinin giriş değerleri yoksa, bu birleştirici ile dönüş veri türü için varsayılan değeri döndürür. Boş giriş verilerini alabilen toplama işlevlerine uygulanır. + +`-OrDefault` diğer birleştiriciler ile kullanılabilir. + +**Sözdizimi** + +``` sql +OrDefault(x) +``` + +**Parametre** + +- `x` — Aggregate function parameters. + +**Döndürülen değerler** + +Toplamak için hiçbir şey yoksa, bir toplama işlevinin dönüş türünün Varsayılan değerini döndürür. + +Türü kullanılan toplama işlevine bağlıdır. + +**Örnek** + +Sorgu: ``` sql SELECT avg(number), avgOrDefault(number) FROM numbers(0) ``` +Sonuç: + ``` text ┌─avg(number)─┬─avgOrDefault(number)─┐ │ nan │ 0 │ └─────────────┴──────────────────────┘ ``` -## - OrNull {#agg-functions-combinator-ornull} +Ayrıca `-OrDefault` başka bir birleştiriciler ile kullanılabilir. Toplama işlevi boş girişi kabul etmediğinde yararlıdır. -Doldurmalar `null` toplamak için hiçbir şey varsa. Dönüş sütun null olur. +Sorgu: ``` sql -SELECT avg(number), avgOrNull(number) FROM numbers(0) +SELECT avgOrDefaultIf(x, x > 10) +FROM +( + SELECT toDecimal32(1.23, 2) AS x +) ``` +Sonuç: + ``` text -┌─avg(number)─┬─avgOrNull(number)─┐ -│ nan │ ᴺᵁᴸᴸ │ -└─────────────┴───────────────────┘ +┌─avgOrDefaultIf(x, greater(x, 10))─┐ +│ 0.00 │ +└───────────────────────────────────┘ ``` -\- OrDefault ve-OrNull diğer birleştiriciler ile kombine edilebilir. Toplama işlevi boş girişi kabul etmediğinde yararlıdır. +## - OrNull {#agg-functions-combinator-ornull} + +Toplama işlevinin davranışını değiştirir. + +Bu birleştirici, bir toplama işlevinin sonucunu [Nullable](../data-types/nullable.md) veri türü. Toplama işlevi hesaplamak için değerleri yoksa döndürür [NULL](../syntax.md#null-literal). + +`-OrNull` diğer birleştiriciler ile kullanılabilir. + +**Sözdizimi** + +``` sql +OrNull(x) +``` + +**Parametre** + +- `x` — Aggregate function parameters. + +**Döndürülen değerler** + +- Toplama işlev resultinin sonucu, `Nullable` veri türü. +- `NULL`, toplamak için bir şey yoksa. + +Tür: `Nullable(aggregate function return type)`. + +**Örnek** + +Eklemek `-orNull` toplama işlevinin sonuna kadar. + +Sorgu: + +``` sql +SELECT sumOrNull(number), toTypeName(sumOrNull(number)) FROM numbers(10) WHERE number > 10 +``` + +Sonuç: + +``` text +┌─sumOrNull(number)─┬─toTypeName(sumOrNull(number))─┐ +│ ᴺᵁᴸᴸ │ Nullable(UInt64) │ +└───────────────────┴───────────────────────────────┘ +``` + +Ayrıca `-OrNull` başka bir birleştiriciler ile kullanılabilir. Toplama işlevi boş girişi kabul etmediğinde yararlıdır. + +Sorgu: ``` sql SELECT avgOrNullIf(x, x > 10) @@ -89,6 +166,8 @@ FROM ) ``` +Sonuç: + ``` text ┌─avgOrNullIf(x, greater(x, 10))─┐ │ ᴺᵁᴸᴸ │ @@ -132,7 +211,7 @@ Düşünün `people` aşağıdaki verilerle tablo: Yaş aralığı içinde olan kişilerin isimlerini alalım `[30,60)` ve `[60,75)`. Yaş için tamsayı temsilini kullandığımızdan, yaşları `[30, 59]` ve `[60,74]` aralıklılar. -Bir dizideki isimleri toplamak için, [groupArray](reference.md#agg_function-grouparray) toplama işlevi. Bir argüman alır. Bizim durumumuzda, bu `name` sütun. Bu `groupArrayResample` fonksiyon kullanmalıdır `age` yaşlara göre isimleri toplamak için sütun. Gerekli aralıkları tanımlamak için `30, 75, 30` argü themanlar içine `groupArrayResample` işlev. +Bir dizideki isimleri toplamak için, [groupArray](reference.md#agg_function-grouparray) toplama işlevi. Bir argüman alır. Bizim durumumuzda, bu `name` sütun. Bu `groupArrayResample` fonksiyon kullanmalıdır `age` yaşlara göre isimleri toplamak için sütun. Gerekli aralıkları tanımlamak için `30, 75, 30` argü themanlar içine `groupArrayResample` İşlev. ``` sql SELECT groupArrayResample(30, 75, 30)(name, age) FROM people diff --git a/docs/tr/sql-reference/aggregate-functions/index.md b/docs/tr/sql-reference/aggregate-functions/index.md index 312868ff8f3..22853f32f17 100644 --- a/docs/tr/sql-reference/aggregate-functions/index.md +++ b/docs/tr/sql-reference/aggregate-functions/index.md @@ -1,12 +1,12 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_folder_title: "Toplama Fonksiyonlar\u0131" toc_priority: 33 toc_title: "Giri\u015F" --- -# Toplama fonksiyonları {#aggregate-functions} +# Toplama Fonksiyonları {#aggregate-functions} Toplama fonksiyonları [normal](http://www.sql-tutorial.com/sql-aggregate-functions-sql-tutorial) veritabanı uzmanları tarafından beklendiği gibi. diff --git a/docs/tr/sql-reference/aggregate-functions/parametric-functions.md b/docs/tr/sql-reference/aggregate-functions/parametric-functions.md index f356c013519..27c359c807c 100644 --- a/docs/tr/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/tr/sql-reference/aggregate-functions/parametric-functions.md @@ -1,8 +1,8 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 38 -toc_title: "Parametrik agrega fonksiyonlar\u0131" +toc_title: Parametrik --- # Parametrik Agrega Fonksiyonları {#aggregate_functions_parametric} @@ -111,7 +111,7 @@ Tür: `UInt8`. - `(?N)` — Matches the condition argument at position `N`. Şartlar numaralandırılmıştır `[1, 32]` Aralık. Mesela, `(?1)` argü theman thela eşleş their `cond1` parametre. -- `.*` — Matches any number of events. You don’t need conditional arguments to match this element of the pattern. +- `.*` — Matches any number of events. You don't need conditional arguments to match this element of the pattern. - `(?t operator value)` — Sets the time in seconds that should separate two events. For example, pattern `(?1)(?t>1800)(?2)` birbirinden 1800 saniyeden fazla meydana gelen olayları eşleşir. Bu olaylar arasında herhangi bir olayın keyfi bir sayısı olabilir. Kullanabilirsiniz `>=`, `>`, `<`, `<=` operatörler. @@ -229,7 +229,7 @@ Kayan bir zaman penceresinde olay zincirlerini arar ve zincirden meydana gelen e Fonksiyon algoritmaya göre çalışır: -- İşlev, zincirdeki ilk koşulu tetikleyen ve olay sayacını 1’e ayarlayan verileri arar. Sürgülü pencerenin başladığı an budur. +- İşlev, zincirdeki ilk koşulu tetikleyen ve olay sayacını 1'e ayarlayan verileri arar. Sürgülü pencerenin başladığı an budur. - Zincirdeki olaylar pencerede sırayla gerçekleşirse, sayaç artırılır. Olayların sırası bozulursa, sayaç artırılmaz. @@ -246,7 +246,7 @@ windowFunnel(window, [mode])(timestamp, cond1, cond2, ..., condN) - `window` — Length of the sliding window in seconds. - `mode` - Bu isteğe bağlı bir argüman. - `'strict'` - Zaman `'strict'` ayarlanırsa, windowFunnel () yalnızca benzersiz değerler için koşullar uygular. -- `timestamp` — Name of the column containing the timestamp. Data types supported: [Tarihli](../../sql-reference/data-types/date.md), [DateTime](../../sql-reference/data-types/datetime.md#data_type-datetime) ve diğer imzasız tamsayı türleri (timestamp’ın `UInt64` yazın, değeri 2^63 - 1 olan Int64 maksimum değerini aşamaz). +- `timestamp` — Name of the column containing the timestamp. Data types supported: [Tarihli](../../sql-reference/data-types/date.md), [DateTime](../../sql-reference/data-types/datetime.md#data_type-datetime) ve diğer imzasız tamsayı türleri (timestamp'ın `UInt64` yazın, değeri 2^63 - 1 olan Int64 maksimum değerini aşamaz). - `cond` — Conditions or data describing the chain of events. [Uİnt8](../../sql-reference/data-types/int-uint.md). **Döndürülen değer** @@ -315,8 +315,8 @@ Sonuç: ## saklama {#retention} -İşlev, bağımsız değişken olarak 1’den 32’ye kadar bir dizi koşul türünü alır `UInt8` bu, etkinlik için belirli bir koşulun karşılanıp karşılanmadığını gösterir. -Herhangi bir koşul bir argüman olarak belirtilebilir (aşağıdaki gibi [WHERE](../../sql-reference/statements/select.md#select-where)). +İşlev, bağımsız değişken olarak 1'den 32'ye kadar bir dizi koşul türünü alır `UInt8` bu, etkinlik için belirli bir koşulun karşılanıp karşılanmadığını gösterir. +Herhangi bir koşul bir argüman olarak belirtilebilir (aşağıdaki gibi [WHERE](../../sql-reference/statements/select/where.md#select-where)). İlk hariç, koşullar çiftler halinde geçerlidir: birinci ve ikinci doğruysa, ikincinin sonucu, birinci ve fird doğruysa, üçüncüsü doğru olacaktır. @@ -335,7 +335,7 @@ retention(cond1, cond2, ..., cond32); 1 veya 0 dizisi. - 1 — condition was met for the event. -- 0 — condition wasn’t met for the event. +- 0 — condition wasn't met for the event. Tür: `UInt8`. @@ -402,7 +402,7 @@ Sonuç: └────────────┴─────┘ ``` -**2.** Kullanıcıları benzersiz kimliğe göre grupla `uid` kullanarak `retention` işlev. +**2.** Kullanıcıları benzersiz kimliğe göre grupla `uid` kullanarak `retention` İşlev. Sorgu: @@ -476,14 +476,14 @@ Nerede: Calculates the number of different argument values ​​if it is less than or equal to N. If the number of different argument values is greater than N, it returns N + 1. -Küçük Ns ile kullanım için tavsiye, kadar 10. N’nin maksimum değeri 100’dür. +Küçük Ns ile kullanım için tavsiye, kadar 10. N'nin maksimum değeri 100'dür. Bir toplama işlevinin durumu için, 1 + n \* bir bayt değerinin boyutuna eşit bellek miktarını kullanır. Dizeler için, 8 baytlık kriptografik olmayan bir karma saklar. Yani, hesaplama dizeler için yaklaşık olarak hesaplanır. İşlev ayrıca birkaç argüman için de çalışır. -Büyük bir N değeri kullanıldığında ve benzersiz değerlerin sayısı n’den biraz daha az olduğu durumlar dışında mümkün olduğunca hızlı çalışır. +Büyük bir N değeri kullanıldığında ve benzersiz değerlerin sayısı n'den biraz daha az olduğu durumlar dışında mümkün olduğunca hızlı çalışır. Kullanım örneği: diff --git a/docs/tr/sql-reference/aggregate-functions/reference.md b/docs/tr/sql-reference/aggregate-functions/reference.md index 7b8e83224e6..9865fa914ed 100644 --- a/docs/tr/sql-reference/aggregate-functions/reference.md +++ b/docs/tr/sql-reference/aggregate-functions/reference.md @@ -1,11 +1,11 @@ --- machine_translated: true -machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 36 toc_title: "Ba\u015Fvurma" --- -# Fonksiyon Referans {#function-reference} +# Toplama Fonksiyonu Referansı {#aggregate-functions-reference} ## sayma {#agg_function-count} @@ -25,13 +25,13 @@ Fonksiyon alabilir: **Döndürülen değer** - Fonksiyon parametreleri olmadan çağrılırsa, satır sayısını sayar. -- Eğer… [ifade](../syntax.md#syntax-expressions) geçirilir, daha sonra işlev bu ifadenin kaç kez NOT null döndürdüğünü sayar. İfad aede bir [Nullable](../../sql-reference/data-types/nullable.md)- type değeri, sonra sonucu `count` kalır değil `Nullable`. İfade döndürülürse işlev 0 döndürür `NULL` tüm satırlar için. +- Eğer... [ifade](../syntax.md#syntax-expressions) geçirilir, daha sonra işlev bu ifadenin kaç kez NOT null döndürdüğünü sayar. İfad aede bir [Nullable](../../sql-reference/data-types/nullable.md)- type değeri, sonra sonucu `count` kalır değil `Nullable`. İfade döndürülürse işlev 0 döndürür `NULL` tüm satırlar için. Her iki durumda da döndürülen değerin türü [Uİnt64](../../sql-reference/data-types/int-uint.md). **Ayrıntı** -ClickHouse destekler `COUNT(DISTINCT ...)` sözdizimi. Bu yapının davranışı Aşağıdakilere bağlıdır [count\_distinct\_implementation](../../operations/settings/settings.md#settings-count_distinct_implementation) ayar. Aşağıdakilerden hang theisini tanımlar [uniq\*](#agg_function-uniq) fonksiyonlar işlemi gerçekleştirmek için kullanılır. Varsayılan değer [uniqExact](#agg_function-uniqexact) işlev. +ClickHouse destekler `COUNT(DISTINCT ...)` sözdizimi. Bu yapının davranışı Aşağıdakilere bağlıdır [count\_distinct\_implementation](../../operations/settings/settings.md#settings-count_distinct_implementation) ayar. Aşağıdakilerden hang theisini tanımlar [uniq\*](#agg_function-uniq) fonksiyonlar işlemi gerçekleştirmek için kullanılır. Varsayılan değer [uniqExact](#agg_function-uniqexact) İşlev. Bu `SELECT count() FROM table` tablodaki girdi sayısı ayrı olarak depolanmadığı için sorgu en iyi duruma getirilmez. Tablodan küçük bir sütun seçer ve içindeki değerlerin sayısını sayar. @@ -81,7 +81,7 @@ Belirli bir sonuç elde etmek için ‘min’ veya ‘max’ fonksiyon yerine Bazı durumlarda, yürütme sırasına güvenebilirsiniz. Bu, select ORDER BY kullanan bir alt sorgudan geldiğinde durumlar için geçerlidir. -Ne zaman bir `SELECT` sorgu vardır `GROUP BY` yan tümce veya en az bir toplama işlevi, ClickHouse (Mysql’in aksine), tüm ifadelerin `SELECT`, `HAVING`, ve `ORDER BY` anahtar functionslardan veya toplama işlev .lerinden hesaplan .malıdır. Başka bir deyişle, tablodan seçilen her sütun, anahtarlarda veya toplama işlevlerinde kullanılmalıdır. Mysql’de olduğu gibi davranış elde etmek için, diğer sütunları `any` toplama işlevi. +Ne zaman bir `SELECT` sorgu vardır `GROUP BY` yan tümce veya en az bir toplama işlevi, ClickHouse (Mysql'in aksine), tüm ifadelerin `SELECT`, `HAVING`, ve `ORDER BY` anahtar functionslardan veya toplama işlev .lerinden hesaplan .malıdır. Başka bir deyişle, tablodan seçilen her sütun, anahtarlarda veya toplama işlevlerinde kullanılmalıdır. Mysql'de olduğu gibi davranış elde etmek için, diğer sütunları `any` toplama işlevi. ## anyHeavy (x) {#anyheavyx} @@ -113,7 +113,7 @@ FROM ontime ## anyLast(x) {#anylastx} Karşılaşılan son değeri seçer. -Sonuç için olduğu kadar belirsiz `any` işlev. +Sonuç için olduğu kadar belirsiz `any` İşlev. ## groupBitAnd {#groupbitand} @@ -332,9 +332,10 @@ Giriş parametreleri için olduğu gibi sonuç için aynı veri türünü kullan Sadece sayılar için çalışır. -## sumMap (anahtar, değer) {#agg_functions-summap} +## sumMap (anahtar, değer), sumMap(Tuple (anahtar, değer)) {#agg_functions-summap} Toplam thelar ‘value’ belirtilen tuş accordinglara göre dizi ‘key’ dizi. +Anahtarları ve değerleri diziler dizi geçen anahtarları ve değerleri iki dizi geçen synonymical var. Eleman sayısı ‘key’ ve ‘value’ toplam her satır için aynı olmalıdır. Returns a tuple of two arrays: keys in sorted order, and values ​​summed for the corresponding keys. @@ -347,25 +348,28 @@ CREATE TABLE sum_map( statusMap Nested( status UInt16, requests UInt64 - ) + ), + statusMapTuple Tuple(Array(Int32), Array(Int32)) ) ENGINE = Log; INSERT INTO sum_map VALUES - ('2000-01-01', '2000-01-01 00:00:00', [1, 2, 3], [10, 10, 10]), - ('2000-01-01', '2000-01-01 00:00:00', [3, 4, 5], [10, 10, 10]), - ('2000-01-01', '2000-01-01 00:01:00', [4, 5, 6], [10, 10, 10]), - ('2000-01-01', '2000-01-01 00:01:00', [6, 7, 8], [10, 10, 10]); + ('2000-01-01', '2000-01-01 00:00:00', [1, 2, 3], [10, 10, 10], ([1, 2, 3], [10, 10, 10])), + ('2000-01-01', '2000-01-01 00:00:00', [3, 4, 5], [10, 10, 10], ([3, 4, 5], [10, 10, 10])), + ('2000-01-01', '2000-01-01 00:01:00', [4, 5, 6], [10, 10, 10], ([4, 5, 6], [10, 10, 10])), + ('2000-01-01', '2000-01-01 00:01:00', [6, 7, 8], [10, 10, 10], ([6, 7, 8], [10, 10, 10])); + SELECT timeslot, - sumMap(statusMap.status, statusMap.requests) + sumMap(statusMap.status, statusMap.requests), + sumMap(statusMapTuple) FROM sum_map GROUP BY timeslot ``` ``` text -┌────────────timeslot─┬─sumMap(statusMap.status, statusMap.requests)─┐ -│ 2000-01-01 00:00:00 │ ([1,2,3,4,5],[10,10,20,10,10]) │ -│ 2000-01-01 00:01:00 │ ([4,5,6,7,8],[10,10,20,10,10]) │ -└─────────────────────┴──────────────────────────────────────────────┘ +┌────────────timeslot─┬─sumMap(statusMap.status, statusMap.requests)─┬─sumMap(statusMapTuple)─────────┐ +│ 2000-01-01 00:00:00 │ ([1,2,3,4,5],[10,10,20,10,10]) │ ([1,2,3,4,5],[10,10,20,10,10]) │ +│ 2000-01-01 00:01:00 │ ([4,5,6,7,8],[10,10,20,10,10]) │ ([4,5,6,7,8],[10,10,20,10,10]) │ +└─────────────────────┴──────────────────────────────────────────────┴────────────────────────────────┘ ``` ## skewPop {#skewpop} @@ -514,10 +518,10 @@ Ve sonuç olacak: ## timeSeriesGroupRateSum(uıd, ts, val) {#agg-function-timeseriesgroupratesum} -Benzer şekilde timeSeriesGroupRateSum, timeSeriesGroupRateSum zaman serisi ve daha sonra toplam oranları birlikte oranını hesaplar. +Benzer şekilde `timeSeriesGroupSum`, `timeSeriesGroupRateSum` zaman serilerinin oranını hesaplar ve daha sonra toplam oranları birlikte hesaplar. Ayrıca, bu işlevi kullanmadan önce zaman damgası yükseliş sırasına göre olmalıdır. -Bu işlevi kullanın, yukarıdaki sonuç olacaktır: +Bu fonksiyon dataun ver theiye uygulanması `timeSeriesGroupSum` örnek, aşağıdaki sonucu alırsınız: ``` text [(2,0),(3,0.1),(7,0.3),(8,0.3),(12,0.3),(17,0.3),(18,0.3),(24,0.3),(25,0.1)] @@ -549,7 +553,7 @@ Türü `x` ve `weight` aynı olmalıdır. **Döndürülen değer** - Ağırlıklı ortalama. -- `NaN`. Tüm ağırlıklar 0’a eşitse. +- `NaN`. Tüm ağırlıklar 0'a eşitse. Tür: [Float64](../data-types/float.md). @@ -592,7 +596,7 @@ Fonksiyon değişken sayıda parametre alır. Parametreler olabilir `Tuple`, `Ar - Toplamdaki tüm parametreler için bir karma hesaplar, daha sonra hesaplamalarda kullanır. -- Bir adaptif örnekleme algoritması kullanır. Hesaplama durumu için işlev, 65536’ya kadar öğe karma değerlerinin bir örneğini kullanır. +- Bir adaptif örnekleme algoritması kullanır. Hesaplama durumu için işlev, 65536'ya kadar öğe karma değerlerinin bir örneğini kullanır. This algorithm is very accurate and very efficient on the CPU. When the query contains several of these functions, using `uniq` is almost as fast as using other aggregate functions. @@ -621,7 +625,7 @@ Bu `uniqCombined` fonksiyon, farklı değerlerin sayısını hesaplamak için iy Fonksiyon değişken sayıda parametre alır. Parametreler olabilir `Tuple`, `Array`, `Date`, `DateTime`, `String` veya sayısal türleri. -`HLL_precision` hücre sayısının baz-2 logaritmasıdır. [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog). İsteğe bağlı olarak işlevi kullanabilirsiniz `uniqCombined(x[, ...])`. İçin varsayılan değer `HLL_precision` etkin bir şekilde 96 KiB alan olan 17’dir (2^17 hücre, her biri 6 bit). +`HLL_precision` hücre sayısının baz-2 logaritmasıdır. [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog). İsteğe bağlı olarak işlevi kullanabilirsiniz `uniqCombined(x[, ...])`. İçin varsayılan değer `HLL_precision` etkin bir şekilde 96 KiB alan olan 17'dir (2^17 hücre, her biri 6 bit). **Döndürülen değer** @@ -687,7 +691,7 @@ Fonksiyon değişken sayıda parametre alır. Parametreler olabilir `Tuple`, `Ar - Belirli sonucu sağlar (sorgu işleme sırasına bağlı değildir). -Bu işlevi kullanmanızı önermiyoruz. Çoğu durumda, kullan [uniq](#agg_function-uniq) veya [uniqCombined](#agg_function-uniqcombined) işlev. +Bu işlevi kullanmanızı önermiyoruz. Çoğu durumda, kullan [uniq](#agg_function-uniq) veya [uniqCombined](#agg_function-uniqcombined) İşlev. **Ayrıca Bakınız** @@ -703,7 +707,7 @@ Farklı bağımsız değişken değerlerinin tam sayısını hesaplar. uniqExact(x[, ...]) ``` -Kullan… `uniqExact` kesinlikle kesin bir sonuca ihtiyacınız varsa işlev. Aksi takdirde kullanın [uniq](#agg_function-uniq) işlev. +Kullan... `uniqExact` kesinlikle kesin bir sonuca ihtiyacınız varsa işlev. Aksi takdirde kullanın [uniq](#agg_function-uniq) İşlev. Bu `uniqExact` fonksiyonu daha fazla bellek kullanır `uniq`, çünkü farklı değerlerin sayısı arttıkça devletin büyüklüğü sınırsız büyümeye sahiptir. @@ -727,19 +731,93 @@ Mesela, `groupArray (1) (x)` eşdeğ toer equivalentdir `[any (x)]`. Bazı durumlarda, hala yürütme sırasına güvenebilirsiniz. Bu, aşağıdaki durumlar için geçerlidir `SELECT` kullanan bir alt sorgudan gelir `ORDER BY`. -## grouparrayınsertat (değer, konum) {#grouparrayinsertatvalue-position} +## grouparrayınsertat {#grouparrayinsertat} Belirtilen konumda diziye bir değer ekler. -!!! note "Not" - Bu işlev, SQL dizileri için geleneksel tek tabanlı konumların aksine sıfır tabanlı konumlar kullanır. +**Sözdizimi** -Accepts the value and position as input. If several values ​​are inserted into the same position, any of them might end up in the resulting array (the first one will be used in the case of single-threaded execution). If no value is inserted into a position, the position is assigned the default value. +``` sql +groupArrayInsertAt(default_x, size)(x, pos); +``` -İsteğe bağlı parametreler: +Bir sorguda aynı konuma birkaç değer eklenirse, işlev aşağıdaki şekillerde davranır: -- Boş pozisyonlarda ikame için varsayılan değer. -- Elde edilen dizinin uzunluğu. Bu, tüm toplama anahtarları için aynı boyuttaki dizileri almanızı sağlar. Bu parametreyi kullanırken, varsayılan değer belirtilmelidir. +- Bir sorgu tek bir iş parçacığında yürütülürse, eklenen değerlerden ilki kullanılır. +- Bir sorgu birden çok iş parçacığında yürütülürse, ortaya çıkan değer, eklenen değerlerden belirsiz bir değerdir. + +**Parametre** + +- `x` — Value to be inserted. [İfade](../syntax.md#syntax-expressions) biri sonuçta [desteklenen veri türleri](../../sql-reference/data-types/index.md). +- `pos` — Position at which the specified element `x` eklen .ecektir. Dizideki dizin numaralandırma sıfırdan başlar. [Uİnt32](../../sql-reference/data-types/int-uint.md#uint-ranges). +- `default_x`— Default value for substituting in empty positions. Optional parameter. [İfade](../syntax.md#syntax-expressions) için yapılandırılmış veri türü ile sonuçlanan `x` parametre. Eğer `default_x` tanımlan ,mamıştır, [varsayılan değerler](../../sql-reference/statements/create.md#create-default-values) kullanılır. +- `size`— Length of the resulting array. Optional parameter. When using this parameter, the default value `default_x` belirt .ilmelidir. [Uİnt32](../../sql-reference/data-types/int-uint.md#uint-ranges). + +**Döndürülen değer** + +- Eklenen değerlerle dizi. + +Tür: [Dizi](../../sql-reference/data-types/array.md#data-type-array). + +**Örnek** + +Sorgu: + +``` sql +SELECT groupArrayInsertAt(toString(number), number * 2) FROM numbers(5); +``` + +Sonuç: + +``` text +┌─groupArrayInsertAt(toString(number), multiply(number, 2))─┐ +│ ['0','','1','','2','','3','','4'] │ +└───────────────────────────────────────────────────────────┘ +``` + +Sorgu: + +``` sql +SELECT groupArrayInsertAt('-')(toString(number), number * 2) FROM numbers(5); +``` + +Sonuç: + +``` text +┌─groupArrayInsertAt('-')(toString(number), multiply(number, 2))─┐ +│ ['0','-','1','-','2','-','3','-','4'] │ +└────────────────────────────────────────────────────────────────┘ +``` + +Sorgu: + +``` sql +SELECT groupArrayInsertAt('-', 5)(toString(number), number * 2) FROM numbers(5); +``` + +Sonuç: + +``` text +┌─groupArrayInsertAt('-', 5)(toString(number), multiply(number, 2))─┐ +│ ['0','-','1','-','2'] │ +└───────────────────────────────────────────────────────────────────┘ +``` + +Elemanların tek bir konuma çok dişli yerleştirilmesi. + +Sorgu: + +``` sql +SELECT groupArrayInsertAt(number, 0) FROM numbers_mt(10) SETTINGS max_block_size = 1; +``` + +Bu sorgu sonucunda rastgele tamsayı elde edersiniz `[0,9]` Aralık. Mesela: + +``` text +┌─groupArrayInsertAt(number, 0)─┐ +│ [7] │ +└───────────────────────────────┘ +``` ## groupArrayMovingSum {#agg_function-grouparraymovingsum} @@ -891,7 +969,7 @@ FROM t ## groupUniqArray (x), groupUniqArray (max\_size)(x) {#groupuniqarrayx-groupuniqarraymax-sizex} -Farklı bağımsız değişken değerlerinden bir dizi oluşturur. Bellek tüketimi için aynıdır `uniqExact` işlev. +Farklı bağımsız değişken değerlerinden bir dizi oluşturur. Bellek tüketimi için aynıdır `uniqExact` İşlev. İkinci versiyonu (ile `max_size` parametre), elde edilen dizinin boyutunu sınırlar `max_size` öğeler. Mesela, `groupUniqArray(1)(x)` eşdeğ toer equivalentdir `[any(x)]`. @@ -900,9 +978,9 @@ Mesela, `groupUniqArray(1)(x)` eşdeğ toer equivalentdir `[any(x)]`. Yaklaşık hesaplar [quantile](https://en.wikipedia.org/wiki/Quantile) sayısal veri dizisinin. -Bu işlev geçerlidir [rezerv reservoiruar örnek samplinglemesi](https://en.wikipedia.org/wiki/Reservoir_sampling) 8192’ye kadar bir rezervuar boyutu ve örnekleme için rastgele sayı üreteci ile. Sonuç deterministik değildir. Tam bir miktar elde etmek için [quantileExact](#quantileexact) işlev. +Bu işlev geçerlidir [rezerv reservoiruar örnek samplinglemesi](https://en.wikipedia.org/wiki/Reservoir_sampling) 8192'ye kadar bir rezervuar boyutu ve örnekleme için rastgele sayı üreteci ile. Sonuç deterministik değildir. Tam bir miktar elde etmek için [quantileExact](#quantileexact) İşlev. -Çoklu kullanırken `quantile*` bir sorguda farklı düzeylerde işlevler, iç durumları birleştirilmez (diğer bir deyişle, sorgu olabilir daha az verimli çalışır). Bu durumda, kullan [quantiles](#quantiles) işlev. +Çoklu kullanırken `quantile*` bir sorguda farklı düzeylerde işlevler, iç durumları birleştirilmez (diğer bir deyişle, sorgu olabilir daha az verimli çalışır). Bu durumda, kullan [quantiles](#quantiles) İşlev. **Sözdizimi** @@ -963,9 +1041,9 @@ Sonuç: Yaklaşık hesaplar [quantile](https://en.wikipedia.org/wiki/Quantile) sayısal veri dizisinin. -Bu işlev geçerlidir [rezerv reservoiruar örnek samplinglemesi](https://en.wikipedia.org/wiki/Reservoir_sampling) 8192’ye kadar bir rezervuar boyutu ve örnekleme deterministik algoritması ile. Sonuç deterministiktir. Tam bir miktar elde etmek için [quantileExact](#quantileexact) işlev. +Bu işlev geçerlidir [rezerv reservoiruar örnek samplinglemesi](https://en.wikipedia.org/wiki/Reservoir_sampling) 8192'ye kadar bir rezervuar boyutu ve örnekleme deterministik algoritması ile. Sonuç deterministiktir. Tam bir miktar elde etmek için [quantileExact](#quantileexact) İşlev. -Çoklu kullanırken `quantile*` bir sorguda farklı düzeylerde işlevler, iç durumları birleştirilmez (diğer bir deyişle, sorgu olabilir daha az verimli çalışır). Bu durumda, kullan [quantiles](#quantiles) işlev. +Çoklu kullanırken `quantile*` bir sorguda farklı düzeylerde işlevler, iç durumları birleştirilmez (diğer bir deyişle, sorgu olabilir daha az verimli çalışır). Bu durumda, kullan [quantiles](#quantiles) İşlev. **Sözdizimi** @@ -1029,7 +1107,7 @@ Tam olarak hesaplar [quantile](https://en.wikipedia.org/wiki/Quantile) sayısal To get exact value, all the passed values ​​are combined into an array, which is then partially sorted. Therefore, the function consumes `O(n)` bellek, nerede `n` geçirilen değerler say .ısıdır. Bununla birlikte, az sayıda değer için, işlev çok etkilidir. -Çoklu kullanırken `quantile*` bir sorguda farklı düzeylerde işlevler, iç durumları birleştirilmez (diğer bir deyişle, sorgu olabilir daha az verimli çalışır). Bu durumda, kullan [quantiles](#quantiles) işlev. +Çoklu kullanırken `quantile*` bir sorguda farklı düzeylerde işlevler, iç durumları birleştirilmez (diğer bir deyişle, sorgu olabilir daha az verimli çalışır). Bu durumda, kullan [quantiles](#quantiles) İşlev. **Sözdizimi** @@ -1081,7 +1159,7 @@ Tam olarak hesaplar [quantile](https://en.wikipedia.org/wiki/Quantile) her elema To get exact value, all the passed values ​​are combined into an array, which is then partially sorted. Each value is counted with its weight, as if it is present `weight` times. A hash table is used in the algorithm. Because of this, if the passed values ​​are frequently repeated, the function consumes less RAM than [quantileExact](#quantileexact). Bunun yerine bu işlevi kullanabilirsiniz `quantileExact` ve 1 ağırlığını belirtin. -Çoklu kullanırken `quantile*` bir sorguda farklı düzeylerde işlevler, iç durumları birleştirilmez (diğer bir deyişle, sorgu olabilir daha az verimli çalışır). Bu durumda, kullan [quantiles](#quantiles) işlev. +Çoklu kullanırken `quantile*` bir sorguda farklı düzeylerde işlevler, iç durumları birleştirilmez (diğer bir deyişle, sorgu olabilir daha az verimli çalışır). Bu durumda, kullan [quantiles](#quantiles) İşlev. **Sözdizimi** @@ -1145,7 +1223,7 @@ Belirlenen hassas hesaplar ile [quantile](https://en.wikipedia.org/wiki/Quantile Sonuç deterministiktir (sorgu işleme sırasına bağlı değildir). Fonksiyon yükleme web sayfaları kez veya arka uç yanıt süreleri gibi dağılımları tanımlamak dizileri ile çalışmak için optimize edilmiştir. -Çoklu kullanırken `quantile*` bir sorguda farklı düzeylerde işlevler, iç durumları birleştirilmez (diğer bir deyişle, sorgu olabilir daha az verimli çalışır). Bu durumda, kullan [quantiles](#quantiles) işlev. +Çoklu kullanırken `quantile*` bir sorguda farklı düzeylerde işlevler, iç durumları birleştirilmez (diğer bir deyişle, sorgu olabilir daha az verimli çalışır). Bu durumda, kullan [quantiles](#quantiles) İşlev. **Sözdizimi** @@ -1168,10 +1246,10 @@ Takma ad: `medianTiming`. Hesaplama doğru ise: -- Toplam değer sayısı 5670’i geçmez. -- Toplam değer sayısı 5670’i aşıyor, ancak sayfa yükleme süresi 1024 ms’den az. +- Toplam değer sayısı 5670'i geçmez. +- Toplam değer sayısı 5670'i aşıyor, ancak sayfa yükleme süresi 1024 ms'den az. -Aksi takdirde, hesaplamanın sonucu 16 MS’nin en yakın katlarına yuvarlanır. +Aksi takdirde, hesaplamanın sonucu 16 MS'nin en yakın katlarına yuvarlanır. !!! note "Not" Sayfa yükleme süresi nicelerini hesaplamak için, bu işlev daha etkili ve doğrudur [quantile](#quantile). @@ -1183,7 +1261,7 @@ Aksi takdirde, hesaplamanın sonucu 16 MS’nin en yakın katlarına yuvarlanır Tür: `Float32`. !!! note "Not" - İşlev valuese hiçbir değer geçir (ilmem (işse (kullanırken `quantileTimingIf`), [Nine](../../sql-reference/data-types/float.md#data_type-float-nan-inf) döndürülür. Bunun amacı, bu vakaları sıfır ile sonuçlanan vakalardan ayırmaktır. Görmek [ORDER BY FLA BYGE](../statements/select.md#select-order-by) sıralama ile ilgili notlar için `NaN` değerler. + İşlev valuese hiçbir değer geçir (ilmem (işse (kullanırken `quantileTimingIf`), [Nine](../../sql-reference/data-types/float.md#data_type-float-nan-inf) döndürülür. Bunun amacı, bu vakaları sıfır ile sonuçlanan vakalardan ayırmaktır. Görmek [ORDER BY FLA BYGE](../statements/select/order-by.md#select-order-by) sıralama ile ilgili notlar için `NaN` değerler. **Örnek** @@ -1228,7 +1306,7 @@ Belirlenen hassas hesaplar ile [quantile](https://en.wikipedia.org/wiki/Quantile Sonuç deterministiktir (sorgu işleme sırasına bağlı değildir). Fonksiyon yükleme web sayfaları kez veya arka uç yanıt süreleri gibi dağılımları tanımlamak dizileri ile çalışmak için optimize edilmiştir. -Çoklu kullanırken `quantile*` bir sorguda farklı düzeylerde işlevler, iç durumları birleştirilmez (diğer bir deyişle, sorgu olabilir daha az verimli çalışır). Bu durumda, kullan [quantiles](#quantiles) işlev. +Çoklu kullanırken `quantile*` bir sorguda farklı düzeylerde işlevler, iç durumları birleştirilmez (diğer bir deyişle, sorgu olabilir daha az verimli çalışır). Bu durumda, kullan [quantiles](#quantiles) İşlev. **Sözdizimi** @@ -1253,10 +1331,10 @@ Takma ad: `medianTimingWeighted`. Hesaplama doğru ise: -- Toplam değer sayısı 5670’i geçmez. -- Toplam değer sayısı 5670’i aşıyor, ancak sayfa yükleme süresi 1024 ms’den az. +- Toplam değer sayısı 5670'i geçmez. +- Toplam değer sayısı 5670'i aşıyor, ancak sayfa yükleme süresi 1024 ms'den az. -Aksi takdirde, hesaplamanın sonucu 16 MS’nin en yakın katlarına yuvarlanır. +Aksi takdirde, hesaplamanın sonucu 16 MS'nin en yakın katlarına yuvarlanır. !!! note "Not" Sayfa yükleme süresi nicelerini hesaplamak için, bu işlev daha etkili ve doğrudur [quantile](#quantile). @@ -1268,7 +1346,7 @@ Aksi takdirde, hesaplamanın sonucu 16 MS’nin en yakın katlarına yuvarlanır Tür: `Float32`. !!! note "Not" - İşlev valuese hiçbir değer geçir (ilmem (işse (kullanırken `quantileTimingIf`), [Nine](../../sql-reference/data-types/float.md#data_type-float-nan-inf) döndürülür. Bunun amacı, bu vakaları sıfır ile sonuçlanan vakalardan ayırmaktır. Görmek [ORDER BY FLA BYGE](../statements/select.md#select-order-by) sıralama ile ilgili notlar için `NaN` değerler. + İşlev valuese hiçbir değer geçir (ilmem (işse (kullanırken `quantileTimingIf`), [Nine](../../sql-reference/data-types/float.md#data_type-float-nan-inf) döndürülür. Bunun amacı, bu vakaları sıfır ile sonuçlanan vakalardan ayırmaktır. Görmek [ORDER BY FLA BYGE](../statements/select/order-by.md#select-order-by) sıralama ile ilgili notlar için `NaN` değerler. **Örnek** @@ -1308,11 +1386,11 @@ Sonuç: Yaklaşık hesaplar [quantile](https://en.wikipedia.org/wiki/Quantile) kullanarak sayısal veri diz ofisinin [t-dig -est](https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf) algoritma. -Maksimum hata %1’dir. Bellek tüketimi `log(n)`, nere `n` değer say isısıdır. Sonuç, sorguyu çalıştırma sırasına bağlıdır ve nondeterministic. +Maksimum hata %1'dir. Bellek tüketimi `log(n)`, nere `n` değer say isısıdır. Sonuç, sorguyu çalıştırma sırasına bağlıdır ve nondeterministic. Fonksiyonun performansı, performanstan daha düşüktür [quantile](#quantile) veya [quantileTiming](#quantiletiming). Durum boyutunun hassasiyete oranı açısından, bu işlev çok daha iyidir `quantile`. -Çoklu kullanırken `quantile*` bir sorguda farklı düzeylerde işlevler, iç durumları birleştirilmez (diğer bir deyişle, sorgu olabilir daha az verimli çalışır). Bu durumda, kullan [quantiles](#quantiles) işlev. +Çoklu kullanırken `quantile*` bir sorguda farklı düzeylerde işlevler, iç durumları birleştirilmez (diğer bir deyişle, sorgu olabilir daha az verimli çalışır). Bu durumda, kullan [quantiles](#quantiles) İşlev. **Sözdizimi** @@ -1360,13 +1438,13 @@ Sonuç: ## quantileTDigestWeighted {#quantiletdigestweighted} -Yaklaşık hesaplar [quantile](https://en.wikipedia.org/wiki/Quantile) kullanarak sayısal veri diz ofisinin [t-dig -est](https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf) algoritma. İşlev, her sıra üyesinin ağırlığını dikkate alır. Maksimum hata %1’dir. Bellek tüketimi `log(n)`, nere `n` değer say isısıdır. +Yaklaşık hesaplar [quantile](https://en.wikipedia.org/wiki/Quantile) kullanarak sayısal veri diz ofisinin [t-dig -est](https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf) algoritma. İşlev, her sıra üyesinin ağırlığını dikkate alır. Maksimum hata %1'dir. Bellek tüketimi `log(n)`, nere `n` değer say isısıdır. Fonksiyonun performansı, performanstan daha düşüktür [quantile](#quantile) veya [quantileTiming](#quantiletiming). Durum boyutunun hassasiyete oranı açısından, bu işlev çok daha iyidir `quantile`. Sonuç, sorguyu çalıştırma sırasına bağlıdır ve nondeterministic. -Çoklu kullanırken `quantile*` bir sorguda farklı düzeylerde işlevler, iç durumları birleştirilmez (diğer bir deyişle, sorgu olabilir daha az verimli çalışır). Bu durumda, kullan [quantiles](#quantiles) işlev. +Çoklu kullanırken `quantile*` bir sorguda farklı düzeylerde işlevler, iç durumları birleştirilmez (diğer bir deyişle, sorgu olabilir daha az verimli çalışır). Bu durumda, kullan [quantiles](#quantiles) İşlev. **Sözdizimi** @@ -1467,20 +1545,32 @@ Bir rassal değişkenin varyansının tarafsız bir tahminini temsil eder, eğer Dönüşler `Float64`. Ne zaman `n <= 1`, dönüşler `+∞`. +!!! note "Not" + Bu işlev sayısal olarak kararsız algoritma kullanır. İhtiyacınız varsa [sayısal kararlılık](https://en.wikipedia.org/wiki/Numerical_stability) hesaplamalarda kullan `varSampStable` İşlev. Daha yavaş çalışır, ancak daha düşük hesaplama hatası sağlar. + ## varPop (x) {#varpopx} Miktarı hesaplar `Σ((x - x̅)^2) / n`, nere `n` örneklem büyüklüğü ve `x̅`ortalama değer isidir `x`. Başka bir deyişle, bir dizi değer için dağılım. Dönüşler `Float64`. +!!! note "Not" + Bu işlev sayısal olarak kararsız algoritma kullanır. İhtiyacınız varsa [sayısal kararlılık](https://en.wikipedia.org/wiki/Numerical_stability) hesaplamalarda kullan `varPopStable` İşlev. Daha yavaş çalışır, ancak daha düşük hesaplama hatası sağlar. + ## stddevSamp(x) {#stddevsampx} Sonuç kareköküne eşittir `varSamp(x)`. +!!! note "Not" + Bu işlev sayısal olarak kararsız algoritma kullanır. İhtiyacınız varsa [sayısal kararlılık](https://en.wikipedia.org/wiki/Numerical_stability) hesaplamalarda kullan `stddevSampStable` İşlev. Daha yavaş çalışır, ancak daha düşük hesaplama hatası sağlar. + ## stddevPop(x) {#stddevpopx} Sonuç kareköküne eşittir `varPop(x)`. +!!! note "Not" + Bu işlev sayısal olarak kararsız algoritma kullanır. İhtiyacınız varsa [sayısal kararlılık](https://en.wikipedia.org/wiki/Numerical_stability) hesaplamalarda kullan `stddevPopStable` İşlev. Daha yavaş çalışır, ancak daha düşük hesaplama hatası sağlar. + ## topK (N) (x) {#topknx} Belirtilen sütundaki yaklaşık en sık değerleri bir dizi döndürür. Elde edilen dizi, değerlerin yaklaşık frekansının azalan sırasına göre sıralanır (değerlerin kendileri tarafından değil). @@ -1503,7 +1593,7 @@ Parametre atlanırsa, varsayılan değer 10 kullanılır. **Değişkenler** -- ’ x ’ – The value to calculate frequency. +- ' x ' – The value to calculate frequency. **Örnek** @@ -1565,14 +1655,23 @@ Değerini hesaplar `Σ((x - x̅)(y - y̅)) / (n - 1)`. Float64 Döndürür. Ne zaman `n <= 1`, returns +∞. +!!! note "Not" + Bu işlev sayısal olarak kararsız algoritma kullanır. İhtiyacınız varsa [sayısal kararlılık](https://en.wikipedia.org/wiki/Numerical_stability) hesaplamalarda kullan `covarSampStable` İşlev. Daha yavaş çalışır, ancak daha düşük hesaplama hatası sağlar. + ## covarPop (x, y) {#covarpopx-y} Değerini hesaplar `Σ((x - x̅)(y - y̅)) / n`. +!!! note "Not" + Bu işlev sayısal olarak kararsız algoritma kullanır. İhtiyacınız varsa [sayısal kararlılık](https://en.wikipedia.org/wiki/Numerical_stability) hesaplamalarda kullan `covarPopStable` İşlev. Daha yavaş çalışır, ancak daha düşük bir hesaplama hatası sağlar. + ## corr(x, y) {#corrx-y} Pearson korelasyon katsayısını hesaplar: `Σ((x - x̅)(y - y̅)) / sqrt(Σ((x - x̅)^2) * Σ((y - y̅)^2))`. +!!! note "Not" + Bu işlev sayısal olarak kararsız algoritma kullanır. İhtiyacınız varsa [sayısal kararlılık](https://en.wikipedia.org/wiki/Numerical_stability) hesaplamalarda kullan `corrStable` İşlev. Daha yavaş çalışır, ancak daha düşük hesaplama hatası sağlar. + ## categoricalınformationvalue {#categoricalinformationvalue} Değerini hesaplar `(P(tag = 1) - P(tag = 0))(log(P(tag = 1)) - log(P(tag = 0)))` her kategori için. diff --git a/docs/tr/sql-reference/ansi.md b/docs/tr/sql-reference/ansi.md deleted file mode 120000 index 3cf6bffed67..00000000000 --- a/docs/tr/sql-reference/ansi.md +++ /dev/null @@ -1 +0,0 @@ -../../en/sql-reference/ansi.md \ No newline at end of file diff --git a/docs/tr/sql-reference/ansi.md b/docs/tr/sql-reference/ansi.md new file mode 100644 index 00000000000..5946da05ce2 --- /dev/null +++ b/docs/tr/sql-reference/ansi.md @@ -0,0 +1,180 @@ +--- +machine_translated: true +machine_translated_rev: ad252bbb4f7e2899c448eb42ecc39ff195c8faa1 +toc_priority: 40 +toc_title: "ANSI uyumlulu\u011Fu" +--- + +# ClickHouse SQL Lehçesinin ANSI SQL uyumluluğu {#ansi-sql-compatibility-of-clickhouse-sql-dialect} + +!!! note "Not" + Bu makale Tablo 38, “Feature taxonomy and definition for mandatory features”, Annex F of ISO/IEC CD 9075-2:2013. + +## Davranış farklılıkları {#differences-in-behaviour} + +Aşağıdaki tabloda, sorgu özelliği ClickHouse çalışır, ancak ANSI SQL'DE belirtildiği gibi davranır durumlarda listeler. + +| Feature ID | Özellik Adı | Fark | +|------------|---------------------------------------|----------------------------------------------------------------------------------------------------| +| E011 | Sayısal veri türleri | Dönem ile sayısal literal yaklaşık olarak yorumlanır (`Float64`) tam yerine (`Decimal`) | +| E051-05 | Seçme öğeler yeniden adlandırılabilir | Öğe yeniden adlandırmaları, yalnızca seçme sonucundan daha geniş bir görünürlük kapsamına sahiptir | +| E141-01 | NOT NULL kısıtlamaları | `NOT NULL` tablo sütunları için varsayılan olarak ima edilir | +| E011-04 | Aritmetik operat operatorsörler | ClickHouse işaretli aritmetik yerine taşar ve sonuç veri türünü özel kurallara göre değiştirir | + +## Özellik Durumu {#feature-status} + +| Feature ID | Özellik Adı | Durum | Yorum | +|------------|-----------------------------------------------------------------------------------------------------------------------------|--------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **E011** | **Sayısal veri türleri** | **Kısmi**{.text-warning} | | +| E011-01 | Tamsayı ve SMALLİNT veri türleri | Evet{.text-success} | | +| E011-02 | Gerçek, çift hassas ve FLOAT veri türleri veri türleri | Kısmi{.text-warning} | `FLOAT()`, `REAL` ve `DOUBLE PRECISION` desteklenmiyor | +| E011-03 | Ondalık ve sayısal veri türleri | Kısmi{.text-warning} | Sadece `DECIMAL(p,s)` desteklenir, değil `NUMERIC` | +| E011-04 | Aritmetik operat operatorsörler | Evet{.text-success} | | +| E011-05 | Sayısal karşılaştırma | Evet{.text-success} | | +| E011-06 | Sayısal veri türleri arasında örtülü döküm | Hayır{.text-danger} | ANSI SQL, sayısal türler arasında rasgele örtülü döküm yapılmasına izin verirken, ClickHouse, örtülü döküm yerine birden fazla aşırı yüke sahip işlevlere dayanır | +| **E021** | **Karakter dizesi türleri** | **Kısmi**{.text-warning} | | +| E021-01 | Karakter veri türü | Hayır{.text-danger} | | +| E021-02 | Karakter değişken veri türü | Hayır{.text-danger} | `String` benzer şekilde davranır, ancak parantez içinde uzunluk sınırı olmadan | +| E021-03 | Karakter değişmezleri | Kısmi{.text-warning} | Ardışık değişmezlerin ve karakter seti desteğinin otomatik olarak birleştirilmesi yok | +| E021-04 | CHARACTER\_LENGTH işlevi | Kısmi{.text-warning} | Hayır `USING` yan | +| E021-05 | OCTET\_LENGTH işlevi | Hayır{.text-danger} | `LENGTH` benzer şekilde davranır | +| E021-06 | SUBSTRING | Kısmi{.text-warning} | İçin destek yok `SIMILAR` ve `ESCAPE` CLA ,us ,es, no `SUBSTRING_REGEX` varyant | +| E021-07 | Karakter birleştirme | Kısmi{.text-warning} | Hayır `COLLATE` yan | +| E021-08 | Üst ve alt fonksiyonlar | Evet{.text-success} | | +| E021-09 | TRİM fonksiyonu | Evet{.text-success} | | +| E021-10 | Sabit uzunlukta ve değişken uzunlukta karakter dizesi türleri arasında örtülü döküm | Hayır{.text-danger} | ANSI SQL, dize türleri arasında rasgele örtük döküm yapılmasına izin verirken, ClickHouse, örtük döküm yerine birden fazla aşırı yüke sahip işlevlere dayanır | +| E021-11 | Pozisyon fonksiyonu | Kısmi{.text-warning} | İçin destek yok `IN` ve `USING` CLA ,us ,es, no `POSITION_REGEX` varyant | +| E021-12 | Karakter karşılaştırma | Evet{.text-success} | | +| **E031** | **Tanıtıcılar** | **Kısmi**{.text-warning} | | +| E031-01 | Ayrılmış tanımlayıcılar | Kısmi{.text-warning} | Unicode literal desteği sınırlıdır | +| E031-02 | Küçük harf tanımlayıcıları | Evet{.text-success} | | +| E031-03 | Sondaki alt çizgi | Evet{.text-success} | | +| **E051** | **Temel sorgu belirtimi** | **Kısmi**{.text-warning} | | +| E051-01 | SELECT DISTINCT | Evet{.text-success} | | +| E051-02 | GROUP BY fık clausera | Evet{.text-success} | | +| E051-04 | GROUP BY içinde olmayan sütunlar içerebilir `` | 是{.text-success} | | +| E051-05 | 选择项目可以重命名 | 是{.text-success} | | +| E051-06 | 有条款 | 是{.text-success} | | +| E051-07 | 合格\*在选择列表中 | 是{.text-success} | | +| E051-08 | FROM子句中的关联名称 | 是{.text-success} | | +| E051-09 | 重命名FROM子句中的列 | 非也。{.text-danger} | | +| **E061** | **基本谓词和搜索条件** | **部分**{.text-warning} | | +| E061-01 | 比较谓词 | 是{.text-success} | | +| E061-02 | 谓词之间 | 部分{.text-warning} | 非也。 `SYMMETRIC` 和 `ASYMMETRIC` 条款 | +| E061-03 | 在具有值列表的谓词中 | 是{.text-success} | | +| E061-04 | 像谓词 | 是{.text-success} | | +| E061-05 | LIKE谓词：逃避条款 | 非也。{.text-danger} | | +| E061-06 | 空谓词 | 是{.text-success} | | +| E061-07 | 量化比较谓词 | 非也。{.text-danger} | | +| E061-08 | 存在谓词 | 非也。{.text-danger} | | +| E061-09 | 比较谓词中的子查询 | 是{.text-success} | | +| E061-11 | 谓词中的子查询 | 是{.text-success} | | +| E061-12 | 量化比较谓词中的子查询 | 非也。{.text-danger} | | +| E061-13 | 相关子查询 | 非也。{.text-danger} | | +| E061-14 | 搜索条件 | 是{.text-success} | | +| **E071** | **基本查询表达式** | **部分**{.text-warning} | | +| E071-01 | UNION DISTINCT table运算符 | 非也。{.text-danger} | | +| E071-02 | 联合所有表运算符 | 是{.text-success} | | +| E071-03 | 除了不同的表运算符 | 非也。{.text-danger} | | +| E071-05 | 通过表运算符组合的列不必具有完全相同的数据类型 | 是{.text-success} | | +| E071-06 | 子查询中的表运算符 | 是{.text-success} | | +| **E081** | **基本特权** | **部分**{.text-warning} | 正在进行的工作 | +| **E091** | **设置函数** | **是**{.text-success} | | +| E091-01 | AVG | 是{.text-success} | | +| E091-02 | COUNT | 是{.text-success} | | +| E091-03 | MAX | 是{.text-success} | | +| E091-04 | MIN | 是{.text-success} | | +| E091-05 | SUM | 是{.text-success} | | +| E091-06 | 全部量词 | 非也。{.text-danger} | | +| E091-07 | 不同的量词 | 部分{.text-warning} | 并非所有聚合函数都受支持 | +| **E101** | **基本数据操作** | **部分**{.text-warning} | | +| E101-01 | 插入语句 | 是{.text-success} | 注：ClickHouse中的主键并不意味着 `UNIQUE` 约束 | +| E101-03 | 搜索更新语句 | 非也。{.text-danger} | 有一个 `ALTER UPDATE` 批量数据修改语句 | +| E101-04 | 搜索的删除语句 | 非也。{.text-danger} | 有一个 `ALTER DELETE` 批量数据删除声明 | +| **E111** | **单行SELECT语句** | **非也。**{.text-danger} | | +| **E121** | **基本光标支持** | **非也。**{.text-danger} | | +| E121-01 | DECLARE CURSOR | 非也。{.text-danger} | | +| E121-02 | 按列排序不需要在选择列表中 | 非也。{.text-danger} | | +| E121-03 | 按顺序排列的值表达式 | 非也。{.text-danger} | | +| E121-04 | 公开声明 | 非也。{.text-danger} | | +| E121-06 | 定位更新语句 | 非也。{.text-danger} | | +| E121-07 | 定位删除语句 | 非也。{.text-danger} | | +| E121-08 | 关闭声明 | 非也。{.text-danger} | | +| E121-10 | FETCH语句：隐式NEXT | 非也。{.text-danger} | | +| E121-17 | 使用保持游标 | 非也。{.text-danger} | | +| **E131** | **空值支持（空值代替值)** | **部分**{.text-warning} | 一些限制适用 | +| **E141** | **基本完整性约束** | **部分**{.text-warning} | | +| E141-01 | 非空约束 | 是{.text-success} | 注: `NOT NULL` 默认情况下，表列隐含 | +| E141-02 | 非空列的唯一约束 | 非也。{.text-danger} | | +| E141-03 | 主键约束 | 非也。{.text-danger} | | +| E141-04 | 对于引用删除操作和引用更新操作，具有默认无操作的基本外键约束 | 非也。{.text-danger} | | +| E141-06 | 检查约束 | 是{.text-success} | | +| E141-07 | 列默认值 | 是{.text-success} | | +| E141-08 | 在主键上推断为非NULL | 是{.text-success} | | +| E141-10 | 可以按任何顺序指定外键中的名称 | 非也。{.text-danger} | | +| **E151** | **交易支持** | **非也。**{.text-danger} | | +| E151-01 | 提交语句 | 非也。{.text-danger} | | +| E151-02 | 回滚语句 | 非也。{.text-danger} | | +| **E152** | **基本设置事务语句** | **非也。**{.text-danger} | | +| E152-01 | SET TRANSACTION语句：隔离级别SERIALIZABLE子句 | 非也。{.text-danger} | | +| E152-02 | SET TRANSACTION语句：只读和读写子句 | 非也。{.text-danger} | | +| **E153** | **具有子查询的可更新查询** | **非也。**{.text-danger} | | +| **E161** | **SQL注释使用前导双减** | **是**{.text-success} | | +| **E171** | **SQLSTATE支持** | **非也。**{.text-danger} | | +| **E182** | **主机语言绑定** | **非也。**{.text-danger} | | +| **F031** | **基本架构操作** | **部分**{.text-warning} | | +| F031-01 | CREATE TABLE语句创建持久基表 | 部分{.text-warning} | 非也。 `SYSTEM VERSIONING`, `ON COMMIT`, `GLOBAL`, `LOCAL`, `PRESERVE`, `DELETE`, `REF IS`, `WITH OPTIONS`, `UNDER`, `LIKE`, `PERIOD FOR` 子句，不支持用户解析的数据类型 | +| F031-02 | 创建视图语句 | 部分{.text-warning} | 非也。 `RECURSIVE`, `CHECK`, `UNDER`, `WITH OPTIONS` 子句，不支持用户解析的数据类型 | +| F031-03 | 赠款声明 | 是{.text-success} | | +| F031-04 | ALTER TABLE语句：ADD COLUMN子句 | 部分{.text-warning} | 不支持 `GENERATED` 条款和系统时间段 | +| F031-13 | DROP TABLE语句：RESTRICT子句 | 非也。{.text-danger} | | +| F031-16 | DROP VIEW语句：RESTRICT子句 | 非也。{.text-danger} | | +| F031-19 | REVOKE语句：RESTRICT子句 | 非也。{.text-danger} | | +| **F041** | **基本连接表** | **部分**{.text-warning} | | +| F041-01 | Inner join（但不一定是INNER关键字) | 是{.text-success} | | +| F041-02 | 内部关键字 | 是{.text-success} | | +| F041-03 | LEFT OUTER JOIN | 是{.text-success} | | +| F041-04 | RIGHT OUTER JOIN | 是{.text-success} | | +| F041-05 | 可以嵌套外部连接 | 是{.text-success} | | +| F041-07 | 左侧或右侧外部联接中的内部表也可用于内部联接 | 是{.text-success} | | +| F041-08 | 支持所有比较运算符（而不仅仅是=) | 非也。{.text-danger} | | +| **F051** | **基本日期和时间** | **部分**{.text-warning} | | +| F051-01 | 日期数据类型（包括对日期文字的支持) | 部分{.text-warning} | 没有文字 | +| F051-02 | 时间数据类型（包括对时间文字的支持），秒小数精度至少为0 | 非也。{.text-danger} | | +| F051-03 | 时间戳数据类型（包括对时间戳文字的支持），小数秒精度至少为0和6 | 非也。{.text-danger} | `DateTime64` 时间提供了类似的功能 | +| F051-04 | 日期、时间和时间戳数据类型的比较谓词 | 部分{.text-warning} | 只有一种数据类型可用 | +| F051-05 | Datetime类型和字符串类型之间的显式转换 | 是{.text-success} | | +| F051-06 | CURRENT\_DATE | 非也。{.text-danger} | `today()` 是相似的 | +| F051-07 | LOCALTIME | 非也。{.text-danger} | `now()` 是相似的 | +| F051-08 | LOCALTIMESTAMP | 非也。{.text-danger} | | +| **F081** | **联盟和视图除外** | **部分**{.text-warning} | | +| **F131** | **分组操作** | **部分**{.text-warning} | | +| F131-01 | WHERE、GROUP BY和HAVING子句在具有分组视图的查询中受支持 | 是{.text-success} | | +| F131-02 | 具有分组视图的查询中支持的多个表 | 是{.text-success} | | +| F131-03 | 设置具有分组视图的查询中支持的函数 | 是{.text-success} | | +| F131-04 | 具有分组依据和具有子句和分组视图的子查询 | 是{.text-success} | | +| F131-05 | 单行选择具有GROUP BY和具有子句和分组视图 | 非也。{.text-danger} | | +| **F181** | **多模块支持** | **非也。**{.text-danger} | | +| **F201** | **投函数** | **是**{.text-success} | | +| **F221** | **显式默认值** | **非也。**{.text-danger} | | +| **F261** | **案例表达式** | **是**{.text-success} | | +| F261-01 | 简单案例 | 是{.text-success} | | +| F261-02 | 检索案例 | 是{.text-success} | | +| F261-03 | NULLIF | 是{.text-success} | | +| F261-04 | COALESCE | 是{.text-success} | | +| **F311** | **架构定义语句** | **部分**{.text-warning} | | +| F311-01 | CREATE SCHEMA | 非也。{.text-danger} | | +| F311-02 | 为持久基表创建表 | 是{.text-success} | | +| F311-03 | CREATE VIEW | 是{.text-success} | | +| F311-04 | CREATE VIEW: WITH CHECK OPTION | 非也。{.text-danger} | | +| F311-05 | 赠款声明 | 是{.text-success} | | +| **F471** | **标量子查询值** | **是**{.text-success} | | +| **F481** | **扩展空谓词** | **是**{.text-success} | | +| **F812** | **基本标记** | **非也。**{.text-danger} | | +| **T321** | **基本的SQL调用例程** | **非也。**{.text-danger} | | +| T321-01 | 无重载的用户定义函数 | 非也。{.text-danger} | | +| T321-02 | 无重载的用户定义存储过程 | 非也。{.text-danger} | | +| T321-03 | 函数调用 | 非也。{.text-danger} | | +| T321-04 | 电话声明 | 非也。{.text-danger} | | +| T321-05 | 退货声明 | 非也。{.text-danger} | | +| **T631** | **在一个列表元素的谓词中** | **是**{.text-success} | | diff --git a/docs/zh/sql-reference/data-types/datetime64.md b/docs/zh/sql-reference/data-types/datetime64.md index 6dc24148393..2442972965a 100644 --- a/docs/zh/sql-reference/data-types/datetime64.md +++ b/docs/zh/sql-reference/data-types/datetime64.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 49 toc_title: DateTime64 --- @@ -99,6 +99,6 @@ FROM dt - [用于处理数组的函数](../../sql-reference/functions/array-functions.md) - [该 `date_time_input_format` 设置](../../operations/settings/settings.md#settings-date_time_input_format) - [该 `timezone` 服务器配置参数](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) -- [使用日期和时间的操作员](../../sql-reference/operators.md#operators-datetime) +- [使用日期和时间的操作员](../../sql-reference/operators/index.md#operators-datetime) - [`Date` 数据类型](date.md) - [`DateTime` 数据类型](datetime.md) diff --git a/docs/zh/sql-reference/data-types/domains/index.md b/docs/zh/sql-reference/data-types/domains/index.md index 7df13d51e54..e05c61e0dbb 100644 --- a/docs/zh/sql-reference/data-types/domains/index.md +++ b/docs/zh/sql-reference/data-types/domains/index.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_folder_title: "\u57DF" toc_priority: 56 --- diff --git a/docs/zh/sql-reference/data-types/special-data-types/interval.md b/docs/zh/sql-reference/data-types/special-data-types/interval.md index b814b8d47b1..df2ce097df0 100644 --- a/docs/zh/sql-reference/data-types/special-data-types/interval.md +++ b/docs/zh/sql-reference/data-types/special-data-types/interval.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 61 toc_title: "\u95F4\u9694" --- # 间隔 {#data-type-interval} -表示时间和日期间隔的数据类型族。 由此产生的类型 [INTERVAL](../../../sql-reference/operators.md#operator-interval) 接线员 +表示时间和日期间隔的数据类型族。 由此产生的类型 [INTERVAL](../../../sql-reference/operators/index.md#operator-interval) 接线员 !!! warning "警告" `Interval` 数据类型值不能存储在表中。 @@ -81,5 +81,5 @@ Code: 43. DB::Exception: Received from localhost:9000. DB::Exception: Wrong argu ## 另请参阅 {#see-also} -- [INTERVAL](../../../sql-reference/operators.md#operator-interval) 接线员 +- [INTERVAL](../../../sql-reference/operators/index.md#operator-interval) 接线员 - [toInterval](../../../sql-reference/functions/type-conversion-functions.md#function-tointerval) 类型转换函数 diff --git a/docs/zh/sql-reference/data-types/uuid.md b/docs/zh/sql-reference/data-types/uuid.md index 6728a5bb178..2ff1e391e81 100644 --- a/docs/zh/sql-reference/data-types/uuid.md +++ b/docs/zh/sql-reference/data-types/uuid.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 46 toc_title: UUID --- diff --git a/docs/zh/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md b/docs/zh/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md index 99c4568ed8b..0b473f0c816 100644 --- a/docs/zh/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md +++ b/docs/zh/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 45 toc_title: "\u5206\u5C42\u5B57\u5178" --- diff --git a/docs/zh/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md b/docs/zh/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md index 956d6a7393e..0843a6a535a 100644 --- a/docs/zh/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md +++ b/docs/zh/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 41 toc_title: "\u5728\u5185\u5B58\u4E2D\u5B58\u50A8\u5B57\u5178" --- @@ -16,7 +16,7 @@ toc_title: "\u5728\u5185\u5B58\u4E2D\u5B58\u50A8\u5B57\u5178" 有几种方法可以提高字典性能: - 调用该函数以使用后的字典 `GROUP BY`. -- 将要提取的属性标记为“注射”。 如果不同的属性值对应于不同的键，则称为注射属性。 所以当 `GROUP BY` 使用由键获取属性值的函数，此函数会自动取出 `GROUP BY`. +- 将要提取的属性标记为"注射"。 如果不同的属性值对应于不同的键，则称为注射属性。 所以当 `GROUP BY` 使用由键获取属性值的函数，此函数会自动取出 `GROUP BY`. ClickHouse为字典中的错误生成异常。 错误示例: @@ -56,6 +56,7 @@ LAYOUT(LAYOUT_TYPE(param value)) -- layout settings - [散列](#dicts-external_dicts_dict_layout-hashed) - [sparse\_hashed](#dicts-external_dicts_dict_layout-sparse_hashed) - [缓存](#cache) +- [直接](#direct) - [range\_hashed](#range-hashed) - [complex\_key\_hashed](#complex-key-hashed) - [complex\_key\_cache](#complex-key-cache) @@ -296,6 +297,28 @@ LAYOUT(CACHE(SIZE_IN_CELLS 1000000000)) 这种类型的存储是用于复合 [键](external-dicts-dict-structure.md). 类似于 `cache`. +### 直接 {#direct} + +字典不存储在内存中，并且在处理请求期间直接转到源。 + +字典键具有 `UInt64` 类型。 + +所有类型的 [来源](external-dicts-dict-sources.md)，除了本地文件，支持。 + +配置示例: + +``` xml + + + +``` + +或 + +``` sql +LAYOUT(DIRECT()) +``` + ### ip\_trie {#ip-trie} 这种类型的存储用于将网络前缀（IP地址）映射到ASN等元数据。 diff --git a/docs/zh/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md b/docs/zh/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md index 1637d513b98..af944fd9aea 100644 --- a/docs/zh/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md +++ b/docs/zh/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 42 toc_title: "\u5B57\u5178\u66F4\u65B0" --- @@ -51,6 +51,11 @@ LIFETIME(300) LIFETIME(MIN 300 MAX 360) ``` +如果 `0` 和 `0`，ClickHouse不会按超时重新加载字典。 +在这种情况下，如果字典配置文件已更改，ClickHouse可以更早地重新加载字典 `SYSTEM RELOAD DICTIONARY` 命令被执行。 + +升级字典时，ClickHouse服务器根据字典的类型应用不同的逻辑 [来源](external-dicts-dict-sources.md): + 升级字典时，ClickHouse服务器根据字典的类型应用不同的逻辑 [来源](external-dicts-dict-sources.md): - 对于文本文件，它检查修改的时间。 如果时间与先前记录的时间不同，则更新字典。 diff --git a/docs/zh/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md b/docs/zh/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md index 47fd4d5fe42..f981f442fb6 100644 --- a/docs/zh/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md +++ b/docs/zh/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 43 toc_title: "\u5916\u90E8\u5B57\u5178\u7684\u6765\u6E90" --- @@ -37,6 +37,28 @@ SOURCE(SOURCE_TYPE(param1 val1 ... paramN valN)) -- Source configuration 源配置在 `source` 科。 +对于源类型 [本地文件](#dicts-external_dicts_dict_sources-local_file), [可执行文件](#dicts-external_dicts_dict_sources-executable), [HTTP(s)](#dicts-external_dicts_dict_sources-http), [ClickHouse](#dicts-external_dicts_dict_sources-clickhouse) +可选设置: + +``` xml + + + /opt/dictionaries/os.tsv + TabSeparated + + + 0 + + +``` + +或 + +``` sql +SOURCE(FILE(path '/opt/dictionaries/os.tsv' format 'TabSeparated')) +SETTINGS(format_csv_allow_single_quotes = 0) +``` + 来源类型 (`source_type`): - [本地文件](#dicts-external_dicts_dict_sources-local_file) diff --git a/docs/zh/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md b/docs/zh/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md index 48ae34fb92b..91497a5767a 100644 --- a/docs/zh/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md +++ b/docs/zh/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 44 toc_title: "\u5B57\u5178\u952E\u548C\u5B57\u6BB5" --- diff --git a/docs/zh/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md b/docs/zh/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md index 8cc599adce3..843b9a5aa41 100644 --- a/docs/zh/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md +++ b/docs/zh/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 40 toc_title: "\u914D\u7F6E\u5916\u90E8\u5B57\u5178" --- diff --git a/docs/zh/sql-reference/dictionaries/external-dictionaries/external-dicts.md b/docs/zh/sql-reference/dictionaries/external-dictionaries/external-dicts.md index 1fd50cf458a..c67deb55401 100644 --- a/docs/zh/sql-reference/dictionaries/external-dictionaries/external-dicts.md +++ b/docs/zh/sql-reference/dictionaries/external-dictionaries/external-dicts.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 39 toc_title: "\u6982\u8FF0" --- @@ -19,6 +19,12 @@ ClickHouse: 字典可以在服务器启动或首次使用时加载，具体取决于 [dictionaries\_lazy\_load](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-dictionaries_lazy_load) 设置。 +该 [字典](../../../operations/system-tables.md#system_tables-dictionaries) 系统表包含有关在服务器上配置的字典的信息。 对于每个字典，你可以在那里找到: + +- 字典的状态。 +- 配置参数。 +- 度量指标，如为字典分配的RAM量或自成功加载字典以来的查询数量。 + 字典配置文件具有以下格式: ``` xml diff --git a/docs/zh/sql-reference/dictionaries/external-dictionaries/index.md b/docs/zh/sql-reference/dictionaries/external-dictionaries/index.md index 25d86ecda96..8d8cc4f187d 100644 --- a/docs/zh/sql-reference/dictionaries/external-dictionaries/index.md +++ b/docs/zh/sql-reference/dictionaries/external-dictionaries/index.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_folder_title: "\u5916\u90E8\u5B57\u5178" toc_priority: 37 --- diff --git a/docs/zh/sql-reference/dictionaries/index.md b/docs/zh/sql-reference/dictionaries/index.md index 75217bf0fde..7e8f5e83aa7 100644 --- a/docs/zh/sql-reference/dictionaries/index.md +++ b/docs/zh/sql-reference/dictionaries/index.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_folder_title: "\u5B57\u5178" toc_priority: 35 toc_title: "\u5BFC\u8A00" @@ -17,6 +17,6 @@ ClickHouse支持使用可用于查询的字典的特殊功能。 这是更容易 ClickHouse支持: - [内置字典](internal-dicts.md#internal_dicts) 具有特定的 [功能集](../../sql-reference/functions/ym-dict-functions.md). -- [插件（外部）字典](external-dictionaries/external-dicts.md#dicts-external-dicts) 用一个 [职能净额](../../sql-reference/functions/ext-dict-functions.md). +- [插件（外部）字典](external-dictionaries/external-dicts.md#dicts-external-dicts) 用一个 [功能集](../../sql-reference/functions/ext-dict-functions.md). [原始文章](https://clickhouse.tech/docs/en/query_language/dicts/) diff --git a/docs/zh/sql-reference/dictionaries/internal-dicts.md b/docs/zh/sql-reference/dictionaries/internal-dicts.md index b5161e31611..706288e30ec 100644 --- a/docs/zh/sql-reference/dictionaries/internal-dicts.md +++ b/docs/zh/sql-reference/dictionaries/internal-dicts.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 39 toc_title: "\u5185\u90E8\u5B57\u5178" --- @@ -39,7 +39,7 @@ Geobase从文本文件加载。 `regions_names_*.txt`：TabSeparated（无标题），列: - 地区ID (`UInt32`) -- 地区名称 (`String`) — Can’t contain tabs or line feeds, even escaped ones. +- 地区名称 (`String`) — Can't contain tabs or line feeds, even escaped ones. 平面阵列用于存储在RAM中。 出于这个原因，Id不应该超过一百万。 diff --git a/docs/zh/sql-reference/functions/conditional-functions.md b/docs/zh/sql-reference/functions/conditional-functions.md index 05a26935f01..eabe253ab1c 100644 --- a/docs/zh/sql-reference/functions/conditional-functions.md +++ b/docs/zh/sql-reference/functions/conditional-functions.md @@ -9,7 +9,7 @@ ## 多 {#multiif} -允许您在查询中更紧凑地编写[CASE](../operators.md#operator_case)运算符。 +允许您在查询中更紧凑地编写[CASE](../operators/index.md#operator_case)运算符。 multiIf(cond_1, then_1, cond_2, then_2...else) diff --git a/docs/zh/sql-reference/functions/in-functions.md b/docs/zh/sql-reference/functions/in-functions.md index 9f911cef06a..aeb46025c36 100644 --- a/docs/zh/sql-reference/functions/in-functions.md +++ b/docs/zh/sql-reference/functions/in-functions.md @@ -2,7 +2,7 @@ ## in,notIn,globalIn,globalNotIn {#in-notin-globalin-globalnotin} -请参阅[IN 运算符](../statements/select.md#select-in-operators)部分。 +请参阅[IN 运算符](../operators/in.md#select-in-operators)部分。 ## tuple(x, y, …), operator (x, y, …) {#tuplex-y-operator-x-y} diff --git a/docs/zh/sql-reference/functions/introspection.md b/docs/zh/sql-reference/functions/introspection.md index 8ea252a1e63..43d8b596dfb 100644 --- a/docs/zh/sql-reference/functions/introspection.md +++ b/docs/zh/sql-reference/functions/introspection.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 65 toc_title: "\u81EA\u7701" --- diff --git a/docs/zh/sql-reference/index.md b/docs/zh/sql-reference/index.md index 59662c886b5..aed96c4b34f 100644 --- a/docs/zh/sql-reference/index.md +++ b/docs/zh/sql-reference/index.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_folder_title: "SQL\u53C2\u8003" toc_hidden: true toc_priority: 28 @@ -9,10 +9,12 @@ toc_title: "\u9690\u85CF" # SQL参考 {#sql-reference} -- [SELECT](statements/select.md) +ClickHouse支持以下类型的查询: + +- [SELECT](statements/select/index.md) - [INSERT INTO](statements/insert-into.md) - [CREATE](statements/create.md) - [ALTER](statements/alter.md#query_language_queries_alter) - [其他类型的查询](statements/misc.md) -[原始文章](https://clickhouse.tech/docs/en/query_language/) +[原始文章](https://clickhouse.tech/docs/en/sql-reference/) diff --git a/docs/zh/sql-reference/operators/in.md b/docs/zh/sql-reference/operators/in.md new file mode 120000 index 00000000000..3a2feda2f61 --- /dev/null +++ b/docs/zh/sql-reference/operators/in.md @@ -0,0 +1 @@ +../../../en/sql-reference/operators/in.md \ No newline at end of file diff --git a/docs/zh/sql-reference/operators.md b/docs/zh/sql-reference/operators/index.md similarity index 87% rename from docs/zh/sql-reference/operators.md rename to docs/zh/sql-reference/operators/index.md index af3608f73af..77aa42755af 100644 --- a/docs/zh/sql-reference/operators.md +++ b/docs/zh/sql-reference/operators/index.md @@ -52,7 +52,7 @@ ## 集合关系运算符 {#ji-he-guan-xi-yun-suan-fu} -*详见此节 [IN 相关操作符](statements/select.md#select-in-operators) 。* +*详见此节 [IN 相关操作符](in.md#select-in-operators) 。* `a IN ...` – 对应函数 `in(a, b)` @@ -80,7 +80,7 @@ 注意: -条件运算符会先计算表达式b和表达式c的值，再根据表达式a的真假，返回相应的值。如果表达式b和表达式c是 [arrayJoin()](../sql-reference/functions/array-join.md#functions_arrayjoin) 函数，则不管表达式a是真是假，每行都会被复制展开。 +条件运算符会先计算表达式b和表达式c的值，再根据表达式a的真假，返回相应的值。如果表达式b和表达式c是 [arrayJoin()](../../sql-reference/functions/array-join.md#functions_arrayjoin) 函数，则不管表达式a是真是假，每行都会被复制展开。 ## 使用日期和时间的操作员 {#operators-datetime} @@ -103,7 +103,7 @@ EXTRACT(part FROM date); 该 `part` 参数不区分大小写。 -该 `date` 参数指定要处理的日期或时间。 无论是 [日期](../sql-reference/data-types/date.md) 或 [日期时间](../sql-reference/data-types/datetime.md) 支持类型。 +该 `date` 参数指定要处理的日期或时间。 无论是 [日期](../../sql-reference/data-types/date.md) 或 [日期时间](../../sql-reference/data-types/datetime.md) 支持类型。 例: @@ -150,7 +150,7 @@ FROM test.Orders; ### INTERVAL {#operator-interval} -创建一个 [间隔](../sql-reference/operators.md)-应在算术运算中使用的类型值 [日期](../sql-reference/data-types/date.md) 和 [日期时间](../sql-reference/data-types/datetime.md)-类型值。 +创建一个 [间隔](../../sql-reference/operators/index.md)-应在算术运算中使用的类型值 [日期](../../sql-reference/data-types/date.md) 和 [日期时间](../../sql-reference/data-types/datetime.md)-类型值。 示例: @@ -166,8 +166,8 @@ SELECT now() AS current_date_time, current_date_time + INTERVAL 4 DAY + INTERVAL **另请参阅** -- [间隔](../sql-reference/operators.md) 数据类型 -- [toInterval](../sql-reference/operators.md#function-tointerval) 类型转换函数 +- [间隔](../../sql-reference/operators/index.md) 数据类型 +- [toInterval](../../sql-reference/operators/index.md#function-tointerval) 类型转换函数 ## CASE条件表达式 {#operator_case} @@ -216,7 +216,7 @@ ClickHouse 支持 `IS NULL` 和 `IS NOT NULL` 。 ### IS NULL {#operator-is-null} -- 对于 [可为空](../sql-reference/operators.md) 类型的值， `IS NULL` 会返回： +- 对于 [可为空](../../sql-reference/operators/index.md) 类型的值， `IS NULL` 会返回： - `1` 值为 `NULL` - `0` 否则 - 对于其他类型的值， `IS NULL` 总会返回 `0` @@ -239,7 +239,7 @@ WHERE isNull(y) ### IS NOT NULL {#is-not-null} -- 对于 [可为空](../sql-reference/operators.md) 类型的值， `IS NOT NULL` 会返回： +- 对于 [可为空](../../sql-reference/operators/index.md) 类型的值， `IS NOT NULL` 会返回： - `0` 值为 `NULL` - `1` 否则 - 对于其他类型的值，`IS NOT NULL` 总会返回 `1` diff --git a/docs/zh/sql-reference/statements/alter.md b/docs/zh/sql-reference/statements/alter.md index ff4a0e0d771..24ca1e47372 100644 --- a/docs/zh/sql-reference/statements/alter.md +++ b/docs/zh/sql-reference/statements/alter.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 36 toc_title: ALTER --- @@ -26,7 +26,7 @@ ALTER TABLE [db].name [ON CLUSTER cluster] ADD|DROP|CLEAR|COMMENT|MODIFY COLUMN - [DROP COLUMN](#alter_drop-column) — Deletes the column. - [CLEAR COLUMN](#alter_clear-column) — Resets column values. - [COMMENT COLUMN](#alter_comment-column) — Adds a text comment to the column. -- [MODIFY COLUMN](#alter_modify-column) — Changes column’s type, default expression and TTL. +- [MODIFY COLUMN](#alter_modify-column) — Changes column's type, default expression and TTL. 下面详细描述这些动作。 @@ -38,7 +38,7 @@ ADD COLUMN [IF NOT EXISTS] name [type] [default_expr] [codec] [AFTER name_after] 将一个新列添加到表中，并指定 `name`, `type`, [`codec`](create.md#codecs) 和 `default_expr` （请参阅部分 [默认表达式](create.md#create-default-values)). -如果 `IF NOT EXISTS` 如果列已经存在，则查询不会返回错误。 如果您指定 `AFTER name_after` （另一列的名称），该列被添加在表列表中指定的一列之后。 否则，该列将添加到表的末尾。 请注意，没有办法将列添加到表的开头。 为了一系列的行动, `name_after` 可将该名称一栏，加入一个以前的行动。 +如果 `IF NOT EXISTS` 如果列已经存在，则查询不会返回错误。 如果您指定 `AFTER name_after` （另一列的名称），该列被添加在表列表中指定的一列之后。 否则，该列将添加到表的末尾。 请注意，没有办法将列添加到表的开头。 为了一系列的行动, `name_after` 可以是在以前的操作之一中添加的列的名称。 添加列只是更改表结构，而不对数据执行任何操作。 数据不会出现在磁盘上后 `ALTER`. 如果从表中读取某一列的数据缺失，则将使用默认值填充该列（如果存在默认表达式，则执行默认表达式，或使用零或空字符串）。 合并数据部分后，该列将出现在磁盘上（请参阅 [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md)). @@ -206,10 +206,9 @@ ALTER TABLE [db].name DROP CONSTRAINT constraint_name; - [DETACH PARTITION](#alter_detach-partition) – Moves a partition to the `detached` 目录和忘记它。 - [DROP PARTITION](#alter_drop-partition) – Deletes a partition. - [ATTACH PART\|PARTITION](#alter_attach-partition) – Adds a part or partition from the `detached` 目录到表。 -- [REPLACE PARTITION](#alter_replace-partition) -将数据分区从一个表复制到另一个表。 - [ATTACH PARTITION FROM](#alter_attach-partition-from) – Copies the data partition from one table to another and adds. - [REPLACE PARTITION](#alter_replace-partition) -将数据分区从一个表复制到另一个表并替换。 -- [MOVE PARTITION TO TABLE](#alter_move_to_table-partition) (\#alter\_move\_to\_table-partition)-将数据分区从一个表移动到另一个表。 +- [MOVE PARTITION TO TABLE](#alter_move_to_table-partition)(\#alter\_move\_to\_table-partition)-将数据分区从一个表移动到另一个表。 - [CLEAR COLUMN IN PARTITION](#alter_clear-column-partition) -重置分区中指定列的值。 - [CLEAR INDEX IN PARTITION](#alter_clear-index-partition) -重置分区中指定的二级索引。 - [FREEZE PARTITION](#alter_freeze-partition) – Creates a backup of a partition. @@ -502,4 +501,102 @@ Mutation查询在添加mutation条目后立即返回（如果将复制的表复 已完成突变的条目不会立即删除（保留条目的数量由 `finished_mutations_to_keep` 存储引擎参数）。 旧的突变条目将被删除。 +## ALTER USER {#alter-user-statement} + +更改ClickHouse用户帐户. + +### 语法 {#alter-user-syntax} + +``` sql +ALTER USER [IF EXISTS] name [ON CLUSTER cluster_name] + [RENAME TO new_name] + [IDENTIFIED [WITH {PLAINTEXT_PASSWORD|SHA256_PASSWORD|DOUBLE_SHA1_PASSWORD}] BY {'password'|'hash'}] + [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] + [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] +``` + +### 产品描述 {#alter-user-dscr} + +使用 `ALTER USER` 你必须有 [ALTER USER](grant.md#grant-access-management) 特权 + +### 例 {#alter-user-examples} + +将授予的角色设置为默认值: + +``` sql +ALTER USER user DEFAULT ROLE role1, role2 +``` + +如果以前未向用户授予角色，ClickHouse将引发异常。 + +将所有授予的角色设置为默认值: + +``` sql +ALTER USER user DEFAULT ROLE ALL +``` + +如果将来将某个角色授予某个用户，它将自动成为默认值。 + +将所有授予的角色设置为默认值，除非 `role1` 和 `role2`: + +``` sql +ALTER USER user DEFAULT ROLE ALL EXCEPT role1, role2 +``` + +## ALTER ROLE {#alter-role-statement} + +更改角色。 + +### 语法 {#alter-role-syntax} + +``` sql +ALTER ROLE [IF EXISTS] name [ON CLUSTER cluster_name] + [RENAME TO new_name] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] +``` + +## ALTER ROW POLICY {#alter-row-policy-statement} + +更改行策略。 + +### 语法 {#alter-row-policy-syntax} + +``` sql +ALTER [ROW] POLICY [IF EXISTS] name [ON CLUSTER cluster_name] ON [database.]table + [RENAME TO new_name] + [AS {PERMISSIVE | RESTRICTIVE}] + [FOR SELECT] + [USING {condition | NONE}][,...] + [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] +``` + +## ALTER QUOTA {#alter-quota-statement} + +更改配额。 + +### 语法 {#alter-quota-syntax} + +``` sql +ALTER QUOTA [IF EXISTS] name [ON CLUSTER cluster_name] + [RENAME TO new_name] + [KEYED BY {'none' | 'user name' | 'ip address' | 'client key' | 'client key or user name' | 'client key or ip address'}] + [FOR [RANDOMIZED] INTERVAL number {SECOND | MINUTE | HOUR | DAY} + {MAX { {QUERIES | ERRORS | RESULT ROWS | RESULT BYTES | READ ROWS | READ BYTES | EXECUTION TIME} = number } [,...] | + NO LIMITS | TRACKING ONLY} [,...]] + [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] +``` + +## ALTER SETTINGS PROFILE {#alter-settings-profile-statement} + +更改配额。 + +### 语法 {#alter-settings-profile-syntax} + +``` sql +ALTER SETTINGS PROFILE [IF EXISTS] name [ON CLUSTER cluster_name] + [RENAME TO new_name] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...] +``` + [原始文章](https://clickhouse.tech/docs/en/query_language/alter/) diff --git a/docs/zh/sql-reference/statements/grant.md b/docs/zh/sql-reference/statements/grant.md new file mode 120000 index 00000000000..f2acbe125b4 --- /dev/null +++ b/docs/zh/sql-reference/statements/grant.md @@ -0,0 +1 @@ +../../../en/sql-reference/statements/grant.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/index.md b/docs/zh/sql-reference/statements/index.md index bb04551dea1..1c5f4e9a7ef 100644 --- a/docs/zh/sql-reference/statements/index.md +++ b/docs/zh/sql-reference/statements/index.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_folder_title: "\u53D1\u8A00" toc_priority: 31 --- diff --git a/docs/zh/sql-reference/statements/misc.md b/docs/zh/sql-reference/statements/misc.md index 91812489e58..aa10350280f 100644 --- a/docs/zh/sql-reference/statements/misc.md +++ b/docs/zh/sql-reference/statements/misc.md @@ -1,7 +1,7 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 -toc_priority: 39 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_priority: 41 toc_title: "\u5176\u4ED6" --- @@ -113,7 +113,65 @@ DROP [TEMPORARY] TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] 删除字典。 如果 `IF EXISTS` 如果表不存在或数据库不存在，则不会返回错误。 -## EXISTS {#exists} +## DROP USER {#drop-user-statement} + +删除用户。 + +### 语法 {#drop-user-syntax} + +``` sql +DROP USER [IF EXISTS] name [,...] [ON CLUSTER cluster_name] +``` + +## DROP ROLE {#drop-role-statement} + +删除角色。 + +已删除的角色将从授予该角色的所有实体撤销。 + +### 语法 {#drop-role-syntax} + +``` sql +DROP ROLE [IF EXISTS] name [,...] [ON CLUSTER cluster_name] +``` + +## DROP ROW POLICY {#drop-row-policy-statement} + +删除行策略。 + +已删除行策略将从分配该策略的所有实体撤销。 + +### 语法 {#drop-row-policy-syntax} + +``` sql +DROP [ROW] POLICY [IF EXISTS] name [,...] ON [database.]table [,...] [ON CLUSTER cluster_name] +``` + +## DROP QUOTA {#drop-quota-statement} + +删除配额。 + +已删除的配额将从分配配额的所有实体撤销。 + +### 语法 {#drop-quota-syntax} + +``` sql +DROP QUOTA [IF EXISTS] name [,...] [ON CLUSTER cluster_name] +``` + +## DROP SETTINGS PROFILE {#drop-settings-profile-statement} + +删除配额。 + +已删除的配额将从分配配额的所有实体撤销。 + +### 语法 {#drop-settings-profile-syntax} + +``` sql +DROP [SETTINGS] PROFILE [IF EXISTS] name [,...] [ON CLUSTER cluster_name] +``` + +## EXISTS {#exists-statement} ``` sql EXISTS [TEMPORARY] [TABLE|DICTIONARY] [db.]name [INTO OUTFILE filename] [FORMAT format] @@ -121,7 +179,7 @@ EXISTS [TEMPORARY] [TABLE|DICTIONARY] [db.]name [INTO OUTFILE filename] [FORMAT 返回单 `UInt8`-type column，其中包含单个值 `0` 如果表或数据库不存在，或 `1` 如果该表存在于指定的数据库中。 -## KILL QUERY {#kill-query} +## KILL QUERY {#kill-query-statement} ``` sql KILL QUERY [ON CLUSTER cluster] @@ -152,7 +210,7 @@ KILL QUERY WHERE user='username' SYNC 1. ‘finished’ – The query was terminated successfully. 2. ‘waiting’ – Waiting for the query to end after sending it a signal to terminate. -3. The other values ​​explain why the query can’t be stopped. +3. The other values ​​explain why the query can't be stopped. 测试查询 (`TEST`）仅检查用户的权限并显示要停止的查询列表。 @@ -229,7 +287,55 @@ SET profile = 'profile-name-from-the-settings-file' 有关详细信息，请参阅 [设置](../../operations/settings/settings.md). -## TRUNCATE {#truncate} +## SET ROLE {#set-role-statement} + +激活当前用户的角色。 + +### 语法 {#set-role-syntax} + +``` sql +SET ROLE {DEFAULT | NONE | role [,...] | ALL | ALL EXCEPT role [,...]} +``` + +## SET DEFAULT ROLE {#set-default-role-statement} + +将默认角色设置为用户。 + +默认角色在用户登录时自动激活。 您只能将以前授予的角色设置为默认值。 如果未向用户授予角色，ClickHouse将引发异常。 + +### 语法 {#set-default-role-syntax} + +``` sql +SET DEFAULT ROLE {NONE | role [,...] | ALL | ALL EXCEPT role [,...]} TO {user|CURRENT_USER} [,...] +``` + +### 例 {#set-default-role-examples} + +为用户设置多个默认角色: + +``` sql +SET DEFAULT ROLE role1, role2, ... TO user +``` + +将所有授予的角色设置为用户的默认值: + +``` sql +SET DEFAULT ROLE ALL TO user +``` + +从用户清除默认角色: + +``` sql +SET DEFAULT ROLE NONE TO user +``` + +将所有授予的角色设置为默认角色，其中一些角色除外: + +``` sql +SET DEFAULT ROLE ALL EXCEPT role1, role2 TO user +``` + +## TRUNCATE {#truncate-statement} ``` sql TRUNCATE TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] diff --git a/docs/zh/sql-reference/statements/revoke.md b/docs/zh/sql-reference/statements/revoke.md new file mode 120000 index 00000000000..4321fdb14a7 --- /dev/null +++ b/docs/zh/sql-reference/statements/revoke.md @@ -0,0 +1 @@ +../../../en/sql-reference/statements/revoke.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/select.md b/docs/zh/sql-reference/statements/select.md deleted file mode 100644 index 4289fd34164..00000000000 --- a/docs/zh/sql-reference/statements/select.md +++ /dev/null @@ -1,1379 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 0f7ef7704d018700049223525bad4a63911b6e70 -toc_priority: 33 -toc_title: SELECT ---- - -# 选择查询语法 {#select-queries-syntax} - -`SELECT` 执行数据检索。 - -``` sql -[WITH expr_list|(subquery)] -SELECT [DISTINCT] expr_list -[FROM [db.]table | (subquery) | table_function] [FINAL] -[SAMPLE sample_coeff] -[ARRAY JOIN ...] -[GLOBAL] [ANY|ALL] [INNER|LEFT|RIGHT|FULL|CROSS] [OUTER] JOIN (subquery)|table USING columns_list -[PREWHERE expr] -[WHERE expr] -[GROUP BY expr_list] [WITH TOTALS] -[HAVING expr] -[ORDER BY expr_list] -[LIMIT [offset_value, ]n BY columns] -[LIMIT [n, ]m] -[UNION ALL ...] -[INTO OUTFILE filename] -[FORMAT format] -``` - -所有子句都是可选的，除了紧接在SELECT之后的必需表达式列表。 -以下子句的描述顺序几乎与查询执行传送器中的顺序相同。 - -如果查询省略 `DISTINCT`, `GROUP BY` 和 `ORDER BY` 条款和 `IN` 和 `JOIN` 子查询，查询将被完全流处理，使用O(1)量的RAM。 -否则，如果未指定适当的限制，则查询可能会消耗大量RAM: `max_memory_usage`, `max_rows_to_group_by`, `max_rows_to_sort`, `max_rows_in_distinct`, `max_bytes_in_distinct`, `max_rows_in_set`, `max_bytes_in_set`, `max_rows_in_join`, `max_bytes_in_join`, `max_bytes_before_external_sort`, `max_bytes_before_external_group_by`. 有关详细信息，请参阅部分 “Settings”. 可以使用外部排序（将临时表保存到磁盘）和外部聚合。 `The system does not have "merge join"`. - -### WITH条款 {#with-clause} - -本节提供对公共表表达式的支持 ([CTE](https://en.wikipedia.org/wiki/Hierarchical_and_recursive_queries_in_SQL)），有一些限制: -1. 不支持递归查询 -2. 当在section中使用子查询时，它的结果应该是只有一行的标量 -3. Expression的结果在子查询中不可用 -WITH子句表达式的结果可以在SELECT子句中使用。 - -示例1：使用常量表达式作为 “variable” - -``` sql -WITH '2019-08-01 15:23:00' as ts_upper_bound -SELECT * -FROM hits -WHERE - EventDate = toDate(ts_upper_bound) AND - EventTime <= ts_upper_bound -``` - -示例2：从SELECT子句列表中逐出sum(bytes)表达式结果 - -``` sql -WITH sum(bytes) as s -SELECT - formatReadableSize(s), - table -FROM system.parts -GROUP BY table -ORDER BY s -``` - -示例3：使用标量子查询的结果 - -``` sql -/* this example would return TOP 10 of most huge tables */ -WITH - ( - SELECT sum(bytes) - FROM system.parts - WHERE active - ) AS total_disk_usage -SELECT - (sum(bytes) / total_disk_usage) * 100 AS table_disk_usage, - table -FROM system.parts -GROUP BY table -ORDER BY table_disk_usage DESC -LIMIT 10 -``` - -示例4：在子查询中重用表达式 -作为子查询中表达式使用的当前限制的解决方法，您可以复制它。 - -``` sql -WITH ['hello'] AS hello -SELECT - hello, - * -FROM -( - WITH ['hello'] AS hello - SELECT hello -) -``` - -``` text -┌─hello─────┬─hello─────┐ -│ ['hello'] │ ['hello'] │ -└───────────┴───────────┘ -``` - -### FROM条款 {#select-from} - -如果FROM子句被省略，数据将从读取 `system.one` 桌子 -该 `system.one` 表只包含一行（此表满足与其他Dbms中找到的双表相同的目的）。 - -该 `FROM` 子句指定从中读取数据的源: - -- 表 -- 子查询 -- [表函数](../table-functions/index.md#table-functions) - -`ARRAY JOIN` 和常规 `JOIN` 也可以包括在内（见下文）。 - -而不是一个表，该 `SELECT` 子查询可以在括号中指定。 -与标准SQL相比，不需要在子查询后指定同义词。 - -若要执行查询，将从相应的表中提取查询中列出的所有列。 外部查询不需要的任何列都将从子查询中抛出。 -如果查询未列出任何列（例如, `SELECT count() FROM t`），无论如何都会从表中提取一些列（最小的列是首选），以便计算行数。 - -#### 最终修饰符 {#select-from-final} - -从表中选择数据时适用 [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md)-发动机系列比其他 `GraphiteMergeTree`. 当 `FINAL` 如果指定，ClickHouse会在返回结果之前完全合并数据，从而执行给定表引擎合并期间发生的所有数据转换。 - -还支持: -- [复制](../../engines/table-engines/mergetree-family/replication.md) 版本 `MergeTree` 引擎 -- [查看](../../engines/table-engines/special/view.md), [缓冲区](../../engines/table-engines/special/buffer.md), [分布](../../engines/table-engines/special/distributed.md)，和 [MaterializedView](../../engines/table-engines/special/materializedview.md) 在其他引擎上运行的引擎，只要它们是在创建 `MergeTree`-发动机表。 - -使用的查询 `FINAL` 执行速度不如类似的查询那么快，因为: - -- 查询在单个线程中执行，并在查询执行期间合并数据。 -- 查询与 `FINAL` 除了读取查询中指定的列之外，还读取主键列。 - -在大多数情况下，避免使用 `FINAL`. - -### 示例子句 {#select-sample-clause} - -该 `SAMPLE` 子句允许近似查询处理。 - -启用数据采样时，不会对所有数据执行查询，而只对特定部分数据（样本）执行查询。 例如，如果您需要计算所有访问的统计信息，只需对所有访问的1/10分数执行查询，然后将结果乘以10即可。 - -近似查询处理在以下情况下可能很有用: - -- 当你有严格的时间requirements（如\<100ms），但你不能证明额外的硬件资源来满足他们的成本。 -- 当您的原始数据不准确时，所以近似不会明显降低质量。 -- 业务需求的目标是近似结果（为了成本效益，或者为了向高级用户推销确切的结果）。 - -!!! note "注" - 您只能使用采样中的表 [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) 家庭，并且只有在表创建过程中指定了采样表达式（请参阅 [MergeTree引擎](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table)). - -下面列出了数据采样的功能: - -- 数据采样是一种确定性机制。 同样的结果 `SELECT .. SAMPLE` 查询始终是相同的。 -- 对于不同的表，采样工作始终如一。 对于具有单个采样键的表，具有相同系数的采样总是选择相同的可能数据子集。 例如，用户Id的示例采用来自不同表的所有可能的用户Id的相同子集的行。 这意味着您可以在子查询中使用示例 [IN](#select-in-operators) 条款 此外，您可以使用 [JOIN](#select-join) 条款 -- 采样允许从磁盘读取更少的数据。 请注意，您必须正确指定采样键。 有关详细信息，请参阅 [创建MergeTree表](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table). - -为 `SAMPLE` 子句支持以下语法: - -| SAMPLE Clause Syntax | 产品描述 | -|----------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `SAMPLE k` | 这里 `k` 是从0到1的数字。
查询执行于 `k` 数据的分数。 例如, `SAMPLE 0.1` 对10%的数据运行查询。 [碌莽禄more拢more](#select-sample-k) | -| `SAMPLE n` | 这里 `n` 是足够大的整数。
该查询是在至少一个样本上执行的 `n` 行（但不超过这个）。 例如, `SAMPLE 10000000` 在至少10,000,000行上运行查询。 [碌莽禄more拢more](#select-sample-n) | -| `SAMPLE k OFFSET m` | 这里 `k` 和 `m` 是从0到1的数字。
查询在以下示例上执行 `k` 数据的分数。 用于采样的数据由以下偏移 `m` 分数。 [碌莽禄more拢more](#select-sample-offset) | - -#### SAMPLE K {#select-sample-k} - -这里 `k` 从0到1的数字（支持小数和小数表示法）。 例如, `SAMPLE 1/2` 或 `SAMPLE 0.5`. - -在一个 `SAMPLE k` 子句，样品是从 `k` 数据的分数。 示例如下所示: - -``` sql -SELECT - Title, - count() * 10 AS PageViews -FROM hits_distributed -SAMPLE 0.1 -WHERE - CounterID = 34 -GROUP BY Title -ORDER BY PageViews DESC LIMIT 1000 -``` - -在此示例中，对0.1(10%)数据的样本执行查询。 聚合函数的值不会自动修正，因此要获得近似结果，值 `count()` 手动乘以10。 - -#### SAMPLE N {#select-sample-n} - -这里 `n` 是足够大的整数。 例如, `SAMPLE 10000000`. - -在这种情况下，查询在至少一个样本上执行 `n` 行（但不超过这个）。 例如, `SAMPLE 10000000` 在至少10,000,000行上运行查询。 - -由于数据读取的最小单位是一个颗粒（其大小由 `index_granularity` 设置），是有意义的设置一个样品，其大小远大于颗粒。 - -使用时 `SAMPLE n` 子句，你不知道处理了哪些数据的相对百分比。 所以你不知道聚合函数应该乘以的系数。 使用 `_sample_factor` 虚拟列得到近似结果。 - -该 `_sample_factor` 列包含动态计算的相对系数。 当您执行以下操作时，将自动创建此列 [创建](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table) 具有指定采样键的表。 的使用示例 `_sample_factor` 列如下所示。 - -让我们考虑表 `visits`，其中包含有关网站访问的统计信息。 第一个示例演示如何计算页面浏览量: - -``` sql -SELECT sum(PageViews * _sample_factor) -FROM visits -SAMPLE 10000000 -``` - -下一个示例演示如何计算访问总数: - -``` sql -SELECT sum(_sample_factor) -FROM visits -SAMPLE 10000000 -``` - -下面的示例显示了如何计算平均会话持续时间。 请注意，您不需要使用相对系数来计算平均值。 - -``` sql -SELECT avg(Duration) -FROM visits -SAMPLE 10000000 -``` - -#### SAMPLE K OFFSET M {#select-sample-offset} - -这里 `k` 和 `m` 是从0到1的数字。 示例如下所示。 - -**示例1** - -``` sql -SAMPLE 1/10 -``` - -在此示例中，示例是所有数据的十分之一: - -`[++------------]` - -**示例2** - -``` sql -SAMPLE 1/10 OFFSET 1/2 -``` - -这里，从数据的后半部分取出10％的样本。 - -`[------++------]` - -### ARRAY JOIN子句 {#select-array-join-clause} - -允许执行 `JOIN` 具有数组或嵌套数据结构。 意图类似于 [arrayJoin](../functions/array-join.md#functions_arrayjoin) 功能，但其功能更广泛。 - -``` sql -SELECT -FROM -[LEFT] ARRAY JOIN -[WHERE|PREWHERE ] -... -``` - -您只能指定一个 `ARRAY JOIN` 查询中的子句。 - -运行时优化查询执行顺序 `ARRAY JOIN`. 虽然 `ARRAY JOIN` 必须始终之前指定 `WHERE/PREWHERE` 子句，它可以执行之前 `WHERE/PREWHERE` （如果结果是需要在这个子句），或完成后（以减少计算量）。 处理顺序由查询优化器控制。 - -支持的类型 `ARRAY JOIN` 下面列出: - -- `ARRAY JOIN` -在这种情况下，空数组不包括在结果中 `JOIN`. -- `LEFT ARRAY JOIN` -的结果 `JOIN` 包含具有空数组的行。 空数组的值设置为数组元素类型的默认值（通常为0、空字符串或NULL）。 - -下面的例子演示的用法 `ARRAY JOIN` 和 `LEFT ARRAY JOIN` 条款 让我们创建一个表 [阵列](../../sql-reference/data-types/array.md) 键入column并在其中插入值: - -``` sql -CREATE TABLE arrays_test -( - s String, - arr Array(UInt8) -) ENGINE = Memory; - -INSERT INTO arrays_test -VALUES ('Hello', [1,2]), ('World', [3,4,5]), ('Goodbye', []); -``` - -``` text -┌─s───────────┬─arr─────┐ -│ Hello │ [1,2] │ -│ World │ [3,4,5] │ -│ Goodbye │ [] │ -└─────────────┴─────────┘ -``` - -下面的例子使用 `ARRAY JOIN` 条款: - -``` sql -SELECT s, arr -FROM arrays_test -ARRAY JOIN arr; -``` - -``` text -┌─s─────┬─arr─┐ -│ Hello │ 1 │ -│ Hello │ 2 │ -│ World │ 3 │ -│ World │ 4 │ -│ World │ 5 │ -└───────┴─────┘ -``` - -下一个示例使用 `LEFT ARRAY JOIN` 条款: - -``` sql -SELECT s, arr -FROM arrays_test -LEFT ARRAY JOIN arr; -``` - -``` text -┌─s───────────┬─arr─┐ -│ Hello │ 1 │ -│ Hello │ 2 │ -│ World │ 3 │ -│ World │ 4 │ -│ World │ 5 │ -│ Goodbye │ 0 │ -└─────────────┴─────┘ -``` - -#### 使用别名 {#using-aliases} - -可以为数组中的别名指定 `ARRAY JOIN` 条款 在这种情况下，数组项目可以通过此别名访问，但数组本身可以通过原始名称访问。 示例: - -``` sql -SELECT s, arr, a -FROM arrays_test -ARRAY JOIN arr AS a; -``` - -``` text -┌─s─────┬─arr─────┬─a─┐ -│ Hello │ [1,2] │ 1 │ -│ Hello │ [1,2] │ 2 │ -│ World │ [3,4,5] │ 3 │ -│ World │ [3,4,5] │ 4 │ -│ World │ [3,4,5] │ 5 │ -└───────┴─────────┴───┘ -``` - -使用别名，您可以执行 `ARRAY JOIN` 与外部阵列。 例如: - -``` sql -SELECT s, arr_external -FROM arrays_test -ARRAY JOIN [1, 2, 3] AS arr_external; -``` - -``` text -┌─s───────────┬─arr_external─┐ -│ Hello │ 1 │ -│ Hello │ 2 │ -│ Hello │ 3 │ -│ World │ 1 │ -│ World │ 2 │ -│ World │ 3 │ -│ Goodbye │ 1 │ -│ Goodbye │ 2 │ -│ Goodbye │ 3 │ -└─────────────┴──────────────┘ -``` - -多个数组可以在逗号分隔 `ARRAY JOIN` 条款 在这种情况下, `JOIN` 与它们同时执行（直接和，而不是笛卡尔积）。 请注意，所有数组必须具有相同的大小。 示例: - -``` sql -SELECT s, arr, a, num, mapped -FROM arrays_test -ARRAY JOIN arr AS a, arrayEnumerate(arr) AS num, arrayMap(x -> x + 1, arr) AS mapped; -``` - -``` text -┌─s─────┬─arr─────┬─a─┬─num─┬─mapped─┐ -│ Hello │ [1,2] │ 1 │ 1 │ 2 │ -│ Hello │ [1,2] │ 2 │ 2 │ 3 │ -│ World │ [3,4,5] │ 3 │ 1 │ 4 │ -│ World │ [3,4,5] │ 4 │ 2 │ 5 │ -│ World │ [3,4,5] │ 5 │ 3 │ 6 │ -└───────┴─────────┴───┴─────┴────────┘ -``` - -下面的例子使用 [arrayEnumerate](../../sql-reference/functions/array-functions.md#array_functions-arrayenumerate) 功能: - -``` sql -SELECT s, arr, a, num, arrayEnumerate(arr) -FROM arrays_test -ARRAY JOIN arr AS a, arrayEnumerate(arr) AS num; -``` - -``` text -┌─s─────┬─arr─────┬─a─┬─num─┬─arrayEnumerate(arr)─┐ -│ Hello │ [1,2] │ 1 │ 1 │ [1,2] │ -│ Hello │ [1,2] │ 2 │ 2 │ [1,2] │ -│ World │ [3,4,5] │ 3 │ 1 │ [1,2,3] │ -│ World │ [3,4,5] │ 4 │ 2 │ [1,2,3] │ -│ World │ [3,4,5] │ 5 │ 3 │ [1,2,3] │ -└───────┴─────────┴───┴─────┴─────────────────────┘ -``` - -#### 具有嵌套数据结构的数组连接 {#array-join-with-nested-data-structure} - -`ARRAY`加入"也适用于 [嵌套数据结构](../../sql-reference/data-types/nested-data-structures/nested.md). 示例: - -``` sql -CREATE TABLE nested_test -( - s String, - nest Nested( - x UInt8, - y UInt32) -) ENGINE = Memory; - -INSERT INTO nested_test -VALUES ('Hello', [1,2], [10,20]), ('World', [3,4,5], [30,40,50]), ('Goodbye', [], []); -``` - -``` text -┌─s───────┬─nest.x──┬─nest.y─────┐ -│ Hello │ [1,2] │ [10,20] │ -│ World │ [3,4,5] │ [30,40,50] │ -│ Goodbye │ [] │ [] │ -└─────────┴─────────┴────────────┘ -``` - -``` sql -SELECT s, `nest.x`, `nest.y` -FROM nested_test -ARRAY JOIN nest; -``` - -``` text -┌─s─────┬─nest.x─┬─nest.y─┐ -│ Hello │ 1 │ 10 │ -│ Hello │ 2 │ 20 │ -│ World │ 3 │ 30 │ -│ World │ 4 │ 40 │ -│ World │ 5 │ 50 │ -└───────┴────────┴────────┘ -``` - -当指定嵌套数据结构的名称 `ARRAY JOIN`，意思是一样的 `ARRAY JOIN` 它包含的所有数组元素。 下面列出了示例: - -``` sql -SELECT s, `nest.x`, `nest.y` -FROM nested_test -ARRAY JOIN `nest.x`, `nest.y`; -``` - -``` text -┌─s─────┬─nest.x─┬─nest.y─┐ -│ Hello │ 1 │ 10 │ -│ Hello │ 2 │ 20 │ -│ World │ 3 │ 30 │ -│ World │ 4 │ 40 │ -│ World │ 5 │ 50 │ -└───────┴────────┴────────┘ -``` - -这种变化也是有道理的: - -``` sql -SELECT s, `nest.x`, `nest.y` -FROM nested_test -ARRAY JOIN `nest.x`; -``` - -``` text -┌─s─────┬─nest.x─┬─nest.y─────┐ -│ Hello │ 1 │ [10,20] │ -│ Hello │ 2 │ [10,20] │ -│ World │ 3 │ [30,40,50] │ -│ World │ 4 │ [30,40,50] │ -│ World │ 5 │ [30,40,50] │ -└───────┴────────┴────────────┘ -``` - -可以将别名用于嵌套数据结构，以便选择 `JOIN` 结果或源数组。 示例: - -``` sql -SELECT s, `n.x`, `n.y`, `nest.x`, `nest.y` -FROM nested_test -ARRAY JOIN nest AS n; -``` - -``` text -┌─s─────┬─n.x─┬─n.y─┬─nest.x──┬─nest.y─────┐ -│ Hello │ 1 │ 10 │ [1,2] │ [10,20] │ -│ Hello │ 2 │ 20 │ [1,2] │ [10,20] │ -│ World │ 3 │ 30 │ [3,4,5] │ [30,40,50] │ -│ World │ 4 │ 40 │ [3,4,5] │ [30,40,50] │ -│ World │ 5 │ 50 │ [3,4,5] │ [30,40,50] │ -└───────┴─────┴─────┴─────────┴────────────┘ -``` - -使用的例子 [arrayEnumerate](../../sql-reference/functions/array-functions.md#array_functions-arrayenumerate) 功能: - -``` sql -SELECT s, `n.x`, `n.y`, `nest.x`, `nest.y`, num -FROM nested_test -ARRAY JOIN nest AS n, arrayEnumerate(`nest.x`) AS num; -``` - -``` text -┌─s─────┬─n.x─┬─n.y─┬─nest.x──┬─nest.y─────┬─num─┐ -│ Hello │ 1 │ 10 │ [1,2] │ [10,20] │ 1 │ -│ Hello │ 2 │ 20 │ [1,2] │ [10,20] │ 2 │ -│ World │ 3 │ 30 │ [3,4,5] │ [30,40,50] │ 1 │ -│ World │ 4 │ 40 │ [3,4,5] │ [30,40,50] │ 2 │ -│ World │ 5 │ 50 │ [3,4,5] │ [30,40,50] │ 3 │ -└───────┴─────┴─────┴─────────┴────────────┴─────┘ -``` - -### JOIN子句 {#select-join} - -加入正常的数据 [SQL JOIN](https://en.wikipedia.org/wiki/Join_(SQL)) 感觉 - -!!! info "注" - 不相关的 [ARRAY JOIN](#select-array-join-clause). - -``` sql -SELECT -FROM -[GLOBAL] [ANY|ALL] [INNER|LEFT|RIGHT|FULL|CROSS] [OUTER] JOIN -(ON )|(USING ) ... -``` - -可以指定表名，而不是 `` 和 ``. 这相当于 `SELECT * FROM table` 子查询，除了在特殊情况下，当表具有 [加入我们](../../engines/table-engines/special/join.md) engine – an array prepared for joining. - -#### 支持的类型 `JOIN` {#select-join-types} - -- `INNER JOIN` （或 `JOIN`) -- `LEFT JOIN` （或 `LEFT OUTER JOIN`) -- `RIGHT JOIN` （或 `RIGHT OUTER JOIN`) -- `FULL JOIN` （或 `FULL OUTER JOIN`) -- `CROSS JOIN` （或 `,` ) - -查看标准 [SQL JOIN](https://en.wikipedia.org/wiki/Join_(SQL)) 描述。 - -#### 多联接 {#multiple-join} - -执行查询时，ClickHouse将多表联接重写为双表联接的序列。 例如，如果有四个连接表ClickHouse连接第一个和第二个，然后将结果与第三个表连接，并在最后一步，它连接第四个表。 - -如果查询包含 `WHERE` 子句，ClickHouse尝试通过中间联接从此子句推下过滤器。 如果无法将筛选器应用于每个中间联接，ClickHouse将在所有联接完成后应用筛选器。 - -我们建议 `JOIN ON` 或 `JOIN USING` 用于创建查询的语法。 例如: - -``` sql -SELECT * FROM t1 JOIN t2 ON t1.a = t2.a JOIN t3 ON t1.a = t3.a -``` - -您可以使用逗号分隔的列表中的表 `FROM` 条款 例如: - -``` sql -SELECT * FROM t1, t2, t3 WHERE t1.a = t2.a AND t1.a = t3.a -``` - -不要混合使用这些语法。 - -ClickHouse不直接支持使用逗号的语法，所以我们不建议使用它们。 该算法尝试重写查询 `CROSS JOIN` 和 `INNER JOIN` 子句，然后继续进行查询处理。 重写查询时，ClickHouse会尝试优化性能和内存消耗。 默认情况下，ClickHouse将逗号视为 `INNER JOIN` 子句和转换 `INNER JOIN` 到 `CROSS JOIN` 当算法不能保证 `INNER JOIN` 返回所需的数据。 - -#### 严格 {#select-join-strictness} - -- `ALL` — If the right table has several matching rows, ClickHouse creates a [笛卡尔积](https://en.wikipedia.org/wiki/Cartesian_product) 从匹配的行。 这是标准 `JOIN` SQL中的行为。 -- `ANY` — If the right table has several matching rows, only the first one found is joined. If the right table has only one matching row, the results of queries with `ANY` 和 `ALL` 关键字是相同的。 -- `ASOF` — For joining sequences with a non-exact match. `ASOF JOIN` 用法描述如下。 - -**ASOF加入使用** - -`ASOF JOIN` 当您需要连接没有完全匹配的记录时非常有用。 - -表 `ASOF JOIN` 必须具有有序序列列。 此列不能单独存在于表中，并且应该是其中一种数据类型: `UInt32`, `UInt64`, `Float32`, `Float64`, `Date`，和 `DateTime`. - -语法 `ASOF JOIN ... ON`: - -``` sql -SELECT expressions_list -FROM table_1 -ASOF LEFT JOIN table_2 -ON equi_cond AND closest_match_cond -``` - -您可以使用任意数量的相等条件和恰好一个最接近的匹配条件。 例如, `SELECT count() FROM table_1 ASOF LEFT JOIN table_2 ON table_1.a == table_2.b AND table_2.t <= table_1.t`. - -支持最接近匹配的条件: `>`, `>=`, `<`, `<=`. - -语法 `ASOF JOIN ... USING`: - -``` sql -SELECT expressions_list -FROM table_1 -ASOF JOIN table_2 -USING (equi_column1, ... equi_columnN, asof_column) -``` - -`ASOF JOIN` 用途 `equi_columnX` 对于加入平等和 `asof_column` 用于加入与最接近的比赛 `table_1.asof_column >= table_2.asof_column` 条件。 该 `asof_column` 列总是在最后一个 `USING` 条款 - -例如，请考虑下表: - - table_1 table_2 - event | ev_time | user_id event | ev_time | user_id - ----------|---------|---------- ----------|---------|---------- - ... ... - event_1_1 | 12:00 | 42 event_2_1 | 11:59 | 42 - ... event_2_2 | 12:30 | 42 - event_1_2 | 13:00 | 42 event_2_3 | 13:00 | 42 - ... ... - -`ASOF JOIN` 可以从用户事件的时间戳 `table_1` 并找到一个事件 `table_2` 其中时间戳最接近事件的时间戳 `table_1` 对应于最接近的匹配条件。 如果可用，则相等的时间戳值是最接近的值。 在这里，该 `user_id` 列可用于连接相等和 `ev_time` 列可用于在最接近的匹配加入。 在我们的例子中, `event_1_1` 可以加入 `event_2_1` 和 `event_1_2` 可以加入 `event_2_3`，但是 `event_2_2` 不能加入。 - -!!! note "注" - `ASOF` 加入是 **不** 支持在 [加入我们](../../engines/table-engines/special/join.md) 表引擎。 - -若要设置默认严格性值，请使用session configuration参数 [join\_default\_strictness](../../operations/settings/settings.md#settings-join_default_strictness). - -#### GLOBAL JOIN {#global-join} - -当使用正常 `JOIN`，将查询发送到远程服务器。 为了创建正确的表，在每个子查询上运行子查询，并使用此表执行联接。 换句话说，在每个服务器上单独形成右表。 - -使用时 `GLOBAL ... JOIN`，首先请求者服务器运行一个子查询来计算正确的表。 此临时表将传递到每个远程服务器，并使用传输的临时数据对其运行查询。 - -使用时要小心 `GLOBAL`. 有关详细信息，请参阅部分 [分布式子查询](#select-distributed-subqueries). - -#### 使用建议 {#usage-recommendations} - -当运行 `JOIN`，与查询的其他阶段相关的执行顺序没有优化。 连接（在右表中搜索）在过滤之前运行 `WHERE` 和聚集之前。 为了明确设置处理顺序，我们建议运行 `JOIN` 具有子查询的子查询。 - -示例: - -``` sql -SELECT - CounterID, - hits, - visits -FROM -( - SELECT - CounterID, - count() AS hits - FROM test.hits - GROUP BY CounterID -) ANY LEFT JOIN -( - SELECT - CounterID, - sum(Sign) AS visits - FROM test.visits - GROUP BY CounterID -) USING CounterID -ORDER BY hits DESC -LIMIT 10 -``` - -``` text -┌─CounterID─┬───hits─┬─visits─┐ -│ 1143050 │ 523264 │ 13665 │ -│ 731962 │ 475698 │ 102716 │ -│ 722545 │ 337212 │ 108187 │ -│ 722889 │ 252197 │ 10547 │ -│ 2237260 │ 196036 │ 9522 │ -│ 23057320 │ 147211 │ 7689 │ -│ 722818 │ 90109 │ 17847 │ -│ 48221 │ 85379 │ 4652 │ -│ 19762435 │ 77807 │ 7026 │ -│ 722884 │ 77492 │ 11056 │ -└───────────┴────────┴────────┘ -``` - -子查询不允许您设置名称或使用它们从特定子查询引用列。 -在指定的列 `USING` 两个子查询中必须具有相同的名称，并且其他列必须以不同的方式命名。 您可以使用别名更改子查询中的列名（此示例使用别名 `hits` 和 `visits`). - -该 `USING` 子句指定一个或多个要联接的列，这将建立这些列的相等性。 列的列表设置不带括号。 不支持更复杂的连接条件。 - -正确的表（子查询结果）驻留在RAM中。 如果没有足够的内存，则无法运行 `JOIN`. - -每次使用相同的查询运行 `JOIN`，子查询再次运行，因为结果未缓存。 为了避免这种情况，使用特殊的 [加入我们](../../engines/table-engines/special/join.md) 表引擎，它是一个用于连接的准备好的数组，总是在RAM中。 - -在某些情况下，使用效率更高 `IN` 而不是 `JOIN`. -在各种类型的 `JOIN`，最有效的是 `ANY LEFT JOIN`，然后 `ANY INNER JOIN`. 效率最低的是 `ALL LEFT JOIN` 和 `ALL INNER JOIN`. - -如果你需要一个 `JOIN` 对于连接维度表（这些是包含维度属性的相对较小的表，例如广告活动的名称）， `JOIN` 由于每个查询都会重新访问正确的表，因此可能不太方便。 对于这种情况下，有一个 “external dictionaries” 您应该使用的功能 `JOIN`. 有关详细信息，请参阅部分 [外部字典](../dictionaries/external-dictionaries/external-dicts.md). - -**内存限制** - -ClickHouse使用 [哈希联接](https://en.wikipedia.org/wiki/Hash_join) 算法。 ClickHouse采取 `` 并在RAM中为其创建哈希表。 如果需要限制联接操作内存消耗，请使用以下设置: - -- [max\_rows\_in\_join](../../operations/settings/query-complexity.md#settings-max_rows_in_join) — Limits number of rows in the hash table. -- [max\_bytes\_in\_join](../../operations/settings/query-complexity.md#settings-max_bytes_in_join) — Limits size of the hash table. - -当任何这些限制达到，ClickHouse作为 [join\_overflow\_mode](../../operations/settings/query-complexity.md#settings-join_overflow_mode) 设置指示。 - -#### 处理空单元格或空单元格 {#processing-of-empty-or-null-cells} - -在连接表时，可能会出现空单元格。 设置 [join\_use\_nulls](../../operations/settings/settings.md#join_use_nulls) 定义ClickHouse如何填充这些单元格。 - -如果 `JOIN` 键是 [可为空](../data-types/nullable.md) 字段，其中至少有一个键具有值的行 [NULL](../syntax.md#null-literal) 没有加入。 - -#### 语法限制 {#syntax-limitations} - -对于多个 `JOIN` 单个子句 `SELECT` 查询: - -- 通过以所有列 `*` 仅在联接表时才可用，而不是子查询。 -- 该 `PREWHERE` 条款不可用。 - -为 `ON`, `WHERE`，和 `GROUP BY` 条款: - -- 任意表达式不能用于 `ON`, `WHERE`，和 `GROUP BY` 子句，但你可以定义一个表达式 `SELECT` 子句，然后通过别名在这些子句中使用它。 - -### WHERE条款 {#select-where} - -如果存在WHERE子句，则必须包含具有UInt8类型的表达式。 这通常是一个带有比较和逻辑运算符的表达式。 -此表达式将用于在所有其他转换之前过滤数据。 - -如果数据库表引擎支持索引，则根据使用索引的能力计算表达式。 - -### PREWHERE条款 {#prewhere-clause} - -此条款与WHERE条款具有相同的含义。 区别在于从表中读取数据。 -使用PREWHERE时，首先只读取执行PREWHERE所需的列。 然后读取运行查询所需的其他列，但只读取PREWHERE表达式为true的那些块。 - -如果查询中的少数列使用过滤条件，但提供强大的数据过滤，则使用PREWHERE是有意义的。 这减少了要读取的数据量。 - -例如，对于提取大量列但仅对少数列进行过滤的查询，编写PREWHERE非常有用。 - -PREWHERE仅由来自 `*MergeTree` 家人 - -查询可以同时指定PREWHERE和WHERE。 在这种情况下，PREWHERE先于WHERE。 - -如果 ‘optimize\_move\_to\_prewhere’ 设置设置为1并省略PREWHERE，系统使用启发式方法自动将部分表达式从哪里移动到哪里。 - -### GROUP BY子句 {#select-group-by-clause} - -这是面向列的DBMS最重要的部分之一。 - -如果存在GROUP BY子句，则必须包含表达式列表。 每个表达式将在这里被称为 “key”. -SELECT、HAVING和ORDER BY子句中的所有表达式都必须从键或聚合函数计算。 换句话说，从表中选择的每个列必须在键或聚合函数内使用。 - -如果查询仅包含聚合函数中的表列，则可以省略GROUP BY子句，并假定通过一组空键进行聚合。 - -示例: - -``` sql -SELECT - count(), - median(FetchTiming > 60 ? 60 : FetchTiming), - count() - sum(Refresh) -FROM hits -``` - -但是，与标准SQL相比，如果表没有任何行（根本没有任何行，或者在使用WHERE to filter之后没有任何行），则返回一个空结果，而不是来自包含聚合函数初始值的行之 - -相对于MySQL（并且符合标准SQL），您无法获取不在键或聚合函数（常量表达式除外）中的某些列的某些值。 要解决此问题，您可以使用 ‘any’ 聚合函数（获取第一个遇到的值）或 ‘min/max’. - -示例: - -``` sql -SELECT - domainWithoutWWW(URL) AS domain, - count(), - any(Title) AS title -- getting the first occurred page header for each domain. -FROM hits -GROUP BY domain -``` - -对于遇到的每个不同的键值，GROUP BY计算一组聚合函数值。 - -数组列不支持分组依据。 - -不能将常量指定为聚合函数的参数。 示例：sum(1)。 相反，你可以摆脱常数。 示例: `count()`. - -#### 空处理 {#null-processing} - -对于GROUP BY子句，ClickHouse将 [NULL](../syntax.md#null-literal) 解释为一个值，并且支持`NULL=NULL`。 - -这里有一个例子来说明这意味着什么。 - -假设你有这张桌子: - -``` text -┌─x─┬────y─┐ -│ 1 │ 2 │ -│ 2 │ ᴺᵁᴸᴸ │ -│ 3 │ 2 │ -│ 3 │ 3 │ -│ 3 │ ᴺᵁᴸᴸ │ -└───┴──────┘ -``` - -查询 `SELECT sum(x), y FROM t_null_big GROUP BY y` 结果: - -``` text -┌─sum(x)─┬────y─┐ -│ 4 │ 2 │ -│ 3 │ 3 │ -│ 5 │ ᴺᵁᴸᴸ │ -└────────┴──────┘ -``` - -你可以看到 `GROUP BY` 为 `y = NULL` 总结 `x`，仿佛 `NULL` 是这个值。 - -如果你通过几个键 `GROUP BY`，结果会给你选择的所有组合，就好像 `NULL` 是一个特定的值。 - -#### 使用总计修饰符 {#with-totals-modifier} - -如果指定了WITH TOTALS修饰符，则将计算另一行。 此行将具有包含默认值（零或空行）的关键列，以及包含跨所有行计算值的聚合函数列（ “total” 值）。 - -这个额外的行以JSON\*，TabSeparated\*和Pretty\*格式输出，与其他行分开。 在其他格式中，此行不输出。 - -在JSON\*格式中，此行作为单独的输出 ‘totals’ 场。 在TabSeparated\*格式中，该行位于主结果之后，前面有一个空行（在其他数据之后）。 在Pretty\*格式中，该行在主结果之后作为单独的表输出。 - -`WITH TOTALS` 当有存在时，可以以不同的方式运行。 该行为取决于 ‘totals\_mode’ 设置。 -默认情况下, `totals_mode = 'before_having'`. 在这种情况下, ‘totals’ 是跨所有行计算，包括那些不通过具有和 ‘max\_rows\_to\_group\_by’. - -其他替代方案仅包括通过具有在 ‘totals’，并与设置不同的行为 `max_rows_to_group_by` 和 `group_by_overflow_mode = 'any'`. - -`after_having_exclusive` – Don't include rows that didn't pass through `max_rows_to_group_by`. 换句话说, ‘totals’ 将有少于或相同数量的行，因为它会 `max_rows_to_group_by` 被省略。 - -`after_having_inclusive` – Include all the rows that didn't pass through ‘max\_rows\_to\_group\_by’ 在 ‘totals’. 换句话说, ‘totals’ 将有多个或相同数量的行，因为它会 `max_rows_to_group_by` 被省略。 - -`after_having_auto` – Count the number of rows that passed through HAVING. If it is more than a certain amount (by default, 50%), include all the rows that didn't pass through ‘max\_rows\_to\_group\_by’ 在 ‘totals’. 否则，不包括它们。 - -`totals_auto_threshold` – By default, 0.5. The coefficient for `after_having_auto`. - -如果 `max_rows_to_group_by` 和 `group_by_overflow_mode = 'any'` 不使用，所有的变化 `after_having` 是相同的，你可以使用它们中的任何一个（例如, `after_having_auto`). - -您可以在子查询中使用总计，包括在JOIN子句中的子查询（在这种情况下，将合并各自的总计值）。 - -#### 在外部存储器中分组 {#select-group-by-in-external-memory} - -您可以启用将临时数据转储到磁盘以限制内存使用期间 `GROUP BY`. -该 [max\_bytes\_before\_external\_group\_by](../../operations/settings/settings.md#settings-max_bytes_before_external_group_by) 设置确定倾销的阈值RAM消耗 `GROUP BY` 临时数据到文件系统。 如果设置为0（默认值），它将被禁用。 - -使用时 `max_bytes_before_external_group_by`，我们建议您设置 `max_memory_usage` 大约两倍高。 这是必要的，因为聚合有两个阶段：读取日期和形成中间数据（1）和合并中间数据（2）。 将数据转储到文件系统只能在阶段1中发生。 如果未转储临时数据，则阶段2可能需要与阶段1相同的内存量。 - -例如，如果 [max\_memory\_usage](../../operations/settings/settings.md#settings_max_memory_usage) 设置为10000000000，你想使用外部聚合，这是有意义的设置 `max_bytes_before_external_group_by` 到10000000000，max\_memory\_usage到20000000000。 当触发外部聚合（如果至少有一个临时数据转储）时，RAM的最大消耗仅略高于 `max_bytes_before_external_group_by`. - -通过分布式查询处理，在远程服务器上执行外部聚合。 为了使请求者服务器只使用少量的RAM，设置 `distributed_aggregation_memory_efficient` 到1。 - -当合并数据刷新到磁盘时，以及当合并来自远程服务器的结果时， `distributed_aggregation_memory_efficient` 设置被启用，消耗高达 `1/256 * the_number_of_threads` 从RAM的总量。 - -当启用外部聚合时，如果有小于 `max_bytes_before_external_group_by` of data (i.e. data was not flushed), the query runs just as fast as without external aggregation. If any temporary data was flushed, the run time will be several times longer (approximately three times). - -如果你有一个 `ORDER BY` 用一个 `LIMIT` 后 `GROUP BY`，然后使用的RAM的量取决于数据的量 `LIMIT`，不是在整个表。 但如果 `ORDER BY` 没有 `LIMIT`，不要忘记启用外部排序 (`max_bytes_before_external_sort`). - -### 限制条款 {#limit-by-clause} - -与查询 `LIMIT n BY expressions` 子句选择第一个 `n` 每个不同值的行 `expressions`. 的关键 `LIMIT BY` 可以包含任意数量的 [表达式](../syntax.md#syntax-expressions). - -ClickHouse支持以下语法: - -- `LIMIT [offset_value, ]n BY expressions` -- `LIMIT n OFFSET offset_value BY expressions` - -在查询处理过程中，ClickHouse会选择按排序键排序的数据。 排序键使用以下命令显式设置 [ORDER BY](#select-order-by) 子句或隐式作为表引擎的属性。 然后ClickHouse应用 `LIMIT n BY expressions` 并返回第一 `n` 每个不同组合的行 `expressions`. 如果 `OFFSET` 被指定，则对于每个数据块属于一个不同的组合 `expressions`，ClickHouse跳过 `offset_value` 从块开始的行数，并返回最大值 `n` 行的结果。 如果 `offset_value` 如果数据块中的行数大于数据块中的行数，ClickHouse将从该块返回零行。 - -`LIMIT BY` 是不相关的 `LIMIT`. 它们都可以在同一个查询中使用。 - -**例** - -样品表: - -``` sql -CREATE TABLE limit_by(id Int, val Int) ENGINE = Memory; -INSERT INTO limit_by values(1, 10), (1, 11), (1, 12), (2, 20), (2, 21); -``` - -查询: - -``` sql -SELECT * FROM limit_by ORDER BY id, val LIMIT 2 BY id -``` - -``` text -┌─id─┬─val─┐ -│ 1 │ 10 │ -│ 1 │ 11 │ -│ 2 │ 20 │ -│ 2 │ 21 │ -└────┴─────┘ -``` - -``` sql -SELECT * FROM limit_by ORDER BY id, val LIMIT 1, 2 BY id -``` - -``` text -┌─id─┬─val─┐ -│ 1 │ 11 │ -│ 1 │ 12 │ -│ 2 │ 21 │ -└────┴─────┘ -``` - -该 `SELECT * FROM limit_by ORDER BY id, val LIMIT 2 OFFSET 1 BY id` 查询返回相同的结果。 - -以下查询返回每个引用的前5个引用 `domain, device_type` 最多可与100行配对 (`LIMIT n BY + LIMIT`). - -``` sql -SELECT - domainWithoutWWW(URL) AS domain, - domainWithoutWWW(REFERRER_URL) AS referrer, - device_type, - count() cnt -FROM hits -GROUP BY domain, referrer, device_type -ORDER BY cnt DESC -LIMIT 5 BY domain, device_type -LIMIT 100 -``` - -### 有条款 {#having-clause} - -允许筛选GROUP BY之后收到的结果，类似于WHERE子句。 -WHERE和HAVING的不同之处在于WHERE是在聚合（GROUP BY）之前执行的，而HAVING是在聚合之后执行的。 -如果不执行聚合，则不能使用HAVING。 - -### 按条款订购 {#select-order-by} - -ORDER BY子句包含一个表达式列表，每个表达式都可以分配DESC或ASC（排序方向）。 如果未指定方向，则假定ASC。 ASC按升序排序，DESC按降序排序。 排序方向适用于单个表达式，而不适用于整个列表。 示例: `ORDER BY Visits DESC, SearchPhrase` - -对于按字符串值排序，可以指定排序规则（比较）。 示例: `ORDER BY SearchPhrase COLLATE 'tr'` -对于按关键字升序排序，使用土耳其字母，不区分大小写，假设字符串是UTF-8编码。 COLLATE可以按顺序独立地为每个表达式指定或不指定。 如果指定了ASC或DESC，则在其后指定COLLATE。 使用COLLATE时，排序始终不区分大小写。 - -我们只建议使用COLLATE对少量行进行最终排序，因为使用COLLATE进行排序的效率低于正常按字节进行排序的效率。 - -对于排序表达式列表具有相同值的行，将以任意顺序输出，也可以是不确定的（每次都不同）。 -如果省略ORDER BY子句，则行的顺序也是未定义的，并且可能也是不确定的。 - -`NaN` 和 `NULL` 排序顺序: - -- 使用修饰符 `NULLS FIRST` — First `NULL`，然后 `NaN`，然后其他值。 -- 使用修饰符 `NULLS LAST` — First the values, then `NaN`，然后 `NULL`. -- Default — The same as with the `NULLS LAST` 修饰符。 - -示例: - -对于表 - -``` text -┌─x─┬────y─┐ -│ 1 │ ᴺᵁᴸᴸ │ -│ 2 │ 2 │ -│ 1 │ nan │ -│ 2 │ 2 │ -│ 3 │ 4 │ -│ 5 │ 6 │ -│ 6 │ nan │ -│ 7 │ ᴺᵁᴸᴸ │ -│ 6 │ 7 │ -│ 8 │ 9 │ -└───┴──────┘ -``` - -运行查询 `SELECT * FROM t_null_nan ORDER BY y NULLS FIRST` 获得: - -``` text -┌─x─┬────y─┐ -│ 1 │ ᴺᵁᴸᴸ │ -│ 7 │ ᴺᵁᴸᴸ │ -│ 1 │ nan │ -│ 6 │ nan │ -│ 2 │ 2 │ -│ 2 │ 2 │ -│ 3 │ 4 │ -│ 5 │ 6 │ -│ 6 │ 7 │ -│ 8 │ 9 │ -└───┴──────┘ -``` - -当对浮点数进行排序时，Nan与其他值是分开的。 无论排序顺序如何，Nan都在最后。 换句话说，对于升序排序，它们被放置为好像它们比所有其他数字大，而对于降序排序，它们被放置为好像它们比其他数字小。 - -如果除了ORDER BY之外指定了足够小的限制，则使用较少的RAM。 否则，所花费的内存量与用于排序的数据量成正比。 对于分布式查询处理，如果省略GROUP BY，则在远程服务器上部分完成排序，并在请求者服务器上合并结果。 这意味着对于分布式排序，要排序的数据量可以大于单个服务器上的内存量。 - -如果没有足够的RAM，则可以在外部存储器中执行排序（在磁盘上创建临时文件）。 使用设置 `max_bytes_before_external_sort` 为此目的。 如果将其设置为0（默认值），则禁用外部排序。 如果启用，则当要排序的数据量达到指定的字节数时，将对收集的数据进行排序并转储到临时文件中。 读取所有数据后，将合并所有已排序的文件并输出结果。 文件被写入配置中的/var/lib/clickhouse/tmp/目录（默认情况下，但您可以使用 ‘tmp\_path’ 参数来更改此设置）。 - -运行查询可能占用的内存比 ‘max\_bytes\_before\_external\_sort’. 因此，此设置的值必须大大小于 ‘max\_memory\_usage’. 例如，如果您的服务器有128GB的RAM，并且您需要运行单个查询，请设置 ‘max\_memory\_usage’ 到100GB，和 ‘max\_bytes\_before\_external\_sort’ 至80GB。 - -外部排序的工作效率远远低于在RAM中进行排序。 - -### SELECT子句 {#select-select} - -[表达式](../syntax.md#syntax-expressions) 在指定 `SELECT` 子句是在上述子句中的所有操作完成后计算的。 这些表达式的工作方式就好像它们应用于结果中的单独行一样。 如果在表达式 `SELECT` 子句包含聚合函数，然后ClickHouse处理过程中用作其参数的聚合函数和表达式 [GROUP BY](#select-group-by-clause) 聚合。 - -如果要在结果中包含所有列，请使用星号 (`*`）符号。 例如, `SELECT * FROM ...`. - -将结果中的某些列与 [re2](https://en.wikipedia.org/wiki/RE2_(software)) 正则表达式，您可以使用 `COLUMNS` 表达。 - -``` sql -COLUMNS('regexp') -``` - -例如，考虑表: - -``` sql -CREATE TABLE default.col_names (aa Int8, ab Int8, bc Int8) ENGINE = TinyLog -``` - -以下查询从包含以下内容的所有列中选择数据 `a` 在他们的名字符号。 - -``` sql -SELECT COLUMNS('a') FROM col_names -``` - -``` text -┌─aa─┬─ab─┐ -│ 1 │ 1 │ -└────┴────┘ -``` - -所选列不按字母顺序返回。 - -您可以使用多个 `COLUMNS` 查询中的表达式并将函数应用于它们。 - -例如: - -``` sql -SELECT COLUMNS('a'), COLUMNS('c'), toTypeName(COLUMNS('c')) FROM col_names -``` - -``` text -┌─aa─┬─ab─┬─bc─┬─toTypeName(bc)─┐ -│ 1 │ 1 │ 1 │ Int8 │ -└────┴────┴────┴────────────────┘ -``` - -由返回的每一列 `COLUMNS` 表达式作为单独的参数传递给函数。 如果函数支持其他参数，您也可以将其他参数传递给函数。 使用函数时要小心。 如果函数不支持您传递给它的参数数，ClickHouse将引发异常。 - -例如: - -``` sql -SELECT COLUMNS('a') + COLUMNS('c') FROM col_names -``` - -``` text -Received exception from server (version 19.14.1): -Code: 42. DB::Exception: Received from localhost:9000. DB::Exception: Number of arguments for function plus doesn't match: passed 3, should be 2. -``` - -在这个例子中, `COLUMNS('a')` 返回两列: `aa` 和 `ab`. `COLUMNS('c')` 返回 `bc` 列。 该 `+` 运算符不能应用于3个参数，因此ClickHouse引发一个带有相关消息的异常。 - -匹配的列 `COLUMNS` 表达式可以具有不同的数据类型。 如果 `COLUMNS` 不匹配任何列，并且是唯一的表达式 `SELECT`，ClickHouse抛出异常。 - -### DISTINCT子句 {#select-distinct} - -如果指定了DISTINCT，则在结果中所有完全匹配的行集中，只有一行将保留。 -结果将与在没有聚合函数的情况下在SELECT中指定的所有字段中指定GROUP BY一样。 但有几个区别，从组通过: - -- DISTINCT可以与GROUP BY一起应用。 -- 如果省略ORDER BY并定义LIMIT，则在读取所需数量的不同行后，查询立即停止运行。 -- 数据块在处理时输出，而无需等待整个查询完成运行。 - -如果SELECT至少有一个数组列，则不支持DISTINCT。 - -`DISTINCT` 适用于 [NULL](../syntax.md#null-literal) 就好像 `NULL` 是一个特定的值，并且 `NULL=NULL`. 换句话说，在 `DISTINCT` 结果，不同的组合 `NULL` 只发生一次。 - -ClickHouse支持使用 `DISTINCT` 和 `ORDER BY` 一个查询中不同列的子句。 该 `DISTINCT` 子句之前执行 `ORDER BY` 条款 - -示例表: - -``` text -┌─a─┬─b─┐ -│ 2 │ 1 │ -│ 1 │ 2 │ -│ 3 │ 3 │ -│ 2 │ 4 │ -└───┴───┘ -``` - -`DISTINCT`可以与 [NULL](../syntax.md#null-literal)一起工作，就好像`NULL`仅是一个特殊的值一样，并且`NULL=NULL`。换而言之，在`DISTINCT`的结果中，与`NULL`不同的组合仅能出现一次。 - -``` text -┌─a─┐ -│ 2 │ -│ 1 │ -│ 3 │ -└───┘ -``` - -如果我们改变排序方向 `SELECT DISTINCT a FROM t1 ORDER BY b DESC`，我们得到以下结果: - -``` text -┌─a─┐ -│ 3 │ -│ 1 │ -│ 2 │ -└───┘ -``` - -行 `2, 4` 分拣前被切割。 - -在编程查询时考虑这种实现特异性。 - -### 限制条款 {#limit-clause} - -`LIMIT m` 允许您选择第一个 `m` 结果中的行。 - -`LIMIT n, m` 允许您选择第一个 `m` 跳过第一个结果后的行 `n` 行。 该 `LIMIT m OFFSET n` 也支持语法。 - -`n` 和 `m` 必须是非负整数。 - -如果没有 `ORDER BY` 明确排序结果的子句，结果可能是任意的和不确定的。 - -### UNION ALL条款 {#union-all-clause} - -您可以使用UNION ALL来组合任意数量的查询。 示例: - -``` sql -SELECT CounterID, 1 AS table, toInt64(count()) AS c - FROM test.hits - GROUP BY CounterID - -UNION ALL - -SELECT CounterID, 2 AS table, sum(Sign) AS c - FROM test.visits - GROUP BY CounterID - HAVING c > 0 -``` - -只支持UNION ALL。 不支持常规联合（UNION DISTINCT）。 如果您需要UNION DISTINCT，则可以从包含UNION ALL的子查询中编写SELECT DISTINCT。 - -作为UNION ALL部分的查询可以同时运行，并且它们的结果可以混合在一起。 - -结果的结构（列的数量和类型）必须与查询匹配。 但列名可能不同。 在这种情况下，最终结果的列名将从第一个查询中获取。 对联合执行类型转换。 例如，如果合并的两个查询具有相同的字段与非-`Nullable` 和 `Nullable` 从兼容类型的类型，由此产生的 `UNION ALL` 有一个 `Nullable` 类型字段。 - -作为UNION ALL部分的查询不能括在括号中。 ORDER BY和LIMIT应用于单独的查询，而不是最终结果。 如果您需要将转换应用于最终结果，则可以将所有带有UNION ALL的查询放在FROM子句的子查询中。 - -### INTO OUTFILE条款 {#into-outfile-clause} - -添加 `INTO OUTFILE filename` 子句（其中filename是字符串文字），用于将查询输出重定向到指定的文件。 -与MySQL相比，该文件是在客户端创建的。 如果具有相同文件名的文件已经存在，则查询将失败。 -此功能在命令行客户端和clickhouse-local中可用（通过HTTP接口发送的查询将失败）。 - -默认输出格式为TabSeparated（与命令行客户端批处理模式相同）。 - -### 格式子句 {#format-clause} - -指定 ‘FORMAT format’ 获取任何指定格式的数据。 -为了方便起见，您可以使用它或创建转储。 -有关详细信息，请参阅部分 “Formats”. -如果省略FORMAT子句，则使用默认格式，这取决于用于访问数据库的设置和接口。 对于http接口和批处理模式下的命令行客户端，默认格式为TabSeparated。 对于交互模式下的命令行客户端，默认格式为PrettyCompact（它具有吸引力和紧凑的表）。 - -使用命令行客户端时，数据以内部高效格式传递给客户端。 客户端独立解释查询的FORMAT子句并格式化数据本身（从而减轻网络和服务器的负载）。 - -### 在运营商 {#select-in-operators} - -该 `IN`, `NOT IN`, `GLOBAL IN`，和 `GLOBAL NOT IN` 运算符是单独复盖的，因为它们的功能相当丰富。 - -运算符的左侧是单列或元组。 - -例: - -``` sql -SELECT UserID IN (123, 456) FROM ... -SELECT (CounterID, UserID) IN ((34, 123), (101500, 456)) FROM ... -``` - -如果左侧是索引中的单列，而右侧是一组常量，则系统将使用索引处理查询。 - -Don't list too many values explicitly (i.e. millions). If a data set is large, put it in a temporary table (for example, see the section “External data for query processing”），然后使用子查询。 - -运算符的右侧可以是一组常量表达式、一组带有常量表达式的元组（如上面的示例所示），或括号中的数据库表或SELECT子查询的名称。 - -如果运算符的右侧是表的名称（例如, `UserID IN users`），这相当于子查询 `UserID IN (SELECT * FROM users)`. 使用与查询一起发送的外部数据时，请使用此选项。 例如，查询可以与一组用户Id一起发送到 ‘users’ 应过滤的临时表。 - -如果运算符的右侧是具有Set引擎的表名（始终位于RAM中的准备好的数据集），则不会为每个查询重新创建数据集。 - -子查询可以指定多个用于筛选元组的列。 -示例: - -``` sql -SELECT (CounterID, UserID) IN (SELECT CounterID, UserID FROM ...) FROM ... -``` - -IN运算符左侧和右侧的列应具有相同的类型。 - -IN运算符和子查询可能出现在查询的任何部分，包括聚合函数和lambda函数。 -示例: - -``` sql -SELECT - EventDate, - avg(UserID IN - ( - SELECT UserID - FROM test.hits - WHERE EventDate = toDate('2014-03-17') - )) AS ratio -FROM test.hits -GROUP BY EventDate -ORDER BY EventDate ASC -``` - -``` text -┌──EventDate─┬────ratio─┐ -│ 2014-03-17 │ 1 │ -│ 2014-03-18 │ 0.807696 │ -│ 2014-03-19 │ 0.755406 │ -│ 2014-03-20 │ 0.723218 │ -│ 2014-03-21 │ 0.697021 │ -│ 2014-03-22 │ 0.647851 │ -│ 2014-03-23 │ 0.648416 │ -└────────────┴──────────┘ -``` - -对于3月17日后的每一天，计算3月17日访问该网站的用户所做的浏览量百分比。 -IN子句中的子查询始终只在单个服务器上运行一次。 没有依赖子查询。 - -#### 空处理 {#null-processing-1} - -在处理中，IN操作符总是假定 [NULL](../syntax.md#null-literal) 值的操作结果总是等于`0`，而不管`NULL`位于左侧还是右侧。`NULL`值不应该包含在任何数据集中，它们彼此不能够对应，并且不能够比较。 - -下面是一个例子 `t_null` 表: - -``` text -┌─x─┬────y─┐ -│ 1 │ ᴺᵁᴸᴸ │ -│ 2 │ 3 │ -└───┴──────┘ -``` - -运行查询 `SELECT x FROM t_null WHERE y IN (NULL,3)` 为您提供以下结果: - -``` text -┌─x─┐ -│ 2 │ -└───┘ -``` - -你可以看到，在其中的行 `y = NULL` 被抛出的查询结果。 这是因为ClickHouse无法决定是否 `NULL` 包含在 `(NULL,3)` 设置，返回 `0` 作为操作的结果，和 `SELECT` 从最终输出中排除此行。 - -``` sql -SELECT y IN (NULL, 3) -FROM t_null -``` - -``` text -┌─in(y, tuple(NULL, 3))─┐ -│ 0 │ -│ 1 │ -└───────────────────────┘ -``` - -#### 分布式子查询 {#select-distributed-subqueries} - -带子查询的IN-s有两个选项（类似于连接）：normal `IN` / `JOIN` 和 `GLOBAL IN` / `GLOBAL JOIN`. 它们在分布式查询处理的运行方式上有所不同。 - -!!! attention "注意" - 请记住，下面描述的算法可能会有不同的工作方式取决于 [设置](../../operations/settings/settings.md) `distributed_product_mode` 设置。 - -当使用常规IN时，查询被发送到远程服务器，并且它们中的每个服务器都在运行子查询 `IN` 或 `JOIN` 条款 - -使用时 `GLOBAL IN` / `GLOBAL JOINs`，首先所有的子查询都运行 `GLOBAL IN` / `GLOBAL JOINs`，并将结果收集在临时表中。 然后将临时表发送到每个远程服务器，其中使用此临时数据运行查询。 - -对于非分布式查询，请使用常规 `IN` / `JOIN`. - -在使用子查询时要小心 `IN` / `JOIN` 用于分布式查询处理的子句。 - -让我们来看看一些例子。 假设集群中的每个服务器都有一个正常的 **local\_table**. 每个服务器还具有 **distributed\_table** 表与 **分布** 类型，它查看群集中的所有服务器。 - -对于查询 **distributed\_table**，查询将被发送到所有远程服务器，并使用以下命令在其上运行 **local\_table**. - -例如，查询 - -``` sql -SELECT uniq(UserID) FROM distributed_table -``` - -将被发送到所有远程服务器 - -``` sql -SELECT uniq(UserID) FROM local_table -``` - -并且并行运行它们中的每一个，直到达到可以结合中间结果的阶段。 然后将中间结果返回给请求者服务器并在其上合并，并将最终结果发送给客户端。 - -现在让我们检查一个查询IN: - -``` sql -SELECT uniq(UserID) FROM distributed_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM local_table WHERE CounterID = 34) -``` - -- 计算两个网站的受众的交集。 - -此查询将以下列方式发送到所有远程服务器 - -``` sql -SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM local_table WHERE CounterID = 34) -``` - -换句话说，IN子句中的数据集将在每台服务器上独立收集，仅在每台服务器上本地存储的数据中收集。 - -如果您已经为此情况做好准备，并且已经将数据分散到群集服务器上，以便单个用户Id的数据完全驻留在单个服务器上，则这将正常和最佳地工作。 在这种情况下，所有必要的数据将在每台服务器上本地提供。 否则，结果将是不准确的。 我们将查询的这种变体称为 “local IN”. - -若要更正数据在群集服务器上随机传播时查询的工作方式，可以指定 **distributed\_table** 在子查询中。 查询如下所示: - -``` sql -SELECT uniq(UserID) FROM distributed_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM distributed_table WHERE CounterID = 34) -``` - -此查询将以下列方式发送到所有远程服务器 - -``` sql -SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM distributed_table WHERE CounterID = 34) -``` - -子查询将开始在每个远程服务器上运行。 由于子查询使用分布式表，因此每个远程服务器上的子查询将重新发送到每个远程服务器 - -``` sql -SELECT UserID FROM local_table WHERE CounterID = 34 -``` - -例如，如果您有100台服务器的集群，则执行整个查询将需要10,000个基本请求，这通常被认为是不可接受的。 - -在这种情况下，应始终使用GLOBAL IN而不是IN。 让我们来看看它是如何工作的查询 - -``` sql -SELECT uniq(UserID) FROM distributed_table WHERE CounterID = 101500 AND UserID GLOBAL IN (SELECT UserID FROM distributed_table WHERE CounterID = 34) -``` - -请求者服务器将运行子查询 - -``` sql -SELECT UserID FROM distributed_table WHERE CounterID = 34 -``` - -结果将被放在RAM中的临时表中。 然后请求将被发送到每个远程服务器 - -``` sql -SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID GLOBAL IN _data1 -``` - -和临时表 `_data1` 将通过查询发送到每个远程服务器（临时表的名称是实现定义的）。 - -这比使用正常IN更优化。 但是，请记住以下几点: - -1. 创建临时表时，数据不是唯一的。 要减少通过网络传输的数据量，请在子查询中指定DISTINCT。 （你不需要为正常人做这个。) -2. 临时表将被发送到所有远程服务器。 传输不考虑网络拓扑。 例如，如果10个远程服务器驻留在与请求者服务器非常远程的数据中心中，则数据将通过通道发送10次到远程数据中心。 使用GLOBAL IN时尽量避免使用大型数据集。 -3. 将数据传输到远程服务器时，无法配置网络带宽限制。 您可能会使网络过载。 -4. 尝试跨服务器分发数据，以便您不需要定期使用GLOBAL IN。 -5. 如果您需要经常使用GLOBAL IN，请规划ClickHouse集群的位置，以便单个副本组驻留在不超过一个数据中心中，并且它们之间具有快速网络，以便可以完全在单个数据中心内处理查询。 - -这也是有意义的，在指定一个本地表 `GLOBAL IN` 子句，以防此本地表仅在请求者服务器上可用，并且您希望在远程服务器上使用来自它的数据。 - -### 极端值 {#extreme-values} - -除了结果之外，还可以获取结果列的最小值和最大值。 要做到这一点，设置 **极端** 设置为1。 最小值和最大值是针对数字类型、日期和带有时间的日期计算的。 对于其他列，默认值为输出。 - -An extra two rows are calculated – the minimums and maximums, respectively. These extra two rows are output in `JSON*`, `TabSeparated*`，和 `Pretty*` [格式](../../interfaces/formats.md)，与其他行分开。 它们不是其他格式的输出。 - -在 `JSON*` 格式时，极端值在一个单独的输出 ‘extremes’ 场。 在 `TabSeparated*` 格式中，该行来的主要结果之后，和之后 ‘totals’ 如果存在。 它前面有一个空行（在其他数据之后）。 在 `Pretty*` 格式中，该行被输出为一个单独的表之后的主结果，和之后 `totals` 如果存在。 - -极值计算之前的行 `LIMIT`，但之后 `LIMIT BY`. 但是，使用时 `LIMIT offset, size`，之前的行 `offset` 都包含在 `extremes`. 在流请求中，结果还可能包括少量通过的行 `LIMIT`. - -### 注 {#notes} - -该 `GROUP BY` 和 `ORDER BY` 子句不支持位置参数。 这与MySQL相矛盾，但符合标准SQL。 -例如, `GROUP BY 1, 2` will be interpreted as grouping by constants (i.e. aggregation of all rows into one). - -您可以使用同义词 (`AS` 别名）在查询的任何部分。 - -您可以在查询的任何部分而不是表达式中添加星号。 分析查询时，星号将展开为所有表列的列表（不包括 `MATERIALIZED` 和 `ALIAS` 列）。 只有少数情况下使用星号是合理的: - -- 创建表转储时。 -- 对于只包含几列的表，例如系统表。 -- 获取有关表中哪些列的信息。 在这种情况下，设置 `LIMIT 1`. 但最好使用 `DESC TABLE` 查询。 -- 当对少量柱进行强过滤时，使用 `PREWHERE`. -- 在子查询中（因为外部查询不需要的列从子查询中排除）。 - -在所有其他情况下，我们不建议使用星号，因为它只给你一个列DBMS的缺点，而不是优点。 换句话说，不建议使用星号。 - -[原始文章](https://clickhouse.tech/docs/en/query_language/select/) diff --git a/docs/zh/sql-reference/statements/select/array-join.md b/docs/zh/sql-reference/statements/select/array-join.md new file mode 120000 index 00000000000..c341801e419 --- /dev/null +++ b/docs/zh/sql-reference/statements/select/array-join.md @@ -0,0 +1 @@ +../../../../en/sql-reference/statements/select/array-join.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/select/distinct.md b/docs/zh/sql-reference/statements/select/distinct.md new file mode 120000 index 00000000000..59319557dc1 --- /dev/null +++ b/docs/zh/sql-reference/statements/select/distinct.md @@ -0,0 +1 @@ +../../../../en/sql-reference/statements/select/distinct.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/select/format.md b/docs/zh/sql-reference/statements/select/format.md new file mode 120000 index 00000000000..106b2d9ebbc --- /dev/null +++ b/docs/zh/sql-reference/statements/select/format.md @@ -0,0 +1 @@ +../../../../en/sql-reference/statements/select/format.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/select/from.md b/docs/zh/sql-reference/statements/select/from.md new file mode 120000 index 00000000000..f8ebfe655cc --- /dev/null +++ b/docs/zh/sql-reference/statements/select/from.md @@ -0,0 +1 @@ +../../../../en/sql-reference/statements/select/from.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/select/group-by.md b/docs/zh/sql-reference/statements/select/group-by.md new file mode 120000 index 00000000000..cf519ad7781 --- /dev/null +++ b/docs/zh/sql-reference/statements/select/group-by.md @@ -0,0 +1 @@ +../../../../en/sql-reference/statements/select/group-by.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/select/having.md b/docs/zh/sql-reference/statements/select/having.md new file mode 120000 index 00000000000..4a038beb126 --- /dev/null +++ b/docs/zh/sql-reference/statements/select/having.md @@ -0,0 +1 @@ +../../../../en/sql-reference/statements/select/having.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/select/index.md b/docs/zh/sql-reference/statements/select/index.md new file mode 120000 index 00000000000..9c649322c82 --- /dev/null +++ b/docs/zh/sql-reference/statements/select/index.md @@ -0,0 +1 @@ +../../../../en/sql-reference/statements/select/index.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/select/into-outfile.md b/docs/zh/sql-reference/statements/select/into-outfile.md new file mode 120000 index 00000000000..2c9c812b3d5 --- /dev/null +++ b/docs/zh/sql-reference/statements/select/into-outfile.md @@ -0,0 +1 @@ +../../../../en/sql-reference/statements/select/into-outfile.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/select/join.md b/docs/zh/sql-reference/statements/select/join.md new file mode 120000 index 00000000000..5951a105137 --- /dev/null +++ b/docs/zh/sql-reference/statements/select/join.md @@ -0,0 +1 @@ +../../../../en/sql-reference/statements/select/join.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/select/limit-by.md b/docs/zh/sql-reference/statements/select/limit-by.md new file mode 120000 index 00000000000..f3a63e9fe22 --- /dev/null +++ b/docs/zh/sql-reference/statements/select/limit-by.md @@ -0,0 +1 @@ +../../../../en/sql-reference/statements/select/limit-by.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/select/limit.md b/docs/zh/sql-reference/statements/select/limit.md new file mode 120000 index 00000000000..e0a0c632dac --- /dev/null +++ b/docs/zh/sql-reference/statements/select/limit.md @@ -0,0 +1 @@ +../../../../en/sql-reference/statements/select/limit.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/select/order-by.md b/docs/zh/sql-reference/statements/select/order-by.md new file mode 120000 index 00000000000..cc2567bce0b --- /dev/null +++ b/docs/zh/sql-reference/statements/select/order-by.md @@ -0,0 +1 @@ +../../../../en/sql-reference/statements/select/order-by.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/select/prewhere.md b/docs/zh/sql-reference/statements/select/prewhere.md new file mode 120000 index 00000000000..567fc95356f --- /dev/null +++ b/docs/zh/sql-reference/statements/select/prewhere.md @@ -0,0 +1 @@ +../../../../en/sql-reference/statements/select/prewhere.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/select/sample.md b/docs/zh/sql-reference/statements/select/sample.md new file mode 120000 index 00000000000..9df6e25d0f3 --- /dev/null +++ b/docs/zh/sql-reference/statements/select/sample.md @@ -0,0 +1 @@ +../../../../en/sql-reference/statements/select/sample.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/select/union-all.md b/docs/zh/sql-reference/statements/select/union-all.md new file mode 120000 index 00000000000..837caae2698 --- /dev/null +++ b/docs/zh/sql-reference/statements/select/union-all.md @@ -0,0 +1 @@ +../../../../en/sql-reference/statements/select/union-all.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/select/where.md b/docs/zh/sql-reference/statements/select/where.md new file mode 120000 index 00000000000..8ba28926879 --- /dev/null +++ b/docs/zh/sql-reference/statements/select/where.md @@ -0,0 +1 @@ +../../../../en/sql-reference/statements/select/where.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/select/with.md b/docs/zh/sql-reference/statements/select/with.md new file mode 120000 index 00000000000..8b7ea4db44c --- /dev/null +++ b/docs/zh/sql-reference/statements/select/with.md @@ -0,0 +1 @@ +../../../../en/sql-reference/statements/select/with.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/show.md b/docs/zh/sql-reference/statements/show.md index fa8cc94a0b6..95404f3d416 100644 --- a/docs/zh/sql-reference/statements/show.md +++ b/docs/zh/sql-reference/statements/show.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 38 toc_title: SHOW --- @@ -102,4 +102,68 @@ SHOW DICTIONARIES FROM db LIKE '%reg%' LIMIT 2 └──────────────┘ ``` +## SHOW GRANTS {#show-grants-statement} + +显示用户的权限。 + +### 语法 {#show-grants-syntax} + +``` sql +SHOW GRANTS [FOR user] +``` + +如果未指定user，则查询返回当前用户的权限。 + +## SHOW CREATE USER {#show-create-user-statement} + +显示了在使用的参数 [用户创建](create.md#create-user-statement). + +`SHOW CREATE USER` 不输出用户密码。 + +### 语法 {#show-create-user-syntax} + +``` sql +SHOW CREATE USER [name | CURRENT_USER] +``` + +## SHOW CREATE ROLE {#show-create-role-statement} + +显示了在使用的参数 [角色创建](create.md#create-role-statement) + +### 语法 {#show-create-role-syntax} + +``` sql +SHOW CREATE ROLE name +``` + +## SHOW CREATE ROW POLICY {#show-create-row-policy-statement} + +显示了在使用的参数 [创建行策略](create.md#create-row-policy-statement) + +### 语法 {#show-create-row-policy-syntax} + +``` sql +SHOW CREATE [ROW] POLICY name ON [database.]table +``` + +## SHOW CREATE QUOTA {#show-create-quota-statement} + +显示了在使用的参数 [创建配额](create.md#create-quota-statement) + +### 语法 {#show-create-row-policy-syntax} + +``` sql +SHOW CREATE QUOTA [name | CURRENT] +``` + +## SHOW CREATE SETTINGS PROFILE {#show-create-settings-profile-statement} + +显示了在使用的参数 [设置配置文件创建](create.md#create-settings-profile-statement) + +### 语法 {#show-create-row-policy-syntax} + +``` sql +SHOW CREATE [SETTINGS] PROFILE name +``` + [原始文章](https://clickhouse.tech/docs/en/query_language/show/) diff --git a/docs/zh/sql-reference/statements/system.md b/docs/zh/sql-reference/statements/system.md index d502bad1e65..067368e02a8 100644 --- a/docs/zh/sql-reference/statements/system.md +++ b/docs/zh/sql-reference/statements/system.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 37 toc_title: SYSTEM --- diff --git a/docs/zh/sql-reference/syntax.md b/docs/zh/sql-reference/syntax.md index 2543b7e0f66..b0aa9e7364f 100644 --- a/docs/zh/sql-reference/syntax.md +++ b/docs/zh/sql-reference/syntax.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 31 toc_title: "\u8BED\u6CD5" --- @@ -9,7 +9,7 @@ toc_title: "\u8BED\u6CD5" 系统中有两种类型的解析器：完整SQL解析器（递归下降解析器）和数据格式解析器（快速流解析器）。 在所有情况下，除了 `INSERT` 查询时，只使用完整的SQL解析器。 -该 `INSERT` 查询使用的分析程序: +该 `INSERT` 查询使用两个解析器: ``` sql INSERT INTO t VALUES (1, 'Hello, world'), (2, 'abc'), (3, 'def') @@ -18,11 +18,11 @@ INSERT INTO t VALUES (1, 'Hello, world'), (2, 'abc'), (3, 'def') 该 `INSERT INTO t VALUES` 片段由完整的解析器解析，并且数据 `(1, 'Hello, world'), (2, 'abc'), (3, 'def')` 由快速流解析器解析。 您也可以通过使用 [input\_format\_values\_interpret\_expressions](../operations/settings/settings.md#settings-input_format_values_interpret_expressions) 设置。 当 `input_format_values_interpret_expressions = 1`，ClickHouse首先尝试使用fast stream解析器解析值。 如果失败，ClickHouse将尝试对数据使用完整的解析器，将其视为SQL [表达式](#syntax-expressions). 数据可以有任何格式。 当接收到查询时，服务器计算不超过 [max\_query\_size](../operations/settings/settings.md#settings-max_query_size) RAM中请求的字节（默认为1MB），其余的是流解析。 -这意味着系统没有大的问题 `INSERT` 查询，就像MySQL一样。 +它允许避免与大的问题 `INSERT` 查询。 使用时 `Values` 格式为 `INSERT` 查询，它可能看起来数据被解析相同的表达式 `SELECT` 查询，但事实并非如此。 该 `Values` 格式更为有限。 -接下来我们将介绍完整的解析器。 有关格式解析器的详细信息，请参阅 [格式](../interfaces/formats.md) 科。 +本文的其余部分将介绍完整的解析器。 有关格式解析器的详细信息，请参阅 [格式](../interfaces/formats.md) 科。 ## 空间 {#spaces} @@ -30,9 +30,9 @@ INSERT INTO t VALUES (1, 'Hello, world'), (2, 'abc'), (3, 'def') ## 评论 {#comments} -支持SQL样式和C样式注释。 -SQL风格的评论：来自 `--` 直到最后 后的空间 `--` 可以省略。 -C风格的评论：来自 `/*` 到 `*/`. 这些注释可以是多行。 这里也不需要空格。 +ClickHouse支持SQL风格和C风格的注释。 +SQL风格的注释以下开头 `--` 并继续到线的末尾，一个空格后 `--` 可以省略。 +C型是从 `/*` 到 `*/`并且可以是多行，也不需要空格。 ## 关键词 {#syntax-keywords} @@ -45,7 +45,7 @@ C风格的评论：来自 `/*` 到 `*/`. 这些注释可以是多行。 这里 与标准SQL相比，所有其他关键字（包括函数名称）都是 **区分大小写**. -不保留关键字（它们只是在相应的上下文中解析为关键字）。 如果您使用 [标识符](#syntax-identifiers) 与关键字相同，将它们括在引号中。 例如，查询 `SELECT "FROM" FROM table_name` 是有效的，如果表 `table_name` 具有名称的列 `"FROM"`. +不保留关键字；它们仅在相应的上下文中被视为保留关键字。 如果您使用 [标识符](#syntax-identifiers) 使用与关键字相同的名称，将它们括在双引号或反引号中。 例如，查询 `SELECT "FROM" FROM table_name` 是有效的，如果表 `table_name` 具有名称的列 `"FROM"`. ## 标识符 {#syntax-identifiers} @@ -56,7 +56,7 @@ C风格的评论：来自 `/*` 到 `*/`. 这些注释可以是多行。 这里 - 数据类型。 - [表达式别名](#syntax-expression_aliases). -标识符可以是引号或非引号。 建议使用非引号标识符。 +标识符可以是引号或非引号。 后者是优选的。 非引号标识符必须与正则表达式匹配 `^[a-zA-Z_][0-9a-zA-Z_]*$` 并且不能等于 [关键词](#syntax-keywords). 例: `x, _1, X_y__Z123_.` @@ -64,34 +64,34 @@ C风格的评论：来自 `/*` 到 `*/`. 这些注释可以是多行。 这里 ## 文字数 {#literals} -有：数字，字符串，复合和 `NULL` 文字。 +有数字，字符串，复合和 `NULL` 文字。 ### 数字 {#numeric} -数值文本尝试进行分析: +数值文字尝试进行分析: -- 首先作为一个64位有符号的数字，使用 [strtoull](https://en.cppreference.com/w/cpp/string/byte/strtoul) 功能。 +- 首先，作为一个64位有符号的数字，使用 [strtoull](https://en.cppreference.com/w/cpp/string/byte/strtoul) 功能。 - 如果不成功，作为64位无符号数，使用 [strtoll](https://en.cppreference.com/w/cpp/string/byte/strtol) 功能。 - 如果不成功，作为一个浮点数使用 [strtod](https://en.cppreference.com/w/cpp/string/byte/strtof) 功能。 - 否则，将返回错误。 -相应的值将具有该值适合的最小类型。 +文本值具有该值适合的最小类型。 例如，1被解析为 `UInt8`，但256被解析为 `UInt16`. 有关详细信息，请参阅 [数据类型](../sql-reference/data-types/index.md). 例: `1`, `18446744073709551615`, `0xDEADBEEF`, `01`, `0.1`, `1e100`, `-1e-100`, `inf`, `nan`. ### 字符串 {#syntax-string-literal} -仅支持单引号中的字符串文字。 封闭的字符可以反斜杠转义。 以下转义序列具有相应的特殊值: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\a`, `\v`, `\xHH`. 在所有其他情况下，转义序列的格式为 `\c`，哪里 `c` 是任何字符，被转换为 `c`. 这意味着您可以使用序列 `\'`和`\\`. 该值将具有 [字符串](../sql-reference/data-types/string.md) 类型。 +仅支持单引号中的字符串文字。 封闭的字符可以反斜杠转义。 以下转义序列具有相应的特殊值: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\a`, `\v`, `\xHH`. 在所有其他情况下，转义序列的格式为 `\c`，哪里 `c` 是任何字符，被转换为 `c`. 这意味着你可以使用序列 `\'`和`\\`. 该值将具有 [字符串](../sql-reference/data-types/string.md) 类型。 -在字符串文字中需要转义的最小字符集: `'` 和 `\`. 单引号可以用单引号，文字转义 `'It\'s'` 和 `'It''s'` 是平等的。 +在字符串文字中，你至少需要转义 `'` 和 `\`. 单引号可以用单引号，文字转义 `'It\'s'` 和 `'It''s'` 是平等的。 ### 化合物 {#compound} -数组支持构造: `[1, 2, 3]` 和元组: `(1, 'Hello, world!', 2)`.. -实际上，这些不是文字，而是分别具有数组创建运算符和元组创建运算符的表达式。 +数组使用方括号构造 `[1, 2, 3]`. Nuples用圆括号构造 `(1, 'Hello, world!', 2)`. +从技术上讲，这些不是文字，而是分别具有数组创建运算符和元组创建运算符的表达式。 数组必须至少包含一个项目，元组必须至少包含两个项目。 -元组有一个特殊的用途用于 `IN` a条款 `SELECT` 查询。 元组可以作为查询的结果获得，但它们不能保存到数据库（除了 [记忆](../engines/table-engines/special/memory.md) 表）。 +有一个单独的情况下，当元组出现在 `IN` a条款 `SELECT` 查询。 查询结果可以包含元组，但元组不能保存到数据库（除了具有以下内容的表 [记忆](../engines/table-engines/special/memory.md) 发动机）。 ### NULL {#null-literal} @@ -101,13 +101,13 @@ C风格的评论：来自 `/*` 到 `*/`. 这些注释可以是多行。 这里 根据数据格式（输入或输出), `NULL` 可能有不同的表示。 有关详细信息，请参阅以下文档 [数据格式](../interfaces/formats.md#formats). -处理有许多细微差别 `NULL`. 例如，如果比较操作的至少一个参数是 `NULL`，此操作的结果也将是 `NULL`. 对于乘法，加法和其他操作也是如此。 有关详细信息，请阅读每个操作的文档。 +处理有许多细微差别 `NULL`. 例如，如果比较操作的至少一个参数是 `NULL`，此操作的结果也是 `NULL`. 对于乘法，加法和其他操作也是如此。 有关详细信息，请阅读每个操作的文档。 -在查询中，您可以检查 `NULL` 使用 [IS NULL](operators.md#operator-is-null) 和 [IS NOT NULL](operators.md) 运算符及相关功能 `isNull` 和 `isNotNull`. +在查询中，您可以检查 `NULL` 使用 [IS NULL](operators/index.md#operator-is-null) 和 [IS NOT NULL](operators/index.md) 运算符及相关功能 `isNull` 和 `isNotNull`. ## 功能 {#functions} -函数像标识符一样写入，并在括号中包含一个参数列表（可能是空的）。 与标准SQL相比，括号是必需的，即使是空的参数列表。 示例: `now()`. +函数调用像一个标识符一样写入，并在圆括号中包含一个参数列表（可能是空的）。 与标准SQL相比，括号是必需的，即使是空的参数列表。 示例: `now()`. 有常规函数和聚合函数（请参阅部分 “Aggregate functions”). 某些聚合函数可以包含括号中的两个参数列表。 示例: `quantile (0.9) (x)`. 这些聚合函数被调用 “parametric” 函数，并在第一个列表中的参数被调用 “parameters”. 不带参数的聚合函数的语法与常规函数的语法相同。 ## 运营商 {#operators} @@ -117,7 +117,7 @@ C风格的评论：来自 `/*` 到 `*/`. 这些注释可以是多行。 这里 ## 数据类型和数据库表引擎 {#data_types-and-database-table-engines} -数据类型和表引擎 `CREATE` 查询的编写方式与标识符或函数相同。 换句话说，它们可能包含也可能不包含括在括号中的参数列表。 有关详细信息，请参阅部分 “Data types,” “Table engines,” 和 “CREATE”. +数据类型和表引擎 `CREATE` 查询的编写方式与标识符或函数相同。 换句话说，它们可能包含也可能不包含括号中的参数列表。 有关详细信息，请参阅部分 “Data types,” “Table engines,” 和 “CREATE”. ## 表达式别名 {#syntax-expression_aliases} @@ -184,4 +184,4 @@ Code: 184. DB::Exception: Received from localhost:9000, 127.0.0.1. DB::Exception 表达式列表是一个或多个用逗号分隔的表达式。 函数和运算符，反过来，可以有表达式作为参数。 -[原始文章](https://clickhouse.tech/docs/en/query_language/syntax/) +[原始文章](https://clickhouse.tech/docs/en/sql_reference/syntax/) diff --git a/docs/zh/sql-reference/table-functions/file.md b/docs/zh/sql-reference/table-functions/file.md index d2f9ae7c0b0..71f84d65f21 100644 --- a/docs/zh/sql-reference/table-functions/file.md +++ b/docs/zh/sql-reference/table-functions/file.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 37 toc_title: "\u6587\u4EF6" --- diff --git a/docs/zh/sql-reference/table-functions/generate.md b/docs/zh/sql-reference/table-functions/generate.md index 84c711711d5..1b535161acb 100644 --- a/docs/zh/sql-reference/table-functions/generate.md +++ b/docs/zh/sql-reference/table-functions/generate.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 47 toc_title: generateRandom --- @@ -19,7 +19,6 @@ generateRandom('name TypeName[, name TypeName]...', [, 'random_seed'[, 'max_stri - `name` — Name of corresponding column. - `TypeName` — Type of corresponding column. -- `limit` — Number of rows to generate. - `max_array_length` — Maximum array length for all generated arrays. Defaults to `10`. - `max_string_length` — Maximum string length for all generated strings. Defaults to `10`. - `random_seed` — Specify random seed manually to produce stable results. If NULL — seed is randomly generated. @@ -31,7 +30,7 @@ generateRandom('name TypeName[, name TypeName]...', [, 'random_seed'[, 'max_stri ## 用法示例 {#usage-example} ``` sql -SELECT * FROM generateRandom('a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3), UUID)', 1, 10, 2); +SELECT * FROM generateRandom('a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3), UUID)', 1, 10, 2) LIMIT 3; ``` ``` text diff --git a/docs/zh/sql-reference/table-functions/hdfs.md b/docs/zh/sql-reference/table-functions/hdfs.md index e9c987999cd..4ea4e71f8fc 100644 --- a/docs/zh/sql-reference/table-functions/hdfs.md +++ b/docs/zh/sql-reference/table-functions/hdfs.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 45 toc_title: hdfs --- diff --git a/docs/zh/sql-reference/table-functions/index.md b/docs/zh/sql-reference/table-functions/index.md index 5f20c49ba95..1fa985a529a 100644 --- a/docs/zh/sql-reference/table-functions/index.md +++ b/docs/zh/sql-reference/table-functions/index.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_folder_title: "\u8868\u51FD\u6570" toc_priority: 34 toc_title: "\u5BFC\u8A00" @@ -12,7 +12,7 @@ toc_title: "\u5BFC\u8A00" 您可以使用表函数: -- [FROM](../statements/select.md#select-from) 《公约》条款 `SELECT` 查询。 +- [FROM](../statements/select/from.md) 《公约》条款 `SELECT` 查询。 The method for creating a temporary table that is available only in the current query. The table is deleted when the query finishes. diff --git a/docs/zh/sql-reference/table-functions/input.md b/docs/zh/sql-reference/table-functions/input.md index 72f71576729..42b354dc935 100644 --- a/docs/zh/sql-reference/table-functions/input.md +++ b/docs/zh/sql-reference/table-functions/input.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 46 toc_title: "\u8F93\u5165" --- diff --git a/docs/zh/sql-reference/table-functions/jdbc.md b/docs/zh/sql-reference/table-functions/jdbc.md index e2268b42e28..c1833462171 100644 --- a/docs/zh/sql-reference/table-functions/jdbc.md +++ b/docs/zh/sql-reference/table-functions/jdbc.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 43 toc_title: jdbc --- diff --git a/docs/zh/sql-reference/table-functions/merge.md b/docs/zh/sql-reference/table-functions/merge.md index 7304c447b1f..0e94dcc4d42 100644 --- a/docs/zh/sql-reference/table-functions/merge.md +++ b/docs/zh/sql-reference/table-functions/merge.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 38 toc_title: "\u5408\u5E76" --- diff --git a/docs/zh/sql-reference/table-functions/mysql.md b/docs/zh/sql-reference/table-functions/mysql.md index 995c45ac1a3..c54cd7d2a06 100644 --- a/docs/zh/sql-reference/table-functions/mysql.md +++ b/docs/zh/sql-reference/table-functions/mysql.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 42 toc_title: mysql --- diff --git a/docs/zh/sql-reference/table-functions/numbers.md b/docs/zh/sql-reference/table-functions/numbers.md index aaee632d5dc..e5f13d60791 100644 --- a/docs/zh/sql-reference/table-functions/numbers.md +++ b/docs/zh/sql-reference/table-functions/numbers.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 39 toc_title: "\u6570\u5B57" --- diff --git a/docs/zh/sql-reference/table-functions/odbc.md b/docs/zh/sql-reference/table-functions/odbc.md index 30000845698..95fb2277474 100644 --- a/docs/zh/sql-reference/table-functions/odbc.md +++ b/docs/zh/sql-reference/table-functions/odbc.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 44 toc_title: odbc --- diff --git a/docs/zh/sql-reference/table-functions/remote.md b/docs/zh/sql-reference/table-functions/remote.md index 46f7cd05b74..1f3bc58111b 100644 --- a/docs/zh/sql-reference/table-functions/remote.md +++ b/docs/zh/sql-reference/table-functions/remote.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 40 toc_title: "\u8FDC\u7A0B" --- diff --git a/docs/zh/sql-reference/table-functions/url.md b/docs/zh/sql-reference/table-functions/url.md index d220bb05c2c..c2efe09913a 100644 --- a/docs/zh/sql-reference/table-functions/url.md +++ b/docs/zh/sql-reference/table-functions/url.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 41 toc_title: url --- diff --git a/docs/zh/whats-new/changelog/2017.md b/docs/zh/whats-new/changelog/2017.md index 74e406cc4b2..de62730b093 100644 --- a/docs/zh/whats-new/changelog/2017.md +++ b/docs/zh/whats-new/changelog/2017.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 79 toc_title: '2017' --- @@ -37,7 +37,7 @@ toc_title: '2017' - IP trie字典的最大大小增加到128M条目。 - 添加了getSizeOfEnumType函数。 - 添加了sumWithOverflow聚合函数。 -- 增加了对Cap’n Proto输入格式的支持。 +- 增加了对Cap'n Proto输入格式的支持。 - 使用zstd算法时，您现在可以自定义压缩级别。 #### 向后不兼容的更改: {#backward-incompatible-changes} @@ -151,7 +151,7 @@ toc_title: '2017' 此版本包含以前版本1.1.54276的错误修复: - 固定 `DB::Exception: Assertion violation: !_path.empty()` 当插入到分布式表中。 -- 如果输入数据以“;”开头，则以RowBinary格式插入时修复了解析。 +- 如果输入数据以";"开头，则以RowBinary格式插入时修复了解析。 - Errors during runtime compilation of certain aggregate functions (e.g. `groupArray()`). ### 碌莽禄,拢,010-68520682\ {#clickhouse-release-1-1-54276-2017-08-16} diff --git a/docs/zh/whats-new/changelog/2018.md b/docs/zh/whats-new/changelog/2018.md index b62d8372d1a..f5f4502925a 100644 --- a/docs/zh/whats-new/changelog/2018.md +++ b/docs/zh/whats-new/changelog/2018.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 78 toc_title: '2018' --- @@ -1060,4 +1060,4 @@ toc_title: '2018' - 当在群集上执行滚动更新时，当某些副本运行旧版本的ClickHouse，而某些副本运行新版本时，复制会暂时停止，并且消息 `unknown parameter 'shard'` 出现在日志中。 更新集群的所有副本后，复制将继续。 - 如果群集服务器上运行不同版本的ClickHouse，则使用以下函数的分布式查询可能会产生不正确的结果: `varSamp`, `varPop`, `stddevSamp`, `stddevPop`, `covarSamp`, `covarPop`, `corr`. 您应该更新所有群集节点。 -## [更新日志2017](https://github.com/ClickHouse/ClickHouse/blob/master/docs/en/changelog/2017.md) {#changelog-for-2017} +## [更新日志2017](./2017.md#clickhouse-release-1-1-54327-2017-12-21) {#changelog-for-2017} diff --git a/docs/zh/whats-new/changelog/2019.md b/docs/zh/whats-new/changelog/2019.md index 05b1ed74bc3..154c92c1d23 100644 --- a/docs/zh/whats-new/changelog/2019.md +++ b/docs/zh/whats-new/changelog/2019.md @@ -1,13 +1,13 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 77 toc_title: '2019' --- -## 碌莽禄.拢.0755-88888888 {#clickhouse-release-v19-17} +## ClickHouse释放19.17 {#clickhouse-release-v19-17} -### ClickHouse释放v19.17.6.36,2019-12-27 {#clickhouse-release-v19-17-6-36-2019-12-27} +### 碌莽禄,拢,010-68520682\戮卤篓拢,010-68520682\ {#clickhouse-release-v19-17-6-36-2019-12-27} #### 错误修复 {#bug-fix} @@ -42,7 +42,7 @@ toc_title: '2019' - 现在在使用WITH TIES和LIMIT BY的情况下，将抛出一个异常。 现在可以使用TOP with LIMIT BY。 [\#7637](https://github.com/ClickHouse/ClickHouse/pull/7637) ([尼基塔\*米哈伊洛夫](https://github.com/nikitamikhaylov)) - 修复字典重新加载，如果它有 `invalidate_query`，停止更新，并在以前的更新尝试一些异常。 [\#8029](https://github.com/ClickHouse/ClickHouse/pull/8029) ([阿利沙平](https://github.com/alesapin)) -### ClickHouse释放v19.17.4.11时，2019-11-22 {#clickhouse-release-v19-17-4-11-2019-11-22} +### 碌莽禄,拢,010-68520682\戮卤篓拢,010-68520682\ {#clickhouse-release-v19-17-4-11-2019-11-22} #### 向后不兼容的更改 {#backward-incompatible-change} @@ -128,13 +128,13 @@ toc_title: '2019' - 为ClickHouse SQL方言添加了ANTLR4语法 [\#7595](https://github.com/ClickHouse/ClickHouse/issues/7595) [\#7596](https://github.com/ClickHouse/ClickHouse/pull/7596) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -## 碌莽禄.拢.0755-88888888 {#clickhouse-release-v19-16} +## ClickHouse释放19.16 {#clickhouse-release-v19-16} -#### ClickHouse版本v19.16.14.65,2020-03-25 {#clickhouse-release-v19-16-14-65-2020-03-25} +#### ClickHouse释放19.16.14.65,2020-03-25 {#clickhouse-release-v19-16-14-65-2020-03-25} - 修复了多个参数（超过10）的三元逻辑运算批量计算中的错误。 [\#8718](https://github.com/ClickHouse/ClickHouse/pull/8718) ([亚历山大\*卡扎科夫](https://github.com/Akazz)）这个错误修正是由Altinity的特殊要求回移到版本19.16的。 -#### ClickHouse释放v19.16.14.65,2020-03-05 {#clickhouse-release-v19-16-14-65-2020-03-05} +#### ClickHouse释放19.16.14.65,2020-03-05 {#clickhouse-release-v19-16-14-65-2020-03-05} - 修复分布式子查询与旧版本的CH不兼容。 修复 [\#7851](https://github.com/ClickHouse/ClickHouse/issues/7851) [（tabplubix)](https://github.com/tavplubix) @@ -154,7 +154,7 @@ toc_title: '2019' - 添加 `deduplicate_blocks_in_dependent_materialized_views` 用于控制具有实例化视图的表中幂等插入的行为的选项。 这个新功能是由Altinity的特殊要求添加到错误修正版本中的。 [\#9070](https://github.com/ClickHouse/ClickHouse/pull/9070) [(urykhy)](https://github.com/urykhy) -### ClickHouse发布版本v19.16.2.2,2019-10-30 {#clickhouse-release-v19-16-2-2-2019-10-30} +### 碌莽禄,拢,010-68520682\戮卤篓拢,010-68520682\ {#clickhouse-release-v19-16-2-2-2019-10-30} #### 向后不兼容的更改 {#backward-incompatible-change-1} @@ -611,7 +611,7 @@ toc_title: '2019' - 修复了服务器过载和全局线程池接近满时挂起查询的可能性。 这在具有大量分片（数百个）的集群上发生的机会更高，因为分布式查询为每个分片分配每个连接的线程。 例如，如果集群330分片正在处理30个并发分布式查询，则此问题可能再现。 此问题会影响从19.2开始的所有版本。 [\#6301](https://github.com/ClickHouse/ClickHouse/pull/6301) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) - 的固定逻辑 `arrayEnumerateUniqRanked` 功能。 [\#6423](https://github.com/ClickHouse/ClickHouse/pull/6423) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) - 解码符号表时修复段错误。 [\#6603](https://github.com/ClickHouse/ClickHouse/pull/6603) ([阿莫斯鸟](https://github.com/amosbird)) -- 在固定不相关的异常转换 `LowCardinality(Nullable)` to not-Nullable column in case if it doesn’t contain Nulls (e.g. in query like `SELECT CAST(CAST('Hello' AS LowCardinality(Nullable(String))) AS String)`. [\#6094](https://github.com/ClickHouse/ClickHouse/issues/6094) [\#6119](https://github.com/ClickHouse/ClickHouse/pull/6119) ([尼古拉\*科切托夫](https://github.com/KochetovNicolai)) +- 在固定不相关的异常转换 `LowCardinality(Nullable)` to not-Nullable column in case if it doesn't contain Nulls (e.g. in query like `SELECT CAST(CAST('Hello' AS LowCardinality(Nullable(String))) AS String)`. [\#6094](https://github.com/ClickHouse/ClickHouse/issues/6094) [\#6119](https://github.com/ClickHouse/ClickHouse/pull/6119) ([尼古拉\*科切托夫](https://github.com/KochetovNicolai)) - 删除描述中的额外引用 `system.settings` 桌子 [\#6696](https://github.com/ClickHouse/ClickHouse/issues/6696) [\#6699](https://github.com/ClickHouse/ClickHouse/pull/6699) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) - 避免可能的死锁 `TRUNCATE` 复制的表。 [\#6695](https://github.com/ClickHouse/ClickHouse/pull/6695) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) - 修复读取排序键的顺序。 [\#6189](https://github.com/ClickHouse/ClickHouse/pull/6189) ([安东\*波波夫](https://github.com/CurtizJ)) @@ -988,7 +988,7 @@ toc_title: '2019' - 修复ExternalLoader::reloadOutdated()中的段错误。 [\#6082](https://github.com/ClickHouse/ClickHouse/pull/6082) ([维塔利\*巴拉诺夫](https://github.com/vitlibar)) - 修复了服务器可能关闭侦听套接字但不关闭并继续提供剩余查询的情况。 您最终可能会有两个正在运行的clickhouse服务器进程。 有时，服务器可能会返回错误 `bad_function_call` 对于剩余的查询。 [\#6231](https://github.com/ClickHouse/ClickHouse/pull/6231) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) - 修复了通过ODBC，MySQL，ClickHouse和HTTP初始加载外部字典的更新字段无用和不正确的条件。 这修复 [\#6069](https://github.com/ClickHouse/ClickHouse/issues/6069) [\#6083](https://github.com/ClickHouse/ClickHouse/pull/6083) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 在固定不相关的异常转换 `LowCardinality(Nullable)` to not-Nullable column in case if it doesn’t contain Nulls (e.g. in query like `SELECT CAST(CAST('Hello' AS LowCardinality(Nullable(String))) AS String)`. [\#6094](https://github.com/ClickHouse/ClickHouse/issues/6094) [\#6119](https://github.com/ClickHouse/ClickHouse/pull/6119) ([尼古拉\*科切托夫](https://github.com/KochetovNicolai)) +- 在固定不相关的异常转换 `LowCardinality(Nullable)` to not-Nullable column in case if it doesn't contain Nulls (e.g. in query like `SELECT CAST(CAST('Hello' AS LowCardinality(Nullable(String))) AS String)`. [\#6094](https://github.com/ClickHouse/ClickHouse/issues/6094) [\#6119](https://github.com/ClickHouse/ClickHouse/pull/6119) ([尼古拉\*科切托夫](https://github.com/KochetovNicolai)) - 修复非确定性结果 “uniq” 在极少数情况下聚合函数。 该错误存在于所有ClickHouse版本。 [\#6058](https://github.com/ClickHouse/ClickHouse/pull/6058) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) - Segfault当我们在函数上设置了一点点太高的CIDR `IPv6CIDRToRange`. [\#6068](https://github.com/ClickHouse/ClickHouse/pull/6068) ([纪尧姆\*塔瑟里](https://github.com/YiuRULE)) - 修复了服务器从许多不同上下文中抛出许多异常时的小内存泄漏。 [\#6144](https://github.com/ClickHouse/ClickHouse/pull/6144) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) @@ -1342,7 +1342,7 @@ toc_title: '2019' #### 错误修复 {#bug-fixes-1} - 修复段错误 `minmax` 具有空值的索引。 [\#5246](https://github.com/ClickHouse/ClickHouse/pull/5246) ([尼基塔\*瓦西列夫](https://github.com/nikvas0)) -- 根据需要输出标记“LIMIT BY”中的所有输入列。 它修复 ‘Not found column’ 某些分布式查询中出错。 [\#5407](https://github.com/ClickHouse/ClickHouse/pull/5407) ([康斯坦丁\*潘](https://github.com/kvap)) +- 根据需要输出标记"LIMIT BY"中的所有输入列。 它修复 ‘Not found column’ 某些分布式查询中出错。 [\#5407](https://github.com/ClickHouse/ClickHouse/pull/5407) ([康斯坦丁\*潘](https://github.com/kvap)) - 修复 “Column ‘0’ already exists” 错误 `SELECT .. PREWHERE` 在具有默认值的列上 [\#5397](https://github.com/ClickHouse/ClickHouse/pull/5397) ([proller](https://github.com/proller)) - 修复 `ALTER MODIFY TTL` 查询开 `ReplicatedMergeTree`. [\#5539](https://github.com/ClickHouse/ClickHouse/pull/5539/commits) ([安东\*波波夫](https://github.com/CurtizJ)) - 当Kafka消费者无法启动时，不要使服务器崩溃。 [\#5285](https://github.com/ClickHouse/ClickHouse/pull/5285) ([伊万](https://github.com/abyss7)) @@ -1476,7 +1476,7 @@ toc_title: '2019' - 修正了潜在的空指针取消引用 `clickhouse-copier`. [\#4900](https://github.com/ClickHouse/ClickHouse/pull/4900) ([proller](https://github.com/proller)) - 修复了使用JOIN+ARRAY JOIN查询的错误 [\#4938](https://github.com/ClickHouse/ClickHouse/pull/4938) ([Artem Zuikov](https://github.com/4ertus2)) - 固定挂在服务器的启动时，字典依赖于另一个字典通过引擎数据库=字典。 [\#4962](https://github.com/ClickHouse/ClickHouse/pull/4962) ([维塔利\*巴拉诺夫](https://github.com/vitlibar)) -- Partially fix distributed\_product\_mode = local. It’s possible to allow columns of local tables in where/having/order by/… via table aliases. Throw exception if table does not have alias. There’s not possible to access to the columns without table aliases yet. [\#4986](https://github.com/ClickHouse/ClickHouse/pull/4986) ([Artem Zuikov](https://github.com/4ertus2)) +- Partially fix distributed\_product\_mode = local. It's possible to allow columns of local tables in where/having/order by/… via table aliases. Throw exception if table does not have alias. There's not possible to access to the columns without table aliases yet. [\#4986](https://github.com/ClickHouse/ClickHouse/pull/4986) ([Artem Zuikov](https://github.com/4ertus2)) - 修复潜在的错误结果 `SELECT DISTINCT` 与 `JOIN` [\#5001](https://github.com/ClickHouse/ClickHouse/pull/5001) ([Artem Zuikov](https://github.com/4ertus2)) - 修复了在执行UNION查询时可能发生的非常罕见的数据争用条件，所有查询都涉及至少两个来自系统的选择。列，系统。表，系统。部件，系统。parts\_tables或Merge系列的表，并同时执行相关表的列的更改。 [\#5189](https://github.com/ClickHouse/ClickHouse/pull/5189) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) @@ -1706,7 +1706,7 @@ toc_title: '2019' #### 改进 {#improvements-3} - 在右表列的连接上部分支持别名。 [\#4412](https://github.com/ClickHouse/ClickHouse/pull/4412) ([Artem Zuikov](https://github.com/4ertus2)) -- 结果多加入了需要正确的结果，名称为使用中子选择. 替换平的别名来源中的名称结果。 [\#4474](https://github.com/ClickHouse/ClickHouse/pull/4474) ([Artem Zuikov](https://github.com/4ertus2)) +- 多个联接的结果需要在子选择中使用正确的结果名称。 将平面别名替换为result中的源名称。 [\#4474](https://github.com/ClickHouse/ClickHouse/pull/4474) ([Artem Zuikov](https://github.com/4ertus2)) - 改进连接语句的下推逻辑。 [\#4387](https://github.com/ClickHouse/ClickHouse/pull/4387) ([伊万](https://github.com/abyss7)) #### 性能改进 {#performance-improvements-3} @@ -2071,4 +2071,4 @@ toc_title: '2019' - 修正了在下面的注释和字符串文字拼写错误 `dbms`. [\#4122](https://github.com/ClickHouse/ClickHouse/pull/4122) ([maiha](https://github.com/maiha)) - 修正了错别字的评论。 [\#4089](https://github.com/ClickHouse/ClickHouse/pull/4089) ([Evgenii Pravda](https://github.com/kvinty)) -## [2018年的更新日志](https://github.com/ClickHouse/ClickHouse/blob/master/docs/en/changelog/2018.md) {#changelog-for-2018} +## [2018年的更新日志](./2018.md#clickhouse-release-18-16) {#changelog-for-2018} diff --git a/docs/zh/whats-new/changelog/index.md b/docs/zh/whats-new/changelog/index.md index 33bb7bfd5f1..8a708afa698 100644 --- a/docs/zh/whats-new/changelog/index.md +++ b/docs/zh/whats-new/changelog/index.md @@ -1,665 +1,9 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd +toc_folder_title: "\u66F4\u65B0\u65E5\u5FD7" +toc_priority: 74 +toc_title: '2020' --- -## 碌莽禄release拢.0755-88888888 {#clickhouse-release-v20-3} - -### ClickHouse版本v20.3.4.10,2020-03-20 {#clickhouse-release-v20-3-4-10-2020-03-20} - -#### 错误修复 {#bug-fix} - -- 此版本还包含20.1.8.41的所有错误修复 -- 修复丢失 `rows_before_limit_at_least` 用于通过http进行查询（使用处理器管道）。 这修复 [\#9730](https://github.com/ClickHouse/ClickHouse/issues/9730). [\#9757](https://github.com/ClickHouse/ClickHouse/pull/9757) ([尼古拉\*科切托夫](https://github.com/KochetovNicolai)) - -### ClickHouse释放v20.3.3.6,2020-03-17 {#clickhouse-release-v20-3-3-6-2020-03-17} - -#### 错误修复 {#bug-fix-1} - -- 此版本还包含20.1.7.38的所有错误修复 -- 修复复制中的错误，如果用户在以前的版本上执行了突变，则不允许复制工作。 这修复 [\#9645](https://github.com/ClickHouse/ClickHouse/issues/9645). [\#9652](https://github.com/ClickHouse/ClickHouse/pull/9652) ([阿利沙平](https://github.com/alesapin)). 它使版本20.3再次向后兼容。 -- 添加设置 `use_compact_format_in_distributed_parts_names` 它允许写文件 `INSERT` 查询到 `Distributed` 表格格式更紧凑。 这修复 [\#9647](https://github.com/ClickHouse/ClickHouse/issues/9647). [\#9653](https://github.com/ClickHouse/ClickHouse/pull/9653) ([阿利沙平](https://github.com/alesapin)). 它使版本20.3再次向后兼容。 - -### ClickHouse版本v20.3.2.1,2020-03-12 {#clickhouse-release-v20-3-2-1-2020-03-12} - -#### 向后不兼容的更改 {#backward-incompatible-change} - -- 修正了这个问题 `file name too long` 当发送数据 `Distributed` 大量副本的表。 修复了服务器日志中显示副本凭据的问题。 磁盘上的目录名格式已更改为 `[shard{shard_index}[_replica{replica_index}]]`. [\#8911](https://github.com/ClickHouse/ClickHouse/pull/8911) ([米哈伊尔\*科罗托夫](https://github.com/millb)）升级到新版本后，您将无法在没有人工干预的情况下降级，因为旧的服务器版本无法识别新的目录格式。 如果要降级，则必须手动将相应的目录重命名为旧格式。 仅当您使用了异步时，此更改才相关 `INSERT`s到 `Distributed` 桌子 在版本20.3.3中，我们将介绍一个设置，让您逐渐启用新格式。 -- 更改了mutation命令的复制日志条目的格式。 在安装新版本之前，您必须等待旧的突变处理。 -- 实现简单的内存分析器，将堆栈跟踪转储到 `system.trace_log` 超过软分配限制的每N个字节 [\#8765](https://github.com/ClickHouse/ClickHouse/pull/8765) ([伊万](https://github.com/abyss7)) [\#9472](https://github.com/ClickHouse/ClickHouse/pull/9472) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)）列 `system.trace_log` 从改名 `timer_type` 到 `trace_type`. 这将需要改变第三方性能分析和flamegraph处理工具。 -- 在任何地方使用操作系统线程id，而不是内部线程编号。 这修复 [\#7477](https://github.com/ClickHouse/ClickHouse/issues/7477) 老 `clickhouse-client` 无法接收从服务器发送的日志，当设置 `send_logs_level` 已启用，因为结构化日志消息的名称和类型已更改。 另一方面，不同的服务器版本可以相互发送不同类型的日志。 当你不使用 `send_logs_level` 设置，你不应该关心。 [\#8954](https://github.com/ClickHouse/ClickHouse/pull/8954) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 删除 `indexHint` 功能 [\#9542](https://github.com/ClickHouse/ClickHouse/pull/9542) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 删除 `findClusterIndex`, `findClusterValue` 功能。 这修复 [\#8641](https://github.com/ClickHouse/ClickHouse/issues/8641). 如果您正在使用这些功能，请发送电子邮件至 `clickhouse-feedback@yandex-team.com` [\#9543](https://github.com/ClickHouse/ClickHouse/pull/9543) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 现在不允许创建列或添加列 `SELECT` 子查询作为默认表达式。 [\#9481](https://github.com/ClickHouse/ClickHouse/pull/9481) ([阿利沙平](https://github.com/alesapin)) -- 需要联接中的子查询的别名。 [\#9274](https://github.com/ClickHouse/ClickHouse/pull/9274) ([Artem Zuikov](https://github.com/4ertus2)) -- 改进 `ALTER MODIFY/ADD` 查询逻辑。 现在你不能 `ADD` 不带类型的列, `MODIFY` 默认表达式不改变列的类型和 `MODIFY` type不会丢失默认表达式值。 修复 [\#8669](https://github.com/ClickHouse/ClickHouse/issues/8669). [\#9227](https://github.com/ClickHouse/ClickHouse/pull/9227) ([阿利沙平](https://github.com/alesapin)) -- 要求重新启动服务器以应用日志记录配置中的更改。 这是一种临时解决方法，可以避免服务器将日志记录到已删除的日志文件中的错误（请参阅 [\#8696](https://github.com/ClickHouse/ClickHouse/issues/8696)). [\#8707](https://github.com/ClickHouse/ClickHouse/pull/8707) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- 设置 `experimental_use_processors` 默认情况下启用。 此设置允许使用新的查询管道。 这是内部重构，我们期望没有明显的变化。 如果您将看到任何问题，请将其设置为返回零。 [\#8768](https://github.com/ClickHouse/ClickHouse/pull/8768) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) - -#### 新功能 {#new-feature} - -- 添加 `Avro` 和 `AvroConfluent` 输入/输出格式 [\#8571](https://github.com/ClickHouse/ClickHouse/pull/8571) ([安德鲁Onyshchuk](https://github.com/oandrew)) [\#8957](https://github.com/ClickHouse/ClickHouse/pull/8957) ([安德鲁Onyshchuk](https://github.com/oandrew)) [\#8717](https://github.com/ClickHouse/ClickHouse/pull/8717) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 过期密钥的多线程和非阻塞更新 `cache` 字典（可选的权限读取旧的）。 [\#8303](https://github.com/ClickHouse/ClickHouse/pull/8303) ([尼基塔\*米哈伊洛夫](https://github.com/nikitamikhaylov)) -- 添加查询 `ALTER ... MATERIALIZE TTL`. 它运行突变，强制通过TTL删除过期的数据，并重新计算所有部分有关ttl的元信息。 [\#8775](https://github.com/ClickHouse/ClickHouse/pull/8775) ([安东\*波波夫](https://github.com/CurtizJ)) -- 如果需要，从HashJoin切换到MergeJoin（在磁盘上 [\#9082](https://github.com/ClickHouse/ClickHouse/pull/9082) ([Artem Zuikov](https://github.com/4ertus2)) -- 已添加 `MOVE PARTITION` 命令 `ALTER TABLE` [\#4729](https://github.com/ClickHouse/ClickHouse/issues/4729) [\#6168](https://github.com/ClickHouse/ClickHouse/pull/6168) ([纪尧姆\*塔瑟里](https://github.com/YiuRULE)) -- 动态地从配置文件重新加载存储配置。 [\#8594](https://github.com/ClickHouse/ClickHouse/pull/8594) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 允许更改 `storage_policy` 为了不那么富有的人。 [\#8107](https://github.com/ClickHouse/ClickHouse/pull/8107) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 增加了对s3存储和表功能的globs/通配符的支持。 [\#8851](https://github.com/ClickHouse/ClickHouse/pull/8851) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 执行 `bitAnd`, `bitOr`, `bitXor`, `bitNot` 为 `FixedString(N)` 数据类型。 [\#9091](https://github.com/ClickHouse/ClickHouse/pull/9091) ([纪尧姆\*塔瑟里](https://github.com/YiuRULE)) -- 添加功能 `bitCount`. 这修复 [\#8702](https://github.com/ClickHouse/ClickHouse/issues/8702). [\#8708](https://github.com/ClickHouse/ClickHouse/pull/8708) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) [\#8749](https://github.com/ClickHouse/ClickHouse/pull/8749) ([ikopylov](https://github.com/ikopylov)) -- 添加 `generateRandom` 表函数生成具有给定模式的随机行。 允许用数据填充任意测试表。 [\#8994](https://github.com/ClickHouse/ClickHouse/pull/8994) ([Ilya Yatsishin](https://github.com/qoega)) -- `JSONEachRowFormat`：当对象包含在顶层数组中时，支持特殊情况。 [\#8860](https://github.com/ClickHouse/ClickHouse/pull/8860) ([克鲁格洛夫\*帕维尔](https://github.com/Avogar)) -- 现在可以创建一个列 `DEFAULT` 取决于默认列的表达式 `ALIAS` 表达。 [\#9489](https://github.com/ClickHouse/ClickHouse/pull/9489) ([阿利沙平](https://github.com/alesapin)) -- 允许指定 `--limit` 超过源数据大小 `clickhouse-obfuscator`. 数据将以不同的随机种子重复。 [\#9155](https://github.com/ClickHouse/ClickHouse/pull/9155) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 已添加 `groupArraySample` 功能（类似于 `groupArray`）与reservior采样算法。 [\#8286](https://github.com/ClickHouse/ClickHouse/pull/8286) ([阿莫斯鸟](https://github.com/amosbird)) -- 现在，您可以监视更新队列的大小 `cache`/`complex_key_cache` 通过系统指标字典。 [\#9413](https://github.com/ClickHouse/ClickHouse/pull/9413) ([尼基塔\*米哈伊洛夫](https://github.com/nikitamikhaylov)) -- 允许使用CRLF作为CSV输出格式的行分隔符与设置 `output_format_csv_crlf_end_of_line` 设置为1 [\#8934](https://github.com/ClickHouse/ClickHouse/pull/8934) [\#8935](https://github.com/ClickHouse/ClickHouse/pull/8935) [\#8963](https://github.com/ClickHouse/ClickHouse/pull/8963) ([米哈伊尔\*科罗托夫](https://github.com/millb)) -- 实现的更多功能 [H3](https://github.com/uber/h3) API: `h3GetBaseCell`, `h3HexAreaM2`, `h3IndexesAreNeighbors`, `h3ToChildren`, `h3ToString` 和 `stringToH3` [\#8938](https://github.com/ClickHouse/ClickHouse/pull/8938) ([Nico Mandery](https://github.com/nmandery)) -- 引入新设置: `max_parser_depth` 控制最大堆栈大小并允许大型复杂查询。 这修复 [\#6681](https://github.com/ClickHouse/ClickHouse/issues/6681) 和 [\#7668](https://github.com/ClickHouse/ClickHouse/issues/7668). [\#8647](https://github.com/ClickHouse/ClickHouse/pull/8647) ([马克西姆\*斯米尔诺夫](https://github.com/qMBQx8GH)) -- 添加设置 `force_optimize_skip_unused_shards` 如果无法跳过未使用的分片，则设置为抛出 [\#8805](https://github.com/ClickHouse/ClickHouse/pull/8805) ([Azat Khuzhin](https://github.com/azat)) -- 允许配置多个磁盘/卷用于存储数据发送 `Distributed` 发动机 [\#8756](https://github.com/ClickHouse/ClickHouse/pull/8756) ([Azat Khuzhin](https://github.com/azat)) -- 支持存储策略 (``）用于存储临时数据。 [\#8750](https://github.com/ClickHouse/ClickHouse/pull/8750) ([Azat Khuzhin](https://github.com/azat)) -- 已添加 `X-ClickHouse-Exception-Code` 如果在发送数据之前引发异常，则设置的HTTP头。 这实现了 [\#4971](https://github.com/ClickHouse/ClickHouse/issues/4971). [\#8786](https://github.com/ClickHouse/ClickHouse/pull/8786) ([米哈伊尔\*科罗托夫](https://github.com/millb)) -- 添加功能 `ifNotFinite`. 这只是一个句法糖: `ifNotFinite(x, y) = isFinite(x) ? x : y`. [\#8710](https://github.com/ClickHouse/ClickHouse/pull/8710) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 已添加 `last_successful_update_time` 列中 `system.dictionaries` 表 [\#9394](https://github.com/ClickHouse/ClickHouse/pull/9394) ([尼基塔\*米哈伊洛夫](https://github.com/nikitamikhaylov)) -- 添加 `blockSerializedSize` 功能（磁盘大小不压缩) [\#8952](https://github.com/ClickHouse/ClickHouse/pull/8952) ([Azat Khuzhin](https://github.com/azat)) -- 添加功能 `moduloOrZero` [\#9358](https://github.com/ClickHouse/ClickHouse/pull/9358) ([hcz](https://github.com/hczhcz)) -- 添加系统表 `system.zeros` 和 `system.zeros_mt` 以及故事功能 `zeros()` 和 `zeros_mt()`. 表（和表函数）包含具有名称的单列 `zero` 和类型 `UInt8`. 此列包含零。 为了测试目的，需要它作为生成许多行的最快方法。 这修复 [\#6604](https://github.com/ClickHouse/ClickHouse/issues/6604) [\#9593](https://github.com/ClickHouse/ClickHouse/pull/9593) ([尼古拉\*科切托夫](https://github.com/KochetovNicolai)) - -#### 实验特点 {#experimental-feature} - -- 添加新的紧凑格式的部件 `MergeTree`-家庭表中的所有列都存储在一个文件中。 它有助于提高小型和频繁插入的性能。 旧的格式（每列一个文件）现在被称为wide。 数据存储格式由设置控制 `min_bytes_for_wide_part` 和 `min_rows_for_wide_part`. [\#8290](https://github.com/ClickHouse/ClickHouse/pull/8290) ([安东\*波波夫](https://github.com/CurtizJ)) -- 支持S3存储 `Log`, `TinyLog` 和 `StripeLog` 桌子 [\#8862](https://github.com/ClickHouse/ClickHouse/pull/8862) ([帕维尔\*科瓦连科](https://github.com/Jokser)) - -#### 错误修复 {#bug-fix-2} - -- 修正了日志消息中不一致的空格。 [\#9322](https://github.com/ClickHouse/ClickHouse/pull/9322) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 修复在创建表时将未命名元组数组展平为嵌套结构的错误。 [\#8866](https://github.com/ClickHouse/ClickHouse/pull/8866) ([achulkov2](https://github.com/achulkov2)) -- 修复了以下问题 “Too many open files” 如果有太多的文件匹配glob模式可能会发生错误 `File` 表或 `file` 表功能。 现在文件懒洋洋地打开。 这修复 [\#8857](https://github.com/ClickHouse/ClickHouse/issues/8857) [\#8861](https://github.com/ClickHouse/ClickHouse/pull/8861) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 删除临时表现在只删除临时表。 [\#8907](https://github.com/ClickHouse/ClickHouse/pull/8907) ([维塔利\*巴拉诺夫](https://github.com/vitlibar)) -- 当我们关闭服务器或分离/附加表时删除过时的分区。 [\#8602](https://github.com/ClickHouse/ClickHouse/pull/8602) ([纪尧姆\*塔瑟里](https://github.com/YiuRULE)) -- 默认磁盘如何计算可用空间 `data` 子目录。 修复了可用空间量计算不正确的问题，如果 `data` 目录被安装到一个单独的设备（罕见的情况）。 这修复 [\#7441](https://github.com/ClickHouse/ClickHouse/issues/7441) [\#9257](https://github.com/ClickHouse/ClickHouse/pull/9257) ([米哈伊尔\*科罗托夫](https://github.com/millb)) -- 允许逗号（交叉）与IN（）内部连接。 [\#9251](https://github.com/ClickHouse/ClickHouse/pull/9251) ([Artem Zuikov](https://github.com/4ertus2)) -- 如果在WHERE部分中有\[NOT\]LIKE运算符，则允许将CROSS重写为INNER JOIN。 [\#9229](https://github.com/ClickHouse/ClickHouse/pull/9229) ([Artem Zuikov](https://github.com/4ertus2)) -- 修复后可能不正确的结果 `GROUP BY` 启用设置 `distributed_aggregation_memory_efficient`. 修复 [\#9134](https://github.com/ClickHouse/ClickHouse/issues/9134). [\#9289](https://github.com/ClickHouse/ClickHouse/pull/9289) ([尼古拉\*科切托夫](https://github.com/KochetovNicolai)) -- 找到的键在缓存字典的指标中被计为错过。 [\#9411](https://github.com/ClickHouse/ClickHouse/pull/9411) ([尼基塔\*米哈伊洛夫](https://github.com/nikitamikhaylov)) -- 修复引入的复制协议不兼容 [\#8598](https://github.com/ClickHouse/ClickHouse/issues/8598). [\#9412](https://github.com/ClickHouse/ClickHouse/pull/9412) ([阿利沙平](https://github.com/alesapin)) -- 在固定的竞争条件 `queue_task_handle` 在启动 `ReplicatedMergeTree` 桌子 [\#9552](https://github.com/ClickHouse/ClickHouse/pull/9552) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 令牌 `NOT` 没有工作 `SHOW TABLES NOT LIKE` 查询 [\#8727](https://github.com/ClickHouse/ClickHouse/issues/8727) [\#8940](https://github.com/ClickHouse/ClickHouse/pull/8940) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 添加范围检查功能 `h3EdgeLengthM`. 如果没有这个检查，缓冲区溢出是可能的。 [\#8945](https://github.com/ClickHouse/ClickHouse/pull/8945) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 修复了多个参数（超过10）的三元逻辑运算批量计算中的错误。 [\#8718](https://github.com/ClickHouse/ClickHouse/pull/8718) ([亚历山大\*卡扎科夫](https://github.com/Akazz)) -- 修复PREWHERE优化的错误，这可能导致段错误或 `Inconsistent number of columns got from MergeTreeRangeReader` 例外。 [\#9024](https://github.com/ClickHouse/ClickHouse/pull/9024) ([安东\*波波夫](https://github.com/CurtizJ)) -- 修复意外 `Timeout exceeded while reading from socket` 异常，在实际超时之前以及启用查询探查器时，在安全连接上随机发生。 还添加 `connect_timeout_with_failover_secure_ms` 设置（默认100ms），这是类似于 `connect_timeout_with_failover_ms`，但用于安全连接（因为SSL握手比普通TCP连接慢) [\#9026](https://github.com/ClickHouse/ClickHouse/pull/9026) ([tavplubix](https://github.com/tavplubix)) -- 修复突变最终确定的错误，当突变可能处于以下状态时 `parts_to_do=0` 和 `is_done=0`. [\#9022](https://github.com/ClickHouse/ClickHouse/pull/9022) ([阿利沙平](https://github.com/alesapin)) -- 使用新的任何连接逻辑 `partial_merge_join` 设置。 有可能使 `ANY|ALL|SEMI LEFT` 和 `ALL INNER` 加入与 `partial_merge_join=1` 现在 [\#8932](https://github.com/ClickHouse/ClickHouse/pull/8932) ([Artem Zuikov](https://github.com/4ertus2)) -- Shard现在将从发起者获得的设置夹到shard的constaints，而不是抛出异常。 此修补程序允许将查询发送到具有另一个约束的分片。 [\#9447](https://github.com/ClickHouse/ClickHouse/pull/9447) ([维塔利\*巴拉诺夫](https://github.com/vitlibar)) -- 修正了内存管理问题 `MergeTreeReadPool`. [\#8791](https://github.com/ClickHouse/ClickHouse/pull/8791) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 修复 `toDecimal*OrNull()` 使用字符串调用时的函数系列 `e`. 修复 [\#8312](https://github.com/ClickHouse/ClickHouse/issues/8312) [\#8764](https://github.com/ClickHouse/ClickHouse/pull/8764) ([Artem Zuikov](https://github.com/4ertus2)) -- 请确保 `FORMAT Null` 不向客户端发送数据。 [\#8767](https://github.com/ClickHouse/ClickHouse/pull/8767) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- 修复时间戳中的错误 `LiveViewBlockInputStream` 不会更新。 `LIVE VIEW` 是一个实验特征。 [\#8644](https://github.com/ClickHouse/ClickHouse/pull/8644) ([vxider](https://github.com/Vxider)) [\#8625](https://github.com/ClickHouse/ClickHouse/pull/8625) ([vxider](https://github.com/Vxider)) -- 固定 `ALTER MODIFY TTL` 不允许删除旧ttl表达式的错误行为。 [\#8422](https://github.com/ClickHouse/ClickHouse/pull/8422) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 修复了MergeTreeIndexSet中的UBSan报告。 这修复 [\#9250](https://github.com/ClickHouse/ClickHouse/issues/9250) [\#9365](https://github.com/ClickHouse/ClickHouse/pull/9365) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 固定的行为 `match` 和 `extract` 当干草堆有零字节的函数。 当干草堆不变时，这种行为是错误的。 这修复 [\#9160](https://github.com/ClickHouse/ClickHouse/issues/9160) [\#9163](https://github.com/ClickHouse/ClickHouse/pull/9163) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) [\#9345](https://github.com/ClickHouse/ClickHouse/pull/9345) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 避免从apache Avro第三方库中的析构函数抛出。 [\#9066](https://github.com/ClickHouse/ClickHouse/pull/9066) ([安德鲁Onyshchuk](https://github.com/oandrew)) -- 不要提交从轮询的批次 `Kafka` 部分，因为它可能会导致数据漏洞。 [\#8876](https://github.com/ClickHouse/ClickHouse/pull/8876) ([filimonov](https://github.com/filimonov)) -- 修复 `joinGet` 使用可为空的返回类型。 https://github.com/ClickHouse/ClickHouse/issues/8919 [\#9014](https://github.com/ClickHouse/ClickHouse/pull/9014) ([阿莫斯鸟](https://github.com/amosbird)) -- 修复压缩时的数据不兼容 `T64` 编解ec [\#9016](https://github.com/ClickHouse/ClickHouse/pull/9016) ([Artem Zuikov](https://github.com/4ertus2)）修复数据类型id `T64` 在受影响的版本中导致错误(de)压缩的压缩编解ec。 [\#9033](https://github.com/ClickHouse/ClickHouse/pull/9033) ([Artem Zuikov](https://github.com/4ertus2)) -- 添加设置 `enable_early_constant_folding` 并禁用它在某些情况下，导致错误。 [\#9010](https://github.com/ClickHouse/ClickHouse/pull/9010) ([Artem Zuikov](https://github.com/4ertus2)) -- 使用VIEW修复下推谓词优化器并启用测试 [\#9011](https://github.com/ClickHouse/ClickHouse/pull/9011) ([张冬](https://github.com/zhang2014)) -- 修复段错误 `Merge` 表，从读取时可能发生 `File` 储存 [\#9387](https://github.com/ClickHouse/ClickHouse/pull/9387) ([tavplubix](https://github.com/tavplubix)) -- 添加了对存储策略的检查 `ATTACH PARTITION FROM`, `REPLACE PARTITION`, `MOVE TO TABLE`. 否则，它可以使部分数据重新启动后无法访问，并阻止ClickHouse启动。 [\#9383](https://github.com/ClickHouse/ClickHouse/pull/9383) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 修复改变，如果有TTL设置表。 [\#8800](https://github.com/ClickHouse/ClickHouse/pull/8800) ([安东\*波波夫](https://github.com/CurtizJ)) -- 修复在以下情况下可能发生的竞争条件 `SYSTEM RELOAD ALL DICTIONARIES` 在某些字典被修改/添加/删除时执行。 [\#8801](https://github.com/ClickHouse/ClickHouse/pull/8801) ([维塔利\*巴拉诺夫](https://github.com/vitlibar)) -- 在以前的版本 `Memory` 数据库引擎使用空数据路径，因此在以下位置创建表 `path` directory (e.g. `/var/lib/clickhouse/`), not in data directory of database (e.g. `/var/lib/clickhouse/db_name`). [\#8753](https://github.com/ClickHouse/ClickHouse/pull/8753) ([tavplubix](https://github.com/tavplubix)) -- 修复了关于缺少默认磁盘或策略的错误日志消息。 [\#9530](https://github.com/ClickHouse/ClickHouse/pull/9530) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 修复数组类型的bloom\_filter索引的not(has())。 [\#9407](https://github.com/ClickHouse/ClickHouse/pull/9407) ([achimbab](https://github.com/achimbab)) -- 允许表中的第一列 `Log` 引擎是别名 [\#9231](https://github.com/ClickHouse/ClickHouse/pull/9231) ([伊万](https://github.com/abyss7)) -- 从读取时修复范围的顺序 `MergeTree` 表中的一个线程。 它可能会导致例外 `MergeTreeRangeReader` 或错误的查询结果。 [\#9050](https://github.com/ClickHouse/ClickHouse/pull/9050) ([安东\*波波夫](https://github.com/CurtizJ)) -- 赂眉露\>\> `reinterpretAsFixedString` 返回 `FixedString` 而不是 `String`. [\#9052](https://github.com/ClickHouse/ClickHouse/pull/9052) ([安德鲁Onyshchuk](https://github.com/oandrew)) -- 避免极少数情况下，当用户可以得到错误的错误消息 (`Success` 而不是详细的错误描述）。 [\#9457](https://github.com/ClickHouse/ClickHouse/pull/9457) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 使用时不要崩溃 `Template` 使用空行模板格式化。 [\#8785](https://github.com/ClickHouse/ClickHouse/pull/8785) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- 系统表的元数据文件可能在错误的位置创建 [\#8653](https://github.com/ClickHouse/ClickHouse/pull/8653) ([tavplubix](https://github.com/tavplubix)）修复 [\#8581](https://github.com/ClickHouse/ClickHouse/issues/8581). -- 修复缓存字典中exception\_ptr上的数据竞赛 [\#8303](https://github.com/ClickHouse/ClickHouse/issues/8303). [\#9379](https://github.com/ClickHouse/ClickHouse/pull/9379) ([尼基塔\*米哈伊洛夫](https://github.com/nikitamikhaylov)) -- 不要为查询引发异常 `ATTACH TABLE IF NOT EXISTS`. 以前它是抛出，如果表已经存在，尽管 `IF NOT EXISTS` 条款 [\#8967](https://github.com/ClickHouse/ClickHouse/pull/8967) ([安东\*波波夫](https://github.com/CurtizJ)) -- 修复了异常消息中丢失的关闭paren。 [\#8811](https://github.com/ClickHouse/ClickHouse/pull/8811) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 避免消息 `Possible deadlock avoided` 在clickhouse客户端在交互模式下启动。 [\#9455](https://github.com/ClickHouse/ClickHouse/pull/9455) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 修复了base64编码值末尾填充格式错误的问题。 更新base64库。 这修复 [\#9491](https://github.com/ClickHouse/ClickHouse/issues/9491)，关闭 [\#9492](https://github.com/ClickHouse/ClickHouse/issues/9492) [\#9500](https://github.com/ClickHouse/ClickHouse/pull/9500) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 防止丢失数据 `Kafka` 在极少数情况下，在读取后缀之后但在提交之前发生异常。 修复 [\#9378](https://github.com/ClickHouse/ClickHouse/issues/9378) [\#9507](https://github.com/ClickHouse/ClickHouse/pull/9507) ([filimonov](https://github.com/filimonov)) -- 在固定的异常 `DROP TABLE IF EXISTS` [\#8663](https://github.com/ClickHouse/ClickHouse/pull/8663) ([尼基塔\*瓦西列夫](https://github.com/nikvas0)) -- 修复当用户尝试崩溃 `ALTER MODIFY SETTING` 对于老格式化 `MergeTree` 表引擎家族. [\#9435](https://github.com/ClickHouse/ClickHouse/pull/9435) ([阿利沙平](https://github.com/alesapin)) -- 支持在JSON相关函数中不适合Int64的UInt64号码。 更新SIMDJSON掌握。 这修复 [\#9209](https://github.com/ClickHouse/ClickHouse/issues/9209) [\#9344](https://github.com/ClickHouse/ClickHouse/pull/9344) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 当使用非严格单调函数索引时，固定执行反转谓词。 [\#9223](https://github.com/ClickHouse/ClickHouse/pull/9223) ([亚历山大\*卡扎科夫](https://github.com/Akazz)) -- 不要试图折叠 `IN` 常量在 `GROUP BY` [\#8868](https://github.com/ClickHouse/ClickHouse/pull/8868) ([阿莫斯鸟](https://github.com/amosbird)) -- 修复bug `ALTER DELETE` 突变导致索引损坏。 这修复 [\#9019](https://github.com/ClickHouse/ClickHouse/issues/9019) 和 [\#8982](https://github.com/ClickHouse/ClickHouse/issues/8982). 另外修复极其罕见的竞争条件 `ReplicatedMergeTree` `ALTER` 查询。 [\#9048](https://github.com/ClickHouse/ClickHouse/pull/9048) ([阿利沙平](https://github.com/alesapin)) -- 当设置 `compile_expressions` 被启用，你可以得到 `unexpected column` 在 `LLVMExecutableFunction` 当我们使用 `Nullable` 类型 [\#8910](https://github.com/ClickHouse/ClickHouse/pull/8910) ([纪尧姆\*塔瑟里](https://github.com/YiuRULE)) -- 多个修复 `Kafka` 引擎：1）修复在消费者组重新平衡期间出现的重复项。 2)修复罕见 ‘holes’ 当数据从一个轮询的几个分区轮询并部分提交时出现（现在我们总是处理/提交整个轮询的消息块）。 3）通过块大小修复刷新（在此之前，只有超时刷新才能正常工作）。 4）更好的订阅程序（与分配反馈）。 5）使测试工作得更快（默认时间间隔和超时）。 由于数据之前没有被块大小刷新（根据文档），pr可能会导致默认设置的性能下降（由于更频繁和更小的刷新不太理想）。 如果您在更改后遇到性能问题-请增加 `kafka_max_block_size` 在表中的更大的值（例如 `CREATE TABLE ...Engine=Kafka ... SETTINGS ... kafka_max_block_size=524288`). 修复 [\#7259](https://github.com/ClickHouse/ClickHouse/issues/7259) [\#8917](https://github.com/ClickHouse/ClickHouse/pull/8917) ([filimonov](https://github.com/filimonov)) -- 修复 `Parameter out of bound` 在PREWHERE优化之后的某些查询中出现异常。 [\#8914](https://github.com/ClickHouse/ClickHouse/pull/8914) ([Baudouin Giard](https://github.com/bgiard)) -- 修正了函数参数混合常量的情况 `arrayZip`. [\#8705](https://github.com/ClickHouse/ClickHouse/pull/8705) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 执行时 `CREATE` 查询，在存储引擎参数中折叠常量表达式。 将空数据库名称替换为当前数据库。 修复 [\#6508](https://github.com/ClickHouse/ClickHouse/issues/6508), [\#3492](https://github.com/ClickHouse/ClickHouse/issues/3492) [\#9262](https://github.com/ClickHouse/ClickHouse/pull/9262) ([tavplubix](https://github.com/tavplubix)) -- 现在不可能创建或添加具有简单循环别名的列，如 `a DEFAULT b, b DEFAULT a`. [\#9603](https://github.com/ClickHouse/ClickHouse/pull/9603) ([阿利沙平](https://github.com/alesapin)) -- 修正了双重移动可能会损坏原始部分的错误。 这是相关的，如果你使用 `ALTER TABLE MOVE` [\#8680](https://github.com/ClickHouse/ClickHouse/pull/8680) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 允许 `interval` 用于正确解析的标识符，而无需反引号。 当一个查询不能被执行，即使固定的问题 `interval` 标识符用反引号或双引号括起来。 这修复 [\#9124](https://github.com/ClickHouse/ClickHouse/issues/9124). [\#9142](https://github.com/ClickHouse/ClickHouse/pull/9142) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 修正了模糊测试和不正确的行为 `bitTestAll`/`bitTestAny` 功能。 [\#9143](https://github.com/ClickHouse/ClickHouse/pull/9143) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 修复可能的崩溃/错误的行数 `LIMIT n WITH TIES` 当有很多行等于第n行时。 [\#9464](https://github.com/ClickHouse/ClickHouse/pull/9464) ([tavplubix](https://github.com/tavplubix)) -- 使用enabled编写的部件修复突变 `insert_quorum`. [\#9463](https://github.com/ClickHouse/ClickHouse/pull/9463) ([阿利沙平](https://github.com/alesapin)) -- 修复数据竞赛破坏 `Poco::HTTPServer`. 当服务器启动并立即关闭时，可能会发生这种情况。 [\#9468](https://github.com/ClickHouse/ClickHouse/pull/9468) ([安东\*波波夫](https://github.com/CurtizJ)) -- 修复运行时显示误导性错误消息的错误 `SHOW CREATE TABLE a_table_that_does_not_exist`. [\#8899](https://github.com/ClickHouse/ClickHouse/pull/8899) ([achulkov2](https://github.com/achulkov2)) -- 固定 `Parameters are out of bound` 例外在一些罕见的情况下，当我们在一个常数 `SELECT` 条款时，我们有一个 `ORDER BY` 和一个 `LIMIT` 条款 [\#8892](https://github.com/ClickHouse/ClickHouse/pull/8892) ([纪尧姆\*塔瑟里](https://github.com/YiuRULE)) -- 修复突变定稿，当已经完成突变可以有状态 `is_done=0`. [\#9217](https://github.com/ClickHouse/ClickHouse/pull/9217) ([阿利沙平](https://github.com/alesapin)) -- 防止执行 `ALTER ADD INDEX` 对于旧语法的MergeTree表，因为它不起作用。 [\#8822](https://github.com/ClickHouse/ClickHouse/pull/8822) ([米哈伊尔\*科罗托夫](https://github.com/millb)) -- 在服务器启动时不要访问表，这 `LIVE VIEW` 取决于，所以服务器将能够启动。 也删除 `LIVE VIEW` 分离时的依赖关系 `LIVE VIEW`. `LIVE VIEW` 是一个实验特征。 [\#8824](https://github.com/ClickHouse/ClickHouse/pull/8824) ([tavplubix](https://github.com/tavplubix)) -- 修复可能的段错误 `MergeTreeRangeReader`，同时执行 `PREWHERE`. [\#9106](https://github.com/ClickHouse/ClickHouse/pull/9106) ([安东\*波波夫](https://github.com/CurtizJ)) -- 修复与列Ttl可能不匹配的校验和。 [\#9451](https://github.com/ClickHouse/ClickHouse/pull/9451) ([安东\*波波夫](https://github.com/CurtizJ)) -- 修正了一个错误，当部分没有被移动的情况下，只有一个卷的TTL规则在后台。 [\#8672](https://github.com/ClickHouse/ClickHouse/pull/8672) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 修正了这个问题 `Method createColumn() is not implemented for data type Set`. 这修复 [\#7799](https://github.com/ClickHouse/ClickHouse/issues/7799). [\#8674](https://github.com/ClickHouse/ClickHouse/pull/8674) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 现在我们将尝试更频繁地完成突变。 [\#9427](https://github.com/ClickHouse/ClickHouse/pull/9427) ([阿利沙平](https://github.com/alesapin)) -- 修复 `intDiv` 减一个常数 [\#9351](https://github.com/ClickHouse/ClickHouse/pull/9351) ([hcz](https://github.com/hczhcz)) -- 修复可能的竞争条件 `BlockIO`. [\#9356](https://github.com/ClickHouse/ClickHouse/pull/9356) ([尼古拉\*科切托夫](https://github.com/KochetovNicolai)) -- 修复尝试使用/删除时导致服务器终止的错误 `Kafka` 使用错误的参数创建的表。 [\#9513](https://github.com/ClickHouse/ClickHouse/pull/9513) ([filimonov](https://github.com/filimonov)) -- 增加了解决方法，如果操作系统返回错误的结果 `timer_create` 功能。 [\#8837](https://github.com/ClickHouse/ClickHouse/pull/8837) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 在使用固定错误 `min_marks_for_seek` 参数。 修复了分布式表中没有分片键时的错误消息，并且我们尝试跳过未使用的分片。 [\#8908](https://github.com/ClickHouse/ClickHouse/pull/8908) ([Azat Khuzhin](https://github.com/azat)) - -#### 改进 {#improvement} - -- 执行 `ALTER MODIFY/DROP` 对突变的顶部查询 `ReplicatedMergeTree*` 引擎家族. 现在 `ALTERS` 仅在元数据更新阶段阻止，之后不阻止。 [\#8701](https://github.com/ClickHouse/ClickHouse/pull/8701) ([阿利沙平](https://github.com/alesapin)) -- 添加重写交叉到内部连接的能力 `WHERE` 包含未编译名称的部分。 [\#9512](https://github.com/ClickHouse/ClickHouse/pull/9512) ([Artem Zuikov](https://github.com/4ertus2)) -- 赂眉露\>\> `SHOW TABLES` 和 `SHOW DATABASES` 查询支持 `WHERE` 表达式和 `FROM`/`IN` [\#9076](https://github.com/ClickHouse/ClickHouse/pull/9076) ([sundyli](https://github.com/sundy-li)) -- 添加了一个设置 `deduplicate_blocks_in_dependent_materialized_views`. [\#9070](https://github.com/ClickHouse/ClickHouse/pull/9070) ([urykhy](https://github.com/urykhy)) -- 在最近的变化之后，MySQL客户端开始以十六进制打印二进制字符串，从而使它们不可读 ([\#9032](https://github.com/ClickHouse/ClickHouse/issues/9032)). ClickHouse中的解决方法是将字符串列标记为UTF-8，这并不总是如此，但通常是这种情况。 [\#9079](https://github.com/ClickHouse/ClickHouse/pull/9079) ([尤里\*巴拉诺夫](https://github.com/yurriy)) -- 添加对字符串和FixedString键的支持 `sumMap` [\#8903](https://github.com/ClickHouse/ClickHouse/pull/8903) ([Baudouin Giard](https://github.com/bgiard)) -- 支持SummingMergeTree地图中的字符串键 [\#8933](https://github.com/ClickHouse/ClickHouse/pull/8933) ([Baudouin Giard](https://github.com/bgiard)) -- 即使线程已抛出异常，也向线程池发送线程终止信号 [\#8736](https://github.com/ClickHouse/ClickHouse/pull/8736) ([丁香飞](https://github.com/dingxiangfei2009)) -- 允许设置 `query_id` 在 `clickhouse-benchmark` [\#9416](https://github.com/ClickHouse/ClickHouse/pull/9416) ([安东\*波波夫](https://github.com/CurtizJ)) -- 不要让奇怪的表达 `ALTER TABLE ... PARTITION partition` 查询。 这个地址 [\#7192](https://github.com/ClickHouse/ClickHouse/issues/7192) [\#8835](https://github.com/ClickHouse/ClickHouse/pull/8835) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 表 `system.table_engines` 现在提供有关功能支持的信息（如 `supports_ttl` 或 `supports_sort_order`). [\#8830](https://github.com/ClickHouse/ClickHouse/pull/8830) ([Max Akhmedov](https://github.com/zlobober)) -- 启用 `system.metric_log` 默认情况下。 它将包含具有ProfileEvents值的行，CurrentMetrics收集与 “collect\_interval\_milliseconds” 间隔（默认情况下为一秒）。 该表非常小（通常以兆字节为单位），默认情况下收集此数据是合理的。 [\#9225](https://github.com/ClickHouse/ClickHouse/pull/9225) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- Initialize query profiler for all threads in a group, e.g. it allows to fully profile insert-queries. Fixes [\#6964](https://github.com/ClickHouse/ClickHouse/issues/6964) [\#8874](https://github.com/ClickHouse/ClickHouse/pull/8874) ([伊万](https://github.com/abyss7)) -- 现在是暂时的 `LIVE VIEW` 创建者 `CREATE LIVE VIEW name WITH TIMEOUT [42] ...` 而不是 `CREATE TEMPORARY LIVE VIEW ...`，因为以前的语法不符合 `CREATE TEMPORARY TABLE ...` [\#9131](https://github.com/ClickHouse/ClickHouse/pull/9131) ([tavplubix](https://github.com/tavplubix)) -- 添加text\_log。级别配置参数，以限制进入 `system.text_log` 表 [\#8809](https://github.com/ClickHouse/ClickHouse/pull/8809) ([Azat Khuzhin](https://github.com/azat)) -- 允许根据TTL规则将下载的部分放入磁盘/卷 [\#8598](https://github.com/ClickHouse/ClickHouse/pull/8598) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 对于外部MySQL字典，允许将MySQL连接池共同化为 “share” 他们在字典中。 此选项显着减少到MySQL服务器的连接数。 [\#9409](https://github.com/ClickHouse/ClickHouse/pull/9409) ([Clément Rodriguez](https://github.com/clemrodriguez)) -- 显示分位数的最近查询执行时间 `clickhouse-benchmark` 输出而不是插值值。 最好显示与某些查询的执行时间相对应的值。 [\#8712](https://github.com/ClickHouse/ClickHouse/pull/8712) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 可以在将数据插入到Kafka时为消息添加密钥和时间戳。 修复 [\#7198](https://github.com/ClickHouse/ClickHouse/issues/7198) [\#8969](https://github.com/ClickHouse/ClickHouse/pull/8969) ([filimonov](https://github.com/filimonov)) -- 如果服务器从终端运行，请按颜色突出显示线程号，查询id和日志优先级。 这是为了提高开发人员相关日志消息的可读性。 [\#8961](https://github.com/ClickHouse/ClickHouse/pull/8961) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 更好的异常消息，同时加载表 `Ordinary` 数据库。 [\#9527](https://github.com/ClickHouse/ClickHouse/pull/9527) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 执行 `arraySlice` 对于具有聚合函数状态的数组。 这修复 [\#9388](https://github.com/ClickHouse/ClickHouse/issues/9388) [\#9391](https://github.com/ClickHouse/ClickHouse/pull/9391) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 允许在in运算符的右侧使用常量函数和常量数组。 [\#8813](https://github.com/ClickHouse/ClickHouse/pull/8813) ([安东\*波波夫](https://github.com/CurtizJ)) -- 如果在获取系统数据时发生了zookeeper异常。副本，将其显示在单独的列中。 这实现了 [\#9137](https://github.com/ClickHouse/ClickHouse/issues/9137) [\#9138](https://github.com/ClickHouse/ClickHouse/pull/9138) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 原子删除destroy上的MergeTree数据部分。 [\#8402](https://github.com/ClickHouse/ClickHouse/pull/8402) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 支持分布式表的行级安全性。 [\#8926](https://github.com/ClickHouse/ClickHouse/pull/8926) ([伊万](https://github.com/abyss7)) -- Now we recognize suffix (like KB, KiB…) in settings values. [\#8072](https://github.com/ClickHouse/ClickHouse/pull/8072) ([米哈伊尔\*科罗托夫](https://github.com/millb)) -- 在构建大型连接的结果时防止内存不足。 [\#8637](https://github.com/ClickHouse/ClickHouse/pull/8637) ([Artem Zuikov](https://github.com/4ertus2)) -- 在交互模式下为建议添加群集名称 `clickhouse-client`. [\#8709](https://github.com/ClickHouse/ClickHouse/pull/8709) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- Initialize query profiler for all threads in a group, e.g. it allows to fully profile insert-queries [\#8820](https://github.com/ClickHouse/ClickHouse/pull/8820) ([伊万](https://github.com/abyss7)) -- 添加列 `exception_code` 在 `system.query_log` 桌子 [\#8770](https://github.com/ClickHouse/ClickHouse/pull/8770) ([米哈伊尔\*科罗托夫](https://github.com/millb)) -- 在端口上启用MySQL兼容服务器 `9004` 在默认服务器配置文件中。 在配置的例子固定密码生成命令。 [\#8771](https://github.com/ClickHouse/ClickHouse/pull/8771) ([尤里\*巴拉诺夫](https://github.com/yurriy)) -- 如果文件系统是只读的，请防止在关闭时中止。 这修复 [\#9094](https://github.com/ClickHouse/ClickHouse/issues/9094) [\#9100](https://github.com/ClickHouse/ClickHouse/pull/9100) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 当HTTP POST查询中需要长度时，更好的异常消息。 [\#9453](https://github.com/ClickHouse/ClickHouse/pull/9453) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 添加 `_path` 和 `_file` 虚拟列 `HDFS` 和 `File` 发动机和 `hdfs` 和 `file` 表函数 [\#8489](https://github.com/ClickHouse/ClickHouse/pull/8489) ([Olga Khvostikova](https://github.com/stavrolia)) -- 修复错误 `Cannot find column` 同时插入到 `MATERIALIZED VIEW` 在情况下，如果新列被添加到视图的内部表。 [\#8766](https://github.com/ClickHouse/ClickHouse/pull/8766) [\#8788](https://github.com/ClickHouse/ClickHouse/pull/8788) ([vzakaznikov](https://github.com/vzakaznikov)) [\#8788](https://github.com/ClickHouse/ClickHouse/issues/8788) [\#8806](https://github.com/ClickHouse/ClickHouse/pull/8806) ([尼古拉\*科切托夫](https://github.com/KochetovNicolai)) [\#8803](https://github.com/ClickHouse/ClickHouse/pull/8803) ([尼古拉\*科切托夫](https://github.com/KochetovNicolai)) -- 通过最终更新后发送进度（如日志）修复本机客户端-服务器协议的进度。 这可能仅与使用本机协议的某些第三方工具相关。 [\#9495](https://github.com/ClickHouse/ClickHouse/pull/9495) ([Azat Khuzhin](https://github.com/azat)) -- 添加系统指标跟踪使用MySQL协议的客户端连接数 ([\#9013](https://github.com/ClickHouse/ClickHouse/issues/9013)). [\#9015](https://github.com/ClickHouse/ClickHouse/pull/9015) ([尤金\*克里莫夫](https://github.com/Slach)) -- 从现在开始，HTTP响应将有 `X-ClickHouse-Timezone` 标题设置为相同的时区值 `SELECT timezone()` 会报告。 [\#9493](https://github.com/ClickHouse/ClickHouse/pull/9493) ([Denis Glazachev](https://github.com/traceon)) - -#### 性能改进 {#performance-improvement} - -- 使用IN提高分析指标的性能 [\#9261](https://github.com/ClickHouse/ClickHouse/pull/9261) ([安东\*波波夫](https://github.com/CurtizJ)) -- 逻辑函数+代码清理更简单，更有效的代码。 跟进到 [\#8718](https://github.com/ClickHouse/ClickHouse/issues/8718) [\#8728](https://github.com/ClickHouse/ClickHouse/pull/8728) ([亚历山大\*卡扎科夫](https://github.com/Akazz)) -- 整体性能改善（范围为5％。.通过确保使用C++20功能进行更严格的别名处理，对于受影响的查询来说，这是200％）。 [\#9304](https://github.com/ClickHouse/ClickHouse/pull/9304) ([阿莫斯鸟](https://github.com/amosbird)) -- 比较函数的内部循环更严格的别名。 [\#9327](https://github.com/ClickHouse/ClickHouse/pull/9327) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 对于算术函数的内部循环更严格的别名。 [\#9325](https://github.com/ClickHouse/ClickHouse/pull/9325) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- ColumnVector::replicate()的实现速度快约3倍，通过该实现ColumnConst::convertToFullColumn()。 在实现常数时，也将在测试中有用。 [\#9293](https://github.com/ClickHouse/ClickHouse/pull/9293) ([亚历山大\*卡扎科夫](https://github.com/Akazz)) -- 另一个小的性能改进 `ColumnVector::replicate()` （这加快了 `materialize` 函数和高阶函数），甚至进一步改进 [\#9293](https://github.com/ClickHouse/ClickHouse/issues/9293) [\#9442](https://github.com/ClickHouse/ClickHouse/pull/9442) ([亚历山大\*卡扎科夫](https://github.com/Akazz)) -- 改进的性能 `stochasticLinearRegression` 聚合函数。 此补丁由英特尔贡献。 [\#8652](https://github.com/ClickHouse/ClickHouse/pull/8652) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 提高性能 `reinterpretAsFixedString` 功能。 [\#9342](https://github.com/ClickHouse/ClickHouse/pull/9342) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 不要向客户端发送块 `Null` 处理器管道中的格式。 [\#8797](https://github.com/ClickHouse/ClickHouse/pull/8797) ([尼古拉\*科切托夫](https://github.com/KochetovNicolai)) [\#8767](https://github.com/ClickHouse/ClickHouse/pull/8767) ([Alexander Kuzmenkov](https://github.com/akuzm)) - -#### 构建/测试/包装改进 {#buildtestingpackaging-improvement} - -- 异常处理现在可以在适用于Linux的Windows子系统上正常工作。 看https://github.com/ClickHouse-Extras/libunwind/pull/3 这修复 [\#6480](https://github.com/ClickHouse/ClickHouse/issues/6480) [\#9564](https://github.com/ClickHouse/ClickHouse/pull/9564) ([sobolevsv](https://github.com/sobolevsv)) -- 替换 `readline` 与 `replxx` 对于在交互式线编辑 `clickhouse-client` [\#8416](https://github.com/ClickHouse/ClickHouse/pull/8416) ([伊万](https://github.com/abyss7)) -- 在FunctionsComparison中更好的构建时间和更少的模板实例化。 [\#9324](https://github.com/ClickHouse/ClickHouse/pull/9324) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 增加了与集成 `clang-tidy` 在线人 另请参阅 [\#6044](https://github.com/ClickHouse/ClickHouse/issues/6044) [\#9566](https://github.com/ClickHouse/ClickHouse/pull/9566) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 现在我们使用CI链接ClickHouse `lld` 即使是 `gcc`. [\#9049](https://github.com/ClickHouse/ClickHouse/pull/9049) ([阿利沙平](https://github.com/alesapin)) -- 允许随机线程调度和插入毛刺时 `THREAD_FUZZER_*` 设置环境变量。 这有助于测试。 [\#9459](https://github.com/ClickHouse/ClickHouse/pull/9459) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 在无状态测试中启用安全套接字 [\#9288](https://github.com/ClickHouse/ClickHouse/pull/9288) ([tavplubix](https://github.com/tavplubix)) -- 使SPLIT\_SHARED\_LIBRARIES=OFF更强大 [\#9156](https://github.com/ClickHouse/ClickHouse/pull/9156) ([Azat Khuzhin](https://github.com/azat)) -- 赂眉露\>\> “performance\_introspection\_and\_logging” 测试可靠的随机服务器卡住。 这可能发生在CI环境中。 另请参阅 [\#9515](https://github.com/ClickHouse/ClickHouse/issues/9515) [\#9528](https://github.com/ClickHouse/ClickHouse/pull/9528) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 在样式检查中验证XML。 [\#9550](https://github.com/ClickHouse/ClickHouse/pull/9550) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 修正了测试中的竞争条件 `00738_lock_for_inner_table`. 这个测试依赖于睡眠。 [\#9555](https://github.com/ClickHouse/ClickHouse/pull/9555) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 删除类型的性能测试 `once`. 这是在统计比较模式下运行所有性能测试（更可靠）所需的。 [\#9557](https://github.com/ClickHouse/ClickHouse/pull/9557) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 增加了算术函数的性能测试。 [\#9326](https://github.com/ClickHouse/ClickHouse/pull/9326) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 增加了性能测试 `sumMap` 和 `sumMapWithOverflow` 聚合函数。 后续行动 [\#8933](https://github.com/ClickHouse/ClickHouse/issues/8933) [\#8947](https://github.com/ClickHouse/ClickHouse/pull/8947) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 通过样式检查确保错误代码的样式。 [\#9370](https://github.com/ClickHouse/ClickHouse/pull/9370) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 为测试历史添加脚本。 [\#8796](https://github.com/ClickHouse/ClickHouse/pull/8796) ([阿利沙平](https://github.com/alesapin)) -- 添加GCC警告 `-Wsuggest-override` 找到并修复所有地方 `override` 必须使用关键字。 [\#8760](https://github.com/ClickHouse/ClickHouse/pull/8760) ([kreuzerkrieg](https://github.com/kreuzerkrieg)) -- 在Mac OS X下忽略弱符号，因为它必须被定义 [\#9538](https://github.com/ClickHouse/ClickHouse/pull/9538) ([已删除用户](https://github.com/ghost)) -- 规范性能测试中某些查询的运行时间。 这是在准备在比较模式下运行所有性能测试时完成的。 [\#9565](https://github.com/ClickHouse/ClickHouse/pull/9565) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 修复一些测试，以支持pytest与查询测试 [\#9062](https://github.com/ClickHouse/ClickHouse/pull/9062) ([伊万](https://github.com/abyss7)) -- 使用MSan在生成中启用SSL，因此在运行无状态测试时，服务器不会在启动时失败 [\#9531](https://github.com/ClickHouse/ClickHouse/pull/9531) ([tavplubix](https://github.com/tavplubix)) -- 修复测试结果中的数据库替换 [\#9384](https://github.com/ClickHouse/ClickHouse/pull/9384) ([Ilya Yatsishin](https://github.com/qoega)) -- 针对其他平台构建修复程序 [\#9381](https://github.com/ClickHouse/ClickHouse/pull/9381) ([proller](https://github.com/proller)) [\#8755](https://github.com/ClickHouse/ClickHouse/pull/8755) ([proller](https://github.com/proller)) [\#8631](https://github.com/ClickHouse/ClickHouse/pull/8631) ([proller](https://github.com/proller)) -- 将磁盘部分添加到无状态复盖率测试docker映像 [\#9213](https://github.com/ClickHouse/ClickHouse/pull/9213) ([帕维尔\*科瓦连科](https://github.com/Jokser)) -- 使用GRPC构建时，摆脱源代码树中的文件 [\#9588](https://github.com/ClickHouse/ClickHouse/pull/9588) ([阿莫斯鸟](https://github.com/amosbird)) -- 通过从上下文中删除SessionCleaner来缩短构建时间。 让SessionCleaner的代码更简单。 [\#9232](https://github.com/ClickHouse/ClickHouse/pull/9232) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 更新了clickhouse-test脚本中挂起查询的检查 [\#8858](https://github.com/ClickHouse/ClickHouse/pull/8858) ([亚历山大\*卡扎科夫](https://github.com/Akazz)) -- 从存储库中删除了一些无用的文件。 [\#8843](https://github.com/ClickHouse/ClickHouse/pull/8843) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 更改类型的数学perftests从 `once` 到 `loop`. [\#8783](https://github.com/ClickHouse/ClickHouse/pull/8783) ([尼古拉\*科切托夫](https://github.com/KochetovNicolai)) -- 添加码头镜像，它允许为我们的代码库构建交互式代码浏览器HTML报告。 [\#8781](https://github.com/ClickHouse/ClickHouse/pull/8781) ([阿利沙平](https://github.com/alesapin)）见 [Woboq代码浏览器](https://clickhouse.tech/codebrowser/html_report///ClickHouse/dbms/index.html) -- 抑制MSan下的一些测试失败。 [\#8780](https://github.com/ClickHouse/ClickHouse/pull/8780) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- 加速 “exception while insert” 测试 此测试通常在具有复盖率的调试版本中超时。 [\#8711](https://github.com/ClickHouse/ClickHouse/pull/8711) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 更新 `libcxx` 和 `libcxxabi` 为了主人 在准备 [\#9304](https://github.com/ClickHouse/ClickHouse/issues/9304) [\#9308](https://github.com/ClickHouse/ClickHouse/pull/9308) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 修复flacky测试 `00910_zookeeper_test_alter_compression_codecs`. [\#9525](https://github.com/ClickHouse/ClickHouse/pull/9525) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 清理重复的链接器标志。 确保链接器不会查找意想不到的符号。 [\#9433](https://github.com/ClickHouse/ClickHouse/pull/9433) ([阿莫斯鸟](https://github.com/amosbird)) -- 添加 `clickhouse-odbc` 驱动程序进入测试图像。 这允许通过自己的ODBC驱动程序测试ClickHouse与ClickHouse的交互。 [\#9348](https://github.com/ClickHouse/ClickHouse/pull/9348) ([filimonov](https://github.com/filimonov)) -- 修复单元测试中的几个错误。 [\#9047](https://github.com/ClickHouse/ClickHouse/pull/9047) ([阿利沙平](https://github.com/alesapin)) -- 启用 `-Wmissing-include-dirs` GCC警告消除所有不存在的包括-主要是由于CMake脚本错误 [\#8704](https://github.com/ClickHouse/ClickHouse/pull/8704) ([kreuzerkrieg](https://github.com/kreuzerkrieg)) -- 描述查询探查器无法工作的原因。 这是用于 [\#9049](https://github.com/ClickHouse/ClickHouse/issues/9049) [\#9144](https://github.com/ClickHouse/ClickHouse/pull/9144) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 将OpenSSL更新到上游主机。 修复了TLS连接可能会失败并显示消息的问题 `OpenSSL SSL_read: error:14094438:SSL routines:ssl3_read_bytes:tlsv1 alert internal error` 和 `SSL Exception: error:2400006E:random number generator::error retrieving entropy`. 该问题出现在版本20.1中。 [\#8956](https://github.com/ClickHouse/ClickHouse/pull/8956) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 更新服务器的Dockerfile [\#8893](https://github.com/ClickHouse/ClickHouse/pull/8893) ([Ilya Mazaev](https://github.com/ne-ray)) -- Build-gcc-from-sources脚本中的小修复 [\#8774](https://github.com/ClickHouse/ClickHouse/pull/8774) ([Michael Nacharov](https://github.com/mnach)) -- 替换 `numbers` 到 `zeros` 在perftests其中 `number` 不使用列。 这将导致更干净的测试结果。 [\#9600](https://github.com/ClickHouse/ClickHouse/pull/9600) ([尼古拉\*科切托夫](https://github.com/KochetovNicolai)) -- 修复列构造函数中使用initializer\_list时堆栈溢出问题。 [\#9367](https://github.com/ClickHouse/ClickHouse/pull/9367) ([已删除用户](https://github.com/ghost)) -- 将librdkafka升级到v1.3.0。 启用bund绑 `rdkafka` 和 `gsasl` mac OS X上的库 [\#9000](https://github.com/ClickHouse/ClickHouse/pull/9000) ([安德鲁Onyshchuk](https://github.com/oandrew)) -- 在GCC9.2.0上构建修复程序 [\#9306](https://github.com/ClickHouse/ClickHouse/pull/9306) ([vxider](https://github.com/Vxider)) - -## 碌莽禄.拢.0755-88888888 {#clickhouse-release-v20-1} - -### ClickHouse版本v20.1.8.41,2020-03-20 {#clickhouse-release-v20-1-8-41-2020-03-20} - -#### 错误修复 {#bug-fix-3} - -- 修复可能的永久性 `Cannot schedule a task` 错误（由于未处理的异常 `ParallelAggregatingBlockInputStream::Handler::onFinish/onFinishThread`). 这修复 [\#6833](https://github.com/ClickHouse/ClickHouse/issues/6833). [\#9154](https://github.com/ClickHouse/ClickHouse/pull/9154) ([Azat Khuzhin](https://github.com/azat)) -- 修复过多的内存消耗 `ALTER` 查询（突变）。 这修复 [\#9533](https://github.com/ClickHouse/ClickHouse/issues/9533) 和 [\#9670](https://github.com/ClickHouse/ClickHouse/issues/9670). [\#9754](https://github.com/ClickHouse/ClickHouse/pull/9754) ([阿利沙平](https://github.com/alesapin)) -- 修复外部字典DDL中反引用的错误。 这修复 [\#9619](https://github.com/ClickHouse/ClickHouse/issues/9619). [\#9734](https://github.com/ClickHouse/ClickHouse/pull/9734) ([阿利沙平](https://github.com/alesapin)) - -### ClickHouse释放v20.1.7.38,2020-03-18 {#clickhouse-release-v20-1-7-38-2020-03-18} - -#### 错误修复 {#bug-fix-4} - -- 修正了不正确的内部函数名称 `sumKahan` 和 `sumWithOverflow`. 在远程查询中使用此函数时，我会导致异常。 [\#9636](https://github.com/ClickHouse/ClickHouse/pull/9636) ([Azat Khuzhin](https://github.com/azat)). 这个问题是在所有ClickHouse版本。 -- 允许 `ALTER ON CLUSTER` 的 `Distributed` 具有内部复制的表。 这修复 [\#3268](https://github.com/ClickHouse/ClickHouse/issues/3268). [\#9617](https://github.com/ClickHouse/ClickHouse/pull/9617) ([shinoi2](https://github.com/shinoi2)). 这个问题是在所有ClickHouse版本。 -- 修复可能的异常 `Size of filter doesn't match size of column` 和 `Invalid number of rows in Chunk` 在 `MergeTreeRangeReader`. 它们可能在执行时出现 `PREWHERE` 在某些情况下。 修复 [\#9132](https://github.com/ClickHouse/ClickHouse/issues/9132). [\#9612](https://github.com/ClickHouse/ClickHouse/pull/9612) ([安东\*波波夫](https://github.com/CurtizJ)) -- 修复了这个问题：如果你编写一个简单的算术表达式，则不会保留时区 `time + 1` （与像这样的表达形成对比 `time + INTERVAL 1 SECOND`). 这修复 [\#5743](https://github.com/ClickHouse/ClickHouse/issues/5743). [\#9323](https://github.com/ClickHouse/ClickHouse/pull/9323) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)). 这个问题是在所有ClickHouse版本。 -- 现在不可能创建或添加具有简单循环别名的列，如 `a DEFAULT b, b DEFAULT a`. [\#9603](https://github.com/ClickHouse/ClickHouse/pull/9603) ([阿利沙平](https://github.com/alesapin)) -- 修复了base64编码值末尾填充格式错误的问题。 更新base64库。 这修复 [\#9491](https://github.com/ClickHouse/ClickHouse/issues/9491)，关闭 [\#9492](https://github.com/ClickHouse/ClickHouse/issues/9492) [\#9500](https://github.com/ClickHouse/ClickHouse/pull/9500) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 修复数据竞赛破坏 `Poco::HTTPServer`. 当服务器启动并立即关闭时，可能会发生这种情况。 [\#9468](https://github.com/ClickHouse/ClickHouse/pull/9468) ([安东\*波波夫](https://github.com/CurtizJ)) -- 修复可能的崩溃/错误的行数 `LIMIT n WITH TIES` 当有很多行等于第n行时。 [\#9464](https://github.com/ClickHouse/ClickHouse/pull/9464) ([tavplubix](https://github.com/tavplubix)) -- 修复与列Ttl可能不匹配的校验和。 [\#9451](https://github.com/ClickHouse/ClickHouse/pull/9451) ([安东\*波波夫](https://github.com/CurtizJ)) -- 修复当用户尝试崩溃 `ALTER MODIFY SETTING` 对于老格式化 `MergeTree` 表引擎家族. [\#9435](https://github.com/ClickHouse/ClickHouse/pull/9435) ([阿利沙平](https://github.com/alesapin)) -- 现在我们将尝试更频繁地完成突变。 [\#9427](https://github.com/ClickHouse/ClickHouse/pull/9427) ([阿利沙平](https://github.com/alesapin)) -- 修复引入的复制协议不兼容 [\#8598](https://github.com/ClickHouse/ClickHouse/issues/8598). [\#9412](https://github.com/ClickHouse/ClickHouse/pull/9412) ([阿利沙平](https://github.com/alesapin)) -- 修复数组类型的bloom\_filter索引的not(has())。 [\#9407](https://github.com/ClickHouse/ClickHouse/pull/9407) ([achimbab](https://github.com/achimbab)) -- 固定的行为 `match` 和 `extract` 当干草堆有零字节的函数。 当干草堆不变时，这种行为是错误的。 这修复 [\#9160](https://github.com/ClickHouse/ClickHouse/issues/9160) [\#9163](https://github.com/ClickHouse/ClickHouse/pull/9163) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) [\#9345](https://github.com/ClickHouse/ClickHouse/pull/9345) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) - -#### 构建/测试/包装改进 {#buildtestingpackaging-improvement-1} - -- 异常处理现在可以在适用于Linux的Windows子系统上正常工作。 看https://github.com/ClickHouse-Extras/libunwind/pull/3 这修复 [\#6480](https://github.com/ClickHouse/ClickHouse/issues/6480) [\#9564](https://github.com/ClickHouse/ClickHouse/pull/9564) ([sobolevsv](https://github.com/sobolevsv)) - -### ClickHouse释放v20.1.6.30,2020-03-05 {#clickhouse-release-v20-1-6-30-2020-03-05} - -#### 错误修复 {#bug-fix-5} - -- 修复压缩时的数据不兼容 `T64` 编解ec - [\#9039](https://github.com/ClickHouse/ClickHouse/pull/9039) [（abyss7)](https://github.com/abyss7) -- 在一个线程中从MergeTree表中读取时修复范围顺序。 修复 [\#8964](https://github.com/ClickHouse/ClickHouse/issues/8964). - [\#9050](https://github.com/ClickHouse/ClickHouse/pull/9050) [(CurtizJ))](https://github.com/CurtizJ) -- 修复可能的段错误 `MergeTreeRangeReader`，同时执行 `PREWHERE`. 修复 [\#9064](https://github.com/ClickHouse/ClickHouse/issues/9064). - [\#9106](https://github.com/ClickHouse/ClickHouse/pull/9106) [(CurtizJ))](https://github.com/CurtizJ) -- 修复 `reinterpretAsFixedString` 返回 `FixedString` 而不是 `String`. - [\#9052](https://github.com/ClickHouse/ClickHouse/pull/9052) [(oandrew)](https://github.com/oandrew) -- 修复 `joinGet` 使用可为空的返回类型。 修复 [\#8919](https://github.com/ClickHouse/ClickHouse/issues/8919) - [\#9014](https://github.com/ClickHouse/ClickHouse/pull/9014) [（amosbird)](https://github.com/amosbird) -- 修复bittestall/bitTestAny函数的模糊测试和不正确的行为。 - [\#9143](https://github.com/ClickHouse/ClickHouse/pull/9143) [（阿列克谢-米洛维多夫)](https://github.com/alexey-milovidov) -- 修复当干草堆有零字节时匹配和提取函数的行为。 当干草堆不变时，这种行为是错误的。 修复 [\#9160](https://github.com/ClickHouse/ClickHouse/issues/9160) - [\#9163](https://github.com/ClickHouse/ClickHouse/pull/9163) [（阿列克谢-米洛维多夫)](https://github.com/alexey-milovidov) -- 当使用非严格单调函数索引时，固定执行反转谓词。 修复 [\#9034](https://github.com/ClickHouse/ClickHouse/issues/9034) - [\#9223](https://github.com/ClickHouse/ClickHouse/pull/9223) [（Akazz)](https://github.com/Akazz) -- 允许重写 `CROSS` 到 `INNER JOIN` 如果有 `[NOT] LIKE` 操作员在 `WHERE` 科。 修复 [\#9191](https://github.com/ClickHouse/ClickHouse/issues/9191) - [\#9229](https://github.com/ClickHouse/ClickHouse/pull/9229) [(4ertus2)](https://github.com/4ertus2) -- 允许使用日志引擎的表中的第一列成为别名。 - [\#9231](https://github.com/ClickHouse/ClickHouse/pull/9231) [（abyss7)](https://github.com/abyss7) -- 允许逗号加入 `IN()` 进去 修复 [\#7314](https://github.com/ClickHouse/ClickHouse/issues/7314). - [\#9251](https://github.com/ClickHouse/ClickHouse/pull/9251) [(4ertus2)](https://github.com/4ertus2) -- 改进 `ALTER MODIFY/ADD` 查询逻辑。 现在你不能 `ADD` 不带类型的列, `MODIFY` 默认表达式不改变列的类型和 `MODIFY` type不会丢失默认表达式值。 修复 [\#8669](https://github.com/ClickHouse/ClickHouse/issues/8669). - [\#9227](https://github.com/ClickHouse/ClickHouse/pull/9227) [(alesapin)](https://github.com/alesapin) -- 修复突变最终确定，当已经完成突变时可以具有状态is\_done=0。 - [\#9217](https://github.com/ClickHouse/ClickHouse/pull/9217) [(alesapin)](https://github.com/alesapin) -- 碌莽禄Support: “Processors” 管道系统.数字和系统.numbers\_mt 这也修复了错误时 `max_execution_time` 不被尊重。 - [\#7796](https://github.com/ClickHouse/ClickHouse/pull/7796) [(KochetovNicolai)](https://github.com/KochetovNicolai) -- 修复错误的计数 `DictCacheKeysRequestedFound` 公制。 - [\#9411](https://github.com/ClickHouse/ClickHouse/pull/9411) [(nikitamikhaylov)](https://github.com/nikitamikhaylov) -- 添加了对存储策略的检查 `ATTACH PARTITION FROM`, `REPLACE PARTITION`, `MOVE TO TABLE` 否则可能使部分数据在重新启动后无法访问，并阻止ClickHouse启动。 - [\#9383](https://github.com/ClickHouse/ClickHouse/pull/9383) [（excitoon)](https://github.com/excitoon) -- 在固定的瑞银报告 `MergeTreeIndexSet`. 这修复 [\#9250](https://github.com/ClickHouse/ClickHouse/issues/9250) - [\#9365](https://github.com/ClickHouse/ClickHouse/pull/9365) [（阿列克谢-米洛维多夫)](https://github.com/alexey-milovidov) -- 在BlockIO中修复可能的数据集。 - [\#9356](https://github.com/ClickHouse/ClickHouse/pull/9356) [(KochetovNicolai)](https://github.com/KochetovNicolai) -- 支持 `UInt64` 在JSON相关函数中不适合Int64的数字。 更新 `SIMDJSON` 为了主人 这修复 [\#9209](https://github.com/ClickHouse/ClickHouse/issues/9209) - [\#9344](https://github.com/ClickHouse/ClickHouse/pull/9344) [（阿列克谢-米洛维多夫)](https://github.com/alexey-milovidov) -- 如果将数据目录挂载到单独的设备，则修复可用空间量计算不正确时的问题。 对于默认磁盘，计算数据子目录的可用空间。 这修复 [\#7441](https://github.com/ClickHouse/ClickHouse/issues/7441) - [\#9257](https://github.com/ClickHouse/ClickHouse/pull/9257) [（米尔布)](https://github.com/millb) -- 修复TLS连接可能会失败并显示消息时的问题 `OpenSSL SSL_read: error:14094438:SSL routines:ssl3_read_bytes:tlsv1 alert internal error and SSL Exception: error:2400006E:random number generator::error retrieving entropy.` 将OpenSSL更新到上游主机。 - [\#8956](https://github.com/ClickHouse/ClickHouse/pull/8956) [（阿列克谢-米洛维多夫)](https://github.com/alexey-milovidov) -- 执行时 `CREATE` 查询，在存储引擎参数中折叠常量表达式。 将空数据库名称替换为当前数据库。 修复 [\#6508](https://github.com/ClickHouse/ClickHouse/issues/6508), [\#3492](https://github.com/ClickHouse/ClickHouse/issues/3492). 还修复了ClickHouseDictionarySource中检查本地地址。 - [\#9262](https://github.com/ClickHouse/ClickHouse/pull/9262) [（tabplubix)](https://github.com/tavplubix) -- 修复段错误 `StorageMerge`，从StorageFile读取时可能发生。 - [\#9387](https://github.com/ClickHouse/ClickHouse/pull/9387) [（tabplubix)](https://github.com/tavplubix) -- 防止丢失数据 `Kafka` 在极少数情况下，在读取后缀之后但在提交之前发生异常。 修复 [\#9378](https://github.com/ClickHouse/ClickHouse/issues/9378). 相关: [\#7175](https://github.com/ClickHouse/ClickHouse/issues/7175) - [\#9507](https://github.com/ClickHouse/ClickHouse/pull/9507) [(菲利蒙诺夫)](https://github.com/filimonov) -- 修复尝试使用/删除时导致服务器终止的错误 `Kafka` 使用错误的参数创建的表。 修复 [\#9494](https://github.com/ClickHouse/ClickHouse/issues/9494). 结合 [\#9507](https://github.com/ClickHouse/ClickHouse/issues/9507). - [\#9513](https://github.com/ClickHouse/ClickHouse/pull/9513) [(菲利蒙诺夫)](https://github.com/filimonov) - -#### 新功能 {#new-feature-1} - -- 添加 `deduplicate_blocks_in_dependent_materialized_views` 用于控制具有实例化视图的表中幂等插入的行为的选项。 这个新功能是由Altinity的特殊要求添加到错误修正版本中的。 - [\#9070](https://github.com/ClickHouse/ClickHouse/pull/9070) [(urykhy)](https://github.com/urykhy) - -### ClickHouse版本v20.1.2.4,2020-01-22 {#clickhouse-release-v20-1-2-4-2020-01-22} - -#### 向后不兼容的更改 {#backward-incompatible-change-1} - -- 使设置 `merge_tree_uniform_read_distribution` 过时了 服务器仍可识别此设置，但无效。 [\#8308](https://github.com/ClickHouse/ClickHouse/pull/8308) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 更改函数的返回类型 `greatCircleDistance` 到 `Float32` 因为现在计算的结果是 `Float32`. [\#7993](https://github.com/ClickHouse/ClickHouse/pull/7993) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 现在预计查询参数表示为 “escaped” 格式。 例如，要传递字符串 `ab` 你必须写 `a\tb` 或 `a\b` 并分别, `a%5Ctb` 或 `a%5C%09b` 在URL中。 这是需要添加传递NULL作为的可能性 `\N`. 这修复 [\#7488](https://github.com/ClickHouse/ClickHouse/issues/7488). [\#8517](https://github.com/ClickHouse/ClickHouse/pull/8517) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 启用 `use_minimalistic_part_header_in_zookeeper` 设置 `ReplicatedMergeTree` 默认情况下。 这将显着减少存储在ZooKeeper中的数据量。 自19.1版本以来支持此设置，我们已经在多个服务的生产中使用它，半年以上没有任何问题。 如果您有机会降级到19.1以前的版本，请禁用此设置。 [\#6850](https://github.com/ClickHouse/ClickHouse/pull/6850) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 数据跳过索引已准备就绪并默认启用。 设置 `allow_experimental_data_skipping_indices`, `allow_experimental_cross_to_join_conversion` 和 `allow_experimental_multiple_joins_emulation` 现在已经过时，什么也不做。 [\#7974](https://github.com/ClickHouse/ClickHouse/pull/7974) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 添加新建 `ANY JOIN` 逻辑 `StorageJoin` 符合 `JOIN` 操作。 要在不改变行为的情况下进行升级，您需要添加 `SETTINGS any_join_distinct_right_table_keys = 1` 引擎联接表元数据或在升级后重新创建这些表。 [\#8400](https://github.com/ClickHouse/ClickHouse/pull/8400) ([Artem Zuikov](https://github.com/4ertus2)) -- 要求重新启动服务器以应用日志记录配置中的更改。 这是一种临时解决方法，可以避免服务器将日志记录到已删除的日志文件中的错误（请参阅 [\#8696](https://github.com/ClickHouse/ClickHouse/issues/8696)). [\#8707](https://github.com/ClickHouse/ClickHouse/pull/8707) ([Alexander Kuzmenkov](https://github.com/akuzm)) - -#### 新功能 {#new-feature-2} - -- 添加了有关部件路径的信息 `system.merges`. [\#8043](https://github.com/ClickHouse/ClickHouse/pull/8043) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 添加执行能力 `SYSTEM RELOAD DICTIONARY` 查询中 `ON CLUSTER` 模式 [\#8288](https://github.com/ClickHouse/ClickHouse/pull/8288) ([纪尧姆\*塔瑟里](https://github.com/YiuRULE)) -- 添加执行能力 `CREATE DICTIONARY` 查询中 `ON CLUSTER` 模式 [\#8163](https://github.com/ClickHouse/ClickHouse/pull/8163) ([阿利沙平](https://github.com/alesapin)) -- 现在用户的个人资料 `users.xml` 可以继承多个配置文件。 [\#8343](https://github.com/ClickHouse/ClickHouse/pull/8343) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) -- 已添加 `system.stack_trace` 允许查看所有服务器线程的堆栈跟踪的表。 这对于开发人员反省服务器状态非常有用。 这修复 [\#7576](https://github.com/ClickHouse/ClickHouse/issues/7576). [\#8344](https://github.com/ClickHouse/ClickHouse/pull/8344) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 添加 `DateTime64` 具有可配置子秒精度的数据类型。 [\#7170](https://github.com/ClickHouse/ClickHouse/pull/7170) ([瓦西里\*内姆科夫](https://github.com/Enmk)) -- 添加表函数 `clusterAllReplicas` 这允许查询集群中的所有节点。 [\#8493](https://github.com/ClickHouse/ClickHouse/pull/8493) ([kiran sunkari](https://github.com/kiransunkari)) -- 添加聚合函数 `categoricalInformationValue` 其计算出离散特征的信息值。 [\#8117](https://github.com/ClickHouse/ClickHouse/pull/8117) ([hcz](https://github.com/hczhcz)) -- 加快数据文件的解析 `CSV`, `TSV` 和 `JSONEachRow` 通过并行进行格式化。 [\#7780](https://github.com/ClickHouse/ClickHouse/pull/7780) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- 添加功能 `bankerRound` 它执行银行家的四舍五入。 [\#8112](https://github.com/ClickHouse/ClickHouse/pull/8112) ([hcz](https://github.com/hczhcz)) -- 支持区域名称的嵌入式字典中的更多语言: ‘ru’, ‘en’, ‘ua’, ‘uk’, ‘by’, ‘kz’, ‘tr’, ‘de’, ‘uz’, ‘lv’, ‘lt’, ‘et’, ‘pt’, ‘he’, ‘vi’. [\#8189](https://github.com/ClickHouse/ClickHouse/pull/8189) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 改进的一致性 `ANY JOIN` 逻辑 现在 `t1 ANY LEFT JOIN t2` 等于 `t2 ANY RIGHT JOIN t1`. [\#7665](https://github.com/ClickHouse/ClickHouse/pull/7665) ([Artem Zuikov](https://github.com/4ertus2)) -- 添加设置 `any_join_distinct_right_table_keys` 这使旧的行为 `ANY INNER JOIN`. [\#7665](https://github.com/ClickHouse/ClickHouse/pull/7665) ([Artem Zuikov](https://github.com/4ertus2)) -- 添加新建 `SEMI` 和 `ANTI JOIN`. 老 `ANY INNER JOIN` 行为现在可作为 `SEMI LEFT JOIN`. [\#7665](https://github.com/ClickHouse/ClickHouse/pull/7665) ([Artem Zuikov](https://github.com/4ertus2)) -- 已添加 `Distributed` 格式 `File` 发动机和 `file` 表函数，它允许从读 `.bin` 通过异步插入生成的文件 `Distributed` 桌子 [\#8535](https://github.com/ClickHouse/ClickHouse/pull/8535) ([尼古拉\*科切托夫](https://github.com/KochetovNicolai)) -- 添加可选的重置列参数 `runningAccumulate` 这允许为每个新的键值重置聚合结果。 [\#8326](https://github.com/ClickHouse/ClickHouse/pull/8326) ([谢尔盖\*科诺年科](https://github.com/kononencheg)) -- 添加使用ClickHouse作为普罗米修斯端点的能力。 [\#7900](https://github.com/ClickHouse/ClickHouse/pull/7900) ([vdimir](https://github.com/Vdimir)) -- 添加部分 `` 在 `config.xml` 这将限制允许的主机用于远程表引擎和表函数 `URL`, `S3`, `HDFS`. [\#7154](https://github.com/ClickHouse/ClickHouse/pull/7154) ([米哈伊尔\*科罗托夫](https://github.com/millb)) -- 添加功能 `greatCircleAngle` 它计算球体上的距离（以度为单位）。 [\#8105](https://github.com/ClickHouse/ClickHouse/pull/8105) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 改变地球半径与h3库一致。 [\#8105](https://github.com/ClickHouse/ClickHouse/pull/8105) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 已添加 `JSONCompactEachRow` 和 `JSONCompactEachRowWithNamesAndTypes` 输入和输出格式。 [\#7841](https://github.com/ClickHouse/ClickHouse/pull/7841) ([米哈伊尔\*科罗托夫](https://github.com/millb)) -- 增加了与文件相关的表引擎和表函数的功能 (`File`, `S3`, `URL`, `HDFS`）它允许读取和写入 `gzip` 基于附加引擎参数或文件扩展名的文件。 [\#7840](https://github.com/ClickHouse/ClickHouse/pull/7840) ([安德烈\*博德罗夫](https://github.com/apbodrov)) -- 添加了 `randomASCII(length)` 函数，生成一个字符串与一个随机集 [ASCII](https://en.wikipedia.org/wiki/ASCII#Printable_characters) 可打印字符。 [\#8401](https://github.com/ClickHouse/ClickHouse/pull/8401) ([刺刀](https://github.com/BayoNet)) -- 添加功能 `JSONExtractArrayRaw` 它返回从未解析的json数组元素上的数组 `JSON` 字符串。 [\#8081](https://github.com/ClickHouse/ClickHouse/pull/8081) ([Oleg Matrokhin](https://github.com/errx)) -- 添加 `arrayZip` 函数允许将多个长度相等的数组合成一个元组数组。 [\#8149](https://github.com/ClickHouse/ClickHouse/pull/8149) ([张冬](https://github.com/zhang2014)) -- 添加根据配置的磁盘之间移动数据的能力 `TTL`-表达式为 `*MergeTree` 表引擎家族. [\#8140](https://github.com/ClickHouse/ClickHouse/pull/8140) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 增加了新的聚合功能 `avgWeighted` 其允许计算加权平均值。 [\#7898](https://github.com/ClickHouse/ClickHouse/pull/7898) ([安德烈\*博德罗夫](https://github.com/apbodrov)) -- 现在并行解析默认启用 `TSV`, `TSKV`, `CSV` 和 `JSONEachRow` 格式。 [\#7894](https://github.com/ClickHouse/ClickHouse/pull/7894) ([尼基塔\*米哈伊洛夫](https://github.com/nikitamikhaylov)) -- 从添加几个地理功能 `H3` 图书馆: `h3GetResolution`, `h3EdgeAngle`, `h3EdgeLength`, `h3IsValid` 和 `h3kRing`. [\#8034](https://github.com/ClickHouse/ClickHouse/pull/8034) ([Konstantin Malanchev](https://github.com/hombit)) -- 增加了对brotli的支持 (`br`）压缩文件相关的存储和表函数。 这修复 [\#8156](https://github.com/ClickHouse/ClickHouse/issues/8156). [\#8526](https://github.com/ClickHouse/ClickHouse/pull/8526) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 添加 `groupBit*` 功能的 `SimpleAggregationFunction` 类型。 [\#8485](https://github.com/ClickHouse/ClickHouse/pull/8485) ([纪尧姆\*塔瑟里](https://github.com/YiuRULE)) - -#### 错误修复 {#bug-fix-6} - -- 修复重命名表 `Distributed` 引擎 修复问题 [\#7868](https://github.com/ClickHouse/ClickHouse/issues/7868). [\#8306](https://github.com/ClickHouse/ClickHouse/pull/8306) ([tavplubix](https://github.com/tavplubix)) -- 现在字典支持 `EXPRESSION` 对于非ClickHouse SQL方言中任意字符串中的属性。 [\#8098](https://github.com/ClickHouse/ClickHouse/pull/8098) ([阿利沙平](https://github.com/alesapin)) -- 修复损坏 `INSERT SELECT FROM mysql(...)` 查询。 这修复 [\#8070](https://github.com/ClickHouse/ClickHouse/issues/8070) 和 [\#7960](https://github.com/ClickHouse/ClickHouse/issues/7960). [\#8234](https://github.com/ClickHouse/ClickHouse/pull/8234) ([tavplubix](https://github.com/tavplubix)) -- 修复错误 “Mismatch column sizes” 插入默认值时 `Tuple` 从 `JSONEachRow`. 这修复 [\#5653](https://github.com/ClickHouse/ClickHouse/issues/5653). [\#8606](https://github.com/ClickHouse/ClickHouse/pull/8606) ([tavplubix](https://github.com/tavplubix)) -- 现在将在使用的情况下抛出一个异常 `WITH TIES` 旁边的 `LIMIT BY`. 还增加了使用能力 `TOP` 与 `LIMIT BY`. 这修复 [\#7472](https://github.com/ClickHouse/ClickHouse/issues/7472). [\#7637](https://github.com/ClickHouse/ClickHouse/pull/7637) ([尼基塔\*米哈伊洛夫](https://github.com/nikitamikhaylov)) -- 从新鲜的glibc版本中修复unintendent依赖关系 `clickhouse-odbc-bridge` 二进制 [\#8046](https://github.com/ClickHouse/ClickHouse/pull/8046) ([阿莫斯鸟](https://github.com/amosbird)) -- 修正错误的检查功能 `*MergeTree` 引擎家族. 现在，当我们在最后一个颗粒和最后一个标记（非最终）中有相同数量的行时，它不会失败。 [\#8047](https://github.com/ClickHouse/ClickHouse/pull/8047) ([阿利沙平](https://github.com/alesapin)) -- 修复插入 `Enum*` 列后 `ALTER` 查询，当基础数值类型等于表指定类型时。 这修复 [\#7836](https://github.com/ClickHouse/ClickHouse/issues/7836). [\#7908](https://github.com/ClickHouse/ClickHouse/pull/7908) ([安东\*波波夫](https://github.com/CurtizJ)) -- 允许非常数负 “size” 函数的参数 `substring`. 这是不允许的错误。 这修复 [\#4832](https://github.com/ClickHouse/ClickHouse/issues/4832). [\#7703](https://github.com/ClickHouse/ClickHouse/pull/7703) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 修复当错误数量的参数传递到解析错误 `(O|J)DBC` 表引擎。 [\#7709](https://github.com/ClickHouse/ClickHouse/pull/7709) ([阿利沙平](https://github.com/alesapin)) -- 将日志发送到syslog时使用正在运行的clickhouse进程的命令名。 在以前的版本中，使用空字符串而不是命令名称。 [\#8460](https://github.com/ClickHouse/ClickHouse/pull/8460) ([Michael Nacharov](https://github.com/mnach)) -- 修复检查允许的主机 `localhost`. 这个公关修复了在提供的解决方案 [\#8241](https://github.com/ClickHouse/ClickHouse/pull/8241). [\#8342](https://github.com/ClickHouse/ClickHouse/pull/8342) ([维塔利\*巴拉诺夫](https://github.com/vitlibar)) -- 修复罕见的崩溃 `argMin` 和 `argMax` 长字符串参数的函数，当结果被用于 `runningAccumulate` 功能。 这修复 [\#8325](https://github.com/ClickHouse/ClickHouse/issues/8325) [\#8341](https://github.com/ClickHouse/ClickHouse/pull/8341) ([恐龙](https://github.com/769344359)) -- 修复表的内存过度使用 `Buffer` 引擎 [\#8345](https://github.com/ClickHouse/ClickHouse/pull/8345) ([Azat Khuzhin](https://github.com/azat)) -- 修正了可以采取的功能中的潜在错误 `NULL` 作为参数之一，并返回非NULL。 [\#8196](https://github.com/ClickHouse/ClickHouse/pull/8196) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 在线程池中更好地计算后台进程的指标 `MergeTree` 表引擎. [\#8194](https://github.com/ClickHouse/ClickHouse/pull/8194) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 修复功能 `IN` 里面 `WHERE` 存在行级表筛选器时的语句。 修复 [\#6687](https://github.com/ClickHouse/ClickHouse/issues/6687) [\#8357](https://github.com/ClickHouse/ClickHouse/pull/8357) ([伊万](https://github.com/abyss7)) -- 现在，如果整数值没有完全解析设置值，则会引发异常。 [\#7678](https://github.com/ClickHouse/ClickHouse/pull/7678) ([米哈伊尔\*科罗托夫](https://github.com/millb)) -- 修复当聚合函数用于查询具有两个以上本地分片的分布式表时出现的异常。 [\#8164](https://github.com/ClickHouse/ClickHouse/pull/8164) ([小路](https://github.com/nicelulu)) -- 现在，bloom filter可以处理零长度数组，并且不执行冗余计算。 [\#8242](https://github.com/ClickHouse/ClickHouse/pull/8242) ([achimbab](https://github.com/achimbab)) -- 修正了通过匹配客户端主机来检查客户端主机是否允许 `host_regexp` 在指定 `users.xml`. [\#8241](https://github.com/ClickHouse/ClickHouse/pull/8241) ([维塔利\*巴拉诺夫](https://github.com/vitlibar)) -- 放松不明确的列检查，导致多个误报 `JOIN ON` 科。 [\#8385](https://github.com/ClickHouse/ClickHouse/pull/8385) ([Artem Zuikov](https://github.com/4ertus2)) -- 修正了可能的服务器崩溃 (`std::terminate`)当服务器不能发送或写入数据 `JSON` 或 `XML` 格式与值 `String` 数据类型（需要 `UTF-8` 验证）或使用Brotli算法或其他一些罕见情况下压缩结果数据时。 这修复 [\#7603](https://github.com/ClickHouse/ClickHouse/issues/7603) [\#8384](https://github.com/ClickHouse/ClickHouse/pull/8384) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 修复竞争条件 `StorageDistributedDirectoryMonitor` 被线人发现 这修复 [\#8364](https://github.com/ClickHouse/ClickHouse/issues/8364). [\#8383](https://github.com/ClickHouse/ClickHouse/pull/8383) ([尼古拉\*科切托夫](https://github.com/KochetovNicolai)) -- 现在背景合并 `*MergeTree` 表引擎家族更准确地保留存储策略卷顺序。 [\#8549](https://github.com/ClickHouse/ClickHouse/pull/8549) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 现在表引擎 `Kafka` 与正常工作 `Native` 格式。 这修复 [\#6731](https://github.com/ClickHouse/ClickHouse/issues/6731) [\#7337](https://github.com/ClickHouse/ClickHouse/issues/7337) [\#8003](https://github.com/ClickHouse/ClickHouse/issues/8003). [\#8016](https://github.com/ClickHouse/ClickHouse/pull/8016) ([filimonov](https://github.com/filimonov)) -- 固定格式与标题（如 `CSVWithNames`）这是抛出关于EOF表引擎的异常 `Kafka`. [\#8016](https://github.com/ClickHouse/ClickHouse/pull/8016) ([filimonov](https://github.com/filimonov)) -- 修复了从子查询右侧部分制作set的错误 `IN` 科。 这修复 [\#5767](https://github.com/ClickHouse/ClickHouse/issues/5767) 和 [\#2542](https://github.com/ClickHouse/ClickHouse/issues/2542). [\#7755](https://github.com/ClickHouse/ClickHouse/pull/7755) ([尼基塔\*米哈伊洛夫](https://github.com/nikitamikhaylov)) -- 从存储读取时修复可能的崩溃 `File`. [\#7756](https://github.com/ClickHouse/ClickHouse/pull/7756) ([尼古拉\*科切托夫](https://github.com/KochetovNicolai)) -- 在固定的文件读取 `Parquet` 包含类型列的格式 `list`. [\#8334](https://github.com/ClickHouse/ClickHouse/pull/8334) ([马苏兰](https://github.com/maxulan)) -- 修复错误 `Not found column` 对于分布式查询 `PREWHERE` 条件取决于采样键if `max_parallel_replicas > 1`. [\#7913](https://github.com/ClickHouse/ClickHouse/pull/7913) ([尼古拉\*科切托夫](https://github.com/KochetovNicolai)) -- 修复错误 `Not found column` 如果使用查询 `PREWHERE` 依赖于表的别名，结果集由于主键条件而为空。 [\#7911](https://github.com/ClickHouse/ClickHouse/pull/7911) ([尼古拉\*科切托夫](https://github.com/KochetovNicolai)) -- 函数的固定返回类型 `rand` 和 `randConstant` 在情况下 `Nullable` 争论。 现在函数总是返回 `UInt32` 而且从来没有 `Nullable(UInt32)`. [\#8204](https://github.com/ClickHouse/ClickHouse/pull/8204) ([尼古拉\*科切托夫](https://github.com/KochetovNicolai)) -- 禁用谓词下推 `WITH FILL` 表达。 这修复 [\#7784](https://github.com/ClickHouse/ClickHouse/issues/7784). [\#7789](https://github.com/ClickHouse/ClickHouse/pull/7789) ([张冬](https://github.com/zhang2014)) -- 修正错误 `count()` 结果 `SummingMergeTree` 当 `FINAL` 部分被使用。 [\#3280](https://github.com/ClickHouse/ClickHouse/issues/3280) [\#7786](https://github.com/ClickHouse/ClickHouse/pull/7786) ([尼基塔\*米哈伊洛夫](https://github.com/nikitamikhaylov)) -- 修复来自远程服务器的常量函数可能不正确的结果。 它发生在具有以下功能的查询中 `version()`, `uptime()` 等。 它为不同的服务器返回不同的常量值。 这修复 [\#7666](https://github.com/ClickHouse/ClickHouse/issues/7666). [\#7689](https://github.com/ClickHouse/ClickHouse/pull/7689) ([尼古拉\*科切托夫](https://github.com/KochetovNicolai)) -- 修复下推谓词优化中导致错误结果的复杂错误。 这解决了下推谓词优化的很多问题。 [\#8503](https://github.com/ClickHouse/ClickHouse/pull/8503) ([张冬](https://github.com/zhang2014)) -- 修复崩溃 `CREATE TABLE .. AS dictionary` 查询。 [\#8508](https://github.com/ClickHouse/ClickHouse/pull/8508) ([Azat Khuzhin](https://github.com/azat)) -- 一些改进ClickHouse语法 `.g4` 文件 [\#8294](https://github.com/ClickHouse/ClickHouse/pull/8294) ([太阳里](https://github.com/taiyang-li)) -- 修复导致崩溃的错误 `JOIN`s与表与发动机 `Join`. 这修复 [\#7556](https://github.com/ClickHouse/ClickHouse/issues/7556) [\#8254](https://github.com/ClickHouse/ClickHouse/issues/8254) [\#7915](https://github.com/ClickHouse/ClickHouse/issues/7915) [\#8100](https://github.com/ClickHouse/ClickHouse/issues/8100). [\#8298](https://github.com/ClickHouse/ClickHouse/pull/8298) ([Artem Zuikov](https://github.com/4ertus2)) -- 修复冗余字典重新加载 `CREATE DATABASE`. [\#7916](https://github.com/ClickHouse/ClickHouse/pull/7916) ([Azat Khuzhin](https://github.com/azat)) -- 限制从读取流的最大数量 `StorageFile` 和 `StorageHDFS`. 修复https://github.com/ClickHouse/ClickHouse/issues/7650. [\#7981](https://github.com/ClickHouse/ClickHouse/pull/7981) ([阿利沙平](https://github.com/alesapin)) -- 修复bug `ALTER ... MODIFY ... CODEC` 查询，当用户同时指定默认表达式和编解ec。 修复 [8593](https://github.com/ClickHouse/ClickHouse/issues/8593). [\#8614](https://github.com/ClickHouse/ClickHouse/pull/8614) ([阿利沙平](https://github.com/alesapin)) -- 修复列的后台合并错误 `SimpleAggregateFunction(LowCardinality)` 类型。 [\#8613](https://github.com/ClickHouse/ClickHouse/pull/8613) ([尼古拉\*科切托夫](https://github.com/KochetovNicolai)) -- 固定类型签入功能 `toDateTime64`. [\#8375](https://github.com/ClickHouse/ClickHouse/pull/8375) ([瓦西里\*内姆科夫](https://github.com/Enmk)) -- 现在服务器不崩溃 `LEFT` 或 `FULL JOIN` 与和加入引擎和不支持 `join_use_nulls` 设置。 [\#8479](https://github.com/ClickHouse/ClickHouse/pull/8479) ([Artem Zuikov](https://github.com/4ertus2)) -- 现在 `DROP DICTIONARY IF EXISTS db.dict` 查询不会抛出异常，如果 `db` 根本不存在 [\#8185](https://github.com/ClickHouse/ClickHouse/pull/8185) ([维塔利\*巴拉诺夫](https://github.com/vitlibar)) -- 修复表函数中可能出现的崩溃 (`file`, `mysql`, `remote`）引用删除引起的 `IStorage` 对象。 修复插入表函数时指定的列的不正确解析。 [\#7762](https://github.com/ClickHouse/ClickHouse/pull/7762) ([tavplubix](https://github.com/tavplubix)) -- 确保网络启动前 `clickhouse-server`. 这修复 [\#7507](https://github.com/ClickHouse/ClickHouse/issues/7507). [\#8570](https://github.com/ClickHouse/ClickHouse/pull/8570) ([余志昌](https://github.com/yuzhichang)) -- 修复安全连接的超时处理，因此查询不会无限挂起。 这修复 [\#8126](https://github.com/ClickHouse/ClickHouse/issues/8126). [\#8128](https://github.com/ClickHouse/ClickHouse/pull/8128) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 修复 `clickhouse-copier`并发工人之间的冗余争用。 [\#7816](https://github.com/ClickHouse/ClickHouse/pull/7816) ([丁香飞](https://github.com/dingxiangfei2009)) -- 现在突变不会跳过附加的部分，即使它们的突变版本比当前的突变版本大。 [\#7812](https://github.com/ClickHouse/ClickHouse/pull/7812) ([余志昌](https://github.com/yuzhichang)) [\#8250](https://github.com/ClickHouse/ClickHouse/pull/8250) ([阿利沙平](https://github.com/alesapin)) -- 忽略冗余副本 `*MergeTree` 数据部分移动到另一个磁盘和服务器重新启动后。 [\#7810](https://github.com/ClickHouse/ClickHouse/pull/7810) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 修复崩溃 `FULL JOIN` 与 `LowCardinality` 在 `JOIN` 钥匙 [\#8252](https://github.com/ClickHouse/ClickHouse/pull/8252) ([Artem Zuikov](https://github.com/4ertus2)) -- 禁止在插入查询中多次使用列名，如 `INSERT INTO tbl (x, y, x)`. 这修复 [\#5465](https://github.com/ClickHouse/ClickHouse/issues/5465), [\#7681](https://github.com/ClickHouse/ClickHouse/issues/7681). [\#7685](https://github.com/ClickHouse/ClickHouse/pull/7685) ([阿利沙平](https://github.com/alesapin)) -- 增加了回退，用于检测未知Cpu的物理CPU内核数量（使用逻辑CPU内核数量）。 这修复 [\#5239](https://github.com/ClickHouse/ClickHouse/issues/5239). [\#7726](https://github.com/ClickHouse/ClickHouse/pull/7726) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 修复 `There's no column` 实例化列和别名列出错。 [\#8210](https://github.com/ClickHouse/ClickHouse/pull/8210) ([Artem Zuikov](https://github.com/4ertus2)) -- 固定切断崩溃时 `EXISTS` 查询没有使用 `TABLE` 或 `DICTIONARY` 预选赛 就像 `EXISTS t`. 这修复 [\#8172](https://github.com/ClickHouse/ClickHouse/issues/8172). 此错误在版本19.17中引入。 [\#8213](https://github.com/ClickHouse/ClickHouse/pull/8213) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 修复罕见错误 `"Sizes of columns doesn't match"` 使用时可能会出现 `SimpleAggregateFunction` 列。 [\#7790](https://github.com/ClickHouse/ClickHouse/pull/7790) ([Boris Granveaud](https://github.com/bgranvea)) -- 修正错误，其中用户空 `allow_databases` 可以访问所有数据库（和相同的 `allow_dictionaries`). [\#7793](https://github.com/ClickHouse/ClickHouse/pull/7793) ([DeifyTheGod](https://github.com/DeifyTheGod)) -- 修复客户端崩溃时，服务器已经从客户端断开连接。 [\#8071](https://github.com/ClickHouse/ClickHouse/pull/8071) ([Azat Khuzhin](https://github.com/azat)) -- 修复 `ORDER BY` 在按主键前缀和非主键后缀排序的情况下的行为。 [\#7759](https://github.com/ClickHouse/ClickHouse/pull/7759) ([安东\*波波夫](https://github.com/CurtizJ)) -- 检查表中是否存在合格列。 这修复 [\#6836](https://github.com/ClickHouse/ClickHouse/issues/6836). [\#7758](https://github.com/ClickHouse/ClickHouse/pull/7758) ([Artem Zuikov](https://github.com/4ertus2)) -- 固定行为 `ALTER MOVE` 合并完成后立即运行移动指定的超部分。 修复 [\#8103](https://github.com/ClickHouse/ClickHouse/issues/8103). [\#8104](https://github.com/ClickHouse/ClickHouse/pull/8104) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 使用时修复可能的服务器崩溃 `UNION` 具有不同数量的列。 修复 [\#7279](https://github.com/ClickHouse/ClickHouse/issues/7279). [\#7929](https://github.com/ClickHouse/ClickHouse/pull/7929) ([尼古拉\*科切托夫](https://github.com/KochetovNicolai)) -- 修复函数结果子字符串的大小 `substr` 负大小。 [\#8589](https://github.com/ClickHouse/ClickHouse/pull/8589) ([尼古拉\*科切托夫](https://github.com/KochetovNicolai)) -- 现在服务器不执行部分突变 `MergeTree` 如果后台池中没有足够的可用线程。 [\#8588](https://github.com/ClickHouse/ClickHouse/pull/8588) ([tavplubix](https://github.com/tavplubix)) -- 修复格式化时的小错字 `UNION ALL` AST. [\#7999](https://github.com/ClickHouse/ClickHouse/pull/7999) ([litao91](https://github.com/litao91)) -- 修正了负数不正确的布隆过滤结果。 这修复 [\#8317](https://github.com/ClickHouse/ClickHouse/issues/8317). [\#8566](https://github.com/ClickHouse/ClickHouse/pull/8566) ([张冬](https://github.com/zhang2014)) -- 在解压缩固定潜在的缓冲区溢出。 恶意用户可以传递捏造的压缩数据，这将导致缓冲区后读取。 这个问题是由Yandex信息安全团队的Eldar Zaitov发现的。 [\#8404](https://github.com/ClickHouse/ClickHouse/pull/8404) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 修复因整数溢出而导致的错误结果 `arrayIntersect`. [\#7777](https://github.com/ClickHouse/ClickHouse/pull/7777) ([尼古拉\*科切托夫](https://github.com/KochetovNicolai)) -- 现在 `OPTIMIZE TABLE` query不会等待脱机副本执行该操作。 [\#8314](https://github.com/ClickHouse/ClickHouse/pull/8314) ([javi santana](https://github.com/javisantana)) -- 固定 `ALTER TTL` 解析器 `Replicated*MergeTree` 桌子 [\#8318](https://github.com/ClickHouse/ClickHouse/pull/8318) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 修复服务器和客户端之间的通信，以便服务器在查询失败后读取临时表信息。 [\#8084](https://github.com/ClickHouse/ClickHouse/pull/8084) ([Azat Khuzhin](https://github.com/azat)) -- 修复 `bitmapAnd` 在聚合位图和标量位图相交时出现函数错误。 [\#8082](https://github.com/ClickHouse/ClickHouse/pull/8082) ([黄月](https://github.com/moon03432)) -- 完善的定义 `ZXid` 根据动物园管理员的程序员指南，它修复了错误 `clickhouse-cluster-copier`. [\#8088](https://github.com/ClickHouse/ClickHouse/pull/8088) ([丁香飞](https://github.com/dingxiangfei2009)) -- `odbc` 表函数现在尊重 `external_table_functions_use_nulls` 设置。 [\#7506](https://github.com/ClickHouse/ClickHouse/pull/7506) ([瓦西里\*内姆科夫](https://github.com/Enmk)) -- 修正了导致罕见的数据竞赛的错误。 [\#8143](https://github.com/ClickHouse/ClickHouse/pull/8143) ([亚历山大\*卡扎科夫](https://github.com/Akazz)) -- 现在 `SYSTEM RELOAD DICTIONARY` 完全重新加载字典，忽略 `update_field`. 这修复 [\#7440](https://github.com/ClickHouse/ClickHouse/issues/7440). [\#8037](https://github.com/ClickHouse/ClickHouse/pull/8037) ([维塔利\*巴拉诺夫](https://github.com/vitlibar)) -- 添加检查字典是否存在于创建查询的能力。 [\#8032](https://github.com/ClickHouse/ClickHouse/pull/8032) ([阿利沙平](https://github.com/alesapin)) -- 修复 `Float*` 解析中 `Values` 格式。 这修复 [\#7817](https://github.com/ClickHouse/ClickHouse/issues/7817). [\#7870](https://github.com/ClickHouse/ClickHouse/pull/7870) ([tavplubix](https://github.com/tavplubix)) -- 修复崩溃时，我们不能在一些后台操作保留空间 `*MergeTree` 表引擎家族. [\#7873](https://github.com/ClickHouse/ClickHouse/pull/7873) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 修复表包含合并操作时的崩溃 `SimpleAggregateFunction(LowCardinality)` 列。 这修复 [\#8515](https://github.com/ClickHouse/ClickHouse/issues/8515). [\#8522](https://github.com/ClickHouse/ClickHouse/pull/8522) ([Azat Khuzhin](https://github.com/azat)) -- 恢复对所有ICU区域设置的支持，并添加对常量表达式应用排序规则的功能。 还添加语言名称 `system.collations` 桌子 [\#8051](https://github.com/ClickHouse/ClickHouse/pull/8051) ([阿利沙平](https://github.com/alesapin)) -- 修正错误时，外部字典与零最小寿命 (`LIFETIME(MIN 0 MAX N)`, `LIFETIME(N)`）不要在后台更新。 [\#7983](https://github.com/ClickHouse/ClickHouse/pull/7983) ([阿利沙平](https://github.com/alesapin)) -- 修复当clickhouse源外部字典在查询中有子查询时崩溃。 [\#8351](https://github.com/ClickHouse/ClickHouse/pull/8351) ([尼古拉\*科切托夫](https://github.com/KochetovNicolai)) -- 修复文件扩展名不正确的解析表与引擎 `URL`. 这修复 [\#8157](https://github.com/ClickHouse/ClickHouse/issues/8157). [\#8419](https://github.com/ClickHouse/ClickHouse/pull/8419) ([安德烈\*博德罗夫](https://github.com/apbodrov)) -- 修复 `CHECK TABLE` 查询为 `*MergeTree` 表没有关键. 修复 [\#7543](https://github.com/ClickHouse/ClickHouse/issues/7543). [\#7979](https://github.com/ClickHouse/ClickHouse/pull/7979) ([阿利沙平](https://github.com/alesapin)) -- 固定转换 `Float64` 到MySQL类型。 [\#8079](https://github.com/ClickHouse/ClickHouse/pull/8079) ([尤里\*巴拉诺夫](https://github.com/yurriy)) -- 现在，如果表没有完全删除，因为服务器崩溃，服务器将尝试恢复并加载它。 [\#8176](https://github.com/ClickHouse/ClickHouse/pull/8176) ([tavplubix](https://github.com/tavplubix)) -- 修复了表函数中的崩溃 `file` 同时插入到不存在的文件。 现在在这种情况下，文件将被创建，然后插入将被处理。 [\#8177](https://github.com/ClickHouse/ClickHouse/pull/8177) ([Olga Khvostikova](https://github.com/stavrolia)) -- 修复罕见的死锁时，可能发生 `trace_log` 处于启用状态。 [\#7838](https://github.com/ClickHouse/ClickHouse/pull/7838) ([filimonov](https://github.com/filimonov)) -- 添加能力与不同类型的工作，除了 `Date` 在 `RangeHashed` 从DDL查询创建的外部字典。 修复 [7899](https://github.com/ClickHouse/ClickHouse/issues/7899). [\#8275](https://github.com/ClickHouse/ClickHouse/pull/8275) ([阿利沙平](https://github.com/alesapin)) -- 修复崩溃时 `now64()` 用另一个函数的结果调用。 [\#8270](https://github.com/ClickHouse/ClickHouse/pull/8270) ([瓦西里\*内姆科夫](https://github.com/Enmk)) -- 修正了通过mysql有线协议检测客户端IP连接的错误。 [\#7743](https://github.com/ClickHouse/ClickHouse/pull/7743) ([Dmitry Muzyka](https://github.com/dmitriy-myz)) -- 修复空阵列处理 `arraySplit` 功能。 这修复 [\#7708](https://github.com/ClickHouse/ClickHouse/issues/7708). [\#7747](https://github.com/ClickHouse/ClickHouse/pull/7747) ([hcz](https://github.com/hczhcz)) -- 修复了以下问题 `pid-file` 另一个运行 `clickhouse-server` 可能会被删除。 [\#8487](https://github.com/ClickHouse/ClickHouse/pull/8487) ([徐伟清](https://github.com/weiqxu)) -- 修复字典重新加载，如果它有 `invalidate_query`，停止更新，并在以前的更新尝试一些异常。 [\#8029](https://github.com/ClickHouse/ClickHouse/pull/8029) ([阿利沙平](https://github.com/alesapin)) -- 修正了功能错误 `arrayReduce` 这可能会导致 “double free” 和聚合函数组合器中的错误 `Resample` 这可能会导致内存泄漏。 添加聚合功能 `aggThrow`. 此功能可用于测试目的。 [\#8446](https://github.com/ClickHouse/ClickHouse/pull/8446) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) - -#### 改进 {#improvement-1} - -- 改进了使用时的日志记录 `S3` 表引擎。 [\#8251](https://github.com/ClickHouse/ClickHouse/pull/8251) ([Grigory Pervakov](https://github.com/GrigoryPervakov)) -- 在调用时未传递任何参数时打印帮助消息 `clickhouse-local`. 这修复 [\#5335](https://github.com/ClickHouse/ClickHouse/issues/5335). [\#8230](https://github.com/ClickHouse/ClickHouse/pull/8230) ([安德烈\*纳戈尔尼](https://github.com/Melancholic)) -- 添加设置 `mutations_sync` 这允许等待 `ALTER UPDATE/DELETE` 同步查询。 [\#8237](https://github.com/ClickHouse/ClickHouse/pull/8237) ([阿利沙平](https://github.com/alesapin)) -- 允许设置相对 `user_files_path` 在 `config.xml` （在类似的方式 `format_schema_path`). [\#7632](https://github.com/ClickHouse/ClickHouse/pull/7632) ([hcz](https://github.com/hczhcz)) -- 为转换函数添加非法类型的异常 `-OrZero` 后缀 [\#7880](https://github.com/ClickHouse/ClickHouse/pull/7880) ([安德烈\*科尼亚耶夫](https://github.com/akonyaev90)) -- 简化在分布式查询中发送到分片的数据头的格式。 [\#8044](https://github.com/ClickHouse/ClickHouse/pull/8044) ([维塔利\*巴拉诺夫](https://github.com/vitlibar)) -- `Live View` 表引擎重构。 [\#8519](https://github.com/ClickHouse/ClickHouse/pull/8519) ([vzakaznikov](https://github.com/vzakaznikov)) -- 为从DDL查询创建的外部字典添加额外的检查。 [\#8127](https://github.com/ClickHouse/ClickHouse/pull/8127) ([阿利沙平](https://github.com/alesapin)) -- 修复错误 `Column ... already exists` 使用时 `FINAL` 和 `SAMPLE` together, e.g. `select count() from table final sample 1/2`. 修复 [\#5186](https://github.com/ClickHouse/ClickHouse/issues/5186). [\#7907](https://github.com/ClickHouse/ClickHouse/pull/7907) ([尼古拉\*科切托夫](https://github.com/KochetovNicolai)) -- 现在表的第一个参数 `joinGet` 函数可以是表标识符。 [\#7707](https://github.com/ClickHouse/ClickHouse/pull/7707) ([阿莫斯鸟](https://github.com/amosbird)) -- 允许使用 `MaterializedView` 与上面的子查询 `Kafka` 桌子 [\#8197](https://github.com/ClickHouse/ClickHouse/pull/8197) ([filimonov](https://github.com/filimonov)) -- 现在后台在磁盘之间移动，运行它的seprate线程池。 [\#7670](https://github.com/ClickHouse/ClickHouse/pull/7670) ([Vladimir Chebotarev](https://github.com/excitoon)) -- `SYSTEM RELOAD DICTIONARY` 现在同步执行。 [\#8240](https://github.com/ClickHouse/ClickHouse/pull/8240) ([维塔利\*巴拉诺夫](https://github.com/vitlibar)) -- 堆栈跟踪现在显示物理地址（对象文件中的偏移量），而不是虚拟内存地址（加载对象文件的位置）。 这允许使用 `addr2line` 当二进制独立于位置并且ASLR处于活动状态时。 这修复 [\#8360](https://github.com/ClickHouse/ClickHouse/issues/8360). [\#8387](https://github.com/ClickHouse/ClickHouse/pull/8387) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 支持行级安全筛选器的新语法: `…
`. 修复 [\#5779](https://github.com/ClickHouse/ClickHouse/issues/5779). [\#8381](https://github.com/ClickHouse/ClickHouse/pull/8381) ([伊万](https://github.com/abyss7)) -- 现在 `cityHash` 功能可以与工作 `Decimal` 和 `UUID` 类型。 修复 [\#5184](https://github.com/ClickHouse/ClickHouse/issues/5184). [\#7693](https://github.com/ClickHouse/ClickHouse/pull/7693) ([米哈伊尔\*科罗托夫](https://github.com/millb)) -- 从系统日志中删除了固定的索引粒度（它是1024），因为它在实现自适应粒度之后已经过时。 [\#7698](https://github.com/ClickHouse/ClickHouse/pull/7698) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 当ClickHouse在没有SSL的情况下编译时，启用MySQL兼容服务器。 [\#7852](https://github.com/ClickHouse/ClickHouse/pull/7852) ([尤里\*巴拉诺夫](https://github.com/yurriy)) -- 现在服务器校验和分布式批处理，这在批处理中损坏数据的情况下提供了更多详细的错误。 [\#7914](https://github.com/ClickHouse/ClickHouse/pull/7914) ([Azat Khuzhin](https://github.com/azat)) -- 碌莽禄Support: `DROP DATABASE`, `DETACH TABLE`, `DROP TABLE` 和 `ATTACH TABLE` 为 `MySQL` 数据库引擎。 [\#8202](https://github.com/ClickHouse/ClickHouse/pull/8202) ([张冬](https://github.com/zhang2014)) -- 在S3表功能和表引擎中添加身份验证。 [\#7623](https://github.com/ClickHouse/ClickHouse/pull/7623) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 增加了检查额外的部分 `MergeTree` 在不同的磁盘上，为了不允许错过未定义磁盘上的数据部分。 [\#8118](https://github.com/ClickHouse/ClickHouse/pull/8118) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 启用Mac客户端和服务器的SSL支持。 [\#8297](https://github.com/ClickHouse/ClickHouse/pull/8297) ([伊万](https://github.com/abyss7)) -- 现在ClickHouse可以作为MySQL联合服务器（参见https://dev.mysql.com/doc/refman/5.7/en/federated-create-server.html）。 [\#7717](https://github.com/ClickHouse/ClickHouse/pull/7717) ([Maxim Fedotov](https://github.com/MaxFedotov)) -- `clickhouse-client` 现在只能启用 `bracketed-paste` 当多查询处于打开状态且多行处于关闭状态时。 这修复（＃7757）\[https://github.com/ClickHouse/ClickHouse/issues/7757。 [\#7761](https://github.com/ClickHouse/ClickHouse/pull/7761) ([阿莫斯鸟](https://github.com/amosbird)) -- 碌莽禄Support: `Array(Decimal)` 在 `if` 功能。 [\#7721](https://github.com/ClickHouse/ClickHouse/pull/7721) ([Artem Zuikov](https://github.com/4ertus2)) -- 支持小数 `arrayDifference`, `arrayCumSum` 和 `arrayCumSumNegative` 功能。 [\#7724](https://github.com/ClickHouse/ClickHouse/pull/7724) ([Artem Zuikov](https://github.com/4ertus2)) -- 已添加 `lifetime` 列到 `system.dictionaries` 桌子 [\#6820](https://github.com/ClickHouse/ClickHouse/issues/6820) [\#7727](https://github.com/ClickHouse/ClickHouse/pull/7727) ([kekekekule](https://github.com/kekekekule)) -- 改进了检查不同磁盘上的现有部件 `*MergeTree` 表引擎. 地址 [\#7660](https://github.com/ClickHouse/ClickHouse/issues/7660). [\#8440](https://github.com/ClickHouse/ClickHouse/pull/8440) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 集成与 `AWS SDK` 为 `S3` 交互允许使用开箱即用的所有S3功能。 [\#8011](https://github.com/ClickHouse/ClickHouse/pull/8011) ([帕维尔\*科瓦连科](https://github.com/Jokser)) -- 增加了对子查询的支持 `Live View` 桌子 [\#7792](https://github.com/ClickHouse/ClickHouse/pull/7792) ([vzakaznikov](https://github.com/vzakaznikov)) -- 检查使用 `Date` 或 `DateTime` 从列 `TTL` 表达式已删除。 [\#7920](https://github.com/ClickHouse/ClickHouse/pull/7920) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 有关磁盘的信息已添加到 `system.detached_parts` 桌子 [\#7833](https://github.com/ClickHouse/ClickHouse/pull/7833) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 现在设置 `max_(table|partition)_size_to_drop` 无需重新启动即可更改。 [\#7779](https://github.com/ClickHouse/ClickHouse/pull/7779) ([Grigory Pervakov](https://github.com/GrigoryPervakov)) -- 错误消息的可用性略好。 要求用户不要删除下面的行 `Stack trace:`. [\#7897](https://github.com/ClickHouse/ClickHouse/pull/7897) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 更好地阅读消息 `Kafka` 引擎在各种格式后 [\#7935](https://github.com/ClickHouse/ClickHouse/issues/7935). [\#8035](https://github.com/ClickHouse/ClickHouse/pull/8035) ([伊万](https://github.com/abyss7)) -- 与不支持MySQL客户端更好的兼容性 `sha2_password` 验证插件。 [\#8036](https://github.com/ClickHouse/ClickHouse/pull/8036) ([尤里\*巴拉诺夫](https://github.com/yurriy)) -- 支持MySQL兼容性服务器中的更多列类型。 [\#7975](https://github.com/ClickHouse/ClickHouse/pull/7975) ([尤里\*巴拉诺夫](https://github.com/yurriy)) -- 执行 `ORDER BY` 优化 `Merge`, `Buffer` 和 `Materilized View` 存储与底层 `MergeTree` 桌子 [\#8130](https://github.com/ClickHouse/ClickHouse/pull/8130) ([安东\*波波夫](https://github.com/CurtizJ)) -- 现在我们总是使用POSIX实现 `getrandom` 与旧内核更好的兼容性（\<3.17）。 [\#7940](https://github.com/ClickHouse/ClickHouse/pull/7940) ([阿莫斯鸟](https://github.com/amosbird)) -- 更好地检查移动ttl规则中的有效目标。 [\#8410](https://github.com/ClickHouse/ClickHouse/pull/8410) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 更好地检查损坏的刀片批次 `Distributed` 表引擎。 [\#7933](https://github.com/ClickHouse/ClickHouse/pull/7933) ([Azat Khuzhin](https://github.com/azat)) -- 添加带有部件名称数组的列，这些部件将来必须处理突变 `system.mutations` 桌子 [\#8179](https://github.com/ClickHouse/ClickHouse/pull/8179) ([阿利沙平](https://github.com/alesapin)) -- 处理器的并行合并排序优化。 [\#8552](https://github.com/ClickHouse/ClickHouse/pull/8552) ([尼古拉\*科切托夫](https://github.com/KochetovNicolai)) -- 设置 `mark_cache_min_lifetime` 现在已经过时了，什么也不做。 在以前的版本中，标记缓存可以在内存中增长大于 `mark_cache_size` 以容纳内的数据 `mark_cache_min_lifetime` 秒。 这导致了混乱和比预期更高的内存使用率，这在内存受限的系统上尤其糟糕。 如果您在安装此版本后会看到性能下降，则应增加 `mark_cache_size`. [\#8484](https://github.com/ClickHouse/ClickHouse/pull/8484) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 准备使用 `tid` 到处都是 这是必要的 [\#7477](https://github.com/ClickHouse/ClickHouse/issues/7477). [\#8276](https://github.com/ClickHouse/ClickHouse/pull/8276) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) - -#### 性能改进 {#performance-improvement-1} - -- 处理器管道中的性能优化。 [\#7988](https://github.com/ClickHouse/ClickHouse/pull/7988) ([尼古拉\*科切托夫](https://github.com/KochetovNicolai)) -- 缓存字典中过期密钥的非阻塞更新（具有读取旧密钥的权限）。 [\#8303](https://github.com/ClickHouse/ClickHouse/pull/8303) ([尼基塔\*米哈伊洛夫](https://github.com/nikitamikhaylov)) -- 没有编译ClickHouse `-fno-omit-frame-pointer` 在全球范围内多余一个寄存器。 [\#8097](https://github.com/ClickHouse/ClickHouse/pull/8097) ([阿莫斯鸟](https://github.com/amosbird)) -- 加速 `greatCircleDistance` 功能，并为它添加性能测试。 [\#7307](https://github.com/ClickHouse/ClickHouse/pull/7307) ([Olga Khvostikova](https://github.com/stavrolia)) -- 改进的功能性能 `roundDown`. [\#8465](https://github.com/ClickHouse/ClickHouse/pull/8465) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 改进的性能 `max`, `min`, `argMin`, `argMax` 为 `DateTime64` 数据类型。 [\#8199](https://github.com/ClickHouse/ClickHouse/pull/8199) ([瓦西里\*内姆科夫](https://github.com/Enmk)) -- 改进了无限制或大限制和外部排序的排序性能。 [\#8545](https://github.com/ClickHouse/ClickHouse/pull/8545) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 改进的性能格式化浮点数高达6倍。 [\#8542](https://github.com/ClickHouse/ClickHouse/pull/8542) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 改进的性能 `modulo` 功能。 [\#7750](https://github.com/ClickHouse/ClickHouse/pull/7750) ([阿莫斯鸟](https://github.com/amosbird)) -- 优化 `ORDER BY` 并与单列键合并。 [\#8335](https://github.com/ClickHouse/ClickHouse/pull/8335) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 更好地实施 `arrayReduce`, `-Array` 和 `-State` 组合子 [\#7710](https://github.com/ClickHouse/ClickHouse/pull/7710) ([阿莫斯鸟](https://github.com/amosbird)) -- 现在 `PREWHERE` 应优化为至少一样高效 `WHERE`. [\#7769](https://github.com/ClickHouse/ClickHouse/pull/7769) ([阿莫斯鸟](https://github.com/amosbird)) -- 改进方式 `round` 和 `roundBankers` 处理负数。 [\#8229](https://github.com/ClickHouse/ClickHouse/pull/8229) ([hcz](https://github.com/hczhcz)) -- 改进的解码性能 `DoubleDelta` 和 `Gorilla` 编解码器大约30-40％。 这修复 [\#7082](https://github.com/ClickHouse/ClickHouse/issues/7082). [\#8019](https://github.com/ClickHouse/ClickHouse/pull/8019) ([瓦西里\*内姆科夫](https://github.com/Enmk)) -- 改进的性能 `base64` 相关功能。 [\#8444](https://github.com/ClickHouse/ClickHouse/pull/8444) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 增加了一个功能 `geoDistance`. 它类似于 `greatCircleDistance` 但使用近似于WGS-84椭球模型。 两个功能的性能几乎相同。 [\#8086](https://github.com/ClickHouse/ClickHouse/pull/8086) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 更快 `min` 和 `max` 聚合函数 `Decimal` 数据类型。 [\#8144](https://github.com/ClickHouse/ClickHouse/pull/8144) ([Artem Zuikov](https://github.com/4ertus2)) -- 矢量化处理 `arrayReduce`. [\#7608](https://github.com/ClickHouse/ClickHouse/pull/7608) ([阿莫斯鸟](https://github.com/amosbird)) -- `if` 链现在优化为 `multiIf`. [\#8355](https://github.com/ClickHouse/ClickHouse/pull/8355) ([kamalov-ruslan](https://github.com/kamalov-ruslan)) -- 修复性能回归 `Kafka` 表引擎在19.15中引入。 这修复 [\#7261](https://github.com/ClickHouse/ClickHouse/issues/7261). [\#7935](https://github.com/ClickHouse/ClickHouse/pull/7935) ([filimonov](https://github.com/filimonov)) -- 已删除 “pie” 代码生成 `gcc` 从Debian软件包偶尔带来默认情况下。 [\#8483](https://github.com/ClickHouse/ClickHouse/pull/8483) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 并行解析数据格式 [\#6553](https://github.com/ClickHouse/ClickHouse/pull/6553) ([尼基塔\*米哈伊洛夫](https://github.com/nikitamikhaylov)) -- 启用优化的解析器 `Values` 默认使用表达式 (`input_format_values_deduce_templates_of_expressions=1`). [\#8231](https://github.com/ClickHouse/ClickHouse/pull/8231) ([tavplubix](https://github.com/tavplubix)) - -#### 构建/测试/包装改进 {#buildtestingpackaging-improvement-2} - -- 构建修复 `ARM` 而在最小模式。 [\#8304](https://github.com/ClickHouse/ClickHouse/pull/8304) ([proller](https://github.com/proller)) -- 添加复盖文件刷新 `clickhouse-server` 当不调用std::atexit时。 还略微改进了无状态测试的复盖率日志记录。 [\#8267](https://github.com/ClickHouse/ClickHouse/pull/8267) ([阿利沙平](https://github.com/alesapin)) -- 更新contrib中的LLVM库。 避免从操作系统包中使用LLVM。 [\#8258](https://github.com/ClickHouse/ClickHouse/pull/8258) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 使bund绑 `curl` 建立完全安静。 [\#8232](https://github.com/ClickHouse/ClickHouse/pull/8232) [\#8203](https://github.com/ClickHouse/ClickHouse/pull/8203) ([帕维尔\*科瓦连科](https://github.com/Jokser)) -- 修复一些 `MemorySanitizer` 警告。 [\#8235](https://github.com/ClickHouse/ClickHouse/pull/8235) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- 使用 `add_warning` 和 `no_warning` 宏 `CMakeLists.txt`. [\#8604](https://github.com/ClickHouse/ClickHouse/pull/8604) ([伊万](https://github.com/abyss7)) -- 添加对Minio S3兼容对象的支持（https://min.io/)为了更好的集成测试。 [\#7863](https://github.com/ClickHouse/ClickHouse/pull/7863) [\#7875](https://github.com/ClickHouse/ClickHouse/pull/7875) ([帕维尔\*科瓦连科](https://github.com/Jokser)) -- 导入 `libc` 标题到contrib。 它允许在各种系统中使构建更加一致（仅适用于 `x86_64-linux-gnu`). [\#5773](https://github.com/ClickHouse/ClickHouse/pull/5773) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 删除 `-fPIC` 从一些图书馆。 [\#8464](https://github.com/ClickHouse/ClickHouse/pull/8464) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 清洁 `CMakeLists.txt` 对于卷曲。 看https://github.com/ClickHouse/ClickHouse/pull/8011\#issuecomment-569478910 [\#8459](https://github.com/ClickHouse/ClickHouse/pull/8459) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 无声警告 `CapNProto` 图书馆. [\#8220](https://github.com/ClickHouse/ClickHouse/pull/8220) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 为短字符串优化哈希表添加性能测试。 [\#7679](https://github.com/ClickHouse/ClickHouse/pull/7679) ([阿莫斯鸟](https://github.com/amosbird)) -- 现在ClickHouse将建立在 `AArch64` 即使 `MADV_FREE` 不可用。 这修复 [\#8027](https://github.com/ClickHouse/ClickHouse/issues/8027). [\#8243](https://github.com/ClickHouse/ClickHouse/pull/8243) ([阿莫斯鸟](https://github.com/amosbird)) -- 更新 `zlib-ng` 来解决记忆消毒的问题 [\#7182](https://github.com/ClickHouse/ClickHouse/pull/7182) [\#8206](https://github.com/ClickHouse/ClickHouse/pull/8206) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- 在非Linux系统上启用内部MySQL库，因为操作系统包的使用非常脆弱，通常根本不起作用。 这修复 [\#5765](https://github.com/ClickHouse/ClickHouse/issues/5765). [\#8426](https://github.com/ClickHouse/ClickHouse/pull/8426) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 修复了启用后在某些系统上构建的问题 `libc++`. 这取代了 [\#8374](https://github.com/ClickHouse/ClickHouse/issues/8374). [\#8380](https://github.com/ClickHouse/ClickHouse/pull/8380) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 赂眉露\>\> `Field` 方法更类型安全，以找到更多的错误。 [\#7386](https://github.com/ClickHouse/ClickHouse/pull/7386) [\#8209](https://github.com/ClickHouse/ClickHouse/pull/8209) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- 添加丢失的文件到 `libc-headers` 子模块。 [\#8507](https://github.com/ClickHouse/ClickHouse/pull/8507) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 修复错误 `JSON` 引用性能测试输出。 [\#8497](https://github.com/ClickHouse/ClickHouse/pull/8497) ([尼古拉\*科切托夫](https://github.com/KochetovNicolai)) -- 现在堆栈跟踪显示 `std::exception` 和 `Poco::Exception`. 在以前的版本中，它仅适用于 `DB::Exception`. 这改进了诊断。 [\#8501](https://github.com/ClickHouse/ClickHouse/pull/8501) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 移植 `clock_gettime` 和 `clock_nanosleep` 对于新鲜的glibc版本。 [\#8054](https://github.com/ClickHouse/ClickHouse/pull/8054) ([阿莫斯鸟](https://github.com/amosbird)) -- 启用 `part_log` 在示例配置开发人员。 [\#8609](https://github.com/ClickHouse/ClickHouse/pull/8609) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 修复重新加载的异步性质 `01036_no_superfluous_dict_reload_on_create_database*`. [\#8111](https://github.com/ClickHouse/ClickHouse/pull/8111) ([Azat Khuzhin](https://github.com/azat)) -- 固定编解码器性能测试。 [\#8615](https://github.com/ClickHouse/ClickHouse/pull/8615) ([瓦西里\*内姆科夫](https://github.com/Enmk)) -- 添加安装脚本 `.tgz` 为他们构建和文档。 [\#8612](https://github.com/ClickHouse/ClickHouse/pull/8612) [\#8591](https://github.com/ClickHouse/ClickHouse/pull/8591) ([阿利沙平](https://github.com/alesapin)) -- 删除旧 `ZSTD` 测试（它是在2016年创建的，以重现zstd1.0版本之前的错误）。 这修复 [\#8618](https://github.com/ClickHouse/ClickHouse/issues/8618). [\#8619](https://github.com/ClickHouse/ClickHouse/pull/8619) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 固定构建在Mac OS卡特琳娜。 [\#8600](https://github.com/ClickHouse/ClickHouse/pull/8600) ([meo](https://github.com/meob)) -- 增加编解码器性能测试中的行数，以使结果显着。 [\#8574](https://github.com/ClickHouse/ClickHouse/pull/8574) ([瓦西里\*内姆科夫](https://github.com/Enmk)) -- 在调试版本中，处理 `LOGICAL_ERROR` 异常作为断言失败，使得它们更容易被注意到。 [\#8475](https://github.com/ClickHouse/ClickHouse/pull/8475) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- 使与格式相关的性能测试更具确定性。 [\#8477](https://github.com/ClickHouse/ClickHouse/pull/8477) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 更新 `lz4` 来修复记忆消毒器的故障 [\#8181](https://github.com/ClickHouse/ClickHouse/pull/8181) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- 在异常处理中抑制已知MemorySanitizer误报。 [\#8182](https://github.com/ClickHouse/ClickHouse/pull/8182) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- 更新 `gcc` 和 `g++` 到版本9在 `build/docker/build.sh` [\#7766](https://github.com/ClickHouse/ClickHouse/pull/7766) ([TLightSky](https://github.com/tlightsky)) -- 添加性能测试用例来测试 `PREWHERE` 比 `WHERE`. [\#7768](https://github.com/ClickHouse/ClickHouse/pull/7768) ([阿莫斯鸟](https://github.com/amosbird)) -- 在修复一个笨拙的测试方面取得了进展。 [\#8621](https://github.com/ClickHouse/ClickHouse/pull/8621) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 避免从MemorySanitizer报告数据 `libunwind`. [\#8539](https://github.com/ClickHouse/ClickHouse/pull/8539) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 更新 `libc++` 到最新版本。 [\#8324](https://github.com/ClickHouse/ClickHouse/pull/8324) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 从源头构建ICU库。 这修复 [\#6460](https://github.com/ClickHouse/ClickHouse/issues/6460). [\#8219](https://github.com/ClickHouse/ClickHouse/pull/8219) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 从切换 `libressl` 到 `openssl`. ClickHouse应在此更改后支持TLS1.3和SNI。 这修复 [\#8171](https://github.com/ClickHouse/ClickHouse/issues/8171). [\#8218](https://github.com/ClickHouse/ClickHouse/pull/8218) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 使用时固定的UBSan报告 `chacha20_poly1305` 从SSL（发生在连接到https://yandex.ru/)。 [\#8214](https://github.com/ClickHouse/ClickHouse/pull/8214) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 修复默认密码文件的模式 `.deb` linux发行版。 [\#8075](https://github.com/ClickHouse/ClickHouse/pull/8075) ([proller](https://github.com/proller)) -- 改进的表达式获取 `clickhouse-server` PID输入 `clickhouse-test`. [\#8063](https://github.com/ClickHouse/ClickHouse/pull/8063) ([亚历山大\*卡扎科夫](https://github.com/Akazz)) -- 更新contrib/googletest到v1.10.0。 [\#8587](https://github.com/ClickHouse/ClickHouse/pull/8587) ([Alexander Burmak](https://github.com/Alex-Burmak)) -- 修复了ThreadSaninitizer报告 `base64` 图书馆. 还将此库更新到最新版本，但无关紧要。 这修复 [\#8397](https://github.com/ClickHouse/ClickHouse/issues/8397). [\#8403](https://github.com/ClickHouse/ClickHouse/pull/8403) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 修复 `00600_replace_running_query` 对于处理器。 [\#8272](https://github.com/ClickHouse/ClickHouse/pull/8272) ([尼古拉\*科切托夫](https://github.com/KochetovNicolai)) -- 删除支持 `tcmalloc` 为了使 `CMakeLists.txt` 更简单 [\#8310](https://github.com/ClickHouse/ClickHouse/pull/8310) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 发布海湾合作委员会构建现在使用 `libc++` 而不是 `libstdc++`. 最近 `libc++` 只与叮当一起使用。 这将提高构建配置的一致性和可移植性。 [\#8311](https://github.com/ClickHouse/ClickHouse/pull/8311) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 使用MemorySanitizer启用ICU库进行构建。 [\#8222](https://github.com/ClickHouse/ClickHouse/pull/8222) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 禁止从警告 `CapNProto` 图书馆. [\#8224](https://github.com/ClickHouse/ClickHouse/pull/8224) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 删除代码的特殊情况 `tcmalloc`，因为它不再受支持。 [\#8225](https://github.com/ClickHouse/ClickHouse/pull/8225) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 在CI coverage任务中，优雅地终止服务器以允许它保存coverage报告。 这修复了我们最近看到的不完整的复盖率报告。 [\#8142](https://github.com/ClickHouse/ClickHouse/pull/8142) ([阿利沙平](https://github.com/alesapin)) -- 针对所有编解码器的性能测试 `Float64` 和 `UInt64` 值。 [\#8349](https://github.com/ClickHouse/ClickHouse/pull/8349) ([瓦西里\*内姆科夫](https://github.com/Enmk)) -- `termcap` 非常不推荐使用，并导致各种问题（f.g.missing “up” 帽和呼应 `^J` 而不是多行）。 帮个忙 `terminfo` 或bund绑 `ncurses`. [\#7737](https://github.com/ClickHouse/ClickHouse/pull/7737) ([阿莫斯鸟](https://github.com/amosbird)) -- 修复 `test_storage_s3` 集成测试。 [\#7734](https://github.com/ClickHouse/ClickHouse/pull/7734) ([尼古拉\*科切托夫](https://github.com/KochetovNicolai)) -- 碌莽禄Support: `StorageFile(, null)` 将块插入给定格式的文件而不实际写入磁盘。 这是性能测试所必需的。 [\#8455](https://github.com/ClickHouse/ClickHouse/pull/8455) ([阿莫斯鸟](https://github.com/amosbird)) -- 添加参数 `--print-time` 功能测试打印每个测试的执行时间。 [\#8001](https://github.com/ClickHouse/ClickHouse/pull/8001) ([尼古拉\*科切托夫](https://github.com/KochetovNicolai)) -- 添加断言 `KeyCondition` 同时评估RPN。 这将修复来自gcc-9的警告。 [\#8279](https://github.com/ClickHouse/ClickHouse/pull/8279) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 在CI构建中转储cmake选项。 [\#8273](https://github.com/ClickHouse/ClickHouse/pull/8273) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- 不要为某些fat库生成调试信息。 [\#8271](https://github.com/ClickHouse/ClickHouse/pull/8271) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 赂眉露\>\> `log_to_console.xml` 始终登录到stderr，无论它是否交互。 [\#8395](https://github.com/ClickHouse/ClickHouse/pull/8395) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- 删除了一些未使用的功能 `clickhouse-performance-test` 工具 [\#8555](https://github.com/ClickHouse/ClickHouse/pull/8555) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 现在我们也将搜索 `lld-X` 与相应的 `clang-X` 版本。 [\#8092](https://github.com/ClickHouse/ClickHouse/pull/8092) ([阿利沙平](https://github.com/alesapin)) -- 实木复合地板建设改善。 [\#8421](https://github.com/ClickHouse/ClickHouse/pull/8421) ([马苏兰](https://github.com/maxulan)) -- 更多海湾合作委员会警告 [\#8221](https://github.com/ClickHouse/ClickHouse/pull/8221) ([kreuzerkrieg](https://github.com/kreuzerkrieg)) -- Arch Linux的软件包现在允许运行ClickHouse服务器，而不仅仅是客户端。 [\#8534](https://github.com/ClickHouse/ClickHouse/pull/8534) ([Vladimir Chebotarev](https://github.com/excitoon)) -- 修复与处理器的测试。 微小的性能修复。 [\#7672](https://github.com/ClickHouse/ClickHouse/pull/7672) ([尼古拉\*科切托夫](https://github.com/KochetovNicolai)) -- 更新contrib/protobuf。 [\#8256](https://github.com/ClickHouse/ClickHouse/pull/8256) ([Matwey V.Kornilov](https://github.com/matwey)) -- 在准备切换到c++20作为新年庆祝活动。 “May the C++ force be with ClickHouse.” [\#8447](https://github.com/ClickHouse/ClickHouse/pull/8447) ([阿莫斯鸟](https://github.com/amosbird)) - -#### 实验特点 {#experimental-feature-1} - -- 增加了实验设置 `min_bytes_to_use_mmap_io`. 它允许读取大文件，而无需将数据从内核复制到用户空间。 默认情况下禁用该设置。 建议的阈值大约是64MB，因为mmap/munmap很慢。 [\#8520](https://github.com/ClickHouse/ClickHouse/pull/8520) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) -- 返工配额作为访问控制系统的一部分。 增加了新表 `system.quotas`，新功能 `currentQuota`, `currentQuotaKey`，新的SQL语法 `CREATE QUOTA`, `ALTER QUOTA`, `DROP QUOTA`, `SHOW QUOTA`. [\#7257](https://github.com/ClickHouse/ClickHouse/pull/7257) ([维塔利\*巴拉诺夫](https://github.com/vitlibar)) -- 允许跳过带有警告的未知设置，而不是引发异常。 [\#7653](https://github.com/ClickHouse/ClickHouse/pull/7653) ([维塔利\*巴拉诺夫](https://github.com/vitlibar)) -- 重新设计的行策略作为访问控制系统的一部分。 增加了新表 `system.row_policies`，新功能 `currentRowPolicies()`，新的SQL语法 `CREATE POLICY`, `ALTER POLICY`, `DROP POLICY`, `SHOW CREATE POLICY`, `SHOW POLICIES`. [\#7808](https://github.com/ClickHouse/ClickHouse/pull/7808) ([维塔利\*巴拉诺夫](https://github.com/vitlibar)) - -#### 安全修复 {#security-fix} - -- 修正了读取目录结构中的表的可能性 `File` 表引擎。 这修复 [\#8536](https://github.com/ClickHouse/ClickHouse/issues/8536). [\#8537](https://github.com/ClickHouse/ClickHouse/pull/8537) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) - -## [更新日志2019](https://github.com/ClickHouse/ClickHouse/blob/master/docs/en/changelog/2019.md) {#changelog-for-2019} +{% include "content/changelog.md" %} diff --git a/docs/zh/whats-new/index.md b/docs/zh/whats-new/index.md index 75a13a72bac..0f248773402 100644 --- a/docs/zh/whats-new/index.md +++ b/docs/zh/whats-new/index.md @@ -1,6 +1,6 @@ --- machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_folder_title: "\u65B0\u589E\u5185\u5BB9" toc_priority: 72 --- From adca04861254fd4a6485525f3c5f805205bb53f6 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 15 May 2020 10:07:06 +0300 Subject: [PATCH 241/738] Update extended-roadmap.md --- docs/ru/whats-new/extended-roadmap.md | 101 ++++++++++++++++++++------ 1 file changed, 79 insertions(+), 22 deletions(-) diff --git a/docs/ru/whats-new/extended-roadmap.md b/docs/ru/whats-new/extended-roadmap.md index b2a732da18a..9c640504aec 100644 --- a/docs/ru/whats-new/extended-roadmap.md +++ b/docs/ru/whats-new/extended-roadmap.md @@ -17,7 +17,7 @@ [Андрей Чулков](https://github.com/achulkov2), ВШЭ. -### 1.2. Wait-free каталог баз данных {#wait-free-katalog-baz-dannykh} +### 1.2. + Wait-free каталог баз данных {#wait-free-katalog-baz-dannykh} Q2. Делает [Александр Токмаков](https://github.com/tavplubix), первый рабочий вариант в декабре 2019. Нужно для DataLens и Яндекс.Метрики. @@ -64,17 +64,19 @@ Upd. Включено для системных таблиц. ### 1.7. Буферизация и WAL в MergeTree {#buferizatsiia-i-wal-v-mergetree} Требует 1.6. Антон Попов. Задача взята в работу. Q2. +Есть pull request. ### 1.8. + Перенос между разделами по TTL {#perenos-mezhdu-razdelami-po-ttl} Делает [Владимир Чеботарёв](https://github.com/excitoon), Altinity. Декабрь 2019. Q1. Закоммичено, но есть технический долг, который исправляется сейчас. -Готово. +Готово. Нет, не готово - там всё ещё технический долг. ### 1.9. Использование TTL для прореживания данных {#ispolzovanie-ttl-dlia-prorezhivaniia-dannykh} Будет делать Сорокин Николай, ВШЭ и Яндекс. +Upd. Есть pull request. Сейчас пользователь может задать в таблице выражение, которое определяет, сколько времени хранятся данные. Обычно это выражение задаётся относительно значения столбца с датой - например: удалять данные через три месяца. https://clickhouse.tech/docs/ru/operations/table_engines/mergetree/\#table_engine-mergetree-ttl @@ -120,7 +122,6 @@ Q2. Делает Александр, Яндекс.Облако (сначала часть для S3), а также Олег Ершов, ВШЭ и Яндекс. Upd. Олег будет делать только часть про HDFS. - Upd. Реализация поверх S3 является рабочей на уровне PoC. ### 1.13. Ускорение запросов с FINAL {#uskorenie-zaprosov-s-final} @@ -149,6 +150,7 @@ Upd: PR [#10463](https://github.com/ClickHouse/ClickHouse/pull/10463) Требует 1.3 и 1.6. Полная замена hard links на sym links, что будет лучше для 1.12. + ## 2. Крупные рефакторинги {#krupnye-refaktoringi} Для обоснования необходимости смотрите ссылки в описании других задач. @@ -236,6 +238,7 @@ Upd. Поползновения наблюдаются. Для нормализации работы materialized views поверх Merge, Distributed, Kafka. + ## 3. Документация {#dokumentatsiia} Здесь задачи только по инфраструктуре документации. @@ -258,6 +261,7 @@ Upd. Иван Блинков сделал эту задачу путём зам Эту задачу сделает [Иван Блинков](https://github.com/blinkov/), до конца декабря 2019. Сделано. + ## 4. Сетевое взаимодействие {#setevoe-vzaimodeistvie} ### 4.1. Уменьшение числа потоков при распределённых запросах {#umenshenie-chisla-potokov-pri-raspredelionnykh-zaprosakh} @@ -298,6 +302,7 @@ Upd. Сейчас обсуждается, как сделать другую з Дмитрий Григорьев, ВШЭ. В очереди. Исправить проблему, что восстанавливающаяся реплика перестаёт мержить. Частично компенсируется 4.3. + ## 5. Операции {#operatsii} ### 5.1. + Разделение задач на более мелкие куски в clickhouse-copier {#razdelenie-zadach-na-bolee-melkie-kuski-v-clickhouse-copier} @@ -308,7 +313,7 @@ Q1. Нужно для Метрики, в очереди. Никита Михай Upd. Задача на финальной стадии разработки. Upd. Сделано. Эффективность работы под вопросом. Есть варианты, как сделать лучше. -### 5.2. Автонастройка лимита на оперативку и размера кэшей {#avtonastroika-limita-na-operativku-i-razmera-keshei} +### 5.2. + Автонастройка лимита на оперативку и размера кэшей {#avtonastroika-limita-na-operativku-i-razmera-keshei} ### 5.3. + Встроенная ручка для Prometheus {#vstroennaia-ruchka-dlia-prometheus} @@ -316,6 +321,8 @@ Upd. Сделано. Эффективность работы под вопрос ### 5.4. Opt-in сообщать в клиенте, если вышла новая версия {#opt-in-soobshchat-v-kliente-esli-vyshla-novaia-versiia} +Есть поползновения. + ### 5.5. + LTS релизы {#lts-relizy} Требует 7.5. Задачу хочет Метрика, Облако, БК, Маркет и Altinity. Первой LTS версией уже стала версия 19.14. @@ -367,6 +374,7 @@ Upd. Появилась вторая версия LTS - 20.3. ### 6.10. Сбор общих системных метрик {#sbor-obshchikh-sistemnykh-metrik} + ## 7. Сопровождение разработки {#soprovozhdenie-razrabotki} ### 7.1. + ICU в submodules {#icu-v-submodules} @@ -392,14 +400,15 @@ Upd. Появилась вторая версия LTS - 20.3. ### 7.6. + Правильный статистический тест для comparison mode в clickhouse-performance-test {#pravilnyi-statisticheskii-test-dlia-comparison-mode-v-clickhouse-performance-test} -Задачу начал делать Дмитрий Рубашкин (ВШЭ). Сейчас продолжает [Александр Кузьменков](https://github.com/akuzm). Сделано, работает в CI. +Задачу начал делать Дмитрий Рубашкин (ВШЭ). Сейчас продолжает [Александр Кузьменков](https://github.com/akuzm). Сделано, работает в CI. Долгое время не были определены надёжные критерии для результата теста. Сейчас проблема решена, но остались не решёнными некоторые вопросы. -### 7.7. Доделать тесты под MSan {#dodelat-testy-pod-msan} +### 7.7. + Доделать тесты под MSan {#dodelat-testy-pod-msan} Уже есть ASan, TSan, UBSan. Не хватает тестов под MSan. Они уже добавлены в CI, но не проходят. [Александр Кузьменков](https://github.com/akuzm) и [Александр Токмаков](https://github.com/tavplubix). Upd. Задача всё ещё медленно тащится. +Upd. Доделал Алексей Миловидов. ### 7.8. + Добавить clang-tidy {#dobavit-clang-tidy} @@ -437,6 +446,7 @@ UBSan включен в функциональных тестах, но не в Подключение replxx вместо readline сделал Иван Лежанкин. Есть технический долг с лицензиями файлов консорциума Unicode. +Есть технический долг с работой \G в multiline режиме. ### 7.14.1. Улучшение возможностей интерактивного режима clickhouse-client {#uluchshenie-vozmozhnostei-interaktivnogo-rezhima-clickhouse-client} @@ -506,11 +516,13 @@ https://github.com/ClickHouse/ClickHouse/issues/8027\#issuecomment-566670282 [Иван Лежанкин](https://github.com/abyss7). Upd. В процессе реализации, есть pull request. -Upd. Есть сборки, [пример](https://clickhouse-builds.s3.yandex.net/0/2cb4c91c0286b774534fcbe80e5ef8835a74a83a/report.html) +Upd. Есть сборки, [пример](https://clickhouse-builds.s3.yandex.net/0/2cb4c91c0286b774534fcbe80e5ef8835a74a83a/report.html) +Всё ещё нет инструкции на сайте! ### 7.21. Автосборка для Linux ppc64 {#avtosborka-dlia-linux-ppc64} [Иван Лежанкин](https://github.com/abyss7). +Как-то медленно тащится. ### 7.22. Дэшборд для pull requests {#deshbord-dlia-pull-requests} @@ -563,6 +575,7 @@ Fuzzing тестирование - это тестирование случай Upd. Сергей Штыков сделал функцию `randomPrintableASCII`. Upd. Илья Яцишин сделал табличную функцию `generateRandom`. Upd. Эльдар Заитов добавляет OSS Fuzz. +Upd. Сделаны randomString, randomFixedString. ### 7.24. Fuzzing лексера и парсера запросов; кодеков и форматов {#fuzzing-leksera-i-parsera-zaprosov-kodekov-i-formatov} @@ -590,9 +603,10 @@ Upd: Задача в процессе реализации. Синхронизи Upd: Есть собирающийся прототип, но сборка как будто ещё не в trunk Аркадии. Upd: Добавлено в Аркадию, но не все файлы (не побайтово). -### 7.26. Побайтовая идентичность репозитория с Аркадией {#pobaitovaia-identichnost-repozitoriia-s-arkadiei} +### 7.26. + Побайтовая идентичность репозитория с Аркадией {#pobaitovaia-identichnost-repozitoriia-s-arkadiei} Команда DevTools. Прогресс по задаче под вопросом. +Upd. Готово (все директории кроме contrib). ### 7.27. Запуск автотестов в Аркадии {#zapusk-avtotestov-v-arkadii} @@ -623,10 +637,12 @@ Upd: Добавлено в Аркадию, но не все файлы (не п ### 7.33. Выкладывать патч релизы в репозиторий автоматически {#vykladyvat-patch-relizy-v-repozitorii-avtomaticheski} В очереди. Иван Лежанкин. +Отсутствует прогресс. ### 7.34. Бэкпортировать bugfix автоматически {#bekportirovat-bugfix-avtomaticheski} В очереди. Иван Лежанкин. +Отсутствует прогресс. ### 7.35. Начальные правила для авто-merge {#nachalnye-pravila-dlia-avto-merge} @@ -646,6 +662,7 @@ Upd: Добавлено в Аркадию, но не все файлы (не п Upd. Иван Блинков настроил CDN repo.clickhouse.tech, что решает проблему с доступностью зарубежом. Вопрос с operations, visibility пока актуален. + ## 8. Интеграция с внешними системами {#integratsiia-s-vneshnimi-sistemami} ### 8.1. Поддержка ALTER MODIFY SETTING для Kafka {#podderzhka-alter-modify-setting-dlia-kafka} @@ -657,6 +674,7 @@ Altinity. Никто не делает эту задачу. ### 8.2. Поддержка Mongo Atlas URI {#podderzhka-mongo-atlas-uri} [Александр Кузьменков](https://github.com/akuzm). +Upd. Задача взята в работу. ### 8.3. + Доработки globs (правильная поддержка диапазонов, уменьшение числа одновременных stream-ов) {#dorabotki-globs-pravilnaia-podderzhka-diapazonov-umenshenie-chisla-odnovremennykh-stream-ov} @@ -690,14 +708,16 @@ Altinity. Никто не делает эту задачу. ### 8.10. Запись в табличную функцию ODBC {#zapis-v-tablichnuiu-funktsiiu-odbc} Артемий Бобровский, ВШЭ +Есть pull request. ### 8.11. Движок таблиц для чтения из Mongo {#dvizhok-tablits-dlia-chteniia-iz-mongo} Артемий Бобровский, ВШЭ +Есть pull request. ### 8.12. Пропуск столбцов в форматах Parquet, ORC {#propusk-stolbtsov-v-formatakh-parquet-orc} -Артемий Бобровский, ВШЭ +Артемий Бобровский, ВШЭ или другой человек. ### 8.13. Поддержка массивов в Parquet, ORC {#podderzhka-massivov-v-parquet-orc} @@ -738,6 +758,7 @@ Upd. Почти готово - есть лишь небольшой технич ### 8.17. ClickHouse как MySQL реплика {#clickhouse-kak-mysql-replika} Ильяс Адюгамов, ВШЭ. +Upd. Задачу внезапно почти сделал другой человек. Реализовать возможность подписаться на row-based репликацию MySQL и сохранять полученные данные в CollapsingMergeTree или ReplacingMergeTree таблицы. Сторонние решения для этой задачи уже существуют: https://www.altinity.com/blog/2018/6/30/realtime-mysql-clickhouse-replication-in-practice Также существует стороннее решение для PostgreSQL: https://github.com/mkabilov/pg2ch @@ -778,6 +799,7 @@ Upd. Юрий Баранов работает в Google, там запрещен Желательно 2.15. + ## 9. Безопасность {#bezopasnost} ### 9.1. + Ограничение на хосты в запросах ко внешним системам {#ogranichenie-na-khosty-v-zaprosakh-ko-vneshnim-sistemam} @@ -798,6 +820,7 @@ ClickHouse предоставляет возможность обратитьс Есть pull request. + ## 10. Внешние словари {#vneshnie-slovari} ### 10.1. + Исправление зависания в библиотеке доступа к YT {#ispravlenie-zavisaniia-v-biblioteke-dostupa-k-yt} @@ -807,13 +830,14 @@ ClickHouse предоставляет возможность обратитьс Цитата: «Оказывается для YT-клиента зависания на несколько минут это нормально. Убрал внутренние ретраи, снизил таймауты. Однозначно станет лучше». -### 10.2. Исправление SIGILL в библиотеке доступа к YT {#ispravlenie-sigill-v-biblioteke-dostupa-k-yt} +### 10.2. + Исправление SIGILL в библиотеке доступа к YT {#ispravlenie-sigill-v-biblioteke-dostupa-k-yt} Код YT использует SIGILL вместо abort. Это, опять же, происходит при учениях. Нужно для БК и Метрики. Поиск причин - [Александр Сапин](https://github.com/alesapin). Дальшейшее исправление возможно на стороне YT. Upd. Одну причину устранили, но ещё что-то неизвестное осталось. Upd. Нас заставляют переписать эту библиотеку с одного API на другое, так как старое внезапно устарело. Кажется, что переписывание случайно исправит все проблемы. +Upd. Ура, нашли причину и исправили. ### 10.3. Возможность чтения данных из статических таблиц в YT словарях {#vozmozhnost-chteniia-dannykh-iz-staticheskikh-tablits-v-yt-slovariakh} @@ -855,6 +879,7 @@ Upd. Нас заставляют переписать эту библиотек Артём Стрельцов, Николай Дегтеринский, Наталия Михненко, ВШЭ. Приступили к этой задаче. +Готов direct, есть pull request complex_key_direct. ### 10.13. Использование Join как generic layout для словарей {#ispolzovanie-join-kak-generic-layout-dlia-slovarei} @@ -872,12 +897,15 @@ Upd. Нас заставляют переписать эту библиотек Использовать эту структуру данных как отдельный вид словарей, как источник для cache словарей или как дополнительный уровень кэширования для cache словарей. +Upd. Задача в финальной стадии готовности. + ### 10.17. Локальный дамп состояния словаря для быстрого старта сервера {#lokalnyi-damp-sostoianiia-slovaria-dlia-bystrogo-starta-servera} ### 10.18. Таблица Join или словарь на удалённом сервере как key-value БД для cache словаря {#tablitsa-join-ili-slovar-na-udalionnom-servere-kak-key-value-bd-dlia-cache-slovaria} ### 10.19. Возможность зарегистрировать некоторые функции, использующие словари, под пользовательскими именами {#vozmozhnost-zaregistrirovat-nekotorye-funktsii-ispolzuiushchie-slovari-pod-polzovatelskimi-imenami} + ## 11. Интерфейсы {#interfeisy} ### 11.1. Вставка состояний агрегатных функций в виде кортежа аргументов или массива кортежей аргументов {#vstavka-sostoianii-agregatnykh-funktsii-v-vide-kortezha-argumentov-ili-massiva-kortezhei-argumentov} @@ -909,17 +937,19 @@ Upd. Александр Крашенинников перешёл в другу В ClickHouse в прошлом году добавили поддержку wire-протокола MySQL. PostgreSQL, так же как MySQL, использует несложный протокол общения между клиентом и сервером, но свой собственный. Поддержка этого протокола является востребованной и откроет новые возможности для ClickHouse. Задача в процессе разработки. +Задача в финальной стадии разработки. ### 11.9. + Доработки ODBC драйвера {#dorabotki-odbc-draivera} Денис Глазачев, Altinity. Хороший прогресс по этой задаче. -### 11.10. Преднастроенные HTTP handlers для запросов {#prednastroennye-http-handlers-dlia-zaprosov} +### 11.10. + Преднастроенные HTTP handlers для запросов {#prednastroennye-http-handlers-dlia-zaprosov} zhang2014, есть pull request. Возможность описать в конфигурационном файле handler (путь в URL) для HTTP запросов к серверу, которому соответствует некоторый параметризованный запрос. Пользователь может вызвать этот обработчик и не должен передавать SQL запрос. + ## 12. Управление пользователями и доступом {#upravlenie-polzovateliami-i-dostupom} ### 12.1. + Role Based Access Control {#role-based-access-control} @@ -935,6 +965,7 @@ Q1. Сделано управление правами полностью, но ### 12.3. Подключение справочника пользователей и прав доступа из LDAP {#podkliuchenie-spravochnika-polzovatelei-i-prav-dostupa-iz-ldap} +Аутентификация через LDAP - Денис Глазачев. [Виталий Баранов](https://github.com/vitlibar) и Денис Глазачев, Altinity. Требует 12.1. Q2. @@ -951,6 +982,7 @@ Q2. [Виталий Баранов](https://github.com/vitlibar). Требует 12.1. Есть pull request. Q2. + ## 13. Разделение ресурсов, multi-tenancy {#razdelenie-resursov-multi-tenancy} ### 13.1. Overcommit запросов по памяти и вытеснение {#overcommit-zaprosov-po-pamiati-i-vytesnenie} @@ -967,6 +999,8 @@ Q2. Обсуждается вариант неудобной реализации. Пока средний приоритет, целимся на Q1/Q2. Вариант реализации выбрал Александр Казаков. Upd. Не уследили, и задачу стали обсуждать менеджеры. +Upd. Задачу смотрит Александр Казаков. + ## 14. Диалект SQL {#dialekt-sql} @@ -1004,6 +1038,8 @@ zhang2014 Результат некоторых агрегатных функций зависит от порядка данных. Предлагается реализовать модификатор ORDER BY, задающий порядок явно. Пример: groupArray(x ORDER BY y, z). +Upd. Есть pull request на DISTINCT. + ### 14.9. Поддержка запроса EXPLAIN {#podderzhka-zaprosa-explain} Требует 2.1. [Николай Кочетов](https://github.com/KochetovNicolai). @@ -1055,7 +1091,6 @@ zhang2014. ### 14.21. Приведение типов для IN (подзапрос) и для JOIN {#privedenie-tipov-dlia-in-podzapros-i-dlia-join} -Павел Потёмкин, ВШЭ. ## 15. Улучшение поддержки JOIN {#uluchshenie-podderzhki-join} @@ -1092,6 +1127,7 @@ Q1. Сделали адаптивный вариант, но вроде он ч Артём Зуйков. + ## 16. Типы данных и функции {#tipy-dannykh-i-funktsii} ### 16.1. + DateTime64 {#datetime64} @@ -1115,6 +1151,7 @@ Upd. Секретного изменения в работе не будет, з ### 16.6. Функции нормализации и хэширования SQL запросов {#funktsii-normalizatsii-i-kheshirovaniia-sql-zaprosov} + ## 17. Работа с географическими данными {#rabota-s-geograficheskimi-dannymi} ### 17.1. Гео-словари для определения региона по координатам {#geo-slovari-dlia-opredeleniia-regiona-po-koordinatam} @@ -1128,6 +1165,7 @@ ClickHouse не является geospatial СУБД. Тем не менее, в Upd. Андрей сделал прототип интерфейса и реализацию-заглушку внутри него. Upd. Андрей сделал прототип более оптимальной структуры данных. +Upd. Есть обнадёживающие результаты. ### 17.2. GIS типы данных и операции {#gis-tipy-dannykh-i-operatsii} @@ -1141,12 +1179,13 @@ Upd. Андрей сделал прототип более оптимально ### 17.4. Ускорение geohash с помощью библиотеки из Аркадии {#uskorenie-geohash-s-pomoshchiu-biblioteki-iz-arkadii} -Предположительно, [Андрей Чулков](https://github.com/achulkov2). Получено одобрение от руководства. +Получено одобрение от руководства. ### 17.5. + Проверки в функции pointInPolygon {#proverki-v-funktsii-pointinpolygon} Сейчас функция тихо не работает в случае полигонов с самопересечениями, надо кидать исключение. + ## 18. Машинное обучение и статистика {#mashinnoe-obuchenie-i-statistika} ### 18.1. Инкрементальная кластеризация данных {#inkrementalnaia-klasterizatsiia-dannykh} @@ -1167,6 +1206,7 @@ Upd. Андрей сделал прототип более оптимально В очереди. + ## 19. Улучшение работы кластера {#uluchshenie-raboty-klastera} ### 19.1. Параллельные кворумные вставки без линеаризуемости {#parallelnye-kvorumnye-vstavki-bez-linearizuemosti} @@ -1236,6 +1276,7 @@ Hold. Полезно для заказчиков внутри Яндекса, н ### 20.4. Поддержка UPDATE с помощью преобразования в DELETE и вставок {#podderzhka-update-s-pomoshchiu-preobrazovaniia-v-delete-i-vstavok} + ## 21. Оптимизации производительности {#optimizatsii-proizvoditelnosti} ### 21.1. + Параллельный парсинг форматов {#parallelnyi-parsing-formatov} @@ -1264,6 +1305,8 @@ Upd. Антон делает эту задачу. Большая часть уж В прошлом году, аналогичное решение сделали для операции ORDER BY. +Upd. Есть pull request для GROUP BY. Приличные результаты. + ### 21.5. + Распараллеливание INSERT при INSERT SELECT, если это необходимо {#rasparallelivanie-insert-pri-insert-select-esli-eto-neobkhodimo} [Vxider](https://github.com/Vxider), ICT @@ -1358,21 +1401,21 @@ Constraints позволяют задать выражение, истиннос В ClickHouse используется неоптимальный вариант top sort. Суть его в том, что из каждого блока достаётся top N записей, а затем, все блоки мержатся. Но доставание top N записей у каждого следующего блока бессмысленно, если мы знаем, что из них в глобальный top N войдёт меньше. Конечно нужно реализовать вариацию на тему priority queue (heap) с быстрым пропуском целых блоков, если ни одна строка не попадёт в накопленный top. -1. Рекурсивный вариант сортировки по кортежам. +2. Рекурсивный вариант сортировки по кортежам. Для сортировки по кортежам используется обычная сортировка с компаратором, который в цикле по элементам кортежа делает виртуальные вызовы `IColumn::compareAt`. Это неоптимально - как из-за короткого цикла по неизвестному в compile-time количеству элементов, так и из-за виртуальных вызовов. Чтобы обойтись без виртуальных вызовов, есть метод `IColumn::getPermutation`. Он используется в случае сортировки по одному столбцу. Есть вариант, что в случае сортировки по кортежу, что-то похожее тоже можно применить… например, сделать метод `updatePermutation`, принимающий аргументы offset и limit, и допереставляющий перестановку в диапазоне значений, в которых предыдущий столбец имел равные значения. -1. RadixSort для сортировки. +3. RadixSort для сортировки. Один наш знакомый начал делать задачу по попытке использования RadixSort для сортировки столбцов. Был сделан вариант indirect сортировки (для `getPermutation`), но не оптимизирован до конца - есть лишние ненужные перекладывания элементов. Для того, чтобы его оптимизировать, придётся добавить немного шаблонной магии (на последнем шаге что-то не копировать, вместо перекладывания индексов - складывать их в готовое место). Также этот человек добавил метод MSD Radix Sort для реализации radix partial sort. Но даже не проверил производительность. Наиболее содержательная часть задачи может состоять в применении Radix Sort для сортировки кортежей, расположенных в оперативке в виде Structure Of Arrays неизвестного в compile-time размера. Это может работать хуже, чем то, что описано в пункте 2… Но попробовать не помешает. -1. Three-way comparison sort. +4. Three-way comparison sort. Виртуальный метод `compareAt` возвращает -1, 0, 1. Но алгоритмы сортировки сравнениями обычно рассчитаны на `operator<` и не могут получить преимущества от three-way comparison. А можно ли написать так, чтобы преимущество было? -1. pdq partial sort +5. pdq partial sort Хороший алгоритм сортировки сравнениями `pdqsort` не имеет варианта partial sort. Заметим, что на практике, почти все сортировки в запросах ClickHouse являются partial\_sort, так как `ORDER BY` почти всегда идёт с `LIMIT`. Кстати, Данила Кутенин уже попробовал это и показал, что в тривиальном случае преимущества нет. Но не очевидно, что нельзя сделать лучше. @@ -1393,6 +1436,7 @@ Constraints позволяют задать выражение, истиннос zhang2014. Есть pull request. + ## 22. Долги и недоделанные возможности {#dolgi-i-nedodelannye-vozmozhnosti} ### 22.1. + Исправление неработающих таймаутов, если используется TLS {#ispravlenie-nerabotaiushchikh-taimautov-esli-ispolzuetsia-tls} @@ -1403,7 +1447,7 @@ zhang2014. N.Vartolomei. -### 22.3. Защита от абсурдно заданных пользователем кодеков {#zashchita-ot-absurdno-zadannykh-polzovatelem-kodekov} +### 22.3. + Защита от абсурдно заданных пользователем кодеков {#zashchita-ot-absurdno-zadannykh-polzovatelem-kodekov} ### 22.4. + Исправление оставшихся deadlocks в табличных RWLock-ах {#ispravlenie-ostavshikhsia-deadlocks-v-tablichnykh-rwlock-akh} @@ -1518,6 +1562,7 @@ Altinity. [Александр Сапин](https://github.com/alesapin) + ## 23. Default Festival {#default-festival} ### 23.1. + Включение minimalistic\_part\_header в ZooKeeper {#vkliuchenie-minimalistic-part-header-v-zookeeper} @@ -1562,6 +1607,7 @@ Q1. [Николай Кочетов](https://github.com/KochetovNicolai). Возможность mlock бинарника сделал Олег Алексеенков [\#3553](https://github.com/ClickHouse/ClickHouse/pull/3553) . Поможет, когда на серверах кроме ClickHouse работает много посторонних программ (мы иногда называем их в шутку «треш-программами»). + ## 24. Экспериментальные задачи {#eksperimentalnye-zadachi} ### 24.1. Веб-интерфейс для просмотра состояния кластера и профилирования запросов {#veb-interfeis-dlia-prosmotra-sostoianiia-klastera-i-profilirovaniia-zaprosov} @@ -1689,6 +1735,8 @@ ClickHouse предоставляет достаточно богатый наб Upd. В компании nVidia выложили прототип, теперь нужна интеграция в систему сборки. Upd. Интеграция в систему сборки - Иван Лежанкин. Upd. Есть прототип bitonic sort. +Upd. Прототип bitonic sort помержен, но целесообразность под вопросом (он работает медленнее). +Наверное надо будет подержать и удалить. ### 24.13. Stream запросы {#stream-zaprosy} @@ -1713,6 +1761,7 @@ Upd. Есть прототип bitonic sort. Максим Серебряков Задача в работе. +Upd. Достигнуты обнадёживающие результаты. ### 24.18. Не TCP протокол передачи файлов при репликации {#ne-tcp-protokol-peredachi-failov-pri-replikatsii} @@ -1761,12 +1810,14 @@ ClickHouse также может использоваться для быстр [\#157](https://github.com/ClickHouse/ClickHouse/issues/157) Есть хороший код в Яндекс.Метрике. Получено согласие от руководства. Михаил Филитов, ВШЭ. +Upd. Есть pull request. Нужно ещё чистить код библиотеки. ### 24.26. Поддержка open tracing или аналогов {#podderzhka-open-tracing-ili-analogov} [\#5182](https://github.com/ClickHouse/ClickHouse/issues/5182) Александр Кожихов, ВШЭ и Яндекс.YT. +Upd. Есть pull request с прототипом. ### 24.27. Реализация алгоритмов min-hash, sim-hash для нечёткого поиска полудубликатов {#realizatsiia-algoritmov-min-hash-sim-hash-dlia-nechiotkogo-poiska-poludublikatov} @@ -1787,6 +1838,7 @@ ucasFL, ICT. [\#7554](https://github.com/ClickHouse/ClickHouse/issues/7554) Жанна Зосимова, ВШЭ. +Upd. Пока поддержали Arrow как формат ввода-вывода. ### 24.30. ClickHouse как графовая СУБД {#clickhouse-kak-grafovaia-subd} @@ -1810,6 +1862,7 @@ Amos Bird, но его решение слишком громоздкое и п Задача в работе, есть pull request. [#10136](https://github.com/ClickHouse/ClickHouse/pull/10136) + ## 25. DevRel {#devrel} ### 25.1. + Перевод инструкции для начинающих разработчиков {#perevod-instruktsii-dlia-nachinaiushchikh-razrabotchikov} @@ -1851,7 +1904,7 @@ Amos Bird, но его решение слишком громоздкое и п ### 25.10. Митапы в России и Беларуси: Москва x2 + митап для разработчиков или хакатон, Санкт-Петербург, Минск, Нижний Новгород, Екатеринбург, Новосибирск и/или Академгородок, Иннополис или Казань {#mitapy-v-rossii-i-belarusi-moskva-x2-mitap-dlia-razrabotchikov-ili-khakaton-sankt-peterburg-minsk-nizhnii-novgorod-ekaterinburg-novosibirsk-iili-akademgorodok-innopolis-ili-kazan} -Екатерина - организация. Upd. Проведено два онлайн митапа на русском. +Екатерина - организация. Upd. Проведено два онлайн митапа на русском и два на английском. ### 25.11. Митапы зарубежные: восток США (Нью Йорк, возможно Raleigh), возможно северо-запад (Сиэтл), Китай (Пекин снова, возможно митап для разработчиков или хакатон), Лондон {#mitapy-zarubezhnye-vostok-ssha-niu-iork-vozmozhno-raleigh-vozmozhno-severo-zapad-sietl-kitai-pekin-snova-vozmozhno-mitap-dlia-razrabotchikov-ili-khakaton-london} @@ -1891,13 +1944,14 @@ Upd. Есть Saint HighLoad online. ### 25.17. Взаимодействие с ВУЗами: ВШЭ, УрФУ, ICT Beijing {#vzaimodeistvie-s-vuzami-vshe-urfu-ict-beijing} -Алексей Миловидов и вся группа разработки +Алексей Миловидов и вся группа разработки. +Благодаря Robert Hodges добавлен CMU. ### 25.18. - Лекция в ШАД {#lektsiia-v-shad} Алексей Миловидов -### 25.19. Участие в курсе разработки на C++ в ШАД {#uchastie-v-kurse-razrabotki-na-c-v-shad} +### 25.19. - Участие в курсе разработки на C++ в ШАД {#uchastie-v-kurse-razrabotki-na-c-v-shad} ### 25.20. Ещё одно сравнение производительности аналитических СУБД {#eshchio-odno-sravnenie-proizvoditelnosti-analiticheskikh-subd} @@ -1905,11 +1959,14 @@ Upd. Есть Saint HighLoad online. Существуют мало известные специализированные СУБД, способные конкурировать с ClickHouse по скорости обработки некоторых классов запросов. Пример: `TDEngine` и `DolphinDB`, `VictoriaMetrics`, а также `Apache Doris` и `LocustDB`. Предлагается изучить и классифицировать архитектурные особенности этих систем - их особенности и преимущества. Установить эти системы, загрузить тестовые данные, изучить производительность. Проанализировать, за счёт чего достигаются преимущества. +Upd. Есть поползновения с TDEngine. + ### 25.21. Повторное награждение контрибьюторов в Китае {#povtornoe-nagrazhdenie-kontribiutorov-v-kitae} ### 25.22. On-site помощь с ClickHouse компаниям в дни рядом с мероприятиями {#on-site-pomoshch-s-clickhouse-kompaniiam-v-dni-riadom-s-meropriiatiiami} -[Иван Блинков](https://github.com/blinkov/) - организация. Проверил мероприятие для турецкой компании. +[Иван Блинков](https://github.com/blinkov/) - организация. Провёл мероприятие для турецкой компании. +Upd. On-site заменяется на Online. ### 25.23. Новый мерч для ClickHouse {#novyi-merch-dlia-clickhouse} From 7a20d95339d64109758c0f4321aade6d2b83376c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 15 May 2020 10:30:26 +0300 Subject: [PATCH 242/738] Enable input_format_with_names_use_header by default --- src/Core/Settings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index ff151e24a99..63375f17d27 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -189,7 +189,7 @@ struct Settings : public SettingsCollection M(SettingUInt64, max_http_get_redirects, 0, "Max number of http GET redirects hops allowed. Make sure additional security measures are in place to prevent a malicious server to redirect your requests to unexpected services.", 0) \ \ M(SettingBool, input_format_skip_unknown_fields, false, "Skip columns with unknown names from input data (it works for JSONEachRow, CSVWithNames, TSVWithNames and TSKV formats).", 0) \ - M(SettingBool, input_format_with_names_use_header, false, "For TSVWithNames and CSVWithNames input formats this controls whether format parser is to assume that column data appear in the input exactly as they are specified in the header.", 0) \ + M(SettingBool, input_format_with_names_use_header, true, "For TSVWithNames and CSVWithNames input formats this controls whether format parser is to assume that column data appear in the input exactly as they are specified in the header.", 0) \ M(SettingBool, input_format_import_nested_json, false, "Map nested JSON data to nested tables (it works for JSONEachRow format).", 0) \ M(SettingBool, input_format_defaults_for_omitted_fields, true, "For input data calculate default expressions for omitted fields (it works for JSONEachRow, CSV and TSV formats).", IMPORTANT) \ M(SettingBool, input_format_tsv_empty_as_default, false, "Treat empty fields in TSV input as default values.", 0) \ From 5710e8068700043bbb319d5748d98efc77a375a5 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 15 May 2020 07:50:01 +0000 Subject: [PATCH 243/738] Bump mkdocs from 1.1.1 to 1.1.2 in /docs/tools Bumps [mkdocs](https://github.com/mkdocs/mkdocs) from 1.1.1 to 1.1.2. - [Release notes](https://github.com/mkdocs/mkdocs/releases) - [Commits](https://github.com/mkdocs/mkdocs/compare/1.1.1...1.1.2) Signed-off-by: dependabot-preview[bot] --- docs/tools/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tools/requirements.txt b/docs/tools/requirements.txt index b675765a2da..246c49738e6 100644 --- a/docs/tools/requirements.txt +++ b/docs/tools/requirements.txt @@ -16,7 +16,7 @@ jsmin==2.2.2 livereload==2.6.1 Markdown==3.2.1 MarkupSafe==1.1.1 -mkdocs==1.1.1 +mkdocs==1.1.2 mkdocs-htmlproofer-plugin==0.0.3 mkdocs-macros-plugin==0.4.7 nltk==3.5 From 0344c9d9dc8c9b3bbee8343b24f0ec8577665e28 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 15 May 2020 11:09:50 +0300 Subject: [PATCH 244/738] Fix build. --- src/Processors/Executors/PullingPipelineExecutor.h | 2 +- src/Processors/tests/processors_test.cpp | 2 -- src/Processors/tests/processors_test_aggregation.cpp | 2 -- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/Processors/Executors/PullingPipelineExecutor.h b/src/Processors/Executors/PullingPipelineExecutor.h index d378a4ee8fe..e51d007e7b1 100644 --- a/src/Processors/Executors/PullingPipelineExecutor.h +++ b/src/Processors/Executors/PullingPipelineExecutor.h @@ -8,7 +8,7 @@ class QueryPipeline; class Block; class Chunk; class LazyOutputFormat; -class BlockStreamProfileInfo; +struct BlockStreamProfileInfo; /// Pulling executor for QueryPipeline. /// Typical usage is: diff --git a/src/Processors/tests/processors_test.cpp b/src/Processors/tests/processors_test.cpp index b0270932234..3c73223e59c 100644 --- a/src/Processors/tests/processors_test.cpp +++ b/src/Processors/tests/processors_test.cpp @@ -9,8 +9,6 @@ #include #include #include -#include -#include #include #include diff --git a/src/Processors/tests/processors_test_aggregation.cpp b/src/Processors/tests/processors_test_aggregation.cpp index af809fab9f2..e3316432ba8 100644 --- a/src/Processors/tests/processors_test_aggregation.cpp +++ b/src/Processors/tests/processors_test_aggregation.cpp @@ -9,8 +9,6 @@ #include #include #include -#include -#include #include #include From d84be03268dc75de0a4408c397e9688a1b71567b Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 15 May 2020 11:17:40 +0300 Subject: [PATCH 245/738] Fix tests. --- src/Processors/Executors/PullingPipelineExecutor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/Executors/PullingPipelineExecutor.cpp b/src/Processors/Executors/PullingPipelineExecutor.cpp index f9b957bf66c..07ca8f2bbf4 100644 --- a/src/Processors/Executors/PullingPipelineExecutor.cpp +++ b/src/Processors/Executors/PullingPipelineExecutor.cpp @@ -123,7 +123,7 @@ bool PullingPipelineExecutor::pull(Block & block, uint64_t milliseconds) { /// In case if timeout exceeded. block.clear(); - return false; + return true; } block = lazy_format->getPort(IOutputFormat::PortKind::Main).getHeader().cloneWithColumns(chunk.detachColumns()); From 9ae37a054c9e672d7143021fc547d43f4bfe7ac4 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 15 May 2020 11:31:10 +0300 Subject: [PATCH 246/738] Fix build without openCL --- programs/server/Server.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 7c119b40ac8..13cb294d03f 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -61,7 +61,7 @@ #include #include "MySQLHandlerFactory.h" -#ifdef USE_OPENCL +#if USE_OPENCL #include "Common/BitonicSort.h" #endif @@ -225,7 +225,7 @@ int Server::main(const std::vector & /*args*/) registerDictionaries(); registerDisks(); -#if defined (USE_OPENCL) +#if USE_OPENCL BitonicSort::getInstance().configure(); #endif From 34ce5bec48295289f3c24acde1896787210d9077 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 15 May 2020 12:06:21 +0300 Subject: [PATCH 247/738] Try fix tests. --- src/Functions/in.cpp | 6 +++++- src/Functions/ya.make | 1 - 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Functions/in.cpp b/src/Functions/in.cpp index 1b08df861cd..81b75f67764 100644 --- a/src/Functions/in.cpp +++ b/src/Functions/in.cpp @@ -70,7 +70,11 @@ public: return std::make_shared(); } - bool useDefaultImplementationForConstants() const override { return true; } + bool useDefaultImplementationForConstants() const override + { + /// Never return constant for -IgnoreSet functions to avoid constant folding. + return !ignore_set; + } bool useDefaultImplementationForNulls() const override { return null_is_skipped; } diff --git a/src/Functions/ya.make b/src/Functions/ya.make index afce90f0dee..04a10d7ce26 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -207,7 +207,6 @@ SRCS( ifNull.cpp IFunction.cpp ignore.cpp - ignoreExceptNull.cpp in.cpp intDiv.cpp intDivOrZero.cpp From 33d4ae89f207db46c54d8a5e68da1dd9969b217c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=82=D0=B5=D0=BC=20=D0=A1=D1=82=D1=80=D0=B5?= =?UTF-8?q?=D0=BB=D1=8C=D1=86=D0=BE=D0=B2?= Date: Fri, 15 May 2020 12:16:01 +0300 Subject: [PATCH 248/738] fixed yandex synchonization error --- src/Dictionaries/ya.make | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Dictionaries/ya.make b/src/Dictionaries/ya.make index 7005f9b90f8..e47b55d5254 100644 --- a/src/Dictionaries/ya.make +++ b/src/Dictionaries/ya.make @@ -25,6 +25,7 @@ SRCS( ComplexKeyCacheDictionary_setAttributeValue.cpp ComplexKeyCacheDictionary_setDefaultAttributeValue.cpp ComplexKeyHashedDictionary.cpp + ComplexKeyDirectDictionary.cpp DictionaryBlockInputStreamBase.cpp DictionaryFactory.cpp DictionarySourceFactory.cpp From ad42ba09c8b8366754332346c4e55006b824b95a Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 15 May 2020 12:48:57 +0300 Subject: [PATCH 249/738] Fix log tables. --- src/Interpreters/TextLog.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Interpreters/TextLog.cpp b/src/Interpreters/TextLog.cpp index b5f1d987b91..35e027616f6 100644 --- a/src/Interpreters/TextLog.cpp +++ b/src/Interpreters/TextLog.cpp @@ -68,6 +68,8 @@ void TextLogElement::appendToBlock(Block & block) const columns[i++]->insert(source_file); columns[i++]->insert(source_line); + + block.setColumns(std::move(columns)); } TextLog::TextLog(Context & context_, const String & database_name_, From 70ce0149b1597ec67992c06e45319dbe3127011f Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 15 May 2020 13:10:13 +0300 Subject: [PATCH 250/738] Try fix build. --- src/Common/COW.h | 2 +- src/Functions/concat.cpp | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Common/COW.h b/src/Common/COW.h index 048bf134295..3c2171436bf 100644 --- a/src/Common/COW.h +++ b/src/Common/COW.h @@ -177,7 +177,7 @@ protected: public: static MutablePtr mutate(Ptr ptr) { - return ptr.shallowMutate(); + return ptr->shallowMutate(); } MutablePtr assumeMutable() const diff --git a/src/Functions/concat.cpp b/src/Functions/concat.cpp index d83fbed30e9..4eea68de1f6 100644 --- a/src/Functions/concat.cpp +++ b/src/Functions/concat.cpp @@ -73,6 +73,12 @@ public: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { + if (arguments.size() < 2) + throw Exception( + "Number of arguments for function " + getName() + " doesn't match: passed " + toString(arguments.size()) + + ", should be at least 2.", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + /// Format function is not proven to be faster for two arguments. /// Actually there is overhead of 2 to 5 extra instructions for each string for checking empty strings in FormatImpl. /// Though, benchmarks are really close, for most examples we saw executeBinary is slightly faster (0-3%). From af0328bf3059ac514c8619cc038b860ed214c050 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 15 May 2020 13:17:07 +0300 Subject: [PATCH 251/738] Fix log tables. --- src/Interpreters/MetricLog.cpp | 2 ++ src/Interpreters/QueryThreadLog.cpp | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/Interpreters/MetricLog.cpp b/src/Interpreters/MetricLog.cpp index bd898170705..96fe55c26e6 100644 --- a/src/Interpreters/MetricLog.cpp +++ b/src/Interpreters/MetricLog.cpp @@ -50,6 +50,8 @@ void MetricLogElement::appendToBlock(Block & block) const for (size_t i = 0, end = CurrentMetrics::end(); i < end; ++i) columns[column_idx++]->insert(current_metrics[i]); + + block.setColumns(std::move(columns)); } diff --git a/src/Interpreters/QueryThreadLog.cpp b/src/Interpreters/QueryThreadLog.cpp index 1a2975bd43b..f539a720449 100644 --- a/src/Interpreters/QueryThreadLog.cpp +++ b/src/Interpreters/QueryThreadLog.cpp @@ -107,6 +107,8 @@ void QueryThreadLogElement::appendToBlock(Block & block) const columns[i++]->insertDefault(); columns[i++]->insertDefault(); } + + block.setColumns(std::move(columns)); } } From 3537c1a65e8a144454b1b9e7bac48238a023108e Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 15 May 2020 13:26:24 +0300 Subject: [PATCH 252/738] Added comment. --- src/Processors/Executors/PullingPipelineExecutor.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Processors/Executors/PullingPipelineExecutor.h b/src/Processors/Executors/PullingPipelineExecutor.h index e51d007e7b1..f3b06fc618a 100644 --- a/src/Processors/Executors/PullingPipelineExecutor.h +++ b/src/Processors/Executors/PullingPipelineExecutor.h @@ -42,6 +42,7 @@ public: /// Get query profile info. BlockStreamProfileInfo & getProfileInfo(); + /// Internal executor data. struct Data; private: From 26606dd64008ffa995b1b649797fdaca5206a9d3 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 15 May 2020 13:26:44 +0300 Subject: [PATCH 253/738] Fix polymorphic parts --- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 15 +++++++++++---- src/Storages/MergeTree/IMergeTreeDataPart.h | 10 +++++++++- .../MergeTree/MergeTreeBlockReadUtils.cpp | 11 ++++++++--- .../01278_alter_rename_combination.reference | 4 ++++ .../01278_alter_rename_combination.sql | 9 +++++++++ 5 files changed, 41 insertions(+), 8 deletions(-) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index a5a7f1f4494..59bd2830789 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -356,19 +356,26 @@ size_t IMergeTreeDataPart::getFileSizeOrZero(const String & file_name) const String IMergeTreeDataPart::getColumnNameWithMinumumCompressedSize() const { const auto & storage_columns = storage.getColumns().getAllPhysical(); - const std::string * minimum_size_column = nullptr; + auto alter_conversions = storage.getAlterConversionsForPart(shared_from_this()); + + std::optional minimum_size_column; UInt64 minimum_size = std::numeric_limits::max(); for (const auto & column : storage_columns) { - if (!hasColumnFiles(column.name, *column.type)) + auto column_name = column.name; + auto column_type = column.type; + if (alter_conversions.isColumnRenamed(column.name)) + column_name = alter_conversions.getColumnOldName(column.name); + + if (!hasColumnFiles(column_name, *column_type)) continue; - const auto size = getColumnSize(column.name, *column.type).data_compressed; + const auto size = getColumnSize(column_name, *column_type).data_compressed; if (size < minimum_size) { minimum_size = size; - minimum_size_column = &column.name; + minimum_size_column = column_name; } } diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index 784a3ff047b..338c1db1f06 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -128,6 +128,10 @@ public: String getNewName(const MergeTreePartInfo & new_part_info) const; /// Returns column position in part structure or std::nullopt if it's missing in part. + /// + /// NOTE: Doesn't take column renames into account, if some column renames + /// take place, you must take original name of column for this part from + /// storage and pass it to this method. std::optional getColumnPosition(const String & column_name) const; /// Returns the name of a column with minimum compressed size (as returned by getColumnSize()). @@ -291,7 +295,11 @@ public: /// Makes full clone of part in detached/ on another disk void makeCloneOnDiskDetached(const ReservationPtr & reservation) const; - /// Checks that .bin and .mrk files exist + /// Checks that .bin and .mrk files exist. + /// + /// NOTE: Doesn't take column renames into account, if some column renames + /// take place, you must take original name of column for this part from + /// storage and pass it to this method. virtual bool hasColumnFiles(const String & /* column */, const IDataType & /* type */) const{ return false; } static UInt64 calculateTotalSizeOnDisk(const DiskPtr & disk_, const String & from); diff --git a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp index cd0827c8a0e..310c566fb19 100644 --- a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp +++ b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp @@ -21,18 +21,23 @@ NameSet injectRequiredColumns(const MergeTreeData & storage, const MergeTreeData auto all_column_files_missing = true; const auto & storage_columns = storage.getColumns(); + auto alter_conversions = storage.getAlterConversionsForPart(part); for (size_t i = 0; i < columns.size(); ++i) { - const auto & column_name = columns[i]; + /// possibly renamed + auto column_name_in_part = columns[i]; + + if (alter_conversions.isColumnRenamed(column_name_in_part)) + column_name_in_part = alter_conversions.getColumnOldName(column_name_in_part); /// column has files and hence does not require evaluation - if (part->hasColumnFiles(column_name, *storage_columns.getPhysical(column_name).type)) + if (part->hasColumnFiles(column_name_in_part, *storage_columns.getPhysical(columns[i]).type)) { all_column_files_missing = false; continue; } - const auto column_default = storage_columns.getDefault(column_name); + const auto column_default = storage_columns.getDefault(columns[i]); if (!column_default) continue; diff --git a/tests/queries/0_stateless/01278_alter_rename_combination.reference b/tests/queries/0_stateless/01278_alter_rename_combination.reference index 3d77561b460..3f00378b4b7 100644 --- a/tests/queries/0_stateless/01278_alter_rename_combination.reference +++ b/tests/queries/0_stateless/01278_alter_rename_combination.reference @@ -9,3 +9,7 @@ k v1 v2 CREATE TABLE default.rename_table_polymorphic\n(\n `key` Int32, \n `old_value1` Int32, \n `value1` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS min_rows_for_wide_part = 10000, index_granularity = 8192 key old_value1 value1 1 2 3 +CREATE TABLE default.rename_table_polymorphic\n(\n `k` Int32, \n `v1` Int32, \n `v2` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS min_rows_for_wide_part = 10000, index_granularity = 8192 +k v1 v2 +1 2 3 +4 5 6 diff --git a/tests/queries/0_stateless/01278_alter_rename_combination.sql b/tests/queries/0_stateless/01278_alter_rename_combination.sql index 06f756cc0fd..fa73362622c 100644 --- a/tests/queries/0_stateless/01278_alter_rename_combination.sql +++ b/tests/queries/0_stateless/01278_alter_rename_combination.sql @@ -43,4 +43,13 @@ SHOW CREATE TABLE rename_table_polymorphic; SELECT * FROM rename_table_polymorphic FORMAT TSVWithNames; +INSERT INTO rename_table_polymorphic VALUES (4, 5, 6); + +-- rename all columns simultaneously +ALTER TABLE rename_table_polymorphic RENAME COLUMN old_value1 TO v1, RENAME COLUMN value1 TO v2, RENAME COLUMN key to k; + +SHOW CREATE TABLE rename_table_polymorphic; + +SELECT * FROM rename_table_polymorphic ORDER BY k FORMAT TSVWithNames; + DROP TABLE IF EXISTS rename_table_polymorphic; From 500b9bac03b15533a561ea6375b50f513c30449e Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 15 May 2020 13:48:55 +0300 Subject: [PATCH 254/738] fix config prefix --- src/Dictionaries/MongoDBDictionarySource.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Dictionaries/MongoDBDictionarySource.cpp b/src/Dictionaries/MongoDBDictionarySource.cpp index 4baa66728c9..d9601f29a03 100644 --- a/src/Dictionaries/MongoDBDictionarySource.cpp +++ b/src/Dictionaries/MongoDBDictionarySource.cpp @@ -11,11 +11,12 @@ void registerDictionarySourceMongoDB(DictionarySourceFactory & factory) auto create_mongo_db_dictionary = []( const DictionaryStructure & dict_struct, const Poco::Util::AbstractConfiguration & config, - const std::string & config_prefix, + const std::string & root_config_prefix, Block & sample_block, const Context &, bool /* check_config */) { + const auto config_prefix = root_config_prefix + ".mongodb"; return std::make_unique(dict_struct, config.getString(config_prefix + ".uri", ""), config.getString(config_prefix + ".host", ""), From bad61a659c639127e582a35b25a512c970d80213 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Fri, 15 May 2020 15:15:13 +0400 Subject: [PATCH 255/738] Allow Linux aarch64 --- cmake/find/ldap.cmake | 11 +- contrib/openldap-cmake/CMakeLists.txt | 4 + .../linux_aarch64/include/lber_types.h | 63 + .../linux_aarch64/include/ldap_config.h | 74 ++ .../linux_aarch64/include/ldap_features.h | 61 + .../linux_aarch64/include/portable.h | 1169 +++++++++++++++++ 6 files changed, 1379 insertions(+), 3 deletions(-) create mode 100644 contrib/openldap-cmake/linux_aarch64/include/lber_types.h create mode 100644 contrib/openldap-cmake/linux_aarch64/include/ldap_config.h create mode 100644 contrib/openldap-cmake/linux_aarch64/include/ldap_features.h create mode 100644 contrib/openldap-cmake/linux_aarch64/include/portable.h diff --git a/cmake/find/ldap.cmake b/cmake/find/ldap.cmake index 204a785b43e..230727819e4 100644 --- a/cmake/find/ldap.cmake +++ b/cmake/find/ldap.cmake @@ -32,12 +32,17 @@ if (ENABLE_LDAP) "${_system_processor}" STREQUAL "x64" ) set (_system_processor "x86_64") + elseif ( + "${_system_processor}" STREQUAL "arm64" + ) + set (_system_processor "aarch64") endif () if ( - ( "${_system_name}" STREQUAL "linux" AND "${_system_processor}" STREQUAL "x86_64" ) OR - ( "${_system_name}" STREQUAL "freebsd" AND "${_system_processor}" STREQUAL "x86_64" ) OR - ( "${_system_name}" STREQUAL "darwin" AND "${_system_processor}" STREQUAL "x86_64" ) + ( "${_system_name}" STREQUAL "linux" AND "${_system_processor}" STREQUAL "x86_64" ) OR + ( "${_system_name}" STREQUAL "linux" AND "${_system_processor}" STREQUAL "aarch64" ) OR + ( "${_system_name}" STREQUAL "freebsd" AND "${_system_processor}" STREQUAL "x86_64" ) OR + ( "${_system_name}" STREQUAL "darwin" AND "${_system_processor}" STREQUAL "x86_64" ) ) set (_ldap_supported_platform TRUE) endif () diff --git a/contrib/openldap-cmake/CMakeLists.txt b/contrib/openldap-cmake/CMakeLists.txt index c69cb1009ff..b0a5f4048ff 100644 --- a/contrib/openldap-cmake/CMakeLists.txt +++ b/contrib/openldap-cmake/CMakeLists.txt @@ -28,6 +28,10 @@ if( "${_system_processor}" STREQUAL "x64" ) set(_system_processor "x86_64") +elseif( + "${_system_processor}" STREQUAL "arm64" +) + set (_system_processor "aarch64") endif() set(_extra_build_dir "${CMAKE_CURRENT_SOURCE_DIR}/${_system_name}_${_system_processor}") diff --git a/contrib/openldap-cmake/linux_aarch64/include/lber_types.h b/contrib/openldap-cmake/linux_aarch64/include/lber_types.h new file mode 100644 index 00000000000..dbd59430527 --- /dev/null +++ b/contrib/openldap-cmake/linux_aarch64/include/lber_types.h @@ -0,0 +1,63 @@ +/* include/lber_types.h. Generated from lber_types.hin by configure. */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software . + * + * Copyright 1998-2020 The OpenLDAP Foundation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +/* + * LBER types + */ + +#ifndef _LBER_TYPES_H +#define _LBER_TYPES_H + +#include + +LDAP_BEGIN_DECL + +/* LBER boolean, enum, integers (32 bits or larger) */ +#define LBER_INT_T int + +/* LBER tags (32 bits or larger) */ +#define LBER_TAG_T long + +/* LBER socket descriptor */ +#define LBER_SOCKET_T int + +/* LBER lengths (32 bits or larger) */ +#define LBER_LEN_T long + +/* ------------------------------------------------------------ */ + +/* booleans, enumerations, and integers */ +typedef LBER_INT_T ber_int_t; + +/* signed and unsigned versions */ +typedef signed LBER_INT_T ber_sint_t; +typedef unsigned LBER_INT_T ber_uint_t; + +/* tags */ +typedef unsigned LBER_TAG_T ber_tag_t; + +/* "socket" descriptors */ +typedef LBER_SOCKET_T ber_socket_t; + +/* lengths */ +typedef unsigned LBER_LEN_T ber_len_t; + +/* signed lengths */ +typedef signed LBER_LEN_T ber_slen_t; + +LDAP_END_DECL + +#endif /* _LBER_TYPES_H */ diff --git a/contrib/openldap-cmake/linux_aarch64/include/ldap_config.h b/contrib/openldap-cmake/linux_aarch64/include/ldap_config.h new file mode 100644 index 00000000000..89f7b40b884 --- /dev/null +++ b/contrib/openldap-cmake/linux_aarch64/include/ldap_config.h @@ -0,0 +1,74 @@ +/* include/ldap_config.h. Generated from ldap_config.hin by configure. */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software . + * + * Copyright 1998-2020 The OpenLDAP Foundation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +/* + * This file works in conjunction with OpenLDAP configure system. + * If you do no like the values below, adjust your configure options. + */ + +#ifndef _LDAP_CONFIG_H +#define _LDAP_CONFIG_H + +/* directory separator */ +#ifndef LDAP_DIRSEP +#ifndef _WIN32 +#define LDAP_DIRSEP "/" +#else +#define LDAP_DIRSEP "\\" +#endif +#endif + +/* directory for temporary files */ +#if defined(_WIN32) +# define LDAP_TMPDIR "C:\\." /* we don't have much of a choice */ +#elif defined( _P_tmpdir ) +# define LDAP_TMPDIR _P_tmpdir +#elif defined( P_tmpdir ) +# define LDAP_TMPDIR P_tmpdir +#elif defined( _PATH_TMPDIR ) +# define LDAP_TMPDIR _PATH_TMPDIR +#else +# define LDAP_TMPDIR LDAP_DIRSEP "tmp" +#endif + +/* directories */ +#ifndef LDAP_BINDIR +#define LDAP_BINDIR "/tmp/ldap-prefix/bin" +#endif +#ifndef LDAP_SBINDIR +#define LDAP_SBINDIR "/tmp/ldap-prefix/sbin" +#endif +#ifndef LDAP_DATADIR +#define LDAP_DATADIR "/tmp/ldap-prefix/share/openldap" +#endif +#ifndef LDAP_SYSCONFDIR +#define LDAP_SYSCONFDIR "/tmp/ldap-prefix/etc/openldap" +#endif +#ifndef LDAP_LIBEXECDIR +#define LDAP_LIBEXECDIR "/tmp/ldap-prefix/libexec" +#endif +#ifndef LDAP_MODULEDIR +#define LDAP_MODULEDIR "/tmp/ldap-prefix/libexec/openldap" +#endif +#ifndef LDAP_RUNDIR +#define LDAP_RUNDIR "/tmp/ldap-prefix/var" +#endif +#ifndef LDAP_LOCALEDIR +#define LDAP_LOCALEDIR "" +#endif + + +#endif /* _LDAP_CONFIG_H */ diff --git a/contrib/openldap-cmake/linux_aarch64/include/ldap_features.h b/contrib/openldap-cmake/linux_aarch64/include/ldap_features.h new file mode 100644 index 00000000000..f0cc7c3626f --- /dev/null +++ b/contrib/openldap-cmake/linux_aarch64/include/ldap_features.h @@ -0,0 +1,61 @@ +/* include/ldap_features.h. Generated from ldap_features.hin by configure. */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software . + * + * Copyright 1998-2020 The OpenLDAP Foundation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +/* + * LDAP Features + */ + +#ifndef _LDAP_FEATURES_H +#define _LDAP_FEATURES_H 1 + +/* OpenLDAP API version macros */ +#define LDAP_VENDOR_VERSION 20501 +#define LDAP_VENDOR_VERSION_MAJOR 2 +#define LDAP_VENDOR_VERSION_MINOR 5 +#define LDAP_VENDOR_VERSION_PATCH X + +/* +** WORK IN PROGRESS! +** +** OpenLDAP reentrancy/thread-safeness should be dynamically +** checked using ldap_get_option(). +** +** The -lldap implementation is not thread-safe. +** +** The -lldap_r implementation is: +** LDAP_API_FEATURE_THREAD_SAFE (basic thread safety) +** but also be: +** LDAP_API_FEATURE_SESSION_THREAD_SAFE +** LDAP_API_FEATURE_OPERATION_THREAD_SAFE +** +** The preprocessor flag LDAP_API_FEATURE_X_OPENLDAP_THREAD_SAFE +** can be used to determine if -lldap_r is available at compile +** time. You must define LDAP_THREAD_SAFE if and only if you +** link with -lldap_r. +** +** If you fail to define LDAP_THREAD_SAFE when linking with +** -lldap_r or define LDAP_THREAD_SAFE when linking with -lldap, +** provided header definitions and declarations may be incorrect. +** +*/ + +/* is -lldap_r available or not */ +#define LDAP_API_FEATURE_X_OPENLDAP_THREAD_SAFE 1 + +/* LDAP v2 Referrals */ +/* #undef LDAP_API_FEATURE_X_OPENLDAP_V2_REFERRALS */ + +#endif /* LDAP_FEATURES */ diff --git a/contrib/openldap-cmake/linux_aarch64/include/portable.h b/contrib/openldap-cmake/linux_aarch64/include/portable.h new file mode 100644 index 00000000000..2924b6713a4 --- /dev/null +++ b/contrib/openldap-cmake/linux_aarch64/include/portable.h @@ -0,0 +1,1169 @@ +/* include/portable.h. Generated from portable.hin by configure. */ +/* include/portable.hin. Generated from configure.in by autoheader. */ + + +/* begin of portable.h.pre */ +/* This work is part of OpenLDAP Software . + * + * Copyright 1998-2020 The OpenLDAP Foundation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#ifndef _LDAP_PORTABLE_H +#define _LDAP_PORTABLE_H + +/* define this if needed to get reentrant functions */ +#ifndef REENTRANT +#define REENTRANT 1 +#endif +#ifndef _REENTRANT +#define _REENTRANT 1 +#endif + +/* define this if needed to get threadsafe functions */ +#ifndef THREADSAFE +#define THREADSAFE 1 +#endif +#ifndef _THREADSAFE +#define _THREADSAFE 1 +#endif +#ifndef THREAD_SAFE +#define THREAD_SAFE 1 +#endif +#ifndef _THREAD_SAFE +#define _THREAD_SAFE 1 +#endif + +#ifndef _SGI_MP_SOURCE +#define _SGI_MP_SOURCE 1 +#endif + +/* end of portable.h.pre */ + + +/* Define if building universal (internal helper macro) */ +/* #undef AC_APPLE_UNIVERSAL_BUILD */ + +/* define to use both and */ +/* #undef BOTH_STRINGS_H */ + +/* define if cross compiling */ +/* #undef CROSS_COMPILING */ + +/* set to the number of arguments ctime_r() expects */ +#define CTIME_R_NARGS 2 + +/* define if toupper() requires islower() */ +/* #undef C_UPPER_LOWER */ + +/* define if sys_errlist is not declared in stdio.h or errno.h */ +/* #undef DECL_SYS_ERRLIST */ + +/* define to enable slapi library */ +/* #undef ENABLE_SLAPI */ + +/* defined to be the EXE extension */ +#define EXEEXT "" + +/* set to the number of arguments gethostbyaddr_r() expects */ +#define GETHOSTBYADDR_R_NARGS 8 + +/* set to the number of arguments gethostbyname_r() expects */ +#define GETHOSTBYNAME_R_NARGS 6 + +/* Define to 1 if `TIOCGWINSZ' requires . */ +#define GWINSZ_IN_SYS_IOCTL 1 + +/* define if you have AIX security lib */ +/* #undef HAVE_AIX_SECURITY */ + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_INET_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_NAMESER_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ASSERT_H 1 + +/* Define to 1 if you have the `bcopy' function. */ +#define HAVE_BCOPY 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_BITS_TYPES_H 1 + +/* Define to 1 if you have the `chroot' function. */ +#define HAVE_CHROOT 1 + +/* Define to 1 if you have the `closesocket' function. */ +/* #undef HAVE_CLOSESOCKET */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_CONIO_H */ + +/* define if crypt(3) is available */ +/* #undef HAVE_CRYPT */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_CRYPT_H */ + +/* define if crypt_r() is also available */ +/* #undef HAVE_CRYPT_R */ + +/* Define to 1 if you have the `ctime_r' function. */ +#define HAVE_CTIME_R 1 + +/* define if you have Cyrus SASL */ +/* #undef HAVE_CYRUS_SASL */ + +/* define if your system supports /dev/poll */ +/* #undef HAVE_DEVPOLL */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_DIRECT_H */ + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +#define HAVE_DIRENT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_DLFCN_H 1 + +/* Define to 1 if you don't have `vprintf' but do have `_doprnt.' */ +/* #undef HAVE_DOPRNT */ + +/* define if system uses EBCDIC instead of ASCII */ +/* #undef HAVE_EBCDIC */ + +/* Define to 1 if you have the `endgrent' function. */ +#define HAVE_ENDGRENT 1 + +/* Define to 1 if you have the `endpwent' function. */ +#define HAVE_ENDPWENT 1 + +/* define if your system supports epoll */ +#define HAVE_EPOLL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ERRNO_H 1 + +/* Define to 1 if you have the `fcntl' function. */ +#define HAVE_FCNTL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_FCNTL_H 1 + +/* define if you actually have FreeBSD fetch(3) */ +/* #undef HAVE_FETCH */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_FILIO_H */ + +/* Define to 1 if you have the `flock' function. */ +#define HAVE_FLOCK 1 + +/* Define to 1 if you have the `fstat' function. */ +#define HAVE_FSTAT 1 + +/* Define to 1 if you have the `gai_strerror' function. */ +#define HAVE_GAI_STRERROR 1 + +/* Define to 1 if you have the `getaddrinfo' function. */ +#define HAVE_GETADDRINFO 1 + +/* Define to 1 if you have the `getdtablesize' function. */ +#define HAVE_GETDTABLESIZE 1 + +/* Define to 1 if you have the `geteuid' function. */ +#define HAVE_GETEUID 1 + +/* Define to 1 if you have the `getgrgid' function. */ +#define HAVE_GETGRGID 1 + +/* Define to 1 if you have the `gethostbyaddr_r' function. */ +#define HAVE_GETHOSTBYADDR_R 1 + +/* Define to 1 if you have the `gethostbyname_r' function. */ +#define HAVE_GETHOSTBYNAME_R 1 + +/* Define to 1 if you have the `gethostname' function. */ +#define HAVE_GETHOSTNAME 1 + +/* Define to 1 if you have the `getnameinfo' function. */ +#define HAVE_GETNAMEINFO 1 + +/* Define to 1 if you have the `getopt' function. */ +#define HAVE_GETOPT 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_GETOPT_H 1 + +/* Define to 1 if you have the `getpassphrase' function. */ +/* #undef HAVE_GETPASSPHRASE */ + +/* Define to 1 if you have the `getpeereid' function. */ +/* #undef HAVE_GETPEEREID */ + +/* Define to 1 if you have the `getpeerucred' function. */ +/* #undef HAVE_GETPEERUCRED */ + +/* Define to 1 if you have the `getpwnam' function. */ +#define HAVE_GETPWNAM 1 + +/* Define to 1 if you have the `getpwuid' function. */ +#define HAVE_GETPWUID 1 + +/* Define to 1 if you have the `getspnam' function. */ +#define HAVE_GETSPNAM 1 + +/* Define to 1 if you have the `gettimeofday' function. */ +#define HAVE_GETTIMEOFDAY 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_GMP_H */ + +/* Define to 1 if you have the `gmtime_r' function. */ +#define HAVE_GMTIME_R 1 + +/* define if you have GNUtls */ +/* #undef HAVE_GNUTLS */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_GNUTLS_GNUTLS_H */ + +/* if you have GNU Pth */ +/* #undef HAVE_GNU_PTH */ + +/* Define to 1 if you have the header file. */ +#define HAVE_GRP_H 1 + +/* Define to 1 if you have the `hstrerror' function. */ +#define HAVE_HSTRERROR 1 + +/* define to you inet_aton(3) is available */ +#define HAVE_INET_ATON 1 + +/* Define to 1 if you have the `inet_ntoa_b' function. */ +/* #undef HAVE_INET_NTOA_B */ + +/* Define to 1 if you have the `inet_ntop' function. */ +#define HAVE_INET_NTOP 1 + +/* Define to 1 if you have the `initgroups' function. */ +#define HAVE_INITGROUPS 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_INTTYPES_H 1 + +/* Define to 1 if you have the `ioctl' function. */ +#define HAVE_IOCTL 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_IO_H */ + +/* define if your system supports kqueue */ +/* #undef HAVE_KQUEUE */ + +/* Define to 1 if you have the `gen' library (-lgen). */ +/* #undef HAVE_LIBGEN */ + +/* Define to 1 if you have the `gmp' library (-lgmp). */ +/* #undef HAVE_LIBGMP */ + +/* Define to 1 if you have the `inet' library (-linet). */ +/* #undef HAVE_LIBINET */ + +/* define if you have libtool -ltdl */ +/* #undef HAVE_LIBLTDL */ + +/* Define to 1 if you have the `net' library (-lnet). */ +/* #undef HAVE_LIBNET */ + +/* Define to 1 if you have the `nsl' library (-lnsl). */ +/* #undef HAVE_LIBNSL */ + +/* Define to 1 if you have the `nsl_s' library (-lnsl_s). */ +/* #undef HAVE_LIBNSL_S */ + +/* Define to 1 if you have the `socket' library (-lsocket). */ +/* #undef HAVE_LIBSOCKET */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_LIBUTIL_H */ + +/* Define to 1 if you have the `V3' library (-lV3). */ +/* #undef HAVE_LIBV3 */ + +/* Define to 1 if you have the header file. */ +#define HAVE_LIMITS_H 1 + +/* if you have LinuxThreads */ +/* #undef HAVE_LINUX_THREADS */ + +/* Define to 1 if you have the header file. */ +#define HAVE_LOCALE_H 1 + +/* Define to 1 if you have the `localtime_r' function. */ +#define HAVE_LOCALTIME_R 1 + +/* Define to 1 if you have the `lockf' function. */ +#define HAVE_LOCKF 1 + +/* Define to 1 if the system has the type `long long'. */ +#define HAVE_LONG_LONG 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_LTDL_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_MALLOC_H 1 + +/* Define to 1 if you have the `memcpy' function. */ +#define HAVE_MEMCPY 1 + +/* Define to 1 if you have the `memmove' function. */ +#define HAVE_MEMMOVE 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_MEMORY_H 1 + +/* Define to 1 if you have the `memrchr' function. */ +#define HAVE_MEMRCHR 1 + +/* Define to 1 if you have the `mkstemp' function. */ +#define HAVE_MKSTEMP 1 + +/* Define to 1 if you have the `mktemp' function. */ +#define HAVE_MKTEMP 1 + +/* define this if you have mkversion */ +#define HAVE_MKVERSION 1 + +/* Define to 1 if you have the header file, and it defines `DIR'. */ +/* #undef HAVE_NDIR_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_NETINET_TCP_H 1 + +/* define if strerror_r returns char* instead of int */ +/* #undef HAVE_NONPOSIX_STRERROR_R */ + +/* if you have NT Event Log */ +/* #undef HAVE_NT_EVENT_LOG */ + +/* if you have NT Service Manager */ +/* #undef HAVE_NT_SERVICE_MANAGER */ + +/* if you have NT Threads */ +/* #undef HAVE_NT_THREADS */ + +/* define if you have OpenSSL */ +#define HAVE_OPENSSL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_OPENSSL_BN_H 1 + +/* define if you have OpenSSL with CRL checking capability */ +#define HAVE_OPENSSL_CRL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_OPENSSL_CRYPTO_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_OPENSSL_SSL_H 1 + +/* Define to 1 if you have the `pipe' function. */ +#define HAVE_PIPE 1 + +/* Define to 1 if you have the `poll' function. */ +#define HAVE_POLL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_POLL_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_PROCESS_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_PSAP_H */ + +/* define to pthreads API spec revision */ +#define HAVE_PTHREADS 10 + +/* define if you have pthread_detach function */ +#define HAVE_PTHREAD_DETACH 1 + +/* Define to 1 if you have the `pthread_getconcurrency' function. */ +#define HAVE_PTHREAD_GETCONCURRENCY 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_PTHREAD_H 1 + +/* Define to 1 if you have the `pthread_kill' function. */ +#define HAVE_PTHREAD_KILL 1 + +/* Define to 1 if you have the `pthread_kill_other_threads_np' function. */ +/* #undef HAVE_PTHREAD_KILL_OTHER_THREADS_NP */ + +/* define if you have pthread_rwlock_destroy function */ +#define HAVE_PTHREAD_RWLOCK_DESTROY 1 + +/* Define to 1 if you have the `pthread_setconcurrency' function. */ +#define HAVE_PTHREAD_SETCONCURRENCY 1 + +/* Define to 1 if you have the `pthread_yield' function. */ +#define HAVE_PTHREAD_YIELD 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_PTH_H */ + +/* Define to 1 if the system has the type `ptrdiff_t'. */ +#define HAVE_PTRDIFF_T 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_PWD_H 1 + +/* Define to 1 if you have the `read' function. */ +#define HAVE_READ 1 + +/* Define to 1 if you have the `recv' function. */ +#define HAVE_RECV 1 + +/* Define to 1 if you have the `recvfrom' function. */ +#define HAVE_RECVFROM 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_REGEX_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_RESOLV_H */ + +/* define if you have res_query() */ +/* #undef HAVE_RES_QUERY */ + +/* define if OpenSSL needs RSAref */ +/* #undef HAVE_RSAREF */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SASL_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SASL_SASL_H */ + +/* define if your SASL library has sasl_version() */ +/* #undef HAVE_SASL_VERSION */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SCHED_H 1 + +/* Define to 1 if you have the `sched_yield' function. */ +#define HAVE_SCHED_YIELD 1 + +/* Define to 1 if you have the `send' function. */ +#define HAVE_SEND 1 + +/* Define to 1 if you have the `sendmsg' function. */ +#define HAVE_SENDMSG 1 + +/* Define to 1 if you have the `sendto' function. */ +#define HAVE_SENDTO 1 + +/* Define to 1 if you have the `setegid' function. */ +#define HAVE_SETEGID 1 + +/* Define to 1 if you have the `seteuid' function. */ +#define HAVE_SETEUID 1 + +/* Define to 1 if you have the `setgid' function. */ +#define HAVE_SETGID 1 + +/* Define to 1 if you have the `setpwfile' function. */ +/* #undef HAVE_SETPWFILE */ + +/* Define to 1 if you have the `setsid' function. */ +#define HAVE_SETSID 1 + +/* Define to 1 if you have the `setuid' function. */ +#define HAVE_SETUID 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SGTTY_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SHADOW_H */ + +/* Define to 1 if you have the `sigaction' function. */ +#define HAVE_SIGACTION 1 + +/* Define to 1 if you have the `signal' function. */ +#define HAVE_SIGNAL 1 + +/* Define to 1 if you have the `sigset' function. */ +#define HAVE_SIGSET 1 + +/* define if you have -lslp */ +/* #undef HAVE_SLP */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SLP_H */ + +/* Define to 1 if you have the `snprintf' function. */ +#define HAVE_SNPRINTF 1 + +/* if you have spawnlp() */ +/* #undef HAVE_SPAWNLP */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SQLEXT_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SQL_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STDDEF_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDINT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDLIB_H 1 + +/* Define to 1 if you have the `strdup' function. */ +#define HAVE_STRDUP 1 + +/* Define to 1 if you have the `strerror' function. */ +#define HAVE_STRERROR 1 + +/* Define to 1 if you have the `strerror_r' function. */ +#define HAVE_STRERROR_R 1 + +/* Define to 1 if you have the `strftime' function. */ +#define HAVE_STRFTIME 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRINGS_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRING_H 1 + +/* Define to 1 if you have the `strpbrk' function. */ +#define HAVE_STRPBRK 1 + +/* Define to 1 if you have the `strrchr' function. */ +#define HAVE_STRRCHR 1 + +/* Define to 1 if you have the `strsep' function. */ +#define HAVE_STRSEP 1 + +/* Define to 1 if you have the `strspn' function. */ +#define HAVE_STRSPN 1 + +/* Define to 1 if you have the `strstr' function. */ +#define HAVE_STRSTR 1 + +/* Define to 1 if you have the `strtol' function. */ +#define HAVE_STRTOL 1 + +/* Define to 1 if you have the `strtoll' function. */ +#define HAVE_STRTOLL 1 + +/* Define to 1 if you have the `strtoq' function. */ +#define HAVE_STRTOQ 1 + +/* Define to 1 if you have the `strtoul' function. */ +#define HAVE_STRTOUL 1 + +/* Define to 1 if you have the `strtoull' function. */ +#define HAVE_STRTOULL 1 + +/* Define to 1 if you have the `strtouq' function. */ +#define HAVE_STRTOUQ 1 + +/* Define to 1 if `msg_accrightslen' is a member of `struct msghdr'. */ +/* #undef HAVE_STRUCT_MSGHDR_MSG_ACCRIGHTSLEN */ + +/* Define to 1 if `msg_control' is a member of `struct msghdr'. */ +#define HAVE_STRUCT_MSGHDR_MSG_CONTROL 1 + +/* Define to 1 if `pw_gecos' is a member of `struct passwd'. */ +#define HAVE_STRUCT_PASSWD_PW_GECOS 1 + +/* Define to 1 if `pw_passwd' is a member of `struct passwd'. */ +#define HAVE_STRUCT_PASSWD_PW_PASSWD 1 + +/* Define to 1 if `st_blksize' is a member of `struct stat'. */ +#define HAVE_STRUCT_STAT_ST_BLKSIZE 1 + +/* Define to 1 if `st_fstype' is a member of `struct stat'. */ +/* #undef HAVE_STRUCT_STAT_ST_FSTYPE */ + +/* define to 1 if st_fstype is char * */ +/* #undef HAVE_STRUCT_STAT_ST_FSTYPE_CHAR */ + +/* define to 1 if st_fstype is int */ +/* #undef HAVE_STRUCT_STAT_ST_FSTYPE_INT */ + +/* Define to 1 if `st_vfstype' is a member of `struct stat'. */ +/* #undef HAVE_STRUCT_STAT_ST_VFSTYPE */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYNCH_H */ + +/* Define to 1 if you have the `sysconf' function. */ +#define HAVE_SYSCONF 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYSEXITS_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYSLOG_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_DEVPOLL_H */ + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +/* #undef HAVE_SYS_DIR_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_EPOLL_H 1 + +/* define if you actually have sys_errlist in your libs */ +#define HAVE_SYS_ERRLIST 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_ERRNO_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_EVENT_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_FILE_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_FILIO_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_FSTYP_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_IOCTL_H 1 + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +/* #undef HAVE_SYS_NDIR_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_PARAM_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_POLL_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_PRIVGRP_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_RESOURCE_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SELECT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SOCKET_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_STAT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SYSLOG_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TIME_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_UCRED_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_UIO_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_UN_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_UUID_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_VMOUNT_H */ + +/* Define to 1 if you have that is POSIX.1 compatible. */ +#define HAVE_SYS_WAIT_H 1 + +/* define if you have -lwrap */ +/* #undef HAVE_TCPD */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_TCPD_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_TERMIOS_H 1 + +/* if you have Solaris LWP (thr) package */ +/* #undef HAVE_THR */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_THREAD_H */ + +/* Define to 1 if you have the `thr_getconcurrency' function. */ +/* #undef HAVE_THR_GETCONCURRENCY */ + +/* Define to 1 if you have the `thr_setconcurrency' function. */ +/* #undef HAVE_THR_SETCONCURRENCY */ + +/* Define to 1 if you have the `thr_yield' function. */ +/* #undef HAVE_THR_YIELD */ + +/* define if you have TLS */ +#define HAVE_TLS 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_UNISTD_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_UTIME_H 1 + +/* define if you have uuid_generate() */ +/* #undef HAVE_UUID_GENERATE */ + +/* define if you have uuid_to_str() */ +/* #undef HAVE_UUID_TO_STR */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_UUID_UUID_H */ + +/* Define to 1 if you have the `vprintf' function. */ +#define HAVE_VPRINTF 1 + +/* Define to 1 if you have the `vsnprintf' function. */ +#define HAVE_VSNPRINTF 1 + +/* Define to 1 if you have the `wait4' function. */ +#define HAVE_WAIT4 1 + +/* Define to 1 if you have the `waitpid' function. */ +#define HAVE_WAITPID 1 + +/* define if you have winsock */ +/* #undef HAVE_WINSOCK */ + +/* define if you have winsock2 */ +/* #undef HAVE_WINSOCK2 */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_WINSOCK2_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_WINSOCK_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_WIREDTIGER_H */ + +/* Define to 1 if you have the `write' function. */ +#define HAVE_WRITE 1 + +/* define if select implicitly yields */ +#define HAVE_YIELDING_SELECT 1 + +/* Define to 1 if you have the `_vsnprintf' function. */ +/* #undef HAVE__VSNPRINTF */ + +/* define to 32-bit or greater integer type */ +#define LBER_INT_T int + +/* define to large integer type */ +#define LBER_LEN_T long + +/* define to socket descriptor type */ +#define LBER_SOCKET_T int + +/* define to large integer type */ +#define LBER_TAG_T long + +/* define to 1 if library is thread safe */ +#define LDAP_API_FEATURE_X_OPENLDAP_THREAD_SAFE 1 + +/* define to LDAP VENDOR VERSION */ +/* #undef LDAP_API_FEATURE_X_OPENLDAP_V2_REFERRALS */ + +/* define this to add debugging code */ +/* #undef LDAP_DEBUG */ + +/* define if LDAP libs are dynamic */ +/* #undef LDAP_LIBS_DYNAMIC */ + +/* define to support PF_INET6 */ +#define LDAP_PF_INET6 1 + +/* define to support PF_LOCAL */ +#define LDAP_PF_LOCAL 1 + +/* define this to add SLAPI code */ +/* #undef LDAP_SLAPI */ + +/* define this to add syslog code */ +/* #undef LDAP_SYSLOG */ + +/* Version */ +#define LDAP_VENDOR_VERSION 20501 + +/* Major */ +#define LDAP_VENDOR_VERSION_MAJOR 2 + +/* Minor */ +#define LDAP_VENDOR_VERSION_MINOR 5 + +/* Patch */ +#define LDAP_VENDOR_VERSION_PATCH X + +/* Define to the sub-directory where libtool stores uninstalled libraries. */ +#define LT_OBJDIR ".libs/" + +/* define if memcmp is not 8-bit clean or is otherwise broken */ +/* #undef NEED_MEMCMP_REPLACEMENT */ + +/* define if you have (or want) no threads */ +/* #undef NO_THREADS */ + +/* define to use the original debug style */ +/* #undef OLD_DEBUG */ + +/* Package */ +#define OPENLDAP_PACKAGE "OpenLDAP" + +/* Version */ +#define OPENLDAP_VERSION "2.5.X" + +/* Define to the address where bug reports for this package should be sent. */ +#define PACKAGE_BUGREPORT "" + +/* Define to the full name of this package. */ +#define PACKAGE_NAME "" + +/* Define to the full name and version of this package. */ +#define PACKAGE_STRING "" + +/* Define to the one symbol short name of this package. */ +#define PACKAGE_TARNAME "" + +/* Define to the home page for this package. */ +#define PACKAGE_URL "" + +/* Define to the version of this package. */ +#define PACKAGE_VERSION "" + +/* define if sched_yield yields the entire process */ +/* #undef REPLACE_BROKEN_YIELD */ + +/* Define as the return type of signal handlers (`int' or `void'). */ +#define RETSIGTYPE void + +/* Define to the type of arg 1 for `select'. */ +#define SELECT_TYPE_ARG1 int + +/* Define to the type of args 2, 3 and 4 for `select'. */ +#define SELECT_TYPE_ARG234 (fd_set *) + +/* Define to the type of arg 5 for `select'. */ +#define SELECT_TYPE_ARG5 (struct timeval *) + +/* The size of `int', as computed by sizeof. */ +#define SIZEOF_INT 4 + +/* The size of `long', as computed by sizeof. */ +#define SIZEOF_LONG 8 + +/* The size of `long long', as computed by sizeof. */ +#define SIZEOF_LONG_LONG 8 + +/* The size of `short', as computed by sizeof. */ +#define SIZEOF_SHORT 2 + +/* The size of `wchar_t', as computed by sizeof. */ +#define SIZEOF_WCHAR_T 4 + +/* define to support per-object ACIs */ +/* #undef SLAPD_ACI_ENABLED */ + +/* define to support LDAP Async Metadirectory backend */ +/* #undef SLAPD_ASYNCMETA */ + +/* define to support cleartext passwords */ +/* #undef SLAPD_CLEARTEXT */ + +/* define to support crypt(3) passwords */ +/* #undef SLAPD_CRYPT */ + +/* define to support DNS SRV backend */ +/* #undef SLAPD_DNSSRV */ + +/* define to support LDAP backend */ +/* #undef SLAPD_LDAP */ + +/* define to support MDB backend */ +/* #undef SLAPD_MDB */ + +/* define to support LDAP Metadirectory backend */ +/* #undef SLAPD_META */ + +/* define to support modules */ +/* #undef SLAPD_MODULES */ + +/* dynamically linked module */ +#define SLAPD_MOD_DYNAMIC 2 + +/* statically linked module */ +#define SLAPD_MOD_STATIC 1 + +/* define to support cn=Monitor backend */ +/* #undef SLAPD_MONITOR */ + +/* define to support NDB backend */ +/* #undef SLAPD_NDB */ + +/* define to support NULL backend */ +/* #undef SLAPD_NULL */ + +/* define for In-Directory Access Logging overlay */ +/* #undef SLAPD_OVER_ACCESSLOG */ + +/* define for Audit Logging overlay */ +/* #undef SLAPD_OVER_AUDITLOG */ + +/* define for Automatic Certificate Authority overlay */ +/* #undef SLAPD_OVER_AUTOCA */ + +/* define for Collect overlay */ +/* #undef SLAPD_OVER_COLLECT */ + +/* define for Attribute Constraint overlay */ +/* #undef SLAPD_OVER_CONSTRAINT */ + +/* define for Dynamic Directory Services overlay */ +/* #undef SLAPD_OVER_DDS */ + +/* define for Dynamic Directory Services overlay */ +/* #undef SLAPD_OVER_DEREF */ + +/* define for Dynamic Group overlay */ +/* #undef SLAPD_OVER_DYNGROUP */ + +/* define for Dynamic List overlay */ +/* #undef SLAPD_OVER_DYNLIST */ + +/* define for Reverse Group Membership overlay */ +/* #undef SLAPD_OVER_MEMBEROF */ + +/* define for Password Policy overlay */ +/* #undef SLAPD_OVER_PPOLICY */ + +/* define for Proxy Cache overlay */ +/* #undef SLAPD_OVER_PROXYCACHE */ + +/* define for Referential Integrity overlay */ +/* #undef SLAPD_OVER_REFINT */ + +/* define for Return Code overlay */ +/* #undef SLAPD_OVER_RETCODE */ + +/* define for Rewrite/Remap overlay */ +/* #undef SLAPD_OVER_RWM */ + +/* define for Sequential Modify overlay */ +/* #undef SLAPD_OVER_SEQMOD */ + +/* define for ServerSideSort/VLV overlay */ +/* #undef SLAPD_OVER_SSSVLV */ + +/* define for Syncrepl Provider overlay */ +/* #undef SLAPD_OVER_SYNCPROV */ + +/* define for Translucent Proxy overlay */ +/* #undef SLAPD_OVER_TRANSLUCENT */ + +/* define for Attribute Uniqueness overlay */ +/* #undef SLAPD_OVER_UNIQUE */ + +/* define for Value Sorting overlay */ +/* #undef SLAPD_OVER_VALSORT */ + +/* define to support PASSWD backend */ +/* #undef SLAPD_PASSWD */ + +/* define to support PERL backend */ +/* #undef SLAPD_PERL */ + +/* define to support relay backend */ +/* #undef SLAPD_RELAY */ + +/* define to support reverse lookups */ +/* #undef SLAPD_RLOOKUPS */ + +/* define to support SHELL backend */ +/* #undef SLAPD_SHELL */ + +/* define to support SOCK backend */ +/* #undef SLAPD_SOCK */ + +/* define to support SASL passwords */ +/* #undef SLAPD_SPASSWD */ + +/* define to support SQL backend */ +/* #undef SLAPD_SQL */ + +/* define to support WiredTiger backend */ +/* #undef SLAPD_WT */ + +/* define to support run-time loadable ACL */ +/* #undef SLAP_DYNACL */ + +/* Define to 1 if you have the ANSI C header files. */ +#define STDC_HEADERS 1 + +/* Define to 1 if you can safely include both and . */ +#define TIME_WITH_SYS_TIME 1 + +/* Define to 1 if your declares `struct tm'. */ +/* #undef TM_IN_SYS_TIME */ + +/* set to urandom device */ +#define URANDOM_DEVICE "/dev/urandom" + +/* define to use OpenSSL BIGNUM for MP */ +/* #undef USE_MP_BIGNUM */ + +/* define to use GMP for MP */ +/* #undef USE_MP_GMP */ + +/* define to use 'long' for MP */ +/* #undef USE_MP_LONG */ + +/* define to use 'long long' for MP */ +/* #undef USE_MP_LONG_LONG */ + +/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most + significant byte first (like Motorola and SPARC, unlike Intel). */ +#if defined AC_APPLE_UNIVERSAL_BUILD +# if defined __BIG_ENDIAN__ +# define WORDS_BIGENDIAN 1 +# endif +#else +# ifndef WORDS_BIGENDIAN +/* # undef WORDS_BIGENDIAN */ +# endif +#endif + +/* Define to the type of arg 3 for `accept'. */ +#define ber_socklen_t socklen_t + +/* Define to `char *' if does not define. */ +/* #undef caddr_t */ + +/* Define to empty if `const' does not conform to ANSI C. */ +/* #undef const */ + +/* Define to `int' if doesn't define. */ +/* #undef gid_t */ + +/* Define to `int' if does not define. */ +/* #undef mode_t */ + +/* Define to `long' if does not define. */ +/* #undef off_t */ + +/* Define to `int' if does not define. */ +/* #undef pid_t */ + +/* Define to `int' if does not define. */ +/* #undef sig_atomic_t */ + +/* Define to `unsigned' if does not define. */ +/* #undef size_t */ + +/* define to snprintf routine */ +/* #undef snprintf */ + +/* Define like ber_socklen_t if does not define. */ +/* #undef socklen_t */ + +/* Define to `signed int' if does not define. */ +/* #undef ssize_t */ + +/* Define to `int' if doesn't define. */ +/* #undef uid_t */ + +/* define as empty if volatile is not supported */ +/* #undef volatile */ + +/* define to snprintf routine */ +/* #undef vsnprintf */ + + +/* begin of portable.h.post */ + +#ifdef _WIN32 +/* don't suck in all of the win32 api */ +# define WIN32_LEAN_AND_MEAN 1 +#endif + +#ifndef LDAP_NEEDS_PROTOTYPES +/* force LDAP_P to always include prototypes */ +#define LDAP_NEEDS_PROTOTYPES 1 +#endif + +#ifndef LDAP_REL_ENG +#if (LDAP_VENDOR_VERSION == 000000) && !defined(LDAP_DEVEL) +#define LDAP_DEVEL +#endif +#if defined(LDAP_DEVEL) && !defined(LDAP_TEST) +#define LDAP_TEST +#endif +#endif + +#ifdef HAVE_STDDEF_H +# include +#endif + +#ifdef HAVE_EBCDIC +/* ASCII/EBCDIC converting replacements for stdio funcs + * vsnprintf and snprintf are used too, but they are already + * checked by the configure script + */ +#define fputs ber_pvt_fputs +#define fgets ber_pvt_fgets +#define printf ber_pvt_printf +#define fprintf ber_pvt_fprintf +#define vfprintf ber_pvt_vfprintf +#define vsprintf ber_pvt_vsprintf +#endif + +#include "ac/fdset.h" + +#include "ldap_cdefs.h" +#include "ldap_features.h" + +#include "ac/assert.h" +#include "ac/localize.h" + +#endif /* _LDAP_PORTABLE_H */ +/* end of portable.h.post */ + From aa17da6b6811593b6fb47f611af4ee665e819a53 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Fri, 15 May 2020 14:25:18 +0300 Subject: [PATCH 256/738] [docs] adjust SELECT ToC meta (#10942) * [docs] adjust SELECT ToC meta * Update array-join.md * Update distinct.md * Update format.md * Update from.md * Update group-by.md * Update having.md * Update into-outfile.md * Update join.md * Update limit-by.md * Update limit.md * Update order-by.md * Update prewhere.md * Update sample.md * Update union-all.md * Update where.md * Update with.md --- docs/en/sql-reference/statements/select/array-join.md | 4 ++++ docs/en/sql-reference/statements/select/distinct.md | 4 ++++ docs/en/sql-reference/statements/select/format.md | 4 ++++ docs/en/sql-reference/statements/select/from.md | 4 ++++ docs/en/sql-reference/statements/select/group-by.md | 4 ++++ docs/en/sql-reference/statements/select/having.md | 4 ++++ docs/en/sql-reference/statements/select/index.md | 3 ++- docs/en/sql-reference/statements/select/into-outfile.md | 4 ++++ docs/en/sql-reference/statements/select/join.md | 4 ++++ docs/en/sql-reference/statements/select/limit-by.md | 4 ++++ docs/en/sql-reference/statements/select/limit.md | 4 ++++ docs/en/sql-reference/statements/select/order-by.md | 4 ++++ docs/en/sql-reference/statements/select/prewhere.md | 4 ++++ docs/en/sql-reference/statements/select/sample.md | 4 ++++ docs/en/sql-reference/statements/select/union-all.md | 4 ++++ docs/en/sql-reference/statements/select/where.md | 4 ++++ docs/en/sql-reference/statements/select/with.md | 4 ++++ 17 files changed, 66 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/select/array-join.md b/docs/en/sql-reference/statements/select/array-join.md index 8e05cf51232..21f6ea40492 100644 --- a/docs/en/sql-reference/statements/select/array-join.md +++ b/docs/en/sql-reference/statements/select/array-join.md @@ -1,3 +1,7 @@ +--- +toc_title: ARRAY JOIN +--- + # ARRAY JOIN Clause {#select-array-join-clause} It is a common operation for tables that contain an array column to produce a new table that has a column with each individual array element of that initial column, while values of other columns are duplicated. This is the basic case of what `ARRAY JOIN` clause does. diff --git a/docs/en/sql-reference/statements/select/distinct.md b/docs/en/sql-reference/statements/select/distinct.md index 1f500bf469b..d7799a8343e 100644 --- a/docs/en/sql-reference/statements/select/distinct.md +++ b/docs/en/sql-reference/statements/select/distinct.md @@ -1,3 +1,7 @@ +--- +toc_title: DISTINCT +--- + # DISTINCT Clause {#select-distinct} If `SELECT DISTINCT` is specified, only unique rows will remain in a query result. Thus only a single row will remain out of all the sets of fully matching rows in the result. diff --git a/docs/en/sql-reference/statements/select/format.md b/docs/en/sql-reference/statements/select/format.md index 75fd7256bfc..ca4b89fae71 100644 --- a/docs/en/sql-reference/statements/select/format.md +++ b/docs/en/sql-reference/statements/select/format.md @@ -1,3 +1,7 @@ +--- +toc_title: FORMAT +--- + # FORMAT Clause {#format-clause} ClickHouse supports a wide range of [serialization formats](../../../interfaces/formats.md) that can be used on query results among other things. There are multiple ways to choose a format for `SELECT` output, one of them is to specify `FORMAT format` at the end of query to get resulting data in any specific format. diff --git a/docs/en/sql-reference/statements/select/from.md b/docs/en/sql-reference/statements/select/from.md index d017537557d..fa29576276c 100644 --- a/docs/en/sql-reference/statements/select/from.md +++ b/docs/en/sql-reference/statements/select/from.md @@ -1,3 +1,7 @@ +--- +toc_title: FROM +--- + # FROM Clause {#select-from} The `FROM` clause specifies the source to read data from: diff --git a/docs/en/sql-reference/statements/select/group-by.md b/docs/en/sql-reference/statements/select/group-by.md index c9cb275129e..49d3a2da8a4 100644 --- a/docs/en/sql-reference/statements/select/group-by.md +++ b/docs/en/sql-reference/statements/select/group-by.md @@ -1,3 +1,7 @@ +--- +toc_title: GROUP BY +--- + # GROUP BY Clause {#select-group-by-clause} `GROUP BY` clause switches the `SELECT` query into an aggregation mode, which works as follows: diff --git a/docs/en/sql-reference/statements/select/having.md b/docs/en/sql-reference/statements/select/having.md index 17413645449..7a42f43bf53 100644 --- a/docs/en/sql-reference/statements/select/having.md +++ b/docs/en/sql-reference/statements/select/having.md @@ -1,3 +1,7 @@ +--- +toc_title: HAVING +--- + # HAVING Clause {#having-clause} Allows filtering the aggregation results produced by [GROUP BY](group-by.md). It is similar to the [WHERE](where.md) clause, but the difference is that `WHERE` is performed before aggregation, while `HAVING` is performed after it. diff --git a/docs/en/sql-reference/statements/select/index.md b/docs/en/sql-reference/statements/select/index.md index 4ee75f7b575..8224bf1e798 100644 --- a/docs/en/sql-reference/statements/select/index.md +++ b/docs/en/sql-reference/statements/select/index.md @@ -1,6 +1,7 @@ --- toc_priority: 33 -toc_title: SELECT +toc_folder_title: SELECT +toc_title: Queries Syntax --- # SELECT Queries Syntax {#select-queries-syntax} diff --git a/docs/en/sql-reference/statements/select/into-outfile.md b/docs/en/sql-reference/statements/select/into-outfile.md index 4385d31438a..26b9c2cf8cb 100644 --- a/docs/en/sql-reference/statements/select/into-outfile.md +++ b/docs/en/sql-reference/statements/select/into-outfile.md @@ -1,3 +1,7 @@ +--- +toc_title: INTO OUTFILE +--- + # INTO OUTFILE Clause {#into-outfile-clause} Add the `INTO OUTFILE filename` clause (where filename is a string literal) to `SELECT query` to redirect its output to the specified file on the client-side. diff --git a/docs/en/sql-reference/statements/select/join.md b/docs/en/sql-reference/statements/select/join.md index bc160d1fc8d..c636e1dab8b 100644 --- a/docs/en/sql-reference/statements/select/join.md +++ b/docs/en/sql-reference/statements/select/join.md @@ -1,3 +1,7 @@ +--- +toc_title: JOIN +--- + # JOIN Clause {#select-join} Join produces a new table by combining columns from one or multiple tables by using values common to each. It is a common operation in databases with SQL support, which corresponds to [relational algebra](https://en.wikipedia.org/wiki/Relational_algebra#Joins_and_join-like_operators) join. The special case of one table join is often referred to as "self-join". diff --git a/docs/en/sql-reference/statements/select/limit-by.md b/docs/en/sql-reference/statements/select/limit-by.md index b6af13fb8d2..05b9e7b9151 100644 --- a/docs/en/sql-reference/statements/select/limit-by.md +++ b/docs/en/sql-reference/statements/select/limit-by.md @@ -1,3 +1,7 @@ +--- +toc_title: LIMIT BY +--- + # LIMIT BY Clause {#limit-by-clause} A query with the `LIMIT n BY expressions` clause selects the first `n` rows for each distinct value of `expressions`. The key for `LIMIT BY` can contain any number of [expressions](../../syntax.md#syntax-expressions). diff --git a/docs/en/sql-reference/statements/select/limit.md b/docs/en/sql-reference/statements/select/limit.md index 8f47904cecb..3f43e1b155e 100644 --- a/docs/en/sql-reference/statements/select/limit.md +++ b/docs/en/sql-reference/statements/select/limit.md @@ -1,3 +1,7 @@ +--- +toc_title: LIMIT +--- + # LIMIT Clause {#limit-clause} `LIMIT m` allows to select the first `m` rows from the result. diff --git a/docs/en/sql-reference/statements/select/order-by.md b/docs/en/sql-reference/statements/select/order-by.md index 39bd19f8eab..10c1f234057 100644 --- a/docs/en/sql-reference/statements/select/order-by.md +++ b/docs/en/sql-reference/statements/select/order-by.md @@ -1,3 +1,7 @@ +--- +toc_title: ORDER BY +--- + # ORDER BY Clause {#select-order-by} The `ORDER BY` clause contains a list of expressions, which can each be attributed with `DESC` (descending) or `ASC` (ascending) modifier which determine the sorting direction. If the direction is not specified, `ASC` is assumed, so it's usually omitted. The sorting direction applies to a single expression, not to the entire list. Example: `ORDER BY Visits DESC, SearchPhrase` diff --git a/docs/en/sql-reference/statements/select/prewhere.md b/docs/en/sql-reference/statements/select/prewhere.md index 765514a6835..38ff11dc548 100644 --- a/docs/en/sql-reference/statements/select/prewhere.md +++ b/docs/en/sql-reference/statements/select/prewhere.md @@ -1,3 +1,7 @@ +--- +toc_title: PREWHERE +--- + # PREWHERE Clause {#prewhere-clause} Prewhere is an optimization to apply filtering more efficiently. It is enabled by default even if `PREWHERE` clause is not specified explicitly. It works by automatically moving part of [WHERE](where.md) condition to prewhere stage. The role of `PREWHERE` clause is only to control this optimization if you think that you know how to do it better than it happens by default. diff --git a/docs/en/sql-reference/statements/select/sample.md b/docs/en/sql-reference/statements/select/sample.md index 955711e74c0..00431d8e44a 100644 --- a/docs/en/sql-reference/statements/select/sample.md +++ b/docs/en/sql-reference/statements/select/sample.md @@ -1,3 +1,7 @@ +--- +toc_title: SAMPLE +--- + # SAMPLE Clause {#select-sample-clause} The `SAMPLE` clause allows for approximated `SELECT` query processing. diff --git a/docs/en/sql-reference/statements/select/union-all.md b/docs/en/sql-reference/statements/select/union-all.md index f89d15e0539..9c5be93a345 100644 --- a/docs/en/sql-reference/statements/select/union-all.md +++ b/docs/en/sql-reference/statements/select/union-all.md @@ -1,3 +1,7 @@ +--- +toc_title: UNION ALL +--- + # UNION ALL Clause {#union-all-clause} You can use `UNION ALL` to combine any number of `SELECT` queries by extending their results. Example: diff --git a/docs/en/sql-reference/statements/select/where.md b/docs/en/sql-reference/statements/select/where.md index c5a6c6add0f..84618be5002 100644 --- a/docs/en/sql-reference/statements/select/where.md +++ b/docs/en/sql-reference/statements/select/where.md @@ -1,3 +1,7 @@ +--- +toc_title: WHERE +--- + # WHERE Clause {#select-where} `WHERE` clause allows to filter the data that is coming from [FROM](from.md) clause of `SELECT`. diff --git a/docs/en/sql-reference/statements/select/with.md b/docs/en/sql-reference/statements/select/with.md index d6530964e64..ac04ce69ae3 100644 --- a/docs/en/sql-reference/statements/select/with.md +++ b/docs/en/sql-reference/statements/select/with.md @@ -1,3 +1,7 @@ +--- +toc_title: WITH +--- + # WITH Clause {#with-clause} This section provides support for Common Table Expressions ([CTE](https://en.wikipedia.org/wiki/Hierarchical_and_recursive_queries_in_SQL)), so the results of `WITH` clause can be used in the rest of `SELECT` query. From 15a827433afeda37116e8031f33020b01633322e Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 15 May 2020 14:26:51 +0300 Subject: [PATCH 257/738] fix build --- programs/odbc-bridge/MainHandler.cpp | 7 ++++++- programs/odbc-bridge/ODBCBlockOutputStream.cpp | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/programs/odbc-bridge/MainHandler.cpp b/programs/odbc-bridge/MainHandler.cpp index 3c16937e7b0..edc92552c44 100644 --- a/programs/odbc-bridge/MainHandler.cpp +++ b/programs/odbc-bridge/MainHandler.cpp @@ -20,8 +20,10 @@ #include #include "getIdentifierQuote.h" +#if USE_ODBC #include #define POCO_SQL_ODBC_CLASS Poco::Data::ODBC +#endif namespace DB { @@ -152,8 +154,11 @@ void ODBCHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne std::string table_name = params.get("table_name"); LOG_TRACE(log, "DB name: '" << db_name << "', table name: '" << table_name << "'"); + auto quoting_style = IdentifierQuotingStyle::None; +#if USE_ODBC POCO_SQL_ODBC_CLASS::SessionImpl session(validateODBCConnectionString(connection_string), DBMS_DEFAULT_CONNECT_TIMEOUT_SEC); - auto quoting_style = getQuotingStyle(session.dbc().handle()); + quoting_style = getQuotingStyle(session.dbc().handle()); +#endif auto pool = getPool(connection_string); ReadBufferFromIStream read_buf(request.stream()); diff --git a/programs/odbc-bridge/ODBCBlockOutputStream.cpp b/programs/odbc-bridge/ODBCBlockOutputStream.cpp index 67990bf1ac0..2d7c093ae67 100644 --- a/programs/odbc-bridge/ODBCBlockOutputStream.cpp +++ b/programs/odbc-bridge/ODBCBlockOutputStream.cpp @@ -80,6 +80,7 @@ namespace case ValueType::vtUUID: return Poco::Dynamic::Var(UUID(field.get()).toUnderType().toHexString()).convert(); } + __builtin_unreachable(); } } From 8ddb5d5c6d33ab4ea1d0bb67318c3a1392d31303 Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Fri, 15 May 2020 16:11:04 +0300 Subject: [PATCH 258/738] [WIP] --- CHANGELOG.md | 311 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 309 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 885b4fc656f..5b5d23f9bb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,315 @@ +## ClickHouse release v20.4 + +### ClickHouse release v20.4.2.9, 2020-05-12 + +#### Backward Incompatible Change +* When string comparison involves FixedString and compared arguments are of different sizes, do comparison as if smaller string is padded to the length of the larger. This is intented for SQL compatibility if we imagine that FixedString data type corresponds to SQL CHAR. This closes [#9272](https://github.com/ClickHouse/ClickHouse/issues/9272). [#10363](https://github.com/ClickHouse/ClickHouse/pull/10363) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Make SHOW CREATE TABLE multiline. Now it is more readable and more like MySQL. [#10049](https://github.com/ClickHouse/ClickHouse/pull/10049) ([Azat Khuzhin](https://github.com/azat)) +* Added a setting `validate_polygons` that is used in `pointInPolygon` function and enabled by default. [#9857](https://github.com/ClickHouse/ClickHouse/pull/9857) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +#### New Feature +* Support custom HTTP handlers. See ISSUES-5436 for description. [#7572](https://github.com/ClickHouse/ClickHouse/pull/7572) ([Winter Zhang](https://github.com/zhang2014)) +* Added custom settings support in DDL-queries for CREATE DICTIONARY [#10465](https://github.com/ClickHouse/ClickHouse/pull/10465) ([Artem Streltsov](https://github.com/kekekekule)) +* Added experimental database engine Atomic. It supports non-blocking `DROP` and `RENAME TABLE` queries and atomic `EXCHANGE TABLES t1 AND t2` query [#7512](https://github.com/ClickHouse/ClickHouse/pull/7512) ([tavplubix](https://github.com/tavplubix)) +* Add simple server-wide memory profiler that will collect allocation contexts when server memory usage becomes higher than the next allocation threshold. [#10444](https://github.com/ClickHouse/ClickHouse/pull/10444) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Added support for custom settings section in dictionaries. Also fixes issue [#2829](https://github.com/ClickHouse/ClickHouse/issues/2829). [#10137](https://github.com/ClickHouse/ClickHouse/pull/10137) ([Artem Streltsov](https://github.com/kekekekule)) +* Add setting `always_fetch_merged_part` which restrict replica to merge parts by itself and always prefer dowloading from other replicas. [#10379](https://github.com/ClickHouse/ClickHouse/pull/10379) ([alesapin](https://github.com/alesapin)) +* Add function JSONExtractKeysAndValuesRaw which extracts raw data from JSON objects [#10378](https://github.com/ClickHouse/ClickHouse/pull/10378) ([hcz](https://github.com/hczhcz)) +* Add CRC32IEEE()/CRC64() support [#7480](https://github.com/ClickHouse/ClickHouse/pull/7480) ([Azat Khuzhin](https://github.com/azat)) +* Add memory usage from OS to `system.asynchronous_metrics`. [#10361](https://github.com/ClickHouse/ClickHouse/pull/10361) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Added generic variants for functions `least` and `greatest`. Now they work with arbitrary number of arguments of arbitrary types. This fixes [#4767](https://github.com/ClickHouse/ClickHouse/issues/4767) [#10318](https://github.com/ClickHouse/ClickHouse/pull/10318) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Now ClickHouse controls timeouts of dictionary sources on its side. Two new settings added to cache dictionary configuration: `strict_max_lifetime_seconds`, which is `max_lifetime` by default, and `query_wait_timeout_milliseconds`, which is one minute by default. The first settings is also useful with `allow_read_expired_keys` settings (to forbid reading very expired keys). [#10337](https://github.com/ClickHouse/ClickHouse/pull/10337) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +* Added output format `Markdown` for embedding tables in markdown documents. [#10317](https://github.com/ClickHouse/ClickHouse/pull/10317) ([Kruglov Pavel](https://github.com/Avogar)) +* Add support for secured connection from ClickHouse to Zookeeper [#10184](https://github.com/ClickHouse/ClickHouse/pull/10184) ([Konstantin Lebedev](https://github.com/xzkostyan)) +* Add log_queries_min_type to filter which entries will be written to query_log [#10053](https://github.com/ClickHouse/ClickHouse/pull/10053) ([Azat Khuzhin](https://github.com/azat)) +* Added function `isConstant`. This function checks whether its argument is constant expression and returns 1 or 0. It is intended for development, debugging and demonstration purposes. [#10198](https://github.com/ClickHouse/ClickHouse/pull/10198) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* add joinGetOrNull to return NULL when key is missing instead of returning the default value. [#10094](https://github.com/ClickHouse/ClickHouse/pull/10094) ([Amos Bird](https://github.com/amosbird)) +* Consider `NULL` to be equal to `NULL` in `IN` operator, if the option `transform_null_in` is set. [#10085](https://github.com/ClickHouse/ClickHouse/pull/10085) ([achimbab](https://github.com/achimbab)) +* Add MessagePack Input/Output format. [#9889](https://github.com/ClickHouse/ClickHouse/pull/9889) ([Kruglov Pavel](https://github.com/Avogar)) +* Add `ALTER TABLE ... RENAME COLUMN` for MergeTree table engines family. [#9948](https://github.com/ClickHouse/ClickHouse/pull/9948) ([alesapin](https://github.com/alesapin)) +* Implemented s3 table function and storage. [#5596](https://github.com/ClickHouse/ClickHouse/pull/5596) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Support parallel distributed INSERT SELECT. [#9759](https://github.com/ClickHouse/ClickHouse/pull/9759) ([vxider](https://github.com/Vxider)) +* Added implementation of LIVE VIEW tables that were originally proposed in [#3925](https://github.com/ClickHouse/ClickHouse/issues/3925), and then updated in [#5541](https://github.com/ClickHouse/ClickHouse/issues/5541). Note that currently only LIVE VIEW tables are supported. See [#5541](https://github.com/ClickHouse/ClickHouse/issues/5541) for detailed description. [#6425](https://github.com/ClickHouse/ClickHouse/pull/6425) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Add ability to query Distributed over Distributed (w/o `distributed_group_by_no_merge`) ... [#9923](https://github.com/ClickHouse/ClickHouse/pull/9923) ([Azat Khuzhin](https://github.com/azat)) +* Add function `arrayReduceInRanges` which aggregates array elements in given ranges. [#9598](https://github.com/ClickHouse/ClickHouse/pull/9598) ([hcz](https://github.com/hczhcz)) +* Add Regexp input format. [#9196](https://github.com/ClickHouse/ClickHouse/pull/9196) ([Kruglov Pavel](https://github.com/Avogar)) +* Add Dictionary Status on prometheus exporter. [#9622](https://github.com/ClickHouse/ClickHouse/pull/9622) ([Guillaume Tassery](https://github.com/YiuRULE)) +* Add array function auc [#8698](https://github.com/ClickHouse/ClickHouse/pull/8698) ([taiyang-li](https://github.com/taiyang-li)) +* Support `DROP VIEW` statement for better TPC-H compatibility. [#9831](https://github.com/ClickHouse/ClickHouse/pull/9831) ([Amos Bird](https://github.com/amosbird)) +* Add 'strict_order' option to windowFunnel() [#9773](https://github.com/ClickHouse/ClickHouse/pull/9773) ([achimbab](https://github.com/achimbab)) +* Add support for configuring MongoDB connections with URI (for Atlas service). [#5387](https://github.com/ClickHouse/ClickHouse/pull/5387) ([Manish Srivastava](https://github.com/manish53)) +* Support `DATE` and `TIMESTAMP` SQL operators, e.g. `SELECT date '2001-01-01'` [#9691](https://github.com/ClickHouse/ClickHouse/pull/9691) ([Artem Zuikov](https://github.com/4ertus2)) + +#### Bug Fix +* Fixed incorrect scalar results inside inner query of `MATERIALIZED VIEW` in case if this query contained dependent table [#10603](https://github.com/ClickHouse/ClickHouse/pull/10603) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fixed bug, which caused HTTP requests to get stuck on client closing connection when `readonly=2` and `cancel_http_readonly_queries_on_client_close=1`. [#10684](https://github.com/ClickHouse/ClickHouse/pull/10684) ([tavplubix](https://github.com/tavplubix)) +* Fix segfault in StorageBuffer when exception is thrown on server startup. Fixes [#10550](https://github.com/ClickHouse/ClickHouse/issues/10550) [#10609](https://github.com/ClickHouse/ClickHouse/pull/10609) ([tavplubix](https://github.com/tavplubix)) +* The query`SYSTEM DROP DNS CACHE` now also drops caches used to check if user is allowed to connect from some IP addresses [#10608](https://github.com/ClickHouse/ClickHouse/pull/10608) ([tavplubix](https://github.com/tavplubix)) +* Fix usage of multiple `IN` operators with an identical set in one query. Fixes [#10539](https://github.com/ClickHouse/ClickHouse/issues/10539) [#10686](https://github.com/ClickHouse/ClickHouse/pull/10686) ([Anton Popov](https://github.com/CurtizJ)) +* Fix crash in `generateRandom` with nested types. Fixes [#10583](https://github.com/ClickHouse/ClickHouse/issues/10583). [#10734](https://github.com/ClickHouse/ClickHouse/pull/10734) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fix data corruption for `LowCardinality(FixedString)` key column in `SummingMergeTree` which could have happened after merge. Fixes [#10489](https://github.com/ClickHouse/ClickHouse/issues/10489). [#10721](https://github.com/ClickHouse/ClickHouse/pull/10721) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fix logic for aggregation_memory_efficient_merge_threads setting. [#10667](https://github.com/ClickHouse/ClickHouse/pull/10667) ([palasonic1](https://github.com/palasonic1)) +* Fix disappearing totals. Totals could have being filtered if query had `JOIN` or subquery with external `WHERE` condition. Fixes [#10674](https://github.com/ClickHouse/ClickHouse/issues/10674) [#10698](https://github.com/ClickHouse/ClickHouse/pull/10698) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fix the lack of parallel execution of remote queries with `distributed_aggregation_memory_efficient` enabled. Fixes [#10655](https://github.com/ClickHouse/ClickHouse/issues/10655) [#10664](https://github.com/ClickHouse/ClickHouse/pull/10664) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fix possible incorrect number of rows for queries with `LIMIT`. Fixes [#10566](https://github.com/ClickHouse/ClickHouse/issues/10566), [#10709](https://github.com/ClickHouse/ClickHouse/issues/10709) [#10660](https://github.com/ClickHouse/ClickHouse/pull/10660) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fix index corruption, which may occur in some cases after merging compact parts into another compact part. [#10531](https://github.com/ClickHouse/ClickHouse/pull/10531) ([Anton Popov](https://github.com/CurtizJ)) +* Fix the situation, when mutation finished all parts, but hung up in `is_done=0`. [#10526](https://github.com/ClickHouse/ClickHouse/pull/10526) ([alesapin](https://github.com/alesapin)) +* Fix overflow at beginning of unix epoch for timezones with fractional offset from UTC. Fixes [#9335](https://github.com/ClickHouse/ClickHouse/issues/9335). [#10513](https://github.com/ClickHouse/ClickHouse/pull/10513) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Better diagnostics for input formats. Fixes [#10204](https://github.com/ClickHouse/ClickHouse/issues/10204) [#10418](https://github.com/ClickHouse/ClickHouse/pull/10418) ([tavplubix](https://github.com/tavplubix)) +* Fix numeric overflow in `simpleLinearRegression()` over large integers [#10474](https://github.com/ClickHouse/ClickHouse/pull/10474) ([hcz](https://github.com/hczhcz)) +* Fix use-after-free in Distributed shutdown, avoid waiting for sending all batches [#10491](https://github.com/ClickHouse/ClickHouse/pull/10491) ([Azat Khuzhin](https://github.com/azat)) +* Add CA certificates to clickhouse-server docker image [#10476](https://github.com/ClickHouse/ClickHouse/pull/10476) ([filimonov](https://github.com/filimonov)) +* Fix a rare endless loop that might have occurred when using the `addressToLine` function or AggregateFunctionState columns. [#10466](https://github.com/ClickHouse/ClickHouse/pull/10466) ([Alexander Kuzmenkov](https://github.com/akuzm)) +* Handle zookeeper "no node error" during distributed query [#10050](https://github.com/ClickHouse/ClickHouse/pull/10050) ([Daniel Chen](https://github.com/Phantomape)) +* Fix bug when server cannot attach table after column's default was altered. [#10441](https://github.com/ClickHouse/ClickHouse/pull/10441) ([alesapin](https://github.com/alesapin)) +* Implicitly cast the default expression type to the column type for the ALIAS columns [#10563](https://github.com/ClickHouse/ClickHouse/pull/10563) ([Azat Khuzhin](https://github.com/azat)) +* Don't remove metadata directory if `ATTACH DATABASE` fails [#10442](https://github.com/ClickHouse/ClickHouse/pull/10442) ([Winter Zhang](https://github.com/zhang2014)) +* Avoid dependency on system tzdata. Fixes loading of `Africa/Casablanca` timezone on CentOS 8. Fixes [#10211](https://github.com/ClickHouse/ClickHouse/issues/10211) [#10425](https://github.com/ClickHouse/ClickHouse/pull/10425) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix some issues if data is inserted with quorum and then gets deleted (DROP PARTITION, TTL, etc.). It led to stuck of INSERTs or false-positive exceptions in SELECTs. Fixes [#9946](https://github.com/ClickHouse/ClickHouse/issues/9946) [#10188](https://github.com/ClickHouse/ClickHouse/pull/10188) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +* Check the number and type of arguments when creating BloomFilter index [#9623](https://github.com/ClickHouse/ClickHouse/issues/9623) [#10431](https://github.com/ClickHouse/ClickHouse/pull/10431) ([Winter Zhang](https://github.com/zhang2014)) +* Prefer `fallback_to_stale_replicas` over `skip_unavailable_shards`, otherwise when both settings specified and there are no up-to-date replicas the query will fail (patch from @alex-zaitsev ) [#10422](https://github.com/ClickHouse/ClickHouse/pull/10422) ([Azat Khuzhin](https://github.com/azat)) +* Fix the issue when a query with ARRAY JOIN, ORDER BY and LIMIT may return incomplete result. Fixes [#10226](https://github.com/ClickHouse/ClickHouse/issues/10226). [#10427](https://github.com/ClickHouse/ClickHouse/pull/10427) ([Vadim Plakhtinskiy](https://github.com/VadimPlh)) +* Add database name to dictionary name after DETACH/ATTACH. Fixes system.dictionaries table and `SYSTEM RELOAD` query [#10415](https://github.com/ClickHouse/ClickHouse/pull/10415) ([Azat Khuzhin](https://github.com/azat)) +* Fix possible incorrect result for extremes in processors pipeline. [#10131](https://github.com/ClickHouse/ClickHouse/pull/10131) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fix possible segfault when the setting `distributed_group_by_no_merge` is enabled (introduced in 20.3.7.46 by [#10131](https://github.com/ClickHouse/ClickHouse/issues/10131)). [#10399](https://github.com/ClickHouse/ClickHouse/pull/10399) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fix wrong flattening of `Array(Tuple(...))` data types. Fixes [#10259](https://github.com/ClickHouse/ClickHouse/issues/10259) [#10390](https://github.com/ClickHouse/ClickHouse/pull/10390) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix column names of constants inside JOIN that may clash with names of constants outside of JOIN [#9950](https://github.com/ClickHouse/ClickHouse/pull/9950) ([Alexander Kuzmenkov](https://github.com/akuzm)) +* Fix order of columns after Block::sortColumns() [#10826](https://github.com/ClickHouse/ClickHouse/pull/10826) ([Azat Khuzhin](https://github.com/azat)) +* Fix possible `Pipeline stuck` error in `ConcatProcessor` which may happen in remote query. [#10381](https://github.com/ClickHouse/ClickHouse/pull/10381) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Don't make disk reservations for aggregations. Fixes [#9241](https://github.com/ClickHouse/ClickHouse/issues/9241) [#10375](https://github.com/ClickHouse/ClickHouse/pull/10375) ([Azat Khuzhin](https://github.com/azat)) +* Fix wrong behaviour of datetime functions for timezones that has altered between positive and negative offsets from UTC (e.g. Pacific/Kiritimati). Fixes [#7202](https://github.com/ClickHouse/ClickHouse/issues/7202) [#10369](https://github.com/ClickHouse/ClickHouse/pull/10369) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Avoid infinite loop in `dictIsIn` function. Fixes #515 [#10365](https://github.com/ClickHouse/ClickHouse/pull/10365) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Disable GROUP BY sharding_key optimization by default and fix it for WITH ROLLUP/CUBE/TOTALS [#10516](https://github.com/ClickHouse/ClickHouse/pull/10516) ([Azat Khuzhin](https://github.com/azat)) +* Check for error code when checking parts and don't mark part as broken if the error is like "not enough memory". Fixes [#6269](https://github.com/ClickHouse/ClickHouse/issues/6269) [#10364](https://github.com/ClickHouse/ClickHouse/pull/10364) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Show information about not loaded dictionaries in system tables. [#10234](https://github.com/ClickHouse/ClickHouse/pull/10234) ([Vitaly Baranov](https://github.com/vitlibar)) +* Fix nullptr dereference in StorageBuffer if server was shutdown before table startup. [#10641](https://github.com/ClickHouse/ClickHouse/pull/10641) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fixed `DROP` vs `OPTIMIZE` race in `ReplicatedMergeTree`. `DROP` could left some garbage in replica path in ZooKeeper if there was concurrent `OPTIMIZE` query. [#10312](https://github.com/ClickHouse/ClickHouse/pull/10312) ([tavplubix](https://github.com/tavplubix)) +* Fix 'Logical error: CROSS JOIN has expressions' error for queries with comma and names joins mix. Fixes [#9910](https://github.com/ClickHouse/ClickHouse/issues/9910) [#10311](https://github.com/ClickHouse/ClickHouse/pull/10311) ([Artem Zuikov](https://github.com/4ertus2)) + +* Fix queries with `max_bytes_before_external_group_by`. [#10302](https://github.com/ClickHouse/ClickHouse/pull/10302) ([Artem Zuikov](https://github.com/4ertus2)) +* Fix the issue with limiting maximum recursion depth in parser in certain cases. This fixes [#10283](https://github.com/ClickHouse/ClickHouse/issues/10283) This fix may introduce minor incompatibility: long and deep queries via clickhouse-client may refuse to work, and you should adjust settings `max_query_size` and `max_parser_depth` accordingly. [#10295](https://github.com/ClickHouse/ClickHouse/pull/10295) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Allow to use `count(*)` with multiple JOINs. Fixes [#9853](https://github.com/ClickHouse/ClickHouse/issues/9853) [#10291](https://github.com/ClickHouse/ClickHouse/pull/10291) ([Artem Zuikov](https://github.com/4ertus2)) +* Remove converting ColumnFixedString to FixedString in MsgPack fromat and add performance test. ... [#10216](https://github.com/ClickHouse/ClickHouse/pull/10216) ([Kruglov Pavel](https://github.com/Avogar)) +* Fix error `Pipeline stuck` with `max_rows_to_group_by` and `group_by_overflow_mode = \'break\'`. [#10279](https://github.com/ClickHouse/ClickHouse/pull/10279) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fix `\'Cannot add column\'` error while creating `range_hashed` dictionary using DDL query. Fixes [#10093](https://github.com/ClickHouse/ClickHouse/issues/10093). [#10235](https://github.com/ClickHouse/ClickHouse/pull/10235) ([alesapin](https://github.com/alesapin)) +* Fix rare possible exception `Cannot drain connections: cancel first`. ... [#10239](https://github.com/ClickHouse/ClickHouse/pull/10239) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fixed bug where ClickHouse would throw "Unknown function lambda." error message when user tries to run ALTER UPDATE/DELETE on tables with ENGINE = Replicated*. Check for nondeterministic functions now handles lambda expressions correctly. ... [#10237](https://github.com/ClickHouse/ClickHouse/pull/10237) ([Alexander Kazakov](https://github.com/Akazz)) +* Fixed reasonably rare segfault in StorageSystemTables that happens when SELECT ... FROM system.tables is run on a database with Lazy engine. [#10209](https://github.com/ClickHouse/ClickHouse/pull/10209) ([Alexander Kazakov](https://github.com/Akazz)) +* Fix possible inifinite query execution when the query actually should stop on LIMIT, while reading from infinite source like `system.numbers` or `system.zeros`. ... [#10206](https://github.com/ClickHouse/ClickHouse/pull/10206) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fixed "generateRandom" function for Date type. This fixes [#9973](https://github.com/ClickHouse/ClickHouse/issues/9973). Fix an edge case when dates with year 2106 are inserted to MergeTree tables with old-style partitioning but partitions are named with year 1970. [#10218](https://github.com/ClickHouse/ClickHouse/pull/10218) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Convert types if the table definition of a View does not correspond to the SELECT query. This fixes [#10180](https://github.com/ClickHouse/ClickHouse/issues/10180) and [#10022](https://github.com/ClickHouse/ClickHouse/issues/10022) [#10217](https://github.com/ClickHouse/ClickHouse/pull/10217) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix `parseDateTimeBestEffort` for strings in RFC-2822 when day of week is Tuesday or Thursday. This fixes [#10082](https://github.com/ClickHouse/ClickHouse/issues/10082) [#10214](https://github.com/ClickHouse/ClickHouse/pull/10214) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix column names of constants inside JOIN that may clash with names of constants outside of JOIN. [#10207](https://github.com/ClickHouse/ClickHouse/pull/10207) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix move-to-prewhere optimization in presense of arrayJoin functions (in certain cases). This fixes [#10092](https://github.com/ClickHouse/ClickHouse/issues/10092) [#10195](https://github.com/ClickHouse/ClickHouse/pull/10195) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix scramble issue for native mysql-connector-java(JDBC) [#10140](https://github.com/ClickHouse/ClickHouse/pull/10140) ([BohuTANG](https://github.com/BohuTANG)) +* Fix using the current database for access checking when the database isn\'t specified. [#10192](https://github.com/ClickHouse/ClickHouse/pull/10192) ([Vitaly Baranov](https://github.com/vitlibar)) +* Fix some kinds of alters with compact parts. ... [#10130](https://github.com/ClickHouse/ClickHouse/pull/10130) ([Anton Popov](https://github.com/CurtizJ)) +* Add the ability to relax the restriction on non-deterministic functions usage in mutations with `allow_nondeterministic_mutations` setting. [#10186](https://github.com/ClickHouse/ClickHouse/pull/10186) ([filimonov](https://github.com/filimonov)) +* Do not break DROP DICTIONARY with DROP TABLE (otherwise after this error you will not be able to remove database) [#10165](https://github.com/ClickHouse/ClickHouse/pull/10165) ([Azat Khuzhin](https://github.com/azat)) +* Revert resetting the settings to their defaults after each query in TCPHandler. This PR reverts https://github.com/ClickHouse/ClickHouse/pull/9970 [#10181](https://github.com/ClickHouse/ClickHouse/pull/10181) ([Vitaly Baranov](https://github.com/vitlibar)) +* Convert blocks if structure does not match on INSERT into Distributed() ... [#10135](https://github.com/ClickHouse/ClickHouse/pull/10135) ([Azat Khuzhin](https://github.com/azat)) +* The number of rows was logged incorrectly (as sum across all parts) when inserted block is split by parts with partition key. [#10138](https://github.com/ClickHouse/ClickHouse/pull/10138) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Add some arguments check and support identifier arguments for MySQL Database Engine [#10077](https://github.com/ClickHouse/ClickHouse/pull/10077) ([Winter Zhang](https://github.com/zhang2014)) +* Fix incorrect `index_granularity_bytes` check while creating new replica. Fixes [#10098](https://github.com/ClickHouse/ClickHouse/issues/10098). [#10121](https://github.com/ClickHouse/ClickHouse/pull/10121) ([alesapin](https://github.com/alesapin)) +* Fix bug in `CHECK TABLE` query when table contain skip indices. [#10068](https://github.com/ClickHouse/ClickHouse/pull/10068) ([alesapin](https://github.com/alesapin)) +* Fix SIGSEGV on INSERT into Distributed table when its structure differs from the underlying tables. ... [#10105](https://github.com/ClickHouse/ClickHouse/pull/10105) ([Azat Khuzhin](https://github.com/azat)) +* Fix Distributed-over-Distributed with one only shard in nested table ... [#9997](https://github.com/ClickHouse/ClickHouse/pull/9997) ([Azat Khuzhin](https://github.com/azat)) +* Fix possible rows loss for queries with `JOIN` and `UNION ALL`. Fixes [#9826](https://github.com/ClickHouse/ClickHouse/issues/9826), [#10113](https://github.com/ClickHouse/ClickHouse/issues/10113). ... [#10099](https://github.com/ClickHouse/ClickHouse/pull/10099) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fix bug in clickhouse dictionary source from localhost clickhouse server. The bug may lead to memory corruption if types in dictionary and source are not compatible. [#10071](https://github.com/ClickHouse/ClickHouse/pull/10071) ([alesapin](https://github.com/alesapin)) +* Fixed replicated tables startup when updating from an old ClickHouse version where `/table/replicas/replica_name/metadata` node doesn\'t exist. Fixes [#10037](https://github.com/ClickHouse/ClickHouse/issues/10037). [#10095](https://github.com/ClickHouse/ClickHouse/pull/10095) ([alesapin](https://github.com/alesapin)) +* Fix error `Cannot clone block with columns because block has 0 columns ... While executing GroupingAggregatedTransform`. It happened when setting `distributed_aggregation_memory_efficient` was enabled, and distributed query read aggregating data with different level from different shards (mixed single and two level aggregation). ... [#10063](https://github.com/ClickHouse/ClickHouse/pull/10063) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fix deadlock on failed database attach at start with materialized view [#10054](https://github.com/ClickHouse/ClickHouse/pull/10054) ([Azat Khuzhin](https://github.com/azat)) +* Fix a segmentation fault that could occur in GROUP BY over string keys containing trailing zero bytes ([#8636](https://github.com/ClickHouse/ClickHouse/issues/8636), [#8925](https://github.com/ClickHouse/ClickHouse/issues/8925)). ... [#10025](https://github.com/ClickHouse/ClickHouse/pull/10025) ([Alexander Kuzmenkov](https://github.com/akuzm)) +* Fix parallel distributed INSERT SELECT for remote table. This PR fixes the solution provided in [#9759](https://github.com/ClickHouse/ClickHouse/pull/9759) ... [#9999](https://github.com/ClickHouse/ClickHouse/pull/9999) ([Vitaly Baranov](https://github.com/vitlibar)) +* Fix wrong results of distributed queries when alias could override qualified column name. Fixes [#9672](https://github.com/ClickHouse/ClickHouse/issues/9672) [#9714](https://github.com/ClickHouse/ClickHouse/issues/9714) [#9972](https://github.com/ClickHouse/ClickHouse/pull/9972) ([Artem Zuikov](https://github.com/4ertus2)) +* Fix possible deadlock in `SYSTEM RESTART REPLICAS` [#9955](https://github.com/ClickHouse/ClickHouse/pull/9955) ([tavplubix](https://github.com/tavplubix)) +* Fix the number of threads used for remote query execution (performance regression, since 20.3). This happened when query from `Distributed` table was executed simultaneously on local and remote shards. Fixes [#9965](https://github.com/ClickHouse/ClickHouse/issues/9965) [#9971](https://github.com/ClickHouse/ClickHouse/pull/9971) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fix reusing connection after changing user\'s default settings via `ALTER USER` or `ALTER SETTINGS PROFILE`. This fixes integration test `test_settings_constraints_distributed`. [#9970](https://github.com/ClickHouse/ClickHouse/pull/9970) ([Vitaly Baranov](https://github.com/vitlibar)) +* Fixed `DeleteOnDestroy` logic in `ATTACH PART` which could lead to automatic removal of attached part and added few tests [#9410](https://github.com/ClickHouse/ClickHouse/pull/9410) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Fix a bug with `ON CLUSTER` DDL queries freezing on server startup. [#9927](https://github.com/ClickHouse/ClickHouse/pull/9927) ([Gagan Arneja](https://github.com/garneja)) +* Fix two bugs in `windowFunnel` ... [#9938](https://github.com/ClickHouse/ClickHouse/pull/9938) ([achimbab](https://github.com/achimbab)) +* Fix bug in which the necessary tables weren\'t retrieved at one of the processing stages of queries to some databases. Fixes [#9699](https://github.com/ClickHouse/ClickHouse/issues/9699). [#9949](https://github.com/ClickHouse/ClickHouse/pull/9949) ([achulkov2](https://github.com/achulkov2)) +* Fix \'Not found column in block\' error when `JOIN` appears with `TOTALS`. Fixes [#9839](https://github.com/ClickHouse/ClickHouse/issues/9839) [#9939](https://github.com/ClickHouse/ClickHouse/pull/9939) ([Artem Zuikov](https://github.com/4ertus2)) +* Fix parsing multiple hosts set in the CREATE USER command, e.g. ... [#9924](https://github.com/ClickHouse/ClickHouse/pull/9924) ([Vitaly Baranov](https://github.com/vitlibar)) +* Fix `TRUNCATE` for Join table engine ([#9917](https://github.com/ClickHouse/ClickHouse/issues/9917)). [#9920](https://github.com/ClickHouse/ClickHouse/pull/9920) ([Amos Bird](https://github.com/amosbird)) +* Fix race condition between drop and optimize in `ReplicatedMergeTree`. [#9901](https://github.com/ClickHouse/ClickHouse/pull/9901) ([alesapin](https://github.com/alesapin)) +* Fix DISTINCT for Distributed when `optimize_skip_unused_shards` is set. [#9808](https://github.com/ClickHouse/ClickHouse/pull/9808) ([Azat Khuzhin](https://github.com/azat)) +* Fix "scalar doesn\'t exist" error in ALTERs ([#9878](https://github.com/ClickHouse/ClickHouse/issues/9878)). ... [#9904](https://github.com/ClickHouse/ClickHouse/pull/9904) ([Amos Bird](https://github.com/amosbird)) +* Fix error with qualified names in `distributed_product_mode=\'local\'`. Fixes [#4756](https://github.com/ClickHouse/ClickHouse/issues/4756) [#9891](https://github.com/ClickHouse/ClickHouse/pull/9891) ([Artem Zuikov](https://github.com/4ertus2)) +* For INSERT queries shard now clamps the settings got from the initiator to the shard\'s constaints instead of throwing an exception. This fix allows to send INSERT queries to a shard with another constraints. This change improves fix [#9447](https://github.com/ClickHouse/ClickHouse/issues/9447). [#9852](https://github.com/ClickHouse/ClickHouse/pull/9852) ([Vitaly Baranov](https://github.com/vitlibar)) +* Introduce commit retry logic to decrease the possibility of getting duplicates from Kafka in rare cases when offset commit was failed. [#9884](https://github.com/ClickHouse/ClickHouse/pull/9884) ([filimonov](https://github.com/filimonov)) +* Sometimes broker can reject commit if during `offsets.commit.timeout.ms` (broker setting, 5000 by default) there were no enough replicas available for the `__consumer_offsets` topic, also some other temporary issues like client-server connectivity problems are possible. Retries make that part more robust. ... [#10144](https://github.com/ClickHouse/ClickHouse/pull/10144) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Fix Distributed engine with virtual columns of the underlying table in WHERE after [#8846](https://github.com/ClickHouse/ClickHouse/issues/8846) (requires `optimize_skip_unused_shards`) [#9847](https://github.com/ClickHouse/ClickHouse/pull/9847) ([Azat Khuzhin](https://github.com/azat)) +* Fixed a few cases when timezone of the function argument wasn\'t used properly. [#9574](https://github.com/ClickHouse/ClickHouse/pull/9574) ([Vasily Nemkov](https://github.com/Enmk)) +* Fix \'Different expressions with the same alias\' error when query has PREWHERE and WHERE on distributed table and `SET distributed_product_mode = \'local\'`. [#9871](https://github.com/ClickHouse/ClickHouse/pull/9871) ([Artem Zuikov](https://github.com/4ertus2)) +* Fix mutations excessive memory consumption for tables with a composite primary key. This fixes [#9850](https://github.com/ClickHouse/ClickHouse/issues/9850). [#9860](https://github.com/ClickHouse/ClickHouse/pull/9860) ([alesapin](https://github.com/alesapin)) +* Fix calculating grants for introspection functions from the setting \'allow_introspection_functions\'. [#9840](https://github.com/ClickHouse/ClickHouse/pull/9840) ([Vitaly Baranov](https://github.com/vitlibar)) +* Fix max_distributed_connections (w/ and w/o Processors) [#9673](https://github.com/ClickHouse/ClickHouse/pull/9673) ([Azat Khuzhin](https://github.com/azat)) +* Fix possible exception `Got 0 in totals chunk, expected 1` on client. It happened for queries with `JOIN` in case if right joined table had zero rows. Example: `select * from system.one t1 join system.one t2 on t1.dummy = t2.dummy limit 0 FORMAT TabSeparated;`. Fixes [#9777](https://github.com/ClickHouse/ClickHouse/issues/9777). ... [#9823](https://github.com/ClickHouse/ClickHouse/pull/9823) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fix \'COMMA to CROSS JOIN rewriter is not enabled or cannot rewrite query\' error in case of subqueries with COMMA JOIN out of tables lists (i.e. in WHERE). Fixes [#9782](https://github.com/ClickHouse/ClickHouse/issues/9782) [#9830](https://github.com/ClickHouse/ClickHouse/pull/9830) ([Artem Zuikov](https://github.com/4ertus2)) +* Fix SIGSEGV with optimize_skip_unused_shards when type cannot be converted [#9804](https://github.com/ClickHouse/ClickHouse/pull/9804) ([Azat Khuzhin](https://github.com/azat)) +* Fix empty string handling in `splitByString`. [#9767](https://github.com/ClickHouse/ClickHouse/pull/9767) ([hcz](https://github.com/hczhcz)) +* Fix broken `ALTER TABLE DELETE COLUMN` query for compact parts. [#9779](https://github.com/ClickHouse/ClickHouse/pull/9779) ([alesapin](https://github.com/alesapin)) +* Fixed missing `rows_before_limit_at_least` for queries over http (with processors pipeline ). Fixes [#9730](https://github.com/ClickHouse/ClickHouse/issues/9730) [#9757](https://github.com/ClickHouse/ClickHouse/pull/9757) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fix completion highlighting issue when case insensitive matches are found. Do not include in changelog as the regression was not in any release. [#9753](https://github.com/ClickHouse/ClickHouse/pull/9753) ([Amos Bird](https://github.com/amosbird)) +* Fix excessive memory consumption in `ALTER` queries (mutations). This fixes [#9533](https://github.com/ClickHouse/ClickHouse/issues/9533) and [#9670](https://github.com/ClickHouse/ClickHouse/issues/9670). [#9754](https://github.com/ClickHouse/ClickHouse/pull/9754) ([alesapin](https://github.com/alesapin)) +* Fix possible permanent "Cannot schedule a task" error (due to unhandled exception in `ParallelAggregatingBlockInputStream::Handler::onFinish/onFinishThread`) ... [#9154](https://github.com/ClickHouse/ClickHouse/pull/9154) ([Azat Khuzhin](https://github.com/azat)) +* Fix bug in backquoting in external dictionaries DDL. Fixes [#9619](https://github.com/ClickHouse/ClickHouse/issues/9619). [#9734](https://github.com/ClickHouse/ClickHouse/pull/9734) ([alesapin](https://github.com/alesapin)) +* Fixed data race in `text_log`. It does not correspond to any real bug. [#9726](https://github.com/ClickHouse/ClickHouse/pull/9726) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix bug in a replication that doesn\'t allow replication to work if the user has executed mutations on the previous version. This fixes [#9645](https://github.com/ClickHouse/ClickHouse/issues/9645). [#9652](https://github.com/ClickHouse/ClickHouse/pull/9652) ([alesapin](https://github.com/alesapin)) +* Fixed incorrect internal function names for `sumKahan` and `sumWithOverflow`. I lead to exception while using this functions in remote queries. ... [#9636](https://github.com/ClickHouse/ClickHouse/pull/9636) ([Azat Khuzhin](https://github.com/azat)) +* Add setting `use_compact_format_in_distributed_parts_names` which allows to write files for `INSERT` queries into `Distributed` table with more compact format. This fixes [#9647](https://github.com/ClickHouse/ClickHouse/issues/9647). [#9653](https://github.com/ClickHouse/ClickHouse/pull/9653) ([alesapin](https://github.com/alesapin)) +* Fix RIGHT and FULL JOIN with LowCardinality in JOIN keys. [#9610](https://github.com/ClickHouse/ClickHouse/pull/9610) ([Artem Zuikov](https://github.com/4ertus2)) +* Fix possible exceptions `Size of filter doesn\'t match size of column` and `Invalid number of rows in Chunk` in `MergeTreeRangeReader`. They could appear while executing `PREWHERE` in some cases. ... [#9612](https://github.com/ClickHouse/ClickHouse/pull/9612) ([Anton Popov](https://github.com/CurtizJ)) +* Allow ALTER ON CLUSTER of Distributed tables with internal replication. This fixes [#3268](https://github.com/ClickHouse/ClickHouse/issues/3268) [#9617](https://github.com/ClickHouse/ClickHouse/pull/9617) ([shinoi2](https://github.com/shinoi2)) +* Fixed the issue: timezone was not preserved if you write a simple arithmetic expression like `time + 1` (in contrast to an expression like `time + INTERVAL 1 SECOND`). This fixes [#5743](https://github.com/ClickHouse/ClickHouse/issues/5743) [#9323](https://github.com/ClickHouse/ClickHouse/pull/9323) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +#### Improvement +* Use AWS SDK libraries from ClickHouse-Extras [#10527](https://github.com/ClickHouse/ClickHouse/pull/10527) ([Pavel Kovalenko](https://github.com/Jokser)) +* Use time zone when comparing DateTime with string literal. This fixes [#5206](https://github.com/ClickHouse/ClickHouse/issues/5206). [#10515](https://github.com/ClickHouse/ClickHouse/pull/10515) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Print verbose diagnostic info if Decimal value cannot be parsed from text input format. [#10205](https://github.com/ClickHouse/ClickHouse/pull/10205) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Add tasks/memory metrics for distributed/buffer schedule pools ... [#10449](https://github.com/ClickHouse/ClickHouse/pull/10449) ([Azat Khuzhin](https://github.com/azat)) +* Display result as soon as it\'s ready for SELECT DISTINCT queries in clickhouse-local and HTTP interface. This fixes [#8951](https://github.com/ClickHouse/ClickHouse/issues/8951) [#9559](https://github.com/ClickHouse/ClickHouse/pull/9559) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Allow to use `SAMPLE OFFSET` query instead of `cityHash64(PRIMARY KEY) % N == n` for splitting in `clickhouse-copier`. To use this feature, pass `--experimental-use-sample-offset 1` as a command line argument. [#10414](https://github.com/ClickHouse/ClickHouse/pull/10414) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +* Allow to parse BOM in TSV if the first column cannot contain BOM in its value. This fixes [#10301](https://github.com/ClickHouse/ClickHouse/issues/10301) [#10424](https://github.com/ClickHouse/ClickHouse/pull/10424) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Avro nested fields insert support [#10354](https://github.com/ClickHouse/ClickHouse/pull/10354) ([Andrew Onyshchuk](https://github.com/oandrew)) +* Allowed to alter column in non-modifying data mode when the same type is specified. ... [#10382](https://github.com/ClickHouse/ClickHouse/pull/10382) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Auto distributed_group_by_no_merge on GROUP BY sharding key (if `optimize_skip_unused_shards` is set) ... [#10341](https://github.com/ClickHouse/ClickHouse/pull/10341) ([Azat Khuzhin](https://github.com/azat)) +* Optimize queries with LIMIT/LIMIT BY/ORDER BY for distributed with GROUP BY sharding_key [#10373](https://github.com/ClickHouse/ClickHouse/pull/10373) ([Azat Khuzhin](https://github.com/azat)) +* Added a setting `max_server_memory_usage` to limit total memory usage of the server. The metric `MemoryTracking` is now calculated without a drift. The setting `max_memory_usage_for_all_queries` is now obsolete and does nothing. This closes [#10293](https://github.com/ClickHouse/ClickHouse/issues/10293). Note: on servers with very low memory amount and swap you may have to add `max_server_memory_usage_to_ram_ratio` in config.xml and set it to a value larger than one. [#10362](https://github.com/ClickHouse/ClickHouse/pull/10362) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Add config option `system_tables_lazy_load` that, if set to false, loads system tables with logs at the server startup. [Alexander Burmak](https://github.com/Alex-Burmak), [Svyatoslav Tkhon Il Pak](https://github.com/DeifyTheGod), [#9642](https://github.com/ClickHouse/ClickHouse/pull/9642) [#10359](https://github.com/ClickHouse/ClickHouse/pull/10359) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Use background thread pool (background_schedule_pool_size) for distributed sends [#10263](https://github.com/ClickHouse/ClickHouse/pull/10263) ([Azat Khuzhin](https://github.com/azat)) +* Use background thread pool for background buffer flushes (this will allow to avoid occupation threads when there are no insert\'s) ... [#10315](https://github.com/ClickHouse/ClickHouse/pull/10315) ([Azat Khuzhin](https://github.com/azat)) +* Support ReplicatedMergeTree over S3 [#10126](https://github.com/ClickHouse/ClickHouse/pull/10126) ([Pavel Kovalenko](https://github.com/Jokser)) +* Support for one special case of removing incompletely written parts. This fixes [#9940](https://github.com/ClickHouse/ClickHouse/issues/9940). [#10221](https://github.com/ClickHouse/ClickHouse/pull/10221) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Use isInjective() over manual list of such functions for GROUP BY optimization ... [#10342](https://github.com/ClickHouse/ClickHouse/pull/10342) ([Azat Khuzhin](https://github.com/azat)) +* Improve MsgPack input format by addition msgpack::visitor concept instead of msgpack::unpack concept. [#10325](https://github.com/ClickHouse/ClickHouse/pull/10325) ([Kruglov Pavel](https://github.com/Avogar)) +* Avoid printing error message in log if client sends RST packet immediately on connect. It is typical behaviour of IPVS balancer with keepalived and VRRP. This fixes [#1851](https://github.com/ClickHouse/ClickHouse/issues/1851) [#10274](https://github.com/ClickHouse/ClickHouse/pull/10274) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Allow to parse `+inf` for floating point types. This closes [#1839](https://github.com/ClickHouse/ClickHouse/issues/1839) [#10272](https://github.com/ClickHouse/ClickHouse/pull/10272) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Followup PR for https://github.com/yandex/ClickHouse/pull/5045. ... [#6256](https://github.com/ClickHouse/ClickHouse/pull/6256) ([Guillaume Tassery](https://github.com/YiuRULE)) +* Followup PR for [#5045](https://github.com/ClickHouse/ClickHouse/issues/5045). ... [#10269](https://github.com/ClickHouse/ClickHouse/pull/10269) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Implemented `generateRandom` table function for Nested types. This closes [#9903](https://github.com/ClickHouse/ClickHouse/issues/9903) [#10219](https://github.com/ClickHouse/ClickHouse/pull/10219) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Provide `max_allowed_packed` in MySQL compatibility interface that will help some clients to communicate with ClickHouse via MySQL protocol. ... [#10199](https://github.com/ClickHouse/ClickHouse/pull/10199) ([BohuTANG](https://github.com/BohuTANG)) +* Allow literals for GLOBAL IN (i.e. `SELECT * FROM remote(\'localhost\', system.one) WHERE dummy global in (0)`) [#10196](https://github.com/ClickHouse/ClickHouse/pull/10196) ([Azat Khuzhin](https://github.com/azat)) +* Fix various small issues in interactive mode of clickhouse-client [#10194](https://github.com/ClickHouse/ClickHouse/pull/10194) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Avoid superfluous dictionaries load (system.tables, DROP/SHOW CREATE TABLE) [#10164](https://github.com/ClickHouse/ClickHouse/pull/10164) ([Azat Khuzhin](https://github.com/azat)) +* Update to RWLock: timeout parameter for getLock() + implementation reworked to be phase fair [#10073](https://github.com/ClickHouse/ClickHouse/pull/10073) ([Alexander Kazakov](https://github.com/Akazz)) +* Enhanced compatibility with native mysql-connector-java(JDBC) [#10021](https://github.com/ClickHouse/ClickHouse/pull/10021) ([BohuTANG](https://github.com/BohuTANG)) +* The function `toString` is considered monotonic and can be used for index analysis even when applied in tautological cases with String or LowCardinality(String) argument. ... [#10110](https://github.com/ClickHouse/ClickHouse/pull/10110) ([Amos Bird](https://github.com/amosbird)) +* Add `ON CLUSTER` clause to commands `{CREATE|DROP} USER/ROLE/ROW POLICY/SETTINGS PROFILE/QUOTA`, `GRANT`. ... [#9811](https://github.com/ClickHouse/ClickHouse/pull/9811) ([Vitaly Baranov](https://github.com/vitlibar)) +* Virtual hosted-style support for S3 URI [#9998](https://github.com/ClickHouse/ClickHouse/pull/9998) ([Pavel Kovalenko](https://github.com/Jokser)) +* Now layout type for dictionaries with no arguments can be specified without round brackets in dictionaries DDL-queries. Fixes [#10057](https://github.com/ClickHouse/ClickHouse/issues/10057). [#10064](https://github.com/ClickHouse/ClickHouse/pull/10064) ([alesapin](https://github.com/alesapin)) +* Add ability to use number ranges with leading zeros in filepath [#9989](https://github.com/ClickHouse/ClickHouse/pull/9989) ([Olga Khvostikova](https://github.com/stavrolia)) +* Better memory usage in CROSS JOIN. [#10029](https://github.com/ClickHouse/ClickHouse/pull/10029) ([Artem Zuikov](https://github.com/4ertus2)) +* Use DFA algorithm for sequenceMatch without `(?t)`. [#4004](https://github.com/ClickHouse/ClickHouse/pull/4004) ([L\xc3\xa9o Ercolanelli](https://github.com/LeoErcolanelli)) +* The `clickhouse-format` tool is now able to format multiple queries when the `-n` argument is used. [#10852](https://github.com/ClickHouse/ClickHouse/pull/10852) ([Dar\xc3\xado](https://github.com/dgrr)) +* Try to all shards in cluster when getting structure of remote table and skip_unavailable_shards is set. ... [#7278](https://github.com/ClickHouse/ClickHouse/pull/7278) ([nvartolomei](https://github.com/nvartolomei)) +* Add `total_rows`/`total_bytes` into the `system.tables` table. [#9919](https://github.com/ClickHouse/ClickHouse/pull/9919) ([Azat Khuzhin](https://github.com/azat)) +* System log tables now use polymorpic parts by default. [#9905](https://github.com/ClickHouse/ClickHouse/pull/9905) ([Anton Popov](https://github.com/CurtizJ)) +* Add type column into system.settings/merge_tree_settings [#9909](https://github.com/ClickHouse/ClickHouse/pull/9909) ([Azat Khuzhin](https://github.com/azat)) +* Check for available CPU instructions at server startup as early as possible. [#9888](https://github.com/ClickHouse/ClickHouse/pull/9888) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Remove order by stage from mutations because we read from a single ordered part in a single thread. Also add check that the order of rows in mutation is ordered in sorting key order and this order is not violated. [#9886](https://github.com/ClickHouse/ClickHouse/pull/9886) ([alesapin](https://github.com/alesapin)) +* Implement operator LIKE for FixedString at left hand side. This is needed to better support TPC-DS queries. [#9890](https://github.com/ClickHouse/ClickHouse/pull/9890) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Add force_optimize_skip_unused_shards_no_nested that will disable force_optimize_skip_unused_shards for nested Distributed table [#9812](https://github.com/ClickHouse/ClickHouse/pull/9812) ([Azat Khuzhin](https://github.com/azat)) +* Now columns size calculated only once for MergeTree data parts. [#9827](https://github.com/ClickHouse/ClickHouse/pull/9827) ([alesapin](https://github.com/alesapin)) +* Evaluate constant expressions for optimize_skip_unused_shards (i.e. `SELECT * FROM foo_dist WHERE key=xxHash32(0)`) [#8846](https://github.com/ClickHouse/ClickHouse/pull/8846) ([Azat Khuzhin](https://github.com/azat)) +* Check for using `Date` or `DateTime` column from TTL expressions was removed. [#9967](https://github.com/ClickHouse/ClickHouse/pull/9967) ([Vladimir Chebotarev](https://github.com/excitoon)) +* DiskS3 hard links optimal implementation. [#9760](https://github.com/ClickHouse/ClickHouse/pull/9760) ([Pavel Kovalenko](https://github.com/Jokser)) +* If `set multiple_joins_rewriter_version = 2` enables second version of multiple JOIN rewrites that keeps not clashed column names as is. It supports multiple JOINs with `USING` and allow `select *` for JOINs with subqueries. ... [#9739](https://github.com/ClickHouse/ClickHouse/pull/9739) ([Artem Zuikov](https://github.com/4ertus2)) +* Implementation of "non-blocking" alter for StorageMergeTree [#9606](https://github.com/ClickHouse/ClickHouse/pull/9606) ([alesapin](https://github.com/alesapin)) +* MergeTree full support for DiskS3 [#9646](https://github.com/ClickHouse/ClickHouse/pull/9646) ([Pavel Kovalenko](https://github.com/Jokser)) +* Extend `splitByString` to support empty strings as separators. [#9742](https://github.com/ClickHouse/ClickHouse/pull/9742) ([hcz](https://github.com/hczhcz)) +* Add a `timestamp_ns` column to `system.trace_log`. It contains a high-definition timestamp of the trace event, and allows to build timelines of thread profiles ("flame charts"). [#9696](https://github.com/ClickHouse/ClickHouse/pull/9696) ([Alexander Kuzmenkov](https://github.com/akuzm)) +* When the setting `send_logs_level` is enabled, avoid intermixing of log messages and query progress. [#9634](https://github.com/ClickHouse/ClickHouse/pull/9634) ([Azat Khuzhin](https://github.com/azat)) +* Added MATERIALIZE TTL IN PARTITION. [#9581](https://github.com/ClickHouse/ClickHouse/pull/9581) ([Vladimir Chebotarev](https://github.com/excitoon)) + +#### Performance Improvement +* Better insert logic for right table for Partial MergeJoin. [#10467](https://github.com/ClickHouse/ClickHouse/pull/10467) ([Artem Zuikov](https://github.com/4ertus2)) +* Improved performance of row-oriented formats (more than 10% for CSV and more than 35% for Avro in case of narrow tables). [#10503](https://github.com/ClickHouse/ClickHouse/pull/10503) ([Andrew Onyshchuk](https://github.com/oandrew)) +* Improved performance of queries with explicitly defined sets at right side of IN operator and tuples in the left side. [#10385](https://github.com/ClickHouse/ClickHouse/pull/10385) ([Anton Popov](https://github.com/CurtizJ)) +* Use less memory for hash table in HashJoin. [#10416](https://github.com/ClickHouse/ClickHouse/pull/10416) ([Artem Zuikov](https://github.com/4ertus2)) +* Special HashJoin over StorageDictionary. Allow rewrite `dictGet()` functions with JOINs. It\'s not backward incompatible itself but could uncover [#8400](https://github.com/ClickHouse/ClickHouse/issues/8400) on some installations. [#10133](https://github.com/ClickHouse/ClickHouse/pull/10133) ([Artem Zuikov](https://github.com/4ertus2)) +* Enable parallel insert of materialized view when its target table supports. [#10052](https://github.com/ClickHouse/ClickHouse/pull/10052) ([vxider](https://github.com/Vxider)) +* Improved performance of index analisys with monotonic functions. [#9607](https://github.com/ClickHouse/ClickHouse/pull/9607) ([Anton Popov](https://github.com/CurtizJ)) +* Improve performance of index analisys with monotonic functions. [#10026](https://github.com/ClickHouse/ClickHouse/pull/10026) ([Anton Popov](https://github.com/CurtizJ)) +* Using SSE2 or SSE4.2 SIMD intrinsics to speed up tokenization in bloom filters. ... [#9968](https://github.com/ClickHouse/ClickHouse/pull/9968) ([Vasily Nemkov](https://github.com/Enmk)) +* Optimize Volnitsky searcher by inlining, should be about 5-10% search improvement for queries with many needles or many similar bigrams ... [#4862](https://github.com/ClickHouse/ClickHouse/pull/4862) ([Danila Kutenin](https://github.com/danlark1)) +* Optimize count() [#6028](https://github.com/ClickHouse/ClickHouse/pull/6028) ([Amos Bird](https://github.com/amosbird)) +* Now prewhere should be at least as efficient as where. As a result, any viable predicates can be moved into prewhere without fancy heuristic ... [#7769](https://github.com/ClickHouse/ClickHouse/pull/7769) ([Amos Bird](https://github.com/amosbird)) +* Enable SSO for IN operator. Improvements varies from 4% - **257%** . Should be merged after https://github.com/ClickHouse/ClickHouse/pull/7662 ... [#7782](https://github.com/ClickHouse/ClickHouse/pull/7782) ([Amos Bird](https://github.com/amosbird)) +* Improved performance of queries with explicitly defined sets at right side of `IN` operator. This fixes performance regression in version 20.3. [#9740](https://github.com/ClickHouse/ClickHouse/pull/9740) ([Anton Popov](https://github.com/CurtizJ)) +* Now clickhouse-copier splits each partition in number of pieces and copies them independently. [#9075](https://github.com/ClickHouse/ClickHouse/pull/9075) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +* Adding more aggregation methods. For example TPCH query 1 will now pick `FixedHashMap` and gets 25% performance gain [#9829](https://github.com/ClickHouse/ClickHouse/pull/9829) ([Amos Bird](https://github.com/amosbird)) +* Use single row counter for multiple streams in pre-limit transform. This helps to avoid uniting pipeline streams in queries with `limit` but without `order by` (like `select f(x) from (select x from t limit 1000000000)`) and use multiple threads for further processing. [#9602](https://github.com/ClickHouse/ClickHouse/pull/9602) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) + +#### Build/Testing/Packaging Improvement +* Adding integration tests for new ALTER RENAME COLUMN query. [#10654](https://github.com/ClickHouse/ClickHouse/pull/10654) ([vzakaznikov](https://github.com/vzakaznikov)) +* Fixed possible signed integer overflow in invocation of function `now64` with wrong arguments. This fixes [#8973](https://github.com/ClickHouse/ClickHouse/issues/8973) [#10511](https://github.com/ClickHouse/ClickHouse/pull/10511) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Split fuzzer and sanitizer configurations to make build config compatible with Oss-fuzz. [#10494](https://github.com/ClickHouse/ClickHouse/pull/10494) ([kyprizel](https://github.com/kyprizel)) +* Fixes for clang-tidy on clang-10. [#10420](https://github.com/ClickHouse/ClickHouse/pull/10420) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Display absolute paths in error messages. Otherwise KDevelop fails to navigate to correct file and opens a new file instead. [#10434](https://github.com/ClickHouse/ClickHouse/pull/10434) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Added `ASAN_OPTIONS` environment variable to investigate errors in CI stress tests with Address sanitizer. [#10440](https://github.com/ClickHouse/ClickHouse/pull/10440) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +* Enable ThinLTO for clang builds (experimental). [#10435](https://github.com/ClickHouse/ClickHouse/pull/10435) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Remove accidential dependency on Z3 that may be introduced if the system has Z3 solver installed. [#10426](https://github.com/ClickHouse/ClickHouse/pull/10426) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Move integration tests docker files to docker/ directory. [#10335](https://github.com/ClickHouse/ClickHouse/pull/10335) ([Ilya Yatsishin](https://github.com/qoega)) +* Allow to use `clang-10` in CI. This is to ensure that [#10238](https://github.com/ClickHouse/ClickHouse/issues/10238) is fixed. [#10384](https://github.com/ClickHouse/ClickHouse/pull/10384) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Update OpenSSL to upstream master. Fixed the issue when TLS connections may fail with the message `OpenSSL SSL_read: error:14094438:SSL routines:ssl3_read_bytes:tlsv1 alert internal error` and `SSL Exception: error:2400006E:random number generator::error retrieving entropy`. The issue was present in version 20.1. [#8956](https://github.com/ClickHouse/ClickHouse/pull/8956) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix clang-10 build. https://github.com/ClickHouse/ClickHouse/issues/10238 [#10370](https://github.com/ClickHouse/ClickHouse/pull/10370) ([Amos Bird](https://github.com/amosbird)) +* Fixing 00964_live_view_watch_events_heartbeat.py test. [#10356](https://github.com/ClickHouse/ClickHouse/pull/10356) ([vzakaznikov](https://github.com/vzakaznikov)) +* This PR adds the performance test for [Parallel INSERT for materialized view](https://github.com/ClickHouse/ClickHouse/pull/10052). [#10345](https://github.com/ClickHouse/ClickHouse/pull/10345) ([vxider](https://github.com/Vxider)) +* Fix flaky test `test_settings_constraints_distributed.test_insert_clamps_settings`. [#10346](https://github.com/ClickHouse/ClickHouse/pull/10346) ([Vitaly Baranov](https://github.com/vitlibar)) +* Add util to test results upload in CI ClickHouse [#10330](https://github.com/ClickHouse/ClickHouse/pull/10330) ([Ilya Yatsishin](https://github.com/qoega)) +* Convert test results to JSONEachRow format in junit_to_html tool [#10323](https://github.com/ClickHouse/ClickHouse/pull/10323) ([Ilya Yatsishin](https://github.com/qoega)) +* Update cctz just in case. [#10215](https://github.com/ClickHouse/ClickHouse/pull/10215) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Allow to create HTML report from purest JUnit XML report. [#10247](https://github.com/ClickHouse/ClickHouse/pull/10247) ([Ilya Yatsishin](https://github.com/qoega)) +* Update the check for minimal compiler version. Fix the root cause of the issue [#10250](https://github.com/ClickHouse/ClickHouse/issues/10250) [#10256](https://github.com/ClickHouse/ClickHouse/pull/10256) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Initial support for live view tables over distributed [#10179](https://github.com/ClickHouse/ClickHouse/pull/10179) ([vzakaznikov](https://github.com/vzakaznikov)) +* Fix (false) MSan report in MergeTreeIndexFullText. The issue first appeared in [#9968](https://github.com/ClickHouse/ClickHouse/issues/9968). [#10801](https://github.com/ClickHouse/ClickHouse/pull/10801) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* clickhouse-docker-util [#10151](https://github.com/ClickHouse/ClickHouse/pull/10151) ([filimonov](https://github.com/filimonov)) +* Update pdqsort to recent version [#10171](https://github.com/ClickHouse/ClickHouse/pull/10171) ([Ivan](https://github.com/abyss7)) +* Update libdivide to v3.0 [#10169](https://github.com/ClickHouse/ClickHouse/pull/10169) ([Ivan](https://github.com/abyss7)) +* Add check with enabled polymorphic parts. [#10086](https://github.com/ClickHouse/ClickHouse/pull/10086) ([Anton Popov](https://github.com/CurtizJ)) +* Add cross-compile build for FreeBSD. This fixes [#9465](https://github.com/ClickHouse/ClickHouse/issues/9465) [#9643](https://github.com/ClickHouse/ClickHouse/pull/9643) ([Ivan](https://github.com/abyss7)) +* performance test for [#6924](https://github.com/ClickHouse/ClickHouse/issues/6924) [#6980](https://github.com/ClickHouse/ClickHouse/pull/6980) ([filimonov](https://github.com/filimonov)) +* support `StorageFile(, null) ` to deser block into given format file without actually write to disk. It\'s used to perf test format parsers. ... [#8455](https://github.com/ClickHouse/ClickHouse/pull/8455) ([Amos Bird](https://github.com/amosbird)) +* Move all folders inside /dbms one level up [#9974](https://github.com/ClickHouse/ClickHouse/pull/9974) ([Ivan](https://github.com/abyss7)) +* Added a test that checks that read from MergeTree with single thread is performed in order. Addition to [#9670](https://github.com/ClickHouse/ClickHouse/issues/9670) [#9762](https://github.com/ClickHouse/ClickHouse/pull/9762) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix the `00964_live_view_watch_events_heartbeat.py` test to avoid race condition. [#9944](https://github.com/ClickHouse/ClickHouse/pull/9944) ([vzakaznikov](https://github.com/vzakaznikov)) +* Fix integration test `test_settings_constraints`. ... [#9962](https://github.com/ClickHouse/ClickHouse/pull/9962) ([Vitaly Baranov](https://github.com/vitlibar)) +* Every function in its own file, part 12. [#9922](https://github.com/ClickHouse/ClickHouse/pull/9922) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Added performance test for the case of extremely slow analysis of array of tuples. [#9872](https://github.com/ClickHouse/ClickHouse/pull/9872) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Update zstd to 1.4.4. It has some minor improvements in performance and compression ratio. If you run replicas with different versions of ClickHouse you may see reasonable error messages `Data after merge is not byte-identical to data on another replicas.` with explanation. These messages are Ok and you should not worry. ... [#10663](https://github.com/ClickHouse/ClickHouse/pull/10663) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix TSan report in `system.stack_trace`. [#9832](https://github.com/ClickHouse/ClickHouse/pull/9832) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Removed dependency on `clock_getres`. [#9833](https://github.com/ClickHouse/ClickHouse/pull/9833) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Preparation to fix https://clickhouse-test-reports.s3.yandex.net/9813/622bca235848bf1a7950d6eeb10241204aa069eb/stress_test_(thread)/stderr.log [#9825](https://github.com/ClickHouse/ClickHouse/pull/9825) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Added identifier names check with clang-tidy. [#9799](https://github.com/ClickHouse/ClickHouse/pull/9799) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Update "builder" docker image. This image is not used in CI but is useful for developers. [#9809](https://github.com/ClickHouse/ClickHouse/pull/9809) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Remove old `performance-test` tool that is no longer used in CI. `clickhouse-performance-test` is great but now we are using way superior tool that is doing comparison testing with sophisticated statistical formulas to achieve confident results regardless to various changes in environment. [#9796](https://github.com/ClickHouse/ClickHouse/pull/9796) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Added most of clang-static-analyzer checks. [#9765](https://github.com/ClickHouse/ClickHouse/pull/9765) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Update Poco to 1.9.3 in preparation for MongoDB URI support. [#6892](https://github.com/ClickHouse/ClickHouse/pull/6892) ([Alexander Kuzmenkov](https://github.com/akuzm)) +* Added even more clang-tidy checks. [#9719](https://github.com/ClickHouse/ClickHouse/pull/9719) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Added more clang-tidy checks. [#9718](https://github.com/ClickHouse/ClickHouse/pull/9718) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Added more clang-tidy checks. [#9717](https://github.com/ClickHouse/ClickHouse/pull/9717) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix build with `-DUSE_STATIC_LIBRARIES=0 -DENABLE_JEMALLOC=0` [#9651](https://github.com/ClickHouse/ClickHouse/pull/9651) ([Artem Zuikov](https://github.com/4ertus2)) +* For change log script, if merge commit was cherry-picked to release branch, take pr name from commit description. [#9708](https://github.com/ClickHouse/ClickHouse/pull/9708) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Support `vX.X-conflicts` tag in backport script. ... [#9705](https://github.com/ClickHouse/ClickHouse/pull/9705) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fix `auto-label` for backporting script. [#9685](https://github.com/ClickHouse/ClickHouse/pull/9685) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Use libc++ in Darwin cross-build to make it consistent with native build. [#9665](https://github.com/ClickHouse/ClickHouse/pull/9665) ([Hui Wang](https://github.com/huiwang)) +* Fix flacky test `01017_uniqCombined_memory_usage`. Continuation of [#7236](https://github.com/ClickHouse/ClickHouse/issues/7236). [#9667](https://github.com/ClickHouse/ClickHouse/pull/9667) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix build for native MacOS Clang compiler [#9649](https://github.com/ClickHouse/ClickHouse/pull/9649) ([Ivan](https://github.com/abyss7)) +* Allow to add various glitches around `pthread_mutex_lock`, `pthread_mutex_unlock` functions. [#9635](https://github.com/ClickHouse/ClickHouse/pull/9635) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Add support for `clang-tidy` in `packager` script. [#9625](https://github.com/ClickHouse/ClickHouse/pull/9625) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Add ability to use unbundled msgpack. [#10168](https://github.com/ClickHouse/ClickHouse/pull/10168) ([Azat Khuzhin](https://github.com/azat)) + +#### New Feature / Improvement +* Support complex types inside Avro nested fields [#10502](https://github.com/ClickHouse/ClickHouse/pull/10502) ([Andrew Onyshchuk](https://github.com/oandrew)) + + ## ClickHouse release v20.3 ### ClickHouse release v20.3.8.53, 2020-04-23 -### Bug Fix +#### Bug Fix * Fixed wrong behaviour of datetime functions for timezones that has altered between positive and negative offsets from UTC (e.g. Pacific/Kiritimati). This fixes [#7202](https://github.com/ClickHouse/ClickHouse/issues/7202) [#10369](https://github.com/ClickHouse/ClickHouse/pull/10369) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Fix possible segfault with `distributed_group_by_no_merge` enabled (introduced in 20.3.7.46 by [#10131](https://github.com/ClickHouse/ClickHouse/issues/10131)). [#10399](https://github.com/ClickHouse/ClickHouse/pull/10399) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) * Fix wrong flattening of `Array(Tuple(...))` data types. This fixes [#10259](https://github.com/ClickHouse/ClickHouse/issues/10259) [#10390](https://github.com/ClickHouse/ClickHouse/pull/10390) ([alexey-milovidov](https://github.com/alexey-milovidov)) @@ -18,7 +325,7 @@ * Fix the issue when a query with ARRAY JOIN, ORDER BY and LIMIT may return incomplete result. This fixes [#10226](https://github.com/ClickHouse/ClickHouse/issues/10226). Author: [Vadim Plakhtinskiy](https://github.com/VadimPlh). [#10427](https://github.com/ClickHouse/ClickHouse/pull/10427) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Check the number and type of arguments when creating BloomFilter index [#9623](https://github.com/ClickHouse/ClickHouse/issues/9623) [#10431](https://github.com/ClickHouse/ClickHouse/pull/10431) ([Winter Zhang](https://github.com/zhang2014)) -### Performance Improvement +#### Performance Improvement * Improved performance of queries with explicitly defined sets at right side of `IN` operator and tuples in the left side. This fixes performance regression in version 20.3. [#9740](https://github.com/ClickHouse/ClickHouse/pull/9740), [#10385](https://github.com/ClickHouse/ClickHouse/pull/10385) ([Anton Popov](https://github.com/CurtizJ)) ### ClickHouse release v20.3.7.46, 2020-04-17 From 111902a0186f098a7ee47628f2bf24bf88b0b39d Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Fri, 15 May 2020 16:14:39 +0300 Subject: [PATCH 259/738] Merge #10600 --- docs/en/operations/server-configuration-parameters/settings.md | 2 +- docs/es/operations/server-configuration-parameters/settings.md | 2 +- docs/fa/operations/server-configuration-parameters/settings.md | 2 +- docs/fr/operations/server-configuration-parameters/settings.md | 2 +- docs/ja/operations/server-configuration-parameters/settings.md | 2 +- docs/ru/operations/server-configuration-parameters/settings.md | 2 +- docs/tr/operations/server-configuration-parameters/settings.md | 2 +- docs/zh/operations/server-configuration-parameters/settings.md | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 3a11785d6ba..41ee085d8d3 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -207,7 +207,7 @@ If `http_port` is specified, the OpenSSL configuration is ignored even if it is **Example** ``` xml -0000 +9999 ``` ## http\_server\_default\_response {#server_configuration_parameters-http_server_default_response} diff --git a/docs/es/operations/server-configuration-parameters/settings.md b/docs/es/operations/server-configuration-parameters/settings.md index 7ca3b628bbc..29f726ee635 100644 --- a/docs/es/operations/server-configuration-parameters/settings.md +++ b/docs/es/operations/server-configuration-parameters/settings.md @@ -209,7 +209,7 @@ Si `http_port` se especifica, la configuración de OpenSSL se ignora incluso si **Ejemplo** ``` xml -0000 +9999 ``` ## http\_server\_default\_response {#server_configuration_parameters-http_server_default_response} diff --git a/docs/fa/operations/server-configuration-parameters/settings.md b/docs/fa/operations/server-configuration-parameters/settings.md index 673f24c1494..1459c20dd5c 100644 --- a/docs/fa/operations/server-configuration-parameters/settings.md +++ b/docs/fa/operations/server-configuration-parameters/settings.md @@ -210,7 +210,7 @@ toc_title: "\u062A\u0646\u0638\u06CC\u0645\u0627\u062A \u06A9\u0627\u0631\u06AF\ **مثال** ``` xml -0000 +9999 ``` ## نقلقولهای جدید از این نویسنده {#server_configuration_parameters-http_server_default_response} diff --git a/docs/fr/operations/server-configuration-parameters/settings.md b/docs/fr/operations/server-configuration-parameters/settings.md index 741bec44421..45be3c5c009 100644 --- a/docs/fr/operations/server-configuration-parameters/settings.md +++ b/docs/fr/operations/server-configuration-parameters/settings.md @@ -209,7 +209,7 @@ Si `http_port` est spécifié, la configuration OpenSSL est ignorée même si el **Exemple** ``` xml -0000 +9999 ``` ## http\_server\_default\_response {#server_configuration_parameters-http_server_default_response} diff --git a/docs/ja/operations/server-configuration-parameters/settings.md b/docs/ja/operations/server-configuration-parameters/settings.md index 2b460e8aca6..98a31fa5f60 100644 --- a/docs/ja/operations/server-configuration-parameters/settings.md +++ b/docs/ja/operations/server-configuration-parameters/settings.md @@ -209,7 +209,7 @@ HTTP経由でサーバーに接続するためのポート。 **例** ``` xml -0000 +9999 ``` ## http\_server\_default\_response {#server_configuration_parameters-http_server_default_response} diff --git a/docs/ru/operations/server-configuration-parameters/settings.md b/docs/ru/operations/server-configuration-parameters/settings.md index 43e7eaac7d7..7a0507724e4 100644 --- a/docs/ru/operations/server-configuration-parameters/settings.md +++ b/docs/ru/operations/server-configuration-parameters/settings.md @@ -195,7 +195,7 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat **Пример** ``` xml -0000 +9999 ``` ## http\_server\_default\_response {#server_configuration_parameters-http_server_default_response} diff --git a/docs/tr/operations/server-configuration-parameters/settings.md b/docs/tr/operations/server-configuration-parameters/settings.md index a944261de8d..cc5ef3e8e21 100644 --- a/docs/tr/operations/server-configuration-parameters/settings.md +++ b/docs/tr/operations/server-configuration-parameters/settings.md @@ -209,7 +209,7 @@ Eğer `http_port` belirtilmişse, OpenSSL yapılandırması ayarlanmış olsa bi **Örnek** ``` xml -0000 +9999 ``` ## http\_server\_default\_response {#server_configuration_parameters-http_server_default_response} diff --git a/docs/zh/operations/server-configuration-parameters/settings.md b/docs/zh/operations/server-configuration-parameters/settings.md index 23db483217a..2c9d611b6a7 100644 --- a/docs/zh/operations/server-configuration-parameters/settings.md +++ b/docs/zh/operations/server-configuration-parameters/settings.md @@ -207,7 +207,7 @@ ClickHouse每x秒重新加载内置字典。 这使得编辑字典 “on the fly **示例** ``` xml -0000 +9999 ``` ## http\_server\_default\_response {#server_configuration_parameters-http_server_default_response} From 833ab950dfd8333d47bb9e150a632c963244bf7b Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 15 May 2020 16:25:27 +0300 Subject: [PATCH 260/738] do not use overloaded comma operator --- programs/odbc-bridge/ODBCBlockOutputStream.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/programs/odbc-bridge/ODBCBlockOutputStream.cpp b/programs/odbc-bridge/ODBCBlockOutputStream.cpp index 2d7c093ae67..c2597805230 100644 --- a/programs/odbc-bridge/ODBCBlockOutputStream.cpp +++ b/programs/odbc-bridge/ODBCBlockOutputStream.cpp @@ -113,9 +113,7 @@ void ODBCBlockOutputStream::write(const Block & block) std::vector row_to_insert(block.columns()); Poco::Data::Statement statement(session << getInsertQuery(db_name, table_name, columns, quoting) + getQuestionMarks(block.columns())); for (size_t i = 0; i < block.columns(); ++i) - { - statement, Poco::Data::Keywords::use(row_to_insert[i]); - } + statement.addBind(Poco::Data::Keywords::use(row_to_insert[i])); for (size_t i = 0; i < block.rows(); ++i) { From 6fb9a19a40d4d9af0675a4f3212230e8e90e659c Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 15 May 2020 17:31:48 +0300 Subject: [PATCH 261/738] Ignore some tests --- .../configs/config.d/storage_configuration.xml | 9 ++++++--- tests/integration/test_rename_column/test.py | 6 +++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_rename_column/configs/config.d/storage_configuration.xml b/tests/integration/test_rename_column/configs/config.d/storage_configuration.xml index 044009512bc..131219abf3d 100644 --- a/tests/integration/test_rename_column/configs/config.d/storage_configuration.xml +++ b/tests/integration/test_rename_column/configs/config.d/storage_configuration.xml @@ -4,6 +4,9 @@ + + /internal/ + /external/ @@ -11,9 +14,9 @@ - - default - + + internal + external diff --git a/tests/integration/test_rename_column/test.py b/tests/integration/test_rename_column/test.py index 85d054fefeb..9fac783e712 100644 --- a/tests/integration/test_rename_column/test.py +++ b/tests/integration/test_rename_column/test.py @@ -14,7 +14,7 @@ node_options = dict( with_zookeeper=True, main_configs=['configs/remote_servers.xml'], config_dir='configs', - tmpfs=['/external:size=200M']) + tmpfs=['/external:size=200M', '/internal:size=1M']) cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', macros={"shard": 0, "replica": 1}, **node_options) @@ -299,6 +299,7 @@ def test_rename_with_parallel_insert(started_cluster): drop_table(nodes, table_name) +@pytest.mark.skip(reason="For unknown reason one of these tests kill Zookeeper") def test_rename_with_parallel_merges(started_cluster): table_name = "test_rename_with_parallel_merges" drop_table(nodes, table_name) @@ -336,6 +337,7 @@ def test_rename_with_parallel_merges(started_cluster): drop_table(nodes, table_name) +@pytest.mark.skip(reason="For unknown reason one of these tests kill Zookeeper") def test_rename_with_parallel_slow_insert(started_cluster): table_name = "test_rename_with_parallel_slow_insert" drop_table(nodes, table_name) @@ -497,6 +499,7 @@ def test_rename_with_parallel_ttl_delete(started_cluster): drop_table(nodes, table_name) +@pytest.mark.skip(reason="For unknown reason one of these tests kill Zookeeper") def test_rename_distributed(started_cluster): table_name = 'test_rename_distributed' try: @@ -513,6 +516,7 @@ def test_rename_distributed(started_cluster): drop_distributed_table(node1, table_name) +@pytest.mark.skip(reason="For unknown reason one of these tests kill Zookeeper") def test_rename_distributed_parallel_insert_and_select(started_cluster): table_name = 'test_rename_distributed_parallel_insert_and_select' try: From a504db34513f41b60bd60bf0792ed5bee64e8cdd Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 15 May 2020 18:11:13 +0300 Subject: [PATCH 262/738] Fix typo --- src/Storages/AlterCommands.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 43f7c15b784..91a48f13a39 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -814,7 +814,7 @@ void AlterCommands::validate(const StorageInMemoryMetadata & metadata, const Con if (next_command.type == AlterCommand::RENAME_COLUMN) { if (next_command.column_name == command.rename_to) - throw Exception{"Transitive renames in a single ALTER query are not allowed (don't make sence)", + throw Exception{"Transitive renames in a single ALTER query are not allowed (don't make sense)", ErrorCodes::NOT_IMPLEMENTED}; else if (next_command.column_name == command.column_name) throw Exception{"Cannot rename column '" + backQuote(command.column_name) From c4a07812f76242b3ba08f2c5fca1d1b87ee5d929 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 15 May 2020 18:48:19 +0300 Subject: [PATCH 263/738] Try fix build. --- src/Functions/concat.cpp | 8 ++------ .../test_string_aggregation.py | 0 2 files changed, 2 insertions(+), 6 deletions(-) create mode 100644 tests/integration/test_backward_compatability/test_string_aggregation.py diff --git a/src/Functions/concat.cpp b/src/Functions/concat.cpp index 4eea68de1f6..c883a56aafd 100644 --- a/src/Functions/concat.cpp +++ b/src/Functions/concat.cpp @@ -73,12 +73,6 @@ public: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - if (arguments.size() < 2) - throw Exception( - "Number of arguments for function " + getName() + " doesn't match: passed " + toString(arguments.size()) - + ", should be at least 2.", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - /// Format function is not proven to be faster for two arguments. /// Actually there is overhead of 2 to 5 extra instructions for each string for checking empty strings in FormatImpl. /// Though, benchmarks are really close, for most examples we saw executeBinary is slightly faster (0-3%). @@ -122,6 +116,8 @@ private: void executeFormatImpl(Block & block, const ColumnNumbers & arguments, const size_t result, size_t input_rows_count) { + assert(arguments.size() > 2); + auto c_res = ColumnString::create(); std::vector data(arguments.size()); std::vector offsets(arguments.size()); diff --git a/tests/integration/test_backward_compatability/test_string_aggregation.py b/tests/integration/test_backward_compatability/test_string_aggregation.py new file mode 100644 index 00000000000..e69de29bb2d From b8052f512a0e48b0dca61376d854c026bb8cfc92 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 15 May 2020 20:40:11 +0300 Subject: [PATCH 264/738] try fix tests. --- .../Executors/PullingPipelineExecutor.cpp | 43 +++++++++++++------ .../Executors/PullingPipelineExecutor.h | 2 + 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/src/Processors/Executors/PullingPipelineExecutor.cpp b/src/Processors/Executors/PullingPipelineExecutor.cpp index 07ca8f2bbf4..eb51ac363f4 100644 --- a/src/Processors/Executors/PullingPipelineExecutor.cpp +++ b/src/Processors/Executors/PullingPipelineExecutor.cpp @@ -23,6 +23,15 @@ struct PullingPipelineExecutor::Data if (thread.joinable()) thread.join(); } + + void rethrowExceptionIfHas() + { + if (has_exception) + { + has_exception = false; + std::rethrow_exception(std::move(exception)); + } + } }; PullingPipelineExecutor::PullingPipelineExecutor(QueryPipeline & pipeline_) : pipeline(pipeline_) @@ -35,25 +44,12 @@ PullingPipelineExecutor::~PullingPipelineExecutor() { try { - /// Cancel execution if it wasn't finished. - if (data && !data->is_executed) - cancel(); - - /// Finish lazy format. Otherwise thread.join() may hung. - if (!lazy_format->isFinished()) - lazy_format->finish(); - - /// Join thread here to wait for possible exception. - if (data && data->thread.joinable()) - data->thread.join(); + wait(); } catch (...) { tryLogCurrentException("PullingPipelineExecutor"); } - - if (data && data->has_exception) - tryLogException(std::move(data->exception), "PullingPipelineExecutor"); } static void threadFunction(PullingPipelineExecutor::Data & data, ThreadGroupStatusPtr thread_group, size_t num_threads) @@ -146,6 +142,25 @@ void PullingPipelineExecutor::cancel() data->executor->cancel(); } +void PullingPipelineExecutor::wait() +{ + /// Cancel execution if it wasn't finished. + if (data && !data->is_executed) + cancel(); + + /// Finish lazy format. Otherwise thread.join() may hung. + if (!lazy_format->isFinished()) + lazy_format->finish(); + + /// Join thread here to wait for possible exception. + if (data && data->thread.joinable()) + data->thread.join(); + + /// Rethrow exception to not swallow it in destructor. + if (data) + data->rethrowExceptionIfHas(); +} + Chunk PullingPipelineExecutor::getTotals() { return lazy_format->getTotals(); diff --git a/src/Processors/Executors/PullingPipelineExecutor.h b/src/Processors/Executors/PullingPipelineExecutor.h index f3b06fc618a..dc2958474d7 100644 --- a/src/Processors/Executors/PullingPipelineExecutor.h +++ b/src/Processors/Executors/PullingPipelineExecutor.h @@ -30,6 +30,8 @@ public: /// Stop execution. It is not necessary, but helps to stop execution before executor is destroyed. void cancel(); + /// Wait query thread and throw exception if has. If not called, possible exception may be swallowed in destructor. + void wait(); /// Get totals and extremes. Returns empty chunk if doesn't have any. Chunk getTotals(); From 83a35a73908a8ca77f2d2524561cc2ad50f3bb55 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 15 May 2020 20:41:12 +0300 Subject: [PATCH 265/738] try fix tests. --- programs/server/TCPHandler.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/programs/server/TCPHandler.cpp b/programs/server/TCPHandler.cpp index 93fa78499a1..c223dbb8a4e 100644 --- a/programs/server/TCPHandler.cpp +++ b/programs/server/TCPHandler.cpp @@ -589,6 +589,9 @@ void TCPHandler::processOrdinaryQueryWithProcessors() } } + /// Rethrow exception if have. + executor.wait(); + /** If data has run out, we will send the profiling data and total values to * the last zero block to be able to use * this information in the suffix output of stream. From 3105be4fea3bcd4943a4eeafad6e3d064544d7d5 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Fri, 15 May 2020 20:44:39 +0300 Subject: [PATCH 266/738] [docs] add info on accessing the playground via CLI (#10944) * [docs] add info on accessing the playground via CLI ...and multiple available versions * Update docs/en/getting-started/playground.md Co-authored-by: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Co-authored-by: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> --- docs/en/getting-started/playground.md | 70 +++++++++++++++++---------- 1 file changed, 44 insertions(+), 26 deletions(-) diff --git a/docs/en/getting-started/playground.md b/docs/en/getting-started/playground.md index 353724b6e3e..36595d1998a 100644 --- a/docs/en/getting-started/playground.md +++ b/docs/en/getting-started/playground.md @@ -5,8 +5,31 @@ toc_title: Playground # ClickHouse Playground {#clickhouse-playground} -[ClickHouse Playground](https://play.clickhouse.tech?file=welcome) allows people to experiment with ClickHouse by running queries instantly, without setting up their server or cluster. -Several example datasets are available in the Playground as well as sample queries that show ClickHouse features. +[ClickHouse Playground](https://play.clickhouse.tech) allows people to experiment with ClickHouse by running queries instantly, without setting up their server or cluster. +Several example datasets are available in the Playground as well as sample queries that show ClickHouse features. There's also a selection of ClickHouse LTS releases to experiment with. + +ClickHouse Playground gives the experience of m2.small [Managed Service for ClickHouse](https://cloud.yandex.com/services/managed-clickhouse) instance hosted in [Yandex.Cloud](https://cloud.yandex.com/). More information about [cloud providers](../commercial/cloud.md). + +You can make queries to playground using any HTTP client, for example [curl](https://curl.haxx.se) or [wget](https://www.gnu.org/software/wget/), or set up a connection using [JDBC](../interfaces/jdbc.md) or [ODBC](../interfaces/odbc.md) drivers. More information about software products that support ClickHouse is available [here](../interfaces/index.md). + +## Credentials + +| Parameter | Value | +|:------------------|:----------------------------------------| +| HTTPS endpoint | `https://play-api.clickhouse.tech:8443` | +| Native endpoint | `play-api.clickhouse.tech:9440` | +| User | `playground` | +| Password | `clickhouse` | + +!!! note "Note" + Note that all endpoints require a secure TLS connection. + +There are additional endpoints with specific ClickHouse releases to experiment with their differences (ports and user/password are the same as above): + +* 20.3 LTS: `play-api-v20-3.clickhouse.tech` +* 19.14 LTS: `play-api-v19-14.clickhouse.tech` + +## Limitations The queries are executed as a read-only user. It implies some limitations: @@ -14,33 +37,28 @@ The queries are executed as a read-only user. It implies some limitations: - INSERT queries are not allowed The following settings are also enforced: -- [`max_result_bytes=10485760`](../operations/settings/query_complexity/#max-result-bytes) -- [`max_result_rows=2000`](../operations/settings/query_complexity/#setting-max_result_rows) -- [`result_overflow_mode=break`](../operations/settings/query_complexity/#result-overflow-mode) -- [`max_execution_time=60000`](../operations/settings/query_complexity/#max-execution-time) +- [max_result_bytes=10485760](../operations/settings/query_complexity/#max-result-bytes) +- [max_result_rows=2000](../operations/settings/query_complexity/#setting-max_result_rows) +- [result_overflow_mode=break](../operations/settings/query_complexity/#result-overflow-mode) +- [max_execution_time=60000](../operations/settings/query_complexity/#max-execution-time) -ClickHouse Playground gives the experience of m2.small -[Managed Service for ClickHouse](https://cloud.yandex.com/services/managed-clickhouse) -instance hosted in [Yandex.Cloud](https://cloud.yandex.com/). -More information about [cloud providers](../commercial/cloud.md). +## Examples -ClickHouse Playground web interface makes requests via ClickHouse [HTTP API](../interfaces/http.md). -The Playground backend is just a ClickHouse cluster without any additional server-side application. -ClickHouse HTTPS endpoint is also available as a part of the Playground. - -You can make queries to playground using any HTTP client, for example [curl](https://curl.haxx.se) or [wget](https://www.gnu.org/software/wget/), or set up a connection using [JDBC](../interfaces/jdbc.md) or [ODBC](../interfaces/odbc.md) drivers. -More information about software products that support ClickHouse is available [here](../interfaces/index.md). - -| Parameter | Value | -|:----------|:--------------------------------------| -| Endpoint | https://play-api.clickhouse.tech:8443 | -| User | `playground` | -| Password | `clickhouse` | - -Note that this endpoint requires a secure connection. - -Example: +HTTPS endpoint example with `curl`: ``` bash curl "https://play-api.clickhouse.tech:8443/?query=SELECT+'Play+ClickHouse!';&user=playground&password=clickhouse&database=datasets" ``` + +TCP endpoint example with [../interfaces/cli.md]: +``` bash +clickhouse client --secure -h play-api.clickhouse.tech --port 9440 -u playground --password clickhouse -q "SELECT 'Play ClickHouse!'" +``` + +## Implementation Details + +ClickHouse Playground web interface makes requests via ClickHouse [HTTP API](../interfaces/http.md). +The Playground backend is just a ClickHouse cluster without any additional server-side application. As mentioned above, ClickHouse HTTPS and TCP/TLS endpoints are also publicly available as a part of the Playground, both are proxied through [Cloudflare Spectrum](https://www.cloudflare.com/products/cloudflare-spectrum/) to add extra layer of protection and improved global connectivity. + +!!! warning "Warning" + Exposing ClickHouse server to public internet in any other situation is **strongly not recommended**. Make sure it listens only on private network and is covered by properly configured firewall. From 6aea9c9a21ca88333e752e78a6c83a74f4a4eef0 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 15 May 2020 21:55:30 +0300 Subject: [PATCH 267/738] Tru fix build. --- src/Functions/concat.cpp | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/Functions/concat.cpp b/src/Functions/concat.cpp index c883a56aafd..56152d2530f 100644 --- a/src/Functions/concat.cpp +++ b/src/Functions/concat.cpp @@ -116,16 +116,17 @@ private: void executeFormatImpl(Block & block, const ColumnNumbers & arguments, const size_t result, size_t input_rows_count) { - assert(arguments.size() > 2); + const size_t num_arguments = arguments.size(); + assert(num_arguments > 2); auto c_res = ColumnString::create(); - std::vector data(arguments.size()); - std::vector offsets(arguments.size()); - std::vector fixed_string_sizes(arguments.size()); - std::vector constant_strings(arguments.size()); + std::vector data(num_arguments); + std::vector offsets(num_arguments); + std::vector fixed_string_sizes(num_arguments); + std::vector constant_strings(num_arguments); bool has_column_string = false; bool has_column_fixed_string = false; - for (size_t i = 0; i < arguments.size(); ++i) + for (size_t i = 0; i < num_arguments; ++i) { const ColumnPtr & column = block.getByPosition(arguments[i]).column; if (const ColumnString * col = checkAndGetColumn(column.get())) @@ -150,9 +151,9 @@ private: } String pattern; - pattern.reserve(2 * arguments.size()); + pattern.reserve(2 * num_arguments); - for (size_t i = 0; i < arguments.size(); ++i) + for (size_t i = 0; i < num_arguments; ++i) pattern += "{}"; FormatImpl::formatExecute( From 485d37b0435f6f86816e656c9f8190cdd88a0f50 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 15 May 2020 22:09:32 +0300 Subject: [PATCH 268/738] Fix incompatibility of two-level aggregation between 19.16 and 20.1 --- src/Core/Defines.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/Defines.h b/src/Core/Defines.h index 5552de3b045..13070c565b4 100644 --- a/src/Core/Defines.h +++ b/src/Core/Defines.h @@ -58,7 +58,7 @@ /// Minimum revision with exactly the same set of aggregation methods and rules to select them. /// Two-level (bucketed) aggregation is incompatible if servers are inconsistent in these rules /// (keys will be placed in different buckets and result will not be fully aggregated). -#define DBMS_MIN_REVISION_WITH_CURRENT_AGGREGATION_VARIANT_SELECTION_METHOD 54408 +#define DBMS_MIN_REVISION_WITH_CURRENT_AGGREGATION_VARIANT_SELECTION_METHOD 54431 #define DBMS_MIN_REVISION_WITH_COLUMN_DEFAULTS_METADATA 54410 #define DBMS_MIN_REVISION_WITH_LOW_CARDINALITY_TYPE 54405 From 5e7a35bbae6e8025ee54d5d7c20a77e069366fa1 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 15 May 2020 22:19:20 +0300 Subject: [PATCH 269/738] Try simplify code. --- .../Executors/PullingPipelineExecutor.cpp | 16 ++++++---------- .../Executors/PullingPipelineExecutor.h | 2 -- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/src/Processors/Executors/PullingPipelineExecutor.cpp b/src/Processors/Executors/PullingPipelineExecutor.cpp index eb51ac363f4..0528e5362bf 100644 --- a/src/Processors/Executors/PullingPipelineExecutor.cpp +++ b/src/Processors/Executors/PullingPipelineExecutor.cpp @@ -73,8 +73,6 @@ static void threadFunction(PullingPipelineExecutor::Data & data, ThreadGroupStat data.exception = std::current_exception(); data.has_exception = true; } - - data.is_executed = true; } @@ -102,7 +100,11 @@ bool PullingPipelineExecutor::pull(Chunk & chunk, uint64_t milliseconds) } if (lazy_format->isFinished()) + { + data->is_executed = true; + cancel(); return false; + } chunk = lazy_format->getChunk(milliseconds); return true; @@ -137,16 +139,10 @@ bool PullingPipelineExecutor::pull(Block & block, uint64_t milliseconds) } void PullingPipelineExecutor::cancel() -{ - if (data && data->executor) - data->executor->cancel(); -} - -void PullingPipelineExecutor::wait() { /// Cancel execution if it wasn't finished. - if (data && !data->is_executed) - cancel(); + if (data && !data->is_executed && data->executor) + data->executor->cancel(); /// Finish lazy format. Otherwise thread.join() may hung. if (!lazy_format->isFinished()) diff --git a/src/Processors/Executors/PullingPipelineExecutor.h b/src/Processors/Executors/PullingPipelineExecutor.h index dc2958474d7..f3b06fc618a 100644 --- a/src/Processors/Executors/PullingPipelineExecutor.h +++ b/src/Processors/Executors/PullingPipelineExecutor.h @@ -30,8 +30,6 @@ public: /// Stop execution. It is not necessary, but helps to stop execution before executor is destroyed. void cancel(); - /// Wait query thread and throw exception if has. If not called, possible exception may be swallowed in destructor. - void wait(); /// Get totals and extremes. Returns empty chunk if doesn't have any. Chunk getTotals(); From 6ac93e9a616cb055966de2da9baf7fe9d1eb5eb9 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 15 May 2020 22:20:02 +0300 Subject: [PATCH 270/738] Try simplify code. --- programs/server/TCPHandler.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/programs/server/TCPHandler.cpp b/programs/server/TCPHandler.cpp index c223dbb8a4e..93fa78499a1 100644 --- a/programs/server/TCPHandler.cpp +++ b/programs/server/TCPHandler.cpp @@ -589,9 +589,6 @@ void TCPHandler::processOrdinaryQueryWithProcessors() } } - /// Rethrow exception if have. - executor.wait(); - /** If data has run out, we will send the profiling data and total values to * the last zero block to be able to use * this information in the suffix output of stream. From 7d6ac4a87b319da0f693c3dd2680a35384356afd Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 15 May 2020 22:21:30 +0300 Subject: [PATCH 271/738] Added test. --- .../test_short_strings_aggregation.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 tests/integration/test_backward_compatability/test_short_strings_aggregation.py diff --git a/tests/integration/test_backward_compatability/test_short_strings_aggregation.py b/tests/integration/test_backward_compatability/test_short_strings_aggregation.py new file mode 100644 index 00000000000..1c264d1e636 --- /dev/null +++ b/tests/integration/test_backward_compatability/test_short_strings_aggregation.py @@ -0,0 +1,28 @@ +import pytest + +import helpers.client as client +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node1 = cluster.add_instance('node1', with_zookeeper=False, image='yandex/clickhouse-server:19.16.9.37', stay_alive=True, with_installed_binary=True) +node2 = cluster.add_instance('node2', with_zookeeper=False, image='yandex/clickhouse-server:19.16.9.37', stay_alive=True, with_installed_binary=True) +node3 = cluster.add_instance('node3', with_zookeeper=False) + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +def test_backward_compatability(start_cluster): + node1.query("create table tab (s String) engine = MergeTree order by s") + node2.query("create table tab (s String) engine = MergeTree order by s") + node1.query("insert into tab select number from numbers(50)") + node2.query("insert into tab select number from numbers(1000000)") + res = node3.query("select s, count() from remote('node{1,2}', default, tab) group by s order by toUInt64(s) limit 50") + print(res) + assert res == ''.join('{}\t2\n'.format(i) for i in range(50)) From fb38c2a30b6ec1c9762ec7c549cfc46fe69b758e Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 15 May 2020 22:28:10 +0300 Subject: [PATCH 272/738] Remove experimental_use_processors setting, part 1. (#10924) Remove experimental_use_processors setting, part 1. --- src/Core/Settings.h | 2 -- .../ClickHouseDictionarySource.cpp | 12 ++++----- src/Interpreters/executeQuery.cpp | 2 +- .../test_aggregation_memory_efficient/test.py | 27 +++++++++---------- ...4_performance_introspection_and_logging.sh | 2 +- tests/queries/0_stateless/00720_with_cube.sql | 2 -- .../0_stateless/00963_achimbab.reference | 8 +++--- tests/queries/0_stateless/00963_achimbab.sql | 3 +-- ...e_read_in_order_with_in_subquery.reference | 2 -- ...ptimize_read_in_order_with_in_subquery.sql | 6 +---- .../01104_distributed_numbers_test.sql | 2 -- tests/queries/0_stateless/01232_extremes.sql | 1 - 12 files changed, 25 insertions(+), 44 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index ff151e24a99..bcacd199a8d 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -385,8 +385,6 @@ struct Settings : public SettingsCollection M(SettingBool, cancel_http_readonly_queries_on_client_close, false, "Cancel HTTP readonly queries when a client closes the connection without waiting for response.", 0) \ M(SettingBool, external_table_functions_use_nulls, true, "If it is set to true, external table functions will implicitly use Nullable type if needed. Otherwise NULLs will be substituted with default values. Currently supported only by 'mysql' and 'odbc' table functions.", 0) \ \ - M(SettingBool, experimental_use_processors, true, "Use processors pipeline.", 0) \ - \ M(SettingBool, allow_hyperscan, true, "Allow functions that use Hyperscan library. Disable to avoid potentially long compilation times and excessive resource usage.", 0) \ M(SettingBool, allow_simdjson, true, "Allow using simdjson library in 'JSON*' functions if AVX2 instructions are available. If disabled rapidjson will be used.", 0) \ M(SettingBool, allow_introspection_functions, false, "Allow functions for introspection of ELF and DWARF for query profiling. These functions are slow and may impose security considerations.", 0) \ diff --git a/src/Dictionaries/ClickHouseDictionarySource.cpp b/src/Dictionaries/ClickHouseDictionarySource.cpp index 98ed7985acb..8c736bc0e56 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.cpp +++ b/src/Dictionaries/ClickHouseDictionarySource.cpp @@ -77,9 +77,6 @@ ClickHouseDictionarySource::ClickHouseDictionarySource( context.setUser(user, password, Poco::Net::SocketAddress("127.0.0.1", 0), {}); context = copyContextAndApplySettings(path_to_settings, context, config); - /// Processors are not supported here yet. - context.setSetting("experimental_use_processors", false); - /// Query context is needed because some code in executeQuery function may assume it exists. /// Current example is Context::getSampleBlockCache from InterpreterSelectWithUnionQuery::getSampleBlock. context.makeQueryContext(); @@ -134,7 +131,7 @@ BlockInputStreamPtr ClickHouseDictionarySource::loadAll() */ if (is_local) { - BlockIO res = executeQuery(load_all_query, context, true); + BlockIO res = executeQuery(load_all_query, context, true, QueryProcessingStage::Complete, false, false); /// FIXME res.in may implicitly use some objects owned be res, but them will be destructed after return res.in = std::make_shared(res.in, sample_block, ConvertingBlockInputStream::MatchColumnsMode::Position); return res.in; @@ -147,7 +144,7 @@ BlockInputStreamPtr ClickHouseDictionarySource::loadUpdatedAll() std::string load_update_query = getUpdateFieldAndDate(); if (is_local) { - auto res = executeQuery(load_update_query, context, true); + auto res = executeQuery(load_update_query, context, true, QueryProcessingStage::Complete, false, false); res.in = std::make_shared(res.in, sample_block, ConvertingBlockInputStream::MatchColumnsMode::Position); return res.in; } @@ -194,7 +191,7 @@ BlockInputStreamPtr ClickHouseDictionarySource::createStreamForSelectiveLoad(con { if (is_local) { - auto res = executeQuery(query, context, true); + auto res = executeQuery(query, context, true, QueryProcessingStage::Complete, false, false); res.in = std::make_shared( res.in, sample_block, ConvertingBlockInputStream::MatchColumnsMode::Position); return res.in; @@ -209,7 +206,8 @@ std::string ClickHouseDictionarySource::doInvalidateQuery(const std::string & re if (is_local) { Context query_context = context; - auto input_block = executeQuery(request, query_context, true).in; + auto input_block = executeQuery(request, query_context, true, + QueryProcessingStage::Complete, false, false).in; return readInvalidateQuery(*input_block); } else diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 62ef4152a2e..4d609395c3a 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -317,7 +317,7 @@ static std::tuple executeQueryImpl( context.resetInputCallbacks(); auto interpreter = InterpreterFactory::get(ast, context, stage); - bool use_processors = settings.experimental_use_processors && allow_processors && interpreter->canExecuteWithProcessors(); + bool use_processors = allow_processors && interpreter->canExecuteWithProcessors(); std::shared_ptr quota; if (!interpreter->ignoreQuota()) diff --git a/tests/integration/test_aggregation_memory_efficient/test.py b/tests/integration/test_aggregation_memory_efficient/test.py index a4e8e2b6295..3a7ada5f02e 100644 --- a/tests/integration/test_aggregation_memory_efficient/test.py +++ b/tests/integration/test_aggregation_memory_efficient/test.py @@ -29,21 +29,18 @@ def start_cluster(): def test_remote(start_cluster): - for flag in (0, 1): - node1.query("set experimental_use_processors = {}".format(flag)) + node1.query("set distributed_aggregation_memory_efficient = 1, group_by_two_level_threshold = 1, group_by_two_level_threshold_bytes=1") + res = node1.query("select sum(a) from (SELECT B, uniqExact(A) a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY B)") + assert res == '200000\n' - node1.query("set distributed_aggregation_memory_efficient = 1, group_by_two_level_threshold = 1, group_by_two_level_threshold_bytes=1") - res = node1.query("select sum(a) from (SELECT B, uniqExact(A) a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY B)") - assert res == '200000\n' + node1.query("set distributed_aggregation_memory_efficient = 0") + res = node1.query("select sum(a) from (SELECT B, uniqExact(A) a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY B)") + assert res == '200000\n' - node1.query("set distributed_aggregation_memory_efficient = 0") - res = node1.query("select sum(a) from (SELECT B, uniqExact(A) a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY B)") - assert res == '200000\n' + node1.query("set distributed_aggregation_memory_efficient = 1, group_by_two_level_threshold = 1, group_by_two_level_threshold_bytes=1") + res = node1.query("SELECT fullHostName() AS h, uniqExact(A) AS a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY h ORDER BY h;") + assert res == 'node1\t100000\nnode2\t100000\n' - node1.query("set distributed_aggregation_memory_efficient = 1, group_by_two_level_threshold = 1, group_by_two_level_threshold_bytes=1") - res = node1.query("SELECT fullHostName() AS h, uniqExact(A) AS a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY h ORDER BY h;") - assert res == 'node1\t100000\nnode2\t100000\n' - - node1.query("set distributed_aggregation_memory_efficient = 0") - res = node1.query("SELECT fullHostName() AS h, uniqExact(A) AS a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY h ORDER BY h;") - assert res == 'node1\t100000\nnode2\t100000\n' + node1.query("set distributed_aggregation_memory_efficient = 0") + res = node1.query("SELECT fullHostName() AS h, uniqExact(A) AS a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY h ORDER BY h;") + assert res == 'node1\t100000\nnode2\t100000\n' diff --git a/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh b/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh index 21014b56b67..5173b5f5772 100755 --- a/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh +++ b/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh @@ -13,7 +13,7 @@ server_logs_file=${CLICKHOUSE_TMP}/$cur_name"_server.logs" server_logs="--server_logs_file=$server_logs_file" rm -f "$server_logs_file" -settings="$server_logs --log_queries=1 --log_query_threads=1 --log_profile_events=1 --log_query_settings=1 --experimental_use_processors=0" +settings="$server_logs --log_queries=1 --log_query_threads=1 --log_profile_events=1 --log_query_settings=1" # Test insert logging on each block and checkPacket() method diff --git a/tests/queries/0_stateless/00720_with_cube.sql b/tests/queries/0_stateless/00720_with_cube.sql index 42b65c8222c..d236e9da34b 100644 --- a/tests/queries/0_stateless/00720_with_cube.sql +++ b/tests/queries/0_stateless/00720_with_cube.sql @@ -1,8 +1,6 @@ DROP TABLE IF EXISTS cube; CREATE TABLE cube(a String, b Int32, s Int32) ENGINE = Memory; --- SET experimental_use_processors=1; - INSERT INTO cube VALUES ('a', 1, 10), ('a', 1, 15), ('a', 2, 20); INSERT INTO cube VALUES ('a', 2, 25), ('b', 1, 10), ('b', 1, 5); INSERT INTO cube VALUES ('b', 2, 20), ('b', 2, 15); diff --git a/tests/queries/0_stateless/00963_achimbab.reference b/tests/queries/0_stateless/00963_achimbab.reference index 5248223efdd..a5f501d1a6a 100644 --- a/tests/queries/0_stateless/00963_achimbab.reference +++ b/tests/queries/0_stateless/00963_achimbab.reference @@ -20,7 +20,7 @@ { "total": 1, "arrayElement(k, 1)": null, - "arrayElement(k, 2)": 4 + "arrayElement(k, 2)": 1 }, { "total": 1, @@ -30,12 +30,12 @@ { "total": 1, "arrayElement(k, 1)": null, - "arrayElement(k, 2)": 1 + "arrayElement(k, 2)": 3 }, { "total": 1, "arrayElement(k, 1)": null, - "arrayElement(k, 2)": 3 + "arrayElement(k, 2)": 4 }, { "total": 1, @@ -53,5 +53,5 @@ "rows": 5, - "rows_before_limit_at_least": 5 + "rows_before_limit_at_least": 1000000 } diff --git a/tests/queries/0_stateless/00963_achimbab.sql b/tests/queries/0_stateless/00963_achimbab.sql index 3f66cfbbf31..758ecf5acf3 100644 --- a/tests/queries/0_stateless/00963_achimbab.sql +++ b/tests/queries/0_stateless/00963_achimbab.sql @@ -1,5 +1,4 @@ SET output_format_write_statistics = 0; -SET experimental_use_processors = 0; select sum(cnt) > 0 as total, @@ -14,6 +13,6 @@ select limit 1000000 ) group by k with totals -order by total desc +order by k[2] SETTINGS max_threads = 100, max_execution_time = 120 format JSON; diff --git a/tests/queries/0_stateless/01018_optimize_read_in_order_with_in_subquery.reference b/tests/queries/0_stateless/01018_optimize_read_in_order_with_in_subquery.reference index d80fc78e03d..b261da18d51 100644 --- a/tests/queries/0_stateless/01018_optimize_read_in_order_with_in_subquery.reference +++ b/tests/queries/0_stateless/01018_optimize_read_in_order_with_in_subquery.reference @@ -1,4 +1,2 @@ 1 0 -1 -0 diff --git a/tests/queries/0_stateless/01018_optimize_read_in_order_with_in_subquery.sql b/tests/queries/0_stateless/01018_optimize_read_in_order_with_in_subquery.sql index 6d2bf5f4863..dcec82461a5 100644 --- a/tests/queries/0_stateless/01018_optimize_read_in_order_with_in_subquery.sql +++ b/tests/queries/0_stateless/01018_optimize_read_in_order_with_in_subquery.sql @@ -8,8 +8,4 @@ INSERT INTO TESTTABLE4 VALUES (0,'1','1'), (1,'0','1'); SELECT _id FROM TESTTABLE4 PREWHERE l IN (select '1') ORDER BY _id DESC LIMIT 10; -SET experimental_use_processors=1; - -SELECT _id FROM TESTTABLE4 PREWHERE l IN (select '1') ORDER BY _id DESC LIMIT 10; - -DROP TABLE TESTTABLE4; \ No newline at end of file +DROP TABLE TESTTABLE4; diff --git a/tests/queries/0_stateless/01104_distributed_numbers_test.sql b/tests/queries/0_stateless/01104_distributed_numbers_test.sql index 7f56a4e08fd..101342b4c34 100644 --- a/tests/queries/0_stateless/01104_distributed_numbers_test.sql +++ b/tests/queries/0_stateless/01104_distributed_numbers_test.sql @@ -1,8 +1,6 @@ DROP TABLE IF EXISTS d_numbers; CREATE TABLE d_numbers (number UInt32) ENGINE = Distributed(test_cluster_two_shards, system, numbers, rand()); -SET experimental_use_processors = 1; - SELECT '100' AS number FROM d_numbers AS n WHERE n.number = 100 LIMIT 2; SET distributed_product_mode = 'local'; diff --git a/tests/queries/0_stateless/01232_extremes.sql b/tests/queries/0_stateless/01232_extremes.sql index 9379dc1cd38..e46a6602766 100644 --- a/tests/queries/0_stateless/01232_extremes.sql +++ b/tests/queries/0_stateless/01232_extremes.sql @@ -1,6 +1,5 @@ set send_logs_level = 'error'; set extremes = 1; --- set experimental_use_processors=0; select * from remote('127.0.0.1', numbers(2)); select '-'; From 8c2d0aa7c0d1d78c86f49f81560e10f6c9376807 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 15 May 2020 22:30:41 +0300 Subject: [PATCH 273/738] Fix comment. --- src/Processors/Executors/PullingPipelineExecutor.cpp | 3 ++- src/Processors/Formats/LazyOutputFormat.h | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Processors/Executors/PullingPipelineExecutor.cpp b/src/Processors/Executors/PullingPipelineExecutor.cpp index 0528e5362bf..c34195a0793 100644 --- a/src/Processors/Executors/PullingPipelineExecutor.cpp +++ b/src/Processors/Executors/PullingPipelineExecutor.cpp @@ -44,7 +44,7 @@ PullingPipelineExecutor::~PullingPipelineExecutor() { try { - wait(); + cancel(); } catch (...) { @@ -102,6 +102,7 @@ bool PullingPipelineExecutor::pull(Chunk & chunk, uint64_t milliseconds) if (lazy_format->isFinished()) { data->is_executed = true; + /// Wait thread ant rethrow exception if any. cancel(); return false; } diff --git a/src/Processors/Formats/LazyOutputFormat.h b/src/Processors/Formats/LazyOutputFormat.h index 24283391355..14d7a7f47d7 100644 --- a/src/Processors/Formats/LazyOutputFormat.h +++ b/src/Processors/Formats/LazyOutputFormat.h @@ -8,8 +8,8 @@ namespace DB { /// LazyOutputFormat is used to retrieve ready data from executing pipeline. -/// You can periodically call `getBlock` from separate thread. -/// Used in TCPHandler. +/// You can periodically call `getChunk` from separate thread. +/// Used in PullingPipelineExecutor. class LazyOutputFormat : public IOutputFormat { From 405cce2c531adf8b9923fa739c7780f198dfc6c9 Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Fri, 15 May 2020 22:19:21 +0200 Subject: [PATCH 274/738] Another variant of fix. It looks like we do not need destructor at all --- .../Kafka/ReadBufferFromKafkaConsumer.cpp | 37 +++++-------------- .../Kafka/ReadBufferFromKafkaConsumer.h | 4 +- tests/integration/test_storage_kafka/test.py | 3 -- 3 files changed, 13 insertions(+), 31 deletions(-) diff --git a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp index 36832a07221..eff4161ffb6 100644 --- a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp +++ b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp @@ -80,31 +80,9 @@ ReadBufferFromKafkaConsumer::ReadBufferFromKafkaConsumer( }); } -ReadBufferFromKafkaConsumer::~ReadBufferFromKafkaConsumer() -{ - try - { - if (!consumer->get_subscription().empty()) - consumer->unsubscribe(); - } - catch (const cppkafka::HandleException & e) - { - LOG_ERROR(log, "Exception from ReadBufferFromKafkaConsumer destructor (unsubscribe): " << e.what()); - } - - try - { - // we need to drain rest of the messages / queued callback calls from the consumer - // after unsubscribe, otherwise consumer will hang on destruction - // see https://github.com/edenhill/librdkafka/issues/2077 - // https://github.com/confluentinc/confluent-kafka-go/issues/189 etc. - while (consumer->poll(100ms)); - } - catch (const cppkafka::HandleException & e) - { - LOG_ERROR(log, "Exception from ReadBufferFromKafkaConsumer destructor (drain): " << e.what()); - } -} +// NOTE on removed desctuctor: There is no need to unsubscribe prior to calling rd_kafka_consumer_close(). +// check: https://github.com/edenhill/librdkafka/blob/master/INTRODUCTION.md#termination +// manual destruction was source of weird errors (hangs during droping kafka table, etc.) void ReadBufferFromKafkaConsumer::commit() { @@ -235,8 +213,13 @@ void ReadBufferFromKafkaConsumer::unsubscribe() // it should not raise exception as used in destructor try { - if (!consumer->get_subscription().empty()) - consumer->unsubscribe(); + // From docs: Any previous subscription will be unassigned and unsubscribed first. + consumer->subscribe(topics); + + // I wanted to avoid explicit unsubscribe as it requires draining the messages + // to close the consumer safely after unsubscribe + // see https://github.com/edenhill/librdkafka/issues/2077 + // https://github.com/confluentinc/confluent-kafka-go/issues/189 etc. } catch (const cppkafka::HandleException & e) { diff --git a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h index 700a69cf49b..c5b72ed6d7c 100644 --- a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h +++ b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h @@ -28,7 +28,6 @@ public: const std::atomic & stopped_, const Names & _topics ); - ~ReadBufferFromKafkaConsumer() override; void allowNext() { allowed = true; } // Allow to read next message. void commit(); // Commit all processed messages. @@ -64,10 +63,13 @@ private: const std::atomic & stopped; + // order is important, need to be destructed before consumer Messages messages; Messages::const_iterator current; bool rebalance_happened = false; + + // order is important, need to be destructed before consumer cppkafka::TopicPartitionList assignment; const Names topics; diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py index cc384d27b36..f546b275190 100644 --- a/tests/integration/test_storage_kafka/test.py +++ b/tests/integration/test_storage_kafka/test.py @@ -242,9 +242,6 @@ def test_kafka_consumer_hang(kafka_cluster): # BROKERFAIL -> |ASSIGN| -> REBALANCE_IN_PROGRESS -> "waiting for rebalance_cb" (repeated forever) # so it was waiting forever while the application will execute queued rebalance callback - # now we drain all queued callbacks (visible as 'Rebalance initiated' after 'Waiting for cleanup') - instance.exec_in_container(["bash", "-c", "tail -n 500 /var/log/clickhouse-server/clickhouse-server.log | grep 'Waiting for cleanup' -A 500 | grep -q 'Rebalance initiated. Revoking partitions'"]) - # from a user perspective: we expect no hanging 'drop' queries # 'dr'||'op' to avoid self matching assert int(instance.query("select count() from system.processes where position(lower(query),'dr'||'op')>0")) == 0 From c701493e3015bceed106246cd5e092f9fa0ca2c8 Mon Sep 17 00:00:00 2001 From: BayoNet Date: Fri, 15 May 2020 23:30:51 +0300 Subject: [PATCH 275/738] DOCS-626: EN review, RU translations for RBAC docs (#10951) * DOCSUP-1062 (#112) * added first draft * minor fixes * fixed anchors * yet another fixes * and the minorest fixes * Apply suggestions from doc review Co-authored-by: BayoNet * fixed terminology in ru (access entity, throws exception) * fixed typo * fixed typo Co-authored-by: Elizaveta Mironyuk Co-authored-by: BayoNet * Fixed link. * CLICKHOUSEDOCS-626: Fixed links. Co-authored-by: Sergei Shtykov Co-authored-by: emironyuk <62014692+emironyuk@users.noreply.github.com> Co-authored-by: Elizaveta Mironyuk --- docs/en/operations/access-rights.md | 44 +- .../operations/settings/settings-profiles.md | 2 +- docs/en/operations/settings/settings-users.md | 2 +- docs/en/sql-reference/statements/alter.md | 12 +- docs/en/sql-reference/statements/create.md | 30 +- docs/en/sql-reference/statements/grant.md | 66 +-- docs/en/sql-reference/statements/misc.md | 8 +- docs/en/sql-reference/statements/revoke.md | 6 +- docs/en/sql-reference/statements/show.md | 8 +- docs/ru/operations/access-rights.md | 190 ++++--- .../settings.md | 10 + .../operations/settings/settings-profiles.md | 19 +- docs/ru/operations/settings/settings-users.md | 16 + docs/ru/sql-reference/statements/alter.md | 106 +++- docs/ru/sql-reference/statements/create.md | 202 ++++++++ docs/ru/sql-reference/statements/grant.md | 480 +++++++++++++++++- docs/ru/sql-reference/statements/misc.md | 123 ++++- docs/ru/sql-reference/statements/revoke.md | 44 +- .../statements/select/prewhere.md | 2 +- .../sql-reference/statements/select/where.md | 4 +- docs/ru/sql-reference/statements/show.md | 74 +++ .../ru/sql-reference/table-functions/index.md | 2 +- 22 files changed, 1269 insertions(+), 181 deletions(-) mode change 120000 => 100644 docs/ru/sql-reference/statements/grant.md mode change 120000 => 100644 docs/ru/sql-reference/statements/revoke.md diff --git a/docs/en/operations/access-rights.md b/docs/en/operations/access-rights.md index aac2befab4c..001afd29fcb 100644 --- a/docs/en/operations/access-rights.md +++ b/docs/en/operations/access-rights.md @@ -22,7 +22,7 @@ You can configure access entities using: - Server [configuration files](configuration-files.md) `users.xml` and `config.xml`. -We recommend using SQL-driven workflow. Both of the configuration methods work simultaneously, so if you use the server configuration files for managing accounts and access rights, you can softly move to SQL-driven workflow. +We recommend using SQL-driven workflow. Both of the configuration methods work simultaneously, so if you use the server configuration files for managing accounts and access rights, you can smoothly switch to SQL-driven workflow. !!! note "Warning" You can't manage the same access entity by both configuration methods simultaneously. @@ -30,32 +30,32 @@ We recommend using SQL-driven workflow. Both of the configuration methods work s ## Usage {#access-control-usage} -By default, the ClickHouse server provides the user account `default` which is not allowed using SQL-driven access control and account management but have all the rights and permissions. The `default` user account is used in any cases when the username is not defined, for example, at login from client or in distributed queries. In distributed query processing a default user account is used, if the configuration of the server or cluster doesn’t specify the [user and password](../engines/table-engines/special/distributed.md) properties. +By default, the ClickHouse server provides the `default` user account which is not allowed using SQL-driven access control and account management but has all the rights and permissions. The `default` user account is used in any cases when the username is not defined, for example, at login from client or in distributed queries. In distributed query processing a default user account is used, if the configuration of the server or cluster doesn’t specify the [user and password](../engines/table-engines/special/distributed.md) properties. -If you just start using ClickHouse, you can use the following scenario: +If you just started using ClickHouse, consider the following scenario: 1. [Enable](#enabling-access-control) SQL-driven access control and account management for the `default` user. -2. Login under the `default` user account and create all the required users. Don't forget to create an administrator account (`GRANT ALL ON *.* WITH GRANT OPTION TO admin_user_account`). +2. Log in to the `default` user account and create all the required users. Don't forget to create an administrator account (`GRANT ALL ON *.* WITH GRANT OPTION TO admin_user_account`). 3. [Restrict permissions](settings/permissions-for-queries.md#permissions_for_queries) for the `default` user and disable SQL-driven access control and account management for it. ### Properties of Current Solution {#access-control-properties} -- You can grant permissions for databases and tables even if they are not exist. -- If a table was deleted, all the privileges that correspond to this table are not revoked. So, if a new table is created later with the same name all the privileges become again actual. To revoke privileges corresponding to the deleted table, you need to perform, for example, the `REVOKE ALL PRIVILEGES ON db.table FROM ALL` query. -- There is no lifetime settings for privileges. +- You can grant permissions for databases and tables even if they do not exist. +- If a table was deleted, all the privileges that correspond to this table are not revoked. This means that even if you create a new table with the same name later, all the privileges remain valid. To revoke privileges corresponding to the deleted table, you need to execute, for example, the `REVOKE ALL PRIVILEGES ON db.table FROM ALL` query. +- There are no lifetime settings for privileges. ## User account {#user-account-management} A user account is an access entity that allows to authorize someone in ClickHouse. A user account contains: - Identification information. -- [Privileges](../sql-reference/statements/grant.md#grant-privileges) that define a scope of queries the user can perform. -- Hosts from which connection to the ClickHouse server is allowed. -- Granted and default roles. -- Settings with their constraints that apply by default at the user's login. +- [Privileges](../sql-reference/statements/grant.md#grant-privileges) that define a scope of queries the user can execute. +- Hosts allowed to connect to the ClickHouse server. +- Assigned and default roles. +- Settings with their constraints applied by default at user login. - Assigned settings profiles. -Privileges to a user account can be granted by the [GRANT](../sql-reference/statements/grant.md) query or by assigning [roles](#role-management). To revoke privileges from a user, ClickHouse provides the [REVOKE](../sql-reference/statements/revoke.md) query. To list privileges for a user, use the - [SHOW GRANTS](../sql-reference/statements/show.md#show-grants-statement) statement. +Privileges can be granted to a user account by the [GRANT](../sql-reference/statements/grant.md) query or by assigning [roles](#role-management). To revoke privileges from a user, ClickHouse provides the [REVOKE](../sql-reference/statements/revoke.md) query. To list privileges for a user, use the [SHOW GRANTS](../sql-reference/statements/show.md#show-grants-statement) statement. Management queries: @@ -66,11 +66,11 @@ Management queries: ### Settings Applying {#access-control-settings-applying} -Settings can be set by different ways: for a user account, in its granted roles and settings profiles. At a user login, if a setting is set in different access entities, the value and constrains of this setting are applied by the following priorities (from higher to lower): +Settings can be configured differently: for a user account, in its granted roles and in settings profiles. At user login, if a setting is configured for different access entities, the value and constraints of this setting are applied as follows (from higher to lower priority): -1. User account setting. -2. The settings of default roles of the user account. If a setting is set in some roles, then order of the setting applying is undefined. -3. The settings in settings profiles assigned to a user or to its default roles. If a setting is set in some profiles, then order of setting applying is undefined. +1. User account settings. +2. The settings of default roles of the user account. If a setting is configured in some roles, then order of the setting application is undefined. +3. The settings from settings profiles assigned to a user or to its default roles. If a setting is configured in some profiles, then order of setting application is undefined. 4. Settings applied to all the server by default or from the [default profile](server-configuration-parameters/settings.md#default-profile). @@ -82,7 +82,7 @@ Role contains: - [Privileges](../sql-reference/statements/grant.md#grant-privileges) - Settings and constraints -- List of granted roles +- List of assigned roles Management queries: @@ -93,11 +93,11 @@ Management queries: - [SET DEFAULT ROLE](../sql-reference/statements/misc.md#set-default-role-statement) - [SHOW CREATE ROLE](../sql-reference/statements/show.md#show-create-role-statement) -Privileges to a role can be granted by the [GRANT](../sql-reference/statements/grant.md) query. To revoke privileges from a role ClickHouse provides the [REVOKE](../sql-reference/statements/revoke.md) query. +Privileges can be granted to a role by the [GRANT](../sql-reference/statements/grant.md) query. To revoke privileges from a role ClickHouse provides the [REVOKE](../sql-reference/statements/revoke.md) query. ## Row Policy {#row-policy-management} -Row policy is a filter that defines which or rows is available for a user or for a role. Row policy contains filters for one specific table and list of roles and/or users which should use this row policy. +Row policy is a filter that defines which of the rows are available to a user or a role. Row policy contains filters for one particular table, as well as a list of roles and/or users which should use this row policy. Management queries: @@ -109,7 +109,7 @@ Management queries: ## Settings Profile {#settings-profiles-management} -Settings profile is a collection of [settings](settings/index.md). Settings profile contains settings and constraints, and list of roles and/or users to which this quota is applied. +Settings profile is a collection of [settings](settings/index.md). Settings profile contains settings and constraints, as well as a list of roles and/or users to which this profile is applied. Management queries: @@ -123,7 +123,7 @@ Management queries: Quota limits resource usage. See [Quotas](quotas.md). -Quota contains a set of limits for some durations, and list of roles and/or users which should use this quota. +Quota contains a set of limits for some durations, as well as a list of roles and/or users which should use this quota. Management queries: @@ -141,7 +141,7 @@ Management queries: - Enable SQL-driven access control and account management for at least one user account. - By default, SQL-driven access control and account management is turned off for all users. You need to configure at least one user in the `users.xml` configuration file and assign 1 to the [access_management](settings/settings-users.md#access_management-user-setting) setting. + By default, SQL-driven access control and account management is disabled for all users. You need to configure at least one user in the `users.xml` configuration file and set the value of the [access_management](settings/settings-users.md#access_management-user-setting) setting to 1. [Original article](https://clickhouse.tech/docs/en/operations/access_rights/) diff --git a/docs/en/operations/settings/settings-profiles.md b/docs/en/operations/settings/settings-profiles.md index 0543f0bd954..3e5d1a02cbd 100644 --- a/docs/en/operations/settings/settings-profiles.md +++ b/docs/en/operations/settings/settings-profiles.md @@ -11,7 +11,7 @@ A settings profile is a collection of settings grouped under the same name. ClickHouse also supports [SQL-driven workflow](../access-rights.md#access-control) for managing settings profiles. We recommend using it. -A profile can have any name. The profile can have any name. You can specify the same profile for different users. The most important thing you can write in the settings profile is `readonly=1`, which ensures read-only access. +The profile can have any name. You can specify the same profile for different users. The most important thing you can write in the settings profile is `readonly=1`, which ensures read-only access. Settings profiles can inherit from each other. To use inheritance, indicate one or multiple `profile` settings before the other settings that are listed in the profile. In case when one setting is defined in different profiles, the latest defined is used. diff --git a/docs/en/operations/settings/settings-users.md b/docs/en/operations/settings/settings-users.md index 3f472ad4879..3c104202801 100644 --- a/docs/en/operations/settings/settings-users.md +++ b/docs/en/operations/settings/settings-users.md @@ -76,7 +76,7 @@ Password can be specified in plaintext or in SHA256 (hex format). ### access_management {#access_management-user-setting} -This setting enables of disables using of SQL-driven [access control and account management](../access-rights.md#access-control) for the user. +This setting enables or disables using of SQL-driven [access control and account management](../access-rights.md#access-control) for the user. Possible values: diff --git a/docs/en/sql-reference/statements/alter.md b/docs/en/sql-reference/statements/alter.md index eb0c42ad9c3..6b1adcdb033 100644 --- a/docs/en/sql-reference/statements/alter.md +++ b/docs/en/sql-reference/statements/alter.md @@ -520,23 +520,23 @@ To use `ALTER USER` you must have the [ALTER USER](grant.md#grant-access-managem ### Examples {#alter-user-examples} -Set granted roles as default: +Set assigned roles as default: ``` sql ALTER USER user DEFAULT ROLE role1, role2 ``` -If roles aren't previously granted to a user, ClickHouse throws an exception. +If roles aren't previously assigned to a user, ClickHouse throws an exception. -Set all the granted roles to default: +Set all the assigned roles to default: ``` sql ALTER USER user DEFAULT ROLE ALL ``` -If a role will be granted to a user in the future it will become default automatically. +If a role is assigned to a user in the future, it will become default automatically. -Set all the granted roles to default excepting `role1` and `role2`: +Set all the assigned roles to default, excepting `role1` and `role2`: ``` sql ALTER USER user DEFAULT ROLE ALL EXCEPT role1, role2 @@ -591,7 +591,7 @@ ALTER QUOTA [IF EXISTS] name [ON CLUSTER cluster_name] ## ALTER SETTINGS PROFILE {#alter-settings-profile-statement} -Changes quotas. +Changes settings profiles. ### Syntax {#alter-settings-profile-syntax} diff --git a/docs/en/sql-reference/statements/create.md b/docs/en/sql-reference/statements/create.md index ea03983e2f9..7a7f4635ec3 100644 --- a/docs/en/sql-reference/statements/create.md +++ b/docs/en/sql-reference/statements/create.md @@ -327,23 +327,23 @@ There are multiple ways of user identification: #### User Host -User host is a host from which a connection to ClickHouse server could be established. Host can be specified in the `HOST` section of query by the following ways: +User host is a host from which a connection to ClickHouse server could be established. The host can be specified in the `HOST` query section in the following ways: - `HOST IP 'ip_address_or_subnetwork'` — User can connect to ClickHouse server only from the specified IP address or a [subnetwork](https://en.wikipedia.org/wiki/Subnetwork). Examples: `HOST IP '192.168.0.0/16'`, `HOST IP '2001:DB8::/32'`. For use in production, only specify `HOST IP` elements (IP addresses and their masks), since using `host` and `host_regexp` might cause extra latency. -- `HOST ANY` — User can connect from any location. This is default option. +- `HOST ANY` — User can connect from any location. This is a default option. - `HOST LOCAL` — User can connect only locally. - `HOST NAME 'fqdn'` — User host can be specified as FQDN. For example, `HOST NAME 'mysite.com'`. - `HOST NAME REGEXP 'regexp'` — You can use [pcre](http://www.pcre.org/) regular expressions when specifying user hosts. For example, `HOST NAME REGEXP '.*\.mysite\.com'`. -- `HOST LIKE 'template'` — Allows you use the [LIKE](../functions/string-search-functions.md#function-like) operator to filter the user hosts. For example, `HOST LIKE '%'` is equivalent to `HOST ANY`, `HOST LIKE '%.mysite.com'` filters all the hosts in the `mysite.com` domain. +- `HOST LIKE 'template'` — Allows you to use the [LIKE](../functions/string-search-functions.md#function-like) operator to filter the user hosts. For example, `HOST LIKE '%'` is equivalent to `HOST ANY`, `HOST LIKE '%.mysite.com'` filters all the hosts in the `mysite.com` domain. -Another way of specifying host is to use `@` syntax with the user name. Examples: +Another way of specifying host is to use `@` syntax following the username. Examples: - `CREATE USER mira@'127.0.0.1'` — Equivalent to the `HOST IP` syntax. - `CREATE USER mira@'localhost'` — Equivalent to the `HOST LOCAL` syntax. - `CREATE USER mira@'192.168.%.%'` — Equivalent to the `HOST LIKE` syntax. !!! info "Warning" - ClickHouse treats `user_name@'address'` as a user name as a whole. Thus, technically you can create multiple users with `user_name` and different constructions after `@`. We don't recommend to do so. + ClickHouse treats `user_name@'address'` as a username as a whole. Thus, technically you can create multiple users with the same `user_name` and different constructions after `@`. However, we don't recommend to do so. ### Examples {#create-user-examples} @@ -369,7 +369,7 @@ Create the user account `john` and make all his future roles default: ALTER USER user DEFAULT ROLE ALL ``` -When some role will be assigned to `john` in the future it will become default automatically. +When some role is assigned to `john` in the future, it will become default automatically. Create the user account `john` and make all his future roles default excepting `role1` and `role2`: @@ -391,15 +391,15 @@ CREATE ROLE [IF NOT EXISTS | OR REPLACE] name ### Description {#create-role-description} -Role is a set of [privileges](grant.md#grant-privileges). A user granted with a role gets all the privileges of this role. +Role is a set of [privileges](grant.md#grant-privileges). A user assigned a role gets all the privileges of this role. -A user can be assigned with multiple roles. Users can apply their granted roles in arbitrary combinations by the [SET ROLE](misc.md#set-role-statement) statement. The final scope of privileges is a combined set of all the privileges of all the applied roles. If a user has privileges granted directly to it's user account, they are also combined with the privileges granted by roles. +A user can be assigned multiple roles. Users can apply their assigned roles in arbitrary combinations by the [SET ROLE](misc.md#set-role-statement) statement. The final scope of privileges is a combined set of all the privileges of all the applied roles. If a user has privileges granted directly to it's user account, they are also combined with the privileges granted by roles. User can have default roles which apply at user login. To set default roles, use the [SET DEFAULT ROLE](misc.md#set-default-role-statement) statement or the [ALTER USER](alter.md#alter-user-statement) statement. To revoke a role, use the [REVOKE](revoke.md) statement. -To delete role, use the [DROP ROLE](misc.md#drop-role-statement) statement. The deleted role is being automatically revoked from all the users and roles to which it was granted. +To delete role, use the [DROP ROLE](misc.md#drop-role-statement) statement. The deleted role is being automatically revoked from all the users and roles to which it was assigned. ### Examples {#create-role-examples} @@ -410,13 +410,13 @@ GRANT SELECT ON db.* TO accountant; This sequence of queries creates the role `accountant` that has the privilege of reading data from the `accounting` database. -Granting the role to the user `mira`: +Assigning the role to the user `mira`: ```sql GRANT accountant TO mira; ``` -After the role is granted, the user can use it and perform the allowed queries. For example: +After the role is assigned, the user can apply it and execute the allowed queries. For example: ```sql SET ROLE accountant; @@ -443,15 +443,15 @@ Using this section you can create permissive or restrictive policies. Permissive policy grants access to rows. Permissive policies which apply to the same table are combined together using the boolean `OR` operator. Policies are permissive by default. -Restrictive policy restricts access to row. Restrictive policies which apply to the same table are combined together using the boolean `AND` operator. +Restrictive policy restricts access to rows. Restrictive policies which apply to the same table are combined together using the boolean `AND` operator. Restrictive policies apply to rows that passed the permissive filters. If you set restrictive policies but no permissive policies, the user can't get any row from the table. #### Section TO {#create-row-policy-to} -In the section `TO` you can give a mixed list of roles and users, for example, `CREATE ROW POLICY ... TO accountant, john@localhost`. +In the section `TO` you can provide a mixed list of roles and users, for example, `CREATE ROW POLICY ... TO accountant, john@localhost`. -Keyword `ALL` means all the ClickHouse users including current user. Keywords `ALL EXCEPT` allow to to exclude some users from the all users list, for example `CREATE ROW POLICY ... TO ALL EXCEPT accountant, john@localhost` +Keyword `ALL` means all the ClickHouse users including current user. Keywords `ALL EXCEPT` allow to exclude some users from the all users list, for example, `CREATE ROW POLICY ... TO ALL EXCEPT accountant, john@localhost` ### Examples @@ -494,7 +494,7 @@ CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] name [ON CLUSTER cluster_na [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...] ``` -# Example {#create-settings-profile-syntax} +### Example {#create-settings-profile-syntax} Create the `max_memory_usage_profile` settings profile with value and constraints for the `max_memory_usage` setting. Assign it to `robin`: diff --git a/docs/en/sql-reference/statements/grant.md b/docs/en/sql-reference/statements/grant.md index f0a1508f1e9..d9e4f2f9309 100644 --- a/docs/en/sql-reference/statements/grant.md +++ b/docs/en/sql-reference/statements/grant.md @@ -7,9 +7,9 @@ toc_title: GRANT # GRANT - Grants [privileges](#grant-privileges) to ClickHouse user accounts or roles. -- Assigns roles to user accounts or to another roles. +- Assigns roles to user accounts or to the other roles. -To revoke privileges, use the [REVOKE](revoke.md) statement. Also you can list granted privileges by the [SHOW GRANTS](show.md#show-grants-statement) statement. +To revoke privileges, use the [REVOKE](revoke.md) statement. Also you can list granted privileges with the [SHOW GRANTS](show.md#show-grants-statement) statement. ## Granting Privilege Syntax {#grant-privigele-syntax} @@ -21,10 +21,10 @@ GRANT [ON CLUSTER cluster_name] privilege[(column_name [,...])] [,...] ON {db.ta - `role` — ClickHouse user role. - `user` — ClickHouse user account. -The `WITH GRANT OPTION` clause grants `user` or `role` with permission to perform the `GRANT` query. Users can grant privileges of the same scope they have and less. +The `WITH GRANT OPTION` clause grants `user` or `role` with permission to execute the `GRANT` query. Users can grant privileges of the same scope they have and less. -## Granting Role Syntax {#assign-role-syntax} +## Assigning Role Syntax {#assign-role-syntax} ```sql GRANT [ON CLUSTER cluster_name] role [,...] TO {user | another_role | CURRENT_USER} [,...] [WITH ADMIN OPTION] @@ -33,7 +33,7 @@ GRANT [ON CLUSTER cluster_name] role [,...] TO {user | another_role | CURRENT_US - `role` — ClickHouse user role. - `user` — ClickHouse user account. -The `WITH ADMIN OPTION` clause sets [ADMIN OPTION](#admin-option-privilege) privilege for `user` or `role`. +The `WITH ADMIN OPTION` clause grants [ADMIN OPTION](#admin-option-privilege) privilege to `user` or `role`. ## Usage {#grant-usage} @@ -45,28 +45,28 @@ For example, administrator has granted privileges to the `john` account by the q GRANT SELECT(x,y) ON db.table TO john WITH GRANT OPTION ``` -It means that `john` has the permission to perform: +It means that `john` has the permission to execute: - `SELECT x,y FROM db.table`. - `SELECT x FROM db.table`. - `SELECT y FROM db.table`. -`john` can't perform `SELECT z FROM db.table`. The `SELECT * FROM db.table` also is not available. Processing this query, ClickHouse doesn't return any data, even `x` and `y`. The only exception is if a table contains only `x` and `y` columns, in this case ClickHouse returns all the data. +`john` can't execute `SELECT z FROM db.table`. The `SELECT * FROM db.table` also is not available. Processing this query, ClickHouse doesn't return any data, even `x` and `y`. The only exception is if a table contains only `x` and `y` columns. In this case ClickHouse returns all the data. -Also `john` has the `GRANT OPTION` privilege, so it can grant other users with privileges of the same or the smaller scope. +Also `john` has the `GRANT OPTION` privilege, so it can grant other users with privileges of the same or smaller scope. -Specifying privileges you can use asterisk (`*`) instead of a table or a database name. For example, the `GRANT SELECT ON db.* TO john` query allows `john` to perform the `SELECT` query over all the tables in `db` database. Also, you can omit database name. In this case privileges are granted for current database, for example: `GRANT SELECT ON * TO john` grants the privilege on all the tables in the current database, `GRANT SELECT ON mytable TO john` grants the privilege on the `mytable` table in the current database. +Specifying privileges you can use asterisk (`*`) instead of a table or a database name. For example, the `GRANT SELECT ON db.* TO john` query allows `john` to execute the `SELECT` query over all the tables in `db` database. Also, you can omit database name. In this case privileges are granted for current database. For example, `GRANT SELECT ON * TO john` grants the privilege on all the tables in the current database, `GRANT SELECT ON mytable TO john` grants the privilege on the `mytable` table in the current database. Access to the `system` database is always allowed (since this database is used for processing queries). -You can grant multiple privileges to multiple accounts in one query. The query `GRANT SELECT, INSERT ON *.* TO john, robin` allows accounts `john` and `robin` to perform the `INSERT` and `SELECT` queries over all the tables in all the databases on the server. +You can grant multiple privileges to multiple accounts in one query. The query `GRANT SELECT, INSERT ON *.* TO john, robin` allows accounts `john` and `robin` to execute the `INSERT` and `SELECT` queries over all the tables in all the databases on the server. ## Privileges {#grant-privileges} -Privilege is a permission to perform specific kind of queries. +Privilege is a permission to execute specific kind of queries. -Privileges have an hierarchical structure. A set of permitted queries depends on the privilege scope. +Privileges have a hierarchical structure. A set of permitted queries depends on the privilege scope. Hierarchy of privileges: @@ -212,20 +212,20 @@ The special privilege [ALL](#grant-all) grants all the privileges to a user acco By default, a user account or a role has no privileges. -If a user or role have no privileges it displayed as [NONE](#grant-none) privilege. +If a user or a role has no privileges, it is displayed as [NONE](#grant-none) privilege. -Some queries by their implementation require a set of privileges. For example, to perform the [RENAME](misc.md#misc_operations-rename) query you need the following privileges: `SELECT`, `CREATE TABLE`, `INSERT` and `DROP TABLE`. +Some queries by their implementation require a set of privileges. For example, to execute the [RENAME](misc.md#misc_operations-rename) query you need the following privileges: `SELECT`, `CREATE TABLE`, `INSERT` and `DROP TABLE`. ### SELECT {#grant-select} -Allows to perform [SELECT](select/index.md) queries. +Allows executing [SELECT](select/index.md) queries. Privilege level: `COLUMN`. **Description** -User granted with this privilege can perform `SELECT` queries over a specified list of columns in the specified table and database. If user includes other columns then specified a query returns no data. +User granted with this privilege can execute `SELECT` queries over a specified list of columns in the specified table and database. If user includes other columns then specified a query returns no data. Consider the following privilege: @@ -233,17 +233,17 @@ Consider the following privilege: GRANT SELECT(x,y) ON db.table TO john ``` -This privilege allows `john` to perform any `SELECT` query that involves data from the `x` and/or `y` columns in `db.table`. For example, `SELECT x FROM db.table`. `john` can't perform `SELECT z FROM db.table`. The `SELECT * FROM db.table` also is not available. Processing this query, ClickHouse doesn't return any data, even `x` and `y`. The only exception is if a table contains only `x` and `y` columns, in this case ClickHouse returns all the data. +This privilege allows `john` to execute any `SELECT` query that involves data from the `x` and/or `y` columns in `db.table`, for example, `SELECT x FROM db.table`. `john` can't execute `SELECT z FROM db.table`. The `SELECT * FROM db.table` also is not available. Processing this query, ClickHouse doesn't return any data, even `x` and `y`. The only exception is if a table contains only `x` and `y` columns, in this case ClickHouse returns all the data. ### INSERT {#grant-insert} -Allows performing [INSERT](insert-into.md) queries. +Allows executing [INSERT](insert-into.md) queries. Privilege level: `COLUMN`. **Description** -User granted with this privilege can perform `INSERT` queries over a specified list of columns in the specified table and database. If user includes other columns then specified a query doesn't insert any data. +User granted with this privilege can execute `INSERT` queries over a specified list of columns in the specified table and database. If user includes other columns then specified a query doesn't insert any data. **Example** @@ -255,7 +255,7 @@ The granted privilege allows `john` to insert data to the `x` and/or `y` columns ### ALTER {#grant-alter} -Allows performing [ALTER](alter.md) queries corresponding to the following hierarchy of privileges: +Allows executing [ALTER](alter.md) queries according to the following hierarchy of privileges: - `ALTER`. Level: `COLUMN`. - `ALTER TABLE`. Level: `GROUP` @@ -294,14 +294,14 @@ Examples of how this hierarchy is treated: **Notes** -- The `MODIFY SETTING` privilege allows to modify table engine settings. In doesn't affect settings or server configuration parameters. +- The `MODIFY SETTING` privilege allows modifying table engine settings. It doesn't affect settings or server configuration parameters. - The `ATTACH` operation needs the [CREATE](#grant-create) privilege. - The `DETACH` operation needs the [DROP](#grant-drop) privilege. - To stop mutation by the [KILL MUTATION](misc.md#kill-mutation) query, you need to have a privilege to start this mutation. For example, if you want to stop the `ALTER UPDATE` query, you need the `ALTER UPDATE`, `ALTER TABLE`, or `ALTER` privilege. ### CREATE {#grant-create} -Allows to perform [CREATE](create.md) and [ATTACH](misc.md#attach) DDL-queries corresponding to the following hierarchy of privileges: +Allows executing [CREATE](create.md) and [ATTACH](misc.md#attach) DDL-queries according to the following hierarchy of privileges: - `CREATE`. Level: `GROUP` - `CREATE DATABASE`. Level: `DATABASE` @@ -316,7 +316,7 @@ Allows to perform [CREATE](create.md) and [ATTACH](misc.md#attach) DDL-queries c ### DROP {#grant-drop} -Allows to perform [DROP](misc.md#drop) and [DETACH](misc.md#detach) queries corresponding to the following hierarchy of privileges: +Allows executing [DROP](misc.md#drop) and [DETACH](misc.md#detach) queries according to the following hierarchy of privileges: - `DROP`. Level: - `DROP DATABASE`. Level: `DATABASE` @@ -327,19 +327,19 @@ Allows to perform [DROP](misc.md#drop) and [DETACH](misc.md#detach) queries corr ### TRUNCATE {#grant-truncate} -Allows to perform [TRUNCATE](misc.md#truncate-statement) queries. +Allows executing [TRUNCATE](misc.md#truncate-statement) queries. Privilege level: `TABLE`. ### OPTIMIZE {#grant-optimize} -Allows to perform the [OPTIMIZE TABLE](misc.md#misc_operations-optimize) queries. +Allows executing [OPTIMIZE TABLE](misc.md#misc_operations-optimize) queries. Privilege level: `TABLE`. ### SHOW {#grant-show} -Allows to perform `SHOW`, `DESCRIBE`, `USE`, and `EXISTS` queries, corresponding to the following hierarchy of privileges: +Allows executing `SHOW`, `DESCRIBE`, `USE`, and `EXISTS` queries according to the following hierarchy of privileges: - `SHOW`. Level: `GROUP` - `SHOW DATABASES`. Level: `DATABASE`. Allows to execute `SHOW DATABASES`, `SHOW CREATE DATABASE`, `USE ` queries. @@ -349,12 +349,12 @@ Allows to perform `SHOW`, `DESCRIBE`, `USE`, and `EXISTS` queries, corresponding **Notes** -A user has the `SHOW` privilege if it has any another privilege concerning the specified table, dictionary or database. +A user has the `SHOW` privilege if it has any other privilege concerning the specified table, dictionary or database. ### KILL QUERY {#grant-kill-query} -Allows to perform the [KILL](misc.md#kill-query-statement) queries corresponding to the following hierarchy of privileges: +Allows executing [KILL](misc.md#kill-query-statement) queries according to the following hierarchy of privileges: Privilege level: `GLOBAL`. @@ -365,7 +365,7 @@ Privilege level: `GLOBAL`. ### ACCESS MANAGEMENT {#grant-access-management} -Allows a user to perform queries that manage users, roles and row policies. +Allows a user to execute queries that manage users, roles and row policies. - `ACCESS MANAGEMENT`. Level: `GROUP` - `CREATE USER`. Level: `GLOBAL` @@ -391,11 +391,11 @@ Allows a user to perform queries that manage users, roles and row policies. - `SHOW_QUOTAS`. Level: `GLOBAL`. Aliases: `SHOW CREATE QUOTA` - `SHOW_SETTINGS_PROFILES`. Level: `GLOBAL`. Aliases: `SHOW PROFILES`, `SHOW CREATE SETTINGS PROFILE`, `SHOW CREATE PROFILE` -The `ROLE ADMIN` privilege allows a user to grant and revoke any roles including those which are not granted to the user with the admin option. +The `ROLE ADMIN` privilege allows a user to assign and revoke any roles including those which are not assigned to the user with the admin option. ### SYSTEM {#grant-system} -Allows a user to perform the [SYSTEM](system.md) queries corresponding to the following hierarchy of privileges. +Allows a user to execute [SYSTEM](system.md) queries according to the following hierarchy of privileges. - `SYSTEM`. Level: `GROUP` - `SYSTEM SHUTDOWN`. Level: `GLOBAL`. Aliases: `SYSTEM KILL`, `SHUTDOWN` @@ -461,7 +461,7 @@ Examples: Allows a user to execute [dictGet](../functions/ext-dict-functions.md#dictget), [dictHas](../functions/ext-dict-functions.md#dicthas), [dictGetHierarchy](../functions/ext-dict-functions.md#dictgethierarchy), [dictIsIn](../functions/ext-dict-functions.md#dictisin) functions. -Level of privilege: `DICTIONARY`. +Privilege level: `DICTIONARY`. **Examples** @@ -480,6 +480,6 @@ Doesn't grant any privileges. ### ADMIN OPTION {#admin-option-privilege} -The `ADMIN OPTION` privilege allows a user granting their role to another user. +The `ADMIN OPTION` privilege allows a user to grant their role to another user. [Original article](https://clickhouse.tech/docs/en/query_language/grant/) diff --git a/docs/en/sql-reference/statements/misc.md b/docs/en/sql-reference/statements/misc.md index 56a82771e2a..18cbf1a90e8 100644 --- a/docs/en/sql-reference/statements/misc.md +++ b/docs/en/sql-reference/statements/misc.md @@ -126,7 +126,7 @@ DROP USER [IF EXISTS] name [,...] [ON CLUSTER cluster_name] Deletes a role. -Deleted role is revoked from all the entities where it was granted. +Deleted role is revoked from all the entities where it was assigned. ### Syntax {#drop-role-syntax} @@ -162,9 +162,9 @@ DROP QUOTA [IF EXISTS] name [,...] [ON CLUSTER cluster_name] ## DROP SETTINGS PROFILE {#drop-settings-profile-statement} -Deletes a quota. +Deletes a settings profile. -Deleted quota is revoked from all the entities where it was assigned. +Deleted settings profile is revoked from all the entities where it was assigned. ### Syntax {#drop-settings-profile-syntax} @@ -360,4 +360,4 @@ Lets you set the current database for the session. The current database is used for searching for tables if the database is not explicitly defined in the query with a dot before the table name. This query can’t be made when using the HTTP protocol, since there is no concept of a session. -[Original article](https://clickhouse.tech/docs/en/query_language/misc/) +[Original article](https://clickhouse.tech/docs/en/query_language/misc/) diff --git a/docs/en/sql-reference/statements/revoke.md b/docs/en/sql-reference/statements/revoke.md index 6bb5fc19c28..66ff978ddfb 100644 --- a/docs/en/sql-reference/statements/revoke.md +++ b/docs/en/sql-reference/statements/revoke.md @@ -23,7 +23,7 @@ REVOKE [ON CLUSTER cluster_name] [ADMIN OPTION FOR] role [,...] FROM {user | rol ## Description {#revoke-description} -To revoke some privilege you can use a privilege of wider scope then you plan to revoke. For example, if a user has the `SELECT (x,y)` privilege, administrator can perform `REVOKE SELECT(x,y) ...`, or `REVOKE SELECT * ...`, or even `REVOKE ALL PRIVILEGES ...` query to revoke this privilege. +To revoke some privilege you can use a privilege of a wider scope than you plan to revoke. For example, if a user has the `SELECT (x,y)` privilege, administrator can execute `REVOKE SELECT(x,y) ...`, or `REVOKE SELECT * ...`, or even `REVOKE ALL PRIVILEGES ...` query to revoke this privilege. ### Partial Revokes {#partial-revokes-dscr} @@ -32,14 +32,14 @@ You can revoke a part of a privilege. For example, if a user has the `SELECT *.* ## Examples {#revoke-example} -Grant the `john` user account with a privilege to select from all the databases excepting the `accounts` one: +Grant the `john` user account with a privilege to select from all the databases, excepting the `accounts` one: ``` sql GRANT SELECT ON *.* TO john; REVOKE SELECT ON accounts.* FROM john; ``` -Grant the `mira` user account with a privilege to select from all the columns of the `accounts.staff` table excepting the `wage` one. +Grant the `mira` user account with a privilege to select from all the columns of the `accounts.staff` table, excepting the `wage` one. ``` sql GRANT SELECT ON accounts.staff TO mira; diff --git a/docs/en/sql-reference/statements/show.md b/docs/en/sql-reference/statements/show.md index 24db87c6171..e892ee7e51b 100644 --- a/docs/en/sql-reference/statements/show.md +++ b/docs/en/sql-reference/statements/show.md @@ -131,7 +131,7 @@ SHOW CREATE USER [name | CURRENT_USER] ## SHOW CREATE ROLE {#show-create-role-statement} -Shows parameters that were used at a [role creation](create.md#create-role-statement) +Shows parameters that were used at a [role creation](create.md#create-role-statement). ### Syntax {#show-create-role-syntax} @@ -143,7 +143,7 @@ SHOW CREATE ROLE name ## SHOW CREATE ROW POLICY {#show-create-row-policy-statement} -Shows parameters that were used at a [row policy creation](create.md#create-row-policy-statement) +Shows parameters that were used at a [row policy creation](create.md#create-row-policy-statement). ### Syntax {#show-create-row-policy-syntax} @@ -154,7 +154,7 @@ SHOW CREATE [ROW] POLICY name ON [database.]table ## SHOW CREATE QUOTA {#show-create-quota-statement} -Shows parameters that were used at a [quota creation](create.md#create-quota-statement) +Shows parameters that were used at a [quota creation](create.md#create-quota-statement). ### Syntax {#show-create-row-policy-syntax} @@ -165,7 +165,7 @@ SHOW CREATE QUOTA [name | CURRENT] ## SHOW CREATE SETTINGS PROFILE {#show-create-settings-profile-statement} -Shows parameters that were used at a [settings profile creation](create.md#create-settings-profile-statement) +Shows parameters that were used at a [settings profile creation](create.md#create-settings-profile-statement). ### Syntax {#show-create-row-policy-syntax} diff --git a/docs/ru/operations/access-rights.md b/docs/ru/operations/access-rights.md index 8c131aa34e4..99da2550e70 100644 --- a/docs/ru/operations/access-rights.md +++ b/docs/ru/operations/access-rights.md @@ -1,101 +1,143 @@ -# Права доступа {#prava-dostupa} +# Управление доступом {#access-control} -Пользователи и права доступа настраиваются в конфиге пользователей. Обычно это `users.xml`. +ClickHouse поддерживает управление доступом на основе подхода [RBAC](https://ru.wikipedia.org/wiki/Управление_доступом_на_основе_ролей). -Пользователи прописаны в секции `users`. Рассмотрим фрагмент файла `users.xml`: +Объекты системы доступа в ClickHouse: -``` xml - - - - - - + Функциональность необходимо [включить](#enabling-access-control). - - +Рекомендуется использовать SQL-воркфлоу. Оба метода конфигурации работают одновременно, поэтому, если для управления доступом вы используете конфигурационные файлы, вы можете плавно перейти на SQL-воркфлоу. - - default +!!! note "Внимание" + Нельзя одновременно использовать оба метода для управления одним и тем же объектом системы доступа. - - default - - - - - - web - default - - test - - -``` +## Использование {#access-control-usage} -Здесь видно объявление двух пользователей - `default` и `web`. Пользователя `web` мы добавили самостоятельно. +По умолчанию сервер ClickHouse предоставляет аккаунт пользователя `default`, для которого выключена функция SQL-ориентированного управления доступом, но у него есть все права и разрешения. Аккаунт `default` используется во всех случаях, когда имя пользователя не определено. Например, при входе с клиента или в распределенных запросах. При распределенной обработке запроса `default` используется, если в конфигурации сервера или кластера не указаны свойства [user и password](../engines/table-engines/special/distributed.md). -Пользователь `default` выбирается в случаях, когда имя пользователя не передаётся. Также пользователь `default` может использоваться при распределённой обработке запроса - если в конфигурации кластера для сервера не указаны `user` и `password`. (см. раздел о движке [Distributed](../engines/table-engines/special/distributed.md)). +Если вы начали пользоваться ClickHouse недавно, попробуйте следующий сценарий: -Пользователь, который используется для обмена информацией между серверами, объединенными в кластер, не должен иметь существенных ограничений или квот - иначе распределённые запросы сломаются. +1. [Включите](#enabling-access-control) SQL-ориентированное управление доступом для пользователя `default`. +2. Войдите под пользователем `default` и создайте всех необходимых пользователей. Не забудьте создать аккаунт администратора (`GRANT ALL ON *.* WITH GRANT OPTION TO admin_user_account`). +3. [Ограничьте разрешения](settings/permissions-for-queries.md#permissions_for_queries) для пользователя `default` и отключите для него SQL-ориентированное управление доступом. -Пароль указывается либо в открытом виде (не рекомендуется), либо в виде SHA-256. Хэш не содержит соль. В связи с этим, не следует рассматривать такие пароли, как защиту от потенциального злоумышленника. Скорее, они нужны для защиты от сотрудников. +### Особенности реализации {#access-control-properties} -Указывается список сетей, из которых разрешён доступ. В этом примере, список сетей для обеих пользователей, загружается из отдельного файла (`/etc/metrika.xml`), содержащего подстановку `networks`. Вот его фрагмент: +- Вы можете выдавать разрешения на базы данных или таблицы, даже если они не существуют. +- При удалении таблицы все связанные с ней привилегии не отзываются. Если вы затем создадите новую таблицу с таким же именем, все привилегии останутся действительными. Чтобы отозвать привилегии, связанные с удаленной таблицей, необходимо выполнить, например, запрос `REVOKE ALL PRIVILEGES ON db.table FROM ALL`. +- У привилегий нет настроек времени жизни. -``` xml - - ... - - ::/64 - 203.0.113.0/24 - 2001:DB8::/32 - ... - - -``` +## Аккаунт пользователя {#user-account-management} -Можно было бы указать этот список сетей непосредственно в `users.xml`, или в файле в директории `users.d` (подробнее смотрите раздел «[Конфигурационные файлы](configuration-files.md#configuration_files)»). +Аккаунт пользователя — это объект системы доступа, позволяющий авторизовать кого-либо в ClickHouse. Аккаунт содержит: -В конфиге приведён комментарий, указывающий, как можно открыть доступ отовсюду. +- Идентификационную информацию. +- [Привилегии](../sql-reference/statements/grant.md#grant-privileges), определяющие область действия запросов, которые могут быть выполнены пользователем. +- Хосты, которые могут подключаться к серверу ClickHouse. +- Назначенные роли и роли по умолчанию. +- Настройки и их ограничения, которые применяются по умолчанию при входе пользователя. +- Присвоенные профили настроек. -Для продакшен использования, указывайте только элементы вида `ip` (IP-адреса и их маски), так как использование `host` и `host_regexp` может вызывать лишние задержки. +Привилегии присваиваются аккаунту пользователя с помощью запроса [GRANT](../sql-reference/statements/grant.md) или через назначение [ролей](#role-management). Отозвать привилегию можно с помощью запроса [REVOKE](../sql-reference/statements/revoke.md). Чтобы вывести список присвоенных привилегий, используется выражение [SHOW GRANTS](../sql-reference/statements/show.md#show-grants-statement). -Далее указывается используемый профиль настроек пользователя (смотрите раздел «[Профили настроек](settings/settings-profiles.md)»). Вы можете указать профиль по умолчанию - `default`. Профиль может называться как угодно; один и тот же профиль может быть указан для разных пользователей. Наиболее важная вещь, которую вы можете прописать в профиле настроек `readonly=1`, что обеспечивает доступ только на чтение. -Затем указывается используемая квота (смотрите раздел «[Квоты](quotas.md#quotas)»). Вы можете указать квоту по умолчанию — `default`. Она настроена в конфиге по умолчанию так, что только считает использование ресурсов, но никак их не ограничивает. Квота может называться как угодно. Одна и та же квота может быть указана для разных пользователей, в этом случае подсчёт использования ресурсов делается для каждого пользователя по отдельности. +Запросы управления: -Также, в необязательном разделе `` можно указать перечень баз, к которым у пользователя будет доступ. По умолчанию пользователю доступны все базы. Можно указать базу данных `default`, в этом случае пользователь получит доступ к базе данных по умолчанию. +- [CREATE USER](../sql-reference/statements/create.md#create-user-statement) +- [ALTER USER](../sql-reference/statements/alter.md#alter-user-statement) +- [DROP USER](../sql-reference/statements/misc.md#drop-user-statement) +- [SHOW CREATE USER](../sql-reference/statements/show.md#show-create-user-statement) -Доступ к БД `system` всегда считается разрешённым (так как эта БД используется для выполнения запросов). +### Применение настроек {#access-control-settings-applying} -Пользователь может получить список всех БД и таблиц в них с помощью запросов `SHOW` или системных таблиц, даже если у него нет доступа к отдельным БД. +Настройки могут быть заданы разными способами: для аккаунта пользователя, для назначенных ему ролей или в профилях настроек. При входе пользователя, если настройка задана для разных объектов системы доступа, значение настройки и ее ограничения применяются в следующем порядке (от высшего приоритета к низшему): + +1. Настройки аккаунта. +2. Настройки ролей по умолчанию для аккаунта. Если настройка задана для нескольких ролей, порядок применения не определен. +3. Настройки из профилей настроек, присвоенных пользователю или его ролям по умолчанию. Если настройка задана в нескольких профилях, порядок применения не определен. +4. Настройки, которые по умолчанию применяются ко всему серверу, или настройки из [профиля по умолчанию](server-configuration-parameters/settings.md#default-profile). + + +## Роль {#role-management} + +Роль — это контейнер объектов системы доступа, которые можно присвоить аккаунту пользователя. + +Роль содержит: + +- [Привилегии](../sql-reference/statements/grant.md#grant-privileges) +- Настройки и ограничения +- Список назначенных ролей + +Запросы управления: + +- [CREATE ROLE](../sql-reference/statements/create.md#create-role-statement) +- [ALTER ROLE](../sql-reference/statements/alter.md#alter-role-statement) +- [DROP ROLE](../sql-reference/statements/misc.md#drop-role-statement) +- [SET ROLE](../sql-reference/statements/misc.md#set-role-statement) +- [SET DEFAULT ROLE](../sql-reference/statements/misc.md#set-default-role-statement) +- [SHOW CREATE ROLE](../sql-reference/statements/show.md#show-create-role-statement) + +Привилегии можно присвоить роли с помощью запроса [GRANT](../sql-reference/statements/grant.md). Для отзыва привилегий у роли ClickHouse предоставляет запрос [REVOKE](../sql-reference/statements/revoke.md). + +## Политика доступа к строкам {#row-policy-management} + +Политика доступа к строкам — это фильтр, определяющий, какие строки доступны пользователю или роли. Политика содержит фильтры для конкретной таблицы, а также список ролей и/или пользователей, которые должны использовать данную политику. + +Запросы управления: + +- [CREATE ROW POLICY](../sql-reference/statements/create.md#create-row-policy-statement) +- [ALTER ROW POLICY](../sql-reference/statements/alter.md#alter-row-policy-statement) +- [DROP ROW POLICY](../sql-reference/statements/misc.md#drop-row-policy-statement) +- [SHOW CREATE ROW POLICY](../sql-reference/statements/show.md#show-create-row-policy-statement) + + +## Профиль настроек {#settings-profiles-management} + +Профиль настроек — это набор [настроек](settings/index.md). Профиль настроек содержит настройки и ограничения, а также список ролей и/или пользователей, по отношению к которым применяется данный профиль. + +Запросы управления: + +- [CREATE SETTINGS PROFILE](../sql-reference/statements/create.md#create-settings-profile-statement) +- [ALTER SETTINGS PROFILE](../sql-reference/statements/alter.md#alter-settings-profile-statement) +- [DROP SETTINGS PROFILE](../sql-reference/statements/misc.md#drop-settings-profile-statement) +- [SHOW CREATE SETTINGS PROFILE](../sql-reference/statements/show.md#show-create-settings-profile-statement) + + +## Квота {#quotas-management} + +Квота ограничивает использование ресурсов. См. [Квоты](quotas.md). + +Квота содержит набор ограничений определенной длительности, а также список ролей и/или пользователей, на которых распространяется данная квота. + +Запросы управления: + +- [CREATE QUOTA](../sql-reference/statements/create.md#create-quota-statement) +- [ALTER QUOTA](../sql-reference/statements/alter.md#alter-quota-statement) +- [DROP QUOTA](../sql-reference/statements/misc.md#drop-quota-statement) +- [SHOW CREATE QUOTA](../sql-reference/statements/show.md#show-create-quota-statement) + + +## Включение SQL-ориентированного управления доступом {#enabling-access-control} + +- Настройте каталог для хранения конфигураций. + + ClickHouse хранит конфигурации объектов системы доступа в каталоге, установленном в конфигурационном параметре сервера [access_control_path](server-configuration-parameters/settings.md#access_control_path). + +- Включите SQL-ориентированное управление доступом как минимум для одного аккаунта. + + По умолчанию управление доступом на основе SQL выключено для всех пользователей. Вам необходимо настроить хотя бы одного пользователя в файле конфигурации `users.xml` и присвоить значение 1 параметру [access_management](settings/settings-users.md#access_management-user-setting). -Доступ к БД не связан с настройкой [readonly](settings/permissions-for-queries.md#settings_readonly). Невозможно дать полный доступ к одной БД и `readonly` к другой. [Оригинальная статья](https://clickhouse.tech/docs/ru/operations/access_rights/) diff --git a/docs/ru/operations/server-configuration-parameters/settings.md b/docs/ru/operations/server-configuration-parameters/settings.md index 43e7eaac7d7..2e0067f99b0 100644 --- a/docs/ru/operations/server-configuration-parameters/settings.md +++ b/docs/ru/operations/server-configuration-parameters/settings.md @@ -832,4 +832,14 @@ ClickHouse использует ZooKeeper для хранения метадан **Значение по умолчанию**: 15. +## access_control_path {#access_control_path} + +Путь к каталогу, где сервер ClickHouse хранит конфигурации пользователей и ролей, созданные командами SQL. + +Значение по умолчанию: `/var/lib/clickhouse/access/`. + +**Смотрите также** + +- [Управление доступом](../access-rights.md#access-control) + [Оригинальная статья](https://clickhouse.tech/docs/ru/operations/server_configuration_parameters/settings/) diff --git a/docs/ru/operations/settings/settings-profiles.md b/docs/ru/operations/settings/settings-profiles.md index a0a8b4ba0ba..d1e24490120 100644 --- a/docs/ru/operations/settings/settings-profiles.md +++ b/docs/ru/operations/settings/settings-profiles.md @@ -1,6 +1,15 @@ -# Профили настроек {#profili-nastroek} +# Профили настроек {#settings-profiles} + +Профиль настроек — это набор настроек, сгруппированных под одним именем. + +!!! note "Информация" + Для управления профилями настроек рекомендуется использовать [SQL-ориентированный воркфлоу](../access-rights.md#access-control), который также поддерживается в ClickHouse. + + +Название профиля может быть любым. Вы можете указать один и тот же профиль для разных пользователей. Самое важное, что можно прописать в профиле — `readonly=1`, это обеспечит доступ только на чтение. + +Профили настроек поддерживают наследование. Это реализуется указанием одной или нескольких настроек `profile` перед остальными настройками, перечисленными в профиле. Если одна настройка указана в нескольких профилях, используется последнее из значений. -Профили настроек - это множество настроек, сгруппированных под одним именем. Для каждого пользователя ClickHouse указывается некоторый профиль. Все настройки профиля можно применить, установив настройку `profile`. Пример: @@ -57,8 +66,10 @@ SET profile = 'web' ``` -В примере задано два профиля: `default` и `web`. Профиль `default` имеет специальное значение - он всегда обязан присутствовать и применяется при запуске сервера. То есть, профиль `default` содержит настройки по умолчанию. Профиль `web` - обычный профиль, который может быть установлен с помощью запроса `SET` или с помощью параметра URL при запросе по HTTP. +В примере задано два профиля: `default` и `web`. -Профили настроек могут наследоваться от друг-друга - это реализуется указанием одной или нескольких настроек `profile` перед остальными настройками, перечисленными в профиле. Если одна настройка указана в нескольких профилях, используется последнее из значений. +Профиль `default` имеет специальное значение — он обязателен и применяется при запуске сервера. Профиль `default` содержит настройки по умолчанию. + +Профиль `web` — обычный профиль, который может быть установлен с помощью запроса `SET` или параметра URL при запросе по HTTP. [Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/settings_profiles/) diff --git a/docs/ru/operations/settings/settings-users.md b/docs/ru/operations/settings/settings-users.md index 2d2e2dec506..95701f0f639 100644 --- a/docs/ru/operations/settings/settings-users.md +++ b/docs/ru/operations/settings/settings-users.md @@ -2,6 +2,9 @@ Раздел `users` конфигурационного файла `user.xml` содержит настройки для пользователей. +!!! note "Информация" + Для управления пользователями рекомендуется использовать [SQL-ориентированный воркфлоу](../access-rights.md#access-control), который также поддерживается в ClickHouse. + Структура раздела `users`: ``` xml @@ -12,6 +15,8 @@ + 0|1 + @@ -67,6 +72,17 @@ Первая строка результата — пароль. Вторая строка — соответствующий ему двойной хэш SHA1. +### access_management {#access_management-user-setting} + +Включает или выключает SQL-ориентированное [управление доступом](../access-rights.md#access-control) для пользователя. + +Возможные значения: + +- 0 — Выключено. +- 1 — Включено. + +Значение по умолчанию: 0. + ### user\_name/networks {#user-namenetworks} Список сетей, из которых пользователь может подключиться к серверу ClickHouse. diff --git a/docs/ru/sql-reference/statements/alter.md b/docs/ru/sql-reference/statements/alter.md index 1b35f411df2..c4afb6981c4 100644 --- a/docs/ru/sql-reference/statements/alter.md +++ b/docs/ru/sql-reference/statements/alter.md @@ -498,8 +498,112 @@ ALTER TABLE [db.]table MATERIALIZE INDEX name IN PARTITION partition_name Мутации линейно упорядочены между собой и накладываются на каждый кусок в порядке добавления. Мутации также упорядочены со вставками - гарантируется, что данные, вставленные в таблицу до начала выполнения запроса мутации, будут изменены, а данные, вставленные после окончания запроса мутации, изменены не будут. При этом мутации никак не блокируют вставки. -Запрос завершается немедленно после добавления информации о мутации (для реплицированных таблиц - в ZooKeeper, для нереплицированных - на файловую систему). Сама мутация выполняется асинхронно, используя настройки системного профиля. Следить за ходом её выполнения можно по таблице [`system.mutations`](../../operations/system-tables.md#system_tables-mutations). Добавленные мутации будут выполняться до конца даже в случае перезапуска серверов ClickHouse. Откатить мутацию после её добавления нельзя, но если мутация по какой-то причине не может выполниться до конца, её можно остановить с помощью запроса [`KILL MUTATION`](misc.md#kill-mutation). +Запрос завершается немедленно после добавления информации о мутации (для реплицированных таблиц - в ZooKeeper, для нереплицированных - на файловую систему). Сама мутация выполняется асинхронно, используя настройки системного профиля. Следить за ходом её выполнения можно по таблице [`system.mutations`](../../operations/system-tables.md#system_tables-mutations). Добавленные мутации будут выполняться до конца даже в случае перезапуска серверов ClickHouse. Откатить мутацию после её добавления нельзя, но если мутация по какой-то причине не может выполниться до конца, её можно остановить с помощью запроса [`KILL MUTATION`](misc.md#kill-mutation-statement). Записи о последних выполненных мутациях удаляются не сразу (количество сохраняемых мутаций определяется параметром движка таблиц `finished_mutations_to_keep`). Более старые записи удаляются. +## ALTER USER {#alter-user-statement} + +Изменяет аккаунт пользователя ClickHouse. + +### Синтаксис {#alter-user-syntax} + +``` sql +ALTER USER [IF EXISTS] name [ON CLUSTER cluster_name] + [RENAME TO new_name] + [IDENTIFIED [WITH {PLAINTEXT_PASSWORD|SHA256_PASSWORD|DOUBLE_SHA1_PASSWORD}] BY {'password'|'hash'}] + [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] + [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] +``` + +### Описание {#alter-user-dscr} + +Для выполнения `ALTER USER` необходима привилегия [ALTER USER](grant.md#grant-access-management). + +### Примеры {#alter-user-examples} + +Установить ролями по умолчанию роли, назначенные пользователю: + +``` sql +ALTER USER user DEFAULT ROLE role1, role2 +``` + +Если роли не были назначены пользователю, ClickHouse выбрасывает исключение. + +Установить ролями по умолчанию все роли, назначенные пользователю: + +``` sql +ALTER USER user DEFAULT ROLE ALL +``` + +Если роль будет впоследствии назначена пользователю, она автоматически станет ролью по умолчанию. + +Установить ролями по умолчанию все назначенные пользователю роли кроме `role1` и `role2`: + +``` sql +ALTER USER user DEFAULT ROLE ALL EXCEPT role1, role2 +``` + + +## ALTER ROLE {#alter-role-statement} + +Изменяет роль. + +### Синтаксис {#alter-role-syntax} + +``` sql +ALTER ROLE [IF EXISTS] name [ON CLUSTER cluster_name] + [RENAME TO new_name] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] +``` + + +## ALTER ROW POLICY {#alter-row-policy-statement} + +Изменяет политику доступа к строкам. + +### Синтаксис {#alter-row-policy-syntax} + +``` sql +ALTER [ROW] POLICY [IF EXISTS] name [ON CLUSTER cluster_name] ON [database.]table + [RENAME TO new_name] + [AS {PERMISSIVE | RESTRICTIVE}] + [FOR SELECT] + [USING {condition | NONE}][,...] + [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] +``` + + +## ALTER QUOTA {#alter-quota-statement} + +Изменяет квоту. + +### Синтаксис {#alter-quota-syntax} + +``` sql +ALTER QUOTA [IF EXISTS] name [ON CLUSTER cluster_name] + [RENAME TO new_name] + [KEYED BY {'none' | 'user name' | 'ip address' | 'client key' | 'client key or user name' | 'client key or ip address'}] + [FOR [RANDOMIZED] INTERVAL number {SECOND | MINUTE | HOUR | DAY} + {MAX { {QUERIES | ERRORS | RESULT ROWS | RESULT BYTES | READ ROWS | READ BYTES | EXECUTION TIME} = number } [,...] | + NO LIMITS | TRACKING ONLY} [,...]] + [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] +``` + + +## ALTER SETTINGS PROFILE {#alter-settings-profile-statement} + +Изменяет профили настроек. + +### Синтаксис {#alter-settings-profile-syntax} + +``` sql +ALTER SETTINGS PROFILE [IF EXISTS] name [ON CLUSTER cluster_name] + [RENAME TO new_name] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...] +``` + + + [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/alter/) diff --git a/docs/ru/sql-reference/statements/create.md b/docs/ru/sql-reference/statements/create.md index 2a57ac0a143..70544c57dfd 100644 --- a/docs/ru/sql-reference/statements/create.md +++ b/docs/ru/sql-reference/statements/create.md @@ -302,4 +302,206 @@ LIFETIME([MIN val1] MAX val2) Смотрите [Внешние словари](../../sql-reference/statements/create.md). +## CREATE USER {#create-user-statement} + +Создает [аккаунт пользователя](../../operations/access-rights.md#user-account-management). + +### Синтаксис {#create-user-syntax} + +```sql +CREATE USER [IF NOT EXISTS | OR REPLACE] name [ON CLUSTER cluster_name] + [IDENTIFIED [WITH {NO_PASSWORD|PLAINTEXT_PASSWORD|SHA256_PASSWORD|SHA256_HASH|DOUBLE_SHA1_PASSWORD|DOUBLE_SHA1_HASH}] BY {'password'|'hash'}] + [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] + [DEFAULT ROLE role [,...]] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] +``` + +#### Идентификация + +Существует несколько способов идентификации пользователя: + +- `IDENTIFIED WITH no_password` +- `IDENTIFIED WITH plaintext_password BY 'qwerty'` +- `IDENTIFIED WITH sha256_password BY 'qwerty'` or `IDENTIFIED BY 'password'` +- `IDENTIFIED WITH sha256_hash BY 'hash'` +- `IDENTIFIED WITH double_sha1_password BY 'qwerty'` +- `IDENTIFIED WITH double_sha1_hash BY 'hash'` + +#### Пользовательский хост + +Пользовательский хост — это хост, с которого можно установить соединение с сервером ClickHouse. Хост задается в секции `HOST` следующими способами: + +- `HOST IP 'ip_address_or_subnetwork'` — Пользователь может подключиться к серверу ClickHouse только с указанного IP-адреса или [подсети](https://ru.wikipedia.org/wiki/Подсеть). Примеры: `HOST IP '192.168.0.0/16'`, `HOST IP '2001:DB8::/32'`. При использовании в эксплуатации указывайте только элементы `HOST IP` (IP-адреса и маски подсети), так как использование `host` и `host_regexp` может привести к дополнительной задержке. +- `HOST ANY` — Пользователь может подключиться с любого хоста. Используется по умолчанию. +- `HOST LOCAL` — Пользователь может подключиться только локально. +- `HOST NAME 'fqdn'` — Хост задается через FQDN. Например, `HOST NAME 'mysite.com'`. +- `HOST NAME REGEXP 'regexp'` — Позволяет использовать регулярные выражения [pcre](http://www.pcre.org/), чтобы задать хосты. Например, `HOST NAME REGEXP '.*\.mysite\.com'`. +- `HOST LIKE 'template'` — Позволяет использовать оператор [LIKE](../functions/string-search-functions.md#function-like) для фильтрации хостов. Например, `HOST LIKE '%'` эквивалентен `HOST ANY`; `HOST LIKE '%.mysite.com'` разрешает подключение со всех хостов в домене `mysite.com`. + +Также, чтобы задать хост, вы можете использовать `@` вместе с именем пользователя. Примеры: + +- `CREATE USER mira@'127.0.0.1'` — Эквивалентно `HOST IP`. +- `CREATE USER mira@'localhost'` — Эквивалентно `HOST LOCAL`. +- `CREATE USER mira@'192.168.%.%'` — Эквивалентно `HOST LIKE`. + +!!! info "Внимание" + ClickHouse трактует конструкцию `user_name@'address'` как имя пользователя целиком. То есть технически вы можете создать несколько пользователей с одинаковыми `user_name`, но разными частями конструкции после `@`, но лучше так не делать. + + +### Примеры {#create-user-examples} + + +Создать аккаунт `mira`, защищенный паролем `qwerty`: + +```sql +CREATE USER mira HOST IP '127.0.0.1' IDENTIFIED WITH sha256_password BY 'qwerty' +``` + +Пользователь `mira` должен запустить клиентское приложение на хосте, где запущен ClickHouse. + +Создать аккаунт `john`, назначить на него роли, сделать данные роли ролями по умолчанию: + +``` sql +CREATE USER john DEFAULT ROLE role1, role2 +``` + +Создать аккаунт `john` и установить ролями по умолчанию все его будущие роли: + +``` sql +ALTER USER user DEFAULT ROLE ALL +``` + +Когда роль будет назначена аккаунту `john`, она автоматически станет ролью по умолчанию. + +Создать аккаунт `john` и установить ролями по умолчанию все его будущие роли, кроме `role1` и `role2`: + +``` sql +ALTER USER john DEFAULT ROLE ALL EXCEPT role1, role2 +``` + + +## CREATE ROLE {#create-role-statement} + +Создает [роль](../../operations/access-rights.md#role-management). + +### Синтаксис {#create-role-syntax} + +```sql +CREATE ROLE [IF NOT EXISTS | OR REPLACE] name + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] +``` + +### Описание {#create-role-description} + +Роль — это набор [привилегий](grant.md#grant-privileges). Пользователь, которому назначена роль, получает все привилегии этой роли. + +Одному пользователю можно назначить несколько ролей. Пользователи могут применять назначенные роли в произвольных комбинациях с помощью выражения [SET ROLE](misc.md#set-role-statement). Конечный объем привилегий — это комбинация всех привилегий всех примененных ролей. Если у пользователя имеются привилегии, присвоенные его аккаунту напрямую, они также прибавляются к привилегиям, присвоенным через роли. + +Роли по умолчанию применяются при входе пользователя в систему. Установить роли по умолчанию можно с помощью выражений [SET DEFAULT ROLE](misc.md#set-default-role-statement) или [ALTER USER](alter.md#alter-user-statement). + +Для отзыва роли используется выражение [REVOKE](revoke.md). + +Для удаления роли используется выражение [DROP ROLE](misc.md#drop-role-statement). Удаленная роль автоматически отзывается у всех пользователей, которым была назначена. + +### Примеры {#create-role-examples} + +```sql +CREATE ROLE accountant; +GRANT SELECT ON db.* TO accountant; +``` + +Такая последовательность запросов создаст роль `accountant`, у которой есть привилегия на чтение из базы данных `accounting`. + +Назначить роль `accountant` аккаунту `mira`: + +```sql +GRANT accountant TO mira; +``` + +После назначения роли пользователь может ее применить и выполнять разрешенные ей запросы. Например: + +```sql +SET ROLE accountant; +SELECT * FROM db.*; +``` + +## CREATE ROW POLICY {#create-row-policy-statement} + +Создает [фильтр для строк](../../operations/access-rights.md#row-policy-management), которые пользователь может прочесть из таблицы. + +### Синтаксис {#create-row-policy-syntax} + +``` sql +CREATE [ROW] POLICY [IF NOT EXISTS | OR REPLACE] policy_name [ON CLUSTER cluster_name] ON [db.]table + [AS {PERMISSIVE | RESTRICTIVE}] + [FOR SELECT] + [USING condition] + [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] +``` + +#### Секция AS {#create-row-policy-as} + +С помощью данной секции можно создать политику разрешения или ограничения. + +Политика разрешения предоставляет доступ к строкам. Разрешительные политики, которые применяются к одной таблице, объединяются с помощью логического оператора `OR`. Политики являются разрешительными по умолчанию. + +Политика ограничения запрещает доступ к строкам. Ограничительные политики, которые применяются к одной таблице, объединяются логическим оператором `AND`. + +Ограничительные политики применяются к строкам, прошедшим фильтр разрешительной политики. Если вы не зададите разрешительные политики, пользователь не сможет обращаться ни к каким строкам из таблицы. + +#### Секция TO {#create-row-policy-to} + +В секции `TO` вы можете перечислить как роли, так и пользователей. Например, `CREATE ROW POLICY ... TO accountant, john@localhost`. + +Ключевым словом `ALL` обозначаются все пользователи, включая текущего. Ключевые слова `ALL EXCEPT` позволяют исключить пользователей из списка всех пользователей. Например, `CREATE ROW POLICY ... TO ALL EXCEPT accountant, john@localhost` + +### Примеры + +- `CREATE ROW POLICY filter ON mydb.mytable FOR SELECT USING a<1000 TO accountant, john@localhost` +- `CREATE ROW POLICY filter ON mydb.mytable FOR SELECT USING a<1000 TO ALL EXCEPT mira` + + +## CREATE QUOTA {#create-quota-statement} + +Создает [квоту](../../operations/access-rights.md#quotas-management), которая может быть присвоена пользователю или роли. + +### Синтаксис {#create-quota-syntax} + +``` sql +CREATE QUOTA [IF NOT EXISTS | OR REPLACE] name [ON CLUSTER cluster_name] + [KEYED BY {'none' | 'user name' | 'ip address' | 'client key' | 'client key or user name' | 'client key or ip address'}] + [FOR [RANDOMIZED] INTERVAL number {SECOND | MINUTE | HOUR | DAY} + {MAX { {QUERIES | ERRORS | RESULT ROWS | RESULT BYTES | READ ROWS | READ BYTES | EXECUTION TIME} = number } [,...] | + NO LIMITS | TRACKING ONLY} [,...]] + [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] +``` + +### Пример {#create-quota-example} + +Ограничить максимальное количество запросов для текущего пользователя до 123 запросов каждые 15 месяцев: + +``` sql +CREATE QUOTA qA FOR INTERVAL 15 MONTH MAX QUERIES 123 TO CURRENT_USER +``` + + +## CREATE SETTINGS PROFILE {#create-settings-profile-statement} + +Создает [профиль настроек](../../operations/access-rights.md#settings-profiles-management), который может быть присвоен пользователю или роли. + +### Синтаксис {#create-settings-profile-syntax} + +``` sql +CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] name [ON CLUSTER cluster_name] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...] +``` + +### Пример {#create-settings-profile-syntax} + +Создать профиль настроек `max_memory_usage_profile`, который содержит значение и ограничения для настройки `max_memory_usage`. Присвоить профиль пользователю `robin`: + +``` sql +CREATE SETTINGS PROFILE max_memory_usage_profile SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000 TO robin +``` + [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/create/) diff --git a/docs/ru/sql-reference/statements/grant.md b/docs/ru/sql-reference/statements/grant.md deleted file mode 120000 index f2acbe125b4..00000000000 --- a/docs/ru/sql-reference/statements/grant.md +++ /dev/null @@ -1 +0,0 @@ -../../../en/sql-reference/statements/grant.md \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/grant.md b/docs/ru/sql-reference/statements/grant.md new file mode 100644 index 00000000000..a328975ec12 --- /dev/null +++ b/docs/ru/sql-reference/statements/grant.md @@ -0,0 +1,479 @@ +# GRANT + +- Присваивает [привилегии](#grant-privileges) пользователям или ролям ClickHouse. +- Назначает роли пользователям или другим ролям. + +Отозвать привилегию можно с помощью выражения [REVOKE](revoke.md). Чтобы вывести список присвоенных привилегий, воспользуйтесь выражением [SHOW GRANTS](show.md#show-grants-statement). + +## Синтаксис присвоения привилегий {#grant-privigele-syntax} + +```sql +GRANT [ON CLUSTER cluster_name] privilege[(column_name [,...])] [,...] ON {db.table|db.*|*.*|table|*} TO {user | role | CURRENT_USER} [,...] [WITH GRANT OPTION] +``` + +- `privilege` — Тип привилегии +- `role` — Роль пользователя ClickHouse. +- `user` — Пользователь ClickHouse. + +`WITH GRANT OPTION` разрешает пользователю или роли выполнять запрос `GRANT`. Пользователь может выдавать только те привилегии, которые есть у него, той же или меньшей области действий. + + +## Синтаксис назначения ролей {#assign-role-syntax} + +```sql +GRANT [ON CLUSTER cluster_name] role [,...] TO {user | another_role | CURRENT_USER} [,...] [WITH ADMIN OPTION] +``` + +- `role` — Роль пользователя ClickHouse. +- `user` — Пользователь ClickHouse. + +`WITH ADMIN OPTION` присваивает привилегию [ADMIN OPTION](#admin-option-privilege) пользователю или роли. + +## Использование {#grant-usage} + +Для использования `GRANT` пользователь должен иметь привилегию `GRANT OPTION`. Пользователь может выдавать привилегии только внутри области действий назначенных ему самому привилегий. + +Например, администратор выдал привилегию пользователю `john`: + +```sql +GRANT SELECT(x,y) ON db.table TO john WITH GRANT OPTION +``` + +Это означает, что пользователю `john` разрешено выполнять: + +- `SELECT x,y FROM db.table`. +- `SELECT x FROM db.table`. +- `SELECT y FROM db.table`. + +`john` не может выполнить `SELECT z FROM db.table` или `SELECT * FROM db.table`. После обработки данных запросов ClickHouse ничего не вернет — даже `x` или `y`. Единственное исключение — если таблица содержит только столбцы `x` и `y`. В таком случае ClickHouse вернет все данные. + +Также у `john` есть привилегия `GRANT OPTION`. `john` может выдать другим пользователям привилегии той же или меньшей области действий из тех, которые есть у него. + +При присвоении привилегий допускается использовать астериск (`*`) вместо имени таблицы или базы данных. Например, запрос `GRANT SELECT ON db.* TO john` позволит пользователю `john` выполнять `SELECT` над всеми таблицам в базе данных `db`. Также вы можете опускать имя базы данных. В таком случае привилегии позволят совершать операции над текущей базой данных. Например, запрос `GRANT SELECT ON * TO john` выдаст привилегию на выполнение `SELECT` над всеми таблицами в текущей базе данных; `GRANT SELECT ON mytable TO john` — только над таблицей `mytable` в текущей базе данных. + +Доступ к базе данных `system` разрешен всегда (данная база данных используется при обработке запросов). + +Вы можете присвоить несколько привилегий нескольким пользователям в одном запросе. Запрос `GRANT SELECT, INSERT ON *.* TO john, robin` позволит пользователям `john` и `robin` выполнять `INSERT` и `SELECT` над всеми таблицами всех баз данных на сервере. + + +## Привилегии {#grant-privileges} + +Привилегия — это разрешение на выполнение определенного типа запросов. + +Привилегии имеют иерархическую структуру. Набор разрешенных запросов зависит от области действия привилегии. + +Иерархия привилегий: + +- [SELECT](#grant-select) +- [INSERT](#grant-insert) +- [ALTER](#grant-alter) + - `ALTER TABLE` + - `ALTER UPDATE` + - `ALTER DELETE` + - `ALTER COLUMN` + - `ALTER ADD COLUMN` + - `ALTER DROP COLUMN` + - `ALTER MODIFY COLUMN` + - `ALTER COMMENT COLUMN` + - `ALTER CLEAR COLUMN` + - `ALTER RENAME COLUMN` + - `ALTER INDEX` + - `ALTER ORDER BY` + - `ALTER ADD INDEX` + - `ALTER DROP INDEX` + - `ALTER MATERIALIZE INDEX` + - `ALTER CLEAR INDEX` + - `ALTER CONSTRAINT` + - `ALTER ADD CONSTRAINT` + - `ALTER DROP CONSTRAINT` + - `ALTER TTL` + - `ALTER MATERIALIZE TTL` + - `ALTER SETTINGS` + - `ALTER MOVE PARTITION` + - `ALTER FETCH PARTITION` + - `ALTER FREEZE PARTITION` + - `ALTER VIEW` + - `ALTER VIEW REFRESH ` + - `ALTER VIEW MODIFY QUERY` +- [CREATE](#grant-create) + - `CREATE DATABASE` + - `CREATE TABLE` + - `CREATE VIEW` + - `CREATE DICTIONARY` + - `CREATE TEMPORARY TABLE` +- [DROP](#grant-drop) + - `DROP DATABASE` + - `DROP TABLE` + - `DROP VIEW` + - `DROP DICTIONARY` +- [TRUNCATE](#grant-truncate) +- [OPTIMIZE](#grant-optimize) +- [SHOW](#grant-show) + - `SHOW DATABASES` + - `SHOW TABLES` + - `SHOW COLUMNS` + - `SHOW DICTIONARIES` +- [KILL QUERY](#grant-kill-query) +- [ACCESS MANAGEMENT](#grant-access-management) + - `CREATE USER` + - `ALTER USER` + - `DROP USER` + - `CREATE ROLE` + - `ALTER ROLE` + - `DROP ROLE` + - `CREATE ROW POLICY` + - `ALTER ROW POLICY` + - `DROP ROW POLICY` + - `CREATE QUOTA` + - `ALTER QUOTA` + - `DROP QUOTA` + - `CREATE SETTINGS PROFILE` + - `ALTER SETTINGS PROFILE` + - `DROP SETTINGS PROFILE` + - `SHOW ACCESS` + - `SHOW_USERS` + - `SHOW_ROLES` + - `SHOW_ROW_POLICIES` + - `SHOW_QUOTAS` + - `SHOW_SETTINGS_PROFILES` + - `ROLE ADMIN` +- [SYSTEM](#grant-system) + - `SYSTEM SHUTDOWN` + - `SYSTEM DROP CACHE` + - `SYSTEM DROP DNS CACHE` + - `SYSTEM DROP MARK CACHE` + - `SYSTEM DROP UNCOMPRESSED CACHE` + - `SYSTEM RELOAD` + - `SYSTEM RELOAD CONFIG` + - `SYSTEM RELOAD DICTIONARY` + - `SYSTEM RELOAD EMBEDDED DICTIONARIES` + - `SYSTEM MERGES` + - `SYSTEM TTL MERGES` + - `SYSTEM FETCHES` + - `SYSTEM MOVES` + - `SYSTEM SENDS` + - `SYSTEM DISTRIBUTED SENDS` + - `SYSTEM REPLICATED SENDS` + - `SYSTEM REPLICATION QUEUES` + - `SYSTEM SYNC REPLICA` + - `SYSTEM RESTART REPLICA` + - `SYSTEM FLUSH` + - `SYSTEM FLUSH DISTRIBUTED` + - `SYSTEM FLUSH LOGS` +- [INTROSPECTION](#grant-introspection) + - `addressToLine` + - `addressToSymbol` + - `demangle` +- [SOURCES](#grant-sources) + - `FILE` + - `URL` + - `REMOTE` + - `YSQL` + - `ODBC` + - `JDBC` + - `HDFS` + - `S3` +- [dictGet](#grant-dictget) + +Примеры того, как трактуется данная иерархия: + +- Привилегия `ALTER` включает все остальные `ALTER*` привилегии. +- `ALTER CONSTRAINT` включает `ALTER ADD CONSTRAINT` и `ALTER DROP CONSTRAINT`. + +Привилегии применяются на разных уровнях. Уровень определяет синтаксис присваивания привилегии. + +Уровни (от низшего к высшему): + +- `COLUMN` — Привилегия присваивается для столбца, таблицы, базы данных или глобально. +- `TABLE` — Привилегия присваивается для таблицы, базы данных или глобально. +- `VIEW` — Привилегия присваивается для представления, базы данных или глобально. +- `DICTIONARY` — Привилегия присваивается для словаря, базы данных или глобально. +- `DATABASE` — Привилегия присваивается для базы данных или глобально. +- `GLOBAL` — Привилегия присваивается только глобально. +- `GROUP` — Группирует привилегии разных уровней. При присвоении привилегии уровня `GROUP` присваиваются только привилегии из группы в соответствии с используемым синтаксисом. + +Примеры допустимого синтаксиса: + +- `GRANT SELECT(x) ON db.table TO user` +- `GRANT SELECT ON db.* TO user` + +Примеры недопустимого синтаксиса: + +- `GRANT CREATE USER(x) ON db.table TO user` +- `GRANT CREATE USER ON db.* TO user` + +Специальная привилегия [ALL](#grant-all) присваивает все привилегии пользователю или роли. + +По умолчанию пользователь или роль не имеют привилегий. + +Отсутствие привилегий у пользователя или роли отображается как привилегия [NONE](#grant-none). + +Выполнение некоторых запросов требует определенного набора привилегий. Например, чтобы выполнить запрос [RENAME](misc.md#misc_operations-rename), нужны следующие привилегии: `SELECT`, `CREATE TABLE`, `INSERT` и `DROP TABLE`. + + +### SELECT {#grant-select} + +Разрешает выполнять запросы [SELECT](select/index.md). + +Уровень: `COLUMN`. + +**Описание** + +Пользователь с данной привилегией может выполнять запросы `SELECT` над определенными столбцами из определенной таблицы и базы данных. При включении в запрос других столбцов запрос ничего не вернет. + +Рассмотрим следующую привилегию: + +```sql +GRANT SELECT(x,y) ON db.table TO john +``` + +Данная привилегия позволяет пользователю `john` выполнять выборку данных из столбцов `x` и/или `y` в `db.table`, например, `SELECT x FROM db.table`. `john` не может выполнить `SELECT z FROM db.table` или `SELECT * FROM db.table`. После обработки данных запросов ClickHouse ничего не вернет — даже `x` или `y`. Единственное исключение — если таблица содержит только столбцы `x` и `y`. В таком случае ClickHouse вернет все данные. + +### INSERT {#grant-insert} + +Разрешает выполнять запросы [INSERT](insert-into.md). + +Уровень: `COLUMN`. + +**Описание** + +Пользователь с данной привилегией может выполнять запросы `INSERT` над определенными столбцами из определенной таблицы и базы данных. При включении в запрос других столбцов запрос не добавит никаких данных. + +**Пример** + +```sql +GRANT INSERT(x,y) ON db.table TO john +``` + +Присвоенная привилегия позволит пользователю `john` вставить данные в столбцы `x` и/или `y` в `db.table`. + +### ALTER {#grant-alter} + +Разрешает выполнять запросы [ALTER](alter.md) в соответствии со следующей иерархией привилегий: + +- `ALTER`. Уровень: `COLUMN`. + - `ALTER TABLE`. Уровень: `GROUP` + - `ALTER UPDATE`. Уровень: `COLUMN`. Алиасы: `UPDATE` + - `ALTER DELETE`. Уровень: `COLUMN`. Алиасы: `DELETE` + - `ALTER COLUMN`. Уровень: `GROUP` + - `ALTER ADD COLUMN`. Уровень: `COLUMN`. Алиасы: `ADD COLUMN` + - `ALTER DROP COLUMN`. Уровень: `COLUMN`. Алиасы: `DROP COLUMN` + - `ALTER MODIFY COLUMN`. Уровень: `COLUMN`. Алиасы: `MODIFY COLUMN` + - `ALTER COMMENT COLUMN`. Уровень: `COLUMN`. Алиасы: `COMMENT COLUMN` + - `ALTER CLEAR COLUMN`. Уровень: `COLUMN`. Алиасы: `CLEAR COLUMN` + - `ALTER RENAME COLUMN`. Уровень: `COLUMN`. Алиасы: `RENAME COLUMN` + - `ALTER INDEX`. Уровень: `GROUP`. Алиасы: `INDEX` + - `ALTER ORDER BY`. Уровень: `TABLE`. Алиасы: `ALTER MODIFY ORDER BY`, `MODIFY ORDER BY` + - `ALTER ADD INDEX`. Уровень: `TABLE`. Алиасы: `ADD INDEX` + - `ALTER DROP INDEX`. Уровень: `TABLE`. Алиасы: `DROP INDEX` + - `ALTER MATERIALIZE INDEX`. Уровень: `TABLE`. Алиасы: `MATERIALIZE INDEX` + - `ALTER CLEAR INDEX`. Уровень: `TABLE`. Алиасы: `CLEAR INDEX` + - `ALTER CONSTRAINT`. Уровень: `GROUP`. Алиасы: `CONSTRAINT` + - `ALTER ADD CONSTRAINT`. Уровень: `TABLE`. Алиасы: `ADD CONSTRAINT` + - `ALTER DROP CONSTRAINT`. Уровень: `TABLE`. Алиасы: `DROP CONSTRAINT` + - `ALTER TTL`. Уровень: `TABLE`. Алиасы: `ALTER MODIFY TTL`, `MODIFY TTL` + - `ALTER MATERIALIZE TTL`. Уровень: `TABLE`. Алиасы: `MATERIALIZE TTL` + - `ALTER SETTINGS`. Уровень: `TABLE`. Алиасы: `ALTER SETTING`, `ALTER MODIFY SETTING`, `MODIFY SETTING` + - `ALTER MOVE PARTITION`. Уровень: `TABLE`. Алиасы: `ALTER MOVE PART`, `MOVE PARTITION`, `MOVE PART` + - `ALTER FETCH PARTITION`. Уровень: `TABLE`. Алиасы: `FETCH PARTITION` + - `ALTER FREEZE PARTITION`. Уровень: `TABLE`. Алиасы: `FREEZE PARTITION` + - `ALTER VIEW` Уровень: `GROUP` + - `ALTER VIEW REFRESH `. Уровень: `VIEW`. Алиасы: `ALTER LIVE VIEW REFRESH`, `REFRESH VIEW` + - `ALTER VIEW MODIFY QUERY`. Уровень: `VIEW`. Алиасы: `ALTER TABLE MODIFY QUERY` + +Примеры того, как трактуется данная иерархия: + +- Привилегия `ALTER` включает все остальные `ALTER*` привилегии. +- `ALTER CONSTRAINT` включает `ALTER ADD CONSTRAINT` и `ALTER DROP CONSTRAINT`. + +**Дополнительно** + +- Привилегия `MODIFY SETTING` позволяет изменять настройки движков таблиц. Не влияет на настройки или конфигурационные параметры сервера. +- Операция `ATTACH` требует наличие привилегии [CREATE](#grant-create). +- Операция `DETACH` требует наличие привилегии [DROP](#grant-drop). +- Для остановки мутации с помощью [KILL MUTATION](misc.md#kill-mutation-statement), необходима привилегия на выполнение данной мутации. Например, чтобы остановить запрос `ALTER UPDATE`, необходима одна из привилегий: `ALTER UPDATE`, `ALTER TABLE` или `ALTER`. + +### CREATE {#grant-create} + +Разрешает выполнять DDL-запросы [CREATE](create.md) и [ATTACH](misc.md#attach) в соответствии со следующей иерархией привилегий: + +- `CREATE`. Уровень: `GROUP` + - `CREATE DATABASE`. Уровень: `DATABASE` + - `CREATE TABLE`. Уровень: `TABLE` + - `CREATE VIEW`. Уровень: `VIEW` + - `CREATE DICTIONARY`. Уровень: `DICTIONARY` + - `CREATE TEMPORARY TABLE`. Уровень: `GLOBAL` + +**Дополнительно** + +- Для удаления созданной таблицы пользователю необходима привилегия [DROP](#grant-drop). + +### DROP {#grant-drop} + +Разрешает выполнять запросы [DROP](misc.md#drop) и [DETACH](misc.md#detach-statement) в соответствии со следующей иерархией привилегий: + +- `DROP`. Уровень: + - `DROP DATABASE`. Уровень: `DATABASE` + - `DROP TABLE`. Уровень: `TABLE` + - `DROP VIEW`. Уровень: `VIEW` + - `DROP DICTIONARY`. Уровень: `DICTIONARY` + + +### TRUNCATE {#grant-truncate} + +Разрешает выполнять запросы [TRUNCATE](misc.md#truncate-statement). + +Уровень: `TABLE`. + +### OPTIMIZE {#grant-optimize} + +Разрешает выполнять запросы [OPTIMIZE TABLE](misc.md#misc_operations-optimize). + +Уровень: `TABLE`. + +### SHOW {#grant-show} + +Разрешает выполнять запросы `SHOW`, `DESCRIBE`, `USE` и `EXISTS` в соответствии со следующей иерархией привилегий: + +- `SHOW`. Уровень: `GROUP` + - `SHOW DATABASES`. Уровень: `DATABASE`. Разрешает выполнять запросы `SHOW DATABASES`, `SHOW CREATE DATABASE`, `USE `. + - `SHOW TABLES`. Уровень: `TABLE`. Разрешает выполнять запросы `SHOW TABLES`, `EXISTS `, `CHECK
`. + - `SHOW COLUMNS`. Уровень: `COLUMN`. Разрешает выполнять запросы `SHOW CREATE TABLE`, `DESCRIBE`. + - `SHOW DICTIONARIES`. Уровень: `DICTIONARY`. Разрешает выполнять запросы `SHOW DICTIONARIES`, `SHOW CREATE DICTIONARY`, `EXISTS `. + +**Дополнительно** + +У пользователя есть привилегия `SHOW`, если ему присвоена любая другая привилегия по отношению к определенной таблице, словарю или базе данных. + + +### KILL QUERY {#grant-kill-query} + +Разрешает выполнять запросы [KILL](misc.md#kill-query-statement) в соответствии со следующей иерархией привилегий: + +Уровень: `GLOBAL`. + +**Дополнительно** + +`KILL QUERY` позволяет пользователю останавливать запросы других пользователей. + + +### ACCESS MANAGEMENT {#grant-access-management} + +Разрешает пользователю выполнять запросы на управление пользователями, ролями и политиками доступа к строкам. + +- `ACCESS MANAGEMENT`. Уровень: `GROUP` + - `CREATE USER`. Уровень: `GLOBAL` + - `ALTER USER`. Уровень: `GLOBAL` + - `DROP USER`. Уровень: `GLOBAL` + - `CREATE ROLE`. Уровень: `GLOBAL` + - `ALTER ROLE`. Уровень: `GLOBAL` + - `DROP ROLE`. Уровень: `GLOBAL` + - `ROLE ADMIN`. Уровень: `GLOBAL` + - `CREATE ROW POLICY`. Уровень: `GLOBAL`. Алиасы: `CREATE POLICY` + - `ALTER ROW POLICY`. Уровень: `GLOBAL`. Алиасы: `ALTER POLICY` + - `DROP ROW POLICY`. Уровень: `GLOBAL`. Алиасы: `DROP POLICY` + - `CREATE QUOTA`. Уровень: `GLOBAL` + - `ALTER QUOTA`. Уровень: `GLOBAL` + - `DROP QUOTA`. Уровень: `GLOBAL` + - `CREATE SETTINGS PROFILE`. Уровень: `GLOBAL`. Алиасы: `CREATE PROFILE` + - `ALTER SETTINGS PROFILE`. Уровень: `GLOBAL`. Алиасы: `ALTER PROFILE` + - `DROP SETTINGS PROFILE`. Уровень: `GLOBAL`. Алиасы: `DROP PROFILE` + - `SHOW ACCESS`. Уровень: `GROUP` + - `SHOW_USERS`. Уровень: `GLOBAL`. Алиасы: `SHOW CREATE USER` + - `SHOW_ROLES`. Уровень: `GLOBAL`. Алиасы: `SHOW CREATE ROLE` + - `SHOW_ROW_POLICIES`. Уровень: `GLOBAL`. Алиасы: `SHOW POLICIES`, `SHOW CREATE ROW POLICY`, `SHOW CREATE POLICY` + - `SHOW_QUOTAS`. Уровень: `GLOBAL`. Алиасы: `SHOW CREATE QUOTA` + - `SHOW_SETTINGS_PROFILES`. Уровень: `GLOBAL`. Алиасы: `SHOW PROFILES`, `SHOW CREATE SETTINGS PROFILE`, `SHOW CREATE PROFILE` + +Привилегия `ROLE ADMIN` разрешает пользователю назначать и отзывать любые роли, включая те, которые не назначены пользователю с опцией администратора. + +### SYSTEM {#grant-system} + +Разрешает выполнять запросы [SYSTEM](system.md) в соответствии со следующей иерархией привилегий: + +- `SYSTEM`. Уровень: `GROUP` + - `SYSTEM SHUTDOWN`. Уровень: `GLOBAL`. Алиасы: `SYSTEM KILL`, `SHUTDOWN` + - `SYSTEM DROP CACHE`. Алиасы: `DROP CACHE` + - `SYSTEM DROP DNS CACHE`. Уровень: `GLOBAL`. Алиасы: `SYSTEM DROP DNS`, `DROP DNS CACHE`, `DROP DNS` + - `SYSTEM DROP MARK CACHE`. Уровень: `GLOBAL`. Алиасы: `SYSTEM DROP MARK`, `DROP MARK CACHE`, `DROP MARKS` + - `SYSTEM DROP UNCOMPRESSED CACHE`. Уровень: `GLOBAL`. Алиасы: `SYSTEM DROP UNCOMPRESSED`, `DROP UNCOMPRESSED CACHE`, `DROP UNCOMPRESSED` + - `SYSTEM RELOAD`. Уровень: `GROUP` + - `SYSTEM RELOAD CONFIG`. Уровень: `GLOBAL`. Алиасы: `RELOAD CONFIG` + - `SYSTEM RELOAD DICTIONARY`. Уровень: `GLOBAL`. Алиасы: `SYSTEM RELOAD DICTIONARIES`, `RELOAD DICTIONARY`, `RELOAD DICTIONARIES` + - `SYSTEM RELOAD EMBEDDED DICTIONARIES`. Уровень: `GLOBAL`. Алиасы: `RELOAD EMBEDDED DICTIONARIES` + - `SYSTEM MERGES`. Уровень: `TABLE`. Алиасы: `SYSTEM STOP MERGES`, `SYSTEM START MERGES`, `STOP MERGES`, `START MERGES` + - `SYSTEM TTL MERGES`. Уровень: `TABLE`. Алиасы: `SYSTEM STOP TTL MERGES`, `SYSTEM START TTL MERGES`, `STOP TTL MERGES`, `START TTL MERGES` + - `SYSTEM FETCHES`. Уровень: `TABLE`. Алиасы: `SYSTEM STOP FETCHES`, `SYSTEM START FETCHES`, `STOP FETCHES`, `START FETCHES` + - `SYSTEM MOVES`. Уровень: `TABLE`. Алиасы: `SYSTEM STOP MOVES`, `SYSTEM START MOVES`, `STOP MOVES`, `START MOVES` + - `SYSTEM SENDS`. Уровень: `GROUP`. Алиасы: `SYSTEM STOP SENDS`, `SYSTEM START SENDS`, `STOP SENDS`, `START SENDS` + - `SYSTEM DISTRIBUTED SENDS`. Уровень: `TABLE`. Алиасы: `SYSTEM STOP DISTRIBUTED SENDS`, `SYSTEM START DISTRIBUTED SENDS`, `STOP DISTRIBUTED SENDS`, `START DISTRIBUTED SENDS` + - `SYSTEM REPLICATED SENDS`. Уровень: `TABLE`. Алиасы: `SYSTEM STOP REPLICATED SENDS`, `SYSTEM START REPLICATED SENDS`, `STOP REPLICATED SENDS`, `START REPLICATED SENDS` + - `SYSTEM REPLICATION QUEUES`. Уровень: `TABLE`. Алиасы: `SYSTEM STOP REPLICATION QUEUES`, `SYSTEM START REPLICATION QUEUES`, `STOP REPLICATION QUEUES`, `START REPLICATION QUEUES` + - `SYSTEM SYNC REPLICA`. Уровень: `TABLE`. Алиасы: `SYNC REPLICA` + - `SYSTEM RESTART REPLICA`. Уровень: `TABLE`. Алиасы: `RESTART REPLICA` + - `SYSTEM FLUSH`. Уровень: `GROUP` + - `SYSTEM FLUSH DISTRIBUTED`. Уровень: `TABLE`. Алиасы: `FLUSH DISTRIBUTED` + - `SYSTEM FLUSH LOGS`. Уровень: `GLOBAL`. Алиасы: `FLUSH LOGS` + +Привилегия `SYSTEM RELOAD EMBEDDED DICTIONARIES` имплицитно присваивается привилегией `SYSTEM RELOAD DICTIONARY ON *.*`. + + +### INTROSPECTION {#grant-introspection} + +Разрешает использовать функции [интроспекции](../../operations/optimizing-performance/sampling-query-profiler.md). + +- `INTROSPECTION`. Уровень: `GROUP`. Алиасы: `INTROSPECTION FUNCTIONS` + - `addressToLine`. Уровень: `GLOBAL` + - `addressToSymbol`. Уровень: `GLOBAL` + - `demangle`. Уровень: `GLOBAL` + + +### SOURCES {#grant-sources} + +Разрешает использовать внешние источники данных. Применяется к [движкам таблиц](../../engines/table-engines/index.md) и [табличным функциям](../table-functions/index.md#table-functions). + +- `SOURCES`. Уровень: `GROUP` + - `FILE`. Уровень: `GLOBAL` + - `URL`. Уровень: `GLOBAL` + - `REMOTE`. Уровень: `GLOBAL` + - `YSQL`. Уровень: `GLOBAL` + - `ODBC`. Уровень: `GLOBAL` + - `JDBC`. Уровень: `GLOBAL` + - `HDFS`. Уровень: `GLOBAL` + - `S3`. Уровень: `GLOBAL` + +Привилегия `SOURCES` разрешает использование всех источников. Также вы можете присвоить привилегию для каждого источника отдельно. Для использования источников необходимы дополнительные привилегии. + +Примеры: + +- Чтобы создать таблицу с [движком MySQL](../../engines/table-engines/integrations/mysql.md), необходимы привилегии `CREATE TABLE (ON db.table_name)` и `MYSQL`. +- Чтобы использовать [табличную функцию mysql](../table-functions/mysql.md), необходимы привилегии `CREATE TEMPORARY TABLE` и `MYSQL`. + +### dictGet {#grant-dictget} + +- `dictGet`. Алиасы: `dictHas`, `dictGetHierarchy`, `dictIsIn` + +Разрешает вызывать функции [dictGet](../functions/ext-dict-functions.md#dictget), [dictHas](../functions/ext-dict-functions.md#dicthas), [dictGetHierarchy](../functions/ext-dict-functions.md#dictgethierarchy), [dictIsIn](../functions/ext-dict-functions.md#dictisin). + +Уровень: `DICTIONARY`. + +**Примеры** + +- `GRANT dictGet ON mydb.mydictionary TO john` +- `GRANT dictGet ON mydictionary TO john` + +### ALL {#grant-all} + +Присваивает пользователю или роли все привилегии на объект с регулируемым доступом. + + +### NONE {#grant-none} + +Не присваивает никаких привилегий. + + +### ADMIN OPTION {#admin-option-privilege} + +Привилегия `ADMIN OPTION` разрешает пользователю назначать свои роли другому пользователю. + +[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/grant/) diff --git a/docs/ru/sql-reference/statements/misc.md b/docs/ru/sql-reference/statements/misc.md index cdd53843ab4..97d2ce5818e 100644 --- a/docs/ru/sql-reference/statements/misc.md +++ b/docs/ru/sql-reference/statements/misc.md @@ -70,7 +70,7 @@ DESC|DESCRIBE TABLE [db.]table [INTO OUTFILE filename] [FORMAT format] Вложенные структуры данных выводятся в «развёрнутом» виде. То есть, каждый столбец - по отдельности, с именем через точку. -## DETACH {#detach} +## DETACH {#detach-statement} Удаляет из сервера информацию о таблице name. Сервер перестаёт знать о существовании таблицы. @@ -91,9 +91,6 @@ DETACH TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] DROP DATABASE [IF EXISTS] db [ON CLUSTER cluster] ``` -Удаляет все таблицы внутри базы данных db, а затем саму базу данных db. -Если указано `IF EXISTS` - не выдавать ошибку, если база данных не существует. - ``` sql DROP [TEMPORARY] TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] ``` @@ -101,6 +98,68 @@ DROP [TEMPORARY] TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] Удаляет таблицу. Если указано `IF EXISTS` - не выдавать ошибку, если таблица не существует или база данных не существует. +## DROP USER {#drop-user-statement} + +Удаляет пользователя. + +### Синтаксис {#drop-user-syntax} + +```sql +DROP USER [IF EXISTS] name [,...] [ON CLUSTER cluster_name] +``` + + +## DROP ROLE {#drop-role-statement} + +Удаляет роль. + +При удалении роль отзывается у всех объектов системы доступа, которым она присвоена. + +### Синтаксис {#drop-role-syntax} + +```sql +DROP ROLE [IF EXISTS] name [,...] [ON CLUSTER cluster_name] +``` + +## DROP ROW POLICY {#drop-row-policy-statement} + +Удаляет политику доступа к строкам. + +При удалении политика отзывается у всех объектов системы доступа, которым она присвоена. + +### Синтаксис {#drop-row-policy-syntax} + +``` sql +DROP [ROW] POLICY [IF EXISTS] name [,...] ON [database.]table [,...] [ON CLUSTER cluster_name] +``` + + +## DROP QUOTA {#drop-quota-statement} + +Удаляет квоту. + +При удалении квота отзывается у всех объектов системы доступа, которым она присвоена. + +### Синтаксис {#drop-quota-syntax} + +``` sql +DROP QUOTA [IF EXISTS] name [,...] [ON CLUSTER cluster_name] +``` + + +## DROP SETTINGS PROFILE {#drop-settings-profile-statement} + +Удаляет профиль настроек. + +При удалении профиль отзывается у всех объектов системы доступа, которым он присвоен. + +### Синтаксис {#drop-settings-profile-syntax} + +``` sql +DROP [SETTINGS] PROFILE [IF EXISTS] name [,...] [ON CLUSTER cluster_name] +``` + + ## EXISTS {#exists} ``` sql @@ -109,7 +168,7 @@ EXISTS [TEMPORARY] TABLE [db.]name [INTO OUTFILE filename] [FORMAT format] Возвращает один столбец типа `UInt8`, содержащий одно значение - `0`, если таблицы или БД не существует и `1`, если таблица в указанной БД существует. -## KILL QUERY {#kill-query} +## KILL QUERY {#kill-query-statement} ``` sql KILL QUERY [ON CLUSTER cluster] @@ -144,7 +203,7 @@ Readonly-пользователи могут останавливать толь Тестовый вариант запроса (`TEST`) только проверяет права пользователя и выводит список запросов для остановки. -## KILL MUTATION {#kill-mutation} +## KILL MUTATION {#kill-mutation-statement} ``` sql KILL MUTATION [ON CLUSTER cluster] @@ -215,7 +274,57 @@ SET profile = 'profile-name-from-the-settings-file' Подробности смотрите в разделе [Настройки](../../operations/settings/settings.md). -## TRUNCATE {#truncate} +## SET ROLE {#set-role-statement} + +Активирует роли для текущего пользователя. + +### Синтаксис {#set-role-syntax} + +``` sql +SET ROLE {DEFAULT | NONE | role [,...] | ALL | ALL EXCEPT role [,...]} +``` + +## SET DEFAULT ROLE {#set-default-role-statement} + +Устанавливает роли по умолчанию для пользователя. + +Роли по умолчанию активируются автоматически при входе пользователя. Ролями по умолчанию могут быть установлены только ранее назначенные роли. Если роль не назначена пользователю, ClickHouse выбрасывает исключение. + + +### Синтаксис {#set-default-role-syntax} + +``` sql +SET DEFAULT ROLE {NONE | role [,...] | ALL | ALL EXCEPT role [,...]} TO {user|CURRENT_USER} [,...] +``` + + +### Примеры {#set-default-role-examples} + +Установить несколько ролей по умолчанию для пользователя: + +``` sql +SET DEFAULT ROLE role1, role2, ... TO user +``` + +Установить ролями по умолчанию все назначенные пользователю роли: + +``` sql +SET DEFAULT ROLE ALL TO user +``` + +Удалить роли по умолчанию для пользователя: + +``` sql +SET DEFAULT ROLE NONE TO user +``` + +Установить ролями по умолчанию все назначенные пользователю роли за исключением указанных: + +```sql +SET DEFAULT ROLE ALL EXCEPT role1, role2 TO user +``` + +## TRUNCATE {#truncate-statement} ``` sql TRUNCATE TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] diff --git a/docs/ru/sql-reference/statements/revoke.md b/docs/ru/sql-reference/statements/revoke.md deleted file mode 120000 index 4321fdb14a7..00000000000 --- a/docs/ru/sql-reference/statements/revoke.md +++ /dev/null @@ -1 +0,0 @@ -../../../en/sql-reference/statements/revoke.md \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/revoke.md b/docs/ru/sql-reference/statements/revoke.md new file mode 100644 index 00000000000..1d2928bb76e --- /dev/null +++ b/docs/ru/sql-reference/statements/revoke.md @@ -0,0 +1,43 @@ +# REVOKE + +Отзывает привилегии у пользователей или ролей. + +## Синтаксис {#revoke-syntax} + +**Отзыв привилегий у пользователей** + +``` sql +REVOKE [ON CLUSTER cluster_name] privilege[(column_name [,...])] [,...] ON {db.table|db.*|*.*|table|*} FROM {user | CURRENT_USER} [,...] | ALL | ALL EXCEPT {user | CURRENT_USER} [,...] +``` + +**Отзыв ролей у пользователей** + +``` sql +REVOKE [ON CLUSTER cluster_name] [ADMIN OPTION FOR] role [,...] FROM {user | role | CURRENT_USER} [,...] | ALL | ALL EXCEPT {user_name | role_name | CURRENT_USER} [,...] +``` + +## Описание {#revoke-description} + +Для отзыва привилегий можно использовать привилегию более широкой области действия. Например, если у пользователя есть привилегия `SELECT (x,y)`, администратор может отозвать ее с помощью одного из запросов: `REVOKE SELECT(x,y) ...`, `REVOKE SELECT * ...` или даже `REVOKE ALL PRIVILEGES ...`. + +### Частичный отзыв {#partial-revokes-dscr} + +Вы можете отозвать часть привилегии. Например, если у пользователя есть привилегия `SELECT *.*`, вы можете отозвать привилегию на чтение данных из какой-то таблицы или базы данных. + +## Примеры {#revoke-example} + +Присвоить пользователю `john` привилегию на `SELECT` из всех баз данных кроме `accounts`: + +``` sql +GRANT SELECT ON *.* TO john; +REVOKE SELECT ON accounts.* FROM john; +``` + +Присвоить пользователю `mira` привилегию на `SELECT` из всех столбцов таблицы `accounts.staff` кроме столбца `wage`: + +``` sql +GRANT SELECT ON accounts.staff TO mira; +REVOKE SELECT(wage) ON accounts.staff FROM mira; +``` + +[Оригинальная статья](https://clickhouse.tech/docs/en/operations/settings/settings/) diff --git a/docs/ru/sql-reference/statements/select/prewhere.md b/docs/ru/sql-reference/statements/select/prewhere.md index d83b7ca3b18..1c8595d8e0c 100644 --- a/docs/ru/sql-reference/statements/select/prewhere.md +++ b/docs/ru/sql-reference/statements/select/prewhere.md @@ -1,4 +1,4 @@ -# Секции PREWHERE {#prewhere-clause} +# Секция PREWHERE {#prewhere-clause} Prewhere — это оптимизация для более эффективного применения фильтрации. Она включена по умолчанию, даже если секция `PREWHERE` явно не указана. В этом случае работает автоматическое перемещение части выражения из [WHERE](where.md) до стадии prewhere. Роль секции `PREWHERE` только для управления этой оптимизацией, если вы думаете, что знаете, как сделать перемещение условия лучше, чем это происходит по умолчанию. diff --git a/docs/ru/sql-reference/statements/select/where.md b/docs/ru/sql-reference/statements/select/where.md index e38d36e6fa8..63d081db43d 100644 --- a/docs/ru/sql-reference/statements/select/where.md +++ b/docs/ru/sql-reference/statements/select/where.md @@ -1,6 +1,6 @@ -# Предложение WHERE {#select-where} +# Секция WHERE {#select-where} -Позволяет задать выражение, которое ClickHouse использует для фильтрации данных перед всеми другими действиями в запросе кроме выражений, содержащихся в секции [PREWHERE](prewhere.md#select-prewhere). Обычно, это выражение с логическими операторами. +Позволяет задать выражение, которое ClickHouse использует для фильтрации данных перед всеми другими действиями в запросе кроме выражений, содержащихся в секции [PREWHERE](prewhere.md#prewhere-clause). Обычно, это выражение с логическими операторами. Результат выражения должен иметь тип `UInt8`. diff --git a/docs/ru/sql-reference/statements/show.md b/docs/ru/sql-reference/statements/show.md index 86f7abd8a1b..72180435b9c 100644 --- a/docs/ru/sql-reference/statements/show.md +++ b/docs/ru/sql-reference/statements/show.md @@ -95,4 +95,78 @@ SHOW DICTIONARIES FROM db LIKE '%reg%' LIMIT 2 └──────────────┘ ``` + + +## SHOW GRANTS {#show-grants-statement} + +Выводит привилегии пользователя. + +### Синтаксис {#show-grants-syntax} + +``` sql +SHOW GRANTS [FOR user] +``` + +Если пользователь не задан, запрос возвращает привилегии текущего пользователя. + + + +## SHOW CREATE USER {#show-create-user-statement} + +Выводит параметры, использованные при [создании пользователя](create.md#create-user-statement). + +`SHOW CREATE USER` не возвращает пароль пользователя. + +### Синтаксис {#show-create-user-syntax} + +``` sql +SHOW CREATE USER [name | CURRENT_USER] +``` + + + +## SHOW CREATE ROLE {#show-create-role-statement} + +Выводит параметры, использованные при [создании роли](create.md#create-role-statement). + +### Синтаксис {#show-create-role-syntax} + +``` sql +SHOW CREATE ROLE name +``` + + + +## SHOW CREATE ROW POLICY {#show-create-row-policy-statement} + +Выводит параметры, использованные при [создании политики доступа к строкам](create.md#create-row-policy-statement). + +### Синтаксис {#show-create-row-policy-syntax} + +```sql +SHOW CREATE [ROW] POLICY name ON [database.]table +``` + + +## SHOW CREATE QUOTA {#show-create-quota-statement} + +Выводит параметры, использованные при [создании квоты](create.md#create-quota-statement). + +### Синтаксис {#show-create-row-policy-syntax} + +```sql +SHOW CREATE QUOTA [name | CURRENT] +``` + + +## SHOW CREATE SETTINGS PROFILE {#show-create-settings-profile-statement} + +Выводит параметры, использованные при [создании профиля настроек](create.md#create-settings-profile-statement). + +### Синтаксис {#show-create-row-policy-syntax} + +```sql +SHOW CREATE [SETTINGS] PROFILE name +``` + [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/show/) diff --git a/docs/ru/sql-reference/table-functions/index.md b/docs/ru/sql-reference/table-functions/index.md index 28f437a814a..6e09dd4e41c 100644 --- a/docs/ru/sql-reference/table-functions/index.md +++ b/docs/ru/sql-reference/table-functions/index.md @@ -5,7 +5,7 @@ toc_priority: 34 toc_title: "\u0412\u0432\u0435\u0434\u0435\u043D\u0438\u0435" --- -# Табличные функции {#tablichnye-funktsii} +# Табличные функции {#table-functions} Табличные функции — это метод создания таблиц. From f748021d9a0458b1857d2dbbcc9a61d47928228a Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 14 May 2020 20:42:32 +0300 Subject: [PATCH 276/738] Use libc-headers only for BUNDLED build (fixes gcc10 unbundled build) libstdc++ from gcc10 uses pthread_mutex_clocklock() which is not yet exists in contrib/libc-headres. P.S. I can prepare an update for libc-contrib --- cmake/linux/default_libs.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/linux/default_libs.cmake b/cmake/linux/default_libs.cmake index d18a996e2c9..0ecdfd2a3ad 100644 --- a/cmake/linux/default_libs.cmake +++ b/cmake/linux/default_libs.cmake @@ -21,7 +21,7 @@ set(CMAKE_C_STANDARD_LIBRARIES ${DEFAULT_LIBS}) # glibc-compatibility library relies to fixed version of libc headers # (because minor changes in function attributes between different glibc versions will introduce incompatibilities) # This is for x86_64. For other architectures we have separate toolchains. -if (ARCH_AMD64) +if (ARCH_AMD64 AND NOT_UNBUNDLED) set(CMAKE_C_STANDARD_INCLUDE_DIRECTORIES ${ClickHouse_SOURCE_DIR}/contrib/libc-headers/x86_64-linux-gnu ${ClickHouse_SOURCE_DIR}/contrib/libc-headers) set(CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES ${ClickHouse_SOURCE_DIR}/contrib/libc-headers/x86_64-linux-gnu ${ClickHouse_SOURCE_DIR}/contrib/libc-headers) endif () From b7d46fe979f7b747ea27ad7b1059193674ff762e Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 14 May 2020 20:42:32 +0300 Subject: [PATCH 277/738] Add missing headers for libstdc++ from gcc10 --- src/Interpreters/addMissingDefaults.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Interpreters/addMissingDefaults.h b/src/Interpreters/addMissingDefaults.h index d9d8d3d4f22..cc84f413b16 100644 --- a/src/Interpreters/addMissingDefaults.h +++ b/src/Interpreters/addMissingDefaults.h @@ -1,6 +1,7 @@ #pragma once #include +#include namespace DB From c059ee008519cff43b42022fc6f6a9d8ee86c56c Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 14 May 2020 20:42:32 +0300 Subject: [PATCH 278/738] Use is_trivial+is_standard_layout over is_pod (later is deprecated) --- src/Common/Dwarf.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/Dwarf.cpp b/src/Common/Dwarf.cpp index 6d3d8fdf0ee..7a697a2c9ef 100644 --- a/src/Common/Dwarf.cpp +++ b/src/Common/Dwarf.cpp @@ -104,7 +104,7 @@ namespace // Read (bitwise) one object of type T template -std::enable_if_t, T> read(std::string_view & sp) +std::enable_if_t && std::is_standard_layout_v, T> read(std::string_view & sp) { SAFE_CHECK(sp.size() >= sizeof(T), "underflow"); T x; From 2498041cc1389415a6be12f8e53cd9a6234c6212 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 15 May 2020 11:13:37 +0300 Subject: [PATCH 279/738] Avoid sending partially written files by the DistributedBlockOutputStream --- .../DistributedBlockOutputStream.cpp | 45 ++++++++++--------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/src/Storages/Distributed/DistributedBlockOutputStream.cpp index ae0af5f9cf4..55d8d4da9d8 100644 --- a/src/Storages/Distributed/DistributedBlockOutputStream.cpp +++ b/src/Storages/Distributed/DistributedBlockOutputStream.cpp @@ -574,31 +574,34 @@ void DistributedBlockOutputStream::writeToShard(const Block & block, const std:: first_file_tmp_path = tmp_path + file_name; - WriteBufferFromFile out{first_file_tmp_path}; - CompressedWriteBuffer compress{out}; - NativeBlockOutputStream stream{compress, ClickHouseRevision::get(), block.cloneEmpty()}; + /// Write batch to temporary location + { + WriteBufferFromFile out{first_file_tmp_path}; + CompressedWriteBuffer compress{out}; + NativeBlockOutputStream stream{compress, ClickHouseRevision::get(), block.cloneEmpty()}; - /// Prepare the header. - /// We wrap the header into a string for compatibility with older versions: - /// a shard will able to read the header partly and ignore other parts based on its version. - WriteBufferFromOwnString header_buf; - writeVarUInt(ClickHouseRevision::get(), header_buf); - writeStringBinary(query_string, header_buf); - context.getSettingsRef().serialize(header_buf); - context.getClientInfo().write(header_buf, ClickHouseRevision::get()); + /// Prepare the header. + /// We wrap the header into a string for compatibility with older versions: + /// a shard will able to read the header partly and ignore other parts based on its version. + WriteBufferFromOwnString header_buf; + writeVarUInt(ClickHouseRevision::get(), header_buf); + writeStringBinary(query_string, header_buf); + context.getSettingsRef().serialize(header_buf); + context.getClientInfo().write(header_buf, ClickHouseRevision::get()); - /// Add new fields here, for example: - /// writeVarUInt(my_new_data, header_buf); + /// Add new fields here, for example: + /// writeVarUInt(my_new_data, header_buf); - /// Write the header. - const StringRef header = header_buf.stringRef(); - writeVarUInt(DBMS_DISTRIBUTED_SIGNATURE_HEADER, out); - writeStringBinary(header, out); - writePODBinary(CityHash_v1_0_2::CityHash128(header.data, header.size), out); + /// Write the header. + const StringRef header = header_buf.stringRef(); + writeVarUInt(DBMS_DISTRIBUTED_SIGNATURE_HEADER, out); + writeStringBinary(header, out); + writePODBinary(CityHash_v1_0_2::CityHash128(header.data, header.size), out); - stream.writePrefix(); - stream.write(block); - stream.writeSuffix(); + stream.writePrefix(); + stream.write(block); + stream.writeSuffix(); + } // Create hardlink here to reuse increment number const std::string block_file_path(path + '/' + file_name); From ec1aff245a37afb7b5e9d1485b3c76a330c184eb Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 16 May 2020 01:11:59 +0300 Subject: [PATCH 280/738] Add an assert() in case of write into a file after close --- src/IO/WriteBufferFromFileDescriptor.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/IO/WriteBufferFromFileDescriptor.cpp b/src/IO/WriteBufferFromFileDescriptor.cpp index fe528bdf810..9710c5a979b 100644 --- a/src/IO/WriteBufferFromFileDescriptor.cpp +++ b/src/IO/WriteBufferFromFileDescriptor.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include @@ -90,6 +91,8 @@ WriteBufferFromFileDescriptor::~WriteBufferFromFileDescriptor() { if (fd >= 0) next(); + else + assert(!offset() && "attempt to write after close"); } catch (...) { From 7309cbbfda2cd980fcbbf7430ca81c2b38cd517e Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Sat, 16 May 2020 10:23:59 +0300 Subject: [PATCH 281/738] Try fix build. --- src/Functions/array/arrayUniq.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Functions/array/arrayUniq.cpp b/src/Functions/array/arrayUniq.cpp index 4b09e725f25..02129781c13 100644 --- a/src/Functions/array/arrayUniq.cpp +++ b/src/Functions/array/arrayUniq.cpp @@ -119,7 +119,8 @@ private: void FunctionArrayUniq::executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) { const ColumnArray::Offsets * offsets = nullptr; - size_t num_arguments = arguments.size(); + const size_t num_arguments = arguments.size(); + assert(num_arguments > 0); ColumnRawPtrs data_columns(num_arguments); Columns array_holders; From a9f64b4c1c0900be09ddf44cd97c4ad6670d562b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 12 May 2020 03:31:44 +0300 Subject: [PATCH 282/738] Ensure that the variance is nonnegative --- .../AggregateFunctionStatisticsSimple.h | 11 +++++++---- .../0_stateless/01278_variance_nonnegative.reference | 8 ++++++++ .../0_stateless/01278_variance_nonnegative.sql | 9 +++++++++ 3 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/01278_variance_nonnegative.reference create mode 100644 tests/queries/0_stateless/01278_variance_nonnegative.sql diff --git a/src/AggregateFunctions/AggregateFunctionStatisticsSimple.h b/src/AggregateFunctions/AggregateFunctionStatisticsSimple.h index 72c2ce014e4..0d214d5677c 100644 --- a/src/AggregateFunctions/AggregateFunctionStatisticsSimple.h +++ b/src/AggregateFunctions/AggregateFunctionStatisticsSimple.h @@ -82,14 +82,17 @@ struct VarMoments T NO_SANITIZE_UNDEFINED getPopulation() const { - return (m[2] - m[1] * m[1] / m[0]) / m[0]; + /// Due to numerical errors, the result can be slightly less than zero, + /// but it should be impossible. Trim to zero. + + return std::max(T{}, (m[2] - m[1] * m[1] / m[0]) / m[0]); } T NO_SANITIZE_UNDEFINED getSample() const { if (m[0] == 0) return std::numeric_limits::quiet_NaN(); - return (m[2] - m[1] * m[1] / m[0]) / (m[0] - 1); + return std::max(T{}, (m[2] - m[1] * m[1] / m[0]) / (m[0] - 1)); } T NO_SANITIZE_UNDEFINED getMoment3() const @@ -180,7 +183,7 @@ struct VarMomentsDecimal if (common::mulOverflow(getM(1), getM(1), tmp) || common::subOverflow(getM(2), NativeType(tmp / m0), tmp)) throw Exception("Decimal math overflow", ErrorCodes::DECIMAL_OVERFLOW); - return convertFromDecimal, DataTypeNumber>(tmp / m0, scale); + return std::max(Float64{}, convertFromDecimal, DataTypeNumber>(tmp / m0, scale)); } Float64 getSample(UInt32 scale) const @@ -194,7 +197,7 @@ struct VarMomentsDecimal if (common::mulOverflow(getM(1), getM(1), tmp) || common::subOverflow(getM(2), NativeType(tmp / m0), tmp)) throw Exception("Decimal math overflow", ErrorCodes::DECIMAL_OVERFLOW); - return convertFromDecimal, DataTypeNumber>(tmp / (m0 - 1), scale); + return std::max(Float64{}, convertFromDecimal, DataTypeNumber>(tmp / (m0 - 1), scale)); } Float64 getMoment3(UInt32 scale) const diff --git a/tests/queries/0_stateless/01278_variance_nonnegative.reference b/tests/queries/0_stateless/01278_variance_nonnegative.reference new file mode 100644 index 00000000000..405d3348775 --- /dev/null +++ b/tests/queries/0_stateless/01278_variance_nonnegative.reference @@ -0,0 +1,8 @@ +0 +0 +0 +0 +0 +0 +0 +0 diff --git a/tests/queries/0_stateless/01278_variance_nonnegative.sql b/tests/queries/0_stateless/01278_variance_nonnegative.sql new file mode 100644 index 00000000000..aa676d8b269 --- /dev/null +++ b/tests/queries/0_stateless/01278_variance_nonnegative.sql @@ -0,0 +1,9 @@ +SELECT varSamp(0.1) FROM numbers(1000000); +SELECT varPop(0.1) FROM numbers(1000000); +SELECT stddevSamp(0.1) FROM numbers(1000000); +SELECT stddevPop(0.1) FROM numbers(1000000); + +SELECT varSampStable(0.1) FROM numbers(1000000); +SELECT varPopStable(0.1) FROM numbers(1000000); +SELECT stddevSampStable(0.1) FROM numbers(1000000); +SELECT stddevPopStable(0.1) FROM numbers(1000000); From e0aaddb262f5acf6a0e25416b09f74787bdec783 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 12 May 2020 05:22:14 +0300 Subject: [PATCH 283/738] Fixed error --- src/AggregateFunctions/AggregateFunctionStatisticsSimple.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AggregateFunctions/AggregateFunctionStatisticsSimple.h b/src/AggregateFunctions/AggregateFunctionStatisticsSimple.h index 0d214d5677c..3920489897c 100644 --- a/src/AggregateFunctions/AggregateFunctionStatisticsSimple.h +++ b/src/AggregateFunctions/AggregateFunctionStatisticsSimple.h @@ -90,7 +90,7 @@ struct VarMoments T NO_SANITIZE_UNDEFINED getSample() const { - if (m[0] == 0) + if (m[0] <= 1) return std::numeric_limits::quiet_NaN(); return std::max(T{}, (m[2] - m[1] * m[1] / m[0]) / (m[0] - 1)); } From ce451bf396685af542561406d542ccedc59aa591 Mon Sep 17 00:00:00 2001 From: Igor Mineev Date: Sat, 16 May 2020 23:06:32 +0300 Subject: [PATCH 284/738] Fix incorrect Column byte size --- src/Columns/ColumnDecimal.h | 2 +- src/Columns/ColumnVector.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Columns/ColumnDecimal.h b/src/Columns/ColumnDecimal.h index 62e414a676b..1f56b7c4242 100644 --- a/src/Columns/ColumnDecimal.h +++ b/src/Columns/ColumnDecimal.h @@ -113,7 +113,7 @@ public: Field operator[](size_t n) const override { return DecimalField(data[n], scale); } - StringRef getRawData() const override { return StringRef(reinterpret_cast(data.data()), data.size()); } + StringRef getRawData() const override { return StringRef(reinterpret_cast(data.data()), byteSize()); } StringRef getDataAt(size_t n) const override { return StringRef(reinterpret_cast(&data[n]), sizeof(data[n])); } void get(size_t n, Field & res) const override { res = (*this)[n]; } bool getBool(size_t n) const override { return bool(data[n]); } diff --git a/src/Columns/ColumnVector.h b/src/Columns/ColumnVector.h index 3551efe890c..2fd177625cc 100644 --- a/src/Columns/ColumnVector.h +++ b/src/Columns/ColumnVector.h @@ -264,7 +264,7 @@ public: bool isFixedAndContiguous() const override { return true; } size_t sizeOfValueIfFixed() const override { return sizeof(T); } - StringRef getRawData() const override { return StringRef(reinterpret_cast(data.data()), data.size()); } + StringRef getRawData() const override { return StringRef(reinterpret_cast(data.data()), byteSize()); } bool structureEquals(const IColumn & rhs) const override From d948fd61d873364a434b55487bff102d097330cd Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Sat, 16 May 2020 23:15:19 +0300 Subject: [PATCH 285/738] try fix tests. --- src/Functions/concat.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/concat.cpp b/src/Functions/concat.cpp index 56152d2530f..5bdc070462c 100644 --- a/src/Functions/concat.cpp +++ b/src/Functions/concat.cpp @@ -117,7 +117,7 @@ private: void executeFormatImpl(Block & block, const ColumnNumbers & arguments, const size_t result, size_t input_rows_count) { const size_t num_arguments = arguments.size(); - assert(num_arguments > 2); + assert(num_arguments >= 2); auto c_res = ColumnString::create(); std::vector data(num_arguments); From 45b84d491e8528b1bacad29165306dec48de824b Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sat, 16 May 2020 23:31:17 +0300 Subject: [PATCH 286/738] small fixes of review remarks --- src/Disks/IDisk.h | 2 +- src/Disks/IVolume.cpp | 23 ------------- src/Disks/IVolume.h | 32 +++---------------- src/Disks/createVolume.cpp | 2 +- src/Disks/createVolume.h | 2 +- src/Storages/MergeTree/IMergeTreeDataPart.h | 2 +- .../MergeTree/IMergeTreeDataPartWriter.cpp | 5 +-- 7 files changed, 12 insertions(+), 56 deletions(-) diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index b89128e7ded..011c75402f4 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -24,7 +24,7 @@ class IDiskDirectoryIterator; using DiskDirectoryIteratorPtr = std::unique_ptr; class IReservation; -using ReservationPtr = std::shared_ptr; +using ReservationPtr = std::unique_ptr; class ReadBufferFromFileBase; class WriteBufferFromFileBase; diff --git a/src/Disks/IVolume.cpp b/src/Disks/IVolume.cpp index 4da9aec1ab9..a71afa4a840 100644 --- a/src/Disks/IVolume.cpp +++ b/src/Disks/IVolume.cpp @@ -13,29 +13,6 @@ namespace ErrorCodes extern const int UNKNOWN_VOLUME_TYPE; } -void VolumeType::fromString(const String & str) -{ - if (str == "JBOD") - value = JBOD; - else if (str == "SINGLE_DISK") - value = SINGLE_DISK; - else - throw DB::Exception("Unexpected string for volume type: " + str, ErrorCodes::UNKNOWN_VOLUME_TYPE); -} - -String VolumeType::toString() const -{ - switch (value) - { - case JBOD: - return "JBOD"; - case SINGLE_DISK: - return "SINGLE_DISK"; - default: - return "Unknown"; - } -} - IVolume::IVolume( String name_, const Poco::Util::AbstractConfiguration & config, const String & config_prefix, DiskSelectorPtr disk_selector) : name(std::move(name_)) diff --git a/src/Disks/IVolume.h b/src/Disks/IVolume.h index 0b1c0a20ad9..a762958a33f 100644 --- a/src/Disks/IVolume.h +++ b/src/Disks/IVolume.h @@ -8,33 +8,11 @@ namespace DB { -class VolumeType +enum class VolumeType { -public: - enum Value - { - JBOD, - SINGLE_DISK, - UNKNOWN - }; - VolumeType() : value(UNKNOWN) {} - VolumeType(Value value_) : value(value_) {} - - bool operator==(const VolumeType & other) const - { - return value == other.value; - } - - bool operator!=(const VolumeType & other) const - { - return !(*this == other); - } - - void fromString(const String & str); - String toString() const; - -private: - Value value; + JBOD, + SINGLE_DISK, + UNKNOWN }; class IVolume; @@ -55,7 +33,7 @@ using Volumes = std::vector; class IVolume : public Space { public: - IVolume(const String & name_, Disks disks_): disks(std::move(disks_)), name(name_) + IVolume(String name_, Disks disks_): disks(std::move(disks_)), name(name_) { } diff --git a/src/Disks/createVolume.cpp b/src/Disks/createVolume.cpp index 3faca365bd4..d7aa7d3c9ad 100644 --- a/src/Disks/createVolume.cpp +++ b/src/Disks/createVolume.cpp @@ -3,7 +3,7 @@ namespace DB { -VolumePtr createVolumeFromReservation(ReservationPtr reservation, VolumePtr other_volume) +VolumePtr createVolumeFromReservation(const ReservationPtr & reservation, VolumePtr other_volume) { if (other_volume->getType() == VolumeType::JBOD || other_volume->getType() == VolumeType::SINGLE_DISK) { diff --git a/src/Disks/createVolume.h b/src/Disks/createVolume.h index 9180d23ea73..52085ec16bc 100644 --- a/src/Disks/createVolume.h +++ b/src/Disks/createVolume.h @@ -7,6 +7,6 @@ namespace DB { -VolumePtr createVolumeFromReservation(ReservationPtr reservation, VolumePtr other_volume); +VolumePtr createVolumeFromReservation(const ReservationPtr & reservation, VolumePtr other_volume); } diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index a2fe9661832..e1b276a4dd7 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -29,7 +29,7 @@ struct ColumnSize; class MergeTreeData; struct FutureMergedMutatedPart; class IReservation; -using ReservationPtr = std::shared_ptr; +using ReservationPtr = std::unique_ptr; class IVolume; using VolumePtr = std::shared_ptr; diff --git a/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp b/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp index 239ccba1d89..763f4fff879 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp @@ -85,8 +85,9 @@ IMergeTreeDataPartWriter::IMergeTreeDataPartWriter( if (settings.blocks_are_granules_size && !index_granularity.empty()) throw Exception("Can't take information about index granularity from blocks, when non empty index_granularity array specified", ErrorCodes::LOGICAL_ERROR); - if (!data_part->volume->getDisk()->exists(part_path)) - data_part->volume->getDisk()->createDirectories(part_path); + auto disk = data_part->volume->getDisk(); + if (!disk->exists(part_path)) + disk->createDirectories(part_path); } IMergeTreeDataPartWriter::~IMergeTreeDataPartWriter() = default; From a56aba0211bfed32f8f8e3f467928c7a940e4fda Mon Sep 17 00:00:00 2001 From: myrrc Date: Sun, 17 May 2020 02:38:30 +0300 Subject: [PATCH 287/738] gtest macro find fix --- src/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5ea862b7db4..f6b33f49d4b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -365,7 +365,7 @@ if (ENABLE_TESTS AND USE_GTEST) endmacro() # attach all dbms gtest sources - grep_gtest_sources(${ClickHouse_SOURCE_DIR}/dbms dbms_gtest_sources) + grep_gtest_sources(${ClickHouse_SOURCE_DIR}/src dbms_gtest_sources) add_executable(unit_tests_dbms ${dbms_gtest_sources}) # gtest framework has substandard code From 791ccd8b05c263bd799d42972e31a2b39ced2208 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Sun, 17 May 2020 04:01:26 +0300 Subject: [PATCH 288/738] never throw from destructor --- src/Databases/DatabaseAtomic.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp index bafa65b10ba..09ff1b8944e 100644 --- a/src/Databases/DatabaseAtomic.cpp +++ b/src/Databases/DatabaseAtomic.cpp @@ -7,6 +7,7 @@ #include #include #include +#include namespace DB @@ -227,7 +228,7 @@ void DatabaseAtomic::commitCreateTable(const ASTCreateQuery & query, const Stora void DatabaseAtomic::commitAlterTable(const StorageID & table_id, const String & table_metadata_tmp_path, const String & table_metadata_path) { - SCOPE_EXIT({ Poco::File(table_metadata_tmp_path).remove(); }); + SCOPE_EXIT({ std::error_code code; std::filesystem::remove(table_metadata_tmp_path, code); }); std::unique_lock lock{mutex}; auto actual_table_id = getTableUnlocked(table_id.table_name, lock)->getStorageID(); From f89bf6155b4ced0832bc6949c5aa47d3d009011b Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Sun, 17 May 2020 04:15:59 +0300 Subject: [PATCH 289/738] fix bad test --- .../0_stateless/01107_atomic_db_detach_attach.reference | 2 ++ tests/queries/0_stateless/01107_atomic_db_detach_attach.sh | 5 +++-- tests/queries/0_stateless/01114_database_atomic.reference | 1 + tests/queries/0_stateless/01114_database_atomic.sh | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/01107_atomic_db_detach_attach.reference b/tests/queries/0_stateless/01107_atomic_db_detach_attach.reference index bf34eef313a..b78efbbb592 100644 --- a/tests/queries/0_stateless/01107_atomic_db_detach_attach.reference +++ b/tests/queries/0_stateless/01107_atomic_db_detach_attach.reference @@ -2,3 +2,5 @@ OK OK 5 10 5 10 +dropped +end diff --git a/tests/queries/0_stateless/01107_atomic_db_detach_attach.sh b/tests/queries/0_stateless/01107_atomic_db_detach_attach.sh index 6ff36318d30..5fa6e194670 100755 --- a/tests/queries/0_stateless/01107_atomic_db_detach_attach.sh +++ b/tests/queries/0_stateless/01107_atomic_db_detach_attach.sh @@ -21,6 +21,7 @@ $CLICKHOUSE_CLIENT -q "DETACH DATABASE test_01107" $CLICKHOUSE_CLIENT --allow_experimental_database_atomic=1 -q "ATTACH DATABASE test_01107 ENGINE=Atomic" $CLICKHOUSE_CLIENT -q "SELECT count(n), sum(n) FROM test_01107.mt" -$CLICKHOUSE_CLIENT -q "INSERT INTO test_01107.mt SELECT number + sleepEachRow(3) FROM numbers(100)" & +$CLICKHOUSE_CLIENT -q "INSERT INTO test_01107.mt SELECT number + sleepEachRow(1) FROM numbers(5)" && echo "end" & sleep 1 -$CLICKHOUSE_CLIENT -q "DROP DATABASE test_01107" +$CLICKHOUSE_CLIENT -q "DROP DATABASE test_01107" && sleep 1 && echo "dropped" +wait diff --git a/tests/queries/0_stateless/01114_database_atomic.reference b/tests/queries/0_stateless/01114_database_atomic.reference index a6c9fc3c4f4..dece5bd9b8b 100644 --- a/tests/queries/0_stateless/01114_database_atomic.reference +++ b/tests/queries/0_stateless/01114_database_atomic.reference @@ -12,5 +12,6 @@ mt 00001114-0000-4000-8000-000000000002 CREATE TABLE test_01114_2.mt UUID \'0000 CREATE TABLE test_01114_1.mt UUID \'00001114-0000-4000-8000-000000000001\'\n(\n `n` UInt64\n)\nENGINE = MergeTree()\nPARTITION BY n % 5\nORDER BY tuple()\nSETTINGS index_granularity = 8192 CREATE TABLE test_01114_2.mt UUID \'00001114-0000-4000-8000-000000000002\'\n(\n `n` UInt64\n)\nENGINE = MergeTree()\nPARTITION BY n % 5\nORDER BY tuple()\nSETTINGS index_granularity = 8192 5 +dropped 20 190 30 435 diff --git a/tests/queries/0_stateless/01114_database_atomic.sh b/tests/queries/0_stateless/01114_database_atomic.sh index 0ef7844b2d1..58c5b2d287d 100755 --- a/tests/queries/0_stateless/01114_database_atomic.sh +++ b/tests/queries/0_stateless/01114_database_atomic.sh @@ -57,7 +57,7 @@ $CLICKHOUSE_CLIENT -q "SELECT count() FROM test_01114_1.mt" # result: 5 $CLICKHOUSE_CLIENT -q "SELECT tuple(s, sleepEachRow(3)) FROM test_01114_1.mt" > /dev/null & # 15s sleep 1 -$CLICKHOUSE_CLIENT -q "DROP DATABASE test_01114_1" +$CLICKHOUSE_CLIENT -q "DROP DATABASE test_01114_1" && echo "dropped" wait # for INSERT From 3320fdce57e3d957d171935f2b9aa380f8d7dc31 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 17 May 2020 05:39:02 +0300 Subject: [PATCH 290/738] Fix error --- .../AggregateFunctionStatisticsSimple.h | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionStatisticsSimple.h b/src/AggregateFunctions/AggregateFunctionStatisticsSimple.h index 3920489897c..96c07cc3d41 100644 --- a/src/AggregateFunctions/AggregateFunctionStatisticsSimple.h +++ b/src/AggregateFunctions/AggregateFunctionStatisticsSimple.h @@ -80,23 +80,28 @@ struct VarMoments readPODBinary(*this, buf); } - T NO_SANITIZE_UNDEFINED getPopulation() const + T getPopulation() const { + if (m[0] == 0) + return std::numeric_limits::quiet_NaN(); + /// Due to numerical errors, the result can be slightly less than zero, /// but it should be impossible. Trim to zero. return std::max(T{}, (m[2] - m[1] * m[1] / m[0]) / m[0]); } - T NO_SANITIZE_UNDEFINED getSample() const + T getSample() const { if (m[0] <= 1) return std::numeric_limits::quiet_NaN(); return std::max(T{}, (m[2] - m[1] * m[1] / m[0]) / (m[0] - 1)); } - T NO_SANITIZE_UNDEFINED getMoment3() const + T getMoment3() const { + if (m[0] == 0) + return std::numeric_limits::quiet_NaN(); // to avoid accuracy problem if (m[0] == 1) return 0; @@ -107,8 +112,10 @@ struct VarMoments ) / m[0]; } - T NO_SANITIZE_UNDEFINED getMoment4() const + T getMoment4() const { + if (m[0] == 0) + return std::numeric_limits::quiet_NaN(); // to avoid accuracy problem if (m[0] == 1) return 0; From 7e24492fb90b03cdb654bbffc3f37b2a4aac1475 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 17 May 2020 06:16:34 +0300 Subject: [PATCH 291/738] Added performance test --- tests/performance/decimal_parse.xml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 tests/performance/decimal_parse.xml diff --git a/tests/performance/decimal_parse.xml b/tests/performance/decimal_parse.xml new file mode 100644 index 00000000000..19e940b13df --- /dev/null +++ b/tests/performance/decimal_parse.xml @@ -0,0 +1,3 @@ + + SELECT count() FROM zeros(10000000) WHERE NOT ignore(toDecimal32OrZero(toString(rand() % 10000), 5)) + From 0636ea9f3891265636dc37e4bd070ef0f0019a62 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 17 May 2020 06:16:58 +0300 Subject: [PATCH 292/738] Add overflow check in Decimal parse --- src/DataTypes/DataTypesDecimal.cpp | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/DataTypes/DataTypesDecimal.cpp b/src/DataTypes/DataTypesDecimal.cpp index b45e26bd5bb..43230f95f63 100644 --- a/src/DataTypes/DataTypesDecimal.cpp +++ b/src/DataTypes/DataTypesDecimal.cpp @@ -58,24 +58,30 @@ void DataTypeDecimal::serializeText(const IColumn & column, size_t row_num, W } template -bool NO_SANITIZE_UNDEFINED DataTypeDecimal::tryReadText(T & x, ReadBuffer & istr, UInt32 precision, UInt32 scale) +bool DataTypeDecimal::tryReadText(T & x, ReadBuffer & istr, UInt32 precision, UInt32 scale) { UInt32 unread_scale = scale; bool done = tryReadDecimalText(istr, x, precision, unread_scale); + if (!done) + return false; - x *= T::getScaleMultiplier(unread_scale); - return done; + if (common::mulOverflow(x.value, T::getScaleMultiplier(unread_scale), x.value)) + return false; + + return true; } template -void NO_SANITIZE_UNDEFINED DataTypeDecimal::readText(T & x, ReadBuffer & istr, UInt32 precision, UInt32 scale, bool csv) +void DataTypeDecimal::readText(T & x, ReadBuffer & istr, UInt32 precision, UInt32 scale, bool csv) { UInt32 unread_scale = scale; if (csv) readCSVDecimalText(istr, x, precision, unread_scale); else readDecimalText(istr, x, precision, unread_scale); - x *= T::getScaleMultiplier(unread_scale); + + if (common::mulOverflow(x.value, T::getScaleMultiplier(unread_scale), x.value)) + throw Exception("Decimal math overflow", ErrorCodes::DECIMAL_OVERFLOW); } template @@ -95,13 +101,15 @@ void DataTypeDecimal::deserializeTextCSV(IColumn & column, ReadBuffer & istr, } template -T NO_SANITIZE_UNDEFINED DataTypeDecimal::parseFromString(const String & str) const +T DataTypeDecimal::parseFromString(const String & str) const { ReadBufferFromMemory buf(str.data(), str.size()); T x; UInt32 unread_scale = this->scale; readDecimalText(buf, x, this->precision, unread_scale, true); - x *= T::getScaleMultiplier(unread_scale); + + if (common::mulOverflow(x.value, T::getScaleMultiplier(unread_scale), x.value)) + throw Exception("Decimal math overflow", ErrorCodes::DECIMAL_OVERFLOW); return x; } From f8a6fa87287c860ab3ae600752fc8f86e9e933f0 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 17 May 2020 06:18:06 +0300 Subject: [PATCH 293/738] Update test --- tests/queries/0_stateless/01260_ubsan_decimal_parse.reference | 2 +- tests/queries/0_stateless/01260_ubsan_decimal_parse.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01260_ubsan_decimal_parse.reference b/tests/queries/0_stateless/01260_ubsan_decimal_parse.reference index 573541ac970..945da8ffd36 100644 --- a/tests/queries/0_stateless/01260_ubsan_decimal_parse.reference +++ b/tests/queries/0_stateless/01260_ubsan_decimal_parse.reference @@ -1 +1 @@ -0 +0.000000 diff --git a/tests/queries/0_stateless/01260_ubsan_decimal_parse.sql b/tests/queries/0_stateless/01260_ubsan_decimal_parse.sql index faf98952a17..2c7cda512e8 100644 --- a/tests/queries/0_stateless/01260_ubsan_decimal_parse.sql +++ b/tests/queries/0_stateless/01260_ubsan_decimal_parse.sql @@ -1 +1 @@ -SELECT ignore(toDecimal32OrZero(CAST(-7174046, 'String'), 6)); +SELECT toDecimal32OrZero(CAST(-7174046, 'String'), 6); From a5d35bfdfb9cf403e0c5c079780321aa0fe1c2bb Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 17 May 2020 06:18:52 +0300 Subject: [PATCH 294/738] Better code --- src/DataTypes/DataTypesDecimal.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/DataTypes/DataTypesDecimal.cpp b/src/DataTypes/DataTypesDecimal.cpp index 43230f95f63..6b085d0e1ed 100644 --- a/src/DataTypes/DataTypesDecimal.cpp +++ b/src/DataTypes/DataTypesDecimal.cpp @@ -61,8 +61,7 @@ template bool DataTypeDecimal::tryReadText(T & x, ReadBuffer & istr, UInt32 precision, UInt32 scale) { UInt32 unread_scale = scale; - bool done = tryReadDecimalText(istr, x, precision, unread_scale); - if (!done) + if (!tryReadDecimalText(istr, x, precision, unread_scale)) return false; if (common::mulOverflow(x.value, T::getScaleMultiplier(unread_scale), x.value)) From 6b650d0e472ef8b3b1d6dae50bc9ff1277fd7e16 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 17 May 2020 06:57:16 +0300 Subject: [PATCH 295/738] Fix mistake in FreeBSD toolchain --- cmake/freebsd/toolchain-x86_64.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/freebsd/toolchain-x86_64.cmake b/cmake/freebsd/toolchain-x86_64.cmake index 0961250ef8e..d9839ec74ee 100644 --- a/cmake/freebsd/toolchain-x86_64.cmake +++ b/cmake/freebsd/toolchain-x86_64.cmake @@ -1,8 +1,8 @@ set (CMAKE_SYSTEM_NAME "FreeBSD") set (CMAKE_SYSTEM_PROCESSOR "x86_64") -set (CMAKE_C_COMPILER_TARGET "x86_64-pc-freebsd12.1") -set (CMAKE_CXX_COMPILER_TARGET "x86_64-pc-freebsd12.1") -set (CMAKE_ASM_COMPILER_TARGET "x86_64-pc-freebsd12.1") +set (CMAKE_C_COMPILER_TARGET "x86_64-pc-freebsd11") +set (CMAKE_CXX_COMPILER_TARGET "x86_64-pc-freebsd11") +set (CMAKE_ASM_COMPILER_TARGET "x86_64-pc-freebsd11") set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../toolchain/freebsd-x86_64") set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) # disable linkage check - it doesn't work in CMake From 88ee76186b57a04e0c1a81159d8e47e06a1c2f5d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 17 May 2020 07:12:33 +0300 Subject: [PATCH 296/738] Try to fix use-after-free error in MergeTree --- src/Storages/StorageMergeTree.cpp | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 68d468233a8..91add9a8104 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -93,9 +93,18 @@ void StorageMergeTree::startup() /// NOTE background task will also do the above cleanups periodically. time_after_previous_cleanup.restart(); - merging_mutating_task_handle = global_context.getBackgroundPool().addTask([this] { return mergeMutateTask(); }); + + auto & pool = global_context.getBackgroundPool(); + + merging_mutating_task_handle = pool.createTask([this] { return mergeMutateTask(); }); + /// Ensure that thread started only after assignment to 'merging_mutating_task_handle' is done. + pool.startTask(merging_mutating_task_handle); + if (areBackgroundMovesNeeded()) - moving_task_handle = global_context.getBackgroundMovePool().addTask([this] { return movePartsTask(); }); + { + moving_task_handle = pool.createTask([this] { return movePartsTask(); }); + pool.startTask(moving_task_handle); + } } From 675509ed2392ee142012b33259509be3bcbb453b Mon Sep 17 00:00:00 2001 From: Andrei Nekrashevich Date: Sun, 17 May 2020 07:43:53 +0300 Subject: [PATCH 297/738] Added function randomStringUTF8 --- src/Functions/randomPrintableASCII.cpp | 23 ++- src/Functions/randomString.cpp | 7 +- src/Functions/randomStringUTF8.cpp | 153 ++++++++++++++++++ src/Functions/registerFunctionsRandom.cpp | 2 + tests/performance/random_string_utf8.xml | 12 ++ .../01278_random_string_utf8.reference | 4 + .../0_stateless/01278_random_string_utf8.sql | 8 + 7 files changed, 193 insertions(+), 16 deletions(-) create mode 100644 src/Functions/randomStringUTF8.cpp create mode 100644 tests/performance/random_string_utf8.xml create mode 100644 tests/queries/0_stateless/01278_random_string_utf8.reference create mode 100644 tests/queries/0_stateless/01278_random_string_utf8.sql diff --git a/src/Functions/randomPrintableASCII.cpp b/src/Functions/randomPrintableASCII.cpp index bd0e84001ab..fedaeb7b9ef 100644 --- a/src/Functions/randomPrintableASCII.cpp +++ b/src/Functions/randomPrintableASCII.cpp @@ -1,15 +1,14 @@ -#include -#include -#include #include #include +#include +#include +#include #include #include namespace DB { - namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; @@ -27,10 +26,7 @@ public: static constexpr auto name = "randomPrintableASCII"; static FunctionPtr create(const Context &) { return std::make_shared(); } - String getName() const override - { - return name; - } + String getName() const override { return name; } bool isVariadic() const override { return true; } size_t getNumberOfArguments() const override { return 0; } @@ -38,11 +34,13 @@ public: DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { if (arguments.empty()) - throw Exception("Function " + getName() + " requires at least one argument: the size of resulting string", + throw Exception( + "Function " + getName() + " requires at least one argument: the size of resulting string", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); if (arguments.size() > 2) - throw Exception("Function " + getName() + " requires at most two arguments: the size of resulting string and optional disambiguation tag", + throw Exception( + "Function " + getName() + " requires at most two arguments: the size of resulting string and optional disambiguation tag", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); const IDataType & length_type = *arguments[0]; @@ -77,8 +75,9 @@ public: data_to.resize(next_offset); offsets_to[row_num] = next_offset; - auto * data_to_ptr = data_to.data(); /// avoid assert on array indexing after end - for (size_t pos = offset, end = offset + length; pos < end; pos += 4) /// We have padding in column buffers that we can overwrite. + auto * data_to_ptr = data_to.data(); /// avoid assert on array indexing after end + for (size_t pos = offset, end = offset + length; pos < end; + pos += 4) /// We have padding in column buffers that we can overwrite. { UInt64 rand = rng(); diff --git a/src/Functions/randomString.cpp b/src/Functions/randomString.cpp index 2b95825d449..1c2d595210b 100644 --- a/src/Functions/randomString.cpp +++ b/src/Functions/randomString.cpp @@ -4,8 +4,8 @@ #include #include #include -#include #include +#include namespace DB @@ -44,9 +44,8 @@ public: "Function " + getName() + " requires at most two arguments: the size of resulting string and optional disambiguation tag", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - const IDataType & length_type = *arguments[0]; - if (!isNumber(length_type)) - throw Exception("First argument of function " + getName() + " must have numeric type", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if (!isUnsignedInteger(*arguments[0])) + throw Exception("First argument for function " + getName() + " must be unsigned integer", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); return std::make_shared(); } diff --git a/src/Functions/randomStringUTF8.cpp b/src/Functions/randomStringUTF8.cpp new file mode 100644 index 00000000000..db8b5028047 --- /dev/null +++ b/src/Functions/randomStringUTF8.cpp @@ -0,0 +1,153 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int TOO_LARGE_STRING_SIZE; +} + + +/* Generate string with a UTF-8 encoded text. + * Take a single argument - length of result string in Unicode code points. + * ATTENTION: Method generate only assignable code points(excluded 4-13 planes). + * See https://en.wikipedia.org/wiki/Plane_(Unicode) */ + +class FunctionRandomStringUTF8 : public IFunction +{ +public: + static constexpr auto name = "randomStringUTF8"; + + static FunctionPtr create(const Context &) { return std::make_shared(); } + + String getName() const override { return name; } + + bool isVariadic() const override { return false; } + + size_t getNumberOfArguments() const override { return 1; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + if (!isUnsignedInteger(*arguments[0])) + throw Exception("First argument for function " + getName() + " must be unsigned integer", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + return std::make_shared(); + } + + bool isDeterministic() const override { return false; } + bool isDeterministicInScopeOfQuery() const override { return false; } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + { + auto col_to = ColumnString::create(); + ColumnString::Chars & data_to = col_to->getChars(); + ColumnString::Offsets & offsets_to = col_to->getOffsets(); + + if (input_rows_count == 0) + { + block.getByPosition(result).column = std::move(col_to); + return; + } + + offsets_to.resize(input_rows_count); + + const IColumn & length_column = *block.getByPosition(arguments[0]).column; + size_t summary_utf8_len = 0; + for (size_t row_num = 0; row_num < input_rows_count; ++row_num) + { + size_t utf8_len = length_column.getUInt(row_num); + summary_utf8_len += utf8_len; + } + + /* As we generate only assigned planes, the mathematical expectation of the number of bytes + * per generated code point ~= 3.85. So, reserving for coefficient 4 will not be an overhead + */ + + if (summary_utf8_len > (1 << 29)) + throw Exception("Too large string size in function " + getName(), ErrorCodes::TOO_LARGE_STRING_SIZE); + + size_t size_in_bytes_with_margin = summary_utf8_len * 4 + input_rows_count; + data_to.resize(size_in_bytes_with_margin); + pcg64_fast rng(randomSeed()); /// TODO It is inefficient. We should use SIMD PRNG instead. + + auto generate_code_point = [](UInt32 rand) { + /// We want to generate number in [0x0, 0x70000) and shift it if need + + /// Generate highest byte in [0, 6] + /// https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ + UInt32 code_point = (rand >> 16) * 7u; + code_point &= ((-1) ^ 0xFFFF); + code_point |= rand & 0xFFFF; // and other bytes obtaining in a simple way + + if (code_point >= 0x40000) + { + code_point += 0xa0000; // shift if it is in 14-16 plane + return code_point; + } + + if (0xD7FF < code_point && code_point < 0xE000) // this range will not be valid in isValidUTF8 + { + /* TODO(reviewer) choose with @axolm variant: + * 1. Not to do this if (isValidUTF8 can return 0) + * 2. just return 0 + * 3. capture rng in lambda and do while(code_point is bad) { recalc... } + * 4. ... + * */ + return 0u; + } + return code_point; + }; + + IColumn::Offset offset = 0; + for (size_t row_num = 0; row_num < input_rows_count; ++row_num) + { + size_t utf8_len = length_column.getUInt(row_num); + auto * pos = data_to.data() + offset; + + size_t last_writed_bytes = 0; + size_t i = 0; + for (; i < utf8_len; i += 2) + { + UInt64 rand = rng(); + + UInt32 code_point1 = generate_code_point(rand); + UInt32 code_point2 = generate_code_point(rand >> 32); + + /// We have padding in column buffers that we can overwrite. + pos += UTF8::convert(*reinterpret_cast(&code_point1), pos, sizeof(int)); + last_writed_bytes = UTF8::convert(*reinterpret_cast(&code_point2), pos, sizeof(int)); + pos += last_writed_bytes; + } + offset = pos - data_to.data() + 1; + if (i > utf8_len) + { + offset -= last_writed_bytes; + } + offsets_to[row_num] = offset; + } + + + /// Put zero bytes in between. + auto * pos = data_to.data(); + for (size_t row_num = 0; row_num < input_rows_count; ++row_num) + pos[offsets_to[row_num] - 1] = 0; + + block.getByPosition(result).column = std::move(col_to); + } +}; + +void registerFunctionRandomStringUTF8(FunctionFactory & factory) +{ + factory.registerFunction(); +} +} diff --git a/src/Functions/registerFunctionsRandom.cpp b/src/Functions/registerFunctionsRandom.cpp index 5826fe78419..1b81fc9e534 100644 --- a/src/Functions/registerFunctionsRandom.cpp +++ b/src/Functions/registerFunctionsRandom.cpp @@ -8,6 +8,7 @@ void registerFunctionRandConstant(FunctionFactory & factory); void registerFunctionGenerateUUIDv4(FunctionFactory & factory); void registerFunctionRandomPrintableASCII(FunctionFactory & factory); void registerFunctionRandomString(FunctionFactory & factory); +void registerFunctionRandomStringUTF8(FunctionFactory & factory); void registerFunctionsRandom(FunctionFactory & factory) { @@ -17,6 +18,7 @@ void registerFunctionsRandom(FunctionFactory & factory) registerFunctionGenerateUUIDv4(factory); registerFunctionRandomPrintableASCII(factory); registerFunctionRandomString(factory); + registerFunctionRandomStringUTF8(factory); } } diff --git a/tests/performance/random_string_utf8.xml b/tests/performance/random_string_utf8.xml new file mode 100644 index 00000000000..60e3b69e79f --- /dev/null +++ b/tests/performance/random_string_utf8.xml @@ -0,0 +1,12 @@ + + + + + SELECT count() FROM zeros(1000000) WHERE NOT ignore(randomStringUTF8(10)) + SELECT count() FROM zeros(1000000) WHERE NOT ignore(randomStringUTF8(100)) + SELECT count() FROM zeros(100000) WHERE NOT ignore(randomStringUTF8(1000)) + SELECT count() FROM zeros(10000) WHERE NOT ignore(randomStringUTF8(10000)) + SELECT count() FROM zeros(10000000) WHERE NOT ignore(randomStringUTF8(rand() % 10)) + SELECT count() FROM zeros(10000000) WHERE NOT ignore(randomStringUTF8(rand() % 100)) + SELECT count() FROM zeros(1000000) WHERE NOT ignore(randomStringUTF8(rand() % 1000)) + diff --git a/tests/queries/0_stateless/01278_random_string_utf8.reference b/tests/queries/0_stateless/01278_random_string_utf8.reference new file mode 100644 index 00000000000..36ae0ace76a --- /dev/null +++ b/tests/queries/0_stateless/01278_random_string_utf8.reference @@ -0,0 +1,4 @@ +100 +String +1 + diff --git a/tests/queries/0_stateless/01278_random_string_utf8.sql b/tests/queries/0_stateless/01278_random_string_utf8.sql new file mode 100644 index 00000000000..72b7a273f56 --- /dev/null +++ b/tests/queries/0_stateless/01278_random_string_utf8.sql @@ -0,0 +1,8 @@ +SELECT randomStringUTF8('string'); -- { serverError 43 } +SELECT randomStringUTF8(-10); -- { serverError 43 } +SELECT lengthUTF8(randomStringUTF8(100)); +SELECT toTypeName(randomStringUTF8(10)); +SELECT isValidUTF8(randomStringUTF8(100000)); +SELECT randomStringUTF8(0); +-- SELECT DISTINCT c > 30000 FROM (SELECT arrayJoin(arrayMap(x -> reinterpretAsUInt8(substring(randomStringUTF8(100), x + 1, 1)), range(100))) AS byte, count() AS c FROM numbers(100000) GROUP BY byte ORDER BY byte); + From 9fcfe5fb14727c07263be3a5ca27ab305375c5ae Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 27 Apr 2020 16:54:31 +0300 Subject: [PATCH 298/738] Better exception message and error code for ALTER of key column #9078 --- src/Common/ErrorCodes.cpp | 1 + src/Storages/MergeTree/MergeTreeData.cpp | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 374315005de..a8bd41162e3 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -495,6 +495,7 @@ namespace ErrorCodes extern const int ATOMIC_RENAME_FAIL = 521; extern const int OPENCL_ERROR = 522; extern const int UNKNOWN_ROW_POLICY = 523; + extern const int ALTER_OF_COLUMN_IS_FORBIDDEN = 524; extern const int KEEPER_EXCEPTION = 999; extern const int POCO_EXCEPTION = 1000; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index edc492efdbe..50a5a209546 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -109,6 +109,7 @@ namespace ErrorCodes extern const int UNEXPECTED_AST_STRUCTURE; extern const int UNKNOWN_DISK; extern const int NOT_ENOUGH_SPACE; + extern const int ALTER_OF_COLUMN_IS_FORBIDDEN; } @@ -1479,7 +1480,7 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, const S else if (command.isModifyingData(getInMemoryMetadata())) { if (columns_alter_type_forbidden.count(command.column_name)) - throw Exception("Trying to ALTER key column " + command.column_name, ErrorCodes::ILLEGAL_COLUMN); + throw Exception("ALTER of key column " + command.column_name + " is forbidden", ErrorCodes::ALTER_OF_COLUMN_IS_FORBIDDEN); if (columns_alter_type_metadata_only.count(command.column_name)) { @@ -1487,7 +1488,8 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, const S { auto it = old_types.find(command.column_name); if (it == old_types.end() || !isMetadataOnlyConversion(it->second, command.data_type.get())) - throw Exception("ALTER of key column " + command.column_name + " must be metadata-only", ErrorCodes::ILLEGAL_COLUMN); + throw Exception("ALTER of key column " + command.column_name + " must be metadata-only", + ErrorCodes::ALTER_OF_COLUMN_IS_FORBIDDEN); } } } From 7396790bd989ac73dc364fc92967f107d7cc2a84 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 17 May 2020 07:47:32 +0300 Subject: [PATCH 299/738] Update tests --- tests/queries/0_stateless/00714_alter_uuid.sql | 4 ++-- tests/queries/0_stateless/00933_alter_ttl.sql | 2 +- .../01213_alter_rename_primary_key_zookeeper.sql | 14 +++++++------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/queries/0_stateless/00714_alter_uuid.sql b/tests/queries/0_stateless/00714_alter_uuid.sql index 6dca79c37ab..00a8f793e0e 100644 --- a/tests/queries/0_stateless/00714_alter_uuid.sql +++ b/tests/queries/0_stateless/00714_alter_uuid.sql @@ -39,7 +39,7 @@ ORDER BY (created_at, id0, id1); SET send_logs_level = 'none'; -ALTER TABLE uuid MODIFY COLUMN id0 UUID; -- { serverError 44 } -ALTER TABLE uuid MODIFY COLUMN id1 UUID; -- { serverError 44 } +ALTER TABLE uuid MODIFY COLUMN id0 UUID; -- { serverError 524 } +ALTER TABLE uuid MODIFY COLUMN id1 UUID; -- { serverError 524 } DROP TABLE uuid; diff --git a/tests/queries/0_stateless/00933_alter_ttl.sql b/tests/queries/0_stateless/00933_alter_ttl.sql index f7f141a9408..d3298b3fbe1 100644 --- a/tests/queries/0_stateless/00933_alter_ttl.sql +++ b/tests/queries/0_stateless/00933_alter_ttl.sql @@ -21,6 +21,6 @@ drop table if exists ttl; create table ttl (d Date, a Int) engine = MergeTree order by tuple() partition by toDayOfMonth(d); alter table ttl modify column a Int ttl d + interval 1 day; desc table ttl; -alter table ttl modify column d Int ttl d + interval 1 day; -- { serverError 44} +alter table ttl modify column d Int ttl d + interval 1 day; -- { serverError 524 } drop table if exists ttl; diff --git a/tests/queries/0_stateless/01213_alter_rename_primary_key_zookeeper.sql b/tests/queries/0_stateless/01213_alter_rename_primary_key_zookeeper.sql index b5ad162fdfb..5c62d5d9107 100644 --- a/tests/queries/0_stateless/01213_alter_rename_primary_key_zookeeper.sql +++ b/tests/queries/0_stateless/01213_alter_rename_primary_key_zookeeper.sql @@ -17,11 +17,11 @@ INSERT INTO table_for_rename_pk SELECT toDate('2019-10-01') + number % 3, number SELECT key1, value1 FROM table_for_rename_pk WHERE key1 = 1 AND key2 = 1 AND key3 = 1; -ALTER TABLE table_for_rename_pk RENAME COLUMN key1 TO renamed_key1; --{serverError 44} +ALTER TABLE table_for_rename_pk RENAME COLUMN key1 TO renamed_key1; --{serverError 524} -ALTER TABLE table_for_rename_pk RENAME COLUMN key3 TO renamed_key3; --{serverError 44} +ALTER TABLE table_for_rename_pk RENAME COLUMN key3 TO renamed_key3; --{serverError 524} -ALTER TABLE table_for_rename_pk RENAME COLUMN key2 TO renamed_key2; --{serverError 44} +ALTER TABLE table_for_rename_pk RENAME COLUMN key2 TO renamed_key2; --{serverError 524} DROP TABLE IF EXISTS table_for_rename_pk NO DELAY; SELECT sleep(1) FORMAT Null; @@ -45,12 +45,12 @@ PRIMARY KEY (key1, key2); INSERT INTO table_for_rename_with_primary_key SELECT toDate('2019-10-01') + number % 3, number, number, number, toString(number), toString(number) from numbers(9); -ALTER TABLE table_for_rename_with_primary_key RENAME COLUMN key1 TO renamed_key1; --{serverError 44} +ALTER TABLE table_for_rename_with_primary_key RENAME COLUMN key1 TO renamed_key1; --{serverError 524} -ALTER TABLE table_for_rename_with_primary_key RENAME COLUMN key2 TO renamed_key2; --{serverError 44} +ALTER TABLE table_for_rename_with_primary_key RENAME COLUMN key2 TO renamed_key2; --{serverError 524} -ALTER TABLE table_for_rename_with_primary_key RENAME COLUMN key3 TO renamed_key3; --{serverError 44} +ALTER TABLE table_for_rename_with_primary_key RENAME COLUMN key3 TO renamed_key3; --{serverError 524} -ALTER TABLE table_for_rename_with_primary_key RENAME COLUMN value1 TO renamed_value1; --{serverError 44} +ALTER TABLE table_for_rename_with_primary_key RENAME COLUMN value1 TO renamed_value1; --{serverError 524} DROP TABLE IF EXISTS table_for_rename_with_primary_key; From c4ff7eec0162bdb2bbd65814f8df296efc8592ce Mon Sep 17 00:00:00 2001 From: Andrei Nekrashevich Date: Sun, 17 May 2020 07:49:41 +0300 Subject: [PATCH 300/738] fix --- src/Functions/randomPrintableASCII.cpp | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/Functions/randomPrintableASCII.cpp b/src/Functions/randomPrintableASCII.cpp index fedaeb7b9ef..bd0e84001ab 100644 --- a/src/Functions/randomPrintableASCII.cpp +++ b/src/Functions/randomPrintableASCII.cpp @@ -1,14 +1,15 @@ -#include -#include +#include #include #include -#include +#include +#include #include #include namespace DB { + namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; @@ -26,7 +27,10 @@ public: static constexpr auto name = "randomPrintableASCII"; static FunctionPtr create(const Context &) { return std::make_shared(); } - String getName() const override { return name; } + String getName() const override + { + return name; + } bool isVariadic() const override { return true; } size_t getNumberOfArguments() const override { return 0; } @@ -34,13 +38,11 @@ public: DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { if (arguments.empty()) - throw Exception( - "Function " + getName() + " requires at least one argument: the size of resulting string", + throw Exception("Function " + getName() + " requires at least one argument: the size of resulting string", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); if (arguments.size() > 2) - throw Exception( - "Function " + getName() + " requires at most two arguments: the size of resulting string and optional disambiguation tag", + throw Exception("Function " + getName() + " requires at most two arguments: the size of resulting string and optional disambiguation tag", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); const IDataType & length_type = *arguments[0]; @@ -75,9 +77,8 @@ public: data_to.resize(next_offset); offsets_to[row_num] = next_offset; - auto * data_to_ptr = data_to.data(); /// avoid assert on array indexing after end - for (size_t pos = offset, end = offset + length; pos < end; - pos += 4) /// We have padding in column buffers that we can overwrite. + auto * data_to_ptr = data_to.data(); /// avoid assert on array indexing after end + for (size_t pos = offset, end = offset + length; pos < end; pos += 4) /// We have padding in column buffers that we can overwrite. { UInt64 rand = rng(); From 7a220b76593c7d71651f59d18887e0544cc67147 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sun, 17 May 2020 08:12:33 +0300 Subject: [PATCH 301/738] Update DataTypesDecimal.cpp --- src/DataTypes/DataTypesDecimal.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DataTypes/DataTypesDecimal.cpp b/src/DataTypes/DataTypesDecimal.cpp index 6b085d0e1ed..0d48845b4fe 100644 --- a/src/DataTypes/DataTypesDecimal.cpp +++ b/src/DataTypes/DataTypesDecimal.cpp @@ -22,9 +22,9 @@ namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int DECIMAL_OVERFLOW; } -// template std::string DataTypeDecimal::doGetName() const From 397859ccb817465b200061263fcbbdd35e261430 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 17 May 2020 08:45:20 +0300 Subject: [PATCH 302/738] Fix error --- src/Client/Connection.cpp | 17 +++-------------- src/Client/ConnectionPool.h | 4 ++-- src/Client/MultiplexedConnections.cpp | 8 ++++---- src/Client/MultiplexedConnections.h | 8 ++++---- src/DataStreams/RemoteBlockInputStream.cpp | 5 ++++- src/DataStreams/RemoteBlockOutputStream.cpp | 13 ++++++++----- src/DataStreams/RemoteBlockOutputStream.h | 6 ++---- src/Storages/Distributed/DirectoryMonitor.cpp | 4 ++-- .../DistributedBlockOutputStream.cpp | 2 +- 9 files changed, 30 insertions(+), 37 deletions(-) diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index 0190d9a8005..64cd7f27cd9 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -399,21 +399,10 @@ void Connection::sendQuery( /// Client info. if (server_revision >= DBMS_MIN_REVISION_WITH_CLIENT_INFO) { - ClientInfo client_info_to_send; - - if (!client_info || client_info->empty()) - { - /// No client info passed - means this query initiated by me. - client_info_to_send.setInitialQuery(); - } + if (client_info && !client_info->empty()) + client_info->write(*out, server_revision); else - { - /// This query is initiated by another query. - client_info_to_send = *client_info; - client_info_to_send.query_kind = ClientInfo::QueryKind::SECONDARY_QUERY; - } - - client_info_to_send.write(*out, server_revision); + ClientInfo().write(*out, server_revision); } /// Per query settings. diff --git a/src/Client/ConnectionPool.h b/src/Client/ConnectionPool.h index 1ecb432c827..be52234b904 100644 --- a/src/Client/ConnectionPool.h +++ b/src/Client/ConnectionPool.h @@ -15,8 +15,8 @@ namespace DB * * void thread() * { - * auto connection = pool.get(); - * connection->sendQuery("SELECT 'Hello, world!' AS world"); + * auto connection = pool.get(); + * connection->sendQuery(...); * } */ diff --git a/src/Client/MultiplexedConnections.cpp b/src/Client/MultiplexedConnections.cpp index b0ff2104ab1..282aaabd119 100644 --- a/src/Client/MultiplexedConnections.cpp +++ b/src/Client/MultiplexedConnections.cpp @@ -94,7 +94,7 @@ void MultiplexedConnections::sendQuery( const String & query, const String & query_id, UInt64 stage, - const ClientInfo * client_info, + const ClientInfo & client_info, bool with_pending_data) { std::lock_guard lock(cancel_mutex); @@ -126,14 +126,14 @@ void MultiplexedConnections::sendQuery( { modified_settings.parallel_replica_offset = i; replica_states[i].connection->sendQuery(timeouts, query, query_id, - stage, &modified_settings, client_info, with_pending_data); + stage, &modified_settings, &client_info, with_pending_data); } } else { /// Use single replica. - replica_states[0].connection->sendQuery(timeouts, query, query_id, stage, - &modified_settings, client_info, with_pending_data); + replica_states[0].connection->sendQuery(timeouts, query, query_id, + stage, &modified_settings, &client_info, with_pending_data); } sent_query = true; diff --git a/src/Client/MultiplexedConnections.h b/src/Client/MultiplexedConnections.h index 9d825adb227..eaec7f744bc 100644 --- a/src/Client/MultiplexedConnections.h +++ b/src/Client/MultiplexedConnections.h @@ -36,10 +36,10 @@ public: void sendQuery( const ConnectionTimeouts & timeouts, const String & query, - const String & query_id = "", - UInt64 stage = QueryProcessingStage::Complete, - const ClientInfo * client_info = nullptr, - bool with_pending_data = false); + const String & query_id, + UInt64 stage, + const ClientInfo & client_info, + bool with_pending_data); /// Get packet from any replica. Packet receivePacket(); diff --git a/src/DataStreams/RemoteBlockInputStream.cpp b/src/DataStreams/RemoteBlockInputStream.cpp index fc1578d1749..e9b8d26f975 100644 --- a/src/DataStreams/RemoteBlockInputStream.cpp +++ b/src/DataStreams/RemoteBlockInputStream.cpp @@ -347,7 +347,10 @@ void RemoteBlockInputStream::sendQuery() established = true; auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(settings); - multiplexed_connections->sendQuery(timeouts, query, query_id, stage, &context.getClientInfo(), true); + ClientInfo modified_client_info = context.getClientInfo(); + modified_client_info.query_kind = ClientInfo::QueryKind::SECONDARY_QUERY; + + multiplexed_connections->sendQuery(timeouts, query, query_id, stage, modified_client_info, true); established = false; sent_query = true; diff --git a/src/DataStreams/RemoteBlockOutputStream.cpp b/src/DataStreams/RemoteBlockOutputStream.cpp index 5200500cd00..327e0204892 100644 --- a/src/DataStreams/RemoteBlockOutputStream.cpp +++ b/src/DataStreams/RemoteBlockOutputStream.cpp @@ -21,14 +21,17 @@ namespace ErrorCodes RemoteBlockOutputStream::RemoteBlockOutputStream(Connection & connection_, const ConnectionTimeouts & timeouts, const String & query_, - const Settings * settings_, - const ClientInfo * client_info_) - : connection(connection_), query(query_), settings(settings_), client_info(client_info_) + const Settings & settings_, + const ClientInfo & client_info_) + : connection(connection_), query(query_) { - /** Send query and receive "header", that describe table structure. + ClientInfo modified_client_info = client_info_; + modified_client_info.query_kind = ClientInfo::QueryKind::SECONDARY_QUERY; + + /** Send query and receive "header", that describes table structure. * Header is needed to know, what structure is required for blocks to be passed to 'write' method. */ - connection.sendQuery(timeouts, query, "", QueryProcessingStage::Complete, settings, client_info); + connection.sendQuery(timeouts, query, "", QueryProcessingStage::Complete, &settings_, &modified_client_info); while (true) { diff --git a/src/DataStreams/RemoteBlockOutputStream.h b/src/DataStreams/RemoteBlockOutputStream.h index f7f1ab46806..40387180997 100644 --- a/src/DataStreams/RemoteBlockOutputStream.h +++ b/src/DataStreams/RemoteBlockOutputStream.h @@ -22,8 +22,8 @@ public: RemoteBlockOutputStream(Connection & connection_, const ConnectionTimeouts & timeouts, const String & query_, - const Settings * settings_ = nullptr, - const ClientInfo * client_info_ = nullptr); + const Settings & settings_, + const ClientInfo & client_info_); Block getHeader() const override { return header; } @@ -38,8 +38,6 @@ public: private: Connection & connection; String query; - const Settings * settings; - const ClientInfo * client_info; Block header; bool finished = false; }; diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp index 2a5e47fdf04..f1593cd73d0 100644 --- a/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/src/Storages/Distributed/DirectoryMonitor.cpp @@ -274,7 +274,7 @@ void StorageDistributedDirectoryMonitor::processFile(const std::string & file_pa ClientInfo client_info; readHeader(in, insert_settings, insert_query, client_info, log); - RemoteBlockOutputStream remote{*connection, timeouts, insert_query, &insert_settings, &client_info}; + RemoteBlockOutputStream remote{*connection, timeouts, insert_query, insert_settings, client_info}; remote.writePrefix(); remote.writePrepared(in); @@ -463,7 +463,7 @@ struct StorageDistributedDirectoryMonitor::Batch if (first) { first = false; - remote = std::make_unique(*connection, timeouts, insert_query, &insert_settings, &client_info); + remote = std::make_unique(*connection, timeouts, insert_query, insert_settings, client_info); remote->writePrefix(); } diff --git a/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/src/Storages/Distributed/DistributedBlockOutputStream.cpp index 55d8d4da9d8..1c4621a6cae 100644 --- a/src/Storages/Distributed/DistributedBlockOutputStream.cpp +++ b/src/Storages/Distributed/DistributedBlockOutputStream.cpp @@ -311,7 +311,7 @@ ThreadPool::Job DistributedBlockOutputStream::runWritingJob(DistributedBlockOutp if (throttler) job.connection_entry->setThrottler(throttler); - job.stream = std::make_shared(*job.connection_entry, timeouts, query_string, &settings, &context.getClientInfo()); + job.stream = std::make_shared(*job.connection_entry, timeouts, query_string, settings, context.getClientInfo()); job.stream->writePrefix(); } From c06995c03c804f6c907b975775da98ba93757b86 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 17 May 2020 09:47:53 +0300 Subject: [PATCH 303/738] Whitespaces --- src/Processors/Transforms/LimitsCheckingTransform.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Processors/Transforms/LimitsCheckingTransform.cpp b/src/Processors/Transforms/LimitsCheckingTransform.cpp index c3ac019f2b6..c4739ee9dde 100644 --- a/src/Processors/Transforms/LimitsCheckingTransform.cpp +++ b/src/Processors/Transforms/LimitsCheckingTransform.cpp @@ -44,7 +44,9 @@ void LimitsCheckingTransform::transform(Chunk & chunk) if (limits.mode == LimitsMode::LIMITS_CURRENT && !limits.size_limits.check(info.rows, info.bytes, "result", ErrorCodes::TOO_MANY_ROWS_OR_BYTES)) + { stopReading(); + } if (quota) checkQuota(chunk); @@ -62,7 +64,10 @@ void LimitsCheckingTransform::checkQuota(Chunk & chunk) case LimitsMode::LIMITS_CURRENT: { UInt64 total_elapsed = info.total_stopwatch.elapsedNanoseconds(); - quota->used({Quota::RESULT_ROWS, chunk.getNumRows()}, {Quota::RESULT_BYTES, chunk.bytes()}, {Quota::EXECUTION_TIME, total_elapsed - prev_elapsed}); + quota->used( + {Quota::RESULT_ROWS, chunk.getNumRows()}, + {Quota::RESULT_BYTES, chunk.bytes()}, + {Quota::EXECUTION_TIME, total_elapsed - prev_elapsed}); prev_elapsed = total_elapsed; break; } From 1ebd71a230af1e2450e5cda064048defaf9b5e8d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 17 May 2020 09:48:49 +0300 Subject: [PATCH 304/738] Updated test reference because new result is correct --- tests/queries/0_stateless/01133_max_result_rows.reference | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/01133_max_result_rows.reference b/tests/queries/0_stateless/01133_max_result_rows.reference index 61ecaca3948..230bb9b4ca2 100644 --- a/tests/queries/0_stateless/01133_max_result_rows.reference +++ b/tests/queries/0_stateless/01133_max_result_rows.reference @@ -97,6 +97,7 @@ 17 18 19 +20 0 1 2 @@ -156,3 +157,4 @@ 17 18 19 +20 From 20c9c2d3ec5f9bb8386655414bc50259d9c1b011 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 17 May 2020 09:52:33 +0300 Subject: [PATCH 305/738] Better test --- .../0_stateless/01133_max_result_rows.reference | 13 +++++++++++++ tests/queries/0_stateless/01133_max_result_rows.sql | 9 +++++++++ 2 files changed, 22 insertions(+) diff --git a/tests/queries/0_stateless/01133_max_result_rows.reference b/tests/queries/0_stateless/01133_max_result_rows.reference index 230bb9b4ca2..7134ee81514 100644 --- a/tests/queries/0_stateless/01133_max_result_rows.reference +++ b/tests/queries/0_stateless/01133_max_result_rows.reference @@ -158,3 +158,16 @@ 18 19 20 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +100 +100 +200 diff --git a/tests/queries/0_stateless/01133_max_result_rows.sql b/tests/queries/0_stateless/01133_max_result_rows.sql index fe86a5902cf..db359959aed 100644 --- a/tests/queries/0_stateless/01133_max_result_rows.sql +++ b/tests/queries/0_stateless/01133_max_result_rows.sql @@ -13,3 +13,12 @@ SELECT DISTINCT intDiv(number, 10) FROM numbers(300); SELECT DISTINCT intDiv(number, 10) FROM numbers(190); SELECT DISTINCT intDiv(number, 10) FROM numbers(200); SELECT DISTINCT intDiv(number, 10) FROM numbers(210); + +SET max_block_size = 10; +SET max_result_rows = 1; +SELECT number FROM system.numbers; +SELECT count() FROM numbers(100); +-- subquery result is not the total result +SELECT count() FROM (SELECT * FROM numbers(100)); +-- remote query result is not the total result +SELECT count() FROM remote('127.0.0.{1,2}', numbers(100)); From de8120d69af8404a420a0a7e56759c747323af8f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 17 May 2020 10:17:54 +0300 Subject: [PATCH 306/738] Improvement; added a test --- src/Processors/ISource.cpp | 5 -- src/Processors/Sources/SourceWithProgress.cpp | 14 ++++- .../Transforms/LimitsCheckingTransform.cpp | 2 +- .../01132_max_rows_to_read.reference | 55 ++++++++++++++++++- .../0_stateless/01132_max_rows_to_read.sql | 15 ++++- 5 files changed, 80 insertions(+), 11 deletions(-) diff --git a/src/Processors/ISource.cpp b/src/Processors/ISource.cpp index 7c620a98a74..90f3962b83e 100644 --- a/src/Processors/ISource.cpp +++ b/src/Processors/ISource.cpp @@ -56,11 +56,6 @@ void ISource::work() finished = true; throw; } -// { -// current_chunk = std::current_exception(); -// has_input = true; -// got_exception = true; -// } } } diff --git a/src/Processors/Sources/SourceWithProgress.cpp b/src/Processors/Sources/SourceWithProgress.cpp index 80844da16cd..8d7a0a3d946 100644 --- a/src/Processors/Sources/SourceWithProgress.cpp +++ b/src/Processors/Sources/SourceWithProgress.cpp @@ -15,7 +15,9 @@ namespace ErrorCodes void SourceWithProgress::work() { if (!limits.speed_limits.checkTimeLimit(total_stopwatch.elapsed(), limits.timeout_overflow_mode)) + { cancel(); + } else { was_progress_called = false; @@ -57,7 +59,13 @@ void SourceWithProgress::progress(const Progress & value) /// The total amount of data processed or intended for processing in all sources, possibly on remote servers. ProgressValues progress = process_list_elem->getProgressIn(); - size_t total_rows_estimate = std::max(progress.read_rows, progress.total_rows_to_read); + + /// If the mode is "throw" and estimate of total rows is known, then throw early if an estimate is too high. + /// If the mode is "break", then allow to read before limit even if estimate is very high. + + size_t rows_to_check_limit = progress.read_rows; + if (limits.size_limits.overflow_mode == OverflowMode::THROW && progress.total_rows_to_read > progress.read_rows) + rows_to_check_limit = progress.total_rows_to_read; /// Check the restrictions on the /// * amount of data to read @@ -67,9 +75,11 @@ void SourceWithProgress::progress(const Progress & value) if (limits.mode == LimitsMode::LIMITS_TOTAL) { - if (!limits.size_limits.check(total_rows_estimate, progress.read_bytes, "rows to read", + if (!limits.size_limits.check(rows_to_check_limit, progress.read_bytes, "rows or bytes to read", ErrorCodes::TOO_MANY_ROWS, ErrorCodes::TOO_MANY_BYTES)) + { cancel(); + } } size_t total_rows = progress.total_rows_to_read; diff --git a/src/Processors/Transforms/LimitsCheckingTransform.cpp b/src/Processors/Transforms/LimitsCheckingTransform.cpp index c4739ee9dde..56edd5f0317 100644 --- a/src/Processors/Transforms/LimitsCheckingTransform.cpp +++ b/src/Processors/Transforms/LimitsCheckingTransform.cpp @@ -58,7 +58,7 @@ void LimitsCheckingTransform::checkQuota(Chunk & chunk) switch (limits.mode) { case LimitsMode::LIMITS_TOTAL: - /// Checked in `progress` method. + /// Checked in SourceWithProgress::progress method. break; case LimitsMode::LIMITS_CURRENT: diff --git a/tests/queries/0_stateless/01132_max_rows_to_read.reference b/tests/queries/0_stateless/01132_max_rows_to_read.reference index 91653a56ef7..5087d15b87c 100644 --- a/tests/queries/0_stateless/01132_max_rows_to_read.reference +++ b/tests/queries/0_stateless/01132_max_rows_to_read.reference @@ -1,6 +1,57 @@ 19 20 -30 19 20 -21 +20 +20 +20 +20 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 diff --git a/tests/queries/0_stateless/01132_max_rows_to_read.sql b/tests/queries/0_stateless/01132_max_rows_to_read.sql index f8e5e32c0b5..b7923a27d04 100644 --- a/tests/queries/0_stateless/01132_max_rows_to_read.sql +++ b/tests/queries/0_stateless/01132_max_rows_to_read.sql @@ -7,11 +7,24 @@ SELECT count() FROM numbers(19); SELECT count() FROM numbers(20); SELECT count() FROM numbers(21); -- { serverError 158 } +-- check early exception if the estimated number of rows is high +SELECT * FROM numbers(30); -- { serverError 158 } + SET read_overflow_mode = 'break'; SELECT count() FROM numbers(19); SELECT count() FROM numbers(20); SELECT count() FROM numbers(21); -SELECT count() FROM numbers(29); -- one extra block is read and it is Ok. +SELECT count() FROM numbers(29); SELECT count() FROM numbers(30); SELECT count() FROM numbers(31); + +-- check that partial result is returned even if the estimated number of rows is high +SELECT * FROM numbers(30); + +-- the same for uneven block sizes +-- NOTE: currently it outputs less amount of data; it will be better to output the latest block also +SET max_block_size = 11; +SELECT * FROM numbers(30); +SET max_block_size = 9; +SELECT * FROM numbers(30); From f2d438b79f5c4af786ddc6827fd5bbbe90929120 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 17 May 2020 10:27:55 +0300 Subject: [PATCH 307/738] Implement max_rows_to_sort again --- src/Interpreters/InterpreterSelectQuery.cpp | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 6c7a17ffd77..d0b5b57fc73 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -2063,10 +2063,9 @@ void InterpreterSelectQuery::executeOrder(QueryPipeline & pipeline, InputSorting const Settings & settings = context->getSettingsRef(); - /// TODO: Limits on sorting -// IBlockInputStream::LocalLimits limits; -// limits.mode = IBlockInputStream::LIMITS_TOTAL; -// limits.size_limits = SizeLimits(settings.max_rows_to_sort, settings.max_bytes_to_sort, settings.sort_overflow_mode); + IBlockInputStream::LocalLimits limits; + limits.mode = IBlockInputStream::LIMITS_CURRENT; + limits.size_limits = SizeLimits(settings.max_rows_to_sort, settings.max_bytes_to_sort, settings.sort_overflow_mode); if (input_sorting_info) { @@ -2103,6 +2102,8 @@ void InterpreterSelectQuery::executeOrder(QueryPipeline & pipeline, InputSorting return std::make_shared(header, output_order_descr, limit); }); + /// NOTE limits are not applied to the size of temporary sets in FinishSortingTransform + pipeline.addSimpleTransform([&](const Block & header) -> ProcessorPtr { return std::make_shared( @@ -2122,6 +2123,15 @@ void InterpreterSelectQuery::executeOrder(QueryPipeline & pipeline, InputSorting return std::make_shared(header, output_order_descr, limit); }); + pipeline.addSimpleTransform([&](const Block & header, QueryPipeline::StreamType stream_type) -> ProcessorPtr + { + if (stream_type == QueryPipeline::StreamType::Totals) + return nullptr; + + auto transform = std::make_shared(header, limits); + return transform; + }); + /// Merge the sorted blocks. pipeline.addSimpleTransform([&](const Block & header, QueryPipeline::StreamType stream_type) -> ProcessorPtr { From c33373f7fb19175bbe338a2bcefa9accd862536b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 17 May 2020 10:43:23 +0300 Subject: [PATCH 308/738] Minor modification --- src/Processors/Transforms/MergeSortingTransform.h | 3 +++ src/Processors/Transforms/SortingTransform.h | 7 ++----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Processors/Transforms/MergeSortingTransform.h b/src/Processors/Transforms/MergeSortingTransform.h index a8786e5a034..043cb3f36c1 100644 --- a/src/Processors/Transforms/MergeSortingTransform.h +++ b/src/Processors/Transforms/MergeSortingTransform.h @@ -38,6 +38,9 @@ private: VolumePtr tmp_volume; size_t min_free_disk_space; + size_t sum_rows_in_blocks = 0; + size_t sum_bytes_in_blocks = 0; + Logger * log = &Logger::get("MergeSortingTransform"); /// If remerge doesn't save memory at least several times, mark it as useless and don't do it anymore. diff --git a/src/Processors/Transforms/SortingTransform.h b/src/Processors/Transforms/SortingTransform.h index 49bdf303c7f..9178991f324 100644 --- a/src/Processors/Transforms/SortingTransform.h +++ b/src/Processors/Transforms/SortingTransform.h @@ -66,8 +66,8 @@ class SortingTransform : public IProcessor public: /// limit - if not 0, allowed to return just first 'limit' rows in sorted order. SortingTransform(const Block & header, - const SortDescription & description_, - size_t max_merged_block_size_, UInt64 limit_); + const SortDescription & description_, + size_t max_merged_block_size_, UInt64 limit_); ~SortingTransform() override; @@ -83,9 +83,6 @@ protected: size_t max_merged_block_size; UInt64 limit; - size_t sum_rows_in_blocks = 0; - size_t sum_bytes_in_blocks = 0; - /// Before operation, will remove constant columns from blocks. And after, place constant columns back. /// (to avoid excessive virtual function calls and because constants cannot be serialized in Native format for temporary files) /// Save original block structure here. From 7971f62776beb7fe89976100fbe9d0ddfc72d349 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 17 May 2020 10:47:53 +0300 Subject: [PATCH 309/738] Fix test --- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 50a5a209546..88094f031a7 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -1474,7 +1474,7 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, const S { throw Exception( "Trying to ALTER RENAME key " + backQuoteIfNeed(command.column_name) + " column which is a part of key expression", - ErrorCodes::ILLEGAL_COLUMN); + ErrorCodes::ALTER_OF_COLUMN_IS_FORBIDDEN); } } else if (command.isModifyingData(getInMemoryMetadata())) From 7e7fd41266d296eb88828bc67b4997fc9f700090 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 17 May 2020 11:00:10 +0300 Subject: [PATCH 310/738] Fix build + improvement --- src/Functions/randomFixedString.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Functions/randomFixedString.cpp b/src/Functions/randomFixedString.cpp index 70c6b013dce..9fb7550346b 100644 --- a/src/Functions/randomFixedString.cpp +++ b/src/Functions/randomFixedString.cpp @@ -42,7 +42,7 @@ public: if (!arguments[0].column || !isColumnConst(*arguments[0].column)) throw Exception("First argument for function " + getName() + " must be constant", ErrorCodes::ILLEGAL_COLUMN); - const size_t n = arguments[0].column->getUInt(0); + const size_t n = assert_cast(*arguments[0].column).getValue(); return std::make_shared(n); } @@ -51,7 +51,7 @@ public: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - const auto n = block.getByPosition(arguments[0]).column->getUInt(0); + const size_t n = assert_cast(*block.getByPosition(arguments[0]).column).getValue(); auto col_to = ColumnFixedString::create(n); ColumnFixedString::Chars & data_to = col_to->getChars(); From 6029b6d7bd67552ef4f94398773c01190737135b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 17 May 2020 11:22:59 +0300 Subject: [PATCH 311/738] Added REAL; remove Int1/2/4; remove LONG because it has conflict meaning in MySQL and MS Access --- src/DataTypes/DataTypeString.cpp | 1 - src/DataTypes/DataTypesNumber.cpp | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/DataTypes/DataTypeString.cpp b/src/DataTypes/DataTypeString.cpp index c1afa8b90ea..efaf844a845 100644 --- a/src/DataTypes/DataTypeString.cpp +++ b/src/DataTypes/DataTypeString.cpp @@ -380,7 +380,6 @@ void registerDataTypeString(DataTypeFactory & factory) factory.registerAlias("TEXT", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("TINYTEXT", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("MEDIUMTEXT", "String", DataTypeFactory::CaseInsensitive); - factory.registerAlias("LONG", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("LONGTEXT", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("BLOB", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("TINYBLOB", "String", DataTypeFactory::CaseInsensitive); diff --git a/src/DataTypes/DataTypesNumber.cpp b/src/DataTypes/DataTypesNumber.cpp index 4da767ae359..8dd36fde9b6 100644 --- a/src/DataTypes/DataTypesNumber.cpp +++ b/src/DataTypes/DataTypesNumber.cpp @@ -27,12 +27,11 @@ void registerDataTypeNumbers(DataTypeFactory & factory) factory.registerAlias("BOOLEAN", "Int8", DataTypeFactory::CaseInsensitive); factory.registerAlias("INT1", "Int8", DataTypeFactory::CaseInsensitive); factory.registerAlias("SMALLINT", "Int16", DataTypeFactory::CaseInsensitive); - factory.registerAlias("INT2", "Int16", DataTypeFactory::CaseInsensitive); factory.registerAlias("INT", "Int32", DataTypeFactory::CaseInsensitive); - factory.registerAlias("INT4", "Int32", DataTypeFactory::CaseInsensitive); factory.registerAlias("INTEGER", "Int32", DataTypeFactory::CaseInsensitive); factory.registerAlias("BIGINT", "Int64", DataTypeFactory::CaseInsensitive); factory.registerAlias("FLOAT", "Float32", DataTypeFactory::CaseInsensitive); + factory.registerAlias("REAL", "Float32", DataTypeFactory::CaseInsensitive); factory.registerAlias("DOUBLE", "Float64", DataTypeFactory::CaseInsensitive); } From 59b0435547c3d663f07e0eaf784a2eb81eee6f17 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 17 May 2020 11:25:29 +0300 Subject: [PATCH 312/738] Fix tests --- tests/queries/0_stateless/00700_decimal_arithm.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/00700_decimal_arithm.sql b/tests/queries/0_stateless/00700_decimal_arithm.sql index 1331ae2800c..3fa2aade743 100644 --- a/tests/queries/0_stateless/00700_decimal_arithm.sql +++ b/tests/queries/0_stateless/00700_decimal_arithm.sql @@ -11,9 +11,9 @@ CREATE TABLE IF NOT EXISTS decimal g Decimal(9, 3), h decimal(18, 9), i deciMAL(38, 18), - j dec(4,2), - k NumEriC(4, 23), - l numeric(3,9), + j dec(4, 2), + k NumEriC(23, 4), + l numeric(9, 3), m NUMEric(18, 9), n FixED(12, 6), o fixed(8, 6) From d1d070fa8da21f66958217d0c428e7172cc221ad Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 17 May 2020 11:35:08 +0300 Subject: [PATCH 313/738] More type aliases --- src/DataTypes/DataTypeString.cpp | 4 ++++ src/DataTypes/DataTypesNumber.cpp | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/DataTypes/DataTypeString.cpp b/src/DataTypes/DataTypeString.cpp index efaf844a845..a7bfb2b635d 100644 --- a/src/DataTypes/DataTypeString.cpp +++ b/src/DataTypes/DataTypeString.cpp @@ -376,15 +376,19 @@ void registerDataTypeString(DataTypeFactory & factory) /// These synonyms are added for compatibility. factory.registerAlias("CHAR", "String", DataTypeFactory::CaseInsensitive); + factory.registerAlias("CHARACTER", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("VARCHAR", "String", DataTypeFactory::CaseInsensitive); + factory.registerAlias("VARCHAR2", "String", DataTypeFactory::CaseInsensitive); /// Oracle factory.registerAlias("TEXT", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("TINYTEXT", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("MEDIUMTEXT", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("LONGTEXT", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("BLOB", "String", DataTypeFactory::CaseInsensitive); + factory.registerAlias("CLOB", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("TINYBLOB", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("MEDIUMBLOB", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("LONGBLOB", "String", DataTypeFactory::CaseInsensitive); + factory.registerAlias("BYTEA", "String", DataTypeFactory::CaseInsensitive); /// PostgreSQL } } diff --git a/src/DataTypes/DataTypesNumber.cpp b/src/DataTypes/DataTypesNumber.cpp index 8dd36fde9b6..18b819c4aa6 100644 --- a/src/DataTypes/DataTypesNumber.cpp +++ b/src/DataTypes/DataTypesNumber.cpp @@ -25,13 +25,15 @@ void registerDataTypeNumbers(DataTypeFactory & factory) factory.registerAlias("TINYINT", "Int8", DataTypeFactory::CaseInsensitive); factory.registerAlias("BOOL", "Int8", DataTypeFactory::CaseInsensitive); factory.registerAlias("BOOLEAN", "Int8", DataTypeFactory::CaseInsensitive); - factory.registerAlias("INT1", "Int8", DataTypeFactory::CaseInsensitive); + factory.registerAlias("INT1", "Int8", DataTypeFactory::CaseInsensitive); /// MySQL + factory.registerAlias("BYTE", "Int8", DataTypeFactory::CaseInsensitive); /// MS Access factory.registerAlias("SMALLINT", "Int16", DataTypeFactory::CaseInsensitive); factory.registerAlias("INT", "Int32", DataTypeFactory::CaseInsensitive); factory.registerAlias("INTEGER", "Int32", DataTypeFactory::CaseInsensitive); factory.registerAlias("BIGINT", "Int64", DataTypeFactory::CaseInsensitive); factory.registerAlias("FLOAT", "Float32", DataTypeFactory::CaseInsensitive); factory.registerAlias("REAL", "Float32", DataTypeFactory::CaseInsensitive); + factory.registerAlias("SINGLE", "Float32", DataTypeFactory::CaseInsensitive); /// MS Access factory.registerAlias("DOUBLE", "Float64", DataTypeFactory::CaseInsensitive); } From c35c89f61aeace947f256657db74cc61976f66c5 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 12 May 2020 21:05:11 +0300 Subject: [PATCH 314/738] Add a test for min_insert_block_size_rows_for_materialized_views --- ...size_rows_for_materialized_views.reference | 4 + ..._block_size_rows_for_materialized_views.sh | 92 +++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 tests/queries/0_stateless/01278_min_insert_block_size_rows_for_materialized_views.reference create mode 100755 tests/queries/0_stateless/01278_min_insert_block_size_rows_for_materialized_views.sh diff --git a/tests/queries/0_stateless/01278_min_insert_block_size_rows_for_materialized_views.reference b/tests/queries/0_stateless/01278_min_insert_block_size_rows_for_materialized_views.reference new file mode 100644 index 00000000000..ed22b7e1e35 --- /dev/null +++ b/tests/queries/0_stateless/01278_min_insert_block_size_rows_for_materialized_views.reference @@ -0,0 +1,4 @@ +0 +0 +100000 +200000 diff --git a/tests/queries/0_stateless/01278_min_insert_block_size_rows_for_materialized_views.sh b/tests/queries/0_stateless/01278_min_insert_block_size_rows_for_materialized_views.sh new file mode 100755 index 00000000000..7e08c930f67 --- /dev/null +++ b/tests/queries/0_stateless/01278_min_insert_block_size_rows_for_materialized_views.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +# just in case +set -o pipefail + +function execute() +{ + ${CLICKHOUSE_CLIENT} -n "$@" +} + +# +# TEST SETTINGS +# +TEST_01278_PARTS=9 +TEST_01278_MEMORY=$((100<<20)) + +function cleanup() +{ + for i in $(seq 1 $TEST_01278_PARTS); do + echo "drop table if exists part_01278_$i;" + echo "drop table if exists mv_01278_$i;" + done | execute + echo 'drop table if exists data_01278;' | execute + echo 'drop table if exists out_01278;' | execute + echo 'drop table if exists null_01278;' | execute +} + +cleanup +trap cleanup EXIT + +# +# CREATE +# +{ +cat < Date: Tue, 12 May 2020 21:05:12 +0300 Subject: [PATCH 315/738] Introduce min_insert_block_size_{rows,bytes}_for_materialized_views With tons of MATERIALIZED VIEW attached to one table pushing to this views can be pretty memory consuming due to blocks squashing, add ability to control this separatelly for MATERIALIZED VIEWs. Plus squashing is useless if the underlying engine is Buffer(). --- src/Core/Settings.h | 2 ++ .../PushingToViewsBlockOutputStream.cpp | 27 ++++++++++++++----- .../PushingToViewsBlockOutputStream.h | 3 ++- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 34d05900f77..d0e86df45c4 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -52,6 +52,8 @@ struct Settings : public SettingsCollection M(SettingUInt64, max_insert_block_size, DEFAULT_INSERT_BLOCK_SIZE, "The maximum block size for insertion, if we control the creation of blocks for insertion.", 0) \ M(SettingUInt64, min_insert_block_size_rows, DEFAULT_INSERT_BLOCK_SIZE, "Squash blocks passed to INSERT query to specified size in rows, if blocks are not big enough.", 0) \ M(SettingUInt64, min_insert_block_size_bytes, (DEFAULT_INSERT_BLOCK_SIZE * 256), "Squash blocks passed to INSERT query to specified size in bytes, if blocks are not big enough.", 0) \ + M(SettingUInt64, min_insert_block_size_rows_for_materialized_views, 0, "Like min_insert_block_size_rows, but applied only during pushing to MATERIALIZED VIEW (default: min_insert_block_size_rows)", 0) \ + M(SettingUInt64, min_insert_block_size_bytes_for_materialized_views, 0, "Like min_insert_block_size_bytes, but applied only during pushing to MATERIALIZED VIEW (default: min_insert_block_size_bytes)", 0) \ M(SettingUInt64, max_joined_block_size_rows, DEFAULT_BLOCK_SIZE, "Maximum block size for JOIN result (if join algorithm supports it). 0 means unlimited.", 0) \ M(SettingUInt64, max_insert_threads, 0, "The maximum number of threads to execute the INSERT SELECT query. Values 0 or 1 means that INSERT SELECT is not run in parallel. Higher values will lead to higher memory usage. Parallel INSERT SELECT has effect only if the SELECT part is run on parallel, see 'max_threads' setting.", 0) \ M(SettingMaxThreads, max_threads, 0, "The maximum number of threads to execute the request. By default, it is determined automatically.", 0) \ diff --git a/src/DataStreams/PushingToViewsBlockOutputStream.cpp b/src/DataStreams/PushingToViewsBlockOutputStream.cpp index ce0922bf282..9a65d948b30 100644 --- a/src/DataStreams/PushingToViewsBlockOutputStream.cpp +++ b/src/DataStreams/PushingToViewsBlockOutputStream.cpp @@ -40,10 +40,23 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream( /// We need special context for materialized views insertions if (!dependencies.empty()) { - views_context = std::make_unique(context); + select_context = std::make_unique(context); + insert_context = std::make_unique(context); + + const auto & insert_settings = insert_context->getSettingsRef(); + // Do not deduplicate insertions into MV if the main insertion is Ok if (disable_deduplication_for_children) - views_context->setSetting("insert_deduplicate", false); + insert_context->setSetting("insert_deduplicate", false); + + // Separate min_insert_block_size_rows/min_insert_block_size_bytes for children + if (insert_settings.min_insert_block_size_rows_for_materialized_views.changed || insert_settings.min_insert_block_size_bytes_for_materialized_views.changed) + { + if (insert_settings.min_insert_block_size_rows_for_materialized_views.changed) + insert_context->setSetting("min_insert_block_size_rows", insert_settings.min_insert_block_size_rows_for_materialized_views.value); + if (insert_settings.min_insert_block_size_bytes_for_materialized_views.changed) + insert_context->setSetting("min_insert_block_size_bytes", insert_settings.min_insert_block_size_bytes_for_materialized_views.value); + } } for (const auto & database_table : dependencies) @@ -67,7 +80,7 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream( insert->table_id = inner_table_id; /// Get list of columns we get from select query. - auto header = InterpreterSelectQuery(query, *views_context, SelectQueryOptions().analyze()) + auto header = InterpreterSelectQuery(query, *select_context, SelectQueryOptions().analyze()) .getSampleBlock(); /// Insert only columns returned by select. @@ -81,14 +94,14 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream( insert->columns = std::move(list); ASTPtr insert_query_ptr(insert.release()); - InterpreterInsertQuery interpreter(insert_query_ptr, *views_context); + InterpreterInsertQuery interpreter(insert_query_ptr, *insert_context); BlockIO io = interpreter.execute(); out = io.out; } else if (dynamic_cast(dependent_table.get())) - out = std::make_shared(dependent_table, *views_context, ASTPtr(), true); + out = std::make_shared(dependent_table, *insert_context, ASTPtr(), true); else - out = std::make_shared(dependent_table, *views_context, ASTPtr()); + out = std::make_shared(dependent_table, *insert_context, ASTPtr()); views.emplace_back(ViewInfo{std::move(query), database_table, std::move(out), nullptr}); } @@ -258,7 +271,7 @@ void PushingToViewsBlockOutputStream::process(const Block & block, size_t view_n /// but it will contain single block (that is INSERT-ed into main table). /// InterpreterSelectQuery will do processing of alias columns. - Context local_context = *views_context; + Context local_context = *select_context; local_context.addViewSource( StorageValues::create( storage->getStorageID(), storage->getColumns(), block, storage->getVirtuals())); diff --git a/src/DataStreams/PushingToViewsBlockOutputStream.h b/src/DataStreams/PushingToViewsBlockOutputStream.h index a2a1ca5caf5..c5fef413a23 100644 --- a/src/DataStreams/PushingToViewsBlockOutputStream.h +++ b/src/DataStreams/PushingToViewsBlockOutputStream.h @@ -44,7 +44,8 @@ private: }; std::vector views; - std::unique_ptr views_context; + std::unique_ptr select_context; + std::unique_ptr insert_context; void process(const Block & block, size_t view_num); }; From 90cd5d8a3cdc78aa8758ff0d5b4449a3e81d152f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 17 May 2020 13:26:55 +0300 Subject: [PATCH 316/738] Remove unused header file --- src/Databases/DatabaseOrdinary.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index 352b9db2672..2c093ae90ad 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -29,7 +29,6 @@ #include #include #include -#include namespace DB From d3c9f5cb65a5485304aa9e1595548c532de66e80 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sun, 17 May 2020 13:31:06 +0300 Subject: [PATCH 317/738] Update RowRef.h --- src/Processors/Merges/Algorithms/RowRef.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Processors/Merges/Algorithms/RowRef.h b/src/Processors/Merges/Algorithms/RowRef.h index 658442e34cc..02a07130bce 100644 --- a/src/Processors/Merges/Algorithms/RowRef.h +++ b/src/Processors/Merges/Algorithms/RowRef.h @@ -109,11 +109,6 @@ private: return; } - /// Release memory. It is not obligatory. -// ptr->clear(); -// ptr->all_columns.clear(); -// ptr->sort_columns.clear(); - free_chunks.push_back(ptr->position); } From fdc4823065e9ee5772a5cbf22537778abcd45760 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 17 May 2020 13:41:54 +0300 Subject: [PATCH 318/738] Simplify min_insert_block_size_{rows,bytes}_for_materialized_views changed detection --- src/DataStreams/PushingToViewsBlockOutputStream.cpp | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/DataStreams/PushingToViewsBlockOutputStream.cpp b/src/DataStreams/PushingToViewsBlockOutputStream.cpp index 9a65d948b30..7f8eccda549 100644 --- a/src/DataStreams/PushingToViewsBlockOutputStream.cpp +++ b/src/DataStreams/PushingToViewsBlockOutputStream.cpp @@ -50,13 +50,10 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream( insert_context->setSetting("insert_deduplicate", false); // Separate min_insert_block_size_rows/min_insert_block_size_bytes for children - if (insert_settings.min_insert_block_size_rows_for_materialized_views.changed || insert_settings.min_insert_block_size_bytes_for_materialized_views.changed) - { - if (insert_settings.min_insert_block_size_rows_for_materialized_views.changed) - insert_context->setSetting("min_insert_block_size_rows", insert_settings.min_insert_block_size_rows_for_materialized_views.value); - if (insert_settings.min_insert_block_size_bytes_for_materialized_views.changed) - insert_context->setSetting("min_insert_block_size_bytes", insert_settings.min_insert_block_size_bytes_for_materialized_views.value); - } + if (insert_settings.min_insert_block_size_rows_for_materialized_views.changed) + insert_context->setSetting("min_insert_block_size_rows", insert_settings.min_insert_block_size_rows_for_materialized_views.value); + if (insert_settings.min_insert_block_size_bytes_for_materialized_views.changed) + insert_context->setSetting("min_insert_block_size_bytes", insert_settings.min_insert_block_size_bytes_for_materialized_views.value); } for (const auto & database_table : dependencies) From 29340d915a4364444cf0539ff6e6b7476075a844 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Sun, 17 May 2020 13:51:52 +0300 Subject: [PATCH 319/738] Try fix performance. --- src/Columns/IColumn.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Columns/IColumn.h b/src/Columns/IColumn.h index 11ade9b3b84..2ef2b71b46e 100644 --- a/src/Columns/IColumn.h +++ b/src/Columns/IColumn.h @@ -306,7 +306,8 @@ public: static MutablePtr mutate(Ptr ptr) { - MutablePtr res = ptr->shallowMutate(); + MutablePtr res = ptr->shallowMutate(); /// Now use_count is 2. + ptr.reset(); /// Reset use_count to 1. res->forEachSubcolumn([](WrappedPtr & subcolumn) { subcolumn = IColumn::mutate(std::move(subcolumn)); }); return res; } From 9d2801e0d828f89d9f3c4a31cf748fd791745252 Mon Sep 17 00:00:00 2001 From: Andrei Nekrashevich Date: Sun, 17 May 2020 16:07:37 +0300 Subject: [PATCH 320/738] fix --- src/Functions/randomString.cpp | 5 +++-- src/Functions/randomStringUTF8.cpp | 21 +++++++------------ .../0_stateless/01278_random_string_utf8.sql | 3 --- 3 files changed, 11 insertions(+), 18 deletions(-) diff --git a/src/Functions/randomString.cpp b/src/Functions/randomString.cpp index 1c2d595210b..4ea470f0a65 100644 --- a/src/Functions/randomString.cpp +++ b/src/Functions/randomString.cpp @@ -44,8 +44,9 @@ public: "Function " + getName() + " requires at most two arguments: the size of resulting string and optional disambiguation tag", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - if (!isUnsignedInteger(*arguments[0])) - throw Exception("First argument for function " + getName() + " must be unsigned integer", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + const IDataType & length_type = *arguments[0]; + if (!isNumber(length_type)) + throw Exception("First argument of function " + getName() + " must have numeric type", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); return std::make_shared(); } diff --git a/src/Functions/randomStringUTF8.cpp b/src/Functions/randomStringUTF8.cpp index db8b5028047..b462849c4d3 100644 --- a/src/Functions/randomStringUTF8.cpp +++ b/src/Functions/randomStringUTF8.cpp @@ -38,8 +38,8 @@ public: DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { - if (!isUnsignedInteger(*arguments[0])) - throw Exception("First argument for function " + getName() + " must be unsigned integer", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if (!isNumber(*arguments[0])) + throw Exception("First argument of function " + getName() + " must have numeric type", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); return std::make_shared(); } @@ -86,7 +86,7 @@ public: /// Generate highest byte in [0, 6] /// https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ UInt32 code_point = (rand >> 16) * 7u; - code_point &= ((-1) ^ 0xFFFF); + code_point &= ~0xFFFF; code_point |= rand & 0xFFFF; // and other bytes obtaining in a simple way if (code_point >= 0x40000) @@ -97,14 +97,9 @@ public: if (0xD7FF < code_point && code_point < 0xE000) // this range will not be valid in isValidUTF8 { - /* TODO(reviewer) choose with @axolm variant: - * 1. Not to do this if (isValidUTF8 can return 0) - * 2. just return 0 - * 3. capture rng in lambda and do while(code_point is bad) { recalc... } - * 4. ... - * */ return 0u; } + return code_point; }; @@ -114,7 +109,7 @@ public: size_t utf8_len = length_column.getUInt(row_num); auto * pos = data_to.data() + offset; - size_t last_writed_bytes = 0; + size_t last_writen_bytes = 0; size_t i = 0; for (; i < utf8_len; i += 2) { @@ -125,13 +120,13 @@ public: /// We have padding in column buffers that we can overwrite. pos += UTF8::convert(*reinterpret_cast(&code_point1), pos, sizeof(int)); - last_writed_bytes = UTF8::convert(*reinterpret_cast(&code_point2), pos, sizeof(int)); - pos += last_writed_bytes; + last_writen_bytes = UTF8::convert(*reinterpret_cast(&code_point2), pos, sizeof(int)); + pos += last_writen_bytes; } offset = pos - data_to.data() + 1; if (i > utf8_len) { - offset -= last_writed_bytes; + offset -= last_writen_bytes; } offsets_to[row_num] = offset; } diff --git a/tests/queries/0_stateless/01278_random_string_utf8.sql b/tests/queries/0_stateless/01278_random_string_utf8.sql index 72b7a273f56..f2c0a6c41c7 100644 --- a/tests/queries/0_stateless/01278_random_string_utf8.sql +++ b/tests/queries/0_stateless/01278_random_string_utf8.sql @@ -1,8 +1,5 @@ SELECT randomStringUTF8('string'); -- { serverError 43 } -SELECT randomStringUTF8(-10); -- { serverError 43 } SELECT lengthUTF8(randomStringUTF8(100)); SELECT toTypeName(randomStringUTF8(10)); SELECT isValidUTF8(randomStringUTF8(100000)); SELECT randomStringUTF8(0); --- SELECT DISTINCT c > 30000 FROM (SELECT arrayJoin(arrayMap(x -> reinterpretAsUInt8(substring(randomStringUTF8(100), x + 1, 1)), range(100))) AS byte, count() AS c FROM numbers(100000) GROUP BY byte ORDER BY byte); - From db07c9f5e9e5c0c1941332df22e6e0b2cb299ef2 Mon Sep 17 00:00:00 2001 From: Andrei Nekrashevich Date: Sun, 17 May 2020 16:22:52 +0300 Subject: [PATCH 321/738] fix --- src/Functions/randomStringUTF8.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Functions/randomStringUTF8.cpp b/src/Functions/randomStringUTF8.cpp index b462849c4d3..8635cd92c9b 100644 --- a/src/Functions/randomStringUTF8.cpp +++ b/src/Functions/randomStringUTF8.cpp @@ -78,9 +78,9 @@ public: size_t size_in_bytes_with_margin = summary_utf8_len * 4 + input_rows_count; data_to.resize(size_in_bytes_with_margin); - pcg64_fast rng(randomSeed()); /// TODO It is inefficient. We should use SIMD PRNG instead. + pcg64_fast rng(randomSeed()); // TODO It is inefficient. We should use SIMD PRNG instead. - auto generate_code_point = [](UInt32 rand) { + const auto generate_code_point = [](UInt32 rand) -> UInt32 { /// We want to generate number in [0x0, 0x70000) and shift it if need /// Generate highest byte in [0, 6] @@ -97,6 +97,7 @@ public: if (0xD7FF < code_point && code_point < 0xE000) // this range will not be valid in isValidUTF8 { + /// The distribution will be slightly non-uniform but we don't care. return 0u; } From 31035a3cd10d6158eb45bcfc20a89f3ffd660346 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 17 May 2020 17:01:10 +0300 Subject: [PATCH 322/738] Allow relative path for renameat2 by using AT_FDCWD Otherwise this will trigger logical error on CREATE TABLE with default config.d overrides (path=./). --- src/Common/renameat2.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Common/renameat2.cpp b/src/Common/renameat2.cpp index 23f2788ac76..323b72267a6 100644 --- a/src/Common/renameat2.cpp +++ b/src/Common/renameat2.cpp @@ -4,6 +4,7 @@ #if defined(linux) || defined(__linux) || defined(__linux__) #include +#include #include #include #include @@ -51,13 +52,11 @@ static void renameat2(const std::string & old_path, const std::string & new_path { if (old_path.empty() || new_path.empty()) throw Exception("Cannot rename " + old_path + " to " + new_path + ": path is empty", ErrorCodes::LOGICAL_ERROR); - if (old_path[0] != '/' || new_path[0] != '/') - throw Exception("Cannot rename " + old_path + " to " + new_path + ": path is relative", ErrorCodes::LOGICAL_ERROR); /// int olddirfd (ignored for absolute oldpath), const char *oldpath, /// int newdirfd (ignored for absolute newpath), const char *newpath, /// unsigned int flags - if (0 == syscall(__NR_renameat2, 0, old_path.c_str(), 0, new_path.c_str(), flags)) + if (0 == syscall(__NR_renameat2, AT_FDCWD, old_path.c_str(), AT_FDCWD, new_path.c_str(), flags)) return; if (errno == EEXIST) From fb6b19d57a4fabdcb733e733970a54b3c9d20ecb Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 17 May 2020 17:14:51 +0300 Subject: [PATCH 323/738] Fix symlinking data directory for Atomic database --- src/Databases/DatabaseAtomic.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp index 09ff1b8944e..2894b65274f 100644 --- a/src/Databases/DatabaseAtomic.cpp +++ b/src/Databases/DatabaseAtomic.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -324,7 +325,7 @@ void DatabaseAtomic::tryCreateSymlink(const String & table_name, const String & try { String link = path_to_table_symlinks + escapeForFileName(table_name); - String data = global_context.getPath() + actual_data_path; + String data = Poco::Path(global_context.getPath()).makeAbsolute().toString() + actual_data_path; Poco::File{data}.linkTo(link, Poco::File::LINK_SYMBOLIC); } catch (...) From dbe781451af775d0a99de80a97cc17e9d53e334b Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 17 May 2020 17:44:21 +0300 Subject: [PATCH 324/738] Redirect librdkafka logs from stderr to clickhouse logger --- src/Storages/Kafka/StorageKafka.cpp | 8 +++++++ src/Storages/Kafka/parseSyslogLevel.cpp | 32 +++++++++++++++++++++++++ src/Storages/Kafka/parseSyslogLevel.h | 7 ++++++ 3 files changed, 47 insertions(+) create mode 100644 src/Storages/Kafka/parseSyslogLevel.cpp create mode 100644 src/Storages/Kafka/parseSyslogLevel.h diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index 2e2797983b6..591ef33eef2 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -278,6 +279,13 @@ void StorageKafka::updateConfiguration(cppkafka::Configuration & conf) if (config.has(topic_config_key)) loadFromConfig(conf, config, topic_config_key); } + + // No need to add any prefix, messages can be distinguished + conf.set_log_callback([this](cppkafka::KafkaHandleBase &, int level, const std::string & /* facility */, const std::string & message) + { + auto [poco_level, client_logs_level] = parseSyslogLevel(level); + LOG_SIMPLE(log, message, client_logs_level, poco_level); + }); } bool StorageKafka::checkDependencies(const StorageID & table_id) diff --git a/src/Storages/Kafka/parseSyslogLevel.cpp b/src/Storages/Kafka/parseSyslogLevel.cpp new file mode 100644 index 00000000000..692bc03000b --- /dev/null +++ b/src/Storages/Kafka/parseSyslogLevel.cpp @@ -0,0 +1,32 @@ +#include "parseSyslogLevel.h" +#include + +/// Must be in a sepearate compilation unit due to macros overlaps: +/// - syslog (LOG_DEBUG/...) +/// - logger_useful.h (LOG_DEBUG()/...) +std::pair parseSyslogLevel(const int level) +{ + using DB::LogsLevel; + using Poco::Message; + + switch (level) + { + case LOG_EMERG: [[fallthrough]]; + case LOG_ALERT: + return std::make_pair(Message::PRIO_FATAL, LogsLevel::error); + case LOG_CRIT: + return std::make_pair(Message::PRIO_CRITICAL, LogsLevel::error); + case LOG_ERR: + return std::make_pair(Message::PRIO_ERROR, LogsLevel::error); + case LOG_WARNING: + return std::make_pair(Message::PRIO_WARNING, LogsLevel::warning); + case LOG_NOTICE: + return std::make_pair(Message::PRIO_NOTICE, LogsLevel::information); + case LOG_INFO: + return std::make_pair(Message::PRIO_INFORMATION, LogsLevel::information); + case LOG_DEBUG: + return std::make_pair(Message::PRIO_DEBUG, LogsLevel::debug); + default: + return std::make_pair(Message::PRIO_TRACE, LogsLevel::trace); + } +} diff --git a/src/Storages/Kafka/parseSyslogLevel.h b/src/Storages/Kafka/parseSyslogLevel.h new file mode 100644 index 00000000000..f0fb4968631 --- /dev/null +++ b/src/Storages/Kafka/parseSyslogLevel.h @@ -0,0 +1,7 @@ +#pragma once + +#include +#include +#include + +std::pair parseSyslogLevel(const int level); From e3d69eea88da8e7a007924840132c0f22373e8db Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 17 May 2020 18:51:48 +0300 Subject: [PATCH 325/738] Configure thread names for librdkafka First of all it is nice to have part of the table name in it. And secondly, librdkafka uses pthread_setname_np(), but due to glibc-compatibility it is a noop in clickhouse sources: # librdkafka uses rdk: prefix for thread names by default $ grep rdk /proc/$(pgrep clickhouse-serv)/task/*/comm # just in case: $ grep rdk /proc/*/task/*/comm $ grep rdk /proc/*/comm (gdb) disas pthread_setname_np Dump of assembler code for function pthread_setname_np: 0x000000000c603250 <+0>: xor %eax,%eax 0x000000000c603252 <+2>: retq End of assembler dump. --- src/Storages/Kafka/StorageKafka.cpp | 61 +++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index 591ef33eef2..5c4657403b7 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -33,6 +33,7 @@ #include #include #include +#include namespace DB @@ -68,6 +69,46 @@ namespace conf.set(key_name, config.getString(key_path)); } } + + rd_kafka_resp_err_t rdKafkaOnThreadStart(rd_kafka_t *, rd_kafka_thread_type_t thread_type, const char *, void * ctx) + { + StorageKafka * self = reinterpret_cast(ctx); + + const auto & storage_id = self->getStorageID(); + const auto & table = storage_id.getTableName(); + + switch (thread_type) + { + case RD_KAFKA_THREAD_MAIN: + setThreadName(("rdk:m/" + table.substr(0, 9)).c_str()); + break; + case RD_KAFKA_THREAD_BACKGROUND: + setThreadName(("rdk:bg/" + table.substr(0, 8)).c_str()); + break; + case RD_KAFKA_THREAD_BROKER: + setThreadName(("rdk:b/" + table.substr(0, 9)).c_str()); + break; + } + return RD_KAFKA_RESP_ERR_NO_ERROR; + } + + rd_kafka_resp_err_t rdKafkaOnNew(rd_kafka_t * rk, const rd_kafka_conf_t *, void * ctx, char * /*errstr*/, size_t /*errstr_size*/) + { + return rd_kafka_interceptor_add_on_thread_start(rk, "setThreadName", rdKafkaOnThreadStart, ctx); + } + + rd_kafka_resp_err_t rdKafkaOnConfDup(rd_kafka_conf_t * new_conf, const rd_kafka_conf_t * /*old_conf*/, size_t /*filter_cnt*/, const char ** /*filter*/, void * ctx) + { + rd_kafka_resp_err_t status; + + // cppkafka copies configuration multiple times + status = rd_kafka_conf_interceptor_add_on_conf_dup(new_conf, "setThreadName", rdKafkaOnConfDup, ctx); + if (status != RD_KAFKA_RESP_ERR_NO_ERROR) + return status; + + status = rd_kafka_conf_interceptor_add_on_new(new_conf, "setThreadName", rdKafkaOnNew, ctx); + return status; + } } StorageKafka::StorageKafka( @@ -286,6 +327,26 @@ void StorageKafka::updateConfiguration(cppkafka::Configuration & conf) auto [poco_level, client_logs_level] = parseSyslogLevel(level); LOG_SIMPLE(log, message, client_logs_level, poco_level); }); + + // Configure interceptor to change thread name + // + // TODO: add interceptors support into the cppkafka. + // XXX: rdkafka uses pthread_set_name_np(), but glibc-compatibliity overrides it to noop. + { + // This should be safe, since we wait the rdkafka object anyway. + void * self = reinterpret_cast(this); + + int status; + + status = rd_kafka_conf_interceptor_add_on_new(conf.get_handle(), "setThreadName", rdKafkaOnNew, self); + if (status != RD_KAFKA_RESP_ERR_NO_ERROR) + LOG_ERROR(log, "Cannot set new interceptor"); + + // cppkafka always copy the configuration + status = rd_kafka_conf_interceptor_add_on_conf_dup(conf.get_handle(), "setThreadName", rdKafkaOnConfDup, self); + if (status != RD_KAFKA_RESP_ERR_NO_ERROR) + LOG_ERROR(log, "Cannot set dup conf interceptor"); + } } bool StorageKafka::checkDependencies(const StorageID & table_id) From c8e7ea0d6c11e15a52e47cfe9e54c729bc17fad5 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Sun, 17 May 2020 21:55:49 +0300 Subject: [PATCH 326/738] Try fix performance. --- src/Common/COW.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Common/COW.h b/src/Common/COW.h index 3c2171436bf..7e4bf3e8056 100644 --- a/src/Common/COW.h +++ b/src/Common/COW.h @@ -216,6 +216,7 @@ protected: operator const immutable_ptr & () const { return value; } operator immutable_ptr & () { return value; } + operator immutable_ptr () && { return std::move(value); } operator bool() const { return value != nullptr; } bool operator! () const { return value == nullptr; } From ca1a9b890db126e04be62ecaad65ab6c1b220ef1 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sun, 17 May 2020 22:36:27 +0300 Subject: [PATCH 327/738] Revert "Try to fix use-after-free error in MergeTree" --- src/Storages/StorageMergeTree.cpp | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 91add9a8104..68d468233a8 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -93,18 +93,9 @@ void StorageMergeTree::startup() /// NOTE background task will also do the above cleanups periodically. time_after_previous_cleanup.restart(); - - auto & pool = global_context.getBackgroundPool(); - - merging_mutating_task_handle = pool.createTask([this] { return mergeMutateTask(); }); - /// Ensure that thread started only after assignment to 'merging_mutating_task_handle' is done. - pool.startTask(merging_mutating_task_handle); - + merging_mutating_task_handle = global_context.getBackgroundPool().addTask([this] { return mergeMutateTask(); }); if (areBackgroundMovesNeeded()) - { - moving_task_handle = pool.createTask([this] { return movePartsTask(); }); - pool.startTask(moving_task_handle); - } + moving_task_handle = global_context.getBackgroundMovePool().addTask([this] { return movePartsTask(); }); } From 20dbd6c3ef3c23e0bfee9d0c022d9b4dadda1526 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 17 May 2020 22:41:49 +0300 Subject: [PATCH 328/738] Fix error --- src/Storages/StorageMergeTree.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 91add9a8104..21d64a3f100 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -94,16 +94,16 @@ void StorageMergeTree::startup() /// NOTE background task will also do the above cleanups periodically. time_after_previous_cleanup.restart(); - auto & pool = global_context.getBackgroundPool(); - - merging_mutating_task_handle = pool.createTask([this] { return mergeMutateTask(); }); + auto & merge_pool = global_context.getBackgroundPool(); + merging_mutating_task_handle = merge_pool.createTask([this] { return mergeMutateTask(); }); /// Ensure that thread started only after assignment to 'merging_mutating_task_handle' is done. - pool.startTask(merging_mutating_task_handle); + merge_pool.startTask(merging_mutating_task_handle); if (areBackgroundMovesNeeded()) { - moving_task_handle = pool.createTask([this] { return movePartsTask(); }); - pool.startTask(moving_task_handle); + auto & move_pool = global_context.getBackgroundMovePool() + moving_task_handle = move_pool.createTask([this] { return movePartsTask(); }); + move_pool.startTask(moving_task_handle); } } From 70c31cbdc496bdeaef294c930c4218ce4ad9ac2c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 17 May 2020 22:44:31 +0300 Subject: [PATCH 329/738] Fix error --- src/Storages/StorageMergeTree.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 21d64a3f100..ad94e2db783 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -101,7 +101,7 @@ void StorageMergeTree::startup() if (areBackgroundMovesNeeded()) { - auto & move_pool = global_context.getBackgroundMovePool() + auto & move_pool = global_context.getBackgroundMovePool(); moving_task_handle = move_pool.createTask([this] { return movePartsTask(); }); move_pool.startTask(moving_task_handle); } From 6deea35fc94d010bd1e3fbebcc8d7c8c02863f25 Mon Sep 17 00:00:00 2001 From: Andrei Nekrashevich Date: Sun, 17 May 2020 22:54:30 +0300 Subject: [PATCH 330/738] submodules --- contrib/aws | 2 +- contrib/boost | 2 +- contrib/poco | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/aws b/contrib/aws index 04d54dfa034..fb5c604525f 160000 --- a/contrib/aws +++ b/contrib/aws @@ -1 +1 @@ -Subproject commit 04d54dfa0342d9465fb2eb3bfd4b77a3f7682e99 +Subproject commit fb5c604525f5151d75a856462653e7e38b559b79 diff --git a/contrib/boost b/contrib/boost index 86be2aef20b..a04e72c0464 160000 --- a/contrib/boost +++ b/contrib/boost @@ -1 +1 @@ -Subproject commit 86be2aef20bee2356b744e5569eed6eaded85dbe +Subproject commit a04e72c0464f0c31d3384f18f0c0db36a05538e0 diff --git a/contrib/poco b/contrib/poco index 9b347d4ab71..be2ab90ba5d 160000 --- a/contrib/poco +++ b/contrib/poco @@ -1 +1 @@ -Subproject commit 9b347d4ab71e4436d8215aace978024e93462731 +Subproject commit be2ab90ba5dccd46919a116e3fe4fa77bb85063b From 361b71efa808629a071532ae989b0d4613984039 Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Sun, 17 May 2020 23:19:03 +0300 Subject: [PATCH 331/738] Update changelog --- CHANGELOG.md | 179 +++++++++++++++++++++++---------------------------- 1 file changed, 81 insertions(+), 98 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b5d23f9bb3..d3826a33fba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,166 +85,155 @@ * Fix nullptr dereference in StorageBuffer if server was shutdown before table startup. [#10641](https://github.com/ClickHouse/ClickHouse/pull/10641) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Fixed `DROP` vs `OPTIMIZE` race in `ReplicatedMergeTree`. `DROP` could left some garbage in replica path in ZooKeeper if there was concurrent `OPTIMIZE` query. [#10312](https://github.com/ClickHouse/ClickHouse/pull/10312) ([tavplubix](https://github.com/tavplubix)) * Fix 'Logical error: CROSS JOIN has expressions' error for queries with comma and names joins mix. Fixes [#9910](https://github.com/ClickHouse/ClickHouse/issues/9910) [#10311](https://github.com/ClickHouse/ClickHouse/pull/10311) ([Artem Zuikov](https://github.com/4ertus2)) - * Fix queries with `max_bytes_before_external_group_by`. [#10302](https://github.com/ClickHouse/ClickHouse/pull/10302) ([Artem Zuikov](https://github.com/4ertus2)) * Fix the issue with limiting maximum recursion depth in parser in certain cases. This fixes [#10283](https://github.com/ClickHouse/ClickHouse/issues/10283) This fix may introduce minor incompatibility: long and deep queries via clickhouse-client may refuse to work, and you should adjust settings `max_query_size` and `max_parser_depth` accordingly. [#10295](https://github.com/ClickHouse/ClickHouse/pull/10295) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Allow to use `count(*)` with multiple JOINs. Fixes [#9853](https://github.com/ClickHouse/ClickHouse/issues/9853) [#10291](https://github.com/ClickHouse/ClickHouse/pull/10291) ([Artem Zuikov](https://github.com/4ertus2)) -* Remove converting ColumnFixedString to FixedString in MsgPack fromat and add performance test. ... [#10216](https://github.com/ClickHouse/ClickHouse/pull/10216) ([Kruglov Pavel](https://github.com/Avogar)) -* Fix error `Pipeline stuck` with `max_rows_to_group_by` and `group_by_overflow_mode = \'break\'`. [#10279](https://github.com/ClickHouse/ClickHouse/pull/10279) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Fix `\'Cannot add column\'` error while creating `range_hashed` dictionary using DDL query. Fixes [#10093](https://github.com/ClickHouse/ClickHouse/issues/10093). [#10235](https://github.com/ClickHouse/ClickHouse/pull/10235) ([alesapin](https://github.com/alesapin)) -* Fix rare possible exception `Cannot drain connections: cancel first`. ... [#10239](https://github.com/ClickHouse/ClickHouse/pull/10239) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Fixed bug where ClickHouse would throw "Unknown function lambda." error message when user tries to run ALTER UPDATE/DELETE on tables with ENGINE = Replicated*. Check for nondeterministic functions now handles lambda expressions correctly. ... [#10237](https://github.com/ClickHouse/ClickHouse/pull/10237) ([Alexander Kazakov](https://github.com/Akazz)) +* Remove converting ColumnFixedString to FixedString in MsgPack format and add performance test. [#10216](https://github.com/ClickHouse/ClickHouse/pull/10216) ([Kruglov Pavel](https://github.com/Avogar)) +* Fix error `Pipeline stuck` with `max_rows_to_group_by` and `group_by_overflow_mode = 'break'`. [#10279](https://github.com/ClickHouse/ClickHouse/pull/10279) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fix 'Cannot add column' error while creating `range_hashed` dictionary using DDL query. Fixes [#10093](https://github.com/ClickHouse/ClickHouse/issues/10093). [#10235](https://github.com/ClickHouse/ClickHouse/pull/10235) ([alesapin](https://github.com/alesapin)) +* Fix rare possible exception `Cannot drain connections: cancel first`. [#10239](https://github.com/ClickHouse/ClickHouse/pull/10239) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fixed bug where ClickHouse would throw "Unknown function lambda." error message when user tries to run ALTER UPDATE/DELETE on tables with ENGINE = Replicated*. Check for nondeterministic functions now handles lambda expressions correctly. [#10237](https://github.com/ClickHouse/ClickHouse/pull/10237) ([Alexander Kazakov](https://github.com/Akazz)) * Fixed reasonably rare segfault in StorageSystemTables that happens when SELECT ... FROM system.tables is run on a database with Lazy engine. [#10209](https://github.com/ClickHouse/ClickHouse/pull/10209) ([Alexander Kazakov](https://github.com/Akazz)) -* Fix possible inifinite query execution when the query actually should stop on LIMIT, while reading from infinite source like `system.numbers` or `system.zeros`. ... [#10206](https://github.com/ClickHouse/ClickHouse/pull/10206) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fix possible infinite query execution when the query actually should stop on LIMIT, while reading from infinite source like `system.numbers` or `system.zeros`. [#10206](https://github.com/ClickHouse/ClickHouse/pull/10206) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) * Fixed "generateRandom" function for Date type. This fixes [#9973](https://github.com/ClickHouse/ClickHouse/issues/9973). Fix an edge case when dates with year 2106 are inserted to MergeTree tables with old-style partitioning but partitions are named with year 1970. [#10218](https://github.com/ClickHouse/ClickHouse/pull/10218) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Convert types if the table definition of a View does not correspond to the SELECT query. This fixes [#10180](https://github.com/ClickHouse/ClickHouse/issues/10180) and [#10022](https://github.com/ClickHouse/ClickHouse/issues/10022) [#10217](https://github.com/ClickHouse/ClickHouse/pull/10217) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Fix `parseDateTimeBestEffort` for strings in RFC-2822 when day of week is Tuesday or Thursday. This fixes [#10082](https://github.com/ClickHouse/ClickHouse/issues/10082) [#10214](https://github.com/ClickHouse/ClickHouse/pull/10214) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Fix column names of constants inside JOIN that may clash with names of constants outside of JOIN. [#10207](https://github.com/ClickHouse/ClickHouse/pull/10207) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Fix move-to-prewhere optimization in presense of arrayJoin functions (in certain cases). This fixes [#10092](https://github.com/ClickHouse/ClickHouse/issues/10092) [#10195](https://github.com/ClickHouse/ClickHouse/pull/10195) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fix scramble issue for native mysql-connector-java(JDBC) [#10140](https://github.com/ClickHouse/ClickHouse/pull/10140) ([BohuTANG](https://github.com/BohuTANG)) -* Fix using the current database for access checking when the database isn\'t specified. [#10192](https://github.com/ClickHouse/ClickHouse/pull/10192) ([Vitaly Baranov](https://github.com/vitlibar)) -* Fix some kinds of alters with compact parts. ... [#10130](https://github.com/ClickHouse/ClickHouse/pull/10130) ([Anton Popov](https://github.com/CurtizJ)) +* Fix issue with separator appearing in SCRAMBLE for native mysql-connector-java(JDBC) [#10140](https://github.com/ClickHouse/ClickHouse/pull/10140) ([BohuTANG](https://github.com/BohuTANG)) +* Fix using the current database for an access checking when the database isn't specified. [#10192](https://github.com/ClickHouse/ClickHouse/pull/10192) ([Vitaly Baranov](https://github.com/vitlibar)) +* Fix ALTER of tables with compact parts. [#10130](https://github.com/ClickHouse/ClickHouse/pull/10130) ([Anton Popov](https://github.com/CurtizJ)) * Add the ability to relax the restriction on non-deterministic functions usage in mutations with `allow_nondeterministic_mutations` setting. [#10186](https://github.com/ClickHouse/ClickHouse/pull/10186) ([filimonov](https://github.com/filimonov)) -* Do not break DROP DICTIONARY with DROP TABLE (otherwise after this error you will not be able to remove database) [#10165](https://github.com/ClickHouse/ClickHouse/pull/10165) ([Azat Khuzhin](https://github.com/azat)) -* Revert resetting the settings to their defaults after each query in TCPHandler. This PR reverts https://github.com/ClickHouse/ClickHouse/pull/9970 [#10181](https://github.com/ClickHouse/ClickHouse/pull/10181) ([Vitaly Baranov](https://github.com/vitlibar)) -* Convert blocks if structure does not match on INSERT into Distributed() ... [#10135](https://github.com/ClickHouse/ClickHouse/pull/10135) ([Azat Khuzhin](https://github.com/azat)) +* Fix `DROP TABLE` invoked for dictionary [#10165](https://github.com/ClickHouse/ClickHouse/pull/10165) ([Azat Khuzhin](https://github.com/azat)) +* Convert blocks if structure does not match when doing `INSERT` into Distributed table [#10135](https://github.com/ClickHouse/ClickHouse/pull/10135) ([Azat Khuzhin](https://github.com/azat)) * The number of rows was logged incorrectly (as sum across all parts) when inserted block is split by parts with partition key. [#10138](https://github.com/ClickHouse/ClickHouse/pull/10138) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Add some arguments check and support identifier arguments for MySQL Database Engine [#10077](https://github.com/ClickHouse/ClickHouse/pull/10077) ([Winter Zhang](https://github.com/zhang2014)) * Fix incorrect `index_granularity_bytes` check while creating new replica. Fixes [#10098](https://github.com/ClickHouse/ClickHouse/issues/10098). [#10121](https://github.com/ClickHouse/ClickHouse/pull/10121) ([alesapin](https://github.com/alesapin)) * Fix bug in `CHECK TABLE` query when table contain skip indices. [#10068](https://github.com/ClickHouse/ClickHouse/pull/10068) ([alesapin](https://github.com/alesapin)) -* Fix SIGSEGV on INSERT into Distributed table when its structure differs from the underlying tables. ... [#10105](https://github.com/ClickHouse/ClickHouse/pull/10105) ([Azat Khuzhin](https://github.com/azat)) -* Fix Distributed-over-Distributed with one only shard in nested table ... [#9997](https://github.com/ClickHouse/ClickHouse/pull/9997) ([Azat Khuzhin](https://github.com/azat)) +* Fix Distributed-over-Distributed with the only one shard in a nested table [#9997](https://github.com/ClickHouse/ClickHouse/pull/9997) ([Azat Khuzhin](https://github.com/azat)) * Fix possible rows loss for queries with `JOIN` and `UNION ALL`. Fixes [#9826](https://github.com/ClickHouse/ClickHouse/issues/9826), [#10113](https://github.com/ClickHouse/ClickHouse/issues/10113). ... [#10099](https://github.com/ClickHouse/ClickHouse/pull/10099) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Fix bug in clickhouse dictionary source from localhost clickhouse server. The bug may lead to memory corruption if types in dictionary and source are not compatible. [#10071](https://github.com/ClickHouse/ClickHouse/pull/10071) ([alesapin](https://github.com/alesapin)) -* Fixed replicated tables startup when updating from an old ClickHouse version where `/table/replicas/replica_name/metadata` node doesn\'t exist. Fixes [#10037](https://github.com/ClickHouse/ClickHouse/issues/10037). [#10095](https://github.com/ClickHouse/ClickHouse/pull/10095) ([alesapin](https://github.com/alesapin)) -* Fix error `Cannot clone block with columns because block has 0 columns ... While executing GroupingAggregatedTransform`. It happened when setting `distributed_aggregation_memory_efficient` was enabled, and distributed query read aggregating data with different level from different shards (mixed single and two level aggregation). ... [#10063](https://github.com/ClickHouse/ClickHouse/pull/10063) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Fix deadlock on failed database attach at start with materialized view [#10054](https://github.com/ClickHouse/ClickHouse/pull/10054) ([Azat Khuzhin](https://github.com/azat)) +* Fix bug in dictionary when local clickhouse server is used as source. It may caused memory corruption if types in dictionary and source are not compatible. [#10071](https://github.com/ClickHouse/ClickHouse/pull/10071) ([alesapin](https://github.com/alesapin)) +* Fixed replicated tables startup when updating from an old ClickHouse version where `/table/replicas/replica_name/metadata` node doesn't exist. Fixes [#10037](https://github.com/ClickHouse/ClickHouse/issues/10037). [#10095](https://github.com/ClickHouse/ClickHouse/pull/10095) ([alesapin](https://github.com/alesapin)) +* Fix error `Cannot clone block with columns because block has 0 columns ... While executing GroupingAggregatedTransform`. It happened when setting `distributed_aggregation_memory_efficient` was enabled, and distributed query read aggregating data with mixed single and two-level aggregation from different shards. [#10063](https://github.com/ClickHouse/ClickHouse/pull/10063) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fix deadlock when database with materialized view failed attach at start [#10054](https://github.com/ClickHouse/ClickHouse/pull/10054) ([Azat Khuzhin](https://github.com/azat)) * Fix a segmentation fault that could occur in GROUP BY over string keys containing trailing zero bytes ([#8636](https://github.com/ClickHouse/ClickHouse/issues/8636), [#8925](https://github.com/ClickHouse/ClickHouse/issues/8925)). ... [#10025](https://github.com/ClickHouse/ClickHouse/pull/10025) ([Alexander Kuzmenkov](https://github.com/akuzm)) -* Fix parallel distributed INSERT SELECT for remote table. This PR fixes the solution provided in [#9759](https://github.com/ClickHouse/ClickHouse/pull/9759) ... [#9999](https://github.com/ClickHouse/ClickHouse/pull/9999) ([Vitaly Baranov](https://github.com/vitlibar)) * Fix wrong results of distributed queries when alias could override qualified column name. Fixes [#9672](https://github.com/ClickHouse/ClickHouse/issues/9672) [#9714](https://github.com/ClickHouse/ClickHouse/issues/9714) [#9972](https://github.com/ClickHouse/ClickHouse/pull/9972) ([Artem Zuikov](https://github.com/4ertus2)) * Fix possible deadlock in `SYSTEM RESTART REPLICAS` [#9955](https://github.com/ClickHouse/ClickHouse/pull/9955) ([tavplubix](https://github.com/tavplubix)) * Fix the number of threads used for remote query execution (performance regression, since 20.3). This happened when query from `Distributed` table was executed simultaneously on local and remote shards. Fixes [#9965](https://github.com/ClickHouse/ClickHouse/issues/9965) [#9971](https://github.com/ClickHouse/ClickHouse/pull/9971) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Fix reusing connection after changing user\'s default settings via `ALTER USER` or `ALTER SETTINGS PROFILE`. This fixes integration test `test_settings_constraints_distributed`. [#9970](https://github.com/ClickHouse/ClickHouse/pull/9970) ([Vitaly Baranov](https://github.com/vitlibar)) * Fixed `DeleteOnDestroy` logic in `ATTACH PART` which could lead to automatic removal of attached part and added few tests [#9410](https://github.com/ClickHouse/ClickHouse/pull/9410) ([Vladimir Chebotarev](https://github.com/excitoon)) * Fix a bug with `ON CLUSTER` DDL queries freezing on server startup. [#9927](https://github.com/ClickHouse/ClickHouse/pull/9927) ([Gagan Arneja](https://github.com/garneja)) -* Fix two bugs in `windowFunnel` ... [#9938](https://github.com/ClickHouse/ClickHouse/pull/9938) ([achimbab](https://github.com/achimbab)) -* Fix bug in which the necessary tables weren\'t retrieved at one of the processing stages of queries to some databases. Fixes [#9699](https://github.com/ClickHouse/ClickHouse/issues/9699). [#9949](https://github.com/ClickHouse/ClickHouse/pull/9949) ([achulkov2](https://github.com/achulkov2)) -* Fix \'Not found column in block\' error when `JOIN` appears with `TOTALS`. Fixes [#9839](https://github.com/ClickHouse/ClickHouse/issues/9839) [#9939](https://github.com/ClickHouse/ClickHouse/pull/9939) ([Artem Zuikov](https://github.com/4ertus2)) -* Fix parsing multiple hosts set in the CREATE USER command, e.g. ... [#9924](https://github.com/ClickHouse/ClickHouse/pull/9924) ([Vitaly Baranov](https://github.com/vitlibar)) +* Fix bug in which the necessary tables weren't retrieved at one of the processing stages of queries to some databases. Fixes [#9699](https://github.com/ClickHouse/ClickHouse/issues/9699). [#9949](https://github.com/ClickHouse/ClickHouse/pull/9949) ([achulkov2](https://github.com/achulkov2)) +* Fix 'Not found column in block' error when `JOIN` appears with `TOTALS`. Fixes [#9839](https://github.com/ClickHouse/ClickHouse/issues/9839) [#9939](https://github.com/ClickHouse/ClickHouse/pull/9939) ([Artem Zuikov](https://github.com/4ertus2)) +* Fix parsing multiple hosts set in the CREATE USER command [#9924](https://github.com/ClickHouse/ClickHouse/pull/9924) ([Vitaly Baranov](https://github.com/vitlibar)) * Fix `TRUNCATE` for Join table engine ([#9917](https://github.com/ClickHouse/ClickHouse/issues/9917)). [#9920](https://github.com/ClickHouse/ClickHouse/pull/9920) ([Amos Bird](https://github.com/amosbird)) * Fix race condition between drop and optimize in `ReplicatedMergeTree`. [#9901](https://github.com/ClickHouse/ClickHouse/pull/9901) ([alesapin](https://github.com/alesapin)) -* Fix DISTINCT for Distributed when `optimize_skip_unused_shards` is set. [#9808](https://github.com/ClickHouse/ClickHouse/pull/9808) ([Azat Khuzhin](https://github.com/azat)) -* Fix "scalar doesn\'t exist" error in ALTERs ([#9878](https://github.com/ClickHouse/ClickHouse/issues/9878)). ... [#9904](https://github.com/ClickHouse/ClickHouse/pull/9904) ([Amos Bird](https://github.com/amosbird)) +* Fix `DISTINCT` for Distributed when `optimize_skip_unused_shards` is set. [#9808](https://github.com/ClickHouse/ClickHouse/pull/9808) ([Azat Khuzhin](https://github.com/azat)) +* Fix "scalar doesn't exist" error in ALTERs ([#9878](https://github.com/ClickHouse/ClickHouse/issues/9878)). ... [#9904](https://github.com/ClickHouse/ClickHouse/pull/9904) ([Amos Bird](https://github.com/amosbird)) * Fix error with qualified names in `distributed_product_mode=\'local\'`. Fixes [#4756](https://github.com/ClickHouse/ClickHouse/issues/4756) [#9891](https://github.com/ClickHouse/ClickHouse/pull/9891) ([Artem Zuikov](https://github.com/4ertus2)) -* For INSERT queries shard now clamps the settings got from the initiator to the shard\'s constaints instead of throwing an exception. This fix allows to send INSERT queries to a shard with another constraints. This change improves fix [#9447](https://github.com/ClickHouse/ClickHouse/issues/9447). [#9852](https://github.com/ClickHouse/ClickHouse/pull/9852) ([Vitaly Baranov](https://github.com/vitlibar)) -* Introduce commit retry logic to decrease the possibility of getting duplicates from Kafka in rare cases when offset commit was failed. [#9884](https://github.com/ClickHouse/ClickHouse/pull/9884) ([filimonov](https://github.com/filimonov)) -* Sometimes broker can reject commit if during `offsets.commit.timeout.ms` (broker setting, 5000 by default) there were no enough replicas available for the `__consumer_offsets` topic, also some other temporary issues like client-server connectivity problems are possible. Retries make that part more robust. ... [#10144](https://github.com/ClickHouse/ClickHouse/pull/10144) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Fix Distributed engine with virtual columns of the underlying table in WHERE after [#8846](https://github.com/ClickHouse/ClickHouse/issues/8846) (requires `optimize_skip_unused_shards`) [#9847](https://github.com/ClickHouse/ClickHouse/pull/9847) ([Azat Khuzhin](https://github.com/azat)) -* Fixed a few cases when timezone of the function argument wasn\'t used properly. [#9574](https://github.com/ClickHouse/ClickHouse/pull/9574) ([Vasily Nemkov](https://github.com/Enmk)) -* Fix \'Different expressions with the same alias\' error when query has PREWHERE and WHERE on distributed table and `SET distributed_product_mode = \'local\'`. [#9871](https://github.com/ClickHouse/ClickHouse/pull/9871) ([Artem Zuikov](https://github.com/4ertus2)) +* For INSERT queries shards now do clamp the settings from the initiator to their constraints instead of throwing an exception. This fix allows to send INSERT queries to a shard with another constraints. This change improves fix [#9447](https://github.com/ClickHouse/ClickHouse/issues/9447). [#9852](https://github.com/ClickHouse/ClickHouse/pull/9852) ([Vitaly Baranov](https://github.com/vitlibar)) +* Add some retries when commiting offsets to Kafka broker, since it can reject commit if during `offsets.commit.timeout.ms` there were no enough replicas available for the `__consumer_offsets` topic [#9884](https://github.com/ClickHouse/ClickHouse/pull/9884) ([filimonov](https://github.com/filimonov)) +* Fix Distributed engine behavior when virtual columns of the underlying table used in `WHERE` [#9847](https://github.com/ClickHouse/ClickHouse/pull/9847) ([Azat Khuzhin](https://github.com/azat)) +* Fixed some cases when timezone of the function argument wasn't used properly. [#9574](https://github.com/ClickHouse/ClickHouse/pull/9574) ([Vasily Nemkov](https://github.com/Enmk)) +* Fix 'Different expressions with the same alias' error when query has PREWHERE and WHERE on distributed table and `SET distributed_product_mode = 'local'`. [#9871](https://github.com/ClickHouse/ClickHouse/pull/9871) ([Artem Zuikov](https://github.com/4ertus2)) * Fix mutations excessive memory consumption for tables with a composite primary key. This fixes [#9850](https://github.com/ClickHouse/ClickHouse/issues/9850). [#9860](https://github.com/ClickHouse/ClickHouse/pull/9860) ([alesapin](https://github.com/alesapin)) -* Fix calculating grants for introspection functions from the setting \'allow_introspection_functions\'. [#9840](https://github.com/ClickHouse/ClickHouse/pull/9840) ([Vitaly Baranov](https://github.com/vitlibar)) +* Fix calculating grants for introspection functions from the setting `allow_introspection_functions`. [#9840](https://github.com/ClickHouse/ClickHouse/pull/9840) ([Vitaly Baranov](https://github.com/vitlibar)) * Fix max_distributed_connections (w/ and w/o Processors) [#9673](https://github.com/ClickHouse/ClickHouse/pull/9673) ([Azat Khuzhin](https://github.com/azat)) * Fix possible exception `Got 0 in totals chunk, expected 1` on client. It happened for queries with `JOIN` in case if right joined table had zero rows. Example: `select * from system.one t1 join system.one t2 on t1.dummy = t2.dummy limit 0 FORMAT TabSeparated;`. Fixes [#9777](https://github.com/ClickHouse/ClickHouse/issues/9777). ... [#9823](https://github.com/ClickHouse/ClickHouse/pull/9823) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Fix \'COMMA to CROSS JOIN rewriter is not enabled or cannot rewrite query\' error in case of subqueries with COMMA JOIN out of tables lists (i.e. in WHERE). Fixes [#9782](https://github.com/ClickHouse/ClickHouse/issues/9782) [#9830](https://github.com/ClickHouse/ClickHouse/pull/9830) ([Artem Zuikov](https://github.com/4ertus2)) -* Fix SIGSEGV with optimize_skip_unused_shards when type cannot be converted [#9804](https://github.com/ClickHouse/ClickHouse/pull/9804) ([Azat Khuzhin](https://github.com/azat)) +* Fix 'COMMA to CROSS JOIN rewriter is not enabled or cannot rewrite query' error in case of subqueries with COMMA JOIN out of tables lists (i.e. in WHERE). Fixes [#9782](https://github.com/ClickHouse/ClickHouse/issues/9782) [#9830](https://github.com/ClickHouse/ClickHouse/pull/9830) ([Artem Zuikov](https://github.com/4ertus2)) +* Fix server crashing when `optimize_skip_unused_shards` is set and expression for key can't be converted to its field type [#9804](https://github.com/ClickHouse/ClickHouse/pull/9804) ([Azat Khuzhin](https://github.com/azat)) * Fix empty string handling in `splitByString`. [#9767](https://github.com/ClickHouse/ClickHouse/pull/9767) ([hcz](https://github.com/hczhcz)) * Fix broken `ALTER TABLE DELETE COLUMN` query for compact parts. [#9779](https://github.com/ClickHouse/ClickHouse/pull/9779) ([alesapin](https://github.com/alesapin)) -* Fixed missing `rows_before_limit_at_least` for queries over http (with processors pipeline ). Fixes [#9730](https://github.com/ClickHouse/ClickHouse/issues/9730) [#9757](https://github.com/ClickHouse/ClickHouse/pull/9757) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Fix completion highlighting issue when case insensitive matches are found. Do not include in changelog as the regression was not in any release. [#9753](https://github.com/ClickHouse/ClickHouse/pull/9753) ([Amos Bird](https://github.com/amosbird)) +* Fixed missing `rows_before_limit_at_least` for queries over http (with processors pipeline). Fixes [#9730](https://github.com/ClickHouse/ClickHouse/issues/9730) [#9757](https://github.com/ClickHouse/ClickHouse/pull/9757) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) * Fix excessive memory consumption in `ALTER` queries (mutations). This fixes [#9533](https://github.com/ClickHouse/ClickHouse/issues/9533) and [#9670](https://github.com/ClickHouse/ClickHouse/issues/9670). [#9754](https://github.com/ClickHouse/ClickHouse/pull/9754) ([alesapin](https://github.com/alesapin)) -* Fix possible permanent "Cannot schedule a task" error (due to unhandled exception in `ParallelAggregatingBlockInputStream::Handler::onFinish/onFinishThread`) ... [#9154](https://github.com/ClickHouse/ClickHouse/pull/9154) ([Azat Khuzhin](https://github.com/azat)) +* Fix possible permanent "Cannot schedule a task" error. [#9154](https://github.com/ClickHouse/ClickHouse/pull/9154) ([Azat Khuzhin](https://github.com/azat)) * Fix bug in backquoting in external dictionaries DDL. Fixes [#9619](https://github.com/ClickHouse/ClickHouse/issues/9619). [#9734](https://github.com/ClickHouse/ClickHouse/pull/9734) ([alesapin](https://github.com/alesapin)) * Fixed data race in `text_log`. It does not correspond to any real bug. [#9726](https://github.com/ClickHouse/ClickHouse/pull/9726) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fix bug in a replication that doesn\'t allow replication to work if the user has executed mutations on the previous version. This fixes [#9645](https://github.com/ClickHouse/ClickHouse/issues/9645). [#9652](https://github.com/ClickHouse/ClickHouse/pull/9652) ([alesapin](https://github.com/alesapin)) -* Fixed incorrect internal function names for `sumKahan` and `sumWithOverflow`. I lead to exception while using this functions in remote queries. ... [#9636](https://github.com/ClickHouse/ClickHouse/pull/9636) ([Azat Khuzhin](https://github.com/azat)) +* Fix bug in a replication that doesn't allow replication to work if the user has executed mutations on the previous version. This fixes [#9645](https://github.com/ClickHouse/ClickHouse/issues/9645). [#9652](https://github.com/ClickHouse/ClickHouse/pull/9652) ([alesapin](https://github.com/alesapin)) +* Fixed incorrect internal function names for `sumKahan` and `sumWithOverflow`. It led to exception while using this functions in remote queries. [#9636](https://github.com/ClickHouse/ClickHouse/pull/9636) ([Azat Khuzhin](https://github.com/azat)) * Add setting `use_compact_format_in_distributed_parts_names` which allows to write files for `INSERT` queries into `Distributed` table with more compact format. This fixes [#9647](https://github.com/ClickHouse/ClickHouse/issues/9647). [#9653](https://github.com/ClickHouse/ClickHouse/pull/9653) ([alesapin](https://github.com/alesapin)) * Fix RIGHT and FULL JOIN with LowCardinality in JOIN keys. [#9610](https://github.com/ClickHouse/ClickHouse/pull/9610) ([Artem Zuikov](https://github.com/4ertus2)) -* Fix possible exceptions `Size of filter doesn\'t match size of column` and `Invalid number of rows in Chunk` in `MergeTreeRangeReader`. They could appear while executing `PREWHERE` in some cases. ... [#9612](https://github.com/ClickHouse/ClickHouse/pull/9612) ([Anton Popov](https://github.com/CurtizJ)) -* Allow ALTER ON CLUSTER of Distributed tables with internal replication. This fixes [#3268](https://github.com/ClickHouse/ClickHouse/issues/3268) [#9617](https://github.com/ClickHouse/ClickHouse/pull/9617) ([shinoi2](https://github.com/shinoi2)) -* Fixed the issue: timezone was not preserved if you write a simple arithmetic expression like `time + 1` (in contrast to an expression like `time + INTERVAL 1 SECOND`). This fixes [#5743](https://github.com/ClickHouse/ClickHouse/issues/5743) [#9323](https://github.com/ClickHouse/ClickHouse/pull/9323) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix possible exceptions `Size of filter doesn't match size of column` and `Invalid number of rows in Chunk` in `MergeTreeRangeReader`. They could appear while executing `PREWHERE` in some cases. [#9612](https://github.com/ClickHouse/ClickHouse/pull/9612) ([Anton Popov](https://github.com/CurtizJ)) +* Allow `ALTER ON CLUSTER` of Distributed tables with internal replication. This fixes [#3268](https://github.com/ClickHouse/ClickHouse/issues/3268) [#9617](https://github.com/ClickHouse/ClickHouse/pull/9617) ([shinoi2](https://github.com/shinoi2)) +* Fix issue when timezone was not preserved if you write a simple arithmetic expression like `time + 1` (in contrast to an expression like `time + INTERVAL 1 SECOND`). This fixes [#5743](https://github.com/ClickHouse/ClickHouse/issues/5743) [#9323](https://github.com/ClickHouse/ClickHouse/pull/9323) ([alexey-milovidov](https://github.com/alexey-milovidov)) #### Improvement * Use AWS SDK libraries from ClickHouse-Extras [#10527](https://github.com/ClickHouse/ClickHouse/pull/10527) ([Pavel Kovalenko](https://github.com/Jokser)) * Use time zone when comparing DateTime with string literal. This fixes [#5206](https://github.com/ClickHouse/ClickHouse/issues/5206). [#10515](https://github.com/ClickHouse/ClickHouse/pull/10515) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Print verbose diagnostic info if Decimal value cannot be parsed from text input format. [#10205](https://github.com/ClickHouse/ClickHouse/pull/10205) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Add tasks/memory metrics for distributed/buffer schedule pools ... [#10449](https://github.com/ClickHouse/ClickHouse/pull/10449) ([Azat Khuzhin](https://github.com/azat)) -* Display result as soon as it\'s ready for SELECT DISTINCT queries in clickhouse-local and HTTP interface. This fixes [#8951](https://github.com/ClickHouse/ClickHouse/issues/8951) [#9559](https://github.com/ClickHouse/ClickHouse/pull/9559) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Add tasks/memory metrics for distributed/buffer schedule pools [#10449](https://github.com/ClickHouse/ClickHouse/pull/10449) ([Azat Khuzhin](https://github.com/azat)) +* Display result as soon as it's ready for SELECT DISTINCT queries in clickhouse-local and HTTP interface. This fixes [#8951](https://github.com/ClickHouse/ClickHouse/issues/8951) [#9559](https://github.com/ClickHouse/ClickHouse/pull/9559) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Allow to use `SAMPLE OFFSET` query instead of `cityHash64(PRIMARY KEY) % N == n` for splitting in `clickhouse-copier`. To use this feature, pass `--experimental-use-sample-offset 1` as a command line argument. [#10414](https://github.com/ClickHouse/ClickHouse/pull/10414) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) * Allow to parse BOM in TSV if the first column cannot contain BOM in its value. This fixes [#10301](https://github.com/ClickHouse/ClickHouse/issues/10301) [#10424](https://github.com/ClickHouse/ClickHouse/pull/10424) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Avro nested fields insert support [#10354](https://github.com/ClickHouse/ClickHouse/pull/10354) ([Andrew Onyshchuk](https://github.com/oandrew)) -* Allowed to alter column in non-modifying data mode when the same type is specified. ... [#10382](https://github.com/ClickHouse/ClickHouse/pull/10382) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Auto distributed_group_by_no_merge on GROUP BY sharding key (if `optimize_skip_unused_shards` is set) ... [#10341](https://github.com/ClickHouse/ClickHouse/pull/10341) ([Azat Khuzhin](https://github.com/azat)) +* Add Avro nested fields insert support [#10354](https://github.com/ClickHouse/ClickHouse/pull/10354) ([Andrew Onyshchuk](https://github.com/oandrew)) +* Allowed to alter column in non-modifying data mode when the same type is specified. [#10382](https://github.com/ClickHouse/ClickHouse/pull/10382) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Auto `distributed_group_by_no_merge` on GROUP BY sharding key (if `optimize_skip_unused_shards` is set) [#10341](https://github.com/ClickHouse/ClickHouse/pull/10341) ([Azat Khuzhin](https://github.com/azat)) * Optimize queries with LIMIT/LIMIT BY/ORDER BY for distributed with GROUP BY sharding_key [#10373](https://github.com/ClickHouse/ClickHouse/pull/10373) ([Azat Khuzhin](https://github.com/azat)) -* Added a setting `max_server_memory_usage` to limit total memory usage of the server. The metric `MemoryTracking` is now calculated without a drift. The setting `max_memory_usage_for_all_queries` is now obsolete and does nothing. This closes [#10293](https://github.com/ClickHouse/ClickHouse/issues/10293). Note: on servers with very low memory amount and swap you may have to add `max_server_memory_usage_to_ram_ratio` in config.xml and set it to a value larger than one. [#10362](https://github.com/ClickHouse/ClickHouse/pull/10362) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Add config option `system_tables_lazy_load` that, if set to false, loads system tables with logs at the server startup. [Alexander Burmak](https://github.com/Alex-Burmak), [Svyatoslav Tkhon Il Pak](https://github.com/DeifyTheGod), [#9642](https://github.com/ClickHouse/ClickHouse/pull/9642) [#10359](https://github.com/ClickHouse/ClickHouse/pull/10359) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Added a setting `max_server_memory_usage` to limit total memory usage of the server. The metric `MemoryTracking` is now calculated without a drift. The setting `max_memory_usage_for_all_queries` is now obsolete and does nothing. This closes [#10293](https://github.com/ClickHouse/ClickHouse/issues/10293). [#10362](https://github.com/ClickHouse/ClickHouse/pull/10362) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Add config option `system_tables_lazy_load`. If it's set to false, then system tables with logs are loaded at the server startup. [Alexander Burmak](https://github.com/Alex-Burmak), [Svyatoslav Tkhon Il Pak](https://github.com/DeifyTheGod), [#9642](https://github.com/ClickHouse/ClickHouse/pull/9642) [#10359](https://github.com/ClickHouse/ClickHouse/pull/10359) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Use background thread pool (background_schedule_pool_size) for distributed sends [#10263](https://github.com/ClickHouse/ClickHouse/pull/10263) ([Azat Khuzhin](https://github.com/azat)) -* Use background thread pool for background buffer flushes (this will allow to avoid occupation threads when there are no insert\'s) ... [#10315](https://github.com/ClickHouse/ClickHouse/pull/10315) ([Azat Khuzhin](https://github.com/azat)) +* Use background thread pool for background buffer flushes. [#10315](https://github.com/ClickHouse/ClickHouse/pull/10315) ([Azat Khuzhin](https://github.com/azat)) * Support ReplicatedMergeTree over S3 [#10126](https://github.com/ClickHouse/ClickHouse/pull/10126) ([Pavel Kovalenko](https://github.com/Jokser)) * Support for one special case of removing incompletely written parts. This fixes [#9940](https://github.com/ClickHouse/ClickHouse/issues/9940). [#10221](https://github.com/ClickHouse/ClickHouse/pull/10221) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Use isInjective() over manual list of such functions for GROUP BY optimization ... [#10342](https://github.com/ClickHouse/ClickHouse/pull/10342) ([Azat Khuzhin](https://github.com/azat)) -* Improve MsgPack input format by addition msgpack::visitor concept instead of msgpack::unpack concept. [#10325](https://github.com/ClickHouse/ClickHouse/pull/10325) ([Kruglov Pavel](https://github.com/Avogar)) +* Use isInjective() over manual list of such functions for GROUP BY optimization. [#10342](https://github.com/ClickHouse/ClickHouse/pull/10342) ([Azat Khuzhin](https://github.com/azat)) +* Improve MsgPack input format by adding msgpack::visitor concept instead of msgpack::unpack concept. [#10325](https://github.com/ClickHouse/ClickHouse/pull/10325) ([Kruglov Pavel](https://github.com/Avogar)) * Avoid printing error message in log if client sends RST packet immediately on connect. It is typical behaviour of IPVS balancer with keepalived and VRRP. This fixes [#1851](https://github.com/ClickHouse/ClickHouse/issues/1851) [#10274](https://github.com/ClickHouse/ClickHouse/pull/10274) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Allow to parse `+inf` for floating point types. This closes [#1839](https://github.com/ClickHouse/ClickHouse/issues/1839) [#10272](https://github.com/ClickHouse/ClickHouse/pull/10272) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Followup PR for https://github.com/yandex/ClickHouse/pull/5045. ... [#6256](https://github.com/ClickHouse/ClickHouse/pull/6256) ([Guillaume Tassery](https://github.com/YiuRULE)) -* Followup PR for [#5045](https://github.com/ClickHouse/ClickHouse/issues/5045). ... [#10269](https://github.com/ClickHouse/ClickHouse/pull/10269) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Implemented `generateRandom` table function for Nested types. This closes [#9903](https://github.com/ClickHouse/ClickHouse/issues/9903) [#10219](https://github.com/ClickHouse/ClickHouse/pull/10219) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Provide `max_allowed_packed` in MySQL compatibility interface that will help some clients to communicate with ClickHouse via MySQL protocol. ... [#10199](https://github.com/ClickHouse/ClickHouse/pull/10199) ([BohuTANG](https://github.com/BohuTANG)) -* Allow literals for GLOBAL IN (i.e. `SELECT * FROM remote(\'localhost\', system.one) WHERE dummy global in (0)`) [#10196](https://github.com/ClickHouse/ClickHouse/pull/10196) ([Azat Khuzhin](https://github.com/azat)) +* Provide `max_allowed_packed` in MySQL compatibility interface that will help some clients to communicate with ClickHouse via MySQL protocol. [#10199](https://github.com/ClickHouse/ClickHouse/pull/10199) ([BohuTANG](https://github.com/BohuTANG)) +* Allow literals for GLOBAL IN (i.e. `SELECT * FROM remote('localhost', system.one) WHERE dummy global in (0)`) [#10196](https://github.com/ClickHouse/ClickHouse/pull/10196) ([Azat Khuzhin](https://github.com/azat)) * Fix various small issues in interactive mode of clickhouse-client [#10194](https://github.com/ClickHouse/ClickHouse/pull/10194) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Avoid superfluous dictionaries load (system.tables, DROP/SHOW CREATE TABLE) [#10164](https://github.com/ClickHouse/ClickHouse/pull/10164) ([Azat Khuzhin](https://github.com/azat)) * Update to RWLock: timeout parameter for getLock() + implementation reworked to be phase fair [#10073](https://github.com/ClickHouse/ClickHouse/pull/10073) ([Alexander Kazakov](https://github.com/Akazz)) * Enhanced compatibility with native mysql-connector-java(JDBC) [#10021](https://github.com/ClickHouse/ClickHouse/pull/10021) ([BohuTANG](https://github.com/BohuTANG)) -* The function `toString` is considered monotonic and can be used for index analysis even when applied in tautological cases with String or LowCardinality(String) argument. ... [#10110](https://github.com/ClickHouse/ClickHouse/pull/10110) ([Amos Bird](https://github.com/amosbird)) -* Add `ON CLUSTER` clause to commands `{CREATE|DROP} USER/ROLE/ROW POLICY/SETTINGS PROFILE/QUOTA`, `GRANT`. ... [#9811](https://github.com/ClickHouse/ClickHouse/pull/9811) ([Vitaly Baranov](https://github.com/vitlibar)) +* The function `toString` is considered monotonic and can be used for index analysis even when applied in tautological cases with String or LowCardinality(String) argument. [#10110](https://github.com/ClickHouse/ClickHouse/pull/10110) ([Amos Bird](https://github.com/amosbird)) +* Add `ON CLUSTER` clause support to commands `{CREATE|DROP} USER/ROLE/ROW POLICY/SETTINGS PROFILE/QUOTA`, `GRANT`. [#9811](https://github.com/ClickHouse/ClickHouse/pull/9811) ([Vitaly Baranov](https://github.com/vitlibar)) * Virtual hosted-style support for S3 URI [#9998](https://github.com/ClickHouse/ClickHouse/pull/9998) ([Pavel Kovalenko](https://github.com/Jokser)) * Now layout type for dictionaries with no arguments can be specified without round brackets in dictionaries DDL-queries. Fixes [#10057](https://github.com/ClickHouse/ClickHouse/issues/10057). [#10064](https://github.com/ClickHouse/ClickHouse/pull/10064) ([alesapin](https://github.com/alesapin)) * Add ability to use number ranges with leading zeros in filepath [#9989](https://github.com/ClickHouse/ClickHouse/pull/9989) ([Olga Khvostikova](https://github.com/stavrolia)) * Better memory usage in CROSS JOIN. [#10029](https://github.com/ClickHouse/ClickHouse/pull/10029) ([Artem Zuikov](https://github.com/4ertus2)) -* Use DFA algorithm for sequenceMatch without `(?t)`. [#4004](https://github.com/ClickHouse/ClickHouse/pull/4004) ([L\xc3\xa9o Ercolanelli](https://github.com/LeoErcolanelli)) -* The `clickhouse-format` tool is now able to format multiple queries when the `-n` argument is used. [#10852](https://github.com/ClickHouse/ClickHouse/pull/10852) ([Dar\xc3\xado](https://github.com/dgrr)) -* Try to all shards in cluster when getting structure of remote table and skip_unavailable_shards is set. ... [#7278](https://github.com/ClickHouse/ClickHouse/pull/7278) ([nvartolomei](https://github.com/nvartolomei)) +* Use DFA algorithm for sequenceMatch without `(?t)`. [#4004](https://github.com/ClickHouse/ClickHouse/pull/4004) ([Léo Ercolanelli](https://github.com/LeoErcolanelli)) +* The `clickhouse-format` tool is now able to format multiple queries when the `-n` argument is used. [#10852](https://github.com/ClickHouse/ClickHouse/pull/10852) ([Darío](https://github.com/dgrr)) +* Try to connect to all shards in cluster when getting structure of remote table and skip_unavailable_shards is set. [#7278](https://github.com/ClickHouse/ClickHouse/pull/7278) ([nvartolomei](https://github.com/nvartolomei)) * Add `total_rows`/`total_bytes` into the `system.tables` table. [#9919](https://github.com/ClickHouse/ClickHouse/pull/9919) ([Azat Khuzhin](https://github.com/azat)) * System log tables now use polymorpic parts by default. [#9905](https://github.com/ClickHouse/ClickHouse/pull/9905) ([Anton Popov](https://github.com/CurtizJ)) * Add type column into system.settings/merge_tree_settings [#9909](https://github.com/ClickHouse/ClickHouse/pull/9909) ([Azat Khuzhin](https://github.com/azat)) * Check for available CPU instructions at server startup as early as possible. [#9888](https://github.com/ClickHouse/ClickHouse/pull/9888) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Remove order by stage from mutations because we read from a single ordered part in a single thread. Also add check that the order of rows in mutation is ordered in sorting key order and this order is not violated. [#9886](https://github.com/ClickHouse/ClickHouse/pull/9886) ([alesapin](https://github.com/alesapin)) +* Remove `ORDER BY` stage from mutations because we read from a single ordered part in a single thread. Also add check that the rows in mutation are ordered by sorting key and this order is not violated. [#9886](https://github.com/ClickHouse/ClickHouse/pull/9886) ([alesapin](https://github.com/alesapin)) * Implement operator LIKE for FixedString at left hand side. This is needed to better support TPC-DS queries. [#9890](https://github.com/ClickHouse/ClickHouse/pull/9890) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Add force_optimize_skip_unused_shards_no_nested that will disable force_optimize_skip_unused_shards for nested Distributed table [#9812](https://github.com/ClickHouse/ClickHouse/pull/9812) ([Azat Khuzhin](https://github.com/azat)) -* Now columns size calculated only once for MergeTree data parts. [#9827](https://github.com/ClickHouse/ClickHouse/pull/9827) ([alesapin](https://github.com/alesapin)) -* Evaluate constant expressions for optimize_skip_unused_shards (i.e. `SELECT * FROM foo_dist WHERE key=xxHash32(0)`) [#8846](https://github.com/ClickHouse/ClickHouse/pull/8846) ([Azat Khuzhin](https://github.com/azat)) +* Add `force_optimize_skip_unused_shards_no_nested` that will disable `force_optimize_skip_unused_shards` for nested Distributed table [#9812](https://github.com/ClickHouse/ClickHouse/pull/9812) ([Azat Khuzhin](https://github.com/azat)) +* Now columns size is calculated only once for MergeTree data parts. [#9827](https://github.com/ClickHouse/ClickHouse/pull/9827) ([alesapin](https://github.com/alesapin)) +* Evaluate constant expressions for `optimize_skip_unused_shards` (i.e. `SELECT * FROM foo_dist WHERE key=xxHash32(0)`) [#8846](https://github.com/ClickHouse/ClickHouse/pull/8846) ([Azat Khuzhin](https://github.com/azat)) * Check for using `Date` or `DateTime` column from TTL expressions was removed. [#9967](https://github.com/ClickHouse/ClickHouse/pull/9967) ([Vladimir Chebotarev](https://github.com/excitoon)) * DiskS3 hard links optimal implementation. [#9760](https://github.com/ClickHouse/ClickHouse/pull/9760) ([Pavel Kovalenko](https://github.com/Jokser)) -* If `set multiple_joins_rewriter_version = 2` enables second version of multiple JOIN rewrites that keeps not clashed column names as is. It supports multiple JOINs with `USING` and allow `select *` for JOINs with subqueries. ... [#9739](https://github.com/ClickHouse/ClickHouse/pull/9739) ([Artem Zuikov](https://github.com/4ertus2)) +* If `set multiple_joins_rewriter_version = 2` enables second version of multiple JOIN rewrites that keeps not clashed column names as is. It supports multiple JOINs with `USING` and allow `select *` for JOINs with subqueries. [#9739](https://github.com/ClickHouse/ClickHouse/pull/9739) ([Artem Zuikov](https://github.com/4ertus2)) * Implementation of "non-blocking" alter for StorageMergeTree [#9606](https://github.com/ClickHouse/ClickHouse/pull/9606) ([alesapin](https://github.com/alesapin)) -* MergeTree full support for DiskS3 [#9646](https://github.com/ClickHouse/ClickHouse/pull/9646) ([Pavel Kovalenko](https://github.com/Jokser)) +* Add MergeTree full support for DiskS3 [#9646](https://github.com/ClickHouse/ClickHouse/pull/9646) ([Pavel Kovalenko](https://github.com/Jokser)) * Extend `splitByString` to support empty strings as separators. [#9742](https://github.com/ClickHouse/ClickHouse/pull/9742) ([hcz](https://github.com/hczhcz)) * Add a `timestamp_ns` column to `system.trace_log`. It contains a high-definition timestamp of the trace event, and allows to build timelines of thread profiles ("flame charts"). [#9696](https://github.com/ClickHouse/ClickHouse/pull/9696) ([Alexander Kuzmenkov](https://github.com/akuzm)) * When the setting `send_logs_level` is enabled, avoid intermixing of log messages and query progress. [#9634](https://github.com/ClickHouse/ClickHouse/pull/9634) ([Azat Khuzhin](https://github.com/azat)) -* Added MATERIALIZE TTL IN PARTITION. [#9581](https://github.com/ClickHouse/ClickHouse/pull/9581) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Added support of `MATERIALIZE TTL IN PARTITION`. [#9581](https://github.com/ClickHouse/ClickHouse/pull/9581) ([Vladimir Chebotarev](https://github.com/excitoon)) #### Performance Improvement * Better insert logic for right table for Partial MergeJoin. [#10467](https://github.com/ClickHouse/ClickHouse/pull/10467) ([Artem Zuikov](https://github.com/4ertus2)) * Improved performance of row-oriented formats (more than 10% for CSV and more than 35% for Avro in case of narrow tables). [#10503](https://github.com/ClickHouse/ClickHouse/pull/10503) ([Andrew Onyshchuk](https://github.com/oandrew)) -* Improved performance of queries with explicitly defined sets at right side of IN operator and tuples in the left side. [#10385](https://github.com/ClickHouse/ClickHouse/pull/10385) ([Anton Popov](https://github.com/CurtizJ)) +* Improved performance of queries with explicitly defined sets at right side of IN operator and tuples on the left side. [#10385](https://github.com/ClickHouse/ClickHouse/pull/10385) ([Anton Popov](https://github.com/CurtizJ)) * Use less memory for hash table in HashJoin. [#10416](https://github.com/ClickHouse/ClickHouse/pull/10416) ([Artem Zuikov](https://github.com/4ertus2)) -* Special HashJoin over StorageDictionary. Allow rewrite `dictGet()` functions with JOINs. It\'s not backward incompatible itself but could uncover [#8400](https://github.com/ClickHouse/ClickHouse/issues/8400) on some installations. [#10133](https://github.com/ClickHouse/ClickHouse/pull/10133) ([Artem Zuikov](https://github.com/4ertus2)) +* Special HashJoin over StorageDictionary. Allow rewrite `dictGet()` functions with JOINs. It's not backward incompatible itself but could uncover [#8400](https://github.com/ClickHouse/ClickHouse/issues/8400) on some installations. [#10133](https://github.com/ClickHouse/ClickHouse/pull/10133) ([Artem Zuikov](https://github.com/4ertus2)) * Enable parallel insert of materialized view when its target table supports. [#10052](https://github.com/ClickHouse/ClickHouse/pull/10052) ([vxider](https://github.com/Vxider)) -* Improved performance of index analisys with monotonic functions. [#9607](https://github.com/ClickHouse/ClickHouse/pull/9607) ([Anton Popov](https://github.com/CurtizJ)) -* Improve performance of index analisys with monotonic functions. [#10026](https://github.com/ClickHouse/ClickHouse/pull/10026) ([Anton Popov](https://github.com/CurtizJ)) -* Using SSE2 or SSE4.2 SIMD intrinsics to speed up tokenization in bloom filters. ... [#9968](https://github.com/ClickHouse/ClickHouse/pull/9968) ([Vasily Nemkov](https://github.com/Enmk)) -* Optimize Volnitsky searcher by inlining, should be about 5-10% search improvement for queries with many needles or many similar bigrams ... [#4862](https://github.com/ClickHouse/ClickHouse/pull/4862) ([Danila Kutenin](https://github.com/danlark1)) +* Improved performance of index analysis with monotonic functions. [#9607](https://github.com/ClickHouse/ClickHouse/pull/9607)[#10026](https://github.com/ClickHouse/ClickHouse/pull/10026) ([Anton Popov](https://github.com/CurtizJ)) +* Using SSE2 or SSE4.2 SIMD intrinsics to speed up tokenization in bloom filters. [#9968](https://github.com/ClickHouse/ClickHouse/pull/9968) ([Vasily Nemkov](https://github.com/Enmk)) +* Optimize Volnitsky searcher by inlining, should be about 5-10% search improvement for queries with many needles or many similar bigrams [#4862](https://github.com/ClickHouse/ClickHouse/pull/4862) ([Danila Kutenin](https://github.com/danlark1)) * Optimize count() [#6028](https://github.com/ClickHouse/ClickHouse/pull/6028) ([Amos Bird](https://github.com/amosbird)) -* Now prewhere should be at least as efficient as where. As a result, any viable predicates can be moved into prewhere without fancy heuristic ... [#7769](https://github.com/ClickHouse/ClickHouse/pull/7769) ([Amos Bird](https://github.com/amosbird)) -* Enable SSO for IN operator. Improvements varies from 4% - **257%** . Should be merged after https://github.com/ClickHouse/ClickHouse/pull/7662 ... [#7782](https://github.com/ClickHouse/ClickHouse/pull/7782) ([Amos Bird](https://github.com/amosbird)) +* Now `PREWHERE` should be at least as efficient as where. As a result, any viable predicates can be moved into `PREWHERE` expression without fancy heuristic [#7769](https://github.com/ClickHouse/ClickHouse/pull/7769) ([Amos Bird](https://github.com/amosbird)) +* Enable SSO for IN operator. Improvements varies from 4% to **257%**. [#7782](https://github.com/ClickHouse/ClickHouse/pull/7782) ([Amos Bird](https://github.com/amosbird)) * Improved performance of queries with explicitly defined sets at right side of `IN` operator. This fixes performance regression in version 20.3. [#9740](https://github.com/ClickHouse/ClickHouse/pull/9740) ([Anton Popov](https://github.com/CurtizJ)) * Now clickhouse-copier splits each partition in number of pieces and copies them independently. [#9075](https://github.com/ClickHouse/ClickHouse/pull/9075) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -* Adding more aggregation methods. For example TPCH query 1 will now pick `FixedHashMap` and gets 25% performance gain [#9829](https://github.com/ClickHouse/ClickHouse/pull/9829) ([Amos Bird](https://github.com/amosbird)) +* Adding more aggregation methods. For example TPC-H query 1 will now pick `FixedHashMap` and gets 25% performance gain [#9829](https://github.com/ClickHouse/ClickHouse/pull/9829) ([Amos Bird](https://github.com/amosbird)) * Use single row counter for multiple streams in pre-limit transform. This helps to avoid uniting pipeline streams in queries with `limit` but without `order by` (like `select f(x) from (select x from t limit 1000000000)`) and use multiple threads for further processing. [#9602](https://github.com/ClickHouse/ClickHouse/pull/9602) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) #### Build/Testing/Packaging Improvement -* Adding integration tests for new ALTER RENAME COLUMN query. [#10654](https://github.com/ClickHouse/ClickHouse/pull/10654) ([vzakaznikov](https://github.com/vzakaznikov)) -* Fixed possible signed integer overflow in invocation of function `now64` with wrong arguments. This fixes [#8973](https://github.com/ClickHouse/ClickHouse/issues/8973) [#10511](https://github.com/ClickHouse/ClickHouse/pull/10511) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Add integration tests for new ALTER RENAME COLUMN query. [#10654](https://github.com/ClickHouse/ClickHouse/pull/10654) ([vzakaznikov](https://github.com/vzakaznikov)) +* Fix possible signed integer overflow in invocation of function `now64` with wrong arguments. This fixes [#8973](https://github.com/ClickHouse/ClickHouse/issues/8973) [#10511](https://github.com/ClickHouse/ClickHouse/pull/10511) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Split fuzzer and sanitizer configurations to make build config compatible with Oss-fuzz. [#10494](https://github.com/ClickHouse/ClickHouse/pull/10494) ([kyprizel](https://github.com/kyprizel)) * Fixes for clang-tidy on clang-10. [#10420](https://github.com/ClickHouse/ClickHouse/pull/10420) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Display absolute paths in error messages. Otherwise KDevelop fails to navigate to correct file and opens a new file instead. [#10434](https://github.com/ClickHouse/ClickHouse/pull/10434) ([alexey-milovidov](https://github.com/alexey-milovidov)) @@ -252,16 +241,15 @@ * Enable ThinLTO for clang builds (experimental). [#10435](https://github.com/ClickHouse/ClickHouse/pull/10435) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Remove accidential dependency on Z3 that may be introduced if the system has Z3 solver installed. [#10426](https://github.com/ClickHouse/ClickHouse/pull/10426) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Move integration tests docker files to docker/ directory. [#10335](https://github.com/ClickHouse/ClickHouse/pull/10335) ([Ilya Yatsishin](https://github.com/qoega)) -* Allow to use `clang-10` in CI. This is to ensure that [#10238](https://github.com/ClickHouse/ClickHouse/issues/10238) is fixed. [#10384](https://github.com/ClickHouse/ClickHouse/pull/10384) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Allow to use `clang-10` in CI. It ensures that [#10238](https://github.com/ClickHouse/ClickHouse/issues/10238) is fixed. [#10384](https://github.com/ClickHouse/ClickHouse/pull/10384) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Update OpenSSL to upstream master. Fixed the issue when TLS connections may fail with the message `OpenSSL SSL_read: error:14094438:SSL routines:ssl3_read_bytes:tlsv1 alert internal error` and `SSL Exception: error:2400006E:random number generator::error retrieving entropy`. The issue was present in version 20.1. [#8956](https://github.com/ClickHouse/ClickHouse/pull/8956) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Fix clang-10 build. https://github.com/ClickHouse/ClickHouse/issues/10238 [#10370](https://github.com/ClickHouse/ClickHouse/pull/10370) ([Amos Bird](https://github.com/amosbird)) -* Fixing 00964_live_view_watch_events_heartbeat.py test. [#10356](https://github.com/ClickHouse/ClickHouse/pull/10356) ([vzakaznikov](https://github.com/vzakaznikov)) -* This PR adds the performance test for [Parallel INSERT for materialized view](https://github.com/ClickHouse/ClickHouse/pull/10052). [#10345](https://github.com/ClickHouse/ClickHouse/pull/10345) ([vxider](https://github.com/Vxider)) +* Add performance test for [Parallel INSERT for materialized view](https://github.com/ClickHouse/ClickHouse/pull/10052). [#10345](https://github.com/ClickHouse/ClickHouse/pull/10345) ([vxider](https://github.com/Vxider)) * Fix flaky test `test_settings_constraints_distributed.test_insert_clamps_settings`. [#10346](https://github.com/ClickHouse/ClickHouse/pull/10346) ([Vitaly Baranov](https://github.com/vitlibar)) * Add util to test results upload in CI ClickHouse [#10330](https://github.com/ClickHouse/ClickHouse/pull/10330) ([Ilya Yatsishin](https://github.com/qoega)) * Convert test results to JSONEachRow format in junit_to_html tool [#10323](https://github.com/ClickHouse/ClickHouse/pull/10323) ([Ilya Yatsishin](https://github.com/qoega)) -* Update cctz just in case. [#10215](https://github.com/ClickHouse/ClickHouse/pull/10215) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Allow to create HTML report from purest JUnit XML report. [#10247](https://github.com/ClickHouse/ClickHouse/pull/10247) ([Ilya Yatsishin](https://github.com/qoega)) +* Update cctz. [#10215](https://github.com/ClickHouse/ClickHouse/pull/10215) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Allow to create HTML report from the purest JUnit XML report. [#10247](https://github.com/ClickHouse/ClickHouse/pull/10247) ([Ilya Yatsishin](https://github.com/qoega)) * Update the check for minimal compiler version. Fix the root cause of the issue [#10250](https://github.com/ClickHouse/ClickHouse/issues/10250) [#10256](https://github.com/ClickHouse/ClickHouse/pull/10256) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Initial support for live view tables over distributed [#10179](https://github.com/ClickHouse/ClickHouse/pull/10179) ([vzakaznikov](https://github.com/vzakaznikov)) * Fix (false) MSan report in MergeTreeIndexFullText. The issue first appeared in [#9968](https://github.com/ClickHouse/ClickHouse/issues/9968). [#10801](https://github.com/ClickHouse/ClickHouse/pull/10801) ([alexey-milovidov](https://github.com/alexey-milovidov)) @@ -270,29 +258,25 @@ * Update libdivide to v3.0 [#10169](https://github.com/ClickHouse/ClickHouse/pull/10169) ([Ivan](https://github.com/abyss7)) * Add check with enabled polymorphic parts. [#10086](https://github.com/ClickHouse/ClickHouse/pull/10086) ([Anton Popov](https://github.com/CurtizJ)) * Add cross-compile build for FreeBSD. This fixes [#9465](https://github.com/ClickHouse/ClickHouse/issues/9465) [#9643](https://github.com/ClickHouse/ClickHouse/pull/9643) ([Ivan](https://github.com/abyss7)) -* performance test for [#6924](https://github.com/ClickHouse/ClickHouse/issues/6924) [#6980](https://github.com/ClickHouse/ClickHouse/pull/6980) ([filimonov](https://github.com/filimonov)) -* support `StorageFile(, null) ` to deser block into given format file without actually write to disk. It\'s used to perf test format parsers. ... [#8455](https://github.com/ClickHouse/ClickHouse/pull/8455) ([Amos Bird](https://github.com/amosbird)) +* Add performance test for [#6924](https://github.com/ClickHouse/ClickHouse/issues/6924) [#6980](https://github.com/ClickHouse/ClickHouse/pull/6980) ([filimonov](https://github.com/filimonov)) +* Add support of `/dev/null` in the `File` engine for better performance testing [#8455](https://github.com/ClickHouse/ClickHouse/pull/8455) ([Amos Bird](https://github.com/amosbird)) * Move all folders inside /dbms one level up [#9974](https://github.com/ClickHouse/ClickHouse/pull/9974) ([Ivan](https://github.com/abyss7)) -* Added a test that checks that read from MergeTree with single thread is performed in order. Addition to [#9670](https://github.com/ClickHouse/ClickHouse/issues/9670) [#9762](https://github.com/ClickHouse/ClickHouse/pull/9762) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Add a test that checks that read from MergeTree with single thread is performed in order. Addition to [#9670](https://github.com/ClickHouse/ClickHouse/issues/9670) [#9762](https://github.com/ClickHouse/ClickHouse/pull/9762) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Fix the `00964_live_view_watch_events_heartbeat.py` test to avoid race condition. [#9944](https://github.com/ClickHouse/ClickHouse/pull/9944) ([vzakaznikov](https://github.com/vzakaznikov)) -* Fix integration test `test_settings_constraints`. ... [#9962](https://github.com/ClickHouse/ClickHouse/pull/9962) ([Vitaly Baranov](https://github.com/vitlibar)) +* Fix integration test `test_settings_constraints` [#9962](https://github.com/ClickHouse/ClickHouse/pull/9962) ([Vitaly Baranov](https://github.com/vitlibar)) * Every function in its own file, part 12. [#9922](https://github.com/ClickHouse/ClickHouse/pull/9922) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Added performance test for the case of extremely slow analysis of array of tuples. [#9872](https://github.com/ClickHouse/ClickHouse/pull/9872) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Update zstd to 1.4.4. It has some minor improvements in performance and compression ratio. If you run replicas with different versions of ClickHouse you may see reasonable error messages `Data after merge is not byte-identical to data on another replicas.` with explanation. These messages are Ok and you should not worry. ... [#10663](https://github.com/ClickHouse/ClickHouse/pull/10663) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Update zstd to 1.4.4. It has some minor improvements in performance and compression ratio. If you run replicas with different versions of ClickHouse you may see reasonable error messages `Data after merge is not byte-identical to data on another replicas.` with explanation. These messages are Ok and you should not worry. [#10663](https://github.com/ClickHouse/ClickHouse/pull/10663) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Fix TSan report in `system.stack_trace`. [#9832](https://github.com/ClickHouse/ClickHouse/pull/9832) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Removed dependency on `clock_getres`. [#9833](https://github.com/ClickHouse/ClickHouse/pull/9833) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Preparation to fix https://clickhouse-test-reports.s3.yandex.net/9813/622bca235848bf1a7950d6eeb10241204aa069eb/stress_test_(thread)/stderr.log [#9825](https://github.com/ClickHouse/ClickHouse/pull/9825) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Added identifier names check with clang-tidy. [#9799](https://github.com/ClickHouse/ClickHouse/pull/9799) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Update "builder" docker image. This image is not used in CI but is useful for developers. [#9809](https://github.com/ClickHouse/ClickHouse/pull/9809) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Remove old `performance-test` tool that is no longer used in CI. `clickhouse-performance-test` is great but now we are using way superior tool that is doing comparison testing with sophisticated statistical formulas to achieve confident results regardless to various changes in environment. [#9796](https://github.com/ClickHouse/ClickHouse/pull/9796) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Added most of clang-static-analyzer checks. [#9765](https://github.com/ClickHouse/ClickHouse/pull/9765) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Update Poco to 1.9.3 in preparation for MongoDB URI support. [#6892](https://github.com/ClickHouse/ClickHouse/pull/6892) ([Alexander Kuzmenkov](https://github.com/akuzm)) -* Added even more clang-tidy checks. [#9719](https://github.com/ClickHouse/ClickHouse/pull/9719) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Added more clang-tidy checks. [#9718](https://github.com/ClickHouse/ClickHouse/pull/9718) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Added more clang-tidy checks. [#9717](https://github.com/ClickHouse/ClickHouse/pull/9717) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Fix build with `-DUSE_STATIC_LIBRARIES=0 -DENABLE_JEMALLOC=0` [#9651](https://github.com/ClickHouse/ClickHouse/pull/9651) ([Artem Zuikov](https://github.com/4ertus2)) -* For change log script, if merge commit was cherry-picked to release branch, take pr name from commit description. [#9708](https://github.com/ClickHouse/ClickHouse/pull/9708) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Support `vX.X-conflicts` tag in backport script. ... [#9705](https://github.com/ClickHouse/ClickHouse/pull/9705) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* For change log script, if merge commit was cherry-picked to release branch, take PR name from commit description. [#9708](https://github.com/ClickHouse/ClickHouse/pull/9708) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Support `vX.X-conflicts` tag in backport script. [#9705](https://github.com/ClickHouse/ClickHouse/pull/9705) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) * Fix `auto-label` for backporting script. [#9685](https://github.com/ClickHouse/ClickHouse/pull/9685) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) * Use libc++ in Darwin cross-build to make it consistent with native build. [#9665](https://github.com/ClickHouse/ClickHouse/pull/9665) ([Hui Wang](https://github.com/huiwang)) * Fix flacky test `01017_uniqCombined_memory_usage`. Continuation of [#7236](https://github.com/ClickHouse/ClickHouse/issues/7236). [#9667](https://github.com/ClickHouse/ClickHouse/pull/9667) ([alexey-milovidov](https://github.com/alexey-milovidov)) @@ -366,7 +350,6 @@ * Fix bug in `CHECK TABLE` query when table contain skip indices. [#10068](https://github.com/ClickHouse/ClickHouse/pull/10068) ([alesapin](https://github.com/alesapin)). * Fix error `Cannot clone block with columns because block has 0 columns ... While executing GroupingAggregatedTransform`. It happened when setting `distributed_aggregation_memory_efficient` was enabled, and distributed query read aggregating data with different level from different shards (mixed single and two level aggregation). [#10063](https://github.com/ClickHouse/ClickHouse/pull/10063) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Fix a segmentation fault that could occur in GROUP BY over string keys containing trailing zero bytes ([#8636](https://github.com/ClickHouse/ClickHouse/issues/8636), [#8925](https://github.com/ClickHouse/ClickHouse/issues/8925)). [#10025](https://github.com/ClickHouse/ClickHouse/pull/10025) ([Alexander Kuzmenkov](https://github.com/akuzm)). -* Fix parallel distributed INSERT SELECT for remote table. This PR fixes the solution provided in [#9759](https://github.com/ClickHouse/ClickHouse/pull/9759). [#9999](https://github.com/ClickHouse/ClickHouse/pull/9999) ([Vitaly Baranov](https://github.com/vitlibar)). * Fix the number of threads used for remote query execution (performance regression, since 20.3). This happened when query from `Distributed` table was executed simultaneously on local and remote shards. Fixes [#9965](https://github.com/ClickHouse/ClickHouse/issues/9965). [#9971](https://github.com/ClickHouse/ClickHouse/pull/9971) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Fix bug in which the necessary tables weren't retrieved at one of the processing stages of queries to some databases. Fixes [#9699](https://github.com/ClickHouse/ClickHouse/issues/9699). [#9949](https://github.com/ClickHouse/ClickHouse/pull/9949) ([achulkov2](https://github.com/achulkov2)). * Fix 'Not found column in block' error when `JOIN` appears with `TOTALS`. Fixes [#9839](https://github.com/ClickHouse/ClickHouse/issues/9839). [#9939](https://github.com/ClickHouse/ClickHouse/pull/9939) ([Artem Zuikov](https://github.com/4ertus2)). From aab2599fad9c518e371b321e50aa091c8ecc1f62 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Sun, 17 May 2020 23:26:53 +0300 Subject: [PATCH 332/738] Fix build. --- src/Columns/IColumn.h | 2 +- src/Common/COW.h | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Columns/IColumn.h b/src/Columns/IColumn.h index 2ef2b71b46e..496a0a5759b 100644 --- a/src/Columns/IColumn.h +++ b/src/Columns/IColumn.h @@ -308,7 +308,7 @@ public: { MutablePtr res = ptr->shallowMutate(); /// Now use_count is 2. ptr.reset(); /// Reset use_count to 1. - res->forEachSubcolumn([](WrappedPtr & subcolumn) { subcolumn = IColumn::mutate(std::move(subcolumn)); }); + res->forEachSubcolumn([](WrappedPtr & subcolumn) { subcolumn = IColumn::mutate(std::move(subcolumn).detach()); }); return res; } diff --git a/src/Common/COW.h b/src/Common/COW.h index 7e4bf3e8056..2bc8e8aba00 100644 --- a/src/Common/COW.h +++ b/src/Common/COW.h @@ -216,7 +216,8 @@ protected: operator const immutable_ptr & () const { return value; } operator immutable_ptr & () { return value; } - operator immutable_ptr () && { return std::move(value); } + + immutable_ptr detach() && { return std::move(value); } operator bool() const { return value != nullptr; } bool operator! () const { return value == nullptr; } From 770b720620591cb7c30819a6df9ec1956f9a8bdf Mon Sep 17 00:00:00 2001 From: Andrei Nekrashevich Date: Mon, 18 May 2020 02:58:19 +0300 Subject: [PATCH 333/738] no reinterpret_cast --- src/Functions/randomStringUTF8.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Functions/randomStringUTF8.cpp b/src/Functions/randomStringUTF8.cpp index 8635cd92c9b..fdea2d29952 100644 --- a/src/Functions/randomStringUTF8.cpp +++ b/src/Functions/randomStringUTF8.cpp @@ -120,8 +120,8 @@ public: UInt32 code_point2 = generate_code_point(rand >> 32); /// We have padding in column buffers that we can overwrite. - pos += UTF8::convert(*reinterpret_cast(&code_point1), pos, sizeof(int)); - last_writen_bytes = UTF8::convert(*reinterpret_cast(&code_point2), pos, sizeof(int)); + pos += UTF8::convert(code_point1, pos, sizeof(int)); + last_writen_bytes = UTF8::convert(code_point2, pos, sizeof(int)); pos += last_writen_bytes; } offset = pos - data_to.data() + 1; From ed6c8054f03fd789ce0859beae96d99201eed8e4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 18 May 2020 03:48:46 +0300 Subject: [PATCH 334/738] Fix bad test --- .../0_stateless/00219_full_right_join_column_order.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/00219_full_right_join_column_order.sql b/tests/queries/0_stateless/00219_full_right_join_column_order.sql index 76098261358..78dfe236923 100644 --- a/tests/queries/0_stateless/00219_full_right_join_column_order.sql +++ b/tests/queries/0_stateless/00219_full_right_join_column_order.sql @@ -1,8 +1,8 @@ SET any_join_distinct_right_table_keys = 1; -SELECT a, b FROM (SELECT 1 AS a, 2000 AS b) js1 ANY RIGHT JOIN (SELECT 2 AS a, 3000 AS b) js2 USING a, b; -SELECT a, b FROM (SELECT 1 AS a, 2000 AS b) js1 ANY RIGHT JOIN (SELECT 2 AS a, 3000 AS b) js2 USING b, a; +SELECT a, b FROM (SELECT 1 AS a, 2000 AS b) js1 ANY RIGHT JOIN (SELECT 2 AS a, 3000 AS b) js2 USING a, b ORDER BY a, b; +SELECT a, b FROM (SELECT 1 AS a, 2000 AS b) js1 ANY RIGHT JOIN (SELECT 2 AS a, 3000 AS b) js2 USING b, a ORDER BY a, b; -SELECT a, b FROM (SELECT 1 AS a, 2000 AS b) js1 ANY RIGHT JOIN (SELECT 2 AS a, 3000 AS b UNION ALL SELECT 1 AS a, 2000 AS b) js2 USING a, b; -SELECT a, b FROM (SELECT 1 AS a, 2000 AS b) js1 ANY RIGHT JOIN (SELECT 2 AS a, 3000 AS b UNION ALL SELECT 1 AS a, 2000 AS b) js2 USING b, a; +SELECT a, b FROM (SELECT 1 AS a, 2000 AS b) js1 ANY RIGHT JOIN (SELECT 2 AS a, 3000 AS b UNION ALL SELECT 1 AS a, 2000 AS b) js2 USING a, b ORDER BY a, b; +SELECT a, b FROM (SELECT 1 AS a, 2000 AS b) js1 ANY RIGHT JOIN (SELECT 2 AS a, 3000 AS b UNION ALL SELECT 1 AS a, 2000 AS b) js2 USING b, a ORDER BY a, b; From 7b5dfd9c00be7e3c1bbb193f348b3e2c3662c997 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 18 May 2020 04:06:04 +0300 Subject: [PATCH 335/738] Wait for odbc-bridge with exponential backoff --- src/Common/XDBCBridgeHelper.h | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Common/XDBCBridgeHelper.h b/src/Common/XDBCBridgeHelper.h index 41aeb421394..a3c538cb065 100644 --- a/src/Common/XDBCBridgeHelper.h +++ b/src/Common/XDBCBridgeHelper.h @@ -151,16 +151,22 @@ public: LOG_TRACE(log, BridgeHelperMixin::serviceAlias() + " is not running, will try to start it"); startBridge(); bool started = false; - for (size_t counter : ext::range(1, 20)) + + uint64_t milliseconds_to_wait = 10; /// Exponential backoff + uint64_t counter = 0; + while (milliseconds_to_wait < 10000) { + ++counter; LOG_TRACE(log, "Checking " + BridgeHelperMixin::serviceAlias() + " is running, try " << counter); if (checkBridgeIsRunning()) { started = true; break; } - std::this_thread::sleep_for(std::chrono::milliseconds(10)); + std::this_thread::sleep_for(std::chrono::milliseconds(milliseconds_to_wait)); + milliseconds_to_wait *= 2; } + if (!started) throw Exception(BridgeHelperMixin::getName() + "BridgeHelper: " + BridgeHelperMixin::serviceAlias() + " is not responding", ErrorCodes::EXTERNAL_SERVER_IS_NOT_RESPONDING); From b0a5ce7743b4f0288bdb06a725033d1045a280ca Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 18 May 2020 04:19:50 +0300 Subject: [PATCH 336/738] Enable clang-tidy for programs and utils --- .clang-tidy | 14 +++++++------- programs/CMakeLists.txt | 4 ++++ programs/odbc-bridge/ODBCBlockOutputStream.cpp | 4 ++-- src/Common/tests/cow_columns.cpp | 8 ++++---- src/Common/tests/cow_compositions.cpp | 8 ++++---- utils/CMakeLists.txt | 4 ++++ utils/convert-month-partitioned-parts/main.cpp | 2 +- utils/iotest/iotest.cpp | 8 ++++---- utils/iotest/iotest_aio.cpp | 18 +++++++----------- utils/iotest/iotest_nonblock.cpp | 6 +++--- utils/test-data-generator/CMakeLists.txt | 3 +++ .../main.cpp | 4 ++-- utils/zookeeper-cli/zookeeper-cli.cpp | 8 +++----- 13 files changed, 48 insertions(+), 43 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 3c222fbf8da..b0971418e0e 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -9,7 +9,7 @@ Checks: '-*, misc-unused-alias-decls, misc-unused-parameters, misc-unused-using-decls, - + modernize-avoid-bind, modernize-loop-convert, modernize-make-shared, @@ -33,7 +33,7 @@ Checks: '-*, performance-no-automatic-move, performance-trivially-destructible, performance-unnecessary-copy-initialization, - + readability-avoid-const-params-in-decls, readability-const-return-type, readability-container-size-empty, @@ -58,7 +58,7 @@ Checks: '-*, readability-simplify-boolean-expr, readability-inconsistent-declaration-parameter-name, readability-identifier-naming, - + bugprone-undelegated-constructor, bugprone-argument-comment, bugprone-bad-signal-to-kill-thread, @@ -102,7 +102,7 @@ Checks: '-*, bugprone-unused-return-value, bugprone-use-after-move, bugprone-virtual-near-miss, - + cert-dcl21-cpp, cert-dcl50-cpp, cert-env33-c, @@ -112,7 +112,7 @@ Checks: '-*, cert-mem57-cpp, cert-msc50-cpp, cert-oop58-cpp, - + google-build-explicit-make-pair, google-build-namespaces, google-default-arguments, @@ -121,9 +121,9 @@ Checks: '-*, google-readability-avoid-underscore-in-googletest-name, google-runtime-int, google-runtime-operator, - + hicpp-exception-baseclass, - + clang-analyzer-core.CallAndMessage, clang-analyzer-core.DivideZero, clang-analyzer-core.NonNullParamChecker, diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index 7cbe2e7a2a6..a3d3188653b 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -1,3 +1,7 @@ +if (USE_CLANG_TIDY) + set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}") +endif () + # 'clickhouse' binary is a multi purpose tool, # that contain multiple execution modes (client, server, etc.) # each of them is built and linked as a separate library, defined below. diff --git a/programs/odbc-bridge/ODBCBlockOutputStream.cpp b/programs/odbc-bridge/ODBCBlockOutputStream.cpp index c2597805230..ab24c008e40 100644 --- a/programs/odbc-bridge/ODBCBlockOutputStream.cpp +++ b/programs/odbc-bridge/ODBCBlockOutputStream.cpp @@ -24,8 +24,8 @@ namespace query.table_id.table_name = table_name; query.columns = std::make_shared(','); query.children.push_back(query.columns); - for (size_t i = 0; i < columns.size(); ++i) - query.columns->children.emplace_back(std::make_shared(columns[i].name)); + for (const auto & column : columns) + query.columns->children.emplace_back(std::make_shared(column.name)); std::stringstream ss; IAST::FormatSettings settings(ss, true); diff --git a/src/Common/tests/cow_columns.cpp b/src/Common/tests/cow_columns.cpp index fa84fc9ebc2..404b478f5a0 100644 --- a/src/Common/tests/cow_columns.cpp +++ b/src/Common/tests/cow_columns.cpp @@ -56,8 +56,8 @@ int main(int, char **) MutableColumnPtr mut = IColumn::mutate(std::move(y)); mut->set(2); - std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << ", " << mut->use_count() << "\n"; - std::cerr << "addresses: " << x.get() << ", " << y.get() << ", " << mut.get() << "\n"; + std::cerr << "refcounts: " << x->use_count() << ", " << mut->use_count() << "\n"; + std::cerr << "addresses: " << x.get() << ", " << mut.get() << "\n"; y = std::move(mut); } @@ -75,8 +75,8 @@ int main(int, char **) MutableColumnPtr mut = IColumn::mutate(std::move(y)); mut->set(3); - std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << ", " << mut->use_count() << "\n"; - std::cerr << "addresses: " << x.get() << ", " << y.get() << ", " << mut.get() << "\n"; + std::cerr << "refcounts: " << x->use_count() << ", " << mut->use_count() << "\n"; + std::cerr << "addresses: " << x.get() << ", " << mut.get() << "\n"; y = std::move(mut); } diff --git a/src/Common/tests/cow_compositions.cpp b/src/Common/tests/cow_compositions.cpp index be33f392497..74369e86300 100644 --- a/src/Common/tests/cow_compositions.cpp +++ b/src/Common/tests/cow_compositions.cpp @@ -75,8 +75,8 @@ int main(int, char **) MutableColumnPtr mut = IColumn::mutate(std::move(y)); mut->set(2); - std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << ", " << mut->use_count() << "\n"; - std::cerr << "addresses: " << x.get() << ", " << y.get() << ", " << mut.get() << "\n"; + std::cerr << "refcounts: " << x->use_count() << ", " << mut->use_count() << "\n"; + std::cerr << "addresses: " << x.get() << ", " << mut.get() << "\n"; y = std::move(mut); } @@ -94,8 +94,8 @@ int main(int, char **) MutableColumnPtr mut = IColumn::mutate(std::move(y)); mut->set(3); - std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << ", " << mut->use_count() << "\n"; - std::cerr << "addresses: " << x.get() << ", " << y.get() << ", " << mut.get() << "\n"; + std::cerr << "refcounts: " << x->use_count() << ", " << mut->use_count() << "\n"; + std::cerr << "addresses: " << x.get() << ", " << ", " << mut.get() << "\n"; y = std::move(mut); } diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index c8531bd63a0..94042ea4090 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -1,3 +1,7 @@ +if (USE_CLANG_TIDY) + set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}") +endif () + if(MAKE_STATIC_LIBRARIES) set(MAX_LINKER_MEMORY 3500) else() diff --git a/utils/convert-month-partitioned-parts/main.cpp b/utils/convert-month-partitioned-parts/main.cpp index 51ea87d35b9..af8e221a10b 100644 --- a/utils/convert-month-partitioned-parts/main.cpp +++ b/utils/convert-month-partitioned-parts/main.cpp @@ -30,7 +30,7 @@ void run(String part_path, String date_column, String dest_path) { std::shared_ptr disk = std::make_shared("local", "/", 0); auto old_part_path = Poco::Path::forDirectory(part_path); - String old_part_name = old_part_path.directory(old_part_path.depth() - 1); + const String & old_part_name = old_part_path.directory(old_part_path.depth() - 1); String old_part_path_str = old_part_path.toString(); auto part_info = MergeTreePartInfo::fromPartName(old_part_name, MergeTreeDataFormatVersion(0)); diff --git a/utils/iotest/iotest.cpp b/utils/iotest/iotest.cpp index ed846e3d46f..e578a539bcd 100644 --- a/utils/iotest/iotest.cpp +++ b/utils/iotest/iotest.cpp @@ -59,9 +59,9 @@ void thread(int fd, int mode, size_t min_offset, size_t max_offset, size_t block for (size_t i = 0; i < count; ++i) { - long rand_result1 = rng(); - long rand_result2 = rng(); - long rand_result3 = rng(); + uint64_t rand_result1 = rng(); + uint64_t rand_result2 = rng(); + uint64_t rand_result3 = rng(); size_t rand_result = rand_result1 ^ (rand_result2 << 22) ^ (rand_result3 << 43); size_t offset; @@ -152,7 +152,7 @@ int mainImpl(int argc, char ** argv) Stopwatch watch; for (size_t i = 0; i < threads; ++i) - pool.scheduleOrThrowOnError(std::bind(thread, fd, mode, min_offset, max_offset, block_size, count)); + pool.scheduleOrThrowOnError([=]{ thread(fd, mode, min_offset, max_offset, block_size, count); }); pool.wait(); fsync(fd); diff --git a/utils/iotest/iotest_aio.cpp b/utils/iotest/iotest_aio.cpp index c0945fbe1e1..24508c1dd9f 100644 --- a/utils/iotest/iotest_aio.cpp +++ b/utils/iotest/iotest_aio.cpp @@ -13,6 +13,8 @@ int main(int, char **) { return 0; } #include #include #include +#include +#include #include #include #include @@ -52,10 +54,7 @@ void thread(int fd, int mode, size_t min_offset, size_t max_offset, size_t block for (size_t i = 0; i < buffers_count; ++i) buffers[i] = Memory<>(block_size, sysconf(_SC_PAGESIZE)); - drand48_data rand_data; - timespec times; - clock_gettime(CLOCK_THREAD_CPUTIME_ID, ×); - srand48_r(times.tv_nsec, &rand_data); + pcg64_fast rng(randomSeed()); size_t in_progress = 0; size_t blocks_sent = 0; @@ -82,12 +81,9 @@ void thread(int fd, int mode, size_t min_offset, size_t max_offset, size_t block char * buf = buffers[i].data(); - long rand_result1 = 0; - long rand_result2 = 0; - long rand_result3 = 0; - lrand48_r(&rand_data, &rand_result1); - lrand48_r(&rand_data, &rand_result2); - lrand48_r(&rand_data, &rand_result3); + uint64_t rand_result1 = rng(); + uint64_t rand_result2 = rng(); + uint64_t rand_result3 = rng(); size_t rand_result = rand_result1 ^ (rand_result2 << 22) ^ (rand_result3 << 43); size_t offset = min_offset + rand_result % ((max_offset - min_offset) / block_size) * block_size; @@ -172,7 +168,7 @@ int mainImpl(int argc, char ** argv) Stopwatch watch; for (size_t i = 0; i < threads_count; ++i) - pool.scheduleOrThrowOnError(std::bind(thread, fd, mode, min_offset, max_offset, block_size, buffers_count, count)); + pool.scheduleOrThrowOnError([=]{ thread(fd, mode, min_offset, max_offset, block_size, buffers_count, count); }); pool.wait(); watch.stop(); diff --git a/utils/iotest/iotest_nonblock.cpp b/utils/iotest/iotest_nonblock.cpp index 9317e7ed47f..524d6298da5 100644 --- a/utils/iotest/iotest_nonblock.cpp +++ b/utils/iotest/iotest_nonblock.cpp @@ -113,9 +113,9 @@ int mainImpl(int argc, char ** argv) polls[i].revents = 0; ++ops; - long rand_result1 = rng(); - long rand_result2 = rng(); - long rand_result3 = rng(); + uint64_t rand_result1 = rng(); + uint64_t rand_result2 = rng(); + uint64_t rand_result3 = rng(); size_t rand_result = rand_result1 ^ (rand_result2 << 22) ^ (rand_result3 << 43); size_t offset; diff --git a/utils/test-data-generator/CMakeLists.txt b/utils/test-data-generator/CMakeLists.txt index d8a2111cf07..3a94358e86d 100644 --- a/utils/test-data-generator/CMakeLists.txt +++ b/utils/test-data-generator/CMakeLists.txt @@ -1,3 +1,6 @@ +# Disable clang-tidy for protobuf generated files +set (CMAKE_CXX_CLANG_TIDY "") + add_compile_options(-Wno-zero-as-null-pointer-constant -Wno-array-bounds) # Protobuf generated files if (USE_PROTOBUF) diff --git a/utils/zookeeper-adjust-block-numbers-to-parts/main.cpp b/utils/zookeeper-adjust-block-numbers-to-parts/main.cpp index 91431c01648..a896129f915 100644 --- a/utils/zookeeper-adjust-block-numbers-to-parts/main.cpp +++ b/utils/zookeeper-adjust-block-numbers-to-parts/main.cpp @@ -102,7 +102,7 @@ std::unordered_map getPartitionsNeedAdjustingBlockNumbers( std::cout << "Shard: " << shard << std::endl; std::vector use_tables = tables.empty() ? getAllTables(zk, root, shard) : removeNotExistingTables(zk, root, shard, tables); - for (auto table : use_tables) + for (const auto & table : use_tables) { std::cout << "\tTable: " << table << std::endl; std::string table_path = root + "/" + shard + "/" + table; @@ -121,7 +121,7 @@ std::unordered_map getPartitionsNeedAdjustingBlockNumbers( continue; } - for (auto partition : partitions) + for (const auto & partition : partitions) { try { diff --git a/utils/zookeeper-cli/zookeeper-cli.cpp b/utils/zookeeper-cli/zookeeper-cli.cpp index 40755fc0160..0a503e77250 100644 --- a/utils/zookeeper-cli/zookeeper-cli.cpp +++ b/utils/zookeeper-cli/zookeeper-cli.cpp @@ -97,10 +97,8 @@ int main(int argc, char ** argv) bool watch = w == "w"; zkutil::EventPtr event = watch ? std::make_shared() : nullptr; std::vector v = zk.getChildren(path, nullptr, event); - for (size_t i = 0; i < v.size(); ++i) - { - std::cout << v[i] << std::endl; - } + for (const auto & child : v) + std::cout << child << std::endl; if (watch) waitForWatch(event); } @@ -193,7 +191,7 @@ int main(int argc, char ** argv) zk.set(path, data, version, &stat); printStat(stat); } - else if (cmd != "") + else if (!cmd.empty()) { std::cout << "commands:\n"; std::cout << " q\n"; From c2b6ed5e212d4e01a91f75d36136cbc8016d49b9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 18 May 2020 04:42:46 +0300 Subject: [PATCH 337/738] Whitespaces --- src/Functions/randomStringUTF8.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Functions/randomStringUTF8.cpp b/src/Functions/randomStringUTF8.cpp index fdea2d29952..51f57d63ea9 100644 --- a/src/Functions/randomStringUTF8.cpp +++ b/src/Functions/randomStringUTF8.cpp @@ -20,7 +20,7 @@ namespace ErrorCodes /* Generate string with a UTF-8 encoded text. * Take a single argument - length of result string in Unicode code points. - * ATTENTION: Method generate only assignable code points(excluded 4-13 planes). + * ATTENTION: Method generate only assignable code points (excluded 4-13 planes). * See https://en.wikipedia.org/wiki/Plane_(Unicode) */ class FunctionRandomStringUTF8 : public IFunction @@ -71,7 +71,7 @@ public: /* As we generate only assigned planes, the mathematical expectation of the number of bytes * per generated code point ~= 3.85. So, reserving for coefficient 4 will not be an overhead - */ + */ if (summary_utf8_len > (1 << 29)) throw Exception("Too large string size in function " + getName(), ErrorCodes::TOO_LARGE_STRING_SIZE); @@ -132,7 +132,6 @@ public: offsets_to[row_num] = offset; } - /// Put zero bytes in between. auto * pos = data_to.data(); for (size_t row_num = 0; row_num < input_rows_count; ++row_num) From bf62c67075bcec339610051b73120e353c87f6e8 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 18 May 2020 04:47:48 +0300 Subject: [PATCH 338/738] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3826a33fba..79cbd86a47a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ### ClickHouse release v20.4.2.9, 2020-05-12 #### Backward Incompatible Change +* System tables (e.g. system.query_log, system.trace_log, system.metric_log) are using compact data part format for parts smaller than 10 MiB in size. Compact data part format is supported since version 20.3. If you are going to downgrade to version less than 20.3, you should manually delete table data for system logs in `/var/lib/clickhouse/data/system/`. * When string comparison involves FixedString and compared arguments are of different sizes, do comparison as if smaller string is padded to the length of the larger. This is intented for SQL compatibility if we imagine that FixedString data type corresponds to SQL CHAR. This closes [#9272](https://github.com/ClickHouse/ClickHouse/issues/9272). [#10363](https://github.com/ClickHouse/ClickHouse/pull/10363) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Make SHOW CREATE TABLE multiline. Now it is more readable and more like MySQL. [#10049](https://github.com/ClickHouse/ClickHouse/pull/10049) ([Azat Khuzhin](https://github.com/azat)) * Added a setting `validate_polygons` that is used in `pointInPolygon` function and enabled by default. [#9857](https://github.com/ClickHouse/ClickHouse/pull/9857) ([alexey-milovidov](https://github.com/alexey-milovidov)) From 24dee5b2a2a652f27f2707f68e61aef9e9345900 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 18 May 2020 04:48:59 +0300 Subject: [PATCH 339/738] Update CHANGELOG.md --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79cbd86a47a..0efcf05cb1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,6 @@ #### New Feature * Support custom HTTP handlers. See ISSUES-5436 for description. [#7572](https://github.com/ClickHouse/ClickHouse/pull/7572) ([Winter Zhang](https://github.com/zhang2014)) * Added custom settings support in DDL-queries for CREATE DICTIONARY [#10465](https://github.com/ClickHouse/ClickHouse/pull/10465) ([Artem Streltsov](https://github.com/kekekekule)) -* Added experimental database engine Atomic. It supports non-blocking `DROP` and `RENAME TABLE` queries and atomic `EXCHANGE TABLES t1 AND t2` query [#7512](https://github.com/ClickHouse/ClickHouse/pull/7512) ([tavplubix](https://github.com/tavplubix)) * Add simple server-wide memory profiler that will collect allocation contexts when server memory usage becomes higher than the next allocation threshold. [#10444](https://github.com/ClickHouse/ClickHouse/pull/10444) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Added support for custom settings section in dictionaries. Also fixes issue [#2829](https://github.com/ClickHouse/ClickHouse/issues/2829). [#10137](https://github.com/ClickHouse/ClickHouse/pull/10137) ([Artem Streltsov](https://github.com/kekekekule)) * Add setting `always_fetch_merged_part` which restrict replica to merge parts by itself and always prefer dowloading from other replicas. [#10379](https://github.com/ClickHouse/ClickHouse/pull/10379) ([alesapin](https://github.com/alesapin)) @@ -41,6 +40,9 @@ * Add support for configuring MongoDB connections with URI (for Atlas service). [#5387](https://github.com/ClickHouse/ClickHouse/pull/5387) ([Manish Srivastava](https://github.com/manish53)) * Support `DATE` and `TIMESTAMP` SQL operators, e.g. `SELECT date '2001-01-01'` [#9691](https://github.com/ClickHouse/ClickHouse/pull/9691) ([Artem Zuikov](https://github.com/4ertus2)) +#### Experimental Feature +* Added experimental database engine Atomic. It supports non-blocking `DROP` and `RENAME TABLE` queries and atomic `EXCHANGE TABLES t1 AND t2` query [#7512](https://github.com/ClickHouse/ClickHouse/pull/7512) ([tavplubix](https://github.com/tavplubix)) + #### Bug Fix * Fixed incorrect scalar results inside inner query of `MATERIALIZED VIEW` in case if this query contained dependent table [#10603](https://github.com/ClickHouse/ClickHouse/pull/10603) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) * Fixed bug, which caused HTTP requests to get stuck on client closing connection when `readonly=2` and `cancel_http_readonly_queries_on_client_close=1`. [#10684](https://github.com/ClickHouse/ClickHouse/pull/10684) ([tavplubix](https://github.com/tavplubix)) From 95af8651617b1b30f5e0e3a5ea563443ab2f097b Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 18 May 2020 04:50:27 +0300 Subject: [PATCH 340/738] Update CHANGELOG.md --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0efcf05cb1c..63814ca67ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,6 @@ * Added support for custom settings section in dictionaries. Also fixes issue [#2829](https://github.com/ClickHouse/ClickHouse/issues/2829). [#10137](https://github.com/ClickHouse/ClickHouse/pull/10137) ([Artem Streltsov](https://github.com/kekekekule)) * Add setting `always_fetch_merged_part` which restrict replica to merge parts by itself and always prefer dowloading from other replicas. [#10379](https://github.com/ClickHouse/ClickHouse/pull/10379) ([alesapin](https://github.com/alesapin)) * Add function JSONExtractKeysAndValuesRaw which extracts raw data from JSON objects [#10378](https://github.com/ClickHouse/ClickHouse/pull/10378) ([hcz](https://github.com/hczhcz)) -* Add CRC32IEEE()/CRC64() support [#7480](https://github.com/ClickHouse/ClickHouse/pull/7480) ([Azat Khuzhin](https://github.com/azat)) * Add memory usage from OS to `system.asynchronous_metrics`. [#10361](https://github.com/ClickHouse/ClickHouse/pull/10361) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Added generic variants for functions `least` and `greatest`. Now they work with arbitrary number of arguments of arbitrary types. This fixes [#4767](https://github.com/ClickHouse/ClickHouse/issues/4767) [#10318](https://github.com/ClickHouse/ClickHouse/pull/10318) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Now ClickHouse controls timeouts of dictionary sources on its side. Two new settings added to cache dictionary configuration: `strict_max_lifetime_seconds`, which is `max_lifetime` by default, and `query_wait_timeout_milliseconds`, which is one minute by default. The first settings is also useful with `allow_read_expired_keys` settings (to forbid reading very expired keys). [#10337](https://github.com/ClickHouse/ClickHouse/pull/10337) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) From 3d62eb17c45a2b15539f57dcc8425d0c0890d345 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 18 May 2020 04:51:48 +0300 Subject: [PATCH 341/738] Update CHANGELOG.md --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63814ca67ee..cc5b799d503 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,7 +26,6 @@ * Consider `NULL` to be equal to `NULL` in `IN` operator, if the option `transform_null_in` is set. [#10085](https://github.com/ClickHouse/ClickHouse/pull/10085) ([achimbab](https://github.com/achimbab)) * Add MessagePack Input/Output format. [#9889](https://github.com/ClickHouse/ClickHouse/pull/9889) ([Kruglov Pavel](https://github.com/Avogar)) * Add `ALTER TABLE ... RENAME COLUMN` for MergeTree table engines family. [#9948](https://github.com/ClickHouse/ClickHouse/pull/9948) ([alesapin](https://github.com/alesapin)) -* Implemented s3 table function and storage. [#5596](https://github.com/ClickHouse/ClickHouse/pull/5596) ([Vladimir Chebotarev](https://github.com/excitoon)) * Support parallel distributed INSERT SELECT. [#9759](https://github.com/ClickHouse/ClickHouse/pull/9759) ([vxider](https://github.com/Vxider)) * Added implementation of LIVE VIEW tables that were originally proposed in [#3925](https://github.com/ClickHouse/ClickHouse/issues/3925), and then updated in [#5541](https://github.com/ClickHouse/ClickHouse/issues/5541). Note that currently only LIVE VIEW tables are supported. See [#5541](https://github.com/ClickHouse/ClickHouse/issues/5541) for detailed description. [#6425](https://github.com/ClickHouse/ClickHouse/pull/6425) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) * Add ability to query Distributed over Distributed (w/o `distributed_group_by_no_merge`) ... [#9923](https://github.com/ClickHouse/ClickHouse/pull/9923) ([Azat Khuzhin](https://github.com/azat)) From be274ca14c010b48666cbecc09d49a6c0c643327 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 18 May 2020 04:53:42 +0300 Subject: [PATCH 342/738] Update CHANGELOG.md --- CHANGELOG.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc5b799d503..7a290085b2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,28 +9,27 @@ * Added a setting `validate_polygons` that is used in `pointInPolygon` function and enabled by default. [#9857](https://github.com/ClickHouse/ClickHouse/pull/9857) ([alexey-milovidov](https://github.com/alexey-milovidov)) #### New Feature +* Add support for secured connection from ClickHouse to Zookeeper [#10184](https://github.com/ClickHouse/ClickHouse/pull/10184) ([Konstantin Lebedev](https://github.com/xzkostyan)) * Support custom HTTP handlers. See ISSUES-5436 for description. [#7572](https://github.com/ClickHouse/ClickHouse/pull/7572) ([Winter Zhang](https://github.com/zhang2014)) +* Add MessagePack Input/Output format. [#9889](https://github.com/ClickHouse/ClickHouse/pull/9889) ([Kruglov Pavel](https://github.com/Avogar)) +* Add Regexp input format. [#9196](https://github.com/ClickHouse/ClickHouse/pull/9196) ([Kruglov Pavel](https://github.com/Avogar)) +* Added output format `Markdown` for embedding tables in markdown documents. [#10317](https://github.com/ClickHouse/ClickHouse/pull/10317) ([Kruglov Pavel](https://github.com/Avogar)) +* Added support for custom settings section in dictionaries. Also fixes issue [#2829](https://github.com/ClickHouse/ClickHouse/issues/2829). [#10137](https://github.com/ClickHouse/ClickHouse/pull/10137) ([Artem Streltsov](https://github.com/kekekekule)) * Added custom settings support in DDL-queries for CREATE DICTIONARY [#10465](https://github.com/ClickHouse/ClickHouse/pull/10465) ([Artem Streltsov](https://github.com/kekekekule)) * Add simple server-wide memory profiler that will collect allocation contexts when server memory usage becomes higher than the next allocation threshold. [#10444](https://github.com/ClickHouse/ClickHouse/pull/10444) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Added support for custom settings section in dictionaries. Also fixes issue [#2829](https://github.com/ClickHouse/ClickHouse/issues/2829). [#10137](https://github.com/ClickHouse/ClickHouse/pull/10137) ([Artem Streltsov](https://github.com/kekekekule)) * Add setting `always_fetch_merged_part` which restrict replica to merge parts by itself and always prefer dowloading from other replicas. [#10379](https://github.com/ClickHouse/ClickHouse/pull/10379) ([alesapin](https://github.com/alesapin)) * Add function JSONExtractKeysAndValuesRaw which extracts raw data from JSON objects [#10378](https://github.com/ClickHouse/ClickHouse/pull/10378) ([hcz](https://github.com/hczhcz)) * Add memory usage from OS to `system.asynchronous_metrics`. [#10361](https://github.com/ClickHouse/ClickHouse/pull/10361) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Added generic variants for functions `least` and `greatest`. Now they work with arbitrary number of arguments of arbitrary types. This fixes [#4767](https://github.com/ClickHouse/ClickHouse/issues/4767) [#10318](https://github.com/ClickHouse/ClickHouse/pull/10318) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Now ClickHouse controls timeouts of dictionary sources on its side. Two new settings added to cache dictionary configuration: `strict_max_lifetime_seconds`, which is `max_lifetime` by default, and `query_wait_timeout_milliseconds`, which is one minute by default. The first settings is also useful with `allow_read_expired_keys` settings (to forbid reading very expired keys). [#10337](https://github.com/ClickHouse/ClickHouse/pull/10337) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -* Added output format `Markdown` for embedding tables in markdown documents. [#10317](https://github.com/ClickHouse/ClickHouse/pull/10317) ([Kruglov Pavel](https://github.com/Avogar)) -* Add support for secured connection from ClickHouse to Zookeeper [#10184](https://github.com/ClickHouse/ClickHouse/pull/10184) ([Konstantin Lebedev](https://github.com/xzkostyan)) * Add log_queries_min_type to filter which entries will be written to query_log [#10053](https://github.com/ClickHouse/ClickHouse/pull/10053) ([Azat Khuzhin](https://github.com/azat)) * Added function `isConstant`. This function checks whether its argument is constant expression and returns 1 or 0. It is intended for development, debugging and demonstration purposes. [#10198](https://github.com/ClickHouse/ClickHouse/pull/10198) ([alexey-milovidov](https://github.com/alexey-milovidov)) * add joinGetOrNull to return NULL when key is missing instead of returning the default value. [#10094](https://github.com/ClickHouse/ClickHouse/pull/10094) ([Amos Bird](https://github.com/amosbird)) * Consider `NULL` to be equal to `NULL` in `IN` operator, if the option `transform_null_in` is set. [#10085](https://github.com/ClickHouse/ClickHouse/pull/10085) ([achimbab](https://github.com/achimbab)) -* Add MessagePack Input/Output format. [#9889](https://github.com/ClickHouse/ClickHouse/pull/9889) ([Kruglov Pavel](https://github.com/Avogar)) * Add `ALTER TABLE ... RENAME COLUMN` for MergeTree table engines family. [#9948](https://github.com/ClickHouse/ClickHouse/pull/9948) ([alesapin](https://github.com/alesapin)) * Support parallel distributed INSERT SELECT. [#9759](https://github.com/ClickHouse/ClickHouse/pull/9759) ([vxider](https://github.com/Vxider)) -* Added implementation of LIVE VIEW tables that were originally proposed in [#3925](https://github.com/ClickHouse/ClickHouse/issues/3925), and then updated in [#5541](https://github.com/ClickHouse/ClickHouse/issues/5541). Note that currently only LIVE VIEW tables are supported. See [#5541](https://github.com/ClickHouse/ClickHouse/issues/5541) for detailed description. [#6425](https://github.com/ClickHouse/ClickHouse/pull/6425) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) * Add ability to query Distributed over Distributed (w/o `distributed_group_by_no_merge`) ... [#9923](https://github.com/ClickHouse/ClickHouse/pull/9923) ([Azat Khuzhin](https://github.com/azat)) * Add function `arrayReduceInRanges` which aggregates array elements in given ranges. [#9598](https://github.com/ClickHouse/ClickHouse/pull/9598) ([hcz](https://github.com/hczhcz)) -* Add Regexp input format. [#9196](https://github.com/ClickHouse/ClickHouse/pull/9196) ([Kruglov Pavel](https://github.com/Avogar)) * Add Dictionary Status on prometheus exporter. [#9622](https://github.com/ClickHouse/ClickHouse/pull/9622) ([Guillaume Tassery](https://github.com/YiuRULE)) * Add array function auc [#8698](https://github.com/ClickHouse/ClickHouse/pull/8698) ([taiyang-li](https://github.com/taiyang-li)) * Support `DROP VIEW` statement for better TPC-H compatibility. [#9831](https://github.com/ClickHouse/ClickHouse/pull/9831) ([Amos Bird](https://github.com/amosbird)) From 3b83ea3e3804dad8f4ab1750c2d58710b76c5729 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 18 May 2020 04:54:17 +0300 Subject: [PATCH 343/738] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a290085b2a..6f719bbd672 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,7 @@ * Add ability to query Distributed over Distributed (w/o `distributed_group_by_no_merge`) ... [#9923](https://github.com/ClickHouse/ClickHouse/pull/9923) ([Azat Khuzhin](https://github.com/azat)) * Add function `arrayReduceInRanges` which aggregates array elements in given ranges. [#9598](https://github.com/ClickHouse/ClickHouse/pull/9598) ([hcz](https://github.com/hczhcz)) * Add Dictionary Status on prometheus exporter. [#9622](https://github.com/ClickHouse/ClickHouse/pull/9622) ([Guillaume Tassery](https://github.com/YiuRULE)) -* Add array function auc [#8698](https://github.com/ClickHouse/ClickHouse/pull/8698) ([taiyang-li](https://github.com/taiyang-li)) +* Add function arrayAUC [#8698](https://github.com/ClickHouse/ClickHouse/pull/8698) ([taiyang-li](https://github.com/taiyang-li)) * Support `DROP VIEW` statement for better TPC-H compatibility. [#9831](https://github.com/ClickHouse/ClickHouse/pull/9831) ([Amos Bird](https://github.com/amosbird)) * Add 'strict_order' option to windowFunnel() [#9773](https://github.com/ClickHouse/ClickHouse/pull/9773) ([achimbab](https://github.com/achimbab)) * Add support for configuring MongoDB connections with URI (for Atlas service). [#5387](https://github.com/ClickHouse/ClickHouse/pull/5387) ([Manish Srivastava](https://github.com/manish53)) From b3ec8c56521974e8f12e7f0d8bc8eb39811d645b Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 18 May 2020 04:54:53 +0300 Subject: [PATCH 344/738] Update CHANGELOG.md --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f719bbd672..c92e80f6390 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,7 +34,6 @@ * Add function arrayAUC [#8698](https://github.com/ClickHouse/ClickHouse/pull/8698) ([taiyang-li](https://github.com/taiyang-li)) * Support `DROP VIEW` statement for better TPC-H compatibility. [#9831](https://github.com/ClickHouse/ClickHouse/pull/9831) ([Amos Bird](https://github.com/amosbird)) * Add 'strict_order' option to windowFunnel() [#9773](https://github.com/ClickHouse/ClickHouse/pull/9773) ([achimbab](https://github.com/achimbab)) -* Add support for configuring MongoDB connections with URI (for Atlas service). [#5387](https://github.com/ClickHouse/ClickHouse/pull/5387) ([Manish Srivastava](https://github.com/manish53)) * Support `DATE` and `TIMESTAMP` SQL operators, e.g. `SELECT date '2001-01-01'` [#9691](https://github.com/ClickHouse/ClickHouse/pull/9691) ([Artem Zuikov](https://github.com/4ertus2)) #### Experimental Feature From 48ff08c057604468230bab688e87e625d7ab4f54 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 18 May 2020 04:58:18 +0300 Subject: [PATCH 345/738] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c92e80f6390..90ec2b47042 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ #### Experimental Feature * Added experimental database engine Atomic. It supports non-blocking `DROP` and `RENAME TABLE` queries and atomic `EXCHANGE TABLES t1 AND t2` query [#7512](https://github.com/ClickHouse/ClickHouse/pull/7512) ([tavplubix](https://github.com/tavplubix)) +* Initial support for ReplicatedMergeTree over S3 (it works in suboptimal way) [#10126](https://github.com/ClickHouse/ClickHouse/pull/10126) ([Pavel Kovalenko](https://github.com/Jokser)) #### Bug Fix * Fixed incorrect scalar results inside inner query of `MATERIALIZED VIEW` in case if this query contained dependent table [#10603](https://github.com/ClickHouse/ClickHouse/pull/10603) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) @@ -171,7 +172,6 @@ * Add config option `system_tables_lazy_load`. If it's set to false, then system tables with logs are loaded at the server startup. [Alexander Burmak](https://github.com/Alex-Burmak), [Svyatoslav Tkhon Il Pak](https://github.com/DeifyTheGod), [#9642](https://github.com/ClickHouse/ClickHouse/pull/9642) [#10359](https://github.com/ClickHouse/ClickHouse/pull/10359) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Use background thread pool (background_schedule_pool_size) for distributed sends [#10263](https://github.com/ClickHouse/ClickHouse/pull/10263) ([Azat Khuzhin](https://github.com/azat)) * Use background thread pool for background buffer flushes. [#10315](https://github.com/ClickHouse/ClickHouse/pull/10315) ([Azat Khuzhin](https://github.com/azat)) -* Support ReplicatedMergeTree over S3 [#10126](https://github.com/ClickHouse/ClickHouse/pull/10126) ([Pavel Kovalenko](https://github.com/Jokser)) * Support for one special case of removing incompletely written parts. This fixes [#9940](https://github.com/ClickHouse/ClickHouse/issues/9940). [#10221](https://github.com/ClickHouse/ClickHouse/pull/10221) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Use isInjective() over manual list of such functions for GROUP BY optimization. [#10342](https://github.com/ClickHouse/ClickHouse/pull/10342) ([Azat Khuzhin](https://github.com/azat)) * Improve MsgPack input format by adding msgpack::visitor concept instead of msgpack::unpack concept. [#10325](https://github.com/ClickHouse/ClickHouse/pull/10325) ([Kruglov Pavel](https://github.com/Avogar)) From 63be230caa58476084903f683f596191a9c9b605 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 18 May 2020 04:59:01 +0300 Subject: [PATCH 346/738] Update CHANGELOG.md --- CHANGELOG.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90ec2b47042..acbb8118016 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,7 +88,6 @@ * Fix queries with `max_bytes_before_external_group_by`. [#10302](https://github.com/ClickHouse/ClickHouse/pull/10302) ([Artem Zuikov](https://github.com/4ertus2)) * Fix the issue with limiting maximum recursion depth in parser in certain cases. This fixes [#10283](https://github.com/ClickHouse/ClickHouse/issues/10283) This fix may introduce minor incompatibility: long and deep queries via clickhouse-client may refuse to work, and you should adjust settings `max_query_size` and `max_parser_depth` accordingly. [#10295](https://github.com/ClickHouse/ClickHouse/pull/10295) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Allow to use `count(*)` with multiple JOINs. Fixes [#9853](https://github.com/ClickHouse/ClickHouse/issues/9853) [#10291](https://github.com/ClickHouse/ClickHouse/pull/10291) ([Artem Zuikov](https://github.com/4ertus2)) -* Remove converting ColumnFixedString to FixedString in MsgPack format and add performance test. [#10216](https://github.com/ClickHouse/ClickHouse/pull/10216) ([Kruglov Pavel](https://github.com/Avogar)) * Fix error `Pipeline stuck` with `max_rows_to_group_by` and `group_by_overflow_mode = 'break'`. [#10279](https://github.com/ClickHouse/ClickHouse/pull/10279) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) * Fix 'Cannot add column' error while creating `range_hashed` dictionary using DDL query. Fixes [#10093](https://github.com/ClickHouse/ClickHouse/issues/10093). [#10235](https://github.com/ClickHouse/ClickHouse/pull/10235) ([alesapin](https://github.com/alesapin)) * Fix rare possible exception `Cannot drain connections: cancel first`. [#10239](https://github.com/ClickHouse/ClickHouse/pull/10239) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) @@ -174,7 +173,6 @@ * Use background thread pool for background buffer flushes. [#10315](https://github.com/ClickHouse/ClickHouse/pull/10315) ([Azat Khuzhin](https://github.com/azat)) * Support for one special case of removing incompletely written parts. This fixes [#9940](https://github.com/ClickHouse/ClickHouse/issues/9940). [#10221](https://github.com/ClickHouse/ClickHouse/pull/10221) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Use isInjective() over manual list of such functions for GROUP BY optimization. [#10342](https://github.com/ClickHouse/ClickHouse/pull/10342) ([Azat Khuzhin](https://github.com/azat)) -* Improve MsgPack input format by adding msgpack::visitor concept instead of msgpack::unpack concept. [#10325](https://github.com/ClickHouse/ClickHouse/pull/10325) ([Kruglov Pavel](https://github.com/Avogar)) * Avoid printing error message in log if client sends RST packet immediately on connect. It is typical behaviour of IPVS balancer with keepalived and VRRP. This fixes [#1851](https://github.com/ClickHouse/ClickHouse/issues/1851) [#10274](https://github.com/ClickHouse/ClickHouse/pull/10274) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Allow to parse `+inf` for floating point types. This closes [#1839](https://github.com/ClickHouse/ClickHouse/issues/1839) [#10272](https://github.com/ClickHouse/ClickHouse/pull/10272) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Implemented `generateRandom` table function for Nested types. This closes [#9903](https://github.com/ClickHouse/ClickHouse/issues/9903) [#10219](https://github.com/ClickHouse/ClickHouse/pull/10219) ([alexey-milovidov](https://github.com/alexey-milovidov)) From cd190c0f0d62e71c68065f1465aad1a95a90e1ff Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 18 May 2020 04:59:34 +0300 Subject: [PATCH 347/738] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index acbb8118016..e247cec8e11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -156,7 +156,6 @@ * Fix issue when timezone was not preserved if you write a simple arithmetic expression like `time + 1` (in contrast to an expression like `time + INTERVAL 1 SECOND`). This fixes [#5743](https://github.com/ClickHouse/ClickHouse/issues/5743) [#9323](https://github.com/ClickHouse/ClickHouse/pull/9323) ([alexey-milovidov](https://github.com/alexey-milovidov)) #### Improvement -* Use AWS SDK libraries from ClickHouse-Extras [#10527](https://github.com/ClickHouse/ClickHouse/pull/10527) ([Pavel Kovalenko](https://github.com/Jokser)) * Use time zone when comparing DateTime with string literal. This fixes [#5206](https://github.com/ClickHouse/ClickHouse/issues/5206). [#10515](https://github.com/ClickHouse/ClickHouse/pull/10515) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Print verbose diagnostic info if Decimal value cannot be parsed from text input format. [#10205](https://github.com/ClickHouse/ClickHouse/pull/10205) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Add tasks/memory metrics for distributed/buffer schedule pools [#10449](https://github.com/ClickHouse/ClickHouse/pull/10449) ([Azat Khuzhin](https://github.com/azat)) @@ -229,6 +228,7 @@ * Use single row counter for multiple streams in pre-limit transform. This helps to avoid uniting pipeline streams in queries with `limit` but without `order by` (like `select f(x) from (select x from t limit 1000000000)`) and use multiple threads for further processing. [#9602](https://github.com/ClickHouse/ClickHouse/pull/9602) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) #### Build/Testing/Packaging Improvement +* Use a fork of AWS SDK libraries from ClickHouse-Extras [#10527](https://github.com/ClickHouse/ClickHouse/pull/10527) ([Pavel Kovalenko](https://github.com/Jokser)) * Add integration tests for new ALTER RENAME COLUMN query. [#10654](https://github.com/ClickHouse/ClickHouse/pull/10654) ([vzakaznikov](https://github.com/vzakaznikov)) * Fix possible signed integer overflow in invocation of function `now64` with wrong arguments. This fixes [#8973](https://github.com/ClickHouse/ClickHouse/issues/8973) [#10511](https://github.com/ClickHouse/ClickHouse/pull/10511) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Split fuzzer and sanitizer configurations to make build config compatible with Oss-fuzz. [#10494](https://github.com/ClickHouse/ClickHouse/pull/10494) ([kyprizel](https://github.com/kyprizel)) From 325cc6be36d2d8c7dfe218d7aa58a46ca3932ae9 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 18 May 2020 05:00:48 +0300 Subject: [PATCH 348/738] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e247cec8e11..c1a347a8fc2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -99,7 +99,7 @@ * Fix `parseDateTimeBestEffort` for strings in RFC-2822 when day of week is Tuesday or Thursday. This fixes [#10082](https://github.com/ClickHouse/ClickHouse/issues/10082) [#10214](https://github.com/ClickHouse/ClickHouse/pull/10214) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Fix column names of constants inside JOIN that may clash with names of constants outside of JOIN. [#10207](https://github.com/ClickHouse/ClickHouse/pull/10207) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Fix move-to-prewhere optimization in presense of arrayJoin functions (in certain cases). This fixes [#10092](https://github.com/ClickHouse/ClickHouse/issues/10092) [#10195](https://github.com/ClickHouse/ClickHouse/pull/10195) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fix issue with separator appearing in SCRAMBLE for native mysql-connector-java(JDBC) [#10140](https://github.com/ClickHouse/ClickHouse/pull/10140) ([BohuTANG](https://github.com/BohuTANG)) +* Fix issue with separator appearing in SCRAMBLE for native mysql-connector-java (JDBC) [#10140](https://github.com/ClickHouse/ClickHouse/pull/10140) ([BohuTANG](https://github.com/BohuTANG)) * Fix using the current database for an access checking when the database isn't specified. [#10192](https://github.com/ClickHouse/ClickHouse/pull/10192) ([Vitaly Baranov](https://github.com/vitlibar)) * Fix ALTER of tables with compact parts. [#10130](https://github.com/ClickHouse/ClickHouse/pull/10130) ([Anton Popov](https://github.com/CurtizJ)) * Add the ability to relax the restriction on non-deterministic functions usage in mutations with `allow_nondeterministic_mutations` setting. [#10186](https://github.com/ClickHouse/ClickHouse/pull/10186) ([filimonov](https://github.com/filimonov)) From 8cb53c39dff0dead49e406d0f1b7de31651e0c88 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 18 May 2020 05:01:44 +0300 Subject: [PATCH 349/738] Update CHANGELOG.md --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1a347a8fc2..1b55285be72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -187,7 +187,6 @@ * Now layout type for dictionaries with no arguments can be specified without round brackets in dictionaries DDL-queries. Fixes [#10057](https://github.com/ClickHouse/ClickHouse/issues/10057). [#10064](https://github.com/ClickHouse/ClickHouse/pull/10064) ([alesapin](https://github.com/alesapin)) * Add ability to use number ranges with leading zeros in filepath [#9989](https://github.com/ClickHouse/ClickHouse/pull/9989) ([Olga Khvostikova](https://github.com/stavrolia)) * Better memory usage in CROSS JOIN. [#10029](https://github.com/ClickHouse/ClickHouse/pull/10029) ([Artem Zuikov](https://github.com/4ertus2)) -* Use DFA algorithm for sequenceMatch without `(?t)`. [#4004](https://github.com/ClickHouse/ClickHouse/pull/4004) ([Léo Ercolanelli](https://github.com/LeoErcolanelli)) * The `clickhouse-format` tool is now able to format multiple queries when the `-n` argument is used. [#10852](https://github.com/ClickHouse/ClickHouse/pull/10852) ([Darío](https://github.com/dgrr)) * Try to connect to all shards in cluster when getting structure of remote table and skip_unavailable_shards is set. [#7278](https://github.com/ClickHouse/ClickHouse/pull/7278) ([nvartolomei](https://github.com/nvartolomei)) * Add `total_rows`/`total_bytes` into the `system.tables` table. [#9919](https://github.com/ClickHouse/ClickHouse/pull/9919) ([Azat Khuzhin](https://github.com/azat)) From 65039118e45da990d7e1609d7407d0a59d6a9631 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 18 May 2020 05:03:18 +0300 Subject: [PATCH 350/738] Update CHANGELOG.md --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b55285be72..99c1a80dcbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -187,7 +187,6 @@ * Now layout type for dictionaries with no arguments can be specified without round brackets in dictionaries DDL-queries. Fixes [#10057](https://github.com/ClickHouse/ClickHouse/issues/10057). [#10064](https://github.com/ClickHouse/ClickHouse/pull/10064) ([alesapin](https://github.com/alesapin)) * Add ability to use number ranges with leading zeros in filepath [#9989](https://github.com/ClickHouse/ClickHouse/pull/9989) ([Olga Khvostikova](https://github.com/stavrolia)) * Better memory usage in CROSS JOIN. [#10029](https://github.com/ClickHouse/ClickHouse/pull/10029) ([Artem Zuikov](https://github.com/4ertus2)) -* The `clickhouse-format` tool is now able to format multiple queries when the `-n` argument is used. [#10852](https://github.com/ClickHouse/ClickHouse/pull/10852) ([Darío](https://github.com/dgrr)) * Try to connect to all shards in cluster when getting structure of remote table and skip_unavailable_shards is set. [#7278](https://github.com/ClickHouse/ClickHouse/pull/7278) ([nvartolomei](https://github.com/nvartolomei)) * Add `total_rows`/`total_bytes` into the `system.tables` table. [#9919](https://github.com/ClickHouse/ClickHouse/pull/9919) ([Azat Khuzhin](https://github.com/azat)) * System log tables now use polymorpic parts by default. [#9905](https://github.com/ClickHouse/ClickHouse/pull/9905) ([Anton Popov](https://github.com/CurtizJ)) From 0692908cbf3ebebf9989f362a79f6ae4fb69c02a Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 18 May 2020 05:09:10 +0300 Subject: [PATCH 351/738] Update CHANGELOG.md --- CHANGELOG.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 99c1a80dcbf..1b1166ecf27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -216,10 +216,6 @@ * Enable parallel insert of materialized view when its target table supports. [#10052](https://github.com/ClickHouse/ClickHouse/pull/10052) ([vxider](https://github.com/Vxider)) * Improved performance of index analysis with monotonic functions. [#9607](https://github.com/ClickHouse/ClickHouse/pull/9607)[#10026](https://github.com/ClickHouse/ClickHouse/pull/10026) ([Anton Popov](https://github.com/CurtizJ)) * Using SSE2 or SSE4.2 SIMD intrinsics to speed up tokenization in bloom filters. [#9968](https://github.com/ClickHouse/ClickHouse/pull/9968) ([Vasily Nemkov](https://github.com/Enmk)) -* Optimize Volnitsky searcher by inlining, should be about 5-10% search improvement for queries with many needles or many similar bigrams [#4862](https://github.com/ClickHouse/ClickHouse/pull/4862) ([Danila Kutenin](https://github.com/danlark1)) -* Optimize count() [#6028](https://github.com/ClickHouse/ClickHouse/pull/6028) ([Amos Bird](https://github.com/amosbird)) -* Now `PREWHERE` should be at least as efficient as where. As a result, any viable predicates can be moved into `PREWHERE` expression without fancy heuristic [#7769](https://github.com/ClickHouse/ClickHouse/pull/7769) ([Amos Bird](https://github.com/amosbird)) -* Enable SSO for IN operator. Improvements varies from 4% to **257%**. [#7782](https://github.com/ClickHouse/ClickHouse/pull/7782) ([Amos Bird](https://github.com/amosbird)) * Improved performance of queries with explicitly defined sets at right side of `IN` operator. This fixes performance regression in version 20.3. [#9740](https://github.com/ClickHouse/ClickHouse/pull/9740) ([Anton Popov](https://github.com/CurtizJ)) * Now clickhouse-copier splits each partition in number of pieces and copies them independently. [#9075](https://github.com/ClickHouse/ClickHouse/pull/9075) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) * Adding more aggregation methods. For example TPC-H query 1 will now pick `FixedHashMap` and gets 25% performance gain [#9829](https://github.com/ClickHouse/ClickHouse/pull/9829) ([Amos Bird](https://github.com/amosbird)) From 24ea1a1656868ebd9172980b76e74c3016e1a4fe Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 18 May 2020 05:11:23 +0300 Subject: [PATCH 352/738] Update CHANGELOG.md --- CHANGELOG.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b1166ecf27..ca14134753a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -206,6 +206,7 @@ * Add a `timestamp_ns` column to `system.trace_log`. It contains a high-definition timestamp of the trace event, and allows to build timelines of thread profiles ("flame charts"). [#9696](https://github.com/ClickHouse/ClickHouse/pull/9696) ([Alexander Kuzmenkov](https://github.com/akuzm)) * When the setting `send_logs_level` is enabled, avoid intermixing of log messages and query progress. [#9634](https://github.com/ClickHouse/ClickHouse/pull/9634) ([Azat Khuzhin](https://github.com/azat)) * Added support of `MATERIALIZE TTL IN PARTITION`. [#9581](https://github.com/ClickHouse/ClickHouse/pull/9581) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Support complex types inside Avro nested fields [#10502](https://github.com/ClickHouse/ClickHouse/pull/10502) ([Andrew Onyshchuk](https://github.com/oandrew)) #### Performance Improvement * Better insert logic for right table for Partial MergeJoin. [#10467](https://github.com/ClickHouse/ClickHouse/pull/10467) ([Artem Zuikov](https://github.com/4ertus2)) @@ -276,9 +277,6 @@ * Add support for `clang-tidy` in `packager` script. [#9625](https://github.com/ClickHouse/ClickHouse/pull/9625) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Add ability to use unbundled msgpack. [#10168](https://github.com/ClickHouse/ClickHouse/pull/10168) ([Azat Khuzhin](https://github.com/azat)) -#### New Feature / Improvement -* Support complex types inside Avro nested fields [#10502](https://github.com/ClickHouse/ClickHouse/pull/10502) ([Andrew Onyshchuk](https://github.com/oandrew)) - ## ClickHouse release v20.3 From 327a48bcf9f3b8b3c233d0bfd65e4c411cf853dd Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 18 May 2020 05:19:08 +0300 Subject: [PATCH 353/738] Update AddingSelectorTransform.h --- src/Processors/Transforms/AddingSelectorTransform.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/Transforms/AddingSelectorTransform.h b/src/Processors/Transforms/AddingSelectorTransform.h index e82dcc964d0..bad97adfa76 100644 --- a/src/Processors/Transforms/AddingSelectorTransform.h +++ b/src/Processors/Transforms/AddingSelectorTransform.h @@ -13,7 +13,7 @@ class AddingSelectorTransform : public ISimpleTransform { public: AddingSelectorTransform(const Block & header, size_t num_outputs_, ColumnNumbers key_columns_); - String getName() const override { return "SplittingByHash"; } + String getName() const override { return "AddingSelector"; } void transform(Chunk & input_chunk, Chunk & output_chunk) override; private: From 6f0c78dfdda139aecdec1f1d1e647ae9190894bb Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 18 May 2020 06:50:53 +0300 Subject: [PATCH 354/738] Vectorize "sum" function --- src/AggregateFunctions/AggregateFunctionSum.h | 94 +++++++++++++++++-- 1 file changed, 85 insertions(+), 9 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionSum.h b/src/AggregateFunctions/AggregateFunctionSum.h index e9a6e50d9ef..b500fe91653 100644 --- a/src/AggregateFunctions/AggregateFunctionSum.h +++ b/src/AggregateFunctions/AggregateFunctionSum.h @@ -25,6 +25,36 @@ struct AggregateFunctionSumData sum += value; } + /// Vectorized version + template + void addMany(const Value * __restrict ptr, size_t count) + { + /// Compiler cannot unroll this loop, do it manually. + + /// Something around the number of SSE registers * the number of elements fit in register. + constexpr size_t unroll_count = 128 / sizeof(T); + T partial_sums[unroll_count]{}; + + const auto * end = ptr + count; + const auto * unrolled_end = ptr + (count / unroll_count * unroll_count); + + while (ptr < unrolled_end) + { + for (size_t i = 0; i < unroll_count; ++i) + partial_sums[i] += ptr[i]; + ptr += unroll_count; + } + + for (size_t i = 0; i < unroll_count; ++i) + sum += partial_sums[i]; + + while (ptr < end) + { + sum += *ptr; + ++ptr; + } + } + void merge(const AggregateFunctionSumData & rhs) { sum += rhs.sum; @@ -55,21 +85,60 @@ struct AggregateFunctionSumKahanData T sum{}; T compensation{}; + template + ALWAYS_INLINE void addImpl(Value value, T & out_sum, T & out_compensation) + { + auto compensated_value = value - out_compensation; + auto new_sum = out_sum + compensated_value; + out_compensation = (new_sum - out_sum) - compensated_value; + out_sum = new_sum; + } + void add(T value) { - auto compensated_value = value - compensation; - auto new_sum = sum + compensated_value; - compensation = (new_sum - sum) - compensated_value; - sum = new_sum; + addImpl(value, sum, compensation); + } + + /// Vectorized version + template + void addMany(const Value * __restrict ptr, size_t count) + { + constexpr size_t unroll_count = 4; // 128 / sizeof(T); + T partial_sums[unroll_count]{}; + T partial_compensations[unroll_count]{}; + + const auto * end = ptr + count; + const auto * unrolled_end = ptr + (count / unroll_count * unroll_count); + + while (ptr < unrolled_end) + { + for (size_t i = 0; i < unroll_count; ++i) + addImpl(ptr[i], partial_sums[i], partial_compensations[i]); + ptr += unroll_count; + } + + for (size_t i = 0; i < unroll_count; ++i) + mergeImpl(sum, compensation, partial_sums[i], partial_compensations[i]); + + while (ptr < end) + { + addImpl(*ptr, sum, compensation); + ++ptr; + } + } + + void ALWAYS_INLINE mergeImpl(T & to_sum, T & to_compensation, T from_sum, T from_compensation) + { + auto raw_sum = to_sum + from_sum; + auto rhs_compensated = raw_sum - to_sum; + auto compensations = ((from_sum - rhs_compensated) + (to_sum - (raw_sum - rhs_compensated))) + compensation + from_compensation; + to_sum = raw_sum + compensations; + to_compensation = compensations - (to_sum - raw_sum); } void merge(const AggregateFunctionSumKahanData & rhs) { - auto raw_sum = sum + rhs.sum; - auto rhs_compensated = raw_sum - sum; - auto compensations = ((rhs.sum - rhs_compensated) + (sum - (raw_sum - rhs_compensated))) + compensation + rhs.compensation; - sum = raw_sum + compensations; - compensation = compensations - (sum - raw_sum); + mergeImpl(sum, compensation, rhs.sum, rhs.compensation); } void write(WriteBuffer & buf) const @@ -141,6 +210,13 @@ public: this->data(place).add(column.getData()[row_num]); } + /// Vectorized version when there is no GROUP BY keys. + void addBatchSinglePlace(size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena *) const override + { + const auto & column = static_cast(*columns[0]); + this->data(place).addMany(column.getData().data(), batch_size); + } + void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override { this->data(place).merge(this->data(rhs)); From 719a3b5a2c61101cd5e6207e61ab1d1c90c04fb9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 18 May 2020 07:49:51 +0300 Subject: [PATCH 355/738] Performance improvement for "sum" over nullable --- .../AggregateFunctionNull.h | 11 ++- src/AggregateFunctions/AggregateFunctionSum.h | 81 +++++++++++++++++-- src/AggregateFunctions/IAggregateFunction.h | 13 +++ 3 files changed, 98 insertions(+), 7 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionNull.h b/src/AggregateFunctions/AggregateFunctionNull.h index e5309e1300a..10a9f207e93 100644 --- a/src/AggregateFunctions/AggregateFunctionNull.h +++ b/src/AggregateFunctions/AggregateFunctionNull.h @@ -194,13 +194,22 @@ public: void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena * arena) const override { const ColumnNullable * column = assert_cast(columns[0]); + const IColumn * nested_column = &column->getNestedColumn(); if (!column->isNullAt(row_num)) { this->setFlag(place); - const IColumn * nested_column = &column->getNestedColumn(); this->nested_function->add(this->nestedPlace(place), &nested_column, row_num, arena); } } + + void addBatchSinglePlace(size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena) const override + { + const ColumnNullable * column = assert_cast(columns[0]); + const IColumn * nested_column = &column->getNestedColumn(); + const UInt8 * null_map = column->getNullMapData().data(); + + this->nested_function->addBatchSinglePlaceNotNull(batch_size, this->nestedPlace(place), &nested_column, null_map, arena); + } }; diff --git a/src/AggregateFunctions/AggregateFunctionSum.h b/src/AggregateFunctions/AggregateFunctionSum.h index b500fe91653..52431060aea 100644 --- a/src/AggregateFunctions/AggregateFunctionSum.h +++ b/src/AggregateFunctions/AggregateFunctionSum.h @@ -20,16 +20,17 @@ struct AggregateFunctionSumData { T sum{}; - void add(T value) + void ALWAYS_INLINE add(T value) { sum += value; } /// Vectorized version template - void addMany(const Value * __restrict ptr, size_t count) + void NO_INLINE addMany(const Value * __restrict ptr, size_t count) { /// Compiler cannot unroll this loop, do it manually. + /// (at least for floats, most likely due to the lack of -fassociative-math) /// Something around the number of SSE registers * the number of elements fit in register. constexpr size_t unroll_count = 128 / sizeof(T); @@ -55,6 +56,36 @@ struct AggregateFunctionSumData } } + template + void NO_INLINE addManyNotNull(const Value * __restrict ptr, const UInt8 * __restrict null_map, size_t count) + { + constexpr size_t unroll_count = 128 / sizeof(T); + T partial_sums[unroll_count]{}; + + const auto * end = ptr + count; + const auto * unrolled_end = ptr + (count / unroll_count * unroll_count); + + while (ptr < unrolled_end) + { + for (size_t i = 0; i < unroll_count; ++i) + if (!null_map[i]) + partial_sums[i] += ptr[i]; + ptr += unroll_count; + null_map += unroll_count; + } + + for (size_t i = 0; i < unroll_count; ++i) + sum += partial_sums[i]; + + while (ptr < end) + { + if (!*null_map) + sum += *ptr; + ++ptr; + ++null_map; + } + } + void merge(const AggregateFunctionSumData & rhs) { sum += rhs.sum; @@ -86,7 +117,7 @@ struct AggregateFunctionSumKahanData T compensation{}; template - ALWAYS_INLINE void addImpl(Value value, T & out_sum, T & out_compensation) + void ALWAYS_INLINE addImpl(Value value, T & out_sum, T & out_compensation) { auto compensated_value = value - out_compensation; auto new_sum = out_sum + compensated_value; @@ -94,16 +125,16 @@ struct AggregateFunctionSumKahanData out_sum = new_sum; } - void add(T value) + void ALWAYS_INLINE add(T value) { addImpl(value, sum, compensation); } /// Vectorized version template - void addMany(const Value * __restrict ptr, size_t count) + void NO_INLINE addMany(const Value * __restrict ptr, size_t count) { - constexpr size_t unroll_count = 4; // 128 / sizeof(T); + constexpr size_t unroll_count = 4; T partial_sums[unroll_count]{}; T partial_compensations[unroll_count]{}; @@ -127,6 +158,37 @@ struct AggregateFunctionSumKahanData } } + template + void NO_INLINE addManyNotNull(const Value * __restrict ptr, const UInt8 * __restrict null_map, size_t count) + { + constexpr size_t unroll_count = 4; + T partial_sums[unroll_count]{}; + T partial_compensations[unroll_count]{}; + + const auto * end = ptr + count; + const auto * unrolled_end = ptr + (count / unroll_count * unroll_count); + + while (ptr < unrolled_end) + { + for (size_t i = 0; i < unroll_count; ++i) + if (!null_map[i]) + addImpl(ptr[i], partial_sums[i], partial_compensations[i]); + ptr += unroll_count; + null_map += unroll_count; + } + + for (size_t i = 0; i < unroll_count; ++i) + mergeImpl(sum, compensation, partial_sums[i], partial_compensations[i]); + + while (ptr < end) + { + if (!*null_map) + addImpl(*ptr, sum, compensation); + ++ptr; + ++null_map; + } + } + void ALWAYS_INLINE mergeImpl(T & to_sum, T & to_compensation, T from_sum, T from_compensation) { auto raw_sum = to_sum + from_sum; @@ -217,6 +279,13 @@ public: this->data(place).addMany(column.getData().data(), batch_size); } + void addBatchSinglePlaceNotNull( + size_t batch_size, AggregateDataPtr place, const IColumn ** columns, const UInt8 * null_map, Arena *) const override + { + const auto & column = static_cast(*columns[0]); + this->data(place).addManyNotNull(column.getData().data(), null_map, batch_size); + } + void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override { this->data(place).merge(this->data(rhs)); diff --git a/src/AggregateFunctions/IAggregateFunction.h b/src/AggregateFunctions/IAggregateFunction.h index ad074feffc5..1870eee07b8 100644 --- a/src/AggregateFunctions/IAggregateFunction.h +++ b/src/AggregateFunctions/IAggregateFunction.h @@ -145,6 +145,11 @@ public: */ virtual void addBatchSinglePlace(size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena) const = 0; + /** The same for single place when need to aggregate only filtered data. + */ + virtual void addBatchSinglePlaceNotNull( + size_t batch_size, AggregateDataPtr place, const IColumn ** columns, const UInt8 * null_map, Arena * arena) const = 0; + /** In addition to addBatch, this method collects multiple rows of arguments into array "places" * as long as they are between offsets[i-1] and offsets[i]. This is used for arrayReduce and * -Array combinator. It might also be used generally to break data dependency when array @@ -201,6 +206,14 @@ public: static_cast(this)->add(place, columns, i, arena); } + void addBatchSinglePlaceNotNull( + size_t batch_size, AggregateDataPtr place, const IColumn ** columns, const UInt8 * null_map, Arena * arena) const override + { + for (size_t i = 0; i < batch_size; ++i) + if (!null_map[i]) + static_cast(this)->add(place, columns, i, arena); + } + void addBatchArray( size_t batch_size, AggregateDataPtr * places, size_t place_offset, const IColumn ** columns, const UInt64 * offsets, Arena * arena) const override From db422434ff3324a825f2507e41ce52448c1f0489 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 18 May 2020 07:55:06 +0300 Subject: [PATCH 356/738] Add performance test --- tests/performance/sum.xml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 tests/performance/sum.xml diff --git a/tests/performance/sum.xml b/tests/performance/sum.xml new file mode 100644 index 00000000000..9bee2a580c3 --- /dev/null +++ b/tests/performance/sum.xml @@ -0,0 +1,19 @@ + + SELECT sum(number) FROM numbers(100000000) + SELECT sum(toUInt32(number)) FROM numbers(100000000) + SELECT sum(toUInt16(number)) FROM numbers(100000000) + SELECT sum(toUInt8(number)) FROM numbers(100000000) + SELECT sum(toFloat32(number)) FROM numbers(100000000) + SELECT sum(toFloat64(number)) FROM numbers(100000000) + SELECT sumKahan(toFloat32(number)) FROM numbers(100000000) + SELECT sumKahan(toFloat64(number)) FROM numbers(100000000) + + SELECT sum(toNullable(number)) FROM numbers(100000000) + SELECT sum(toNullable(toUInt32(number))) FROM numbers(100000000) + SELECT sum(toNullable(toUInt16(number))) FROM numbers(100000000) + SELECT sum(toNullable(toUInt8(number))) FROM numbers(100000000) + SELECT sum(toNullable(toFloat32(number))) FROM numbers(100000000) + SELECT sum(toNullable(toFloat64(number))) FROM numbers(100000000) + SELECT sumKahan(toNullable(toFloat32(number))) FROM numbers(100000000) + SELECT sumKahan(toNullable(toFloat64(number))) FROM numbers(100000000) + From f5072aab18b692d44cad29af71f347e6bfd06fc1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 18 May 2020 08:01:55 +0300 Subject: [PATCH 357/738] Fix error --- src/AggregateFunctions/AggregateFunctionNull.h | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionNull.h b/src/AggregateFunctions/AggregateFunctionNull.h index 10a9f207e93..55d610207f1 100644 --- a/src/AggregateFunctions/AggregateFunctionNull.h +++ b/src/AggregateFunctions/AggregateFunctionNull.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -53,13 +54,13 @@ protected: static void initFlag(AggregateDataPtr place) noexcept { - if (result_is_nullable) + if constexpr (result_is_nullable) place[0] = 0; } static void setFlag(AggregateDataPtr place) noexcept { - if (result_is_nullable) + if constexpr (result_is_nullable) place[0] = 1; } @@ -72,7 +73,7 @@ public: AggregateFunctionNullBase(AggregateFunctionPtr nested_function_, const DataTypes & arguments, const Array & params) : IAggregateFunctionHelper(arguments, params), nested_function{nested_function_} { - if (result_is_nullable) + if constexpr (result_is_nullable) prefix_size = nested_function->alignOfData(); else prefix_size = 0; @@ -128,7 +129,7 @@ public: void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override { bool flag = getFlag(place); - if (result_is_nullable) + if constexpr (result_is_nullable) writeBinary(flag, buf); if (flag) nested_function->serialize(nestedPlace(place), buf); @@ -137,7 +138,7 @@ public: void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena * arena) const override { bool flag = 1; - if (result_is_nullable) + if constexpr (result_is_nullable) readBinary(flag, buf); if (flag) { @@ -148,7 +149,7 @@ public: void insertResultInto(AggregateDataPtr place, IColumn & to) const override { - if (result_is_nullable) + if constexpr (result_is_nullable) { ColumnNullable & to_concrete = assert_cast(to); if (getFlag(place)) @@ -209,6 +210,10 @@ public: const UInt8 * null_map = column->getNullMapData().data(); this->nested_function->addBatchSinglePlaceNotNull(batch_size, this->nestedPlace(place), &nested_column, null_map, arena); + + if constexpr (result_is_nullable) + if (!memoryIsByte(null_map, batch_size, 1)) + this->setFlag(place); } }; From 65e387f1a3496e0fd3715574ce25a5bf3e8b1820 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 18 May 2020 08:03:58 +0300 Subject: [PATCH 358/738] Added a test --- tests/queries/0_stateless/01281_sum_nullable.reference | 6 ++++++ tests/queries/0_stateless/01281_sum_nullable.sql | 6 ++++++ 2 files changed, 12 insertions(+) create mode 100644 tests/queries/0_stateless/01281_sum_nullable.reference create mode 100644 tests/queries/0_stateless/01281_sum_nullable.sql diff --git a/tests/queries/0_stateless/01281_sum_nullable.reference b/tests/queries/0_stateless/01281_sum_nullable.reference new file mode 100644 index 00000000000..be8b67fd296 --- /dev/null +++ b/tests/queries/0_stateless/01281_sum_nullable.reference @@ -0,0 +1,6 @@ +45 +45 +45 +1 +45 +\N diff --git a/tests/queries/0_stateless/01281_sum_nullable.sql b/tests/queries/0_stateless/01281_sum_nullable.sql new file mode 100644 index 00000000000..35d593da75d --- /dev/null +++ b/tests/queries/0_stateless/01281_sum_nullable.sql @@ -0,0 +1,6 @@ +SELECT sumKahan(toFloat64(number)) FROM numbers(10); +SELECT sumKahan(toNullable(toFloat64(number))) FROM numbers(10); +SELECT sum(toNullable(number)) FROM numbers(10); +SELECT sum(x) FROM (SELECT 1 AS x UNION ALL SELECT NULL); +SELECT sum(number) FROM numbers(10); +SELECT sum(number < 1000 ? NULL : number) FROM numbers(10); From cdb742dd4b5d0d922a54836f89eaf83b0f73daa4 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 18 May 2020 09:06:48 +0300 Subject: [PATCH 359/738] Added comment. --- src/Common/COW.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Common/COW.h b/src/Common/COW.h index 2bc8e8aba00..23733a8635d 100644 --- a/src/Common/COW.h +++ b/src/Common/COW.h @@ -217,6 +217,7 @@ protected: operator const immutable_ptr & () const { return value; } operator immutable_ptr & () { return value; } + /// Get internal immutable ptr. Does not change internal use counter. immutable_ptr detach() && { return std::move(value); } operator bool() const { return value != nullptr; } From 738842b846424369808077b4a5e62e9c60231dbd Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 18 May 2020 09:20:17 +0300 Subject: [PATCH 360/738] Moved test #4277 to bugs --- .../{0_stateless => bugs}/leak_when_memory_limit_exceeded.sql | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/queries/{0_stateless => bugs}/leak_when_memory_limit_exceeded.sql (100%) diff --git a/tests/queries/0_stateless/leak_when_memory_limit_exceeded.sql b/tests/queries/bugs/leak_when_memory_limit_exceeded.sql similarity index 100% rename from tests/queries/0_stateless/leak_when_memory_limit_exceeded.sql rename to tests/queries/bugs/leak_when_memory_limit_exceeded.sql From 07759d5d011ca3978d33fd1c8371d8606d8f8c3e Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 07:56:20 +0000 Subject: [PATCH 361/738] Bump protobuf from 3.11.3 to 3.12.0 in /docs/tools Bumps [protobuf](https://github.com/protocolbuffers/protobuf) from 3.11.3 to 3.12.0. - [Release notes](https://github.com/protocolbuffers/protobuf/releases) - [Changelog](https://github.com/protocolbuffers/protobuf/blob/master/generate_changelog.py) - [Commits](https://github.com/protocolbuffers/protobuf/compare/v3.11.3...v3.12.0) Signed-off-by: dependabot-preview[bot] --- docs/tools/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tools/requirements.txt b/docs/tools/requirements.txt index 246c49738e6..3d5957cf9b6 100644 --- a/docs/tools/requirements.txt +++ b/docs/tools/requirements.txt @@ -21,7 +21,7 @@ mkdocs-htmlproofer-plugin==0.0.3 mkdocs-macros-plugin==0.4.7 nltk==3.5 nose==1.3.7 -protobuf==3.11.3 +protobuf==3.12.0 numpy==1.18.4 Pygments==2.5.2 pymdown-extensions==7.1 From 380dce33f39b238684bc7f4ab0eaa95e76f44147 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 07:56:39 +0000 Subject: [PATCH 362/738] Bump beautifulsoup4 from 4.9.0 to 4.9.1 in /docs/tools Bumps [beautifulsoup4](http://www.crummy.com/software/BeautifulSoup/bs4/) from 4.9.0 to 4.9.1. Signed-off-by: dependabot-preview[bot] --- docs/tools/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tools/requirements.txt b/docs/tools/requirements.txt index 246c49738e6..94dd8ed6f35 100644 --- a/docs/tools/requirements.txt +++ b/docs/tools/requirements.txt @@ -1,7 +1,7 @@ Babel==2.8.0 backports-abc==0.5 backports.functools-lru-cache==1.6.1 -beautifulsoup4==4.9.0 +beautifulsoup4==4.9.1 certifi==2020.4.5.1 chardet==3.0.4 click==7.1.2 From 0d1f63d1c59e4883576a1a3a9fae5fbfe81db8db Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 07:57:03 +0000 Subject: [PATCH 363/738] Bump soupsieve from 2.0 to 2.0.1 in /docs/tools Bumps [soupsieve](https://github.com/facelessuser/soupsieve) from 2.0 to 2.0.1. - [Release notes](https://github.com/facelessuser/soupsieve/releases) - [Commits](https://github.com/facelessuser/soupsieve/compare/2.0.0...2.0.1) Signed-off-by: dependabot-preview[bot] --- docs/tools/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tools/requirements.txt b/docs/tools/requirements.txt index 246c49738e6..f9070475dfb 100644 --- a/docs/tools/requirements.txt +++ b/docs/tools/requirements.txt @@ -31,7 +31,7 @@ repackage==0.7.3 requests==2.23.0 singledispatch==3.4.0.3 six==1.14.0 -soupsieve==2.0 +soupsieve==2.0.1 termcolor==1.1.0 tornado==5.1.1 Unidecode==1.1.1 From f54435e7fda3d377c21c2cbecd49345044dc14ee Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 18 May 2020 11:08:55 +0300 Subject: [PATCH 364/738] Fix clang-tidy --- programs/benchmark/Benchmark.cpp | 2 +- programs/client/Client.cpp | 4 +- programs/client/ConnectionParameters.cpp | 2 +- programs/copier/ClusterCopier.cpp | 4 +- programs/copier/ClusterCopierApp.cpp | 2 +- programs/copier/Internals.cpp | 4 +- programs/obfuscator/Obfuscator.cpp | 4 +- programs/server/HTTPHandler.cpp | 4 +- programs/server/HTTPHandlerFactory.cpp | 133 ++++++++++-------- programs/server/ReplicasStatusHandler.cpp | 2 +- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 12 +- .../main.cpp | 2 +- 12 files changed, 98 insertions(+), 77 deletions(-) diff --git a/programs/benchmark/Benchmark.cpp b/programs/benchmark/Benchmark.cpp index ce59f5cac7f..91c43160e0f 100644 --- a/programs/benchmark/Benchmark.cpp +++ b/programs/benchmark/Benchmark.cpp @@ -289,7 +289,7 @@ private: connection_entries.emplace_back(std::make_shared( connection->get(ConnectionTimeouts::getTCPTimeoutsWithoutFailover(settings)))); - pool.scheduleOrThrowOnError(std::bind(&Benchmark::thread, this, connection_entries)); + pool.scheduleOrThrowOnError([this, connection_entries]() mutable { thread(connection_entries); }); } } catch (...) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 53947283faf..d6cac7a7b02 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -485,7 +485,7 @@ private: history_file = config().getString("history_file"); else { - auto history_file_from_env = getenv("CLICKHOUSE_HISTORY_FILE"); + auto * history_file_from_env = getenv("CLICKHOUSE_HISTORY_FILE"); if (history_file_from_env) history_file = history_file_from_env; else if (!home_path.empty()) @@ -1480,7 +1480,7 @@ private: "\033[1m↗\033[0m", }; - auto indicator = indicators[increment % 8]; + const char * indicator = indicators[increment % 8]; if (!send_logs && written_progress_chars) message << '\r'; diff --git a/programs/client/ConnectionParameters.cpp b/programs/client/ConnectionParameters.cpp index 50cac3b7800..f0ef3ae5694 100644 --- a/programs/client/ConnectionParameters.cpp +++ b/programs/client/ConnectionParameters.cpp @@ -51,7 +51,7 @@ ConnectionParameters::ConnectionParameters(const Poco::Util::AbstractConfigurati { std::string prompt{"Password for user (" + user + "): "}; char buf[1000] = {}; - if (auto result = readpassphrase(prompt.c_str(), buf, sizeof(buf), 0)) + if (auto * result = readpassphrase(prompt.c_str(), buf, sizeof(buf), 0)) password = result; } diff --git a/programs/copier/ClusterCopier.cpp b/programs/copier/ClusterCopier.cpp index 8df55b63407..45cfb4963a3 100644 --- a/programs/copier/ClusterCopier.cpp +++ b/programs/copier/ClusterCopier.cpp @@ -442,7 +442,7 @@ bool ClusterCopier::checkPartitionPieceIsDone(const TaskTable & task_table, cons /// Collect all shards that contain partition piece number piece_number. Strings piece_status_paths; - for (auto & shard : shards_with_partition) + for (const auto & shard : shards_with_partition) { ShardPartition & task_shard_partition = shard->partition_tasks.find(partition_name)->second; ShardPartitionPiece & shard_partition_piece = task_shard_partition.pieces[piece_number]; @@ -702,7 +702,7 @@ ASTPtr ClusterCopier::removeAliasColumnsFromCreateQuery(const ASTPtr & query_ast auto new_columns_list = std::make_shared(); new_columns_list->set(new_columns_list->columns, new_columns); - if (auto indices = query_ast->as()->columns_list->indices) + if (const auto * indices = query_ast->as()->columns_list->indices) new_columns_list->set(new_columns_list->indices, indices->clone()); new_query.replace(new_query.columns_list, new_columns_list); diff --git a/programs/copier/ClusterCopierApp.cpp b/programs/copier/ClusterCopierApp.cpp index 52a37c75c72..5ab6064f0f3 100644 --- a/programs/copier/ClusterCopierApp.cpp +++ b/programs/copier/ClusterCopierApp.cpp @@ -94,7 +94,7 @@ void ClusterCopierApp::mainImpl() StatusFile status_file(process_path + "/status"); ThreadStatus thread_status; - auto log = &logger(); + auto * log = &logger(); LOG_INFO(log, "Starting clickhouse-copier (" << "id " << process_id << ", " << "host_id " << host_id << ", " diff --git a/programs/copier/Internals.cpp b/programs/copier/Internals.cpp index 545df2e779c..0613381a763 100644 --- a/programs/copier/Internals.cpp +++ b/programs/copier/Internals.cpp @@ -260,7 +260,7 @@ ShardPriority getReplicasPriority(const Cluster::Addresses & replicas, const std return res; res.is_remote = 1; - for (auto & replica : replicas) + for (const auto & replica : replicas) { if (isLocalAddress(DNSResolver::instance().resolveHost(replica.host_name))) { @@ -270,7 +270,7 @@ ShardPriority getReplicasPriority(const Cluster::Addresses & replicas, const std } res.hostname_difference = std::numeric_limits::max(); - for (auto & replica : replicas) + for (const auto & replica : replicas) { size_t difference = getHostNameDifference(local_hostname, replica.host_name); res.hostname_difference = std::min(difference, res.hostname_difference); diff --git a/programs/obfuscator/Obfuscator.cpp b/programs/obfuscator/Obfuscator.cpp index 8b5a8c73ca4..f3ac0549573 100644 --- a/programs/obfuscator/Obfuscator.cpp +++ b/programs/obfuscator/Obfuscator.cpp @@ -937,10 +937,10 @@ public: if (typeid_cast(&data_type)) return std::make_unique(seed); - if (auto type = typeid_cast(&data_type)) + if (const auto * type = typeid_cast(&data_type)) return std::make_unique(get(*type->getNestedType(), seed, markov_model_params)); - if (auto type = typeid_cast(&data_type)) + if (const auto * type = typeid_cast(&data_type)) return std::make_unique(get(*type->getNestedType(), seed, markov_model_params)); throw Exception("Unsupported data type", ErrorCodes::NOT_IMPLEMENTED); diff --git a/programs/server/HTTPHandler.cpp b/programs/server/HTTPHandler.cpp index bceeec306cf..701b5f7d735 100644 --- a/programs/server/HTTPHandler.cpp +++ b/programs/server/HTTPHandler.cpp @@ -195,7 +195,7 @@ void HTTPHandler::pushDelayedResults(Output & used_output) std::vector read_buffers; std::vector read_buffers_raw_ptr; - auto cascade_buffer = typeid_cast(used_output.out_maybe_delayed_and_compressed.get()); + auto * cascade_buffer = typeid_cast(used_output.out_maybe_delayed_and_compressed.get()); if (!cascade_buffer) throw Exception("Expected CascadeWriteBuffer", ErrorCodes::LOGICAL_ERROR); @@ -383,7 +383,7 @@ void HTTPHandler::processQuery( { auto push_memory_buffer_and_continue = [next_buffer = used_output.out_maybe_compressed] (const WriteBufferPtr & prev_buf) { - auto prev_memory_buffer = typeid_cast(prev_buf.get()); + auto * prev_memory_buffer = typeid_cast(prev_buf.get()); if (!prev_memory_buffer) throw Exception("Expected MemoryWriteBuffer", ErrorCodes::LOGICAL_ERROR); diff --git a/programs/server/HTTPHandlerFactory.cpp b/programs/server/HTTPHandlerFactory.cpp index 91cf9ddf25b..4caea1e92e8 100644 --- a/programs/server/HTTPHandlerFactory.cpp +++ b/programs/server/HTTPHandlerFactory.cpp @@ -28,7 +28,7 @@ HTTPRequestHandlerFactoryMain::HTTPRequestHandlerFactoryMain(const std::string & { } -Poco::Net::HTTPRequestHandler * HTTPRequestHandlerFactoryMain::createRequestHandler(const Poco::Net::HTTPServerRequest & request) // override +Poco::Net::HTTPRequestHandler * HTTPRequestHandlerFactoryMain::createRequestHandler(const Poco::Net::HTTPServerRequest & request) { LOG_TRACE(log, "HTTP Request for " << name << ". " << "Method: " << request.getMethod() @@ -40,7 +40,7 @@ Poco::Net::HTTPRequestHandler * HTTPRequestHandlerFactoryMain::createRequestHand for (auto & handler_factory : child_factories) { - auto handler = handler_factory->createRequestHandler(request); + auto * handler = handler_factory->createRequestHandler(request); if (handler != nullptr) return handler; } @@ -72,80 +72,96 @@ HTTPRequestHandlerFactoryMain::TThis * HTTPRequestHandlerFactoryMain::addHandler static inline auto createHandlersFactoryFromConfig(IServer & server, const std::string & name, const String & prefix) { - auto main_handler_factory = new HTTPRequestHandlerFactoryMain(name); + auto main_handler_factory = std::make_unique(name); - try + Poco::Util::AbstractConfiguration::Keys keys; + server.config().keys(prefix, keys); + + for (const auto & key : keys) { - Poco::Util::AbstractConfiguration::Keys keys; - server.config().keys(prefix, keys); + if (!startsWith(key, "rule")) + throw Exception("Unknown element in config: " + prefix + "." + key + ", must be 'rule'", ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); - for (const auto & key : keys) - { - if (!startsWith(key, "rule")) - throw Exception("Unknown element in config: " + prefix + "." + key + ", must be 'rule'", ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); + const auto & handler_type = server.config().getString(prefix + "." + key + ".handler.type", ""); - const auto & handler_type = server.config().getString(prefix + "." + key + ".handler.type", ""); - - if (handler_type == "static") - main_handler_factory->addHandler(createStaticHandlerFactory(server, prefix + "." + key)); - else if (handler_type == "dynamic_query_handler") - main_handler_factory->addHandler(createDynamicHandlerFactory(server, prefix + "." + key)); - else if (handler_type == "predefined_query_handler") - main_handler_factory->addHandler(createPredefinedHandlerFactory(server, prefix + "." + key)); - else if (handler_type.empty()) - throw Exception("Handler type in config is not specified here: " + - prefix + "." + key + ".handler.type", ErrorCodes::INVALID_CONFIG_PARAMETER); - else - throw Exception("Unknown handler type '" + handler_type +"' in config here: " + - prefix + "." + key + ".handler.type",ErrorCodes::INVALID_CONFIG_PARAMETER); - } - - return main_handler_factory; - } - catch (...) - { - delete main_handler_factory; - throw; + if (handler_type == "static") + main_handler_factory->addHandler(createStaticHandlerFactory(server, prefix + "." + key)); + else if (handler_type == "dynamic_query_handler") + main_handler_factory->addHandler(createDynamicHandlerFactory(server, prefix + "." + key)); + else if (handler_type == "predefined_query_handler") + main_handler_factory->addHandler(createPredefinedHandlerFactory(server, prefix + "." + key)); + else if (handler_type.empty()) + throw Exception("Handler type in config is not specified here: " + + prefix + "." + key + ".handler.type", ErrorCodes::INVALID_CONFIG_PARAMETER); + else + throw Exception("Unknown handler type '" + handler_type +"' in config here: " + + prefix + "." + key + ".handler.type",ErrorCodes::INVALID_CONFIG_PARAMETER); } + + return main_handler_factory.release(); } static const auto ping_response_expression = "Ok.\n"; static const auto root_response_expression = "config://http_server_default_response"; -static inline Poco::Net::HTTPRequestHandlerFactory * createHTTPHandlerFactory(IServer & server, const std::string & name, AsynchronousMetrics & async_metrics) +static inline Poco::Net::HTTPRequestHandlerFactory * createHTTPHandlerFactory( + IServer & server, const std::string & name, AsynchronousMetrics & async_metrics) { if (server.config().has("http_handlers")) return createHandlersFactoryFromConfig(server, name, "http_handlers"); else { - auto factory = (new HTTPRequestHandlerFactoryMain(name)) - ->addHandler((new HandlingRuleHTTPHandlerFactory(server, root_response_expression)) - ->attachStrictPath("/")->allowGetAndHeadRequest()) - ->addHandler((new HandlingRuleHTTPHandlerFactory(server, ping_response_expression)) - ->attachStrictPath("/ping")->allowGetAndHeadRequest()) - ->addHandler((new HandlingRuleHTTPHandlerFactory(server)) - ->attachNonStrictPath("/replicas_status")->allowGetAndHeadRequest()) - ->addHandler((new HandlingRuleHTTPHandlerFactory(server, "query"))->allowPostAndGetParamsRequest()); + auto factory = std::make_unique(name); + + auto root_handler = std::make_unique>(server, root_response_expression); + root_handler->attachStrictPath("/")->allowGetAndHeadRequest(); + factory->addHandler(root_handler.release()); + + auto ping_handler = std::make_unique>(server, ping_response_expression); + ping_handler->attachStrictPath("/ping")->allowGetAndHeadRequest(); + factory->addHandler(ping_handler.release()); + + auto replicas_status_handler = std::make_unique>(server); + replicas_status_handler->attachNonStrictPath("/replicas_status")->allowGetAndHeadRequest(); + factory->addHandler(replicas_status_handler.release()); + + auto query_handler = std::make_unique>(server, "query"); + query_handler->allowPostAndGetParamsRequest(); + factory->addHandler(query_handler.release()); if (server.config().has("prometheus") && server.config().getInt("prometheus.port", 0) == 0) - factory->addHandler((new HandlingRuleHTTPHandlerFactory( - server, PrometheusMetricsWriter(server.config(), "prometheus", async_metrics))) - ->attachStrictPath(server.config().getString("prometheus.endpoint", "/metrics"))->allowGetAndHeadRequest()); + { + auto prometheus_handler = std::make_unique>( + server, PrometheusMetricsWriter(server.config(), "prometheus", async_metrics)); + prometheus_handler->attachStrictPath(server.config().getString("prometheus.endpoint", "/metrics"))->allowGetAndHeadRequest(); + factory->addHandler(prometheus_handler.release()); + } - return factory; + return factory.release(); } } static inline Poco::Net::HTTPRequestHandlerFactory * createInterserverHTTPHandlerFactory(IServer & server, const std::string & name) { - return (new HTTPRequestHandlerFactoryMain(name)) - ->addHandler((new HandlingRuleHTTPHandlerFactory(server, root_response_expression)) - ->attachStrictPath("/")->allowGetAndHeadRequest()) - ->addHandler((new HandlingRuleHTTPHandlerFactory(server, ping_response_expression)) - ->attachStrictPath("/ping")->allowGetAndHeadRequest()) - ->addHandler((new HandlingRuleHTTPHandlerFactory(server)) - ->attachNonStrictPath("/replicas_status")->allowGetAndHeadRequest()) - ->addHandler((new HandlingRuleHTTPHandlerFactory(server))->allowPostAndGetParamsRequest()); + auto factory = std::make_unique(name); + + auto root_handler = std::make_unique>(server, root_response_expression); + root_handler->attachStrictPath("/")->allowGetAndHeadRequest(); + factory->addHandler(root_handler.release()); + + auto ping_handler = std::make_unique>(server, ping_response_expression); + ping_handler->attachStrictPath("/ping")->allowGetAndHeadRequest(); + factory->addHandler(ping_handler.release()); + + auto replicas_status_handler = std::make_unique>(server); + replicas_status_handler->attachNonStrictPath("/replicas_status")->allowGetAndHeadRequest(); + factory->addHandler(replicas_status_handler.release()); + + auto main_handler = std::make_unique>(server); + main_handler->allowPostAndGetParamsRequest(); + factory->addHandler(main_handler.release()); + + return factory.release(); } Poco::Net::HTTPRequestHandlerFactory * createHandlerFactory(IServer & server, AsynchronousMetrics & async_metrics, const std::string & name) @@ -155,9 +171,14 @@ Poco::Net::HTTPRequestHandlerFactory * createHandlerFactory(IServer & server, As else if (name == "InterserverIOHTTPHandler-factory" || name == "InterserverIOHTTPSHandler-factory") return createInterserverHTTPHandlerFactory(server, name); else if (name == "PrometheusHandler-factory") - return (new HTTPRequestHandlerFactoryMain(name))->addHandler((new HandlingRuleHTTPHandlerFactory( - server, PrometheusMetricsWriter(server.config(), "prometheus", async_metrics))) - ->attachStrictPath(server.config().getString("prometheus.endpoint", "/metrics"))->allowGetAndHeadRequest()); + { + auto factory = std::make_unique(name); + auto handler = std::make_unique>( + server, PrometheusMetricsWriter(server.config(), "prometheus", async_metrics)); + handler->attachStrictPath(server.config().getString("prometheus.endpoint", "/metrics"))->allowGetAndHeadRequest(); + factory->addHandler(handler.release()); + return factory.release(); + } throw Exception("LOGICAL ERROR: Unknown HTTP handler factory name.", ErrorCodes::LOGICAL_ERROR); } diff --git a/programs/server/ReplicasStatusHandler.cpp b/programs/server/ReplicasStatusHandler.cpp index 2f2aa5953b6..f2d1ffe2ee5 100644 --- a/programs/server/ReplicasStatusHandler.cpp +++ b/programs/server/ReplicasStatusHandler.cpp @@ -46,7 +46,7 @@ void ReplicasStatusHandler::handleRequest(Poco::Net::HTTPServerRequest & request for (auto iterator = db.second->getTablesIterator(); iterator->isValid(); iterator->next()) { - auto & table = iterator->table(); + const auto & table = iterator->table(); StorageReplicatedMergeTree * table_replicated = dynamic_cast(table.get()); if (!table_replicated) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 80e7e033525..2e71bc902e9 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -331,18 +331,13 @@ void ReplicatedMergeTreeQueue::updateTimesInZooKeeper( void ReplicatedMergeTreeQueue::removeProcessedEntry(zkutil::ZooKeeperPtr zookeeper, LogEntryPtr & entry) { - auto code = zookeeper->tryRemove(replica_path + "/queue/" + entry->znode_name); - - if (code) - LOG_ERROR(log, "Couldn't remove " << replica_path << "/queue/" << entry->znode_name << ": " - << zkutil::ZooKeeper::error2string(code) << ". This shouldn't happen often."); - std::optional min_unprocessed_insert_time_changed; std::optional max_processed_insert_time_changed; bool found = false; size_t queue_size = 0; + /// First remove from memory then from ZooKeeper { std::unique_lock lock(state_mutex); @@ -372,6 +367,11 @@ void ReplicatedMergeTreeQueue::removeProcessedEntry(zkutil::ZooKeeperPtr zookeep notifySubscribers(queue_size); + auto code = zookeeper->tryRemove(replica_path + "/queue/" + entry->znode_name); + if (code) + LOG_ERROR(log, "Couldn't remove " << replica_path << "/queue/" << entry->znode_name << ": " + << zkutil::ZooKeeper::error2string(code) << ". This shouldn't happen often."); + updateTimesInZooKeeper(zookeeper, min_unprocessed_insert_time_changed, max_processed_insert_time_changed); } diff --git a/utils/zookeeper-adjust-block-numbers-to-parts/main.cpp b/utils/zookeeper-adjust-block-numbers-to-parts/main.cpp index a896129f915..8550675cb9e 100644 --- a/utils/zookeeper-adjust-block-numbers-to-parts/main.cpp +++ b/utils/zookeeper-adjust-block-numbers-to-parts/main.cpp @@ -199,7 +199,7 @@ void setCurrentBlockNumber(zkutil::ZooKeeper & zk, const std::string & path, Int create_ephemeral_nodes(1); /// Firstly try to create just a single node. /// Create other nodes in batches of 50 nodes. - while (current_block_number + 50 <= new_current_block_number) + while (current_block_number + 50 <= new_current_block_number) // NOLINT: clang-tidy thinks that the loop is infinite create_ephemeral_nodes(50); create_ephemeral_nodes(new_current_block_number - current_block_number); From e7fa91c5ad2047a6c135c114faf69f0bb031967f Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Mon, 18 May 2020 11:17:43 +0300 Subject: [PATCH 365/738] Update playground.md --- docs/en/getting-started/playground.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/getting-started/playground.md b/docs/en/getting-started/playground.md index 36595d1998a..e80673f1b1f 100644 --- a/docs/en/getting-started/playground.md +++ b/docs/en/getting-started/playground.md @@ -50,7 +50,7 @@ HTTPS endpoint example with `curl`: curl "https://play-api.clickhouse.tech:8443/?query=SELECT+'Play+ClickHouse!';&user=playground&password=clickhouse&database=datasets" ``` -TCP endpoint example with [../interfaces/cli.md]: +TCP endpoint example with [CLI](../interfaces/cli.md): ``` bash clickhouse client --secure -h play-api.clickhouse.tech --port 9440 -u playground --password clickhouse -q "SELECT 'Play ClickHouse!'" ``` From ab491c0dd989ea6f51a39ab00ebdd8d15778262c Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 18 May 2020 11:42:39 +0300 Subject: [PATCH 366/738] boop CI --- src/Disks/createVolume.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Disks/createVolume.cpp b/src/Disks/createVolume.cpp index d7aa7d3c9ad..111f05b8db8 100644 --- a/src/Disks/createVolume.cpp +++ b/src/Disks/createVolume.cpp @@ -7,8 +7,8 @@ VolumePtr createVolumeFromReservation(const ReservationPtr & reservation, Volume { if (other_volume->getType() == VolumeType::JBOD || other_volume->getType() == VolumeType::SINGLE_DISK) { - // Since reservation on JBOD chosies one of disks and makes reservation there, volume - // for such type of reservation will be with one disk. + /// Since reservation on JBOD chosies one of disks and makes reservation there, volume + /// for such type of reservation will be with one disk. return std::make_shared(other_volume->getName(), reservation->getDisk()); } return nullptr; From 0395f59b73ab7ef8f59317b75cb94d03ecff20a8 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 18 May 2020 12:03:33 +0300 Subject: [PATCH 367/738] Just wait slightly more in test --- .../0_stateless/01076_parallel_alter_replicated_zookeeper.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh b/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh index b6522b12895..15bb851fc77 100755 --- a/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh +++ b/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh @@ -105,7 +105,7 @@ sleep 1 counter=0 while [[ $($CLICKHOUSE_CLIENT --query "select * from system.mutations where table like 'concurrent_mutate_mt_%' and is_done=0" 2>&1) ]]; do - if [ "$counter" -gt 20 ] + if [ "$counter" -gt 40 ] then break fi From 1ccd4fb9788e4f40cc774da560bda9bda17ce183 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 18 May 2020 12:24:48 +0300 Subject: [PATCH 368/738] Add missed drop in test --- .../0_stateless/00029_test_zookeeper_optimize_exception.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/00029_test_zookeeper_optimize_exception.sh b/tests/queries/0_stateless/00029_test_zookeeper_optimize_exception.sh index 7c55f59a3f5..0600da63af4 100755 --- a/tests/queries/0_stateless/00029_test_zookeeper_optimize_exception.sh +++ b/tests/queries/0_stateless/00029_test_zookeeper_optimize_exception.sh @@ -5,6 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS test_optimize_exception" +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS test_optimize_exception_replicated" ${CLICKHOUSE_CLIENT} --query="CREATE TABLE test_optimize_exception (date Date) ENGINE=MergeTree() PARTITION BY toYYYYMM(date) ORDER BY date" ${CLICKHOUSE_CLIENT} --query="CREATE TABLE test_optimize_exception_replicated (date Date) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test/optimize', 'r1') PARTITION BY toYYYYMM(date) ORDER BY date" @@ -23,4 +24,5 @@ echo `${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --server_logs_file=/dev/nu | grep -c 'Code: 388. DB::Exception: .* DB::Exception:.* Cannot select parts for optimization' ${CLICKHOUSE_CLIENT} --query="DROP TABLE test_optimize_exception NO DELAY" +${CLICKHOUSE_CLIENT} --query="DROP TABLE test_optimize_exception_replicated NO DELAY" sleep 1 From b4652625c72cdff34cb64ed8eafff8fa67b4a1dc Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 18 May 2020 13:02:52 +0300 Subject: [PATCH 369/738] Try to enable skipped tests --- .../compose/docker_compose_zookeeper.yml | 22 ++++++++++++++----- tests/integration/test_rename_column/test.py | 4 ---- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/docker/test/integration/compose/docker_compose_zookeeper.yml b/docker/test/integration/compose/docker_compose_zookeeper.yml index 284e4a855dd..974e329481c 100644 --- a/docker/test/integration/compose/docker_compose_zookeeper.yml +++ b/docker/test/integration/compose/docker_compose_zookeeper.yml @@ -1,25 +1,37 @@ version: '2.2' services: +# our compose version doesn't support inheritance :( so we copy-paste images +# configuration zoo1: image: zookeeper:3.4.12 restart: always environment: ZOO_TICK_TIME: 500 - ZOO_MY_ID: 1 ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888 - + ZOO_MY_ID: 1 + JVMFLAGS: -Dzookeeper.forceSync=no + tmpfs: + - /data + - /datalog zoo2: image: zookeeper:3.4.12 restart: always environment: ZOO_TICK_TIME: 500 - ZOO_MY_ID: 2 ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888 - + ZOO_MY_ID: 2 + JVMFLAGS: -Dzookeeper.forceSync=no + tmpfs: + - /data + - /datalog zoo3: image: zookeeper:3.4.12 restart: always environment: ZOO_TICK_TIME: 500 - ZOO_MY_ID: 3 ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888 + ZOO_MY_ID: 3 + JVMFLAGS: -Dzookeeper.forceSync=no + tmpfs: + - /data + - /datalog diff --git a/tests/integration/test_rename_column/test.py b/tests/integration/test_rename_column/test.py index 9fac783e712..029d140d0ed 100644 --- a/tests/integration/test_rename_column/test.py +++ b/tests/integration/test_rename_column/test.py @@ -299,7 +299,6 @@ def test_rename_with_parallel_insert(started_cluster): drop_table(nodes, table_name) -@pytest.mark.skip(reason="For unknown reason one of these tests kill Zookeeper") def test_rename_with_parallel_merges(started_cluster): table_name = "test_rename_with_parallel_merges" drop_table(nodes, table_name) @@ -337,7 +336,6 @@ def test_rename_with_parallel_merges(started_cluster): drop_table(nodes, table_name) -@pytest.mark.skip(reason="For unknown reason one of these tests kill Zookeeper") def test_rename_with_parallel_slow_insert(started_cluster): table_name = "test_rename_with_parallel_slow_insert" drop_table(nodes, table_name) @@ -499,7 +497,6 @@ def test_rename_with_parallel_ttl_delete(started_cluster): drop_table(nodes, table_name) -@pytest.mark.skip(reason="For unknown reason one of these tests kill Zookeeper") def test_rename_distributed(started_cluster): table_name = 'test_rename_distributed' try: @@ -516,7 +513,6 @@ def test_rename_distributed(started_cluster): drop_distributed_table(node1, table_name) -@pytest.mark.skip(reason="For unknown reason one of these tests kill Zookeeper") def test_rename_distributed_parallel_insert_and_select(started_cluster): table_name = 'test_rename_distributed_parallel_insert_and_select' try: From ca1bf06498617c24e21094aec943a2f50ad1586f Mon Sep 17 00:00:00 2001 From: Vasily Morozov Date: Tue, 12 May 2020 03:58:58 +0300 Subject: [PATCH 370/738] updatePermutation --- src/Columns/ColumnAggregateFunction.cpp | 2 + src/Columns/ColumnAggregateFunction.h | 1 + src/Columns/ColumnArray.cpp | 58 ++++++++++ src/Columns/ColumnArray.h | 1 + src/Columns/ColumnConst.cpp | 2 + src/Columns/ColumnConst.h | 1 + src/Columns/ColumnDecimal.cpp | 59 +++++++++++ src/Columns/ColumnDecimal.h | 1 + src/Columns/ColumnFixedString.cpp | 58 ++++++++++ src/Columns/ColumnFixedString.h | 2 + src/Columns/ColumnFunction.h | 5 + src/Columns/ColumnLowCardinality.cpp | 63 +++++++++++ src/Columns/ColumnLowCardinality.h | 2 + src/Columns/ColumnNullable.cpp | 70 ++++++++++++ src/Columns/ColumnNullable.h | 1 + src/Columns/ColumnString.cpp | 135 ++++++++++++++++++++++++ src/Columns/ColumnString.h | 4 + src/Columns/ColumnTuple.cpp | 12 +++ src/Columns/ColumnTuple.h | 1 + src/Columns/ColumnUnique.h | 26 +++++ src/Columns/ColumnVector.cpp | 60 +++++++++++ src/Columns/ColumnVector.h | 2 + src/Columns/IColumn.h | 4 + src/Columns/IColumnDummy.h | 2 + src/Interpreters/sortBlock.cpp | 43 ++++++-- 25 files changed, 604 insertions(+), 11 deletions(-) diff --git a/src/Columns/ColumnAggregateFunction.cpp b/src/Columns/ColumnAggregateFunction.cpp index 2f3a766b8f5..3b1f99bc5be 100644 --- a/src/Columns/ColumnAggregateFunction.cpp +++ b/src/Columns/ColumnAggregateFunction.cpp @@ -549,6 +549,8 @@ void ColumnAggregateFunction::getPermutation(bool /*reverse*/, size_t /*limit*/, res[i] = i; } +void ColumnAggregateFunction::updatePermutation(bool, size_t, int, Permutation &, EqualRanges&) const {} + void ColumnAggregateFunction::gather(ColumnGathererStream & gatherer) { gatherer.gather(*this); diff --git a/src/Columns/ColumnAggregateFunction.h b/src/Columns/ColumnAggregateFunction.h index f257351a4d0..2a692d889a4 100644 --- a/src/Columns/ColumnAggregateFunction.h +++ b/src/Columns/ColumnAggregateFunction.h @@ -193,6 +193,7 @@ public: } void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override; + void updatePermutation(bool reverse, size_t limit, int, Permutation & res, EqualRanges& equal_range) const override; /** More efficient manipulation methods */ Container & getData() diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index 7dba8e857cc..163185a7511 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -737,6 +737,64 @@ void ColumnArray::getPermutation(bool reverse, size_t limit, int nan_direction_h } } +void ColumnArray::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res, EqualRanges& equal_range) const +{ + if (limit >= size() || limit >= equal_range.back().second) { + limit = 0; + } + size_t n = equal_range.size(); + if (limit) + { + --n; + } + EqualRanges new_ranges; + for (size_t i = 0; i < n; ++i) { + const auto& [first, last] = equal_range[i]; + if (reverse) + std::sort(res.begin() + first, res.begin() + last, Less(*this, nan_direction_hint)); + else + std::sort(res.begin() + first, res.begin() + last, Less(*this, nan_direction_hint)); + auto new_first = first; + for (auto j = first + 1; j < last; ++j) { + if (compareAt(res[new_first], res[j], *this, nan_direction_hint) != 0) { + if (j - new_first > 1) { + new_ranges.emplace_back(new_first, j); + } + new_first = j; + } + } + if (last - new_first > 1) { + new_ranges.emplace_back(new_first, last); + } + } + if (limit) { + const auto& [first, last] = equal_range.back(); + if (reverse) + std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, Less(*this, nan_direction_hint)); + else + std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, Less(*this, nan_direction_hint)); + auto new_first = first; + for (auto j = first + 1; j < limit; ++j) { + if (compareAt(res[new_first], res[j], *this, nan_direction_hint) != 0) { + if (j - new_first > 1) { + new_ranges.emplace_back(new_first, j); + } + new_first = j; + } + } + auto new_last = limit; + for (auto j = limit; j < last; ++j) { + if (compareAt(res[new_first], res[j], *this, nan_direction_hint) == 0) { + std::swap(res[new_last], res[j]); + ++new_last; + } + } + if (new_last - new_first > 1) { + new_ranges.emplace_back(new_first, new_last); + } + } + equal_range = std::move(new_ranges); +} ColumnPtr ColumnArray::replicate(const Offsets & replicate_offsets) const { diff --git a/src/Columns/ColumnArray.h b/src/Columns/ColumnArray.h index 15a1d1bd91a..c21628a8cf2 100644 --- a/src/Columns/ColumnArray.h +++ b/src/Columns/ColumnArray.h @@ -73,6 +73,7 @@ public: template ColumnPtr indexImpl(const PaddedPODArray & indexes, size_t limit) const; int compareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const override; void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override; + void updatePermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res, EqualRanges& equal_range) const override; void reserve(size_t n) override; size_t byteSize() const override; size_t allocatedBytes() const override; diff --git a/src/Columns/ColumnConst.cpp b/src/Columns/ColumnConst.cpp index 63f520a4c05..546b909c09a 100644 --- a/src/Columns/ColumnConst.cpp +++ b/src/Columns/ColumnConst.cpp @@ -120,6 +120,8 @@ void ColumnConst::getPermutation(bool /*reverse*/, size_t /*limit*/, int /*nan_d res[i] = i; } +void ColumnConst::updatePermutation(bool, size_t, int, Permutation &, EqualRanges&) const {} + void ColumnConst::updateWeakHash32(WeakHash32 & hash) const { if (hash.getData().size() != s) diff --git a/src/Columns/ColumnConst.h b/src/Columns/ColumnConst.h index 560d4d63a10..aecc04c0e11 100644 --- a/src/Columns/ColumnConst.h +++ b/src/Columns/ColumnConst.h @@ -170,6 +170,7 @@ public: ColumnPtr permute(const Permutation & perm, size_t limit) const override; ColumnPtr index(const IColumn & indexes, size_t limit) const override; void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override; + void updatePermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res, EqualRanges& equal_range) const override; size_t byteSize() const override { diff --git a/src/Columns/ColumnDecimal.cpp b/src/Columns/ColumnDecimal.cpp index 5396389294a..d6f7dba92e1 100644 --- a/src/Columns/ColumnDecimal.cpp +++ b/src/Columns/ColumnDecimal.cpp @@ -108,6 +108,65 @@ void ColumnDecimal::getPermutation(bool reverse, size_t limit, int , IColumn: permutation(reverse, limit, res); } +template +void ColumnDecimal::updatePermutation(bool reverse, size_t limit, int, IColumn::Permutation & res, EqualRanges& equal_range) const +{ + if (limit >= data.size() || limit >= equal_range.back().second) { + limit = 0; + } + size_t n = equal_range.size(); + if (limit) { + --n; + } + EqualRanges new_ranges; + for (size_t i = 0; i < n; ++i) { + const auto& [first, last] = equal_range[i]; + if (reverse) + std::partial_sort(res.begin() + first, res.begin() + last, res.begin() + last, [this](size_t a, size_t b) { return data[a] > data[b]; }); + else + std::partial_sort(res.begin() + first, res.begin() + last, res.begin() + last, [this](size_t a, size_t b) { return data[a] < data[b]; }); + auto new_first = first; + for (auto j = first + 1; j < last; ++j) { + if (data[res[new_first]] != data[res[j]]) { + if (j - new_first > 1) { + new_ranges.emplace_back(new_first, j); + } + new_first = j; + } + } + if (last - new_first > 1) { + new_ranges.emplace_back(new_first, last); + } + } + if (limit) { + const auto& [first, last] = equal_range.back(); + if (reverse) + std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, [this](size_t a, size_t b) { return data[a] > data[b]; }); + else + std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, [this](size_t a, size_t b) { return data[a] > data[b]; }); + auto new_first = first; + for (auto j = first + 1; j < limit; ++j) { + if (data[res[new_first]] != data[res[j]]) { + if (j - new_first > 1) { + new_ranges.emplace_back(new_first, j); + } + new_first = j; + } + } + auto new_last = limit; + for (auto j = limit; j < last; ++j) { + if (data[res[new_first]] == data[res[j]]) { + std::swap(res[new_last], res[j]); + ++new_last; + } + } + if (new_last - new_first > 1) { + new_ranges.emplace_back(new_first, new_last); + } + } + equal_range = std::move(new_ranges); +} + template ColumnPtr ColumnDecimal::permute(const IColumn::Permutation & perm, size_t limit) const { diff --git a/src/Columns/ColumnDecimal.h b/src/Columns/ColumnDecimal.h index 1f56b7c4242..9c2e232568f 100644 --- a/src/Columns/ColumnDecimal.h +++ b/src/Columns/ColumnDecimal.h @@ -108,6 +108,7 @@ public: void updateWeakHash32(WeakHash32 & hash) const override; int compareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const override; void getPermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res) const override; + void updatePermutation(bool reverse, size_t limit, int, IColumn::Permutation & res, EqualRanges& equal_range) const override; MutableColumnPtr cloneResized(size_t size) const override; diff --git a/src/Columns/ColumnFixedString.cpp b/src/Columns/ColumnFixedString.cpp index 57ae4cbdedf..d16cac0a40e 100644 --- a/src/Columns/ColumnFixedString.cpp +++ b/src/Columns/ColumnFixedString.cpp @@ -162,6 +162,64 @@ void ColumnFixedString::getPermutation(bool reverse, size_t limit, int /*nan_dir } } +void ColumnFixedString::updatePermutation(bool reverse, size_t limit, int, Permutation & res, EqualRanges& equal_range) const { + if (limit >= size() || limit >= equal_range.back().second) + limit = 0; + + size_t k = equal_range.size(); + if (limit) { + --k; + } + EqualRanges new_ranges; + + for (size_t i = 0; i < k; ++i) { + const auto& [first, last] = equal_range[i]; + if (reverse) + std::sort(res.begin() + first, res.begin() + last, less(*this)); + else + std::sort(res.begin() + first, res.begin() + last, less(*this)); + auto new_first = first; + for (auto j = first + 1; j < last; ++j) { + if (memcmpSmallAllowOverflow15(chars.data() + j * n, chars.data() + new_first * n, n) != 0) { + if (j - new_first > 1) { + new_ranges.emplace_back(new_first, j); + } + new_first = j; + } + } + if (last - new_first > 1) { + new_ranges.emplace_back(new_first, last); + } + } + if (limit) { + const auto& [first, last] = equal_range.back(); + if (reverse) + std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less(*this)); + else + std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less(*this)); + auto new_first = first; + for (auto j = first + 1; j < limit; ++j) { + if (memcmpSmallAllowOverflow15(chars.data() + j * n, chars.data() + new_first * n, n) != 0) { + if (j - new_first > 1) { + new_ranges.emplace_back(new_first, j); + } + new_first = j; + } + } + auto new_last = limit; + for (auto j = limit; j < last; ++j) { + if (memcmpSmallAllowOverflow15(chars.data() + j * n, chars.data() + new_first * n, n) == 0) { + std::swap(res[new_last], res[j]); + ++new_last; + } + } + if (new_last - new_first > 1) { + new_ranges.emplace_back(new_first, new_last); + } + } + equal_range = std::move(new_ranges); +} + void ColumnFixedString::insertRangeFrom(const IColumn & src, size_t start, size_t length) { const ColumnFixedString & src_concrete = assert_cast(src); diff --git a/src/Columns/ColumnFixedString.h b/src/Columns/ColumnFixedString.h index 74c4f3c74f2..be78651f7cb 100644 --- a/src/Columns/ColumnFixedString.h +++ b/src/Columns/ColumnFixedString.h @@ -118,6 +118,8 @@ public: void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override; + void updatePermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res, EqualRanges& equal_range) const override; + void insertRangeFrom(const IColumn & src, size_t start, size_t length) override; ColumnPtr filter(const IColumn::Filter & filt, ssize_t result_size_hint) const override; diff --git a/src/Columns/ColumnFunction.h b/src/Columns/ColumnFunction.h index 1bde48559fe..7b3252e3379 100644 --- a/src/Columns/ColumnFunction.h +++ b/src/Columns/ColumnFunction.h @@ -121,6 +121,11 @@ public: throw Exception("getPermutation is not implemented for " + getName(), ErrorCodes::NOT_IMPLEMENTED); } + void updatePermutation(bool, size_t, int, Permutation &, EqualRanges&) const override + { + throw Exception("updatePermutation is not implemented for " + getName(), ErrorCodes::NOT_IMPLEMENTED); + } + void gather(ColumnGathererStream &) override { throw Exception("Method gather is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED); diff --git a/src/Columns/ColumnLowCardinality.cpp b/src/Columns/ColumnLowCardinality.cpp index d6f0df1d53a..e5b1e1990ea 100644 --- a/src/Columns/ColumnLowCardinality.cpp +++ b/src/Columns/ColumnLowCardinality.cpp @@ -314,6 +314,69 @@ void ColumnLowCardinality::getPermutation(bool reverse, size_t limit, int nan_di } } +void ColumnLowCardinality::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges& equal_range) const +{ + if (limit >= size() || limit >= equal_range.back().second) { + limit = 0; + } + size_t n = equal_range.size(); + if (limit) + { + --n; + } + EqualRanges new_ranges; + for (size_t i = 0; i < n; ++i) { + const auto& [first, last] = equal_range[i]; + if (reverse) + std::sort(res.begin() + first, res.begin() + last, [this, nan_direction_hint](size_t a, size_t b) { + return getDictionary().compareAt(getIndexes().getUInt(a), getIndexes().getUInt(b), getDictionary(), nan_direction_hint) > 0; }); + else + std::sort(res.begin() + first, res.begin() + last, [this, nan_direction_hint](size_t a, size_t b) { + return getDictionary().compareAt(getIndexes().getUInt(a), getIndexes().getUInt(b), getDictionary(), nan_direction_hint) < 0; }); + auto new_first = first; + for (auto j = first + 1; j < last; ++j) { + if (compareAt(new_first, j, *this, nan_direction_hint) != 0) { + if (j - new_first > 1) { + new_ranges.emplace_back(new_first, j); + } + new_first = j; + } + } + if (last - new_first > 1) { + new_ranges.emplace_back(new_first, last); + } + } + if (limit) { + const auto& [first, last] = equal_range.back(); + if (reverse) + std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, [this, nan_direction_hint](size_t a, size_t b) { + return getDictionary().compareAt(getIndexes().getUInt(a), getIndexes().getUInt(b), getDictionary(), nan_direction_hint) > 0; }); + else + std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, [this, nan_direction_hint](size_t a, size_t b) { + return getDictionary().compareAt(getIndexes().getUInt(a), getIndexes().getUInt(b), getDictionary(), nan_direction_hint) < 0; }); + auto new_first = first; + for (auto j = first + 1; j < limit; ++j) { + if (getDictionary().compareAt(getIndexes().getUInt(new_first), getIndexes().getUInt(j), getDictionary(), nan_direction_hint) != 0) { + if (j - new_first > 1) { + new_ranges.emplace_back(new_first, j); + } + new_first = j; + } + } + auto new_last = limit; + for (auto j = limit; j < last; ++j) { + if (getDictionary().compareAt(getIndexes().getUInt(new_first), getIndexes().getUInt(j), getDictionary(), nan_direction_hint) == 0) { + std::swap(res[new_last], res[j]); + ++new_last; + } + } + if (new_last - new_first > 1) { + new_ranges.emplace_back(new_first, new_last); + } + } + equal_range = std::move(new_ranges); +} + std::vector ColumnLowCardinality::scatter(ColumnIndex num_columns, const Selector & selector) const { auto columns = getIndexes().scatter(num_columns, selector); diff --git a/src/Columns/ColumnLowCardinality.h b/src/Columns/ColumnLowCardinality.h index e641cc177f3..4a5fe7b7517 100644 --- a/src/Columns/ColumnLowCardinality.h +++ b/src/Columns/ColumnLowCardinality.h @@ -111,6 +111,8 @@ public: void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override; + void updatePermutation(bool reverse, size_t limit, int, IColumn::Permutation & res, EqualRanges& equal_range) const override; + ColumnPtr replicate(const Offsets & offsets) const override { return ColumnLowCardinality::create(dictionary.getColumnUniquePtr(), getIndexes().replicate(offsets)); diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp index 55ce1401073..b26a43745cd 100644 --- a/src/Columns/ColumnNullable.cpp +++ b/src/Columns/ColumnNullable.cpp @@ -321,6 +321,76 @@ void ColumnNullable::getPermutation(bool reverse, size_t limit, int null_directi } } +void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_direction_hint, IColumn::Permutation & res, EqualRanges& equal_range) const { + if (limit >= equal_range.back().second || limit >= size()) { + limit = 0; + } + + EqualRanges new_ranges, temp_ranges; + + for (const auto &[first, last] : equal_range) { + bool direction = ((null_direction_hint > 0) != reverse); + /// Shift all NULL values to the end. + + size_t read_idx = first; + size_t write_idx = first; + while (read_idx < last && (isNullAt(res[read_idx])^direction)) + { + ++read_idx; + ++write_idx; + } + + ++read_idx; + + /// Invariants: + /// write_idx < read_idx + /// write_idx points to NULL + /// read_idx will be incremented to position of next not-NULL + /// there are range of NULLs between write_idx and read_idx - 1, + /// We are moving elements from end to begin of this range, + /// so range will "bubble" towards the end. + /// Relative order of NULL elements could be changed, + /// but relative order of non-NULLs is preserved. + + while (read_idx < last && write_idx < last) + { + if (isNullAt(res[read_idx])^direction) + { + std::swap(res[read_idx], res[write_idx]); + ++write_idx; + } + ++read_idx; + } + if (write_idx - first > 1) + { + if (direction) { + temp_ranges.emplace_back(first, write_idx); + } else { + new_ranges.emplace_back(first, write_idx); + } + } + if (last - write_idx > 1) + { + if (direction) + { + new_ranges.emplace_back(write_idx, last); + } else + { + temp_ranges.emplace_back(write_idx, last); + } + } + } + while (!new_ranges.empty() && limit && limit <= new_ranges.back().first) { + new_ranges.pop_back(); + } + if (!temp_ranges.empty()) + { + getNestedColumn().updatePermutation(reverse, limit, null_direction_hint, res, temp_ranges); + } + equal_range.resize(temp_ranges.size() + new_ranges.size()); + std::merge(temp_ranges.begin(), temp_ranges.end(), new_ranges.begin(), new_ranges.end(), equal_range.begin()); +} + void ColumnNullable::gather(ColumnGathererStream & gatherer) { gatherer.gather(*this); diff --git a/src/Columns/ColumnNullable.h b/src/Columns/ColumnNullable.h index 5443d8b0187..3c18fc44ffa 100644 --- a/src/Columns/ColumnNullable.h +++ b/src/Columns/ColumnNullable.h @@ -78,6 +78,7 @@ public: ColumnPtr index(const IColumn & indexes, size_t limit) const override; int compareAt(size_t n, size_t m, const IColumn & rhs_, int null_direction_hint) const override; void getPermutation(bool reverse, size_t limit, int null_direction_hint, Permutation & res) const override; + void updatePermutation(bool reverse, size_t limit, int, Permutation & res, EqualRanges& equal_range) const override; void reserve(size_t n) override; size_t byteSize() const override; size_t allocatedBytes() const override; diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp index 12f7a5632db..a341cc8b648 100644 --- a/src/Columns/ColumnString.cpp +++ b/src/Columns/ColumnString.cpp @@ -302,6 +302,74 @@ void ColumnString::getPermutation(bool reverse, size_t limit, int /*nan_directio } } +void ColumnString::updatePermutation(bool reverse, size_t limit, int /*nan_direction_hint*/, Permutation & res, EqualRanges &equal_range) const +{ + if (limit >= size() || limit > equal_range.back().second) + limit = 0; + + EqualRanges new_ranges; + auto less_true = less(*this); + auto less_false = less(*this); + size_t n = equal_range.size(); + if (limit) + --n; + + for (size_t i = 0; i < n; ++i) { + const auto &[first, last] = equal_range[i]; + if (reverse) + std::sort(res.begin() + first, res.begin() + last, less_false); + else + std::sort(res.begin() + first, res.begin() + last, less_true); + size_t new_first = first; + for (size_t j = first + 1; j < last; ++j) { + if (memcmpSmallAllowOverflow15( + chars.data() + offsetAt(res[j]), sizeAt(res[j]) - 1, + chars.data() + offsetAt(res[new_first]), sizeAt(res[new_first]) - 1) != 0) + { + if (j - new_first > 1) { + new_ranges.emplace_back(new_first, j); + } + new_first = j; + } + } + if (last - new_first > 1) { + new_ranges.emplace_back(new_first, last); + } + } + + if (limit) + { + const auto &[first, last] = equal_range.back(); + if (reverse) + std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less_false); + else + std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less_true); + size_t new_first = first; + for (size_t j = first + 1; j < limit; ++j) { + if (memcmpSmallAllowOverflow15( + chars.data() + offsetAt(res[j]), sizeAt(res[j]) - 1, + chars.data() + offsetAt(res[new_first]), sizeAt(res[new_first]) - 1) != 0) { + if (j - new_first > 1) { + new_ranges.emplace_back(new_first, j); + } + new_first = j; + } + } + size_t new_last = limit; + for (size_t j = limit; j < last; ++j) { + if (memcmpSmallAllowOverflow15( + chars.data() + offsetAt(res[j]), sizeAt(res[j]) - 1, + chars.data() + offsetAt(res[new_first]), sizeAt(res[new_first]) - 1) == 0) { + std::swap(res[j], res[new_last]); + ++new_last; + } + } + if (new_last - new_first > 1) { + new_ranges.emplace_back(new_first, new_last); + } + } + equal_range = std::move(new_ranges); +} ColumnPtr ColumnString::replicate(const Offsets & replicate_offsets) const { @@ -440,6 +508,73 @@ void ColumnString::getPermutationWithCollation(const Collator & collator, bool r } } +void ColumnString::updatePermutationWithCollation(const Collator & collator, bool reverse, size_t limit, int, Permutation &res, EqualRanges &equal_range) const +{ + if (limit >= size() || limit >= equal_range.back().second) { + limit = 0; + } + size_t n = equal_range.size(); + if (limit) + { + --n; + } + EqualRanges new_ranges; + for (size_t i = 0; i < n; ++i) { + const auto& [first, last] = equal_range[i]; + if (reverse) + std::sort(res.begin() + first, res.begin() + last, lessWithCollation(*this, collator)); + else + std::sort(res.begin() + first, res.begin() + last, lessWithCollation(*this, collator)); + auto new_first = first; + for (auto j = first + 1; j < last; ++j) { + if (collator.compare( + reinterpret_cast(&chars[offsetAt(res[new_first])]), sizeAt(res[new_first]), + reinterpret_cast(&chars[offsetAt(res[j])]), sizeAt(res[j])) != 0) + { + if (j - new_first > 1) { + new_ranges.emplace_back(new_first, j); + } + new_first = j; + } + } + if (last - new_first > 1) { + new_ranges.emplace_back(new_first, last); + } + } + if (limit) { + const auto& [first, last] = equal_range.back(); + if (reverse) + std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, lessWithCollation(*this, collator)); + else + std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, lessWithCollation(*this, collator)); + auto new_first = first; + for (auto j = first + 1; j < limit; ++j) { + if (collator.compare( + reinterpret_cast(&chars[offsetAt(res[new_first])]), sizeAt(res[new_first]), + reinterpret_cast(&chars[offsetAt(res[j])]), sizeAt(res[j])) != 0) + { + if (j - new_first > 1) { + new_ranges.emplace_back(new_first, j); + } + new_first = j; + } + } + auto new_last = limit; + for (auto j = limit; j < last; ++j) { + if (collator.compare( + reinterpret_cast(&chars[offsetAt(res[new_first])]), sizeAt(res[new_first]), + reinterpret_cast(&chars[offsetAt(res[j])]), sizeAt(res[j])) == 0) + { + std::swap(res[new_last], res[j]); + ++new_last; + } + } + if (new_last - new_first > 1) { + new_ranges.emplace_back(new_first, new_last); + } + } + equal_range = std::move(new_ranges); +} void ColumnString::protect() { diff --git a/src/Columns/ColumnString.h b/src/Columns/ColumnString.h index 32116880014..74803205a20 100644 --- a/src/Columns/ColumnString.h +++ b/src/Columns/ColumnString.h @@ -225,9 +225,13 @@ public: void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override; + void updatePermutation(bool reverse, size_t limit, int, Permutation & res, EqualRanges& equal_range) const override; + /// Sorting with respect of collation. void getPermutationWithCollation(const Collator & collator, bool reverse, size_t limit, Permutation & res) const; + void updatePermutationWithCollation(const Collator & collator, bool reverse, size_t limit, int, Permutation & res, EqualRanges& equal_range) const; + ColumnPtr replicate(const Offsets & replicate_offsets) const override; MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override diff --git a/src/Columns/ColumnTuple.cpp b/src/Columns/ColumnTuple.cpp index 59552c67f14..ae0d70036b0 100644 --- a/src/Columns/ColumnTuple.cpp +++ b/src/Columns/ColumnTuple.cpp @@ -329,6 +329,18 @@ void ColumnTuple::getPermutation(bool reverse, size_t limit, int nan_direction_h } } +void ColumnTuple::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges& equal_range) const { + for (const auto& column : columns) { + column->updatePermutation(reverse, limit, nan_direction_hint, res, equal_range); + while (limit && limit <= equal_range.back().first) { + equal_range.pop_back(); + } + if (equal_range.empty()) { + break; + } + } +} + void ColumnTuple::gather(ColumnGathererStream & gatherer) { gatherer.gather(*this); diff --git a/src/Columns/ColumnTuple.h b/src/Columns/ColumnTuple.h index 3533b602a1b..cceba49c4fa 100644 --- a/src/Columns/ColumnTuple.h +++ b/src/Columns/ColumnTuple.h @@ -72,6 +72,7 @@ public: int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override; void getExtremes(Field & min, Field & max) const override; void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override; + void updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges& equal_range) const override; void reserve(size_t n) override; size_t byteSize() const override; size_t allocatedBytes() const override; diff --git a/src/Columns/ColumnUnique.h b/src/Columns/ColumnUnique.h index da96e4a5ea2..883879f929b 100644 --- a/src/Columns/ColumnUnique.h +++ b/src/Columns/ColumnUnique.h @@ -77,6 +77,7 @@ public: } int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override; + void updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges& equal_range) const override; void getExtremes(Field & min, Field & max) const override { column_holder->getExtremes(min, max); } bool valuesHaveFixedSize() const override { return column_holder->valuesHaveFixedSize(); } @@ -374,6 +375,31 @@ int ColumnUnique::compareAt(size_t n, size_t m, const IColumn & rhs, return getNestedColumn()->compareAt(n, m, *column_unique.getNestedColumn(), nan_direction_hint); } +template +void ColumnUnique::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges& equal_range) const { + bool found_null_value_index = false; + for (size_t i = 0; i < equal_range.size() && !found_null_value_index; ++i){ + auto& [first, last] = equal_range[i]; + for (auto j = first; j < last; ++j) { + if (res[i] == getNullValueIndex()) { + if ((nan_direction_hint > 0) != reverse) { + std::swap(res[i], res[last - 1]); + --last; + } else { + std::swap(res[i], first); + ++first; + } + if (last - first <= 1) { + equal_range.erase(equal_range.begin() + i); + } + found_null_value_index = true; + break; + } + } + } + getNestedColumn()->updatePermutation(reverse, limit, nan_direction_hint, res, equal_range); +} + template static void checkIndexes(const ColumnVector & indexes, size_t max_dictionary_size) { diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 9b128fcffec..d12af0b5032 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -222,6 +222,66 @@ void ColumnVector::getPermutation(bool reverse, size_t limit, int nan_directi } } +template +void ColumnVector::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges &equal_range) const +{ + if (limit >= data.size() || limit >= equal_range.back().second) + limit = 0; + + EqualRanges new_ranges; + + for (size_t i = 0; i < equal_range.size() - bool(limit); ++i) + { + const auto & [first, last] = equal_range[i]; + if (reverse) + pdqsort(res.begin() + first, res.begin() + last, greater(*this, nan_direction_hint)); + else + pdqsort(res.begin() + first, res.begin() + last, less(*this, nan_direction_hint)); + size_t new_first = first; + for (size_t j = first + 1; j < last; ++j) { + if (less(*this, nan_direction_hint)(res[j], res[new_first]) || greater(*this, nan_direction_hint)(res[j], res[new_first])) + { + if (j - new_first > 1) { + new_ranges.emplace_back(new_first, j); + } + new_first = j; + } + } + if (last - new_first > 1) { + new_ranges.emplace_back(new_first, last); + } + } + if (limit) { + const auto & [first, last] = equal_range.back(); + if (reverse) + std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, greater(*this, nan_direction_hint)); + else + std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less(*this, nan_direction_hint)); + + size_t new_first = first; + for (size_t j = first + 1; j < limit; ++j) { + if (less(*this, nan_direction_hint)(res[j], res[new_first]) || greater(*this, nan_direction_hint)(res[j], res[new_first])) { + if (j - new_first > 1) { + new_ranges.emplace_back(new_first, j); + } + new_first = j; + } + } + + size_t new_last = limit; + for (size_t j = limit; j < last; ++j) { + if (!less(*this, nan_direction_hint)(res[j], res[new_first]) && !greater(*this, nan_direction_hint)(res[j], res[new_first])) { + std::swap(res[j], res[new_last]); + ++new_last; + } + } + if (new_last - new_first > 1) { + new_ranges.emplace_back(new_first, new_last); + } + } + equal_range = std::move(new_ranges); +} + template const char * ColumnVector::getFamilyName() const diff --git a/src/Columns/ColumnVector.h b/src/Columns/ColumnVector.h index 2fd177625cc..3fca2aec4a3 100644 --- a/src/Columns/ColumnVector.h +++ b/src/Columns/ColumnVector.h @@ -190,6 +190,8 @@ public: void getPermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res) const override; + void updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges& equal_range) const override; + void reserve(size_t n) override { data.reserve(n); diff --git a/src/Columns/IColumn.h b/src/Columns/IColumn.h index 496a0a5759b..dfddff5a718 100644 --- a/src/Columns/IColumn.h +++ b/src/Columns/IColumn.h @@ -25,6 +25,8 @@ class ColumnGathererStream; class Field; class WeakHash32; +typedef std::vector> EqualRanges; + /// Declares interface to store columns in memory. class IColumn : public COW { @@ -245,6 +247,8 @@ public: */ virtual void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const = 0; + virtual void updatePermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res, EqualRanges &equal_ranges) const = 0; + /** Copies each element according offsets parameter. * (i-th element should be copied offsets[i] - offsets[i - 1] times.) * It is necessary in ARRAY JOIN operation. diff --git a/src/Columns/IColumnDummy.h b/src/Columns/IColumnDummy.h index 00604fb87d0..f0d0981e82a 100644 --- a/src/Columns/IColumnDummy.h +++ b/src/Columns/IColumnDummy.h @@ -107,6 +107,8 @@ public: res[i] = i; } + void updatePermutation(bool, size_t, int, Permutation & res, EqualRanges&) const override {} + ColumnPtr replicate(const Offsets & offsets) const override { if (s != offsets.size()) diff --git a/src/Interpreters/sortBlock.cpp b/src/Interpreters/sortBlock.cpp index 0e98dc0eb4b..d982086d0b6 100644 --- a/src/Interpreters/sortBlock.cpp +++ b/src/Interpreters/sortBlock.cpp @@ -102,6 +102,7 @@ struct PartialSortingLessWithCollation } }; + void sortBlock(Block & block, const SortDescription & description, UInt64 limit) { if (!block) @@ -187,21 +188,41 @@ void sortBlock(Block & block, const SortDescription & description, UInt64 limit) if (need_collation) { - PartialSortingLessWithCollation less_with_collation(columns_with_sort_desc); + EqualRanges ranges; + ranges.emplace_back(0, perm.size()); + for (const auto& column : columns_with_sort_desc) + { + while (!ranges.empty() && limit && limit <= ranges.back().first) { + ranges.pop_back(); + } + if (ranges.empty()) { + break; + } - if (limit) - std::partial_sort(perm.begin(), perm.begin() + limit, perm.end(), less_with_collation); - else - pdqsort(perm.begin(), perm.end(), less_with_collation); + if (isCollationRequired(column.description)) { + const ColumnString & column_string = assert_cast(*column.column); + column_string.updatePermutationWithCollation(*column.description.collator, column.description.direction < 0, limit, column.description.nulls_direction, perm, ranges); + } + else + { + column.column->updatePermutation( + column.description.direction < 0, limit, column.description.nulls_direction, perm, ranges); + } + } } else { - PartialSortingLess less(columns_with_sort_desc); - - if (limit) - std::partial_sort(perm.begin(), perm.begin() + limit, perm.end(), less); - else - pdqsort(perm.begin(), perm.end(), less); + EqualRanges ranges; + ranges.emplace_back(0, perm.size()); + for (const auto& column : columns_with_sort_desc) + { + while (!ranges.empty() && limit && limit <= ranges.back().first) { + ranges.pop_back(); + } + if (ranges.empty()) { + break; + } + } } size_t columns = block.columns(); From 82af700eb8c8479a425bf423bc6824640654e1b8 Mon Sep 17 00:00:00 2001 From: Vasily Morozov Date: Tue, 12 May 2020 04:50:42 +0300 Subject: [PATCH 371/738] updatePermutation fix --- src/Columns/ColumnUnique.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Columns/ColumnUnique.h b/src/Columns/ColumnUnique.h index 883879f929b..c1e06475b94 100644 --- a/src/Columns/ColumnUnique.h +++ b/src/Columns/ColumnUnique.h @@ -381,12 +381,12 @@ void ColumnUnique::updatePermutation(bool reverse, size_t limit, int for (size_t i = 0; i < equal_range.size() && !found_null_value_index; ++i){ auto& [first, last] = equal_range[i]; for (auto j = first; j < last; ++j) { - if (res[i] == getNullValueIndex()) { + if (res[j] == getNullValueIndex()) { if ((nan_direction_hint > 0) != reverse) { - std::swap(res[i], res[last - 1]); + std::swap(res[j], res[last - 1]); --last; } else { - std::swap(res[i], first); + std::swap(res[j], res[first]); ++first; } if (last - first <= 1) { From 876fff193867e70efb262cd95dfd75733a614744 Mon Sep 17 00:00:00 2001 From: Vasily Morozov Date: Fri, 15 May 2020 00:00:56 +0300 Subject: [PATCH 372/738] fix build and style --- src/Columns/ColumnArray.cpp | 39 ++++++++++++------ src/Columns/ColumnDecimal.cpp | 42 ++++++++++++------- src/Columns/ColumnFixedString.cpp | 42 ++++++++++++------- src/Columns/ColumnLowCardinality.cpp | 53 +++++++++++++++--------- src/Columns/ColumnNullable.cpp | 19 ++++++--- src/Columns/ColumnString.cpp | 60 ++++++++++++++++++---------- src/Columns/ColumnTuple.cpp | 12 ++++-- src/Columns/ColumnUnique.h | 22 ++++++---- src/Columns/ColumnVector.cpp | 30 +++++++++----- src/Columns/IColumnDummy.h | 2 +- src/Interpreters/sortBlock.cpp | 17 +++++--- 11 files changed, 225 insertions(+), 113 deletions(-) diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index 163185a7511..d7f2964b13b 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -739,7 +739,8 @@ void ColumnArray::getPermutation(bool reverse, size_t limit, int nan_direction_h void ColumnArray::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res, EqualRanges& equal_range) const { - if (limit >= size() || limit >= equal_range.back().second) { + if (limit >= size() || limit >= equal_range.back().second) + { limit = 0; } size_t n = equal_range.size(); @@ -748,48 +749,60 @@ void ColumnArray::updatePermutation(bool reverse, size_t limit, int nan_directio --n; } EqualRanges new_ranges; - for (size_t i = 0; i < n; ++i) { + for (size_t i = 0; i < n; ++i) + { const auto& [first, last] = equal_range[i]; if (reverse) std::sort(res.begin() + first, res.begin() + last, Less(*this, nan_direction_hint)); else std::sort(res.begin() + first, res.begin() + last, Less(*this, nan_direction_hint)); auto new_first = first; - for (auto j = first + 1; j < last; ++j) { - if (compareAt(res[new_first], res[j], *this, nan_direction_hint) != 0) { - if (j - new_first > 1) { + for (auto j = first + 1; j < last; ++j) + { + if (compareAt(res[new_first], res[j], *this, nan_direction_hint) != 0) + { + if (j - new_first > 1) + { new_ranges.emplace_back(new_first, j); } new_first = j; } } - if (last - new_first > 1) { + if (last - new_first > 1) + { new_ranges.emplace_back(new_first, last); } } - if (limit) { + if (limit) + { const auto& [first, last] = equal_range.back(); if (reverse) std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, Less(*this, nan_direction_hint)); else std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, Less(*this, nan_direction_hint)); auto new_first = first; - for (auto j = first + 1; j < limit; ++j) { - if (compareAt(res[new_first], res[j], *this, nan_direction_hint) != 0) { - if (j - new_first > 1) { + for (auto j = first + 1; j < limit; ++j) + { + if (compareAt(res[new_first], res[j], *this, nan_direction_hint) != 0) + { + if (j - new_first > 1) + { new_ranges.emplace_back(new_first, j); } new_first = j; } } auto new_last = limit; - for (auto j = limit; j < last; ++j) { - if (compareAt(res[new_first], res[j], *this, nan_direction_hint) == 0) { + for (auto j = limit; j < last; ++j) + { + if (compareAt(res[new_first], res[j], *this, nan_direction_hint) == 0) + { std::swap(res[new_last], res[j]); ++new_last; } } - if (new_last - new_first > 1) { + if (new_last - new_first > 1) + { new_ranges.emplace_back(new_first, new_last); } } diff --git a/src/Columns/ColumnDecimal.cpp b/src/Columns/ColumnDecimal.cpp index d6f7dba92e1..94f20e845fd 100644 --- a/src/Columns/ColumnDecimal.cpp +++ b/src/Columns/ColumnDecimal.cpp @@ -111,56 +111,70 @@ void ColumnDecimal::getPermutation(bool reverse, size_t limit, int , IColumn: template void ColumnDecimal::updatePermutation(bool reverse, size_t limit, int, IColumn::Permutation & res, EqualRanges& equal_range) const { - if (limit >= data.size() || limit >= equal_range.back().second) { + if (limit >= data.size() || limit >= equal_range.back().second) + { limit = 0; } size_t n = equal_range.size(); - if (limit) { + if (limit) + { --n; } EqualRanges new_ranges; - for (size_t i = 0; i < n; ++i) { + for (size_t i = 0; i < n; ++i) + { const auto& [first, last] = equal_range[i]; if (reverse) std::partial_sort(res.begin() + first, res.begin() + last, res.begin() + last, [this](size_t a, size_t b) { return data[a] > data[b]; }); else std::partial_sort(res.begin() + first, res.begin() + last, res.begin() + last, [this](size_t a, size_t b) { return data[a] < data[b]; }); auto new_first = first; - for (auto j = first + 1; j < last; ++j) { - if (data[res[new_first]] != data[res[j]]) { - if (j - new_first > 1) { + for (auto j = first + 1; j < last; ++j) + { + if (data[res[new_first]] != data[res[j]]) + { + if (j - new_first > 1) + { new_ranges.emplace_back(new_first, j); } new_first = j; } } - if (last - new_first > 1) { + if (last - new_first > 1) + { new_ranges.emplace_back(new_first, last); } } - if (limit) { + if (limit) + { const auto& [first, last] = equal_range.back(); if (reverse) std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, [this](size_t a, size_t b) { return data[a] > data[b]; }); else std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, [this](size_t a, size_t b) { return data[a] > data[b]; }); auto new_first = first; - for (auto j = first + 1; j < limit; ++j) { - if (data[res[new_first]] != data[res[j]]) { - if (j - new_first > 1) { + for (auto j = first + 1; j < limit; ++j) + { + if (data[res[new_first]] != data[res[j]]) + { + if (j - new_first > 1) + { new_ranges.emplace_back(new_first, j); } new_first = j; } } auto new_last = limit; - for (auto j = limit; j < last; ++j) { - if (data[res[new_first]] == data[res[j]]) { + for (auto j = limit; j < last; ++j) + { + if (data[res[new_first]] == data[res[j]]) + { std::swap(res[new_last], res[j]); ++new_last; } } - if (new_last - new_first > 1) { + if (new_last - new_first > 1) + { new_ranges.emplace_back(new_first, new_last); } } diff --git a/src/Columns/ColumnFixedString.cpp b/src/Columns/ColumnFixedString.cpp index d16cac0a40e..52176925ae0 100644 --- a/src/Columns/ColumnFixedString.cpp +++ b/src/Columns/ColumnFixedString.cpp @@ -162,58 +162,72 @@ void ColumnFixedString::getPermutation(bool reverse, size_t limit, int /*nan_dir } } -void ColumnFixedString::updatePermutation(bool reverse, size_t limit, int, Permutation & res, EqualRanges& equal_range) const { +void ColumnFixedString::updatePermutation(bool reverse, size_t limit, int, Permutation & res, EqualRanges& equal_range) const +{ if (limit >= size() || limit >= equal_range.back().second) limit = 0; size_t k = equal_range.size(); - if (limit) { + if (limit) + { --k; } EqualRanges new_ranges; - for (size_t i = 0; i < k; ++i) { + for (size_t i = 0; i < k; ++i) + { const auto& [first, last] = equal_range[i]; if (reverse) std::sort(res.begin() + first, res.begin() + last, less(*this)); else std::sort(res.begin() + first, res.begin() + last, less(*this)); auto new_first = first; - for (auto j = first + 1; j < last; ++j) { - if (memcmpSmallAllowOverflow15(chars.data() + j * n, chars.data() + new_first * n, n) != 0) { - if (j - new_first > 1) { + for (auto j = first + 1; j < last; ++j) + { + if (memcmpSmallAllowOverflow15(chars.data() + j * n, chars.data() + new_first * n, n) != 0) + { + if (j - new_first > 1) + { new_ranges.emplace_back(new_first, j); } new_first = j; } } - if (last - new_first > 1) { + if (last - new_first > 1) + { new_ranges.emplace_back(new_first, last); } } - if (limit) { + if (limit) + { const auto& [first, last] = equal_range.back(); if (reverse) std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less(*this)); else std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less(*this)); auto new_first = first; - for (auto j = first + 1; j < limit; ++j) { - if (memcmpSmallAllowOverflow15(chars.data() + j * n, chars.data() + new_first * n, n) != 0) { - if (j - new_first > 1) { + for (auto j = first + 1; j < limit; ++j) + { + if (memcmpSmallAllowOverflow15(chars.data() + j * n, chars.data() + new_first * n, n) != 0) + { + if (j - new_first > 1) + { new_ranges.emplace_back(new_first, j); } new_first = j; } } auto new_last = limit; - for (auto j = limit; j < last; ++j) { - if (memcmpSmallAllowOverflow15(chars.data() + j * n, chars.data() + new_first * n, n) == 0) { + for (auto j = limit; j < last; ++j) + { + if (memcmpSmallAllowOverflow15(chars.data() + j * n, chars.data() + new_first * n, n) == 0) + { std::swap(res[new_last], res[j]); ++new_last; } } - if (new_last - new_first > 1) { + if (new_last - new_first > 1) + { new_ranges.emplace_back(new_first, new_last); } } diff --git a/src/Columns/ColumnLowCardinality.cpp b/src/Columns/ColumnLowCardinality.cpp index e5b1e1990ea..0cf0e6f9be6 100644 --- a/src/Columns/ColumnLowCardinality.cpp +++ b/src/Columns/ColumnLowCardinality.cpp @@ -316,7 +316,8 @@ void ColumnLowCardinality::getPermutation(bool reverse, size_t limit, int nan_di void ColumnLowCardinality::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges& equal_range) const { - if (limit >= size() || limit >= equal_range.back().second) { + if (limit >= size() || limit >= equal_range.back().second) + { limit = 0; } size_t n = equal_range.size(); @@ -325,52 +326,66 @@ void ColumnLowCardinality::updatePermutation(bool reverse, size_t limit, int nan --n; } EqualRanges new_ranges; - for (size_t i = 0; i < n; ++i) { + for (size_t i = 0; i < n; ++i) + { const auto& [first, last] = equal_range[i]; if (reverse) - std::sort(res.begin() + first, res.begin() + last, [this, nan_direction_hint](size_t a, size_t b) { + std::sort(res.begin() + first, res.begin() + last, [this, nan_direction_hint](size_t a, size_t b) + { return getDictionary().compareAt(getIndexes().getUInt(a), getIndexes().getUInt(b), getDictionary(), nan_direction_hint) > 0; }); else - std::sort(res.begin() + first, res.begin() + last, [this, nan_direction_hint](size_t a, size_t b) { + std::sort(res.begin() + first, res.begin() + last, [this, nan_direction_hint](size_t a, size_t b) + { return getDictionary().compareAt(getIndexes().getUInt(a), getIndexes().getUInt(b), getDictionary(), nan_direction_hint) < 0; }); auto new_first = first; - for (auto j = first + 1; j < last; ++j) { - if (compareAt(new_first, j, *this, nan_direction_hint) != 0) { - if (j - new_first > 1) { + for (auto j = first + 1; j < last; ++j) + { + if (compareAt(new_first, j, *this, nan_direction_hint) != 0) + { + if (j - new_first > 1) + { new_ranges.emplace_back(new_first, j); } new_first = j; } } - if (last - new_first > 1) { + if (last - new_first > 1) + { new_ranges.emplace_back(new_first, last); } } - if (limit) { + if (limit) + { const auto& [first, last] = equal_range.back(); if (reverse) - std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, [this, nan_direction_hint](size_t a, size_t b) { - return getDictionary().compareAt(getIndexes().getUInt(a), getIndexes().getUInt(b), getDictionary(), nan_direction_hint) > 0; }); + std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, [this, nan_direction_hint](size_t a, size_t b) + {return getDictionary().compareAt(getIndexes().getUInt(a), getIndexes().getUInt(b), getDictionary(), nan_direction_hint) > 0; }); else - std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, [this, nan_direction_hint](size_t a, size_t b) { - return getDictionary().compareAt(getIndexes().getUInt(a), getIndexes().getUInt(b), getDictionary(), nan_direction_hint) < 0; }); + std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, [this, nan_direction_hint](size_t a, size_t b) + {return getDictionary().compareAt(getIndexes().getUInt(a), getIndexes().getUInt(b), getDictionary(), nan_direction_hint) < 0; }); auto new_first = first; - for (auto j = first + 1; j < limit; ++j) { - if (getDictionary().compareAt(getIndexes().getUInt(new_first), getIndexes().getUInt(j), getDictionary(), nan_direction_hint) != 0) { - if (j - new_first > 1) { + for (auto j = first + 1; j < limit; ++j) + { + if (getDictionary().compareAt(getIndexes().getUInt(new_first), getIndexes().getUInt(j), getDictionary(), nan_direction_hint) != 0) + { + if (j - new_first > 1) + { new_ranges.emplace_back(new_first, j); } new_first = j; } } auto new_last = limit; - for (auto j = limit; j < last; ++j) { - if (getDictionary().compareAt(getIndexes().getUInt(new_first), getIndexes().getUInt(j), getDictionary(), nan_direction_hint) == 0) { + for (auto j = limit; j < last; ++j) + { + if (getDictionary().compareAt(getIndexes().getUInt(new_first), getIndexes().getUInt(j), getDictionary(), nan_direction_hint) == 0) + { std::swap(res[new_last], res[j]); ++new_last; } } - if (new_last - new_first > 1) { + if (new_last - new_first > 1) + { new_ranges.emplace_back(new_first, new_last); } } diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp index b26a43745cd..0a2986160e3 100644 --- a/src/Columns/ColumnNullable.cpp +++ b/src/Columns/ColumnNullable.cpp @@ -321,14 +321,17 @@ void ColumnNullable::getPermutation(bool reverse, size_t limit, int null_directi } } -void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_direction_hint, IColumn::Permutation & res, EqualRanges& equal_range) const { - if (limit >= equal_range.back().second || limit >= size()) { +void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_direction_hint, IColumn::Permutation & res, EqualRanges& equal_range) const +{ + if (limit >= equal_range.back().second || limit >= size()) + { limit = 0; } EqualRanges new_ranges, temp_ranges; - for (const auto &[first, last] : equal_range) { + for (const auto &[first, last] : equal_range) + { bool direction = ((null_direction_hint > 0) != reverse); /// Shift all NULL values to the end. @@ -363,9 +366,12 @@ void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_dire } if (write_idx - first > 1) { - if (direction) { + if (direction) + { temp_ranges.emplace_back(first, write_idx); - } else { + } + else + { new_ranges.emplace_back(first, write_idx); } } @@ -380,7 +386,8 @@ void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_dire } } } - while (!new_ranges.empty() && limit && limit <= new_ranges.back().first) { + while (!new_ranges.empty() && limit && limit <= new_ranges.back().first) + { new_ranges.pop_back(); } if (!temp_ranges.empty()) diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp index a341cc8b648..5a386856c93 100644 --- a/src/Columns/ColumnString.cpp +++ b/src/Columns/ColumnString.cpp @@ -314,25 +314,29 @@ void ColumnString::updatePermutation(bool reverse, size_t limit, int /*nan_direc if (limit) --n; - for (size_t i = 0; i < n; ++i) { + for (size_t i = 0; i < n; ++i) + { const auto &[first, last] = equal_range[i]; if (reverse) std::sort(res.begin() + first, res.begin() + last, less_false); else std::sort(res.begin() + first, res.begin() + last, less_true); size_t new_first = first; - for (size_t j = first + 1; j < last; ++j) { + for (size_t j = first + 1; j < last; ++j) + { if (memcmpSmallAllowOverflow15( chars.data() + offsetAt(res[j]), sizeAt(res[j]) - 1, chars.data() + offsetAt(res[new_first]), sizeAt(res[new_first]) - 1) != 0) { - if (j - new_first > 1) { + if (j - new_first > 1) + { new_ranges.emplace_back(new_first, j); } new_first = j; } } - if (last - new_first > 1) { + if (last - new_first > 1) + { new_ranges.emplace_back(new_first, last); } } @@ -345,26 +349,32 @@ void ColumnString::updatePermutation(bool reverse, size_t limit, int /*nan_direc else std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less_true); size_t new_first = first; - for (size_t j = first + 1; j < limit; ++j) { + for (size_t j = first + 1; j < limit; ++j) + { if (memcmpSmallAllowOverflow15( chars.data() + offsetAt(res[j]), sizeAt(res[j]) - 1, - chars.data() + offsetAt(res[new_first]), sizeAt(res[new_first]) - 1) != 0) { - if (j - new_first > 1) { + chars.data() + offsetAt(res[new_first]), sizeAt(res[new_first]) - 1) != 0) + { + if (j - new_first > 1) + { new_ranges.emplace_back(new_first, j); } new_first = j; } } size_t new_last = limit; - for (size_t j = limit; j < last; ++j) { + for (size_t j = limit; j < last; ++j) + { if (memcmpSmallAllowOverflow15( chars.data() + offsetAt(res[j]), sizeAt(res[j]) - 1, - chars.data() + offsetAt(res[new_first]), sizeAt(res[new_first]) - 1) == 0) { + chars.data() + offsetAt(res[new_first]), sizeAt(res[new_first]) - 1) == 0) + { std::swap(res[j], res[new_last]); ++new_last; } } - if (new_last - new_first > 1) { + if (new_last - new_first > 1) + { new_ranges.emplace_back(new_first, new_last); } } @@ -510,7 +520,8 @@ void ColumnString::getPermutationWithCollation(const Collator & collator, bool r void ColumnString::updatePermutationWithCollation(const Collator & collator, bool reverse, size_t limit, int, Permutation &res, EqualRanges &equal_range) const { - if (limit >= size() || limit >= equal_range.back().second) { + if (limit >= size() || limit >= equal_range.back().second) + { limit = 0; } size_t n = equal_range.size(); @@ -519,48 +530,56 @@ void ColumnString::updatePermutationWithCollation(const Collator & collator, boo --n; } EqualRanges new_ranges; - for (size_t i = 0; i < n; ++i) { + for (size_t i = 0; i < n; ++i) + { const auto& [first, last] = equal_range[i]; if (reverse) std::sort(res.begin() + first, res.begin() + last, lessWithCollation(*this, collator)); else std::sort(res.begin() + first, res.begin() + last, lessWithCollation(*this, collator)); auto new_first = first; - for (auto j = first + 1; j < last; ++j) { + for (auto j = first + 1; j < last; ++j) + { if (collator.compare( reinterpret_cast(&chars[offsetAt(res[new_first])]), sizeAt(res[new_first]), reinterpret_cast(&chars[offsetAt(res[j])]), sizeAt(res[j])) != 0) { - if (j - new_first > 1) { + if (j - new_first > 1) + { new_ranges.emplace_back(new_first, j); } new_first = j; } } - if (last - new_first > 1) { + if (last - new_first > 1) + { new_ranges.emplace_back(new_first, last); } } - if (limit) { + if (limit) + { const auto& [first, last] = equal_range.back(); if (reverse) std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, lessWithCollation(*this, collator)); else std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, lessWithCollation(*this, collator)); auto new_first = first; - for (auto j = first + 1; j < limit; ++j) { + for (auto j = first + 1; j < limit; ++j) + { if (collator.compare( reinterpret_cast(&chars[offsetAt(res[new_first])]), sizeAt(res[new_first]), reinterpret_cast(&chars[offsetAt(res[j])]), sizeAt(res[j])) != 0) { - if (j - new_first > 1) { + if (j - new_first > 1) + { new_ranges.emplace_back(new_first, j); } new_first = j; } } auto new_last = limit; - for (auto j = limit; j < last; ++j) { + for (auto j = limit; j < last; ++j) + { if (collator.compare( reinterpret_cast(&chars[offsetAt(res[new_first])]), sizeAt(res[new_first]), reinterpret_cast(&chars[offsetAt(res[j])]), sizeAt(res[j])) == 0) @@ -569,7 +588,8 @@ void ColumnString::updatePermutationWithCollation(const Collator & collator, boo ++new_last; } } - if (new_last - new_first > 1) { + if (new_last - new_first > 1) + { new_ranges.emplace_back(new_first, new_last); } } diff --git a/src/Columns/ColumnTuple.cpp b/src/Columns/ColumnTuple.cpp index ae0d70036b0..2e99c378e00 100644 --- a/src/Columns/ColumnTuple.cpp +++ b/src/Columns/ColumnTuple.cpp @@ -329,13 +329,17 @@ void ColumnTuple::getPermutation(bool reverse, size_t limit, int nan_direction_h } } -void ColumnTuple::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges& equal_range) const { - for (const auto& column : columns) { +void ColumnTuple::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges& equal_range) const +{ + for (const auto& column : columns) + { column->updatePermutation(reverse, limit, nan_direction_hint, res, equal_range); - while (limit && limit <= equal_range.back().first) { + while (limit && limit <= equal_range.back().first) + { equal_range.pop_back(); } - if (equal_range.empty()) { + if (equal_range.empty()) + { break; } } diff --git a/src/Columns/ColumnUnique.h b/src/Columns/ColumnUnique.h index c1e06475b94..b5c32ade9dc 100644 --- a/src/Columns/ColumnUnique.h +++ b/src/Columns/ColumnUnique.h @@ -376,20 +376,28 @@ int ColumnUnique::compareAt(size_t n, size_t m, const IColumn & rhs, } template -void ColumnUnique::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges& equal_range) const { +void ColumnUnique::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges& equal_range) const +{ bool found_null_value_index = false; - for (size_t i = 0; i < equal_range.size() && !found_null_value_index; ++i){ + for (size_t i = 0; i < equal_range.size() && !found_null_value_index; ++i) + { auto& [first, last] = equal_range[i]; - for (auto j = first; j < last; ++j) { - if (res[j] == getNullValueIndex()) { - if ((nan_direction_hint > 0) != reverse) { + for (auto j = first; j < last; ++j) + { + if (res[j] == getNullValueIndex()) + { + if ((nan_direction_hint > 0) != reverse) + { std::swap(res[j], res[last - 1]); --last; - } else { + } + else + { std::swap(res[j], res[first]); ++first; } - if (last - first <= 1) { + if (last - first <= 1) + { equal_range.erase(equal_range.begin() + i); } found_null_value_index = true; diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index d12af0b5032..c0b97fec499 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -238,20 +238,24 @@ void ColumnVector::updatePermutation(bool reverse, size_t limit, int nan_dire else pdqsort(res.begin() + first, res.begin() + last, less(*this, nan_direction_hint)); size_t new_first = first; - for (size_t j = first + 1; j < last; ++j) { + for (size_t j = first + 1; j < last; ++j) + { if (less(*this, nan_direction_hint)(res[j], res[new_first]) || greater(*this, nan_direction_hint)(res[j], res[new_first])) { - if (j - new_first > 1) { + if (j - new_first > 1) + { new_ranges.emplace_back(new_first, j); } new_first = j; } } - if (last - new_first > 1) { + if (last - new_first > 1) + { new_ranges.emplace_back(new_first, last); } } - if (limit) { + if (limit) + { const auto & [first, last] = equal_range.back(); if (reverse) std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, greater(*this, nan_direction_hint)); @@ -259,9 +263,12 @@ void ColumnVector::updatePermutation(bool reverse, size_t limit, int nan_dire std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less(*this, nan_direction_hint)); size_t new_first = first; - for (size_t j = first + 1; j < limit; ++j) { - if (less(*this, nan_direction_hint)(res[j], res[new_first]) || greater(*this, nan_direction_hint)(res[j], res[new_first])) { - if (j - new_first > 1) { + for (size_t j = first + 1; j < limit; ++j) + { + if (less(*this, nan_direction_hint)(res[j], res[new_first]) || greater(*this, nan_direction_hint)(res[j], res[new_first])) + { + if (j - new_first > 1) + { new_ranges.emplace_back(new_first, j); } new_first = j; @@ -269,13 +276,16 @@ void ColumnVector::updatePermutation(bool reverse, size_t limit, int nan_dire } size_t new_last = limit; - for (size_t j = limit; j < last; ++j) { - if (!less(*this, nan_direction_hint)(res[j], res[new_first]) && !greater(*this, nan_direction_hint)(res[j], res[new_first])) { + for (size_t j = limit; j < last; ++j) + { + if (!less(*this, nan_direction_hint)(res[j], res[new_first]) && !greater(*this, nan_direction_hint)(res[j], res[new_first])) + { std::swap(res[j], res[new_last]); ++new_last; } } - if (new_last - new_first > 1) { + if (new_last - new_first > 1) + { new_ranges.emplace_back(new_first, new_last); } } diff --git a/src/Columns/IColumnDummy.h b/src/Columns/IColumnDummy.h index f0d0981e82a..b0c479c46c7 100644 --- a/src/Columns/IColumnDummy.h +++ b/src/Columns/IColumnDummy.h @@ -107,7 +107,7 @@ public: res[i] = i; } - void updatePermutation(bool, size_t, int, Permutation & res, EqualRanges&) const override {} + void updatePermutation(bool, size_t, int, Permutation &, EqualRanges&) const override {} ColumnPtr replicate(const Offsets & offsets) const override { diff --git a/src/Interpreters/sortBlock.cpp b/src/Interpreters/sortBlock.cpp index d982086d0b6..cea975ff5f4 100644 --- a/src/Interpreters/sortBlock.cpp +++ b/src/Interpreters/sortBlock.cpp @@ -192,14 +192,17 @@ void sortBlock(Block & block, const SortDescription & description, UInt64 limit) ranges.emplace_back(0, perm.size()); for (const auto& column : columns_with_sort_desc) { - while (!ranges.empty() && limit && limit <= ranges.back().first) { + while (!ranges.empty() && limit && limit <= ranges.back().first) + { ranges.pop_back(); } - if (ranges.empty()) { + if (ranges.empty()) + { break; } - if (isCollationRequired(column.description)) { + if (isCollationRequired(column.description)) + { const ColumnString & column_string = assert_cast(*column.column); column_string.updatePermutationWithCollation(*column.description.collator, column.description.direction < 0, limit, column.description.nulls_direction, perm, ranges); } @@ -216,12 +219,16 @@ void sortBlock(Block & block, const SortDescription & description, UInt64 limit) ranges.emplace_back(0, perm.size()); for (const auto& column : columns_with_sort_desc) { - while (!ranges.empty() && limit && limit <= ranges.back().first) { + while (!ranges.empty() && limit && limit <= ranges.back().first) + { ranges.pop_back(); } - if (ranges.empty()) { + if (ranges.empty()) + { break; } + column.column->updatePermutation( + column.description.direction < 0, limit, column.description.nulls_direction, perm, ranges); } } From 4688729396a481989f61c184f6a73475506781f8 Mon Sep 17 00:00:00 2001 From: Vasily Morozov Date: Sun, 17 May 2020 00:59:05 +0300 Subject: [PATCH 373/738] fix --- src/Columns/ColumnDecimal.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Columns/ColumnDecimal.cpp b/src/Columns/ColumnDecimal.cpp index 94f20e845fd..f8c59c93efd 100644 --- a/src/Columns/ColumnDecimal.cpp +++ b/src/Columns/ColumnDecimal.cpp @@ -151,7 +151,7 @@ void ColumnDecimal::updatePermutation(bool reverse, size_t limit, int, IColum if (reverse) std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, [this](size_t a, size_t b) { return data[a] > data[b]; }); else - std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, [this](size_t a, size_t b) { return data[a] > data[b]; }); + std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, [this](size_t a, size_t b) { return data[a] < data[b]; }); auto new_first = first; for (auto j = first + 1; j < limit; ++j) { From 0fe723acc9a7e8833af84ed76a3ac9da8e7433f3 Mon Sep 17 00:00:00 2001 From: Artem Zuikov Date: Mon, 18 May 2020 13:26:23 +0300 Subject: [PATCH 374/738] Try to fix arcadia sync (#10959) --- programs/server/Server.cpp | 13 +++++++------ src/Columns/ColumnVector.cpp | 4 +++- src/Common/BitonicSort.h | 2 +- src/Common/{oclBasics.cpp => oclBasics.h} | 2 ++ src/Common/tests/CMakeLists.txt | 8 ++++---- 5 files changed, 17 insertions(+), 12 deletions(-) rename src/Common/{oclBasics.cpp => oclBasics.h} (99%) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 13cb294d03f..bcafdcb0308 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -61,13 +61,12 @@ #include #include "MySQLHandlerFactory.h" -#if USE_OPENCL -#include "Common/BitonicSort.h" -#endif - #if !defined(ARCADIA_BUILD) -# include "config_core.h" -# include "Common/config_version.h" +# include "config_core.h" +# include "Common/config_version.h" +# if USE_OPENCL +# include "Common/BitonicSort.h" +# endif #endif #if defined(OS_LINUX) @@ -225,8 +224,10 @@ int Server::main(const std::vector & /*args*/) registerDictionaries(); registerDisks(); +#if !defined(ARCADIA_BUILD) #if USE_OPENCL BitonicSort::getInstance().configure(); +#endif #endif CurrentMetrics::set(CurrentMetrics::Revision, ClickHouseRevision::get()); diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 9b128fcffec..42013bdffd1 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -21,7 +21,7 @@ #if !defined(ARCADIA_BUILD) # include # if USE_OPENCL -# include "Common/BitonicSort.h" +# include "Common/BitonicSort.h" // Y_IGNORE # endif #else #undef USE_OPENCL @@ -144,10 +144,12 @@ void ColumnVector::getPermutation(bool reverse, size_t limit, int nan_directi } else { +#if !defined(ARCADIA_BUILD) #if USE_OPENCL /// If bitonic sort if specified as preferred than `nan_direction_hint` equals specific value 42. if (nan_direction_hint == 42 && BitonicSort::getInstance().sort(data, res, !reverse)) return; +#endif #endif /// A case for radix sort diff --git a/src/Common/BitonicSort.h b/src/Common/BitonicSort.h index 840ab477a45..cbe5b5dc0a4 100644 --- a/src/Common/BitonicSort.h +++ b/src/Common/BitonicSort.h @@ -24,7 +24,7 @@ #include #include -#include "oclBasics.cpp" +#include "oclBasics.h" #include "bitonicSortKernels.cl" class BitonicSort diff --git a/src/Common/oclBasics.cpp b/src/Common/oclBasics.h similarity index 99% rename from src/Common/oclBasics.cpp rename to src/Common/oclBasics.h index cd90cd945c1..550f42a32d0 100644 --- a/src/Common/oclBasics.cpp +++ b/src/Common/oclBasics.h @@ -1,3 +1,5 @@ +#pragma once + #include #if USE_OPENCL diff --git a/src/Common/tests/CMakeLists.txt b/src/Common/tests/CMakeLists.txt index 495663bc5d8..1c8a39a4fa5 100644 --- a/src/Common/tests/CMakeLists.txt +++ b/src/Common/tests/CMakeLists.txt @@ -35,10 +35,10 @@ target_link_libraries (compact_array PRIVATE clickhouse_common_io) add_executable (radix_sort radix_sort.cpp) target_link_libraries (radix_sort PRIVATE clickhouse_common_io) -if (USE_OPENCL) - add_executable (bitonic_sort bitonic_sort.cpp) - target_link_libraries (bitonic_sort PRIVATE clickhouse_common_io ${OPENCL_LINKER_FLAGS}) -endif () +# if (USE_OPENCL) +# add_executable (bitonic_sort bitonic_sort.cpp) +# target_link_libraries (bitonic_sort PRIVATE clickhouse_common_io ${OPENCL_LINKER_FLAGS}) +# endif () add_executable (arena_with_free_lists arena_with_free_lists.cpp) target_link_libraries (arena_with_free_lists PRIVATE dbms) From 528ac30a019ce173e5b613d6767aa7f4b326e02a Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Mon, 18 May 2020 13:31:01 +0300 Subject: [PATCH 375/738] [docs] handle incorrect hash in url (#10999) * [docs] handle incorrect hash in url * fix mistype --- website/js/base.js | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/website/js/base.js b/website/js/base.js index 92d34cf2661..7311da8d7a4 100644 --- a/website/js/base.js +++ b/website/js/base.js @@ -20,8 +20,11 @@ var offset = 0; if (selector !== '#') { - offset = $(selector).offset().top - $('#top-nav').height() * 1.5; - dst += selector; + var destination = $(selector); + if (destination.length) { + offset = destination.offset().top - $('#top-nav').height() * 1.5; + dst += selector; + } } $('html, body').animate({ scrollTop: offset @@ -32,10 +35,13 @@ var top_nav = $('#top-nav.sticky-top'); if (window.location.hash.length > 1 && top_nav.length) { - var offset = $(window.location.hash).offset().top - top_nav.height() * 1.5; - $('html, body').animate({ - scrollTop: offset - }, 70); + var hash_destination = $(window.location.hash); + if (hash_destination.length) { + var offset = hash_destination.offset().top - top_nav.height() * 1.5; + $('html, body').animate({ + scrollTop: offset + }, 70); + } } $('img').each(function() { From fcaee7dc9831c84dfc255af0d9fe6be961b6c1e6 Mon Sep 17 00:00:00 2001 From: Artem Zuikov Date: Mon, 18 May 2020 13:37:13 +0300 Subject: [PATCH 376/738] add file to ya.make --- src/Processors/ya.make | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Processors/ya.make b/src/Processors/ya.make index 09fb9d64ad1..8659fffd741 100644 --- a/src/Processors/ya.make +++ b/src/Processors/ya.make @@ -98,6 +98,7 @@ SRCS( Merges/ReplacingSortedTransform.h Merges/SummingSortedTransform.h Merges/VersionedCollapsingTransform.h + OffsetTransform.cpp Pipe.cpp Port.cpp QueryPipeline.cpp From 62f77abbc5e0f0d5ca0ee07f4097e2c4a50fc300 Mon Sep 17 00:00:00 2001 From: Artem Zuikov Date: Mon, 18 May 2020 13:40:02 +0300 Subject: [PATCH 377/738] add file to ya.make --- src/Functions/ya.make | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Functions/ya.make b/src/Functions/ya.make index 999a63e8fa3..900c286d94b 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -283,6 +283,7 @@ SRCS( rand.cpp randomPrintableASCII.cpp randomString.cpp + randomStringUTF8.cpp regexpQuoteMeta.cpp registerFunctionsArithmetic.cpp registerFunctionsComparison.cpp From ebce2c7e7c84e78219bb4cc3f5ba297b1cd0d6b0 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 18 May 2020 14:38:22 +0300 Subject: [PATCH 378/738] only style changes --- src/Columns/ColumnArray.cpp | 19 +++++++++--------- src/Columns/ColumnDecimal.cpp | 29 +++++++++++++--------------- src/Columns/ColumnFixedString.cpp | 13 +++---------- src/Columns/ColumnLowCardinality.cpp | 24 ++++++++--------------- src/Columns/ColumnNullable.cpp | 20 ++++++------------- 5 files changed, 39 insertions(+), 66 deletions(-) diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index d7f2964b13b..49678682713 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -740,39 +740,39 @@ void ColumnArray::getPermutation(bool reverse, size_t limit, int nan_direction_h void ColumnArray::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res, EqualRanges& equal_range) const { if (limit >= size() || limit >= equal_range.back().second) - { limit = 0; - } + size_t n = equal_range.size(); + if (limit) - { --n; - } + EqualRanges new_ranges; for (size_t i = 0; i < n; ++i) { const auto& [first, last] = equal_range[i]; + if (reverse) std::sort(res.begin() + first, res.begin() + last, Less(*this, nan_direction_hint)); else std::sort(res.begin() + first, res.begin() + last, Less(*this, nan_direction_hint)); auto new_first = first; + for (auto j = first + 1; j < last; ++j) { if (compareAt(res[new_first], res[j], *this, nan_direction_hint) != 0) { if (j - new_first > 1) - { new_ranges.emplace_back(new_first, j); - } + new_first = j; } } + if (last - new_first > 1) - { new_ranges.emplace_back(new_first, last); - } } + if (limit) { const auto& [first, last] = equal_range.back(); @@ -786,9 +786,8 @@ void ColumnArray::updatePermutation(bool reverse, size_t limit, int nan_directio if (compareAt(res[new_first], res[j], *this, nan_direction_hint) != 0) { if (j - new_first > 1) - { new_ranges.emplace_back(new_first, j); - } + new_first = j; } } diff --git a/src/Columns/ColumnDecimal.cpp b/src/Columns/ColumnDecimal.cpp index f8c59c93efd..604a2c27ab2 100644 --- a/src/Columns/ColumnDecimal.cpp +++ b/src/Columns/ColumnDecimal.cpp @@ -112,55 +112,54 @@ template void ColumnDecimal::updatePermutation(bool reverse, size_t limit, int, IColumn::Permutation & res, EqualRanges& equal_range) const { if (limit >= data.size() || limit >= equal_range.back().second) - { limit = 0; - } + size_t n = equal_range.size(); if (limit) - { --n; - } + EqualRanges new_ranges; for (size_t i = 0; i < n; ++i) { const auto& [first, last] = equal_range[i]; if (reverse) - std::partial_sort(res.begin() + first, res.begin() + last, res.begin() + last, [this](size_t a, size_t b) { return data[a] > data[b]; }); + std::partial_sort(res.begin() + first, res.begin() + last, res.begin() + last, + [this](size_t a, size_t b) { return data[a] > data[b]; }); else - std::partial_sort(res.begin() + first, res.begin() + last, res.begin() + last, [this](size_t a, size_t b) { return data[a] < data[b]; }); + std::partial_sort(res.begin() + first, res.begin() + last, res.begin() + last, + [this](size_t a, size_t b) { return data[a] < data[b]; }); auto new_first = first; for (auto j = first + 1; j < last; ++j) { if (data[res[new_first]] != data[res[j]]) { if (j - new_first > 1) - { new_ranges.emplace_back(new_first, j); - } + new_first = j; } } if (last - new_first > 1) - { new_ranges.emplace_back(new_first, last); - } } + if (limit) { const auto& [first, last] = equal_range.back(); if (reverse) - std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, [this](size_t a, size_t b) { return data[a] > data[b]; }); + std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, + [this](size_t a, size_t b) { return data[a] > data[b]; }); else - std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, [this](size_t a, size_t b) { return data[a] < data[b]; }); + std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, + [this](size_t a, size_t b) { return data[a] < data[b]; }); auto new_first = first; for (auto j = first + 1; j < limit; ++j) { if (data[res[new_first]] != data[res[j]]) { if (j - new_first > 1) - { new_ranges.emplace_back(new_first, j); - } + new_first = j; } } @@ -174,9 +173,7 @@ void ColumnDecimal::updatePermutation(bool reverse, size_t limit, int, IColum } } if (new_last - new_first > 1) - { new_ranges.emplace_back(new_first, new_last); - } } equal_range = std::move(new_ranges); } diff --git a/src/Columns/ColumnFixedString.cpp b/src/Columns/ColumnFixedString.cpp index 52176925ae0..ffd707111d8 100644 --- a/src/Columns/ColumnFixedString.cpp +++ b/src/Columns/ColumnFixedString.cpp @@ -169,9 +169,8 @@ void ColumnFixedString::updatePermutation(bool reverse, size_t limit, int, Permu size_t k = equal_range.size(); if (limit) - { --k; - } + EqualRanges new_ranges; for (size_t i = 0; i < k; ++i) @@ -187,16 +186,13 @@ void ColumnFixedString::updatePermutation(bool reverse, size_t limit, int, Permu if (memcmpSmallAllowOverflow15(chars.data() + j * n, chars.data() + new_first * n, n) != 0) { if (j - new_first > 1) - { new_ranges.emplace_back(new_first, j); - } + new_first = j; } } if (last - new_first > 1) - { new_ranges.emplace_back(new_first, last); - } } if (limit) { @@ -211,9 +207,8 @@ void ColumnFixedString::updatePermutation(bool reverse, size_t limit, int, Permu if (memcmpSmallAllowOverflow15(chars.data() + j * n, chars.data() + new_first * n, n) != 0) { if (j - new_first > 1) - { new_ranges.emplace_back(new_first, j); - } + new_first = j; } } @@ -227,9 +222,7 @@ void ColumnFixedString::updatePermutation(bool reverse, size_t limit, int, Permu } } if (new_last - new_first > 1) - { new_ranges.emplace_back(new_first, new_last); - } } equal_range = std::move(new_ranges); } diff --git a/src/Columns/ColumnLowCardinality.cpp b/src/Columns/ColumnLowCardinality.cpp index 0cf0e6f9be6..47fa570a7cb 100644 --- a/src/Columns/ColumnLowCardinality.cpp +++ b/src/Columns/ColumnLowCardinality.cpp @@ -317,43 +317,38 @@ void ColumnLowCardinality::getPermutation(bool reverse, size_t limit, int nan_di void ColumnLowCardinality::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges& equal_range) const { if (limit >= size() || limit >= equal_range.back().second) - { limit = 0; - } + size_t n = equal_range.size(); if (limit) - { --n; - } + EqualRanges new_ranges; for (size_t i = 0; i < n; ++i) { const auto& [first, last] = equal_range[i]; if (reverse) std::sort(res.begin() + first, res.begin() + last, [this, nan_direction_hint](size_t a, size_t b) - { - return getDictionary().compareAt(getIndexes().getUInt(a), getIndexes().getUInt(b), getDictionary(), nan_direction_hint) > 0; }); + {return getDictionary().compareAt(getIndexes().getUInt(a), getIndexes().getUInt(b), getDictionary(), nan_direction_hint) > 0; }); else std::sort(res.begin() + first, res.begin() + last, [this, nan_direction_hint](size_t a, size_t b) - { - return getDictionary().compareAt(getIndexes().getUInt(a), getIndexes().getUInt(b), getDictionary(), nan_direction_hint) < 0; }); + {return getDictionary().compareAt(getIndexes().getUInt(a), getIndexes().getUInt(b), getDictionary(), nan_direction_hint) < 0; }); + auto new_first = first; for (auto j = first + 1; j < last; ++j) { if (compareAt(new_first, j, *this, nan_direction_hint) != 0) { if (j - new_first > 1) - { new_ranges.emplace_back(new_first, j); - } + new_first = j; } } if (last - new_first > 1) - { new_ranges.emplace_back(new_first, last); - } } + if (limit) { const auto& [first, last] = equal_range.back(); @@ -369,9 +364,8 @@ void ColumnLowCardinality::updatePermutation(bool reverse, size_t limit, int nan if (getDictionary().compareAt(getIndexes().getUInt(new_first), getIndexes().getUInt(j), getDictionary(), nan_direction_hint) != 0) { if (j - new_first > 1) - { new_ranges.emplace_back(new_first, j); - } + new_first = j; } } @@ -385,9 +379,7 @@ void ColumnLowCardinality::updatePermutation(bool reverse, size_t limit, int nan } } if (new_last - new_first > 1) - { new_ranges.emplace_back(new_first, new_last); - } } equal_range = std::move(new_ranges); } diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp index 0a2986160e3..42c948a45b1 100644 --- a/src/Columns/ColumnNullable.cpp +++ b/src/Columns/ColumnNullable.cpp @@ -324,9 +324,7 @@ void ColumnNullable::getPermutation(bool reverse, size_t limit, int null_directi void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_direction_hint, IColumn::Permutation & res, EqualRanges& equal_range) const { if (limit >= equal_range.back().second || limit >= size()) - { limit = 0; - } EqualRanges new_ranges, temp_ranges; @@ -364,36 +362,30 @@ void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_dire } ++read_idx; } + if (write_idx - first > 1) { if (direction) - { temp_ranges.emplace_back(first, write_idx); - } else - { new_ranges.emplace_back(first, write_idx); - } + } + if (last - write_idx > 1) { if (direction) - { new_ranges.emplace_back(write_idx, last); - } else - { + else temp_ranges.emplace_back(write_idx, last); - } } } while (!new_ranges.empty() && limit && limit <= new_ranges.back().first) - { new_ranges.pop_back(); - } + if (!temp_ranges.empty()) - { getNestedColumn().updatePermutation(reverse, limit, null_direction_hint, res, temp_ranges); - } + equal_range.resize(temp_ranges.size() + new_ranges.size()); std::merge(temp_ranges.begin(), temp_ranges.end(), new_ranges.begin(), new_ranges.end(), equal_range.begin()); } From bfcbc08bba6db36cad7e3f2991cdea8c84e07ab7 Mon Sep 17 00:00:00 2001 From: Artem Zuikov Date: Mon, 18 May 2020 15:50:23 +0300 Subject: [PATCH 379/738] arcadia sync fixes --- programs/server/Server.cpp | 2 +- src/Functions/ya.make | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index bcafdcb0308..4958a7976de 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -65,7 +65,7 @@ # include "config_core.h" # include "Common/config_version.h" # if USE_OPENCL -# include "Common/BitonicSort.h" +# include "Common/BitonicSort.h" // Y_IGNORE # endif #endif diff --git a/src/Functions/ya.make b/src/Functions/ya.make index 900c286d94b..8e53ffe493d 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -284,6 +284,7 @@ SRCS( randomPrintableASCII.cpp randomString.cpp randomStringUTF8.cpp + randomFixedString.cpp regexpQuoteMeta.cpp registerFunctionsArithmetic.cpp registerFunctionsComparison.cpp From e542e814de3d594fbc2526d8c65d4280b2d6b490 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 18 May 2020 16:34:54 +0300 Subject: [PATCH 380/738] better background_pool_size description in docs --- docs/en/operations/settings/settings.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 67097103cc9..130a56fb131 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -1250,7 +1250,9 @@ Default value: Empty ## background\_pool\_size {#background_pool_size} -Sets the number of threads performing background operations in table engines (for example, merges in [MergeTree engine](../../engines/table-engines/mergetree-family/index.md) tables). This setting is applied at ClickHouse server start and can’t be changed in a user session. By adjusting this setting, you manage CPU and disk load. Smaller pool size utilizes less CPU and disk resources, but background processes advance slower which might eventually impact query performance. +Sets the number of threads performing background operations in table engines (for example, merges in [MergeTree engine](../../engines/table-engines/mergetree-family/index.md) tables). This setting is applied from `default` profile at ClickHouse server start and can’t be changed in a user session. By adjusting this setting, you manage CPU and disk load. Smaller pool size utilizes less CPU and disk resources, but background processes advance slower which might eventually impact query performance. + +Before changing it, please also take a look at related [MergeTree settings](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-merge_tree), such as `number_of_free_entries_in_pool_to_lower_max_size_of_merge` and `number_of_free_entries_in_pool_to_execute_mutation` Possible values: From 03f09986fd9b75e54d196bb5f4cc9cb1285f38aa Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 18 May 2020 16:47:30 +0300 Subject: [PATCH 381/738] Add dot. --- docs/en/operations/settings/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 130a56fb131..5adba7cfa07 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -1252,7 +1252,7 @@ Default value: Empty Sets the number of threads performing background operations in table engines (for example, merges in [MergeTree engine](../../engines/table-engines/mergetree-family/index.md) tables). This setting is applied from `default` profile at ClickHouse server start and can’t be changed in a user session. By adjusting this setting, you manage CPU and disk load. Smaller pool size utilizes less CPU and disk resources, but background processes advance slower which might eventually impact query performance. -Before changing it, please also take a look at related [MergeTree settings](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-merge_tree), such as `number_of_free_entries_in_pool_to_lower_max_size_of_merge` and `number_of_free_entries_in_pool_to_execute_mutation` +Before changing it, please also take a look at related [MergeTree settings](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-merge_tree), such as `number_of_free_entries_in_pool_to_lower_max_size_of_merge` and `number_of_free_entries_in_pool_to_execute_mutation`. Possible values: From bec7d1ce934fb9c8fbd464ffe4839266062d3509 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 18 May 2020 16:49:19 +0300 Subject: [PATCH 382/738] Fix style check --- src/Disks/IVolume.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Disks/IVolume.cpp b/src/Disks/IVolume.cpp index a71afa4a840..6a122a3e3b2 100644 --- a/src/Disks/IVolume.cpp +++ b/src/Disks/IVolume.cpp @@ -10,7 +10,6 @@ namespace DB namespace ErrorCodes { extern const int EXCESSIVE_ELEMENT_IN_CONFIG; - extern const int UNKNOWN_VOLUME_TYPE; } IVolume::IVolume( From 6af9c7847667bd0db95af6597de23d59972a3d84 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 18 May 2020 16:51:01 +0300 Subject: [PATCH 383/738] remove data on DROP DATABASE --- src/Databases/DatabaseMemory.cpp | 7 +++++++ src/Databases/DatabaseMemory.h | 2 ++ 2 files changed, 9 insertions(+) diff --git a/src/Databases/DatabaseMemory.cpp b/src/Databases/DatabaseMemory.cpp index 417761f81e7..84fec6bcc22 100644 --- a/src/Databases/DatabaseMemory.cpp +++ b/src/Databases/DatabaseMemory.cpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB @@ -84,4 +85,10 @@ UUID DatabaseMemory::tryGetTableUUID(const String & table_name) const return UUIDHelpers::Nil; } +void DatabaseMemory::drop(const Context & context) +{ + /// Remove data on explicit DROP DATABASE + std::filesystem::remove_all(context.getPath() + data_path); +} + } diff --git a/src/Databases/DatabaseMemory.h b/src/Databases/DatabaseMemory.h index 29a9abc9d75..ad34c4d9097 100644 --- a/src/Databases/DatabaseMemory.h +++ b/src/Databases/DatabaseMemory.h @@ -46,6 +46,8 @@ public: UUID tryGetTableUUID(const String & table_name) const override; + void drop(const Context & context) override; + private: String data_path; using NameToASTCreate = std::unordered_map; From 0e48cb1f802995933b5217d94be40f036a883c2f Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 18 May 2020 16:55:07 +0300 Subject: [PATCH 384/738] Removed allow_processors flag from executeQuery(). --- src/DataStreams/BlockIO.cpp | 22 ++++ src/DataStreams/BlockIO.h | 3 + .../ClickHouseDictionarySource.cpp | 23 ++-- .../InterpreterKillQueryQuery.cpp | 5 +- src/Interpreters/executeQuery.cpp | 12 +- src/Interpreters/executeQuery.h | 3 +- .../PipelineExecutingBlockInputStream.cpp | 106 ++++++++++++++++++ .../PipelineExecutingBlockInputStream.h | 39 +++++++ .../Executors/PullingPipelineExecutor.cpp | 5 + .../Executors/PullingPipelineExecutor.h | 3 + 10 files changed, 198 insertions(+), 23 deletions(-) create mode 100644 src/Processors/Executors/PipelineExecutingBlockInputStream.cpp create mode 100644 src/Processors/Executors/PipelineExecutingBlockInputStream.h diff --git a/src/DataStreams/BlockIO.cpp b/src/DataStreams/BlockIO.cpp index 60a0b415237..150995962bf 100644 --- a/src/DataStreams/BlockIO.cpp +++ b/src/DataStreams/BlockIO.cpp @@ -1,9 +1,31 @@ #include #include +#include namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +BlockInputStreamPtr BlockIO::getInputStream() +{ + if (out) + throw Exception("Cannot get input stream from BlockIO because output stream is not empty", + ErrorCodes::LOGICAL_ERROR); + + if (in) + return in; + + if (pipeline.initialized()) + return std::make_shared(std::move(pipeline)); + + throw Exception("Cannot get input stream from BlockIO because query pipeline was not initialized", + ErrorCodes::LOGICAL_ERROR); +} + void BlockIO::reset() { /** process_list_entry should be destroyed after in, after out and after pipeline, diff --git a/src/DataStreams/BlockIO.h b/src/DataStreams/BlockIO.h index 08a5f819fd6..d4733e6aebe 100644 --- a/src/DataStreams/BlockIO.h +++ b/src/DataStreams/BlockIO.h @@ -50,6 +50,9 @@ struct BlockIO exception_callback(); } + /// Returns in or converts pipeline to stream. Throws if out is not empty. + BlockInputStreamPtr getInputStream(); + private: void reset(); }; diff --git a/src/Dictionaries/ClickHouseDictionarySource.cpp b/src/Dictionaries/ClickHouseDictionarySource.cpp index 7bf147dd539..aefde10a873 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.cpp +++ b/src/Dictionaries/ClickHouseDictionarySource.cpp @@ -131,10 +131,10 @@ BlockInputStreamPtr ClickHouseDictionarySource::loadAll() */ if (is_local) { - BlockIO res = executeQuery(load_all_query, context, true, QueryProcessingStage::Complete, false, false); + auto stream = executeQuery(load_all_query, context, true).getInputStream(); /// FIXME res.in may implicitly use some objects owned be res, but them will be destructed after return - res.in = std::make_shared(res.in, sample_block, ConvertingBlockInputStream::MatchColumnsMode::Position); - return res.in; + stream = std::make_shared(stream, sample_block, ConvertingBlockInputStream::MatchColumnsMode::Position); + return stream; } return std::make_shared(pool, load_all_query, sample_block, context); } @@ -144,9 +144,9 @@ BlockInputStreamPtr ClickHouseDictionarySource::loadUpdatedAll() std::string load_update_query = getUpdateFieldAndDate(); if (is_local) { - auto res = executeQuery(load_update_query, context, true, QueryProcessingStage::Complete, false, false); - res.in = std::make_shared(res.in, sample_block, ConvertingBlockInputStream::MatchColumnsMode::Position); - return res.in; + auto stream = executeQuery(load_update_query, context, true).getInputStream(); + stream = std::make_shared(stream, sample_block, ConvertingBlockInputStream::MatchColumnsMode::Position); + return stream; } return std::make_shared(pool, load_update_query, sample_block, context); } @@ -191,10 +191,10 @@ BlockInputStreamPtr ClickHouseDictionarySource::createStreamForSelectiveLoad(con { if (is_local) { - auto res = executeQuery(query, context, true, QueryProcessingStage::Complete, false, false); - res.in = std::make_shared( - res.in, sample_block, ConvertingBlockInputStream::MatchColumnsMode::Position); - return res.in; + auto res = executeQuery(query, context, true).getInputStream(); + res = std::make_shared( + res, sample_block, ConvertingBlockInputStream::MatchColumnsMode::Position); + return res; } return std::make_shared(pool, query, sample_block, context); @@ -206,8 +206,7 @@ std::string ClickHouseDictionarySource::doInvalidateQuery(const std::string & re if (is_local) { Context query_context = context; - auto input_block = executeQuery(request, query_context, true, - QueryProcessingStage::Complete, false, false).in; + auto input_block = executeQuery(request, query_context, true).getInputStream(); return readInvalidateQuery(*input_block); } else diff --git a/src/Interpreters/InterpreterKillQueryQuery.cpp b/src/Interpreters/InterpreterKillQueryQuery.cpp index 23f39ab3fc5..39e432195fd 100644 --- a/src/Interpreters/InterpreterKillQueryQuery.cpp +++ b/src/Interpreters/InterpreterKillQueryQuery.cpp @@ -302,8 +302,9 @@ Block InterpreterKillQueryQuery::getSelectResult(const String & columns, const S if (where_expression) select_query += " WHERE " + queryToString(where_expression); - BlockIO block_io = executeQuery(select_query, context.getGlobalContext(), true, QueryProcessingStage::Complete, false, false); - Block res = block_io.in->read(); + BlockIO block_io = executeQuery(select_query, context.getGlobalContext(), true); + auto stream = block_io.getInputStream(); + Block res = stream->read(); if (res && block_io.in->read()) throw Exception("Expected one block from input stream", ErrorCodes::LOGICAL_ERROR); diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 4d609395c3a..8c6e5cea73c 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -196,8 +196,7 @@ static std::tuple executeQueryImpl( bool internal, QueryProcessingStage::Enum stage, bool has_query_tail, - ReadBuffer * istr, - bool allow_processors) + ReadBuffer * istr) { time_t current_time = time(nullptr); @@ -317,7 +316,7 @@ static std::tuple executeQueryImpl( context.resetInputCallbacks(); auto interpreter = InterpreterFactory::get(ast, context, stage); - bool use_processors = allow_processors && interpreter->canExecuteWithProcessors(); + bool use_processors = interpreter->canExecuteWithProcessors(); std::shared_ptr quota; if (!interpreter->ignoreQuota()) @@ -580,13 +579,12 @@ BlockIO executeQuery( Context & context, bool internal, QueryProcessingStage::Enum stage, - bool may_have_embedded_data, - bool allow_processors) + bool may_have_embedded_data) { ASTPtr ast; BlockIO streams; std::tie(ast, streams) = executeQueryImpl(query.data(), query.data() + query.size(), context, - internal, stage, !may_have_embedded_data, nullptr, allow_processors); + internal, stage, !may_have_embedded_data, nullptr); if (const auto * ast_query_with_output = dynamic_cast(ast.get())) { @@ -647,7 +645,7 @@ void executeQuery( ASTPtr ast; BlockIO streams; - std::tie(ast, streams) = executeQueryImpl(begin, end, context, false, QueryProcessingStage::Complete, may_have_tail, &istr, true); + std::tie(ast, streams) = executeQueryImpl(begin, end, context, false, QueryProcessingStage::Complete, may_have_tail, &istr); auto & pipeline = streams.pipeline; diff --git a/src/Interpreters/executeQuery.h b/src/Interpreters/executeQuery.h index b6cb56b9e24..22b353488ad 100644 --- a/src/Interpreters/executeQuery.h +++ b/src/Interpreters/executeQuery.h @@ -42,8 +42,7 @@ BlockIO executeQuery( Context & context, /// DB, tables, data types, storage engines, functions, aggregate functions... bool internal = false, /// If true, this query is caused by another query and thus needn't be registered in the ProcessList. QueryProcessingStage::Enum stage = QueryProcessingStage::Complete, /// To which stage the query must be executed. - bool may_have_embedded_data = false, /// If insert query may have embedded data - bool allow_processors = true /// If can use processors pipeline + bool may_have_embedded_data = false /// If insert query may have embedded data ); diff --git a/src/Processors/Executors/PipelineExecutingBlockInputStream.cpp b/src/Processors/Executors/PipelineExecutingBlockInputStream.cpp new file mode 100644 index 00000000000..e7846f31bd5 --- /dev/null +++ b/src/Processors/Executors/PipelineExecutingBlockInputStream.cpp @@ -0,0 +1,106 @@ +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +PipelineExecutingBlockInputStream::PipelineExecutingBlockInputStream(QueryPipeline pipeline_) + : pipeline(std::make_unique(std::move(pipeline_))) +{ +} + +PipelineExecutingBlockInputStream::~PipelineExecutingBlockInputStream() = default; + +Block PipelineExecutingBlockInputStream::getHeader() const +{ + return executor ? executor->getHeader() + : pipeline->getHeader(); +} + +void PipelineExecutingBlockInputStream::readPrefixImpl() +{ + executor = std::make_unique(*pipeline); +} + +Block PipelineExecutingBlockInputStream::readImpl() +{ + Block block; + while (executor->pull(block)) + { + if (block) + return block; + } + + return {}; +} + +inline static void throwIfExecutionStarted(bool is_execution_started, const char * method) +{ + if (is_execution_started) + throw Exception(String("Cannot call ") + method + + " for PipelineExecutingBlockInputStream because execution was started", + ErrorCodes::LOGICAL_ERROR); +} + +inline static void throwIfExecutionNotStarted(bool is_execution_started, const char * method) +{ + if (!is_execution_started) + throw Exception(String("Cannot call ") + method + + " for PipelineExecutingBlockInputStream because execution was not started", + ErrorCodes::LOGICAL_ERROR); +} + +void PipelineExecutingBlockInputStream::cancel(bool kill) +{ + throwIfExecutionNotStarted(executor != nullptr, "cancel"); + IBlockInputStream::cancel(kill); + executor->cancel(); +} + + +void PipelineExecutingBlockInputStream::setProgressCallback(const ProgressCallback & callback) +{ + throwIfExecutionStarted(executor != nullptr, "setProgressCallback"); + pipeline->setProgressCallback(callback); +} + +void PipelineExecutingBlockInputStream::setProcessListElement(QueryStatus * elem) +{ + throwIfExecutionStarted(executor != nullptr, "setProcessListElement"); + IBlockInputStream::setProcessListElement(elem); + pipeline->setProcessListElement(elem); +} + +void PipelineExecutingBlockInputStream::setLimits(const IBlockInputStream::LocalLimits & limits_) +{ + throwIfExecutionStarted(executor != nullptr, "setLimits"); + + if (limits_.mode == LimitsMode::LIMITS_TOTAL) + throw Exception("Total limits are not supported by PipelineExecutingBlockInputStream", + ErrorCodes::LOGICAL_ERROR); + + /// Local limits may be checked by IBlockInputStream itself. + IBlockInputStream::setLimits(limits_); +} + +void PipelineExecutingBlockInputStream::setQuota(const std::shared_ptr &) +{ + throw Exception("Quota is not supported by PipelineExecutingBlockInputStream", + ErrorCodes::LOGICAL_ERROR); +} + +void PipelineExecutingBlockInputStream::addTotalRowsApprox(size_t) +{ + throw Exception("Progress is not supported by PipelineExecutingBlockInputStream", + ErrorCodes::LOGICAL_ERROR); +} + + +} diff --git a/src/Processors/Executors/PipelineExecutingBlockInputStream.h b/src/Processors/Executors/PipelineExecutingBlockInputStream.h new file mode 100644 index 00000000000..773332f69c8 --- /dev/null +++ b/src/Processors/Executors/PipelineExecutingBlockInputStream.h @@ -0,0 +1,39 @@ +#pragma once +#include + +namespace DB +{ + +class QueryPipeline; +class PullingPipelineExecutor; + +/// Implement IBlockInputStream from QueryPipeline. +/// It's a temporary wrapper. +class PipelineExecutingBlockInputStream : public IBlockInputStream +{ +public: + explicit PipelineExecutingBlockInputStream(QueryPipeline pipeline_); + ~PipelineExecutingBlockInputStream(); + + String getName() const override { return "PipelineExecuting"; } + Block getHeader() const override; + + void cancel(bool kill) override; + + /// Implement IBlockInputStream methods via QueryPipeline. + void setProgressCallback(const ProgressCallback & callback) final; + void setProcessListElement(QueryStatus * elem) final; + void setLimits(const LocalLimits & limits_) final; + void setQuota(const std::shared_ptr & quota_) final; + void addTotalRowsApprox(size_t value) final; + +protected: + void readPrefixImpl() override; + Block readImpl() override; + +private: + std::unique_ptr executor; + std::unique_ptr pipeline; +}; + +} diff --git a/src/Processors/Executors/PullingPipelineExecutor.cpp b/src/Processors/Executors/PullingPipelineExecutor.cpp index c34195a0793..223c22e59db 100644 --- a/src/Processors/Executors/PullingPipelineExecutor.cpp +++ b/src/Processors/Executors/PullingPipelineExecutor.cpp @@ -52,6 +52,11 @@ PullingPipelineExecutor::~PullingPipelineExecutor() } } +const Block & PullingPipelineExecutor::getHeader() const +{ + return lazy_format->getPort(IOutputFormat::PortKind::Main).getHeader(); +} + static void threadFunction(PullingPipelineExecutor::Data & data, ThreadGroupStatusPtr thread_group, size_t num_threads) { if (thread_group) diff --git a/src/Processors/Executors/PullingPipelineExecutor.h b/src/Processors/Executors/PullingPipelineExecutor.h index f3b06fc618a..7a093fe0022 100644 --- a/src/Processors/Executors/PullingPipelineExecutor.h +++ b/src/Processors/Executors/PullingPipelineExecutor.h @@ -22,6 +22,9 @@ public: explicit PullingPipelineExecutor(QueryPipeline & pipeline_); ~PullingPipelineExecutor(); + /// Get structure of returned block or chunk. + const Block & getHeader() const; + /// Methods return false if query is finished. /// If milliseconds > 0, returns empty object and `true` after timeout exceeded. /// You can use any pull method. From bf66693a85c6b628b839b1cabbef5e6c91e4ed66 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Mon, 18 May 2020 17:29:05 +0300 Subject: [PATCH 385/738] Get rid of "Note Note" --- docs/en/getting-started/playground.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/getting-started/playground.md b/docs/en/getting-started/playground.md index e80673f1b1f..c552e07eded 100644 --- a/docs/en/getting-started/playground.md +++ b/docs/en/getting-started/playground.md @@ -22,7 +22,7 @@ You can make queries to playground using any HTTP client, for example [curl](htt | Password | `clickhouse` | !!! note "Note" - Note that all endpoints require a secure TLS connection. + All these endpoints require a secure TLS connection. There are additional endpoints with specific ClickHouse releases to experiment with their differences (ports and user/password are the same as above): From 99cdb1211d3d60dd0e16f38fcff964960103a5a4 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Mon, 18 May 2020 17:37:24 +0300 Subject: [PATCH 386/738] [docs] minor playground.md improvements --- docs/en/getting-started/playground.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/en/getting-started/playground.md b/docs/en/getting-started/playground.md index c552e07eded..bed1618314b 100644 --- a/docs/en/getting-started/playground.md +++ b/docs/en/getting-started/playground.md @@ -8,27 +8,27 @@ toc_title: Playground [ClickHouse Playground](https://play.clickhouse.tech) allows people to experiment with ClickHouse by running queries instantly, without setting up their server or cluster. Several example datasets are available in the Playground as well as sample queries that show ClickHouse features. There's also a selection of ClickHouse LTS releases to experiment with. -ClickHouse Playground gives the experience of m2.small [Managed Service for ClickHouse](https://cloud.yandex.com/services/managed-clickhouse) instance hosted in [Yandex.Cloud](https://cloud.yandex.com/). More information about [cloud providers](../commercial/cloud.md). +ClickHouse Playground gives the experience of m2.small [Managed Service for ClickHouse](https://cloud.yandex.com/services/managed-clickhouse) instance (4 vCPU, 32 GB RAM) hosted in [Yandex.Cloud](https://cloud.yandex.com/). More information about [cloud providers](../commercial/cloud.md). You can make queries to playground using any HTTP client, for example [curl](https://curl.haxx.se) or [wget](https://www.gnu.org/software/wget/), or set up a connection using [JDBC](../interfaces/jdbc.md) or [ODBC](../interfaces/odbc.md) drivers. More information about software products that support ClickHouse is available [here](../interfaces/index.md). ## Credentials -| Parameter | Value | -|:------------------|:----------------------------------------| -| HTTPS endpoint | `https://play-api.clickhouse.tech:8443` | -| Native endpoint | `play-api.clickhouse.tech:9440` | -| User | `playground` | -| Password | `clickhouse` | - -!!! note "Note" - All these endpoints require a secure TLS connection. +| Parameter | Value | +|:--------------------|:----------------------------------------| +| HTTPS endpoint | `https://play-api.clickhouse.tech:8443` | +| Native TCP endpoint | `play-api.clickhouse.tech:9440` | +| User | `playground` | +| Password | `clickhouse` | There are additional endpoints with specific ClickHouse releases to experiment with their differences (ports and user/password are the same as above): * 20.3 LTS: `play-api-v20-3.clickhouse.tech` * 19.14 LTS: `play-api-v19-14.clickhouse.tech` +!!! note "Note" + All these endpoints require a secure TLS connection. + ## Limitations The queries are executed as a read-only user. It implies some limitations: From 452ad9bf2ef48ae9054257fef597681d10aed015 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Mon, 18 May 2020 17:38:36 +0300 Subject: [PATCH 387/738] [docs] add edit link on narrow screens (mobile) (#11014) --- website/js/docs.js | 6 +++++- website/templates/docs/nav.html | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/website/js/docs.js b/website/js/docs.js index c83beb9ba7d..c4460adfdde 100644 --- a/website/js/docs.js +++ b/website/js/docs.js @@ -7,6 +7,7 @@ function onResize() { $('body').attr('data-offset', window_height.toString()); var sidebar = $('#sidebar'); var languages = $('#languages-dropdown') + var edit = $('#edit-link'); var single_page_switch = $('#single-page-switch'); if ((sidebar.width() - single_page_switch.width() - sidebar.find('.dropdown-toggle').width()) >= 36) { single_page_switch.addClass('float-right'); @@ -15,10 +16,13 @@ function onResize() { } if (is_wide) { sidebar.removeClass('collapse'); + edit.detach().appendTo($('#edit-wrapper')); languages.detach().appendTo($('#languages-wrapper')); } else { sidebar.addClass('collapse'); - languages.detach().insertBefore(single_page_switch); + edit.detach().insertBefore(single_page_switch); + edit.addClass('float-right'); + languages.detach().insertBefore(edit); languages.addClass('float-right'); single_page_switch.removeClass('float-right'); } diff --git a/website/templates/docs/nav.html b/website/templates/docs/nav.html index adc7231658f..5e8f92ebb9a 100644 --- a/website/templates/docs/nav.html +++ b/website/templates/docs/nav.html @@ -23,8 +23,8 @@ {% set edit_url = page.edit_url %} {% endif %} {% if edit_url %} -
  • - Edit this article +
    • {% endif %}
  • From c7ec354362028f8ec626caf2707eb2f07fed224d Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Mon, 18 May 2020 17:43:38 +0300 Subject: [PATCH 388/738] [docs] rel=canonical match symlinks in sources (#11013) --- docs/tools/mdx_clickhouse.py | 5 +++++ website/main.html | 4 ++++ website/templates/common_meta.html | 2 +- website/templates/docs/amp.html | 2 +- 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/tools/mdx_clickhouse.py b/docs/tools/mdx_clickhouse.py index 6e41c7d07bb..393658be2d7 100755 --- a/docs/tools/mdx_clickhouse.py +++ b/docs/tools/mdx_clickhouse.py @@ -119,6 +119,11 @@ class PatchedMacrosPlugin(macros.plugin.MacrosPlugin): def on_page_markdown(self, markdown, page, config, files): markdown = super(PatchedMacrosPlugin, self).on_page_markdown(markdown, page, config, files) + + if os.path.islink(page.file.abs_src_path): + lang = config.data['theme']['language'] + page.canonical_url = page.canonical_url.replace(f'/{lang}/', '/en/', 1) + if config.data['extra'].get('version_prefix') or config.data['extra'].get('single_page'): return markdown if self.skip_git_log: diff --git a/website/main.html b/website/main.html index 18aa61d192e..8fcc52034f0 100644 --- a/website/main.html +++ b/website/main.html @@ -11,6 +11,10 @@ {% set url = 'https://clickhouse.tech' + page.abs_url %} {% endif %} +{% if page and page.canonical_url %} + {% set canonical_url = page.canonical_url %} +{% endif %} + {% if page and page.meta and page.meta.title %} {% set title = page.meta.title %} {% elif page and page.title and not page.is_homepage and page.title != 'hidden' %} diff --git a/website/templates/common_meta.html b/website/templates/common_meta.html index adcae49f41b..09bef26d575 100644 --- a/website/templates/common_meta.html +++ b/website/templates/common_meta.html @@ -12,7 +12,7 @@ - + {% if page and not single_page %} {% endif %} diff --git a/website/templates/docs/amp.html b/website/templates/docs/amp.html index 7aac9ea6df1..f3e167644f8 100644 --- a/website/templates/docs/amp.html +++ b/website/templates/docs/amp.html @@ -4,7 +4,7 @@ {{ title }} - + {% include "templates/docs/ld_json.html" %} From 2d71089ad0e062703fa0485c22659184ece455f3 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Mon, 18 May 2020 18:28:33 +0300 Subject: [PATCH 389/738] =?UTF-8?q?[docs]=20improve=20headerlink=20handlin?= =?UTF-8?q?g=20(=C2=B6)=20(#11016)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/tools/website.py | 28 +++++++++++++++++----------- website/css/base.css | 6 ++++++ website/css/docs.css | 1 - website/js/docs.js | 4 ++-- 4 files changed, 25 insertions(+), 14 deletions(-) diff --git a/docs/tools/website.py b/docs/tools/website.py index 050a31c3803..47425df380b 100644 --- a/docs/tools/website.py +++ b/docs/tools/website.py @@ -20,25 +20,31 @@ def adjust_markdown_html(content): content, features='html.parser' ) + for a in soup.find_all('a'): + a_class = a.attrs.get('class') + if a_class and 'headerlink' in a_class: + a.string = '\xa0' for details in soup.find_all('details'): for summary in details.find_all('summary'): if summary.parent != details: summary.extract() details.insert(0, summary) for div in soup.find_all('div'): - div.attrs['role'] = 'alert' div_class = div.attrs.get('class') - for a in div.find_all('a'): - a_class = a.attrs.get('class') - if a_class: - a.attrs['class'] = a_class + ['alert-link'] - else: - a.attrs['class'] = 'alert-link' + is_admonition = div_class and 'admonition' in div.attrs.get('class') + if is_admonition: + for a in div.find_all('a'): + a_class = a.attrs.get('class') + if a_class: + a.attrs['class'] = a_class + ['alert-link'] + else: + a.attrs['class'] = 'alert-link' for p in div.find_all('p'): p_class = p.attrs.get('class') - if p_class and ('admonition-title' in p_class): - p.attrs['class'] = p_class + ['alert-heading', 'display-5', 'mb-2'] - if div_class and 'admonition' in div.attrs.get('class'): + if is_admonition and p_class and ('admonition-title' in p_class): + p.attrs['class'] = p_class + ['alert-heading', 'display-6', 'mb-2'] + if is_admonition: + div.attrs['role'] = 'alert' if ('info' in div_class) or ('note' in div_class): mode = 'alert-primary' elif ('attention' in div_class) or ('warning' in div_class): @@ -49,7 +55,7 @@ def adjust_markdown_html(content): mode = 'alert-info' else: mode = 'alert-secondary' - div.attrs['class'] = div_class + ['alert', 'lead', 'pb-0', 'mb-4', mode] + div.attrs['class'] = div_class + ['alert', 'pb-0', 'mb-4', mode] return str(soup) diff --git a/website/css/base.css b/website/css/base.css index 1546337d4cf..a63d1d7b166 100644 --- a/website/css/base.css +++ b/website/css/base.css @@ -39,6 +39,12 @@ a:hover, a:active { line-height: 1.2; } +.display-6 { + font-size: 1.75rem; + font-weight: 300; + line-height: 1.2; +} + .bg-dark-alt, .bg-dark-alt:focus { background: #36363F; } diff --git a/website/css/docs.css b/website/css/docs.css index dfbf6d12b2d..8d041ec9320 100644 --- a/website/css/docs.css +++ b/website/css/docs.css @@ -33,7 +33,6 @@ summary { } .headerlink { - display: none; text-decoration: none !important; margin-left: .5rem; } diff --git a/website/js/docs.js b/website/js/docs.js index c4460adfdde..f43b8f06af5 100644 --- a/website/js/docs.js +++ b/website/js/docs.js @@ -82,10 +82,10 @@ $(document).ready(function () { var headers = $('#content h1, #content h2, #content h3, #content h4, #content h5, #content h6'); headers.mouseenter(function() { - $(this).find('.headerlink').show(); + $(this).find('.headerlink').html('¶'); }); headers.mouseleave(function() { - $(this).find('.headerlink').hide(); + $(this).find('.headerlink').html(' '); }); if ($('#docsearch-input').length) { From deeae6645502cab4fef80250ed3c9ca6d586ddcc Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 18 May 2020 18:52:00 +0300 Subject: [PATCH 390/738] Fix clang build. --- src/Processors/Executors/PipelineExecutingBlockInputStream.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/Executors/PipelineExecutingBlockInputStream.h b/src/Processors/Executors/PipelineExecutingBlockInputStream.h index 773332f69c8..781396159e6 100644 --- a/src/Processors/Executors/PipelineExecutingBlockInputStream.h +++ b/src/Processors/Executors/PipelineExecutingBlockInputStream.h @@ -13,7 +13,7 @@ class PipelineExecutingBlockInputStream : public IBlockInputStream { public: explicit PipelineExecutingBlockInputStream(QueryPipeline pipeline_); - ~PipelineExecutingBlockInputStream(); + ~PipelineExecutingBlockInputStream() override; String getName() const override { return "PipelineExecuting"; } Block getHeader() const override; From 0ce9afd1e3ee53539267d9ce6893710e22ed9b46 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 18 May 2020 19:19:43 +0300 Subject: [PATCH 391/738] Try fix tests. --- src/Processors/Executors/PipelineExecutingBlockInputStream.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Processors/Executors/PipelineExecutingBlockInputStream.cpp b/src/Processors/Executors/PipelineExecutingBlockInputStream.cpp index e7846f31bd5..75324cb25b8 100644 --- a/src/Processors/Executors/PipelineExecutingBlockInputStream.cpp +++ b/src/Processors/Executors/PipelineExecutingBlockInputStream.cpp @@ -31,6 +31,9 @@ void PipelineExecutingBlockInputStream::readPrefixImpl() Block PipelineExecutingBlockInputStream::readImpl() { + if (!executor) + executor = std::make_unique(*pipeline); + Block block; while (executor->pull(block)) { From 15f704095a8479c014d12d3f125f9457bd9829af Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Mon, 18 May 2020 19:44:11 +0300 Subject: [PATCH 392/738] [docs] support dark mode in AMP version (#11019) --- website/css/docs.css | 7 ++++++- website/templates/docs/amp.html | 6 +++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/website/css/docs.css b/website/css/docs.css index 8d041ec9320..18bf40de13b 100644 --- a/website/css/docs.css +++ b/website/css/docs.css @@ -170,11 +170,15 @@ summary { } @media (prefers-color-scheme: dark) { - body[data-spy] { + body[data-spy], body.amp, #to-full-website { background: #1c1c1c; color: #efefef; } + .invert-dark { + filter: invert(100%); + } + #sidebar { background: #333; } @@ -190,6 +194,7 @@ summary { #content td { border: 1px solid #444451; } + #content code { background: #444; color: #eee; diff --git a/website/templates/docs/amp.html b/website/templates/docs/amp.html index f3e167644f8..a60270d23d0 100644 --- a/website/templates/docs/amp.html +++ b/website/templates/docs/amp.html @@ -11,12 +11,12 @@ - +
    - +
    @@ -31,7 +31,7 @@ {% include "templates/docs/footer.html" %}
    -
    +
    To full website
    From 8552d5e9b75d196615a8cf2e8dce00b4072f64b6 Mon Sep 17 00:00:00 2001 From: Vladimir Chebotarev Date: Mon, 18 May 2020 19:49:06 +0300 Subject: [PATCH 393/738] Fixed link to external dictionaries --- docs/ru/sql-reference/functions/ext-dict-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/functions/ext-dict-functions.md b/docs/ru/sql-reference/functions/ext-dict-functions.md index f8d2ebb2b98..a260ec1e16e 100644 --- a/docs/ru/sql-reference/functions/ext-dict-functions.md +++ b/docs/ru/sql-reference/functions/ext-dict-functions.md @@ -1,6 +1,6 @@ # Функции для работы с внешними словарями {#ext_dict_functions} -Информацию о подключении и настройке внешних словарей смотрите в разделе [Внешние словари](../../sql-reference/functions/ext-dict-functions.md). +Информацию о подключении и настройке внешних словарей смотрите в разделе [Внешние словари](../../sql-reference/dictionaries/external-dictionaries/external-dicts.md). ## dictGet {#dictget} From 59a1c230cdef13a1d2ddf34374baaddb2f4d4e36 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Mon, 18 May 2020 20:17:29 +0200 Subject: [PATCH 394/738] Fixing tests/queries/0_stateless/00979_live_view_watch_continuous_aggregates.py. --- ...9_live_view_watch_continuous_aggregates.py | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/queries/0_stateless/00979_live_view_watch_continuous_aggregates.py b/tests/queries/0_stateless/00979_live_view_watch_continuous_aggregates.py index 5aec9ac0a8b..d184fea7183 100755 --- a/tests/queries/0_stateless/00979_live_view_watch_continuous_aggregates.py +++ b/tests/queries/0_stateless/00979_live_view_watch_continuous_aggregates.py @@ -33,28 +33,28 @@ with client(name='client1>', log=log) as client1, client(name='client2>', log=lo client1.expect(r'0.*1' + end_of_block) client2.send("INSERT INTO test.mt VALUES ('2019-01-01 00:00:00','New York',60),('2019-01-01 00:10:00','New York',70)") client2.expect(prompt) - client1.expect(r'"2019-01-01 00:00:00","New York",65,2') + client1.expect(r'"2019-01-01 00:00:00","New York",65') client2.send("INSERT INTO test.mt VALUES ('2019-01-01 00:00:00','Moscow',30),('2019-01-01 00:10:00', 'Moscow', 40)") client2.expect(prompt) - client1.expect(r'"2019-01-01 00:00:00","Moscow",35,3') - client1.expect(r'"2019-01-01 00:00:00","New York",65,3') + client1.expect(r'"2019-01-01 00:00:00","Moscow",35') + client1.expect(r'"2019-01-01 00:00:00","New York",65') client2.send("INSERT INTO test.mt VALUES ('2019-01-02 00:00:00','New York',50),('2019-01-02 00:10:00','New York',60)") client2.expect(prompt) - client1.expect(r'"2019-01-01 00:00:00","Moscow",35,4') - client1.expect(r'"2019-01-01 00:00:00","New York",65,4') - client1.expect(r'"2019-01-02 00:00:00","New York",55,4') + client1.expect(r'"2019-01-01 00:00:00","Moscow",35') + client1.expect(r'"2019-01-01 00:00:00","New York",65') + client1.expect(r'"2019-01-02 00:00:00","New York",55') client2.send("INSERT INTO test.mt VALUES ('2019-01-02 00:00:00','Moscow',20),('2019-01-02 00:10:00', 'Moscow', 30)") client2.expect(prompt) - client1.expect(r'"2019-01-01 00:00:00","Moscow",35,5') - client1.expect(r'"2019-01-01 00:00:00","New York",65,5') - client1.expect(r'"2019-01-02 00:00:00","Moscow",25,5') - client1.expect(r'"2019-01-02 00:00:00","New York",55,5') + client1.expect(r'"2019-01-01 00:00:00","Moscow",35') + client1.expect(r'"2019-01-01 00:00:00","New York",65') + client1.expect(r'"2019-01-02 00:00:00","Moscow",25') + client1.expect(r'"2019-01-02 00:00:00","New York",55') client2.send("INSERT INTO test.mt VALUES ('2019-01-02 00:03:00','New York',40),('2019-01-02 00:06:00','New York',30)") client2.expect(prompt) - client1.expect(r'"2019-01-01 00:00:00","Moscow",35,6') - client1.expect(r'"2019-01-01 00:00:00","New York",65,6') - client1.expect(r'"2019-01-02 00:00:00","Moscow",25,6') - client1.expect(r'"2019-01-02 00:00:00","New York",45,6') + client1.expect(r'"2019-01-01 00:00:00","Moscow",35') + client1.expect(r'"2019-01-01 00:00:00","New York",65') + client1.expect(r'"2019-01-02 00:00:00","Moscow",25') + client1.expect(r'"2019-01-02 00:00:00","New York",45') # send Ctrl-C client1.send('\x03', eol='') match = client1.expect('(%s)|([#\$] )' % prompt) From bdbfa08e024aac9db21c8399e28c18003138b724 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 18 May 2020 21:18:37 +0300 Subject: [PATCH 395/738] Truncate system.*_log by default in tests Enable it by default due to: - it is a sane default - avoids modifying docker images --- tests/clickhouse-test | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 1a2735296e8..76a61e1d558 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -234,6 +234,11 @@ def run_tests_array(all_tests_with_params): clickhouse_proc = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE) clickhouse_proc.communicate("SELECT 'Running test {suite}/{case} from pid={pid}';".format(pid = os.getpid(), case = case, suite = suite)) + if not args.no_system_log_cleanup: + for table in ['query_log', 'query_thread_log', 'trace_log', 'metric_log']: + clickhouse_proc = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE) + clickhouse_proc.communicate("TRUNCATE TABLE IF EXISTS system.{}".format(table)) + reference_file = os.path.join(suite_dir, name) + '.reference' stdout_file = os.path.join(suite_tmp_dir, name) + '.stdout' stderr_file = os.path.join(suite_tmp_dir, name) + '.stderr' @@ -564,6 +569,7 @@ if __name__ == '__main__': parser.add_argument('--stop', action='store_true', default=None, dest='stop', help='Stop on network errors') parser.add_argument('--order', default='desc', choices=['asc', 'desc', 'random'], help='Run order') parser.add_argument('--testname', action='store_true', default=None, dest='testname', help='Make query with test name before test run') + parser.add_argument('--no-system-log-cleanup', action='store_true', default=None, help='Do not cleanup system.*_log tables') parser.add_argument('--hung-check', action='store_true', default=False) parser.add_argument('--force-color', action='store_true', default=False) parser.add_argument('--database', help='Database for tests (random name test_XXXXXX by default)') From 32071ba08096c12545104be46fe8595073e86006 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 18 May 2020 21:18:37 +0300 Subject: [PATCH 396/738] Do not add event_date/event_time condition for system.*_log Also some of them can be even more simplified, since the system.*_log are empty most of the time there should not be any conditions at all (i.e. `query` LIKE and so on). --- .../00634_performance_introspection_and_logging.sh | 6 +++--- .../00933_test_fix_extra_seek_on_compressed_cache.sh | 2 +- .../0_stateless/00956_sensitive_data_masking.sh | 6 +++--- tests/queries/0_stateless/00974_query_profiler.sql | 4 ++-- .../0_stateless/00974_text_log_table_not_empty.sh | 2 +- .../01070_exception_code_in_query_log_table.sql | 2 +- tests/queries/0_stateless/01091_num_threads.sql | 12 ++++++------ tests/queries/0_stateless/01092_memory_profiler.sql | 2 +- tests/queries/0_stateless/01198_client_quota_key.sh | 2 +- .../0_stateless/01231_log_queries_min_type.sql | 6 +++--- 10 files changed, 22 insertions(+), 22 deletions(-) diff --git a/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh b/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh index 5173b5f5772..684287da761 100755 --- a/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh +++ b/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh @@ -47,7 +47,7 @@ SELECT threads_realtime >= threads_time_user_system_io, any(length(thread_ids)) >= 1 FROM - (SELECT * FROM system.query_log PREWHERE query='$heavy_cpu_query' WHERE event_date >= today()-1 AND type=2 ORDER BY event_time DESC LIMIT 1) + (SELECT * FROM system.query_log PREWHERE query='$heavy_cpu_query' WHERE type=2 ORDER BY event_time DESC LIMIT 1) ARRAY JOIN ProfileEvents.Names AS PN, ProfileEvents.Values AS PV" # Check per-thread and per-query ProfileEvents consistency @@ -58,7 +58,7 @@ SELECT PN, PVq, PVt FROM SELECT PN, sum(PV) AS PVt FROM system.query_thread_log ARRAY JOIN ProfileEvents.Names AS PN, ProfileEvents.Values AS PV - WHERE event_date >= today()-1 AND query_id='$query_id' + WHERE query_id='$query_id' GROUP BY PN ) js1 ANY INNER JOIN @@ -66,7 +66,7 @@ ANY INNER JOIN SELECT PN, PV AS PVq FROM system.query_log ARRAY JOIN ProfileEvents.Names AS PN, ProfileEvents.Values AS PV - WHERE event_date >= today()-1 AND query_id='$query_id' + WHERE query_id='$query_id' ) js2 USING PN WHERE diff --git a/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh b/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh index 1f7571a2404..b0fd9a70bd4 100755 --- a/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh +++ b/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh @@ -19,7 +19,7 @@ $CLICKHOUSE_CLIENT --use_uncompressed_cache=1 --query_id="test-query-uncompresse sleep 1 $CLICKHOUSE_CLIENT --query="SYSTEM FLUSH LOGS" -$CLICKHOUSE_CLIENT --query="SELECT ProfileEvents.Values[indexOf(ProfileEvents.Names, 'Seek')], ProfileEvents.Values[indexOf(ProfileEvents.Names, 'ReadCompressedBytes')], ProfileEvents.Values[indexOf(ProfileEvents.Names, 'UncompressedCacheHits')] AS hit FROM system.query_log WHERE (query_id = 'test-query-uncompressed-cache') AND (type = 2) AND event_date >= yesterday() ORDER BY event_time DESC LIMIT 1" +$CLICKHOUSE_CLIENT --query="SELECT ProfileEvents.Values[indexOf(ProfileEvents.Names, 'Seek')], ProfileEvents.Values[indexOf(ProfileEvents.Names, 'ReadCompressedBytes')], ProfileEvents.Values[indexOf(ProfileEvents.Names, 'UncompressedCacheHits')] AS hit FROM system.query_log WHERE (query_id = 'test-query-uncompressed-cache') AND (type = 2) ORDER BY event_time DESC LIMIT 1" $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS small_table" diff --git a/tests/queries/0_stateless/00956_sensitive_data_masking.sh b/tests/queries/0_stateless/00956_sensitive_data_masking.sh index 0f76c34eaff..c492fd35b89 100755 --- a/tests/queries/0_stateless/00956_sensitive_data_masking.sh +++ b/tests/queries/0_stateless/00956_sensitive_data_masking.sh @@ -95,7 +95,7 @@ echo 7 # and finally querylog $CLICKHOUSE_CLIENT \ --server_logs_file=/dev/null \ - --query="select * from system.query_log where event_time>now() - 10 and query like '%TOPSECRET%';" + --query="select * from system.query_log where query like '%TOPSECRET%';" rm -f $tmp_file >/dev/null 2>&1 @@ -117,8 +117,8 @@ sleep 0.1; echo 9 $CLICKHOUSE_CLIENT \ --server_logs_file=/dev/null \ - --query="SELECT if( count() > 0, 'text_log non empty', 'text_log empty') FROM system.text_log WHERE event_time>now() - 60 and message like '%find_me%'; - select * from system.text_log where event_time>now() - 60 and message like '%TOPSECRET=TOPSECRET%';" --ignore-error --multiquery + --query="SELECT if( count() > 0, 'text_log non empty', 'text_log empty') FROM system.text_log WHERE message like '%find_me%'; + select * from system.text_log where message like '%TOPSECRET=TOPSECRET%';" --ignore-error --multiquery echo 'finish' rm -f $tmp_file >/dev/null 2>&1 diff --git a/tests/queries/0_stateless/00974_query_profiler.sql b/tests/queries/0_stateless/00974_query_profiler.sql index 4d0b46dd51f..9e2723c67d8 100644 --- a/tests/queries/0_stateless/00974_query_profiler.sql +++ b/tests/queries/0_stateless/00974_query_profiler.sql @@ -5,7 +5,7 @@ SET log_queries = 1; SELECT sleep(0.5), ignore('test real time query profiler'); SET log_queries = 0; SYSTEM FLUSH LOGS; -WITH addressToSymbol(arrayJoin(trace)) AS symbol SELECT count() > 0 FROM system.trace_log t WHERE event_date >= yesterday() AND query_id = (SELECT query_id FROM system.query_log WHERE event_date >= yesterday() AND query LIKE '%test real time query profiler%' AND query NOT LIKE '%system%' ORDER BY event_time DESC LIMIT 1) AND symbol LIKE '%FunctionSleep%'; +WITH addressToSymbol(arrayJoin(trace)) AS symbol SELECT count() > 0 FROM system.trace_log t WHERE query_id = (SELECT query_id FROM system.query_log WHERE query LIKE '%test real time query profiler%' AND query NOT LIKE '%system%' ORDER BY event_time DESC LIMIT 1) AND symbol LIKE '%FunctionSleep%'; SET query_profiler_real_time_period_ns = 0; SET query_profiler_cpu_time_period_ns = 1000000; @@ -13,4 +13,4 @@ SET log_queries = 1; SELECT count(), ignore('test cpu time query profiler') FROM numbers(1000000000); SET log_queries = 0; SYSTEM FLUSH LOGS; -WITH addressToSymbol(arrayJoin(trace)) AS symbol SELECT count() > 0 FROM system.trace_log t WHERE event_date >= yesterday() AND query_id = (SELECT query_id FROM system.query_log WHERE event_date >= yesterday() AND query LIKE '%test cpu time query profiler%' AND query NOT LIKE '%system%' ORDER BY event_time DESC LIMIT 1) AND symbol LIKE '%Source%'; +WITH addressToSymbol(arrayJoin(trace)) AS symbol SELECT count() > 0 FROM system.trace_log t WHERE query_id = (SELECT query_id FROM system.query_log WHERE query LIKE '%test cpu time query profiler%' AND query NOT LIKE '%system%' ORDER BY event_time DESC LIMIT 1) AND symbol LIKE '%Source%'; diff --git a/tests/queries/0_stateless/00974_text_log_table_not_empty.sh b/tests/queries/0_stateless/00974_text_log_table_not_empty.sh index c3cde4c08bb..149f0668bd1 100755 --- a/tests/queries/0_stateless/00974_text_log_table_not_empty.sh +++ b/tests/queries/0_stateless/00974_text_log_table_not_empty.sh @@ -10,7 +10,7 @@ do ${CLICKHOUSE_CLIENT} --query="SYSTEM FLUSH LOGS" sleep 0.1; -if [[ $($CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL" -d "SELECT count() > 0 FROM system.text_log WHERE position(system.text_log.message, 'SELECT 6103') > 0 AND event_date >= yesterday()") == 1 ]]; then echo 1; exit; fi; +if [[ $($CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL" -d "SELECT count() > 0 FROM system.text_log WHERE position(system.text_log.message, 'SELECT 6103') > 0") == 1 ]]; then echo 1; exit; fi; done; diff --git a/tests/queries/0_stateless/01070_exception_code_in_query_log_table.sql b/tests/queries/0_stateless/01070_exception_code_in_query_log_table.sql index 2c99ba54112..e1e81614ab7 100644 --- a/tests/queries/0_stateless/01070_exception_code_in_query_log_table.sql +++ b/tests/queries/0_stateless/01070_exception_code_in_query_log_table.sql @@ -3,5 +3,5 @@ SELECT * FROM test_table_for_01070_exception_code_in_query_log_table; -- { serve CREATE TABLE test_table_for_01070_exception_code_in_query_log_table (value UInt64) ENGINE=Memory(); SELECT * FROM test_table_for_01070_exception_code_in_query_log_table; SYSTEM FLUSH LOGS; -SELECT exception_code FROM system.query_log WHERE query = 'SELECT * FROM test_table_for_01070_exception_code_in_query_log_table' AND event_date >= yesterday() AND event_time > now() - INTERVAL 5 MINUTE ORDER BY exception_code; +SELECT exception_code FROM system.query_log WHERE query = 'SELECT * FROM test_table_for_01070_exception_code_in_query_log_table' ORDER BY exception_code; DROP TABLE IF EXISTS test_table_for_01070_exception_code_in_query_log_table; diff --git a/tests/queries/0_stateless/01091_num_threads.sql b/tests/queries/0_stateless/01091_num_threads.sql index 876a2d15d1a..a93568fcee5 100644 --- a/tests/queries/0_stateless/01091_num_threads.sql +++ b/tests/queries/0_stateless/01091_num_threads.sql @@ -8,13 +8,13 @@ WITH ( SELECT query_id FROM system.query_log - WHERE (query = 'SELECT 1') AND (event_date >= (today() - 1)) + WHERE (query = 'SELECT 1') ORDER BY event_time DESC LIMIT 1 ) AS id SELECT uniqExact(thread_id) FROM system.query_thread_log -WHERE (event_date >= (today() - 1)) AND (query_id = id) AND (thread_id != master_thread_id); +WHERE (query_id = id) AND (thread_id != master_thread_id); select sum(number) from numbers(1000000); SYSTEM FLUSH LOGS; @@ -23,13 +23,13 @@ WITH ( SELECT query_id FROM system.query_log - WHERE (query = 'SELECT sum(number) FROM numbers(1000000)') AND (event_date >= (today() - 1)) + WHERE (query = 'SELECT sum(number) FROM numbers(1000000)') ORDER BY event_time DESC LIMIT 1 ) AS id SELECT uniqExact(thread_id) FROM system.query_thread_log -WHERE (event_date >= (today() - 1)) AND (query_id = id) AND (thread_id != master_thread_id); +WHERE (query_id = id) AND (thread_id != master_thread_id); select sum(number) from numbers_mt(1000000); SYSTEM FLUSH LOGS; @@ -38,10 +38,10 @@ WITH ( SELECT query_id FROM system.query_log - WHERE (query = 'SELECT sum(number) FROM numbers_mt(1000000)') AND (event_date >= (today() - 1)) + WHERE (query = 'SELECT sum(number) FROM numbers_mt(1000000)') ORDER BY event_time DESC LIMIT 1 ) AS id SELECT uniqExact(thread_id) > 2 FROM system.query_thread_log -WHERE (event_date >= (today() - 1)) AND (query_id = id) AND (thread_id != master_thread_id); +WHERE (query_id = id) AND (thread_id != master_thread_id); diff --git a/tests/queries/0_stateless/01092_memory_profiler.sql b/tests/queries/0_stateless/01092_memory_profiler.sql index c20b5c79cdb..980f7f73d5d 100644 --- a/tests/queries/0_stateless/01092_memory_profiler.sql +++ b/tests/queries/0_stateless/01092_memory_profiler.sql @@ -3,4 +3,4 @@ SET allow_introspection_functions = 1; SET memory_profiler_step = 1000000; SELECT ignore(groupArray(number), 'test memory profiler') FROM numbers(10000000); SYSTEM FLUSH LOGS; -WITH addressToSymbol(arrayJoin(trace)) AS symbol SELECT count() > 0 FROM system.trace_log t WHERE event_date >= yesterday() AND trace_type = 'Memory' AND query_id = (SELECT query_id FROM system.query_log WHERE event_date >= yesterday() AND query LIKE '%test memory profiler%' ORDER BY event_time DESC LIMIT 1); +WITH addressToSymbol(arrayJoin(trace)) AS symbol SELECT count() > 0 FROM system.trace_log t WHERE trace_type = 'Memory' AND query_id = (SELECT query_id FROM system.query_log WHERE query LIKE '%test memory profiler%' ORDER BY event_time DESC LIMIT 1); diff --git a/tests/queries/0_stateless/01198_client_quota_key.sh b/tests/queries/0_stateless/01198_client_quota_key.sh index f4b66aea6ac..b3bc845cd06 100755 --- a/tests/queries/0_stateless/01198_client_quota_key.sh +++ b/tests/queries/0_stateless/01198_client_quota_key.sh @@ -3,4 +3,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh -$CLICKHOUSE_CLIENT --quota_key Hello --query_id test_quota_key --log_queries 1 --multiquery --query "SELECT 1; SYSTEM FLUSH LOGS; SELECT DISTINCT quota_key FROM system.query_log WHERE event_date >= yesterday() AND event_time >= now() - 300 AND query_id = 'test_quota_key'" +$CLICKHOUSE_CLIENT --quota_key Hello --query_id test_quota_key --log_queries 1 --multiquery --query "SELECT 1; SYSTEM FLUSH LOGS; SELECT DISTINCT quota_key FROM system.query_log WHERE query_id = 'test_quota_key'" diff --git a/tests/queries/0_stateless/01231_log_queries_min_type.sql b/tests/queries/0_stateless/01231_log_queries_min_type.sql index f2229c94a8a..565a5880b84 100644 --- a/tests/queries/0_stateless/01231_log_queries_min_type.sql +++ b/tests/queries/0_stateless/01231_log_queries_min_type.sql @@ -2,14 +2,14 @@ set log_queries=1; select '01231_log_queries_min_type/QUERY_START'; system flush logs; -select count() from system.query_log where query like '%01231_log_queries_min_type/%' and query not like '%system.query_log%' and event_date = today() and event_time >= now() - interval 1 minute; +select count() from system.query_log where query like '%01231_log_queries_min_type/%' and query not like '%system.query_log%'; set log_queries_min_type='EXCEPTION_BEFORE_START'; select '01231_log_queries_min_type/EXCEPTION_BEFORE_START'; system flush logs; -select count() from system.query_log where query like '%01231_log_queries_min_type/%' and query not like '%system.query_log%' and event_date = today() and event_time >= now() - interval 1 minute; +select count() from system.query_log where query like '%01231_log_queries_min_type/%' and query not like '%system.query_log%'; set log_queries_min_type='EXCEPTION_WHILE_PROCESSING'; select '01231_log_queries_min_type/', max(number) from system.numbers limit 1e6 settings max_rows_to_read='100K'; -- { serverError 158; } system flush logs; -select count() from system.query_log where query like '%01231_log_queries_min_type/%' and query not like '%system.query_log%' and event_date = today() and event_time >= now() - interval 1 minute; +select count() from system.query_log where query like '%01231_log_queries_min_type/%' and query not like '%system.query_log%'; From bd86f15680c51f7b48d35e73e490882814c54006 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 18 May 2020 21:18:37 +0300 Subject: [PATCH 397/738] Use QueryFinish over magic const in tests Find all usages with: $ git grep 'type[ ]*=[ ]*2' tests/queries/0_stateless/ Sure this is not all... --- .../0_stateless/00634_performance_introspection_and_logging.sh | 2 +- .../00933_test_fix_extra_seek_on_compressed_cache.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh b/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh index 684287da761..efb4b3569fb 100755 --- a/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh +++ b/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh @@ -47,7 +47,7 @@ SELECT threads_realtime >= threads_time_user_system_io, any(length(thread_ids)) >= 1 FROM - (SELECT * FROM system.query_log PREWHERE query='$heavy_cpu_query' WHERE type=2 ORDER BY event_time DESC LIMIT 1) + (SELECT * FROM system.query_log PREWHERE query='$heavy_cpu_query' WHERE type='QueryFinish' ORDER BY event_time DESC LIMIT 1) ARRAY JOIN ProfileEvents.Names AS PN, ProfileEvents.Values AS PV" # Check per-thread and per-query ProfileEvents consistency diff --git a/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh b/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh index b0fd9a70bd4..9e32c30ce20 100755 --- a/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh +++ b/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh @@ -19,7 +19,7 @@ $CLICKHOUSE_CLIENT --use_uncompressed_cache=1 --query_id="test-query-uncompresse sleep 1 $CLICKHOUSE_CLIENT --query="SYSTEM FLUSH LOGS" -$CLICKHOUSE_CLIENT --query="SELECT ProfileEvents.Values[indexOf(ProfileEvents.Names, 'Seek')], ProfileEvents.Values[indexOf(ProfileEvents.Names, 'ReadCompressedBytes')], ProfileEvents.Values[indexOf(ProfileEvents.Names, 'UncompressedCacheHits')] AS hit FROM system.query_log WHERE (query_id = 'test-query-uncompressed-cache') AND (type = 2) ORDER BY event_time DESC LIMIT 1" +$CLICKHOUSE_CLIENT --query="SELECT ProfileEvents.Values[indexOf(ProfileEvents.Names, 'Seek')], ProfileEvents.Values[indexOf(ProfileEvents.Names, 'ReadCompressedBytes')], ProfileEvents.Values[indexOf(ProfileEvents.Names, 'UncompressedCacheHits')] AS hit FROM system.query_log WHERE (query_id = 'test-query-uncompressed-cache') AND (type = 'QueryFinish') ORDER BY event_time DESC LIMIT 1" $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS small_table" From 4ce0ef6055872b3070aad66e69b90808f3254035 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 18 May 2020 21:40:01 +0300 Subject: [PATCH 398/738] Fix tests. --- src/Processors/Executors/PipelineExecutingBlockInputStream.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/Executors/PipelineExecutingBlockInputStream.h b/src/Processors/Executors/PipelineExecutingBlockInputStream.h index 781396159e6..7555e1dec97 100644 --- a/src/Processors/Executors/PipelineExecutingBlockInputStream.h +++ b/src/Processors/Executors/PipelineExecutingBlockInputStream.h @@ -32,8 +32,8 @@ protected: Block readImpl() override; private: - std::unique_ptr executor; std::unique_ptr pipeline; + std::unique_ptr executor; }; } From 08ff5b59663ed0e82d033bc5f5cf72afcb4ccab1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 18 May 2020 22:13:13 +0300 Subject: [PATCH 399/738] Add NCHAR and NVARCHAR synonims --- src/DataTypes/DataTypeString.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/DataTypes/DataTypeString.cpp b/src/DataTypes/DataTypeString.cpp index a7bfb2b635d..4a9a6e9ab78 100644 --- a/src/DataTypes/DataTypeString.cpp +++ b/src/DataTypes/DataTypeString.cpp @@ -376,8 +376,10 @@ void registerDataTypeString(DataTypeFactory & factory) /// These synonyms are added for compatibility. factory.registerAlias("CHAR", "String", DataTypeFactory::CaseInsensitive); + factory.registerAlias("NCHAR", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("CHARACTER", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("VARCHAR", "String", DataTypeFactory::CaseInsensitive); + factory.registerAlias("NVARCHAR", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("VARCHAR2", "String", DataTypeFactory::CaseInsensitive); /// Oracle factory.registerAlias("TEXT", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("TINYTEXT", "String", DataTypeFactory::CaseInsensitive); From d2211f3acb891f368d4a1887d9698998df003466 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Mon, 18 May 2020 22:20:49 +0300 Subject: [PATCH 400/738] Remove incorrect comment --- website/templates/index/performance.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/templates/index/performance.html b/website/templates/index/performance.html index 505059e42c6..230eb0de974 100644 --- a/website/templates/index/performance.html +++ b/website/templates/index/performance.html @@ -38,7 +38,7 @@
  • ClickHouse and Vertica comparison
    zhtsh (machine translation from Chinese)
  • MySQL vs ColumnStore vs ClickHouse
    Mafiree (machine translation from Chinese)
    • + rel="external nofollow noreferrer" target="_blank" class="text-reset">MySQL vs ColumnStore vs ClickHouse
    Mafiree
    From a54773f94739fa48934f446c8dd34f2832819ffd Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 18 May 2020 22:33:12 +0300 Subject: [PATCH 401/738] Add more logs to debug trace_log overflow --- docker/test/performance-comparison/compare.sh | 2 +- src/Interpreters/SystemLog.h | 65 ++++++++++++++----- 2 files changed, 51 insertions(+), 16 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 05c055107ac..023c931de1c 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -542,7 +542,7 @@ case "$stage" in # to collect the logs. Prefer not to restart, because addresses might change # and we won't be able to process trace_log data. Start in a subshell, so that # it doesn't interfere with the watchdog through `wait`. - ( time get_profiles || restart || get_profiles ||: ) + ( time get_profiles || restart || get_profiles ||: ) 2>> profile-errors.log # Kill the whole process group, because somehow when the subshell is killed, # the sleep inside remains alive and orphaned. diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h index f5a024c7768..10dcb84c1ce 100644 --- a/src/Interpreters/SystemLog.h +++ b/src/Interpreters/SystemLog.h @@ -165,9 +165,9 @@ private: // Queue is bounded. But its size is quite large to not block in all normal cases. std::vector queue; // An always-incrementing index of the first message currently in the queue. - // We use it to give a global sequential index to every message, so that we can wait - // until a particular message is flushed. This is used to implement synchronous log - // flushing for SYSTEM FLUSH LOGS. + // We use it to give a global sequential index to every message, so that we + // can wait until a particular message is flushed. This is used to implement + // synchronous log flushing for SYSTEM FLUSH LOGS. uint64_t queue_front_index = 0; bool is_shutdown = false; std::condition_variable flush_event; @@ -175,6 +175,8 @@ private: uint64_t requested_flush_before = 0; // Flushed log up to this index, exclusive uint64_t flushed_before = 0; + // Logged overflow message at this queue front index + uint64_t logged_queue_full_at_index = -1; void savingThreadFunction(); @@ -244,9 +246,22 @@ void SystemLog::add(const LogElement & element) if (queue.size() >= DBMS_SYSTEM_LOG_QUEUE_SIZE) { - // TextLog sets its logger level to 0, so this log is a noop and there - // is no recursive logging. - LOG_ERROR(log, "Queue is full for system log '" + demangle(typeid(*this).name()) + "'."); + // Ignore all further entries until the queue is flushed. + // Log a message about that. Don't spam it -- this might be especially + // problematic in case of trace log. Remember what the front index of the + // queue was when we last logged the message. If it changed, it means the + // queue was flushed, and we can log again. + if (queue_front_index != logged_queue_full_at_index) + { + logged_queue_full_at_index = queue_front_index; + + // TextLog sets its logger level to 0, so this log is a noop and + // there is no recursive logging. + LOG_ERROR(log, "Queue is full for system log '" + << demangle(typeid(*this).name()) << "'" + << " at " << queue_front_index); + } + return; } @@ -325,9 +340,15 @@ void SystemLog::savingThreadFunction() uint64_t to_flush_end = 0; { + LOG_TRACE(log, "Sleeping"); std::unique_lock lock(mutex); - flush_event.wait_for(lock, std::chrono::milliseconds(flush_interval_milliseconds), - [&] () { return requested_flush_before > flushed_before || is_shutdown; }); + const bool predicate = flush_event.wait_for(lock, + std::chrono::milliseconds(flush_interval_milliseconds), + [&] () { + return requested_flush_before > flushed_before + || is_shutdown; + } + ); queue_front_index += queue.size(); to_flush_end = queue_front_index; @@ -337,6 +358,13 @@ void SystemLog::savingThreadFunction() queue.swap(to_flush); exit_this_thread = is_shutdown; + + LOG_TRACE(log, "Woke up" + << (predicate ? " by condition" : " by timeout (" + + toString(flush_interval_milliseconds) + " ms)") + << ", " << to_flush.size() << " elements to flush" + << " up to " << to_flush_end + << (is_shutdown ? ", shutdown requested" : "")); } if (to_flush.empty()) @@ -351,6 +379,7 @@ void SystemLog::savingThreadFunction() tryLogCurrentException(__PRETTY_FUNCTION__); } } + LOG_TRACE(log, "Terminating"); } @@ -359,11 +388,13 @@ void SystemLog::flushImpl(const std::vector & to_flush, { try { - LOG_TRACE(log, "Flushing system log"); + LOG_TRACE(log, "Flushing system log, " + << to_flush.size() << " entries to flush"); - /// We check for existence of the table and create it as needed at every flush. - /// This is done to allow user to drop the table at any moment (new empty table will be created automatically). - /// BTW, flush method is called from single thread. + /// We check for existence of the table and create it as needed at every + /// flush. This is done to allow user to drop the table at any moment + /// (new empty table will be created automatically). BTW, flush method + /// is called from single thread. prepareTable(); Block block = LogElement::createBlock(); @@ -389,9 +420,13 @@ void SystemLog::flushImpl(const std::vector & to_flush, tryLogCurrentException(__PRETTY_FUNCTION__); } - std::unique_lock lock(mutex); - flushed_before = to_flush_end; - flush_event.notify_all(); + { + std::unique_lock lock(mutex); + flushed_before = to_flush_end; + flush_event.notify_all(); + } + + LOG_TRACE(log, "Flushed system log"); } From 551fbf8519485e2d827290a439a924e44e651b09 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 18 May 2020 22:46:32 +0300 Subject: [PATCH 402/738] Update HTTPHandlerFactory.cpp --- programs/server/HTTPHandlerFactory.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/programs/server/HTTPHandlerFactory.cpp b/programs/server/HTTPHandlerFactory.cpp index 4caea1e92e8..955ff6b8834 100644 --- a/programs/server/HTTPHandlerFactory.cpp +++ b/programs/server/HTTPHandlerFactory.cpp @@ -129,6 +129,8 @@ static inline Poco::Net::HTTPRequestHandlerFactory * createHTTPHandlerFactory( query_handler->allowPostAndGetParamsRequest(); factory->addHandler(query_handler.release()); + /// We check that prometheus handler will be served on current (default) port. + /// Otherwise it will be created separately, see below. if (server.config().has("prometheus") && server.config().getInt("prometheus.port", 0) == 0) { auto prometheus_handler = std::make_unique>( From 0ec321a811b3e7c5776e9ce9b9d9cc5681f99f5c Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 19 May 2020 00:01:33 +0300 Subject: [PATCH 403/738] Fix ya.make --- src/Processors/ya.make | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Processors/ya.make b/src/Processors/ya.make index 09fb9d64ad1..f5a3c0ea30a 100644 --- a/src/Processors/ya.make +++ b/src/Processors/ya.make @@ -10,6 +10,7 @@ SRCS( Chunk.cpp ConcatProcessor.cpp DelayedPortsProcessor.cpp + Executors/PipelineExecutingBlockInputStream.h Executors/PipelineExecutor.cpp Executors/PullingPipelineExecutor.cpp Executors/TreeExecutorBlockInputStream.cpp From c34ea2736283e23ac13f5dd16775482719b1c9d0 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 19 May 2020 00:14:43 +0300 Subject: [PATCH 404/738] performance comparison --- docker/test/performance-comparison/compare.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 05c055107ac..4c4f40a8e90 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -314,6 +314,25 @@ create table queries_old_format engine File(TSVWithNamesAndTypes, 'queries.rep') from queries ; +-- save all test runs as JSON for the new comparison page +create table all_query_funs_json engine File(JSON, 'report/all-query-runs.json') as + select test, query, versions_runs[1] runs_left, versions_runs[2] runs_right + from ( + select + test, query, + groupArrayInsertAt(runs, version) versions_runs + from ( + select + replaceAll(_file, '-queries.tsv', '') test, + query, version, + groupArray(time) runs + from file('*-queries.tsv', TSV, 'query text, run int, version UInt32, time float') + group by test, query, version + ) + group by test, query + ) + ; + create table changed_perf_tsv engine File(TSV, 'report/changed-perf.tsv') as select left, right, diff, stat_threshold, changed_fail, test, query from queries where changed_show order by abs(diff) desc; From 075ccceee971935505b42064fed526ec602a1aa1 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 19 May 2020 00:38:28 +0300 Subject: [PATCH 405/738] Fix tests. --- src/Interpreters/InterpreterKillQueryQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/InterpreterKillQueryQuery.cpp b/src/Interpreters/InterpreterKillQueryQuery.cpp index 39e432195fd..42afd0ef477 100644 --- a/src/Interpreters/InterpreterKillQueryQuery.cpp +++ b/src/Interpreters/InterpreterKillQueryQuery.cpp @@ -306,7 +306,7 @@ Block InterpreterKillQueryQuery::getSelectResult(const String & columns, const S auto stream = block_io.getInputStream(); Block res = stream->read(); - if (res && block_io.in->read()) + if (res && stream->read()) throw Exception("Expected one block from input stream", ErrorCodes::LOGICAL_ERROR); return res; From 97c74471301aca3c4af32665cdeebd3adbcabe15 Mon Sep 17 00:00:00 2001 From: Artem Zuikov Date: Tue, 19 May 2020 00:41:23 +0300 Subject: [PATCH 406/738] Bitonic sort improvements (#10934) --- CMakeLists.txt | 3 - cmake/find/opencl.cmake | 10 +- programs/server/Server.cpp | 2 +- src/Columns/ColumnVector.cpp | 33 +++-- src/Columns/ColumnVector.h | 2 + src/Columns/IColumn.h | 11 ++ src/Common/BitonicSort.h | 229 ++++++++++++------------------ src/Common/oclBasics.h | 73 ++++------ src/Common/tests/CMakeLists.txt | 8 +- src/Common/tests/bitonic_sort.cpp | 186 ++++++++---------------- src/Core/config_core.h.in | 1 + src/Interpreters/sortBlock.cpp | 19 +-- 12 files changed, 234 insertions(+), 343 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fb36aff6603..53dfd1df1cb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -385,9 +385,6 @@ if (OS_LINUX AND NOT ENABLE_JEMALLOC) endif () if (USE_OPENCL) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DUSE_OPENCL=1") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_OPENCL=1") - if (OS_DARWIN) set(OPENCL_LINKER_FLAGS "-framework OpenCL") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OPENCL_LINKER_FLAGS}") diff --git a/cmake/find/opencl.cmake b/cmake/find/opencl.cmake index b1bf4630990..0f307350cb8 100644 --- a/cmake/find/opencl.cmake +++ b/cmake/find/opencl.cmake @@ -1,13 +1,19 @@ +# TODO: enable by default +if(0) + option(ENABLE_OPENCL "Enable OpenCL support" ${ENABLE_LIBRARIES}) +endif() + if(ENABLE_OPENCL) # Intel OpenCl driver: sudo apt install intel-opencl-icd -# TODO It's possible to add it as submodules: https://github.com/intel/compute-runtime/releases +# @sa https://github.com/intel/compute-runtime/releases # OpenCL applications should link wiht ICD loader # sudo apt install opencl-headers ocl-icd-libopencl1 # sudo ln -s /usr/lib/x86_64-linux-gnu/libOpenCL.so.1.0.0 /usr/lib/libOpenCL.so +# TODO: add https://github.com/OCL-dev/ocl-icd as submodule instead -find_package(OpenCL REQUIRED) +find_package(OpenCL) if(OpenCL_FOUND) set(USE_OPENCL 1) endif() diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 4958a7976de..bcb239ce083 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -226,7 +226,7 @@ int Server::main(const std::vector & /*args*/) #if !defined(ARCADIA_BUILD) #if USE_OPENCL - BitonicSort::getInstance().configure(); + BitonicSort::getInstance().configure(); #endif #endif diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 42013bdffd1..74f1438de14 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -38,6 +38,7 @@ namespace ErrorCodes { extern const int PARAMETER_OUT_OF_BOUND; extern const int SIZES_OF_COLUMNS_DOESNT_MATCH; + extern const int OPENCL_ERROR; extern const int LOGICAL_ERROR; } @@ -120,6 +121,30 @@ namespace }; } +template +void ColumnVector::getSpecialPermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, + IColumn::SpecialSort special_sort) const +{ + if (special_sort == IColumn::SpecialSort::OPENCL_BITONIC) + { +#if !defined(ARCADIA_BUILD) +#if USE_OPENCL + if (!limit || limit >= data.size()) + { + res.resize(data.size()); + + if (data.empty() || BitonicSort::getInstance().sort(data, res, !reverse)) + return; + } +#else + throw DB::Exception("'special_sort = bitonic' specified but OpenCL not available", DB::ErrorCodes::OPENCL_ERROR); +#endif +#endif + } + + getPermutation(reverse, limit, nan_direction_hint, res); +} + template void ColumnVector::getPermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res) const { @@ -144,14 +169,6 @@ void ColumnVector::getPermutation(bool reverse, size_t limit, int nan_directi } else { -#if !defined(ARCADIA_BUILD) -#if USE_OPENCL - /// If bitonic sort if specified as preferred than `nan_direction_hint` equals specific value 42. - if (nan_direction_hint == 42 && BitonicSort::getInstance().sort(data, res, !reverse)) - return; -#endif -#endif - /// A case for radix sort if constexpr (is_arithmetic_v && !std::is_same_v) { diff --git a/src/Columns/ColumnVector.h b/src/Columns/ColumnVector.h index 2fd177625cc..43b7c607f64 100644 --- a/src/Columns/ColumnVector.h +++ b/src/Columns/ColumnVector.h @@ -189,6 +189,8 @@ public: } void getPermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res) const override; + void getSpecialPermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, + IColumn::SpecialSort) const override; void reserve(size_t n) override { diff --git a/src/Columns/IColumn.h b/src/Columns/IColumn.h index 496a0a5759b..2a38fd5365b 100644 --- a/src/Columns/IColumn.h +++ b/src/Columns/IColumn.h @@ -245,6 +245,17 @@ public: */ virtual void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const = 0; + enum class SpecialSort + { + NONE = 0, + OPENCL_BITONIC, + }; + + virtual void getSpecialPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res, SpecialSort) const + { + getPermutation(reverse, limit, nan_direction_hint, res); + } + /** Copies each element according offsets parameter. * (i-th element should be copied offsets[i] - offsets[i - 1] times.) * It is necessary in ARRAY JOIN operation. diff --git a/src/Common/BitonicSort.h b/src/Common/BitonicSort.h index cbe5b5dc0a4..6bf10ebe835 100644 --- a/src/Common/BitonicSort.h +++ b/src/Common/BitonicSort.h @@ -11,13 +11,6 @@ #include #endif -#include -#include -#include -#include -#include -#include - #include #include #include @@ -30,6 +23,20 @@ class BitonicSort { public: + using KernelType = OCL::KernelType; + + enum Types + { + KernelInt8 = 0, + KernelUInt8, + KernelInt16, + KernelUInt16, + KernelInt32, + KernelUInt32, + KernelInt64, + KernelUInt64, + KernelMax + }; static BitonicSort & getInstance() { @@ -39,40 +46,50 @@ public: /// Sorts given array in specified order. Returns `true` if given sequence was sorted, `false` otherwise. template - bool sort(const DB::PaddedPODArray & data, DB::IColumn::Permutation & res, cl_uint sort_ascending) + bool sort(const DB::PaddedPODArray & data, DB::IColumn::Permutation & res, cl_uint sort_ascending [[maybe_unused]]) const { - size_t s = data.size(); - - /// Getting the nearest power of 2. - size_t power = 1; - - if (s <= 8) power = 8; - else while (power < s) power <<= 1; - - /// Allocates more space for additional stubs to be added if needed. - std::vector pairs_content(power); - std::vector pairs_indices(power); - for (UInt32 i = 0; i < s; ++i) + if constexpr ( + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v) { - pairs_content[i] = data[i]; - pairs_indices[i] = i; - } + size_t data_size = data.size(); - bool result = sort(pairs_content.data(), pairs_indices.data(), s, power - s, sort_ascending); + /// Getting the nearest power of 2. + size_t power = 8; + while (power < data_size) + power <<= 1; - if (!result) return false; + /// Allocates more space for additional stubs to be added if needed. + std::vector pairs_content(power); + std::vector pairs_indices(power); - for (size_t i = 0, shift = 0; i < power; ++i) - { - if (pairs_indices[i] >= s) + memcpy(&pairs_content[0], &data[0], sizeof(T) * data_size); + for (UInt32 i = 0; i < data_size; ++i) + pairs_indices[i] = i; + + fillWithStubs(pairs_content.data(), pairs_indices.data(), data_size, power - data_size, sort_ascending); + sort(pairs_content.data(), pairs_indices.data(), power, sort_ascending); + + for (size_t i = 0, shift = 0; i < power; ++i) { - ++shift; - continue; + if (pairs_indices[i] >= data_size) + { + ++shift; + continue; + } + res[i - shift] = pairs_indices[i]; } - res[i - shift] = pairs_indices[i]; + + return true; } - return true; + return false; } /// Creating a configuration instance with making all OpenCl required variables @@ -84,29 +101,36 @@ public: cl_platform_id platform = OCL::getPlatformID(settings); cl_device_id device = OCL::getDeviceID(platform, settings); cl_context gpu_context = OCL::makeContext(device, settings); - cl_command_queue command_queue = OCL::makeCommandQueue(device, gpu_context, settings); + cl_command_queue command_queue = OCL::makeCommandQueue<2>(device, gpu_context, settings); cl_program program = OCL::makeProgram(bitonic_sort_kernels, gpu_context, device, settings); /// Creating kernels for each specified data type. cl_int error = 0; + kernels.resize(KernelMax); - kernels["char"] = std::shared_ptr(clCreateKernel(program, "bitonicSort_char", &error), - clReleaseKernel); - kernels["uchar"] = std::shared_ptr(clCreateKernel(program, "bitonicSort_uchar", &error), - clReleaseKernel); - kernels["short"] = std::shared_ptr(clCreateKernel(program, "bitonicSort_short", &error), - clReleaseKernel); - kernels["ushort"] = std::shared_ptr(clCreateKernel(program, "bitonicSort_ushort", &error), - clReleaseKernel); - kernels["int"] = std::shared_ptr(clCreateKernel(program, "bitonicSort_int", &error), - clReleaseKernel); - kernels["uint"] = std::shared_ptr(clCreateKernel(program, "bitonicSort_uint", &error), - clReleaseKernel); - kernels["long"] = std::shared_ptr(clCreateKernel(program, "bitonicSort_long", &error), - clReleaseKernel); - kernels["ulong"] = std::shared_ptr(clCreateKernel(program, "bitonicSort_ulong", &error), - clReleaseKernel); + kernels[KernelInt8] = std::shared_ptr(clCreateKernel(program, "bitonicSort_char", &error), clReleaseKernel); + OCL::checkError(error); + + kernels[KernelUInt8] = std::shared_ptr(clCreateKernel(program, "bitonicSort_uchar", &error), clReleaseKernel); + OCL::checkError(error); + + kernels[KernelInt16] = std::shared_ptr(clCreateKernel(program, "bitonicSort_short", &error), clReleaseKernel); + OCL::checkError(error); + + kernels[KernelUInt16] = std::shared_ptr(clCreateKernel(program, "bitonicSort_ushort", &error), clReleaseKernel); + OCL::checkError(error); + + kernels[KernelInt32] = std::shared_ptr(clCreateKernel(program, "bitonicSort_int", &error), clReleaseKernel); + OCL::checkError(error); + + kernels[KernelUInt32] = std::shared_ptr(clCreateKernel(program, "bitonicSort_uint", &error), clReleaseKernel); + OCL::checkError(error); + + kernels[KernelInt64] = std::shared_ptr(clCreateKernel(program, "bitonicSort_long", &error), clReleaseKernel); + OCL::checkError(error); + + kernels[KernelUInt64] = std::shared_ptr(clCreateKernel(program, "bitonicSort_ulong", &error), clReleaseKernel); OCL::checkError(error); configuration = std::shared_ptr(new OCL::Configuration(device, gpu_context, command_queue, program)); @@ -114,97 +138,24 @@ public: private: /// Dictionary with kernels for each type from list: uchar, char, ushort, short, uint, int, ulong and long. - std::map> kernels; + std::vector> kernels; /// Current configuration with core OpenCL instances. std::shared_ptr configuration = nullptr; - /// Returns `true` if given sequence was sorted, `false` otherwise. - template - bool sort(T * p_input, cl_uint * indices, cl_int array_size, cl_int number_of_stubs, cl_uint sort_ascending) - { - if (typeid(T).name() == typeid(cl_char).name()) - sort_char(reinterpret_cast(p_input), indices, array_size, number_of_stubs, sort_ascending); - else if (typeid(T) == typeid(cl_uchar)) - sort_uchar(reinterpret_cast(p_input), indices, array_size, number_of_stubs, sort_ascending); - else if (typeid(T) == typeid(cl_short)) - sort_short(reinterpret_cast(p_input), indices, array_size, number_of_stubs, sort_ascending); - else if (typeid(T) == typeid(cl_ushort)) - sort_ushort(reinterpret_cast(p_input), indices, array_size, number_of_stubs, sort_ascending); - else if (typeid(T) == typeid(cl_int)) - sort_int(reinterpret_cast(p_input), indices, array_size, number_of_stubs, sort_ascending); - else if (typeid(T) == typeid(cl_uint)) - sort_uint(reinterpret_cast(p_input), indices, array_size, number_of_stubs, sort_ascending); - else if (typeid(T) == typeid(cl_long)) - sort_long(reinterpret_cast(p_input), indices, array_size, number_of_stubs, sort_ascending); - else if (typeid(T) == typeid(cl_ulong)) - sort_ulong(reinterpret_cast(p_input), indices, array_size, number_of_stubs, sort_ascending); - else - return false; - - return true; - } - - /// Specific functions for each integer type. - void sort_char(cl_char * p_input, cl_uint * indices, cl_int array_size, cl_int number_of_stubs, cl_uint sort_ascending) - { - cl_char stubs_value = sort_ascending ? CHAR_MAX : CHAR_MIN; - fillWithStubs(number_of_stubs, stubs_value, p_input, indices, array_size); - sort(kernels["char"].get(), p_input, indices, array_size + number_of_stubs, sort_ascending); - } - - void sort_uchar(cl_uchar * p_input, cl_uint * indices, cl_int array_size, cl_int number_of_stubs, cl_uint sort_ascending) - { - cl_uchar stubs_value = sort_ascending ? UCHAR_MAX : 0; - fillWithStubs(number_of_stubs, stubs_value, p_input, indices, array_size); - sort(kernels["uchar"].get(), p_input, indices, array_size + number_of_stubs, sort_ascending); - } - - void sort_short(cl_short * p_input, cl_uint * indices, cl_int array_size, cl_int number_of_stubs, cl_uint sort_ascending) - { - cl_short stubs_value = sort_ascending ? SHRT_MAX : SHRT_MIN; - fillWithStubs(number_of_stubs, stubs_value, p_input, indices, array_size); - sort(kernels["short"].get(), p_input, indices, array_size + number_of_stubs, sort_ascending); - } - - void sort_ushort(cl_ushort * p_input, cl_uint * indices, cl_int array_size, cl_int number_of_stubs, cl_uint sort_ascending) - { - cl_ushort stubs_value = sort_ascending ? USHRT_MAX : 0; - fillWithStubs(number_of_stubs, stubs_value, p_input, indices, array_size); - sort(kernels["ushort"].get(), p_input, indices, array_size + number_of_stubs, sort_ascending); - } - - void sort_int(cl_int * p_input, cl_uint * indices, cl_int array_size, cl_int number_of_stubs, cl_uint sort_ascending) - { - cl_int stubs_value = sort_ascending ? INT_MAX : INT_MIN; - fillWithStubs(number_of_stubs, stubs_value, p_input, indices, array_size); - sort(kernels["int"].get(), p_input, indices, array_size + number_of_stubs, sort_ascending); - } - - void sort_uint(cl_uint * p_input, cl_uint * indices, cl_int array_size, cl_int number_of_stubs, cl_uint sort_ascending) - { - cl_uint stubs_value = sort_ascending ? UINT_MAX : 0; - fillWithStubs(number_of_stubs, stubs_value, p_input, indices, array_size); - sort(kernels["uint"].get(), p_input, indices, array_size + number_of_stubs, sort_ascending); - } - - void sort_long(cl_long * p_input, cl_uint * indices, cl_int array_size, cl_int number_of_stubs, cl_uint sort_ascending) - { - cl_long stubs_value = sort_ascending ? LONG_MAX : LONG_MIN; - fillWithStubs(number_of_stubs, stubs_value, p_input, indices, array_size); - sort(kernels["long"].get(), p_input, indices, array_size + number_of_stubs, sort_ascending); - } - - void sort_ulong(cl_ulong * p_input, cl_uint * indices, cl_int array_size, cl_int number_of_stubs, cl_uint sort_ascending) - { - cl_ulong stubs_value = sort_ascending ? ULONG_MAX : 0; - fillWithStubs(number_of_stubs, stubs_value, p_input, indices, array_size); - sort(kernels["ulong"].get(), p_input, indices, array_size + number_of_stubs, sort_ascending); - } + cl_kernel getKernel(Int8) const { return kernels[KernelInt8].get(); } + cl_kernel getKernel(UInt8) const { return kernels[KernelUInt8].get(); } + cl_kernel getKernel(Int16) const { return kernels[KernelInt16].get(); } + cl_kernel getKernel(UInt16) const { return kernels[KernelUInt16].get(); } + cl_kernel getKernel(Int32) const { return kernels[KernelInt32].get(); } + cl_kernel getKernel(UInt32) const { return kernels[KernelUInt32].get(); } + cl_kernel getKernel(Int64) const { return kernels[KernelInt64].get(); } + cl_kernel getKernel(UInt64) const { return kernels[KernelUInt64].get(); } /// Sorts p_input inplace with indices. Works only with arrays which size equals to power of two. template - void sort(cl_kernel kernel, T * p_input, cl_uint * indices, cl_int array_size, cl_uint sort_ascending) + void sort(T * p_input, cl_uint * indices, cl_int array_size, cl_uint sort_ascending) const { + cl_kernel kernel = getKernel(T(0)); cl_int error = CL_SUCCESS; cl_int num_stages = 0; @@ -246,7 +197,7 @@ private: } template - void configureKernel(cl_kernel kernel, int number_of_argument, void * source) + void configureKernel(cl_kernel kernel, int number_of_argument, void * source) const { cl_int error = clSetKernelArg(kernel, number_of_argument, sizeof(T), source); OCL::checkError(error); @@ -254,9 +205,9 @@ private: /// Fills given sequences from `arraySize` index with `numberOfStubs` values. template - void fillWithStubs(cl_int number_of_stubs, T value, T * p_input, - cl_uint * indices, cl_int array_size) + void fillWithStubs(T * p_input, cl_uint * indices, cl_int array_size, cl_int number_of_stubs, cl_uint sort_ascending) const { + T value = sort_ascending ? std::numeric_limits::max() : std::numeric_limits::min(); for (cl_int index = 0; index < number_of_stubs; ++index) { p_input[array_size + index] = value; @@ -264,7 +215,7 @@ private: } } - BitonicSort() {} - BitonicSort(BitonicSort const &); - void operator=(BitonicSort const &); + BitonicSort() = default; + BitonicSort(BitonicSort const &) = delete; + void operator = (BitonicSort const &) = delete; }; diff --git a/src/Common/oclBasics.h b/src/Common/oclBasics.h index 550f42a32d0..7c977830e82 100644 --- a/src/Common/oclBasics.h +++ b/src/Common/oclBasics.h @@ -17,24 +17,18 @@ #include #include -#ifndef CL_VERSION_2_0 -#define CL_USE_DEPRECATED_OPENCL_1_2_APIS -#endif - - -using KernelType = std::remove_reference::type; - namespace DB { - namespace ErrorCodes - { - extern const int OPENCL_ERROR; - } +namespace ErrorCodes +{ + extern const int OPENCL_ERROR; +} } struct OCL { + using KernelType = std::remove_reference::type; /** * Structure which represents the most essential settings of common OpenCl entities. @@ -211,7 +205,7 @@ struct OCL static void checkError(cl_int error) { if (error != CL_SUCCESS) - throw DB::Exception("OpenCL error " + opencl_error_to_str(error), DB::ErrorCodes::OPENCL_ERROR); + throw DB::Exception("OpenCL error: " + opencl_error_to_str(error), DB::ErrorCodes::OPENCL_ERROR); } @@ -223,22 +217,18 @@ struct OCL cl_int error = clGetPlatformIDs(settings.number_of_platform_entries, &platform, settings.number_of_available_platforms); checkError(error); - return platform; } - static cl_device_id getDeviceID(cl_platform_id & platform, const Settings & settings) { cl_device_id device; cl_int error = clGetDeviceIDs(platform, CL_DEVICE_TYPE_GPU, settings.number_of_devices_entries, &device, settings.number_of_available_devices); OCL::checkError(error); - return device; } - static cl_context makeContext(cl_device_id & device, const Settings & settings) { cl_int error; @@ -246,32 +236,43 @@ struct OCL &device, settings.context_callback, settings.context_callback_data, &error); OCL::checkError(error); - return gpu_context; } - + template static cl_command_queue makeCommandQueue(cl_device_id & device, cl_context & context, const Settings & settings [[maybe_unused]]) { cl_int error; -#ifdef CL_USE_DEPRECATED_OPENCL_1_2_APIS - cl_command_queue command_queue = clCreateCommandQueue(context, device, settings.command_queue_properties, &error); -#else - cl_command_queue command_queue = clCreateCommandQueueWithProperties(context, device, nullptr, &error); -#endif - OCL::checkError(error); + cl_command_queue command_queue; + if constexpr (version == 1) + { +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" + command_queue = clCreateCommandQueue(context, device, settings.command_queue_properties, &error); +#pragma GCC diagnostic pop + } + else + { +#ifdef CL_VERSION_2_0 + command_queue = clCreateCommandQueueWithProperties(context, device, nullptr, &error); +#else + throw DB::Exception("Binary is built with OpenCL version < 2.0", DB::ErrorCodes::OPENCL_ERROR); +#endif + } + + OCL::checkError(error); return command_queue; } - static cl_program makeProgram(const char * source_code, cl_context context, cl_device_id device_id, const Settings & settings) { cl_int error = 0; size_t source_size = strlen(source_code); - cl_program program = clCreateProgramWithSource(context, settings.number_of_program_source_pointers, &source_code, &source_size, &error); + cl_program program = clCreateProgramWithSource(context, settings.number_of_program_source_pointers, + &source_code, &source_size, &error); checkError(error); error = clBuildProgram(program, settings.number_of_devices_entries, &device_id, settings.build_options, @@ -293,39 +294,30 @@ struct OCL } checkError(error); - return program; } - /// Configuring buffer for given input data template - static cl_mem createBuffer(K * p_input, cl_int array_size, cl_context context, - cl_int elements_size = sizeof(K)) + static cl_mem createBuffer(K * p_input, cl_int array_size, cl_context context, cl_int elements_size = sizeof(K)) { cl_int error = CL_SUCCESS; - cl_mem cl_input_buffer = - clCreateBuffer - ( + cl_mem cl_input_buffer = clCreateBuffer( context, CL_MEM_USE_HOST_PTR, zeroCopySizeAlignment(elements_size * array_size), p_input, - &error - ); + &error); checkError(error); - return cl_input_buffer; } - static size_t zeroCopySizeAlignment(size_t required_size) { return required_size + (~required_size + 1) % 64; } - /// Manipulating with common OpenCL variables. static void finishCommandQueue(cl_command_queue command_queue) @@ -335,10 +327,8 @@ struct OCL OCL::checkError(error); } - template - static void releaseData(T * origin, cl_int array_size, cl_mem cl_buffer, - cl_command_queue command_queue, size_t offset = 0) + static void releaseData(T * origin, cl_int array_size, cl_mem cl_buffer, cl_command_queue command_queue, size_t offset = 0) { cl_int error = CL_SUCCESS; @@ -359,7 +349,6 @@ struct OCL error = clReleaseMemObject(cl_buffer); checkError(error); } - }; #endif diff --git a/src/Common/tests/CMakeLists.txt b/src/Common/tests/CMakeLists.txt index 1c8a39a4fa5..72c47d1ef49 100644 --- a/src/Common/tests/CMakeLists.txt +++ b/src/Common/tests/CMakeLists.txt @@ -35,10 +35,10 @@ target_link_libraries (compact_array PRIVATE clickhouse_common_io) add_executable (radix_sort radix_sort.cpp) target_link_libraries (radix_sort PRIVATE clickhouse_common_io) -# if (USE_OPENCL) -# add_executable (bitonic_sort bitonic_sort.cpp) -# target_link_libraries (bitonic_sort PRIVATE clickhouse_common_io ${OPENCL_LINKER_FLAGS}) -# endif () +if (USE_OPENCL) + add_executable (bitonic_sort bitonic_sort.cpp) + target_link_libraries (bitonic_sort PRIVATE clickhouse_common_io ${OPENCL_LINKER_FLAGS} ${OpenCL_LIBRARIES}) +endif () add_executable (arena_with_free_lists arena_with_free_lists.cpp) target_link_libraries (arena_with_free_lists PRIVATE dbms) diff --git a/src/Common/tests/bitonic_sort.cpp b/src/Common/tests/bitonic_sort.cpp index adaef94ed4c..2545662c8cb 100644 --- a/src/Common/tests/bitonic_sort.cpp +++ b/src/Common/tests/bitonic_sort.cpp @@ -1,8 +1,6 @@ #include #include -#if USE_OPENCL - #if !defined(__APPLE__) && !defined(__FreeBSD__) #include #endif @@ -16,13 +14,10 @@ #include "Common/BitonicSort.h" -using Key = cl_ulong; - - /// Generates vector of size 8 for testing. /// Vector contains max possible value, min possible value and duplicate values. template -static void generateTest(std::vector& data, Type min_value, Type max_value) +static void generateTest(std::vector & data, Type min_value, Type max_value) { int size = 10; @@ -62,8 +57,7 @@ static void check(const std::vector & indices, bool reverse = true) template -static void sortBitonicSortWithPodArrays(const std::vector& data, - std::vector & indices, bool ascending = true) +static void sortBitonicSortWithPodArrays(const std::vector & data, std::vector & indices, bool ascending = true) { DB::PaddedPODArray pod_array_data = DB::PaddedPODArray(data.size()); DB::IColumn::Permutation pod_array_indices = DB::IColumn::Permutation(data.size()); @@ -74,7 +68,6 @@ static void sortBitonicSortWithPodArrays(const std::vector& data, *(pod_array_indices.data() + index) = index; } - BitonicSort::getInstance().configure(); BitonicSort::getInstance().sort(pod_array_data, pod_array_indices, ascending); for (size_t index = 0; index < data.size(); ++index) @@ -83,7 +76,7 @@ static void sortBitonicSortWithPodArrays(const std::vector& data, template -static void testBitonicSort(std::string test_name, Type min_value, Type max_value) +static void testBitonicSort(const std::string & test_name, Type min_value, Type max_value) { std::cerr << test_name << std::endl; @@ -102,147 +95,80 @@ static void testBitonicSort(std::string test_name, Type min_value, Type max_valu static void straightforwardTests() { - testBitonicSort("Test 01: cl_char.", CHAR_MIN, CHAR_MAX); - testBitonicSort("Test 02: cl_uchar.", 0, UCHAR_MAX); - testBitonicSort("Test 03: cl_short.", SHRT_MIN, SHRT_MAX); - testBitonicSort("Test 04: cl_ushort.", 0, USHRT_MAX); - testBitonicSort("Test 05: cl_int.", INT_MIN, INT_MAX); - testBitonicSort("Test 06: cl_uint.", 0, UINT_MAX); - testBitonicSort("Test 07: cl_long.", LONG_MIN, LONG_MAX); - testBitonicSort("Test 08: cl_ulong.", 0, ULONG_MAX); + testBitonicSort("Test 01: Int8.", CHAR_MIN, CHAR_MAX); + testBitonicSort("Test 02: UInt8.", 0, UCHAR_MAX); + testBitonicSort("Test 03: Int16.", SHRT_MIN, SHRT_MAX); + testBitonicSort("Test 04: UInt16.", 0, USHRT_MAX); + testBitonicSort("Test 05: Int32.", INT_MIN, INT_MAX); + testBitonicSort("Test 06: UInt32.", 0, UINT_MAX); + testBitonicSort("Test 07: Int64.", LONG_MIN, LONG_MAX); + testBitonicSort("Test 08: UInt64.", 0, ULONG_MAX); } -static void NO_INLINE sort1(Key * data, size_t size) +template +static void bitonicSort(std::vector & data) { - std::sort(data, data + size); -} - - -static void NO_INLINE sort2(std::vector & data, std::vector & indices) -{ - BitonicSort::getInstance().configure(); + size_t size = data.size(); + std::vector indices(size); + for (size_t i = 0; i < size; ++i) + indices[i] = i; sortBitonicSortWithPodArrays(data, indices); - std::vector result(data.size()); - for (size_t index = 0; index < data.size(); ++index) - result[index] = data[indices[index]]; + std::vector result(size); + for (size_t i = 0; i < size; ++i) + result[i] = data[indices[i]]; data = std::move(result); } -int main(int argc, char ** argv) +template +static bool checkSort(const std::vector & data, size_t size) { - straightforwardTests(); + std::vector copy1(data.begin(), data.begin() + size); + std::vector copy2(data.begin(), data.begin() + size); - if (argc < 3) - { - std::cerr << "Not enough arguments were passed\n"; - return 1; - } + std::sort(copy1.data(), copy1.data() + size); + bitonicSort(copy2); - size_t n = DB::parse(argv[1]); - size_t method = DB::parse(argv[2]); + for (size_t i = 0; i < size; ++i) + if (copy1[i] != copy2[i]) + return false; - std::vector data(n); - std::vector indices(n); - - { - Stopwatch watch; - - for (auto & elem : data) - elem = static_cast(rand()); - - for (size_t i = 0; i < n; ++i) - indices[i] = i; - - watch.stop(); - double elapsed = watch.elapsedSeconds(); - std::cerr - << "Filled in " << elapsed - << " (" << n / elapsed << " elem/sec., " - << n * sizeof(Key) / elapsed / 1048576 << " MB/sec.)" - << std::endl; - } - - if (n <= 100) - { - std::cerr << std::endl; - for (const auto & elem : data) - std::cerr << elem << ' '; - std::cerr << std::endl; - for (const auto & index : indices) - std::cerr << index << ' '; - std::cerr << std::endl; - } - - { - Stopwatch watch; - - if (method == 1) sort1(data.data(), n); - if (method == 2) sort2(data, indices); - - watch.stop(); - double elapsed = watch.elapsedSeconds(); - std::cerr - << "Sorted in " << elapsed - << " (" << n / elapsed << " elem/sec., " - << n * sizeof(Key) / elapsed / 1048576 << " MB/sec.)" - << std::endl; - } - - { - Stopwatch watch; - - size_t i = 1; - while (i < n) - { - if (!(data[i - 1] <= data[i])) - break; - ++i; - } - - watch.stop(); - double elapsed = watch.elapsedSeconds(); - std::cerr - << "Checked in " << elapsed - << " (" << n / elapsed << " elem/sec., " - << n * sizeof(Key) / elapsed / 1048576 << " MB/sec.)" - << std::endl - << "Result: " << (i == n ? "Ok." : "Fail!") << std::endl; - } - - if (n <= 1000) - { - std::cerr << std::endl; - - std::cerr << data[0] << ' '; - for (size_t i = 1; i < n; ++i) - { - if (!(data[i - 1] <= data[i])) - std::cerr << "*** "; - std::cerr << data[i] << ' '; - } - - std::cerr << std::endl; - - for (const auto & index : indices) - std::cerr << index << ' '; - std::cerr << std::endl; - } - - return 0; + return true; } -#else int main() { - std::cerr << "Openc CL disabled."; + BitonicSort::getInstance().configure(); + + straightforwardTests(); + + size_t size = 1100; + std::vector data(size); + for (size_t i = 0; i < size; ++i) + data[i] = rand(); + + for (size_t i = 0; i < 128; ++i) + { + if (!checkSort(data, i)) + { + std::cerr << "fail at length " << i << std::endl; + return 1; + } + } + + for (size_t i = 128; i < size; i += 7) + { + if (!checkSort(data, i)) + { + std::cerr << "fail at length " << i << std::endl; + return 1; + } + } return 0; } - -#endif diff --git a/src/Core/config_core.h.in b/src/Core/config_core.h.in index 054ee9a80b7..620c23c21cc 100644 --- a/src/Core/config_core.h.in +++ b/src/Core/config_core.h.in @@ -8,3 +8,4 @@ #cmakedefine01 USE_EMBEDDED_COMPILER #cmakedefine01 USE_INTERNAL_LLVM_LIBRARY #cmakedefine01 USE_SSL +#cmakedefine01 USE_OPENCL diff --git a/src/Interpreters/sortBlock.cpp b/src/Interpreters/sortBlock.cpp index 0e98dc0eb4b..ec0865c2fb5 100644 --- a/src/Interpreters/sortBlock.cpp +++ b/src/Interpreters/sortBlock.cpp @@ -13,7 +13,6 @@ namespace DB namespace ErrorCodes { extern const int BAD_COLLATION; - extern const int OPENCL_ERROR; } static bool isCollationRequired(const SortColumnDescription & description) @@ -134,20 +133,12 @@ void sortBlock(Block & block, const SortDescription & description, UInt64 limit) else if (!isColumnConst(*column)) { int nan_direction_hint = description[0].nulls_direction; + auto special_sort = description[0].special_sort; - /// If in Settings `special_sort` option has been set as `bitonic_sort`, - /// then via `nan_direction_hint` variable a flag which specifies bitonic sort as preferred - /// will be passed to `getPermutation` method with value 42. - if (description[0].special_sort == SpecialSort::OPENCL_BITONIC) - { -#ifdef USE_OPENCL - nan_direction_hint = 42; -#else - throw DB::Exception("Bitonic sort specified as preferred, but OpenCL not available", DB::ErrorCodes::OPENCL_ERROR); -#endif - } - - column->getPermutation(reverse, limit, nan_direction_hint, perm); + if (special_sort == SpecialSort::OPENCL_BITONIC) + column->getSpecialPermutation(reverse, limit, nan_direction_hint, perm, IColumn::SpecialSort::OPENCL_BITONIC); + else + column->getPermutation(reverse, limit, nan_direction_hint, perm); } else /// we don't need to do anything with const column From e9dc2cbcf70968996a1afbae3c409c5177f77f74 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 19 May 2020 01:12:51 +0300 Subject: [PATCH 407/738] Add missing SYSTEM FLUSH LOGS before TRUNCATE TABLE in clickhouse-test Suggested-by: @filimonov --- tests/clickhouse-test | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 76a61e1d558..324fd13aac2 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -235,6 +235,9 @@ def run_tests_array(all_tests_with_params): clickhouse_proc.communicate("SELECT 'Running test {suite}/{case} from pid={pid}';".format(pid = os.getpid(), case = case, suite = suite)) if not args.no_system_log_cleanup: + clickhouse_proc = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE) + clickhouse_proc.communicate("SYSTEM FLUSH LOGS") + for table in ['query_log', 'query_thread_log', 'trace_log', 'metric_log']: clickhouse_proc = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE) clickhouse_proc.communicate("TRUNCATE TABLE IF EXISTS system.{}".format(table)) From ef01eb954fa903eb0422ae8280e22b0108867de3 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 19 May 2020 02:40:45 +0300 Subject: [PATCH 408/738] Add note about system_tables_lazy_load into config.xml --- programs/server/config.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/programs/server/config.xml b/programs/server/config.xml index 226cbc01178..dd01b377485 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -405,6 +405,9 @@ --> + + + @@ -307,13 +307,13 @@ For the query to run successfully, the following conditions must be met: ALTER TABLE table_source MOVE PARTITION partition_expr TO TABLE table_dest ``` -This query move the data partition from the `table_source` to `table_dest` with deleting the data from `table_source`. +This query moves the data partition from the `table_source` to `table_dest` with deleting the data from `table_source`. For the query to run successfully, the following conditions must be met: - Both tables must have the same structure. - Both tables must have the same partition key. -- Both tables must be the same engine family. (replicated or non-replicated) +- Both tables must be the same engine family (replicated or non-replicated). - Both tables must have the same storage policy. #### CLEAR COLUMN IN PARTITION {#alter_clear-column-partition} diff --git a/docs/ru/getting-started/install.md b/docs/ru/getting-started/install.md index fb36b9c87ec..04712328844 100644 --- a/docs/ru/getting-started/install.md +++ b/docs/ru/getting-started/install.md @@ -38,7 +38,7 @@ sudo rpm --import https://repo.yandex.ru/clickhouse/CLICKHOUSE-KEY.GPG sudo yum-config-manager --add-repo https://repo.yandex.ru/clickhouse/rpm/stable/x86_64 ``` -Для использования наиболее свежих версий нужно заменить `stable` на `testing` (рекомендуется для тестовых окружений). +Для использования наиболее свежих версий нужно заменить `stable` на `testing` (рекомендуется для тестовых окружений). Также иногда доступен `prestable`. Для, собственно, установки пакетов необходимо выполнить следующие команды: diff --git a/docs/ru/operations/server-configuration-parameters/settings.md b/docs/ru/operations/server-configuration-parameters/settings.md index f86fc1baa75..c664580c659 100644 --- a/docs/ru/operations/server-configuration-parameters/settings.md +++ b/docs/ru/operations/server-configuration-parameters/settings.md @@ -686,7 +686,7 @@ TCP порт для защищённого обмена данными с кли 9004 ``` -## tmp\_path {#tmp-path} +## tmp_path {#tmp-path} Путь ко временным данным для обработки больших запросов. @@ -698,6 +698,17 @@ TCP порт для защищённого обмена данными с кли ``` xml /var/lib/clickhouse/tmp/ ``` +## tmp_policy {#tmp-policy} + +Политика из [storage_configuration](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes) для хранения временных файлов. + +Если политика не задана, используется [tmp_path](#tmp-path). В противном случае `tmp_path` игнорируется. + +!!! note "Примечание" + - `move_factor` игнорируется. + - `keep_free_space_bytes` игнорируется. + - `max_data_part_size_bytes` игнорируется. + - В данной политике у вас должен быть ровно один том. ## uncompressed\_cache\_size {#server-settings-uncompressed_cache_size} diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 16d840fe4b1..56c3042bfa3 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -1025,6 +1025,29 @@ ClickHouse генерирует исключение Значение по умолчанию: 0. +## optimize_skip_unused_shards {#optimize-skip-unused-shards} + +Включает или отключает пропуск неиспользуемых шардов для запросов [SELECT](../../sql-reference/statements/select/index.md) , в которых условие ключа шардирования задано в секции `WHERE/PREWHERE`. Предполагается, что данные распределены с помощью ключа шардирования, в противном случае настройка ничего не делает. + +Возможные значения: + +- 0 — Выключена. +- 1 — Включена. + +Значение по умолчанию: 0 + +## force_optimize_skip_unused_shards {#force-optimize-skip-unused-shards} + +Разрешает или запрещает выполнение запроса, если настройка [optimize_skip_unused_shards](#optimize-skip-unused-shards) включена, а пропуск неиспользуемых шардов невозможен. Если данная настройка включена и пропуск невозможен, ClickHouse генерирует исключение. + +Возможные значения: + +- 0 — Выключена. ClickHouse не генерирует исключение. +- 1 — Включена. Выполнение запроса запрещается, только если у таблицы есть ключ шардирования. +- 2 — Включена. Выполнение запроса запрещается, даже если для таблицы не определен ключ шардирования. + +Значение по умолчанию: 0 + ## optimize\_throw\_if\_noop {#setting-optimize_throw_if_noop} Включает или отключает генерирование исключения в в случаях, когда запрос [OPTIMIZE](../../sql-reference/statements/misc.md#misc_operations-optimize) не выполняет мёрж. diff --git a/docs/ru/operations/system-tables.md b/docs/ru/operations/system-tables.md index bc4a4c2751f..ae0e67a4515 100644 --- a/docs/ru/operations/system-tables.md +++ b/docs/ru/operations/system-tables.md @@ -517,6 +517,33 @@ CurrentMetric_ReplicatedChecks: 0 - `query` (String) – текст запроса. Для запросов `INSERT` не содержит встаявляемые данные. - `query_id` (String) – идентификатор запроса, если был задан. +## system.text\_log {#system-tables-text-log} + +Содержит записи логов. Уровень логирования для таблицы может быть ограничен параметром сервера `text_log.level`. + +Столбцы: + +- `event_date` (Date) — Дата создания записи. +- `event_time` (DateTime) — Время создания записи. +- `microseconds` (UInt32) — Время создания записи в микросекундах. +- `thread_name` (String) — Название потока, из которого была сделана запись. +- `thread_id` (UInt64) — Идентификатор потока ОС. +- `level` (Enum8) — Уровень логирования записи. Возможные значения: + - `1` или `'Fatal'`. + - `2` или `'Critical'`. + - `3` или `'Error'`. + - `4` или `'Warning'`. + - `5` или `'Notice'`. + - `6` или `'Information'`. + - `7` или `'Debug'`. + - `8` или `'Trace'`. +- `query_id` (String) — Идентификатор запроса. +- `logger_name` (LowCardinality(String)) — Название логгера (`DDLWorker`). +- `message` (String) — Само тело записи. +- `revision` (UInt32) — Ревизия ClickHouse. +- `source_file` (LowCardinality(String)) — Исходный файл, из которого была сделана запись. +- `source_line` (UInt64) — Исходная строка, из которой была сделана запись. + ## system.query\_log {#system_tables-query_log} Содержит информацию о выполнении запросов. Для каждого запроса вы можете увидеть время начала обработки, продолжительность обработки, сообщения об ошибках и другую информацию. diff --git a/docs/ru/sql-reference/functions/arithmetic-functions.md b/docs/ru/sql-reference/functions/arithmetic-functions.md index 85c597143fd..8513737f025 100644 --- a/docs/ru/sql-reference/functions/arithmetic-functions.md +++ b/docs/ru/sql-reference/functions/arithmetic-functions.md @@ -48,13 +48,17 @@ SELECT toTypeName(0), toTypeName(0 + 0), toTypeName(0 + 0 + 0), toTypeName(0 + 0 Отличается от intDiv тем, что при делении на ноль или при делении минимального отрицательного числа на минус единицу, возвращается ноль. -## modulo(a, b), оператор a % b {#moduloa-b-operator-a-b} +## modulo(a, b), оператор a % b {#modulo} Вычисляет остаток от деления. Если аргументы - числа с плавающей запятой, то они предварительно преобразуются в целые числа, путём отбрасывания дробной части. Берётся остаток в том же смысле, как это делается в C++. По факту, для отрицательных чисел, используется truncated division. При делении на ноль или при делении минимального отрицательного числа на минус единицу, кидается исключение. +## moduloOrZero(a, b) {#modulo-or-zero} + +В отличие от [modulo](#modulo), возвращает ноль при делении на ноль. + ## negate(a), оператор -a {#negatea-operator-a} Вычисляет число, обратное по знаку. Результат всегда имеет знаковый тип. diff --git a/docs/ru/sql-reference/statements/alter.md b/docs/ru/sql-reference/statements/alter.md index c4afb6981c4..1cf061c174f 100644 --- a/docs/ru/sql-reference/statements/alter.md +++ b/docs/ru/sql-reference/statements/alter.md @@ -204,17 +204,17 @@ ALTER TABLE [db].name DROP CONSTRAINT constraint_name; Для работы с [партициями](../../sql-reference/statements/alter.md) доступны следующие операции: -- [DETACH PARTITION](#alter_detach-partition) – перенести партицию в директорию `detached`; -- [DROP PARTITION](#alter_drop-partition) – удалить партицию; -- [ATTACH PARTITION\|PART](#alter_attach-partition) – добавить партицию/кусок в таблицу из директории `detached`; -- [ATTACH PARTITION FROM](#alter_attach-partition-from) – скопировать партицию из другой таблицы; -- [REPLACE PARTITION](#alter_replace-partition) – скопировать партицию из другой таблицы с заменой; -- [MOVE PARTITION TO TABLE](#alter_move_to_table-partition) (\#alter\_move\_to\_table-partition) - переместить партицию в другую таблицу; -- [CLEAR COLUMN IN PARTITION](#alter_clear-column-partition) – удалить все значения в столбце для заданной партиции; -- [CLEAR INDEX IN PARTITION](#alter_clear-index-partition) - очистить построенные вторичные индексы для заданной партиции; -- [FREEZE PARTITION](#alter_freeze-partition) – создать резервную копию партиции; -- [FETCH PARTITION](#alter_fetch-partition) – скачать партицию с другого сервера; -- [MOVE PARTITION\|PART](#alter_move-partition) – переместить партицию/кускок на другой диск или том. +- [DETACH PARTITION](#alter_detach-partition) — перенести партицию в директорию `detached`; +- [DROP PARTITION](#alter_drop-partition) — удалить партицию; +- [ATTACH PARTITION\|PART](#alter_attach-partition) — добавить партицию/кусок в таблицу из директории `detached`; +- [ATTACH PARTITION FROM](#alter_attach-partition-from) — скопировать партицию из другой таблицы; +- [REPLACE PARTITION](#alter_replace-partition) — скопировать партицию из другой таблицы с заменой; +- [MOVE PARTITION TO TABLE](#alter_move_to_table-partition) — переместить партицию в другую таблицу; +- [CLEAR COLUMN IN PARTITION](#alter_clear-column-partition) — удалить все значения в столбце для заданной партиции; +- [CLEAR INDEX IN PARTITION](#alter_clear-index-partition) — очистить построенные вторичные индексы для заданной партиции; +- [FREEZE PARTITION](#alter_freeze-partition) — создать резервную копию партиции; +- [FETCH PARTITION](#alter_fetch-partition) — скачать партицию с другого сервера; +- [MOVE PARTITION\|PART](#alter_move-partition) — переместить партицию/кускок на другой диск или том. #### DETACH PARTITION {#alter_detach-partition} @@ -312,12 +312,14 @@ ALTER TABLE table2 REPLACE PARTITION partition_expr FROM table1 ALTER TABLE table_source MOVE PARTITION partition_expr TO TABLE table_dest ``` -Перемещает партицию из таблицы `table_source` в таблицу `table_dest` (добавляет к существующим данным в `table_dest`), с удалением данных из таблицы `table_source`. +Перемещает партицию из таблицы `table_source` в таблицу `table_dest` (добавляет к существующим данным в `table_dest`) с удалением данных из таблицы `table_source`. Следует иметь в виду: - Таблицы должны иметь одинаковую структуру. - Для таблиц должен быть задан одинаковый ключ партиционирования. +- Движки таблиц должны быть одинакового семейства (реплицированные или нереплицированные). +- Для таблиц должна быть задана одинаковая политика хранения. #### CLEAR COLUMN IN PARTITION {#alter_clear-column-partition} From 249a1eb7f98b7883e1942523b5ba064b325d0a3c Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 19 May 2020 10:43:18 +0300 Subject: [PATCH 415/738] Fix test_server_initialization, but looking into default database only --- tests/integration/test_server_initialization/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_server_initialization/test.py b/tests/integration/test_server_initialization/test.py index 0ba8f688566..ebf50e7dc51 100644 --- a/tests/integration/test_server_initialization/test.py +++ b/tests/integration/test_server_initialization/test.py @@ -30,7 +30,7 @@ def test_sophisticated_default(started_cluster): def test_partially_dropped_tables(started_cluster): instance = started_cluster.instances['dummy'] - assert instance.exec_in_container(['bash', '-c', 'find /var/lib/clickhouse -name *.sql* | sort'], privileged=True, user='root') \ + assert instance.exec_in_container(['bash', '-c', 'find /var/lib/clickhouse/*/default -name *.sql* | sort'], privileged=True, user='root') \ == "/var/lib/clickhouse/metadata/default/should_be_restored.sql\n" \ "/var/lib/clickhouse/metadata/default/sophisticated_default.sql\n" assert instance.query("SELECT n FROM should_be_restored") == "1\n2\n3\n" From f57ac3d3f025de15dc1255671a976bfeac641650 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 19 May 2020 10:38:21 +0300 Subject: [PATCH 416/738] Update test_part_log_table - system_tables_lazy_load - SYSTEM FLUSH LOGS --- .../config_with_non_standard_part_log.xml | 4 ++-- tests/integration/test_part_log_table/test.py | 19 +++++++++---------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/tests/integration/test_part_log_table/configs/config_with_non_standard_part_log.xml b/tests/integration/test_part_log_table/configs/config_with_non_standard_part_log.xml index 2a8655de830..bb8fa5742f3 100644 --- a/tests/integration/test_part_log_table/configs/config_with_non_standard_part_log.xml +++ b/tests/integration/test_part_log_table/configs/config_with_non_standard_part_log.xml @@ -1,6 +1,6 @@ - database_name -
    table_name
    + database_name + own_part_log
    diff --git a/tests/integration/test_part_log_table/test.py b/tests/integration/test_part_log_table/test.py index a97c8cf7b3a..876f52246fa 100644 --- a/tests/integration/test_part_log_table/test.py +++ b/tests/integration/test_part_log_table/test.py @@ -21,22 +21,21 @@ def test_config_without_part_log(start_cluster): node1.query("CREATE TABLE test_table(word String, value UInt64) ENGINE=MergeTree() ORDER BY value") assert "Table system.part_log doesn't exist" in node1.query_and_get_error("SELECT * FROM system.part_log") node1.query("INSERT INTO test_table VALUES ('name', 1)") - time.sleep(10) + node1.query("SYSTEM FLUSH LOGS") assert "Table system.part_log doesn't exist" in node1.query_and_get_error("SELECT * FROM system.part_log") def test_config_with_standard_part_log(start_cluster): - assert "Table system.part_log doesn't exist" in node2.query_and_get_error("SELECT * FROM system.part_log") + assert node2.query("SELECT * FROM system.part_log") == '' node2.query("CREATE TABLE test_table(word String, value UInt64) ENGINE=MergeTree() Order by value") - assert "Table system.part_log doesn't exist" in node2.query_and_get_error("SELECT * FROM system.part_log") + assert node2.query("SELECT * FROM system.part_log") == '' node2.query("INSERT INTO test_table VALUES ('name', 1)") - time.sleep(10) - assert node2.query("SELECT * FROM system.part_log") != "" + node2.query("SYSTEM FLUSH LOGS") + assert int(node2.query("SELECT count() FROM system.part_log")) == 1 def test_config_with_non_standard_part_log(start_cluster): - assert "Table system.table_name doesn't exist" in node3.query_and_get_error("SELECT * FROM system.table_name") + assert node3.query("SELECT * FROM system.own_part_log") == '' node3.query("CREATE TABLE test_table(word String, value UInt64) ENGINE=MergeTree() Order by value") - assert "Table system.table_name doesn't exist" in node3.query_and_get_error("SELECT * FROM system.table_name") + assert node3.query("SELECT * FROM system.own_part_log") == '' node3.query("INSERT INTO test_table VALUES ('name', 1)") - time.sleep(10) - assert node3.query("SELECT * FROM system.table_name") != "" - + node3.query("SYSTEM FLUSH LOGS") + assert int(node3.query("SELECT count() FROM system.own_part_log")) == 1 From 1086cf769184c4d6f83a748c16d670d4fe453061 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Mon, 6 Apr 2020 11:37:16 +0300 Subject: [PATCH 417/738] wip --- docker/packager/deb/build.sh | 7 +++++++ docker/packager/packager | 11 +++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/docker/packager/deb/build.sh b/docker/packager/deb/build.sh index 1efed3628a0..586fe3bfcd9 100755 --- a/docker/packager/deb/build.sh +++ b/docker/packager/deb/build.sh @@ -10,5 +10,12 @@ mv *.changes /output mv *.buildinfo /output mv /*.rpm /output ||: # if exists mv /*.tgz /output ||: # if exists + +if [ "binary" == "$BINARY_OUTPUT" ] +then + mkdir /output/binary + mv ./programs/clickhouse* /output/binary + mv ./dbms/unit_tests_dbms /output/binary +fi ccache --show-stats ||: ln -s /usr/lib/x86_64-linux-gnu/libOpenCL.so.1.0.0 /usr/lib/libOpenCL.so ||: diff --git a/docker/packager/packager b/docker/packager/packager index 025ca3bf398..eff9eabf388 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -54,7 +54,7 @@ def run_docker_image_with_env(image_name, output, env_variables, ch_root, ccache subprocess.check_call(cmd, shell=True) -def parse_env_variables(build_type, compiler, sanitizer, package_type, image_type, cache, distcc_hosts, unbundled, split_binary, clang_tidy, version, author, official, alien_pkgs, with_coverage): +def parse_env_variables(build_type, compiler, sanitizer, package_type, image_type, cache, distcc_hosts, unbundled, split_binary, clang_tidy, version, author, official, alien_pkgs, with_coverage, with_binaries): CLANG_PREFIX = "clang" DARWIN_SUFFIX = "-darwin" ARM_SUFFIX = "-aarch64" @@ -131,6 +131,9 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ if alien_pkgs: result.append("ALIEN_PKGS='" + ' '.join(['--' + pkg for pkg in alien_pkgs]) + "'") + if with_binaries: + result.append('BINARY_OUTPUT=binary') + if unbundled: # TODO: fix build with ENABLE_RDKAFKA cmake_flags.append('-DUNBUNDLED=1 -DENABLE_MYSQL=0 -DENABLE_ODBC=0 -DENABLE_REPLXX=0 -DENABLE_RDKAFKA=0') @@ -179,6 +182,7 @@ if __name__ == "__main__": parser.add_argument("--official", action="store_true") parser.add_argument("--alien-pkgs", nargs='+', default=[]) parser.add_argument("--with-coverage", action="store_true") + parser.add_argument("--with-binaries", action="store_true") args = parser.parse_args() if not os.path.isabs(args.output_dir): @@ -195,6 +199,9 @@ if __name__ == "__main__": if args.alien_pkgs and not image_type == "deb": raise Exception("Can add alien packages only in deb build") + if args.with_binaries and not image_type == "deb": + raise Exception("Can add additional binaries only in deb build") + dockerfile = os.path.join(ch_root, "docker/packager", image_type, "Dockerfile") if image_type != "freebsd" and not check_image_exists_locally(image_name) or args.force_build_image: if not pull_image(image_name) or args.force_build_image: @@ -202,6 +209,6 @@ if __name__ == "__main__": env_prepared = parse_env_variables( args.build_type, args.compiler, args.sanitizer, args.package_type, image_type, args.cache, args.distcc_hosts, args.unbundled, args.split_binary, args.clang_tidy, - args.version, args.author, args.official, args.alien_pkgs, args.with_coverage) + args.version, args.author, args.official, args.alien_pkgs, args.with_coverage, args.with_binaries) run_docker_image_with_env(image_name, args.output_dir, env_prepared, ch_root, args.ccache_dir) logging.info("Output placed into {}".format(args.output_dir)) From 60f9d2088d89e2536704513420faff0c5b462f96 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Tue, 7 Apr 2020 14:33:50 +0300 Subject: [PATCH 418/738] wip --- docker/packager/deb/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/packager/deb/build.sh b/docker/packager/deb/build.sh index 586fe3bfcd9..65499b87df2 100755 --- a/docker/packager/deb/build.sh +++ b/docker/packager/deb/build.sh @@ -14,8 +14,8 @@ mv /*.tgz /output ||: # if exists if [ "binary" == "$BINARY_OUTPUT" ] then mkdir /output/binary - mv ./programs/clickhouse* /output/binary - mv ./dbms/unit_tests_dbms /output/binary + mv /build/obj-x86_64-linux-gnu/programs/clickhouse* /output/binary + mv /build/obj-x86_64-linux-gnu/dbms/unit_tests_dbms /output/binary fi ccache --show-stats ||: ln -s /usr/lib/x86_64-linux-gnu/libOpenCL.so.1.0.0 /usr/lib/libOpenCL.so ||: From b780adb241202dd3f9a7e33ce90c278c534c8f82 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Tue, 7 Apr 2020 18:15:22 +0300 Subject: [PATCH 419/738] wip --- docker/packager/deb/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/packager/deb/build.sh b/docker/packager/deb/build.sh index 65499b87df2..04233fbed45 100755 --- a/docker/packager/deb/build.sh +++ b/docker/packager/deb/build.sh @@ -15,7 +15,7 @@ if [ "binary" == "$BINARY_OUTPUT" ] then mkdir /output/binary mv /build/obj-x86_64-linux-gnu/programs/clickhouse* /output/binary - mv /build/obj-x86_64-linux-gnu/dbms/unit_tests_dbms /output/binary + mv /build/obj-x86_64-linux-gnu/src/unit_tests_dbms /output/binary fi ccache --show-stats ||: ln -s /usr/lib/x86_64-linux-gnu/libOpenCL.so.1.0.0 /usr/lib/libOpenCL.so ||: From 8d190f316deba8e0e9c87c59f7ce13d234f2b02a Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Thu, 9 Apr 2020 12:04:30 +0300 Subject: [PATCH 420/738] wip --- debian/rules | 2 +- docker/packager/deb/build.sh | 10 +++++++--- docker/packager/packager | 14 ++++++++++---- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/debian/rules b/debian/rules index dabebb516cd..d2ea9857ff6 100755 --- a/debian/rules +++ b/debian/rules @@ -92,7 +92,7 @@ override_dh_auto_test: override_dh_clean: rm -rf debian/copyright debian/clickhouse-client.docs debian/clickhouse-common-static.docs - dh_clean -X contrib + dh_clean # -X contrib override_dh_strip: dh_strip -pclickhouse-common-static --dbg-package=clickhouse-common-static-dbg diff --git a/docker/packager/deb/build.sh b/docker/packager/deb/build.sh index 04233fbed45..cfdca7abc45 100755 --- a/docker/packager/deb/build.sh +++ b/docker/packager/deb/build.sh @@ -11,11 +11,15 @@ mv *.buildinfo /output mv /*.rpm /output ||: # if exists mv /*.tgz /output ||: # if exists -if [ "binary" == "$BINARY_OUTPUT" ] +if [ -n "$BINARY_OUTPUT" ] && { [ "$BINARY_OUTPUT" = "programs" ] || [ "$BINARY_OUTPUT" = "tests" ] ;} then - mkdir /output/binary + echo Place $BINARY_OUTPUT to output + mkdir /output/binary ||: # if exists mv /build/obj-x86_64-linux-gnu/programs/clickhouse* /output/binary - mv /build/obj-x86_64-linux-gnu/src/unit_tests_dbms /output/binary + if [ "$BINARY_OUTPUT" = "tests" ] + then + mv /build/obj-x86_64-linux-gnu/src/unit_tests_dbms /output/binary + fi fi ccache --show-stats ||: ln -s /usr/lib/x86_64-linux-gnu/libOpenCL.so.1.0.0 /usr/lib/libOpenCL.so ||: diff --git a/docker/packager/packager b/docker/packager/packager index eff9eabf388..cee012a4699 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -131,8 +131,11 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ if alien_pkgs: result.append("ALIEN_PKGS='" + ' '.join(['--' + pkg for pkg in alien_pkgs]) + "'") - if with_binaries: - result.append('BINARY_OUTPUT=binary') + if with_binaries == "programs": + result.append('BINARY_OUTPUT=programs') + elif with_binaries == "tests": + result.append('BINARY_OUTPUT=tests') + cmake_flags.append('-DENABLE_TESTS=1') if unbundled: # TODO: fix build with ENABLE_RDKAFKA @@ -182,7 +185,7 @@ if __name__ == "__main__": parser.add_argument("--official", action="store_true") parser.add_argument("--alien-pkgs", nargs='+', default=[]) parser.add_argument("--with-coverage", action="store_true") - parser.add_argument("--with-binaries", action="store_true") + parser.add_argument("--with-binaries", choices=("programs", "tests", ""), default="") args = parser.parse_args() if not os.path.isabs(args.output_dir): @@ -199,9 +202,12 @@ if __name__ == "__main__": if args.alien_pkgs and not image_type == "deb": raise Exception("Can add alien packages only in deb build") - if args.with_binaries and not image_type == "deb": + if args.with_binaries != "" and not image_type == "deb": raise Exception("Can add additional binaries only in deb build") + if args.with_binaries != "" and image_type == "deb": + logging.info("Should place {} to output".format(args.with_binaries)) + dockerfile = os.path.join(ch_root, "docker/packager", image_type, "Dockerfile") if image_type != "freebsd" and not check_image_exists_locally(image_name) or args.force_build_image: if not pull_image(image_name) or args.force_build_image: From d961707512cad27c9fbf1d07e0ed26da1d2a4208 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Wed, 13 May 2020 17:47:25 +0300 Subject: [PATCH 421/738] add targets dependecies to tests --- debian/rules | 1 + programs/CMakeLists.txt | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/debian/rules b/debian/rules index d2ea9857ff6..e032dbad000 100755 --- a/debian/rules +++ b/debian/rules @@ -21,6 +21,7 @@ ifeq ($(THREADS_COUNT),) THREADS_COUNT=$(shell nproc || grep -c ^processor /proc/cpuinfo || sysctl -n hw.ncpu || echo 4) endif DEB_BUILD_OPTIONS+=parallel=$(THREADS_COUNT) +DEB_BUILD_OPTIONS+=nocheck ifndef ENABLE_TESTS CMAKE_FLAGS += -DENABLE_TESTS=0 diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index 7cbe2e7a2a6..7bc31452aa4 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -201,3 +201,9 @@ endif () if (TARGET clickhouse-server AND TARGET copy-headers) add_dependencies(clickhouse-server copy-headers) endif () + +if (ENABLE_TESTS AND USE_GTEST) + set (CLICKHOUSE_ALL_TESTS_TARGETS local_date_time_comparison unit_tests_libcommon unit_tests_dbms hashing_write_buffer hashing_read_buffer in_join_subqueries_preprocessor expression_analyzer) + add_custom_target (clickhouse-tests ALL DEPENDS ${CLICKHOUSE_ALL_TESTS_TARGETS}) + add_dependencies(clickhouse-bundle clickhouse-tests) +endif() From 9c551b8b4d2e88da2669cf51d6d134649864cd62 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Thu, 14 May 2020 10:44:15 +0300 Subject: [PATCH 422/738] set USE_GTEST for tests build --- docker/packager/packager | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/packager/packager b/docker/packager/packager index cee012a4699..ff385e168b2 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -136,6 +136,7 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ elif with_binaries == "tests": result.append('BINARY_OUTPUT=tests') cmake_flags.append('-DENABLE_TESTS=1') + cmake_flags.append('-DUSE_GTEST=1') if unbundled: # TODO: fix build with ENABLE_RDKAFKA From c66d1a03eb6e182fec9b60ac21ff6a5707dd5dca Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Mon, 18 May 2020 10:09:43 +0300 Subject: [PATCH 423/738] set nostrip for build with binaries export --- debian/rules | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/debian/rules b/debian/rules index e032dbad000..194350ec26b 100755 --- a/debian/rules +++ b/debian/rules @@ -21,10 +21,13 @@ ifeq ($(THREADS_COUNT),) THREADS_COUNT=$(shell nproc || grep -c ^processor /proc/cpuinfo || sysctl -n hw.ncpu || echo 4) endif DEB_BUILD_OPTIONS+=parallel=$(THREADS_COUNT) -DEB_BUILD_OPTIONS+=nocheck ifndef ENABLE_TESTS CMAKE_FLAGS += -DENABLE_TESTS=0 +else +# To export binaries and from deb build we do not strip them. No need to run tests in deb build as we run them in CI + DEB_BUILD_OPTIONS+=nocheck + DEB_BUILD_OPTIONS+=nostrip endif ifndef MAKE_TARGET From e3c692bc48759637d0d27cfb3a9f0d9d8afe1167 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Mon, 18 May 2020 15:32:06 +0300 Subject: [PATCH 424/738] fix issues --- debian/rules | 3 +-- docker/packager/deb/build.sh | 4 ++-- docker/packager/packager | 1 + 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/debian/rules b/debian/rules index 194350ec26b..467b86e5d79 100755 --- a/debian/rules +++ b/debian/rules @@ -26,8 +26,7 @@ ifndef ENABLE_TESTS CMAKE_FLAGS += -DENABLE_TESTS=0 else # To export binaries and from deb build we do not strip them. No need to run tests in deb build as we run them in CI - DEB_BUILD_OPTIONS+=nocheck - DEB_BUILD_OPTIONS+=nostrip + DEB_BUILD_OPTIONS+=" nocheck nostrip" endif ifndef MAKE_TARGET diff --git a/docker/packager/deb/build.sh b/docker/packager/deb/build.sh index cfdca7abc45..fbaa0151c6b 100755 --- a/docker/packager/deb/build.sh +++ b/docker/packager/deb/build.sh @@ -15,10 +15,10 @@ if [ -n "$BINARY_OUTPUT" ] && { [ "$BINARY_OUTPUT" = "programs" ] || [ "$BINARY_ then echo Place $BINARY_OUTPUT to output mkdir /output/binary ||: # if exists - mv /build/obj-x86_64-linux-gnu/programs/clickhouse* /output/binary + mv /build/obj-*/programs/clickhouse* /output/binary if [ "$BINARY_OUTPUT" = "tests" ] then - mv /build/obj-x86_64-linux-gnu/src/unit_tests_dbms /output/binary + mv /build/obj-*/src/unit_tests_dbms /output/binary fi fi ccache --show-stats ||: diff --git a/docker/packager/packager b/docker/packager/packager index ff385e168b2..8a5bdda60e8 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -134,6 +134,7 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ if with_binaries == "programs": result.append('BINARY_OUTPUT=programs') elif with_binaries == "tests": + result.append('ENABLE_TESTS=1') result.append('BINARY_OUTPUT=tests') cmake_flags.append('-DENABLE_TESTS=1') cmake_flags.append('-DUSE_GTEST=1') From 62b302e698759e0136eda3eee5152d0a08367acf Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 19 May 2020 11:48:24 +0300 Subject: [PATCH 425/738] Better link @ adopters.md --- docs/en/introduction/adopters.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/introduction/adopters.md b/docs/en/introduction/adopters.md index c3fa509165e..8868522e977 100644 --- a/docs/en/introduction/adopters.md +++ b/docs/en/introduction/adopters.md @@ -11,7 +11,7 @@ toc_title: Adopters | Company | Industry | Usecase | Cluster Size | (Un)Compressed Data Size\* | Reference | |---------------------------------------------------------------------|---------------------------------|-----------------------|------------------------------------------------------------|------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | [2gis](https://2gis.ru){.favicon} | Maps | Monitoring | — | — | [Talk in Russian, July 2019](https://youtu.be/58sPkXfq6nw) | -| [Aloha Browser](https://alohabrowser.com/){.favicon} | Mobile App | Browser backend | — | — | [Slides in Russian, May 2019](https://github.com/yandex/clickhouse-presentations/blob/master/meetup22/aloha.pdf) | +| [Aloha Browser](https://alohabrowser.com/){.favicon} | Mobile App | Browser backend | — | — | [Slides in Russian, May 2019](https://presentations.clickhouse.tech/meetup22/aloha.pdf) | | [Amadeus](https://amadeus.com/){.favicon} | Travel | Analytics | — | — | [Press Release, April 2018](https://www.altinity.com/blog/2018/4/5/amadeus-technologies-launches-investment-and-insights-tool-based-on-machine-learning-and-strategy-algorithms) | | [Appsflyer](https://www.appsflyer.com){.favicon} | Mobile analytics | Main product | — | — | [Talk in Russian, July 2019](https://www.youtube.com/watch?v=M3wbRlcpBbY) | | [ArenaData](https://arenadata.tech/){.favicon} | Data Platform | Main product | — | — | [Slides in Russian, December 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup38/indexes.pdf) | From daa519a654ee102f0f2212d7dd31064cc79822d1 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 19 May 2020 12:04:56 +0300 Subject: [PATCH 426/738] Fix ya.make --- src/Processors/ya.make | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/ya.make b/src/Processors/ya.make index f5a3c0ea30a..03a9b939d42 100644 --- a/src/Processors/ya.make +++ b/src/Processors/ya.make @@ -10,7 +10,7 @@ SRCS( Chunk.cpp ConcatProcessor.cpp DelayedPortsProcessor.cpp - Executors/PipelineExecutingBlockInputStream.h + Executors/PipelineExecutingBlockInputStream.cpp Executors/PipelineExecutor.cpp Executors/PullingPipelineExecutor.cpp Executors/TreeExecutorBlockInputStream.cpp From 665c6b7f59e0d0cba351908da424d47c43983644 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Tue, 19 May 2020 12:38:43 +0300 Subject: [PATCH 427/738] Added new files to ya make --- src/Disks/ya.make | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Disks/ya.make b/src/Disks/ya.make index a14024e7af3..0187bd54b48 100644 --- a/src/Disks/ya.make +++ b/src/Disks/ya.make @@ -5,6 +5,7 @@ PEERDIR( ) SRCS( + createVolume.cpp DiskFactory.cpp DiskLocal.cpp DiskMemory.cpp @@ -12,6 +13,7 @@ SRCS( IDisk.cpp IVolume.cpp registerDisks.cpp + SingleDiskVolume.cpp StoragePolicy.cpp VolumeJBOD.cpp ) From 70e55532041eaeac55e6c885f2602fc365fe3eaf Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 19 May 2020 12:54:56 +0300 Subject: [PATCH 428/738] At least something works --- src/Interpreters/MutationsInterpreter.cpp | 2 + src/Storages/AlterCommands.cpp | 34 +++++--- src/Storages/AlterCommands.h | 4 +- .../MergeTree/MergeTreeDataMergerMutator.cpp | 84 ++++++++++++++----- .../MergeTree/MergeTreeDataMergerMutator.h | 4 +- ...1_alter_rename_and_other_renames.reference | 24 ++++++ .../01281_alter_rename_and_other_renames.sql | 67 +++++++++++++++ 7 files changed, 182 insertions(+), 37 deletions(-) create mode 100644 tests/queries/0_stateless/01281_alter_rename_and_other_renames.reference create mode 100644 tests/queries/0_stateless/01281_alter_rename_and_other_renames.sql diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index 5ba6424653c..a1cabdc4199 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -1,5 +1,6 @@ #include "MutationsInterpreter.h" +#include #include #include #include @@ -635,6 +636,7 @@ ASTPtr MutationsInterpreter::prepareInterpreterSelectQuery(std::vector & } select->setExpression(ASTSelectQuery::Expression::WHERE, std::move(where_expression)); } + LOG_DEBUG(&Poco::Logger::get("DEBUG"), "QUERY:" << queryToString(select)); return select; } diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 91a48f13a39..1921177d0d4 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -594,7 +594,7 @@ bool AlterCommand::isCommentAlter() const return false; } -std::optional AlterCommand::tryConvertToMutationCommand(const StorageInMemoryMetadata & metadata) const +std::optional AlterCommand::tryConvertToMutationCommand(StorageInMemoryMetadata & metadata) const { if (!isRequireMutationStage(metadata)) return {}; @@ -637,6 +637,7 @@ std::optional AlterCommand::tryConvertToMutationCommand(const S } result.ast = ast->clone(); + apply(metadata); return result; } @@ -733,6 +734,7 @@ void AlterCommands::validate(const StorageInMemoryMetadata & metadata, const Con auto all_columns = metadata.columns; /// Default expression for all added/modified columns ASTPtr default_expr_list = std::make_shared(); + NameSet modified_columns, renamed_columns; for (size_t i = 0; i < size(); ++i) { const auto & command = (*this)[i]; @@ -740,7 +742,7 @@ void AlterCommands::validate(const StorageInMemoryMetadata & metadata, const Con const auto & column_name = command.column_name; if (command.type == AlterCommand::ADD_COLUMN) { - if (metadata.columns.has(column_name) || metadata.columns.hasNested(column_name)) + if (all_columns.has(column_name) || all_columns.hasNested(column_name)) { if (!command.if_not_exists) throw Exception{"Cannot add column " + backQuote(column_name) + ": column with this name already exists", @@ -757,7 +759,7 @@ void AlterCommands::validate(const StorageInMemoryMetadata & metadata, const Con } else if (command.type == AlterCommand::MODIFY_COLUMN) { - if (!metadata.columns.has(column_name)) + if (!all_columns.has(column_name)) { if (!command.if_exists) throw Exception{"Wrong column name. Cannot find column " + backQuote(column_name) + " to modify", @@ -765,18 +767,23 @@ void AlterCommands::validate(const StorageInMemoryMetadata & metadata, const Con else continue; } + + if (renamed_columns.count(column_name)) + throw Exception{"Cannot rename and modify the same column " + backQuote(column_name) + " in a single ALTER query", + ErrorCodes::NOT_IMPLEMENTED}; + modified_columns.emplace(column_name); } else if (command.type == AlterCommand::DROP_COLUMN) { - if (metadata.columns.has(command.column_name) || metadata.columns.hasNested(command.column_name)) + if (all_columns.has(command.column_name) || all_columns.hasNested(command.column_name)) { - for (const ColumnDescription & column : metadata.columns) + for (const ColumnDescription & column : all_columns) { const auto & default_expression = column.default_desc.expression; if (default_expression) { ASTPtr query = default_expression->clone(); - auto syntax_result = SyntaxAnalyzer(context).analyze(query, metadata.columns.getAll()); + auto syntax_result = SyntaxAnalyzer(context).analyze(query, all_columns.getAll()); const auto actions = ExpressionAnalyzer(query, syntax_result, context).getActions(true); const auto required_columns = actions->getRequiredColumns(); @@ -786,6 +793,7 @@ void AlterCommands::validate(const StorageInMemoryMetadata & metadata, const Con ErrorCodes::ILLEGAL_COLUMN); } } + all_columns.remove(command.column_name); } else if (!command.if_exists) throw Exception( @@ -794,7 +802,7 @@ void AlterCommands::validate(const StorageInMemoryMetadata & metadata, const Con } else if (command.type == AlterCommand::COMMENT_COLUMN) { - if (!metadata.columns.has(command.column_name)) + if (!all_columns.has(command.column_name)) { if (!command.if_exists) throw Exception{"Wrong column name. Cannot find column " + backQuote(command.column_name) + " to comment", @@ -842,6 +850,10 @@ void AlterCommands::validate(const StorageInMemoryMetadata & metadata, const Con throw Exception{"Cannot rename to " + backQuote(command.rename_to) + ": column with this name already exists", ErrorCodes::DUPLICATE_COLUMN}; + if (modified_columns.count(column_name)) + throw Exception{"Cannot rename and modify the same column " + backQuote(column_name) + " in a single ALTER query", + ErrorCodes::NOT_IMPLEMENTED}; + String from_nested_table_name = Nested::extractTableName(command.column_name); String to_nested_table_name = Nested::extractTableName(command.rename_to); bool from_nested = from_nested_table_name != command.column_name; @@ -855,6 +867,8 @@ void AlterCommands::validate(const StorageInMemoryMetadata & metadata, const Con else if (!from_nested && !to_nested) { all_columns.rename(command.column_name, command.rename_to); + renamed_columns.emplace(command.column_name); + renamed_columns.emplace(command.rename_to); } else { @@ -886,9 +900,9 @@ void AlterCommands::validate(const StorageInMemoryMetadata & metadata, const Con default_expr_list->children.emplace_back(setAlias(command.default_expression->clone(), tmp_column_name)); } } /// if we change data type for column with default - else if (metadata.columns.has(column_name) && command.data_type) + else if (all_columns.has(column_name) && command.data_type) { - auto column_in_table = metadata.columns.get(column_name); + auto column_in_table = all_columns.get(column_name); /// Column doesn't have a default, nothing to check if (!column_in_table.default_desc.expression) continue; @@ -931,7 +945,7 @@ bool AlterCommands::isCommentAlter() const } -MutationCommands AlterCommands::getMutationCommands(const StorageInMemoryMetadata & metadata) const +MutationCommands AlterCommands::getMutationCommands(StorageInMemoryMetadata metadata) const { MutationCommands result; for (const auto & alter_cmd : *this) diff --git a/src/Storages/AlterCommands.h b/src/Storages/AlterCommands.h index c1c913dad73..94cf2a2ba37 100644 --- a/src/Storages/AlterCommands.h +++ b/src/Storages/AlterCommands.h @@ -121,7 +121,7 @@ struct AlterCommand /// If possible, convert alter command to mutation command. In other case /// return empty optional. Some storages may execute mutations after /// metadata changes. - std::optional tryConvertToMutationCommand(const StorageInMemoryMetadata & metadata) const; + std::optional tryConvertToMutationCommand(StorageInMemoryMetadata & metadata) const; }; /// Return string representation of AlterCommand::Type @@ -162,7 +162,7 @@ public: /// Return mutation commands which some storages may execute as part of /// alter. If alter can be performed is pure metadata update, than result is /// empty. - MutationCommands getMutationCommands(const StorageInMemoryMetadata & metadata) const; + MutationCommands getMutationCommands(StorageInMemoryMetadata metadata) const; }; } diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index d84a82dd372..cf88414436e 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -407,7 +407,7 @@ MergeTreeData::DataPartsVector MergeTreeDataMergerMutator::selectAllPartsFromPar /// PK columns are sorted and merged, ordinary columns are gathered using info from merge step static void extractMergingAndGatheringColumns( - const NamesAndTypesList & all_columns, + const NamesAndTypesList & storage_columns, const ExpressionActionsPtr & sorting_key_expr, const MergeTreeIndices & indexes, const MergeTreeData::MergingParams & merging_params, @@ -437,11 +437,11 @@ static void extractMergingAndGatheringColumns( /// Force to merge at least one column in case of empty key if (key_columns.empty()) - key_columns.emplace(all_columns.front().name); + key_columns.emplace(storage_columns.front().name); /// TODO: also force "summing" and "aggregating" columns to make Horizontal merge only for such columns - for (const auto & column : all_columns) + for (const auto & column : storage_columns) { if (key_columns.count(column.name)) { @@ -600,14 +600,14 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor MergeTreeData::DataPart::ColumnToSize merged_column_to_size; Names all_column_names = data.getColumns().getNamesOfPhysical(); - NamesAndTypesList all_columns = data.getColumns().getAllPhysical(); + NamesAndTypesList storage_columns = data.getColumns().getAllPhysical(); const auto data_settings = data.getSettings(); NamesAndTypesList gathering_columns; NamesAndTypesList merging_columns; Names gathering_column_names, merging_column_names; extractMergingAndGatheringColumns( - all_columns, data.sorting_key_expr, data.skip_indices, + storage_columns, data.sorting_key_expr, data.skip_indices, data.merging_params, gathering_columns, gathering_column_names, merging_columns, merging_column_names); MergeTreeData::MutableDataPartPtr new_data_part = data.createPart( @@ -617,7 +617,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor disk, TMP_PREFIX + future_part.name); - new_data_part->setColumns(all_columns); + new_data_part->setColumns(storage_columns); new_data_part->partition.assign(future_part.getPartition()); new_data_part->is_temp = true; @@ -669,7 +669,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor } else { - merging_columns = all_columns; + merging_columns = storage_columns; merging_column_names = all_column_names; gathering_columns.clear(); gathering_column_names.clear(); @@ -959,7 +959,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor if (merge_alg != MergeAlgorithm::Vertical) to.writeSuffixAndFinalizePart(new_data_part); else - to.writeSuffixAndFinalizePart(new_data_part, &all_columns, &checksums_gathered_columns); + to.writeSuffixAndFinalizePart(new_data_part, &storage_columns, &checksums_gathered_columns); return new_data_part; } @@ -1013,12 +1013,16 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor const auto data_settings = data.getSettings(); MutationCommands for_interpreter, for_file_renames; + LOG_DEBUG(&Poco::Logger::get("DEBUG"), "COMMANDS FOR PART:" << commands_for_part.size()); + LOG_DEBUG(&Poco::Logger::get("DEBUG"), "TOTAL COLUMNS:" << commands.size()); splitMutationCommands(source_part, commands_for_part, for_interpreter, for_file_renames); + LOG_DEBUG(&Poco::Logger::get("DEBUG"), "COMMANDS FOR INTERPRETER:" << for_interpreter.size()); + LOG_DEBUG(&Poco::Logger::get("DEBUG"), "COMMANDS FOR FILE RENAMES:" << for_file_renames.size()); UInt64 watch_prev_elapsed = 0; MergeStageProgress stage_progress(1.0); - NamesAndTypesList all_columns = data.getColumns().getAllPhysical(); + NamesAndTypesList storage_columns = data.getColumns().getAllPhysical(); if (!for_interpreter.empty()) { @@ -1026,6 +1030,9 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor in = interpreter->execute(table_lock_holder); updated_header = interpreter->getUpdatedHeader(); in->setProgressCallback(MergeProgressCallback(merge_entry, watch_prev_elapsed, stage_progress)); + + LOG_DEBUG(&Poco::Logger::get("DEBUG"), "UPDATED HEADER:" << updated_header.dumpStructure()); + LOG_DEBUG(&Poco::Logger::get("DEBUG"), "STREAM HEADER:" << in->getHeader().dumpStructure()); } auto new_data_part = data.createPart( @@ -1036,7 +1043,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor /// It shouldn't be changed by mutation. new_data_part->index_granularity_info = source_part->index_granularity_info; - new_data_part->setColumns(getColumnsForNewDataPart(source_part, updated_header, all_columns, for_file_renames)); + new_data_part->setColumns(getColumnsForNewDataPart(source_part, updated_header, storage_columns, for_file_renames)); new_data_part->partition.assign(source_part->partition); auto disk = new_data_part->disk; @@ -1113,7 +1120,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor disk->createHardLink(it->path(), destination); } - merge_entry->columns_written = all_columns.size() - updated_header.columns(); + merge_entry->columns_written = storage_columns.size() - updated_header.columns(); new_data_part->checksums = source_part->checksums; @@ -1258,6 +1265,18 @@ void MergeTreeDataMergerMutator::splitMutationCommands( NameSet removed_columns_from_compact_part; NameSet already_changed_columns; bool is_compact_part = isCompactPart(part); + ColumnsDescription part_columns(part->getColumns()); + NameSet modified_columns; + for (const auto & command : commands) + { + if (command.type == MutationCommand::Type::DELETE || command.type == MutationCommand::Type::UPDATE + || command.type == MutationCommand::Type::MATERIALIZE_INDEX || command.type == MutationCommand::Type::MATERIALIZE_TTL || command.type == MutationCommand::Type::READ_COLUMN) + { + modified_columns.emplace(command.column_name); + } + } + + for (const auto & command : commands) { if (command.type == MutationCommand::Type::DELETE @@ -1269,28 +1288,30 @@ void MergeTreeDataMergerMutator::splitMutationCommands( for (const auto & [column_name, expr] : command.column_to_update_expression) already_changed_columns.emplace(column_name); } - else if (command.type == MutationCommand::Type::READ_COLUMN) + else if (command.type == MutationCommand::Type::READ_COLUMN && part_columns.has(command.column_name)) { + LOG_DEBUG(&Poco::Logger::get("DEBUG"), "READ COLUMN:" << command.column_name); /// If we don't have this column in source part, than we don't /// need to materialize it - if (part->getColumns().contains(command.column_name)) + if (part_columns.has(command.column_name)) { + LOG_DEBUG(&Poco::Logger::get("DEBUG"), "PART HAS COLUMN:" << command.column_name); for_interpreter.push_back(command); - if (!command.column_name.empty()) - already_changed_columns.emplace(command.column_name); + already_changed_columns.emplace(command.column_name); } else for_file_renames.push_back(command); } - else if (is_compact_part && command.type == MutationCommand::Type::DROP_COLUMN) + else if (is_compact_part && command.type == MutationCommand::Type::DROP_COLUMN && part_columns.has(command.column_name)) { removed_columns_from_compact_part.emplace(command.column_name); for_file_renames.push_back(command); + part_columns.remove(command.column_name); } - else if (command.type == MutationCommand::Type::RENAME_COLUMN) + else if (command.type == MutationCommand::Type::RENAME_COLUMN && part_columns.has(command.column_name)) { - if (is_compact_part) + if (is_compact_part || modified_columns.count(command.rename_to)) { for_interpreter.push_back( { @@ -1298,11 +1319,15 @@ void MergeTreeDataMergerMutator::splitMutationCommands( .column_name = command.rename_to, }); already_changed_columns.emplace(command.column_name); + part_columns.rename(command.column_name, command.rename_to); } else + { + part_columns.rename(command.column_name, command.rename_to); for_file_renames.push_back(command); + } } - else + else if (part_columns.has(command.column_name)) { for_file_renames.push_back(command); } @@ -1429,9 +1454,12 @@ NameSet MergeTreeDataMergerMutator::collectFilesToSkip( NamesAndTypesList MergeTreeDataMergerMutator::getColumnsForNewDataPart( MergeTreeData::DataPartPtr source_part, const Block & updated_header, - NamesAndTypesList all_columns, + NamesAndTypesList storage_columns, const MutationCommands & commands_for_removes) { + LOG_DEBUG(&Poco::Logger::get("DEBUG"), "COLUMNS FOr NEW PART START:" << storage_columns.toString()); + if (isCompactPart(source_part)) + return updated_header.getNamesAndTypesList(); NameSet removed_columns; NameToNameMap renamed_columns; for (const auto & command : commands_for_removes) @@ -1439,12 +1467,17 @@ NamesAndTypesList MergeTreeDataMergerMutator::getColumnsForNewDataPart( if (command.type == MutationCommand::DROP_COLUMN) removed_columns.insert(command.column_name); if (command.type == MutationCommand::RENAME_COLUMN) + { + LOG_DEBUG(&Poco::Logger::get("DEBUG"), "RENAME FROM:" << command.column_name << " TO:" << command.rename_to); renamed_columns.emplace(command.rename_to, command.column_name); + } } Names source_column_names = source_part->getColumns().getNames(); NameSet source_columns_name_set(source_column_names.begin(), source_column_names.end()); - for (auto it = all_columns.begin(); it != all_columns.end();) + NameSet columns_that_was_renamed; + for (auto it = storage_columns.begin(); it != storage_columns.end();) { + LOG_DEBUG(&Poco::Logger::get("DEBUG"), "LOOKING AT:" << it->name); if (updated_header.has(it->name)) { auto updated_type = updated_header.getByName(it->name).type; @@ -1461,9 +1494,14 @@ NamesAndTypesList MergeTreeDataMergerMutator::getColumnsForNewDataPart( ++it; } else - it = all_columns.erase(it); + { + LOG_DEBUG(&Poco::Logger::get("DEBUG"), "ERASE:" << it->name); + it = storage_columns.erase(it); + } } - return all_columns; + + LOG_DEBUG(&Poco::Logger::get("DEBUG"), "COLUMNS FOr NEW PART FINISH:" << storage_columns.toString()); + return storage_columns; } MergeTreeIndices MergeTreeDataMergerMutator::getIndicesForNewDataPart( diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.h b/src/Storages/MergeTree/MergeTreeDataMergerMutator.h index 78de7375d70..431d059ba60 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.h +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.h @@ -153,11 +153,11 @@ private: /// Because we will generate new versions of them after we perform mutation. static NameSet collectFilesToSkip(const Block & updated_header, const std::set & indices_to_recalc, const String & mrk_extension); - /// Get the columns list of the resulting part in the same order as all_columns. + /// Get the columns list of the resulting part in the same order as storage_columns. static NamesAndTypesList getColumnsForNewDataPart( MergeTreeData::DataPartPtr source_part, const Block & updated_header, - NamesAndTypesList all_columns, + NamesAndTypesList storage_columns, const MutationCommands & commands_for_removes); /// Get skip indcies, that should exists in the resulting data part. diff --git a/tests/queries/0_stateless/01281_alter_rename_and_other_renames.reference b/tests/queries/0_stateless/01281_alter_rename_and_other_renames.reference new file mode 100644 index 00000000000..f0a906147ac --- /dev/null +++ b/tests/queries/0_stateless/01281_alter_rename_and_other_renames.reference @@ -0,0 +1,24 @@ +CREATE TABLE default.rename_table_multiple\n(\n `key` Int32, \n `value1_string` String, \n `value2` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +key value1_string value2 +1 2 3 +CREATE TABLE default.rename_table_multiple\n(\n `key` Int32, \n `value1_string` String, \n `value2_old` Int32, \n `value2` Int64 DEFAULT 7\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +key value1_string value2_old value2 +1 2 3 7 +4 5 6 7 +CREATE TABLE default.rename_table_multiple\n(\n `key` Int32, \n `value1_string` String, \n `value2_old` Int64 DEFAULT 7\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +key value1_string value2_old +1 2 7 +4 5 7 +7 8 10 +CREATE TABLE default.rename_table_multiple_compact\n(\n `key` Int32, \n `value1_string` String, \n `value2` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS min_rows_for_wide_part = 100000, index_granularity = 8192 +key value1_string value2 +1 2 3 +CREATE TABLE default.rename_table_multiple_compact\n(\n `key` Int32, \n `value1_string` String, \n `value2_old` Int32, \n `value2` Int64 DEFAULT 7\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS min_rows_for_wide_part = 100000, index_granularity = 8192 +key value1_string value2_old value2 +1 2 3 7 +4 5 6 7 +CREATE TABLE default.rename_table_multiple_compact\n(\n `key` Int32, \n `value1_string` String, \n `value2_old` Int64 DEFAULT 7\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS min_rows_for_wide_part = 100000, index_granularity = 8192 +key value1_string value2_old +1 2 7 +4 5 7 +7 8 10 diff --git a/tests/queries/0_stateless/01281_alter_rename_and_other_renames.sql b/tests/queries/0_stateless/01281_alter_rename_and_other_renames.sql new file mode 100644 index 00000000000..f9462f0478e --- /dev/null +++ b/tests/queries/0_stateless/01281_alter_rename_and_other_renames.sql @@ -0,0 +1,67 @@ +DROP TABLE IF EXISTS rename_table_multiple; + +CREATE TABLE rename_table_multiple (key Int32, value1 String, value2 Int32) ENGINE = MergeTree ORDER BY tuple(); + +INSERT INTO rename_table_multiple VALUES (1, 2, 3); + +ALTER TABLE rename_table_multiple RENAME COLUMN value1 TO value1_string, MODIFY COLUMN value1_string String; --{serverError 48} +ALTER TABLE rename_table_multiple MODIFY COLUMN value1 String, RENAME COLUMN value1 to value1_string; --{serverError 48} + +ALTER TABLE rename_table_multiple RENAME COLUMN value1 TO value1_string; +ALTER TABLE rename_table_multiple MODIFY COLUMN value1_string String; + +SHOW CREATE TABLE rename_table_multiple; + +SELECT * FROM rename_table_multiple FORMAT TSVWithNames; + +INSERT INTO rename_table_multiple VALUES (4, '5', 6); + +ALTER TABLE rename_table_multiple RENAME COLUMN value2 TO value2_old, ADD COLUMN value2 Int64 DEFAULT 7; + +SHOW CREATE TABLE rename_table_multiple; + +SELECT * FROM rename_table_multiple ORDER BY key FORMAT TSVWithNames; + +INSERT INTO rename_table_multiple VALUES (7, '8', 9, 10); + +ALTER TABLE rename_table_multiple DROP COLUMN value2_old, RENAME COLUMN value2 TO value2_old; + +SHOW CREATE TABLE rename_table_multiple; + +SELECT * FROM rename_table_multiple ORDER BY key FORMAT TSVWithNames; + +DROP TABLE IF EXISTS rename_table_multiple; + +DROP TABLE IF EXISTS rename_table_multiple_compact; + +CREATE TABLE rename_table_multiple_compact (key Int32, value1 String, value2 Int32) ENGINE = MergeTree ORDER BY tuple() SETTINGS min_rows_for_wide_part = 100000; + +INSERT INTO rename_table_multiple_compact VALUES (1, 2, 3); + +ALTER TABLE rename_table_multiple_compact RENAME COLUMN value1 TO value1_string, MODIFY COLUMN value1_string String; --{serverError 48} +ALTER TABLE rename_table_multiple_compact MODIFY COLUMN value1 String, RENAME COLUMN value1 to value1_string; --{serverError 48} + +ALTER TABLE rename_table_multiple_compact RENAME COLUMN value1 TO value1_string; +ALTER TABLE rename_table_multiple_compact MODIFY COLUMN value1_string String; + +SHOW CREATE TABLE rename_table_multiple_compact; + +SELECT * FROM rename_table_multiple_compact FORMAT TSVWithNames; + +INSERT INTO rename_table_multiple_compact VALUES (4, '5', 6); + +ALTER TABLE rename_table_multiple_compact RENAME COLUMN value2 TO value2_old, ADD COLUMN value2 Int64 DEFAULT 7; + +SHOW CREATE TABLE rename_table_multiple_compact; + +SELECT * FROM rename_table_multiple_compact ORDER BY key FORMAT TSVWithNames; + +INSERT INTO rename_table_multiple_compact VALUES (7, '8', 9, 10); + +ALTER TABLE rename_table_multiple_compact DROP COLUMN value2_old, RENAME COLUMN value2 TO value2_old; + +SHOW CREATE TABLE rename_table_multiple_compact; + +SELECT * FROM rename_table_multiple_compact ORDER BY key FORMAT TSVWithNames; + +DROP TABLE IF EXISTS rename_table_multiple_compact; From 111c576579a983f23dee89967637c174e632c415 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 19 May 2020 13:03:05 +0300 Subject: [PATCH 429/738] Slightly less garbage --- .../MergeTree/MergeTreeDataMergerMutator.cpp | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index cf88414436e..6610629c159 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -1266,16 +1266,6 @@ void MergeTreeDataMergerMutator::splitMutationCommands( NameSet already_changed_columns; bool is_compact_part = isCompactPart(part); ColumnsDescription part_columns(part->getColumns()); - NameSet modified_columns; - for (const auto & command : commands) - { - if (command.type == MutationCommand::Type::DELETE || command.type == MutationCommand::Type::UPDATE - || command.type == MutationCommand::Type::MATERIALIZE_INDEX || command.type == MutationCommand::Type::MATERIALIZE_TTL || command.type == MutationCommand::Type::READ_COLUMN) - { - modified_columns.emplace(command.column_name); - } - } - for (const auto & command : commands) { @@ -1311,7 +1301,7 @@ void MergeTreeDataMergerMutator::splitMutationCommands( } else if (command.type == MutationCommand::Type::RENAME_COLUMN && part_columns.has(command.column_name)) { - if (is_compact_part || modified_columns.count(command.rename_to)) + if (is_compact_part) { for_interpreter.push_back( { From 2b37a418f2df083ce5b2bba5d0b96fa9fcb564fc Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 19 May 2020 13:44:53 +0300 Subject: [PATCH 430/738] More clear logic --- .../MergeTree/MergeTreeDataMergerMutator.cpp | 198 +++++++++++------- 1 file changed, 122 insertions(+), 76 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 6610629c159..aa09d6ddf9e 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -1013,11 +1013,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor const auto data_settings = data.getSettings(); MutationCommands for_interpreter, for_file_renames; - LOG_DEBUG(&Poco::Logger::get("DEBUG"), "COMMANDS FOR PART:" << commands_for_part.size()); - LOG_DEBUG(&Poco::Logger::get("DEBUG"), "TOTAL COLUMNS:" << commands.size()); splitMutationCommands(source_part, commands_for_part, for_interpreter, for_file_renames); - LOG_DEBUG(&Poco::Logger::get("DEBUG"), "COMMANDS FOR INTERPRETER:" << for_interpreter.size()); - LOG_DEBUG(&Poco::Logger::get("DEBUG"), "COMMANDS FOR FILE RENAMES:" << for_file_renames.size()); UInt64 watch_prev_elapsed = 0; MergeStageProgress stage_progress(1.0); @@ -1030,9 +1026,6 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor in = interpreter->execute(table_lock_holder); updated_header = interpreter->getUpdatedHeader(); in->setProgressCallback(MergeProgressCallback(merge_entry, watch_prev_elapsed, stage_progress)); - - LOG_DEBUG(&Poco::Logger::get("DEBUG"), "UPDATED HEADER:" << updated_header.dumpStructure()); - LOG_DEBUG(&Poco::Logger::get("DEBUG"), "STREAM HEADER:" << in->getHeader().dumpStructure()); } auto new_data_part = data.createPart( @@ -1262,85 +1255,143 @@ void MergeTreeDataMergerMutator::splitMutationCommands( MutationCommands & for_interpreter, MutationCommands & for_file_renames) { - NameSet removed_columns_from_compact_part; - NameSet already_changed_columns; - bool is_compact_part = isCompactPart(part); ColumnsDescription part_columns(part->getColumns()); - for (const auto & command : commands) + if (isCompactPart(part)) { - if (command.type == MutationCommand::Type::DELETE - || command.type == MutationCommand::Type::UPDATE - || command.type == MutationCommand::Type::MATERIALIZE_INDEX - || command.type == MutationCommand::Type::MATERIALIZE_TTL) + NameSet mutated_columns; + for (const auto & command : commands) { - for_interpreter.push_back(command); - for (const auto & [column_name, expr] : command.column_to_update_expression) - already_changed_columns.emplace(column_name); - } - else if (command.type == MutationCommand::Type::READ_COLUMN && part_columns.has(command.column_name)) - { - LOG_DEBUG(&Poco::Logger::get("DEBUG"), "READ COLUMN:" << command.column_name); - /// If we don't have this column in source part, than we don't - /// need to materialize it - if (part_columns.has(command.column_name)) + if (command.type == MutationCommand::Type::MATERIALIZE_INDEX || command.type == MutationCommand::Type::MATERIALIZE_TTL + || command.type == MutationCommand::Type::DELETE || command.type == MutationCommand::Type::UPDATE) { - LOG_DEBUG(&Poco::Logger::get("DEBUG"), "PART HAS COLUMN:" << command.column_name); for_interpreter.push_back(command); - already_changed_columns.emplace(command.column_name); + for (const auto & [column_name, expr] : command.column_to_update_expression) + mutated_columns.emplace(column_name); } - else - for_file_renames.push_back(command); - - } - else if (is_compact_part && command.type == MutationCommand::Type::DROP_COLUMN && part_columns.has(command.column_name)) - { - removed_columns_from_compact_part.emplace(command.column_name); - for_file_renames.push_back(command); - part_columns.remove(command.column_name); - } - else if (command.type == MutationCommand::Type::RENAME_COLUMN && part_columns.has(command.column_name)) - { - if (is_compact_part) + else if (part_columns.has(command.column_name)) { - for_interpreter.push_back( + if (command.type == MutationCommand::Type::DROP_COLUMN) { - .type = MutationCommand::Type::READ_COLUMN, - .column_name = command.rename_to, - }); - already_changed_columns.emplace(command.column_name); - part_columns.rename(command.column_name, command.rename_to); - } - else - { - part_columns.rename(command.column_name, command.rename_to); - for_file_renames.push_back(command); + mutated_columns.emplace(command.column_name); + } + else if (command.type == MutationCommand::Type::RENAME_COLUMN) + { + for_interpreter.push_back({ + .type = MutationCommand::Type::READ_COLUMN, + .column_name = command.rename_to, + }); + mutated_columns.emplace(command.column_name); + part_columns.rename(command.column_name, command.rename_to); + } } } - else if (part_columns.has(command.column_name)) - { - for_file_renames.push_back(command); - } - } - - if (is_compact_part) - { - /// If it's compact part than we don't need to actually remove files from disk - /// we just don't read dropped columns + /// If it's compact part than we don't need to actually remove files + /// from disk we just don't read dropped columns for (const auto & column : part->getColumns()) { - if (!removed_columns_from_compact_part.count(column.name) - && !already_changed_columns.count(column.name)) + if (!mutated_columns.count(column.name)) + for_interpreter.emplace_back( + MutationCommand{.type = MutationCommand::Type::READ_COLUMN, .column_name = column.name, .data_type = column.type}); + } + } + else + { + for (const auto & command : commands) + { + if (command.type == MutationCommand::Type::MATERIALIZE_INDEX || command.type == MutationCommand::Type::MATERIALIZE_TTL + || command.type == MutationCommand::Type::DELETE || command.type == MutationCommand::Type::UPDATE) { - for_interpreter.emplace_back(MutationCommand + for_interpreter.push_back(command); + } + /// If we don't have this column in source part, than we don't need + /// to materialize it + else if (part_columns.has(command.column_name)) + { + if (command.type == MutationCommand::Type::READ_COLUMN) { - .type = MutationCommand::Type::READ_COLUMN, - .column_name = column.name, - .data_type = column.type - }); + for_interpreter.push_back(command); + } + else if (command.type == MutationCommand::Type::RENAME_COLUMN) + { + part_columns.rename(command.column_name, command.rename_to); + for_file_renames.push_back(command); + } + else + { + for_file_renames.push_back(command); + } } } } + //for (const auto & command : commands) + //{ + // if (command.type == MutationCommand::Type::MATERIALIZE_INDEX || command.type == MutationCommand::Type::MATERIALIZE_TTL) + // { + // for_interpreter.push_back(command); + // for (const auto & [column_name, expr] : command.column_to_update_expression) + // already_changed_columns.emplace(column_name); + // } + // if (command.type == MutationCommand::Type::DELETE + // || command.type == MutationCommand::Type::UPDATE + // || + // || ) + // { + // } + // else if (command.type == MutationCommand::Type::READ_COLUMN && ) + // { + // if (part_columns.has(command.column_name)) + // { + // for_interpreter.push_back(command); + // already_changed_columns.emplace(command.column_name); + // } + // else + // for_file_renames.push_back(command); + + // } + // else if (is_compact_part && command.type == MutationCommand::Type::DROP_COLUMN && part_columns.has(command.column_name)) + // { + // for_file_renames.push_back(command); + // part_columns.remove(command.column_name); + // } + // else if (command.type == MutationCommand::Type::RENAME_COLUMN && part_columns.has(command.column_name)) + // { + // if (is_compact_part) + // { + // for_interpreter.push_back( + // { + // .type = MutationCommand::Type::READ_COLUMN, + // .column_name = command.rename_to, + // }); + // already_changed_columns.emplace(command.column_name); + // part_columns.rename(command.column_name, command.rename_to); + // } + // else + // { + // } + // } + // else if (part_columns.has(command.column_name)) + // { + // for_file_renames.push_back(command); + // } + //} + + //if (is_compact_part) + //{ + // for (const auto & column : part->getColumns()) + // { + // if (!removed_columns_from_compact_part.count(column.name) + // && !already_changed_columns.count(column.name)) + // { + // for_interpreter.emplace_back(MutationCommand + // { + // .type = MutationCommand::Type::READ_COLUMN, + // .column_name = column.name, + // .data_type = column.type + // }); + // } + // } + //} } @@ -1447,9 +1498,11 @@ NamesAndTypesList MergeTreeDataMergerMutator::getColumnsForNewDataPart( NamesAndTypesList storage_columns, const MutationCommands & commands_for_removes) { - LOG_DEBUG(&Poco::Logger::get("DEBUG"), "COLUMNS FOr NEW PART START:" << storage_columns.toString()); + /// In compact parts we read all columns, because they all stored in a + /// single file if (isCompactPart(source_part)) return updated_header.getNamesAndTypesList(); + NameSet removed_columns; NameToNameMap renamed_columns; for (const auto & command : commands_for_removes) @@ -1457,17 +1510,12 @@ NamesAndTypesList MergeTreeDataMergerMutator::getColumnsForNewDataPart( if (command.type == MutationCommand::DROP_COLUMN) removed_columns.insert(command.column_name); if (command.type == MutationCommand::RENAME_COLUMN) - { - LOG_DEBUG(&Poco::Logger::get("DEBUG"), "RENAME FROM:" << command.column_name << " TO:" << command.rename_to); renamed_columns.emplace(command.rename_to, command.column_name); - } } Names source_column_names = source_part->getColumns().getNames(); NameSet source_columns_name_set(source_column_names.begin(), source_column_names.end()); - NameSet columns_that_was_renamed; for (auto it = storage_columns.begin(); it != storage_columns.end();) { - LOG_DEBUG(&Poco::Logger::get("DEBUG"), "LOOKING AT:" << it->name); if (updated_header.has(it->name)) { auto updated_type = updated_header.getByName(it->name).type; @@ -1485,12 +1533,10 @@ NamesAndTypesList MergeTreeDataMergerMutator::getColumnsForNewDataPart( } else { - LOG_DEBUG(&Poco::Logger::get("DEBUG"), "ERASE:" << it->name); it = storage_columns.erase(it); } } - LOG_DEBUG(&Poco::Logger::get("DEBUG"), "COLUMNS FOr NEW PART FINISH:" << storage_columns.toString()); return storage_columns; } From bc5f68f1eed3bb815463121a8d349859946b0ebf Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 19 May 2020 13:45:02 +0300 Subject: [PATCH 431/738] [docs] initial support for article rating (#11033) --- website/js/base.js | 3 +- website/js/docs.js | 60 ++++++++++++++++++++++ website/js/index.js | 5 +- website/locale/en/LC_MESSAGES/messages.po | 54 +++++++++++++++---- website/locale/es/LC_MESSAGES/messages.mo | Bin 6274 -> 6641 bytes website/locale/es/LC_MESSAGES/messages.po | 54 +++++++++++++++---- website/locale/fa/LC_MESSAGES/messages.mo | Bin 7059 -> 7453 bytes website/locale/fa/LC_MESSAGES/messages.po | 54 +++++++++++++++---- website/locale/fr/LC_MESSAGES/messages.mo | Bin 6135 -> 6490 bytes website/locale/fr/LC_MESSAGES/messages.po | 54 +++++++++++++++---- website/locale/ja/LC_MESSAGES/messages.mo | Bin 6258 -> 6613 bytes website/locale/ja/LC_MESSAGES/messages.po | 54 +++++++++++++++---- website/locale/messages.pot | 54 +++++++++++++++---- website/locale/ru/LC_MESSAGES/messages.mo | Bin 8110 -> 8554 bytes website/locale/ru/LC_MESSAGES/messages.po | 54 +++++++++++++++---- website/locale/tr/LC_MESSAGES/messages.mo | Bin 6029 -> 6378 bytes website/locale/tr/LC_MESSAGES/messages.po | 54 +++++++++++++++---- website/locale/zh/LC_MESSAGES/messages.mo | Bin 5582 -> 5926 bytes website/locale/zh/LC_MESSAGES/messages.po | 54 +++++++++++++++---- website/main.html | 1 + website/templates/docs/footer.html | 11 ++++ website/templates/docs/ld_json.html | 11 ++-- 22 files changed, 489 insertions(+), 88 deletions(-) diff --git a/website/js/base.js b/website/js/base.js index 7311da8d7a4..4d0768ef919 100644 --- a/website/js/base.js +++ b/website/js/base.js @@ -5,6 +5,7 @@ var selector = target.attr('href'); var is_tab = target.attr('role') === 'tab'; var is_collapse = target.attr('data-toggle') === 'collapse'; + var is_rating = target.attr('role') === 'rating'; var navbar_toggle = $('#navbar-toggle'); navbar_toggle.collapse('hide'); @@ -14,7 +15,7 @@ selector = '#'; } - if (selector && selector.startsWith('#') && !is_tab && !is_collapse) { + if (selector && selector.startsWith('#') && !is_tab && !is_collapse && !is_rating) { event.preventDefault(); var dst = window.location.href.replace(window.location.hash, ''); var offset = 0; diff --git a/website/js/docs.js b/website/js/docs.js index f43b8f06af5..7c1d8645cd2 100644 --- a/website/js/docs.js +++ b/website/js/docs.js @@ -105,4 +105,64 @@ $(document).ready(function () { debug: false }); } + + var rating_stars = $('#rating-stars'); + if (rating_stars.length) { + var key = ''; + var stars_text = rating_stars.text().replace(/^\s+|\s+$/g, ''); + var url_match = window.location.pathname.match(/^\/docs(?:\/v[0-9]+\.[0-9]+)?(?:\/[a-z]{2})(.*)/); + if (url_match && url_match.length > 1) { + key = url_match[1]; + if (key.length > 2) { + key = key.replace(/^\/|\/$/g, ''); + } + key = 'rating_' + key; + var local_stars = localStorage.getItem(key); + if (local_stars) { + stars_text = local_stars; + rating_stars.addClass('text-orange'); + } + } + var titles = JSON.parse(rating_stars.attr('data-titles').replace(/'/g, '"')); + var documentation = rating_stars.attr('data-documentation'); + var replacement = ''; + for (var i = 0; i < 5; i++) { + var star = stars_text.charAt(i); + replacement += '' + star + ''; + } + rating_stars.html(replacement); + $('[data-toggle="tooltip"]').tooltip(); + var rating_star_item = $('.rating-star'); + rating_star_item.hover(function() { + var current = $(this); + current.text('★'); + current.prevAll().text('★'); + current.nextAll().text('☆'); + }); + rating_stars.mouseleave(function() { + for (var i = 0; i < 5; i++) { + rating_stars.children().eq(i).text(stars_text[i]); + } + }); + rating_star_item.click(function(e) { + e.preventDefault(); + rating_stars.addClass('text-orange'); + stars_text = rating_stars.text(); + localStorage.setItem(key, stars_text); + $.ajax({ + url: window.location.pathname + 'rate/', + type: 'POST', + dataType: 'json', + data: {rating: $(this).prevAll().length + 1}, + success: function () { + }, + error: function () { + rating_stars.removeClass('text-orange'); + localStorage.removeItem(key); + } + }); + }); + } }); diff --git a/website/js/index.js b/website/js/index.js index 0e5a62dadd9..21c3147db98 100644 --- a/website/js/index.js +++ b/website/js/index.js @@ -23,18 +23,17 @@ $(document).ready(function () { }); if (valid) { var data = JSON.stringify(serializeForm(meetup_form)); - console.log(data); $.ajax({ url: '/meet-form/', type: 'POST', dataType: 'json', data: data, success: function () { - meetup_form.html('

    Thanks!

    We\'ll be in touch soon.

    ') + meetup_form.html('

    Thanks!

    We\'ll be in touch soon.

    '); $('#meetup-form-error').html(''); }, error: function () { - $('#meetup-form-error').html('
    Error! Unfortunately it didn\'t work for some reason, please try again or use the email address below.
    ') + $('#meetup-form-error').html('
    Error! Unfortunately it didn\'t work for some reason, please try again or use the email address below.
    '); } }); } diff --git a/website/locale/en/LC_MESSAGES/messages.po b/website/locale/en/LC_MESSAGES/messages.po index 0caf8970259..ee9b7b2cc4f 100644 --- a/website/locale/en/LC_MESSAGES/messages.po +++ b/website/locale/en/LC_MESSAGES/messages.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2020-05-08 17:57+0300\n" +"POT-Creation-Date: 2020-05-19 11:11+0300\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language: en\n" @@ -37,27 +37,27 @@ msgstr "ClickHouse - fast open-source OLAP DBMS" msgid "ClickHouse DBMS" msgstr "ClickHouse DBMS" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "open-source" msgstr "open-source" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "relational" msgstr "relational" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "analytics" msgstr "analytics" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "analytical" msgstr "analytical" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "Big Data" msgstr "Big Data" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "web-analytics" msgstr "web-analytics" @@ -77,15 +77,51 @@ msgstr "" msgid "Yandex LLC" msgstr "Yandex LLC" +#: templates/docs/footer.html:3 +msgid "Rating" +msgstr "Rating" + +#: templates/docs/footer.html:3 +msgid "votes" +msgstr "votes" + #: templates/docs/footer.html:4 +msgid "Article Rating" +msgstr "Article Rating" + +#: templates/docs/footer.html:4 +msgid "Was this content helpful?" +msgstr "Was this content helpful?" + +#: templates/docs/footer.html:7 +msgid "Unusable" +msgstr "Unusable" + +#: templates/docs/footer.html:7 +msgid "Poor" +msgstr "Poor" + +#: templates/docs/footer.html:7 +msgid "Good" +msgstr "Good" + +#: templates/docs/footer.html:7 +msgid "Excellent" +msgstr "Excellent" + +#: templates/docs/footer.html:8 +msgid "documentation" +msgstr "documentation" + +#: templates/docs/footer.html:15 msgid "Built from" msgstr "Built from" -#: templates/docs/footer.html:4 +#: templates/docs/footer.html:15 msgid "published on" msgstr "published on" -#: templates/docs/footer.html:4 +#: templates/docs/footer.html:15 msgid "modified on" msgstr "modified on" diff --git a/website/locale/es/LC_MESSAGES/messages.mo b/website/locale/es/LC_MESSAGES/messages.mo index 8a44b54eaf5620e43d05a83c70d2ed75ae6a4679..b6bd03868f8b16c36c591d92f0487347fa8fbe0a 100644 GIT binary patch delta 1504 zcmY+@O=uKn7{Kvo&Dz9#Y&6FBmFhsm&k!8jgVrYLsxc@zEQv`Fz0B@TbaXN!yECPV z>Dq`NhuTm>u`Q*D2l1dDGM-FJO9Be!AgEA^q5(e+wo-a3=t2LF8I&%w^PBgbo#%Px zdEXs2|NW@+urB$X@Y&9{k#E;59lie?O^VECID{$u7U$u4tiUm>#Bp4NH;_Xfaax5> zuokOl&)A6id^0wQl%$P|B}`;vhj1z5@38@|VLeXa4>*ng;gJtS(pV@L;gD;bxFx@! zCYXrl@8KNA4^Rtxh8yrNTtR+W#%okFu@$wD4Ce44&c)lf7^krgU!oRRLsSj82HUU& zt8g%$KaA8UUn05WL_B{6wWDV-B)?qbLMv{hk4qB-LUPDXqUnP@s0Hl7W(=_tFW^Re zjvd&Dju){J^mpq~m$t2z{x9dEmkBK(L=%r$$`+6VD@$BxYc14P zdZ>W|)IyGDA#if8ZFF0KN9?}V9WoXjWYGG6Xp&t3>?=v^1fKY%EWIS z#lVhhb@IM35ZcD%b$;0%^*4+pQt#@0TgK?qg~DL z2B}s`(6UW;7et1+h^UPONnkBRs5H_=kB#*G)j-3{{rvy`JNJLiJ?Gx#o!6=pbNQYz zfgZTa3d#vV*v)&n61YOe1J{(6F*=R&a5@_Vvqw%R)Pu;My=O!sKEv-!+y-hF(j}( z#17)yGX}XVY~i+wumYs$1WXV%ReBwO=Vc+^#B< zi@Ngy4%#=29paS17kHS;7Y&>__MDC%uF zj{SHM+1tKiJBhC%?YDzOQ)`dmVZ4j2IFD_(BSD>2vh&!CH&H!$n{pP}&*o9RUO;X9 z8P%EN-%&*IE30~8#ZDUD$W2}Q)g#)yt^-U@@%QEqp!EOD;6Je$2|?{?pN=q zY&Ykq{cHB_TUXR-o@T9rdgR743 W?-f&S%zaXL+ex+;4JT(pd4B*%=zwSd diff --git a/website/locale/es/LC_MESSAGES/messages.po b/website/locale/es/LC_MESSAGES/messages.po index 1a1aa85fb44..10fb0a88aa1 100644 --- a/website/locale/es/LC_MESSAGES/messages.po +++ b/website/locale/es/LC_MESSAGES/messages.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2020-05-08 17:57+0300\n" +"POT-Creation-Date: 2020-05-19 11:11+0300\n" "PO-Revision-Date: 2020-03-26 10:19+0300\n" "Last-Translator: FULL NAME \n" "Language: es\n" @@ -36,27 +36,27 @@ msgstr "ClickHouse - DBMS OLAP de código abierto rápido" msgid "ClickHouse DBMS" msgstr "Sistema abierto." -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "open-source" msgstr "de código abierto" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "relational" msgstr "relacional" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "analytics" msgstr "analítica" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "analytical" msgstr "analítico" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "Big Data" msgstr "Grandes Datos" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "web-analytics" msgstr "Sistema abierto." @@ -76,15 +76,51 @@ msgstr "" msgid "Yandex LLC" msgstr "Sistema abierto." +#: templates/docs/footer.html:3 +msgid "Rating" +msgstr "Clasificación" + +#: templates/docs/footer.html:3 +msgid "votes" +msgstr "voto" + #: templates/docs/footer.html:4 +msgid "Article Rating" +msgstr "Clasificación del artículo" + +#: templates/docs/footer.html:4 +msgid "Was this content helpful?" +msgstr "¿Este contenido fue útil?" + +#: templates/docs/footer.html:7 +msgid "Unusable" +msgstr "Inutilizable" + +#: templates/docs/footer.html:7 +msgid "Poor" +msgstr "Pobre" + +#: templates/docs/footer.html:7 +msgid "Good" +msgstr "Bien" + +#: templates/docs/footer.html:7 +msgid "Excellent" +msgstr "Excelente" + +#: templates/docs/footer.html:8 +msgid "documentation" +msgstr "documentación" + +#: templates/docs/footer.html:15 msgid "Built from" msgstr "Construido a partir de" -#: templates/docs/footer.html:4 +#: templates/docs/footer.html:15 msgid "published on" msgstr "publicado en" -#: templates/docs/footer.html:4 +#: templates/docs/footer.html:15 msgid "modified on" msgstr "modificado en" diff --git a/website/locale/fa/LC_MESSAGES/messages.mo b/website/locale/fa/LC_MESSAGES/messages.mo index 1b3bd3ff610465730796f4be0adc4236c39e153a..ef35154a13b39f998b0ca95f3fe4898c7375ad17 100644 GIT binary patch delta 1532 zcmYM!YiJZ#6u|MbNz|CfMzf}7lV}DNAMw=<1zV{~5RFPdh#CVvKE%zcOEx=_ofRuh zmq^S8x<%{yMnMF-cBwAO1jH9;3yKQni{K0i8hrFaOVRd&r4_~h!G$i9`Q3YG?m6e) zxg@>h?9#-O%A#Y!Z#92){C!oRz5jpT6p8rgcB2=6z{z+Hi*XQ3@H$rG9psXy+~(j5 ztiZByAFV^3zW`^7BxE^_8U}Xc?Z#UAC-D;;!YTLwPvIE;gZsyeG-1~S5iS|x#v{3b zn&6lG_#>Q1e-yQ_->?z?z**#%=`5p+ft9F*tj8dB;3OPIKaOD~{)bv%IZ-v>JY0^8 zuoOG<<6k2+%C|@^IhY^+9(6~LV4VE&BMq&%j(x245D3X3>xiZkH=-7>6&GL}oAEq; zj=$q7oXINmQ-pCDTKWDUl0t4`A>PGue1Hl4+%65RbQFmt&-1=Q-O{(nB_3vJ0x!-% zKkD-)|2&g12DGxj zP)~UYCr(%DM}2V?uEvG<8Cs|X{DhkL9uDAZoQCJAdIsJ??LS9dsE1|iLg%A}jR_jM zwHeezL#PGa%=aIn7V;SN&^^P$_#UU?A#S%>Sq3#xn1iBmlBfk7#4tlG`RbJKhNXku$bJ^|JH*BA;zrgSIxCcg3&VV>MrvB_EdDQ3R zMpE|asxt*^ooo6~BQf4TV ze~=KONC^7Fjh>3i*vo{1B$^50qvQ&Tz=9t7{&LV|_kPYfyZ4;m`TfqFrgTGc&260( zu4e84x7DJ(|5v_9q=N2O+=Yf!#EuoX0|QuzVdN(#c!aSPT{u{99QFGt4BVio;j49NeKS%vOTj;N2 z9sQrU534y4L2Shu>~oQSt;G}rI>7?E3FHOd$M@8woAEK~K{n+BYN=LH4_zLaTZ;L( z4Lx`iPopk4gSyZYe2Sm23MV;O2k?Q1{OdrL0o~aK>dxG3BY}P-DKd>3z#M7{@1dTJ z$2fyaxEs4k(;EU$puT^MC{N;Xyn&P0hd~b72u>tv{Hqy`Ghihhy5JSm9bQMR@dE0E z4^cDl42e}Ts3rW2dNwvtOZ5XaPzO\n" "Language: fa\n" @@ -36,27 +36,27 @@ msgstr "ClickHouse - سریع باز-منبع OLAP DBMS" msgid "ClickHouse DBMS" msgstr "خانه عروسکی" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "open-source" msgstr "متن باز" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "relational" msgstr "رابطه" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "analytics" msgstr "تجزیه و تحلیل" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "analytical" msgstr "تحلیلی" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "Big Data" msgstr "داده های بزرگ" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "web-analytics" msgstr "تجزیه و تحلیل وب سایت" @@ -76,15 +76,51 @@ msgstr "" msgid "Yandex LLC" msgstr "Yandex, LLC" +#: templates/docs/footer.html:3 +msgid "Rating" +msgstr "درجهبندی" + +#: templates/docs/footer.html:3 +msgid "votes" +msgstr "رای" + #: templates/docs/footer.html:4 +msgid "Article Rating" +msgstr "رتبه مقاله" + +#: templates/docs/footer.html:4 +msgid "Was this content helpful?" +msgstr "این مطالب مفید بود?" + +#: templates/docs/footer.html:7 +msgid "Unusable" +msgstr "غیرقابل استفاده" + +#: templates/docs/footer.html:7 +msgid "Poor" +msgstr "فقیر" + +#: templates/docs/footer.html:7 +msgid "Good" +msgstr "خوبه" + +#: templates/docs/footer.html:7 +msgid "Excellent" +msgstr "عالیه" + +#: templates/docs/footer.html:8 +msgid "documentation" +msgstr "مستندات" + +#: templates/docs/footer.html:15 msgid "Built from" msgstr "ساخته شده از" -#: templates/docs/footer.html:4 +#: templates/docs/footer.html:15 msgid "published on" msgstr "منتشر شده در" -#: templates/docs/footer.html:4 +#: templates/docs/footer.html:15 msgid "modified on" msgstr "تغییریافته" diff --git a/website/locale/fr/LC_MESSAGES/messages.mo b/website/locale/fr/LC_MESSAGES/messages.mo index 1b56afa453381e7931d5d2f40a4e7172484f8c90..a5fe91d114e7db67f96c45963df30ee3067dd686 100644 GIT binary patch delta 1487 zcmY+?TWkzr6u|K_T3WhYY!#(sT%v9zD;^NHq84db_lS6yZih}xcVu^_?lvAgxJ3}R zH<2Lq;Gye*;6nDCj; zw~}vNHy!=`>?jl|W4IHeco_TQX)MN8?1fixAhsiyJmXe{udz2qx_4QL`u=zvA(E4s zJPcx@K6EDzW_%L+<0b5eckvWH$8Wf;he!=(ibS~N5;uO5Yp4lsg!2!uC*uy(!rtH{ ze2*i^FGE>Igo(MRg(NYB8?g`G!wP(kG3-Pwu#~78a174G30Q(n;rv#lM%jhrl0D)4 zLDY^O!GQd7jt8x{l0FVL2!!O2MMTpVm!lT27RO_N2|R;S@Ez9T2v(t6;ouDP!*MH; zLat*0-ojG6i#gryM?7ey9Y`#B75WLarQeZD4BpZNQLMrW)Zc56OA_1;uz-~~gz-xb zOgZWS(9ZTpU057-Yp3<5|6_Pq%7g~mjv9D3a!E_*71R#h#^LxR^apC-VeFG0-s!00 zHK+-Eyovj9Hiab#GLP|L_E8t|Fh>8iq8Ci)>3xUV@&YzUTULU)$K^N~2jM7Of?U$f zO%v|JES^Ozw1N|>1&u?Ey8t=uVxexq9$bfeb3ACv-=c28mvG#P+L0dgPYa5m2I_~6 zSdHY8Rnq_*_kXcWwn(x{YEEC=(NwvW~1$HS|7Nx@+W$&HcA`3RG=A2NdMB9bkRPc`H5Y4LdeUSzgWURU*}qmoz54;9viWw6*OmT zY1`Xuxxw$?`Hr#=#)O6Gz;|5d{~G7oVReCNnQbk8TZ^gxvS;P*m!CHx|CJaF@XNGB P*=*NWTl8!zjmpX|oxJ%bLQH#rC=?RLi@84EifizqdyN(U{C#-Zm>af<-HXpv7&kj zD?y}!K#-ufg6?BPQTY%(gqV;<68KcEz=yD)hrT~{!Etv#=bU@)@0|1d-MRSk_Tqe< zIVSwN_&4%5E3~)rD|tjhbbsO+G)xgchHy1DVgt4ymn68gU=Ic{?|K#W{W}=Lag2%- zoAUWcm#Drr;$C$5bnckScR{USLK8Id>S=VpYZ~gP&d?9NgPAO zMx8%G+FJ1eCRtx5X>?+Ms6BW9HMPU2Wx4P6XOI~34XK9wK&{p>E@{&nCO1G+#S-{DP6u%Uz0Wk3DfMAMyrM6xP#s2|+|YD)j% zZZxUeX552XiZiJ3ZM5(nGF3c0)D46~g;GTaNfV8vw3I7-u4dXbs%9lk>GES6AjH8jyGId ad84`hi7`6UFj#Xb)7O?_k;`_j-;)2~--34l diff --git a/website/locale/fr/LC_MESSAGES/messages.po b/website/locale/fr/LC_MESSAGES/messages.po index aa332939179..dcbe4b5ee8b 100644 --- a/website/locale/fr/LC_MESSAGES/messages.po +++ b/website/locale/fr/LC_MESSAGES/messages.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2020-05-08 17:57+0300\n" +"POT-Creation-Date: 2020-05-19 11:11+0300\n" "PO-Revision-Date: 2020-03-30 15:12+0300\n" "Last-Translator: FULL NAME \n" "Language: fr\n" @@ -36,27 +36,27 @@ msgstr "ClickHouse-SGBD OLAP open-source rapide" msgid "ClickHouse DBMS" msgstr "SGBD ClickHouse" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "open-source" msgstr "open-source" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "relational" msgstr "relationnel" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "analytics" msgstr "Analytics" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "analytical" msgstr "analytique" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "Big Data" msgstr "Big Data" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "web-analytics" msgstr "web-analytics" @@ -76,15 +76,51 @@ msgstr "" msgid "Yandex LLC" msgstr "Yandex LLC" +#: templates/docs/footer.html:3 +msgid "Rating" +msgstr "Évaluation" + +#: templates/docs/footer.html:3 +msgid "votes" +msgstr "vote" + #: templates/docs/footer.html:4 +msgid "Article Rating" +msgstr "L'Article De Notation" + +#: templates/docs/footer.html:4 +msgid "Was this content helpful?" +msgstr "Ce contenu a été utile?" + +#: templates/docs/footer.html:7 +msgid "Unusable" +msgstr "Inutilisable" + +#: templates/docs/footer.html:7 +msgid "Poor" +msgstr "Pauvre" + +#: templates/docs/footer.html:7 +msgid "Good" +msgstr "Bien" + +#: templates/docs/footer.html:7 +msgid "Excellent" +msgstr "Excellent" + +#: templates/docs/footer.html:8 +msgid "documentation" +msgstr "documentation" + +#: templates/docs/footer.html:15 msgid "Built from" msgstr "Construit à partir de" -#: templates/docs/footer.html:4 +#: templates/docs/footer.html:15 msgid "published on" msgstr "publié le" -#: templates/docs/footer.html:4 +#: templates/docs/footer.html:15 msgid "modified on" msgstr "modifié sur" diff --git a/website/locale/ja/LC_MESSAGES/messages.mo b/website/locale/ja/LC_MESSAGES/messages.mo index 879e68bd8ebc18b93fd68c6b6aaede784e3dc979..191276fb16d5ecc45840f0f97058f252a7eeccd1 100644 GIT binary patch delta 1497 zcmYMzZERCj7{KwTTY<^eZH&o87E*bKEZSm1WC5fhb7C;^vIzvfbY0yVyI!;28d0I# zZVUt`xG`RWk$2dL8Och*P+k%ffyf79gb&6=Nlsgc#$fb=i7(OrlikGT?*7g>_uS`s z&N+9#1@1qe{-Me{ApBn9ua>_xMcVuS_r6zTD&6f^jvwO`Jb@)Rf|KzJdY&K60_3K~x{(BirsXV5=_)9?~j;~hMT5AY9sccMrLJ12>7$t7+) zlB=i*zI4a$;S==7Pzx*IV*DBBkY8r9j4}pRp%$_hBlsp(;$57M53maVL@m%qR1LTg zSKzZ)iXHCwHl#+`f#i~1?szZij`m@S{Bn+lR$R+I&hQWj$sw;0O<&xATEG^p!xT2+ zDSQEc!UmkpD)cB~xEyV_KZ2x?Ygml8(1&+0t;hWx4Xtzxi6zGMG3u87gVL(Gh9hwA+~uv{R~y;g6~(c|GG6}3_OVi z_rc$&a=5#c+|f>Y{wF9Bi`l3@m*atfwkaRQYJ+JZ#E6&ny&MvD=Wt7Cv?7NSAO`4aV{1<`GC2`IbodBL^sYpvon+3Fs;us kv2bX2;n-+@u4YF~Vei1hvxB5QXEH=Nl;6`+`1FAM2UF$;rvLx| delta 1143 zcmYMyUr1AN6u|Lw=kiZFXI`gSDw>fYV$(1+nrxIx&_ji2;6vmeN@!ZSAPAc|8a-5u z%&Z6ni3)n_V)RfHg%8n-U{WDSaWN>Zs2(b!=zH=)W4oW<@7&$r@0|0yvzx#Aqx0GB z5#bZySHREh(%JvdQj&a{KQ=Jtq{q<3cQLT{El-N!kJ`|6!dW+B!#FOl%mFKxKv{R*W(e)z!79) znZQo=mxm0p2$WD(K2~Ek?m|syKa!K2#0DJ1bbN$7D$kw!GpL<Iss*Tu$+S+rd$1>{7XOT7J4RRXt0rj*#ArHZ7Ip!vp)#5W< qUOSesV-M`1`*!S>9kZ<4c^8f0Q2s!cJ#^C^B6NA-Z2a9=LjD37iH%nP diff --git a/website/locale/ja/LC_MESSAGES/messages.po b/website/locale/ja/LC_MESSAGES/messages.po index ba40c41edf1..54dc1d401b6 100644 --- a/website/locale/ja/LC_MESSAGES/messages.po +++ b/website/locale/ja/LC_MESSAGES/messages.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2020-05-08 17:57+0300\n" +"POT-Creation-Date: 2020-05-19 11:11+0300\n" "PO-Revision-Date: 2020-03-26 10:19+0300\n" "Last-Translator: FULL NAME \n" "Language: ja\n" @@ -33,27 +33,27 @@ msgstr "ﾂ環板篠ｮﾂ嘉ｯﾂ偲青エﾂδﾂ-ﾂエﾂスﾂ-ﾂシﾂ" msgid "ClickHouse DBMS" msgstr "クリックハウスDBMS" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "open-source" msgstr "オープンソース" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "relational" msgstr "関係" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "analytics" msgstr "analytics" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "analytical" msgstr "分析" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "Big Data" msgstr "ビッグデータ" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "web-analytics" msgstr "ウェブ分析" @@ -71,15 +71,51 @@ msgstr "ソフトウェアは、明示または黙示を問わず、いかなる msgid "Yandex LLC" msgstr "Yandex LLC" +#: templates/docs/footer.html:3 +msgid "Rating" +msgstr "評価" + +#: templates/docs/footer.html:3 +msgid "votes" +msgstr "投票" + #: templates/docs/footer.html:4 +msgid "Article Rating" +msgstr "記事の評価" + +#: templates/docs/footer.html:4 +msgid "Was this content helpful?" +msgstr "この内容は有用だったか。" + +#: templates/docs/footer.html:7 +msgid "Unusable" +msgstr "使用不可" + +#: templates/docs/footer.html:7 +msgid "Poor" +msgstr "貧しい" + +#: templates/docs/footer.html:7 +msgid "Good" +msgstr "よし" + +#: templates/docs/footer.html:7 +msgid "Excellent" +msgstr "優れた" + +#: templates/docs/footer.html:8 +msgid "documentation" +msgstr "文書" + +#: templates/docs/footer.html:15 msgid "Built from" msgstr "から構築" -#: templates/docs/footer.html:4 +#: templates/docs/footer.html:15 msgid "published on" msgstr "掲載され" -#: templates/docs/footer.html:4 +#: templates/docs/footer.html:15 msgid "modified on" msgstr "に変更" diff --git a/website/locale/messages.pot b/website/locale/messages.pot index b209f40175b..2e1259fc0c6 100644 --- a/website/locale/messages.pot +++ b/website/locale/messages.pot @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2020-05-08 17:57+0300\n" +"POT-Creation-Date: 2020-05-19 11:11+0300\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -32,27 +32,27 @@ msgstr "" msgid "ClickHouse DBMS" msgstr "" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "open-source" msgstr "" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "relational" msgstr "" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "analytics" msgstr "" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "analytical" msgstr "" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "Big Data" msgstr "" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "web-analytics" msgstr "" @@ -70,15 +70,51 @@ msgstr "" msgid "Yandex LLC" msgstr "" +#: templates/docs/footer.html:3 +msgid "Rating" +msgstr "" + +#: templates/docs/footer.html:3 +msgid "votes" +msgstr "" + #: templates/docs/footer.html:4 +msgid "Article Rating" +msgstr "" + +#: templates/docs/footer.html:4 +msgid "Was this content helpful?" +msgstr "" + +#: templates/docs/footer.html:7 +msgid "Unusable" +msgstr "" + +#: templates/docs/footer.html:7 +msgid "Poor" +msgstr "" + +#: templates/docs/footer.html:7 +msgid "Good" +msgstr "" + +#: templates/docs/footer.html:7 +msgid "Excellent" +msgstr "" + +#: templates/docs/footer.html:8 +msgid "documentation" +msgstr "" + +#: templates/docs/footer.html:15 msgid "Built from" msgstr "" -#: templates/docs/footer.html:4 +#: templates/docs/footer.html:15 msgid "published on" msgstr "" -#: templates/docs/footer.html:4 +#: templates/docs/footer.html:15 msgid "modified on" msgstr "" diff --git a/website/locale/ru/LC_MESSAGES/messages.mo b/website/locale/ru/LC_MESSAGES/messages.mo index 6d0187452dddfec6d8305eaa480f493d9eb6f01d..d6d6af10b607b6396950b7b7aa1bbbf89433fe30 100644 GIT binary patch delta 1569 zcmY+@TWl0n7{Kvw+foX>DN^l?Wdu|1BrBK5txZd?KzwL~S`s6=Y)4qx?wGwaVp^9= zP!mELS`BKHTd0W-$hyK!WkbL|^=W*)fF8%>2$dGxL4l zIdk^Qmb-14sW~Md3cu$$8#!Mn(%!>gUx~;Qbo;RiPvGNt2FoyqvvC}2@d|RtG?zvA z8&+e*jDn4*?=Q!NA{l9>QP030-+pYM{}DckBRCf)@ngJ&f8*PwBCVL1DZ(KmT(~8d zP!oLbk6*)C^lzdT_B%d{_wX6=OOR<)Ft8c5kPeLCetZnCVjbSXIrtB1ft5tnfJ?9$ zS7JH#_~QqV8s!}%mmKxSkE5>WAg0MLpVQEa8(GJO0D+JkvYlx9;ttdTUdH8^#&$f1 z&*5F%f(x02eu^k=!j#|7At_`6i}44n#7WHP=e|KhE4_)tlH0xyP?z*yB9R2ZAE-|wGCU4ieaS^xDkt}&p2t6A?ZtVewy;rlK= zMgJ3Q!5r$$|H65=jOEir+fet6jq|Y&Pg2mQIE$ICCg`h-H`f$C?Wr2p|7iv;GOz(X z)Xi2&6k^K))Lp*{-@rE10xzKM_8C;gE|wtvqW4k%MRnY4!?m~!5BU9I)Rn%2X?&2O zp^0ohur6^gYQiJFgQ%PCEH1$j`~#;@S7?$Q+r-SKk(#8G<mExYH7e1N^E|AKO>khlqTkE}y#m5qMC6Sc5is1w+ai?I)Nvkjv@e-X8? zA5s5R)2Ib*X4#p;V(HNJZ|&}mrlL+f9J3-$Pmdi>C8afyibi6#^

    `jqlBsY$~qW z@>ax-#kec&j??XKoH-CE?{J)i6rOQ^F1r#~+8IwL!+T=3yksYm#FVaZ(n{@%CJ7x+ z@v^ngj=lMMI@aQzoZS_u>~EE3b|J+&IM}E zm{IeMci4=X3FBJkEX^$EsM}NZMQQL9&2cm4y~o|GHy~!%IY zKB2Yf)iRgqj??*??lE(z#r>&as%Yrfx|$+yU}!k_XOo84QVyA{m>f@jwZddIoyH~3 I7!kAbAMndUZ2$lO delta 1127 zcmYMyUr19?9KiA4{-mzCT$;0$eCao=pI&I(-OoAa-h0mP{LXK+^GQb{TWp^a ze(n6r`P;49`~S*Eak?KRG=KtqI$44~}3VP9Y0R z1_xMQW@)$?sAgMUY{FvfK@BK`#3WbnFecH7FOg5>gZX?3HB-wN#~f-vrya~=5s9I` zKSA2+@jiC5zRb}Gpo>|%@EB@pZ=oK`wAufJ%pohtX~-(-X{{m2kagseKc;rJtC?}5 z?%cyo=dD3M?#G0SMlX#oL>R^c^xL+H)Zz$g2BuMWW?(D6L!HRV0r#N`_4%ObWh|pV zh8;MAy20;Qft93N=WTP7e?9Ad2C6WM(*!b(4z}j!972pYQV0ym9jwL-9>TZCMG=Uc zRZs&;q88;cYO$^3XMN6MTI4zGL`NC<-$$c|aCP7kYAT;&9ABd@9AGv!Eyq!7qaXL; zIn3b{Y6fmlK(z#Z4@suXvdxqD9@ALI>bjwM+~_CB2W!YYPT~q`b;c_<7n(rklZU2{ z%<;#lfxW;w{D@i`zmQlq(y5vCR%w-sG@usuBuX$G9*PY`hfkgN6%0fpk-_2EsFAj} z+l(Liw=Kp^$DAd0>`z5f6M5Qw<| diff --git a/website/locale/ru/LC_MESSAGES/messages.po b/website/locale/ru/LC_MESSAGES/messages.po index 5bb1fa8cef2..df9e1492c27 100644 --- a/website/locale/ru/LC_MESSAGES/messages.po +++ b/website/locale/ru/LC_MESSAGES/messages.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2020-05-08 17:57+0300\n" +"POT-Creation-Date: 2020-05-19 11:11+0300\n" "PO-Revision-Date: 2020-03-26 10:19+0300\n" "Last-Translator: FULL NAME \n" "Language: ru\n" @@ -38,27 +38,27 @@ msgstr "ClickHouse-быстрая СУБД OLAP с открытым исходн msgid "ClickHouse DBMS" msgstr "СУБД ClickHouse" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "open-source" msgstr "открытый исходный код" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "relational" msgstr "реляционный" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "analytics" msgstr "аналитика" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "analytical" msgstr "аналитический" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "Big Data" msgstr "большие данные" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "web-analytics" msgstr "веб-аналитика" @@ -78,15 +78,51 @@ msgstr "" msgid "Yandex LLC" msgstr "ООО «Яндекс»" +#: templates/docs/footer.html:3 +msgid "Rating" +msgstr "Рейтинг" + +#: templates/docs/footer.html:3 +msgid "votes" +msgstr "голоса" + #: templates/docs/footer.html:4 +msgid "Article Rating" +msgstr "Рейтинг Статей" + +#: templates/docs/footer.html:4 +msgid "Was this content helpful?" +msgstr "Был ли этот контент полезным?" + +#: templates/docs/footer.html:7 +msgid "Unusable" +msgstr "Непригодная" + +#: templates/docs/footer.html:7 +msgid "Poor" +msgstr "Плохая" + +#: templates/docs/footer.html:7 +msgid "Good" +msgstr "Хорошая" + +#: templates/docs/footer.html:7 +msgid "Excellent" +msgstr "Отличная" + +#: templates/docs/footer.html:8 +msgid "documentation" +msgstr "документация" + +#: templates/docs/footer.html:15 msgid "Built from" msgstr "Собрано из" -#: templates/docs/footer.html:4 +#: templates/docs/footer.html:15 msgid "published on" msgstr "опубликовано" -#: templates/docs/footer.html:4 +#: templates/docs/footer.html:15 msgid "modified on" msgstr "изменено" diff --git a/website/locale/tr/LC_MESSAGES/messages.mo b/website/locale/tr/LC_MESSAGES/messages.mo index 938a800f4305d14cabcdb6fa8593fb8ea76dc593..a8bb89012865a5b30bf3beed3b763a4637e8180a 100644 GIT binary patch delta 1483 zcmYMzU2GIp6u|K_bfty13$(U~RG3)hD@C(4AylDc<)aAJfPfA9&`ftnI_d0AGc#LR zV%?YsF(!?{hA#~P3?IgXz;^oP`eIi|;DJQ?f@zdQqXZ!)KG;MP;{Sw6+|17J-n%>Z zoO|w_u@~>BN$9Y(ZIlF)I@e+28VDV-ohn#7n|@O)C3!uRVQ4H z-S`65V=f*qB00(jNGv%VkB^~N^fN3FUoO(njN7Q=QiF+*7_x)e^uax-3A}?HSim0q z64&E>+=}h2Lbt-fjTpxL2_%Gki#2#18}TNVbi41+&`f8Lx#XAFKT%8iFLH^&QyL(N zD{%?x?`h8F!m^F zh0dXg5$bc(s9QFR%dv_5(lgVE8u&H5iv8Lrv4u>MroV!H)XEMvQGZ>*Faw-M89`m) z=cpySfV#(5P)j+mlmrmSWopT{l?uphbPHaAV^chp2`Il-FfGpH3`kGt`e5)Ccg z80wyXftt{H)DnG-not>aq6!Y<4{?7pgBw`ce$?0VILp_ST|j-EFXMK+f%~zUedxj* zZo$$e8XDjMZo^sBM7nvIHGv+~jNd>_Y&U9vKGeh1kGj_x)P;DcdpsJCe}X#yaqPr% zsPjxC6x7P%w_G#h<#KjD3?%J`PR6y(H?7dgADBpNtVwP?n6X`#U!}+M zveAaxPmTIrp6AQs|3o+Ht{H3gwx@74;Cotf!bdXwaOxg;mJzG}9$7=aAk~mr)M|Z0k|B%8B|j}~99J__ zj(T!GH(fV`VcdZQH;p)rYed+Ey1*QRYjFt~5>MIk1_7+0AF}$b7@>a(wbqv{Z=jz1 z9tQC#>b$q8ssE1ESn47FTI=o1_Tdp6!?W5aASZQ+(XVDTOwqYF)JU z4%AY)Py?(&E!9?Qd^hU4hcJpKP&1N8iFI|Qa;Z#r;$qOznMtRU-MOqeYHKbr=j^u( zbI>_u*yG8*EU%ck(srZx(RHEN=I(XuY6{jj?r&%`U%Iy$=AW|1CFXPQSHpZ#Id9m{ hOpf\n" "Language: tr\n" @@ -36,27 +36,27 @@ msgstr "ClickHouse - hızlı açık kaynak OLAP DBMS" msgid "ClickHouse DBMS" msgstr "ClickHouse DBMS" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "open-source" msgstr "açık kaynak" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "relational" msgstr "ilişkisel" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "analytics" msgstr "analiz" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "analytical" msgstr "analitik" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "Big Data" msgstr "Büyük Veri" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "web-analytics" msgstr "web-analyt -ics" @@ -76,15 +76,51 @@ msgstr "" msgid "Yandex LLC" msgstr "Yandex LLC" +#: templates/docs/footer.html:3 +msgid "Rating" +msgstr "Verim" + +#: templates/docs/footer.html:3 +msgid "votes" +msgstr "oylar" + #: templates/docs/footer.html:4 +msgid "Article Rating" +msgstr "Makale Değerlendirme" + +#: templates/docs/footer.html:4 +msgid "Was this content helpful?" +msgstr "Bu içerik yararlı mıydı?" + +#: templates/docs/footer.html:7 +msgid "Unusable" +msgstr "Kullanışsız" + +#: templates/docs/footer.html:7 +msgid "Poor" +msgstr "Zavallı" + +#: templates/docs/footer.html:7 +msgid "Good" +msgstr "İyi" + +#: templates/docs/footer.html:7 +msgid "Excellent" +msgstr "Mükemmel" + +#: templates/docs/footer.html:8 +msgid "documentation" +msgstr "belge" + +#: templates/docs/footer.html:15 msgid "Built from" msgstr "Dahili" -#: templates/docs/footer.html:4 +#: templates/docs/footer.html:15 msgid "published on" msgstr "yayınlanan" -#: templates/docs/footer.html:4 +#: templates/docs/footer.html:15 msgid "modified on" msgstr "modifiye on" diff --git a/website/locale/zh/LC_MESSAGES/messages.mo b/website/locale/zh/LC_MESSAGES/messages.mo index 9c36aa66bc42e54c3ca4e623256ce2fcd19aceb3..0b2efedd38bf7e407fa5e2ef3c3a668e41d64471 100644 GIT binary patch delta 1475 zcmYMze@K*f9Ki9gC1a2;v^AK@ZQVl5uQ3Vet) zIG4B3qln;2Oqli{l0rr>3xCA|9L1C#_Z=Rz(s3k~+%ry~Zs{}R5{p%upb+O{3F>$? za!D<>HeO%@>iltjFzN%;-@qf(LUL$mzNu5#e|@uJ(-A-oT!GsE3BHayjE9WpaUT7* z@J)PzGqH^G()mkp4%Xv#wDDhjfXnbU+pUEa7P0@Vl~N8?&$gVKR#u5Rp~ke=Vma*w z)U$6zUH1cO!Dq1n@1Pd)4hO8`Yf$4fp!RP;jlUC@;@2r2bW1Lyp3OBg;Wy)$@iuCp zKk);6X!JHYS79PSn?8FuL2WkPSV*X=#rXQkqG@u6PH1?nt zcG$F!nDzK+I2vVCYHd5@yf=BDm0xe$E%IWH^Ly@f>#Ys3 zWV~^6G%Op#EpcMXrpCBG(G-ajI+kFuzbPF3d`mJ~<@DrjvI;_WFsT_yW5>Gw%a3Gl zwG-j4^}cv+V7WgKcsCGm#(WDcpLcLi=I8$Vr*~&A|L8m}sLz`29^9S2^sU>~F){p= zxL4ZUqa)7hX;CZBy)Yn5blcmUvqeqT+g?w<=X9q>u4cYFE}4#YukWPXzuKF=_>JD7;`a0Z;1Mc1vRTIPCJ8#UWvU13l9y;Ik@yyU{uj5PiN>AqKaj*M~WJdO9hWd!? I9qyF>0Vibo`Tzg` delta 1132 zcmYMyT}V@57{Kx8tSB?ho96mP5pe9xT8L+=R8rC2iblu^mgW&v*@W{%s843~m-l z%Imd;EMYkVf3OU_>qRzVJ>J6*{>CpD!c?wE9(uVE56$Qtq$sfK(S(ApXu*HU8u#xBj~4p)p*}H zi&}zDSc8AjioAO3vH{fR z?xG%W2D|VL>c-Vf*72a5{OiJD0y=OEb-{DE2ji$2cx>X&jL%W$&Ep|_XX53204wxs zP}lMC1?$18@E~^JUYtZ-Kb54RH7Q_s4R}!}b{LPM*7}6;B+SKPUyxoT&yr7397yS7>OO7YPpXzrC&^c8yQ}HFe?85;2wf= diff --git a/website/locale/zh/LC_MESSAGES/messages.po b/website/locale/zh/LC_MESSAGES/messages.po index f509557fed1..31195654358 100644 --- a/website/locale/zh/LC_MESSAGES/messages.po +++ b/website/locale/zh/LC_MESSAGES/messages.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2020-05-08 17:57+0300\n" +"POT-Creation-Date: 2020-05-19 11:11+0300\n" "PO-Revision-Date: 2020-03-26 10:19+0300\n" "Last-Translator: FULL NAME \n" "Language: zh\n" @@ -33,27 +33,27 @@ msgstr "ﾂ暗ｪﾂ氾环催ﾂ団ﾂ法ﾂ人" msgid "ClickHouse DBMS" msgstr "ﾂ环板msｮﾂ嘉ｯﾂ偲" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "open-source" msgstr "开源" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "relational" msgstr "关系" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "analytics" msgstr "分析" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "analytical" msgstr "分析" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "Big Data" msgstr "大数据" -#: templates/common_meta.html:21 +#: templates/common_meta.html:24 msgid "web-analytics" msgstr "网络分析" @@ -71,15 +71,51 @@ msgstr "软件按\"原样\"分发，不附带任何明示或暗示的担保或 msgid "Yandex LLC" msgstr "Yandex LLC" +#: templates/docs/footer.html:3 +msgid "Rating" +msgstr "评分" + +#: templates/docs/footer.html:3 +msgid "votes" +msgstr "所得票数" + #: templates/docs/footer.html:4 +msgid "Article Rating" +msgstr "文章评级" + +#: templates/docs/footer.html:4 +msgid "Was this content helpful?" +msgstr "这个内容有帮助?？" + +#: templates/docs/footer.html:7 +msgid "Unusable" +msgstr "无法使用" + +#: templates/docs/footer.html:7 +msgid "Poor" +msgstr "差" + +#: templates/docs/footer.html:7 +msgid "Good" +msgstr "很好" + +#: templates/docs/footer.html:7 +msgid "Excellent" +msgstr "善乎哉！" + +#: templates/docs/footer.html:8 +msgid "documentation" +msgstr "文件" + +#: templates/docs/footer.html:15 msgid "Built from" msgstr "建于" -#: templates/docs/footer.html:4 +#: templates/docs/footer.html:15 msgid "published on" msgstr "发表于" -#: templates/docs/footer.html:4 +#: templates/docs/footer.html:15 msgid "modified on" msgstr "修改于" diff --git a/website/main.html b/website/main.html index 8fcc52034f0..07647c24a2d 100644 --- a/website/main.html +++ b/website/main.html @@ -24,6 +24,7 @@ {% else %} {% set title = config.site_name %} {% endif %} +{% set title = title.strip() %} {% if page and page.content and not single_page %} {% set description = page.content|striptags %} {% set description = description.replace('¶','')[0:120] %} diff --git a/website/templates/docs/footer.html b/website/templates/docs/footer.html index 8b23e602069..efe53b3c24c 100644 --- a/website/templates/docs/footer.html +++ b/website/templates/docs/footer.html @@ -1,4 +1,15 @@ {% if not single_page %} +

    +

    {{ _('Rating') }}: RATING_VALUE - RATING_COUNT {{ _('votes') }}

    +

    {% if is_amp %}{{ _('Article Rating') }}{% else %}{{ _('Was this content helpful?') }}{% endif %}

    +
    RATING_STARS
    +
    +
    {{ config.copyright }}
    {{ _('Built from') }} {{ config.extra.rev_short }}{% if page and page.meta %}{% if page.meta.published_date %}, {{ _('published on') }} {{ page.meta.published_date.date().isoformat() }}{% endif %}{% if page.meta.modified_date %}, {{ _('modified on') }} {{ page.meta.modified_date.date().isoformat() }}{% endif %} diff --git a/website/templates/docs/ld_json.html b/website/templates/docs/ld_json.html index 07940c66b5e..051bac0c8a7 100644 --- a/website/templates/docs/ld_json.html +++ b/website/templates/docs/ld_json.html @@ -1,5 +1,5 @@ {% if page and page.meta %} - +}, { +"@context": "http://schema.org", +"@type": "Episode", +"name": "{{ title }}", +"ratingValue": RATING_VALUE, +"ratingCount": RATING_COUNT +}] {% endif %} From 4906dbef410a8e96d4a2c2f007528cbb79a7230d Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 19 May 2020 13:46:56 +0300 Subject: [PATCH 432/738] Remove redundant logs --- src/Interpreters/MutationsInterpreter.cpp | 2 - .../MergeTree/MergeTreeDataMergerMutator.cpp | 68 ------------------- 2 files changed, 70 deletions(-) diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index a1cabdc4199..5ba6424653c 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -1,6 +1,5 @@ #include "MutationsInterpreter.h" -#include #include #include #include @@ -636,7 +635,6 @@ ASTPtr MutationsInterpreter::prepareInterpreterSelectQuery(std::vector & } select->setExpression(ASTSelectQuery::Expression::WHERE, std::move(where_expression)); } - LOG_DEBUG(&Poco::Logger::get("DEBUG"), "QUERY:" << queryToString(select)); return select; } diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index aa09d6ddf9e..0c7d749d786 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -1324,74 +1324,6 @@ void MergeTreeDataMergerMutator::splitMutationCommands( } } } - //for (const auto & command : commands) - //{ - // if (command.type == MutationCommand::Type::MATERIALIZE_INDEX || command.type == MutationCommand::Type::MATERIALIZE_TTL) - // { - // for_interpreter.push_back(command); - // for (const auto & [column_name, expr] : command.column_to_update_expression) - // already_changed_columns.emplace(column_name); - // } - // if (command.type == MutationCommand::Type::DELETE - // || command.type == MutationCommand::Type::UPDATE - // || - // || ) - // { - // } - // else if (command.type == MutationCommand::Type::READ_COLUMN && ) - // { - // if (part_columns.has(command.column_name)) - // { - // for_interpreter.push_back(command); - // already_changed_columns.emplace(command.column_name); - // } - // else - // for_file_renames.push_back(command); - - // } - // else if (is_compact_part && command.type == MutationCommand::Type::DROP_COLUMN && part_columns.has(command.column_name)) - // { - // for_file_renames.push_back(command); - // part_columns.remove(command.column_name); - // } - // else if (command.type == MutationCommand::Type::RENAME_COLUMN && part_columns.has(command.column_name)) - // { - // if (is_compact_part) - // { - // for_interpreter.push_back( - // { - // .type = MutationCommand::Type::READ_COLUMN, - // .column_name = command.rename_to, - // }); - // already_changed_columns.emplace(command.column_name); - // part_columns.rename(command.column_name, command.rename_to); - // } - // else - // { - // } - // } - // else if (part_columns.has(command.column_name)) - // { - // for_file_renames.push_back(command); - // } - //} - - //if (is_compact_part) - //{ - // for (const auto & column : part->getColumns()) - // { - // if (!removed_columns_from_compact_part.count(column.name) - // && !already_changed_columns.count(column.name)) - // { - // for_interpreter.emplace_back(MutationCommand - // { - // .type = MutationCommand::Type::READ_COLUMN, - // .column_name = column.name, - // .data_type = column.type - // }); - // } - // } - //} } From 8d26cdd3889416119b89bba1db23603dede3ab68 Mon Sep 17 00:00:00 2001 From: Saulius Valatka Date: Tue, 19 May 2020 14:51:58 +0300 Subject: [PATCH 433/738] add reference to stream-loader in Third Party Integration docs (#11040) --- docs/es/interfaces/third-party/integrations.md | 1 + docs/fa/interfaces/third-party/integrations.md | 1 + docs/fr/interfaces/third-party/integrations.md | 1 + docs/ja/interfaces/third-party/integrations.md | 1 + docs/ru/interfaces/third-party/integrations.md | 1 + docs/tr/interfaces/third-party/integrations.md | 1 + docs/zh/interfaces/third-party/integrations.md | 1 + 7 files changed, 7 insertions(+) diff --git a/docs/es/interfaces/third-party/integrations.md b/docs/es/interfaces/third-party/integrations.md index 5e9e8f841bf..716e774871b 100644 --- a/docs/es/interfaces/third-party/integrations.md +++ b/docs/es/interfaces/third-party/integrations.md @@ -25,6 +25,7 @@ toc_title: Integrations - Message queues - [Kafka](https://kafka.apache.org) - [clickhouse\_sinker](https://github.com/housepower/clickhouse_sinker) (uses [Go client](https://github.com/ClickHouse/clickhouse-go/)) + - [stream-loader-clickhouse](https://github.com/adform/stream-loader) - Stream processing - [Flink](https://flink.apache.org) - [flink-clickhouse-sink](https://github.com/ivi-ru/flink-clickhouse-sink) diff --git a/docs/fa/interfaces/third-party/integrations.md b/docs/fa/interfaces/third-party/integrations.md index 2e3bb52fa3d..657432c7958 100644 --- a/docs/fa/interfaces/third-party/integrations.md +++ b/docs/fa/interfaces/third-party/integrations.md @@ -27,6 +27,7 @@ toc_title: "\u06CC\u06A9\u067E\u0627\u0631\u0686\u06AF\u06CC" - صف پیام - [کافکا](https://kafka.apache.org) - [در حال بارگذاری](https://github.com/housepower/clickhouse_sinker) (استفاده [برو کارگیر](https://github.com/ClickHouse/clickhouse-go/)) + - [stream-loader-clickhouse](https://github.com/adform/stream-loader) - پردازش جریان - [لرزش](https://flink.apache.org) - [سینک فلینک-کلیک](https://github.com/ivi-ru/flink-clickhouse-sink) diff --git a/docs/fr/interfaces/third-party/integrations.md b/docs/fr/interfaces/third-party/integrations.md index 6edb04ce518..f252fd6229b 100644 --- a/docs/fr/interfaces/third-party/integrations.md +++ b/docs/fr/interfaces/third-party/integrations.md @@ -27,6 +27,7 @@ toc_title: "Int\xE9gration" - Files d'attente de messages - [Kafka](https://kafka.apache.org) - [clickhouse\_sinker](https://github.com/housepower/clickhouse_sinker) (utiliser [Allez client](https://github.com/ClickHouse/clickhouse-go/)) + - [stream-loader-clickhouse](https://github.com/adform/stream-loader) - Traitement de flux - [Flink](https://flink.apache.org) - [flink-clickhouse-évier](https://github.com/ivi-ru/flink-clickhouse-sink) diff --git a/docs/ja/interfaces/third-party/integrations.md b/docs/ja/interfaces/third-party/integrations.md index 59876cd63a0..3e38d578093 100644 --- a/docs/ja/interfaces/third-party/integrations.md +++ b/docs/ja/interfaces/third-party/integrations.md @@ -27,6 +27,7 @@ toc_title: "\u7D71\u5408" - メッセージキュ - [カフカ](https://kafka.apache.org) - [clickhouse\_sinker](https://github.com/housepower/clickhouse_sinker) （用途 [Goクライアント](https://github.com/ClickHouse/clickhouse-go/)) + - [stream-loader-clickhouse](https://github.com/adform/stream-loader) - ストリーム処理 - [フリンク](https://flink.apache.org) - [フリンク-クリックハウス-シンク](https://github.com/ivi-ru/flink-clickhouse-sink) diff --git a/docs/ru/interfaces/third-party/integrations.md b/docs/ru/interfaces/third-party/integrations.md index bc4b9f22950..39449b54df8 100644 --- a/docs/ru/interfaces/third-party/integrations.md +++ b/docs/ru/interfaces/third-party/integrations.md @@ -20,6 +20,7 @@ - Очереди сообщений - [Kafka](https://kafka.apache.org) - [clickhouse\_sinker](https://github.com/housepower/clickhouse_sinker) (использует [Go client](https://github.com/ClickHouse/clickhouse-go/)) + - [stream-loader-clickhouse](https://github.com/adform/stream-loader) - Потоковая обработка - [Flink](https://flink.apache.org) - [flink-clickhouse-sink](https://github.com/ivi-ru/flink-clickhouse-sink) diff --git a/docs/tr/interfaces/third-party/integrations.md b/docs/tr/interfaces/third-party/integrations.md index 91f0151fa8a..8a1d5c239f6 100644 --- a/docs/tr/interfaces/third-party/integrations.md +++ b/docs/tr/interfaces/third-party/integrations.md @@ -27,6 +27,7 @@ toc_title: Entegrasyonlar - Mesaj kuyrukları - [Kafka](https://kafka.apache.org) - [clickhouse\_sinker](https://github.com/housepower/clickhouse_sinker) (kullanma [Go client](https://github.com/ClickHouse/clickhouse-go/)) + - [stream-loader-clickhouse](https://github.com/adform/stream-loader) - Akış işleme - [Flink](https://flink.apache.org) - [flink-clickhouse-lavabo](https://github.com/ivi-ru/flink-clickhouse-sink) diff --git a/docs/zh/interfaces/third-party/integrations.md b/docs/zh/interfaces/third-party/integrations.md index 0d551829991..014fdc88304 100644 --- a/docs/zh/interfaces/third-party/integrations.md +++ b/docs/zh/interfaces/third-party/integrations.md @@ -19,6 +19,7 @@ - 消息队列 - [卡夫卡](https://kafka.apache.org) - [clickhouse\_sinker](https://github.com/housepower/clickhouse_sinker) (使用 [去客户](https://github.com/ClickHouse/clickhouse-go/)) + - [stream-loader-clickhouse](https://github.com/adform/stream-loader) - 流处理 - [Flink](https://flink.apache.org) - [flink-clickhouse-sink](https://github.com/ivi-ru/flink-clickhouse-sink) From d7d70b788d6927066158ef7c07a0201a9b147c24 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Tue, 19 May 2020 15:07:16 +0300 Subject: [PATCH 434/738] finally fix rules --- debian/rules | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/debian/rules b/debian/rules index 467b86e5d79..ad7b73380f4 100755 --- a/debian/rules +++ b/debian/rules @@ -26,7 +26,8 @@ ifndef ENABLE_TESTS CMAKE_FLAGS += -DENABLE_TESTS=0 else # To export binaries and from deb build we do not strip them. No need to run tests in deb build as we run them in CI - DEB_BUILD_OPTIONS+=" nocheck nostrip" + DEB_BUILD_OPTIONS+= nocheck + DEB_BUILD_OPTIONS+= nostrip endif ifndef MAKE_TARGET @@ -91,14 +92,18 @@ override_dh_auto_build: $(MAKE) $(THREADS_COUNT) -C $(BUILDDIR) $(MAKE_TARGET) override_dh_auto_test: +ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS))) cd $(BUILDDIR) && ctest $(THREADS_COUNT) -V -E with_server +endif override_dh_clean: rm -rf debian/copyright debian/clickhouse-client.docs debian/clickhouse-common-static.docs dh_clean # -X contrib override_dh_strip: +ifeq (,$(filter nostrip,$(DEB_BUILD_OPTIONS))) dh_strip -pclickhouse-common-static --dbg-package=clickhouse-common-static-dbg +endif override_dh_install: # Making docs From dbe36dd37dd9ae5a7d0019ef18dd347bf7cc32f1 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 19 May 2020 15:19:55 +0300 Subject: [PATCH 435/738] Update docs.js --- website/js/docs.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/js/docs.js b/website/js/docs.js index 7c1d8645cd2..f5a497ca033 100644 --- a/website/js/docs.js +++ b/website/js/docs.js @@ -155,7 +155,7 @@ $(document).ready(function () { url: window.location.pathname + 'rate/', type: 'POST', dataType: 'json', - data: {rating: $(this).prevAll().length + 1}, + data: JSON.stringify({rating: $(this).prevAll().length + 1}), success: function () { }, error: function () { From 2f28dbe7d6770df3642a78140c47545d07d39824 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 19 May 2020 15:31:08 +0300 Subject: [PATCH 436/738] [website] report client-side errors to sentry.io (#11039) --- docs/tools/website.py | 1 + website/js/base.js | 4 +- website/js/sentry.js | 5699 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 5701 insertions(+), 3 deletions(-) create mode 100644 website/js/sentry.js diff --git a/docs/tools/website.py b/docs/tools/website.py index 47425df380b..4dccc66693f 100644 --- a/docs/tools/website.py +++ b/docs/tools/website.py @@ -144,6 +144,7 @@ def get_js_in(args): f"'{args.website_dir}/js/jquery.js'", f"'{args.website_dir}/js/popper.js'", f"'{args.website_dir}/js/bootstrap.js'", + f"'{args.website_dir}/js/sentry.js'", f"'{args.website_dir}/js/base.js'", f"'{args.website_dir}/js/index.js'", f"'{args.website_dir}/js/docsearch.js'", diff --git a/website/js/base.js b/website/js/base.js index 4d0768ef919..46f06f70f30 100644 --- a/website/js/base.js +++ b/website/js/base.js @@ -1,4 +1,5 @@ (function () { + Sentry.init({ dsn: 'https://2b95b52c943f4ad99baccab7a9048e4d@o388870.ingest.sentry.io/5246103' }); $(document).click(function (event) { var target = $(event.target); var target_id = target.attr('id'); @@ -7,14 +8,11 @@ var is_collapse = target.attr('data-toggle') === 'collapse'; var is_rating = target.attr('role') === 'rating'; var navbar_toggle = $('#navbar-toggle'); - navbar_toggle.collapse('hide'); $('.algolia-autocomplete .ds-dropdown-menu').hide(); - if (target_id && target_id.startsWith('logo-')) { selector = '#'; } - if (selector && selector.startsWith('#') && !is_tab && !is_collapse && !is_rating) { event.preventDefault(); var dst = window.location.href.replace(window.location.hash, ''); diff --git a/website/js/sentry.js b/website/js/sentry.js new file mode 100644 index 00000000000..7d171824be7 --- /dev/null +++ b/website/js/sentry.js @@ -0,0 +1,5699 @@ +/*! @sentry/browser 5.15.5 (2accf045) | https://github.com/getsentry/sentry-javascript */ +var Sentry = (function (exports) { + /*! ***************************************************************************** + Copyright (c) Microsoft Corporation. All rights reserved. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use + this file except in compliance with the License. You may obtain a copy of the + License at http://www.apache.org/licenses/LICENSE-2.0 + + THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED + WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, + MERCHANTABLITY OR NON-INFRINGEMENT. + + See the Apache Version 2.0 License for specific language governing permissions + and limitations under the License. + ***************************************************************************** */ + /* global Reflect, Promise */ + + var extendStatics = function(d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + + function __extends(d, b) { + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + } + + var __assign = function() { + __assign = Object.assign || function __assign(t) { + for (var s, i = 1, n = arguments.length; i < n; i++) { + s = arguments[i]; + for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p)) t[p] = s[p]; + } + return t; + }; + return __assign.apply(this, arguments); + }; + + function __rest(s, e) { + var t = {}; + for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0) + t[p] = s[p]; + if (s != null && typeof Object.getOwnPropertySymbols === "function") + for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) if (e.indexOf(p[i]) < 0) + t[p[i]] = s[p[i]]; + return t; + } + + function __decorate(decorators, target, key, desc) { + var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d; + if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc); + else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r; + return c > 3 && r && Object.defineProperty(target, key, r), r; + } + + function __param(paramIndex, decorator) { + return function (target, key) { decorator(target, key, paramIndex); } + } + + function __metadata(metadataKey, metadataValue) { + if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(metadataKey, metadataValue); + } + + function __awaiter(thisArg, _arguments, P, generator) { + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); + } + + function __generator(thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (_) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } + } + + function __exportStar(m, exports) { + for (var p in m) if (!exports.hasOwnProperty(p)) exports[p] = m[p]; + } + + function __values(o) { + var m = typeof Symbol === "function" && o[Symbol.iterator], i = 0; + if (m) return m.call(o); + return { + next: function () { + if (o && i >= o.length) o = void 0; + return { value: o && o[i++], done: !o }; + } + }; + } + + function __read(o, n) { + var m = typeof Symbol === "function" && o[Symbol.iterator]; + if (!m) return o; + var i = m.call(o), r, ar = [], e; + try { + while ((n === void 0 || n-- > 0) && !(r = i.next()).done) ar.push(r.value); + } + catch (error) { e = { error: error }; } + finally { + try { + if (r && !r.done && (m = i["return"])) m.call(i); + } + finally { if (e) throw e.error; } + } + return ar; + } + + function __spread() { + for (var ar = [], i = 0; i < arguments.length; i++) + ar = ar.concat(__read(arguments[i])); + return ar; + } + + function __await(v) { + return this instanceof __await ? (this.v = v, this) : new __await(v); + } + + function __asyncGenerator(thisArg, _arguments, generator) { + if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined."); + var g = generator.apply(thisArg, _arguments || []), i, q = []; + return i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i; + function verb(n) { if (g[n]) i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; } + function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } } + function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r); } + function fulfill(value) { resume("next", value); } + function reject(value) { resume("throw", value); } + function settle(f, v) { if (f(v), q.shift(), q.length) resume(q[0][0], q[0][1]); } + } + + function __asyncDelegator(o) { + var i, p; + return i = {}, verb("next"), verb("throw", function (e) { throw e; }), verb("return"), i[Symbol.iterator] = function () { return this; }, i; + function verb(n, f) { i[n] = o[n] ? function (v) { return (p = !p) ? { value: __await(o[n](v)), done: n === "return" } : f ? f(v) : v; } : f; } + } + + function __asyncValues(o) { + if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined."); + var m = o[Symbol.asyncIterator], i; + return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i); + function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; } + function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); } + } + + function __makeTemplateObject(cooked, raw) { + if (Object.defineProperty) { Object.defineProperty(cooked, "raw", { value: raw }); } else { cooked.raw = raw; } + return cooked; + } + function __importStar(mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (Object.hasOwnProperty.call(mod, k)) result[k] = mod[k]; + result.default = mod; + return result; + } + + function __importDefault(mod) { + return (mod && mod.__esModule) ? mod : { default: mod }; + } + + var tslib_1 = /*#__PURE__*/Object.freeze({ + __extends: __extends, + get __assign () { return __assign; }, + __rest: __rest, + __decorate: __decorate, + __param: __param, + __metadata: __metadata, + __awaiter: __awaiter, + __generator: __generator, + __exportStar: __exportStar, + __values: __values, + __read: __read, + __spread: __spread, + __await: __await, + __asyncGenerator: __asyncGenerator, + __asyncDelegator: __asyncDelegator, + __asyncValues: __asyncValues, + __makeTemplateObject: __makeTemplateObject, + __importStar: __importStar, + __importDefault: __importDefault + }); + + /** Console logging verbosity for the SDK. */ + var LogLevel; + (function (LogLevel) { + /** No logs will be generated. */ + LogLevel[LogLevel["None"] = 0] = "None"; + /** Only SDK internal errors will be logged. */ + LogLevel[LogLevel["Error"] = 1] = "Error"; + /** Information useful for debugging the SDK will be logged. */ + LogLevel[LogLevel["Debug"] = 2] = "Debug"; + /** All SDK actions will be logged. */ + LogLevel[LogLevel["Verbose"] = 3] = "Verbose"; + })(LogLevel || (LogLevel = {})); + + /** JSDoc */ + (function (Severity) { + /** JSDoc */ + Severity["Fatal"] = "fatal"; + /** JSDoc */ + Severity["Error"] = "error"; + /** JSDoc */ + Severity["Warning"] = "warning"; + /** JSDoc */ + Severity["Log"] = "log"; + /** JSDoc */ + Severity["Info"] = "info"; + /** JSDoc */ + Severity["Debug"] = "debug"; + /** JSDoc */ + Severity["Critical"] = "critical"; + })(exports.Severity || (exports.Severity = {})); + // tslint:disable:completed-docs + // tslint:disable:no-unnecessary-qualifier no-namespace + (function (Severity) { + /** + * Converts a string-based level into a {@link Severity}. + * + * @param level string representation of Severity + * @returns Severity + */ + function fromString(level) { + switch (level) { + case 'debug': + return Severity.Debug; + case 'info': + return Severity.Info; + case 'warn': + case 'warning': + return Severity.Warning; + case 'error': + return Severity.Error; + case 'fatal': + return Severity.Fatal; + case 'critical': + return Severity.Critical; + case 'log': + default: + return Severity.Log; + } + } + Severity.fromString = fromString; + })(exports.Severity || (exports.Severity = {})); + + /** The status of an Span. */ + var SpanStatus; + (function (SpanStatus) { + /** The operation completed successfully. */ + SpanStatus["Ok"] = "ok"; + /** Deadline expired before operation could complete. */ + SpanStatus["DeadlineExceeded"] = "deadline_exceeded"; + /** 401 Unauthorized (actually does mean unauthenticated according to RFC 7235) */ + SpanStatus["Unauthenticated"] = "unauthenticated"; + /** 403 Forbidden */ + SpanStatus["PermissionDenied"] = "permission_denied"; + /** 404 Not Found. Some requested entity (file or directory) was not found. */ + SpanStatus["NotFound"] = "not_found"; + /** 429 Too Many Requests */ + SpanStatus["ResourceExhausted"] = "resource_exhausted"; + /** Client specified an invalid argument. 4xx. */ + SpanStatus["InvalidArgument"] = "invalid_argument"; + /** 501 Not Implemented */ + SpanStatus["Unimplemented"] = "unimplemented"; + /** 503 Service Unavailable */ + SpanStatus["Unavailable"] = "unavailable"; + /** Other/generic 5xx. */ + SpanStatus["InternalError"] = "internal_error"; + /** Unknown. Any non-standard HTTP status code. */ + SpanStatus["UnknownError"] = "unknown_error"; + /** The operation was cancelled (typically by the user). */ + SpanStatus["Cancelled"] = "cancelled"; + /** Already exists (409) */ + SpanStatus["AlreadyExists"] = "already_exists"; + /** Operation was rejected because the system is not in a state required for the operation's */ + SpanStatus["FailedPrecondition"] = "failed_precondition"; + /** The operation was aborted, typically due to a concurrency issue. */ + SpanStatus["Aborted"] = "aborted"; + /** Operation was attempted past the valid range. */ + SpanStatus["OutOfRange"] = "out_of_range"; + /** Unrecoverable data loss or corruption */ + SpanStatus["DataLoss"] = "data_loss"; + })(SpanStatus || (SpanStatus = {})); + // tslint:disable:no-unnecessary-qualifier no-namespace + (function (SpanStatus) { + /** + * Converts a HTTP status code into a {@link SpanStatus}. + * + * @param httpStatus The HTTP response status code. + * @returns The span status or {@link SpanStatus.UnknownError}. + */ + // tslint:disable-next-line:completed-docs + function fromHttpCode(httpStatus) { + if (httpStatus < 400) { + return SpanStatus.Ok; + } + if (httpStatus >= 400 && httpStatus < 500) { + switch (httpStatus) { + case 401: + return SpanStatus.Unauthenticated; + case 403: + return SpanStatus.PermissionDenied; + case 404: + return SpanStatus.NotFound; + case 409: + return SpanStatus.AlreadyExists; + case 413: + return SpanStatus.FailedPrecondition; + case 429: + return SpanStatus.ResourceExhausted; + default: + return SpanStatus.InvalidArgument; + } + } + if (httpStatus >= 500 && httpStatus < 600) { + switch (httpStatus) { + case 501: + return SpanStatus.Unimplemented; + case 503: + return SpanStatus.Unavailable; + case 504: + return SpanStatus.DeadlineExceeded; + default: + return SpanStatus.InternalError; + } + } + return SpanStatus.UnknownError; + } + SpanStatus.fromHttpCode = fromHttpCode; + })(SpanStatus || (SpanStatus = {})); + + /** The status of an event. */ + (function (Status) { + /** The status could not be determined. */ + Status["Unknown"] = "unknown"; + /** The event was skipped due to configuration or callbacks. */ + Status["Skipped"] = "skipped"; + /** The event was sent to Sentry successfully. */ + Status["Success"] = "success"; + /** The client is currently rate limited and will try again later. */ + Status["RateLimit"] = "rate_limit"; + /** The event could not be processed. */ + Status["Invalid"] = "invalid"; + /** A server-side error ocurred during submission. */ + Status["Failed"] = "failed"; + })(exports.Status || (exports.Status = {})); + // tslint:disable:completed-docs + // tslint:disable:no-unnecessary-qualifier no-namespace + (function (Status) { + /** + * Converts a HTTP status code into a {@link Status}. + * + * @param code The HTTP response status code. + * @returns The send status or {@link Status.Unknown}. + */ + function fromHttpCode(code) { + if (code >= 200 && code < 300) { + return Status.Success; + } + if (code === 429) { + return Status.RateLimit; + } + if (code >= 400 && code < 500) { + return Status.Invalid; + } + if (code >= 500) { + return Status.Failed; + } + return Status.Unknown; + } + Status.fromHttpCode = fromHttpCode; + })(exports.Status || (exports.Status = {})); + + /** + * Consumes the promise and logs the error when it rejects. + * @param promise A promise to forget. + */ + + var setPrototypeOf = Object.setPrototypeOf || ({ __proto__: [] } instanceof Array ? setProtoOf : mixinProperties); // tslint:disable-line:no-unbound-method + /** + * setPrototypeOf polyfill using __proto__ + */ + function setProtoOf(obj, proto) { + // @ts-ignore + obj.__proto__ = proto; + return obj; + } + /** + * setPrototypeOf polyfill using mixin + */ + function mixinProperties(obj, proto) { + for (var prop in proto) { + if (!obj.hasOwnProperty(prop)) { + // @ts-ignore + obj[prop] = proto[prop]; + } + } + return obj; + } + + /** An error emitted by Sentry SDKs and related utilities. */ + var SentryError = /** @class */ (function (_super) { + __extends(SentryError, _super); + function SentryError(message) { + var _newTarget = this.constructor; + var _this = _super.call(this, message) || this; + _this.message = message; + // tslint:disable:no-unsafe-any + _this.name = _newTarget.prototype.constructor.name; + setPrototypeOf(_this, _newTarget.prototype); + return _this; + } + return SentryError; + }(Error)); + + /** + * Checks whether given value's type is one of a few Error or Error-like + * {@link isError}. + * + * @param wat A value to be checked. + * @returns A boolean representing the result. + */ + function isError(wat) { + switch (Object.prototype.toString.call(wat)) { + case '[object Error]': + return true; + case '[object Exception]': + return true; + case '[object DOMException]': + return true; + default: + return isInstanceOf(wat, Error); + } + } + /** + * Checks whether given value's type is ErrorEvent + * {@link isErrorEvent}. + * + * @param wat A value to be checked. + * @returns A boolean representing the result. + */ + function isErrorEvent(wat) { + return Object.prototype.toString.call(wat) === '[object ErrorEvent]'; + } + /** + * Checks whether given value's type is DOMError + * {@link isDOMError}. + * + * @param wat A value to be checked. + * @returns A boolean representing the result. + */ + function isDOMError(wat) { + return Object.prototype.toString.call(wat) === '[object DOMError]'; + } + /** + * Checks whether given value's type is DOMException + * {@link isDOMException}. + * + * @param wat A value to be checked. + * @returns A boolean representing the result. + */ + function isDOMException(wat) { + return Object.prototype.toString.call(wat) === '[object DOMException]'; + } + /** + * Checks whether given value's type is a string + * {@link isString}. + * + * @param wat A value to be checked. + * @returns A boolean representing the result. + */ + function isString(wat) { + return Object.prototype.toString.call(wat) === '[object String]'; + } + /** + * Checks whether given value's is a primitive (undefined, null, number, boolean, string) + * {@link isPrimitive}. + * + * @param wat A value to be checked. + * @returns A boolean representing the result. + */ + function isPrimitive(wat) { + return wat === null || (typeof wat !== 'object' && typeof wat !== 'function'); + } + /** + * Checks whether given value's type is an object literal + * {@link isPlainObject}. + * + * @param wat A value to be checked. + * @returns A boolean representing the result. + */ + function isPlainObject(wat) { + return Object.prototype.toString.call(wat) === '[object Object]'; + } + /** + * Checks whether given value's type is an Event instance + * {@link isEvent}. + * + * @param wat A value to be checked. + * @returns A boolean representing the result. + */ + function isEvent(wat) { + // tslint:disable-next-line:strict-type-predicates + return typeof Event !== 'undefined' && isInstanceOf(wat, Event); + } + /** + * Checks whether given value's type is an Element instance + * {@link isElement}. + * + * @param wat A value to be checked. + * @returns A boolean representing the result. + */ + function isElement(wat) { + // tslint:disable-next-line:strict-type-predicates + return typeof Element !== 'undefined' && isInstanceOf(wat, Element); + } + /** + * Checks whether given value's type is an regexp + * {@link isRegExp}. + * + * @param wat A value to be checked. + * @returns A boolean representing the result. + */ + function isRegExp(wat) { + return Object.prototype.toString.call(wat) === '[object RegExp]'; + } + /** + * Checks whether given value has a then function. + * @param wat A value to be checked. + */ + function isThenable(wat) { + // tslint:disable:no-unsafe-any + return Boolean(wat && wat.then && typeof wat.then === 'function'); + // tslint:enable:no-unsafe-any + } + /** + * Checks whether given value's type is a SyntheticEvent + * {@link isSyntheticEvent}. + * + * @param wat A value to be checked. + * @returns A boolean representing the result. + */ + function isSyntheticEvent(wat) { + // tslint:disable-next-line:no-unsafe-any + return isPlainObject(wat) && 'nativeEvent' in wat && 'preventDefault' in wat && 'stopPropagation' in wat; + } + /** + * Checks whether given value's type is an instance of provided constructor. + * {@link isInstanceOf}. + * + * @param wat A value to be checked. + * @param base A constructor to be used in a check. + * @returns A boolean representing the result. + */ + function isInstanceOf(wat, base) { + try { + // tslint:disable-next-line:no-unsafe-any + return wat instanceof base; + } + catch (_e) { + return false; + } + } + + /** + * Truncates given string to the maximum characters count + * + * @param str An object that contains serializable values + * @param max Maximum number of characters in truncated string + * @returns string Encoded + */ + function truncate(str, max) { + if (max === void 0) { max = 0; } + // tslint:disable-next-line:strict-type-predicates + if (typeof str !== 'string' || max === 0) { + return str; + } + return str.length <= max ? str : str.substr(0, max) + "..."; + } + /** + * Join values in array + * @param input array of values to be joined together + * @param delimiter string to be placed in-between values + * @returns Joined values + */ + function safeJoin(input, delimiter) { + if (!Array.isArray(input)) { + return ''; + } + var output = []; + // tslint:disable-next-line:prefer-for-of + for (var i = 0; i < input.length; i++) { + var value = input[i]; + try { + output.push(String(value)); + } + catch (e) { + output.push('[value cannot be serialized]'); + } + } + return output.join(delimiter); + } + /** + * Checks if the value matches a regex or includes the string + * @param value The string value to be checked against + * @param pattern Either a regex or a string that must be contained in value + */ + function isMatchingPattern(value, pattern) { + if (!isString(value)) { + return false; + } + if (isRegExp(pattern)) { + return pattern.test(value); + } + if (typeof pattern === 'string') { + return value.indexOf(pattern) !== -1; + } + return false; + } + + /** + * Requires a module which is protected against bundler minification. + * + * @param request The module path to resolve + */ + function dynamicRequire(mod, request) { + // tslint:disable-next-line: no-unsafe-any + return mod.require(request); + } + /** + * Checks whether we're in the Node.js or Browser environment + * + * @returns Answer to given question + */ + function isNodeEnv() { + // tslint:disable:strict-type-predicates + return Object.prototype.toString.call(typeof process !== 'undefined' ? process : 0) === '[object process]'; + } + var fallbackGlobalObject = {}; + /** + * Safely get global scope object + * + * @returns Global scope object + */ + function getGlobalObject() { + return (isNodeEnv() + ? global + : typeof window !== 'undefined' + ? window + : typeof self !== 'undefined' + ? self + : fallbackGlobalObject); + } + /** + * UUID4 generator + * + * @returns string Generated UUID4. + */ + function uuid4() { + var global = getGlobalObject(); + var crypto = global.crypto || global.msCrypto; + if (!(crypto === void 0) && crypto.getRandomValues) { + // Use window.crypto API if available + var arr = new Uint16Array(8); + crypto.getRandomValues(arr); + // set 4 in byte 7 + // tslint:disable-next-line:no-bitwise + arr[3] = (arr[3] & 0xfff) | 0x4000; + // set 2 most significant bits of byte 9 to '10' + // tslint:disable-next-line:no-bitwise + arr[4] = (arr[4] & 0x3fff) | 0x8000; + var pad = function (num) { + var v = num.toString(16); + while (v.length < 4) { + v = "0" + v; + } + return v; + }; + return (pad(arr[0]) + pad(arr[1]) + pad(arr[2]) + pad(arr[3]) + pad(arr[4]) + pad(arr[5]) + pad(arr[6]) + pad(arr[7])); + } + // http://stackoverflow.com/questions/105034/how-to-create-a-guid-uuid-in-javascript/2117523#2117523 + return 'xxxxxxxxxxxx4xxxyxxxxxxxxxxxxxxx'.replace(/[xy]/g, function (c) { + // tslint:disable-next-line:no-bitwise + var r = (Math.random() * 16) | 0; + // tslint:disable-next-line:no-bitwise + var v = c === 'x' ? r : (r & 0x3) | 0x8; + return v.toString(16); + }); + } + /** + * Parses string form of URL into an object + * // borrowed from https://tools.ietf.org/html/rfc3986#appendix-B + * // intentionally using regex and not href parsing trick because React Native and other + * // environments where DOM might not be available + * @returns parsed URL object + */ + function parseUrl(url) { + if (!url) { + return {}; + } + var match = url.match(/^(([^:\/?#]+):)?(\/\/([^\/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?$/); + if (!match) { + return {}; + } + // coerce to undefined values to empty string so we don't get 'undefined' + var query = match[6] || ''; + var fragment = match[8] || ''; + return { + host: match[4], + path: match[5], + protocol: match[2], + relative: match[5] + query + fragment, + }; + } + /** + * Extracts either message or type+value from an event that can be used for user-facing logs + * @returns event's description + */ + function getEventDescription(event) { + if (event.message) { + return event.message; + } + if (event.exception && event.exception.values && event.exception.values[0]) { + var exception = event.exception.values[0]; + if (exception.type && exception.value) { + return exception.type + ": " + exception.value; + } + return exception.type || exception.value || event.event_id || ''; + } + return event.event_id || ''; + } + /** JSDoc */ + function consoleSandbox(callback) { + var global = getGlobalObject(); + var levels = ['debug', 'info', 'warn', 'error', 'log', 'assert']; + if (!('console' in global)) { + return callback(); + } + var originalConsole = global.console; + var wrappedLevels = {}; + // Restore all wrapped console methods + levels.forEach(function (level) { + if (level in global.console && originalConsole[level].__sentry_original__) { + wrappedLevels[level] = originalConsole[level]; + originalConsole[level] = originalConsole[level].__sentry_original__; + } + }); + // Perform callback manipulations + var result = callback(); + // Revert restoration to wrapped state + Object.keys(wrappedLevels).forEach(function (level) { + originalConsole[level] = wrappedLevels[level]; + }); + return result; + } + /** + * Adds exception values, type and value to an synthetic Exception. + * @param event The event to modify. + * @param value Value of the exception. + * @param type Type of the exception. + * @hidden + */ + function addExceptionTypeValue(event, value, type) { + event.exception = event.exception || {}; + event.exception.values = event.exception.values || []; + event.exception.values[0] = event.exception.values[0] || {}; + event.exception.values[0].value = event.exception.values[0].value || value || ''; + event.exception.values[0].type = event.exception.values[0].type || type || 'Error'; + } + /** + * Adds exception mechanism to a given event. + * @param event The event to modify. + * @param mechanism Mechanism of the mechanism. + * @hidden + */ + function addExceptionMechanism(event, mechanism) { + if (mechanism === void 0) { mechanism = {}; } + // TODO: Use real type with `keyof Mechanism` thingy and maybe make it better? + try { + // @ts-ignore + // tslint:disable:no-non-null-assertion + event.exception.values[0].mechanism = event.exception.values[0].mechanism || {}; + Object.keys(mechanism).forEach(function (key) { + // @ts-ignore + event.exception.values[0].mechanism[key] = mechanism[key]; + }); + } + catch (_oO) { + // no-empty + } + } + /** + * A safe form of location.href + */ + function getLocationHref() { + try { + return document.location.href; + } + catch (oO) { + return ''; + } + } + /** + * Given a child DOM element, returns a query-selector statement describing that + * and its ancestors + * e.g. [HTMLElement] => body > div > input#foo.btn[name=baz] + * @returns generated DOM path + */ + function htmlTreeAsString(elem) { + // try/catch both: + // - accessing event.target (see getsentry/raven-js#838, #768) + // - `htmlTreeAsString` because it's complex, and just accessing the DOM incorrectly + // - can throw an exception in some circumstances. + try { + var currentElem = elem; + var MAX_TRAVERSE_HEIGHT = 5; + var MAX_OUTPUT_LEN = 80; + var out = []; + var height = 0; + var len = 0; + var separator = ' > '; + var sepLength = separator.length; + var nextStr = void 0; + while (currentElem && height++ < MAX_TRAVERSE_HEIGHT) { + nextStr = _htmlElementAsString(currentElem); + // bail out if + // - nextStr is the 'html' element + // - the length of the string that would be created exceeds MAX_OUTPUT_LEN + // (ignore this limit if we are on the first iteration) + if (nextStr === 'html' || (height > 1 && len + out.length * sepLength + nextStr.length >= MAX_OUTPUT_LEN)) { + break; + } + out.push(nextStr); + len += nextStr.length; + currentElem = currentElem.parentNode; + } + return out.reverse().join(separator); + } + catch (_oO) { + return ''; + } + } + /** + * Returns a simple, query-selector representation of a DOM element + * e.g. [HTMLElement] => input#foo.btn[name=baz] + * @returns generated DOM path + */ + function _htmlElementAsString(el) { + var elem = el; + var out = []; + var className; + var classes; + var key; + var attr; + var i; + if (!elem || !elem.tagName) { + return ''; + } + out.push(elem.tagName.toLowerCase()); + if (elem.id) { + out.push("#" + elem.id); + } + className = elem.className; + if (className && isString(className)) { + classes = className.split(/\s+/); + for (i = 0; i < classes.length; i++) { + out.push("." + classes[i]); + } + } + var attrWhitelist = ['type', 'name', 'title', 'alt']; + for (i = 0; i < attrWhitelist.length; i++) { + key = attrWhitelist[i]; + attr = elem.getAttribute(key); + if (attr) { + out.push("[" + key + "=\"" + attr + "\"]"); + } + } + return out.join(''); + } + var INITIAL_TIME = Date.now(); + var prevNow = 0; + var performanceFallback = { + now: function () { + var now = Date.now() - INITIAL_TIME; + if (now < prevNow) { + now = prevNow; + } + prevNow = now; + return now; + }, + timeOrigin: INITIAL_TIME, + }; + var crossPlatformPerformance = (function () { + if (isNodeEnv()) { + try { + var perfHooks = dynamicRequire(module, 'perf_hooks'); + return perfHooks.performance; + } + catch (_) { + return performanceFallback; + } + } + if (getGlobalObject().performance) { + // Polyfill for performance.timeOrigin. + // + // While performance.timing.navigationStart is deprecated in favor of performance.timeOrigin, performance.timeOrigin + // is not as widely supported. Namely, performance.timeOrigin is undefined in Safari as of writing. + // tslint:disable-next-line:strict-type-predicates + if (performance.timeOrigin === undefined) { + // As of writing, performance.timing is not available in Web Workers in mainstream browsers, so it is not always a + // valid fallback. In the absence of a initial time provided by the browser, fallback to INITIAL_TIME. + // @ts-ignore + // tslint:disable-next-line:deprecation + performance.timeOrigin = (performance.timing && performance.timing.navigationStart) || INITIAL_TIME; + } + } + return getGlobalObject().performance || performanceFallback; + })(); + /** + * Returns a timestamp in seconds with milliseconds precision since the UNIX epoch calculated with the monotonic clock. + */ + function timestampWithMs() { + return (crossPlatformPerformance.timeOrigin + crossPlatformPerformance.now()) / 1000; + } + var defaultRetryAfter = 60 * 1000; // 60 seconds + /** + * Extracts Retry-After value from the request header or returns default value + * @param now current unix timestamp + * @param header string representation of 'Retry-After' header + */ + function parseRetryAfterHeader(now, header) { + if (!header) { + return defaultRetryAfter; + } + var headerDelay = parseInt("" + header, 10); + if (!isNaN(headerDelay)) { + return headerDelay * 1000; + } + var headerDate = Date.parse("" + header); + if (!isNaN(headerDate)) { + return headerDate - now; + } + return defaultRetryAfter; + } + var defaultFunctionName = ''; + /** + * Safely extract function name from itself + */ + function getFunctionName(fn) { + try { + if (!fn || typeof fn !== 'function') { + return defaultFunctionName; + } + return fn.name || defaultFunctionName; + } + catch (e) { + // Just accessing custom props in some Selenium environments + // can cause a "Permission denied" exception (see raven-js#495). + return defaultFunctionName; + } + } + + // TODO: Implement different loggers for different environments + var global$1 = getGlobalObject(); + /** Prefix for logging strings */ + var PREFIX = 'Sentry Logger '; + /** JSDoc */ + var Logger = /** @class */ (function () { + /** JSDoc */ + function Logger() { + this._enabled = false; + } + /** JSDoc */ + Logger.prototype.disable = function () { + this._enabled = false; + }; + /** JSDoc */ + Logger.prototype.enable = function () { + this._enabled = true; + }; + /** JSDoc */ + Logger.prototype.log = function () { + var args = []; + for (var _i = 0; _i < arguments.length; _i++) { + args[_i] = arguments[_i]; + } + if (!this._enabled) { + return; + } + consoleSandbox(function () { + global$1.console.log(PREFIX + "[Log]: " + args.join(' ')); // tslint:disable-line:no-console + }); + }; + /** JSDoc */ + Logger.prototype.warn = function () { + var args = []; + for (var _i = 0; _i < arguments.length; _i++) { + args[_i] = arguments[_i]; + } + if (!this._enabled) { + return; + } + consoleSandbox(function () { + global$1.console.warn(PREFIX + "[Warn]: " + args.join(' ')); // tslint:disable-line:no-console + }); + }; + /** JSDoc */ + Logger.prototype.error = function () { + var args = []; + for (var _i = 0; _i < arguments.length; _i++) { + args[_i] = arguments[_i]; + } + if (!this._enabled) { + return; + } + consoleSandbox(function () { + global$1.console.error(PREFIX + "[Error]: " + args.join(' ')); // tslint:disable-line:no-console + }); + }; + return Logger; + }()); + // Ensure we only have a single logger instance, even if multiple versions of @sentry/utils are being used + global$1.__SENTRY__ = global$1.__SENTRY__ || {}; + var logger = global$1.__SENTRY__.logger || (global$1.__SENTRY__.logger = new Logger()); + + // tslint:disable:no-unsafe-any + /** + * Memo class used for decycle json objects. Uses WeakSet if available otherwise array. + */ + var Memo = /** @class */ (function () { + function Memo() { + // tslint:disable-next-line + this._hasWeakSet = typeof WeakSet === 'function'; + this._inner = this._hasWeakSet ? new WeakSet() : []; + } + /** + * Sets obj to remember. + * @param obj Object to remember + */ + Memo.prototype.memoize = function (obj) { + if (this._hasWeakSet) { + if (this._inner.has(obj)) { + return true; + } + this._inner.add(obj); + return false; + } + // tslint:disable-next-line:prefer-for-of + for (var i = 0; i < this._inner.length; i++) { + var value = this._inner[i]; + if (value === obj) { + return true; + } + } + this._inner.push(obj); + return false; + }; + /** + * Removes object from internal storage. + * @param obj Object to forget + */ + Memo.prototype.unmemoize = function (obj) { + if (this._hasWeakSet) { + this._inner.delete(obj); + } + else { + for (var i = 0; i < this._inner.length; i++) { + if (this._inner[i] === obj) { + this._inner.splice(i, 1); + break; + } + } + } + }; + return Memo; + }()); + + /** + * Wrap a given object method with a higher-order function + * + * @param source An object that contains a method to be wrapped. + * @param name A name of method to be wrapped. + * @param replacement A function that should be used to wrap a given method. + * @returns void + */ + function fill(source, name, replacement) { + if (!(name in source)) { + return; + } + var original = source[name]; + var wrapped = replacement(original); + // Make sure it's a function first, as we need to attach an empty prototype for `defineProperties` to work + // otherwise it'll throw "TypeError: Object.defineProperties called on non-object" + // tslint:disable-next-line:strict-type-predicates + if (typeof wrapped === 'function') { + try { + wrapped.prototype = wrapped.prototype || {}; + Object.defineProperties(wrapped, { + __sentry_original__: { + enumerable: false, + value: original, + }, + }); + } + catch (_Oo) { + // This can throw if multiple fill happens on a global object like XMLHttpRequest + // Fixes https://github.com/getsentry/sentry-javascript/issues/2043 + } + } + source[name] = wrapped; + } + /** + * Encodes given object into url-friendly format + * + * @param object An object that contains serializable values + * @returns string Encoded + */ + function urlEncode(object) { + return Object.keys(object) + .map( + // tslint:disable-next-line:no-unsafe-any + function (key) { return encodeURIComponent(key) + "=" + encodeURIComponent(object[key]); }) + .join('&'); + } + /** + * Transforms any object into an object literal with all it's attributes + * attached to it. + * + * @param value Initial source that we have to transform in order to be usable by the serializer + */ + function getWalkSource(value) { + if (isError(value)) { + var error = value; + var err = { + message: error.message, + name: error.name, + stack: error.stack, + }; + for (var i in error) { + if (Object.prototype.hasOwnProperty.call(error, i)) { + err[i] = error[i]; + } + } + return err; + } + if (isEvent(value)) { + var event_1 = value; + var source = {}; + source.type = event_1.type; + // Accessing event.target can throw (see getsentry/raven-js#838, #768) + try { + source.target = isElement(event_1.target) + ? htmlTreeAsString(event_1.target) + : Object.prototype.toString.call(event_1.target); + } + catch (_oO) { + source.target = ''; + } + try { + source.currentTarget = isElement(event_1.currentTarget) + ? htmlTreeAsString(event_1.currentTarget) + : Object.prototype.toString.call(event_1.currentTarget); + } + catch (_oO) { + source.currentTarget = ''; + } + // tslint:disable-next-line:strict-type-predicates + if (typeof CustomEvent !== 'undefined' && isInstanceOf(value, CustomEvent)) { + source.detail = event_1.detail; + } + for (var i in event_1) { + if (Object.prototype.hasOwnProperty.call(event_1, i)) { + source[i] = event_1; + } + } + return source; + } + return value; + } + /** Calculates bytes size of input string */ + function utf8Length(value) { + // tslint:disable-next-line:no-bitwise + return ~-encodeURI(value).split(/%..|./).length; + } + /** Calculates bytes size of input object */ + function jsonSize(value) { + return utf8Length(JSON.stringify(value)); + } + /** JSDoc */ + function normalizeToSize(object, + // Default Node.js REPL depth + depth, + // 100kB, as 200kB is max payload size, so half sounds reasonable + maxSize) { + if (depth === void 0) { depth = 3; } + if (maxSize === void 0) { maxSize = 100 * 1024; } + var serialized = normalize(object, depth); + if (jsonSize(serialized) > maxSize) { + return normalizeToSize(object, depth - 1, maxSize); + } + return serialized; + } + /** Transforms any input value into a string form, either primitive value or a type of the input */ + function serializeValue(value) { + var type = Object.prototype.toString.call(value); + // Node.js REPL notation + if (typeof value === 'string') { + return value; + } + if (type === '[object Object]') { + return '[Object]'; + } + if (type === '[object Array]') { + return '[Array]'; + } + var normalized = normalizeValue(value); + return isPrimitive(normalized) ? normalized : type; + } + /** + * normalizeValue() + * + * Takes unserializable input and make it serializable friendly + * + * - translates undefined/NaN values to "[undefined]"/"[NaN]" respectively, + * - serializes Error objects + * - filter global objects + */ + // tslint:disable-next-line:cyclomatic-complexity + function normalizeValue(value, key) { + if (key === 'domain' && value && typeof value === 'object' && value._events) { + return '[Domain]'; + } + if (key === 'domainEmitter') { + return '[DomainEmitter]'; + } + if (typeof global !== 'undefined' && value === global) { + return '[Global]'; + } + if (typeof window !== 'undefined' && value === window) { + return '[Window]'; + } + if (typeof document !== 'undefined' && value === document) { + return '[Document]'; + } + // React's SyntheticEvent thingy + if (isSyntheticEvent(value)) { + return '[SyntheticEvent]'; + } + // tslint:disable-next-line:no-tautology-expression + if (typeof value === 'number' && value !== value) { + return '[NaN]'; + } + if (value === void 0) { + return '[undefined]'; + } + if (typeof value === 'function') { + return "[Function: " + getFunctionName(value) + "]"; + } + return value; + } + /** + * Walks an object to perform a normalization on it + * + * @param key of object that's walked in current iteration + * @param value object to be walked + * @param depth Optional number indicating how deep should walking be performed + * @param memo Optional Memo class handling decycling + */ + function walk(key, value, depth, memo) { + if (depth === void 0) { depth = +Infinity; } + if (memo === void 0) { memo = new Memo(); } + // If we reach the maximum depth, serialize whatever has left + if (depth === 0) { + return serializeValue(value); + } + // If value implements `toJSON` method, call it and return early + // tslint:disable:no-unsafe-any + if (value !== null && value !== undefined && typeof value.toJSON === 'function') { + return value.toJSON(); + } + // tslint:enable:no-unsafe-any + // If normalized value is a primitive, there are no branches left to walk, so we can just bail out, as theres no point in going down that branch any further + var normalized = normalizeValue(value, key); + if (isPrimitive(normalized)) { + return normalized; + } + // Create source that we will use for next itterations, either objectified error object (Error type with extracted keys:value pairs) or the input itself + var source = getWalkSource(value); + // Create an accumulator that will act as a parent for all future itterations of that branch + var acc = Array.isArray(value) ? [] : {}; + // If we already walked that branch, bail out, as it's circular reference + if (memo.memoize(value)) { + return '[Circular ~]'; + } + // Walk all keys of the source + for (var innerKey in source) { + // Avoid iterating over fields in the prototype if they've somehow been exposed to enumeration. + if (!Object.prototype.hasOwnProperty.call(source, innerKey)) { + continue; + } + // Recursively walk through all the child nodes + acc[innerKey] = walk(innerKey, source[innerKey], depth - 1, memo); + } + // Once walked through all the branches, remove the parent from memo storage + memo.unmemoize(value); + // Return accumulated values + return acc; + } + /** + * normalize() + * + * - Creates a copy to prevent original input mutation + * - Skip non-enumerablers + * - Calls `toJSON` if implemented + * - Removes circular references + * - Translates non-serializeable values (undefined/NaN/Functions) to serializable format + * - Translates known global objects/Classes to a string representations + * - Takes care of Error objects serialization + * - Optionally limit depth of final output + */ + function normalize(input, depth) { + try { + // tslint:disable-next-line:no-unsafe-any + return JSON.parse(JSON.stringify(input, function (key, value) { return walk(key, value, depth); })); + } + catch (_oO) { + return '**non-serializable**'; + } + } + /** + * Given any captured exception, extract its keys and create a sorted + * and truncated list that will be used inside the event message. + * eg. `Non-error exception captured with keys: foo, bar, baz` + */ + function extractExceptionKeysForMessage(exception, maxLength) { + if (maxLength === void 0) { maxLength = 40; } + // tslint:disable:strict-type-predicates + var keys = Object.keys(getWalkSource(exception)); + keys.sort(); + if (!keys.length) { + return '[object has no keys]'; + } + if (keys[0].length >= maxLength) { + return truncate(keys[0], maxLength); + } + for (var includedKeys = keys.length; includedKeys > 0; includedKeys--) { + var serialized = keys.slice(0, includedKeys).join(', '); + if (serialized.length > maxLength) { + continue; + } + if (includedKeys === keys.length) { + return serialized; + } + return truncate(serialized, maxLength); + } + return ''; + } + + // Slightly modified (no IE8 support, ES6) and transcribed to TypeScript + + /** SyncPromise internal states */ + var States; + (function (States) { + /** Pending */ + States["PENDING"] = "PENDING"; + /** Resolved / OK */ + States["RESOLVED"] = "RESOLVED"; + /** Rejected / Error */ + States["REJECTED"] = "REJECTED"; + })(States || (States = {})); + /** + * Thenable class that behaves like a Promise and follows it's interface + * but is not async internally + */ + var SyncPromise = /** @class */ (function () { + function SyncPromise(executor) { + var _this = this; + this._state = States.PENDING; + this._handlers = []; + /** JSDoc */ + this._resolve = function (value) { + _this._setResult(States.RESOLVED, value); + }; + /** JSDoc */ + this._reject = function (reason) { + _this._setResult(States.REJECTED, reason); + }; + /** JSDoc */ + this._setResult = function (state, value) { + if (_this._state !== States.PENDING) { + return; + } + if (isThenable(value)) { + value.then(_this._resolve, _this._reject); + return; + } + _this._state = state; + _this._value = value; + _this._executeHandlers(); + }; + // TODO: FIXME + /** JSDoc */ + this._attachHandler = function (handler) { + _this._handlers = _this._handlers.concat(handler); + _this._executeHandlers(); + }; + /** JSDoc */ + this._executeHandlers = function () { + if (_this._state === States.PENDING) { + return; + } + var cachedHandlers = _this._handlers.slice(); + _this._handlers = []; + cachedHandlers.forEach(function (handler) { + if (handler.done) { + return; + } + if (_this._state === States.RESOLVED) { + if (handler.onfulfilled) { + handler.onfulfilled(_this._value); + } + } + if (_this._state === States.REJECTED) { + if (handler.onrejected) { + handler.onrejected(_this._value); + } + } + handler.done = true; + }); + }; + try { + executor(this._resolve, this._reject); + } + catch (e) { + this._reject(e); + } + } + /** JSDoc */ + SyncPromise.prototype.toString = function () { + return '[object SyncPromise]'; + }; + /** JSDoc */ + SyncPromise.resolve = function (value) { + return new SyncPromise(function (resolve) { + resolve(value); + }); + }; + /** JSDoc */ + SyncPromise.reject = function (reason) { + return new SyncPromise(function (_, reject) { + reject(reason); + }); + }; + /** JSDoc */ + SyncPromise.all = function (collection) { + return new SyncPromise(function (resolve, reject) { + if (!Array.isArray(collection)) { + reject(new TypeError("Promise.all requires an array as input.")); + return; + } + if (collection.length === 0) { + resolve([]); + return; + } + var counter = collection.length; + var resolvedCollection = []; + collection.forEach(function (item, index) { + SyncPromise.resolve(item) + .then(function (value) { + resolvedCollection[index] = value; + counter -= 1; + if (counter !== 0) { + return; + } + resolve(resolvedCollection); + }) + .then(null, reject); + }); + }); + }; + /** JSDoc */ + SyncPromise.prototype.then = function (onfulfilled, onrejected) { + var _this = this; + return new SyncPromise(function (resolve, reject) { + _this._attachHandler({ + done: false, + onfulfilled: function (result) { + if (!onfulfilled) { + // TODO: ¯\_(ツ)_/¯ + // TODO: FIXME + resolve(result); + return; + } + try { + resolve(onfulfilled(result)); + return; + } + catch (e) { + reject(e); + return; + } + }, + onrejected: function (reason) { + if (!onrejected) { + reject(reason); + return; + } + try { + resolve(onrejected(reason)); + return; + } + catch (e) { + reject(e); + return; + } + }, + }); + }); + }; + /** JSDoc */ + SyncPromise.prototype.catch = function (onrejected) { + return this.then(function (val) { return val; }, onrejected); + }; + /** JSDoc */ + SyncPromise.prototype.finally = function (onfinally) { + var _this = this; + return new SyncPromise(function (resolve, reject) { + var val; + var isRejected; + return _this.then(function (value) { + isRejected = false; + val = value; + if (onfinally) { + onfinally(); + } + }, function (reason) { + isRejected = true; + val = reason; + if (onfinally) { + onfinally(); + } + }).then(function () { + if (isRejected) { + reject(val); + return; + } + resolve(val); + }); + }); + }; + return SyncPromise; + }()); + + /** A simple queue that holds promises. */ + var PromiseBuffer = /** @class */ (function () { + function PromiseBuffer(_limit) { + this._limit = _limit; + /** Internal set of queued Promises */ + this._buffer = []; + } + /** + * Says if the buffer is ready to take more requests + */ + PromiseBuffer.prototype.isReady = function () { + return this._limit === undefined || this.length() < this._limit; + }; + /** + * Add a promise to the queue. + * + * @param task Can be any PromiseLike + * @returns The original promise. + */ + PromiseBuffer.prototype.add = function (task) { + var _this = this; + if (!this.isReady()) { + return SyncPromise.reject(new SentryError('Not adding Promise due to buffer limit reached.')); + } + if (this._buffer.indexOf(task) === -1) { + this._buffer.push(task); + } + task + .then(function () { return _this.remove(task); }) + .then(null, function () { + return _this.remove(task).then(null, function () { + // We have to add this catch here otherwise we have an unhandledPromiseRejection + // because it's a new Promise chain. + }); + }); + return task; + }; + /** + * Remove a promise to the queue. + * + * @param task Can be any PromiseLike + * @returns Removed promise. + */ + PromiseBuffer.prototype.remove = function (task) { + var removedTask = this._buffer.splice(this._buffer.indexOf(task), 1)[0]; + return removedTask; + }; + /** + * This function returns the number of unresolved promises in the queue. + */ + PromiseBuffer.prototype.length = function () { + return this._buffer.length; + }; + /** + * This will drain the whole queue, returns true if queue is empty or drained. + * If timeout is provided and the queue takes longer to drain, the promise still resolves but with false. + * + * @param timeout Number in ms to wait until it resolves with false. + */ + PromiseBuffer.prototype.drain = function (timeout) { + var _this = this; + return new SyncPromise(function (resolve) { + var capturedSetTimeout = setTimeout(function () { + if (timeout && timeout > 0) { + resolve(false); + } + }, timeout); + SyncPromise.all(_this._buffer) + .then(function () { + clearTimeout(capturedSetTimeout); + resolve(true); + }) + .then(null, function () { + resolve(true); + }); + }); + }; + return PromiseBuffer; + }()); + + /** + * Tells whether current environment supports Fetch API + * {@link supportsFetch}. + * + * @returns Answer to the given question. + */ + function supportsFetch() { + if (!('fetch' in getGlobalObject())) { + return false; + } + try { + // tslint:disable-next-line:no-unused-expression + new Headers(); + // tslint:disable-next-line:no-unused-expression + new Request(''); + // tslint:disable-next-line:no-unused-expression + new Response(); + return true; + } + catch (e) { + return false; + } + } + /** + * isNativeFetch checks if the given function is a native implementation of fetch() + */ + function isNativeFetch(func) { + return func && /^function fetch\(\)\s+\{\s+\[native code\]\s+\}$/.test(func.toString()); + } + /** + * Tells whether current environment supports Fetch API natively + * {@link supportsNativeFetch}. + * + * @returns true if `window.fetch` is natively implemented, false otherwise + */ + function supportsNativeFetch() { + if (!supportsFetch()) { + return false; + } + var global = getGlobalObject(); + // Fast path to avoid DOM I/O + // tslint:disable-next-line:no-unbound-method + if (isNativeFetch(global.fetch)) { + return true; + } + // window.fetch is implemented, but is polyfilled or already wrapped (e.g: by a chrome extension) + // so create a "pure" iframe to see if that has native fetch + var result = false; + var doc = global.document; + // tslint:disable-next-line:no-unbound-method deprecation + if (doc && typeof doc.createElement === "function") { + try { + var sandbox = doc.createElement('iframe'); + sandbox.hidden = true; + doc.head.appendChild(sandbox); + if (sandbox.contentWindow && sandbox.contentWindow.fetch) { + // tslint:disable-next-line:no-unbound-method + result = isNativeFetch(sandbox.contentWindow.fetch); + } + doc.head.removeChild(sandbox); + } + catch (err) { + logger.warn('Could not create sandbox iframe for pure fetch check, bailing to window.fetch: ', err); + } + } + return result; + } + /** + * Tells whether current environment supports Referrer Policy API + * {@link supportsReferrerPolicy}. + * + * @returns Answer to the given question. + */ + function supportsReferrerPolicy() { + // Despite all stars in the sky saying that Edge supports old draft syntax, aka 'never', 'always', 'origin' and 'default + // https://caniuse.com/#feat=referrer-policy + // It doesn't. And it throw exception instead of ignoring this parameter... + // REF: https://github.com/getsentry/raven-js/issues/1233 + if (!supportsFetch()) { + return false; + } + try { + // tslint:disable:no-unused-expression + new Request('_', { + referrerPolicy: 'origin', + }); + return true; + } + catch (e) { + return false; + } + } + /** + * Tells whether current environment supports History API + * {@link supportsHistory}. + * + * @returns Answer to the given question. + */ + function supportsHistory() { + // NOTE: in Chrome App environment, touching history.pushState, *even inside + // a try/catch block*, will cause Chrome to output an error to console.error + // borrowed from: https://github.com/angular/angular.js/pull/13945/files + var global = getGlobalObject(); + var chrome = global.chrome; + // tslint:disable-next-line:no-unsafe-any + var isChromePackagedApp = chrome && chrome.app && chrome.app.runtime; + var hasHistoryApi = 'history' in global && !!global.history.pushState && !!global.history.replaceState; + return !isChromePackagedApp && hasHistoryApi; + } + + /* tslint:disable:only-arrow-functions no-unsafe-any */ + var global$2 = getGlobalObject(); + /** + * Instrument native APIs to call handlers that can be used to create breadcrumbs, APM spans etc. + * - Console API + * - Fetch API + * - XHR API + * - History API + * - DOM API (click/typing) + * - Error API + * - UnhandledRejection API + */ + var handlers = {}; + var instrumented = {}; + /** Instruments given API */ + function instrument(type) { + if (instrumented[type]) { + return; + } + instrumented[type] = true; + switch (type) { + case 'console': + instrumentConsole(); + break; + case 'dom': + instrumentDOM(); + break; + case 'xhr': + instrumentXHR(); + break; + case 'fetch': + instrumentFetch(); + break; + case 'history': + instrumentHistory(); + break; + case 'error': + instrumentError(); + break; + case 'unhandledrejection': + instrumentUnhandledRejection(); + break; + default: + logger.warn('unknown instrumentation type:', type); + } + } + /** + * Add handler that will be called when given type of instrumentation triggers. + * Use at your own risk, this might break without changelog notice, only used internally. + * @hidden + */ + function addInstrumentationHandler(handler) { + // tslint:disable-next-line:strict-type-predicates + if (!handler || typeof handler.type !== 'string' || typeof handler.callback !== 'function') { + return; + } + handlers[handler.type] = handlers[handler.type] || []; + handlers[handler.type].push(handler.callback); + instrument(handler.type); + } + /** JSDoc */ + function triggerHandlers(type, data) { + var e_1, _a; + if (!type || !handlers[type]) { + return; + } + try { + for (var _b = __values(handlers[type] || []), _c = _b.next(); !_c.done; _c = _b.next()) { + var handler = _c.value; + try { + handler(data); + } + catch (e) { + logger.error("Error while triggering instrumentation handler.\nType: " + type + "\nName: " + getFunctionName(handler) + "\nError: " + e); + } + } + } + catch (e_1_1) { e_1 = { error: e_1_1 }; } + finally { + try { + if (_c && !_c.done && (_a = _b.return)) _a.call(_b); + } + finally { if (e_1) throw e_1.error; } + } + } + /** JSDoc */ + function instrumentConsole() { + if (!('console' in global$2)) { + return; + } + ['debug', 'info', 'warn', 'error', 'log', 'assert'].forEach(function (level) { + if (!(level in global$2.console)) { + return; + } + fill(global$2.console, level, function (originalConsoleLevel) { + return function () { + var args = []; + for (var _i = 0; _i < arguments.length; _i++) { + args[_i] = arguments[_i]; + } + triggerHandlers('console', { args: args, level: level }); + // this fails for some browsers. :( + if (originalConsoleLevel) { + Function.prototype.apply.call(originalConsoleLevel, global$2.console, args); + } + }; + }); + }); + } + /** JSDoc */ + function instrumentFetch() { + if (!supportsNativeFetch()) { + return; + } + fill(global$2, 'fetch', function (originalFetch) { + return function () { + var args = []; + for (var _i = 0; _i < arguments.length; _i++) { + args[_i] = arguments[_i]; + } + var commonHandlerData = { + args: args, + fetchData: { + method: getFetchMethod(args), + url: getFetchUrl(args), + }, + startTimestamp: Date.now(), + }; + triggerHandlers('fetch', __assign({}, commonHandlerData)); + return originalFetch.apply(global$2, args).then(function (response) { + triggerHandlers('fetch', __assign({}, commonHandlerData, { endTimestamp: Date.now(), response: response })); + return response; + }, function (error) { + triggerHandlers('fetch', __assign({}, commonHandlerData, { endTimestamp: Date.now(), error: error })); + throw error; + }); + }; + }); + } + /** Extract `method` from fetch call arguments */ + function getFetchMethod(fetchArgs) { + if (fetchArgs === void 0) { fetchArgs = []; } + if ('Request' in global$2 && isInstanceOf(fetchArgs[0], Request) && fetchArgs[0].method) { + return String(fetchArgs[0].method).toUpperCase(); + } + if (fetchArgs[1] && fetchArgs[1].method) { + return String(fetchArgs[1].method).toUpperCase(); + } + return 'GET'; + } + /** Extract `url` from fetch call arguments */ + function getFetchUrl(fetchArgs) { + if (fetchArgs === void 0) { fetchArgs = []; } + if (typeof fetchArgs[0] === 'string') { + return fetchArgs[0]; + } + if ('Request' in global$2 && isInstanceOf(fetchArgs[0], Request)) { + return fetchArgs[0].url; + } + return String(fetchArgs[0]); + } + /** JSDoc */ + function instrumentXHR() { + if (!('XMLHttpRequest' in global$2)) { + return; + } + var xhrproto = XMLHttpRequest.prototype; + fill(xhrproto, 'open', function (originalOpen) { + return function () { + var args = []; + for (var _i = 0; _i < arguments.length; _i++) { + args[_i] = arguments[_i]; + } + var url = args[1]; + this.__sentry_xhr__ = { + method: isString(args[0]) ? args[0].toUpperCase() : args[0], + url: args[1], + }; + // if Sentry key appears in URL, don't capture it as a request + if (isString(url) && this.__sentry_xhr__.method === 'POST' && url.match(/sentry_key/)) { + this.__sentry_own_request__ = true; + } + return originalOpen.apply(this, args); + }; + }); + fill(xhrproto, 'send', function (originalSend) { + return function () { + var args = []; + for (var _i = 0; _i < arguments.length; _i++) { + args[_i] = arguments[_i]; + } + var xhr = this; // tslint:disable-line:no-this-assignment + var commonHandlerData = { + args: args, + startTimestamp: Date.now(), + xhr: xhr, + }; + triggerHandlers('xhr', __assign({}, commonHandlerData)); + xhr.addEventListener('readystatechange', function () { + if (xhr.readyState === 4) { + try { + // touching statusCode in some platforms throws + // an exception + if (xhr.__sentry_xhr__) { + xhr.__sentry_xhr__.status_code = xhr.status; + } + } + catch (e) { + /* do nothing */ + } + triggerHandlers('xhr', __assign({}, commonHandlerData, { endTimestamp: Date.now() })); + } + }); + return originalSend.apply(this, args); + }; + }); + } + var lastHref; + /** JSDoc */ + function instrumentHistory() { + if (!supportsHistory()) { + return; + } + var oldOnPopState = global$2.onpopstate; + global$2.onpopstate = function () { + var args = []; + for (var _i = 0; _i < arguments.length; _i++) { + args[_i] = arguments[_i]; + } + var to = global$2.location.href; + // keep track of the current URL state, as we always receive only the updated state + var from = lastHref; + lastHref = to; + triggerHandlers('history', { + from: from, + to: to, + }); + if (oldOnPopState) { + return oldOnPopState.apply(this, args); + } + }; + /** @hidden */ + function historyReplacementFunction(originalHistoryFunction) { + return function () { + var args = []; + for (var _i = 0; _i < arguments.length; _i++) { + args[_i] = arguments[_i]; + } + var url = args.length > 2 ? args[2] : undefined; + if (url) { + // coerce to string (this is what pushState does) + var from = lastHref; + var to = String(url); + // keep track of the current URL state, as we always receive only the updated state + lastHref = to; + triggerHandlers('history', { + from: from, + to: to, + }); + } + return originalHistoryFunction.apply(this, args); + }; + } + fill(global$2.history, 'pushState', historyReplacementFunction); + fill(global$2.history, 'replaceState', historyReplacementFunction); + } + /** JSDoc */ + function instrumentDOM() { + if (!('document' in global$2)) { + return; + } + // Capture breadcrumbs from any click that is unhandled / bubbled up all the way + // to the document. Do this before we instrument addEventListener. + global$2.document.addEventListener('click', domEventHandler('click', triggerHandlers.bind(null, 'dom')), false); + global$2.document.addEventListener('keypress', keypressEventHandler(triggerHandlers.bind(null, 'dom')), false); + // After hooking into document bubbled up click and keypresses events, we also hook into user handled click & keypresses. + ['EventTarget', 'Node'].forEach(function (target) { + var proto = global$2[target] && global$2[target].prototype; + if (!proto || !proto.hasOwnProperty || !proto.hasOwnProperty('addEventListener')) { + return; + } + fill(proto, 'addEventListener', function (original) { + return function (eventName, fn, options) { + if (fn && fn.handleEvent) { + if (eventName === 'click') { + fill(fn, 'handleEvent', function (innerOriginal) { + return function (event) { + domEventHandler('click', triggerHandlers.bind(null, 'dom'))(event); + return innerOriginal.call(this, event); + }; + }); + } + if (eventName === 'keypress') { + fill(fn, 'handleEvent', function (innerOriginal) { + return function (event) { + keypressEventHandler(triggerHandlers.bind(null, 'dom'))(event); + return innerOriginal.call(this, event); + }; + }); + } + } + else { + if (eventName === 'click') { + domEventHandler('click', triggerHandlers.bind(null, 'dom'), true)(this); + } + if (eventName === 'keypress') { + keypressEventHandler(triggerHandlers.bind(null, 'dom'))(this); + } + } + return original.call(this, eventName, fn, options); + }; + }); + fill(proto, 'removeEventListener', function (original) { + return function (eventName, fn, options) { + var callback = fn; + try { + callback = callback && (callback.__sentry_wrapped__ || callback); + } + catch (e) { + // ignore, accessing __sentry_wrapped__ will throw in some Selenium environments + } + return original.call(this, eventName, callback, options); + }; + }); + }); + } + var debounceDuration = 1000; + var debounceTimer = 0; + var keypressTimeout; + var lastCapturedEvent; + /** + * Wraps addEventListener to capture UI breadcrumbs + * @param name the event name (e.g. "click") + * @param handler function that will be triggered + * @param debounce decides whether it should wait till another event loop + * @returns wrapped breadcrumb events handler + * @hidden + */ + function domEventHandler(name, handler, debounce) { + if (debounce === void 0) { debounce = false; } + return function (event) { + // reset keypress timeout; e.g. triggering a 'click' after + // a 'keypress' will reset the keypress debounce so that a new + // set of keypresses can be recorded + keypressTimeout = undefined; + // It's possible this handler might trigger multiple times for the same + // event (e.g. event propagation through node ancestors). Ignore if we've + // already captured the event. + if (!event || lastCapturedEvent === event) { + return; + } + lastCapturedEvent = event; + if (debounceTimer) { + clearTimeout(debounceTimer); + } + if (debounce) { + debounceTimer = setTimeout(function () { + handler({ event: event, name: name }); + }); + } + else { + handler({ event: event, name: name }); + } + }; + } + /** + * Wraps addEventListener to capture keypress UI events + * @param handler function that will be triggered + * @returns wrapped keypress events handler + * @hidden + */ + function keypressEventHandler(handler) { + // TODO: if somehow user switches keypress target before + // debounce timeout is triggered, we will only capture + // a single breadcrumb from the FIRST target (acceptable?) + return function (event) { + var target; + try { + target = event.target; + } + catch (e) { + // just accessing event properties can throw an exception in some rare circumstances + // see: https://github.com/getsentry/raven-js/issues/838 + return; + } + var tagName = target && target.tagName; + // only consider keypress events on actual input elements + // this will disregard keypresses targeting body (e.g. tabbing + // through elements, hotkeys, etc) + if (!tagName || (tagName !== 'INPUT' && tagName !== 'TEXTAREA' && !target.isContentEditable)) { + return; + } + // record first keypress in a series, but ignore subsequent + // keypresses until debounce clears + if (!keypressTimeout) { + domEventHandler('input', handler)(event); + } + clearTimeout(keypressTimeout); + keypressTimeout = setTimeout(function () { + keypressTimeout = undefined; + }, debounceDuration); + }; + } + var _oldOnErrorHandler = null; + /** JSDoc */ + function instrumentError() { + _oldOnErrorHandler = global$2.onerror; + global$2.onerror = function (msg, url, line, column, error) { + triggerHandlers('error', { + column: column, + error: error, + line: line, + msg: msg, + url: url, + }); + if (_oldOnErrorHandler) { + return _oldOnErrorHandler.apply(this, arguments); + } + return false; + }; + } + var _oldOnUnhandledRejectionHandler = null; + /** JSDoc */ + function instrumentUnhandledRejection() { + _oldOnUnhandledRejectionHandler = global$2.onunhandledrejection; + global$2.onunhandledrejection = function (e) { + triggerHandlers('unhandledrejection', e); + if (_oldOnUnhandledRejectionHandler) { + return _oldOnUnhandledRejectionHandler.apply(this, arguments); + } + return true; + }; + } + + /** Regular expression used to parse a Dsn. */ + var DSN_REGEX = /^(?:(\w+):)\/\/(?:(\w+)(?::(\w+))?@)([\w\.-]+)(?::(\d+))?\/(.+)/; + /** Error message */ + var ERROR_MESSAGE = 'Invalid Dsn'; + /** The Sentry Dsn, identifying a Sentry instance and project. */ + var Dsn = /** @class */ (function () { + /** Creates a new Dsn component */ + function Dsn(from) { + if (typeof from === 'string') { + this._fromString(from); + } + else { + this._fromComponents(from); + } + this._validate(); + } + /** + * Renders the string representation of this Dsn. + * + * By default, this will render the public representation without the password + * component. To get the deprecated private representation, set `withPassword` + * to true. + * + * @param withPassword When set to true, the password will be included. + */ + Dsn.prototype.toString = function (withPassword) { + if (withPassword === void 0) { withPassword = false; } + // tslint:disable-next-line:no-this-assignment + var _a = this, host = _a.host, path = _a.path, pass = _a.pass, port = _a.port, projectId = _a.projectId, protocol = _a.protocol, user = _a.user; + return (protocol + "://" + user + (withPassword && pass ? ":" + pass : '') + + ("@" + host + (port ? ":" + port : '') + "/" + (path ? path + "/" : path) + projectId)); + }; + /** Parses a string into this Dsn. */ + Dsn.prototype._fromString = function (str) { + var match = DSN_REGEX.exec(str); + if (!match) { + throw new SentryError(ERROR_MESSAGE); + } + var _a = __read(match.slice(1), 6), protocol = _a[0], user = _a[1], _b = _a[2], pass = _b === void 0 ? '' : _b, host = _a[3], _c = _a[4], port = _c === void 0 ? '' : _c, lastPath = _a[5]; + var path = ''; + var projectId = lastPath; + var split = projectId.split('/'); + if (split.length > 1) { + path = split.slice(0, -1).join('/'); + projectId = split.pop(); + } + this._fromComponents({ host: host, pass: pass, path: path, projectId: projectId, port: port, protocol: protocol, user: user }); + }; + /** Maps Dsn components into this instance. */ + Dsn.prototype._fromComponents = function (components) { + this.protocol = components.protocol; + this.user = components.user; + this.pass = components.pass || ''; + this.host = components.host; + this.port = components.port || ''; + this.path = components.path || ''; + this.projectId = components.projectId; + }; + /** Validates this Dsn and throws on error. */ + Dsn.prototype._validate = function () { + var _this = this; + ['protocol', 'user', 'host', 'projectId'].forEach(function (component) { + if (!_this[component]) { + throw new SentryError(ERROR_MESSAGE); + } + }); + if (this.protocol !== 'http' && this.protocol !== 'https') { + throw new SentryError(ERROR_MESSAGE); + } + if (this.port && isNaN(parseInt(this.port, 10))) { + throw new SentryError(ERROR_MESSAGE); + } + }; + return Dsn; + }()); + + /** + * Holds additional event information. {@link Scope.applyToEvent} will be + * called by the client before an event will be sent. + */ + var Scope = /** @class */ (function () { + function Scope() { + /** Flag if notifiying is happening. */ + this._notifyingListeners = false; + /** Callback for client to receive scope changes. */ + this._scopeListeners = []; + /** Callback list that will be called after {@link applyToEvent}. */ + this._eventProcessors = []; + /** Array of breadcrumbs. */ + this._breadcrumbs = []; + /** User */ + this._user = {}; + /** Tags */ + this._tags = {}; + /** Extra */ + this._extra = {}; + /** Contexts */ + this._context = {}; + } + /** + * Add internal on change listener. Used for sub SDKs that need to store the scope. + * @hidden + */ + Scope.prototype.addScopeListener = function (callback) { + this._scopeListeners.push(callback); + }; + /** + * @inheritDoc + */ + Scope.prototype.addEventProcessor = function (callback) { + this._eventProcessors.push(callback); + return this; + }; + /** + * This will be called on every set call. + */ + Scope.prototype._notifyScopeListeners = function () { + var _this = this; + if (!this._notifyingListeners) { + this._notifyingListeners = true; + setTimeout(function () { + _this._scopeListeners.forEach(function (callback) { + callback(_this); + }); + _this._notifyingListeners = false; + }); + } + }; + /** + * This will be called after {@link applyToEvent} is finished. + */ + Scope.prototype._notifyEventProcessors = function (processors, event, hint, index) { + var _this = this; + if (index === void 0) { index = 0; } + return new SyncPromise(function (resolve, reject) { + var processor = processors[index]; + // tslint:disable-next-line:strict-type-predicates + if (event === null || typeof processor !== 'function') { + resolve(event); + } + else { + var result = processor(__assign({}, event), hint); + if (isThenable(result)) { + result + .then(function (final) { return _this._notifyEventProcessors(processors, final, hint, index + 1).then(resolve); }) + .then(null, reject); + } + else { + _this._notifyEventProcessors(processors, result, hint, index + 1) + .then(resolve) + .then(null, reject); + } + } + }); + }; + /** + * @inheritDoc + */ + Scope.prototype.setUser = function (user) { + this._user = user || {}; + this._notifyScopeListeners(); + return this; + }; + /** + * @inheritDoc + */ + Scope.prototype.setTags = function (tags) { + this._tags = __assign({}, this._tags, tags); + this._notifyScopeListeners(); + return this; + }; + /** + * @inheritDoc + */ + Scope.prototype.setTag = function (key, value) { + var _a; + this._tags = __assign({}, this._tags, (_a = {}, _a[key] = value, _a)); + this._notifyScopeListeners(); + return this; + }; + /** + * @inheritDoc + */ + Scope.prototype.setExtras = function (extras) { + this._extra = __assign({}, this._extra, extras); + this._notifyScopeListeners(); + return this; + }; + /** + * @inheritDoc + */ + Scope.prototype.setExtra = function (key, extra) { + var _a; + this._extra = __assign({}, this._extra, (_a = {}, _a[key] = extra, _a)); + this._notifyScopeListeners(); + return this; + }; + /** + * @inheritDoc + */ + Scope.prototype.setFingerprint = function (fingerprint) { + this._fingerprint = fingerprint; + this._notifyScopeListeners(); + return this; + }; + /** + * @inheritDoc + */ + Scope.prototype.setLevel = function (level) { + this._level = level; + this._notifyScopeListeners(); + return this; + }; + /** + * @inheritDoc + */ + Scope.prototype.setTransaction = function (transaction) { + this._transaction = transaction; + if (this._span) { + this._span.transaction = transaction; + } + this._notifyScopeListeners(); + return this; + }; + /** + * @inheritDoc + */ + Scope.prototype.setContext = function (key, context) { + var _a; + this._context = __assign({}, this._context, (_a = {}, _a[key] = context, _a)); + this._notifyScopeListeners(); + return this; + }; + /** + * @inheritDoc + */ + Scope.prototype.setSpan = function (span) { + this._span = span; + this._notifyScopeListeners(); + return this; + }; + /** + * Internal getter for Span, used in Hub. + * @hidden + */ + Scope.prototype.getSpan = function () { + return this._span; + }; + /** + * Inherit values from the parent scope. + * @param scope to clone. + */ + Scope.clone = function (scope) { + var newScope = new Scope(); + if (scope) { + newScope._breadcrumbs = __spread(scope._breadcrumbs); + newScope._tags = __assign({}, scope._tags); + newScope._extra = __assign({}, scope._extra); + newScope._context = __assign({}, scope._context); + newScope._user = scope._user; + newScope._level = scope._level; + newScope._span = scope._span; + newScope._transaction = scope._transaction; + newScope._fingerprint = scope._fingerprint; + newScope._eventProcessors = __spread(scope._eventProcessors); + } + return newScope; + }; + /** + * @inheritDoc + */ + Scope.prototype.clear = function () { + this._breadcrumbs = []; + this._tags = {}; + this._extra = {}; + this._user = {}; + this._context = {}; + this._level = undefined; + this._transaction = undefined; + this._fingerprint = undefined; + this._span = undefined; + this._notifyScopeListeners(); + return this; + }; + /** + * @inheritDoc + */ + Scope.prototype.addBreadcrumb = function (breadcrumb, maxBreadcrumbs) { + var mergedBreadcrumb = __assign({ timestamp: timestampWithMs() }, breadcrumb); + this._breadcrumbs = + maxBreadcrumbs !== undefined && maxBreadcrumbs >= 0 + ? __spread(this._breadcrumbs, [mergedBreadcrumb]).slice(-maxBreadcrumbs) + : __spread(this._breadcrumbs, [mergedBreadcrumb]); + this._notifyScopeListeners(); + return this; + }; + /** + * @inheritDoc + */ + Scope.prototype.clearBreadcrumbs = function () { + this._breadcrumbs = []; + this._notifyScopeListeners(); + return this; + }; + /** + * Applies fingerprint from the scope to the event if there's one, + * uses message if there's one instead or get rid of empty fingerprint + */ + Scope.prototype._applyFingerprint = function (event) { + // Make sure it's an array first and we actually have something in place + event.fingerprint = event.fingerprint + ? Array.isArray(event.fingerprint) + ? event.fingerprint + : [event.fingerprint] + : []; + // If we have something on the scope, then merge it with event + if (this._fingerprint) { + event.fingerprint = event.fingerprint.concat(this._fingerprint); + } + // If we have no data at all, remove empty array default + if (event.fingerprint && !event.fingerprint.length) { + delete event.fingerprint; + } + }; + /** + * Applies the current context and fingerprint to the event. + * Note that breadcrumbs will be added by the client. + * Also if the event has already breadcrumbs on it, we do not merge them. + * @param event Event + * @param hint May contain additional informartion about the original exception. + * @hidden + */ + Scope.prototype.applyToEvent = function (event, hint) { + if (this._extra && Object.keys(this._extra).length) { + event.extra = __assign({}, this._extra, event.extra); + } + if (this._tags && Object.keys(this._tags).length) { + event.tags = __assign({}, this._tags, event.tags); + } + if (this._user && Object.keys(this._user).length) { + event.user = __assign({}, this._user, event.user); + } + if (this._context && Object.keys(this._context).length) { + event.contexts = __assign({}, this._context, event.contexts); + } + if (this._level) { + event.level = this._level; + } + if (this._transaction) { + event.transaction = this._transaction; + } + if (this._span) { + event.contexts = __assign({ trace: this._span.getTraceContext() }, event.contexts); + } + this._applyFingerprint(event); + event.breadcrumbs = __spread((event.breadcrumbs || []), this._breadcrumbs); + event.breadcrumbs = event.breadcrumbs.length > 0 ? event.breadcrumbs : undefined; + return this._notifyEventProcessors(__spread(getGlobalEventProcessors(), this._eventProcessors), event, hint); + }; + return Scope; + }()); + /** + * Retruns the global event processors. + */ + function getGlobalEventProcessors() { + var global = getGlobalObject(); + global.__SENTRY__ = global.__SENTRY__ || {}; + global.__SENTRY__.globalEventProcessors = global.__SENTRY__.globalEventProcessors || []; + return global.__SENTRY__.globalEventProcessors; + } + /** + * Add a EventProcessor to be kept globally. + * @param callback EventProcessor to add + */ + function addGlobalEventProcessor(callback) { + getGlobalEventProcessors().push(callback); + } + + /** + * API compatibility version of this hub. + * + * WARNING: This number should only be incresed when the global interface + * changes a and new methods are introduced. + * + * @hidden + */ + var API_VERSION = 3; + /** + * Default maximum number of breadcrumbs added to an event. Can be overwritten + * with {@link Options.maxBreadcrumbs}. + */ + var DEFAULT_BREADCRUMBS = 100; + /** + * Absolute maximum number of breadcrumbs added to an event. The + * `maxBreadcrumbs` option cannot be higher than this value. + */ + var MAX_BREADCRUMBS = 100; + /** + * @inheritDoc + */ + var Hub = /** @class */ (function () { + /** + * Creates a new instance of the hub, will push one {@link Layer} into the + * internal stack on creation. + * + * @param client bound to the hub. + * @param scope bound to the hub. + * @param version number, higher number means higher priority. + */ + function Hub(client, scope, _version) { + if (scope === void 0) { scope = new Scope(); } + if (_version === void 0) { _version = API_VERSION; } + this._version = _version; + /** Is a {@link Layer}[] containing the client and scope */ + this._stack = []; + this._stack.push({ client: client, scope: scope }); + } + /** + * Internal helper function to call a method on the top client if it exists. + * + * @param method The method to call on the client. + * @param args Arguments to pass to the client function. + */ + Hub.prototype._invokeClient = function (method) { + var _a; + var args = []; + for (var _i = 1; _i < arguments.length; _i++) { + args[_i - 1] = arguments[_i]; + } + var top = this.getStackTop(); + if (top && top.client && top.client[method]) { + (_a = top.client)[method].apply(_a, __spread(args, [top.scope])); + } + }; + /** + * @inheritDoc + */ + Hub.prototype.isOlderThan = function (version) { + return this._version < version; + }; + /** + * @inheritDoc + */ + Hub.prototype.bindClient = function (client) { + var top = this.getStackTop(); + top.client = client; + if (client && client.setupIntegrations) { + client.setupIntegrations(); + } + }; + /** + * @inheritDoc + */ + Hub.prototype.pushScope = function () { + // We want to clone the content of prev scope + var stack = this.getStack(); + var parentScope = stack.length > 0 ? stack[stack.length - 1].scope : undefined; + var scope = Scope.clone(parentScope); + this.getStack().push({ + client: this.getClient(), + scope: scope, + }); + return scope; + }; + /** + * @inheritDoc + */ + Hub.prototype.popScope = function () { + return this.getStack().pop() !== undefined; + }; + /** + * @inheritDoc + */ + Hub.prototype.withScope = function (callback) { + var scope = this.pushScope(); + try { + callback(scope); + } + finally { + this.popScope(); + } + }; + /** + * @inheritDoc + */ + Hub.prototype.getClient = function () { + return this.getStackTop().client; + }; + /** Returns the scope of the top stack. */ + Hub.prototype.getScope = function () { + return this.getStackTop().scope; + }; + /** Returns the scope stack for domains or the process. */ + Hub.prototype.getStack = function () { + return this._stack; + }; + /** Returns the topmost scope layer in the order domain > local > process. */ + Hub.prototype.getStackTop = function () { + return this._stack[this._stack.length - 1]; + }; + /** + * @inheritDoc + */ + Hub.prototype.captureException = function (exception, hint) { + var eventId = (this._lastEventId = uuid4()); + var finalHint = hint; + // If there's no explicit hint provided, mimick the same thing that would happen + // in the minimal itself to create a consistent behavior. + // We don't do this in the client, as it's the lowest level API, and doing this, + // would prevent user from having full control over direct calls. + if (!hint) { + var syntheticException = void 0; + try { + throw new Error('Sentry syntheticException'); + } + catch (exception) { + syntheticException = exception; + } + finalHint = { + originalException: exception, + syntheticException: syntheticException, + }; + } + this._invokeClient('captureException', exception, __assign({}, finalHint, { event_id: eventId })); + return eventId; + }; + /** + * @inheritDoc + */ + Hub.prototype.captureMessage = function (message, level, hint) { + var eventId = (this._lastEventId = uuid4()); + var finalHint = hint; + // If there's no explicit hint provided, mimick the same thing that would happen + // in the minimal itself to create a consistent behavior. + // We don't do this in the client, as it's the lowest level API, and doing this, + // would prevent user from having full control over direct calls. + if (!hint) { + var syntheticException = void 0; + try { + throw new Error(message); + } + catch (exception) { + syntheticException = exception; + } + finalHint = { + originalException: message, + syntheticException: syntheticException, + }; + } + this._invokeClient('captureMessage', message, level, __assign({}, finalHint, { event_id: eventId })); + return eventId; + }; + /** + * @inheritDoc + */ + Hub.prototype.captureEvent = function (event, hint) { + var eventId = (this._lastEventId = uuid4()); + this._invokeClient('captureEvent', event, __assign({}, hint, { event_id: eventId })); + return eventId; + }; + /** + * @inheritDoc + */ + Hub.prototype.lastEventId = function () { + return this._lastEventId; + }; + /** + * @inheritDoc + */ + Hub.prototype.addBreadcrumb = function (breadcrumb, hint) { + var top = this.getStackTop(); + if (!top.scope || !top.client) { + return; + } + var _a = (top.client.getOptions && top.client.getOptions()) || {}, _b = _a.beforeBreadcrumb, beforeBreadcrumb = _b === void 0 ? null : _b, _c = _a.maxBreadcrumbs, maxBreadcrumbs = _c === void 0 ? DEFAULT_BREADCRUMBS : _c; + if (maxBreadcrumbs <= 0) { + return; + } + var timestamp = timestampWithMs(); + var mergedBreadcrumb = __assign({ timestamp: timestamp }, breadcrumb); + var finalBreadcrumb = beforeBreadcrumb + ? consoleSandbox(function () { return beforeBreadcrumb(mergedBreadcrumb, hint); }) + : mergedBreadcrumb; + if (finalBreadcrumb === null) { + return; + } + top.scope.addBreadcrumb(finalBreadcrumb, Math.min(maxBreadcrumbs, MAX_BREADCRUMBS)); + }; + /** + * @inheritDoc + */ + Hub.prototype.setUser = function (user) { + var top = this.getStackTop(); + if (!top.scope) { + return; + } + top.scope.setUser(user); + }; + /** + * @inheritDoc + */ + Hub.prototype.setTags = function (tags) { + var top = this.getStackTop(); + if (!top.scope) { + return; + } + top.scope.setTags(tags); + }; + /** + * @inheritDoc + */ + Hub.prototype.setExtras = function (extras) { + var top = this.getStackTop(); + if (!top.scope) { + return; + } + top.scope.setExtras(extras); + }; + /** + * @inheritDoc + */ + Hub.prototype.setTag = function (key, value) { + var top = this.getStackTop(); + if (!top.scope) { + return; + } + top.scope.setTag(key, value); + }; + /** + * @inheritDoc + */ + Hub.prototype.setExtra = function (key, extra) { + var top = this.getStackTop(); + if (!top.scope) { + return; + } + top.scope.setExtra(key, extra); + }; + /** + * @inheritDoc + */ + Hub.prototype.setContext = function (name, context) { + var top = this.getStackTop(); + if (!top.scope) { + return; + } + top.scope.setContext(name, context); + }; + /** + * @inheritDoc + */ + Hub.prototype.configureScope = function (callback) { + var top = this.getStackTop(); + if (top.scope && top.client) { + callback(top.scope); + } + }; + /** + * @inheritDoc + */ + Hub.prototype.run = function (callback) { + var oldHub = makeMain(this); + try { + callback(this); + } + finally { + makeMain(oldHub); + } + }; + /** + * @inheritDoc + */ + Hub.prototype.getIntegration = function (integration) { + var client = this.getClient(); + if (!client) { + return null; + } + try { + return client.getIntegration(integration); + } + catch (_oO) { + logger.warn("Cannot retrieve integration " + integration.id + " from the current Hub"); + return null; + } + }; + /** + * @inheritDoc + */ + Hub.prototype.startSpan = function (spanOrSpanContext, forceNoChild) { + if (forceNoChild === void 0) { forceNoChild = false; } + return this._callExtensionMethod('startSpan', spanOrSpanContext, forceNoChild); + }; + /** + * @inheritDoc + */ + Hub.prototype.traceHeaders = function () { + return this._callExtensionMethod('traceHeaders'); + }; + /** + * Calls global extension method and binding current instance to the function call + */ + // @ts-ignore + Hub.prototype._callExtensionMethod = function (method) { + var args = []; + for (var _i = 1; _i < arguments.length; _i++) { + args[_i - 1] = arguments[_i]; + } + var carrier = getMainCarrier(); + var sentry = carrier.__SENTRY__; + // tslint:disable-next-line: strict-type-predicates + if (sentry && sentry.extensions && typeof sentry.extensions[method] === 'function') { + return sentry.extensions[method].apply(this, args); + } + logger.warn("Extension method " + method + " couldn't be found, doing nothing."); + }; + return Hub; + }()); + /** Returns the global shim registry. */ + function getMainCarrier() { + var carrier = getGlobalObject(); + carrier.__SENTRY__ = carrier.__SENTRY__ || { + extensions: {}, + hub: undefined, + }; + return carrier; + } + /** + * Replaces the current main hub with the passed one on the global object + * + * @returns The old replaced hub + */ + function makeMain(hub) { + var registry = getMainCarrier(); + var oldHub = getHubFromCarrier(registry); + setHubOnCarrier(registry, hub); + return oldHub; + } + /** + * Returns the default hub instance. + * + * If a hub is already registered in the global carrier but this module + * contains a more recent version, it replaces the registered version. + * Otherwise, the currently registered hub will be returned. + */ + function getCurrentHub() { + // Get main carrier (global for every environment) + var registry = getMainCarrier(); + // If there's no hub, or its an old API, assign a new one + if (!hasHubOnCarrier(registry) || getHubFromCarrier(registry).isOlderThan(API_VERSION)) { + setHubOnCarrier(registry, new Hub()); + } + // Prefer domains over global if they are there (applicable only to Node environment) + if (isNodeEnv()) { + return getHubFromActiveDomain(registry); + } + // Return hub that lives on a global object + return getHubFromCarrier(registry); + } + /** + * Try to read the hub from an active domain, fallback to the registry if one doesnt exist + * @returns discovered hub + */ + function getHubFromActiveDomain(registry) { + try { + var property = 'domain'; + var carrier = getMainCarrier(); + var sentry = carrier.__SENTRY__; + // tslint:disable-next-line: strict-type-predicates + if (!sentry || !sentry.extensions || !sentry.extensions[property]) { + return getHubFromCarrier(registry); + } + var domain = sentry.extensions[property]; + var activeDomain = domain.active; + // If there no active domain, just return global hub + if (!activeDomain) { + return getHubFromCarrier(registry); + } + // If there's no hub on current domain, or its an old API, assign a new one + if (!hasHubOnCarrier(activeDomain) || getHubFromCarrier(activeDomain).isOlderThan(API_VERSION)) { + var registryHubTopStack = getHubFromCarrier(registry).getStackTop(); + setHubOnCarrier(activeDomain, new Hub(registryHubTopStack.client, Scope.clone(registryHubTopStack.scope))); + } + // Return hub that lives on a domain + return getHubFromCarrier(activeDomain); + } + catch (_Oo) { + // Return hub that lives on a global object + return getHubFromCarrier(registry); + } + } + /** + * This will tell whether a carrier has a hub on it or not + * @param carrier object + */ + function hasHubOnCarrier(carrier) { + if (carrier && carrier.__SENTRY__ && carrier.__SENTRY__.hub) { + return true; + } + return false; + } + /** + * This will create a new {@link Hub} and add to the passed object on + * __SENTRY__.hub. + * @param carrier object + * @hidden + */ + function getHubFromCarrier(carrier) { + if (carrier && carrier.__SENTRY__ && carrier.__SENTRY__.hub) { + return carrier.__SENTRY__.hub; + } + carrier.__SENTRY__ = carrier.__SENTRY__ || {}; + carrier.__SENTRY__.hub = new Hub(); + return carrier.__SENTRY__.hub; + } + /** + * This will set passed {@link Hub} on the passed object's __SENTRY__.hub attribute + * @param carrier object + * @param hub Hub + */ + function setHubOnCarrier(carrier, hub) { + if (!carrier) { + return false; + } + carrier.__SENTRY__ = carrier.__SENTRY__ || {}; + carrier.__SENTRY__.hub = hub; + return true; + } + + /** + * This calls a function on the current hub. + * @param method function to call on hub. + * @param args to pass to function. + */ + function callOnHub(method) { + var args = []; + for (var _i = 1; _i < arguments.length; _i++) { + args[_i - 1] = arguments[_i]; + } + var hub = getCurrentHub(); + if (hub && hub[method]) { + // tslint:disable-next-line:no-unsafe-any + return hub[method].apply(hub, __spread(args)); + } + throw new Error("No hub defined or " + method + " was not found on the hub, please open a bug report."); + } + /** + * Captures an exception event and sends it to Sentry. + * + * @param exception An exception-like object. + * @returns The generated eventId. + */ + function captureException(exception) { + var syntheticException; + try { + throw new Error('Sentry syntheticException'); + } + catch (exception) { + syntheticException = exception; + } + return callOnHub('captureException', exception, { + originalException: exception, + syntheticException: syntheticException, + }); + } + /** + * Captures a message event and sends it to Sentry. + * + * @param message The message to send to Sentry. + * @param level Define the level of the message. + * @returns The generated eventId. + */ + function captureMessage(message, level) { + var syntheticException; + try { + throw new Error(message); + } + catch (exception) { + syntheticException = exception; + } + return callOnHub('captureMessage', message, level, { + originalException: message, + syntheticException: syntheticException, + }); + } + /** + * Captures a manually created event and sends it to Sentry. + * + * @param event The event to send to Sentry. + * @returns The generated eventId. + */ + function captureEvent(event) { + return callOnHub('captureEvent', event); + } + /** + * Callback to set context information onto the scope. + * @param callback Callback function that receives Scope. + */ + function configureScope(callback) { + callOnHub('configureScope', callback); + } + /** + * Records a new breadcrumb which will be attached to future events. + * + * Breadcrumbs will be added to subsequent events to provide more context on + * user's actions prior to an error or crash. + * + * @param breadcrumb The breadcrumb to record. + */ + function addBreadcrumb(breadcrumb) { + callOnHub('addBreadcrumb', breadcrumb); + } + /** + * Sets context data with the given name. + * @param name of the context + * @param context Any kind of data. This data will be normailzed. + */ + function setContext(name, context) { + callOnHub('setContext', name, context); + } + /** + * Set an object that will be merged sent as extra data with the event. + * @param extras Extras object to merge into current context. + */ + function setExtras(extras) { + callOnHub('setExtras', extras); + } + /** + * Set an object that will be merged sent as tags data with the event. + * @param tags Tags context object to merge into current context. + */ + function setTags(tags) { + callOnHub('setTags', tags); + } + /** + * Set key:value that will be sent as extra data with the event. + * @param key String of extra + * @param extra Any kind of data. This data will be normailzed. + */ + function setExtra(key, extra) { + callOnHub('setExtra', key, extra); + } + /** + * Set key:value that will be sent as tags data with the event. + * @param key String key of tag + * @param value String value of tag + */ + function setTag(key, value) { + callOnHub('setTag', key, value); + } + /** + * Updates user context information for future events. + * + * @param user User context object to be set in the current context. Pass `null` to unset the user. + */ + function setUser(user) { + callOnHub('setUser', user); + } + /** + * Creates a new scope with and executes the given operation within. + * The scope is automatically removed once the operation + * finishes or throws. + * + * This is essentially a convenience function for: + * + * pushScope(); + * callback(); + * popScope(); + * + * @param callback that will be enclosed into push/popScope. + */ + function withScope(callback) { + callOnHub('withScope', callback); + } + + var SENTRY_API_VERSION = '7'; + /** Helper class to provide urls to different Sentry endpoints. */ + var API = /** @class */ (function () { + /** Create a new instance of API */ + function API(dsn) { + this.dsn = dsn; + this._dsnObject = new Dsn(dsn); + } + /** Returns the Dsn object. */ + API.prototype.getDsn = function () { + return this._dsnObject; + }; + /** Returns a string with auth headers in the url to the store endpoint. */ + API.prototype.getStoreEndpoint = function () { + return "" + this._getBaseUrl() + this.getStoreEndpointPath(); + }; + /** Returns the store endpoint with auth added in url encoded. */ + API.prototype.getStoreEndpointWithUrlEncodedAuth = function () { + var dsn = this._dsnObject; + var auth = { + sentry_key: dsn.user, + sentry_version: SENTRY_API_VERSION, + }; + // Auth is intentionally sent as part of query string (NOT as custom HTTP header) + // to avoid preflight CORS requests + return this.getStoreEndpoint() + "?" + urlEncode(auth); + }; + /** Returns the base path of the url including the port. */ + API.prototype._getBaseUrl = function () { + var dsn = this._dsnObject; + var protocol = dsn.protocol ? dsn.protocol + ":" : ''; + var port = dsn.port ? ":" + dsn.port : ''; + return protocol + "//" + dsn.host + port; + }; + /** Returns only the path component for the store endpoint. */ + API.prototype.getStoreEndpointPath = function () { + var dsn = this._dsnObject; + return (dsn.path ? "/" + dsn.path : '') + "/api/" + dsn.projectId + "/store/"; + }; + /** Returns an object that can be used in request headers. */ + API.prototype.getRequestHeaders = function (clientName, clientVersion) { + var dsn = this._dsnObject; + var header = ["Sentry sentry_version=" + SENTRY_API_VERSION]; + header.push("sentry_client=" + clientName + "/" + clientVersion); + header.push("sentry_key=" + dsn.user); + if (dsn.pass) { + header.push("sentry_secret=" + dsn.pass); + } + return { + 'Content-Type': 'application/json', + 'X-Sentry-Auth': header.join(', '), + }; + }; + /** Returns the url to the report dialog endpoint. */ + API.prototype.getReportDialogEndpoint = function (dialogOptions) { + if (dialogOptions === void 0) { dialogOptions = {}; } + var dsn = this._dsnObject; + var endpoint = "" + this._getBaseUrl() + (dsn.path ? "/" + dsn.path : '') + "/api/embed/error-page/"; + var encodedOptions = []; + encodedOptions.push("dsn=" + dsn.toString()); + for (var key in dialogOptions) { + if (key === 'user') { + if (!dialogOptions.user) { + continue; + } + if (dialogOptions.user.name) { + encodedOptions.push("name=" + encodeURIComponent(dialogOptions.user.name)); + } + if (dialogOptions.user.email) { + encodedOptions.push("email=" + encodeURIComponent(dialogOptions.user.email)); + } + } + else { + encodedOptions.push(encodeURIComponent(key) + "=" + encodeURIComponent(dialogOptions[key])); + } + } + if (encodedOptions.length) { + return endpoint + "?" + encodedOptions.join('&'); + } + return endpoint; + }; + return API; + }()); + + var installedIntegrations = []; + /** Gets integration to install */ + function getIntegrationsToSetup(options) { + var defaultIntegrations = (options.defaultIntegrations && __spread(options.defaultIntegrations)) || []; + var userIntegrations = options.integrations; + var integrations = []; + if (Array.isArray(userIntegrations)) { + var userIntegrationsNames_1 = userIntegrations.map(function (i) { return i.name; }); + var pickedIntegrationsNames_1 = []; + // Leave only unique default integrations, that were not overridden with provided user integrations + defaultIntegrations.forEach(function (defaultIntegration) { + if (userIntegrationsNames_1.indexOf(defaultIntegration.name) === -1 && + pickedIntegrationsNames_1.indexOf(defaultIntegration.name) === -1) { + integrations.push(defaultIntegration); + pickedIntegrationsNames_1.push(defaultIntegration.name); + } + }); + // Don't add same user integration twice + userIntegrations.forEach(function (userIntegration) { + if (pickedIntegrationsNames_1.indexOf(userIntegration.name) === -1) { + integrations.push(userIntegration); + pickedIntegrationsNames_1.push(userIntegration.name); + } + }); + } + else if (typeof userIntegrations === 'function') { + integrations = userIntegrations(defaultIntegrations); + integrations = Array.isArray(integrations) ? integrations : [integrations]; + } + else { + integrations = __spread(defaultIntegrations); + } + // Make sure that if present, `Debug` integration will always run last + var integrationsNames = integrations.map(function (i) { return i.name; }); + var alwaysLastToRun = 'Debug'; + if (integrationsNames.indexOf(alwaysLastToRun) !== -1) { + integrations.push.apply(integrations, __spread(integrations.splice(integrationsNames.indexOf(alwaysLastToRun), 1))); + } + return integrations; + } + /** Setup given integration */ + function setupIntegration(integration) { + if (installedIntegrations.indexOf(integration.name) !== -1) { + return; + } + integration.setupOnce(addGlobalEventProcessor, getCurrentHub); + installedIntegrations.push(integration.name); + logger.log("Integration installed: " + integration.name); + } + /** + * Given a list of integration instances this installs them all. When `withDefaults` is set to `true` then all default + * integrations are added unless they were already provided before. + * @param integrations array of integration instances + * @param withDefault should enable default integrations + */ + function setupIntegrations(options) { + var integrations = {}; + getIntegrationsToSetup(options).forEach(function (integration) { + integrations[integration.name] = integration; + setupIntegration(integration); + }); + return integrations; + } + + /** + * Base implementation for all JavaScript SDK clients. + * + * Call the constructor with the corresponding backend constructor and options + * specific to the client subclass. To access these options later, use + * {@link Client.getOptions}. Also, the Backend instance is available via + * {@link Client.getBackend}. + * + * If a Dsn is specified in the options, it will be parsed and stored. Use + * {@link Client.getDsn} to retrieve the Dsn at any moment. In case the Dsn is + * invalid, the constructor will throw a {@link SentryException}. Note that + * without a valid Dsn, the SDK will not send any events to Sentry. + * + * Before sending an event via the backend, it is passed through + * {@link BaseClient.prepareEvent} to add SDK information and scope data + * (breadcrumbs and context). To add more custom information, override this + * method and extend the resulting prepared event. + * + * To issue automatically created events (e.g. via instrumentation), use + * {@link Client.captureEvent}. It will prepare the event and pass it through + * the callback lifecycle. To issue auto-breadcrumbs, use + * {@link Client.addBreadcrumb}. + * + * @example + * class NodeClient extends BaseClient { + * public constructor(options: NodeOptions) { + * super(NodeBackend, options); + * } + * + * // ... + * } + */ + var BaseClient = /** @class */ (function () { + /** + * Initializes this client instance. + * + * @param backendClass A constructor function to create the backend. + * @param options Options for the client. + */ + function BaseClient(backendClass, options) { + /** Array of used integrations. */ + this._integrations = {}; + /** Is the client still processing a call? */ + this._processing = false; + this._backend = new backendClass(options); + this._options = options; + if (options.dsn) { + this._dsn = new Dsn(options.dsn); + } + } + /** + * @inheritDoc + */ + BaseClient.prototype.captureException = function (exception, hint, scope) { + var _this = this; + var eventId = hint && hint.event_id; + this._processing = true; + this._getBackend() + .eventFromException(exception, hint) + .then(function (event) { return _this._processEvent(event, hint, scope); }) + .then(function (finalEvent) { + // We need to check for finalEvent in case beforeSend returned null + eventId = finalEvent && finalEvent.event_id; + _this._processing = false; + }) + .then(null, function (reason) { + logger.error(reason); + _this._processing = false; + }); + return eventId; + }; + /** + * @inheritDoc + */ + BaseClient.prototype.captureMessage = function (message, level, hint, scope) { + var _this = this; + var eventId = hint && hint.event_id; + this._processing = true; + var promisedEvent = isPrimitive(message) + ? this._getBackend().eventFromMessage("" + message, level, hint) + : this._getBackend().eventFromException(message, hint); + promisedEvent + .then(function (event) { return _this._processEvent(event, hint, scope); }) + .then(function (finalEvent) { + // We need to check for finalEvent in case beforeSend returned null + eventId = finalEvent && finalEvent.event_id; + _this._processing = false; + }) + .then(null, function (reason) { + logger.error(reason); + _this._processing = false; + }); + return eventId; + }; + /** + * @inheritDoc + */ + BaseClient.prototype.captureEvent = function (event, hint, scope) { + var _this = this; + var eventId = hint && hint.event_id; + this._processing = true; + this._processEvent(event, hint, scope) + .then(function (finalEvent) { + // We need to check for finalEvent in case beforeSend returned null + eventId = finalEvent && finalEvent.event_id; + _this._processing = false; + }) + .then(null, function (reason) { + logger.error(reason); + _this._processing = false; + }); + return eventId; + }; + /** + * @inheritDoc + */ + BaseClient.prototype.getDsn = function () { + return this._dsn; + }; + /** + * @inheritDoc + */ + BaseClient.prototype.getOptions = function () { + return this._options; + }; + /** + * @inheritDoc + */ + BaseClient.prototype.flush = function (timeout) { + var _this = this; + return this._isClientProcessing(timeout).then(function (status) { + clearInterval(status.interval); + return _this._getBackend() + .getTransport() + .close(timeout) + .then(function (transportFlushed) { return status.ready && transportFlushed; }); + }); + }; + /** + * @inheritDoc + */ + BaseClient.prototype.close = function (timeout) { + var _this = this; + return this.flush(timeout).then(function (result) { + _this.getOptions().enabled = false; + return result; + }); + }; + /** + * Sets up the integrations + */ + BaseClient.prototype.setupIntegrations = function () { + if (this._isEnabled()) { + this._integrations = setupIntegrations(this._options); + } + }; + /** + * @inheritDoc + */ + BaseClient.prototype.getIntegration = function (integration) { + try { + return this._integrations[integration.id] || null; + } + catch (_oO) { + logger.warn("Cannot retrieve integration " + integration.id + " from the current Client"); + return null; + } + }; + /** Waits for the client to be done with processing. */ + BaseClient.prototype._isClientProcessing = function (timeout) { + var _this = this; + return new SyncPromise(function (resolve) { + var ticked = 0; + var tick = 1; + var interval = 0; + clearInterval(interval); + interval = setInterval(function () { + if (!_this._processing) { + resolve({ + interval: interval, + ready: true, + }); + } + else { + ticked += tick; + if (timeout && ticked >= timeout) { + resolve({ + interval: interval, + ready: false, + }); + } + } + }, tick); + }); + }; + /** Returns the current backend. */ + BaseClient.prototype._getBackend = function () { + return this._backend; + }; + /** Determines whether this SDK is enabled and a valid Dsn is present. */ + BaseClient.prototype._isEnabled = function () { + return this.getOptions().enabled !== false && this._dsn !== undefined; + }; + /** + * Adds common information to events. + * + * The information includes release and environment from `options`, + * breadcrumbs and context (extra, tags and user) from the scope. + * + * Information that is already present in the event is never overwritten. For + * nested objects, such as the context, keys are merged. + * + * @param event The original event. + * @param hint May contain additional informartion about the original exception. + * @param scope A scope containing event metadata. + * @returns A new event with more information. + */ + BaseClient.prototype._prepareEvent = function (event, scope, hint) { + var _this = this; + var _a = this.getOptions(), environment = _a.environment, release = _a.release, dist = _a.dist, _b = _a.maxValueLength, maxValueLength = _b === void 0 ? 250 : _b, _c = _a.normalizeDepth, normalizeDepth = _c === void 0 ? 3 : _c; + var prepared = __assign({}, event); + if (prepared.environment === undefined && environment !== undefined) { + prepared.environment = environment; + } + if (prepared.release === undefined && release !== undefined) { + prepared.release = release; + } + if (prepared.dist === undefined && dist !== undefined) { + prepared.dist = dist; + } + if (prepared.message) { + prepared.message = truncate(prepared.message, maxValueLength); + } + var exception = prepared.exception && prepared.exception.values && prepared.exception.values[0]; + if (exception && exception.value) { + exception.value = truncate(exception.value, maxValueLength); + } + var request = prepared.request; + if (request && request.url) { + request.url = truncate(request.url, maxValueLength); + } + if (prepared.event_id === undefined) { + prepared.event_id = hint && hint.event_id ? hint.event_id : uuid4(); + } + this._addIntegrations(prepared.sdk); + // We prepare the result here with a resolved Event. + var result = SyncPromise.resolve(prepared); + // This should be the last thing called, since we want that + // {@link Hub.addEventProcessor} gets the finished prepared event. + if (scope) { + // In case we have a hub we reassign it. + result = scope.applyToEvent(prepared, hint); + } + return result.then(function (evt) { + // tslint:disable-next-line:strict-type-predicates + if (typeof normalizeDepth === 'number' && normalizeDepth > 0) { + return _this._normalizeEvent(evt, normalizeDepth); + } + return evt; + }); + }; + /** + * Applies `normalize` function on necessary `Event` attributes to make them safe for serialization. + * Normalized keys: + * - `breadcrumbs.data` + * - `user` + * - `contexts` + * - `extra` + * @param event Event + * @returns Normalized event + */ + BaseClient.prototype._normalizeEvent = function (event, depth) { + if (!event) { + return null; + } + // tslint:disable:no-unsafe-any + return __assign({}, event, (event.breadcrumbs && { + breadcrumbs: event.breadcrumbs.map(function (b) { return (__assign({}, b, (b.data && { + data: normalize(b.data, depth), + }))); }), + }), (event.user && { + user: normalize(event.user, depth), + }), (event.contexts && { + contexts: normalize(event.contexts, depth), + }), (event.extra && { + extra: normalize(event.extra, depth), + })); + }; + /** + * This function adds all used integrations to the SDK info in the event. + * @param sdkInfo The sdkInfo of the event that will be filled with all integrations. + */ + BaseClient.prototype._addIntegrations = function (sdkInfo) { + var integrationsArray = Object.keys(this._integrations); + if (sdkInfo && integrationsArray.length > 0) { + sdkInfo.integrations = integrationsArray; + } + }; + /** + * Processes an event (either error or message) and sends it to Sentry. + * + * This also adds breadcrumbs and context information to the event. However, + * platform specific meta data (such as the User's IP address) must be added + * by the SDK implementor. + * + * + * @param event The event to send to Sentry. + * @param hint May contain additional informartion about the original exception. + * @param scope A scope containing event metadata. + * @returns A SyncPromise that resolves with the event or rejects in case event was/will not be send. + */ + BaseClient.prototype._processEvent = function (event, hint, scope) { + var _this = this; + var _a = this.getOptions(), beforeSend = _a.beforeSend, sampleRate = _a.sampleRate; + if (!this._isEnabled()) { + return SyncPromise.reject('SDK not enabled, will not send event.'); + } + // 1.0 === 100% events are sent + // 0.0 === 0% events are sent + if (typeof sampleRate === 'number' && Math.random() > sampleRate) { + return SyncPromise.reject('This event has been sampled, will not send event.'); + } + return new SyncPromise(function (resolve, reject) { + _this._prepareEvent(event, scope, hint) + .then(function (prepared) { + if (prepared === null) { + reject('An event processor returned null, will not send event.'); + return; + } + var finalEvent = prepared; + var isInternalException = hint && hint.data && hint.data.__sentry__ === true; + if (isInternalException || !beforeSend) { + _this._getBackend().sendEvent(finalEvent); + resolve(finalEvent); + return; + } + var beforeSendResult = beforeSend(prepared, hint); + // tslint:disable-next-line:strict-type-predicates + if (typeof beforeSendResult === 'undefined') { + logger.error('`beforeSend` method has to return `null` or a valid event.'); + } + else if (isThenable(beforeSendResult)) { + _this._handleAsyncBeforeSend(beforeSendResult, resolve, reject); + } + else { + finalEvent = beforeSendResult; + if (finalEvent === null) { + logger.log('`beforeSend` returned `null`, will not send event.'); + resolve(null); + return; + } + // From here on we are really async + _this._getBackend().sendEvent(finalEvent); + resolve(finalEvent); + } + }) + .then(null, function (reason) { + _this.captureException(reason, { + data: { + __sentry__: true, + }, + originalException: reason, + }); + reject("Event processing pipeline threw an error, original event will not be sent. Details have been sent as a new event.\nReason: " + reason); + }); + }); + }; + /** + * Resolves before send Promise and calls resolve/reject on parent SyncPromise. + */ + BaseClient.prototype._handleAsyncBeforeSend = function (beforeSend, resolve, reject) { + var _this = this; + beforeSend + .then(function (processedEvent) { + if (processedEvent === null) { + reject('`beforeSend` returned `null`, will not send event.'); + return; + } + // From here on we are really async + _this._getBackend().sendEvent(processedEvent); + resolve(processedEvent); + }) + .then(null, function (e) { + reject("beforeSend rejected with " + e); + }); + }; + return BaseClient; + }()); + + /** Noop transport */ + var NoopTransport = /** @class */ (function () { + function NoopTransport() { + } + /** + * @inheritDoc + */ + NoopTransport.prototype.sendEvent = function (_) { + return SyncPromise.resolve({ + reason: "NoopTransport: Event has been skipped because no Dsn is configured.", + status: exports.Status.Skipped, + }); + }; + /** + * @inheritDoc + */ + NoopTransport.prototype.close = function (_) { + return SyncPromise.resolve(true); + }; + return NoopTransport; + }()); + + /** + * This is the base implemention of a Backend. + * @hidden + */ + var BaseBackend = /** @class */ (function () { + /** Creates a new backend instance. */ + function BaseBackend(options) { + this._options = options; + if (!this._options.dsn) { + logger.warn('No DSN provided, backend will not do anything.'); + } + this._transport = this._setupTransport(); + } + /** + * Sets up the transport so it can be used later to send requests. + */ + BaseBackend.prototype._setupTransport = function () { + return new NoopTransport(); + }; + /** + * @inheritDoc + */ + BaseBackend.prototype.eventFromException = function (_exception, _hint) { + throw new SentryError('Backend has to implement `eventFromException` method'); + }; + /** + * @inheritDoc + */ + BaseBackend.prototype.eventFromMessage = function (_message, _level, _hint) { + throw new SentryError('Backend has to implement `eventFromMessage` method'); + }; + /** + * @inheritDoc + */ + BaseBackend.prototype.sendEvent = function (event) { + this._transport.sendEvent(event).then(null, function (reason) { + logger.error("Error while sending event: " + reason); + }); + }; + /** + * @inheritDoc + */ + BaseBackend.prototype.getTransport = function () { + return this._transport; + }; + return BaseBackend; + }()); + + /** + * Internal function to create a new SDK client instance. The client is + * installed and then bound to the current scope. + * + * @param clientClass The client class to instanciate. + * @param options Options to pass to the client. + */ + function initAndBind(clientClass, options) { + if (options.debug === true) { + logger.enable(); + } + var hub = getCurrentHub(); + var client = new clientClass(options); + hub.bindClient(client); + } + + var originalFunctionToString; + /** Patch toString calls to return proper name for wrapped functions */ + var FunctionToString = /** @class */ (function () { + function FunctionToString() { + /** + * @inheritDoc + */ + this.name = FunctionToString.id; + } + /** + * @inheritDoc + */ + FunctionToString.prototype.setupOnce = function () { + originalFunctionToString = Function.prototype.toString; + Function.prototype.toString = function () { + var args = []; + for (var _i = 0; _i < arguments.length; _i++) { + args[_i] = arguments[_i]; + } + var context = this.__sentry_original__ || this; + // tslint:disable-next-line:no-unsafe-any + return originalFunctionToString.apply(context, args); + }; + }; + /** + * @inheritDoc + */ + FunctionToString.id = 'FunctionToString'; + return FunctionToString; + }()); + + // "Script error." is hard coded into browsers for errors that it can't read. + // this is the result of a script being pulled in from an external domain and CORS. + var DEFAULT_IGNORE_ERRORS = [/^Script error\.?$/, /^Javascript error: Script error\.? on line 0$/]; + /** Inbound filters configurable by the user */ + var InboundFilters = /** @class */ (function () { + function InboundFilters(_options) { + if (_options === void 0) { _options = {}; } + this._options = _options; + /** + * @inheritDoc + */ + this.name = InboundFilters.id; + } + /** + * @inheritDoc + */ + InboundFilters.prototype.setupOnce = function () { + addGlobalEventProcessor(function (event) { + var hub = getCurrentHub(); + if (!hub) { + return event; + } + var self = hub.getIntegration(InboundFilters); + if (self) { + var client = hub.getClient(); + var clientOptions = client ? client.getOptions() : {}; + var options = self._mergeOptions(clientOptions); + if (self._shouldDropEvent(event, options)) { + return null; + } + } + return event; + }); + }; + /** JSDoc */ + InboundFilters.prototype._shouldDropEvent = function (event, options) { + if (this._isSentryError(event, options)) { + logger.warn("Event dropped due to being internal Sentry Error.\nEvent: " + getEventDescription(event)); + return true; + } + if (this._isIgnoredError(event, options)) { + logger.warn("Event dropped due to being matched by `ignoreErrors` option.\nEvent: " + getEventDescription(event)); + return true; + } + if (this._isBlacklistedUrl(event, options)) { + logger.warn("Event dropped due to being matched by `blacklistUrls` option.\nEvent: " + getEventDescription(event) + ".\nUrl: " + this._getEventFilterUrl(event)); + return true; + } + if (!this._isWhitelistedUrl(event, options)) { + logger.warn("Event dropped due to not being matched by `whitelistUrls` option.\nEvent: " + getEventDescription(event) + ".\nUrl: " + this._getEventFilterUrl(event)); + return true; + } + return false; + }; + /** JSDoc */ + InboundFilters.prototype._isSentryError = function (event, options) { + if (options === void 0) { options = {}; } + if (!options.ignoreInternal) { + return false; + } + try { + return ((event && + event.exception && + event.exception.values && + event.exception.values[0] && + event.exception.values[0].type === 'SentryError') || + false); + } + catch (_oO) { + return false; + } + }; + /** JSDoc */ + InboundFilters.prototype._isIgnoredError = function (event, options) { + if (options === void 0) { options = {}; } + if (!options.ignoreErrors || !options.ignoreErrors.length) { + return false; + } + return this._getPossibleEventMessages(event).some(function (message) { + // Not sure why TypeScript complains here... + return options.ignoreErrors.some(function (pattern) { return isMatchingPattern(message, pattern); }); + }); + }; + /** JSDoc */ + InboundFilters.prototype._isBlacklistedUrl = function (event, options) { + if (options === void 0) { options = {}; } + // TODO: Use Glob instead? + if (!options.blacklistUrls || !options.blacklistUrls.length) { + return false; + } + var url = this._getEventFilterUrl(event); + return !url ? false : options.blacklistUrls.some(function (pattern) { return isMatchingPattern(url, pattern); }); + }; + /** JSDoc */ + InboundFilters.prototype._isWhitelistedUrl = function (event, options) { + if (options === void 0) { options = {}; } + // TODO: Use Glob instead? + if (!options.whitelistUrls || !options.whitelistUrls.length) { + return true; + } + var url = this._getEventFilterUrl(event); + return !url ? true : options.whitelistUrls.some(function (pattern) { return isMatchingPattern(url, pattern); }); + }; + /** JSDoc */ + InboundFilters.prototype._mergeOptions = function (clientOptions) { + if (clientOptions === void 0) { clientOptions = {}; } + return { + blacklistUrls: __spread((this._options.blacklistUrls || []), (clientOptions.blacklistUrls || [])), + ignoreErrors: __spread((this._options.ignoreErrors || []), (clientOptions.ignoreErrors || []), DEFAULT_IGNORE_ERRORS), + ignoreInternal: typeof this._options.ignoreInternal !== 'undefined' ? this._options.ignoreInternal : true, + whitelistUrls: __spread((this._options.whitelistUrls || []), (clientOptions.whitelistUrls || [])), + }; + }; + /** JSDoc */ + InboundFilters.prototype._getPossibleEventMessages = function (event) { + if (event.message) { + return [event.message]; + } + if (event.exception) { + try { + var _a = (event.exception.values && event.exception.values[0]) || {}, _b = _a.type, type = _b === void 0 ? '' : _b, _c = _a.value, value = _c === void 0 ? '' : _c; + return ["" + value, type + ": " + value]; + } + catch (oO) { + logger.error("Cannot extract message for event " + getEventDescription(event)); + return []; + } + } + return []; + }; + /** JSDoc */ + InboundFilters.prototype._getEventFilterUrl = function (event) { + try { + if (event.stacktrace) { + var frames_1 = event.stacktrace.frames; + return (frames_1 && frames_1[frames_1.length - 1].filename) || null; + } + if (event.exception) { + var frames_2 = event.exception.values && event.exception.values[0].stacktrace && event.exception.values[0].stacktrace.frames; + return (frames_2 && frames_2[frames_2.length - 1].filename) || null; + } + return null; + } + catch (oO) { + logger.error("Cannot extract url for event " + getEventDescription(event)); + return null; + } + }; + /** + * @inheritDoc + */ + InboundFilters.id = 'InboundFilters'; + return InboundFilters; + }()); + + + + var CoreIntegrations = /*#__PURE__*/Object.freeze({ + FunctionToString: FunctionToString, + InboundFilters: InboundFilters + }); + + // tslint:disable:object-literal-sort-keys + // global reference to slice + var UNKNOWN_FUNCTION = '?'; + // Chromium based browsers: Chrome, Brave, new Opera, new Edge + var chrome = /^\s*at (?:(.*?) ?\()?((?:file|https?|blob|chrome-extension|address|native|eval|webpack||[-a-z]+:|.*bundle|\/).*?)(?::(\d+))?(?::(\d+))?\)?\s*$/i; + // gecko regex: `(?:bundle|\d+\.js)`: `bundle` is for react native, `\d+\.js` also but specifically for ram bundles because it + // generates filenames without a prefix like `file://` the filenames in the stacktrace are just 42.js + // We need this specific case for now because we want no other regex to match. + var gecko = /^\s*(.*?)(?:\((.*?)\))?(?:^|@)?((?:file|https?|blob|chrome|webpack|resource|moz-extension).*?:\/.*?|\[native code\]|[^@]*(?:bundle|\d+\.js))(?::(\d+))?(?::(\d+))?\s*$/i; + var winjs = /^\s*at (?:((?:\[object object\])?.+) )?\(?((?:file|ms-appx|https?|webpack|blob):.*?):(\d+)(?::(\d+))?\)?\s*$/i; + var geckoEval = /(\S+) line (\d+)(?: > eval line \d+)* > eval/i; + var chromeEval = /\((\S*)(?::(\d+))(?::(\d+))\)/; + /** JSDoc */ + function computeStackTrace(ex) { + // tslint:disable:no-unsafe-any + var stack = null; + var popSize = ex && ex.framesToPop; + try { + // This must be tried first because Opera 10 *destroys* + // its stacktrace property if you try to access the stack + // property first!! + stack = computeStackTraceFromStacktraceProp(ex); + if (stack) { + return popFrames(stack, popSize); + } + } + catch (e) { + // no-empty + } + try { + stack = computeStackTraceFromStackProp(ex); + if (stack) { + return popFrames(stack, popSize); + } + } + catch (e) { + // no-empty + } + return { + message: extractMessage(ex), + name: ex && ex.name, + stack: [], + failed: true, + }; + } + /** JSDoc */ + // tslint:disable-next-line:cyclomatic-complexity + function computeStackTraceFromStackProp(ex) { + // tslint:disable:no-conditional-assignment + if (!ex || !ex.stack) { + return null; + } + var stack = []; + var lines = ex.stack.split('\n'); + var isEval; + var submatch; + var parts; + var element; + for (var i = 0; i < lines.length; ++i) { + if ((parts = chrome.exec(lines[i]))) { + var isNative = parts[2] && parts[2].indexOf('native') === 0; // start of line + isEval = parts[2] && parts[2].indexOf('eval') === 0; // start of line + if (isEval && (submatch = chromeEval.exec(parts[2]))) { + // throw out eval line/column and use top-most line/column number + parts[2] = submatch[1]; // url + parts[3] = submatch[2]; // line + parts[4] = submatch[3]; // column + } + element = { + // working with the regexp above is super painful. it is quite a hack, but just stripping the `address at ` + // prefix here seems like the quickest solution for now. + url: parts[2] && parts[2].indexOf('address at ') === 0 ? parts[2].substr('address at '.length) : parts[2], + func: parts[1] || UNKNOWN_FUNCTION, + args: isNative ? [parts[2]] : [], + line: parts[3] ? +parts[3] : null, + column: parts[4] ? +parts[4] : null, + }; + } + else if ((parts = winjs.exec(lines[i]))) { + element = { + url: parts[2], + func: parts[1] || UNKNOWN_FUNCTION, + args: [], + line: +parts[3], + column: parts[4] ? +parts[4] : null, + }; + } + else if ((parts = gecko.exec(lines[i]))) { + isEval = parts[3] && parts[3].indexOf(' > eval') > -1; + if (isEval && (submatch = geckoEval.exec(parts[3]))) { + // throw out eval line/column and use top-most line number + parts[1] = parts[1] || "eval"; + parts[3] = submatch[1]; + parts[4] = submatch[2]; + parts[5] = ''; // no column when eval + } + else if (i === 0 && !parts[5] && ex.columnNumber !== void 0) { + // FireFox uses this awesome columnNumber property for its top frame + // Also note, Firefox's column number is 0-based and everything else expects 1-based, + // so adding 1 + // NOTE: this hack doesn't work if top-most frame is eval + stack[0].column = ex.columnNumber + 1; + } + element = { + url: parts[3], + func: parts[1] || UNKNOWN_FUNCTION, + args: parts[2] ? parts[2].split(',') : [], + line: parts[4] ? +parts[4] : null, + column: parts[5] ? +parts[5] : null, + }; + } + else { + continue; + } + if (!element.func && element.line) { + element.func = UNKNOWN_FUNCTION; + } + stack.push(element); + } + if (!stack.length) { + return null; + } + return { + message: extractMessage(ex), + name: ex.name, + stack: stack, + }; + } + /** JSDoc */ + function computeStackTraceFromStacktraceProp(ex) { + if (!ex || !ex.stacktrace) { + return null; + } + // Access and store the stacktrace property before doing ANYTHING + // else to it because Opera is not very good at providing it + // reliably in other circumstances. + var stacktrace = ex.stacktrace; + var opera10Regex = / line (\d+).*script (?:in )?(\S+)(?:: in function (\S+))?$/i; + var opera11Regex = / line (\d+), column (\d+)\s*(?:in (?:]+)>|([^\)]+))\((.*)\))? in (.*):\s*$/i; + var lines = stacktrace.split('\n'); + var stack = []; + var parts; + for (var line = 0; line < lines.length; line += 2) { + // tslint:disable:no-conditional-assignment + var element = null; + if ((parts = opera10Regex.exec(lines[line]))) { + element = { + url: parts[2], + func: parts[3], + args: [], + line: +parts[1], + column: null, + }; + } + else if ((parts = opera11Regex.exec(lines[line]))) { + element = { + url: parts[6], + func: parts[3] || parts[4], + args: parts[5] ? parts[5].split(',') : [], + line: +parts[1], + column: +parts[2], + }; + } + if (element) { + if (!element.func && element.line) { + element.func = UNKNOWN_FUNCTION; + } + stack.push(element); + } + } + if (!stack.length) { + return null; + } + return { + message: extractMessage(ex), + name: ex.name, + stack: stack, + }; + } + /** Remove N number of frames from the stack */ + function popFrames(stacktrace, popSize) { + try { + return __assign({}, stacktrace, { stack: stacktrace.stack.slice(popSize) }); + } + catch (e) { + return stacktrace; + } + } + /** + * There are cases where stacktrace.message is an Event object + * https://github.com/getsentry/sentry-javascript/issues/1949 + * In this specific case we try to extract stacktrace.message.error.message + */ + function extractMessage(ex) { + var message = ex && ex.message; + if (!message) { + return 'No error message'; + } + if (message.error && typeof message.error.message === 'string') { + return message.error.message; + } + return message; + } + + var STACKTRACE_LIMIT = 50; + /** + * This function creates an exception from an TraceKitStackTrace + * @param stacktrace TraceKitStackTrace that will be converted to an exception + * @hidden + */ + function exceptionFromStacktrace(stacktrace) { + var frames = prepareFramesForEvent(stacktrace.stack); + var exception = { + type: stacktrace.name, + value: stacktrace.message, + }; + if (frames && frames.length) { + exception.stacktrace = { frames: frames }; + } + // tslint:disable-next-line:strict-type-predicates + if (exception.type === undefined && exception.value === '') { + exception.value = 'Unrecoverable error caught'; + } + return exception; + } + /** + * @hidden + */ + function eventFromPlainObject(exception, syntheticException, rejection) { + var event = { + exception: { + values: [ + { + type: isEvent(exception) ? exception.constructor.name : rejection ? 'UnhandledRejection' : 'Error', + value: "Non-Error " + (rejection ? 'promise rejection' : 'exception') + " captured with keys: " + extractExceptionKeysForMessage(exception), + }, + ], + }, + extra: { + __serialized__: normalizeToSize(exception), + }, + }; + if (syntheticException) { + var stacktrace = computeStackTrace(syntheticException); + var frames_1 = prepareFramesForEvent(stacktrace.stack); + event.stacktrace = { + frames: frames_1, + }; + } + return event; + } + /** + * @hidden + */ + function eventFromStacktrace(stacktrace) { + var exception = exceptionFromStacktrace(stacktrace); + return { + exception: { + values: [exception], + }, + }; + } + /** + * @hidden + */ + function prepareFramesForEvent(stack) { + if (!stack || !stack.length) { + return []; + } + var localStack = stack; + var firstFrameFunction = localStack[0].func || ''; + var lastFrameFunction = localStack[localStack.length - 1].func || ''; + // If stack starts with one of our API calls, remove it (starts, meaning it's the top of the stack - aka last call) + if (firstFrameFunction.indexOf('captureMessage') !== -1 || firstFrameFunction.indexOf('captureException') !== -1) { + localStack = localStack.slice(1); + } + // If stack ends with one of our internal API calls, remove it (ends, meaning it's the bottom of the stack - aka top-most call) + if (lastFrameFunction.indexOf('sentryWrapped') !== -1) { + localStack = localStack.slice(0, -1); + } + // The frame where the crash happened, should be the last entry in the array + return localStack + .map(function (frame) { return ({ + colno: frame.column === null ? undefined : frame.column, + filename: frame.url || localStack[0].url, + function: frame.func || '?', + in_app: true, + lineno: frame.line === null ? undefined : frame.line, + }); }) + .slice(0, STACKTRACE_LIMIT) + .reverse(); + } + + /** JSDoc */ + function eventFromUnknownInput(exception, syntheticException, options) { + if (options === void 0) { options = {}; } + var event; + if (isErrorEvent(exception) && exception.error) { + // If it is an ErrorEvent with `error` property, extract it to get actual Error + var errorEvent = exception; + exception = errorEvent.error; // tslint:disable-line:no-parameter-reassignment + event = eventFromStacktrace(computeStackTrace(exception)); + return event; + } + if (isDOMError(exception) || isDOMException(exception)) { + // If it is a DOMError or DOMException (which are legacy APIs, but still supported in some browsers) + // then we just extract the name and message, as they don't provide anything else + // https://developer.mozilla.org/en-US/docs/Web/API/DOMError + // https://developer.mozilla.org/en-US/docs/Web/API/DOMException + var domException = exception; + var name_1 = domException.name || (isDOMError(domException) ? 'DOMError' : 'DOMException'); + var message = domException.message ? name_1 + ": " + domException.message : name_1; + event = eventFromString(message, syntheticException, options); + addExceptionTypeValue(event, message); + return event; + } + if (isError(exception)) { + // we have a real Error object, do nothing + event = eventFromStacktrace(computeStackTrace(exception)); + return event; + } + if (isPlainObject(exception) || isEvent(exception)) { + // If it is plain Object or Event, serialize it manually and extract options + // This will allow us to group events based on top-level keys + // which is much better than creating new group when any key/value change + var objectException = exception; + event = eventFromPlainObject(objectException, syntheticException, options.rejection); + addExceptionMechanism(event, { + synthetic: true, + }); + return event; + } + // If none of previous checks were valid, then it means that it's not: + // - an instance of DOMError + // - an instance of DOMException + // - an instance of Event + // - an instance of Error + // - a valid ErrorEvent (one with an error property) + // - a plain Object + // + // So bail out and capture it as a simple message: + event = eventFromString(exception, syntheticException, options); + addExceptionTypeValue(event, "" + exception, undefined); + addExceptionMechanism(event, { + synthetic: true, + }); + return event; + } + // this._options.attachStacktrace + /** JSDoc */ + function eventFromString(input, syntheticException, options) { + if (options === void 0) { options = {}; } + var event = { + message: input, + }; + if (options.attachStacktrace && syntheticException) { + var stacktrace = computeStackTrace(syntheticException); + var frames_1 = prepareFramesForEvent(stacktrace.stack); + event.stacktrace = { + frames: frames_1, + }; + } + return event; + } + + /** Base Transport class implementation */ + var BaseTransport = /** @class */ (function () { + function BaseTransport(options) { + this.options = options; + /** A simple buffer holding all requests. */ + this._buffer = new PromiseBuffer(30); + this.url = new API(this.options.dsn).getStoreEndpointWithUrlEncodedAuth(); + } + /** + * @inheritDoc + */ + BaseTransport.prototype.sendEvent = function (_) { + throw new SentryError('Transport Class has to implement `sendEvent` method'); + }; + /** + * @inheritDoc + */ + BaseTransport.prototype.close = function (timeout) { + return this._buffer.drain(timeout); + }; + return BaseTransport; + }()); + + var global$3 = getGlobalObject(); + /** `fetch` based transport */ + var FetchTransport = /** @class */ (function (_super) { + __extends(FetchTransport, _super); + function FetchTransport() { + var _this = _super !== null && _super.apply(this, arguments) || this; + /** Locks transport after receiving 429 response */ + _this._disabledUntil = new Date(Date.now()); + return _this; + } + /** + * @inheritDoc + */ + FetchTransport.prototype.sendEvent = function (event) { + var _this = this; + if (new Date(Date.now()) < this._disabledUntil) { + return Promise.reject({ + event: event, + reason: "Transport locked till " + this._disabledUntil + " due to too many requests.", + status: 429, + }); + } + var defaultOptions = { + body: JSON.stringify(event), + method: 'POST', + // Despite all stars in the sky saying that Edge supports old draft syntax, aka 'never', 'always', 'origin' and 'default + // https://caniuse.com/#feat=referrer-policy + // It doesn't. And it throw exception instead of ignoring this parameter... + // REF: https://github.com/getsentry/raven-js/issues/1233 + referrerPolicy: (supportsReferrerPolicy() ? 'origin' : ''), + }; + if (this.options.headers !== undefined) { + defaultOptions.headers = this.options.headers; + } + return this._buffer.add(new SyncPromise(function (resolve, reject) { + global$3 + .fetch(_this.url, defaultOptions) + .then(function (response) { + var status = exports.Status.fromHttpCode(response.status); + if (status === exports.Status.Success) { + resolve({ status: status }); + return; + } + if (status === exports.Status.RateLimit) { + var now = Date.now(); + _this._disabledUntil = new Date(now + parseRetryAfterHeader(now, response.headers.get('Retry-After'))); + logger.warn("Too many requests, backing off till: " + _this._disabledUntil); + } + reject(response); + }) + .catch(reject); + })); + }; + return FetchTransport; + }(BaseTransport)); + + /** `XHR` based transport */ + var XHRTransport = /** @class */ (function (_super) { + __extends(XHRTransport, _super); + function XHRTransport() { + var _this = _super !== null && _super.apply(this, arguments) || this; + /** Locks transport after receiving 429 response */ + _this._disabledUntil = new Date(Date.now()); + return _this; + } + /** + * @inheritDoc + */ + XHRTransport.prototype.sendEvent = function (event) { + var _this = this; + if (new Date(Date.now()) < this._disabledUntil) { + return Promise.reject({ + event: event, + reason: "Transport locked till " + this._disabledUntil + " due to too many requests.", + status: 429, + }); + } + return this._buffer.add(new SyncPromise(function (resolve, reject) { + var request = new XMLHttpRequest(); + request.onreadystatechange = function () { + if (request.readyState !== 4) { + return; + } + var status = exports.Status.fromHttpCode(request.status); + if (status === exports.Status.Success) { + resolve({ status: status }); + return; + } + if (status === exports.Status.RateLimit) { + var now = Date.now(); + _this._disabledUntil = new Date(now + parseRetryAfterHeader(now, request.getResponseHeader('Retry-After'))); + logger.warn("Too many requests, backing off till: " + _this._disabledUntil); + } + reject(request); + }; + request.open('POST', _this.url); + for (var header in _this.options.headers) { + if (_this.options.headers.hasOwnProperty(header)) { + request.setRequestHeader(header, _this.options.headers[header]); + } + } + request.send(JSON.stringify(event)); + })); + }; + return XHRTransport; + }(BaseTransport)); + + + + var index = /*#__PURE__*/Object.freeze({ + BaseTransport: BaseTransport, + FetchTransport: FetchTransport, + XHRTransport: XHRTransport + }); + + /** + * The Sentry Browser SDK Backend. + * @hidden + */ + var BrowserBackend = /** @class */ (function (_super) { + __extends(BrowserBackend, _super); + function BrowserBackend() { + return _super !== null && _super.apply(this, arguments) || this; + } + /** + * @inheritDoc + */ + BrowserBackend.prototype._setupTransport = function () { + if (!this._options.dsn) { + // We return the noop transport here in case there is no Dsn. + return _super.prototype._setupTransport.call(this); + } + var transportOptions = __assign({}, this._options.transportOptions, { dsn: this._options.dsn }); + if (this._options.transport) { + return new this._options.transport(transportOptions); + } + if (supportsFetch()) { + return new FetchTransport(transportOptions); + } + return new XHRTransport(transportOptions); + }; + /** + * @inheritDoc + */ + BrowserBackend.prototype.eventFromException = function (exception, hint) { + var syntheticException = (hint && hint.syntheticException) || undefined; + var event = eventFromUnknownInput(exception, syntheticException, { + attachStacktrace: this._options.attachStacktrace, + }); + addExceptionMechanism(event, { + handled: true, + type: 'generic', + }); + event.level = exports.Severity.Error; + if (hint && hint.event_id) { + event.event_id = hint.event_id; + } + return SyncPromise.resolve(event); + }; + /** + * @inheritDoc + */ + BrowserBackend.prototype.eventFromMessage = function (message, level, hint) { + if (level === void 0) { level = exports.Severity.Info; } + var syntheticException = (hint && hint.syntheticException) || undefined; + var event = eventFromString(message, syntheticException, { + attachStacktrace: this._options.attachStacktrace, + }); + event.level = level; + if (hint && hint.event_id) { + event.event_id = hint.event_id; + } + return SyncPromise.resolve(event); + }; + return BrowserBackend; + }(BaseBackend)); + + var SDK_NAME = 'sentry.javascript.browser'; + var SDK_VERSION = '5.15.5'; + + /** + * The Sentry Browser SDK Client. + * + * @see BrowserOptions for documentation on configuration options. + * @see SentryClient for usage documentation. + */ + var BrowserClient = /** @class */ (function (_super) { + __extends(BrowserClient, _super); + /** + * Creates a new Browser SDK instance. + * + * @param options Configuration options for this SDK. + */ + function BrowserClient(options) { + if (options === void 0) { options = {}; } + return _super.call(this, BrowserBackend, options) || this; + } + /** + * @inheritDoc + */ + BrowserClient.prototype._prepareEvent = function (event, scope, hint) { + event.platform = event.platform || 'javascript'; + event.sdk = __assign({}, event.sdk, { name: SDK_NAME, packages: __spread(((event.sdk && event.sdk.packages) || []), [ + { + name: 'npm:@sentry/browser', + version: SDK_VERSION, + }, + ]), version: SDK_VERSION }); + return _super.prototype._prepareEvent.call(this, event, scope, hint); + }; + /** + * Show a report dialog to the user to send feedback to a specific event. + * + * @param options Set individual options for the dialog + */ + BrowserClient.prototype.showReportDialog = function (options) { + if (options === void 0) { options = {}; } + // doesn't work without a document (React Native) + var document = getGlobalObject().document; + if (!document) { + return; + } + if (!this._isEnabled()) { + logger.error('Trying to call showReportDialog with Sentry Client is disabled'); + return; + } + var dsn = options.dsn || this.getDsn(); + if (!options.eventId) { + logger.error('Missing `eventId` option in showReportDialog call'); + return; + } + if (!dsn) { + logger.error('Missing `Dsn` option in showReportDialog call'); + return; + } + var script = document.createElement('script'); + script.async = true; + script.src = new API(dsn).getReportDialogEndpoint(options); + if (options.onLoad) { + script.onload = options.onLoad; + } + (document.head || document.body).appendChild(script); + }; + return BrowserClient; + }(BaseClient)); + + var ignoreOnError = 0; + /** + * @hidden + */ + function shouldIgnoreOnError() { + return ignoreOnError > 0; + } + /** + * @hidden + */ + function ignoreNextOnError() { + // onerror should trigger before setTimeout + ignoreOnError += 1; + setTimeout(function () { + ignoreOnError -= 1; + }); + } + /** + * Instruments the given function and sends an event to Sentry every time the + * function throws an exception. + * + * @param fn A function to wrap. + * @returns The wrapped function. + * @hidden + */ + function wrap(fn, options, before) { + if (options === void 0) { options = {}; } + // tslint:disable-next-line:strict-type-predicates + if (typeof fn !== 'function') { + return fn; + } + try { + // We don't wanna wrap it twice + if (fn.__sentry__) { + return fn; + } + // If this has already been wrapped in the past, return that wrapped function + if (fn.__sentry_wrapped__) { + return fn.__sentry_wrapped__; + } + } + catch (e) { + // Just accessing custom props in some Selenium environments + // can cause a "Permission denied" exception (see raven-js#495). + // Bail on wrapping and return the function as-is (defers to window.onerror). + return fn; + } + var sentryWrapped = function () { + var args = Array.prototype.slice.call(arguments); + // tslint:disable:no-unsafe-any + try { + // tslint:disable-next-line:strict-type-predicates + if (before && typeof before === 'function') { + before.apply(this, arguments); + } + var wrappedArguments = args.map(function (arg) { return wrap(arg, options); }); + if (fn.handleEvent) { + // Attempt to invoke user-land function + // NOTE: If you are a Sentry user, and you are seeing this stack frame, it + // means the sentry.javascript SDK caught an error invoking your application code. This + // is expected behavior and NOT indicative of a bug with sentry.javascript. + return fn.handleEvent.apply(this, wrappedArguments); + } + // Attempt to invoke user-land function + // NOTE: If you are a Sentry user, and you are seeing this stack frame, it + // means the sentry.javascript SDK caught an error invoking your application code. This + // is expected behavior and NOT indicative of a bug with sentry.javascript. + return fn.apply(this, wrappedArguments); + // tslint:enable:no-unsafe-any + } + catch (ex) { + ignoreNextOnError(); + withScope(function (scope) { + scope.addEventProcessor(function (event) { + var processedEvent = __assign({}, event); + if (options.mechanism) { + addExceptionTypeValue(processedEvent, undefined, undefined); + addExceptionMechanism(processedEvent, options.mechanism); + } + processedEvent.extra = __assign({}, processedEvent.extra, { arguments: args }); + return processedEvent; + }); + captureException(ex); + }); + throw ex; + } + }; + // Accessing some objects may throw + // ref: https://github.com/getsentry/sentry-javascript/issues/1168 + try { + for (var property in fn) { + if (Object.prototype.hasOwnProperty.call(fn, property)) { + sentryWrapped[property] = fn[property]; + } + } + } + catch (_oO) { } // tslint:disable-line:no-empty + fn.prototype = fn.prototype || {}; + sentryWrapped.prototype = fn.prototype; + Object.defineProperty(fn, '__sentry_wrapped__', { + enumerable: false, + value: sentryWrapped, + }); + // Signal that this function has been wrapped/filled already + // for both debugging and to prevent it to being wrapped/filled twice + Object.defineProperties(sentryWrapped, { + __sentry__: { + enumerable: false, + value: true, + }, + __sentry_original__: { + enumerable: false, + value: fn, + }, + }); + // Restore original function name (not all browsers allow that) + try { + var descriptor = Object.getOwnPropertyDescriptor(sentryWrapped, 'name'); + if (descriptor.configurable) { + Object.defineProperty(sentryWrapped, 'name', { + get: function () { + return fn.name; + }, + }); + } + } + catch (_oO) { + /*no-empty*/ + } + return sentryWrapped; + } + + /** Global handlers */ + var GlobalHandlers = /** @class */ (function () { + /** JSDoc */ + function GlobalHandlers(options) { + /** + * @inheritDoc + */ + this.name = GlobalHandlers.id; + /** JSDoc */ + this._onErrorHandlerInstalled = false; + /** JSDoc */ + this._onUnhandledRejectionHandlerInstalled = false; + this._options = __assign({ onerror: true, onunhandledrejection: true }, options); + } + /** + * @inheritDoc + */ + GlobalHandlers.prototype.setupOnce = function () { + Error.stackTraceLimit = 50; + if (this._options.onerror) { + logger.log('Global Handler attached: onerror'); + this._installGlobalOnErrorHandler(); + } + if (this._options.onunhandledrejection) { + logger.log('Global Handler attached: onunhandledrejection'); + this._installGlobalOnUnhandledRejectionHandler(); + } + }; + /** JSDoc */ + GlobalHandlers.prototype._installGlobalOnErrorHandler = function () { + var _this = this; + if (this._onErrorHandlerInstalled) { + return; + } + addInstrumentationHandler({ + callback: function (data) { + var error = data.error; + var currentHub = getCurrentHub(); + var hasIntegration = currentHub.getIntegration(GlobalHandlers); + var isFailedOwnDelivery = error && error.__sentry_own_request__ === true; + if (!hasIntegration || shouldIgnoreOnError() || isFailedOwnDelivery) { + return; + } + var client = currentHub.getClient(); + var event = isPrimitive(error) + ? _this._eventFromIncompleteOnError(data.msg, data.url, data.line, data.column) + : _this._enhanceEventWithInitialFrame(eventFromUnknownInput(error, undefined, { + attachStacktrace: client && client.getOptions().attachStacktrace, + rejection: false, + }), data.url, data.line, data.column); + addExceptionMechanism(event, { + handled: false, + type: 'onerror', + }); + currentHub.captureEvent(event, { + originalException: error, + }); + }, + type: 'error', + }); + this._onErrorHandlerInstalled = true; + }; + /** JSDoc */ + GlobalHandlers.prototype._installGlobalOnUnhandledRejectionHandler = function () { + var _this = this; + if (this._onUnhandledRejectionHandlerInstalled) { + return; + } + addInstrumentationHandler({ + callback: function (e) { + var error = e; + // dig the object of the rejection out of known event types + try { + // PromiseRejectionEvents store the object of the rejection under 'reason' + // see https://developer.mozilla.org/en-US/docs/Web/API/PromiseRejectionEvent + if ('reason' in e) { + error = e.reason; + } + // something, somewhere, (likely a browser extension) effectively casts PromiseRejectionEvents + // to CustomEvents, moving the `promise` and `reason` attributes of the PRE into + // the CustomEvent's `detail` attribute, since they're not part of CustomEvent's spec + // see https://developer.mozilla.org/en-US/docs/Web/API/CustomEvent and + // https://github.com/getsentry/sentry-javascript/issues/2380 + else if ('detail' in e && 'reason' in e.detail) { + error = e.detail.reason; + } + } + catch (_oO) { + // no-empty + } + var currentHub = getCurrentHub(); + var hasIntegration = currentHub.getIntegration(GlobalHandlers); + var isFailedOwnDelivery = error && error.__sentry_own_request__ === true; + if (!hasIntegration || shouldIgnoreOnError() || isFailedOwnDelivery) { + return true; + } + var client = currentHub.getClient(); + var event = isPrimitive(error) + ? _this._eventFromIncompleteRejection(error) + : eventFromUnknownInput(error, undefined, { + attachStacktrace: client && client.getOptions().attachStacktrace, + rejection: true, + }); + event.level = exports.Severity.Error; + addExceptionMechanism(event, { + handled: false, + type: 'onunhandledrejection', + }); + currentHub.captureEvent(event, { + originalException: error, + }); + return; + }, + type: 'unhandledrejection', + }); + this._onUnhandledRejectionHandlerInstalled = true; + }; + /** + * This function creates a stack from an old, error-less onerror handler. + */ + GlobalHandlers.prototype._eventFromIncompleteOnError = function (msg, url, line, column) { + var ERROR_TYPES_RE = /^(?:[Uu]ncaught (?:exception: )?)?(?:((?:Eval|Internal|Range|Reference|Syntax|Type|URI|)Error): )?(.*)$/i; + // If 'message' is ErrorEvent, get real message from inside + var message = isErrorEvent(msg) ? msg.message : msg; + var name; + if (isString(message)) { + var groups = message.match(ERROR_TYPES_RE); + if (groups) { + name = groups[1]; + message = groups[2]; + } + } + var event = { + exception: { + values: [ + { + type: name || 'Error', + value: message, + }, + ], + }, + }; + return this._enhanceEventWithInitialFrame(event, url, line, column); + }; + /** + * This function creates an Event from an TraceKitStackTrace that has part of it missing. + */ + GlobalHandlers.prototype._eventFromIncompleteRejection = function (error) { + return { + exception: { + values: [ + { + type: 'UnhandledRejection', + value: "Non-Error promise rejection captured with value: " + error, + }, + ], + }, + }; + }; + /** JSDoc */ + GlobalHandlers.prototype._enhanceEventWithInitialFrame = function (event, url, line, column) { + event.exception = event.exception || {}; + event.exception.values = event.exception.values || []; + event.exception.values[0] = event.exception.values[0] || {}; + event.exception.values[0].stacktrace = event.exception.values[0].stacktrace || {}; + event.exception.values[0].stacktrace.frames = event.exception.values[0].stacktrace.frames || []; + var colno = isNaN(parseInt(column, 10)) ? undefined : column; + var lineno = isNaN(parseInt(line, 10)) ? undefined : line; + var filename = isString(url) && url.length > 0 ? url : getLocationHref(); + if (event.exception.values[0].stacktrace.frames.length === 0) { + event.exception.values[0].stacktrace.frames.push({ + colno: colno, + filename: filename, + function: '?', + in_app: true, + lineno: lineno, + }); + } + return event; + }; + /** + * @inheritDoc + */ + GlobalHandlers.id = 'GlobalHandlers'; + return GlobalHandlers; + }()); + + /** Wrap timer functions and event targets to catch errors and provide better meta data */ + var TryCatch = /** @class */ (function () { + function TryCatch() { + /** JSDoc */ + this._ignoreOnError = 0; + /** + * @inheritDoc + */ + this.name = TryCatch.id; + } + /** JSDoc */ + TryCatch.prototype._wrapTimeFunction = function (original) { + return function () { + var args = []; + for (var _i = 0; _i < arguments.length; _i++) { + args[_i] = arguments[_i]; + } + var originalCallback = args[0]; + args[0] = wrap(originalCallback, { + mechanism: { + data: { function: getFunctionName(original) }, + handled: true, + type: 'instrument', + }, + }); + return original.apply(this, args); + }; + }; + /** JSDoc */ + TryCatch.prototype._wrapRAF = function (original) { + return function (callback) { + return original(wrap(callback, { + mechanism: { + data: { + function: 'requestAnimationFrame', + handler: getFunctionName(original), + }, + handled: true, + type: 'instrument', + }, + })); + }; + }; + /** JSDoc */ + TryCatch.prototype._wrapEventTarget = function (target) { + var global = getGlobalObject(); + var proto = global[target] && global[target].prototype; + if (!proto || !proto.hasOwnProperty || !proto.hasOwnProperty('addEventListener')) { + return; + } + fill(proto, 'addEventListener', function (original) { + return function (eventName, fn, options) { + try { + // tslint:disable-next-line:no-unbound-method strict-type-predicates + if (typeof fn.handleEvent === 'function') { + fn.handleEvent = wrap(fn.handleEvent.bind(fn), { + mechanism: { + data: { + function: 'handleEvent', + handler: getFunctionName(fn), + target: target, + }, + handled: true, + type: 'instrument', + }, + }); + } + } + catch (err) { + // can sometimes get 'Permission denied to access property "handle Event' + } + return original.call(this, eventName, wrap(fn, { + mechanism: { + data: { + function: 'addEventListener', + handler: getFunctionName(fn), + target: target, + }, + handled: true, + type: 'instrument', + }, + }), options); + }; + }); + fill(proto, 'removeEventListener', function (original) { + return function (eventName, fn, options) { + var callback = fn; + try { + callback = callback && (callback.__sentry_wrapped__ || callback); + } + catch (e) { + // ignore, accessing __sentry_wrapped__ will throw in some Selenium environments + } + return original.call(this, eventName, callback, options); + }; + }); + }; + /** JSDoc */ + TryCatch.prototype._wrapXHR = function (originalSend) { + return function () { + var args = []; + for (var _i = 0; _i < arguments.length; _i++) { + args[_i] = arguments[_i]; + } + var xhr = this; // tslint:disable-line:no-this-assignment + var xmlHttpRequestProps = ['onload', 'onerror', 'onprogress', 'onreadystatechange']; + xmlHttpRequestProps.forEach(function (prop) { + if (prop in xhr && typeof xhr[prop] === 'function') { + fill(xhr, prop, function (original) { + var wrapOptions = { + mechanism: { + data: { + function: prop, + handler: getFunctionName(original), + }, + handled: true, + type: 'instrument', + }, + }; + // If Instrument integration has been called before TryCatch, get the name of original function + if (original.__sentry_original__) { + wrapOptions.mechanism.data.handler = getFunctionName(original.__sentry_original__); + } + // Otherwise wrap directly + return wrap(original, wrapOptions); + }); + } + }); + return originalSend.apply(this, args); + }; + }; + /** + * Wrap timer functions and event targets to catch errors + * and provide better metadata. + */ + TryCatch.prototype.setupOnce = function () { + this._ignoreOnError = this._ignoreOnError; + var global = getGlobalObject(); + fill(global, 'setTimeout', this._wrapTimeFunction.bind(this)); + fill(global, 'setInterval', this._wrapTimeFunction.bind(this)); + fill(global, 'requestAnimationFrame', this._wrapRAF.bind(this)); + if ('XMLHttpRequest' in global) { + fill(XMLHttpRequest.prototype, 'send', this._wrapXHR.bind(this)); + } + [ + 'EventTarget', + 'Window', + 'Node', + 'ApplicationCache', + 'AudioTrackList', + 'ChannelMergerNode', + 'CryptoOperation', + 'EventSource', + 'FileReader', + 'HTMLUnknownElement', + 'IDBDatabase', + 'IDBRequest', + 'IDBTransaction', + 'KeyOperation', + 'MediaController', + 'MessagePort', + 'ModalWindow', + 'Notification', + 'SVGElementInstance', + 'Screen', + 'TextTrack', + 'TextTrackCue', + 'TextTrackList', + 'WebSocket', + 'WebSocketWorker', + 'Worker', + 'XMLHttpRequest', + 'XMLHttpRequestEventTarget', + 'XMLHttpRequestUpload', + ].forEach(this._wrapEventTarget.bind(this)); + }; + /** + * @inheritDoc + */ + TryCatch.id = 'TryCatch'; + return TryCatch; + }()); + + /** + * Default Breadcrumbs instrumentations + * TODO: Deprecated - with v6, this will be renamed to `Instrument` + */ + var Breadcrumbs = /** @class */ (function () { + /** + * @inheritDoc + */ + function Breadcrumbs(options) { + /** + * @inheritDoc + */ + this.name = Breadcrumbs.id; + this._options = __assign({ console: true, dom: true, fetch: true, history: true, sentry: true, xhr: true }, options); + } + /** + * Creates breadcrumbs from console API calls + */ + Breadcrumbs.prototype._consoleBreadcrumb = function (handlerData) { + var breadcrumb = { + category: 'console', + data: { + arguments: handlerData.args, + logger: 'console', + }, + level: exports.Severity.fromString(handlerData.level), + message: safeJoin(handlerData.args, ' '), + }; + if (handlerData.level === 'assert') { + if (handlerData.args[0] === false) { + breadcrumb.message = "Assertion failed: " + (safeJoin(handlerData.args.slice(1), ' ') || 'console.assert'); + breadcrumb.data.arguments = handlerData.args.slice(1); + } + else { + // Don't capture a breadcrumb for passed assertions + return; + } + } + getCurrentHub().addBreadcrumb(breadcrumb, { + input: handlerData.args, + level: handlerData.level, + }); + }; + /** + * Creates breadcrumbs from DOM API calls + */ + Breadcrumbs.prototype._domBreadcrumb = function (handlerData) { + var target; + // Accessing event.target can throw (see getsentry/raven-js#838, #768) + try { + target = handlerData.event.target + ? htmlTreeAsString(handlerData.event.target) + : htmlTreeAsString(handlerData.event); + } + catch (e) { + target = ''; + } + if (target.length === 0) { + return; + } + getCurrentHub().addBreadcrumb({ + category: "ui." + handlerData.name, + message: target, + }, { + event: handlerData.event, + name: handlerData.name, + }); + }; + /** + * Creates breadcrumbs from XHR API calls + */ + Breadcrumbs.prototype._xhrBreadcrumb = function (handlerData) { + if (handlerData.endTimestamp) { + // We only capture complete, non-sentry requests + if (handlerData.xhr.__sentry_own_request__) { + return; + } + getCurrentHub().addBreadcrumb({ + category: 'xhr', + data: handlerData.xhr.__sentry_xhr__, + type: 'http', + }, { + xhr: handlerData.xhr, + }); + return; + } + // We only capture issued sentry requests + if (this._options.sentry && handlerData.xhr.__sentry_own_request__) { + addSentryBreadcrumb(handlerData.args[0]); + } + }; + /** + * Creates breadcrumbs from fetch API calls + */ + Breadcrumbs.prototype._fetchBreadcrumb = function (handlerData) { + // We only capture complete fetch requests + if (!handlerData.endTimestamp) { + return; + } + var client = getCurrentHub().getClient(); + var dsn = client && client.getDsn(); + if (this._options.sentry && dsn) { + var filterUrl = new API(dsn).getStoreEndpoint(); + // if Sentry key appears in URL, don't capture it as a request + // but rather as our own 'sentry' type breadcrumb + if (filterUrl && + handlerData.fetchData.url.indexOf(filterUrl) !== -1 && + handlerData.fetchData.method === 'POST' && + handlerData.args[1] && + handlerData.args[1].body) { + addSentryBreadcrumb(handlerData.args[1].body); + return; + } + } + if (handlerData.error) { + getCurrentHub().addBreadcrumb({ + category: 'fetch', + data: __assign({}, handlerData.fetchData, { status_code: handlerData.response.status }), + level: exports.Severity.Error, + type: 'http', + }, { + data: handlerData.error, + input: handlerData.args, + }); + } + else { + getCurrentHub().addBreadcrumb({ + category: 'fetch', + data: __assign({}, handlerData.fetchData, { status_code: handlerData.response.status }), + type: 'http', + }, { + input: handlerData.args, + response: handlerData.response, + }); + } + }; + /** + * Creates breadcrumbs from history API calls + */ + Breadcrumbs.prototype._historyBreadcrumb = function (handlerData) { + var global = getGlobalObject(); + var from = handlerData.from; + var to = handlerData.to; + var parsedLoc = parseUrl(global.location.href); + var parsedFrom = parseUrl(from); + var parsedTo = parseUrl(to); + // Initial pushState doesn't provide `from` information + if (!parsedFrom.path) { + parsedFrom = parsedLoc; + } + // Use only the path component of the URL if the URL matches the current + // document (almost all the time when using pushState) + if (parsedLoc.protocol === parsedTo.protocol && parsedLoc.host === parsedTo.host) { + // tslint:disable-next-line:no-parameter-reassignment + to = parsedTo.relative; + } + if (parsedLoc.protocol === parsedFrom.protocol && parsedLoc.host === parsedFrom.host) { + // tslint:disable-next-line:no-parameter-reassignment + from = parsedFrom.relative; + } + getCurrentHub().addBreadcrumb({ + category: 'navigation', + data: { + from: from, + to: to, + }, + }); + }; + /** + * Instrument browser built-ins w/ breadcrumb capturing + * - Console API + * - DOM API (click/typing) + * - XMLHttpRequest API + * - Fetch API + * - History API + */ + Breadcrumbs.prototype.setupOnce = function () { + var _this = this; + if (this._options.console) { + addInstrumentationHandler({ + callback: function () { + var args = []; + for (var _i = 0; _i < arguments.length; _i++) { + args[_i] = arguments[_i]; + } + _this._consoleBreadcrumb.apply(_this, __spread(args)); + }, + type: 'console', + }); + } + if (this._options.dom) { + addInstrumentationHandler({ + callback: function () { + var args = []; + for (var _i = 0; _i < arguments.length; _i++) { + args[_i] = arguments[_i]; + } + _this._domBreadcrumb.apply(_this, __spread(args)); + }, + type: 'dom', + }); + } + if (this._options.xhr) { + addInstrumentationHandler({ + callback: function () { + var args = []; + for (var _i = 0; _i < arguments.length; _i++) { + args[_i] = arguments[_i]; + } + _this._xhrBreadcrumb.apply(_this, __spread(args)); + }, + type: 'xhr', + }); + } + if (this._options.fetch) { + addInstrumentationHandler({ + callback: function () { + var args = []; + for (var _i = 0; _i < arguments.length; _i++) { + args[_i] = arguments[_i]; + } + _this._fetchBreadcrumb.apply(_this, __spread(args)); + }, + type: 'fetch', + }); + } + if (this._options.history) { + addInstrumentationHandler({ + callback: function () { + var args = []; + for (var _i = 0; _i < arguments.length; _i++) { + args[_i] = arguments[_i]; + } + _this._historyBreadcrumb.apply(_this, __spread(args)); + }, + type: 'history', + }); + } + }; + /** + * @inheritDoc + */ + Breadcrumbs.id = 'Breadcrumbs'; + return Breadcrumbs; + }()); + /** + * Create a breadcrumb of `sentry` from the events themselves + */ + function addSentryBreadcrumb(serializedData) { + // There's always something that can go wrong with deserialization... + try { + var event_1 = JSON.parse(serializedData); + getCurrentHub().addBreadcrumb({ + category: "sentry." + (event_1.type === 'transaction' ? 'transaction' : 'event'), + event_id: event_1.event_id, + level: event_1.level || exports.Severity.fromString('error'), + message: getEventDescription(event_1), + }, { + event: event_1, + }); + } + catch (_oO) { + logger.error('Error while adding sentry type breadcrumb'); + } + } + + var DEFAULT_KEY = 'cause'; + var DEFAULT_LIMIT = 5; + /** Adds SDK info to an event. */ + var LinkedErrors = /** @class */ (function () { + /** + * @inheritDoc + */ + function LinkedErrors(options) { + if (options === void 0) { options = {}; } + /** + * @inheritDoc + */ + this.name = LinkedErrors.id; + this._key = options.key || DEFAULT_KEY; + this._limit = options.limit || DEFAULT_LIMIT; + } + /** + * @inheritDoc + */ + LinkedErrors.prototype.setupOnce = function () { + addGlobalEventProcessor(function (event, hint) { + var self = getCurrentHub().getIntegration(LinkedErrors); + if (self) { + return self._handler(event, hint); + } + return event; + }); + }; + /** + * @inheritDoc + */ + LinkedErrors.prototype._handler = function (event, hint) { + if (!event.exception || !event.exception.values || !hint || !isInstanceOf(hint.originalException, Error)) { + return event; + } + var linkedErrors = this._walkErrorTree(hint.originalException, this._key); + event.exception.values = __spread(linkedErrors, event.exception.values); + return event; + }; + /** + * @inheritDoc + */ + LinkedErrors.prototype._walkErrorTree = function (error, key, stack) { + if (stack === void 0) { stack = []; } + if (!isInstanceOf(error[key], Error) || stack.length + 1 >= this._limit) { + return stack; + } + var stacktrace = computeStackTrace(error[key]); + var exception = exceptionFromStacktrace(stacktrace); + return this._walkErrorTree(error[key], key, __spread([exception], stack)); + }; + /** + * @inheritDoc + */ + LinkedErrors.id = 'LinkedErrors'; + return LinkedErrors; + }()); + + var global$4 = getGlobalObject(); + /** UserAgent */ + var UserAgent = /** @class */ (function () { + function UserAgent() { + /** + * @inheritDoc + */ + this.name = UserAgent.id; + } + /** + * @inheritDoc + */ + UserAgent.prototype.setupOnce = function () { + addGlobalEventProcessor(function (event) { + if (getCurrentHub().getIntegration(UserAgent)) { + if (!global$4.navigator || !global$4.location) { + return event; + } + // Request Interface: https://docs.sentry.io/development/sdk-dev/event-payloads/request/ + var request = event.request || {}; + request.url = request.url || global$4.location.href; + request.headers = request.headers || {}; + request.headers['User-Agent'] = global$4.navigator.userAgent; + return __assign({}, event, { request: request }); + } + return event; + }); + }; + /** + * @inheritDoc + */ + UserAgent.id = 'UserAgent'; + return UserAgent; + }()); + + + + var BrowserIntegrations = /*#__PURE__*/Object.freeze({ + GlobalHandlers: GlobalHandlers, + TryCatch: TryCatch, + Breadcrumbs: Breadcrumbs, + LinkedErrors: LinkedErrors, + UserAgent: UserAgent + }); + + var defaultIntegrations = [ + new InboundFilters(), + new FunctionToString(), + new TryCatch(), + new Breadcrumbs(), + new GlobalHandlers(), + new LinkedErrors(), + new UserAgent(), + ]; + /** + * The Sentry Browser SDK Client. + * + * To use this SDK, call the {@link init} function as early as possible when + * loading the web page. To set context information or send manual events, use + * the provided methods. + * + * @example + * + * ``` + * + * import { init } from '@sentry/browser'; + * + * init({ + * dsn: '__DSN__', + * // ... + * }); + * ``` + * + * @example + * ``` + * + * import { configureScope } from '@sentry/browser'; + * configureScope((scope: Scope) => { + * scope.setExtra({ battery: 0.7 }); + * scope.setTag({ user_mode: 'admin' }); + * scope.setUser({ id: '4711' }); + * }); + * ``` + * + * @example + * ``` + * + * import { addBreadcrumb } from '@sentry/browser'; + * addBreadcrumb({ + * message: 'My Breadcrumb', + * // ... + * }); + * ``` + * + * @example + * + * ``` + * + * import * as Sentry from '@sentry/browser'; + * Sentry.captureMessage('Hello, world!'); + * Sentry.captureException(new Error('Good bye')); + * Sentry.captureEvent({ + * message: 'Manual', + * stacktrace: [ + * // ... + * ], + * }); + * ``` + * + * @see {@link BrowserOptions} for documentation on configuration options. + */ + function init(options) { + if (options === void 0) { options = {}; } + if (options.defaultIntegrations === undefined) { + options.defaultIntegrations = defaultIntegrations; + } + if (options.release === undefined) { + var window_1 = getGlobalObject(); + // This supports the variable that sentry-webpack-plugin injects + if (window_1.SENTRY_RELEASE && window_1.SENTRY_RELEASE.id) { + options.release = window_1.SENTRY_RELEASE.id; + } + } + initAndBind(BrowserClient, options); + } + /** + * Present the user with a report dialog. + * + * @param options Everything is optional, we try to fetch all info need from the global scope. + */ + function showReportDialog(options) { + if (options === void 0) { options = {}; } + if (!options.eventId) { + options.eventId = getCurrentHub().lastEventId(); + } + var client = getCurrentHub().getClient(); + if (client) { + client.showReportDialog(options); + } + } + /** + * This is the getter for lastEventId. + * + * @returns The last event id of a captured event. + */ + function lastEventId() { + return getCurrentHub().lastEventId(); + } + /** + * This function is here to be API compatible with the loader. + * @hidden + */ + function forceLoad() { + // Noop + } + /** + * This function is here to be API compatible with the loader. + * @hidden + */ + function onLoad(callback) { + callback(); + } + /** + * A promise that resolves when all current events have been sent. + * If you provide a timeout and the queue takes longer to drain the promise returns false. + * + * @param timeout Maximum time in ms the client should wait. + */ + function flush(timeout) { + var client = getCurrentHub().getClient(); + if (client) { + return client.flush(timeout); + } + return SyncPromise.reject(false); + } + /** + * A promise that resolves when all current events have been sent. + * If you provide a timeout and the queue takes longer to drain the promise returns false. + * + * @param timeout Maximum time in ms the client should wait. + */ + function close(timeout) { + var client = getCurrentHub().getClient(); + if (client) { + return client.close(timeout); + } + return SyncPromise.reject(false); + } + /** + * Wrap code within a try/catch block so the SDK is able to capture errors. + * + * @param fn A function to wrap. + * + * @returns The result of wrapped function call. + */ + function wrap$1(fn) { + return wrap(fn)(); // tslint:disable-line:no-unsafe-any + } + + var windowIntegrations = {}; + // This block is needed to add compatibility with the integrations packages when used with a CDN + // tslint:disable: no-unsafe-any + var _window = getGlobalObject(); + if (_window.Sentry && _window.Sentry.Integrations) { + windowIntegrations = _window.Sentry.Integrations; + } + // tslint:enable: no-unsafe-any + var INTEGRATIONS = __assign({}, windowIntegrations, CoreIntegrations, BrowserIntegrations); + + exports.BrowserClient = BrowserClient; + exports.Hub = Hub; + exports.Integrations = INTEGRATIONS; + exports.SDK_NAME = SDK_NAME; + exports.SDK_VERSION = SDK_VERSION; + exports.Scope = Scope; + exports.Transports = index; + exports.addBreadcrumb = addBreadcrumb; + exports.addGlobalEventProcessor = addGlobalEventProcessor; + exports.captureEvent = captureEvent; + exports.captureException = captureException; + exports.captureMessage = captureMessage; + exports.close = close; + exports.configureScope = configureScope; + exports.defaultIntegrations = defaultIntegrations; + exports.flush = flush; + exports.forceLoad = forceLoad; + exports.getCurrentHub = getCurrentHub; + exports.getHubFromCarrier = getHubFromCarrier; + exports.init = init; + exports.lastEventId = lastEventId; + exports.onLoad = onLoad; + exports.setContext = setContext; + exports.setExtra = setExtra; + exports.setExtras = setExtras; + exports.setTag = setTag; + exports.setTags = setTags; + exports.setUser = setUser; + exports.showReportDialog = showReportDialog; + exports.withScope = withScope; + exports.wrap = wrap$1; + + return exports; + +}({})); +//# sourceMappingURL=bundle.js.map From 6a1d02a1fb822dfa90b798318be467fecbbf8c29 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 19 May 2020 15:41:45 +0300 Subject: [PATCH 437/738] Update ld_json.html --- website/templates/docs/ld_json.html | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/website/templates/docs/ld_json.html b/website/templates/docs/ld_json.html index 051bac0c8a7..29b5b0dac9f 100644 --- a/website/templates/docs/ld_json.html +++ b/website/templates/docs/ld_json.html @@ -20,11 +20,14 @@ "width": 163, "height": 60 } -}, { +}}, { "@context": "http://schema.org", "@type": "Episode", "name": "{{ title }}", -"ratingValue": RATING_VALUE, -"ratingCount": RATING_COUNT +"aggregateRating": { + "@type": "AggregateRating" + "ratingValue": RATING_VALUE, + "ratingCount": RATING_COUNT +} }] {% endif %} From 1f6697004015bfe4bd6bae5cbf20482594fc9380 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 19 May 2020 15:44:59 +0300 Subject: [PATCH 438/738] trigger ci --- docs/en/engines/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/engines/index.md b/docs/en/engines/index.md index c4b0b299858..fe7e46fb534 100644 --- a/docs/en/engines/index.md +++ b/docs/en/engines/index.md @@ -3,4 +3,4 @@ toc_folder_title: Engines toc_priority: 25 --- - +{## [Original article](https://clickhouse.tech/docs/en/engines/) ##} From 35b916433a12706398b28a710c2c0066895ecf1a Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Tue, 19 May 2020 16:00:32 +0300 Subject: [PATCH 439/738] fix TTL with defaults after modify --- src/DataStreams/TTLBlockInputStream.cpp | 21 +++++---- tests/integration/test_ttl_replicated/test.py | 44 +++++++++++++++++++ 2 files changed, 56 insertions(+), 9 deletions(-) diff --git a/src/DataStreams/TTLBlockInputStream.cpp b/src/DataStreams/TTLBlockInputStream.cpp index 692d2c0d5cf..c9a40768166 100644 --- a/src/DataStreams/TTLBlockInputStream.cpp +++ b/src/DataStreams/TTLBlockInputStream.cpp @@ -34,22 +34,25 @@ TTLBlockInputStream::TTLBlockInputStream( const auto & storage_columns = storage.getColumns(); const auto & column_defaults = storage_columns.getDefaults(); + ASTPtr default_expr_list = std::make_shared(); + for (const auto & [name, _] : storage.column_ttl_entries_by_name) + { + auto it = column_defaults.find(name); + if (it != column_defaults.end()) + { + auto column = storage_columns.get(name); + auto expression = it->second.expression->clone(); + default_expr_list->children.emplace_back(setAlias(addTypeConversionToAST(std::move(expression), column.type->getName()), it->first)); + } + } + for (const auto & [name, ttl_info] : old_ttl_infos.columns_ttl) { if (force || isTTLExpired(ttl_info.min)) { new_ttl_infos.columns_ttl.emplace(name, IMergeTreeDataPart::TTLInfo{}); empty_columns.emplace(name); - - auto it = column_defaults.find(name); - - if (it != column_defaults.end()) - { - auto column = storage_columns.get(name); - auto expression = it->second.expression->clone(); - default_expr_list->children.emplace_back(setAlias(addTypeConversionToAST(std::move(expression), column.type->getName()), it->first)); - } } else new_ttl_infos.columns_ttl.emplace(name, ttl_info); diff --git a/tests/integration/test_ttl_replicated/test.py b/tests/integration/test_ttl_replicated/test.py index 78ff703f0ec..29169ad3c0e 100644 --- a/tests/integration/test_ttl_replicated/test.py +++ b/tests/integration/test_ttl_replicated/test.py @@ -109,6 +109,50 @@ def test_ttl_table(started_cluster, delete_suffix): assert TSV(node1.query("SELECT * FROM test_ttl")) == TSV("") assert TSV(node2.query("SELECT * FROM test_ttl")) == TSV("") +def test_modify_ttl(started_cluster): + drop_table([node1, node2], "test_ttl") + for node in [node1, node2]: + node.query( + ''' + CREATE TABLE test_ttl(d DateTime, id UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl', '{replica}') + ORDER BY id + '''.format(replica=node.name)) + + node1.query("INSERT INTO test_ttl VALUES (now() - INTERVAL 5 HOUR, 1), (now() - INTERVAL 3 HOUR, 2), (now() - INTERVAL 1 HOUR, 3)") + node2.query("SYSTEM SYNC REPLICA test_ttl", timeout=20) + + node1.query("ALTER TABLE test_ttl MODIFY TTL d + INTERVAL 4 HOUR SETTINGS mutations_sync = 2") + assert node2.query("SELECT id FROM test_ttl") == "2\n3\n" + + node2.query("ALTER TABLE test_ttl MODIFY TTL d + INTERVAL 2 HOUR SETTINGS mutations_sync = 2") + assert node1.query("SELECT id FROM test_ttl") == "3\n" + + node1.query("ALTER TABLE test_ttl MODIFY TTL d + INTERVAL 30 MINUTE SETTINGS mutations_sync = 2") + assert node2.query("SELECT id FROM test_ttl") == "" + +def test_modify_column_ttl(started_cluster): + drop_table([node1, node2], "test_ttl") + for node in [node1, node2]: + node.query( + ''' + CREATE TABLE test_ttl(d DateTime, id UInt32 DEFAULT 42) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl', '{replica}') + ORDER BY d + '''.format(replica=node.name)) + + node1.query("INSERT INTO test_ttl VALUES (now() - INTERVAL 5 HOUR, 1), (now() - INTERVAL 3 HOUR, 2), (now() - INTERVAL 1 HOUR, 3)") + node2.query("SYSTEM SYNC REPLICA test_ttl", timeout=20) + + node1.query("ALTER TABLE test_ttl MODIFY COLUMN id UInt32 TTL d + INTERVAL 4 HOUR SETTINGS mutations_sync = 2") + assert node2.query("SELECT id FROM test_ttl") == "42\n2\n3\n" + + node1.query("ALTER TABLE test_ttl MODIFY COLUMN id UInt32 TTL d + INTERVAL 2 HOUR SETTINGS mutations_sync = 2") + assert node1.query("SELECT id FROM test_ttl") == "42\n42\n3\n" + + node1.query("ALTER TABLE test_ttl MODIFY COLUMN id UInt32 TTL d + INTERVAL 30 MINUTE SETTINGS mutations_sync = 2") + assert node2.query("SELECT id FROM test_ttl") == "42\n42\n42\n" + def test_ttl_double_delete_rule_returns_error(started_cluster): drop_table([node1, node2], "test_ttl") try: From e2a49e8a8063e47d8375f0fcc468d746ce967b97 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 19 May 2020 16:22:29 +0300 Subject: [PATCH 440/738] Update roadmap.md --- docs/en/whats-new/roadmap.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/en/whats-new/roadmap.md b/docs/en/whats-new/roadmap.md index e50d663fcb0..c2ebc5260e6 100644 --- a/docs/en/whats-new/roadmap.md +++ b/docs/en/whats-new/roadmap.md @@ -5,13 +5,12 @@ toc_title: Roadmap # Roadmap {#roadmap} -## Q1 2020 {#q1-2020} - -- Role-based access control - ## Q2 2020 {#q2-2020} - Integration with external authentication services + +## Q3 2020 {#q3-2020} + - Resource pools for more precise distribution of cluster capacity between users {## [Original article](https://clickhouse.tech/docs/en/roadmap/) ##} From b0d9552e1eb7b2e7f50978f52d1a3e730eff9ad7 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 19 May 2020 17:06:33 +0300 Subject: [PATCH 441/738] Try fix compatibility. --- src/Interpreters/executeQuery.cpp | 16 ++++++++++++++++ src/Interpreters/executeQuery.h | 15 ++++++++------- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 8c6e5cea73c..468ce36daf9 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -599,6 +599,22 @@ BlockIO executeQuery( return streams; } +BlockIO executeQuery( + const String & query, + Context & context, + bool internal, + QueryProcessingStage::Enum stage, + bool may_have_embedded_data, + bool allow_processors) +{ + BlockIO res = executeQuery(query, context, internal, stage, may_have_embedded_data, allow_processors); + + if (!allow_processors && res.pipeline.initialized()) + res.in = res.getInputStream(); + + return res; +} + void executeQuery( ReadBuffer & istr, diff --git a/src/Interpreters/executeQuery.h b/src/Interpreters/executeQuery.h index 22b353488ad..2850bb3baf4 100644 --- a/src/Interpreters/executeQuery.h +++ b/src/Interpreters/executeQuery.h @@ -45,13 +45,14 @@ BlockIO executeQuery( bool may_have_embedded_data = false /// If insert query may have embedded data ); - -QueryPipeline executeQueryWithProcessors( - const String & query, /// Query text without INSERT data. The latter must be written to BlockIO::out. - Context & context, /// DB, tables, data types, storage engines, functions, aggregate functions... - bool internal = false, /// If true, this query is caused by another query and thus needn't be registered in the ProcessList. - QueryProcessingStage::Enum stage = QueryProcessingStage::Complete, /// To which stage the query must be executed. - bool may_have_embedded_data = false /// If insert query may have embedded data +/// Old interface with allow_processors flag. For compatibility. +BlockIO executeQuery( + const String & query, + Context & context, + bool internal, + QueryProcessingStage::Enum stage, + bool may_have_embedded_data, + bool allow_processors /// If can use processors pipeline ); } From b78a93aa2e22a21bf33713b1218d9f3e6b81e291 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 19 May 2020 17:32:24 +0300 Subject: [PATCH 442/738] Tune cache purge --- docs/tools/release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tools/release.sh b/docs/tools/release.sh index 8c61d16966f..ff3d2971e6e 100755 --- a/docs/tools/release.sh +++ b/docs/tools/release.sh @@ -44,7 +44,7 @@ then if [[ ! -z "${CLOUDFLARE_TOKEN}" ]] then sleep 1m - git diff --stat="9999,9999" --diff-filter=M HEAD~1 | grep '|' | awk '$1 ~ /\.html$/ { if ($3>4) { url="https://clickhouse.tech/"$1; sub(/\/index.html/, "/", url); print "\""url"\""; }}' | split -l 25 /dev/stdin PURGE + git diff --stat="9999,9999" --diff-filter=M HEAD~1 | grep '|' | awk '$1 ~ /\.html$/ { if ($3>8) { url="https://content.clickhouse.tech/"$1; sub(/\/index.html/, "/", url); print "\""url"\""; }}' | split -l 25 /dev/stdin PURGE for FILENAME in $(ls PURGE*) do POST_DATA=$(cat "${FILENAME}" | sed -n -e 'H;${x;s/\n/,/g;s/^,//;p;}' | awk '{print "{\"files\":["$0"]}";}') From 284d3aba1278752462de631d82a8f366025e28e5 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 19 May 2020 17:41:36 +0300 Subject: [PATCH 443/738] Adjust footer margins --- website/templates/docs/footer.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/templates/docs/footer.html b/website/templates/docs/footer.html index efe53b3c24c..c9432a0b4dc 100644 --- a/website/templates/docs/footer.html +++ b/website/templates/docs/footer.html @@ -1,5 +1,5 @@ {% if not single_page %} -
    +

    {{ _('Rating') }}: RATING_VALUE - RATING_COUNT {{ _('votes') }}

    {% if is_amp %}{{ _('Article Rating') }}{% else %}{{ _('Was this content helpful?') }}{% endif %}

    RATING_STARS
    -
    +
    {{ config.copyright }}
    {{ _('Built from') }} {{ config.extra.rev_short }}{% if page and page.meta %}{% if page.meta.published_date %}, {{ _('published on') }} {{ page.meta.published_date.date().isoformat() }}{% endif %}{% if page.meta.modified_date %}, {{ _('modified on') }} {{ page.meta.modified_date.date().isoformat() }}{% endif %} {% endif %} From a700b527ce43ff7b772935b9e6781e084a566ad2 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 19 May 2020 17:51:51 +0300 Subject: [PATCH 444/738] fixup --- docker/test/performance-comparison/compare.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 023c931de1c..1833c9a7a9e 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -27,7 +27,7 @@ function configure kill -0 $left_pid disown $left_pid set +m - while ! clickhouse-client --port 9001 --query "select 1" ; do kill -0 $left_pid ; echo . ; sleep 1 ; done + while ! clickhouse-client --port 9001 --query "select 1" && kill -0 $left_pid ; do echo . ; sleep 1 ; done echo server for setup started clickhouse-client --port 9001 --query "create database test" ||: @@ -71,9 +71,9 @@ function restart set +m - while ! clickhouse-client --port 9001 --query "select 1" ; do kill -0 $left_pid ; echo . ; sleep 1 ; done + while ! clickhouse-client --port 9001 --query "select 1" && kill -0 $left_pid ; do echo . ; sleep 1 ; done echo left ok - while ! clickhouse-client --port 9002 --query "select 1" ; do kill -0 $right_pid ; echo . ; sleep 1 ; done + while ! clickhouse-client --port 9002 --query "select 1" && kill -0 $right_pid ; do echo . ; sleep 1 ; done echo right ok clickhouse-client --port 9001 --query "select * from system.tables where database != 'system'" @@ -263,7 +263,7 @@ done wait unset IFS -parallel --verbose --null < analyze-commands.txt +parallel --null < analyze-commands.txt } # Analyze results @@ -542,7 +542,7 @@ case "$stage" in # to collect the logs. Prefer not to restart, because addresses might change # and we won't be able to process trace_log data. Start in a subshell, so that # it doesn't interfere with the watchdog through `wait`. - ( time get_profiles || restart || get_profiles ||: ) 2>> profile-errors.log + ( get_profiles || restart || get_profiles ||: ) # Kill the whole process group, because somehow when the subshell is killed, # the sleep inside remains alive and orphaned. From 838b3e95332b83ab7cd28962e974543c708608ff Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 19 May 2020 17:55:27 +0300 Subject: [PATCH 445/738] fixup --- src/Interpreters/SystemLog.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h index 10dcb84c1ce..b2a4eec7883 100644 --- a/src/Interpreters/SystemLog.h +++ b/src/Interpreters/SystemLog.h @@ -344,7 +344,8 @@ void SystemLog::savingThreadFunction() std::unique_lock lock(mutex); const bool predicate = flush_event.wait_for(lock, std::chrono::milliseconds(flush_interval_milliseconds), - [&] () { + [&] () + { return requested_flush_before > flushed_before || is_shutdown; } From a120fcb8256cca55f9a170dab9199c589c8222fb Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 19 May 2020 18:09:49 +0300 Subject: [PATCH 446/738] fix mistype --- website/templates/docs/ld_json.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/templates/docs/ld_json.html b/website/templates/docs/ld_json.html index 29b5b0dac9f..3db89657221 100644 --- a/website/templates/docs/ld_json.html +++ b/website/templates/docs/ld_json.html @@ -25,7 +25,7 @@ "@type": "Episode", "name": "{{ title }}", "aggregateRating": { - "@type": "AggregateRating" + "@type": "AggregateRating", "ratingValue": RATING_VALUE, "ratingCount": RATING_COUNT } From 87e8288e964fa41d1047d851735e80d40581c8bb Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 19 May 2020 18:10:49 +0300 Subject: [PATCH 447/738] trigger ci --- docs/en/introduction/performance.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/introduction/performance.md b/docs/en/introduction/performance.md index 73aacc293b1..d2780aedccb 100644 --- a/docs/en/introduction/performance.md +++ b/docs/en/introduction/performance.md @@ -27,4 +27,4 @@ Under the same conditions, ClickHouse can handle several hundred queries per sec We recommend inserting data in packets of at least 1000 rows, or no more than a single request per second. When inserting to a MergeTree table from a tab-separated dump, the insertion speed can be from 50 to 200 MB/s. If the inserted rows are around 1 Kb in size, the speed will be from 50,000 to 200,000 rows per second. If the rows are small, the performance can be higher in rows per second (on Banner System data -`>` 500,000 rows per second; on Graphite data -`>` 1,000,000 rows per second). To improve performance, you can make multiple INSERT queries in parallel, which scales linearly. -[Original article](https://clickhouse.tech/docs/en/introduction/performance/) +{## [Original article](https://clickhouse.tech/docs/en/introduction/performance/) ##} From 274a48c353f5245362b15627c5682c9c8221637d Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 19 May 2020 18:11:23 +0300 Subject: [PATCH 448/738] [website] parallel minify (#11043) --- docs/tools/website.py | 57 ++++++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/docs/tools/website.py b/docs/tools/website.py index 4dccc66693f..ed950bd06e3 100644 --- a/docs/tools/website.py +++ b/docs/tools/website.py @@ -1,9 +1,11 @@ +import concurrent.futures import hashlib import json import logging import os import shutil import subprocess +import sys import bs4 import closure @@ -152,6 +154,28 @@ def get_js_in(args): ] +def minify_file(path, css_digest, js_digest): + if not ( + path.endswith('.html') or + path.endswith('.css') + ): + return + + logging.info('Minifying %s', path) + with open(path, 'rb') as f: + content = f.read().decode('utf-8') + if path.endswith('.html'): + content = minify_html(content) + content = content.replace('base.css?css_digest', f'base.css?{css_digest}') + content = content.replace('base.js?js_digest', f'base.js?{js_digest}') + elif path.endswith('.css'): + content = cssmin.cssmin(content) + elif path.endswith('.js'): + content = jsmin.jsmin(content) + with open(path, 'wb') as f: + f.write(content.encode('utf-8')) + + def minify_website(args): css_in = ' '.join(get_css_in(args)) css_out = f'{args.output_dir}/css/base.css' @@ -197,28 +221,17 @@ def minify_website(args): if args.minify: logging.info('Minifying website') - for root, _, filenames in os.walk(args.output_dir): - for filename in filenames: - path = os.path.join(root, filename) - if not ( - filename.endswith('.html') or - filename.endswith('.css') - ): - continue - - logging.info('Minifying %s', path) - with open(path, 'rb') as f: - content = f.read().decode('utf-8') - if filename.endswith('.html'): - content = minify_html(content) - content = content.replace('base.css?css_digest', f'base.css?{css_digest}') - content = content.replace('base.js?js_digest', f'base.js?{js_digest}') - elif filename.endswith('.css'): - content = cssmin.cssmin(content) - elif filename.endswith('.js'): - content = jsmin.jsmin(content) - with open(path, 'wb') as f: - f.write(content.encode('utf-8')) + with concurrent.futures.ThreadPoolExecutor() as executor: + futures = [] + for root, _, filenames in os.walk(args.output_dir): + for filename in filenames: + path = os.path.join(root, filename) + futures.append(executor.submit(minify_file, path, css_digest, js_digest)) + for future in futures: + exc = future.exception() + if exc: + logging.error(exc) + sys.exit(1) def process_benchmark_results(args): From a17f034271fca012109f5750179a071bc03a361d Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 19 May 2020 18:27:10 +0300 Subject: [PATCH 449/738] Bump compose version to 2.3, and make tmpfs for zookeeper optional --- .../compose/docker_compose_hdfs.yml | 2 +- .../compose/docker_compose_kafka.yml | 2 +- .../compose/docker_compose_minio.yml | 2 +- .../compose/docker_compose_mongo.yml | 2 +- .../compose/docker_compose_mysql.yml | 2 +- .../compose/docker_compose_net.yml | 2 +- .../compose/docker_compose_postgres.yml | 2 +- .../compose/docker_compose_redis.yml | 2 +- .../compose/docker_compose_zookeeper.yml | 34 ++++++++++++------- tests/integration/helpers/cluster.py | 30 ++++++++++++++-- .../clients/golang/docker_compose.yml | 2 +- .../clients/java/docker_compose.yml | 2 +- .../clients/mysql/docker_compose.yml | 2 +- .../clients/mysqljs/docker_compose.yml | 2 +- .../clients/php-mysqlnd/docker_compose.yml | 2 +- .../integration/test_zookeeper_config/test.py | 20 +++++------ 16 files changed, 72 insertions(+), 38 deletions(-) diff --git a/docker/test/integration/compose/docker_compose_hdfs.yml b/docker/test/integration/compose/docker_compose_hdfs.yml index 010a6cd37c1..d19aacfc8d4 100644 --- a/docker/test/integration/compose/docker_compose_hdfs.yml +++ b/docker/test/integration/compose/docker_compose_hdfs.yml @@ -1,4 +1,4 @@ -version: '2.2' +version: '2.3' services: hdfs1: image: sequenceiq/hadoop-docker:2.7.0 diff --git a/docker/test/integration/compose/docker_compose_kafka.yml b/docker/test/integration/compose/docker_compose_kafka.yml index 16cbec5f333..219d977ffd9 100644 --- a/docker/test/integration/compose/docker_compose_kafka.yml +++ b/docker/test/integration/compose/docker_compose_kafka.yml @@ -1,4 +1,4 @@ -version: '2.2' +version: '2.3' services: kafka_zookeeper: diff --git a/docker/test/integration/compose/docker_compose_minio.yml b/docker/test/integration/compose/docker_compose_minio.yml index 79023e24244..c52c45b9d69 100644 --- a/docker/test/integration/compose/docker_compose_minio.yml +++ b/docker/test/integration/compose/docker_compose_minio.yml @@ -1,4 +1,4 @@ -version: '2.2' +version: '2.3' services: minio1: diff --git a/docker/test/integration/compose/docker_compose_mongo.yml b/docker/test/integration/compose/docker_compose_mongo.yml index a593c3e123b..f6afe951349 100644 --- a/docker/test/integration/compose/docker_compose_mongo.yml +++ b/docker/test/integration/compose/docker_compose_mongo.yml @@ -1,4 +1,4 @@ -version: '2.2' +version: '2.3' services: mongo1: image: mongo:3.6 diff --git a/docker/test/integration/compose/docker_compose_mysql.yml b/docker/test/integration/compose/docker_compose_mysql.yml index 30c76ef1981..2e3afce117d 100644 --- a/docker/test/integration/compose/docker_compose_mysql.yml +++ b/docker/test/integration/compose/docker_compose_mysql.yml @@ -1,4 +1,4 @@ -version: '2.2' +version: '2.3' services: mysql1: image: mysql:5.7 diff --git a/docker/test/integration/compose/docker_compose_net.yml b/docker/test/integration/compose/docker_compose_net.yml index 4e8cabe5403..eff43681f2e 100644 --- a/docker/test/integration/compose/docker_compose_net.yml +++ b/docker/test/integration/compose/docker_compose_net.yml @@ -1,4 +1,4 @@ -version: '2.2' +version: '2.3' networks: default: driver: bridge diff --git a/docker/test/integration/compose/docker_compose_postgres.yml b/docker/test/integration/compose/docker_compose_postgres.yml index 0745fe827c0..ffc4d3164b5 100644 --- a/docker/test/integration/compose/docker_compose_postgres.yml +++ b/docker/test/integration/compose/docker_compose_postgres.yml @@ -1,4 +1,4 @@ -version: '2.2' +version: '2.3' services: postgres1: image: postgres diff --git a/docker/test/integration/compose/docker_compose_redis.yml b/docker/test/integration/compose/docker_compose_redis.yml index 205409b3a21..2dc79ed5910 100644 --- a/docker/test/integration/compose/docker_compose_redis.yml +++ b/docker/test/integration/compose/docker_compose_redis.yml @@ -1,4 +1,4 @@ -version: '2.2' +version: '2.3' services: redis1: image: redis diff --git a/docker/test/integration/compose/docker_compose_zookeeper.yml b/docker/test/integration/compose/docker_compose_zookeeper.yml index 974e329481c..49e285b5515 100644 --- a/docker/test/integration/compose/docker_compose_zookeeper.yml +++ b/docker/test/integration/compose/docker_compose_zookeeper.yml @@ -1,7 +1,5 @@ -version: '2.2' +version: '2.3' services: -# our compose version doesn't support inheritance :( so we copy-paste images -# configuration zoo1: image: zookeeper:3.4.12 restart: always @@ -10,9 +8,13 @@ services: ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888 ZOO_MY_ID: 1 JVMFLAGS: -Dzookeeper.forceSync=no - tmpfs: - - /data - - /datalog + volumes: + - type: ${ZK_FS:-tmpfs} + source: ${ZK_DATA1:-} + target: /data + - type: ${ZK_FS:-tmpfs} + source: ${ZK_DATA_LOG1:-} + target: /datalog zoo2: image: zookeeper:3.4.12 restart: always @@ -21,9 +23,13 @@ services: ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888 ZOO_MY_ID: 2 JVMFLAGS: -Dzookeeper.forceSync=no - tmpfs: - - /data - - /datalog + volumes: + - type: ${ZK_FS:-tmpfs} + source: ${ZK_DATA2:-} + target: /data + - type: ${ZK_FS:-tmpfs} + source: ${ZK_DATA_LOG2:-} + target: /datalog zoo3: image: zookeeper:3.4.12 restart: always @@ -32,6 +38,10 @@ services: ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888 ZOO_MY_ID: 3 JVMFLAGS: -Dzookeeper.forceSync=no - tmpfs: - - /data - - /datalog + volumes: + - type: ${ZK_FS:-tmpfs} + source: ${ZK_DATA3:-} + target: /data + - type: ${ZK_FS:-tmpfs} + source: ${ZK_DATA_LOG3:-} + target: /datalog diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 53c36ff8924..f6e4eb9768d 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -133,6 +133,8 @@ class ClickHouseCluster: self.schema_registry_host = "schema-registry" self.schema_registry_port = 8081 + self.zookeeper_use_tmpfs = True + self.docker_client = None self.is_up = False @@ -148,7 +150,7 @@ class ClickHouseCluster: with_redis=False, with_minio=False, hostname=None, env_variables=None, image="yandex/clickhouse-integration-test", stay_alive=False, ipv4_address=None, ipv6_address=None, with_installed_binary=False, tmpfs=None, - zookeeper_docker_compose_path=None): + zookeeper_docker_compose_path=None, zookeeper_use_tmpfs=True): """Add an instance to the cluster. name - the name of the instance directory and the value of the 'instance' macro in ClickHouse. @@ -187,6 +189,7 @@ class ClickHouseCluster: zookeeper_docker_compose_path = p.join(DOCKER_COMPOSE_DIR, 'docker_compose_zookeeper.yml') self.with_zookeeper = True + self.zookeeper_use_tmpfs = zookeeper_use_tmpfs self.base_cmd.extend(['--file', zookeeper_docker_compose_path]) self.base_zookeeper_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', self.project_name, '--file', zookeeper_docker_compose_path] @@ -464,7 +467,19 @@ class ClickHouseCluster: common_opts = ['up', '-d', '--force-recreate'] if self.with_zookeeper and self.base_zookeeper_cmd: - subprocess_check_call(self.base_zookeeper_cmd + common_opts) + env = os.environ.copy() + if not self.zookeeper_use_tmpfs: + env['ZK_FS'] = 'bind' + for i in range(1, 4): + zk_data_path = self.instances_dir + '/zkdata' + str(i) + zk_log_data_path = self.instances_dir + '/zklog' + str(i) + if not os.path.exists(zk_data_path): + os.mkdir(zk_data_path) + if not os.path.exists(zk_log_data_path): + os.mkdir(zk_log_data_path) + env['ZK_DATA' + str(i)] = zk_data_path + env['ZK_DATA_LOG' + str(i)] = zk_log_data_path + subprocess.check_call(self.base_zookeeper_cmd + common_opts, env=env) for command in self.pre_zookeeper_commands: self.run_kazoo_commands_with_retries(command, repeats=5) self.wait_zookeeper_to_start(120) @@ -547,6 +562,15 @@ class ClickHouseCluster: instance.ip_address = None instance.client = None + if not self.zookeeper_use_tmpfs: + for i in range(1, 4): + zk_data_path = self.instances_dir + '/zkdata' + str(i) + zk_log_data_path = self.instances_dir + '/zklog' + str(i) + if os.path.exists(zk_data_path): + shutil.rmtree(zk_data_path) + if os.path.exists(zk_log_data_path): + shutil.rmtree(zk_log_data_path) + if sanitizer_assert_instance is not None: raise Exception("Sanitizer assert found in {} for instance {}".format(self.docker_logs_path, sanitizer_assert_instance)) @@ -586,7 +610,7 @@ CLICKHOUSE_START_COMMAND = "clickhouse server --config-file=/etc/clickhouse-serv CLICKHOUSE_STAY_ALIVE_COMMAND = 'bash -c "{} --daemon; tail -f /dev/null"'.format(CLICKHOUSE_START_COMMAND) DOCKER_COMPOSE_TEMPLATE = ''' -version: '2.2' +version: '2.3' services: {name}: image: {image} diff --git a/tests/integration/test_mysql_protocol/clients/golang/docker_compose.yml b/tests/integration/test_mysql_protocol/clients/golang/docker_compose.yml index 7850298dd7c..4fe6fdaeecd 100644 --- a/tests/integration/test_mysql_protocol/clients/golang/docker_compose.yml +++ b/tests/integration/test_mysql_protocol/clients/golang/docker_compose.yml @@ -1,4 +1,4 @@ -version: '2.2' +version: '2.3' services: golang1: build: diff --git a/tests/integration/test_mysql_protocol/clients/java/docker_compose.yml b/tests/integration/test_mysql_protocol/clients/java/docker_compose.yml index dbe404232a0..522f404cde6 100644 --- a/tests/integration/test_mysql_protocol/clients/java/docker_compose.yml +++ b/tests/integration/test_mysql_protocol/clients/java/docker_compose.yml @@ -1,4 +1,4 @@ -version: '2.2' +version: '2.3' services: java1: build: diff --git a/tests/integration/test_mysql_protocol/clients/mysql/docker_compose.yml b/tests/integration/test_mysql_protocol/clients/mysql/docker_compose.yml index 752e59a2d08..802151c4d7b 100644 --- a/tests/integration/test_mysql_protocol/clients/mysql/docker_compose.yml +++ b/tests/integration/test_mysql_protocol/clients/mysql/docker_compose.yml @@ -1,4 +1,4 @@ -version: '2.2' +version: '2.3' services: mysql1: image: mysql:5.7 diff --git a/tests/integration/test_mysql_protocol/clients/mysqljs/docker_compose.yml b/tests/integration/test_mysql_protocol/clients/mysqljs/docker_compose.yml index 9485662dd6c..ebb73bc611a 100644 --- a/tests/integration/test_mysql_protocol/clients/mysqljs/docker_compose.yml +++ b/tests/integration/test_mysql_protocol/clients/mysqljs/docker_compose.yml @@ -1,4 +1,4 @@ -version: '2.2' +version: '2.3' services: mysqljs1: build: diff --git a/tests/integration/test_mysql_protocol/clients/php-mysqlnd/docker_compose.yml b/tests/integration/test_mysql_protocol/clients/php-mysqlnd/docker_compose.yml index 28145864024..c197944f375 100644 --- a/tests/integration/test_mysql_protocol/clients/php-mysqlnd/docker_compose.yml +++ b/tests/integration/test_mysql_protocol/clients/php-mysqlnd/docker_compose.yml @@ -1,4 +1,4 @@ -version: '2.2' +version: '2.3' services: php1: build: diff --git a/tests/integration/test_zookeeper_config/test.py b/tests/integration/test_zookeeper_config/test.py index 4be99c8711d..5ee6a8af021 100644 --- a/tests/integration/test_zookeeper_config/test.py +++ b/tests/integration/test_zookeeper_config/test.py @@ -12,8 +12,8 @@ def test_chroot_with_same_root(): cluster_1 = ClickHouseCluster(__file__, zookeeper_config_path='configs/zookeeper_config_root_a.xml') cluster_2 = ClickHouseCluster(__file__, zookeeper_config_path='configs/zookeeper_config_root_a.xml') - node1 = cluster_1.add_instance('node1', config_dir='configs', with_zookeeper=True) - node2 = cluster_2.add_instance('node2', config_dir='configs', with_zookeeper=True) + node1 = cluster_1.add_instance('node1', config_dir='configs', with_zookeeper=True, zookeeper_use_tmpfs=False) + node2 = cluster_2.add_instance('node2', config_dir='configs', with_zookeeper=True, zookeeper_use_tmpfs=False) nodes = [node1, node2] def create_zk_root(zk): @@ -51,8 +51,8 @@ def test_chroot_with_different_root(): cluster_1 = ClickHouseCluster(__file__, zookeeper_config_path='configs/zookeeper_config_root_a.xml') cluster_2 = ClickHouseCluster(__file__, zookeeper_config_path='configs/zookeeper_config_root_b.xml') - node1 = cluster_1.add_instance('node1', config_dir='configs', with_zookeeper=True) - node2 = cluster_2.add_instance('node2', config_dir='configs', with_zookeeper=True) + node1 = cluster_1.add_instance('node1', config_dir='configs', with_zookeeper=True, zookeeper_use_tmpfs=False) + node2 = cluster_2.add_instance('node2', config_dir='configs', with_zookeeper=True, zookeeper_use_tmpfs=False) nodes = [node1, node2] def create_zk_roots(zk): @@ -69,7 +69,7 @@ def test_chroot_with_different_root(): for i, node in enumerate(nodes): node.query(''' - CREATE TABLE simple (date Date, id UInt32) + CREATE TABLE simple (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', '{replica}', date, id, 8192); '''.format(replica=node.name)) for j in range(2): # Second insert to test deduplication @@ -90,8 +90,8 @@ def test_identity(): cluster_1 = ClickHouseCluster(__file__, zookeeper_config_path='configs/zookeeper_config_with_password.xml') cluster_2 = ClickHouseCluster(__file__) - node1 = cluster_1.add_instance('node1', config_dir='configs', with_zookeeper=True) - node2 = cluster_2.add_instance('node2', config_dir='configs', with_zookeeper=True) + node1 = cluster_1.add_instance('node1', config_dir='configs', with_zookeeper=True, zookeeper_use_tmpfs=False) + node2 = cluster_2.add_instance('node2', config_dir='configs', with_zookeeper=True, zookeeper_use_tmpfs=False) try: cluster_1.start() @@ -138,7 +138,7 @@ def test_secure_connection(): docker_compose = NamedTemporaryFile(delete=False) docker_compose.write( - "version: '2.2'\nservices:\n" + + "version: '2.3'\nservices:\n" + TEMPLATE.format(zoo_id=1, configs_dir=configs_dir, helpers_dir=helpers_dir) + TEMPLATE.format(zoo_id=2, configs_dir=configs_dir, helpers_dir=helpers_dir) + TEMPLATE.format(zoo_id=3, configs_dir=configs_dir, helpers_dir=helpers_dir) @@ -146,9 +146,9 @@ def test_secure_connection(): docker_compose.close() node1 = cluster.add_instance('node1', config_dir='configs_secure', with_zookeeper=True, - zookeeper_docker_compose_path=docker_compose.name) + zookeeper_docker_compose_path=docker_compose.name, zookeeper_use_tmpfs=False) node2 = cluster.add_instance('node2', config_dir='configs_secure', with_zookeeper=True, - zookeeper_docker_compose_path=docker_compose.name) + zookeeper_docker_compose_path=docker_compose.name, zookeeper_use_tmpfs=False) try: cluster.start() From c3d9bb829b91c44b11301928252bd93a38debbb2 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 19 May 2020 18:27:44 +0300 Subject: [PATCH 450/738] performance comparison --- tests/performance/cryptographic_hashes.xml | 2 +- tests/performance/logical_functions_large.xml | 2 +- tests/performance/logical_functions_medium.xml | 2 +- tests/performance/logical_functions_small.xml | 2 +- tests/performance/mingroupby-orderbylimit1.xml | 4 +--- tests/performance/number_formatting_formats.xml | 2 +- 6 files changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/performance/cryptographic_hashes.xml b/tests/performance/cryptographic_hashes.xml index 455a6598ad5..89947368cac 100644 --- a/tests/performance/cryptographic_hashes.xml +++ b/tests/performance/cryptographic_hashes.xml @@ -1,4 +1,4 @@ - + crypto_hash_func diff --git a/tests/performance/logical_functions_large.xml b/tests/performance/logical_functions_large.xml index 00ba31c1218..51478455c45 100644 --- a/tests/performance/logical_functions_large.xml +++ b/tests/performance/logical_functions_large.xml @@ -1,4 +1,4 @@ - + 1 diff --git a/tests/performance/logical_functions_medium.xml b/tests/performance/logical_functions_medium.xml index 19572191532..0ea4023a275 100644 --- a/tests/performance/logical_functions_medium.xml +++ b/tests/performance/logical_functions_medium.xml @@ -1,4 +1,4 @@ - + 1 diff --git a/tests/performance/logical_functions_small.xml b/tests/performance/logical_functions_small.xml index d5f6a7b99cb..9876e5188d5 100644 --- a/tests/performance/logical_functions_small.xml +++ b/tests/performance/logical_functions_small.xml @@ -1,4 +1,4 @@ - + 1 diff --git a/tests/performance/mingroupby-orderbylimit1.xml b/tests/performance/mingroupby-orderbylimit1.xml index 5b23432255d..8cf401235a9 100644 --- a/tests/performance/mingroupby-orderbylimit1.xml +++ b/tests/performance/mingroupby-orderbylimit1.xml @@ -1,6 +1,4 @@ - - - + 1 diff --git a/tests/performance/number_formatting_formats.xml b/tests/performance/number_formatting_formats.xml index ea970b5dba5..e6b5a8344dc 100644 --- a/tests/performance/number_formatting_formats.xml +++ b/tests/performance/number_formatting_formats.xml @@ -1,4 +1,4 @@ - + CREATE TABLE IF NOT EXISTS table_{format} (x UInt64) ENGINE = File(`{format}`) From 6109e87aeddd0b1799e3ff7fd41efc44f177e4b5 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 19 May 2020 18:40:15 +0300 Subject: [PATCH 451/738] Try fix compatibility. --- src/Interpreters/executeQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 468ce36daf9..e0377615dd6 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -607,7 +607,7 @@ BlockIO executeQuery( bool may_have_embedded_data, bool allow_processors) { - BlockIO res = executeQuery(query, context, internal, stage, may_have_embedded_data, allow_processors); + BlockIO res = executeQuery(query, context, internal, stage, may_have_embedded_data); if (!allow_processors && res.pipeline.initialized()) res.in = res.getInputStream(); From b5620c070c818bdfe9aa55391751d093bf0dc632 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 19 May 2020 18:47:00 +0300 Subject: [PATCH 452/738] [docs] preprocess single-page html as well (#11045) --- website/templates/docs/content.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/templates/docs/content.html b/website/templates/docs/content.html index 04c07e7802b..c5e92fa38a0 100644 --- a/website/templates/docs/content.html +++ b/website/templates/docs/content.html @@ -34,7 +34,7 @@ {% if single_page and page.content %} (function() { - {% for chunk in page.content|chunks %} + {% for chunk in page.content|adjust_markdown_html|chunks %} document.write({{ chunk|tojson|safe }}); {% endfor %} })(); From 7d382cc7b59d8659e9d28a2bf0b5c0211d51e82c Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 19 May 2020 19:03:10 +0300 Subject: [PATCH 453/738] Return missed command --- .../MergeTree/MergeTreeDataMergerMutator.cpp | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 0c7d749d786..8e6533575f3 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -1262,13 +1262,19 @@ void MergeTreeDataMergerMutator::splitMutationCommands( NameSet mutated_columns; for (const auto & command : commands) { - if (command.type == MutationCommand::Type::MATERIALIZE_INDEX || command.type == MutationCommand::Type::MATERIALIZE_TTL - || command.type == MutationCommand::Type::DELETE || command.type == MutationCommand::Type::UPDATE) + if (command.type == MutationCommand::Type::MATERIALIZE_INDEX + || command.type == MutationCommand::Type::MATERIALIZE_TTL + || command.type == MutationCommand::Type::DELETE + || command.type == MutationCommand::Type::UPDATE) { for_interpreter.push_back(command); for (const auto & [column_name, expr] : command.column_to_update_expression) mutated_columns.emplace(column_name); } + else if (command.type == MutationCommand::Type::DROP_INDEX) + { + for_file_renames.push_back(command); + } else if (part_columns.has(command.column_name)) { if (command.type == MutationCommand::Type::DROP_COLUMN) @@ -1277,7 +1283,8 @@ void MergeTreeDataMergerMutator::splitMutationCommands( } else if (command.type == MutationCommand::Type::RENAME_COLUMN) { - for_interpreter.push_back({ + for_interpreter.push_back( + { .type = MutationCommand::Type::READ_COLUMN, .column_name = command.rename_to, }); @@ -1299,11 +1306,17 @@ void MergeTreeDataMergerMutator::splitMutationCommands( { for (const auto & command : commands) { - if (command.type == MutationCommand::Type::MATERIALIZE_INDEX || command.type == MutationCommand::Type::MATERIALIZE_TTL - || command.type == MutationCommand::Type::DELETE || command.type == MutationCommand::Type::UPDATE) + if (command.type == MutationCommand::Type::MATERIALIZE_INDEX + || command.type == MutationCommand::Type::MATERIALIZE_TTL + || command.type == MutationCommand::Type::DELETE + || command.type == MutationCommand::Type::UPDATE) { for_interpreter.push_back(command); } + else if (command.type == MutationCommand::Type::DROP_INDEX) + { + for_file_renames.push_back(command); + } /// If we don't have this column in source part, than we don't need /// to materialize it else if (part_columns.has(command.column_name)) From f18eedc5e785bcaa3aa65aa40a936cbe4fefd26e Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 19 May 2020 19:11:00 +0300 Subject: [PATCH 454/738] [website] set sentry environment --- website/js/base.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/website/js/base.js b/website/js/base.js index 46f06f70f30..de9ba3b5e43 100644 --- a/website/js/base.js +++ b/website/js/base.js @@ -1,5 +1,8 @@ (function () { - Sentry.init({ dsn: 'https://2b95b52c943f4ad99baccab7a9048e4d@o388870.ingest.sentry.io/5246103' }); + Sentry.init({ + dsn: 'https://2b95b52c943f4ad99baccab7a9048e4d@o388870.ingest.sentry.io/5246103', + environment: window.location.hostname === 'clickhouse.tech' ? 'prod' : 'test' + }); $(document).click(function (event) { var target = $(event.target); var target_id = target.attr('id'); From b50480cbde937e0e67c6c95b7cf956ff5342d99b Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Tue, 19 May 2020 18:55:37 +0200 Subject: [PATCH 455/738] Fixes the potential missed data during termination of Kafka engine table --- src/Storages/Kafka/KafkaBlockInputStream.cpp | 8 +- .../Kafka/ReadBufferFromKafkaConsumer.cpp | 9 +- .../Kafka/ReadBufferFromKafkaConsumer.h | 3 +- src/Storages/Kafka/StorageKafka.cpp | 5 +- tests/integration/test_storage_kafka/test.py | 92 +++++++++++++++++++ 5 files changed, 110 insertions(+), 7 deletions(-) diff --git a/src/Storages/Kafka/KafkaBlockInputStream.cpp b/src/Storages/Kafka/KafkaBlockInputStream.cpp index 55ff8610941..9f19bd464ff 100644 --- a/src/Storages/Kafka/KafkaBlockInputStream.cpp +++ b/src/Storages/Kafka/KafkaBlockInputStream.cpp @@ -134,7 +134,11 @@ Block KafkaBlockInputStream::readImpl() auto new_rows = read_kafka_message(); - buffer->storeLastReadMessageOffset(); + // we can't store the offser after rebalance, when consumer is stalled, or if it's terminating + if (!buffer->storeLastReadMessageOffset()) { + total_rows = 0; + break; + } auto topic = buffer->currentTopic(); auto key = buffer->currentKey(); @@ -172,7 +176,7 @@ Block KafkaBlockInputStream::readImpl() } } - if (buffer->rebalanceHappened() || total_rows == 0) + if (total_rows == 0) return Block(); /// MATERIALIZED columns can be added here, but I think diff --git a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp index eff4161ffb6..aeda60e90a7 100644 --- a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp +++ b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp @@ -336,12 +336,17 @@ bool ReadBufferFromKafkaConsumer::nextImpl() return true; } -void ReadBufferFromKafkaConsumer::storeLastReadMessageOffset() +bool ReadBufferFromKafkaConsumer::storeLastReadMessageOffset() { - if (!stalled && !rebalance_happened) + if (!stalled && !rebalance_happened && !stopped) { consumer->store_offset(*(current - 1)); ++offsets_stored; + return true; + } + else + { + return false; } } diff --git a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h index c5b72ed6d7c..435b5f2a7c8 100644 --- a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h +++ b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h @@ -37,9 +37,8 @@ public: auto pollTimeout() const { return poll_timeout; } bool hasMorePolledMessages() const; - auto rebalanceHappened() const { return rebalance_happened; } - void storeLastReadMessageOffset(); + bool storeLastReadMessageOffset(); void resetToLastCommitted(const char * msg); // Return values for the message that's being read. diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index 5c4657403b7..793a9a29676 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -454,7 +454,10 @@ bool StorageKafka::streamToViews() else in = streams[0]; - copyData(*in, *block_io.out, &stream_cancelled); + // We can't cancel during copyData, as it's not aware of commits and other kafka-related stuff. + // It will be cancelled on underlying layer (kafka buffer) + std::atomic stub = {false}; + copyData(*in, *block_io.out, &stub); for (auto & stream : streams) stream->as()->commit(); diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py index cbe96df3c29..9154ad67c05 100644 --- a/tests/integration/test_storage_kafka/test.py +++ b/tests/integration/test_storage_kafka/test.py @@ -1241,6 +1241,98 @@ def test_exception_from_destructor(kafka_cluster): assert TSV(instance.query('SELECT 1')) == TSV('1') +@pytest.mark.timeout(120) +def test_commits_of_unprocessed_messages_on_drop(kafka_cluster): + messages = [json.dumps({'key': j+1, 'value': j+1}) for j in range(1)] + kafka_produce('commits_of_unprocessed_messages_on_drop', messages) + + instance.query(''' + DROP TABLE IF EXISTS test.destination; + CREATE TABLE test.destination ( + key UInt64, + value UInt64, + _topic String, + _key String, + _offset UInt64, + _partition UInt64, + _timestamp Nullable(DateTime), + _consumed_by LowCardinality(String) + ) + ENGINE = MergeTree() + ORDER BY key; + + CREATE TABLE test.kafka (key UInt64, value UInt64) + ENGINE = Kafka + SETTINGS kafka_broker_list = 'kafka1:19092', + kafka_topic_list = 'commits_of_unprocessed_messages_on_drop', + kafka_group_name = 'commits_of_unprocessed_messages_on_drop_test_group', + kafka_format = 'JSONEachRow', + kafka_max_block_size = 1000; + + CREATE MATERIALIZED VIEW test.kafka_consumer TO test.destination AS + SELECT + key, + value, + _topic, + _key, + _offset, + _partition, + _timestamp + FROM test.kafka; + ''') + + while int(instance.query("SELECT count() FROM test.destination")) == 0: + print("Waiting for test.kafka_consumer to start consume") + time.sleep(1) + + cancel = threading.Event() + + i = [2] + def produce(): + while not cancel.is_set(): + messages = [] + for _ in range(113): + messages.append(json.dumps({'key': i[0], 'value': i[0]})) + i[0] += 1 + kafka_produce('commits_of_unprocessed_messages_on_drop', messages) + time.sleep(1) + + kafka_thread = threading.Thread(target=produce) + kafka_thread.start() + time.sleep(12) + + instance.query(''' + DROP TABLE test.kafka; + ''') + + instance.query(''' + CREATE TABLE test.kafka (key UInt64, value UInt64) + ENGINE = Kafka + SETTINGS kafka_broker_list = 'kafka1:19092', + kafka_topic_list = 'commits_of_unprocessed_messages_on_drop', + kafka_group_name = 'commits_of_unprocessed_messages_on_drop_test_group', + kafka_format = 'JSONEachRow', + kafka_max_block_size = 10000; + ''') + + cancel.set() + time.sleep(15) + + #kafka_cluster.open_bash_shell('instance') + # SELECT key, _timestamp, _offset FROM test.destination where runningDifference(key) <> 1 ORDER BY key; + + result = instance.query('SELECT count(), uniqExact(key), max(key) FROM test.destination') + print(result) + + instance.query(''' + DROP TABLE test.kafka_consumer; + DROP TABLE test.destination; + ''') + + kafka_thread.join() + assert TSV(result) == TSV('{0}\t{0}\t{0}'.format(i[0]-1)), 'Missing data!' + + @pytest.mark.timeout(1200) def test_kafka_duplicates_when_commit_failed(kafka_cluster): messages = [json.dumps({'key': j+1, 'value': 'x' * 300}) for j in range(22)] From 34198336eddb7bd07ae7831652fb1fe9d5e02318 Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Tue, 19 May 2020 19:02:37 +0200 Subject: [PATCH 456/738] Style --- src/Storages/Kafka/KafkaBlockInputStream.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Storages/Kafka/KafkaBlockInputStream.cpp b/src/Storages/Kafka/KafkaBlockInputStream.cpp index 9f19bd464ff..4da5de69085 100644 --- a/src/Storages/Kafka/KafkaBlockInputStream.cpp +++ b/src/Storages/Kafka/KafkaBlockInputStream.cpp @@ -135,7 +135,8 @@ Block KafkaBlockInputStream::readImpl() auto new_rows = read_kafka_message(); // we can't store the offser after rebalance, when consumer is stalled, or if it's terminating - if (!buffer->storeLastReadMessageOffset()) { + if (!buffer->storeLastReadMessageOffset()) + { total_rows = 0; break; } From ce3c53a00a03ad5fe750902ff93f19ae20c1ad3c Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Sun, 22 Dec 2019 01:48:33 +0800 Subject: [PATCH 457/738] dictGet* with table name --- src/Interpreters/ActionsVisitor.cpp | 4 ++-- src/Interpreters/MarkTableIdentifiersVisitor.cpp | 2 +- src/Interpreters/misc.h | 7 +++++++ .../01018_ddl_dictionaries_select.reference | 4 ++++ .../0_stateless/01018_ddl_dictionaries_select.sql | 10 ++++++++++ 5 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 81f19d0b3dc..7bb2dda0ab4 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -471,7 +471,7 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & argument_types.push_back(column.type); argument_names.push_back(column.name); } - else if (identifier && node.name == "joinGet" && arg == 0) + else if (identifier && functionIsJoinGetOrDictGet(node.name) && arg == 0) { auto table_id = IdentifierSemantic::extractDatabaseAndTable(*identifier); table_id = data.context.resolveStorageID(table_id, Context::ResolveOrdinary); @@ -480,7 +480,7 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & ColumnWithTypeAndName column( ColumnConst::create(std::move(column_string), 1), std::make_shared(), - data.getUniqueName("__joinGet")); + data.getUniqueName("__joinGetOrDictGet")); data.addAction(ExpressionAction::addColumn(column)); argument_types.push_back(column.type); argument_names.push_back(column.name); diff --git a/src/Interpreters/MarkTableIdentifiersVisitor.cpp b/src/Interpreters/MarkTableIdentifiersVisitor.cpp index c68a7373680..93477064862 100644 --- a/src/Interpreters/MarkTableIdentifiersVisitor.cpp +++ b/src/Interpreters/MarkTableIdentifiersVisitor.cpp @@ -44,7 +44,7 @@ void MarkTableIdentifiersMatcher::visit(const ASTFunction & func, ASTPtr &, Data } // first argument of joinGet can be a table identifier - if (func.name == "joinGet") + if (functionIsJoinGetOrDictGet(func.name)) { auto & ast = func.arguments->children.at(0); if (auto opt_name = tryGetIdentifierName(ast)) diff --git a/src/Interpreters/misc.h b/src/Interpreters/misc.h index 0fd0e12a4bb..97cfbf252f9 100644 --- a/src/Interpreters/misc.h +++ b/src/Interpreters/misc.h @@ -1,5 +1,7 @@ #pragma once +#include + namespace DB { @@ -18,4 +20,9 @@ inline bool functionIsLikeOperator(const std::string & name) return name == "like" || name == "notLike"; } +inline bool functionIsJoinGetOrDictGet(const std::string & name) +{ + return name == "joinGet" || startsWith(name, "dictGet"); +} + } diff --git a/tests/queries/0_stateless/01018_ddl_dictionaries_select.reference b/tests/queries/0_stateless/01018_ddl_dictionaries_select.reference index a7f56f9ee03..7c13bf305ee 100644 --- a/tests/queries/0_stateless/01018_ddl_dictionaries_select.reference +++ b/tests/queries/0_stateless/01018_ddl_dictionaries_select.reference @@ -17,3 +17,7 @@ database_for_dict dict1 ComplexKeyCache database_for_dict dict2 Hashed 6 6 +6 +6 +6 +6 diff --git a/tests/queries/0_stateless/01018_ddl_dictionaries_select.sql b/tests/queries/0_stateless/01018_ddl_dictionaries_select.sql index 9b8edab4d41..f3dd7682ffd 100644 --- a/tests/queries/0_stateless/01018_ddl_dictionaries_select.sql +++ b/tests/queries/0_stateless/01018_ddl_dictionaries_select.sql @@ -105,6 +105,16 @@ LAYOUT(HASHED()); SELECT dictGetString('database_for_dict.dict3', 'some_column', toUInt64(12)); +-- dictGet with table name +USE database_for_dict; +SELECT dictGetString(dict3, 'some_column', toUInt64(12)); +SELECT dictGetString(database_for_dict.dict3, 'some_column', toUInt64(12)); +SELECT dictGetString(default.dict3, 'some_column', toUInt64(12)); -- {serverError 36} +SELECT dictGet(dict3, 'some_column', toUInt64(12)); +SELECT dictGet(database_for_dict.dict3, 'some_column', toUInt64(12)); +SELECT dictGet(default.dict3, 'some_column', toUInt64(12)); -- {serverError 36} +USE default; + DROP TABLE database_for_dict.table_for_dict; SYSTEM RELOAD DICTIONARIES; -- {serverError 60} From 57bd7afc0316c4649b0385f959d04374668be7da Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Tue, 19 May 2020 21:12:30 +0300 Subject: [PATCH 458/738] Fix passing dictionary's name to dictGet() with alias. --- src/Interpreters/ActionsVisitor.cpp | 4 ++-- src/Interpreters/MarkTableIdentifiersVisitor.cpp | 14 ++++++++------ src/Interpreters/misc.h | 7 ++++++- .../01018_ddl_dictionaries_select.reference | 1 + .../0_stateless/01018_ddl_dictionaries_select.sql | 3 +++ 5 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 7bb2dda0ab4..3fa53e3e694 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -471,7 +471,7 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & argument_types.push_back(column.type); argument_names.push_back(column.name); } - else if (identifier && functionIsJoinGetOrDictGet(node.name) && arg == 0) + else if (identifier && (functionIsJoinGet(node.name) || functionIsDictGet(node.name)) && arg == 0) { auto table_id = IdentifierSemantic::extractDatabaseAndTable(*identifier); table_id = data.context.resolveStorageID(table_id, Context::ResolveOrdinary); @@ -480,7 +480,7 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & ColumnWithTypeAndName column( ColumnConst::create(std::move(column_string), 1), std::make_shared(), - data.getUniqueName("__joinGetOrDictGet")); + data.getUniqueName("__" + node.name)); data.addAction(ExpressionAction::addColumn(column)); argument_types.push_back(column.type); argument_names.push_back(column.name); diff --git a/src/Interpreters/MarkTableIdentifiersVisitor.cpp b/src/Interpreters/MarkTableIdentifiersVisitor.cpp index 93477064862..c7b8701c4a0 100644 --- a/src/Interpreters/MarkTableIdentifiersVisitor.cpp +++ b/src/Interpreters/MarkTableIdentifiersVisitor.cpp @@ -38,16 +38,18 @@ void MarkTableIdentifiersMatcher::visit(const ASTFunction & func, ASTPtr &, Data if (functionIsInOrGlobalInOperator(func.name)) { auto & ast = func.arguments->children.at(1); - if (auto opt_name = tryGetIdentifierName(ast)) - if (!data.aliases.count(*opt_name)) - setIdentifierSpecial(ast); + auto opt_name = tryGetIdentifierName(ast); + if (opt_name && !data.aliases.count(*opt_name)) + setIdentifierSpecial(ast); } - // first argument of joinGet can be a table identifier - if (functionIsJoinGetOrDictGet(func.name)) + // First argument of joinGet can be a table name, perhaps with a database. + // First argument of dictGet can be a dictionary name, perhaps with a database. + if (functionIsJoinGet(func.name) || functionIsDictGet(func.name)) { auto & ast = func.arguments->children.at(0); - if (auto opt_name = tryGetIdentifierName(ast)) + auto opt_name = tryGetIdentifierName(ast); + if (opt_name && !data.aliases.count(*opt_name)) setIdentifierSpecial(ast); } } diff --git a/src/Interpreters/misc.h b/src/Interpreters/misc.h index 97cfbf252f9..bc994194336 100644 --- a/src/Interpreters/misc.h +++ b/src/Interpreters/misc.h @@ -20,9 +20,14 @@ inline bool functionIsLikeOperator(const std::string & name) return name == "like" || name == "notLike"; } -inline bool functionIsJoinGetOrDictGet(const std::string & name) +inline bool functionIsJoinGet(const std::string & name) { return name == "joinGet" || startsWith(name, "dictGet"); } +inline bool functionIsDictGet(const std::string & name) +{ + return startsWith(name, "dictGet") || (name == "dictHas") || (name == "dictIsIn"); +} + } diff --git a/tests/queries/0_stateless/01018_ddl_dictionaries_select.reference b/tests/queries/0_stateless/01018_ddl_dictionaries_select.reference index 7c13bf305ee..28ee2a8690b 100644 --- a/tests/queries/0_stateless/01018_ddl_dictionaries_select.reference +++ b/tests/queries/0_stateless/01018_ddl_dictionaries_select.reference @@ -20,4 +20,5 @@ database_for_dict dict2 Hashed 6 6 6 +database_for_dict.dict3 6 6 diff --git a/tests/queries/0_stateless/01018_ddl_dictionaries_select.sql b/tests/queries/0_stateless/01018_ddl_dictionaries_select.sql index f3dd7682ffd..7f2e1d08671 100644 --- a/tests/queries/0_stateless/01018_ddl_dictionaries_select.sql +++ b/tests/queries/0_stateless/01018_ddl_dictionaries_select.sql @@ -115,6 +115,9 @@ SELECT dictGet(database_for_dict.dict3, 'some_column', toUInt64(12)); SELECT dictGet(default.dict3, 'some_column', toUInt64(12)); -- {serverError 36} USE default; +-- alias should be handled correctly +SELECT 'database_for_dict.dict3' as n, dictGet(n, 'some_column', toUInt64(12)); + DROP TABLE database_for_dict.table_for_dict; SYSTEM RELOAD DICTIONARIES; -- {serverError 60} From bc34e0ff94fb30414ce8b0a77dc294c04c33ef6a Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 19 May 2020 23:12:10 +0300 Subject: [PATCH 459/738] fixup --- .../compose/docker_compose_mongo.yml | 1 + src/Dictionaries/DictionaryFactory.cpp | 5 ++++ src/Dictionaries/MongoDBDictionarySource.cpp | 27 ++++++++++++++++++- src/Dictionaries/MongoDBDictionarySource.h | 8 +++--- .../external_sources.py | 5 ++++ .../test.py | 4 +-- 6 files changed, 43 insertions(+), 7 deletions(-) diff --git a/docker/test/integration/compose/docker_compose_mongo.yml b/docker/test/integration/compose/docker_compose_mongo.yml index a593c3e123b..9137f0aebd1 100644 --- a/docker/test/integration/compose/docker_compose_mongo.yml +++ b/docker/test/integration/compose/docker_compose_mongo.yml @@ -8,3 +8,4 @@ services: MONGO_INITDB_ROOT_PASSWORD: clickhouse ports: - 27018:27017 + command: --profile=2 --verbose diff --git a/src/Dictionaries/DictionaryFactory.cpp b/src/Dictionaries/DictionaryFactory.cpp index 1e985e81969..326e0f08d89 100644 --- a/src/Dictionaries/DictionaryFactory.cpp +++ b/src/Dictionaries/DictionaryFactory.cpp @@ -5,6 +5,8 @@ #include "DictionaryStructure.h" #include "getDictionaryConfigurationFromAST.h" +#include + namespace DB { namespace ErrorCodes @@ -41,6 +43,9 @@ DictionaryPtr DictionaryFactory::create( const DictionaryStructure dict_struct{config, config_prefix + ".structure"}; DictionarySourcePtr source_ptr = DictionarySourceFactory::instance().create(name, config, config_prefix + ".source", dict_struct, context, check_source_config); + LOG_TRACE(&Poco::Logger::get("DictionaryFactory"), + "Created dictionary source '" << source_ptr->toString() + << "' for dictionary '" << name << "'"); const auto & layout_type = keys.front(); diff --git a/src/Dictionaries/MongoDBDictionarySource.cpp b/src/Dictionaries/MongoDBDictionarySource.cpp index d9601f29a03..a4bd22cd848 100644 --- a/src/Dictionaries/MongoDBDictionarySource.cpp +++ b/src/Dictionaries/MongoDBDictionarySource.cpp @@ -35,11 +35,13 @@ void registerDictionarySourceMongoDB(DictionarySourceFactory & factory) } +#include #include #include #include #include #include +#include #include #include @@ -187,15 +189,38 @@ MongoDBDictionarySource::MongoDBDictionarySource( , db{db_} , collection{collection_} , sample_block{sample_block_} - , connection{std::make_shared(host, port)} + , connection{std::make_shared()} { if (!uri.empty()) { + Poco::URI poco_uri(uri); + + // Parse database from URI. This is required for correctness -- the + // cursor is created using database name and colleciton name, so we have + // to specify them properly. + db = poco_uri.getPath(); + // getPath() may return a leading slash, remove it. + if (!db.empty() && db[0] == '/') + { + db.erase(0, 1); + } + + // Parse some other parts from URI, for logging and display purposes. + host = poco_uri.getHost(); + port = poco_uri.getPort(); + user = poco_uri.getUserInfo(); + if (size_t separator = user.find(':'); separator != std::string::npos) + { + user.resize(separator); + } + + // Connect with URI. Poco::MongoDB::Connection::SocketFactory socket_factory; connection->connect(uri, socket_factory); } else { + // Connect with host/port/user/etc. connection->connect(host, port); if (!user.empty()) { diff --git a/src/Dictionaries/MongoDBDictionarySource.h b/src/Dictionaries/MongoDBDictionarySource.h index d90f28e1e74..36c9e82474c 100644 --- a/src/Dictionaries/MongoDBDictionarySource.h +++ b/src/Dictionaries/MongoDBDictionarySource.h @@ -72,12 +72,12 @@ public: private: const DictionaryStructure dict_struct; const std::string uri; - const std::string host; - const UInt16 port; - const std::string user; + std::string host; + UInt16 port; + std::string user; const std::string password; const std::string method; - const std::string db; + std::string db; const std::string collection; Block sample_block; diff --git a/tests/integration/test_dictionaries_all_layouts_and_sources/external_sources.py b/tests/integration/test_dictionaries_all_layouts_and_sources/external_sources.py index 7d1ded04bdc..7f8a480704c 100644 --- a/tests/integration/test_dictionaries_all_layouts_and_sources/external_sources.py +++ b/tests/integration/test_dictionaries_all_layouts_and_sources/external_sources.py @@ -179,6 +179,11 @@ class SourceMongo(ExternalSource): result = tbl.insert_many(to_insert) class SourceMongoURI(SourceMongo): + def compatible_with_layout(self, layout): + # It is enough to test one layout for this dictionary, since we're + # only testing that the connection with URI works. + return layout.name == 'flat' + def get_source_str(self, table_name): return ''' diff --git a/tests/integration/test_dictionaries_all_layouts_and_sources/test.py b/tests/integration/test_dictionaries_all_layouts_and_sources/test.py index 36034bab357..9cf5806b9e8 100644 --- a/tests/integration/test_dictionaries_all_layouts_and_sources/test.py +++ b/tests/integration/test_dictionaries_all_layouts_and_sources/test.py @@ -106,9 +106,9 @@ VALUES = { LAYOUTS = [ + Layout("flat"), Layout("hashed"), Layout("cache"), - Layout("flat"), Layout("complex_key_hashed"), Layout("complex_key_cache"), Layout("range_hashed"), @@ -117,7 +117,7 @@ LAYOUTS = [ SOURCES = [ SourceMongo("MongoDB", "localhost", "27018", "mongo1", "27017", "root", "clickhouse"), - SourceMongoURI("MongoDB", "localhost", "27018", "mongo1", "27017", "root", "clickhouse"), + SourceMongoURI("MongoDB_URI", "localhost", "27018", "mongo1", "27017", "root", "clickhouse"), SourceMySQL("MySQL", "localhost", "3308", "mysql1", "3306", "root", "clickhouse"), SourceClickHouse("RemoteClickHouse", "localhost", "9000", "clickhouse1", "9000", "default", ""), SourceClickHouse("LocalClickHouse", "localhost", "9000", "node", "9000", "default", ""), From 7ba5419190ff8fe38f9d841882d45e27978a5308 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 19 May 2020 23:24:06 +0300 Subject: [PATCH 460/738] Update PipelineExecutor. --- src/Processors/Executors/PipelineExecutor.cpp | 163 +++++++++++------- src/Processors/Executors/PipelineExecutor.h | 26 ++- 2 files changed, 127 insertions(+), 62 deletions(-) diff --git a/src/Processors/Executors/PipelineExecutor.cpp b/src/Processors/Executors/PipelineExecutor.cpp index 687736eef61..59f37115333 100644 --- a/src/Processors/Executors/PipelineExecutor.cpp +++ b/src/Processors/Executors/PipelineExecutor.cpp @@ -491,6 +491,34 @@ void PipelineExecutor::execute(size_t num_threads) throw; } + finalizeExecution(); +} + +bool PipelineExecutor::executeStep(std::atomic_bool * yield_flag) +{ + if (finished) + return false; + + if (!is_execution_initialized) + initExecution(1); + + executeStep(yield_flag); + + if (!finished) + return true; + + /// Execution can be stopped because of exception. Check and rethrow if any. + for (auto & node : graph) + if (node.execution_state->exception) + std::rethrow_exception(node.execution_state->exception); + + finalizeExecution(); + + return false; +} + +void PipelineExecutor::finalizeExecution() +{ if (process_list_element && process_list_element->isKilled()) throw Exception("Query was cancelled", ErrorCodes::QUERY_WAS_CANCELLED); @@ -506,33 +534,39 @@ void PipelineExecutor::execute(size_t num_threads) throw Exception("Pipeline stuck. Current state:\n" + dumpPipeline(), ErrorCodes::LOGICAL_ERROR); } +void PipelineExecutor::wakeUpExecutor(size_t thread_num) +{ + std::lock_guard guard(executor_contexts[thread_num]->mutex); + executor_contexts[thread_num]->wake_flag = true; + executor_contexts[thread_num]->condvar.notify_one(); +} + void PipelineExecutor::executeSingleThread(size_t thread_num, size_t num_threads) { -#ifndef NDEBUG - UInt64 total_time_ns = 0; - UInt64 execution_time_ns = 0; - UInt64 processing_time_ns = 0; - UInt64 wait_time_ns = 0; + executeStepImpl(thread_num, num_threads); +#ifndef NDEBUG + auto & context = executor_contexts[thread_num]; + LOG_TRACE(log, std::fixed << std::setprecision(3) + << "Thread finished." + << " Total time: " << (context.total_time_ns / 1e9) << " sec." + << " Execution time: " << (context.execution_time_ns / 1e9) << " sec." + << " Processing time: " << (context.processing_time_ns / 1e9) << " sec." + << " Wait time: " << (context.wait_time_ns / 1e9) << " sec."); +#endif +} + +void PipelineExecutor::executeStepImpl(size_t thread_num, size_t num_threads, std::atomic_bool * yield_flag) +{ +#ifndef NDEBUG Stopwatch total_time_watch; #endif - ExecutionState * state = nullptr; + auto & context = executor_contexts[thread_num]; + auto & state = context->state; + bool yield = false; - auto prepare_processor = [&](UInt64 pid, Queue & queue) - { - if (!prepareProcessor(pid, thread_num, queue, std::unique_lock(*graph[pid].status_mutex))) - finish(); - }; - - auto wake_up_executor = [&](size_t executor) - { - std::lock_guard guard(executor_contexts[executor]->mutex); - executor_contexts[executor]->wake_flag = true; - executor_contexts[executor]->condvar.notify_one(); - }; - - while (!finished) + while (!finished && !yield) { /// First, find any processor to execute. /// Just travers graph and prepare any processor. @@ -555,7 +589,7 @@ void PipelineExecutor::executeSingleThread(size_t thread_num, size_t num_threads thread_to_wake = threads_queue.pop_any(); lock.unlock(); - wake_up_executor(thread_to_wake); + wakeUpExecutor(thread_to_wake); } break; @@ -572,21 +606,21 @@ void PipelineExecutor::executeSingleThread(size_t thread_num, size_t num_threads } { - std::unique_lock lock(executor_contexts[thread_num]->mutex); + std::unique_lock lock(context->mutex); - executor_contexts[thread_num]->condvar.wait(lock, [&] + context->condvar.wait(lock, [&] { - return finished || executor_contexts[thread_num]->wake_flag; + return finished || context->wake_flag; }); - executor_contexts[thread_num]->wake_flag = false; + context->wake_flag = false; } } if (finished) break; - while (state) + while (state && !yield) { if (finished) break; @@ -601,7 +635,7 @@ void PipelineExecutor::executeSingleThread(size_t thread_num, size_t num_threads state->job(); #ifndef NDEBUG - execution_time_ns += execution_time_watch.elapsed(); + context->execution_time_ns += execution_time_watch.elapsed(); #endif } @@ -623,8 +657,13 @@ void PipelineExecutor::executeSingleThread(size_t thread_num, size_t num_threads while (auto * task = expand_pipeline_task.load()) doExpandPipeline(task, true); - /// Execute again if can. - prepare_processor(state->processors_id, queue); + /// Prepare processor after execution. + { + auto lock = std::unique_lock(*graph[state->processors_id].status_mutex); + if (!prepareProcessor(state->processors_id, thread_num, queue, std::move(lock))) + finish(); + } + state = nullptr; /// Take local task from queue if has one. @@ -656,7 +695,7 @@ void PipelineExecutor::executeSingleThread(size_t thread_num, size_t num_threads lock.unlock(); - wake_up_executor(thread_to_wake); + wakeUpExecutor(thread_to_wake); } } @@ -666,27 +705,24 @@ void PipelineExecutor::executeSingleThread(size_t thread_num, size_t num_threads } #ifndef NDEBUG - processing_time_ns += processing_time_watch.elapsed(); + context->processing_time_ns += processing_time_watch.elapsed(); #endif + + /// We have executed single processor. Check if we need to yield execution. + if (yield_flag && *yield_flag) + yield = true; } } #ifndef NDEBUG - total_time_ns = total_time_watch.elapsed(); - wait_time_ns = total_time_ns - execution_time_ns - processing_time_ns; - - LOG_TRACE(log, std::fixed << std::setprecision(3) - << "Thread finished." - << " Total time: " << (total_time_ns / 1e9) << " sec." - << " Execution time: " << (execution_time_ns / 1e9) << " sec." - << " Processing time: " << (processing_time_ns / 1e9) << " sec." - << " Wait time: " << (wait_time_ns / 1e9) << " sec."); + context->total_time_ns += total_time_watch.elapsed(); + context->wait_time_ns = total_time_ns - execution_time_ns - processing_time_ns; #endif } -void PipelineExecutor::executeImpl(size_t num_threads) +void PipelineExecutor::initExecution(size_t num_threads) { - Stack stack; + is_execution_initialized = true; threads_queue.init(num_threads); task_queue.init(num_threads); @@ -699,25 +735,7 @@ void PipelineExecutor::executeImpl(size_t num_threads) executor_contexts.emplace_back(std::make_unique()); } - auto thread_group = CurrentThread::getGroup(); - - using ThreadsData = std::vector; - ThreadsData threads; - threads.reserve(num_threads); - - bool finished_flag = false; - - SCOPE_EXIT( - if (!finished_flag) - { - finish(); - - for (auto & thread : threads) - if (thread.joinable()) - thread.join(); - } - ); - + Stack stack; addChildlessProcessorsToStack(stack); { @@ -744,9 +762,32 @@ void PipelineExecutor::executeImpl(size_t num_threads) } } } +} + +void PipelineExecutor::executeImpl(size_t num_threads) +{ + initExecution(num_threads); + + using ThreadsData = std::vector; + ThreadsData threads; + threads.reserve(num_threads); + + bool finished_flag = false; + + SCOPE_EXIT( + if (!finished_flag) + { + finish(); + + for (auto & thread : threads) + if (thread.joinable()) + thread.join(); + } + ); if (num_threads > 1) { + auto thread_group = CurrentThread::getGroup(); for (size_t i = 0; i < num_threads; ++i) { diff --git a/src/Processors/Executors/PipelineExecutor.h b/src/Processors/Executors/PipelineExecutor.h index c0ce9053e5a..0a15f5cc974 100644 --- a/src/Processors/Executors/PipelineExecutor.h +++ b/src/Processors/Executors/PipelineExecutor.h @@ -35,6 +35,11 @@ public: /// In case of exception during execution throws any occurred. void execute(size_t num_threads); + /// Execute single step. Step will be stopped when yield_flag is true. + /// Execution is happened in single thread. + /// Return true if execution should be continued. + bool executeStep(std::atomic_bool * yield_flag = nullptr); + String getName() const { return "PipelineExecutor"; } const Processors & getProcessors() const { return processors; } @@ -203,6 +208,8 @@ private: ThreadsQueue threads_queue; std::mutex task_queue_mutex; + /// Flag that checks that initExecution was called. + bool is_execution_initialized = false; std::atomic_bool cancelled; std::atomic_bool finished; @@ -235,7 +242,17 @@ private: std::mutex mutex; bool wake_flag = false; - /// std::queue pinned_tasks; + /// Currently processing state. + ExecutionState * state = nullptr; + +#ifndef NDEBUG + /// Time for different processing stages. + UInt64 total_time_ns = 0; + UInt64 execution_time_ns = 0; + UInt64 processing_time_ns = 0; + UInt64 wait_time_ns = 0; +#endif + }; std::vector> executor_contexts; @@ -267,7 +284,14 @@ private: bool prepareProcessor(UInt64 pid, size_t thread_number, Queue & queue, std::unique_lock node_lock); bool doExpandPipeline(ExpandPipelineTask * task, bool processing); + /// Continue executor (in case there are tasks in queue). + void wakeUpExecutor(size_t thread_num); + + void initExecution(size_t num_threads); /// Initialize executor contexts and task_queue. + void finalizeExecution(); /// Check all processors are finished. + void executeImpl(size_t num_threads); + void executeStepImpl(size_t thread_num, size_t num_threads, std::atomic_bool * yield_flag = nullptr); void executeSingleThread(size_t thread_num, size_t num_threads); void finish(); From 994884399c39d6caa12dc15ad5e695b1559fe66a Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 20 May 2020 00:47:48 +0300 Subject: [PATCH 461/738] trigger ci --- docs/en/faq/index.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/en/faq/index.md b/docs/en/faq/index.md index 2ee9d51e83b..11395301904 100644 --- a/docs/en/faq/index.md +++ b/docs/en/faq/index.md @@ -4,3 +4,4 @@ toc_priority: 76 --- +{## [Original article](https://clickhouse.tech/docs/en/faq) ##} From 8eac46c79f7be28f4084dc0fd498a7365b76907e Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 20 May 2020 00:49:43 +0300 Subject: [PATCH 462/738] [docs] mark submitting feedback as Metrika goal (#11046) --- website/js/docs.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/website/js/docs.js b/website/js/docs.js index f5a497ca033..f6d11497005 100644 --- a/website/js/docs.js +++ b/website/js/docs.js @@ -157,6 +157,11 @@ $(document).ready(function () { dataType: 'json', data: JSON.stringify({rating: $(this).prevAll().length + 1}), success: function () { + try { + window.yaCounter18343495.reachGoal('docs_feedback'); + } catch (e) { + Sentry.captureException(e); + } }, error: function () { rating_stars.removeClass('text-orange'); From bd3e7307d068cb6b826be9ba962aef66e6b864c8 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Wed, 20 May 2020 01:43:16 +0300 Subject: [PATCH 463/738] fix integration test --- tests/integration/test_ttl_move/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_ttl_move/test.py b/tests/integration/test_ttl_move/test.py index ab348ea0cb1..243268260d0 100644 --- a/tests/integration/test_ttl_move/test.py +++ b/tests/integration/test_ttl_move/test.py @@ -626,7 +626,7 @@ def test_materialize_ttl_in_partition(started_cluster, name, engine): node1.query(""" ALTER TABLE {name} MODIFY TTL - d1 TO DISK 'external' + d1 TO DISK 'external' SETTINGS materialize_ttl_after_modify = 0 """.format(name=name)) time.sleep(0.5) From 5e84769c741d48b343b87848802d978c4142a9a6 Mon Sep 17 00:00:00 2001 From: BayoNet Date: Wed, 20 May 2020 08:35:51 +0300 Subject: [PATCH 464/738] DOCS-628: Stable versions of statistical functions v2 (#11047) * CLICKHOUSEDOCS-628: Restore deleted changes. * CLICKHOUSEDOCS-628: pillar * CLICKHOUSEDOCS-628: Fixed grammar. * CLICKHOUSEDOCS-628: Translated #8933 Co-authored-by: Sergei Shtykov --- .../aggregate-functions/reference.md | 21 +++++++++++++++++ .../mergetree-family/summingmergetree.md | 2 +- .../third-party/client-libraries.md | 1 + .../aggregate-functions/reference.md | 23 +++++++++++++++++++ 4 files changed, 46 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/aggregate-functions/reference.md b/docs/en/sql-reference/aggregate-functions/reference.md index 7aa37d67dbe..4c505a46fd1 100644 --- a/docs/en/sql-reference/aggregate-functions/reference.md +++ b/docs/en/sql-reference/aggregate-functions/reference.md @@ -1543,20 +1543,32 @@ It represents an unbiased estimate of the variance of a random variable if passe Returns `Float64`. When `n <= 1`, returns `+∞`. +!!! note "Note" + This function uses a numerically unstable algorithm. If you need [numerical stability](https://en.wikipedia.org/wiki/Numerical_stability) in calculations, use the `varSampStable` function. It works slower but provides a lower computational error. + ## varPop(x) {#varpopx} Calculates the amount `Σ((x - x̅)^2) / n`, where `n` is the sample size and `x̅`is the average value of `x`. In other words, dispersion for a set of values. Returns `Float64`. +!!! note "Note" + This function uses a numerically unstable algorithm. If you need [numerical stability](https://en.wikipedia.org/wiki/Numerical_stability) in calculations, use the `varPopStable` function. It works slower but provides a lower computational error. + ## stddevSamp(x) {#stddevsampx} The result is equal to the square root of `varSamp(x)`. +!!! note "Note" + This function uses a numerically unstable algorithm. If you need [numerical stability](https://en.wikipedia.org/wiki/Numerical_stability) in calculations, use the `stddevSampStable` function. It works slower but provides a lower computational error. + ## stddevPop(x) {#stddevpopx} The result is equal to the square root of `varPop(x)`. +!!! note "Note" + This function uses a numerically unstable algorithm. If you need [numerical stability](https://en.wikipedia.org/wiki/Numerical_stability) in calculations, use the `stddevPopStable` function. It works slower but provides a lower computational error. + ## topK(N)(x) {#topknx} Returns an array of the approximately most frequent values in the specified column. The resulting array is sorted in descending order of approximate frequency of values (not by the values themselves). @@ -1641,14 +1653,23 @@ Calculates the value of `Σ((x - x̅)(y - y̅)) / (n - 1)`. Returns Float64. When `n <= 1`, returns +∞. +!!! note "Note" + This function uses a numerically unstable algorithm. If you need [numerical stability](https://en.wikipedia.org/wiki/Numerical_stability) in calculations, use the `covarSampStable` function. It works slower but provides a lower computational error. + ## covarPop(x, y) {#covarpopx-y} Calculates the value of `Σ((x - x̅)(y - y̅)) / n`. +!!! note "Note" + This function uses a numerically unstable algorithm. If you need [numerical stability](https://en.wikipedia.org/wiki/Numerical_stability) in calculations, use the `covarPopStable` function. It works slower but provides a lower computational error. + ## corr(x, y) {#corrx-y} Calculates the Pearson correlation coefficient: `Σ((x - x̅)(y - y̅)) / sqrt(Σ((x - x̅)^2) * Σ((y - y̅)^2))`. +!!! note "Note" + This function uses a numerically unstable algorithm. If you need [numerical stability](https://en.wikipedia.org/wiki/Numerical_stability) in calculations, use the `corrStable` function. It works slower but provides a lower computational error. + ## categoricalInformationValue {#categoricalinformationvalue} Calculates the value of `(P(tag = 1) - P(tag = 0))(log(P(tag = 1)) - log(P(tag = 0)))` for each category. diff --git a/docs/ru/engines/table-engines/mergetree-family/summingmergetree.md b/docs/ru/engines/table-engines/mergetree-family/summingmergetree.md index c4c34fed0eb..070cd109c30 100644 --- a/docs/ru/engines/table-engines/mergetree-family/summingmergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/summingmergetree.md @@ -113,7 +113,7 @@ ClickHouse может слить куски данных таким образо Если название вложенной таблицы заканчивается на `Map` и она содержит не менее двух столбцов, удовлетворяющих критериям: -- первый столбец - числовой `(*Int*, Date, DateTime)`, назовем его условно `key`, +- первый столбец - числовой `(*Int*, Date, DateTime)` или строковый `(String, FixedString)`, назовем его условно `key`, - остальные столбцы - арифметические `(*Int*, Float32/64)`, условно `(values...)`, то вложенная таблица воспринимается как отображение `key => (values...)` и при слиянии её строк выполняется слияние элементов двух множеств по `key` со сложением соответствующих `(values...)`. diff --git a/docs/ru/interfaces/third-party/client-libraries.md b/docs/ru/interfaces/third-party/client-libraries.md index 27fb3f67512..928c1393c26 100644 --- a/docs/ru/interfaces/third-party/client-libraries.md +++ b/docs/ru/interfaces/third-party/client-libraries.md @@ -45,6 +45,7 @@ - [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net) - Elixir - [clickhousex](https://github.com/appodeal/clickhousex/) + - [pillar](https://github.com/sofakingworld/pillar) - Nim - [nim-clickhouse](https://github.com/leonardoce/nim-clickhouse) diff --git a/docs/ru/sql-reference/aggregate-functions/reference.md b/docs/ru/sql-reference/aggregate-functions/reference.md index 3705ac91180..612d073c484 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference.md +++ b/docs/ru/sql-reference/aggregate-functions/reference.md @@ -1533,20 +1533,33 @@ SELECT medianDeterministic(val, 1) FROM t Возвращает `Float64`. В случае, когда `n <= 1`, возвращается `+∞`. +!!! note "Примечание" + Функция использует вычислительно неустойчивый алгоритм. Если для ваших расчётов необходима [вычислительная устойчивость](https://ru.wikipedia.org/wiki/Вычислительная_устойчивость), используйте функцию `varSampStable`. Она работает медленнее, но обеспечиват меньшую вычислительную ошибку. + ## varPop(x) {#varpopx} Вычисляет величину `Σ((x - x̅)^2) / n`, где `n` - размер выборки, `x̅`- среднее значение `x`. То есть, дисперсию для множества значений. Возвращает `Float64`. +!!! note "Примечание" + Функция использует вычислительно неустойчивый алгоритм. Если для ваших расчётов необходима [вычислительная устойчивость](https://ru.wikipedia.org/wiki/Вычислительная_устойчивость), используйте функцию `varPopStable`. Она работает медленнее, но обеспечивает меньшую вычислительную ошибку. + ## stddevSamp(x) {#stddevsampx} Результат равен квадратному корню от `varSamp(x)`. +!!! note "Примечание" + Функция использует вычислительно неустойчивый алгоритм. Если для ваших расчётов необходима [вычислительная устойчивость](https://ru.wikipedia.org/wiki/Вычислительная_устойчивость), используйте функцию `stddevSampStable`. Она работает медленнее, но обеспечивает меньшую вычислительную ошибку. + ## stddevPop(x) {#stddevpopx} Результат равен квадратному корню от `varPop(x)`. +!!! note "Примечание" + Функция использует вычислительно неустойчивый алгоритм. Если для ваших расчётов необходима [вычислительная устойчивость](https://ru.wikipedia.org/wiki/Вычислительная_устойчивость), используйте функцию `stddevPopStable`. Она работает медленнее, но обеспечивает меньшую вычислительную ошибку. + + ## topK(N)(column) {#topkncolumn} Возвращает массив наиболее часто встречающихся значений в указанном столбце. Результирующий массив упорядочен по убыванию частоты значения (не по самим значениям). @@ -1626,14 +1639,24 @@ SELECT topKWeighted(10)(number, number) FROM numbers(1000) Возвращает Float64. В случае, когда `n <= 1`, возвращается +∞. +!!! note "Примечание" + Функция использует вычислительно неустойчивый алгоритм. Если для ваших расчётов необходима [вычислительная устойчивость](https://ru.wikipedia.org/wiki/Вычислительная_устойчивость), используйте функцию `covarSampStable`. Она работает медленнее, но обеспечивает меньшую вычислительную ошибку. + ## covarPop(x, y) {#covarpopx-y} Вычисляет величину `Σ((x - x̅)(y - y̅)) / n`. +!!! note "Примечание" + Функция использует вычислительно неустойчивый алгоритм. Если для ваших расчётов необходима [вычислительная устойчивость](https://ru.wikipedia.org/wiki/Вычислительная_устойчивость), используйте функцию `covarPopStable`. Она работает медленнее, но обеспечивает меньшую вычислительную ошибку. + + ## corr(x, y) {#corrx-y} Вычисляет коэффициент корреляции Пирсона: `Σ((x - x̅)(y - y̅)) / sqrt(Σ((x - x̅)^2) * Σ((y - y̅)^2))`. +!!! note "Примечание" + Функция использует вычислительно неустойчивый алгоритм. Если для ваших расчётов необходима [вычислительная устойчивость](https://ru.wikipedia.org/wiki/Вычислительная_устойчивость), используйте функцию `corrStable`. Она работает медленнее, но обеспечивает меньшую вычислительную ошибку. + ## simpleLinearRegression {#simplelinearregression} Выполняет простую (одномерную) линейную регрессию. From 2b8ccac0841a6e5787a9b9a9eb90cb6ca26461b8 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 20 May 2020 07:33:10 +0000 Subject: [PATCH 465/738] Bump mkdocs-macros-plugin from 0.4.7 to 0.4.9 in /docs/tools Bumps [mkdocs-macros-plugin](https://github.com/fralau/mkdocs_macros_plugin) from 0.4.7 to 0.4.9. - [Release notes](https://github.com/fralau/mkdocs_macros_plugin/releases) - [Commits](https://github.com/fralau/mkdocs_macros_plugin/commits) Signed-off-by: dependabot-preview[bot] --- docs/tools/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tools/requirements.txt b/docs/tools/requirements.txt index 79ee2e96f82..e4751272ebb 100644 --- a/docs/tools/requirements.txt +++ b/docs/tools/requirements.txt @@ -18,7 +18,7 @@ Markdown==3.2.1 MarkupSafe==1.1.1 mkdocs==1.1.2 mkdocs-htmlproofer-plugin==0.0.3 -mkdocs-macros-plugin==0.4.7 +mkdocs-macros-plugin==0.4.9 nltk==3.5 nose==1.3.7 protobuf==3.12.0 From e8a13842afc9e40c3a693d983d1edb1830ac2dcc Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Wed, 20 May 2020 11:02:02 +0200 Subject: [PATCH 466/738] Better fix, previous fix was wrong (was leaving the cycle by eof condition) --- src/Storages/Kafka/KafkaBlockInputStream.cpp | 9 ++---- .../Kafka/ReadBufferFromKafkaConsumer.cpp | 31 +++++++++++++------ .../Kafka/ReadBufferFromKafkaConsumer.h | 5 ++- 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/src/Storages/Kafka/KafkaBlockInputStream.cpp b/src/Storages/Kafka/KafkaBlockInputStream.cpp index 4da5de69085..a2403e66c50 100644 --- a/src/Storages/Kafka/KafkaBlockInputStream.cpp +++ b/src/Storages/Kafka/KafkaBlockInputStream.cpp @@ -134,12 +134,7 @@ Block KafkaBlockInputStream::readImpl() auto new_rows = read_kafka_message(); - // we can't store the offser after rebalance, when consumer is stalled, or if it's terminating - if (!buffer->storeLastReadMessageOffset()) - { - total_rows = 0; - break; - } + buffer->storeLastReadMessageOffset(); auto topic = buffer->currentTopic(); auto key = buffer->currentKey(); @@ -177,7 +172,7 @@ Block KafkaBlockInputStream::readImpl() } } - if (total_rows == 0) + if (buffer->polledDataUnusable() || total_rows == 0) return Block(); /// MATERIALIZED columns can be added here, but I think diff --git a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp index aeda60e90a7..ad9d660a989 100644 --- a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp +++ b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp @@ -250,11 +250,23 @@ void ReadBufferFromKafkaConsumer::resetToLastCommitted(const char * msg) /// Do commit messages implicitly after we processed the previous batch. bool ReadBufferFromKafkaConsumer::nextImpl() { + /// NOTE: ReadBuffer was implemented with an immutable underlying contents in mind. /// If we failed to poll any message once - don't try again. /// Otherwise, the |poll_timeout| expectations get flawn. - if (stalled || stopped || !allowed || rebalance_happened) + + // we can react on stop only during fetching data + // after block is formed (i.e. during copying data to MV / commiting) we ignore stop attempts + if (stopped) + { + was_stopped = true; + offsets_stored = 0; return false; + } + + if (stalled || was_stopped || !allowed || rebalance_happened) + return false; + if (current == messages.end()) { @@ -267,7 +279,13 @@ bool ReadBufferFromKafkaConsumer::nextImpl() /// Don't drop old messages immediately, since we may need them for virtual columns. auto new_messages = consumer->poll_batch(batch_size, std::chrono::milliseconds(poll_timeout)); - if (rebalance_happened) + if (stopped) + { + was_stopped = true; + offsets_stored = 0; + return false; + } + else if (rebalance_happened) { if (!new_messages.empty()) { @@ -336,17 +354,12 @@ bool ReadBufferFromKafkaConsumer::nextImpl() return true; } -bool ReadBufferFromKafkaConsumer::storeLastReadMessageOffset() +void ReadBufferFromKafkaConsumer::storeLastReadMessageOffset() { - if (!stalled && !rebalance_happened && !stopped) + if (!stalled && !was_stopped && !rebalance_happened) { consumer->store_offset(*(current - 1)); ++offsets_stored; - return true; - } - else - { - return false; } } diff --git a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h index 435b5f2a7c8..46dace827d0 100644 --- a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h +++ b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h @@ -37,8 +37,9 @@ public: auto pollTimeout() const { return poll_timeout; } bool hasMorePolledMessages() const; + bool polledDataUnusable() const { return (was_stopped || rebalance_happened); } - bool storeLastReadMessageOffset(); + void storeLastReadMessageOffset(); void resetToLastCommitted(const char * msg); // Return values for the message that's being read. @@ -68,6 +69,8 @@ private: bool rebalance_happened = false; + bool was_stopped = false; + // order is important, need to be destructed before consumer cppkafka::TopicPartitionList assignment; const Names topics; From 9b38324028fb33984431cb1bcc317b2d979eefa6 Mon Sep 17 00:00:00 2001 From: Artem Zuikov Date: Wed, 20 May 2020 12:29:23 +0300 Subject: [PATCH 467/738] fix JOIN with PREWHERE with second analyze() (#11051) --- src/Interpreters/ExpressionAnalyzer.cpp | 6 ++ src/Interpreters/ExpressionAnalyzer.h | 5 +- src/Interpreters/InterpreterSelectQuery.cpp | 58 +++++++------------ src/Interpreters/InterpreterSelectQuery.h | 2 +- src/Interpreters/SyntaxAnalyzer.cpp | 9 ++- src/Interpreters/TableJoin.cpp | 12 ++++ src/Interpreters/TableJoin.h | 1 + .../01281_join_with_prewhere_fix.reference | 8 +++ .../01281_join_with_prewhere_fix.sql | 21 +++++++ 9 files changed, 80 insertions(+), 42 deletions(-) create mode 100644 tests/queries/0_stateless/01281_join_with_prewhere_fix.reference create mode 100644 tests/queries/0_stateless/01281_join_with_prewhere_fix.sql diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 4a94bc83368..8bff2816df1 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -1025,6 +1025,12 @@ ExpressionAnalysisResult::ExpressionAnalysisResult( chain.clear(); }; + if (storage) + { + query_analyzer.makeSetsForIndex(query.where()); + query_analyzer.makeSetsForIndex(query.prewhere()); + } + { ExpressionActionsChain chain(context); Names additional_required_columns_after_prewhere; diff --git a/src/Interpreters/ExpressionAnalyzer.h b/src/Interpreters/ExpressionAnalyzer.h index 72a4ead6a14..1afb289430e 100644 --- a/src/Interpreters/ExpressionAnalyzer.h +++ b/src/Interpreters/ExpressionAnalyzer.h @@ -243,8 +243,6 @@ public: const NamesAndTypesList & aggregationKeys() const { return aggregation_keys; } const AggregateDescriptions & aggregates() const { return aggregate_descriptions; } - /// Create Set-s that we make from IN section to use index on them. - void makeSetsForIndex(const ASTPtr & node); const PreparedSets & getPreparedSets() const { return prepared_sets; } /// Tables that will need to be sent to remote servers for distributed query processing. @@ -275,6 +273,9 @@ private: */ SetPtr isPlainStorageSetInSubquery(const ASTPtr & subquery_or_table_name); + /// Create Set-s that we make from IN section to use index on them. + void makeSetsForIndex(const ASTPtr & node); + JoinPtr makeTableJoin(const ASTTablesInSelectQueryElement & join_element); const ASTSelectQuery * getAggregatingQuery() const; diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index aa124d9843f..1e300ae3c5a 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -309,12 +309,29 @@ InterpreterSelectQuery::InterpreterSelectQuery( ASTSelectQuery & query = getSelectQuery(); std::shared_ptr table_join = joined_tables.makeTableJoin(query); - auto analyze = [&] (bool try_move_to_prewhere = true) + ASTPtr row_policy_filter; + if (storage) + row_policy_filter = context->getRowPolicyCondition(table_id.getDatabaseName(), table_id.getTableName(), RowPolicy::SELECT_FILTER); + + auto analyze = [&] (bool try_move_to_prewhere) { syntax_analyzer_result = SyntaxAnalyzer(*context).analyzeSelect( query_ptr, SyntaxAnalyzerResult(source_header.getNamesAndTypesList(), storage), options, joined_tables.tablesWithColumns(), required_result_column_names, table_join); + if (try_move_to_prewhere && storage && !row_policy_filter && query.where() && !query.prewhere() && !query.final()) + { + /// PREWHERE optimization: transfer some condition from WHERE to PREWHERE if enabled and viable + if (const auto * merge_tree = dynamic_cast(storage.get())) + { + SelectQueryInfo current_info; + current_info.query = query_ptr; + current_info.syntax_analyzer_result = syntax_analyzer_result; + + MergeTreeWhereOptimizer{current_info, *context, *merge_tree, syntax_analyzer_result->requiredSourceColumns(), log}; + } + } + /// Save scalar sub queries's results in the query context if (!options.only_analyze && context->hasQueryContext()) for (const auto & it : syntax_analyzer_result->getScalars()) @@ -365,7 +382,6 @@ InterpreterSelectQuery::InterpreterSelectQuery( source_header = storage->getSampleBlockForColumns(required_columns); /// Fix source_header for filter actions. - auto row_policy_filter = context->getRowPolicyCondition(table_id.getDatabaseName(), table_id.getTableName(), RowPolicy::SELECT_FILTER); if (row_policy_filter) { filter_info = std::make_shared(); @@ -378,10 +394,10 @@ InterpreterSelectQuery::InterpreterSelectQuery( throw Exception("PREWHERE is not supported if the table is filtered by row-level security expression", ErrorCodes::ILLEGAL_PREWHERE); /// Calculate structure of the result. - result_header = getSampleBlockImpl(try_move_to_prewhere); + result_header = getSampleBlockImpl(); }; - analyze(); + analyze(settings.optimize_move_to_prewhere); bool need_analyze_again = false; if (analysis_result.prewhere_constant_filter_description.always_false || analysis_result.prewhere_constant_filter_description.always_true) @@ -481,40 +497,8 @@ QueryPipeline InterpreterSelectQuery::executeWithProcessors() } -Block InterpreterSelectQuery::getSampleBlockImpl(bool try_move_to_prewhere) +Block InterpreterSelectQuery::getSampleBlockImpl() { - auto & query = getSelectQuery(); - const Settings & settings = context->getSettingsRef(); - - /// Do all AST changes here, because actions from analysis_result will be used later in readImpl. - - if (storage) - { - query_analyzer->makeSetsForIndex(query.where()); - query_analyzer->makeSetsForIndex(query.prewhere()); - - /// PREWHERE optimization. - /// Turn off, if the table filter (row-level security) is applied. - if (!context->getRowPolicyCondition(table_id.getDatabaseName(), table_id.getTableName(), RowPolicy::SELECT_FILTER)) - { - auto optimize_prewhere = [&](auto & merge_tree) - { - SelectQueryInfo current_info; - current_info.query = query_ptr; - current_info.syntax_analyzer_result = syntax_analyzer_result; - current_info.sets = query_analyzer->getPreparedSets(); - - /// Try transferring some condition from WHERE to PREWHERE if enabled and viable - if (settings.optimize_move_to_prewhere && try_move_to_prewhere && query.where() && !query.prewhere() && !query.final()) - MergeTreeWhereOptimizer{current_info, *context, merge_tree, - syntax_analyzer_result->requiredSourceColumns(), log}; - }; - - if (const auto * merge_tree_data = dynamic_cast(storage.get())) - optimize_prewhere(*merge_tree_data); - } - } - if (storage && !options.only_analyze) from_stage = storage->getQueryProcessingStage(*context, options.to_stage, query_ptr); diff --git a/src/Interpreters/InterpreterSelectQuery.h b/src/Interpreters/InterpreterSelectQuery.h index 1415143dd63..87a95c5b836 100644 --- a/src/Interpreters/InterpreterSelectQuery.h +++ b/src/Interpreters/InterpreterSelectQuery.h @@ -106,7 +106,7 @@ private: ASTSelectQuery & getSelectQuery() { return query_ptr->as(); } - Block getSampleBlockImpl(bool try_move_to_prewhere); + Block getSampleBlockImpl(); struct Pipeline { diff --git a/src/Interpreters/SyntaxAnalyzer.cpp b/src/Interpreters/SyntaxAnalyzer.cpp index 84d11122de4..b3d566dbdc8 100644 --- a/src/Interpreters/SyntaxAnalyzer.cpp +++ b/src/Interpreters/SyntaxAnalyzer.cpp @@ -767,8 +767,13 @@ SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyzeSelect( const auto & settings = context.getSettingsRef(); const NameSet & source_columns_set = result.source_columns_set; - result.analyzed_join = table_join; - if (!result.analyzed_join) /// ExpressionAnalyzer expects some not empty object here + + if (table_join) + { + result.analyzed_join = table_join; + result.analyzed_join->resetCollected(); + } + else /// TODO: remove. For now ExpressionAnalyzer expects some not empty object here result.analyzed_join = std::make_shared(); if (remove_duplicates) diff --git a/src/Interpreters/TableJoin.cpp b/src/Interpreters/TableJoin.cpp index 02eb321fdd8..aa88df2e921 100644 --- a/src/Interpreters/TableJoin.cpp +++ b/src/Interpreters/TableJoin.cpp @@ -29,6 +29,18 @@ TableJoin::TableJoin(const Settings & settings, VolumeJBODPtr tmp_volume_) join_algorithm = JoinAlgorithm::PREFER_PARTIAL_MERGE; } +void TableJoin::resetCollected() +{ + key_names_left.clear(); + key_names_right.clear(); + key_asts_left.clear(); + key_asts_right.clear(); + columns_from_joined_table.clear(); + columns_added_by_join.clear(); + original_names.clear(); + renames.clear(); +} + void TableJoin::addUsingKey(const ASTPtr & ast) { key_names_left.push_back(ast->getColumnName()); diff --git a/src/Interpreters/TableJoin.h b/src/Interpreters/TableJoin.h index 0eeb724ab7b..46d2c2e457c 100644 --- a/src/Interpreters/TableJoin.h +++ b/src/Interpreters/TableJoin.h @@ -112,6 +112,7 @@ public: const String & temporaryFilesCodec() const { return temporary_files_codec; } bool enablePartialMergeJoinOptimizations() const { return partial_merge_join_optimizations; } + void resetCollected(); void addUsingKey(const ASTPtr & ast); void addOnKeys(ASTPtr & left_table_ast, ASTPtr & right_table_ast); diff --git a/tests/queries/0_stateless/01281_join_with_prewhere_fix.reference b/tests/queries/0_stateless/01281_join_with_prewhere_fix.reference new file mode 100644 index 00000000000..535d2b01d33 --- /dev/null +++ b/tests/queries/0_stateless/01281_join_with_prewhere_fix.reference @@ -0,0 +1,8 @@ +1 +2 +3 +4 +5 +6 +7 +8 diff --git a/tests/queries/0_stateless/01281_join_with_prewhere_fix.sql b/tests/queries/0_stateless/01281_join_with_prewhere_fix.sql new file mode 100644 index 00000000000..3d772c7ad12 --- /dev/null +++ b/tests/queries/0_stateless/01281_join_with_prewhere_fix.sql @@ -0,0 +1,21 @@ +drop table if exists t; + +create table t (x UInt8, id UInt8) ENGINE = MergeTree() order by (id); +insert into t values (1, 1); + +set enable_optimize_predicate_expression = 0; + +select 1 from t as l join t as r on l.id = r.id prewhere l.x; +select 2 from t as l join t as r on l.id = r.id where r.x; +select 3 from t as l join t as r on l.id = r.id prewhere l.x where r.x; +select 4 from t as l join t as r using id prewhere l.x where r.x; + +select 5 from t as l join t as r on l.id = r.id where l.x and r.x; +select 6 from t as l join t as r using id where l.x and r.x; + +set optimize_move_to_prewhere = 0; + +select 7 from t as l join t as r on l.id = r.id where l.x and r.x; +select 8 from t as l join t as r using id where l.x and r.x; + +drop table t; From b9f35aad02aa67e0dc5e4da654e4a5823141f69d Mon Sep 17 00:00:00 2001 From: Vladimir Chebotarev Date: Fri, 24 Jan 2020 13:17:29 +0300 Subject: [PATCH 468/738] Revert "Disable some flappy tests" This reverts commit e6958c0e498e595099836651f083e9a639a40775. --- tests/integration/test_multiple_disks/test.py | 6 ------ tests/integration/test_ttl_move/test.py | 13 ------------- 2 files changed, 19 deletions(-) diff --git a/tests/integration/test_multiple_disks/test.py b/tests/integration/test_multiple_disks/test.py index 87f0b4ec3be..b2e54f53ba8 100644 --- a/tests/integration/test_multiple_disks/test.py +++ b/tests/integration/test_multiple_disks/test.py @@ -360,7 +360,6 @@ def test_max_data_part_size(start_cluster, name, engine): finally: node1.query("DROP TABLE IF EXISTS {}".format(name)) -@pytest.mark.skip(reason="Flappy test") @pytest.mark.parametrize("name,engine", [ ("mt_with_overflow","MergeTree()"), ("replicated_mt_with_overflow","ReplicatedMergeTree('/clickhouse/replicated_mt_with_overflow', '1')",), @@ -455,7 +454,6 @@ def test_background_move(start_cluster, name, engine): finally: node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) -@pytest.mark.skip(reason="Flappy test") @pytest.mark.parametrize("name,engine", [ ("stopped_moving_mt","MergeTree()"), ("stopped_moving_replicated_mt","ReplicatedMergeTree('/clickhouse/stopped_moving_replicated_mt', '1')",), @@ -722,7 +720,6 @@ def produce_alter_move(node, name): pass -@pytest.mark.skip(reason="Flappy test") @pytest.mark.parametrize("name,engine", [ ("concurrently_altering_mt","MergeTree()"), ("concurrently_altering_replicated_mt","ReplicatedMergeTree('/clickhouse/concurrently_altering_replicated_mt', '1')",), @@ -776,7 +773,6 @@ def test_concurrent_alter_move(start_cluster, name, engine): finally: node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) -@pytest.mark.skip(reason="Flappy test") @pytest.mark.parametrize("name,engine", [ ("concurrently_dropping_mt","MergeTree()"), ("concurrently_dropping_replicated_mt","ReplicatedMergeTree('/clickhouse/concurrently_dropping_replicated_mt', '1')",), @@ -905,8 +901,6 @@ def test_mutate_to_another_disk(start_cluster, name, engine): finally: node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) - -@pytest.mark.skip(reason="Flappy test") @pytest.mark.parametrize("name,engine", [ ("alter_modifying_mt","MergeTree()"), ("replicated_alter_modifying_mt","ReplicatedMergeTree('/clickhouse/replicated_alter_modifying_mt', '1')",), diff --git a/tests/integration/test_ttl_move/test.py b/tests/integration/test_ttl_move/test.py index ab348ea0cb1..d09fc9d4b24 100644 --- a/tests/integration/test_ttl_move/test.py +++ b/tests/integration/test_ttl_move/test.py @@ -59,7 +59,6 @@ def get_used_disks_for_table(node, table_name, partition=None): """.format(name=table_name, suffix=suffix)).strip().split('\n') -@pytest.mark.skip(reason="Flappy test") @pytest.mark.parametrize("name,engine,alter", [ ("mt_test_rule_with_invalid_destination","MergeTree()",0), ("replicated_mt_test_rule_with_invalid_destination","ReplicatedMergeTree('/clickhouse/replicated_test_rule_with_invalid_destination', '1')",0), @@ -119,7 +118,6 @@ def test_rule_with_invalid_destination(started_cluster, name, engine, alter): node1.query("DROP TABLE IF EXISTS {}".format(name)) -@pytest.mark.skip(reason="Flappy test") @pytest.mark.parametrize("name,engine,positive", [ ("mt_test_inserts_to_disk_do_not_work","MergeTree()",0), ("replicated_mt_test_inserts_to_disk_do_not_work","ReplicatedMergeTree('/clickhouse/replicated_test_inserts_to_disk_do_not_work', '1')",0), @@ -152,7 +150,6 @@ def test_inserts_to_disk_work(started_cluster, name, engine, positive): node1.query("DROP TABLE IF EXISTS {}".format(name)) -@pytest.mark.skip(reason="Flappy test") @pytest.mark.parametrize("name,engine,positive", [ ("mt_test_moves_to_disk_do_not_work","MergeTree()",0), ("replicated_mt_test_moves_to_disk_do_not_work","ReplicatedMergeTree('/clickhouse/replicated_test_moves_to_disk_do_not_work', '1')",0), @@ -199,7 +196,6 @@ def test_moves_to_disk_work(started_cluster, name, engine, positive): node1.query("DROP TABLE IF EXISTS {}".format(name)) -@pytest.mark.skip(reason="Flappy test") @pytest.mark.parametrize("name,engine", [ ("mt_test_moves_to_volume_work","MergeTree()"), ("replicated_mt_test_moves_to_volume_work","ReplicatedMergeTree('/clickhouse/replicated_test_moves_to_volume_work', '1')"), @@ -246,7 +242,6 @@ def test_moves_to_volume_work(started_cluster, name, engine): node1.query("DROP TABLE IF EXISTS {}".format(name)) -@pytest.mark.skip(reason="Flappy test") @pytest.mark.parametrize("name,engine,positive", [ ("mt_test_inserts_to_volume_do_not_work","MergeTree()",0), ("replicated_mt_test_inserts_to_volume_do_not_work","ReplicatedMergeTree('/clickhouse/replicated_test_inserts_to_volume_do_not_work', '1')",0), @@ -285,7 +280,6 @@ def test_inserts_to_volume_work(started_cluster, name, engine, positive): node1.query("DROP TABLE IF EXISTS {}".format(name)) -@pytest.mark.skip(reason="Flappy test") @pytest.mark.parametrize("name,engine", [ ("mt_test_moves_to_disk_eventually_work","MergeTree()"), ("replicated_mt_test_moves_to_disk_eventually_work","ReplicatedMergeTree('/clickhouse/replicated_test_moves_to_disk_eventually_work', '1')"), @@ -374,7 +368,6 @@ def test_replicated_download_ttl_info(started_cluster): continue -@pytest.mark.skip(reason="Flappy test") @pytest.mark.parametrize("name,engine,positive", [ ("mt_test_merges_to_disk_do_not_work","MergeTree()",0), ("replicated_mt_test_merges_to_disk_do_not_work","ReplicatedMergeTree('/clickhouse/replicated_test_merges_to_disk_do_not_work', '1')",0), @@ -432,7 +425,6 @@ def test_merges_to_disk_work(started_cluster, name, engine, positive): node1.query("DROP TABLE IF EXISTS {}".format(name)) -@pytest.mark.skip(reason="Flappy test") @pytest.mark.parametrize("name,engine", [ ("mt_test_merges_with_full_disk_work","MergeTree()"), ("replicated_mt_test_merges_with_full_disk_work","ReplicatedMergeTree('/clickhouse/replicated_test_merges_with_full_disk_work', '1')"), @@ -499,7 +491,6 @@ def test_merges_with_full_disk_work(started_cluster, name, engine): node1.query("DROP TABLE IF EXISTS {}".format(name)) -@pytest.mark.skip(reason="Flappy test") @pytest.mark.parametrize("name,engine,positive", [ ("mt_test_moves_after_merges_do_not_work","MergeTree()",0), ("replicated_mt_test_moves_after_merges_do_not_work","ReplicatedMergeTree('/clickhouse/replicated_test_moves_after_merges_do_not_work', '1')",0), @@ -552,7 +543,6 @@ def test_moves_after_merges_work(started_cluster, name, engine, positive): node1.query("DROP TABLE IF EXISTS {}".format(name)) -@pytest.mark.skip(reason="Flappy test") @pytest.mark.parametrize("name,engine,positive,bar", [ ("mt_test_moves_after_alter_do_not_work","MergeTree()",0,"DELETE"), ("replicated_mt_test_moves_after_alter_do_not_work","ReplicatedMergeTree('/clickhouse/replicated_test_moves_after_alter_do_not_work', '1')",0,"DELETE"), @@ -658,7 +648,6 @@ def test_materialize_ttl_in_partition(started_cluster, name, engine): node1.query("DROP TABLE IF EXISTS {}".format(name)) -@pytest.mark.skip(reason="Flappy test") @pytest.mark.parametrize("name,engine,positive", [ ("mt_test_alter_multiple_ttls_positive", "MergeTree()", True), ("mt_replicated_test_alter_multiple_ttls_positive", "ReplicatedMergeTree('/clickhouse/replicated_test_alter_multiple_ttls_positive', '1')", True), @@ -741,7 +730,6 @@ limitations under the License.""" node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) -@pytest.mark.skip(reason="Flappy test") @pytest.mark.parametrize("name,engine", [ ("concurrently_altering_ttl_mt","MergeTree()"), ("concurrently_altering_ttl_replicated_mt","ReplicatedMergeTree('/clickhouse/concurrently_altering_ttl_replicated_mt', '1')",), @@ -832,7 +820,6 @@ def test_concurrent_alter_with_ttl_move(started_cluster, name, engine): finally: node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) -@pytest.mark.skip(reason="Flappy test") @pytest.mark.parametrize("name,positive", [ ("test_double_move_while_select_negative", 0), ("test_double_move_while_select_positive", 1), From a9a7da8e68a3e026be64d63811759f4bea3afe6b Mon Sep 17 00:00:00 2001 From: Vladimir Chebotarev Date: Tue, 28 Jan 2020 10:57:54 +0300 Subject: [PATCH 469/738] Fixed `test_ttl_move::test_alter_multiple_ttls`. --- tests/integration/test_ttl_move/test.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_ttl_move/test.py b/tests/integration/test_ttl_move/test.py index d09fc9d4b24..3d2268e9f16 100644 --- a/tests/integration/test_ttl_move/test.py +++ b/tests/integration/test_ttl_move/test.py @@ -648,6 +648,12 @@ def test_materialize_ttl_in_partition(started_cluster, name, engine): node1.query("DROP TABLE IF EXISTS {}".format(name)) +def start_thread(*args, **kwargs): + thread = threading.Thread(*args, **kwargs) + thread.start() + return thread + + @pytest.mark.parametrize("name,engine,positive", [ ("mt_test_alter_multiple_ttls_positive", "MergeTree()", True), ("mt_replicated_test_alter_multiple_ttls_positive", "ReplicatedMergeTree('/clickhouse/replicated_test_alter_multiple_ttls_positive', '1')", True), @@ -678,6 +684,8 @@ limitations under the License.""" """ now = time.time() try: + sleeps = { delay : start_thread(target=time.sleep, args=(delay,)) for delay in [12, 22] } + node1.query(""" CREATE TABLE {name} ( p1 Int64, @@ -694,8 +702,8 @@ limitations under the License.""" node1.query(""" ALTER TABLE {name} MODIFY TTL d1 + INTERVAL 0 SECOND TO DISK 'jbod2', - d1 + INTERVAL 5 SECOND TO VOLUME 'external', - d1 + INTERVAL 10 SECOND DELETE + d1 + INTERVAL 10 SECOND TO VOLUME 'external', + d1 + INTERVAL 20 SECOND DELETE """.format(name=name)) for p in range(3): @@ -713,14 +721,14 @@ limitations under the License.""" assert node1.query("SELECT count() FROM {name}".format(name=name)).splitlines() == ["6"] - time.sleep(5) + sleeps[12].join() used_disks = get_used_disks_for_table(node1, name) assert set(used_disks) == {"external"} if positive else {"jbod1", "jbod2"} assert node1.query("SELECT count() FROM {name}".format(name=name)).splitlines() == ["6"] - time.sleep(5) + sleeps[22].join() node1.query("OPTIMIZE TABLE {name} FINAL".format(name=name)) From c0947ab609d6bcb5567b115a2167e9a961bc2b8e Mon Sep 17 00:00:00 2001 From: Vladimir Chebotarev Date: Thu, 9 Apr 2020 07:53:57 +0300 Subject: [PATCH 470/738] Minor fixes in `test_ttl_move`. --- tests/integration/test_multiple_disks/test.py | 6 +++- tests/integration/test_ttl_move/test.py | 34 ++++++++++++------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/tests/integration/test_multiple_disks/test.py b/tests/integration/test_multiple_disks/test.py index b2e54f53ba8..cbfeb9dac3a 100644 --- a/tests/integration/test_multiple_disks/test.py +++ b/tests/integration/test_multiple_disks/test.py @@ -933,7 +933,11 @@ def test_concurrent_alter_modify(start_cluster, name, engine): def alter_modify(num): for i in range(num): column_type = random.choice(["UInt64", "String"]) - node1.query("ALTER TABLE {} MODIFY COLUMN number {}".format(name, column_type)) + try: + node1.query("ALTER TABLE {} MODIFY COLUMN number {}".format(name, column_type)) + except: + if "Replicated" not in engine: + raise insert(100) diff --git a/tests/integration/test_ttl_move/test.py b/tests/integration/test_ttl_move/test.py index 3d2268e9f16..a4091304352 100644 --- a/tests/integration/test_ttl_move/test.py +++ b/tests/integration/test_ttl_move/test.py @@ -147,7 +147,10 @@ def test_inserts_to_disk_work(started_cluster, name, engine, positive): assert node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "10" finally: - node1.query("DROP TABLE IF EXISTS {}".format(name)) + try: + node1.query("DROP TABLE IF EXISTS {}".format(name)) + except: + pass @pytest.mark.parametrize("name,engine,positive", [ @@ -168,7 +171,7 @@ def test_moves_to_disk_work(started_cluster, name, engine, positive): SETTINGS storage_policy='small_jbod_with_external' """.format(name=name, engine=engine)) - wait_expire_1 = 6 + wait_expire_1 = 12 wait_expire_2 = 4 time_1 = time.time() + wait_expire_1 time_2 = time.time() + wait_expire_1 + wait_expire_2 @@ -389,7 +392,7 @@ def test_merges_to_disk_work(started_cluster, name, engine, positive): node1.query("SYSTEM STOP MERGES {}".format(name)) node1.query("SYSTEM STOP MOVES {}".format(name)) - wait_expire_1 = 10 + wait_expire_1 = 16 wait_expire_2 = 4 time_1 = time.time() + wait_expire_1 time_2 = time.time() + wait_expire_1 + wait_expire_2 @@ -509,7 +512,7 @@ def test_moves_after_merges_work(started_cluster, name, engine, positive): SETTINGS storage_policy='small_jbod_with_external' """.format(name=name, engine=engine)) - wait_expire_1 = 10 + wait_expire_1 = 16 wait_expire_2 = 4 time_1 = time.time() + wait_expire_1 time_2 = time.time() + wait_expire_1 + wait_expire_2 @@ -684,7 +687,7 @@ limitations under the License.""" """ now = time.time() try: - sleeps = { delay : start_thread(target=time.sleep, args=(delay,)) for delay in [12, 22] } + sleeps = { delay : start_thread(target=time.sleep, args=(delay,)) for delay in [16, 26] } node1.query(""" CREATE TABLE {name} ( @@ -694,16 +697,16 @@ limitations under the License.""" ) ENGINE = {engine} ORDER BY tuple() PARTITION BY p1 - TTL d1 + INTERVAL 30 SECOND TO DISK 'jbod2', - d1 + INTERVAL 60 SECOND TO VOLUME 'external' + TTL d1 + INTERVAL 34 SECOND TO DISK 'jbod2', + d1 + INTERVAL 64 SECOND TO VOLUME 'external' SETTINGS storage_policy='jbods_with_external', merge_with_ttl_timeout=0 """.format(name=name, engine=engine)) node1.query(""" ALTER TABLE {name} MODIFY TTL d1 + INTERVAL 0 SECOND TO DISK 'jbod2', - d1 + INTERVAL 10 SECOND TO VOLUME 'external', - d1 + INTERVAL 20 SECOND DELETE + d1 + INTERVAL 14 SECOND TO VOLUME 'external', + d1 + INTERVAL 24 SECOND DELETE """.format(name=name)) for p in range(3): @@ -721,14 +724,14 @@ limitations under the License.""" assert node1.query("SELECT count() FROM {name}".format(name=name)).splitlines() == ["6"] - sleeps[12].join() + sleeps[16].join() used_disks = get_used_disks_for_table(node1, name) assert set(used_disks) == {"external"} if positive else {"jbod1", "jbod2"} assert node1.query("SELECT count() FROM {name}".format(name=name)).splitlines() == ["6"] - sleeps[22].join() + sleeps[26].join() node1.query("OPTIMIZE TABLE {name} FINAL".format(name=name)) @@ -788,7 +791,7 @@ def test_concurrent_alter_with_ttl_move(started_cluster, name, engine): try: node1.query("ALTER TABLE {} MOVE {mt} {mp} TO {md} {mv}".format( name, mt=move_type, mp=move_part, md=move_disk, mv=move_volume)) - except QueryRuntimeException as ex: + except QueryRuntimeException: pass for i in range(num): @@ -805,7 +808,10 @@ def test_concurrent_alter_with_ttl_move(started_cluster, name, engine): what = random.choice(["TO VOLUME 'main'", "TO VOLUME 'external'", "TO DISK 'jbod1'", "TO DISK 'jbod2'", "TO DISK 'external'"]) when = "now()+{}".format(random.randint(-1, 5)) ttls.append("{} {}".format(when, what)) - node1.query("ALTER TABLE {} MODIFY TTL {}".format(name, ", ".join(ttls))) + try: + node1.query("ALTER TABLE {} MODIFY TTL {}".format(name, ", ".join(ttls))) + except QueryRuntimeException: + pass def optimize_table(num): for i in range(num): @@ -865,6 +871,8 @@ def test_double_move_while_select(started_cluster, name, positive): node1.query("INSERT INTO {name} VALUES (3, '{string}')".format(name=name, string=get_random_string(9 * 1024 * 1024))) node1.query("INSERT INTO {name} VALUES (4, '{string}')".format(name=name, string=get_random_string(9 * 1024 * 1024))) + time.sleep(1) + # If SELECT locked old part on external, move shall fail. assert node1.query("SELECT disk_name FROM system.parts WHERE table = '{name}' AND active = 1 AND name = '{part}'" .format(name=name, part=parts[0])).splitlines() == ["jbod1" if positive else "external"] From 3ed5d9434a9f1ca8c1a9a82b59a9fec9f75c8070 Mon Sep 17 00:00:00 2001 From: Vladimir Chebotarev Date: Thu, 30 Apr 2020 10:21:19 +0300 Subject: [PATCH 471/738] Added `move_ttl_info` to `system.parts`. --- src/Storages/System/StorageSystemParts.cpp | 93 ++++++++++++++-------- 1 file changed, 60 insertions(+), 33 deletions(-) diff --git a/src/Storages/System/StorageSystemParts.cpp b/src/Storages/System/StorageSystemParts.cpp index 2418594899e..39246498849 100644 --- a/src/Storages/System/StorageSystemParts.cpp +++ b/src/Storages/System/StorageSystemParts.cpp @@ -3,6 +3,8 @@ #include #include #include +#include +#include #include #include #include @@ -17,41 +19,51 @@ namespace DB StorageSystemParts::StorageSystemParts(const std::string & name_) : StorageSystemPartsBase(name_, { - {"partition", std::make_shared()}, - {"name", std::make_shared()}, - {"part_type", std::make_shared()}, - {"active", std::make_shared()}, - {"marks", std::make_shared()}, - {"rows", std::make_shared()}, - {"bytes_on_disk", std::make_shared()}, - {"data_compressed_bytes", std::make_shared()}, - {"data_uncompressed_bytes", std::make_shared()}, - {"marks_bytes", std::make_shared()}, - {"modification_time", std::make_shared()}, - {"remove_time", std::make_shared()}, - {"refcount", std::make_shared()}, - {"min_date", std::make_shared()}, - {"max_date", std::make_shared()}, - {"min_time", std::make_shared()}, - {"max_time", std::make_shared()}, - {"partition_id", std::make_shared()}, - {"min_block_number", std::make_shared()}, - {"max_block_number", std::make_shared()}, - {"level", std::make_shared()}, - {"data_version", std::make_shared()}, - {"primary_key_bytes_in_memory", std::make_shared()}, - {"primary_key_bytes_in_memory_allocated", std::make_shared()}, - {"is_frozen", std::make_shared()}, + {"partition", std::make_shared()}, + {"name", std::make_shared()}, + {"part_type", std::make_shared()}, + {"active", std::make_shared()}, + {"marks", std::make_shared()}, + {"rows", std::make_shared()}, + {"bytes_on_disk", std::make_shared()}, + {"data_compressed_bytes", std::make_shared()}, + {"data_uncompressed_bytes", std::make_shared()}, + {"marks_bytes", std::make_shared()}, + {"modification_time", std::make_shared()}, + {"remove_time", std::make_shared()}, + {"refcount", std::make_shared()}, + {"min_date", std::make_shared()}, + {"max_date", std::make_shared()}, + {"min_time", std::make_shared()}, + {"max_time", std::make_shared()}, + {"partition_id", std::make_shared()}, + {"min_block_number", std::make_shared()}, + {"max_block_number", std::make_shared()}, + {"level", std::make_shared()}, + {"data_version", std::make_shared()}, + {"primary_key_bytes_in_memory", std::make_shared()}, + {"primary_key_bytes_in_memory_allocated", std::make_shared()}, + {"is_frozen", std::make_shared()}, - {"database", std::make_shared()}, - {"table", std::make_shared()}, - {"engine", std::make_shared()}, - {"disk_name", std::make_shared()}, - {"path", std::make_shared()}, + {"database", std::make_shared()}, + {"table", std::make_shared()}, + {"engine", std::make_shared()}, + {"disk_name", std::make_shared()}, + {"path", std::make_shared()}, - {"hash_of_all_files", std::make_shared()}, - {"hash_of_uncompressed_files", std::make_shared()}, - {"uncompressed_hash_of_compressed_files", std::make_shared()} + {"hash_of_all_files", std::make_shared()}, + {"hash_of_uncompressed_files", std::make_shared()}, + {"uncompressed_hash_of_compressed_files", std::make_shared()}, + + {"move_ttl_info", std::make_shared( + std::make_shared( + DataTypes({ + std::make_shared(), + std::make_shared(), + std::make_shared() + }), + Strings({"expression", "min", "max"}) + ))}, } ) { @@ -128,6 +140,21 @@ void StorageSystemParts::processNextStorage(MutableColumns & columns_, const Sto checksum = helper.uncompressed_hash_of_compressed_files; columns_[i++]->insert(getHexUIntLowercase(checksum.first) + getHexUIntLowercase(checksum.second)); + + /// move_ttl_info + { + Array move_ttl_info_array; + move_ttl_info_array.reserve(part->ttl_infos.moves_ttl.size()); + for (const auto & [expression, move_ttl_info] : part->ttl_infos.moves_ttl) + { + Tuple move_ttl_info_tuple; + move_ttl_info_tuple.push_back(expression); + move_ttl_info_tuple.push_back(static_cast(move_ttl_info.min)); + move_ttl_info_tuple.push_back(static_cast(move_ttl_info.max)); + move_ttl_info_array.emplace_back(std::move(move_ttl_info_tuple)); + } + columns_[i++]->insert(move_ttl_info_array); + } } } From 6f68979662cad8d6005d500e8cbcd865aa5c0ec3 Mon Sep 17 00:00:00 2001 From: Vladimir Chebotarev Date: Mon, 11 May 2020 10:31:26 +0300 Subject: [PATCH 472/738] Switched `system.parts.move_ttl_info` to separate arrays. --- src/Storages/System/StorageSystemParts.cpp | 31 +++++++++------------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/src/Storages/System/StorageSystemParts.cpp b/src/Storages/System/StorageSystemParts.cpp index 39246498849..8c1fdd667c4 100644 --- a/src/Storages/System/StorageSystemParts.cpp +++ b/src/Storages/System/StorageSystemParts.cpp @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #include @@ -55,15 +54,9 @@ StorageSystemParts::StorageSystemParts(const std::string & name_) {"hash_of_uncompressed_files", std::make_shared()}, {"uncompressed_hash_of_compressed_files", std::make_shared()}, - {"move_ttl_info", std::make_shared( - std::make_shared( - DataTypes({ - std::make_shared(), - std::make_shared(), - std::make_shared() - }), - Strings({"expression", "min", "max"}) - ))}, + {"move_ttl_info.expression", std::make_shared(std::make_shared())}, + {"move_ttl_info.min", std::make_shared(std::make_shared())}, + {"move_ttl_info.max", std::make_shared(std::make_shared())}, } ) { @@ -143,17 +136,19 @@ void StorageSystemParts::processNextStorage(MutableColumns & columns_, const Sto /// move_ttl_info { - Array move_ttl_info_array; - move_ttl_info_array.reserve(part->ttl_infos.moves_ttl.size()); + Array expression_array, min_array, max_array; + expression_array.reserve(part->ttl_infos.moves_ttl.size()); + min_array.reserve(part->ttl_infos.moves_ttl.size()); + max_array.reserve(part->ttl_infos.moves_ttl.size()); for (const auto & [expression, move_ttl_info] : part->ttl_infos.moves_ttl) { - Tuple move_ttl_info_tuple; - move_ttl_info_tuple.push_back(expression); - move_ttl_info_tuple.push_back(static_cast(move_ttl_info.min)); - move_ttl_info_tuple.push_back(static_cast(move_ttl_info.max)); - move_ttl_info_array.emplace_back(std::move(move_ttl_info_tuple)); + expression_array.emplace_back(expression); + min_array.push_back(static_cast(move_ttl_info.min)); + max_array.push_back(static_cast(move_ttl_info.max)); } - columns_[i++]->insert(move_ttl_info_array); + columns_[i++]->insert(expression_array); + columns_[i++]->insert(min_array); + columns_[i++]->insert(max_array); } } } From 02b52df86dac2967a3118e41536d4f047c54d84e Mon Sep 17 00:00:00 2001 From: Vladimir Chebotarev Date: Mon, 11 May 2020 10:35:13 +0300 Subject: [PATCH 473/738] Added `delete_ttl_info_min` and `delete_ttl_info_max` to `system.parts`. --- src/Storages/System/StorageSystemParts.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/Storages/System/StorageSystemParts.cpp b/src/Storages/System/StorageSystemParts.cpp index 8c1fdd667c4..a5d5da5bd22 100644 --- a/src/Storages/System/StorageSystemParts.cpp +++ b/src/Storages/System/StorageSystemParts.cpp @@ -54,6 +54,9 @@ StorageSystemParts::StorageSystemParts(const std::string & name_) {"hash_of_uncompressed_files", std::make_shared()}, {"uncompressed_hash_of_compressed_files", std::make_shared()}, + {"delete_ttl_info_min", std::make_shared(std::make_shared())}, + {"delete_ttl_info_max", std::make_shared(std::make_shared())}, + {"move_ttl_info.expression", std::make_shared(std::make_shared())}, {"move_ttl_info.min", std::make_shared(std::make_shared())}, {"move_ttl_info.max", std::make_shared(std::make_shared())}, @@ -134,6 +137,12 @@ void StorageSystemParts::processNextStorage(MutableColumns & columns_, const Sto checksum = helper.uncompressed_hash_of_compressed_files; columns_[i++]->insert(getHexUIntLowercase(checksum.first) + getHexUIntLowercase(checksum.second)); + /// delete_ttl_info + { + columns_[i++]->insert(static_cast(part->ttl_infos.table_ttl.min)); + columns_[i++]->insert(static_cast(part->ttl_infos.table_ttl.max)); + } + /// move_ttl_info { Array expression_array, min_array, max_array; From da33858d392c6b08f608b12df1b30a4be236efe7 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 18 May 2020 19:53:14 +0300 Subject: [PATCH 474/738] Update StorageSystemParts.cpp --- src/Storages/System/StorageSystemParts.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/System/StorageSystemParts.cpp b/src/Storages/System/StorageSystemParts.cpp index a5d5da5bd22..745d2938d86 100644 --- a/src/Storages/System/StorageSystemParts.cpp +++ b/src/Storages/System/StorageSystemParts.cpp @@ -54,8 +54,8 @@ StorageSystemParts::StorageSystemParts(const std::string & name_) {"hash_of_uncompressed_files", std::make_shared()}, {"uncompressed_hash_of_compressed_files", std::make_shared()}, - {"delete_ttl_info_min", std::make_shared(std::make_shared())}, - {"delete_ttl_info_max", std::make_shared(std::make_shared())}, + {"delete_ttl_info_min", std::make_shared()}, + {"delete_ttl_info_max", std::make_shared()}, {"move_ttl_info.expression", std::make_shared(std::make_shared())}, {"move_ttl_info.min", std::make_shared(std::make_shared())}, From 8bc527eecde1758060b3d58a28e73ebddbcdec97 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 20 May 2020 15:16:55 +0300 Subject: [PATCH 475/738] Compile partition key --- src/Interpreters/MutationsInterpreter.cpp | 5 +-- src/Storages/IStorage.cpp | 23 +++++++++++ src/Storages/IStorage.h | 18 ++++++++- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 6 +-- src/Storages/MergeTree/MergeTreeData.cpp | 39 ++++++++++--------- src/Storages/MergeTree/MergeTreeData.h | 8 +--- .../MergeTree/MergeTreeDataWriter.cpp | 11 +++--- src/Storages/MergeTree/MergeTreePartition.cpp | 20 +++++----- .../ReplicatedMergeTreeTableMetadata.cpp | 2 +- src/Storages/StorageInMemoryMetadata.h | 20 ++++++++++ 10 files changed, 105 insertions(+), 47 deletions(-) diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index 5ba6424653c..7ea3401a2c2 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -221,9 +221,8 @@ static NameSet getKeyColumns(const StoragePtr & storage) NameSet key_columns; - if (merge_tree_data->partition_key_expr) - for (const String & col : merge_tree_data->partition_key_expr->getRequiredColumns()) - key_columns.insert(col); + for (const String & col : merge_tree_data->getColumnsRequiredForPartitionKey()) + key_columns.insert(col); auto sorting_key_expr = merge_tree_data->sorting_key_expr; if (sorting_key_expr) diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index bc17efaafd7..2fefe844408 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include @@ -433,4 +434,26 @@ NamesAndTypesList IStorage::getVirtuals() const return {}; } +const StorageMetadataKeyField & IStorage::getPartitionKey() const +{ + return partition_key; +} + +void IStorage::setPartitionKey(const StorageMetadataKeyField & partition_key_) +{ + partition_key = partition_key_; +} + +Names IStorage::getColumnsRequiredForPartitionKey() const +{ + if (partition_key.expressions) + return partition_key.expressions->getRequiredColumns(); + return {}; +} + +bool IStorage::hasPartitionKey() const +{ + return partition_key.expressions != nullptr; +} + } diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 8c36fb64f72..0ed30ecc16f 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -183,6 +183,9 @@ public: /// thread-unsafe part. lockStructure must be acquired /// /// By default return empty list of columns. virtual NamesAndTypesList getVirtuals() const; + const StorageMetadataKeyField & getPartitionKey() const; + void setPartitionKey(const StorageMetadataKeyField & partition_key_); + bool hasPartitionKey() const; protected: /// still thread-unsafe part. void setIndices(IndicesDescription indices_); @@ -195,10 +198,21 @@ protected: /// still thread-unsafe part. private: StorageID storage_id; mutable std::mutex id_mutex; + ColumnsDescription columns; IndicesDescription indices; ConstraintsDescription constraints; + StorageMetadataKeyField partition_key; + //StorageMetadataKeyField primary_key; + //StorageMetadataKeyField sorting_key; + //StorageMetadataKeyField sample_key; + + //StorageMetadataField rows_ttl_entry; + + //std::vector column_ttl_entries; + //std::vector move_ttl_entries; + private: RWLockImpl::LockHolder tryLockTimed( const RWLock & rwlock, RWLockImpl::Type type, const String & query_id, const SettingSeconds & acquire_timeout) const; @@ -441,7 +455,7 @@ public: virtual Strings getDataPaths() const { return {}; } /// Returns ASTExpressionList of partition key expression for storage or nullptr if there is none. - virtual ASTPtr getPartitionKeyAST() const { return nullptr; } + virtual ASTPtr getPartitionKeyAST() const { return partition_key.definition_ast; } /// Returns ASTExpressionList of sorting key expression for storage or nullptr if there is none. virtual ASTPtr getSortingKeyAST() const { return nullptr; } @@ -453,7 +467,7 @@ public: virtual ASTPtr getSamplingKeyAST() const { return nullptr; } /// Returns column names that need to be read to calculate partition key. - virtual Names getColumnsRequiredForPartitionKey() const { return {}; } + virtual Names getColumnsRequiredForPartitionKey() const; /// Returns column names that need to be read to calculate sorting key. virtual Names getColumnsRequiredForSortingKey() const { return {}; } diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 44ad4af626e..0dec588a738 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -494,7 +494,7 @@ void IMergeTreeDataPart::loadPartitionAndMinMaxIndex() minmax_idx.load(storage, volume->getDisk(), path); } - String calculated_partition_id = partition.getID(storage.partition_key_sample); + String calculated_partition_id = partition.getID(storage.getPartitionKey().sample_block); if (calculated_partition_id != info.partition_id) throw Exception( "While loading part " + getFullPath() + ": calculated partition ID: " + calculated_partition_id @@ -852,7 +852,7 @@ void IMergeTreeDataPart::checkConsistencyBase() const if (!checksums.files.count("count.txt")) throw Exception("No checksum for count.txt", ErrorCodes::NO_FILE_IN_DATA_PART); - if (storage.partition_key_expr && !checksums.files.count("partition.dat")) + if (storage.hasPartitionKey() && !checksums.files.count("partition.dat")) throw Exception("No checksum for partition.dat", ErrorCodes::NO_FILE_IN_DATA_PART); if (!isEmpty()) @@ -885,7 +885,7 @@ void IMergeTreeDataPart::checkConsistencyBase() const { check_file_not_empty(volume->getDisk(), path + "count.txt"); - if (storage.partition_key_expr) + if (storage.hasPartitionKey()) check_file_not_empty(volume->getDisk(), path + "partition.dat"); for (const String & col_name : storage.minmax_idx_columns) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index de9d3f6e981..bc10bbc238b 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -130,7 +130,6 @@ MergeTreeData::MergeTreeData( : IStorage(table_id_) , global_context(context_) , merging_params(merging_params_) - , partition_by_ast(metadata.partition_by_ast) , sample_by_ast(metadata.sample_by_ast) , settings_ast(metadata.settings_ast) , require_part_metadata(require_part_metadata_) @@ -169,8 +168,8 @@ MergeTreeData::MergeTreeData( { try { - partition_by_ast = makeASTFunction("toYYYYMM", std::make_shared(date_column_name)); - initPartitionKey(); + auto partition_by_ast = makeASTFunction("toYYYYMM", std::make_shared(date_column_name)); + initPartitionKey(partition_by_ast); if (minmax_idx_date_column_pos == -1) throw Exception("Could not find Date column", ErrorCodes::BAD_TYPE_OF_FIELD); @@ -185,7 +184,7 @@ MergeTreeData::MergeTreeData( else { is_custom_partitioned = true; - initPartitionKey(); + initPartitionKey(metadata.partition_by_ast); min_format_version = MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING; } @@ -252,8 +251,8 @@ StorageInMemoryMetadata MergeTreeData::getInMemoryMetadata() const { StorageInMemoryMetadata metadata(getColumns(), getIndices(), getConstraints()); - if (partition_by_ast) - metadata.partition_by_ast = partition_by_ast->clone(); + if (hasPartitionKey()) + metadata.partition_by_ast = getPartitionKey().definition_ast->clone(); if (order_by_ast) metadata.order_by_ast = order_by_ast->clone(); @@ -511,28 +510,31 @@ ASTPtr MergeTreeData::extractKeyExpressionList(const ASTPtr & node) } -void MergeTreeData::initPartitionKey() +void MergeTreeData::initPartitionKey(ASTPtr partition_by_ast) { ASTPtr partition_key_expr_list = extractKeyExpressionList(partition_by_ast); + StorageMetadataKeyField new_partition_key; + new_partition_key.definition_ast = partition_by_ast; + new_partition_key.expression_ast = partition_key_expr_list; if (partition_key_expr_list->children.empty()) return; { auto syntax_result = SyntaxAnalyzer(global_context).analyze(partition_key_expr_list, getColumns().getAllPhysical()); - partition_key_expr = ExpressionAnalyzer(partition_key_expr_list, syntax_result, global_context).getActions(false); + new_partition_key.expressions = ExpressionAnalyzer(partition_key_expr_list, syntax_result, global_context).getActions(false); } for (const ASTPtr & ast : partition_key_expr_list->children) { String col_name = ast->getColumnName(); - partition_key_sample.insert(partition_key_expr->getSampleBlock().getByName(col_name)); + new_partition_key.sample_block.insert(new_partition_key.expressions->getSampleBlock().getByName(col_name)); } - checkKeyExpression(*partition_key_expr, partition_key_sample, "Partition"); + checkKeyExpression(*new_partition_key.expressions, new_partition_key.sample_block, "Partition"); /// Add all columns used in the partition key to the min-max index. - const NamesAndTypesList & minmax_idx_columns_with_types = partition_key_expr->getRequiredColumnsWithTypes(); + const NamesAndTypesList & minmax_idx_columns_with_types = new_partition_key.expressions->getRequiredColumnsWithTypes(); minmax_idx_expr = std::make_shared(minmax_idx_columns_with_types, global_context); for (const NameAndTypePair & column : minmax_idx_columns_with_types) { @@ -577,6 +579,7 @@ void MergeTreeData::initPartitionKey() } } } + setPartitionKey(new_partition_key); } namespace @@ -631,8 +634,8 @@ void MergeTreeData::setTTLExpressions(const ColumnsDescription & new_columns, { NameSet columns_ttl_forbidden; - if (partition_key_expr) - for (const auto & col : partition_key_expr->getRequiredColumns()) + if (hasPartitionKey()) + for (const auto & col : getPartitionKey().expressions->getRequiredColumns()) columns_ttl_forbidden.insert(col); if (sorting_key_expr) @@ -1419,12 +1422,12 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, const S /// (and not as a part of some expression) and if the ALTER only affects column metadata. NameSet columns_alter_type_metadata_only; - if (partition_key_expr) + if (hasPartitionKey()) { /// Forbid altering partition key columns because it can change partition ID format. /// TODO: in some cases (e.g. adding an Enum value) a partition key column can still be ALTERed. /// We should allow it. - for (const String & col : partition_key_expr->getRequiredColumns()) + for (const String & col : getColumnsRequiredForPartitionKey()) columns_alter_type_forbidden.insert(col); } @@ -2621,7 +2624,7 @@ String MergeTreeData::getPartitionIDFromQuery(const ASTPtr & ast, const Context /// Re-parse partition key fields using the information about expected field types. - size_t fields_count = partition_key_sample.columns(); + size_t fields_count = getPartitionKey().sample_block.columns(); if (partition_ast.fields_count != fields_count) throw Exception( "Wrong number of fields in the partition expression: " + toString(partition_ast.fields_count) + @@ -2638,7 +2641,7 @@ String MergeTreeData::getPartitionIDFromQuery(const ASTPtr & ast, const Context ReadBufferFromMemory right_paren_buf(")", 1); ConcatReadBuffer buf({&left_paren_buf, &fields_buf, &right_paren_buf}); - auto input_stream = FormatFactory::instance().getInput("Values", buf, partition_key_sample, context, context.getSettingsRef().max_block_size); + auto input_stream = FormatFactory::instance().getInput("Values", buf, getPartitionKey().sample_block, context, context.getSettingsRef().max_block_size); auto block = input_stream->read(); if (!block || !block.rows()) @@ -3161,7 +3164,7 @@ MergeTreeData & MergeTreeData::checkStructureAndGetMergeTreeData(IStorage & sour if (query_to_string(order_by_ast) != query_to_string(src_data->order_by_ast)) throw Exception("Tables have different ordering", ErrorCodes::BAD_ARGUMENTS); - if (query_to_string(partition_by_ast) != query_to_string(src_data->partition_by_ast)) + if (query_to_string(getPartitionKey().definition_ast) != query_to_string(src_data->getPartitionKey().definition_ast)) throw Exception("Tables have different partition key", ErrorCodes::BAD_ARGUMENTS); if (format_version != src_data->format_version) diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 394260096a7..87eabea364b 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -334,12 +334,11 @@ public: /// See comments about methods below in IStorage interface StorageInMemoryMetadata getInMemoryMetadata() const override; - ASTPtr getPartitionKeyAST() const override { return partition_by_ast; } ASTPtr getSortingKeyAST() const override { return sorting_key_expr_ast; } ASTPtr getPrimaryKeyAST() const override { return primary_key_expr_ast; } ASTPtr getSamplingKeyAST() const override { return sample_by_ast; } - Names getColumnsRequiredForPartitionKey() const override { return (partition_key_expr ? partition_key_expr->getRequiredColumns() : Names{}); } + //Names getColumnsRequiredForPartitionKey() const override { return (partition_key_expr ? partition_key_expr->getRequiredColumns() : Names{}); } Names getColumnsRequiredForSortingKey() const override { return sorting_key_expr->getRequiredColumns(); } Names getColumnsRequiredForPrimaryKey() const override { return primary_key_expr->getRequiredColumns(); } Names getColumnsRequiredForSampling() const override { return columns_required_for_sampling; } @@ -648,8 +647,6 @@ public: const MergingParams merging_params; bool is_custom_partitioned = false; - ExpressionActionsPtr partition_key_expr; - Block partition_key_sample; ExpressionActionsPtr minmax_idx_expr; Names minmax_idx_columns; @@ -738,7 +735,6 @@ protected: friend struct ReplicatedMergeTreeTableMetadata; friend class StorageReplicatedMergeTree; - ASTPtr partition_by_ast; ASTPtr order_by_ast; ASTPtr primary_key_ast; ASTPtr sample_by_ast; @@ -853,7 +849,7 @@ protected: void setProperties(const StorageInMemoryMetadata & metadata, bool only_check = false, bool attach = false); - void initPartitionKey(); + void initPartitionKey(ASTPtr partition_by_ast); void setTTLExpressions(const ColumnsDescription & columns, const ASTPtr & new_ttl_table_ast, bool only_check = false); diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index a78e2e5ae32..34377bb29f6 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -138,18 +138,19 @@ BlocksWithPartition MergeTreeDataWriter::splitBlockIntoParts(const Block & block data.check(block, true); block.checkNumberOfRows(); - if (!data.partition_key_expr) /// Table is not partitioned. + if (!data.hasPartitionKey()) /// Table is not partitioned. { result.emplace_back(Block(block), Row()); return result; } Block block_copy = block; - data.partition_key_expr->execute(block_copy); + const auto & partition_key = data.getPartitionKey(); + partition_key.expressions->execute(block_copy); ColumnRawPtrs partition_columns; - partition_columns.reserve(data.partition_key_sample.columns()); - for (const ColumnWithTypeAndName & element : data.partition_key_sample) + partition_columns.reserve(partition_key.sample_block.columns()); + for (const ColumnWithTypeAndName & element : partition_key.sample_block) partition_columns.emplace_back(block_copy.getByName(element.name).column.get()); PODArray partition_num_to_first_row; @@ -203,7 +204,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa MergeTreePartition partition(std::move(block_with_partition.partition)); - MergeTreePartInfo new_part_info(partition.getID(data.partition_key_sample), temp_index, temp_index, 0); + MergeTreePartInfo new_part_info(partition.getID(data.getPartitionKey().sample_block), temp_index, temp_index, 0); String part_name; if (data.format_version < MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) { diff --git a/src/Storages/MergeTree/MergeTreePartition.cpp b/src/Storages/MergeTree/MergeTreePartition.cpp index 000d0abad43..54e213fafac 100644 --- a/src/Storages/MergeTree/MergeTreePartition.cpp +++ b/src/Storages/MergeTree/MergeTreePartition.cpp @@ -26,7 +26,7 @@ static std::unique_ptr openForReading(const DiskPtr & di String MergeTreePartition::getID(const MergeTreeData & storage) const { - return getID(storage.partition_key_sample); + return getID(storage.getPartitionKey().sample_block); } /// NOTE: This ID is used to create part names which are then persisted in ZK and as directory names on the file system. @@ -89,7 +89,8 @@ String MergeTreePartition::getID(const Block & partition_key_sample) const void MergeTreePartition::serializeText(const MergeTreeData & storage, WriteBuffer & out, const FormatSettings & format_settings) const { - size_t key_size = storage.partition_key_sample.columns(); + const auto & partition_key_sample = storage.getPartitionKey().sample_block; + size_t key_size = partition_key_sample.columns(); if (key_size == 0) { @@ -97,7 +98,7 @@ void MergeTreePartition::serializeText(const MergeTreeData & storage, WriteBuffe } else if (key_size == 1) { - const DataTypePtr & type = storage.partition_key_sample.getByPosition(0).type; + const DataTypePtr & type = partition_key_sample.getByPosition(0).type; auto column = type->createColumn(); column->insert(value[0]); type->serializeAsText(*column, 0, out, format_settings); @@ -108,7 +109,7 @@ void MergeTreePartition::serializeText(const MergeTreeData & storage, WriteBuffe Columns columns; for (size_t i = 0; i < key_size; ++i) { - const auto & type = storage.partition_key_sample.getByPosition(i).type; + const auto & type = partition_key_sample.getByPosition(i).type; types.push_back(type); auto column = type->createColumn(); column->insert(value[i]); @@ -123,19 +124,20 @@ void MergeTreePartition::serializeText(const MergeTreeData & storage, WriteBuffe void MergeTreePartition::load(const MergeTreeData & storage, const DiskPtr & disk, const String & part_path) { - if (!storage.partition_key_expr) + if (!storage.hasPartitionKey()) return; + const auto & partition_key_sample = storage.getPartitionKey().sample_block; auto partition_file_path = part_path + "partition.dat"; auto file = openForReading(disk, partition_file_path); - value.resize(storage.partition_key_sample.columns()); - for (size_t i = 0; i < storage.partition_key_sample.columns(); ++i) - storage.partition_key_sample.getByPosition(i).type->deserializeBinary(value[i], *file); + value.resize(partition_key_sample.columns()); + for (size_t i = 0; i < partition_key_sample.columns(); ++i) + partition_key_sample.getByPosition(i).type->deserializeBinary(value[i], *file); } void MergeTreePartition::store(const MergeTreeData & storage, const DiskPtr & disk, const String & part_path, MergeTreeDataPartChecksums & checksums) const { - store(storage.partition_key_sample, disk, part_path, checksums); + store(storage.getPartitionKey().sample_block, disk, part_path, checksums); } void MergeTreePartition::store(const Block & partition_key_sample, const DiskPtr & disk, const String & part_path, MergeTreeDataPartChecksums & checksums) const diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp index fe4ea79d106..8a5ce2e8846 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp @@ -51,7 +51,7 @@ ReplicatedMergeTreeTableMetadata::ReplicatedMergeTreeTableMetadata(const MergeTr data_format_version = data.format_version; if (data.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) - partition_key = formattedAST(MergeTreeData::extractKeyExpressionList(data.partition_by_ast)); + partition_key = formattedAST(data.getPartitionKey().expression_ast); ttl_table = formattedAST(data.ttl_table_ast); diff --git a/src/Storages/StorageInMemoryMetadata.h b/src/Storages/StorageInMemoryMetadata.h index 361b6be009b..7b8a0dab7ee 100644 --- a/src/Storages/StorageInMemoryMetadata.h +++ b/src/Storages/StorageInMemoryMetadata.h @@ -43,4 +43,24 @@ struct StorageInMemoryMetadata StorageInMemoryMetadata & operator=(const StorageInMemoryMetadata & other); }; +struct StorageMetadataField +{ + ASTPtr ast; + ExpressionActionsPtr actions; +}; + +struct StorageMetadataKeyField +{ + ASTPtr definition_ast; + + ASTPtr expression_ast; + ExpressionActionsPtr expressions; + Names expression_columns; + + Block sample_block; + DataTypes data_types; + + StorageMetadataKeyField & operator=(const StorageMetadataKeyField & other) = default; +}; + } From fcf58fa4a15ea6b8b1d4cb21ac4785f12f19ad40 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 20 May 2020 17:04:24 +0300 Subject: [PATCH 476/738] Fix clang build. --- src/Processors/Executors/PipelineExecutor.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Processors/Executors/PipelineExecutor.cpp b/src/Processors/Executors/PipelineExecutor.cpp index 59f37115333..5825a8c90d6 100644 --- a/src/Processors/Executors/PipelineExecutor.cpp +++ b/src/Processors/Executors/PipelineExecutor.cpp @@ -549,10 +549,10 @@ void PipelineExecutor::executeSingleThread(size_t thread_num, size_t num_threads auto & context = executor_contexts[thread_num]; LOG_TRACE(log, std::fixed << std::setprecision(3) << "Thread finished." - << " Total time: " << (context.total_time_ns / 1e9) << " sec." - << " Execution time: " << (context.execution_time_ns / 1e9) << " sec." - << " Processing time: " << (context.processing_time_ns / 1e9) << " sec." - << " Wait time: " << (context.wait_time_ns / 1e9) << " sec."); + << " Total time: " << (context->total_time_ns / 1e9) << " sec." + << " Execution time: " << (context->execution_time_ns / 1e9) << " sec." + << " Processing time: " << (context->processing_time_ns / 1e9) << " sec." + << " Wait time: " << (context->wait_time_ns / 1e9) << " sec."); #endif } From 9fb28f5ac0390f44a44b457fa370d19100baf93a Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 20 May 2020 18:16:39 +0300 Subject: [PATCH 477/738] Add sampling key --- src/Storages/IStorage.cpp | 29 +++++++++- src/Storages/IStorage.h | 14 +++-- src/Storages/MergeTree/MergeTreeData.cpp | 34 +++-------- src/Storages/MergeTree/MergeTreeData.h | 7 --- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 13 +++-- .../ReplicatedMergeTreeTableMetadata.cpp | 2 +- src/Storages/StorageInMemoryMetadata.cpp | 58 +++++++++++++++++++ src/Storages/StorageInMemoryMetadata.h | 5 +- 8 files changed, 116 insertions(+), 46 deletions(-) diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index 2fefe844408..873e5d617bb 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -446,14 +446,41 @@ void IStorage::setPartitionKey(const StorageMetadataKeyField & partition_key_) Names IStorage::getColumnsRequiredForPartitionKey() const { - if (partition_key.expressions) + if (hasPartitionKey()) return partition_key.expressions->getRequiredColumns(); return {}; } +Names IStorage::getColumnsRequiredForSampling() const +{ + if (hasSamplingKey()) + return sampling_key.expressions->getRequiredColumns(); + return {}; +} + bool IStorage::hasPartitionKey() const { return partition_key.expressions != nullptr; } + +bool IStorage::supportsSampling() const +{ + return hasSamplingKey(); +} + +const StorageMetadataKeyField & IStorage::getSamplingKey() const +{ + return sampling_key; +} +void IStorage::setSamplingKey(const StorageMetadataKeyField & sampling_key_) +{ + sampling_key = sampling_key_; +} + +bool IStorage::hasSamplingKey() const +{ + return sampling_key.expressions != nullptr; +} + } diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 0ed30ecc16f..dd672349a73 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -101,7 +101,7 @@ public: virtual bool isView() const { return false; } /// Returns true if the storage supports queries with the SAMPLE section. - virtual bool supportsSampling() const { return false; } + virtual bool supportsSampling() const; /// Returns true if the storage supports queries with the FINAL section. virtual bool supportsFinal() const { return false; } @@ -183,10 +183,16 @@ public: /// thread-unsafe part. lockStructure must be acquired /// /// By default return empty list of columns. virtual NamesAndTypesList getVirtuals() const; + const StorageMetadataKeyField & getPartitionKey() const; void setPartitionKey(const StorageMetadataKeyField & partition_key_); bool hasPartitionKey() const; + const StorageMetadataKeyField & getSamplingKey() const; + void setSamplingKey(const StorageMetadataKeyField & sampling_key_); + bool hasSamplingKey() const; + + protected: /// still thread-unsafe part. void setIndices(IndicesDescription indices_); @@ -206,7 +212,7 @@ private: StorageMetadataKeyField partition_key; //StorageMetadataKeyField primary_key; //StorageMetadataKeyField sorting_key; - //StorageMetadataKeyField sample_key; + StorageMetadataKeyField sampling_key; //StorageMetadataField rows_ttl_entry; @@ -464,7 +470,7 @@ public: virtual ASTPtr getPrimaryKeyAST() const { return nullptr; } /// Returns sampling expression AST for storage or nullptr if there is none. - virtual ASTPtr getSamplingKeyAST() const { return nullptr; } + virtual ASTPtr getSamplingKeyAST() const { return sampling_key.definition_ast; } /// Returns column names that need to be read to calculate partition key. virtual Names getColumnsRequiredForPartitionKey() const; @@ -476,7 +482,7 @@ public: virtual Names getColumnsRequiredForPrimaryKey() const { return {}; } /// Returns column names that need to be read to calculate sampling key. - virtual Names getColumnsRequiredForSampling() const { return {}; } + virtual Names getColumnsRequiredForSampling() const; /// Returns column names that need to be read for FINAL to work. virtual Names getColumnsRequiredForFinal() const { return {}; } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index bc10bbc238b..f6da3d56912 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -130,7 +130,6 @@ MergeTreeData::MergeTreeData( : IStorage(table_id_) , global_context(context_) , merging_params(merging_params_) - , sample_by_ast(metadata.sample_by_ast) , settings_ast(metadata.settings_ast) , require_part_metadata(require_part_metadata_) , relative_data_path(relative_data_path_) @@ -151,16 +150,15 @@ MergeTreeData::MergeTreeData( /// NOTE: using the same columns list as is read when performing actual merges. merging_params.check(getColumns().getAllPhysical()); - if (sample_by_ast) + if (metadata.sample_by_ast != nullptr) { - sampling_expr_column_name = sample_by_ast->getColumnName(); + StorageMetadataKeyField sampling_key = StorageMetadataKeyField::getKeyFromAST(metadata.sample_by_ast, getColumns(), global_context); - if (!primary_key_sample.has(sampling_expr_column_name) + if (!primary_key_sample.has(sampling_key.expression_column_names[0]) && !attach && !settings->compatibility_allow_sampling_expression_not_in_primary_key) /// This is for backward compatibility. throw Exception("Sampling expression must be present in the primary key", ErrorCodes::BAD_ARGUMENTS); - auto syntax = SyntaxAnalyzer(global_context).analyze(sample_by_ast, getColumns().getAllPhysical()); - columns_required_for_sampling = syntax->requiredSourceColumns(); + setSamplingKey(sampling_key); } MergeTreeDataFormatVersion min_format_version(0); @@ -252,7 +250,7 @@ StorageInMemoryMetadata MergeTreeData::getInMemoryMetadata() const StorageInMemoryMetadata metadata(getColumns(), getIndices(), getConstraints()); if (hasPartitionKey()) - metadata.partition_by_ast = getPartitionKey().definition_ast->clone(); + metadata.partition_by_ast = getPartitionKeyAST()->clone(); if (order_by_ast) metadata.order_by_ast = order_by_ast->clone(); @@ -263,8 +261,8 @@ StorageInMemoryMetadata MergeTreeData::getInMemoryMetadata() const if (ttl_table_ast) metadata.ttl_for_table_ast = ttl_table_ast->clone(); - if (sample_by_ast) - metadata.sample_by_ast = sample_by_ast->clone(); + if (hasSamplingKey()) + metadata.sample_by_ast = getSamplingKeyAST()->clone(); if (settings_ast) metadata.settings_ast = settings_ast->clone(); @@ -512,25 +510,11 @@ ASTPtr MergeTreeData::extractKeyExpressionList(const ASTPtr & node) void MergeTreeData::initPartitionKey(ASTPtr partition_by_ast) { - ASTPtr partition_key_expr_list = extractKeyExpressionList(partition_by_ast); - StorageMetadataKeyField new_partition_key; - new_partition_key.definition_ast = partition_by_ast; - new_partition_key.expression_ast = partition_key_expr_list; + StorageMetadataKeyField new_partition_key = StorageMetadataKeyField::getKeyFromAST(partition_by_ast, getColumns(), global_context); - if (partition_key_expr_list->children.empty()) + if (new_partition_key.expression_ast->children.empty()) return; - { - auto syntax_result = SyntaxAnalyzer(global_context).analyze(partition_key_expr_list, getColumns().getAllPhysical()); - new_partition_key.expressions = ExpressionAnalyzer(partition_key_expr_list, syntax_result, global_context).getActions(false); - } - - for (const ASTPtr & ast : partition_key_expr_list->children) - { - String col_name = ast->getColumnName(); - new_partition_key.sample_block.insert(new_partition_key.expressions->getSampleBlock().getByName(col_name)); - } - checkKeyExpression(*new_partition_key.expressions, new_partition_key.sample_block, "Partition"); /// Add all columns used in the partition key to the min-max index. diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 87eabea364b..ca27dc6c385 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -336,12 +336,10 @@ public: ASTPtr getSortingKeyAST() const override { return sorting_key_expr_ast; } ASTPtr getPrimaryKeyAST() const override { return primary_key_expr_ast; } - ASTPtr getSamplingKeyAST() const override { return sample_by_ast; } //Names getColumnsRequiredForPartitionKey() const override { return (partition_key_expr ? partition_key_expr->getRequiredColumns() : Names{}); } Names getColumnsRequiredForSortingKey() const override { return sorting_key_expr->getRequiredColumns(); } Names getColumnsRequiredForPrimaryKey() const override { return primary_key_expr->getRequiredColumns(); } - Names getColumnsRequiredForSampling() const override { return columns_required_for_sampling; } Names getColumnsRequiredForFinal() const override { return sorting_key_expr->getRequiredColumns(); } Names getSortingKeyColumns() const override { return sorting_key_columns; } @@ -350,7 +348,6 @@ public: StoragePolicyPtr getStoragePolicy() const override; bool supportsPrewhere() const override { return true; } - bool supportsSampling() const override { return sample_by_ast != nullptr; } bool supportsFinal() const override { @@ -706,9 +703,6 @@ public: /// Vector rw operations have to be done under "move_ttl_entries_mutex". std::vector move_ttl_entries; - String sampling_expr_column_name; - Names columns_required_for_sampling; - /// Limiting parallel sends per one table, used in DataPartsExchange std::atomic_uint current_table_sends {0}; @@ -737,7 +731,6 @@ protected: ASTPtr order_by_ast; ASTPtr primary_key_ast; - ASTPtr sample_by_ast; ASTPtr ttl_table_ast; ASTPtr settings_ast; diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 662fb067e48..3d18b8decc6 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -387,7 +387,8 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( used_sample_factor = 1.0 / boost::rational_cast(relative_sample_size); RelativeSize size_of_universum = 0; - DataTypePtr sampling_column_type = data.primary_key_sample.getByName(data.sampling_expr_column_name).type; + const auto & sampling_key = data.getSamplingKey(); + DataTypePtr sampling_column_type = sampling_key.data_types[0]; if (typeid_cast(sampling_column_type.get())) size_of_universum = RelativeSize(std::numeric_limits::max()) + RelativeSize(1); @@ -456,17 +457,17 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( /// The first time it was calculated for final, because sample key is a part of the PK. /// So, assume that we already have calculated column. ASTPtr sampling_key_ast = data.getSamplingKeyAST(); + if (select.final()) { - sampling_key_ast = std::make_shared(data.sampling_expr_column_name); - + sampling_key_ast = std::make_shared(sampling_key.expression_column_names[0]); /// We do spoil available_real_columns here, but it is not used later. - available_real_columns.emplace_back(data.sampling_expr_column_name, std::move(sampling_column_type)); + available_real_columns.emplace_back(sampling_key.expression_column_names[0], std::move(sampling_column_type)); } if (has_lower_limit) { - if (!key_condition.addCondition(data.sampling_expr_column_name, Range::createLeftBounded(lower, true))) + if (!key_condition.addCondition(sampling_key.expression_column_names[0], Range::createLeftBounded(lower, true))) throw Exception("Sampling column not in primary key", ErrorCodes::ILLEGAL_COLUMN); ASTPtr args = std::make_shared(); @@ -483,7 +484,7 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( if (has_upper_limit) { - if (!key_condition.addCondition(data.sampling_expr_column_name, Range::createRightBounded(upper, false))) + if (!key_condition.addCondition(sampling_key.expression_column_names[0], Range::createRightBounded(upper, false))) throw Exception("Sampling column not in primary key", ErrorCodes::ILLEGAL_COLUMN); ASTPtr args = std::make_shared(); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp index 8a5ce2e8846..0cf10384ba5 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp @@ -29,7 +29,7 @@ ReplicatedMergeTreeTableMetadata::ReplicatedMergeTreeTableMetadata(const MergeTr date_column = data.minmax_idx_columns[data.minmax_idx_date_column_pos]; const auto data_settings = data.getSettings(); - sampling_expression = formattedAST(data.sample_by_ast); + sampling_expression = formattedAST(data.getSamplingKeyAST()); index_granularity = data_settings->index_granularity; merging_params_mode = static_cast(data.merging_params.mode); sign_column = data.merging_params.sign_column; diff --git a/src/Storages/StorageInMemoryMetadata.cpp b/src/Storages/StorageInMemoryMetadata.cpp index a05872234de..277d9dfbfdd 100644 --- a/src/Storages/StorageInMemoryMetadata.cpp +++ b/src/Storages/StorageInMemoryMetadata.cpp @@ -1,4 +1,10 @@ +#include +#include +#include +#include +#include #include +#include namespace DB { @@ -79,4 +85,56 @@ StorageInMemoryMetadata & StorageInMemoryMetadata::operator=(const StorageInMemo return *this; } + +namespace +{ + ASTPtr extractKeyExpressionList(const ASTPtr & node) + { + if (!node) + return std::make_shared(); + + const auto * expr_func = node->as(); + + if (expr_func && expr_func->name == "tuple") + { + /// Primary key is specified in tuple, extract its arguments. + return expr_func->arguments->clone(); + } + else + { + /// Primary key consists of one column. + auto res = std::make_shared(); + res->children.push_back(node); + return res; + } + } +} + +StorageMetadataKeyField StorageMetadataKeyField::getKeyFromAST(const ASTPtr & definition_ast, const ColumnsDescription & columns, const Context & context) +{ + StorageMetadataKeyField result; + result.definition_ast = definition_ast; + result.expression_ast = extractKeyExpressionList(definition_ast); + + if (result.expression_ast->children.empty()) + return result; + + + const auto & children = result.expression_ast->children; + for (const auto & child : children) + result.expression_column_names.emplace_back(child->getColumnName()); + + { + auto syntax_result = SyntaxAnalyzer(context).analyze(result.expression_ast, columns.getAllPhysical()); + result.expressions = ExpressionAnalyzer(result.expression_ast->clone(), syntax_result, context).getActions(true); + result.sample_block = result.expressions->getSampleBlock(); + } + + + for (size_t i = 0; i < result.sample_block.columns(); ++i) + result.data_types.emplace_back(result.sample_block.getByPosition(i).type); + + return result; +} + } diff --git a/src/Storages/StorageInMemoryMetadata.h b/src/Storages/StorageInMemoryMetadata.h index 7b8a0dab7ee..a98c00bc4c9 100644 --- a/src/Storages/StorageInMemoryMetadata.h +++ b/src/Storages/StorageInMemoryMetadata.h @@ -55,12 +55,13 @@ struct StorageMetadataKeyField ASTPtr expression_ast; ExpressionActionsPtr expressions; - Names expression_columns; + Names expression_column_names; Block sample_block; DataTypes data_types; - StorageMetadataKeyField & operator=(const StorageMetadataKeyField & other) = default; + static StorageMetadataKeyField getKeyFromAST(const ASTPtr & definition_ast, const ColumnsDescription & columns, const Context & context); }; + } From 555087512ae4164cac6f93aa08a2f472cd356f0a Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Wed, 20 May 2020 18:28:44 +0300 Subject: [PATCH 478/738] Update debian/rules Co-authored-by: alesapin --- debian/rules | 1 + 1 file changed, 1 insertion(+) diff --git a/debian/rules b/debian/rules index ad7b73380f4..7218e196baa 100755 --- a/debian/rules +++ b/debian/rules @@ -101,6 +101,7 @@ override_dh_clean: dh_clean # -X contrib override_dh_strip: +#https://www.debian.org/doc/debian-policy/ch-source.html#debian-rules-and-deb-build-options ifeq (,$(filter nostrip,$(DEB_BUILD_OPTIONS))) dh_strip -pclickhouse-common-static --dbg-package=clickhouse-common-static-dbg endif From 138154ba6aceecf621f7095aab5a80c719abd182 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Wed, 20 May 2020 18:42:45 +0300 Subject: [PATCH 479/738] Fix memory-leak in registerDiskS3 during config->createView invocations. --- src/Disks/S3/registerDiskS3.cpp | 38 ++++++++++++++++----------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/src/Disks/S3/registerDiskS3.cpp b/src/Disks/S3/registerDiskS3.cpp index 2b72f872dd2..81879800b7f 100644 --- a/src/Disks/S3/registerDiskS3.cpp +++ b/src/Disks/S3/registerDiskS3.cpp @@ -37,13 +37,14 @@ namespace void checkRemoveAccess(IDisk & disk) { disk.remove("test_acl"); } - std::shared_ptr getProxyResolverConfiguration(const Poco::Util::AbstractConfiguration * proxy_resolver_config) + std::shared_ptr getProxyResolverConfiguration( + const String & prefix, const Poco::Util::AbstractConfiguration & proxy_resolver_config) { - auto endpoint = Poco::URI(proxy_resolver_config->getString("endpoint")); - auto proxy_scheme = proxy_resolver_config->getString("proxy_scheme"); + auto endpoint = Poco::URI(proxy_resolver_config.getString(prefix + ".endpoint")); + auto proxy_scheme = proxy_resolver_config.getString(prefix + ".proxy_scheme"); if (proxy_scheme != "http" && proxy_scheme != "https") throw Exception("Only HTTP/HTTPS schemas allowed in proxy resolver config: " + proxy_scheme, ErrorCodes::BAD_ARGUMENTS); - auto proxy_port = proxy_resolver_config->getUInt("proxy_port"); + auto proxy_port = proxy_resolver_config.getUInt(prefix + ".proxy_port"); LOG_DEBUG( &Logger::get("DiskS3"), "Configured proxy resolver: " << endpoint.toString() << ", Scheme: " << proxy_scheme << ", Port: " << proxy_port); @@ -51,16 +52,17 @@ namespace return std::make_shared(endpoint, proxy_scheme, proxy_port); } - std::shared_ptr getProxyListConfiguration(const Poco::Util::AbstractConfiguration * proxy_config) + std::shared_ptr getProxyListConfiguration( + const String & prefix, const Poco::Util::AbstractConfiguration & proxy_config) { std::vector keys; - proxy_config->keys(keys); + proxy_config.keys(prefix, keys); std::vector proxies; for (const auto & key : keys) if (startsWith(key, "uri")) { - Poco::URI proxy_uri(proxy_config->getString(key)); + Poco::URI proxy_uri(proxy_config.getString(prefix + "." + key)); if (proxy_uri.getScheme() != "http" && proxy_uri.getScheme() != "https") throw Exception("Only HTTP/HTTPS schemas allowed in proxy uri: " + proxy_uri.toString(), ErrorCodes::BAD_ARGUMENTS); @@ -78,25 +80,23 @@ namespace return nullptr; } - std::shared_ptr getProxyConfiguration(const Poco::Util::AbstractConfiguration * config) + std::shared_ptr getProxyConfiguration(const String & prefix, const Poco::Util::AbstractConfiguration & config) { - if (!config->has("proxy")) + if (!config.has(prefix + ".proxy")) return nullptr; - const auto * proxy_config = config->createView("proxy"); - std::vector config_keys; - proxy_config->keys(config_keys); + config.keys(prefix + ".proxy", config_keys); if (auto resolver_configs = std::count(config_keys.begin(), config_keys.end(), "resolver")) { if (resolver_configs > 1) throw Exception("Multiple proxy resolver configurations aren't allowed", ErrorCodes::BAD_ARGUMENTS); - return getProxyResolverConfiguration(proxy_config->createView("resolver")); + return getProxyResolverConfiguration(prefix + ".proxy.resolver", config); } - return getProxyListConfiguration(proxy_config); + return getProxyListConfiguration(prefix + ".proxy", config); } } @@ -107,27 +107,25 @@ void registerDiskS3(DiskFactory & factory) const Poco::Util::AbstractConfiguration & config, const String & config_prefix, const Context & context) -> DiskPtr { - const auto * disk_config = config.createView(config_prefix); - Poco::File disk{context.getPath() + "disks/" + name}; disk.createDirectories(); Aws::Client::ClientConfiguration cfg; - S3::URI uri(Poco::URI(disk_config->getString("endpoint"))); + S3::URI uri(Poco::URI(config.getString(config_prefix + ".endpoint"))); if (uri.key.back() != '/') throw Exception("S3 path must ends with '/', but '" + uri.key + "' doesn't.", ErrorCodes::BAD_ARGUMENTS); cfg.endpointOverride = uri.endpoint; - auto proxy_config = getProxyConfiguration(disk_config); + auto proxy_config = getProxyConfiguration(config_prefix, config); if (proxy_config) cfg.perRequestConfiguration = [proxy_config](const auto & request) { return proxy_config->getConfiguration(request); }; auto client = S3::ClientFactory::instance().create( cfg, - disk_config->getString("access_key_id", ""), - disk_config->getString("secret_access_key", "")); + config.getString(config_prefix + ".access_key_id", ""), + config.getString(config_prefix + ".secret_access_key", "")); String metadata_path = context.getPath() + "disks/" + name + "/"; From d98c1589edf16281f687099d9e0772a1015ba174 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Wed, 20 May 2020 19:01:05 +0300 Subject: [PATCH 480/738] Better error handling in Proxy Resolver. --- src/Disks/S3/ProxyResolverConfiguration.cpp | 12 ++++++++++-- src/Disks/S3/ProxyResolverConfiguration.h | 2 +- .../configs/config.d/storage_conf.xml | 4 ++-- .../test_s3_with_proxy/proxy-resolver/resolver.py | 2 +- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/Disks/S3/ProxyResolverConfiguration.cpp b/src/Disks/S3/ProxyResolverConfiguration.cpp index a574809596f..c36432d933e 100644 --- a/src/Disks/S3/ProxyResolverConfiguration.cpp +++ b/src/Disks/S3/ProxyResolverConfiguration.cpp @@ -7,6 +7,11 @@ #include #include +namespace DB::ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + namespace DB::S3 { ProxyResolverConfiguration::ProxyResolverConfiguration(const Poco::URI & endpoint_, String proxy_scheme_, unsigned proxy_port_) @@ -30,13 +35,16 @@ Aws::Client::ClientConfigurationPerRequest ProxyResolverConfiguration::getConfig Aws::Client::ClientConfigurationPerRequest cfg; try { - /// It should be just empty GET / request. - Poco::Net::HTTPRequest request(Poco::Net::HTTPRequest::HTTP_1_1); + /// It should be just empty GET request. + Poco::Net::HTTPRequest request(Poco::Net::HTTPRequest::HTTP_GET, endpoint.getPath(), Poco::Net::HTTPRequest::HTTP_1_1); session->sendRequest(request); Poco::Net::HTTPResponse response; auto & response_body_stream = session->receiveResponse(response); + if (response.getStatus() != Poco::Net::HTTPResponse::HTTP_OK) + throw Exception("Proxy resolver returned not OK status: " + response.getReason(), ErrorCodes::BAD_ARGUMENTS); + String proxy_host; /// Read proxy host as string from response body. Poco::StreamCopier::copyToString(response_body_stream, proxy_host); diff --git a/src/Disks/S3/ProxyResolverConfiguration.h b/src/Disks/S3/ProxyResolverConfiguration.h index 0b23ae77c4a..113ee6ea035 100644 --- a/src/Disks/S3/ProxyResolverConfiguration.h +++ b/src/Disks/S3/ProxyResolverConfiguration.h @@ -6,7 +6,7 @@ namespace DB::S3 { /** * Proxy configuration where proxy host is obtained each time from specified endpoint. - * For each request to S3 it makes GET request to specified endpoint and reads proxy host from a response body. + * For each request to S3 it makes GET request to specified endpoint URL and reads proxy host from a response body. * Specified scheme and port added to obtained proxy host to form completed proxy URL. */ class ProxyResolverConfiguration : public ProxyConfiguration diff --git a/tests/integration/test_s3_with_proxy/configs/config.d/storage_conf.xml b/tests/integration/test_s3_with_proxy/configs/config.d/storage_conf.xml index a83c875b134..ec543d64cdb 100644 --- a/tests/integration/test_s3_with_proxy/configs/config.d/storage_conf.xml +++ b/tests/integration/test_s3_with_proxy/configs/config.d/storage_conf.xml @@ -18,12 +18,12 @@ minio123 - http://resolver:8080 + http://resolver:8080/hostname http 8888 diff --git a/tests/integration/test_s3_with_proxy/proxy-resolver/resolver.py b/tests/integration/test_s3_with_proxy/proxy-resolver/resolver.py index ecafe92cb83..520c018cbad 100644 --- a/tests/integration/test_s3_with_proxy/proxy-resolver/resolver.py +++ b/tests/integration/test_s3_with_proxy/proxy-resolver/resolver.py @@ -2,7 +2,7 @@ import bottle import random -@bottle.route('/') +@bottle.route('/hostname') def index(): if random.randrange(2) == 0: return 'proxy1' From a62e1834d2807a7336ced5fa954b92bedd69f456 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 20 May 2020 19:30:27 +0300 Subject: [PATCH 481/738] Fix data rot @blinkov --- docs/en/development/developer-instruction.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/development/developer-instruction.md b/docs/en/development/developer-instruction.md index a7826810030..abd93e83116 100644 --- a/docs/en/development/developer-instruction.md +++ b/docs/en/development/developer-instruction.md @@ -7,7 +7,7 @@ Building of ClickHouse is supported on Linux, FreeBSD and Mac OS X. # If You Use Windows {#if-you-use-windows} -If you use Windows, you need to create a virtual machine with Ubuntu. To start working with a virtual machine please install VirtualBox. You can download Ubuntu from the website: https://www.ubuntu.com/\#download. Please create a virtual machine from the downloaded image (you should reserve at least 4GB of RAM for it). To run a command-line terminal in Ubuntu, please locate a program containing the word “terminal” in its name (gnome-terminal, konsole etc.) or just press Ctrl+Alt+T. +If you use Windows, you need to create a virtual machine with Ubuntu. To start working with a virtual machine please install VirtualBox. You can download Ubuntu from the website: https://www.ubuntu.com/#download. Please create a virtual machine from the downloaded image (you should reserve at least 4GB of RAM for it). To run a command-line terminal in Ubuntu, please locate a program containing the word “terminal” in its name (gnome-terminal, konsole etc.) or just press Ctrl+Alt+T. # If You Use a 32-bit System {#if-you-use-a-32-bit-system} From 616902a995863b4157623b09385c6893ea920349 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 20 May 2020 21:11:38 +0300 Subject: [PATCH 482/738] Sorting and primary key (broken) --- src/Interpreters/MutationsInterpreter.cpp | 6 +- src/Storages/IStorage.cpp | 55 +++++++++++++++- src/Storages/IStorage.h | 24 ++++--- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 11 ++-- src/Storages/MergeTree/MergeTreeData.cpp | 63 ++++++++++++------- src/Storages/MergeTree/MergeTreeData.h | 35 +++++------ .../MergeTree/MergeTreeDataMergerMutator.cpp | 4 +- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 22 ++++--- .../MergeTree/MergeTreeDataWriter.cpp | 2 +- .../MergeTree/MergeTreeWhereOptimizer.cpp | 5 +- .../MergeTree/MergedBlockOutputStream.cpp | 2 +- .../ReplicatedMergeTreeTableMetadata.cpp | 8 +-- src/Storages/StorageReplicatedMergeTree.cpp | 4 +- 13 files changed, 158 insertions(+), 83 deletions(-) diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index 7ea3401a2c2..6962885534c 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -224,10 +224,8 @@ static NameSet getKeyColumns(const StoragePtr & storage) for (const String & col : merge_tree_data->getColumnsRequiredForPartitionKey()) key_columns.insert(col); - auto sorting_key_expr = merge_tree_data->sorting_key_expr; - if (sorting_key_expr) - for (const String & col : sorting_key_expr->getRequiredColumns()) - key_columns.insert(col); + for (const String & col : merge_tree_data->getColumnsRequiredForSortingKey()) + key_columns.insert(col); /// We don't process sample_by_ast separately because it must be among the primary key columns. if (!merge_tree_data->merging_params.sign_column.empty()) diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index 873e5d617bb..7a8da63f76f 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -460,7 +460,7 @@ Names IStorage::getColumnsRequiredForSampling() const bool IStorage::hasPartitionKey() const { - return partition_key.expressions != nullptr; + return partition_key.definition_ast != nullptr; } @@ -469,6 +469,21 @@ bool IStorage::supportsSampling() const return hasSamplingKey(); } +Names IStorage::getColumnsRequiredForSortingKey() const +{ + if (hasSortingKey()) + return sorting_key.expressions->getRequiredColumns(); + return {}; +} + + +Names IStorage::getSortingKeyColumns() const +{ + if (hasSortingKey()) + return sorting_key.expression_column_names; + return {}; +} + const StorageMetadataKeyField & IStorage::getSamplingKey() const { return sampling_key; @@ -480,7 +495,43 @@ void IStorage::setSamplingKey(const StorageMetadataKeyField & sampling_key_) bool IStorage::hasSamplingKey() const { - return sampling_key.expressions != nullptr; + return sampling_key.definition_ast != nullptr; } +bool IStorage::hasSortingKey() const +{ + return sorting_key.definition_ast != nullptr; +} + +void IStorage::setSortingKey(const StorageMetadataKeyField & sorting_key_) +{ + sorting_key = sorting_key_; +} + +const StorageMetadataKeyField & IStorage::getSortingKey() const +{ + return sorting_key; +} + +const StorageMetadataKeyField & IStorage::getPrimaryKey() const +{ + return primary_key; +} + +void IStorage::setPrimaryKey(const StorageMetadataKeyField & primary_key_) +{ + primary_key = primary_key_; +} + +bool IStorage::hasPrimaryKey() const +{ + return primary_key.definition_ast != nullptr; +} + +Names IStorage::getColumnsRequiredForPrimaryKey() const +{ + if (hasPrimaryKey()) + return primary_key.expressions->getRequiredColumns(); + return {}; +} } diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index dd672349a73..6c91997a21c 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -192,6 +192,14 @@ public: /// thread-unsafe part. lockStructure must be acquired void setSamplingKey(const StorageMetadataKeyField & sampling_key_); bool hasSamplingKey() const; + const StorageMetadataKeyField & getSortingKey() const; + void setSortingKey(const StorageMetadataKeyField & sorting_key_); + bool hasSortingKey() const; + + const StorageMetadataKeyField & getPrimaryKey() const; + void setPrimaryKey(const StorageMetadataKeyField & primary_key_); + bool hasPrimaryKey() const; + protected: /// still thread-unsafe part. void setIndices(IndicesDescription indices_); @@ -210,8 +218,8 @@ private: ConstraintsDescription constraints; StorageMetadataKeyField partition_key; - //StorageMetadataKeyField primary_key; - //StorageMetadataKeyField sorting_key; + StorageMetadataKeyField primary_key; + StorageMetadataKeyField sorting_key; StorageMetadataKeyField sampling_key; //StorageMetadataField rows_ttl_entry; @@ -464,10 +472,10 @@ public: virtual ASTPtr getPartitionKeyAST() const { return partition_key.definition_ast; } /// Returns ASTExpressionList of sorting key expression for storage or nullptr if there is none. - virtual ASTPtr getSortingKeyAST() const { return nullptr; } + virtual ASTPtr getSortingKeyAST() const { return sorting_key.definition_ast; } /// Returns ASTExpressionList of primary key expression for storage or nullptr if there is none. - virtual ASTPtr getPrimaryKeyAST() const { return nullptr; } + virtual ASTPtr getPrimaryKeyAST() const { return primary_key.definition_ast; } /// Returns sampling expression AST for storage or nullptr if there is none. virtual ASTPtr getSamplingKeyAST() const { return sampling_key.definition_ast; } @@ -476,20 +484,20 @@ public: virtual Names getColumnsRequiredForPartitionKey() const; /// Returns column names that need to be read to calculate sorting key. - virtual Names getColumnsRequiredForSortingKey() const { return {}; } + virtual Names getColumnsRequiredForSortingKey() const; /// Returns column names that need to be read to calculate primary key. - virtual Names getColumnsRequiredForPrimaryKey() const { return {}; } + virtual Names getColumnsRequiredForPrimaryKey() const; /// Returns column names that need to be read to calculate sampling key. virtual Names getColumnsRequiredForSampling() const; /// Returns column names that need to be read for FINAL to work. - virtual Names getColumnsRequiredForFinal() const { return {}; } + virtual Names getColumnsRequiredForFinal() const { return getColumnsRequiredForSortingKey(); } /// Returns columns names in sorting key specified by user in ORDER BY /// expression. For example: 'a', 'x * y', 'toStartOfMonth(date)', etc. - virtual Names getSortingKeyColumns() const { return {}; } + virtual Names getSortingKeyColumns() const; /// Returns columns, which will be needed to calculate dependencies /// (skip indices, TTL expressions) if we update @updated_columns set of columns. diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 0dec588a738..0487aabfed3 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -438,7 +438,8 @@ void IMergeTreeDataPart::loadIndex() if (!index_granularity.isInitialized()) throw Exception("Index granularity is not loaded before index loading", ErrorCodes::LOGICAL_ERROR); - size_t key_size = storage.primary_key_columns.size(); + const auto & primary_key = storage.getPrimaryKey(); + size_t key_size = primary_key.expression_column_names.size(); if (key_size) { @@ -447,7 +448,7 @@ void IMergeTreeDataPart::loadIndex() for (size_t i = 0; i < key_size; ++i) { - loaded_index[i] = storage.primary_key_data_types[i]->createColumn(); + loaded_index[i] = primary_key.data_types[i]->createColumn(); loaded_index[i]->reserve(index_granularity.getMarksCount()); } @@ -456,7 +457,7 @@ void IMergeTreeDataPart::loadIndex() for (size_t i = 0; i < index_granularity.getMarksCount(); ++i) //-V756 for (size_t j = 0; j < key_size; ++j) - storage.primary_key_data_types[j]->deserializeBinary(*loaded_index[j], *index_file); + primary_key.data_types[j]->deserializeBinary(*loaded_index[j], *index_file); for (size_t i = 0; i < key_size; ++i) { @@ -844,7 +845,7 @@ void IMergeTreeDataPart::checkConsistencyBase() const if (!checksums.empty()) { - if (!storage.primary_key_columns.empty() && !checksums.files.count("primary.idx")) + if (!storage.getPrimaryKey().expression_column_names.empty() && !checksums.files.count("primary.idx")) throw Exception("No checksum for primary.idx", ErrorCodes::NO_FILE_IN_DATA_PART); if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) @@ -878,7 +879,7 @@ void IMergeTreeDataPart::checkConsistencyBase() const }; /// Check that the primary key index is not empty. - if (!storage.primary_key_columns.empty()) + if (!storage.getPrimaryKey().expression_column_names.empty()) check_file_not_empty(volume->getDisk(), path + "primary.idx"); if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index f6da3d56912..094864f5642 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -154,7 +154,8 @@ MergeTreeData::MergeTreeData( { StorageMetadataKeyField sampling_key = StorageMetadataKeyField::getKeyFromAST(metadata.sample_by_ast, getColumns(), global_context); - if (!primary_key_sample.has(sampling_key.expression_column_names[0]) + const auto & primary_key = getPrimaryKey(); + if (!primary_key.sample_block.has(sampling_key.expression_column_names[0]) && !attach && !settings->compatibility_allow_sampling_expression_not_in_primary_key) /// This is for backward compatibility. throw Exception("Sampling expression must be present in the primary key", ErrorCodes::BAD_ARGUMENTS); @@ -252,11 +253,11 @@ StorageInMemoryMetadata MergeTreeData::getInMemoryMetadata() const if (hasPartitionKey()) metadata.partition_by_ast = getPartitionKeyAST()->clone(); - if (order_by_ast) - metadata.order_by_ast = order_by_ast->clone(); + if (hasSortingKey()) + metadata.order_by_ast = getSortingKeyAST()->clone(); - if (primary_key_ast) - metadata.primary_key_ast = primary_key_ast->clone(); + if (hasPrimaryKey()) + metadata.primary_key_ast = getPrimaryKeyAST()->clone(); if (ttl_table_ast) metadata.ttl_for_table_ast = ttl_table_ast->clone(); @@ -349,17 +350,18 @@ void MergeTreeData::setProperties(const StorageInMemoryMetadata & metadata, bool auto all_columns = metadata.columns.getAllPhysical(); /// Order by check AST - if (order_by_ast && only_check) + if (hasSortingKey() && only_check) { /// This is ALTER, not CREATE/ATTACH TABLE. Let us check that all new columns used in the sorting key /// expression have just been added (so that the sorting order is guaranteed to be valid with the new key). ASTPtr added_key_column_expr_list = std::make_shared(); + const auto & old_sorting_key_columns = getSortingKeyColumns(); for (size_t new_i = 0, old_i = 0; new_i < sorting_key_size; ++new_i) { - if (old_i < sorting_key_columns.size()) + if (old_i < old_sorting_key_columns.size()) { - if (new_sorting_key_columns[new_i] != sorting_key_columns[old_i]) + if (new_sorting_key_columns[new_i] != old_sorting_key_columns[old_i]) added_key_column_expr_list->children.push_back(new_sorting_key_expr_list->children[new_i]); else ++old_i; @@ -414,6 +416,12 @@ void MergeTreeData::setProperties(const StorageInMemoryMetadata & metadata, bool new_primary_key_data_types.push_back(elem.type); } + DataTypes new_sorting_key_data_types; + for (size_t i = 0; i < sorting_key_size; ++i) + { + new_sorting_key_data_types.push_back(new_sorting_key_sample.getByPosition(i).type); + } + ASTPtr skip_indices_with_primary_key_expr_list = new_primary_key_expr_list->clone(); ASTPtr skip_indices_with_sorting_key_expr_list = new_sorting_key_expr_list->clone(); @@ -463,17 +471,23 @@ void MergeTreeData::setProperties(const StorageInMemoryMetadata & metadata, bool { setColumns(std::move(metadata.columns)); - order_by_ast = metadata.order_by_ast; - sorting_key_columns = std::move(new_sorting_key_columns); - sorting_key_expr_ast = std::move(new_sorting_key_expr_list); - sorting_key_expr = std::move(new_sorting_key_expr); + StorageMetadataKeyField new_sorting_key; + new_sorting_key.definition_ast = metadata.order_by_ast; + new_sorting_key.expression_column_names = std::move(new_sorting_key_columns); + new_sorting_key.expression_ast = std::move(new_sorting_key_expr_list); + new_sorting_key.expressions = std::move(new_sorting_key_expr); + new_sorting_key.sample_block = std::move(new_sorting_key_sample); + new_sorting_key.data_types = std::move(new_sorting_key_data_types); + setSortingKey(new_sorting_key); - primary_key_ast = metadata.primary_key_ast; - primary_key_columns = std::move(new_primary_key_columns); - primary_key_expr_ast = std::move(new_primary_key_expr_list); - primary_key_expr = std::move(new_primary_key_expr); - primary_key_sample = std::move(new_primary_key_sample); - primary_key_data_types = std::move(new_primary_key_data_types); + StorageMetadataKeyField new_primary_key; + new_primary_key.definition_ast = metadata.primary_key_ast; + new_primary_key.expression_column_names = std::move(new_primary_key_columns); + new_primary_key.expression_ast = std::move(new_primary_key_expr_list); + new_primary_key.expressions = std::move(new_primary_key_expr); + new_primary_key.sample_block = std::move(new_primary_key_sample); + new_primary_key.data_types = std::move(new_primary_key_data_types); + setPrimaryKey(new_primary_key); setIndices(metadata.indices); skip_indices = std::move(new_indices); @@ -622,8 +636,8 @@ void MergeTreeData::setTTLExpressions(const ColumnsDescription & new_columns, for (const auto & col : getPartitionKey().expressions->getRequiredColumns()) columns_ttl_forbidden.insert(col); - if (sorting_key_expr) - for (const auto & col : sorting_key_expr->getRequiredColumns()) + if (hasSortingKey()) + for (const auto & col : getColumnsRequiredForSortingKey()) columns_ttl_forbidden.insert(col); for (const auto & [name, ast] : new_column_ttls) @@ -1421,8 +1435,9 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, const S columns_alter_type_forbidden.insert(col); } - if (sorting_key_expr) + if (hasSortingKey()) { + auto sorting_key_expr = getSortingKey().expressions; for (const ExpressionAction & action : sorting_key_expr->getActions()) { auto action_columns = action.getNeededColumns(); @@ -3085,7 +3100,7 @@ bool MergeTreeData::isPrimaryOrMinMaxKeyColumnPossiblyWrappedInFunctions(const A { const String column_name = node->getColumnName(); - for (const auto & name : primary_key_columns) + for (const auto & name : getPrimaryKey().expression_column_names) if (column_name == name) return true; @@ -3145,10 +3160,10 @@ MergeTreeData & MergeTreeData::checkStructureAndGetMergeTreeData(IStorage & sour return ast ? queryToString(ast) : ""; }; - if (query_to_string(order_by_ast) != query_to_string(src_data->order_by_ast)) + if (query_to_string(getSortingKeyAST()) != query_to_string(src_data->getSortingKeyAST())) throw Exception("Tables have different ordering", ErrorCodes::BAD_ARGUMENTS); - if (query_to_string(getPartitionKey().definition_ast) != query_to_string(src_data->getPartitionKey().definition_ast)) + if (query_to_string(getPartitionKeyAST()) != query_to_string(src_data->getPartitionKeyAST())) throw Exception("Tables have different partition key", ErrorCodes::BAD_ARGUMENTS); if (format_version != src_data->format_version) diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index ca27dc6c385..f9b7326b11d 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -334,14 +334,14 @@ public: /// See comments about methods below in IStorage interface StorageInMemoryMetadata getInMemoryMetadata() const override; - ASTPtr getSortingKeyAST() const override { return sorting_key_expr_ast; } - ASTPtr getPrimaryKeyAST() const override { return primary_key_expr_ast; } + // ASTPtr getSortingKeyAST() const override { return sorting_key_expr_ast; } + //ASTPtr getPrimaryKeyAST() const override { return primary_key_expr_ast; } //Names getColumnsRequiredForPartitionKey() const override { return (partition_key_expr ? partition_key_expr->getRequiredColumns() : Names{}); } - Names getColumnsRequiredForSortingKey() const override { return sorting_key_expr->getRequiredColumns(); } - Names getColumnsRequiredForPrimaryKey() const override { return primary_key_expr->getRequiredColumns(); } - Names getColumnsRequiredForFinal() const override { return sorting_key_expr->getRequiredColumns(); } - Names getSortingKeyColumns() const override { return sorting_key_columns; } + //Names getColumnsRequiredForSortingKey() const override { return sorting_key_expr->getRequiredColumns(); } + //Names getColumnsRequiredForPrimaryKey() const override { return primary_key_expr->getRequiredColumns(); } + //Names getColumnsRequiredForFinal() const override { return sorting_key_expr->getRequiredColumns(); } + //Names getSortingKeyColumns() const override { return sorting_key_columns; } ColumnDependencies getColumnDependencies(const NameSet & updated_columns) const override; @@ -525,8 +525,7 @@ public: */ static ASTPtr extractKeyExpressionList(const ASTPtr & node); - bool hasSortingKey() const { return !sorting_key_columns.empty(); } - bool hasPrimaryKey() const { return !primary_key_columns.empty(); } + //bool hasPrimaryKey() const { return !primary_key_columns.empty(); } bool hasSkipIndices() const { return !skip_indices.empty(); } bool hasAnyColumnTTL() const { return !column_ttl_entries_by_name.empty(); } @@ -659,16 +658,16 @@ public: /// Names of sorting key columns in ORDER BY expression. For example: 'a', /// 'x * y', 'toStartOfMonth(date)', etc. - Names sorting_key_columns; - ASTPtr sorting_key_expr_ast; - ExpressionActionsPtr sorting_key_expr; + //Names sorting_key_columns; + //ASTPtr sorting_key_expr_ast; + //ExpressionActionsPtr sorting_key_expr; /// Names of columns for primary key. - Names primary_key_columns; - ASTPtr primary_key_expr_ast; - ExpressionActionsPtr primary_key_expr; - Block primary_key_sample; - DataTypes primary_key_data_types; + //Names primary_key_columns; + //ASTPtr primary_key_expr_ast; + //ExpressionActionsPtr primary_key_expr; + //Block primary_key_sample; + //DataTypes primary_key_data_types; struct TTLEntry { @@ -729,8 +728,8 @@ protected: friend struct ReplicatedMergeTreeTableMetadata; friend class StorageReplicatedMergeTree; - ASTPtr order_by_ast; - ASTPtr primary_key_ast; + ///ASTPtr order_by_ast; + ///ASTPtr primary_key_ast; ASTPtr ttl_table_ast; ASTPtr settings_ast; diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 625dc9f367a..edbc00a2b54 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -607,7 +607,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor NamesAndTypesList merging_columns; Names gathering_column_names, merging_column_names; extractMergingAndGatheringColumns( - storage_columns, data.sorting_key_expr, data.skip_indices, + storage_columns, data.getSortingKey().expressions, data.skip_indices, data.merging_params, gathering_columns, gathering_column_names, merging_columns, merging_column_names); auto single_disk_volume = std::make_shared("volume_" + future_part.name, disk); @@ -726,7 +726,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor pipes.emplace_back(std::move(pipe)); } - Names sort_columns = data.sorting_key_columns; + Names sort_columns = data.getSortingKey().expression_column_names; SortDescription sort_description; size_t sort_columns_size = sort_columns.size(); sort_description.reserve(sort_columns_size); diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 3d18b8decc6..eb6fcb67f82 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -222,9 +222,10 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( data.check(real_column_names); const Settings & settings = context.getSettingsRef(); - Names primary_key_columns = data.primary_key_columns; + const auto & primary_key = data.getPrimaryKey(); + Names primary_key_columns = primary_key.expression_column_names; - KeyCondition key_condition(query_info, context, primary_key_columns, data.primary_key_expr); + KeyCondition key_condition(query_info, context, primary_key_columns, primary_key.expressions); if (settings.force_primary_key && key_condition.alwaysUnknownOrTrue()) { @@ -613,7 +614,7 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( if (select.final()) { /// Add columns needed to calculate the sorting expression and the sign. - std::vector add_columns = data.sorting_key_expr->getRequiredColumns(); + std::vector add_columns = data.getColumnsRequiredForSortingKey(); column_names_to_read.insert(column_names_to_read.end(), add_columns.begin(), add_columns.end()); if (!data.merging_params.sign_column.empty()) @@ -639,7 +640,7 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( else if (settings.optimize_read_in_order && query_info.input_sorting_info) { size_t prefix_size = query_info.input_sorting_info->order_key_prefix_descr.size(); - auto order_key_prefix_ast = data.sorting_key_expr_ast->clone(); + auto order_key_prefix_ast = data.getSortingKey().expression_ast->clone(); order_key_prefix_ast->children.resize(prefix_size); auto syntax_result = SyntaxAnalyzer(context).analyze(order_key_prefix_ast, data.getColumns().getAllPhysical()); @@ -1024,7 +1025,7 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder( { SortDescription sort_description; for (size_t j = 0; j < input_sorting_info->order_key_prefix_descr.size(); ++j) - sort_description.emplace_back(data.sorting_key_columns[j], + sort_description.emplace_back(data.getSortingKey().expression_column_names[j], input_sorting_info->direction, 1); /// Drop temporary columns, added by 'sorting_key_prefix_expr' @@ -1097,11 +1098,11 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal( if (!out_projection) out_projection = createProjection(pipe, data); - pipe.addSimpleTransform(std::make_shared(pipe.getHeader(), data.sorting_key_expr)); + pipe.addSimpleTransform(std::make_shared(pipe.getHeader(), data.getSortingKey().expressions)); pipes.emplace_back(std::move(pipe)); } - Names sort_columns = data.sorting_key_columns; + Names sort_columns = data.getSortingKey().expression_column_names; SortDescription sort_description; size_t sort_columns_size = sort_columns.size(); sort_description.reserve(sort_columns_size); @@ -1294,11 +1295,12 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange( std::function create_field_ref; /// If there are no monotonic functions, there is no need to save block reference. /// Passing explicit field to FieldRef allows to optimize ranges and shows better performance. + const auto & primary_key = data.getPrimaryKey(); if (key_condition.hasMonotonicFunctionsChain()) { auto index_block = std::make_shared(); for (size_t i = 0; i < used_key_size; ++i) - index_block->insert({index[i], data.primary_key_data_types[i], data.primary_key_columns[i]}); + index_block->insert({index[i], primary_key.data_types[i], primary_key.expression_column_names[i]}); create_field_ref = [index_block](size_t row, size_t column, FieldRef & field) { @@ -1329,7 +1331,7 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange( create_field_ref(range.begin, i, index_left[i]); may_be_true = key_condition.mayBeTrueAfter( - used_key_size, index_left.data(), data.primary_key_data_types); + used_key_size, index_left.data(), primary_key.data_types); } else { @@ -1343,7 +1345,7 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange( } may_be_true = key_condition.mayBeTrueInRange( - used_key_size, index_left.data(), index_right.data(), data.primary_key_data_types); + used_key_size, index_left.data(), index_right.data(), primary_key.data_types); } if (!may_be_true) diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 34377bb29f6..e79765c3603 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -262,7 +262,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa if (data.hasSortingKey() || data.hasSkipIndices()) data.sorting_key_and_skip_indices_expr->execute(block); - Names sort_columns = data.sorting_key_columns; + Names sort_columns = data.getSortingKey().expression_column_names; SortDescription sort_description; size_t sort_columns_size = sort_columns.size(); sort_description.reserve(sort_columns_size); diff --git a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp index 623da6318ff..759241669ff 100644 --- a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp +++ b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp @@ -39,8 +39,9 @@ MergeTreeWhereOptimizer::MergeTreeWhereOptimizer( block_with_constants{KeyCondition::getBlockWithConstants(query_info.query, query_info.syntax_analyzer_result, context)}, log{log_} { - if (!data.primary_key_columns.empty()) - first_primary_key_column = data.primary_key_columns[0]; + const auto & primary_key = data.getPrimaryKey(); + if (!primary_key.expression_column_names.empty()) + first_primary_key_column = primary_key.expression_column_names[0]; calculateColumnSizes(data, queried_columns); determineArrayJoinedNames(query_info.query->as()); diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/src/Storages/MergeTree/MergedBlockOutputStream.cpp index e0760e87d00..2cc7cafb01d 100644 --- a/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -161,7 +161,7 @@ void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Perm std::inserter(skip_indexes_column_names_set, skip_indexes_column_names_set.end())); Names skip_indexes_column_names(skip_indexes_column_names_set.begin(), skip_indexes_column_names_set.end()); - Block primary_key_block = getBlockAndPermute(block, storage.primary_key_columns, permutation); + Block primary_key_block = getBlockAndPermute(block, storage.getPrimaryKey().expression_column_names, permutation); Block skip_indexes_block = getBlockAndPermute(block, skip_indexes_column_names, permutation); writer->write(block, permutation, primary_key_block, skip_indexes_block); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp index 0cf10384ba5..5755603ee93 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp @@ -40,12 +40,12 @@ ReplicatedMergeTreeTableMetadata::ReplicatedMergeTreeTableMetadata(const MergeTr /// So rules in zookeeper metadata is following: /// - When we have only ORDER BY, than store it in "primary key:" row of /metadata /// - When we have both, than store PRIMARY KEY in "primary key:" row and ORDER BY in "sorting key:" row of /metadata - if (!data.primary_key_ast) - primary_key = formattedAST(MergeTreeData::extractKeyExpressionList(data.order_by_ast)); + if (!data.hasPrimaryKey()) + primary_key = formattedAST(data.getSortingKey().expression_ast); else { - primary_key = formattedAST(MergeTreeData::extractKeyExpressionList(data.primary_key_ast)); - sorting_key = formattedAST(MergeTreeData::extractKeyExpressionList(data.order_by_ast)); + primary_key = formattedAST(data.getPrimaryKey().expression_ast); + sorting_key = formattedAST(data.getSortingKey().expression_ast); } data_format_version = data.format_version; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index e6fc32ecbf9..df6564b3368 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -493,11 +493,11 @@ void StorageReplicatedMergeTree::setTableStructure(ColumnsDescription new_column metadata.order_by_ast = tuple; } - if (!primary_key_ast) + if (!hasPrimaryKey()) { /// Primary and sorting key become independent after this ALTER so we have to /// save the old ORDER BY expression as the new primary key. - metadata.primary_key_ast = order_by_ast->clone(); + metadata.primary_key_ast = getSortingKeyAST()->clone(); } } From 84fda62e2c6cb1b687597f891eee9cb343278cdf Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 20 May 2020 21:13:03 +0300 Subject: [PATCH 483/738] [docs] minor translation update (#11072) --- website/locale/en/LC_MESSAGES/messages.po | 2 +- website/locale/es/LC_MESSAGES/messages.mo | Bin 6641 -> 6641 bytes website/locale/es/LC_MESSAGES/messages.po | 2 +- website/locale/fa/LC_MESSAGES/messages.mo | Bin 7453 -> 7453 bytes website/locale/fa/LC_MESSAGES/messages.po | 2 +- website/locale/fr/LC_MESSAGES/messages.mo | Bin 6490 -> 6490 bytes website/locale/fr/LC_MESSAGES/messages.po | 2 +- website/locale/ja/LC_MESSAGES/messages.mo | Bin 6613 -> 6613 bytes website/locale/ja/LC_MESSAGES/messages.po | 2 +- website/locale/messages.pot | 2 +- website/locale/ru/LC_MESSAGES/messages.mo | Bin 8554 -> 8556 bytes website/locale/ru/LC_MESSAGES/messages.po | 6 +++--- website/locale/tr/LC_MESSAGES/messages.mo | Bin 6378 -> 6378 bytes website/locale/tr/LC_MESSAGES/messages.po | 2 +- website/locale/zh/LC_MESSAGES/messages.mo | Bin 5926 -> 5926 bytes website/locale/zh/LC_MESSAGES/messages.po | 2 +- 16 files changed, 11 insertions(+), 11 deletions(-) diff --git a/website/locale/en/LC_MESSAGES/messages.po b/website/locale/en/LC_MESSAGES/messages.po index ee9b7b2cc4f..c2c37ebf2b1 100644 --- a/website/locale/en/LC_MESSAGES/messages.po +++ b/website/locale/en/LC_MESSAGES/messages.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2020-05-19 11:11+0300\n" +"POT-Creation-Date: 2020-05-20 15:32+0300\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language: en\n" diff --git a/website/locale/es/LC_MESSAGES/messages.mo b/website/locale/es/LC_MESSAGES/messages.mo index b6bd03868f8b16c36c591d92f0487347fa8fbe0a..9b61bb401bb8d56b03085f60b85d14bdc3c28009 100644 GIT binary patch delta 21 ccmexp{Ly&B4qgr;0|i4\n" "Language: es\n" diff --git a/website/locale/fa/LC_MESSAGES/messages.mo b/website/locale/fa/LC_MESSAGES/messages.mo index ef35154a13b39f998b0ca95f3fe4898c7375ad17..36d0063da7a33529fbb93d0dc770594a8f758d80 100644 GIT binary patch delta 21 ccmbPhHP>px4qgr;0|i4px4qgsJO9ewiD?`K02Y6%o0Z)Vmpa1{> diff --git a/website/locale/fa/LC_MESSAGES/messages.po b/website/locale/fa/LC_MESSAGES/messages.po index 9340903800d..65eeb6f605a 100644 --- a/website/locale/fa/LC_MESSAGES/messages.po +++ b/website/locale/fa/LC_MESSAGES/messages.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2020-05-19 11:11+0300\n" +"POT-Creation-Date: 2020-05-20 15:32+0300\n" "PO-Revision-Date: 2020-03-26 10:19+0300\n" "Last-Translator: FULL NAME \n" "Language: fa\n" diff --git a/website/locale/fr/LC_MESSAGES/messages.mo b/website/locale/fr/LC_MESSAGES/messages.mo index a5fe91d114e7db67f96c45963df30ee3067dd686..6d5d70236c139ece8255fb2ceaee84ce6c7d96b3 100644 GIT binary patch delta 21 dcmca*bjxVN4qgr;0|i4\n" "Language: fr\n" diff --git a/website/locale/ja/LC_MESSAGES/messages.mo b/website/locale/ja/LC_MESSAGES/messages.mo index 191276fb16d5ecc45840f0f97058f252a7eeccd1..4577e84e199bbcf81da749052a34b8ce358e4e1f 100644 GIT binary patch delta 21 ccmca=eARfv4qgr;0|i4\n" "Language: ja\n" diff --git a/website/locale/messages.pot b/website/locale/messages.pot index 2e1259fc0c6..75aae739cff 100644 --- a/website/locale/messages.pot +++ b/website/locale/messages.pot @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2020-05-19 11:11+0300\n" +"POT-Creation-Date: 2020-05-20 15:32+0300\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" diff --git a/website/locale/ru/LC_MESSAGES/messages.mo b/website/locale/ru/LC_MESSAGES/messages.mo index d6d6af10b607b6396950b7b7aa1bbbf89433fe30..38fc03212af92bef1bc9b77364958447cf048c5d 100644 GIT binary patch delta 66 zcmaFm^u}pJ4ht_I0|SGzA_IdYkPg~h$zspRVPv3SXli9_wD|z<21cHXjTf6PY`EBT WvFE~$%{Ba&1ey0;*fiN+Q5pb~Ru_l> delta 64 zcmaFk^vY>N4ht_20|SGzA_IdYkPg^f$zspRVQ8sfXlP|3okZZ*Z{;^ SFYMf0!+%MTal_<5MQH$KKNdXz diff --git a/website/locale/ru/LC_MESSAGES/messages.po b/website/locale/ru/LC_MESSAGES/messages.po index df9e1492c27..aed233c1275 100644 --- a/website/locale/ru/LC_MESSAGES/messages.po +++ b/website/locale/ru/LC_MESSAGES/messages.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2020-05-19 11:11+0300\n" +"POT-Creation-Date: 2020-05-20 15:32+0300\n" "PO-Revision-Date: 2020-03-26 10:19+0300\n" "Last-Translator: FULL NAME \n" "Language: ru\n" @@ -84,11 +84,11 @@ msgstr "Рейтинг" #: templates/docs/footer.html:3 msgid "votes" -msgstr "голоса" +msgstr "голосов" #: templates/docs/footer.html:4 msgid "Article Rating" -msgstr "Рейтинг Статей" +msgstr "Рейтинг статьи" #: templates/docs/footer.html:4 msgid "Was this content helpful?" diff --git a/website/locale/tr/LC_MESSAGES/messages.mo b/website/locale/tr/LC_MESSAGES/messages.mo index a8bb89012865a5b30bf3beed3b763a4637e8180a..1cfc686a5ed1392629766d00ad23bfdb739bd789 100644 GIT binary patch delta 21 ccmaE5_{wm@4qgr;0|i4L6@Bjb+ delta 21 ccmaE5_{wm@4qgsJO9ewiD?`K02Y9u(0a?rj@c;k- diff --git a/website/locale/tr/LC_MESSAGES/messages.po b/website/locale/tr/LC_MESSAGES/messages.po index 35cd573b0b3..ddffb96f232 100644 --- a/website/locale/tr/LC_MESSAGES/messages.po +++ b/website/locale/tr/LC_MESSAGES/messages.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2020-05-19 11:11+0300\n" +"POT-Creation-Date: 2020-05-20 15:32+0300\n" "PO-Revision-Date: 2020-04-15 13:17+0000\n" "Last-Translator: FULL NAME \n" "Language: tr\n" diff --git a/website/locale/zh/LC_MESSAGES/messages.mo b/website/locale/zh/LC_MESSAGES/messages.mo index 0b2efedd38bf7e407fa5e2ef3c3a668e41d64471..7260ce95e89b510777ffebc6c96cb915de5e9939 100644 GIT binary patch delta 21 ccmZ3cw@h!t4qgr;0|i4\n" "Language: zh\n" From 6f50700f3524b943cd66612ace6c55cdb2c02604 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 20 May 2020 21:53:18 +0300 Subject: [PATCH 484/738] Add PullingOutputFormat and PullingPipelineExecutor. --- .../PipelineExecutingBlockInputStream.cpp | 47 +++- .../PipelineExecutingBlockInputStream.h | 5 + src/Processors/Executors/PipelineExecutor.cpp | 6 +- src/Processors/Executors/PipelineExecutor.h | 8 +- .../PullingAsyncPipelineExecutor.cpp | 203 ++++++++++++++++++ .../Executors/PullingAsyncPipelineExecutor.h | 58 +++++ .../Executors/PullingPipelineExecutor.cpp | 129 ++--------- .../Executors/PullingPipelineExecutor.h | 22 +- src/Processors/Formats/LazyOutputFormat.h | 2 +- .../Formats/PullingOutputFormat.cpp | 39 ++++ src/Processors/Formats/PullingOutputFormat.h | 45 ++++ src/Processors/ya.make | 1 + 12 files changed, 429 insertions(+), 136 deletions(-) create mode 100644 src/Processors/Executors/PullingAsyncPipelineExecutor.cpp create mode 100644 src/Processors/Executors/PullingAsyncPipelineExecutor.h create mode 100644 src/Processors/Formats/PullingOutputFormat.cpp create mode 100644 src/Processors/Formats/PullingOutputFormat.h diff --git a/src/Processors/Executors/PipelineExecutingBlockInputStream.cpp b/src/Processors/Executors/PipelineExecutingBlockInputStream.cpp index 75324cb25b8..d39938b57c5 100644 --- a/src/Processors/Executors/PipelineExecutingBlockInputStream.cpp +++ b/src/Processors/Executors/PipelineExecutingBlockInputStream.cpp @@ -1,7 +1,7 @@ #include +#include #include #include -#include namespace DB { @@ -20,27 +20,54 @@ PipelineExecutingBlockInputStream::~PipelineExecutingBlockInputStream() = defaul Block PipelineExecutingBlockInputStream::getHeader() const { - return executor ? executor->getHeader() - : pipeline->getHeader(); + if (executor) + return executor->getHeader(); + + if (async_executor) + return async_executor->getHeader(); + + return pipeline->getHeader(); +} + +void PipelineExecutingBlockInputStream::createExecutor() +{ + if (pipeline->getNumThreads() > 1) + async_executor = std::make_unique(*pipeline); + else + executor = std::make_unique(*pipeline); + + is_execution_started = true; } void PipelineExecutingBlockInputStream::readPrefixImpl() { - executor = std::make_unique(*pipeline); + createExecutor(); } Block PipelineExecutingBlockInputStream::readImpl() { - if (!executor) - executor = std::make_unique(*pipeline); + if (!is_execution_started) + createExecutor(); Block block; - while (executor->pull(block)) + bool can_continue = true; + while (can_continue) { + if (executor) + can_continue = executor->pull(block); + else + can_continue = async_executor->pull(block); + if (block) return block; } + totals = executor ? executor->getTotalsBlock() + : async_executor->getTotalsBlock(); + + extremes = executor ? executor->getExtremesBlock() + : async_executor->getExtremesBlock(); + return {}; } @@ -70,20 +97,20 @@ void PipelineExecutingBlockInputStream::cancel(bool kill) void PipelineExecutingBlockInputStream::setProgressCallback(const ProgressCallback & callback) { - throwIfExecutionStarted(executor != nullptr, "setProgressCallback"); + throwIfExecutionStarted(is_execution_started, "setProgressCallback"); pipeline->setProgressCallback(callback); } void PipelineExecutingBlockInputStream::setProcessListElement(QueryStatus * elem) { - throwIfExecutionStarted(executor != nullptr, "setProcessListElement"); + throwIfExecutionStarted(is_execution_started, "setProcessListElement"); IBlockInputStream::setProcessListElement(elem); pipeline->setProcessListElement(elem); } void PipelineExecutingBlockInputStream::setLimits(const IBlockInputStream::LocalLimits & limits_) { - throwIfExecutionStarted(executor != nullptr, "setLimits"); + throwIfExecutionStarted(is_execution_started, "setLimits"); if (limits_.mode == LimitsMode::LIMITS_TOTAL) throw Exception("Total limits are not supported by PipelineExecutingBlockInputStream", diff --git a/src/Processors/Executors/PipelineExecutingBlockInputStream.h b/src/Processors/Executors/PipelineExecutingBlockInputStream.h index 7555e1dec97..11ccb71f17a 100644 --- a/src/Processors/Executors/PipelineExecutingBlockInputStream.h +++ b/src/Processors/Executors/PipelineExecutingBlockInputStream.h @@ -5,6 +5,7 @@ namespace DB { class QueryPipeline; +class PullingAsyncPipelineExecutor; class PullingPipelineExecutor; /// Implement IBlockInputStream from QueryPipeline. @@ -34,6 +35,10 @@ protected: private: std::unique_ptr pipeline; std::unique_ptr executor; + std::unique_ptr async_executor; + bool is_execution_started = false; + + void createExecutor(); }; } diff --git a/src/Processors/Executors/PipelineExecutor.cpp b/src/Processors/Executors/PipelineExecutor.cpp index 5825a8c90d6..a5a43a9f546 100644 --- a/src/Processors/Executors/PipelineExecutor.cpp +++ b/src/Processors/Executors/PipelineExecutor.cpp @@ -500,7 +500,7 @@ bool PipelineExecutor::executeStep(std::atomic_bool * yield_flag) return false; if (!is_execution_initialized) - initExecution(1); + initializeExecution(1); executeStep(yield_flag); @@ -720,7 +720,7 @@ void PipelineExecutor::executeStepImpl(size_t thread_num, size_t num_threads, st #endif } -void PipelineExecutor::initExecution(size_t num_threads) +void PipelineExecutor::initializeExecution(size_t num_threads) { is_execution_initialized = true; @@ -766,7 +766,7 @@ void PipelineExecutor::initExecution(size_t num_threads) void PipelineExecutor::executeImpl(size_t num_threads) { - initExecution(num_threads); + initializeExecution(num_threads); using ThreadsData = std::vector; ThreadsData threads; diff --git a/src/Processors/Executors/PipelineExecutor.h b/src/Processors/Executors/PipelineExecutor.h index 0a15f5cc974..395f42a3316 100644 --- a/src/Processors/Executors/PipelineExecutor.h +++ b/src/Processors/Executors/PipelineExecutor.h @@ -36,12 +36,10 @@ public: void execute(size_t num_threads); /// Execute single step. Step will be stopped when yield_flag is true. - /// Execution is happened in single thread. + /// Execution is happened in a single thread. /// Return true if execution should be continued. bool executeStep(std::atomic_bool * yield_flag = nullptr); - String getName() const { return "PipelineExecutor"; } - const Processors & getProcessors() const { return processors; } /// Cancel execution. May be called from another thread. @@ -208,7 +206,7 @@ private: ThreadsQueue threads_queue; std::mutex task_queue_mutex; - /// Flag that checks that initExecution was called. + /// Flag that checks that initializeExecution was called. bool is_execution_initialized = false; std::atomic_bool cancelled; std::atomic_bool finished; @@ -287,7 +285,7 @@ private: /// Continue executor (in case there are tasks in queue). void wakeUpExecutor(size_t thread_num); - void initExecution(size_t num_threads); /// Initialize executor contexts and task_queue. + void initializeExecution(size_t num_threads); /// Initialize executor contexts and task_queue. void finalizeExecution(); /// Check all processors are finished. void executeImpl(size_t num_threads); diff --git a/src/Processors/Executors/PullingAsyncPipelineExecutor.cpp b/src/Processors/Executors/PullingAsyncPipelineExecutor.cpp new file mode 100644 index 00000000000..003508ab86f --- /dev/null +++ b/src/Processors/Executors/PullingAsyncPipelineExecutor.cpp @@ -0,0 +1,203 @@ +#include +#include +#include +#include +#include + +#include +#include + +namespace DB +{ + +struct PullingAsyncPipelineExecutor::Data +{ + PipelineExecutorPtr executor; + std::exception_ptr exception; + std::atomic_bool is_executed = false; + std::atomic_bool has_exception = false; + ThreadFromGlobalPool thread; + + ~Data() + { + if (thread.joinable()) + thread.join(); + } + + void rethrowExceptionIfHas() + { + if (has_exception) + { + has_exception = false; + std::rethrow_exception(std::move(exception)); + } + } +}; + +PullingAsyncPipelineExecutor::PullingAsyncPipelineExecutor(QueryPipeline & pipeline_) : pipeline(pipeline_) +{ + lazy_format = std::make_shared(pipeline.getHeader()); + pipeline.setOutput(lazy_format); +} + +PullingAsyncPipelineExecutor::~PullingAsyncPipelineExecutor() +{ + try + { + cancel(); + } + catch (...) + { + tryLogCurrentException("PullingAsyncPipelineExecutor"); + } +} + +const Block & PullingAsyncPipelineExecutor::getHeader() const +{ + return lazy_format->getPort(IOutputFormat::PortKind::Main).getHeader(); +} + +static void threadFunction(PullingAsyncPipelineExecutor::Data & data, ThreadGroupStatusPtr thread_group, size_t num_threads) +{ + if (thread_group) + CurrentThread::attachTo(thread_group); + + SCOPE_EXIT( + if (thread_group) + CurrentThread::detachQueryIfNotDetached(); + ); + + setThreadName("QueryPipelineEx"); + + try + { + data.executor->execute(num_threads); + } + catch (...) + { + data.exception = std::current_exception(); + data.has_exception = true; + } +} + + +bool PullingAsyncPipelineExecutor::pull(Chunk & chunk, uint64_t milliseconds) +{ + if (!data) + { + data = std::make_unique(); + data->executor = pipeline.execute(); + + auto func = [&, thread_group = CurrentThread::getGroup()]() + { + threadFunction(*data, thread_group, pipeline.getNumThreads()); + }; + + data->thread = ThreadFromGlobalPool(std::move(func)); + } + + if (data->has_exception) + { + /// Finish lazy format in case of exception. Otherwise thread.join() may hung. + lazy_format->finish(); + data->has_exception = false; + std::rethrow_exception(std::move(data->exception)); + } + + if (lazy_format->isFinished()) + { + data->is_executed = true; + /// Wait thread ant rethrow exception if any. + cancel(); + return false; + } + + chunk = lazy_format->getChunk(milliseconds); + return true; +} + +bool PullingAsyncPipelineExecutor::pull(Block & block, uint64_t milliseconds) +{ + Chunk chunk; + + if (!pull(chunk, milliseconds)) + return false; + + if (!chunk) + { + /// In case if timeout exceeded. + block.clear(); + return true; + } + + block = lazy_format->getPort(IOutputFormat::PortKind::Main).getHeader().cloneWithColumns(chunk.detachColumns()); + + if (auto chunk_info = chunk.getChunkInfo()) + { + if (const auto * agg_info = typeid_cast(chunk_info.get())) + { + block.info.bucket_num = agg_info->bucket_num; + block.info.is_overflows = agg_info->is_overflows; + } + } + + return true; +} + +void PullingAsyncPipelineExecutor::cancel() +{ + /// Cancel execution if it wasn't finished. + if (data && !data->is_executed && data->executor) + data->executor->cancel(); + + /// Finish lazy format. Otherwise thread.join() may hung. + if (!lazy_format->isFinished()) + lazy_format->finish(); + + /// Join thread here to wait for possible exception. + if (data && data->thread.joinable()) + data->thread.join(); + + /// Rethrow exception to not swallow it in destructor. + if (data) + data->rethrowExceptionIfHas(); +} + +Chunk PullingAsyncPipelineExecutor::getTotals() +{ + return lazy_format->getTotals(); +} + +Chunk PullingAsyncPipelineExecutor::getExtremes() +{ + return lazy_format->getExtremes(); +} + +Block PullingAsyncPipelineExecutor::getTotalsBlock() +{ + auto totals = getTotals(); + + if (totals.empty()) + return {}; + + const auto & header = lazy_format->getPort(IOutputFormat::PortKind::Totals).getHeader(); + return header.cloneWithColumns(totals.detachColumns()); +} + +Block PullingAsyncPipelineExecutor::getExtremesBlock() +{ + auto extremes = getExtremes(); + + if (extremes.empty()) + return {}; + + const auto & header = lazy_format->getPort(IOutputFormat::PortKind::Extremes).getHeader(); + return header.cloneWithColumns(extremes.detachColumns()); +} + +BlockStreamProfileInfo & PullingAsyncPipelineExecutor::getProfileInfo() +{ + return lazy_format->getProfileInfo(); +} + +} diff --git a/src/Processors/Executors/PullingAsyncPipelineExecutor.h b/src/Processors/Executors/PullingAsyncPipelineExecutor.h new file mode 100644 index 00000000000..2ce75aecab7 --- /dev/null +++ b/src/Processors/Executors/PullingAsyncPipelineExecutor.h @@ -0,0 +1,58 @@ +#pragma once +#include + +namespace DB +{ + +class QueryPipeline; +class Block; +class Chunk; +class LazyOutputFormat; +struct BlockStreamProfileInfo; + +/// Asynchronous pulling executor for QueryPipeline. +/// Always creates extra thread. If query is executed in single thread, use PullingPipelineExecutor. +/// Typical usage is: +/// +/// PullingAsyncPipelineExecutor executor(query_pipeline); +/// while (executor.pull(chunk, timeout)) +/// ... process chunk ... +class PullingAsyncPipelineExecutor +{ +public: + explicit PullingAsyncPipelineExecutor(QueryPipeline & pipeline_); + ~PullingAsyncPipelineExecutor(); + + /// Get structure of returned block or chunk. + const Block & getHeader() const; + + /// Methods return false if query is finished. + /// If milliseconds > 0, returns empty object and `true` after timeout exceeded. Otherwise method is blocking. + /// You can use any pull method. + bool pull(Chunk & chunk, uint64_t milliseconds = 0); + bool pull(Block & block, uint64_t milliseconds = 0); + + /// Stop execution. It is not necessary, but helps to stop execution before executor is destroyed. + void cancel(); + + /// Get totals and extremes. Returns empty chunk if doesn't have any. + Chunk getTotals(); + Chunk getExtremes(); + + /// Get totals and extremes. Returns empty chunk if doesn't have any. + Block getTotalsBlock(); + Block getExtremesBlock(); + + /// Get query profile info. + BlockStreamProfileInfo & getProfileInfo(); + + /// Internal executor data. + struct Data; + +private: + QueryPipeline & pipeline; + std::shared_ptr lazy_format; + std::unique_ptr data; +}; + +} diff --git a/src/Processors/Executors/PullingPipelineExecutor.cpp b/src/Processors/Executors/PullingPipelineExecutor.cpp index 223c22e59db..375f6c9ed0e 100644 --- a/src/Processors/Executors/PullingPipelineExecutor.cpp +++ b/src/Processors/Executors/PullingPipelineExecutor.cpp @@ -1,43 +1,15 @@ #include -#include -#include -#include +#include #include - -#include -#include +#include namespace DB { -struct PullingPipelineExecutor::Data -{ - PipelineExecutorPtr executor; - std::exception_ptr exception; - std::atomic_bool is_executed = false; - std::atomic_bool has_exception = false; - ThreadFromGlobalPool thread; - - ~Data() - { - if (thread.joinable()) - thread.join(); - } - - void rethrowExceptionIfHas() - { - if (has_exception) - { - has_exception = false; - std::rethrow_exception(std::move(exception)); - } - } -}; - PullingPipelineExecutor::PullingPipelineExecutor(QueryPipeline & pipeline_) : pipeline(pipeline_) { - lazy_format = std::make_shared(pipeline.getHeader()); - pipeline.setOutput(lazy_format); + pulling_format = std::make_shared(pipeline.getHeader(), has_data_flag); + pipeline.setOutput(pulling_format); } PullingPipelineExecutor::~PullingPipelineExecutor() @@ -54,73 +26,26 @@ PullingPipelineExecutor::~PullingPipelineExecutor() const Block & PullingPipelineExecutor::getHeader() const { - return lazy_format->getPort(IOutputFormat::PortKind::Main).getHeader(); + return pulling_format->getPort(IOutputFormat::PortKind::Main).getHeader(); } -static void threadFunction(PullingPipelineExecutor::Data & data, ThreadGroupStatusPtr thread_group, size_t num_threads) +bool PullingPipelineExecutor::pull(Chunk & chunk) { - if (thread_group) - CurrentThread::attachTo(thread_group); + if (!executor) + executor = pipeline.execute(); - SCOPE_EXIT( - if (thread_group) - CurrentThread::detachQueryIfNotDetached(); - ); - - setThreadName("QueryPipelineEx"); - - try - { - data.executor->execute(num_threads); - } - catch (...) - { - data.exception = std::current_exception(); - data.has_exception = true; - } -} - - -bool PullingPipelineExecutor::pull(Chunk & chunk, uint64_t milliseconds) -{ - if (!data) - { - data = std::make_unique(); - data->executor = pipeline.execute(); - - auto func = [&, thread_group = CurrentThread::getGroup()]() - { - threadFunction(*data, thread_group, pipeline.getNumThreads()); - }; - - data->thread = ThreadFromGlobalPool(std::move(func)); - } - - if (data->has_exception) - { - /// Finish lazy format in case of exception. Otherwise thread.join() may hung. - lazy_format->finish(); - data->has_exception = false; - std::rethrow_exception(std::move(data->exception)); - } - - if (lazy_format->isFinished()) - { - data->is_executed = true; - /// Wait thread ant rethrow exception if any. - cancel(); + if (!executor->executeStep(&has_data_flag)) return false; - } - chunk = lazy_format->getChunk(milliseconds); + chunk = pulling_format->getChunk(); return true; } -bool PullingPipelineExecutor::pull(Block & block, uint64_t milliseconds) +bool PullingPipelineExecutor::pull(Block & block) { Chunk chunk; - if (!pull(chunk, milliseconds)) + if (!pull(chunk)) return false; if (!chunk) @@ -130,7 +55,7 @@ bool PullingPipelineExecutor::pull(Block & block, uint64_t milliseconds) return true; } - block = lazy_format->getPort(IOutputFormat::PortKind::Main).getHeader().cloneWithColumns(chunk.detachColumns()); + block = pulling_format->getPort(IOutputFormat::PortKind::Main).getHeader().cloneWithColumns(chunk.detachColumns()); if (auto chunk_info = chunk.getChunkInfo()) { @@ -147,30 +72,22 @@ bool PullingPipelineExecutor::pull(Block & block, uint64_t milliseconds) void PullingPipelineExecutor::cancel() { /// Cancel execution if it wasn't finished. - if (data && !data->is_executed && data->executor) - data->executor->cancel(); + if (executor) + executor->cancel(); - /// Finish lazy format. Otherwise thread.join() may hung. - if (!lazy_format->isFinished()) - lazy_format->finish(); - - /// Join thread here to wait for possible exception. - if (data && data->thread.joinable()) - data->thread.join(); - - /// Rethrow exception to not swallow it in destructor. - if (data) - data->rethrowExceptionIfHas(); + /// Read all data and finish execution. + Chunk chunk; + while (pull(chunk)); } Chunk PullingPipelineExecutor::getTotals() { - return lazy_format->getTotals(); + return pulling_format->getTotals(); } Chunk PullingPipelineExecutor::getExtremes() { - return lazy_format->getExtremes(); + return pulling_format->getExtremes(); } Block PullingPipelineExecutor::getTotalsBlock() @@ -180,7 +97,7 @@ Block PullingPipelineExecutor::getTotalsBlock() if (totals.empty()) return {}; - const auto & header = lazy_format->getPort(IOutputFormat::PortKind::Totals).getHeader(); + const auto & header = pulling_format->getPort(IOutputFormat::PortKind::Totals).getHeader(); return header.cloneWithColumns(totals.detachColumns()); } @@ -191,13 +108,13 @@ Block PullingPipelineExecutor::getExtremesBlock() if (extremes.empty()) return {}; - const auto & header = lazy_format->getPort(IOutputFormat::PortKind::Extremes).getHeader(); + const auto & header = pulling_format->getPort(IOutputFormat::PortKind::Extremes).getHeader(); return header.cloneWithColumns(extremes.detachColumns()); } BlockStreamProfileInfo & PullingPipelineExecutor::getProfileInfo() { - return lazy_format->getProfileInfo(); + return pulling_format->getProfileInfo(); } } diff --git a/src/Processors/Executors/PullingPipelineExecutor.h b/src/Processors/Executors/PullingPipelineExecutor.h index 7a093fe0022..67ef4f7cf71 100644 --- a/src/Processors/Executors/PullingPipelineExecutor.h +++ b/src/Processors/Executors/PullingPipelineExecutor.h @@ -4,13 +4,16 @@ namespace DB { -class QueryPipeline; class Block; class Chunk; -class LazyOutputFormat; +class QueryPipeline; +class PipelineExecutor; +class PullingOutputFormat; struct BlockStreamProfileInfo; -/// Pulling executor for QueryPipeline. +using PipelineExecutorPtr = std::shared_ptr; + +/// Pulling executor for QueryPipeline. Always execute pipeline in single thread. /// Typical usage is: /// /// PullingPipelineExecutor executor(query_pipeline); @@ -26,10 +29,9 @@ public: const Block & getHeader() const; /// Methods return false if query is finished. - /// If milliseconds > 0, returns empty object and `true` after timeout exceeded. /// You can use any pull method. - bool pull(Chunk & chunk, uint64_t milliseconds = 0); - bool pull(Block & block, uint64_t milliseconds = 0); + bool pull(Chunk & chunk); + bool pull(Block & block); /// Stop execution. It is not necessary, but helps to stop execution before executor is destroyed. void cancel(); @@ -45,13 +47,11 @@ public: /// Get query profile info. BlockStreamProfileInfo & getProfileInfo(); - /// Internal executor data. - struct Data; - private: + std::atomic_bool has_data_flag = false; QueryPipeline & pipeline; - std::shared_ptr lazy_format; - std::unique_ptr data; + std::shared_ptr pulling_format; + PipelineExecutorPtr executor; }; } diff --git a/src/Processors/Formats/LazyOutputFormat.h b/src/Processors/Formats/LazyOutputFormat.h index 14d7a7f47d7..06ec116f3dd 100644 --- a/src/Processors/Formats/LazyOutputFormat.h +++ b/src/Processors/Formats/LazyOutputFormat.h @@ -9,7 +9,7 @@ namespace DB /// LazyOutputFormat is used to retrieve ready data from executing pipeline. /// You can periodically call `getChunk` from separate thread. -/// Used in PullingPipelineExecutor. +/// Used in PullingAsyncPipelineExecutor. class LazyOutputFormat : public IOutputFormat { diff --git a/src/Processors/Formats/PullingOutputFormat.cpp b/src/Processors/Formats/PullingOutputFormat.cpp new file mode 100644 index 00000000000..e137fee3dd1 --- /dev/null +++ b/src/Processors/Formats/PullingOutputFormat.cpp @@ -0,0 +1,39 @@ +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +void PullingOutputFormat::consume(Chunk chunk) +{ + if (data) + throw Exception("PullingOutputFormat cannot consume chunk because it already has data", + ErrorCodes::LOGICAL_ERROR); + + if (chunk) + info.update(chunk.getNumRows(), chunk.allocatedBytes()); + + data = std::move(chunk); + has_data_flag = true; +} + +Chunk PullingOutputFormat::getChunk() +{ + auto chunk = std::move(data); + has_data_flag = false; + return chunk; +} + +Chunk PullingOutputFormat::getTotals() { return std::move(totals); } +Chunk PullingOutputFormat::getExtremes() { return std::move(extremes); } + +void PullingOutputFormat::setRowsBeforeLimit(size_t rows_before_limit) +{ + info.setRowsBeforeLimit(rows_before_limit); +} + +} diff --git a/src/Processors/Formats/PullingOutputFormat.h b/src/Processors/Formats/PullingOutputFormat.h new file mode 100644 index 00000000000..0864b5a02ef --- /dev/null +++ b/src/Processors/Formats/PullingOutputFormat.h @@ -0,0 +1,45 @@ +#pragma once +#include +#include + +namespace DB +{ + +/// Output format which is used in PullingPipelineExecutor. +class PullingOutputFormat : public IOutputFormat +{ +public: + explicit PullingOutputFormat(const Block & header, std::atomic_bool & consume_data_flag_) + : IOutputFormat(header, out) + , has_data_flag(consume_data_flag_) + {} + + String getName() const override { return "PullingOutputFormat"; } + + Chunk getChunk(); + Chunk getTotals(); + Chunk getExtremes(); + + BlockStreamProfileInfo & getProfileInfo() { return info; } + + void setRowsBeforeLimit(size_t rows_before_limit) override; + +protected: + void consume(Chunk chunk) override; + void consumeTotals(Chunk chunk) override { totals = std::move(chunk); } + void consumeExtremes(Chunk chunk) override { extremes = std::move(chunk); } + +private: + Chunk data; + Chunk totals; + Chunk extremes; + + std::atomic_bool & has_data_flag; + + BlockStreamProfileInfo info; + + /// Is not used. + static WriteBuffer out; +}; + +} diff --git a/src/Processors/ya.make b/src/Processors/ya.make index 03a9b939d42..4412aa4748c 100644 --- a/src/Processors/ya.make +++ b/src/Processors/ya.make @@ -12,6 +12,7 @@ SRCS( DelayedPortsProcessor.cpp Executors/PipelineExecutingBlockInputStream.cpp Executors/PipelineExecutor.cpp + Executors/PullingAsyncPipelineExecutor.cpp Executors/PullingPipelineExecutor.cpp Executors/TreeExecutorBlockInputStream.cpp ForkProcessor.cpp From 9e410444e752299cb6c920f6e84771e7c2f1bc77 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 20 May 2020 21:57:20 +0300 Subject: [PATCH 485/738] Better error messages about joins --- src/Interpreters/CrossToInnerJoinVisitor.cpp | 20 ++++++++++++- src/Interpreters/JoinedTables.cpp | 30 +++++++++++--------- src/Parsers/IAST.cpp | 6 ++++ src/Parsers/IAST.h | 3 ++ 4 files changed, 44 insertions(+), 15 deletions(-) diff --git a/src/Interpreters/CrossToInnerJoinVisitor.cpp b/src/Interpreters/CrossToInnerJoinVisitor.cpp index 49b52b0fb7f..4f3fc380341 100644 --- a/src/Interpreters/CrossToInnerJoinVisitor.cpp +++ b/src/Interpreters/CrossToInnerJoinVisitor.cpp @@ -239,6 +239,10 @@ bool getTables(ASTSelectQuery & select, std::vector & joined_tabl size_t num_array_join = 0; size_t num_using = 0; + // For diagnostic messages. + std::vector tables_with_using; + tables_with_using.reserve(num_tables); + for (const auto & child : tables->children) { auto * table_element = child->as(); @@ -257,6 +261,7 @@ bool getTables(ASTSelectQuery & select, std::vector & joined_tabl if (t.hasUsing()) { ++num_using; + tables_with_using.push_back(table_element); continue; } @@ -275,7 +280,20 @@ bool getTables(ASTSelectQuery & select, std::vector & joined_tabl } if (num_using && (num_tables - num_array_join) > 2) - throw Exception("Multiple CROSS/COMMA JOIN do not support USING", ErrorCodes::NOT_IMPLEMENTED); + { + std::stringstream ss; + ss << "Multiple CROSS/COMMA JOIN do not support USING (while processing '"; + for (size_t i = 0; i < tables_with_using.size(); ++i) + { + if (i) + { + ss << ", "; + } + tables_with_using[i]->format(IAST::FormatSettings(ss, true /* one line */)); + } + ss << "')"; + throw Exception(ss.str(), ErrorCodes::NOT_IMPLEMENTED); + } return !(num_array_join || num_using); } diff --git a/src/Interpreters/JoinedTables.cpp b/src/Interpreters/JoinedTables.cpp index b2591d94310..9fc867c7ae2 100644 --- a/src/Interpreters/JoinedTables.cpp +++ b/src/Interpreters/JoinedTables.cpp @@ -62,19 +62,6 @@ void replaceJoinedTable(const ASTSelectQuery & select_query) } } -template -void checkTablesWithColumns(const std::vector & tables_with_columns, const Context & context) -{ - const auto & settings = context.getSettingsRef(); - if (settings.joined_subquery_requires_alias && tables_with_columns.size() > 1) - { - for (auto & t : tables_with_columns) - if (t.table.table.empty() && t.table.alias.empty()) - throw Exception("No alias for subquery or table function in JOIN (set joined_subquery_requires_alias=0 to disable restriction).", - ErrorCodes::ALIAS_REQUIRED); - } -} - class RenameQualifiedIdentifiersMatcher { public: @@ -201,7 +188,22 @@ StoragePtr JoinedTables::getLeftTableStorage() bool JoinedTables::resolveTables() { tables_with_columns = getDatabaseAndTablesWithColumns(table_expressions, context); - checkTablesWithColumns(tables_with_columns, context); + assert(tables_with_columns.size == table_expressions.size); + + const auto & settings = context.getSettingsRef(); + if (settings.joined_subquery_requires_alias && tables_with_columns.size() > 1) + { + for (size_t i = 0; i < tables_with_columns.size(); ++i) + { + const auto & t = tables_with_columns[i]; + if (t.table.table.empty() && t.table.alias.empty()) + { + throw Exception("No alias for subquery or table function in JOIN (set joined_subquery_requires_alias=0 to disable restriction). While processing '" + + table_expressions[i]->formatForErrorMessage() + "'", + ErrorCodes::ALIAS_REQUIRED); + } + } + } return !tables_with_columns.empty(); } diff --git a/src/Parsers/IAST.cpp b/src/Parsers/IAST.cpp index eb9f1462666..8ee4154541b 100644 --- a/src/Parsers/IAST.cpp +++ b/src/Parsers/IAST.cpp @@ -87,6 +87,12 @@ size_t IAST::checkDepthImpl(size_t max_depth, size_t level) const return res; } +std::string IAST::formatForErrorMessage() const +{ + std::stringstream ss; + format(FormatSettings(ss, true /* one line */)); + return ss.str(); +} void IAST::cloneChildren() { diff --git a/src/Parsers/IAST.h b/src/Parsers/IAST.h index 246b88f8c24..4eb6662b57c 100644 --- a/src/Parsers/IAST.h +++ b/src/Parsers/IAST.h @@ -215,6 +215,9 @@ public: throw Exception("Unknown element in AST: " + getID(), ErrorCodes::UNKNOWN_ELEMENT_IN_AST); } + // A simple way to add some user-readable context to an error message. + std::string formatForErrorMessage() const; + void cloneChildren(); public: From 1183bad4d066f1c03ec2af2388fd551bdf37e1b2 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 20 May 2020 22:01:36 +0300 Subject: [PATCH 486/738] Fix build. --- programs/server/TCPHandler.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/server/TCPHandler.cpp b/programs/server/TCPHandler.cpp index a792af30cf2..37195b21a86 100644 --- a/programs/server/TCPHandler.cpp +++ b/programs/server/TCPHandler.cpp @@ -28,7 +28,7 @@ #include #include -#include +#include #include "TCPHandler.h" @@ -560,7 +560,7 @@ void TCPHandler::processOrdinaryQueryWithProcessors() } { - PullingPipelineExecutor executor(pipeline); + PullingAsyncPipelineExecutor executor(pipeline); CurrentMetrics::Increment query_thread_metric_increment{CurrentMetrics::QueryThread}; Block block; From bcd38ac207b3a07f745bbfef4ef4883a99cbb9ac Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 20 May 2020 22:08:20 +0300 Subject: [PATCH 487/738] Fix build. --- src/Processors/Formats/PullingOutputFormat.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Processors/Formats/PullingOutputFormat.cpp b/src/Processors/Formats/PullingOutputFormat.cpp index e137fee3dd1..af237037a72 100644 --- a/src/Processors/Formats/PullingOutputFormat.cpp +++ b/src/Processors/Formats/PullingOutputFormat.cpp @@ -1,4 +1,5 @@ #include +#include namespace DB { @@ -8,6 +9,8 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } +WriteBuffer PullingOutputFormat::out(nullptr, 0); + void PullingOutputFormat::consume(Chunk chunk) { if (data) From 50df893dc63b6af266d809e2d82fd92a332c5c5d Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 15 May 2020 19:23:31 +0300 Subject: [PATCH 488/738] Assert that allocator & container are consistent wrt. inline memory There are lots of places where HashTable is used with AllocatorWithStackMemory, but the size of allocator inline memory is set incorrectly, and it's less than the initial HashTable buffer size. Because of this, HashTable is always allocated on the heap, and the inline memory becomes a useless dead weight. For PODArray, we have previously added a helper template that makes sure these values are in sync, so there was only one such discrepancy left, in the unit test. --- .../AggregateFunctionEntropy.h | 14 +++------ .../AggregateFunctionGroupUniqArray.h | 15 ++++------ .../AggregateFunctionTopK.h | 17 ++--------- .../QuantileExactWeighted.h | 8 ++--- src/Common/Allocator.h | 28 ++++++++++++----- src/Common/HashTable/HashTable.h | 30 +++++++++++++++++++ src/Common/HashTable/HashTableAllocator.h | 4 +-- src/Common/PODArray.h | 5 ++++ src/Common/SpaceSaving.h | 7 ++--- src/Common/tests/pod_array.cpp | 18 +++++------ src/Functions/array/arrayDistinct.cpp | 16 ++++------ src/Functions/array/arrayEnumerateExtended.h | 30 ++++++++++++------- src/Functions/array/arrayEnumerateRanked.h | 10 +++---- src/Functions/array/arrayIntersect.cpp | 26 ++++++++-------- src/Functions/array/arrayUniq.cpp | 25 +++++++++------- src/Interpreters/tests/hash_map3.cpp | 5 ++-- 16 files changed, 145 insertions(+), 113 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionEntropy.h b/src/AggregateFunctions/AggregateFunctionEntropy.h index 7586cebd8ec..59cacd0e9ca 100644 --- a/src/AggregateFunctions/AggregateFunctionEntropy.h +++ b/src/AggregateFunctions/AggregateFunctionEntropy.h @@ -23,18 +23,12 @@ struct EntropyData { using Weight = UInt64; - using HashingMap = HashMap< - Value, Weight, - HashCRC32, - HashTableGrower<4>, - HashTableAllocatorWithStackMemory) * (1 << 3)>>; + using HashingMap = HASH_TABLE_WITH_STACK_MEMORY(HashMap, Value, Weight, + HashCRC32, HashTableGrower<4>); /// For the case of pre-hashed values. - using TrivialMap = HashMap< - Value, Weight, - UInt128TrivialHash, - HashTableGrower<4>, - HashTableAllocatorWithStackMemory) * (1 << 3)>>; + using TrivialMap = HASH_TABLE_WITH_STACK_MEMORY(HashMap, Value, Weight, + UInt128TrivialHash, HashTableGrower<4>); using Map = std::conditional_t, TrivialMap, HashingMap>; diff --git a/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h b/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h index 9dbf2c921c2..af97df209f6 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h +++ b/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h @@ -28,12 +28,8 @@ template struct AggregateFunctionGroupUniqArrayData { /// When creating, the hash table must be small. - using Set = HashSet< - T, - DefaultHash, - HashTableGrower<4>, - HashTableAllocatorWithStackMemory - >; + using Set = HASH_TABLE_WITH_STACK_MEMORY(HashSet, T, DefaultHash, + HashTableGrower<4>); Set value; }; @@ -126,9 +122,10 @@ public: /// Generic implementation, it uses serialized representation as object descriptor. struct AggregateFunctionGroupUniqArrayGenericData { - static constexpr size_t INIT_ELEMS = 2; /// adjustable - static constexpr size_t ELEM_SIZE = sizeof(HashSetCellWithSavedHash); - using Set = HashSetWithSavedHash, HashTableAllocatorWithStackMemory>; + static constexpr size_t INITIAL_SIZE_DEGREE = 1; /// adjustable + + using Set = HASH_TABLE_WITH_STACK_MEMORY(HashSetWithSavedHash, + StringRef, StringRefHash, HashTableGrower); Set value; }; diff --git a/src/AggregateFunctions/AggregateFunctionTopK.h b/src/AggregateFunctions/AggregateFunctionTopK.h index 9c5e62bb6d7..23eb0e7ff09 100644 --- a/src/AggregateFunctions/AggregateFunctionTopK.h +++ b/src/AggregateFunctions/AggregateFunctionTopK.h @@ -23,13 +23,8 @@ namespace DB template struct AggregateFunctionTopKData { - using Set = SpaceSaving - < - T, - HashCRC32, - HashTableGrower<4>, - HashTableAllocatorWithStackMemory - >; + using Set = SpaceSaving>; + Set value; }; @@ -109,13 +104,7 @@ public: /// Generic implementation, it uses serialized representation as object descriptor. struct AggregateFunctionTopKGenericData { - using Set = SpaceSaving - < - StringRef, - StringRefHash, - HashTableGrower<4>, - HashTableAllocatorWithStackMemory - >; + using Set = SpaceSaving; Set value; }; diff --git a/src/AggregateFunctions/QuantileExactWeighted.h b/src/AggregateFunctions/QuantileExactWeighted.h index 6053bddc947..7731ff980fe 100644 --- a/src/AggregateFunctions/QuantileExactWeighted.h +++ b/src/AggregateFunctions/QuantileExactWeighted.h @@ -33,12 +33,8 @@ struct QuantileExactWeighted using Hasher = std::conditional_t, Int128Hash, HashCRC32>; /// When creating, the hash table must be small. - using Map = HashMap< - UnderlyingType, Weight, - Hasher, - HashTableGrower<4>, - HashTableAllocatorWithStackMemory) * (1 << 3)> - >; + using Map = HASH_TABLE_WITH_STACK_MEMORY(HashMap, UnderlyingType, Weight, + Hasher, HashTableGrower<4>); Map map; diff --git a/src/Common/Allocator.h b/src/Common/Allocator.h index 9add9299430..43d7e67c4bb 100644 --- a/src/Common/Allocator.h +++ b/src/Common/Allocator.h @@ -278,13 +278,15 @@ private: /** Allocator with optimization to place small memory ranges in automatic memory. */ -template +template class AllocatorWithStackMemory : private Base { private: - alignas(Alignment) char stack_memory[N]; + alignas(Alignment) char stack_memory[_initial_bytes]; public: + static constexpr size_t initial_bytes = _initial_bytes; + /// Do not use boost::noncopyable to avoid the warning about direct base /// being inaccessible due to ambiguity, when derived classes are also /// noncopiable (-Winaccessible-base). @@ -295,10 +297,10 @@ public: void * alloc(size_t size) { - if (size <= N) + if (size <= initial_bytes) { if constexpr (Base::clear_memory) - memset(stack_memory, 0, N); + memset(stack_memory, 0, initial_bytes); return stack_memory; } @@ -307,18 +309,18 @@ public: void free(void * buf, size_t size) { - if (size > N) + if (size > initial_bytes) Base::free(buf, size); } void * realloc(void * buf, size_t old_size, size_t new_size) { /// Was in stack_memory, will remain there. - if (new_size <= N) + if (new_size <= initial_bytes) return buf; /// Already was big enough to not fit in stack_memory. - if (old_size > N) + if (old_size > initial_bytes) return Base::realloc(buf, old_size, new_size, Alignment); /// Was in stack memory, but now will not fit there. @@ -330,10 +332,20 @@ public: protected: static constexpr size_t getStackThreshold() { - return N; + return initial_bytes; } }; +// A constant that gives the number of initially available bytes in +// the allocator. Used to check that this number is in sync with the +// initial size of array or hash table that uses the allocator. +template +constexpr size_t allocatorInitialBytes = 0; + +template +constexpr size_t allocatorInitialBytes> = initial_bytes; + #if !__clang__ #pragma GCC diagnostic pop diff --git a/src/Common/HashTable/HashTable.h b/src/Common/HashTable/HashTable.h index 58b7cd81901..2f7272ca8ec 100644 --- a/src/Common/HashTable/HashTable.h +++ b/src/Common/HashTable/HashTable.h @@ -308,7 +308,16 @@ struct ZeroValueStorage const Cell * zeroValue() const { return nullptr; } }; +// These templates give the initial hash table size, so that we can check +// that it is in sync with initial allocator size. +template +constexpr size_t growerInitialCount = 0; +template +constexpr size_t growerInitialCount> + = 1ULL << initial_size_degree; + +// The HashTable template < typename Key, @@ -324,6 +333,17 @@ class HashTable : protected Cell::State, protected ZeroValueStorage /// empty base optimization { +public: + // Export the initial buffer sizes for the ease of using allocators with + // inline memory. + static constexpr size_t initial_buffer_bytes + = growerInitialCount * sizeof(Cell); + + // If we use an allocator with inline memory, check that the initial + // size of the hash table is in sync with the amount of this memory. + static_assert(allocatorInitialBytes == 0 + || allocatorInitialBytes == initial_buffer_bytes); + protected: friend class const_iterator; friend class iterator; @@ -1075,3 +1095,13 @@ public: } #endif }; + +// A helper macro that declares hash table with allocator with stack memory, +// and the initial size of the allocator is in sync with initial size of the +// hash table. +#define HASH_TABLE_WITH_STACK_MEMORY(HASH_TABLE_VARIANT, ...) \ + HASH_TABLE_VARIANT<__VA_ARGS__, \ + HashTableAllocatorWithStackMemory< \ + HASH_TABLE_VARIANT<__VA_ARGS__>::initial_buffer_bytes \ + > \ + > diff --git a/src/Common/HashTable/HashTableAllocator.h b/src/Common/HashTable/HashTableAllocator.h index 99f9c979685..47e3fdfc4b6 100644 --- a/src/Common/HashTable/HashTableAllocator.h +++ b/src/Common/HashTable/HashTableAllocator.h @@ -10,5 +10,5 @@ */ using HashTableAllocator = Allocator; -template -using HashTableAllocatorWithStackMemory = AllocatorWithStackMemory; +template +using HashTableAllocatorWithStackMemory = AllocatorWithStackMemory; diff --git a/src/Common/PODArray.h b/src/Common/PODArray.h index 8fe1f74484e..8b30b314e91 100644 --- a/src/Common/PODArray.h +++ b/src/Common/PODArray.h @@ -85,6 +85,11 @@ protected: static_assert(pad_left <= EmptyPODArraySize && "Left Padding exceeds EmptyPODArraySize. Is the element size too large?"); + // If we are using allocator with inline memory, the minimal size of + // array must be in sync with the size of this memory. + static_assert(allocatorInitialBytes == 0 + || allocatorInitialBytes == initial_bytes); + char * c_start = null; /// Does not include pad_left. char * c_end = null; char * c_end_of_storage = null; /// Does not include pad_right. diff --git a/src/Common/SpaceSaving.h b/src/Common/SpaceSaving.h index 9ad7f6275d6..47b49add060 100644 --- a/src/Common/SpaceSaving.h +++ b/src/Common/SpaceSaving.h @@ -67,9 +67,7 @@ private: template < typename TKey, - typename Hash = DefaultHash, - typename Grower = HashTableGrower<>, - typename Allocator = HashTableAllocator + typename Hash = DefaultHash > class SpaceSaving { @@ -380,7 +378,8 @@ private: counter_map[counter->key] = counter; } - using CounterMap = HashMap; + using CounterMap = HASH_TABLE_WITH_STACK_MEMORY(HashMap, + TKey, Counter *, Hash, HashTableGrower<4>); CounterMap counter_map; std::vector counter_list; diff --git a/src/Common/tests/pod_array.cpp b/src/Common/tests/pod_array.cpp index de15b485411..6e9634ba3cf 100644 --- a/src/Common/tests/pod_array.cpp +++ b/src/Common/tests/pod_array.cpp @@ -18,9 +18,9 @@ static void test1() { using namespace DB; - static constexpr size_t initial_size = 8; - static constexpr size_t stack_threshold = 32; - using Array = PODArray, stack_threshold>>; + static constexpr size_t initial_bytes = 32; + using Array = PODArray, initial_bytes>>; bool res = true; @@ -139,9 +139,9 @@ static void test2() { using namespace DB; - static constexpr size_t initial_size = 8; - static constexpr size_t stack_threshold = 32; - using Array = PODArray, stack_threshold>>; + static constexpr size_t initial_bytes = 32; + using Array = PODArray, initial_bytes>>; bool res = true; @@ -389,9 +389,9 @@ static void test3() { using namespace DB; - static constexpr size_t initial_size = 8; - static constexpr size_t stack_threshold = 32; - using Array = PODArray, stack_threshold>>; + static constexpr size_t initial_bytes = 32; + using Array = PODArray, initial_bytes>>; bool res = true; diff --git a/src/Functions/array/arrayDistinct.cpp b/src/Functions/array/arrayDistinct.cpp index d24de638865..d3da77e43d2 100644 --- a/src/Functions/array/arrayDistinct.cpp +++ b/src/Functions/array/arrayDistinct.cpp @@ -153,10 +153,8 @@ bool FunctionArrayDistinct::executeNumber( if (nullable_col) src_null_map = &nullable_col->getNullMapData(); - using Set = ClearableHashSet, - HashTableGrower, - HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(T)>>; + using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashSet, + T, DefaultHash, HashTableGrower); Set set; @@ -201,10 +199,8 @@ bool FunctionArrayDistinct::executeString( ColumnString & res_data_column_string = typeid_cast(res_data_col); - using Set = ClearableHashSet, - HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(StringRef)>>; + using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashSet, + StringRef, StringRefHash, HashTableGrower); const PaddedPODArray * src_null_map = nullptr; @@ -249,8 +245,8 @@ void FunctionArrayDistinct::executeHashed( ColumnArray::Offsets & res_offsets, const ColumnNullable * nullable_col) { - using Set = ClearableHashSet, - HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(UInt128)>>; + using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashSet, + UInt128, UInt128TrivialHash, HashTableGrower); const PaddedPODArray * src_null_map = nullptr; diff --git a/src/Functions/array/arrayEnumerateExtended.h b/src/Functions/array/arrayEnumerateExtended.h index 69db59e698e..e97766c5827 100644 --- a/src/Functions/array/arrayEnumerateExtended.h +++ b/src/Functions/array/arrayEnumerateExtended.h @@ -64,36 +64,46 @@ private: template struct MethodOneNumber { - using Set = ClearableHashMap, HashTableGrower, - HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(T)>>; + using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap, + T, UInt32, DefaultHash, + HashTableGrower); + using Method = ColumnsHashing::HashMethodOneNumber; }; struct MethodString { - using Set = ClearableHashMap, - HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(StringRef)>>; + using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap, + StringRef, UInt32, StringRefHash, + HashTableGrower); + using Method = ColumnsHashing::HashMethodString; }; struct MethodFixedString { - using Set = ClearableHashMap, - HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(StringRef)>>; + using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap, + StringRef, UInt32, StringRefHash, + HashTableGrower); + using Method = ColumnsHashing::HashMethodFixedString; }; struct MethodFixed { - using Set = ClearableHashMap, - HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(UInt128)>>; + using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap, + UInt128, UInt32, UInt128HashCRC32, + HashTableGrower); + using Method = ColumnsHashing::HashMethodKeysFixed; }; struct MethodHashed { - using Set = ClearableHashMap, - HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(UInt128)>>; + using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap, + UInt128, UInt32, UInt128TrivialHash, + HashTableGrower); + using Method = ColumnsHashing::HashMethodHashed; }; diff --git a/src/Functions/array/arrayEnumerateRanked.h b/src/Functions/array/arrayEnumerateRanked.h index 133f5d7cb81..1700260dd9f 100644 --- a/src/Functions/array/arrayEnumerateRanked.h +++ b/src/Functions/array/arrayEnumerateRanked.h @@ -308,12 +308,10 @@ void FunctionArrayEnumerateRankedExtended::executeMethodImpl( const size_t depth_to_look = arrays_depths.max_array_depth; const auto & offsets = *offsets_by_depth[depth_to_look - 1]; - using Map = ClearableHashMap< - UInt128, - UInt32, - UInt128TrivialHash, - HashTableGrower, - HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(UInt128)>>; + using Map = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap, + UInt128, UInt32, UInt128TrivialHash, + HashTableGrower); + Map indices; std::vector indices_by_depth(depth_to_look); diff --git a/src/Functions/array/arrayIntersect.cpp b/src/Functions/array/arrayIntersect.cpp index cccad7c7a03..acd162974c9 100644 --- a/src/Functions/array/arrayIntersect.cpp +++ b/src/Functions/array/arrayIntersect.cpp @@ -418,16 +418,18 @@ void FunctionArrayIntersect::executeImpl(Block & block, const ColumnNumbers & ar TypeListNativeNumbers::forEach(NumberExecutor(arrays, not_nullable_nested_return_type, result_column)); TypeListDecimalNumbers::forEach(DecimalExecutor(arrays, not_nullable_nested_return_type, result_column)); - using DateMap = ClearableHashMap, - HashTableGrower, - HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(DataTypeDate::FieldType)>>; + using DateMap = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap, + DataTypeDate::FieldType, size_t, DefaultHash, + HashTableGrower); - using DateTimeMap = ClearableHashMap, - HashTableGrower, - HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(DataTypeDateTime::FieldType)>>; + using DateTimeMap = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap, + DataTypeDateTime::FieldType, size_t, + DefaultHash, + HashTableGrower); - using StringMap = ClearableHashMap, - HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(StringRef)>>; + using StringMap = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap, + StringRef, size_t, StringRefHash, + HashTableGrower); if (!result_column) { @@ -455,8 +457,8 @@ void FunctionArrayIntersect::executeImpl(Block & block, const ColumnNumbers & ar template void FunctionArrayIntersect::NumberExecutor::operator()() { - using Map = ClearableHashMap, HashTableGrower, - HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(T)>>; + using Map = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap, + T, size_t, DefaultHash, HashTableGrower); if (!result && typeid_cast *>(data_type.get())) result = execute, true>(arrays, ColumnVector::create()); @@ -465,8 +467,8 @@ void FunctionArrayIntersect::NumberExecutor::operator()() template void FunctionArrayIntersect::DecimalExecutor::operator()() { - using Map = ClearableHashMap, HashTableGrower, - HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(T)>>; + using Map = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap, + T, size_t, DefaultHash, HashTableGrower); if (!result) if (auto * decimal = typeid_cast *>(data_type.get())) diff --git a/src/Functions/array/arrayUniq.cpp b/src/Functions/array/arrayUniq.cpp index 02129781c13..a83c535d88d 100644 --- a/src/Functions/array/arrayUniq.cpp +++ b/src/Functions/array/arrayUniq.cpp @@ -66,36 +66,41 @@ private: template struct MethodOneNumber { - using Set = ClearableHashSet, HashTableGrower, - HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(T)>>; + using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashSet, + T, DefaultHash, HashTableGrower); + using Method = ColumnsHashing::HashMethodOneNumber; }; struct MethodString { - using Set = ClearableHashSet, - HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(StringRef)>>; + using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashSet, + StringRef, StringRefHash, HashTableGrower); + using Method = ColumnsHashing::HashMethodString; }; struct MethodFixedString { - using Set = ClearableHashSet, - HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(StringRef)>>; + using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashSet, + StringRef, StringRefHash, HashTableGrower); + using Method = ColumnsHashing::HashMethodFixedString; }; struct MethodFixed { - using Set = ClearableHashSet, - HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(UInt128)>>; + using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashSet, + UInt128, UInt128HashCRC32, HashTableGrower); + using Method = ColumnsHashing::HashMethodKeysFixed; }; struct MethodHashed { - using Set = ClearableHashSet, - HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(UInt128)>>; + using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashSet, + UInt128, UInt128TrivialHash, HashTableGrower); + using Method = ColumnsHashing::HashMethodHashed; }; diff --git a/src/Interpreters/tests/hash_map3.cpp b/src/Interpreters/tests/hash_map3.cpp index 2207edc6cc1..804b084b2a1 100644 --- a/src/Interpreters/tests/hash_map3.cpp +++ b/src/Interpreters/tests/hash_map3.cpp @@ -61,12 +61,11 @@ struct Grower : public HashTableGrower<2> int main(int, char **) { - using Map = HashMapWithDump< + using Map = HASH_TABLE_WITH_STACK_MEMORY(HashMapWithDump, StringRef, UInt64, SimpleHash, - Grower, - HashTableAllocatorWithStackMemory<4 * 24>>; + Grower); Map map; From 09ea7253cd0466fa29fe4f02414472aaa9cc72b5 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 20 May 2020 05:19:02 +0300 Subject: [PATCH 489/738] performance test for some functions that use hash tables --- tests/performance/functions_with_hash_tables.xml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 tests/performance/functions_with_hash_tables.xml diff --git a/tests/performance/functions_with_hash_tables.xml b/tests/performance/functions_with_hash_tables.xml new file mode 100644 index 00000000000..3ff2633c4e7 --- /dev/null +++ b/tests/performance/functions_with_hash_tables.xml @@ -0,0 +1,8 @@ + + select arrayUniq(range(1 + (number % 10) * 10)) from numbers(1000000) format Null + select arrayDistinct(range(1 + (number % 10) * 10)) from numbers(1000000) format Null + select arrayEnumerateUniq(range(1 + (number % 10) * 10)) from numbers(1000000) format Null + select arrayIntersect(range((1 + number % 10)), range(1, (1 + number % 10) + 1)) from numbers(1000000) format Null + select groupUniqArray(number / 10) from numbers(1000 * 1000) group by number / 1000 format Null + select entropy(number / 10) from numbers(1000 * 1000) group by number / 1000 format Null + From 7ad8228854edecce8e20bd8a8db00ea97580d786 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 20 May 2020 05:19:19 +0300 Subject: [PATCH 490/738] [wip] Add memory stats to performance test --- docker/test/performance-comparison/compare.sh | 311 +++++++++++------- .../test/performance-comparison/entrypoint.sh | 2 +- docker/test/performance-comparison/eqmed.sql | 41 ++- docker/test/performance-comparison/perf.py | 23 +- docker/test/performance-comparison/report.py | 45 +-- 5 files changed, 256 insertions(+), 166 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index af88a3954d7..c748a09497d 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -133,7 +133,7 @@ function run_tests fi # Delete old report files. - for x in {test-times,skipped-tests,wall-clock-times,report-thresholds,client-times}.tsv + for x in {test-times,wall-clock-times}.tsv do rm -v "$x" ||: touch "$x" @@ -220,66 +220,117 @@ function get_profiles # Build and analyze randomization distribution for all queries. function analyze_queries { -rm -v analyze-commands.txt analyze-errors.log all-queries.tsv unstable-queries.tsv ./*-report.tsv raw-queries.tsv client-times.tsv report-thresholds.tsv ||: +rm -v analyze-commands.txt analyze-errors.log all-queries.tsv unstable-queries.tsv ./*-report.tsv raw-queries.tsv ||: +rm -rfv analyze ||: +mkdir analyze ||: + +# FIXME This loop builds column definitons from TSVWithNamesAndTypes in an +# absolutely atrocious way. This should be done by the file() function itself. +for x in {right,left}-{addresses,{query,query-thread,trace,metric}-log}.tsv +do + paste -d' ' \ + <(sed -n '1{s/\t/\n/g;p;q}' "$x" | sed 's/\(^.*$\)/"\1"/') \ + <(sed -n '2{s/\t/\n/g;p;q}' "$x" ) \ + | tr '\n' ', ' | sed 's/,$//' > "$x.columns" +done # Split the raw test output into files suitable for analysis. IFS=$'\n' for test_file in $(find . -maxdepth 1 -name "*-raw.tsv" -print) do test_name=$(basename "$test_file" "-raw.tsv") - sed -n "s/^query\t//p" < "$test_file" > "$test_name-queries.tsv" - sed -n "s/^client-time/$test_name/p" < "$test_file" >> "client-times.tsv" - sed -n "s/^report-threshold/$test_name/p" < "$test_file" >> "report-thresholds.tsv" - sed -n "s/^skipped/$test_name/p" < "$test_file" >> "skipped-tests.tsv" + sed -n "s/^query\t/$test_name\t/p" < "$test_file" >> "analyze/query-runs.tsv" + sed -n "s/^client-time/$test_name/p" < "$test_file" >> "analyze/client-times.tsv" + sed -n "s/^report-threshold/$test_name/p" < "$test_file" >> "analyze/report-thresholds.tsv" + sed -n "s/^skipped/$test_name/p" < "$test_file" >> "analyze/skipped-tests.tsv" + sed -n "s/^display-name/$test_name/p" < "$test_file" >> "analyze/query-display-names.tsv" done unset IFS +# for each query run, prepare array of metrics from query log +clickhouse-local --query " +create view query_runs as select * from file('analyze/query-runs.tsv', TSV, + 'test text, query_index int, query_id text, version UInt8, time float'); + +create view left_query_log as select * + from file('left-query-log.tsv', TSVWithNamesAndTypes, + '$(cat "left-query-log.tsv.columns")'); + +create view right_query_log as select * + from file('right-query-log.tsv', TSVWithNamesAndTypes, + '$(cat "right-query-log.tsv.columns")'); + +create table query_metrics engine File(TSV, -- do not add header -- will parse with grep + 'analyze/query-run-metrics.tsv') + as select + test, query_index, 0 run, version, + [ + -- FIXME server-reported duration might be significantly less than + -- the client-reported one, when there is some slow cleanup that + -- happens after we reported the completion to the client (this did + -- happen with some queries). + -- To fix this, generate a unique query id for each run, that would + -- let us match the client and server times. + query_duration_ms / toFloat64(1000) + , toFloat64(memory_usage) + , query_runs.time + ] metrics + from ( + select *, 0 version from left_query_log + union all + select *, 1 version from right_query_log + ) query_logs + right join query_runs + using (query_id, version) + ; +" + # This is a lateral join in bash... please forgive me. -# We don't have arrayPermute(), so I have to make random permutations with +# We don't have arrayPermute(), so I have to make random permutations with # `order by rand`, and it becomes really slow if I do it for more than one # query. We also don't have lateral joins. So I just put all runs of each # query into a separate file, and then compute randomization distribution # for each file. I do this in parallel using GNU parallel. +query_index=1 IFS=$'\n' -for test_file in $(find . -maxdepth 1 -name "*-queries.tsv" -print) +for prefix in $(cut -f1,2 "analyze/query-run-metrics.tsv" | sort | uniq) do - test_name=$(basename "$test_file" "-queries.tsv") - query_index=1 - for query in $(cut -d' ' -f1 "$test_file" | sort | uniq) - do - query_prefix="$test_name.q$query_index" - query_index=$((query_index + 1)) - grep -F "$query " "$test_file" > "$query_prefix.tmp" - printf "%s\0\n" \ - "clickhouse-local \ - --file \"$query_prefix.tmp\" \ - --structure 'query text, run int, version UInt32, time float' \ - --query \"$(cat "$script_dir/eqmed.sql")\" \ - >> \"$test_name-report.tsv\"" \ - 2>> analyze-errors.log \ - >> analyze-commands.txt - done + file="analyze/q$query_index.tmp" + grep -F "$prefix " "analyze/query-run-metrics.tsv" > "$file" & + printf "%s\0\n" \ + "clickhouse-local \ + --file \"$file\" \ + --structure 'test text, query text, run int, version UInt8, metrics Array(float)' \ + --query \"$(cat "$script_dir/eqmed.sql")\" \ + >> \"analyze/query-reports.tsv\"" \ + 2>> analyze/errors.log \ + >> analyze/commands.txt + + query_index=$((query_index + 1)) done wait unset IFS -parallel --null < analyze-commands.txt +parallel --joblog analyze/parallel-log.txt --null < analyze/commands.txt } # Analyze results function report { - rm -r report ||: mkdir report ||: - rm ./*.{rep,svg} test-times.tsv test-dump.tsv unstable.tsv unstable-query-ids.tsv unstable-query-metrics.tsv changed-perf.tsv unstable-tests.tsv unstable-queries.tsv bad-tests.tsv slow-on-client.tsv all-queries.tsv ||: -cat analyze-errors.log >> report/errors.log ||: +cat analyze/errors.log >> report/errors.log ||: cat profile-errors.log >> report/errors.log ||: clickhouse-local --query " +create view query_display_names as select * from + file('analyze/query-display-names.tsv', TSV, + 'test text, query_index int, query_display_name text') + ; + create table queries engine File(TSVWithNamesAndTypes, 'report/queries.tsv') as select -- FIXME Comparison mode doesn't make sense for queries that complete @@ -296,21 +347,26 @@ create table queries engine File(TSVWithNamesAndTypes, 'report/queries.tsv') left, right, diff, stat_threshold, if(report_threshold > 0, report_threshold, 0.10) as report_threshold, - reports.test, - query + reports.test test, query_index, query_display_name from ( - select *, - replaceAll(_file, '-report.tsv', '') test - from file('*-report.tsv', TSV, 'left float, right float, diff float, stat_threshold float, query text') + select left[1] as left, right[1] as right, diff[1] diff, + stat_threshold[1] stat_threshold, test, query_index + from file ('analyze/query-reports.tsv', TSV, + 'left Array(float), right Array(float), diff Array(float), + stat_threshold Array(float), test text, query_index int') ) reports - left join file('report-thresholds.tsv', TSV, 'test text, report_threshold float') thresholds - using test + left join file('analyze/report-thresholds.tsv', TSV, 'test text, report_threshold float') thresholds + on reports.test = thresholds.test + left join query_display_names + on reports.test = query_display_names.test + and reports.query_index = query_display_names.query_index ; -- keep the table in old format so that we can analyze new and old data together create table queries_old_format engine File(TSVWithNamesAndTypes, 'queries.rep') - as select short, changed_fail, unstable_fail, left, right, diff, stat_threshold, test, query + as select short, changed_fail, unstable_fail, left, right, diff, + stat_threshold, test, query_display_name query from queries ; @@ -334,15 +390,16 @@ create table all_query_funs_json engine File(JSON, 'report/all-query-runs.json') ; create table changed_perf_tsv engine File(TSV, 'report/changed-perf.tsv') as - select left, right, diff, stat_threshold, changed_fail, test, query from queries where changed_show - order by abs(diff) desc; + select left, right, diff, stat_threshold, changed_fail, test, query_display_name + from queries where changed_show order by abs(diff) desc; create table unstable_queries_tsv engine File(TSV, 'report/unstable-queries.tsv') as - select left, right, diff, stat_threshold, unstable_fail, test, query from queries where unstable_show - order by stat_threshold desc; + select left, right, diff, stat_threshold, unstable_fail, test, query_display_name + from queries where unstable_show order by stat_threshold desc; -create table queries_for_flamegraph engine File(TSVWithNamesAndTypes, 'report/queries-for-flamegraph.tsv') as - select query, test from queries where unstable_show or changed_show +create table queries_for_flamegraph engine File(TSVWithNamesAndTypes, + 'report/queries-for-flamegraph.tsv') as + select test, query_index from queries where unstable_show or changed_show ; create table unstable_tests_tsv engine File(TSV, 'report/bad-tests.tsv') as @@ -350,23 +407,23 @@ create table unstable_tests_tsv engine File(TSV, 'report/bad-tests.tsv') as group by test having s > 0 order by s desc; create table query_time engine Memory as select * - from file('client-times.tsv', TSV, 'test text, query text, client float, server float'); + from file('analyze/client-times.tsv', TSV, + 'test text, query_index int, client float, server float'); create table wall_clock engine Memory as select * from file('wall-clock-times.tsv', TSV, 'test text, real float, user float, system float'); create table slow_on_client_tsv engine File(TSV, 'report/slow-on-client.tsv') as - select client, server, floor(client/server, 3) p, query - from query_time where p > 1.02 order by p desc; + select client, server, floor(client/server, 3) p, query_display_name + from query_time left join query_display_names using (test, query_index) + where p > 1.02 order by p desc; create table test_time engine Memory as select test, sum(client) total_client_time, maxIf(client, not short) query_max, minIf(client, not short) query_min, - count(*) queries, - sum(short) short_queries - from query_time full join queries - using test, query + count(*) queries, sum(short) short_queries + from query_time full join queries using (test, query_index) group by test; create table test_times_tsv engine File(TSV, 'report/test-times.tsv') as @@ -378,40 +435,80 @@ create table test_times_tsv engine File(TSV, 'report/test-times.tsv') as floor(real / queries, 3) avg_real_per_query, floor(query_min, 3) from test_time - -- wall clock times are also measured for skipped tests, so don't - -- do full join - left join wall_clock using test + -- wall clock times are also measured for skipped tests, so don't + -- do full join + left join wall_clock using test order by avg_real_per_query desc; create table all_tests_tsv engine File(TSV, 'report/all-queries.tsv') as select changed_fail, unstable_fail, left, right, diff, floor(left > right ? left / right : right / left, 3), - stat_threshold, test, query - from queries order by test, query; + stat_threshold, test, query_display_name + from queries order by test, query_display_name; " 2> >(tee -a report/errors.log 1>&2) -for x in {right,left}-{addresses,{query,query-thread,trace,metric}-log}.tsv -do - # FIXME This loop builds column definitons from TSVWithNamesAndTypes in an - # absolutely atrocious way. This should be done by the file() function itself. - paste -d' ' \ - <(sed -n '1{s/\t/\n/g;p;q}' "$x" | sed 's/\(^.*$\)/"\1"/') \ - <(sed -n '2{s/\t/\n/g;p;q}' "$x" ) \ - | tr '\n' ', ' | sed 's/,$//' > "$x.columns" -done - +# Prepare source data for metrics and flamegraphs for unstable queries. for version in {right,left} -do -clickhouse-local --query " + do + rm -rf data + clickhouse-local --query " create view queries_for_flamegraph as select * from file('report/queries-for-flamegraph.tsv', TSVWithNamesAndTypes, - 'query text, test text'); + 'test text, query_index int'); + +create view query_runs as + with 0 as left, 1 as right + select * from file('analyze/query-runs.tsv', TSV, + 'test text, query_index int, query_id text, version UInt8, time float') + where version = $version + ; + +create view query_display_names as select * from + file('analyze/query-display-names.tsv', TSV, + 'test text, query_index int, query_display_name text') + ; + +create table unstable_query_runs engine File(TSVWithNamesAndTypes, + 'unstable-query-runs.$version.rep') as + select test, query_index, query_display_name, query_id + from query_runs + join queries_for_flamegraph on + query_runs.test = queries_for_flamegraph.test + and query_runs.query_index = queries_for_flamegraph.query_index + left join query_display_names on + query_runs.test = query_display_names.test + and query_runs.query_index = query_display_names.query_index + ; create view query_log as select * from file('$version-query-log.tsv', TSVWithNamesAndTypes, '$(cat "$version-query-log.tsv.columns")'); +create table unstable_run_metrics engine File(TSVWithNamesAndTypes, + 'unstable-run-metrics.$version.rep') as + select + test, query_index, query_id, + ProfileEvents.Values value, ProfileEvents.Names metric + from query_log array join ProfileEvents + join unstable_query_runs using (query_id) + ; + +create table unstable_run_metrics_2 engine File(TSVWithNamesAndTypes, + 'unstable-run-metrics-2.$version.rep') as + select + test, query_index, query_id, + v, n + from ( + select + test, query_index, query_id, + ['memory_usage', 'read_bytes', 'written_bytes', 'query_duration_ms'] n, + [memory_usage, read_bytes, written_bytes, query_duration_ms] v + from query_log + join unstable_query_runs using (query_id) + ) + array join v, n; + create view trace_log as select * from file('$version-trace-log.tsv', TSVWithNamesAndTypes, '$(cat "$version-trace-log.tsv.columns")'); @@ -423,88 +520,64 @@ create view addresses_src as select * create table addresses_join_$version engine Join(any, left, address) as select addr address, name from addresses_src; -create table unstable_query_runs engine File(TSVWithNamesAndTypes, - 'unstable-query-runs.$version.rep') as - select query, query_id from query_log - where query in (select query from queries_for_flamegraph) - and query_id not like 'prewarm %' - ; - -create table unstable_query_log engine File(Vertical, - 'unstable-query-log.$version.rep') as - select * from query_log - where query_id in (select query_id from unstable_query_runs); - -create table unstable_run_metrics engine File(TSVWithNamesAndTypes, - 'unstable-run-metrics.$version.rep') as - select ProfileEvents.Values value, ProfileEvents.Names metric, query_id, query - from query_log array join ProfileEvents - where query_id in (select query_id from unstable_query_runs) - ; - -create table unstable_run_metrics_2 engine File(TSVWithNamesAndTypes, - 'unstable-run-metrics-2.$version.rep') as - select v, n, query_id, query - from - (select - ['memory_usage', 'read_bytes', 'written_bytes', 'query_duration_ms'] n, - [memory_usage, read_bytes, written_bytes, query_duration_ms] v, - query, - query_id - from query_log - where query_id in (select query_id from unstable_query_runs)) - array join n, v; - create table unstable_run_traces engine File(TSVWithNamesAndTypes, 'unstable-run-traces.$version.rep') as select + test, query_index, query_id, count() value, - joinGet(addresses_join_$version, 'name', arrayJoin(trace)) metric, - unstable_query_runs.query_id, - any(unstable_query_runs.query) query - from unstable_query_runs - join trace_log on trace_log.query_id = unstable_query_runs.query_id - group by unstable_query_runs.query_id, metric + joinGet(addresses_join_$version, 'name', arrayJoin(trace)) metric + from trace_log + join unstable_query_runs using query_id + group by test, query_index, query_id, metric order by count() desc ; create table metric_devation engine File(TSVWithNamesAndTypes, 'metric-deviation.$version.rep') as - select query, floor((q[3] - q[1])/q[2], 3) d, - quantilesExact(0, 0.5, 1)(value) q, metric - from (select * from unstable_run_metrics - union all select * from unstable_run_traces - union all select * from unstable_run_metrics_2) mm - join queries_for_flamegraph using query - group by query, metric - having d > 0.5 - order by query desc, d desc + -- first goes the key used to split the file with grep + select test, query_index, query_display_name, + d, q, metric + from ( + select + test, query_index, + floor((q[3] - q[1])/q[2], 3) d, + quantilesExact(0, 0.5, 1)(value) q, metric + from (select * from unstable_run_metrics + union all select * from unstable_run_traces + union all select * from unstable_run_metrics_2) mm + group by test, query_index, metric + having d > 0.5 + ) metrics + left join unstable_query_runs using (test, query_index) + order by test, query_index, d desc ; create table stacks engine File(TSV, 'stacks.$version.rep') as select - query, + -- first goes the key used to split the file with grep + test, query_index, any(query_display_name), arrayStringConcat( arrayMap(x -> joinGet(addresses_join_$version, 'name', x), arrayReverse(trace) ), ';' ) readable_trace, - count() + count() c from trace_log join unstable_query_runs using query_id - group by query, trace + group by test, query_index, trace ; " 2> >(tee -a report/errors.log 1>&2) # do not run in parallel because they use the same data dir for StorageJoins which leads to weird errors. done wait +# Create per-query flamegraphs and files with metrics IFS=$'\n' for version in {right,left} do - for query in $(cut -d' ' -f1 "stacks.$version.rep" | sort | uniq) + for query in $(cut -d' ' -f1,2,3 "stacks.$version.rep" | sort | uniq) do - query_file=$(echo "$query" | cut -c-120 | sed 's/[/]/_/g') + query_file=$(echo "$query" | cut -c-120 | sed 's/[/ ]/_/g') # Build separate .svg flamegraph for each query. grep -F "$query " "stacks.$version.rep" \ diff --git a/docker/test/performance-comparison/entrypoint.sh b/docker/test/performance-comparison/entrypoint.sh index 500dd872c8d..3b8a24ad5da 100755 --- a/docker/test/performance-comparison/entrypoint.sh +++ b/docker/test/performance-comparison/entrypoint.sh @@ -119,5 +119,5 @@ done dmesg -T > dmesg.log -7z a /output/output.7z ./*.{log,tsv,html,txt,rep,svg} {right,left}/{performance,db/preprocessed_configs,scripts} ./report +7z a /output/output.7z ./*.{log,tsv,html,txt,rep,svg} {right,left}/{performance,db/preprocessed_configs,scripts} report analyze cp compare.log /output diff --git a/docker/test/performance-comparison/eqmed.sql b/docker/test/performance-comparison/eqmed.sql index 4f8482ea74b..f7f8d6ac40d 100644 --- a/docker/test/performance-comparison/eqmed.sql +++ b/docker/test/performance-comparison/eqmed.sql @@ -1,32 +1,37 @@ --- input is table(query text, run UInt32, version int, time float) +-- input is table(test text, query text, run UInt32, version int, metrics Array(float)) select - floor(original_medians_array.time_by_version[1], 4) l, - floor(original_medians_array.time_by_version[2], 4) r, - floor((r - l) / l, 3) diff_percent, - floor(threshold / l, 3) threshold_percent, - query + arrayMap(x -> floor(x, 4), original_medians_array.medians_by_version[1] as l) l_rounded, + arrayMap(x -> floor(x, 4), original_medians_array.medians_by_version[2] as r) r_rounded, + arrayMap(x, y -> floor((y - x) / x, 3), l, r) diff_percent, + arrayMap(x, y -> floor(x / y, 3), threshold, l) threshold_percent, + test, query from ( -- quantiles of randomization distributions - select quantileExact(0.999)(abs(time_by_label[1] - time_by_label[2]) as d) threshold + select quantileExactForEach(0.999)( + arrayMap(x, y -> abs(x - y), metrics_by_label[1], metrics_by_label[2]) as d + ) threshold ---- uncomment to see what the distribution is really like - --, uniqExact(d) u + --, uniqExact(d.1) u --, arraySort(x->x.1, -- arrayZip( - -- (sumMap([d], [1]) as f).1, + -- (sumMap([d.1], [1]) as f).1, -- f.2)) full_histogram from ( - select virtual_run, groupArrayInsertAt(median_time, random_label) time_by_label -- make array 'random label' -> 'median time' + -- make array 'random label' -> '[median metric]' + select virtual_run, groupArrayInsertAt(median_metrics, random_label) metrics_by_label from ( - select medianExact(time) median_time, virtual_run, random_label -- get median times, grouping by random label + -- get [median metric] arrays among virtual runs, grouping by random label + select medianExactForEach(metrics) median_metrics, virtual_run, random_label from ( - select *, toUInt32(rowNumberInAllBlocks() % 2) random_label -- randomly relabel measurements + -- randomly relabel measurements + select *, toUInt32(rowNumberInAllBlocks() % 2) random_label from ( - select time, number virtual_run + select metrics, number virtual_run from -- strip the query away before the join -- it might be several kB long; - (select time, run, version from table) no_query, + (select metrics, run, version from table) no_query, -- duplicate input measurements into many virtual runs numbers(1, 100000) nn -- for each virtual run, randomly reorder measurements @@ -40,19 +45,19 @@ from -- this select aggregates by virtual_run ) rd, ( - select groupArrayInsertAt(median_time, version) time_by_version + select groupArrayInsertAt(median_metrics, version) medians_by_version from ( - select medianExact(time) median_time, version + select medianExactForEach(metrics) median_metrics, version from table group by version ) original_medians ) original_medians_array, ( - select any(query) query from table + select any(test) test, any(query) query from table ) any_query, ( - select throwIf(uniq(query) != 1) from table + select throwIf(uniq((test, query)) != 1) from table ) check_single_query -- this subselect checks that there is only one query in the input table; -- written this way so that it is not optimized away (#10523) ; diff --git a/docker/test/performance-comparison/perf.py b/docker/test/performance-comparison/perf.py index a749b158e83..be8155cdddb 100755 --- a/docker/test/performance-comparison/perf.py +++ b/docker/test/performance-comparison/perf.py @@ -29,6 +29,8 @@ parser.add_argument('--runs', type=int, default=int(os.environ.get('CHPC_RUNS', parser.add_argument('--no-long', type=bool, default=True, help='Skip the tests tagged as long.') args = parser.parse_args() +test_name = os.path.splitext(os.path.basename(args.file[0].name))[0] + tree = et.parse(args.file[0]) root = tree.getroot() @@ -141,19 +143,25 @@ test_queries = substitute_parameters(test_query_templates) report_stage_end('substitute2') -for i, q in enumerate(test_queries): +for query_index, q in enumerate(test_queries): + query_prefix = f'{test_name}.query{query_index}' + # We have some crazy long queries (about 100kB), so trim them to a sane - # length. + # length. This means we can't use query text as an identifier and have to + # use the test name + the test-wide query index. query_display_name = q if len(query_display_name) > 1000: query_display_name = f'{query_display_name[:1000]}...({i})' + print(f'display-name\t{query_index}\t{tsv_escape(query_display_name)}') + # Prewarm: run once on both servers. Helps to bring the data into memory, # precompile the queries, etc. try: for conn_index, c in enumerate(connections): - res = c.execute(q, query_id = f'prewarm {0} {query_display_name}') - print(f'prewarm\t{tsv_escape(query_display_name)}\t{conn_index}\t{c.last_query.elapsed}') + prewarm_id = f'{query_prefix}.prewarm0' + res = c.execute(q, query_id = prewarm_id) + print(f'prewarm\t{query_index}\t{prewarm_id}\t{conn_index}\t{c.last_query.elapsed}') except: # If prewarm fails for some query -- skip it, and try to test the others. # This might happen if the new test introduces some function that the @@ -170,13 +178,14 @@ for i, q in enumerate(test_queries): start_seconds = time.perf_counter() server_seconds = 0 for run in range(0, args.runs): + run_id = f'{query_prefix}.run{run}' for conn_index, c in enumerate(connections): - res = c.execute(q) - print(f'query\t{tsv_escape(query_display_name)}\t{run}\t{conn_index}\t{c.last_query.elapsed}') + res = c.execute(q, query_id = run_id) + print(f'query\t{query_index}\t{run_id}\t{conn_index}\t{c.last_query.elapsed}') server_seconds += c.last_query.elapsed client_seconds = time.perf_counter() - start_seconds - print(f'client-time\t{tsv_escape(query_display_name)}\t{client_seconds}\t{server_seconds}') + print(f'client-time\t{query_index}\t{client_seconds}\t{server_seconds}') report_stage_end('benchmark') diff --git a/docker/test/performance-comparison/report.py b/docker/test/performance-comparison/report.py index a6203ec40e6..c6b743a606e 100755 --- a/docker/test/performance-comparison/report.py +++ b/docker/test/performance-comparison/report.py @@ -25,6 +25,9 @@ very_unstable_queries = 0 # max seconds to run one query by itself, not counting preparation allowed_single_run_time = 2 +color_bad='#ffb0c0' +color_good='#b0d050' + header_template = """ @@ -188,8 +191,8 @@ if args.report == 'main': print(tableStart('Changes in performance')) columns = [ - 'Old, s.', # 0 - 'New, s.', # 1 + 'Old, s', # 0 + 'New, s', # 1 'Relative difference (new − old) / old', # 2 'p < 0.001 threshold', # 3 # Failed # 4 @@ -221,7 +224,7 @@ if args.report == 'main': slow_on_client_rows = tsvRows('report/slow-on-client.tsv') error_tests += len(slow_on_client_rows) printSimpleTable('Slow on client', - ['Client time, s.', 'Server time, s.', 'Ratio', 'Query'], + ['Client time, s', 'Server time, s', 'Ratio', 'Query'], slow_on_client_rows) def print_unstable_queries(): @@ -252,7 +255,7 @@ if args.report == 'main': for r in unstable_rows: if int(r[4]): very_unstable_queries += 1 - attrs[3] = 'style="background: #ffb0a0"' + attrs[3] = f'style="background: {color_bad}"' else: attrs[3] = '' @@ -266,7 +269,7 @@ if args.report == 'main': error_tests += len(run_error_rows) printSimpleTable('Run errors', ['Test', 'Error'], run_error_rows) - skipped_tests_rows = tsvRows('skipped-tests.tsv') + skipped_tests_rows = tsvRows('analyze/skipped-tests.tsv') printSimpleTable('Skipped tests', ['Test', 'Reason'], skipped_tests_rows) printSimpleTable('Tests with most unstable queries', @@ -281,13 +284,13 @@ if args.report == 'main': columns = [ 'Test', #0 - 'Wall clock time, s.', #1 - 'Total client time, s.', #2 + 'Wall clock time, s', #1 + 'Total client time, s', #2 'Total queries', #3 'Ignored short queries', #4 - 'Longest query
    (sum for all runs), s.', #5 - 'Avg wall clock time
    (sum for all runs), s.', #6 - 'Shortest query
    (sum for all runs), s.', #7 + 'Longest query
    (sum for all runs), s', #5 + 'Avg wall clock time
    (sum for all runs), s', #6 + 'Shortest query
    (sum for all runs), s', #7 ] print(tableStart('Test times')) @@ -300,13 +303,13 @@ if args.report == 'main': if float(r[6]) > 1.5 * total_runs: # FIXME should be 15s max -- investigate parallel_insert slow_average_tests += 1 - attrs[6] = 'style="background: #ffb0a0"' + attrs[6] = f'style="background: {color_bad}"' else: attrs[6] = '' if float(r[5]) > allowed_single_run_time * total_runs: slow_average_tests += 1 - attrs[5] = 'style="background: #ffb0a0"' + attrs[5] = f'style="background: {color_bad}"' else: attrs[5] = '' @@ -320,9 +323,9 @@ if args.report == 'main': print("""

    - Test output All queries Log + Test output

    @@ -382,8 +385,8 @@ elif args.report == 'all-queries': columns = [ # Changed #0 # Unstable #1 - 'Old, s.', #2 - 'New, s.', #3 + 'Old, s', #2 + 'New, s', #3 'Relative difference (new − old) / old', #4 'Times speedup / slowdown', #5 'p < 0.001 threshold', #6 @@ -399,21 +402,21 @@ elif args.report == 'all-queries': attrs[1] = None for r in rows: if int(r[1]): - attrs[6] = 'style="background: #ffb0a0"' + attrs[6] = f'style="background: {color_bad}"' else: attrs[6] = '' if int(r[0]): if float(r[4]) > 0.: - attrs[4] = 'style="background: #ffb0a0"' + attrs[4] = f'style="background: {color_bad}"' else: - attrs[4] = 'style="background: #adbdff"' + attrs[4] = f'style="background: {color_good}"' else: attrs[4] = '' if (float(r[2]) + float(r[3])) / 2 > allowed_single_run_time: - attrs[2] = 'style="background: #ffb0a0"' - attrs[3] = 'style="background: #ffb0a0"' + attrs[2] = f'style="background: {color_bad}"' + attrs[3] = f'style="background: {color_bad}"' else: attrs[2] = '' attrs[3] = '' @@ -428,9 +431,9 @@ elif args.report == 'all-queries': print("""

    - Test output Main report Log + Test output

    From ebef7b9a853f4e6d56a37ad022c896e5d9ed6d72 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Thu, 21 May 2020 00:01:18 +0300 Subject: [PATCH 491/738] [website] ld+json for the front page (#11077) * [website] ld+json for the front page * Update index.html * Update index.html --- website/index.html | 34 ++++++++++++++++++++++++++++++++++ website/templates/base.html | 1 + 2 files changed, 35 insertions(+) diff --git a/website/index.html b/website/index.html index e2ac6e31441..a1f85f0bbfb 100644 --- a/website/index.html +++ b/website/index.html @@ -4,6 +4,40 @@ {% extends "templates/base.html" %} +{% block extra_meta %} + +{% endblock %} + {% block content %} {% include "templates/index/nav.html" %} diff --git a/website/templates/base.html b/website/templates/base.html index 3a2977ac2b7..700f71d61ec 100644 --- a/website/templates/base.html +++ b/website/templates/base.html @@ -2,6 +2,7 @@ {% include "templates/common_meta.html" %} + {% block extra_meta %}{% endblock %} {% include "templates/common_css.html" %} From 7d0ac4e20fe2333a31936f696defa9d2c7f75f31 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Thu, 21 May 2020 00:08:04 +0300 Subject: [PATCH 492/738] trigger ci --- docs/en/sql-reference/syntax.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/en/sql-reference/syntax.md b/docs/en/sql-reference/syntax.md index 9ff5a388ad3..70db90b38be 100644 --- a/docs/en/sql-reference/syntax.md +++ b/docs/en/sql-reference/syntax.md @@ -28,9 +28,10 @@ There may be any number of space symbols between syntactical constructions (incl ## Comments {#comments} -ClickHouse supports either SQL-style and C-style comments. -SQL-style comments start with `--` and continue to the end of the line, a space after `--` can be omitted. -C-style are from `/*` to `*/`and can be multiline, spaces are not required either. +ClickHouse supports either SQL-style and C-style comments: + +- SQL-style comments start with `--` and continue to the end of the line, a space after `--` can be omitted. +- C-style are from `/*` to `*/`and can be multiline, spaces are not required either. ## Keywords {#syntax-keywords} From 58ffa4c59ce5e0dfc9793a708d5067feb126f0ad Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 21 May 2020 01:01:28 +0300 Subject: [PATCH 493/738] Enable percpu_arena:percpu for jemalloc This will reduce memory fragmentation due to thread pool. --- .../jemalloc/internal/jemalloc_internal_defs.h | 2 +- .../jemalloc/internal/jemalloc_internal_defs.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h index 9c46a3a9320..e989020d651 100644 --- a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h +++ b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h @@ -369,7 +369,7 @@ /* #undef JEMALLOC_EXPORT */ /* config.malloc_conf options string. */ -#define JEMALLOC_CONFIG_MALLOC_CONF "" +#define JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu" /* If defined, jemalloc takes the malloc/free/etc. symbol names. */ #define JEMALLOC_IS_MALLOC 1 diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h index 43936e8eba0..596dffa8d26 100644 --- a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h +++ b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h @@ -360,7 +360,7 @@ /* #undef JEMALLOC_EXPORT */ /* config.malloc_conf options string. */ -#define JEMALLOC_CONFIG_MALLOC_CONF "" +#define JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu" /* If defined, jemalloc takes the malloc/free/etc. symbol names. */ #define JEMALLOC_IS_MALLOC 1 From d93b9a57f65a8790d4c3a6e2affc0ffefd84feb8 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 20 May 2020 23:16:32 +0300 Subject: [PATCH 494/738] Forward declaration for Context as much as possible. Now after changing Context.h 488 modules will be recompiled instead of 582. --- programs/client/Suggest.h | 1 + programs/local/LocalServer.cpp | 1 - programs/local/LocalServer.h | 3 +- src/Client/Connection.h | 2 ++ .../AddingDefaultBlockOutputStream.h | 2 +- .../AddingDefaultsBlockInputStream.h | 3 +- .../CreatingSetsBlockInputStream.cpp | 1 + .../ParallelParsingBlockInputStream.h | 1 - .../PushingToViewsBlockOutputStream.cpp | 1 + src/DataStreams/RemoteBlockOutputStream.h | 1 + src/Databases/DatabaseAtomic.cpp | 1 + src/Databases/DatabaseLazy.h | 2 +- src/Databases/DatabaseMySQL.h | 4 ++- src/Databases/DatabaseOnDisk.h | 3 +- src/Databases/DatabaseWithDictionaries.h | 3 ++ src/Disks/DiskSelector.cpp | 1 + src/Disks/DiskSelector.h | 2 +- src/Functions/pointInPolygon.cpp | 1 + src/IO/ReadWriteBufferFromHTTP.h | 1 + src/Interpreters/ActionsVisitor.cpp | 1 + src/Interpreters/AsynchronousMetrics.cpp | 1 + .../ClusterProxy/SelectStreamFactory.h | 3 ++ src/Interpreters/DDLWorker.cpp | 1 + src/Interpreters/DDLWorker.h | 8 ++++- src/Interpreters/ExpressionActions.cpp | 28 ++++++++++++++++++ src/Interpreters/ExpressionActions.h | 29 ++++--------------- src/Interpreters/ExpressionAnalyzer.cpp | 1 + src/Interpreters/GlobalSubqueriesVisitor.h | 2 +- src/Interpreters/InterpreterExplainQuery.cpp | 1 + src/Interpreters/InterpreterExplainQuery.h | 3 +- src/Interpreters/InterpreterFactory.cpp | 1 + src/Interpreters/InterpreterInsertQuery.h | 3 +- src/Interpreters/InterpreterSelectQuery.h | 2 +- .../InterpreterSelectWithUnionQuery.cpp | 1 + .../InterpreterSelectWithUnionQuery.h | 3 +- src/Interpreters/InterpreterSetRoleQuery.h | 2 ++ src/Interpreters/InterpreterWatchQuery.cpp | 1 + src/Interpreters/InterpreterWatchQuery.h | 2 +- src/Interpreters/JoinedTables.cpp | 1 - src/Interpreters/JoinedTables.h | 3 +- src/Interpreters/MutationsInterpreter.h | 1 + src/Interpreters/PartLog.cpp | 1 + src/Interpreters/QueryLog.h | 1 + src/Interpreters/QueryThreadLog.h | 1 + src/Interpreters/Set.cpp | 1 + src/Interpreters/Set.h | 2 +- src/Interpreters/SystemLog.h | 7 ++++- src/Interpreters/executeQuery.cpp | 1 + src/Interpreters/getTableExpressions.cpp | 1 + src/Interpreters/interpretSubquery.cpp | 1 + .../Impl/ConstantExpressionTemplate.cpp | 3 +- src/Storages/Distributed/DirectoryMonitor.cpp | 16 +++++----- .../DistributedBlockOutputStream.cpp | 1 + .../DistributedBlockOutputStream.h | 2 +- src/Storages/MergeTree/IMergeTreeDataPart.h | 1 + src/Storages/MergeTree/KeyCondition.h | 4 +-- src/Storages/MergeTree/MergeList.h | 1 - src/Storages/MergeTree/MergeTreeData.cpp | 1 + src/Storages/MergeTree/MergeTreeData.h | 3 +- .../MergeTree/MergeTreeDataMergerMutator.cpp | 1 + .../MergeTree/MergeTreeDataPartWriterWide.cpp | 1 + .../MergeTree/MergeTreeDataSelectExecutor.cpp | 1 + .../MergeTree/MergeTreeDataWriter.cpp | 1 + src/Storages/MergeTree/MergeTreeDataWriter.h | 1 - .../MergeTreeReverseSelectProcessor.cpp | 1 + .../MergeTree/MergeTreeSelectProcessor.cpp | 1 + .../MergeTree/MergeTreeSequentialSource.cpp | 1 + ...rgeTreeThreadSelectBlockInputProcessor.cpp | 1 + .../MergeTree/MergedBlockOutputStream.cpp | 1 + .../MergedColumnOnlyOutputStream.cpp | 1 + src/Storages/MergeTree/RPNBuilder.h | 2 +- .../ReplicatedMergeTreeCleanupThread.cpp | 1 + .../ReplicatedMergeTreePartCheckThread.cpp | 1 + .../ReplicatedMergeTreeRestartingThread.cpp | 1 + .../MergeTree/registerStorageMergeTree.cpp | 2 ++ src/Storages/StorageBuffer.h | 3 +- src/Storages/StorageDistributed.cpp | 19 ++++++------ src/Storages/StorageDistributed.h | 2 +- src/Storages/StorageMergeTree.cpp | 5 ++-- src/Storages/StorageReplicatedMergeTree.cpp | 1 + src/Storages/StorageSet.cpp | 1 + src/Storages/StorageView.cpp | 1 + src/Storages/System/StorageSystemColumns.cpp | 1 + src/Storages/System/StorageSystemDisks.cpp | 1 + .../System/StorageSystemMutations.cpp | 1 + .../System/StorageSystemPartsBase.cpp | 1 + .../System/StorageSystemStoragePolicies.cpp | 1 + src/TableFunctions/ITableFunctionFileLike.h | 2 +- src/TableFunctions/TableFunctionFile.cpp | 1 + src/TableFunctions/TableFunctionFile.h | 2 +- src/TableFunctions/TableFunctionHDFS.h | 5 +++- src/TableFunctions/TableFunctionInput.cpp | 1 - src/TableFunctions/TableFunctionInput.h | 5 +++- src/TableFunctions/TableFunctionURL.h | 5 +++- 94 files changed, 175 insertions(+), 81 deletions(-) diff --git a/programs/client/Suggest.h b/programs/client/Suggest.h index 6c81a388ea7..b13289ac322 100644 --- a/programs/client/Suggest.h +++ b/programs/client/Suggest.h @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 06790038a5e..eae30fddfee 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include diff --git a/programs/local/LocalServer.h b/programs/local/LocalServer.h index 77d0b9ff5dc..5733bbc1a7c 100644 --- a/programs/local/LocalServer.h +++ b/programs/local/LocalServer.h @@ -4,13 +4,12 @@ #include #include #include +#include namespace DB { -class Context; - /// Lightweight Application for clickhouse-local /// No networking, no extra configs and working directories, no pid and status files, no dictionaries, no logging. /// Quiet mode by default diff --git a/src/Client/Connection.h b/src/Client/Connection.h index de04e3f0ef4..e056a4323df 100644 --- a/src/Client/Connection.h +++ b/src/Client/Connection.h @@ -50,6 +50,8 @@ class Connection; using ConnectionPtr = std::shared_ptr; using Connections = std::vector; +using Scalars = std::map; + /// Packet that could be received from server. struct Packet diff --git a/src/DataStreams/AddingDefaultBlockOutputStream.h b/src/DataStreams/AddingDefaultBlockOutputStream.h index e04fdc0faeb..89134920730 100644 --- a/src/DataStreams/AddingDefaultBlockOutputStream.h +++ b/src/DataStreams/AddingDefaultBlockOutputStream.h @@ -3,12 +3,12 @@ #include #include #include -#include namespace DB { +class Context; /** This stream adds three types of columns into block * 1. Columns, that are missed inside request, but present in table without defaults (missed columns) diff --git a/src/DataStreams/AddingDefaultsBlockInputStream.h b/src/DataStreams/AddingDefaultsBlockInputStream.h index cf16ec3fb75..10fa20f7ad1 100644 --- a/src/DataStreams/AddingDefaultsBlockInputStream.h +++ b/src/DataStreams/AddingDefaultsBlockInputStream.h @@ -2,12 +2,13 @@ #include #include -#include namespace DB { +class Context; + /// Adds defaults to columns using BlockDelayedDefaults bitmask attached to Block by child InputStream. class AddingDefaultsBlockInputStream : public IBlockInputStream { diff --git a/src/DataStreams/CreatingSetsBlockInputStream.cpp b/src/DataStreams/CreatingSetsBlockInputStream.cpp index 1a67031df5d..be89e0e87b0 100644 --- a/src/DataStreams/CreatingSetsBlockInputStream.cpp +++ b/src/DataStreams/CreatingSetsBlockInputStream.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include diff --git a/src/DataStreams/ParallelParsingBlockInputStream.h b/src/DataStreams/ParallelParsingBlockInputStream.h index 03f0d508227..a904c686e47 100644 --- a/src/DataStreams/ParallelParsingBlockInputStream.h +++ b/src/DataStreams/ParallelParsingBlockInputStream.h @@ -8,7 +8,6 @@ #include #include #include -#include namespace DB { diff --git a/src/DataStreams/PushingToViewsBlockOutputStream.cpp b/src/DataStreams/PushingToViewsBlockOutputStream.cpp index ce0922bf282..159199b2828 100644 --- a/src/DataStreams/PushingToViewsBlockOutputStream.cpp +++ b/src/DataStreams/PushingToViewsBlockOutputStream.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include diff --git a/src/DataStreams/RemoteBlockOutputStream.h b/src/DataStreams/RemoteBlockOutputStream.h index 40387180997..2c89a7358ad 100644 --- a/src/DataStreams/RemoteBlockOutputStream.h +++ b/src/DataStreams/RemoteBlockOutputStream.h @@ -4,6 +4,7 @@ #include #include #include +#include namespace DB diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp index 2894b65274f..27a2441cec6 100644 --- a/src/Databases/DatabaseAtomic.cpp +++ b/src/Databases/DatabaseAtomic.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include diff --git a/src/Databases/DatabaseLazy.h b/src/Databases/DatabaseLazy.h index 77ad1a3ab20..4306e61b37b 100644 --- a/src/Databases/DatabaseLazy.h +++ b/src/Databases/DatabaseLazy.h @@ -1,7 +1,6 @@ #pragma once #include -#include #include @@ -10,6 +9,7 @@ namespace DB class DatabaseLazyIterator; +class Context; /** Lazy engine of databases. * Works like DatabaseOrdinary, but stores in memory only cache. diff --git a/src/Databases/DatabaseMySQL.h b/src/Databases/DatabaseMySQL.h index d729c62a8bb..a43da5d1762 100644 --- a/src/Databases/DatabaseMySQL.h +++ b/src/Databases/DatabaseMySQL.h @@ -5,14 +5,16 @@ #include #include -#include #include #include +#include namespace DB { +class Context; + /** Real-time access to table list and table structure from remote MySQL * It doesn't make any manipulations with filesystem. * All tables are created by calling code after real-time pull-out structure from remote MySQL diff --git a/src/Databases/DatabaseOnDisk.h b/src/Databases/DatabaseOnDisk.h index a07a143ea92..75609e231af 100644 --- a/src/Databases/DatabaseOnDisk.h +++ b/src/Databases/DatabaseOnDisk.h @@ -3,7 +3,6 @@ #include #include #include -#include #include #include @@ -11,6 +10,8 @@ namespace DB { +class Context; + std::pair createTableFromAST( ASTCreateQuery ast_create_query, const String & database_name, diff --git a/src/Databases/DatabaseWithDictionaries.h b/src/Databases/DatabaseWithDictionaries.h index a02b68a56da..eb9e105e31d 100644 --- a/src/Databases/DatabaseWithDictionaries.h +++ b/src/Databases/DatabaseWithDictionaries.h @@ -5,6 +5,9 @@ namespace DB { +class Context; +class ExternalDictionariesLoader; + class DatabaseWithDictionaries : public DatabaseOnDisk { diff --git a/src/Disks/DiskSelector.cpp b/src/Disks/DiskSelector.cpp index 0ae8763eef3..69549e4520d 100644 --- a/src/Disks/DiskSelector.cpp +++ b/src/Disks/DiskSelector.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include diff --git a/src/Disks/DiskSelector.h b/src/Disks/DiskSelector.h index 85b67f55d0a..8ae8de3be61 100644 --- a/src/Disks/DiskSelector.h +++ b/src/Disks/DiskSelector.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include @@ -10,6 +9,7 @@ namespace DB { +class Context; class DiskSelector; using DiskSelectorPtr = std::shared_ptr; diff --git a/src/Functions/pointInPolygon.cpp b/src/Functions/pointInPolygon.cpp index 460c60d6e4c..2dbfd6f6641 100644 --- a/src/Functions/pointInPolygon.cpp +++ b/src/Functions/pointInPolygon.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include diff --git a/src/IO/ReadWriteBufferFromHTTP.h b/src/IO/ReadWriteBufferFromHTTP.h index 2d4e3f0472e..4b4e4453360 100644 --- a/src/IO/ReadWriteBufferFromHTTP.h +++ b/src/IO/ReadWriteBufferFromHTTP.h @@ -14,6 +14,7 @@ #include #include #include +#include #include #include diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 3fa53e3e694..7c2133e629f 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -32,6 +32,7 @@ #include #include +#include #include #include #include diff --git a/src/Interpreters/AsynchronousMetrics.cpp b/src/Interpreters/AsynchronousMetrics.cpp index 7ba3886afb3..59ff01bf972 100644 --- a/src/Interpreters/AsynchronousMetrics.cpp +++ b/src/Interpreters/AsynchronousMetrics.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.h b/src/Interpreters/ClusterProxy/SelectStreamFactory.h index daf4dd48b4d..da1d18b6dd9 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.h +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.h @@ -2,11 +2,14 @@ #include #include +#include #include namespace DB { +using Scalars = std::map; + namespace ClusterProxy { diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index fd8ca66e85c..cb499577272 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/DDLWorker.h b/src/Interpreters/DDLWorker.h index 62eba97032e..2f63d9dadee 100644 --- a/src/Interpreters/DDLWorker.h +++ b/src/Interpreters/DDLWorker.h @@ -1,5 +1,5 @@ #pragma once -#include + #include #include #include @@ -13,9 +13,15 @@ #include #include +namespace zkutil +{ + class ZooKeeper; +} + namespace DB { +class Context; class ASTAlterQuery; class AccessRightsElements; struct DDLLogEntry; diff --git a/src/Interpreters/ExpressionActions.cpp b/src/Interpreters/ExpressionActions.cpp index cb65ec35b9e..363d4765019 100644 --- a/src/Interpreters/ExpressionActions.cpp +++ b/src/Interpreters/ExpressionActions.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -509,6 +510,33 @@ std::string ExpressionAction::toString() const return ss.str(); } +ExpressionActions::ExpressionActions(const NamesAndTypesList & input_columns_, const Context & context_) + : input_columns(input_columns_), settings(context_.getSettingsRef()) +{ + for (const auto & input_elem : input_columns) + sample_block.insert(ColumnWithTypeAndName(nullptr, input_elem.type, input_elem.name)); + +#if USE_EMBEDDED_COMPILER +compilation_cache = context_.getCompiledExpressionCache(); +#endif +} + +/// For constant columns the columns themselves can be contained in `input_columns_`. +ExpressionActions::ExpressionActions(const ColumnsWithTypeAndName & input_columns_, const Context & context_) + : settings(context_.getSettingsRef()) +{ + for (const auto & input_elem : input_columns_) + { + input_columns.emplace_back(input_elem.name, input_elem.type); + sample_block.insert(input_elem); + } +#if USE_EMBEDDED_COMPILER + compilation_cache = context_.getCompiledExpressionCache(); +#endif +} + +ExpressionActions::~ExpressionActions() = default; + void ExpressionActions::checkLimits(Block & block) const { if (settings.max_temporary_columns && block.columns() > settings.max_temporary_columns) diff --git a/src/Interpreters/ExpressionActions.h b/src/Interpreters/ExpressionActions.h index 5a29eaaab9e..080e8f8a10f 100644 --- a/src/Interpreters/ExpressionActions.h +++ b/src/Interpreters/ExpressionActions.h @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #include @@ -25,6 +24,7 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } +class Context; class TableJoin; class IJoin; using JoinPtr = std::shared_ptr; @@ -42,6 +42,7 @@ class IDataType; using DataTypePtr = std::shared_ptr; class ExpressionActions; +class CompiledExpressionCache; /** Action on the block. */ @@ -155,30 +156,12 @@ class ExpressionActions public: using Actions = std::vector; - ExpressionActions(const NamesAndTypesList & input_columns_, const Context & context_) - : input_columns(input_columns_), settings(context_.getSettingsRef()) - { - for (const auto & input_elem : input_columns) - sample_block.insert(ColumnWithTypeAndName(nullptr, input_elem.type, input_elem.name)); - -#if USE_EMBEDDED_COMPILER - compilation_cache = context_.getCompiledExpressionCache(); -#endif - } + ExpressionActions(const NamesAndTypesList & input_columns_, const Context & context_); /// For constant columns the columns themselves can be contained in `input_columns_`. - ExpressionActions(const ColumnsWithTypeAndName & input_columns_, const Context & context_) - : settings(context_.getSettingsRef()) - { - for (const auto & input_elem : input_columns_) - { - input_columns.emplace_back(input_elem.name, input_elem.type); - sample_block.insert(input_elem); - } -#if USE_EMBEDDED_COMPILER - compilation_cache = context_.getCompiledExpressionCache(); -#endif - } + ExpressionActions(const ColumnsWithTypeAndName & input_columns_, const Context & context_); + + ~ExpressionActions(); /// Add the input column. /// The name of the column must not match the names of the intermediate columns that occur when evaluating the expression. diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 8bff2816df1..3341855b8c6 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -32,6 +32,7 @@ #include #include #include +#include #include #include diff --git a/src/Interpreters/GlobalSubqueriesVisitor.h b/src/Interpreters/GlobalSubqueriesVisitor.h index 78d98805814..37a358c3d28 100644 --- a/src/Interpreters/GlobalSubqueriesVisitor.h +++ b/src/Interpreters/GlobalSubqueriesVisitor.h @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include @@ -17,6 +16,7 @@ #include #include #include +#include namespace DB { diff --git a/src/Interpreters/InterpreterExplainQuery.cpp b/src/Interpreters/InterpreterExplainQuery.cpp index 841635b8d01..1c1e21fc32c 100644 --- a/src/Interpreters/InterpreterExplainQuery.cpp +++ b/src/Interpreters/InterpreterExplainQuery.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/InterpreterExplainQuery.h b/src/Interpreters/InterpreterExplainQuery.h index 058c51737b0..fbc8a998f2c 100644 --- a/src/Interpreters/InterpreterExplainQuery.h +++ b/src/Interpreters/InterpreterExplainQuery.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include @@ -8,6 +7,8 @@ namespace DB { +class Context; + /// Returns single row with explain results class InterpreterExplainQuery : public IInterpreter { diff --git a/src/Interpreters/InterpreterFactory.cpp b/src/Interpreters/InterpreterFactory.cpp index 60302848367..ccaa8fa4067 100644 --- a/src/Interpreters/InterpreterFactory.cpp +++ b/src/Interpreters/InterpreterFactory.cpp @@ -61,6 +61,7 @@ #include #include #include +#include #include diff --git a/src/Interpreters/InterpreterInsertQuery.h b/src/Interpreters/InterpreterInsertQuery.h index 476e86898d7..fef962d24a3 100644 --- a/src/Interpreters/InterpreterInsertQuery.h +++ b/src/Interpreters/InterpreterInsertQuery.h @@ -2,13 +2,14 @@ #include #include -#include #include #include namespace DB { +class Context; + /** Interprets the INSERT query. */ diff --git a/src/Interpreters/InterpreterSelectQuery.h b/src/Interpreters/InterpreterSelectQuery.h index 87a95c5b836..b97ff65e988 100644 --- a/src/Interpreters/InterpreterSelectQuery.h +++ b/src/Interpreters/InterpreterSelectQuery.h @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include @@ -25,6 +24,7 @@ namespace DB struct SubqueryForSet; class InterpreterSelectWithUnionQuery; +class Context; struct SyntaxAnalyzerResult; using SyntaxAnalyzerResultPtr = std::shared_ptr; diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index 9cdb19b1934..378cb943c04 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.h b/src/Interpreters/InterpreterSelectWithUnionQuery.h index ad78572ab77..c7a8e09578b 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.h +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.h @@ -1,15 +1,16 @@ #pragma once #include -#include #include #include +#include #include namespace DB { +class Context; class InterpreterSelectQuery; diff --git a/src/Interpreters/InterpreterSetRoleQuery.h b/src/Interpreters/InterpreterSetRoleQuery.h index afb53014c87..91cf5fc1b2e 100644 --- a/src/Interpreters/InterpreterSetRoleQuery.h +++ b/src/Interpreters/InterpreterSetRoleQuery.h @@ -6,6 +6,8 @@ namespace DB { + +class Context; class ASTSetRoleQuery; struct ExtendedRoleSet; struct User; diff --git a/src/Interpreters/InterpreterWatchQuery.cpp b/src/Interpreters/InterpreterWatchQuery.cpp index 91e7bfac2a3..dc1ae6a7cad 100644 --- a/src/Interpreters/InterpreterWatchQuery.cpp +++ b/src/Interpreters/InterpreterWatchQuery.cpp @@ -13,6 +13,7 @@ limitations under the License. */ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/InterpreterWatchQuery.h b/src/Interpreters/InterpreterWatchQuery.h index 7929b86b1c8..a0dacd08ea8 100644 --- a/src/Interpreters/InterpreterWatchQuery.h +++ b/src/Interpreters/InterpreterWatchQuery.h @@ -18,11 +18,11 @@ limitations under the License. */ #include #include #include -#include namespace DB { +class Context; class IAST; using ASTPtr = std::shared_ptr; using StoragePtr = std::shared_ptr; diff --git a/src/Interpreters/JoinedTables.cpp b/src/Interpreters/JoinedTables.cpp index b2591d94310..67363737670 100644 --- a/src/Interpreters/JoinedTables.cpp +++ b/src/Interpreters/JoinedTables.cpp @@ -1,6 +1,5 @@ #include #include -#include #include #include #include diff --git a/src/Interpreters/JoinedTables.h b/src/Interpreters/JoinedTables.h index 399acdc0768..55244e1225c 100644 --- a/src/Interpreters/JoinedTables.h +++ b/src/Interpreters/JoinedTables.h @@ -3,13 +3,14 @@ #include #include #include +#include +#include #include namespace DB { class ASTSelectQuery; -class Context; class TableJoin; struct SelectQueryOptions; diff --git a/src/Interpreters/MutationsInterpreter.h b/src/Interpreters/MutationsInterpreter.h index 9b12ce79e91..35c4f8ece0a 100644 --- a/src/Interpreters/MutationsInterpreter.h +++ b/src/Interpreters/MutationsInterpreter.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include diff --git a/src/Interpreters/PartLog.cpp b/src/Interpreters/PartLog.cpp index 7e04c5f6abb..ab22e73a0ca 100644 --- a/src/Interpreters/PartLog.cpp +++ b/src/Interpreters/PartLog.cpp @@ -10,6 +10,7 @@ #include #include #include +#include namespace DB diff --git a/src/Interpreters/QueryLog.h b/src/Interpreters/QueryLog.h index ec14f5e97fb..e503b417638 100644 --- a/src/Interpreters/QueryLog.h +++ b/src/Interpreters/QueryLog.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include diff --git a/src/Interpreters/QueryThreadLog.h b/src/Interpreters/QueryThreadLog.h index c50daa7bc88..e2c082eb54c 100644 --- a/src/Interpreters/QueryThreadLog.h +++ b/src/Interpreters/QueryThreadLog.h @@ -1,6 +1,7 @@ #pragma once #include +#include namespace ProfileEvents diff --git a/src/Interpreters/Set.cpp b/src/Interpreters/Set.cpp index afc95e11fd8..6f3031d5e7d 100644 --- a/src/Interpreters/Set.cpp +++ b/src/Interpreters/Set.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include diff --git a/src/Interpreters/Set.h b/src/Interpreters/Set.h index 0f84c2f5da8..848abc9aa8a 100644 --- a/src/Interpreters/Set.h +++ b/src/Interpreters/Set.h @@ -5,7 +5,6 @@ #include #include #include -#include #include #include @@ -17,6 +16,7 @@ namespace DB struct Range; +class Context; class IFunctionBase; using FunctionBasePtr = std::shared_ptr; diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h index b2a4eec7883..218555bebbc 100644 --- a/src/Interpreters/SystemLog.h +++ b/src/Interpreters/SystemLog.h @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include @@ -65,6 +64,12 @@ namespace ErrorCodes class Context; +class QueryLog; +class QueryThreadLog; +class PartLog; +class TextLog; +class TraceLog; +class MetricLog; class ISystemLog diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 4d609395c3a..8b3dc709ab2 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #include #include diff --git a/src/Interpreters/getTableExpressions.cpp b/src/Interpreters/getTableExpressions.cpp index 68f18c1397a..b5444f73b35 100644 --- a/src/Interpreters/getTableExpressions.cpp +++ b/src/Interpreters/getTableExpressions.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include diff --git a/src/Interpreters/interpretSubquery.cpp b/src/Interpreters/interpretSubquery.cpp index 0521ab7fd8b..e108db1af30 100644 --- a/src/Interpreters/interpretSubquery.cpp +++ b/src/Interpreters/interpretSubquery.cpp @@ -12,6 +12,7 @@ #include #include +#include namespace DB { diff --git a/src/Processors/Formats/Impl/ConstantExpressionTemplate.cpp b/src/Processors/Formats/Impl/ConstantExpressionTemplate.cpp index 7254cf91245..24fff4203bb 100644 --- a/src/Processors/Formats/Impl/ConstantExpressionTemplate.cpp +++ b/src/Processors/Formats/Impl/ConstantExpressionTemplate.cpp @@ -8,6 +8,8 @@ #include #include #include +#include +#include #include #include #include @@ -17,7 +19,6 @@ #include #include #include -#include #include diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp index d90a7974968..2947b47eefa 100644 --- a/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/src/Storages/Distributed/DirectoryMonitor.cpp @@ -81,13 +81,13 @@ StorageDistributedDirectoryMonitor::StorageDistributedDirectoryMonitor( : storage(storage_) , pool(std::move(pool_)) , path{path_ + '/'} - , should_batch_inserts(storage.global_context.getSettingsRef().distributed_directory_monitor_batch_inserts) - , min_batched_block_size_rows(storage.global_context.getSettingsRef().min_insert_block_size_rows) - , min_batched_block_size_bytes(storage.global_context.getSettingsRef().min_insert_block_size_bytes) + , should_batch_inserts(storage.global_context->getSettingsRef().distributed_directory_monitor_batch_inserts) + , min_batched_block_size_rows(storage.global_context->getSettingsRef().min_insert_block_size_rows) + , min_batched_block_size_bytes(storage.global_context->getSettingsRef().min_insert_block_size_bytes) , current_batch_file_path{path + "current_batch.txt"} - , default_sleep_time{storage.global_context.getSettingsRef().distributed_directory_monitor_sleep_time_ms.totalMilliseconds()} + , default_sleep_time{storage.global_context->getSettingsRef().distributed_directory_monitor_sleep_time_ms.totalMilliseconds()} , sleep_time{default_sleep_time} - , max_sleep_time{storage.global_context.getSettingsRef().distributed_directory_monitor_max_sleep_time_ms.totalMilliseconds()} + , max_sleep_time{storage.global_context->getSettingsRef().distributed_directory_monitor_max_sleep_time_ms.totalMilliseconds()} , log{&Logger::get(getLoggerName())} , monitor_blocker(monitor_blocker_) , bg_pool(bg_pool_) @@ -214,7 +214,7 @@ ConnectionPoolPtr StorageDistributedDirectoryMonitor::createPool(const std::stri auto pools = createPoolsForAddresses(name, pool_factory); - const auto settings = storage.global_context.getSettings(); + const auto settings = storage.global_context->getSettings(); return pools.size() == 1 ? pools.front() : std::make_shared(pools, settings.load_balancing, settings.distributed_replica_error_half_life.totalSeconds(), @@ -262,7 +262,7 @@ bool StorageDistributedDirectoryMonitor::processFiles() void StorageDistributedDirectoryMonitor::processFile(const std::string & file_path) { LOG_TRACE(log, "Started processing `" << file_path << '`'); - auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(storage.global_context.getSettingsRef()); + auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(storage.global_context->getSettingsRef()); auto connection = pool->get(timeouts); try @@ -437,7 +437,7 @@ struct StorageDistributedDirectoryMonitor::Batch Poco::File{tmp_file}.renameTo(parent.current_batch_file_path); } - auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(parent.storage.global_context.getSettingsRef()); + auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(parent.storage.global_context->getSettingsRef()); auto connection = parent.pool->get(timeouts); bool batch_broken = false; diff --git a/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/src/Storages/Distributed/DistributedBlockOutputStream.cpp index c7b0436a9a3..e08c4b7fd34 100644 --- a/src/Storages/Distributed/DistributedBlockOutputStream.cpp +++ b/src/Storages/Distributed/DistributedBlockOutputStream.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include diff --git a/src/Storages/Distributed/DistributedBlockOutputStream.h b/src/Storages/Distributed/DistributedBlockOutputStream.h index 319664e1723..17db955431c 100644 --- a/src/Storages/Distributed/DistributedBlockOutputStream.h +++ b/src/Storages/Distributed/DistributedBlockOutputStream.h @@ -11,7 +11,6 @@ #include #include #include -#include namespace Poco @@ -22,6 +21,7 @@ namespace Poco namespace DB { +class Context; class StorageDistributed; /** If insert_sync_ is true, the write is synchronous. Uses insert_timeout_ if it is not zero. diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index f0ae8b40c5b..bed57dfe68d 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -36,6 +36,7 @@ using VolumePtr = std::shared_ptr; class IMergeTreeReader; class IMergeTreeDataPartWriter; +class MarkCache; namespace ErrorCodes { diff --git a/src/Storages/MergeTree/KeyCondition.h b/src/Storages/MergeTree/KeyCondition.h index 3a3768f0e4c..f12acdbf7bf 100644 --- a/src/Storages/MergeTree/KeyCondition.h +++ b/src/Storages/MergeTree/KeyCondition.h @@ -3,7 +3,6 @@ #include #include -#include #include #include #include @@ -15,10 +14,9 @@ namespace DB { - +class Context; class IFunction; using FunctionBasePtr = std::shared_ptr; - class ExpressionActions; using ExpressionActionsPtr = std::shared_ptr; diff --git a/src/Storages/MergeTree/MergeList.h b/src/Storages/MergeTree/MergeList.h index 107b03a018a..4ee8a75a868 100644 --- a/src/Storages/MergeTree/MergeList.h +++ b/src/Storages/MergeTree/MergeList.h @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index de9d3f6e981..d9ca89d721b 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 394260096a7..2868ecaf290 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include #include #include @@ -34,6 +34,7 @@ class MergeListEntry; class AlterCommands; class MergeTreePartsMover; class MutationCommands; +class Context; class ExpressionActions; using ExpressionActionsPtr = std::shared_ptr; diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 625dc9f367a..00b474a7792 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index 4cdf57a4700..1ab10b55409 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -1,4 +1,5 @@ #include +#include namespace DB { diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 662fb067e48..92f4f6107b8 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -18,6 +18,7 @@ #include #include #include +#include /// Allow to use __uint128_t as a template parameter for boost::rational. // https://stackoverflow.com/questions/41198673/uint128-t-not-working-with-clang-and-libstdc diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index a78e2e5ae32..75d6ebe7cb7 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.h b/src/Storages/MergeTree/MergeTreeDataWriter.h index c2878145a50..e199aa3b43a 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.h +++ b/src/Storages/MergeTree/MergeTreeDataWriter.h @@ -9,7 +9,6 @@ #include #include -#include #include diff --git a/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp index 265dba0e6fe..cc1f713a57e 100644 --- a/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp @@ -1,6 +1,7 @@ #include #include #include +#include namespace DB diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp index def01b192d5..9af72e60123 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp @@ -1,6 +1,7 @@ #include #include #include +#include namespace DB diff --git a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp index cfda7fdf562..755ff87eab5 100644 --- a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp +++ b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp @@ -1,5 +1,6 @@ #include #include +#include namespace DB { diff --git a/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputProcessor.cpp b/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputProcessor.cpp index aa8c550839d..0b09fad91d1 100644 --- a/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputProcessor.cpp @@ -1,6 +1,7 @@ #include #include #include +#include namespace DB diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/src/Storages/MergeTree/MergedBlockOutputStream.cpp index e0760e87d00..bd1312f0c59 100644 --- a/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -1,4 +1,5 @@ #include +#include #include diff --git a/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp b/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp index 892b4eccfbc..1a99636534b 100644 --- a/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp +++ b/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp @@ -1,4 +1,5 @@ #include +#include namespace DB { diff --git a/src/Storages/MergeTree/RPNBuilder.h b/src/Storages/MergeTree/RPNBuilder.h index 2e457147cf4..ed25252642b 100644 --- a/src/Storages/MergeTree/RPNBuilder.h +++ b/src/Storages/MergeTree/RPNBuilder.h @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include @@ -13,6 +12,7 @@ namespace DB { +class Context; /// Builds reverse polish notation template diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp index b1164f6621c..c36879750a1 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp index 98b30498839..9d055545457 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp @@ -2,6 +2,7 @@ #include #include #include +#include namespace ProfileEvents diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index cd82a865827..75f15556edf 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index 721d58539ef..40cc8edca74 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -17,6 +17,8 @@ #include #include +#include + namespace DB { diff --git a/src/Storages/StorageBuffer.h b/src/Storages/StorageBuffer.h index f8fd1d2eaaf..6d6c1f66569 100644 --- a/src/Storages/StorageBuffer.h +++ b/src/Storages/StorageBuffer.h @@ -8,6 +8,7 @@ #include #include #include +#include namespace Poco { class Logger; } @@ -16,8 +17,6 @@ namespace Poco { class Logger; } namespace DB { -class Context; - /** During insertion, buffers the data in the RAM until certain thresholds are exceeded. * When thresholds are exceeded, flushes the data to another table. diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index eb75f53ef9c..10870993dd8 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -276,8 +277,8 @@ StorageDistributed::StorageDistributed( : IStorage(id_) , remote_database(remote_database_) , remote_table(remote_table_) - , global_context(context_) - , cluster_name(global_context.getMacros()->expand(cluster_name_)) + , global_context(std::make_unique(context_)) + , cluster_name(global_context->getMacros()->expand(cluster_name_)) , has_sharding_key(sharding_key_) , storage_policy(storage_policy_) , relative_data_path(relative_data_path_) @@ -287,7 +288,7 @@ StorageDistributed::StorageDistributed( if (sharding_key_) { - sharding_key_expr = buildShardingKeyExpression(sharding_key_, global_context, getColumns().getAllPhysical(), false); + sharding_key_expr = buildShardingKeyExpression(sharding_key_, *global_context, getColumns().getAllPhysical(), false); sharding_key_column_name = sharding_key_->getColumnName(); } @@ -297,7 +298,7 @@ StorageDistributed::StorageDistributed( /// Sanity check. Skip check if the table is already created to allow the server to start. if (!attach_ && !cluster_name.empty()) { - size_t num_local_shards = global_context.getCluster(cluster_name)->getLocalShardCount(); + size_t num_local_shards = global_context->getCluster(cluster_name)->getLocalShardCount(); if (num_local_shards && remote_database == id_.database_name && remote_table == id_.table_name) throw Exception("Distributed table " + id_.table_name + " looks at itself", ErrorCodes::INFINITE_LOOP); } @@ -325,7 +326,7 @@ void StorageDistributed::createStorage() /// Create default policy with the relative_data_path_ if (storage_policy.empty()) { - std::string path(global_context.getPath()); + std::string path(global_context->getPath()); /// Disk must ends with '/' if (!path.ends_with('/')) path += '/'; @@ -334,7 +335,7 @@ void StorageDistributed::createStorage() } else { - auto policy = global_context.getStoragePolicySelector()->get(storage_policy); + auto policy = global_context->getStoragePolicySelector()->get(storage_policy); if (policy->getVolumes().size() != 1) throw Exception("Policy for Distributed table, should have exactly one volume", ErrorCodes::BAD_ARGUMENTS); volume = policy->getVolume(0); @@ -628,7 +629,7 @@ StoragePolicyPtr StorageDistributed::getStoragePolicy() const { if (storage_policy.empty()) return {}; - return global_context.getStoragePolicySelector()->get(storage_policy); + return global_context->getStoragePolicySelector()->get(storage_policy); } void StorageDistributed::createDirectoryMonitors(const std::string & disk) @@ -655,7 +656,7 @@ StorageDistributedDirectoryMonitor& StorageDistributed::requireDirectoryMonitor( { node_data.conneciton_pool = StorageDistributedDirectoryMonitor::createPool(name, *this); node_data.directory_monitor = std::make_unique( - *this, path, node_data.conneciton_pool, monitors_blocker, global_context.getDistributedSchedulePool()); + *this, path, node_data.conneciton_pool, monitors_blocker, global_context->getDistributedSchedulePool()); } return *node_data.directory_monitor; } @@ -672,7 +673,7 @@ std::pair StorageDistributed::getPath( ClusterPtr StorageDistributed::getCluster() const { - return owned_cluster ? owned_cluster : global_context.getCluster(cluster_name); + return owned_cluster ? owned_cluster : global_context->getCluster(cluster_name); } ClusterPtr StorageDistributed::getOptimizedCluster(const Context & context, const ASTPtr & query_ptr) const diff --git a/src/Storages/StorageDistributed.h b/src/Storages/StorageDistributed.h index 125e1dee1e6..c934b3870f4 100644 --- a/src/Storages/StorageDistributed.h +++ b/src/Storages/StorageDistributed.h @@ -127,7 +127,7 @@ public: String remote_table; ASTPtr remote_table_function_ptr; - Context global_context; + std::unique_ptr global_context; Logger * log = &Logger::get("StorageDistributed"); /// Used to implement TableFunctionRemote. diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index e22e81d5041..a25b4d3beba 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include #include #include @@ -21,9 +23,8 @@ #include #include #include -#include -#include #include +#include namespace DB diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index e6fc32ecbf9..a553fbbe970 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -44,6 +44,7 @@ #include #include +#include #include #include diff --git a/src/Storages/StorageSet.cpp b/src/Storages/StorageSet.cpp index 613f5b6a2ed..9a3a2b0ac94 100644 --- a/src/Storages/StorageSet.cpp +++ b/src/Storages/StorageSet.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include diff --git a/src/Storages/StorageView.cpp b/src/Storages/StorageView.cpp index 287e93fc99a..636c7f9d64d 100644 --- a/src/Storages/StorageView.cpp +++ b/src/Storages/StorageView.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include diff --git a/src/Storages/System/StorageSystemColumns.cpp b/src/Storages/System/StorageSystemColumns.cpp index 283c7f2c4a5..ab824fc8bdc 100644 --- a/src/Storages/System/StorageSystemColumns.cpp +++ b/src/Storages/System/StorageSystemColumns.cpp @@ -12,6 +12,7 @@ #include #include #include +#include namespace DB diff --git a/src/Storages/System/StorageSystemDisks.cpp b/src/Storages/System/StorageSystemDisks.cpp index 5ddf7de9ec8..c7d134645dc 100644 --- a/src/Storages/System/StorageSystemDisks.cpp +++ b/src/Storages/System/StorageSystemDisks.cpp @@ -1,6 +1,7 @@ #include #include #include +#include namespace DB { diff --git a/src/Storages/System/StorageSystemMutations.cpp b/src/Storages/System/StorageSystemMutations.cpp index 968c67bc14a..d4a262860dc 100644 --- a/src/Storages/System/StorageSystemMutations.cpp +++ b/src/Storages/System/StorageSystemMutations.cpp @@ -9,6 +9,7 @@ #include #include #include +#include namespace DB diff --git a/src/Storages/System/StorageSystemPartsBase.cpp b/src/Storages/System/StorageSystemPartsBase.cpp index 0fce41d8398..6356e6d699e 100644 --- a/src/Storages/System/StorageSystemPartsBase.cpp +++ b/src/Storages/System/StorageSystemPartsBase.cpp @@ -14,6 +14,7 @@ #include #include #include +#include namespace DB diff --git a/src/Storages/System/StorageSystemStoragePolicies.cpp b/src/Storages/System/StorageSystemStoragePolicies.cpp index 81b6ddd465a..fdfdc277fd2 100644 --- a/src/Storages/System/StorageSystemStoragePolicies.cpp +++ b/src/Storages/System/StorageSystemStoragePolicies.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB diff --git a/src/TableFunctions/ITableFunctionFileLike.h b/src/TableFunctions/ITableFunctionFileLike.h index a18ca8ea4c8..55df9a7484d 100644 --- a/src/TableFunctions/ITableFunctionFileLike.h +++ b/src/TableFunctions/ITableFunctionFileLike.h @@ -1,11 +1,11 @@ #pragma once #include -#include namespace DB { class ColumnsDescription; +class Context; /* * function(source, format, structure) - creates a temporary storage from formated source diff --git a/src/TableFunctions/TableFunctionFile.cpp b/src/TableFunctions/TableFunctionFile.cpp index 0a68ed59aa2..4f67f4cfd10 100644 --- a/src/TableFunctions/TableFunctionFile.cpp +++ b/src/TableFunctions/TableFunctionFile.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include "registerTableFunctions.h" namespace DB diff --git a/src/TableFunctions/TableFunctionFile.h b/src/TableFunctions/TableFunctionFile.h index ead924f6828..e0d8c20ac61 100644 --- a/src/TableFunctions/TableFunctionFile.h +++ b/src/TableFunctions/TableFunctionFile.h @@ -1,11 +1,11 @@ #pragma once #include -#include namespace DB { +class Context; /* file(path, format, structure) - creates a temporary storage from file * * diff --git a/src/TableFunctions/TableFunctionHDFS.h b/src/TableFunctions/TableFunctionHDFS.h index 443ce0aa93b..5e8503b318e 100644 --- a/src/TableFunctions/TableFunctionHDFS.h +++ b/src/TableFunctions/TableFunctionHDFS.h @@ -5,11 +5,13 @@ #if USE_HDFS #include -#include namespace DB { + +class Context; + /* hdfs(name_node_ip:name_node_port, format, structure) - creates a temporary storage from hdfs file * */ @@ -27,6 +29,7 @@ private: const String & source, const String & format, const ColumnsDescription & columns, Context & global_context, const std::string & table_name, const String & compression_method) const override; const char * getStorageTypeName() const override { return "HDFS"; } }; + } #endif diff --git a/src/TableFunctions/TableFunctionInput.cpp b/src/TableFunctions/TableFunctionInput.cpp index e8f3453da06..5b4a3d97ee4 100644 --- a/src/TableFunctions/TableFunctionInput.cpp +++ b/src/TableFunctions/TableFunctionInput.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include "registerTableFunctions.h" diff --git a/src/TableFunctions/TableFunctionInput.h b/src/TableFunctions/TableFunctionInput.h index 92c2e3a6e54..a2791533c5d 100644 --- a/src/TableFunctions/TableFunctionInput.h +++ b/src/TableFunctions/TableFunctionInput.h @@ -1,11 +1,13 @@ #pragma once #include -#include namespace DB { + +class Context; + /* input(structure) - allows to make INSERT SELECT from incoming stream of data */ class TableFunctionInput : public ITableFunction @@ -18,4 +20,5 @@ private: StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const override; const char * getStorageTypeName() const override { return "Input"; } }; + } diff --git a/src/TableFunctions/TableFunctionURL.h b/src/TableFunctions/TableFunctionURL.h index 61dca561f0c..366d573bcf4 100644 --- a/src/TableFunctions/TableFunctionURL.h +++ b/src/TableFunctions/TableFunctionURL.h @@ -1,11 +1,13 @@ #pragma once #include -#include namespace DB { + +class Context; + /* url(source, format, structure) - creates a temporary storage from url */ class TableFunctionURL : public ITableFunctionFileLike @@ -22,4 +24,5 @@ private: const String & source, const String & format, const ColumnsDescription & columns, Context & global_context, const std::string & table_name, const String & compression_method) const override; const char * getStorageTypeName() const override { return "URL"; } }; + } From 09474ebea51996c668e1a5ebcdd4a3f89e284de1 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 18 May 2020 21:13:56 +0300 Subject: [PATCH 495/738] Fix memory tracking for two-level GROUP BY when not all rows read from Aggregator (TCP) Example of such cases: - SELECT GROUP BY LIMIT - SELECT GROUP BY with subsequent MEMORY_LIMIT_EXCEEDED error And it should be two-level aggregation, since otherwise there will be only one hashtable which will be cleared correctly, only if you have two-level GROUP BY some of hashtables will not be cleared since nobody consume rows. Before this patch: 09:39.015292 [ 3070801 ] {609a0610-e377-4132-9cf3-f49454cf3c96} executeQuery: Read 1000000 rows, 7.63 MiB in 0.707 sec., 1413826 rows/sec., 10.79 MiB/sec. 09:39.015348 [ 3070801 ] {609a0610-e377-4132-9cf3-f49454cf3c96} MemoryTracker: Peak memory usage (for query): 51.93 MiB. 09:39.015942 [ 3070801 ] {} Aggregator: Destroying aggregate states <-- **problem** 09:39.017057 [ 3070801 ] {} Aggregator: Destroying aggregate states <-- 09:39.017961 [ 3070801 ] {} MemoryTracker: Peak memory usage (for query): 51.93 MiB. 09:39.018029 [ 3070801 ] {} TCPHandler: Processed in 0.711 sec. After this patch: 16:24.544030 [ 3087333 ] {79da208a-b3c0-48d4-9943-c974a3cbb6ea} executeQuery: Read 1000000 rows, 7.63 MiB in 0.599 sec., 1670199 rows/sec., 12.74 MiB/sec. 16:24.544084 [ 3087333 ] {79da208a-b3c0-48d4-9943-c974a3cbb6ea} MemoryTracker: Peak memory usage (for query): 72.11 MiB. 16:24.544398 [ 3087333 ] {79da208a-b3c0-48d4-9943-c974a3cbb6ea} Aggregator: Destroying aggregate states 16:24.545485 [ 3087333 ] {79da208a-b3c0-48d4-9943-c974a3cbb6ea} Aggregator: Destroying aggregate states 16:24.547053 [ 3087333 ] {} MemoryTracker: Peak memory usage (for query): 72.11 MiB. 16:24.547093 [ 3087333 ] {} TCPHandler: Processed in 0.603 sec. --- programs/server/TCPHandler.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/programs/server/TCPHandler.cpp b/programs/server/TCPHandler.cpp index a792af30cf2..3ae87a57725 100644 --- a/programs/server/TCPHandler.cpp +++ b/programs/server/TCPHandler.cpp @@ -278,8 +278,11 @@ void TCPHandler::runImpl() sendLogs(); sendEndOfStream(); - query_scope.reset(); + /// QueryState should be cleared before QueryScope, since otherwise + /// the MemoryTracker will be wrong for possible deallocations. + /// (i.e. deallocations from the Aggregator with two-level aggregation) state.reset(); + query_scope.reset(); } catch (const Exception & e) { @@ -359,8 +362,11 @@ void TCPHandler::runImpl() try { - query_scope.reset(); + /// QueryState should be cleared before QueryScope, since otherwise + /// the MemoryTracker will be wrong for possible deallocations. + /// (i.e. deallocations from the Aggregator with two-level aggregation) state.reset(); + query_scope.reset(); } catch (...) { From 77bdb5f4629bc72014daa1bad67ab3bd015cbd3f Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 18 May 2020 21:13:56 +0300 Subject: [PATCH 496/738] Add a test for GROUP BY memory accounting - GROUP BY - GROUP BY LIMIT - GROUP BY MEMORY_LIMIT_EXCEEDED --- ...1_group_by_limit_memory_tracking.reference | 0 .../01281_group_by_limit_memory_tracking.sql | 84 +++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 tests/queries/0_stateless/01281_group_by_limit_memory_tracking.reference create mode 100644 tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sql diff --git a/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.reference b/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sql b/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sql new file mode 100644 index 00000000000..fc8f81975f0 --- /dev/null +++ b/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sql @@ -0,0 +1,84 @@ +DROP TABLE IF EXISTS trace_log_01281; +DROP TABLE IF EXISTS trace_log_01281_mv; +DROP TABLE IF EXISTS trace_log_01281_assert; + +-- better alternative will be to TRUNCATE TABLE system.*_log +-- but this will be a separate issue +CREATE TABLE trace_log_01281 AS system.trace_log Engine=Memory(); +CREATE MATERIALIZED VIEW trace_log_01281_mv TO trace_log_01281 AS SELECT * FROM system.trace_log WHERE trace_type = 'MemorySample'; +CREATE VIEW trace_log_01281_assert AS SELECT + *, + throwIf(cnt < 0, 'no memory profile captured'), + throwIf(queries != 1, 'too much queries'), + throwIf(alloc < 100e6, 'minimal allocation had not been done'), + throwIf((alloc+free)/alloc > 0.05, 'memory accounting leaked more than 5%') + FROM ( + SELECT + count() cnt, + uniq(query_id) queries, + sumIf(size, size > 0) alloc, + sumIf(size, size < 0) free + FROM trace_log_01281 + ); + +-- +-- Basic +-- NOTE: 0 (and even 1e6) is too small, will make SYSTEM FLUSH LOGS too slow +-- (in debug build at least) +-- +SET max_untracked_memory=4e6; + +TRUNCATE TABLE trace_log_01281; +-- single { +SET max_threads=1; +SET memory_profiler_sample_probability=1; +SELECT uniqExactState(number) FROM numbers(toUInt64(2e6)) GROUP BY number % 2e5 FORMAT Null; +SET memory_profiler_sample_probability=0; +SYSTEM FLUSH LOGS; +-- } +SELECT * FROM trace_log_01281_assert FORMAT Null; + +TRUNCATE TABLE trace_log_01281; +-- single limit { +SET max_threads=1; +SET memory_profiler_sample_probability=1; +SELECT uniqExactState(number) FROM numbers(toUInt64(2e6)) GROUP BY number % 2e5 LIMIT 10 FORMAT Null; +SET memory_profiler_sample_probability=0; +SYSTEM FLUSH LOGS; +-- } +SELECT * FROM trace_log_01281_assert FORMAT Null; + +TRUNCATE TABLE trace_log_01281; +-- two-level { +-- need to have multiple threads for two-level aggregation +SET max_threads=2; +SET memory_profiler_sample_probability=1; +SELECT uniqExactState(number) FROM numbers_mt(toUInt64(2e6)) GROUP BY number % 2e5 FORMAT Null; +SET memory_profiler_sample_probability=0; +SYSTEM FLUSH LOGS; +-- } +SELECT * FROM trace_log_01281_assert FORMAT Null; + +TRUNCATE TABLE trace_log_01281; +-- two-level limit { +-- need to have multiple threads for two-level aggregation +SET max_threads=2; +SET memory_profiler_sample_probability=1; +SELECT uniqExactState(number) FROM numbers_mt(toUInt64(2e6)) GROUP BY number % 2e5 LIMIT 10 FORMAT Null; +SET memory_profiler_sample_probability=0; +SYSTEM FLUSH LOGS; +-- } +SELECT * FROM trace_log_01281_assert FORMAT Null; + +TRUNCATE TABLE trace_log_01281; +-- two-level MEMORY_LIMIT_EXCEEDED { +-- need to have multiple threads for two-level aggregation +SET max_threads=2; +SET memory_profiler_sample_probability=1; +SET max_memory_usage='150M'; +SELECT uniqExactState(number) FROM numbers_mt(toUInt64(10e6)) GROUP BY number % 1e6 FORMAT Null; -- { serverError 241; } +SET memory_profiler_sample_probability=0; +SET max_memory_usage=0; +SYSTEM FLUSH LOGS; +-- } +SELECT * FROM trace_log_01281_assert FORMAT Null; From 34508b8f641e8872ff99e449705d50813f3eb87b Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 20 May 2020 05:19:19 +0300 Subject: [PATCH 497/738] [wip] Add memory stats to performance test --- docker/test/performance-comparison/compare.sh | 353 +++++++++++------- .../test/performance-comparison/entrypoint.sh | 2 +- docker/test/performance-comparison/eqmed.sql | 41 +- docker/test/performance-comparison/perf.py | 23 +- docker/test/performance-comparison/report.py | 49 +-- 5 files changed, 286 insertions(+), 182 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index af88a3954d7..0c0289d811c 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -133,7 +133,7 @@ function run_tests fi # Delete old report files. - for x in {test-times,skipped-tests,wall-clock-times,report-thresholds,client-times}.tsv + for x in {test-times,wall-clock-times}.tsv do rm -v "$x" ||: touch "$x" @@ -220,66 +220,127 @@ function get_profiles # Build and analyze randomization distribution for all queries. function analyze_queries { -rm -v analyze-commands.txt analyze-errors.log all-queries.tsv unstable-queries.tsv ./*-report.tsv raw-queries.tsv client-times.tsv report-thresholds.tsv ||: +rm -v analyze-commands.txt analyze-errors.log all-queries.tsv unstable-queries.tsv ./*-report.tsv raw-queries.tsv ||: +rm -rfv analyze ||: +mkdir analyze ||: + +# FIXME This loop builds column definitons from TSVWithNamesAndTypes in an +# absolutely atrocious way. This should be done by the file() function itself. +for x in {right,left}-{addresses,{query,query-thread,trace,metric}-log}.tsv +do + paste -d' ' \ + <(sed -n '1{s/\t/\n/g;p;q}' "$x" | sed 's/\(^.*$\)/"\1"/') \ + <(sed -n '2{s/\t/\n/g;p;q}' "$x" ) \ + | tr '\n' ', ' | sed 's/,$//' > "$x.columns" +done # Split the raw test output into files suitable for analysis. IFS=$'\n' for test_file in $(find . -maxdepth 1 -name "*-raw.tsv" -print) do test_name=$(basename "$test_file" "-raw.tsv") - sed -n "s/^query\t//p" < "$test_file" > "$test_name-queries.tsv" - sed -n "s/^client-time/$test_name/p" < "$test_file" >> "client-times.tsv" - sed -n "s/^report-threshold/$test_name/p" < "$test_file" >> "report-thresholds.tsv" - sed -n "s/^skipped/$test_name/p" < "$test_file" >> "skipped-tests.tsv" + sed -n "s/^query\t/$test_name\t/p" < "$test_file" >> "analyze/query-runs.tsv" + sed -n "s/^client-time/$test_name/p" < "$test_file" >> "analyze/client-times.tsv" + sed -n "s/^report-threshold/$test_name/p" < "$test_file" >> "analyze/report-thresholds.tsv" + sed -n "s/^skipped/$test_name/p" < "$test_file" >> "analyze/skipped-tests.tsv" + sed -n "s/^display-name/$test_name/p" < "$test_file" >> "analyze/query-display-names.tsv" done unset IFS +# for each query run, prepare array of metrics from query log +clickhouse-local --query " +create view query_runs as select * from file('analyze/query-runs.tsv', TSV, + 'test text, query_index int, query_id text, version UInt8, time float'); + +create view left_query_log as select * + from file('left-query-log.tsv', TSVWithNamesAndTypes, + '$(cat "left-query-log.tsv.columns")'); + +create view right_query_log as select * + from file('right-query-log.tsv', TSVWithNamesAndTypes, + '$(cat "right-query-log.tsv.columns")'); + +create table query_metrics engine File(TSV, -- do not add header -- will parse with grep + 'analyze/query-run-metrics.tsv') + as select + test, query_index, 0 run, version, + [ + -- server-reported time + query_duration_ms / toFloat64(1000) + , toFloat64(memory_usage) + -- client-reported time + , query_runs.time + ] metrics + from ( + select *, 0 version from left_query_log + union all + select *, 1 version from right_query_log + ) query_logs + right join query_runs + using (query_id, version) + ; +" + # This is a lateral join in bash... please forgive me. -# We don't have arrayPermute(), so I have to make random permutations with +# We don't have arrayPermute(), so I have to make random permutations with # `order by rand`, and it becomes really slow if I do it for more than one # query. We also don't have lateral joins. So I just put all runs of each # query into a separate file, and then compute randomization distribution # for each file. I do this in parallel using GNU parallel. +query_index=1 IFS=$'\n' -for test_file in $(find . -maxdepth 1 -name "*-queries.tsv" -print) +for prefix in $(cut -f1,2 "analyze/query-run-metrics.tsv" | sort | uniq) do - test_name=$(basename "$test_file" "-queries.tsv") - query_index=1 - for query in $(cut -d' ' -f1 "$test_file" | sort | uniq) - do - query_prefix="$test_name.q$query_index" - query_index=$((query_index + 1)) - grep -F "$query " "$test_file" > "$query_prefix.tmp" - printf "%s\0\n" \ - "clickhouse-local \ - --file \"$query_prefix.tmp\" \ - --structure 'query text, run int, version UInt32, time float' \ - --query \"$(cat "$script_dir/eqmed.sql")\" \ - >> \"$test_name-report.tsv\"" \ - 2>> analyze-errors.log \ - >> analyze-commands.txt - done + file="analyze/q$query_index.tmp" + grep -F "$prefix " "analyze/query-run-metrics.tsv" > "$file" & + printf "%s\0\n" \ + "clickhouse-local \ + --file \"$file\" \ + --structure 'test text, query text, run int, version UInt8, metrics Array(float)' \ + --query \"$(cat "$script_dir/eqmed.sql")\" \ + >> \"analyze/query-reports.tsv\"" \ + 2>> analyze/errors.log \ + >> analyze/commands.txt + + query_index=$((query_index + 1)) done wait unset IFS -parallel --null < analyze-commands.txt +parallel --joblog analyze/parallel-log.txt --null < analyze/commands.txt } # Analyze results function report { - rm -r report ||: mkdir report ||: - rm ./*.{rep,svg} test-times.tsv test-dump.tsv unstable.tsv unstable-query-ids.tsv unstable-query-metrics.tsv changed-perf.tsv unstable-tests.tsv unstable-queries.tsv bad-tests.tsv slow-on-client.tsv all-queries.tsv ||: -cat analyze-errors.log >> report/errors.log ||: +cat analyze/errors.log >> report/errors.log ||: cat profile-errors.log >> report/errors.log ||: clickhouse-local --query " +create view query_display_names as select * from + file('analyze/query-display-names.tsv', TSV, + 'test text, query_index int, query_display_name text') + ; + +create table query_metric_stats engine File(TSVWithNamesAndTypes, + 'report/query-metric-stats.tsv') as + select *, metric_name + from file ('analyze/query-reports.tsv', TSV, 'left Array(float), + right Array(float), diff Array(float), stat_threshold Array(float), + test text, query_index int') reports + left array join ['server_time', 'memory', 'client_time'] as metric_name, + left, right, diff, stat_threshold + left join query_display_names + on reports.test = query_display_names.test + and reports.query_index = query_display_names.query_index + ; + +-- Main statistics for queries -- query time as reported in query log. create table queries engine File(TSVWithNamesAndTypes, 'report/queries.tsv') as select -- FIXME Comparison mode doesn't make sense for queries that complete @@ -296,53 +357,54 @@ create table queries engine File(TSVWithNamesAndTypes, 'report/queries.tsv') left, right, diff, stat_threshold, if(report_threshold > 0, report_threshold, 0.10) as report_threshold, - reports.test, - query - from - ( - select *, - replaceAll(_file, '-report.tsv', '') test - from file('*-report.tsv', TSV, 'left float, right float, diff float, stat_threshold float, query text') - ) reports - left join file('report-thresholds.tsv', TSV, 'test text, report_threshold float') thresholds - using test - ; + test, query_index, query_display_name + from query_metric_stats + left join file('analyze/report-thresholds.tsv', TSV, + 'test text, report_threshold float') thresholds + on query_metric_stats.test = thresholds.test + where metric_name = 'server_time' + order by test, query_index, metric_name + ; -- keep the table in old format so that we can analyze new and old data together create table queries_old_format engine File(TSVWithNamesAndTypes, 'queries.rep') - as select short, changed_fail, unstable_fail, left, right, diff, stat_threshold, test, query + as select short, changed_fail, unstable_fail, left, right, diff, + stat_threshold, test, query_display_name query from queries ; -- save all test runs as JSON for the new comparison page -create table all_query_funs_json engine File(JSON, 'report/all-query-runs.json') as - select test, query, versions_runs[1] runs_left, versions_runs[2] runs_right +create table all_query_runs_json engine File(JSON, 'report/all-query-runs.json') as + select test, query_display_name query, + versions_runs[1] runs_left, versions_runs[2] runs_right from ( select - test, query, + test, query_index, groupArrayInsertAt(runs, version) versions_runs from ( select - replaceAll(_file, '-queries.tsv', '') test, - query, version, - groupArray(time) runs - from file('*-queries.tsv', TSV, 'query text, run int, version UInt32, time float') - group by test, query, version + test, query_index, version, + groupArray(metrics[1]) runs + from file('analyze/query-run-metrics.tsv', TSV, + 'test text, query_index int, run int, version UInt8, metrics Array(float)') + group by test, query_index, version ) - group by test, query - ) + group by test, query_index + ) runs + left join query_display_names using (test, query_index) ; create table changed_perf_tsv engine File(TSV, 'report/changed-perf.tsv') as - select left, right, diff, stat_threshold, changed_fail, test, query from queries where changed_show - order by abs(diff) desc; + select left, right, diff, stat_threshold, changed_fail, test, query_display_name + from queries where changed_show order by abs(diff) desc; create table unstable_queries_tsv engine File(TSV, 'report/unstable-queries.tsv') as - select left, right, diff, stat_threshold, unstable_fail, test, query from queries where unstable_show - order by stat_threshold desc; + select left, right, diff, stat_threshold, unstable_fail, test, query_display_name + from queries where unstable_show order by stat_threshold desc; -create table queries_for_flamegraph engine File(TSVWithNamesAndTypes, 'report/queries-for-flamegraph.tsv') as - select query, test from queries where unstable_show or changed_show +create table queries_for_flamegraph engine File(TSVWithNamesAndTypes, + 'report/queries-for-flamegraph.tsv') as + select test, query_index from queries where unstable_show or changed_show ; create table unstable_tests_tsv engine File(TSV, 'report/bad-tests.tsv') as @@ -350,23 +412,23 @@ create table unstable_tests_tsv engine File(TSV, 'report/bad-tests.tsv') as group by test having s > 0 order by s desc; create table query_time engine Memory as select * - from file('client-times.tsv', TSV, 'test text, query text, client float, server float'); + from file('analyze/client-times.tsv', TSV, + 'test text, query_index int, client float, server float'); create table wall_clock engine Memory as select * from file('wall-clock-times.tsv', TSV, 'test text, real float, user float, system float'); create table slow_on_client_tsv engine File(TSV, 'report/slow-on-client.tsv') as - select client, server, floor(client/server, 3) p, query - from query_time where p > 1.02 order by p desc; + select client, server, floor(client/server, 3) p, query_display_name + from query_time left join query_display_names using (test, query_index) + where p > 1.02 order by p desc; create table test_time engine Memory as select test, sum(client) total_client_time, maxIf(client, not short) query_max, minIf(client, not short) query_min, - count(*) queries, - sum(short) short_queries - from query_time full join queries - using test, query + count(*) queries, sum(short) short_queries + from query_time full join queries using (test, query_index) group by test; create table test_times_tsv engine File(TSV, 'report/test-times.tsv') as @@ -378,40 +440,89 @@ create table test_times_tsv engine File(TSV, 'report/test-times.tsv') as floor(real / queries, 3) avg_real_per_query, floor(query_min, 3) from test_time - -- wall clock times are also measured for skipped tests, so don't - -- do full join - left join wall_clock using test + -- wall clock times are also measured for skipped tests, so don't + -- do full join + left join wall_clock using test order by avg_real_per_query desc; +-- report for all queries page, only main metric create table all_tests_tsv engine File(TSV, 'report/all-queries.tsv') as select changed_fail, unstable_fail, left, right, diff, floor(left > right ? left / right : right / left, 3), - stat_threshold, test, query - from queries order by test, query; + stat_threshold, test, query_display_name + from queries order by test, query_display_name; + +-- new report for all queries with all metrics (no page yet) +create table all_query_metrics_tsv engine File(TSV, 'report/all-query-metrics.tsv') as + select metric_name, left, right, diff, + floor(left > right ? left / right : right / left, 3), + stat_threshold, test, query_index, query_display_name + from query_metric_stats + order by test, query_index; " 2> >(tee -a report/errors.log 1>&2) -for x in {right,left}-{addresses,{query,query-thread,trace,metric}-log}.tsv -do - # FIXME This loop builds column definitons from TSVWithNamesAndTypes in an - # absolutely atrocious way. This should be done by the file() function itself. - paste -d' ' \ - <(sed -n '1{s/\t/\n/g;p;q}' "$x" | sed 's/\(^.*$\)/"\1"/') \ - <(sed -n '2{s/\t/\n/g;p;q}' "$x" ) \ - | tr '\n' ', ' | sed 's/,$//' > "$x.columns" -done - +# Prepare source data for metrics and flamegraphs for unstable queries. for version in {right,left} -do -clickhouse-local --query " + do + rm -rf data + clickhouse-local --query " create view queries_for_flamegraph as select * from file('report/queries-for-flamegraph.tsv', TSVWithNamesAndTypes, - 'query text, test text'); + 'test text, query_index int'); + +create view query_runs as + with 0 as left, 1 as right + select * from file('analyze/query-runs.tsv', TSV, + 'test text, query_index int, query_id text, version UInt8, time float') + where version = $version + ; + +create view query_display_names as select * from + file('analyze/query-display-names.tsv', TSV, + 'test text, query_index int, query_display_name text') + ; + +create table unstable_query_runs engine File(TSVWithNamesAndTypes, + 'unstable-query-runs.$version.rep') as + select test, query_index, query_display_name, query_id + from query_runs + join queries_for_flamegraph on + query_runs.test = queries_for_flamegraph.test + and query_runs.query_index = queries_for_flamegraph.query_index + left join query_display_names on + query_runs.test = query_display_names.test + and query_runs.query_index = query_display_names.query_index + ; create view query_log as select * from file('$version-query-log.tsv', TSVWithNamesAndTypes, '$(cat "$version-query-log.tsv.columns")'); +create table unstable_run_metrics engine File(TSVWithNamesAndTypes, + 'unstable-run-metrics.$version.rep') as + select + test, query_index, query_id, + ProfileEvents.Values value, ProfileEvents.Names metric + from query_log array join ProfileEvents + join unstable_query_runs using (query_id) + ; + +create table unstable_run_metrics_2 engine File(TSVWithNamesAndTypes, + 'unstable-run-metrics-2.$version.rep') as + select + test, query_index, query_id, + v, n + from ( + select + test, query_index, query_id, + ['memory_usage', 'read_bytes', 'written_bytes', 'query_duration_ms'] n, + [memory_usage, read_bytes, written_bytes, query_duration_ms] v + from query_log + join unstable_query_runs using (query_id) + ) + array join v, n; + create view trace_log as select * from file('$version-trace-log.tsv', TSVWithNamesAndTypes, '$(cat "$version-trace-log.tsv.columns")'); @@ -423,88 +534,64 @@ create view addresses_src as select * create table addresses_join_$version engine Join(any, left, address) as select addr address, name from addresses_src; -create table unstable_query_runs engine File(TSVWithNamesAndTypes, - 'unstable-query-runs.$version.rep') as - select query, query_id from query_log - where query in (select query from queries_for_flamegraph) - and query_id not like 'prewarm %' - ; - -create table unstable_query_log engine File(Vertical, - 'unstable-query-log.$version.rep') as - select * from query_log - where query_id in (select query_id from unstable_query_runs); - -create table unstable_run_metrics engine File(TSVWithNamesAndTypes, - 'unstable-run-metrics.$version.rep') as - select ProfileEvents.Values value, ProfileEvents.Names metric, query_id, query - from query_log array join ProfileEvents - where query_id in (select query_id from unstable_query_runs) - ; - -create table unstable_run_metrics_2 engine File(TSVWithNamesAndTypes, - 'unstable-run-metrics-2.$version.rep') as - select v, n, query_id, query - from - (select - ['memory_usage', 'read_bytes', 'written_bytes', 'query_duration_ms'] n, - [memory_usage, read_bytes, written_bytes, query_duration_ms] v, - query, - query_id - from query_log - where query_id in (select query_id from unstable_query_runs)) - array join n, v; - create table unstable_run_traces engine File(TSVWithNamesAndTypes, 'unstable-run-traces.$version.rep') as select + test, query_index, query_id, count() value, - joinGet(addresses_join_$version, 'name', arrayJoin(trace)) metric, - unstable_query_runs.query_id, - any(unstable_query_runs.query) query - from unstable_query_runs - join trace_log on trace_log.query_id = unstable_query_runs.query_id - group by unstable_query_runs.query_id, metric + joinGet(addresses_join_$version, 'name', arrayJoin(trace)) metric + from trace_log + join unstable_query_runs using query_id + group by test, query_index, query_id, metric order by count() desc ; create table metric_devation engine File(TSVWithNamesAndTypes, 'metric-deviation.$version.rep') as - select query, floor((q[3] - q[1])/q[2], 3) d, - quantilesExact(0, 0.5, 1)(value) q, metric - from (select * from unstable_run_metrics - union all select * from unstable_run_traces - union all select * from unstable_run_metrics_2) mm - join queries_for_flamegraph using query - group by query, metric - having d > 0.5 - order by query desc, d desc + -- first goes the key used to split the file with grep + select test, query_index, query_display_name, + d, q, metric + from ( + select + test, query_index, + floor((q[3] - q[1])/q[2], 3) d, + quantilesExact(0, 0.5, 1)(value) q, metric + from (select * from unstable_run_metrics + union all select * from unstable_run_traces + union all select * from unstable_run_metrics_2) mm + group by test, query_index, metric + having d > 0.5 + ) metrics + left join unstable_query_runs using (test, query_index) + order by test, query_index, d desc ; create table stacks engine File(TSV, 'stacks.$version.rep') as select - query, + -- first goes the key used to split the file with grep + test, query_index, any(query_display_name), arrayStringConcat( arrayMap(x -> joinGet(addresses_join_$version, 'name', x), arrayReverse(trace) ), ';' ) readable_trace, - count() + count() c from trace_log join unstable_query_runs using query_id - group by query, trace + group by test, query_index, trace ; " 2> >(tee -a report/errors.log 1>&2) # do not run in parallel because they use the same data dir for StorageJoins which leads to weird errors. done wait +# Create per-query flamegraphs and files with metrics IFS=$'\n' for version in {right,left} do - for query in $(cut -d' ' -f1 "stacks.$version.rep" | sort | uniq) + for query in $(cut -d' ' -f1,2,3 "stacks.$version.rep" | sort | uniq) do - query_file=$(echo "$query" | cut -c-120 | sed 's/[/]/_/g') + query_file=$(echo "$query" | cut -c-120 | sed 's/[/ ]/_/g') # Build separate .svg flamegraph for each query. grep -F "$query " "stacks.$version.rep" \ diff --git a/docker/test/performance-comparison/entrypoint.sh b/docker/test/performance-comparison/entrypoint.sh index 500dd872c8d..3b8a24ad5da 100755 --- a/docker/test/performance-comparison/entrypoint.sh +++ b/docker/test/performance-comparison/entrypoint.sh @@ -119,5 +119,5 @@ done dmesg -T > dmesg.log -7z a /output/output.7z ./*.{log,tsv,html,txt,rep,svg} {right,left}/{performance,db/preprocessed_configs,scripts} ./report +7z a /output/output.7z ./*.{log,tsv,html,txt,rep,svg} {right,left}/{performance,db/preprocessed_configs,scripts} report analyze cp compare.log /output diff --git a/docker/test/performance-comparison/eqmed.sql b/docker/test/performance-comparison/eqmed.sql index 4f8482ea74b..f7f8d6ac40d 100644 --- a/docker/test/performance-comparison/eqmed.sql +++ b/docker/test/performance-comparison/eqmed.sql @@ -1,32 +1,37 @@ --- input is table(query text, run UInt32, version int, time float) +-- input is table(test text, query text, run UInt32, version int, metrics Array(float)) select - floor(original_medians_array.time_by_version[1], 4) l, - floor(original_medians_array.time_by_version[2], 4) r, - floor((r - l) / l, 3) diff_percent, - floor(threshold / l, 3) threshold_percent, - query + arrayMap(x -> floor(x, 4), original_medians_array.medians_by_version[1] as l) l_rounded, + arrayMap(x -> floor(x, 4), original_medians_array.medians_by_version[2] as r) r_rounded, + arrayMap(x, y -> floor((y - x) / x, 3), l, r) diff_percent, + arrayMap(x, y -> floor(x / y, 3), threshold, l) threshold_percent, + test, query from ( -- quantiles of randomization distributions - select quantileExact(0.999)(abs(time_by_label[1] - time_by_label[2]) as d) threshold + select quantileExactForEach(0.999)( + arrayMap(x, y -> abs(x - y), metrics_by_label[1], metrics_by_label[2]) as d + ) threshold ---- uncomment to see what the distribution is really like - --, uniqExact(d) u + --, uniqExact(d.1) u --, arraySort(x->x.1, -- arrayZip( - -- (sumMap([d], [1]) as f).1, + -- (sumMap([d.1], [1]) as f).1, -- f.2)) full_histogram from ( - select virtual_run, groupArrayInsertAt(median_time, random_label) time_by_label -- make array 'random label' -> 'median time' + -- make array 'random label' -> '[median metric]' + select virtual_run, groupArrayInsertAt(median_metrics, random_label) metrics_by_label from ( - select medianExact(time) median_time, virtual_run, random_label -- get median times, grouping by random label + -- get [median metric] arrays among virtual runs, grouping by random label + select medianExactForEach(metrics) median_metrics, virtual_run, random_label from ( - select *, toUInt32(rowNumberInAllBlocks() % 2) random_label -- randomly relabel measurements + -- randomly relabel measurements + select *, toUInt32(rowNumberInAllBlocks() % 2) random_label from ( - select time, number virtual_run + select metrics, number virtual_run from -- strip the query away before the join -- it might be several kB long; - (select time, run, version from table) no_query, + (select metrics, run, version from table) no_query, -- duplicate input measurements into many virtual runs numbers(1, 100000) nn -- for each virtual run, randomly reorder measurements @@ -40,19 +45,19 @@ from -- this select aggregates by virtual_run ) rd, ( - select groupArrayInsertAt(median_time, version) time_by_version + select groupArrayInsertAt(median_metrics, version) medians_by_version from ( - select medianExact(time) median_time, version + select medianExactForEach(metrics) median_metrics, version from table group by version ) original_medians ) original_medians_array, ( - select any(query) query from table + select any(test) test, any(query) query from table ) any_query, ( - select throwIf(uniq(query) != 1) from table + select throwIf(uniq((test, query)) != 1) from table ) check_single_query -- this subselect checks that there is only one query in the input table; -- written this way so that it is not optimized away (#10523) ; diff --git a/docker/test/performance-comparison/perf.py b/docker/test/performance-comparison/perf.py index a749b158e83..be8155cdddb 100755 --- a/docker/test/performance-comparison/perf.py +++ b/docker/test/performance-comparison/perf.py @@ -29,6 +29,8 @@ parser.add_argument('--runs', type=int, default=int(os.environ.get('CHPC_RUNS', parser.add_argument('--no-long', type=bool, default=True, help='Skip the tests tagged as long.') args = parser.parse_args() +test_name = os.path.splitext(os.path.basename(args.file[0].name))[0] + tree = et.parse(args.file[0]) root = tree.getroot() @@ -141,19 +143,25 @@ test_queries = substitute_parameters(test_query_templates) report_stage_end('substitute2') -for i, q in enumerate(test_queries): +for query_index, q in enumerate(test_queries): + query_prefix = f'{test_name}.query{query_index}' + # We have some crazy long queries (about 100kB), so trim them to a sane - # length. + # length. This means we can't use query text as an identifier and have to + # use the test name + the test-wide query index. query_display_name = q if len(query_display_name) > 1000: query_display_name = f'{query_display_name[:1000]}...({i})' + print(f'display-name\t{query_index}\t{tsv_escape(query_display_name)}') + # Prewarm: run once on both servers. Helps to bring the data into memory, # precompile the queries, etc. try: for conn_index, c in enumerate(connections): - res = c.execute(q, query_id = f'prewarm {0} {query_display_name}') - print(f'prewarm\t{tsv_escape(query_display_name)}\t{conn_index}\t{c.last_query.elapsed}') + prewarm_id = f'{query_prefix}.prewarm0' + res = c.execute(q, query_id = prewarm_id) + print(f'prewarm\t{query_index}\t{prewarm_id}\t{conn_index}\t{c.last_query.elapsed}') except: # If prewarm fails for some query -- skip it, and try to test the others. # This might happen if the new test introduces some function that the @@ -170,13 +178,14 @@ for i, q in enumerate(test_queries): start_seconds = time.perf_counter() server_seconds = 0 for run in range(0, args.runs): + run_id = f'{query_prefix}.run{run}' for conn_index, c in enumerate(connections): - res = c.execute(q) - print(f'query\t{tsv_escape(query_display_name)}\t{run}\t{conn_index}\t{c.last_query.elapsed}') + res = c.execute(q, query_id = run_id) + print(f'query\t{query_index}\t{run_id}\t{conn_index}\t{c.last_query.elapsed}') server_seconds += c.last_query.elapsed client_seconds = time.perf_counter() - start_seconds - print(f'client-time\t{tsv_escape(query_display_name)}\t{client_seconds}\t{server_seconds}') + print(f'client-time\t{query_index}\t{client_seconds}\t{server_seconds}') report_stage_end('benchmark') diff --git a/docker/test/performance-comparison/report.py b/docker/test/performance-comparison/report.py index a6203ec40e6..83c5e7344f7 100755 --- a/docker/test/performance-comparison/report.py +++ b/docker/test/performance-comparison/report.py @@ -25,6 +25,9 @@ very_unstable_queries = 0 # max seconds to run one query by itself, not counting preparation allowed_single_run_time = 2 +color_bad='#ffb0c0' +color_good='#b0d050' + header_template = """ @@ -188,8 +191,8 @@ if args.report == 'main': print(tableStart('Changes in performance')) columns = [ - 'Old, s.', # 0 - 'New, s.', # 1 + 'Old, s', # 0 + 'New, s', # 1 'Relative difference (new − old) / old', # 2 'p < 0.001 threshold', # 3 # Failed # 4 @@ -205,10 +208,10 @@ if args.report == 'main': if int(row[4]): if float(row[2]) < 0.: faster_queries += 1 - attrs[2] = 'style="background: #00ff00"' + attrs[2] = f'style="background: {color_good}"' else: slower_queries += 1 - attrs[2] = 'style="background: #ff0000"' + attrs[2] = f'style="background: {color_bad}"' else: attrs[2] = '' @@ -221,7 +224,7 @@ if args.report == 'main': slow_on_client_rows = tsvRows('report/slow-on-client.tsv') error_tests += len(slow_on_client_rows) printSimpleTable('Slow on client', - ['Client time, s.', 'Server time, s.', 'Ratio', 'Query'], + ['Client time, s', 'Server time, s', 'Ratio', 'Query'], slow_on_client_rows) def print_unstable_queries(): @@ -252,7 +255,7 @@ if args.report == 'main': for r in unstable_rows: if int(r[4]): very_unstable_queries += 1 - attrs[3] = 'style="background: #ffb0a0"' + attrs[3] = f'style="background: {color_bad}"' else: attrs[3] = '' @@ -266,7 +269,7 @@ if args.report == 'main': error_tests += len(run_error_rows) printSimpleTable('Run errors', ['Test', 'Error'], run_error_rows) - skipped_tests_rows = tsvRows('skipped-tests.tsv') + skipped_tests_rows = tsvRows('analyze/skipped-tests.tsv') printSimpleTable('Skipped tests', ['Test', 'Reason'], skipped_tests_rows) printSimpleTable('Tests with most unstable queries', @@ -281,13 +284,13 @@ if args.report == 'main': columns = [ 'Test', #0 - 'Wall clock time, s.', #1 - 'Total client time, s.', #2 + 'Wall clock time, s', #1 + 'Total client time, s', #2 'Total queries', #3 'Ignored short queries', #4 - 'Longest query
    (sum for all runs), s.', #5 - 'Avg wall clock time
    (sum for all runs), s.', #6 - 'Shortest query
    (sum for all runs), s.', #7 + 'Longest query
    (sum for all runs), s', #5 + 'Avg wall clock time
    (sum for all runs), s', #6 + 'Shortest query
    (sum for all runs), s', #7 ] print(tableStart('Test times')) @@ -300,13 +303,13 @@ if args.report == 'main': if float(r[6]) > 1.5 * total_runs: # FIXME should be 15s max -- investigate parallel_insert slow_average_tests += 1 - attrs[6] = 'style="background: #ffb0a0"' + attrs[6] = f'style="background: {color_bad}"' else: attrs[6] = '' if float(r[5]) > allowed_single_run_time * total_runs: slow_average_tests += 1 - attrs[5] = 'style="background: #ffb0a0"' + attrs[5] = f'style="background: {color_bad}"' else: attrs[5] = '' @@ -320,9 +323,9 @@ if args.report == 'main': print("""

    - Test output All queries Log + Test output

    @@ -382,8 +385,8 @@ elif args.report == 'all-queries': columns = [ # Changed #0 # Unstable #1 - 'Old, s.', #2 - 'New, s.', #3 + 'Old, s', #2 + 'New, s', #3 'Relative difference (new − old) / old', #4 'Times speedup / slowdown', #5 'p < 0.001 threshold', #6 @@ -399,21 +402,21 @@ elif args.report == 'all-queries': attrs[1] = None for r in rows: if int(r[1]): - attrs[6] = 'style="background: #ffb0a0"' + attrs[6] = f'style="background: {color_bad}"' else: attrs[6] = '' if int(r[0]): if float(r[4]) > 0.: - attrs[4] = 'style="background: #ffb0a0"' + attrs[4] = f'style="background: {color_bad}"' else: - attrs[4] = 'style="background: #adbdff"' + attrs[4] = f'style="background: {color_good}"' else: attrs[4] = '' if (float(r[2]) + float(r[3])) / 2 > allowed_single_run_time: - attrs[2] = 'style="background: #ffb0a0"' - attrs[3] = 'style="background: #ffb0a0"' + attrs[2] = f'style="background: {color_bad}"' + attrs[3] = f'style="background: {color_bad}"' else: attrs[2] = '' attrs[3] = '' @@ -428,9 +431,9 @@ elif args.report == 'all-queries': print("""

    - Test output Main report Log + Test output

    From 4f9abad9dd7cc7bde46f44512ff6033be00b7b75 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 21 May 2020 09:15:52 +0300 Subject: [PATCH 498/738] Added .arcignore --- .arcignore | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .arcignore diff --git a/.arcignore b/.arcignore new file mode 100644 index 00000000000..cd917b06fb7 --- /dev/null +++ b/.arcignore @@ -0,0 +1,12 @@ +# .arcignore is the same as .gitignore but for Arc VCS. +# Arc VCS is a proprietary VCS in Yandex that is very similar to Git +# from the user perspective but with the following differences: +# 1. Data is stored in distributed object storage. +# 2. Local copy works via FUSE without downloading all the objects. +# For this reason, it is better suited for huge monorepositories that can be found in large companies (e.g. Yandex, Google). +# As ClickHouse developers, we don't use Arc as a VCS (we use Git). +# But the ClickHouse source code is also mirrored into internal monorepository and our collegues are using Arc. +# You can read more about Arc here: https://habr.com/en/company/yandex/blog/482926/ + +# Repository is synchronized without 3rd-party submodules. +contrib From 8c2b7c31a5a79952b323c6080aba43bae0b6ea8a Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 21 May 2020 09:49:15 +0300 Subject: [PATCH 499/738] Fix too long perf test. --- tests/performance/random_string_utf8.xml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/performance/random_string_utf8.xml b/tests/performance/random_string_utf8.xml index 60e3b69e79f..0185f519c31 100644 --- a/tests/performance/random_string_utf8.xml +++ b/tests/performance/random_string_utf8.xml @@ -3,10 +3,10 @@ SELECT count() FROM zeros(1000000) WHERE NOT ignore(randomStringUTF8(10)) - SELECT count() FROM zeros(1000000) WHERE NOT ignore(randomStringUTF8(100)) - SELECT count() FROM zeros(100000) WHERE NOT ignore(randomStringUTF8(1000)) - SELECT count() FROM zeros(10000) WHERE NOT ignore(randomStringUTF8(10000)) - SELECT count() FROM zeros(10000000) WHERE NOT ignore(randomStringUTF8(rand() % 10)) - SELECT count() FROM zeros(10000000) WHERE NOT ignore(randomStringUTF8(rand() % 100)) - SELECT count() FROM zeros(1000000) WHERE NOT ignore(randomStringUTF8(rand() % 1000)) + SELECT count() FROM zeros(100000) WHERE NOT ignore(randomStringUTF8(100)) + SELECT count() FROM zeros(10000) WHERE NOT ignore(randomStringUTF8(1000)) + SELECT count() FROM zeros(1000) WHERE NOT ignore(randomStringUTF8(10000)) + SELECT count() FROM zeros(1000000) WHERE NOT ignore(randomStringUTF8(rand() % 10)) + SELECT count() FROM zeros(100000) WHERE NOT ignore(randomStringUTF8(rand() % 100)) + SELECT count() FROM zeros(10000) WHERE NOT ignore(randomStringUTF8(rand() % 1000))     From 72ca56c43b6f8ca21837860369ba25cf0fd71a4b Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 21 May 2020 07:28:30 +0000 Subject: [PATCH 500/738] Bump protobuf from 3.12.0 to 3.12.1 in /docs/tools Bumps [protobuf](https://github.com/protocolbuffers/protobuf) from 3.12.0 to 3.12.1. - [Release notes](https://github.com/protocolbuffers/protobuf/releases) - [Changelog](https://github.com/protocolbuffers/protobuf/blob/master/generate_changelog.py) - [Commits](https://github.com/protocolbuffers/protobuf/compare/v3.12.0...v3.12.1) Signed-off-by: dependabot-preview[bot] --- docs/tools/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tools/requirements.txt b/docs/tools/requirements.txt index e4751272ebb..adb68cfb2d7 100644 --- a/docs/tools/requirements.txt +++ b/docs/tools/requirements.txt @@ -21,7 +21,7 @@ mkdocs-htmlproofer-plugin==0.0.3 mkdocs-macros-plugin==0.4.9 nltk==3.5 nose==1.3.7 -protobuf==3.12.0 +protobuf==3.12.1 numpy==1.18.4 Pygments==2.5.2 pymdown-extensions==7.1 From 6efc51335a7a6dfc9da258965558f562b5f52860 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 21 May 2020 10:31:43 +0300 Subject: [PATCH 501/738] Fix build. --- src/Processors/Executors/PipelineExecutor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/Executors/PipelineExecutor.cpp b/src/Processors/Executors/PipelineExecutor.cpp index a5a43a9f546..fb1527e7cfd 100644 --- a/src/Processors/Executors/PipelineExecutor.cpp +++ b/src/Processors/Executors/PipelineExecutor.cpp @@ -716,7 +716,7 @@ void PipelineExecutor::executeStepImpl(size_t thread_num, size_t num_threads, st #ifndef NDEBUG context->total_time_ns += total_time_watch.elapsed(); - context->wait_time_ns = total_time_ns - execution_time_ns - processing_time_ns; + context->wait_time_ns = context->total_time_ns - context->execution_time_ns - context->processing_time_ns; #endif } From f6d5961915991bf22722160a5a004b23c400b17f Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 21 May 2020 10:40:01 +0300 Subject: [PATCH 502/738] Fix build. --- src/Processors/Executors/PipelineExecutor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/Executors/PipelineExecutor.cpp b/src/Processors/Executors/PipelineExecutor.cpp index fb1527e7cfd..4552262040b 100644 --- a/src/Processors/Executors/PipelineExecutor.cpp +++ b/src/Processors/Executors/PipelineExecutor.cpp @@ -502,7 +502,7 @@ bool PipelineExecutor::executeStep(std::atomic_bool * yield_flag) if (!is_execution_initialized) initializeExecution(1); - executeStep(yield_flag); + executeStepImpl(0, 1, yield_flag); if (!finished) return true; From adac47a8452e9f07a6cc4ed16059cae735842c68 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 21 May 2020 10:47:38 +0300 Subject: [PATCH 503/738] Fix build. --- .../Executors/PipelineExecutingBlockInputStream.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Processors/Executors/PipelineExecutingBlockInputStream.cpp b/src/Processors/Executors/PipelineExecutingBlockInputStream.cpp index d39938b57c5..3c127d08c28 100644 --- a/src/Processors/Executors/PipelineExecutingBlockInputStream.cpp +++ b/src/Processors/Executors/PipelineExecutingBlockInputStream.cpp @@ -89,11 +89,14 @@ inline static void throwIfExecutionNotStarted(bool is_execution_started, const c void PipelineExecutingBlockInputStream::cancel(bool kill) { - throwIfExecutionNotStarted(executor != nullptr, "cancel"); IBlockInputStream::cancel(kill); - executor->cancel(); -} + if (is_execution_started) + { + executor ? executor->cancel() + : async_executor->cancel(); + } +} void PipelineExecutingBlockInputStream::setProgressCallback(const ProgressCallback & callback) { From 5da41d5cd6217f7737107226295a6ff01aed8985 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 21 May 2020 10:35:37 +0300 Subject: [PATCH 504/738] Configure jemalloc options via cmake --- contrib/CMakeLists.txt | 1 + contrib/jemalloc-cmake/CMakeLists.txt | 15 +++++++++++++-- .../jemalloc-cmake/include_linux_aarch64/README | 1 + ...nternal_defs.h => jemalloc_internal_defs.h.in} | 2 +- .../jemalloc-cmake/include_linux_x86_64/README | 1 + ...nternal_defs.h => jemalloc_internal_defs.h.in} | 2 +- 6 files changed, 18 insertions(+), 4 deletions(-) rename contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/{jemalloc_internal_defs.h => jemalloc_internal_defs.h.in} (99%) rename contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/{jemalloc_internal_defs.h => jemalloc_internal_defs.h.in} (99%) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 1031285eac7..981a60df70a 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -22,6 +22,7 @@ add_subdirectory (consistent-hashing) add_subdirectory (croaring) add_subdirectory (FastMemcpy) add_subdirectory (grpc-cmake) +set(JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu") add_subdirectory (jemalloc-cmake) add_subdirectory (libcpuid-cmake) add_subdirectory (murmurhash) diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt index 5ae09bec8aa..aa005400bd1 100644 --- a/contrib/jemalloc-cmake/CMakeLists.txt +++ b/contrib/jemalloc-cmake/CMakeLists.txt @@ -9,6 +9,9 @@ if (ENABLE_JEMALLOC) option (USE_INTERNAL_JEMALLOC "Use internal jemalloc library" ${NOT_UNBUNDLED}) if (USE_INTERNAL_JEMALLOC) + option (JEMALLOC_CONFIG_MALLOC_CONF "Change default configuration string" "") + message (STATUS "jemalloc malloc_conf: ${JEMALLOC_CONFIG_MALLOC_CONF}") + set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/jemalloc") set (SRCS @@ -52,11 +55,19 @@ if (ENABLE_JEMALLOC) add_library(jemalloc ${SRCS}) target_include_directories(jemalloc PRIVATE ${LIBRARY_DIR}/include) target_include_directories(jemalloc SYSTEM PUBLIC include) + + set(JEMALLOC_INCLUDE) if (ARCH_AMD64) - target_include_directories(jemalloc SYSTEM PUBLIC include_linux_x86_64) + set(JEMALLOC_INCLUDE_PREFIX include_linux_x86_64) elseif (ARCH_ARM) - target_include_directories(jemalloc SYSTEM PUBLIC include_linux_aarch64) + set(JEMALLOC_INCLUDE_PREFIX include_linux_aarch64) endif () + target_include_directories(jemalloc SYSTEM PUBLIC + ${JEMALLOC_INCLUDE_PREFIX}) + configure_file(${JEMALLOC_INCLUDE_PREFIX}/jemalloc/internal/jemalloc_internal_defs.h.in + ${JEMALLOC_INCLUDE_PREFIX}/jemalloc/internal/jemalloc_internal_defs.h) + target_include_directories(jemalloc SYSTEM PRIVATE + ${CMAKE_CURRENT_BINARY_DIR}/${JEMALLOC_INCLUDE_PREFIX}/jemalloc/internal) target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_NO_PRIVATE_NAMESPACE) diff --git a/contrib/jemalloc-cmake/include_linux_aarch64/README b/contrib/jemalloc-cmake/include_linux_aarch64/README index 2ab582803a2..3cecf7fa36d 100644 --- a/contrib/jemalloc-cmake/include_linux_aarch64/README +++ b/contrib/jemalloc-cmake/include_linux_aarch64/README @@ -5,3 +5,4 @@ Added #define GNU_SOURCE Added JEMALLOC_OVERRIDE___POSIX_MEMALIGN because why not. Removed JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF because it's non standard. Removed JEMALLOC_PURGE_MADVISE_FREE because it's available only from Linux 4.5. +Added JEMALLOC_CONFIG_MALLOC_CONF substitution diff --git a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in similarity index 99% rename from contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h rename to contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in index e989020d651..089f1a773aa 100644 --- a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h +++ b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -369,7 +369,7 @@ /* #undef JEMALLOC_EXPORT */ /* config.malloc_conf options string. */ -#define JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu" +#define JEMALLOC_CONFIG_MALLOC_CONF "@JEMALLOC_CONFIG_MALLOC_CONF@" /* If defined, jemalloc takes the malloc/free/etc. symbol names. */ #define JEMALLOC_IS_MALLOC 1 diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/README b/contrib/jemalloc-cmake/include_linux_x86_64/README index bf7663bda8d..8b93e0d4dcd 100644 --- a/contrib/jemalloc-cmake/include_linux_x86_64/README +++ b/contrib/jemalloc-cmake/include_linux_x86_64/README @@ -5,3 +5,4 @@ Added #define GNU_SOURCE Added JEMALLOC_OVERRIDE___POSIX_MEMALIGN because why not. Removed JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF because it's non standard. Removed JEMALLOC_PURGE_MADVISE_FREE because it's available only from Linux 4.5. +Added JEMALLOC_CONFIG_MALLOC_CONF substitution diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in similarity index 99% rename from contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h rename to contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in index 596dffa8d26..63f7f765023 100644 --- a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h +++ b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -360,7 +360,7 @@ /* #undef JEMALLOC_EXPORT */ /* config.malloc_conf options string. */ -#define JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu" +#define JEMALLOC_CONFIG_MALLOC_CONF "@JEMALLOC_CONFIG_MALLOC_CONF@" /* If defined, jemalloc takes the malloc/free/etc. symbol names. */ #define JEMALLOC_IS_MALLOC 1 From 111fa49df593a29d7b25245e31b1d61ffd093908 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 21 May 2020 10:45:59 +0300 Subject: [PATCH 505/738] Add comment for enabling jemalloc percpu_arena --- contrib/CMakeLists.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 981a60df70a..baf1fb0c254 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -22,6 +22,14 @@ add_subdirectory (consistent-hashing) add_subdirectory (croaring) add_subdirectory (FastMemcpy) add_subdirectory (grpc-cmake) +# ThreadPool select job randomly, and there can be some threads that had been +# performed some memory heavy task before and will be inactive for some time, +# but until it will became active again, the memory will not be freed since by +# default each thread has it's own arena, but there should be not more then +# 4*CPU arenas (see opt.nareans description). +# +# By enabling percpu_arena number of arenas limited to number of CPUs and hence +# this problem should go away. set(JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu") add_subdirectory (jemalloc-cmake) add_subdirectory (libcpuid-cmake) From cfef21be866e7d25a89f9c94ddfe2864e86a94ab Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 21 May 2020 11:54:41 +0300 Subject: [PATCH 506/738] Fix build. --- .../Executors/PipelineExecutingBlockInputStream.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/Processors/Executors/PipelineExecutingBlockInputStream.cpp b/src/Processors/Executors/PipelineExecutingBlockInputStream.cpp index 3c127d08c28..ba42d858d50 100644 --- a/src/Processors/Executors/PipelineExecutingBlockInputStream.cpp +++ b/src/Processors/Executors/PipelineExecutingBlockInputStream.cpp @@ -79,14 +79,6 @@ inline static void throwIfExecutionStarted(bool is_execution_started, const char ErrorCodes::LOGICAL_ERROR); } -inline static void throwIfExecutionNotStarted(bool is_execution_started, const char * method) -{ - if (!is_execution_started) - throw Exception(String("Cannot call ") + method + - " for PipelineExecutingBlockInputStream because execution was not started", - ErrorCodes::LOGICAL_ERROR); -} - void PipelineExecutingBlockInputStream::cancel(bool kill) { IBlockInputStream::cancel(kill); From c9d315654c4b55825cea847cc9a7319e0cebf7e2 Mon Sep 17 00:00:00 2001 From: BohuTANG Date: Thu, 21 May 2020 17:00:44 +0800 Subject: [PATCH 507/738] Fix Storages/MergeTree typo --- src/Storages/MergeTree/IMergeTreeDataPart.h | 4 ++-- src/Storages/MergeTree/IMergeTreeDataPartWriter.h | 2 +- src/Storages/MergeTree/MergeTreeData.h | 2 +- src/Storages/MergeTree/MergeTreeDataMergerMutator.h | 2 +- src/Storages/MergeTree/MergeTreeDataPartCompact.h | 4 ++-- src/Storages/MergeTree/MergeTreeSettings.h | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index f0ae8b40c5b..4b2d6102b2f 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -319,7 +319,7 @@ protected: /// checksums.txt and columns.txt. 0 - if not counted; UInt64 bytes_on_disk{0}; - /// Columns description. Cannot be changed, after part initialiation. + /// Columns description. Cannot be changed, after part initialization. NamesAndTypesList columns; const Type part_type; @@ -352,7 +352,7 @@ private: /// For the older format version calculates rows count from the size of a column with a fixed size. void loadRowsCount(); - /// Loads ttl infos in json format from file ttl.txt. If file doesn`t exists assigns ttl infos with all zeros + /// Loads ttl infos in json format from file ttl.txt. If file doesn't exists assigns ttl infos with all zeros void loadTTLInfos(); void loadPartitionAndMinMaxIndex(); diff --git a/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index ffdba570544..149aeaa2f0d 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -136,7 +136,7 @@ protected: size_t next_mark = 0; size_t next_index_offset = 0; - /// Number of marsk in data from which skip indices have to start + /// Number of marks in data from which skip indices have to start /// aggregation. I.e. it's data mark number, not skip indices mark. size_t skip_index_data_mark = 0; diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 394260096a7..852b3b68caf 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -70,7 +70,7 @@ namespace ErrorCodes /// [Column].mrk - marks, pointing to seek positions allowing to skip n * k rows. /// /// File structure of tables with custom partitioning (format_version >= 1): -/// Part directory - / partiiton-id _ min-id _ max-id _ level / +/// Part directory - / partition-id _ min-id _ max-id _ level / /// Inside the part directory: /// The same files as for month-partitioned tables, plus /// count.txt - contains total number of rows in this part. diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.h b/src/Storages/MergeTree/MergeTreeDataMergerMutator.h index 431d059ba60..84a3fcf1be2 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.h +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.h @@ -160,7 +160,7 @@ private: NamesAndTypesList storage_columns, const MutationCommands & commands_for_removes); - /// Get skip indcies, that should exists in the resulting data part. + /// Get skip indices, that should exists in the resulting data part. static MergeTreeIndices getIndicesForNewDataPart( const MergeTreeIndices & all_indices, const MutationCommands & commands_for_removes); diff --git a/src/Storages/MergeTree/MergeTreeDataPartCompact.h b/src/Storages/MergeTree/MergeTreeDataPartCompact.h index 3ce1fd830d5..b4a2b5fa797 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartCompact.h +++ b/src/Storages/MergeTree/MergeTreeDataPartCompact.h @@ -6,13 +6,13 @@ namespace DB { /** In compact format all columns are stored in one file (`data.bin`). - * Data is splitted in granules and columns are serialized sequentially in one granule. + * Data is split in granules and columns are serialized sequentially in one granule. * Granules are written one by one in data file. * Marks are also stored in single file (`data.mrk3`). * In compact format one mark is an array of marks for every column and a number of rows in granule. * Format of other data part files is not changed. * It's considered to store only small parts in compact format (up to 10M). - * NOTE: Compact parts aren't supported for tables with non-adaptive granularty. + * NOTE: Compact parts aren't supported for tables with non-adaptive granularity. * NOTE: In compact part compressed and uncompressed size of single column is unknown. */ class MergeTreeDataPartCompact : public IMergeTreeDataPart diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index dfe3643e60c..f29be3d692f 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -28,7 +28,7 @@ struct MergeTreeSettings : public SettingsCollection #define LIST_OF_MERGE_TREE_SETTINGS(M) \ M(SettingUInt64, index_granularity, 8192, "How many rows correspond to one primary key value.", 0) \ \ - /** Data storing format settigns. */ \ + /** Data storing format settings. */ \ M(SettingUInt64, min_bytes_for_wide_part, 0, "Minimal uncompressed size in bytes to create part in wide format instead of compact", 0) \ M(SettingUInt64, min_rows_for_wide_part, 0, "Minimal number of rows to create part in wide format instead of compact", 0) \ \ From 37e5da737cfd51b10ab11715e23a01e1e5697e73 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 21 May 2020 12:27:55 +0300 Subject: [PATCH 508/738] Fix ya.make. --- src/Processors/ya.make | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Processors/ya.make b/src/Processors/ya.make index 4412aa4748c..a17cab9c0a1 100644 --- a/src/Processors/ya.make +++ b/src/Processors/ya.make @@ -59,6 +59,7 @@ SRCS( Formats/IRowOutputFormat.cpp Formats/LazyOutputFormat.cpp Formats/OutputStreamToOutputFormat.cpp + Formats/PullingOutputFormat.cpp Formats/RowInputFormatWithDiagnosticInfo.cpp IAccumulatingTransform.cpp IInflatingTransform.cpp From 2fa264d83edf4a1453a64a77d73c0be48638b72e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 21 May 2020 12:45:44 +0300 Subject: [PATCH 509/738] Fix style #10591 --- src/Storages/System/StorageSystemParts.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Storages/System/StorageSystemParts.cpp b/src/Storages/System/StorageSystemParts.cpp index 745d2938d86..83e3c34bfb8 100644 --- a/src/Storages/System/StorageSystemParts.cpp +++ b/src/Storages/System/StorageSystemParts.cpp @@ -145,7 +145,9 @@ void StorageSystemParts::processNextStorage(MutableColumns & columns_, const Sto /// move_ttl_info { - Array expression_array, min_array, max_array; + Array expression_array; + Array min_array; + Array max_array; expression_array.reserve(part->ttl_infos.moves_ttl.size()); min_array.reserve(part->ttl_infos.moves_ttl.size()); max_array.reserve(part->ttl_infos.moves_ttl.size()); From 7e4cd8f8d4d3c093333adef8b44d49a5a5215e8d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 21 May 2020 12:55:39 +0300 Subject: [PATCH 510/738] Test for TTL info in system.parts --- .../0_stateless/01282_system_parts_ttl_info.reference | 2 ++ .../queries/0_stateless/01282_system_parts_ttl_info.sql | 9 +++++++++ 2 files changed, 11 insertions(+) create mode 100644 tests/queries/0_stateless/01282_system_parts_ttl_info.reference create mode 100644 tests/queries/0_stateless/01282_system_parts_ttl_info.sql diff --git a/tests/queries/0_stateless/01282_system_parts_ttl_info.reference b/tests/queries/0_stateless/01282_system_parts_ttl_info.reference new file mode 100644 index 00000000000..1d9fe9eeb36 --- /dev/null +++ b/tests/queries/0_stateless/01282_system_parts_ttl_info.reference @@ -0,0 +1,2 @@ +2 2000-01-11 01:02:03 2000-02-13 04:05:06 [] [] [] +0 0000-00-00 00:00:00 0000-00-00 00:00:00 [] [] [] diff --git a/tests/queries/0_stateless/01282_system_parts_ttl_info.sql b/tests/queries/0_stateless/01282_system_parts_ttl_info.sql new file mode 100644 index 00000000000..3a1b1cc79ce --- /dev/null +++ b/tests/queries/0_stateless/01282_system_parts_ttl_info.sql @@ -0,0 +1,9 @@ +DROP TABLE IF EXISTS ttl; +CREATE TABLE ttl (d DateTime) ENGINE = MergeTree ORDER BY tuple() TTL d + INTERVAL 10 DAY; +SYSTEM STOP MERGES; +INSERT INTO ttl VALUES ('2000-01-01 01:02:03'), ('2000-02-03 04:05:06'); +SELECT rows, delete_ttl_info_min, delete_ttl_info_max, move_ttl_info.expression, move_ttl_info.min, move_ttl_info.max FROM system.parts WHERE database = currentDatabase() AND table = 'ttl'; +SYSTEM START MERGES; +OPTIMIZE TABLE ttl FINAL; +SELECT rows, delete_ttl_info_min, delete_ttl_info_max, move_ttl_info.expression, move_ttl_info.min, move_ttl_info.max FROM system.parts WHERE database = currentDatabase() AND table = 'ttl' AND active; +DROP TABLE ttl; From c816916fba75311ae5f30b84ef86c09c6fd4d1e8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 21 May 2020 13:46:36 +0300 Subject: [PATCH 511/738] Fix typo --- .../0_stateless/01281_group_by_limit_memory_tracking.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sql b/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sql index fc8f81975f0..e2f1671acb9 100644 --- a/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sql +++ b/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sql @@ -9,7 +9,7 @@ CREATE MATERIALIZED VIEW trace_log_01281_mv TO trace_log_01281 AS SELECT * FROM CREATE VIEW trace_log_01281_assert AS SELECT *, throwIf(cnt < 0, 'no memory profile captured'), - throwIf(queries != 1, 'too much queries'), + throwIf(queries != 1, 'too many queries'), throwIf(alloc < 100e6, 'minimal allocation had not been done'), throwIf((alloc+free)/alloc > 0.05, 'memory accounting leaked more than 5%') FROM ( From 1fca0294fe9769b997b4aa1afd06d2b999a940fd Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 21 May 2020 13:48:48 +0300 Subject: [PATCH 512/738] Fix logical error in test --- .../0_stateless/01281_group_by_limit_memory_tracking.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sql b/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sql index e2f1671acb9..18d9988551f 100644 --- a/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sql +++ b/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sql @@ -9,7 +9,7 @@ CREATE MATERIALIZED VIEW trace_log_01281_mv TO trace_log_01281 AS SELECT * FROM CREATE VIEW trace_log_01281_assert AS SELECT *, throwIf(cnt < 0, 'no memory profile captured'), - throwIf(queries != 1, 'too many queries'), + throwIf(queries > 1, 'too many queries'), throwIf(alloc < 100e6, 'minimal allocation had not been done'), throwIf((alloc+free)/alloc > 0.05, 'memory accounting leaked more than 5%') FROM ( From 886266fdad160c04b441cdb74603523bcacb871a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 21 May 2020 13:57:24 +0300 Subject: [PATCH 513/738] Attempt to fix an error --- .../0_stateless/01281_group_by_limit_memory_tracking.sql | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sql b/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sql index 18d9988551f..b9e467ba301 100644 --- a/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sql +++ b/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sql @@ -38,6 +38,7 @@ SYSTEM FLUSH LOGS; -- } SELECT * FROM trace_log_01281_assert FORMAT Null; +SYSTEM FLUSH LOGS; TRUNCATE TABLE trace_log_01281; -- single limit { SET max_threads=1; @@ -48,6 +49,7 @@ SYSTEM FLUSH LOGS; -- } SELECT * FROM trace_log_01281_assert FORMAT Null; +SYSTEM FLUSH LOGS; TRUNCATE TABLE trace_log_01281; -- two-level { -- need to have multiple threads for two-level aggregation @@ -59,6 +61,7 @@ SYSTEM FLUSH LOGS; -- } SELECT * FROM trace_log_01281_assert FORMAT Null; +SYSTEM FLUSH LOGS; TRUNCATE TABLE trace_log_01281; -- two-level limit { -- need to have multiple threads for two-level aggregation @@ -70,6 +73,7 @@ SYSTEM FLUSH LOGS; -- } SELECT * FROM trace_log_01281_assert FORMAT Null; +SYSTEM FLUSH LOGS; TRUNCATE TABLE trace_log_01281; -- two-level MEMORY_LIMIT_EXCEEDED { -- need to have multiple threads for two-level aggregation From 80c7e85f38fc815cd952ffe709b41cd7162c282d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 21 May 2020 13:58:05 +0300 Subject: [PATCH 514/738] Disable potentially unreliable check --- .../0_stateless/01281_group_by_limit_memory_tracking.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sql b/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sql index b9e467ba301..fa606898fe8 100644 --- a/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sql +++ b/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sql @@ -9,7 +9,7 @@ CREATE MATERIALIZED VIEW trace_log_01281_mv TO trace_log_01281 AS SELECT * FROM CREATE VIEW trace_log_01281_assert AS SELECT *, throwIf(cnt < 0, 'no memory profile captured'), - throwIf(queries > 1, 'too many queries'), +-- this check is disabled because it's not reliable: throwIf(queries > 1, 'too many queries'), throwIf(alloc < 100e6, 'minimal allocation had not been done'), throwIf((alloc+free)/alloc > 0.05, 'memory accounting leaked more than 5%') FROM ( From cce805c42cbe742f2fe745913f54d0e3fe9eb1bd Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Thu, 21 May 2020 14:50:12 +0300 Subject: [PATCH 515/738] Revert "Allow to build binaries and tests from deb package build. " --- debian/rules | 11 +---------- docker/packager/deb/build.sh | 11 ----------- docker/packager/packager | 19 ++----------------- programs/CMakeLists.txt | 6 ------ 4 files changed, 3 insertions(+), 44 deletions(-) diff --git a/debian/rules b/debian/rules index 7218e196baa..dabebb516cd 100755 --- a/debian/rules +++ b/debian/rules @@ -24,10 +24,6 @@ DEB_BUILD_OPTIONS+=parallel=$(THREADS_COUNT) ifndef ENABLE_TESTS CMAKE_FLAGS += -DENABLE_TESTS=0 -else -# To export binaries and from deb build we do not strip them. No need to run tests in deb build as we run them in CI - DEB_BUILD_OPTIONS+= nocheck - DEB_BUILD_OPTIONS+= nostrip endif ifndef MAKE_TARGET @@ -92,19 +88,14 @@ override_dh_auto_build: $(MAKE) $(THREADS_COUNT) -C $(BUILDDIR) $(MAKE_TARGET) override_dh_auto_test: -ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS))) cd $(BUILDDIR) && ctest $(THREADS_COUNT) -V -E with_server -endif override_dh_clean: rm -rf debian/copyright debian/clickhouse-client.docs debian/clickhouse-common-static.docs - dh_clean # -X contrib + dh_clean -X contrib override_dh_strip: -#https://www.debian.org/doc/debian-policy/ch-source.html#debian-rules-and-deb-build-options -ifeq (,$(filter nostrip,$(DEB_BUILD_OPTIONS))) dh_strip -pclickhouse-common-static --dbg-package=clickhouse-common-static-dbg -endif override_dh_install: # Making docs diff --git a/docker/packager/deb/build.sh b/docker/packager/deb/build.sh index fbaa0151c6b..1efed3628a0 100755 --- a/docker/packager/deb/build.sh +++ b/docker/packager/deb/build.sh @@ -10,16 +10,5 @@ mv *.changes /output mv *.buildinfo /output mv /*.rpm /output ||: # if exists mv /*.tgz /output ||: # if exists - -if [ -n "$BINARY_OUTPUT" ] && { [ "$BINARY_OUTPUT" = "programs" ] || [ "$BINARY_OUTPUT" = "tests" ] ;} -then - echo Place $BINARY_OUTPUT to output - mkdir /output/binary ||: # if exists - mv /build/obj-*/programs/clickhouse* /output/binary - if [ "$BINARY_OUTPUT" = "tests" ] - then - mv /build/obj-*/src/unit_tests_dbms /output/binary - fi -fi ccache --show-stats ||: ln -s /usr/lib/x86_64-linux-gnu/libOpenCL.so.1.0.0 /usr/lib/libOpenCL.so ||: diff --git a/docker/packager/packager b/docker/packager/packager index 8a5bdda60e8..025ca3bf398 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -54,7 +54,7 @@ def run_docker_image_with_env(image_name, output, env_variables, ch_root, ccache subprocess.check_call(cmd, shell=True) -def parse_env_variables(build_type, compiler, sanitizer, package_type, image_type, cache, distcc_hosts, unbundled, split_binary, clang_tidy, version, author, official, alien_pkgs, with_coverage, with_binaries): +def parse_env_variables(build_type, compiler, sanitizer, package_type, image_type, cache, distcc_hosts, unbundled, split_binary, clang_tidy, version, author, official, alien_pkgs, with_coverage): CLANG_PREFIX = "clang" DARWIN_SUFFIX = "-darwin" ARM_SUFFIX = "-aarch64" @@ -131,14 +131,6 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ if alien_pkgs: result.append("ALIEN_PKGS='" + ' '.join(['--' + pkg for pkg in alien_pkgs]) + "'") - if with_binaries == "programs": - result.append('BINARY_OUTPUT=programs') - elif with_binaries == "tests": - result.append('ENABLE_TESTS=1') - result.append('BINARY_OUTPUT=tests') - cmake_flags.append('-DENABLE_TESTS=1') - cmake_flags.append('-DUSE_GTEST=1') - if unbundled: # TODO: fix build with ENABLE_RDKAFKA cmake_flags.append('-DUNBUNDLED=1 -DENABLE_MYSQL=0 -DENABLE_ODBC=0 -DENABLE_REPLXX=0 -DENABLE_RDKAFKA=0') @@ -187,7 +179,6 @@ if __name__ == "__main__": parser.add_argument("--official", action="store_true") parser.add_argument("--alien-pkgs", nargs='+', default=[]) parser.add_argument("--with-coverage", action="store_true") - parser.add_argument("--with-binaries", choices=("programs", "tests", ""), default="") args = parser.parse_args() if not os.path.isabs(args.output_dir): @@ -204,12 +195,6 @@ if __name__ == "__main__": if args.alien_pkgs and not image_type == "deb": raise Exception("Can add alien packages only in deb build") - if args.with_binaries != "" and not image_type == "deb": - raise Exception("Can add additional binaries only in deb build") - - if args.with_binaries != "" and image_type == "deb": - logging.info("Should place {} to output".format(args.with_binaries)) - dockerfile = os.path.join(ch_root, "docker/packager", image_type, "Dockerfile") if image_type != "freebsd" and not check_image_exists_locally(image_name) or args.force_build_image: if not pull_image(image_name) or args.force_build_image: @@ -217,6 +202,6 @@ if __name__ == "__main__": env_prepared = parse_env_variables( args.build_type, args.compiler, args.sanitizer, args.package_type, image_type, args.cache, args.distcc_hosts, args.unbundled, args.split_binary, args.clang_tidy, - args.version, args.author, args.official, args.alien_pkgs, args.with_coverage, args.with_binaries) + args.version, args.author, args.official, args.alien_pkgs, args.with_coverage) run_docker_image_with_env(image_name, args.output_dir, env_prepared, ch_root, args.ccache_dir) logging.info("Output placed into {}".format(args.output_dir)) diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index 7bc31452aa4..7cbe2e7a2a6 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -201,9 +201,3 @@ endif () if (TARGET clickhouse-server AND TARGET copy-headers) add_dependencies(clickhouse-server copy-headers) endif () - -if (ENABLE_TESTS AND USE_GTEST) - set (CLICKHOUSE_ALL_TESTS_TARGETS local_date_time_comparison unit_tests_libcommon unit_tests_dbms hashing_write_buffer hashing_read_buffer in_join_subqueries_preprocessor expression_analyzer) - add_custom_target (clickhouse-tests ALL DEPENDS ${CLICKHOUSE_ALL_TESTS_TARGETS}) - add_dependencies(clickhouse-bundle clickhouse-tests) -endif() From d7b1615ac4edb5325a8f300824c670388176dc4b Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 21 May 2020 14:52:28 +0300 Subject: [PATCH 516/738] Update extended-roadmap.md --- docs/ru/whats-new/extended-roadmap.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/ru/whats-new/extended-roadmap.md b/docs/ru/whats-new/extended-roadmap.md index 9c640504aec..b7756f7fb79 100644 --- a/docs/ru/whats-new/extended-roadmap.md +++ b/docs/ru/whats-new/extended-roadmap.md @@ -174,7 +174,7 @@ Upd. Всё ещё ждём удаление старого кода, котор ### 2.3. Перенос столбцового ser/de из DataType в Column {#perenos-stolbtsovogo-serde-iz-datatype-v-column} -В очереди. +В очереди. Антон Попов. ### 2.4. Перевод LowCardinality из DataType в Column. Добавление ColumnSparse {#perevod-lowcardinality-iz-datatype-v-column-dobavlenie-columnsparse} @@ -977,10 +977,10 @@ Q2. [Виталий Баранов](https://github.com/vitlibar) и Денис Глазачев, Altinity. Требует 12.1. -### 12.6. Информация о пользователях и квотах в системной таблице {#informatsiia-o-polzovateliakh-i-kvotakh-v-sistemnoi-tablitse} +### 12.6. + Информация о пользователях и квотах в системной таблице {#informatsiia-o-polzovateliakh-i-kvotakh-v-sistemnoi-tablitse} [Виталий Баранов](https://github.com/vitlibar). Требует 12.1. -Есть pull request. Q2. +Есть pull request. Q2. Готово. ## 13. Разделение ресурсов, multi-tenancy {#razdelenie-resursov-multi-tenancy} From d6ce1ed7b931115c53f066e2c423b98bdd78eef3 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 21 May 2020 14:54:26 +0300 Subject: [PATCH 517/738] Added comments. --- src/Processors/Executors/PipelineExecutingBlockInputStream.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Processors/Executors/PipelineExecutingBlockInputStream.h b/src/Processors/Executors/PipelineExecutingBlockInputStream.h index 11ccb71f17a..e2e6eda0e24 100644 --- a/src/Processors/Executors/PipelineExecutingBlockInputStream.h +++ b/src/Processors/Executors/PipelineExecutingBlockInputStream.h @@ -34,8 +34,9 @@ protected: private: std::unique_ptr pipeline; - std::unique_ptr executor; - std::unique_ptr async_executor; + /// One of executors is used. + std::unique_ptr executor; /// for singe thread. + std::unique_ptr async_executor; /// for many threads. bool is_execution_started = false; void createExecutor(); From 95668def24d8cbc2b5e6ee5b06d8ed58d2e6b7eb Mon Sep 17 00:00:00 2001 From: g-arslan Date: Sun, 17 May 2020 17:22:17 +0300 Subject: [PATCH 518/738] Try to remove redundant data move in radix sort --- src/AggregateFunctions/QuantileTDigest.h | 2 ++ src/Columns/ColumnVector.cpp | 46 ++++++++---------------- src/Common/RadixSort.h | 36 ++++++++++++++++--- 3 files changed, 48 insertions(+), 36 deletions(-) diff --git a/src/AggregateFunctions/QuantileTDigest.h b/src/AggregateFunctions/QuantileTDigest.h index 91211d25173..ea9b2c681b4 100644 --- a/src/AggregateFunctions/QuantileTDigest.h +++ b/src/AggregateFunctions/QuantileTDigest.h @@ -103,6 +103,7 @@ class QuantileTDigest struct RadixSortTraits { using Element = Centroid; + using Index = Element; using Key = Value; using CountType = UInt32; using KeyBits = UInt32; @@ -114,6 +115,7 @@ class QuantileTDigest /// The function to get the key from an array element. static Key & extractKey(Element & elem) { return elem.mean; } + static Index & extractIndex(Element & elem) { return elem; } }; /** Adds a centroid `c` to the digest diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 74f1438de14..8e26a204cb3 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -117,10 +117,19 @@ namespace struct RadixSortTraits : RadixSortNumTraits { using Element = ValueWithIndex; + using Index = size_t; static T & extractKey(Element & elem) { return elem.value; } + static size_t extractIndex(Element & elem) { return elem.index; } }; } +template +void inplacePartialSwapHelper(T& arr, size_t bound, bool reverse) { + std::reverse(std::begin(arr), std::begin(arr) + bound); + std::reverse(std::begin(arr) + bound, std::end(arr)); + if (!reverse) std::reverse(std::begin(arr), std::end(arr)); +} + template void ColumnVector::getSpecialPermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, IColumn::SpecialSort special_sort) const @@ -179,7 +188,7 @@ void ColumnVector::getPermutation(bool reverse, size_t limit, int nan_directi for (UInt32 i = 0; i < UInt32(s); ++i) pairs[i] = {data[i], i}; - RadixSort>::executeLSD(pairs.data(), s); + RadixSort>::executeLSD(pairs.data(), s, res.data()); /// Radix sort treats all NaNs to be greater than all numbers. /// If the user needs the opposite, we must move them accordingly. @@ -188,42 +197,17 @@ void ColumnVector::getPermutation(bool reverse, size_t limit, int nan_directi { for (ssize_t i = s - 1; i >= 0; --i) { - if (isNaN(pairs[i].value)) + if (isNaN(data[res[i]])) ++nans_to_move; else break; } } - if (reverse) - { - if (nans_to_move) - { - for (size_t i = 0; i < s - nans_to_move; ++i) - res[i] = pairs[s - nans_to_move - 1 - i].index; - for (size_t i = s - nans_to_move; i < s; ++i) - res[i] = pairs[s - 1 - (i - (s - nans_to_move))].index; - } - else - { - for (size_t i = 0; i < s; ++i) - res[s - 1 - i] = pairs[i].index; - } - } - else - { - if (nans_to_move) - { - for (size_t i = 0; i < nans_to_move; ++i) - res[i] = pairs[i + s - nans_to_move].index; - for (size_t i = nans_to_move; i < s; ++i) - res[i] = pairs[i - nans_to_move].index; - } - else - { - for (size_t i = 0; i < s; ++i) - res[i] = pairs[i].index; - } + if (nans_to_move) { + inplacePartialSwapHelper(res, s - nans_to_move, reverse); + } else if (reverse) { + std::reverse(std::begin(res), std::end(res)); } return; diff --git a/src/Common/RadixSort.h b/src/Common/RadixSort.h index 0eeb1e086b4..914384ff607 100644 --- a/src/Common/RadixSort.h +++ b/src/Common/RadixSort.h @@ -71,6 +71,7 @@ template struct RadixSortFloatTraits { using Element = TElement; /// The type of the element. It can be a structure with a key and some other payload. Or just a key. + using Index = Element; /// The index type to store permutation if needed using Key = Element; /// The key to sort by. using CountType = uint32_t; /// Type for calculating histograms. In the case of a known small number of elements, it can be less than size_t. @@ -90,6 +91,9 @@ struct RadixSortFloatTraits /// The function to get the key from an array element. static Key & extractKey(Element & elem) { return elem; } + /// The function to get the index from an array. + static Index & extractIndex(Element & elem) { return elem; } + /// Used when fallback to comparison based sorting is needed. /// TODO: Correct handling of NaNs, NULLs, etc static bool less(Key x, Key y) @@ -113,6 +117,7 @@ template struct RadixSortUIntTraits { using Element = TElement; + using Index = Element; using Key = Element; using CountType = uint32_t; using KeyBits = Key; @@ -123,6 +128,7 @@ struct RadixSortUIntTraits using Allocator = RadixSortMallocAllocator; static Key & extractKey(Element & elem) { return elem; } + static Index & extractIndex(Element & elem) { return elem; } static bool less(Key x, Key y) { @@ -145,6 +151,7 @@ template struct RadixSortIntTraits { using Element = TElement; + using Index = Element; using Key = Element; using CountType = uint32_t; using KeyBits = std::make_unsigned_t; @@ -155,6 +162,7 @@ struct RadixSortIntTraits using Allocator = RadixSortMallocAllocator; static Key & extractKey(Element & elem) { return elem; } + static Index & extractIndex(Element & elem) { return elem; } static bool less(Key x, Key y) { @@ -175,6 +183,7 @@ struct RadixSort { private: using Element = typename Traits::Element; + using Index = typename Traits::Index; using Key = typename Traits::Key; using CountType = typename Traits::CountType; using KeyBits = typename Traits::KeyBits; @@ -304,7 +313,7 @@ private: public: /// Least significant digit radix sort (stable) - static void executeLSD(Element * arr, size_t size) + static void executeLSD(Element * arr, size_t size, Index * destination = NULL) { /// If the array is smaller than 256, then it is better to use another algorithm. @@ -344,8 +353,10 @@ public: } } + bool direct_copy_to_destination = (destination); + /// Move the elements in the order starting from the least bit piece, and then do a few passes on the number of pieces. - for (size_t pass = 0; pass < NUM_PASSES; ++pass) + for (size_t pass = 0; pass < NUM_PASSES - direct_copy_to_destination; ++pass) { Element * writer = pass % 2 ? arr : swap_buffer; Element * reader = pass % 2 ? swap_buffer : arr; @@ -364,10 +375,25 @@ public: } } - /// If the number of passes is odd, the result array is in a temporary buffer. Copy it to the place of the original array. - /// NOTE Sometimes it will be more optimal to provide non-destructive interface, that will not modify original array. - if (NUM_PASSES % 2) + if (direct_copy_to_destination) + { + size_t pass = NUM_PASSES - 1; + Index * writer = destination; + Element * reader = pass % 2 ? swap_buffer : arr; + + for (size_t i = 0; i < size; ++i) + { + size_t pos = getPart(pass, keyToBits(Traits::extractKey(reader[i]))); + + /// Place the element on the next free position. + writer[++histograms[pass * HISTOGRAM_SIZE + pos]] = Traits::extractIndex(reader[i]); + } + } else if (NUM_PASSES % 2) + { + /// If the number of passes is odd, the result array is in a temporary buffer. Copy it to the place of the original array. + /// NOTE Sometimes it will be more optimal to provide non-destructive interface, that will not modify original array. memcpy(arr, swap_buffer, size * sizeof(Element)); + } allocator.deallocate(swap_buffer, size * sizeof(Element)); } From c01a7ef0e25049b7958d6828d60243814f40de43 Mon Sep 17 00:00:00 2001 From: g-arslan Date: Mon, 18 May 2020 16:13:59 +0300 Subject: [PATCH 519/738] Implement reverse in radix sort, fix style, fix build --- src/Columns/ColumnVector.cpp | 24 ++++++++++++------------ src/Common/RadixSort.h | 10 ++++++++-- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 8e26a204cb3..9cb9cedb86c 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -124,10 +124,11 @@ namespace } template -void inplacePartialSwapHelper(T& arr, size_t bound, bool reverse) { +void inplacePartialSwapHelper(T& arr, size_t bound) +{ std::reverse(std::begin(arr), std::begin(arr) + bound); std::reverse(std::begin(arr) + bound, std::end(arr)); - if (!reverse) std::reverse(std::begin(arr), std::end(arr)); + std::reverse(std::begin(arr), std::end(arr)); } template @@ -188,28 +189,27 @@ void ColumnVector::getPermutation(bool reverse, size_t limit, int nan_directi for (UInt32 i = 0; i < UInt32(s); ++i) pairs[i] = {data[i], i}; - RadixSort>::executeLSD(pairs.data(), s, res.data()); + RadixSort>::executeLSD(pairs.data(), s, reverse, res.data()); /// Radix sort treats all NaNs to be greater than all numbers. /// If the user needs the opposite, we must move them accordingly. - size_t nans_to_move = 0; if (std::is_floating_point_v && nan_direction_hint < 0) { - for (ssize_t i = s - 1; i >= 0; --i) + size_t nans_to_move = 0; + + for (size_t i = 0; i < s; ++i) { - if (isNaN(data[res[i]])) + if (isNaN(data[res[reverse ? i : s - 1 - i]])) ++nans_to_move; else break; } - } - if (nans_to_move) { - inplacePartialSwapHelper(res, s - nans_to_move, reverse); - } else if (reverse) { - std::reverse(std::begin(res), std::end(res)); + if (nans_to_move) + { + inplacePartialSwapHelper(res, reverse ? nans_to_move : s - nans_to_move); + } } - return; } } diff --git a/src/Common/RadixSort.h b/src/Common/RadixSort.h index 914384ff607..9953e3253cf 100644 --- a/src/Common/RadixSort.h +++ b/src/Common/RadixSort.h @@ -313,7 +313,7 @@ private: public: /// Least significant digit radix sort (stable) - static void executeLSD(Element * arr, size_t size, Index * destination = NULL) + static void executeLSD(Element * arr, size_t size, bool reverse = false, Index * destination = nullptr) { /// If the array is smaller than 256, then it is better to use another algorithm. @@ -386,13 +386,19 @@ public: size_t pos = getPart(pass, keyToBits(Traits::extractKey(reader[i]))); /// Place the element on the next free position. - writer[++histograms[pass * HISTOGRAM_SIZE + pos]] = Traits::extractIndex(reader[i]); + if (reverse) + writer[size - 1 - (++histograms[pass * HISTOGRAM_SIZE + pos])] = Traits::extractIndex(reader[i]); + else + writer[++histograms[pass * HISTOGRAM_SIZE + pos]] = Traits::extractIndex(reader[i]); } } else if (NUM_PASSES % 2) { /// If the number of passes is odd, the result array is in a temporary buffer. Copy it to the place of the original array. /// NOTE Sometimes it will be more optimal to provide non-destructive interface, that will not modify original array. memcpy(arr, swap_buffer, size * sizeof(Element)); + } else if (reverse) + { + std::reverse(arr, arr + size); } allocator.deallocate(swap_buffer, size * sizeof(Element)); From b88d0dec1c625c4b20e5a6a99f7f626c80d41420 Mon Sep 17 00:00:00 2001 From: g-arslan Date: Thu, 21 May 2020 14:35:54 +0300 Subject: [PATCH 520/738] some little improvements --- src/Columns/ColumnVector.cpp | 19 ++++++++++++++----- src/Common/RadixSort.h | 19 ++++++++++++++----- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 9cb9cedb86c..cf6d9cf030b 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #if !defined(ARCADIA_BUILD) # include @@ -124,11 +125,19 @@ namespace } template -void inplacePartialSwapHelper(T& arr, size_t bound) +void shiftArrayRight(T& arr, size_t bound) { - std::reverse(std::begin(arr), std::begin(arr) + bound); - std::reverse(std::begin(arr) + bound, std::end(arr)); - std::reverse(std::begin(arr), std::end(arr)); + bound %= arr.size(); + size_t gcd = std::gcd(arr.size(), bound); + for (size_t i = 0; i < gcd; ++i) { + size_t j = i; + do { + size_t next = j + bound; + if (next >= arr.size()) next -= arr.size(); + std::swap(arr[j], arr[next]); + j = next; + } while (j != i); + } } template @@ -207,7 +216,7 @@ void ColumnVector::getPermutation(bool reverse, size_t limit, int nan_directi if (nans_to_move) { - inplacePartialSwapHelper(res, reverse ? nans_to_move : s - nans_to_move); + shiftArrayRight(res, reverse ? s - nans_to_move : nans_to_move); } } return; diff --git a/src/Common/RadixSort.h b/src/Common/RadixSort.h index 9953e3253cf..88ddc3a1d15 100644 --- a/src/Common/RadixSort.h +++ b/src/Common/RadixSort.h @@ -381,15 +381,24 @@ public: Index * writer = destination; Element * reader = pass % 2 ? swap_buffer : arr; - for (size_t i = 0; i < size; ++i) + if (reverse) { - size_t pos = getPart(pass, keyToBits(Traits::extractKey(reader[i]))); + for (size_t i = 0; i < size; ++i) + { + size_t pos = getPart(pass, keyToBits(Traits::extractKey(reader[i]))); - /// Place the element on the next free position. - if (reverse) + /// Place the element on the next free position. writer[size - 1 - (++histograms[pass * HISTOGRAM_SIZE + pos])] = Traits::extractIndex(reader[i]); - else + } + } else + { + for (size_t i = 0; i < size; ++i) + { + size_t pos = getPart(pass, keyToBits(Traits::extractKey(reader[i]))); + + /// Place the element on the next free position. writer[++histograms[pass * HISTOGRAM_SIZE + pos]] = Traits::extractIndex(reader[i]); + } } } else if (NUM_PASSES % 2) { From 3267eaa49840a21de35386f15912a7c8bac2bac0 Mon Sep 17 00:00:00 2001 From: g-arslan Date: Thu, 21 May 2020 14:39:40 +0300 Subject: [PATCH 521/738] fix style --- src/Columns/ColumnVector.cpp | 11 +++++++---- src/Common/RadixSort.h | 3 ++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index cf6d9cf030b..20fc68c0a14 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -125,15 +125,18 @@ namespace } template -void shiftArrayRight(T& arr, size_t bound) +void shiftArrayRight(T & arr, size_t bound) { bound %= arr.size(); size_t gcd = std::gcd(arr.size(), bound); - for (size_t i = 0; i < gcd; ++i) { + for (size_t i = 0; i < gcd; ++i) + { size_t j = i; - do { + do + { size_t next = j + bound; - if (next >= arr.size()) next -= arr.size(); + if (next >= arr.size()) + next -= arr.size(); std::swap(arr[j], arr[next]); j = next; } while (j != i); diff --git a/src/Common/RadixSort.h b/src/Common/RadixSort.h index 88ddc3a1d15..fea27ae36db 100644 --- a/src/Common/RadixSort.h +++ b/src/Common/RadixSort.h @@ -390,7 +390,8 @@ public: /// Place the element on the next free position. writer[size - 1 - (++histograms[pass * HISTOGRAM_SIZE + pos])] = Traits::extractIndex(reader[i]); } - } else + } + else { for (size_t i = 0; i < size; ++i) { From 3c763908c3570bc73dd18574c9046293fede337a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 21 May 2020 15:04:15 +0300 Subject: [PATCH 522/738] Added comment --- src/AggregateFunctions/AggregateFunctionSum.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/AggregateFunctions/AggregateFunctionSum.h b/src/AggregateFunctions/AggregateFunctionSum.h index 52431060aea..1d89b028150 100644 --- a/src/AggregateFunctions/AggregateFunctionSum.h +++ b/src/AggregateFunctions/AggregateFunctionSum.h @@ -134,6 +134,8 @@ struct AggregateFunctionSumKahanData template void NO_INLINE addMany(const Value * __restrict ptr, size_t count) { + /// Less than in ordinary sum, because the algorithm is more complicated and too large loop unrolling is questionable. + /// But this is just a guess. constexpr size_t unroll_count = 4; T partial_sums[unroll_count]{}; T partial_compensations[unroll_count]{}; From 581b2358de1219dac615994440944cb7a6becfb5 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 21 May 2020 15:07:12 +0300 Subject: [PATCH 523/738] Added comment --- src/AggregateFunctions/AggregateFunctionSum.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/AggregateFunctions/AggregateFunctionSum.h b/src/AggregateFunctions/AggregateFunctionSum.h index 1d89b028150..9d3d559ecee 100644 --- a/src/AggregateFunctions/AggregateFunctionSum.h +++ b/src/AggregateFunctions/AggregateFunctionSum.h @@ -195,6 +195,8 @@ struct AggregateFunctionSumKahanData { auto raw_sum = to_sum + from_sum; auto rhs_compensated = raw_sum - to_sum; + /// Kahan summation is tricky because it depends on non-associativity of float arithmetic. + /// Do not simplify this expression if you are not sure. auto compensations = ((from_sum - rhs_compensated) + (to_sum - (raw_sum - rhs_compensated))) + compensation + from_compensation; to_sum = raw_sum + compensations; to_compensation = compensations - (to_sum - raw_sum); From 5e388d5519f7e05e6bdeb1e6ffeb6a22d3f97266 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E7=92=9E?= Date: Thu, 21 May 2020 20:14:39 +0800 Subject: [PATCH 524/738] table name typo fix (#11100) According to the context, the table name of queue2 should be queue3. --- docs/en/engines/table-engines/integrations/kafka.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/engines/table-engines/integrations/kafka.md b/docs/en/engines/table-engines/integrations/kafka.md index 4b29baed70d..d7824443c1d 100644 --- a/docs/en/engines/table-engines/integrations/kafka.md +++ b/docs/en/engines/table-engines/integrations/kafka.md @@ -72,7 +72,7 @@ Examples: kafka_format = 'JSONEachRow', kafka_num_consumers = 4; - CREATE TABLE queue2 ( + CREATE TABLE queue3 ( timestamp UInt64, level String, message String From 4e72ede169cfd2f425dccc2307f1d6816ee10d8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9E=81=E5=AE=A2=E9=9D=92=E5=B9=B4?= Date: Thu, 21 May 2020 20:40:44 +0800 Subject: [PATCH 525/738] Update install.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://repo.yandex.ru/clickhouse/rpm/stable/x86_64/ change url | 修正地址 --- docs/zh/getting-started/install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/getting-started/install.md b/docs/zh/getting-started/install.md index 9f06317ebd0..32eb7fa0a82 100644 --- a/docs/zh/getting-started/install.md +++ b/docs/zh/getting-started/install.md @@ -46,7 +46,7 @@ sudo yum-config-manager --add-repo https://repo.yandex.ru/clickhouse/rpm/stable/ sudo yum install clickhouse-server clickhouse-client ``` -您也可以从此处手动下载和安装软件包：https://repo.yandex.ru/clickhouse/rpm/stable/x86\_64。 +您也可以从此处手动下载和安装软件包：https://repo.yandex.ru/clickhouse/rpm/stable/x86_64。 ### 来自Docker {#from-docker-image} From 9e350bea33feb24078747f33ebafe091f2cc5a03 Mon Sep 17 00:00:00 2001 From: g-arslan Date: Thu, 21 May 2020 17:10:47 +0300 Subject: [PATCH 526/738] replace shift array with std::rotate --- src/Columns/ColumnVector.cpp | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 20fc68c0a14..58c4645d0c5 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -124,25 +124,6 @@ namespace }; } -template -void shiftArrayRight(T & arr, size_t bound) -{ - bound %= arr.size(); - size_t gcd = std::gcd(arr.size(), bound); - for (size_t i = 0; i < gcd; ++i) - { - size_t j = i; - do - { - size_t next = j + bound; - if (next >= arr.size()) - next -= arr.size(); - std::swap(arr[j], arr[next]); - j = next; - } while (j != i); - } -} - template void ColumnVector::getSpecialPermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, IColumn::SpecialSort special_sort) const @@ -219,7 +200,7 @@ void ColumnVector::getPermutation(bool reverse, size_t limit, int nan_directi if (nans_to_move) { - shiftArrayRight(res, reverse ? s - nans_to_move : nans_to_move); + std::rotate(std::begin(res), std::begin(res) + (reverse ? nans_to_move : s - nans_to_move), std::end(res)); } } return; From c036af0261b972cbb6f3bf6585d7672dca778816 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 21 May 2020 17:11:56 +0300 Subject: [PATCH 527/738] Fix deadlock after clickhouse-server update (with changes in one of system log tables structure) during startup between concurrent merge and table rename. --- src/Interpreters/Context.cpp | 56 +++++++++++-------- src/Interpreters/Context.h | 10 ++-- src/Interpreters/SystemLog.h | 4 +- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- src/Storages/StorageDistributed.cpp | 4 +- src/Storages/System/StorageSystemDisks.cpp | 4 +- .../System/StorageSystemStoragePolicies.cpp | 4 +- 7 files changed, 47 insertions(+), 37 deletions(-) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 6739aab134f..3b285e77305 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -295,6 +295,10 @@ struct ContextShared mutable std::mutex embedded_dictionaries_mutex; mutable std::mutex external_dictionaries_mutex; mutable std::mutex external_models_mutex; + /// Separate mutex for storage policies. During server startup we may + /// initialize some important storages (system tables with MergeTree engine) + /// under context lock. + mutable std::mutex storage_policies_mutex; /// Separate mutex for re-initialization of zookeeper session. This operation could take a long time and must not interfere with another operations. mutable std::mutex zookeeper_mutex; @@ -567,7 +571,7 @@ void Context::setPath(const String & path) VolumeJBODPtr Context::setTemporaryStorage(const String & path, const String & policy_name) { - auto lock = getLock(); + std::lock_guard lock(shared->storage_policies_mutex); if (policy_name.empty()) { @@ -580,7 +584,7 @@ VolumeJBODPtr Context::setTemporaryStorage(const String & path, const String & p } else { - StoragePolicyPtr tmp_policy = getStoragePolicySelector()->get(policy_name); + StoragePolicyPtr tmp_policy = getStoragePolicySelector(lock)->get(policy_name); if (tmp_policy->getVolumes().size() != 1) throw Exception("Policy " + policy_name + " is used temporary files, such policy should have exactly one volume", ErrorCodes::NO_ELEMENTS_IN_CONFIG); shared->tmp_volume = tmp_policy->getVolume(0); @@ -1688,18 +1692,37 @@ CompressionCodecPtr Context::chooseCompressionCodec(size_t part_size, double par DiskPtr Context::getDisk(const String & name) const { - auto lock = getLock(); + std::lock_guard lock(shared->storage_policies_mutex); - auto disk_selector = getDiskSelector(); + auto disk_selector = getDiskSelector(lock); return disk_selector->get(name); } - -DiskSelectorPtr Context::getDiskSelector() const +StoragePolicyPtr Context::getStoragePolicy(const String & name) const { - auto lock = getLock(); + std::lock_guard lock(shared->storage_policies_mutex); + auto policy_selector = getStoragePolicySelector(lock); + + return policy_selector->get(name); +} + + +const std::map & Context::getDisksMap() const +{ + std::lock_guard lock(shared->storage_policies_mutex); + return getDiskSelector(lock)->getDisksMap(); +} + +const std::map & Context::getPoliciesMap() const +{ + std::lock_guard lock(shared->storage_policies_mutex); + return getStoragePolicySelector(lock)->getPoliciesMap(); +} + +DiskSelectorPtr Context::getDiskSelector(std::lock_guard & /* lock */) const +{ if (!shared->merge_tree_disk_selector) { constexpr auto config_name = "storage_configuration.disks"; @@ -1710,27 +1733,14 @@ DiskSelectorPtr Context::getDiskSelector() const return shared->merge_tree_disk_selector; } - -StoragePolicyPtr Context::getStoragePolicy(const String & name) const +StoragePolicySelectorPtr Context::getStoragePolicySelector(std::lock_guard & lock) const { - auto lock = getLock(); - - auto policy_selector = getStoragePolicySelector(); - - return policy_selector->get(name); -} - - -StoragePolicySelectorPtr Context::getStoragePolicySelector() const -{ - auto lock = getLock(); - if (!shared->merge_tree_storage_policy_selector) { constexpr auto config_name = "storage_configuration.policies"; const auto & config = getConfigRef(); - shared->merge_tree_storage_policy_selector = std::make_shared(config, config_name, getDiskSelector()); + shared->merge_tree_storage_policy_selector = std::make_shared(config, config_name, getDiskSelector(lock)); } return shared->merge_tree_storage_policy_selector; } @@ -1738,7 +1748,7 @@ StoragePolicySelectorPtr Context::getStoragePolicySelector() const void Context::updateStorageConfiguration(const Poco::Util::AbstractConfiguration & config) { - auto lock = getLock(); + std::lock_guard lock(shared->storage_policies_mutex); if (shared->merge_tree_disk_selector) shared->merge_tree_disk_selector = shared->merge_tree_disk_selector->updateFromConfig(config, "storage_configuration.disks", *this); diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 4ba135d0b1a..65c9ed6749f 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -541,14 +541,12 @@ public: /// Lets you select the compression codec according to the conditions described in the configuration file. std::shared_ptr chooseCompressionCodec(size_t part_size, double part_size_ratio) const; - DiskSelectorPtr getDiskSelector() const; /// Provides storage disks DiskPtr getDisk(const String & name) const; - DiskPtr getDefaultDisk() const { return getDisk("default"); } - - StoragePolicySelectorPtr getStoragePolicySelector() const; + const std::map & getPoliciesMap() const; + const std::map & getDisksMap() const; void updateStorageConfiguration(const Poco::Util::AbstractConfiguration & config); /// Provides storage politics schemes @@ -626,6 +624,10 @@ private: EmbeddedDictionaries & getEmbeddedDictionariesImpl(bool throw_on_error) const; void checkCanBeDropped(const String & database, const String & table, const size_t & size, const size_t & max_size_to_drop) const; + + StoragePolicySelectorPtr getStoragePolicySelector(std::lock_guard & lock) const; + + DiskSelectorPtr getDiskSelector(std::lock_guard & /* lock */) const; }; diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h index b2a4eec7883..f9b1154f442 100644 --- a/src/Interpreters/SystemLog.h +++ b/src/Interpreters/SystemLog.h @@ -199,8 +199,8 @@ SystemLog::SystemLog(Context & context_, size_t flush_interval_milliseconds_) : context(context_) , table_id(database_name_, table_name_) - , storage_def(storage_def_), - flush_interval_milliseconds(flush_interval_milliseconds_) + , storage_def(storage_def_) + , flush_interval_milliseconds(flush_interval_milliseconds_) { assert(database_name_ == DatabaseCatalog::SYSTEM_DATABASE); log = &Logger::get("SystemLog (" + database_name_ + "." + table_name_ + ")"); diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index de9d3f6e981..64c95b8b8a5 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -866,7 +866,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) for (const auto & disk_ptr : disks) defined_disk_names.insert(disk_ptr->getName()); - for (const auto & [disk_name, disk] : global_context.getDiskSelector()->getDisksMap()) + for (const auto & [disk_name, disk] : global_context.getDisksMap()) { if (defined_disk_names.count(disk_name) == 0 && disk->exists(relative_data_path)) { diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index eb75f53ef9c..8d85b5a1db2 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -334,7 +334,7 @@ void StorageDistributed::createStorage() } else { - auto policy = global_context.getStoragePolicySelector()->get(storage_policy); + auto policy = global_context.getStoragePolicy(storage_policy); if (policy->getVolumes().size() != 1) throw Exception("Policy for Distributed table, should have exactly one volume", ErrorCodes::BAD_ARGUMENTS); volume = policy->getVolume(0); @@ -628,7 +628,7 @@ StoragePolicyPtr StorageDistributed::getStoragePolicy() const { if (storage_policy.empty()) return {}; - return global_context.getStoragePolicySelector()->get(storage_policy); + return global_context.getStoragePolicy(storage_policy); } void StorageDistributed::createDirectoryMonitors(const std::string & disk) diff --git a/src/Storages/System/StorageSystemDisks.cpp b/src/Storages/System/StorageSystemDisks.cpp index 5ddf7de9ec8..cadaaff6cbc 100644 --- a/src/Storages/System/StorageSystemDisks.cpp +++ b/src/Storages/System/StorageSystemDisks.cpp @@ -39,9 +39,7 @@ Pipes StorageSystemDisks::read( MutableColumnPtr col_total = ColumnUInt64::create(); MutableColumnPtr col_keep = ColumnUInt64::create(); - const auto & disk_selector = context.getDiskSelector(); - - for (const auto & [disk_name, disk_ptr] : disk_selector->getDisksMap()) + for (const auto & [disk_name, disk_ptr] : context.getDisksMap()) { col_name->insert(disk_name); col_path->insert(disk_ptr->getPath()); diff --git a/src/Storages/System/StorageSystemStoragePolicies.cpp b/src/Storages/System/StorageSystemStoragePolicies.cpp index 81b6ddd465a..474d2b5bcc4 100644 --- a/src/Storages/System/StorageSystemStoragePolicies.cpp +++ b/src/Storages/System/StorageSystemStoragePolicies.cpp @@ -44,9 +44,9 @@ Pipes StorageSystemStoragePolicies::read( MutableColumnPtr col_max_part_size = ColumnUInt64::create(); MutableColumnPtr col_move_factor = ColumnFloat32::create(); - const auto & policy_selector = context.getStoragePolicySelector(); + const auto & policies_map = context.getPoliciesMap(); - for (const auto & [policy_name, policy_ptr] : policy_selector->getPoliciesMap()) + for (const auto & [policy_name, policy_ptr] : policies_map) { const auto & volumes = policy_ptr->getVolumes(); for (size_t i = 0; i != volumes.size(); ++i) From 403aae91265cfe50f8a5c989147680c68a9e93a0 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Sat, 16 May 2020 15:11:17 +0300 Subject: [PATCH 528/738] Fixed parseDateTime64BestEffort implementation Fixed argument resolution issues. Added tests and made sure -orNull and -orZero variants alwo work correctly. --- src/Functions/FunctionsConversion.h | 139 +++++++++--------- src/IO/parseDateTimeBestEffort.cpp | 13 +- .../01281_parseDateTime64BestEffort.reference | 15 ++ .../01281_parseDateTime64BestEffort.sql | 33 +++++ 4 files changed, 132 insertions(+), 68 deletions(-) create mode 100644 tests/queries/0_stateless/01281_parseDateTime64BestEffort.reference create mode 100644 tests/queries/0_stateless/01281_parseDateTime64BestEffort.sql diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index 64708f45598..9d0b764e84b 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -1110,6 +1110,8 @@ public: std::is_same_v> || std::is_same_v>; + static constexpr bool to_datetime64 = std::is_same_v; + static FunctionPtr create(const Context &) { return std::make_shared(); } static FunctionPtr create() { return std::make_shared(); } @@ -1126,67 +1128,17 @@ public: DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { - if ((arguments.size() != 1 && arguments.size() != 2) || (to_decimal && arguments.size() != 2)) - throw Exception("Number of arguments for function " + getName() + " doesn't match: passed " + toString(arguments.size()) + - ", should be 1 or 2. Second argument only make sense for DateTime (time zone, optional) and Decimal (scale).", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - - if (!isStringOrFixedString(arguments[0].type)) - { - if (this->getName().find("OrZero") != std::string::npos || - this->getName().find("OrNull") != std::string::npos) - throw Exception("Illegal type " + arguments[0].type->getName() + " of first argument of function " + getName() + - ". Conversion functions with postfix 'OrZero' or 'OrNull' should take String argument", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - else - throw Exception("Illegal type " + arguments[0].type->getName() + " of first argument of function " + getName(), - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - } - - if (arguments.size() == 2) - { - if constexpr (std::is_same_v) - { - if (!isString(arguments[1].type)) - throw Exception("Illegal type " + arguments[1].type->getName() + " of 2nd argument of function " + getName(), - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - } - else if constexpr (to_decimal) - { - if (!isInteger(arguments[1].type)) - throw Exception("Illegal type " + arguments[1].type->getName() + " of 2nd argument of function " + getName(), - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - if (!arguments[1].column) - throw Exception("Second argument for function " + getName() + " must be constant", ErrorCodes::ILLEGAL_COLUMN); - } - else - { - throw Exception("Number of arguments for function " + getName() + " doesn't match: passed " - + toString(arguments.size()) + ", should be 1. Second argument makes sense only for DateTime and Decimal.", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - } - } - DataTypePtr res; - - if constexpr (std::is_same_v) - res = std::make_shared(extractTimeZoneNameFromFunctionArguments(arguments, 1, 0)); - else if constexpr (to_decimal) + if constexpr (to_datetime64) { - UInt64 scale = extractToDecimalScale(arguments[1]); + validateFunctionArgumentTypes(*this, arguments, + FunctionArgumentDescriptors{{"string", isStringOrFixedString, nullptr, "String or FixedString"}}, + // optional + FunctionArgumentDescriptors{ + {"precision", isUInt8, isColumnConst, "const UInt8"}, + {"timezone", isStringOrFixedString, isColumnConst, "const String or FixedString"}, + }); - if constexpr (std::is_same_v>) - res = createDecimal(9, scale); - else if constexpr (std::is_same_v>) - res = createDecimal(18, scale); - else if constexpr (std::is_same_v>) - res = createDecimal(38, scale); - - if (!res) - throw Exception("Someting wrong with toDecimalNNOrZero() or toDecimalNNOrNull()", ErrorCodes::LOGICAL_ERROR); - } - else if constexpr (std::is_same_v) - { UInt64 scale = DataTypeDateTime64::default_scale; if (arguments.size() > 1) scale = extractToDecimalScale(arguments[1]); @@ -1194,7 +1146,67 @@ public: res = std::make_shared(scale, timezone); } else - res = std::make_shared(); + { + if ((arguments.size() != 1 && arguments.size() != 2) || (to_decimal && arguments.size() != 2)) + throw Exception("Number of arguments for function " + getName() + " doesn't match: passed " + toString(arguments.size()) + + ", should be 1 or 2. Second argument only make sense for DateTime (time zone, optional) and Decimal (scale).", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + if (!isStringOrFixedString(arguments[0].type)) + { + if (this->getName().find("OrZero") != std::string::npos || + this->getName().find("OrNull") != std::string::npos) + throw Exception("Illegal type " + arguments[0].type->getName() + " of first argument of function " + getName() + + ". Conversion functions with postfix 'OrZero' or 'OrNull' should take String argument", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + else + throw Exception("Illegal type " + arguments[0].type->getName() + " of first argument of function " + getName(), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + + if (arguments.size() == 2) + { + if constexpr (std::is_same_v) + { + if (!isString(arguments[1].type)) + throw Exception("Illegal type " + arguments[1].type->getName() + " of 2nd argument of function " + getName(), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + else if constexpr (to_decimal) + { + if (!isInteger(arguments[1].type)) + throw Exception("Illegal type " + arguments[1].type->getName() + " of 2nd argument of function " + getName(), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if (!arguments[1].column) + throw Exception("Second argument for function " + getName() + " must be constant", ErrorCodes::ILLEGAL_COLUMN); + } + else + { + throw Exception("Number of arguments for function " + getName() + " doesn't match: passed " + + toString(arguments.size()) + ", should be 1. Second argument makes sense only for DateTime and Decimal.", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + } + } + + if constexpr (std::is_same_v) + res = std::make_shared(extractTimeZoneNameFromFunctionArguments(arguments, 1, 0)); + else if constexpr (to_decimal) + { + UInt64 scale = extractToDecimalScale(arguments[1]); + + if constexpr (std::is_same_v>) + res = createDecimal(9, scale); + else if constexpr (std::is_same_v>) + res = createDecimal(18, scale); + else if constexpr (std::is_same_v>) + res = createDecimal(38, scale); + + if (!res) + throw Exception("Someting wrong with toDecimalNNOrZero() or toDecimalNNOrNull()", ErrorCodes::LOGICAL_ERROR); + } + else + res = std::make_shared(); + } if constexpr (exception_mode == ConvertFromStringExceptionMode::Null) res = std::make_shared(res); @@ -1207,12 +1219,9 @@ public: const IDataType * from_type = block.getByPosition(arguments[0]).type.get(); bool ok = true; - if constexpr (to_decimal || std::is_same_v) + if constexpr (to_decimal || to_datetime64) { - if (arguments.size() != 2) - throw Exception{"Function " + getName() + " expects 2 arguments for Decimal.", ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION}; - - UInt32 scale = extractToDecimalScale(block.getByPosition(arguments[1])); + const UInt32 scale = assert_cast(*removeNullable(block.getByPosition(result).type)).getScale(); if (checkAndGetDataType(from_type)) { @@ -1241,7 +1250,6 @@ public: } else ok = false; - } if (!ok) @@ -1252,7 +1260,6 @@ public: } }; - /** Conversion to fixed string is implemented only for strings. */ class FunctionToFixedString : public IFunction diff --git a/src/IO/parseDateTimeBestEffort.cpp b/src/IO/parseDateTimeBestEffort.cpp index 7e40909226c..4b6183e9c0b 100644 --- a/src/IO/parseDateTimeBestEffort.cpp +++ b/src/IO/parseDateTimeBestEffort.cpp @@ -562,8 +562,17 @@ ReturnType parseDateTime64BestEffortImpl(DateTime64 & res, UInt32 scale, ReadBuf { time_t whole; DateTimeSubsecondPart subsecond = {0, 0}; // needs to be explicitly initialized sine it could be missing from input string - if (!parseDateTimeBestEffortImpl(whole, in, local_time_zone, utc_time_zone, &subsecond)) - return ReturnType(false); + + if constexpr (std::is_same_v) + { + if (!parseDateTimeBestEffortImpl(whole, in, local_time_zone, utc_time_zone, &subsecond)) + return false; + } + else + { + parseDateTimeBestEffortImpl(whole, in, local_time_zone, utc_time_zone, &subsecond); + } + DateTime64::NativeType fractional = subsecond.value; if (scale < subsecond.digits) diff --git a/tests/queries/0_stateless/01281_parseDateTime64BestEffort.reference b/tests/queries/0_stateless/01281_parseDateTime64BestEffort.reference new file mode 100644 index 00000000000..e55e50c15d5 --- /dev/null +++ b/tests/queries/0_stateless/01281_parseDateTime64BestEffort.reference @@ -0,0 +1,15 @@ +orNull +2020-05-14 03:37:03.253 +\N +orZero +2020-05-14 03:37:03.253 +0000-00-00 00:00:00.000 +non-const +2020-05-14 03:37:03.253 +Timezones +2020-05-14 03:37:03.253 +2020-05-14 06:37:03.253 +Formats +2020-05-14 03:37:03.253 +2020-05-14 03:37:03.000 +2020-05-14 03:37:03.000 diff --git a/tests/queries/0_stateless/01281_parseDateTime64BestEffort.sql b/tests/queries/0_stateless/01281_parseDateTime64BestEffort.sql new file mode 100644 index 00000000000..5c0bbe1b4c2 --- /dev/null +++ b/tests/queries/0_stateless/01281_parseDateTime64BestEffort.sql @@ -0,0 +1,33 @@ +-- Error cases +SELECT parseDateTime64BestEffort(); -- {serverError 42} +SELECT parseDateTime64BestEffort(123); -- {serverError 43} +SELECT parseDateTime64BestEffort('foo'); -- {serverError 41} + +SELECT parseDateTime64BestEffort('2020-05-14T03:37:03.253184Z', 'bar'); -- {serverError 43} -- invalid scale parameter +SELECT parseDateTime64BestEffort('2020-05-14T03:37:03.253184Z', 3, 4); -- {serverError 43} -- invalid timezone parameter +SELECT parseDateTime64BestEffort('2020-05-14T03:37:03.253184Z', 3, 'baz'); -- {serverError 1000} -- unknown timezone + +SELECT parseDateTime64BestEffort('2020-05-14T03:37:03.253184Z', materialize(3), 4); -- {serverError 44} -- non-const precision +SELECT parseDateTime64BestEffort('2020-05-14T03:37:03.253184Z', 3, materialize('UTC')); -- {serverError 44} -- non-const timezone + +SELECT parseDateTime64BestEffort('2020-05-14T03:37:03.253184012345678910111213141516171819Z', 3, 'UTC'); -- {serverError 6} + +SELECT 'orNull'; +SELECT parseDateTime64BestEffortOrNull('2020-05-14T03:37:03.253184Z', 3, 'UTC'); +SELECT parseDateTime64BestEffortOrNull('foo', 3, 'UTC'); + +SELECT 'orZero'; +SELECT parseDateTime64BestEffortOrZero('2020-05-14T03:37:03.253184Z', 3, 'UTC'); +SELECT parseDateTime64BestEffortOrZero('bar', 3, 'UTC'); + +SELECT 'non-const'; +SELECT parseDateTime64BestEffort(materialize('2020-05-14T03:37:03.253184Z'), 3, 'UTC'); + +SELECT 'Timezones'; +SELECT parseDateTime64BestEffort('2020-05-14T03:37:03.253184Z', 3, 'UTC'); +SELECT parseDateTime64BestEffort('2020-05-14T03:37:03.253184Z', 3, 'Europe/Minsk'); + +SELECT 'Formats'; +SELECT parseDateTime64BestEffort('2020-05-14T03:37:03.253184', 3, 'UTC'); +SELECT parseDateTime64BestEffort('2020-05-14T03:37:03', 3, 'UTC'); +SELECT parseDateTime64BestEffort('2020-05-14 03:37:03', 3, 'UTC'); \ No newline at end of file From 9dc493e063c20459c2941c3d3ff28996da3205e3 Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Thu, 21 May 2020 18:02:52 +0300 Subject: [PATCH 529/738] Revert "Revert "Allow to build binaries and tests from deb package build. "" --- debian/rules | 11 ++++++++++- docker/packager/deb/build.sh | 11 +++++++++++ docker/packager/packager | 19 +++++++++++++++++-- programs/CMakeLists.txt | 6 ++++++ 4 files changed, 44 insertions(+), 3 deletions(-) diff --git a/debian/rules b/debian/rules index dabebb516cd..7218e196baa 100755 --- a/debian/rules +++ b/debian/rules @@ -24,6 +24,10 @@ DEB_BUILD_OPTIONS+=parallel=$(THREADS_COUNT) ifndef ENABLE_TESTS CMAKE_FLAGS += -DENABLE_TESTS=0 +else +# To export binaries and from deb build we do not strip them. No need to run tests in deb build as we run them in CI + DEB_BUILD_OPTIONS+= nocheck + DEB_BUILD_OPTIONS+= nostrip endif ifndef MAKE_TARGET @@ -88,14 +92,19 @@ override_dh_auto_build: $(MAKE) $(THREADS_COUNT) -C $(BUILDDIR) $(MAKE_TARGET) override_dh_auto_test: +ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS))) cd $(BUILDDIR) && ctest $(THREADS_COUNT) -V -E with_server +endif override_dh_clean: rm -rf debian/copyright debian/clickhouse-client.docs debian/clickhouse-common-static.docs - dh_clean -X contrib + dh_clean # -X contrib override_dh_strip: +#https://www.debian.org/doc/debian-policy/ch-source.html#debian-rules-and-deb-build-options +ifeq (,$(filter nostrip,$(DEB_BUILD_OPTIONS))) dh_strip -pclickhouse-common-static --dbg-package=clickhouse-common-static-dbg +endif override_dh_install: # Making docs diff --git a/docker/packager/deb/build.sh b/docker/packager/deb/build.sh index 1efed3628a0..fbaa0151c6b 100755 --- a/docker/packager/deb/build.sh +++ b/docker/packager/deb/build.sh @@ -10,5 +10,16 @@ mv *.changes /output mv *.buildinfo /output mv /*.rpm /output ||: # if exists mv /*.tgz /output ||: # if exists + +if [ -n "$BINARY_OUTPUT" ] && { [ "$BINARY_OUTPUT" = "programs" ] || [ "$BINARY_OUTPUT" = "tests" ] ;} +then + echo Place $BINARY_OUTPUT to output + mkdir /output/binary ||: # if exists + mv /build/obj-*/programs/clickhouse* /output/binary + if [ "$BINARY_OUTPUT" = "tests" ] + then + mv /build/obj-*/src/unit_tests_dbms /output/binary + fi +fi ccache --show-stats ||: ln -s /usr/lib/x86_64-linux-gnu/libOpenCL.so.1.0.0 /usr/lib/libOpenCL.so ||: diff --git a/docker/packager/packager b/docker/packager/packager index 025ca3bf398..8a5bdda60e8 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -54,7 +54,7 @@ def run_docker_image_with_env(image_name, output, env_variables, ch_root, ccache subprocess.check_call(cmd, shell=True) -def parse_env_variables(build_type, compiler, sanitizer, package_type, image_type, cache, distcc_hosts, unbundled, split_binary, clang_tidy, version, author, official, alien_pkgs, with_coverage): +def parse_env_variables(build_type, compiler, sanitizer, package_type, image_type, cache, distcc_hosts, unbundled, split_binary, clang_tidy, version, author, official, alien_pkgs, with_coverage, with_binaries): CLANG_PREFIX = "clang" DARWIN_SUFFIX = "-darwin" ARM_SUFFIX = "-aarch64" @@ -131,6 +131,14 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ if alien_pkgs: result.append("ALIEN_PKGS='" + ' '.join(['--' + pkg for pkg in alien_pkgs]) + "'") + if with_binaries == "programs": + result.append('BINARY_OUTPUT=programs') + elif with_binaries == "tests": + result.append('ENABLE_TESTS=1') + result.append('BINARY_OUTPUT=tests') + cmake_flags.append('-DENABLE_TESTS=1') + cmake_flags.append('-DUSE_GTEST=1') + if unbundled: # TODO: fix build with ENABLE_RDKAFKA cmake_flags.append('-DUNBUNDLED=1 -DENABLE_MYSQL=0 -DENABLE_ODBC=0 -DENABLE_REPLXX=0 -DENABLE_RDKAFKA=0') @@ -179,6 +187,7 @@ if __name__ == "__main__": parser.add_argument("--official", action="store_true") parser.add_argument("--alien-pkgs", nargs='+', default=[]) parser.add_argument("--with-coverage", action="store_true") + parser.add_argument("--with-binaries", choices=("programs", "tests", ""), default="") args = parser.parse_args() if not os.path.isabs(args.output_dir): @@ -195,6 +204,12 @@ if __name__ == "__main__": if args.alien_pkgs and not image_type == "deb": raise Exception("Can add alien packages only in deb build") + if args.with_binaries != "" and not image_type == "deb": + raise Exception("Can add additional binaries only in deb build") + + if args.with_binaries != "" and image_type == "deb": + logging.info("Should place {} to output".format(args.with_binaries)) + dockerfile = os.path.join(ch_root, "docker/packager", image_type, "Dockerfile") if image_type != "freebsd" and not check_image_exists_locally(image_name) or args.force_build_image: if not pull_image(image_name) or args.force_build_image: @@ -202,6 +217,6 @@ if __name__ == "__main__": env_prepared = parse_env_variables( args.build_type, args.compiler, args.sanitizer, args.package_type, image_type, args.cache, args.distcc_hosts, args.unbundled, args.split_binary, args.clang_tidy, - args.version, args.author, args.official, args.alien_pkgs, args.with_coverage) + args.version, args.author, args.official, args.alien_pkgs, args.with_coverage, args.with_binaries) run_docker_image_with_env(image_name, args.output_dir, env_prepared, ch_root, args.ccache_dir) logging.info("Output placed into {}".format(args.output_dir)) diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index a3d3188653b..65742697333 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -205,3 +205,9 @@ endif () if (TARGET clickhouse-server AND TARGET copy-headers) add_dependencies(clickhouse-server copy-headers) endif () + +if (ENABLE_TESTS AND USE_GTEST) + set (CLICKHOUSE_ALL_TESTS_TARGETS local_date_time_comparison unit_tests_libcommon unit_tests_dbms hashing_write_buffer hashing_read_buffer in_join_subqueries_preprocessor expression_analyzer) + add_custom_target (clickhouse-tests ALL DEPENDS ${CLICKHOUSE_ALL_TESTS_TARGETS}) + add_dependencies(clickhouse-bundle clickhouse-tests) +endif() From 20d41a3f9834d2c2d4f2badfa94b6054bd6c24ae Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 21 May 2020 18:30:08 +0300 Subject: [PATCH 530/738] Update dpkg-deb (https://github.com/ClickHouse-Extras/dpkg) for compatibility with ubuntu eoan packages format --- docker/packager/deb/Dockerfile | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/docker/packager/deb/Dockerfile b/docker/packager/deb/Dockerfile index 78f396f5c75..ae70bc8c594 100644 --- a/docker/packager/deb/Dockerfile +++ b/docker/packager/deb/Dockerfile @@ -5,6 +5,7 @@ RUN apt-get --allow-unauthenticated update -y && apt-get install --yes wget gnup RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - RUN echo "deb [trusted=yes] http://apt.llvm.org/eoan/ llvm-toolchain-eoan-10 main" >> /etc/apt/sources.list + RUN apt-get --allow-unauthenticated update -y \ && env DEBIAN_FRONTEND=noninteractive \ apt-get --allow-unauthenticated install --yes --no-install-recommends \ @@ -17,6 +18,14 @@ RUN apt-get --allow-unauthenticated update -y \ apt-transport-https \ ca-certificates +# Special dpkg-deb (https://github.com/ClickHouse-Extras/dpkg) version which is able +# to compress files using pigz (https://zlib.net/pigz/) instead of gzip. +# Significantly increase deb packaging speed and compatible with old systems +RUN curl -O https://clickhouse-builds.s3.yandex.net/utils/dpkg-deb +RUN chmod +x dpkg-deb +RUN cp dpkg-deb /usr/bin + + # Libraries from OS are only needed to test the "unbundled" build (that is not used in production). RUN apt-get --allow-unauthenticated update -y \ && env DEBIAN_FRONTEND=noninteractive \ @@ -74,12 +83,6 @@ RUN apt-get --allow-unauthenticated update -y \ libldap2-dev -# Special dpkg-deb (https://github.com/ClickHouse-Extras/dpkg) version which is able -# to compress files using pigz (https://zlib.net/pigz/) instead of gzip. -# Significantly increase deb packaging speed and compatible with old systems -RUN curl -O https://clickhouse-builds.s3.yandex.net/utils/dpkg-deb -RUN chmod +x dpkg-deb -RUN cp dpkg-deb /usr/bin # This symlink required by gcc to find lld compiler RUN ln -s /usr/bin/lld-10 /usr/bin/ld.lld From 5676340f6b4378c953b3462700839eecf6c9dbd2 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 21 May 2020 18:52:11 +0300 Subject: [PATCH 531/738] Less verbose logging in mutation finalization task --- src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp | 2 ++ src/Storages/StorageReplicatedMergeTree.cpp | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 2e71bc902e9..94497b2a850 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -1405,6 +1405,8 @@ bool ReplicatedMergeTreeQueue::tryFinalizeMutations(zkutil::ZooKeeperPtr zookeep if (candidates.empty()) return false; + else + LOG_DEBUG(log, "Trying to finalize " << candidates.size() << " mutations"); auto merge_pred = getMergePredicate(zookeeper); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index e6fc32ecbf9..f05c078bd57 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -2272,7 +2272,6 @@ void StorageReplicatedMergeTree::mergeSelectingTask() void StorageReplicatedMergeTree::mutationsFinalizingTask() { - LOG_DEBUG(log, "Trying to finalize mutations"); bool needs_reschedule = false; try From adfa77be9dd958a109c7481642189ec67775d00f Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 21 May 2020 20:07:22 +0300 Subject: [PATCH 532/738] Added another one comment. --- src/Processors/Executors/PipelineExecutor.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Processors/Executors/PipelineExecutor.h b/src/Processors/Executors/PipelineExecutor.h index 395f42a3316..4b87a1147c6 100644 --- a/src/Processors/Executors/PipelineExecutor.h +++ b/src/Processors/Executors/PipelineExecutor.h @@ -288,6 +288,7 @@ private: void initializeExecution(size_t num_threads); /// Initialize executor contexts and task_queue. void finalizeExecution(); /// Check all processors are finished. + /// Methods connected to execution. void executeImpl(size_t num_threads); void executeStepImpl(size_t thread_num, size_t num_threads, std::atomic_bool * yield_flag = nullptr); void executeSingleThread(size_t thread_num, size_t num_threads); From ebb29d48fab6741a436ad80ae2d250899540e54d Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Thu, 21 May 2020 21:06:08 +0300 Subject: [PATCH 533/738] Update perf.py --- docker/test/performance-comparison/perf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/performance-comparison/perf.py b/docker/test/performance-comparison/perf.py index be8155cdddb..fd16673bd89 100755 --- a/docker/test/performance-comparison/perf.py +++ b/docker/test/performance-comparison/perf.py @@ -151,7 +151,7 @@ for query_index, q in enumerate(test_queries): # use the test name + the test-wide query index. query_display_name = q if len(query_display_name) > 1000: - query_display_name = f'{query_display_name[:1000]}...({i})' + query_display_name = f'{query_display_name[:1000]}...({query_index})' print(f'display-name\t{query_index}\t{tsv_escape(query_display_name)}') From 23044ac02cff79b629974e6705ca8491214b32f2 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 21 May 2020 21:18:34 +0300 Subject: [PATCH 534/738] Disable -Wsequence-point on gcc10 (otherwise it stuck on GatherUtils compiling) clang (10.0.0 is fine BTW) will warn about this anyway on CI. For the debug build gcc10: - before patch: - concat.cpp -> >5m (stuck on cc1plus, not as) - has.cpp -> >10m (stuck on cc1plus, not as) - after this patch: - concat.cpp -> 1m16s - has.cpp -> 4m (and most of the time eats, as from binutils 2.34.50.20200508) Command for build: - ninja src/Functions/GatherUtils/CMakeFiles/clickhouse_functions_gatherutils.dir/concat.cpp.o - ninja src/Functions/GatherUtils/CMakeFiles/clickhouse_functions_gatherutils.dir/has.cpp.o The test case should be reduced and then it can be reported to the gcc bugzilla. P.S. Looks like a signal not to switch to gcc10 for now --- cmake/warnings.cmake | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmake/warnings.cmake b/cmake/warnings.cmake index cc97e727101..63cb153a0b4 100644 --- a/cmake/warnings.cmake +++ b/cmake/warnings.cmake @@ -162,4 +162,10 @@ elseif (COMPILER_GCC) add_cxx_compile_options(-Wunused) # Warn if vector operation is not implemented via SIMD capabilities of the architecture add_cxx_compile_options(-Wvector-operation-performance) + + # XXX: gcc10 stuck with this option while compiling GatherUtils code + # (anyway there are builds with clang, that will warn) + if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 10) + add_cxx_compile_options(-Wno-sequence-point) + endif() endif () From 37fa26971e0bf4da290cf7542d8e54efcdbcb653 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 21 May 2020 21:49:31 +0300 Subject: [PATCH 535/738] boop ci --- src/Interpreters/Context.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 3b285e77305..e099721acdc 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -296,7 +296,7 @@ struct ContextShared mutable std::mutex external_dictionaries_mutex; mutable std::mutex external_models_mutex; /// Separate mutex for storage policies. During server startup we may - /// initialize some important storages (system tables with MergeTree engine) + /// initialize some important storages (system logs with MergeTree engine) /// under context lock. mutable std::mutex storage_policies_mutex; /// Separate mutex for re-initialization of zookeeper session. This operation could take a long time and must not interfere with another operations. From 41914588fd3adffc3a4534af2ff1c2ffbb0127cf Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 21 May 2020 22:07:18 +0300 Subject: [PATCH 536/738] More clear --- src/Storages/IStorage.cpp | 16 +++++++++----- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- .../ReplicatedMergeTreeTableMetadata.cpp | 2 +- .../MergeTree/StorageFromMergeTreeDataPart.h | 5 +---- src/Storages/StorageInMemoryMetadata.cpp | 6 +++--- src/Storages/StorageInMemoryMetadata.h | 21 +++++++++++++------ src/Storages/StorageReplicatedMergeTree.cpp | 2 +- 7 files changed, 33 insertions(+), 21 deletions(-) diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index 7a8da63f76f..2f1f4902952 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -460,7 +460,7 @@ Names IStorage::getColumnsRequiredForSampling() const bool IStorage::hasPartitionKey() const { - return partition_key.definition_ast != nullptr; + return partition_key.expressions != nullptr; } @@ -495,12 +495,12 @@ void IStorage::setSamplingKey(const StorageMetadataKeyField & sampling_key_) bool IStorage::hasSamplingKey() const { - return sampling_key.definition_ast != nullptr; + return sampling_key.expressions != nullptr; } bool IStorage::hasSortingKey() const { - return sorting_key.definition_ast != nullptr; + return sorting_key.expressions != nullptr; } void IStorage::setSortingKey(const StorageMetadataKeyField & sorting_key_) @@ -525,13 +525,19 @@ void IStorage::setPrimaryKey(const StorageMetadataKeyField & primary_key_) bool IStorage::hasPrimaryKey() const { - return primary_key.definition_ast != nullptr; + return primary_key.expressions != nullptr; } Names IStorage::getColumnsRequiredForPrimaryKey() const { - if (hasPrimaryKey()) + if (primary_key.expressions != nullptr) return primary_key.expressions->getRequiredColumns(); return {}; } + +bool IStorage::isPrimaryKeyDefined() const +{ + return primary_key.definition_ast != nullptr; +} + } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 094864f5642..fbcaeb350ad 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -256,7 +256,7 @@ StorageInMemoryMetadata MergeTreeData::getInMemoryMetadata() const if (hasSortingKey()) metadata.order_by_ast = getSortingKeyAST()->clone(); - if (hasPrimaryKey()) + if (isPrimaryKeyDefined()) metadata.primary_key_ast = getPrimaryKeyAST()->clone(); if (ttl_table_ast) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp index 5755603ee93..d6ca26730cf 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp @@ -40,7 +40,7 @@ ReplicatedMergeTreeTableMetadata::ReplicatedMergeTreeTableMetadata(const MergeTr /// So rules in zookeeper metadata is following: /// - When we have only ORDER BY, than store it in "primary key:" row of /metadata /// - When we have both, than store PRIMARY KEY in "primary key:" row and ORDER BY in "sorting key:" row of /metadata - if (!data.hasPrimaryKey()) + if (!data.isPrimaryKeyDefined()) primary_key = formattedAST(data.getSortingKey().expression_ast); else { diff --git a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h index 20a3bb5132b..a25a3a25337 100644 --- a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h +++ b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h @@ -52,10 +52,6 @@ public: return part->storage.getInMemoryMetadata(); } - bool hasSortingKey() const { return part->storage.hasSortingKey(); } - - Names getSortingKeyColumns() const override { return part->storage.getSortingKeyColumns(); } - NamesAndTypesList getVirtuals() const override { return part->storage.getVirtuals(); @@ -68,6 +64,7 @@ protected: { setColumns(part_->storage.getColumns()); setIndices(part_->storage.getIndices()); + setSortingKey(part_->storage.getSortingKey()); } private: diff --git a/src/Storages/StorageInMemoryMetadata.cpp b/src/Storages/StorageInMemoryMetadata.cpp index 277d9dfbfdd..0cda4d10f36 100644 --- a/src/Storages/StorageInMemoryMetadata.cpp +++ b/src/Storages/StorageInMemoryMetadata.cpp @@ -1,13 +1,15 @@ +#include + #include #include #include #include #include -#include #include namespace DB { + StorageInMemoryMetadata::StorageInMemoryMetadata( const ColumnsDescription & columns_, const IndicesDescription & indices_, @@ -119,7 +121,6 @@ StorageMetadataKeyField StorageMetadataKeyField::getKeyFromAST(const ASTPtr & de if (result.expression_ast->children.empty()) return result; - const auto & children = result.expression_ast->children; for (const auto & child : children) result.expression_column_names.emplace_back(child->getColumnName()); @@ -130,7 +131,6 @@ StorageMetadataKeyField StorageMetadataKeyField::getKeyFromAST(const ASTPtr & de result.sample_block = result.expressions->getSampleBlock(); } - for (size_t i = 0; i < result.sample_block.columns(); ++i) result.data_types.emplace_back(result.sample_block.getByPosition(i).type); diff --git a/src/Storages/StorageInMemoryMetadata.h b/src/Storages/StorageInMemoryMetadata.h index a98c00bc4c9..f32982bb49d 100644 --- a/src/Storages/StorageInMemoryMetadata.h +++ b/src/Storages/StorageInMemoryMetadata.h @@ -43,23 +43,32 @@ struct StorageInMemoryMetadata StorageInMemoryMetadata & operator=(const StorageInMemoryMetadata & other); }; -struct StorageMetadataField -{ - ASTPtr ast; - ExpressionActionsPtr actions; -}; - +/// Common structure for primary, partition and other storage keys struct StorageMetadataKeyField { + /// User defined AST in CREATE/ALTER query. This field may be empty, but key + /// can exists because some of them maybe set implicitly (for example, + /// primary key in merge tree can be part of sorting key) ASTPtr definition_ast; + /// ASTExpressionList with key fields, example: (x, toStartOfMonth(date))). ASTPtr expression_ast; + + /// Expressions from expression_list_ast from expression_analyzer. Useful, + /// when you need to get required columns for key, example: a, date, b. ExpressionActionsPtr expressions; + + /// Column names in key definition, example: x, toStartOfMonth(date), a * b. Names expression_column_names; + /// Sample block with key columns (names, types, empty column) Block sample_block; + + /// Types from sample block ordered in columns order. DataTypes data_types; + /// Parse key structure from key definition. Requires all columns available + /// in storage. static StorageMetadataKeyField getKeyFromAST(const ASTPtr & definition_ast, const ColumnsDescription & columns, const Context & context); }; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index df6564b3368..c1c670ea03d 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -493,7 +493,7 @@ void StorageReplicatedMergeTree::setTableStructure(ColumnsDescription new_column metadata.order_by_ast = tuple; } - if (!hasPrimaryKey()) + if (!isPrimaryKeyDefined()) { /// Primary and sorting key become independent after this ALTER so we have to /// save the old ORDER BY expression as the new primary key. From e0668f48e18bd3a44d77e0ead8b1a9200438865b Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Thu, 21 May 2020 22:34:50 +0300 Subject: [PATCH 537/738] [docs] fix sitemap (#11113) --- docs/tools/build.py | 11 ----------- website/robots.txt | 2 +- website/{sitemap.xml => sitemap-index.xml} | 2 +- website/{sitemap_static.xml => sitemap-static.xml} | 0 4 files changed, 2 insertions(+), 13 deletions(-) rename website/{sitemap.xml => sitemap-index.xml} (92%) rename website/{sitemap_static.xml => sitemap-static.xml} (100%) diff --git a/docs/tools/build.py b/docs/tools/build.py index 406f5689bc4..95e887f046f 100755 --- a/docs/tools/build.py +++ b/docs/tools/build.py @@ -58,17 +58,6 @@ def build_for_lang(lang, args): 'custom_dir': os.path.join(os.path.dirname(__file__), '..', args.theme_dir), 'language': lang, 'direction': 'rtl' if lang == 'fa' else 'ltr', - # TODO: cleanup - 'feature': { - 'tabs': False - }, - 'palette': { - 'primary': 'white', - 'accent': 'white' - }, - 'font': False, - 'logo': 'images/logo.svg', - 'favicon': 'assets/images/favicon.ico', 'static_templates': ['404.html'], 'extra': { 'now': int(time.mktime(datetime.datetime.now().timetuple())) # TODO better way to avoid caching diff --git a/website/robots.txt b/website/robots.txt index f9970836f18..fa3a68b6d69 100644 --- a/website/robots.txt +++ b/website/robots.txt @@ -12,4 +12,4 @@ Disallow: /docs/v3* Disallow: /cdn-cgi/ Allow: / Host: https://clickhouse.tech -Sitemap: https://clickhouse.tech/sitemap.xml +Sitemap: https://clickhouse.tech/sitemap-index.xml diff --git a/website/sitemap.xml b/website/sitemap-index.xml similarity index 92% rename from website/sitemap.xml rename to website/sitemap-index.xml index a147404ec6f..e53d6c29c54 100644 --- a/website/sitemap.xml +++ b/website/sitemap-index.xml @@ -22,6 +22,6 @@ https://clickhouse.tech/docs/fa/sitemap.xml - https://clickhouse.tech/sitemap_static.xml + https://clickhouse.tech/sitemap-static.xml diff --git a/website/sitemap_static.xml b/website/sitemap-static.xml similarity index 100% rename from website/sitemap_static.xml rename to website/sitemap-static.xml From 327d17ac6a62c4562c91e090e089c48d2b041b07 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 21 May 2020 22:46:03 +0300 Subject: [PATCH 538/738] Better --- src/Storages/IStorage.cpp | 129 +++++++++--------- src/Storages/IStorage.h | 102 +++++++------- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 6 +- src/Storages/MergeTree/MergeTreeData.cpp | 26 ++-- .../MergeTree/MergeTreeDataMergerMutator.cpp | 4 +- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 22 +-- .../MergeTree/MergeTreeDataWriter.cpp | 4 +- .../MergeTree/MergeTreeWhereOptimizer.cpp | 4 +- .../MergeTree/MergedBlockOutputStream.cpp | 2 +- .../ReplicatedMergeTreeTableMetadata.cpp | 8 +- src/Storages/StorageInMemoryMetadata.cpp | 15 +- src/Storages/StorageInMemoryMetadata.h | 14 +- 12 files changed, 173 insertions(+), 163 deletions(-) diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index 2f1f4902952..64e06a2534e 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -444,63 +444,21 @@ void IStorage::setPartitionKey(const StorageMetadataKeyField & partition_key_) partition_key = partition_key_; } +bool IStorage::hasPartitionKey() const +{ + return partition_key.expression != nullptr; +} + Names IStorage::getColumnsRequiredForPartitionKey() const { if (hasPartitionKey()) - return partition_key.expressions->getRequiredColumns(); + return partition_key.expression->getRequiredColumns(); return {}; } -Names IStorage::getColumnsRequiredForSampling() const +const StorageMetadataKeyField & IStorage::getSortingKey() const { - if (hasSamplingKey()) - return sampling_key.expressions->getRequiredColumns(); - return {}; -} - -bool IStorage::hasPartitionKey() const -{ - return partition_key.expressions != nullptr; -} - - -bool IStorage::supportsSampling() const -{ - return hasSamplingKey(); -} - -Names IStorage::getColumnsRequiredForSortingKey() const -{ - if (hasSortingKey()) - return sorting_key.expressions->getRequiredColumns(); - return {}; -} - - -Names IStorage::getSortingKeyColumns() const -{ - if (hasSortingKey()) - return sorting_key.expression_column_names; - return {}; -} - -const StorageMetadataKeyField & IStorage::getSamplingKey() const -{ - return sampling_key; -} -void IStorage::setSamplingKey(const StorageMetadataKeyField & sampling_key_) -{ - sampling_key = sampling_key_; -} - -bool IStorage::hasSamplingKey() const -{ - return sampling_key.expressions != nullptr; -} - -bool IStorage::hasSortingKey() const -{ - return sorting_key.expressions != nullptr; + return sorting_key; } void IStorage::setSortingKey(const StorageMetadataKeyField & sorting_key_) @@ -508,9 +466,23 @@ void IStorage::setSortingKey(const StorageMetadataKeyField & sorting_key_) sorting_key = sorting_key_; } -const StorageMetadataKeyField & IStorage::getSortingKey() const +bool IStorage::hasSortingKey() const { - return sorting_key; + return sorting_key.expression != nullptr; +} + +Names IStorage::getColumnsRequiredForSortingKey() const +{ + if (hasSortingKey()) + return sorting_key.expression->getRequiredColumns(); + return {}; +} + +Names IStorage::getSortingKeyColumns() const +{ + if (hasSortingKey()) + return sorting_key.column_names; + return {}; } const StorageMetadataKeyField & IStorage::getPrimaryKey() const @@ -523,21 +495,50 @@ void IStorage::setPrimaryKey(const StorageMetadataKeyField & primary_key_) primary_key = primary_key_; } -bool IStorage::hasPrimaryKey() const -{ - return primary_key.expressions != nullptr; -} - -Names IStorage::getColumnsRequiredForPrimaryKey() const -{ - if (primary_key.expressions != nullptr) - return primary_key.expressions->getRequiredColumns(); - return {}; -} - bool IStorage::isPrimaryKeyDefined() const { return primary_key.definition_ast != nullptr; } +bool IStorage::hasPrimaryKey() const +{ + return primary_key.expression != nullptr; +} + +Names IStorage::getColumnsRequiredForPrimaryKey() const +{ + if (hasPrimaryKey()) + return primary_key.expression->getRequiredColumns(); + return {}; +} + +Names IStorage::getPrimaryKeyColumns() const +{ + if (hasSortingKey()) + return primary_key.column_names; + return {}; +} + +const StorageMetadataKeyField & IStorage::getSamplingKey() const +{ + return sampling_key; +} + +void IStorage::setSamplingKey(const StorageMetadataKeyField & sampling_key_) +{ + sampling_key = sampling_key_; +} + +bool IStorage::hasSamplingKey() const +{ + return sampling_key.expression != nullptr; +} + +Names IStorage::getColumnsRequiredForSampling() const +{ + if (hasSamplingKey()) + return sampling_key.expression->getRequiredColumns(); + return {}; +} + } diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 6c91997a21c..62c94a22c02 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -101,7 +101,7 @@ public: virtual bool isView() const { return false; } /// Returns true if the storage supports queries with the SAMPLE section. - virtual bool supportsSampling() const; + virtual bool supportsSampling() const { return hasSamplingKey(); } /// Returns true if the storage supports queries with the FINAL section. virtual bool supportsFinal() const { return false; } @@ -184,23 +184,6 @@ public: /// thread-unsafe part. lockStructure must be acquired /// By default return empty list of columns. virtual NamesAndTypesList getVirtuals() const; - const StorageMetadataKeyField & getPartitionKey() const; - void setPartitionKey(const StorageMetadataKeyField & partition_key_); - bool hasPartitionKey() const; - - const StorageMetadataKeyField & getSamplingKey() const; - void setSamplingKey(const StorageMetadataKeyField & sampling_key_); - bool hasSamplingKey() const; - - const StorageMetadataKeyField & getSortingKey() const; - void setSortingKey(const StorageMetadataKeyField & sorting_key_); - bool hasSortingKey() const; - - const StorageMetadataKeyField & getPrimaryKey() const; - void setPrimaryKey(const StorageMetadataKeyField & primary_key_); - bool hasPrimaryKey() const; - - protected: /// still thread-unsafe part. void setIndices(IndicesDescription indices_); @@ -222,11 +205,6 @@ private: StorageMetadataKeyField sorting_key; StorageMetadataKeyField sampling_key; - //StorageMetadataField rows_ttl_entry; - - //std::vector column_ttl_entries; - //std::vector move_ttl_entries; - private: RWLockImpl::LockHolder tryLockTimed( const RWLock & rwlock, RWLockImpl::Type type, const String & query_id, const SettingSeconds & acquire_timeout) const; @@ -468,36 +446,66 @@ public: /// Returns data paths if storage supports it, empty vector otherwise. virtual Strings getDataPaths() const { return {}; } + /// Returns structure with partition key + const StorageMetadataKeyField & getPartitionKey() const; + /// Set partition key for storage (methods bellow, are just wrappers for this + /// struct) + void setPartitionKey(const StorageMetadataKeyField & partition_key_); /// Returns ASTExpressionList of partition key expression for storage or nullptr if there is none. - virtual ASTPtr getPartitionKeyAST() const { return partition_key.definition_ast; } - - /// Returns ASTExpressionList of sorting key expression for storage or nullptr if there is none. - virtual ASTPtr getSortingKeyAST() const { return sorting_key.definition_ast; } - - /// Returns ASTExpressionList of primary key expression for storage or nullptr if there is none. - virtual ASTPtr getPrimaryKeyAST() const { return primary_key.definition_ast; } - - /// Returns sampling expression AST for storage or nullptr if there is none. - virtual ASTPtr getSamplingKeyAST() const { return sampling_key.definition_ast; } - + ASTPtr getPartitionKeyAST() const { return partition_key.definition_ast; } + /// Storage has partition key + bool hasPartitionKey() const; /// Returns column names that need to be read to calculate partition key. - virtual Names getColumnsRequiredForPartitionKey() const; + Names getColumnsRequiredForPartitionKey() const; + + /// Returns structure with sorting key + const StorageMetadataKeyField & getSortingKey() const; + /// Set sorting key for storage (methods bellow, are just wrappers for this + /// struct) + void setSortingKey(const StorageMetadataKeyField & sorting_key_); + /// Returns ASTExpressionList of sorting key expression for storage or nullptr if there is none. + ASTPtr getSortingKeyAST() const { return sorting_key.definition_ast; } + /// Storage has sorting key + bool hasSortingKey() const; /// Returns column names that need to be read to calculate sorting key. - virtual Names getColumnsRequiredForSortingKey() const; - - /// Returns column names that need to be read to calculate primary key. - virtual Names getColumnsRequiredForPrimaryKey() const; - - /// Returns column names that need to be read to calculate sampling key. - virtual Names getColumnsRequiredForSampling() const; - - /// Returns column names that need to be read for FINAL to work. - virtual Names getColumnsRequiredForFinal() const { return getColumnsRequiredForSortingKey(); } - + Names getColumnsRequiredForSortingKey() const; /// Returns columns names in sorting key specified by user in ORDER BY /// expression. For example: 'a', 'x * y', 'toStartOfMonth(date)', etc. - virtual Names getSortingKeyColumns() const; + Names getSortingKeyColumns() const; + + /// Returns structure with primary key + const StorageMetadataKeyField & getPrimaryKey() const; + /// Set primary key for storage (methods bellow, are just wrappers for this + /// struct) + void setPrimaryKey(const StorageMetadataKeyField & primary_key_); + /// Returns ASTExpressionList of primary key expression for storage or nullptr if there is none. + ASTPtr getPrimaryKeyAST() const { return primary_key.definition_ast; } + /// Storage has user-defined (in CREATE query) sorting key + bool isPrimaryKeyDefined() const; + /// Storage has primary key (maybe part of some other key) + bool hasPrimaryKey() const; + /// Returns column names that need to be read to calculate primary key. + Names getColumnsRequiredForPrimaryKey() const; + /// Returns columns names in sorting key specified by. For example: 'a', 'x + /// * y', 'toStartOfMonth(date)', etc. + Names getPrimaryKeyColumns() const; + + /// Returns structure with sampling key + const StorageMetadataKeyField & getSamplingKey() const; + /// Set sampling key for storage (methods bellow, are just wrappers for this + /// struct) + void setSamplingKey(const StorageMetadataKeyField & sampling_key_); + /// Returns sampling expression AST for storage or nullptr if there is none. + ASTPtr getSamplingKeyAST() const { return sampling_key.definition_ast; } + /// Storage has sampling key + bool hasSamplingKey() const; + /// Returns column names that need to be read to calculate sampling key. + Names getColumnsRequiredForSampling() const; + + /// Returns column names that need to be read for FINAL to work. + Names getColumnsRequiredForFinal() const { return getColumnsRequiredForSortingKey(); } + /// Returns columns, which will be needed to calculate dependencies /// (skip indices, TTL expressions) if we update @updated_columns set of columns. diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 0487aabfed3..c1b1f06d840 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -439,7 +439,7 @@ void IMergeTreeDataPart::loadIndex() throw Exception("Index granularity is not loaded before index loading", ErrorCodes::LOGICAL_ERROR); const auto & primary_key = storage.getPrimaryKey(); - size_t key_size = primary_key.expression_column_names.size(); + size_t key_size = primary_key.column_names.size(); if (key_size) { @@ -845,7 +845,7 @@ void IMergeTreeDataPart::checkConsistencyBase() const if (!checksums.empty()) { - if (!storage.getPrimaryKey().expression_column_names.empty() && !checksums.files.count("primary.idx")) + if (storage.hasPrimaryKey() && !checksums.files.count("primary.idx")) throw Exception("No checksum for primary.idx", ErrorCodes::NO_FILE_IN_DATA_PART); if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) @@ -879,7 +879,7 @@ void IMergeTreeDataPart::checkConsistencyBase() const }; /// Check that the primary key index is not empty. - if (!storage.getPrimaryKey().expression_column_names.empty()) + if (storage.hasPrimaryKey()) check_file_not_empty(volume->getDisk(), path + "primary.idx"); if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index fbcaeb350ad..728bfa18bd8 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -155,7 +155,7 @@ MergeTreeData::MergeTreeData( StorageMetadataKeyField sampling_key = StorageMetadataKeyField::getKeyFromAST(metadata.sample_by_ast, getColumns(), global_context); const auto & primary_key = getPrimaryKey(); - if (!primary_key.sample_block.has(sampling_key.expression_column_names[0]) + if (!primary_key.sample_block.has(sampling_key.column_names[0]) && !attach && !settings->compatibility_allow_sampling_expression_not_in_primary_key) /// This is for backward compatibility. throw Exception("Sampling expression must be present in the primary key", ErrorCodes::BAD_ARGUMENTS); @@ -473,18 +473,18 @@ void MergeTreeData::setProperties(const StorageInMemoryMetadata & metadata, bool StorageMetadataKeyField new_sorting_key; new_sorting_key.definition_ast = metadata.order_by_ast; - new_sorting_key.expression_column_names = std::move(new_sorting_key_columns); - new_sorting_key.expression_ast = std::move(new_sorting_key_expr_list); - new_sorting_key.expressions = std::move(new_sorting_key_expr); + new_sorting_key.column_names = std::move(new_sorting_key_columns); + new_sorting_key.expression_list_ast = std::move(new_sorting_key_expr_list); + new_sorting_key.expression = std::move(new_sorting_key_expr); new_sorting_key.sample_block = std::move(new_sorting_key_sample); new_sorting_key.data_types = std::move(new_sorting_key_data_types); setSortingKey(new_sorting_key); StorageMetadataKeyField new_primary_key; new_primary_key.definition_ast = metadata.primary_key_ast; - new_primary_key.expression_column_names = std::move(new_primary_key_columns); - new_primary_key.expression_ast = std::move(new_primary_key_expr_list); - new_primary_key.expressions = std::move(new_primary_key_expr); + new_primary_key.column_names = std::move(new_primary_key_columns); + new_primary_key.expression_list_ast = std::move(new_primary_key_expr_list); + new_primary_key.expression = std::move(new_primary_key_expr); new_primary_key.sample_block = std::move(new_primary_key_sample); new_primary_key.data_types = std::move(new_primary_key_data_types); setPrimaryKey(new_primary_key); @@ -526,13 +526,13 @@ void MergeTreeData::initPartitionKey(ASTPtr partition_by_ast) { StorageMetadataKeyField new_partition_key = StorageMetadataKeyField::getKeyFromAST(partition_by_ast, getColumns(), global_context); - if (new_partition_key.expression_ast->children.empty()) + if (new_partition_key.expression_list_ast->children.empty()) return; - checkKeyExpression(*new_partition_key.expressions, new_partition_key.sample_block, "Partition"); + checkKeyExpression(*new_partition_key.expression, new_partition_key.sample_block, "Partition"); /// Add all columns used in the partition key to the min-max index. - const NamesAndTypesList & minmax_idx_columns_with_types = new_partition_key.expressions->getRequiredColumnsWithTypes(); + const NamesAndTypesList & minmax_idx_columns_with_types = new_partition_key.expression->getRequiredColumnsWithTypes(); minmax_idx_expr = std::make_shared(minmax_idx_columns_with_types, global_context); for (const NameAndTypePair & column : minmax_idx_columns_with_types) { @@ -633,7 +633,7 @@ void MergeTreeData::setTTLExpressions(const ColumnsDescription & new_columns, NameSet columns_ttl_forbidden; if (hasPartitionKey()) - for (const auto & col : getPartitionKey().expressions->getRequiredColumns()) + for (const auto & col : getColumnsRequiredForPartitionKey()) columns_ttl_forbidden.insert(col); if (hasSortingKey()) @@ -1437,7 +1437,7 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, const S if (hasSortingKey()) { - auto sorting_key_expr = getSortingKey().expressions; + auto sorting_key_expr = getSortingKey().expression; for (const ExpressionAction & action : sorting_key_expr->getActions()) { auto action_columns = action.getNeededColumns(); @@ -3100,7 +3100,7 @@ bool MergeTreeData::isPrimaryOrMinMaxKeyColumnPossiblyWrappedInFunctions(const A { const String column_name = node->getColumnName(); - for (const auto & name : getPrimaryKey().expression_column_names) + for (const auto & name : getPrimaryKeyColumns()) if (column_name == name) return true; diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index edbc00a2b54..17746509ff6 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -607,7 +607,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor NamesAndTypesList merging_columns; Names gathering_column_names, merging_column_names; extractMergingAndGatheringColumns( - storage_columns, data.getSortingKey().expressions, data.skip_indices, + storage_columns, data.getSortingKey().expression, data.skip_indices, data.merging_params, gathering_columns, gathering_column_names, merging_columns, merging_column_names); auto single_disk_volume = std::make_shared("volume_" + future_part.name, disk); @@ -726,7 +726,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor pipes.emplace_back(std::move(pipe)); } - Names sort_columns = data.getSortingKey().expression_column_names; + Names sort_columns = data.getSortingKeyColumns(); SortDescription sort_description; size_t sort_columns_size = sort_columns.size(); sort_description.reserve(sort_columns_size); diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index eb6fcb67f82..cf6048efcd4 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -223,9 +223,9 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( const Settings & settings = context.getSettingsRef(); const auto & primary_key = data.getPrimaryKey(); - Names primary_key_columns = primary_key.expression_column_names; + Names primary_key_columns = primary_key.column_names; - KeyCondition key_condition(query_info, context, primary_key_columns, primary_key.expressions); + KeyCondition key_condition(query_info, context, primary_key_columns, primary_key.expression); if (settings.force_primary_key && key_condition.alwaysUnknownOrTrue()) { @@ -461,14 +461,14 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( if (select.final()) { - sampling_key_ast = std::make_shared(sampling_key.expression_column_names[0]); + sampling_key_ast = std::make_shared(sampling_key.column_names[0]); /// We do spoil available_real_columns here, but it is not used later. - available_real_columns.emplace_back(sampling_key.expression_column_names[0], std::move(sampling_column_type)); + available_real_columns.emplace_back(sampling_key.column_names[0], std::move(sampling_column_type)); } if (has_lower_limit) { - if (!key_condition.addCondition(sampling_key.expression_column_names[0], Range::createLeftBounded(lower, true))) + if (!key_condition.addCondition(sampling_key.column_names[0], Range::createLeftBounded(lower, true))) throw Exception("Sampling column not in primary key", ErrorCodes::ILLEGAL_COLUMN); ASTPtr args = std::make_shared(); @@ -485,7 +485,7 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( if (has_upper_limit) { - if (!key_condition.addCondition(sampling_key.expression_column_names[0], Range::createRightBounded(upper, false))) + if (!key_condition.addCondition(sampling_key.column_names[0], Range::createRightBounded(upper, false))) throw Exception("Sampling column not in primary key", ErrorCodes::ILLEGAL_COLUMN); ASTPtr args = std::make_shared(); @@ -640,7 +640,7 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( else if (settings.optimize_read_in_order && query_info.input_sorting_info) { size_t prefix_size = query_info.input_sorting_info->order_key_prefix_descr.size(); - auto order_key_prefix_ast = data.getSortingKey().expression_ast->clone(); + auto order_key_prefix_ast = data.getSortingKey().expression_list_ast->clone(); order_key_prefix_ast->children.resize(prefix_size); auto syntax_result = SyntaxAnalyzer(context).analyze(order_key_prefix_ast, data.getColumns().getAllPhysical()); @@ -1025,7 +1025,7 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder( { SortDescription sort_description; for (size_t j = 0; j < input_sorting_info->order_key_prefix_descr.size(); ++j) - sort_description.emplace_back(data.getSortingKey().expression_column_names[j], + sort_description.emplace_back(data.getSortingKey().column_names[j], input_sorting_info->direction, 1); /// Drop temporary columns, added by 'sorting_key_prefix_expr' @@ -1098,11 +1098,11 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal( if (!out_projection) out_projection = createProjection(pipe, data); - pipe.addSimpleTransform(std::make_shared(pipe.getHeader(), data.getSortingKey().expressions)); + pipe.addSimpleTransform(std::make_shared(pipe.getHeader(), data.getSortingKey().expression)); pipes.emplace_back(std::move(pipe)); } - Names sort_columns = data.getSortingKey().expression_column_names; + Names sort_columns = data.getSortingKeyColumns(); SortDescription sort_description; size_t sort_columns_size = sort_columns.size(); sort_description.reserve(sort_columns_size); @@ -1300,7 +1300,7 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange( { auto index_block = std::make_shared(); for (size_t i = 0; i < used_key_size; ++i) - index_block->insert({index[i], primary_key.data_types[i], primary_key.expression_column_names[i]}); + index_block->insert({index[i], primary_key.data_types[i], primary_key.column_names[i]}); create_field_ref = [index_block](size_t row, size_t column, FieldRef & field) { diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index e79765c3603..48dbf66e72b 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -146,7 +146,7 @@ BlocksWithPartition MergeTreeDataWriter::splitBlockIntoParts(const Block & block Block block_copy = block; const auto & partition_key = data.getPartitionKey(); - partition_key.expressions->execute(block_copy); + partition_key.expression->execute(block_copy); ColumnRawPtrs partition_columns; partition_columns.reserve(partition_key.sample_block.columns()); @@ -262,7 +262,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa if (data.hasSortingKey() || data.hasSkipIndices()) data.sorting_key_and_skip_indices_expr->execute(block); - Names sort_columns = data.getSortingKey().expression_column_names; + Names sort_columns = data.getSortingKeyColumns(); SortDescription sort_description; size_t sort_columns_size = sort_columns.size(); sort_description.reserve(sort_columns_size); diff --git a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp index 759241669ff..03abc23cfd7 100644 --- a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp +++ b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp @@ -40,8 +40,8 @@ MergeTreeWhereOptimizer::MergeTreeWhereOptimizer( log{log_} { const auto & primary_key = data.getPrimaryKey(); - if (!primary_key.expression_column_names.empty()) - first_primary_key_column = primary_key.expression_column_names[0]; + if (!primary_key.column_names.empty()) + first_primary_key_column = primary_key.column_names[0]; calculateColumnSizes(data, queried_columns); determineArrayJoinedNames(query_info.query->as()); diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 2cc7cafb01d..68071008665 100644 --- a/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -161,7 +161,7 @@ void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Perm std::inserter(skip_indexes_column_names_set, skip_indexes_column_names_set.end())); Names skip_indexes_column_names(skip_indexes_column_names_set.begin(), skip_indexes_column_names_set.end()); - Block primary_key_block = getBlockAndPermute(block, storage.getPrimaryKey().expression_column_names, permutation); + Block primary_key_block = getBlockAndPermute(block, storage.getPrimaryKeyColumns(), permutation); Block skip_indexes_block = getBlockAndPermute(block, skip_indexes_column_names, permutation); writer->write(block, permutation, primary_key_block, skip_indexes_block); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp index d6ca26730cf..5925c4a6935 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp @@ -41,17 +41,17 @@ ReplicatedMergeTreeTableMetadata::ReplicatedMergeTreeTableMetadata(const MergeTr /// - When we have only ORDER BY, than store it in "primary key:" row of /metadata /// - When we have both, than store PRIMARY KEY in "primary key:" row and ORDER BY in "sorting key:" row of /metadata if (!data.isPrimaryKeyDefined()) - primary_key = formattedAST(data.getSortingKey().expression_ast); + primary_key = formattedAST(data.getSortingKey().expression_list_ast); else { - primary_key = formattedAST(data.getPrimaryKey().expression_ast); - sorting_key = formattedAST(data.getSortingKey().expression_ast); + primary_key = formattedAST(data.getPrimaryKey().expression_list_ast); + sorting_key = formattedAST(data.getSortingKey().expression_list_ast); } data_format_version = data.format_version; if (data.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) - partition_key = formattedAST(data.getPartitionKey().expression_ast); + partition_key = formattedAST(data.getPartitionKey().expression_list_ast); ttl_table = formattedAST(data.ttl_table_ast); diff --git a/src/Storages/StorageInMemoryMetadata.cpp b/src/Storages/StorageInMemoryMetadata.cpp index 0cda4d10f36..6c13033da1e 100644 --- a/src/Storages/StorageInMemoryMetadata.cpp +++ b/src/Storages/StorageInMemoryMetadata.cpp @@ -116,19 +116,20 @@ StorageMetadataKeyField StorageMetadataKeyField::getKeyFromAST(const ASTPtr & de { StorageMetadataKeyField result; result.definition_ast = definition_ast; - result.expression_ast = extractKeyExpressionList(definition_ast); + result.expression_list_ast = extractKeyExpressionList(definition_ast); - if (result.expression_ast->children.empty()) + if (result.expression_list_ast->children.empty()) return result; - const auto & children = result.expression_ast->children; + const auto & children = result.expression_list_ast->children; for (const auto & child : children) - result.expression_column_names.emplace_back(child->getColumnName()); + result.column_names.emplace_back(child->getColumnName()); { - auto syntax_result = SyntaxAnalyzer(context).analyze(result.expression_ast, columns.getAllPhysical()); - result.expressions = ExpressionAnalyzer(result.expression_ast->clone(), syntax_result, context).getActions(true); - result.sample_block = result.expressions->getSampleBlock(); + auto expr = result.expression_list_ast->clone(); + auto syntax_result = SyntaxAnalyzer(context).analyze(expr, columns.getAllPhysical()); + result.expression = ExpressionAnalyzer(expr, syntax_result, context).getActions(true); + result.sample_block = result.expression->getSampleBlock(); } for (size_t i = 0; i < result.sample_block.columns(); ++i) diff --git a/src/Storages/StorageInMemoryMetadata.h b/src/Storages/StorageInMemoryMetadata.h index f32982bb49d..37dce5a78b9 100644 --- a/src/Storages/StorageInMemoryMetadata.h +++ b/src/Storages/StorageInMemoryMetadata.h @@ -52,22 +52,22 @@ struct StorageMetadataKeyField ASTPtr definition_ast; /// ASTExpressionList with key fields, example: (x, toStartOfMonth(date))). - ASTPtr expression_ast; + ASTPtr expression_list_ast; - /// Expressions from expression_list_ast from expression_analyzer. Useful, + /// Expression from expression_list_ast created by ExpressionAnalyzer. Useful, /// when you need to get required columns for key, example: a, date, b. - ExpressionActionsPtr expressions; - - /// Column names in key definition, example: x, toStartOfMonth(date), a * b. - Names expression_column_names; + ExpressionActionsPtr expression; /// Sample block with key columns (names, types, empty column) Block sample_block; + /// Column names in key definition, example: x, toStartOfMonth(date), a * b. + Names column_names; + /// Types from sample block ordered in columns order. DataTypes data_types; - /// Parse key structure from key definition. Requires all columns available + /// Parse key structure from key definition. Requires all columns, available /// in storage. static StorageMetadataKeyField getKeyFromAST(const ASTPtr & definition_ast, const ColumnsDescription & columns, const Context & context); }; From 09034157cdfdad0d9fa68148309eec49d18f9b54 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 21 May 2020 22:53:46 +0300 Subject: [PATCH 539/738] Remove comments --- src/Storages/MergeTree/MergeTreeData.h | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 2b80027f47c..70d3a1c3514 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -334,15 +334,6 @@ public: /// See comments about methods below in IStorage interface StorageInMemoryMetadata getInMemoryMetadata() const override; - // ASTPtr getSortingKeyAST() const override { return sorting_key_expr_ast; } - //ASTPtr getPrimaryKeyAST() const override { return primary_key_expr_ast; } - - //Names getColumnsRequiredForPartitionKey() const override { return (partition_key_expr ? partition_key_expr->getRequiredColumns() : Names{}); } - //Names getColumnsRequiredForSortingKey() const override { return sorting_key_expr->getRequiredColumns(); } - //Names getColumnsRequiredForPrimaryKey() const override { return primary_key_expr->getRequiredColumns(); } - //Names getColumnsRequiredForFinal() const override { return sorting_key_expr->getRequiredColumns(); } - //Names getSortingKeyColumns() const override { return sorting_key_columns; } - ColumnDependencies getColumnDependencies(const NameSet & updated_columns) const override; StoragePolicyPtr getStoragePolicy() const override; @@ -525,7 +516,6 @@ public: */ static ASTPtr extractKeyExpressionList(const ASTPtr & node); - //bool hasPrimaryKey() const { return !primary_key_columns.empty(); } bool hasSkipIndices() const { return !skip_indices.empty(); } bool hasAnyColumnTTL() const { return !column_ttl_entries_by_name.empty(); } @@ -656,19 +646,6 @@ public: ExpressionActionsPtr primary_key_and_skip_indices_expr; ExpressionActionsPtr sorting_key_and_skip_indices_expr; - /// Names of sorting key columns in ORDER BY expression. For example: 'a', - /// 'x * y', 'toStartOfMonth(date)', etc. - //Names sorting_key_columns; - //ASTPtr sorting_key_expr_ast; - //ExpressionActionsPtr sorting_key_expr; - - /// Names of columns for primary key. - //Names primary_key_columns; - //ASTPtr primary_key_expr_ast; - //ExpressionActionsPtr primary_key_expr; - //Block primary_key_sample; - //DataTypes primary_key_data_types; - struct TTLEntry { ExpressionActionsPtr expression; @@ -728,8 +705,6 @@ protected: friend struct ReplicatedMergeTreeTableMetadata; friend class StorageReplicatedMergeTree; - ///ASTPtr order_by_ast; - ///ASTPtr primary_key_ast; ASTPtr ttl_table_ast; ASTPtr settings_ast; From c81dd49686693d1eeb29bffe2b95e0f256fbf246 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 21 May 2020 22:55:27 +0300 Subject: [PATCH 540/738] Try fix build. --- src/Processors/Executors/PullingPipelineExecutor.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Processors/Executors/PullingPipelineExecutor.h b/src/Processors/Executors/PullingPipelineExecutor.h index 67ef4f7cf71..878d66bd3d4 100644 --- a/src/Processors/Executors/PullingPipelineExecutor.h +++ b/src/Processors/Executors/PullingPipelineExecutor.h @@ -1,5 +1,6 @@ #pragma once #include +#include namespace DB { From 087d8ab020d0ef11783fe0620767a03666252dd8 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 21 May 2020 23:15:18 +0300 Subject: [PATCH 541/738] A follow-up for trace log overflow in perftests The start was in https://github.com/ClickHouse/ClickHouse/pull/11026 It turned out that the problem was due to the incorrect mutate() implementation that lead to quadratic amount of column copying. This problem has since been fixed. Remove the excessively verbose logging, and also change appendToBlock of LogElement's to accept mutable columns instead of accepting a block and mutating it on each call. It looks wasteful, even though it is almost a noop. --- src/Interpreters/MetricLog.cpp | 6 +----- src/Interpreters/MetricLog.h | 2 +- src/Interpreters/PartLog.cpp | 6 +----- src/Interpreters/PartLog.h | 2 +- src/Interpreters/QueryLog.cpp | 6 +----- src/Interpreters/QueryLog.h | 2 +- src/Interpreters/QueryThreadLog.cpp | 6 +----- src/Interpreters/QueryThreadLog.h | 2 +- src/Interpreters/SystemLog.h | 16 +++++----------- src/Interpreters/TextLog.cpp | 6 +----- src/Interpreters/TextLog.h | 2 +- src/Interpreters/TraceLog.cpp | 6 +----- src/Interpreters/TraceLog.h | 2 +- 13 files changed, 17 insertions(+), 47 deletions(-) diff --git a/src/Interpreters/MetricLog.cpp b/src/Interpreters/MetricLog.cpp index 96fe55c26e6..437d91e3743 100644 --- a/src/Interpreters/MetricLog.cpp +++ b/src/Interpreters/MetricLog.cpp @@ -35,10 +35,8 @@ Block MetricLogElement::createBlock() } -void MetricLogElement::appendToBlock(Block & block) const +void MetricLogElement::appendToBlock(MutableColumns & columns) const { - MutableColumns columns = block.mutateColumns(); - size_t column_idx = 0; columns[column_idx++]->insert(DateLUT::instance().toDayNum(event_time)); @@ -50,8 +48,6 @@ void MetricLogElement::appendToBlock(Block & block) const for (size_t i = 0, end = CurrentMetrics::end(); i < end; ++i) columns[column_idx++]->insert(current_metrics[i]); - - block.setColumns(std::move(columns)); } diff --git a/src/Interpreters/MetricLog.h b/src/Interpreters/MetricLog.h index a90ce923494..7774a45d7e1 100644 --- a/src/Interpreters/MetricLog.h +++ b/src/Interpreters/MetricLog.h @@ -25,7 +25,7 @@ struct MetricLogElement static std::string name() { return "MetricLog"; } static Block createBlock(); - void appendToBlock(Block & block) const; + void appendToBlock(MutableColumns & columns) const; }; diff --git a/src/Interpreters/PartLog.cpp b/src/Interpreters/PartLog.cpp index 7e04c5f6abb..91b7bd800b6 100644 --- a/src/Interpreters/PartLog.cpp +++ b/src/Interpreters/PartLog.cpp @@ -58,10 +58,8 @@ Block PartLogElement::createBlock() }; } -void PartLogElement::appendToBlock(Block & block) const +void PartLogElement::appendToBlock(MutableColumns & columns) const { - MutableColumns columns = block.mutateColumns(); - size_t i = 0; columns[i++]->insert(event_type); @@ -92,8 +90,6 @@ void PartLogElement::appendToBlock(Block & block) const columns[i++]->insert(error); columns[i++]->insert(exception); - - block.setColumns(std::move(columns)); } diff --git a/src/Interpreters/PartLog.h b/src/Interpreters/PartLog.h index b84138159a2..9c8bf414099 100644 --- a/src/Interpreters/PartLog.h +++ b/src/Interpreters/PartLog.h @@ -50,7 +50,7 @@ struct PartLogElement static std::string name() { return "PartLog"; } static Block createBlock(); - void appendToBlock(Block & block) const; + void appendToBlock(MutableColumns & columns) const; }; class IMergeTreeDataPart; diff --git a/src/Interpreters/QueryLog.cpp b/src/Interpreters/QueryLog.cpp index 1b104fe1a6c..007a118b4ce 100644 --- a/src/Interpreters/QueryLog.cpp +++ b/src/Interpreters/QueryLog.cpp @@ -85,10 +85,8 @@ Block QueryLogElement::createBlock() } -void QueryLogElement::appendToBlock(Block & block) const +void QueryLogElement::appendToBlock(MutableColumns & columns) const { - MutableColumns columns = block.mutateColumns(); - size_t i = 0; columns[i++]->insert(type); @@ -146,8 +144,6 @@ void QueryLogElement::appendToBlock(Block & block) const columns[i++]->insertDefault(); columns[i++]->insertDefault(); } - - block.setColumns(std::move(columns)); } void QueryLogElement::appendClientInfo(const ClientInfo & client_info, MutableColumns & columns, size_t & i) diff --git a/src/Interpreters/QueryLog.h b/src/Interpreters/QueryLog.h index ec14f5e97fb..ac7c2365a07 100644 --- a/src/Interpreters/QueryLog.h +++ b/src/Interpreters/QueryLog.h @@ -62,7 +62,7 @@ struct QueryLogElement static std::string name() { return "QueryLog"; } static Block createBlock(); - void appendToBlock(Block & block) const; + void appendToBlock(MutableColumns & columns) const; static void appendClientInfo(const ClientInfo & client_info, MutableColumns & columns, size_t & i); }; diff --git a/src/Interpreters/QueryThreadLog.cpp b/src/Interpreters/QueryThreadLog.cpp index f539a720449..97d35659d48 100644 --- a/src/Interpreters/QueryThreadLog.cpp +++ b/src/Interpreters/QueryThreadLog.cpp @@ -67,10 +67,8 @@ Block QueryThreadLogElement::createBlock() }; } -void QueryThreadLogElement::appendToBlock(Block & block) const +void QueryThreadLogElement::appendToBlock(MutableColumns & columns) const { - MutableColumns columns = block.mutateColumns(); - size_t i = 0; columns[i++]->insert(DateLUT::instance().toDayNum(event_time)); @@ -107,8 +105,6 @@ void QueryThreadLogElement::appendToBlock(Block & block) const columns[i++]->insertDefault(); columns[i++]->insertDefault(); } - - block.setColumns(std::move(columns)); } } diff --git a/src/Interpreters/QueryThreadLog.h b/src/Interpreters/QueryThreadLog.h index c50daa7bc88..7b4e8ca5787 100644 --- a/src/Interpreters/QueryThreadLog.h +++ b/src/Interpreters/QueryThreadLog.h @@ -43,7 +43,7 @@ struct QueryThreadLogElement static std::string name() { return "QueryThreadLog"; } static Block createBlock(); - void appendToBlock(Block & block) const; + void appendToBlock(MutableColumns & columns) const; }; diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h index b2a4eec7883..f32a05123b1 100644 --- a/src/Interpreters/SystemLog.h +++ b/src/Interpreters/SystemLog.h @@ -52,7 +52,7 @@ namespace DB static std::string name(); static Block createBlock(); - void appendToBlock(Block & block) const; + void appendToBlock(MutableColumns & columns) const; }; */ @@ -340,9 +340,8 @@ void SystemLog::savingThreadFunction() uint64_t to_flush_end = 0; { - LOG_TRACE(log, "Sleeping"); std::unique_lock lock(mutex); - const bool predicate = flush_event.wait_for(lock, + flush_event.wait_for(lock, std::chrono::milliseconds(flush_interval_milliseconds), [&] () { @@ -359,13 +358,6 @@ void SystemLog::savingThreadFunction() queue.swap(to_flush); exit_this_thread = is_shutdown; - - LOG_TRACE(log, "Woke up" - << (predicate ? " by condition" : " by timeout (" - + toString(flush_interval_milliseconds) + " ms)") - << ", " << to_flush.size() << " elements to flush" - << " up to " << to_flush_end - << (is_shutdown ? ", shutdown requested" : "")); } if (to_flush.empty()) @@ -399,8 +391,10 @@ void SystemLog::flushImpl(const std::vector & to_flush, prepareTable(); Block block = LogElement::createBlock(); + MutableColumns columns = block.mutateColumns(); for (const auto & elem : to_flush) - elem.appendToBlock(block); + elem.appendToBlock(columns); + block.setColumns(std::move(columns)); /// We write to table indirectly, using InterpreterInsertQuery. /// This is needed to support DEFAULT-columns in table. diff --git a/src/Interpreters/TextLog.cpp b/src/Interpreters/TextLog.cpp index 35e027616f6..d166b24ef4f 100644 --- a/src/Interpreters/TextLog.cpp +++ b/src/Interpreters/TextLog.cpp @@ -46,10 +46,8 @@ Block TextLogElement::createBlock() }; } -void TextLogElement::appendToBlock(Block & block) const +void TextLogElement::appendToBlock(MutableColumns & columns) const { - MutableColumns columns = block.mutateColumns(); - size_t i = 0; columns[i++]->insert(DateLUT::instance().toDayNum(event_time)); @@ -68,8 +66,6 @@ void TextLogElement::appendToBlock(Block & block) const columns[i++]->insert(source_file); columns[i++]->insert(source_line); - - block.setColumns(std::move(columns)); } TextLog::TextLog(Context & context_, const String & database_name_, diff --git a/src/Interpreters/TextLog.h b/src/Interpreters/TextLog.h index 73c38429662..d5d1610dfb5 100644 --- a/src/Interpreters/TextLog.h +++ b/src/Interpreters/TextLog.h @@ -25,7 +25,7 @@ struct TextLogElement static std::string name() { return "TextLog"; } static Block createBlock(); - void appendToBlock(Block & block) const; + void appendToBlock(MutableColumns & columns) const; }; class TextLog : public SystemLog diff --git a/src/Interpreters/TraceLog.cpp b/src/Interpreters/TraceLog.cpp index 2367cc56ac0..3df182a61ca 100644 --- a/src/Interpreters/TraceLog.cpp +++ b/src/Interpreters/TraceLog.cpp @@ -35,10 +35,8 @@ Block TraceLogElement::createBlock() }; } -void TraceLogElement::appendToBlock(Block & block) const +void TraceLogElement::appendToBlock(MutableColumns & columns) const { - MutableColumns columns = block.mutateColumns(); - size_t i = 0; columns[i++]->insert(DateLUT::instance().toDayNum(event_time)); @@ -50,6 +48,4 @@ void TraceLogElement::appendToBlock(Block & block) const columns[i++]->insertData(query_id.data(), query_id.size()); columns[i++]->insert(trace); columns[i++]->insert(size); - - block.setColumns(std::move(columns)); } diff --git a/src/Interpreters/TraceLog.h b/src/Interpreters/TraceLog.h index c882ae69420..f97cdaaec40 100644 --- a/src/Interpreters/TraceLog.h +++ b/src/Interpreters/TraceLog.h @@ -24,7 +24,7 @@ struct TraceLogElement static std::string name() { return "TraceLog"; } static Block createBlock(); - void appendToBlock(Block & block) const; + void appendToBlock(MutableColumns & columns) const; }; class TraceLog : public SystemLog From 7b747fabe20c91c24bc6328f8a605cff47cd6fb9 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 22 May 2020 02:03:41 +0300 Subject: [PATCH 542/738] performance comparison --- docker/test/performance-comparison/compare.sh | 60 ++++++++++++++----- 1 file changed, 45 insertions(+), 15 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 0c0289d811c..5731b3c88af 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -217,13 +217,8 @@ function get_profiles clickhouse-client --port 9002 --query "select 1" } -# Build and analyze randomization distribution for all queries. -function analyze_queries +function build_log_column_definitions { -rm -v analyze-commands.txt analyze-errors.log all-queries.tsv unstable-queries.tsv ./*-report.tsv raw-queries.tsv ||: -rm -rfv analyze ||: -mkdir analyze ||: - # FIXME This loop builds column definitons from TSVWithNamesAndTypes in an # absolutely atrocious way. This should be done by the file() function itself. for x in {right,left}-{addresses,{query,query-thread,trace,metric}-log}.tsv @@ -233,6 +228,16 @@ do <(sed -n '2{s/\t/\n/g;p;q}' "$x" ) \ | tr '\n' ', ' | sed 's/,$//' > "$x.columns" done +} + +# Build and analyze randomization distribution for all queries. +function analyze_queries +{ +rm -v analyze-commands.txt analyze-errors.log all-queries.tsv unstable-queries.tsv ./*-report.tsv raw-queries.tsv ||: +rm -rfv analyze ||: +mkdir analyze ||: + +build_log_column_definitions # Split the raw test output into files suitable for analysis. IFS=$'\n' @@ -278,6 +283,7 @@ create table query_metrics engine File(TSV, -- do not add header -- will parse w ) query_logs right join query_runs using (query_id, version) + order by test, query_index ; " @@ -291,8 +297,8 @@ query_index=1 IFS=$'\n' for prefix in $(cut -f1,2 "analyze/query-run-metrics.tsv" | sort | uniq) do - file="analyze/q$query_index.tmp" - grep -F "$prefix " "analyze/query-run-metrics.tsv" > "$file" & + file="analyze/$(echo "$prefix" | sed 's/\t/_/g').tmp" + grep "^$prefix " "analyze/query-run-metrics.tsv" > "$file" & printf "%s\0\n" \ "clickhouse-local \ --file \"$file\" \ @@ -301,13 +307,11 @@ do >> \"analyze/query-reports.tsv\"" \ 2>> analyze/errors.log \ >> analyze/commands.txt - - query_index=$((query_index + 1)) done wait unset IFS -parallel --joblog analyze/parallel-log.txt --null < analyze/commands.txt +parallel --joblog analyze/parallel-log.txt --null < analyze/commands.txt 2>> analyze/errors.log } # Analyze results @@ -318,6 +322,8 @@ mkdir report ||: rm ./*.{rep,svg} test-times.tsv test-dump.tsv unstable.tsv unstable-query-ids.tsv unstable-query-metrics.tsv changed-perf.tsv unstable-tests.tsv unstable-queries.tsv bad-tests.tsv slow-on-client.tsv all-queries.tsv ||: +build_log_column_definitions + cat analyze/errors.log >> report/errors.log ||: cat profile-errors.log >> report/errors.log ||: @@ -570,8 +576,16 @@ create table stacks engine File(TSV, 'stacks.$version.rep') as select -- first goes the key used to split the file with grep test, query_index, any(query_display_name), + -- next go the stacks in flamegraph format: 'func1;...;funcN count' arrayStringConcat( - arrayMap(x -> joinGet(addresses_join_$version, 'name', x), + arrayMap(addr -> replaceRegexpOne( + joinGet(addresses_join_$version, 'name', addr), + -- This function is at the base of the stack, and its name changes + -- surprisingly often between builds, e.g. '__clone' or 'clone' or + -- even '__GI__clone'. This breaks differential flame graphs, so + -- filter it out here. + '^clone\\.S.*', 'clone.S (name filtered by comparison script)' + ), arrayReverse(trace) ), ';' @@ -580,6 +594,7 @@ create table stacks engine File(TSV, 'stacks.$version.rep') as from trace_log join unstable_query_runs using query_id group by test, query_index, trace + order by test, query_index, trace ; " 2> >(tee -a report/errors.log 1>&2) # do not run in parallel because they use the same data dir for StorageJoins which leads to weird errors. done @@ -592,22 +607,37 @@ do for query in $(cut -d' ' -f1,2,3 "stacks.$version.rep" | sort | uniq) do query_file=$(echo "$query" | cut -c-120 | sed 's/[/ ]/_/g') + echo "$query_file" >> report/query-files.txt # Build separate .svg flamegraph for each query. + # -F is somewhat unsafe because it might match not the beginning of the + # string, but this is unlikely and escaping the query for grep is a pain. grep -F "$query " "stacks.$version.rep" \ - | cut -d' ' -f 2- \ + | cut -f 4- \ | sed 's/\t/ /g' \ | tee "$query_file.stacks.$version.rep" \ - | ~/fg/flamegraph.pl > "$query_file.$version.svg" & + | ~/fg/flamegraph.pl --hash > "$query_file.$version.svg" & # Copy metric stats into separate files as well. + # Ditto the above comment about -F. grep -F "$query " "metric-deviation.$version.rep" \ - | cut -f2- > "$query_file.$version.metrics.rep" & + | cut -f4- > "$query_file.$version.metrics.rep" & done done wait unset IFS +# Create differential flamegraphs. +IFS=$'\n' +for query_file in $(cat report/query-files.txt) +do + ~/fg/difffolded.pl "$query_file.stacks.left.rep" "$query_file.stacks.right.rep" \ + | tee "$query_file.stacks.diff.rep" \ + | ~/fg/flamegraph.pl > "$query_file.diff.svg" & +done +unset IFS +wait + # Remember that grep sets error code when nothing is found, hence the bayan # operator. grep -H -m2 -i '\(Exception\|Error\):[^:]' ./*-err.log | sed 's/:/\t/' >> run-errors.tsv ||: From 322d731d28c1dab7b722867a98d6949d2dbef37a Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 22 May 2020 03:01:35 +0300 Subject: [PATCH 543/738] Remove extra copying from addressToLine The line is already serialized to the same arena, so there is no need to copy it. Also add this function to query profiler test, so that it has some coverage. --- src/Functions/addressToLine.cpp | 4 +--- tests/queries/0_stateless/00974_query_profiler.sql | 8 ++++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Functions/addressToLine.cpp b/src/Functions/addressToLine.cpp index 6489fac3371..420938bc4b1 100644 --- a/src/Functions/addressToLine.cpp +++ b/src/Functions/addressToLine.cpp @@ -118,9 +118,7 @@ private: writeChar(':', out); writeIntText(location.line, out); - StringRef out_str = out.finish(); - out_str.data = arena.insert(out_str.data, out_str.size); - return out_str; + return out.finish(); } else { diff --git a/tests/queries/0_stateless/00974_query_profiler.sql b/tests/queries/0_stateless/00974_query_profiler.sql index 9e2723c67d8..c5c073abfa9 100644 --- a/tests/queries/0_stateless/00974_query_profiler.sql +++ b/tests/queries/0_stateless/00974_query_profiler.sql @@ -5,7 +5,9 @@ SET log_queries = 1; SELECT sleep(0.5), ignore('test real time query profiler'); SET log_queries = 0; SYSTEM FLUSH LOGS; -WITH addressToSymbol(arrayJoin(trace)) AS symbol SELECT count() > 0 FROM system.trace_log t WHERE query_id = (SELECT query_id FROM system.query_log WHERE query LIKE '%test real time query profiler%' AND query NOT LIKE '%system%' ORDER BY event_time DESC LIMIT 1) AND symbol LIKE '%FunctionSleep%'; + +WITH addressToLine(arrayJoin(trace) AS addr) || '#' || demangle(addressToSymbol(addr)) AS symbol +SELECT count() > 0 FROM system.trace_log t WHERE query_id = (SELECT query_id FROM system.query_log WHERE query LIKE '%test real time query profiler%' AND query NOT LIKE '%system%' ORDER BY event_time DESC LIMIT 1) AND symbol LIKE '%FunctionSleep%'; SET query_profiler_real_time_period_ns = 0; SET query_profiler_cpu_time_period_ns = 1000000; @@ -13,4 +15,6 @@ SET log_queries = 1; SELECT count(), ignore('test cpu time query profiler') FROM numbers(1000000000); SET log_queries = 0; SYSTEM FLUSH LOGS; -WITH addressToSymbol(arrayJoin(trace)) AS symbol SELECT count() > 0 FROM system.trace_log t WHERE query_id = (SELECT query_id FROM system.query_log WHERE query LIKE '%test cpu time query profiler%' AND query NOT LIKE '%system%' ORDER BY event_time DESC LIMIT 1) AND symbol LIKE '%Source%'; + +WITH addressToLine(arrayJoin(trace) AS addr) || '#' || demangle(addressToSymbol(addr)) AS symbol +SELECT count() > 0 FROM system.trace_log t WHERE query_id = (SELECT query_id FROM system.query_log WHERE query LIKE '%test cpu time query profiler%' AND query NOT LIKE '%system%' ORDER BY event_time DESC LIMIT 1) AND symbol LIKE '%Source%'; From 432648ca1ee3a5055bc6cd91bc35dbd88457b952 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 22 May 2020 03:51:42 +0300 Subject: [PATCH 544/738] performance comparison --- docker/test/performance-comparison/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/performance-comparison/entrypoint.sh b/docker/test/performance-comparison/entrypoint.sh index 3b8a24ad5da..b425acd1a72 100755 --- a/docker/test/performance-comparison/entrypoint.sh +++ b/docker/test/performance-comparison/entrypoint.sh @@ -82,7 +82,7 @@ if [ "$REF_PR" == "" ]; then echo Reference PR is not specified ; exit 1 ; fi ) | tee right-commit.txt # Prepare the list of changed tests for use by compare.sh -git -C ch diff --name-only "$SHA_TO_TEST" "$(git -C ch merge-base "$SHA_TO_TEST"~ master)" -- tests/performance | tee changed-tests.txt +git -C ch diff --name-only "$REF_SHA" "$SHA_TO_TEST" -- tests/performance | tee changed-tests.txt # Set python output encoding so that we can print queries with Russian letters. export PYTHONIOENCODING=utf-8 From 3073fc9eb00f72c87e39109df43e644b586cf88f Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 22 May 2020 05:51:26 +0300 Subject: [PATCH 545/738] performance comparison --- docker/test/performance-comparison/report.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docker/test/performance-comparison/report.py b/docker/test/performance-comparison/report.py index 83c5e7344f7..b171603700d 100755 --- a/docker/test/performance-comparison/report.py +++ b/docker/test/performance-comparison/report.py @@ -182,6 +182,16 @@ if args.report == 'main': print_tested_commits() + run_error_rows = tsvRows('run-errors.tsv') + error_tests += len(run_error_rows) + printSimpleTable('Run errors', ['Test', 'Error'], run_error_rows) + + slow_on_client_rows = tsvRows('report/slow-on-client.tsv') + error_tests += len(slow_on_client_rows) + printSimpleTable('Slow on client', + ['Client time, s', 'Server time, s', 'Ratio', 'Query'], + slow_on_client_rows) + def print_changes(): rows = tsvRows('report/changed-perf.tsv') if not rows: @@ -221,12 +231,6 @@ if args.report == 'main': print_changes() - slow_on_client_rows = tsvRows('report/slow-on-client.tsv') - error_tests += len(slow_on_client_rows) - printSimpleTable('Slow on client', - ['Client time, s', 'Server time, s', 'Ratio', 'Query'], - slow_on_client_rows) - def print_unstable_queries(): global unstable_queries global very_unstable_queries @@ -265,10 +269,6 @@ if args.report == 'main': print_unstable_queries() - run_error_rows = tsvRows('run-errors.tsv') - error_tests += len(run_error_rows) - printSimpleTable('Run errors', ['Test', 'Error'], run_error_rows) - skipped_tests_rows = tsvRows('analyze/skipped-tests.tsv') printSimpleTable('Skipped tests', ['Test', 'Reason'], skipped_tests_rows) From 6a670ba7ecb73f9b5dc2ad019b58644f5b27bef2 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Fri, 22 May 2020 08:51:46 +0300 Subject: [PATCH 546/738] Update index.md --- docs/en/faq/index.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/en/faq/index.md b/docs/en/faq/index.md index 11395301904..bdbd59f7880 100644 --- a/docs/en/faq/index.md +++ b/docs/en/faq/index.md @@ -1,6 +1,8 @@ --- toc_folder_title: F.A.Q. toc_priority: 76 +toc_title: hidden +toc_hidden: true --- From caa788b4f8035f25835299ddc1fb02801c07ea5c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 22 May 2020 07:28:01 +0000 Subject: [PATCH 547/738] Bump six from 1.14.0 to 1.15.0 in /docs/tools Bumps [six](https://github.com/benjaminp/six) from 1.14.0 to 1.15.0. - [Release notes](https://github.com/benjaminp/six/releases) - [Changelog](https://github.com/benjaminp/six/blob/master/CHANGES) - [Commits](https://github.com/benjaminp/six/compare/1.14.0...1.15.0) Signed-off-by: dependabot-preview[bot] --- docs/tools/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tools/requirements.txt b/docs/tools/requirements.txt index adb68cfb2d7..4d4e9f98780 100644 --- a/docs/tools/requirements.txt +++ b/docs/tools/requirements.txt @@ -30,7 +30,7 @@ PyYAML==5.3.1 repackage==0.7.3 requests==2.23.0 singledispatch==3.4.0.3 -six==1.14.0 +six==1.15.0 soupsieve==2.0.1 termcolor==1.1.0 tornado==5.1.1 From 02cb08ea2f6c267c1a3cfb246d2cbac5d523380d Mon Sep 17 00:00:00 2001 From: Vladimir Chebotarev Date: Tue, 19 May 2020 13:00:40 +0300 Subject: [PATCH 548/738] Fixed parsing of S3 URL. --- src/IO/S3Common.cpp | 33 +++++++++++++++++++++++---------- src/Storages/StorageS3.cpp | 4 +++- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp index 599b12d8593..8989634b8c3 100644 --- a/src/IO/S3Common.cpp +++ b/src/IO/S3Common.cpp @@ -9,10 +9,11 @@ # include # include # include +# include +# include # include # include - namespace { const std::pair & convertLogLevel(Aws::Utils::Logging::LogLevel log_level) @@ -90,8 +91,23 @@ namespace S3 const String & secret_access_key) { Aws::Client::ClientConfiguration cfg; + if (!endpoint.empty()) + { + static const RE2 region_pattern(R"(s3\.([a-z0-9\-]+)\.amazonaws\.)"); + cfg.endpointOverride = endpoint; + Poco::URI uri(endpoint); + if (uri.getScheme() == "http") + cfg.scheme = Aws::Http::Scheme::HTTP; + + String region; + if (re2::RE2::PartialMatch(uri.getHost(), region_pattern, ®ion)) + { + boost::algorithm::to_lower(region); + cfg.region = region; + } + } return create(cfg, access_key_id, secret_access_key); } @@ -116,11 +132,11 @@ namespace S3 /// Case when bucket name represented in domain name of S3 URL. /// E.g. (https://bucket-name.s3.Region.amazonaws.com/key) /// https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html#virtual-hosted-style-access - static const RE2 virtual_hosted_style_pattern(R"((.+\.)?s3[.\-][a-z0-9\-.]+)"); + static const RE2 virtual_hosted_style_pattern(R"((.+)\.s3[.\-][a-z0-9\-.:]+)"); /// Case when bucket name and key represented in path of S3 URL. /// E.g. (https://s3.Region.amazonaws.com/bucket-name/key) /// https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html#path-style-access - static const RE2 path_style_pattern("([^/]+)/(.*)"); + static const RE2 path_style_pattern("^/([^/]*)/(.*)"); uri = uri_; @@ -131,19 +147,16 @@ namespace S3 if (re2::RE2::FullMatch(uri.getAuthority(), virtual_hosted_style_pattern, &bucket)) { - if (!bucket.empty()) - bucket.pop_back(); /// Remove '.' character from the end of the bucket name. - /// S3 specification requires at least 3 and at most 63 characters in bucket name. /// https://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-s3-bucket-naming-requirements.html if (bucket.length() < 3 || bucket.length() > 63) throw Exception( - "Bucket name length out of bounds in S3 URI: " + bucket + " (" + uri.toString() + ")", ErrorCodes::BAD_ARGUMENTS); + "Bucket name length is out of bounds in virtual hosted style S3 URI: " + bucket + " (" + uri.toString() + ")", ErrorCodes::BAD_ARGUMENTS); /// Remove leading '/' from path to extract key. key = uri.getPath().substr(1); if (key.empty() || key == "/") - throw Exception("Key name is empty in S3 URI: " + key + " (" + uri.toString() + ")", ErrorCodes::BAD_ARGUMENTS); + throw Exception("Key name is empty in virtual hosted style S3 URI: " + key + " (" + uri.toString() + ")", ErrorCodes::BAD_ARGUMENTS); } else if (re2::RE2::PartialMatch(uri.getPath(), path_style_pattern, &bucket, &key)) { @@ -151,10 +164,10 @@ namespace S3 /// https://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-s3-bucket-naming-requirements.html if (bucket.length() < 3 || bucket.length() > 63) throw Exception( - "Bucket name length out of bounds in S3 URI: " + bucket + " (" + uri.toString() + ")", ErrorCodes::BAD_ARGUMENTS); + "Bucket name length is out of bounds in path style S3 URI: " + bucket + " (" + uri.toString() + ")", ErrorCodes::BAD_ARGUMENTS); if (key.empty() || key == "/") - throw Exception("Key name is empty in S3 URI: " + key + " (" + uri.toString() + ")", ErrorCodes::BAD_ARGUMENTS); + throw Exception("Key name is empty in path style S3 URI: " + key + " (" + uri.toString() + ")", ErrorCodes::BAD_ARGUMENTS); } else throw Exception("Bucket or key name are invalid in S3 URI: " + uri.toString(), ErrorCodes::BAD_ARGUMENTS); diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index a568152e055..4946240deb5 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -244,7 +244,9 @@ Strings listFilesWithRegexpMatching(Aws::S3::S3Client & client, const S3::URI & { throw Exception("Could not list objects in bucket " + quoteString(request.GetBucket()) + " with prefix " + quoteString(request.GetPrefix()) - + ", page " + std::to_string(page), ErrorCodes::S3_ERROR); + + ", page " + std::to_string(page) + + ", S3 exception " + outcome.GetError().GetExceptionName() + " " + outcome.GetError().GetMessage() + , ErrorCodes::S3_ERROR); } for (const auto & row : outcome.GetResult().GetContents()) From cbcdb6219eb14a0f49475edc042def6ec3e988d1 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 22 May 2020 11:17:57 +0300 Subject: [PATCH 549/738] Slighty more verbose comment --- src/Storages/AlterCommands.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Storages/AlterCommands.h b/src/Storages/AlterCommands.h index a42e7cce646..82090cb1aaf 100644 --- a/src/Storages/AlterCommands.h +++ b/src/Storages/AlterCommands.h @@ -163,8 +163,9 @@ public: bool isCommentAlter() const; /// Return mutation commands which some storages may execute as part of - /// alter. If alter can be performed is pure metadata update, than result is - /// empty. + /// alter. If alter can be performed as pure metadata update, than result is + /// empty. If some TTL changes happened than, depending on materialize_ttl + /// additional mutation command (MATERIALIZE_TTL) will be returned. MutationCommands getMutationCommands(StorageInMemoryMetadata metadata, bool materialize_ttl) const; }; From 98f31374d549c12ec71e2930739364aff1a43cd3 Mon Sep 17 00:00:00 2001 From: Vladimir Chebotarev Date: Wed, 20 May 2020 16:47:53 +0300 Subject: [PATCH 550/738] Added improved S3 URL parsing to `DiskS3`. --- src/Disks/S3/registerDiskS3.cpp | 1 + src/IO/S3Common.cpp | 51 ++++++++++++++++++++------------- src/IO/S3Common.h | 4 +++ src/Storages/StorageS3.cpp | 2 +- 4 files changed, 37 insertions(+), 21 deletions(-) diff --git a/src/Disks/S3/registerDiskS3.cpp b/src/Disks/S3/registerDiskS3.cpp index 81879800b7f..1912579275e 100644 --- a/src/Disks/S3/registerDiskS3.cpp +++ b/src/Disks/S3/registerDiskS3.cpp @@ -124,6 +124,7 @@ void registerDiskS3(DiskFactory & factory) auto client = S3::ClientFactory::instance().create( cfg, + uri.is_virtual_hosted_style, config.getString(config_prefix + ".access_key_id", ""), config.getString(config_prefix + ".secret_access_key", "")); diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp index 8989634b8c3..a6b518afcdc 100644 --- a/src/IO/S3Common.cpp +++ b/src/IO/S3Common.cpp @@ -87,43 +87,48 @@ namespace S3 /// This method is not static because it requires ClientFactory to be initialized. std::shared_ptr ClientFactory::create( // NOLINT const String & endpoint, + bool is_virtual_hosted_style, const String & access_key_id, const String & secret_access_key) { Aws::Client::ClientConfiguration cfg; if (!endpoint.empty()) - { - static const RE2 region_pattern(R"(s3\.([a-z0-9\-]+)\.amazonaws\.)"); - cfg.endpointOverride = endpoint; - Poco::URI uri(endpoint); - if (uri.getScheme() == "http") - cfg.scheme = Aws::Http::Scheme::HTTP; - String region; - if (re2::RE2::PartialMatch(uri.getHost(), region_pattern, ®ion)) - { - boost::algorithm::to_lower(region); - cfg.region = region; - } - } - - return create(cfg, access_key_id, secret_access_key); + return create(cfg, is_virtual_hosted_style, access_key_id, secret_access_key); } std::shared_ptr ClientFactory::create( // NOLINT Aws::Client::ClientConfiguration & cfg, + bool is_virtual_hosted_style, const String & access_key_id, const String & secret_access_key) { Aws::Auth::AWSCredentials credentials(access_key_id, secret_access_key); + Aws::Client::ClientConfiguration client_configuration = cfg; + + if (!client_configuration.endpointOverride.empty()) + { + static const RE2 region_pattern(R"(s3[.\-]([a-z0-9\-]+)\.amazonaws\.)"); + Poco::URI uri(client_configuration.endpointOverride); + if (uri.getScheme() == "http") + client_configuration.scheme = Aws::Http::Scheme::HTTP; + + String region; + if (re2::RE2::PartialMatch(uri.getHost(), region_pattern, ®ion)) + { + boost::algorithm::to_lower(region); + client_configuration.region = region; + } + } + return std::make_shared( credentials, // Aws credentials. - std::move(cfg), // Client configuration. + std::move(client_configuration), // Client configuration. Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, // Sign policy. - cfg.endpointOverride.empty() // Use virtual addressing only if endpoint is not specified. + is_virtual_hosted_style || cfg.endpointOverride.empty() // Use virtual addressing if endpoint is not specified. ); } @@ -132,7 +137,7 @@ namespace S3 /// Case when bucket name represented in domain name of S3 URL. /// E.g. (https://bucket-name.s3.Region.amazonaws.com/key) /// https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html#virtual-hosted-style-access - static const RE2 virtual_hosted_style_pattern(R"((.+)\.s3[.\-][a-z0-9\-.:]+)"); + static const RE2 virtual_hosted_style_pattern(R"((.+)\.(s3[.\-][a-z0-9\-.:]+))"); /// Case when bucket name and key represented in path of S3 URL. /// E.g. (https://s3.Region.amazonaws.com/bucket-name/key) /// https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html#path-style-access @@ -143,10 +148,13 @@ namespace S3 if (uri.getHost().empty()) throw Exception("Host is empty in S3 URI: " + uri.toString(), ErrorCodes::BAD_ARGUMENTS); - endpoint = uri.getScheme() + "://" + uri.getAuthority(); + String endpoint_authority_from_uri; - if (re2::RE2::FullMatch(uri.getAuthority(), virtual_hosted_style_pattern, &bucket)) + if (re2::RE2::FullMatch(uri.getAuthority(), virtual_hosted_style_pattern, &bucket, &endpoint_authority_from_uri)) { + is_virtual_hosted_style = true; + endpoint = uri.getScheme() + "://" + endpoint_authority_from_uri; + /// S3 specification requires at least 3 and at most 63 characters in bucket name. /// https://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-s3-bucket-naming-requirements.html if (bucket.length() < 3 || bucket.length() > 63) @@ -160,6 +168,9 @@ namespace S3 } else if (re2::RE2::PartialMatch(uri.getPath(), path_style_pattern, &bucket, &key)) { + is_virtual_hosted_style = false; + endpoint = uri.getScheme() + "://" + uri.getAuthority(); + /// S3 specification requires at least 3 and at most 63 characters in bucket name. /// https://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-s3-bucket-naming-requirements.html if (bucket.length() < 3 || bucket.length() > 63) diff --git a/src/IO/S3Common.h b/src/IO/S3Common.h index b4cc9031b81..84795a4b39a 100644 --- a/src/IO/S3Common.h +++ b/src/IO/S3Common.h @@ -25,11 +25,13 @@ public: std::shared_ptr create( const String & endpoint, + bool is_virtual_hosted_style, const String & access_key_id, const String & secret_access_key); std::shared_ptr create( Aws::Client::ClientConfiguration & cfg, + bool is_virtual_hosted_style, const String & access_key_id, const String & secret_access_key); private: @@ -54,6 +56,8 @@ struct URI String bucket; String key; + bool is_virtual_hosted_style; + explicit URI(const Poco::URI & uri_); }; diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 4946240deb5..b8710de507b 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -201,7 +201,7 @@ StorageS3::StorageS3( , format_name(format_name_) , min_upload_part_size(min_upload_part_size_) , compression_method(compression_method_) - , client(S3::ClientFactory::instance().create(uri_.endpoint, access_key_id_, secret_access_key_)) + , client(S3::ClientFactory::instance().create(uri_.endpoint, uri_.is_virtual_hosted_style, access_key_id_, secret_access_key_)) { context_global.getRemoteHostFilter().checkURL(uri_.uri); setColumns(columns_); From 86da115018f290e66c7c097149bb079a2c94af9c Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 22 May 2020 11:40:02 +0300 Subject: [PATCH 551/738] performance comparison --- docker/test/performance-comparison/download.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docker/test/performance-comparison/download.sh b/docker/test/performance-comparison/download.sh index dd030e9f39d..8e09fac77f8 100755 --- a/docker/test/performance-comparison/download.sh +++ b/docker/test/performance-comparison/download.sh @@ -46,7 +46,13 @@ function download done mkdir ~/fg ||: - cd ~/fg && wget -nv -nd -c "https://raw.githubusercontent.com/brendangregg/FlameGraph/master/flamegraph.pl" && chmod +x ~/fg/flamegraph.pl & + ( + cd ~/fg + wget -nv -nd -c "https://raw.githubusercontent.com/brendangregg/FlameGraph/master/flamegraph.pl" + wget -nv -nd -c "https://raw.githubusercontent.com/brendangregg/FlameGraph/master/difffolded.pl" + chmod +x ~/fg/difffolded.pl + chmod +x ~/fg/flamegraph.pl + ) & wait } From f04c5c6100231255f871653b1294b2a2d13927bd Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 22 May 2020 11:54:35 +0300 Subject: [PATCH 552/738] performance comparison --- docker/test/performance-comparison/entrypoint.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docker/test/performance-comparison/entrypoint.sh b/docker/test/performance-comparison/entrypoint.sh index b425acd1a72..ef62c8981e9 100755 --- a/docker/test/performance-comparison/entrypoint.sh +++ b/docker/test/performance-comparison/entrypoint.sh @@ -81,8 +81,13 @@ if [ "$REF_PR" == "" ]; then echo Reference PR is not specified ; exit 1 ; fi fi ) | tee right-commit.txt -# Prepare the list of changed tests for use by compare.sh -git -C ch diff --name-only "$REF_SHA" "$SHA_TO_TEST" -- tests/performance | tee changed-tests.txt +if [ "$PR_TO_TEST" != "0" ] +then + # Prepare the list of tests changed in the PR for use by compare.sh. Compare to + # merge base, because master might be far in the future and have unrelated test + # changes. + git -C ch diff --name-only "$SHA_TO_TEST" "$(git -C ch merge-base "$SHA_TO_TEST" master)" -- tests/performance | tee changed-tests.txt +fi # Set python output encoding so that we can print queries with Russian letters. export PYTHONIOENCODING=utf-8 From a2fabd7ff03f32cb5580c2883bbeda08723e62b4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 22 May 2020 12:36:26 +0300 Subject: [PATCH 553/738] Remove unreliable test #11119 --- ...1_group_by_limit_memory_tracking.reference | 0 .../01281_group_by_limit_memory_tracking.sql | 88 ------------------- 2 files changed, 88 deletions(-) delete mode 100644 tests/queries/0_stateless/01281_group_by_limit_memory_tracking.reference delete mode 100644 tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sql diff --git a/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.reference b/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sql b/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sql deleted file mode 100644 index fa606898fe8..00000000000 --- a/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sql +++ /dev/null @@ -1,88 +0,0 @@ -DROP TABLE IF EXISTS trace_log_01281; -DROP TABLE IF EXISTS trace_log_01281_mv; -DROP TABLE IF EXISTS trace_log_01281_assert; - --- better alternative will be to TRUNCATE TABLE system.*_log --- but this will be a separate issue -CREATE TABLE trace_log_01281 AS system.trace_log Engine=Memory(); -CREATE MATERIALIZED VIEW trace_log_01281_mv TO trace_log_01281 AS SELECT * FROM system.trace_log WHERE trace_type = 'MemorySample'; -CREATE VIEW trace_log_01281_assert AS SELECT - *, - throwIf(cnt < 0, 'no memory profile captured'), --- this check is disabled because it's not reliable: throwIf(queries > 1, 'too many queries'), - throwIf(alloc < 100e6, 'minimal allocation had not been done'), - throwIf((alloc+free)/alloc > 0.05, 'memory accounting leaked more than 5%') - FROM ( - SELECT - count() cnt, - uniq(query_id) queries, - sumIf(size, size > 0) alloc, - sumIf(size, size < 0) free - FROM trace_log_01281 - ); - --- --- Basic --- NOTE: 0 (and even 1e6) is too small, will make SYSTEM FLUSH LOGS too slow --- (in debug build at least) --- -SET max_untracked_memory=4e6; - -TRUNCATE TABLE trace_log_01281; --- single { -SET max_threads=1; -SET memory_profiler_sample_probability=1; -SELECT uniqExactState(number) FROM numbers(toUInt64(2e6)) GROUP BY number % 2e5 FORMAT Null; -SET memory_profiler_sample_probability=0; -SYSTEM FLUSH LOGS; --- } -SELECT * FROM trace_log_01281_assert FORMAT Null; - -SYSTEM FLUSH LOGS; -TRUNCATE TABLE trace_log_01281; --- single limit { -SET max_threads=1; -SET memory_profiler_sample_probability=1; -SELECT uniqExactState(number) FROM numbers(toUInt64(2e6)) GROUP BY number % 2e5 LIMIT 10 FORMAT Null; -SET memory_profiler_sample_probability=0; -SYSTEM FLUSH LOGS; --- } -SELECT * FROM trace_log_01281_assert FORMAT Null; - -SYSTEM FLUSH LOGS; -TRUNCATE TABLE trace_log_01281; --- two-level { --- need to have multiple threads for two-level aggregation -SET max_threads=2; -SET memory_profiler_sample_probability=1; -SELECT uniqExactState(number) FROM numbers_mt(toUInt64(2e6)) GROUP BY number % 2e5 FORMAT Null; -SET memory_profiler_sample_probability=0; -SYSTEM FLUSH LOGS; --- } -SELECT * FROM trace_log_01281_assert FORMAT Null; - -SYSTEM FLUSH LOGS; -TRUNCATE TABLE trace_log_01281; --- two-level limit { --- need to have multiple threads for two-level aggregation -SET max_threads=2; -SET memory_profiler_sample_probability=1; -SELECT uniqExactState(number) FROM numbers_mt(toUInt64(2e6)) GROUP BY number % 2e5 LIMIT 10 FORMAT Null; -SET memory_profiler_sample_probability=0; -SYSTEM FLUSH LOGS; --- } -SELECT * FROM trace_log_01281_assert FORMAT Null; - -SYSTEM FLUSH LOGS; -TRUNCATE TABLE trace_log_01281; --- two-level MEMORY_LIMIT_EXCEEDED { --- need to have multiple threads for two-level aggregation -SET max_threads=2; -SET memory_profiler_sample_probability=1; -SET max_memory_usage='150M'; -SELECT uniqExactState(number) FROM numbers_mt(toUInt64(10e6)) GROUP BY number % 1e6 FORMAT Null; -- { serverError 241; } -SET memory_profiler_sample_probability=0; -SET max_memory_usage=0; -SYSTEM FLUSH LOGS; --- } -SELECT * FROM trace_log_01281_assert FORMAT Null; From 98ffefd90c161b8073e019546d84a607b9c46950 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 22 May 2020 13:09:24 +0300 Subject: [PATCH 554/738] Fix bug with return type --- src/Interpreters/Context.cpp | 4 ++-- src/Interpreters/Context.h | 4 ++-- src/Storages/System/StorageSystemStoragePolicies.cpp | 4 +--- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index e099721acdc..f6aa2890816 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1709,13 +1709,13 @@ StoragePolicyPtr Context::getStoragePolicy(const String & name) const } -const std::map & Context::getDisksMap() const +std::map Context::getDisksMap() const { std::lock_guard lock(shared->storage_policies_mutex); return getDiskSelector(lock)->getDisksMap(); } -const std::map & Context::getPoliciesMap() const +std::map Context::getPoliciesMap() const { std::lock_guard lock(shared->storage_policies_mutex); return getStoragePolicySelector(lock)->getPoliciesMap(); diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 65c9ed6749f..1f59648a525 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -545,8 +545,8 @@ public: /// Provides storage disks DiskPtr getDisk(const String & name) const; - const std::map & getPoliciesMap() const; - const std::map & getDisksMap() const; + std::map getPoliciesMap() const; + std::map getDisksMap() const; void updateStorageConfiguration(const Poco::Util::AbstractConfiguration & config); /// Provides storage politics schemes diff --git a/src/Storages/System/StorageSystemStoragePolicies.cpp b/src/Storages/System/StorageSystemStoragePolicies.cpp index c4f43b0a5bf..acbc9d72a20 100644 --- a/src/Storages/System/StorageSystemStoragePolicies.cpp +++ b/src/Storages/System/StorageSystemStoragePolicies.cpp @@ -45,9 +45,7 @@ Pipes StorageSystemStoragePolicies::read( MutableColumnPtr col_max_part_size = ColumnUInt64::create(); MutableColumnPtr col_move_factor = ColumnFloat32::create(); - const auto & policies_map = context.getPoliciesMap(); - - for (const auto & [policy_name, policy_ptr] : policies_map) + for (const auto & [policy_name, policy_ptr] : context.getPoliciesMap()) { const auto & volumes = policy_ptr->getVolumes(); for (size_t i = 0; i != volumes.size(); ++i) From 70fe1194261ea839cc333019e12722170a12e64f Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 22 May 2020 13:33:57 +0300 Subject: [PATCH 555/738] Add clear names --- src/Disks/DiskSelector.h | 6 ++++-- src/Disks/StoragePolicy.h | 5 +++-- src/Interpreters/Context.cpp | 4 ++-- src/Interpreters/Context.h | 6 ++++-- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/Disks/DiskSelector.h b/src/Disks/DiskSelector.h index 8ae8de3be61..3f19dfba381 100644 --- a/src/Disks/DiskSelector.h +++ b/src/Disks/DiskSelector.h @@ -9,9 +9,11 @@ namespace DB { + class Context; class DiskSelector; using DiskSelectorPtr = std::shared_ptr; +using DisksMap = std::map; /// Parse .xml configuration and store information about disks /// Mostly used for introspection. @@ -28,14 +30,14 @@ public: DiskPtr get(const String & name) const; /// Get all disks with names - const auto & getDisksMap() const { return disks; } + const DisksMap & getDisksMap() const { return disks; } void addToDiskMap(String name, DiskPtr disk) { disks.emplace(name, disk); } private: - std::map disks; + DisksMap disks; }; } diff --git a/src/Disks/StoragePolicy.h b/src/Disks/StoragePolicy.h index 00ac2c2c9bb..a8cba9abaeb 100644 --- a/src/Disks/StoragePolicy.h +++ b/src/Disks/StoragePolicy.h @@ -99,6 +99,7 @@ private: class StoragePolicySelector; using StoragePolicySelectorPtr = std::shared_ptr; +using StoragePoliciesMap = std::map; /// Parse .xml configuration and store information about policies /// Mostly used for introspection. @@ -113,10 +114,10 @@ public: StoragePolicyPtr get(const String & name) const; /// All policies - const std::map & getPoliciesMap() const { return policies; } + const StoragePoliciesMap & getPoliciesMap() const { return policies; } private: - std::map policies; + StoragePoliciesMap policies; }; } diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index f6aa2890816..45eeb47c035 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1709,13 +1709,13 @@ StoragePolicyPtr Context::getStoragePolicy(const String & name) const } -std::map Context::getDisksMap() const +DisksMap Context::getDisksMap() const { std::lock_guard lock(shared->storage_policies_mutex); return getDiskSelector(lock)->getDisksMap(); } -std::map Context::getPoliciesMap() const +StoragePoliciesMap Context::getPoliciesMap() const { std::lock_guard lock(shared->storage_policies_mutex); return getStoragePolicySelector(lock)->getPoliciesMap(); diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 1f59648a525..864468c0663 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -97,8 +97,10 @@ class IDisk; using DiskPtr = std::shared_ptr; class DiskSelector; using DiskSelectorPtr = std::shared_ptr; +using DisksMap = std::map; class StoragePolicy; using StoragePolicyPtr = std::shared_ptr; +using StoragePoliciesMap = std::map; class StoragePolicySelector; using StoragePolicySelectorPtr = std::shared_ptr; @@ -545,8 +547,8 @@ public: /// Provides storage disks DiskPtr getDisk(const String & name) const; - std::map getPoliciesMap() const; - std::map getDisksMap() const; + StoragePoliciesMap getPoliciesMap() const; + DisksMap getDisksMap() const; void updateStorageConfiguration(const Poco::Util::AbstractConfiguration & config); /// Provides storage politics schemes From 138e4562f30c650d5ce8cc4181705eb66e3a8a2a Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 22 May 2020 13:38:56 +0300 Subject: [PATCH 556/738] Skip flacky test --- tests/integration/test_replicated_merge_tree_s3/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_replicated_merge_tree_s3/test.py b/tests/integration/test_replicated_merge_tree_s3/test.py index d6b6015a388..e38186ead3e 100644 --- a/tests/integration/test_replicated_merge_tree_s3/test.py +++ b/tests/integration/test_replicated_merge_tree_s3/test.py @@ -85,7 +85,7 @@ def drop_table(cluster): for obj in list(minio.list_objects(cluster.minio_bucket, 'data/')): minio.remove_object(cluster.minio_bucket, obj.object_name) - +@pytest.mark.skip(reason="Cannot correctly start server") def test_insert_select_replicated(cluster): create_table(cluster) From 2547874b3badc3bf8254689edf9f3d5fbfdfc5a6 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Fri, 22 May 2020 14:00:41 +0300 Subject: [PATCH 557/738] Update index.md --- docs/en/engines/index.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/en/engines/index.md b/docs/en/engines/index.md index fe7e46fb534..0ab3b5fe172 100644 --- a/docs/en/engines/index.md +++ b/docs/en/engines/index.md @@ -1,6 +1,8 @@ --- toc_folder_title: Engines toc_priority: 25 +toc_title: hidden +toc_hidden: true --- {## [Original article](https://clickhouse.tech/docs/en/engines/) ##} From f4b0d475afcf16fd2edf39be652e9417d348198a Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 22 May 2020 14:04:15 +0300 Subject: [PATCH 558/738] Fix system.tables output --- src/Storages/System/StorageSystemTables.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/System/StorageSystemTables.cpp b/src/Storages/System/StorageSystemTables.cpp index d6951d8467f..2caf599fe51 100644 --- a/src/Storages/System/StorageSystemTables.cpp +++ b/src/Storages/System/StorageSystemTables.cpp @@ -372,7 +372,7 @@ protected: if (columns_mask[src_index++]) { assert(table != nullptr); - if ((expression_ptr = table->getSortingKeyAST())) + if ((expression_ptr = table->getSortingKey().expression_list_ast)) res_columns[res_index++]->insert(queryToString(expression_ptr)); else res_columns[res_index++]->insertDefault(); @@ -381,7 +381,7 @@ protected: if (columns_mask[src_index++]) { assert(table != nullptr); - if ((expression_ptr = table->getPrimaryKeyAST())) + if ((expression_ptr = table->getPrimaryKey().expression_list_ast)) res_columns[res_index++]->insert(queryToString(expression_ptr)); else res_columns[res_index++]->insertDefault(); From b6c04fb6ec881b1baf2c8cb649929c3aea54b5e6 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Fri, 22 May 2020 14:09:33 +0300 Subject: [PATCH 559/738] Update footer.html --- website/templates/docs/footer.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/templates/docs/footer.html b/website/templates/docs/footer.html index c9432a0b4dc..a99a723c036 100644 --- a/website/templates/docs/footer.html +++ b/website/templates/docs/footer.html @@ -1,7 +1,7 @@ {% if not single_page %}

    {{ _('Rating') }}: RATING_VALUE - RATING_COUNT {{ _('votes') }}

    -

    {% if is_amp %}{{ _('Article Rating') }}{% else %}{{ _('Was this content helpful?') }}{% endif %}

    + {% if is_amp %}{{ _('Article Rating') }}{% else %}{{ _('Was this content helpful?') }}{% endif %}
    Date: Fri, 22 May 2020 14:16:24 +0300 Subject: [PATCH 560/738] [docs] tune titles --- website/main.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/main.html b/website/main.html index 07647c24a2d..73693204e5d 100644 --- a/website/main.html +++ b/website/main.html @@ -18,9 +18,9 @@ {% if page and page.meta and page.meta.title %} {% set title = page.meta.title %} {% elif page and page.title and not page.is_homepage and page.title != 'hidden' %} - {% set title = page.title + ' - ' + config.site_name %} + {% set title = page.title + ' | ' + config.site_name %} {% elif page and page.title and not page.is_homepage and page.title == 'hidden' and page.ancestors %} - {% set title = (page.ancestors | first).title + ' - ' + config.site_name %} + {% set title = (page.ancestors | first).title + ' | ' + config.site_name %} {% else %} {% set title = config.site_name %} {% endif %} From 1a67c89c26b43ea4fa2eda9fda51101dc3ce92b4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 22 May 2020 14:31:27 +0300 Subject: [PATCH 561/738] Enable mlock by default --- programs/server/config.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/server/config.xml b/programs/server/config.xml index dd01b377485..e16af9d75d7 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -227,7 +227,7 @@ and to prevent clickhouse executable from being paged out under high IO load. Enabling this option is recommended but will lead to increased startup time for up to a few seconds. --> - false + true - - + + From 9d1d7c61d60acf405246670bcc72275386a9da6a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 17:52:32 +0300 Subject: [PATCH 611/738] Replace hits_1000m to hits_100m --- tests/performance/columns_hashing.xml | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/tests/performance/columns_hashing.xml b/tests/performance/columns_hashing.xml index 7297b690ee4..186d3a8ba62 100644 --- a/tests/performance/columns_hashing.xml +++ b/tests/performance/columns_hashing.xml @@ -5,28 +5,11 @@ hits_100m_single - hits_100m_single - - - - - - + + From 2c76899f530f3204dcf068e0998847744b8152c0 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 17:53:54 +0300 Subject: [PATCH 612/738] Edited strange test --- tests/performance/date_time_64.xml | 2 +- tests/performance/early_constant_folding.xml | 4 +-- tests/performance/leftpad.xml | 26 +++++++------------- tests/performance/merge_table_streams.xml | 2 +- tests/performance/nyc_taxi.xml | 2 +- tests/performance/order_by_read_in_order.xml | 2 +- tests/performance/parallel_insert.xml | 2 +- tests/performance/prewhere.xml | 2 +- tests/performance/string_join.xml | 2 +- 9 files changed, 18 insertions(+), 26 deletions(-) diff --git a/tests/performance/date_time_64.xml b/tests/performance/date_time_64.xml index a69b1910d7a..5f9c76766c2 100644 --- a/tests/performance/date_time_64.xml +++ b/tests/performance/date_time_64.xml @@ -1,7 +1,7 @@ - default.hits_100m_single + hits_100m_single CREATE TABLE dt (x DateTime) ENGINE = MergeTree ORDER BY tuple() diff --git a/tests/performance/early_constant_folding.xml b/tests/performance/early_constant_folding.xml index f6f1be8598d..aa7d540801f 100644 --- a/tests/performance/early_constant_folding.xml +++ b/tests/performance/early_constant_folding.xml @@ -3,8 +3,8 @@ - default.hits_100m_single + hits_100m_single - SELECT count(JavaEnable) FROM default.hits_100m_single WHERE WatchID = 1 OR Title = 'next' OR URL = 'prev' OR OriginalURL = '???' OR 1 + SELECT count(JavaEnable) FROM hits_100m_single WHERE WatchID = 1 OR Title = 'next' OR URL = 'prev' OR OriginalURL = '???' OR 1 diff --git a/tests/performance/leftpad.xml b/tests/performance/leftpad.xml index 3f747054122..4349c6ac0e5 100644 --- a/tests/performance/leftpad.xml +++ b/tests/performance/leftpad.xml @@ -1,22 +1,14 @@ - - - string - - - hashfile + hits_100m_single - - - - - - - - - - - + + + + + + + + diff --git a/tests/performance/merge_table_streams.xml b/tests/performance/merge_table_streams.xml index 01f0444e54c..c3484275606 100644 --- a/tests/performance/merge_table_streams.xml +++ b/tests/performance/merge_table_streams.xml @@ -19,7 +19,7 @@ CREATE TABLE merge_table_streams_3 AS hits_100m_single CREATE TABLE merge_table_streams_4 AS hits_100m_single -SELECT UserID FROM merge(default, '^(hits_100m_single|merge_table_streams_\\d)$') WHERE UserID = 12345678901234567890 +SELECT UserID FROM merge( '^(hits_100m_single|merge_table_streams_\\d)$') WHERE UserID = 12345678901234567890 DROP TABLE merge_table_streams_1 DROP TABLE merge_table_streams_2 diff --git a/tests/performance/nyc_taxi.xml b/tests/performance/nyc_taxi.xml index b7c1cf58146..b8d9621e3eb 100644 --- a/tests/performance/nyc_taxi.xml +++ b/tests/performance/nyc_taxi.xml @@ -3,7 +3,7 @@ - default.trips_mergetree + trips_mergetree SELECT cab_type, count(*) FROM trips_mergetree GROUP BY cab_type diff --git a/tests/performance/order_by_read_in_order.xml b/tests/performance/order_by_read_in_order.xml index 5749a49a3aa..bb3a921c862 100644 --- a/tests/performance/order_by_read_in_order.xml +++ b/tests/performance/order_by_read_in_order.xml @@ -4,7 +4,7 @@ - default.hits_100m_single + hits_100m_single SELECT * FROM hits_100m_single ORDER BY CounterID, EventDate LIMIT 100 diff --git a/tests/performance/parallel_insert.xml b/tests/performance/parallel_insert.xml index 34c45e08bc0..4050e771cb8 100644 --- a/tests/performance/parallel_insert.xml +++ b/tests/performance/parallel_insert.xml @@ -1,7 +1,7 @@ - default.hits_10m_single + hits_10m_single diff --git a/tests/performance/prewhere.xml b/tests/performance/prewhere.xml index 40a12a68bb9..43ce52d8bd4 100644 --- a/tests/performance/prewhere.xml +++ b/tests/performance/prewhere.xml @@ -3,7 +3,7 @@ - default.hits_10m_single + hits_10m_single diff --git a/tests/performance/string_join.xml b/tests/performance/string_join.xml index 477f62c1327..e4f9979bc09 100644 --- a/tests/performance/string_join.xml +++ b/tests/performance/string_join.xml @@ -3,7 +3,7 @@ - default.hits_10m_single + hits_10m_single CREATE TABLE hits_10m_words (word String, UserID UInt64) ENGINE Memory From 3b29ea09c16e7054e3abf49d0b38e8ff7660a4b9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 17:56:35 +0300 Subject: [PATCH 613/738] Fix another strange test --- tests/performance/if_to_multiif.xml | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/tests/performance/if_to_multiif.xml b/tests/performance/if_to_multiif.xml index 88fc38d48f0..61b76bdb870 100644 --- a/tests/performance/if_to_multiif.xml +++ b/tests/performance/if_to_multiif.xml @@ -1,17 +1,8 @@ - - - - - nonexistent_table_if_multiif - - - - + - + From 82f03fa8cedb029544c7455492fa2d4b102b76d1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 18:22:04 +0300 Subject: [PATCH 614/738] Fix loop unrolling --- src/AggregateFunctions/QuantileTDigest.h | 1 + src/Columns/ColumnVector.cpp | 2 ++ src/Common/RadixSort.h | 31 +++++++++++------------- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/AggregateFunctions/QuantileTDigest.h b/src/AggregateFunctions/QuantileTDigest.h index b90979c02b9..0f1cf0c8577 100644 --- a/src/AggregateFunctions/QuantileTDigest.h +++ b/src/AggregateFunctions/QuantileTDigest.h @@ -109,6 +109,7 @@ class QuantileTDigest using KeyBits = UInt32; static constexpr size_t PART_SIZE_BITS = 8; + static constexpr bool DIRECT_WRITE_TO_DESTINATION = false; using Transform = RadixSortFloatTransform; using Allocator = RadixSortMallocAllocator; diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index f12171b7f94..d13f126bf16 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -119,6 +119,8 @@ namespace { using Element = ValueWithIndex; using Result = size_t; + static constexpr bool DIRECT_WRITE_TO_DESTINATION = true; + static T & extractKey(Element & elem) { return elem.value; } static size_t extractResult(Element & elem) { return elem.index; } }; diff --git a/src/Common/RadixSort.h b/src/Common/RadixSort.h index e54e7f2d383..6d6a9daf6f4 100644 --- a/src/Common/RadixSort.h +++ b/src/Common/RadixSort.h @@ -87,6 +87,7 @@ struct RadixSortFloatTraits using KeyBits = std::conditional_t; static constexpr size_t PART_SIZE_BITS = 8; /// With what pieces of the key, in bits, to do one pass - reshuffle of the array. + static constexpr bool DIRECT_WRITE_TO_DESTINATION = false; /// Converting a key into KeyBits is such that the order relation over the key corresponds to the order relation over KeyBits. using Transform = RadixSortFloatTransform; @@ -131,6 +132,7 @@ struct RadixSortUIntTraits using KeyBits = Key; static constexpr size_t PART_SIZE_BITS = 8; + static constexpr bool DIRECT_WRITE_TO_DESTINATION = false; using Transform = RadixSortIdentityTransform; using Allocator = RadixSortMallocAllocator; @@ -165,6 +167,7 @@ struct RadixSortIntTraits using KeyBits = std::make_unsigned_t; static constexpr size_t PART_SIZE_BITS = 8; + static constexpr bool DIRECT_WRITE_TO_DESTINATION = false; using Transform = RadixSortSignedTransform; using Allocator = RadixSortMallocAllocator; @@ -328,7 +331,7 @@ public: * In this case it will fill only Result parts of the Element into destination. * It is handy to avoid unnecessary data movements. */ - static void executeLSD(Element * arr, size_t size, bool reverse = false, Result * destination = nullptr) + static NO_INLINE void executeLSD(Element * arr, size_t size, bool reverse = false, Result * destination = nullptr) { /// If the array is smaller than 256, then it is better to use another algorithm. @@ -368,10 +371,8 @@ public: } } - bool direct_copy_to_destination = (destination); - /// Move the elements in the order starting from the least bit piece, and then do a few passes on the number of pieces. - for (size_t pass = 0; pass < NUM_PASSES - direct_copy_to_destination; ++pass) + for (size_t pass = 0; pass < NUM_PASSES - Traits::DIRECT_WRITE_TO_DESTINATION; ++pass) { Element * writer = pass % 2 ? arr : swap_buffer; Element * reader = pass % 2 ? swap_buffer : arr; @@ -390,9 +391,9 @@ public: } } - if (direct_copy_to_destination) + if (Traits::DIRECT_WRITE_TO_DESTINATION) { - size_t pass = NUM_PASSES - 1; + constexpr size_t pass = NUM_PASSES - 1; Result * writer = destination; Element * reader = pass % 2 ? swap_buffer : arr; @@ -401,8 +402,6 @@ public: for (size_t i = 0; i < size; ++i) { size_t pos = getPart(pass, keyToBits(Traits::extractKey(reader[i]))); - - /// Place the element on the next free position. writer[size - 1 - (++histograms[pass * HISTOGRAM_SIZE + pos])] = Traits::extractResult(reader[i]); } } @@ -411,21 +410,19 @@ public: for (size_t i = 0; i < size; ++i) { size_t pos = getPart(pass, keyToBits(Traits::extractKey(reader[i]))); - - /// Place the element on the next free position. writer[++histograms[pass * HISTOGRAM_SIZE + pos]] = Traits::extractResult(reader[i]); } } } - else if (NUM_PASSES % 2) + else { /// If the number of passes is odd, the result array is in a temporary buffer. Copy it to the place of the original array. - /// NOTE Sometimes it will be more optimal to provide non-destructive interface, that will not modify original array. - memcpy(arr, swap_buffer, size * sizeof(Element)); - } - else if (reverse) - { - std::reverse(arr, arr + size); + if (NUM_PASSES % 2) + memcpy(arr, swap_buffer, size * sizeof(Element)); + + /// This is suboptimal, we can embed it to the last pass. + if (reverse) + std::reverse(arr, arr + size); } allocator.deallocate(swap_buffer, size * sizeof(Element)); From 9c15e3ad928f37c223f925aef1ebab83b4076a3a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 18:29:43 +0300 Subject: [PATCH 615/738] Make it right --- src/AggregateFunctions/QuantileTDigest.h | 1 - src/Columns/ColumnVector.cpp | 1 - src/Common/RadixSort.h | 39 +++++++++++++++--------- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/src/AggregateFunctions/QuantileTDigest.h b/src/AggregateFunctions/QuantileTDigest.h index 0f1cf0c8577..b90979c02b9 100644 --- a/src/AggregateFunctions/QuantileTDigest.h +++ b/src/AggregateFunctions/QuantileTDigest.h @@ -109,7 +109,6 @@ class QuantileTDigest using KeyBits = UInt32; static constexpr size_t PART_SIZE_BITS = 8; - static constexpr bool DIRECT_WRITE_TO_DESTINATION = false; using Transform = RadixSortFloatTransform; using Allocator = RadixSortMallocAllocator; diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index d13f126bf16..41ed17d6d82 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -119,7 +119,6 @@ namespace { using Element = ValueWithIndex; using Result = size_t; - static constexpr bool DIRECT_WRITE_TO_DESTINATION = true; static T & extractKey(Element & elem) { return elem.value; } static size_t extractResult(Element & elem) { return elem.index; } diff --git a/src/Common/RadixSort.h b/src/Common/RadixSort.h index 6d6a9daf6f4..59a241e5533 100644 --- a/src/Common/RadixSort.h +++ b/src/Common/RadixSort.h @@ -87,7 +87,6 @@ struct RadixSortFloatTraits using KeyBits = std::conditional_t; static constexpr size_t PART_SIZE_BITS = 8; /// With what pieces of the key, in bits, to do one pass - reshuffle of the array. - static constexpr bool DIRECT_WRITE_TO_DESTINATION = false; /// Converting a key into KeyBits is such that the order relation over the key corresponds to the order relation over KeyBits. using Transform = RadixSortFloatTransform; @@ -132,7 +131,6 @@ struct RadixSortUIntTraits using KeyBits = Key; static constexpr size_t PART_SIZE_BITS = 8; - static constexpr bool DIRECT_WRITE_TO_DESTINATION = false; using Transform = RadixSortIdentityTransform; using Allocator = RadixSortMallocAllocator; @@ -167,7 +165,6 @@ struct RadixSortIntTraits using KeyBits = std::make_unsigned_t; static constexpr size_t PART_SIZE_BITS = 8; - static constexpr bool DIRECT_WRITE_TO_DESTINATION = false; using Transform = RadixSortSignedTransform; using Allocator = RadixSortMallocAllocator; @@ -322,16 +319,8 @@ private: radixSortMSDInternal(arr, size, limit); } -public: - /** Least significant digit radix sort (stable). - * - * This function will sort inplace (modify 'arr') - * but if 'destination' is provided, it will write result directly to destination - * instead of finishing sorting 'arr' at the last step. - * In this case it will fill only Result parts of the Element into destination. - * It is handy to avoid unnecessary data movements. - */ - static NO_INLINE void executeLSD(Element * arr, size_t size, bool reverse = false, Result * destination = nullptr) + template + static NO_INLINE void radixSortLSDInternal(Element * arr, size_t size, bool reverse, Result * destination) { /// If the array is smaller than 256, then it is better to use another algorithm. @@ -372,7 +361,7 @@ public: } /// Move the elements in the order starting from the least bit piece, and then do a few passes on the number of pieces. - for (size_t pass = 0; pass < NUM_PASSES - Traits::DIRECT_WRITE_TO_DESTINATION; ++pass) + for (size_t pass = 0; pass < NUM_PASSES - DIRECT_WRITE_TO_DESTINATION; ++pass) { Element * writer = pass % 2 ? arr : swap_buffer; Element * reader = pass % 2 ? swap_buffer : arr; @@ -391,7 +380,7 @@ public: } } - if (Traits::DIRECT_WRITE_TO_DESTINATION) + if (DIRECT_WRITE_TO_DESTINATION) { constexpr size_t pass = NUM_PASSES - 1; Result * writer = destination; @@ -428,6 +417,26 @@ public: allocator.deallocate(swap_buffer, size * sizeof(Element)); } +public: + /** Least significant digit radix sort (stable). + * This function will sort inplace (modify 'arr') + */ + static void executeLSD(Element * arr, size_t size) + { + radixSortLSDInternal(arr, size, false, nullptr); + } + + /** This function will start to sort inplace (modify 'arr') + * but on the last step it will write result directly to the destination + * instead of finishing sorting 'arr'. + * In this case it will fill only Result parts of the Element into destination. + * It is handy to avoid unnecessary data movements. + */ + static void executeLSD(Element * arr, size_t size, bool reverse, Result * destination) + { + radixSortLSDInternal(arr, size, reverse, destination); + } + /* Most significant digit radix sort * Usually slower than LSD and is not stable, but allows partial sorting * From 35d79e12528379ba97ecae33876e3863229ac3e1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 22 May 2020 13:58:29 +0300 Subject: [PATCH 616/738] fmt-style logging, part 1 --- .gitmodules | 3 ++ base/common/CMakeLists.txt | 1 + base/common/logger_useful.h | 47 +++++++++++++++++++++++------ contrib/CMakeLists.txt | 2 ++ contrib/fmtlib | 1 + contrib/fmtlib-cmake/CMakeLists.txt | 20 ++++++++++++ src/IO/S3Common.cpp | 4 +-- src/Storages/Kafka/StorageKafka.cpp | 4 +-- 8 files changed, 68 insertions(+), 14 deletions(-) create mode 160000 contrib/fmtlib create mode 100644 contrib/fmtlib-cmake/CMakeLists.txt diff --git a/.gitmodules b/.gitmodules index f7a16b84d37..7f5d1307a6e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -157,3 +157,6 @@ [submodule "contrib/openldap"] path = contrib/openldap url = https://github.com/openldap/openldap.git +[submodule "contrib/fmtlib"] + path = contrib/fmtlib + url = https://github.com/fmtlib/fmt.git diff --git a/base/common/CMakeLists.txt b/base/common/CMakeLists.txt index c3eddb4d26c..9b827cdb468 100644 --- a/base/common/CMakeLists.txt +++ b/base/common/CMakeLists.txt @@ -79,6 +79,7 @@ target_link_libraries (common Poco::Util Poco::Foundation replxx + fmt PRIVATE cctz diff --git a/base/common/logger_useful.h b/base/common/logger_useful.h index ea1a25cc8fa..4cedb3dfc30 100644 --- a/base/common/logger_useful.h +++ b/base/common/logger_useful.h @@ -3,15 +3,14 @@ /// Macros for convenient usage of Poco logger. #include +#include #include #include #include #include -#ifndef QUERY_PREVIEW_LENGTH -#define QUERY_PREVIEW_LENGTH 160 -#endif +/// TODO Remove this. using Poco::Logger; using Poco::Message; using DB::LogsLevel; @@ -19,7 +18,7 @@ using DB::CurrentThread; /// Logs a message to a specified logger with that level. -#define LOG_SIMPLE(logger, message, priority, PRIORITY) do \ +#define LOG_IMPL(logger, priority, PRIORITY, message) do \ { \ const bool is_clients_log = (CurrentThread::getGroup() != nullptr) && \ (CurrentThread::getGroup()->client_logs_level >= (priority)); \ @@ -41,10 +40,38 @@ using DB::CurrentThread; } while (false) -#define LOG_TRACE(logger, message) LOG_SIMPLE(logger, message, LogsLevel::trace, Message::PRIO_TRACE) -#define LOG_DEBUG(logger, message) LOG_SIMPLE(logger, message, LogsLevel::debug, Message::PRIO_DEBUG) -#define LOG_INFO(logger, message) LOG_SIMPLE(logger, message, LogsLevel::information, Message::PRIO_INFORMATION) -#define LOG_WARNING(logger, message) LOG_SIMPLE(logger, message, LogsLevel::warning, Message::PRIO_WARNING) -#define LOG_ERROR(logger, message) LOG_SIMPLE(logger, message, LogsLevel::error, Message::PRIO_ERROR) -#define LOG_FATAL(logger, message) LOG_SIMPLE(logger, message, LogsLevel::error, Message::PRIO_FATAL) +#define LOG_TRACE(logger, message) LOG_IMPL(logger, LogsLevel::trace, Message::PRIO_TRACE, message) +#define LOG_DEBUG(logger, message) LOG_IMPL(logger, LogsLevel::debug, Message::PRIO_DEBUG, message) +#define LOG_INFO(logger, message) LOG_IMPL(logger, LogsLevel::information, Message::PRIO_INFORMATION, message) +#define LOG_WARNING(logger, message) LOG_IMPL(logger, LogsLevel::warning, Message::PRIO_WARNING, message) +#define LOG_ERROR(logger, message) LOG_IMPL(logger, LogsLevel::error, Message::PRIO_ERROR, message) +#define LOG_FATAL(logger, message) LOG_IMPL(logger, LogsLevel::error, Message::PRIO_FATAL, message) + +#define LOG_IMPL_FORMATTED(logger, priority, PRIORITY, message, ...) do \ +{ \ + const bool is_clients_log = (CurrentThread::getGroup() != nullptr) && \ + (CurrentThread::getGroup()->client_logs_level >= (priority)); \ + if ((logger)->is((PRIORITY)) || is_clients_log) \ + { \ + std::string formatted_message = fmt::format(message, __VA_ARGS__); \ + if (auto channel = (logger)->getChannel()) \ + { \ + std::string file_function; \ + file_function += __FILE__; \ + file_function += "; "; \ + file_function += __PRETTY_FUNCTION__; \ + Message poco_message((logger)->name(), formatted_message, \ + (PRIORITY), file_function.c_str(), __LINE__); \ + channel->log(poco_message); \ + } \ + } \ +} while (false) + + +#define LOG_TRACE_FORMATTED(logger, message, ...) LOG_IMPL_FORMATTED(logger, LogsLevel::trace, Message::PRIO_TRACE, message, __VA_ARGS__) +#define LOG_DEBUG_FORMATTED(logger, message, ...) LOG_IMPL_FORMATTED(logger, LogsLevel::debug, Message::PRIO_DEBUG, message, __VA_ARGS__) +#define LOG_INFO_FORMATTED(logger, message, ...) LOG_IMPL_FORMATTED(logger, LogsLevel::information, Message::PRIO_INFORMATION, message, __VA_ARGS__) +#define LOG_WARNING_FORMATTED(logger, message, ...) LOG_IMPL_FORMATTED(logger, LogsLevel::warning, Message::PRIO_WARNING, message, __VA_ARGS__) +#define LOG_ERROR_FORMATTED(logger, message, ...) LOG_IMPL_FORMATTED(logger, LogsLevel::error, Message::PRIO_ERROR, message, __VA_ARGS__) +#define LOG_FATAL_FORMATTED(logger, message, ...) LOG_IMPL_FORMATTED(logger, LogsLevel::error, Message::PRIO_FATAL, message, __VA_ARGS__) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 1031285eac7..99f7be2cbb7 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -317,3 +317,5 @@ endif() if (USE_FASTOPS) add_subdirectory (fastops-cmake) endif() + +add_subdirectory (fmtlib-cmake) diff --git a/contrib/fmtlib b/contrib/fmtlib new file mode 160000 index 00000000000..297c3b2ed55 --- /dev/null +++ b/contrib/fmtlib @@ -0,0 +1 @@ +Subproject commit 297c3b2ed551a4989826fc8c4780bf533e964bd9 diff --git a/contrib/fmtlib-cmake/CMakeLists.txt b/contrib/fmtlib-cmake/CMakeLists.txt new file mode 100644 index 00000000000..f3bf73d7dbc --- /dev/null +++ b/contrib/fmtlib-cmake/CMakeLists.txt @@ -0,0 +1,20 @@ +set (SRCS + ../fmtlib/src/format.cc + ../fmtlib/src/os.cc + + ../fmtlib/include/fmt/chrono.h + ../fmtlib/include/fmt/color.h + ../fmtlib/include/fmt/compile.h + ../fmtlib/include/fmt/core.h + ../fmtlib/include/fmt/format.h + ../fmtlib/include/fmt/format-inl.h + ../fmtlib/include/fmt/locale.h + ../fmtlib/include/fmt/os.h + ../fmtlib/include/fmt/ostream.h + ../fmtlib/include/fmt/posix.h + ../fmtlib/include/fmt/printf.h + ../fmtlib/include/fmt/ranges.h +) + +add_library(fmt ${SRCS}) +target_include_directories(fmt SYSTEM PUBLIC ../fmtlib/include) diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp index a6b518afcdc..0cf8cf22326 100644 --- a/src/IO/S3Common.cpp +++ b/src/IO/S3Common.cpp @@ -40,13 +40,13 @@ public: void Log(Aws::Utils::Logging::LogLevel log_level, const char * tag, const char * format_str, ...) final // NOLINT { const auto & [level, prio] = convertLogLevel(log_level); - LOG_SIMPLE(log, std::string(tag) + ": " + format_str, level, prio); + LOG_IMPL(log, level, prio, std::string(tag) + ": " + format_str); } void LogStream(Aws::Utils::Logging::LogLevel log_level, const char * tag, const Aws::OStringStream & message_stream) final { const auto & [level, prio] = convertLogLevel(log_level); - LOG_SIMPLE(log, std::string(tag) + ": " + message_stream.str(), level, prio); + LOG_IMPL(log, level, prio, std::string(tag) + ": " + message_stream.str()); } void Flush() final {} diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index 793a9a29676..6d5d569ce5d 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -325,7 +325,7 @@ void StorageKafka::updateConfiguration(cppkafka::Configuration & conf) conf.set_log_callback([this](cppkafka::KafkaHandleBase &, int level, const std::string & /* facility */, const std::string & message) { auto [poco_level, client_logs_level] = parseSyslogLevel(level); - LOG_SIMPLE(log, message, client_logs_level, poco_level); + LOG_IMPL(log, client_logs_level, poco_level, message); }); // Configure interceptor to change thread name @@ -334,7 +334,7 @@ void StorageKafka::updateConfiguration(cppkafka::Configuration & conf) // XXX: rdkafka uses pthread_set_name_np(), but glibc-compatibliity overrides it to noop. { // This should be safe, since we wait the rdkafka object anyway. - void * self = reinterpret_cast(this); + void * self = static_cast(this); int status; From e88480066214538657c4745b9efa3bdf6575966c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 19:38:59 +0300 Subject: [PATCH 617/738] Remove commented code --- src/Client/Connection.cpp | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index 64cd7f27cd9..63822b58659 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -132,8 +132,6 @@ void Connection::connect(const ConnectionTimeouts & timeouts) void Connection::disconnect() { - //LOG_TRACE(log_wrapper.get(), "Disconnecting"); - in = nullptr; last_input_packet_type.reset(); out = nullptr; // can write to socket @@ -186,8 +184,6 @@ void Connection::sendHello() void Connection::receiveHello() { - //LOG_TRACE(log_wrapper.get(), "Receiving hello"); - /// Receive hello packet. UInt64 packet_type = 0; @@ -391,8 +387,6 @@ void Connection::sendQuery( query_id = query_id_; - //LOG_TRACE(log_wrapper.get(), "Sending query"); - writeVarUInt(Protocol::Client::Query, *out); writeStringBinary(query_id, *out); @@ -441,8 +435,6 @@ void Connection::sendCancel() if (!out) return; - //LOG_TRACE(log_wrapper.get(), "Sending cancel"); - writeVarUInt(Protocol::Client::Cancel, *out); out->next(); } @@ -450,8 +442,6 @@ void Connection::sendCancel() void Connection::sendData(const Block & block, const String & name, bool scalar) { - //LOG_TRACE(log_wrapper.get(), "Sending data"); - if (!block_out) { if (compression == Protocol::Compression::Enable) @@ -682,12 +672,9 @@ Packet Connection::receivePacket() } else { - //LOG_TRACE(log_wrapper.get(), "Receiving packet type"); readVarUInt(res.type, *in); } - //LOG_TRACE(log_wrapper.get(), "Receiving packet " << res.type << " " << Protocol::Server::toString(res.type)); - //std::cerr << "Client got packet: " << Protocol::Server::toString(res.type) << "\n"; switch (res.type) { case Protocol::Server::Data: [[fallthrough]]; @@ -740,8 +727,6 @@ Packet Connection::receivePacket() Block Connection::receiveData() { - //LOG_TRACE(log_wrapper.get(), "Receiving data"); - initBlockInput(); return receiveDataImpl(block_in); } @@ -820,8 +805,6 @@ void Connection::setDescription() std::unique_ptr Connection::receiveException() { - //LOG_TRACE(log_wrapper.get(), "Receiving exception"); - return std::make_unique(readException(*in, "Received from " + getDescription())); } @@ -838,8 +821,6 @@ std::vector Connection::receiveMultistringMessage(UInt64 msg_type) Progress Connection::receiveProgress() { - //LOG_TRACE(log_wrapper.get(), "Receiving progress"); - Progress progress; progress.read(*in, server_revision); return progress; From 3c582fc96fadf3dc80521f034f2296964f6e74df Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 19:42:16 +0300 Subject: [PATCH 618/738] Fix error --- base/common/logger_useful.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/base/common/logger_useful.h b/base/common/logger_useful.h index 4cedb3dfc30..08bfb8cdee5 100644 --- a/base/common/logger_useful.h +++ b/base/common/logger_useful.h @@ -48,13 +48,13 @@ using DB::CurrentThread; #define LOG_FATAL(logger, message) LOG_IMPL(logger, LogsLevel::error, Message::PRIO_FATAL, message) -#define LOG_IMPL_FORMATTED(logger, priority, PRIORITY, message, ...) do \ +#define LOG_IMPL_FORMATTED(logger, priority, PRIORITY, ...) do \ { \ const bool is_clients_log = (CurrentThread::getGroup() != nullptr) && \ (CurrentThread::getGroup()->client_logs_level >= (priority)); \ if ((logger)->is((PRIORITY)) || is_clients_log) \ { \ - std::string formatted_message = fmt::format(message, __VA_ARGS__); \ + std::string formatted_message = fmt::format(__VA_ARGS__); \ if (auto channel = (logger)->getChannel()) \ { \ std::string file_function; \ @@ -69,9 +69,9 @@ using DB::CurrentThread; } while (false) -#define LOG_TRACE_FORMATTED(logger, message, ...) LOG_IMPL_FORMATTED(logger, LogsLevel::trace, Message::PRIO_TRACE, message, __VA_ARGS__) -#define LOG_DEBUG_FORMATTED(logger, message, ...) LOG_IMPL_FORMATTED(logger, LogsLevel::debug, Message::PRIO_DEBUG, message, __VA_ARGS__) -#define LOG_INFO_FORMATTED(logger, message, ...) LOG_IMPL_FORMATTED(logger, LogsLevel::information, Message::PRIO_INFORMATION, message, __VA_ARGS__) -#define LOG_WARNING_FORMATTED(logger, message, ...) LOG_IMPL_FORMATTED(logger, LogsLevel::warning, Message::PRIO_WARNING, message, __VA_ARGS__) -#define LOG_ERROR_FORMATTED(logger, message, ...) LOG_IMPL_FORMATTED(logger, LogsLevel::error, Message::PRIO_ERROR, message, __VA_ARGS__) -#define LOG_FATAL_FORMATTED(logger, message, ...) LOG_IMPL_FORMATTED(logger, LogsLevel::error, Message::PRIO_FATAL, message, __VA_ARGS__) +#define LOG_TRACE_FORMATTED(logger, ...) LOG_IMPL_FORMATTED(logger, LogsLevel::trace, Message::PRIO_TRACE, __VA_ARGS__) +#define LOG_DEBUG_FORMATTED(logger, ...) LOG_IMPL_FORMATTED(logger, LogsLevel::debug, Message::PRIO_DEBUG, __VA_ARGS__) +#define LOG_INFO_FORMATTED(logger, ...) LOG_IMPL_FORMATTED(logger, LogsLevel::information, Message::PRIO_INFORMATION, __VA_ARGS__) +#define LOG_WARNING_FORMATTED(logger, ...) LOG_IMPL_FORMATTED(logger, LogsLevel::warning, Message::PRIO_WARNING, __VA_ARGS__) +#define LOG_ERROR_FORMATTED(logger, ...) LOG_IMPL_FORMATTED(logger, LogsLevel::error, Message::PRIO_ERROR, __VA_ARGS__) +#define LOG_FATAL_FORMATTED(logger, ...) LOG_IMPL_FORMATTED(logger, LogsLevel::error, Message::PRIO_FATAL, __VA_ARGS__) From 8d2e80a5e2c74569f7a388e42452adeb81b7a337 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 19:42:39 +0300 Subject: [PATCH 619/738] find {base,src,programs} -name '*.h' -or -name '*.cpp' | xargs grep -l -P 'LOG_\w+\([^,]+, "[^"]+"\)' | xargs sed -i -r -e 's/(LOG_\w+)\(([^,]+, "[^"]+")\)/\1_FORMATTED(\2)/' --- base/daemon/BaseDaemon.cpp | 10 +++--- programs/copier/ClusterCopier.cpp | 18 +++++----- programs/local/LocalServer.cpp | 2 +- programs/odbc-bridge/ODBCBridge.cpp | 6 ++-- programs/server/HTTPHandler.cpp | 2 +- programs/server/InterserverIOHTTPHandler.cpp | 2 +- programs/server/MySQLHandler.cpp | 2 +- programs/server/MySQLHandlerFactory.cpp | 2 +- programs/server/ReplicasStatusHandler.cpp | 2 +- programs/server/Server.cpp | 34 +++++++++---------- programs/server/TCPHandler.cpp | 16 ++++----- programs/server/TCPHandlerFactory.h | 2 +- src/Access/AccessRights.cpp | 2 +- src/Client/Connection.cpp | 6 ++-- src/Common/LRUCache.h | 4 +-- src/Common/PoolBase.h | 2 +- src/Common/TraceCollector.cpp | 2 +- src/Common/ZooKeeper/Increment.h | 2 +- src/Common/ZooKeeper/ZooKeeperHolder.cpp | 2 +- src/Core/BackgroundSchedulePool.cpp | 2 +- src/Core/MySQLProtocol.h | 14 ++++---- .../CreatingSetsBlockInputStream.cpp | 4 +-- ...regatedMemoryEfficientBlockInputStream.cpp | 6 ++-- .../ParallelAggregatingBlockInputStream.cpp | 2 +- src/DataStreams/UnionBlockInputStream.h | 4 +-- src/Databases/DatabaseOrdinary.cpp | 2 +- .../ClickHouseDictionarySource.cpp | 2 +- .../Embedded/RegionsHierarchies.cpp | 2 +- .../Embedded/RegionsHierarchy.cpp | 2 +- src/Dictionaries/Embedded/RegionsNames.cpp | 2 +- src/Dictionaries/MySQLDictionarySource.cpp | 4 +-- src/IO/WriteBufferFromS3.cpp | 2 +- src/Interpreters/Aggregator.cpp | 22 ++++++------ src/Interpreters/DDLWorker.cpp | 12 +++---- src/Interpreters/EmbeddedDictionaries.cpp | 4 +-- src/Interpreters/InternalTextLogsQueue.cpp | 2 +- src/Interpreters/InterpreterSystemQuery.cpp | 2 +- src/Interpreters/ProcessList.cpp | 4 +-- src/Interpreters/SystemLog.h | 4 +-- src/Interpreters/ThreadStatusExt.cpp | 2 +- .../Transforms/AggregatingTransform.cpp | 2 +- .../Transforms/CreatingSetsTransform.cpp | 2 +- .../Transforms/MergingAggregatedTransform.cpp | 2 +- src/Storages/Distributed/DirectoryMonitor.cpp | 2 +- .../Kafka/ReadBufferFromKafkaConsumer.cpp | 10 +++--- src/Storages/Kafka/StorageKafka.cpp | 6 ++-- src/Storages/MergeTree/MergeTreeData.cpp | 14 ++++---- .../MergeTree/MergeTreeDataMergerMutator.cpp | 2 +- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 2 +- .../ReplicatedMergeTreeBlockOutputStream.cpp | 4 +-- .../ReplicatedMergeTreePartCheckThread.cpp | 6 ++-- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 2 +- .../ReplicatedMergeTreeRestartingThread.cpp | 12 +++---- src/Storages/StorageDistributed.cpp | 2 +- src/Storages/StorageMergeTree.cpp | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 30 ++++++++-------- 56 files changed, 159 insertions(+), 159 deletions(-) diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index 6625ba80b7b..c703dc29b1b 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -184,14 +184,14 @@ public: if (sig == Signals::StopThread) { - LOG_INFO(log, "Stop SignalListener thread"); + LOG_INFO_FORMATTED(log, "Stop SignalListener thread"); break; } else if (sig == SIGHUP || sig == SIGUSR1) { - LOG_DEBUG(log, "Received signal to close logs."); + LOG_DEBUG_FORMATTED(log, "Received signal to close logs."); BaseDaemon::instance().closeLogs(BaseDaemon::instance().logger()); - LOG_INFO(log, "Opened new log file after received signal."); + LOG_INFO_FORMATTED(log, "Opened new log file after received signal."); } else if (sig == Signals::StdTerminate) { @@ -247,7 +247,7 @@ private: UInt32 thread_num, const std::string & query_id) const { - LOG_FATAL(log, "########################################"); + LOG_FATAL_FORMATTED(log, "########################################"); { std::stringstream message; @@ -854,7 +854,7 @@ void BaseDaemon::onInterruptSignals(int signal_id) if (sigint_signals_counter >= 2) { - LOG_INFO(&logger(), "Received second signal Interrupt. Immediately terminate."); + LOG_INFO_FORMATTED(&logger(), "Received second signal Interrupt. Immediately terminate."); kill(); } } diff --git a/programs/copier/ClusterCopier.cpp b/programs/copier/ClusterCopier.cpp index 45cfb4963a3..dbd2d96c6f2 100644 --- a/programs/copier/ClusterCopier.cpp +++ b/programs/copier/ClusterCopier.cpp @@ -241,7 +241,7 @@ void ClusterCopier::updateConfigIfNeeded() if (!is_outdated_version && !is_expired_session) return; - LOG_DEBUG(log, "Updating task description"); + LOG_DEBUG_FORMATTED(log, "Updating task description"); reloadTaskDescription(); task_description_current_version = version_to_update; @@ -384,7 +384,7 @@ zkutil::EphemeralNodeHolder::Ptr ClusterCopier::createTaskWorkerNodeAndWaitIfNee /// Try to make fast retries if (num_bad_version_errors > 3) { - LOG_DEBUG(log, "A concurrent worker has just been added, will check free worker slots again"); + LOG_DEBUG_FORMATTED(log, "A concurrent worker has just been added, will check free worker slots again"); std::chrono::milliseconds random_sleep_time(std::uniform_int_distribution(1, 1000)(task_cluster->random_engine)); std::this_thread::sleep_for(random_sleep_time); num_bad_version_errors = 0; @@ -854,7 +854,7 @@ bool ClusterCopier::tryDropPartitionPiece( } else { - LOG_DEBUG(log, "Clean state is altered when dropping the partition, cowardly bailing"); + LOG_DEBUG_FORMATTED(log, "Clean state is altered when dropping the partition, cowardly bailing"); /// clean state is stale return false; } @@ -1165,17 +1165,17 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( auto create_is_dirty_node = [&] (const CleanStateClock & clock) { if (clock.is_stale()) - LOG_DEBUG(log, "Clean state clock is stale while setting dirty flag, cowardly bailing"); + LOG_DEBUG_FORMATTED(log, "Clean state clock is stale while setting dirty flag, cowardly bailing"); else if (!clock.is_clean()) - LOG_DEBUG(log, "Thank you, Captain Obvious"); + LOG_DEBUG_FORMATTED(log, "Thank you, Captain Obvious"); else if (clock.discovery_version) { - LOG_DEBUG(log, "Updating clean state clock"); + LOG_DEBUG_FORMATTED(log, "Updating clean state clock"); zookeeper->set(piece_is_dirty_flag_path, host_id, clock.discovery_version.value()); } else { - LOG_DEBUG(log, "Creating clean state clock"); + LOG_DEBUG_FORMATTED(log, "Creating clean state clock"); zookeeper->create(piece_is_dirty_flag_path, host_id, zkutil::CreateMode::Persistent); } }; @@ -1571,7 +1571,7 @@ void ClusterCopier::dropLocalTableIfExists(const DatabaseAndTableName & table_na void ClusterCopier::dropHelpingTables(const TaskTable & task_table) { - LOG_DEBUG(log, "Removing helping tables"); + LOG_DEBUG_FORMATTED(log, "Removing helping tables"); for (size_t current_piece_number = 0; current_piece_number < task_table.number_of_splits; ++current_piece_number) { DatabaseAndTableName original_table = task_table.table_push; @@ -1598,7 +1598,7 @@ void ClusterCopier::dropHelpingTables(const TaskTable & task_table) void ClusterCopier::dropParticularPartitionPieceFromAllHelpingTables(const TaskTable & task_table, const String & partition_name) { - LOG_DEBUG(log, "Try drop partition partition from all helping tables."); + LOG_DEBUG_FORMATTED(log, "Try drop partition partition from all helping tables."); for (size_t current_piece_number = 0; current_piece_number < task_table.number_of_splits; ++current_piece_number) { DatabaseAndTableName original_table = task_table.table_push; diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index eae30fddfee..7a164e2b108 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -216,7 +216,7 @@ try attachSystemTables(); loadMetadata(*context); DatabaseCatalog::instance().loadDatabases(); - LOG_DEBUG(log, "Loaded metadata."); + LOG_DEBUG_FORMATTED(log, "Loaded metadata."); } else { diff --git a/programs/odbc-bridge/ODBCBridge.cpp b/programs/odbc-bridge/ODBCBridge.cpp index 819a5026fec..b878a7c840f 100644 --- a/programs/odbc-bridge/ODBCBridge.cpp +++ b/programs/odbc-bridge/ODBCBridge.cpp @@ -165,7 +165,7 @@ int ODBCBridge::main(const std::vector & /*args*/) if (is_help) return Application::EXIT_OK; - LOG_INFO(log, "Starting up"); + LOG_INFO_FORMATTED(log, "Starting up"); Poco::Net::ServerSocket socket; auto address = socketBindListen(socket, hostname, port, log); socket.setReceiveTimeout(http_timeout); @@ -191,8 +191,8 @@ int ODBCBridge::main(const std::vector & /*args*/) LOG_INFO(log, "Listening http://" + address.toString()); SCOPE_EXIT({ - LOG_DEBUG(log, "Received termination signal."); - LOG_DEBUG(log, "Waiting for current connections to close."); + LOG_DEBUG_FORMATTED(log, "Received termination signal."); + LOG_DEBUG_FORMATTED(log, "Waiting for current connections to close."); server.stop(); for (size_t count : ext::range(1, 6)) { diff --git a/programs/server/HTTPHandler.cpp b/programs/server/HTTPHandler.cpp index 701b5f7d735..b50cba97d90 100644 --- a/programs/server/HTTPHandler.cpp +++ b/programs/server/HTTPHandler.cpp @@ -705,7 +705,7 @@ void HTTPHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne } processQuery(request, params, response, used_output); - LOG_INFO(log, "Done processing query"); + LOG_INFO_FORMATTED(log, "Done processing query"); } catch (...) { diff --git a/programs/server/InterserverIOHTTPHandler.cpp b/programs/server/InterserverIOHTTPHandler.cpp index 481ba8b00d0..1dc7fa88dc0 100644 --- a/programs/server/InterserverIOHTTPHandler.cpp +++ b/programs/server/InterserverIOHTTPHandler.cpp @@ -96,7 +96,7 @@ void InterserverIOHTTPHandler::handleRequest(Poco::Net::HTTPServerRequest & requ if (auto [message, success] = checkAuthentication(request); success) { processQuery(request, response, used_output); - LOG_INFO(log, "Done processing query"); + LOG_INFO_FORMATTED(log, "Done processing query"); } else { diff --git a/programs/server/MySQLHandler.cpp b/programs/server/MySQLHandler.cpp index a3ac3601e01..b6919c2f188 100644 --- a/programs/server/MySQLHandler.cpp +++ b/programs/server/MySQLHandler.cpp @@ -73,7 +73,7 @@ void MySQLHandler::run() Handshake handshake(server_capability_flags, connection_id, VERSION_STRING + String("-") + VERSION_NAME, auth_plugin->getName(), auth_plugin->getAuthPluginData()); packet_sender->sendPacket(handshake, true); - LOG_TRACE(log, "Sent handshake"); + LOG_TRACE_FORMATTED(log, "Sent handshake"); HandshakeResponse handshake_response; finishHandshake(handshake_response); diff --git a/programs/server/MySQLHandlerFactory.cpp b/programs/server/MySQLHandlerFactory.cpp index 02a478b8efa..3fbab08d413 100644 --- a/programs/server/MySQLHandlerFactory.cpp +++ b/programs/server/MySQLHandlerFactory.cpp @@ -100,7 +100,7 @@ void MySQLHandlerFactory::readRSAKeys() void MySQLHandlerFactory::generateRSAKeys() { - LOG_TRACE(log, "Generating new RSA key pair."); + LOG_TRACE_FORMATTED(log, "Generating new RSA key pair."); public_key.reset(RSA_new()); if (!public_key) throw Exception("Failed to allocate RSA key. Error: " + getOpenSSLErrors(), ErrorCodes::OPENSSL_ERROR); diff --git a/programs/server/ReplicasStatusHandler.cpp b/programs/server/ReplicasStatusHandler.cpp index f2d1ffe2ee5..0a59a727236 100644 --- a/programs/server/ReplicasStatusHandler.cpp +++ b/programs/server/ReplicasStatusHandler.cpp @@ -96,7 +96,7 @@ void ReplicasStatusHandler::handleRequest(Poco::Net::HTTPServerRequest & request } catch (...) { - LOG_ERROR((&Logger::get("ReplicasStatusHandler")), "Cannot send exception to client"); + LOG_ERROR_FORMATTED((&Logger::get("ReplicasStatusHandler")), "Cannot send exception to client"); } } } diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 9015c16c140..e3fbd234dfd 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -234,7 +234,7 @@ int Server::main(const std::vector & /*args*/) CurrentMetrics::set(CurrentMetrics::VersionInteger, ClickHouseRevision::getVersionInteger()); if (ThreadFuzzer::instance().isEffective()) - LOG_WARNING(log, "ThreadFuzzer is enabled. Application will run slowly and unstable."); + LOG_WARNING_FORMATTED(log, "ThreadFuzzer is enabled. Application will run slowly and unstable."); /** Context contains all that query execution is dependent: * settings, available functions, data types, aggregate functions, databases... @@ -274,11 +274,11 @@ int Server::main(const std::vector & /*args*/) { if (hasLinuxCapability(CAP_IPC_LOCK)) { - LOG_TRACE(log, "Will mlockall to prevent executable memory from being paged out. It may take a few seconds."); + LOG_TRACE_FORMATTED(log, "Will mlockall to prevent executable memory from being paged out. It may take a few seconds."); if (0 != mlockall(MCL_CURRENT)) LOG_WARNING(log, "Failed mlockall: " + errnoToString(ErrorCodes::SYSTEM_ERROR)); else - LOG_TRACE(log, "The memory map of clickhouse executable has been mlock'ed"); + LOG_TRACE_FORMATTED(log, "The memory map of clickhouse executable has been mlock'ed"); } else { @@ -326,11 +326,11 @@ int Server::main(const std::vector & /*args*/) * It is important to do early, not in destructor of Context, because * table engines could use Context on destroy. */ - LOG_INFO(log, "Shutting down storages."); + LOG_INFO_FORMATTED(log, "Shutting down storages."); global_context->shutdown(); - LOG_DEBUG(log, "Shut down storages."); + LOG_DEBUG_FORMATTED(log, "Shut down storages."); /** Explicitly destroy Context. It is more convenient than in destructor of Server, because logger is still available. * At this moment, no one could own shared part of Context. @@ -338,7 +338,7 @@ int Server::main(const std::vector & /*args*/) global_context_ptr = nullptr; global_context.reset(); shared_context.reset(); - LOG_DEBUG(log, "Destroyed global context."); + LOG_DEBUG_FORMATTED(log, "Destroyed global context."); }); /// Try to increase limit on number of open files. @@ -370,7 +370,7 @@ int Server::main(const std::vector & /*args*/) Poco::ErrorHandler::set(&error_handler); /// Initialize DateLUT early, to not interfere with running time of first query. - LOG_DEBUG(log, "Initializing DateLUT."); + LOG_DEBUG_FORMATTED(log, "Initializing DateLUT."); DateLUT::instance(); LOG_TRACE(log, "Initialized DateLUT with time zone '" << DateLUT::instance().getTimeZone() << "'."); @@ -550,7 +550,7 @@ int Server::main(const std::vector & /*args*/) /// Size of cache for marks (index of MergeTree family of tables). It is mandatory. size_t mark_cache_size = config().getUInt64("mark_cache_size"); if (!mark_cache_size) - LOG_ERROR(log, "Too low mark cache size will lead to severe performance degradation."); + LOG_ERROR_FORMATTED(log, "Too low mark cache size will lead to severe performance degradation."); if (mark_cache_size > max_cache_size) { mark_cache_size = max_cache_size; @@ -610,7 +610,7 @@ int Server::main(const std::vector & /*args*/) tryLogCurrentException(log, "Caught exception while loading metadata"); throw; } - LOG_DEBUG(log, "Loaded metadata."); + LOG_DEBUG_FORMATTED(log, "Loaded metadata."); /// Init trace collector only after trace_log system table was created /// Disable it if we collect test coverage information, because it will work extremely slow. @@ -650,11 +650,11 @@ int Server::main(const std::vector & /*args*/) /// Describe multiple reasons when query profiler cannot work. #if !USE_UNWIND - LOG_INFO(log, "Query Profiler and TraceCollector are disabled because they cannot work without bundled unwind (stack unwinding) library."); + LOG_INFO_FORMATTED(log, "Query Profiler and TraceCollector are disabled because they cannot work without bundled unwind (stack unwinding) library."); #endif #if WITH_COVERAGE - LOG_INFO(log, "Query Profiler and TraceCollector are disabled because they work extremely slow with test coverage."); + LOG_INFO_FORMATTED(log, "Query Profiler and TraceCollector are disabled because they work extremely slow with test coverage."); #endif #if defined(SANITIZER) @@ -707,7 +707,7 @@ int Server::main(const std::vector & /*args*/) " Note that it will not work on 'nosuid' mounted filesystems."); } #else - LOG_INFO(log, "TaskStats is not implemented for this OS. IO accounting will be disabled."); + LOG_INFO_FORMATTED(log, "TaskStats is not implemented for this OS. IO accounting will be disabled."); #endif { @@ -974,11 +974,11 @@ int Server::main(const std::vector & /*args*/) LOG_INFO(log, message.str()); } - LOG_INFO(log, "Ready for connections."); + LOG_INFO_FORMATTED(log, "Ready for connections."); SCOPE_EXIT({ - LOG_DEBUG(log, "Received termination signal."); - LOG_DEBUG(log, "Waiting for current connections to close."); + LOG_DEBUG_FORMATTED(log, "Received termination signal."); + LOG_DEBUG_FORMATTED(log, "Waiting for current connections to close."); is_cancelled = true; @@ -1030,7 +1030,7 @@ int Server::main(const std::vector & /*args*/) /// Dump coverage here, because std::atexit callback would not be called. dumpCoverageReportIfPossible(); - LOG_INFO(log, "Will shutdown forcefully."); + LOG_INFO_FORMATTED(log, "Will shutdown forcefully."); _exit(Application::EXIT_OK); } }); @@ -1051,7 +1051,7 @@ int Server::main(const std::vector & /*args*/) } catch (...) { - LOG_ERROR(log, "Caught exception while loading dictionaries."); + LOG_ERROR_FORMATTED(log, "Caught exception while loading dictionaries."); throw; } diff --git a/programs/server/TCPHandler.cpp b/programs/server/TCPHandler.cpp index e64d7799dcb..2005aa931ba 100644 --- a/programs/server/TCPHandler.cpp +++ b/programs/server/TCPHandler.cpp @@ -76,7 +76,7 @@ void TCPHandler::runImpl() if (in->eof()) { - LOG_WARNING(log, "Client has not sent any data."); + LOG_WARNING_FORMATTED(log, "Client has not sent any data."); return; } @@ -89,13 +89,13 @@ void TCPHandler::runImpl() { if (e.code() == ErrorCodes::CLIENT_HAS_CONNECTED_TO_WRONG_PORT) { - LOG_DEBUG(log, "Client has connected to wrong port."); + LOG_DEBUG_FORMATTED(log, "Client has connected to wrong port."); return; } if (e.code() == ErrorCodes::ATTEMPT_TO_READ_AFTER_EOF) { - LOG_WARNING(log, "Client has gone away."); + LOG_WARNING_FORMATTED(log, "Client has gone away."); return; } @@ -140,7 +140,7 @@ void TCPHandler::runImpl() { if (idle_time.elapsedSeconds() > connection_settings.idle_connection_timeout) { - LOG_TRACE(log, "Closing idle connection"); + LOG_TRACE_FORMATTED(log, "Closing idle connection"); return; } } @@ -345,7 +345,7 @@ void TCPHandler::runImpl() { /** Could not send exception information to the client. */ network_error = true; - LOG_WARNING(log, "Client has gone away."); + LOG_WARNING_FORMATTED(log, "Client has gone away."); } try @@ -356,7 +356,7 @@ void TCPHandler::runImpl() catch (...) { network_error = true; - LOG_WARNING(log, "Can't read external tables after query failure."); + LOG_WARNING_FORMATTED(log, "Can't read external tables after query failure."); } @@ -1080,7 +1080,7 @@ bool TCPHandler::isQueryCancelled() case Protocol::Client::Cancel: if (state.empty()) throw NetException("Unexpected packet Cancel received from client", ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT); - LOG_INFO(log, "Query was cancelled."); + LOG_INFO_FORMATTED(log, "Query was cancelled."); state.is_cancelled = true; return true; @@ -1198,7 +1198,7 @@ void TCPHandler::run() { runImpl(); - LOG_INFO(log, "Done processing connection."); + LOG_INFO_FORMATTED(log, "Done processing connection."); } catch (Poco::Exception & e) { diff --git a/programs/server/TCPHandlerFactory.h b/programs/server/TCPHandlerFactory.h index 04a5e394c62..ab000718d6f 100644 --- a/programs/server/TCPHandlerFactory.h +++ b/programs/server/TCPHandlerFactory.h @@ -40,7 +40,7 @@ public: } catch (const Poco::Net::NetException &) { - LOG_TRACE(log, "TCP Request. Client is not connected (most likely RST packet was sent)."); + LOG_TRACE_FORMATTED(log, "TCP Request. Client is not connected (most likely RST packet was sent)."); return new DummyTCPHandler(socket); } } diff --git a/src/Access/AccessRights.cpp b/src/Access/AccessRights.cpp index 086d740ede1..ed5e1abdb2a 100644 --- a/src/Access/AccessRights.cpp +++ b/src/Access/AccessRights.cpp @@ -794,6 +794,6 @@ void AccessRights::logTree() const if (root) root->logTree(log); else - LOG_TRACE(log, "Tree: NULL"); + LOG_TRACE_FORMATTED(log, "Tree: NULL"); } } diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index 63822b58659..1b9dfde9177 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -287,14 +287,14 @@ void Connection::forceConnected(const ConnectionTimeouts & timeouts) } else if (!ping()) { - LOG_TRACE(log_wrapper.get(), "Connection was closed, will reconnect."); + LOG_TRACE_FORMATTED(log_wrapper.get(), "Connection was closed, will reconnect."); connect(timeouts); } } bool Connection::ping() { - // LOG_TRACE(log_wrapper.get(), "Ping"); + // LOG_TRACE_FORMATTED(log_wrapper.get(), "Ping"); TimeoutSetter timeout_setter(*socket, sync_request_timeout, true); try @@ -646,7 +646,7 @@ std::optional Connection::checkPacket(size_t timeout_microseconds) if (hasReadPendingData() || poll(timeout_microseconds)) { - // LOG_TRACE(log_wrapper.get(), "Receiving packet type"); + // LOG_TRACE_FORMATTED(log_wrapper.get(), "Receiving packet type"); UInt64 packet_type; readVarUInt(packet_type, *in); diff --git a/src/Common/LRUCache.h b/src/Common/LRUCache.h index 5bcfc8fc2db..988b7826977 100644 --- a/src/Common/LRUCache.h +++ b/src/Common/LRUCache.h @@ -306,7 +306,7 @@ private: auto it = cells.find(key); if (it == cells.end()) { - LOG_ERROR(&Logger::get("LRUCache"), "LRUCache became inconsistent. There must be a bug in it."); + LOG_ERROR_FORMATTED(&Logger::get("LRUCache"), "LRUCache became inconsistent. There must be a bug in it."); abort(); } @@ -324,7 +324,7 @@ private: if (current_size > (1ull << 63)) { - LOG_ERROR(&Logger::get("LRUCache"), "LRUCache became inconsistent. There must be a bug in it."); + LOG_ERROR_FORMATTED(&Logger::get("LRUCache"), "LRUCache became inconsistent. There must be a bug in it."); abort(); } } diff --git a/src/Common/PoolBase.h b/src/Common/PoolBase.h index 201f9ec1d00..8c2ad8c9857 100644 --- a/src/Common/PoolBase.h +++ b/src/Common/PoolBase.h @@ -122,7 +122,7 @@ public: return Entry(*items.back()); } - LOG_INFO(log, "No free connections in pool. Waiting."); + LOG_INFO_FORMATTED(log, "No free connections in pool. Waiting."); if (timeout < 0) available.wait(lock); diff --git a/src/Common/TraceCollector.cpp b/src/Common/TraceCollector.cpp index 7df06dc7892..5b489bd32a9 100644 --- a/src/Common/TraceCollector.cpp +++ b/src/Common/TraceCollector.cpp @@ -48,7 +48,7 @@ TraceCollector::TraceCollector(std::shared_ptr trace_log_) TraceCollector::~TraceCollector() { if (!thread.joinable()) - LOG_ERROR(&Poco::Logger::get("TraceCollector"), "TraceCollector thread is malformed and cannot be joined"); + LOG_ERROR_FORMATTED(&Poco::Logger::get("TraceCollector"), "TraceCollector thread is malformed and cannot be joined"); else stop(); diff --git a/src/Common/ZooKeeper/Increment.h b/src/Common/ZooKeeper/Increment.h index 26128480635..4dc4aac78d2 100644 --- a/src/Common/ZooKeeper/Increment.h +++ b/src/Common/ZooKeeper/Increment.h @@ -16,7 +16,7 @@ public: size_t get() { - LOG_TRACE(log, "Get increment"); + LOG_TRACE_FORMATTED(log, "Get increment"); size_t result = 0; std::string result_str; diff --git a/src/Common/ZooKeeper/ZooKeeperHolder.cpp b/src/Common/ZooKeeper/ZooKeeperHolder.cpp index 41a36a51082..d21bfa9f10d 100644 --- a/src/Common/ZooKeeper/ZooKeeperHolder.cpp +++ b/src/Common/ZooKeeper/ZooKeeperHolder.cpp @@ -34,7 +34,7 @@ bool ZooKeeperHolder::replaceZooKeeperSessionToNewOne() } else { - LOG_ERROR(log, "replaceZooKeeperSessionToNewOne(): Fail to replace zookeeper session to new one because handlers for old zookeeper session still exists."); + LOG_ERROR_FORMATTED(log, "replaceZooKeeperSessionToNewOne(): Fail to replace zookeeper session to new one because handlers for old zookeeper session still exists."); return false; } } diff --git a/src/Core/BackgroundSchedulePool.cpp b/src/Core/BackgroundSchedulePool.cpp index 6b98b8dc5fc..3974cf0071d 100644 --- a/src/Core/BackgroundSchedulePool.cpp +++ b/src/Core/BackgroundSchedulePool.cpp @@ -179,7 +179,7 @@ BackgroundSchedulePool::~BackgroundSchedulePool() queue.wakeUpAll(); delayed_thread.join(); - LOG_TRACE(&Logger::get("BackgroundSchedulePool/" + thread_name), "Waiting for threads to finish."); + LOG_TRACE_FORMATTED(&Logger::get("BackgroundSchedulePool/" + thread_name), "Waiting for threads to finish."); for (auto & thread : threads) thread.join(); } diff --git a/src/Core/MySQLProtocol.h b/src/Core/MySQLProtocol.h index b548b573a90..f0cacca6358 100644 --- a/src/Core/MySQLProtocol.h +++ b/src/Core/MySQLProtocol.h @@ -1044,17 +1044,17 @@ public: AuthSwitchResponse response; packet_sender->receivePacket(response); auth_response = response.value; - LOG_TRACE(log, "Authentication method mismatch."); + LOG_TRACE_FORMATTED(log, "Authentication method mismatch."); } else { - LOG_TRACE(log, "Authentication method match."); + LOG_TRACE_FORMATTED(log, "Authentication method match."); } bool sent_public_key = false; if (auth_response == "\1") { - LOG_TRACE(log, "Client requests public key."); + LOG_TRACE_FORMATTED(log, "Client requests public key."); BIO * mem = BIO_new(BIO_s_mem()); SCOPE_EXIT(BIO_free(mem)); if (PEM_write_bio_RSA_PUBKEY(mem, &public_key) != 1) @@ -1080,7 +1080,7 @@ public: } else { - LOG_TRACE(log, "Client didn't request public key."); + LOG_TRACE_FORMATTED(log, "Client didn't request public key."); } String password; @@ -1092,7 +1092,7 @@ public: */ if (!is_secure_connection && !auth_response->empty() && auth_response != String("\0", 1)) { - LOG_TRACE(log, "Received nonempty password."); + LOG_TRACE_FORMATTED(log, "Received nonempty password."); auto ciphertext = reinterpret_cast(auth_response->data()); unsigned char plaintext[RSA_size(&private_key)]; @@ -1100,7 +1100,7 @@ public: if (plaintext_size == -1) { if (!sent_public_key) - LOG_WARNING(log, "Client could have encrypted password with different public key since it didn't request it from server."); + LOG_WARNING_FORMATTED(log, "Client could have encrypted password with different public key since it didn't request it from server."); throw Exception("Failed to decrypt auth data. Error: " + getOpenSSLErrors(), ErrorCodes::OPENSSL_ERROR); } @@ -1116,7 +1116,7 @@ public: } else { - LOG_TRACE(log, "Received empty password"); + LOG_TRACE_FORMATTED(log, "Received empty password"); } if (!password.empty() && password.back() == 0) diff --git a/src/DataStreams/CreatingSetsBlockInputStream.cpp b/src/DataStreams/CreatingSetsBlockInputStream.cpp index be89e0e87b0..967029b6dc1 100644 --- a/src/DataStreams/CreatingSetsBlockInputStream.cpp +++ b/src/DataStreams/CreatingSetsBlockInputStream.cpp @@ -113,7 +113,7 @@ void CreatingSetsBlockInputStream::createOne(SubqueryForSet & subquery) { if (isCancelled()) { - LOG_DEBUG(log, "Query was cancelled during set / join or temporary table creation."); + LOG_DEBUG_FORMATTED(log, "Query was cancelled during set / join or temporary table creation."); return; } @@ -181,7 +181,7 @@ void CreatingSetsBlockInputStream::createOne(SubqueryForSet & subquery) } else { - LOG_DEBUG(log, "Subquery has empty result."); + LOG_DEBUG_FORMATTED(log, "Subquery has empty result."); } } diff --git a/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp b/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp index b241f9fd00b..4dbd1e0d9ec 100644 --- a/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp +++ b/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp @@ -292,12 +292,12 @@ void MergingAggregatedMemoryEfficientBlockInputStream::finalize() if (!started) return; - LOG_TRACE(log, "Waiting for threads to finish"); + LOG_TRACE_FORMATTED(log, "Waiting for threads to finish"); if (parallel_merge_data) parallel_merge_data->pool.wait(); - LOG_TRACE(log, "Waited for threads to finish"); + LOG_TRACE_FORMATTED(log, "Waited for threads to finish"); } @@ -555,7 +555,7 @@ MergingAggregatedMemoryEfficientBlockInputStream::BlocksToMerge MergingAggregate /// Not yet partitioned (splitted to buckets) block. Will partition it and place result to 'splitted_blocks'. if (input.block.info.bucket_num == -1 && input.block && input.splitted_blocks.empty()) { - LOG_TRACE(&Logger::get("MergingAggregatedMemoryEfficient"), "Having block without bucket: will split."); + LOG_TRACE_FORMATTED(&Logger::get("MergingAggregatedMemoryEfficient"), "Having block without bucket: will split."); input.splitted_blocks = aggregator.convertBlockToTwoLevel(input.block); input.block = Block(); diff --git a/src/DataStreams/ParallelAggregatingBlockInputStream.cpp b/src/DataStreams/ParallelAggregatingBlockInputStream.cpp index 951ba7d5f2e..91663d9fc7a 100644 --- a/src/DataStreams/ParallelAggregatingBlockInputStream.cpp +++ b/src/DataStreams/ParallelAggregatingBlockInputStream.cpp @@ -157,7 +157,7 @@ void ParallelAggregatingBlockInputStream::execute() for (size_t i = 0; i < max_threads; ++i) threads_data.emplace_back(keys_size, aggregates_size); - LOG_TRACE(log, "Aggregating"); + LOG_TRACE_FORMATTED(log, "Aggregating"); Stopwatch watch; diff --git a/src/DataStreams/UnionBlockInputStream.h b/src/DataStreams/UnionBlockInputStream.h index c4e84e85845..091de967e09 100644 --- a/src/DataStreams/UnionBlockInputStream.h +++ b/src/DataStreams/UnionBlockInputStream.h @@ -107,7 +107,7 @@ protected: if (!started) return; - LOG_TRACE(log, "Waiting for threads to finish"); + LOG_TRACE_FORMATTED(log, "Waiting for threads to finish"); std::exception_ptr exception; if (!all_read) @@ -137,7 +137,7 @@ protected: processor.wait(); - LOG_TRACE(log, "Waited for threads to finish"); + LOG_TRACE_FORMATTED(log, "Waited for threads to finish"); if (exception) std::rethrow_exception(exception); diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index 2c093ae90ad..9eae75f71bb 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -194,7 +194,7 @@ void DatabaseOrdinary::loadStoredObjects( void DatabaseOrdinary::startupTables(ThreadPool & thread_pool) { - LOG_INFO(log, "Starting up tables."); + LOG_INFO_FORMATTED(log, "Starting up tables."); const size_t total_tables = tables.size(); if (!total_tables) diff --git a/src/Dictionaries/ClickHouseDictionarySource.cpp b/src/Dictionaries/ClickHouseDictionarySource.cpp index aefde10a873..89ce56df369 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.cpp +++ b/src/Dictionaries/ClickHouseDictionarySource.cpp @@ -202,7 +202,7 @@ BlockInputStreamPtr ClickHouseDictionarySource::createStreamForSelectiveLoad(con std::string ClickHouseDictionarySource::doInvalidateQuery(const std::string & request) const { - LOG_TRACE(log, "Performing invalidate query"); + LOG_TRACE_FORMATTED(log, "Performing invalidate query"); if (is_local) { Context query_context = context; diff --git a/src/Dictionaries/Embedded/RegionsHierarchies.cpp b/src/Dictionaries/Embedded/RegionsHierarchies.cpp index 1ee21b63c80..906ae381a45 100644 --- a/src/Dictionaries/Embedded/RegionsHierarchies.cpp +++ b/src/Dictionaries/Embedded/RegionsHierarchies.cpp @@ -8,7 +8,7 @@ RegionsHierarchies::RegionsHierarchies(IRegionsHierarchiesDataProviderPtr data_p { Logger * log = &Logger::get("RegionsHierarchies"); - LOG_DEBUG(log, "Adding default regions hierarchy"); + LOG_DEBUG_FORMATTED(log, "Adding default regions hierarchy"); data.emplace("", data_provider->getDefaultHierarchySource()); for (const auto & name : data_provider->listCustomHierarchies()) diff --git a/src/Dictionaries/Embedded/RegionsHierarchy.cpp b/src/Dictionaries/Embedded/RegionsHierarchy.cpp index dac1e948e0a..866d6b750fc 100644 --- a/src/Dictionaries/Embedded/RegionsHierarchy.cpp +++ b/src/Dictionaries/Embedded/RegionsHierarchy.cpp @@ -28,7 +28,7 @@ void RegionsHierarchy::reload() if (!data_source->isModified()) return; - LOG_DEBUG(log, "Reloading regions hierarchy"); + LOG_DEBUG_FORMATTED(log, "Reloading regions hierarchy"); const size_t initial_size = 10000; const size_t max_size = 15000000; diff --git a/src/Dictionaries/Embedded/RegionsNames.cpp b/src/Dictionaries/Embedded/RegionsNames.cpp index 2f5362f404c..d696b35c5d0 100644 --- a/src/Dictionaries/Embedded/RegionsNames.cpp +++ b/src/Dictionaries/Embedded/RegionsNames.cpp @@ -43,7 +43,7 @@ std::string RegionsNames::dumpSupportedLanguagesNames() void RegionsNames::reload() { Logger * log = &Logger::get("RegionsNames"); - LOG_DEBUG(log, "Reloading regions names"); + LOG_DEBUG_FORMATTED(log, "Reloading regions names"); RegionID max_region_id = 0; for (size_t language_id = 0; language_id < total_languages; ++language_id) diff --git a/src/Dictionaries/MySQLDictionarySource.cpp b/src/Dictionaries/MySQLDictionarySource.cpp index 4ba16ca19de..632951613ac 100644 --- a/src/Dictionaries/MySQLDictionarySource.cpp +++ b/src/Dictionaries/MySQLDictionarySource.cpp @@ -241,10 +241,10 @@ LocalDateTime MySQLDictionarySource::getLastModification(mysqlxx::Pool::Entry & } if (0 == fetched_rows) - LOG_ERROR(log, "Cannot find table in SHOW TABLE STATUS result."); + LOG_ERROR_FORMATTED(log, "Cannot find table in SHOW TABLE STATUS result."); if (fetched_rows > 1) - LOG_ERROR(log, "Found more than one table in SHOW TABLE STATUS result."); + LOG_ERROR_FORMATTED(log, "Found more than one table in SHOW TABLE STATUS result."); } catch (...) { diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 38456627db2..34b1c6b07ae 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -115,7 +115,7 @@ void WriteBufferFromS3::writePart(const String & data) if (part_tags.size() == S3_WARN_MAX_PARTS) { // Don't throw exception here by ourselves but leave the decision to take by S3 server. - LOG_WARNING(log, "Maximum part number in S3 protocol has reached (too many parts). Server may not accept this whole upload."); + LOG_WARNING_FORMATTED(log, "Maximum part number in S3 protocol has reached (too many parts). Server may not accept this whole upload."); } Aws::S3::Model::UploadPartRequest req; diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 7cd6a8341cd..d1b9b8637d3 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -75,7 +75,7 @@ AggregatedDataVariants::~AggregatedDataVariants() void AggregatedDataVariants::convertToTwoLevel() { if (aggregator) - LOG_TRACE(aggregator->log, "Converting aggregation data to two-level."); + LOG_TRACE_FORMATTED(aggregator->log, "Converting aggregation data to two-level."); switch (type) { @@ -912,7 +912,7 @@ void Aggregator::execute(const BlockInputStreamPtr & stream, AggregatedDataVaria */ bool no_more_keys = false; - LOG_TRACE(log, "Aggregating"); + LOG_TRACE_FORMATTED(log, "Aggregating"); Stopwatch watch; @@ -1255,7 +1255,7 @@ BlocksList Aggregator::convertToBlocks(AggregatedDataVariants & data_variants, b if (isCancelled()) return BlocksList(); - LOG_TRACE(log, "Converting aggregated data to blocks"); + LOG_TRACE_FORMATTED(log, "Converting aggregated data to blocks"); Stopwatch watch; @@ -1562,7 +1562,7 @@ public: ~MergingAndConvertingBlockInputStream() override { - LOG_TRACE(&Logger::get(__PRETTY_FUNCTION__), "Waiting for threads to finish"); + LOG_TRACE_FORMATTED(&Logger::get(__PRETTY_FUNCTION__), "Waiting for threads to finish"); /// We need to wait for threads to finish before destructor of 'parallel_merge_data', /// because the threads access 'parallel_merge_data'. @@ -1737,7 +1737,7 @@ ManyAggregatedDataVariants Aggregator::prepareVariantsToMerge(ManyAggregatedData if (data_variants.empty()) throw Exception("Empty data passed to Aggregator::mergeAndConvertToBlocks.", ErrorCodes::EMPTY_DATA_PASSED); - LOG_TRACE(log, "Merging aggregated data"); + LOG_TRACE_FORMATTED(log, "Merging aggregated data"); ManyAggregatedDataVariants non_empty_data; non_empty_data.reserve(data_variants.size()); @@ -1929,7 +1929,7 @@ void Aggregator::mergeStream(const BlockInputStreamPtr & stream, AggregatedDataV BucketToBlocks bucket_to_blocks; /// Read all the data. - LOG_TRACE(log, "Reading blocks of partially aggregated data."); + LOG_TRACE_FORMATTED(log, "Reading blocks of partially aggregated data."); size_t total_input_rows = 0; size_t total_input_blocks = 0; @@ -1997,7 +1997,7 @@ void Aggregator::mergeBlocks(BucketToBlocks bucket_to_blocks, AggregatedDataVari * That is, the keys in the end can be significantly larger than max_rows_to_group_by. */ - LOG_TRACE(log, "Merging partially aggregated two-level data."); + LOG_TRACE_FORMATTED(log, "Merging partially aggregated two-level data."); auto merge_bucket = [&bucket_to_blocks, &result, this](Int32 bucket, Arena * aggregates_pool, ThreadGroupStatusPtr thread_group) { @@ -2046,7 +2046,7 @@ void Aggregator::mergeBlocks(BucketToBlocks bucket_to_blocks, AggregatedDataVari if (thread_pool) thread_pool->wait(); - LOG_TRACE(log, "Merged partially aggregated two-level data."); + LOG_TRACE_FORMATTED(log, "Merged partially aggregated two-level data."); } if (isCancelled()) @@ -2057,7 +2057,7 @@ void Aggregator::mergeBlocks(BucketToBlocks bucket_to_blocks, AggregatedDataVari if (has_blocks_with_unknown_bucket) { - LOG_TRACE(log, "Merging partially aggregated single-level data."); + LOG_TRACE_FORMATTED(log, "Merging partially aggregated single-level data."); bool no_more_keys = false; @@ -2086,7 +2086,7 @@ void Aggregator::mergeBlocks(BucketToBlocks bucket_to_blocks, AggregatedDataVari throw Exception("Unknown aggregated data variant.", ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT); } - LOG_TRACE(log, "Merged partially aggregated single-level data."); + LOG_TRACE_FORMATTED(log, "Merged partially aggregated single-level data."); } } @@ -2347,7 +2347,7 @@ void Aggregator::destroyAllAggregateStates(AggregatedDataVariants & result) if (result.empty()) return; - LOG_TRACE(log, "Destroying aggregate states"); + LOG_TRACE_FORMATTED(log, "Destroying aggregate states"); /// In what data structure is the data aggregated? if (result.type == AggregatedDataVariants::Type::without_key || params.overflow_row) diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index cb499577272..c982ddfd83e 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -364,7 +364,7 @@ static void filterAndSortQueueNodes(Strings & all_nodes) void DDLWorker::processTasks() { - LOG_DEBUG(log, "Processing tasks"); + LOG_DEBUG_FORMATTED(log, "Processing tasks"); auto zookeeper = tryGetZooKeeper(); Strings queue_nodes = zookeeper->getChildren(queue_dir, nullptr, queue_updated_event); @@ -823,7 +823,7 @@ bool DDLWorker::tryExecuteQueryOnLeaderReplica( void DDLWorker::cleanupQueue(Int64 current_time_seconds, const ZooKeeperPtr & zookeeper) { - LOG_DEBUG(log, "Cleaning queue"); + LOG_DEBUG_FORMATTED(log, "Cleaning queue"); Strings queue_nodes = zookeeper->getChildren(queue_dir); filterAndSortQueueNodes(queue_nodes); @@ -958,7 +958,7 @@ String DDLWorker::enqueueQuery(DDLLogEntry & entry) void DDLWorker::runMainThread() { setThreadName("DDLWorker"); - LOG_DEBUG(log, "Started DDLWorker thread"); + LOG_DEBUG_FORMATTED(log, "Started DDLWorker thread"); bool initialized = false; do @@ -996,7 +996,7 @@ void DDLWorker::runMainThread() cleanup_event->set(); processTasks(); - LOG_DEBUG(log, "Waiting a watch"); + LOG_DEBUG_FORMATTED(log, "Waiting a watch"); queue_updated_event->wait(); } catch (const Coordination::Exception & e) @@ -1043,7 +1043,7 @@ void DDLWorker::runMainThread() void DDLWorker::runCleanupThread() { setThreadName("DDLWorkerClnr"); - LOG_DEBUG(log, "Started DDLWorker cleanup thread"); + LOG_DEBUG_FORMATTED(log, "Started DDLWorker cleanup thread"); Int64 last_cleanup_time_seconds = 0; while (!stop_flag) @@ -1057,7 +1057,7 @@ void DDLWorker::runCleanupThread() Int64 current_time_seconds = Poco::Timestamp().epochTime(); if (last_cleanup_time_seconds && current_time_seconds < last_cleanup_time_seconds + cleanup_delay_period) { - LOG_TRACE(log, "Too early to clean queue, will do it later."); + LOG_TRACE_FORMATTED(log, "Too early to clean queue, will do it later."); continue; } diff --git a/src/Interpreters/EmbeddedDictionaries.cpp b/src/Interpreters/EmbeddedDictionaries.cpp index 9ab3cf2dcbe..70d1e7144f4 100644 --- a/src/Interpreters/EmbeddedDictionaries.cpp +++ b/src/Interpreters/EmbeddedDictionaries.cpp @@ -68,7 +68,7 @@ bool EmbeddedDictionaries::reloadImpl(const bool throw_on_error, const bool forc * An attempt is made to load each directory separately. */ - LOG_INFO(log, "Loading dictionaries."); + LOG_INFO_FORMATTED(log, "Loading dictionaries."); bool was_exception = false; @@ -89,7 +89,7 @@ bool EmbeddedDictionaries::reloadImpl(const bool throw_on_error, const bool forc was_exception = true; if (!was_exception) - LOG_INFO(log, "Loaded dictionaries."); + LOG_INFO_FORMATTED(log, "Loaded dictionaries."); return !was_exception; } diff --git a/src/Interpreters/InternalTextLogsQueue.cpp b/src/Interpreters/InternalTextLogsQueue.cpp index a61b0d6dd9e..a28ee0c3fd7 100644 --- a/src/Interpreters/InternalTextLogsQueue.cpp +++ b/src/Interpreters/InternalTextLogsQueue.cpp @@ -43,7 +43,7 @@ void InternalTextLogsQueue::pushBlock(Block && log_block) if (blocksHaveEqualStructure(sample_block, log_block)) emplace(log_block.mutateColumns()); else - LOG_WARNING(&Poco::Logger::get("InternalTextLogsQueue"), "Log block have different structure"); + LOG_WARNING_FORMATTED(&Poco::Logger::get("InternalTextLogsQueue"), "Log block have different structure"); } const char * InternalTextLogsQueue::getPriorityName(int priority) diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index caa7dfb69d5..984707b904c 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -398,7 +398,7 @@ void InterpreterSystemQuery::syncReplica(ASTSystemQuery &) if (auto * storage_replicated = dynamic_cast(table.get())) { - LOG_TRACE(log, "Synchronizing entries in replica's queue with table's log and waiting for it to become empty"); + LOG_TRACE_FORMATTED(log, "Synchronizing entries in replica's queue with table's log and waiting for it to become empty"); if (!storage_replicated->waitForShrinkingQueueSize(0, context.getSettingsRef().receive_timeout.totalMilliseconds())) { LOG_ERROR(log, "SYNC REPLICA " + table_id.getNameForLogs() + ": Timed out!"); diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index 47f3828e109..3d576d79b6c 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -231,7 +231,7 @@ ProcessListEntry::~ProcessListEntry() auto user_process_list_it = parent.user_to_queries.find(user); if (user_process_list_it == parent.user_to_queries.end()) { - LOG_ERROR(&Logger::get("ProcessList"), "Logical error: cannot find user in ProcessList"); + LOG_ERROR_FORMATTED(&Logger::get("ProcessList"), "Logical error: cannot find user in ProcessList"); std::terminate(); } @@ -250,7 +250,7 @@ ProcessListEntry::~ProcessListEntry() if (!found) { - LOG_ERROR(&Logger::get("ProcessList"), "Logical error: cannot find query by query_id and pointer to ProcessListElement in ProcessListForUser"); + LOG_ERROR_FORMATTED(&Logger::get("ProcessList"), "Logical error: cannot find query by query_id and pointer to ProcessListElement in ProcessListForUser"); std::terminate(); } parent.have_space.notify_all(); diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h index 53c41fe0b08..3c07a2e4e6f 100644 --- a/src/Interpreters/SystemLog.h +++ b/src/Interpreters/SystemLog.h @@ -377,7 +377,7 @@ void SystemLog::savingThreadFunction() tryLogCurrentException(__PRETTY_FUNCTION__); } } - LOG_TRACE(log, "Terminating"); + LOG_TRACE_FORMATTED(log, "Terminating"); } @@ -426,7 +426,7 @@ void SystemLog::flushImpl(const std::vector & to_flush, flush_event.notify_all(); } - LOG_TRACE(log, "Flushed system log"); + LOG_TRACE_FORMATTED(log, "Flushed system log"); } diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index 080c9c433df..18fe2735c49 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -218,7 +218,7 @@ void ThreadStatus::detachQuery(bool exit_if_already_detached, bool thread_exits) #if defined(__linux__) if (os_thread_priority) { - LOG_TRACE(log, "Resetting nice"); + LOG_TRACE_FORMATTED(log, "Resetting nice"); if (0 != setpriority(PRIO_PROCESS, thread_id, 0)) LOG_ERROR(log, "Cannot 'setpriority' back to zero: " << errnoToString(ErrorCodes::CANNOT_SET_THREAD_PRIORITY, errno)); diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index 134894232e3..d0d521e3e2c 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -515,7 +515,7 @@ void AggregatingTransform::consume(Chunk chunk) if (!is_consume_started) { - LOG_TRACE(log, "Aggregating"); + LOG_TRACE_FORMATTED(log, "Aggregating"); is_consume_started = true; } diff --git a/src/Processors/Transforms/CreatingSetsTransform.cpp b/src/Processors/Transforms/CreatingSetsTransform.cpp index f5637a21ede..d35938e23b0 100644 --- a/src/Processors/Transforms/CreatingSetsTransform.cpp +++ b/src/Processors/Transforms/CreatingSetsTransform.cpp @@ -103,7 +103,7 @@ void CreatingSetsTransform::finishSubquery(SubqueryForSet & subquery) } else { - LOG_DEBUG(log, "Subquery has empty result."); + LOG_DEBUG_FORMATTED(log, "Subquery has empty result."); } } diff --git a/src/Processors/Transforms/MergingAggregatedTransform.cpp b/src/Processors/Transforms/MergingAggregatedTransform.cpp index 81341456faf..8340238f011 100644 --- a/src/Processors/Transforms/MergingAggregatedTransform.cpp +++ b/src/Processors/Transforms/MergingAggregatedTransform.cpp @@ -20,7 +20,7 @@ void MergingAggregatedTransform::consume(Chunk chunk) if (!consume_started) { consume_started = true; - LOG_TRACE(log, "Reading blocks of partially aggregated data."); + LOG_TRACE_FORMATTED(log, "Reading blocks of partially aggregated data."); } total_input_rows += chunk.getNumRows(); diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp index 2947b47eefa..e777b2122b0 100644 --- a/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/src/Storages/Distributed/DirectoryMonitor.cpp @@ -153,7 +153,7 @@ void StorageDistributedDirectoryMonitor::run() } else { - LOG_DEBUG(log, "Skipping send data over distributed table."); + LOG_DEBUG_FORMATTED(log, "Skipping send data over distributed table."); } const auto now = std::chrono::system_clock::now(); diff --git a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp index ad9d660a989..a118fdf23ca 100644 --- a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp +++ b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp @@ -163,7 +163,7 @@ void ReadBufferFromKafkaConsumer::commit() } else { - LOG_TRACE(log, "Nothing to commit."); + LOG_TRACE_FORMATTED(log, "Nothing to commit."); } offsets_stored = 0; @@ -204,7 +204,7 @@ void ReadBufferFromKafkaConsumer::subscribe() void ReadBufferFromKafkaConsumer::unsubscribe() { - LOG_TRACE(log, "Re-joining claimed consumer after failure"); + LOG_TRACE_FORMATTED(log, "Re-joining claimed consumer after failure"); messages.clear(); current = messages.begin(); @@ -239,7 +239,7 @@ void ReadBufferFromKafkaConsumer::resetToLastCommitted(const char * msg) { if (assignment.empty()) { - LOG_TRACE(log, "Not assignned. Can't reset to last committed position."); + LOG_TRACE_FORMATTED(log, "Not assignned. Can't reset to last committed position."); return; } auto committed_offset = consumer->get_offsets_committed(consumer->get_assignment()); @@ -312,7 +312,7 @@ bool ReadBufferFromKafkaConsumer::nextImpl() } else { - LOG_TRACE(log, "Can't get assignment"); + LOG_TRACE_FORMATTED(log, "Can't get assignment"); stalled = true; return false; } @@ -320,7 +320,7 @@ bool ReadBufferFromKafkaConsumer::nextImpl() } else { - LOG_TRACE(log, "Stalled"); + LOG_TRACE_FORMATTED(log, "Stalled"); stalled = true; return false; } diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index 6d5d569ce5d..7fb4a9cf71d 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -212,7 +212,7 @@ void StorageKafka::shutdown() // Interrupt streaming thread stream_cancelled = true; - LOG_TRACE(log, "Waiting for cleanup"); + LOG_TRACE_FORMATTED(log, "Waiting for cleanup"); task->deactivate(); // Close all consumers @@ -340,12 +340,12 @@ void StorageKafka::updateConfiguration(cppkafka::Configuration & conf) status = rd_kafka_conf_interceptor_add_on_new(conf.get_handle(), "setThreadName", rdKafkaOnNew, self); if (status != RD_KAFKA_RESP_ERR_NO_ERROR) - LOG_ERROR(log, "Cannot set new interceptor"); + LOG_ERROR_FORMATTED(log, "Cannot set new interceptor"); // cppkafka always copy the configuration status = rd_kafka_conf_interceptor_add_on_conf_dup(conf.get_handle(), "setThreadName", rdKafkaOnConfDup, self); if (status != RD_KAFKA_RESP_ERR_NO_ERROR) - LOG_ERROR(log, "Cannot set dup conf interceptor"); + LOG_ERROR_FORMATTED(log, "Cannot set dup conf interceptor"); } } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 88a396ce763..12a6a5dbde4 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -851,7 +851,7 @@ Int64 MergeTreeData::getMaxBlockNumber() const void MergeTreeData::loadDataParts(bool skip_sanity_checks) { - LOG_DEBUG(log, "Loading data parts"); + LOG_DEBUG_FORMATTED(log, "Loading data parts"); const auto settings = getSettings(); std::vector> part_names_with_disks; @@ -901,7 +901,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) if (part_names_with_disks.empty()) { - LOG_DEBUG(log, "There is no data parts"); + LOG_DEBUG_FORMATTED(log, "There is no data parts"); return; } @@ -1323,11 +1323,11 @@ void MergeTreeData::rename(const String & new_table_path, const StorageID & new_ void MergeTreeData::dropAllData() { - LOG_TRACE(log, "dropAllData: waiting for locks."); + LOG_TRACE_FORMATTED(log, "dropAllData: waiting for locks."); auto lock = lockParts(); - LOG_TRACE(log, "dropAllData: removing data from memory."); + LOG_TRACE_FORMATTED(log, "dropAllData: removing data from memory."); DataPartsVector all_parts(data_parts_by_info.begin(), data_parts_by_info.end()); @@ -1336,7 +1336,7 @@ void MergeTreeData::dropAllData() global_context.dropCaches(); - LOG_TRACE(log, "dropAllData: removing data from filesystem."); + LOG_TRACE_FORMATTED(log, "dropAllData: removing data from filesystem."); /// Removing of each data part before recursive removal of directory is to speed-up removal, because there will be less number of syscalls. clearPartsFromFilesystem(all_parts); @@ -1344,7 +1344,7 @@ void MergeTreeData::dropAllData() for (const auto & [path, disk] : getRelativeDataPathsWithDisks()) disk->removeRecursive(path); - LOG_TRACE(log, "dropAllData: done."); + LOG_TRACE_FORMATTED(log, "dropAllData: done."); } namespace @@ -2844,7 +2844,7 @@ MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const renamed_parts.tryRenameAll(); /// Synchronously check that added parts exist and are not broken. We will write checksums.txt if it does not exist. - LOG_DEBUG(log, "Checking parts"); + LOG_DEBUG_FORMATTED(log, "Checking parts"); MutableDataPartsVector loaded_parts; loaded_parts.reserve(renamed_parts.old_and_new_names.size()); for (const auto & part_names : renamed_parts.old_and_new_names) diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 00b474a7792..4165e0a4551 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -694,7 +694,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor total_size += part->getBytesOnDisk(); if (total_size >= data_settings->min_merge_bytes_to_use_direct_io) { - LOG_DEBUG(log, "Will merge parts reading files in O_DIRECT"); + LOG_DEBUG_FORMATTED(log, "Will merge parts reading files in O_DIRECT"); read_with_direct_io = true; break; diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 92f4f6107b8..dc2da4ff03b 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -530,7 +530,7 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( if (no_data) { - LOG_DEBUG(log, "Sampling yields no data."); + LOG_DEBUG_FORMATTED(log, "Sampling yields no data."); return {}; } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp index 72255081e6b..fa4adfc0ea2 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp @@ -367,7 +367,7 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zoo if (quorum) { /// We are waiting for quorum to be satisfied. - LOG_TRACE(log, "Waiting for quorum"); + LOG_TRACE_FORMATTED(log, "Waiting for quorum"); String quorum_status_path = storage.zookeeper_path + "/quorum/status"; @@ -406,7 +406,7 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zoo ErrorCodes::UNKNOWN_STATUS_OF_INSERT); } - LOG_TRACE(log, "Quorum satisfied"); + LOG_TRACE_FORMATTED(log, "Quorum satisfied"); } } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp index 9d055545457..5f405189fd3 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp @@ -243,7 +243,7 @@ CheckResult ReplicatedMergeTreePartCheckThread::checkPart(const String & part_na if (need_stop) { - LOG_INFO(log, "Checking part was cancelled."); + LOG_INFO_FORMATTED(log, "Checking part was cancelled."); return {part_name, false, "Checking part was cancelled"}; } @@ -325,7 +325,7 @@ void ReplicatedMergeTreePartCheckThread::run() { if (!parts_set.empty()) { - LOG_ERROR(log, "Non-empty parts_set with empty parts_queue. This is a bug."); + LOG_ERROR_FORMATTED(log, "Non-empty parts_set with empty parts_queue. This is a bug."); parts_set.clear(); } } @@ -359,7 +359,7 @@ void ReplicatedMergeTreePartCheckThread::run() if (parts_queue.empty()) { - LOG_ERROR(log, "Someone erased cheking part from parts_queue. This is a bug."); + LOG_ERROR_FORMATTED(log, "Someone erased cheking part from parts_queue. This is a bug."); } else { diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 94497b2a850..78f981f39d9 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -106,7 +106,7 @@ bool ReplicatedMergeTreeQueue::load(zkutil::ZooKeeperPtr zookeeper) updateTimesInZooKeeper(zookeeper, min_unprocessed_insert_time_changed, {}); - LOG_TRACE(log, "Loaded queue"); + LOG_TRACE_FORMATTED(log, "Loaded queue"); return updated; } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index 75f15556edf..ccec2df91c2 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -68,11 +68,11 @@ void ReplicatedMergeTreeRestartingThread::run() if (first_time) { - LOG_DEBUG(log, "Activating replica."); + LOG_DEBUG_FORMATTED(log, "Activating replica."); } else { - LOG_WARNING(log, "ZooKeeper session has expired. Switching to a new session."); + LOG_WARNING_FORMATTED(log, "ZooKeeper session has expired. Switching to a new session."); bool old_val = false; if (storage.is_readonly.compare_exchange_strong(old_val, true)) @@ -189,7 +189,7 @@ bool ReplicatedMergeTreeRestartingThread::tryStartup() if (storage_settings->replicated_can_become_leader) storage.enterLeaderElection(); else - LOG_INFO(log, "Will not enter leader election because replicated_can_become_leader=0"); + LOG_INFO_FORMATTED(log, "Will not enter leader election because replicated_can_become_leader=0"); /// Anything above can throw a KeeperException if something is wrong with ZK. /// Anything below should not throw exceptions. @@ -337,7 +337,7 @@ void ReplicatedMergeTreeRestartingThread::partialShutdown() storage.partial_shutdown_event.set(); storage.replica_is_active_node = nullptr; - LOG_TRACE(log, "Waiting for threads to finish"); + LOG_TRACE_FORMATTED(log, "Waiting for threads to finish"); storage.exitLeaderElection(); @@ -348,7 +348,7 @@ void ReplicatedMergeTreeRestartingThread::partialShutdown() storage.cleanup_thread.stop(); storage.part_check_thread.stop(); - LOG_TRACE(log, "Threads finished"); + LOG_TRACE_FORMATTED(log, "Threads finished"); } @@ -357,7 +357,7 @@ void ReplicatedMergeTreeRestartingThread::shutdown() /// Stop restarting_thread before stopping other tasks - so that it won't restart them again. need_stop = true; task->deactivate(); - LOG_TRACE(log, "Restarting thread finished"); + LOG_TRACE_FORMATTED(log, "Restarting thread finished"); /// Stop other tasks. partialShutdown(); diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index be18267649c..54eccea5a83 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -573,7 +573,7 @@ void StorageDistributed::alter(const AlterCommands & params, const Context & con void StorageDistributed::startup() { if (remote_database.empty() && !remote_table_function_ptr) - LOG_WARNING(log, "Name of remote database is empty. Default database will be used implicitly."); + LOG_WARNING_FORMATTED(log, "Name of remote database is empty. Default database will be used implicitly."); if (!volume) return; diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 23dc68ca173..fbe5b6b4d4d 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -1074,7 +1074,7 @@ void StorageMergeTree::attachPartition(const ASTPtr & partition, bool attach_par LOG_INFO(log, "Attaching part " << loaded_parts[i]->name << " from " << renamed_parts.old_and_new_names[i].second); renameTempPartAndAdd(loaded_parts[i], &increment); renamed_parts.old_and_new_names[i].first.clear(); - LOG_INFO(log, "Finished attaching part"); + LOG_INFO_FORMATTED(log, "Finished attaching part"); } /// New parts with other data may appear in place of deleted parts. diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 219fa79b1a4..8bae12e2a19 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -221,7 +221,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( { skip_sanity_checks = true; - LOG_WARNING(log, "Skipping the limits on severity of changes to data parts and columns (flag force_restore_data)."); + LOG_WARNING_FORMATTED(log, "Skipping the limits on severity of changes to data parts and columns (flag force_restore_data)."); } loadDataParts(skip_sanity_checks); @@ -232,14 +232,14 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( throw Exception("Can't create replicated table without ZooKeeper", ErrorCodes::NO_ZOOKEEPER); /// Do not activate the replica. It will be readonly. - LOG_ERROR(log, "No ZooKeeper: table will be in readonly mode."); + LOG_ERROR_FORMATTED(log, "No ZooKeeper: table will be in readonly mode."); is_readonly = true; return; } if (attach && !current_zookeeper->exists(zookeeper_path + "/metadata")) { - LOG_WARNING(log, "No metadata in ZooKeeper: table will be in readonly mode."); + LOG_WARNING_FORMATTED(log, "No metadata in ZooKeeper: table will be in readonly mode."); is_readonly = true; return; } @@ -579,7 +579,7 @@ void StorageReplicatedMergeTree::createReplica() if (code == Coordination::Error::ZNODEEXISTS) throw Exception("Replica " + replica_path + " already exists.", ErrorCodes::REPLICA_IS_ALREADY_EXIST); else if (code == Coordination::Error::ZBADVERSION) - LOG_ERROR(log, "Retrying createReplica(), because some other replicas were created at the same time"); + LOG_ERROR_FORMATTED(log, "Retrying createReplica(), because some other replicas were created at the same time"); else zkutil::KeeperMultiException::check(code, ops, responses); } while (code == Coordination::Error::ZBADVERSION); @@ -1576,14 +1576,14 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) if (parts_to_add.empty()) { - LOG_INFO(log, "All parts from REPLACE PARTITION command have been already attached"); + LOG_INFO_FORMATTED(log, "All parts from REPLACE PARTITION command have been already attached"); tryRemovePartsFromZooKeeperWithRetries(parts_to_remove); return true; } if (parts_to_add.size() < all_parts.size()) { - LOG_WARNING(log, "Some (but not all) parts from REPLACE PARTITION command already exist. REPLACE PARTITION will not be atomic."); + LOG_WARNING_FORMATTED(log, "Some (but not all) parts from REPLACE PARTITION command already exist. REPLACE PARTITION will not be atomic."); } StoragePtr source_table; @@ -1839,7 +1839,7 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo zkutil::EventPtr event = std::make_shared(); if (zookeeper->exists(source_path + "/columns", nullptr, event)) { - LOG_WARNING(log, "Oops, a watch has leaked"); + LOG_WARNING_FORMATTED(log, "Oops, a watch has leaked"); break; } @@ -2451,7 +2451,7 @@ void StorageReplicatedMergeTree::enterLeaderElection() auto callback = [this]() { CurrentMetrics::add(CurrentMetrics::LeaderReplica); - LOG_INFO(log, "Became leader"); + LOG_INFO_FORMATTED(log, "Became leader"); is_leader = true; merge_selecting_task->activateAndSchedule(); @@ -2486,7 +2486,7 @@ void StorageReplicatedMergeTree::exitLeaderElection() if (is_leader) { CurrentMetrics::sub(CurrentMetrics::LeaderReplica); - LOG_INFO(log, "Stopped being leader"); + LOG_INFO_FORMATTED(log, "Stopped being leader"); is_leader = false; merge_selecting_task->deactivate(); @@ -3277,7 +3277,7 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer /// TODO (relax this lock) auto table_lock = lockExclusively(RWLockImpl::NO_QUERY, getSettings()->lock_acquire_timeout_for_background_operations); - LOG_INFO(log, "Metadata changed in ZooKeeper. Applying changes locally."); + LOG_INFO_FORMATTED(log, "Metadata changed in ZooKeeper. Applying changes locally."); auto metadata_diff = ReplicatedMergeTreeTableMetadata(*this).checkAndFindDiff(metadata_from_entry); setTableStructure(std::move(columns_from_entry), metadata_diff); @@ -3463,12 +3463,12 @@ void StorageReplicatedMergeTree::alter( std::vector unwaited; if (query_context.getSettingsRef().replication_alter_partitions_sync == 2) { - LOG_DEBUG(log, "Updated shared metadata nodes in ZooKeeper. Waiting for replicas to apply changes."); + LOG_DEBUG_FORMATTED(log, "Updated shared metadata nodes in ZooKeeper. Waiting for replicas to apply changes."); unwaited = waitForAllReplicasToProcessLogEntry(*alter_entry, false); } else if (query_context.getSettingsRef().replication_alter_partitions_sync == 1) { - LOG_DEBUG(log, "Updated shared metadata nodes in ZooKeeper. Waiting for replicas to apply changes."); + LOG_DEBUG_FORMATTED(log, "Updated shared metadata nodes in ZooKeeper. Waiting for replicas to apply changes."); waitForReplicaToProcessLogEntry(replica_name, *alter_entry); } @@ -3477,9 +3477,9 @@ void StorageReplicatedMergeTree::alter( if (mutation_znode) { - LOG_DEBUG(log, "Metadata changes applied. Will wait for data changes."); + LOG_DEBUG_FORMATTED(log, "Metadata changes applied. Will wait for data changes."); waitMutation(*mutation_znode, query_context.getSettingsRef().replication_alter_partitions_sync); - LOG_DEBUG(log, "Data changes applied."); + LOG_DEBUG_FORMATTED(log, "Data changes applied."); } } @@ -4519,7 +4519,7 @@ void StorageReplicatedMergeTree::mutate(const MutationCommands & commands, const } else if (rc == Coordination::ZBADVERSION) { - LOG_TRACE(log, "Version conflict when trying to create a mutation node, retrying..."); + LOG_TRACE_FORMATTED(log, "Version conflict when trying to create a mutation node, retrying..."); continue; } else From ee4ffbc33279bfd65bff8fcf72994ed6015dfa83 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 19:47:56 +0300 Subject: [PATCH 620/738] find {base,src,programs} -name '*.h' -or -name '*.cpp' | xargs grep -l -P 'LOG_\w+\([^,]+, "[^"]+" << [^<]+\);' | xargs sed -i -r -e 's/(LOG_\w+)\(([^,]+), "([^"]+)" << ([^<]+)\);/\1_FORMATTED(\2, "\3{}", \4);/' --- base/daemon/BaseDaemon.cpp | 2 +- programs/copier/ClusterCopier.cpp | 34 +++++++++---------- programs/copier/ZooKeeperStaff.h | 4 +-- programs/local/LocalServer.cpp | 2 +- programs/odbc-bridge/MainHandler.cpp | 2 +- programs/server/HTTPHandler.cpp | 2 +- programs/server/InterserverIOHTTPHandler.cpp | 2 +- programs/server/MySQLHandler.cpp | 4 +-- programs/server/MySQLHandlerFactory.cpp | 4 +-- programs/server/Server.cpp | 8 ++--- programs/server/TCPHandlerFactory.h | 2 +- src/Common/DNSResolver.cpp | 2 +- src/Common/PipeFDs.cpp | 2 +- src/Common/ShellCommand.cpp | 2 +- src/Core/MySQLProtocol.h | 2 +- ...regatedMemoryEfficientBlockInputStream.cpp | 2 +- src/DataStreams/ParallelInputsProcessor.h | 2 +- src/Databases/DatabaseOnDisk.cpp | 4 +-- .../Embedded/RegionsHierarchies.cpp | 2 +- src/Dictionaries/Embedded/RegionsNames.cpp | 2 +- src/Dictionaries/MySQLDictionarySource.cpp | 2 +- src/Disks/DiskLocal.cpp | 2 +- src/Disks/S3/DiskS3.cpp | 4 +-- src/Disks/S3/ProxyListConfiguration.cpp | 2 +- src/Disks/S3/ProxyResolverConfiguration.cpp | 2 +- src/Disks/S3/registerDiskS3.cpp | 2 +- src/IO/AIOContextPool.cpp | 2 +- src/IO/ReadWriteBufferFromHTTP.h | 2 +- src/IO/WriteBufferFromHTTP.cpp | 2 +- src/IO/WriteBufferFromS3.cpp | 4 +-- src/Interpreters/Aggregator.cpp | 2 +- src/Interpreters/Context.cpp | 2 +- src/Interpreters/DDLWorker.cpp | 8 ++--- src/Interpreters/HashJoin.cpp | 2 +- src/Interpreters/ThreadStatusExt.cpp | 4 +-- src/Interpreters/executeQuery.cpp | 2 +- src/Processors/Executors/PipelineExecutor.cpp | 2 +- .../Formats/Impl/AvroRowInputFormat.cpp | 2 +- .../Kafka/ReadBufferFromKafkaConsumer.cpp | 16 ++++----- src/Storages/MergeTree/DataPartsExchange.cpp | 2 +- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 4 +-- src/Storages/MergeTree/MergeTreeData.cpp | 24 ++++++------- .../MergeTree/MergeTreeDataMergerMutator.cpp | 2 +- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 10 +++--- .../MergeTree/MergeTreePartsMover.cpp | 2 +- src/Storages/MergeTree/MergeTreeReadPool.cpp | 2 +- .../ReplicatedMergeTreePartCheckThread.cpp | 2 +- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 4 +-- src/Storages/StorageDistributed.cpp | 4 +-- src/Storages/StorageMergeTree.cpp | 12 +++---- src/Storages/StorageReplicatedMergeTree.cpp | 30 ++++++++-------- 51 files changed, 124 insertions(+), 124 deletions(-) diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index c703dc29b1b..692c1809139 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -180,7 +180,7 @@ public: // levels and more info, but for completeness we log all signals // here at trace level. // Don't use strsignal here, because it's not thread-safe. - LOG_TRACE(log, "Received signal " << sig); + LOG_TRACE_FORMATTED(log, "Received signal {}", sig); if (sig == Signals::StopThread) { diff --git a/programs/copier/ClusterCopier.cpp b/programs/copier/ClusterCopier.cpp index dbd2d96c6f2..d9a75ec080d 100644 --- a/programs/copier/ClusterCopier.cpp +++ b/programs/copier/ClusterCopier.cpp @@ -26,7 +26,7 @@ void ClusterCopier::init() if (response.error != Coordination::ZOK) return; UInt64 version = ++task_description_version; - LOG_DEBUG(log, "Task description should be updated, local version " << version); + LOG_DEBUG_FORMATTED(log, "Task description should be updated, local version {}", version); }; task_description_path = task_zookeeper_path + "/description"; @@ -85,7 +85,7 @@ void ClusterCopier::discoverShardPartitions(const ConnectionTimeouts & timeouts, { TaskTable & task_table = task_shard->task_table; - LOG_INFO(log, "Discover partitions of shard " << task_shard->getDescription()); + LOG_INFO_FORMATTED(log, "Discover partitions of shard {}", task_shard->getDescription()); auto get_partitions = [&] () { return getShardPartitions(timeouts, *task_shard); }; auto existing_partitions_names = retry(get_partitions, 60); @@ -221,7 +221,7 @@ void ClusterCopier::reloadTaskDescription() if (code) throw Exception("Can't get description node " + task_description_path, ErrorCodes::BAD_ARGUMENTS); - LOG_DEBUG(log, "Loading description, zxid=" << task_description_current_stat.czxid); + LOG_DEBUG_FORMATTED(log, "Loading description, zxid={}", task_description_current_stat.czxid); auto config = getConfigurationFromXMLString(task_config_str); /// Setup settings @@ -548,7 +548,7 @@ TaskStatus ClusterCopier::tryMoveAllPiecesToDestinationTable(const TaskTable & t { if (e.code == Coordination::ZNODEEXISTS) { - LOG_DEBUG(log, "Someone is already moving pieces " << current_partition_attach_is_active); + LOG_DEBUG_FORMATTED(log, "Someone is already moving pieces {}", current_partition_attach_is_active); return TaskStatus::Active; } @@ -614,7 +614,7 @@ TaskStatus ClusterCopier::tryMoveAllPiecesToDestinationTable(const TaskTable & t " ATTACH PARTITION " + partition_name + " FROM " + getQuotedTable(helping_table); - LOG_DEBUG(log, "Executing ALTER query: " << query_alter_ast_string); + LOG_DEBUG_FORMATTED(log, "Executing ALTER query: {}", query_alter_ast_string); try { @@ -626,7 +626,7 @@ TaskStatus ClusterCopier::tryMoveAllPiecesToDestinationTable(const TaskTable & t PoolMode::GET_MANY, ClusterExecutionMode::ON_EACH_NODE); - LOG_INFO(log, "Number of nodes that executed ALTER query successfully : " << toString(num_nodes)); + LOG_INFO_FORMATTED(log, "Number of nodes that executed ALTER query successfully : {}", toString(num_nodes)); } catch (...) { @@ -647,7 +647,7 @@ TaskStatus ClusterCopier::tryMoveAllPiecesToDestinationTable(const TaskTable & t query_deduplicate_ast_string += " OPTIMIZE TABLE " + getQuotedTable(original_table) + " PARTITION " + partition_name + " DEDUPLICATE;"; - LOG_DEBUG(log, "Executing OPTIMIZE DEDUPLICATE query: " << query_alter_ast_string); + LOG_DEBUG_FORMATTED(log, "Executing OPTIMIZE DEDUPLICATE query: {}", query_alter_ast_string); UInt64 num_nodes = executeQueryOnCluster( task_table.cluster_push, @@ -832,7 +832,7 @@ bool ClusterCopier::tryDropPartitionPiece( /// It is important, DROP PARTITION must be done synchronously settings_push.replication_alter_partitions_sync = 2; - LOG_DEBUG(log, "Execute distributed DROP PARTITION: " << query); + LOG_DEBUG_FORMATTED(log, "Execute distributed DROP PARTITION: {}", query); /// We have to drop partition_piece on each replica size_t num_shards = executeQueryOnCluster( cluster_push, query, @@ -1210,7 +1210,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( /// Load balancing auto worker_node_holder = createTaskWorkerNodeAndWaitIfNeed(zookeeper, current_task_piece_status_path, is_unprioritized_task); - LOG_DEBUG(log, "Processing " << current_task_piece_status_path); + LOG_DEBUG_FORMATTED(log, "Processing {}", current_task_piece_status_path); const String piece_status_path = partition_piece.getPartitionPieceShardsPath(); @@ -1253,7 +1253,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( { if (e.code == Coordination::ZNODEEXISTS) { - LOG_DEBUG(log, "Someone is already processing " << current_task_piece_is_active_path); + LOG_DEBUG_FORMATTED(log, "Someone is already processing {}", current_task_piece_is_active_path); return TaskStatus::Active; } @@ -1387,7 +1387,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( create_query_push_ast->as().if_not_exists = true; String query = queryToString(create_query_push_ast); - LOG_DEBUG(log, "Create destination tables. Query: " << query); + LOG_DEBUG_FORMATTED(log, "Create destination tables. Query: {}", query); UInt64 shards = executeQueryOnCluster(task_table.cluster_push, query, create_query_push_ast, &task_cluster->settings_push, PoolMode::GET_MANY); @@ -1419,7 +1419,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( const auto & settings = context.getSettingsRef(); query_insert_ast = parseQuery(p_query, query, settings.max_query_size, settings.max_parser_depth); - LOG_DEBUG(log, "Executing INSERT query: " << query); + LOG_DEBUG_FORMATTED(log, "Executing INSERT query: {}", query); } try @@ -1513,7 +1513,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( create_query_push_ast->as().if_not_exists = true; String query = queryToString(create_query_push_ast); - LOG_DEBUG(log, "Create destination tables. Query: " << query); + LOG_DEBUG_FORMATTED(log, "Create destination tables. Query: {}", query); UInt64 shards = executeQueryOnCluster(task_table.cluster_push, query, create_query_push_ast, &task_cluster->settings_push, PoolMode::GET_MANY); @@ -1582,7 +1582,7 @@ void ClusterCopier::dropHelpingTables(const TaskTable & task_table) const ClusterPtr & cluster_push = task_table.cluster_push; Settings settings_push = task_cluster->settings_push; - LOG_DEBUG(log, "Execute distributed DROP TABLE: " << query); + LOG_DEBUG_FORMATTED(log, "Execute distributed DROP TABLE: {}", query); /// We have to drop partition_piece on each replica UInt64 num_nodes = executeQueryOnCluster( cluster_push, query, @@ -1609,7 +1609,7 @@ void ClusterCopier::dropParticularPartitionPieceFromAllHelpingTables(const TaskT const ClusterPtr & cluster_push = task_table.cluster_push; Settings settings_push = task_cluster->settings_push; - LOG_DEBUG(log, "Execute distributed DROP PARTITION: " << query); + LOG_DEBUG_FORMATTED(log, "Execute distributed DROP PARTITION: {}", query); /// We have to drop partition_piece on each replica UInt64 num_nodes = executeQueryOnCluster( cluster_push, query, @@ -1620,7 +1620,7 @@ void ClusterCopier::dropParticularPartitionPieceFromAllHelpingTables(const TaskT LOG_DEBUG(log, "DROP PARTITION query was successfully executed on " << toString(num_nodes) << " nodes."); } - LOG_DEBUG(log, "All helping tables dropped partition " << partition_name); + LOG_DEBUG_FORMATTED(log, "All helping tables dropped partition {}", partition_name); } String ClusterCopier::getRemoteCreateTable(const DatabaseAndTableName & table, Connection & connection, const Settings * settings) @@ -1724,7 +1724,7 @@ std::set ClusterCopier::getShardPartitions(const ConnectionTimeouts & ti const auto & settings = context.getSettingsRef(); ASTPtr query_ast = parseQuery(parser_query, query, settings.max_query_size, settings.max_parser_depth); - LOG_DEBUG(log, "Computing destination partition set, executing query: " << query); + LOG_DEBUG_FORMATTED(log, "Computing destination partition set, executing query: {}", query); Context local_context = context; local_context.setSettings(task_cluster->settings_pull); diff --git a/programs/copier/ZooKeeperStaff.h b/programs/copier/ZooKeeperStaff.h index 2fc4d35400d..47a2df51ffb 100644 --- a/programs/copier/ZooKeeperStaff.h +++ b/programs/copier/ZooKeeperStaff.h @@ -183,11 +183,11 @@ public: switch (rsp.type) { case Coordination::CREATED: - LOG_DEBUG(logger, "CleanStateClock change: CREATED, at " << rsp.path); + LOG_DEBUG_FORMATTED(logger, "CleanStateClock change: CREATED, at {}", rsp.path); stale->store(true); break; case Coordination::CHANGED: - LOG_DEBUG(logger, "CleanStateClock change: CHANGED, at" << rsp.path); + LOG_DEBUG_FORMATTED(logger, "CleanStateClock change: CHANGED, at{}", rsp.path); stale->store(true); } } diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 7a164e2b108..0ac05658181 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -211,7 +211,7 @@ try /// Lock path directory before read status.emplace(context->getPath() + "status"); - LOG_DEBUG(log, "Loading metadata from " << context->getPath()); + LOG_DEBUG_FORMATTED(log, "Loading metadata from {}", context->getPath()); loadMetadataSystem(*context); attachSystemTables(); loadMetadata(*context); diff --git a/programs/odbc-bridge/MainHandler.cpp b/programs/odbc-bridge/MainHandler.cpp index edc92552c44..05d1cb5d162 100644 --- a/programs/odbc-bridge/MainHandler.cpp +++ b/programs/odbc-bridge/MainHandler.cpp @@ -171,7 +171,7 @@ void ODBCHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne else { std::string query = params.get("query"); - LOG_TRACE(log, "Query: " << query); + LOG_TRACE_FORMATTED(log, "Query: {}", query); BlockOutputStreamPtr writer = FormatFactory::instance().getOutput(format, out, *sample_block, context); auto pool = getPool(connection_string); diff --git a/programs/server/HTTPHandler.cpp b/programs/server/HTTPHandler.cpp index b50cba97d90..7569ee057bc 100644 --- a/programs/server/HTTPHandler.cpp +++ b/programs/server/HTTPHandler.cpp @@ -241,7 +241,7 @@ void HTTPHandler::processQuery( CurrentThread::QueryScope query_scope(context); - LOG_TRACE(log, "Request URI: " << request.getURI()); + LOG_TRACE_FORMATTED(log, "Request URI: {}", request.getURI()); std::istream & istr = request.stream(); diff --git a/programs/server/InterserverIOHTTPHandler.cpp b/programs/server/InterserverIOHTTPHandler.cpp index 1dc7fa88dc0..bedd38c28ef 100644 --- a/programs/server/InterserverIOHTTPHandler.cpp +++ b/programs/server/InterserverIOHTTPHandler.cpp @@ -53,7 +53,7 @@ void InterserverIOHTTPHandler::processQuery(Poco::Net::HTTPServerRequest & reque { HTMLForm params(request); - LOG_TRACE(log, "Request URI: " << request.getURI()); + LOG_TRACE_FORMATTED(log, "Request URI: {}", request.getURI()); String endpoint_name = params.get("endpoint"); bool compress = params.get("compress") == "true"; diff --git a/programs/server/MySQLHandler.cpp b/programs/server/MySQLHandler.cpp index b6919c2f188..f070c850509 100644 --- a/programs/server/MySQLHandler.cpp +++ b/programs/server/MySQLHandler.cpp @@ -197,7 +197,7 @@ void MySQLHandler::finishHandshake(MySQLProtocol::HandshakeResponse & packet) read_bytes(3); /// We can find out whether it is SSLRequest of HandshakeResponse by first 3 bytes. size_t payload_size = unalignedLoad(buf) & 0xFFFFFFu; - LOG_TRACE(log, "payload size: " << payload_size); + LOG_TRACE_FORMATTED(log, "payload size: {}", payload_size); if (payload_size == SSL_REQUEST_PAYLOAD_SIZE) { @@ -245,7 +245,7 @@ void MySQLHandler::comInitDB(ReadBuffer & payload) { String database; readStringUntilEOF(database, payload); - LOG_DEBUG(log, "Setting current database to " << database); + LOG_DEBUG_FORMATTED(log, "Setting current database to {}", database); connection_context.setCurrentDatabase(database); packet_sender->sendPacket(OK_Packet(0, client_capability_flags, 0, 0, 1), true); } diff --git a/programs/server/MySQLHandlerFactory.cpp b/programs/server/MySQLHandlerFactory.cpp index 3fbab08d413..9c98739380e 100644 --- a/programs/server/MySQLHandlerFactory.cpp +++ b/programs/server/MySQLHandlerFactory.cpp @@ -32,7 +32,7 @@ MySQLHandlerFactory::MySQLHandlerFactory(IServer & server_) } catch (...) { - LOG_TRACE(log, "Failed to create SSL context. SSL will be disabled. Error: " << getCurrentExceptionMessage(false)); + LOG_TRACE_FORMATTED(log, "Failed to create SSL context. SSL will be disabled. Error: {}", getCurrentExceptionMessage(false)); ssl_enabled = false; } @@ -43,7 +43,7 @@ MySQLHandlerFactory::MySQLHandlerFactory(IServer & server_) } catch (...) { - LOG_TRACE(log, "Failed to read RSA key pair from server certificate. Error: " << getCurrentExceptionMessage(false)); + LOG_TRACE_FORMATTED(log, "Failed to read RSA key pair from server certificate. Error: {}", getCurrentExceptionMessage(false)); generateRSAKeys(); } #endif diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index e3fbd234dfd..a097b12f2d8 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -101,11 +101,11 @@ void setupTmpPath(Logger * log, const std::string & path) { if (it->isFile() && startsWith(it.name(), "tmp")) { - LOG_DEBUG(log, "Removing old temporary file " << it->path()); + LOG_DEBUG_FORMATTED(log, "Removing old temporary file {}", it->path()); it->remove(); } else - LOG_DEBUG(log, "Skipped file in temporary path " << it->path()); + LOG_DEBUG_FORMATTED(log, "Skipped file in temporary path {}", it->path()); } } @@ -349,7 +349,7 @@ int Server::main(const std::vector & /*args*/) if (rlim.rlim_cur == rlim.rlim_max) { - LOG_DEBUG(log, "rlimit on number of file descriptors is " << rlim.rlim_cur); + LOG_DEBUG_FORMATTED(log, "rlimit on number of file descriptors is {}", rlim.rlim_cur); } else { @@ -579,7 +579,7 @@ int Server::main(const std::vector & /*args*/) if (max_server_memory_usage == 0) { max_server_memory_usage = default_max_server_memory_usage; - LOG_INFO(log, "Setting max_server_memory_usage was set to " << formatReadableSizeWithBinarySuffix(max_server_memory_usage)); + LOG_INFO_FORMATTED(log, "Setting max_server_memory_usage was set to {}", formatReadableSizeWithBinarySuffix(max_server_memory_usage)); } else if (max_server_memory_usage > default_max_server_memory_usage) { diff --git a/programs/server/TCPHandlerFactory.h b/programs/server/TCPHandlerFactory.h index ab000718d6f..dc08fc8825d 100644 --- a/programs/server/TCPHandlerFactory.h +++ b/programs/server/TCPHandlerFactory.h @@ -35,7 +35,7 @@ public: { try { - LOG_TRACE(log, "TCP Request. Address: " << socket.peerAddress().toString()); + LOG_TRACE_FORMATTED(log, "TCP Request. Address: {}", socket.peerAddress().toString()); return new TCPHandler(server, socket); } catch (const Poco::Net::NetException &) diff --git a/src/Common/DNSResolver.cpp b/src/Common/DNSResolver.cpp index b0b9c61bc84..4b039bb810a 100644 --- a/src/Common/DNSResolver.cpp +++ b/src/Common/DNSResolver.cpp @@ -202,7 +202,7 @@ bool DNSResolver::updateCache() } if (!lost_hosts.empty()) - LOG_INFO(&Logger::get("DNSResolver"), "Cached hosts not found: " << lost_hosts); + LOG_INFO_FORMATTED(&Logger::get("DNSResolver"), "Cached hosts not found: {}", lost_hosts); return updated; } diff --git a/src/Common/PipeFDs.cpp b/src/Common/PipeFDs.cpp index 17eeb9aaef7..bb1dad4e2d3 100644 --- a/src/Common/PipeFDs.cpp +++ b/src/Common/PipeFDs.cpp @@ -102,7 +102,7 @@ void LazyPipeFDs::tryIncreaseSize(int desired_size) if (-1 == fcntl(fds_rw[1], F_SETPIPE_SZ, pipe_size * 2) && errno != EPERM) throwFromErrno("Cannot increase pipe capacity to " + std::to_string(pipe_size * 2), ErrorCodes::CANNOT_FCNTL); - LOG_TRACE(log, "Pipe capacity is " << formatReadableSizeWithBinarySuffix(std::min(pipe_size, desired_size))); + LOG_TRACE_FORMATTED(log, "Pipe capacity is {}", formatReadableSizeWithBinarySuffix(std::min(pipe_size, desired_size))); } #else (void)desired_size; diff --git a/src/Common/ShellCommand.cpp b/src/Common/ShellCommand.cpp index cf7ddb559c1..98a01e18511 100644 --- a/src/Common/ShellCommand.cpp +++ b/src/Common/ShellCommand.cpp @@ -188,7 +188,7 @@ int ShellCommand::tryWait() { wait_called = true; - LOG_TRACE(getLogger(), "Will wait for shell command pid " << pid); + LOG_TRACE_FORMATTED(getLogger(), "Will wait for shell command pid {}", pid); int status = 0; if (-1 == waitpid(pid, &status, 0)) diff --git a/src/Core/MySQLProtocol.h b/src/Core/MySQLProtocol.h index f0cacca6358..fded91c116c 100644 --- a/src/Core/MySQLProtocol.h +++ b/src/Core/MySQLProtocol.h @@ -1068,7 +1068,7 @@ public: # pragma GCC diagnostic pop String pem(pem_buf, pem_size); - LOG_TRACE(log, "Key: " << pem); + LOG_TRACE_FORMATTED(log, "Key: {}", pem); AuthMoreData data(pem); packet_sender->sendPacket(data, true); diff --git a/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp b/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp index 4dbd1e0d9ec..382dbc37063 100644 --- a/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp +++ b/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp @@ -149,7 +149,7 @@ void MergingAggregatedMemoryEfficientBlockInputStream::cancel(bool kill) * (example: connection reset during distributed query execution) * - then don't care. */ - LOG_ERROR(log, "Exception while cancelling " << input.stream->getName()); + LOG_ERROR_FORMATTED(log, "Exception while cancelling {}", input.stream->getName()); } } } diff --git a/src/DataStreams/ParallelInputsProcessor.h b/src/DataStreams/ParallelInputsProcessor.h index a786dac7497..62d26fada2e 100644 --- a/src/DataStreams/ParallelInputsProcessor.h +++ b/src/DataStreams/ParallelInputsProcessor.h @@ -134,7 +134,7 @@ public: * (for example, the connection is broken for distributed query processing) * - then do not care. */ - LOG_ERROR(log, "Exception while cancelling " << input->getName()); + LOG_ERROR_FORMATTED(log, "Exception while cancelling {}", input->getName()); } } } diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index aed9f38c2ea..e5c8c5cf287 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -381,7 +381,7 @@ void DatabaseOnDisk::iterateMetadataFiles(const Context & context, const Iterati } else { - LOG_INFO(log, "Removing file " << getMetadataPath() + file_name); + LOG_INFO_FORMATTED(log, "Removing file {}", getMetadataPath() + file_name); Poco::File(getMetadataPath() + file_name).remove(); } }; @@ -406,7 +406,7 @@ void DatabaseOnDisk::iterateMetadataFiles(const Context & context, const Iterati else if (endsWith(dir_it.name(), ".sql.tmp")) { /// There are files .sql.tmp - delete - LOG_INFO(log, "Removing file " << dir_it->path()); + LOG_INFO_FORMATTED(log, "Removing file {}", dir_it->path()); Poco::File(dir_it->path()).remove(); } else if (endsWith(dir_it.name(), ".sql")) diff --git a/src/Dictionaries/Embedded/RegionsHierarchies.cpp b/src/Dictionaries/Embedded/RegionsHierarchies.cpp index 906ae381a45..eaab1fac214 100644 --- a/src/Dictionaries/Embedded/RegionsHierarchies.cpp +++ b/src/Dictionaries/Embedded/RegionsHierarchies.cpp @@ -13,7 +13,7 @@ RegionsHierarchies::RegionsHierarchies(IRegionsHierarchiesDataProviderPtr data_p for (const auto & name : data_provider->listCustomHierarchies()) { - LOG_DEBUG(log, "Adding regions hierarchy for " << name); + LOG_DEBUG_FORMATTED(log, "Adding regions hierarchy for {}", name); data.emplace(name, data_provider->getHierarchySource(name)); } diff --git a/src/Dictionaries/Embedded/RegionsNames.cpp b/src/Dictionaries/Embedded/RegionsNames.cpp index d696b35c5d0..12362c7e50e 100644 --- a/src/Dictionaries/Embedded/RegionsNames.cpp +++ b/src/Dictionaries/Embedded/RegionsNames.cpp @@ -55,7 +55,7 @@ void RegionsNames::reload() if (!names_source || !names_source->isModified()) continue; - LOG_DEBUG(log, "Reloading regions names for language: " << language); + LOG_DEBUG_FORMATTED(log, "Reloading regions names for language: {}", language); auto names_reader = names_source->createReader(); diff --git a/src/Dictionaries/MySQLDictionarySource.cpp b/src/Dictionaries/MySQLDictionarySource.cpp index 632951613ac..884a88c1c3e 100644 --- a/src/Dictionaries/MySQLDictionarySource.cpp +++ b/src/Dictionaries/MySQLDictionarySource.cpp @@ -227,7 +227,7 @@ LocalDateTime MySQLDictionarySource::getLastModification(mysqlxx::Pool::Entry & if (!update_time_value.isNull()) { modification_time = update_time_value.getDateTime(); - LOG_TRACE(log, "Got modification time: " << modification_time); + LOG_TRACE_FORMATTED(log, "Got modification time: {}", modification_time); } /// fetch remaining rows to avoid "commands out of sync" error diff --git a/src/Disks/DiskLocal.cpp b/src/Disks/DiskLocal.cpp index 7e089ba8b51..3b3dbf155f8 100644 --- a/src/Disks/DiskLocal.cpp +++ b/src/Disks/DiskLocal.cpp @@ -91,7 +91,7 @@ bool DiskLocal::tryReserve(UInt64 bytes) std::lock_guard lock(DiskLocal::reservation_mutex); if (bytes == 0) { - LOG_DEBUG(&Logger::get("DiskLocal"), "Reserving 0 bytes on disk " << backQuote(name)); + LOG_DEBUG_FORMATTED(&Logger::get("DiskLocal"), "Reserving 0 bytes on disk {}", backQuote(name)); ++reservation_count; return true; } diff --git a/src/Disks/S3/DiskS3.cpp b/src/Disks/S3/DiskS3.cpp index 1b058354545..af83fdbfe10 100644 --- a/src/Disks/S3/DiskS3.cpp +++ b/src/Disks/S3/DiskS3.cpp @@ -556,7 +556,7 @@ std::unique_ptr DiskS3::writeFile(const String & path, void DiskS3::remove(const String & path) { - LOG_DEBUG(&Logger::get("DiskS3"), "Remove file by path: " << backQuote(metadata_path + path)); + LOG_DEBUG_FORMATTED(&Logger::get("DiskS3"), "Remove file by path: {}", backQuote(metadata_path + path)); Poco::File file(metadata_path + path); if (file.isFile()) @@ -610,7 +610,7 @@ bool DiskS3::tryReserve(UInt64 bytes) std::lock_guard lock(reservation_mutex); if (bytes == 0) { - LOG_DEBUG(&Logger::get("DiskS3"), "Reserving 0 bytes on s3 disk " << backQuote(name)); + LOG_DEBUG_FORMATTED(&Logger::get("DiskS3"), "Reserving 0 bytes on s3 disk {}", backQuote(name)); ++reservation_count; return true; } diff --git a/src/Disks/S3/ProxyListConfiguration.cpp b/src/Disks/S3/ProxyListConfiguration.cpp index bc68635fa7b..3842fc727fd 100644 --- a/src/Disks/S3/ProxyListConfiguration.cpp +++ b/src/Disks/S3/ProxyListConfiguration.cpp @@ -20,7 +20,7 @@ Aws::Client::ClientConfigurationPerRequest ProxyListConfiguration::getConfigurat cfg.proxyHost = proxies[index].getHost(); cfg.proxyPort = proxies[index].getPort(); - LOG_DEBUG(&Logger::get("AWSClient"), "Use proxy: " << proxies[index].toString()); + LOG_DEBUG_FORMATTED(&Logger::get("AWSClient"), "Use proxy: {}", proxies[index].toString()); return cfg; } diff --git a/src/Disks/S3/ProxyResolverConfiguration.cpp b/src/Disks/S3/ProxyResolverConfiguration.cpp index c36432d933e..202c10aa7e4 100644 --- a/src/Disks/S3/ProxyResolverConfiguration.cpp +++ b/src/Disks/S3/ProxyResolverConfiguration.cpp @@ -21,7 +21,7 @@ ProxyResolverConfiguration::ProxyResolverConfiguration(const Poco::URI & endpoin Aws::Client::ClientConfigurationPerRequest ProxyResolverConfiguration::getConfiguration(const Aws::Http::HttpRequest &) { - LOG_DEBUG(&Logger::get("AWSClient"), "Obtain proxy using resolver: " << endpoint.toString()); + LOG_DEBUG_FORMATTED(&Logger::get("AWSClient"), "Obtain proxy using resolver: {}", endpoint.toString()); /// 1 second is enough for now. /// TODO: Make timeouts configurable. diff --git a/src/Disks/S3/registerDiskS3.cpp b/src/Disks/S3/registerDiskS3.cpp index 1912579275e..5da407bf9f0 100644 --- a/src/Disks/S3/registerDiskS3.cpp +++ b/src/Disks/S3/registerDiskS3.cpp @@ -71,7 +71,7 @@ namespace proxies.push_back(proxy_uri); - LOG_DEBUG(&Logger::get("DiskS3"), "Configured proxy: " << proxy_uri.toString()); + LOG_DEBUG_FORMATTED(&Logger::get("DiskS3"), "Configured proxy: {}", proxy_uri.toString()); } if (!proxies.empty()) diff --git a/src/IO/AIOContextPool.cpp b/src/IO/AIOContextPool.cpp index 31d0948fedc..43ffea00512 100644 --- a/src/IO/AIOContextPool.cpp +++ b/src/IO/AIOContextPool.cpp @@ -98,7 +98,7 @@ void AIOContextPool::fulfillPromises(const io_event events[], const int num_even const auto it = promises.find(completed_id); if (it == std::end(promises)) { - LOG_ERROR(&Poco::Logger::get("AIOcontextPool"), "Found io_event with unknown id " << completed_id); + LOG_ERROR_FORMATTED(&Poco::Logger::get("AIOcontextPool"), "Found io_event with unknown id {}", completed_id); continue; } diff --git a/src/IO/ReadWriteBufferFromHTTP.h b/src/IO/ReadWriteBufferFromHTTP.h index 4b4e4453360..51402a1f062 100644 --- a/src/IO/ReadWriteBufferFromHTTP.h +++ b/src/IO/ReadWriteBufferFromHTTP.h @@ -127,7 +127,7 @@ namespace detail if (!credentials.getUsername().empty()) credentials.authenticate(request); - LOG_TRACE((&Logger::get("ReadWriteBufferFromHTTP")), "Sending request to " << uri.toString()); + LOG_TRACE_FORMATTED((&Logger::get("ReadWriteBufferFromHTTP")), "Sending request to {}", uri.toString()); auto sess = session->getSession(); diff --git a/src/IO/WriteBufferFromHTTP.cpp b/src/IO/WriteBufferFromHTTP.cpp index 0a8095b960f..79add0d90c0 100644 --- a/src/IO/WriteBufferFromHTTP.cpp +++ b/src/IO/WriteBufferFromHTTP.cpp @@ -15,7 +15,7 @@ WriteBufferFromHTTP::WriteBufferFromHTTP( request.setHost(uri.getHost()); request.setChunkedTransferEncoding(true); - LOG_TRACE((&Logger::get("WriteBufferToHTTP")), "Sending request to " << uri.toString()); + LOG_TRACE_FORMATTED((&Logger::get("WriteBufferToHTTP")), "Sending request to {}", uri.toString()); ostr = &session->sendRequest(request); } diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 34b1c6b07ae..699db5f92fe 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -100,7 +100,7 @@ void WriteBufferFromS3::initiate() if (outcome.IsSuccess()) { upload_id = outcome.GetResult().GetUploadId(); - LOG_DEBUG(log, "Multipart upload initiated. Upload id: " << upload_id); + LOG_DEBUG_FORMATTED(log, "Multipart upload initiated. Upload id: {}", upload_id); } else throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR); @@ -170,7 +170,7 @@ void WriteBufferFromS3::complete() auto outcome = client_ptr->CompleteMultipartUpload(req); if (outcome.IsSuccess()) - LOG_DEBUG(log, "Multipart upload completed. Upload_id: " << upload_id); + LOG_DEBUG_FORMATTED(log, "Multipart upload completed. Upload_id: {}", upload_id); else throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR); } diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index d1b9b8637d3..5c4455f3281 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -551,7 +551,7 @@ bool Aggregator::executeOnBlock(Columns columns, UInt64 num_rows, AggregatedData result.init(method_chosen); result.keys_size = params.keys_size; result.key_sizes = key_sizes; - LOG_TRACE(log, "Aggregation method: " << result.getMethodName()); + LOG_TRACE_FORMATTED(log, "Aggregation method: {}", result.getMethodName()); } if (isCancelled()) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 45eeb47c035..aa983196d02 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1761,7 +1761,7 @@ void Context::updateStorageConfiguration(const Poco::Util::AbstractConfiguration } catch (Exception & e) { - LOG_ERROR(shared->log, "An error has occured while reloading storage policies, storage policies were not applied: " << e.message()); + LOG_ERROR_FORMATTED(shared->log, "An error has occured while reloading storage policies, storage policies were not applied: {}", e.message()); } } } diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index c982ddfd83e..6fa2b171981 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -579,7 +579,7 @@ bool DDLWorker::tryExecuteQuery(const String & query, const DDLTask & task, Exec } status = ExecutionStatus(0); - LOG_DEBUG(log, "Executed query: " << query); + LOG_DEBUG_FORMATTED(log, "Executed query: {}", query); return true; } @@ -631,7 +631,7 @@ void DDLWorker::processTask(DDLTask & task, const ZooKeeperPtr & zookeeper) ASTPtr rewritten_ast = task.query_on_cluster->getRewrittenASTWithoutOnCluster(task.address_in_cluster.default_database); String rewritten_query = queryToString(rewritten_ast); - LOG_DEBUG(log, "Executing query: " << rewritten_query); + LOG_DEBUG_FORMATTED(log, "Executing query: {}", rewritten_query); if (auto * query_with_table = dynamic_cast(rewritten_ast.get()); query_with_table) { @@ -1003,7 +1003,7 @@ void DDLWorker::runMainThread() { if (Coordination::isHardwareError(e.code)) { - LOG_DEBUG(log, "Recovering ZooKeeper session after: " << getCurrentExceptionMessage(false)); + LOG_DEBUG_FORMATTED(log, "Recovering ZooKeeper session after: {}", getCurrentExceptionMessage(false)); while (!stop_flag) { @@ -1023,7 +1023,7 @@ void DDLWorker::runMainThread() } else if (e.code == Coordination::ZNONODE) { - LOG_ERROR(log, "ZooKeeper error: " << getCurrentExceptionMessage(true)); + LOG_ERROR_FORMATTED(log, "ZooKeeper error: {}", getCurrentExceptionMessage(true)); } else { diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index 0e463a84dc4..776d06e645d 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -422,7 +422,7 @@ void HashJoin::setSampleBlock(const Block & block) /// You have to restore this lock if you call the function outside of ctor. //std::unique_lock lock(rwlock); - LOG_DEBUG(log, "setSampleBlock: " << block.dumpStructure()); + LOG_DEBUG_FORMATTED(log, "setSampleBlock: {}", block.dumpStructure()); if (!empty()) return; diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index 18fe2735c49..a27508303c4 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -95,7 +95,7 @@ void ThreadStatus::setupState(const ThreadGroupStatusPtr & thread_group_) Int32 new_os_thread_priority = settings.os_thread_priority; if (new_os_thread_priority && hasLinuxCapability(CAP_SYS_NICE)) { - LOG_TRACE(log, "Setting nice to " << new_os_thread_priority); + LOG_TRACE_FORMATTED(log, "Setting nice to {}", new_os_thread_priority); if (0 != setpriority(PRIO_PROCESS, thread_id, new_os_thread_priority)) throwFromErrno("Cannot 'setpriority'", ErrorCodes::CANNOT_SET_THREAD_PRIORITY); @@ -221,7 +221,7 @@ void ThreadStatus::detachQuery(bool exit_if_already_detached, bool thread_exits) LOG_TRACE_FORMATTED(log, "Resetting nice"); if (0 != setpriority(PRIO_PROCESS, thread_id, 0)) - LOG_ERROR(log, "Cannot 'setpriority' back to zero: " << errnoToString(ErrorCodes::CANNOT_SET_THREAD_PRIORITY, errno)); + LOG_ERROR_FORMATTED(log, "Cannot 'setpriority' back to zero: {}", errnoToString(ErrorCodes::CANNOT_SET_THREAD_PRIORITY, errno)); os_thread_priority = 0; } diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index edb03bcfb68..00f8c1bfdc5 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -105,7 +105,7 @@ static void logQuery(const String & query, const Context & context, bool interna { if (internal) { - LOG_DEBUG(&Logger::get("executeQuery"), "(internal) " << joinLines(query)); + LOG_DEBUG_FORMATTED(&Logger::get("executeQuery"), "(internal) {}", joinLines(query)); } else { diff --git a/src/Processors/Executors/PipelineExecutor.cpp b/src/Processors/Executors/PipelineExecutor.cpp index 4552262040b..5bf7ec0ff15 100644 --- a/src/Processors/Executors/PipelineExecutor.cpp +++ b/src/Processors/Executors/PipelineExecutor.cpp @@ -486,7 +486,7 @@ void PipelineExecutor::execute(size_t num_threads) catch (...) { #ifndef NDEBUG - LOG_TRACE(log, "Exception while executing query. Current state:\n" << dumpPipeline()); + LOG_TRACE_FORMATTED(log, "Exception while executing query. Current state:\n{}", dumpPipeline()); #endif throw; } diff --git a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp index d92cefbde7d..d0272e6f277 100644 --- a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp @@ -612,7 +612,7 @@ private: try { Poco::URI url(base_url, "/schemas/ids/" + std::to_string(id)); - LOG_TRACE((&Logger::get("AvroConfluentRowInputFormat")), "Fetching schema id = " << id); + LOG_TRACE_FORMATTED((&Logger::get("AvroConfluentRowInputFormat")), "Fetching schema id = {}", id); /// One second for connect/send/receive. Just in case. ConnectionTimeouts timeouts({1, 0}, {1, 0}, {1, 0}); diff --git a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp index a118fdf23ca..873e03580bc 100644 --- a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp +++ b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp @@ -38,7 +38,7 @@ ReadBufferFromKafkaConsumer::ReadBufferFromKafkaConsumer( // called (synchroniously, during poll) when we enter the consumer group consumer->set_assignment_callback([this](const cppkafka::TopicPartitionList & topic_partitions) { - LOG_TRACE(log, "Topics/partitions assigned: " << topic_partitions); + LOG_TRACE_FORMATTED(log, "Topics/partitions assigned: {}", topic_partitions); assignment = topic_partitions; }); @@ -47,7 +47,7 @@ ReadBufferFromKafkaConsumer::ReadBufferFromKafkaConsumer( { // Rebalance is happening now, and now we have a chance to finish the work // with topics/partitions we were working with before rebalance - LOG_TRACE(log, "Rebalance initiated. Revoking partitions: " << topic_partitions); + LOG_TRACE_FORMATTED(log, "Rebalance initiated. Revoking partitions: {}", topic_partitions); // we can not flush data to target from that point (it is pulled, not pushed) // so the best we can now it to @@ -70,13 +70,13 @@ ReadBufferFromKafkaConsumer::ReadBufferFromKafkaConsumer( // } // catch (cppkafka::HandleException & e) // { - // LOG_WARNING(log, "Commit error: " << e.what()); + // LOG_WARNING_FORMATTED(log, "Commit error: {}", e.what()); // } }); consumer->set_rebalance_error_callback([this](cppkafka::Error err) { - LOG_ERROR(log, "Rebalance error: " << err); + LOG_ERROR_FORMATTED(log, "Rebalance error: {}", err); }); } @@ -150,7 +150,7 @@ void ReadBufferFromKafkaConsumer::commit() } catch (const cppkafka::HandleException & e) { - LOG_ERROR(log, "Exception during commit attempt: " << e.what()); + LOG_ERROR_FORMATTED(log, "Exception during commit attempt: {}", e.what()); } --max_retries; } @@ -176,7 +176,7 @@ void ReadBufferFromKafkaConsumer::subscribe() << boost::algorithm::join(consumer->get_subscription(), ", ") << " ]"); - LOG_TRACE(log, "Already assigned to : " << assignment); + LOG_TRACE_FORMATTED(log, "Already assigned to : {}", assignment); size_t max_retries = 5; @@ -223,7 +223,7 @@ void ReadBufferFromKafkaConsumer::unsubscribe() } catch (const cppkafka::HandleException & e) { - LOG_ERROR(log, "Exception from ReadBufferFromKafkaConsumer::unsubscribe: " << e.what()); + LOG_ERROR_FORMATTED(log, "Exception from ReadBufferFromKafkaConsumer::unsubscribe: {}", e.what()); } } @@ -340,7 +340,7 @@ bool ReadBufferFromKafkaConsumer::nextImpl() ++current; // TODO: should throw exception instead - LOG_ERROR(log, "Consumer error: " << err); + LOG_ERROR_FORMATTED(log, "Consumer error: {}", err); return false; } diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index 2b491f9ede6..1fe26eb06da 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -82,7 +82,7 @@ void Service::processQuery(const Poco::Net::HTMLForm & params, ReadBuffer & /*bo ++data.current_table_sends; SCOPE_EXIT({--data.current_table_sends;}); - LOG_TRACE(log, "Sending part " << part_name); + LOG_TRACE_FORMATTED(log, "Sending part {}", part_name); try { diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 44ad4af626e..882c1226273 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -267,7 +267,7 @@ void IMergeTreeDataPart::removeIfNeeded() if (state == State::DeleteOnDestroy) { - LOG_TRACE(storage.log, "Removed part from old location " << path); + LOG_TRACE_FORMATTED(storage.log, "Removed part from old location {}", path); } } catch (...) @@ -812,7 +812,7 @@ void IMergeTreeDataPart::renameToDetached(const String & prefix) const void IMergeTreeDataPart::makeCloneInDetached(const String & prefix) const { assertOnDisk(); - LOG_INFO(storage.log, "Detaching " << relative_path); + LOG_INFO_FORMATTED(storage.log, "Detaching {}", relative_path); String destination_path = storage.relative_data_path + getRelativePathForDetachedPart(prefix); diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 12a6a5dbde4..0d29f0808b2 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -1144,7 +1144,7 @@ void MergeTreeData::clearOldTemporaryDirectories(ssize_t custom_directories_life { if (disk->isDirectory(it->path()) && isOldPartDirectory(disk, it->path(), deadline)) { - LOG_WARNING(log, "Removing temporary directory " << fullPath(disk, it->path())); + LOG_WARNING_FORMATTED(log, "Removing temporary directory {}", fullPath(disk, it->path())); disk->removeRecursive(it->path()); } } @@ -1281,7 +1281,7 @@ void MergeTreeData::clearPartsFromFilesystem(const DataPartsVector & parts_to_re { pool.scheduleOrThrowOnError([&] { - LOG_DEBUG(log, "Removing part from filesystem " << part->name); + LOG_DEBUG_FORMATTED(log, "Removing part from filesystem {}", part->name); part->remove(); }); } @@ -1292,7 +1292,7 @@ void MergeTreeData::clearPartsFromFilesystem(const DataPartsVector & parts_to_re { for (const DataPartPtr & part : parts_to_remove) { - LOG_DEBUG(log, "Removing part from filesystem " << part->name); + LOG_DEBUG_FORMATTED(log, "Removing part from filesystem {}", part->name); part->remove(); } } @@ -1684,7 +1684,7 @@ void MergeTreeData::PartsTemporaryRename::tryRenameAll() catch (...) { old_and_new_names.resize(i); - LOG_WARNING(storage.log, "Cannot rename parts to perform operation on them: " << getCurrentExceptionMessage(false)); + LOG_WARNING_FORMATTED(storage.log, "Cannot rename parts to perform operation on them: {}", getCurrentExceptionMessage(false)); throw; } } @@ -2004,7 +2004,7 @@ restore_covered) if (restore_covered && part->info.level == 0) { - LOG_WARNING(log, "Will not recover parts covered by zero-level part " << part->name); + LOG_WARNING_FORMATTED(log, "Will not recover parts covered by zero-level part {}", part->name); return; } @@ -2085,7 +2085,7 @@ restore_covered) for (const String & name : restored) { - LOG_INFO(log, "Activated part " << name); + LOG_INFO_FORMATTED(log, "Activated part {}", name); } if (error) @@ -2104,7 +2104,7 @@ void MergeTreeData::tryRemovePartImmediately(DataPartPtr && part) { auto lock = lockParts(); - LOG_TRACE(log, "Trying to immediately remove part " << part->getNameWithState()); + LOG_TRACE_FORMATTED(log, "Trying to immediately remove part {}", part->getNameWithState()); auto it = data_parts_by_info.find(part->info); if (it == data_parts_by_info.end() || (*it).get() != part.get()) @@ -2130,7 +2130,7 @@ void MergeTreeData::tryRemovePartImmediately(DataPartPtr && part) } removePartsFinally({part_to_delete}); - LOG_TRACE(log, "Removed part " << part_to_delete->name); + LOG_TRACE_FORMATTED(log, "Removed part {}", part_to_delete->name); } @@ -2779,7 +2779,7 @@ void MergeTreeData::dropDetached(const ASTPtr & partition, bool part, const Cont { const auto & [path, disk] = renamed_parts.old_part_name_to_path_and_disk[old_name]; disk->removeRecursive(path + "detached/" + new_name + "/"); - LOG_DEBUG(log, "Dropped detached part " << old_name); + LOG_DEBUG_FORMATTED(log, "Dropped detached part {}", old_name); old_name.clear(); } } @@ -2819,7 +2819,7 @@ MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const { continue; } - LOG_DEBUG(log, "Found part " << name); + LOG_DEBUG_FORMATTED(log, "Found part {}", name); active_parts.add(name); name_to_disk[name] = disk; } @@ -2849,7 +2849,7 @@ MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const loaded_parts.reserve(renamed_parts.old_and_new_names.size()); for (const auto & part_names : renamed_parts.old_and_new_names) { - LOG_DEBUG(log, "Checking part " << part_names.second); + LOG_DEBUG_FORMATTED(log, "Checking part {}", part_names.second); auto single_disk_volume = std::make_shared("volume_" + part_names.first, name_to_disk[part_names.first]); MutableDataPartPtr part = createPart(part_names.first, single_disk_volume, source_dir + part_names.second); loadPartAndFixMetadataImpl(part); @@ -3042,7 +3042,7 @@ void MergeTreeData::Transaction::rollback() for (const auto & part : precommitted_parts) ss << " " << part->relative_path; ss << "."; - LOG_DEBUG(data.log, "Undoing transaction." << ss.str()); + LOG_DEBUG_FORMATTED(data.log, "Undoing transaction.{}", ss.str()); data.removePartsFromWorkingSet( DataPartsVector(precommitted_parts.begin(), precommitted_parts.end()), diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 4165e0a4551..2d482e676da 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -640,7 +640,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor size_t sum_input_rows_upper_bound = merge_entry->total_rows_count; MergeAlgorithm merge_alg = chooseMergeAlgorithm(parts, sum_input_rows_upper_bound, gathering_columns, deduplicate, need_remove_expired_values); - LOG_DEBUG(log, "Selected MergeAlgorithm: " << ((merge_alg == MergeAlgorithm::Vertical) ? "Vertical" : "Horizontal")); + LOG_DEBUG_FORMATTED(log, "Selected MergeAlgorithm: {}", ((merge_alg == MergeAlgorithm::Vertical) ? "Vertical" : "Horizontal")); /// Note: this is done before creating input streams, because otherwise data.data_parts_mutex /// (which is locked in data.getTotalActiveSizeInBytes()) diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index dc2da4ff03b..f51e56f7735 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -107,7 +107,7 @@ size_t MergeTreeDataSelectExecutor::getApproximateTotalRowsToRead( size_t rows_count = 0; /// We will find out how many rows we would have read without sampling. - LOG_DEBUG(log, "Preliminary index scan with condition: " << key_condition.toString()); + LOG_DEBUG_FORMATTED(log, "Preliminary index scan with condition: {}", key_condition.toString()); for (const auto & part : parts) { @@ -329,7 +329,7 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( if (relative_sample_size > 1) { relative_sample_size = convertAbsoluteSampleSizeToRelative(select_sample_size, approx_total_rows); - LOG_DEBUG(log, "Selected relative sample size: " << toString(relative_sample_size)); + LOG_DEBUG_FORMATTED(log, "Selected relative sample size: {}", toString(relative_sample_size)); } /// SAMPLE 1 is the same as the absence of SAMPLE. @@ -342,7 +342,7 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( if (relative_sample_offset > 1) { relative_sample_offset = convertAbsoluteSampleSizeToRelative(select_sample_offset, approx_total_rows); - LOG_DEBUG(log, "Selected relative sample offset: " << toString(relative_sample_offset)); + LOG_DEBUG_FORMATTED(log, "Selected relative sample offset: {}", toString(relative_sample_offset)); } } @@ -534,9 +534,9 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( return {}; } - LOG_DEBUG(log, "Key condition: " << key_condition.toString()); + LOG_DEBUG_FORMATTED(log, "Key condition: {}", key_condition.toString()); if (minmax_idx_condition) - LOG_DEBUG(log, "MinMax index condition: " << minmax_idx_condition->toString()); + LOG_DEBUG_FORMATTED(log, "MinMax index condition: {}", minmax_idx_condition->toString()); /// PREWHERE String prewhere_column; diff --git a/src/Storages/MergeTree/MergeTreePartsMover.cpp b/src/Storages/MergeTree/MergeTreePartsMover.cpp index 1609e1e517e..d9c7492fb41 100644 --- a/src/Storages/MergeTree/MergeTreePartsMover.cpp +++ b/src/Storages/MergeTree/MergeTreePartsMover.cpp @@ -193,7 +193,7 @@ MergeTreeData::DataPartPtr MergeTreePartsMover::clonePart(const MergeTreeMoveEnt if (moves_blocker.isCancelled()) throw Exception("Cancelled moving parts.", ErrorCodes::ABORTED); - LOG_TRACE(log, "Cloning part " << moving_part.part->name); + LOG_TRACE_FORMATTED(log, "Cloning part {}", moving_part.part->name); moving_part.part->makeCloneOnDiskDetached(moving_part.reserved_space); auto single_disk_volume = std::make_shared("volume_" + moving_part.part->name, moving_part.reserved_space->getDisk()); diff --git a/src/Storages/MergeTree/MergeTreeReadPool.cpp b/src/Storages/MergeTree/MergeTreeReadPool.cpp index 8efa7bd31fe..8c8f240203b 100644 --- a/src/Storages/MergeTree/MergeTreeReadPool.cpp +++ b/src/Storages/MergeTree/MergeTreeReadPool.cpp @@ -178,7 +178,7 @@ void MergeTreeReadPool::profileFeedback(const ReadBufferFromFileBase::ProfileInf --backoff_state.current_threads; ProfileEvents::increment(ProfileEvents::ReadBackoff); - LOG_DEBUG(log, "Will lower number of threads to " << backoff_state.current_threads); + LOG_DEBUG_FORMATTED(log, "Will lower number of threads to {}", backoff_state.current_threads); } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp index 5f405189fd3..4bb7d4cff52 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp @@ -184,7 +184,7 @@ void ReplicatedMergeTreePartCheckThread::searchForMissingPart(const String & par CheckResult ReplicatedMergeTreePartCheckThread::checkPart(const String & part_name) { - LOG_WARNING(log, "Checking part " << part_name); + LOG_WARNING_FORMATTED(log, "Checking part {}", part_name); ProfileEvents::increment(ProfileEvents::ReplicatedPartChecks); /// If the part is still in the PreCommitted -> Committed transition, it is not lost diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 78f981f39d9..acc10624df8 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -49,7 +49,7 @@ bool ReplicatedMergeTreeQueue::isVirtualPart(const MergeTreeData::DataPartPtr & bool ReplicatedMergeTreeQueue::load(zkutil::ZooKeeperPtr zookeeper) { auto queue_path = replica_path + "/queue"; - LOG_DEBUG(log, "Loading queue from " << queue_path); + LOG_DEBUG_FORMATTED(log, "Loading queue from {}", queue_path); bool updated = false; std::optional min_unprocessed_insert_time_changed; @@ -234,7 +234,7 @@ void ReplicatedMergeTreeQueue::updateStateOnQueueEntryRemoval( if (entry->type == LogEntry::ALTER_METADATA) { - LOG_TRACE(log, "Finishing metadata alter with version " << entry->alter_version); + LOG_TRACE_FORMATTED(log, "Finishing metadata alter with version {}", entry->alter_version); alter_sequence.finishMetadataAlter(entry->alter_version, state_lock); } } diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 54eccea5a83..840408e467f 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -587,7 +587,7 @@ void StorageDistributed::startup() if (inc > file_names_increment.value) file_names_increment.value.store(inc); } - LOG_DEBUG(log, "Auto-increment is " << file_names_increment.value); + LOG_DEBUG_FORMATTED(log, "Auto-increment is {}", file_names_increment.value); } @@ -816,7 +816,7 @@ void StorageDistributed::renameOnDisk(const String & new_path_to_table_data) auto new_path = path + new_path_to_table_data; Poco::File(path + relative_data_path).renameTo(new_path); - LOG_DEBUG(log, "Updating path to " << new_path); + LOG_DEBUG_FORMATTED(log, "Updating path to {}", new_path); std::lock_guard lock(cluster_nodes_mutex); for (auto & node : cluster_nodes_data) diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index fbe5b6b4d4d..678bf510954 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -384,14 +384,14 @@ Int64 StorageMergeTree::startMutation(const MutationCommands & commands, String auto insertion = current_mutations_by_id.emplace(mutation_file_name, std::move(entry)); current_mutations_by_version.emplace(version, insertion.first->second); - LOG_INFO(log, "Added mutation: " << mutation_file_name); + LOG_INFO_FORMATTED(log, "Added mutation: {}", mutation_file_name); merging_mutating_task_handle->wake(); return version; } void StorageMergeTree::waitForMutation(Int64 version, const String & file_name) { - LOG_INFO(log, "Waiting mutation: " << file_name); + LOG_INFO_FORMATTED(log, "Waiting mutation: {}", file_name); auto check = [version, this]() { return shutdown_called || isMutationDone(version); }; std::unique_lock lock(mutation_wait_mutex); mutation_wait_event.wait(lock, check); @@ -492,7 +492,7 @@ std::vector StorageMergeTree::getMutationsStatus() cons CancellationCode StorageMergeTree::killMutation(const String & mutation_id) { - LOG_TRACE(log, "Killing mutation " << mutation_id); + LOG_TRACE_FORMATTED(log, "Killing mutation {}", mutation_id); std::optional to_kill; { @@ -511,7 +511,7 @@ CancellationCode StorageMergeTree::killMutation(const String & mutation_id) global_context.getMergeList().cancelPartMutations({}, to_kill->block_number); to_kill->removeFile(); - LOG_TRACE(log, "Cancelled part mutations and removed mutation file " << mutation_id); + LOG_TRACE_FORMATTED(log, "Cancelled part mutations and removed mutation file {}", mutation_id); { std::lock_guard lock(mutation_wait_mutex); mutation_wait_event.notify_all(); @@ -896,7 +896,7 @@ void StorageMergeTree::clearOldMutations(bool truncate) for (auto & mutation : mutations_to_delete) { - LOG_TRACE(log, "Removing mutation: " << mutation.file_name); + LOG_TRACE_FORMATTED(log, "Removing mutation: {}", mutation.file_name); mutation.removeFile(); } } @@ -1050,7 +1050,7 @@ void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, cons /// If DETACH clone parts to detached/ directory for (const auto & part : parts_to_remove) { - LOG_INFO(log, "Detaching " << part->relative_path); + LOG_INFO_FORMATTED(log, "Detaching {}", part->relative_path); part->makeCloneInDetached(""); } } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 8bae12e2a19..b8319c45dca 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -412,7 +412,7 @@ void StorageReplicatedMergeTree::createTableIfNotExists() if (zookeeper->exists(zookeeper_path)) return; - LOG_DEBUG(log, "Creating table " << zookeeper_path); + LOG_DEBUG_FORMATTED(log, "Creating table {}", zookeeper_path); zookeeper->createAncestors(zookeeper_path); @@ -548,7 +548,7 @@ void StorageReplicatedMergeTree::createReplica() { auto zookeeper = getZooKeeper(); - LOG_DEBUG(log, "Creating replica " << replica_path); + LOG_DEBUG_FORMATTED(log, "Creating replica {}", replica_path); int32_t code; @@ -643,7 +643,7 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks) if (MergeTreePartInfo::tryParsePartName(part_name, &part_info, format_version)) return part_info.getBlocksCount(); - LOG_ERROR(log, "Unexpected part name: " << part_name); + LOG_ERROR_FORMATTED(log, "Unexpected part name: {}", part_name); return 0; }; @@ -701,7 +701,7 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks) for (size_t i = 0; i < parts_to_fetch.size(); ++i) { const String & part_name = parts_to_fetch[i]; - LOG_ERROR(log, "Removing locally missing part from ZooKeeper and queueing a fetch: " << part_name); + LOG_ERROR_FORMATTED(log, "Removing locally missing part from ZooKeeper and queueing a fetch: {}", part_name); Coordination::Requests ops; @@ -1483,7 +1483,7 @@ void StorageReplicatedMergeTree::executeDropRange(const LogEntry & entry) /// If DETACH clone parts to detached/ directory for (const auto & part : parts_to_remove) { - LOG_INFO(log, "Detaching " << part->relative_path); + LOG_INFO_FORMATTED(log, "Detaching {}", part->relative_path); part->makeCloneInDetached(""); } } @@ -1824,7 +1824,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coordination::Stat source_is_lost_stat, zkutil::ZooKeeperPtr & zookeeper) { - LOG_INFO(log, "Will mimic " << source_replica); + LOG_INFO_FORMATTED(log, "Will mimic {}", source_replica); String source_path = zookeeper_path + "/replicas/" + source_replica; @@ -3283,7 +3283,7 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer setTableStructure(std::move(columns_from_entry), metadata_diff); metadata_version = entry.alter_version; - LOG_INFO(log, "Applied changes to the metadata of the table. Current metadata version: " << metadata_version); + LOG_INFO_FORMATTED(log, "Applied changes to the metadata of the table. Current metadata version: {}", metadata_version); } /// This transaction may not happen, but it's OK, because on the next retry we will eventually create/update this node @@ -3722,7 +3722,7 @@ void StorageReplicatedMergeTree::drop() if (zookeeper->expired()) throw Exception("Table was not dropped because ZooKeeper session has expired.", ErrorCodes::TABLE_WAS_NOT_DROPPED); - LOG_INFO(log, "Removing replica " << replica_path); + LOG_INFO_FORMATTED(log, "Removing replica {}", replica_path); replica_is_active_node = nullptr; /// It may left some garbage if replica_path subtree are concurently modified zookeeper->tryRemoveRecursive(replica_path); @@ -3833,7 +3833,7 @@ StorageReplicatedMergeTree::allocateBlockNumber( Strings StorageReplicatedMergeTree::waitForAllReplicasToProcessLogEntry(const ReplicatedMergeTreeLogEntryData & entry, bool wait_for_non_active) { - LOG_DEBUG(log, "Waiting for all replicas to process " << entry.znode_name); + LOG_DEBUG_FORMATTED(log, "Waiting for all replicas to process {}", entry.znode_name); auto zookeeper = getZooKeeper(); Strings replicas = zookeeper->getChildren(zookeeper_path + "/replicas"); @@ -3851,7 +3851,7 @@ Strings StorageReplicatedMergeTree::waitForAllReplicasToProcessLogEntry(const Re } } - LOG_DEBUG(log, "Finished waiting for all replicas to process " << entry.znode_name); + LOG_DEBUG_FORMATTED(log, "Finished waiting for all replicas to process {}", entry.znode_name); return unwaited; } @@ -4397,7 +4397,7 @@ void StorageReplicatedMergeTree::fetchPartition(const ASTPtr & partition, const } } - LOG_INFO(log, "Parts to fetch: " << parts_to_fetch.size()); + LOG_INFO_FORMATTED(log, "Parts to fetch: {}", parts_to_fetch.size()); missing_parts.clear(); for (const String & part : parts_to_fetch) @@ -4514,7 +4514,7 @@ void StorageReplicatedMergeTree::mutate(const MutationCommands & commands, const const String & path_created = dynamic_cast(responses[1].get())->path_created; entry.znode_name = path_created.substr(path_created.find_last_of('/') + 1); - LOG_TRACE(log, "Created mutation with ID " << entry.znode_name); + LOG_TRACE_FORMATTED(log, "Created mutation with ID {}", entry.znode_name); break; } else if (rc == Coordination::ZBADVERSION) @@ -4556,7 +4556,7 @@ CancellationCode StorageReplicatedMergeTree::killMutation(const String & mutatio zkutil::ZooKeeperPtr zookeeper = getZooKeeper(); - LOG_TRACE(log, "Killing mutation " << mutation_id); + LOG_TRACE_FORMATTED(log, "Killing mutation {}", mutation_id); auto mutation_entry = queue.removeMutation(zookeeper, mutation_id); if (!mutation_entry) @@ -4639,7 +4639,7 @@ void StorageReplicatedMergeTree::clearOldPartsAndRemoveFromZK() } catch (...) { - LOG_ERROR(log, "There is a problem with deleting parts from ZooKeeper: " << getCurrentExceptionMessage(true)); + LOG_ERROR_FORMATTED(log, "There is a problem with deleting parts from ZooKeeper: {}", getCurrentExceptionMessage(true)); } /// Part names that were reliably deleted from ZooKeeper should be deleted from filesystem @@ -5386,7 +5386,7 @@ bool StorageReplicatedMergeTree::dropPartsInPartition( queue.disableMergesInBlockRange(drop_range_fake_part_name); } - LOG_DEBUG(log, "Disabled merges covered by range " << drop_range_fake_part_name); + LOG_DEBUG_FORMATTED(log, "Disabled merges covered by range {}", drop_range_fake_part_name); /// Finally, having achieved the necessary invariants, you can put an entry in the log. entry.type = LogEntry::DROP_RANGE; From 979e5d77c19eca1f1155755ed4835328a4a56284 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 19:53:51 +0300 Subject: [PATCH 621/738] Fix build --- base/common/logger_useful.h | 1 + 1 file changed, 1 insertion(+) diff --git a/base/common/logger_useful.h b/base/common/logger_useful.h index 08bfb8cdee5..526a1a6ce44 100644 --- a/base/common/logger_useful.h +++ b/base/common/logger_useful.h @@ -4,6 +4,7 @@ #include #include +#include #include #include #include From e391b77d81b62dc0b187f27c487131068a62133e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 19:56:05 +0300 Subject: [PATCH 622/738] find {base,src,programs} -name '*.h' -or -name '*.cpp' | xargs grep -l -P 'LOG_\w+\([^,]+, "[^"]+" << [^<]+ << "[^"]+"\);' | xargs sed -i -r -e 's/(LOG_\w+)\(([^,]+), "([^"]+)" << ([^<]+) << "([^"]+)"\);/\1_FORMATTED(\2, "\3{}\5", \4);/' --- base/daemon/BaseDaemon.cpp | 2 +- programs/copier/ClusterCopier.cpp | 20 +++---- programs/odbc-bridge/ColumnInfoHandler.cpp | 8 +-- programs/odbc-bridge/MainHandler.cpp | 2 +- programs/server/InterserverIOHTTPHandler.cpp | 2 +- programs/server/MySQLHandler.cpp | 4 +- programs/server/Server.cpp | 4 +- src/Common/Config/ConfigProcessor.cpp | 2 +- src/Common/Config/ConfigReloader.cpp | 2 +- src/Common/PipeFDs.cpp | 2 +- src/Common/ShellCommand.cpp | 2 +- src/Common/StatusFile.cpp | 2 +- src/Core/BackgroundSchedulePool.cpp | 4 +- src/Core/SettingsCollection.cpp | 2 +- .../MergeSortingBlockInputStream.cpp | 2 +- src/Databases/DatabaseLazy.cpp | 10 ++-- src/Databases/DatabaseOnDisk.cpp | 4 +- src/Disks/StoragePolicy.cpp | 2 +- src/Interpreters/Aggregator.cpp | 4 +- src/Interpreters/DDLWorker.cpp | 14 ++--- src/Interpreters/ExternalLoader.cpp | 10 ++-- src/Interpreters/ProcessList.cpp | 2 +- src/Processors/Formats/IRowInputFormat.cpp | 2 +- .../Transforms/MergeSortingTransform.cpp | 2 +- src/Storages/Distributed/DirectoryMonitor.cpp | 8 +-- src/Storages/Kafka/StorageKafka.cpp | 4 +- src/Storages/MergeTree/MergeTreeData.cpp | 10 ++-- .../MergeTree/MergeTreeDataMergerMutator.cpp | 2 +- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 2 +- .../ReplicatedMergeTreeBlockOutputStream.cpp | 4 +- .../ReplicatedMergeTreeCleanupThread.cpp | 2 +- .../ReplicatedMergeTreePartCheckThread.cpp | 10 ++-- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 10 ++-- .../ReplicatedMergeTreeRestartingThread.cpp | 2 +- src/Storages/StorageBuffer.cpp | 2 +- src/Storages/StorageDistributed.cpp | 2 +- src/Storages/StorageMergeTree.cpp | 4 +- src/Storages/StorageReplicatedMergeTree.cpp | 60 +++++++++---------- 38 files changed, 116 insertions(+), 116 deletions(-) diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index 692c1809139..bb58c3633d8 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -850,7 +850,7 @@ void BaseDaemon::handleSignal(int signal_id) void BaseDaemon::onInterruptSignals(int signal_id) { is_cancelled = true; - LOG_INFO(&logger(), "Received termination signal (" << strsignal(signal_id) << ")"); + LOG_INFO_FORMATTED(&logger(), "Received termination signal ({})", strsignal(signal_id)); if (sigint_signals_counter >= 2) { diff --git a/programs/copier/ClusterCopier.cpp b/programs/copier/ClusterCopier.cpp index d9a75ec080d..9a7716a9a57 100644 --- a/programs/copier/ClusterCopier.cpp +++ b/programs/copier/ClusterCopier.cpp @@ -47,7 +47,7 @@ void ClusterCopier::init() task_table.initShards(task_cluster->random_engine); } - LOG_DEBUG(log, "Will process " << task_cluster->table_tasks.size() << " table tasks"); + LOG_DEBUG_FORMATTED(log, "Will process {} table tasks", task_cluster->table_tasks.size()); /// Do not initialize tables, will make deferred initialization in process() @@ -181,7 +181,7 @@ void ClusterCopier::discoverTablePartitions(const ConnectionTimeouts & timeouts, for (const TaskShardPtr & task_shard : task_table.all_shards) thread_pool.scheduleOrThrowOnError([this, timeouts, task_shard]() { discoverShardPartitions(timeouts, task_shard); }); - LOG_DEBUG(log, "Waiting for " << thread_pool.active() << " setup jobs"); + LOG_DEBUG_FORMATTED(log, "Waiting for {} setup jobs", thread_pool.active()); thread_pool.wait(); } } @@ -484,7 +484,7 @@ bool ClusterCopier::checkPartitionPieceIsDone(const TaskTable & task_table, cons if (!is_clean) { - LOG_INFO(log, "Partition " << partition_name << " become dirty"); + LOG_INFO_FORMATTED(log, "Partition {} become dirty", partition_name); return false; } @@ -511,7 +511,7 @@ bool ClusterCopier::checkPartitionPieceIsDone(const TaskTable & task_table, cons { if (zxid1[shard_num] != zxid2[shard_num]) { - LOG_INFO(log, "The task " << piece_status_paths[shard_num] << " is being modified now. Partition piece will be rechecked"); + LOG_INFO_FORMATTED(log, "The task {} is being modified now. Partition piece will be rechecked", piece_status_paths[shard_num]); return false; } } @@ -530,7 +530,7 @@ TaskStatus ClusterCopier::tryMoveAllPiecesToDestinationTable(const TaskTable & t inject_fault = value < move_fault_probability; } - LOG_DEBUG(log, "Try to move " << partition_name << " to destionation table"); + LOG_DEBUG_FORMATTED(log, "Try to move {} to destionation table", partition_name); auto zookeeper = context.getZooKeeper(); @@ -794,7 +794,7 @@ bool ClusterCopier::tryDropPartitionPiece( { if (e.code == Coordination::ZNODEEXISTS) { - LOG_DEBUG(log, "Partition " << task_partition.name << " is being filled now by somebody, sleep"); + LOG_DEBUG_FORMATTED(log, "Partition {} is being filled now by somebody, sleep", task_partition.name); return false; } @@ -841,7 +841,7 @@ bool ClusterCopier::tryDropPartitionPiece( PoolMode::GET_MANY, ClusterExecutionMode::ON_EACH_NODE); - LOG_INFO(log, "DROP PARTITION was successfully executed on " << num_shards << " nodes of a cluster."); + LOG_INFO_FORMATTED(log, "DROP PARTITION was successfully executed on {} nodes of a cluster.", num_shards); /// Update the locking node if (!my_clock.is_stale()) @@ -889,7 +889,7 @@ bool ClusterCopier::tryProcessTable(const ConnectionTimeouts & timeouts, TaskTab ++cluster_partition.total_tries; - LOG_DEBUG(log, "Processing partition " << partition_name << " for the whole cluster"); + LOG_DEBUG_FORMATTED(log, "Processing partition {} for the whole cluster", partition_name); /// Process each source shard having current partition and copy current partition /// NOTE: shards are sorted by "distance" to current host @@ -1591,7 +1591,7 @@ void ClusterCopier::dropHelpingTables(const TaskTable & task_table) PoolMode::GET_MANY, ClusterExecutionMode::ON_EACH_NODE); - LOG_DEBUG(log, "DROP TABLE query was successfully executed on " << toString(num_nodes) << " nodes."); + LOG_DEBUG_FORMATTED(log, "DROP TABLE query was successfully executed on {} nodes.", toString(num_nodes)); } } @@ -1618,7 +1618,7 @@ void ClusterCopier::dropParticularPartitionPieceFromAllHelpingTables(const TaskT PoolMode::GET_MANY, ClusterExecutionMode::ON_EACH_NODE); - LOG_DEBUG(log, "DROP PARTITION query was successfully executed on " << toString(num_nodes) << " nodes."); + LOG_DEBUG_FORMATTED(log, "DROP PARTITION query was successfully executed on {} nodes.", toString(num_nodes)); } LOG_DEBUG_FORMATTED(log, "All helping tables dropped partition {}", partition_name); } diff --git a/programs/odbc-bridge/ColumnInfoHandler.cpp b/programs/odbc-bridge/ColumnInfoHandler.cpp index e62aad48b90..d3badc23f38 100644 --- a/programs/odbc-bridge/ColumnInfoHandler.cpp +++ b/programs/odbc-bridge/ColumnInfoHandler.cpp @@ -89,11 +89,11 @@ void ODBCColumnsInfoHandler::handleRequest(Poco::Net::HTTPServerRequest & reques if (params.has("schema")) { schema_name = params.get("schema"); - LOG_TRACE(log, "Will fetch info for table '" << schema_name + "." + table_name << "'"); + LOG_TRACE_FORMATTED(log, "Will fetch info for table '{}'", schema_name + "." + table_name); } else - LOG_TRACE(log, "Will fetch info for table '" << table_name << "'"); - LOG_TRACE(log, "Got connection str '" << connection_string << "'"); + LOG_TRACE_FORMATTED(log, "Will fetch info for table '{}'", table_name); + LOG_TRACE_FORMATTED(log, "Got connection str '{}'", connection_string); try { @@ -124,7 +124,7 @@ void ODBCColumnsInfoHandler::handleRequest(Poco::Net::HTTPServerRequest & reques select->format(settings); std::string query = ss.str(); - LOG_TRACE(log, "Inferring structure with query '" << query << "'"); + LOG_TRACE_FORMATTED(log, "Inferring structure with query '{}'", query); if (POCO_SQL_ODBC_CLASS::Utility::isError(POCO_SQL_ODBC_CLASS::SQLPrepare(hstmt, reinterpret_cast(query.data()), query.size()))) throw POCO_SQL_ODBC_CLASS::DescriptorException(session.dbc()); diff --git a/programs/odbc-bridge/MainHandler.cpp b/programs/odbc-bridge/MainHandler.cpp index 05d1cb5d162..e3ee0071fa1 100644 --- a/programs/odbc-bridge/MainHandler.cpp +++ b/programs/odbc-bridge/MainHandler.cpp @@ -132,7 +132,7 @@ void ODBCHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne std::string format = params.get("format", "RowBinary"); std::string connection_string = params.get("connection_string"); - LOG_TRACE(log, "Connection string: '" << connection_string << "'"); + LOG_TRACE_FORMATTED(log, "Connection string: '{}'", connection_string); WriteBufferFromHTTPServerResponse out(request, response, keep_alive_timeout); diff --git a/programs/server/InterserverIOHTTPHandler.cpp b/programs/server/InterserverIOHTTPHandler.cpp index bedd38c28ef..d86876800b8 100644 --- a/programs/server/InterserverIOHTTPHandler.cpp +++ b/programs/server/InterserverIOHTTPHandler.cpp @@ -103,7 +103,7 @@ void InterserverIOHTTPHandler::handleRequest(Poco::Net::HTTPServerRequest & requ response.setStatusAndReason(Poco::Net::HTTPServerResponse::HTTP_UNAUTHORIZED); if (!response.sent()) writeString(message, *used_output.out); - LOG_WARNING(log, "Query processing failed request: '" << request.getURI() << "' authentication failed"); + LOG_WARNING_FORMATTED(log, "Query processing failed request: '{}' authentication failed", request.getURI()); } } catch (Exception & e) diff --git a/programs/server/MySQLHandler.cpp b/programs/server/MySQLHandler.cpp index f070c850509..c1de7bfcb58 100644 --- a/programs/server/MySQLHandler.cpp +++ b/programs/server/MySQLHandler.cpp @@ -234,11 +234,11 @@ void MySQLHandler::authenticate(const String & user_name, const String & auth_pl } catch (const Exception & exc) { - LOG_ERROR(log, "Authentication for user " << user_name << " failed."); + LOG_ERROR_FORMATTED(log, "Authentication for user {} failed.", user_name); packet_sender->sendPacket(ERR_Packet(exc.code(), "00000", exc.message()), true); throw; } - LOG_INFO(log, "Authentication for user " << user_name << " succeeded."); + LOG_INFO_FORMATTED(log, "Authentication for user {} succeeded.", user_name); } void MySQLHandler::comInitDB(ReadBuffer & payload) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index a097b12f2d8..eb14cdad6b7 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -91,7 +91,7 @@ namespace void setupTmpPath(Logger * log, const std::string & path) { - LOG_DEBUG(log, "Setting up " << path << " to store temporary data in it"); + LOG_DEBUG_FORMATTED(log, "Setting up {} to store temporary data in it", path); Poco::File(path).createDirectories(); @@ -372,7 +372,7 @@ int Server::main(const std::vector & /*args*/) /// Initialize DateLUT early, to not interfere with running time of first query. LOG_DEBUG_FORMATTED(log, "Initializing DateLUT."); DateLUT::instance(); - LOG_TRACE(log, "Initialized DateLUT with time zone '" << DateLUT::instance().getTimeZone() << "'."); + LOG_TRACE_FORMATTED(log, "Initialized DateLUT with time zone '{}'.", DateLUT::instance().getTimeZone()); /// Storage with temporary data for processing of heavy queries. diff --git a/src/Common/Config/ConfigProcessor.cpp b/src/Common/Config/ConfigProcessor.cpp index 1e765a0b8a0..d5f92f05886 100644 --- a/src/Common/Config/ConfigProcessor.cpp +++ b/src/Common/Config/ConfigProcessor.cpp @@ -619,7 +619,7 @@ void ConfigProcessor::savePreprocessedConfig(const LoadedConfig & loaded_config, Poco::File(preprocessed_path_parent).createDirectories(); } DOMWriter().writeNode(preprocessed_path, loaded_config.preprocessed_xml); - LOG_DEBUG(log, "Saved preprocessed configuration to '" << preprocessed_path << "'."); + LOG_DEBUG_FORMATTED(log, "Saved preprocessed configuration to '{}'.", preprocessed_path); } catch (Poco::Exception & e) { diff --git a/src/Common/Config/ConfigReloader.cpp b/src/Common/Config/ConfigReloader.cpp index 0e3cf9d1dba..4a9cbd213ff 100644 --- a/src/Common/Config/ConfigReloader.cpp +++ b/src/Common/Config/ConfigReloader.cpp @@ -87,7 +87,7 @@ void ConfigReloader::reloadIfNewer(bool force, bool throw_on_error, bool fallbac ConfigProcessor::LoadedConfig loaded_config; try { - LOG_DEBUG(log, "Loading config '" << path << "'"); + LOG_DEBUG_FORMATTED(log, "Loading config '{}'", path); loaded_config = config_processor.loadConfig(/* allow_zk_includes = */ true); if (loaded_config.has_zk_includes) diff --git a/src/Common/PipeFDs.cpp b/src/Common/PipeFDs.cpp index bb1dad4e2d3..b16feaf51ef 100644 --- a/src/Common/PipeFDs.cpp +++ b/src/Common/PipeFDs.cpp @@ -90,7 +90,7 @@ void LazyPipeFDs::tryIncreaseSize(int desired_size) { if (errno == EINVAL) { - LOG_INFO(log, "Cannot get pipe capacity, " << errnoToString(ErrorCodes::CANNOT_FCNTL) << ". Very old Linux kernels have no support for this fcntl."); + LOG_INFO_FORMATTED(log, "Cannot get pipe capacity, {}. Very old Linux kernels have no support for this fcntl.", errnoToString(ErrorCodes::CANNOT_FCNTL)); /// It will work nevertheless. } else diff --git a/src/Common/ShellCommand.cpp b/src/Common/ShellCommand.cpp index 98a01e18511..f19ea031c6a 100644 --- a/src/Common/ShellCommand.cpp +++ b/src/Common/ShellCommand.cpp @@ -50,7 +50,7 @@ ShellCommand::~ShellCommand() { if (terminate_in_destructor) { - LOG_TRACE(getLogger(), "Will kill shell command pid " << pid << " with SIGTERM"); + LOG_TRACE_FORMATTED(getLogger(), "Will kill shell command pid {} with SIGTERM", pid); int retcode = kill(pid, SIGTERM); if (retcode != 0) LOG_WARNING(getLogger(), "Cannot kill shell command pid " << pid << " errno '" << errnoToString(retcode) << "'"); diff --git a/src/Common/StatusFile.cpp b/src/Common/StatusFile.cpp index 7fa72ef15ab..7609e1f1c73 100644 --- a/src/Common/StatusFile.cpp +++ b/src/Common/StatusFile.cpp @@ -45,7 +45,7 @@ StatusFile::StatusFile(const std::string & path_) if (!contents.empty()) LOG_INFO(&Logger::get("StatusFile"), "Status file " << path << " already exists - unclean restart. Contents:\n" << contents); else - LOG_INFO(&Logger::get("StatusFile"), "Status file " << path << " already exists and is empty - probably unclean hardware restart."); + LOG_INFO_FORMATTED(&Logger::get("StatusFile"), "Status file {} already exists and is empty - probably unclean hardware restart.", path); } fd = ::open(path.c_str(), O_WRONLY | O_CREAT | O_CLOEXEC, 0666); diff --git a/src/Core/BackgroundSchedulePool.cpp b/src/Core/BackgroundSchedulePool.cpp index 3974cf0071d..41fec3c0f1f 100644 --- a/src/Core/BackgroundSchedulePool.cpp +++ b/src/Core/BackgroundSchedulePool.cpp @@ -111,7 +111,7 @@ void BackgroundSchedulePoolTaskInfo::execute() static const int32_t slow_execution_threshold_ms = 200; if (milliseconds >= slow_execution_threshold_ms) - LOG_TRACE(&Logger::get(log_name), "Execution took " << milliseconds << " ms."); + LOG_TRACE_FORMATTED(&Logger::get(log_name), "Execution took {} ms.", milliseconds); { std::lock_guard lock_schedule(schedule_mutex); @@ -156,7 +156,7 @@ BackgroundSchedulePool::BackgroundSchedulePool(size_t size_, CurrentMetrics::Met , memory_metric(memory_metric_) , thread_name(thread_name_) { - LOG_INFO(&Logger::get("BackgroundSchedulePool/" + thread_name), "Create BackgroundSchedulePool with " << size << " threads"); + LOG_INFO_FORMATTED(&Logger::get("BackgroundSchedulePool/" + thread_name), "Create BackgroundSchedulePool with {} threads", size); threads.resize(size); for (auto & thread : threads) diff --git a/src/Core/SettingsCollection.cpp b/src/Core/SettingsCollection.cpp index 364b90e8ce8..6753ef9ab0a 100644 --- a/src/Core/SettingsCollection.cpp +++ b/src/Core/SettingsCollection.cpp @@ -599,7 +599,7 @@ namespace details void SettingsCollectionUtils::warningNameNotFound(const StringRef & name) { static auto * log = &Logger::get("Settings"); - LOG_WARNING(log, "Unknown setting " << name << ", skipping"); + LOG_WARNING_FORMATTED(log, "Unknown setting {}, skipping", name); } void SettingsCollectionUtils::throwNameNotFound(const StringRef & name) diff --git a/src/DataStreams/MergeSortingBlockInputStream.cpp b/src/DataStreams/MergeSortingBlockInputStream.cpp index 5b26f4822b9..5e484fa4803 100644 --- a/src/DataStreams/MergeSortingBlockInputStream.cpp +++ b/src/DataStreams/MergeSortingBlockInputStream.cpp @@ -118,7 +118,7 @@ Block MergeSortingBlockInputStream::readImpl() /// If there was temporary files. ProfileEvents::increment(ProfileEvents::ExternalSortMerge); - LOG_INFO(log, "There are " << temporary_files.size() << " temporary sorted parts to merge."); + LOG_INFO_FORMATTED(log, "There are {} temporary sorted parts to merge.", temporary_files.size()); /// Create sorted streams to merge. for (const auto & file : temporary_files) diff --git a/src/Databases/DatabaseLazy.cpp b/src/Databases/DatabaseLazy.cpp index d287ece0fde..e740a4f2e36 100644 --- a/src/Databases/DatabaseLazy.cpp +++ b/src/Databases/DatabaseLazy.cpp @@ -152,7 +152,7 @@ bool DatabaseLazy::empty() const void DatabaseLazy::attachTable(const String & table_name, const StoragePtr & table, const String &) { - LOG_DEBUG(log, "Attach table " << backQuote(table_name) << "."); + LOG_DEBUG_FORMATTED(log, "Attach table {}.", backQuote(table_name)); std::lock_guard lock(mutex); time_t current_time = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); @@ -169,7 +169,7 @@ StoragePtr DatabaseLazy::detachTable(const String & table_name) { StoragePtr res; { - LOG_DEBUG(log, "Detach table " << backQuote(table_name) << "."); + LOG_DEBUG_FORMATTED(log, "Detach table {}.", backQuote(table_name)); std::lock_guard lock(mutex); auto it = tables_cache.find(table_name); if (it == tables_cache.end()) @@ -216,7 +216,7 @@ StoragePtr DatabaseLazy::loadTable(const String & table_name) const { SCOPE_EXIT({ clearExpiredTables(); }); - LOG_DEBUG(log, "Load table " << backQuote(table_name) << " to cache."); + LOG_DEBUG_FORMATTED(log, "Load table {} to cache.", backQuote(table_name)); const String table_metadata_path = getMetadataPath() + "/" + escapeForFileName(table_name) + ".sql"; @@ -277,14 +277,14 @@ void DatabaseLazy::clearExpiredTables() const if (!it->second.table || it->second.table.unique()) { - LOG_DEBUG(log, "Drop table " << backQuote(it->first) << " from cache."); + LOG_DEBUG_FORMATTED(log, "Drop table {} from cache.", backQuote(it->first)); it->second.table.reset(); expired_tables.erase(it->second.expiration_iterator); it->second.expiration_iterator = cache_expiration_queue.end(); } else { - LOG_DEBUG(log, "Table " << backQuote(it->first) << " is busy."); + LOG_DEBUG_FORMATTED(log, "Table {} is busy.", backQuote(it->first)); busy_tables.splice(busy_tables.end(), expired_tables, it->second.expiration_iterator); } } diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index e5c8c5cf287..6f59fe7c595 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -376,7 +376,7 @@ void DatabaseOnDisk::iterateMetadataFiles(const Context & context, const Iterati if (Poco::File(context.getPath() + getDataPath() + '/' + object_name).exists()) { Poco::File(getMetadataPath() + file_name).renameTo(getMetadataPath() + object_name + ".sql"); - LOG_WARNING(log, "Object " << backQuote(object_name) << " was not dropped previously and will be restored"); + LOG_WARNING_FORMATTED(log, "Object {} was not dropped previously and will be restored", backQuote(object_name)); process_metadata_file(object_name + ".sql"); } else @@ -442,7 +442,7 @@ ASTPtr DatabaseOnDisk::parseQueryFromMetadata(Poco::Logger * loger, const Contex */ if (remove_empty && query.empty()) { - LOG_ERROR(loger, "File " << metadata_file_path << " is empty. Removing."); + LOG_ERROR_FORMATTED(loger, "File {} is empty. Removing.", metadata_file_path); Poco::File(metadata_file_path).remove(); return nullptr; } diff --git a/src/Disks/StoragePolicy.cpp b/src/Disks/StoragePolicy.cpp index e1ae718d9b9..c11b5b79581 100644 --- a/src/Disks/StoragePolicy.cpp +++ b/src/Disks/StoragePolicy.cpp @@ -245,7 +245,7 @@ StoragePolicySelector::StoragePolicySelector( "StoragePolicy name can contain only alphanumeric and '_' (" + name + ")", ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG); policies.emplace(name, std::make_shared(name, config, config_prefix + "." + name, disks)); - LOG_INFO(&Logger::get("StoragePolicySelector"), "Storage policy " << backQuote(name) << " loaded"); + LOG_INFO_FORMATTED(&Logger::get("StoragePolicySelector"), "Storage policy {} loaded", backQuote(name)); } constexpr auto default_storage_policy_name = "default"; diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 5c4455f3281..51fdf57c50c 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -722,7 +722,7 @@ void Aggregator::writeToTemporaryFile(AggregatedDataVariants & data_variants, co CompressedWriteBuffer compressed_buf(file_buf); NativeBlockOutputStream block_out(compressed_buf, ClickHouseRevision::get(), getHeader(false)); - LOG_DEBUG(log, "Writing part of aggregation data into temporary file " << path << "."); + LOG_DEBUG_FORMATTED(log, "Writing part of aggregation data into temporary file {}.", path); ProfileEvents::increment(ProfileEvents::ExternalAggregationWritePart); /// Flush only two-level data and possibly overflow data. @@ -2099,7 +2099,7 @@ Block Aggregator::mergeBlocks(BlocksList & blocks, bool final) auto bucket_num = blocks.front().info.bucket_num; bool is_overflows = blocks.front().info.is_overflows; - LOG_TRACE(log, "Merging partially aggregated blocks (bucket = " << bucket_num << ")."); + LOG_TRACE_FORMATTED(log, "Merging partially aggregated blocks (bucket = {}).", bucket_num); Stopwatch watch; /** If possible, change 'method' to some_hash64. Otherwise, leave as is. diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index 6fa2b171981..0924ae433f8 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -386,11 +386,11 @@ void DDLWorker::processTasks() { if (current_task->entry_name == entry_name) { - LOG_INFO(log, "Trying to process task " << entry_name << " again"); + LOG_INFO_FORMATTED(log, "Trying to process task {} again", entry_name); } else { - LOG_INFO(log, "Task " << current_task->entry_name << " was deleted from ZooKeeper before current host committed it"); + LOG_INFO_FORMATTED(log, "Task {} was deleted from ZooKeeper before current host committed it", current_task->entry_name); current_task = nullptr; } } @@ -863,7 +863,7 @@ void DDLWorker::cleanupQueue(Int64 current_time_seconds, const ZooKeeperPtr & zo /// Skip if there are active nodes (it is weak guard) if (zookeeper->exists(node_path + "/active", &stat) && stat.numChildren > 0) { - LOG_INFO(log, "Task " << node_name << " should be deleted, but there are active workers. Skipping it."); + LOG_INFO_FORMATTED(log, "Task {} should be deleted, but there are active workers. Skipping it.", node_name); continue; } @@ -872,14 +872,14 @@ void DDLWorker::cleanupQueue(Int64 current_time_seconds, const ZooKeeperPtr & zo auto lock = createSimpleZooKeeperLock(zookeeper, node_path, "lock", host_fqdn_id); if (!lock->tryLock()) { - LOG_INFO(log, "Task " << node_name << " should be deleted, but it is locked. Skipping it."); + LOG_INFO_FORMATTED(log, "Task {} should be deleted, but it is locked. Skipping it.", node_name); continue; } if (node_lifetime_is_expired) - LOG_INFO(log, "Lifetime of task " << node_name << " is expired, deleting it"); + LOG_INFO_FORMATTED(log, "Lifetime of task {} is expired, deleting it", node_name); else if (node_is_outside_max_window) - LOG_INFO(log, "Task " << node_name << " is outdated, deleting it"); + LOG_INFO_FORMATTED(log, "Task {} is outdated, deleting it", node_name); /// Deleting { @@ -1027,7 +1027,7 @@ void DDLWorker::runMainThread() } else { - LOG_ERROR(log, "Unexpected ZooKeeper error: " << getCurrentExceptionMessage(true) << ". Terminating."); + LOG_ERROR_FORMATTED(log, "Unexpected ZooKeeper error: {}. Terminating.", getCurrentExceptionMessage(true)); return; } } diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index acde799de6a..eff80a37e86 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -271,7 +271,7 @@ private: return false; } - LOG_TRACE(log, "Loading config file '" << path << "'."); + LOG_TRACE_FORMATTED(log, "Loading config file '{}'.", path); file_info.file_contents = repository.load(path); auto & file_contents = *file_info.file_contents; @@ -475,7 +475,7 @@ public: { const auto & info = it->second; if (info.loaded() || info.isLoading()) - LOG_TRACE(log, "Unloading '" << name << "' because its configuration has been removed or detached"); + LOG_TRACE_FORMATTED(log, "Unloading '{}' because its configuration has been removed or detached", name); infos.erase(it); } } @@ -931,7 +931,7 @@ private: /// Does the loading, possibly in the separate thread. void doLoading(const String & name, size_t loading_id, bool forced_to_reload, size_t min_id_to_finish_loading_dependencies_, bool async) { - LOG_TRACE(log, "Start loading object '" << name << "'"); + LOG_TRACE_FORMATTED(log, "Start loading object '{}'", name); try { /// Prepare for loading. @@ -1045,12 +1045,12 @@ private: /// This is necessary because the object could be removed or load with another config while the `mutex` was unlocked. if (!info) { - LOG_TRACE(log, "Next update time for '" << name << "' will not be set because this object was not found."); + LOG_TRACE_FORMATTED(log, "Next update time for '{}' will not be set because this object was not found.", name); return; } if (!info->isLoading()) { - LOG_TRACE(log, "Next update time for '" << name << "' will not be set because this object is not currently loading."); + LOG_TRACE_FORMATTED(log, "Next update time for '{}' will not be set because this object is not currently loading.", name); return; } if (info->loading_id != loading_id) diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index 3d576d79b6c..b22bdfa9666 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -84,7 +84,7 @@ ProcessList::EntryPtr ProcessList::insert(const String & query_, const IAST * as if (!is_unlimited_query && max_size && processes.size() >= max_size) { if (queue_max_wait_ms) - LOG_WARNING(&Logger::get("ProcessList"), "Too many simultaneous queries, will wait " << queue_max_wait_ms << " ms."); + LOG_WARNING_FORMATTED(&Logger::get("ProcessList"), "Too many simultaneous queries, will wait {} ms.", queue_max_wait_ms); if (!queue_max_wait_ms || !have_space.wait_for(lock, std::chrono::milliseconds(queue_max_wait_ms), [&]{ return processes.size() < max_size; })) throw Exception("Too many simultaneous queries. Maximum: " + toString(max_size), ErrorCodes::TOO_MANY_SIMULTANEOUS_QUERIES); } diff --git a/src/Processors/Formats/IRowInputFormat.cpp b/src/Processors/Formats/IRowInputFormat.cpp index dbd2e7aef83..e717ab6f717 100644 --- a/src/Processors/Formats/IRowInputFormat.cpp +++ b/src/Processors/Formats/IRowInputFormat.cpp @@ -150,7 +150,7 @@ Chunk IRowInputFormat::generate() if (num_errors && (params.allow_errors_num > 0 || params.allow_errors_ratio > 0)) { Logger * log = &Logger::get("IRowInputFormat"); - LOG_TRACE(log, "Skipped " << num_errors << " rows with errors while reading the input stream"); + LOG_TRACE_FORMATTED(log, "Skipped {} rows with errors while reading the input stream", num_errors); } readSuffix(); diff --git a/src/Processors/Transforms/MergeSortingTransform.cpp b/src/Processors/Transforms/MergeSortingTransform.cpp index e2a84d3f1c2..71c75d5386e 100644 --- a/src/Processors/Transforms/MergeSortingTransform.cpp +++ b/src/Processors/Transforms/MergeSortingTransform.cpp @@ -229,7 +229,7 @@ void MergeSortingTransform::generate() else { ProfileEvents::increment(ProfileEvents::ExternalSortMerge); - LOG_INFO(log, "There are " << temporary_files.size() << " temporary sorted parts to merge."); + LOG_INFO_FORMATTED(log, "There are {} temporary sorted parts to merge.", temporary_files.size()); if (!chunks.empty()) processors.emplace_back(std::make_shared( diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp index e777b2122b0..2795b98dcc2 100644 --- a/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/src/Storages/Distributed/DirectoryMonitor.cpp @@ -428,7 +428,7 @@ struct StorageDistributedDirectoryMonitor::Batch String tmp_file{parent.current_batch_file_path + ".tmp"}; if (Poco::File{tmp_file}.exists()) - LOG_ERROR(parent.log, "Temporary file " << backQuote(tmp_file) << " exists. Unclean shutdown?"); + LOG_ERROR_FORMATTED(parent.log, "Temporary file {} exists. Unclean shutdown?", backQuote(tmp_file)); { WriteBufferFromFile out{tmp_file, O_WRONLY | O_TRUNC | O_CREAT}; @@ -454,7 +454,7 @@ struct StorageDistributedDirectoryMonitor::Batch auto file_path = file_index_to_path.find(file_idx); if (file_path == file_index_to_path.end()) { - LOG_ERROR(parent.log, "Failed to send batch: file with index " << file_idx << " is absent"); + LOG_ERROR_FORMATTED(parent.log, "Failed to send batch: file with index {} is absent", file_idx); batch_broken = true; break; } @@ -488,14 +488,14 @@ struct StorageDistributedDirectoryMonitor::Batch if (!batch_broken) { - LOG_TRACE(parent.log, "Sent a batch of " << file_indices.size() << " files."); + LOG_TRACE_FORMATTED(parent.log, "Sent a batch of {} files.", file_indices.size()); for (UInt64 file_index : file_indices) Poco::File{file_index_to_path.at(file_index)}.remove(); } else { - LOG_ERROR(parent.log, "Marking a batch of " << file_indices.size() << " files as broken."); + LOG_ERROR_FORMATTED(parent.log, "Marking a batch of {} files as broken.", file_indices.size()); for (UInt64 file_idx : file_indices) { diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index 7fb4a9cf71d..ae09706ea5d 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -174,7 +174,7 @@ Pipes StorageKafka::read( pipes.emplace_back(std::make_shared(std::make_shared(*this, context, column_names, 1))); } - LOG_DEBUG(log, "Starting reading " << pipes.size() << " streams"); + LOG_DEBUG_FORMATTED(log, "Starting reading {} streams", pipes.size()); return pipes; } @@ -391,7 +391,7 @@ void StorageKafka::threadFunc() if (!checkDependencies(table_id)) break; - LOG_DEBUG(log, "Started streaming to " << dependencies_count << " attached views"); + LOG_DEBUG_FORMATTED(log, "Started streaming to {} attached views", dependencies_count); // Reschedule if not limited if (!streamToViews()) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 0d29f0808b2..0fe1681e819 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -1100,7 +1100,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) calculateColumnSizesImpl(); - LOG_DEBUG(log, "Loaded data parts (" << data_parts_indexes.size() << " items)"); + LOG_DEBUG_FORMATTED(log, "Loaded data parts ({} items)", data_parts_indexes.size()); } @@ -1197,7 +1197,7 @@ MergeTreeData::DataPartsVector MergeTreeData::grabOldParts(bool force) } if (!res.empty()) - LOG_TRACE(log, "Found " << res.size() << " old parts to remove."); + LOG_TRACE_FORMATTED(log, "Found {} old parts to remove.", res.size()); return res; } @@ -2771,7 +2771,7 @@ void MergeTreeData::dropDetached(const ASTPtr & partition, bool part, const Cont renamed_parts.addPart(part_info.dir_name, "deleting_" + part_info.dir_name); } - LOG_DEBUG(log, "Will drop " << renamed_parts.old_and_new_names.size() << " detached parts."); + LOG_DEBUG_FORMATTED(log, "Will drop {} detached parts.", renamed_parts.old_and_new_names.size()); renamed_parts.tryRenameAll(); @@ -3300,7 +3300,7 @@ void MergeTreeData::freezePartitionsByMatcher(MatcherFn matcher, const String & ++parts_processed; } - LOG_DEBUG(log, "Freezed " << parts_processed << " parts"); + LOG_DEBUG_FORMATTED(log, "Freezed {} parts", parts_processed); } bool MergeTreeData::canReplacePartition(const DataPartPtr & src_part) const @@ -3486,7 +3486,7 @@ MergeTreeData::CurrentlyMovingPartsTagger MergeTreeData::checkPartsForMove(const bool MergeTreeData::moveParts(CurrentlyMovingPartsTagger && moving_tagger) { - LOG_INFO(log, "Got " << moving_tagger.parts_to_move.size() << " parts to move."); + LOG_INFO_FORMATTED(log, "Got {} parts to move.", moving_tagger.parts_to_move.size()); for (const auto & moving_part : moving_tagger.parts_to_move) { diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 2d482e676da..8feeeadea45 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -633,7 +633,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor if (need_remove_expired_values && ttl_merges_blocker.isCancelled()) { - LOG_INFO(log, "Part " << new_data_part->name << " has values with expired TTL, but merges with TTL are cancelled."); + LOG_INFO_FORMATTED(log, "Part {} has values with expired TTL, but merges with TTL are cancelled.", new_data_part->name); need_remove_expired_values = false; } diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index f51e56f7735..49f3c97a8aa 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -1383,7 +1383,7 @@ MarkRanges MergeTreeDataSelectExecutor::filterMarksUsingIndex( { if (!part->volume->getDisk()->exists(part->getFullRelativePath() + index->getFileName() + ".idx")) { - LOG_DEBUG(log, "File for index " << backQuote(index->name) << " does not exist. Skipping it."); + LOG_DEBUG_FORMATTED(log, "File for index {} does not exist. Skipping it.", backQuote(index->name)); return ranges; } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp index fa4adfc0ea2..82d7c079020 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp @@ -151,7 +151,7 @@ void ReplicatedMergeTreeBlockOutputStream::write(const Block & block) } else { - LOG_DEBUG(log, "Wrote block with " << current_block.block.rows() << " rows"); + LOG_DEBUG_FORMATTED(log, "Wrote block with {} rows", current_block.block.rows()); } try @@ -214,7 +214,7 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zoo if (!block_number_lock) { - LOG_INFO(log, "Block with ID " << block_id << " already exists; ignoring it."); + LOG_INFO_FORMATTED(log, "Block with ID {} already exists; ignoring it.", block_id); part->is_duplicate = true; last_block_is_duplicate = true; ProfileEvents::increment(ProfileEvents::DuplicatedInsertedBlocks); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp index c36879750a1..c11b7444dcf 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp @@ -339,7 +339,7 @@ void ReplicatedMergeTreeCleanupThread::clearOldBlocks() auto num_nodes_to_delete = timed_blocks.end() - first_outdated_block; if (num_nodes_to_delete) - LOG_TRACE(log, "Cleared " << num_nodes_to_delete << " old blocks from ZooKeeper"); + LOG_TRACE_FORMATTED(log, "Cleared {} old blocks from ZooKeeper", num_nodes_to_delete); } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp index 4bb7d4cff52..ae7a5c23c01 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp @@ -116,7 +116,7 @@ void ReplicatedMergeTreePartCheckThread::searchForMissingPart(const String & par * and don't delete the queue entry when in doubt. */ - LOG_WARNING(log, "Checking if anyone has a part covering " << part_name << "."); + LOG_WARNING_FORMATTED(log, "Checking if anyone has a part covering {}.", part_name); bool found_part_with_the_same_min_block = false; bool found_part_with_the_same_max_block = false; @@ -170,14 +170,14 @@ void ReplicatedMergeTreePartCheckThread::searchForMissingPart(const String & par if (!storage.queue.remove(zookeeper, part_name)) { /// The part was not in our queue. Why did it happen? - LOG_ERROR(log, "Missing part " << part_name << " is not in our queue."); + LOG_ERROR_FORMATTED(log, "Missing part {} is not in our queue.", part_name); return; } /** This situation is possible if on all the replicas where the part was, it deteriorated. * For example, a replica that has just written it has power turned off and the data has not been written from cache to disk. */ - LOG_ERROR(log, "Part " << part_name << " is lost forever."); + LOG_ERROR_FORMATTED(log, "Part {} is lost forever.", part_name); ProfileEvents::increment(ProfileEvents::ReplicatedDataLoss); } @@ -216,7 +216,7 @@ CheckResult ReplicatedMergeTreePartCheckThread::checkPart(const String & part_na /// If the part is in ZooKeeper, check its data with its checksums, and them with ZooKeeper. if (zookeeper->tryGet(part_path, part_znode)) { - LOG_WARNING(log, "Checking data of part " << part_name << "."); + LOG_WARNING_FORMATTED(log, "Checking data of part {}.", part_name); try { @@ -247,7 +247,7 @@ CheckResult ReplicatedMergeTreePartCheckThread::checkPart(const String & part_na return {part_name, false, "Checking part was cancelled"}; } - LOG_INFO(log, "Part " << part_name << " looks good."); + LOG_INFO_FORMATTED(log, "Part {} looks good.", part_name); } catch (const Exception & e) { diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index acc10624df8..58fa6255a6d 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -152,7 +152,7 @@ void ReplicatedMergeTreeQueue::insertUnlocked( } if (entry->type == LogEntry::ALTER_METADATA) { - LOG_TRACE(log, "Adding alter metadata version " << entry->alter_version << " to the queue"); + LOG_TRACE_FORMATTED(log, "Adding alter metadata version {} to the queue", entry->alter_version); alter_sequence.addMetadataAlter(entry->alter_version, entry->have_mutation, state_lock); } } @@ -560,7 +560,7 @@ void ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper, C } if (!copied_entries.empty()) - LOG_DEBUG(log, "Pulled " << copied_entries.size() << " entries to queue."); + LOG_DEBUG_FORMATTED(log, "Pulled {} entries to queue.", copied_entries.size()); } if (storage.queue_task_handle) @@ -1397,7 +1397,7 @@ bool ReplicatedMergeTreeQueue::tryFinalizeMutations(zkutil::ZooKeeperPtr zookeep } else if (mutation.parts_to_do.size() == 0) { - LOG_TRACE(log, "Will check if mutation " << mutation.entry->znode_name << " is done"); + LOG_TRACE_FORMATTED(log, "Will check if mutation {} is done", mutation.entry->znode_name); candidates.push_back(mutation.entry); } } @@ -1406,7 +1406,7 @@ bool ReplicatedMergeTreeQueue::tryFinalizeMutations(zkutil::ZooKeeperPtr zookeep if (candidates.empty()) return false; else - LOG_DEBUG(log, "Trying to finalize " << candidates.size() << " mutations"); + LOG_DEBUG_FORMATTED(log, "Trying to finalize {} mutations", candidates.size()); auto merge_pred = getMergePredicate(zookeeper); @@ -1430,7 +1430,7 @@ bool ReplicatedMergeTreeQueue::tryFinalizeMutations(zkutil::ZooKeeperPtr zookeep auto it = mutations_by_znode.find(entry->znode_name); if (it != mutations_by_znode.end()) { - LOG_TRACE(log, "Mutation " << entry->znode_name << " is done"); + LOG_TRACE_FORMATTED(log, "Mutation {} is done", entry->znode_name); it->second.is_done = true; if (entry->isAlterMutation()) { diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index ccec2df91c2..e76fc8d4019 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -248,7 +248,7 @@ void ReplicatedMergeTreeRestartingThread::removeFailedQuorumParts() if (part) { - LOG_DEBUG(log, "Found part " << part_name << " with failed quorum. Moving to detached. This shouldn't happen often."); + LOG_DEBUG_FORMATTED(log, "Found part {} with failed quorum. Moving to detached. This shouldn't happen often.", part_name); storage.forgetPartAndMoveToDetached(part, "noquorum"); storage.queue.removeFromVirtualParts(part->info); } diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index e71d75816e8..c515b845a10 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -636,7 +636,7 @@ void StorageBuffer::writeBlockToDestination(const Block & block, StoragePtr tabl if (!table) { - LOG_ERROR(log, "Destination table " << destination_id.getNameForLogs() << " doesn't exist. Block of data is discarded."); + LOG_ERROR_FORMATTED(log, "Destination table {} doesn't exist. Block of data is discarded.", destination_id.getNameForLogs()); return; } diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 840408e467f..fd21393c96d 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -439,7 +439,7 @@ bool StorageDistributed::canForceGroupByNoMerge(const Context &context, QueryPro reason = "GROUP BY " + backQuote(serializeAST(*group_by, true)); } - LOG_DEBUG(log, "Force distributed_group_by_no_merge for " << reason << " (injective)"); + LOG_DEBUG_FORMATTED(log, "Force distributed_group_by_no_merge for {} (injective)", reason); return true; } diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 678bf510954..820d95dd54a 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -214,7 +214,7 @@ void StorageMergeTree::truncate(const ASTPtr &, const Context &, TableStructureW auto parts_to_remove = getDataPartsVector(); removePartsFromWorkingSet(parts_to_remove, true); - LOG_INFO(log, "Removed " << parts_to_remove.size() << " parts."); + LOG_INFO_FORMATTED(log, "Removed {} parts.", parts_to_remove.size()); } clearOldMutations(true); @@ -395,7 +395,7 @@ void StorageMergeTree::waitForMutation(Int64 version, const String & file_name) auto check = [version, this]() { return shutdown_called || isMutationDone(version); }; std::unique_lock lock(mutation_wait_mutex); mutation_wait_event.wait(lock, check); - LOG_INFO(log, "Mutation " << file_name << " done"); + LOG_INFO_FORMATTED(log, "Mutation {} done", file_name); } void StorageMergeTree::mutate(const MutationCommands & commands, const Context & query_context) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index b8319c45dca..204a71997b7 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -335,7 +335,7 @@ void StorageReplicatedMergeTree::waitMutationToFinishOnReplicas( Coordination::Stat exists_stat; if (!getZooKeeper()->exists(zookeeper_path + "/mutations/" + mutation_id, &exists_stat, wait_event)) { - LOG_WARNING(log, "Mutation " << mutation_id << " was killed or manually removed. Nothing to wait."); + LOG_WARNING_FORMATTED(log, "Mutation {} was killed or manually removed. Nothing to wait.", mutation_id); return; } @@ -883,7 +883,7 @@ MergeTreeData::DataPartsVector StorageReplicatedMergeTree::checkPartChecksumsAnd if (failed_op_index < num_check_ops && e.code == Coordination::ZNODEEXISTS) { - LOG_INFO(log, "The part " << e.getPathForFirstFailedOp() << " on a replica suddenly appeared, will recheck checksums"); + LOG_INFO_FORMATTED(log, "The part {} on a replica suddenly appeared, will recheck checksums", e.getPathForFirstFailedOp()); } else throw; @@ -935,12 +935,12 @@ bool StorageReplicatedMergeTree::executeLogEntry(LogEntry & entry) } if (entry.type == LogEntry::GET_PART && entry.source_replica == replica_name) - LOG_WARNING(log, "Part " << entry.new_part_name << " from own log doesn't exist."); + LOG_WARNING_FORMATTED(log, "Part {} from own log doesn't exist.", entry.new_part_name); /// Perhaps we don't need this part, because during write with quorum, the quorum has failed (see below about `/quorum/failed_parts`). if (entry.quorum && getZooKeeper()->exists(zookeeper_path + "/quorum/failed_parts/" + entry.new_part_name)) { - LOG_DEBUG(log, "Skipping action for part " << entry.new_part_name << " because quorum for that part was failed."); + LOG_DEBUG_FORMATTED(log, "Skipping action for part {} because quorum for that part was failed.", entry.new_part_name); return true; /// NOTE Deletion from `virtual_parts` is not done, but it is only necessary for merge. } @@ -992,7 +992,7 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) if (storage_settings_ptr->always_fetch_merged_part) { - LOG_INFO(log, "Will fetch part " << entry.new_part_name << " because setting 'always_fetch_merged_part' is true"); + LOG_INFO_FORMATTED(log, "Will fetch part {} because setting 'always_fetch_merged_part' is true", entry.new_part_name); return false; } @@ -1019,7 +1019,7 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) if (!have_all_parts) { /// If you do not have all the necessary parts, try to take some already merged part from someone. - LOG_DEBUG(log, "Don't have all parts for merge " << entry.new_part_name << "; will try to fetch it instead"); + LOG_DEBUG_FORMATTED(log, "Don't have all parts for merge {}; will try to fetch it instead", entry.new_part_name); return false; } else if (entry.create_time + storage_settings_ptr->prefer_fetch_merged_part_time_threshold.totalSeconds() <= time(nullptr)) @@ -1378,7 +1378,7 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) if (code == Coordination::ZOK) { - LOG_DEBUG(log, "Marked quorum for part " << entry.new_part_name << " as failed."); + LOG_DEBUG_FORMATTED(log, "Marked quorum for part {} as failed.", entry.new_part_name); queue.removeFromVirtualParts(part_info); return true; } @@ -1437,7 +1437,7 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) if (!parts_for_merge.empty() && replica.empty()) { - LOG_INFO(log, "No active replica has part " << entry.new_part_name << ". Will fetch merged part instead."); + LOG_INFO_FORMATTED(log, "No active replica has part {}. Will fetch merged part instead.", entry.new_part_name); return false; } @@ -1595,7 +1595,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) source_table = DatabaseCatalog::instance().tryGetTable(source_table_id); if (!source_table) { - LOG_DEBUG(log, "Can't use " << source_table_id.getNameForLogs() << " as source table for REPLACE PARTITION command. It does not exist."); + LOG_DEBUG_FORMATTED(log, "Can't use {} as source table for REPLACE PARTITION command. It does not exist.", source_table_id.getNameForLogs()); return 0; } @@ -1669,7 +1669,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) if (replica.empty()) { - LOG_DEBUG(log, "Part " << part_desc->new_part_name << " is not found on remote replicas"); + LOG_DEBUG_FORMATTED(log, "Part {} is not found on remote replicas", part_desc->new_part_name); /// Fallback to covering part replica = findReplicaHavingCoveringPart(part_desc->new_part_name, true, found_part_name); @@ -1677,7 +1677,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) if (replica.empty()) { /// It is not fail, since adjacent parts could cover current part - LOG_DEBUG(log, "Parts covering " << part_desc->new_part_name << " are not found on remote replicas"); + LOG_DEBUG_FORMATTED(log, "Parts covering {} are not found on remote replicas", part_desc->new_part_name); continue; } } @@ -1834,7 +1834,7 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo */ while (!zookeeper->exists(source_path + "/columns")) { - LOG_INFO(log, "Waiting for replica " << source_path << " to be fully created"); + LOG_INFO_FORMATTED(log, "Waiting for replica {} to be fully created", source_path); zkutil::EventPtr event = std::make_shared(); if (zookeeper->exists(source_path + "/columns", nullptr, event)) @@ -1901,7 +1901,7 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo { /// If source replica's log_pointer changed than we probably read /// stale state of /queue and have to try one more time. - LOG_WARNING(log, "Log pointer of source replica " << source_replica << " changed while we loading queue nodes. Will retry."); + LOG_WARNING_FORMATTED(log, "Log pointer of source replica {} changed while we loading queue nodes. Will retry.", source_replica); continue; } else @@ -1936,7 +1936,7 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo { queue.remove(zookeeper, part); parts_to_remove_from_zk.emplace_back(part); - LOG_WARNING(log, "Source replica does not have part " << part << ". Removing it from ZooKeeper."); + LOG_WARNING_FORMATTED(log, "Source replica does not have part {}. Removing it from ZooKeeper.", part); } } tryRemovePartsFromZooKeeperWithRetries(parts_to_remove_from_zk); @@ -1948,7 +1948,7 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo if (active_parts_set.getContainingPart(part->name).empty()) { parts_to_remove_from_working_set.emplace_back(part); - LOG_WARNING(log, "Source replica does not have part " << part->name << ". Removing it from working set."); + LOG_WARNING_FORMATTED(log, "Source replica does not have part {}. Removing it from working set.", part->name); } } removePartsFromWorkingSet(parts_to_remove_from_working_set, true); @@ -1964,7 +1964,7 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo zookeeper->create(replica_path + "/queue/queue-", log_entry.toString(), zkutil::CreateMode::PersistentSequential); } - LOG_DEBUG(log, "Queued " << active_parts.size() << " parts to be fetched"); + LOG_DEBUG_FORMATTED(log, "Queued {} parts to be fetched", active_parts.size()); /// Add content of the reference/master replica queue to the queue. for (const String & entry : source_queue) @@ -1972,7 +1972,7 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo zookeeper->create(replica_path + "/queue/queue-", entry, zkutil::CreateMode::PersistentSequential); } - LOG_DEBUG(log, "Copied " << source_queue.size() << " queue entries"); + LOG_DEBUG_FORMATTED(log, "Copied {} queue entries", source_queue.size()); } @@ -2772,7 +2772,7 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Strin if (auto part = getPartIfExists(part_info, {IMergeTreeDataPart::State::Outdated, IMergeTreeDataPart::State::Deleting})) { - LOG_DEBUG(log, "Part " << part->name << " should be deleted after previous attempt before fetch"); + LOG_DEBUG_FORMATTED(log, "Part {} should be deleted after previous attempt before fetch", part->name); /// Force immediate parts cleanup to delete the part that was left from the previous fetch attempt. cleanup_thread.wakeup(); return false; @@ -2782,7 +2782,7 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Strin std::lock_guard lock(currently_fetching_parts_mutex); if (!currently_fetching_parts.insert(part_name).second) { - LOG_DEBUG(log, "Part " << part_name << " is already fetching right now"); + LOG_DEBUG_FORMATTED(log, "Part {} is already fetching right now", part_name); return false; } } @@ -3734,7 +3734,7 @@ void StorageReplicatedMergeTree::drop() Strings replicas; if (zookeeper->tryGetChildren(zookeeper_path + "/replicas", replicas) == Coordination::ZOK && replicas.empty()) { - LOG_INFO(log, "Removing table " << zookeeper_path << " (this might take several minutes)"); + LOG_INFO_FORMATTED(log, "Removing table {} (this might take several minutes)", zookeeper_path); zookeeper->tryRemoveRecursive(zookeeper_path); if (zookeeper->exists(zookeeper_path)) LOG_ERROR(log, "Table was not completely removed from ZooKeeper, " @@ -3963,7 +3963,7 @@ bool StorageReplicatedMergeTree::waitForReplicaToProcessLogEntry(const String & if (!log_node_name.empty()) LOG_DEBUG(log, "Looking for node corresponding to " << log_node_name << " in " << replica << " queue"); else - LOG_DEBUG(log, "Looking for corresponding node in " << replica << " queue"); + LOG_DEBUG_FORMATTED(log, "Looking for corresponding node in {} queue", replica); /** Second - find the corresponding entry in the queue of the specified replica. * Its number may match neither the `log` node nor the `queue` node of the current replica (for us). @@ -4359,7 +4359,7 @@ void StorageReplicatedMergeTree::fetchPartition(const ASTPtr & partition, const do { if (try_no) - LOG_INFO(log, "Some of parts (" << missing_parts.size() << ") are missing. Will try to fetch covering parts."); + LOG_INFO_FORMATTED(log, "Some of parts ({}) are missing. Will try to fetch covering parts.", missing_parts.size()); if (try_no >= query_context.getSettings().max_fetch_partition_retries_count) throw Exception("Too many retries to fetch parts from " + best_replica_path, ErrorCodes::TOO_MANY_RETRIES_TO_FETCH_PARTS); @@ -4623,7 +4623,7 @@ void StorageReplicatedMergeTree::clearOldPartsAndRemoveFromZK() remove_parts_from_filesystem(parts_to_delete_only_from_filesystem); removePartsFinally(parts_to_delete_only_from_filesystem); - LOG_DEBUG(log, "Removed " << parts_to_delete_only_from_filesystem.size() << " old duplicate parts"); + LOG_DEBUG_FORMATTED(log, "Removed {} old duplicate parts", parts_to_delete_only_from_filesystem.size()); } /// Delete normal parts from ZooKeeper @@ -4634,7 +4634,7 @@ void StorageReplicatedMergeTree::clearOldPartsAndRemoveFromZK() for (const auto & part : parts_to_delete_completely) part_names_to_delete_completely.emplace_back(part->name); - LOG_DEBUG(log, "Removing " << parts_to_delete_completely.size() << " old parts from ZooKeeper"); + LOG_DEBUG_FORMATTED(log, "Removing {} old parts from ZooKeeper", parts_to_delete_completely.size()); removePartsFromZooKeeper(zookeeper, part_names_to_delete_completely, &part_names_to_retry_deletion); } catch (...) @@ -4644,7 +4644,7 @@ void StorageReplicatedMergeTree::clearOldPartsAndRemoveFromZK() /// Part names that were reliably deleted from ZooKeeper should be deleted from filesystem auto num_reliably_deleted_parts = parts_to_delete_completely.size() - part_names_to_retry_deletion.size(); - LOG_DEBUG(log, "Removed " << num_reliably_deleted_parts << " old parts from ZooKeeper. Removing them from filesystem."); + LOG_DEBUG_FORMATTED(log, "Removed {} old parts from ZooKeeper. Removing them from filesystem.", num_reliably_deleted_parts); /// Delete normal parts on two sets for (auto & part : parts_to_delete_completely) @@ -4659,7 +4659,7 @@ void StorageReplicatedMergeTree::clearOldPartsAndRemoveFromZK() if (!parts_to_retry_deletion.empty()) { rollbackDeletingParts(parts_to_retry_deletion); - LOG_DEBUG(log, "Will retry deletion of " << parts_to_retry_deletion.size() << " parts in the next time"); + LOG_DEBUG_FORMATTED(log, "Will retry deletion of {} parts in the next time", parts_to_retry_deletion.size()); } /// Remove parts from filesystem and finally from data_parts @@ -4668,7 +4668,7 @@ void StorageReplicatedMergeTree::clearOldPartsAndRemoveFromZK() remove_parts_from_filesystem(parts_to_remove_from_filesystem); removePartsFinally(parts_to_remove_from_filesystem); - LOG_DEBUG(log, "Removed " << parts_to_remove_from_filesystem.size() << " old parts"); + LOG_DEBUG_FORMATTED(log, "Removed {} old parts", parts_to_remove_from_filesystem.size()); } } @@ -4890,7 +4890,7 @@ void StorageReplicatedMergeTree::replacePartitionFrom(const StoragePtr & source_ Strings part_checksums; std::vector ephemeral_locks; - LOG_DEBUG(log, "Cloning " << src_all_parts.size() << " parts"); + LOG_DEBUG_FORMATTED(log, "Cloning {} parts", src_all_parts.size()); static const String TMP_PREFIX = "tmp_replace_from_"; auto zookeeper = getZooKeeper(); @@ -5080,7 +5080,7 @@ void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_ta Strings part_checksums; std::vector ephemeral_locks; - LOG_DEBUG(log, "Cloning " << src_all_parts.size() << " parts"); + LOG_DEBUG_FORMATTED(log, "Cloning {} parts", src_all_parts.size()); static const String TMP_PREFIX = "tmp_move_from_"; auto zookeeper = getZooKeeper(); @@ -5371,7 +5371,7 @@ bool StorageReplicatedMergeTree::dropPartsInPartition( MergeTreePartInfo drop_range_info; if (!getFakePartCoveringAllPartsInPartition(partition_id, drop_range_info)) { - LOG_INFO(log, "Will not drop partition " << partition_id << ", it is empty."); + LOG_INFO_FORMATTED(log, "Will not drop partition {}, it is empty.", partition_id); return false; } From 8042e5febefd1b28336652a5ac2cb1e32ca31ca4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 19:58:15 +0300 Subject: [PATCH 623/738] find {base,src,programs} -name '*.h' -or -name '*.cpp' | xargs grep -l -P 'LOG_\w+\([^,]+, "[^"]+" << [^<]+ << "[^"]+" << [^<]+\);' | xargs sed -i -r -e 's/(LOG_\w+)\(([^,]+), "([^"]+)" << ([^<]+) << "([^"]+)" << ([^<]+)\);/\1_FORMATTED(\2, "\3{}\5{}", \4, \6);/' --- base/daemon/BaseDaemon.cpp | 2 +- programs/copier/ClusterCopier.cpp | 8 ++++---- programs/odbc-bridge/ODBCBridge.cpp | 2 +- programs/server/MySQLHandlerFactory.cpp | 2 +- src/Access/ContextAccess.cpp | 2 +- src/Common/Config/ConfigProcessor.cpp | 2 +- src/Common/ShellCommand.cpp | 4 ++-- src/Common/StatusFile.cpp | 6 +++--- src/Common/ZooKeeper/Lock.cpp | 2 +- src/Common/ZooKeeper/ZooKeeper.cpp | 2 +- .../CollapsingFinalBlockInputStream.cpp | 2 +- src/DataStreams/RemoteBlockInputStream.cpp | 2 +- src/DataStreams/TTLBlockInputStream.cpp | 2 +- .../ClickHouseDictionarySource.cpp | 2 +- .../ExecutableDictionarySource.cpp | 4 ++-- src/Dictionaries/HTTPDictionarySource.cpp | 4 ++-- src/Dictionaries/LibraryDictionarySource.cpp | 4 ++-- src/Disks/S3/DiskS3.cpp | 2 +- src/Interpreters/DDLWorker.cpp | 2 +- src/Interpreters/DatabaseCatalog.cpp | 6 +++--- .../DistributedBlockOutputStream.cpp | 2 +- .../Kafka/ReadBufferFromKafkaConsumer.cpp | 2 +- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 2 +- src/Storages/MergeTree/MergeTreeData.cpp | 6 +++--- .../MergeTree/MergeTreeDataMergerMutator.cpp | 6 +++--- .../MergeTree/MergeTreePartsMover.cpp | 4 ++-- .../ReplicatedMergeTreePartCheckThread.cpp | 2 +- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 2 +- .../ReplicatedMergeTreeRestartingThread.cpp | 4 ++-- src/Storages/StorageMergeTree.cpp | 4 ++-- src/Storages/StorageReplicatedMergeTree.cpp | 20 +++++++++---------- 31 files changed, 58 insertions(+), 58 deletions(-) diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index bb58c3633d8..18773a3ff5e 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -734,7 +734,7 @@ void BaseDaemon::handleNotification(Poco::TaskFailedNotification *_tfn) task_failed = true; Poco::AutoPtr fn(_tfn); Logger *lg = &(logger()); - LOG_ERROR(lg, "Task '" << fn->task()->name() << "' failed. Daemon is shutting down. Reason - " << fn->reason().displayText()); + LOG_ERROR_FORMATTED(lg, "Task '{}' failed. Daemon is shutting down. Reason - {}", fn->task()->name(), fn->reason().displayText()); ServerApplication::terminate(); } diff --git a/programs/copier/ClusterCopier.cpp b/programs/copier/ClusterCopier.cpp index 9a7716a9a57..9551408e2b7 100644 --- a/programs/copier/ClusterCopier.cpp +++ b/programs/copier/ClusterCopier.cpp @@ -169,7 +169,7 @@ void ClusterCopier::discoverShardPartitions(const ConnectionTimeouts & timeouts, << task_shard->getDescription() << " :" << ss.str()); } - LOG_DEBUG(log, "Will copy " << task_shard->partition_tasks.size() << " partitions from shard " << task_shard->getDescription()); + LOG_DEBUG_FORMATTED(log, "Will copy {} partitions from shard {}", task_shard->partition_tasks.size(), task_shard->getDescription()); } void ClusterCopier::discoverTablePartitions(const ConnectionTimeouts & timeouts, TaskTable & task_table, UInt64 num_threads) @@ -911,7 +911,7 @@ bool ClusterCopier::tryProcessTable(const ConnectionTimeouts & timeouts, TaskTab { const size_t number_of_splits = task_table.number_of_splits; shard->partition_tasks.emplace(partition_name, ShardPartition(*shard, partition_name, number_of_splits)); - LOG_DEBUG(log, "Discovered partition " << partition_name << " in shard " << shard->getDescription()); + LOG_DEBUG_FORMATTED(log, "Discovered partition {} in shard {}", partition_name, shard->getDescription()); /// To save references in the future. auto shard_partition_it = shard->partition_tasks.find(partition_name); PartitionPieces & shard_partition_pieces = shard_partition_it->second.pieces; @@ -924,7 +924,7 @@ bool ClusterCopier::tryProcessTable(const ConnectionTimeouts & timeouts, TaskTab } else { - LOG_DEBUG(log, "Found that shard " << shard->getDescription() << " does not contain current partition " << partition_name); + LOG_DEBUG_FORMATTED(log, "Found that shard {} does not contain current partition {}", shard->getDescription(), partition_name); continue; } } @@ -1744,7 +1744,7 @@ std::set ClusterCopier::getShardPartitions(const ConnectionTimeouts & ti } } - LOG_DEBUG(log, "There are " << res.size() << " destination partitions in shard " << task_shard.getDescription()); + LOG_DEBUG_FORMATTED(log, "There are {} destination partitions in shard {}", res.size(), task_shard.getDescription()); return res; } diff --git a/programs/odbc-bridge/ODBCBridge.cpp b/programs/odbc-bridge/ODBCBridge.cpp index b878a7c840f..e707606acac 100644 --- a/programs/odbc-bridge/ODBCBridge.cpp +++ b/programs/odbc-bridge/ODBCBridge.cpp @@ -198,7 +198,7 @@ int ODBCBridge::main(const std::vector & /*args*/) { if (server.currentConnections() == 0) break; - LOG_DEBUG(log, "Waiting for " << server.currentConnections() << " connections, try " << count); + LOG_DEBUG_FORMATTED(log, "Waiting for {} connections, try {}", server.currentConnections(), count); std::this_thread::sleep_for(std::chrono::milliseconds(1000)); } }); diff --git a/programs/server/MySQLHandlerFactory.cpp b/programs/server/MySQLHandlerFactory.cpp index 9c98739380e..f5d463be1ad 100644 --- a/programs/server/MySQLHandlerFactory.cpp +++ b/programs/server/MySQLHandlerFactory.cpp @@ -122,7 +122,7 @@ void MySQLHandlerFactory::generateRSAKeys() Poco::Net::TCPServerConnection * MySQLHandlerFactory::createConnection(const Poco::Net::StreamSocket & socket) { size_t connection_id = last_connection_id++; - LOG_TRACE(log, "MySQL connection. Id: " << connection_id << ". Address: " << socket.peerAddress().toString()); + LOG_TRACE_FORMATTED(log, "MySQL connection. Id: {}. Address: {}", connection_id, socket.peerAddress().toString()); #if USE_SSL return new MySQLHandlerSSL(server, socket, ssl_enabled, connection_id, *public_key, *private_key); #else diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index f973e93c76b..a8e22180490 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -200,7 +200,7 @@ bool ContextAccess::calculateResultAccessAndCheck(Poco::Logger * log_, const Acc bool is_granted = access->isGranted(flags, args...); if (trace_log) - LOG_TRACE(trace_log, "Access " << (is_granted ? "granted" : "denied") << ": " << (AccessRightsElement{flags, args...}.toString())); + LOG_TRACE_FORMATTED(trace_log, "Access {}: {}", (is_granted ? "granted" : "denied"), (AccessRightsElement{flags, args...}.toString())); if (is_granted) return true; diff --git a/src/Common/Config/ConfigProcessor.cpp b/src/Common/Config/ConfigProcessor.cpp index d5f92f05886..7daf5321d24 100644 --- a/src/Common/Config/ConfigProcessor.cpp +++ b/src/Common/Config/ConfigProcessor.cpp @@ -623,7 +623,7 @@ void ConfigProcessor::savePreprocessedConfig(const LoadedConfig & loaded_config, } catch (Poco::Exception & e) { - LOG_WARNING(log, "Couldn't save preprocessed config to " << preprocessed_path << ": " << e.displayText()); + LOG_WARNING_FORMATTED(log, "Couldn't save preprocessed config to {}: {}", preprocessed_path, e.displayText()); } } diff --git a/src/Common/ShellCommand.cpp b/src/Common/ShellCommand.cpp index f19ea031c6a..ec536d956e5 100644 --- a/src/Common/ShellCommand.cpp +++ b/src/Common/ShellCommand.cpp @@ -134,7 +134,7 @@ std::unique_ptr ShellCommand::executeImpl(const char * filename, c std::unique_ptr res(new ShellCommand(pid, pipe_stdin.fds_rw[1], pipe_stdout.fds_rw[0], pipe_stderr.fds_rw[0], terminate_in_destructor)); - LOG_TRACE(getLogger(), "Started shell command '" << filename << "' with pid " << pid); + LOG_TRACE_FORMATTED(getLogger(), "Started shell command '{}' with pid {}", filename, pid); /// Now the ownership of the file descriptors is passed to the result. pipe_stdin.fds_rw[1] = -1; @@ -194,7 +194,7 @@ int ShellCommand::tryWait() if (-1 == waitpid(pid, &status, 0)) throwFromErrno("Cannot waitpid", ErrorCodes::CANNOT_WAITPID); - LOG_TRACE(getLogger(), "Wait for shell command pid " << pid << " completed with status " << status); + LOG_TRACE_FORMATTED(getLogger(), "Wait for shell command pid {} completed with status {}", pid, status); if (WIFEXITED(status)) return WEXITSTATUS(status); diff --git a/src/Common/StatusFile.cpp b/src/Common/StatusFile.cpp index 7609e1f1c73..a3a8cbda593 100644 --- a/src/Common/StatusFile.cpp +++ b/src/Common/StatusFile.cpp @@ -43,7 +43,7 @@ StatusFile::StatusFile(const std::string & path_) } if (!contents.empty()) - LOG_INFO(&Logger::get("StatusFile"), "Status file " << path << " already exists - unclean restart. Contents:\n" << contents); + LOG_INFO_FORMATTED(&Logger::get("StatusFile"), "Status file {} already exists - unclean restart. Contents:\n{}", path, contents); else LOG_INFO_FORMATTED(&Logger::get("StatusFile"), "Status file {} already exists and is empty - probably unclean hardware restart.", path); } @@ -90,10 +90,10 @@ StatusFile::StatusFile(const std::string & path_) StatusFile::~StatusFile() { if (0 != close(fd)) - LOG_ERROR(&Logger::get("StatusFile"), "Cannot close file " << path << ", " << errnoToString(ErrorCodes::CANNOT_CLOSE_FILE)); + LOG_ERROR_FORMATTED(&Logger::get("StatusFile"), "Cannot close file {}, {}", path, errnoToString(ErrorCodes::CANNOT_CLOSE_FILE)); if (0 != unlink(path.c_str())) - LOG_ERROR(&Logger::get("StatusFile"), "Cannot unlink file " << path << ", " << errnoToString(ErrorCodes::CANNOT_CLOSE_FILE)); + LOG_ERROR_FORMATTED(&Logger::get("StatusFile"), "Cannot unlink file {}, {}", path, errnoToString(ErrorCodes::CANNOT_CLOSE_FILE)); } } diff --git a/src/Common/ZooKeeper/Lock.cpp b/src/Common/ZooKeeper/Lock.cpp index c434ec3e172..ccea3c75dd7 100644 --- a/src/Common/ZooKeeper/Lock.cpp +++ b/src/Common/ZooKeeper/Lock.cpp @@ -64,7 +64,7 @@ Lock::Status Lock::tryCheck() const } if (locked && lock_status != LOCKED_BY_ME) - LOG_WARNING(log, "Lock is lost. It is normal if session was expired. Path: " << lock_path << "/" << lock_message); + LOG_WARNING_FORMATTED(log, "Lock is lost. It is normal if session was expired. Path: {}/{}", lock_path, lock_message); return lock_status; } diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index 2e94cfe9992..a6040a6ed5b 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -79,7 +79,7 @@ void ZooKeeper::init(const std::string & implementation_, const std::string & ho } catch (const Poco::Net::DNSException & e) { - LOG_ERROR(log, "Cannot use ZooKeeper host " << host_string << ", reason: " << e.displayText()); + LOG_ERROR_FORMATTED(log, "Cannot use ZooKeeper host {}, reason: {}", host_string, e.displayText()); } } diff --git a/src/DataStreams/CollapsingFinalBlockInputStream.cpp b/src/DataStreams/CollapsingFinalBlockInputStream.cpp index a90d41b23ba..744a3560477 100644 --- a/src/DataStreams/CollapsingFinalBlockInputStream.cpp +++ b/src/DataStreams/CollapsingFinalBlockInputStream.cpp @@ -135,7 +135,7 @@ Block CollapsingFinalBlockInputStream::readImpl() if (output_blocks.empty()) { if (blocks_fetched != blocks_output) - LOG_ERROR(log, "Logical error: CollapsingFinalBlockInputStream has output " << blocks_output << " blocks instead of " << blocks_fetched); + LOG_ERROR_FORMATTED(log, "Logical error: CollapsingFinalBlockInputStream has output {} blocks instead of {}", blocks_output, blocks_fetched); return Block(); } diff --git a/src/DataStreams/RemoteBlockInputStream.cpp b/src/DataStreams/RemoteBlockInputStream.cpp index e9b8d26f975..12ac5032395 100644 --- a/src/DataStreams/RemoteBlockInputStream.cpp +++ b/src/DataStreams/RemoteBlockInputStream.cpp @@ -372,7 +372,7 @@ void RemoteBlockInputStream::tryCancel(const char * reason) multiplexed_connections->sendCancel(); } - LOG_TRACE(log, "(" << multiplexed_connections->dumpAddresses() << ") " << reason); + LOG_TRACE_FORMATTED(log, "({}) {}", multiplexed_connections->dumpAddresses(), reason); } bool RemoteBlockInputStream::isQueryPending() const diff --git a/src/DataStreams/TTLBlockInputStream.cpp b/src/DataStreams/TTLBlockInputStream.cpp index c9a40768166..ea970cd564d 100644 --- a/src/DataStreams/TTLBlockInputStream.cpp +++ b/src/DataStreams/TTLBlockInputStream.cpp @@ -108,7 +108,7 @@ void TTLBlockInputStream::readSuffixImpl() data_part->expired_columns = std::move(empty_columns); if (rows_removed) - LOG_INFO(log, "Removed " << rows_removed << " rows with expired TTL from part " << data_part->name); + LOG_INFO_FORMATTED(log, "Removed {} rows with expired TTL from part {}", rows_removed, data_part->name); } void TTLBlockInputStream::removeRowsWithExpiredTableTTL(Block & block) diff --git a/src/Dictionaries/ClickHouseDictionarySource.cpp b/src/Dictionaries/ClickHouseDictionarySource.cpp index 89ce56df369..99f294c1f3f 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.cpp +++ b/src/Dictionaries/ClickHouseDictionarySource.cpp @@ -168,7 +168,7 @@ bool ClickHouseDictionarySource::isModified() const if (!invalidate_query.empty()) { auto response = doInvalidateQuery(invalidate_query); - LOG_TRACE(log, "Invalidate query has returned: " << response << ", previous value: " << invalidate_query_response); + LOG_TRACE_FORMATTED(log, "Invalidate query has returned: {}, previous value: {}", response, invalidate_query_response); if (invalidate_query_response == response) return false; invalidate_query_response = response; diff --git a/src/Dictionaries/ExecutableDictionarySource.cpp b/src/Dictionaries/ExecutableDictionarySource.cpp index 34943d62b44..6d392cd2b74 100644 --- a/src/Dictionaries/ExecutableDictionarySource.cpp +++ b/src/Dictionaries/ExecutableDictionarySource.cpp @@ -163,7 +163,7 @@ namespace BlockInputStreamPtr ExecutableDictionarySource::loadIds(const std::vector & ids) { - LOG_TRACE(log, "loadIds " << toString() << " size = " << ids.size()); + LOG_TRACE_FORMATTED(log, "loadIds {} size = {}", toString(), ids.size()); auto process = ShellCommand::execute(command); auto output_stream = context.getOutputFormat(format, process->in, sample_block); @@ -175,7 +175,7 @@ BlockInputStreamPtr ExecutableDictionarySource::loadIds(const std::vector & requested_rows) { - LOG_TRACE(log, "loadKeys " << toString() << " size = " << requested_rows.size()); + LOG_TRACE_FORMATTED(log, "loadKeys {} size = {}", toString(), requested_rows.size()); auto process = ShellCommand::execute(command); auto output_stream = context.getOutputFormat(format, process->in, sample_block); diff --git a/src/Dictionaries/HTTPDictionarySource.cpp b/src/Dictionaries/HTTPDictionarySource.cpp index 61f16797ce0..f0847af2e8c 100644 --- a/src/Dictionaries/HTTPDictionarySource.cpp +++ b/src/Dictionaries/HTTPDictionarySource.cpp @@ -128,7 +128,7 @@ BlockInputStreamPtr HTTPDictionarySource::loadUpdatedAll() BlockInputStreamPtr HTTPDictionarySource::loadIds(const std::vector & ids) { - LOG_TRACE(log, "loadIds " << toString() << " size = " << ids.size()); + LOG_TRACE_FORMATTED(log, "loadIds {} size = {}", toString(), ids.size()); ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback = [&](std::ostream & ostr) { @@ -147,7 +147,7 @@ BlockInputStreamPtr HTTPDictionarySource::loadIds(const std::vector & id BlockInputStreamPtr HTTPDictionarySource::loadKeys(const Columns & key_columns, const std::vector & requested_rows) { - LOG_TRACE(log, "loadKeys " << toString() << " size = " << requested_rows.size()); + LOG_TRACE_FORMATTED(log, "loadKeys {} size = {}", toString(), requested_rows.size()); ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback = [&](std::ostream & ostr) { diff --git a/src/Dictionaries/LibraryDictionarySource.cpp b/src/Dictionaries/LibraryDictionarySource.cpp index 79f90fcf22c..6f887ee29e3 100644 --- a/src/Dictionaries/LibraryDictionarySource.cpp +++ b/src/Dictionaries/LibraryDictionarySource.cpp @@ -207,7 +207,7 @@ BlockInputStreamPtr LibraryDictionarySource::loadAll() BlockInputStreamPtr LibraryDictionarySource::loadIds(const std::vector & ids) { - LOG_TRACE(log, "loadIds " << toString() << " size = " << ids.size()); + LOG_TRACE_FORMATTED(log, "loadIds {} size = {}", toString(), ids.size()); const ClickHouseLibrary::VectorUInt64 ids_data{ext::bit_cast(ids.data()), ids.size()}; auto columns_holder = std::make_unique(dict_struct.attributes.size()); @@ -234,7 +234,7 @@ BlockInputStreamPtr LibraryDictionarySource::loadIds(const std::vector & BlockInputStreamPtr LibraryDictionarySource::loadKeys(const Columns & key_columns, const std::vector & requested_rows) { - LOG_TRACE(log, "loadKeys " << toString() << " size = " << requested_rows.size()); + LOG_TRACE_FORMATTED(log, "loadKeys {} size = {}", toString(), requested_rows.size()); auto holder = std::make_unique(key_columns.size()); std::vector> column_data_holders; diff --git a/src/Disks/S3/DiskS3.cpp b/src/Disks/S3/DiskS3.cpp index af83fdbfe10..8014c797909 100644 --- a/src/Disks/S3/DiskS3.cpp +++ b/src/Disks/S3/DiskS3.cpp @@ -537,7 +537,7 @@ std::unique_ptr DiskS3::writeFile(const String & path, /// Save empty metadata to disk to have ability to get file size while buffer is not finalized. metadata.save(); - LOG_DEBUG(&Logger::get("DiskS3"), "Write to file by path: " << backQuote(metadata_path + path) << " New S3 path: " << s3_path); + LOG_DEBUG_FORMATTED(&Logger::get("DiskS3"), "Write to file by path: {} New S3 path: {}", backQuote(metadata_path + path), s3_path); return std::make_unique(client, bucket, metadata, s3_path, min_upload_part_size, buf_size); } diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index 0924ae433f8..48b443a30f5 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -400,7 +400,7 @@ void DDLWorker::processTasks() String reason; if (!initAndCheckTask(entry_name, reason, zookeeper)) { - LOG_DEBUG(log, "Will not execute task " << entry_name << ": " << reason); + LOG_DEBUG_FORMATTED(log, "Will not execute task {}: {}", entry_name, reason); last_processed_task_name = entry_name; continue; } diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index bbefa016507..4180690ac86 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -572,7 +572,7 @@ void DatabaseCatalog::enqueueDroppedTableCleanup(StorageID table_id, StoragePtr else { /// Try load table from metadata to drop it correctly (e.g. remove metadata from zk or remove data from all volumes) - LOG_INFO(log, "Trying load partially dropped table " << table_id.getNameForLogs() << " from " << dropped_metadata_path); + LOG_INFO_FORMATTED(log, "Trying load partially dropped table {} from {}", table_id.getNameForLogs(), dropped_metadata_path); ASTPtr ast = DatabaseOnDisk::parseQueryFromMetadata(log, *global_context, dropped_metadata_path, /*throw_on_error*/ false, /*remove_empty*/false); auto * create = typeid_cast(ast.get()); assert(!create || create->uuid == table_id.uuid); @@ -685,11 +685,11 @@ void DatabaseCatalog::dropTableFinally(const TableMarkedAsDropped & table) const Poco::File table_data_dir{data_path}; if (table_data_dir.exists()) { - LOG_INFO(log, "Removing data directory " << data_path << " of dropped table " << table.table_id.getNameForLogs()); + LOG_INFO_FORMATTED(log, "Removing data directory {} of dropped table {}", data_path, table.table_id.getNameForLogs()); table_data_dir.remove(true); } - LOG_INFO(log, "Removing metadata " << table.metadata_path << " of dropped table " << table.table_id.getNameForLogs()); + LOG_INFO_FORMATTED(log, "Removing metadata {} of dropped table {}", table.metadata_path, table.table_id.getNameForLogs()); Poco::File(table.metadata_path).remove(); } diff --git a/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/src/Storages/Distributed/DistributedBlockOutputStream.cpp index e08c4b7fd34..315c380838d 100644 --- a/src/Storages/Distributed/DistributedBlockOutputStream.cpp +++ b/src/Storages/Distributed/DistributedBlockOutputStream.cpp @@ -229,7 +229,7 @@ void DistributedBlockOutputStream::waitForJobs() size_t num_finished_jobs = finished_jobs_count; if (num_finished_jobs < jobs_count) - LOG_WARNING(log, "Expected " << jobs_count << " writing jobs, but finished only " << num_finished_jobs); + LOG_WARNING_FORMATTED(log, "Expected {} writing jobs, but finished only {}", jobs_count, num_finished_jobs); } diff --git a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp index 873e03580bc..875c29c8835 100644 --- a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp +++ b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp @@ -329,7 +329,7 @@ bool ReadBufferFromKafkaConsumer::nextImpl() { messages = std::move(new_messages); current = messages.begin(); - LOG_TRACE(log, "Polled batch of " << messages.size() << " messages. Offset position: " << consumer->get_offsets_position(consumer->get_assignment())); + LOG_TRACE_FORMATTED(log, "Polled batch of {} messages. Offset position: {}", messages.size(), consumer->get_offsets_position(consumer->get_assignment())); break; } } diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 882c1226273..ed6a0784f62 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -721,7 +721,7 @@ void IMergeTreeDataPart::remove() const } catch (...) { - LOG_ERROR(storage.log, "Cannot recursively remove directory " << fullPath(volume->getDisk(), to) << ". Exception: " << getCurrentExceptionMessage(false)); + LOG_ERROR_FORMATTED(storage.log, "Cannot recursively remove directory {}. Exception: {}", fullPath(volume->getDisk(), to), getCurrentExceptionMessage(false)); throw; } } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 0fe1681e819..46cd7da87b9 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -1838,7 +1838,7 @@ void MergeTreeData::renameTempPartAndReplace( if (covering_part) { - LOG_WARNING(log, "Tried to add obsolete part " << part_name << " covered by " << covering_part->getNameWithState()); + LOG_WARNING_FORMATTED(log, "Tried to add obsolete part {} covered by {}", part_name, covering_part->getNameWithState()); return; } @@ -2802,7 +2802,7 @@ MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const else { String partition_id = getPartitionIDFromQuery(partition, context); - LOG_DEBUG(log, "Looking for parts for partition " << partition_id << " in " << source_dir); + LOG_DEBUG_FORMATTED(log, "Looking for parts for partition {} in {}", partition_id, source_dir); ActiveDataPartSet active_parts(format_version); const auto disks = getStoragePolicy()->getDisks(); @@ -3205,7 +3205,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::cloneAndLoadDataPartOnSameDisk( if (disk->exists(dst_part_path)) throw Exception("Part in " + fullPath(disk, dst_part_path) + " already exists", ErrorCodes::DIRECTORY_ALREADY_EXISTS); - LOG_DEBUG(log, "Cloning part " << fullPath(disk, src_part_path) << " to " << fullPath(disk, dst_part_path)); + LOG_DEBUG_FORMATTED(log, "Cloning part {} to {}", fullPath(disk, src_part_path), fullPath(disk, dst_part_path)); localBackup(disk, src_part_path, dst_part_path); disk->removeIfExists(dst_part_path + "/" + DELETE_ON_DESTROY_MARKER_PATH); diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 8feeeadea45..a3dc8be5023 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -268,7 +268,7 @@ bool MergeTreeDataMergerMutator::selectPartsToMerge( if (prev_part && part->info.partition_id == (*prev_part)->info.partition_id && part->info.min_block <= (*prev_part)->info.max_block) { - LOG_ERROR(log, "Part " << part->name << " intersects previous part " << (*prev_part)->name); + LOG_ERROR_FORMATTED(log, "Part {} intersects previous part {}", part->name, (*prev_part)->name); } prev_part = ∂ @@ -1000,12 +1000,12 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor if (!isStorageTouchedByMutations(storage_from_source_part, commands_for_part, context_for_reading)) { - LOG_TRACE(log, "Part " << source_part->name << " doesn't change up to mutation version " << future_part.part_info.mutation); + LOG_TRACE_FORMATTED(log, "Part {} doesn't change up to mutation version {}", source_part->name, future_part.part_info.mutation); return data.cloneAndLoadDataPartOnSameDisk(source_part, "tmp_clone_", future_part.part_info); } else { - LOG_TRACE(log, "Mutating part " << source_part->name << " to mutation version " << future_part.part_info.mutation); + LOG_TRACE_FORMATTED(log, "Mutating part {} to mutation version {}", source_part->name, future_part.part_info.mutation); } BlockInputStreamPtr in = nullptr; diff --git a/src/Storages/MergeTree/MergeTreePartsMover.cpp b/src/Storages/MergeTree/MergeTreePartsMover.cpp index d9c7492fb41..1244ae61556 100644 --- a/src/Storages/MergeTree/MergeTreePartsMover.cpp +++ b/src/Storages/MergeTree/MergeTreePartsMover.cpp @@ -199,7 +199,7 @@ MergeTreeData::DataPartPtr MergeTreePartsMover::clonePart(const MergeTreeMoveEnt auto single_disk_volume = std::make_shared("volume_" + moving_part.part->name, moving_part.reserved_space->getDisk()); MergeTreeData::MutableDataPartPtr cloned_part = data->createPart(moving_part.part->name, single_disk_volume, "detached/" + moving_part.part->name); - LOG_TRACE(log, "Part " << moving_part.part->name << " was cloned to " << cloned_part->getFullPath()); + LOG_TRACE_FORMATTED(log, "Part {} was cloned to {}", moving_part.part->name, cloned_part->getFullPath()); cloned_part->loadColumnsChecksumsIndexes(true, true); return cloned_part; @@ -228,7 +228,7 @@ void MergeTreePartsMover::swapClonedPart(const MergeTreeData::DataPartPtr & clon /// TODO what happen if server goes down here? data->swapActivePart(cloned_part); - LOG_TRACE(log, "Part " << cloned_part->name << " was moved to " << cloned_part->getFullPath()); + LOG_TRACE_FORMATTED(log, "Part {} was moved to {}", cloned_part->name, cloned_part->getFullPath()); } } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp index ae7a5c23c01..ad425ee44fa 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp @@ -298,7 +298,7 @@ CheckResult ReplicatedMergeTreePartCheckThread::checkPart(const String & part_na { /// If we have a covering part, ignore all the problems with this part. /// In the worst case, errors will still appear `old_parts_lifetime` seconds in error log until the part is removed as the old one. - LOG_WARNING(log, "We have part " << part->name << " covering part " << part_name); + LOG_WARNING_FORMATTED(log, "We have part {} covering part {}", part->name, part_name); } return {part_name, true, ""}; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 58fa6255a6d..b820b2e4745 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -1434,7 +1434,7 @@ bool ReplicatedMergeTreeQueue::tryFinalizeMutations(zkutil::ZooKeeperPtr zookeep it->second.is_done = true; if (entry->isAlterMutation()) { - LOG_TRACE(log, "Finishing data alter with version " << entry->alter_version << " for entry " << entry->znode_name); + LOG_TRACE_FORMATTED(log, "Finishing data alter with version {} for entry {}", entry->alter_version, entry->znode_name); alter_sequence.finishDataAlter(entry->alter_version, lock); } } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index e76fc8d4019..2e1230eaba4 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -215,7 +215,7 @@ bool ReplicatedMergeTreeRestartingThread::tryStartup() } catch (const Coordination::Exception & e) { - LOG_ERROR(log, "Couldn't start replication: " << e.what() << ". " << DB::getCurrentExceptionMessage(true)); + LOG_ERROR_FORMATTED(log, "Couldn't start replication: {}. {}", e.what(), DB::getCurrentExceptionMessage(true)); return false; } catch (const Exception & e) @@ -223,7 +223,7 @@ bool ReplicatedMergeTreeRestartingThread::tryStartup() if (e.code() != ErrorCodes::REPLICA_IS_ALREADY_ACTIVE) throw; - LOG_ERROR(log, "Couldn't start replication: " << e.what() << ". " << DB::getCurrentExceptionMessage(true)); + LOG_ERROR_FORMATTED(log, "Couldn't start replication: {}. {}", e.what(), DB::getCurrentExceptionMessage(true)); return false; } } diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 820d95dd54a..f83b0c61d29 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -534,7 +534,7 @@ void StorageMergeTree::loadMutations() { MergeTreeMutationEntry entry(disk, path, it->name()); Int64 block_number = entry.block_number; - LOG_DEBUG(log, "Loading mutation: " << it->name() << " entry, commands size: " << entry.commands.size()); + LOG_DEBUG_FORMATTED(log, "Loading mutation: {} entry, commands size: {}", it->name(), entry.commands.size()); auto insertion = current_mutations_by_id.emplace(it->name(), std::move(entry)); current_mutations_by_version.emplace(block_number, insertion.first->second); } @@ -1071,7 +1071,7 @@ void StorageMergeTree::attachPartition(const ASTPtr & partition, bool attach_par for (size_t i = 0; i < loaded_parts.size(); ++i) { - LOG_INFO(log, "Attaching part " << loaded_parts[i]->name << " from " << renamed_parts.old_and_new_names[i].second); + LOG_INFO_FORMATTED(log, "Attaching part {} from {}", loaded_parts[i]->name, renamed_parts.old_and_new_names[i].second); renameTempPartAndAdd(loaded_parts[i], &increment); renamed_parts.old_and_new_names[i].first.clear(); LOG_INFO_FORMATTED(log, "Finished attaching part"); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 204a71997b7..e3bf43cd9c0 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -1036,7 +1036,7 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) String replica = findReplicaHavingPart(entry.new_part_name, true); /// NOTE excessive ZK requests for same data later, may remove. if (!replica.empty()) { - LOG_DEBUG(log, "Prefer to fetch " << entry.new_part_name << " from replica " << replica); + LOG_DEBUG_FORMATTED(log, "Prefer to fetch {} from replica {}", entry.new_part_name, replica); return false; } } @@ -1151,7 +1151,7 @@ bool StorageReplicatedMergeTree::tryExecutePartMutation(const StorageReplicatedM { const String & source_part_name = entry.source_parts.at(0); const auto storage_settings_ptr = getSettings(); - LOG_TRACE(log, "Executing log entry to mutate part " << source_part_name << " to " << entry.new_part_name); + LOG_TRACE_FORMATTED(log, "Executing log entry to mutate part {} to {}", source_part_name, entry.new_part_name); DataPartPtr source_part = getActiveContainingPart(source_part_name); if (!source_part) @@ -1178,7 +1178,7 @@ bool StorageReplicatedMergeTree::tryExecutePartMutation(const StorageReplicatedM String replica = findReplicaHavingPart(entry.new_part_name, true); /// NOTE excessive ZK requests for same data later, may remove. if (!replica.empty()) { - LOG_DEBUG(log, "Prefer to fetch " << entry.new_part_name << " from replica " << replica); + LOG_DEBUG_FORMATTED(log, "Prefer to fetch {} from replica {}", entry.new_part_name, replica); return false; } } @@ -1623,7 +1623,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) auto src_part = src_data->getPartIfExists(part_desc->src_part_info, valid_states); if (!src_part) { - LOG_DEBUG(log, "There is no part " << part_desc->src_part_name << " in " << source_table_id.getNameForLogs()); + LOG_DEBUG_FORMATTED(log, "There is no part {} in {}", part_desc->src_part_name, source_table_id.getNameForLogs()); continue; } @@ -2793,7 +2793,7 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Strin currently_fetching_parts.erase(part_name); }); - LOG_DEBUG(log, "Fetching part " << part_name << " from " << source_replica_path); + LOG_DEBUG_FORMATTED(log, "Fetching part {} from {}", part_name, source_replica_path); TableStructureReadLockHolder table_lock_holder; if (!to_detached) @@ -2841,7 +2841,7 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Strin if (source_part_checksums == desired_checksums) { - LOG_TRACE(log, "Found local part " << source_part->name << " with the same checksums as " << part_name); + LOG_TRACE_FORMATTED(log, "Found local part {} with the same checksums as {}", source_part->name, part_name); part_to_clone = source_part; } } @@ -2903,7 +2903,7 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Strin for (const auto & replaced_part : replaced_parts) { - LOG_DEBUG(log, "Part " << replaced_part->name << " is rendered obsolete by fetching part " << part_name); + LOG_DEBUG_FORMATTED(log, "Part {} is rendered obsolete by fetching part {}", replaced_part->name, part_name); ProfileEvents::increment(ProfileEvents::ObsoleteReplicatedParts); } @@ -3682,7 +3682,7 @@ void StorageReplicatedMergeTree::attachPartition(const ASTPtr & partition, bool String old_name = loaded_parts[i]->name; output.writeExistingPart(loaded_parts[i]); renamed_parts.old_and_new_names[i].first.clear(); - LOG_DEBUG(log, "Attached part " << old_name << " as " << loaded_parts[i]->name); + LOG_DEBUG_FORMATTED(log, "Attached part {} as {}", old_name, loaded_parts[i]->name); } } @@ -4271,7 +4271,7 @@ void StorageReplicatedMergeTree::fetchPartition(const ASTPtr & partition, const if (from.back() == '/') from.resize(from.size() - 1); - LOG_INFO(log, "Will fetch partition " << partition_id << " from shard " << from_); + LOG_INFO_FORMATTED(log, "Will fetch partition {} from shard {}", partition_id, from_); /** Let's check that there is no such partition in the `detached` directory (where we will write the downloaded parts). * Unreliable (there is a race condition) - such a partition may appear a little later. @@ -4869,7 +4869,7 @@ void StorageReplicatedMergeTree::clearBlocksInPartition( "Error while deleting ZooKeeper path `" << path << "`: " + zkutil::ZooKeeper::error2string(rc) << ", ignoring."); } - LOG_TRACE(log, "Deleted " << to_delete_futures.size() << " deduplication block IDs in partition ID " << partition_id); + LOG_TRACE_FORMATTED(log, "Deleted {} deduplication block IDs in partition ID {}", to_delete_futures.size(), partition_id); } void StorageReplicatedMergeTree::replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, From 533f86278a7a27fc72ec415363697a597d274063 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 20:00:41 +0300 Subject: [PATCH 624/738] find {base,src,programs} -name '*.h' -or -name '*.cpp' | xargs grep -l -P 'LOG_\w+\([^,]+, "[^"]+" << [^<]+ << "[^"]+" << [^<]+ << "[^"]+"\);' | xargs sed -i -r -e 's/(LOG_\w+)\(([^,]+), "([^"]+)" << ([^<]+) << "([^"]+)" << ([^<]+) << "([^"]+)"\);/\1_FORMATTED(\2, "\3{}\5{}\7", \4, \6);/' --- programs/copier/ClusterCopier.cpp | 2 +- programs/odbc-bridge/MainHandler.cpp | 2 +- programs/server/Server.cpp | 2 +- src/Common/ShellCommand.cpp | 2 +- .../MergeSortingBlockInputStream.cpp | 2 +- src/Databases/DatabaseOrdinary.cpp | 2 +- src/IO/HTTPCommon.cpp | 2 +- src/Interpreters/DDLWorker.cpp | 4 ++-- .../MergeTree/BackgroundProcessingPool.cpp | 2 +- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 4 ++-- .../ReplicatedMergeTreeBlockOutputStream.cpp | 4 ++-- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 4 ++-- .../ReplicatedMergeTreeRestartingThread.cpp | 2 +- src/Storages/StorageBuffer.cpp | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 18 +++++++++--------- 16 files changed, 28 insertions(+), 28 deletions(-) diff --git a/programs/copier/ClusterCopier.cpp b/programs/copier/ClusterCopier.cpp index 9551408e2b7..50804d5a8a1 100644 --- a/programs/copier/ClusterCopier.cpp +++ b/programs/copier/ClusterCopier.cpp @@ -516,7 +516,7 @@ bool ClusterCopier::checkPartitionPieceIsDone(const TaskTable & task_table, cons } } - LOG_INFO(log, "Partition " << partition_name << " piece number " << toString(piece_number) << " is copied successfully"); + LOG_INFO_FORMATTED(log, "Partition {} piece number {} is copied successfully", partition_name, toString(piece_number)); return true; } diff --git a/programs/odbc-bridge/MainHandler.cpp b/programs/odbc-bridge/MainHandler.cpp index e3ee0071fa1..a430bf72be7 100644 --- a/programs/odbc-bridge/MainHandler.cpp +++ b/programs/odbc-bridge/MainHandler.cpp @@ -152,7 +152,7 @@ void ODBCHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne } std::string db_name = params.get("db_name"); std::string table_name = params.get("table_name"); - LOG_TRACE(log, "DB name: '" << db_name << "', table name: '" << table_name << "'"); + LOG_TRACE_FORMATTED(log, "DB name: '{}', table name: '{}'", db_name, table_name); auto quoting_style = IdentifierQuotingStyle::None; #if USE_ODBC diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index eb14cdad6b7..bd2dc21b322 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -362,7 +362,7 @@ int Server::main(const std::vector & /*args*/) << ". Try to specify max_open_files according to your system limits. error: " << strerror(errno)); else - LOG_DEBUG(log, "Set max number of file descriptors to " << rlim.rlim_cur << " (was " << old << ")."); + LOG_DEBUG_FORMATTED(log, "Set max number of file descriptors to {} (was {}).", rlim.rlim_cur, old); } } diff --git a/src/Common/ShellCommand.cpp b/src/Common/ShellCommand.cpp index ec536d956e5..01bcea34da1 100644 --- a/src/Common/ShellCommand.cpp +++ b/src/Common/ShellCommand.cpp @@ -53,7 +53,7 @@ ShellCommand::~ShellCommand() LOG_TRACE_FORMATTED(getLogger(), "Will kill shell command pid {} with SIGTERM", pid); int retcode = kill(pid, SIGTERM); if (retcode != 0) - LOG_WARNING(getLogger(), "Cannot kill shell command pid " << pid << " errno '" << errnoToString(retcode) << "'"); + LOG_WARNING_FORMATTED(getLogger(), "Cannot kill shell command pid {} errno '{}'", pid, errnoToString(retcode)); } else if (!wait_called) tryWait(); diff --git a/src/DataStreams/MergeSortingBlockInputStream.cpp b/src/DataStreams/MergeSortingBlockInputStream.cpp index 5e484fa4803..b40fd698836 100644 --- a/src/DataStreams/MergeSortingBlockInputStream.cpp +++ b/src/DataStreams/MergeSortingBlockInputStream.cpp @@ -246,7 +246,7 @@ Block MergeSortingBlocksBlockInputStream::mergeImpl(TSortingHeap & queue) void MergeSortingBlockInputStream::remerge() { - LOG_DEBUG(log, "Re-merging intermediate ORDER BY data (" << blocks.size() << " blocks with " << sum_rows_in_blocks << " rows) to save memory consumption"); + LOG_DEBUG_FORMATTED(log, "Re-merging intermediate ORDER BY data ({} blocks with {} rows) to save memory consumption", blocks.size(), sum_rows_in_blocks); /// NOTE Maybe concat all blocks and partial sort will be faster than merge? MergeSortingBlocksBlockInputStream merger(blocks, description, max_merged_block_size, limit); diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index 9eae75f71bb..09a4d28c6e7 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -150,7 +150,7 @@ void DatabaseOrdinary::loadStoredObjects( size_t total_tables = file_names.size() - total_dictionaries; - LOG_INFO(log, "Total " << total_tables << " tables and " << total_dictionaries << " dictionaries."); + LOG_INFO_FORMATTED(log, "Total {} tables and {} dictionaries.", total_tables, total_dictionaries); AtomicStopwatch watch; std::atomic tables_processed{0}; diff --git a/src/IO/HTTPCommon.cpp b/src/IO/HTTPCommon.cpp index 7200dfe558d..70ba323808a 100644 --- a/src/IO/HTTPCommon.cpp +++ b/src/IO/HTTPCommon.cpp @@ -173,7 +173,7 @@ namespace auto msg = Poco::AnyCast(session_data); if (!msg.empty()) { - LOG_TRACE((&Logger::get("HTTPCommon")), "Failed communicating with " << host << " with error '" << msg << "' will try to reconnect session"); + LOG_TRACE_FORMATTED((&Logger::get("HTTPCommon")), "Failed communicating with {} with error '{}' will try to reconnect session", host, msg); /// Host can change IP const auto ip = DNSResolver::instance().resolveHost(host).toString(); if (ip != session->getHost()) diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index 48b443a30f5..d50723ccc8e 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -443,7 +443,7 @@ void DDLWorker::processTasks() } else { - LOG_DEBUG(log, "Task " << task.entry_name << " (" << task.entry.query << ") has been already processed"); + LOG_DEBUG_FORMATTED(log, "Task {} ({}) has been already processed", task.entry_name, task.entry.query); } last_processed_task_name = task.entry_name; @@ -601,7 +601,7 @@ void DDLWorker::attachToThreadGroup() void DDLWorker::processTask(DDLTask & task, const ZooKeeperPtr & zookeeper) { - LOG_DEBUG(log, "Processing task " << task.entry_name << " (" << task.entry.query << ")"); + LOG_DEBUG_FORMATTED(log, "Processing task {} ({})", task.entry_name, task.entry.query); String dummy; String active_node_path = task.entry_path + "/active/" + task.host_id_str; diff --git a/src/Storages/MergeTree/BackgroundProcessingPool.cpp b/src/Storages/MergeTree/BackgroundProcessingPool.cpp index 58018324158..5b815f461bb 100644 --- a/src/Storages/MergeTree/BackgroundProcessingPool.cpp +++ b/src/Storages/MergeTree/BackgroundProcessingPool.cpp @@ -52,7 +52,7 @@ BackgroundProcessingPool::BackgroundProcessingPool(int size_, , settings(pool_settings) { logger = &Logger::get(log_name); - LOG_INFO(logger, "Create " << log_name << " with " << size << " threads"); + LOG_INFO_FORMATTED(logger, "Create {} with {} threads", log_name, size); threads.resize(size); for (auto & thread : threads) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 46cd7da87b9..c85be37f894 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -1820,7 +1820,7 @@ void MergeTreeData::renameTempPartAndReplace( else part_name = part->name; - LOG_TRACE(log, "Renaming temporary part " << part->relative_path << " to " << part_name << "."); + LOG_TRACE_FORMATTED(log, "Renaming temporary part {} to {}.", part->relative_path, part_name); auto it_duplicate = data_parts_by_info.find(part_info); if (it_duplicate != data_parts_by_info.end()) diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 49f3c97a8aa..c8018b651b6 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -786,7 +786,7 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams( column_names, MergeTreeReadPool::BackoffSettings(settings), settings.preferred_block_size_bytes, false); /// Let's estimate total number of rows for progress bar. - LOG_TRACE(log, "Reading approx. " << total_rows << " rows with " << num_streams << " streams"); + LOG_TRACE_FORMATTED(log, "Reading approx. {} rows with {} streams", total_rows, num_streams); for (size_t i = 0; i < num_streams; ++i) { @@ -1443,7 +1443,7 @@ MarkRanges MergeTreeDataSelectExecutor::filterMarksUsingIndex( last_index_mark = index_range.end - 1; } - LOG_DEBUG(log, "Index " << backQuote(index->name) << " has dropped " << granules_dropped << " granules."); + LOG_DEBUG_FORMATTED(log, "Index {} has dropped {} granules.", backQuote(index->name), granules_dropped); return res; } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp index 82d7c079020..709976005f2 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp @@ -147,7 +147,7 @@ void ReplicatedMergeTreeBlockOutputStream::write(const Block & block) /// That is, do not insert the same data to the same partition twice. block_id = part->info.partition_id + "_" + toString(hash_value.words[0]) + "_" + toString(hash_value.words[1]); - LOG_DEBUG(log, "Wrote block with ID '" << block_id << "', " << current_block.block.rows() << " rows"); + LOG_DEBUG_FORMATTED(log, "Wrote block with ID '{}', {} rows", block_id, current_block.block.rows()); } else { @@ -329,7 +329,7 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zoo if (multi_code == Coordination::ZNODEEXISTS && deduplicate_block && failed_op_path == block_id_path) { /// Block with the same id have just appeared in table (or other replica), rollback thee insertion. - LOG_INFO(log, "Block with ID " << block_id << " already exists; ignoring it (removing part " << part->name << ")"); + LOG_INFO_FORMATTED(log, "Block with ID {} already exists; ignoring it (removing part {})", block_id, part->name); part->is_duplicate = true; transaction.rollback(); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index b820b2e4745..5e9cb06fdb8 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -706,7 +706,7 @@ void ReplicatedMergeTreeQueue::updateMutations(zkutil::ZooKeeperPtr zookeeper, C /// otherwise it's already done if (entry->isAlterMutation() && entry->znode_name > mutation_pointer) { - LOG_TRACE(log, "Adding mutation " << entry->znode_name << " with alter version " << entry->alter_version << " to the queue"); + LOG_TRACE_FORMATTED(log, "Adding mutation {} with alter version {} to the queue", entry->znode_name, entry->alter_version); alter_sequence.addMutationForAlter(entry->alter_version, state_lock); } } @@ -1385,7 +1385,7 @@ bool ReplicatedMergeTreeQueue::tryFinalizeMutations(zkutil::ZooKeeperPtr zookeep if (znode <= mutation_pointer) { - LOG_TRACE(log, "Marking mutation " << znode << " done because it is <= mutation_pointer (" << mutation_pointer << ")"); + LOG_TRACE_FORMATTED(log, "Marking mutation {} done because it is <= mutation_pointer ({})", znode, mutation_pointer); mutation.is_done = true; alter_sequence.finishDataAlter(mutation.entry->alter_version, lock); if (mutation.parts_to_do.size() != 0) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index 2e1230eaba4..5e47cd9739d 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -133,7 +133,7 @@ void ReplicatedMergeTreeRestartingThread::run() storage.getReplicaDelays(absolute_delay, relative_delay); if (absolute_delay) - LOG_TRACE(log, "Absolute delay: " << absolute_delay << ". Relative delay: " << relative_delay << "."); + LOG_TRACE_FORMATTED(log, "Absolute delay: {}. Relative delay: {}.", absolute_delay, relative_delay); prev_time_of_check_delay = current_time; diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index c515b845a10..960cb1ea4d2 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -350,7 +350,7 @@ public: { if (storage.destination_id) { - LOG_TRACE(storage.log, "Writing block with " << rows << " rows, " << bytes << " bytes directly."); + LOG_TRACE_FORMATTED(storage.log, "Writing block with {} rows, {} bytes directly.", rows, bytes); storage.writeBlockToDestination(block, destination); } return; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index e3bf43cd9c0..27361651d67 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -1631,7 +1631,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) if (checksum_hex != part_desc->checksum_hex) { - LOG_DEBUG(log, "Part " << part_desc->src_part_name << " of " << source_table_id.getNameForLogs() << " has inappropriate checksum"); + LOG_DEBUG_FORMATTED(log, "Part {} of {} has inappropriate checksum", part_desc->src_part_name, source_table_id.getNameForLogs()); /// TODO: check version continue; } @@ -1647,7 +1647,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) }; size_t num_clonable_parts = clone_data_parts_from_source_table(); - LOG_DEBUG(log, "Found " << num_clonable_parts << " parts that could be cloned (of " << parts_to_add.size() << " required parts)"); + LOG_DEBUG_FORMATTED(log, "Found {} parts that could be cloned (of {} required parts)", num_clonable_parts, parts_to_add.size()); ActiveDataPartSet adding_parts_active_set(format_version); std::unordered_map part_name_to_desc; @@ -3891,7 +3891,7 @@ bool StorageReplicatedMergeTree::waitForReplicaToProcessLogEntry(const String & UInt64 log_index = parse(entry.znode_name.substr(entry.znode_name.size() - 10)); log_node_name = entry.znode_name; - LOG_DEBUG(log, "Waiting for " << replica << " to pull " << log_node_name << " to queue"); + LOG_DEBUG_FORMATTED(log, "Waiting for {} to pull {} to queue", replica, log_node_name); /// Let's wait until entry gets into the replica queue. while (wait_for_non_active || !check_replica_become_inactive()) @@ -3939,7 +3939,7 @@ bool StorageReplicatedMergeTree::waitForReplicaToProcessLogEntry(const String & if (found) { - LOG_DEBUG(log, "Waiting for " << replica << " to pull " << log_node_name << " to queue"); + LOG_DEBUG_FORMATTED(log, "Waiting for {} to pull {} to queue", replica, log_node_name); /// Let's wait until the entry gets into the replica queue. while (wait_for_non_active || !check_replica_become_inactive()) @@ -3961,7 +3961,7 @@ bool StorageReplicatedMergeTree::waitForReplicaToProcessLogEntry(const String & throw Exception("Logical error: unexpected name of log node: " + entry.znode_name, ErrorCodes::LOGICAL_ERROR); if (!log_node_name.empty()) - LOG_DEBUG(log, "Looking for node corresponding to " << log_node_name << " in " << replica << " queue"); + LOG_DEBUG_FORMATTED(log, "Looking for node corresponding to {} in {} queue", log_node_name, replica); else LOG_DEBUG_FORMATTED(log, "Looking for corresponding node in {} queue", replica); @@ -3991,7 +3991,7 @@ bool StorageReplicatedMergeTree::waitForReplicaToProcessLogEntry(const String & return true; } - LOG_DEBUG(log, "Waiting for " << queue_entry_to_wait_for << " to disappear from " << replica << " queue"); + LOG_DEBUG_FORMATTED(log, "Waiting for {} to disappear from {} queue", queue_entry_to_wait_for, replica); /// Third - wait until the entry disappears from the replica queue or replica become inactive. String path_to_wait_on = zookeeper_path + "/replicas/" + replica + "/queue/" + queue_entry_to_wait_for; @@ -4393,7 +4393,7 @@ void StorageReplicatedMergeTree::fetchPartition(const ASTPtr & partition, const if (!containing_part.empty()) parts_to_fetch.push_back(containing_part); else - LOG_WARNING(log, "Part " << missing_part << " on replica " << best_replica_path << " has been vanished."); + LOG_WARNING_FORMATTED(log, "Part {} on replica {} has been vanished.", missing_part, best_replica_path); } } @@ -4941,7 +4941,7 @@ void StorageReplicatedMergeTree::replacePartitionFrom(const StoragePtr & source_ auto lock = allocateBlockNumber(partition_id, zookeeper, block_id_path); if (!lock) { - LOG_INFO(log, "Part " << src_part->name << " (hash " << hash_hex << ") has been already attached"); + LOG_INFO_FORMATTED(log, "Part {} (hash {}) has been already attached", src_part->name, hash_hex); continue; } @@ -5116,7 +5116,7 @@ void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_ta auto lock = dest_table_storage->allocateBlockNumber(partition_id, zookeeper, block_id_path); if (!lock) { - LOG_INFO(log, "Part " << src_part->name << " (hash " << hash_hex << ") has been already attached"); + LOG_INFO_FORMATTED(log, "Part {} (hash {}) has been already attached", src_part->name, hash_hex); continue; } From f68d1ceb4f00b4518af2efbd9623eb8d359a3603 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 20:02:09 +0300 Subject: [PATCH 625/738] find {base,src,programs} -name '*.h' -or -name '*.cpp' | xargs grep -l -P 'LOG_\w+\([^,]+, "[^"]+" << [^<]+ << "[^"]+" << [^<]+ << "[^"]+" << [^<]+\);' | xargs sed -i -r -e 's/(LOG_\w+)\(([^,]+), "([^"]+)" << ([^<]+) << "([^"]+)" << ([^<]+) << "([^"]+)" << ([^<]+)\);/\1_FORMATTED(\2, "\3{}\5{}\7{}", \4, \6, \8);/' --- src/Access/ContextAccess.cpp | 2 +- src/Disks/S3/ProxyResolverConfiguration.cpp | 2 +- src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp | 6 +++--- src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp | 4 ++-- .../MergeTree/ReplicatedMergeTreePartCheckThread.cpp | 2 +- src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index a8e22180490..40eadd1bee0 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -459,7 +459,7 @@ boost::shared_ptr ContextAccess::calculateResultAccess(bool "Current_roles: " << boost::algorithm::join(roles_info->getCurrentRolesNames(), ", ") << ", enabled_roles: " << boost::algorithm::join(roles_info->getEnabledRolesNames(), ", ")); } - LOG_TRACE(trace_log, "Settings: readonly=" << readonly_ << ", allow_ddl=" << allow_ddl_ << ", allow_introspection_functions=" << allow_introspection_); + LOG_TRACE_FORMATTED(trace_log, "Settings: readonly={}, allow_ddl={}, allow_introspection_functions={}", readonly_, allow_ddl_, allow_introspection_); } res = std::move(merged_access); diff --git a/src/Disks/S3/ProxyResolverConfiguration.cpp b/src/Disks/S3/ProxyResolverConfiguration.cpp index 202c10aa7e4..f5acdab369c 100644 --- a/src/Disks/S3/ProxyResolverConfiguration.cpp +++ b/src/Disks/S3/ProxyResolverConfiguration.cpp @@ -49,7 +49,7 @@ Aws::Client::ClientConfigurationPerRequest ProxyResolverConfiguration::getConfig /// Read proxy host as string from response body. Poco::StreamCopier::copyToString(response_body_stream, proxy_host); - LOG_DEBUG(&Logger::get("AWSClient"), "Use proxy: " << proxy_scheme << "://" << proxy_host << ":" << proxy_port); + LOG_DEBUG_FORMATTED(&Logger::get("AWSClient"), "Use proxy: {}://{}:{}", proxy_scheme, proxy_host, proxy_port); cfg.proxyScheme = Aws::Http::SchemeMapper::FromString(proxy_scheme.c_str()); cfg.proxyHost = proxy_host; diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index a3dc8be5023..76ce3daf820 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -315,7 +315,7 @@ bool MergeTreeDataMergerMutator::selectPartsToMerge( parts.push_back(part); } - LOG_DEBUG(log, "Selected " << parts.size() << " parts from " << parts.front()->name << " to " << parts.back()->name); + LOG_DEBUG_FORMATTED(log, "Selected {} parts from {} to {}", parts.size(), parts.front()->name, parts.back()->name); future_part.assign(std::move(parts)); return true; } @@ -381,7 +381,7 @@ bool MergeTreeDataMergerMutator::selectAllPartsToMergeWithinPartition( return false; } - LOG_DEBUG(log, "Selected " << parts.size() << " parts from " << parts.front()->name << " to " << parts.back()->name); + LOG_DEBUG_FORMATTED(log, "Selected {} parts from {} to {}", parts.size(), parts.front()->name, parts.back()->name); future_part.assign(std::move(parts)); available_disk_space -= required_disk_space; return true; @@ -1238,7 +1238,7 @@ MergeTreeData::DataPartPtr MergeTreeDataMergerMutator::renameMergedTemporaryPart + " instead of " + parts[i]->name, ErrorCodes::LOGICAL_ERROR); } - LOG_TRACE(log, "Merged " << parts.size() << " parts: from " << parts.front()->name << " to " << parts.back()->name); + LOG_TRACE_FORMATTED(log, "Merged {} parts: from {} to {}", parts.size(), parts.front()->name, parts.back()->name); return new_data_part; } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp index c11b7444dcf..e0e131dc78b 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp @@ -234,7 +234,7 @@ void ReplicatedMergeTreeCleanupThread::clearOldLogs() } } - LOG_DEBUG(log, "Removed " << entries.size() << " old log entries: " << entries.front() << " - " << entries.back()); + LOG_DEBUG_FORMATTED(log, "Removed {} old log entries: {} - {}", entries.size(), entries.front(), entries.back()); } @@ -456,7 +456,7 @@ void ReplicatedMergeTreeCleanupThread::clearOldMutations() /// Simultaneously with clearing the log, we check to see if replica was added since we received replicas list. ops.emplace_back(zkutil::makeCheckRequest(storage.zookeeper_path + "/replicas", replicas_stat.version)); zookeeper->multi(ops); - LOG_DEBUG(log, "Removed " << (i + 1 - batch_start_i) << " old mutation entries: " << entries[batch_start_i] << " - " << entries[i]); + LOG_DEBUG_FORMATTED(log, "Removed {} old mutation entries: {} - {}", (i + 1 - batch_start_i), entries[batch_start_i], entries[i]); batch_start_i = i + 1; ops.clear(); } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp index ad425ee44fa..345f6a37cf8 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp @@ -131,7 +131,7 @@ void ReplicatedMergeTreePartCheckThread::searchForMissingPart(const String & par if (part_on_replica_info.contains(part_info)) { - LOG_WARNING(log, "Found part " << part_on_replica << " on " << replica << " that covers the missing part " << part_name); + LOG_WARNING_FORMATTED(log, "Found part {} on {} that covers the missing part {}", part_on_replica, replica, part_name); return; } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 5e9cb06fdb8..ffd9a8f0a74 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -486,7 +486,7 @@ void ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper, C UInt64 last_entry_index = parse(last_entry.substr(strlen("log-"))); - LOG_DEBUG(log, "Pulling " << (end - begin) << " entries to queue: " << *begin << " - " << *last); + LOG_DEBUG_FORMATTED(log, "Pulling {} entries to queue: {} - {}", (end - begin), *begin, *last); zkutil::AsyncResponses futures; futures.reserve(end - begin); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 27361651d67..a520ebe15ce 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -2564,7 +2564,7 @@ String StorageReplicatedMergeTree::findReplicaHavingCoveringPart(LogEntry & entr String reject_reason; if (!queue.addFuturePartIfNotCoveredByThem(largest_part_found, entry, reject_reason)) { - LOG_INFO(log, "Will not fetch part " << largest_part_found << " covering " << entry.new_part_name << ". " << reject_reason); + LOG_INFO_FORMATTED(log, "Will not fetch part {} covering {}. {}", largest_part_found, entry.new_part_name, reject_reason); return {}; } } From f69cbdcbfc77a6fae760460fb126abbd557e0be9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 20:09:37 +0300 Subject: [PATCH 626/738] find {base,src,programs} -name '*.h' -or -name '*.cpp' | xargs grep -l -P 'LOG_\w+\([^,]+, "[^"]+" \+ [^+]+\);' | xargs sed -i -r -e 's/(LOG_\w+)\(([^,]+), "([^"]+)" \+ ([^+]+)\);/\1_FORMATTED(\2, "\3{}", \4);/' --- base/daemon/BaseDaemon.cpp | 2 +- programs/odbc-bridge/ColumnInfoHandler.cpp | 2 +- programs/odbc-bridge/HandlerFactory.cpp | 2 +- .../odbc-bridge/IdentifierQuoteHandler.cpp | 2 +- programs/odbc-bridge/MainHandler.cpp | 2 +- programs/odbc-bridge/ODBCBridge.cpp | 2 +- programs/server/Server.cpp | 20 +++++++++---------- src/Access/DiskAccessStorage.cpp | 2 +- src/Common/QueryProfiler.cpp | 4 ++-- src/Common/XDBCBridgeHelper.h | 2 +- .../MergeSortingBlockInputStream.cpp | 4 ++-- .../ExecutableDictionarySource.cpp | 4 ++-- src/Dictionaries/FileDictionarySource.cpp | 2 +- src/Dictionaries/HTTPDictionarySource.cpp | 4 ++-- src/Dictionaries/LibraryDictionarySource.cpp | 2 +- src/Interpreters/DatabaseCatalog.cpp | 2 +- .../Transforms/MergeSortingTransform.cpp | 4 ++-- src/Storages/MergeTree/MergeTreeData.cpp | 4 ++-- .../MergeTree/MergeTreeDataWriter.cpp | 2 +- 19 files changed, 34 insertions(+), 34 deletions(-) diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index 18773a3ff5e..c48e92a1b7b 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -502,7 +502,7 @@ void debugIncreaseOOMScore() e.displayText() + "'."); return; } - LOG_INFO(&Logger::root(), "Set OOM score adjustment to " + new_score); + LOG_INFO_FORMATTED(&Logger::root(), "Set OOM score adjustment to {}", new_score); } #else void debugIncreaseOOMScore() {} diff --git a/programs/odbc-bridge/ColumnInfoHandler.cpp b/programs/odbc-bridge/ColumnInfoHandler.cpp index d3badc23f38..4bc3522b220 100644 --- a/programs/odbc-bridge/ColumnInfoHandler.cpp +++ b/programs/odbc-bridge/ColumnInfoHandler.cpp @@ -62,7 +62,7 @@ namespace void ODBCColumnsInfoHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) { Poco::Net::HTMLForm params(request, request.stream()); - LOG_TRACE(log, "Request URI: " + request.getURI()); + LOG_TRACE_FORMATTED(log, "Request URI: {}", request.getURI()); auto process_error = [&response, this](const std::string & message) { diff --git a/programs/odbc-bridge/HandlerFactory.cpp b/programs/odbc-bridge/HandlerFactory.cpp index 156c2fcce61..ebf37925d0d 100644 --- a/programs/odbc-bridge/HandlerFactory.cpp +++ b/programs/odbc-bridge/HandlerFactory.cpp @@ -10,7 +10,7 @@ namespace DB Poco::Net::HTTPRequestHandler * HandlerFactory::createRequestHandler(const Poco::Net::HTTPServerRequest & request) { Poco::URI uri{request.getURI()}; - LOG_TRACE(log, "Request URI: " + uri.toString()); + LOG_TRACE_FORMATTED(log, "Request URI: {}", uri.toString()); if (uri.getPath() == "/ping" && request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET) return new PingHandler(keep_alive_timeout); diff --git a/programs/odbc-bridge/IdentifierQuoteHandler.cpp b/programs/odbc-bridge/IdentifierQuoteHandler.cpp index f89cea5ab7e..83768a3d941 100644 --- a/programs/odbc-bridge/IdentifierQuoteHandler.cpp +++ b/programs/odbc-bridge/IdentifierQuoteHandler.cpp @@ -25,7 +25,7 @@ namespace DB void IdentifierQuoteHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) { Poco::Net::HTMLForm params(request, request.stream()); - LOG_TRACE(log, "Request URI: " + request.getURI()); + LOG_TRACE_FORMATTED(log, "Request URI: {}", request.getURI()); auto process_error = [&response, this](const std::string & message) { diff --git a/programs/odbc-bridge/MainHandler.cpp b/programs/odbc-bridge/MainHandler.cpp index a430bf72be7..b46a03472ab 100644 --- a/programs/odbc-bridge/MainHandler.cpp +++ b/programs/odbc-bridge/MainHandler.cpp @@ -84,7 +84,7 @@ void ODBCHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne Poco::Net::HTMLForm params(request); if (mode == "read") params.read(request.stream()); - LOG_TRACE(log, "Request URI: " + request.getURI()); + LOG_TRACE_FORMATTED(log, "Request URI: {}", request.getURI()); if (mode == "read" && !params.has("query")) { diff --git a/programs/odbc-bridge/ODBCBridge.cpp b/programs/odbc-bridge/ODBCBridge.cpp index e707606acac..991eda151cf 100644 --- a/programs/odbc-bridge/ODBCBridge.cpp +++ b/programs/odbc-bridge/ODBCBridge.cpp @@ -188,7 +188,7 @@ int ODBCBridge::main(const std::vector & /*args*/) new HandlerFactory("ODBCRequestHandlerFactory-factory", keep_alive_timeout, context), server_pool, socket, http_params); server.start(); - LOG_INFO(log, "Listening http://" + address.toString()); + LOG_INFO_FORMATTED(log, "Listening http://{}", address.toString()); SCOPE_EXIT({ LOG_DEBUG_FORMATTED(log, "Received termination signal."); diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index bd2dc21b322..517a32f8d91 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -276,7 +276,7 @@ int Server::main(const std::vector & /*args*/) { LOG_TRACE_FORMATTED(log, "Will mlockall to prevent executable memory from being paged out. It may take a few seconds."); if (0 != mlockall(MCL_CURRENT)) - LOG_WARNING(log, "Failed mlockall: " + errnoToString(ErrorCodes::SYSTEM_ERROR)); + LOG_WARNING_FORMATTED(log, "Failed mlockall: {}", errnoToString(ErrorCodes::SYSTEM_ERROR)); else LOG_TRACE_FORMATTED(log, "The memory map of clickhouse executable has been mlock'ed"); } @@ -592,7 +592,7 @@ int Server::main(const std::vector & /*args*/) total_memory_tracker.setDescription("(total)"); total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking); - LOG_INFO(log, "Loading metadata from " + path); + LOG_INFO_FORMATTED(log, "Loading metadata from {}", path); try { @@ -826,7 +826,7 @@ int Server::main(const std::vector & /*args*/) servers.emplace_back(std::make_unique( createHandlerFactory(*this, async_metrics, "HTTPHandler-factory"), server_pool, socket, http_params)); - LOG_INFO(log, "Listening for http://" + address.toString()); + LOG_INFO_FORMATTED(log, "Listening for http://{}", address.toString()); }); /// HTTPS @@ -840,7 +840,7 @@ int Server::main(const std::vector & /*args*/) servers.emplace_back(std::make_unique( createHandlerFactory(*this, async_metrics, "HTTPSHandler-factory"), server_pool, socket, http_params)); - LOG_INFO(log, "Listening for https://" + address.toString()); + LOG_INFO_FORMATTED(log, "Listening for https://{}", address.toString()); #else UNUSED(port); throw Exception{"HTTPS protocol is disabled because Poco library was built without NetSSL support.", @@ -861,7 +861,7 @@ int Server::main(const std::vector & /*args*/) socket, new Poco::Net::TCPServerParams)); - LOG_INFO(log, "Listening for connections with native protocol (tcp): " + address.toString()); + LOG_INFO_FORMATTED(log, "Listening for connections with native protocol (tcp): {}", address.toString()); }); /// TCP with SSL @@ -877,7 +877,7 @@ int Server::main(const std::vector & /*args*/) server_pool, socket, new Poco::Net::TCPServerParams)); - LOG_INFO(log, "Listening for connections with secure native protocol (tcp_secure): " + address.toString()); + LOG_INFO_FORMATTED(log, "Listening for connections with secure native protocol (tcp_secure): {}", address.toString()); #else UNUSED(port); throw Exception{"SSL support for TCP protocol is disabled because Poco library was built without NetSSL support.", @@ -895,7 +895,7 @@ int Server::main(const std::vector & /*args*/) servers.emplace_back(std::make_unique( createHandlerFactory(*this, async_metrics, "InterserverIOHTTPHandler-factory"), server_pool, socket, http_params)); - LOG_INFO(log, "Listening for replica communication (interserver): http://" + address.toString()); + LOG_INFO_FORMATTED(log, "Listening for replica communication (interserver): http://{}", address.toString()); }); create_server("interserver_https_port", [&](UInt16 port) @@ -908,7 +908,7 @@ int Server::main(const std::vector & /*args*/) servers.emplace_back(std::make_unique( createHandlerFactory(*this, async_metrics, "InterserverIOHTTPSHandler-factory"), server_pool, socket, http_params)); - LOG_INFO(log, "Listening for secure replica communication (interserver): https://" + address.toString()); + LOG_INFO_FORMATTED(log, "Listening for secure replica communication (interserver): https://{}", address.toString()); #else UNUSED(port); throw Exception{"SSL support for TCP protocol is disabled because Poco library was built without NetSSL support.", @@ -928,7 +928,7 @@ int Server::main(const std::vector & /*args*/) socket, new Poco::Net::TCPServerParams)); - LOG_INFO(log, "Listening for MySQL compatibility protocol: " + address.toString()); + LOG_INFO_FORMATTED(log, "Listening for MySQL compatibility protocol: {}", address.toString()); }); /// Prometheus (if defined and not setup yet with http_port) @@ -941,7 +941,7 @@ int Server::main(const std::vector & /*args*/) servers.emplace_back(std::make_unique( createHandlerFactory(*this, async_metrics, "PrometheusHandler-factory"), server_pool, socket, http_params)); - LOG_INFO(log, "Listening for Prometheus: http://" + address.toString()); + LOG_INFO_FORMATTED(log, "Listening for Prometheus: http://{}", address.toString()); }); } diff --git a/src/Access/DiskAccessStorage.cpp b/src/Access/DiskAccessStorage.cpp index bc3e35a4fc7..a49751c18b8 100644 --- a/src/Access/DiskAccessStorage.cpp +++ b/src/Access/DiskAccessStorage.cpp @@ -496,7 +496,7 @@ void DiskAccessStorage::listsWritingThreadFunc() /// and then saves the files "users.list", "roles.list", etc. to the same directory. bool DiskAccessStorage::rebuildLists() { - LOG_WARNING(getLogger(), "Recovering lists in directory " + directory_path); + LOG_WARNING_FORMATTED(getLogger(), "Recovering lists in directory {}", directory_path); clear(); for (const auto & directory_entry : std::filesystem::directory_iterator(directory_path)) diff --git a/src/Common/QueryProfiler.cpp b/src/Common/QueryProfiler.cpp index a78c2b2ea6c..b1306f2a936 100644 --- a/src/Common/QueryProfiler.cpp +++ b/src/Common/QueryProfiler.cpp @@ -165,10 +165,10 @@ void QueryProfilerBase::tryCleanup() { #if USE_UNWIND if (timer_id != nullptr && timer_delete(timer_id)) - LOG_ERROR(log, "Failed to delete query profiler timer " + errnoToString(ErrorCodes::CANNOT_DELETE_TIMER)); + LOG_ERROR_FORMATTED(log, "Failed to delete query profiler timer {}", errnoToString(ErrorCodes::CANNOT_DELETE_TIMER)); if (previous_handler != nullptr && sigaction(pause_signal, previous_handler, nullptr)) - LOG_ERROR(log, "Failed to restore signal handler after query profiler " + errnoToString(ErrorCodes::CANNOT_SET_SIGNAL_HANDLER)); + LOG_ERROR_FORMATTED(log, "Failed to restore signal handler after query profiler {}", errnoToString(ErrorCodes::CANNOT_SET_SIGNAL_HANDLER)); #endif } diff --git a/src/Common/XDBCBridgeHelper.h b/src/Common/XDBCBridgeHelper.h index a3c538cb065..975ee498ae1 100644 --- a/src/Common/XDBCBridgeHelper.h +++ b/src/Common/XDBCBridgeHelper.h @@ -309,7 +309,7 @@ struct ODBCBridgeMixin cmd_args.push_back(config.getString("logger." + configPrefix() + "_level")); } - LOG_TRACE(log, "Starting " + serviceAlias()); + LOG_TRACE_FORMATTED(log, "Starting {}", serviceAlias()); return ShellCommand::executeDirect(path.toString(), cmd_args, true); } diff --git a/src/DataStreams/MergeSortingBlockInputStream.cpp b/src/DataStreams/MergeSortingBlockInputStream.cpp index b40fd698836..8e1a1d0719e 100644 --- a/src/DataStreams/MergeSortingBlockInputStream.cpp +++ b/src/DataStreams/MergeSortingBlockInputStream.cpp @@ -95,10 +95,10 @@ Block MergeSortingBlockInputStream::readImpl() const std::string & path = temporary_files.back()->path(); MergeSortingBlocksBlockInputStream block_in(blocks, description, max_merged_block_size, limit); - LOG_INFO(log, "Sorting and writing part of data into temporary file " + path); + LOG_INFO_FORMATTED(log, "Sorting and writing part of data into temporary file {}", path); ProfileEvents::increment(ProfileEvents::ExternalSortWritePart); TemporaryFileStream::write(path, header_without_constants, block_in, &is_cancelled, codec); /// NOTE. Possibly limit disk usage. - LOG_INFO(log, "Done writing part of data into temporary file " + path); + LOG_INFO_FORMATTED(log, "Done writing part of data into temporary file {}", path); blocks.clear(); sum_bytes_in_blocks = 0; diff --git a/src/Dictionaries/ExecutableDictionarySource.cpp b/src/Dictionaries/ExecutableDictionarySource.cpp index 6d392cd2b74..56cdce58c90 100644 --- a/src/Dictionaries/ExecutableDictionarySource.cpp +++ b/src/Dictionaries/ExecutableDictionarySource.cpp @@ -77,7 +77,7 @@ ExecutableDictionarySource::ExecutableDictionarySource(const ExecutableDictionar BlockInputStreamPtr ExecutableDictionarySource::loadAll() { - LOG_TRACE(log, "loadAll " + toString()); + LOG_TRACE_FORMATTED(log, "loadAll {}", toString()); auto process = ShellCommand::execute(command); auto input_stream = context.getInputFormat(format, process->out, sample_block, max_block_size); return std::make_shared(input_stream, std::move(process)); @@ -92,7 +92,7 @@ BlockInputStreamPtr ExecutableDictionarySource::loadUpdatedAll() if (update_time) command_with_update_field += " " + update_field + " " + DB::toString(LocalDateTime(update_time - 1)); - LOG_TRACE(log, "loadUpdatedAll " + command_with_update_field); + LOG_TRACE_FORMATTED(log, "loadUpdatedAll {}", command_with_update_field); auto process = ShellCommand::execute(command_with_update_field); auto input_stream = context.getInputFormat(format, process->out, sample_block, max_block_size); return std::make_shared(input_stream, std::move(process)); diff --git a/src/Dictionaries/FileDictionarySource.cpp b/src/Dictionaries/FileDictionarySource.cpp index 0f20ab8edc4..81578ca3050 100644 --- a/src/Dictionaries/FileDictionarySource.cpp +++ b/src/Dictionaries/FileDictionarySource.cpp @@ -49,7 +49,7 @@ FileDictionarySource::FileDictionarySource(const FileDictionarySource & other) BlockInputStreamPtr FileDictionarySource::loadAll() { - LOG_TRACE(&Poco::Logger::get("FileDictionary"), "loadAll " + toString()); + LOG_TRACE_FORMATTED(&Poco::Logger::get("FileDictionary"), "loadAll {}", toString()); auto in_ptr = std::make_unique(filepath); auto stream = context.getInputFormat(format, *in_ptr, sample_block, max_block_size); last_modification = getLastModification(); diff --git a/src/Dictionaries/HTTPDictionarySource.cpp b/src/Dictionaries/HTTPDictionarySource.cpp index f0847af2e8c..722d25e0d95 100644 --- a/src/Dictionaries/HTTPDictionarySource.cpp +++ b/src/Dictionaries/HTTPDictionarySource.cpp @@ -105,7 +105,7 @@ void HTTPDictionarySource::getUpdateFieldAndDate(Poco::URI & uri) BlockInputStreamPtr HTTPDictionarySource::loadAll() { - LOG_TRACE(log, "loadAll " + toString()); + LOG_TRACE_FORMATTED(log, "loadAll {}", toString()); Poco::URI uri(url); auto in_ptr = std::make_unique( uri, Poco::Net::HTTPRequest::HTTP_GET, ReadWriteBufferFromHTTP::OutStreamCallback(), timeouts, @@ -118,7 +118,7 @@ BlockInputStreamPtr HTTPDictionarySource::loadUpdatedAll() { Poco::URI uri(url); getUpdateFieldAndDate(uri); - LOG_TRACE(log, "loadUpdatedAll " + uri.toString()); + LOG_TRACE_FORMATTED(log, "loadUpdatedAll {}", uri.toString()); auto in_ptr = std::make_unique( uri, Poco::Net::HTTPRequest::HTTP_GET, ReadWriteBufferFromHTTP::OutStreamCallback(), timeouts, 0, credentials, DBMS_DEFAULT_BUFFER_SIZE, header_entries); diff --git a/src/Dictionaries/LibraryDictionarySource.cpp b/src/Dictionaries/LibraryDictionarySource.cpp index 6f887ee29e3..11b453ce745 100644 --- a/src/Dictionaries/LibraryDictionarySource.cpp +++ b/src/Dictionaries/LibraryDictionarySource.cpp @@ -182,7 +182,7 @@ LibraryDictionarySource::~LibraryDictionarySource() BlockInputStreamPtr LibraryDictionarySource::loadAll() { - LOG_TRACE(log, "loadAll " + toString()); + LOG_TRACE_FORMATTED(log, "loadAll {}", toString()); auto columns_holder = std::make_unique(dict_struct.attributes.size()); ClickHouseLibrary::CStrings columns{static_cast(columns_holder.get()), diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 4180690ac86..a159c93e881 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -635,7 +635,7 @@ void DatabaseCatalog::dropTableDataTask() if (it != tables_marked_dropped.end()) { table = std::move(*it); - LOG_INFO(log, "Will try drop " + table.table_id.getNameForLogs()); + LOG_INFO_FORMATTED(log, "Will try drop {}", table.table_id.getNameForLogs()); tables_marked_dropped.erase(it); } need_reschedule = !tables_marked_dropped.empty(); diff --git a/src/Processors/Transforms/MergeSortingTransform.cpp b/src/Processors/Transforms/MergeSortingTransform.cpp index 71c75d5386e..f432948e8e0 100644 --- a/src/Processors/Transforms/MergeSortingTransform.cpp +++ b/src/Processors/Transforms/MergeSortingTransform.cpp @@ -36,7 +36,7 @@ public: , path(std::move(path_)), file_buf_out(path), compressed_buf_out(file_buf_out) , out_stream(std::make_shared(compressed_buf_out, 0, header)) { - LOG_INFO(log, "Sorting and writing part of data into temporary file " + path); + LOG_INFO_FORMATTED(log, "Sorting and writing part of data into temporary file {}", path); ProfileEvents::increment(ProfileEvents::ExternalSortWritePart); out_stream->writePrefix(); } @@ -55,7 +55,7 @@ public: out_stream->writeSuffix(); compressed_buf_out.next(); file_buf_out.next(); - LOG_INFO(log, "Done writing part of data into temporary file " + path); + LOG_INFO_FORMATTED(log, "Done writing part of data into temporary file {}", path); out_stream.reset(); diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index c85be37f894..02d4d4aa370 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -2477,9 +2477,9 @@ void MergeTreeData::freezePartition(const ASTPtr & partition_ast, const String & partition_id = getPartitionIDFromQuery(partition_ast, context); if (prefix) - LOG_DEBUG(log, "Freezing parts with prefix " + *prefix); + LOG_DEBUG_FORMATTED(log, "Freezing parts with prefix {}", *prefix); else - LOG_DEBUG(log, "Freezing parts with partition ID " + partition_id); + LOG_DEBUG_FORMATTED(log, "Freezing parts with partition ID {}", partition_id); freezePartitionsByMatcher( diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 75d6ebe7cb7..98ca6613134 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -252,7 +252,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa if (new_data_part->volume->getDisk()->exists(full_path)) { - LOG_WARNING(log, "Removing old temporary directory " + fullPath(new_data_part->volume->getDisk(), full_path)); + LOG_WARNING_FORMATTED(log, "Removing old temporary directory {}", fullPath(new_data_part->volume->getDisk(), full_path)); new_data_part->volume->getDisk()->removeRecursive(full_path); } From 241f8c54316d7abc15ec675e4d875b307fd9a2fe Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 20:10:21 +0300 Subject: [PATCH 627/738] find {base,src,programs} -name '*.h' -or -name '*.cpp' | xargs grep -l -P 'LOG_\w+\([^,]+, "[^"]+" \+ [^+]+ \+ "[^"]+"\);' | xargs sed -i -r -e 's/(LOG_\w+)\(([^,]+), "([^"]+)" \+ ([^+]+) \+ "([^"]+)"\);/\1_FORMATTED(\2, "\3{}\5", \4);/' --- src/Access/DiskAccessStorage.cpp | 2 +- src/Common/Config/ConfigProcessor.cpp | 6 +++--- src/Common/Exception.cpp | 2 +- src/Disks/DiskLocal.cpp | 4 ++-- src/Disks/S3/DiskS3.cpp | 4 ++-- src/Interpreters/DDLWorker.cpp | 2 +- src/Interpreters/ExternalLoader.cpp | 2 +- src/Interpreters/InterpreterSystemQuery.cpp | 4 ++-- src/Interpreters/SystemLog.h | 2 +- src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp | 12 ++++++------ 10 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/Access/DiskAccessStorage.cpp b/src/Access/DiskAccessStorage.cpp index a49751c18b8..00da0dfe2ae 100644 --- a/src/Access/DiskAccessStorage.cpp +++ b/src/Access/DiskAccessStorage.cpp @@ -367,7 +367,7 @@ bool DiskAccessStorage::readLists() auto file_path = getListFilePath(directory_path, type); if (!std::filesystem::exists(file_path)) { - LOG_WARNING(getLogger(), "File " + file_path.string() + " doesn't exist"); + LOG_WARNING_FORMATTED(getLogger(), "File {} doesn't exist", file_path.string()); ok = false; break; } diff --git a/src/Common/Config/ConfigProcessor.cpp b/src/Common/Config/ConfigProcessor.cpp index 7daf5321d24..be104990187 100644 --- a/src/Common/Config/ConfigProcessor.cpp +++ b/src/Common/Config/ConfigProcessor.cpp @@ -440,7 +440,7 @@ XMLDocumentPtr ConfigProcessor::processConfig( zkutil::ZooKeeperNodeCache * zk_node_cache, const zkutil::EventPtr & zk_changed_event) { - LOG_DEBUG(log, "Processing configuration file '" + path + "'."); + LOG_DEBUG_FORMATTED(log, "Processing configuration file '{}'.", path); XMLDocumentPtr config = dom_parser.parse(path); @@ -451,7 +451,7 @@ XMLDocumentPtr ConfigProcessor::processConfig( { try { - LOG_DEBUG(log, "Merging configuration file '" + merge_file + "'."); + LOG_DEBUG_FORMATTED(log, "Merging configuration file '{}'.", merge_file); XMLDocumentPtr with = dom_parser.parse(merge_file); merge(config, with); @@ -488,7 +488,7 @@ XMLDocumentPtr ConfigProcessor::processConfig( } if (!include_from_path.empty()) { - LOG_DEBUG(log, "Including configuration file '" + include_from_path + "'."); + LOG_DEBUG_FORMATTED(log, "Including configuration file '{}'.", include_from_path); contributing_files.push_back(include_from_path); include_from = dom_parser.parse(include_from_path); diff --git a/src/Common/Exception.cpp b/src/Common/Exception.cpp index 354895b1361..6c8d3e02535 100644 --- a/src/Common/Exception.cpp +++ b/src/Common/Exception.cpp @@ -36,7 +36,7 @@ Exception::Exception(const std::string & msg, int code) #ifndef NDEBUG if (code == ErrorCodes::LOGICAL_ERROR) { - LOG_ERROR(&Poco::Logger::root(), "Logical error: '" + msg + "'."); + LOG_ERROR_FORMATTED(&Poco::Logger::root(), "Logical error: '{}'.", msg); assert(false); } #endif diff --git a/src/Disks/DiskLocal.cpp b/src/Disks/DiskLocal.cpp index 3b3dbf155f8..f7c1d836085 100644 --- a/src/Disks/DiskLocal.cpp +++ b/src/Disks/DiskLocal.cpp @@ -312,7 +312,7 @@ DiskLocalReservation::~DiskLocalReservation() if (disk->reserved_bytes < size) { disk->reserved_bytes = 0; - LOG_ERROR(&Logger::get("DiskLocal"), "Unbalanced reservations size for disk '" + disk->getName() + "'."); + LOG_ERROR_FORMATTED(&Logger::get("DiskLocal"), "Unbalanced reservations size for disk '{}'.", disk->getName()); } else { @@ -320,7 +320,7 @@ DiskLocalReservation::~DiskLocalReservation() } if (disk->reservation_count == 0) - LOG_ERROR(&Logger::get("DiskLocal"), "Unbalanced reservation count for disk '" + disk->getName() + "'."); + LOG_ERROR_FORMATTED(&Logger::get("DiskLocal"), "Unbalanced reservation count for disk '{}'.", disk->getName()); else --disk->reservation_count; } diff --git a/src/Disks/S3/DiskS3.cpp b/src/Disks/S3/DiskS3.cpp index 8014c797909..8dd7be00465 100644 --- a/src/Disks/S3/DiskS3.cpp +++ b/src/Disks/S3/DiskS3.cpp @@ -677,7 +677,7 @@ DiskS3Reservation::~DiskS3Reservation() if (disk->reserved_bytes < size) { disk->reserved_bytes = 0; - LOG_ERROR(&Logger::get("DiskLocal"), "Unbalanced reservations size for disk '" + disk->getName() + "'."); + LOG_ERROR_FORMATTED(&Logger::get("DiskLocal"), "Unbalanced reservations size for disk '{}'.", disk->getName()); } else { @@ -685,7 +685,7 @@ DiskS3Reservation::~DiskS3Reservation() } if (disk->reservation_count == 0) - LOG_ERROR(&Logger::get("DiskLocal"), "Unbalanced reservation count for disk '" + disk->getName() + "'."); + LOG_ERROR_FORMATTED(&Logger::get("DiskLocal"), "Unbalanced reservation count for disk '{}'.", disk->getName()); else --disk->reservation_count; } diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index d50723ccc8e..b8c7dded878 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -701,7 +701,7 @@ void DDLWorker::checkShardConfig(const String & table, const DDLTask & task, Sto if (dynamic_cast(storage.get())) { - LOG_TRACE(log, "Table " + backQuote(table) + " is distributed, skip checking config."); + LOG_TRACE_FORMATTED(log, "Table {} is distributed, skip checking config.", backQuote(table)); return; } diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index eff80a37e86..b5e22619803 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -250,7 +250,7 @@ private: { if (path.empty() || !repository.exists(path)) { - LOG_WARNING(log, "Config file '" + path + "' does not exist"); + LOG_WARNING_FORMATTED(log, "Config file '{}' does not exist", path); return false; } diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 984707b904c..376facabca3 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -401,12 +401,12 @@ void InterpreterSystemQuery::syncReplica(ASTSystemQuery &) LOG_TRACE_FORMATTED(log, "Synchronizing entries in replica's queue with table's log and waiting for it to become empty"); if (!storage_replicated->waitForShrinkingQueueSize(0, context.getSettingsRef().receive_timeout.totalMilliseconds())) { - LOG_ERROR(log, "SYNC REPLICA " + table_id.getNameForLogs() + ": Timed out!"); + LOG_ERROR_FORMATTED(log, "SYNC REPLICA {}: Timed out!", table_id.getNameForLogs()); throw Exception( "SYNC REPLICA " + table_id.getNameForLogs() + ": command timed out! " "See the 'receive_timeout' setting", ErrorCodes::TIMEOUT_EXCEEDED); } - LOG_TRACE(log, "SYNC REPLICA " + table_id.getNameForLogs() + ": OK"); + LOG_TRACE_FORMATTED(log, "SYNC REPLICA {}: OK", table_id.getNameForLogs()); } else throw Exception("Table " + table_id.getNameForLogs() + " is not replicated", ErrorCodes::BAD_ARGUMENTS); diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h index 3c07a2e4e6f..c4de60e0514 100644 --- a/src/Interpreters/SystemLog.h +++ b/src/Interpreters/SystemLog.h @@ -246,7 +246,7 @@ void SystemLog::add(const LogElement & element) requested_flush_before = queue_end; flush_event.notify_all(); - LOG_INFO(log, "Queue is half full for system log '" + demangle(typeid(*this).name()) + "'."); + LOG_INFO_FORMATTED(log, "Queue is half full for system log '{}'.", demangle(typeid(*this).name())); } if (queue.size() >= DBMS_SYSTEM_LOG_QUEUE_SIZE) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index ffd9a8f0a74..2a95e939cd8 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -618,11 +618,11 @@ void ReplicatedMergeTreeQueue::updateMutations(zkutil::ZooKeeperPtr zookeeper, C { if (!it->second.is_done) { - LOG_DEBUG(log, "Removing killed mutation " + entry.znode_name + " from local state."); + LOG_DEBUG_FORMATTED(log, "Removing killed mutation {} from local state.", entry.znode_name); some_active_mutations_were_killed = true; } else - LOG_DEBUG(log, "Removing obsolete mutation " + entry.znode_name + " from local state."); + LOG_DEBUG_FORMATTED(log, "Removing obsolete mutation {} from local state.", entry.znode_name); for (const auto & partition_and_block_num : entry.block_numbers) { @@ -727,7 +727,7 @@ ReplicatedMergeTreeMutationEntryPtr ReplicatedMergeTreeQueue::removeMutation( auto rc = zookeeper->tryRemove(zookeeper_path + "/mutations/" + mutation_id); if (rc == Coordination::ZOK) - LOG_DEBUG(log, "Removed mutation " + mutation_id + " from ZooKeeper."); + LOG_DEBUG_FORMATTED(log, "Removed mutation {} from ZooKeeper.", mutation_id); ReplicatedMergeTreeMutationEntryPtr entry; bool mutation_was_active = false; @@ -756,7 +756,7 @@ ReplicatedMergeTreeMutationEntryPtr ReplicatedMergeTreeQueue::removeMutation( } mutations_by_znode.erase(it); - LOG_DEBUG(log, "Removed mutation " + entry->znode_name + " from local state."); + LOG_DEBUG_FORMATTED(log, "Removed mutation {} from local state.", entry->znode_name); } if (mutation_was_active) @@ -1161,13 +1161,13 @@ ReplicatedMergeTreeQueue::CurrentlyExecuting::~CurrentlyExecuting() for (const String & new_part_name : entry->getBlockingPartNames()) { if (!queue.future_parts.erase(new_part_name)) - LOG_ERROR(queue.log, "Untagging already untagged future part " + new_part_name + ". This is a bug."); + LOG_ERROR_FORMATTED(queue.log, "Untagging already untagged future part {}. This is a bug.", new_part_name); } if (!entry->actual_new_part_name.empty()) { if (entry->actual_new_part_name != entry->new_part_name && !queue.future_parts.erase(entry->actual_new_part_name)) - LOG_ERROR(queue.log, "Untagging already untagged future part " + entry->actual_new_part_name + ". This is a bug."); + LOG_ERROR_FORMATTED(queue.log, "Untagging already untagged future part {}. This is a bug.", entry->actual_new_part_name); entry->actual_new_part_name.clear(); } From 7bc2b6bbbc5402a2660e94fcaeabacd04ca64e0a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 20:10:45 +0300 Subject: [PATCH 628/738] find {base,src,programs} -name '*.h' -or -name '*.cpp' | xargs grep -l -P 'LOG_\w+\([^,]+, "[^"]+" \+ [^+]+ \+ "[^"]+" \+ [^+]+\);' | xargs sed -i -r -e 's/(LOG_\w+)\(([^,]+), "([^"]+)" \+ ([^+]+) \+ "([^"]+)" \+ ([^+]+)\);/\1_FORMATTED(\2, "\3{}\5{}", \4, \6);/' --- src/Interpreters/DDLWorker.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index b8c7dded878..fbdbcb77cb7 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -901,7 +901,7 @@ void DDLWorker::cleanupQueue(Int64 current_time_seconds, const ZooKeeperPtr & zo } catch (...) { - LOG_INFO(log, "An error occured while checking and cleaning task " + node_name + " from queue: " + getCurrentExceptionMessage(false)); + LOG_INFO_FORMATTED(log, "An error occured while checking and cleaning task {} from queue: {}", node_name, getCurrentExceptionMessage(false)); } } } From 57f6dc66d356669969a2d73c613c930bef67b4d4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 20:11:09 +0300 Subject: [PATCH 629/738] find {base,src,programs} -name '*.h' -or -name '*.cpp' | xargs grep -l -P 'LOG_\w+\([^,]+, "[^"]+" \+ [^+]+ \+ "[^"]+" \+ [^+]+ \+ "[^"]+" \+ [^+]+\);' | xargs sed -i -r -e 's/(LOG_\w+)\(([^,]+), "([^"]+)" \+ ([^+]+) \+ "([^"]+)" \+ ([^+]+) \+ "([^"]+)" \+ ([^+]+)\);/\1_FORMATTED(\2, "\3{}\5{}\7{}", \4, \6, \8);/' --- src/IO/ReadBufferFromS3.cpp | 2 +- src/IO/WriteBufferFromS3.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp index 1fa318cf629..c751716f42b 100644 --- a/src/IO/ReadBufferFromS3.cpp +++ b/src/IO/ReadBufferFromS3.cpp @@ -66,7 +66,7 @@ off_t ReadBufferFromS3::getPosition() std::unique_ptr ReadBufferFromS3::initialize() { - LOG_TRACE(log, "Read S3 object. Bucket: " + bucket + ", Key: " + key + ", Offset: " + std::to_string(offset)); + LOG_TRACE_FORMATTED(log, "Read S3 object. Bucket: {}, Key: {}, Offset: {}", bucket, key, std::to_string(offset)); Aws::S3::Model::GetObjectRequest req; req.SetBucket(bucket); diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 699db5f92fe..8830d60e0a0 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -148,7 +148,7 @@ void WriteBufferFromS3::writePart(const String & data) void WriteBufferFromS3::complete() { - LOG_DEBUG(log, "Completing multipart upload. Bucket: " + bucket + ", Key: " + key + ", Upload_id: " + upload_id); + LOG_DEBUG_FORMATTED(log, "Completing multipart upload. Bucket: {}, Key: {}, Upload_id: {}", bucket, key, upload_id); Aws::S3::Model::CompleteMultipartUploadRequest req; req.SetBucket(bucket); From d6bc4d2bd0c0c2676b155565f985b385b79107b4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 20:14:57 +0300 Subject: [PATCH 630/738] find {base,src,programs} -name '*.h' -or -name '*.cpp' | xargs grep -l -P 'LOG_\w+\([^,]+, [^<]+ << "[^"]+" << [^<]+\);' | xargs sed -i -r -e 's/(LOG_\w+)\(([^,]+), ([^<]+) << "([^"]+)" << ([^<]+)\);/\1_FORMATTED(\2, "{}\4{}", \3, \5);/' --- src/Interpreters/InterpreterSelectQuery.cpp | 2 +- src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 1e300ae3c5a..c4a08f0f837 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -776,7 +776,7 @@ void InterpreterSelectQuery::executeImpl(TPipeline & pipeline, const BlockInputS /** Read the data from Storage. from_stage - to what stage the request was completed in Storage. */ executeFetchColumns(from_stage, pipeline, expressions.prewhere_info, expressions.columns_to_remove_after_prewhere, save_context_and_storage); - LOG_TRACE(log, QueryProcessingStage::toString(from_stage) << " -> " << QueryProcessingStage::toString(options.to_stage)); + LOG_TRACE_FORMATTED(log, "{} -> {}", QueryProcessingStage::toString(from_stage), QueryProcessingStage::toString(options.to_stage)); } if (options.to_stage > QueryProcessingStage::FetchColumns) diff --git a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp index 875c29c8835..eb373d40d8f 100644 --- a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp +++ b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp @@ -244,7 +244,7 @@ void ReadBufferFromKafkaConsumer::resetToLastCommitted(const char * msg) } auto committed_offset = consumer->get_offsets_committed(consumer->get_assignment()); consumer->assign(committed_offset); - LOG_TRACE(log, msg << " Returned to committed position: " << committed_offset); + LOG_TRACE_FORMATTED(log, "{} Returned to committed position: {}", msg, committed_offset); } /// Do commit messages implicitly after we processed the previous batch. From b3f4fe12ddd584abf0802132257d810823652b24 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 20:25:19 +0300 Subject: [PATCH 631/738] Progress on task --- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- src/Storages/StorageSet.cpp | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index a520ebe15ce..29794b1da76 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -3987,7 +3987,7 @@ bool StorageReplicatedMergeTree::waitForReplicaToProcessLogEntry(const String & /// While looking for the record, it has already been executed and deleted. if (queue_entry_to_wait_for.empty()) { - LOG_DEBUG(log, "No corresponding node found. Assuming it has been already processed." " Found " << queue_entries.size() << " nodes."); + LOG_DEBUG_FORMATTED(log, "No corresponding node found. Assuming it has been already processed. Found {} nodes", queue_entries.size()); return true; } diff --git a/src/Storages/StorageSet.cpp b/src/Storages/StorageSet.cpp index 9a3a2b0ac94..939a8367f60 100644 --- a/src/Storages/StorageSet.cpp +++ b/src/Storages/StorageSet.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -190,11 +191,8 @@ void StorageSetOrJoinBase::restoreFromFile(const String & file_path) backup_stream.readSuffix(); /// TODO Add speed, compressed bytes, data volume in memory, compression ratio ... Generalize all statistics logging in project. - LOG_INFO(&Logger::get("StorageSetOrJoinBase"), std::fixed << std::setprecision(2) - << "Loaded from backup file " << file_path << ". " - << backup_stream.getProfileInfo().rows << " rows, " - << backup_stream.getProfileInfo().bytes / 1048576.0 << " MiB. " - << "State has " << getSize() << " unique rows."); + LOG_INFO_FORMATTED(&Logger::get("StorageSetOrJoinBase"), "Loaded from backup file {}. {} rows, {}. State has {} unique rows.", + file_path, backup_stream.getProfileInfo().rows, formatReadableSizeWithBinarySuffix(backup_stream.getProfileInfo().bytes), getSize()); } From aa656f95dd331d00d152e76ff8bca70d1bbd3441 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 23 May 2020 20:28:40 +0300 Subject: [PATCH 632/738] Fix compilation on osx (where UInt64 and size_t is different) --- src/Interpreters/InterpreterSelectWithUnionQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index e667029dcda..7fe124b31e6 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -275,7 +275,7 @@ QueryPipeline InterpreterSelectWithUnionQuery::executeWithProcessors() // nested queries can force 1 thread (due to simplicity) // but in case of union this cannot be done. UInt64 max_threads = context->getSettingsRef().max_threads; - main_pipeline.setMaxThreads(std::min(nested_interpreters.size(), max_threads)); + main_pipeline.setMaxThreads(std::min(nested_interpreters.size(), max_threads)); } main_pipeline.addInterpreterContext(context); From 3a09f9ca1c4b286e27580a6fbdf2c3bb21176b43 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 20:29:56 +0300 Subject: [PATCH 633/738] find {base,src,programs} -name '*.h' -or -name '*.cpp' | xargs grep -l -P 'LOG_[^\W_]+\([^,]+, [^<+)"]+\);' | xargs sed -i -r -e 's/(LOG_[^\W_]+)\(([^,]+), ([^<+)"]+)\);/\1_FORMATTED(\2, \3);/' --- base/daemon/BaseDaemon.cpp | 4 ++-- programs/server/InterserverIOHTTPHandler.cpp | 6 +++--- src/Dictionaries/MySQLDictionarySource.cpp | 4 ++-- src/Dictionaries/XDBCDictionarySource.cpp | 4 ++-- src/Storages/Kafka/StorageKafka.cpp | 2 +- .../MergeTree/ReplicatedMergeTreePartCheckThread.cpp | 4 ++-- src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp | 10 +++++----- 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index c48e92a1b7b..b091d01dea0 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -278,7 +278,7 @@ private: } /// Write symbolized stack trace line by line for better grep-ability. - stack_trace.toStringEveryLine([&](const std::string & s) { LOG_FATAL(log, s); }); + stack_trace.toStringEveryLine([&](const std::string & s) { LOG_FATAL_FORMATTED(log, s); }); } }; @@ -318,7 +318,7 @@ static void sanitizerDeathCallback() } /// Write symbolized stack trace line by line for better grep-ability. - stack_trace.toStringEveryLine([&](const std::string & s) { LOG_FATAL(log, s); }); + stack_trace.toStringEveryLine([&](const std::string & s) { LOG_FATAL_FORMATTED(log, s); }); } #endif diff --git a/programs/server/InterserverIOHTTPHandler.cpp b/programs/server/InterserverIOHTTPHandler.cpp index d86876800b8..71b6010f87a 100644 --- a/programs/server/InterserverIOHTTPHandler.cpp +++ b/programs/server/InterserverIOHTTPHandler.cpp @@ -121,9 +121,9 @@ void InterserverIOHTTPHandler::handleRequest(Poco::Net::HTTPServerRequest & requ writeString(message, *used_output.out); if (is_real_error) - LOG_ERROR(log, message); + LOG_ERROR_FORMATTED(log, message); else - LOG_INFO(log, message); + LOG_INFO_FORMATTED(log, message); } catch (...) { @@ -132,7 +132,7 @@ void InterserverIOHTTPHandler::handleRequest(Poco::Net::HTTPServerRequest & requ if (!response.sent()) writeString(message, *used_output.out); - LOG_ERROR(log, message); + LOG_ERROR_FORMATTED(log, message); } } diff --git a/src/Dictionaries/MySQLDictionarySource.cpp b/src/Dictionaries/MySQLDictionarySource.cpp index 884a88c1c3e..0c4ce686f62 100644 --- a/src/Dictionaries/MySQLDictionarySource.cpp +++ b/src/Dictionaries/MySQLDictionarySource.cpp @@ -118,7 +118,7 @@ BlockInputStreamPtr MySQLDictionarySource::loadAll() auto connection = pool.get(); last_modification = getLastModification(connection, false); - LOG_TRACE(log, load_all_query); + LOG_TRACE_FORMATTED(log, load_all_query); return std::make_shared(connection, load_all_query, sample_block, max_block_size, close_connection); } @@ -128,7 +128,7 @@ BlockInputStreamPtr MySQLDictionarySource::loadUpdatedAll() last_modification = getLastModification(connection, false); std::string load_update_query = getUpdateFieldAndDate(); - LOG_TRACE(log, load_update_query); + LOG_TRACE_FORMATTED(log, load_update_query); return std::make_shared(connection, load_update_query, sample_block, max_block_size, close_connection); } diff --git a/src/Dictionaries/XDBCDictionarySource.cpp b/src/Dictionaries/XDBCDictionarySource.cpp index 03d31fa001f..27839678c6d 100644 --- a/src/Dictionaries/XDBCDictionarySource.cpp +++ b/src/Dictionaries/XDBCDictionarySource.cpp @@ -134,7 +134,7 @@ std::string XDBCDictionarySource::getUpdateFieldAndDate() BlockInputStreamPtr XDBCDictionarySource::loadAll() { - LOG_TRACE(log, load_all_query); + LOG_TRACE_FORMATTED(log, load_all_query); return loadBase(load_all_query); } @@ -142,7 +142,7 @@ BlockInputStreamPtr XDBCDictionarySource::loadUpdatedAll() { std::string load_query_update = getUpdateFieldAndDate(); - LOG_TRACE(log, load_query_update); + LOG_TRACE_FORMATTED(log, load_query_update); return loadBase(load_query_update); } diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index ae09706ea5d..4e8cb916c38 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -325,7 +325,7 @@ void StorageKafka::updateConfiguration(cppkafka::Configuration & conf) conf.set_log_callback([this](cppkafka::KafkaHandleBase &, int level, const std::string & /* facility */, const std::string & message) { auto [poco_level, client_logs_level] = parseSyslogLevel(level); - LOG_IMPL(log, client_logs_level, poco_level, message); + LOG_IMPL_FORMATTED(log, client_logs_level, poco_level, message); }); // Configure interceptor to change thread name diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp index 345f6a37cf8..ade201c04de 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp @@ -260,7 +260,7 @@ CheckResult ReplicatedMergeTreePartCheckThread::checkPart(const String & part_na tryLogCurrentException(log, __PRETTY_FUNCTION__); String message = "Part " + part_name + " looks broken. Removing it and queueing a fetch."; - LOG_ERROR(log, message); + LOG_ERROR_FORMATTED(log, message); ProfileEvents::increment(ProfileEvents::ReplicatedPartChecksFailed); storage.removePartAndEnqueueFetch(part_name); @@ -278,7 +278,7 @@ CheckResult ReplicatedMergeTreePartCheckThread::checkPart(const String & part_na ProfileEvents::increment(ProfileEvents::ReplicatedPartChecksFailed); String message = "Unexpected part " + part_name + " in filesystem. Removing."; - LOG_ERROR(log, message); + LOG_ERROR_FORMATTED(log, message); storage.forgetPartAndMoveToDetached(part, "unexpected"); return {part_name, false, message}; } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 2a95e939cd8..3d04a51daf9 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -987,7 +987,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( if (!isNotCoveredByFuturePartsImpl(new_part_name, out_postpone_reason, state_lock)) { if (!out_postpone_reason.empty()) - LOG_DEBUG(log, out_postpone_reason); + LOG_DEBUG_FORMATTED(log, out_postpone_reason); return false; } } @@ -1007,7 +1007,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( { String reason = "Not merging into part " + entry.new_part_name + " because part " + name + " is not ready yet (log entry for that part is being processed)."; - LOG_TRACE(log, reason); + LOG_TRACE_FORMATTED(log, reason); out_postpone_reason = reason; return false; } @@ -1020,7 +1020,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( if (merger_mutator.merges_blocker.isCancelled()) { String reason = "Not executing log entry for part " + entry.new_part_name + " because merges and mutations are cancelled now."; - LOG_DEBUG(log, reason); + LOG_DEBUG_FORMATTED(log, reason); out_postpone_reason = reason; return false; } @@ -1041,7 +1041,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( String reason = "Not executing log entry " + entry.typeToString() + " for part " + entry.new_part_name + " because source parts size (" + formatReadableSizeWithBinarySuffix(sum_parts_size_in_bytes) + ") is greater than the current maximum (" + formatReadableSizeWithBinarySuffix(max_source_parts_size) + ")."; - LOG_DEBUG(log, reason); + LOG_DEBUG_FORMATTED(log, reason); out_postpone_reason = reason; return false; } @@ -1056,7 +1056,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( if (0 != getConflictsCountForRange(range, entry, &conflicts_description, state_lock)) { - LOG_DEBUG(log, conflicts_description); + LOG_DEBUG_FORMATTED(log, conflicts_description); return false; } } From 409cff11df96b810f3dbe8adb75c62d71894d10f Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 22 May 2020 00:37:47 +0300 Subject: [PATCH 634/738] Add port() function It uses domain() (getURLHost()) internally so it has the same limitations: - no ipv6 support - no host-without-dots support (i.e. "foobar") --- .../sql-reference/functions/url-functions.md | 4 + src/Functions/URL/port.cpp | 95 +++++++++++++++++++ src/Functions/URL/registerFunctionsURL.cpp | 2 + .../queries/0_stateless/01284_port.reference | 21 ++++ tests/queries/0_stateless/01284_port.sql | 25 +++++ 5 files changed, 147 insertions(+) create mode 100644 src/Functions/URL/port.cpp create mode 100644 tests/queries/0_stateless/01284_port.reference create mode 100644 tests/queries/0_stateless/01284_port.sql diff --git a/docs/en/sql-reference/functions/url-functions.md b/docs/en/sql-reference/functions/url-functions.md index 143bd42b08e..f35c621a6e2 100644 --- a/docs/en/sql-reference/functions/url-functions.md +++ b/docs/en/sql-reference/functions/url-functions.md @@ -117,6 +117,10 @@ Returns the part of the domain that includes top-level subdomains up to the “f For example, `cutToFirstSignificantSubdomain('https://news.yandex.com.tr/') = 'yandex.com.tr'`. +### port(URL) {#port} + +Returns the port or zero if there is no port in the URL (or in case of validation error). + ### path {#path} Returns the path. Example: `/top/news.html` The path does not include the query string. diff --git a/src/Functions/URL/port.cpp b/src/Functions/URL/port.cpp new file mode 100644 index 00000000000..d4876e9f647 --- /dev/null +++ b/src/Functions/URL/port.cpp @@ -0,0 +1,95 @@ +#include +#include +#include +#include "domain.h" + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + +struct ExtractPort +{ + static constexpr auto name = "port"; + static constexpr auto is_fixed_to_constant = true; + + static void vector(const ColumnString::Chars & data, const ColumnString::Offsets & offsets, PaddedPODArray & res) + { + size_t size = offsets.size(); + + ColumnString::Offset prev_offset = 0; + for (size_t i = 0; i < size; ++i) + { + res[i] = parse(data, prev_offset, offsets[i] - prev_offset - 1); + prev_offset = offsets[i]; + } + } + + static void vectorFixedToConstant(const ColumnString::Chars & data, size_t n, UInt16 & res) { res = parse(data, 0, n); } + + static void vectorFixedToVector(const ColumnString::Chars & data, size_t n, PaddedPODArray & res) + { + size_t size = data.size() / n; + + for (size_t i = 0; i < size; ++i) + { + res[i] = parse(data, i * n, n); + } + } + + [[noreturn]] static void array(const ColumnString::Offsets &, PaddedPODArray &) + { + throw Exception("Cannot apply function port to Array argument", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + +private: + static UInt16 parse(const ColumnString::Chars & buf, size_t offset, size_t size) + { + const char * p = reinterpret_cast(&buf[0]) + offset; + const char * end = p + size; + + StringRef host = getURLHost(p, size); + if (!host.size) + return 0; + if (host.size == size) + return 0; + + p = host.data + host.size; + if (*p++ != ':') + return 0; + + Int64 port = 0; + while (p < end) + { + if (*p == '/') + break; + if (!isNumericASCII(*p)) + return 0; + + port = (port * 10) + (*p - '0'); + if (port < 0 || port > UInt16(-1)) + return 0; + ++p; + } + return port; + } +}; + +struct NamePort +{ + static constexpr auto name = "port"; +}; + +using FunctionPort = FunctionStringOrArrayToT; + +void registerFunctionPort(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} + diff --git a/src/Functions/URL/registerFunctionsURL.cpp b/src/Functions/URL/registerFunctionsURL.cpp index 66a847185f3..9ba5261f728 100644 --- a/src/Functions/URL/registerFunctionsURL.cpp +++ b/src/Functions/URL/registerFunctionsURL.cpp @@ -8,6 +8,7 @@ void registerFunctionDomain(FunctionFactory & factory); void registerFunctionDomainWithoutWWW(FunctionFactory & factory); void registerFunctionFirstSignificantSubdomain(FunctionFactory & factory); void registerFunctionTopLevelDomain(FunctionFactory & factory); +void registerFunctionPort(FunctionFactory & factory); void registerFunctionPath(FunctionFactory & factory); void registerFunctionPathFull(FunctionFactory & factory); void registerFunctionQueryString(FunctionFactory & factory); @@ -33,6 +34,7 @@ void registerFunctionsURL(FunctionFactory & factory) registerFunctionDomainWithoutWWW(factory); registerFunctionFirstSignificantSubdomain(factory); registerFunctionTopLevelDomain(factory); + registerFunctionPort(factory); registerFunctionPath(factory); registerFunctionPathFull(factory); registerFunctionQueryString(factory); diff --git a/tests/queries/0_stateless/01284_port.reference b/tests/queries/0_stateless/01284_port.reference new file mode 100644 index 00000000000..5bf1c035ca8 --- /dev/null +++ b/tests/queries/0_stateless/01284_port.reference @@ -0,0 +1,21 @@ +ipv4 +0 +80 +80 +80 +80 +hostname +0 +80 +80 +80 +80 +ipv6 +0 +0 +0 +0 +0 +0 +host-no-dot +0 diff --git a/tests/queries/0_stateless/01284_port.sql b/tests/queries/0_stateless/01284_port.sql new file mode 100644 index 00000000000..3ba5aecd691 --- /dev/null +++ b/tests/queries/0_stateless/01284_port.sql @@ -0,0 +1,25 @@ +select 'ipv4'; +select port('http://127.0.0.1/'); +select port('http://127.0.0.1:80'); +select port('http://127.0.0.1:80/'); +select port('//127.0.0.1:80/'); +select port('127.0.0.1:80'); +select 'hostname'; +select port('http://foobar.com/'); +select port('http://foobar.com:80'); +select port('http://foobar.com:80/'); +select port('//foobar.com:80/'); +select port('foobar.com:80'); + +-- +-- Known limitations of domain() (getURLHost()) +-- +select 'ipv6'; +select port('http://[2001:db8::8a2e:370:7334]/'); +select port('http://[2001:db8::8a2e:370:7334]:80'); +select port('http://[2001:db8::8a2e:370:7334]:80/'); +select port('//[2001:db8::8a2e:370:7334]:80/'); +select port('[2001:db8::8a2e:370:7334]:80'); +select port('2001:db8::8a2e:370:7334:80'); +select 'host-no-dot'; +select port('//foobar:80/'); From 9d24908e536d08ba5a72de882e5650003ee72553 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 20:52:11 +0300 Subject: [PATCH 635/738] Progress on task --- src/Access/AccessRights.cpp | 11 +++++----- src/Access/AllowedClientHosts.cpp | 12 +++++------ src/Access/ContextAccess.cpp | 15 ++++++++------ src/Client/Connection.cpp | 19 ++++++++---------- .../ParallelAggregatingBlockInputStream.cpp | 8 ++++---- src/Databases/DatabaseOrdinary.cpp | 2 +- src/Dictionaries/CacheDictionary.cpp | 3 +-- src/Dictionaries/DictionaryFactory.cpp | 4 +--- src/Dictionaries/MySQLDictionarySource.cpp | 2 +- src/Disks/DiskLocal.cpp | 6 ++---- src/Disks/DiskSelector.cpp | 2 +- src/Disks/IDisk.cpp | 4 +--- src/Disks/S3/DiskS3.cpp | 17 ++++++---------- src/Interpreters/Aggregator.cpp | 20 +++++++++---------- src/Processors/Executors/PipelineExecutor.cpp | 12 +++++------ .../Transforms/AggregatingTransform.cpp | 8 ++++---- .../MergeTree/MergeTreeDataMergerMutator.cpp | 4 ++-- src/Storages/MergeTree/MergeTreeReadPool.cpp | 4 ++-- 18 files changed, 70 insertions(+), 83 deletions(-) diff --git a/src/Access/AccessRights.cpp b/src/Access/AccessRights.cpp index ed5e1abdb2a..966fcdfc062 100644 --- a/src/Access/AccessRights.cpp +++ b/src/Access/AccessRights.cpp @@ -251,12 +251,11 @@ public: void logTree(Poco::Logger * log) const { - LOG_TRACE(log, "Tree(" << level << "): name=" << (node_name ? *node_name : "NULL") - << ", access=" << access.toString() - << ", final_access=" << final_access.toString() - << ", min_access=" << min_access.toString() - << ", max_access=" << max_access.toString() - << ", num_children=" << (children ? children->size() : 0)); + LOG_TRACE_FORMATTED(log, "Tree({}): name={}, access={}, final_access={}, min_access={}, max_access={}, num_children={}", + level, node_name ? *node_name : "NULL", access.toString(), + final_access.toString(), min_access.toString(), max_access.toString(), + (children ? children->size() : 0)); + if (children) { for (auto & child : *children | boost::adaptors::map_values) diff --git a/src/Access/AllowedClientHosts.cpp b/src/Access/AllowedClientHosts.cpp index 1db7e6d508d..201efa499be 100644 --- a/src/Access/AllowedClientHosts.cpp +++ b/src/Access/AllowedClientHosts.cpp @@ -308,10 +308,10 @@ bool AllowedClientHosts::contains(const IPAddress & client_address) const if (e.code() != ErrorCodes::DNS_ERROR) throw; /// Try to ignore DNS errors: if host cannot be resolved, skip it and try next. - LOG_WARNING( + LOG_WARNING_FORMATTED( &Logger::get("AddressPatterns"), - "Failed to check if the allowed client hosts contain address " << client_address.toString() << ". " << e.displayText() - << ", code = " << e.code()); + "Failed to check if the allowed client hosts contain address {}. {}, code = {}", + client_address.toString(), e.displayText(), e.code()); return false; } }; @@ -341,10 +341,10 @@ bool AllowedClientHosts::contains(const IPAddress & client_address) const if (e.code() != ErrorCodes::DNS_ERROR) throw; /// Try to ignore DNS errors: if host cannot be resolved, skip it and try next. - LOG_WARNING( + LOG_WARNING_FORMATTED( &Logger::get("AddressPatterns"), - "Failed to check if the allowed client hosts contain address " << client_address.toString() << ". " << e.displayText() - << ", code = " << e.code()); + "Failed to check if the allowed client hosts contain address {}. {}, code = {}", + client_address.toString(), e.displayText(), e.code()); return false; } }; diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index 40eadd1bee0..644f4a5817d 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -219,7 +219,7 @@ bool ContextAccess::calculateResultAccessAndCheck(Poco::Logger * log_, const Acc if constexpr (mode == THROW_IF_ACCESS_DENIED) throw Exception(user_name + ": " + msg, error_code); else if constexpr (mode == LOG_WARNING_IF_ACCESS_DENIED) - LOG_WARNING(log_, user_name + ": " + msg + formatSkippedMessage(args...)); + LOG_WARNING_FORMATTED(log_, "{}: {}{}", user_name, msg, formatSkippedMessage(args...)); }; if (!user) @@ -451,13 +451,16 @@ boost::shared_ptr ContextAccess::calculateResultAccess(bool if (trace_log && (params.readonly == readonly_) && (params.allow_ddl == allow_ddl_) && (params.allow_introspection == allow_introspection_)) { - LOG_TRACE(trace_log, "List of all grants: " << merged_access->toString() << (grant_option ? " WITH GRANT OPTION" : "")); + if (grant_option) + LOG_TRACE_FORMATTED(trace_log, "List of all grants: {} WITH GRANT OPTION", merged_access->toString()); + else + LOG_TRACE_FORMATTED(trace_log, "List of all grants: {}", merged_access->toString()); + if (roles_info && !roles_info->getCurrentRolesNames().empty()) { - LOG_TRACE( - trace_log, - "Current_roles: " << boost::algorithm::join(roles_info->getCurrentRolesNames(), ", ") - << ", enabled_roles: " << boost::algorithm::join(roles_info->getEnabledRolesNames(), ", ")); + LOG_TRACE_FORMATTED(trace_log, "Current_roles: {}, enabled_roles: {}", + boost::algorithm::join(roles_info->getCurrentRolesNames(), ", "), + boost::algorithm::join(roles_info->getEnabledRolesNames(), ", ")); } LOG_TRACE_FORMATTED(trace_log, "Settings: readonly={}, allow_ddl={}, allow_introspection_functions={}", readonly_, allow_ddl_, allow_introspection_); } diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index 1b9dfde9177..0173291e76c 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -61,11 +61,11 @@ void Connection::connect(const ConnectionTimeouts & timeouts) if (connected) disconnect(); - LOG_TRACE(log_wrapper.get(), "Connecting. Database: " - << (default_database.empty() ? "(not specified)" : default_database) - << ". User: " << user - << (static_cast(secure) ? ". Secure" : "") - << (static_cast(compression) ? "" : ". Uncompressed")); + LOG_TRACE_FORMATTED(log_wrapper.get(), "Connecting. Database: {}. User: {}{}{}", + default_database.empty() ? "(not specified)" : default_database, + user, + static_cast(secure) ? ". Secure" : "", + static_cast(compression) ? "" : ". Uncompressed"); if (static_cast(secure)) { @@ -107,11 +107,8 @@ void Connection::connect(const ConnectionTimeouts & timeouts) sendHello(); receiveHello(); - LOG_TRACE(log_wrapper.get(), "Connected to " << server_name - << " server version " << server_version_major - << "." << server_version_minor - << "." << server_version_patch - << "."); + LOG_TRACE_FORMATTED(log_wrapper.get(), "Connected to {} server version {}.{}.{}.", + server_name, server_version_major, server_version_minor, server_version_patch); } catch (Poco::Net::NetException & e) { @@ -324,7 +321,7 @@ bool Connection::ping() } catch (const Poco::Exception & e) { - LOG_TRACE(log_wrapper.get(), e.displayText()); + LOG_TRACE_FORMATTED(log_wrapper.get(), e.displayText()); return false; } diff --git a/src/DataStreams/ParallelAggregatingBlockInputStream.cpp b/src/DataStreams/ParallelAggregatingBlockInputStream.cpp index 91663d9fc7a..979999ab745 100644 --- a/src/DataStreams/ParallelAggregatingBlockInputStream.cpp +++ b/src/DataStreams/ParallelAggregatingBlockInputStream.cpp @@ -179,8 +179,8 @@ void ParallelAggregatingBlockInputStream::execute() for (size_t i = 0; i < max_threads; ++i) { size_t rows = many_data[i]->size(); - LOG_TRACE(log, std::fixed << std::setprecision(3) - << "Aggregated. " << threads_data[i].src_rows << " to " << rows << " rows" + LOG_TRACE(log, + "Aggregated. " << threads_data[i].src_rows << " to " << rows << " rows" << " (from " << threads_data[i].src_bytes / 1048576.0 << " MiB)" << " in " << elapsed_seconds << " sec." << " (" << threads_data[i].src_rows / elapsed_seconds << " rows/sec., " @@ -189,8 +189,8 @@ void ParallelAggregatingBlockInputStream::execute() total_src_rows += threads_data[i].src_rows; total_src_bytes += threads_data[i].src_bytes; } - LOG_TRACE(log, std::fixed << std::setprecision(3) - << "Total aggregated. " << total_src_rows << " rows (from " << total_src_bytes / 1048576.0 << " MiB)" + LOG_TRACE(log, + "Total aggregated. " << total_src_rows << " rows (from " << total_src_bytes / 1048576.0 << " MiB)" << " in " << elapsed_seconds << " sec." << " (" << total_src_rows / elapsed_seconds << " rows/sec., " << total_src_bytes / elapsed_seconds / 1048576.0 << " MiB/sec.)"); diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index 09a4d28c6e7..5c93d2cbfca 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -95,7 +95,7 @@ namespace { if (processed % PRINT_MESSAGE_EACH_N_OBJECTS == 0 || watch.compareAndRestart(PRINT_MESSAGE_EACH_N_SECONDS)) { - LOG_INFO(log, std::fixed << std::setprecision(2) << processed * 100.0 / total << "%"); + LOG_INFO_FORMATTED(log, "{}%", processed * 100.0 / total); watch.restart(); } } diff --git a/src/Dictionaries/CacheDictionary.cpp b/src/Dictionaries/CacheDictionary.cpp index 479f6ba8922..7ac9e5401e5 100644 --- a/src/Dictionaries/CacheDictionary.cpp +++ b/src/Dictionaries/CacheDictionary.cpp @@ -768,8 +768,7 @@ void CacheDictionary::updateThreadFunction() const size_t current_queue_size = update_queue.size(); if (current_queue_size > 0) - LOG_TRACE(log, "Performing bunch of keys update in cache dictionary with " - << current_queue_size + 1 << " keys"); + LOG_TRACE_FORMATTED(log, "Performing bunch of keys update in cache dictionary with {} keys", current_queue_size + 1); std::vector update_request; update_request.reserve(current_queue_size + 1); diff --git a/src/Dictionaries/DictionaryFactory.cpp b/src/Dictionaries/DictionaryFactory.cpp index 326e0f08d89..056a6861a33 100644 --- a/src/Dictionaries/DictionaryFactory.cpp +++ b/src/Dictionaries/DictionaryFactory.cpp @@ -43,9 +43,7 @@ DictionaryPtr DictionaryFactory::create( const DictionaryStructure dict_struct{config, config_prefix + ".structure"}; DictionarySourcePtr source_ptr = DictionarySourceFactory::instance().create(name, config, config_prefix + ".source", dict_struct, context, check_source_config); - LOG_TRACE(&Poco::Logger::get("DictionaryFactory"), - "Created dictionary source '" << source_ptr->toString() - << "' for dictionary '" << name << "'"); + LOG_TRACE_FORMATTED(&Poco::Logger::get("DictionaryFactory"), "Created dictionary source '{}' for dictionary '{}'", source_ptr->toString(), name); const auto & layout_type = keys.front(); diff --git a/src/Dictionaries/MySQLDictionarySource.cpp b/src/Dictionaries/MySQLDictionarySource.cpp index 0c4ce686f62..f8d035f2037 100644 --- a/src/Dictionaries/MySQLDictionarySource.cpp +++ b/src/Dictionaries/MySQLDictionarySource.cpp @@ -213,7 +213,7 @@ LocalDateTime MySQLDictionarySource::getLastModification(mysqlxx::Pool::Entry & { auto query = connection->query("SHOW TABLE STATUS LIKE " + quoteForLike(table)); - LOG_TRACE(log, query.str()); + LOG_TRACE_FORMATTED(log, query.str()); auto result = query.use(); diff --git a/src/Disks/DiskLocal.cpp b/src/Disks/DiskLocal.cpp index f7c1d836085..31ea07482a7 100644 --- a/src/Disks/DiskLocal.cpp +++ b/src/Disks/DiskLocal.cpp @@ -100,10 +100,8 @@ bool DiskLocal::tryReserve(UInt64 bytes) UInt64 unreserved_space = available_space - std::min(available_space, reserved_bytes); if (unreserved_space >= bytes) { - LOG_DEBUG( - &Logger::get("DiskLocal"), - "Reserving " << formatReadableSizeWithBinarySuffix(bytes) << " on disk " << backQuote(name) << ", having unreserved " - << formatReadableSizeWithBinarySuffix(unreserved_space) << "."); + LOG_DEBUG_FORMATTED(&Logger::get("DiskLocal"), "Reserving {} on disk {}, having unreserved {}.", + formatReadableSizeWithBinarySuffix(bytes), backQuote(name), formatReadableSizeWithBinarySuffix(unreserved_space)); ++reservation_count; reserved_bytes += bytes; return true; diff --git a/src/Disks/DiskSelector.cpp b/src/Disks/DiskSelector.cpp index 69549e4520d..fc74f8f789b 100644 --- a/src/Disks/DiskSelector.cpp +++ b/src/Disks/DiskSelector.cpp @@ -99,7 +99,7 @@ DiskSelectorPtr DiskSelector::updateFromConfig( } writeString(" disappeared from configuration, this change will be applied after restart of ClickHouse", warning); - LOG_WARNING(&Logger::get("DiskSelector"), warning.str()); + LOG_WARNING_FORMATTED(&Logger::get("DiskSelector"), warning.str()); } return result; diff --git a/src/Disks/IDisk.cpp b/src/Disks/IDisk.cpp index 36ab2a49573..aeb54deead8 100644 --- a/src/Disks/IDisk.cpp +++ b/src/Disks/IDisk.cpp @@ -15,9 +15,7 @@ bool IDisk::isDirectoryEmpty(const String & path) void copyFile(IDisk & from_disk, const String & from_path, IDisk & to_disk, const String & to_path) { - LOG_DEBUG( - &Poco::Logger::get("IDisk"), - "Copying from " << from_disk.getName() << " " << from_path << " to " << to_disk.getName() << " " << to_path); + LOG_DEBUG_FORMATTED(&Poco::Logger::get("IDisk"), "Copying from {} {} to {} {}.", from_disk.getName(), from_path, to_disk.getName(), to_path); auto in = from_disk.readFile(from_path); auto out = to_disk.writeFile(to_path); diff --git a/src/Disks/S3/DiskS3.cpp b/src/Disks/S3/DiskS3.cpp index 8dd7be00465..7bf151fba5f 100644 --- a/src/Disks/S3/DiskS3.cpp +++ b/src/Disks/S3/DiskS3.cpp @@ -515,9 +515,8 @@ std::unique_ptr DiskS3::readFile(const String & path, si { Metadata metadata(metadata_path, path); - LOG_DEBUG( - &Logger::get("DiskS3"), - "Read from file by path: " << backQuote(metadata_path + path) << " Existing S3 objects: " << metadata.s3_objects.size()); + LOG_DEBUG_FORMATTED(&Logger::get("DiskS3"), "Read from file by path: {}. Existing S3 objects: {}", + backQuote(metadata_path + path), metadata.s3_objects.size()); return std::make_unique(client, bucket, metadata, buf_size); } @@ -545,10 +544,8 @@ std::unique_ptr DiskS3::writeFile(const String & path, { Metadata metadata(metadata_path, path); - LOG_DEBUG( - &Logger::get("DiskS3"), - "Append to file by path: " << backQuote(metadata_path + path) << " New S3 path: " << s3_path - << " Existing S3 objects: " << metadata.s3_objects.size()); + LOG_DEBUG_FORMATTED(&Logger::get("DiskS3"), "Append to file by path: {}. New S3 path: {}. Existing S3 objects: {}.", + backQuote(metadata_path + path), s3_path, metadata.s3_objects.size()); return std::make_unique(client, bucket, metadata, s3_path, min_upload_part_size, buf_size); } @@ -619,10 +616,8 @@ bool DiskS3::tryReserve(UInt64 bytes) UInt64 unreserved_space = available_space - std::min(available_space, reserved_bytes); if (unreserved_space >= bytes) { - LOG_DEBUG( - &Logger::get("DiskS3"), - "Reserving " << formatReadableSizeWithBinarySuffix(bytes) << " on disk " << backQuote(name) << ", having unreserved " - << formatReadableSizeWithBinarySuffix(unreserved_space) << "."); + LOG_DEBUG_FORMATTED(&Logger::get("DiskS3"), "Reserving {} on disk {}, having unreserved {}.", + formatReadableSizeWithBinarySuffix(bytes), backQuote(name), formatReadableSizeWithBinarySuffix(unreserved_space)); ++reservation_count; reserved_bytes += bytes; return true; diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 51fdf57c50c..6dc53befc9d 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -761,8 +761,8 @@ void Aggregator::writeToTemporaryFile(AggregatedDataVariants & data_variants, co ProfileEvents::increment(ProfileEvents::ExternalAggregationCompressedBytes, compressed_bytes); ProfileEvents::increment(ProfileEvents::ExternalAggregationUncompressedBytes, uncompressed_bytes); - LOG_TRACE(log, std::fixed << std::setprecision(3) - << "Written part in " << elapsed_seconds << " sec., " + LOG_TRACE(log, + "Written part in " << elapsed_seconds << " sec., " << rows << " rows, " << (uncompressed_bytes / 1048576.0) << " MiB uncompressed, " << (compressed_bytes / 1048576.0) << " MiB compressed, " @@ -867,8 +867,8 @@ void Aggregator::writeToTemporaryFileImpl( /// `data_variants` will not destroy them in the destructor, they are now owned by ColumnAggregateFunction objects. data_variants.aggregator = nullptr; - LOG_TRACE(log, std::fixed << std::setprecision(3) - << "Max size of temporary block: " << max_temporary_block_size_rows << " rows, " + LOG_TRACE(log, + "Max size of temporary block: " << max_temporary_block_size_rows << " rows, " << (max_temporary_block_size_bytes / 1048576.0) << " MiB."); } @@ -939,8 +939,8 @@ void Aggregator::execute(const BlockInputStreamPtr & stream, AggregatedDataVaria double elapsed_seconds = watch.elapsedSeconds(); size_t rows = result.sizeWithoutOverflowRow(); - LOG_TRACE(log, std::fixed << std::setprecision(3) - << "Aggregated. " << src_rows << " to " << rows << " rows (from " << src_bytes / 1048576.0 << " MiB)" + LOG_TRACE(log, + "Aggregated. " << src_rows << " to " << rows << " rows (from " << src_bytes / 1048576.0 << " MiB)" << " in " << elapsed_seconds << " sec." << " (" << src_rows / elapsed_seconds << " rows/sec., " << src_bytes / elapsed_seconds / 1048576.0 << " MiB/sec.)"); } @@ -1308,8 +1308,8 @@ BlocksList Aggregator::convertToBlocks(AggregatedDataVariants & data_variants, b } double elapsed_seconds = watch.elapsedSeconds(); - LOG_TRACE(log, std::fixed << std::setprecision(3) - << "Converted aggregated data to blocks. " + LOG_TRACE(log, + "Converted aggregated data to blocks. " << rows << " rows, " << bytes / 1048576.0 << " MiB" << " in " << elapsed_seconds << " sec." << " (" << rows / elapsed_seconds << " rows/sec., " << bytes / elapsed_seconds / 1048576.0 << " MiB/sec.)"); @@ -2175,8 +2175,8 @@ Block Aggregator::mergeBlocks(BlocksList & blocks, bool final) size_t rows = block.rows(); size_t bytes = block.bytes(); double elapsed_seconds = watch.elapsedSeconds(); - LOG_TRACE(log, std::fixed << std::setprecision(3) - << "Merged partially aggregated blocks. " + LOG_TRACE(log, + "Merged partially aggregated blocks. " << rows << " rows, " << bytes / 1048576.0 << " MiB." << " in " << elapsed_seconds << " sec." << " (" << rows / elapsed_seconds << " rows/sec., " << bytes / elapsed_seconds / 1048576.0 << " MiB/sec.)"); diff --git a/src/Processors/Executors/PipelineExecutor.cpp b/src/Processors/Executors/PipelineExecutor.cpp index 5bf7ec0ff15..2a843f23319 100644 --- a/src/Processors/Executors/PipelineExecutor.cpp +++ b/src/Processors/Executors/PipelineExecutor.cpp @@ -547,12 +547,12 @@ void PipelineExecutor::executeSingleThread(size_t thread_num, size_t num_threads #ifndef NDEBUG auto & context = executor_contexts[thread_num]; - LOG_TRACE(log, std::fixed << std::setprecision(3) - << "Thread finished." - << " Total time: " << (context->total_time_ns / 1e9) << " sec." - << " Execution time: " << (context->execution_time_ns / 1e9) << " sec." - << " Processing time: " << (context->processing_time_ns / 1e9) << " sec." - << " Wait time: " << (context->wait_time_ns / 1e9) << " sec."); + LOG_TRACE(log, + "Thread finished." + << " Total time: " << (context->total_time_ns / 1e9) << " sec." + << " Execution time: " << (context->execution_time_ns / 1e9) << " sec." + << " Processing time: " << (context->processing_time_ns / 1e9) << " sec." + << " Wait time: " << (context->wait_time_ns / 1e9) << " sec."); #endif } diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index d0d521e3e2c..a14e649b4d1 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -540,10 +540,10 @@ void AggregatingTransform::initGenerate() double elapsed_seconds = watch.elapsedSeconds(); size_t rows = variants.sizeWithoutOverflowRow(); - LOG_TRACE(log, std::fixed << std::setprecision(3) - << "Aggregated. " << src_rows << " to " << rows << " rows (from " << src_bytes / 1048576.0 << " MiB)" - << " in " << elapsed_seconds << " sec." - << " (" << src_rows / elapsed_seconds << " rows/sec., " << src_bytes / elapsed_seconds / 1048576.0 << " MiB/sec.)"); + LOG_TRACE(log, + "Aggregated. " << src_rows << " to " << rows << " rows (from " << src_bytes / 1048576.0 << " MiB)" + << " in " << elapsed_seconds << " sec." + << " (" << src_rows / elapsed_seconds << " rows/sec., " << src_bytes / elapsed_seconds / 1048576.0 << " MiB/sec.)"); if (params->aggregator.hasTemporaryFiles()) { diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 76ce3daf820..11cac2245cb 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -949,8 +949,8 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor /// Print overall profiling info. NOTE: it may duplicates previous messages { double elapsed_seconds = merge_entry->watch.elapsedSeconds(); - LOG_DEBUG(log, std::fixed << std::setprecision(2) - << "Merge sorted " << merge_entry->rows_read << " rows" + LOG_DEBUG(log, + "Merge sorted " << merge_entry->rows_read << " rows" << ", containing " << all_column_names.size() << " columns" << " (" << merging_column_names.size() << " merged, " << gathering_column_names.size() << " gathered)" << " in " << elapsed_seconds << " sec., " diff --git a/src/Storages/MergeTree/MergeTreeReadPool.cpp b/src/Storages/MergeTree/MergeTreeReadPool.cpp index 8c8f240203b..5a51b9ee216 100644 --- a/src/Storages/MergeTree/MergeTreeReadPool.cpp +++ b/src/Storages/MergeTree/MergeTreeReadPool.cpp @@ -166,8 +166,8 @@ void MergeTreeReadPool::profileFeedback(const ReadBufferFromFileBase::ProfileInf ++backoff_state.num_events; ProfileEvents::increment(ProfileEvents::SlowRead); - LOG_DEBUG(log, std::fixed << std::setprecision(3) - << "Slow read, event №" << backoff_state.num_events + LOG_DEBUG(log, + "Slow read, event №" << backoff_state.num_events << ": read " << info.bytes_read << " bytes in " << info.nanoseconds / 1000000000.0 << " sec., " << info.bytes_read * 1000.0 / info.nanoseconds << " MB/s."); From fc412f2d9de4dc283283901c5b385823ded622fb Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 20:59:03 +0300 Subject: [PATCH 636/738] Progress on task --- src/DataStreams/MergingSortedBlockInputStream.cpp | 4 ++-- src/Storages/MergeTree/MergeTreeReadPool.cpp | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/DataStreams/MergingSortedBlockInputStream.cpp b/src/DataStreams/MergingSortedBlockInputStream.cpp index ae10617d45a..f78522ccffa 100644 --- a/src/DataStreams/MergingSortedBlockInputStream.cpp +++ b/src/DataStreams/MergingSortedBlockInputStream.cpp @@ -270,8 +270,8 @@ void MergingSortedBlockInputStream::readSuffixImpl() if (seconds) message << ", " - << profile_info.rows / seconds << " rows/sec., " - << profile_info.bytes / 1000000.0 / seconds << " MB/sec."; + << profile_info.rows / seconds << " rows/sec., " + << profile_info.bytes / 1000000.0 / seconds << " MB/sec."; LOG_DEBUG(log, message.str()); } diff --git a/src/Storages/MergeTree/MergeTreeReadPool.cpp b/src/Storages/MergeTree/MergeTreeReadPool.cpp index 5a51b9ee216..1fcabfafcab 100644 --- a/src/Storages/MergeTree/MergeTreeReadPool.cpp +++ b/src/Storages/MergeTree/MergeTreeReadPool.cpp @@ -1,6 +1,7 @@ #include -#include #include +#include +#include namespace ProfileEvents @@ -166,10 +167,9 @@ void MergeTreeReadPool::profileFeedback(const ReadBufferFromFileBase::ProfileInf ++backoff_state.num_events; ProfileEvents::increment(ProfileEvents::SlowRead); - LOG_DEBUG(log, - "Slow read, event №" << backoff_state.num_events - << ": read " << info.bytes_read << " bytes in " << info.nanoseconds / 1000000000.0 << " sec., " - << info.bytes_read * 1000.0 / info.nanoseconds << " MB/s."); + LOG_DEBUG_FORMATTED(log, "Slow read, event №{}: read {} bytes in {} sec., {}/s.", + backoff_state.num_events, info.bytes_read, info.nanoseconds / 1e9, + formatReadableSizeWithBinarySuffix(throughput)); if (backoff_state.num_events < backoff_settings.min_events) return; From 3384f3b77e6fe71f8b8f4e6b74fb48d54d6b94f9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 21:07:38 +0300 Subject: [PATCH 637/738] Progress on task --- .../MergingSortedBlockInputStream.cpp | 18 +++++++----------- .../Merges/MergingSortedTransform.cpp | 18 +++++++----------- .../MergeTree/MergeTreeDataMergerMutator.cpp | 11 ++++------- 3 files changed, 18 insertions(+), 29 deletions(-) diff --git a/src/DataStreams/MergingSortedBlockInputStream.cpp b/src/DataStreams/MergingSortedBlockInputStream.cpp index f78522ccffa..bba58fd19a3 100644 --- a/src/DataStreams/MergingSortedBlockInputStream.cpp +++ b/src/DataStreams/MergingSortedBlockInputStream.cpp @@ -263,17 +263,13 @@ void MergingSortedBlockInputStream::readSuffixImpl() const BlockStreamProfileInfo & profile_info = getProfileInfo(); double seconds = profile_info.total_stopwatch.elapsedSeconds(); - std::stringstream message; - message << std::fixed << std::setprecision(2) - << "Merge sorted " << profile_info.blocks << " blocks, " << profile_info.rows << " rows" - << " in " << seconds << " sec."; - - if (seconds) - message << ", " - << profile_info.rows / seconds << " rows/sec., " - << profile_info.bytes / 1000000.0 / seconds << " MB/sec."; - - LOG_DEBUG(log, message.str()); + if (!seconds) + LOG_DEBUG_FORMATTED(log, "Merge sorted {} blocks, {} rows in 0 sec.", profile_info.blocks, profile_info.rows); + else + LOG_DEBUG_FORMATTED(log, "Merge sorted {} blocks, {} rows in {} sec., {} rows/sec., {}/sec", + profile_info.blocks, profile_info.rows, seconds, + profile_info.rows / seconds, + formatReadableSizeWithBinarySuffix(profile_info.bytes / seconds)); } } diff --git a/src/Processors/Merges/MergingSortedTransform.cpp b/src/Processors/Merges/MergingSortedTransform.cpp index c7597e4da62..11246c30aaa 100644 --- a/src/Processors/Merges/MergingSortedTransform.cpp +++ b/src/Processors/Merges/MergingSortedTransform.cpp @@ -47,17 +47,13 @@ void MergingSortedTransform::onFinish() double seconds = total_stopwatch.elapsedSeconds(); - std::stringstream message; - message << std::fixed << std::setprecision(2) - << "Merge sorted " << merged_data.totalChunks() << " blocks, " << merged_data.totalMergedRows() << " rows" - << " in " << seconds << " sec."; - - if (seconds != 0) - message << ", " - << merged_data.totalMergedRows() / seconds << " rows/sec., " - << merged_data.totalAllocatedBytes() / 1000000.0 / seconds << " MB/sec."; - - LOG_DEBUG(log, message.str()); + if (!seconds) + LOG_DEBUG_FORMATTED(log, "Merge sorted {} blocks, {} rows in 0 sec.", merged_data.totalChunks(), merged_data.totalMergedRows()); + else + LOG_DEBUG_FORMATTED(log, "Merge sorted {} blocks, {} rows in {} sec., {} rows/sec., {}/sec", + merged_data.totalChunks(), merged_data.totalMergedRows(), seconds, + merged_data.totalMergedRows() / seconds, + formatReadableSizeWithBinarySuffix(merged_data.totalAllocatedBytes() / seconds)); } } diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 11cac2245cb..e124189b31b 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -949,13 +949,10 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor /// Print overall profiling info. NOTE: it may duplicates previous messages { double elapsed_seconds = merge_entry->watch.elapsedSeconds(); - LOG_DEBUG(log, - "Merge sorted " << merge_entry->rows_read << " rows" - << ", containing " << all_column_names.size() << " columns" - << " (" << merging_column_names.size() << " merged, " << gathering_column_names.size() << " gathered)" - << " in " << elapsed_seconds << " sec., " - << merge_entry->rows_read / elapsed_seconds << " rows/sec., " - << merge_entry->bytes_read_uncompressed / 1000000.0 / elapsed_seconds << " MB/sec."); + LOG_DEBUG_FORMATTED(log, "Merge sorted {} rows, containing {} columns ({} merged, {} gathered) in {} sec., {} rows/sec., {}/sec.", + merge_entry->rows_read, all_column_names.size(), merging_column_names.size(), gathering_column_names.size(), + elapsed_seconds, merge_entry->rows_read / elapsed_seconds, + formatReadableSizeWithBinarySuffix(merge_entry->bytes_read_uncompressed / elapsed_seconds)); } if (merge_alg != MergeAlgorithm::Vertical) From 45b6abdfee344851d627694de8a69d730a52bc24 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 21:22:40 +0300 Subject: [PATCH 638/738] Progress on task --- src/Client/Connection.cpp | 9 +++++---- src/DataStreams/ColumnGathererStream.cpp | 19 ++++++++----------- .../ParallelAggregatingBlockInputStream.cpp | 4 ++-- src/Interpreters/Aggregator.cpp | 16 ++++++++-------- .../Transforms/AggregatingTransform.cpp | 2 +- 5 files changed, 24 insertions(+), 26 deletions(-) diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index 0173291e76c..ce060197f4a 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -507,11 +507,12 @@ void Connection::sendScalarsData(Scalars & data) msg << std::fixed << std::setprecision(3); msg << "Sent data for " << data.size() << " scalars, total " << rows << " rows in " << elapsed << " sec., " << static_cast(rows / watch.elapsedSeconds()) << " rows/sec., " - << maybe_compressed_out_bytes / 1048576.0 << " MiB (" << maybe_compressed_out_bytes / 1048576.0 / watch.elapsedSeconds() << " MiB/sec.)"; + << formatReadableSizeWithBinarySuffix(maybe_compressed_out_bytes) << " (" + << formatReadableSizeWithBinarySuffix(maybe_compressed_out_bytes / watch.elapsedSeconds()) << "/sec.)"; if (compression == Protocol::Compression::Enable) msg << ", compressed " << static_cast(maybe_compressed_out_bytes) / out_bytes << " times to " - << out_bytes / 1048576.0 << " MiB (" << out_bytes / 1048576.0 / watch.elapsedSeconds() << " MiB/sec.)"; + << formatReadableSizeWithBinarySuffix(out_bytes) << " (" << formatReadableSizeWithBinarySuffix(out_bytes / watch.elapsedSeconds()) << "/sec.)"; else msg << ", no compression."; @@ -607,11 +608,11 @@ void Connection::sendExternalTablesData(ExternalTablesData & data) msg << std::fixed << std::setprecision(3); msg << "Sent data for " << data.size() << " external tables, total " << rows << " rows in " << elapsed << " sec., " << static_cast(rows / watch.elapsedSeconds()) << " rows/sec., " - << maybe_compressed_out_bytes / 1048576.0 << " MiB (" << maybe_compressed_out_bytes / 1048576.0 / watch.elapsedSeconds() << " MiB/sec.)"; + << formatReadableSizeWithBinarySuffix(maybe_compressed_out_bytes) << " (" << formatReadableSizeWithBinarySuffix(maybe_compressed_out_bytes / watch.elapsedSeconds()) << "/sec.)"; if (compression == Protocol::Compression::Enable) msg << ", compressed " << static_cast(maybe_compressed_out_bytes) / out_bytes << " times to " - << out_bytes / 1048576.0 << " MiB (" << out_bytes / 1048576.0 / watch.elapsedSeconds() << " MiB/sec.)"; + << formatReadableSizeWithBinarySuffix(out_bytes) << " (" << formatReadableSizeWithBinarySuffix(out_bytes / watch.elapsedSeconds()) << "/sec.)"; else msg << ", no compression."; diff --git a/src/DataStreams/ColumnGathererStream.cpp b/src/DataStreams/ColumnGathererStream.cpp index 6115de8c1ab..f828ab92a47 100644 --- a/src/DataStreams/ColumnGathererStream.cpp +++ b/src/DataStreams/ColumnGathererStream.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include @@ -98,17 +99,13 @@ void ColumnGathererStream::readSuffixImpl() double seconds = profile_info.total_stopwatch.elapsedSeconds(); - std::stringstream message; - message << std::fixed << std::setprecision(2) - << "Gathered column " << column_name - << " (" << static_cast(profile_info.bytes) / profile_info.rows << " bytes/elem.)" - << " in " << seconds << " sec."; - - if (seconds) - message << ", " << profile_info.rows / seconds << " rows/sec., " - << profile_info.bytes / 1048576.0 / seconds << " MiB/sec."; - - LOG_TRACE(log, message.str()); + if (!seconds) + LOG_DEBUG_FORMATTED(log, "Gathered column {} ({} bytes/elem.) in 0 sec.", + column_name, static_cast(profile_info.bytes) / profile_info.rows); + else + LOG_DEBUG_FORMATTED(log, "Gathered column {} ({} bytes/elem.) in {} sec., {} rows/sec., {}/sec.", + column_name, static_cast(profile_info.bytes) / profile_info.rows, seconds, + profile_info.rows / seconds, formatReadableSizeWithBinarySuffix(profile_info.bytes / seconds)); } } diff --git a/src/DataStreams/ParallelAggregatingBlockInputStream.cpp b/src/DataStreams/ParallelAggregatingBlockInputStream.cpp index 979999ab745..8fcaecafd2a 100644 --- a/src/DataStreams/ParallelAggregatingBlockInputStream.cpp +++ b/src/DataStreams/ParallelAggregatingBlockInputStream.cpp @@ -184,7 +184,7 @@ void ParallelAggregatingBlockInputStream::execute() << " (from " << threads_data[i].src_bytes / 1048576.0 << " MiB)" << " in " << elapsed_seconds << " sec." << " (" << threads_data[i].src_rows / elapsed_seconds << " rows/sec., " - << threads_data[i].src_bytes / elapsed_seconds / 1048576.0 << " MiB/sec.)"); + << formatReadableSizeWithBinarySuffix(threads_data[i].src_bytes / elapsed_seconds) << "/sec.)"); total_src_rows += threads_data[i].src_rows; total_src_bytes += threads_data[i].src_bytes; @@ -192,7 +192,7 @@ void ParallelAggregatingBlockInputStream::execute() LOG_TRACE(log, "Total aggregated. " << total_src_rows << " rows (from " << total_src_bytes / 1048576.0 << " MiB)" << " in " << elapsed_seconds << " sec." - << " (" << total_src_rows / elapsed_seconds << " rows/sec., " << total_src_bytes / elapsed_seconds / 1048576.0 << " MiB/sec.)"); + << " (" << total_src_rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(total_src_bytes / elapsed_seconds) << "/sec.)"); /// If there was no data, and we aggregate without keys, we must return single row with the result of empty aggregation. /// To do this, we pass a block with zero rows to aggregate. diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 6dc53befc9d..73f8840e2c9 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -764,14 +764,14 @@ void Aggregator::writeToTemporaryFile(AggregatedDataVariants & data_variants, co LOG_TRACE(log, "Written part in " << elapsed_seconds << " sec., " << rows << " rows, " - << (uncompressed_bytes / 1048576.0) << " MiB uncompressed, " - << (compressed_bytes / 1048576.0) << " MiB compressed, " + << formatReadableSizeWithBinarySuffix(uncompressed_bytes) << " uncompressed, " + << formatReadableSizeWithBinarySuffix(compressed_bytes) << " compressed, " << (uncompressed_bytes / rows) << " uncompressed bytes per row, " << (compressed_bytes / rows) << " compressed bytes per row, " << "compression rate: " << (uncompressed_bytes / compressed_bytes) << " (" << (rows / elapsed_seconds) << " rows/sec., " - << (uncompressed_bytes / elapsed_seconds / 1048576.0) << " MiB/sec. uncompressed, " - << (compressed_bytes / elapsed_seconds / 1048576.0) << " MiB/sec. compressed)"); + << formatReadableSizeWithBinarySuffix(uncompressed_bytes / elapsed_seconds) << "/sec. uncompressed, " + << formatReadableSizeWithBinarySuffix(compressed_bytes / elapsed_seconds) << "/sec. compressed)"); } void Aggregator::writeToTemporaryFile(AggregatedDataVariants & data_variants) { @@ -940,9 +940,9 @@ void Aggregator::execute(const BlockInputStreamPtr & stream, AggregatedDataVaria double elapsed_seconds = watch.elapsedSeconds(); size_t rows = result.sizeWithoutOverflowRow(); LOG_TRACE(log, - "Aggregated. " << src_rows << " to " << rows << " rows (from " << src_bytes / 1048576.0 << " MiB)" + "Aggregated. " << src_rows << " to " << rows << " rows (from " << formatReadableSizeWithBinarySuffix(src_bytes) << ")" << " in " << elapsed_seconds << " sec." - << " (" << src_rows / elapsed_seconds << " rows/sec., " << src_bytes / elapsed_seconds / 1048576.0 << " MiB/sec.)"); + << " (" << src_rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(src_bytes / elapsed_seconds) << "/sec.)"); } @@ -1312,7 +1312,7 @@ BlocksList Aggregator::convertToBlocks(AggregatedDataVariants & data_variants, b "Converted aggregated data to blocks. " << rows << " rows, " << bytes / 1048576.0 << " MiB" << " in " << elapsed_seconds << " sec." - << " (" << rows / elapsed_seconds << " rows/sec., " << bytes / elapsed_seconds / 1048576.0 << " MiB/sec.)"); + << " (" << rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(bytes / elapsed_seconds) << "/sec.)"); return blocks; } @@ -2179,7 +2179,7 @@ Block Aggregator::mergeBlocks(BlocksList & blocks, bool final) "Merged partially aggregated blocks. " << rows << " rows, " << bytes / 1048576.0 << " MiB." << " in " << elapsed_seconds << " sec." - << " (" << rows / elapsed_seconds << " rows/sec., " << bytes / elapsed_seconds / 1048576.0 << " MiB/sec.)"); + << " (" << rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(bytes / elapsed_seconds) << "/sec.)"); if (isCancelled()) return {}; diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index a14e649b4d1..3672e23f302 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -543,7 +543,7 @@ void AggregatingTransform::initGenerate() LOG_TRACE(log, "Aggregated. " << src_rows << " to " << rows << " rows (from " << src_bytes / 1048576.0 << " MiB)" << " in " << elapsed_seconds << " sec." - << " (" << src_rows / elapsed_seconds << " rows/sec., " << src_bytes / elapsed_seconds / 1048576.0 << " MiB/sec.)"); + << " (" << src_rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(src_bytes / elapsed_seconds) << "/sec.)"); if (params->aggregator.hasTemporaryFiles()) { From 1f13515a65932a7123358c3168db0dfa889c8d34 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 21:31:37 +0300 Subject: [PATCH 639/738] Make all LOG in single line (preparation) --- base/daemon/BaseDaemon.cpp | 3 +- base/daemon/GraphiteWriter.h | 3 +- programs/copier/ClusterCopier.cpp | 137 ++++++------------ programs/copier/ClusterCopierApp.cpp | 6 +- programs/odbc-bridge/ODBCBridge.cpp | 7 +- programs/server/HTTPHandlerFactory.cpp | 8 +- programs/server/MySQLHandler.cpp | 14 +- programs/server/Server.cpp | 62 ++------ programs/server/TCPHandler.cpp | 18 +-- src/Common/MemoryTracker.cpp | 8 +- src/Common/PoolWithFailoverBase.h | 3 +- .../AggregatingBlockInputStream.cpp | 4 +- .../CollapsingFinalBlockInputStream.cpp | 4 +- .../CreatingSetsBlockInputStream.cpp | 4 +- .../MergeSortingBlockInputStream.cpp | 4 +- .../ParallelAggregatingBlockInputStream.cpp | 16 +- src/Databases/DatabaseOnDisk.cpp | 3 +- src/Interpreters/Aggregator.cpp | 36 +---- src/Interpreters/DDLWorker.cpp | 24 +-- src/Interpreters/DNSCacheUpdater.cpp | 3 +- src/Interpreters/DatabaseCatalog.cpp | 4 +- src/Interpreters/ExternalLoader.cpp | 62 ++------ src/Interpreters/SystemLog.cpp | 4 +- src/Interpreters/SystemLog.h | 22 +-- src/Interpreters/executeQuery.cpp | 17 +-- src/Processors/Executors/PipelineExecutor.cpp | 7 +- .../Formats/Impl/AvroRowInputFormat.cpp | 3 +- src/Processors/Merges/Algorithms/RowRef.h | 8 +- .../Transforms/AggregatingTransform.cpp | 9 +- .../Transforms/CreatingSetsTransform.cpp | 4 +- .../Transforms/MergeSortingTransform.cpp | 7 +- .../Transforms/MergingAggregatedTransform.cpp | 3 +- .../DistributedBlockOutputStream.cpp | 8 +- .../Kafka/ReadBufferFromKafkaConsumer.cpp | 14 +- src/Storages/Kafka/parseSyslogLevel.cpp | 21 +-- .../MergeTree/EphemeralLockInZooKeeper.cpp | 3 +- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 18 +-- src/Storages/MergeTree/MergeTreeData.cpp | 36 ++--- .../MergeTree/MergeTreeDataMergerMutator.cpp | 19 +-- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 3 +- .../MergeTree/MergeTreePartsMover.cpp | 7 +- .../MergeTreeReverseSelectProcessor.cpp | 7 +- .../MergeTree/MergeTreeSelectProcessor.cpp | 7 +- .../ReplicatedMergeTreeCleanupThread.cpp | 6 +- .../ReplicatedMergeTreePartCheckThread.cpp | 14 +- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 40 ++--- .../ReplicatedMergeTreeRestartingThread.cpp | 6 +- src/Storages/StorageBuffer.cpp | 24 +-- src/Storages/StorageDistributed.cpp | 7 +- src/Storages/StorageReplicatedMergeTree.cpp | 86 +++-------- 50 files changed, 217 insertions(+), 626 deletions(-) diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index b091d01dea0..818721b9e6d 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -498,8 +498,7 @@ void debugIncreaseOOMScore() } catch (const Poco::Exception & e) { - LOG_WARNING(&Logger::root(), "Failed to adjust OOM score: '" + - e.displayText() + "'."); + LOG_WARNING(&Logger::root(), "Failed to adjust OOM score: '" + e.displayText() + "'."); return; } LOG_INFO_FORMATTED(&Logger::root(), "Set OOM score adjustment to {}", new_score); diff --git a/base/daemon/GraphiteWriter.h b/base/daemon/GraphiteWriter.h index a19f6396831..d368470e5e6 100644 --- a/base/daemon/GraphiteWriter.h +++ b/base/daemon/GraphiteWriter.h @@ -52,8 +52,7 @@ private: } catch (const Poco::Exception & e) { - LOG_WARNING(&Poco::Util::Application::instance().logger(), - "Fail to write to Graphite " << host << ":" << port << ". e.what() = " << e.what() << ", e.message() = " << e.message()); + LOG_WARNING(&Poco::Util::Application::instance().logger(), "Fail to write to Graphite " << host << ":" << port << ". e.what() = " << e.what() << ", e.message() = " << e.message()); } } diff --git a/programs/copier/ClusterCopier.cpp b/programs/copier/ClusterCopier.cpp index 50804d5a8a1..5ad2f6497e6 100644 --- a/programs/copier/ClusterCopier.cpp +++ b/programs/copier/ClusterCopier.cpp @@ -132,8 +132,7 @@ void ClusterCopier::discoverShardPartitions(const ConnectionTimeouts & timeouts, { if (!task_table.enabled_partitions_set.count(partition_name)) { - LOG_DEBUG(log, "Partition " << partition_name << " will not be processed, since it is not in " - << "enabled_partitions of " << task_table.table_id); + LOG_DEBUG(log, "Partition " << partition_name << " will not be processed, since it is not in " << "enabled_partitions of " << task_table.table_id); } } } @@ -165,8 +164,7 @@ void ClusterCopier::discoverShardPartitions(const ConnectionTimeouts & timeouts, for (const String & missing_partition : missing_partitions) ss << " " << missing_partition; - LOG_WARNING(log, "There are no " << missing_partitions.size() << " partitions from enabled_partitions in shard " - << task_shard->getDescription() << " :" << ss.str()); + LOG_WARNING(log, "There are no " << missing_partitions.size() << " partitions from enabled_partitions in shard " << task_shard->getDescription() << " :" << ss.str()); } LOG_DEBUG_FORMATTED(log, "Will copy {} partitions from shard {}", task_shard->partition_tasks.size(), task_shard->getDescription()); @@ -251,9 +249,7 @@ void ClusterCopier::process(const ConnectionTimeouts & timeouts) { for (TaskTable & task_table : task_cluster->table_tasks) { - LOG_INFO(log, "Process table task " << task_table.table_id << " with " - << task_table.all_shards.size() << " shards, " - << task_table.local_shards.size() << " of them are local ones"); + LOG_INFO(log, "Process table task " << task_table.table_id << " with " << task_table.all_shards.size() << " shards, " << task_table.local_shards.size() << " of them are local ones"); if (task_table.all_shards.empty()) continue; @@ -357,8 +353,7 @@ zkutil::EphemeralNodeHolder::Ptr ClusterCopier::createTaskWorkerNodeAndWaitIfNee if (static_cast(stat.numChildren) >= task_cluster->max_workers) { - LOG_DEBUG(log, "Too many workers (" << stat.numChildren << ", maximum " << task_cluster->max_workers << ")" - << ". Postpone processing " << description); + LOG_DEBUG(log, "Too many workers (" << stat.numChildren << ", maximum " << task_cluster->max_workers << ")" << ". Postpone processing " << description); if (unprioritized) current_sleep_time = std::min(max_sleep_time, current_sleep_time + default_sleep_time); @@ -435,8 +430,7 @@ bool ClusterCopier::checkAllPiecesInPartitionAreDone(const TaskTable & task_tabl bool ClusterCopier::checkPartitionPieceIsDone(const TaskTable & task_table, const String & partition_name, size_t piece_number, const TasksShard & shards_with_partition) { - LOG_DEBUG(log, "Check that all shards processed partition " << partition_name - << " piece " + toString(piece_number) + " successfully"); + LOG_DEBUG(log, "Check that all shards processed partition " << partition_name << " piece " + toString(piece_number) + " successfully"); auto zookeeper = context.getZooKeeper(); @@ -465,8 +459,7 @@ bool ClusterCopier::checkPartitionPieceIsDone(const TaskTable & task_table, cons TaskStateWithOwner status = TaskStateWithOwner::fromString(res.data); if (status.state != TaskState::Finished) { - LOG_INFO(log, "The task " << res.data << " is being rewritten by " - << status.owner << ". Partition piece will be rechecked"); + LOG_INFO(log, "The task " << res.data << " is being rewritten by " << status.owner << ". Partition piece will be rechecked"); return false; } @@ -501,8 +494,7 @@ bool ClusterCopier::checkPartitionPieceIsDone(const TaskTable & task_table, cons } catch (const Coordination::Exception & e) { - LOG_INFO(log, "A ZooKeeper error occurred while checking partition " << partition_name << " piece number " - << toString(piece_number) << ". Will recheck the partition. Error: " << e.displayText()); + LOG_INFO(log, "A ZooKeeper error occurred while checking partition " << partition_name << " piece number " << toString(piece_number) << ". Will recheck the partition. Error: " << e.displayText()); return false; } @@ -565,16 +557,13 @@ TaskStatus ClusterCopier::tryMoveAllPiecesToDestinationTable(const TaskTable & t TaskStateWithOwner status = TaskStateWithOwner::fromString(status_data); if (status.state == TaskState::Finished) { - LOG_DEBUG(log, "All pieces for partition from this task " << current_partition_attach_is_active - << " has been successfully moved to destination table by " << status.owner); + LOG_DEBUG(log, "All pieces for partition from this task " << current_partition_attach_is_active << " has been successfully moved to destination table by " << status.owner); return TaskStatus::Finished; } /// Task is abandoned, because previously we created ephemeral node, possibly in other copier's process. /// Initialize DROP PARTITION - LOG_DEBUG(log, "Moving piece for partition " << current_partition_attach_is_active - << " has not been successfully finished by " << status.owner - << ". Will try to move by myself."); + LOG_DEBUG(log, "Moving piece for partition " << current_partition_attach_is_active << " has not been successfully finished by " << status.owner << ". Will try to move by myself."); /// Remove is_done marker. zookeeper->remove(current_partition_attach_is_done); @@ -591,9 +580,7 @@ TaskStatus ClusterCopier::tryMoveAllPiecesToDestinationTable(const TaskTable & t /// Move partition to original destination table. for (size_t current_piece_number = 0; current_piece_number < task_table.number_of_splits; ++current_piece_number) { - LOG_DEBUG(log, "Trying to move partition " << partition_name - << " piece " << toString(current_piece_number) - << " to original table"); + LOG_DEBUG(log, "Trying to move partition " << partition_name << " piece " << toString(current_piece_number) << " to original table"); ASTPtr query_alter_ast; String query_alter_ast_string; @@ -630,9 +617,7 @@ TaskStatus ClusterCopier::tryMoveAllPiecesToDestinationTable(const TaskTable & t } catch (...) { - LOG_DEBUG(log, "Error while moving partition " << partition_name - << " piece " << toString(current_piece_number) - << "to original table"); + LOG_DEBUG(log, "Error while moving partition " << partition_name << " piece " << toString(current_piece_number) << "to original table"); throw; } @@ -656,14 +641,12 @@ TaskStatus ClusterCopier::tryMoveAllPiecesToDestinationTable(const TaskTable & t &task_cluster->settings_push, PoolMode::GET_MANY); - LOG_INFO(log, "Number of shard that executed OPTIMIZE DEDUPLICATE query successfully : " - << toString(num_nodes)); + LOG_INFO(log, "Number of shard that executed OPTIMIZE DEDUPLICATE query successfully : " << toString(num_nodes)); } } catch (...) { - LOG_DEBUG(log, "Error while executing OPTIMIZE DEDUPLICATE partition " << partition_name - << "in the original table"); + LOG_DEBUG(log, "Error while executing OPTIMIZE DEDUPLICATE partition " << partition_name << "in the original table"); throw; } } @@ -759,8 +742,7 @@ bool ClusterCopier::tryDropPartitionPiece( { if (e.code == Coordination::ZNODEEXISTS) { - LOG_DEBUG(log, "Partition " << task_partition.name << " piece " - << toString(current_piece_number) << " is cleaning now by somebody, sleep"); + LOG_DEBUG(log, "Partition " << task_partition.name << " piece " << toString(current_piece_number) << " is cleaning now by somebody, sleep"); std::this_thread::sleep_for(default_sleep_time); return false; } @@ -773,8 +755,7 @@ bool ClusterCopier::tryDropPartitionPiece( { if (stat.numChildren != 0) { - LOG_DEBUG(log, "Partition " << task_partition.name << " contains " << stat.numChildren - << " active workers while trying to drop it. Going to sleep."); + LOG_DEBUG(log, "Partition " << task_partition.name << " contains " << stat.numChildren << " active workers while trying to drop it. Going to sleep."); std::this_thread::sleep_for(default_sleep_time); return false; } @@ -859,8 +840,7 @@ bool ClusterCopier::tryDropPartitionPiece( return false; } - LOG_INFO(log, "Partition " << task_partition.name << " piece " << toString(current_piece_number) - << " was dropped on cluster " << task_table.cluster_push_name); + LOG_INFO(log, "Partition " << task_partition.name << " piece " << toString(current_piece_number) << " was dropped on cluster " << task_table.cluster_push_name); if (zookeeper->tryCreate(current_shards_path, host_id, zkutil::CreateMode::Persistent) == Coordination::ZNODEEXISTS) zookeeper->set(current_shards_path, host_id); } @@ -1030,21 +1010,16 @@ bool ClusterCopier::tryProcessTable(const ConnectionTimeouts & timeouts, TaskTab task_table.rows_copied += cluster_partition.rows_copied; double elapsed = cluster_partition.elapsed_time_seconds; - LOG_INFO(log, "It took " << std::fixed << std::setprecision(2) << elapsed << " seconds to copy partition " << partition_name - << ": " << formatReadableSizeWithDecimalSuffix(cluster_partition.bytes_copied) << " uncompressed bytes" - << ", " << formatReadableQuantity(cluster_partition.rows_copied) << " rows" - << " and " << cluster_partition.blocks_copied << " source blocks are copied"); + LOG_INFO(log, "It took " << std::fixed << std::setprecision(2) << elapsed << " seconds to copy partition " << partition_name << ": " << formatReadableSizeWithDecimalSuffix(cluster_partition.bytes_copied) << " uncompressed bytes" << ", " << formatReadableQuantity(cluster_partition.rows_copied) << " rows" << " and " << cluster_partition.blocks_copied << " source blocks are copied"); if (cluster_partition.rows_copied) { - LOG_INFO(log, "Average partition speed: " - << formatReadableSizeWithDecimalSuffix(cluster_partition.bytes_copied / elapsed) << " per second."); + LOG_INFO(log, "Average partition speed: " << formatReadableSizeWithDecimalSuffix(cluster_partition.bytes_copied / elapsed) << " per second."); } if (task_table.rows_copied) { - LOG_INFO(log, "Average table " << task_table.table_id << " speed: " - << formatReadableSizeWithDecimalSuffix(task_table.bytes_copied / elapsed) << " per second."); + LOG_INFO(log, "Average table " << task_table.table_id << " speed: " << formatReadableSizeWithDecimalSuffix(task_table.bytes_copied / elapsed) << " per second."); } } } @@ -1055,8 +1030,7 @@ bool ClusterCopier::tryProcessTable(const ConnectionTimeouts & timeouts, TaskTab if (!table_is_done) { - LOG_INFO(log, "Table " + task_table.table_id + " is not processed yet." - << "Copied " << finished_partitions << " of " << required_partitions << ", will retry"); + LOG_INFO(log, "Table " + task_table.table_id + " is not processed yet." << "Copied " << finished_partitions << " of " << required_partitions << ", will retry"); } return table_is_done; @@ -1104,9 +1078,7 @@ TaskStatus ClusterCopier::iterateThroughAllPiecesInPartition(const ConnectionTim { for (UInt64 try_num = 0; try_num < max_shard_partition_tries; ++try_num) { - LOG_INFO(log, "Attempt number " << try_num << " to process partition " << task_partition.name - << " piece number " << piece_number << " on shard number " << task_partition.task_shard.numberInCluster() - << " with index " << task_partition.task_shard.indexInCluster()); + LOG_INFO(log, "Attempt number " << try_num << " to process partition " << task_partition.name << " piece number " << piece_number << " on shard number " << task_partition.task_shard.numberInCluster() << " with index " << task_partition.task_shard.indexInCluster()); res = processPartitionPieceTaskImpl(timeouts, task_partition, piece_number, is_unprioritized_task); /// Exit if success @@ -1221,14 +1193,12 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( /// Do not start if partition piece is dirty, try to clean it if (is_clean) { - LOG_DEBUG(log, "Partition " << task_partition.name - << " piece " + toString(current_piece_number) + " appears to be clean"); + LOG_DEBUG(log, "Partition " << task_partition.name << " piece " + toString(current_piece_number) + " appears to be clean"); zookeeper->createAncestors(current_task_piece_status_path); } else { - LOG_DEBUG(log, "Partition " << task_partition.name - << " piece " + toString(current_piece_number) + " is dirty, try to drop it"); + LOG_DEBUG(log, "Partition " << task_partition.name << " piece " + toString(current_piece_number) + " is dirty, try to drop it"); try { @@ -1269,16 +1239,13 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( TaskStateWithOwner status = TaskStateWithOwner::fromString(status_data); if (status.state == TaskState::Finished) { - LOG_DEBUG(log, "Task " << current_task_piece_status_path - << " has been successfully executed by " << status.owner); + LOG_DEBUG(log, "Task " << current_task_piece_status_path << " has been successfully executed by " << status.owner); return TaskStatus::Finished; } /// Task is abandoned, because previously we created ephemeral node, possibly in other copier's process. /// Initialize DROP PARTITION - LOG_DEBUG(log, "Task " << current_task_piece_status_path - << " has not been successfully finished by " << status.owner - << ". Partition will be dropped and refilled."); + LOG_DEBUG(log, "Task " << current_task_piece_status_path << " has not been successfully finished by " << status.owner << ". Partition will be dropped and refilled."); create_is_dirty_node(clean_state_clock); return TaskStatus::Error; @@ -1293,11 +1260,9 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( String state_finished = TaskStateWithOwner::getData(TaskState::Finished, host_id); auto res = zookeeper->tryCreate(current_task_piece_status_path, state_finished, zkutil::CreateMode::Persistent); if (res == Coordination::ZNODEEXISTS) - LOG_DEBUG(log, "Partition " << task_partition.name << " piece " - + toString(current_piece_number) + " is absent on current replica of a shard. But other replicas have already marked it as done."); + LOG_DEBUG(log, "Partition " << task_partition.name << " piece " + toString(current_piece_number) + " is absent on current replica of a shard. But other replicas have already marked it as done."); if (res == Coordination::ZOK) - LOG_DEBUG(log, "Partition " << task_partition.name << " piece " - + toString(current_piece_number) + " is absent on current replica of a shard. Will mark it as done. Other replicas will do the same."); + LOG_DEBUG(log, "Partition " << task_partition.name << " piece " + toString(current_piece_number) + " is absent on current replica of a shard. Will mark it as done. Other replicas will do the same."); return TaskStatus::Finished; } @@ -1325,18 +1290,14 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( if (count != 0) { - LOG_INFO(log, "Partition " << task_partition.name << " piece " - << current_piece_number << "is not empty. In contains " << count << " rows."); + LOG_INFO(log, "Partition " << task_partition.name << " piece " << current_piece_number << "is not empty. In contains " << count << " rows."); Coordination::Stat stat_shards{}; zookeeper->get(partition_piece.getPartitionPieceShardsPath(), &stat_shards); /// NOTE: partition is still fresh if dirt discovery happens before cleaning if (stat_shards.numChildren == 0) { - LOG_WARNING(log, "There are no workers for partition " << task_partition.name - << " piece " << toString(current_piece_number) - << ", but destination table contains " << count << " rows" - << ". Partition will be dropped and refilled."); + LOG_WARNING(log, "There are no workers for partition " << task_partition.name << " piece " << toString(current_piece_number) << ", but destination table contains " << count << " rows" << ". Partition will be dropped and refilled."); create_is_dirty_node(clean_state_clock); return TaskStatus::Error; @@ -1353,14 +1314,12 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( CleanStateClock new_clean_state_clock (zookeeper, piece_is_dirty_flag_path, piece_is_dirty_cleaned_path); if (clean_state_clock != new_clean_state_clock) { - LOG_INFO(log, "Partition " << task_partition.name << " piece " - << toString(current_piece_number) << " clean state changed, cowardly bailing"); + LOG_INFO(log, "Partition " << task_partition.name << " piece " << toString(current_piece_number) << " clean state changed, cowardly bailing"); return TaskStatus::Error; } else if (!new_clean_state_clock.is_clean()) { - LOG_INFO(log, "Partition " << task_partition.name << " piece " - << toString(current_piece_number) << " is dirty and will be dropped and refilled"); + LOG_INFO(log, "Partition " << task_partition.name << " piece " << toString(current_piece_number) << " is dirty and will be dropped and refilled"); create_is_dirty_node(new_clean_state_clock); return TaskStatus::Error; } @@ -1391,8 +1350,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( UInt64 shards = executeQueryOnCluster(task_table.cluster_push, query, create_query_push_ast, &task_cluster->settings_push, PoolMode::GET_MANY); - LOG_DEBUG(log, "Destination tables " << getQuotedTable(task_table.table_push) - << " have been created on " << shards << " shards of " << task_table.cluster_push->getShardCount()); + LOG_DEBUG(log, "Destination tables " << getQuotedTable(task_table.table_push) << " have been created on " << shards << " shards of " << task_table.cluster_push->getShardCount()); } /// Do the copying @@ -1407,8 +1365,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( // Select all fields ASTPtr query_select_ast = get_select_query(task_shard.table_read_shard, "*", /*enable_splitting*/ true, inject_fault ? "1" : ""); - LOG_DEBUG(log, "Executing SELECT query and pull from " << task_shard.getDescription() - << " : " << queryToString(query_select_ast)); + LOG_DEBUG(log, "Executing SELECT query and pull from " << task_shard.getDescription() << " : " << queryToString(query_select_ast)); ASTPtr query_insert_ast; { @@ -1501,8 +1458,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( } } - LOG_INFO(log, "Partition " << task_partition.name << " piece " - << toString(current_piece_number) << " copied. But not moved to original destination table."); + LOG_INFO(log, "Partition " << task_partition.name << " piece " << toString(current_piece_number) << " copied. But not moved to original destination table."); /// Try create original table (if not exists) on each shard @@ -1517,8 +1473,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( UInt64 shards = executeQueryOnCluster(task_table.cluster_push, query, create_query_push_ast, &task_cluster->settings_push, PoolMode::GET_MANY); - LOG_DEBUG(log, "Destination tables " << getQuotedTable(task_table.table_push) - << " have been created on " << shards << " shards of " << task_table.cluster_push->getShardCount()); + LOG_DEBUG(log, "Destination tables " << getQuotedTable(task_table.table_push) << " have been created on " << shards << " shards of " << task_table.cluster_push->getShardCount()); } catch (...) { @@ -1531,14 +1486,12 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( CleanStateClock new_clean_state_clock (zookeeper, piece_is_dirty_flag_path, piece_is_dirty_cleaned_path); if (clean_state_clock != new_clean_state_clock) { - LOG_INFO(log, "Partition " << task_partition.name << " piece " - << toString(current_piece_number) << " clean state changed, cowardly bailing"); + LOG_INFO(log, "Partition " << task_partition.name << " piece " << toString(current_piece_number) << " clean state changed, cowardly bailing"); return TaskStatus::Error; } else if (!new_clean_state_clock.is_clean()) { - LOG_INFO(log, "Partition " << task_partition.name << " piece " - << toString(current_piece_number) << " became dirty and will be dropped and refilled"); + LOG_INFO(log, "Partition " << task_partition.name << " piece " << toString(current_piece_number) << " became dirty and will be dropped and refilled"); create_is_dirty_node(new_clean_state_clock); return TaskStatus::Error; } @@ -1765,8 +1718,7 @@ bool ClusterCopier::checkShardHasPartition(const ConnectionTimeouts & timeouts, query += " LIMIT 1"; - LOG_DEBUG(log, "Checking shard " << task_shard.getDescription() << " for partition " - << partition_quoted_name << " existence, executing query: " << query); + LOG_DEBUG(log, "Checking shard " << task_shard.getDescription() << " for partition " << partition_quoted_name << " existence, executing query: " << query); ParserQuery parser_query(query.data() + query.size()); const auto & settings = context.getSettingsRef(); @@ -1805,9 +1757,7 @@ bool ClusterCopier::checkPresentPartitionPiecesOnCurrentShard(const ConnectionTi query += " LIMIT 1"; - LOG_DEBUG(log, "Checking shard " << task_shard.getDescription() << " for partition " - << partition_quoted_name << " piece " << std::to_string(current_piece_number) - << "existence, executing query: " << query); + LOG_DEBUG(log, "Checking shard " << task_shard.getDescription() << " for partition " << partition_quoted_name << " piece " << std::to_string(current_piece_number) << "existence, executing query: " << query); ParserQuery parser_query(query.data() + query.size()); const auto & settings = context.getSettingsRef(); @@ -1817,11 +1767,9 @@ bool ClusterCopier::checkPresentPartitionPiecesOnCurrentShard(const ConnectionTi local_context.setSettings(task_cluster->settings_pull); auto result = InterpreterFactory::get(query_ast, local_context)->execute().in->read().rows(); if (result != 0) - LOG_DEBUG(log, "Partition " << partition_quoted_name << " piece number " - << std::to_string(current_piece_number) << " is PRESENT on shard " << task_shard.getDescription()); + LOG_DEBUG(log, "Partition " << partition_quoted_name << " piece number " << std::to_string(current_piece_number) << " is PRESENT on shard " << task_shard.getDescription()); else - LOG_DEBUG(log, "Partition " << partition_quoted_name << " piece number " - << std::to_string(current_piece_number) << " is ABSENT on shard " << task_shard.getDescription()); + LOG_DEBUG(log, "Partition " << partition_quoted_name << " piece number " << std::to_string(current_piece_number) << " is ABSENT on shard " << task_shard.getDescription()); return result != 0; } @@ -1938,8 +1886,7 @@ UInt64 ClusterCopier::executeQueryOnCluster( if (execution_mode == ClusterExecutionMode::ON_EACH_NODE && successful_nodes != origin_replicas_number) { - LOG_INFO(log, "There was an error while executing ALTER on each node. Query was executed on " - << toString(successful_nodes) << " nodes. But had to be executed on " << toString(origin_replicas_number.load())); + LOG_INFO(log, "There was an error while executing ALTER on each node. Query was executed on " << toString(successful_nodes) << " nodes. But had to be executed on " << toString(origin_replicas_number.load())); } diff --git a/programs/copier/ClusterCopierApp.cpp b/programs/copier/ClusterCopierApp.cpp index 5ab6064f0f3..1afbdba2c4f 100644 --- a/programs/copier/ClusterCopierApp.cpp +++ b/programs/copier/ClusterCopierApp.cpp @@ -95,11 +95,7 @@ void ClusterCopierApp::mainImpl() ThreadStatus thread_status; auto * log = &logger(); - LOG_INFO(log, "Starting clickhouse-copier (" - << "id " << process_id << ", " - << "host_id " << host_id << ", " - << "path " << process_path << ", " - << "revision " << ClickHouseRevision::get() << ")"); + LOG_INFO(log, "Starting clickhouse-copier (" << "id " << process_id << ", " << "host_id " << host_id << ", " << "path " << process_path << ", " << "revision " << ClickHouseRevision::get() << ")"); SharedContextHolder shared_context = Context::createShared(); auto context = std::make_unique(Context::createGlobal(shared_context.get())); diff --git a/programs/odbc-bridge/ODBCBridge.cpp b/programs/odbc-bridge/ODBCBridge.cpp index 991eda151cf..fefe927142a 100644 --- a/programs/odbc-bridge/ODBCBridge.cpp +++ b/programs/odbc-bridge/ODBCBridge.cpp @@ -48,12 +48,7 @@ namespace #endif ) { - LOG_ERROR(log, - "Cannot resolve listen_host (" << host << "), error " << e.code() << ": " << e.message() - << ". " - "If it is an IPv6 address and your host has disabled IPv6, then consider to " - "specify IPv4 address to listen in element of configuration " - "file. Example: 0.0.0.0"); + LOG_ERROR(log, "Cannot resolve listen_host (" << host << "), error " << e.code() << ": " << e.message() << ". " "If it is an IPv6 address and your host has disabled IPv6, then consider to " "specify IPv4 address to listen in element of configuration " "file. Example: 0.0.0.0"); } throw; diff --git a/programs/server/HTTPHandlerFactory.cpp b/programs/server/HTTPHandlerFactory.cpp index 955ff6b8834..9074df20b15 100644 --- a/programs/server/HTTPHandlerFactory.cpp +++ b/programs/server/HTTPHandlerFactory.cpp @@ -30,13 +30,7 @@ HTTPRequestHandlerFactoryMain::HTTPRequestHandlerFactoryMain(const std::string & Poco::Net::HTTPRequestHandler * HTTPRequestHandlerFactoryMain::createRequestHandler(const Poco::Net::HTTPServerRequest & request) { - LOG_TRACE(log, "HTTP Request for " << name << ". " - << "Method: " << request.getMethod() - << ", Address: " << request.clientAddress().toString() - << ", User-Agent: " << (request.has("User-Agent") ? request.get("User-Agent") : "none") - << (request.hasContentLength() ? (", Length: " + std::to_string(request.getContentLength())) : ("")) - << ", Content Type: " << request.getContentType() - << ", Transfer Encoding: " << request.getTransferEncoding()); + LOG_TRACE(log, "HTTP Request for " << name << ". " << "Method: " << request.getMethod() << ", Address: " << request.clientAddress().toString() << ", User-Agent: " << (request.has("User-Agent") ? request.get("User-Agent") : "none") << (request.hasContentLength() ? (", Length: " + std::to_string(request.getContentLength())) : ("")) << ", Content Type: " << request.getContentType() << ", Transfer Encoding: " << request.getTransferEncoding()); for (auto & handler_factory : child_factories) { diff --git a/programs/server/MySQLHandler.cpp b/programs/server/MySQLHandler.cpp index c1de7bfcb58..cc37b8a73b0 100644 --- a/programs/server/MySQLHandler.cpp +++ b/programs/server/MySQLHandler.cpp @@ -83,19 +83,7 @@ void MySQLHandler::run() if (!connection_context.mysql.max_packet_size) connection_context.mysql.max_packet_size = MAX_PACKET_LENGTH; - LOG_TRACE(log, "Capabilities: " << handshake_response.capability_flags - << ", max_packet_size: " - << handshake_response.max_packet_size - << ", character_set: " - << static_cast(handshake_response.character_set) - << ", user: " - << handshake_response.username - << ", auth_response length: " - << handshake_response.auth_response.length() - << ", database: " - << handshake_response.database - << ", auth_plugin_name: " - << handshake_response.auth_plugin_name); + LOG_TRACE(log, "Capabilities: " << handshake_response.capability_flags << ", max_packet_size: " << handshake_response.max_packet_size << ", character_set: " << static_cast(handshake_response.character_set) << ", user: " << handshake_response.username << ", auth_response length: " << handshake_response.auth_response.length() << ", database: " << handshake_response.database << ", auth_plugin_name: " << handshake_response.auth_plugin_name); client_capability_flags = handshake_response.capability_flags; if (!(client_capability_flags & CLIENT_PROTOCOL_41)) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 517a32f8d91..5ee2b3bd631 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -282,10 +282,7 @@ int Server::main(const std::vector & /*args*/) } else { - LOG_INFO(log, "It looks like the process has no CAP_IPC_LOCK capability, binary mlock will be disabled." - " It could happen due to incorrect ClickHouse package installation." - " You could resolve the problem manually with 'sudo setcap cap_ipc_lock=+ep " << executable_path << "'." - " Note that it will not work on 'nosuid' mounted filesystems."); + LOG_INFO(log, "It looks like the process has no CAP_IPC_LOCK capability, binary mlock will be disabled." " It could happen due to incorrect ClickHouse package installation." " You could resolve the problem manually with 'sudo setcap cap_ipc_lock=+ep " << executable_path << "'." " Note that it will not work on 'nosuid' mounted filesystems."); } } } @@ -357,10 +354,7 @@ int Server::main(const std::vector & /*args*/) rlim.rlim_cur = config().getUInt("max_open_files", rlim.rlim_max); int rc = setrlimit(RLIMIT_NOFILE, &rlim); if (rc != 0) - LOG_WARNING(log, - "Cannot set max number of file descriptors to " << rlim.rlim_cur - << ". Try to specify max_open_files according to your system limits. error: " - << strerror(errno)); + LOG_WARNING(log, "Cannot set max number of file descriptors to " << rlim.rlim_cur << ". Try to specify max_open_files according to your system limits. error: " << strerror(errno)); else LOG_DEBUG_FORMATTED(log, "Set max number of file descriptors to {} (was {}).", rlim.rlim_cur, old); } @@ -431,9 +425,7 @@ int Server::main(const std::vector & /*args*/) if (this_host.empty()) { this_host = getFQDNOrHostName(); - LOG_DEBUG(log, - "Configuration parameter '" + String(host_tag) + "' doesn't exist or exists and empty. Will use '" + this_host - + "' as replica host."); + LOG_DEBUG(log, "Configuration parameter '" + String(host_tag) + "' doesn't exist or exists and empty. Will use '" + this_host + "' as replica host."); } String port_str = config().getString(port_tag); @@ -538,8 +530,7 @@ int Server::main(const std::vector & /*args*/) if (uncompressed_cache_size > max_cache_size) { uncompressed_cache_size = max_cache_size; - LOG_INFO(log, "Uncompressed cache size was lowered to " << formatReadableSizeWithBinarySuffix(uncompressed_cache_size) - << " because the system has low amount of memory"); + LOG_INFO(log, "Uncompressed cache size was lowered to " << formatReadableSizeWithBinarySuffix(uncompressed_cache_size) << " because the system has low amount of memory"); } global_context->setUncompressedCache(uncompressed_cache_size); @@ -554,8 +545,7 @@ int Server::main(const std::vector & /*args*/) if (mark_cache_size > max_cache_size) { mark_cache_size = max_cache_size; - LOG_INFO(log, "Mark cache size was lowered to " << formatReadableSizeWithBinarySuffix(uncompressed_cache_size) - << " because the system has low amount of memory"); + LOG_INFO(log, "Mark cache size was lowered to " << formatReadableSizeWithBinarySuffix(uncompressed_cache_size) << " because the system has low amount of memory"); } global_context->setMarkCache(mark_cache_size); @@ -584,8 +574,7 @@ int Server::main(const std::vector & /*args*/) else if (max_server_memory_usage > default_max_server_memory_usage) { max_server_memory_usage = default_max_server_memory_usage; - LOG_INFO(log, "Setting max_server_memory_usage was lowered to " << formatReadableSizeWithBinarySuffix(max_server_memory_usage) - << " because the system has low amount of memory"); + LOG_INFO(log, "Setting max_server_memory_usage was lowered to " << formatReadableSizeWithBinarySuffix(max_server_memory_usage) << " because the system has low amount of memory"); } total_memory_tracker.setOrRaiseHardLimit(max_server_memory_usage); @@ -658,13 +647,11 @@ int Server::main(const std::vector & /*args*/) #endif #if defined(SANITIZER) - LOG_INFO(log, "Query Profiler and TraceCollector are disabled because they cannot work under sanitizers" - " when two different stack unwinding methods will interfere with each other."); + LOG_INFO(log, "Query Profiler and TraceCollector are disabled because they cannot work under sanitizers" " when two different stack unwinding methods will interfere with each other."); #endif if (!hasPHDRCache()) - LOG_INFO(log, "Query Profiler and TraceCollector are disabled because they require PHDR cache to be created" - " (otherwise the function 'dl_iterate_phdr' is not lock free and not async-signal safe)."); + LOG_INFO(log, "Query Profiler and TraceCollector are disabled because they require PHDR cache to be created" " (otherwise the function 'dl_iterate_phdr' is not lock free and not async-signal safe)."); global_context->setCurrentDatabase(default_database); @@ -690,21 +677,12 @@ int Server::main(const std::vector & /*args*/) #if defined(OS_LINUX) if (!TasksStatsCounters::checkIfAvailable()) { - LOG_INFO(log, "It looks like this system does not have procfs mounted at /proc location," - " neither clickhouse-server process has CAP_NET_ADMIN capability." - " 'taskstats' performance statistics will be disabled." - " It could happen due to incorrect ClickHouse package installation." - " You can try to resolve the problem manually with 'sudo setcap cap_net_admin=+ep " << executable_path << "'." - " Note that it will not work on 'nosuid' mounted filesystems." - " It also doesn't work if you run clickhouse-server inside network namespace as it happens in some containers."); + LOG_INFO(log, "It looks like this system does not have procfs mounted at /proc location," " neither clickhouse-server process has CAP_NET_ADMIN capability." " 'taskstats' performance statistics will be disabled." " It could happen due to incorrect ClickHouse package installation." " You can try to resolve the problem manually with 'sudo setcap cap_net_admin=+ep " << executable_path << "'." " Note that it will not work on 'nosuid' mounted filesystems." " It also doesn't work if you run clickhouse-server inside network namespace as it happens in some containers."); } if (!hasLinuxCapability(CAP_SYS_NICE)) { - LOG_INFO(log, "It looks like the process has no CAP_SYS_NICE capability, the setting 'os_thread_nice' will have no effect." - " It could happen due to incorrect ClickHouse package installation." - " You could resolve the problem manually with 'sudo setcap cap_sys_nice=+ep " << executable_path << "'." - " Note that it will not work on 'nosuid' mounted filesystems."); + LOG_INFO(log, "It looks like the process has no CAP_SYS_NICE capability, the setting 'os_thread_nice' will have no effect." " It could happen due to incorrect ClickHouse package installation." " You could resolve the problem manually with 'sudo setcap cap_sys_nice=+ep " << executable_path << "'." " Note that it will not work on 'nosuid' mounted filesystems."); } #else LOG_INFO_FORMATTED(log, "TaskStats is not implemented for this OS. IO accounting will be disabled."); @@ -746,11 +724,7 @@ int Server::main(const std::vector & /*args*/) #endif ) { - LOG_ERROR(log, - "Cannot resolve listen_host (" << host << "), error " << e.code() << ": " << e.message() << ". " - "If it is an IPv6 address and your host has disabled IPv6, then consider to " - "specify IPv4 address to listen in element of configuration " - "file. Example: 0.0.0.0"); + LOG_ERROR(log, "Cannot resolve listen_host (" << host << "), error " << e.code() << ": " << e.message() << ". " "If it is an IPv6 address and your host has disabled IPv6, then consider to " "specify IPv4 address to listen in element of configuration " "file. Example: 0.0.0.0"); } throw; @@ -802,11 +776,7 @@ int Server::main(const std::vector & /*args*/) if (listen_try) { - LOG_ERROR(log, message - << ". If it is an IPv6 or IPv4 address and your host has disabled IPv6 or IPv4, then consider to " - "specify not disabled IPv4 or IPv6 address to listen in element of configuration " - "file. Example for disabled IPv6: 0.0.0.0 ." - " Example for disabled IPv4: ::"); + LOG_ERROR(log, message << ". If it is an IPv6 or IPv4 address and your host has disabled IPv6 or IPv4, then consider to " "specify not disabled IPv4 or IPv6 address to listen in element of configuration " "file. Example for disabled IPv6: 0.0.0.0 ." " Example for disabled IPv4: ::"); } else { @@ -989,9 +959,7 @@ int Server::main(const std::vector & /*args*/) current_connections += server->currentConnections(); } - LOG_INFO(log, - "Closed all listening sockets." - << (current_connections ? " Waiting for " + toString(current_connections) + " outstanding connections." : "")); + LOG_INFO(log, "Closed all listening sockets." << (current_connections ? " Waiting for " + toString(current_connections) + " outstanding connections." : "")); /// Killing remaining queries. global_context->getProcessList().killAllQueries(); @@ -1013,9 +981,7 @@ int Server::main(const std::vector & /*args*/) } } - LOG_INFO( - log, "Closed connections." << (current_connections ? " But " + toString(current_connections) + " remains." - " Tip: To increase wait time add to config: 60" : "")); + LOG_INFO( log, "Closed connections." << (current_connections ? " But " + toString(current_connections) + " remains." " Tip: To increase wait time add to config: 60" : "")); dns_cache_updater.reset(); main_config_reloader.reset(); diff --git a/programs/server/TCPHandler.cpp b/programs/server/TCPHandler.cpp index 2005aa931ba..fec67686195 100644 --- a/programs/server/TCPHandler.cpp +++ b/programs/server/TCPHandler.cpp @@ -115,8 +115,7 @@ void TCPHandler::runImpl() if (!DatabaseCatalog::instance().isDatabaseExist(default_database)) { Exception e("Database " + backQuote(default_database) + " doesn't exist", ErrorCodes::UNKNOWN_DATABASE); - LOG_ERROR(log, "Code: " << e.code() << ", e.displayText() = " << e.displayText() - << ", Stack trace:\n\n" << e.getStackTraceString()); + LOG_ERROR(log, "Code: " << e.code() << ", e.displayText() = " << e.displayText() << ", Stack trace:\n\n" << e.getStackTraceString()); sendException(e, connection_context.getSettingsRef().calculate_text_stack_trace); return; } @@ -379,8 +378,7 @@ void TCPHandler::runImpl() watch.stop(); - LOG_INFO(log, std::fixed << std::setprecision(3) - << "Processed in " << watch.elapsedSeconds() << " sec."); + LOG_INFO(log, std::fixed << std::setprecision(3) << "Processed in " << watch.elapsedSeconds() << " sec."); /// It is important to destroy query context here. We do not want it to live arbitrarily longer than the query. query_context.reset(); @@ -732,14 +730,7 @@ void TCPHandler::receiveHello() readStringBinary(user, *in); readStringBinary(password, *in); - LOG_DEBUG(log, "Connected " << client_name - << " version " << client_version_major - << "." << client_version_minor - << "." << client_version_patch - << ", revision: " << client_revision - << (!default_database.empty() ? ", database: " + default_database : "") - << (!user.empty() ? ", user: " + user : "") - << "."); + LOG_DEBUG(log, "Connected " << client_name << " version " << client_version_major << "." << client_version_minor << "." << client_version_patch << ", revision: " << client_revision << (!default_database.empty() ? ", database: " + default_database : "") << (!user.empty() ? ", user: " + user : "") << "."); connection_context.setUser(user, password, socket().peerAddress()); } @@ -1205,8 +1196,7 @@ void TCPHandler::run() /// Timeout - not an error. if (!strcmp(e.what(), "Timeout")) { - LOG_DEBUG(log, "Poco::Exception. Code: " << ErrorCodes::POCO_EXCEPTION << ", e.code() = " << e.code() - << ", e.displayText() = " << e.displayText() << ", e.what() = " << e.what()); + LOG_DEBUG(log, "Poco::Exception. Code: " << ErrorCodes::POCO_EXCEPTION << ", e.code() = " << e.code() << ", e.displayText() = " << e.displayText() << ", e.what() = " << e.what()); } else throw; diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp index 7e51401a86b..022a4bafc06 100644 --- a/src/Common/MemoryTracker.cpp +++ b/src/Common/MemoryTracker.cpp @@ -49,16 +49,12 @@ MemoryTracker::~MemoryTracker() void MemoryTracker::logPeakMemoryUsage() const { - LOG_DEBUG(&Logger::get("MemoryTracker"), - "Peak memory usage" << (description ? " " + std::string(description) : "") - << ": " << formatReadableSizeWithBinarySuffix(peak) << "."); + LOG_DEBUG(&Logger::get("MemoryTracker"), "Peak memory usage" << (description ? " " + std::string(description) : "") << ": " << formatReadableSizeWithBinarySuffix(peak) << "."); } void MemoryTracker::logMemoryUsage(Int64 current) const { - LOG_DEBUG(&Logger::get("MemoryTracker"), - "Current memory usage" << (description ? " " + std::string(description) : "") - << ": " << formatReadableSizeWithBinarySuffix(current) << "."); + LOG_DEBUG(&Logger::get("MemoryTracker"), "Current memory usage" << (description ? " " + std::string(description) : "") << ": " << formatReadableSizeWithBinarySuffix(current) << "."); } diff --git a/src/Common/PoolWithFailoverBase.h b/src/Common/PoolWithFailoverBase.h index e0f251e9ed5..b83648f6d0e 100644 --- a/src/Common/PoolWithFailoverBase.h +++ b/src/Common/PoolWithFailoverBase.h @@ -239,8 +239,7 @@ PoolWithFailoverBase::getMany( } else { - LOG_WARNING(log, "Connection failed at try №" - << (shuffled_pool.error_count + 1) << ", reason: " << fail_message); + LOG_WARNING(log, "Connection failed at try №" << (shuffled_pool.error_count + 1) << ", reason: " << fail_message); ProfileEvents::increment(ProfileEvents::DistributedConnectionFailTry); shuffled_pool.error_count = std::min(max_error_cap, shuffled_pool.error_count + 1); diff --git a/src/DataStreams/AggregatingBlockInputStream.cpp b/src/DataStreams/AggregatingBlockInputStream.cpp index 04447474add..475a5529f9e 100644 --- a/src/DataStreams/AggregatingBlockInputStream.cpp +++ b/src/DataStreams/AggregatingBlockInputStream.cpp @@ -60,9 +60,7 @@ Block AggregatingBlockInputStream::readImpl() input_streams.emplace_back(temporary_inputs.back()->block_in); } - LOG_TRACE(log, "Will merge " << files.files.size() << " temporary files of size " - << (files.sum_size_compressed / 1048576.0) << " MiB compressed, " - << (files.sum_size_uncompressed / 1048576.0) << " MiB uncompressed."); + LOG_TRACE(log, "Will merge " << files.files.size() << " temporary files of size " << (files.sum_size_compressed / 1048576.0) << " MiB compressed, " << (files.sum_size_uncompressed / 1048576.0) << " MiB uncompressed."); impl = std::make_unique(input_streams, params, final, 1, 1); } diff --git a/src/DataStreams/CollapsingFinalBlockInputStream.cpp b/src/DataStreams/CollapsingFinalBlockInputStream.cpp index 744a3560477..2ac1c46f0bc 100644 --- a/src/DataStreams/CollapsingFinalBlockInputStream.cpp +++ b/src/DataStreams/CollapsingFinalBlockInputStream.cpp @@ -18,9 +18,7 @@ CollapsingFinalBlockInputStream::~CollapsingFinalBlockInputStream() void CollapsingFinalBlockInputStream::reportBadCounts() { /// With inconsistent data, this is an unavoidable error that can not be easily fixed by admins. Therefore Warning. - LOG_WARNING(log, "Incorrect data: number of rows with sign = 1 (" << count_positive - << ") differs with number of rows with sign = -1 (" << count_negative - << ") by more than one"); + LOG_WARNING(log, "Incorrect data: number of rows with sign = 1 (" << count_positive << ") differs with number of rows with sign = -1 (" << count_negative << ") by more than one"); } void CollapsingFinalBlockInputStream::reportBadSign(Int8 sign) diff --git a/src/DataStreams/CreatingSetsBlockInputStream.cpp b/src/DataStreams/CreatingSetsBlockInputStream.cpp index 967029b6dc1..9722feec1bd 100644 --- a/src/DataStreams/CreatingSetsBlockInputStream.cpp +++ b/src/DataStreams/CreatingSetsBlockInputStream.cpp @@ -90,9 +90,7 @@ void CreatingSetsBlockInputStream::createAll() void CreatingSetsBlockInputStream::createOne(SubqueryForSet & subquery) { - LOG_TRACE(log, (subquery.set ? "Creating set. " : "") - << (subquery.join ? "Creating join. " : "") - << (subquery.table ? "Filling temporary table. " : "")); + LOG_TRACE(log, (subquery.set ? "Creating set. " : "") << (subquery.join ? "Creating join. " : "") << (subquery.table ? "Filling temporary table. " : "")); Stopwatch watch; BlockOutputStreamPtr table_out; diff --git a/src/DataStreams/MergeSortingBlockInputStream.cpp b/src/DataStreams/MergeSortingBlockInputStream.cpp index 8e1a1d0719e..08ff290b9ea 100644 --- a/src/DataStreams/MergeSortingBlockInputStream.cpp +++ b/src/DataStreams/MergeSortingBlockInputStream.cpp @@ -264,9 +264,7 @@ void MergeSortingBlockInputStream::remerge() } merger.readSuffix(); - LOG_DEBUG(log, "Memory usage is lowered from " - << formatReadableSizeWithBinarySuffix(sum_bytes_in_blocks) << " to " - << formatReadableSizeWithBinarySuffix(new_sum_bytes_in_blocks)); + LOG_DEBUG(log, "Memory usage is lowered from " << formatReadableSizeWithBinarySuffix(sum_bytes_in_blocks) << " to " << formatReadableSizeWithBinarySuffix(new_sum_bytes_in_blocks)); /// If the memory consumption was not lowered enough - we will not perform remerge anymore. 2 is a guess. if (new_sum_bytes_in_blocks * 2 > sum_bytes_in_blocks) diff --git a/src/DataStreams/ParallelAggregatingBlockInputStream.cpp b/src/DataStreams/ParallelAggregatingBlockInputStream.cpp index 8fcaecafd2a..03abb13c580 100644 --- a/src/DataStreams/ParallelAggregatingBlockInputStream.cpp +++ b/src/DataStreams/ParallelAggregatingBlockInputStream.cpp @@ -82,9 +82,7 @@ Block ParallelAggregatingBlockInputStream::readImpl() input_streams.emplace_back(temporary_inputs.back()->block_in); } - LOG_TRACE(log, "Will merge " << files.files.size() << " temporary files of size " - << (files.sum_size_compressed / 1048576.0) << " MiB compressed, " - << (files.sum_size_uncompressed / 1048576.0) << " MiB uncompressed."); + LOG_TRACE(log, "Will merge " << files.files.size() << " temporary files of size " << (files.sum_size_compressed / 1048576.0) << " MiB compressed, " << (files.sum_size_uncompressed / 1048576.0) << " MiB uncompressed."); impl = std::make_unique( input_streams, params, final, temporary_data_merge_threads, temporary_data_merge_threads); @@ -179,20 +177,12 @@ void ParallelAggregatingBlockInputStream::execute() for (size_t i = 0; i < max_threads; ++i) { size_t rows = many_data[i]->size(); - LOG_TRACE(log, - "Aggregated. " << threads_data[i].src_rows << " to " << rows << " rows" - << " (from " << threads_data[i].src_bytes / 1048576.0 << " MiB)" - << " in " << elapsed_seconds << " sec." - << " (" << threads_data[i].src_rows / elapsed_seconds << " rows/sec., " - << formatReadableSizeWithBinarySuffix(threads_data[i].src_bytes / elapsed_seconds) << "/sec.)"); + LOG_TRACE(log, "Aggregated. " << threads_data[i].src_rows << " to " << rows << " rows" << " (from " << threads_data[i].src_bytes / 1048576.0 << " MiB)" << " in " << elapsed_seconds << " sec." << " (" << threads_data[i].src_rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(threads_data[i].src_bytes / elapsed_seconds) << "/sec.)"); total_src_rows += threads_data[i].src_rows; total_src_bytes += threads_data[i].src_bytes; } - LOG_TRACE(log, - "Total aggregated. " << total_src_rows << " rows (from " << total_src_bytes / 1048576.0 << " MiB)" - << " in " << elapsed_seconds << " sec." - << " (" << total_src_rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(total_src_bytes / elapsed_seconds) << "/sec.)"); + LOG_TRACE(log, "Total aggregated. " << total_src_rows << " rows (from " << total_src_bytes / 1048576.0 << " MiB)" << " in " << elapsed_seconds << " sec." << " (" << total_src_rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(total_src_bytes / elapsed_seconds) << "/sec.)"); /// If there was no data, and we aggregate without keys, we must return single row with the result of empty aggregation. /// To do this, we pass a block with zero rows to aggregate. diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index 6f59fe7c595..3d563b09e0d 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -466,8 +466,7 @@ ASTPtr DatabaseOnDisk::parseQueryFromMetadata(Poco::Logger * loger, const Contex table_name = unescapeForFileName(table_name); if (create.table != TABLE_WITH_UUID_NAME_PLACEHOLDER) - LOG_WARNING(loger, "File " << metadata_file_path << " contains both UUID and table name. " - "Will use name `" << table_name << "` instead of `" << create.table << "`"); + LOG_WARNING(loger, "File " << metadata_file_path << " contains both UUID and table name. " "Will use name `" << table_name << "` instead of `" << create.table << "`"); create.table = table_name; } diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 73f8840e2c9..c6df6b01090 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -761,17 +761,7 @@ void Aggregator::writeToTemporaryFile(AggregatedDataVariants & data_variants, co ProfileEvents::increment(ProfileEvents::ExternalAggregationCompressedBytes, compressed_bytes); ProfileEvents::increment(ProfileEvents::ExternalAggregationUncompressedBytes, uncompressed_bytes); - LOG_TRACE(log, - "Written part in " << elapsed_seconds << " sec., " - << rows << " rows, " - << formatReadableSizeWithBinarySuffix(uncompressed_bytes) << " uncompressed, " - << formatReadableSizeWithBinarySuffix(compressed_bytes) << " compressed, " - << (uncompressed_bytes / rows) << " uncompressed bytes per row, " - << (compressed_bytes / rows) << " compressed bytes per row, " - << "compression rate: " << (uncompressed_bytes / compressed_bytes) - << " (" << (rows / elapsed_seconds) << " rows/sec., " - << formatReadableSizeWithBinarySuffix(uncompressed_bytes / elapsed_seconds) << "/sec. uncompressed, " - << formatReadableSizeWithBinarySuffix(compressed_bytes / elapsed_seconds) << "/sec. compressed)"); + LOG_TRACE(log, "Written part in " << elapsed_seconds << " sec., " << rows << " rows, " << formatReadableSizeWithBinarySuffix(uncompressed_bytes) << " uncompressed, " << formatReadableSizeWithBinarySuffix(compressed_bytes) << " compressed, " << (uncompressed_bytes / rows) << " uncompressed bytes per row, " << (compressed_bytes / rows) << " compressed bytes per row, " << "compression rate: " << (uncompressed_bytes / compressed_bytes) << " (" << (rows / elapsed_seconds) << " rows/sec., " << formatReadableSizeWithBinarySuffix(uncompressed_bytes / elapsed_seconds) << "/sec. uncompressed, " << formatReadableSizeWithBinarySuffix(compressed_bytes / elapsed_seconds) << "/sec. compressed)"); } void Aggregator::writeToTemporaryFile(AggregatedDataVariants & data_variants) { @@ -867,9 +857,7 @@ void Aggregator::writeToTemporaryFileImpl( /// `data_variants` will not destroy them in the destructor, they are now owned by ColumnAggregateFunction objects. data_variants.aggregator = nullptr; - LOG_TRACE(log, - "Max size of temporary block: " << max_temporary_block_size_rows << " rows, " - << (max_temporary_block_size_bytes / 1048576.0) << " MiB."); + LOG_TRACE(log, "Max size of temporary block: " << max_temporary_block_size_rows << " rows, " << (max_temporary_block_size_bytes / 1048576.0) << " MiB."); } @@ -939,10 +927,7 @@ void Aggregator::execute(const BlockInputStreamPtr & stream, AggregatedDataVaria double elapsed_seconds = watch.elapsedSeconds(); size_t rows = result.sizeWithoutOverflowRow(); - LOG_TRACE(log, - "Aggregated. " << src_rows << " to " << rows << " rows (from " << formatReadableSizeWithBinarySuffix(src_bytes) << ")" - << " in " << elapsed_seconds << " sec." - << " (" << src_rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(src_bytes / elapsed_seconds) << "/sec.)"); + LOG_TRACE(log, "Aggregated. " << src_rows << " to " << rows << " rows (from " << formatReadableSizeWithBinarySuffix(src_bytes) << ")" << " in " << elapsed_seconds << " sec." << " (" << src_rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(src_bytes / elapsed_seconds) << "/sec.)"); } @@ -1308,11 +1293,7 @@ BlocksList Aggregator::convertToBlocks(AggregatedDataVariants & data_variants, b } double elapsed_seconds = watch.elapsedSeconds(); - LOG_TRACE(log, - "Converted aggregated data to blocks. " - << rows << " rows, " << bytes / 1048576.0 << " MiB" - << " in " << elapsed_seconds << " sec." - << " (" << rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(bytes / elapsed_seconds) << "/sec.)"); + LOG_TRACE(log, "Converted aggregated data to blocks. " << rows << " rows, " << bytes / 1048576.0 << " MiB" << " in " << elapsed_seconds << " sec." << " (" << rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(bytes / elapsed_seconds) << "/sec.)"); return blocks; } @@ -1943,8 +1924,7 @@ void Aggregator::mergeStream(const BlockInputStreamPtr & stream, AggregatedDataV bucket_to_blocks[block.info.bucket_num].emplace_back(std::move(block)); } - LOG_TRACE(log, "Read " << total_input_blocks << " blocks of partially aggregated data, total " << total_input_rows - << " rows."); + LOG_TRACE(log, "Read " << total_input_blocks << " blocks of partially aggregated data, total " << total_input_rows << " rows."); mergeBlocks(bucket_to_blocks, result, max_threads); } @@ -2175,11 +2155,7 @@ Block Aggregator::mergeBlocks(BlocksList & blocks, bool final) size_t rows = block.rows(); size_t bytes = block.bytes(); double elapsed_seconds = watch.elapsedSeconds(); - LOG_TRACE(log, - "Merged partially aggregated blocks. " - << rows << " rows, " << bytes / 1048576.0 << " MiB." - << " in " << elapsed_seconds << " sec." - << " (" << rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(bytes / elapsed_seconds) << "/sec.)"); + LOG_TRACE(log, "Merged partially aggregated blocks. " << rows << " rows, " << bytes / 1048576.0 << " MiB." << " in " << elapsed_seconds << " sec." << " (" << rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(bytes / elapsed_seconds) << "/sec.)"); if (isCancelled()) return {}; diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index fbdbcb77cb7..6ce3be6e651 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -238,8 +238,7 @@ DDLWorker::DDLWorker(const std::string & zk_root_dir, Context & context_, const if (context.getSettingsRef().readonly) { - LOG_WARNING(log, "Distributed DDL worker is run with readonly settings, it will not be able to execute DDL queries" - << " Set appropriate system_profile or distributed_ddl.profile to fix this."); + LOG_WARNING(log, "Distributed DDL worker is run with readonly settings, it will not be able to execute DDL queries" << " Set appropriate system_profile or distributed_ddl.profile to fix this."); } host_fqdn = getFQDNOrHostName(); @@ -335,8 +334,7 @@ bool DDLWorker::initAndCheckTask(const String & entry_name, String & out_reason, if (host_in_hostlist) { /// This check could be slow a little bit - LOG_WARNING(log, "There are two the same ClickHouse instances in task " << entry_name - << ": " << task->host_id.readableString() << " and " << host.readableString() << ". Will use the first one only."); + LOG_WARNING(log, "There are two the same ClickHouse instances in task " << entry_name << ": " << task->host_id.readableString() << " and " << host.readableString() << ". Will use the first one only."); } else { @@ -426,8 +424,7 @@ void DDLWorker::processTasks() { if (server_startup && e.code == Coordination::ZNONODE) { - LOG_WARNING(log, "ZooKeeper NONODE error during startup. Ignoring entry " << - task.entry_name << " (" << task.entry.query << ") : " << getCurrentExceptionMessage(true)); + LOG_WARNING(log, "ZooKeeper NONODE error during startup. Ignoring entry " << task.entry_name << " (" << task.entry.query << ") : " << getCurrentExceptionMessage(true)); } else { @@ -436,8 +433,7 @@ void DDLWorker::processTasks() } catch (...) { - LOG_WARNING(log, "An error occurred while processing task " << task.entry_name << " (" << task.entry.query << ") : " - << getCurrentExceptionMessage(true)); + LOG_WARNING(log, "An error occurred while processing task " << task.entry_name << " (" << task.entry.query << ") : " << getCurrentExceptionMessage(true)); throw; } } @@ -510,8 +506,7 @@ void DDLWorker::parseQueryAndResolveHost(DDLTask & task) if (found_exact_match) return; - LOG_WARNING(log, "Not found the exact match of host " << task.host_id.readableString() << " from task " << task.entry_name - << " in cluster " << task.cluster_name << " definition. Will try to find it using host name resolving."); + LOG_WARNING(log, "Not found the exact match of host " << task.host_id.readableString() << " from task " << task.entry_name << " in cluster " << task.cluster_name << " definition. Will try to find it using host name resolving."); bool found_via_resolving = false; for (size_t shard_num = 0; shard_num < shards.size(); ++shard_num) @@ -547,8 +542,7 @@ void DDLWorker::parseQueryAndResolveHost(DDLTask & task) } else { - LOG_INFO(log, "Resolved host " << task.host_id.readableString() << " from task " << task.entry_name - << " as host " << task.address_in_cluster.readableString() << " in definition of cluster " << task.cluster_name); + LOG_INFO(log, "Resolved host " << task.host_id.readableString() << " from task " << task.entry_name << " as host " << task.address_in_cluster.readableString() << " in definition of cluster " << task.cluster_name); } } @@ -761,8 +755,7 @@ bool DDLWorker::tryExecuteQueryOnLeaderReplica( String executed_by; if (zookeeper->tryGet(is_executed_path, executed_by)) { - LOG_DEBUG(log, "Task " << task.entry_name << " has already been executed by leader replica (" - << executed_by << ") of the same shard."); + LOG_DEBUG(log, "Task " << task.entry_name << " has already been executed by leader replica (" << executed_by << ") of the same shard."); return true; } @@ -947,8 +940,7 @@ String DDLWorker::enqueueQuery(DDLLogEntry & entry) } catch (...) { - LOG_INFO(log, "An error occurred while creating auxiliary ZooKeeper directories in " << node_path << " . They will be created later" - << ". Error : " << getCurrentExceptionMessage(true)); + LOG_INFO(log, "An error occurred while creating auxiliary ZooKeeper directories in " << node_path << " . They will be created later" << ". Error : " << getCurrentExceptionMessage(true)); } return node_path; diff --git a/src/Interpreters/DNSCacheUpdater.cpp b/src/Interpreters/DNSCacheUpdater.cpp index 90c27ea2854..d387cccc0be 100644 --- a/src/Interpreters/DNSCacheUpdater.cpp +++ b/src/Interpreters/DNSCacheUpdater.cpp @@ -21,8 +21,7 @@ void DNSCacheUpdater::run() /// Reload cluster config if IP of any host has been changed since last update. if (resolver.updateCache()) { - LOG_INFO(&Poco::Logger::get("DNSCacheUpdater"), - "IPs of some hosts have been changed. Will reload cluster config."); + LOG_INFO(&Poco::Logger::get("DNSCacheUpdater"), "IPs of some hosts have been changed. Will reload cluster config."); try { context.reloadClusterConfig(); diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index a159c93e881..40ae756f48d 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -597,9 +597,7 @@ void DatabaseCatalog::enqueueDroppedTableCleanup(StorageID table_id, StoragePtr } else { - LOG_WARNING(log, "Cannot parse metadata of partially dropped table " - << table_id.getNameForLogs() << " from " << dropped_metadata_path - << ". Will remove metadata file and data directory. Garbage may be left in /store directory and ZooKeeper."); + LOG_WARNING(log, "Cannot parse metadata of partially dropped table " << table_id.getNameForLogs() << " from " << dropped_metadata_path << ". Will remove metadata file and data directory. Garbage may be left in /store directory and ZooKeeper."); } drop_time = Poco::File(dropped_metadata_path).getLastModified().epochTime(); diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index b5e22619803..92f4dc5863c 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -350,12 +350,7 @@ private: const auto & already_added = already_added_it->second; if (!already_added->from_temp_repository && !repository->isTemporary()) { - LOG_WARNING( - log, - type_name << " '" << object_name << "' is found " - << (((path == already_added->path) && (repository->getName() == already_added->repository_name)) - ? ("twice in the same file '" + path + "'") - : ("both in file '" + already_added->path + "' and '" + path + "'"))); + LOG_WARNING( log, type_name << " '" << object_name << "' is found " << (((path == already_added->path) && (repository->getName() == already_added->repository_name)) ? ("twice in the same file '" + path + "'") : ("both in file '" + already_added->path + "' and '" + path + "'"))); } } } @@ -444,9 +439,7 @@ public: { /// The object has been tried to load before, so it is currently in use or was in use /// and we should try to reload it with the new config. - LOG_TRACE(log, "Will reload '" << name << "'" - " because its configuration has been changed and" - " there were attempts to load it before"); + LOG_TRACE(log, "Will reload '" << name << "'" " because its configuration has been changed and" " there were attempts to load it before"); startLoading(info, true); } } @@ -461,8 +454,7 @@ public: Info & info = infos.emplace(name, Info{name, config}).first->second; if (always_load_everything) { - LOG_TRACE(log, "Will load '" << name << "'" - " because always_load_everything flag is set."); + LOG_TRACE(log, "Will load '" << name << "'" " because always_load_everything flag is set."); startLoading(info); } } @@ -672,10 +664,7 @@ public: if (!should_update_flag) { info.next_update_time = calculateNextUpdateTime(info.object, info.error_count); - LOG_TRACE(log, "Object '" << info.name << "'" - " not modified, will not reload. " - "Next update at " - << ext::to_string(info.next_update_time)); + LOG_TRACE(log, "Object '" << info.name << "'" " not modified, will not reload. " "Next update at " << ext::to_string(info.next_update_time)); continue; } @@ -687,8 +676,7 @@ public: /// Object was never loaded successfully and should be reloaded. startLoading(info); } - LOG_TRACE(log, "Object '" << info.name << "' is neither" - " loaded nor failed, so it will not be reloaded as outdated."); + LOG_TRACE(log, "Object '" << info.name << "' is neither" " loaded nor failed, so it will not be reloaded as outdated."); } } } @@ -881,8 +869,7 @@ private: { if (info.isLoading()) { - LOG_TRACE(log, "The object '" << info.name << - "' is already being loaded, force = " << forced_to_reload << "."); + LOG_TRACE(log, "The object '" << info.name << "' is already being loaded, force = " << forced_to_reload << "."); if (!forced_to_reload) { @@ -898,11 +885,7 @@ private: info.loading_start_time = std::chrono::system_clock::now(); info.loading_end_time = TimePoint{}; - LOG_TRACE(log, "Will load the object '" << info.name << "' " - << (enable_async_loading ? std::string("in background") - : "immediately") - << ", force = " << forced_to_reload - << ", loading_id = " << info.loading_id); + LOG_TRACE(log, "Will load the object '" << info.name << "' " << (enable_async_loading ? std::string("in background") : "immediately") << ", force = " << forced_to_reload << ", loading_id = " << info.loading_id); if (enable_async_loading) { @@ -941,8 +924,7 @@ private: info = prepareToLoadSingleObject(name, loading_id, min_id_to_finish_loading_dependencies_, lock); if (!info) { - LOG_TRACE(log, "Could not lock object '" << name - << "' for loading"); + LOG_TRACE(log, "Could not lock object '" << name << "' for loading"); return; } } @@ -1055,8 +1037,7 @@ private: } if (info->loading_id != loading_id) { - LOG_TRACE(log, "Next update time for '" << name << "' will not be set because this object's current loading_id " - << info->loading_id << " is different from the specified " << loading_id << "."); + LOG_TRACE(log, "Next update time for '" << name << "' will not be set because this object's current loading_id " << info->loading_id << " is different from the specified " << loading_id << "."); return; } @@ -1086,8 +1067,7 @@ private: info->last_successful_update_time = current_time; info->state_id = info->loading_id; info->next_update_time = next_update_time; - LOG_TRACE(log, "Next update time for '" << info->name - << "' was set to " << ext::to_string(next_update_time)); + LOG_TRACE(log, "Next update time for '" << info->name << "' was set to " << ext::to_string(next_update_time)); } /// Removes the references to the loading thread from the maps. @@ -1119,9 +1099,7 @@ private: { if (!loaded_object->supportUpdates()) { - LOG_TRACE(log, "Supposed update time for " - "'" + loaded_object->getLoadableName() + "'" - " is never (loaded, does not support updates)"); + LOG_TRACE(log, "Supposed update time for " "'" + loaded_object->getLoadableName() + "'" " is never (loaded, does not support updates)"); return never; } @@ -1130,9 +1108,7 @@ private: const auto & lifetime = loaded_object->getLifetime(); if (lifetime.min_sec == 0 && lifetime.max_sec == 0) { - LOG_TRACE(log, "Supposed update time for " - "'" + loaded_object->getLoadableName() + "'" - " is never (loaded, lifetime 0)"); + LOG_TRACE(log, "Supposed update time for " "'" + loaded_object->getLoadableName() + "'" " is never (loaded, lifetime 0)"); return never; } @@ -1140,26 +1116,18 @@ private: { std::uniform_int_distribution distribution{lifetime.min_sec, lifetime.max_sec}; auto result = std::chrono::system_clock::now() + std::chrono::seconds{distribution(rnd_engine)}; - LOG_TRACE(log, "Supposed update time for " - "'" << loaded_object->getLoadableName() << "'" - " is " << ext::to_string(result) - << " (loaded, lifetime [" << lifetime.min_sec - << ", " << lifetime.max_sec << "], no errors)"); + LOG_TRACE(log, "Supposed update time for " "'" << loaded_object->getLoadableName() << "'" " is " << ext::to_string(result) << " (loaded, lifetime [" << lifetime.min_sec << ", " << lifetime.max_sec << "], no errors)"); return result; } auto result = std::chrono::system_clock::now() + std::chrono::seconds(calculateDurationWithBackoff(rnd_engine, error_count)); - LOG_TRACE(log, "Supposed update time for '" << loaded_object->getLoadableName() << "'" - " is " << ext::to_string(result) - << " (backoff, " << error_count << " errors)"); + LOG_TRACE(log, "Supposed update time for '" << loaded_object->getLoadableName() << "'" " is " << ext::to_string(result) << " (backoff, " << error_count << " errors)"); return result; } else { auto result = std::chrono::system_clock::now() + std::chrono::seconds(calculateDurationWithBackoff(rnd_engine, error_count)); - LOG_TRACE(log, "Supposed update time for unspecified object " - " is " << ext::to_string(result) - << " (backoff, " << error_count << " errors."); + LOG_TRACE(log, "Supposed update time for unspecified object " " is " << ext::to_string(result) << " (backoff, " << error_count << " errors."); return result; } } diff --git a/src/Interpreters/SystemLog.cpp b/src/Interpreters/SystemLog.cpp index 3de94598603..69509b7968b 100644 --- a/src/Interpreters/SystemLog.cpp +++ b/src/Interpreters/SystemLog.cpp @@ -41,9 +41,7 @@ std::shared_ptr createSystemLog( if (database != default_database_name) { /// System tables must be loaded before other tables, but loading order is undefined for all databases except `system` - LOG_ERROR(&Logger::get("SystemLog"), "Custom database name for a system table specified in config. " - "Table `" << table << "` will be created in `system` database " - "instead of `" << database << "`"); + LOG_ERROR(&Logger::get("SystemLog"), "Custom database name for a system table specified in config. " "Table `" << table << "` will be created in `system` database " "instead of `" << database << "`"); database = default_database_name; } diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h index c4de60e0514..c61f295f341 100644 --- a/src/Interpreters/SystemLog.h +++ b/src/Interpreters/SystemLog.h @@ -60,9 +60,7 @@ namespace ErrorCodes extern const int TIMEOUT_EXCEEDED; } -#define DBMS_SYSTEM_LOG_QUEUE_SIZE 1048576 - - +#define DBMS_SYSTEM_LOG_QUEUE_SIZE 1048576 class Context; class QueryLog; class QueryThreadLog; @@ -234,9 +232,7 @@ void SystemLog::add(const LogElement & element) if (is_shutdown) return; - if (queue.size() == DBMS_SYSTEM_LOG_QUEUE_SIZE / 2) - { - // The queue more than half full, time to flush. + if (queue.size() == DBMS_SYSTEM_LOG_QUEUE_SIZE / 2) { // The queue more than half full, time to flush. // We only check for strict equality, because messages are added one // by one, under exclusive lock, so we will see each message count. // It is enough to only wake the flushing thread once, after the message @@ -249,9 +245,7 @@ void SystemLog::add(const LogElement & element) LOG_INFO_FORMATTED(log, "Queue is half full for system log '{}'.", demangle(typeid(*this).name())); } - if (queue.size() >= DBMS_SYSTEM_LOG_QUEUE_SIZE) - { - // Ignore all further entries until the queue is flushed. + if (queue.size() >= DBMS_SYSTEM_LOG_QUEUE_SIZE) { // Ignore all further entries until the queue is flushed. // Log a message about that. Don't spam it -- this might be especially // problematic in case of trace log. Remember what the front index of the // queue was when we last logged the message. If it changed, it means the @@ -262,9 +256,7 @@ void SystemLog::add(const LogElement & element) // TextLog sets its logger level to 0, so this log is a noop and // there is no recursive logging. - LOG_ERROR(log, "Queue is full for system log '" - << demangle(typeid(*this).name()) << "'" - << " at " << queue_front_index); + LOG_ERROR(log, "Queue is full for system log '" << demangle(typeid(*this).name()) << "'" << " at " << queue_front_index); } return; @@ -386,8 +378,7 @@ void SystemLog::flushImpl(const std::vector & to_flush, { try { - LOG_TRACE(log, "Flushing system log, " - << to_flush.size() << " entries to flush"); + LOG_TRACE(log, "Flushing system log, " << to_flush.size() << " entries to flush"); /// We check for existence of the table and create it as needed at every /// flush. This is done to allow user to drop the table at any moment @@ -465,8 +456,7 @@ void SystemLog::prepareTable() rename->elements.emplace_back(elem); - LOG_DEBUG(log, "Existing table " << description << " for system log has obsolete or different structure." - " Renaming it to " << backQuoteIfNeed(to.table)); + LOG_DEBUG(log, "Existing table " << description << " for system log has obsolete or different structure." " Renaming it to " << backQuoteIfNeed(to.table)); InterpreterRenameQuery(rename, context).execute(); diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 00f8c1bfdc5..83ff94642c7 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -113,11 +113,7 @@ static void logQuery(const String & query, const Context & context, bool interna const auto & initial_query_id = context.getClientInfo().initial_query_id; const auto & current_user = context.getClientInfo().current_user; - LOG_DEBUG(&Logger::get("executeQuery"), "(from " << context.getClientInfo().current_address.toString() - << (current_user != "default" ? ", user: " + context.getClientInfo().current_user : "") - << (!initial_query_id.empty() && current_query_id != initial_query_id ? ", initial_query_id: " + initial_query_id : std::string()) - << ") " - << joinLines(query)); + LOG_DEBUG(&Logger::get("executeQuery"), "(from " << context.getClientInfo().current_address.toString() << (current_user != "default" ? ", user: " + context.getClientInfo().current_user : "") << (!initial_query_id.empty() && current_query_id != initial_query_id ? ", initial_query_id: " + initial_query_id : std::string()) << ") " << joinLines(query)); } } @@ -144,10 +140,7 @@ static void setExceptionStackTrace(QueryLogElement & elem) /// Log exception (with query info) into text log (not into system table). static void logException(Context & context, QueryLogElement & elem) { - LOG_ERROR(&Logger::get("executeQuery"), elem.exception - << " (from " << context.getClientInfo().current_address.toString() << ")" - << " (in query: " << joinLines(elem.query) << ")" - << (!elem.stack_trace.empty() ? ", Stack trace (when copying this message, always include the lines below):\n\n" + elem.stack_trace : "")); + LOG_ERROR(&Logger::get("executeQuery"), elem.exception << " (from " << context.getClientInfo().current_address.toString() << ")" << " (in query: " << joinLines(elem.query) << ")" << (!elem.stack_trace.empty() ? ", Stack trace (when copying this message, always include the lines below):\n\n" + elem.stack_trace : "")); } @@ -502,11 +495,7 @@ static std::tuple executeQueryImpl( if (elem.read_rows != 0) { - LOG_INFO(&Logger::get("executeQuery"), std::fixed << std::setprecision(3) - << "Read " << elem.read_rows << " rows, " - << formatReadableSizeWithBinarySuffix(elem.read_bytes) << " in " << elapsed_seconds << " sec., " - << static_cast(elem.read_rows / elapsed_seconds) << " rows/sec., " - << formatReadableSizeWithBinarySuffix(elem.read_bytes / elapsed_seconds) << "/sec."); + LOG_INFO(&Logger::get("executeQuery"), std::fixed << std::setprecision(3) << "Read " << elem.read_rows << " rows, " << formatReadableSizeWithBinarySuffix(elem.read_bytes) << " in " << elapsed_seconds << " sec., " << static_cast(elem.read_rows / elapsed_seconds) << " rows/sec., " << formatReadableSizeWithBinarySuffix(elem.read_bytes / elapsed_seconds) << "/sec."); } elem.thread_ids = std::move(info.thread_ids); diff --git a/src/Processors/Executors/PipelineExecutor.cpp b/src/Processors/Executors/PipelineExecutor.cpp index 2a843f23319..5c3734f377f 100644 --- a/src/Processors/Executors/PipelineExecutor.cpp +++ b/src/Processors/Executors/PipelineExecutor.cpp @@ -547,12 +547,7 @@ void PipelineExecutor::executeSingleThread(size_t thread_num, size_t num_threads #ifndef NDEBUG auto & context = executor_contexts[thread_num]; - LOG_TRACE(log, - "Thread finished." - << " Total time: " << (context->total_time_ns / 1e9) << " sec." - << " Execution time: " << (context->execution_time_ns / 1e9) << " sec." - << " Processing time: " << (context->processing_time_ns / 1e9) << " sec." - << " Wait time: " << (context->wait_time_ns / 1e9) << " sec."); + LOG_TRACE(log, "Thread finished." << " Total time: " << (context->total_time_ns / 1e9) << " sec." << " Execution time: " << (context->execution_time_ns / 1e9) << " sec." << " Processing time: " << (context->processing_time_ns / 1e9) << " sec." << " Wait time: " << (context->wait_time_ns / 1e9) << " sec."); #endif } diff --git a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp index d0272e6f277..7ad2285f84e 100644 --- a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp @@ -629,8 +629,7 @@ private: Poco::JSON::Parser parser; auto json_body = parser.parse(*response_body).extract(); auto schema = json_body->getValue("schema"); - LOG_TRACE((&Logger::get("AvroConfluentRowInputFormat")), - "Succesfully fetched schema id = " << id << "\n" << schema); + LOG_TRACE((&Logger::get("AvroConfluentRowInputFormat")), "Succesfully fetched schema id = " << id << "\n" << schema); return avro::compileJsonSchemaFromString(schema); } catch (const Exception &) diff --git a/src/Processors/Merges/Algorithms/RowRef.h b/src/Processors/Merges/Algorithms/RowRef.h index 02a07130bce..4958c05ab14 100644 --- a/src/Processors/Merges/Algorithms/RowRef.h +++ b/src/Processors/Merges/Algorithms/RowRef.h @@ -83,9 +83,7 @@ public: { if (free_chunks.size() != chunks.size()) { - LOG_ERROR(&Logger::get("SharedChunkAllocator"), - "SharedChunkAllocator was destroyed before RowRef was released. StackTrace: " - << StackTrace().toString()); + LOG_ERROR(&Logger::get("SharedChunkAllocator"), "SharedChunkAllocator was destroyed before RowRef was released. StackTrace: " << StackTrace().toString()); return; } @@ -102,9 +100,7 @@ private: /// This may happen if allocator was removed before chunks. /// Log message and exit, because we don't want to throw exception in destructor. - LOG_ERROR(&Logger::get("SharedChunkAllocator"), - "SharedChunkAllocator was destroyed before RowRef was released. StackTrace: " - << StackTrace().toString()); + LOG_ERROR(&Logger::get("SharedChunkAllocator"), "SharedChunkAllocator was destroyed before RowRef was released. StackTrace: " << StackTrace().toString()); return; } diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index 3672e23f302..64ca98f3808 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -540,10 +540,7 @@ void AggregatingTransform::initGenerate() double elapsed_seconds = watch.elapsedSeconds(); size_t rows = variants.sizeWithoutOverflowRow(); - LOG_TRACE(log, - "Aggregated. " << src_rows << " to " << rows << " rows (from " << src_bytes / 1048576.0 << " MiB)" - << " in " << elapsed_seconds << " sec." - << " (" << src_rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(src_bytes / elapsed_seconds) << "/sec.)"); + LOG_TRACE(log, "Aggregated. " << src_rows << " to " << rows << " rows (from " << src_bytes / 1048576.0 << " MiB)" << " in " << elapsed_seconds << " sec." << " (" << src_rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(src_bytes / elapsed_seconds) << "/sec.)"); if (params->aggregator.hasTemporaryFiles()) { @@ -592,9 +589,7 @@ void AggregatingTransform::initGenerate() for (const auto & file : files.files) processors.emplace_back(std::make_unique(header, file->path())); - LOG_TRACE(log, "Will merge " << files.files.size() << " temporary files of size " - << (files.sum_size_compressed / 1048576.0) << " MiB compressed, " - << (files.sum_size_uncompressed / 1048576.0) << " MiB uncompressed."); + LOG_TRACE(log, "Will merge " << files.files.size() << " temporary files of size " << (files.sum_size_compressed / 1048576.0) << " MiB compressed, " << (files.sum_size_uncompressed / 1048576.0) << " MiB uncompressed."); auto pipe = createMergingAggregatedMemoryEfficientPipe( header, params, files.files.size(), temporary_data_merge_threads); diff --git a/src/Processors/Transforms/CreatingSetsTransform.cpp b/src/Processors/Transforms/CreatingSetsTransform.cpp index d35938e23b0..7bdc084c211 100644 --- a/src/Processors/Transforms/CreatingSetsTransform.cpp +++ b/src/Processors/Transforms/CreatingSetsTransform.cpp @@ -56,9 +56,7 @@ IProcessor::Status CreatingSetsTransform::prepare() void CreatingSetsTransform::startSubquery(SubqueryForSet & subquery) { - LOG_TRACE(log, (subquery.set ? "Creating set. " : "") - << (subquery.join ? "Creating join. " : "") - << (subquery.table ? "Filling temporary table. " : "")); + LOG_TRACE(log, (subquery.set ? "Creating set. " : "") << (subquery.join ? "Creating join. " : "") << (subquery.table ? "Filling temporary table. " : "")); elapsed_nanoseconds = 0; diff --git a/src/Processors/Transforms/MergeSortingTransform.cpp b/src/Processors/Transforms/MergeSortingTransform.cpp index f432948e8e0..4d72cced8b1 100644 --- a/src/Processors/Transforms/MergeSortingTransform.cpp +++ b/src/Processors/Transforms/MergeSortingTransform.cpp @@ -251,8 +251,7 @@ void MergeSortingTransform::generate() void MergeSortingTransform::remerge() { - LOG_DEBUG(log, "Re-merging intermediate ORDER BY data (" << chunks.size() - << " blocks with " << sum_rows_in_blocks << " rows) to save memory consumption"); + LOG_DEBUG(log, "Re-merging intermediate ORDER BY data (" << chunks.size() << " blocks with " << sum_rows_in_blocks << " rows) to save memory consumption"); /// NOTE Maybe concat all blocks and partial sort will be faster than merge? MergeSorter remerge_sorter(std::move(chunks), description, max_merged_block_size, limit); @@ -268,9 +267,7 @@ void MergeSortingTransform::remerge() new_chunks.emplace_back(std::move(chunk)); } - LOG_DEBUG(log, "Memory usage is lowered from " - << formatReadableSizeWithBinarySuffix(sum_bytes_in_blocks) << " to " - << formatReadableSizeWithBinarySuffix(new_sum_bytes_in_blocks)); + LOG_DEBUG(log, "Memory usage is lowered from " << formatReadableSizeWithBinarySuffix(sum_bytes_in_blocks) << " to " << formatReadableSizeWithBinarySuffix(new_sum_bytes_in_blocks)); /// If the memory consumption was not lowered enough - we will not perform remerge anymore. 2 is a guess. if (new_sum_bytes_in_blocks * 2 > sum_bytes_in_blocks) diff --git a/src/Processors/Transforms/MergingAggregatedTransform.cpp b/src/Processors/Transforms/MergingAggregatedTransform.cpp index 8340238f011..b7176925986 100644 --- a/src/Processors/Transforms/MergingAggregatedTransform.cpp +++ b/src/Processors/Transforms/MergingAggregatedTransform.cpp @@ -46,8 +46,7 @@ Chunk MergingAggregatedTransform::generate() if (!generate_started) { generate_started = true; - LOG_TRACE(log, "Read " << total_input_blocks << " blocks of partially aggregated data, total " << total_input_rows - << " rows."); + LOG_TRACE(log, "Read " << total_input_blocks << " blocks of partially aggregated data, total " << total_input_rows << " rows."); /// Exception safety. Make iterator valid in case any method below throws. next_block = blocks.begin(); diff --git a/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/src/Storages/Distributed/DistributedBlockOutputStream.cpp index 315c380838d..884add0b3fa 100644 --- a/src/Storages/Distributed/DistributedBlockOutputStream.cpp +++ b/src/Storages/Distributed/DistributedBlockOutputStream.cpp @@ -114,9 +114,7 @@ void DistributedBlockOutputStream::write(const Block & block) if (ordinary_block.has(col.name)) { ordinary_block.erase(col.name); - LOG_DEBUG(log, storage.getStorageID().getNameForLogs() - << ": column " + col.name + " will be removed, " - << "because it is MATERIALIZED"); + LOG_DEBUG(log, storage.getStorageID().getNameForLogs() << ": column " + col.name + " will be removed, " << "because it is MATERIALIZED"); } } @@ -413,9 +411,7 @@ void DistributedBlockOutputStream::writeSuffix() auto log_performance = [this] () { double elapsed = watch.elapsedSeconds(); - LOG_DEBUG(log, "It took " << std::fixed << std::setprecision(1) << elapsed << " sec. to insert " << inserted_blocks << " blocks" - << ", " << std::fixed << std::setprecision(1) << inserted_rows / elapsed << " rows per second" - << ". " << getCurrentStateDescription()); + LOG_DEBUG(log, "It took " << std::fixed << std::setprecision(1) << elapsed << " sec. to insert " << inserted_blocks << " blocks" << ", " << std::fixed << std::setprecision(1) << inserted_rows / elapsed << " rows per second" << ". " << getCurrentStateDescription()); }; if (insert_sync && pool) diff --git a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp index eb373d40d8f..9af073b13ef 100644 --- a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp +++ b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp @@ -104,17 +104,11 @@ void ReadBufferFromKafkaConsumer::commit() if (topic_part.get_offset() < 0) { - LOG_TRACE( - log, - prefix << " " << print_special_offset() << " (topic: " << topic_part.get_topic() - << ", partition: " << topic_part.get_partition() << ")"); + LOG_TRACE( log, prefix << " " << print_special_offset() << " (topic: " << topic_part.get_topic() << ", partition: " << topic_part.get_partition() << ")"); } else { - LOG_TRACE( - log, - prefix << " " << topic_part.get_offset() << " (topic: " << topic_part.get_topic() - << ", partition: " << topic_part.get_partition() << ")"); + LOG_TRACE( log, prefix << " " << topic_part.get_offset() << " (topic: " << topic_part.get_topic() << ", partition: " << topic_part.get_partition() << ")"); } } }; @@ -172,9 +166,7 @@ void ReadBufferFromKafkaConsumer::commit() void ReadBufferFromKafkaConsumer::subscribe() { - LOG_TRACE(log,"Already subscribed to topics: [ " - << boost::algorithm::join(consumer->get_subscription(), ", ") - << " ]"); + LOG_TRACE(log,"Already subscribed to topics: [ " << boost::algorithm::join(consumer->get_subscription(), ", ") << " ]"); LOG_TRACE_FORMATTED(log, "Already assigned to : {}", assignment); diff --git a/src/Storages/Kafka/parseSyslogLevel.cpp b/src/Storages/Kafka/parseSyslogLevel.cpp index 692bc03000b..db948d5c668 100644 --- a/src/Storages/Kafka/parseSyslogLevel.cpp +++ b/src/Storages/Kafka/parseSyslogLevel.cpp @@ -12,20 +12,13 @@ std::pair parseSyslogLevel(const int lev switch (level) { case LOG_EMERG: [[fallthrough]]; - case LOG_ALERT: - return std::make_pair(Message::PRIO_FATAL, LogsLevel::error); - case LOG_CRIT: - return std::make_pair(Message::PRIO_CRITICAL, LogsLevel::error); - case LOG_ERR: - return std::make_pair(Message::PRIO_ERROR, LogsLevel::error); - case LOG_WARNING: - return std::make_pair(Message::PRIO_WARNING, LogsLevel::warning); - case LOG_NOTICE: - return std::make_pair(Message::PRIO_NOTICE, LogsLevel::information); - case LOG_INFO: - return std::make_pair(Message::PRIO_INFORMATION, LogsLevel::information); - case LOG_DEBUG: - return std::make_pair(Message::PRIO_DEBUG, LogsLevel::debug); + case LOG_ALERT: return std::make_pair(Message::PRIO_FATAL, LogsLevel::error); + case LOG_CRIT: return std::make_pair(Message::PRIO_CRITICAL, LogsLevel::error); + case LOG_ERR: return std::make_pair(Message::PRIO_ERROR, LogsLevel::error); + case LOG_WARNING: return std::make_pair(Message::PRIO_WARNING, LogsLevel::warning); + case LOG_NOTICE: return std::make_pair(Message::PRIO_NOTICE, LogsLevel::information); + case LOG_INFO: return std::make_pair(Message::PRIO_INFORMATION, LogsLevel::information); + case LOG_DEBUG: return std::make_pair(Message::PRIO_DEBUG, LogsLevel::debug); default: return std::make_pair(Message::PRIO_TRACE, LogsLevel::trace); } diff --git a/src/Storages/MergeTree/EphemeralLockInZooKeeper.cpp b/src/Storages/MergeTree/EphemeralLockInZooKeeper.cpp index 154c5992d1e..909930d6aec 100644 --- a/src/Storages/MergeTree/EphemeralLockInZooKeeper.cpp +++ b/src/Storages/MergeTree/EphemeralLockInZooKeeper.cpp @@ -107,8 +107,7 @@ EphemeralLocksInAllPartitions::EphemeralLocksInAllPartitions( int rc = zookeeper.tryMulti(lock_ops, lock_responses); if (rc == Coordination::ZBADVERSION) { - LOG_TRACE(&Logger::get("EphemeralLocksInAllPartitions"), - "Someone has inserted a block in a new partition while we were creating locks. Retry."); + LOG_TRACE(&Logger::get("EphemeralLocksInAllPartitions"), "Someone has inserted a block in a new partition while we were creating locks. Retry."); continue; } else if (rc != Coordination::ZOK) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index ed6a0784f62..456c3665d5a 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -257,8 +257,7 @@ void IMergeTreeDataPart::removeIfNeeded() if (!startsWith(file_name, "tmp")) { - LOG_ERROR(storage.log, "~DataPart() should remove part " << path - << " but its name doesn't start with tmp. Too suspicious, keeping the part."); + LOG_ERROR(storage.log, "~DataPart() should remove part " << path << " but its name doesn't start with tmp. Too suspicious, keeping the part."); return; } } @@ -669,8 +668,7 @@ void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_ Names files; volume->getDisk()->listFiles(to, files); - LOG_WARNING(storage.log, "Part directory " << fullPath(volume->getDisk(), to) << " already exists" - << " and contains " << files.size() << " files. Removing it."); + LOG_WARNING(storage.log, "Part directory " << fullPath(volume->getDisk(), to) << " already exists" << " and contains " << files.size() << " files. Removing it."); volume->getDisk()->removeRecursive(to); } @@ -712,8 +710,7 @@ void IMergeTreeDataPart::remove() const if (volume->getDisk()->exists(to)) { - LOG_WARNING(storage.log, "Directory " << fullPath(volume->getDisk(), to) << " (to which part must be renamed before removing) already exists." - " Most likely this is due to unclean restart. Removing it."); + LOG_WARNING(storage.log, "Directory " << fullPath(volume->getDisk(), to) << " (to which part must be renamed before removing) already exists." " Most likely this is due to unclean restart. Removing it."); try { @@ -732,8 +729,7 @@ void IMergeTreeDataPart::remove() const } catch (const Poco::FileNotFoundException &) { - LOG_ERROR(storage.log, "Directory " << fullPath(volume->getDisk(), to) << " (part to remove) doesn't exist or one of nested files has gone." - " Most likely this is due to manual removing. This should be discouraged. Ignoring."); + LOG_ERROR(storage.log, "Directory " << fullPath(volume->getDisk(), to) << " (part to remove) doesn't exist or one of nested files has gone." " Most likely this is due to manual removing. This should be discouraged. Ignoring."); return; } @@ -769,8 +765,7 @@ void IMergeTreeDataPart::remove() const { /// Recursive directory removal does many excessive "stat" syscalls under the hood. - LOG_ERROR(storage.log, "Cannot quickly remove directory " << fullPath(volume->getDisk(), to) << " by removing files; fallback to recursive removal. Reason: " - << getCurrentExceptionMessage(false)); + LOG_ERROR(storage.log, "Cannot quickly remove directory " << fullPath(volume->getDisk(), to) << " by removing files; fallback to recursive removal. Reason: " << getCurrentExceptionMessage(false)); volume->getDisk()->removeRecursive(to + "/"); } @@ -796,8 +791,7 @@ String IMergeTreeDataPart::getRelativePathForDetachedPart(const String & prefix) if (!volume->getDisk()->exists(getFullRelativePath() + res)) return res; - LOG_WARNING(storage.log, "Directory " << res << " (to detach to) already exists." - " Will detach to directory with '_tryN' suffix."); + LOG_WARNING(storage.log, "Directory " << res << " (to detach to) already exists." " Will detach to directory with '_tryN' suffix."); } return res; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 02d4d4aa370..e9c3fe13d97 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -204,8 +204,7 @@ MergeTreeData::MergeTreeData( { if (!version_file.first.empty()) { - LOG_ERROR(log, "Duplication of version file " << - fullPath(version_file.second, version_file.first) << " and " << current_version_file_path); + LOG_ERROR(log, "Duplication of version file " << fullPath(version_file.second, version_file.first) << " and " << current_version_file_path); throw Exception("Multiple format_version.txt file", ErrorCodes::CORRUPTED_DATA); } version_file = {current_version_file_path, disk}; @@ -1009,8 +1008,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) } else { - LOG_ERROR(log, "Detaching broken part " << getFullPathOnDisk(part_disk_ptr) << part_name - << " because it covers less than 2 parts. You need to resolve this manually"); + LOG_ERROR(log, "Detaching broken part " << getFullPathOnDisk(part_disk_ptr) << part_name << " because it covers less than 2 parts. You need to resolve this manually"); std::lock_guard loading_lock(mutex); broken_parts_to_detach.push_back(part); ++suspicious_broken_parts; @@ -2090,9 +2088,7 @@ restore_covered) if (error) { - LOG_ERROR(log, "The set of parts restored in place of " << part->name << " looks incomplete." - << " There might or might not be a data loss." - << (error_parts.empty() ? "" : " Suspicious parts: " + error_parts)); + LOG_ERROR(log, "The set of parts restored in place of " << part->name << " looks incomplete." << " There might or might not be a data loss." << (error_parts.empty() ? "" : " Suspicious parts: " + error_parts)); } } } @@ -2245,8 +2241,7 @@ void MergeTreeData::delayInsertOrThrowIfNeeded(Poco::Event * until) const CurrentMetrics::Increment metric_increment(CurrentMetrics::DelayedInserts); - LOG_INFO(log, "Delaying inserting block by " - << std::fixed << std::setprecision(4) << delay_milliseconds << " ms. because there are " << parts_count_in_partition << " parts"); + LOG_INFO(log, "Delaying inserting block by " << std::fixed << std::setprecision(4) << delay_milliseconds << " ms. because there are " << parts_count_in_partition << " parts"); if (until) until->tryWait(delay_milliseconds); @@ -2444,9 +2439,7 @@ void MergeTreeData::removePartContributionToColumnSizes(const DataPartPtr & part auto log_subtract = [&](size_t & from, size_t value, const char * field) { if (value > from) - LOG_ERROR(log, "Possibly incorrect column size subtraction: " - << from << " - " << value << " = " << from - value - << ", column: " << column.name << ", field: " << field); + LOG_ERROR(log, "Possibly incorrect column size subtraction: " << from << " - " << value << " = " << from - value << ", column: " << column.name << ", field: " << field); from -= value; }; @@ -2921,13 +2914,9 @@ ReservationPtr MergeTreeData::tryReserveSpacePreferringTTLRules(UInt64 expected_ if (!destination_ptr) { if (ttl_entry->destination_type == PartDestinationType::VOLUME) - LOG_WARNING(log, "Would like to reserve space on volume '" - << ttl_entry->destination_name << "' by TTL rule of table '" - << log_name << "' but volume was not found"); + LOG_WARNING(log, "Would like to reserve space on volume '" << ttl_entry->destination_name << "' by TTL rule of table '" << log_name << "' but volume was not found"); else if (ttl_entry->destination_type == PartDestinationType::DISK) - LOG_WARNING(log, "Would like to reserve space on disk '" - << ttl_entry->destination_name << "' by TTL rule of table '" - << log_name << "' but disk was not found"); + LOG_WARNING(log, "Would like to reserve space on disk '" << ttl_entry->destination_name << "' by TTL rule of table '" << log_name << "' but disk was not found"); } else { @@ -2936,13 +2925,9 @@ ReservationPtr MergeTreeData::tryReserveSpacePreferringTTLRules(UInt64 expected_ return reservation; else if (ttl_entry->destination_type == PartDestinationType::VOLUME) - LOG_WARNING(log, "Would like to reserve space on volume '" - << ttl_entry->destination_name << "' by TTL rule of table '" - << log_name << "' but there is not enough space"); + LOG_WARNING(log, "Would like to reserve space on volume '" << ttl_entry->destination_name << "' by TTL rule of table '" << log_name << "' but there is not enough space"); else if (ttl_entry->destination_type == PartDestinationType::DISK) - LOG_WARNING(log, "Would like to reserve space on disk '" - << ttl_entry->destination_name << "' by TTL rule of table '" - << log_name << "' but there is not enough space"); + LOG_WARNING(log, "Would like to reserve space on disk '" << ttl_entry->destination_name << "' by TTL rule of table '" << log_name << "' but there is not enough space"); } } @@ -3068,8 +3053,7 @@ MergeTreeData::DataPartsVector MergeTreeData::Transaction::commit(MergeTreeData: DataPartsVector covered_parts = data.getActivePartsToReplace(part->info, part->name, covering_part, *owing_parts_lock); if (covering_part) { - LOG_WARNING(data.log, "Tried to commit obsolete part " << part->name - << " covered by " << covering_part->getNameWithState()); + LOG_WARNING(data.log, "Tried to commit obsolete part " << part->name << " covered by " << covering_part->getNameWithState()); part->remove_time.store(0, std::memory_order_relaxed); /// The part will be removed without waiting for old_parts_lifetime seconds. data.modifyPartState(part, DataPartState::Outdated); diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index e124189b31b..fe07bea828f 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -366,12 +366,7 @@ bool MergeTreeDataMergerMutator::selectAllPartsToMergeWithinPartition( if (now - disk_space_warning_time > 3600) { disk_space_warning_time = now; - LOG_WARNING(log, "Won't merge parts from " << parts.front()->name << " to " << (*prev_it)->name - << " because not enough free space: " - << formatReadableSizeWithBinarySuffix(available_disk_space) << " free and unreserved, " - << formatReadableSizeWithBinarySuffix(sum_bytes) - << " required now (+" << static_cast((DISK_USAGE_COEFFICIENT_TO_SELECT - 1.0) * 100) - << "% on overhead); suppressing similar warnings for the next hour"); + LOG_WARNING(log, "Won't merge parts from " << parts.front()->name << " to " << (*prev_it)->name << " because not enough free space: " << formatReadableSizeWithBinarySuffix(available_disk_space) << " free and unreserved, " << formatReadableSizeWithBinarySuffix(sum_bytes) << " required now (+" << static_cast((DISK_USAGE_COEFFICIENT_TO_SELECT - 1.0) * 100) << "% on overhead); suppressing similar warnings for the next hour"); } if (out_disable_reason) @@ -588,9 +583,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor const MergeTreeData::DataPartsVector & parts = future_part.parts; - LOG_DEBUG(log, "Merging " << parts.size() << " parts: from " - << parts.front()->name << " to " << parts.back()->name - << " into " << TMP_PREFIX + future_part.name + " with type " + future_part.type.toString()); + LOG_DEBUG(log, "Merging " << parts.size() << " parts: from " << parts.front()->name << " to " << parts.back()->name << " into " << TMP_PREFIX + future_part.name + " with type " + future_part.type.toString()); auto disk = space_reservation->getDisk(); String part_path = data.relative_data_path; @@ -949,10 +942,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor /// Print overall profiling info. NOTE: it may duplicates previous messages { double elapsed_seconds = merge_entry->watch.elapsedSeconds(); - LOG_DEBUG_FORMATTED(log, "Merge sorted {} rows, containing {} columns ({} merged, {} gathered) in {} sec., {} rows/sec., {}/sec.", - merge_entry->rows_read, all_column_names.size(), merging_column_names.size(), gathering_column_names.size(), - elapsed_seconds, merge_entry->rows_read / elapsed_seconds, - formatReadableSizeWithBinarySuffix(merge_entry->bytes_read_uncompressed / elapsed_seconds)); + LOG_DEBUG_FORMATTED(log, "Merge sorted {} rows, containing {} columns ({} merged, {} gathered) in {} sec., {} rows/sec., {}/sec.", merge_entry->rows_read, all_column_names.size(), merging_column_names.size(), gathering_column_names.size(), elapsed_seconds, merge_entry->rows_read / elapsed_seconds, formatReadableSizeWithBinarySuffix(merge_entry->bytes_read_uncompressed / elapsed_seconds)); } if (merge_alg != MergeAlgorithm::Vertical) @@ -1224,8 +1214,7 @@ MergeTreeData::DataPartPtr MergeTreeDataMergerMutator::renameMergedTemporaryPart * (NOTE: Merging with part that is not in ZK is not possible, see checks in 'createLogEntryToMergeParts'.) * - and after merge, this part will be removed in addition to parts that was merged. */ - LOG_WARNING(log, "Unexpected number of parts removed when adding " << new_data_part->name << ": " << replaced_parts.size() - << " instead of " << parts.size()); + LOG_WARNING(log, "Unexpected number of parts removed when adding " << new_data_part->name << ": " << replaced_parts.size() << " instead of " << parts.size()); } else { diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index c8018b651b6..5a7f1690d42 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -586,8 +586,7 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( } } - LOG_DEBUG(log, "Selected " << parts.size() << " parts by date, " << parts_with_ranges.size() << " parts by key, " - << sum_marks << " marks to read from " << sum_ranges << " ranges"); + LOG_DEBUG(log, "Selected " << parts.size() << " parts by date, " << parts_with_ranges.size() << " parts by key, " << sum_marks << " marks to read from " << sum_ranges << " ranges"); if (parts_with_ranges.empty()) return {}; diff --git a/src/Storages/MergeTree/MergeTreePartsMover.cpp b/src/Storages/MergeTree/MergeTreePartsMover.cpp index 1244ae61556..10e9314018a 100644 --- a/src/Storages/MergeTree/MergeTreePartsMover.cpp +++ b/src/Storages/MergeTree/MergeTreePartsMover.cpp @@ -179,9 +179,7 @@ bool MergeTreePartsMover::selectPartsForMove( if (!parts_to_move.empty()) { - LOG_TRACE(log, "Selected " << parts_to_move_by_policy_rules << " parts to move according to storage policy rules and " - << parts_to_move_by_ttl_rules << " parts according to TTL rules, " - << formatReadableSizeWithBinarySuffix(parts_to_move_total_size_bytes) << " total"); + LOG_TRACE(log, "Selected " << parts_to_move_by_policy_rules << " parts to move according to storage policy rules and " << parts_to_move_by_ttl_rules << " parts according to TTL rules, " << formatReadableSizeWithBinarySuffix(parts_to_move_total_size_bytes) << " total"); return true; } else @@ -217,8 +215,7 @@ void MergeTreePartsMover::swapClonedPart(const MergeTreeData::DataPartPtr & clon /// It's ok, because we don't block moving parts for merges or mutations if (!active_part || active_part->name != cloned_part->name) { - LOG_INFO(log, "Failed to swap " << cloned_part->name << ". Active part doesn't exist." - << " Possible it was merged or mutated. Will remove copy on path '" << cloned_part->getFullPath() << "'."); + LOG_INFO(log, "Failed to swap " << cloned_part->name << ". Active part doesn't exist." << " Possible it was merged or mutated. Will remove copy on path '" << cloned_part->getFullPath() << "'."); return; } diff --git a/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp index cc1f713a57e..69e514f9822 100644 --- a/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp @@ -66,12 +66,7 @@ MergeTreeReverseSelectProcessor::MergeTreeReverseSelectProcessor( size_t total_rows = data_part->index_granularity.getTotalRows(); if (!quiet) - LOG_TRACE(log, "Reading " << all_mark_ranges.size() << " ranges in reverse order from part " << data_part->name - << ", approx. " << total_rows - << (all_mark_ranges.size() > 1 - ? ", up to " + toString(data_part->index_granularity.getRowsCountInRanges(all_mark_ranges)) - : "") - << " rows starting from " << data_part->index_granularity.getMarkStartingRow(all_mark_ranges.front().begin)); + LOG_TRACE(log, "Reading " << all_mark_ranges.size() << " ranges in reverse order from part " << data_part->name << ", approx. " << total_rows << (all_mark_ranges.size() > 1 ? ", up to " + toString(data_part->index_granularity.getRowsCountInRanges(all_mark_ranges)) : "") << " rows starting from " << data_part->index_granularity.getMarkStartingRow(all_mark_ranges.front().begin)); addTotalRowsApprox(total_rows); diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp index 9af72e60123..5aaaf6c280a 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp @@ -47,12 +47,7 @@ MergeTreeSelectProcessor::MergeTreeSelectProcessor( size_t total_rows = data_part->index_granularity.getRowsCountInRanges(all_mark_ranges); if (!quiet) - LOG_TRACE(log, "Reading " << all_mark_ranges.size() << " ranges from part " << data_part->name - << ", approx. " << total_rows - << (all_mark_ranges.size() > 1 - ? ", up to " + toString(total_rows) - : "") - << " rows starting from " << data_part->index_granularity.getMarkStartingRow(all_mark_ranges.front().begin)); + LOG_TRACE(log, "Reading " << all_mark_ranges.size() << " ranges from part " << data_part->name << ", approx. " << total_rows << (all_mark_ranges.size() > 1 ? ", up to " + toString(total_rows) : "") << " rows starting from " << data_part->index_granularity.getMarkStartingRow(all_mark_ranges.front().begin)); addTotalRowsApprox(total_rows); ordered_names = header_without_virtual_columns.getNames(); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp index e0e131dc78b..3141bb84acf 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp @@ -327,8 +327,7 @@ void ReplicatedMergeTreeCleanupThread::clearOldBlocks() cached_block_stats.erase(first_outdated_block->node); } else if (rc) - LOG_WARNING(log, - "Error while deleting ZooKeeper path `" << path << "`: " + zkutil::ZooKeeper::error2string(rc) << ", ignoring."); + LOG_WARNING(log, "Error while deleting ZooKeeper path `" << path << "`: " + zkutil::ZooKeeper::error2string(rc) << ", ignoring."); else { /// Successfully removed blocks have to be removed from cache @@ -369,8 +368,7 @@ void ReplicatedMergeTreeCleanupThread::getBlocksSortedByTime(zkutil::ZooKeeper & auto not_cached_blocks = stat.numChildren - cached_block_stats.size(); if (not_cached_blocks) { - LOG_TRACE(log, "Checking " << stat.numChildren << " blocks (" << not_cached_blocks << " are not cached)" - << " to clear old ones from ZooKeeper."); + LOG_TRACE(log, "Checking " << stat.numChildren << " blocks (" << not_cached_blocks << " are not cached)" << " to clear old ones from ZooKeeper."); } zkutil::AsyncResponses exists_futures; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp index ade201c04de..e61ce4e4abf 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp @@ -82,8 +82,7 @@ void ReplicatedMergeTreePartCheckThread::searchForMissingPart(const String & par /// If the part is in ZooKeeper, remove it from there and add the task to download it to the queue. if (zookeeper->exists(part_path)) { - LOG_WARNING(log, "Part " << part_name << " exists in ZooKeeper but not locally. " - "Removing from ZooKeeper and queueing a fetch."); + LOG_WARNING(log, "Part " << part_name << " exists in ZooKeeper but not locally. " "Removing from ZooKeeper and queueing a fetch."); ProfileEvents::increment(ProfileEvents::ReplicatedPartChecksFailed); storage.removePartAndEnqueueFetch(part_name); @@ -144,9 +143,7 @@ void ReplicatedMergeTreePartCheckThread::searchForMissingPart(const String & par if (found_part_with_the_same_min_block && found_part_with_the_same_max_block) { - LOG_WARNING(log, - "Found parts with the same min block and with the same max block as the missing part " - << part_name << ". Hoping that it will eventually appear as a result of a merge."); + LOG_WARNING(log, "Found parts with the same min block and with the same max block as the missing part " << part_name << ". Hoping that it will eventually appear as a result of a merge."); return; } } @@ -161,8 +158,7 @@ void ReplicatedMergeTreePartCheckThread::searchForMissingPart(const String & par not_found_msg = "a smaller part with the same min block."; else not_found_msg = "smaller parts with either the same min block or the same max block."; - LOG_ERROR(log, "No replica has part covering " << part_name - << " and a merge is impossible: we didn't find " << not_found_msg); + LOG_ERROR(log, "No replica has part covering " << part_name << " and a merge is impossible: we didn't find " << not_found_msg); ProfileEvents::increment(ProfileEvents::ReplicatedPartChecksFailed); @@ -289,9 +285,7 @@ CheckResult ReplicatedMergeTreePartCheckThread::checkPart(const String & part_na /// but remained in the filesystem and in a number of active parts. /// And then for a long time (before restarting), the data on the replicas will be different. - LOG_TRACE(log, "Young part " << part_name - << " with age " << (time(nullptr) - part->modification_time) - << " seconds hasn't been added to ZooKeeper yet. It's ok."); + LOG_TRACE(log, "Young part " << part_name << " with age " << (time(nullptr) - part->modification_time) << " seconds hasn't been added to ZooKeeper yet. It's ok."); } } else diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 3d04a51daf9..eaf77a637d8 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -75,9 +75,7 @@ bool ReplicatedMergeTreeQueue::load(zkutil::ZooKeeperPtr zookeeper) return already_loaded_paths.count(path); }); - LOG_DEBUG(log, - "Having " << (to_remove_it - children.begin()) << " queue entries to load, " - << (children.end() - to_remove_it) << " entries already loaded."); + LOG_DEBUG(log, "Having " << (to_remove_it - children.begin()) << " queue entries to load, " << (children.end() - to_remove_it) << " entries already loaded."); children.erase(to_remove_it, children.end()); std::sort(children.begin(), children.end()); @@ -322,9 +320,7 @@ void ReplicatedMergeTreeQueue::updateTimesInZooKeeper( auto code = zookeeper->tryMulti(ops, responses); if (code) - LOG_ERROR(log, "Couldn't set value of nodes for insert times (" - << replica_path << "/min_unprocessed_insert_time, max_processed_insert_time)" << ": " - << zkutil::ZooKeeper::error2string(code) + ". This shouldn't happen often."); + LOG_ERROR(log, "Couldn't set value of nodes for insert times (" << replica_path << "/min_unprocessed_insert_time, max_processed_insert_time)" << ": " << zkutil::ZooKeeper::error2string(code) + ". This shouldn't happen often."); } } @@ -369,8 +365,7 @@ void ReplicatedMergeTreeQueue::removeProcessedEntry(zkutil::ZooKeeperPtr zookeep auto code = zookeeper->tryRemove(replica_path + "/queue/" + entry->znode_name); if (code) - LOG_ERROR(log, "Couldn't remove " << replica_path << "/queue/" << entry->znode_name << ": " - << zkutil::ZooKeeper::error2string(code) << ". This shouldn't happen often."); + LOG_ERROR(log, "Couldn't remove " << replica_path << "/queue/" << entry->znode_name << ": " << zkutil::ZooKeeper::error2string(code) << ". This shouldn't happen often."); updateTimesInZooKeeper(zookeeper, min_unprocessed_insert_time_changed, max_processed_insert_time_changed); } @@ -650,8 +645,7 @@ void ReplicatedMergeTreeQueue::updateMutations(zkutil::ZooKeeperPtr zookeeper, C if (!entries_to_load.empty()) { - LOG_INFO(log, "Loading " + toString(entries_to_load.size()) + " mutation entries: " - + entries_to_load.front() + " - " + entries_to_load.back()); + LOG_INFO(log, "Loading " + toString(entries_to_load.size()) + " mutation entries: " + entries_to_load.front() + " - " + entries_to_load.back()); std::vector> futures; for (const String & entry : entries_to_load) @@ -851,8 +845,7 @@ void ReplicatedMergeTreeQueue::removePartProducingOpsInRange( to_wait.push_back(*it); auto code = zookeeper->tryRemove(replica_path + "/queue/" + (*it)->znode_name); if (code) - LOG_INFO(log, "Couldn't remove " << replica_path + "/queue/" + (*it)->znode_name << ": " - << zkutil::ZooKeeper::error2string(code)); + LOG_INFO(log, "Couldn't remove " << replica_path + "/queue/" + (*it)->znode_name << ": " << zkutil::ZooKeeper::error2string(code)); updateStateOnQueueEntryRemoval( *it, /* is_successful = */ false, @@ -866,8 +859,7 @@ void ReplicatedMergeTreeQueue::removePartProducingOpsInRange( updateTimesInZooKeeper(zookeeper, min_unprocessed_insert_time_changed, max_processed_insert_time_changed); - LOG_DEBUG(log, "Removed " << removed_entries << " entries from queue. " - "Waiting for " << to_wait.size() << " entries that are currently executing."); + LOG_DEBUG(log, "Removed " << removed_entries << " entries from queue. " "Waiting for " << to_wait.size() << " entries that are currently executing."); /// Let's wait for the operations with the parts contained in the range to be deleted. for (LogEntryPtr & entry : to_wait) @@ -1336,8 +1328,7 @@ MutationCommands ReplicatedMergeTreeQueue::getMutationCommands( if (part->info.getDataVersion() > desired_mutation_version) { - LOG_WARNING(log, "Data version of part " << part->name << " is already greater than " - "desired mutation version " << desired_mutation_version); + LOG_WARNING(log, "Data version of part " << part->name << " is already greater than " "desired mutation version " << desired_mutation_version); return MutationCommands{}; } @@ -1346,8 +1337,7 @@ MutationCommands ReplicatedMergeTreeQueue::getMutationCommands( auto in_partition = mutations_by_partition.find(part->info.partition_id); if (in_partition == mutations_by_partition.end()) { - LOG_WARNING(log, "There are no mutations for partition ID " << part->info.partition_id - << " (trying to mutate part " << part->name << " to " << toString(desired_mutation_version) << ")"); + LOG_WARNING(log, "There are no mutations for partition ID " << part->info.partition_id << " (trying to mutate part " << part->name << " to " << toString(desired_mutation_version) << ")"); return MutationCommands{}; } @@ -1355,9 +1345,7 @@ MutationCommands ReplicatedMergeTreeQueue::getMutationCommands( auto end = in_partition->second.lower_bound(desired_mutation_version); if (end == in_partition->second.end() || end->first != desired_mutation_version) - LOG_WARNING(log, "Mutation with version " << desired_mutation_version - << " not found in partition ID " << part->info.partition_id - << " (trying to mutate part " << part->name + ")"); + LOG_WARNING(log, "Mutation with version " << desired_mutation_version << " not found in partition ID " << part->info.partition_id << " (trying to mutate part " << part->name + ")"); else ++end; @@ -1390,8 +1378,7 @@ bool ReplicatedMergeTreeQueue::tryFinalizeMutations(zkutil::ZooKeeperPtr zookeep alter_sequence.finishDataAlter(mutation.entry->alter_version, lock); if (mutation.parts_to_do.size() != 0) { - LOG_INFO(log, "Seems like we jumped over mutation " << znode << " when downloaded part with bigger mutation number." - << " It's OK, tasks for rest parts will be skipped, but probably a lot of mutations were executed concurrently on different replicas."); + LOG_INFO(log, "Seems like we jumped over mutation " << znode << " when downloaded part with bigger mutation number." << " It's OK, tasks for rest parts will be skipped, but probably a lot of mutations were executed concurrently on different replicas."); mutation.parts_to_do.clear(); } } @@ -1898,9 +1885,7 @@ bool ReplicatedMergeTreeMergePredicate::isMutationFinished(const ReplicatedMerge partition_it->second.begin(), partition_it->second.lower_bound(block_num)); if (blocks_count) { - LOG_TRACE(queue.log, "Mutation " << mutation.znode_name << " is not done yet because " - << "in partition ID " << partition_id << " there are still " - << blocks_count << " uncommitted blocks."); + LOG_TRACE(queue.log, "Mutation " << mutation.znode_name << " is not done yet because " << "in partition ID " << partition_id << " there are still " << blocks_count << " uncommitted blocks."); return false; } } @@ -1912,8 +1897,7 @@ bool ReplicatedMergeTreeMergePredicate::isMutationFinished(const ReplicatedMerge size_t suddenly_appeared_parts = getPartNamesToMutate(mutation, queue.virtual_parts).size(); if (suddenly_appeared_parts) { - LOG_TRACE(queue.log, "Mutation " << mutation.znode_name << " is not done yet because " - << suddenly_appeared_parts << " parts to mutate suddenly appeared."); + LOG_TRACE(queue.log, "Mutation " << mutation.znode_name << " is not done yet because " << suddenly_appeared_parts << " parts to mutate suddenly appeared."); return false; } } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index 5e47cd9739d..3d51b611c03 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -141,8 +141,7 @@ void ReplicatedMergeTreeRestartingThread::run() if (storage.is_leader && relative_delay > static_cast(storage_settings->min_relative_delay_to_yield_leadership)) { - LOG_INFO(log, "Relative replica delay (" << relative_delay << " seconds) is bigger than threshold (" - << storage_settings->min_relative_delay_to_yield_leadership << "). Will yield leadership."); + LOG_INFO(log, "Relative replica delay (" << relative_delay << " seconds) is bigger than threshold (" << storage_settings->min_relative_delay_to_yield_leadership << "). Will yield leadership."); ProfileEvents::increment(ProfileEvents::ReplicaYieldLeadership); @@ -269,8 +268,7 @@ void ReplicatedMergeTreeRestartingThread::updateQuorumIfWeHavePart() if (!quorum_entry.replicas.count(storage.replica_name) && zookeeper->exists(storage.replica_path + "/parts/" + quorum_entry.part_name)) { - LOG_WARNING(log, "We have part " << quorum_entry.part_name - << " but we is not in quorum. Updating quorum. This shouldn't happen often."); + LOG_WARNING(log, "We have part " << quorum_entry.part_name << " but we is not in quorum. Updating quorum. This shouldn't happen often."); storage.updateQuorum(quorum_entry.part_name); } } diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index 960cb1ea4d2..1f76ed2624c 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -189,8 +189,7 @@ Pipes StorageBuffer::read( { if (!dest_columns.hasPhysical(column_name)) { - LOG_WARNING(log, "Destination table " << destination_id.getNameForLogs() - << " doesn't have column " << backQuoteIfNeed(column_name) << ". The default values are used."); + LOG_WARNING(log, "Destination table " << destination_id.getNameForLogs() << " doesn't have column " << backQuoteIfNeed(column_name) << ". The default values are used."); boost::range::remove_erase(columns_intersection, column_name); continue; } @@ -198,17 +197,14 @@ Pipes StorageBuffer::read( const auto & col = our_columns.getPhysical(column_name); if (!dst_col.type->equals(*col.type)) { - LOG_WARNING(log, "Destination table " << destination_id.getNameForLogs() - << " has different type of column " << backQuoteIfNeed(column_name) << " (" - << dst_col.type->getName() << " != " << col.type->getName() << "). Data from destination table are converted."); + LOG_WARNING(log, "Destination table " << destination_id.getNameForLogs() << " has different type of column " << backQuoteIfNeed(column_name) << " (" << dst_col.type->getName() << " != " << col.type->getName() << "). Data from destination table are converted."); header_after_adding_defaults.getByName(column_name) = ColumnWithTypeAndName(dst_col.type, column_name); } } if (columns_intersection.empty()) { - LOG_WARNING(log, "Destination table " << destination_id.getNameForLogs() - << " has no common columns with block in buffer. Block of data is skipped."); + LOG_WARNING(log, "Destination table " << destination_id.getNameForLogs() << " has no common columns with block in buffer. Block of data is skipped."); } else { @@ -451,8 +447,7 @@ void StorageBuffer::startup() { if (global_context.getSettingsRef().readonly) { - LOG_WARNING(log, "Storage " << getName() << " is run with readonly settings, it will not be able to insert data." - << " Set appropriate system_profile to fix this."); + LOG_WARNING(log, "Storage " << getName() << " is run with readonly settings, it will not be able to insert data." << " Set appropriate system_profile to fix this."); } @@ -658,10 +653,7 @@ void StorageBuffer::writeBlockToDestination(const Block & block, StoragePtr tabl auto column = block.getByName(dst_col.name); if (!column.type->equals(*dst_col.type)) { - LOG_WARNING(log, "Destination table " << destination_id.getNameForLogs() - << " have different type of column " << backQuoteIfNeed(column.name) << " (" - << dst_col.type->getName() << " != " << column.type->getName() - << "). Block of data is converted."); + LOG_WARNING(log, "Destination table " << destination_id.getNameForLogs() << " have different type of column " << backQuoteIfNeed(column.name) << " (" << dst_col.type->getName() << " != " << column.type->getName() << "). Block of data is converted."); column.column = castColumn(column, dst_col.type); column.type = dst_col.type; } @@ -672,14 +664,12 @@ void StorageBuffer::writeBlockToDestination(const Block & block, StoragePtr tabl if (block_to_write.columns() == 0) { - LOG_ERROR(log, "Destination table " << destination_id.getNameForLogs() - << " have no common columns with block in buffer. Block of data is discarded."); + LOG_ERROR(log, "Destination table " << destination_id.getNameForLogs() << " have no common columns with block in buffer. Block of data is discarded."); return; } if (block_to_write.columns() != block.columns()) - LOG_WARNING(log, "Not all columns from block in buffer exist in destination table " - << destination_id.getNameForLogs() << ". Some columns are discarded."); + LOG_WARNING(log, "Not all columns from block in buffer exist in destination table " << destination_id.getNameForLogs() << ". Some columns are discarded."); auto list_of_columns = std::make_shared(); insert->columns = list_of_columns; diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index fd21393c96d..6acf26f82ce 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -477,15 +477,12 @@ Pipes StorageDistributed::read( ClusterPtr optimized_cluster = getOptimizedCluster(context, query_info.query); if (optimized_cluster) { - LOG_DEBUG(log, "Skipping irrelevant shards - the query will be sent to the following shards of the cluster (shard numbers): " << - makeFormattedListOfShards(optimized_cluster)); + LOG_DEBUG(log, "Skipping irrelevant shards - the query will be sent to the following shards of the cluster (shard numbers): " << makeFormattedListOfShards(optimized_cluster)); cluster = optimized_cluster; } else { - LOG_DEBUG(log, (has_sharding_key ? "" : " (no sharding key)") << ": " - "Unable to figure out irrelevant shards from WHERE/PREWHERE clauses - " - "the query will be sent to all shards of the cluster"); + LOG_DEBUG(log, (has_sharding_key ? "" : " (no sharding key)") << ": " "Unable to figure out irrelevant shards from WHERE/PREWHERE clauses - " "the query will be sent to all shards of the cluster"); } } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 29794b1da76..fb8f1df9fa6 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -214,8 +214,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( skip_sanity_checks = true; current_zookeeper->remove(replica_path + "/flags/force_restore_data"); - LOG_WARNING(log, "Skipping the limits on severity of changes to data parts and columns (flag " - << replica_path << "/flags/force_restore_data)."); + LOG_WARNING(log, "Skipping the limits on severity of changes to data parts and columns (flag " << replica_path << "/flags/force_restore_data)."); } else if (has_force_restore_data_flag) { @@ -343,8 +342,7 @@ void StorageReplicatedMergeTree::waitMutationToFinishOnReplicas( /// Replica could be inactive. if (!zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/is_active")) { - LOG_WARNING(log, "Replica " << replica << " is not active during mutation. " - "Mutation will be done asynchronously when replica becomes active."); + LOG_WARNING(log, "Replica " << replica << " is not active during mutation. " "Mutation will be done asynchronously when replica becomes active."); inactive_replicas.emplace(replica); break; @@ -779,8 +777,7 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: !zookeeper->exists(current_part_path + "/columns", &columns_stat_after) || columns_stat_before.version != columns_stat_after.version) { - LOG_INFO(log, "Not checking checksums of part " << part_name << " with replica " << replica - << " because part changed while we were reading its checksums"); + LOG_INFO(log, "Not checking checksums of part " << part_name << " with replica " << replica << " because part changed while we were reading its checksums"); continue; } @@ -790,8 +787,7 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: if (replica_part_header.getColumnsHash() != local_part_header.getColumnsHash()) { - LOG_INFO(log, "Not checking checksums of part " << part_name << " with replica " << replica - << " because columns are different"); + LOG_INFO(log, "Not checking checksums of part " << part_name << " with replica " << replica << " because columns are different"); continue; } @@ -834,8 +830,7 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: } else { - LOG_WARNING(log, "checkPartAndAddToZooKeeper: node " << replica_path + "/parts/" + part_name << " already exists." - << " Will not commit any nodes."); + LOG_WARNING(log, "checkPartAndAddToZooKeeper: node " << replica_path + "/parts/" + part_name << " already exists." << " Will not commit any nodes."); } } @@ -1008,8 +1003,7 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) } if (part->name != name) { - LOG_WARNING(log, "Part " << name << " is covered by " << part->name - << " but should be merged into " << entry.new_part_name << ". This shouldn't happen often."); + LOG_WARNING(log, "Part " << name << " is covered by " << part->name << " but should be merged into " << entry.new_part_name << ". This shouldn't happen often."); have_all_parts = false; break; } @@ -1101,19 +1095,7 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) ProfileEvents::increment(ProfileEvents::DataAfterMergeDiffersFromReplica); - LOG_ERROR(log, getCurrentExceptionMessage(false) << ". " - "Data after merge is not byte-identical to data on another replicas. " - "There could be several reasons: " - "1. Using newer version of compression library after server update. " - "2. Using another compression method. " - "3. Non-deterministic compression algorithm (highly unlikely). " - "4. Non-deterministic merge algorithm due to logical error in code. " - "5. Data corruption in memory due to bug in code. " - "6. Data corruption in memory due to hardware issue. " - "7. Manual modification of source data after server startup. " - "8. Manual modification of checksums stored in ZooKeeper. " - "9. Part format related settings like 'enable_mixed_granularity_parts' are different on different replicas. " - "We will download merged part from replica to force byte-identical result."); + LOG_ERROR(log, getCurrentExceptionMessage(false) << ". " "Data after merge is not byte-identical to data on another replicas. " "There could be several reasons: " "1. Using newer version of compression library after server update. " "2. Using another compression method. " "3. Non-deterministic compression algorithm (highly unlikely). " "4. Non-deterministic merge algorithm due to logical error in code. " "5. Data corruption in memory due to bug in code. " "6. Data corruption in memory due to hardware issue. " "7. Manual modification of source data after server startup. " "8. Manual modification of checksums stored in ZooKeeper. " "9. Part format related settings like 'enable_mixed_granularity_parts' are different on different replicas. " "We will download merged part from replica to force byte-identical result."); write_part_log(ExecutionStatus::fromCurrentException()); @@ -1235,9 +1217,7 @@ bool StorageReplicatedMergeTree::tryExecutePartMutation(const StorageReplicatedM ProfileEvents::increment(ProfileEvents::DataAfterMutationDiffersFromReplica); - LOG_ERROR(log, getCurrentExceptionMessage(false) << ". " - "Data after mutation is not byte-identical to data on another replicas. " - "We will download merged part from replica to force byte-identical result."); + LOG_ERROR(log, getCurrentExceptionMessage(false) << ". " "Data after mutation is not byte-identical to data on another replicas. " "We will download merged part from replica to force byte-identical result."); write_part_log(ExecutionStatus::fromCurrentException()); @@ -1308,8 +1288,7 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) if (entry.type != LogEntry::GET_PART) throw Exception("Logical error: log entry with quorum but type is not GET_PART", ErrorCodes::LOGICAL_ERROR); - LOG_DEBUG(log, "No active replica has part " << entry.new_part_name << " which needs to be written with quorum." - " Will try to mark that quorum as failed."); + LOG_DEBUG(log, "No active replica has part " << entry.new_part_name << " which needs to be written with quorum." " Will try to mark that quorum as failed."); /** Atomically: * - if replicas do not become active; @@ -1384,19 +1363,14 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) } else if (code == Coordination::ZBADVERSION || code == Coordination::ZNONODE || code == Coordination::ZNODEEXISTS) { - LOG_DEBUG(log, "State was changed or isn't expected when trying to mark quorum for part " - << entry.new_part_name << " as failed. Code: " << zkutil::ZooKeeper::error2string(code)); + LOG_DEBUG(log, "State was changed or isn't expected when trying to mark quorum for part " << entry.new_part_name << " as failed. Code: " << zkutil::ZooKeeper::error2string(code)); } else throw Coordination::Exception(code); } else { - LOG_WARNING(log, "No active replica has part " << entry.new_part_name - << ", but that part needs quorum and /quorum/status contains entry about another part " << quorum_entry.part_name - << ". It means that part was successfully written to " << entry.quorum - << " replicas, but then all of them goes offline." - << " Or it is a bug."); + LOG_WARNING(log, "No active replica has part " << entry.new_part_name << ", but that part needs quorum and /quorum/status contains entry about another part " << quorum_entry.part_name << ". It means that part was successfully written to " << entry.quorum << " replicas, but then all of them goes offline." << " Or it is a bug."); } } } @@ -1606,8 +1580,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) } catch (Exception &) { - LOG_INFO(log, "Can't use " << source_table_id.getNameForLogs() << " as source table for REPLACE PARTITION command. Will fetch all parts." - << " Reason: " << getCurrentExceptionMessage(false)); + LOG_INFO(log, "Can't use " << source_table_id.getNameForLogs() << " as source table for REPLACE PARTITION command. Will fetch all parts." << " Reason: " << getCurrentExceptionMessage(false)); return 0; } @@ -2214,9 +2187,7 @@ void StorageReplicatedMergeTree::mergeSelectingTask() size_t merges_and_mutations_sum = merges_and_mutations_queued.first + merges_and_mutations_queued.second; if (merges_and_mutations_sum >= storage_settings_ptr->max_replicated_merges_in_queue) { - LOG_TRACE(log, "Number of queued merges (" << merges_and_mutations_queued.first << ") and part mutations (" - << merges_and_mutations_queued.second << ") is greater than max_replicated_merges_in_queue (" - << storage_settings_ptr->max_replicated_merges_in_queue << "), so won't select new parts to merge or mutate."); + LOG_TRACE(log, "Number of queued merges (" << merges_and_mutations_queued.first << ") and part mutations (" << merges_and_mutations_queued.second << ") is greater than max_replicated_merges_in_queue (" << storage_settings_ptr->max_replicated_merges_in_queue << "), so won't select new parts to merge or mutate."); } else { @@ -2326,10 +2297,7 @@ bool StorageReplicatedMergeTree::createLogEntryToMergeParts( const auto & part = parts[i]; if (part->modification_time + MAX_AGE_OF_LOCAL_PART_THAT_WASNT_ADDED_TO_ZOOKEEPER < time(nullptr)) { - LOG_WARNING(log, "Part " << part->name << " (that was selected for merge)" - << " with age " << (time(nullptr) - part->modification_time) - << " seconds exists locally but not in ZooKeeper." - << " Won't do merge with that part and will check it."); + LOG_WARNING(log, "Part " << part->name << " (that was selected for merge)" << " with age " << (time(nullptr) - part->modification_time) << " seconds exists locally but not in ZooKeeper." << " Won't do merge with that part and will check it."); enqueuePartForCheck(part->name); } } @@ -2371,10 +2339,7 @@ bool StorageReplicatedMergeTree::createLogEntryToMutatePart(const IMergeTreeData { if (part.modification_time + MAX_AGE_OF_LOCAL_PART_THAT_WASNT_ADDED_TO_ZOOKEEPER < time(nullptr)) { - LOG_WARNING(log, "Part " << part.name << " (that was selected for mutation)" - << " with age " << (time(nullptr) - part.modification_time) - << " seconds exists locally but not in ZooKeeper." - << " Won't mutate that part and will check it."); + LOG_WARNING(log, "Part " << part.name << " (that was selected for mutation)" << " with age " << (time(nullptr) - part.modification_time) << " seconds exists locally but not in ZooKeeper." << " Won't mutate that part and will check it."); enqueuePartForCheck(part.name); } @@ -3727,8 +3692,7 @@ void StorageReplicatedMergeTree::drop() /// It may left some garbage if replica_path subtree are concurently modified zookeeper->tryRemoveRecursive(replica_path); if (zookeeper->exists(replica_path)) - LOG_ERROR(log, "Replica was not completely removed from ZooKeeper, " - << replica_path << " still exists and may contain some garbage."); + LOG_ERROR(log, "Replica was not completely removed from ZooKeeper, " << replica_path << " still exists and may contain some garbage."); /// Check that `zookeeper_path` exists: it could have been deleted by another replica after execution of previous line. Strings replicas; @@ -3737,8 +3701,7 @@ void StorageReplicatedMergeTree::drop() LOG_INFO_FORMATTED(log, "Removing table {} (this might take several minutes)", zookeeper_path); zookeeper->tryRemoveRecursive(zookeeper_path); if (zookeeper->exists(zookeeper_path)) - LOG_ERROR(log, "Table was not completely removed from ZooKeeper, " - << zookeeper_path << " still exists and may contain some garbage."); + LOG_ERROR(log, "Table was not completely removed from ZooKeeper, " << zookeeper_path << " still exists and may contain some garbage."); } } @@ -4342,8 +4305,7 @@ void StorageReplicatedMergeTree::fetchPartition(const ASTPtr & partition, const if (best_replica.empty()) throw Exception("Logical error: cannot choose best replica.", ErrorCodes::LOGICAL_ERROR); - LOG_INFO(log, "Found " << replicas.size() << " replicas, " << active_replicas.size() << " of them are active." - << " Selected " << best_replica << " to fetch from."); + LOG_INFO(log, "Found " << replicas.size() << " replicas, " << active_replicas.size() << " of them are active." << " Selected " << best_replica << " to fetch from."); String best_replica_path = from + "/replicas/" + best_replica; @@ -4778,8 +4740,7 @@ void StorageReplicatedMergeTree::removePartsFromZooKeeper( } else { - LOG_DEBUG(log, - "There is no part " << part_names[i] << " in ZooKeeper, it was only in filesystem"); + LOG_DEBUG(log, "There is no part " << part_names[i] << " in ZooKeeper, it was only in filesystem"); // emplace invalid future so that the total number of futures is the same as part_names.size(); remove_futures.emplace_back(); } @@ -4804,8 +4765,7 @@ void StorageReplicatedMergeTree::removePartsFromZooKeeper( continue; else if (response.error == Coordination::ZNONODE) { - LOG_DEBUG(log, - "There is no part " << part_names[i] << " in ZooKeeper, it was only in filesystem"); + LOG_DEBUG(log, "There is no part " << part_names[i] << " in ZooKeeper, it was only in filesystem"); continue; } else if (Coordination::isHardwareError(response.error)) @@ -4815,8 +4775,7 @@ void StorageReplicatedMergeTree::removePartsFromZooKeeper( continue; } else - LOG_WARNING(log, "Cannot remove part " << part_names[i] << " from ZooKeeper: " - << zkutil::ZooKeeper::error2string(response.error)); + LOG_WARNING(log, "Cannot remove part " << part_names[i] << " from ZooKeeper: " << zkutil::ZooKeeper::error2string(response.error)); } } @@ -4865,8 +4824,7 @@ void StorageReplicatedMergeTree::clearBlocksInPartition( zookeeper.removeRecursive(path); } else if (rc) - LOG_WARNING(log, - "Error while deleting ZooKeeper path `" << path << "`: " + zkutil::ZooKeeper::error2string(rc) << ", ignoring."); + LOG_WARNING(log, "Error while deleting ZooKeeper path `" << path << "`: " + zkutil::ZooKeeper::error2string(rc) << ", ignoring."); } LOG_TRACE_FORMATTED(log, "Deleted {} deduplication block IDs in partition ID {}", to_delete_futures.size(), partition_id); From a2ad11897fdaf8f50529f1c8883e0b9e475f1f83 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 21:53:58 +0300 Subject: [PATCH 640/738] Remove duplicate whitespaces (preparation) --- programs/copier/ClusterCopier.cpp | 82 +++++++++---------- programs/copier/ClusterCopierApp.cpp | 2 +- programs/odbc-bridge/ODBCBridge.cpp | 2 +- programs/server/HTTPHandlerFactory.cpp | 2 +- programs/server/MySQLHandler.cpp | 2 +- programs/server/Server.cpp | 26 +++--- programs/server/TCPHandler.cpp | 8 +- src/Common/MemoryTracker.cpp | 4 +- .../AggregatingBlockInputStream.cpp | 2 +- .../CreatingSetsBlockInputStream.cpp | 2 +- .../MergeSortingBlockInputStream.cpp | 2 +- .../ParallelAggregatingBlockInputStream.cpp | 6 +- src/Interpreters/Aggregator.cpp | 12 +-- src/Interpreters/DDLWorker.cpp | 6 +- src/Interpreters/DNSCacheUpdater.cpp | 2 +- src/Interpreters/ExternalLoader.cpp | 28 +++---- src/Interpreters/SystemLog.cpp | 2 +- src/Interpreters/SystemLog.h | 6 +- src/Interpreters/executeQuery.cpp | 6 +- src/Processors/Executors/PipelineExecutor.cpp | 2 +- .../Formats/Impl/AvroRowInputFormat.cpp | 2 +- src/Processors/Merges/Algorithms/RowRef.h | 4 +- .../Transforms/AggregatingTransform.cpp | 4 +- .../Transforms/CreatingSetsTransform.cpp | 2 +- .../Transforms/MergeSortingTransform.cpp | 4 +- .../Transforms/MergingAggregatedTransform.cpp | 2 +- .../DistributedBlockOutputStream.cpp | 4 +- .../Kafka/ReadBufferFromKafkaConsumer.cpp | 6 +- .../MergeTree/EphemeralLockInZooKeeper.cpp | 2 +- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 6 +- src/Storages/MergeTree/MergeTreeData.cpp | 10 +-- .../MergeTree/MergeTreeDataMergerMutator.cpp | 2 +- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 2 +- .../MergeTree/MergeTreePartsMover.cpp | 4 +- .../MergeTreeReverseSelectProcessor.cpp | 2 +- .../MergeTree/MergeTreeSelectProcessor.cpp | 2 +- .../ReplicatedMergeTreeCleanupThread.cpp | 2 +- .../ReplicatedMergeTreePartCheckThread.cpp | 4 +- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 18 ++-- .../ReplicatedMergeTreeRestartingThread.cpp | 2 +- src/Storages/StorageBuffer.cpp | 2 +- src/Storages/StorageDistributed.cpp | 4 +- src/Storages/StorageReplicatedMergeTree.cpp | 26 +++--- 43 files changed, 160 insertions(+), 160 deletions(-) diff --git a/programs/copier/ClusterCopier.cpp b/programs/copier/ClusterCopier.cpp index 5ad2f6497e6..3c6afab175c 100644 --- a/programs/copier/ClusterCopier.cpp +++ b/programs/copier/ClusterCopier.cpp @@ -132,7 +132,7 @@ void ClusterCopier::discoverShardPartitions(const ConnectionTimeouts & timeouts, { if (!task_table.enabled_partitions_set.count(partition_name)) { - LOG_DEBUG(log, "Partition " << partition_name << " will not be processed, since it is not in " << "enabled_partitions of " << task_table.table_id); + LOG_DEBUG(log, "Partition " << partition_name << " will not be processed, since it is not in " << "enabled_partitions of " << task_table.table_id); } } } @@ -249,7 +249,7 @@ void ClusterCopier::process(const ConnectionTimeouts & timeouts) { for (TaskTable & task_table : task_cluster->table_tasks) { - LOG_INFO(log, "Process table task " << task_table.table_id << " with " << task_table.all_shards.size() << " shards, " << task_table.local_shards.size() << " of them are local ones"); + LOG_INFO(log, "Process table task " << task_table.table_id << " with " << task_table.all_shards.size() << " shards, " << task_table.local_shards.size() << " of them are local ones"); if (task_table.all_shards.empty()) continue; @@ -353,7 +353,7 @@ zkutil::EphemeralNodeHolder::Ptr ClusterCopier::createTaskWorkerNodeAndWaitIfNee if (static_cast(stat.numChildren) >= task_cluster->max_workers) { - LOG_DEBUG(log, "Too many workers (" << stat.numChildren << ", maximum " << task_cluster->max_workers << ")" << ". Postpone processing " << description); + LOG_DEBUG(log, "Too many workers (" << stat.numChildren << ", maximum " << task_cluster->max_workers << ")" << ". Postpone processing " << description); if (unprioritized) current_sleep_time = std::min(max_sleep_time, current_sleep_time + default_sleep_time); @@ -430,7 +430,7 @@ bool ClusterCopier::checkAllPiecesInPartitionAreDone(const TaskTable & task_tabl bool ClusterCopier::checkPartitionPieceIsDone(const TaskTable & task_table, const String & partition_name, size_t piece_number, const TasksShard & shards_with_partition) { - LOG_DEBUG(log, "Check that all shards processed partition " << partition_name << " piece " + toString(piece_number) + " successfully"); + LOG_DEBUG(log, "Check that all shards processed partition " << partition_name << " piece " + toString(piece_number) + " successfully"); auto zookeeper = context.getZooKeeper(); @@ -459,7 +459,7 @@ bool ClusterCopier::checkPartitionPieceIsDone(const TaskTable & task_table, cons TaskStateWithOwner status = TaskStateWithOwner::fromString(res.data); if (status.state != TaskState::Finished) { - LOG_INFO(log, "The task " << res.data << " is being rewritten by " << status.owner << ". Partition piece will be rechecked"); + LOG_INFO(log, "The task " << res.data << " is being rewritten by " << status.owner << ". Partition piece will be rechecked"); return false; } @@ -494,7 +494,7 @@ bool ClusterCopier::checkPartitionPieceIsDone(const TaskTable & task_table, cons } catch (const Coordination::Exception & e) { - LOG_INFO(log, "A ZooKeeper error occurred while checking partition " << partition_name << " piece number " << toString(piece_number) << ". Will recheck the partition. Error: " << e.displayText()); + LOG_INFO(log, "A ZooKeeper error occurred while checking partition " << partition_name << " piece number " << toString(piece_number) << ". Will recheck the partition. Error: " << e.displayText()); return false; } @@ -557,13 +557,13 @@ TaskStatus ClusterCopier::tryMoveAllPiecesToDestinationTable(const TaskTable & t TaskStateWithOwner status = TaskStateWithOwner::fromString(status_data); if (status.state == TaskState::Finished) { - LOG_DEBUG(log, "All pieces for partition from this task " << current_partition_attach_is_active << " has been successfully moved to destination table by " << status.owner); + LOG_DEBUG(log, "All pieces for partition from this task " << current_partition_attach_is_active << " has been successfully moved to destination table by " << status.owner); return TaskStatus::Finished; } /// Task is abandoned, because previously we created ephemeral node, possibly in other copier's process. /// Initialize DROP PARTITION - LOG_DEBUG(log, "Moving piece for partition " << current_partition_attach_is_active << " has not been successfully finished by " << status.owner << ". Will try to move by myself."); + LOG_DEBUG(log, "Moving piece for partition " << current_partition_attach_is_active << " has not been successfully finished by " << status.owner << ". Will try to move by myself."); /// Remove is_done marker. zookeeper->remove(current_partition_attach_is_done); @@ -580,7 +580,7 @@ TaskStatus ClusterCopier::tryMoveAllPiecesToDestinationTable(const TaskTable & t /// Move partition to original destination table. for (size_t current_piece_number = 0; current_piece_number < task_table.number_of_splits; ++current_piece_number) { - LOG_DEBUG(log, "Trying to move partition " << partition_name << " piece " << toString(current_piece_number) << " to original table"); + LOG_DEBUG(log, "Trying to move partition " << partition_name << " piece " << toString(current_piece_number) << " to original table"); ASTPtr query_alter_ast; String query_alter_ast_string; @@ -617,7 +617,7 @@ TaskStatus ClusterCopier::tryMoveAllPiecesToDestinationTable(const TaskTable & t } catch (...) { - LOG_DEBUG(log, "Error while moving partition " << partition_name << " piece " << toString(current_piece_number) << "to original table"); + LOG_DEBUG(log, "Error while moving partition " << partition_name << " piece " << toString(current_piece_number) << "to original table"); throw; } @@ -641,12 +641,12 @@ TaskStatus ClusterCopier::tryMoveAllPiecesToDestinationTable(const TaskTable & t &task_cluster->settings_push, PoolMode::GET_MANY); - LOG_INFO(log, "Number of shard that executed OPTIMIZE DEDUPLICATE query successfully : " << toString(num_nodes)); + LOG_INFO(log, "Number of shard that executed OPTIMIZE DEDUPLICATE query successfully : " << toString(num_nodes)); } } catch (...) { - LOG_DEBUG(log, "Error while executing OPTIMIZE DEDUPLICATE partition " << partition_name << "in the original table"); + LOG_DEBUG(log, "Error while executing OPTIMIZE DEDUPLICATE partition " << partition_name << "in the original table"); throw; } } @@ -742,7 +742,7 @@ bool ClusterCopier::tryDropPartitionPiece( { if (e.code == Coordination::ZNODEEXISTS) { - LOG_DEBUG(log, "Partition " << task_partition.name << " piece " << toString(current_piece_number) << " is cleaning now by somebody, sleep"); + LOG_DEBUG(log, "Partition " << task_partition.name << " piece " << toString(current_piece_number) << " is cleaning now by somebody, sleep"); std::this_thread::sleep_for(default_sleep_time); return false; } @@ -755,7 +755,7 @@ bool ClusterCopier::tryDropPartitionPiece( { if (stat.numChildren != 0) { - LOG_DEBUG(log, "Partition " << task_partition.name << " contains " << stat.numChildren << " active workers while trying to drop it. Going to sleep."); + LOG_DEBUG(log, "Partition " << task_partition.name << " contains " << stat.numChildren << " active workers while trying to drop it. Going to sleep."); std::this_thread::sleep_for(default_sleep_time); return false; } @@ -840,12 +840,12 @@ bool ClusterCopier::tryDropPartitionPiece( return false; } - LOG_INFO(log, "Partition " << task_partition.name << " piece " << toString(current_piece_number) << " was dropped on cluster " << task_table.cluster_push_name); + LOG_INFO(log, "Partition " << task_partition.name << " piece " << toString(current_piece_number) << " was dropped on cluster " << task_table.cluster_push_name); if (zookeeper->tryCreate(current_shards_path, host_id, zkutil::CreateMode::Persistent) == Coordination::ZNODEEXISTS) zookeeper->set(current_shards_path, host_id); } - LOG_INFO(log, "Partition " << task_partition.name << " piece " << toString(current_piece_number) << " is safe for work now."); + LOG_INFO(log, "Partition " << task_partition.name << " piece " << toString(current_piece_number) << " is safe for work now."); return true; } @@ -1010,16 +1010,16 @@ bool ClusterCopier::tryProcessTable(const ConnectionTimeouts & timeouts, TaskTab task_table.rows_copied += cluster_partition.rows_copied; double elapsed = cluster_partition.elapsed_time_seconds; - LOG_INFO(log, "It took " << std::fixed << std::setprecision(2) << elapsed << " seconds to copy partition " << partition_name << ": " << formatReadableSizeWithDecimalSuffix(cluster_partition.bytes_copied) << " uncompressed bytes" << ", " << formatReadableQuantity(cluster_partition.rows_copied) << " rows" << " and " << cluster_partition.blocks_copied << " source blocks are copied"); + LOG_INFO(log, "It took " << std::fixed << std::setprecision(2) << elapsed << " seconds to copy partition " << partition_name << ": " << formatReadableSizeWithDecimalSuffix(cluster_partition.bytes_copied) << " uncompressed bytes" << ", " << formatReadableQuantity(cluster_partition.rows_copied) << " rows" << " and " << cluster_partition.blocks_copied << " source blocks are copied"); if (cluster_partition.rows_copied) { - LOG_INFO(log, "Average partition speed: " << formatReadableSizeWithDecimalSuffix(cluster_partition.bytes_copied / elapsed) << " per second."); + LOG_INFO(log, "Average partition speed: " << formatReadableSizeWithDecimalSuffix(cluster_partition.bytes_copied / elapsed) << " per second."); } if (task_table.rows_copied) { - LOG_INFO(log, "Average table " << task_table.table_id << " speed: " << formatReadableSizeWithDecimalSuffix(task_table.bytes_copied / elapsed) << " per second."); + LOG_INFO(log, "Average table " << task_table.table_id << " speed: " << formatReadableSizeWithDecimalSuffix(task_table.bytes_copied / elapsed) << " per second."); } } } @@ -1030,7 +1030,7 @@ bool ClusterCopier::tryProcessTable(const ConnectionTimeouts & timeouts, TaskTab if (!table_is_done) { - LOG_INFO(log, "Table " + task_table.table_id + " is not processed yet." << "Copied " << finished_partitions << " of " << required_partitions << ", will retry"); + LOG_INFO(log, "Table " + task_table.table_id + " is not processed yet." << "Copied " << finished_partitions << " of " << required_partitions << ", will retry"); } return table_is_done; @@ -1078,7 +1078,7 @@ TaskStatus ClusterCopier::iterateThroughAllPiecesInPartition(const ConnectionTim { for (UInt64 try_num = 0; try_num < max_shard_partition_tries; ++try_num) { - LOG_INFO(log, "Attempt number " << try_num << " to process partition " << task_partition.name << " piece number " << piece_number << " on shard number " << task_partition.task_shard.numberInCluster() << " with index " << task_partition.task_shard.indexInCluster()); + LOG_INFO(log, "Attempt number " << try_num << " to process partition " << task_partition.name << " piece number " << piece_number << " on shard number " << task_partition.task_shard.numberInCluster() << " with index " << task_partition.task_shard.indexInCluster()); res = processPartitionPieceTaskImpl(timeouts, task_partition, piece_number, is_unprioritized_task); /// Exit if success @@ -1193,12 +1193,12 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( /// Do not start if partition piece is dirty, try to clean it if (is_clean) { - LOG_DEBUG(log, "Partition " << task_partition.name << " piece " + toString(current_piece_number) + " appears to be clean"); + LOG_DEBUG(log, "Partition " << task_partition.name << " piece " + toString(current_piece_number) + " appears to be clean"); zookeeper->createAncestors(current_task_piece_status_path); } else { - LOG_DEBUG(log, "Partition " << task_partition.name << " piece " + toString(current_piece_number) + " is dirty, try to drop it"); + LOG_DEBUG(log, "Partition " << task_partition.name << " piece " + toString(current_piece_number) + " is dirty, try to drop it"); try { @@ -1239,13 +1239,13 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( TaskStateWithOwner status = TaskStateWithOwner::fromString(status_data); if (status.state == TaskState::Finished) { - LOG_DEBUG(log, "Task " << current_task_piece_status_path << " has been successfully executed by " << status.owner); + LOG_DEBUG(log, "Task " << current_task_piece_status_path << " has been successfully executed by " << status.owner); return TaskStatus::Finished; } /// Task is abandoned, because previously we created ephemeral node, possibly in other copier's process. /// Initialize DROP PARTITION - LOG_DEBUG(log, "Task " << current_task_piece_status_path << " has not been successfully finished by " << status.owner << ". Partition will be dropped and refilled."); + LOG_DEBUG(log, "Task " << current_task_piece_status_path << " has not been successfully finished by " << status.owner << ". Partition will be dropped and refilled."); create_is_dirty_node(clean_state_clock); return TaskStatus::Error; @@ -1260,9 +1260,9 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( String state_finished = TaskStateWithOwner::getData(TaskState::Finished, host_id); auto res = zookeeper->tryCreate(current_task_piece_status_path, state_finished, zkutil::CreateMode::Persistent); if (res == Coordination::ZNODEEXISTS) - LOG_DEBUG(log, "Partition " << task_partition.name << " piece " + toString(current_piece_number) + " is absent on current replica of a shard. But other replicas have already marked it as done."); + LOG_DEBUG(log, "Partition " << task_partition.name << " piece " + toString(current_piece_number) + " is absent on current replica of a shard. But other replicas have already marked it as done."); if (res == Coordination::ZOK) - LOG_DEBUG(log, "Partition " << task_partition.name << " piece " + toString(current_piece_number) + " is absent on current replica of a shard. Will mark it as done. Other replicas will do the same."); + LOG_DEBUG(log, "Partition " << task_partition.name << " piece " + toString(current_piece_number) + " is absent on current replica of a shard. Will mark it as done. Other replicas will do the same."); return TaskStatus::Finished; } @@ -1290,7 +1290,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( if (count != 0) { - LOG_INFO(log, "Partition " << task_partition.name << " piece " << current_piece_number << "is not empty. In contains " << count << " rows."); + LOG_INFO(log, "Partition " << task_partition.name << " piece " << current_piece_number << "is not empty. In contains " << count << " rows."); Coordination::Stat stat_shards{}; zookeeper->get(partition_piece.getPartitionPieceShardsPath(), &stat_shards); @@ -1314,12 +1314,12 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( CleanStateClock new_clean_state_clock (zookeeper, piece_is_dirty_flag_path, piece_is_dirty_cleaned_path); if (clean_state_clock != new_clean_state_clock) { - LOG_INFO(log, "Partition " << task_partition.name << " piece " << toString(current_piece_number) << " clean state changed, cowardly bailing"); + LOG_INFO(log, "Partition " << task_partition.name << " piece " << toString(current_piece_number) << " clean state changed, cowardly bailing"); return TaskStatus::Error; } else if (!new_clean_state_clock.is_clean()) { - LOG_INFO(log, "Partition " << task_partition.name << " piece " << toString(current_piece_number) << " is dirty and will be dropped and refilled"); + LOG_INFO(log, "Partition " << task_partition.name << " piece " << toString(current_piece_number) << " is dirty and will be dropped and refilled"); create_is_dirty_node(new_clean_state_clock); return TaskStatus::Error; } @@ -1350,7 +1350,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( UInt64 shards = executeQueryOnCluster(task_table.cluster_push, query, create_query_push_ast, &task_cluster->settings_push, PoolMode::GET_MANY); - LOG_DEBUG(log, "Destination tables " << getQuotedTable(task_table.table_push) << " have been created on " << shards << " shards of " << task_table.cluster_push->getShardCount()); + LOG_DEBUG(log, "Destination tables " << getQuotedTable(task_table.table_push) << " have been created on " << shards << " shards of " << task_table.cluster_push->getShardCount()); } /// Do the copying @@ -1365,7 +1365,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( // Select all fields ASTPtr query_select_ast = get_select_query(task_shard.table_read_shard, "*", /*enable_splitting*/ true, inject_fault ? "1" : ""); - LOG_DEBUG(log, "Executing SELECT query and pull from " << task_shard.getDescription() << " : " << queryToString(query_select_ast)); + LOG_DEBUG(log, "Executing SELECT query and pull from " << task_shard.getDescription() << " : " << queryToString(query_select_ast)); ASTPtr query_insert_ast; { @@ -1458,7 +1458,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( } } - LOG_INFO(log, "Partition " << task_partition.name << " piece " << toString(current_piece_number) << " copied. But not moved to original destination table."); + LOG_INFO(log, "Partition " << task_partition.name << " piece " << toString(current_piece_number) << " copied. But not moved to original destination table."); /// Try create original table (if not exists) on each shard @@ -1473,7 +1473,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( UInt64 shards = executeQueryOnCluster(task_table.cluster_push, query, create_query_push_ast, &task_cluster->settings_push, PoolMode::GET_MANY); - LOG_DEBUG(log, "Destination tables " << getQuotedTable(task_table.table_push) << " have been created on " << shards << " shards of " << task_table.cluster_push->getShardCount()); + LOG_DEBUG(log, "Destination tables " << getQuotedTable(task_table.table_push) << " have been created on " << shards << " shards of " << task_table.cluster_push->getShardCount()); } catch (...) { @@ -1486,12 +1486,12 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( CleanStateClock new_clean_state_clock (zookeeper, piece_is_dirty_flag_path, piece_is_dirty_cleaned_path); if (clean_state_clock != new_clean_state_clock) { - LOG_INFO(log, "Partition " << task_partition.name << " piece " << toString(current_piece_number) << " clean state changed, cowardly bailing"); + LOG_INFO(log, "Partition " << task_partition.name << " piece " << toString(current_piece_number) << " clean state changed, cowardly bailing"); return TaskStatus::Error; } else if (!new_clean_state_clock.is_clean()) { - LOG_INFO(log, "Partition " << task_partition.name << " piece " << toString(current_piece_number) << " became dirty and will be dropped and refilled"); + LOG_INFO(log, "Partition " << task_partition.name << " piece " << toString(current_piece_number) << " became dirty and will be dropped and refilled"); create_is_dirty_node(new_clean_state_clock); return TaskStatus::Error; } @@ -1718,7 +1718,7 @@ bool ClusterCopier::checkShardHasPartition(const ConnectionTimeouts & timeouts, query += " LIMIT 1"; - LOG_DEBUG(log, "Checking shard " << task_shard.getDescription() << " for partition " << partition_quoted_name << " existence, executing query: " << query); + LOG_DEBUG(log, "Checking shard " << task_shard.getDescription() << " for partition " << partition_quoted_name << " existence, executing query: " << query); ParserQuery parser_query(query.data() + query.size()); const auto & settings = context.getSettingsRef(); @@ -1757,7 +1757,7 @@ bool ClusterCopier::checkPresentPartitionPiecesOnCurrentShard(const ConnectionTi query += " LIMIT 1"; - LOG_DEBUG(log, "Checking shard " << task_shard.getDescription() << " for partition " << partition_quoted_name << " piece " << std::to_string(current_piece_number) << "existence, executing query: " << query); + LOG_DEBUG(log, "Checking shard " << task_shard.getDescription() << " for partition " << partition_quoted_name << " piece " << std::to_string(current_piece_number) << "existence, executing query: " << query); ParserQuery parser_query(query.data() + query.size()); const auto & settings = context.getSettingsRef(); @@ -1767,9 +1767,9 @@ bool ClusterCopier::checkPresentPartitionPiecesOnCurrentShard(const ConnectionTi local_context.setSettings(task_cluster->settings_pull); auto result = InterpreterFactory::get(query_ast, local_context)->execute().in->read().rows(); if (result != 0) - LOG_DEBUG(log, "Partition " << partition_quoted_name << " piece number " << std::to_string(current_piece_number) << " is PRESENT on shard " << task_shard.getDescription()); + LOG_DEBUG(log, "Partition " << partition_quoted_name << " piece number " << std::to_string(current_piece_number) << " is PRESENT on shard " << task_shard.getDescription()); else - LOG_DEBUG(log, "Partition " << partition_quoted_name << " piece number " << std::to_string(current_piece_number) << " is ABSENT on shard " << task_shard.getDescription()); + LOG_DEBUG(log, "Partition " << partition_quoted_name << " piece number " << std::to_string(current_piece_number) << " is ABSENT on shard " << task_shard.getDescription()); return result != 0; } @@ -1886,7 +1886,7 @@ UInt64 ClusterCopier::executeQueryOnCluster( if (execution_mode == ClusterExecutionMode::ON_EACH_NODE && successful_nodes != origin_replicas_number) { - LOG_INFO(log, "There was an error while executing ALTER on each node. Query was executed on " << toString(successful_nodes) << " nodes. But had to be executed on " << toString(origin_replicas_number.load())); + LOG_INFO(log, "There was an error while executing ALTER on each node. Query was executed on " << toString(successful_nodes) << " nodes. But had to be executed on " << toString(origin_replicas_number.load())); } diff --git a/programs/copier/ClusterCopierApp.cpp b/programs/copier/ClusterCopierApp.cpp index 1afbdba2c4f..f34cc9d48b6 100644 --- a/programs/copier/ClusterCopierApp.cpp +++ b/programs/copier/ClusterCopierApp.cpp @@ -95,7 +95,7 @@ void ClusterCopierApp::mainImpl() ThreadStatus thread_status; auto * log = &logger(); - LOG_INFO(log, "Starting clickhouse-copier (" << "id " << process_id << ", " << "host_id " << host_id << ", " << "path " << process_path << ", " << "revision " << ClickHouseRevision::get() << ")"); + LOG_INFO(log, "Starting clickhouse-copier (" << "id " << process_id << ", " << "host_id " << host_id << ", " << "path " << process_path << ", " << "revision " << ClickHouseRevision::get() << ")"); SharedContextHolder shared_context = Context::createShared(); auto context = std::make_unique(Context::createGlobal(shared_context.get())); diff --git a/programs/odbc-bridge/ODBCBridge.cpp b/programs/odbc-bridge/ODBCBridge.cpp index fefe927142a..dec071dc1cb 100644 --- a/programs/odbc-bridge/ODBCBridge.cpp +++ b/programs/odbc-bridge/ODBCBridge.cpp @@ -48,7 +48,7 @@ namespace #endif ) { - LOG_ERROR(log, "Cannot resolve listen_host (" << host << "), error " << e.code() << ": " << e.message() << ". " "If it is an IPv6 address and your host has disabled IPv6, then consider to " "specify IPv4 address to listen in element of configuration " "file. Example: 0.0.0.0"); + LOG_ERROR(log, "Cannot resolve listen_host (" << host << "), error " << e.code() << ": " << e.message() << ". " "If it is an IPv6 address and your host has disabled IPv6, then consider to " "specify IPv4 address to listen in element of configuration " "file. Example: 0.0.0.0"); } throw; diff --git a/programs/server/HTTPHandlerFactory.cpp b/programs/server/HTTPHandlerFactory.cpp index 9074df20b15..af5900ee18e 100644 --- a/programs/server/HTTPHandlerFactory.cpp +++ b/programs/server/HTTPHandlerFactory.cpp @@ -30,7 +30,7 @@ HTTPRequestHandlerFactoryMain::HTTPRequestHandlerFactoryMain(const std::string & Poco::Net::HTTPRequestHandler * HTTPRequestHandlerFactoryMain::createRequestHandler(const Poco::Net::HTTPServerRequest & request) { - LOG_TRACE(log, "HTTP Request for " << name << ". " << "Method: " << request.getMethod() << ", Address: " << request.clientAddress().toString() << ", User-Agent: " << (request.has("User-Agent") ? request.get("User-Agent") : "none") << (request.hasContentLength() ? (", Length: " + std::to_string(request.getContentLength())) : ("")) << ", Content Type: " << request.getContentType() << ", Transfer Encoding: " << request.getTransferEncoding()); + LOG_TRACE(log, "HTTP Request for " << name << ". " << "Method: " << request.getMethod() << ", Address: " << request.clientAddress().toString() << ", User-Agent: " << (request.has("User-Agent") ? request.get("User-Agent") : "none") << (request.hasContentLength() ? (", Length: " + std::to_string(request.getContentLength())) : ("")) << ", Content Type: " << request.getContentType() << ", Transfer Encoding: " << request.getTransferEncoding()); for (auto & handler_factory : child_factories) { diff --git a/programs/server/MySQLHandler.cpp b/programs/server/MySQLHandler.cpp index cc37b8a73b0..f05abe085cc 100644 --- a/programs/server/MySQLHandler.cpp +++ b/programs/server/MySQLHandler.cpp @@ -83,7 +83,7 @@ void MySQLHandler::run() if (!connection_context.mysql.max_packet_size) connection_context.mysql.max_packet_size = MAX_PACKET_LENGTH; - LOG_TRACE(log, "Capabilities: " << handshake_response.capability_flags << ", max_packet_size: " << handshake_response.max_packet_size << ", character_set: " << static_cast(handshake_response.character_set) << ", user: " << handshake_response.username << ", auth_response length: " << handshake_response.auth_response.length() << ", database: " << handshake_response.database << ", auth_plugin_name: " << handshake_response.auth_plugin_name); + LOG_TRACE(log, "Capabilities: " << handshake_response.capability_flags << ", max_packet_size: " << handshake_response.max_packet_size << ", character_set: " << static_cast(handshake_response.character_set) << ", user: " << handshake_response.username << ", auth_response length: " << handshake_response.auth_response.length() << ", database: " << handshake_response.database << ", auth_plugin_name: " << handshake_response.auth_plugin_name); client_capability_flags = handshake_response.capability_flags; if (!(client_capability_flags & CLIENT_PROTOCOL_41)) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 5ee2b3bd631..5e01fbdac4d 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -282,7 +282,7 @@ int Server::main(const std::vector & /*args*/) } else { - LOG_INFO(log, "It looks like the process has no CAP_IPC_LOCK capability, binary mlock will be disabled." " It could happen due to incorrect ClickHouse package installation." " You could resolve the problem manually with 'sudo setcap cap_ipc_lock=+ep " << executable_path << "'." " Note that it will not work on 'nosuid' mounted filesystems."); + LOG_INFO(log, "It looks like the process has no CAP_IPC_LOCK capability, binary mlock will be disabled." " It could happen due to incorrect ClickHouse package installation." " You could resolve the problem manually with 'sudo setcap cap_ipc_lock=+ep " << executable_path << "'." " Note that it will not work on 'nosuid' mounted filesystems."); } } } @@ -425,7 +425,7 @@ int Server::main(const std::vector & /*args*/) if (this_host.empty()) { this_host = getFQDNOrHostName(); - LOG_DEBUG(log, "Configuration parameter '" + String(host_tag) + "' doesn't exist or exists and empty. Will use '" + this_host + "' as replica host."); + LOG_DEBUG(log, "Configuration parameter '" + String(host_tag) + "' doesn't exist or exists and empty. Will use '" + this_host + "' as replica host."); } String port_str = config().getString(port_tag); @@ -530,7 +530,7 @@ int Server::main(const std::vector & /*args*/) if (uncompressed_cache_size > max_cache_size) { uncompressed_cache_size = max_cache_size; - LOG_INFO(log, "Uncompressed cache size was lowered to " << formatReadableSizeWithBinarySuffix(uncompressed_cache_size) << " because the system has low amount of memory"); + LOG_INFO(log, "Uncompressed cache size was lowered to " << formatReadableSizeWithBinarySuffix(uncompressed_cache_size) << " because the system has low amount of memory"); } global_context->setUncompressedCache(uncompressed_cache_size); @@ -545,7 +545,7 @@ int Server::main(const std::vector & /*args*/) if (mark_cache_size > max_cache_size) { mark_cache_size = max_cache_size; - LOG_INFO(log, "Mark cache size was lowered to " << formatReadableSizeWithBinarySuffix(uncompressed_cache_size) << " because the system has low amount of memory"); + LOG_INFO(log, "Mark cache size was lowered to " << formatReadableSizeWithBinarySuffix(uncompressed_cache_size) << " because the system has low amount of memory"); } global_context->setMarkCache(mark_cache_size); @@ -574,7 +574,7 @@ int Server::main(const std::vector & /*args*/) else if (max_server_memory_usage > default_max_server_memory_usage) { max_server_memory_usage = default_max_server_memory_usage; - LOG_INFO(log, "Setting max_server_memory_usage was lowered to " << formatReadableSizeWithBinarySuffix(max_server_memory_usage) << " because the system has low amount of memory"); + LOG_INFO(log, "Setting max_server_memory_usage was lowered to " << formatReadableSizeWithBinarySuffix(max_server_memory_usage) << " because the system has low amount of memory"); } total_memory_tracker.setOrRaiseHardLimit(max_server_memory_usage); @@ -647,11 +647,11 @@ int Server::main(const std::vector & /*args*/) #endif #if defined(SANITIZER) - LOG_INFO(log, "Query Profiler and TraceCollector are disabled because they cannot work under sanitizers" " when two different stack unwinding methods will interfere with each other."); + LOG_INFO(log, "Query Profiler and TraceCollector are disabled because they cannot work under sanitizers" " when two different stack unwinding methods will interfere with each other."); #endif if (!hasPHDRCache()) - LOG_INFO(log, "Query Profiler and TraceCollector are disabled because they require PHDR cache to be created" " (otherwise the function 'dl_iterate_phdr' is not lock free and not async-signal safe)."); + LOG_INFO(log, "Query Profiler and TraceCollector are disabled because they require PHDR cache to be created" " (otherwise the function 'dl_iterate_phdr' is not lock free and not async-signal safe)."); global_context->setCurrentDatabase(default_database); @@ -677,12 +677,12 @@ int Server::main(const std::vector & /*args*/) #if defined(OS_LINUX) if (!TasksStatsCounters::checkIfAvailable()) { - LOG_INFO(log, "It looks like this system does not have procfs mounted at /proc location," " neither clickhouse-server process has CAP_NET_ADMIN capability." " 'taskstats' performance statistics will be disabled." " It could happen due to incorrect ClickHouse package installation." " You can try to resolve the problem manually with 'sudo setcap cap_net_admin=+ep " << executable_path << "'." " Note that it will not work on 'nosuid' mounted filesystems." " It also doesn't work if you run clickhouse-server inside network namespace as it happens in some containers."); + LOG_INFO(log, "It looks like this system does not have procfs mounted at /proc location," " neither clickhouse-server process has CAP_NET_ADMIN capability." " 'taskstats' performance statistics will be disabled." " It could happen due to incorrect ClickHouse package installation." " You can try to resolve the problem manually with 'sudo setcap cap_net_admin=+ep " << executable_path << "'." " Note that it will not work on 'nosuid' mounted filesystems." " It also doesn't work if you run clickhouse-server inside network namespace as it happens in some containers."); } if (!hasLinuxCapability(CAP_SYS_NICE)) { - LOG_INFO(log, "It looks like the process has no CAP_SYS_NICE capability, the setting 'os_thread_nice' will have no effect." " It could happen due to incorrect ClickHouse package installation." " You could resolve the problem manually with 'sudo setcap cap_sys_nice=+ep " << executable_path << "'." " Note that it will not work on 'nosuid' mounted filesystems."); + LOG_INFO(log, "It looks like the process has no CAP_SYS_NICE capability, the setting 'os_thread_nice' will have no effect." " It could happen due to incorrect ClickHouse package installation." " You could resolve the problem manually with 'sudo setcap cap_sys_nice=+ep " << executable_path << "'." " Note that it will not work on 'nosuid' mounted filesystems."); } #else LOG_INFO_FORMATTED(log, "TaskStats is not implemented for this OS. IO accounting will be disabled."); @@ -724,7 +724,7 @@ int Server::main(const std::vector & /*args*/) #endif ) { - LOG_ERROR(log, "Cannot resolve listen_host (" << host << "), error " << e.code() << ": " << e.message() << ". " "If it is an IPv6 address and your host has disabled IPv6, then consider to " "specify IPv4 address to listen in element of configuration " "file. Example: 0.0.0.0"); + LOG_ERROR(log, "Cannot resolve listen_host (" << host << "), error " << e.code() << ": " << e.message() << ". " "If it is an IPv6 address and your host has disabled IPv6, then consider to " "specify IPv4 address to listen in element of configuration " "file. Example: 0.0.0.0"); } throw; @@ -776,7 +776,7 @@ int Server::main(const std::vector & /*args*/) if (listen_try) { - LOG_ERROR(log, message << ". If it is an IPv6 or IPv4 address and your host has disabled IPv6 or IPv4, then consider to " "specify not disabled IPv4 or IPv6 address to listen in element of configuration " "file. Example for disabled IPv6: 0.0.0.0 ." " Example for disabled IPv4: ::"); + LOG_ERROR(log, message << ". If it is an IPv6 or IPv4 address and your host has disabled IPv6 or IPv4, then consider to " "specify not disabled IPv4 or IPv6 address to listen in element of configuration " "file. Example for disabled IPv6: 0.0.0.0 ." " Example for disabled IPv4: ::"); } else { @@ -959,7 +959,7 @@ int Server::main(const std::vector & /*args*/) current_connections += server->currentConnections(); } - LOG_INFO(log, "Closed all listening sockets." << (current_connections ? " Waiting for " + toString(current_connections) + " outstanding connections." : "")); + LOG_INFO(log, "Closed all listening sockets." << (current_connections ? " Waiting for " + toString(current_connections) + " outstanding connections." : "")); /// Killing remaining queries. global_context->getProcessList().killAllQueries(); @@ -981,7 +981,7 @@ int Server::main(const std::vector & /*args*/) } } - LOG_INFO( log, "Closed connections." << (current_connections ? " But " + toString(current_connections) + " remains." " Tip: To increase wait time add to config: 60" : "")); + LOG_INFO( log, "Closed connections." << (current_connections ? " But " + toString(current_connections) + " remains." " Tip: To increase wait time add to config: 60" : "")); dns_cache_updater.reset(); main_config_reloader.reset(); diff --git a/programs/server/TCPHandler.cpp b/programs/server/TCPHandler.cpp index fec67686195..612fc678505 100644 --- a/programs/server/TCPHandler.cpp +++ b/programs/server/TCPHandler.cpp @@ -115,7 +115,7 @@ void TCPHandler::runImpl() if (!DatabaseCatalog::instance().isDatabaseExist(default_database)) { Exception e("Database " + backQuote(default_database) + " doesn't exist", ErrorCodes::UNKNOWN_DATABASE); - LOG_ERROR(log, "Code: " << e.code() << ", e.displayText() = " << e.displayText() << ", Stack trace:\n\n" << e.getStackTraceString()); + LOG_ERROR(log, "Code: " << e.code() << ", e.displayText() = " << e.displayText() << ", Stack trace:\n\n" << e.getStackTraceString()); sendException(e, connection_context.getSettingsRef().calculate_text_stack_trace); return; } @@ -378,7 +378,7 @@ void TCPHandler::runImpl() watch.stop(); - LOG_INFO(log, std::fixed << std::setprecision(3) << "Processed in " << watch.elapsedSeconds() << " sec."); + LOG_INFO(log, std::fixed << std::setprecision(3) << "Processed in " << watch.elapsedSeconds() << " sec."); /// It is important to destroy query context here. We do not want it to live arbitrarily longer than the query. query_context.reset(); @@ -730,7 +730,7 @@ void TCPHandler::receiveHello() readStringBinary(user, *in); readStringBinary(password, *in); - LOG_DEBUG(log, "Connected " << client_name << " version " << client_version_major << "." << client_version_minor << "." << client_version_patch << ", revision: " << client_revision << (!default_database.empty() ? ", database: " + default_database : "") << (!user.empty() ? ", user: " + user : "") << "."); + LOG_DEBUG(log, "Connected " << client_name << " version " << client_version_major << "." << client_version_minor << "." << client_version_patch << ", revision: " << client_revision << (!default_database.empty() ? ", database: " + default_database : "") << (!user.empty() ? ", user: " + user : "") << "."); connection_context.setUser(user, password, socket().peerAddress()); } @@ -1196,7 +1196,7 @@ void TCPHandler::run() /// Timeout - not an error. if (!strcmp(e.what(), "Timeout")) { - LOG_DEBUG(log, "Poco::Exception. Code: " << ErrorCodes::POCO_EXCEPTION << ", e.code() = " << e.code() << ", e.displayText() = " << e.displayText() << ", e.what() = " << e.what()); + LOG_DEBUG(log, "Poco::Exception. Code: " << ErrorCodes::POCO_EXCEPTION << ", e.code() = " << e.code() << ", e.displayText() = " << e.displayText() << ", e.what() = " << e.what()); } else throw; diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp index 022a4bafc06..4c729a25f4c 100644 --- a/src/Common/MemoryTracker.cpp +++ b/src/Common/MemoryTracker.cpp @@ -49,12 +49,12 @@ MemoryTracker::~MemoryTracker() void MemoryTracker::logPeakMemoryUsage() const { - LOG_DEBUG(&Logger::get("MemoryTracker"), "Peak memory usage" << (description ? " " + std::string(description) : "") << ": " << formatReadableSizeWithBinarySuffix(peak) << "."); + LOG_DEBUG(&Logger::get("MemoryTracker"), "Peak memory usage" << (description ? " " + std::string(description) : "") << ": " << formatReadableSizeWithBinarySuffix(peak) << "."); } void MemoryTracker::logMemoryUsage(Int64 current) const { - LOG_DEBUG(&Logger::get("MemoryTracker"), "Current memory usage" << (description ? " " + std::string(description) : "") << ": " << formatReadableSizeWithBinarySuffix(current) << "."); + LOG_DEBUG(&Logger::get("MemoryTracker"), "Current memory usage" << (description ? " " + std::string(description) : "") << ": " << formatReadableSizeWithBinarySuffix(current) << "."); } diff --git a/src/DataStreams/AggregatingBlockInputStream.cpp b/src/DataStreams/AggregatingBlockInputStream.cpp index 475a5529f9e..83197ef447f 100644 --- a/src/DataStreams/AggregatingBlockInputStream.cpp +++ b/src/DataStreams/AggregatingBlockInputStream.cpp @@ -60,7 +60,7 @@ Block AggregatingBlockInputStream::readImpl() input_streams.emplace_back(temporary_inputs.back()->block_in); } - LOG_TRACE(log, "Will merge " << files.files.size() << " temporary files of size " << (files.sum_size_compressed / 1048576.0) << " MiB compressed, " << (files.sum_size_uncompressed / 1048576.0) << " MiB uncompressed."); + LOG_TRACE(log, "Will merge " << files.files.size() << " temporary files of size " << (files.sum_size_compressed / 1048576.0) << " MiB compressed, " << (files.sum_size_uncompressed / 1048576.0) << " MiB uncompressed."); impl = std::make_unique(input_streams, params, final, 1, 1); } diff --git a/src/DataStreams/CreatingSetsBlockInputStream.cpp b/src/DataStreams/CreatingSetsBlockInputStream.cpp index 9722feec1bd..2a8e6c216a7 100644 --- a/src/DataStreams/CreatingSetsBlockInputStream.cpp +++ b/src/DataStreams/CreatingSetsBlockInputStream.cpp @@ -90,7 +90,7 @@ void CreatingSetsBlockInputStream::createAll() void CreatingSetsBlockInputStream::createOne(SubqueryForSet & subquery) { - LOG_TRACE(log, (subquery.set ? "Creating set. " : "") << (subquery.join ? "Creating join. " : "") << (subquery.table ? "Filling temporary table. " : "")); + LOG_TRACE(log, (subquery.set ? "Creating set. " : "") << (subquery.join ? "Creating join. " : "") << (subquery.table ? "Filling temporary table. " : "")); Stopwatch watch; BlockOutputStreamPtr table_out; diff --git a/src/DataStreams/MergeSortingBlockInputStream.cpp b/src/DataStreams/MergeSortingBlockInputStream.cpp index 08ff290b9ea..20833f717d3 100644 --- a/src/DataStreams/MergeSortingBlockInputStream.cpp +++ b/src/DataStreams/MergeSortingBlockInputStream.cpp @@ -264,7 +264,7 @@ void MergeSortingBlockInputStream::remerge() } merger.readSuffix(); - LOG_DEBUG(log, "Memory usage is lowered from " << formatReadableSizeWithBinarySuffix(sum_bytes_in_blocks) << " to " << formatReadableSizeWithBinarySuffix(new_sum_bytes_in_blocks)); + LOG_DEBUG(log, "Memory usage is lowered from " << formatReadableSizeWithBinarySuffix(sum_bytes_in_blocks) << " to " << formatReadableSizeWithBinarySuffix(new_sum_bytes_in_blocks)); /// If the memory consumption was not lowered enough - we will not perform remerge anymore. 2 is a guess. if (new_sum_bytes_in_blocks * 2 > sum_bytes_in_blocks) diff --git a/src/DataStreams/ParallelAggregatingBlockInputStream.cpp b/src/DataStreams/ParallelAggregatingBlockInputStream.cpp index 03abb13c580..5fd89bbc3dc 100644 --- a/src/DataStreams/ParallelAggregatingBlockInputStream.cpp +++ b/src/DataStreams/ParallelAggregatingBlockInputStream.cpp @@ -82,7 +82,7 @@ Block ParallelAggregatingBlockInputStream::readImpl() input_streams.emplace_back(temporary_inputs.back()->block_in); } - LOG_TRACE(log, "Will merge " << files.files.size() << " temporary files of size " << (files.sum_size_compressed / 1048576.0) << " MiB compressed, " << (files.sum_size_uncompressed / 1048576.0) << " MiB uncompressed."); + LOG_TRACE(log, "Will merge " << files.files.size() << " temporary files of size " << (files.sum_size_compressed / 1048576.0) << " MiB compressed, " << (files.sum_size_uncompressed / 1048576.0) << " MiB uncompressed."); impl = std::make_unique( input_streams, params, final, temporary_data_merge_threads, temporary_data_merge_threads); @@ -177,12 +177,12 @@ void ParallelAggregatingBlockInputStream::execute() for (size_t i = 0; i < max_threads; ++i) { size_t rows = many_data[i]->size(); - LOG_TRACE(log, "Aggregated. " << threads_data[i].src_rows << " to " << rows << " rows" << " (from " << threads_data[i].src_bytes / 1048576.0 << " MiB)" << " in " << elapsed_seconds << " sec." << " (" << threads_data[i].src_rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(threads_data[i].src_bytes / elapsed_seconds) << "/sec.)"); + LOG_TRACE(log, "Aggregated. " << threads_data[i].src_rows << " to " << rows << " rows" << " (from " << threads_data[i].src_bytes / 1048576.0 << " MiB)" << " in " << elapsed_seconds << " sec." << " (" << threads_data[i].src_rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(threads_data[i].src_bytes / elapsed_seconds) << "/sec.)"); total_src_rows += threads_data[i].src_rows; total_src_bytes += threads_data[i].src_bytes; } - LOG_TRACE(log, "Total aggregated. " << total_src_rows << " rows (from " << total_src_bytes / 1048576.0 << " MiB)" << " in " << elapsed_seconds << " sec." << " (" << total_src_rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(total_src_bytes / elapsed_seconds) << "/sec.)"); + LOG_TRACE(log, "Total aggregated. " << total_src_rows << " rows (from " << total_src_bytes / 1048576.0 << " MiB)" << " in " << elapsed_seconds << " sec." << " (" << total_src_rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(total_src_bytes / elapsed_seconds) << "/sec.)"); /// If there was no data, and we aggregate without keys, we must return single row with the result of empty aggregation. /// To do this, we pass a block with zero rows to aggregate. diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index c6df6b01090..fc9f8d52065 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -761,7 +761,7 @@ void Aggregator::writeToTemporaryFile(AggregatedDataVariants & data_variants, co ProfileEvents::increment(ProfileEvents::ExternalAggregationCompressedBytes, compressed_bytes); ProfileEvents::increment(ProfileEvents::ExternalAggregationUncompressedBytes, uncompressed_bytes); - LOG_TRACE(log, "Written part in " << elapsed_seconds << " sec., " << rows << " rows, " << formatReadableSizeWithBinarySuffix(uncompressed_bytes) << " uncompressed, " << formatReadableSizeWithBinarySuffix(compressed_bytes) << " compressed, " << (uncompressed_bytes / rows) << " uncompressed bytes per row, " << (compressed_bytes / rows) << " compressed bytes per row, " << "compression rate: " << (uncompressed_bytes / compressed_bytes) << " (" << (rows / elapsed_seconds) << " rows/sec., " << formatReadableSizeWithBinarySuffix(uncompressed_bytes / elapsed_seconds) << "/sec. uncompressed, " << formatReadableSizeWithBinarySuffix(compressed_bytes / elapsed_seconds) << "/sec. compressed)"); + LOG_TRACE(log, "Written part in " << elapsed_seconds << " sec., " << rows << " rows, " << formatReadableSizeWithBinarySuffix(uncompressed_bytes) << " uncompressed, " << formatReadableSizeWithBinarySuffix(compressed_bytes) << " compressed, " << (uncompressed_bytes / rows) << " uncompressed bytes per row, " << (compressed_bytes / rows) << " compressed bytes per row, " << "compression rate: " << (uncompressed_bytes / compressed_bytes) << " (" << (rows / elapsed_seconds) << " rows/sec., " << formatReadableSizeWithBinarySuffix(uncompressed_bytes / elapsed_seconds) << "/sec. uncompressed, " << formatReadableSizeWithBinarySuffix(compressed_bytes / elapsed_seconds) << "/sec. compressed)"); } void Aggregator::writeToTemporaryFile(AggregatedDataVariants & data_variants) { @@ -857,7 +857,7 @@ void Aggregator::writeToTemporaryFileImpl( /// `data_variants` will not destroy them in the destructor, they are now owned by ColumnAggregateFunction objects. data_variants.aggregator = nullptr; - LOG_TRACE(log, "Max size of temporary block: " << max_temporary_block_size_rows << " rows, " << (max_temporary_block_size_bytes / 1048576.0) << " MiB."); + LOG_TRACE(log, "Max size of temporary block: " << max_temporary_block_size_rows << " rows, " << (max_temporary_block_size_bytes / 1048576.0) << " MiB."); } @@ -927,7 +927,7 @@ void Aggregator::execute(const BlockInputStreamPtr & stream, AggregatedDataVaria double elapsed_seconds = watch.elapsedSeconds(); size_t rows = result.sizeWithoutOverflowRow(); - LOG_TRACE(log, "Aggregated. " << src_rows << " to " << rows << " rows (from " << formatReadableSizeWithBinarySuffix(src_bytes) << ")" << " in " << elapsed_seconds << " sec." << " (" << src_rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(src_bytes / elapsed_seconds) << "/sec.)"); + LOG_TRACE(log, "Aggregated. " << src_rows << " to " << rows << " rows (from " << formatReadableSizeWithBinarySuffix(src_bytes) << ")" << " in " << elapsed_seconds << " sec." << " (" << src_rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(src_bytes / elapsed_seconds) << "/sec.)"); } @@ -1293,7 +1293,7 @@ BlocksList Aggregator::convertToBlocks(AggregatedDataVariants & data_variants, b } double elapsed_seconds = watch.elapsedSeconds(); - LOG_TRACE(log, "Converted aggregated data to blocks. " << rows << " rows, " << bytes / 1048576.0 << " MiB" << " in " << elapsed_seconds << " sec." << " (" << rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(bytes / elapsed_seconds) << "/sec.)"); + LOG_TRACE(log, "Converted aggregated data to blocks. " << rows << " rows, " << bytes / 1048576.0 << " MiB" << " in " << elapsed_seconds << " sec." << " (" << rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(bytes / elapsed_seconds) << "/sec.)"); return blocks; } @@ -1924,7 +1924,7 @@ void Aggregator::mergeStream(const BlockInputStreamPtr & stream, AggregatedDataV bucket_to_blocks[block.info.bucket_num].emplace_back(std::move(block)); } - LOG_TRACE(log, "Read " << total_input_blocks << " blocks of partially aggregated data, total " << total_input_rows << " rows."); + LOG_TRACE(log, "Read " << total_input_blocks << " blocks of partially aggregated data, total " << total_input_rows << " rows."); mergeBlocks(bucket_to_blocks, result, max_threads); } @@ -2155,7 +2155,7 @@ Block Aggregator::mergeBlocks(BlocksList & blocks, bool final) size_t rows = block.rows(); size_t bytes = block.bytes(); double elapsed_seconds = watch.elapsedSeconds(); - LOG_TRACE(log, "Merged partially aggregated blocks. " << rows << " rows, " << bytes / 1048576.0 << " MiB." << " in " << elapsed_seconds << " sec." << " (" << rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(bytes / elapsed_seconds) << "/sec.)"); + LOG_TRACE(log, "Merged partially aggregated blocks. " << rows << " rows, " << bytes / 1048576.0 << " MiB." << " in " << elapsed_seconds << " sec." << " (" << rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(bytes / elapsed_seconds) << "/sec.)"); if (isCancelled()) return {}; diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index 6ce3be6e651..3f911c249d5 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -542,7 +542,7 @@ void DDLWorker::parseQueryAndResolveHost(DDLTask & task) } else { - LOG_INFO(log, "Resolved host " << task.host_id.readableString() << " from task " << task.entry_name << " as host " << task.address_in_cluster.readableString() << " in definition of cluster " << task.cluster_name); + LOG_INFO(log, "Resolved host " << task.host_id.readableString() << " from task " << task.entry_name << " as host " << task.address_in_cluster.readableString() << " in definition of cluster " << task.cluster_name); } } @@ -755,7 +755,7 @@ bool DDLWorker::tryExecuteQueryOnLeaderReplica( String executed_by; if (zookeeper->tryGet(is_executed_path, executed_by)) { - LOG_DEBUG(log, "Task " << task.entry_name << " has already been executed by leader replica (" << executed_by << ") of the same shard."); + LOG_DEBUG(log, "Task " << task.entry_name << " has already been executed by leader replica (" << executed_by << ") of the same shard."); return true; } @@ -940,7 +940,7 @@ String DDLWorker::enqueueQuery(DDLLogEntry & entry) } catch (...) { - LOG_INFO(log, "An error occurred while creating auxiliary ZooKeeper directories in " << node_path << " . They will be created later" << ". Error : " << getCurrentExceptionMessage(true)); + LOG_INFO(log, "An error occurred while creating auxiliary ZooKeeper directories in " << node_path << " . They will be created later" << ". Error : " << getCurrentExceptionMessage(true)); } return node_path; diff --git a/src/Interpreters/DNSCacheUpdater.cpp b/src/Interpreters/DNSCacheUpdater.cpp index d387cccc0be..e5a97dc76d9 100644 --- a/src/Interpreters/DNSCacheUpdater.cpp +++ b/src/Interpreters/DNSCacheUpdater.cpp @@ -21,7 +21,7 @@ void DNSCacheUpdater::run() /// Reload cluster config if IP of any host has been changed since last update. if (resolver.updateCache()) { - LOG_INFO(&Poco::Logger::get("DNSCacheUpdater"), "IPs of some hosts have been changed. Will reload cluster config."); + LOG_INFO(&Poco::Logger::get("DNSCacheUpdater"), "IPs of some hosts have been changed. Will reload cluster config."); try { context.reloadClusterConfig(); diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index 92f4dc5863c..d2a286f1e78 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -439,7 +439,7 @@ public: { /// The object has been tried to load before, so it is currently in use or was in use /// and we should try to reload it with the new config. - LOG_TRACE(log, "Will reload '" << name << "'" " because its configuration has been changed and" " there were attempts to load it before"); + LOG_TRACE(log, "Will reload '" << name << "'" " because its configuration has been changed and" " there were attempts to load it before"); startLoading(info, true); } } @@ -454,7 +454,7 @@ public: Info & info = infos.emplace(name, Info{name, config}).first->second; if (always_load_everything) { - LOG_TRACE(log, "Will load '" << name << "'" " because always_load_everything flag is set."); + LOG_TRACE(log, "Will load '" << name << "'" " because always_load_everything flag is set."); startLoading(info); } } @@ -664,7 +664,7 @@ public: if (!should_update_flag) { info.next_update_time = calculateNextUpdateTime(info.object, info.error_count); - LOG_TRACE(log, "Object '" << info.name << "'" " not modified, will not reload. " "Next update at " << ext::to_string(info.next_update_time)); + LOG_TRACE(log, "Object '" << info.name << "'" " not modified, will not reload. " "Next update at " << ext::to_string(info.next_update_time)); continue; } @@ -676,7 +676,7 @@ public: /// Object was never loaded successfully and should be reloaded. startLoading(info); } - LOG_TRACE(log, "Object '" << info.name << "' is neither" " loaded nor failed, so it will not be reloaded as outdated."); + LOG_TRACE(log, "Object '" << info.name << "' is neither" " loaded nor failed, so it will not be reloaded as outdated."); } } } @@ -869,7 +869,7 @@ private: { if (info.isLoading()) { - LOG_TRACE(log, "The object '" << info.name << "' is already being loaded, force = " << forced_to_reload << "."); + LOG_TRACE(log, "The object '" << info.name << "' is already being loaded, force = " << forced_to_reload << "."); if (!forced_to_reload) { @@ -885,7 +885,7 @@ private: info.loading_start_time = std::chrono::system_clock::now(); info.loading_end_time = TimePoint{}; - LOG_TRACE(log, "Will load the object '" << info.name << "' " << (enable_async_loading ? std::string("in background") : "immediately") << ", force = " << forced_to_reload << ", loading_id = " << info.loading_id); + LOG_TRACE(log, "Will load the object '" << info.name << "' " << (enable_async_loading ? std::string("in background") : "immediately") << ", force = " << forced_to_reload << ", loading_id = " << info.loading_id); if (enable_async_loading) { @@ -924,7 +924,7 @@ private: info = prepareToLoadSingleObject(name, loading_id, min_id_to_finish_loading_dependencies_, lock); if (!info) { - LOG_TRACE(log, "Could not lock object '" << name << "' for loading"); + LOG_TRACE(log, "Could not lock object '" << name << "' for loading"); return; } } @@ -1037,7 +1037,7 @@ private: } if (info->loading_id != loading_id) { - LOG_TRACE(log, "Next update time for '" << name << "' will not be set because this object's current loading_id " << info->loading_id << " is different from the specified " << loading_id << "."); + LOG_TRACE(log, "Next update time for '" << name << "' will not be set because this object's current loading_id " << info->loading_id << " is different from the specified " << loading_id << "."); return; } @@ -1067,7 +1067,7 @@ private: info->last_successful_update_time = current_time; info->state_id = info->loading_id; info->next_update_time = next_update_time; - LOG_TRACE(log, "Next update time for '" << info->name << "' was set to " << ext::to_string(next_update_time)); + LOG_TRACE(log, "Next update time for '" << info->name << "' was set to " << ext::to_string(next_update_time)); } /// Removes the references to the loading thread from the maps. @@ -1099,7 +1099,7 @@ private: { if (!loaded_object->supportUpdates()) { - LOG_TRACE(log, "Supposed update time for " "'" + loaded_object->getLoadableName() + "'" " is never (loaded, does not support updates)"); + LOG_TRACE(log, "Supposed update time for " "'" + loaded_object->getLoadableName() + "'" " is never (loaded, does not support updates)"); return never; } @@ -1108,7 +1108,7 @@ private: const auto & lifetime = loaded_object->getLifetime(); if (lifetime.min_sec == 0 && lifetime.max_sec == 0) { - LOG_TRACE(log, "Supposed update time for " "'" + loaded_object->getLoadableName() + "'" " is never (loaded, lifetime 0)"); + LOG_TRACE(log, "Supposed update time for " "'" + loaded_object->getLoadableName() + "'" " is never (loaded, lifetime 0)"); return never; } @@ -1116,18 +1116,18 @@ private: { std::uniform_int_distribution distribution{lifetime.min_sec, lifetime.max_sec}; auto result = std::chrono::system_clock::now() + std::chrono::seconds{distribution(rnd_engine)}; - LOG_TRACE(log, "Supposed update time for " "'" << loaded_object->getLoadableName() << "'" " is " << ext::to_string(result) << " (loaded, lifetime [" << lifetime.min_sec << ", " << lifetime.max_sec << "], no errors)"); + LOG_TRACE(log, "Supposed update time for " "'" << loaded_object->getLoadableName() << "'" " is " << ext::to_string(result) << " (loaded, lifetime [" << lifetime.min_sec << ", " << lifetime.max_sec << "], no errors)"); return result; } auto result = std::chrono::system_clock::now() + std::chrono::seconds(calculateDurationWithBackoff(rnd_engine, error_count)); - LOG_TRACE(log, "Supposed update time for '" << loaded_object->getLoadableName() << "'" " is " << ext::to_string(result) << " (backoff, " << error_count << " errors)"); + LOG_TRACE(log, "Supposed update time for '" << loaded_object->getLoadableName() << "'" " is " << ext::to_string(result) << " (backoff, " << error_count << " errors)"); return result; } else { auto result = std::chrono::system_clock::now() + std::chrono::seconds(calculateDurationWithBackoff(rnd_engine, error_count)); - LOG_TRACE(log, "Supposed update time for unspecified object " " is " << ext::to_string(result) << " (backoff, " << error_count << " errors."); + LOG_TRACE(log, "Supposed update time for unspecified object " " is " << ext::to_string(result) << " (backoff, " << error_count << " errors."); return result; } } diff --git a/src/Interpreters/SystemLog.cpp b/src/Interpreters/SystemLog.cpp index 69509b7968b..f604c05401b 100644 --- a/src/Interpreters/SystemLog.cpp +++ b/src/Interpreters/SystemLog.cpp @@ -41,7 +41,7 @@ std::shared_ptr createSystemLog( if (database != default_database_name) { /// System tables must be loaded before other tables, but loading order is undefined for all databases except `system` - LOG_ERROR(&Logger::get("SystemLog"), "Custom database name for a system table specified in config. " "Table `" << table << "` will be created in `system` database " "instead of `" << database << "`"); + LOG_ERROR(&Logger::get("SystemLog"), "Custom database name for a system table specified in config. " "Table `" << table << "` will be created in `system` database " "instead of `" << database << "`"); database = default_database_name; } diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h index c61f295f341..40550705aed 100644 --- a/src/Interpreters/SystemLog.h +++ b/src/Interpreters/SystemLog.h @@ -256,7 +256,7 @@ void SystemLog::add(const LogElement & element) // TextLog sets its logger level to 0, so this log is a noop and // there is no recursive logging. - LOG_ERROR(log, "Queue is full for system log '" << demangle(typeid(*this).name()) << "'" << " at " << queue_front_index); + LOG_ERROR(log, "Queue is full for system log '" << demangle(typeid(*this).name()) << "'" << " at " << queue_front_index); } return; @@ -378,7 +378,7 @@ void SystemLog::flushImpl(const std::vector & to_flush, { try { - LOG_TRACE(log, "Flushing system log, " << to_flush.size() << " entries to flush"); + LOG_TRACE(log, "Flushing system log, " << to_flush.size() << " entries to flush"); /// We check for existence of the table and create it as needed at every /// flush. This is done to allow user to drop the table at any moment @@ -456,7 +456,7 @@ void SystemLog::prepareTable() rename->elements.emplace_back(elem); - LOG_DEBUG(log, "Existing table " << description << " for system log has obsolete or different structure." " Renaming it to " << backQuoteIfNeed(to.table)); + LOG_DEBUG(log, "Existing table " << description << " for system log has obsolete or different structure." " Renaming it to " << backQuoteIfNeed(to.table)); InterpreterRenameQuery(rename, context).execute(); diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 83ff94642c7..4a629babb60 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -113,7 +113,7 @@ static void logQuery(const String & query, const Context & context, bool interna const auto & initial_query_id = context.getClientInfo().initial_query_id; const auto & current_user = context.getClientInfo().current_user; - LOG_DEBUG(&Logger::get("executeQuery"), "(from " << context.getClientInfo().current_address.toString() << (current_user != "default" ? ", user: " + context.getClientInfo().current_user : "") << (!initial_query_id.empty() && current_query_id != initial_query_id ? ", initial_query_id: " + initial_query_id : std::string()) << ") " << joinLines(query)); + LOG_DEBUG(&Logger::get("executeQuery"), "(from " << context.getClientInfo().current_address.toString() << (current_user != "default" ? ", user: " + context.getClientInfo().current_user : "") << (!initial_query_id.empty() && current_query_id != initial_query_id ? ", initial_query_id: " + initial_query_id : std::string()) << ") " << joinLines(query)); } } @@ -140,7 +140,7 @@ static void setExceptionStackTrace(QueryLogElement & elem) /// Log exception (with query info) into text log (not into system table). static void logException(Context & context, QueryLogElement & elem) { - LOG_ERROR(&Logger::get("executeQuery"), elem.exception << " (from " << context.getClientInfo().current_address.toString() << ")" << " (in query: " << joinLines(elem.query) << ")" << (!elem.stack_trace.empty() ? ", Stack trace (when copying this message, always include the lines below):\n\n" + elem.stack_trace : "")); + LOG_ERROR(&Logger::get("executeQuery"), elem.exception << " (from " << context.getClientInfo().current_address.toString() << ")" << " (in query: " << joinLines(elem.query) << ")" << (!elem.stack_trace.empty() ? ", Stack trace (when copying this message, always include the lines below):\n\n" + elem.stack_trace : "")); } @@ -495,7 +495,7 @@ static std::tuple executeQueryImpl( if (elem.read_rows != 0) { - LOG_INFO(&Logger::get("executeQuery"), std::fixed << std::setprecision(3) << "Read " << elem.read_rows << " rows, " << formatReadableSizeWithBinarySuffix(elem.read_bytes) << " in " << elapsed_seconds << " sec., " << static_cast(elem.read_rows / elapsed_seconds) << " rows/sec., " << formatReadableSizeWithBinarySuffix(elem.read_bytes / elapsed_seconds) << "/sec."); + LOG_INFO(&Logger::get("executeQuery"), std::fixed << std::setprecision(3) << "Read " << elem.read_rows << " rows, " << formatReadableSizeWithBinarySuffix(elem.read_bytes) << " in " << elapsed_seconds << " sec., " << static_cast(elem.read_rows / elapsed_seconds) << " rows/sec., " << formatReadableSizeWithBinarySuffix(elem.read_bytes / elapsed_seconds) << "/sec."); } elem.thread_ids = std::move(info.thread_ids); diff --git a/src/Processors/Executors/PipelineExecutor.cpp b/src/Processors/Executors/PipelineExecutor.cpp index 5c3734f377f..d190f441b65 100644 --- a/src/Processors/Executors/PipelineExecutor.cpp +++ b/src/Processors/Executors/PipelineExecutor.cpp @@ -547,7 +547,7 @@ void PipelineExecutor::executeSingleThread(size_t thread_num, size_t num_threads #ifndef NDEBUG auto & context = executor_contexts[thread_num]; - LOG_TRACE(log, "Thread finished." << " Total time: " << (context->total_time_ns / 1e9) << " sec." << " Execution time: " << (context->execution_time_ns / 1e9) << " sec." << " Processing time: " << (context->processing_time_ns / 1e9) << " sec." << " Wait time: " << (context->wait_time_ns / 1e9) << " sec."); + LOG_TRACE(log, "Thread finished." << " Total time: " << (context->total_time_ns / 1e9) << " sec." << " Execution time: " << (context->execution_time_ns / 1e9) << " sec." << " Processing time: " << (context->processing_time_ns / 1e9) << " sec." << " Wait time: " << (context->wait_time_ns / 1e9) << " sec."); #endif } diff --git a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp index 7ad2285f84e..7f6bc70f8a5 100644 --- a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp @@ -629,7 +629,7 @@ private: Poco::JSON::Parser parser; auto json_body = parser.parse(*response_body).extract(); auto schema = json_body->getValue("schema"); - LOG_TRACE((&Logger::get("AvroConfluentRowInputFormat")), "Succesfully fetched schema id = " << id << "\n" << schema); + LOG_TRACE((&Logger::get("AvroConfluentRowInputFormat")), "Succesfully fetched schema id = " << id << "\n" << schema); return avro::compileJsonSchemaFromString(schema); } catch (const Exception &) diff --git a/src/Processors/Merges/Algorithms/RowRef.h b/src/Processors/Merges/Algorithms/RowRef.h index 4958c05ab14..e51046cac2a 100644 --- a/src/Processors/Merges/Algorithms/RowRef.h +++ b/src/Processors/Merges/Algorithms/RowRef.h @@ -83,7 +83,7 @@ public: { if (free_chunks.size() != chunks.size()) { - LOG_ERROR(&Logger::get("SharedChunkAllocator"), "SharedChunkAllocator was destroyed before RowRef was released. StackTrace: " << StackTrace().toString()); + LOG_ERROR(&Logger::get("SharedChunkAllocator"), "SharedChunkAllocator was destroyed before RowRef was released. StackTrace: " << StackTrace().toString()); return; } @@ -100,7 +100,7 @@ private: /// This may happen if allocator was removed before chunks. /// Log message and exit, because we don't want to throw exception in destructor. - LOG_ERROR(&Logger::get("SharedChunkAllocator"), "SharedChunkAllocator was destroyed before RowRef was released. StackTrace: " << StackTrace().toString()); + LOG_ERROR(&Logger::get("SharedChunkAllocator"), "SharedChunkAllocator was destroyed before RowRef was released. StackTrace: " << StackTrace().toString()); return; } diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index 64ca98f3808..71063050a40 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -540,7 +540,7 @@ void AggregatingTransform::initGenerate() double elapsed_seconds = watch.elapsedSeconds(); size_t rows = variants.sizeWithoutOverflowRow(); - LOG_TRACE(log, "Aggregated. " << src_rows << " to " << rows << " rows (from " << src_bytes / 1048576.0 << " MiB)" << " in " << elapsed_seconds << " sec." << " (" << src_rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(src_bytes / elapsed_seconds) << "/sec.)"); + LOG_TRACE(log, "Aggregated. " << src_rows << " to " << rows << " rows (from " << src_bytes / 1048576.0 << " MiB)" << " in " << elapsed_seconds << " sec." << " (" << src_rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(src_bytes / elapsed_seconds) << "/sec.)"); if (params->aggregator.hasTemporaryFiles()) { @@ -589,7 +589,7 @@ void AggregatingTransform::initGenerate() for (const auto & file : files.files) processors.emplace_back(std::make_unique(header, file->path())); - LOG_TRACE(log, "Will merge " << files.files.size() << " temporary files of size " << (files.sum_size_compressed / 1048576.0) << " MiB compressed, " << (files.sum_size_uncompressed / 1048576.0) << " MiB uncompressed."); + LOG_TRACE(log, "Will merge " << files.files.size() << " temporary files of size " << (files.sum_size_compressed / 1048576.0) << " MiB compressed, " << (files.sum_size_uncompressed / 1048576.0) << " MiB uncompressed."); auto pipe = createMergingAggregatedMemoryEfficientPipe( header, params, files.files.size(), temporary_data_merge_threads); diff --git a/src/Processors/Transforms/CreatingSetsTransform.cpp b/src/Processors/Transforms/CreatingSetsTransform.cpp index 7bdc084c211..a8080376169 100644 --- a/src/Processors/Transforms/CreatingSetsTransform.cpp +++ b/src/Processors/Transforms/CreatingSetsTransform.cpp @@ -56,7 +56,7 @@ IProcessor::Status CreatingSetsTransform::prepare() void CreatingSetsTransform::startSubquery(SubqueryForSet & subquery) { - LOG_TRACE(log, (subquery.set ? "Creating set. " : "") << (subquery.join ? "Creating join. " : "") << (subquery.table ? "Filling temporary table. " : "")); + LOG_TRACE(log, (subquery.set ? "Creating set. " : "") << (subquery.join ? "Creating join. " : "") << (subquery.table ? "Filling temporary table. " : "")); elapsed_nanoseconds = 0; diff --git a/src/Processors/Transforms/MergeSortingTransform.cpp b/src/Processors/Transforms/MergeSortingTransform.cpp index 4d72cced8b1..7a19a2a6bf9 100644 --- a/src/Processors/Transforms/MergeSortingTransform.cpp +++ b/src/Processors/Transforms/MergeSortingTransform.cpp @@ -251,7 +251,7 @@ void MergeSortingTransform::generate() void MergeSortingTransform::remerge() { - LOG_DEBUG(log, "Re-merging intermediate ORDER BY data (" << chunks.size() << " blocks with " << sum_rows_in_blocks << " rows) to save memory consumption"); + LOG_DEBUG(log, "Re-merging intermediate ORDER BY data (" << chunks.size() << " blocks with " << sum_rows_in_blocks << " rows) to save memory consumption"); /// NOTE Maybe concat all blocks and partial sort will be faster than merge? MergeSorter remerge_sorter(std::move(chunks), description, max_merged_block_size, limit); @@ -267,7 +267,7 @@ void MergeSortingTransform::remerge() new_chunks.emplace_back(std::move(chunk)); } - LOG_DEBUG(log, "Memory usage is lowered from " << formatReadableSizeWithBinarySuffix(sum_bytes_in_blocks) << " to " << formatReadableSizeWithBinarySuffix(new_sum_bytes_in_blocks)); + LOG_DEBUG(log, "Memory usage is lowered from " << formatReadableSizeWithBinarySuffix(sum_bytes_in_blocks) << " to " << formatReadableSizeWithBinarySuffix(new_sum_bytes_in_blocks)); /// If the memory consumption was not lowered enough - we will not perform remerge anymore. 2 is a guess. if (new_sum_bytes_in_blocks * 2 > sum_bytes_in_blocks) diff --git a/src/Processors/Transforms/MergingAggregatedTransform.cpp b/src/Processors/Transforms/MergingAggregatedTransform.cpp index b7176925986..6242075eff6 100644 --- a/src/Processors/Transforms/MergingAggregatedTransform.cpp +++ b/src/Processors/Transforms/MergingAggregatedTransform.cpp @@ -46,7 +46,7 @@ Chunk MergingAggregatedTransform::generate() if (!generate_started) { generate_started = true; - LOG_TRACE(log, "Read " << total_input_blocks << " blocks of partially aggregated data, total " << total_input_rows << " rows."); + LOG_TRACE(log, "Read " << total_input_blocks << " blocks of partially aggregated data, total " << total_input_rows << " rows."); /// Exception safety. Make iterator valid in case any method below throws. next_block = blocks.begin(); diff --git a/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/src/Storages/Distributed/DistributedBlockOutputStream.cpp index 884add0b3fa..569452f3ba8 100644 --- a/src/Storages/Distributed/DistributedBlockOutputStream.cpp +++ b/src/Storages/Distributed/DistributedBlockOutputStream.cpp @@ -114,7 +114,7 @@ void DistributedBlockOutputStream::write(const Block & block) if (ordinary_block.has(col.name)) { ordinary_block.erase(col.name); - LOG_DEBUG(log, storage.getStorageID().getNameForLogs() << ": column " + col.name + " will be removed, " << "because it is MATERIALIZED"); + LOG_DEBUG(log, storage.getStorageID().getNameForLogs() << ": column " + col.name + " will be removed, " << "because it is MATERIALIZED"); } } @@ -411,7 +411,7 @@ void DistributedBlockOutputStream::writeSuffix() auto log_performance = [this] () { double elapsed = watch.elapsedSeconds(); - LOG_DEBUG(log, "It took " << std::fixed << std::setprecision(1) << elapsed << " sec. to insert " << inserted_blocks << " blocks" << ", " << std::fixed << std::setprecision(1) << inserted_rows / elapsed << " rows per second" << ". " << getCurrentStateDescription()); + LOG_DEBUG(log, "It took " << std::fixed << std::setprecision(1) << elapsed << " sec. to insert " << inserted_blocks << " blocks" << ", " << std::fixed << std::setprecision(1) << inserted_rows / elapsed << " rows per second" << ". " << getCurrentStateDescription()); }; if (insert_sync && pool) diff --git a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp index 9af073b13ef..28e6a22977a 100644 --- a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp +++ b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp @@ -104,11 +104,11 @@ void ReadBufferFromKafkaConsumer::commit() if (topic_part.get_offset() < 0) { - LOG_TRACE( log, prefix << " " << print_special_offset() << " (topic: " << topic_part.get_topic() << ", partition: " << topic_part.get_partition() << ")"); + LOG_TRACE( log, prefix << " " << print_special_offset() << " (topic: " << topic_part.get_topic() << ", partition: " << topic_part.get_partition() << ")"); } else { - LOG_TRACE( log, prefix << " " << topic_part.get_offset() << " (topic: " << topic_part.get_topic() << ", partition: " << topic_part.get_partition() << ")"); + LOG_TRACE( log, prefix << " " << topic_part.get_offset() << " (topic: " << topic_part.get_topic() << ", partition: " << topic_part.get_partition() << ")"); } } }; @@ -166,7 +166,7 @@ void ReadBufferFromKafkaConsumer::commit() void ReadBufferFromKafkaConsumer::subscribe() { - LOG_TRACE(log,"Already subscribed to topics: [ " << boost::algorithm::join(consumer->get_subscription(), ", ") << " ]"); + LOG_TRACE(log,"Already subscribed to topics: [ " << boost::algorithm::join(consumer->get_subscription(), ", ") << " ]"); LOG_TRACE_FORMATTED(log, "Already assigned to : {}", assignment); diff --git a/src/Storages/MergeTree/EphemeralLockInZooKeeper.cpp b/src/Storages/MergeTree/EphemeralLockInZooKeeper.cpp index 909930d6aec..758401b7f4f 100644 --- a/src/Storages/MergeTree/EphemeralLockInZooKeeper.cpp +++ b/src/Storages/MergeTree/EphemeralLockInZooKeeper.cpp @@ -107,7 +107,7 @@ EphemeralLocksInAllPartitions::EphemeralLocksInAllPartitions( int rc = zookeeper.tryMulti(lock_ops, lock_responses); if (rc == Coordination::ZBADVERSION) { - LOG_TRACE(&Logger::get("EphemeralLocksInAllPartitions"), "Someone has inserted a block in a new partition while we were creating locks. Retry."); + LOG_TRACE(&Logger::get("EphemeralLocksInAllPartitions"), "Someone has inserted a block in a new partition while we were creating locks. Retry."); continue; } else if (rc != Coordination::ZOK) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 456c3665d5a..a71f4a09d3f 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -257,7 +257,7 @@ void IMergeTreeDataPart::removeIfNeeded() if (!startsWith(file_name, "tmp")) { - LOG_ERROR(storage.log, "~DataPart() should remove part " << path << " but its name doesn't start with tmp. Too suspicious, keeping the part."); + LOG_ERROR(storage.log, "~DataPart() should remove part " << path << " but its name doesn't start with tmp. Too suspicious, keeping the part."); return; } } @@ -729,7 +729,7 @@ void IMergeTreeDataPart::remove() const } catch (const Poco::FileNotFoundException &) { - LOG_ERROR(storage.log, "Directory " << fullPath(volume->getDisk(), to) << " (part to remove) doesn't exist or one of nested files has gone." " Most likely this is due to manual removing. This should be discouraged. Ignoring."); + LOG_ERROR(storage.log, "Directory " << fullPath(volume->getDisk(), to) << " (part to remove) doesn't exist or one of nested files has gone." " Most likely this is due to manual removing. This should be discouraged. Ignoring."); return; } @@ -765,7 +765,7 @@ void IMergeTreeDataPart::remove() const { /// Recursive directory removal does many excessive "stat" syscalls under the hood. - LOG_ERROR(storage.log, "Cannot quickly remove directory " << fullPath(volume->getDisk(), to) << " by removing files; fallback to recursive removal. Reason: " << getCurrentExceptionMessage(false)); + LOG_ERROR(storage.log, "Cannot quickly remove directory " << fullPath(volume->getDisk(), to) << " by removing files; fallback to recursive removal. Reason: " << getCurrentExceptionMessage(false)); volume->getDisk()->removeRecursive(to + "/"); } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index e9c3fe13d97..7d3058cfdbf 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -204,7 +204,7 @@ MergeTreeData::MergeTreeData( { if (!version_file.first.empty()) { - LOG_ERROR(log, "Duplication of version file " << fullPath(version_file.second, version_file.first) << " and " << current_version_file_path); + LOG_ERROR(log, "Duplication of version file " << fullPath(version_file.second, version_file.first) << " and " << current_version_file_path); throw Exception("Multiple format_version.txt file", ErrorCodes::CORRUPTED_DATA); } version_file = {current_version_file_path, disk}; @@ -1008,7 +1008,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) } else { - LOG_ERROR(log, "Detaching broken part " << getFullPathOnDisk(part_disk_ptr) << part_name << " because it covers less than 2 parts. You need to resolve this manually"); + LOG_ERROR(log, "Detaching broken part " << getFullPathOnDisk(part_disk_ptr) << part_name << " because it covers less than 2 parts. You need to resolve this manually"); std::lock_guard loading_lock(mutex); broken_parts_to_detach.push_back(part); ++suspicious_broken_parts; @@ -2088,7 +2088,7 @@ restore_covered) if (error) { - LOG_ERROR(log, "The set of parts restored in place of " << part->name << " looks incomplete." << " There might or might not be a data loss." << (error_parts.empty() ? "" : " Suspicious parts: " + error_parts)); + LOG_ERROR(log, "The set of parts restored in place of " << part->name << " looks incomplete." << " There might or might not be a data loss." << (error_parts.empty() ? "" : " Suspicious parts: " + error_parts)); } } } @@ -2241,7 +2241,7 @@ void MergeTreeData::delayInsertOrThrowIfNeeded(Poco::Event * until) const CurrentMetrics::Increment metric_increment(CurrentMetrics::DelayedInserts); - LOG_INFO(log, "Delaying inserting block by " << std::fixed << std::setprecision(4) << delay_milliseconds << " ms. because there are " << parts_count_in_partition << " parts"); + LOG_INFO(log, "Delaying inserting block by " << std::fixed << std::setprecision(4) << delay_milliseconds << " ms. because there are " << parts_count_in_partition << " parts"); if (until) until->tryWait(delay_milliseconds); @@ -2439,7 +2439,7 @@ void MergeTreeData::removePartContributionToColumnSizes(const DataPartPtr & part auto log_subtract = [&](size_t & from, size_t value, const char * field) { if (value > from) - LOG_ERROR(log, "Possibly incorrect column size subtraction: " << from << " - " << value << " = " << from - value << ", column: " << column.name << ", field: " << field); + LOG_ERROR(log, "Possibly incorrect column size subtraction: " << from << " - " << value << " = " << from - value << ", column: " << column.name << ", field: " << field); from -= value; }; diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index fe07bea828f..f56ef85015a 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -583,7 +583,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor const MergeTreeData::DataPartsVector & parts = future_part.parts; - LOG_DEBUG(log, "Merging " << parts.size() << " parts: from " << parts.front()->name << " to " << parts.back()->name << " into " << TMP_PREFIX + future_part.name + " with type " + future_part.type.toString()); + LOG_DEBUG(log, "Merging " << parts.size() << " parts: from " << parts.front()->name << " to " << parts.back()->name << " into " << TMP_PREFIX + future_part.name + " with type " + future_part.type.toString()); auto disk = space_reservation->getDisk(); String part_path = data.relative_data_path; diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 5a7f1690d42..3a74f8664b8 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -586,7 +586,7 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( } } - LOG_DEBUG(log, "Selected " << parts.size() << " parts by date, " << parts_with_ranges.size() << " parts by key, " << sum_marks << " marks to read from " << sum_ranges << " ranges"); + LOG_DEBUG(log, "Selected " << parts.size() << " parts by date, " << parts_with_ranges.size() << " parts by key, " << sum_marks << " marks to read from " << sum_ranges << " ranges"); if (parts_with_ranges.empty()) return {}; diff --git a/src/Storages/MergeTree/MergeTreePartsMover.cpp b/src/Storages/MergeTree/MergeTreePartsMover.cpp index 10e9314018a..60883429b32 100644 --- a/src/Storages/MergeTree/MergeTreePartsMover.cpp +++ b/src/Storages/MergeTree/MergeTreePartsMover.cpp @@ -179,7 +179,7 @@ bool MergeTreePartsMover::selectPartsForMove( if (!parts_to_move.empty()) { - LOG_TRACE(log, "Selected " << parts_to_move_by_policy_rules << " parts to move according to storage policy rules and " << parts_to_move_by_ttl_rules << " parts according to TTL rules, " << formatReadableSizeWithBinarySuffix(parts_to_move_total_size_bytes) << " total"); + LOG_TRACE(log, "Selected " << parts_to_move_by_policy_rules << " parts to move according to storage policy rules and " << parts_to_move_by_ttl_rules << " parts according to TTL rules, " << formatReadableSizeWithBinarySuffix(parts_to_move_total_size_bytes) << " total"); return true; } else @@ -215,7 +215,7 @@ void MergeTreePartsMover::swapClonedPart(const MergeTreeData::DataPartPtr & clon /// It's ok, because we don't block moving parts for merges or mutations if (!active_part || active_part->name != cloned_part->name) { - LOG_INFO(log, "Failed to swap " << cloned_part->name << ". Active part doesn't exist." << " Possible it was merged or mutated. Will remove copy on path '" << cloned_part->getFullPath() << "'."); + LOG_INFO(log, "Failed to swap " << cloned_part->name << ". Active part doesn't exist." << " Possible it was merged or mutated. Will remove copy on path '" << cloned_part->getFullPath() << "'."); return; } diff --git a/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp index 69e514f9822..1ea47d2aa23 100644 --- a/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp @@ -66,7 +66,7 @@ MergeTreeReverseSelectProcessor::MergeTreeReverseSelectProcessor( size_t total_rows = data_part->index_granularity.getTotalRows(); if (!quiet) - LOG_TRACE(log, "Reading " << all_mark_ranges.size() << " ranges in reverse order from part " << data_part->name << ", approx. " << total_rows << (all_mark_ranges.size() > 1 ? ", up to " + toString(data_part->index_granularity.getRowsCountInRanges(all_mark_ranges)) : "") << " rows starting from " << data_part->index_granularity.getMarkStartingRow(all_mark_ranges.front().begin)); + LOG_TRACE(log, "Reading " << all_mark_ranges.size() << " ranges in reverse order from part " << data_part->name << ", approx. " << total_rows << (all_mark_ranges.size() > 1 ? ", up to " + toString(data_part->index_granularity.getRowsCountInRanges(all_mark_ranges)) : "") << " rows starting from " << data_part->index_granularity.getMarkStartingRow(all_mark_ranges.front().begin)); addTotalRowsApprox(total_rows); diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp index 5aaaf6c280a..468c016cc5e 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp @@ -47,7 +47,7 @@ MergeTreeSelectProcessor::MergeTreeSelectProcessor( size_t total_rows = data_part->index_granularity.getRowsCountInRanges(all_mark_ranges); if (!quiet) - LOG_TRACE(log, "Reading " << all_mark_ranges.size() << " ranges from part " << data_part->name << ", approx. " << total_rows << (all_mark_ranges.size() > 1 ? ", up to " + toString(total_rows) : "") << " rows starting from " << data_part->index_granularity.getMarkStartingRow(all_mark_ranges.front().begin)); + LOG_TRACE(log, "Reading " << all_mark_ranges.size() << " ranges from part " << data_part->name << ", approx. " << total_rows << (all_mark_ranges.size() > 1 ? ", up to " + toString(total_rows) : "") << " rows starting from " << data_part->index_granularity.getMarkStartingRow(all_mark_ranges.front().begin)); addTotalRowsApprox(total_rows); ordered_names = header_without_virtual_columns.getNames(); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp index 3141bb84acf..6e830932f28 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp @@ -368,7 +368,7 @@ void ReplicatedMergeTreeCleanupThread::getBlocksSortedByTime(zkutil::ZooKeeper & auto not_cached_blocks = stat.numChildren - cached_block_stats.size(); if (not_cached_blocks) { - LOG_TRACE(log, "Checking " << stat.numChildren << " blocks (" << not_cached_blocks << " are not cached)" << " to clear old ones from ZooKeeper."); + LOG_TRACE(log, "Checking " << stat.numChildren << " blocks (" << not_cached_blocks << " are not cached)" << " to clear old ones from ZooKeeper."); } zkutil::AsyncResponses exists_futures; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp index e61ce4e4abf..9da64bd02ac 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp @@ -158,7 +158,7 @@ void ReplicatedMergeTreePartCheckThread::searchForMissingPart(const String & par not_found_msg = "a smaller part with the same min block."; else not_found_msg = "smaller parts with either the same min block or the same max block."; - LOG_ERROR(log, "No replica has part covering " << part_name << " and a merge is impossible: we didn't find " << not_found_msg); + LOG_ERROR(log, "No replica has part covering " << part_name << " and a merge is impossible: we didn't find " << not_found_msg); ProfileEvents::increment(ProfileEvents::ReplicatedPartChecksFailed); @@ -285,7 +285,7 @@ CheckResult ReplicatedMergeTreePartCheckThread::checkPart(const String & part_na /// but remained in the filesystem and in a number of active parts. /// And then for a long time (before restarting), the data on the replicas will be different. - LOG_TRACE(log, "Young part " << part_name << " with age " << (time(nullptr) - part->modification_time) << " seconds hasn't been added to ZooKeeper yet. It's ok."); + LOG_TRACE(log, "Young part " << part_name << " with age " << (time(nullptr) - part->modification_time) << " seconds hasn't been added to ZooKeeper yet. It's ok."); } } else diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index eaf77a637d8..6ddfdca92e7 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -75,7 +75,7 @@ bool ReplicatedMergeTreeQueue::load(zkutil::ZooKeeperPtr zookeeper) return already_loaded_paths.count(path); }); - LOG_DEBUG(log, "Having " << (to_remove_it - children.begin()) << " queue entries to load, " << (children.end() - to_remove_it) << " entries already loaded."); + LOG_DEBUG(log, "Having " << (to_remove_it - children.begin()) << " queue entries to load, " << (children.end() - to_remove_it) << " entries already loaded."); children.erase(to_remove_it, children.end()); std::sort(children.begin(), children.end()); @@ -320,7 +320,7 @@ void ReplicatedMergeTreeQueue::updateTimesInZooKeeper( auto code = zookeeper->tryMulti(ops, responses); if (code) - LOG_ERROR(log, "Couldn't set value of nodes for insert times (" << replica_path << "/min_unprocessed_insert_time, max_processed_insert_time)" << ": " << zkutil::ZooKeeper::error2string(code) + ". This shouldn't happen often."); + LOG_ERROR(log, "Couldn't set value of nodes for insert times (" << replica_path << "/min_unprocessed_insert_time, max_processed_insert_time)" << ": " << zkutil::ZooKeeper::error2string(code) + ". This shouldn't happen often."); } } @@ -365,7 +365,7 @@ void ReplicatedMergeTreeQueue::removeProcessedEntry(zkutil::ZooKeeperPtr zookeep auto code = zookeeper->tryRemove(replica_path + "/queue/" + entry->znode_name); if (code) - LOG_ERROR(log, "Couldn't remove " << replica_path << "/queue/" << entry->znode_name << ": " << zkutil::ZooKeeper::error2string(code) << ". This shouldn't happen often."); + LOG_ERROR(log, "Couldn't remove " << replica_path << "/queue/" << entry->znode_name << ": " << zkutil::ZooKeeper::error2string(code) << ". This shouldn't happen often."); updateTimesInZooKeeper(zookeeper, min_unprocessed_insert_time_changed, max_processed_insert_time_changed); } @@ -645,7 +645,7 @@ void ReplicatedMergeTreeQueue::updateMutations(zkutil::ZooKeeperPtr zookeeper, C if (!entries_to_load.empty()) { - LOG_INFO(log, "Loading " + toString(entries_to_load.size()) + " mutation entries: " + entries_to_load.front() + " - " + entries_to_load.back()); + LOG_INFO(log, "Loading " + toString(entries_to_load.size()) + " mutation entries: " + entries_to_load.front() + " - " + entries_to_load.back()); std::vector> futures; for (const String & entry : entries_to_load) @@ -845,7 +845,7 @@ void ReplicatedMergeTreeQueue::removePartProducingOpsInRange( to_wait.push_back(*it); auto code = zookeeper->tryRemove(replica_path + "/queue/" + (*it)->znode_name); if (code) - LOG_INFO(log, "Couldn't remove " << replica_path + "/queue/" + (*it)->znode_name << ": " << zkutil::ZooKeeper::error2string(code)); + LOG_INFO(log, "Couldn't remove " << replica_path + "/queue/" + (*it)->znode_name << ": " << zkutil::ZooKeeper::error2string(code)); updateStateOnQueueEntryRemoval( *it, /* is_successful = */ false, @@ -859,7 +859,7 @@ void ReplicatedMergeTreeQueue::removePartProducingOpsInRange( updateTimesInZooKeeper(zookeeper, min_unprocessed_insert_time_changed, max_processed_insert_time_changed); - LOG_DEBUG(log, "Removed " << removed_entries << " entries from queue. " "Waiting for " << to_wait.size() << " entries that are currently executing."); + LOG_DEBUG(log, "Removed " << removed_entries << " entries from queue. " "Waiting for " << to_wait.size() << " entries that are currently executing."); /// Let's wait for the operations with the parts contained in the range to be deleted. for (LogEntryPtr & entry : to_wait) @@ -1378,7 +1378,7 @@ bool ReplicatedMergeTreeQueue::tryFinalizeMutations(zkutil::ZooKeeperPtr zookeep alter_sequence.finishDataAlter(mutation.entry->alter_version, lock); if (mutation.parts_to_do.size() != 0) { - LOG_INFO(log, "Seems like we jumped over mutation " << znode << " when downloaded part with bigger mutation number." << " It's OK, tasks for rest parts will be skipped, but probably a lot of mutations were executed concurrently on different replicas."); + LOG_INFO(log, "Seems like we jumped over mutation " << znode << " when downloaded part with bigger mutation number." << " It's OK, tasks for rest parts will be skipped, but probably a lot of mutations were executed concurrently on different replicas."); mutation.parts_to_do.clear(); } } @@ -1885,7 +1885,7 @@ bool ReplicatedMergeTreeMergePredicate::isMutationFinished(const ReplicatedMerge partition_it->second.begin(), partition_it->second.lower_bound(block_num)); if (blocks_count) { - LOG_TRACE(queue.log, "Mutation " << mutation.znode_name << " is not done yet because " << "in partition ID " << partition_id << " there are still " << blocks_count << " uncommitted blocks."); + LOG_TRACE(queue.log, "Mutation " << mutation.znode_name << " is not done yet because " << "in partition ID " << partition_id << " there are still " << blocks_count << " uncommitted blocks."); return false; } } @@ -1897,7 +1897,7 @@ bool ReplicatedMergeTreeMergePredicate::isMutationFinished(const ReplicatedMerge size_t suddenly_appeared_parts = getPartNamesToMutate(mutation, queue.virtual_parts).size(); if (suddenly_appeared_parts) { - LOG_TRACE(queue.log, "Mutation " << mutation.znode_name << " is not done yet because " << suddenly_appeared_parts << " parts to mutate suddenly appeared."); + LOG_TRACE(queue.log, "Mutation " << mutation.znode_name << " is not done yet because " << suddenly_appeared_parts << " parts to mutate suddenly appeared."); return false; } } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index 3d51b611c03..9d9551d00bf 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -141,7 +141,7 @@ void ReplicatedMergeTreeRestartingThread::run() if (storage.is_leader && relative_delay > static_cast(storage_settings->min_relative_delay_to_yield_leadership)) { - LOG_INFO(log, "Relative replica delay (" << relative_delay << " seconds) is bigger than threshold (" << storage_settings->min_relative_delay_to_yield_leadership << "). Will yield leadership."); + LOG_INFO(log, "Relative replica delay (" << relative_delay << " seconds) is bigger than threshold (" << storage_settings->min_relative_delay_to_yield_leadership << "). Will yield leadership."); ProfileEvents::increment(ProfileEvents::ReplicaYieldLeadership); diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index 1f76ed2624c..0bed05841c4 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -664,7 +664,7 @@ void StorageBuffer::writeBlockToDestination(const Block & block, StoragePtr tabl if (block_to_write.columns() == 0) { - LOG_ERROR(log, "Destination table " << destination_id.getNameForLogs() << " have no common columns with block in buffer. Block of data is discarded."); + LOG_ERROR(log, "Destination table " << destination_id.getNameForLogs() << " have no common columns with block in buffer. Block of data is discarded."); return; } diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 6acf26f82ce..3ad06a12a14 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -477,12 +477,12 @@ Pipes StorageDistributed::read( ClusterPtr optimized_cluster = getOptimizedCluster(context, query_info.query); if (optimized_cluster) { - LOG_DEBUG(log, "Skipping irrelevant shards - the query will be sent to the following shards of the cluster (shard numbers): " << makeFormattedListOfShards(optimized_cluster)); + LOG_DEBUG(log, "Skipping irrelevant shards - the query will be sent to the following shards of the cluster (shard numbers): " << makeFormattedListOfShards(optimized_cluster)); cluster = optimized_cluster; } else { - LOG_DEBUG(log, (has_sharding_key ? "" : " (no sharding key)") << ": " "Unable to figure out irrelevant shards from WHERE/PREWHERE clauses - " "the query will be sent to all shards of the cluster"); + LOG_DEBUG(log, (has_sharding_key ? "" : " (no sharding key)") << ": " "Unable to figure out irrelevant shards from WHERE/PREWHERE clauses - " "the query will be sent to all shards of the cluster"); } } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index fb8f1df9fa6..d0820e89614 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -777,7 +777,7 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: !zookeeper->exists(current_part_path + "/columns", &columns_stat_after) || columns_stat_before.version != columns_stat_after.version) { - LOG_INFO(log, "Not checking checksums of part " << part_name << " with replica " << replica << " because part changed while we were reading its checksums"); + LOG_INFO(log, "Not checking checksums of part " << part_name << " with replica " << replica << " because part changed while we were reading its checksums"); continue; } @@ -787,7 +787,7 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: if (replica_part_header.getColumnsHash() != local_part_header.getColumnsHash()) { - LOG_INFO(log, "Not checking checksums of part " << part_name << " with replica " << replica << " because columns are different"); + LOG_INFO(log, "Not checking checksums of part " << part_name << " with replica " << replica << " because columns are different"); continue; } @@ -1095,7 +1095,7 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) ProfileEvents::increment(ProfileEvents::DataAfterMergeDiffersFromReplica); - LOG_ERROR(log, getCurrentExceptionMessage(false) << ". " "Data after merge is not byte-identical to data on another replicas. " "There could be several reasons: " "1. Using newer version of compression library after server update. " "2. Using another compression method. " "3. Non-deterministic compression algorithm (highly unlikely). " "4. Non-deterministic merge algorithm due to logical error in code. " "5. Data corruption in memory due to bug in code. " "6. Data corruption in memory due to hardware issue. " "7. Manual modification of source data after server startup. " "8. Manual modification of checksums stored in ZooKeeper. " "9. Part format related settings like 'enable_mixed_granularity_parts' are different on different replicas. " "We will download merged part from replica to force byte-identical result."); + LOG_ERROR(log, getCurrentExceptionMessage(false) << ". " "Data after merge is not byte-identical to data on another replicas. " "There could be several reasons: " "1. Using newer version of compression library after server update. " "2. Using another compression method. " "3. Non-deterministic compression algorithm (highly unlikely). " "4. Non-deterministic merge algorithm due to logical error in code. " "5. Data corruption in memory due to bug in code. " "6. Data corruption in memory due to hardware issue. " "7. Manual modification of source data after server startup. " "8. Manual modification of checksums stored in ZooKeeper. " "9. Part format related settings like 'enable_mixed_granularity_parts' are different on different replicas. " "We will download merged part from replica to force byte-identical result."); write_part_log(ExecutionStatus::fromCurrentException()); @@ -1217,7 +1217,7 @@ bool StorageReplicatedMergeTree::tryExecutePartMutation(const StorageReplicatedM ProfileEvents::increment(ProfileEvents::DataAfterMutationDiffersFromReplica); - LOG_ERROR(log, getCurrentExceptionMessage(false) << ". " "Data after mutation is not byte-identical to data on another replicas. " "We will download merged part from replica to force byte-identical result."); + LOG_ERROR(log, getCurrentExceptionMessage(false) << ". " "Data after mutation is not byte-identical to data on another replicas. " "We will download merged part from replica to force byte-identical result."); write_part_log(ExecutionStatus::fromCurrentException()); @@ -1288,7 +1288,7 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) if (entry.type != LogEntry::GET_PART) throw Exception("Logical error: log entry with quorum but type is not GET_PART", ErrorCodes::LOGICAL_ERROR); - LOG_DEBUG(log, "No active replica has part " << entry.new_part_name << " which needs to be written with quorum." " Will try to mark that quorum as failed."); + LOG_DEBUG(log, "No active replica has part " << entry.new_part_name << " which needs to be written with quorum." " Will try to mark that quorum as failed."); /** Atomically: * - if replicas do not become active; @@ -1363,7 +1363,7 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) } else if (code == Coordination::ZBADVERSION || code == Coordination::ZNONODE || code == Coordination::ZNODEEXISTS) { - LOG_DEBUG(log, "State was changed or isn't expected when trying to mark quorum for part " << entry.new_part_name << " as failed. Code: " << zkutil::ZooKeeper::error2string(code)); + LOG_DEBUG(log, "State was changed or isn't expected when trying to mark quorum for part " << entry.new_part_name << " as failed. Code: " << zkutil::ZooKeeper::error2string(code)); } else throw Coordination::Exception(code); @@ -1580,7 +1580,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) } catch (Exception &) { - LOG_INFO(log, "Can't use " << source_table_id.getNameForLogs() << " as source table for REPLACE PARTITION command. Will fetch all parts." << " Reason: " << getCurrentExceptionMessage(false)); + LOG_INFO(log, "Can't use " << source_table_id.getNameForLogs() << " as source table for REPLACE PARTITION command. Will fetch all parts." << " Reason: " << getCurrentExceptionMessage(false)); return 0; } @@ -2187,7 +2187,7 @@ void StorageReplicatedMergeTree::mergeSelectingTask() size_t merges_and_mutations_sum = merges_and_mutations_queued.first + merges_and_mutations_queued.second; if (merges_and_mutations_sum >= storage_settings_ptr->max_replicated_merges_in_queue) { - LOG_TRACE(log, "Number of queued merges (" << merges_and_mutations_queued.first << ") and part mutations (" << merges_and_mutations_queued.second << ") is greater than max_replicated_merges_in_queue (" << storage_settings_ptr->max_replicated_merges_in_queue << "), so won't select new parts to merge or mutate."); + LOG_TRACE(log, "Number of queued merges (" << merges_and_mutations_queued.first << ") and part mutations (" << merges_and_mutations_queued.second << ") is greater than max_replicated_merges_in_queue (" << storage_settings_ptr->max_replicated_merges_in_queue << "), so won't select new parts to merge or mutate."); } else { @@ -3692,7 +3692,7 @@ void StorageReplicatedMergeTree::drop() /// It may left some garbage if replica_path subtree are concurently modified zookeeper->tryRemoveRecursive(replica_path); if (zookeeper->exists(replica_path)) - LOG_ERROR(log, "Replica was not completely removed from ZooKeeper, " << replica_path << " still exists and may contain some garbage."); + LOG_ERROR(log, "Replica was not completely removed from ZooKeeper, " << replica_path << " still exists and may contain some garbage."); /// Check that `zookeeper_path` exists: it could have been deleted by another replica after execution of previous line. Strings replicas; @@ -3701,7 +3701,7 @@ void StorageReplicatedMergeTree::drop() LOG_INFO_FORMATTED(log, "Removing table {} (this might take several minutes)", zookeeper_path); zookeeper->tryRemoveRecursive(zookeeper_path); if (zookeeper->exists(zookeeper_path)) - LOG_ERROR(log, "Table was not completely removed from ZooKeeper, " << zookeeper_path << " still exists and may contain some garbage."); + LOG_ERROR(log, "Table was not completely removed from ZooKeeper, " << zookeeper_path << " still exists and may contain some garbage."); } } @@ -4305,7 +4305,7 @@ void StorageReplicatedMergeTree::fetchPartition(const ASTPtr & partition, const if (best_replica.empty()) throw Exception("Logical error: cannot choose best replica.", ErrorCodes::LOGICAL_ERROR); - LOG_INFO(log, "Found " << replicas.size() << " replicas, " << active_replicas.size() << " of them are active." << " Selected " << best_replica << " to fetch from."); + LOG_INFO(log, "Found " << replicas.size() << " replicas, " << active_replicas.size() << " of them are active." << " Selected " << best_replica << " to fetch from."); String best_replica_path = from + "/replicas/" + best_replica; @@ -4740,7 +4740,7 @@ void StorageReplicatedMergeTree::removePartsFromZooKeeper( } else { - LOG_DEBUG(log, "There is no part " << part_names[i] << " in ZooKeeper, it was only in filesystem"); + LOG_DEBUG(log, "There is no part " << part_names[i] << " in ZooKeeper, it was only in filesystem"); // emplace invalid future so that the total number of futures is the same as part_names.size(); remove_futures.emplace_back(); } @@ -4765,7 +4765,7 @@ void StorageReplicatedMergeTree::removePartsFromZooKeeper( continue; else if (response.error == Coordination::ZNONODE) { - LOG_DEBUG(log, "There is no part " << part_names[i] << " in ZooKeeper, it was only in filesystem"); + LOG_DEBUG(log, "There is no part " << part_names[i] << " in ZooKeeper, it was only in filesystem"); continue; } else if (Coordination::isHardwareError(response.error)) From 9d2a0d2dd7205f2beb96575ca926a180858a895c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 21:59:49 +0300 Subject: [PATCH 641/738] Apply all transformations again --- programs/copier/ClusterCopier.cpp | 56 +++++++++---------- programs/server/Server.cpp | 8 +-- programs/server/TCPHandler.cpp | 2 +- src/Common/MemoryTracker.cpp | 4 +- .../MergeSortingBlockInputStream.cpp | 2 +- src/Interpreters/Aggregator.cpp | 4 +- src/Interpreters/DDLWorker.cpp | 2 +- src/Interpreters/DNSCacheUpdater.cpp | 2 +- src/Interpreters/ExternalLoader.cpp | 6 +- src/Interpreters/SystemLog.h | 2 +- .../Formats/Impl/AvroRowInputFormat.cpp | 2 +- src/Processors/Merges/Algorithms/RowRef.h | 4 +- .../Transforms/MergeSortingTransform.cpp | 4 +- .../Transforms/MergingAggregatedTransform.cpp | 2 +- .../MergeTree/EphemeralLockInZooKeeper.cpp | 2 +- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 4 +- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- .../ReplicatedMergeTreeCleanupThread.cpp | 2 +- .../ReplicatedMergeTreePartCheckThread.cpp | 4 +- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 10 ++-- .../ReplicatedMergeTreeRestartingThread.cpp | 2 +- src/Storages/StorageBuffer.cpp | 2 +- src/Storages/StorageDistributed.cpp | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 14 ++--- 24 files changed, 72 insertions(+), 72 deletions(-) diff --git a/programs/copier/ClusterCopier.cpp b/programs/copier/ClusterCopier.cpp index 3c6afab175c..50b32676bce 100644 --- a/programs/copier/ClusterCopier.cpp +++ b/programs/copier/ClusterCopier.cpp @@ -459,7 +459,7 @@ bool ClusterCopier::checkPartitionPieceIsDone(const TaskTable & task_table, cons TaskStateWithOwner status = TaskStateWithOwner::fromString(res.data); if (status.state != TaskState::Finished) { - LOG_INFO(log, "The task " << res.data << " is being rewritten by " << status.owner << ". Partition piece will be rechecked"); + LOG_INFO_FORMATTED(log, "The task {} is being rewritten by {}. Partition piece will be rechecked", res.data, status.owner); return false; } @@ -494,7 +494,7 @@ bool ClusterCopier::checkPartitionPieceIsDone(const TaskTable & task_table, cons } catch (const Coordination::Exception & e) { - LOG_INFO(log, "A ZooKeeper error occurred while checking partition " << partition_name << " piece number " << toString(piece_number) << ". Will recheck the partition. Error: " << e.displayText()); + LOG_INFO_FORMATTED(log, "A ZooKeeper error occurred while checking partition {} piece number {}. Will recheck the partition. Error: {}", partition_name, toString(piece_number), e.displayText()); return false; } @@ -557,13 +557,13 @@ TaskStatus ClusterCopier::tryMoveAllPiecesToDestinationTable(const TaskTable & t TaskStateWithOwner status = TaskStateWithOwner::fromString(status_data); if (status.state == TaskState::Finished) { - LOG_DEBUG(log, "All pieces for partition from this task " << current_partition_attach_is_active << " has been successfully moved to destination table by " << status.owner); + LOG_DEBUG_FORMATTED(log, "All pieces for partition from this task {} has been successfully moved to destination table by {}", current_partition_attach_is_active, status.owner); return TaskStatus::Finished; } /// Task is abandoned, because previously we created ephemeral node, possibly in other copier's process. /// Initialize DROP PARTITION - LOG_DEBUG(log, "Moving piece for partition " << current_partition_attach_is_active << " has not been successfully finished by " << status.owner << ". Will try to move by myself."); + LOG_DEBUG_FORMATTED(log, "Moving piece for partition {} has not been successfully finished by {}. Will try to move by myself.", current_partition_attach_is_active, status.owner); /// Remove is_done marker. zookeeper->remove(current_partition_attach_is_done); @@ -580,7 +580,7 @@ TaskStatus ClusterCopier::tryMoveAllPiecesToDestinationTable(const TaskTable & t /// Move partition to original destination table. for (size_t current_piece_number = 0; current_piece_number < task_table.number_of_splits; ++current_piece_number) { - LOG_DEBUG(log, "Trying to move partition " << partition_name << " piece " << toString(current_piece_number) << " to original table"); + LOG_DEBUG_FORMATTED(log, "Trying to move partition {} piece {} to original table", partition_name, toString(current_piece_number)); ASTPtr query_alter_ast; String query_alter_ast_string; @@ -617,7 +617,7 @@ TaskStatus ClusterCopier::tryMoveAllPiecesToDestinationTable(const TaskTable & t } catch (...) { - LOG_DEBUG(log, "Error while moving partition " << partition_name << " piece " << toString(current_piece_number) << "to original table"); + LOG_DEBUG_FORMATTED(log, "Error while moving partition {} piece {} to original table", partition_name, toString(current_piece_number)); throw; } @@ -641,12 +641,12 @@ TaskStatus ClusterCopier::tryMoveAllPiecesToDestinationTable(const TaskTable & t &task_cluster->settings_push, PoolMode::GET_MANY); - LOG_INFO(log, "Number of shard that executed OPTIMIZE DEDUPLICATE query successfully : " << toString(num_nodes)); + LOG_INFO_FORMATTED(log, "Number of shard that executed OPTIMIZE DEDUPLICATE query successfully : {}", toString(num_nodes)); } } catch (...) { - LOG_DEBUG(log, "Error while executing OPTIMIZE DEDUPLICATE partition " << partition_name << "in the original table"); + LOG_DEBUG_FORMATTED(log, "Error while executing OPTIMIZE DEDUPLICATE partition {}in the original table", partition_name); throw; } } @@ -742,7 +742,7 @@ bool ClusterCopier::tryDropPartitionPiece( { if (e.code == Coordination::ZNODEEXISTS) { - LOG_DEBUG(log, "Partition " << task_partition.name << " piece " << toString(current_piece_number) << " is cleaning now by somebody, sleep"); + LOG_DEBUG_FORMATTED(log, "Partition {} piece {} is cleaning now by somebody, sleep", task_partition.name, toString(current_piece_number)); std::this_thread::sleep_for(default_sleep_time); return false; } @@ -755,7 +755,7 @@ bool ClusterCopier::tryDropPartitionPiece( { if (stat.numChildren != 0) { - LOG_DEBUG(log, "Partition " << task_partition.name << " contains " << stat.numChildren << " active workers while trying to drop it. Going to sleep."); + LOG_DEBUG_FORMATTED(log, "Partition {} contains {} active workers while trying to drop it. Going to sleep.", task_partition.name, stat.numChildren); std::this_thread::sleep_for(default_sleep_time); return false; } @@ -840,12 +840,12 @@ bool ClusterCopier::tryDropPartitionPiece( return false; } - LOG_INFO(log, "Partition " << task_partition.name << " piece " << toString(current_piece_number) << " was dropped on cluster " << task_table.cluster_push_name); + LOG_INFO_FORMATTED(log, "Partition {} piece {} was dropped on cluster {}", task_partition.name, toString(current_piece_number), task_table.cluster_push_name); if (zookeeper->tryCreate(current_shards_path, host_id, zkutil::CreateMode::Persistent) == Coordination::ZNODEEXISTS) zookeeper->set(current_shards_path, host_id); } - LOG_INFO(log, "Partition " << task_partition.name << " piece " << toString(current_piece_number) << " is safe for work now."); + LOG_INFO_FORMATTED(log, "Partition {} piece {} is safe for work now.", task_partition.name, toString(current_piece_number)); return true; } @@ -1014,12 +1014,12 @@ bool ClusterCopier::tryProcessTable(const ConnectionTimeouts & timeouts, TaskTab if (cluster_partition.rows_copied) { - LOG_INFO(log, "Average partition speed: " << formatReadableSizeWithDecimalSuffix(cluster_partition.bytes_copied / elapsed) << " per second."); + LOG_INFO_FORMATTED(log, "Average partition speed: {} per second.", formatReadableSizeWithDecimalSuffix(cluster_partition.bytes_copied / elapsed)); } if (task_table.rows_copied) { - LOG_INFO(log, "Average table " << task_table.table_id << " speed: " << formatReadableSizeWithDecimalSuffix(task_table.bytes_copied / elapsed) << " per second."); + LOG_INFO_FORMATTED(log, "Average table {} speed: {} per second.", task_table.table_id, formatReadableSizeWithDecimalSuffix(task_table.bytes_copied / elapsed)); } } } @@ -1239,13 +1239,13 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( TaskStateWithOwner status = TaskStateWithOwner::fromString(status_data); if (status.state == TaskState::Finished) { - LOG_DEBUG(log, "Task " << current_task_piece_status_path << " has been successfully executed by " << status.owner); + LOG_DEBUG_FORMATTED(log, "Task {} has been successfully executed by {}", current_task_piece_status_path, status.owner); return TaskStatus::Finished; } /// Task is abandoned, because previously we created ephemeral node, possibly in other copier's process. /// Initialize DROP PARTITION - LOG_DEBUG(log, "Task " << current_task_piece_status_path << " has not been successfully finished by " << status.owner << ". Partition will be dropped and refilled."); + LOG_DEBUG_FORMATTED(log, "Task {} has not been successfully finished by {}. Partition will be dropped and refilled.", current_task_piece_status_path, status.owner); create_is_dirty_node(clean_state_clock); return TaskStatus::Error; @@ -1314,12 +1314,12 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( CleanStateClock new_clean_state_clock (zookeeper, piece_is_dirty_flag_path, piece_is_dirty_cleaned_path); if (clean_state_clock != new_clean_state_clock) { - LOG_INFO(log, "Partition " << task_partition.name << " piece " << toString(current_piece_number) << " clean state changed, cowardly bailing"); + LOG_INFO_FORMATTED(log, "Partition {} piece {} clean state changed, cowardly bailing", task_partition.name, toString(current_piece_number)); return TaskStatus::Error; } else if (!new_clean_state_clock.is_clean()) { - LOG_INFO(log, "Partition " << task_partition.name << " piece " << toString(current_piece_number) << " is dirty and will be dropped and refilled"); + LOG_INFO_FORMATTED(log, "Partition {} piece {} is dirty and will be dropped and refilled", task_partition.name, toString(current_piece_number)); create_is_dirty_node(new_clean_state_clock); return TaskStatus::Error; } @@ -1350,7 +1350,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( UInt64 shards = executeQueryOnCluster(task_table.cluster_push, query, create_query_push_ast, &task_cluster->settings_push, PoolMode::GET_MANY); - LOG_DEBUG(log, "Destination tables " << getQuotedTable(task_table.table_push) << " have been created on " << shards << " shards of " << task_table.cluster_push->getShardCount()); + LOG_DEBUG_FORMATTED(log, "Destination tables {} have been created on {} shards of {}", getQuotedTable(task_table.table_push), shards, task_table.cluster_push->getShardCount()); } /// Do the copying @@ -1365,7 +1365,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( // Select all fields ASTPtr query_select_ast = get_select_query(task_shard.table_read_shard, "*", /*enable_splitting*/ true, inject_fault ? "1" : ""); - LOG_DEBUG(log, "Executing SELECT query and pull from " << task_shard.getDescription() << " : " << queryToString(query_select_ast)); + LOG_DEBUG_FORMATTED(log, "Executing SELECT query and pull from {} : {}", task_shard.getDescription(), queryToString(query_select_ast)); ASTPtr query_insert_ast; { @@ -1458,7 +1458,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( } } - LOG_INFO(log, "Partition " << task_partition.name << " piece " << toString(current_piece_number) << " copied. But not moved to original destination table."); + LOG_INFO_FORMATTED(log, "Partition {} piece {} copied. But not moved to original destination table.", task_partition.name, toString(current_piece_number)); /// Try create original table (if not exists) on each shard @@ -1473,7 +1473,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( UInt64 shards = executeQueryOnCluster(task_table.cluster_push, query, create_query_push_ast, &task_cluster->settings_push, PoolMode::GET_MANY); - LOG_DEBUG(log, "Destination tables " << getQuotedTable(task_table.table_push) << " have been created on " << shards << " shards of " << task_table.cluster_push->getShardCount()); + LOG_DEBUG_FORMATTED(log, "Destination tables {} have been created on {} shards of {}", getQuotedTable(task_table.table_push), shards, task_table.cluster_push->getShardCount()); } catch (...) { @@ -1486,12 +1486,12 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( CleanStateClock new_clean_state_clock (zookeeper, piece_is_dirty_flag_path, piece_is_dirty_cleaned_path); if (clean_state_clock != new_clean_state_clock) { - LOG_INFO(log, "Partition " << task_partition.name << " piece " << toString(current_piece_number) << " clean state changed, cowardly bailing"); + LOG_INFO_FORMATTED(log, "Partition {} piece {} clean state changed, cowardly bailing", task_partition.name, toString(current_piece_number)); return TaskStatus::Error; } else if (!new_clean_state_clock.is_clean()) { - LOG_INFO(log, "Partition " << task_partition.name << " piece " << toString(current_piece_number) << " became dirty and will be dropped and refilled"); + LOG_INFO_FORMATTED(log, "Partition {} piece {} became dirty and will be dropped and refilled", task_partition.name, toString(current_piece_number)); create_is_dirty_node(new_clean_state_clock); return TaskStatus::Error; } @@ -1718,7 +1718,7 @@ bool ClusterCopier::checkShardHasPartition(const ConnectionTimeouts & timeouts, query += " LIMIT 1"; - LOG_DEBUG(log, "Checking shard " << task_shard.getDescription() << " for partition " << partition_quoted_name << " existence, executing query: " << query); + LOG_DEBUG_FORMATTED(log, "Checking shard {} for partition {} existence, executing query: {}", task_shard.getDescription(), partition_quoted_name, query); ParserQuery parser_query(query.data() + query.size()); const auto & settings = context.getSettingsRef(); @@ -1767,9 +1767,9 @@ bool ClusterCopier::checkPresentPartitionPiecesOnCurrentShard(const ConnectionTi local_context.setSettings(task_cluster->settings_pull); auto result = InterpreterFactory::get(query_ast, local_context)->execute().in->read().rows(); if (result != 0) - LOG_DEBUG(log, "Partition " << partition_quoted_name << " piece number " << std::to_string(current_piece_number) << " is PRESENT on shard " << task_shard.getDescription()); + LOG_DEBUG_FORMATTED(log, "Partition {} piece number {} is PRESENT on shard {}", partition_quoted_name, std::to_string(current_piece_number), task_shard.getDescription()); else - LOG_DEBUG(log, "Partition " << partition_quoted_name << " piece number " << std::to_string(current_piece_number) << " is ABSENT on shard " << task_shard.getDescription()); + LOG_DEBUG_FORMATTED(log, "Partition {} piece number {} is ABSENT on shard {}", partition_quoted_name, std::to_string(current_piece_number), task_shard.getDescription()); return result != 0; } @@ -1886,7 +1886,7 @@ UInt64 ClusterCopier::executeQueryOnCluster( if (execution_mode == ClusterExecutionMode::ON_EACH_NODE && successful_nodes != origin_replicas_number) { - LOG_INFO(log, "There was an error while executing ALTER on each node. Query was executed on " << toString(successful_nodes) << " nodes. But had to be executed on " << toString(origin_replicas_number.load())); + LOG_INFO_FORMATTED(log, "There was an error while executing ALTER on each node. Query was executed on {} nodes. But had to be executed on {}", toString(successful_nodes), toString(origin_replicas_number.load())); } diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 5e01fbdac4d..c352a6ce8ed 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -530,7 +530,7 @@ int Server::main(const std::vector & /*args*/) if (uncompressed_cache_size > max_cache_size) { uncompressed_cache_size = max_cache_size; - LOG_INFO(log, "Uncompressed cache size was lowered to " << formatReadableSizeWithBinarySuffix(uncompressed_cache_size) << " because the system has low amount of memory"); + LOG_INFO_FORMATTED(log, "Uncompressed cache size was lowered to {} because the system has low amount of memory", formatReadableSizeWithBinarySuffix(uncompressed_cache_size)); } global_context->setUncompressedCache(uncompressed_cache_size); @@ -545,7 +545,7 @@ int Server::main(const std::vector & /*args*/) if (mark_cache_size > max_cache_size) { mark_cache_size = max_cache_size; - LOG_INFO(log, "Mark cache size was lowered to " << formatReadableSizeWithBinarySuffix(uncompressed_cache_size) << " because the system has low amount of memory"); + LOG_INFO_FORMATTED(log, "Mark cache size was lowered to {} because the system has low amount of memory", formatReadableSizeWithBinarySuffix(uncompressed_cache_size)); } global_context->setMarkCache(mark_cache_size); @@ -574,7 +574,7 @@ int Server::main(const std::vector & /*args*/) else if (max_server_memory_usage > default_max_server_memory_usage) { max_server_memory_usage = default_max_server_memory_usage; - LOG_INFO(log, "Setting max_server_memory_usage was lowered to " << formatReadableSizeWithBinarySuffix(max_server_memory_usage) << " because the system has low amount of memory"); + LOG_INFO_FORMATTED(log, "Setting max_server_memory_usage was lowered to {} because the system has low amount of memory", formatReadableSizeWithBinarySuffix(max_server_memory_usage)); } total_memory_tracker.setOrRaiseHardLimit(max_server_memory_usage); @@ -959,7 +959,7 @@ int Server::main(const std::vector & /*args*/) current_connections += server->currentConnections(); } - LOG_INFO(log, "Closed all listening sockets." << (current_connections ? " Waiting for " + toString(current_connections) + " outstanding connections." : "")); + LOG_INFO_FORMATTED(log, "Closed all listening sockets.{}", (current_connections ? " Waiting for " + toString(current_connections) + " outstanding connections." : "")); /// Killing remaining queries. global_context->getProcessList().killAllQueries(); diff --git a/programs/server/TCPHandler.cpp b/programs/server/TCPHandler.cpp index 612fc678505..b3e329a74e5 100644 --- a/programs/server/TCPHandler.cpp +++ b/programs/server/TCPHandler.cpp @@ -115,7 +115,7 @@ void TCPHandler::runImpl() if (!DatabaseCatalog::instance().isDatabaseExist(default_database)) { Exception e("Database " + backQuote(default_database) + " doesn't exist", ErrorCodes::UNKNOWN_DATABASE); - LOG_ERROR(log, "Code: " << e.code() << ", e.displayText() = " << e.displayText() << ", Stack trace:\n\n" << e.getStackTraceString()); + LOG_ERROR_FORMATTED(log, "Code: {}, e.displayText() = {}, Stack trace:\n\n{}", e.code(), e.displayText(), e.getStackTraceString()); sendException(e, connection_context.getSettingsRef().calculate_text_stack_trace); return; } diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp index 4c729a25f4c..79825b90880 100644 --- a/src/Common/MemoryTracker.cpp +++ b/src/Common/MemoryTracker.cpp @@ -49,12 +49,12 @@ MemoryTracker::~MemoryTracker() void MemoryTracker::logPeakMemoryUsage() const { - LOG_DEBUG(&Logger::get("MemoryTracker"), "Peak memory usage" << (description ? " " + std::string(description) : "") << ": " << formatReadableSizeWithBinarySuffix(peak) << "."); + LOG_DEBUG_FORMATTED(&Logger::get("MemoryTracker"), "Peak memory usage{}: {}.", (description ? " " + std::string(description) : ""), formatReadableSizeWithBinarySuffix(peak)); } void MemoryTracker::logMemoryUsage(Int64 current) const { - LOG_DEBUG(&Logger::get("MemoryTracker"), "Current memory usage" << (description ? " " + std::string(description) : "") << ": " << formatReadableSizeWithBinarySuffix(current) << "."); + LOG_DEBUG_FORMATTED(&Logger::get("MemoryTracker"), "Current memory usage{}: {}.", (description ? " " + std::string(description) : ""), formatReadableSizeWithBinarySuffix(current)); } diff --git a/src/DataStreams/MergeSortingBlockInputStream.cpp b/src/DataStreams/MergeSortingBlockInputStream.cpp index 20833f717d3..7c633460680 100644 --- a/src/DataStreams/MergeSortingBlockInputStream.cpp +++ b/src/DataStreams/MergeSortingBlockInputStream.cpp @@ -264,7 +264,7 @@ void MergeSortingBlockInputStream::remerge() } merger.readSuffix(); - LOG_DEBUG(log, "Memory usage is lowered from " << formatReadableSizeWithBinarySuffix(sum_bytes_in_blocks) << " to " << formatReadableSizeWithBinarySuffix(new_sum_bytes_in_blocks)); + LOG_DEBUG_FORMATTED(log, "Memory usage is lowered from {} to {}", formatReadableSizeWithBinarySuffix(sum_bytes_in_blocks), formatReadableSizeWithBinarySuffix(new_sum_bytes_in_blocks)); /// If the memory consumption was not lowered enough - we will not perform remerge anymore. 2 is a guess. if (new_sum_bytes_in_blocks * 2 > sum_bytes_in_blocks) diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index fc9f8d52065..cfcdf009d29 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -857,7 +857,7 @@ void Aggregator::writeToTemporaryFileImpl( /// `data_variants` will not destroy them in the destructor, they are now owned by ColumnAggregateFunction objects. data_variants.aggregator = nullptr; - LOG_TRACE(log, "Max size of temporary block: " << max_temporary_block_size_rows << " rows, " << (max_temporary_block_size_bytes / 1048576.0) << " MiB."); + LOG_TRACE_FORMATTED(log, "Max size of temporary block: {} rows, {} MiB.", max_temporary_block_size_rows, (max_temporary_block_size_bytes / 1048576.0)); } @@ -1924,7 +1924,7 @@ void Aggregator::mergeStream(const BlockInputStreamPtr & stream, AggregatedDataV bucket_to_blocks[block.info.bucket_num].emplace_back(std::move(block)); } - LOG_TRACE(log, "Read " << total_input_blocks << " blocks of partially aggregated data, total " << total_input_rows << " rows."); + LOG_TRACE_FORMATTED(log, "Read {} blocks of partially aggregated data, total {} rows.", total_input_blocks, total_input_rows); mergeBlocks(bucket_to_blocks, result, max_threads); } diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index 3f911c249d5..53be380e029 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -755,7 +755,7 @@ bool DDLWorker::tryExecuteQueryOnLeaderReplica( String executed_by; if (zookeeper->tryGet(is_executed_path, executed_by)) { - LOG_DEBUG(log, "Task " << task.entry_name << " has already been executed by leader replica (" << executed_by << ") of the same shard."); + LOG_DEBUG_FORMATTED(log, "Task {} has already been executed by leader replica ({}) of the same shard.", task.entry_name, executed_by); return true; } diff --git a/src/Interpreters/DNSCacheUpdater.cpp b/src/Interpreters/DNSCacheUpdater.cpp index e5a97dc76d9..98957555c24 100644 --- a/src/Interpreters/DNSCacheUpdater.cpp +++ b/src/Interpreters/DNSCacheUpdater.cpp @@ -21,7 +21,7 @@ void DNSCacheUpdater::run() /// Reload cluster config if IP of any host has been changed since last update. if (resolver.updateCache()) { - LOG_INFO(&Poco::Logger::get("DNSCacheUpdater"), "IPs of some hosts have been changed. Will reload cluster config."); + LOG_INFO_FORMATTED(&Poco::Logger::get("DNSCacheUpdater"), "IPs of some hosts have been changed. Will reload cluster config."); try { context.reloadClusterConfig(); diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index d2a286f1e78..29d645e89d9 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -869,7 +869,7 @@ private: { if (info.isLoading()) { - LOG_TRACE(log, "The object '" << info.name << "' is already being loaded, force = " << forced_to_reload << "."); + LOG_TRACE_FORMATTED(log, "The object '{}' is already being loaded, force = {}.", info.name, forced_to_reload); if (!forced_to_reload) { @@ -924,7 +924,7 @@ private: info = prepareToLoadSingleObject(name, loading_id, min_id_to_finish_loading_dependencies_, lock); if (!info) { - LOG_TRACE(log, "Could not lock object '" << name << "' for loading"); + LOG_TRACE_FORMATTED(log, "Could not lock object '{}' for loading", name); return; } } @@ -1067,7 +1067,7 @@ private: info->last_successful_update_time = current_time; info->state_id = info->loading_id; info->next_update_time = next_update_time; - LOG_TRACE(log, "Next update time for '" << info->name << "' was set to " << ext::to_string(next_update_time)); + LOG_TRACE_FORMATTED(log, "Next update time for '{}' was set to {}", info->name, ext::to_string(next_update_time)); } /// Removes the references to the loading thread from the maps. diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h index 40550705aed..a7c488fab86 100644 --- a/src/Interpreters/SystemLog.h +++ b/src/Interpreters/SystemLog.h @@ -378,7 +378,7 @@ void SystemLog::flushImpl(const std::vector & to_flush, { try { - LOG_TRACE(log, "Flushing system log, " << to_flush.size() << " entries to flush"); + LOG_TRACE_FORMATTED(log, "Flushing system log, {} entries to flush", to_flush.size()); /// We check for existence of the table and create it as needed at every /// flush. This is done to allow user to drop the table at any moment diff --git a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp index 7f6bc70f8a5..01d7f6c9d91 100644 --- a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp @@ -629,7 +629,7 @@ private: Poco::JSON::Parser parser; auto json_body = parser.parse(*response_body).extract(); auto schema = json_body->getValue("schema"); - LOG_TRACE((&Logger::get("AvroConfluentRowInputFormat")), "Succesfully fetched schema id = " << id << "\n" << schema); + LOG_TRACE_FORMATTED((&Logger::get("AvroConfluentRowInputFormat")), "Succesfully fetched schema id = {}\n{}", id, schema); return avro::compileJsonSchemaFromString(schema); } catch (const Exception &) diff --git a/src/Processors/Merges/Algorithms/RowRef.h b/src/Processors/Merges/Algorithms/RowRef.h index e51046cac2a..4ceec885a0b 100644 --- a/src/Processors/Merges/Algorithms/RowRef.h +++ b/src/Processors/Merges/Algorithms/RowRef.h @@ -83,7 +83,7 @@ public: { if (free_chunks.size() != chunks.size()) { - LOG_ERROR(&Logger::get("SharedChunkAllocator"), "SharedChunkAllocator was destroyed before RowRef was released. StackTrace: " << StackTrace().toString()); + LOG_ERROR_FORMATTED(&Logger::get("SharedChunkAllocator"), "SharedChunkAllocator was destroyed before RowRef was released. StackTrace: {}", StackTrace().toString()); return; } @@ -100,7 +100,7 @@ private: /// This may happen if allocator was removed before chunks. /// Log message and exit, because we don't want to throw exception in destructor. - LOG_ERROR(&Logger::get("SharedChunkAllocator"), "SharedChunkAllocator was destroyed before RowRef was released. StackTrace: " << StackTrace().toString()); + LOG_ERROR_FORMATTED(&Logger::get("SharedChunkAllocator"), "SharedChunkAllocator was destroyed before RowRef was released. StackTrace: {}", StackTrace().toString()); return; } diff --git a/src/Processors/Transforms/MergeSortingTransform.cpp b/src/Processors/Transforms/MergeSortingTransform.cpp index 7a19a2a6bf9..7147781b4e8 100644 --- a/src/Processors/Transforms/MergeSortingTransform.cpp +++ b/src/Processors/Transforms/MergeSortingTransform.cpp @@ -251,7 +251,7 @@ void MergeSortingTransform::generate() void MergeSortingTransform::remerge() { - LOG_DEBUG(log, "Re-merging intermediate ORDER BY data (" << chunks.size() << " blocks with " << sum_rows_in_blocks << " rows) to save memory consumption"); + LOG_DEBUG_FORMATTED(log, "Re-merging intermediate ORDER BY data ({} blocks with {} rows) to save memory consumption", chunks.size(), sum_rows_in_blocks); /// NOTE Maybe concat all blocks and partial sort will be faster than merge? MergeSorter remerge_sorter(std::move(chunks), description, max_merged_block_size, limit); @@ -267,7 +267,7 @@ void MergeSortingTransform::remerge() new_chunks.emplace_back(std::move(chunk)); } - LOG_DEBUG(log, "Memory usage is lowered from " << formatReadableSizeWithBinarySuffix(sum_bytes_in_blocks) << " to " << formatReadableSizeWithBinarySuffix(new_sum_bytes_in_blocks)); + LOG_DEBUG_FORMATTED(log, "Memory usage is lowered from {} to {}", formatReadableSizeWithBinarySuffix(sum_bytes_in_blocks), formatReadableSizeWithBinarySuffix(new_sum_bytes_in_blocks)); /// If the memory consumption was not lowered enough - we will not perform remerge anymore. 2 is a guess. if (new_sum_bytes_in_blocks * 2 > sum_bytes_in_blocks) diff --git a/src/Processors/Transforms/MergingAggregatedTransform.cpp b/src/Processors/Transforms/MergingAggregatedTransform.cpp index 6242075eff6..2d787c5c3b6 100644 --- a/src/Processors/Transforms/MergingAggregatedTransform.cpp +++ b/src/Processors/Transforms/MergingAggregatedTransform.cpp @@ -46,7 +46,7 @@ Chunk MergingAggregatedTransform::generate() if (!generate_started) { generate_started = true; - LOG_TRACE(log, "Read " << total_input_blocks << " blocks of partially aggregated data, total " << total_input_rows << " rows."); + LOG_TRACE_FORMATTED(log, "Read {} blocks of partially aggregated data, total {} rows.", total_input_blocks, total_input_rows); /// Exception safety. Make iterator valid in case any method below throws. next_block = blocks.begin(); diff --git a/src/Storages/MergeTree/EphemeralLockInZooKeeper.cpp b/src/Storages/MergeTree/EphemeralLockInZooKeeper.cpp index 758401b7f4f..2f4a42848e5 100644 --- a/src/Storages/MergeTree/EphemeralLockInZooKeeper.cpp +++ b/src/Storages/MergeTree/EphemeralLockInZooKeeper.cpp @@ -107,7 +107,7 @@ EphemeralLocksInAllPartitions::EphemeralLocksInAllPartitions( int rc = zookeeper.tryMulti(lock_ops, lock_responses); if (rc == Coordination::ZBADVERSION) { - LOG_TRACE(&Logger::get("EphemeralLocksInAllPartitions"), "Someone has inserted a block in a new partition while we were creating locks. Retry."); + LOG_TRACE_FORMATTED(&Logger::get("EphemeralLocksInAllPartitions"), "Someone has inserted a block in a new partition while we were creating locks. Retry."); continue; } else if (rc != Coordination::ZOK) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index a71f4a09d3f..0ea1dfddabc 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -257,7 +257,7 @@ void IMergeTreeDataPart::removeIfNeeded() if (!startsWith(file_name, "tmp")) { - LOG_ERROR(storage.log, "~DataPart() should remove part " << path << " but its name doesn't start with tmp. Too suspicious, keeping the part."); + LOG_ERROR_FORMATTED(storage.log, "~DataPart() should remove part {} but its name doesn't start with tmp. Too suspicious, keeping the part.", path); return; } } @@ -765,7 +765,7 @@ void IMergeTreeDataPart::remove() const { /// Recursive directory removal does many excessive "stat" syscalls under the hood. - LOG_ERROR(storage.log, "Cannot quickly remove directory " << fullPath(volume->getDisk(), to) << " by removing files; fallback to recursive removal. Reason: " << getCurrentExceptionMessage(false)); + LOG_ERROR_FORMATTED(storage.log, "Cannot quickly remove directory {} by removing files; fallback to recursive removal. Reason: {}", fullPath(volume->getDisk(), to), getCurrentExceptionMessage(false)); volume->getDisk()->removeRecursive(to + "/"); } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 7d3058cfdbf..3e57a48b135 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -204,7 +204,7 @@ MergeTreeData::MergeTreeData( { if (!version_file.first.empty()) { - LOG_ERROR(log, "Duplication of version file " << fullPath(version_file.second, version_file.first) << " and " << current_version_file_path); + LOG_ERROR_FORMATTED(log, "Duplication of version file {} and {}", fullPath(version_file.second, version_file.first), current_version_file_path); throw Exception("Multiple format_version.txt file", ErrorCodes::CORRUPTED_DATA); } version_file = {current_version_file_path, disk}; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp index 6e830932f28..80577bd1b20 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp @@ -368,7 +368,7 @@ void ReplicatedMergeTreeCleanupThread::getBlocksSortedByTime(zkutil::ZooKeeper & auto not_cached_blocks = stat.numChildren - cached_block_stats.size(); if (not_cached_blocks) { - LOG_TRACE(log, "Checking " << stat.numChildren << " blocks (" << not_cached_blocks << " are not cached)" << " to clear old ones from ZooKeeper."); + LOG_TRACE_FORMATTED(log, "Checking {} blocks ({} are not cached){}", stat.numChildren, not_cached_blocks, " to clear old ones from ZooKeeper."); } zkutil::AsyncResponses exists_futures; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp index 9da64bd02ac..383a1e5e485 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp @@ -158,7 +158,7 @@ void ReplicatedMergeTreePartCheckThread::searchForMissingPart(const String & par not_found_msg = "a smaller part with the same min block."; else not_found_msg = "smaller parts with either the same min block or the same max block."; - LOG_ERROR(log, "No replica has part covering " << part_name << " and a merge is impossible: we didn't find " << not_found_msg); + LOG_ERROR_FORMATTED(log, "No replica has part covering {} and a merge is impossible: we didn't find {}", part_name, not_found_msg); ProfileEvents::increment(ProfileEvents::ReplicatedPartChecksFailed); @@ -285,7 +285,7 @@ CheckResult ReplicatedMergeTreePartCheckThread::checkPart(const String & part_na /// but remained in the filesystem and in a number of active parts. /// And then for a long time (before restarting), the data on the replicas will be different. - LOG_TRACE(log, "Young part " << part_name << " with age " << (time(nullptr) - part->modification_time) << " seconds hasn't been added to ZooKeeper yet. It's ok."); + LOG_TRACE_FORMATTED(log, "Young part {} with age {} seconds hasn't been added to ZooKeeper yet. It's ok.", part_name, (time(nullptr) - part->modification_time)); } } else diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 6ddfdca92e7..027e33a1373 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -75,7 +75,7 @@ bool ReplicatedMergeTreeQueue::load(zkutil::ZooKeeperPtr zookeeper) return already_loaded_paths.count(path); }); - LOG_DEBUG(log, "Having " << (to_remove_it - children.begin()) << " queue entries to load, " << (children.end() - to_remove_it) << " entries already loaded."); + LOG_DEBUG_FORMATTED(log, "Having {} queue entries to load, {} entries already loaded.", (to_remove_it - children.begin()), (children.end() - to_remove_it)); children.erase(to_remove_it, children.end()); std::sort(children.begin(), children.end()); @@ -645,7 +645,7 @@ void ReplicatedMergeTreeQueue::updateMutations(zkutil::ZooKeeperPtr zookeeper, C if (!entries_to_load.empty()) { - LOG_INFO(log, "Loading " + toString(entries_to_load.size()) + " mutation entries: " + entries_to_load.front() + " - " + entries_to_load.back()); + LOG_INFO_FORMATTED(log, "Loading {} mutation entries: {} - {}", toString(entries_to_load.size()), entries_to_load.front(), entries_to_load.back()); std::vector> futures; for (const String & entry : entries_to_load) @@ -845,7 +845,7 @@ void ReplicatedMergeTreeQueue::removePartProducingOpsInRange( to_wait.push_back(*it); auto code = zookeeper->tryRemove(replica_path + "/queue/" + (*it)->znode_name); if (code) - LOG_INFO(log, "Couldn't remove " << replica_path + "/queue/" + (*it)->znode_name << ": " << zkutil::ZooKeeper::error2string(code)); + LOG_INFO_FORMATTED(log, "Couldn't remove {}: {}", replica_path + "/queue/" + (*it)->znode_name, zkutil::ZooKeeper::error2string(code)); updateStateOnQueueEntryRemoval( *it, /* is_successful = */ false, @@ -1378,7 +1378,7 @@ bool ReplicatedMergeTreeQueue::tryFinalizeMutations(zkutil::ZooKeeperPtr zookeep alter_sequence.finishDataAlter(mutation.entry->alter_version, lock); if (mutation.parts_to_do.size() != 0) { - LOG_INFO(log, "Seems like we jumped over mutation " << znode << " when downloaded part with bigger mutation number." << " It's OK, tasks for rest parts will be skipped, but probably a lot of mutations were executed concurrently on different replicas."); + LOG_INFO_FORMATTED(log, "Seems like we jumped over mutation {} when downloaded part with bigger mutation number.{}", znode, " It's OK, tasks for rest parts will be skipped, but probably a lot of mutations were executed concurrently on different replicas."); mutation.parts_to_do.clear(); } } @@ -1897,7 +1897,7 @@ bool ReplicatedMergeTreeMergePredicate::isMutationFinished(const ReplicatedMerge size_t suddenly_appeared_parts = getPartNamesToMutate(mutation, queue.virtual_parts).size(); if (suddenly_appeared_parts) { - LOG_TRACE(queue.log, "Mutation " << mutation.znode_name << " is not done yet because " << suddenly_appeared_parts << " parts to mutate suddenly appeared."); + LOG_TRACE_FORMATTED(queue.log, "Mutation {} is not done yet because {} parts to mutate suddenly appeared.", mutation.znode_name, suddenly_appeared_parts); return false; } } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index 9d9551d00bf..c60c6953edf 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -141,7 +141,7 @@ void ReplicatedMergeTreeRestartingThread::run() if (storage.is_leader && relative_delay > static_cast(storage_settings->min_relative_delay_to_yield_leadership)) { - LOG_INFO(log, "Relative replica delay (" << relative_delay << " seconds) is bigger than threshold (" << storage_settings->min_relative_delay_to_yield_leadership << "). Will yield leadership."); + LOG_INFO_FORMATTED(log, "Relative replica delay ({} seconds) is bigger than threshold ({}). Will yield leadership.", relative_delay, storage_settings->min_relative_delay_to_yield_leadership); ProfileEvents::increment(ProfileEvents::ReplicaYieldLeadership); diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index 0bed05841c4..82f781b3193 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -664,7 +664,7 @@ void StorageBuffer::writeBlockToDestination(const Block & block, StoragePtr tabl if (block_to_write.columns() == 0) { - LOG_ERROR(log, "Destination table " << destination_id.getNameForLogs() << " have no common columns with block in buffer. Block of data is discarded."); + LOG_ERROR_FORMATTED(log, "Destination table {} have no common columns with block in buffer. Block of data is discarded.", destination_id.getNameForLogs()); return; } diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 3ad06a12a14..771ec0abaca 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -477,7 +477,7 @@ Pipes StorageDistributed::read( ClusterPtr optimized_cluster = getOptimizedCluster(context, query_info.query); if (optimized_cluster) { - LOG_DEBUG(log, "Skipping irrelevant shards - the query will be sent to the following shards of the cluster (shard numbers): " << makeFormattedListOfShards(optimized_cluster)); + LOG_DEBUG_FORMATTED(log, "Skipping irrelevant shards - the query will be sent to the following shards of the cluster (shard numbers): {}", makeFormattedListOfShards(optimized_cluster)); cluster = optimized_cluster; } else diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index d0820e89614..d539bd53a11 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -777,7 +777,7 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: !zookeeper->exists(current_part_path + "/columns", &columns_stat_after) || columns_stat_before.version != columns_stat_after.version) { - LOG_INFO(log, "Not checking checksums of part " << part_name << " with replica " << replica << " because part changed while we were reading its checksums"); + LOG_INFO_FORMATTED(log, "Not checking checksums of part {} with replica {} because part changed while we were reading its checksums", part_name, replica); continue; } @@ -787,7 +787,7 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: if (replica_part_header.getColumnsHash() != local_part_header.getColumnsHash()) { - LOG_INFO(log, "Not checking checksums of part " << part_name << " with replica " << replica << " because columns are different"); + LOG_INFO_FORMATTED(log, "Not checking checksums of part {} with replica {} because columns are different", part_name, replica); continue; } @@ -1363,7 +1363,7 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) } else if (code == Coordination::ZBADVERSION || code == Coordination::ZNONODE || code == Coordination::ZNODEEXISTS) { - LOG_DEBUG(log, "State was changed or isn't expected when trying to mark quorum for part " << entry.new_part_name << " as failed. Code: " << zkutil::ZooKeeper::error2string(code)); + LOG_DEBUG_FORMATTED(log, "State was changed or isn't expected when trying to mark quorum for part {} as failed. Code: {}", entry.new_part_name, zkutil::ZooKeeper::error2string(code)); } else throw Coordination::Exception(code); @@ -3692,7 +3692,7 @@ void StorageReplicatedMergeTree::drop() /// It may left some garbage if replica_path subtree are concurently modified zookeeper->tryRemoveRecursive(replica_path); if (zookeeper->exists(replica_path)) - LOG_ERROR(log, "Replica was not completely removed from ZooKeeper, " << replica_path << " still exists and may contain some garbage."); + LOG_ERROR_FORMATTED(log, "Replica was not completely removed from ZooKeeper, {} still exists and may contain some garbage.", replica_path); /// Check that `zookeeper_path` exists: it could have been deleted by another replica after execution of previous line. Strings replicas; @@ -3701,7 +3701,7 @@ void StorageReplicatedMergeTree::drop() LOG_INFO_FORMATTED(log, "Removing table {} (this might take several minutes)", zookeeper_path); zookeeper->tryRemoveRecursive(zookeeper_path); if (zookeeper->exists(zookeeper_path)) - LOG_ERROR(log, "Table was not completely removed from ZooKeeper, " << zookeeper_path << " still exists and may contain some garbage."); + LOG_ERROR_FORMATTED(log, "Table was not completely removed from ZooKeeper, {} still exists and may contain some garbage.", zookeeper_path); } } @@ -4740,7 +4740,7 @@ void StorageReplicatedMergeTree::removePartsFromZooKeeper( } else { - LOG_DEBUG(log, "There is no part " << part_names[i] << " in ZooKeeper, it was only in filesystem"); + LOG_DEBUG_FORMATTED(log, "There is no part {} in ZooKeeper, it was only in filesystem", part_names[i]); // emplace invalid future so that the total number of futures is the same as part_names.size(); remove_futures.emplace_back(); } @@ -4765,7 +4765,7 @@ void StorageReplicatedMergeTree::removePartsFromZooKeeper( continue; else if (response.error == Coordination::ZNONODE) { - LOG_DEBUG(log, "There is no part " << part_names[i] << " in ZooKeeper, it was only in filesystem"); + LOG_DEBUG_FORMATTED(log, "There is no part {} in ZooKeeper, it was only in filesystem", part_names[i]); continue; } else if (Coordination::isHardwareError(response.error)) From ab0562a574a9fbb076eae7a591ffcfb4f0d681e6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 22:05:41 +0300 Subject: [PATCH 642/738] Make all LOG in single line (preparation) --- src/Client/ConnectionPoolWithFailover.cpp | 5 +---- src/Common/Config/ConfigProcessor.cpp | 5 +---- src/Disks/S3/registerDiskS3.cpp | 3 +-- src/Disks/VolumeJBOD.cpp | 11 ++--------- src/IO/WriteBufferFromS3.cpp | 8 ++------ .../ClusterProxy/SelectStreamFactory.cpp | 13 +++---------- src/Storages/Distributed/DirectoryMonitor.cpp | 5 +---- 7 files changed, 11 insertions(+), 39 deletions(-) diff --git a/src/Client/ConnectionPoolWithFailover.cpp b/src/Client/ConnectionPoolWithFailover.cpp index 52bd73c9de4..c40ea290f00 100644 --- a/src/Client/ConnectionPoolWithFailover.cpp +++ b/src/Client/ConnectionPoolWithFailover.cpp @@ -248,10 +248,7 @@ ConnectionPoolWithFailover::tryGetEntry( result.is_up_to_date = false; result.staleness = delay; - LOG_TRACE( - log, "Server " << result.entry->getDescription() << " has unacceptable replica delay " - << "for table " << table_to_check->database << "." << table_to_check->table - << ": " << delay); + LOG_TRACE( log, "Server " << result.entry->getDescription() << " has unacceptable replica delay " << "for table " << table_to_check->database << "." << table_to_check->table << ": " << delay); ProfileEvents::increment(ProfileEvents::DistributedConnectionStaleReplica); } } diff --git a/src/Common/Config/ConfigProcessor.cpp b/src/Common/Config/ConfigProcessor.cpp index be104990187..0e8a67a156b 100644 --- a/src/Common/Config/ConfigProcessor.cpp +++ b/src/Common/Config/ConfigProcessor.cpp @@ -568,10 +568,7 @@ ConfigProcessor::LoadedConfig ConfigProcessor::loadConfigWithZooKeeperIncludes( if (!zk_exception) throw; - LOG_WARNING( - log, - "Error while processing from_zk config includes: " + zk_exception->message() + - ". Config will be loaded from preprocessed file: " + preprocessed_path); + LOG_WARNING( log, "Error while processing from_zk config includes: " + zk_exception->message() + ". Config will be loaded from preprocessed file: " + preprocessed_path); config_xml = dom_parser.parse(preprocessed_path); } diff --git a/src/Disks/S3/registerDiskS3.cpp b/src/Disks/S3/registerDiskS3.cpp index 5da407bf9f0..fce502ed110 100644 --- a/src/Disks/S3/registerDiskS3.cpp +++ b/src/Disks/S3/registerDiskS3.cpp @@ -46,8 +46,7 @@ namespace throw Exception("Only HTTP/HTTPS schemas allowed in proxy resolver config: " + proxy_scheme, ErrorCodes::BAD_ARGUMENTS); auto proxy_port = proxy_resolver_config.getUInt(prefix + ".proxy_port"); - LOG_DEBUG( - &Logger::get("DiskS3"), "Configured proxy resolver: " << endpoint.toString() << ", Scheme: " << proxy_scheme << ", Port: " << proxy_port); + LOG_DEBUG( &Logger::get("DiskS3"), "Configured proxy resolver: " << endpoint.toString() << ", Scheme: " << proxy_scheme << ", Port: " << proxy_port); return std::make_shared(endpoint, proxy_scheme, proxy_port); } diff --git a/src/Disks/VolumeJBOD.cpp b/src/Disks/VolumeJBOD.cpp index 79320aee7f2..250c10ea3fd 100644 --- a/src/Disks/VolumeJBOD.cpp +++ b/src/Disks/VolumeJBOD.cpp @@ -48,18 +48,11 @@ VolumeJBOD::VolumeJBOD( max_data_part_size = static_cast(sum_size * ratio / disks.size()); for (size_t i = 0; i < disks.size(); ++i) if (sizes[i] < max_data_part_size) - LOG_WARNING( - logger, - "Disk " << backQuote(disks[i]->getName()) << " on volume " << backQuote(config_prefix) << " have not enough space (" - << formatReadableSizeWithBinarySuffix(sizes[i]) << ") for containing part the size of max_data_part_size (" - << formatReadableSizeWithBinarySuffix(max_data_part_size) << ")"); + LOG_WARNING( logger, "Disk " << backQuote(disks[i]->getName()) << " on volume " << backQuote(config_prefix) << " have not enough space (" << formatReadableSizeWithBinarySuffix(sizes[i]) << ") for containing part the size of max_data_part_size (" << formatReadableSizeWithBinarySuffix(max_data_part_size) << ")"); } static constexpr UInt64 MIN_PART_SIZE = 8u * 1024u * 1024u; if (max_data_part_size != 0 && max_data_part_size < MIN_PART_SIZE) - LOG_WARNING( - logger, - "Volume " << backQuote(name) << " max_data_part_size is too low (" << formatReadableSizeWithBinarySuffix(max_data_part_size) - << " < " << formatReadableSizeWithBinarySuffix(MIN_PART_SIZE) << ")"); + LOG_WARNING( logger, "Volume " << backQuote(name) << " max_data_part_size is too low (" << formatReadableSizeWithBinarySuffix(max_data_part_size) << " < " << formatReadableSizeWithBinarySuffix(MIN_PART_SIZE) << ")"); } DiskPtr VolumeJBOD::getNextDisk() diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 8830d60e0a0..9b24640895c 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -129,17 +129,13 @@ void WriteBufferFromS3::writePart(const String & data) auto outcome = client_ptr->UploadPart(req); - LOG_TRACE( - log, "Writing part. Bucket: " << bucket << ", Key: " << key << ", Upload_id: " << upload_id << ", Data size: " << data.size()); + LOG_TRACE( log, "Writing part. Bucket: " << bucket << ", Key: " << key << ", Upload_id: " << upload_id << ", Data size: " << data.size()); if (outcome.IsSuccess()) { auto etag = outcome.GetResult().GetETag(); part_tags.push_back(etag); - LOG_DEBUG( - log, - "Writing part finished. " - << "Total parts: " << part_tags.size() << ", Upload_id: " << upload_id << ", Etag: " << etag); + LOG_DEBUG( log, "Writing part finished. " << "Total parts: " << part_tags.size() << ", Upload_id: " << upload_id << ", Etag: " << etag); } else throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR); diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index e9668eb84c6..8b8ff7077dd 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -200,10 +200,7 @@ void SelectStreamFactory::createForShard( ProfileEvents::increment(ProfileEvents::DistributedConnectionMissingTable); if (shard_info.hasRemoteConnections()) { - LOG_WARNING( - &Logger::get("ClusterProxy::SelectStreamFactory"), - "There is no table " << main_table.getNameForLogs() - << " on local replica of shard " << shard_info.shard_num << ", will try remote replicas."); + LOG_WARNING( &Logger::get("ClusterProxy::SelectStreamFactory"), "There is no table " << main_table.getNameForLogs() << " on local replica of shard " << shard_info.shard_num << ", will try remote replicas."); emplace_remote_stream(); } else @@ -239,9 +236,7 @@ void SelectStreamFactory::createForShard( /// If we reached this point, local replica is stale. ProfileEvents::increment(ProfileEvents::DistributedConnectionStaleReplica); - LOG_WARNING( - &Logger::get("ClusterProxy::SelectStreamFactory"), - "Local replica of shard " << shard_info.shard_num << " is stale (delay: " << local_delay << "s.)"); + LOG_WARNING( &Logger::get("ClusterProxy::SelectStreamFactory"), "Local replica of shard " << shard_info.shard_num << " is stale (delay: " << local_delay << "s.)"); if (!settings.fallback_to_stale_replicas_for_distributed_queries) { @@ -289,9 +284,7 @@ void SelectStreamFactory::createForShard( catch (const Exception & ex) { if (ex.code() == ErrorCodes::ALL_CONNECTION_TRIES_FAILED) - LOG_WARNING( - &Logger::get("ClusterProxy::SelectStreamFactory"), - "Connections to remote replicas of local shard " << shard_num << " failed, will use stale local replica"); + LOG_WARNING( &Logger::get("ClusterProxy::SelectStreamFactory"), "Connections to remote replicas of local shard " << shard_num << " failed, will use stale local replica"); else throw; } diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp index 2795b98dcc2..ac9742b42a1 100644 --- a/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/src/Storages/Distributed/DirectoryMonitor.cpp @@ -317,10 +317,7 @@ void StorageDistributedDirectoryMonitor::readHeader( readVarUInt(initiator_revision, header_buf); if (ClickHouseRevision::get() < initiator_revision) { - LOG_WARNING( - log, - "ClickHouse shard version is older than ClickHouse initiator version. " - << "It may lack support for new features."); + LOG_WARNING( log, "ClickHouse shard version is older than ClickHouse initiator version. " << "It may lack support for new features."); } readStringBinary(insert_query, header_buf); From 86b4d5a86f0e18c7f1b9157c003384f2058476df Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 22:09:06 +0300 Subject: [PATCH 643/738] Remove duplicate whitespaces (preparation) --- programs/server/Server.cpp | 2 +- src/Client/ConnectionPoolWithFailover.cpp | 2 +- src/Disks/S3/registerDiskS3.cpp | 2 +- src/IO/WriteBufferFromS3.cpp | 4 ++-- src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index c352a6ce8ed..fa24b106a09 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -981,7 +981,7 @@ int Server::main(const std::vector & /*args*/) } } - LOG_INFO( log, "Closed connections." << (current_connections ? " But " + toString(current_connections) + " remains." " Tip: To increase wait time add to config: 60" : "")); + LOG_INFO(log, "Closed connections." << (current_connections ? " But " + toString(current_connections) + " remains." " Tip: To increase wait time add to config: 60" : "")); dns_cache_updater.reset(); main_config_reloader.reset(); diff --git a/src/Client/ConnectionPoolWithFailover.cpp b/src/Client/ConnectionPoolWithFailover.cpp index c40ea290f00..fc47138e4a3 100644 --- a/src/Client/ConnectionPoolWithFailover.cpp +++ b/src/Client/ConnectionPoolWithFailover.cpp @@ -248,7 +248,7 @@ ConnectionPoolWithFailover::tryGetEntry( result.is_up_to_date = false; result.staleness = delay; - LOG_TRACE( log, "Server " << result.entry->getDescription() << " has unacceptable replica delay " << "for table " << table_to_check->database << "." << table_to_check->table << ": " << delay); + LOG_TRACE(log, "Server " << result.entry->getDescription() << " has unacceptable replica delay " << "for table " << table_to_check->database << "." << table_to_check->table << ": " << delay); ProfileEvents::increment(ProfileEvents::DistributedConnectionStaleReplica); } } diff --git a/src/Disks/S3/registerDiskS3.cpp b/src/Disks/S3/registerDiskS3.cpp index fce502ed110..b711f7360c8 100644 --- a/src/Disks/S3/registerDiskS3.cpp +++ b/src/Disks/S3/registerDiskS3.cpp @@ -46,7 +46,7 @@ namespace throw Exception("Only HTTP/HTTPS schemas allowed in proxy resolver config: " + proxy_scheme, ErrorCodes::BAD_ARGUMENTS); auto proxy_port = proxy_resolver_config.getUInt(prefix + ".proxy_port"); - LOG_DEBUG( &Logger::get("DiskS3"), "Configured proxy resolver: " << endpoint.toString() << ", Scheme: " << proxy_scheme << ", Port: " << proxy_port); + LOG_DEBUG(&Logger::get("DiskS3"), "Configured proxy resolver: " << endpoint.toString() << ", Scheme: " << proxy_scheme << ", Port: " << proxy_port); return std::make_shared(endpoint, proxy_scheme, proxy_port); } diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 9b24640895c..21db726d074 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -129,13 +129,13 @@ void WriteBufferFromS3::writePart(const String & data) auto outcome = client_ptr->UploadPart(req); - LOG_TRACE( log, "Writing part. Bucket: " << bucket << ", Key: " << key << ", Upload_id: " << upload_id << ", Data size: " << data.size()); + LOG_TRACE(log, "Writing part. Bucket: " << bucket << ", Key: " << key << ", Upload_id: " << upload_id << ", Data size: " << data.size()); if (outcome.IsSuccess()) { auto etag = outcome.GetResult().GetETag(); part_tags.push_back(etag); - LOG_DEBUG( log, "Writing part finished. " << "Total parts: " << part_tags.size() << ", Upload_id: " << upload_id << ", Etag: " << etag); + LOG_DEBUG(log, "Writing part finished. " << "Total parts: " << part_tags.size() << ", Upload_id: " << upload_id << ", Etag: " << etag); } else throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR); diff --git a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp index 28e6a22977a..45d5edce480 100644 --- a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp +++ b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp @@ -104,11 +104,11 @@ void ReadBufferFromKafkaConsumer::commit() if (topic_part.get_offset() < 0) { - LOG_TRACE( log, prefix << " " << print_special_offset() << " (topic: " << topic_part.get_topic() << ", partition: " << topic_part.get_partition() << ")"); + LOG_TRACE(log, prefix << " " << print_special_offset() << " (topic: " << topic_part.get_topic() << ", partition: " << topic_part.get_partition() << ")"); } else { - LOG_TRACE( log, prefix << " " << topic_part.get_offset() << " (topic: " << topic_part.get_topic() << ", partition: " << topic_part.get_partition() << ")"); + LOG_TRACE(log, prefix << " " << topic_part.get_offset() << " (topic: " << topic_part.get_topic() << ", partition: " << topic_part.get_partition() << ")"); } } }; From 7fed65cbe2718c22ca157eb1a63f577e8ca22715 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 22:14:58 +0300 Subject: [PATCH 644/738] Remove duplicate whitespaces (preparation) --- src/Common/Config/ConfigProcessor.cpp | 2 +- src/Disks/VolumeJBOD.cpp | 4 ++-- src/Interpreters/ClusterProxy/SelectStreamFactory.cpp | 6 +++--- src/Interpreters/ExternalLoader.cpp | 2 +- src/Storages/Distributed/DirectoryMonitor.cpp | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Common/Config/ConfigProcessor.cpp b/src/Common/Config/ConfigProcessor.cpp index 0e8a67a156b..5e9f7f066ca 100644 --- a/src/Common/Config/ConfigProcessor.cpp +++ b/src/Common/Config/ConfigProcessor.cpp @@ -568,7 +568,7 @@ ConfigProcessor::LoadedConfig ConfigProcessor::loadConfigWithZooKeeperIncludes( if (!zk_exception) throw; - LOG_WARNING( log, "Error while processing from_zk config includes: " + zk_exception->message() + ". Config will be loaded from preprocessed file: " + preprocessed_path); + LOG_WARNING(log, "Error while processing from_zk config includes: " + zk_exception->message() + ". Config will be loaded from preprocessed file: " + preprocessed_path); config_xml = dom_parser.parse(preprocessed_path); } diff --git a/src/Disks/VolumeJBOD.cpp b/src/Disks/VolumeJBOD.cpp index 250c10ea3fd..59ab3087499 100644 --- a/src/Disks/VolumeJBOD.cpp +++ b/src/Disks/VolumeJBOD.cpp @@ -48,11 +48,11 @@ VolumeJBOD::VolumeJBOD( max_data_part_size = static_cast(sum_size * ratio / disks.size()); for (size_t i = 0; i < disks.size(); ++i) if (sizes[i] < max_data_part_size) - LOG_WARNING( logger, "Disk " << backQuote(disks[i]->getName()) << " on volume " << backQuote(config_prefix) << " have not enough space (" << formatReadableSizeWithBinarySuffix(sizes[i]) << ") for containing part the size of max_data_part_size (" << formatReadableSizeWithBinarySuffix(max_data_part_size) << ")"); + LOG_WARNING(logger, "Disk " << backQuote(disks[i]->getName()) << " on volume " << backQuote(config_prefix) << " have not enough space (" << formatReadableSizeWithBinarySuffix(sizes[i]) << ") for containing part the size of max_data_part_size (" << formatReadableSizeWithBinarySuffix(max_data_part_size) << ")"); } static constexpr UInt64 MIN_PART_SIZE = 8u * 1024u * 1024u; if (max_data_part_size != 0 && max_data_part_size < MIN_PART_SIZE) - LOG_WARNING( logger, "Volume " << backQuote(name) << " max_data_part_size is too low (" << formatReadableSizeWithBinarySuffix(max_data_part_size) << " < " << formatReadableSizeWithBinarySuffix(MIN_PART_SIZE) << ")"); + LOG_WARNING(logger, "Volume " << backQuote(name) << " max_data_part_size is too low (" << formatReadableSizeWithBinarySuffix(max_data_part_size) << " < " << formatReadableSizeWithBinarySuffix(MIN_PART_SIZE) << ")"); } DiskPtr VolumeJBOD::getNextDisk() diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index 8b8ff7077dd..5eceac625a2 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -200,7 +200,7 @@ void SelectStreamFactory::createForShard( ProfileEvents::increment(ProfileEvents::DistributedConnectionMissingTable); if (shard_info.hasRemoteConnections()) { - LOG_WARNING( &Logger::get("ClusterProxy::SelectStreamFactory"), "There is no table " << main_table.getNameForLogs() << " on local replica of shard " << shard_info.shard_num << ", will try remote replicas."); + LOG_WARNING(&Logger::get("ClusterProxy::SelectStreamFactory"), "There is no table " << main_table.getNameForLogs() << " on local replica of shard " << shard_info.shard_num << ", will try remote replicas."); emplace_remote_stream(); } else @@ -236,7 +236,7 @@ void SelectStreamFactory::createForShard( /// If we reached this point, local replica is stale. ProfileEvents::increment(ProfileEvents::DistributedConnectionStaleReplica); - LOG_WARNING( &Logger::get("ClusterProxy::SelectStreamFactory"), "Local replica of shard " << shard_info.shard_num << " is stale (delay: " << local_delay << "s.)"); + LOG_WARNING(&Logger::get("ClusterProxy::SelectStreamFactory"), "Local replica of shard " << shard_info.shard_num << " is stale (delay: " << local_delay << "s.)"); if (!settings.fallback_to_stale_replicas_for_distributed_queries) { @@ -284,7 +284,7 @@ void SelectStreamFactory::createForShard( catch (const Exception & ex) { if (ex.code() == ErrorCodes::ALL_CONNECTION_TRIES_FAILED) - LOG_WARNING( &Logger::get("ClusterProxy::SelectStreamFactory"), "Connections to remote replicas of local shard " << shard_num << " failed, will use stale local replica"); + LOG_WARNING(&Logger::get("ClusterProxy::SelectStreamFactory"), "Connections to remote replicas of local shard " << shard_num << " failed, will use stale local replica"); else throw; } diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index 29d645e89d9..721f4812afe 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -350,7 +350,7 @@ private: const auto & already_added = already_added_it->second; if (!already_added->from_temp_repository && !repository->isTemporary()) { - LOG_WARNING( log, type_name << " '" << object_name << "' is found " << (((path == already_added->path) && (repository->getName() == already_added->repository_name)) ? ("twice in the same file '" + path + "'") : ("both in file '" + already_added->path + "' and '" + path + "'"))); + LOG_WARNING(log, type_name << " '" << object_name << "' is found " << (((path == already_added->path) && (repository->getName() == already_added->repository_name)) ? ("twice in the same file '" + path + "'") : ("both in file '" + already_added->path + "' and '" + path + "'"))); } } } diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp index ac9742b42a1..b6742157335 100644 --- a/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/src/Storages/Distributed/DirectoryMonitor.cpp @@ -317,7 +317,7 @@ void StorageDistributedDirectoryMonitor::readHeader( readVarUInt(initiator_revision, header_buf); if (ClickHouseRevision::get() < initiator_revision) { - LOG_WARNING( log, "ClickHouse shard version is older than ClickHouse initiator version. " << "It may lack support for new features."); + LOG_WARNING(log, "ClickHouse shard version is older than ClickHouse initiator version. " << "It may lack support for new features."); } readStringBinary(insert_query, header_buf); From 29762240de60e9f2cb2b547b8b211c0500913429 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 22:31:54 +0300 Subject: [PATCH 645/738] Remove duplicate whitespaces (preparation) --- base/daemon/BaseDaemon.cpp | 2 +- base/daemon/GraphiteWriter.h | 2 +- programs/copier/ClusterCopier.cpp | 4 ++-- programs/server/Server.cpp | 2 +- src/Common/Config/ConfigProcessor.cpp | 2 +- src/Common/PoolWithFailoverBase.h | 2 +- .../CollapsingFinalBlockInputStream.cpp | 2 +- src/Databases/DatabaseOnDisk.cpp | 2 +- src/Disks/VolumeJBOD.cpp | 4 ++-- .../ClusterProxy/SelectStreamFactory.cpp | 6 +++--- src/Interpreters/DDLWorker.cpp | 10 +++++----- src/Interpreters/DatabaseCatalog.cpp | 2 +- src/Interpreters/ExternalLoader.cpp | 2 +- src/Storages/Distributed/DirectoryMonitor.cpp | 2 +- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 6 +++--- src/Storages/MergeTree/MergeTreeData.cpp | 10 +++++----- .../MergeTree/MergeTreeDataMergerMutator.cpp | 4 ++-- .../ReplicatedMergeTreeCleanupThread.cpp | 2 +- .../ReplicatedMergeTreePartCheckThread.cpp | 4 ++-- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 6 +++--- .../ReplicatedMergeTreeRestartingThread.cpp | 2 +- src/Storages/StorageBuffer.cpp | 12 ++++++------ src/Storages/StorageReplicatedMergeTree.cpp | 18 +++++++++--------- 23 files changed, 54 insertions(+), 54 deletions(-) diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index 818721b9e6d..d5b7d1a0f6c 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -498,7 +498,7 @@ void debugIncreaseOOMScore() } catch (const Poco::Exception & e) { - LOG_WARNING(&Logger::root(), "Failed to adjust OOM score: '" + e.displayText() + "'."); + LOG_WARNING(&Logger::root(), "Failed to adjust OOM score: '" + e.displayText() + "'."); return; } LOG_INFO_FORMATTED(&Logger::root(), "Set OOM score adjustment to {}", new_score); diff --git a/base/daemon/GraphiteWriter.h b/base/daemon/GraphiteWriter.h index d368470e5e6..400c99066b2 100644 --- a/base/daemon/GraphiteWriter.h +++ b/base/daemon/GraphiteWriter.h @@ -52,7 +52,7 @@ private: } catch (const Poco::Exception & e) { - LOG_WARNING(&Poco::Util::Application::instance().logger(), "Fail to write to Graphite " << host << ":" << port << ". e.what() = " << e.what() << ", e.message() = " << e.message()); + LOG_WARNING(&Poco::Util::Application::instance().logger(), "Fail to write to Graphite " << host << ":" << port << ". e.what() = " << e.what() << ", e.message() = " << e.message()); } } diff --git a/programs/copier/ClusterCopier.cpp b/programs/copier/ClusterCopier.cpp index 50b32676bce..eb35768b955 100644 --- a/programs/copier/ClusterCopier.cpp +++ b/programs/copier/ClusterCopier.cpp @@ -164,7 +164,7 @@ void ClusterCopier::discoverShardPartitions(const ConnectionTimeouts & timeouts, for (const String & missing_partition : missing_partitions) ss << " " << missing_partition; - LOG_WARNING(log, "There are no " << missing_partitions.size() << " partitions from enabled_partitions in shard " << task_shard->getDescription() << " :" << ss.str()); + LOG_WARNING(log, "There are no " << missing_partitions.size() << " partitions from enabled_partitions in shard " << task_shard->getDescription() << " :" << ss.str()); } LOG_DEBUG_FORMATTED(log, "Will copy {} partitions from shard {}", task_shard->partition_tasks.size(), task_shard->getDescription()); @@ -1297,7 +1297,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( /// NOTE: partition is still fresh if dirt discovery happens before cleaning if (stat_shards.numChildren == 0) { - LOG_WARNING(log, "There are no workers for partition " << task_partition.name << " piece " << toString(current_piece_number) << ", but destination table contains " << count << " rows" << ". Partition will be dropped and refilled."); + LOG_WARNING(log, "There are no workers for partition " << task_partition.name << " piece " << toString(current_piece_number) << ", but destination table contains " << count << " rows" << ". Partition will be dropped and refilled."); create_is_dirty_node(clean_state_clock); return TaskStatus::Error; diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index fa24b106a09..78ca90a4280 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -354,7 +354,7 @@ int Server::main(const std::vector & /*args*/) rlim.rlim_cur = config().getUInt("max_open_files", rlim.rlim_max); int rc = setrlimit(RLIMIT_NOFILE, &rlim); if (rc != 0) - LOG_WARNING(log, "Cannot set max number of file descriptors to " << rlim.rlim_cur << ". Try to specify max_open_files according to your system limits. error: " << strerror(errno)); + LOG_WARNING(log, "Cannot set max number of file descriptors to " << rlim.rlim_cur << ". Try to specify max_open_files according to your system limits. error: " << strerror(errno)); else LOG_DEBUG_FORMATTED(log, "Set max number of file descriptors to {} (was {}).", rlim.rlim_cur, old); } diff --git a/src/Common/Config/ConfigProcessor.cpp b/src/Common/Config/ConfigProcessor.cpp index 5e9f7f066ca..8ee26226405 100644 --- a/src/Common/Config/ConfigProcessor.cpp +++ b/src/Common/Config/ConfigProcessor.cpp @@ -568,7 +568,7 @@ ConfigProcessor::LoadedConfig ConfigProcessor::loadConfigWithZooKeeperIncludes( if (!zk_exception) throw; - LOG_WARNING(log, "Error while processing from_zk config includes: " + zk_exception->message() + ". Config will be loaded from preprocessed file: " + preprocessed_path); + LOG_WARNING(log, "Error while processing from_zk config includes: " + zk_exception->message() + ". Config will be loaded from preprocessed file: " + preprocessed_path); config_xml = dom_parser.parse(preprocessed_path); } diff --git a/src/Common/PoolWithFailoverBase.h b/src/Common/PoolWithFailoverBase.h index b83648f6d0e..10ab3885be6 100644 --- a/src/Common/PoolWithFailoverBase.h +++ b/src/Common/PoolWithFailoverBase.h @@ -239,7 +239,7 @@ PoolWithFailoverBase::getMany( } else { - LOG_WARNING(log, "Connection failed at try №" << (shuffled_pool.error_count + 1) << ", reason: " << fail_message); + LOG_WARNING(log, "Connection failed at try №" << (shuffled_pool.error_count + 1) << ", reason: " << fail_message); ProfileEvents::increment(ProfileEvents::DistributedConnectionFailTry); shuffled_pool.error_count = std::min(max_error_cap, shuffled_pool.error_count + 1); diff --git a/src/DataStreams/CollapsingFinalBlockInputStream.cpp b/src/DataStreams/CollapsingFinalBlockInputStream.cpp index 2ac1c46f0bc..73d3bda5860 100644 --- a/src/DataStreams/CollapsingFinalBlockInputStream.cpp +++ b/src/DataStreams/CollapsingFinalBlockInputStream.cpp @@ -18,7 +18,7 @@ CollapsingFinalBlockInputStream::~CollapsingFinalBlockInputStream() void CollapsingFinalBlockInputStream::reportBadCounts() { /// With inconsistent data, this is an unavoidable error that can not be easily fixed by admins. Therefore Warning. - LOG_WARNING(log, "Incorrect data: number of rows with sign = 1 (" << count_positive << ") differs with number of rows with sign = -1 (" << count_negative << ") by more than one"); + LOG_WARNING(log, "Incorrect data: number of rows with sign = 1 (" << count_positive << ") differs with number of rows with sign = -1 (" << count_negative << ") by more than one"); } void CollapsingFinalBlockInputStream::reportBadSign(Int8 sign) diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index 3d563b09e0d..27d0d9a7a10 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -466,7 +466,7 @@ ASTPtr DatabaseOnDisk::parseQueryFromMetadata(Poco::Logger * loger, const Contex table_name = unescapeForFileName(table_name); if (create.table != TABLE_WITH_UUID_NAME_PLACEHOLDER) - LOG_WARNING(loger, "File " << metadata_file_path << " contains both UUID and table name. " "Will use name `" << table_name << "` instead of `" << create.table << "`"); + LOG_WARNING(loger, "File " << metadata_file_path << " contains both UUID and table name. " "Will use name `" << table_name << "` instead of `" << create.table << "`"); create.table = table_name; } diff --git a/src/Disks/VolumeJBOD.cpp b/src/Disks/VolumeJBOD.cpp index 59ab3087499..986d69111b3 100644 --- a/src/Disks/VolumeJBOD.cpp +++ b/src/Disks/VolumeJBOD.cpp @@ -48,11 +48,11 @@ VolumeJBOD::VolumeJBOD( max_data_part_size = static_cast(sum_size * ratio / disks.size()); for (size_t i = 0; i < disks.size(); ++i) if (sizes[i] < max_data_part_size) - LOG_WARNING(logger, "Disk " << backQuote(disks[i]->getName()) << " on volume " << backQuote(config_prefix) << " have not enough space (" << formatReadableSizeWithBinarySuffix(sizes[i]) << ") for containing part the size of max_data_part_size (" << formatReadableSizeWithBinarySuffix(max_data_part_size) << ")"); + LOG_WARNING(logger, "Disk " << backQuote(disks[i]->getName()) << " on volume " << backQuote(config_prefix) << " have not enough space (" << formatReadableSizeWithBinarySuffix(sizes[i]) << ") for containing part the size of max_data_part_size (" << formatReadableSizeWithBinarySuffix(max_data_part_size) << ")"); } static constexpr UInt64 MIN_PART_SIZE = 8u * 1024u * 1024u; if (max_data_part_size != 0 && max_data_part_size < MIN_PART_SIZE) - LOG_WARNING(logger, "Volume " << backQuote(name) << " max_data_part_size is too low (" << formatReadableSizeWithBinarySuffix(max_data_part_size) << " < " << formatReadableSizeWithBinarySuffix(MIN_PART_SIZE) << ")"); + LOG_WARNING(logger, "Volume " << backQuote(name) << " max_data_part_size is too low (" << formatReadableSizeWithBinarySuffix(max_data_part_size) << " < " << formatReadableSizeWithBinarySuffix(MIN_PART_SIZE) << ")"); } DiskPtr VolumeJBOD::getNextDisk() diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index 5eceac625a2..00c1356d002 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -200,7 +200,7 @@ void SelectStreamFactory::createForShard( ProfileEvents::increment(ProfileEvents::DistributedConnectionMissingTable); if (shard_info.hasRemoteConnections()) { - LOG_WARNING(&Logger::get("ClusterProxy::SelectStreamFactory"), "There is no table " << main_table.getNameForLogs() << " on local replica of shard " << shard_info.shard_num << ", will try remote replicas."); + LOG_WARNING(&Logger::get("ClusterProxy::SelectStreamFactory"), "There is no table " << main_table.getNameForLogs() << " on local replica of shard " << shard_info.shard_num << ", will try remote replicas."); emplace_remote_stream(); } else @@ -236,7 +236,7 @@ void SelectStreamFactory::createForShard( /// If we reached this point, local replica is stale. ProfileEvents::increment(ProfileEvents::DistributedConnectionStaleReplica); - LOG_WARNING(&Logger::get("ClusterProxy::SelectStreamFactory"), "Local replica of shard " << shard_info.shard_num << " is stale (delay: " << local_delay << "s.)"); + LOG_WARNING(&Logger::get("ClusterProxy::SelectStreamFactory"), "Local replica of shard " << shard_info.shard_num << " is stale (delay: " << local_delay << "s.)"); if (!settings.fallback_to_stale_replicas_for_distributed_queries) { @@ -284,7 +284,7 @@ void SelectStreamFactory::createForShard( catch (const Exception & ex) { if (ex.code() == ErrorCodes::ALL_CONNECTION_TRIES_FAILED) - LOG_WARNING(&Logger::get("ClusterProxy::SelectStreamFactory"), "Connections to remote replicas of local shard " << shard_num << " failed, will use stale local replica"); + LOG_WARNING(&Logger::get("ClusterProxy::SelectStreamFactory"), "Connections to remote replicas of local shard " << shard_num << " failed, will use stale local replica"); else throw; } diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index 53be380e029..0ba1c184545 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -238,7 +238,7 @@ DDLWorker::DDLWorker(const std::string & zk_root_dir, Context & context_, const if (context.getSettingsRef().readonly) { - LOG_WARNING(log, "Distributed DDL worker is run with readonly settings, it will not be able to execute DDL queries" << " Set appropriate system_profile or distributed_ddl.profile to fix this."); + LOG_WARNING(log, "Distributed DDL worker is run with readonly settings, it will not be able to execute DDL queries" << " Set appropriate system_profile or distributed_ddl.profile to fix this."); } host_fqdn = getFQDNOrHostName(); @@ -334,7 +334,7 @@ bool DDLWorker::initAndCheckTask(const String & entry_name, String & out_reason, if (host_in_hostlist) { /// This check could be slow a little bit - LOG_WARNING(log, "There are two the same ClickHouse instances in task " << entry_name << ": " << task->host_id.readableString() << " and " << host.readableString() << ". Will use the first one only."); + LOG_WARNING(log, "There are two the same ClickHouse instances in task " << entry_name << ": " << task->host_id.readableString() << " and " << host.readableString() << ". Will use the first one only."); } else { @@ -424,7 +424,7 @@ void DDLWorker::processTasks() { if (server_startup && e.code == Coordination::ZNONODE) { - LOG_WARNING(log, "ZooKeeper NONODE error during startup. Ignoring entry " << task.entry_name << " (" << task.entry.query << ") : " << getCurrentExceptionMessage(true)); + LOG_WARNING(log, "ZooKeeper NONODE error during startup. Ignoring entry " << task.entry_name << " (" << task.entry.query << ") : " << getCurrentExceptionMessage(true)); } else { @@ -433,7 +433,7 @@ void DDLWorker::processTasks() } catch (...) { - LOG_WARNING(log, "An error occurred while processing task " << task.entry_name << " (" << task.entry.query << ") : " << getCurrentExceptionMessage(true)); + LOG_WARNING(log, "An error occurred while processing task " << task.entry_name << " (" << task.entry.query << ") : " << getCurrentExceptionMessage(true)); throw; } } @@ -506,7 +506,7 @@ void DDLWorker::parseQueryAndResolveHost(DDLTask & task) if (found_exact_match) return; - LOG_WARNING(log, "Not found the exact match of host " << task.host_id.readableString() << " from task " << task.entry_name << " in cluster " << task.cluster_name << " definition. Will try to find it using host name resolving."); + LOG_WARNING(log, "Not found the exact match of host " << task.host_id.readableString() << " from task " << task.entry_name << " in cluster " << task.cluster_name << " definition. Will try to find it using host name resolving."); bool found_via_resolving = false; for (size_t shard_num = 0; shard_num < shards.size(); ++shard_num) diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 40ae756f48d..6c1d00df3bc 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -597,7 +597,7 @@ void DatabaseCatalog::enqueueDroppedTableCleanup(StorageID table_id, StoragePtr } else { - LOG_WARNING(log, "Cannot parse metadata of partially dropped table " << table_id.getNameForLogs() << " from " << dropped_metadata_path << ". Will remove metadata file and data directory. Garbage may be left in /store directory and ZooKeeper."); + LOG_WARNING(log, "Cannot parse metadata of partially dropped table " << table_id.getNameForLogs() << " from " << dropped_metadata_path << ". Will remove metadata file and data directory. Garbage may be left in /store directory and ZooKeeper."); } drop_time = Poco::File(dropped_metadata_path).getLastModified().epochTime(); diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index 721f4812afe..7157fcd58e9 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -350,7 +350,7 @@ private: const auto & already_added = already_added_it->second; if (!already_added->from_temp_repository && !repository->isTemporary()) { - LOG_WARNING(log, type_name << " '" << object_name << "' is found " << (((path == already_added->path) && (repository->getName() == already_added->repository_name)) ? ("twice in the same file '" + path + "'") : ("both in file '" + already_added->path + "' and '" + path + "'"))); + LOG_WARNING(log, type_name << " '" << object_name << "' is found " << (((path == already_added->path) && (repository->getName() == already_added->repository_name)) ? ("twice in the same file '" + path + "'") : ("both in file '" + already_added->path + "' and '" + path + "'"))); } } } diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp index b6742157335..30879433024 100644 --- a/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/src/Storages/Distributed/DirectoryMonitor.cpp @@ -317,7 +317,7 @@ void StorageDistributedDirectoryMonitor::readHeader( readVarUInt(initiator_revision, header_buf); if (ClickHouseRevision::get() < initiator_revision) { - LOG_WARNING(log, "ClickHouse shard version is older than ClickHouse initiator version. " << "It may lack support for new features."); + LOG_WARNING(log, "ClickHouse shard version is older than ClickHouse initiator version. " << "It may lack support for new features."); } readStringBinary(insert_query, header_buf); diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 0ea1dfddabc..494e5809683 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -668,7 +668,7 @@ void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_ Names files; volume->getDisk()->listFiles(to, files); - LOG_WARNING(storage.log, "Part directory " << fullPath(volume->getDisk(), to) << " already exists" << " and contains " << files.size() << " files. Removing it."); + LOG_WARNING(storage.log, "Part directory " << fullPath(volume->getDisk(), to) << " already exists" << " and contains " << files.size() << " files. Removing it."); volume->getDisk()->removeRecursive(to); } @@ -710,7 +710,7 @@ void IMergeTreeDataPart::remove() const if (volume->getDisk()->exists(to)) { - LOG_WARNING(storage.log, "Directory " << fullPath(volume->getDisk(), to) << " (to which part must be renamed before removing) already exists." " Most likely this is due to unclean restart. Removing it."); + LOG_WARNING(storage.log, "Directory " << fullPath(volume->getDisk(), to) << " (to which part must be renamed before removing) already exists." " Most likely this is due to unclean restart. Removing it."); try { @@ -791,7 +791,7 @@ String IMergeTreeDataPart::getRelativePathForDetachedPart(const String & prefix) if (!volume->getDisk()->exists(getFullRelativePath() + res)) return res; - LOG_WARNING(storage.log, "Directory " << res << " (to detach to) already exists." " Will detach to directory with '_tryN' suffix."); + LOG_WARNING(storage.log, "Directory " << res << " (to detach to) already exists." " Will detach to directory with '_tryN' suffix."); } return res; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 3e57a48b135..c35cb29ca62 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -2914,9 +2914,9 @@ ReservationPtr MergeTreeData::tryReserveSpacePreferringTTLRules(UInt64 expected_ if (!destination_ptr) { if (ttl_entry->destination_type == PartDestinationType::VOLUME) - LOG_WARNING(log, "Would like to reserve space on volume '" << ttl_entry->destination_name << "' by TTL rule of table '" << log_name << "' but volume was not found"); + LOG_WARNING(log, "Would like to reserve space on volume '" << ttl_entry->destination_name << "' by TTL rule of table '" << log_name << "' but volume was not found"); else if (ttl_entry->destination_type == PartDestinationType::DISK) - LOG_WARNING(log, "Would like to reserve space on disk '" << ttl_entry->destination_name << "' by TTL rule of table '" << log_name << "' but disk was not found"); + LOG_WARNING(log, "Would like to reserve space on disk '" << ttl_entry->destination_name << "' by TTL rule of table '" << log_name << "' but disk was not found"); } else { @@ -2925,9 +2925,9 @@ ReservationPtr MergeTreeData::tryReserveSpacePreferringTTLRules(UInt64 expected_ return reservation; else if (ttl_entry->destination_type == PartDestinationType::VOLUME) - LOG_WARNING(log, "Would like to reserve space on volume '" << ttl_entry->destination_name << "' by TTL rule of table '" << log_name << "' but there is not enough space"); + LOG_WARNING(log, "Would like to reserve space on volume '" << ttl_entry->destination_name << "' by TTL rule of table '" << log_name << "' but there is not enough space"); else if (ttl_entry->destination_type == PartDestinationType::DISK) - LOG_WARNING(log, "Would like to reserve space on disk '" << ttl_entry->destination_name << "' by TTL rule of table '" << log_name << "' but there is not enough space"); + LOG_WARNING(log, "Would like to reserve space on disk '" << ttl_entry->destination_name << "' by TTL rule of table '" << log_name << "' but there is not enough space"); } } @@ -3053,7 +3053,7 @@ MergeTreeData::DataPartsVector MergeTreeData::Transaction::commit(MergeTreeData: DataPartsVector covered_parts = data.getActivePartsToReplace(part->info, part->name, covering_part, *owing_parts_lock); if (covering_part) { - LOG_WARNING(data.log, "Tried to commit obsolete part " << part->name << " covered by " << covering_part->getNameWithState()); + LOG_WARNING(data.log, "Tried to commit obsolete part " << part->name << " covered by " << covering_part->getNameWithState()); part->remove_time.store(0, std::memory_order_relaxed); /// The part will be removed without waiting for old_parts_lifetime seconds. data.modifyPartState(part, DataPartState::Outdated); diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index f56ef85015a..8eee53cde4a 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -366,7 +366,7 @@ bool MergeTreeDataMergerMutator::selectAllPartsToMergeWithinPartition( if (now - disk_space_warning_time > 3600) { disk_space_warning_time = now; - LOG_WARNING(log, "Won't merge parts from " << parts.front()->name << " to " << (*prev_it)->name << " because not enough free space: " << formatReadableSizeWithBinarySuffix(available_disk_space) << " free and unreserved, " << formatReadableSizeWithBinarySuffix(sum_bytes) << " required now (+" << static_cast((DISK_USAGE_COEFFICIENT_TO_SELECT - 1.0) * 100) << "% on overhead); suppressing similar warnings for the next hour"); + LOG_WARNING(log, "Won't merge parts from " << parts.front()->name << " to " << (*prev_it)->name << " because not enough free space: " << formatReadableSizeWithBinarySuffix(available_disk_space) << " free and unreserved, " << formatReadableSizeWithBinarySuffix(sum_bytes) << " required now (+" << static_cast((DISK_USAGE_COEFFICIENT_TO_SELECT - 1.0) * 100) << "% on overhead); suppressing similar warnings for the next hour"); } if (out_disable_reason) @@ -1214,7 +1214,7 @@ MergeTreeData::DataPartPtr MergeTreeDataMergerMutator::renameMergedTemporaryPart * (NOTE: Merging with part that is not in ZK is not possible, see checks in 'createLogEntryToMergeParts'.) * - and after merge, this part will be removed in addition to parts that was merged. */ - LOG_WARNING(log, "Unexpected number of parts removed when adding " << new_data_part->name << ": " << replaced_parts.size() << " instead of " << parts.size()); + LOG_WARNING(log, "Unexpected number of parts removed when adding " << new_data_part->name << ": " << replaced_parts.size() << " instead of " << parts.size()); } else { diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp index 80577bd1b20..17dc1a508d2 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp @@ -327,7 +327,7 @@ void ReplicatedMergeTreeCleanupThread::clearOldBlocks() cached_block_stats.erase(first_outdated_block->node); } else if (rc) - LOG_WARNING(log, "Error while deleting ZooKeeper path `" << path << "`: " + zkutil::ZooKeeper::error2string(rc) << ", ignoring."); + LOG_WARNING(log, "Error while deleting ZooKeeper path `" << path << "`: " + zkutil::ZooKeeper::error2string(rc) << ", ignoring."); else { /// Successfully removed blocks have to be removed from cache diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp index 383a1e5e485..8ec073282c9 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp @@ -82,7 +82,7 @@ void ReplicatedMergeTreePartCheckThread::searchForMissingPart(const String & par /// If the part is in ZooKeeper, remove it from there and add the task to download it to the queue. if (zookeeper->exists(part_path)) { - LOG_WARNING(log, "Part " << part_name << " exists in ZooKeeper but not locally. " "Removing from ZooKeeper and queueing a fetch."); + LOG_WARNING(log, "Part " << part_name << " exists in ZooKeeper but not locally. " "Removing from ZooKeeper and queueing a fetch."); ProfileEvents::increment(ProfileEvents::ReplicatedPartChecksFailed); storage.removePartAndEnqueueFetch(part_name); @@ -143,7 +143,7 @@ void ReplicatedMergeTreePartCheckThread::searchForMissingPart(const String & par if (found_part_with_the_same_min_block && found_part_with_the_same_max_block) { - LOG_WARNING(log, "Found parts with the same min block and with the same max block as the missing part " << part_name << ". Hoping that it will eventually appear as a result of a merge."); + LOG_WARNING(log, "Found parts with the same min block and with the same max block as the missing part " << part_name << ". Hoping that it will eventually appear as a result of a merge."); return; } } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 027e33a1373..9ee4f7d3f9a 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -1328,7 +1328,7 @@ MutationCommands ReplicatedMergeTreeQueue::getMutationCommands( if (part->info.getDataVersion() > desired_mutation_version) { - LOG_WARNING(log, "Data version of part " << part->name << " is already greater than " "desired mutation version " << desired_mutation_version); + LOG_WARNING(log, "Data version of part " << part->name << " is already greater than " "desired mutation version " << desired_mutation_version); return MutationCommands{}; } @@ -1337,7 +1337,7 @@ MutationCommands ReplicatedMergeTreeQueue::getMutationCommands( auto in_partition = mutations_by_partition.find(part->info.partition_id); if (in_partition == mutations_by_partition.end()) { - LOG_WARNING(log, "There are no mutations for partition ID " << part->info.partition_id << " (trying to mutate part " << part->name << " to " << toString(desired_mutation_version) << ")"); + LOG_WARNING(log, "There are no mutations for partition ID " << part->info.partition_id << " (trying to mutate part " << part->name << " to " << toString(desired_mutation_version) << ")"); return MutationCommands{}; } @@ -1345,7 +1345,7 @@ MutationCommands ReplicatedMergeTreeQueue::getMutationCommands( auto end = in_partition->second.lower_bound(desired_mutation_version); if (end == in_partition->second.end() || end->first != desired_mutation_version) - LOG_WARNING(log, "Mutation with version " << desired_mutation_version << " not found in partition ID " << part->info.partition_id << " (trying to mutate part " << part->name + ")"); + LOG_WARNING(log, "Mutation with version " << desired_mutation_version << " not found in partition ID " << part->info.partition_id << " (trying to mutate part " << part->name + ")"); else ++end; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index c60c6953edf..519c063f088 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -268,7 +268,7 @@ void ReplicatedMergeTreeRestartingThread::updateQuorumIfWeHavePart() if (!quorum_entry.replicas.count(storage.replica_name) && zookeeper->exists(storage.replica_path + "/parts/" + quorum_entry.part_name)) { - LOG_WARNING(log, "We have part " << quorum_entry.part_name << " but we is not in quorum. Updating quorum. This shouldn't happen often."); + LOG_WARNING(log, "We have part " << quorum_entry.part_name << " but we is not in quorum. Updating quorum. This shouldn't happen often."); storage.updateQuorum(quorum_entry.part_name); } } diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index 82f781b3193..1bb30201e82 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -189,7 +189,7 @@ Pipes StorageBuffer::read( { if (!dest_columns.hasPhysical(column_name)) { - LOG_WARNING(log, "Destination table " << destination_id.getNameForLogs() << " doesn't have column " << backQuoteIfNeed(column_name) << ". The default values are used."); + LOG_WARNING(log, "Destination table " << destination_id.getNameForLogs() << " doesn't have column " << backQuoteIfNeed(column_name) << ". The default values are used."); boost::range::remove_erase(columns_intersection, column_name); continue; } @@ -197,14 +197,14 @@ Pipes StorageBuffer::read( const auto & col = our_columns.getPhysical(column_name); if (!dst_col.type->equals(*col.type)) { - LOG_WARNING(log, "Destination table " << destination_id.getNameForLogs() << " has different type of column " << backQuoteIfNeed(column_name) << " (" << dst_col.type->getName() << " != " << col.type->getName() << "). Data from destination table are converted."); + LOG_WARNING(log, "Destination table " << destination_id.getNameForLogs() << " has different type of column " << backQuoteIfNeed(column_name) << " (" << dst_col.type->getName() << " != " << col.type->getName() << "). Data from destination table are converted."); header_after_adding_defaults.getByName(column_name) = ColumnWithTypeAndName(dst_col.type, column_name); } } if (columns_intersection.empty()) { - LOG_WARNING(log, "Destination table " << destination_id.getNameForLogs() << " has no common columns with block in buffer. Block of data is skipped."); + LOG_WARNING(log, "Destination table " << destination_id.getNameForLogs() << " has no common columns with block in buffer. Block of data is skipped."); } else { @@ -447,7 +447,7 @@ void StorageBuffer::startup() { if (global_context.getSettingsRef().readonly) { - LOG_WARNING(log, "Storage " << getName() << " is run with readonly settings, it will not be able to insert data." << " Set appropriate system_profile to fix this."); + LOG_WARNING(log, "Storage " << getName() << " is run with readonly settings, it will not be able to insert data." << " Set appropriate system_profile to fix this."); } @@ -653,7 +653,7 @@ void StorageBuffer::writeBlockToDestination(const Block & block, StoragePtr tabl auto column = block.getByName(dst_col.name); if (!column.type->equals(*dst_col.type)) { - LOG_WARNING(log, "Destination table " << destination_id.getNameForLogs() << " have different type of column " << backQuoteIfNeed(column.name) << " (" << dst_col.type->getName() << " != " << column.type->getName() << "). Block of data is converted."); + LOG_WARNING(log, "Destination table " << destination_id.getNameForLogs() << " have different type of column " << backQuoteIfNeed(column.name) << " (" << dst_col.type->getName() << " != " << column.type->getName() << "). Block of data is converted."); column.column = castColumn(column, dst_col.type); column.type = dst_col.type; } @@ -669,7 +669,7 @@ void StorageBuffer::writeBlockToDestination(const Block & block, StoragePtr tabl } if (block_to_write.columns() != block.columns()) - LOG_WARNING(log, "Not all columns from block in buffer exist in destination table " << destination_id.getNameForLogs() << ". Some columns are discarded."); + LOG_WARNING(log, "Not all columns from block in buffer exist in destination table " << destination_id.getNameForLogs() << ". Some columns are discarded."); auto list_of_columns = std::make_shared(); insert->columns = list_of_columns; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index d539bd53a11..6adadc6ac72 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -214,7 +214,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( skip_sanity_checks = true; current_zookeeper->remove(replica_path + "/flags/force_restore_data"); - LOG_WARNING(log, "Skipping the limits on severity of changes to data parts and columns (flag " << replica_path << "/flags/force_restore_data)."); + LOG_WARNING(log, "Skipping the limits on severity of changes to data parts and columns (flag " << replica_path << "/flags/force_restore_data)."); } else if (has_force_restore_data_flag) { @@ -342,7 +342,7 @@ void StorageReplicatedMergeTree::waitMutationToFinishOnReplicas( /// Replica could be inactive. if (!zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/is_active")) { - LOG_WARNING(log, "Replica " << replica << " is not active during mutation. " "Mutation will be done asynchronously when replica becomes active."); + LOG_WARNING(log, "Replica " << replica << " is not active during mutation. " "Mutation will be done asynchronously when replica becomes active."); inactive_replicas.emplace(replica); break; @@ -830,7 +830,7 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: } else { - LOG_WARNING(log, "checkPartAndAddToZooKeeper: node " << replica_path + "/parts/" + part_name << " already exists." << " Will not commit any nodes."); + LOG_WARNING(log, "checkPartAndAddToZooKeeper: node " << replica_path + "/parts/" + part_name << " already exists." << " Will not commit any nodes."); } } @@ -1003,7 +1003,7 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) } if (part->name != name) { - LOG_WARNING(log, "Part " << name << " is covered by " << part->name << " but should be merged into " << entry.new_part_name << ". This shouldn't happen often."); + LOG_WARNING(log, "Part " << name << " is covered by " << part->name << " but should be merged into " << entry.new_part_name << ". This shouldn't happen often."); have_all_parts = false; break; } @@ -1370,7 +1370,7 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) } else { - LOG_WARNING(log, "No active replica has part " << entry.new_part_name << ", but that part needs quorum and /quorum/status contains entry about another part " << quorum_entry.part_name << ". It means that part was successfully written to " << entry.quorum << " replicas, but then all of them goes offline." << " Or it is a bug."); + LOG_WARNING(log, "No active replica has part " << entry.new_part_name << ", but that part needs quorum and /quorum/status contains entry about another part " << quorum_entry.part_name << ". It means that part was successfully written to " << entry.quorum << " replicas, but then all of them goes offline." << " Or it is a bug."); } } } @@ -2297,7 +2297,7 @@ bool StorageReplicatedMergeTree::createLogEntryToMergeParts( const auto & part = parts[i]; if (part->modification_time + MAX_AGE_OF_LOCAL_PART_THAT_WASNT_ADDED_TO_ZOOKEEPER < time(nullptr)) { - LOG_WARNING(log, "Part " << part->name << " (that was selected for merge)" << " with age " << (time(nullptr) - part->modification_time) << " seconds exists locally but not in ZooKeeper." << " Won't do merge with that part and will check it."); + LOG_WARNING(log, "Part " << part->name << " (that was selected for merge)" << " with age " << (time(nullptr) - part->modification_time) << " seconds exists locally but not in ZooKeeper." << " Won't do merge with that part and will check it."); enqueuePartForCheck(part->name); } } @@ -2339,7 +2339,7 @@ bool StorageReplicatedMergeTree::createLogEntryToMutatePart(const IMergeTreeData { if (part.modification_time + MAX_AGE_OF_LOCAL_PART_THAT_WASNT_ADDED_TO_ZOOKEEPER < time(nullptr)) { - LOG_WARNING(log, "Part " << part.name << " (that was selected for mutation)" << " with age " << (time(nullptr) - part.modification_time) << " seconds exists locally but not in ZooKeeper." << " Won't mutate that part and will check it."); + LOG_WARNING(log, "Part " << part.name << " (that was selected for mutation)" << " with age " << (time(nullptr) - part.modification_time) << " seconds exists locally but not in ZooKeeper." << " Won't mutate that part and will check it."); enqueuePartForCheck(part.name); } @@ -4775,7 +4775,7 @@ void StorageReplicatedMergeTree::removePartsFromZooKeeper( continue; } else - LOG_WARNING(log, "Cannot remove part " << part_names[i] << " from ZooKeeper: " << zkutil::ZooKeeper::error2string(response.error)); + LOG_WARNING(log, "Cannot remove part " << part_names[i] << " from ZooKeeper: " << zkutil::ZooKeeper::error2string(response.error)); } } @@ -4824,7 +4824,7 @@ void StorageReplicatedMergeTree::clearBlocksInPartition( zookeeper.removeRecursive(path); } else if (rc) - LOG_WARNING(log, "Error while deleting ZooKeeper path `" << path << "`: " + zkutil::ZooKeeper::error2string(rc) << ", ignoring."); + LOG_WARNING(log, "Error while deleting ZooKeeper path `" << path << "`: " + zkutil::ZooKeeper::error2string(rc) << ", ignoring."); } LOG_TRACE_FORMATTED(log, "Deleted {} deduplication block IDs in partition ID {}", to_delete_futures.size(), partition_id); From 0ab3ca853437164cdfa49ca0d0cb747f4e918511 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 22:32:44 +0300 Subject: [PATCH 646/738] Apply all transformations again --- src/Disks/S3/registerDiskS3.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Disks/S3/registerDiskS3.cpp b/src/Disks/S3/registerDiskS3.cpp index b711f7360c8..cf347b75931 100644 --- a/src/Disks/S3/registerDiskS3.cpp +++ b/src/Disks/S3/registerDiskS3.cpp @@ -46,7 +46,7 @@ namespace throw Exception("Only HTTP/HTTPS schemas allowed in proxy resolver config: " + proxy_scheme, ErrorCodes::BAD_ARGUMENTS); auto proxy_port = proxy_resolver_config.getUInt(prefix + ".proxy_port"); - LOG_DEBUG(&Logger::get("DiskS3"), "Configured proxy resolver: " << endpoint.toString() << ", Scheme: " << proxy_scheme << ", Port: " << proxy_port); + LOG_DEBUG_FORMATTED(&Logger::get("DiskS3"), "Configured proxy resolver: {}, Scheme: {}, Port: {}", endpoint.toString(), proxy_scheme, proxy_port); return std::make_shared(endpoint, proxy_scheme, proxy_port); } From eacff92d0ed9e22c484e244ed9c348fa0aa745b6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 22:35:08 +0300 Subject: [PATCH 647/738] Progress on task --- src/Client/ConnectionPoolWithFailover.cpp | 2 +- src/Common/tests/cow_compositions.cpp | 2 +- src/Common/tests/gtest_shell_command.cpp | 2 +- src/Compression/tests/gtest_compressionCodec.cpp | 2 +- .../ParallelAggregatingBlockInputStream.cpp | 4 ++-- src/IO/WriteBufferFromS3.cpp | 2 +- src/Interpreters/Aggregator.cpp | 8 ++++---- src/Interpreters/DDLWorker.cpp | 6 +++--- src/Interpreters/SystemLog.h | 2 +- src/Interpreters/executeQuery.cpp | 2 +- src/Parsers/ASTWatchQuery.h | 2 +- src/Processors/Executors/PipelineExecutor.cpp | 2 +- src/Processors/Transforms/AggregatingTransform.cpp | 2 +- src/Processors/printPipeline.h | 2 +- src/Storages/Distributed/DirectoryMonitor.cpp | 2 +- .../Distributed/DistributedBlockOutputStream.cpp | 4 ++-- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 2 +- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- src/Storages/MergeTree/MergeTreePartsMover.cpp | 2 +- src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp | 4 ++-- .../MergeTree/ReplicatedMergeTreeTableMetadata.cpp | 2 +- src/Storages/StorageBuffer.cpp | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 12 ++++++------ 23 files changed, 36 insertions(+), 36 deletions(-) diff --git a/src/Client/ConnectionPoolWithFailover.cpp b/src/Client/ConnectionPoolWithFailover.cpp index fc47138e4a3..e59fab2a162 100644 --- a/src/Client/ConnectionPoolWithFailover.cpp +++ b/src/Client/ConnectionPoolWithFailover.cpp @@ -248,7 +248,7 @@ ConnectionPoolWithFailover::tryGetEntry( result.is_up_to_date = false; result.staleness = delay; - LOG_TRACE(log, "Server " << result.entry->getDescription() << " has unacceptable replica delay " << "for table " << table_to_check->database << "." << table_to_check->table << ": " << delay); + LOG_TRACE(log, "Server " << result.entry->getDescription() << " has unacceptable replica delay for table " << table_to_check->database << "." << table_to_check->table << ": " << delay); ProfileEvents::increment(ProfileEvents::DistributedConnectionStaleReplica); } } diff --git a/src/Common/tests/cow_compositions.cpp b/src/Common/tests/cow_compositions.cpp index 74369e86300..427fe9130e9 100644 --- a/src/Common/tests/cow_compositions.cpp +++ b/src/Common/tests/cow_compositions.cpp @@ -95,7 +95,7 @@ int main(int, char **) mut->set(3); std::cerr << "refcounts: " << x->use_count() << ", " << mut->use_count() << "\n"; - std::cerr << "addresses: " << x.get() << ", " << ", " << mut.get() << "\n"; + std::cerr << "addresses: " << x.get() << ", " << mut.get() << "\n"; y = std::move(mut); } diff --git a/src/Common/tests/gtest_shell_command.cpp b/src/Common/tests/gtest_shell_command.cpp index 79b8309d0fa..057a4d22648 100644 --- a/src/Common/tests/gtest_shell_command.cpp +++ b/src/Common/tests/gtest_shell_command.cpp @@ -62,6 +62,6 @@ TEST(ShellCommand, AutoWait) //command->wait(); // now automatic } - // std::cerr << "inspect me: ps auxwwf" << "\n"; + // std::cerr << "inspect me: ps auxwwf\n"; // std::this_thread::sleep_for(std::chrono::seconds(100)); } diff --git a/src/Compression/tests/gtest_compressionCodec.cpp b/src/Compression/tests/gtest_compressionCodec.cpp index eca1c3c0a8f..36fc7eba8f8 100644 --- a/src/Compression/tests/gtest_compressionCodec.cpp +++ b/src/Compression/tests/gtest_compressionCodec.cpp @@ -220,7 +220,7 @@ template if (l_size != r_size) { - result = ::testing::AssertionFailure() << "size mismatch" << " expected: " << l_size << " got:" << r_size; + result = ::testing::AssertionFailure() << "size mismatch expected: " << l_size << " got:" << r_size; } if (l_size == 0 || r_size == 0) { diff --git a/src/DataStreams/ParallelAggregatingBlockInputStream.cpp b/src/DataStreams/ParallelAggregatingBlockInputStream.cpp index 5fd89bbc3dc..eb8cf912eec 100644 --- a/src/DataStreams/ParallelAggregatingBlockInputStream.cpp +++ b/src/DataStreams/ParallelAggregatingBlockInputStream.cpp @@ -177,12 +177,12 @@ void ParallelAggregatingBlockInputStream::execute() for (size_t i = 0; i < max_threads; ++i) { size_t rows = many_data[i]->size(); - LOG_TRACE(log, "Aggregated. " << threads_data[i].src_rows << " to " << rows << " rows" << " (from " << threads_data[i].src_bytes / 1048576.0 << " MiB)" << " in " << elapsed_seconds << " sec." << " (" << threads_data[i].src_rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(threads_data[i].src_bytes / elapsed_seconds) << "/sec.)"); + LOG_TRACE(log, "Aggregated. " << threads_data[i].src_rows << " to " << rows << " rows (from " << threads_data[i].src_bytes / 1048576.0 << " MiB) in " << elapsed_seconds << " sec. (" << threads_data[i].src_rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(threads_data[i].src_bytes / elapsed_seconds) << "/sec.)"); total_src_rows += threads_data[i].src_rows; total_src_bytes += threads_data[i].src_bytes; } - LOG_TRACE(log, "Total aggregated. " << total_src_rows << " rows (from " << total_src_bytes / 1048576.0 << " MiB)" << " in " << elapsed_seconds << " sec." << " (" << total_src_rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(total_src_bytes / elapsed_seconds) << "/sec.)"); + LOG_TRACE(log, "Total aggregated. " << total_src_rows << " rows (from " << total_src_bytes / 1048576.0 << " MiB) in " << elapsed_seconds << " sec. (" << total_src_rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(total_src_bytes / elapsed_seconds) << "/sec.)"); /// If there was no data, and we aggregate without keys, we must return single row with the result of empty aggregation. /// To do this, we pass a block with zero rows to aggregate. diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 21db726d074..70cf8503254 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -135,7 +135,7 @@ void WriteBufferFromS3::writePart(const String & data) { auto etag = outcome.GetResult().GetETag(); part_tags.push_back(etag); - LOG_DEBUG(log, "Writing part finished. " << "Total parts: " << part_tags.size() << ", Upload_id: " << upload_id << ", Etag: " << etag); + LOG_DEBUG_FORMATTED(log, "Writing part finished. Total parts: {}, Upload_id: {}, Etag: {}", part_tags.size(), upload_id, etag); } else throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR); diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index cfcdf009d29..72543f5a4cb 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -761,7 +761,7 @@ void Aggregator::writeToTemporaryFile(AggregatedDataVariants & data_variants, co ProfileEvents::increment(ProfileEvents::ExternalAggregationCompressedBytes, compressed_bytes); ProfileEvents::increment(ProfileEvents::ExternalAggregationUncompressedBytes, uncompressed_bytes); - LOG_TRACE(log, "Written part in " << elapsed_seconds << " sec., " << rows << " rows, " << formatReadableSizeWithBinarySuffix(uncompressed_bytes) << " uncompressed, " << formatReadableSizeWithBinarySuffix(compressed_bytes) << " compressed, " << (uncompressed_bytes / rows) << " uncompressed bytes per row, " << (compressed_bytes / rows) << " compressed bytes per row, " << "compression rate: " << (uncompressed_bytes / compressed_bytes) << " (" << (rows / elapsed_seconds) << " rows/sec., " << formatReadableSizeWithBinarySuffix(uncompressed_bytes / elapsed_seconds) << "/sec. uncompressed, " << formatReadableSizeWithBinarySuffix(compressed_bytes / elapsed_seconds) << "/sec. compressed)"); + LOG_TRACE(log, "Written part in " << elapsed_seconds << " sec., " << rows << " rows, " << formatReadableSizeWithBinarySuffix(uncompressed_bytes) << " uncompressed, " << formatReadableSizeWithBinarySuffix(compressed_bytes) << " compressed, " << (uncompressed_bytes / rows) << " uncompressed bytes per row, " << (compressed_bytes / rows) << " compressed bytes per row, compression rate: " << (uncompressed_bytes / compressed_bytes) << " (" << (rows / elapsed_seconds) << " rows/sec., " << formatReadableSizeWithBinarySuffix(uncompressed_bytes / elapsed_seconds) << "/sec. uncompressed, " << formatReadableSizeWithBinarySuffix(compressed_bytes / elapsed_seconds) << "/sec. compressed)"); } void Aggregator::writeToTemporaryFile(AggregatedDataVariants & data_variants) { @@ -927,7 +927,7 @@ void Aggregator::execute(const BlockInputStreamPtr & stream, AggregatedDataVaria double elapsed_seconds = watch.elapsedSeconds(); size_t rows = result.sizeWithoutOverflowRow(); - LOG_TRACE(log, "Aggregated. " << src_rows << " to " << rows << " rows (from " << formatReadableSizeWithBinarySuffix(src_bytes) << ")" << " in " << elapsed_seconds << " sec." << " (" << src_rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(src_bytes / elapsed_seconds) << "/sec.)"); + LOG_TRACE(log, "Aggregated. " << src_rows << " to " << rows << " rows (from " << formatReadableSizeWithBinarySuffix(src_bytes) << ") in " << elapsed_seconds << " sec. (" << src_rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(src_bytes / elapsed_seconds) << "/sec.)"); } @@ -1293,7 +1293,7 @@ BlocksList Aggregator::convertToBlocks(AggregatedDataVariants & data_variants, b } double elapsed_seconds = watch.elapsedSeconds(); - LOG_TRACE(log, "Converted aggregated data to blocks. " << rows << " rows, " << bytes / 1048576.0 << " MiB" << " in " << elapsed_seconds << " sec." << " (" << rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(bytes / elapsed_seconds) << "/sec.)"); + LOG_TRACE(log, "Converted aggregated data to blocks. " << rows << " rows, " << bytes / 1048576.0 << " MiB in " << elapsed_seconds << " sec. (" << rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(bytes / elapsed_seconds) << "/sec.)"); return blocks; } @@ -2155,7 +2155,7 @@ Block Aggregator::mergeBlocks(BlocksList & blocks, bool final) size_t rows = block.rows(); size_t bytes = block.bytes(); double elapsed_seconds = watch.elapsedSeconds(); - LOG_TRACE(log, "Merged partially aggregated blocks. " << rows << " rows, " << bytes / 1048576.0 << " MiB." << " in " << elapsed_seconds << " sec." << " (" << rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(bytes / elapsed_seconds) << "/sec.)"); + LOG_TRACE(log, "Merged partially aggregated blocks. " << rows << " rows, " << bytes / 1048576.0 << " MiB. in " << elapsed_seconds << " sec. (" << rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(bytes / elapsed_seconds) << "/sec.)"); if (isCancelled()) return {}; diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index 0ba1c184545..f065f381b38 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -238,7 +238,7 @@ DDLWorker::DDLWorker(const std::string & zk_root_dir, Context & context_, const if (context.getSettingsRef().readonly) { - LOG_WARNING(log, "Distributed DDL worker is run with readonly settings, it will not be able to execute DDL queries" << " Set appropriate system_profile or distributed_ddl.profile to fix this."); + LOG_WARNING(log, "Distributed DDL worker is run with readonly settings, it will not be able to execute DDL queries Set appropriate system_profile or distributed_ddl.profile to fix this."); } host_fqdn = getFQDNOrHostName(); @@ -940,7 +940,7 @@ String DDLWorker::enqueueQuery(DDLLogEntry & entry) } catch (...) { - LOG_INFO(log, "An error occurred while creating auxiliary ZooKeeper directories in " << node_path << " . They will be created later" << ". Error : " << getCurrentExceptionMessage(true)); + LOG_INFO_FORMATTED(log, "An error occurred while creating auxiliary ZooKeeper directories in {} . They will be created later. Error : {}", node_path, getCurrentExceptionMessage(true)); } return node_path; @@ -1217,7 +1217,7 @@ private: if (!ignoring_hosts.count(host)) { ignoring_hosts.emplace(host); - LOG_INFO(log, "Unexpected host " << host << " appeared " << " in task " << node_path); + LOG_INFO_FORMATTED(log, "Unexpected host {} appeared in task {}", host, node_path); } continue; } diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h index a7c488fab86..033a46d3960 100644 --- a/src/Interpreters/SystemLog.h +++ b/src/Interpreters/SystemLog.h @@ -256,7 +256,7 @@ void SystemLog::add(const LogElement & element) // TextLog sets its logger level to 0, so this log is a noop and // there is no recursive logging. - LOG_ERROR(log, "Queue is full for system log '" << demangle(typeid(*this).name()) << "'" << " at " << queue_front_index); + LOG_ERROR_FORMATTED(log, "Queue is full for system log '{}' at {}", demangle(typeid(*this).name()), queue_front_index); } return; diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 4a629babb60..1a10edf3ab3 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -140,7 +140,7 @@ static void setExceptionStackTrace(QueryLogElement & elem) /// Log exception (with query info) into text log (not into system table). static void logException(Context & context, QueryLogElement & elem) { - LOG_ERROR(&Logger::get("executeQuery"), elem.exception << " (from " << context.getClientInfo().current_address.toString() << ")" << " (in query: " << joinLines(elem.query) << ")" << (!elem.stack_trace.empty() ? ", Stack trace (when copying this message, always include the lines below):\n\n" + elem.stack_trace : "")); + LOG_ERROR(&Logger::get("executeQuery"), elem.exception << " (from " << context.getClientInfo().current_address.toString() << ") (in query: " << joinLines(elem.query) << ")" << (!elem.stack_trace.empty() ? ", Stack trace (when copying this message, always include the lines below):\n\n" + elem.stack_trace : "")); } diff --git a/src/Parsers/ASTWatchQuery.h b/src/Parsers/ASTWatchQuery.h index c4046a8771f..c6e6908cf97 100644 --- a/src/Parsers/ASTWatchQuery.h +++ b/src/Parsers/ASTWatchQuery.h @@ -41,7 +41,7 @@ protected: { std::string indent_str = s.one_line ? "" : std::string(4 * frame.indent, ' '); - s.ostr << (s.hilite ? hilite_keyword : "") << "WATCH" << " " << (s.hilite ? hilite_none : "") + s.ostr << (s.hilite ? hilite_keyword : "") << "WATCH " << (s.hilite ? hilite_none : "") << (!database.empty() ? backQuoteIfNeed(database) + "." : "") << backQuoteIfNeed(table); if (is_watch_events) diff --git a/src/Processors/Executors/PipelineExecutor.cpp b/src/Processors/Executors/PipelineExecutor.cpp index d190f441b65..ba7883f01d6 100644 --- a/src/Processors/Executors/PipelineExecutor.cpp +++ b/src/Processors/Executors/PipelineExecutor.cpp @@ -547,7 +547,7 @@ void PipelineExecutor::executeSingleThread(size_t thread_num, size_t num_threads #ifndef NDEBUG auto & context = executor_contexts[thread_num]; - LOG_TRACE(log, "Thread finished." << " Total time: " << (context->total_time_ns / 1e9) << " sec." << " Execution time: " << (context->execution_time_ns / 1e9) << " sec." << " Processing time: " << (context->processing_time_ns / 1e9) << " sec." << " Wait time: " << (context->wait_time_ns / 1e9) << " sec."); + LOG_TRACE(log, "Thread finished. Total time: " << (context->total_time_ns / 1e9) << " sec. Execution time: " << (context->execution_time_ns / 1e9) << " sec. Processing time: " << (context->processing_time_ns / 1e9) << " sec. Wait time: " << (context->wait_time_ns / 1e9) << " sec."); #endif } diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index 71063050a40..bea1ca0be76 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -540,7 +540,7 @@ void AggregatingTransform::initGenerate() double elapsed_seconds = watch.elapsedSeconds(); size_t rows = variants.sizeWithoutOverflowRow(); - LOG_TRACE(log, "Aggregated. " << src_rows << " to " << rows << " rows (from " << src_bytes / 1048576.0 << " MiB)" << " in " << elapsed_seconds << " sec." << " (" << src_rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(src_bytes / elapsed_seconds) << "/sec.)"); + LOG_TRACE(log, "Aggregated. " << src_rows << " to " << rows << " rows (from " << src_bytes / 1048576.0 << " MiB) in " << elapsed_seconds << " sec. (" << src_rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(src_bytes / elapsed_seconds) << "/sec.)"); if (params->aggregator.hasTemporaryFiles()) { diff --git a/src/Processors/printPipeline.h b/src/Processors/printPipeline.h index f7e350de3dc..ce7306ec4cf 100644 --- a/src/Processors/printPipeline.h +++ b/src/Processors/printPipeline.h @@ -48,7 +48,7 @@ void printPipeline(const Processors & processors, const Statuses & statuses, Wri const IProcessor & curr = *processor; const IProcessor & next = port.getInputPort().getProcessor(); - out << "n" << get_proc_id(curr) << " -> " << "n" << get_proc_id(next) << ";\n"; + out << "n" << get_proc_id(curr) << " -> n" << get_proc_id(next) << ";\n"; } } out << "}\n"; diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp index 30879433024..c699fde552e 100644 --- a/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/src/Storages/Distributed/DirectoryMonitor.cpp @@ -317,7 +317,7 @@ void StorageDistributedDirectoryMonitor::readHeader( readVarUInt(initiator_revision, header_buf); if (ClickHouseRevision::get() < initiator_revision) { - LOG_WARNING(log, "ClickHouse shard version is older than ClickHouse initiator version. " << "It may lack support for new features."); + LOG_WARNING(log, "ClickHouse shard version is older than ClickHouse initiator version. It may lack support for new features."); } readStringBinary(insert_query, header_buf); diff --git a/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/src/Storages/Distributed/DistributedBlockOutputStream.cpp index 569452f3ba8..a5811b39865 100644 --- a/src/Storages/Distributed/DistributedBlockOutputStream.cpp +++ b/src/Storages/Distributed/DistributedBlockOutputStream.cpp @@ -114,7 +114,7 @@ void DistributedBlockOutputStream::write(const Block & block) if (ordinary_block.has(col.name)) { ordinary_block.erase(col.name); - LOG_DEBUG(log, storage.getStorageID().getNameForLogs() << ": column " + col.name + " will be removed, " << "because it is MATERIALIZED"); + LOG_DEBUG(log, storage.getStorageID().getNameForLogs() << ": column " + col.name + " will be removed, because it is MATERIALIZED"); } } @@ -411,7 +411,7 @@ void DistributedBlockOutputStream::writeSuffix() auto log_performance = [this] () { double elapsed = watch.elapsedSeconds(); - LOG_DEBUG(log, "It took " << std::fixed << std::setprecision(1) << elapsed << " sec. to insert " << inserted_blocks << " blocks" << ", " << std::fixed << std::setprecision(1) << inserted_rows / elapsed << " rows per second" << ". " << getCurrentStateDescription()); + LOG_DEBUG(log, "It took " << std::fixed << std::setprecision(1) << elapsed << " sec. to insert " << inserted_blocks << " blocks, " << std::fixed << std::setprecision(1) << inserted_rows / elapsed << " rows per second. " << getCurrentStateDescription()); }; if (insert_sync && pool) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 494e5809683..d2b637943d1 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -668,7 +668,7 @@ void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_ Names files; volume->getDisk()->listFiles(to, files); - LOG_WARNING(storage.log, "Part directory " << fullPath(volume->getDisk(), to) << " already exists" << " and contains " << files.size() << " files. Removing it."); + LOG_WARNING(storage.log, "Part directory " << fullPath(volume->getDisk(), to) << " already exists and contains " << files.size() << " files. Removing it."); volume->getDisk()->removeRecursive(to); } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index c35cb29ca62..dd5319962bf 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -2088,7 +2088,7 @@ restore_covered) if (error) { - LOG_ERROR(log, "The set of parts restored in place of " << part->name << " looks incomplete." << " There might or might not be a data loss." << (error_parts.empty() ? "" : " Suspicious parts: " + error_parts)); + LOG_ERROR_FORMATTED(log, "The set of parts restored in place of {} looks incomplete. There might or might not be a data loss.{}", part->name, (error_parts.empty() ? "" : " Suspicious parts: " + error_parts)); } } } diff --git a/src/Storages/MergeTree/MergeTreePartsMover.cpp b/src/Storages/MergeTree/MergeTreePartsMover.cpp index 60883429b32..e18bbe0fdce 100644 --- a/src/Storages/MergeTree/MergeTreePartsMover.cpp +++ b/src/Storages/MergeTree/MergeTreePartsMover.cpp @@ -215,7 +215,7 @@ void MergeTreePartsMover::swapClonedPart(const MergeTreeData::DataPartPtr & clon /// It's ok, because we don't block moving parts for merges or mutations if (!active_part || active_part->name != cloned_part->name) { - LOG_INFO(log, "Failed to swap " << cloned_part->name << ". Active part doesn't exist." << " Possible it was merged or mutated. Will remove copy on path '" << cloned_part->getFullPath() << "'."); + LOG_INFO_FORMATTED(log, "Failed to swap {}. Active part doesn't exist. Possible it was merged or mutated. Will remove copy on path '{}'.", cloned_part->name, cloned_part->getFullPath()); return; } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 9ee4f7d3f9a..b99df9a3352 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -320,7 +320,7 @@ void ReplicatedMergeTreeQueue::updateTimesInZooKeeper( auto code = zookeeper->tryMulti(ops, responses); if (code) - LOG_ERROR(log, "Couldn't set value of nodes for insert times (" << replica_path << "/min_unprocessed_insert_time, max_processed_insert_time)" << ": " << zkutil::ZooKeeper::error2string(code) + ". This shouldn't happen often."); + LOG_ERROR_FORMATTED(log, "Couldn't set value of nodes for insert times ({}/min_unprocessed_insert_time, max_processed_insert_time): {}", replica_path, zkutil::ZooKeeper::error2string(code) + ". This shouldn't happen often."); } } @@ -1885,7 +1885,7 @@ bool ReplicatedMergeTreeMergePredicate::isMutationFinished(const ReplicatedMerge partition_it->second.begin(), partition_it->second.lower_bound(block_num)); if (blocks_count) { - LOG_TRACE(queue.log, "Mutation " << mutation.znode_name << " is not done yet because " << "in partition ID " << partition_id << " there are still " << blocks_count << " uncommitted blocks."); + LOG_TRACE(queue.log, "Mutation " << mutation.znode_name << " is not done yet because in partition ID " << partition_id << " there are still " << blocks_count << " uncommitted blocks."); return false; } } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp index fe4ea79d106..ba9bbd1da61 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp @@ -66,7 +66,7 @@ ReplicatedMergeTreeTableMetadata::ReplicatedMergeTreeTableMetadata(const MergeTr void ReplicatedMergeTreeTableMetadata::write(WriteBuffer & out) const { - out << "metadata format version: 1" << "\n" + out << "metadata format version: 1\n" << "date column: " << date_column << "\n" << "sampling expression: " << sampling_expression << "\n" << "index granularity: " << index_granularity << "\n" diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index 1bb30201e82..5708e5f2b05 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -447,7 +447,7 @@ void StorageBuffer::startup() { if (global_context.getSettingsRef().readonly) { - LOG_WARNING(log, "Storage " << getName() << " is run with readonly settings, it will not be able to insert data." << " Set appropriate system_profile to fix this."); + LOG_WARNING(log, "Storage " << getName() << " is run with readonly settings, it will not be able to insert data. Set appropriate system_profile to fix this."); } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 6adadc6ac72..094569e91dd 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -830,7 +830,7 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: } else { - LOG_WARNING(log, "checkPartAndAddToZooKeeper: node " << replica_path + "/parts/" + part_name << " already exists." << " Will not commit any nodes."); + LOG_WARNING(log, "checkPartAndAddToZooKeeper: node " << replica_path + "/parts/" + part_name << " already exists. Will not commit any nodes."); } } @@ -1370,7 +1370,7 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) } else { - LOG_WARNING(log, "No active replica has part " << entry.new_part_name << ", but that part needs quorum and /quorum/status contains entry about another part " << quorum_entry.part_name << ". It means that part was successfully written to " << entry.quorum << " replicas, but then all of them goes offline." << " Or it is a bug."); + LOG_WARNING(log, "No active replica has part " << entry.new_part_name << ", but that part needs quorum and /quorum/status contains entry about another part " << quorum_entry.part_name << ". It means that part was successfully written to " << entry.quorum << " replicas, but then all of them goes offline. Or it is a bug."); } } } @@ -1580,7 +1580,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) } catch (Exception &) { - LOG_INFO(log, "Can't use " << source_table_id.getNameForLogs() << " as source table for REPLACE PARTITION command. Will fetch all parts." << " Reason: " << getCurrentExceptionMessage(false)); + LOG_INFO_FORMATTED(log, "Can't use {} as source table for REPLACE PARTITION command. Will fetch all parts. Reason: {}", source_table_id.getNameForLogs(), getCurrentExceptionMessage(false)); return 0; } @@ -2297,7 +2297,7 @@ bool StorageReplicatedMergeTree::createLogEntryToMergeParts( const auto & part = parts[i]; if (part->modification_time + MAX_AGE_OF_LOCAL_PART_THAT_WASNT_ADDED_TO_ZOOKEEPER < time(nullptr)) { - LOG_WARNING(log, "Part " << part->name << " (that was selected for merge)" << " with age " << (time(nullptr) - part->modification_time) << " seconds exists locally but not in ZooKeeper." << " Won't do merge with that part and will check it."); + LOG_WARNING(log, "Part " << part->name << " (that was selected for merge) with age " << (time(nullptr) - part->modification_time) << " seconds exists locally but not in ZooKeeper. Won't do merge with that part and will check it."); enqueuePartForCheck(part->name); } } @@ -2339,7 +2339,7 @@ bool StorageReplicatedMergeTree::createLogEntryToMutatePart(const IMergeTreeData { if (part.modification_time + MAX_AGE_OF_LOCAL_PART_THAT_WASNT_ADDED_TO_ZOOKEEPER < time(nullptr)) { - LOG_WARNING(log, "Part " << part.name << " (that was selected for mutation)" << " with age " << (time(nullptr) - part.modification_time) << " seconds exists locally but not in ZooKeeper." << " Won't mutate that part and will check it."); + LOG_WARNING(log, "Part " << part.name << " (that was selected for mutation) with age " << (time(nullptr) - part.modification_time) << " seconds exists locally but not in ZooKeeper. Won't mutate that part and will check it."); enqueuePartForCheck(part.name); } @@ -4305,7 +4305,7 @@ void StorageReplicatedMergeTree::fetchPartition(const ASTPtr & partition, const if (best_replica.empty()) throw Exception("Logical error: cannot choose best replica.", ErrorCodes::LOGICAL_ERROR); - LOG_INFO(log, "Found " << replicas.size() << " replicas, " << active_replicas.size() << " of them are active." << " Selected " << best_replica << " to fetch from."); + LOG_INFO(log, "Found " << replicas.size() << " replicas, " << active_replicas.size() << " of them are active. Selected " << best_replica << " to fetch from."); String best_replica_path = from + "/replicas/" + best_replica; From 7e2fb9ad653dd13522828a1b8615d77b2fbbbd9f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 22:38:30 +0300 Subject: [PATCH 648/738] Apply all transformations again --- base/daemon/BaseDaemon.cpp | 2 +- programs/copier/ClusterCopier.cpp | 2 +- programs/server/Server.cpp | 2 +- src/Common/Config/ConfigProcessor.cpp | 2 +- src/Common/PoolWithFailoverBase.h | 2 +- src/DataStreams/CollapsingFinalBlockInputStream.cpp | 2 +- src/Databases/DatabaseOnDisk.cpp | 2 +- src/Interpreters/ClusterProxy/SelectStreamFactory.cpp | 6 +++--- src/Interpreters/DDLWorker.cpp | 6 +++--- src/Interpreters/DatabaseCatalog.cpp | 2 +- src/Storages/Distributed/DirectoryMonitor.cpp | 2 +- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 2 +- src/Storages/MergeTree/MergeTreeData.cpp | 10 +++++----- src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp | 2 +- .../MergeTree/ReplicatedMergeTreePartCheckThread.cpp | 2 +- src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp | 2 +- .../MergeTree/ReplicatedMergeTreeRestartingThread.cpp | 2 +- src/Storages/StorageBuffer.cpp | 8 ++++---- src/Storages/StorageReplicatedMergeTree.cpp | 10 +++++----- 19 files changed, 34 insertions(+), 34 deletions(-) diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index d5b7d1a0f6c..79dc9282931 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -498,7 +498,7 @@ void debugIncreaseOOMScore() } catch (const Poco::Exception & e) { - LOG_WARNING(&Logger::root(), "Failed to adjust OOM score: '" + e.displayText() + "'."); + LOG_WARNING_FORMATTED(&Logger::root(), "Failed to adjust OOM score: '{}'.", e.displayText()); return; } LOG_INFO_FORMATTED(&Logger::root(), "Set OOM score adjustment to {}", new_score); diff --git a/programs/copier/ClusterCopier.cpp b/programs/copier/ClusterCopier.cpp index eb35768b955..3e082ebef2d 100644 --- a/programs/copier/ClusterCopier.cpp +++ b/programs/copier/ClusterCopier.cpp @@ -164,7 +164,7 @@ void ClusterCopier::discoverShardPartitions(const ConnectionTimeouts & timeouts, for (const String & missing_partition : missing_partitions) ss << " " << missing_partition; - LOG_WARNING(log, "There are no " << missing_partitions.size() << " partitions from enabled_partitions in shard " << task_shard->getDescription() << " :" << ss.str()); + LOG_WARNING_FORMATTED(log, "There are no {} partitions from enabled_partitions in shard {} :{}", missing_partitions.size(), task_shard->getDescription(), ss.str()); } LOG_DEBUG_FORMATTED(log, "Will copy {} partitions from shard {}", task_shard->partition_tasks.size(), task_shard->getDescription()); diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 78ca90a4280..6399fb440cf 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -354,7 +354,7 @@ int Server::main(const std::vector & /*args*/) rlim.rlim_cur = config().getUInt("max_open_files", rlim.rlim_max); int rc = setrlimit(RLIMIT_NOFILE, &rlim); if (rc != 0) - LOG_WARNING(log, "Cannot set max number of file descriptors to " << rlim.rlim_cur << ". Try to specify max_open_files according to your system limits. error: " << strerror(errno)); + LOG_WARNING_FORMATTED(log, "Cannot set max number of file descriptors to {}. Try to specify max_open_files according to your system limits. error: {}", rlim.rlim_cur, strerror(errno)); else LOG_DEBUG_FORMATTED(log, "Set max number of file descriptors to {} (was {}).", rlim.rlim_cur, old); } diff --git a/src/Common/Config/ConfigProcessor.cpp b/src/Common/Config/ConfigProcessor.cpp index 8ee26226405..af9cdeeb716 100644 --- a/src/Common/Config/ConfigProcessor.cpp +++ b/src/Common/Config/ConfigProcessor.cpp @@ -568,7 +568,7 @@ ConfigProcessor::LoadedConfig ConfigProcessor::loadConfigWithZooKeeperIncludes( if (!zk_exception) throw; - LOG_WARNING(log, "Error while processing from_zk config includes: " + zk_exception->message() + ". Config will be loaded from preprocessed file: " + preprocessed_path); + LOG_WARNING_FORMATTED(log, "Error while processing from_zk config includes: {}. Config will be loaded from preprocessed file: {}", zk_exception->message(), preprocessed_path); config_xml = dom_parser.parse(preprocessed_path); } diff --git a/src/Common/PoolWithFailoverBase.h b/src/Common/PoolWithFailoverBase.h index 10ab3885be6..2b9f7212a4f 100644 --- a/src/Common/PoolWithFailoverBase.h +++ b/src/Common/PoolWithFailoverBase.h @@ -239,7 +239,7 @@ PoolWithFailoverBase::getMany( } else { - LOG_WARNING(log, "Connection failed at try №" << (shuffled_pool.error_count + 1) << ", reason: " << fail_message); + LOG_WARNING_FORMATTED(log, "Connection failed at try №{}, reason: {}", (shuffled_pool.error_count + 1), fail_message); ProfileEvents::increment(ProfileEvents::DistributedConnectionFailTry); shuffled_pool.error_count = std::min(max_error_cap, shuffled_pool.error_count + 1); diff --git a/src/DataStreams/CollapsingFinalBlockInputStream.cpp b/src/DataStreams/CollapsingFinalBlockInputStream.cpp index 73d3bda5860..2c359f81b7c 100644 --- a/src/DataStreams/CollapsingFinalBlockInputStream.cpp +++ b/src/DataStreams/CollapsingFinalBlockInputStream.cpp @@ -18,7 +18,7 @@ CollapsingFinalBlockInputStream::~CollapsingFinalBlockInputStream() void CollapsingFinalBlockInputStream::reportBadCounts() { /// With inconsistent data, this is an unavoidable error that can not be easily fixed by admins. Therefore Warning. - LOG_WARNING(log, "Incorrect data: number of rows with sign = 1 (" << count_positive << ") differs with number of rows with sign = -1 (" << count_negative << ") by more than one"); + LOG_WARNING_FORMATTED(log, "Incorrect data: number of rows with sign = 1 ({}) differs with number of rows with sign = -1 ({}) by more than one", count_positive, count_negative); } void CollapsingFinalBlockInputStream::reportBadSign(Int8 sign) diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index 27d0d9a7a10..395c05df5e3 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -466,7 +466,7 @@ ASTPtr DatabaseOnDisk::parseQueryFromMetadata(Poco::Logger * loger, const Contex table_name = unescapeForFileName(table_name); if (create.table != TABLE_WITH_UUID_NAME_PLACEHOLDER) - LOG_WARNING(loger, "File " << metadata_file_path << " contains both UUID and table name. " "Will use name `" << table_name << "` instead of `" << create.table << "`"); + LOG_WARNING(loger, "File " << metadata_file_path << " contains both UUID and table name. Will use name `" << table_name << "` instead of `" << create.table << "`"); create.table = table_name; } diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index 00c1356d002..7fe6bb260c5 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -200,7 +200,7 @@ void SelectStreamFactory::createForShard( ProfileEvents::increment(ProfileEvents::DistributedConnectionMissingTable); if (shard_info.hasRemoteConnections()) { - LOG_WARNING(&Logger::get("ClusterProxy::SelectStreamFactory"), "There is no table " << main_table.getNameForLogs() << " on local replica of shard " << shard_info.shard_num << ", will try remote replicas."); + LOG_WARNING_FORMATTED(&Logger::get("ClusterProxy::SelectStreamFactory"), "There is no table {} on local replica of shard {}, will try remote replicas.", main_table.getNameForLogs(), shard_info.shard_num); emplace_remote_stream(); } else @@ -236,7 +236,7 @@ void SelectStreamFactory::createForShard( /// If we reached this point, local replica is stale. ProfileEvents::increment(ProfileEvents::DistributedConnectionStaleReplica); - LOG_WARNING(&Logger::get("ClusterProxy::SelectStreamFactory"), "Local replica of shard " << shard_info.shard_num << " is stale (delay: " << local_delay << "s.)"); + LOG_WARNING_FORMATTED(&Logger::get("ClusterProxy::SelectStreamFactory"), "Local replica of shard {} is stale (delay: {}s.)", shard_info.shard_num, local_delay); if (!settings.fallback_to_stale_replicas_for_distributed_queries) { @@ -284,7 +284,7 @@ void SelectStreamFactory::createForShard( catch (const Exception & ex) { if (ex.code() == ErrorCodes::ALL_CONNECTION_TRIES_FAILED) - LOG_WARNING(&Logger::get("ClusterProxy::SelectStreamFactory"), "Connections to remote replicas of local shard " << shard_num << " failed, will use stale local replica"); + LOG_WARNING_FORMATTED(&Logger::get("ClusterProxy::SelectStreamFactory"), "Connections to remote replicas of local shard {} failed, will use stale local replica", shard_num); else throw; } diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index f065f381b38..3d6d9801591 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -238,7 +238,7 @@ DDLWorker::DDLWorker(const std::string & zk_root_dir, Context & context_, const if (context.getSettingsRef().readonly) { - LOG_WARNING(log, "Distributed DDL worker is run with readonly settings, it will not be able to execute DDL queries Set appropriate system_profile or distributed_ddl.profile to fix this."); + LOG_WARNING_FORMATTED(log, "Distributed DDL worker is run with readonly settings, it will not be able to execute DDL queries Set appropriate system_profile or distributed_ddl.profile to fix this."); } host_fqdn = getFQDNOrHostName(); @@ -424,7 +424,7 @@ void DDLWorker::processTasks() { if (server_startup && e.code == Coordination::ZNONODE) { - LOG_WARNING(log, "ZooKeeper NONODE error during startup. Ignoring entry " << task.entry_name << " (" << task.entry.query << ") : " << getCurrentExceptionMessage(true)); + LOG_WARNING_FORMATTED(log, "ZooKeeper NONODE error during startup. Ignoring entry {} ({}) : {}", task.entry_name, task.entry.query, getCurrentExceptionMessage(true)); } else { @@ -433,7 +433,7 @@ void DDLWorker::processTasks() } catch (...) { - LOG_WARNING(log, "An error occurred while processing task " << task.entry_name << " (" << task.entry.query << ") : " << getCurrentExceptionMessage(true)); + LOG_WARNING_FORMATTED(log, "An error occurred while processing task {} ({}) : {}", task.entry_name, task.entry.query, getCurrentExceptionMessage(true)); throw; } } diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 6c1d00df3bc..6bcccdff436 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -597,7 +597,7 @@ void DatabaseCatalog::enqueueDroppedTableCleanup(StorageID table_id, StoragePtr } else { - LOG_WARNING(log, "Cannot parse metadata of partially dropped table " << table_id.getNameForLogs() << " from " << dropped_metadata_path << ". Will remove metadata file and data directory. Garbage may be left in /store directory and ZooKeeper."); + LOG_WARNING_FORMATTED(log, "Cannot parse metadata of partially dropped table {} from {}. Will remove metadata file and data directory. Garbage may be left in /store directory and ZooKeeper.", table_id.getNameForLogs(), dropped_metadata_path); } drop_time = Poco::File(dropped_metadata_path).getLastModified().epochTime(); diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp index c699fde552e..40980d1830a 100644 --- a/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/src/Storages/Distributed/DirectoryMonitor.cpp @@ -317,7 +317,7 @@ void StorageDistributedDirectoryMonitor::readHeader( readVarUInt(initiator_revision, header_buf); if (ClickHouseRevision::get() < initiator_revision) { - LOG_WARNING(log, "ClickHouse shard version is older than ClickHouse initiator version. It may lack support for new features."); + LOG_WARNING_FORMATTED(log, "ClickHouse shard version is older than ClickHouse initiator version. It may lack support for new features."); } readStringBinary(insert_query, header_buf); diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index d2b637943d1..a5f831a983e 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -668,7 +668,7 @@ void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_ Names files; volume->getDisk()->listFiles(to, files); - LOG_WARNING(storage.log, "Part directory " << fullPath(volume->getDisk(), to) << " already exists and contains " << files.size() << " files. Removing it."); + LOG_WARNING_FORMATTED(storage.log, "Part directory {} already exists and contains {} files. Removing it.", fullPath(volume->getDisk(), to), files.size()); volume->getDisk()->removeRecursive(to); } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index dd5319962bf..25a4fdd255e 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -2914,9 +2914,9 @@ ReservationPtr MergeTreeData::tryReserveSpacePreferringTTLRules(UInt64 expected_ if (!destination_ptr) { if (ttl_entry->destination_type == PartDestinationType::VOLUME) - LOG_WARNING(log, "Would like to reserve space on volume '" << ttl_entry->destination_name << "' by TTL rule of table '" << log_name << "' but volume was not found"); + LOG_WARNING_FORMATTED(log, "Would like to reserve space on volume '{}' by TTL rule of table '{}' but volume was not found", ttl_entry->destination_name, log_name); else if (ttl_entry->destination_type == PartDestinationType::DISK) - LOG_WARNING(log, "Would like to reserve space on disk '" << ttl_entry->destination_name << "' by TTL rule of table '" << log_name << "' but disk was not found"); + LOG_WARNING_FORMATTED(log, "Would like to reserve space on disk '{}' by TTL rule of table '{}' but disk was not found", ttl_entry->destination_name, log_name); } else { @@ -2925,9 +2925,9 @@ ReservationPtr MergeTreeData::tryReserveSpacePreferringTTLRules(UInt64 expected_ return reservation; else if (ttl_entry->destination_type == PartDestinationType::VOLUME) - LOG_WARNING(log, "Would like to reserve space on volume '" << ttl_entry->destination_name << "' by TTL rule of table '" << log_name << "' but there is not enough space"); + LOG_WARNING_FORMATTED(log, "Would like to reserve space on volume '{}' by TTL rule of table '{}' but there is not enough space", ttl_entry->destination_name, log_name); else if (ttl_entry->destination_type == PartDestinationType::DISK) - LOG_WARNING(log, "Would like to reserve space on disk '" << ttl_entry->destination_name << "' by TTL rule of table '" << log_name << "' but there is not enough space"); + LOG_WARNING_FORMATTED(log, "Would like to reserve space on disk '{}' by TTL rule of table '{}' but there is not enough space", ttl_entry->destination_name, log_name); } } @@ -3053,7 +3053,7 @@ MergeTreeData::DataPartsVector MergeTreeData::Transaction::commit(MergeTreeData: DataPartsVector covered_parts = data.getActivePartsToReplace(part->info, part->name, covering_part, *owing_parts_lock); if (covering_part) { - LOG_WARNING(data.log, "Tried to commit obsolete part " << part->name << " covered by " << covering_part->getNameWithState()); + LOG_WARNING_FORMATTED(data.log, "Tried to commit obsolete part {} covered by {}", part->name, covering_part->getNameWithState()); part->remove_time.store(0, std::memory_order_relaxed); /// The part will be removed without waiting for old_parts_lifetime seconds. data.modifyPartState(part, DataPartState::Outdated); diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 8eee53cde4a..86fce4c3ef6 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -1214,7 +1214,7 @@ MergeTreeData::DataPartPtr MergeTreeDataMergerMutator::renameMergedTemporaryPart * (NOTE: Merging with part that is not in ZK is not possible, see checks in 'createLogEntryToMergeParts'.) * - and after merge, this part will be removed in addition to parts that was merged. */ - LOG_WARNING(log, "Unexpected number of parts removed when adding " << new_data_part->name << ": " << replaced_parts.size() << " instead of " << parts.size()); + LOG_WARNING_FORMATTED(log, "Unexpected number of parts removed when adding {}: {} instead of {}", new_data_part->name, replaced_parts.size(), parts.size()); } else { diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp index 8ec073282c9..1ec1c041826 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp @@ -143,7 +143,7 @@ void ReplicatedMergeTreePartCheckThread::searchForMissingPart(const String & par if (found_part_with_the_same_min_block && found_part_with_the_same_max_block) { - LOG_WARNING(log, "Found parts with the same min block and with the same max block as the missing part " << part_name << ". Hoping that it will eventually appear as a result of a merge."); + LOG_WARNING_FORMATTED(log, "Found parts with the same min block and with the same max block as the missing part {}. Hoping that it will eventually appear as a result of a merge.", part_name); return; } } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index b99df9a3352..2b2eadd8b69 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -1345,7 +1345,7 @@ MutationCommands ReplicatedMergeTreeQueue::getMutationCommands( auto end = in_partition->second.lower_bound(desired_mutation_version); if (end == in_partition->second.end() || end->first != desired_mutation_version) - LOG_WARNING(log, "Mutation with version " << desired_mutation_version << " not found in partition ID " << part->info.partition_id << " (trying to mutate part " << part->name + ")"); + LOG_WARNING_FORMATTED(log, "Mutation with version {} not found in partition ID {} (trying to mutate part {}", desired_mutation_version, part->info.partition_id, part->name + ")"); else ++end; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index 519c063f088..36805788f13 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -268,7 +268,7 @@ void ReplicatedMergeTreeRestartingThread::updateQuorumIfWeHavePart() if (!quorum_entry.replicas.count(storage.replica_name) && zookeeper->exists(storage.replica_path + "/parts/" + quorum_entry.part_name)) { - LOG_WARNING(log, "We have part " << quorum_entry.part_name << " but we is not in quorum. Updating quorum. This shouldn't happen often."); + LOG_WARNING_FORMATTED(log, "We have part {} but we is not in quorum. Updating quorum. This shouldn't happen often.", quorum_entry.part_name); storage.updateQuorum(quorum_entry.part_name); } } diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index 5708e5f2b05..0c9f1a79a5d 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -189,7 +189,7 @@ Pipes StorageBuffer::read( { if (!dest_columns.hasPhysical(column_name)) { - LOG_WARNING(log, "Destination table " << destination_id.getNameForLogs() << " doesn't have column " << backQuoteIfNeed(column_name) << ". The default values are used."); + LOG_WARNING_FORMATTED(log, "Destination table {} doesn't have column {}. The default values are used.", destination_id.getNameForLogs(), backQuoteIfNeed(column_name)); boost::range::remove_erase(columns_intersection, column_name); continue; } @@ -204,7 +204,7 @@ Pipes StorageBuffer::read( if (columns_intersection.empty()) { - LOG_WARNING(log, "Destination table " << destination_id.getNameForLogs() << " has no common columns with block in buffer. Block of data is skipped."); + LOG_WARNING_FORMATTED(log, "Destination table {} has no common columns with block in buffer. Block of data is skipped.", destination_id.getNameForLogs()); } else { @@ -447,7 +447,7 @@ void StorageBuffer::startup() { if (global_context.getSettingsRef().readonly) { - LOG_WARNING(log, "Storage " << getName() << " is run with readonly settings, it will not be able to insert data. Set appropriate system_profile to fix this."); + LOG_WARNING_FORMATTED(log, "Storage {} is run with readonly settings, it will not be able to insert data. Set appropriate system_profile to fix this.", getName()); } @@ -669,7 +669,7 @@ void StorageBuffer::writeBlockToDestination(const Block & block, StoragePtr tabl } if (block_to_write.columns() != block.columns()) - LOG_WARNING(log, "Not all columns from block in buffer exist in destination table " << destination_id.getNameForLogs() << ". Some columns are discarded."); + LOG_WARNING_FORMATTED(log, "Not all columns from block in buffer exist in destination table {}. Some columns are discarded.", destination_id.getNameForLogs()); auto list_of_columns = std::make_shared(); insert->columns = list_of_columns; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 094569e91dd..468be2dde79 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -214,7 +214,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( skip_sanity_checks = true; current_zookeeper->remove(replica_path + "/flags/force_restore_data"); - LOG_WARNING(log, "Skipping the limits on severity of changes to data parts and columns (flag " << replica_path << "/flags/force_restore_data)."); + LOG_WARNING_FORMATTED(log, "Skipping the limits on severity of changes to data parts and columns (flag {}/flags/force_restore_data).", replica_path); } else if (has_force_restore_data_flag) { @@ -830,7 +830,7 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: } else { - LOG_WARNING(log, "checkPartAndAddToZooKeeper: node " << replica_path + "/parts/" + part_name << " already exists. Will not commit any nodes."); + LOG_WARNING_FORMATTED(log, "checkPartAndAddToZooKeeper: node {} already exists. Will not commit any nodes.", replica_path + "/parts/" + part_name); } } @@ -2297,7 +2297,7 @@ bool StorageReplicatedMergeTree::createLogEntryToMergeParts( const auto & part = parts[i]; if (part->modification_time + MAX_AGE_OF_LOCAL_PART_THAT_WASNT_ADDED_TO_ZOOKEEPER < time(nullptr)) { - LOG_WARNING(log, "Part " << part->name << " (that was selected for merge) with age " << (time(nullptr) - part->modification_time) << " seconds exists locally but not in ZooKeeper. Won't do merge with that part and will check it."); + LOG_WARNING_FORMATTED(log, "Part {} (that was selected for merge) with age {} seconds exists locally but not in ZooKeeper. Won't do merge with that part and will check it.", part->name, (time(nullptr) - part->modification_time)); enqueuePartForCheck(part->name); } } @@ -2339,7 +2339,7 @@ bool StorageReplicatedMergeTree::createLogEntryToMutatePart(const IMergeTreeData { if (part.modification_time + MAX_AGE_OF_LOCAL_PART_THAT_WASNT_ADDED_TO_ZOOKEEPER < time(nullptr)) { - LOG_WARNING(log, "Part " << part.name << " (that was selected for mutation) with age " << (time(nullptr) - part.modification_time) << " seconds exists locally but not in ZooKeeper. Won't mutate that part and will check it."); + LOG_WARNING_FORMATTED(log, "Part {} (that was selected for mutation) with age {} seconds exists locally but not in ZooKeeper. Won't mutate that part and will check it.", part.name, (time(nullptr) - part.modification_time)); enqueuePartForCheck(part.name); } @@ -4775,7 +4775,7 @@ void StorageReplicatedMergeTree::removePartsFromZooKeeper( continue; } else - LOG_WARNING(log, "Cannot remove part " << part_names[i] << " from ZooKeeper: " << zkutil::ZooKeeper::error2string(response.error)); + LOG_WARNING_FORMATTED(log, "Cannot remove part {} from ZooKeeper: {}", part_names[i], zkutil::ZooKeeper::error2string(response.error)); } } From d5b78834514e86d11421156cb7f69881ab68505f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 22:41:08 +0300 Subject: [PATCH 649/738] find {base,src,programs} -name '*.h' -or -name '*.cpp' | xargs grep -l -P 'LOG_[^\_(]+\([^,]+, "[^"]+" << [^<]+ << "[^"]+" << [^<]+ << "[^"]+" << [^<]+ << "[^"]+"\);' | xargs sed -i -r -e 's/(LOG_[^\_(]+)\(([^,]+), "([^"]+)" << ([^<]+) << "([^"]+)" << ([^<]+) << "([^"]+)" << ([^<]+) << "([^"]+)"\);/\1_FORMATTED(\2, "\3{}\5{}\7{}\9", \4, \6, \8);/' --- programs/copier/ClusterCopier.cpp | 4 ++-- src/DataStreams/AggregatingBlockInputStream.cpp | 2 +- src/DataStreams/ParallelAggregatingBlockInputStream.cpp | 2 +- src/Databases/DatabaseOnDisk.cpp | 2 +- src/Disks/VolumeJBOD.cpp | 2 +- src/Interpreters/DDLWorker.cpp | 4 ++-- src/Interpreters/ExternalLoader.cpp | 2 +- src/Processors/Transforms/AggregatingTransform.cpp | 2 +- src/Storages/MergeTree/MergeTreePartsMover.cpp | 2 +- src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp | 6 +++--- src/Storages/StorageReplicatedMergeTree.cpp | 8 ++++---- 11 files changed, 18 insertions(+), 18 deletions(-) diff --git a/programs/copier/ClusterCopier.cpp b/programs/copier/ClusterCopier.cpp index 3e082ebef2d..ed8edd866e9 100644 --- a/programs/copier/ClusterCopier.cpp +++ b/programs/copier/ClusterCopier.cpp @@ -249,7 +249,7 @@ void ClusterCopier::process(const ConnectionTimeouts & timeouts) { for (TaskTable & task_table : task_cluster->table_tasks) { - LOG_INFO(log, "Process table task " << task_table.table_id << " with " << task_table.all_shards.size() << " shards, " << task_table.local_shards.size() << " of them are local ones"); + LOG_INFO_FORMATTED(log, "Process table task {} with {} shards, {} of them are local ones", task_table.table_id, task_table.all_shards.size(), task_table.local_shards.size()); if (task_table.all_shards.empty()) continue; @@ -1290,7 +1290,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( if (count != 0) { - LOG_INFO(log, "Partition " << task_partition.name << " piece " << current_piece_number << "is not empty. In contains " << count << " rows."); + LOG_INFO_FORMATTED(log, "Partition {} piece {}is not empty. In contains {} rows.", task_partition.name, current_piece_number, count); Coordination::Stat stat_shards{}; zookeeper->get(partition_piece.getPartitionPieceShardsPath(), &stat_shards); diff --git a/src/DataStreams/AggregatingBlockInputStream.cpp b/src/DataStreams/AggregatingBlockInputStream.cpp index 83197ef447f..1a06d18b3aa 100644 --- a/src/DataStreams/AggregatingBlockInputStream.cpp +++ b/src/DataStreams/AggregatingBlockInputStream.cpp @@ -60,7 +60,7 @@ Block AggregatingBlockInputStream::readImpl() input_streams.emplace_back(temporary_inputs.back()->block_in); } - LOG_TRACE(log, "Will merge " << files.files.size() << " temporary files of size " << (files.sum_size_compressed / 1048576.0) << " MiB compressed, " << (files.sum_size_uncompressed / 1048576.0) << " MiB uncompressed."); + LOG_TRACE_FORMATTED(log, "Will merge {} temporary files of size {} MiB compressed, {} MiB uncompressed.", files.files.size(), (files.sum_size_compressed / 1048576.0), (files.sum_size_uncompressed / 1048576.0)); impl = std::make_unique(input_streams, params, final, 1, 1); } diff --git a/src/DataStreams/ParallelAggregatingBlockInputStream.cpp b/src/DataStreams/ParallelAggregatingBlockInputStream.cpp index eb8cf912eec..b3770ae45b1 100644 --- a/src/DataStreams/ParallelAggregatingBlockInputStream.cpp +++ b/src/DataStreams/ParallelAggregatingBlockInputStream.cpp @@ -82,7 +82,7 @@ Block ParallelAggregatingBlockInputStream::readImpl() input_streams.emplace_back(temporary_inputs.back()->block_in); } - LOG_TRACE(log, "Will merge " << files.files.size() << " temporary files of size " << (files.sum_size_compressed / 1048576.0) << " MiB compressed, " << (files.sum_size_uncompressed / 1048576.0) << " MiB uncompressed."); + LOG_TRACE_FORMATTED(log, "Will merge {} temporary files of size {} MiB compressed, {} MiB uncompressed.", files.files.size(), (files.sum_size_compressed / 1048576.0), (files.sum_size_uncompressed / 1048576.0)); impl = std::make_unique( input_streams, params, final, temporary_data_merge_threads, temporary_data_merge_threads); diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index 395c05df5e3..480f03fb7b2 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -466,7 +466,7 @@ ASTPtr DatabaseOnDisk::parseQueryFromMetadata(Poco::Logger * loger, const Contex table_name = unescapeForFileName(table_name); if (create.table != TABLE_WITH_UUID_NAME_PLACEHOLDER) - LOG_WARNING(loger, "File " << metadata_file_path << " contains both UUID and table name. Will use name `" << table_name << "` instead of `" << create.table << "`"); + LOG_WARNING_FORMATTED(loger, "File {} contains both UUID and table name. Will use name `{}` instead of `{}`", metadata_file_path, table_name, create.table); create.table = table_name; } diff --git a/src/Disks/VolumeJBOD.cpp b/src/Disks/VolumeJBOD.cpp index 986d69111b3..73ea2b5ae34 100644 --- a/src/Disks/VolumeJBOD.cpp +++ b/src/Disks/VolumeJBOD.cpp @@ -52,7 +52,7 @@ VolumeJBOD::VolumeJBOD( } static constexpr UInt64 MIN_PART_SIZE = 8u * 1024u * 1024u; if (max_data_part_size != 0 && max_data_part_size < MIN_PART_SIZE) - LOG_WARNING(logger, "Volume " << backQuote(name) << " max_data_part_size is too low (" << formatReadableSizeWithBinarySuffix(max_data_part_size) << " < " << formatReadableSizeWithBinarySuffix(MIN_PART_SIZE) << ")"); + LOG_WARNING_FORMATTED(logger, "Volume {} max_data_part_size is too low ({} < {})", backQuote(name), formatReadableSizeWithBinarySuffix(max_data_part_size), formatReadableSizeWithBinarySuffix(MIN_PART_SIZE)); } DiskPtr VolumeJBOD::getNextDisk() diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index 3d6d9801591..e33e65f5879 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -334,7 +334,7 @@ bool DDLWorker::initAndCheckTask(const String & entry_name, String & out_reason, if (host_in_hostlist) { /// This check could be slow a little bit - LOG_WARNING(log, "There are two the same ClickHouse instances in task " << entry_name << ": " << task->host_id.readableString() << " and " << host.readableString() << ". Will use the first one only."); + LOG_WARNING_FORMATTED(log, "There are two the same ClickHouse instances in task {}: {} and {}. Will use the first one only.", entry_name, task->host_id.readableString(), host.readableString()); } else { @@ -506,7 +506,7 @@ void DDLWorker::parseQueryAndResolveHost(DDLTask & task) if (found_exact_match) return; - LOG_WARNING(log, "Not found the exact match of host " << task.host_id.readableString() << " from task " << task.entry_name << " in cluster " << task.cluster_name << " definition. Will try to find it using host name resolving."); + LOG_WARNING_FORMATTED(log, "Not found the exact match of host {} from task {} in cluster {} definition. Will try to find it using host name resolving.", task.host_id.readableString(), task.entry_name, task.cluster_name); bool found_via_resolving = false; for (size_t shard_num = 0; shard_num < shards.size(); ++shard_num) diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index 7157fcd58e9..5e308d5702f 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -1037,7 +1037,7 @@ private: } if (info->loading_id != loading_id) { - LOG_TRACE(log, "Next update time for '" << name << "' will not be set because this object's current loading_id " << info->loading_id << " is different from the specified " << loading_id << "."); + LOG_TRACE_FORMATTED(log, "Next update time for '{}' will not be set because this object's current loading_id {} is different from the specified {}.", name, info->loading_id, loading_id); return; } diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index bea1ca0be76..17105de9803 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -589,7 +589,7 @@ void AggregatingTransform::initGenerate() for (const auto & file : files.files) processors.emplace_back(std::make_unique(header, file->path())); - LOG_TRACE(log, "Will merge " << files.files.size() << " temporary files of size " << (files.sum_size_compressed / 1048576.0) << " MiB compressed, " << (files.sum_size_uncompressed / 1048576.0) << " MiB uncompressed."); + LOG_TRACE_FORMATTED(log, "Will merge {} temporary files of size {} MiB compressed, {} MiB uncompressed.", files.files.size(), (files.sum_size_compressed / 1048576.0), (files.sum_size_uncompressed / 1048576.0)); auto pipe = createMergingAggregatedMemoryEfficientPipe( header, params, files.files.size(), temporary_data_merge_threads); diff --git a/src/Storages/MergeTree/MergeTreePartsMover.cpp b/src/Storages/MergeTree/MergeTreePartsMover.cpp index e18bbe0fdce..464c469ad7b 100644 --- a/src/Storages/MergeTree/MergeTreePartsMover.cpp +++ b/src/Storages/MergeTree/MergeTreePartsMover.cpp @@ -179,7 +179,7 @@ bool MergeTreePartsMover::selectPartsForMove( if (!parts_to_move.empty()) { - LOG_TRACE(log, "Selected " << parts_to_move_by_policy_rules << " parts to move according to storage policy rules and " << parts_to_move_by_ttl_rules << " parts according to TTL rules, " << formatReadableSizeWithBinarySuffix(parts_to_move_total_size_bytes) << " total"); + LOG_TRACE_FORMATTED(log, "Selected {} parts to move according to storage policy rules and {} parts according to TTL rules, {} total", parts_to_move_by_policy_rules, parts_to_move_by_ttl_rules, formatReadableSizeWithBinarySuffix(parts_to_move_total_size_bytes)); return true; } else diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 2b2eadd8b69..e73f832225f 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -365,7 +365,7 @@ void ReplicatedMergeTreeQueue::removeProcessedEntry(zkutil::ZooKeeperPtr zookeep auto code = zookeeper->tryRemove(replica_path + "/queue/" + entry->znode_name); if (code) - LOG_ERROR(log, "Couldn't remove " << replica_path << "/queue/" << entry->znode_name << ": " << zkutil::ZooKeeper::error2string(code) << ". This shouldn't happen often."); + LOG_ERROR_FORMATTED(log, "Couldn't remove {}/queue/{}: {}. This shouldn't happen often.", replica_path, entry->znode_name, zkutil::ZooKeeper::error2string(code)); updateTimesInZooKeeper(zookeeper, min_unprocessed_insert_time_changed, max_processed_insert_time_changed); } @@ -1337,7 +1337,7 @@ MutationCommands ReplicatedMergeTreeQueue::getMutationCommands( auto in_partition = mutations_by_partition.find(part->info.partition_id); if (in_partition == mutations_by_partition.end()) { - LOG_WARNING(log, "There are no mutations for partition ID " << part->info.partition_id << " (trying to mutate part " << part->name << " to " << toString(desired_mutation_version) << ")"); + LOG_WARNING_FORMATTED(log, "There are no mutations for partition ID {} (trying to mutate part {} to {})", part->info.partition_id, part->name, toString(desired_mutation_version)); return MutationCommands{}; } @@ -1885,7 +1885,7 @@ bool ReplicatedMergeTreeMergePredicate::isMutationFinished(const ReplicatedMerge partition_it->second.begin(), partition_it->second.lower_bound(block_num)); if (blocks_count) { - LOG_TRACE(queue.log, "Mutation " << mutation.znode_name << " is not done yet because in partition ID " << partition_id << " there are still " << blocks_count << " uncommitted blocks."); + LOG_TRACE_FORMATTED(queue.log, "Mutation {} is not done yet because in partition ID {} there are still {} uncommitted blocks.", mutation.znode_name, partition_id, blocks_count); return false; } } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 468be2dde79..03f80206ade 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -1003,7 +1003,7 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) } if (part->name != name) { - LOG_WARNING(log, "Part " << name << " is covered by " << part->name << " but should be merged into " << entry.new_part_name << ". This shouldn't happen often."); + LOG_WARNING_FORMATTED(log, "Part {} is covered by {} but should be merged into {}. This shouldn't happen often.", name, part->name, entry.new_part_name); have_all_parts = false; break; } @@ -1370,7 +1370,7 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) } else { - LOG_WARNING(log, "No active replica has part " << entry.new_part_name << ", but that part needs quorum and /quorum/status contains entry about another part " << quorum_entry.part_name << ". It means that part was successfully written to " << entry.quorum << " replicas, but then all of them goes offline. Or it is a bug."); + LOG_WARNING_FORMATTED(log, "No active replica has part {}, but that part needs quorum and /quorum/status contains entry about another part {}. It means that part was successfully written to {} replicas, but then all of them goes offline. Or it is a bug.", entry.new_part_name, quorum_entry.part_name, entry.quorum); } } } @@ -2187,7 +2187,7 @@ void StorageReplicatedMergeTree::mergeSelectingTask() size_t merges_and_mutations_sum = merges_and_mutations_queued.first + merges_and_mutations_queued.second; if (merges_and_mutations_sum >= storage_settings_ptr->max_replicated_merges_in_queue) { - LOG_TRACE(log, "Number of queued merges (" << merges_and_mutations_queued.first << ") and part mutations (" << merges_and_mutations_queued.second << ") is greater than max_replicated_merges_in_queue (" << storage_settings_ptr->max_replicated_merges_in_queue << "), so won't select new parts to merge or mutate."); + LOG_TRACE_FORMATTED(log, "Number of queued merges ({}) and part mutations ({}) is greater than max_replicated_merges_in_queue ({}), so won't select new parts to merge or mutate.", merges_and_mutations_queued.first, merges_and_mutations_queued.second, storage_settings_ptr->max_replicated_merges_in_queue); } else { @@ -4305,7 +4305,7 @@ void StorageReplicatedMergeTree::fetchPartition(const ASTPtr & partition, const if (best_replica.empty()) throw Exception("Logical error: cannot choose best replica.", ErrorCodes::LOGICAL_ERROR); - LOG_INFO(log, "Found " << replicas.size() << " replicas, " << active_replicas.size() << " of them are active. Selected " << best_replica << " to fetch from."); + LOG_INFO_FORMATTED(log, "Found {} replicas, {} of them are active. Selected {} to fetch from.", replicas.size(), active_replicas.size(), best_replica); String best_replica_path = from + "/replicas/" + best_replica; From f13ccfc8e86cf2a71508f0505dfe61ae9a46d604 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 22:43:13 +0300 Subject: [PATCH 650/738] Progress on task --- base/daemon/BaseDaemon.cpp | 2 +- programs/benchmark/Benchmark.cpp | 8 ++++---- programs/copier/ClusterCopier.cpp | 10 +++++----- programs/copier/ClusterCopierApp.cpp | 2 +- programs/server/HTTPHandlerFactory.cpp | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index 79dc9282931..751a658ea6f 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -257,7 +257,7 @@ private: message << " (no query)"; else message << " (query_id: " << query_id << ")"; - message << " Received signal " << strsignal(sig) << " (" << sig << ")" << "."; + message << " Received signal " << strsignal(sig) << " (" << sig << ")."; LOG_FATAL(log, message.rdbuf()); } diff --git a/programs/benchmark/Benchmark.cpp b/programs/benchmark/Benchmark.cpp index 91c43160e0f..1b2867940ea 100644 --- a/programs/benchmark/Benchmark.cpp +++ b/programs/benchmark/Benchmark.cpp @@ -424,7 +424,7 @@ private: std::cerr << percent << "%\t\t"; for (const auto & info : infos) { - std::cerr << info->sampler.quantileNearest(percent / 100.0) << " sec." << "\t"; + std::cerr << info->sampler.quantileNearest(percent / 100.0) << " sec.\t"; } std::cerr << "\n"; }; @@ -459,7 +459,7 @@ private: auto print_percentile = [&json_out](Stats & info, auto percent, bool with_comma = true) { - json_out << "\"" << percent << "\"" << ": " << info.sampler.quantileNearest(percent / 100.0) << (with_comma ? ",\n" : "\n"); + json_out << "\"" << percent << "\": " << info.sampler.quantileNearest(percent / 100.0) << (with_comma ? ",\n" : "\n"); }; json_out << "{\n"; @@ -469,7 +469,7 @@ private: const auto & info = infos[i]; json_out << double_quote << connections[i]->getDescription() << ": {\n"; - json_out << double_quote << "statistics" << ": {\n"; + json_out << double_quote << "statistics: {\n"; print_key_value("QPS", info->queries / info->work_time); print_key_value("RPS", info->read_rows / info->work_time); @@ -479,7 +479,7 @@ private: print_key_value("num_queries", info->queries.load(), false); json_out << "},\n"; - json_out << double_quote << "query_time_percentiles" << ": {\n"; + json_out << double_quote << "query_time_percentiles: {\n"; for (int percent = 0; percent <= 90; percent += 10) print_percentile(*info, percent); diff --git a/programs/copier/ClusterCopier.cpp b/programs/copier/ClusterCopier.cpp index ed8edd866e9..06c910ccd3f 100644 --- a/programs/copier/ClusterCopier.cpp +++ b/programs/copier/ClusterCopier.cpp @@ -132,7 +132,7 @@ void ClusterCopier::discoverShardPartitions(const ConnectionTimeouts & timeouts, { if (!task_table.enabled_partitions_set.count(partition_name)) { - LOG_DEBUG(log, "Partition " << partition_name << " will not be processed, since it is not in " << "enabled_partitions of " << task_table.table_id); + LOG_DEBUG(log, "Partition " << partition_name << " will not be processed, since it is not in enabled_partitions of " << task_table.table_id); } } } @@ -353,7 +353,7 @@ zkutil::EphemeralNodeHolder::Ptr ClusterCopier::createTaskWorkerNodeAndWaitIfNee if (static_cast(stat.numChildren) >= task_cluster->max_workers) { - LOG_DEBUG(log, "Too many workers (" << stat.numChildren << ", maximum " << task_cluster->max_workers << ")" << ". Postpone processing " << description); + LOG_DEBUG(log, "Too many workers (" << stat.numChildren << ", maximum " << task_cluster->max_workers << "). Postpone processing " << description); if (unprioritized) current_sleep_time = std::min(max_sleep_time, current_sleep_time + default_sleep_time); @@ -1010,7 +1010,7 @@ bool ClusterCopier::tryProcessTable(const ConnectionTimeouts & timeouts, TaskTab task_table.rows_copied += cluster_partition.rows_copied; double elapsed = cluster_partition.elapsed_time_seconds; - LOG_INFO(log, "It took " << std::fixed << std::setprecision(2) << elapsed << " seconds to copy partition " << partition_name << ": " << formatReadableSizeWithDecimalSuffix(cluster_partition.bytes_copied) << " uncompressed bytes" << ", " << formatReadableQuantity(cluster_partition.rows_copied) << " rows" << " and " << cluster_partition.blocks_copied << " source blocks are copied"); + LOG_INFO(log, "It took " << std::fixed << std::setprecision(2) << elapsed << " seconds to copy partition " << partition_name << ": " << formatReadableSizeWithDecimalSuffix(cluster_partition.bytes_copied) << " uncompressed bytes, " << formatReadableQuantity(cluster_partition.rows_copied) << " rows and " << cluster_partition.blocks_copied << " source blocks are copied"); if (cluster_partition.rows_copied) { @@ -1030,7 +1030,7 @@ bool ClusterCopier::tryProcessTable(const ConnectionTimeouts & timeouts, TaskTab if (!table_is_done) { - LOG_INFO(log, "Table " + task_table.table_id + " is not processed yet." << "Copied " << finished_partitions << " of " << required_partitions << ", will retry"); + LOG_INFO(log, "Table " + task_table.table_id + " is not processed yet.Copied " << finished_partitions << " of " << required_partitions << ", will retry"); } return table_is_done; @@ -1297,7 +1297,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( /// NOTE: partition is still fresh if dirt discovery happens before cleaning if (stat_shards.numChildren == 0) { - LOG_WARNING(log, "There are no workers for partition " << task_partition.name << " piece " << toString(current_piece_number) << ", but destination table contains " << count << " rows" << ". Partition will be dropped and refilled."); + LOG_WARNING(log, "There are no workers for partition " << task_partition.name << " piece " << toString(current_piece_number) << ", but destination table contains " << count << " rows. Partition will be dropped and refilled."); create_is_dirty_node(clean_state_clock); return TaskStatus::Error; diff --git a/programs/copier/ClusterCopierApp.cpp b/programs/copier/ClusterCopierApp.cpp index f34cc9d48b6..e2a20eb965a 100644 --- a/programs/copier/ClusterCopierApp.cpp +++ b/programs/copier/ClusterCopierApp.cpp @@ -95,7 +95,7 @@ void ClusterCopierApp::mainImpl() ThreadStatus thread_status; auto * log = &logger(); - LOG_INFO(log, "Starting clickhouse-copier (" << "id " << process_id << ", " << "host_id " << host_id << ", " << "path " << process_path << ", " << "revision " << ClickHouseRevision::get() << ")"); + LOG_INFO(log, "Starting clickhouse-copier (id " << process_id << ", host_id " << host_id << ", path " << process_path << ", revision " << ClickHouseRevision::get() << ")"); SharedContextHolder shared_context = Context::createShared(); auto context = std::make_unique(Context::createGlobal(shared_context.get())); diff --git a/programs/server/HTTPHandlerFactory.cpp b/programs/server/HTTPHandlerFactory.cpp index af5900ee18e..abb1fd10be9 100644 --- a/programs/server/HTTPHandlerFactory.cpp +++ b/programs/server/HTTPHandlerFactory.cpp @@ -30,7 +30,7 @@ HTTPRequestHandlerFactoryMain::HTTPRequestHandlerFactoryMain(const std::string & Poco::Net::HTTPRequestHandler * HTTPRequestHandlerFactoryMain::createRequestHandler(const Poco::Net::HTTPServerRequest & request) { - LOG_TRACE(log, "HTTP Request for " << name << ". " << "Method: " << request.getMethod() << ", Address: " << request.clientAddress().toString() << ", User-Agent: " << (request.has("User-Agent") ? request.get("User-Agent") : "none") << (request.hasContentLength() ? (", Length: " + std::to_string(request.getContentLength())) : ("")) << ", Content Type: " << request.getContentType() << ", Transfer Encoding: " << request.getTransferEncoding()); + LOG_TRACE(log, "HTTP Request for " << name << ". Method: " << request.getMethod() << ", Address: " << request.clientAddress().toString() << ", User-Agent: " << (request.has("User-Agent") ? request.get("User-Agent") : "none") << (request.hasContentLength() ? (", Length: " + std::to_string(request.getContentLength())) : ("")) << ", Content Type: " << request.getContentType() << ", Transfer Encoding: " << request.getTransferEncoding()); for (auto & handler_factory : child_factories) { From cd6d3c75095a00b7b046ef286d7369e5df6c7f67 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 22:55:06 +0300 Subject: [PATCH 651/738] Progress on task --- base/daemon/GraphiteWriter.h | 2 +- programs/copier/ClusterCopier.cpp | 8 ++++---- programs/server/TCPHandler.cpp | 2 +- src/Client/ConnectionPoolWithFailover.cpp | 2 +- src/IO/WriteBufferFromS3.cpp | 2 +- src/Interpreters/DDLWorker.cpp | 2 +- src/Interpreters/ExternalLoader.cpp | 2 +- src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp | 2 +- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/base/daemon/GraphiteWriter.h b/base/daemon/GraphiteWriter.h index 400c99066b2..401ed709b8f 100644 --- a/base/daemon/GraphiteWriter.h +++ b/base/daemon/GraphiteWriter.h @@ -52,7 +52,7 @@ private: } catch (const Poco::Exception & e) { - LOG_WARNING(&Poco::Util::Application::instance().logger(), "Fail to write to Graphite " << host << ":" << port << ". e.what() = " << e.what() << ", e.message() = " << e.message()); + LOG_WARNING_FORMATTED(&Poco::Util::Application::instance().logger(), "Fail to write to Graphite {}:{}. e.what() = {}, e.message() = {}", host, port, e.what(), e.message()); } } diff --git a/programs/copier/ClusterCopier.cpp b/programs/copier/ClusterCopier.cpp index 06c910ccd3f..00e60355209 100644 --- a/programs/copier/ClusterCopier.cpp +++ b/programs/copier/ClusterCopier.cpp @@ -132,7 +132,7 @@ void ClusterCopier::discoverShardPartitions(const ConnectionTimeouts & timeouts, { if (!task_table.enabled_partitions_set.count(partition_name)) { - LOG_DEBUG(log, "Partition " << partition_name << " will not be processed, since it is not in enabled_partitions of " << task_table.table_id); + LOG_DEBUG_FORMATTED(log, "Partition {} will not be processed, since it is not in enabled_partitions of {}", partition_name, task_table.table_id); } } } @@ -353,7 +353,7 @@ zkutil::EphemeralNodeHolder::Ptr ClusterCopier::createTaskWorkerNodeAndWaitIfNee if (static_cast(stat.numChildren) >= task_cluster->max_workers) { - LOG_DEBUG(log, "Too many workers (" << stat.numChildren << ", maximum " << task_cluster->max_workers << "). Postpone processing " << description); + LOG_DEBUG_FORMATTED(log, "Too many workers ({}, maximum {}). Postpone processing {}", stat.numChildren, task_cluster->max_workers, description); if (unprioritized) current_sleep_time = std::min(max_sleep_time, current_sleep_time + default_sleep_time); @@ -1297,7 +1297,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( /// NOTE: partition is still fresh if dirt discovery happens before cleaning if (stat_shards.numChildren == 0) { - LOG_WARNING(log, "There are no workers for partition " << task_partition.name << " piece " << toString(current_piece_number) << ", but destination table contains " << count << " rows. Partition will be dropped and refilled."); + LOG_WARNING_FORMATTED(log, "There are no workers for partition {} piece {}, but destination table contains {} rows. Partition will be dropped and refilled.", task_partition.name, toString(current_piece_number), count); create_is_dirty_node(clean_state_clock); return TaskStatus::Error; @@ -1757,7 +1757,7 @@ bool ClusterCopier::checkPresentPartitionPiecesOnCurrentShard(const ConnectionTi query += " LIMIT 1"; - LOG_DEBUG(log, "Checking shard " << task_shard.getDescription() << " for partition " << partition_quoted_name << " piece " << std::to_string(current_piece_number) << "existence, executing query: " << query); + LOG_DEBUG_FORMATTED(log, "Checking shard {} for partition {} piece {} existence, executing query: {}", task_shard.getDescription(), partition_quoted_name, std::to_string(current_piece_number), query); ParserQuery parser_query(query.data() + query.size()); const auto & settings = context.getSettingsRef(); diff --git a/programs/server/TCPHandler.cpp b/programs/server/TCPHandler.cpp index b3e329a74e5..c2a36ed4b9d 100644 --- a/programs/server/TCPHandler.cpp +++ b/programs/server/TCPHandler.cpp @@ -1196,7 +1196,7 @@ void TCPHandler::run() /// Timeout - not an error. if (!strcmp(e.what(), "Timeout")) { - LOG_DEBUG(log, "Poco::Exception. Code: " << ErrorCodes::POCO_EXCEPTION << ", e.code() = " << e.code() << ", e.displayText() = " << e.displayText() << ", e.what() = " << e.what()); + LOG_DEBUG_FORMATTED(log, "Poco::Exception. Code: {}, e.code() = {}, e.displayText() = {}, e.what() = {}", ErrorCodes::POCO_EXCEPTION, e.code(), e.displayText(), e.what()); } else throw; diff --git a/src/Client/ConnectionPoolWithFailover.cpp b/src/Client/ConnectionPoolWithFailover.cpp index e59fab2a162..c925f6a7187 100644 --- a/src/Client/ConnectionPoolWithFailover.cpp +++ b/src/Client/ConnectionPoolWithFailover.cpp @@ -248,7 +248,7 @@ ConnectionPoolWithFailover::tryGetEntry( result.is_up_to_date = false; result.staleness = delay; - LOG_TRACE(log, "Server " << result.entry->getDescription() << " has unacceptable replica delay for table " << table_to_check->database << "." << table_to_check->table << ": " << delay); + LOG_TRACE_FORMATTED(log, "Server {} has unacceptable replica delay for table {}.{}: {}", result.entry->getDescription(), table_to_check->database, table_to_check->table, delay); ProfileEvents::increment(ProfileEvents::DistributedConnectionStaleReplica); } } diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 70cf8503254..593c3451d47 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -129,7 +129,7 @@ void WriteBufferFromS3::writePart(const String & data) auto outcome = client_ptr->UploadPart(req); - LOG_TRACE(log, "Writing part. Bucket: " << bucket << ", Key: " << key << ", Upload_id: " << upload_id << ", Data size: " << data.size()); + LOG_TRACE_FORMATTED(log, "Writing part. Bucket: {}, Key: {}, Upload_id: {}, Data size: {}", bucket, key, upload_id, data.size()); if (outcome.IsSuccess()) { diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index e33e65f5879..8836343b6be 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -542,7 +542,7 @@ void DDLWorker::parseQueryAndResolveHost(DDLTask & task) } else { - LOG_INFO(log, "Resolved host " << task.host_id.readableString() << " from task " << task.entry_name << " as host " << task.address_in_cluster.readableString() << " in definition of cluster " << task.cluster_name); + LOG_INFO_FORMATTED(log, "Resolved host {} from task {} as host {} in definition of cluster {}", task.host_id.readableString(), task.entry_name, task.address_in_cluster.readableString(), task.cluster_name); } } diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index 5e308d5702f..c097b2e7507 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -885,7 +885,7 @@ private: info.loading_start_time = std::chrono::system_clock::now(); info.loading_end_time = TimePoint{}; - LOG_TRACE(log, "Will load the object '" << info.name << "' " << (enable_async_loading ? std::string("in background") : "immediately") << ", force = " << forced_to_reload << ", loading_id = " << info.loading_id); + LOG_TRACE_FORMATTED(log, "Will load the object '{}' {}, force = {}, loading_id = {}", info.name, (enable_async_loading ? std::string("in background") : "immediately"), forced_to_reload, info.loading_id); if (enable_async_loading) { diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 86fce4c3ef6..e4d2ff520d5 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -583,7 +583,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor const MergeTreeData::DataPartsVector & parts = future_part.parts; - LOG_DEBUG(log, "Merging " << parts.size() << " parts: from " << parts.front()->name << " to " << parts.back()->name << " into " << TMP_PREFIX + future_part.name + " with type " + future_part.type.toString()); + LOG_DEBUG_FORMATTED(log, "Merging {} parts: from {} to {} into {}", parts.size(), parts.front()->name, parts.back()->name, future_part.type.toString()); auto disk = space_reservation->getDisk(); String part_path = data.relative_data_path; From 18febd7b972af21394745666ac57f7a244a90761 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 22:56:05 +0300 Subject: [PATCH 652/738] find {base,src,programs} -name '*.h' -or -name '*.cpp' | xargs grep -l -P 'LOG_[^\_(]+\([^,]+, "[^"]+" << [^<]+ << "[^"]+" << [^<]+ << "[^"]+" << [^<]+ << "[^"]+" << [^<]+ << "[^"]+"\);' | while read file; do perl -pne 's/(LOG_[^\_(]+)\(([^,]+), "([^"]+)" << ([^<]+) << "([^"]+)" << ([^<]+) << "([^"]+)" << ([^<]+) << "([^"]+)" << ([^<]+) << "([^"]+)"\);/${1}_FORMATTED(${2}, "${3}{}${5}{}${7}{}${9}{}${11}", ${4}, ${6}, ${8}, ${10});/' $file > ${file}.tmp; mv ${file}.tmp $file; done --- programs/copier/ClusterCopierApp.cpp | 2 +- src/Disks/VolumeJBOD.cpp | 2 +- src/Processors/Executors/PipelineExecutor.cpp | 2 +- src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp | 2 +- src/Storages/StorageBuffer.cpp | 6 +++--- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/programs/copier/ClusterCopierApp.cpp b/programs/copier/ClusterCopierApp.cpp index e2a20eb965a..e49be154b7f 100644 --- a/programs/copier/ClusterCopierApp.cpp +++ b/programs/copier/ClusterCopierApp.cpp @@ -95,7 +95,7 @@ void ClusterCopierApp::mainImpl() ThreadStatus thread_status; auto * log = &logger(); - LOG_INFO(log, "Starting clickhouse-copier (id " << process_id << ", host_id " << host_id << ", path " << process_path << ", revision " << ClickHouseRevision::get() << ")"); + LOG_INFO_FORMATTED(log, "Starting clickhouse-copier (id {}, host_id {}, path {}, revision {})", process_id, host_id, process_path, ClickHouseRevision::get()); SharedContextHolder shared_context = Context::createShared(); auto context = std::make_unique(Context::createGlobal(shared_context.get())); diff --git a/src/Disks/VolumeJBOD.cpp b/src/Disks/VolumeJBOD.cpp index 73ea2b5ae34..c48a9250c71 100644 --- a/src/Disks/VolumeJBOD.cpp +++ b/src/Disks/VolumeJBOD.cpp @@ -48,7 +48,7 @@ VolumeJBOD::VolumeJBOD( max_data_part_size = static_cast(sum_size * ratio / disks.size()); for (size_t i = 0; i < disks.size(); ++i) if (sizes[i] < max_data_part_size) - LOG_WARNING(logger, "Disk " << backQuote(disks[i]->getName()) << " on volume " << backQuote(config_prefix) << " have not enough space (" << formatReadableSizeWithBinarySuffix(sizes[i]) << ") for containing part the size of max_data_part_size (" << formatReadableSizeWithBinarySuffix(max_data_part_size) << ")"); + LOG_WARNING_FORMATTED(logger, "Disk {} on volume {} have not enough space ({}) for containing part the size of max_data_part_size ({})", backQuote(disks[i]->getName()), backQuote(config_prefix), formatReadableSizeWithBinarySuffix(sizes[i]), formatReadableSizeWithBinarySuffix(max_data_part_size)); } static constexpr UInt64 MIN_PART_SIZE = 8u * 1024u * 1024u; if (max_data_part_size != 0 && max_data_part_size < MIN_PART_SIZE) diff --git a/src/Processors/Executors/PipelineExecutor.cpp b/src/Processors/Executors/PipelineExecutor.cpp index ba7883f01d6..2ae621fb3b8 100644 --- a/src/Processors/Executors/PipelineExecutor.cpp +++ b/src/Processors/Executors/PipelineExecutor.cpp @@ -547,7 +547,7 @@ void PipelineExecutor::executeSingleThread(size_t thread_num, size_t num_threads #ifndef NDEBUG auto & context = executor_contexts[thread_num]; - LOG_TRACE(log, "Thread finished. Total time: " << (context->total_time_ns / 1e9) << " sec. Execution time: " << (context->execution_time_ns / 1e9) << " sec. Processing time: " << (context->processing_time_ns / 1e9) << " sec. Wait time: " << (context->wait_time_ns / 1e9) << " sec."); + LOG_TRACE_FORMATTED(log, "Thread finished. Total time: {} sec. Execution time: {} sec. Processing time: {} sec. Wait time: {} sec.", (context->total_time_ns / 1e9), (context->execution_time_ns / 1e9), (context->processing_time_ns / 1e9), (context->wait_time_ns / 1e9)); #endif } diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 3a74f8664b8..005e7364b8d 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -586,7 +586,7 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( } } - LOG_DEBUG(log, "Selected " << parts.size() << " parts by date, " << parts_with_ranges.size() << " parts by key, " << sum_marks << " marks to read from " << sum_ranges << " ranges"); + LOG_DEBUG_FORMATTED(log, "Selected {} parts by date, {} parts by key, {} marks to read from {} ranges", parts.size(), parts_with_ranges.size(), sum_marks, sum_ranges); if (parts_with_ranges.empty()) return {}; diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index 0c9f1a79a5d..39372314e2e 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -197,7 +197,7 @@ Pipes StorageBuffer::read( const auto & col = our_columns.getPhysical(column_name); if (!dst_col.type->equals(*col.type)) { - LOG_WARNING(log, "Destination table " << destination_id.getNameForLogs() << " has different type of column " << backQuoteIfNeed(column_name) << " (" << dst_col.type->getName() << " != " << col.type->getName() << "). Data from destination table are converted."); + LOG_WARNING_FORMATTED(log, "Destination table {} has different type of column {} ({} != {}). Data from destination table are converted.", destination_id.getNameForLogs(), backQuoteIfNeed(column_name), dst_col.type->getName(), col.type->getName()); header_after_adding_defaults.getByName(column_name) = ColumnWithTypeAndName(dst_col.type, column_name); } } @@ -589,7 +589,7 @@ void StorageBuffer::flushBuffer(Buffer & buffer, bool check_thresholds, bool loc ProfileEvents::increment(ProfileEvents::StorageBufferFlush); - LOG_TRACE(log, "Flushing buffer with " << rows << " rows, " << bytes << " bytes, age " << time_passed << " seconds " << (check_thresholds ? "(bg)" : "(direct)") << "."); + LOG_TRACE_FORMATTED(log, "Flushing buffer with {} rows, {} bytes, age {} seconds {}.", rows, bytes, time_passed, (check_thresholds ? "(bg)" : "(direct)")); if (!destination_id) return; @@ -653,7 +653,7 @@ void StorageBuffer::writeBlockToDestination(const Block & block, StoragePtr tabl auto column = block.getByName(dst_col.name); if (!column.type->equals(*dst_col.type)) { - LOG_WARNING(log, "Destination table " << destination_id.getNameForLogs() << " have different type of column " << backQuoteIfNeed(column.name) << " (" << dst_col.type->getName() << " != " << column.type->getName() << "). Block of data is converted."); + LOG_WARNING_FORMATTED(log, "Destination table {} have different type of column {} ({} != {}). Block of data is converted.", destination_id.getNameForLogs(), backQuoteIfNeed(column.name), dst_col.type->getName(), column.type->getName()); column.column = castColumn(column, dst_col.type); column.type = dst_col.type; } From aa5247f1fed85e64479985dc0de65621118a249d Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 21 May 2020 23:58:55 +0300 Subject: [PATCH 653/738] Get back 01281_group_by_limit_memory_tracking (without flackiness) --- ...1_group_by_limit_memory_tracking.reference | 0 .../01281_group_by_limit_memory_tracking.sh | 34 +++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 tests/queries/0_stateless/01281_group_by_limit_memory_tracking.reference create mode 100755 tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sh diff --git a/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.reference b/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sh b/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sh new file mode 100755 index 00000000000..79a731b9273 --- /dev/null +++ b/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +# Regression for MemoryTracker that had been incorrectly accounted +# (it was reseted before deallocation) +# +# For this will be used: +# - two-level group by +# - max_memory_usage_for_user +# - one users' query in background (to avoid reseting max_memory_usage_for_user) + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +set -o pipefail + +function execute_null() +{ + ${CLICKHOUSE_CLIENT} --format Null -n "$@" +} + +function execute_group_by() +{ + local opts=( + --max_memory_usage_for_user=$((150<<20)) + --max_threads=2 + ) + execute_null "${opts[@]}" <<<'SELECT uniq(number) FROM numbers_mt(toUInt64(1e6)) GROUP BY number % 5e5' +} + +execute_null <<<'SELECT sleep(3)' & +execute_group_by +# if memory accounting will be incorrect, the second query will be failed with MEMORY_LIMIT_EXCEEDED +execute_group_by +wait From f75587e562d428626d5b82dcb2f738239cc5de12 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 23:13:34 +0300 Subject: [PATCH 654/738] Progress on task --- src/Client/TimeoutSetter.cpp | 2 +- src/Common/Config/ConfigProcessor.cpp | 2 +- src/Common/Exception.cpp | 5 ++++- src/Common/SensitiveDataMasker.cpp | 6 ++---- src/Common/ShellCommand.cpp | 11 ++++------ src/Common/XDBCBridgeHelper.h | 4 ++-- src/Common/ZooKeeper/ZooKeeper.cpp | 5 ++++- .../CollapsingFinalBlockInputStream.cpp | 2 +- .../CreatingSetsBlockInputStream.cpp | 21 ++++++++++--------- src/Storages/MergeTree/MergeTreeData.cpp | 3 ++- 10 files changed, 32 insertions(+), 29 deletions(-) diff --git a/src/Client/TimeoutSetter.cpp b/src/Client/TimeoutSetter.cpp index f61c505390d..5fb0392ced3 100644 --- a/src/Client/TimeoutSetter.cpp +++ b/src/Client/TimeoutSetter.cpp @@ -35,7 +35,7 @@ TimeoutSetter::~TimeoutSetter() catch (std::exception & e) { // Sometimes catched on macos - LOG_ERROR(&Logger::get("Client"), std::string{"TimeoutSetter: Can't reset timeouts: "} + e.what()); + LOG_ERROR_FORMATTED(&Logger::get("Client"), "TimeoutSetter: Can't reset timeouts: {}", e.what()); } } } diff --git a/src/Common/Config/ConfigProcessor.cpp b/src/Common/Config/ConfigProcessor.cpp index af9cdeeb716..ca7ccaf50c1 100644 --- a/src/Common/Config/ConfigProcessor.cpp +++ b/src/Common/Config/ConfigProcessor.cpp @@ -303,7 +303,7 @@ void ConfigProcessor::doIncludesRecursive( else if (throw_on_bad_incl) throw Poco::Exception(error_msg + name); else - LOG_WARNING(log, error_msg << name); + LOG_WARNING_FORMATTED(log, "{}{}", error_msg, name); } else { diff --git a/src/Common/Exception.cpp b/src/Common/Exception.cpp index 6c8d3e02535..e7f0adb2807 100644 --- a/src/Common/Exception.cpp +++ b/src/Common/Exception.cpp @@ -125,7 +125,10 @@ void tryLogCurrentException(Poco::Logger * logger, const std::string & start_of_ { try { - LOG_ERROR(logger, start_of_message << (start_of_message.empty() ? "" : ": ") << getCurrentExceptionMessage(true)); + if (start_of_message.empty()) + LOG_ERROR_FORMATTED(logger, "{}", getCurrentExceptionMessage(true)); + else + LOG_ERROR_FORMATTED(logger, "{}: {}", start_of_message, getCurrentExceptionMessage(true)); } catch (...) { diff --git a/src/Common/SensitiveDataMasker.cpp b/src/Common/SensitiveDataMasker.cpp index 9bd692a9c64..3fad8b4fc43 100644 --- a/src/Common/SensitiveDataMasker.cpp +++ b/src/Common/SensitiveDataMasker.cpp @@ -144,15 +144,13 @@ SensitiveDataMasker::SensitiveDataMasker(const Poco::Util::AbstractConfiguration } else { - LOG_WARNING(logger, "Unused param " << config_prefix << '.' << rule); + LOG_WARNING_FORMATTED(logger, "Unused param {}.{}", config_prefix, rule); } } auto rules_count = rulesCount(); if (rules_count > 0) - { - LOG_INFO(logger, rules_count << " query masking rules loaded."); - } + LOG_INFO_FORMATTED(logger, "{} query masking rules loaded.", rules_count); } void SensitiveDataMasker::addMaskingRule( diff --git a/src/Common/ShellCommand.cpp b/src/Common/ShellCommand.cpp index 01bcea34da1..d87372fbb8d 100644 --- a/src/Common/ShellCommand.cpp +++ b/src/Common/ShellCommand.cpp @@ -61,19 +61,16 @@ ShellCommand::~ShellCommand() void ShellCommand::logCommand(const char * filename, char * const argv[]) { - std::stringstream log_message; - log_message << "Will start shell command '" << filename << "' with arguments "; + std::stringstream args; for (int i = 0; argv != nullptr && argv[i] != nullptr; ++i) { if (i > 0) - { - log_message << ", "; - } + args << ", "; /// NOTE: No escaping is performed. - log_message << "'" << argv[i] << "'"; + args << "'" << argv[i] << "'"; } - LOG_TRACE(ShellCommand::getLogger(), log_message.str()); + LOG_TRACE_FORMATTED(ShellCommand::getLogger(), "Will start shell command '{}' with arguments {}", filename, args.str()); } std::unique_ptr ShellCommand::executeImpl(const char * filename, char * const argv[], bool pipe_stdin_only, bool terminate_in_destructor) diff --git a/src/Common/XDBCBridgeHelper.h b/src/Common/XDBCBridgeHelper.h index 975ee498ae1..05176ee6aab 100644 --- a/src/Common/XDBCBridgeHelper.h +++ b/src/Common/XDBCBridgeHelper.h @@ -148,7 +148,7 @@ public: { if (!checkBridgeIsRunning()) { - LOG_TRACE(log, BridgeHelperMixin::serviceAlias() + " is not running, will try to start it"); + LOG_TRACE_FORMATTED(log, "{} is not running, will try to start it", BridgeHelperMixin::serviceAlias()); startBridge(); bool started = false; @@ -157,7 +157,7 @@ public: while (milliseconds_to_wait < 10000) { ++counter; - LOG_TRACE(log, "Checking " + BridgeHelperMixin::serviceAlias() + " is running, try " << counter); + LOG_TRACE_FORMATTED(log, "Checking {} is running, try {}", BridgeHelperMixin::serviceAlias(), counter); if (checkBridgeIsRunning()) { started = true; diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index a6040a6ed5b..6bb63f88030 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -95,7 +95,10 @@ void ZooKeeper::init(const std::string & implementation_, const std::string & ho Poco::Timespan(0, ZOOKEEPER_CONNECTION_TIMEOUT_MS * 1000), Poco::Timespan(0, operation_timeout_ms_ * 1000)); - LOG_TRACE(log, "initialized, hosts: " << hosts << (chroot.empty() ? "" : ", chroot: " + chroot)); + if (chroot.empty()) + LOG_TRACE_FORMATTED(log, "Initialized, hosts: {}", hosts); + else + LOG_TRACE_FORMATTED(log, "Initialized, hosts: {}, chroot: {}", hosts, chroot); } else if (implementation == "testkeeper") { diff --git a/src/DataStreams/CollapsingFinalBlockInputStream.cpp b/src/DataStreams/CollapsingFinalBlockInputStream.cpp index 2c359f81b7c..27aaca37073 100644 --- a/src/DataStreams/CollapsingFinalBlockInputStream.cpp +++ b/src/DataStreams/CollapsingFinalBlockInputStream.cpp @@ -23,7 +23,7 @@ void CollapsingFinalBlockInputStream::reportBadCounts() void CollapsingFinalBlockInputStream::reportBadSign(Int8 sign) { - LOG_ERROR(log, "Invalid sign: " << static_cast(sign)); + LOG_ERROR_FORMATTED(log, "Invalid sign: {}", static_cast(sign)); } void CollapsingFinalBlockInputStream::fetchNextBlock(size_t input_index) diff --git a/src/DataStreams/CreatingSetsBlockInputStream.cpp b/src/DataStreams/CreatingSetsBlockInputStream.cpp index 2a8e6c216a7..5cd42296ea0 100644 --- a/src/DataStreams/CreatingSetsBlockInputStream.cpp +++ b/src/DataStreams/CreatingSetsBlockInputStream.cpp @@ -90,7 +90,13 @@ void CreatingSetsBlockInputStream::createAll() void CreatingSetsBlockInputStream::createOne(SubqueryForSet & subquery) { - LOG_TRACE(log, (subquery.set ? "Creating set. " : "") << (subquery.join ? "Creating join. " : "") << (subquery.table ? "Filling temporary table. " : "")); + if (subquery.set) + LOG_TRACE_FORMATTED(log, "Creating set."); + if (subquery.join) + LOG_TRACE_FORMATTED(log, "Creating join."); + if (subquery.table) + LOG_TRACE_FORMATTED(log, "Filling temporary table."); + Stopwatch watch; BlockOutputStreamPtr table_out; @@ -163,19 +169,14 @@ void CreatingSetsBlockInputStream::createOne(SubqueryForSet & subquery) if (head_rows != 0) { - std::stringstream msg; - msg << std::fixed << std::setprecision(3); - msg << "Created. "; + auto seconds = watch.elapsedSeconds(); if (subquery.set) - msg << "Set with " << subquery.set->getTotalRowCount() << " entries from " << head_rows << " rows. "; + LOG_DEBUG_FORMATTED(log, "Created Set with {} entries from {} rows in {} sec.", subquery.set->getTotalRowCount(), head_rows, seconds); if (subquery.join) - msg << "Join with " << subquery.join->getTotalRowCount() << " entries from " << head_rows << " rows. "; + LOG_DEBUG_FORMATTED(log, "Created Join with {} entries from {} rows in {} sec.", subquery.join->getTotalRowCount(), head_rows, seconds); if (subquery.table) - msg << "Table with " << head_rows << " rows. "; - - msg << "In " << watch.elapsedSeconds() << " sec."; - LOG_DEBUG(log, msg.rdbuf()); + LOG_DEBUG_FORMATTED(log, "Created Table with {} rows in {} sec.", head_rows, seconds); } else { diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 25a4fdd255e..aeb8930f8fb 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -2439,7 +2439,8 @@ void MergeTreeData::removePartContributionToColumnSizes(const DataPartPtr & part auto log_subtract = [&](size_t & from, size_t value, const char * field) { if (value > from) - LOG_ERROR(log, "Possibly incorrect column size subtraction: " << from << " - " << value << " = " << from - value << ", column: " << column.name << ", field: " << field); + LOG_ERROR_FORMATTED(log, "Possibly incorrect column size subtraction: {} - {} = {}, column: {}, field: {}", + from, value, from - value, column.name, field); from -= value; }; From 9de1a231c9b003c306570cedbf380b4bff922294 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 23 May 2020 21:14:52 +0300 Subject: [PATCH 655/738] Add optional default_port argument for the port() function --- .../sql-reference/functions/url-functions.md | 4 +- src/Functions/URL/port.cpp | 110 ++++++++++++------ .../queries/0_stateless/01284_port.reference | 3 + tests/queries/0_stateless/01284_port.sql | 9 ++ 4 files changed, 86 insertions(+), 40 deletions(-) diff --git a/docs/en/sql-reference/functions/url-functions.md b/docs/en/sql-reference/functions/url-functions.md index f35c621a6e2..34477529649 100644 --- a/docs/en/sql-reference/functions/url-functions.md +++ b/docs/en/sql-reference/functions/url-functions.md @@ -117,9 +117,9 @@ Returns the part of the domain that includes top-level subdomains up to the “f For example, `cutToFirstSignificantSubdomain('https://news.yandex.com.tr/') = 'yandex.com.tr'`. -### port(URL) {#port} +### port(URL[, default_port = 0]) {#port} -Returns the port or zero if there is no port in the URL (or in case of validation error). +Returns the port or `default_port` if there is no port in the URL (or in case of validation error). ### path {#path} diff --git a/src/Functions/URL/port.cpp b/src/Functions/URL/port.cpp index d4876e9f647..e703da2602b 100644 --- a/src/Functions/URL/port.cpp +++ b/src/Functions/URL/port.cpp @@ -1,6 +1,10 @@ #include -#include +#include #include +#include +#include +#include +#include #include "domain.h" @@ -9,83 +13,113 @@ namespace DB namespace ErrorCodes { + extern const int ILLEGAL_COLUMN; extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } -struct ExtractPort +struct FunctionPort : public IFunction { static constexpr auto name = "port"; - static constexpr auto is_fixed_to_constant = true; + static FunctionPtr create(const Context &) { return std::make_shared(); } - static void vector(const ColumnString::Chars & data, const ColumnString::Offsets & offsets, PaddedPODArray & res) + String getName() const override { return name; } + bool isVariadic() const override { return true; } + size_t getNumberOfArguments() const override { return 0; } + bool useDefaultImplementationForConstants() const override { return true; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + if (arguments.size() != 1 && arguments.size() != 2) + throw Exception("Number of arguments for function " + getName() + " doesn't match: passed " + + std::to_string(arguments.size()) + ", should be 1 or 2", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + if (!WhichDataType(arguments[0].type).isString()) + throw Exception("Illegal type " + arguments[0].type->getName() + " of first argument of function " + getName() + ". Must be String.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + if (arguments.size() == 2 && !WhichDataType(arguments[1].type).isUInt16()) + throw Exception("Illegal type " + arguments[1].type->getName() + " of second argument of function " + getName() + ". Must be UInt16.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + return std::make_shared(); + } + + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t) override + { + UInt16 default_port = 0; + if (arguments.size() == 2) + { + const auto * port_column = checkAndGetColumn(block.getByPosition(arguments[1]).column.get()); + if (!port_column) + throw Exception("Second argument for function " + getName() + " must be constant UInt16", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + default_port = port_column->getValue(); + } + + const ColumnPtr url_column = block.getByPosition(arguments[0]).column; + if (const ColumnString * url_strs = checkAndGetColumn(url_column.get())) + { + auto col_res = ColumnVector::create(); + typename ColumnVector::Container & vec_res = col_res->getData(); + vec_res.resize(url_column->size()); + + vector(default_port, url_strs->getChars(), url_strs->getOffsets(), vec_res); + block.getByPosition(result).column = std::move(col_res); + } + else + throw Exception( + "Illegal column " + block.getByPosition(arguments[0]).column->getName() + " of argument of function " + getName(), + ErrorCodes::ILLEGAL_COLUMN); +} + +private: + static void vector(UInt16 default_port, const ColumnString::Chars & data, const ColumnString::Offsets & offsets, PaddedPODArray & res) { size_t size = offsets.size(); ColumnString::Offset prev_offset = 0; for (size_t i = 0; i < size; ++i) { - res[i] = parse(data, prev_offset, offsets[i] - prev_offset - 1); + res[i] = extractPort(default_port, data, prev_offset, offsets[i] - prev_offset - 1); prev_offset = offsets[i]; } - } +} - static void vectorFixedToConstant(const ColumnString::Chars & data, size_t n, UInt16 & res) { res = parse(data, 0, n); } - - static void vectorFixedToVector(const ColumnString::Chars & data, size_t n, PaddedPODArray & res) - { - size_t size = data.size() / n; - - for (size_t i = 0; i < size; ++i) - { - res[i] = parse(data, i * n, n); - } - } - - [[noreturn]] static void array(const ColumnString::Offsets &, PaddedPODArray &) - { - throw Exception("Cannot apply function port to Array argument", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - } - -private: - static UInt16 parse(const ColumnString::Chars & buf, size_t offset, size_t size) + static UInt16 extractPort(UInt16 default_port, const ColumnString::Chars & buf, size_t offset, size_t size) { const char * p = reinterpret_cast(&buf[0]) + offset; const char * end = p + size; StringRef host = getURLHost(p, size); if (!host.size) - return 0; + return default_port; if (host.size == size) - return 0; + return default_port; p = host.data + host.size; if (*p++ != ':') - return 0; + return default_port; - Int64 port = 0; + Int64 port = default_port; while (p < end) { if (*p == '/') break; if (!isNumericASCII(*p)) - return 0; + return default_port; port = (port * 10) + (*p - '0'); if (port < 0 || port > UInt16(-1)) - return 0; + return default_port; ++p; } return port; } }; -struct NamePort -{ - static constexpr auto name = "port"; -}; - -using FunctionPort = FunctionStringOrArrayToT; - void registerFunctionPort(FunctionFactory & factory) { factory.registerFunction(); diff --git a/tests/queries/0_stateless/01284_port.reference b/tests/queries/0_stateless/01284_port.reference index 5bf1c035ca8..7e776595065 100644 --- a/tests/queries/0_stateless/01284_port.reference +++ b/tests/queries/0_stateless/01284_port.reference @@ -10,6 +10,9 @@ hostname 80 80 80 +default-port +80 +80 ipv6 0 0 diff --git a/tests/queries/0_stateless/01284_port.sql b/tests/queries/0_stateless/01284_port.sql index 3ba5aecd691..9c31a5d42ad 100644 --- a/tests/queries/0_stateless/01284_port.sql +++ b/tests/queries/0_stateless/01284_port.sql @@ -11,6 +11,15 @@ select port('http://foobar.com:80/'); select port('//foobar.com:80/'); select port('foobar.com:80'); +select 'default-port'; +select port('http://127.0.0.1/', toUInt16(80)); +select port('http://foobar.com/', toUInt16(80)); + +-- unsupported +/* ILLEGAL_TYPE_OF_ARGUMENT */ select port(toFixedString('', 1)); -- { serverError 43; } +/* ILLEGAL_TYPE_OF_ARGUMENT */ select port('', 1); -- { serverError 43; } +/* NUMBER_OF_ARGUMENTS_DOESNT_MATCH */ select port('', 1, 1); -- { serverError 42; } + -- -- Known limitations of domain() (getURLHost()) -- From 85f84550bac8aecec739578237bcf51b96b1a28b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 23:37:37 +0300 Subject: [PATCH 656/738] Progress on task --- src/Storages/Distributed/DirectoryMonitor.cpp | 6 +-- .../Kafka/ReadBufferFromKafkaConsumer.cpp | 11 +++--- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 6 +-- src/Storages/MergeTree/MergeTreeData.cpp | 26 ++++++------- .../MergeTree/MergeTreeWhereOptimizer.cpp | 2 +- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 4 +- src/Storages/StorageDistributed.cpp | 2 +- src/Storages/StorageMergeTree.cpp | 5 ++- src/Storages/StorageReplicatedMergeTree.cpp | 39 ++++++++++++------- 9 files changed, 56 insertions(+), 45 deletions(-) diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp index 40980d1830a..65048625d91 100644 --- a/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/src/Storages/Distributed/DirectoryMonitor.cpp @@ -261,7 +261,7 @@ bool StorageDistributedDirectoryMonitor::processFiles() void StorageDistributedDirectoryMonitor::processFile(const std::string & file_path) { - LOG_TRACE(log, "Started processing `" << file_path << '`'); + LOG_TRACE(log, "Started processing `" << file_path << "`"); auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(storage.global_context->getSettingsRef()); auto connection = pool->get(timeouts); @@ -290,7 +290,7 @@ void StorageDistributedDirectoryMonitor::processFile(const std::string & file_pa Poco::File{file_path}.remove(); - LOG_TRACE(log, "Finished processing `" << file_path << '`'); + LOG_TRACE(log, "Finished processing `" << file_path << "`"); } void StorageDistributedDirectoryMonitor::readHeader( @@ -692,7 +692,7 @@ void StorageDistributedDirectoryMonitor::markAsBroken(const std::string & file_p Poco::File{broken_path}.createDirectory(); Poco::File{file_path}.renameTo(broken_file_path); - LOG_ERROR(log, "Renamed `" << file_path << "` to `" << broken_file_path << '`'); + LOG_ERROR(log, "Renamed `" << file_path << "` to `" << broken_file_path << "`"); } bool StorageDistributedDirectoryMonitor::maybeMarkAsBroken(const std::string & file_path, const Exception & e) const diff --git a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp index 45d5edce480..0421c706e12 100644 --- a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp +++ b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp @@ -104,11 +104,11 @@ void ReadBufferFromKafkaConsumer::commit() if (topic_part.get_offset() < 0) { - LOG_TRACE(log, prefix << " " << print_special_offset() << " (topic: " << topic_part.get_topic() << ", partition: " << topic_part.get_partition() << ")"); + LOG_TRACE_FORMATTED(log, "{} {} (topic: {}, partition: {})", prefix, print_special_offset(), topic_part.get_topic(), topic_part.get_partition()); } else { - LOG_TRACE(log, prefix << " " << topic_part.get_offset() << " (topic: " << topic_part.get_topic() << ", partition: " << topic_part.get_partition() << ")"); + LOG_TRACE_FORMATTED(log, "{} {} (topic: {}, partition: {})", prefix, topic_part.get_offset(), topic_part.get_topic(), topic_part.get_partition()); } } }; @@ -117,7 +117,7 @@ void ReadBufferFromKafkaConsumer::commit() if (hasMorePolledMessages()) { - LOG_WARNING(log,"Logical error. Non all polled messages were processed."); + LOG_WARNING_FORMATTED(log, "Logical error. Non all polled messages were processed."); } if (offsets_stored > 0) @@ -166,9 +166,8 @@ void ReadBufferFromKafkaConsumer::commit() void ReadBufferFromKafkaConsumer::subscribe() { - LOG_TRACE(log,"Already subscribed to topics: [ " << boost::algorithm::join(consumer->get_subscription(), ", ") << " ]"); - - LOG_TRACE_FORMATTED(log, "Already assigned to : {}", assignment); + LOG_TRACE_FORMATTED(log, "Already subscribed to topics: [{}]", boost::algorithm::join(consumer->get_subscription(), ", ")); + LOG_TRACE_FORMATTED(log, "Already assigned to: {}", assignment); size_t max_retries = 5; diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index a5f831a983e..2f1552801d5 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -710,7 +710,7 @@ void IMergeTreeDataPart::remove() const if (volume->getDisk()->exists(to)) { - LOG_WARNING(storage.log, "Directory " << fullPath(volume->getDisk(), to) << " (to which part must be renamed before removing) already exists." " Most likely this is due to unclean restart. Removing it."); + LOG_WARNING(storage.log, "Directory " << fullPath(volume->getDisk(), to) << " (to which part must be renamed before removing) already exists. Most likely this is due to unclean restart. Removing it."); try { @@ -729,7 +729,7 @@ void IMergeTreeDataPart::remove() const } catch (const Poco::FileNotFoundException &) { - LOG_ERROR(storage.log, "Directory " << fullPath(volume->getDisk(), to) << " (part to remove) doesn't exist or one of nested files has gone." " Most likely this is due to manual removing. This should be discouraged. Ignoring."); + LOG_ERROR(storage.log, "Directory " << fullPath(volume->getDisk(), to) << " (part to remove) doesn't exist or one of nested files has gone. Most likely this is due to manual removing. This should be discouraged. Ignoring."); return; } @@ -791,7 +791,7 @@ String IMergeTreeDataPart::getRelativePathForDetachedPart(const String & prefix) if (!volume->getDisk()->exists(getFullRelativePath() + res)) return res; - LOG_WARNING(storage.log, "Directory " << res << " (to detach to) already exists." " Will detach to directory with '_tryN' suffix."); + LOG_WARNING(storage.log, "Directory " << res << " (to detach to) already exists. Will detach to directory with '_tryN' suffix."); } return res; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index aeb8930f8fb..84dd84f7068 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -244,7 +244,7 @@ MergeTreeData::MergeTreeData( String reason; if (!canUsePolymorphicParts(*settings, &reason) && !reason.empty()) - LOG_WARNING(log, reason + " Settings 'min_bytes_for_wide_part' and 'min_bytes_for_wide_part' will be ignored."); + LOG_WARNING_FORMATTED(log, "{} Settings 'min_bytes_for_wide_part' and 'min_bytes_for_wide_part' will be ignored.", reason); } @@ -937,7 +937,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) String marker_path = part_path + "/" + DELETE_ON_DESTROY_MARKER_PATH; if (part_disk_ptr->exists(marker_path)) { - LOG_WARNING(log, "Detaching stale part " << getFullPathOnDisk(part_disk_ptr) << part_name << ", which should have been deleted after a move. That can only happen after unclean restart of ClickHouse after move of a part having an operation blocking that stale copy of part."); + LOG_WARNING_FORMATTED(log, "Detaching stale part {}{}, which should have been deleted after a move. That can only happen after unclean restart of ClickHouse after move of a part having an operation blocking that stale copy of part.", getFullPathOnDisk(part_disk_ptr), part_name); std::lock_guard loading_lock(mutex); broken_parts_to_detach.push_back(part); ++suspicious_broken_parts; @@ -971,7 +971,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) if (part->info.level == 0) { /// It is impossible to restore level 0 parts. - LOG_ERROR(log, "Considering to remove broken part " << getFullPathOnDisk(part_disk_ptr) << part_name << " because it's impossible to repair."); + LOG_ERROR_FORMATTED(log, "Considering to remove broken part {}{} because it's impossible to repair.", getFullPathOnDisk(part_disk_ptr), part_name); std::lock_guard loading_lock(mutex); broken_parts_to_remove.push_back(part); } @@ -982,7 +982,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) /// delete it. size_t contained_parts = 0; - LOG_ERROR(log, "Part " << getFullPathOnDisk(part_disk_ptr) << part_name << " is broken. Looking for parts to replace it."); + LOG_ERROR_FORMATTED(log, "Part {}{} is broken. Looking for parts to replace it.", getFullPathOnDisk(part_disk_ptr), part_name); for (const auto & [contained_name, contained_disk_ptr] : part_names_with_disks) { @@ -995,20 +995,20 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) if (part->info.contains(contained_part_info)) { - LOG_ERROR(log, "Found part " << getFullPathOnDisk(contained_disk_ptr) << contained_name); + LOG_ERROR_FORMATTED(log, "Found part {}{}", getFullPathOnDisk(contained_disk_ptr), contained_name); ++contained_parts; } } if (contained_parts >= 2) { - LOG_ERROR(log, "Considering to remove broken part " << getFullPathOnDisk(part_disk_ptr) << part_name << " because it covers at least 2 other parts"); + LOG_ERROR_FORMATTED(log, "Considering to remove broken part {}{} because it covers at least 2 other parts", getFullPathOnDisk(part_disk_ptr), part_name); std::lock_guard loading_lock(mutex); broken_parts_to_remove.push_back(part); } else { - LOG_ERROR(log, "Detaching broken part " << getFullPathOnDisk(part_disk_ptr) << part_name << " because it covers less than 2 parts. You need to resolve this manually"); + LOG_ERROR_FORMATTED(log, "Detaching broken part {}{} because it covers less than 2 parts. You need to resolve this manually", getFullPathOnDisk(part_disk_ptr), part_name); std::lock_guard loading_lock(mutex); broken_parts_to_detach.push_back(part); ++suspicious_broken_parts; @@ -1948,7 +1948,7 @@ MergeTreeData::DataPartsVector MergeTreeData::removePartsInRangeFromWorkingSet(c if (!skip_intersecting_parts) throw Exception(error, ErrorCodes::LOGICAL_ERROR); - LOG_WARNING(log, error); + LOG_WARNING_FORMATTED(log, error); } continue; @@ -1965,7 +1965,7 @@ MergeTreeData::DataPartsVector MergeTreeData::removePartsInRangeFromWorkingSet(c if (!skip_intersecting_parts) throw Exception(error, ErrorCodes::LOGICAL_ERROR); - LOG_WARNING(log, error); + LOG_WARNING_FORMATTED(log, error); continue; } @@ -1981,7 +1981,7 @@ MergeTreeData::DataPartsVector MergeTreeData::removePartsInRangeFromWorkingSet(c void MergeTreeData::forgetPartAndMoveToDetached(const MergeTreeData::DataPartPtr & part_to_detach, const String & prefix, bool restore_covered) { - LOG_INFO(log, "Renaming " << part_to_detach->relative_path << " to " << prefix << part_to_detach->name << " and forgiving it."); + LOG_INFO_FORMATTED(log, "Renaming {} to {}{} and forgiving it.", part_to_detach->relative_path, prefix, part_to_detach->name); auto lock = lockParts(); @@ -2241,7 +2241,7 @@ void MergeTreeData::delayInsertOrThrowIfNeeded(Poco::Event * until) const CurrentMetrics::Increment metric_increment(CurrentMetrics::DelayedInserts); - LOG_INFO(log, "Delaying inserting block by " << std::fixed << std::setprecision(4) << delay_milliseconds << " ms. because there are " << parts_count_in_partition << " parts"); + LOG_INFO_FORMATTED(log, "Delaying inserting block by {} ms. because there are {} parts", delay_milliseconds, parts_count_in_partition); if (until) until->tryWait(delay_milliseconds); @@ -2319,7 +2319,7 @@ void MergeTreeData::swapActivePart(MergeTreeData::DataPartPtr part_copy) } catch (Poco::Exception & e) { - LOG_ERROR(log, e.what() << " (while creating DeleteOnDestroy marker: " + backQuote(fullPath(disk, marker_path)) + ")"); + LOG_ERROR_FORMATTED(log, "{} (while creating DeleteOnDestroy marker: {})", e.what(), backQuote(fullPath(disk, marker_path))); } return; } @@ -2818,7 +2818,7 @@ MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const name_to_disk[name] = disk; } } - LOG_DEBUG(log, active_parts.size() << " of them are active"); + LOG_DEBUG_FORMATTED(log, "{} of them are active", active_parts.size()); /// Inactive parts rename so they can not be attached in case of repeated ATTACH. for (const auto & [name, disk] : name_to_disk) { diff --git a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp index 623da6318ff..b7388f52340 100644 --- a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp +++ b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp @@ -238,7 +238,7 @@ void MergeTreeWhereOptimizer::optimize(ASTSelectQuery & select) const select.setExpression(ASTSelectQuery::Expression::WHERE, reconstruct(where_conditions)); select.setExpression(ASTSelectQuery::Expression::PREWHERE, reconstruct(prewhere_conditions)); - LOG_DEBUG(log, "MergeTreeWhereOptimizer: condition \"" << select.prewhere() << "\" moved to PREWHERE"); + LOG_DEBUG_FORMATTED(log, "MergeTreeWhereOptimizer: condition \"{}\" moved to PREWHERE", select.prewhere()); } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index e73f832225f..4389bca586e 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -859,7 +859,7 @@ void ReplicatedMergeTreeQueue::removePartProducingOpsInRange( updateTimesInZooKeeper(zookeeper, min_unprocessed_insert_time_changed, max_processed_insert_time_changed); - LOG_DEBUG(log, "Removed " << removed_entries << " entries from queue. " "Waiting for " << to_wait.size() << " entries that are currently executing."); + LOG_DEBUG(log, "Removed " << removed_entries << " entries from queue. Waiting for " << to_wait.size() << " entries that are currently executing."); /// Let's wait for the operations with the parts contained in the range to be deleted. for (LogEntryPtr & entry : to_wait) @@ -1328,7 +1328,7 @@ MutationCommands ReplicatedMergeTreeQueue::getMutationCommands( if (part->info.getDataVersion() > desired_mutation_version) { - LOG_WARNING(log, "Data version of part " << part->name << " is already greater than " "desired mutation version " << desired_mutation_version); + LOG_WARNING(log, "Data version of part " << part->name << " is already greater than desired mutation version " << desired_mutation_version); return MutationCommands{}; } diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 771ec0abaca..f3e5e0a837c 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -482,7 +482,7 @@ Pipes StorageDistributed::read( } else { - LOG_DEBUG(log, (has_sharding_key ? "" : " (no sharding key)") << ": " "Unable to figure out irrelevant shards from WHERE/PREWHERE clauses - " "the query will be sent to all shards of the cluster"); + LOG_DEBUG_FORMATTED(log, "Unable to figure out irrelevant shards from WHERE/PREWHERE clauses - the query will be sent to all shards of the cluster{}", has_sharding_key ? "" : " (no sharding key)"); } } diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index f83b0c61d29..7e652deceb6 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -1055,7 +1055,10 @@ void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, cons } } - LOG_INFO(log, (detach ? "Detached " : "Removed ") << parts_to_remove.size() << " parts inside partition ID " << partition_id << "."); + if (detach) + LOG_INFO(log, "Detached " << parts_to_remove.size() << " parts inside partition ID " << partition_id << "."); + else + LOG_INFO(log, "Removed " << parts_to_remove.size() << " parts inside partition ID " << partition_id << "."); } clearOldPartsFromFilesystem(); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 03f80206ade..29df2bf7b2b 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -325,7 +325,7 @@ void StorageReplicatedMergeTree::waitMutationToFinishOnReplicas( for (const String & replica : replicas) { - LOG_DEBUG(log, "Waiting for " << replica << " to apply mutation " + mutation_id); + LOG_DEBUG(log, "Waiting for " << replica << " to apply mutation " << mutation_id); while (!partial_shutdown_called) { @@ -342,7 +342,7 @@ void StorageReplicatedMergeTree::waitMutationToFinishOnReplicas( /// Replica could be inactive. if (!zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/is_active")) { - LOG_WARNING(log, "Replica " << replica << " is not active during mutation. " "Mutation will be done asynchronously when replica becomes active."); + LOG_WARNING(log, "Replica " << replica << " is not active during mutation. Mutation will be done asynchronously when replica becomes active."); inactive_replicas.emplace(replica); break; @@ -354,7 +354,7 @@ void StorageReplicatedMergeTree::waitMutationToFinishOnReplicas( /// Replica could be removed if (!zookeeper->tryGet(mutation_pointer, mutation_pointer_value, &get_stat, wait_event)) { - LOG_WARNING(log, replica << " was removed"); + LOG_WARNING_FORMATTED(log, "Replica {} was removed", replica); break; } else if (mutation_pointer_value >= mutation_id) /// Maybe we already processed more fresh mutation @@ -683,7 +683,7 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks) } if (unexpected_parts_nonnew_rows > 0) - LOG_WARNING(log, sanity_report.str()); + LOG_WARNING_FORMATTED(log, sanity_report.str()); /// Add to the queue jobs to pick up the missing parts from other replicas and remove from ZK the information that we have them. std::vector> exists_futures; @@ -730,7 +730,7 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks) /// Remove extra local parts. for (const DataPartPtr & part : unexpected_parts) { - LOG_ERROR(log, "Renaming unexpected part " << part->name << " to ignored_" + part->name); + LOG_ERROR(log, "Renaming unexpected part " << part->name << " to ignored_" << part->name); forgetPartAndMoveToDetached(part, "ignored", true); } } @@ -923,7 +923,7 @@ bool StorageReplicatedMergeTree::executeLogEntry(LogEntry & entry) { if (!(entry.type == LogEntry::GET_PART && entry.source_replica == replica_name)) { - LOG_DEBUG(log, "Skipping action for part " << entry.new_part_name << " because part " + existing_part->name + " already exists."); + LOG_DEBUG(log, "Skipping action for part " << entry.new_part_name << " because part " << existing_part->name << " already exists."); } return true; } @@ -1095,7 +1095,7 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) ProfileEvents::increment(ProfileEvents::DataAfterMergeDiffersFromReplica); - LOG_ERROR(log, getCurrentExceptionMessage(false) << ". " "Data after merge is not byte-identical to data on another replicas. " "There could be several reasons: " "1. Using newer version of compression library after server update. " "2. Using another compression method. " "3. Non-deterministic compression algorithm (highly unlikely). " "4. Non-deterministic merge algorithm due to logical error in code. " "5. Data corruption in memory due to bug in code. " "6. Data corruption in memory due to hardware issue. " "7. Manual modification of source data after server startup. " "8. Manual modification of checksums stored in ZooKeeper. " "9. Part format related settings like 'enable_mixed_granularity_parts' are different on different replicas. " "We will download merged part from replica to force byte-identical result."); + LOG_ERROR_FORMATTED(log, "{}. Data after merge is not byte-identical to data on another replicas. There could be several reasons: 1. Using newer version of compression library after server update. 2. Using another compression method. 3. Non-deterministic compression algorithm (highly unlikely). 4. Non-deterministic merge algorithm due to logical error in code. 5. Data corruption in memory due to bug in code. 6. Data corruption in memory due to hardware issue. 7. Manual modification of source data after server startup. 8. Manual modification of checksums stored in ZooKeeper. 9. Part format related settings like 'enable_mixed_granularity_parts' are different on different replicas. We will download merged part from replica to force byte-identical result.", getCurrentExceptionMessage(false)); write_part_log(ExecutionStatus::fromCurrentException()); @@ -1138,7 +1138,7 @@ bool StorageReplicatedMergeTree::tryExecutePartMutation(const StorageReplicatedM DataPartPtr source_part = getActiveContainingPart(source_part_name); if (!source_part) { - LOG_DEBUG(log, "Source part " + source_part_name + " for " << entry.new_part_name << " is not ready; will try to fetch it instead"); + LOG_DEBUG(log, "Source part " << source_part_name << " for " << entry.new_part_name << " is not ready; will try to fetch it instead"); return false; } @@ -1217,7 +1217,7 @@ bool StorageReplicatedMergeTree::tryExecutePartMutation(const StorageReplicatedM ProfileEvents::increment(ProfileEvents::DataAfterMutationDiffersFromReplica); - LOG_ERROR(log, getCurrentExceptionMessage(false) << ". " "Data after mutation is not byte-identical to data on another replicas. " "We will download merged part from replica to force byte-identical result."); + LOG_ERROR_FORMATTED(log, "{}. Data after mutation is not byte-identical to data on another replicas. We will download merged part from replica to force byte-identical result.", getCurrentExceptionMessage(false)); write_part_log(ExecutionStatus::fromCurrentException()); @@ -1288,7 +1288,7 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) if (entry.type != LogEntry::GET_PART) throw Exception("Logical error: log entry with quorum but type is not GET_PART", ErrorCodes::LOGICAL_ERROR); - LOG_DEBUG(log, "No active replica has part " << entry.new_part_name << " which needs to be written with quorum." " Will try to mark that quorum as failed."); + LOG_DEBUG(log, "No active replica has part " << entry.new_part_name << " which needs to be written with quorum. Will try to mark that quorum as failed."); /** Atomically: * - if replicas do not become active; @@ -1438,7 +1438,10 @@ void StorageReplicatedMergeTree::executeDropRange(const LogEntry & entry) auto drop_range_info = MergeTreePartInfo::fromPartName(entry.new_part_name, format_version); queue.removePartProducingOpsInRange(getZooKeeper(), drop_range_info, entry); - LOG_DEBUG(log, (entry.detach ? "Detaching" : "Removing") << " parts."); + if (entry.detach) + LOG_DEBUG_FORMATTED(log, "Detaching parts."); + else + LOG_DEBUG_FORMATTED(log, "Removing parts."); /// Delete the parts contained in the range to be deleted. /// It's important that no old parts remain (after the merge), because otherwise, @@ -1465,7 +1468,10 @@ void StorageReplicatedMergeTree::executeDropRange(const LogEntry & entry) /// Forcibly remove parts from ZooKeeper tryRemovePartsFromZooKeeperWithRetries(parts_to_remove); - LOG_INFO(log, (entry.detach ? "Detached " : "Removed ") << parts_to_remove.size() << " parts inside " << entry.new_part_name << "."); + if (entry.detach) + LOG_DEBUG_FORMATTED(log, "Detached {} parts inside {}.", parts_to_remove.size(), entry.new_part_name); + else + LOG_DEBUG_FORMATTED(log, "Removed {} parts inside {}.", parts_to_remove.size(), entry.new_part_name); /// We want to remove dropped parts from disk as soon as possible /// To be removed a partition should have zero refcount, therefore call the cleanup thread at exit @@ -2890,9 +2896,9 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Strin ProfileEvents::increment(ProfileEvents::ReplicatedPartFetches); if (part_to_clone) - LOG_DEBUG(log, "Cloned part " << part_name << " from " << part_to_clone->name << (to_detached ? " (to 'detached' directory)" : "")); + LOG_DEBUG_FORMATTED(log, "Cloned part {} from {}{}", part_name, part_to_clone->name, to_detached ? " (to 'detached' directory)" : ""); else - LOG_DEBUG(log, "Fetched part " << part_name << " from " << source_replica_path << (to_detached ? " (to 'detached' directory)" : "")); + LOG_DEBUG_FORMATTED(log, "Fetched part {} from {}{}", part_name, part_to_clone->name, to_detached ? " (to 'detached' directory)" : ""); return true; } @@ -4892,7 +4898,10 @@ void StorageReplicatedMergeTree::replacePartitionFrom(const StoragePtr & source_ String hash_hex = src_part->checksums.getTotalChecksumHex(); - LOG_INFO(log, "Trying to " << (replace ? "replace " : "attach ") << src_part->name << " with hash_hex " << hash_hex); + if (replace) + LOG_INFO(log, "Trying to replace " << src_part->name << " with hash_hex " << hash_hex); + else + LOG_INFO(log, "Trying to attach " << src_part->name << " with hash_hex " << hash_hex); String block_id_path = replace ? "" : (zookeeper_path + "/blocks/" + partition_id + "_replace_from_" + hash_hex); From 38e3e273cdc3841c6b87bc610706f099525895af Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 23:39:59 +0300 Subject: [PATCH 657/738] Progress on task --- src/Parsers/DumpASTNode.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Parsers/DumpASTNode.h b/src/Parsers/DumpASTNode.h index 6b945cd5c8d..2fad5793304 100644 --- a/src/Parsers/DumpASTNode.h +++ b/src/Parsers/DumpASTNode.h @@ -102,7 +102,7 @@ public: ~DebugASTLog() { if constexpr (_enable) - LOG_DEBUG(log, ss.str()); + LOG_DEBUG_FORMATTED(log, ss.str()); } std::ostream * stream() { return (_enable ? &ss : nullptr); } From e1695feb7f93b04d773d52daad61a05c31430a37 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 23:40:32 +0300 Subject: [PATCH 658/738] Apply all transformations again --- src/Storages/Distributed/DirectoryMonitor.cpp | 6 +++--- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 6 +++--- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 4 ++-- src/Storages/StorageMergeTree.cpp | 4 ++-- src/Storages/StorageReplicatedMergeTree.cpp | 16 ++++++++-------- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp index 65048625d91..c17c2090a04 100644 --- a/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/src/Storages/Distributed/DirectoryMonitor.cpp @@ -261,7 +261,7 @@ bool StorageDistributedDirectoryMonitor::processFiles() void StorageDistributedDirectoryMonitor::processFile(const std::string & file_path) { - LOG_TRACE(log, "Started processing `" << file_path << "`"); + LOG_TRACE_FORMATTED(log, "Started processing `{}`", file_path); auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(storage.global_context->getSettingsRef()); auto connection = pool->get(timeouts); @@ -290,7 +290,7 @@ void StorageDistributedDirectoryMonitor::processFile(const std::string & file_pa Poco::File{file_path}.remove(); - LOG_TRACE(log, "Finished processing `" << file_path << "`"); + LOG_TRACE_FORMATTED(log, "Finished processing `{}`", file_path); } void StorageDistributedDirectoryMonitor::readHeader( @@ -692,7 +692,7 @@ void StorageDistributedDirectoryMonitor::markAsBroken(const std::string & file_p Poco::File{broken_path}.createDirectory(); Poco::File{file_path}.renameTo(broken_file_path); - LOG_ERROR(log, "Renamed `" << file_path << "` to `" << broken_file_path << "`"); + LOG_ERROR_FORMATTED(log, "Renamed `{}` to `{}`", file_path, broken_file_path); } bool StorageDistributedDirectoryMonitor::maybeMarkAsBroken(const std::string & file_path, const Exception & e) const diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 2f1552801d5..18416cebda6 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -710,7 +710,7 @@ void IMergeTreeDataPart::remove() const if (volume->getDisk()->exists(to)) { - LOG_WARNING(storage.log, "Directory " << fullPath(volume->getDisk(), to) << " (to which part must be renamed before removing) already exists. Most likely this is due to unclean restart. Removing it."); + LOG_WARNING_FORMATTED(storage.log, "Directory {} (to which part must be renamed before removing) already exists. Most likely this is due to unclean restart. Removing it.", fullPath(volume->getDisk(), to)); try { @@ -729,7 +729,7 @@ void IMergeTreeDataPart::remove() const } catch (const Poco::FileNotFoundException &) { - LOG_ERROR(storage.log, "Directory " << fullPath(volume->getDisk(), to) << " (part to remove) doesn't exist or one of nested files has gone. Most likely this is due to manual removing. This should be discouraged. Ignoring."); + LOG_ERROR_FORMATTED(storage.log, "Directory {} (part to remove) doesn't exist or one of nested files has gone. Most likely this is due to manual removing. This should be discouraged. Ignoring.", fullPath(volume->getDisk(), to)); return; } @@ -791,7 +791,7 @@ String IMergeTreeDataPart::getRelativePathForDetachedPart(const String & prefix) if (!volume->getDisk()->exists(getFullRelativePath() + res)) return res; - LOG_WARNING(storage.log, "Directory " << res << " (to detach to) already exists. Will detach to directory with '_tryN' suffix."); + LOG_WARNING_FORMATTED(storage.log, "Directory {} (to detach to) already exists. Will detach to directory with '_tryN' suffix.", res); } return res; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 4389bca586e..ae8ee086bb3 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -859,7 +859,7 @@ void ReplicatedMergeTreeQueue::removePartProducingOpsInRange( updateTimesInZooKeeper(zookeeper, min_unprocessed_insert_time_changed, max_processed_insert_time_changed); - LOG_DEBUG(log, "Removed " << removed_entries << " entries from queue. Waiting for " << to_wait.size() << " entries that are currently executing."); + LOG_DEBUG_FORMATTED(log, "Removed {} entries from queue. Waiting for {} entries that are currently executing.", removed_entries, to_wait.size()); /// Let's wait for the operations with the parts contained in the range to be deleted. for (LogEntryPtr & entry : to_wait) @@ -1328,7 +1328,7 @@ MutationCommands ReplicatedMergeTreeQueue::getMutationCommands( if (part->info.getDataVersion() > desired_mutation_version) { - LOG_WARNING(log, "Data version of part " << part->name << " is already greater than desired mutation version " << desired_mutation_version); + LOG_WARNING_FORMATTED(log, "Data version of part {} is already greater than desired mutation version {}", part->name, desired_mutation_version); return MutationCommands{}; } diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 7e652deceb6..55645c3c2b7 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -1056,9 +1056,9 @@ void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, cons } if (detach) - LOG_INFO(log, "Detached " << parts_to_remove.size() << " parts inside partition ID " << partition_id << "."); + LOG_INFO_FORMATTED(log, "Detached {} parts inside partition ID {}.", parts_to_remove.size(), partition_id); else - LOG_INFO(log, "Removed " << parts_to_remove.size() << " parts inside partition ID " << partition_id << "."); + LOG_INFO_FORMATTED(log, "Removed {} parts inside partition ID {}.", parts_to_remove.size(), partition_id); } clearOldPartsFromFilesystem(); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 29df2bf7b2b..e9f8b69aacf 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -325,7 +325,7 @@ void StorageReplicatedMergeTree::waitMutationToFinishOnReplicas( for (const String & replica : replicas) { - LOG_DEBUG(log, "Waiting for " << replica << " to apply mutation " << mutation_id); + LOG_DEBUG_FORMATTED(log, "Waiting for {} to apply mutation {}", replica, mutation_id); while (!partial_shutdown_called) { @@ -342,7 +342,7 @@ void StorageReplicatedMergeTree::waitMutationToFinishOnReplicas( /// Replica could be inactive. if (!zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/is_active")) { - LOG_WARNING(log, "Replica " << replica << " is not active during mutation. Mutation will be done asynchronously when replica becomes active."); + LOG_WARNING_FORMATTED(log, "Replica {} is not active during mutation. Mutation will be done asynchronously when replica becomes active.", replica); inactive_replicas.emplace(replica); break; @@ -730,7 +730,7 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks) /// Remove extra local parts. for (const DataPartPtr & part : unexpected_parts) { - LOG_ERROR(log, "Renaming unexpected part " << part->name << " to ignored_" << part->name); + LOG_ERROR_FORMATTED(log, "Renaming unexpected part {} to ignored_{}", part->name, part->name); forgetPartAndMoveToDetached(part, "ignored", true); } } @@ -923,7 +923,7 @@ bool StorageReplicatedMergeTree::executeLogEntry(LogEntry & entry) { if (!(entry.type == LogEntry::GET_PART && entry.source_replica == replica_name)) { - LOG_DEBUG(log, "Skipping action for part " << entry.new_part_name << " because part " << existing_part->name << " already exists."); + LOG_DEBUG_FORMATTED(log, "Skipping action for part {} because part {} already exists.", entry.new_part_name, existing_part->name); } return true; } @@ -1138,7 +1138,7 @@ bool StorageReplicatedMergeTree::tryExecutePartMutation(const StorageReplicatedM DataPartPtr source_part = getActiveContainingPart(source_part_name); if (!source_part) { - LOG_DEBUG(log, "Source part " << source_part_name << " for " << entry.new_part_name << " is not ready; will try to fetch it instead"); + LOG_DEBUG_FORMATTED(log, "Source part {} for {} is not ready; will try to fetch it instead", source_part_name, entry.new_part_name); return false; } @@ -1288,7 +1288,7 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) if (entry.type != LogEntry::GET_PART) throw Exception("Logical error: log entry with quorum but type is not GET_PART", ErrorCodes::LOGICAL_ERROR); - LOG_DEBUG(log, "No active replica has part " << entry.new_part_name << " which needs to be written with quorum. Will try to mark that quorum as failed."); + LOG_DEBUG_FORMATTED(log, "No active replica has part {} which needs to be written with quorum. Will try to mark that quorum as failed.", entry.new_part_name); /** Atomically: * - if replicas do not become active; @@ -4899,9 +4899,9 @@ void StorageReplicatedMergeTree::replacePartitionFrom(const StoragePtr & source_ String hash_hex = src_part->checksums.getTotalChecksumHex(); if (replace) - LOG_INFO(log, "Trying to replace " << src_part->name << " with hash_hex " << hash_hex); + LOG_INFO_FORMATTED(log, "Trying to replace {} with hash_hex {}", src_part->name, hash_hex); else - LOG_INFO(log, "Trying to attach " << src_part->name << " with hash_hex " << hash_hex); + LOG_INFO_FORMATTED(log, "Trying to attach {} with hash_hex {}", src_part->name, hash_hex); String block_id_path = replace ? "" : (zookeeper_path + "/blocks/" + partition_id + "_replace_from_" + hash_hex); From 17df54785e97de1d546a3610e08d438da30cbd37 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sat, 23 May 2020 23:45:28 +0300 Subject: [PATCH 659/738] Update merge_table_streams.xml --- tests/performance/merge_table_streams.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/performance/merge_table_streams.xml b/tests/performance/merge_table_streams.xml index c3484275606..01f0444e54c 100644 --- a/tests/performance/merge_table_streams.xml +++ b/tests/performance/merge_table_streams.xml @@ -19,7 +19,7 @@ CREATE TABLE merge_table_streams_3 AS hits_100m_single CREATE TABLE merge_table_streams_4 AS hits_100m_single -SELECT UserID FROM merge( '^(hits_100m_single|merge_table_streams_\\d)$') WHERE UserID = 12345678901234567890 +SELECT UserID FROM merge(default, '^(hits_100m_single|merge_table_streams_\\d)$') WHERE UserID = 12345678901234567890 DROP TABLE merge_table_streams_1 DROP TABLE merge_table_streams_2 From d2621fee304bfa257e9774a6f30350666afd1766 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 23 May 2020 23:57:48 +0300 Subject: [PATCH 660/738] Progress on task --- src/Client/ConnectionPoolWithFailover.cpp | 6 +++--- src/DataStreams/AggregatingBlockInputStream.cpp | 4 ++-- src/DataStreams/ParallelAggregatingBlockInputStream.cpp | 8 ++++---- src/Interpreters/Aggregator.cpp | 7 ++++--- src/Processors/Transforms/AggregatingTransform.cpp | 4 ++-- 5 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/Client/ConnectionPoolWithFailover.cpp b/src/Client/ConnectionPoolWithFailover.cpp index c925f6a7187..288292683a5 100644 --- a/src/Client/ConnectionPoolWithFailover.cpp +++ b/src/Client/ConnectionPoolWithFailover.cpp @@ -222,9 +222,9 @@ ConnectionPoolWithFailover::tryGetEntry( auto table_status_it = status_response.table_states_by_id.find(*table_to_check); if (table_status_it == status_response.table_states_by_id.end()) { - fail_message = "There is no table " + table_to_check->database + "." + table_to_check->table - + " on server: " + result.entry->getDescription(); - LOG_WARNING(log, fail_message); + auto message_pattern = "There is no table {}.{} on server: {}"; + fail_message = fmt::format(message_pattern, backQuote(table_to_check->database), backQuote(table_to_check->table), result.entry->getDescription()); + LOG_WARNING_FORMATTED(log, fail_message); ProfileEvents::increment(ProfileEvents::DistributedConnectionMissingTable); return result; diff --git a/src/DataStreams/AggregatingBlockInputStream.cpp b/src/DataStreams/AggregatingBlockInputStream.cpp index 1a06d18b3aa..34162deb3d3 100644 --- a/src/DataStreams/AggregatingBlockInputStream.cpp +++ b/src/DataStreams/AggregatingBlockInputStream.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include @@ -60,7 +60,7 @@ Block AggregatingBlockInputStream::readImpl() input_streams.emplace_back(temporary_inputs.back()->block_in); } - LOG_TRACE_FORMATTED(log, "Will merge {} temporary files of size {} MiB compressed, {} MiB uncompressed.", files.files.size(), (files.sum_size_compressed / 1048576.0), (files.sum_size_uncompressed / 1048576.0)); + LOG_TRACE_FORMATTED(log, "Will merge {} temporary files of size {} compressed, {} uncompressed.", files.files.size(), formatReadableSizeWithBinarySuffix(files.sum_size_compressed), formatReadableSizeWithBinarySuffix(files.sum_size_uncompressed)); impl = std::make_unique(input_streams, params, final, 1, 1); } diff --git a/src/DataStreams/ParallelAggregatingBlockInputStream.cpp b/src/DataStreams/ParallelAggregatingBlockInputStream.cpp index b3770ae45b1..1fda575ed1a 100644 --- a/src/DataStreams/ParallelAggregatingBlockInputStream.cpp +++ b/src/DataStreams/ParallelAggregatingBlockInputStream.cpp @@ -2,7 +2,7 @@ #include #include #include -#include +#include namespace ProfileEvents @@ -82,7 +82,7 @@ Block ParallelAggregatingBlockInputStream::readImpl() input_streams.emplace_back(temporary_inputs.back()->block_in); } - LOG_TRACE_FORMATTED(log, "Will merge {} temporary files of size {} MiB compressed, {} MiB uncompressed.", files.files.size(), (files.sum_size_compressed / 1048576.0), (files.sum_size_uncompressed / 1048576.0)); + LOG_TRACE_FORMATTED(log, "Will merge {} temporary files of size {} compressed, {} uncompressed.", files.files.size(), formatReadableSizeWithBinarySuffix(files.sum_size_compressed), formatReadableSizeWithBinarySuffix(files.sum_size_uncompressed)); impl = std::make_unique( input_streams, params, final, temporary_data_merge_threads, temporary_data_merge_threads); @@ -177,12 +177,12 @@ void ParallelAggregatingBlockInputStream::execute() for (size_t i = 0; i < max_threads; ++i) { size_t rows = many_data[i]->size(); - LOG_TRACE(log, "Aggregated. " << threads_data[i].src_rows << " to " << rows << " rows (from " << threads_data[i].src_bytes / 1048576.0 << " MiB) in " << elapsed_seconds << " sec. (" << threads_data[i].src_rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(threads_data[i].src_bytes / elapsed_seconds) << "/sec.)"); + LOG_TRACE(log, "Aggregated. " << threads_data[i].src_rows << " to " << rows << " rows (from " << formatReadableSizeWithBinarySuffix(threads_data[i].src_bytes) << ") in " << elapsed_seconds << " sec. (" << threads_data[i].src_rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(threads_data[i].src_bytes / elapsed_seconds) << "/sec.)"); total_src_rows += threads_data[i].src_rows; total_src_bytes += threads_data[i].src_bytes; } - LOG_TRACE(log, "Total aggregated. " << total_src_rows << " rows (from " << total_src_bytes / 1048576.0 << " MiB) in " << elapsed_seconds << " sec. (" << total_src_rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(total_src_bytes / elapsed_seconds) << "/sec.)"); + LOG_TRACE(log, "Total aggregated. " << total_src_rows << " rows (from " << formatReadableSizeWithBinarySuffix(total_src_bytes) << ") in " << elapsed_seconds << " sec. (" << total_src_rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(total_src_bytes / elapsed_seconds) << "/sec.)"); /// If there was no data, and we aggregate without keys, we must return single row with the result of empty aggregation. /// To do this, we pass a block with zero rows to aggregate. diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 72543f5a4cb..2114d8f3d0a 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -857,7 +858,7 @@ void Aggregator::writeToTemporaryFileImpl( /// `data_variants` will not destroy them in the destructor, they are now owned by ColumnAggregateFunction objects. data_variants.aggregator = nullptr; - LOG_TRACE_FORMATTED(log, "Max size of temporary block: {} rows, {} MiB.", max_temporary_block_size_rows, (max_temporary_block_size_bytes / 1048576.0)); + LOG_TRACE_FORMATTED(log, "Max size of temporary block: {} rows, {}.", max_temporary_block_size_rows, formatReadableSizeWithBinarySuffix(max_temporary_block_size_bytes)); } @@ -1293,7 +1294,7 @@ BlocksList Aggregator::convertToBlocks(AggregatedDataVariants & data_variants, b } double elapsed_seconds = watch.elapsedSeconds(); - LOG_TRACE(log, "Converted aggregated data to blocks. " << rows << " rows, " << bytes / 1048576.0 << " MiB in " << elapsed_seconds << " sec. (" << rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(bytes / elapsed_seconds) << "/sec.)"); + LOG_TRACE(log, "Converted aggregated data to blocks. " << rows << " rows, " << formatReadableSizeWithBinarySuffix(bytes) << " in " << elapsed_seconds << " sec. (" << rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(bytes / elapsed_seconds) << "/sec.)"); return blocks; } @@ -2155,7 +2156,7 @@ Block Aggregator::mergeBlocks(BlocksList & blocks, bool final) size_t rows = block.rows(); size_t bytes = block.bytes(); double elapsed_seconds = watch.elapsedSeconds(); - LOG_TRACE(log, "Merged partially aggregated blocks. " << rows << " rows, " << bytes / 1048576.0 << " MiB. in " << elapsed_seconds << " sec. (" << rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(bytes / elapsed_seconds) << "/sec.)"); + LOG_TRACE(log, "Merged partially aggregated blocks. " << rows << " rows, " << formatReadableSizeWithBinarySuffix(bytes) << ". in " << elapsed_seconds << " sec. (" << rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(bytes / elapsed_seconds) << "/sec.)"); if (isCancelled()) return {}; diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index 17105de9803..cea310e7862 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -540,7 +540,7 @@ void AggregatingTransform::initGenerate() double elapsed_seconds = watch.elapsedSeconds(); size_t rows = variants.sizeWithoutOverflowRow(); - LOG_TRACE(log, "Aggregated. " << src_rows << " to " << rows << " rows (from " << src_bytes / 1048576.0 << " MiB) in " << elapsed_seconds << " sec. (" << src_rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(src_bytes / elapsed_seconds) << "/sec.)"); + LOG_TRACE(log, "Aggregated. " << src_rows << " to " << rows << " rows (from " << formatReadableSizeWithBinarySuffix(src_bytes) << ") in " << elapsed_seconds << " sec. (" << src_rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(src_bytes / elapsed_seconds) << "/sec.)"); if (params->aggregator.hasTemporaryFiles()) { @@ -589,7 +589,7 @@ void AggregatingTransform::initGenerate() for (const auto & file : files.files) processors.emplace_back(std::make_unique(header, file->path())); - LOG_TRACE_FORMATTED(log, "Will merge {} temporary files of size {} MiB compressed, {} MiB uncompressed.", files.files.size(), (files.sum_size_compressed / 1048576.0), (files.sum_size_uncompressed / 1048576.0)); + LOG_TRACE_FORMATTED(log, "Will merge {} temporary files of size {} compressed, {} uncompressed.", files.files.size(), formatReadableSizeWithBinarySuffix(files.sum_size_compressed), formatReadableSizeWithBinarySuffix(files.sum_size_uncompressed)); auto pipe = createMergingAggregatedMemoryEfficientPipe( header, params, files.files.size(), temporary_data_merge_threads); From 7c0c328a350fab365f1188debbace55050b0db14 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 24 May 2020 00:05:47 +0300 Subject: [PATCH 661/738] Progress on task --- src/Client/Connection.cpp | 27 ++++++++++--------- .../ParallelAggregatingBlockInputStream.cpp | 9 +++++-- src/Databases/DatabaseOnDisk.cpp | 2 +- src/IO/S3Common.cpp | 4 +-- 4 files changed, 25 insertions(+), 17 deletions(-) diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index ce060197f4a..fdb5177c66d 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -503,20 +503,23 @@ void Connection::sendScalarsData(Scalars & data) maybe_compressed_out_bytes = maybe_compressed_out->count() - maybe_compressed_out_bytes; double elapsed = watch.elapsedSeconds(); - std::stringstream msg; - msg << std::fixed << std::setprecision(3); - msg << "Sent data for " << data.size() << " scalars, total " << rows << " rows in " << elapsed << " sec., " - << static_cast(rows / watch.elapsedSeconds()) << " rows/sec., " - << formatReadableSizeWithBinarySuffix(maybe_compressed_out_bytes) << " (" - << formatReadableSizeWithBinarySuffix(maybe_compressed_out_bytes / watch.elapsedSeconds()) << "/sec.)"; - if (compression == Protocol::Compression::Enable) - msg << ", compressed " << static_cast(maybe_compressed_out_bytes) / out_bytes << " times to " - << formatReadableSizeWithBinarySuffix(out_bytes) << " (" << formatReadableSizeWithBinarySuffix(out_bytes / watch.elapsedSeconds()) << "/sec.)"; + LOG_DEBUG_FORMATTED(log_wrapper.get(), + "Sent data for {} scalars, total {} rows in {} sec., {} rows/sec., {} ({}/sec.), compressed {} times to {} ({}/sec.)", + data.size(), rows, elapsed, + static_cast(rows / watch.elapsedSeconds()), + formatReadableSizeWithBinarySuffix(maybe_compressed_out_bytes), + formatReadableSizeWithBinarySuffix(maybe_compressed_out_bytes / watch.elapsedSeconds()), + static_cast(maybe_compressed_out_bytes) / out_bytes, + formatReadableSizeWithBinarySuffix(out_bytes), + formatReadableSizeWithBinarySuffix(out_bytes / watch.elapsedSeconds())); else - msg << ", no compression."; - - LOG_DEBUG(log_wrapper.get(), msg.rdbuf()); + LOG_DEBUG_FORMATTED(log_wrapper.get(), + "Sent data for {} scalars, total {} rows in {} sec., {} rows/sec., {} ({}/sec.), no compression.", + data.size(), rows, elapsed, + static_cast(rows / watch.elapsedSeconds()), + formatReadableSizeWithBinarySuffix(maybe_compressed_out_bytes), + formatReadableSizeWithBinarySuffix(maybe_compressed_out_bytes / watch.elapsedSeconds())); } namespace diff --git a/src/DataStreams/ParallelAggregatingBlockInputStream.cpp b/src/DataStreams/ParallelAggregatingBlockInputStream.cpp index 1fda575ed1a..037f421c733 100644 --- a/src/DataStreams/ParallelAggregatingBlockInputStream.cpp +++ b/src/DataStreams/ParallelAggregatingBlockInputStream.cpp @@ -177,12 +177,17 @@ void ParallelAggregatingBlockInputStream::execute() for (size_t i = 0; i < max_threads; ++i) { size_t rows = many_data[i]->size(); - LOG_TRACE(log, "Aggregated. " << threads_data[i].src_rows << " to " << rows << " rows (from " << formatReadableSizeWithBinarySuffix(threads_data[i].src_bytes) << ") in " << elapsed_seconds << " sec. (" << threads_data[i].src_rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(threads_data[i].src_bytes / elapsed_seconds) << "/sec.)"); + LOG_TRACE_FORMATTED(log, "Aggregated. {} to {} rows (from {}) in {} sec. ({} rows/sec., {}/sec.)", + threads_data[i].src_rows, rows, formatReadableSizeWithBinarySuffix(threads_data[i].src_bytes), + elapsed_seconds, threads_data[i].src_rows / elapsed_seconds, + formatReadableSizeWithBinarySuffix(threads_data[i].src_bytes / elapsed_seconds)); total_src_rows += threads_data[i].src_rows; total_src_bytes += threads_data[i].src_bytes; } - LOG_TRACE(log, "Total aggregated. " << total_src_rows << " rows (from " << formatReadableSizeWithBinarySuffix(total_src_bytes) << ") in " << elapsed_seconds << " sec. (" << total_src_rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(total_src_bytes / elapsed_seconds) << "/sec.)"); + LOG_TRACE_FORMATTED(log, "Total aggregated. {} rows (from {}) in {} sec. ({} rows/sec., {}/sec.)", + total_src_rows, formatReadableSizeWithBinarySuffix(total_src_bytes), elapsed_seconds, + total_src_rows / elapsed_seconds, formatReadableSizeWithBinarySuffix(total_src_bytes / elapsed_seconds)); /// If there was no data, and we aggregate without keys, we must return single row with the result of empty aggregation. /// To do this, we pass a block with zero rows to aggregate. diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index 480f03fb7b2..9e606296624 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -230,7 +230,7 @@ void DatabaseOnDisk::dropTable(const Context & context, const String & table_nam } catch (...) { - LOG_WARNING(log, getCurrentExceptionMessage(__PRETTY_FUNCTION__)); + LOG_WARNING_FORMATTED(log, getCurrentExceptionMessage(__PRETTY_FUNCTION__)); attachTable(table_name, table, table_data_path_relative); if (renamed) Poco::File(table_metadata_path_drop).renameTo(table_metadata_path); diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp index 0cf8cf22326..49fa1dc0c7a 100644 --- a/src/IO/S3Common.cpp +++ b/src/IO/S3Common.cpp @@ -40,13 +40,13 @@ public: void Log(Aws::Utils::Logging::LogLevel log_level, const char * tag, const char * format_str, ...) final // NOLINT { const auto & [level, prio] = convertLogLevel(log_level); - LOG_IMPL(log, level, prio, std::string(tag) + ": " + format_str); + LOG_IMPL_FORMATTED(log, level, prio, "{}: {}", tag, format_str); } void LogStream(Aws::Utils::Logging::LogLevel log_level, const char * tag, const Aws::OStringStream & message_stream) final { const auto & [level, prio] = convertLogLevel(log_level); - LOG_IMPL(log, level, prio, std::string(tag) + ": " + message_stream.str()); + LOG_IMPL_FORMATTED(log, level, prio, "{}: {}", tag, message_stream.str()); } void Flush() final {} From bab24879e9796d16e93baf9c903879c2797d0d94 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 24 May 2020 00:16:05 +0300 Subject: [PATCH 662/738] Progress on task --- src/Client/Connection.cpp | 26 +++++++++++-------- src/Interpreters/Aggregator.cpp | 21 +++++++++++++-- .../Transforms/AggregatingTransform.cpp | 6 ++++- .../Transforms/CreatingSetsTransform.cpp | 20 +++++++------- .../DistributedBlockOutputStream.cpp | 2 +- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- .../ReplicatedMergeTreeCleanupThread.cpp | 2 +- .../ReplicatedMergeTreePartCheckThread.cpp | 2 +- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 2 +- src/Storages/StorageMergeTree.cpp | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 12 ++++----- 11 files changed, 61 insertions(+), 36 deletions(-) diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index fdb5177c66d..9d51a27408a 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -607,19 +607,23 @@ void Connection::sendExternalTablesData(ExternalTablesData & data) maybe_compressed_out_bytes = maybe_compressed_out->count() - maybe_compressed_out_bytes; double elapsed = watch.elapsedSeconds(); - std::stringstream msg; - msg << std::fixed << std::setprecision(3); - msg << "Sent data for " << data.size() << " external tables, total " << rows << " rows in " << elapsed << " sec., " - << static_cast(rows / watch.elapsedSeconds()) << " rows/sec., " - << formatReadableSizeWithBinarySuffix(maybe_compressed_out_bytes) << " (" << formatReadableSizeWithBinarySuffix(maybe_compressed_out_bytes / watch.elapsedSeconds()) << "/sec.)"; - if (compression == Protocol::Compression::Enable) - msg << ", compressed " << static_cast(maybe_compressed_out_bytes) / out_bytes << " times to " - << formatReadableSizeWithBinarySuffix(out_bytes) << " (" << formatReadableSizeWithBinarySuffix(out_bytes / watch.elapsedSeconds()) << "/sec.)"; + LOG_DEBUG_FORMATTED(log_wrapper.get(), + "Sent data for {} external tables, total {} rows in {} sec., {} rows/sec., {} ({}/sec.), compressed {} times to {} ({}/sec.)", + data.size(), rows, elapsed, + static_cast(rows / watch.elapsedSeconds()), + formatReadableSizeWithBinarySuffix(maybe_compressed_out_bytes), + formatReadableSizeWithBinarySuffix(maybe_compressed_out_bytes / watch.elapsedSeconds()), + static_cast(maybe_compressed_out_bytes) / out_bytes, + formatReadableSizeWithBinarySuffix(out_bytes), + formatReadableSizeWithBinarySuffix(out_bytes / watch.elapsedSeconds())); else - msg << ", no compression."; - - LOG_DEBUG(log_wrapper.get(), msg.rdbuf()); + LOG_DEBUG_FORMATTED(log_wrapper.get(), + "Sent data for {} external tables, total {} rows in {} sec., {} rows/sec., {} ({}/sec.), no compression.", + data.size(), rows, elapsed, + static_cast(rows / watch.elapsedSeconds()), + formatReadableSizeWithBinarySuffix(maybe_compressed_out_bytes), + formatReadableSizeWithBinarySuffix(maybe_compressed_out_bytes / watch.elapsedSeconds())); } std::optional Connection::getResolvedAddress() const diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 2114d8f3d0a..a30f62d14d7 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -762,7 +762,20 @@ void Aggregator::writeToTemporaryFile(AggregatedDataVariants & data_variants, co ProfileEvents::increment(ProfileEvents::ExternalAggregationCompressedBytes, compressed_bytes); ProfileEvents::increment(ProfileEvents::ExternalAggregationUncompressedBytes, uncompressed_bytes); - LOG_TRACE(log, "Written part in " << elapsed_seconds << " sec., " << rows << " rows, " << formatReadableSizeWithBinarySuffix(uncompressed_bytes) << " uncompressed, " << formatReadableSizeWithBinarySuffix(compressed_bytes) << " compressed, " << (uncompressed_bytes / rows) << " uncompressed bytes per row, " << (compressed_bytes / rows) << " compressed bytes per row, compression rate: " << (uncompressed_bytes / compressed_bytes) << " (" << (rows / elapsed_seconds) << " rows/sec., " << formatReadableSizeWithBinarySuffix(uncompressed_bytes / elapsed_seconds) << "/sec. uncompressed, " << formatReadableSizeWithBinarySuffix(compressed_bytes / elapsed_seconds) << "/sec. compressed)"); + LOG_TRACE_FORMATTED(log, + "Written part in {} sec., {} rows, {} uncompressed, {} compressed," + " {} uncompressed bytes per row, {} compressed bytes per row, compression rate: {}" + " ({} rows/sec., {}/sec. uncompressed, {}/sec. compressed)", + elapsed_seconds, + rows, + formatReadableSizeWithBinarySuffix(uncompressed_bytes), + formatReadableSizeWithBinarySuffix(compressed_bytes), + uncompressed_bytes / rows, + compressed_bytes / rows, + uncompressed_bytes / compressed_bytes, + rows / elapsed_seconds, + formatReadableSizeWithBinarySuffix(uncompressed_bytes / elapsed_seconds), + formatReadableSizeWithBinarySuffix(compressed_bytes / elapsed_seconds)); } void Aggregator::writeToTemporaryFile(AggregatedDataVariants & data_variants) { @@ -928,7 +941,11 @@ void Aggregator::execute(const BlockInputStreamPtr & stream, AggregatedDataVaria double elapsed_seconds = watch.elapsedSeconds(); size_t rows = result.sizeWithoutOverflowRow(); - LOG_TRACE(log, "Aggregated. " << src_rows << " to " << rows << " rows (from " << formatReadableSizeWithBinarySuffix(src_bytes) << ") in " << elapsed_seconds << " sec. (" << src_rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(src_bytes / elapsed_seconds) << "/sec.)"); + + LOG_TRACE_FORMATTED(log, "Aggregated. {} to {} rows (from {}) in {} sec. ({} rows/sec., {}/sec.)", + src_rows, rows, formatReadableSizeWithBinarySuffix(src_bytes), + elapsed_seconds, src_rows / elapsed_seconds, + formatReadableSizeWithBinarySuffix(src_bytes / elapsed_seconds)); } diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index cea310e7862..2a10223022e 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -540,7 +540,11 @@ void AggregatingTransform::initGenerate() double elapsed_seconds = watch.elapsedSeconds(); size_t rows = variants.sizeWithoutOverflowRow(); - LOG_TRACE(log, "Aggregated. " << src_rows << " to " << rows << " rows (from " << formatReadableSizeWithBinarySuffix(src_bytes) << ") in " << elapsed_seconds << " sec. (" << src_rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(src_bytes / elapsed_seconds) << "/sec.)"); + + LOG_TRACE_FORMATTED(log, "Aggregated. {} to {} rows (from {}) in {} sec. ({} rows/sec., {}/sec.)", + src_rows, rows, formatReadableSizeWithBinarySuffix(src_bytes), + elapsed_seconds, src_rows / elapsed_seconds, + formatReadableSizeWithBinarySuffix(src_bytes / elapsed_seconds)); if (params->aggregator.hasTemporaryFiles()) { diff --git a/src/Processors/Transforms/CreatingSetsTransform.cpp b/src/Processors/Transforms/CreatingSetsTransform.cpp index a8080376169..19ed3067703 100644 --- a/src/Processors/Transforms/CreatingSetsTransform.cpp +++ b/src/Processors/Transforms/CreatingSetsTransform.cpp @@ -56,7 +56,12 @@ IProcessor::Status CreatingSetsTransform::prepare() void CreatingSetsTransform::startSubquery(SubqueryForSet & subquery) { - LOG_TRACE(log, (subquery.set ? "Creating set. " : "") << (subquery.join ? "Creating join. " : "") << (subquery.table ? "Filling temporary table. " : "")); + if (subquery.set) + LOG_TRACE_FORMATTED(log, "Creating set."); + if (subquery.join) + LOG_TRACE_FORMATTED(log, "Creating join."); + if (subquery.table) + LOG_TRACE_FORMATTED(log, "Filling temporary table."); elapsed_nanoseconds = 0; @@ -85,19 +90,14 @@ void CreatingSetsTransform::finishSubquery(SubqueryForSet & subquery) if (head_rows != 0) { - std::stringstream msg; - msg << std::fixed << std::setprecision(3); - msg << "Created. "; + auto seconds = elapsed_nanoseconds / 1e9; if (subquery.set) - msg << "Set with " << subquery.set->getTotalRowCount() << " entries from " << head_rows << " rows. "; + LOG_DEBUG_FORMATTED(log, "Created Set with {} entries from {} rows in {} sec.", subquery.set->getTotalRowCount(), head_rows, seconds); if (subquery.join) - msg << "Join with " << subquery.join->getTotalRowCount() << " entries from " << head_rows << " rows. "; + LOG_DEBUG_FORMATTED(log, "Created Join with {} entries from {} rows in {} sec.", subquery.join->getTotalRowCount(), head_rows, seconds); if (subquery.table) - msg << "Table with " << head_rows << " rows. "; - - msg << "In " << (static_cast(elapsed_nanoseconds) / 1000000000ULL) << " sec."; - LOG_DEBUG(log, msg.rdbuf()); + LOG_DEBUG_FORMATTED(log, "Created Table with {} rows in {} sec.", head_rows, seconds); } else { diff --git a/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/src/Storages/Distributed/DistributedBlockOutputStream.cpp index a5811b39865..48c7270a4bb 100644 --- a/src/Storages/Distributed/DistributedBlockOutputStream.cpp +++ b/src/Storages/Distributed/DistributedBlockOutputStream.cpp @@ -411,7 +411,7 @@ void DistributedBlockOutputStream::writeSuffix() auto log_performance = [this] () { double elapsed = watch.elapsedSeconds(); - LOG_DEBUG(log, "It took " << std::fixed << std::setprecision(1) << elapsed << " sec. to insert " << inserted_blocks << " blocks, " << std::fixed << std::setprecision(1) << inserted_rows / elapsed << " rows per second. " << getCurrentStateDescription()); + LOG_DEBUG(log, "It took " << elapsed << " sec. to insert " << inserted_blocks << " blocks, " << inserted_rows / elapsed << " rows per second. " << getCurrentStateDescription()); }; if (insert_sync && pool) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 84dd84f7068..896bf38f254 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -3275,7 +3275,7 @@ void MergeTreeData::freezePartitionsByMatcher(MatcherFn matcher, const String & : toString(increment)) + "/"; - LOG_DEBUG(log, "Freezing part " << part->name << " snapshot will be placed at " + backup_path); + LOG_DEBUG(log, "Freezing part " << part->name << " snapshot will be placed at " << backup_path); String backup_part_path = backup_path + relative_data_path + part->relative_path; localBackup(part->volume->getDisk(), part->getFullRelativePath(), backup_part_path); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp index 17dc1a508d2..b7ca328647e 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp @@ -327,7 +327,7 @@ void ReplicatedMergeTreeCleanupThread::clearOldBlocks() cached_block_stats.erase(first_outdated_block->node); } else if (rc) - LOG_WARNING(log, "Error while deleting ZooKeeper path `" << path << "`: " + zkutil::ZooKeeper::error2string(rc) << ", ignoring."); + LOG_WARNING(log, "Error while deleting ZooKeeper path `" << path << "`: " << zkutil::ZooKeeper::error2string(rc) << ", ignoring."); else { /// Successfully removed blocks have to be removed from cache diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp index 1ec1c041826..424909d8575 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp @@ -82,7 +82,7 @@ void ReplicatedMergeTreePartCheckThread::searchForMissingPart(const String & par /// If the part is in ZooKeeper, remove it from there and add the task to download it to the queue. if (zookeeper->exists(part_path)) { - LOG_WARNING(log, "Part " << part_name << " exists in ZooKeeper but not locally. " "Removing from ZooKeeper and queueing a fetch."); + LOG_WARNING(log, "Part " << part_name << " exists in ZooKeeper but not locally. Removing from ZooKeeper and queueing a fetch."); ProfileEvents::increment(ProfileEvents::ReplicatedPartChecksFailed); storage.removePartAndEnqueueFetch(part_name); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index ae8ee086bb3..00d15bd8049 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -745,7 +745,7 @@ ReplicatedMergeTreeMutationEntryPtr ReplicatedMergeTreeQueue::removeMutation( if (entry->isAlterMutation()) { - LOG_DEBUG(log, "Removed alter " << entry->alter_version << " because mutation " + entry->znode_name + " were killed."); + LOG_DEBUG(log, "Removed alter " << entry->alter_version << " because mutation " << entry->znode_name << " were killed."); alter_sequence.finishDataAlter(entry->alter_version, state_lock); } diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 55645c3c2b7..d5fd36b89a2 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -837,7 +837,7 @@ BackgroundProcessingPoolTaskResult StorageMergeTree::mergeMutateTask() { if (e.code() == ErrorCodes::ABORTED) { - LOG_INFO(log, e.message()); + LOG_INFO_FORMATTED(log, e.message()); return BackgroundProcessingPoolTaskResult::ERROR; } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index e9f8b69aacf..f431a73668f 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -2105,17 +2105,17 @@ BackgroundProcessingPoolTaskResult StorageReplicatedMergeTree::queueTask() if (e.code() == ErrorCodes::NO_REPLICA_HAS_PART) { /// If no one has the right part, probably not all replicas work; We will not write to log with Error level. - LOG_INFO(log, e.displayText()); + LOG_INFO_FORMATTED(log, e.displayText()); } else if (e.code() == ErrorCodes::ABORTED) { /// Interrupted merge or downloading a part is not an error. - LOG_INFO(log, e.message()); + LOG_INFO_FORMATTED(log, e.message()); } else if (e.code() == ErrorCodes::PART_IS_TEMPORARILY_LOCKED) { /// Part cannot be added temporarily - LOG_INFO(log, e.displayText()); + LOG_INFO_FORMATTED(log, e.displayText()); cleanup_thread.wakeup(); } else @@ -3205,7 +3205,7 @@ bool StorageReplicatedMergeTree::optimize(const ASTPtr & query, const ASTPtr & p message << "Cannot select parts for optimization"; if (!disable_reason.empty()) message << ": " << disable_reason; - LOG_INFO(log, message.rdbuf()); + LOG_INFO_FORMATTED(log, message.str()); return handle_noop(message.str()); } @@ -4380,7 +4380,7 @@ void StorageReplicatedMergeTree::fetchPartition(const ASTPtr & partition, const && e.code() != ErrorCodes::CANNOT_READ_ALL_DATA) throw; - LOG_INFO(log, e.displayText()); + LOG_INFO_FORMATTED(log, e.displayText()); missing_parts.push_back(part); } } @@ -4830,7 +4830,7 @@ void StorageReplicatedMergeTree::clearBlocksInPartition( zookeeper.removeRecursive(path); } else if (rc) - LOG_WARNING(log, "Error while deleting ZooKeeper path `" << path << "`: " + zkutil::ZooKeeper::error2string(rc) << ", ignoring."); + LOG_WARNING_FORMATTED(log, "Error while deleting ZooKeeper path `{}`: {}, ignoring.", path, zkutil::ZooKeeper::error2string(rc)); } LOG_TRACE_FORMATTED(log, "Deleted {} deduplication block IDs in partition ID {}", to_delete_futures.size(), partition_id); From 2d7d5a1547e2c7d273a8bc40a13306933c993b8c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 24 May 2020 00:16:27 +0300 Subject: [PATCH 663/738] Apply all transformations again --- src/Storages/Distributed/DistributedBlockOutputStream.cpp | 2 +- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp | 2 +- src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp | 2 +- src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/src/Storages/Distributed/DistributedBlockOutputStream.cpp index 48c7270a4bb..ba883b3a357 100644 --- a/src/Storages/Distributed/DistributedBlockOutputStream.cpp +++ b/src/Storages/Distributed/DistributedBlockOutputStream.cpp @@ -411,7 +411,7 @@ void DistributedBlockOutputStream::writeSuffix() auto log_performance = [this] () { double elapsed = watch.elapsedSeconds(); - LOG_DEBUG(log, "It took " << elapsed << " sec. to insert " << inserted_blocks << " blocks, " << inserted_rows / elapsed << " rows per second. " << getCurrentStateDescription()); + LOG_DEBUG_FORMATTED(log, "It took {} sec. to insert {} blocks, {} rows per second. {}", elapsed, inserted_blocks, inserted_rows / elapsed, LOG_DEBUG0); }; if (insert_sync && pool) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 896bf38f254..a5310b0608d 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -3275,7 +3275,7 @@ void MergeTreeData::freezePartitionsByMatcher(MatcherFn matcher, const String & : toString(increment)) + "/"; - LOG_DEBUG(log, "Freezing part " << part->name << " snapshot will be placed at " << backup_path); + LOG_DEBUG_FORMATTED(log, "Freezing part {} snapshot will be placed at {}", part->name, backup_path); String backup_part_path = backup_path + relative_data_path + part->relative_path; localBackup(part->volume->getDisk(), part->getFullRelativePath(), backup_part_path); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp index b7ca328647e..3c911fbaa0d 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp @@ -327,7 +327,7 @@ void ReplicatedMergeTreeCleanupThread::clearOldBlocks() cached_block_stats.erase(first_outdated_block->node); } else if (rc) - LOG_WARNING(log, "Error while deleting ZooKeeper path `" << path << "`: " << zkutil::ZooKeeper::error2string(rc) << ", ignoring."); + LOG_WARNING_FORMATTED(log, "Error while deleting ZooKeeper path `{}`: {}, ignoring.", path, zkutil::ZooKeeper::error2string(rc)); else { /// Successfully removed blocks have to be removed from cache diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp index 424909d8575..f3c24d9f8fb 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp @@ -82,7 +82,7 @@ void ReplicatedMergeTreePartCheckThread::searchForMissingPart(const String & par /// If the part is in ZooKeeper, remove it from there and add the task to download it to the queue. if (zookeeper->exists(part_path)) { - LOG_WARNING(log, "Part " << part_name << " exists in ZooKeeper but not locally. Removing from ZooKeeper and queueing a fetch."); + LOG_WARNING_FORMATTED(log, "Part {} exists in ZooKeeper but not locally. Removing from ZooKeeper and queueing a fetch.", part_name); ProfileEvents::increment(ProfileEvents::ReplicatedPartChecksFailed); storage.removePartAndEnqueueFetch(part_name); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 00d15bd8049..5177438ffd1 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -745,7 +745,7 @@ ReplicatedMergeTreeMutationEntryPtr ReplicatedMergeTreeQueue::removeMutation( if (entry->isAlterMutation()) { - LOG_DEBUG(log, "Removed alter " << entry->alter_version << " because mutation " << entry->znode_name << " were killed."); + LOG_DEBUG_FORMATTED(log, "Removed alter {} because mutation {} were killed.", entry->alter_version, entry->znode_name); alter_sequence.finishDataAlter(entry->alter_version, state_lock); } From ce0619dabf132366a70250d3950aba115d00d2cf Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 24 May 2020 00:26:45 +0300 Subject: [PATCH 664/738] Progress on task --- src/Interpreters/Aggregator.cpp | 11 +++++++++-- src/Interpreters/ExternalLoader.cpp | 18 +++++++++--------- src/Interpreters/SystemLog.cpp | 2 +- src/Interpreters/SystemLog.h | 8 ++++---- src/Interpreters/executeQuery.cpp | 2 +- .../Algorithms/CollapsingSortedAlgorithm.cpp | 9 +++------ .../DistributedBlockOutputStream.cpp | 7 ++++--- 7 files changed, 31 insertions(+), 26 deletions(-) diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index a30f62d14d7..b96d9c0638d 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -1311,7 +1311,11 @@ BlocksList Aggregator::convertToBlocks(AggregatedDataVariants & data_variants, b } double elapsed_seconds = watch.elapsedSeconds(); - LOG_TRACE(log, "Converted aggregated data to blocks. " << rows << " rows, " << formatReadableSizeWithBinarySuffix(bytes) << " in " << elapsed_seconds << " sec. (" << rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(bytes / elapsed_seconds) << "/sec.)"); + LOG_TRACE_FORMATTED(log, + "Converted aggregated data to blocks. {} rows, {} in {} sec. ({} rows/sec., {}/sec.)", + rows, formatReadableSizeWithBinarySuffix(bytes), + elapsed_seconds, rows / elapsed_seconds, + formatReadableSizeWithBinarySuffix(bytes / elapsed_seconds)); return blocks; } @@ -2173,7 +2177,10 @@ Block Aggregator::mergeBlocks(BlocksList & blocks, bool final) size_t rows = block.rows(); size_t bytes = block.bytes(); double elapsed_seconds = watch.elapsedSeconds(); - LOG_TRACE(log, "Merged partially aggregated blocks. " << rows << " rows, " << formatReadableSizeWithBinarySuffix(bytes) << ". in " << elapsed_seconds << " sec. (" << rows / elapsed_seconds << " rows/sec., " << formatReadableSizeWithBinarySuffix(bytes / elapsed_seconds) << "/sec.)"); + LOG_TRACE_FORMATTED(log, "Merged partially aggregated blocks. {} rows, {}. in {} sec. ({} rows/sec., {}/sec.)", + rows, formatReadableSizeWithBinarySuffix(bytes), + elapsed_seconds, rows / elapsed_seconds, + formatReadableSizeWithBinarySuffix(bytes / elapsed_seconds)); if (isCancelled()) return {}; diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index c097b2e7507..f81fed8c8ac 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -439,7 +439,7 @@ public: { /// The object has been tried to load before, so it is currently in use or was in use /// and we should try to reload it with the new config. - LOG_TRACE(log, "Will reload '" << name << "'" " because its configuration has been changed and" " there were attempts to load it before"); + LOG_TRACE(log, "Will reload '" << name << "' because its configuration has been changed and there were attempts to load it before"); startLoading(info, true); } } @@ -454,7 +454,7 @@ public: Info & info = infos.emplace(name, Info{name, config}).first->second; if (always_load_everything) { - LOG_TRACE(log, "Will load '" << name << "'" " because always_load_everything flag is set."); + LOG_TRACE(log, "Will load '" << name << "' because always_load_everything flag is set."); startLoading(info); } } @@ -664,7 +664,7 @@ public: if (!should_update_flag) { info.next_update_time = calculateNextUpdateTime(info.object, info.error_count); - LOG_TRACE(log, "Object '" << info.name << "'" " not modified, will not reload. " "Next update at " << ext::to_string(info.next_update_time)); + LOG_TRACE(log, "Object '" << info.name << "' not modified, will not reload. Next update at " << ext::to_string(info.next_update_time)); continue; } @@ -676,7 +676,7 @@ public: /// Object was never loaded successfully and should be reloaded. startLoading(info); } - LOG_TRACE(log, "Object '" << info.name << "' is neither" " loaded nor failed, so it will not be reloaded as outdated."); + LOG_TRACE(log, "Object '" << info.name << "' is neither loaded nor failed, so it will not be reloaded as outdated."); } } } @@ -1099,7 +1099,7 @@ private: { if (!loaded_object->supportUpdates()) { - LOG_TRACE(log, "Supposed update time for " "'" + loaded_object->getLoadableName() + "'" " is never (loaded, does not support updates)"); + LOG_TRACE(log, "Supposed update time for '" + loaded_object->getLoadableName() + "' is never (loaded, does not support updates)"); return never; } @@ -1108,7 +1108,7 @@ private: const auto & lifetime = loaded_object->getLifetime(); if (lifetime.min_sec == 0 && lifetime.max_sec == 0) { - LOG_TRACE(log, "Supposed update time for " "'" + loaded_object->getLoadableName() + "'" " is never (loaded, lifetime 0)"); + LOG_TRACE(log, "Supposed update time for '" + loaded_object->getLoadableName() + "' is never (loaded, lifetime 0)"); return never; } @@ -1116,18 +1116,18 @@ private: { std::uniform_int_distribution distribution{lifetime.min_sec, lifetime.max_sec}; auto result = std::chrono::system_clock::now() + std::chrono::seconds{distribution(rnd_engine)}; - LOG_TRACE(log, "Supposed update time for " "'" << loaded_object->getLoadableName() << "'" " is " << ext::to_string(result) << " (loaded, lifetime [" << lifetime.min_sec << ", " << lifetime.max_sec << "], no errors)"); + LOG_TRACE(log, "Supposed update time for '" << loaded_object->getLoadableName() << "' is " << ext::to_string(result) << " (loaded, lifetime [" << lifetime.min_sec << ", " << lifetime.max_sec << "], no errors)"); return result; } auto result = std::chrono::system_clock::now() + std::chrono::seconds(calculateDurationWithBackoff(rnd_engine, error_count)); - LOG_TRACE(log, "Supposed update time for '" << loaded_object->getLoadableName() << "'" " is " << ext::to_string(result) << " (backoff, " << error_count << " errors)"); + LOG_TRACE(log, "Supposed update time for '" << loaded_object->getLoadableName() << "' is " << ext::to_string(result) << " (backoff, " << error_count << " errors)"); return result; } else { auto result = std::chrono::system_clock::now() + std::chrono::seconds(calculateDurationWithBackoff(rnd_engine, error_count)); - LOG_TRACE(log, "Supposed update time for unspecified object " " is " << ext::to_string(result) << " (backoff, " << error_count << " errors."); + LOG_TRACE(log, "Supposed update time for unspecified object is " << ext::to_string(result) << " (backoff, " << error_count << " errors."); return result; } } diff --git a/src/Interpreters/SystemLog.cpp b/src/Interpreters/SystemLog.cpp index f604c05401b..c9b00e04f10 100644 --- a/src/Interpreters/SystemLog.cpp +++ b/src/Interpreters/SystemLog.cpp @@ -41,7 +41,7 @@ std::shared_ptr createSystemLog( if (database != default_database_name) { /// System tables must be loaded before other tables, but loading order is undefined for all databases except `system` - LOG_ERROR(&Logger::get("SystemLog"), "Custom database name for a system table specified in config. " "Table `" << table << "` will be created in `system` database " "instead of `" << database << "`"); + LOG_ERROR(&Logger::get("SystemLog"), "Custom database name for a system table specified in config. Table `" << table << "` will be created in `system` database instead of `" << database << "`"); database = default_database_name; } diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h index 033a46d3960..8f40ff0dc28 100644 --- a/src/Interpreters/SystemLog.h +++ b/src/Interpreters/SystemLog.h @@ -60,7 +60,7 @@ namespace ErrorCodes extern const int TIMEOUT_EXCEEDED; } -#define DBMS_SYSTEM_LOG_QUEUE_SIZE 1048576 +#define DBMS_SYSTEM_LOG_QUEUE_SIZE 1048576 class Context; class QueryLog; class QueryThreadLog; @@ -456,7 +456,7 @@ void SystemLog::prepareTable() rename->elements.emplace_back(elem); - LOG_DEBUG(log, "Existing table " << description << " for system log has obsolete or different structure." " Renaming it to " << backQuoteIfNeed(to.table)); + LOG_DEBUG(log, "Existing table " << description << " for system log has obsolete or different structure. Renaming it to " << backQuoteIfNeed(to.table)); InterpreterRenameQuery(rename, context).execute(); @@ -464,13 +464,13 @@ void SystemLog::prepareTable() table = nullptr; } else if (!is_prepared) - LOG_DEBUG(log, "Will use existing table " << description << " for " + LogElement::name()); + LOG_DEBUG(log, "Will use existing table " << description << " for " << LogElement::name()); } if (!table) { /// Create the table. - LOG_DEBUG(log, "Creating new table " << description << " for " + LogElement::name()); + LOG_DEBUG(log, "Creating new table " << description << " for " << LogElement::name()); auto create = getCreateTableQuery(); diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 1a10edf3ab3..8fe40a0a109 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -572,7 +572,7 @@ static std::tuple executeQueryImpl( std::stringstream log_str; log_str << "Query pipeline:\n"; res.in->dumpTree(log_str); - LOG_DEBUG(&Logger::get("executeQuery"), log_str.str()); + LOG_DEBUG_FORMATTED(&Logger::get("executeQuery"), log_str.str()); } } } diff --git a/src/Processors/Merges/Algorithms/CollapsingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/CollapsingSortedAlgorithm.cpp index 8e799664fae..b13f0f06de0 100644 --- a/src/Processors/Merges/Algorithms/CollapsingSortedAlgorithm.cpp +++ b/src/Processors/Merges/Algorithms/CollapsingSortedAlgorithm.cpp @@ -42,10 +42,6 @@ void CollapsingSortedAlgorithm::reportIncorrectData() return; std::stringstream s; - s << "Incorrect data: number of rows with sign = 1 (" << count_positive - << ") differs with number of rows with sign = -1 (" << count_negative - << ") by more than one (for key: "; - auto & sort_columns = *last_row.sort_columns; for (size_t i = 0, size = sort_columns.size(); i < size; ++i) { @@ -53,14 +49,15 @@ void CollapsingSortedAlgorithm::reportIncorrectData() s << ", "; s << applyVisitor(FieldVisitorToString(), (*sort_columns[i])[last_row.row_num]); } - s << ")."; /** Fow now we limit ourselves to just logging such situations, * since the data is generated by external programs. * With inconsistent data, this is an unavoidable error that can not be easily corrected by admins. Therefore Warning. */ - LOG_WARNING(log, s.rdbuf()); + LOG_WARNING_FORMATTED(log, + "Incorrect data: number of rows with sign = 1 ({}) differs with number of rows with sign = -1 ({}) by more than one (for key: {}).", + count_positive, count_negative, s.str()); } void CollapsingSortedAlgorithm::insertRow(RowRef & row) diff --git a/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/src/Storages/Distributed/DistributedBlockOutputStream.cpp index ba883b3a357..3ce472f300d 100644 --- a/src/Storages/Distributed/DistributedBlockOutputStream.cpp +++ b/src/Storages/Distributed/DistributedBlockOutputStream.cpp @@ -114,7 +114,8 @@ void DistributedBlockOutputStream::write(const Block & block) if (ordinary_block.has(col.name)) { ordinary_block.erase(col.name); - LOG_DEBUG(log, storage.getStorageID().getNameForLogs() << ": column " + col.name + " will be removed, because it is MATERIALIZED"); + LOG_DEBUG_FORMATTED(log, "{}: column {} will be removed, because it is MATERIALIZED", + storage.getStorageID().getNameForLogs(), col.name); } } @@ -408,10 +409,10 @@ void DistributedBlockOutputStream::writeSync(const Block & block) void DistributedBlockOutputStream::writeSuffix() { - auto log_performance = [this] () + auto log_performance = [this]() { double elapsed = watch.elapsedSeconds(); - LOG_DEBUG_FORMATTED(log, "It took {} sec. to insert {} blocks, {} rows per second. {}", elapsed, inserted_blocks, inserted_rows / elapsed, LOG_DEBUG0); + LOG_DEBUG_FORMATTED(log, "It took {} sec. to insert {} blocks, {} rows per second. {}", elapsed, inserted_blocks, inserted_rows / elapsed, getCurrentStateDescription()); }; if (insert_sync && pool) From 93626701b801543ff8987775916e6d56a47d1676 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 24 May 2020 00:27:27 +0300 Subject: [PATCH 665/738] Apply all transformations again --- src/Interpreters/ExternalLoader.cpp | 12 ++++++------ src/Interpreters/SystemLog.cpp | 2 +- src/Interpreters/SystemLog.h | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index f81fed8c8ac..6ac36cff941 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -439,7 +439,7 @@ public: { /// The object has been tried to load before, so it is currently in use or was in use /// and we should try to reload it with the new config. - LOG_TRACE(log, "Will reload '" << name << "' because its configuration has been changed and there were attempts to load it before"); + LOG_TRACE_FORMATTED(log, "Will reload '{}' because its configuration has been changed and there were attempts to load it before", name); startLoading(info, true); } } @@ -454,7 +454,7 @@ public: Info & info = infos.emplace(name, Info{name, config}).first->second; if (always_load_everything) { - LOG_TRACE(log, "Will load '" << name << "' because always_load_everything flag is set."); + LOG_TRACE_FORMATTED(log, "Will load '{}' because always_load_everything flag is set.", name); startLoading(info); } } @@ -664,7 +664,7 @@ public: if (!should_update_flag) { info.next_update_time = calculateNextUpdateTime(info.object, info.error_count); - LOG_TRACE(log, "Object '" << info.name << "' not modified, will not reload. Next update at " << ext::to_string(info.next_update_time)); + LOG_TRACE_FORMATTED(log, "Object '{}' not modified, will not reload. Next update at {}", info.name, ext::to_string(info.next_update_time)); continue; } @@ -676,7 +676,7 @@ public: /// Object was never loaded successfully and should be reloaded. startLoading(info); } - LOG_TRACE(log, "Object '" << info.name << "' is neither loaded nor failed, so it will not be reloaded as outdated."); + LOG_TRACE_FORMATTED(log, "Object '{}' is neither loaded nor failed, so it will not be reloaded as outdated.", info.name); } } } @@ -1121,13 +1121,13 @@ private: } auto result = std::chrono::system_clock::now() + std::chrono::seconds(calculateDurationWithBackoff(rnd_engine, error_count)); - LOG_TRACE(log, "Supposed update time for '" << loaded_object->getLoadableName() << "' is " << ext::to_string(result) << " (backoff, " << error_count << " errors)"); + LOG_TRACE_FORMATTED(log, "Supposed update time for '{}' is {} (backoff, {} errors)", loaded_object->getLoadableName(), ext::to_string(result), error_count); return result; } else { auto result = std::chrono::system_clock::now() + std::chrono::seconds(calculateDurationWithBackoff(rnd_engine, error_count)); - LOG_TRACE(log, "Supposed update time for unspecified object is " << ext::to_string(result) << " (backoff, " << error_count << " errors."); + LOG_TRACE_FORMATTED(log, "Supposed update time for unspecified object is {} (backoff, {} errors.", ext::to_string(result), error_count); return result; } } diff --git a/src/Interpreters/SystemLog.cpp b/src/Interpreters/SystemLog.cpp index c9b00e04f10..9ce15f12138 100644 --- a/src/Interpreters/SystemLog.cpp +++ b/src/Interpreters/SystemLog.cpp @@ -41,7 +41,7 @@ std::shared_ptr createSystemLog( if (database != default_database_name) { /// System tables must be loaded before other tables, but loading order is undefined for all databases except `system` - LOG_ERROR(&Logger::get("SystemLog"), "Custom database name for a system table specified in config. Table `" << table << "` will be created in `system` database instead of `" << database << "`"); + LOG_ERROR_FORMATTED(&Logger::get("SystemLog"), "Custom database name for a system table specified in config. Table `{}` will be created in `system` database instead of `{}`", table, database); database = default_database_name; } diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h index 8f40ff0dc28..72d85e10f14 100644 --- a/src/Interpreters/SystemLog.h +++ b/src/Interpreters/SystemLog.h @@ -456,7 +456,7 @@ void SystemLog::prepareTable() rename->elements.emplace_back(elem); - LOG_DEBUG(log, "Existing table " << description << " for system log has obsolete or different structure. Renaming it to " << backQuoteIfNeed(to.table)); + LOG_DEBUG_FORMATTED(log, "Existing table {} for system log has obsolete or different structure. Renaming it to {}", description, backQuoteIfNeed(to.table)); InterpreterRenameQuery(rename, context).execute(); @@ -464,13 +464,13 @@ void SystemLog::prepareTable() table = nullptr; } else if (!is_prepared) - LOG_DEBUG(log, "Will use existing table " << description << " for " << LogElement::name()); + LOG_DEBUG_FORMATTED(log, "Will use existing table {} for {}", description, LogElement::name()); } if (!table) { /// Create the table. - LOG_DEBUG(log, "Creating new table " << description << " for " << LogElement::name()); + LOG_DEBUG_FORMATTED(log, "Creating new table {} for {}", description, LogElement::name()); auto create = getCreateTableQuery(); From c87fb91532c68a1fbc4c0026fb7f62aa6fc334ea Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 24 May 2020 00:41:35 +0300 Subject: [PATCH 666/738] Progress on task --- src/Interpreters/ExternalLoader.cpp | 4 ++-- src/Interpreters/executeQuery.cpp | 19 ++++++++++++++++--- .../MergeTree/MergeTreeDataMergerMutator.cpp | 9 ++++++++- .../MergeTreeReverseSelectProcessor.cpp | 5 ++++- .../MergeTree/MergeTreeSelectProcessor.cpp | 4 +++- 5 files changed, 33 insertions(+), 8 deletions(-) diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index 6ac36cff941..c827c4a8419 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -1099,7 +1099,7 @@ private: { if (!loaded_object->supportUpdates()) { - LOG_TRACE(log, "Supposed update time for '" + loaded_object->getLoadableName() + "' is never (loaded, does not support updates)"); + LOG_TRACE(log, "Supposed update time for '" << loaded_object->getLoadableName() << "' is never (loaded, does not support updates)"); return never; } @@ -1108,7 +1108,7 @@ private: const auto & lifetime = loaded_object->getLifetime(); if (lifetime.min_sec == 0 && lifetime.max_sec == 0) { - LOG_TRACE(log, "Supposed update time for '" + loaded_object->getLoadableName() + "' is never (loaded, lifetime 0)"); + LOG_TRACE(log, "Supposed update time for '" << loaded_object->getLoadableName() << "' is never (loaded, lifetime 0)"); return never; } diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 8fe40a0a109..ad899aa31a9 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -113,7 +113,11 @@ static void logQuery(const String & query, const Context & context, bool interna const auto & initial_query_id = context.getClientInfo().initial_query_id; const auto & current_user = context.getClientInfo().current_user; - LOG_DEBUG(&Logger::get("executeQuery"), "(from " << context.getClientInfo().current_address.toString() << (current_user != "default" ? ", user: " + context.getClientInfo().current_user : "") << (!initial_query_id.empty() && current_query_id != initial_query_id ? ", initial_query_id: " + initial_query_id : std::string()) << ") " << joinLines(query)); + LOG_DEBUG_FORMATTED(&Logger::get("executeQuery"), "(from {}{}{}) {}", + context.getClientInfo().current_address.toString(), + (current_user != "default" ? ", user: " + context.getClientInfo().current_user : ""), + (!initial_query_id.empty() && current_query_id != initial_query_id ? ", initial_query_id: " + initial_query_id : std::string()), + joinLines(query)); } } @@ -140,7 +144,13 @@ static void setExceptionStackTrace(QueryLogElement & elem) /// Log exception (with query info) into text log (not into system table). static void logException(Context & context, QueryLogElement & elem) { - LOG_ERROR(&Logger::get("executeQuery"), elem.exception << " (from " << context.getClientInfo().current_address.toString() << ") (in query: " << joinLines(elem.query) << ")" << (!elem.stack_trace.empty() ? ", Stack trace (when copying this message, always include the lines below):\n\n" + elem.stack_trace : "")); + if (elem.stack_trace.empty()) + LOG_ERROR_FORMATTED(&Logger::get("executeQuery"), "{} (from {}) (in query: {})", + elem.exception, context.getClientInfo().current_address.toString(), joinLines(elem.query)); + else + LOG_ERROR_FORMATTED(&Logger::get("executeQuery"), "{} (from {}) (in query: {})" + ", Stack trace (when copying this message, always include the lines below):\n\n{}", + elem.exception, context.getClientInfo().current_address.toString(), joinLines(elem.query), elem.stack_trace); } @@ -495,7 +505,10 @@ static std::tuple executeQueryImpl( if (elem.read_rows != 0) { - LOG_INFO(&Logger::get("executeQuery"), std::fixed << std::setprecision(3) << "Read " << elem.read_rows << " rows, " << formatReadableSizeWithBinarySuffix(elem.read_bytes) << " in " << elapsed_seconds << " sec., " << static_cast(elem.read_rows / elapsed_seconds) << " rows/sec., " << formatReadableSizeWithBinarySuffix(elem.read_bytes / elapsed_seconds) << "/sec."); + LOG_INFO_FORMATTED(&Logger::get("executeQuery"), "Read {} rows, {} in {} sec., {} rows/sec., {}/sec.", + elem.read_rows, formatReadableSizeWithBinarySuffix(elem.read_bytes), elapsed_seconds, + static_cast(elem.read_rows / elapsed_seconds), + formatReadableSizeWithBinarySuffix(elem.read_bytes / elapsed_seconds)); } elem.thread_ids = std::move(info.thread_ids); diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index e4d2ff520d5..9a570664f15 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -366,7 +366,14 @@ bool MergeTreeDataMergerMutator::selectAllPartsToMergeWithinPartition( if (now - disk_space_warning_time > 3600) { disk_space_warning_time = now; - LOG_WARNING(log, "Won't merge parts from " << parts.front()->name << " to " << (*prev_it)->name << " because not enough free space: " << formatReadableSizeWithBinarySuffix(available_disk_space) << " free and unreserved, " << formatReadableSizeWithBinarySuffix(sum_bytes) << " required now (+" << static_cast((DISK_USAGE_COEFFICIENT_TO_SELECT - 1.0) * 100) << "% on overhead); suppressing similar warnings for the next hour"); + LOG_WARNING_FORMATTED(log, + "Won't merge parts from {} to {} because not enough free space: {} free and unreserved" + ", {} required now (+{}% on overhead); suppressing similar warnings for the next hour", + parts.front()->name, + (*prev_it)->name, + formatReadableSizeWithBinarySuffix(available_disk_space), + formatReadableSizeWithBinarySuffix(sum_bytes), + static_cast((DISK_USAGE_COEFFICIENT_TO_SELECT - 1.0) * 100)); } if (out_disable_reason) diff --git a/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp index 1ea47d2aa23..f59543ac254 100644 --- a/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp @@ -66,7 +66,10 @@ MergeTreeReverseSelectProcessor::MergeTreeReverseSelectProcessor( size_t total_rows = data_part->index_granularity.getTotalRows(); if (!quiet) - LOG_TRACE(log, "Reading " << all_mark_ranges.size() << " ranges in reverse order from part " << data_part->name << ", approx. " << total_rows << (all_mark_ranges.size() > 1 ? ", up to " + toString(data_part->index_granularity.getRowsCountInRanges(all_mark_ranges)) : "") << " rows starting from " << data_part->index_granularity.getMarkStartingRow(all_mark_ranges.front().begin)); + LOG_TRACE_FORMATTED(log, "Reading {} ranges in reverse order from part {}, approx. {}, up to {} rows starting from {}", + all_mark_ranges.size(), data_part->name, total_rows, + data_part->index_granularity.getRowsCountInRanges(all_mark_ranges), + data_part->index_granularity.getMarkStartingRow(all_mark_ranges.front().begin)); addTotalRowsApprox(total_rows); diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp index 468c016cc5e..051989108d0 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp @@ -47,7 +47,9 @@ MergeTreeSelectProcessor::MergeTreeSelectProcessor( size_t total_rows = data_part->index_granularity.getRowsCountInRanges(all_mark_ranges); if (!quiet) - LOG_TRACE(log, "Reading " << all_mark_ranges.size() << " ranges from part " << data_part->name << ", approx. " << total_rows << (all_mark_ranges.size() > 1 ? ", up to " + toString(total_rows) : "") << " rows starting from " << data_part->index_granularity.getMarkStartingRow(all_mark_ranges.front().begin)); + LOG_TRACE_FORMATTED(log, "Reading {} ranges from part {}, approx. {} rows starting from {}", + all_mark_ranges.size(), data_part->name, total_rows, + data_part->index_granularity.getMarkStartingRow(all_mark_ranges.front().begin)); addTotalRowsApprox(total_rows); ordered_names = header_without_virtual_columns.getNames(); From d853bc39c5caab9d6b1bac570f607059174bdb69 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 24 May 2020 00:41:48 +0300 Subject: [PATCH 667/738] Apply all transformations again --- src/Interpreters/ExternalLoader.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index c827c4a8419..da8db263ebf 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -1099,7 +1099,7 @@ private: { if (!loaded_object->supportUpdates()) { - LOG_TRACE(log, "Supposed update time for '" << loaded_object->getLoadableName() << "' is never (loaded, does not support updates)"); + LOG_TRACE_FORMATTED(log, "Supposed update time for '{}' is never (loaded, does not support updates)", loaded_object->getLoadableName()); return never; } @@ -1108,7 +1108,7 @@ private: const auto & lifetime = loaded_object->getLifetime(); if (lifetime.min_sec == 0 && lifetime.max_sec == 0) { - LOG_TRACE(log, "Supposed update time for '" << loaded_object->getLoadableName() << "' is never (loaded, lifetime 0)"); + LOG_TRACE_FORMATTED(log, "Supposed update time for '{}' is never (loaded, lifetime 0)", loaded_object->getLoadableName()); return never; } From af2a59e49b1b44b7d2427236371ea596213917a5 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 24 May 2020 00:50:34 +0300 Subject: [PATCH 668/738] Progress on task --- src/Interpreters/ExternalLoader.cpp | 14 ++++++++++---- src/Storages/Kafka/parseSyslogLevel.cpp | 2 +- .../MergeTree/MergeTreeSequentialSource.cpp | 15 +++++++-------- src/Storages/StorageMergeTree.cpp | 4 ++-- src/Storages/StorageReplicatedMergeTree.cpp | 8 +++----- 5 files changed, 23 insertions(+), 20 deletions(-) diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index da8db263ebf..048b4b91f4b 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -286,14 +286,14 @@ private: if (!startsWith(key, settings.external_config)) { if (!startsWith(key, "comment") && !startsWith(key, "include_from")) - LOG_WARNING(log, path << ": file contains unknown node '" << key << "', expected '" << settings.external_config << "'"); + LOG_WARNING_FORMATTED(log, "{}: file contains unknown node '{}', expected '{}'", path, key, settings.external_config); continue; } String object_name = file_contents.getString(key + "." + settings.external_name); if (object_name.empty()) { - LOG_WARNING(log, path << ": node '" << key << "' defines " << type_name << " with an empty name. It's not allowed"); + LOG_WARNING_FORMATTED(log, "{}: node '{}' defines {} with an empty name. It's not allowed", path, key, type_name); continue; } @@ -350,7 +350,12 @@ private: const auto & already_added = already_added_it->second; if (!already_added->from_temp_repository && !repository->isTemporary()) { - LOG_WARNING(log, type_name << " '" << object_name << "' is found " << (((path == already_added->path) && (repository->getName() == already_added->repository_name)) ? ("twice in the same file '" + path + "'") : ("both in file '" + already_added->path + "' and '" + path + "'"))); + if (path == already_added->path && repository->getName() == already_added->repository_name) + LOG_WARNING_FORMATTED(log, "{} '{}' is found twice in the same file '{}'", + type_name, object_name, path); + else + LOG_WARNING_FORMATTED(log, "{} '{}' is found both in file '{}' and '{}'", + type_name, object_name, already_added->path, path); } } } @@ -1116,7 +1121,8 @@ private: { std::uniform_int_distribution distribution{lifetime.min_sec, lifetime.max_sec}; auto result = std::chrono::system_clock::now() + std::chrono::seconds{distribution(rnd_engine)}; - LOG_TRACE(log, "Supposed update time for '" << loaded_object->getLoadableName() << "' is " << ext::to_string(result) << " (loaded, lifetime [" << lifetime.min_sec << ", " << lifetime.max_sec << "], no errors)"); + LOG_TRACE_FORMATTED(log, "Supposed update time for '{}' is {} (loaded, lifetime [{}, {}], no errors)", + loaded_object->getLoadableName(), ext::to_string(result), lifetime.min_sec, lifetime.max_sec); return result; } diff --git a/src/Storages/Kafka/parseSyslogLevel.cpp b/src/Storages/Kafka/parseSyslogLevel.cpp index db948d5c668..96a6885111a 100644 --- a/src/Storages/Kafka/parseSyslogLevel.cpp +++ b/src/Storages/Kafka/parseSyslogLevel.cpp @@ -3,7 +3,7 @@ /// Must be in a sepearate compilation unit due to macros overlaps: /// - syslog (LOG_DEBUG/...) -/// - logger_useful.h (LOG_DEBUG()/...) +/// - logger_useful.h (LOG_DEBUG(...)/...) std::pair parseSyslogLevel(const int level) { using DB::LogsLevel; diff --git a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp index 755ff87eab5..b27d8bcfa97 100644 --- a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp +++ b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp @@ -25,14 +25,13 @@ MergeTreeSequentialSource::MergeTreeSequentialSource( { if (!quiet) { - std::stringstream message; - message << "Reading " << data_part->getMarksCount() << " marks from part " << data_part->name - << ", total " << data_part->rows_count - << " rows starting from the beginning of the part"; - if (columns_to_read.size() == 1) /// Print column name but don't pollute logs in case of many columns. - message << ", column " << columns_to_read.front(); - - LOG_TRACE(log, message.rdbuf()); + /// Print column name but don't pollute logs in case of many columns. + if (columns_to_read.size() == 1) + LOG_TRACE_FORMATTED(log, "Reading {} marks from part {}, total {} rows starting from the beginning of the part, column {}", + data_part->getMarksCount(), data_part->name, data_part->rows_count, columns_to_read.front()); + else + LOG_TRACE_FORMATTED(log, "Reading {} marks from part {}, total {} rows starting from the beginning of the part", + data_part->getMarksCount(), data_part->name, data_part->rows_count); } addTotalRowsApprox(data_part->rows_count); diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index d5fd36b89a2..8070432c2db 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -923,7 +923,7 @@ bool StorageMergeTree::optimize( message << ": " << disable_reason; else message << " by some reason."; - LOG_INFO(log, message.rdbuf()); + LOG_INFO_FORMATTED(log, message.str()); if (context.getSettingsRef().optimize_throw_if_noop) throw Exception(message.str(), ErrorCodes::CANNOT_ASSIGN_OPTIMIZE); @@ -945,7 +945,7 @@ bool StorageMergeTree::optimize( message << ": " << disable_reason; else message << " by some reason."; - LOG_INFO(log, message.rdbuf()); + LOG_INFO_FORMATTED(log, message.str()); if (context.getSettingsRef().optimize_throw_if_noop) throw Exception(message.str(), ErrorCodes::CANNOT_ASSIGN_OPTIMIZE); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index f431a73668f..49569c96894 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -974,13 +974,11 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) { // Log source part names just in case { - std::stringstream log_message; - log_message << "Executing log entry to merge parts "; + std::stringstream source_parts_msg; for (auto i : ext::range(0, entry.source_parts.size())) - log_message << (i != 0 ? ", " : "") << entry.source_parts[i]; - log_message << " to " << entry.new_part_name; + source_parts_msg << (i != 0 ? ", " : "") << entry.source_parts[i]; - LOG_TRACE(log, log_message.rdbuf()); + LOG_TRACE_FORMATTED(log, "Executing log entry to merge parts {} to {}", source_parts_msg.str(), entry.new_part_name); } const auto storage_settings_ptr = getSettings(); From 61cc605ee7210d8d5b37b819118b02a43f3af662 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 24 May 2020 01:21:29 +0300 Subject: [PATCH 669/738] Remove old method --- base/common/logger_useful.h | 31 --------- base/daemon/BaseDaemon.cpp | 10 +-- base/ext/chrono_io.h | 1 + programs/copier/ClusterCopier.cpp | 31 +++++---- programs/odbc-bridge/ColumnInfoHandler.cpp | 2 +- .../odbc-bridge/IdentifierQuoteHandler.cpp | 2 +- programs/odbc-bridge/MainHandler.cpp | 4 +- programs/odbc-bridge/ODBCBridge.cpp | 2 +- programs/server/HTTPHandlerFactory.cpp | 5 +- programs/server/MySQLHandler.cpp | 14 +++- programs/server/Server.cpp | 64 ++++++++++++++----- programs/server/TCPHandler.cpp | 9 ++- src/Common/Config/ConfigProcessor.cpp | 1 + 13 files changed, 102 insertions(+), 74 deletions(-) diff --git a/base/common/logger_useful.h b/base/common/logger_useful.h index 526a1a6ce44..8980ba9f94d 100644 --- a/base/common/logger_useful.h +++ b/base/common/logger_useful.h @@ -2,7 +2,6 @@ /// Macros for convenient usage of Poco logger. -#include #include #include #include @@ -19,36 +18,6 @@ using DB::CurrentThread; /// Logs a message to a specified logger with that level. -#define LOG_IMPL(logger, priority, PRIORITY, message) do \ -{ \ - const bool is_clients_log = (CurrentThread::getGroup() != nullptr) && \ - (CurrentThread::getGroup()->client_logs_level >= (priority)); \ - if ((logger)->is((PRIORITY)) || is_clients_log) \ - { \ - std::stringstream oss_internal_rare; \ - oss_internal_rare << message; \ - if (auto channel = (logger)->getChannel()) \ - { \ - std::string file_function; \ - file_function += __FILE__; \ - file_function += "; "; \ - file_function += __PRETTY_FUNCTION__; \ - Message poco_message((logger)->name(), oss_internal_rare.str(), \ - (PRIORITY), file_function.c_str(), __LINE__); \ - channel->log(poco_message); \ - } \ - } \ -} while (false) - - -#define LOG_TRACE(logger, message) LOG_IMPL(logger, LogsLevel::trace, Message::PRIO_TRACE, message) -#define LOG_DEBUG(logger, message) LOG_IMPL(logger, LogsLevel::debug, Message::PRIO_DEBUG, message) -#define LOG_INFO(logger, message) LOG_IMPL(logger, LogsLevel::information, Message::PRIO_INFORMATION, message) -#define LOG_WARNING(logger, message) LOG_IMPL(logger, LogsLevel::warning, Message::PRIO_WARNING, message) -#define LOG_ERROR(logger, message) LOG_IMPL(logger, LogsLevel::error, Message::PRIO_ERROR, message) -#define LOG_FATAL(logger, message) LOG_IMPL(logger, LogsLevel::error, Message::PRIO_FATAL, message) - - #define LOG_IMPL_FORMATTED(logger, priority, PRIORITY, ...) do \ { \ const bool is_clients_log = (CurrentThread::getGroup() != nullptr) && \ diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index 751a658ea6f..7665612f10e 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -236,7 +236,7 @@ private: void onTerminate(const std::string & message, UInt32 thread_num) const { - LOG_FATAL(log, "(version " << VERSION_STRING << VERSION_OFFICIAL << ") (from thread " << thread_num << ") " << message); + LOG_FATAL_FORMATTED(log, "(version {}{}) (from thread {}) {}", VERSION_STRING, VERSION_OFFICIAL, thread_num, message); } void onFault( @@ -259,10 +259,10 @@ private: message << " (query_id: " << query_id << ")"; message << " Received signal " << strsignal(sig) << " (" << sig << ")."; - LOG_FATAL(log, message.rdbuf()); + LOG_FATAL_FORMATTED(log, message.str()); } - LOG_FATAL(log, signalToErrorMessage(sig, info, context)); + LOG_FATAL_FORMATTED(log, signalToErrorMessage(sig, info, context)); if (stack_trace.getSize()) { @@ -274,7 +274,7 @@ private: for (size_t i = stack_trace.getOffset(); i < stack_trace.getSize(); ++i) bare_stacktrace << ' ' << stack_trace.getFrames()[i]; - LOG_FATAL(log, bare_stacktrace.rdbuf()); + LOG_FATAL_FORMATTED(log, bare_stacktrace.str()); } /// Write symbolized stack trace line by line for better grep-ability. @@ -379,7 +379,7 @@ static bool tryCreateDirectories(Poco::Logger * logger, const std::string & path } catch (...) { - LOG_WARNING(logger, __PRETTY_FUNCTION__ << ": when creating " << path << ", " << DB::getCurrentExceptionMessage(true)); + LOG_WARNING_FORMATTED(logger, "{}: when creating {}, {}", __PRETTY_FUNCTION__, path, DB::getCurrentExceptionMessage(true)); } return false; } diff --git a/base/ext/chrono_io.h b/base/ext/chrono_io.h index 40b76a96478..392ec25d526 100644 --- a/base/ext/chrono_io.h +++ b/base/ext/chrono_io.h @@ -4,6 +4,7 @@ #include #include #include +#include namespace ext diff --git a/programs/copier/ClusterCopier.cpp b/programs/copier/ClusterCopier.cpp index 00e60355209..cb0bcbaca83 100644 --- a/programs/copier/ClusterCopier.cpp +++ b/programs/copier/ClusterCopier.cpp @@ -203,7 +203,8 @@ void ClusterCopier::uploadTaskDescription(const std::string & task_path, const s if (code && force) zookeeper->createOrUpdate(local_task_description_path, task_config_str, zkutil::CreateMode::Persistent); - LOG_DEBUG(log, "Task description " << ((code && !force) ? "not " : "") << "uploaded to " << local_task_description_path << " with result " << code << " ("<< zookeeper->error2string(code) << ")"); + LOG_DEBUG_FORMATTED(log, "Task description {} uploaded to {} with result {} ({})", + ((code && !force) ? "not " : ""), local_task_description_path, code, zookeeper->error2string(code)); } void ClusterCopier::reloadTaskDescription() @@ -414,7 +415,7 @@ bool ClusterCopier::checkAllPiecesInPartitionAreDone(const TaskTable & task_tabl { bool piece_is_done = checkPartitionPieceIsDone(task_table, partition_name, piece_number, shards_with_partition); if (!piece_is_done) - LOG_DEBUG(log, "Partition " << partition_name << " piece " + toString(piece_number) + " is not already done."); + LOG_DEBUG_FORMATTED(log, "Partition {} piece {} is not already done.", partition_name, piece_number); answer &= piece_is_done; } @@ -430,7 +431,7 @@ bool ClusterCopier::checkAllPiecesInPartitionAreDone(const TaskTable & task_tabl bool ClusterCopier::checkPartitionPieceIsDone(const TaskTable & task_table, const String & partition_name, size_t piece_number, const TasksShard & shards_with_partition) { - LOG_DEBUG(log, "Check that all shards processed partition " << partition_name << " piece " + toString(piece_number) + " successfully"); + LOG_DEBUG_FORMATTED(log, "Check that all shards processed partition {} piece {} successfully", partition_name, piece_number); auto zookeeper = context.getZooKeeper(); @@ -1010,7 +1011,11 @@ bool ClusterCopier::tryProcessTable(const ConnectionTimeouts & timeouts, TaskTab task_table.rows_copied += cluster_partition.rows_copied; double elapsed = cluster_partition.elapsed_time_seconds; - LOG_INFO(log, "It took " << std::fixed << std::setprecision(2) << elapsed << " seconds to copy partition " << partition_name << ": " << formatReadableSizeWithDecimalSuffix(cluster_partition.bytes_copied) << " uncompressed bytes, " << formatReadableQuantity(cluster_partition.rows_copied) << " rows and " << cluster_partition.blocks_copied << " source blocks are copied"); + LOG_INFO_FORMATTED(log, "It took {} seconds to copy partition {}: {} uncompressed bytes, {} rows and {} source blocks are copied", + elapsed, partition_name, + formatReadableSizeWithDecimalSuffix(cluster_partition.bytes_copied), + formatReadableQuantity(cluster_partition.rows_copied), + cluster_partition.blocks_copied); if (cluster_partition.rows_copied) { @@ -1030,7 +1035,7 @@ bool ClusterCopier::tryProcessTable(const ConnectionTimeouts & timeouts, TaskTab if (!table_is_done) { - LOG_INFO(log, "Table " + task_table.table_id + " is not processed yet.Copied " << finished_partitions << " of " << required_partitions << ", will retry"); + LOG_INFO_FORMATTED(log, "Table {} is not processed yet.Copied {} of {}, will retry", task_table.table_id, finished_partitions, required_partitions); } return table_is_done; @@ -1078,7 +1083,11 @@ TaskStatus ClusterCopier::iterateThroughAllPiecesInPartition(const ConnectionTim { for (UInt64 try_num = 0; try_num < max_shard_partition_tries; ++try_num) { - LOG_INFO(log, "Attempt number " << try_num << " to process partition " << task_partition.name << " piece number " << piece_number << " on shard number " << task_partition.task_shard.numberInCluster() << " with index " << task_partition.task_shard.indexInCluster()); + LOG_INFO_FORMATTED(log, "Attempt number {} to process partition {} piece number {} on shard number {} with index {}.", + try_num, task_partition.name, piece_number, + task_partition.task_shard.numberInCluster(), + task_partition.task_shard.indexInCluster()); + res = processPartitionPieceTaskImpl(timeouts, task_partition, piece_number, is_unprioritized_task); /// Exit if success @@ -1193,12 +1202,12 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( /// Do not start if partition piece is dirty, try to clean it if (is_clean) { - LOG_DEBUG(log, "Partition " << task_partition.name << " piece " + toString(current_piece_number) + " appears to be clean"); + LOG_DEBUG_FORMATTED(log, "Partition {} piece {} appears to be clean", task_partition.name, current_piece_number); zookeeper->createAncestors(current_task_piece_status_path); } else { - LOG_DEBUG(log, "Partition " << task_partition.name << " piece " + toString(current_piece_number) + " is dirty, try to drop it"); + LOG_DEBUG_FORMATTED(log, "Partition {} piece {} is dirty, try to drop it", task_partition.name, current_piece_number); try { @@ -1260,9 +1269,9 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( String state_finished = TaskStateWithOwner::getData(TaskState::Finished, host_id); auto res = zookeeper->tryCreate(current_task_piece_status_path, state_finished, zkutil::CreateMode::Persistent); if (res == Coordination::ZNODEEXISTS) - LOG_DEBUG(log, "Partition " << task_partition.name << " piece " + toString(current_piece_number) + " is absent on current replica of a shard. But other replicas have already marked it as done."); + LOG_DEBUG_FORMATTED(log, "Partition {} piece {} is absent on current replica of a shard. But other replicas have already marked it as done.", task_partition.name, current_piece_number); if (res == Coordination::ZOK) - LOG_DEBUG(log, "Partition " << task_partition.name << " piece " + toString(current_piece_number) + " is absent on current replica of a shard. Will mark it as done. Other replicas will do the same."); + LOG_DEBUG_FORMATTED(log, "Partition {} piece {} is absent on current replica of a shard. Will mark it as done. Other replicas will do the same.", task_partition.name, current_piece_number); return TaskStatus::Finished; } @@ -1859,7 +1868,7 @@ UInt64 ClusterCopier::executeQueryOnCluster( } catch (const Exception &) { - LOG_INFO(log, getCurrentExceptionMessage(false, true)); + LOG_INFO_FORMATTED(log, getCurrentExceptionMessage(false, true)); } } } diff --git a/programs/odbc-bridge/ColumnInfoHandler.cpp b/programs/odbc-bridge/ColumnInfoHandler.cpp index 4bc3522b220..d8579a5d171 100644 --- a/programs/odbc-bridge/ColumnInfoHandler.cpp +++ b/programs/odbc-bridge/ColumnInfoHandler.cpp @@ -69,7 +69,7 @@ void ODBCColumnsInfoHandler::handleRequest(Poco::Net::HTTPServerRequest & reques response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); if (!response.sent()) response.send() << message << std::endl; - LOG_WARNING(log, message); + LOG_WARNING_FORMATTED(log, message); }; if (!params.has("table")) diff --git a/programs/odbc-bridge/IdentifierQuoteHandler.cpp b/programs/odbc-bridge/IdentifierQuoteHandler.cpp index 83768a3d941..c7c9ab941bf 100644 --- a/programs/odbc-bridge/IdentifierQuoteHandler.cpp +++ b/programs/odbc-bridge/IdentifierQuoteHandler.cpp @@ -32,7 +32,7 @@ void IdentifierQuoteHandler::handleRequest(Poco::Net::HTTPServerRequest & reques response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); if (!response.sent()) response.send() << message << std::endl; - LOG_WARNING(log, message); + LOG_WARNING_FORMATTED(log, message); }; if (!params.has("connection_string")) diff --git a/programs/odbc-bridge/MainHandler.cpp b/programs/odbc-bridge/MainHandler.cpp index b46a03472ab..63d24336de1 100644 --- a/programs/odbc-bridge/MainHandler.cpp +++ b/programs/odbc-bridge/MainHandler.cpp @@ -76,7 +76,7 @@ void ODBCHandler::processError(Poco::Net::HTTPServerResponse & response, const s response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); if (!response.sent()) response.send() << message << std::endl; - LOG_WARNING(log, message); + LOG_WARNING_FORMATTED(log, message); } void ODBCHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) @@ -125,7 +125,7 @@ void ODBCHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne catch (const Exception & ex) { processError(response, "Invalid 'columns' parameter in request body '" + ex.message() + "'"); - LOG_WARNING(log, ex.getStackTraceString()); + LOG_WARNING_FORMATTED(log, ex.getStackTraceString()); return; } diff --git a/programs/odbc-bridge/ODBCBridge.cpp b/programs/odbc-bridge/ODBCBridge.cpp index dec071dc1cb..4a868cdee8c 100644 --- a/programs/odbc-bridge/ODBCBridge.cpp +++ b/programs/odbc-bridge/ODBCBridge.cpp @@ -48,7 +48,7 @@ namespace #endif ) { - LOG_ERROR(log, "Cannot resolve listen_host (" << host << "), error " << e.code() << ": " << e.message() << ". " "If it is an IPv6 address and your host has disabled IPv6, then consider to " "specify IPv4 address to listen in element of configuration " "file. Example: 0.0.0.0"); + LOG_ERROR_FORMATTED(log, "Cannot resolve listen_host ({}), error {}: {}. If it is an IPv6 address and your host has disabled IPv6, then consider to specify IPv4 address to listen in element of configuration file. Example: 0.0.0.0", host, e.code(), e.message()); } throw; diff --git a/programs/server/HTTPHandlerFactory.cpp b/programs/server/HTTPHandlerFactory.cpp index abb1fd10be9..d5e984685f3 100644 --- a/programs/server/HTTPHandlerFactory.cpp +++ b/programs/server/HTTPHandlerFactory.cpp @@ -30,7 +30,10 @@ HTTPRequestHandlerFactoryMain::HTTPRequestHandlerFactoryMain(const std::string & Poco::Net::HTTPRequestHandler * HTTPRequestHandlerFactoryMain::createRequestHandler(const Poco::Net::HTTPServerRequest & request) { - LOG_TRACE(log, "HTTP Request for " << name << ". Method: " << request.getMethod() << ", Address: " << request.clientAddress().toString() << ", User-Agent: " << (request.has("User-Agent") ? request.get("User-Agent") : "none") << (request.hasContentLength() ? (", Length: " + std::to_string(request.getContentLength())) : ("")) << ", Content Type: " << request.getContentType() << ", Transfer Encoding: " << request.getTransferEncoding()); + LOG_TRACE_FORMATTED(log, "HTTP Request for {}. Method: {}, Address: {}, User-Agent: {}{}, Content Type: {}, Transfer Encoding: {}", + name, request.getMethod(), request.clientAddress().toString(), request.has("User-Agent") ? request.get("User-Agent") : "none", + (request.hasContentLength() ? (", Length: " + std::to_string(request.getContentLength())) : ("")), + request.getContentType(), request.getTransferEncoding()); for (auto & handler_factory : child_factories) { diff --git a/programs/server/MySQLHandler.cpp b/programs/server/MySQLHandler.cpp index f05abe085cc..fe1cd45bedb 100644 --- a/programs/server/MySQLHandler.cpp +++ b/programs/server/MySQLHandler.cpp @@ -83,7 +83,15 @@ void MySQLHandler::run() if (!connection_context.mysql.max_packet_size) connection_context.mysql.max_packet_size = MAX_PACKET_LENGTH; - LOG_TRACE(log, "Capabilities: " << handshake_response.capability_flags << ", max_packet_size: " << handshake_response.max_packet_size << ", character_set: " << static_cast(handshake_response.character_set) << ", user: " << handshake_response.username << ", auth_response length: " << handshake_response.auth_response.length() << ", database: " << handshake_response.database << ", auth_plugin_name: " << handshake_response.auth_plugin_name); + LOG_TRACE_FORMATTED(log, + "Capabilities: {}, max_packet_size: {}, character_set: {}, user: {}, auth_response length: {}, database: {}, auth_plugin_name: {}", + handshake_response.capability_flags, + handshake_response.max_packet_size, + static_cast(handshake_response.character_set), + handshake_response.username, + handshake_response.auth_response.length(), + handshake_response.database, + handshake_response.auth_plugin_name); client_capability_flags = handshake_response.capability_flags; if (!(client_capability_flags & CLIENT_PROTOCOL_41)) @@ -117,7 +125,9 @@ void MySQLHandler::run() // For commands which are executed without MemoryTracker. LimitReadBuffer limited_payload(payload, 10000, true, "too long MySQL packet."); - LOG_DEBUG(log, "Received command: " << static_cast(static_cast(command)) << ". Connection id: " << connection_id << "."); + LOG_DEBUG_FORMATTED(log, "Received command: {}. Connection id: {}.", + static_cast(static_cast(command)), connection_id); + try { switch (command) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 6399fb440cf..8c7d9b76c9c 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -282,7 +282,10 @@ int Server::main(const std::vector & /*args*/) } else { - LOG_INFO(log, "It looks like the process has no CAP_IPC_LOCK capability, binary mlock will be disabled." " It could happen due to incorrect ClickHouse package installation." " You could resolve the problem manually with 'sudo setcap cap_ipc_lock=+ep " << executable_path << "'." " Note that it will not work on 'nosuid' mounted filesystems."); + LOG_INFO_FORMATTED(log, "It looks like the process has no CAP_IPC_LOCK capability, binary mlock will be disabled." + " It could happen due to incorrect ClickHouse package installation." + " You could resolve the problem manually with 'sudo setcap cap_ipc_lock=+ep {}'." + " Note that it will not work on 'nosuid' mounted filesystems.", executable_path); } } } @@ -309,7 +312,7 @@ int Server::main(const std::vector & /*args*/) } else { - LOG_WARNING(log, message); + LOG_WARNING_FORMATTED(log, message); } } @@ -425,7 +428,8 @@ int Server::main(const std::vector & /*args*/) if (this_host.empty()) { this_host = getFQDNOrHostName(); - LOG_DEBUG(log, "Configuration parameter '" + String(host_tag) + "' doesn't exist or exists and empty. Will use '" + this_host + "' as replica host."); + LOG_DEBUG_FORMATTED(log, "Configuration parameter '{}' doesn't exist or exists and empty. Will use '{}' as replica host.", + host_tag, this_host); } String port_str = config().getString(port_tag); @@ -647,11 +651,13 @@ int Server::main(const std::vector & /*args*/) #endif #if defined(SANITIZER) - LOG_INFO(log, "Query Profiler and TraceCollector are disabled because they cannot work under sanitizers" " when two different stack unwinding methods will interfere with each other."); + LOG_INFO_FORMATTED(log, "Query Profiler and TraceCollector are disabled because they cannot work under sanitizers" + " when two different stack unwinding methods will interfere with each other."); #endif if (!hasPHDRCache()) - LOG_INFO(log, "Query Profiler and TraceCollector are disabled because they require PHDR cache to be created" " (otherwise the function 'dl_iterate_phdr' is not lock free and not async-signal safe)."); + LOG_INFO_FORMATTED(log, "Query Profiler and TraceCollector are disabled because they require PHDR cache to be created" + " (otherwise the function 'dl_iterate_phdr' is not lock free and not async-signal safe)."); global_context->setCurrentDatabase(default_database); @@ -677,12 +683,23 @@ int Server::main(const std::vector & /*args*/) #if defined(OS_LINUX) if (!TasksStatsCounters::checkIfAvailable()) { - LOG_INFO(log, "It looks like this system does not have procfs mounted at /proc location," " neither clickhouse-server process has CAP_NET_ADMIN capability." " 'taskstats' performance statistics will be disabled." " It could happen due to incorrect ClickHouse package installation." " You can try to resolve the problem manually with 'sudo setcap cap_net_admin=+ep " << executable_path << "'." " Note that it will not work on 'nosuid' mounted filesystems." " It also doesn't work if you run clickhouse-server inside network namespace as it happens in some containers."); + LOG_INFO_FORMATTED(log, "It looks like this system does not have procfs mounted at /proc location," + " neither clickhouse-server process has CAP_NET_ADMIN capability." + " 'taskstats' performance statistics will be disabled." + " It could happen due to incorrect ClickHouse package installation." + " You can try to resolve the problem manually with 'sudo setcap cap_net_admin=+ep {}'." + " Note that it will not work on 'nosuid' mounted filesystems." + " It also doesn't work if you run clickhouse-server inside network namespace as it happens in some containers.", + executable_path); } if (!hasLinuxCapability(CAP_SYS_NICE)) { - LOG_INFO(log, "It looks like the process has no CAP_SYS_NICE capability, the setting 'os_thread_nice' will have no effect." " It could happen due to incorrect ClickHouse package installation." " You could resolve the problem manually with 'sudo setcap cap_sys_nice=+ep " << executable_path << "'." " Note that it will not work on 'nosuid' mounted filesystems."); + LOG_INFO_FORMATTED(log, "It looks like the process has no CAP_SYS_NICE capability, the setting 'os_thread_nice' will have no effect." + " It could happen due to incorrect ClickHouse package installation." + " You could resolve the problem manually with 'sudo setcap cap_sys_nice=+ep {}'." + " Note that it will not work on 'nosuid' mounted filesystems.", + executable_path); } #else LOG_INFO_FORMATTED(log, "TaskStats is not implemented for this OS. IO accounting will be disabled."); @@ -724,7 +741,11 @@ int Server::main(const std::vector & /*args*/) #endif ) { - LOG_ERROR(log, "Cannot resolve listen_host (" << host << "), error " << e.code() << ": " << e.message() << ". " "If it is an IPv6 address and your host has disabled IPv6, then consider to " "specify IPv4 address to listen in element of configuration " "file. Example: 0.0.0.0"); + LOG_ERROR_FORMATTED(log, "Cannot resolve listen_host ({}), error {}: {}. " + "If it is an IPv6 address and your host has disabled IPv6, then consider to " + "specify IPv4 address to listen in element of configuration " + "file. Example: 0.0.0.0", + host, e.code(), e.message()); } throw; @@ -776,7 +797,11 @@ int Server::main(const std::vector & /*args*/) if (listen_try) { - LOG_ERROR(log, message << ". If it is an IPv6 or IPv4 address and your host has disabled IPv6 or IPv4, then consider to " "specify not disabled IPv4 or IPv6 address to listen in element of configuration " "file. Example for disabled IPv6: 0.0.0.0 ." " Example for disabled IPv4: ::"); + LOG_ERROR_FORMATTED(log, "{}. If it is an IPv6 or IPv4 address and your host has disabled IPv6 or IPv4, then consider to " + "specify not disabled IPv4 or IPv6 address to listen in element of configuration " + "file. Example for disabled IPv6: 0.0.0.0 ." + " Example for disabled IPv4: ::", + message); } else { @@ -936,12 +961,10 @@ int Server::main(const std::vector & /*args*/) dns_cache_updater->start(); { - std::stringstream message; - message << "Available RAM: " << formatReadableSizeWithBinarySuffix(memory_amount) << ";" - << " physical cores: " << getNumberOfPhysicalCPUCores() << ";" - // on ARM processors it can show only enabled at current moment cores - << " logical cores: " << std::thread::hardware_concurrency() << "."; - LOG_INFO(log, message.str()); + LOG_INFO_FORMATTED(log, "Available RAM: {}; physical cores: {}; logical cores: {}.", + formatReadableSizeWithBinarySuffix(memory_amount), + getNumberOfPhysicalCPUCores(), // on ARM processors it can show only enabled at current moment cores + std::thread::hardware_concurrency()); } LOG_INFO_FORMATTED(log, "Ready for connections."); @@ -959,7 +982,10 @@ int Server::main(const std::vector & /*args*/) current_connections += server->currentConnections(); } - LOG_INFO_FORMATTED(log, "Closed all listening sockets.{}", (current_connections ? " Waiting for " + toString(current_connections) + " outstanding connections." : "")); + if (current_connections) + LOG_INFO_FORMATTED(log, "Closed all listening sockets. Waiting for {} outstanding connections.", current_connections); + else + LOG_INFO_FORMATTED(log, "Closed all listening sockets."); /// Killing remaining queries. global_context->getProcessList().killAllQueries(); @@ -981,7 +1007,11 @@ int Server::main(const std::vector & /*args*/) } } - LOG_INFO(log, "Closed connections." << (current_connections ? " But " + toString(current_connections) + " remains." " Tip: To increase wait time add to config: 60" : "")); + if (current_connections) + LOG_INFO_FORMATTED(log, "Closed connections. But {} remain." + " Tip: To increase wait time add to config: 60", current_connections); + else + LOG_INFO_FORMATTED(log, "Closed connections."); dns_cache_updater.reset(); main_config_reloader.reset(); diff --git a/programs/server/TCPHandler.cpp b/programs/server/TCPHandler.cpp index c2a36ed4b9d..8de03adae30 100644 --- a/programs/server/TCPHandler.cpp +++ b/programs/server/TCPHandler.cpp @@ -378,7 +378,7 @@ void TCPHandler::runImpl() watch.stop(); - LOG_INFO(log, std::fixed << std::setprecision(3) << "Processed in " << watch.elapsedSeconds() << " sec."); + LOG_INFO_FORMATTED(log, "Processed in {} sec.", watch.elapsedSeconds()); /// It is important to destroy query context here. We do not want it to live arbitrarily longer than the query. query_context.reset(); @@ -730,7 +730,12 @@ void TCPHandler::receiveHello() readStringBinary(user, *in); readStringBinary(password, *in); - LOG_DEBUG(log, "Connected " << client_name << " version " << client_version_major << "." << client_version_minor << "." << client_version_patch << ", revision: " << client_revision << (!default_database.empty() ? ", database: " + default_database : "") << (!user.empty() ? ", user: " + user : "") << "."); + LOG_DEBUG_FORMATTED(log, "Connected {} version {}.{}.{}, revision: {}{}{}.", + client_name, + client_version_major, client_version_minor, client_version_patch, + client_revision, + (!default_database.empty() ? ", database: " + default_database : ""), + (!user.empty() ? ", user: " + user : "")); connection_context.setUser(user, password, socket().peerAddress()); } diff --git a/src/Common/Config/ConfigProcessor.cpp b/src/Common/Config/ConfigProcessor.cpp index ca7ccaf50c1..0e0430b7870 100644 --- a/src/Common/Config/ConfigProcessor.cpp +++ b/src/Common/Config/ConfigProcessor.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include From 7e1813825b425cb5f142c799e831e29f6f153368 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 24 May 2020 01:24:01 +0300 Subject: [PATCH 670/738] Return old names of macros --- base/common/logger_useful.h | 15 +- base/daemon/BaseDaemon.cpp | 34 +-- base/daemon/GraphiteWriter.h | 2 +- programs/copier/ClusterCopier.cpp | 174 ++++++------- programs/copier/ClusterCopierApp.cpp | 2 +- programs/copier/ZooKeeperStaff.h | 4 +- programs/local/LocalServer.cpp | 4 +- programs/odbc-bridge/ColumnInfoHandler.cpp | 12 +- programs/odbc-bridge/HandlerFactory.cpp | 2 +- .../odbc-bridge/IdentifierQuoteHandler.cpp | 4 +- programs/odbc-bridge/MainHandler.cpp | 12 +- programs/odbc-bridge/ODBCBridge.cpp | 12 +- programs/server/HTTPHandler.cpp | 4 +- programs/server/HTTPHandlerFactory.cpp | 2 +- programs/server/InterserverIOHTTPHandler.cpp | 12 +- programs/server/MySQLHandler.cpp | 14 +- programs/server/MySQLHandlerFactory.cpp | 8 +- programs/server/ReplicasStatusHandler.cpp | 2 +- programs/server/Server.cpp | 104 ++++---- programs/server/TCPHandler.cpp | 24 +- programs/server/TCPHandlerFactory.h | 4 +- src/Access/AccessRights.cpp | 4 +- src/Access/AllowedClientHosts.cpp | 4 +- src/Access/ContextAccess.cpp | 12 +- src/Access/DiskAccessStorage.cpp | 4 +- src/Client/Connection.cpp | 20 +- src/Client/ConnectionPoolWithFailover.cpp | 4 +- src/Client/TimeoutSetter.cpp | 2 +- src/Common/Config/ConfigProcessor.cpp | 14 +- src/Common/Config/ConfigReloader.cpp | 2 +- src/Common/DNSResolver.cpp | 2 +- src/Common/Exception.cpp | 6 +- src/Common/LRUCache.h | 4 +- src/Common/MemoryTracker.cpp | 4 +- src/Common/PipeFDs.cpp | 4 +- src/Common/PoolBase.h | 2 +- src/Common/PoolWithFailoverBase.h | 2 +- src/Common/QueryProfiler.cpp | 4 +- src/Common/SensitiveDataMasker.cpp | 4 +- src/Common/ShellCommand.cpp | 12 +- src/Common/StatusFile.cpp | 8 +- src/Common/TraceCollector.cpp | 2 +- src/Common/XDBCBridgeHelper.h | 6 +- src/Common/ZooKeeper/Increment.h | 2 +- src/Common/ZooKeeper/Lock.cpp | 2 +- src/Common/ZooKeeper/ZooKeeper.cpp | 6 +- src/Common/ZooKeeper/ZooKeeperHolder.cpp | 2 +- src/Core/BackgroundSchedulePool.cpp | 6 +- src/Core/MySQLProtocol.h | 16 +- src/Core/SettingsCollection.cpp | 2 +- .../AggregatingBlockInputStream.cpp | 2 +- .../CollapsingFinalBlockInputStream.cpp | 6 +- src/DataStreams/ColumnGathererStream.cpp | 4 +- .../CreatingSetsBlockInputStream.cpp | 16 +- .../MergeSortingBlockInputStream.cpp | 10 +- ...regatedMemoryEfficientBlockInputStream.cpp | 8 +- .../MergingSortedBlockInputStream.cpp | 4 +- .../ParallelAggregatingBlockInputStream.cpp | 8 +- src/DataStreams/ParallelInputsProcessor.h | 2 +- src/DataStreams/RemoteBlockInputStream.cpp | 2 +- src/DataStreams/TTLBlockInputStream.cpp | 2 +- src/DataStreams/UnionBlockInputStream.h | 4 +- src/Databases/DatabaseLazy.cpp | 10 +- src/Databases/DatabaseOnDisk.cpp | 12 +- src/Databases/DatabaseOrdinary.cpp | 6 +- src/Dictionaries/CacheDictionary.cpp | 2 +- .../ClickHouseDictionarySource.cpp | 4 +- src/Dictionaries/DictionaryFactory.cpp | 2 +- .../Embedded/RegionsHierarchies.cpp | 4 +- .../Embedded/RegionsHierarchy.cpp | 2 +- src/Dictionaries/Embedded/RegionsNames.cpp | 4 +- .../ExecutableDictionarySource.cpp | 8 +- src/Dictionaries/FileDictionarySource.cpp | 2 +- src/Dictionaries/HTTPDictionarySource.cpp | 8 +- src/Dictionaries/LibraryDictionarySource.cpp | 6 +- src/Dictionaries/MySQLDictionarySource.cpp | 12 +- src/Dictionaries/XDBCDictionarySource.cpp | 4 +- src/Disks/DiskLocal.cpp | 8 +- src/Disks/DiskSelector.cpp | 2 +- src/Disks/IDisk.cpp | 2 +- src/Disks/S3/DiskS3.cpp | 16 +- src/Disks/S3/ProxyListConfiguration.cpp | 2 +- src/Disks/S3/ProxyResolverConfiguration.cpp | 4 +- src/Disks/S3/registerDiskS3.cpp | 4 +- src/Disks/StoragePolicy.cpp | 2 +- src/Disks/VolumeJBOD.cpp | 4 +- src/IO/AIOContextPool.cpp | 2 +- src/IO/HTTPCommon.cpp | 2 +- src/IO/ReadBufferFromS3.cpp | 2 +- src/IO/ReadWriteBufferFromHTTP.h | 2 +- src/IO/S3Common.cpp | 4 +- src/IO/WriteBufferFromHTTP.cpp | 2 +- src/IO/WriteBufferFromS3.cpp | 12 +- src/Interpreters/Aggregator.cpp | 40 +-- .../ClusterProxy/SelectStreamFactory.cpp | 6 +- src/Interpreters/Context.cpp | 2 +- src/Interpreters/DDLWorker.cpp | 62 ++--- src/Interpreters/DNSCacheUpdater.cpp | 2 +- src/Interpreters/DatabaseCatalog.cpp | 10 +- src/Interpreters/EmbeddedDictionaries.cpp | 4 +- src/Interpreters/ExternalLoader.cpp | 48 ++-- src/Interpreters/HashJoin.cpp | 2 +- src/Interpreters/InternalTextLogsQueue.cpp | 2 +- src/Interpreters/InterpreterSelectQuery.cpp | 2 +- src/Interpreters/InterpreterSystemQuery.cpp | 6 +- src/Interpreters/ProcessList.cpp | 6 +- src/Interpreters/SystemLog.cpp | 2 +- src/Interpreters/SystemLog.h | 16 +- src/Interpreters/ThreadStatusExt.cpp | 6 +- src/Interpreters/executeQuery.cpp | 12 +- src/Parsers/DumpASTNode.h | 2 +- src/Processors/Executors/PipelineExecutor.cpp | 4 +- src/Processors/Formats/IRowInputFormat.cpp | 2 +- .../Formats/Impl/AvroRowInputFormat.cpp | 4 +- .../Algorithms/CollapsingSortedAlgorithm.cpp | 2 +- src/Processors/Merges/Algorithms/RowRef.h | 4 +- .../Merges/MergingSortedTransform.cpp | 4 +- .../Transforms/AggregatingTransform.cpp | 6 +- .../Transforms/CreatingSetsTransform.cpp | 14 +- .../Transforms/MergeSortingTransform.cpp | 10 +- .../Transforms/MergingAggregatedTransform.cpp | 4 +- src/Storages/Distributed/DirectoryMonitor.cpp | 18 +- .../DistributedBlockOutputStream.cpp | 6 +- .../Kafka/ReadBufferFromKafkaConsumer.cpp | 38 +-- src/Storages/Kafka/StorageKafka.cpp | 12 +- .../MergeTree/BackgroundProcessingPool.cpp | 2 +- src/Storages/MergeTree/DataPartsExchange.cpp | 2 +- .../MergeTree/EphemeralLockInZooKeeper.cpp | 2 +- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 18 +- src/Storages/MergeTree/MergeTreeData.cpp | 104 ++++---- .../MergeTree/MergeTreeDataMergerMutator.cpp | 26 +- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 20 +- .../MergeTree/MergeTreeDataWriter.cpp | 2 +- .../MergeTree/MergeTreePartsMover.cpp | 10 +- src/Storages/MergeTree/MergeTreeReadPool.cpp | 4 +- .../MergeTreeReverseSelectProcessor.cpp | 2 +- .../MergeTree/MergeTreeSelectProcessor.cpp | 2 +- .../MergeTree/MergeTreeSequentialSource.cpp | 4 +- .../MergeTree/MergeTreeWhereOptimizer.cpp | 2 +- .../ReplicatedMergeTreeBlockOutputStream.cpp | 12 +- .../ReplicatedMergeTreeCleanupThread.cpp | 10 +- .../ReplicatedMergeTreePartCheckThread.cpp | 34 +-- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 72 ++--- .../ReplicatedMergeTreeRestartingThread.cpp | 24 +- src/Storages/StorageBuffer.cpp | 20 +- src/Storages/StorageDistributed.cpp | 12 +- src/Storages/StorageMergeTree.cpp | 32 +-- src/Storages/StorageReplicatedMergeTree.cpp | 246 +++++++++--------- src/Storages/StorageSet.cpp | 2 +- 149 files changed, 929 insertions(+), 930 deletions(-) diff --git a/base/common/logger_useful.h b/base/common/logger_useful.h index 8980ba9f94d..c89ec63ef22 100644 --- a/base/common/logger_useful.h +++ b/base/common/logger_useful.h @@ -6,7 +6,6 @@ #include #include #include -#include #include @@ -18,7 +17,7 @@ using DB::CurrentThread; /// Logs a message to a specified logger with that level. -#define LOG_IMPL_FORMATTED(logger, priority, PRIORITY, ...) do \ +#define LOG_IMPL(logger, priority, PRIORITY, ...) do \ { \ const bool is_clients_log = (CurrentThread::getGroup() != nullptr) && \ (CurrentThread::getGroup()->client_logs_level >= (priority)); \ @@ -39,9 +38,9 @@ using DB::CurrentThread; } while (false) -#define LOG_TRACE_FORMATTED(logger, ...) LOG_IMPL_FORMATTED(logger, LogsLevel::trace, Message::PRIO_TRACE, __VA_ARGS__) -#define LOG_DEBUG_FORMATTED(logger, ...) LOG_IMPL_FORMATTED(logger, LogsLevel::debug, Message::PRIO_DEBUG, __VA_ARGS__) -#define LOG_INFO_FORMATTED(logger, ...) LOG_IMPL_FORMATTED(logger, LogsLevel::information, Message::PRIO_INFORMATION, __VA_ARGS__) -#define LOG_WARNING_FORMATTED(logger, ...) LOG_IMPL_FORMATTED(logger, LogsLevel::warning, Message::PRIO_WARNING, __VA_ARGS__) -#define LOG_ERROR_FORMATTED(logger, ...) LOG_IMPL_FORMATTED(logger, LogsLevel::error, Message::PRIO_ERROR, __VA_ARGS__) -#define LOG_FATAL_FORMATTED(logger, ...) LOG_IMPL_FORMATTED(logger, LogsLevel::error, Message::PRIO_FATAL, __VA_ARGS__) +#define LOG_TRACE(logger, ...) LOG_IMPL(logger, LogsLevel::trace, Message::PRIO_TRACE, __VA_ARGS__) +#define LOG_DEBUG(logger, ...) LOG_IMPL(logger, LogsLevel::debug, Message::PRIO_DEBUG, __VA_ARGS__) +#define LOG_INFO(logger, ...) LOG_IMPL(logger, LogsLevel::information, Message::PRIO_INFORMATION, __VA_ARGS__) +#define LOG_WARNING(logger, ...) LOG_IMPL(logger, LogsLevel::warning, Message::PRIO_WARNING, __VA_ARGS__) +#define LOG_ERROR(logger, ...) LOG_IMPL(logger, LogsLevel::error, Message::PRIO_ERROR, __VA_ARGS__) +#define LOG_FATAL(logger, ...) LOG_IMPL(logger, LogsLevel::error, Message::PRIO_FATAL, __VA_ARGS__) diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index 7665612f10e..04f92b91874 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -180,18 +180,18 @@ public: // levels and more info, but for completeness we log all signals // here at trace level. // Don't use strsignal here, because it's not thread-safe. - LOG_TRACE_FORMATTED(log, "Received signal {}", sig); + LOG_TRACE(log, "Received signal {}", sig); if (sig == Signals::StopThread) { - LOG_INFO_FORMATTED(log, "Stop SignalListener thread"); + LOG_INFO(log, "Stop SignalListener thread"); break; } else if (sig == SIGHUP || sig == SIGUSR1) { - LOG_DEBUG_FORMATTED(log, "Received signal to close logs."); + LOG_DEBUG(log, "Received signal to close logs."); BaseDaemon::instance().closeLogs(BaseDaemon::instance().logger()); - LOG_INFO_FORMATTED(log, "Opened new log file after received signal."); + LOG_INFO(log, "Opened new log file after received signal."); } else if (sig == Signals::StdTerminate) { @@ -236,7 +236,7 @@ private: void onTerminate(const std::string & message, UInt32 thread_num) const { - LOG_FATAL_FORMATTED(log, "(version {}{}) (from thread {}) {}", VERSION_STRING, VERSION_OFFICIAL, thread_num, message); + LOG_FATAL(log, "(version {}{}) (from thread {}) {}", VERSION_STRING, VERSION_OFFICIAL, thread_num, message); } void onFault( @@ -247,7 +247,7 @@ private: UInt32 thread_num, const std::string & query_id) const { - LOG_FATAL_FORMATTED(log, "########################################"); + LOG_FATAL(log, "########################################"); { std::stringstream message; @@ -259,10 +259,10 @@ private: message << " (query_id: " << query_id << ")"; message << " Received signal " << strsignal(sig) << " (" << sig << ")."; - LOG_FATAL_FORMATTED(log, message.str()); + LOG_FATAL(log, message.str()); } - LOG_FATAL_FORMATTED(log, signalToErrorMessage(sig, info, context)); + LOG_FATAL(log, signalToErrorMessage(sig, info, context)); if (stack_trace.getSize()) { @@ -274,11 +274,11 @@ private: for (size_t i = stack_trace.getOffset(); i < stack_trace.getSize(); ++i) bare_stacktrace << ' ' << stack_trace.getFrames()[i]; - LOG_FATAL_FORMATTED(log, bare_stacktrace.str()); + LOG_FATAL(log, bare_stacktrace.str()); } /// Write symbolized stack trace line by line for better grep-ability. - stack_trace.toStringEveryLine([&](const std::string & s) { LOG_FATAL_FORMATTED(log, s); }); + stack_trace.toStringEveryLine([&](const std::string & s) { LOG_FATAL(log, s); }); } }; @@ -318,7 +318,7 @@ static void sanitizerDeathCallback() } /// Write symbolized stack trace line by line for better grep-ability. - stack_trace.toStringEveryLine([&](const std::string & s) { LOG_FATAL_FORMATTED(log, s); }); + stack_trace.toStringEveryLine([&](const std::string & s) { LOG_FATAL(log, s); }); } #endif @@ -379,7 +379,7 @@ static bool tryCreateDirectories(Poco::Logger * logger, const std::string & path } catch (...) { - LOG_WARNING_FORMATTED(logger, "{}: when creating {}, {}", __PRETTY_FUNCTION__, path, DB::getCurrentExceptionMessage(true)); + LOG_WARNING(logger, "{}: when creating {}, {}", __PRETTY_FUNCTION__, path, DB::getCurrentExceptionMessage(true)); } return false; } @@ -498,10 +498,10 @@ void debugIncreaseOOMScore() } catch (const Poco::Exception & e) { - LOG_WARNING_FORMATTED(&Logger::root(), "Failed to adjust OOM score: '{}'.", e.displayText()); + LOG_WARNING(&Logger::root(), "Failed to adjust OOM score: '{}'.", e.displayText()); return; } - LOG_INFO_FORMATTED(&Logger::root(), "Set OOM score adjustment to {}", new_score); + LOG_INFO(&Logger::root(), "Set OOM score adjustment to {}", new_score); } #else void debugIncreaseOOMScore() {} @@ -733,7 +733,7 @@ void BaseDaemon::handleNotification(Poco::TaskFailedNotification *_tfn) task_failed = true; Poco::AutoPtr fn(_tfn); Logger *lg = &(logger()); - LOG_ERROR_FORMATTED(lg, "Task '{}' failed. Daemon is shutting down. Reason - {}", fn->task()->name(), fn->reason().displayText()); + LOG_ERROR(lg, "Task '{}' failed. Daemon is shutting down. Reason - {}", fn->task()->name(), fn->reason().displayText()); ServerApplication::terminate(); } @@ -849,11 +849,11 @@ void BaseDaemon::handleSignal(int signal_id) void BaseDaemon::onInterruptSignals(int signal_id) { is_cancelled = true; - LOG_INFO_FORMATTED(&logger(), "Received termination signal ({})", strsignal(signal_id)); + LOG_INFO(&logger(), "Received termination signal ({})", strsignal(signal_id)); if (sigint_signals_counter >= 2) { - LOG_INFO_FORMATTED(&logger(), "Received second signal Interrupt. Immediately terminate."); + LOG_INFO(&logger(), "Received second signal Interrupt. Immediately terminate."); kill(); } } diff --git a/base/daemon/GraphiteWriter.h b/base/daemon/GraphiteWriter.h index 401ed709b8f..fc07d2b7a40 100644 --- a/base/daemon/GraphiteWriter.h +++ b/base/daemon/GraphiteWriter.h @@ -52,7 +52,7 @@ private: } catch (const Poco::Exception & e) { - LOG_WARNING_FORMATTED(&Poco::Util::Application::instance().logger(), "Fail to write to Graphite {}:{}. e.what() = {}, e.message() = {}", host, port, e.what(), e.message()); + LOG_WARNING(&Poco::Util::Application::instance().logger(), "Fail to write to Graphite {}:{}. e.what() = {}, e.message() = {}", host, port, e.what(), e.message()); } } diff --git a/programs/copier/ClusterCopier.cpp b/programs/copier/ClusterCopier.cpp index cb0bcbaca83..d50b89738aa 100644 --- a/programs/copier/ClusterCopier.cpp +++ b/programs/copier/ClusterCopier.cpp @@ -26,7 +26,7 @@ void ClusterCopier::init() if (response.error != Coordination::ZOK) return; UInt64 version = ++task_description_version; - LOG_DEBUG_FORMATTED(log, "Task description should be updated, local version {}", version); + LOG_DEBUG(log, "Task description should be updated, local version {}", version); }; task_description_path = task_zookeeper_path + "/description"; @@ -47,7 +47,7 @@ void ClusterCopier::init() task_table.initShards(task_cluster->random_engine); } - LOG_DEBUG_FORMATTED(log, "Will process {} table tasks", task_cluster->table_tasks.size()); + LOG_DEBUG(log, "Will process {} table tasks", task_cluster->table_tasks.size()); /// Do not initialize tables, will make deferred initialization in process() @@ -85,7 +85,7 @@ void ClusterCopier::discoverShardPartitions(const ConnectionTimeouts & timeouts, { TaskTable & task_table = task_shard->task_table; - LOG_INFO_FORMATTED(log, "Discover partitions of shard {}", task_shard->getDescription()); + LOG_INFO(log, "Discover partitions of shard {}", task_shard->getDescription()); auto get_partitions = [&] () { return getShardPartitions(timeouts, *task_shard); }; auto existing_partitions_names = retry(get_partitions, 60); @@ -132,7 +132,7 @@ void ClusterCopier::discoverShardPartitions(const ConnectionTimeouts & timeouts, { if (!task_table.enabled_partitions_set.count(partition_name)) { - LOG_DEBUG_FORMATTED(log, "Partition {} will not be processed, since it is not in enabled_partitions of {}", partition_name, task_table.table_id); + LOG_DEBUG(log, "Partition {} will not be processed, since it is not in enabled_partitions of {}", partition_name, task_table.table_id); } } } @@ -164,10 +164,10 @@ void ClusterCopier::discoverShardPartitions(const ConnectionTimeouts & timeouts, for (const String & missing_partition : missing_partitions) ss << " " << missing_partition; - LOG_WARNING_FORMATTED(log, "There are no {} partitions from enabled_partitions in shard {} :{}", missing_partitions.size(), task_shard->getDescription(), ss.str()); + LOG_WARNING(log, "There are no {} partitions from enabled_partitions in shard {} :{}", missing_partitions.size(), task_shard->getDescription(), ss.str()); } - LOG_DEBUG_FORMATTED(log, "Will copy {} partitions from shard {}", task_shard->partition_tasks.size(), task_shard->getDescription()); + LOG_DEBUG(log, "Will copy {} partitions from shard {}", task_shard->partition_tasks.size(), task_shard->getDescription()); } void ClusterCopier::discoverTablePartitions(const ConnectionTimeouts & timeouts, TaskTable & task_table, UInt64 num_threads) @@ -179,7 +179,7 @@ void ClusterCopier::discoverTablePartitions(const ConnectionTimeouts & timeouts, for (const TaskShardPtr & task_shard : task_table.all_shards) thread_pool.scheduleOrThrowOnError([this, timeouts, task_shard]() { discoverShardPartitions(timeouts, task_shard); }); - LOG_DEBUG_FORMATTED(log, "Waiting for {} setup jobs", thread_pool.active()); + LOG_DEBUG(log, "Waiting for {} setup jobs", thread_pool.active()); thread_pool.wait(); } } @@ -203,7 +203,7 @@ void ClusterCopier::uploadTaskDescription(const std::string & task_path, const s if (code && force) zookeeper->createOrUpdate(local_task_description_path, task_config_str, zkutil::CreateMode::Persistent); - LOG_DEBUG_FORMATTED(log, "Task description {} uploaded to {} with result {} ({})", + LOG_DEBUG(log, "Task description {} uploaded to {} with result {} ({})", ((code && !force) ? "not " : ""), local_task_description_path, code, zookeeper->error2string(code)); } @@ -220,7 +220,7 @@ void ClusterCopier::reloadTaskDescription() if (code) throw Exception("Can't get description node " + task_description_path, ErrorCodes::BAD_ARGUMENTS); - LOG_DEBUG_FORMATTED(log, "Loading description, zxid={}", task_description_current_stat.czxid); + LOG_DEBUG(log, "Loading description, zxid={}", task_description_current_stat.czxid); auto config = getConfigurationFromXMLString(task_config_str); /// Setup settings @@ -240,7 +240,7 @@ void ClusterCopier::updateConfigIfNeeded() if (!is_outdated_version && !is_expired_session) return; - LOG_DEBUG_FORMATTED(log, "Updating task description"); + LOG_DEBUG(log, "Updating task description"); reloadTaskDescription(); task_description_current_version = version_to_update; @@ -250,7 +250,7 @@ void ClusterCopier::process(const ConnectionTimeouts & timeouts) { for (TaskTable & task_table : task_cluster->table_tasks) { - LOG_INFO_FORMATTED(log, "Process table task {} with {} shards, {} of them are local ones", task_table.table_id, task_table.all_shards.size(), task_table.local_shards.size()); + LOG_INFO(log, "Process table task {} with {} shards, {} of them are local ones", task_table.table_id, task_table.all_shards.size(), task_table.local_shards.size()); if (task_table.all_shards.empty()) continue; @@ -354,7 +354,7 @@ zkutil::EphemeralNodeHolder::Ptr ClusterCopier::createTaskWorkerNodeAndWaitIfNee if (static_cast(stat.numChildren) >= task_cluster->max_workers) { - LOG_DEBUG_FORMATTED(log, "Too many workers ({}, maximum {}). Postpone processing {}", stat.numChildren, task_cluster->max_workers, description); + LOG_DEBUG(log, "Too many workers ({}, maximum {}). Postpone processing {}", stat.numChildren, task_cluster->max_workers, description); if (unprioritized) current_sleep_time = std::min(max_sleep_time, current_sleep_time + default_sleep_time); @@ -380,7 +380,7 @@ zkutil::EphemeralNodeHolder::Ptr ClusterCopier::createTaskWorkerNodeAndWaitIfNee /// Try to make fast retries if (num_bad_version_errors > 3) { - LOG_DEBUG_FORMATTED(log, "A concurrent worker has just been added, will check free worker slots again"); + LOG_DEBUG(log, "A concurrent worker has just been added, will check free worker slots again"); std::chrono::milliseconds random_sleep_time(std::uniform_int_distribution(1, 1000)(task_cluster->random_engine)); std::this_thread::sleep_for(random_sleep_time); num_bad_version_errors = 0; @@ -415,7 +415,7 @@ bool ClusterCopier::checkAllPiecesInPartitionAreDone(const TaskTable & task_tabl { bool piece_is_done = checkPartitionPieceIsDone(task_table, partition_name, piece_number, shards_with_partition); if (!piece_is_done) - LOG_DEBUG_FORMATTED(log, "Partition {} piece {} is not already done.", partition_name, piece_number); + LOG_DEBUG(log, "Partition {} piece {} is not already done.", partition_name, piece_number); answer &= piece_is_done; } @@ -431,7 +431,7 @@ bool ClusterCopier::checkAllPiecesInPartitionAreDone(const TaskTable & task_tabl bool ClusterCopier::checkPartitionPieceIsDone(const TaskTable & task_table, const String & partition_name, size_t piece_number, const TasksShard & shards_with_partition) { - LOG_DEBUG_FORMATTED(log, "Check that all shards processed partition {} piece {} successfully", partition_name, piece_number); + LOG_DEBUG(log, "Check that all shards processed partition {} piece {} successfully", partition_name, piece_number); auto zookeeper = context.getZooKeeper(); @@ -460,7 +460,7 @@ bool ClusterCopier::checkPartitionPieceIsDone(const TaskTable & task_table, cons TaskStateWithOwner status = TaskStateWithOwner::fromString(res.data); if (status.state != TaskState::Finished) { - LOG_INFO_FORMATTED(log, "The task {} is being rewritten by {}. Partition piece will be rechecked", res.data, status.owner); + LOG_INFO(log, "The task {} is being rewritten by {}. Partition piece will be rechecked", res.data, status.owner); return false; } @@ -478,7 +478,7 @@ bool ClusterCopier::checkPartitionPieceIsDone(const TaskTable & task_table, cons if (!is_clean) { - LOG_INFO_FORMATTED(log, "Partition {} become dirty", partition_name); + LOG_INFO(log, "Partition {} become dirty", partition_name); return false; } @@ -495,7 +495,7 @@ bool ClusterCopier::checkPartitionPieceIsDone(const TaskTable & task_table, cons } catch (const Coordination::Exception & e) { - LOG_INFO_FORMATTED(log, "A ZooKeeper error occurred while checking partition {} piece number {}. Will recheck the partition. Error: {}", partition_name, toString(piece_number), e.displayText()); + LOG_INFO(log, "A ZooKeeper error occurred while checking partition {} piece number {}. Will recheck the partition. Error: {}", partition_name, toString(piece_number), e.displayText()); return false; } @@ -504,12 +504,12 @@ bool ClusterCopier::checkPartitionPieceIsDone(const TaskTable & task_table, cons { if (zxid1[shard_num] != zxid2[shard_num]) { - LOG_INFO_FORMATTED(log, "The task {} is being modified now. Partition piece will be rechecked", piece_status_paths[shard_num]); + LOG_INFO(log, "The task {} is being modified now. Partition piece will be rechecked", piece_status_paths[shard_num]); return false; } } - LOG_INFO_FORMATTED(log, "Partition {} piece number {} is copied successfully", partition_name, toString(piece_number)); + LOG_INFO(log, "Partition {} piece number {} is copied successfully", partition_name, toString(piece_number)); return true; } @@ -523,7 +523,7 @@ TaskStatus ClusterCopier::tryMoveAllPiecesToDestinationTable(const TaskTable & t inject_fault = value < move_fault_probability; } - LOG_DEBUG_FORMATTED(log, "Try to move {} to destionation table", partition_name); + LOG_DEBUG(log, "Try to move {} to destionation table", partition_name); auto zookeeper = context.getZooKeeper(); @@ -541,7 +541,7 @@ TaskStatus ClusterCopier::tryMoveAllPiecesToDestinationTable(const TaskTable & t { if (e.code == Coordination::ZNODEEXISTS) { - LOG_DEBUG_FORMATTED(log, "Someone is already moving pieces {}", current_partition_attach_is_active); + LOG_DEBUG(log, "Someone is already moving pieces {}", current_partition_attach_is_active); return TaskStatus::Active; } @@ -558,13 +558,13 @@ TaskStatus ClusterCopier::tryMoveAllPiecesToDestinationTable(const TaskTable & t TaskStateWithOwner status = TaskStateWithOwner::fromString(status_data); if (status.state == TaskState::Finished) { - LOG_DEBUG_FORMATTED(log, "All pieces for partition from this task {} has been successfully moved to destination table by {}", current_partition_attach_is_active, status.owner); + LOG_DEBUG(log, "All pieces for partition from this task {} has been successfully moved to destination table by {}", current_partition_attach_is_active, status.owner); return TaskStatus::Finished; } /// Task is abandoned, because previously we created ephemeral node, possibly in other copier's process. /// Initialize DROP PARTITION - LOG_DEBUG_FORMATTED(log, "Moving piece for partition {} has not been successfully finished by {}. Will try to move by myself.", current_partition_attach_is_active, status.owner); + LOG_DEBUG(log, "Moving piece for partition {} has not been successfully finished by {}. Will try to move by myself.", current_partition_attach_is_active, status.owner); /// Remove is_done marker. zookeeper->remove(current_partition_attach_is_done); @@ -581,7 +581,7 @@ TaskStatus ClusterCopier::tryMoveAllPiecesToDestinationTable(const TaskTable & t /// Move partition to original destination table. for (size_t current_piece_number = 0; current_piece_number < task_table.number_of_splits; ++current_piece_number) { - LOG_DEBUG_FORMATTED(log, "Trying to move partition {} piece {} to original table", partition_name, toString(current_piece_number)); + LOG_DEBUG(log, "Trying to move partition {} piece {} to original table", partition_name, toString(current_piece_number)); ASTPtr query_alter_ast; String query_alter_ast_string; @@ -602,7 +602,7 @@ TaskStatus ClusterCopier::tryMoveAllPiecesToDestinationTable(const TaskTable & t " ATTACH PARTITION " + partition_name + " FROM " + getQuotedTable(helping_table); - LOG_DEBUG_FORMATTED(log, "Executing ALTER query: {}", query_alter_ast_string); + LOG_DEBUG(log, "Executing ALTER query: {}", query_alter_ast_string); try { @@ -614,11 +614,11 @@ TaskStatus ClusterCopier::tryMoveAllPiecesToDestinationTable(const TaskTable & t PoolMode::GET_MANY, ClusterExecutionMode::ON_EACH_NODE); - LOG_INFO_FORMATTED(log, "Number of nodes that executed ALTER query successfully : {}", toString(num_nodes)); + LOG_INFO(log, "Number of nodes that executed ALTER query successfully : {}", toString(num_nodes)); } catch (...) { - LOG_DEBUG_FORMATTED(log, "Error while moving partition {} piece {} to original table", partition_name, toString(current_piece_number)); + LOG_DEBUG(log, "Error while moving partition {} piece {} to original table", partition_name, toString(current_piece_number)); throw; } @@ -633,7 +633,7 @@ TaskStatus ClusterCopier::tryMoveAllPiecesToDestinationTable(const TaskTable & t query_deduplicate_ast_string += " OPTIMIZE TABLE " + getQuotedTable(original_table) + " PARTITION " + partition_name + " DEDUPLICATE;"; - LOG_DEBUG_FORMATTED(log, "Executing OPTIMIZE DEDUPLICATE query: {}", query_alter_ast_string); + LOG_DEBUG(log, "Executing OPTIMIZE DEDUPLICATE query: {}", query_alter_ast_string); UInt64 num_nodes = executeQueryOnCluster( task_table.cluster_push, @@ -642,12 +642,12 @@ TaskStatus ClusterCopier::tryMoveAllPiecesToDestinationTable(const TaskTable & t &task_cluster->settings_push, PoolMode::GET_MANY); - LOG_INFO_FORMATTED(log, "Number of shard that executed OPTIMIZE DEDUPLICATE query successfully : {}", toString(num_nodes)); + LOG_INFO(log, "Number of shard that executed OPTIMIZE DEDUPLICATE query successfully : {}", toString(num_nodes)); } } catch (...) { - LOG_DEBUG_FORMATTED(log, "Error while executing OPTIMIZE DEDUPLICATE partition {}in the original table", partition_name); + LOG_DEBUG(log, "Error while executing OPTIMIZE DEDUPLICATE partition {}in the original table", partition_name); throw; } } @@ -743,7 +743,7 @@ bool ClusterCopier::tryDropPartitionPiece( { if (e.code == Coordination::ZNODEEXISTS) { - LOG_DEBUG_FORMATTED(log, "Partition {} piece {} is cleaning now by somebody, sleep", task_partition.name, toString(current_piece_number)); + LOG_DEBUG(log, "Partition {} piece {} is cleaning now by somebody, sleep", task_partition.name, toString(current_piece_number)); std::this_thread::sleep_for(default_sleep_time); return false; } @@ -756,7 +756,7 @@ bool ClusterCopier::tryDropPartitionPiece( { if (stat.numChildren != 0) { - LOG_DEBUG_FORMATTED(log, "Partition {} contains {} active workers while trying to drop it. Going to sleep.", task_partition.name, stat.numChildren); + LOG_DEBUG(log, "Partition {} contains {} active workers while trying to drop it. Going to sleep.", task_partition.name, stat.numChildren); std::this_thread::sleep_for(default_sleep_time); return false; } @@ -776,7 +776,7 @@ bool ClusterCopier::tryDropPartitionPiece( { if (e.code == Coordination::ZNODEEXISTS) { - LOG_DEBUG_FORMATTED(log, "Partition {} is being filled now by somebody, sleep", task_partition.name); + LOG_DEBUG(log, "Partition {} is being filled now by somebody, sleep", task_partition.name); return false; } @@ -814,7 +814,7 @@ bool ClusterCopier::tryDropPartitionPiece( /// It is important, DROP PARTITION must be done synchronously settings_push.replication_alter_partitions_sync = 2; - LOG_DEBUG_FORMATTED(log, "Execute distributed DROP PARTITION: {}", query); + LOG_DEBUG(log, "Execute distributed DROP PARTITION: {}", query); /// We have to drop partition_piece on each replica size_t num_shards = executeQueryOnCluster( cluster_push, query, @@ -823,7 +823,7 @@ bool ClusterCopier::tryDropPartitionPiece( PoolMode::GET_MANY, ClusterExecutionMode::ON_EACH_NODE); - LOG_INFO_FORMATTED(log, "DROP PARTITION was successfully executed on {} nodes of a cluster.", num_shards); + LOG_INFO(log, "DROP PARTITION was successfully executed on {} nodes of a cluster.", num_shards); /// Update the locking node if (!my_clock.is_stale()) @@ -836,17 +836,17 @@ bool ClusterCopier::tryDropPartitionPiece( } else { - LOG_DEBUG_FORMATTED(log, "Clean state is altered when dropping the partition, cowardly bailing"); + LOG_DEBUG(log, "Clean state is altered when dropping the partition, cowardly bailing"); /// clean state is stale return false; } - LOG_INFO_FORMATTED(log, "Partition {} piece {} was dropped on cluster {}", task_partition.name, toString(current_piece_number), task_table.cluster_push_name); + LOG_INFO(log, "Partition {} piece {} was dropped on cluster {}", task_partition.name, toString(current_piece_number), task_table.cluster_push_name); if (zookeeper->tryCreate(current_shards_path, host_id, zkutil::CreateMode::Persistent) == Coordination::ZNODEEXISTS) zookeeper->set(current_shards_path, host_id); } - LOG_INFO_FORMATTED(log, "Partition {} piece {} is safe for work now.", task_partition.name, toString(current_piece_number)); + LOG_INFO(log, "Partition {} piece {} is safe for work now.", task_partition.name, toString(current_piece_number)); return true; } @@ -870,7 +870,7 @@ bool ClusterCopier::tryProcessTable(const ConnectionTimeouts & timeouts, TaskTab ++cluster_partition.total_tries; - LOG_DEBUG_FORMATTED(log, "Processing partition {} for the whole cluster", partition_name); + LOG_DEBUG(log, "Processing partition {} for the whole cluster", partition_name); /// Process each source shard having current partition and copy current partition /// NOTE: shards are sorted by "distance" to current host @@ -892,7 +892,7 @@ bool ClusterCopier::tryProcessTable(const ConnectionTimeouts & timeouts, TaskTab { const size_t number_of_splits = task_table.number_of_splits; shard->partition_tasks.emplace(partition_name, ShardPartition(*shard, partition_name, number_of_splits)); - LOG_DEBUG_FORMATTED(log, "Discovered partition {} in shard {}", partition_name, shard->getDescription()); + LOG_DEBUG(log, "Discovered partition {} in shard {}", partition_name, shard->getDescription()); /// To save references in the future. auto shard_partition_it = shard->partition_tasks.find(partition_name); PartitionPieces & shard_partition_pieces = shard_partition_it->second.pieces; @@ -905,7 +905,7 @@ bool ClusterCopier::tryProcessTable(const ConnectionTimeouts & timeouts, TaskTab } else { - LOG_DEBUG_FORMATTED(log, "Found that shard {} does not contain current partition {}", shard->getDescription(), partition_name); + LOG_DEBUG(log, "Found that shard {} does not contain current partition {}", shard->getDescription(), partition_name); continue; } } @@ -1011,7 +1011,7 @@ bool ClusterCopier::tryProcessTable(const ConnectionTimeouts & timeouts, TaskTab task_table.rows_copied += cluster_partition.rows_copied; double elapsed = cluster_partition.elapsed_time_seconds; - LOG_INFO_FORMATTED(log, "It took {} seconds to copy partition {}: {} uncompressed bytes, {} rows and {} source blocks are copied", + LOG_INFO(log, "It took {} seconds to copy partition {}: {} uncompressed bytes, {} rows and {} source blocks are copied", elapsed, partition_name, formatReadableSizeWithDecimalSuffix(cluster_partition.bytes_copied), formatReadableQuantity(cluster_partition.rows_copied), @@ -1019,12 +1019,12 @@ bool ClusterCopier::tryProcessTable(const ConnectionTimeouts & timeouts, TaskTab if (cluster_partition.rows_copied) { - LOG_INFO_FORMATTED(log, "Average partition speed: {} per second.", formatReadableSizeWithDecimalSuffix(cluster_partition.bytes_copied / elapsed)); + LOG_INFO(log, "Average partition speed: {} per second.", formatReadableSizeWithDecimalSuffix(cluster_partition.bytes_copied / elapsed)); } if (task_table.rows_copied) { - LOG_INFO_FORMATTED(log, "Average table {} speed: {} per second.", task_table.table_id, formatReadableSizeWithDecimalSuffix(task_table.bytes_copied / elapsed)); + LOG_INFO(log, "Average table {} speed: {} per second.", task_table.table_id, formatReadableSizeWithDecimalSuffix(task_table.bytes_copied / elapsed)); } } } @@ -1035,7 +1035,7 @@ bool ClusterCopier::tryProcessTable(const ConnectionTimeouts & timeouts, TaskTab if (!table_is_done) { - LOG_INFO_FORMATTED(log, "Table {} is not processed yet.Copied {} of {}, will retry", task_table.table_id, finished_partitions, required_partitions); + LOG_INFO(log, "Table {} is not processed yet.Copied {} of {}, will retry", task_table.table_id, finished_partitions, required_partitions); } return table_is_done; @@ -1083,7 +1083,7 @@ TaskStatus ClusterCopier::iterateThroughAllPiecesInPartition(const ConnectionTim { for (UInt64 try_num = 0; try_num < max_shard_partition_tries; ++try_num) { - LOG_INFO_FORMATTED(log, "Attempt number {} to process partition {} piece number {} on shard number {} with index {}.", + LOG_INFO(log, "Attempt number {} to process partition {} piece number {} on shard number {} with index {}.", try_num, task_partition.name, piece_number, task_partition.task_shard.numberInCluster(), task_partition.task_shard.indexInCluster()); @@ -1146,17 +1146,17 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( auto create_is_dirty_node = [&] (const CleanStateClock & clock) { if (clock.is_stale()) - LOG_DEBUG_FORMATTED(log, "Clean state clock is stale while setting dirty flag, cowardly bailing"); + LOG_DEBUG(log, "Clean state clock is stale while setting dirty flag, cowardly bailing"); else if (!clock.is_clean()) - LOG_DEBUG_FORMATTED(log, "Thank you, Captain Obvious"); + LOG_DEBUG(log, "Thank you, Captain Obvious"); else if (clock.discovery_version) { - LOG_DEBUG_FORMATTED(log, "Updating clean state clock"); + LOG_DEBUG(log, "Updating clean state clock"); zookeeper->set(piece_is_dirty_flag_path, host_id, clock.discovery_version.value()); } else { - LOG_DEBUG_FORMATTED(log, "Creating clean state clock"); + LOG_DEBUG(log, "Creating clean state clock"); zookeeper->create(piece_is_dirty_flag_path, host_id, zkutil::CreateMode::Persistent); } }; @@ -1191,7 +1191,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( /// Load balancing auto worker_node_holder = createTaskWorkerNodeAndWaitIfNeed(zookeeper, current_task_piece_status_path, is_unprioritized_task); - LOG_DEBUG_FORMATTED(log, "Processing {}", current_task_piece_status_path); + LOG_DEBUG(log, "Processing {}", current_task_piece_status_path); const String piece_status_path = partition_piece.getPartitionPieceShardsPath(); @@ -1202,12 +1202,12 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( /// Do not start if partition piece is dirty, try to clean it if (is_clean) { - LOG_DEBUG_FORMATTED(log, "Partition {} piece {} appears to be clean", task_partition.name, current_piece_number); + LOG_DEBUG(log, "Partition {} piece {} appears to be clean", task_partition.name, current_piece_number); zookeeper->createAncestors(current_task_piece_status_path); } else { - LOG_DEBUG_FORMATTED(log, "Partition {} piece {} is dirty, try to drop it", task_partition.name, current_piece_number); + LOG_DEBUG(log, "Partition {} piece {} is dirty, try to drop it", task_partition.name, current_piece_number); try { @@ -1232,7 +1232,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( { if (e.code == Coordination::ZNODEEXISTS) { - LOG_DEBUG_FORMATTED(log, "Someone is already processing {}", current_task_piece_is_active_path); + LOG_DEBUG(log, "Someone is already processing {}", current_task_piece_is_active_path); return TaskStatus::Active; } @@ -1248,13 +1248,13 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( TaskStateWithOwner status = TaskStateWithOwner::fromString(status_data); if (status.state == TaskState::Finished) { - LOG_DEBUG_FORMATTED(log, "Task {} has been successfully executed by {}", current_task_piece_status_path, status.owner); + LOG_DEBUG(log, "Task {} has been successfully executed by {}", current_task_piece_status_path, status.owner); return TaskStatus::Finished; } /// Task is abandoned, because previously we created ephemeral node, possibly in other copier's process. /// Initialize DROP PARTITION - LOG_DEBUG_FORMATTED(log, "Task {} has not been successfully finished by {}. Partition will be dropped and refilled.", current_task_piece_status_path, status.owner); + LOG_DEBUG(log, "Task {} has not been successfully finished by {}. Partition will be dropped and refilled.", current_task_piece_status_path, status.owner); create_is_dirty_node(clean_state_clock); return TaskStatus::Error; @@ -1269,9 +1269,9 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( String state_finished = TaskStateWithOwner::getData(TaskState::Finished, host_id); auto res = zookeeper->tryCreate(current_task_piece_status_path, state_finished, zkutil::CreateMode::Persistent); if (res == Coordination::ZNODEEXISTS) - LOG_DEBUG_FORMATTED(log, "Partition {} piece {} is absent on current replica of a shard. But other replicas have already marked it as done.", task_partition.name, current_piece_number); + LOG_DEBUG(log, "Partition {} piece {} is absent on current replica of a shard. But other replicas have already marked it as done.", task_partition.name, current_piece_number); if (res == Coordination::ZOK) - LOG_DEBUG_FORMATTED(log, "Partition {} piece {} is absent on current replica of a shard. Will mark it as done. Other replicas will do the same.", task_partition.name, current_piece_number); + LOG_DEBUG(log, "Partition {} piece {} is absent on current replica of a shard. Will mark it as done. Other replicas will do the same.", task_partition.name, current_piece_number); return TaskStatus::Finished; } @@ -1299,14 +1299,14 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( if (count != 0) { - LOG_INFO_FORMATTED(log, "Partition {} piece {}is not empty. In contains {} rows.", task_partition.name, current_piece_number, count); + LOG_INFO(log, "Partition {} piece {}is not empty. In contains {} rows.", task_partition.name, current_piece_number, count); Coordination::Stat stat_shards{}; zookeeper->get(partition_piece.getPartitionPieceShardsPath(), &stat_shards); /// NOTE: partition is still fresh if dirt discovery happens before cleaning if (stat_shards.numChildren == 0) { - LOG_WARNING_FORMATTED(log, "There are no workers for partition {} piece {}, but destination table contains {} rows. Partition will be dropped and refilled.", task_partition.name, toString(current_piece_number), count); + LOG_WARNING(log, "There are no workers for partition {} piece {}, but destination table contains {} rows. Partition will be dropped and refilled.", task_partition.name, toString(current_piece_number), count); create_is_dirty_node(clean_state_clock); return TaskStatus::Error; @@ -1323,12 +1323,12 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( CleanStateClock new_clean_state_clock (zookeeper, piece_is_dirty_flag_path, piece_is_dirty_cleaned_path); if (clean_state_clock != new_clean_state_clock) { - LOG_INFO_FORMATTED(log, "Partition {} piece {} clean state changed, cowardly bailing", task_partition.name, toString(current_piece_number)); + LOG_INFO(log, "Partition {} piece {} clean state changed, cowardly bailing", task_partition.name, toString(current_piece_number)); return TaskStatus::Error; } else if (!new_clean_state_clock.is_clean()) { - LOG_INFO_FORMATTED(log, "Partition {} piece {} is dirty and will be dropped and refilled", task_partition.name, toString(current_piece_number)); + LOG_INFO(log, "Partition {} piece {} is dirty and will be dropped and refilled", task_partition.name, toString(current_piece_number)); create_is_dirty_node(new_clean_state_clock); return TaskStatus::Error; } @@ -1355,11 +1355,11 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( create_query_push_ast->as().if_not_exists = true; String query = queryToString(create_query_push_ast); - LOG_DEBUG_FORMATTED(log, "Create destination tables. Query: {}", query); + LOG_DEBUG(log, "Create destination tables. Query: {}", query); UInt64 shards = executeQueryOnCluster(task_table.cluster_push, query, create_query_push_ast, &task_cluster->settings_push, PoolMode::GET_MANY); - LOG_DEBUG_FORMATTED(log, "Destination tables {} have been created on {} shards of {}", getQuotedTable(task_table.table_push), shards, task_table.cluster_push->getShardCount()); + LOG_DEBUG(log, "Destination tables {} have been created on {} shards of {}", getQuotedTable(task_table.table_push), shards, task_table.cluster_push->getShardCount()); } /// Do the copying @@ -1374,7 +1374,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( // Select all fields ASTPtr query_select_ast = get_select_query(task_shard.table_read_shard, "*", /*enable_splitting*/ true, inject_fault ? "1" : ""); - LOG_DEBUG_FORMATTED(log, "Executing SELECT query and pull from {} : {}", task_shard.getDescription(), queryToString(query_select_ast)); + LOG_DEBUG(log, "Executing SELECT query and pull from {} : {}", task_shard.getDescription(), queryToString(query_select_ast)); ASTPtr query_insert_ast; { @@ -1385,7 +1385,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( const auto & settings = context.getSettingsRef(); query_insert_ast = parseQuery(p_query, query, settings.max_query_size, settings.max_parser_depth); - LOG_DEBUG_FORMATTED(log, "Executing INSERT query: {}", query); + LOG_DEBUG(log, "Executing INSERT query: {}", query); } try @@ -1467,7 +1467,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( } } - LOG_INFO_FORMATTED(log, "Partition {} piece {} copied. But not moved to original destination table.", task_partition.name, toString(current_piece_number)); + LOG_INFO(log, "Partition {} piece {} copied. But not moved to original destination table.", task_partition.name, toString(current_piece_number)); /// Try create original table (if not exists) on each shard @@ -1478,11 +1478,11 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( create_query_push_ast->as().if_not_exists = true; String query = queryToString(create_query_push_ast); - LOG_DEBUG_FORMATTED(log, "Create destination tables. Query: {}", query); + LOG_DEBUG(log, "Create destination tables. Query: {}", query); UInt64 shards = executeQueryOnCluster(task_table.cluster_push, query, create_query_push_ast, &task_cluster->settings_push, PoolMode::GET_MANY); - LOG_DEBUG_FORMATTED(log, "Destination tables {} have been created on {} shards of {}", getQuotedTable(task_table.table_push), shards, task_table.cluster_push->getShardCount()); + LOG_DEBUG(log, "Destination tables {} have been created on {} shards of {}", getQuotedTable(task_table.table_push), shards, task_table.cluster_push->getShardCount()); } catch (...) { @@ -1495,12 +1495,12 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( CleanStateClock new_clean_state_clock (zookeeper, piece_is_dirty_flag_path, piece_is_dirty_cleaned_path); if (clean_state_clock != new_clean_state_clock) { - LOG_INFO_FORMATTED(log, "Partition {} piece {} clean state changed, cowardly bailing", task_partition.name, toString(current_piece_number)); + LOG_INFO(log, "Partition {} piece {} clean state changed, cowardly bailing", task_partition.name, toString(current_piece_number)); return TaskStatus::Error; } else if (!new_clean_state_clock.is_clean()) { - LOG_INFO_FORMATTED(log, "Partition {} piece {} became dirty and will be dropped and refilled", task_partition.name, toString(current_piece_number)); + LOG_INFO(log, "Partition {} piece {} became dirty and will be dropped and refilled", task_partition.name, toString(current_piece_number)); create_is_dirty_node(new_clean_state_clock); return TaskStatus::Error; } @@ -1533,7 +1533,7 @@ void ClusterCopier::dropLocalTableIfExists(const DatabaseAndTableName & table_na void ClusterCopier::dropHelpingTables(const TaskTable & task_table) { - LOG_DEBUG_FORMATTED(log, "Removing helping tables"); + LOG_DEBUG(log, "Removing helping tables"); for (size_t current_piece_number = 0; current_piece_number < task_table.number_of_splits; ++current_piece_number) { DatabaseAndTableName original_table = task_table.table_push; @@ -1544,7 +1544,7 @@ void ClusterCopier::dropHelpingTables(const TaskTable & task_table) const ClusterPtr & cluster_push = task_table.cluster_push; Settings settings_push = task_cluster->settings_push; - LOG_DEBUG_FORMATTED(log, "Execute distributed DROP TABLE: {}", query); + LOG_DEBUG(log, "Execute distributed DROP TABLE: {}", query); /// We have to drop partition_piece on each replica UInt64 num_nodes = executeQueryOnCluster( cluster_push, query, @@ -1553,14 +1553,14 @@ void ClusterCopier::dropHelpingTables(const TaskTable & task_table) PoolMode::GET_MANY, ClusterExecutionMode::ON_EACH_NODE); - LOG_DEBUG_FORMATTED(log, "DROP TABLE query was successfully executed on {} nodes.", toString(num_nodes)); + LOG_DEBUG(log, "DROP TABLE query was successfully executed on {} nodes.", toString(num_nodes)); } } void ClusterCopier::dropParticularPartitionPieceFromAllHelpingTables(const TaskTable & task_table, const String & partition_name) { - LOG_DEBUG_FORMATTED(log, "Try drop partition partition from all helping tables."); + LOG_DEBUG(log, "Try drop partition partition from all helping tables."); for (size_t current_piece_number = 0; current_piece_number < task_table.number_of_splits; ++current_piece_number) { DatabaseAndTableName original_table = task_table.table_push; @@ -1571,7 +1571,7 @@ void ClusterCopier::dropParticularPartitionPieceFromAllHelpingTables(const TaskT const ClusterPtr & cluster_push = task_table.cluster_push; Settings settings_push = task_cluster->settings_push; - LOG_DEBUG_FORMATTED(log, "Execute distributed DROP PARTITION: {}", query); + LOG_DEBUG(log, "Execute distributed DROP PARTITION: {}", query); /// We have to drop partition_piece on each replica UInt64 num_nodes = executeQueryOnCluster( cluster_push, query, @@ -1580,9 +1580,9 @@ void ClusterCopier::dropParticularPartitionPieceFromAllHelpingTables(const TaskT PoolMode::GET_MANY, ClusterExecutionMode::ON_EACH_NODE); - LOG_DEBUG_FORMATTED(log, "DROP PARTITION query was successfully executed on {} nodes.", toString(num_nodes)); + LOG_DEBUG(log, "DROP PARTITION query was successfully executed on {} nodes.", toString(num_nodes)); } - LOG_DEBUG_FORMATTED(log, "All helping tables dropped partition {}", partition_name); + LOG_DEBUG(log, "All helping tables dropped partition {}", partition_name); } String ClusterCopier::getRemoteCreateTable(const DatabaseAndTableName & table, Connection & connection, const Settings * settings) @@ -1686,7 +1686,7 @@ std::set ClusterCopier::getShardPartitions(const ConnectionTimeouts & ti const auto & settings = context.getSettingsRef(); ASTPtr query_ast = parseQuery(parser_query, query, settings.max_query_size, settings.max_parser_depth); - LOG_DEBUG_FORMATTED(log, "Computing destination partition set, executing query: {}", query); + LOG_DEBUG(log, "Computing destination partition set, executing query: {}", query); Context local_context = context; local_context.setSettings(task_cluster->settings_pull); @@ -1706,7 +1706,7 @@ std::set ClusterCopier::getShardPartitions(const ConnectionTimeouts & ti } } - LOG_DEBUG_FORMATTED(log, "There are {} destination partitions in shard {}", res.size(), task_shard.getDescription()); + LOG_DEBUG(log, "There are {} destination partitions in shard {}", res.size(), task_shard.getDescription()); return res; } @@ -1727,7 +1727,7 @@ bool ClusterCopier::checkShardHasPartition(const ConnectionTimeouts & timeouts, query += " LIMIT 1"; - LOG_DEBUG_FORMATTED(log, "Checking shard {} for partition {} existence, executing query: {}", task_shard.getDescription(), partition_quoted_name, query); + LOG_DEBUG(log, "Checking shard {} for partition {} existence, executing query: {}", task_shard.getDescription(), partition_quoted_name, query); ParserQuery parser_query(query.data() + query.size()); const auto & settings = context.getSettingsRef(); @@ -1766,7 +1766,7 @@ bool ClusterCopier::checkPresentPartitionPiecesOnCurrentShard(const ConnectionTi query += " LIMIT 1"; - LOG_DEBUG_FORMATTED(log, "Checking shard {} for partition {} piece {} existence, executing query: {}", task_shard.getDescription(), partition_quoted_name, std::to_string(current_piece_number), query); + LOG_DEBUG(log, "Checking shard {} for partition {} piece {} existence, executing query: {}", task_shard.getDescription(), partition_quoted_name, std::to_string(current_piece_number), query); ParserQuery parser_query(query.data() + query.size()); const auto & settings = context.getSettingsRef(); @@ -1776,9 +1776,9 @@ bool ClusterCopier::checkPresentPartitionPiecesOnCurrentShard(const ConnectionTi local_context.setSettings(task_cluster->settings_pull); auto result = InterpreterFactory::get(query_ast, local_context)->execute().in->read().rows(); if (result != 0) - LOG_DEBUG_FORMATTED(log, "Partition {} piece number {} is PRESENT on shard {}", partition_quoted_name, std::to_string(current_piece_number), task_shard.getDescription()); + LOG_DEBUG(log, "Partition {} piece number {} is PRESENT on shard {}", partition_quoted_name, std::to_string(current_piece_number), task_shard.getDescription()); else - LOG_DEBUG_FORMATTED(log, "Partition {} piece number {} is ABSENT on shard {}", partition_quoted_name, std::to_string(current_piece_number), task_shard.getDescription()); + LOG_DEBUG(log, "Partition {} piece number {} is ABSENT on shard {}", partition_quoted_name, std::to_string(current_piece_number), task_shard.getDescription()); return result != 0; } @@ -1868,7 +1868,7 @@ UInt64 ClusterCopier::executeQueryOnCluster( } catch (const Exception &) { - LOG_INFO_FORMATTED(log, getCurrentExceptionMessage(false, true)); + LOG_INFO(log, getCurrentExceptionMessage(false, true)); } } } @@ -1895,7 +1895,7 @@ UInt64 ClusterCopier::executeQueryOnCluster( if (execution_mode == ClusterExecutionMode::ON_EACH_NODE && successful_nodes != origin_replicas_number) { - LOG_INFO_FORMATTED(log, "There was an error while executing ALTER on each node. Query was executed on {} nodes. But had to be executed on {}", toString(successful_nodes), toString(origin_replicas_number.load())); + LOG_INFO(log, "There was an error while executing ALTER on each node. Query was executed on {} nodes. But had to be executed on {}", toString(successful_nodes), toString(origin_replicas_number.load())); } diff --git a/programs/copier/ClusterCopierApp.cpp b/programs/copier/ClusterCopierApp.cpp index e49be154b7f..9b1393204d4 100644 --- a/programs/copier/ClusterCopierApp.cpp +++ b/programs/copier/ClusterCopierApp.cpp @@ -95,7 +95,7 @@ void ClusterCopierApp::mainImpl() ThreadStatus thread_status; auto * log = &logger(); - LOG_INFO_FORMATTED(log, "Starting clickhouse-copier (id {}, host_id {}, path {}, revision {})", process_id, host_id, process_path, ClickHouseRevision::get()); + LOG_INFO(log, "Starting clickhouse-copier (id {}, host_id {}, path {}, revision {})", process_id, host_id, process_path, ClickHouseRevision::get()); SharedContextHolder shared_context = Context::createShared(); auto context = std::make_unique(Context::createGlobal(shared_context.get())); diff --git a/programs/copier/ZooKeeperStaff.h b/programs/copier/ZooKeeperStaff.h index 47a2df51ffb..f5224857259 100644 --- a/programs/copier/ZooKeeperStaff.h +++ b/programs/copier/ZooKeeperStaff.h @@ -183,11 +183,11 @@ public: switch (rsp.type) { case Coordination::CREATED: - LOG_DEBUG_FORMATTED(logger, "CleanStateClock change: CREATED, at {}", rsp.path); + LOG_DEBUG(logger, "CleanStateClock change: CREATED, at {}", rsp.path); stale->store(true); break; case Coordination::CHANGED: - LOG_DEBUG_FORMATTED(logger, "CleanStateClock change: CHANGED, at{}", rsp.path); + LOG_DEBUG(logger, "CleanStateClock change: CHANGED, at{}", rsp.path); stale->store(true); } } diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 0ac05658181..01acf250b1b 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -211,12 +211,12 @@ try /// Lock path directory before read status.emplace(context->getPath() + "status"); - LOG_DEBUG_FORMATTED(log, "Loading metadata from {}", context->getPath()); + LOG_DEBUG(log, "Loading metadata from {}", context->getPath()); loadMetadataSystem(*context); attachSystemTables(); loadMetadata(*context); DatabaseCatalog::instance().loadDatabases(); - LOG_DEBUG_FORMATTED(log, "Loaded metadata."); + LOG_DEBUG(log, "Loaded metadata."); } else { diff --git a/programs/odbc-bridge/ColumnInfoHandler.cpp b/programs/odbc-bridge/ColumnInfoHandler.cpp index d8579a5d171..33723b4cc8e 100644 --- a/programs/odbc-bridge/ColumnInfoHandler.cpp +++ b/programs/odbc-bridge/ColumnInfoHandler.cpp @@ -62,14 +62,14 @@ namespace void ODBCColumnsInfoHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) { Poco::Net::HTMLForm params(request, request.stream()); - LOG_TRACE_FORMATTED(log, "Request URI: {}", request.getURI()); + LOG_TRACE(log, "Request URI: {}", request.getURI()); auto process_error = [&response, this](const std::string & message) { response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); if (!response.sent()) response.send() << message << std::endl; - LOG_WARNING_FORMATTED(log, message); + LOG_WARNING(log, message); }; if (!params.has("table")) @@ -89,11 +89,11 @@ void ODBCColumnsInfoHandler::handleRequest(Poco::Net::HTTPServerRequest & reques if (params.has("schema")) { schema_name = params.get("schema"); - LOG_TRACE_FORMATTED(log, "Will fetch info for table '{}'", schema_name + "." + table_name); + LOG_TRACE(log, "Will fetch info for table '{}'", schema_name + "." + table_name); } else - LOG_TRACE_FORMATTED(log, "Will fetch info for table '{}'", table_name); - LOG_TRACE_FORMATTED(log, "Got connection str '{}'", connection_string); + LOG_TRACE(log, "Will fetch info for table '{}'", table_name); + LOG_TRACE(log, "Got connection str '{}'", connection_string); try { @@ -124,7 +124,7 @@ void ODBCColumnsInfoHandler::handleRequest(Poco::Net::HTTPServerRequest & reques select->format(settings); std::string query = ss.str(); - LOG_TRACE_FORMATTED(log, "Inferring structure with query '{}'", query); + LOG_TRACE(log, "Inferring structure with query '{}'", query); if (POCO_SQL_ODBC_CLASS::Utility::isError(POCO_SQL_ODBC_CLASS::SQLPrepare(hstmt, reinterpret_cast(query.data()), query.size()))) throw POCO_SQL_ODBC_CLASS::DescriptorException(session.dbc()); diff --git a/programs/odbc-bridge/HandlerFactory.cpp b/programs/odbc-bridge/HandlerFactory.cpp index ebf37925d0d..a02fcadea52 100644 --- a/programs/odbc-bridge/HandlerFactory.cpp +++ b/programs/odbc-bridge/HandlerFactory.cpp @@ -10,7 +10,7 @@ namespace DB Poco::Net::HTTPRequestHandler * HandlerFactory::createRequestHandler(const Poco::Net::HTTPServerRequest & request) { Poco::URI uri{request.getURI()}; - LOG_TRACE_FORMATTED(log, "Request URI: {}", uri.toString()); + LOG_TRACE(log, "Request URI: {}", uri.toString()); if (uri.getPath() == "/ping" && request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET) return new PingHandler(keep_alive_timeout); diff --git a/programs/odbc-bridge/IdentifierQuoteHandler.cpp b/programs/odbc-bridge/IdentifierQuoteHandler.cpp index c7c9ab941bf..2c3701cfff9 100644 --- a/programs/odbc-bridge/IdentifierQuoteHandler.cpp +++ b/programs/odbc-bridge/IdentifierQuoteHandler.cpp @@ -25,14 +25,14 @@ namespace DB void IdentifierQuoteHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) { Poco::Net::HTMLForm params(request, request.stream()); - LOG_TRACE_FORMATTED(log, "Request URI: {}", request.getURI()); + LOG_TRACE(log, "Request URI: {}", request.getURI()); auto process_error = [&response, this](const std::string & message) { response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); if (!response.sent()) response.send() << message << std::endl; - LOG_WARNING_FORMATTED(log, message); + LOG_WARNING(log, message); }; if (!params.has("connection_string")) diff --git a/programs/odbc-bridge/MainHandler.cpp b/programs/odbc-bridge/MainHandler.cpp index 63d24336de1..9bbffed3e23 100644 --- a/programs/odbc-bridge/MainHandler.cpp +++ b/programs/odbc-bridge/MainHandler.cpp @@ -76,7 +76,7 @@ void ODBCHandler::processError(Poco::Net::HTTPServerResponse & response, const s response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); if (!response.sent()) response.send() << message << std::endl; - LOG_WARNING_FORMATTED(log, message); + LOG_WARNING(log, message); } void ODBCHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) @@ -84,7 +84,7 @@ void ODBCHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne Poco::Net::HTMLForm params(request); if (mode == "read") params.read(request.stream()); - LOG_TRACE_FORMATTED(log, "Request URI: {}", request.getURI()); + LOG_TRACE(log, "Request URI: {}", request.getURI()); if (mode == "read" && !params.has("query")) { @@ -125,14 +125,14 @@ void ODBCHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne catch (const Exception & ex) { processError(response, "Invalid 'columns' parameter in request body '" + ex.message() + "'"); - LOG_WARNING_FORMATTED(log, ex.getStackTraceString()); + LOG_WARNING(log, ex.getStackTraceString()); return; } std::string format = params.get("format", "RowBinary"); std::string connection_string = params.get("connection_string"); - LOG_TRACE_FORMATTED(log, "Connection string: '{}'", connection_string); + LOG_TRACE(log, "Connection string: '{}'", connection_string); WriteBufferFromHTTPServerResponse out(request, response, keep_alive_timeout); @@ -152,7 +152,7 @@ void ODBCHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne } std::string db_name = params.get("db_name"); std::string table_name = params.get("table_name"); - LOG_TRACE_FORMATTED(log, "DB name: '{}', table name: '{}'", db_name, table_name); + LOG_TRACE(log, "DB name: '{}', table name: '{}'", db_name, table_name); auto quoting_style = IdentifierQuotingStyle::None; #if USE_ODBC @@ -171,7 +171,7 @@ void ODBCHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne else { std::string query = params.get("query"); - LOG_TRACE_FORMATTED(log, "Query: {}", query); + LOG_TRACE(log, "Query: {}", query); BlockOutputStreamPtr writer = FormatFactory::instance().getOutput(format, out, *sample_block, context); auto pool = getPool(connection_string); diff --git a/programs/odbc-bridge/ODBCBridge.cpp b/programs/odbc-bridge/ODBCBridge.cpp index 4a868cdee8c..3f119fbf7ba 100644 --- a/programs/odbc-bridge/ODBCBridge.cpp +++ b/programs/odbc-bridge/ODBCBridge.cpp @@ -48,7 +48,7 @@ namespace #endif ) { - LOG_ERROR_FORMATTED(log, "Cannot resolve listen_host ({}), error {}: {}. If it is an IPv6 address and your host has disabled IPv6, then consider to specify IPv4 address to listen in element of configuration file. Example: 0.0.0.0", host, e.code(), e.message()); + LOG_ERROR(log, "Cannot resolve listen_host ({}), error {}: {}. If it is an IPv6 address and your host has disabled IPv6, then consider to specify IPv4 address to listen in element of configuration file. Example: 0.0.0.0", host, e.code(), e.message()); } throw; @@ -160,7 +160,7 @@ int ODBCBridge::main(const std::vector & /*args*/) if (is_help) return Application::EXIT_OK; - LOG_INFO_FORMATTED(log, "Starting up"); + LOG_INFO(log, "Starting up"); Poco::Net::ServerSocket socket; auto address = socketBindListen(socket, hostname, port, log); socket.setReceiveTimeout(http_timeout); @@ -183,17 +183,17 @@ int ODBCBridge::main(const std::vector & /*args*/) new HandlerFactory("ODBCRequestHandlerFactory-factory", keep_alive_timeout, context), server_pool, socket, http_params); server.start(); - LOG_INFO_FORMATTED(log, "Listening http://{}", address.toString()); + LOG_INFO(log, "Listening http://{}", address.toString()); SCOPE_EXIT({ - LOG_DEBUG_FORMATTED(log, "Received termination signal."); - LOG_DEBUG_FORMATTED(log, "Waiting for current connections to close."); + LOG_DEBUG(log, "Received termination signal."); + LOG_DEBUG(log, "Waiting for current connections to close."); server.stop(); for (size_t count : ext::range(1, 6)) { if (server.currentConnections() == 0) break; - LOG_DEBUG_FORMATTED(log, "Waiting for {} connections, try {}", server.currentConnections(), count); + LOG_DEBUG(log, "Waiting for {} connections, try {}", server.currentConnections(), count); std::this_thread::sleep_for(std::chrono::milliseconds(1000)); } }); diff --git a/programs/server/HTTPHandler.cpp b/programs/server/HTTPHandler.cpp index 7569ee057bc..e5c784fd409 100644 --- a/programs/server/HTTPHandler.cpp +++ b/programs/server/HTTPHandler.cpp @@ -241,7 +241,7 @@ void HTTPHandler::processQuery( CurrentThread::QueryScope query_scope(context); - LOG_TRACE_FORMATTED(log, "Request URI: {}", request.getURI()); + LOG_TRACE(log, "Request URI: {}", request.getURI()); std::istream & istr = request.stream(); @@ -705,7 +705,7 @@ void HTTPHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne } processQuery(request, params, response, used_output); - LOG_INFO_FORMATTED(log, "Done processing query"); + LOG_INFO(log, "Done processing query"); } catch (...) { diff --git a/programs/server/HTTPHandlerFactory.cpp b/programs/server/HTTPHandlerFactory.cpp index d5e984685f3..f302216e22b 100644 --- a/programs/server/HTTPHandlerFactory.cpp +++ b/programs/server/HTTPHandlerFactory.cpp @@ -30,7 +30,7 @@ HTTPRequestHandlerFactoryMain::HTTPRequestHandlerFactoryMain(const std::string & Poco::Net::HTTPRequestHandler * HTTPRequestHandlerFactoryMain::createRequestHandler(const Poco::Net::HTTPServerRequest & request) { - LOG_TRACE_FORMATTED(log, "HTTP Request for {}. Method: {}, Address: {}, User-Agent: {}{}, Content Type: {}, Transfer Encoding: {}", + LOG_TRACE(log, "HTTP Request for {}. Method: {}, Address: {}, User-Agent: {}{}, Content Type: {}, Transfer Encoding: {}", name, request.getMethod(), request.clientAddress().toString(), request.has("User-Agent") ? request.get("User-Agent") : "none", (request.hasContentLength() ? (", Length: " + std::to_string(request.getContentLength())) : ("")), request.getContentType(), request.getTransferEncoding()); diff --git a/programs/server/InterserverIOHTTPHandler.cpp b/programs/server/InterserverIOHTTPHandler.cpp index 71b6010f87a..4b733c7f1fd 100644 --- a/programs/server/InterserverIOHTTPHandler.cpp +++ b/programs/server/InterserverIOHTTPHandler.cpp @@ -53,7 +53,7 @@ void InterserverIOHTTPHandler::processQuery(Poco::Net::HTTPServerRequest & reque { HTMLForm params(request); - LOG_TRACE_FORMATTED(log, "Request URI: {}", request.getURI()); + LOG_TRACE(log, "Request URI: {}", request.getURI()); String endpoint_name = params.get("endpoint"); bool compress = params.get("compress") == "true"; @@ -96,14 +96,14 @@ void InterserverIOHTTPHandler::handleRequest(Poco::Net::HTTPServerRequest & requ if (auto [message, success] = checkAuthentication(request); success) { processQuery(request, response, used_output); - LOG_INFO_FORMATTED(log, "Done processing query"); + LOG_INFO(log, "Done processing query"); } else { response.setStatusAndReason(Poco::Net::HTTPServerResponse::HTTP_UNAUTHORIZED); if (!response.sent()) writeString(message, *used_output.out); - LOG_WARNING_FORMATTED(log, "Query processing failed request: '{}' authentication failed", request.getURI()); + LOG_WARNING(log, "Query processing failed request: '{}' authentication failed", request.getURI()); } } catch (Exception & e) @@ -121,9 +121,9 @@ void InterserverIOHTTPHandler::handleRequest(Poco::Net::HTTPServerRequest & requ writeString(message, *used_output.out); if (is_real_error) - LOG_ERROR_FORMATTED(log, message); + LOG_ERROR(log, message); else - LOG_INFO_FORMATTED(log, message); + LOG_INFO(log, message); } catch (...) { @@ -132,7 +132,7 @@ void InterserverIOHTTPHandler::handleRequest(Poco::Net::HTTPServerRequest & requ if (!response.sent()) writeString(message, *used_output.out); - LOG_ERROR_FORMATTED(log, message); + LOG_ERROR(log, message); } } diff --git a/programs/server/MySQLHandler.cpp b/programs/server/MySQLHandler.cpp index fe1cd45bedb..dabb0c12208 100644 --- a/programs/server/MySQLHandler.cpp +++ b/programs/server/MySQLHandler.cpp @@ -73,7 +73,7 @@ void MySQLHandler::run() Handshake handshake(server_capability_flags, connection_id, VERSION_STRING + String("-") + VERSION_NAME, auth_plugin->getName(), auth_plugin->getAuthPluginData()); packet_sender->sendPacket(handshake, true); - LOG_TRACE_FORMATTED(log, "Sent handshake"); + LOG_TRACE(log, "Sent handshake"); HandshakeResponse handshake_response; finishHandshake(handshake_response); @@ -83,7 +83,7 @@ void MySQLHandler::run() if (!connection_context.mysql.max_packet_size) connection_context.mysql.max_packet_size = MAX_PACKET_LENGTH; - LOG_TRACE_FORMATTED(log, + LOG_TRACE(log, "Capabilities: {}, max_packet_size: {}, character_set: {}, user: {}, auth_response length: {}, database: {}, auth_plugin_name: {}", handshake_response.capability_flags, handshake_response.max_packet_size, @@ -125,7 +125,7 @@ void MySQLHandler::run() // For commands which are executed without MemoryTracker. LimitReadBuffer limited_payload(payload, 10000, true, "too long MySQL packet."); - LOG_DEBUG_FORMATTED(log, "Received command: {}. Connection id: {}.", + LOG_DEBUG(log, "Received command: {}. Connection id: {}.", static_cast(static_cast(command)), connection_id); try @@ -195,7 +195,7 @@ void MySQLHandler::finishHandshake(MySQLProtocol::HandshakeResponse & packet) read_bytes(3); /// We can find out whether it is SSLRequest of HandshakeResponse by first 3 bytes. size_t payload_size = unalignedLoad(buf) & 0xFFFFFFu; - LOG_TRACE_FORMATTED(log, "payload size: {}", payload_size); + LOG_TRACE(log, "payload size: {}", payload_size); if (payload_size == SSL_REQUEST_PAYLOAD_SIZE) { @@ -232,18 +232,18 @@ void MySQLHandler::authenticate(const String & user_name, const String & auth_pl } catch (const Exception & exc) { - LOG_ERROR_FORMATTED(log, "Authentication for user {} failed.", user_name); + LOG_ERROR(log, "Authentication for user {} failed.", user_name); packet_sender->sendPacket(ERR_Packet(exc.code(), "00000", exc.message()), true); throw; } - LOG_INFO_FORMATTED(log, "Authentication for user {} succeeded.", user_name); + LOG_INFO(log, "Authentication for user {} succeeded.", user_name); } void MySQLHandler::comInitDB(ReadBuffer & payload) { String database; readStringUntilEOF(database, payload); - LOG_DEBUG_FORMATTED(log, "Setting current database to {}", database); + LOG_DEBUG(log, "Setting current database to {}", database); connection_context.setCurrentDatabase(database); packet_sender->sendPacket(OK_Packet(0, client_capability_flags, 0, 0, 1), true); } diff --git a/programs/server/MySQLHandlerFactory.cpp b/programs/server/MySQLHandlerFactory.cpp index f5d463be1ad..022167fe766 100644 --- a/programs/server/MySQLHandlerFactory.cpp +++ b/programs/server/MySQLHandlerFactory.cpp @@ -32,7 +32,7 @@ MySQLHandlerFactory::MySQLHandlerFactory(IServer & server_) } catch (...) { - LOG_TRACE_FORMATTED(log, "Failed to create SSL context. SSL will be disabled. Error: {}", getCurrentExceptionMessage(false)); + LOG_TRACE(log, "Failed to create SSL context. SSL will be disabled. Error: {}", getCurrentExceptionMessage(false)); ssl_enabled = false; } @@ -43,7 +43,7 @@ MySQLHandlerFactory::MySQLHandlerFactory(IServer & server_) } catch (...) { - LOG_TRACE_FORMATTED(log, "Failed to read RSA key pair from server certificate. Error: {}", getCurrentExceptionMessage(false)); + LOG_TRACE(log, "Failed to read RSA key pair from server certificate. Error: {}", getCurrentExceptionMessage(false)); generateRSAKeys(); } #endif @@ -100,7 +100,7 @@ void MySQLHandlerFactory::readRSAKeys() void MySQLHandlerFactory::generateRSAKeys() { - LOG_TRACE_FORMATTED(log, "Generating new RSA key pair."); + LOG_TRACE(log, "Generating new RSA key pair."); public_key.reset(RSA_new()); if (!public_key) throw Exception("Failed to allocate RSA key. Error: " + getOpenSSLErrors(), ErrorCodes::OPENSSL_ERROR); @@ -122,7 +122,7 @@ void MySQLHandlerFactory::generateRSAKeys() Poco::Net::TCPServerConnection * MySQLHandlerFactory::createConnection(const Poco::Net::StreamSocket & socket) { size_t connection_id = last_connection_id++; - LOG_TRACE_FORMATTED(log, "MySQL connection. Id: {}. Address: {}", connection_id, socket.peerAddress().toString()); + LOG_TRACE(log, "MySQL connection. Id: {}. Address: {}", connection_id, socket.peerAddress().toString()); #if USE_SSL return new MySQLHandlerSSL(server, socket, ssl_enabled, connection_id, *public_key, *private_key); #else diff --git a/programs/server/ReplicasStatusHandler.cpp b/programs/server/ReplicasStatusHandler.cpp index 0a59a727236..f2d1ffe2ee5 100644 --- a/programs/server/ReplicasStatusHandler.cpp +++ b/programs/server/ReplicasStatusHandler.cpp @@ -96,7 +96,7 @@ void ReplicasStatusHandler::handleRequest(Poco::Net::HTTPServerRequest & request } catch (...) { - LOG_ERROR_FORMATTED((&Logger::get("ReplicasStatusHandler")), "Cannot send exception to client"); + LOG_ERROR((&Logger::get("ReplicasStatusHandler")), "Cannot send exception to client"); } } } diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 8c7d9b76c9c..c1a520030f4 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -91,7 +91,7 @@ namespace void setupTmpPath(Logger * log, const std::string & path) { - LOG_DEBUG_FORMATTED(log, "Setting up {} to store temporary data in it", path); + LOG_DEBUG(log, "Setting up {} to store temporary data in it", path); Poco::File(path).createDirectories(); @@ -101,11 +101,11 @@ void setupTmpPath(Logger * log, const std::string & path) { if (it->isFile() && startsWith(it.name(), "tmp")) { - LOG_DEBUG_FORMATTED(log, "Removing old temporary file {}", it->path()); + LOG_DEBUG(log, "Removing old temporary file {}", it->path()); it->remove(); } else - LOG_DEBUG_FORMATTED(log, "Skipped file in temporary path {}", it->path()); + LOG_DEBUG(log, "Skipped file in temporary path {}", it->path()); } } @@ -234,7 +234,7 @@ int Server::main(const std::vector & /*args*/) CurrentMetrics::set(CurrentMetrics::VersionInteger, ClickHouseRevision::getVersionInteger()); if (ThreadFuzzer::instance().isEffective()) - LOG_WARNING_FORMATTED(log, "ThreadFuzzer is enabled. Application will run slowly and unstable."); + LOG_WARNING(log, "ThreadFuzzer is enabled. Application will run slowly and unstable."); /** Context contains all that query execution is dependent: * settings, available functions, data types, aggregate functions, databases... @@ -274,15 +274,15 @@ int Server::main(const std::vector & /*args*/) { if (hasLinuxCapability(CAP_IPC_LOCK)) { - LOG_TRACE_FORMATTED(log, "Will mlockall to prevent executable memory from being paged out. It may take a few seconds."); + LOG_TRACE(log, "Will mlockall to prevent executable memory from being paged out. It may take a few seconds."); if (0 != mlockall(MCL_CURRENT)) - LOG_WARNING_FORMATTED(log, "Failed mlockall: {}", errnoToString(ErrorCodes::SYSTEM_ERROR)); + LOG_WARNING(log, "Failed mlockall: {}", errnoToString(ErrorCodes::SYSTEM_ERROR)); else - LOG_TRACE_FORMATTED(log, "The memory map of clickhouse executable has been mlock'ed"); + LOG_TRACE(log, "The memory map of clickhouse executable has been mlock'ed"); } else { - LOG_INFO_FORMATTED(log, "It looks like the process has no CAP_IPC_LOCK capability, binary mlock will be disabled." + LOG_INFO(log, "It looks like the process has no CAP_IPC_LOCK capability, binary mlock will be disabled." " It could happen due to incorrect ClickHouse package installation." " You could resolve the problem manually with 'sudo setcap cap_ipc_lock=+ep {}'." " Note that it will not work on 'nosuid' mounted filesystems.", executable_path); @@ -312,7 +312,7 @@ int Server::main(const std::vector & /*args*/) } else { - LOG_WARNING_FORMATTED(log, message); + LOG_WARNING(log, message); } } @@ -326,11 +326,11 @@ int Server::main(const std::vector & /*args*/) * It is important to do early, not in destructor of Context, because * table engines could use Context on destroy. */ - LOG_INFO_FORMATTED(log, "Shutting down storages."); + LOG_INFO(log, "Shutting down storages."); global_context->shutdown(); - LOG_DEBUG_FORMATTED(log, "Shut down storages."); + LOG_DEBUG(log, "Shut down storages."); /** Explicitly destroy Context. It is more convenient than in destructor of Server, because logger is still available. * At this moment, no one could own shared part of Context. @@ -338,7 +338,7 @@ int Server::main(const std::vector & /*args*/) global_context_ptr = nullptr; global_context.reset(); shared_context.reset(); - LOG_DEBUG_FORMATTED(log, "Destroyed global context."); + LOG_DEBUG(log, "Destroyed global context."); }); /// Try to increase limit on number of open files. @@ -349,7 +349,7 @@ int Server::main(const std::vector & /*args*/) if (rlim.rlim_cur == rlim.rlim_max) { - LOG_DEBUG_FORMATTED(log, "rlimit on number of file descriptors is {}", rlim.rlim_cur); + LOG_DEBUG(log, "rlimit on number of file descriptors is {}", rlim.rlim_cur); } else { @@ -357,9 +357,9 @@ int Server::main(const std::vector & /*args*/) rlim.rlim_cur = config().getUInt("max_open_files", rlim.rlim_max); int rc = setrlimit(RLIMIT_NOFILE, &rlim); if (rc != 0) - LOG_WARNING_FORMATTED(log, "Cannot set max number of file descriptors to {}. Try to specify max_open_files according to your system limits. error: {}", rlim.rlim_cur, strerror(errno)); + LOG_WARNING(log, "Cannot set max number of file descriptors to {}. Try to specify max_open_files according to your system limits. error: {}", rlim.rlim_cur, strerror(errno)); else - LOG_DEBUG_FORMATTED(log, "Set max number of file descriptors to {} (was {}).", rlim.rlim_cur, old); + LOG_DEBUG(log, "Set max number of file descriptors to {} (was {}).", rlim.rlim_cur, old); } } @@ -367,9 +367,9 @@ int Server::main(const std::vector & /*args*/) Poco::ErrorHandler::set(&error_handler); /// Initialize DateLUT early, to not interfere with running time of first query. - LOG_DEBUG_FORMATTED(log, "Initializing DateLUT."); + LOG_DEBUG(log, "Initializing DateLUT."); DateLUT::instance(); - LOG_TRACE_FORMATTED(log, "Initialized DateLUT with time zone '{}'.", DateLUT::instance().getTimeZone()); + LOG_TRACE(log, "Initialized DateLUT with time zone '{}'.", DateLUT::instance().getTimeZone()); /// Storage with temporary data for processing of heavy queries. @@ -428,7 +428,7 @@ int Server::main(const std::vector & /*args*/) if (this_host.empty()) { this_host = getFQDNOrHostName(); - LOG_DEBUG_FORMATTED(log, "Configuration parameter '{}' doesn't exist or exists and empty. Will use '{}' as replica host.", + LOG_DEBUG(log, "Configuration parameter '{}' doesn't exist or exists and empty. Will use '{}' as replica host.", host_tag, this_host); } @@ -534,7 +534,7 @@ int Server::main(const std::vector & /*args*/) if (uncompressed_cache_size > max_cache_size) { uncompressed_cache_size = max_cache_size; - LOG_INFO_FORMATTED(log, "Uncompressed cache size was lowered to {} because the system has low amount of memory", formatReadableSizeWithBinarySuffix(uncompressed_cache_size)); + LOG_INFO(log, "Uncompressed cache size was lowered to {} because the system has low amount of memory", formatReadableSizeWithBinarySuffix(uncompressed_cache_size)); } global_context->setUncompressedCache(uncompressed_cache_size); @@ -545,11 +545,11 @@ int Server::main(const std::vector & /*args*/) /// Size of cache for marks (index of MergeTree family of tables). It is mandatory. size_t mark_cache_size = config().getUInt64("mark_cache_size"); if (!mark_cache_size) - LOG_ERROR_FORMATTED(log, "Too low mark cache size will lead to severe performance degradation."); + LOG_ERROR(log, "Too low mark cache size will lead to severe performance degradation."); if (mark_cache_size > max_cache_size) { mark_cache_size = max_cache_size; - LOG_INFO_FORMATTED(log, "Mark cache size was lowered to {} because the system has low amount of memory", formatReadableSizeWithBinarySuffix(uncompressed_cache_size)); + LOG_INFO(log, "Mark cache size was lowered to {} because the system has low amount of memory", formatReadableSizeWithBinarySuffix(uncompressed_cache_size)); } global_context->setMarkCache(mark_cache_size); @@ -573,19 +573,19 @@ int Server::main(const std::vector & /*args*/) if (max_server_memory_usage == 0) { max_server_memory_usage = default_max_server_memory_usage; - LOG_INFO_FORMATTED(log, "Setting max_server_memory_usage was set to {}", formatReadableSizeWithBinarySuffix(max_server_memory_usage)); + LOG_INFO(log, "Setting max_server_memory_usage was set to {}", formatReadableSizeWithBinarySuffix(max_server_memory_usage)); } else if (max_server_memory_usage > default_max_server_memory_usage) { max_server_memory_usage = default_max_server_memory_usage; - LOG_INFO_FORMATTED(log, "Setting max_server_memory_usage was lowered to {} because the system has low amount of memory", formatReadableSizeWithBinarySuffix(max_server_memory_usage)); + LOG_INFO(log, "Setting max_server_memory_usage was lowered to {} because the system has low amount of memory", formatReadableSizeWithBinarySuffix(max_server_memory_usage)); } total_memory_tracker.setOrRaiseHardLimit(max_server_memory_usage); total_memory_tracker.setDescription("(total)"); total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking); - LOG_INFO_FORMATTED(log, "Loading metadata from {}", path); + LOG_INFO(log, "Loading metadata from {}", path); try { @@ -603,7 +603,7 @@ int Server::main(const std::vector & /*args*/) tryLogCurrentException(log, "Caught exception while loading metadata"); throw; } - LOG_DEBUG_FORMATTED(log, "Loaded metadata."); + LOG_DEBUG(log, "Loaded metadata."); /// Init trace collector only after trace_log system table was created /// Disable it if we collect test coverage information, because it will work extremely slow. @@ -643,20 +643,20 @@ int Server::main(const std::vector & /*args*/) /// Describe multiple reasons when query profiler cannot work. #if !USE_UNWIND - LOG_INFO_FORMATTED(log, "Query Profiler and TraceCollector are disabled because they cannot work without bundled unwind (stack unwinding) library."); + LOG_INFO(log, "Query Profiler and TraceCollector are disabled because they cannot work without bundled unwind (stack unwinding) library."); #endif #if WITH_COVERAGE - LOG_INFO_FORMATTED(log, "Query Profiler and TraceCollector are disabled because they work extremely slow with test coverage."); + LOG_INFO(log, "Query Profiler and TraceCollector are disabled because they work extremely slow with test coverage."); #endif #if defined(SANITIZER) - LOG_INFO_FORMATTED(log, "Query Profiler and TraceCollector are disabled because they cannot work under sanitizers" + LOG_INFO(log, "Query Profiler and TraceCollector are disabled because they cannot work under sanitizers" " when two different stack unwinding methods will interfere with each other."); #endif if (!hasPHDRCache()) - LOG_INFO_FORMATTED(log, "Query Profiler and TraceCollector are disabled because they require PHDR cache to be created" + LOG_INFO(log, "Query Profiler and TraceCollector are disabled because they require PHDR cache to be created" " (otherwise the function 'dl_iterate_phdr' is not lock free and not async-signal safe)."); global_context->setCurrentDatabase(default_database); @@ -683,7 +683,7 @@ int Server::main(const std::vector & /*args*/) #if defined(OS_LINUX) if (!TasksStatsCounters::checkIfAvailable()) { - LOG_INFO_FORMATTED(log, "It looks like this system does not have procfs mounted at /proc location," + LOG_INFO(log, "It looks like this system does not have procfs mounted at /proc location," " neither clickhouse-server process has CAP_NET_ADMIN capability." " 'taskstats' performance statistics will be disabled." " It could happen due to incorrect ClickHouse package installation." @@ -695,14 +695,14 @@ int Server::main(const std::vector & /*args*/) if (!hasLinuxCapability(CAP_SYS_NICE)) { - LOG_INFO_FORMATTED(log, "It looks like the process has no CAP_SYS_NICE capability, the setting 'os_thread_nice' will have no effect." + LOG_INFO(log, "It looks like the process has no CAP_SYS_NICE capability, the setting 'os_thread_nice' will have no effect." " It could happen due to incorrect ClickHouse package installation." " You could resolve the problem manually with 'sudo setcap cap_sys_nice=+ep {}'." " Note that it will not work on 'nosuid' mounted filesystems.", executable_path); } #else - LOG_INFO_FORMATTED(log, "TaskStats is not implemented for this OS. IO accounting will be disabled."); + LOG_INFO(log, "TaskStats is not implemented for this OS. IO accounting will be disabled."); #endif { @@ -741,7 +741,7 @@ int Server::main(const std::vector & /*args*/) #endif ) { - LOG_ERROR_FORMATTED(log, "Cannot resolve listen_host ({}), error {}: {}. " + LOG_ERROR(log, "Cannot resolve listen_host ({}), error {}: {}. " "If it is an IPv6 address and your host has disabled IPv6, then consider to " "specify IPv4 address to listen in element of configuration " "file. Example: 0.0.0.0", @@ -797,7 +797,7 @@ int Server::main(const std::vector & /*args*/) if (listen_try) { - LOG_ERROR_FORMATTED(log, "{}. If it is an IPv6 or IPv4 address and your host has disabled IPv6 or IPv4, then consider to " + LOG_ERROR(log, "{}. If it is an IPv6 or IPv4 address and your host has disabled IPv6 or IPv4, then consider to " "specify not disabled IPv4 or IPv6 address to listen in element of configuration " "file. Example for disabled IPv6: 0.0.0.0 ." " Example for disabled IPv4: ::", @@ -821,7 +821,7 @@ int Server::main(const std::vector & /*args*/) servers.emplace_back(std::make_unique( createHandlerFactory(*this, async_metrics, "HTTPHandler-factory"), server_pool, socket, http_params)); - LOG_INFO_FORMATTED(log, "Listening for http://{}", address.toString()); + LOG_INFO(log, "Listening for http://{}", address.toString()); }); /// HTTPS @@ -835,7 +835,7 @@ int Server::main(const std::vector & /*args*/) servers.emplace_back(std::make_unique( createHandlerFactory(*this, async_metrics, "HTTPSHandler-factory"), server_pool, socket, http_params)); - LOG_INFO_FORMATTED(log, "Listening for https://{}", address.toString()); + LOG_INFO(log, "Listening for https://{}", address.toString()); #else UNUSED(port); throw Exception{"HTTPS protocol is disabled because Poco library was built without NetSSL support.", @@ -856,7 +856,7 @@ int Server::main(const std::vector & /*args*/) socket, new Poco::Net::TCPServerParams)); - LOG_INFO_FORMATTED(log, "Listening for connections with native protocol (tcp): {}", address.toString()); + LOG_INFO(log, "Listening for connections with native protocol (tcp): {}", address.toString()); }); /// TCP with SSL @@ -872,7 +872,7 @@ int Server::main(const std::vector & /*args*/) server_pool, socket, new Poco::Net::TCPServerParams)); - LOG_INFO_FORMATTED(log, "Listening for connections with secure native protocol (tcp_secure): {}", address.toString()); + LOG_INFO(log, "Listening for connections with secure native protocol (tcp_secure): {}", address.toString()); #else UNUSED(port); throw Exception{"SSL support for TCP protocol is disabled because Poco library was built without NetSSL support.", @@ -890,7 +890,7 @@ int Server::main(const std::vector & /*args*/) servers.emplace_back(std::make_unique( createHandlerFactory(*this, async_metrics, "InterserverIOHTTPHandler-factory"), server_pool, socket, http_params)); - LOG_INFO_FORMATTED(log, "Listening for replica communication (interserver): http://{}", address.toString()); + LOG_INFO(log, "Listening for replica communication (interserver): http://{}", address.toString()); }); create_server("interserver_https_port", [&](UInt16 port) @@ -903,7 +903,7 @@ int Server::main(const std::vector & /*args*/) servers.emplace_back(std::make_unique( createHandlerFactory(*this, async_metrics, "InterserverIOHTTPSHandler-factory"), server_pool, socket, http_params)); - LOG_INFO_FORMATTED(log, "Listening for secure replica communication (interserver): https://{}", address.toString()); + LOG_INFO(log, "Listening for secure replica communication (interserver): https://{}", address.toString()); #else UNUSED(port); throw Exception{"SSL support for TCP protocol is disabled because Poco library was built without NetSSL support.", @@ -923,7 +923,7 @@ int Server::main(const std::vector & /*args*/) socket, new Poco::Net::TCPServerParams)); - LOG_INFO_FORMATTED(log, "Listening for MySQL compatibility protocol: {}", address.toString()); + LOG_INFO(log, "Listening for MySQL compatibility protocol: {}", address.toString()); }); /// Prometheus (if defined and not setup yet with http_port) @@ -936,7 +936,7 @@ int Server::main(const std::vector & /*args*/) servers.emplace_back(std::make_unique( createHandlerFactory(*this, async_metrics, "PrometheusHandler-factory"), server_pool, socket, http_params)); - LOG_INFO_FORMATTED(log, "Listening for Prometheus: http://{}", address.toString()); + LOG_INFO(log, "Listening for Prometheus: http://{}", address.toString()); }); } @@ -961,17 +961,17 @@ int Server::main(const std::vector & /*args*/) dns_cache_updater->start(); { - LOG_INFO_FORMATTED(log, "Available RAM: {}; physical cores: {}; logical cores: {}.", + LOG_INFO(log, "Available RAM: {}; physical cores: {}; logical cores: {}.", formatReadableSizeWithBinarySuffix(memory_amount), getNumberOfPhysicalCPUCores(), // on ARM processors it can show only enabled at current moment cores std::thread::hardware_concurrency()); } - LOG_INFO_FORMATTED(log, "Ready for connections."); + LOG_INFO(log, "Ready for connections."); SCOPE_EXIT({ - LOG_DEBUG_FORMATTED(log, "Received termination signal."); - LOG_DEBUG_FORMATTED(log, "Waiting for current connections to close."); + LOG_DEBUG(log, "Received termination signal."); + LOG_DEBUG(log, "Waiting for current connections to close."); is_cancelled = true; @@ -983,9 +983,9 @@ int Server::main(const std::vector & /*args*/) } if (current_connections) - LOG_INFO_FORMATTED(log, "Closed all listening sockets. Waiting for {} outstanding connections.", current_connections); + LOG_INFO(log, "Closed all listening sockets. Waiting for {} outstanding connections.", current_connections); else - LOG_INFO_FORMATTED(log, "Closed all listening sockets."); + LOG_INFO(log, "Closed all listening sockets."); /// Killing remaining queries. global_context->getProcessList().killAllQueries(); @@ -1008,10 +1008,10 @@ int Server::main(const std::vector & /*args*/) } if (current_connections) - LOG_INFO_FORMATTED(log, "Closed connections. But {} remain." + LOG_INFO(log, "Closed connections. But {} remain." " Tip: To increase wait time add to config: 60", current_connections); else - LOG_INFO_FORMATTED(log, "Closed connections."); + LOG_INFO(log, "Closed connections."); dns_cache_updater.reset(); main_config_reloader.reset(); @@ -1026,7 +1026,7 @@ int Server::main(const std::vector & /*args*/) /// Dump coverage here, because std::atexit callback would not be called. dumpCoverageReportIfPossible(); - LOG_INFO_FORMATTED(log, "Will shutdown forcefully."); + LOG_INFO(log, "Will shutdown forcefully."); _exit(Application::EXIT_OK); } }); @@ -1047,7 +1047,7 @@ int Server::main(const std::vector & /*args*/) } catch (...) { - LOG_ERROR_FORMATTED(log, "Caught exception while loading dictionaries."); + LOG_ERROR(log, "Caught exception while loading dictionaries."); throw; } diff --git a/programs/server/TCPHandler.cpp b/programs/server/TCPHandler.cpp index 8de03adae30..190c549361f 100644 --- a/programs/server/TCPHandler.cpp +++ b/programs/server/TCPHandler.cpp @@ -76,7 +76,7 @@ void TCPHandler::runImpl() if (in->eof()) { - LOG_WARNING_FORMATTED(log, "Client has not sent any data."); + LOG_WARNING(log, "Client has not sent any data."); return; } @@ -89,13 +89,13 @@ void TCPHandler::runImpl() { if (e.code() == ErrorCodes::CLIENT_HAS_CONNECTED_TO_WRONG_PORT) { - LOG_DEBUG_FORMATTED(log, "Client has connected to wrong port."); + LOG_DEBUG(log, "Client has connected to wrong port."); return; } if (e.code() == ErrorCodes::ATTEMPT_TO_READ_AFTER_EOF) { - LOG_WARNING_FORMATTED(log, "Client has gone away."); + LOG_WARNING(log, "Client has gone away."); return; } @@ -115,7 +115,7 @@ void TCPHandler::runImpl() if (!DatabaseCatalog::instance().isDatabaseExist(default_database)) { Exception e("Database " + backQuote(default_database) + " doesn't exist", ErrorCodes::UNKNOWN_DATABASE); - LOG_ERROR_FORMATTED(log, "Code: {}, e.displayText() = {}, Stack trace:\n\n{}", e.code(), e.displayText(), e.getStackTraceString()); + LOG_ERROR(log, "Code: {}, e.displayText() = {}, Stack trace:\n\n{}", e.code(), e.displayText(), e.getStackTraceString()); sendException(e, connection_context.getSettingsRef().calculate_text_stack_trace); return; } @@ -139,7 +139,7 @@ void TCPHandler::runImpl() { if (idle_time.elapsedSeconds() > connection_settings.idle_connection_timeout) { - LOG_TRACE_FORMATTED(log, "Closing idle connection"); + LOG_TRACE(log, "Closing idle connection"); return; } } @@ -344,7 +344,7 @@ void TCPHandler::runImpl() { /** Could not send exception information to the client. */ network_error = true; - LOG_WARNING_FORMATTED(log, "Client has gone away."); + LOG_WARNING(log, "Client has gone away."); } try @@ -355,7 +355,7 @@ void TCPHandler::runImpl() catch (...) { network_error = true; - LOG_WARNING_FORMATTED(log, "Can't read external tables after query failure."); + LOG_WARNING(log, "Can't read external tables after query failure."); } @@ -378,7 +378,7 @@ void TCPHandler::runImpl() watch.stop(); - LOG_INFO_FORMATTED(log, "Processed in {} sec.", watch.elapsedSeconds()); + LOG_INFO(log, "Processed in {} sec.", watch.elapsedSeconds()); /// It is important to destroy query context here. We do not want it to live arbitrarily longer than the query. query_context.reset(); @@ -730,7 +730,7 @@ void TCPHandler::receiveHello() readStringBinary(user, *in); readStringBinary(password, *in); - LOG_DEBUG_FORMATTED(log, "Connected {} version {}.{}.{}, revision: {}{}{}.", + LOG_DEBUG(log, "Connected {} version {}.{}.{}, revision: {}{}{}.", client_name, client_version_major, client_version_minor, client_version_patch, client_revision, @@ -1076,7 +1076,7 @@ bool TCPHandler::isQueryCancelled() case Protocol::Client::Cancel: if (state.empty()) throw NetException("Unexpected packet Cancel received from client", ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT); - LOG_INFO_FORMATTED(log, "Query was cancelled."); + LOG_INFO(log, "Query was cancelled."); state.is_cancelled = true; return true; @@ -1194,14 +1194,14 @@ void TCPHandler::run() { runImpl(); - LOG_INFO_FORMATTED(log, "Done processing connection."); + LOG_INFO(log, "Done processing connection."); } catch (Poco::Exception & e) { /// Timeout - not an error. if (!strcmp(e.what(), "Timeout")) { - LOG_DEBUG_FORMATTED(log, "Poco::Exception. Code: {}, e.code() = {}, e.displayText() = {}, e.what() = {}", ErrorCodes::POCO_EXCEPTION, e.code(), e.displayText(), e.what()); + LOG_DEBUG(log, "Poco::Exception. Code: {}, e.code() = {}, e.displayText() = {}, e.what() = {}", ErrorCodes::POCO_EXCEPTION, e.code(), e.displayText(), e.what()); } else throw; diff --git a/programs/server/TCPHandlerFactory.h b/programs/server/TCPHandlerFactory.h index dc08fc8825d..68652540192 100644 --- a/programs/server/TCPHandlerFactory.h +++ b/programs/server/TCPHandlerFactory.h @@ -35,12 +35,12 @@ public: { try { - LOG_TRACE_FORMATTED(log, "TCP Request. Address: {}", socket.peerAddress().toString()); + LOG_TRACE(log, "TCP Request. Address: {}", socket.peerAddress().toString()); return new TCPHandler(server, socket); } catch (const Poco::Net::NetException &) { - LOG_TRACE_FORMATTED(log, "TCP Request. Client is not connected (most likely RST packet was sent)."); + LOG_TRACE(log, "TCP Request. Client is not connected (most likely RST packet was sent)."); return new DummyTCPHandler(socket); } } diff --git a/src/Access/AccessRights.cpp b/src/Access/AccessRights.cpp index 966fcdfc062..a4e446750a7 100644 --- a/src/Access/AccessRights.cpp +++ b/src/Access/AccessRights.cpp @@ -251,7 +251,7 @@ public: void logTree(Poco::Logger * log) const { - LOG_TRACE_FORMATTED(log, "Tree({}): name={}, access={}, final_access={}, min_access={}, max_access={}, num_children={}", + LOG_TRACE(log, "Tree({}): name={}, access={}, final_access={}, min_access={}, max_access={}, num_children={}", level, node_name ? *node_name : "NULL", access.toString(), final_access.toString(), min_access.toString(), max_access.toString(), (children ? children->size() : 0)); @@ -793,6 +793,6 @@ void AccessRights::logTree() const if (root) root->logTree(log); else - LOG_TRACE_FORMATTED(log, "Tree: NULL"); + LOG_TRACE(log, "Tree: NULL"); } } diff --git a/src/Access/AllowedClientHosts.cpp b/src/Access/AllowedClientHosts.cpp index 201efa499be..9c8a7cc12f5 100644 --- a/src/Access/AllowedClientHosts.cpp +++ b/src/Access/AllowedClientHosts.cpp @@ -308,7 +308,7 @@ bool AllowedClientHosts::contains(const IPAddress & client_address) const if (e.code() != ErrorCodes::DNS_ERROR) throw; /// Try to ignore DNS errors: if host cannot be resolved, skip it and try next. - LOG_WARNING_FORMATTED( + LOG_WARNING( &Logger::get("AddressPatterns"), "Failed to check if the allowed client hosts contain address {}. {}, code = {}", client_address.toString(), e.displayText(), e.code()); @@ -341,7 +341,7 @@ bool AllowedClientHosts::contains(const IPAddress & client_address) const if (e.code() != ErrorCodes::DNS_ERROR) throw; /// Try to ignore DNS errors: if host cannot be resolved, skip it and try next. - LOG_WARNING_FORMATTED( + LOG_WARNING( &Logger::get("AddressPatterns"), "Failed to check if the allowed client hosts contain address {}. {}, code = {}", client_address.toString(), e.displayText(), e.code()); diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index 644f4a5817d..82ed5920243 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -200,7 +200,7 @@ bool ContextAccess::calculateResultAccessAndCheck(Poco::Logger * log_, const Acc bool is_granted = access->isGranted(flags, args...); if (trace_log) - LOG_TRACE_FORMATTED(trace_log, "Access {}: {}", (is_granted ? "granted" : "denied"), (AccessRightsElement{flags, args...}.toString())); + LOG_TRACE(trace_log, "Access {}: {}", (is_granted ? "granted" : "denied"), (AccessRightsElement{flags, args...}.toString())); if (is_granted) return true; @@ -219,7 +219,7 @@ bool ContextAccess::calculateResultAccessAndCheck(Poco::Logger * log_, const Acc if constexpr (mode == THROW_IF_ACCESS_DENIED) throw Exception(user_name + ": " + msg, error_code); else if constexpr (mode == LOG_WARNING_IF_ACCESS_DENIED) - LOG_WARNING_FORMATTED(log_, "{}: {}{}", user_name, msg, formatSkippedMessage(args...)); + LOG_WARNING(log_, "{}: {}{}", user_name, msg, formatSkippedMessage(args...)); }; if (!user) @@ -452,17 +452,17 @@ boost::shared_ptr ContextAccess::calculateResultAccess(bool if (trace_log && (params.readonly == readonly_) && (params.allow_ddl == allow_ddl_) && (params.allow_introspection == allow_introspection_)) { if (grant_option) - LOG_TRACE_FORMATTED(trace_log, "List of all grants: {} WITH GRANT OPTION", merged_access->toString()); + LOG_TRACE(trace_log, "List of all grants: {} WITH GRANT OPTION", merged_access->toString()); else - LOG_TRACE_FORMATTED(trace_log, "List of all grants: {}", merged_access->toString()); + LOG_TRACE(trace_log, "List of all grants: {}", merged_access->toString()); if (roles_info && !roles_info->getCurrentRolesNames().empty()) { - LOG_TRACE_FORMATTED(trace_log, "Current_roles: {}, enabled_roles: {}", + LOG_TRACE(trace_log, "Current_roles: {}, enabled_roles: {}", boost::algorithm::join(roles_info->getCurrentRolesNames(), ", "), boost::algorithm::join(roles_info->getEnabledRolesNames(), ", ")); } - LOG_TRACE_FORMATTED(trace_log, "Settings: readonly={}, allow_ddl={}, allow_introspection_functions={}", readonly_, allow_ddl_, allow_introspection_); + LOG_TRACE(trace_log, "Settings: readonly={}, allow_ddl={}, allow_introspection_functions={}", readonly_, allow_ddl_, allow_introspection_); } res = std::move(merged_access); diff --git a/src/Access/DiskAccessStorage.cpp b/src/Access/DiskAccessStorage.cpp index 00da0dfe2ae..1195bcf842c 100644 --- a/src/Access/DiskAccessStorage.cpp +++ b/src/Access/DiskAccessStorage.cpp @@ -367,7 +367,7 @@ bool DiskAccessStorage::readLists() auto file_path = getListFilePath(directory_path, type); if (!std::filesystem::exists(file_path)) { - LOG_WARNING_FORMATTED(getLogger(), "File {} doesn't exist", file_path.string()); + LOG_WARNING(getLogger(), "File {} doesn't exist", file_path.string()); ok = false; break; } @@ -496,7 +496,7 @@ void DiskAccessStorage::listsWritingThreadFunc() /// and then saves the files "users.list", "roles.list", etc. to the same directory. bool DiskAccessStorage::rebuildLists() { - LOG_WARNING_FORMATTED(getLogger(), "Recovering lists in directory {}", directory_path); + LOG_WARNING(getLogger(), "Recovering lists in directory {}", directory_path); clear(); for (const auto & directory_entry : std::filesystem::directory_iterator(directory_path)) diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index 9d51a27408a..6c1f7f8e953 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -61,7 +61,7 @@ void Connection::connect(const ConnectionTimeouts & timeouts) if (connected) disconnect(); - LOG_TRACE_FORMATTED(log_wrapper.get(), "Connecting. Database: {}. User: {}{}{}", + LOG_TRACE(log_wrapper.get(), "Connecting. Database: {}. User: {}{}{}", default_database.empty() ? "(not specified)" : default_database, user, static_cast(secure) ? ". Secure" : "", @@ -107,7 +107,7 @@ void Connection::connect(const ConnectionTimeouts & timeouts) sendHello(); receiveHello(); - LOG_TRACE_FORMATTED(log_wrapper.get(), "Connected to {} server version {}.{}.{}.", + LOG_TRACE(log_wrapper.get(), "Connected to {} server version {}.{}.{}.", server_name, server_version_major, server_version_minor, server_version_patch); } catch (Poco::Net::NetException & e) @@ -284,14 +284,14 @@ void Connection::forceConnected(const ConnectionTimeouts & timeouts) } else if (!ping()) { - LOG_TRACE_FORMATTED(log_wrapper.get(), "Connection was closed, will reconnect."); + LOG_TRACE(log_wrapper.get(), "Connection was closed, will reconnect."); connect(timeouts); } } bool Connection::ping() { - // LOG_TRACE_FORMATTED(log_wrapper.get(), "Ping"); + // LOG_TRACE(log_wrapper.get(), "Ping"); TimeoutSetter timeout_setter(*socket, sync_request_timeout, true); try @@ -321,7 +321,7 @@ bool Connection::ping() } catch (const Poco::Exception & e) { - LOG_TRACE_FORMATTED(log_wrapper.get(), e.displayText()); + LOG_TRACE(log_wrapper.get(), e.displayText()); return false; } @@ -504,7 +504,7 @@ void Connection::sendScalarsData(Scalars & data) double elapsed = watch.elapsedSeconds(); if (compression == Protocol::Compression::Enable) - LOG_DEBUG_FORMATTED(log_wrapper.get(), + LOG_DEBUG(log_wrapper.get(), "Sent data for {} scalars, total {} rows in {} sec., {} rows/sec., {} ({}/sec.), compressed {} times to {} ({}/sec.)", data.size(), rows, elapsed, static_cast(rows / watch.elapsedSeconds()), @@ -514,7 +514,7 @@ void Connection::sendScalarsData(Scalars & data) formatReadableSizeWithBinarySuffix(out_bytes), formatReadableSizeWithBinarySuffix(out_bytes / watch.elapsedSeconds())); else - LOG_DEBUG_FORMATTED(log_wrapper.get(), + LOG_DEBUG(log_wrapper.get(), "Sent data for {} scalars, total {} rows in {} sec., {} rows/sec., {} ({}/sec.), no compression.", data.size(), rows, elapsed, static_cast(rows / watch.elapsedSeconds()), @@ -608,7 +608,7 @@ void Connection::sendExternalTablesData(ExternalTablesData & data) double elapsed = watch.elapsedSeconds(); if (compression == Protocol::Compression::Enable) - LOG_DEBUG_FORMATTED(log_wrapper.get(), + LOG_DEBUG(log_wrapper.get(), "Sent data for {} external tables, total {} rows in {} sec., {} rows/sec., {} ({}/sec.), compressed {} times to {} ({}/sec.)", data.size(), rows, elapsed, static_cast(rows / watch.elapsedSeconds()), @@ -618,7 +618,7 @@ void Connection::sendExternalTablesData(ExternalTablesData & data) formatReadableSizeWithBinarySuffix(out_bytes), formatReadableSizeWithBinarySuffix(out_bytes / watch.elapsedSeconds())); else - LOG_DEBUG_FORMATTED(log_wrapper.get(), + LOG_DEBUG(log_wrapper.get(), "Sent data for {} external tables, total {} rows in {} sec., {} rows/sec., {} ({}/sec.), no compression.", data.size(), rows, elapsed, static_cast(rows / watch.elapsedSeconds()), @@ -651,7 +651,7 @@ std::optional Connection::checkPacket(size_t timeout_microseconds) if (hasReadPendingData() || poll(timeout_microseconds)) { - // LOG_TRACE_FORMATTED(log_wrapper.get(), "Receiving packet type"); + // LOG_TRACE(log_wrapper.get(), "Receiving packet type"); UInt64 packet_type; readVarUInt(packet_type, *in); diff --git a/src/Client/ConnectionPoolWithFailover.cpp b/src/Client/ConnectionPoolWithFailover.cpp index 288292683a5..0bfebd15ef7 100644 --- a/src/Client/ConnectionPoolWithFailover.cpp +++ b/src/Client/ConnectionPoolWithFailover.cpp @@ -224,7 +224,7 @@ ConnectionPoolWithFailover::tryGetEntry( { auto message_pattern = "There is no table {}.{} on server: {}"; fail_message = fmt::format(message_pattern, backQuote(table_to_check->database), backQuote(table_to_check->table), result.entry->getDescription()); - LOG_WARNING_FORMATTED(log, fail_message); + LOG_WARNING(log, fail_message); ProfileEvents::increment(ProfileEvents::DistributedConnectionMissingTable); return result; @@ -248,7 +248,7 @@ ConnectionPoolWithFailover::tryGetEntry( result.is_up_to_date = false; result.staleness = delay; - LOG_TRACE_FORMATTED(log, "Server {} has unacceptable replica delay for table {}.{}: {}", result.entry->getDescription(), table_to_check->database, table_to_check->table, delay); + LOG_TRACE(log, "Server {} has unacceptable replica delay for table {}.{}: {}", result.entry->getDescription(), table_to_check->database, table_to_check->table, delay); ProfileEvents::increment(ProfileEvents::DistributedConnectionStaleReplica); } } diff --git a/src/Client/TimeoutSetter.cpp b/src/Client/TimeoutSetter.cpp index 5fb0392ced3..10cc9fbffef 100644 --- a/src/Client/TimeoutSetter.cpp +++ b/src/Client/TimeoutSetter.cpp @@ -35,7 +35,7 @@ TimeoutSetter::~TimeoutSetter() catch (std::exception & e) { // Sometimes catched on macos - LOG_ERROR_FORMATTED(&Logger::get("Client"), "TimeoutSetter: Can't reset timeouts: {}", e.what()); + LOG_ERROR(&Logger::get("Client"), "TimeoutSetter: Can't reset timeouts: {}", e.what()); } } } diff --git a/src/Common/Config/ConfigProcessor.cpp b/src/Common/Config/ConfigProcessor.cpp index 0e0430b7870..7eb7e13a93e 100644 --- a/src/Common/Config/ConfigProcessor.cpp +++ b/src/Common/Config/ConfigProcessor.cpp @@ -304,7 +304,7 @@ void ConfigProcessor::doIncludesRecursive( else if (throw_on_bad_incl) throw Poco::Exception(error_msg + name); else - LOG_WARNING_FORMATTED(log, "{}{}", error_msg, name); + LOG_WARNING(log, "{}{}", error_msg, name); } else { @@ -441,7 +441,7 @@ XMLDocumentPtr ConfigProcessor::processConfig( zkutil::ZooKeeperNodeCache * zk_node_cache, const zkutil::EventPtr & zk_changed_event) { - LOG_DEBUG_FORMATTED(log, "Processing configuration file '{}'.", path); + LOG_DEBUG(log, "Processing configuration file '{}'.", path); XMLDocumentPtr config = dom_parser.parse(path); @@ -452,7 +452,7 @@ XMLDocumentPtr ConfigProcessor::processConfig( { try { - LOG_DEBUG_FORMATTED(log, "Merging configuration file '{}'.", merge_file); + LOG_DEBUG(log, "Merging configuration file '{}'.", merge_file); XMLDocumentPtr with = dom_parser.parse(merge_file); merge(config, with); @@ -489,7 +489,7 @@ XMLDocumentPtr ConfigProcessor::processConfig( } if (!include_from_path.empty()) { - LOG_DEBUG_FORMATTED(log, "Including configuration file '{}'.", include_from_path); + LOG_DEBUG(log, "Including configuration file '{}'.", include_from_path); contributing_files.push_back(include_from_path); include_from = dom_parser.parse(include_from_path); @@ -569,7 +569,7 @@ ConfigProcessor::LoadedConfig ConfigProcessor::loadConfigWithZooKeeperIncludes( if (!zk_exception) throw; - LOG_WARNING_FORMATTED(log, "Error while processing from_zk config includes: {}. Config will be loaded from preprocessed file: {}", zk_exception->message(), preprocessed_path); + LOG_WARNING(log, "Error while processing from_zk config includes: {}. Config will be loaded from preprocessed file: {}", zk_exception->message(), preprocessed_path); config_xml = dom_parser.parse(preprocessed_path); } @@ -617,11 +617,11 @@ void ConfigProcessor::savePreprocessedConfig(const LoadedConfig & loaded_config, Poco::File(preprocessed_path_parent).createDirectories(); } DOMWriter().writeNode(preprocessed_path, loaded_config.preprocessed_xml); - LOG_DEBUG_FORMATTED(log, "Saved preprocessed configuration to '{}'.", preprocessed_path); + LOG_DEBUG(log, "Saved preprocessed configuration to '{}'.", preprocessed_path); } catch (Poco::Exception & e) { - LOG_WARNING_FORMATTED(log, "Couldn't save preprocessed config to {}: {}", preprocessed_path, e.displayText()); + LOG_WARNING(log, "Couldn't save preprocessed config to {}: {}", preprocessed_path, e.displayText()); } } diff --git a/src/Common/Config/ConfigReloader.cpp b/src/Common/Config/ConfigReloader.cpp index 4a9cbd213ff..6da4832210e 100644 --- a/src/Common/Config/ConfigReloader.cpp +++ b/src/Common/Config/ConfigReloader.cpp @@ -87,7 +87,7 @@ void ConfigReloader::reloadIfNewer(bool force, bool throw_on_error, bool fallbac ConfigProcessor::LoadedConfig loaded_config; try { - LOG_DEBUG_FORMATTED(log, "Loading config '{}'", path); + LOG_DEBUG(log, "Loading config '{}'", path); loaded_config = config_processor.loadConfig(/* allow_zk_includes = */ true); if (loaded_config.has_zk_includes) diff --git a/src/Common/DNSResolver.cpp b/src/Common/DNSResolver.cpp index 4b039bb810a..1d2d89f477c 100644 --- a/src/Common/DNSResolver.cpp +++ b/src/Common/DNSResolver.cpp @@ -202,7 +202,7 @@ bool DNSResolver::updateCache() } if (!lost_hosts.empty()) - LOG_INFO_FORMATTED(&Logger::get("DNSResolver"), "Cached hosts not found: {}", lost_hosts); + LOG_INFO(&Logger::get("DNSResolver"), "Cached hosts not found: {}", lost_hosts); return updated; } diff --git a/src/Common/Exception.cpp b/src/Common/Exception.cpp index e7f0adb2807..23e5e0a762e 100644 --- a/src/Common/Exception.cpp +++ b/src/Common/Exception.cpp @@ -36,7 +36,7 @@ Exception::Exception(const std::string & msg, int code) #ifndef NDEBUG if (code == ErrorCodes::LOGICAL_ERROR) { - LOG_ERROR_FORMATTED(&Poco::Logger::root(), "Logical error: '{}'.", msg); + LOG_ERROR(&Poco::Logger::root(), "Logical error: '{}'.", msg); assert(false); } #endif @@ -126,9 +126,9 @@ void tryLogCurrentException(Poco::Logger * logger, const std::string & start_of_ try { if (start_of_message.empty()) - LOG_ERROR_FORMATTED(logger, "{}", getCurrentExceptionMessage(true)); + LOG_ERROR(logger, "{}", getCurrentExceptionMessage(true)); else - LOG_ERROR_FORMATTED(logger, "{}: {}", start_of_message, getCurrentExceptionMessage(true)); + LOG_ERROR(logger, "{}: {}", start_of_message, getCurrentExceptionMessage(true)); } catch (...) { diff --git a/src/Common/LRUCache.h b/src/Common/LRUCache.h index 988b7826977..5bcfc8fc2db 100644 --- a/src/Common/LRUCache.h +++ b/src/Common/LRUCache.h @@ -306,7 +306,7 @@ private: auto it = cells.find(key); if (it == cells.end()) { - LOG_ERROR_FORMATTED(&Logger::get("LRUCache"), "LRUCache became inconsistent. There must be a bug in it."); + LOG_ERROR(&Logger::get("LRUCache"), "LRUCache became inconsistent. There must be a bug in it."); abort(); } @@ -324,7 +324,7 @@ private: if (current_size > (1ull << 63)) { - LOG_ERROR_FORMATTED(&Logger::get("LRUCache"), "LRUCache became inconsistent. There must be a bug in it."); + LOG_ERROR(&Logger::get("LRUCache"), "LRUCache became inconsistent. There must be a bug in it."); abort(); } } diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp index 79825b90880..082b78682a2 100644 --- a/src/Common/MemoryTracker.cpp +++ b/src/Common/MemoryTracker.cpp @@ -49,12 +49,12 @@ MemoryTracker::~MemoryTracker() void MemoryTracker::logPeakMemoryUsage() const { - LOG_DEBUG_FORMATTED(&Logger::get("MemoryTracker"), "Peak memory usage{}: {}.", (description ? " " + std::string(description) : ""), formatReadableSizeWithBinarySuffix(peak)); + LOG_DEBUG(&Logger::get("MemoryTracker"), "Peak memory usage{}: {}.", (description ? " " + std::string(description) : ""), formatReadableSizeWithBinarySuffix(peak)); } void MemoryTracker::logMemoryUsage(Int64 current) const { - LOG_DEBUG_FORMATTED(&Logger::get("MemoryTracker"), "Current memory usage{}: {}.", (description ? " " + std::string(description) : ""), formatReadableSizeWithBinarySuffix(current)); + LOG_DEBUG(&Logger::get("MemoryTracker"), "Current memory usage{}: {}.", (description ? " " + std::string(description) : ""), formatReadableSizeWithBinarySuffix(current)); } diff --git a/src/Common/PipeFDs.cpp b/src/Common/PipeFDs.cpp index b16feaf51ef..cec8cb1cf29 100644 --- a/src/Common/PipeFDs.cpp +++ b/src/Common/PipeFDs.cpp @@ -90,7 +90,7 @@ void LazyPipeFDs::tryIncreaseSize(int desired_size) { if (errno == EINVAL) { - LOG_INFO_FORMATTED(log, "Cannot get pipe capacity, {}. Very old Linux kernels have no support for this fcntl.", errnoToString(ErrorCodes::CANNOT_FCNTL)); + LOG_INFO(log, "Cannot get pipe capacity, {}. Very old Linux kernels have no support for this fcntl.", errnoToString(ErrorCodes::CANNOT_FCNTL)); /// It will work nevertheless. } else @@ -102,7 +102,7 @@ void LazyPipeFDs::tryIncreaseSize(int desired_size) if (-1 == fcntl(fds_rw[1], F_SETPIPE_SZ, pipe_size * 2) && errno != EPERM) throwFromErrno("Cannot increase pipe capacity to " + std::to_string(pipe_size * 2), ErrorCodes::CANNOT_FCNTL); - LOG_TRACE_FORMATTED(log, "Pipe capacity is {}", formatReadableSizeWithBinarySuffix(std::min(pipe_size, desired_size))); + LOG_TRACE(log, "Pipe capacity is {}", formatReadableSizeWithBinarySuffix(std::min(pipe_size, desired_size))); } #else (void)desired_size; diff --git a/src/Common/PoolBase.h b/src/Common/PoolBase.h index 8c2ad8c9857..201f9ec1d00 100644 --- a/src/Common/PoolBase.h +++ b/src/Common/PoolBase.h @@ -122,7 +122,7 @@ public: return Entry(*items.back()); } - LOG_INFO_FORMATTED(log, "No free connections in pool. Waiting."); + LOG_INFO(log, "No free connections in pool. Waiting."); if (timeout < 0) available.wait(lock); diff --git a/src/Common/PoolWithFailoverBase.h b/src/Common/PoolWithFailoverBase.h index 2b9f7212a4f..e9f3d7d5d23 100644 --- a/src/Common/PoolWithFailoverBase.h +++ b/src/Common/PoolWithFailoverBase.h @@ -239,7 +239,7 @@ PoolWithFailoverBase::getMany( } else { - LOG_WARNING_FORMATTED(log, "Connection failed at try №{}, reason: {}", (shuffled_pool.error_count + 1), fail_message); + LOG_WARNING(log, "Connection failed at try №{}, reason: {}", (shuffled_pool.error_count + 1), fail_message); ProfileEvents::increment(ProfileEvents::DistributedConnectionFailTry); shuffled_pool.error_count = std::min(max_error_cap, shuffled_pool.error_count + 1); diff --git a/src/Common/QueryProfiler.cpp b/src/Common/QueryProfiler.cpp index b1306f2a936..cd68edd3ba1 100644 --- a/src/Common/QueryProfiler.cpp +++ b/src/Common/QueryProfiler.cpp @@ -165,10 +165,10 @@ void QueryProfilerBase::tryCleanup() { #if USE_UNWIND if (timer_id != nullptr && timer_delete(timer_id)) - LOG_ERROR_FORMATTED(log, "Failed to delete query profiler timer {}", errnoToString(ErrorCodes::CANNOT_DELETE_TIMER)); + LOG_ERROR(log, "Failed to delete query profiler timer {}", errnoToString(ErrorCodes::CANNOT_DELETE_TIMER)); if (previous_handler != nullptr && sigaction(pause_signal, previous_handler, nullptr)) - LOG_ERROR_FORMATTED(log, "Failed to restore signal handler after query profiler {}", errnoToString(ErrorCodes::CANNOT_SET_SIGNAL_HANDLER)); + LOG_ERROR(log, "Failed to restore signal handler after query profiler {}", errnoToString(ErrorCodes::CANNOT_SET_SIGNAL_HANDLER)); #endif } diff --git a/src/Common/SensitiveDataMasker.cpp b/src/Common/SensitiveDataMasker.cpp index 3fad8b4fc43..0d8e155d44a 100644 --- a/src/Common/SensitiveDataMasker.cpp +++ b/src/Common/SensitiveDataMasker.cpp @@ -144,13 +144,13 @@ SensitiveDataMasker::SensitiveDataMasker(const Poco::Util::AbstractConfiguration } else { - LOG_WARNING_FORMATTED(logger, "Unused param {}.{}", config_prefix, rule); + LOG_WARNING(logger, "Unused param {}.{}", config_prefix, rule); } } auto rules_count = rulesCount(); if (rules_count > 0) - LOG_INFO_FORMATTED(logger, "{} query masking rules loaded.", rules_count); + LOG_INFO(logger, "{} query masking rules loaded.", rules_count); } void SensitiveDataMasker::addMaskingRule( diff --git a/src/Common/ShellCommand.cpp b/src/Common/ShellCommand.cpp index d87372fbb8d..1b97ed5689c 100644 --- a/src/Common/ShellCommand.cpp +++ b/src/Common/ShellCommand.cpp @@ -50,10 +50,10 @@ ShellCommand::~ShellCommand() { if (terminate_in_destructor) { - LOG_TRACE_FORMATTED(getLogger(), "Will kill shell command pid {} with SIGTERM", pid); + LOG_TRACE(getLogger(), "Will kill shell command pid {} with SIGTERM", pid); int retcode = kill(pid, SIGTERM); if (retcode != 0) - LOG_WARNING_FORMATTED(getLogger(), "Cannot kill shell command pid {} errno '{}'", pid, errnoToString(retcode)); + LOG_WARNING(getLogger(), "Cannot kill shell command pid {} errno '{}'", pid, errnoToString(retcode)); } else if (!wait_called) tryWait(); @@ -70,7 +70,7 @@ void ShellCommand::logCommand(const char * filename, char * const argv[]) /// NOTE: No escaping is performed. args << "'" << argv[i] << "'"; } - LOG_TRACE_FORMATTED(ShellCommand::getLogger(), "Will start shell command '{}' with arguments {}", filename, args.str()); + LOG_TRACE(ShellCommand::getLogger(), "Will start shell command '{}' with arguments {}", filename, args.str()); } std::unique_ptr ShellCommand::executeImpl(const char * filename, char * const argv[], bool pipe_stdin_only, bool terminate_in_destructor) @@ -131,7 +131,7 @@ std::unique_ptr ShellCommand::executeImpl(const char * filename, c std::unique_ptr res(new ShellCommand(pid, pipe_stdin.fds_rw[1], pipe_stdout.fds_rw[0], pipe_stderr.fds_rw[0], terminate_in_destructor)); - LOG_TRACE_FORMATTED(getLogger(), "Started shell command '{}' with pid {}", filename, pid); + LOG_TRACE(getLogger(), "Started shell command '{}' with pid {}", filename, pid); /// Now the ownership of the file descriptors is passed to the result. pipe_stdin.fds_rw[1] = -1; @@ -185,13 +185,13 @@ int ShellCommand::tryWait() { wait_called = true; - LOG_TRACE_FORMATTED(getLogger(), "Will wait for shell command pid {}", pid); + LOG_TRACE(getLogger(), "Will wait for shell command pid {}", pid); int status = 0; if (-1 == waitpid(pid, &status, 0)) throwFromErrno("Cannot waitpid", ErrorCodes::CANNOT_WAITPID); - LOG_TRACE_FORMATTED(getLogger(), "Wait for shell command pid {} completed with status {}", pid, status); + LOG_TRACE(getLogger(), "Wait for shell command pid {} completed with status {}", pid, status); if (WIFEXITED(status)) return WEXITSTATUS(status); diff --git a/src/Common/StatusFile.cpp b/src/Common/StatusFile.cpp index a3a8cbda593..79e34d244bc 100644 --- a/src/Common/StatusFile.cpp +++ b/src/Common/StatusFile.cpp @@ -43,9 +43,9 @@ StatusFile::StatusFile(const std::string & path_) } if (!contents.empty()) - LOG_INFO_FORMATTED(&Logger::get("StatusFile"), "Status file {} already exists - unclean restart. Contents:\n{}", path, contents); + LOG_INFO(&Logger::get("StatusFile"), "Status file {} already exists - unclean restart. Contents:\n{}", path, contents); else - LOG_INFO_FORMATTED(&Logger::get("StatusFile"), "Status file {} already exists and is empty - probably unclean hardware restart.", path); + LOG_INFO(&Logger::get("StatusFile"), "Status file {} already exists and is empty - probably unclean hardware restart.", path); } fd = ::open(path.c_str(), O_WRONLY | O_CREAT | O_CLOEXEC, 0666); @@ -90,10 +90,10 @@ StatusFile::StatusFile(const std::string & path_) StatusFile::~StatusFile() { if (0 != close(fd)) - LOG_ERROR_FORMATTED(&Logger::get("StatusFile"), "Cannot close file {}, {}", path, errnoToString(ErrorCodes::CANNOT_CLOSE_FILE)); + LOG_ERROR(&Logger::get("StatusFile"), "Cannot close file {}, {}", path, errnoToString(ErrorCodes::CANNOT_CLOSE_FILE)); if (0 != unlink(path.c_str())) - LOG_ERROR_FORMATTED(&Logger::get("StatusFile"), "Cannot unlink file {}, {}", path, errnoToString(ErrorCodes::CANNOT_CLOSE_FILE)); + LOG_ERROR(&Logger::get("StatusFile"), "Cannot unlink file {}, {}", path, errnoToString(ErrorCodes::CANNOT_CLOSE_FILE)); } } diff --git a/src/Common/TraceCollector.cpp b/src/Common/TraceCollector.cpp index 5b489bd32a9..7df06dc7892 100644 --- a/src/Common/TraceCollector.cpp +++ b/src/Common/TraceCollector.cpp @@ -48,7 +48,7 @@ TraceCollector::TraceCollector(std::shared_ptr trace_log_) TraceCollector::~TraceCollector() { if (!thread.joinable()) - LOG_ERROR_FORMATTED(&Poco::Logger::get("TraceCollector"), "TraceCollector thread is malformed and cannot be joined"); + LOG_ERROR(&Poco::Logger::get("TraceCollector"), "TraceCollector thread is malformed and cannot be joined"); else stop(); diff --git a/src/Common/XDBCBridgeHelper.h b/src/Common/XDBCBridgeHelper.h index 05176ee6aab..1609737107e 100644 --- a/src/Common/XDBCBridgeHelper.h +++ b/src/Common/XDBCBridgeHelper.h @@ -148,7 +148,7 @@ public: { if (!checkBridgeIsRunning()) { - LOG_TRACE_FORMATTED(log, "{} is not running, will try to start it", BridgeHelperMixin::serviceAlias()); + LOG_TRACE(log, "{} is not running, will try to start it", BridgeHelperMixin::serviceAlias()); startBridge(); bool started = false; @@ -157,7 +157,7 @@ public: while (milliseconds_to_wait < 10000) { ++counter; - LOG_TRACE_FORMATTED(log, "Checking {} is running, try {}", BridgeHelperMixin::serviceAlias(), counter); + LOG_TRACE(log, "Checking {} is running, try {}", BridgeHelperMixin::serviceAlias(), counter); if (checkBridgeIsRunning()) { started = true; @@ -309,7 +309,7 @@ struct ODBCBridgeMixin cmd_args.push_back(config.getString("logger." + configPrefix() + "_level")); } - LOG_TRACE_FORMATTED(log, "Starting {}", serviceAlias()); + LOG_TRACE(log, "Starting {}", serviceAlias()); return ShellCommand::executeDirect(path.toString(), cmd_args, true); } diff --git a/src/Common/ZooKeeper/Increment.h b/src/Common/ZooKeeper/Increment.h index 4dc4aac78d2..26128480635 100644 --- a/src/Common/ZooKeeper/Increment.h +++ b/src/Common/ZooKeeper/Increment.h @@ -16,7 +16,7 @@ public: size_t get() { - LOG_TRACE_FORMATTED(log, "Get increment"); + LOG_TRACE(log, "Get increment"); size_t result = 0; std::string result_str; diff --git a/src/Common/ZooKeeper/Lock.cpp b/src/Common/ZooKeeper/Lock.cpp index ccea3c75dd7..c781d8ba2bf 100644 --- a/src/Common/ZooKeeper/Lock.cpp +++ b/src/Common/ZooKeeper/Lock.cpp @@ -64,7 +64,7 @@ Lock::Status Lock::tryCheck() const } if (locked && lock_status != LOCKED_BY_ME) - LOG_WARNING_FORMATTED(log, "Lock is lost. It is normal if session was expired. Path: {}/{}", lock_path, lock_message); + LOG_WARNING(log, "Lock is lost. It is normal if session was expired. Path: {}/{}", lock_path, lock_message); return lock_status; } diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index 6bb63f88030..6e0cc22a952 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -79,7 +79,7 @@ void ZooKeeper::init(const std::string & implementation_, const std::string & ho } catch (const Poco::Net::DNSException & e) { - LOG_ERROR_FORMATTED(log, "Cannot use ZooKeeper host {}, reason: {}", host_string, e.displayText()); + LOG_ERROR(log, "Cannot use ZooKeeper host {}, reason: {}", host_string, e.displayText()); } } @@ -96,9 +96,9 @@ void ZooKeeper::init(const std::string & implementation_, const std::string & ho Poco::Timespan(0, operation_timeout_ms_ * 1000)); if (chroot.empty()) - LOG_TRACE_FORMATTED(log, "Initialized, hosts: {}", hosts); + LOG_TRACE(log, "Initialized, hosts: {}", hosts); else - LOG_TRACE_FORMATTED(log, "Initialized, hosts: {}, chroot: {}", hosts, chroot); + LOG_TRACE(log, "Initialized, hosts: {}, chroot: {}", hosts, chroot); } else if (implementation == "testkeeper") { diff --git a/src/Common/ZooKeeper/ZooKeeperHolder.cpp b/src/Common/ZooKeeper/ZooKeeperHolder.cpp index d21bfa9f10d..41a36a51082 100644 --- a/src/Common/ZooKeeper/ZooKeeperHolder.cpp +++ b/src/Common/ZooKeeper/ZooKeeperHolder.cpp @@ -34,7 +34,7 @@ bool ZooKeeperHolder::replaceZooKeeperSessionToNewOne() } else { - LOG_ERROR_FORMATTED(log, "replaceZooKeeperSessionToNewOne(): Fail to replace zookeeper session to new one because handlers for old zookeeper session still exists."); + LOG_ERROR(log, "replaceZooKeeperSessionToNewOne(): Fail to replace zookeeper session to new one because handlers for old zookeeper session still exists."); return false; } } diff --git a/src/Core/BackgroundSchedulePool.cpp b/src/Core/BackgroundSchedulePool.cpp index 41fec3c0f1f..cc9ee23c9b8 100644 --- a/src/Core/BackgroundSchedulePool.cpp +++ b/src/Core/BackgroundSchedulePool.cpp @@ -111,7 +111,7 @@ void BackgroundSchedulePoolTaskInfo::execute() static const int32_t slow_execution_threshold_ms = 200; if (milliseconds >= slow_execution_threshold_ms) - LOG_TRACE_FORMATTED(&Logger::get(log_name), "Execution took {} ms.", milliseconds); + LOG_TRACE(&Logger::get(log_name), "Execution took {} ms.", milliseconds); { std::lock_guard lock_schedule(schedule_mutex); @@ -156,7 +156,7 @@ BackgroundSchedulePool::BackgroundSchedulePool(size_t size_, CurrentMetrics::Met , memory_metric(memory_metric_) , thread_name(thread_name_) { - LOG_INFO_FORMATTED(&Logger::get("BackgroundSchedulePool/" + thread_name), "Create BackgroundSchedulePool with {} threads", size); + LOG_INFO(&Logger::get("BackgroundSchedulePool/" + thread_name), "Create BackgroundSchedulePool with {} threads", size); threads.resize(size); for (auto & thread : threads) @@ -179,7 +179,7 @@ BackgroundSchedulePool::~BackgroundSchedulePool() queue.wakeUpAll(); delayed_thread.join(); - LOG_TRACE_FORMATTED(&Logger::get("BackgroundSchedulePool/" + thread_name), "Waiting for threads to finish."); + LOG_TRACE(&Logger::get("BackgroundSchedulePool/" + thread_name), "Waiting for threads to finish."); for (auto & thread : threads) thread.join(); } diff --git a/src/Core/MySQLProtocol.h b/src/Core/MySQLProtocol.h index fded91c116c..2fd15f78c05 100644 --- a/src/Core/MySQLProtocol.h +++ b/src/Core/MySQLProtocol.h @@ -1044,17 +1044,17 @@ public: AuthSwitchResponse response; packet_sender->receivePacket(response); auth_response = response.value; - LOG_TRACE_FORMATTED(log, "Authentication method mismatch."); + LOG_TRACE(log, "Authentication method mismatch."); } else { - LOG_TRACE_FORMATTED(log, "Authentication method match."); + LOG_TRACE(log, "Authentication method match."); } bool sent_public_key = false; if (auth_response == "\1") { - LOG_TRACE_FORMATTED(log, "Client requests public key."); + LOG_TRACE(log, "Client requests public key."); BIO * mem = BIO_new(BIO_s_mem()); SCOPE_EXIT(BIO_free(mem)); if (PEM_write_bio_RSA_PUBKEY(mem, &public_key) != 1) @@ -1068,7 +1068,7 @@ public: # pragma GCC diagnostic pop String pem(pem_buf, pem_size); - LOG_TRACE_FORMATTED(log, "Key: {}", pem); + LOG_TRACE(log, "Key: {}", pem); AuthMoreData data(pem); packet_sender->sendPacket(data, true); @@ -1080,7 +1080,7 @@ public: } else { - LOG_TRACE_FORMATTED(log, "Client didn't request public key."); + LOG_TRACE(log, "Client didn't request public key."); } String password; @@ -1092,7 +1092,7 @@ public: */ if (!is_secure_connection && !auth_response->empty() && auth_response != String("\0", 1)) { - LOG_TRACE_FORMATTED(log, "Received nonempty password."); + LOG_TRACE(log, "Received nonempty password."); auto ciphertext = reinterpret_cast(auth_response->data()); unsigned char plaintext[RSA_size(&private_key)]; @@ -1100,7 +1100,7 @@ public: if (plaintext_size == -1) { if (!sent_public_key) - LOG_WARNING_FORMATTED(log, "Client could have encrypted password with different public key since it didn't request it from server."); + LOG_WARNING(log, "Client could have encrypted password with different public key since it didn't request it from server."); throw Exception("Failed to decrypt auth data. Error: " + getOpenSSLErrors(), ErrorCodes::OPENSSL_ERROR); } @@ -1116,7 +1116,7 @@ public: } else { - LOG_TRACE_FORMATTED(log, "Received empty password"); + LOG_TRACE(log, "Received empty password"); } if (!password.empty() && password.back() == 0) diff --git a/src/Core/SettingsCollection.cpp b/src/Core/SettingsCollection.cpp index 6753ef9ab0a..3b291406df9 100644 --- a/src/Core/SettingsCollection.cpp +++ b/src/Core/SettingsCollection.cpp @@ -599,7 +599,7 @@ namespace details void SettingsCollectionUtils::warningNameNotFound(const StringRef & name) { static auto * log = &Logger::get("Settings"); - LOG_WARNING_FORMATTED(log, "Unknown setting {}, skipping", name); + LOG_WARNING(log, "Unknown setting {}, skipping", name); } void SettingsCollectionUtils::throwNameNotFound(const StringRef & name) diff --git a/src/DataStreams/AggregatingBlockInputStream.cpp b/src/DataStreams/AggregatingBlockInputStream.cpp index 34162deb3d3..7c891271468 100644 --- a/src/DataStreams/AggregatingBlockInputStream.cpp +++ b/src/DataStreams/AggregatingBlockInputStream.cpp @@ -60,7 +60,7 @@ Block AggregatingBlockInputStream::readImpl() input_streams.emplace_back(temporary_inputs.back()->block_in); } - LOG_TRACE_FORMATTED(log, "Will merge {} temporary files of size {} compressed, {} uncompressed.", files.files.size(), formatReadableSizeWithBinarySuffix(files.sum_size_compressed), formatReadableSizeWithBinarySuffix(files.sum_size_uncompressed)); + LOG_TRACE(log, "Will merge {} temporary files of size {} compressed, {} uncompressed.", files.files.size(), formatReadableSizeWithBinarySuffix(files.sum_size_compressed), formatReadableSizeWithBinarySuffix(files.sum_size_uncompressed)); impl = std::make_unique(input_streams, params, final, 1, 1); } diff --git a/src/DataStreams/CollapsingFinalBlockInputStream.cpp b/src/DataStreams/CollapsingFinalBlockInputStream.cpp index 27aaca37073..29b9a5d9ab6 100644 --- a/src/DataStreams/CollapsingFinalBlockInputStream.cpp +++ b/src/DataStreams/CollapsingFinalBlockInputStream.cpp @@ -18,12 +18,12 @@ CollapsingFinalBlockInputStream::~CollapsingFinalBlockInputStream() void CollapsingFinalBlockInputStream::reportBadCounts() { /// With inconsistent data, this is an unavoidable error that can not be easily fixed by admins. Therefore Warning. - LOG_WARNING_FORMATTED(log, "Incorrect data: number of rows with sign = 1 ({}) differs with number of rows with sign = -1 ({}) by more than one", count_positive, count_negative); + LOG_WARNING(log, "Incorrect data: number of rows with sign = 1 ({}) differs with number of rows with sign = -1 ({}) by more than one", count_positive, count_negative); } void CollapsingFinalBlockInputStream::reportBadSign(Int8 sign) { - LOG_ERROR_FORMATTED(log, "Invalid sign: {}", static_cast(sign)); + LOG_ERROR(log, "Invalid sign: {}", static_cast(sign)); } void CollapsingFinalBlockInputStream::fetchNextBlock(size_t input_index) @@ -133,7 +133,7 @@ Block CollapsingFinalBlockInputStream::readImpl() if (output_blocks.empty()) { if (blocks_fetched != blocks_output) - LOG_ERROR_FORMATTED(log, "Logical error: CollapsingFinalBlockInputStream has output {} blocks instead of {}", blocks_output, blocks_fetched); + LOG_ERROR(log, "Logical error: CollapsingFinalBlockInputStream has output {} blocks instead of {}", blocks_output, blocks_fetched); return Block(); } diff --git a/src/DataStreams/ColumnGathererStream.cpp b/src/DataStreams/ColumnGathererStream.cpp index f828ab92a47..b98ee96d26a 100644 --- a/src/DataStreams/ColumnGathererStream.cpp +++ b/src/DataStreams/ColumnGathererStream.cpp @@ -100,10 +100,10 @@ void ColumnGathererStream::readSuffixImpl() double seconds = profile_info.total_stopwatch.elapsedSeconds(); if (!seconds) - LOG_DEBUG_FORMATTED(log, "Gathered column {} ({} bytes/elem.) in 0 sec.", + LOG_DEBUG(log, "Gathered column {} ({} bytes/elem.) in 0 sec.", column_name, static_cast(profile_info.bytes) / profile_info.rows); else - LOG_DEBUG_FORMATTED(log, "Gathered column {} ({} bytes/elem.) in {} sec., {} rows/sec., {}/sec.", + LOG_DEBUG(log, "Gathered column {} ({} bytes/elem.) in {} sec., {} rows/sec., {}/sec.", column_name, static_cast(profile_info.bytes) / profile_info.rows, seconds, profile_info.rows / seconds, formatReadableSizeWithBinarySuffix(profile_info.bytes / seconds)); } diff --git a/src/DataStreams/CreatingSetsBlockInputStream.cpp b/src/DataStreams/CreatingSetsBlockInputStream.cpp index 5cd42296ea0..e40b5979b05 100644 --- a/src/DataStreams/CreatingSetsBlockInputStream.cpp +++ b/src/DataStreams/CreatingSetsBlockInputStream.cpp @@ -91,11 +91,11 @@ void CreatingSetsBlockInputStream::createAll() void CreatingSetsBlockInputStream::createOne(SubqueryForSet & subquery) { if (subquery.set) - LOG_TRACE_FORMATTED(log, "Creating set."); + LOG_TRACE(log, "Creating set."); if (subquery.join) - LOG_TRACE_FORMATTED(log, "Creating join."); + LOG_TRACE(log, "Creating join."); if (subquery.table) - LOG_TRACE_FORMATTED(log, "Filling temporary table."); + LOG_TRACE(log, "Filling temporary table."); Stopwatch watch; @@ -117,7 +117,7 @@ void CreatingSetsBlockInputStream::createOne(SubqueryForSet & subquery) { if (isCancelled()) { - LOG_DEBUG_FORMATTED(log, "Query was cancelled during set / join or temporary table creation."); + LOG_DEBUG(log, "Query was cancelled during set / join or temporary table creation."); return; } @@ -172,15 +172,15 @@ void CreatingSetsBlockInputStream::createOne(SubqueryForSet & subquery) auto seconds = watch.elapsedSeconds(); if (subquery.set) - LOG_DEBUG_FORMATTED(log, "Created Set with {} entries from {} rows in {} sec.", subquery.set->getTotalRowCount(), head_rows, seconds); + LOG_DEBUG(log, "Created Set with {} entries from {} rows in {} sec.", subquery.set->getTotalRowCount(), head_rows, seconds); if (subquery.join) - LOG_DEBUG_FORMATTED(log, "Created Join with {} entries from {} rows in {} sec.", subquery.join->getTotalRowCount(), head_rows, seconds); + LOG_DEBUG(log, "Created Join with {} entries from {} rows in {} sec.", subquery.join->getTotalRowCount(), head_rows, seconds); if (subquery.table) - LOG_DEBUG_FORMATTED(log, "Created Table with {} rows in {} sec.", head_rows, seconds); + LOG_DEBUG(log, "Created Table with {} rows in {} sec.", head_rows, seconds); } else { - LOG_DEBUG_FORMATTED(log, "Subquery has empty result."); + LOG_DEBUG(log, "Subquery has empty result."); } } diff --git a/src/DataStreams/MergeSortingBlockInputStream.cpp b/src/DataStreams/MergeSortingBlockInputStream.cpp index 7c633460680..587f21d2467 100644 --- a/src/DataStreams/MergeSortingBlockInputStream.cpp +++ b/src/DataStreams/MergeSortingBlockInputStream.cpp @@ -95,10 +95,10 @@ Block MergeSortingBlockInputStream::readImpl() const std::string & path = temporary_files.back()->path(); MergeSortingBlocksBlockInputStream block_in(blocks, description, max_merged_block_size, limit); - LOG_INFO_FORMATTED(log, "Sorting and writing part of data into temporary file {}", path); + LOG_INFO(log, "Sorting and writing part of data into temporary file {}", path); ProfileEvents::increment(ProfileEvents::ExternalSortWritePart); TemporaryFileStream::write(path, header_without_constants, block_in, &is_cancelled, codec); /// NOTE. Possibly limit disk usage. - LOG_INFO_FORMATTED(log, "Done writing part of data into temporary file {}", path); + LOG_INFO(log, "Done writing part of data into temporary file {}", path); blocks.clear(); sum_bytes_in_blocks = 0; @@ -118,7 +118,7 @@ Block MergeSortingBlockInputStream::readImpl() /// If there was temporary files. ProfileEvents::increment(ProfileEvents::ExternalSortMerge); - LOG_INFO_FORMATTED(log, "There are {} temporary sorted parts to merge.", temporary_files.size()); + LOG_INFO(log, "There are {} temporary sorted parts to merge.", temporary_files.size()); /// Create sorted streams to merge. for (const auto & file : temporary_files) @@ -246,7 +246,7 @@ Block MergeSortingBlocksBlockInputStream::mergeImpl(TSortingHeap & queue) void MergeSortingBlockInputStream::remerge() { - LOG_DEBUG_FORMATTED(log, "Re-merging intermediate ORDER BY data ({} blocks with {} rows) to save memory consumption", blocks.size(), sum_rows_in_blocks); + LOG_DEBUG(log, "Re-merging intermediate ORDER BY data ({} blocks with {} rows) to save memory consumption", blocks.size(), sum_rows_in_blocks); /// NOTE Maybe concat all blocks and partial sort will be faster than merge? MergeSortingBlocksBlockInputStream merger(blocks, description, max_merged_block_size, limit); @@ -264,7 +264,7 @@ void MergeSortingBlockInputStream::remerge() } merger.readSuffix(); - LOG_DEBUG_FORMATTED(log, "Memory usage is lowered from {} to {}", formatReadableSizeWithBinarySuffix(sum_bytes_in_blocks), formatReadableSizeWithBinarySuffix(new_sum_bytes_in_blocks)); + LOG_DEBUG(log, "Memory usage is lowered from {} to {}", formatReadableSizeWithBinarySuffix(sum_bytes_in_blocks), formatReadableSizeWithBinarySuffix(new_sum_bytes_in_blocks)); /// If the memory consumption was not lowered enough - we will not perform remerge anymore. 2 is a guess. if (new_sum_bytes_in_blocks * 2 > sum_bytes_in_blocks) diff --git a/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp b/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp index 382dbc37063..9f514b6701d 100644 --- a/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp +++ b/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp @@ -149,7 +149,7 @@ void MergingAggregatedMemoryEfficientBlockInputStream::cancel(bool kill) * (example: connection reset during distributed query execution) * - then don't care. */ - LOG_ERROR_FORMATTED(log, "Exception while cancelling {}", input.stream->getName()); + LOG_ERROR(log, "Exception while cancelling {}", input.stream->getName()); } } } @@ -292,12 +292,12 @@ void MergingAggregatedMemoryEfficientBlockInputStream::finalize() if (!started) return; - LOG_TRACE_FORMATTED(log, "Waiting for threads to finish"); + LOG_TRACE(log, "Waiting for threads to finish"); if (parallel_merge_data) parallel_merge_data->pool.wait(); - LOG_TRACE_FORMATTED(log, "Waited for threads to finish"); + LOG_TRACE(log, "Waited for threads to finish"); } @@ -555,7 +555,7 @@ MergingAggregatedMemoryEfficientBlockInputStream::BlocksToMerge MergingAggregate /// Not yet partitioned (splitted to buckets) block. Will partition it and place result to 'splitted_blocks'. if (input.block.info.bucket_num == -1 && input.block && input.splitted_blocks.empty()) { - LOG_TRACE_FORMATTED(&Logger::get("MergingAggregatedMemoryEfficient"), "Having block without bucket: will split."); + LOG_TRACE(&Logger::get("MergingAggregatedMemoryEfficient"), "Having block without bucket: will split."); input.splitted_blocks = aggregator.convertBlockToTwoLevel(input.block); input.block = Block(); diff --git a/src/DataStreams/MergingSortedBlockInputStream.cpp b/src/DataStreams/MergingSortedBlockInputStream.cpp index bba58fd19a3..877213294ec 100644 --- a/src/DataStreams/MergingSortedBlockInputStream.cpp +++ b/src/DataStreams/MergingSortedBlockInputStream.cpp @@ -264,9 +264,9 @@ void MergingSortedBlockInputStream::readSuffixImpl() double seconds = profile_info.total_stopwatch.elapsedSeconds(); if (!seconds) - LOG_DEBUG_FORMATTED(log, "Merge sorted {} blocks, {} rows in 0 sec.", profile_info.blocks, profile_info.rows); + LOG_DEBUG(log, "Merge sorted {} blocks, {} rows in 0 sec.", profile_info.blocks, profile_info.rows); else - LOG_DEBUG_FORMATTED(log, "Merge sorted {} blocks, {} rows in {} sec., {} rows/sec., {}/sec", + LOG_DEBUG(log, "Merge sorted {} blocks, {} rows in {} sec., {} rows/sec., {}/sec", profile_info.blocks, profile_info.rows, seconds, profile_info.rows / seconds, formatReadableSizeWithBinarySuffix(profile_info.bytes / seconds)); diff --git a/src/DataStreams/ParallelAggregatingBlockInputStream.cpp b/src/DataStreams/ParallelAggregatingBlockInputStream.cpp index 037f421c733..aeda8ea5be9 100644 --- a/src/DataStreams/ParallelAggregatingBlockInputStream.cpp +++ b/src/DataStreams/ParallelAggregatingBlockInputStream.cpp @@ -82,7 +82,7 @@ Block ParallelAggregatingBlockInputStream::readImpl() input_streams.emplace_back(temporary_inputs.back()->block_in); } - LOG_TRACE_FORMATTED(log, "Will merge {} temporary files of size {} compressed, {} uncompressed.", files.files.size(), formatReadableSizeWithBinarySuffix(files.sum_size_compressed), formatReadableSizeWithBinarySuffix(files.sum_size_uncompressed)); + LOG_TRACE(log, "Will merge {} temporary files of size {} compressed, {} uncompressed.", files.files.size(), formatReadableSizeWithBinarySuffix(files.sum_size_compressed), formatReadableSizeWithBinarySuffix(files.sum_size_uncompressed)); impl = std::make_unique( input_streams, params, final, temporary_data_merge_threads, temporary_data_merge_threads); @@ -155,7 +155,7 @@ void ParallelAggregatingBlockInputStream::execute() for (size_t i = 0; i < max_threads; ++i) threads_data.emplace_back(keys_size, aggregates_size); - LOG_TRACE_FORMATTED(log, "Aggregating"); + LOG_TRACE(log, "Aggregating"); Stopwatch watch; @@ -177,7 +177,7 @@ void ParallelAggregatingBlockInputStream::execute() for (size_t i = 0; i < max_threads; ++i) { size_t rows = many_data[i]->size(); - LOG_TRACE_FORMATTED(log, "Aggregated. {} to {} rows (from {}) in {} sec. ({} rows/sec., {}/sec.)", + LOG_TRACE(log, "Aggregated. {} to {} rows (from {}) in {} sec. ({} rows/sec., {}/sec.)", threads_data[i].src_rows, rows, formatReadableSizeWithBinarySuffix(threads_data[i].src_bytes), elapsed_seconds, threads_data[i].src_rows / elapsed_seconds, formatReadableSizeWithBinarySuffix(threads_data[i].src_bytes / elapsed_seconds)); @@ -185,7 +185,7 @@ void ParallelAggregatingBlockInputStream::execute() total_src_rows += threads_data[i].src_rows; total_src_bytes += threads_data[i].src_bytes; } - LOG_TRACE_FORMATTED(log, "Total aggregated. {} rows (from {}) in {} sec. ({} rows/sec., {}/sec.)", + LOG_TRACE(log, "Total aggregated. {} rows (from {}) in {} sec. ({} rows/sec., {}/sec.)", total_src_rows, formatReadableSizeWithBinarySuffix(total_src_bytes), elapsed_seconds, total_src_rows / elapsed_seconds, formatReadableSizeWithBinarySuffix(total_src_bytes / elapsed_seconds)); diff --git a/src/DataStreams/ParallelInputsProcessor.h b/src/DataStreams/ParallelInputsProcessor.h index 62d26fada2e..714174e6ac1 100644 --- a/src/DataStreams/ParallelInputsProcessor.h +++ b/src/DataStreams/ParallelInputsProcessor.h @@ -134,7 +134,7 @@ public: * (for example, the connection is broken for distributed query processing) * - then do not care. */ - LOG_ERROR_FORMATTED(log, "Exception while cancelling {}", input->getName()); + LOG_ERROR(log, "Exception while cancelling {}", input->getName()); } } } diff --git a/src/DataStreams/RemoteBlockInputStream.cpp b/src/DataStreams/RemoteBlockInputStream.cpp index 12ac5032395..d5e77bb0759 100644 --- a/src/DataStreams/RemoteBlockInputStream.cpp +++ b/src/DataStreams/RemoteBlockInputStream.cpp @@ -372,7 +372,7 @@ void RemoteBlockInputStream::tryCancel(const char * reason) multiplexed_connections->sendCancel(); } - LOG_TRACE_FORMATTED(log, "({}) {}", multiplexed_connections->dumpAddresses(), reason); + LOG_TRACE(log, "({}) {}", multiplexed_connections->dumpAddresses(), reason); } bool RemoteBlockInputStream::isQueryPending() const diff --git a/src/DataStreams/TTLBlockInputStream.cpp b/src/DataStreams/TTLBlockInputStream.cpp index ea970cd564d..d2d5d6a92f9 100644 --- a/src/DataStreams/TTLBlockInputStream.cpp +++ b/src/DataStreams/TTLBlockInputStream.cpp @@ -108,7 +108,7 @@ void TTLBlockInputStream::readSuffixImpl() data_part->expired_columns = std::move(empty_columns); if (rows_removed) - LOG_INFO_FORMATTED(log, "Removed {} rows with expired TTL from part {}", rows_removed, data_part->name); + LOG_INFO(log, "Removed {} rows with expired TTL from part {}", rows_removed, data_part->name); } void TTLBlockInputStream::removeRowsWithExpiredTableTTL(Block & block) diff --git a/src/DataStreams/UnionBlockInputStream.h b/src/DataStreams/UnionBlockInputStream.h index 091de967e09..c4e84e85845 100644 --- a/src/DataStreams/UnionBlockInputStream.h +++ b/src/DataStreams/UnionBlockInputStream.h @@ -107,7 +107,7 @@ protected: if (!started) return; - LOG_TRACE_FORMATTED(log, "Waiting for threads to finish"); + LOG_TRACE(log, "Waiting for threads to finish"); std::exception_ptr exception; if (!all_read) @@ -137,7 +137,7 @@ protected: processor.wait(); - LOG_TRACE_FORMATTED(log, "Waited for threads to finish"); + LOG_TRACE(log, "Waited for threads to finish"); if (exception) std::rethrow_exception(exception); diff --git a/src/Databases/DatabaseLazy.cpp b/src/Databases/DatabaseLazy.cpp index e740a4f2e36..51ee664a42a 100644 --- a/src/Databases/DatabaseLazy.cpp +++ b/src/Databases/DatabaseLazy.cpp @@ -152,7 +152,7 @@ bool DatabaseLazy::empty() const void DatabaseLazy::attachTable(const String & table_name, const StoragePtr & table, const String &) { - LOG_DEBUG_FORMATTED(log, "Attach table {}.", backQuote(table_name)); + LOG_DEBUG(log, "Attach table {}.", backQuote(table_name)); std::lock_guard lock(mutex); time_t current_time = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); @@ -169,7 +169,7 @@ StoragePtr DatabaseLazy::detachTable(const String & table_name) { StoragePtr res; { - LOG_DEBUG_FORMATTED(log, "Detach table {}.", backQuote(table_name)); + LOG_DEBUG(log, "Detach table {}.", backQuote(table_name)); std::lock_guard lock(mutex); auto it = tables_cache.find(table_name); if (it == tables_cache.end()) @@ -216,7 +216,7 @@ StoragePtr DatabaseLazy::loadTable(const String & table_name) const { SCOPE_EXIT({ clearExpiredTables(); }); - LOG_DEBUG_FORMATTED(log, "Load table {} to cache.", backQuote(table_name)); + LOG_DEBUG(log, "Load table {} to cache.", backQuote(table_name)); const String table_metadata_path = getMetadataPath() + "/" + escapeForFileName(table_name) + ".sql"; @@ -277,14 +277,14 @@ void DatabaseLazy::clearExpiredTables() const if (!it->second.table || it->second.table.unique()) { - LOG_DEBUG_FORMATTED(log, "Drop table {} from cache.", backQuote(it->first)); + LOG_DEBUG(log, "Drop table {} from cache.", backQuote(it->first)); it->second.table.reset(); expired_tables.erase(it->second.expiration_iterator); it->second.expiration_iterator = cache_expiration_queue.end(); } else { - LOG_DEBUG_FORMATTED(log, "Table {} is busy.", backQuote(it->first)); + LOG_DEBUG(log, "Table {} is busy.", backQuote(it->first)); busy_tables.splice(busy_tables.end(), expired_tables, it->second.expiration_iterator); } } diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index 9e606296624..70f86b19f1f 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -230,7 +230,7 @@ void DatabaseOnDisk::dropTable(const Context & context, const String & table_nam } catch (...) { - LOG_WARNING_FORMATTED(log, getCurrentExceptionMessage(__PRETTY_FUNCTION__)); + LOG_WARNING(log, getCurrentExceptionMessage(__PRETTY_FUNCTION__)); attachTable(table_name, table, table_data_path_relative); if (renamed) Poco::File(table_metadata_path_drop).renameTo(table_metadata_path); @@ -376,12 +376,12 @@ void DatabaseOnDisk::iterateMetadataFiles(const Context & context, const Iterati if (Poco::File(context.getPath() + getDataPath() + '/' + object_name).exists()) { Poco::File(getMetadataPath() + file_name).renameTo(getMetadataPath() + object_name + ".sql"); - LOG_WARNING_FORMATTED(log, "Object {} was not dropped previously and will be restored", backQuote(object_name)); + LOG_WARNING(log, "Object {} was not dropped previously and will be restored", backQuote(object_name)); process_metadata_file(object_name + ".sql"); } else { - LOG_INFO_FORMATTED(log, "Removing file {}", getMetadataPath() + file_name); + LOG_INFO(log, "Removing file {}", getMetadataPath() + file_name); Poco::File(getMetadataPath() + file_name).remove(); } }; @@ -406,7 +406,7 @@ void DatabaseOnDisk::iterateMetadataFiles(const Context & context, const Iterati else if (endsWith(dir_it.name(), ".sql.tmp")) { /// There are files .sql.tmp - delete - LOG_INFO_FORMATTED(log, "Removing file {}", dir_it->path()); + LOG_INFO(log, "Removing file {}", dir_it->path()); Poco::File(dir_it->path()).remove(); } else if (endsWith(dir_it.name(), ".sql")) @@ -442,7 +442,7 @@ ASTPtr DatabaseOnDisk::parseQueryFromMetadata(Poco::Logger * loger, const Contex */ if (remove_empty && query.empty()) { - LOG_ERROR_FORMATTED(loger, "File {} is empty. Removing.", metadata_file_path); + LOG_ERROR(loger, "File {} is empty. Removing.", metadata_file_path); Poco::File(metadata_file_path).remove(); return nullptr; } @@ -466,7 +466,7 @@ ASTPtr DatabaseOnDisk::parseQueryFromMetadata(Poco::Logger * loger, const Contex table_name = unescapeForFileName(table_name); if (create.table != TABLE_WITH_UUID_NAME_PLACEHOLDER) - LOG_WARNING_FORMATTED(loger, "File {} contains both UUID and table name. Will use name `{}` instead of `{}`", metadata_file_path, table_name, create.table); + LOG_WARNING(loger, "File {} contains both UUID and table name. Will use name `{}` instead of `{}`", metadata_file_path, table_name, create.table); create.table = table_name; } diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index 5c93d2cbfca..09f71f5af4a 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -95,7 +95,7 @@ namespace { if (processed % PRINT_MESSAGE_EACH_N_OBJECTS == 0 || watch.compareAndRestart(PRINT_MESSAGE_EACH_N_SECONDS)) { - LOG_INFO_FORMATTED(log, "{}%", processed * 100.0 / total); + LOG_INFO(log, "{}%", processed * 100.0 / total); watch.restart(); } } @@ -150,7 +150,7 @@ void DatabaseOrdinary::loadStoredObjects( size_t total_tables = file_names.size() - total_dictionaries; - LOG_INFO_FORMATTED(log, "Total {} tables and {} dictionaries.", total_tables, total_dictionaries); + LOG_INFO(log, "Total {} tables and {} dictionaries.", total_tables, total_dictionaries); AtomicStopwatch watch; std::atomic tables_processed{0}; @@ -194,7 +194,7 @@ void DatabaseOrdinary::loadStoredObjects( void DatabaseOrdinary::startupTables(ThreadPool & thread_pool) { - LOG_INFO_FORMATTED(log, "Starting up tables."); + LOG_INFO(log, "Starting up tables."); const size_t total_tables = tables.size(); if (!total_tables) diff --git a/src/Dictionaries/CacheDictionary.cpp b/src/Dictionaries/CacheDictionary.cpp index 7ac9e5401e5..fffeb796c56 100644 --- a/src/Dictionaries/CacheDictionary.cpp +++ b/src/Dictionaries/CacheDictionary.cpp @@ -768,7 +768,7 @@ void CacheDictionary::updateThreadFunction() const size_t current_queue_size = update_queue.size(); if (current_queue_size > 0) - LOG_TRACE_FORMATTED(log, "Performing bunch of keys update in cache dictionary with {} keys", current_queue_size + 1); + LOG_TRACE(log, "Performing bunch of keys update in cache dictionary with {} keys", current_queue_size + 1); std::vector update_request; update_request.reserve(current_queue_size + 1); diff --git a/src/Dictionaries/ClickHouseDictionarySource.cpp b/src/Dictionaries/ClickHouseDictionarySource.cpp index 99f294c1f3f..9d3f6063a21 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.cpp +++ b/src/Dictionaries/ClickHouseDictionarySource.cpp @@ -168,7 +168,7 @@ bool ClickHouseDictionarySource::isModified() const if (!invalidate_query.empty()) { auto response = doInvalidateQuery(invalidate_query); - LOG_TRACE_FORMATTED(log, "Invalidate query has returned: {}, previous value: {}", response, invalidate_query_response); + LOG_TRACE(log, "Invalidate query has returned: {}, previous value: {}", response, invalidate_query_response); if (invalidate_query_response == response) return false; invalidate_query_response = response; @@ -202,7 +202,7 @@ BlockInputStreamPtr ClickHouseDictionarySource::createStreamForSelectiveLoad(con std::string ClickHouseDictionarySource::doInvalidateQuery(const std::string & request) const { - LOG_TRACE_FORMATTED(log, "Performing invalidate query"); + LOG_TRACE(log, "Performing invalidate query"); if (is_local) { Context query_context = context; diff --git a/src/Dictionaries/DictionaryFactory.cpp b/src/Dictionaries/DictionaryFactory.cpp index 056a6861a33..03b4fe106c7 100644 --- a/src/Dictionaries/DictionaryFactory.cpp +++ b/src/Dictionaries/DictionaryFactory.cpp @@ -43,7 +43,7 @@ DictionaryPtr DictionaryFactory::create( const DictionaryStructure dict_struct{config, config_prefix + ".structure"}; DictionarySourcePtr source_ptr = DictionarySourceFactory::instance().create(name, config, config_prefix + ".source", dict_struct, context, check_source_config); - LOG_TRACE_FORMATTED(&Poco::Logger::get("DictionaryFactory"), "Created dictionary source '{}' for dictionary '{}'", source_ptr->toString(), name); + LOG_TRACE(&Poco::Logger::get("DictionaryFactory"), "Created dictionary source '{}' for dictionary '{}'", source_ptr->toString(), name); const auto & layout_type = keys.front(); diff --git a/src/Dictionaries/Embedded/RegionsHierarchies.cpp b/src/Dictionaries/Embedded/RegionsHierarchies.cpp index eaab1fac214..9fb51d72457 100644 --- a/src/Dictionaries/Embedded/RegionsHierarchies.cpp +++ b/src/Dictionaries/Embedded/RegionsHierarchies.cpp @@ -8,12 +8,12 @@ RegionsHierarchies::RegionsHierarchies(IRegionsHierarchiesDataProviderPtr data_p { Logger * log = &Logger::get("RegionsHierarchies"); - LOG_DEBUG_FORMATTED(log, "Adding default regions hierarchy"); + LOG_DEBUG(log, "Adding default regions hierarchy"); data.emplace("", data_provider->getDefaultHierarchySource()); for (const auto & name : data_provider->listCustomHierarchies()) { - LOG_DEBUG_FORMATTED(log, "Adding regions hierarchy for {}", name); + LOG_DEBUG(log, "Adding regions hierarchy for {}", name); data.emplace(name, data_provider->getHierarchySource(name)); } diff --git a/src/Dictionaries/Embedded/RegionsHierarchy.cpp b/src/Dictionaries/Embedded/RegionsHierarchy.cpp index 866d6b750fc..dac1e948e0a 100644 --- a/src/Dictionaries/Embedded/RegionsHierarchy.cpp +++ b/src/Dictionaries/Embedded/RegionsHierarchy.cpp @@ -28,7 +28,7 @@ void RegionsHierarchy::reload() if (!data_source->isModified()) return; - LOG_DEBUG_FORMATTED(log, "Reloading regions hierarchy"); + LOG_DEBUG(log, "Reloading regions hierarchy"); const size_t initial_size = 10000; const size_t max_size = 15000000; diff --git a/src/Dictionaries/Embedded/RegionsNames.cpp b/src/Dictionaries/Embedded/RegionsNames.cpp index 12362c7e50e..b31debdbc26 100644 --- a/src/Dictionaries/Embedded/RegionsNames.cpp +++ b/src/Dictionaries/Embedded/RegionsNames.cpp @@ -43,7 +43,7 @@ std::string RegionsNames::dumpSupportedLanguagesNames() void RegionsNames::reload() { Logger * log = &Logger::get("RegionsNames"); - LOG_DEBUG_FORMATTED(log, "Reloading regions names"); + LOG_DEBUG(log, "Reloading regions names"); RegionID max_region_id = 0; for (size_t language_id = 0; language_id < total_languages; ++language_id) @@ -55,7 +55,7 @@ void RegionsNames::reload() if (!names_source || !names_source->isModified()) continue; - LOG_DEBUG_FORMATTED(log, "Reloading regions names for language: {}", language); + LOG_DEBUG(log, "Reloading regions names for language: {}", language); auto names_reader = names_source->createReader(); diff --git a/src/Dictionaries/ExecutableDictionarySource.cpp b/src/Dictionaries/ExecutableDictionarySource.cpp index 56cdce58c90..0bb8c8543e8 100644 --- a/src/Dictionaries/ExecutableDictionarySource.cpp +++ b/src/Dictionaries/ExecutableDictionarySource.cpp @@ -77,7 +77,7 @@ ExecutableDictionarySource::ExecutableDictionarySource(const ExecutableDictionar BlockInputStreamPtr ExecutableDictionarySource::loadAll() { - LOG_TRACE_FORMATTED(log, "loadAll {}", toString()); + LOG_TRACE(log, "loadAll {}", toString()); auto process = ShellCommand::execute(command); auto input_stream = context.getInputFormat(format, process->out, sample_block, max_block_size); return std::make_shared(input_stream, std::move(process)); @@ -92,7 +92,7 @@ BlockInputStreamPtr ExecutableDictionarySource::loadUpdatedAll() if (update_time) command_with_update_field += " " + update_field + " " + DB::toString(LocalDateTime(update_time - 1)); - LOG_TRACE_FORMATTED(log, "loadUpdatedAll {}", command_with_update_field); + LOG_TRACE(log, "loadUpdatedAll {}", command_with_update_field); auto process = ShellCommand::execute(command_with_update_field); auto input_stream = context.getInputFormat(format, process->out, sample_block, max_block_size); return std::make_shared(input_stream, std::move(process)); @@ -163,7 +163,7 @@ namespace BlockInputStreamPtr ExecutableDictionarySource::loadIds(const std::vector & ids) { - LOG_TRACE_FORMATTED(log, "loadIds {} size = {}", toString(), ids.size()); + LOG_TRACE(log, "loadIds {} size = {}", toString(), ids.size()); auto process = ShellCommand::execute(command); auto output_stream = context.getOutputFormat(format, process->in, sample_block); @@ -175,7 +175,7 @@ BlockInputStreamPtr ExecutableDictionarySource::loadIds(const std::vector & requested_rows) { - LOG_TRACE_FORMATTED(log, "loadKeys {} size = {}", toString(), requested_rows.size()); + LOG_TRACE(log, "loadKeys {} size = {}", toString(), requested_rows.size()); auto process = ShellCommand::execute(command); auto output_stream = context.getOutputFormat(format, process->in, sample_block); diff --git a/src/Dictionaries/FileDictionarySource.cpp b/src/Dictionaries/FileDictionarySource.cpp index 81578ca3050..a581f77ed3a 100644 --- a/src/Dictionaries/FileDictionarySource.cpp +++ b/src/Dictionaries/FileDictionarySource.cpp @@ -49,7 +49,7 @@ FileDictionarySource::FileDictionarySource(const FileDictionarySource & other) BlockInputStreamPtr FileDictionarySource::loadAll() { - LOG_TRACE_FORMATTED(&Poco::Logger::get("FileDictionary"), "loadAll {}", toString()); + LOG_TRACE(&Poco::Logger::get("FileDictionary"), "loadAll {}", toString()); auto in_ptr = std::make_unique(filepath); auto stream = context.getInputFormat(format, *in_ptr, sample_block, max_block_size); last_modification = getLastModification(); diff --git a/src/Dictionaries/HTTPDictionarySource.cpp b/src/Dictionaries/HTTPDictionarySource.cpp index 722d25e0d95..7600253d5b8 100644 --- a/src/Dictionaries/HTTPDictionarySource.cpp +++ b/src/Dictionaries/HTTPDictionarySource.cpp @@ -105,7 +105,7 @@ void HTTPDictionarySource::getUpdateFieldAndDate(Poco::URI & uri) BlockInputStreamPtr HTTPDictionarySource::loadAll() { - LOG_TRACE_FORMATTED(log, "loadAll {}", toString()); + LOG_TRACE(log, "loadAll {}", toString()); Poco::URI uri(url); auto in_ptr = std::make_unique( uri, Poco::Net::HTTPRequest::HTTP_GET, ReadWriteBufferFromHTTP::OutStreamCallback(), timeouts, @@ -118,7 +118,7 @@ BlockInputStreamPtr HTTPDictionarySource::loadUpdatedAll() { Poco::URI uri(url); getUpdateFieldAndDate(uri); - LOG_TRACE_FORMATTED(log, "loadUpdatedAll {}", uri.toString()); + LOG_TRACE(log, "loadUpdatedAll {}", uri.toString()); auto in_ptr = std::make_unique( uri, Poco::Net::HTTPRequest::HTTP_GET, ReadWriteBufferFromHTTP::OutStreamCallback(), timeouts, 0, credentials, DBMS_DEFAULT_BUFFER_SIZE, header_entries); @@ -128,7 +128,7 @@ BlockInputStreamPtr HTTPDictionarySource::loadUpdatedAll() BlockInputStreamPtr HTTPDictionarySource::loadIds(const std::vector & ids) { - LOG_TRACE_FORMATTED(log, "loadIds {} size = {}", toString(), ids.size()); + LOG_TRACE(log, "loadIds {} size = {}", toString(), ids.size()); ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback = [&](std::ostream & ostr) { @@ -147,7 +147,7 @@ BlockInputStreamPtr HTTPDictionarySource::loadIds(const std::vector & id BlockInputStreamPtr HTTPDictionarySource::loadKeys(const Columns & key_columns, const std::vector & requested_rows) { - LOG_TRACE_FORMATTED(log, "loadKeys {} size = {}", toString(), requested_rows.size()); + LOG_TRACE(log, "loadKeys {} size = {}", toString(), requested_rows.size()); ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback = [&](std::ostream & ostr) { diff --git a/src/Dictionaries/LibraryDictionarySource.cpp b/src/Dictionaries/LibraryDictionarySource.cpp index 11b453ce745..83d081707bd 100644 --- a/src/Dictionaries/LibraryDictionarySource.cpp +++ b/src/Dictionaries/LibraryDictionarySource.cpp @@ -182,7 +182,7 @@ LibraryDictionarySource::~LibraryDictionarySource() BlockInputStreamPtr LibraryDictionarySource::loadAll() { - LOG_TRACE_FORMATTED(log, "loadAll {}", toString()); + LOG_TRACE(log, "loadAll {}", toString()); auto columns_holder = std::make_unique(dict_struct.attributes.size()); ClickHouseLibrary::CStrings columns{static_cast(columns_holder.get()), @@ -207,7 +207,7 @@ BlockInputStreamPtr LibraryDictionarySource::loadAll() BlockInputStreamPtr LibraryDictionarySource::loadIds(const std::vector & ids) { - LOG_TRACE_FORMATTED(log, "loadIds {} size = {}", toString(), ids.size()); + LOG_TRACE(log, "loadIds {} size = {}", toString(), ids.size()); const ClickHouseLibrary::VectorUInt64 ids_data{ext::bit_cast(ids.data()), ids.size()}; auto columns_holder = std::make_unique(dict_struct.attributes.size()); @@ -234,7 +234,7 @@ BlockInputStreamPtr LibraryDictionarySource::loadIds(const std::vector & BlockInputStreamPtr LibraryDictionarySource::loadKeys(const Columns & key_columns, const std::vector & requested_rows) { - LOG_TRACE_FORMATTED(log, "loadKeys {} size = {}", toString(), requested_rows.size()); + LOG_TRACE(log, "loadKeys {} size = {}", toString(), requested_rows.size()); auto holder = std::make_unique(key_columns.size()); std::vector> column_data_holders; diff --git a/src/Dictionaries/MySQLDictionarySource.cpp b/src/Dictionaries/MySQLDictionarySource.cpp index f8d035f2037..00f592460a7 100644 --- a/src/Dictionaries/MySQLDictionarySource.cpp +++ b/src/Dictionaries/MySQLDictionarySource.cpp @@ -118,7 +118,7 @@ BlockInputStreamPtr MySQLDictionarySource::loadAll() auto connection = pool.get(); last_modification = getLastModification(connection, false); - LOG_TRACE_FORMATTED(log, load_all_query); + LOG_TRACE(log, load_all_query); return std::make_shared(connection, load_all_query, sample_block, max_block_size, close_connection); } @@ -128,7 +128,7 @@ BlockInputStreamPtr MySQLDictionarySource::loadUpdatedAll() last_modification = getLastModification(connection, false); std::string load_update_query = getUpdateFieldAndDate(); - LOG_TRACE_FORMATTED(log, load_update_query); + LOG_TRACE(log, load_update_query); return std::make_shared(connection, load_update_query, sample_block, max_block_size, close_connection); } @@ -213,7 +213,7 @@ LocalDateTime MySQLDictionarySource::getLastModification(mysqlxx::Pool::Entry & { auto query = connection->query("SHOW TABLE STATUS LIKE " + quoteForLike(table)); - LOG_TRACE_FORMATTED(log, query.str()); + LOG_TRACE(log, query.str()); auto result = query.use(); @@ -227,7 +227,7 @@ LocalDateTime MySQLDictionarySource::getLastModification(mysqlxx::Pool::Entry & if (!update_time_value.isNull()) { modification_time = update_time_value.getDateTime(); - LOG_TRACE_FORMATTED(log, "Got modification time: {}", modification_time); + LOG_TRACE(log, "Got modification time: {}", modification_time); } /// fetch remaining rows to avoid "commands out of sync" error @@ -241,10 +241,10 @@ LocalDateTime MySQLDictionarySource::getLastModification(mysqlxx::Pool::Entry & } if (0 == fetched_rows) - LOG_ERROR_FORMATTED(log, "Cannot find table in SHOW TABLE STATUS result."); + LOG_ERROR(log, "Cannot find table in SHOW TABLE STATUS result."); if (fetched_rows > 1) - LOG_ERROR_FORMATTED(log, "Found more than one table in SHOW TABLE STATUS result."); + LOG_ERROR(log, "Found more than one table in SHOW TABLE STATUS result."); } catch (...) { diff --git a/src/Dictionaries/XDBCDictionarySource.cpp b/src/Dictionaries/XDBCDictionarySource.cpp index 27839678c6d..03d31fa001f 100644 --- a/src/Dictionaries/XDBCDictionarySource.cpp +++ b/src/Dictionaries/XDBCDictionarySource.cpp @@ -134,7 +134,7 @@ std::string XDBCDictionarySource::getUpdateFieldAndDate() BlockInputStreamPtr XDBCDictionarySource::loadAll() { - LOG_TRACE_FORMATTED(log, load_all_query); + LOG_TRACE(log, load_all_query); return loadBase(load_all_query); } @@ -142,7 +142,7 @@ BlockInputStreamPtr XDBCDictionarySource::loadUpdatedAll() { std::string load_query_update = getUpdateFieldAndDate(); - LOG_TRACE_FORMATTED(log, load_query_update); + LOG_TRACE(log, load_query_update); return loadBase(load_query_update); } diff --git a/src/Disks/DiskLocal.cpp b/src/Disks/DiskLocal.cpp index 31ea07482a7..cea900d5bad 100644 --- a/src/Disks/DiskLocal.cpp +++ b/src/Disks/DiskLocal.cpp @@ -91,7 +91,7 @@ bool DiskLocal::tryReserve(UInt64 bytes) std::lock_guard lock(DiskLocal::reservation_mutex); if (bytes == 0) { - LOG_DEBUG_FORMATTED(&Logger::get("DiskLocal"), "Reserving 0 bytes on disk {}", backQuote(name)); + LOG_DEBUG(&Logger::get("DiskLocal"), "Reserving 0 bytes on disk {}", backQuote(name)); ++reservation_count; return true; } @@ -100,7 +100,7 @@ bool DiskLocal::tryReserve(UInt64 bytes) UInt64 unreserved_space = available_space - std::min(available_space, reserved_bytes); if (unreserved_space >= bytes) { - LOG_DEBUG_FORMATTED(&Logger::get("DiskLocal"), "Reserving {} on disk {}, having unreserved {}.", + LOG_DEBUG(&Logger::get("DiskLocal"), "Reserving {} on disk {}, having unreserved {}.", formatReadableSizeWithBinarySuffix(bytes), backQuote(name), formatReadableSizeWithBinarySuffix(unreserved_space)); ++reservation_count; reserved_bytes += bytes; @@ -310,7 +310,7 @@ DiskLocalReservation::~DiskLocalReservation() if (disk->reserved_bytes < size) { disk->reserved_bytes = 0; - LOG_ERROR_FORMATTED(&Logger::get("DiskLocal"), "Unbalanced reservations size for disk '{}'.", disk->getName()); + LOG_ERROR(&Logger::get("DiskLocal"), "Unbalanced reservations size for disk '{}'.", disk->getName()); } else { @@ -318,7 +318,7 @@ DiskLocalReservation::~DiskLocalReservation() } if (disk->reservation_count == 0) - LOG_ERROR_FORMATTED(&Logger::get("DiskLocal"), "Unbalanced reservation count for disk '{}'.", disk->getName()); + LOG_ERROR(&Logger::get("DiskLocal"), "Unbalanced reservation count for disk '{}'.", disk->getName()); else --disk->reservation_count; } diff --git a/src/Disks/DiskSelector.cpp b/src/Disks/DiskSelector.cpp index fc74f8f789b..69549e4520d 100644 --- a/src/Disks/DiskSelector.cpp +++ b/src/Disks/DiskSelector.cpp @@ -99,7 +99,7 @@ DiskSelectorPtr DiskSelector::updateFromConfig( } writeString(" disappeared from configuration, this change will be applied after restart of ClickHouse", warning); - LOG_WARNING_FORMATTED(&Logger::get("DiskSelector"), warning.str()); + LOG_WARNING(&Logger::get("DiskSelector"), warning.str()); } return result; diff --git a/src/Disks/IDisk.cpp b/src/Disks/IDisk.cpp index aeb54deead8..837ddf1b6b2 100644 --- a/src/Disks/IDisk.cpp +++ b/src/Disks/IDisk.cpp @@ -15,7 +15,7 @@ bool IDisk::isDirectoryEmpty(const String & path) void copyFile(IDisk & from_disk, const String & from_path, IDisk & to_disk, const String & to_path) { - LOG_DEBUG_FORMATTED(&Poco::Logger::get("IDisk"), "Copying from {} {} to {} {}.", from_disk.getName(), from_path, to_disk.getName(), to_path); + LOG_DEBUG(&Poco::Logger::get("IDisk"), "Copying from {} {} to {} {}.", from_disk.getName(), from_path, to_disk.getName(), to_path); auto in = from_disk.readFile(from_path); auto out = to_disk.writeFile(to_path); diff --git a/src/Disks/S3/DiskS3.cpp b/src/Disks/S3/DiskS3.cpp index 7bf151fba5f..4b36deb7e98 100644 --- a/src/Disks/S3/DiskS3.cpp +++ b/src/Disks/S3/DiskS3.cpp @@ -515,7 +515,7 @@ std::unique_ptr DiskS3::readFile(const String & path, si { Metadata metadata(metadata_path, path); - LOG_DEBUG_FORMATTED(&Logger::get("DiskS3"), "Read from file by path: {}. Existing S3 objects: {}", + LOG_DEBUG(&Logger::get("DiskS3"), "Read from file by path: {}. Existing S3 objects: {}", backQuote(metadata_path + path), metadata.s3_objects.size()); return std::make_unique(client, bucket, metadata, buf_size); @@ -536,7 +536,7 @@ std::unique_ptr DiskS3::writeFile(const String & path, /// Save empty metadata to disk to have ability to get file size while buffer is not finalized. metadata.save(); - LOG_DEBUG_FORMATTED(&Logger::get("DiskS3"), "Write to file by path: {} New S3 path: {}", backQuote(metadata_path + path), s3_path); + LOG_DEBUG(&Logger::get("DiskS3"), "Write to file by path: {} New S3 path: {}", backQuote(metadata_path + path), s3_path); return std::make_unique(client, bucket, metadata, s3_path, min_upload_part_size, buf_size); } @@ -544,7 +544,7 @@ std::unique_ptr DiskS3::writeFile(const String & path, { Metadata metadata(metadata_path, path); - LOG_DEBUG_FORMATTED(&Logger::get("DiskS3"), "Append to file by path: {}. New S3 path: {}. Existing S3 objects: {}.", + LOG_DEBUG(&Logger::get("DiskS3"), "Append to file by path: {}. New S3 path: {}. Existing S3 objects: {}.", backQuote(metadata_path + path), s3_path, metadata.s3_objects.size()); return std::make_unique(client, bucket, metadata, s3_path, min_upload_part_size, buf_size); @@ -553,7 +553,7 @@ std::unique_ptr DiskS3::writeFile(const String & path, void DiskS3::remove(const String & path) { - LOG_DEBUG_FORMATTED(&Logger::get("DiskS3"), "Remove file by path: {}", backQuote(metadata_path + path)); + LOG_DEBUG(&Logger::get("DiskS3"), "Remove file by path: {}", backQuote(metadata_path + path)); Poco::File file(metadata_path + path); if (file.isFile()) @@ -607,7 +607,7 @@ bool DiskS3::tryReserve(UInt64 bytes) std::lock_guard lock(reservation_mutex); if (bytes == 0) { - LOG_DEBUG_FORMATTED(&Logger::get("DiskS3"), "Reserving 0 bytes on s3 disk {}", backQuote(name)); + LOG_DEBUG(&Logger::get("DiskS3"), "Reserving 0 bytes on s3 disk {}", backQuote(name)); ++reservation_count; return true; } @@ -616,7 +616,7 @@ bool DiskS3::tryReserve(UInt64 bytes) UInt64 unreserved_space = available_space - std::min(available_space, reserved_bytes); if (unreserved_space >= bytes) { - LOG_DEBUG_FORMATTED(&Logger::get("DiskS3"), "Reserving {} on disk {}, having unreserved {}.", + LOG_DEBUG(&Logger::get("DiskS3"), "Reserving {} on disk {}, having unreserved {}.", formatReadableSizeWithBinarySuffix(bytes), backQuote(name), formatReadableSizeWithBinarySuffix(unreserved_space)); ++reservation_count; reserved_bytes += bytes; @@ -672,7 +672,7 @@ DiskS3Reservation::~DiskS3Reservation() if (disk->reserved_bytes < size) { disk->reserved_bytes = 0; - LOG_ERROR_FORMATTED(&Logger::get("DiskLocal"), "Unbalanced reservations size for disk '{}'.", disk->getName()); + LOG_ERROR(&Logger::get("DiskLocal"), "Unbalanced reservations size for disk '{}'.", disk->getName()); } else { @@ -680,7 +680,7 @@ DiskS3Reservation::~DiskS3Reservation() } if (disk->reservation_count == 0) - LOG_ERROR_FORMATTED(&Logger::get("DiskLocal"), "Unbalanced reservation count for disk '{}'.", disk->getName()); + LOG_ERROR(&Logger::get("DiskLocal"), "Unbalanced reservation count for disk '{}'.", disk->getName()); else --disk->reservation_count; } diff --git a/src/Disks/S3/ProxyListConfiguration.cpp b/src/Disks/S3/ProxyListConfiguration.cpp index 3842fc727fd..430b6412408 100644 --- a/src/Disks/S3/ProxyListConfiguration.cpp +++ b/src/Disks/S3/ProxyListConfiguration.cpp @@ -20,7 +20,7 @@ Aws::Client::ClientConfigurationPerRequest ProxyListConfiguration::getConfigurat cfg.proxyHost = proxies[index].getHost(); cfg.proxyPort = proxies[index].getPort(); - LOG_DEBUG_FORMATTED(&Logger::get("AWSClient"), "Use proxy: {}", proxies[index].toString()); + LOG_DEBUG(&Logger::get("AWSClient"), "Use proxy: {}", proxies[index].toString()); return cfg; } diff --git a/src/Disks/S3/ProxyResolverConfiguration.cpp b/src/Disks/S3/ProxyResolverConfiguration.cpp index f5acdab369c..c0cbe4ac5bf 100644 --- a/src/Disks/S3/ProxyResolverConfiguration.cpp +++ b/src/Disks/S3/ProxyResolverConfiguration.cpp @@ -21,7 +21,7 @@ ProxyResolverConfiguration::ProxyResolverConfiguration(const Poco::URI & endpoin Aws::Client::ClientConfigurationPerRequest ProxyResolverConfiguration::getConfiguration(const Aws::Http::HttpRequest &) { - LOG_DEBUG_FORMATTED(&Logger::get("AWSClient"), "Obtain proxy using resolver: {}", endpoint.toString()); + LOG_DEBUG(&Logger::get("AWSClient"), "Obtain proxy using resolver: {}", endpoint.toString()); /// 1 second is enough for now. /// TODO: Make timeouts configurable. @@ -49,7 +49,7 @@ Aws::Client::ClientConfigurationPerRequest ProxyResolverConfiguration::getConfig /// Read proxy host as string from response body. Poco::StreamCopier::copyToString(response_body_stream, proxy_host); - LOG_DEBUG_FORMATTED(&Logger::get("AWSClient"), "Use proxy: {}://{}:{}", proxy_scheme, proxy_host, proxy_port); + LOG_DEBUG(&Logger::get("AWSClient"), "Use proxy: {}://{}:{}", proxy_scheme, proxy_host, proxy_port); cfg.proxyScheme = Aws::Http::SchemeMapper::FromString(proxy_scheme.c_str()); cfg.proxyHost = proxy_host; diff --git a/src/Disks/S3/registerDiskS3.cpp b/src/Disks/S3/registerDiskS3.cpp index cf347b75931..53292bb30bc 100644 --- a/src/Disks/S3/registerDiskS3.cpp +++ b/src/Disks/S3/registerDiskS3.cpp @@ -46,7 +46,7 @@ namespace throw Exception("Only HTTP/HTTPS schemas allowed in proxy resolver config: " + proxy_scheme, ErrorCodes::BAD_ARGUMENTS); auto proxy_port = proxy_resolver_config.getUInt(prefix + ".proxy_port"); - LOG_DEBUG_FORMATTED(&Logger::get("DiskS3"), "Configured proxy resolver: {}, Scheme: {}, Port: {}", endpoint.toString(), proxy_scheme, proxy_port); + LOG_DEBUG(&Logger::get("DiskS3"), "Configured proxy resolver: {}, Scheme: {}, Port: {}", endpoint.toString(), proxy_scheme, proxy_port); return std::make_shared(endpoint, proxy_scheme, proxy_port); } @@ -70,7 +70,7 @@ namespace proxies.push_back(proxy_uri); - LOG_DEBUG_FORMATTED(&Logger::get("DiskS3"), "Configured proxy: {}", proxy_uri.toString()); + LOG_DEBUG(&Logger::get("DiskS3"), "Configured proxy: {}", proxy_uri.toString()); } if (!proxies.empty()) diff --git a/src/Disks/StoragePolicy.cpp b/src/Disks/StoragePolicy.cpp index c11b5b79581..3f01ed38e1c 100644 --- a/src/Disks/StoragePolicy.cpp +++ b/src/Disks/StoragePolicy.cpp @@ -245,7 +245,7 @@ StoragePolicySelector::StoragePolicySelector( "StoragePolicy name can contain only alphanumeric and '_' (" + name + ")", ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG); policies.emplace(name, std::make_shared(name, config, config_prefix + "." + name, disks)); - LOG_INFO_FORMATTED(&Logger::get("StoragePolicySelector"), "Storage policy {} loaded", backQuote(name)); + LOG_INFO(&Logger::get("StoragePolicySelector"), "Storage policy {} loaded", backQuote(name)); } constexpr auto default_storage_policy_name = "default"; diff --git a/src/Disks/VolumeJBOD.cpp b/src/Disks/VolumeJBOD.cpp index c48a9250c71..b9eb8038cac 100644 --- a/src/Disks/VolumeJBOD.cpp +++ b/src/Disks/VolumeJBOD.cpp @@ -48,11 +48,11 @@ VolumeJBOD::VolumeJBOD( max_data_part_size = static_cast(sum_size * ratio / disks.size()); for (size_t i = 0; i < disks.size(); ++i) if (sizes[i] < max_data_part_size) - LOG_WARNING_FORMATTED(logger, "Disk {} on volume {} have not enough space ({}) for containing part the size of max_data_part_size ({})", backQuote(disks[i]->getName()), backQuote(config_prefix), formatReadableSizeWithBinarySuffix(sizes[i]), formatReadableSizeWithBinarySuffix(max_data_part_size)); + LOG_WARNING(logger, "Disk {} on volume {} have not enough space ({}) for containing part the size of max_data_part_size ({})", backQuote(disks[i]->getName()), backQuote(config_prefix), formatReadableSizeWithBinarySuffix(sizes[i]), formatReadableSizeWithBinarySuffix(max_data_part_size)); } static constexpr UInt64 MIN_PART_SIZE = 8u * 1024u * 1024u; if (max_data_part_size != 0 && max_data_part_size < MIN_PART_SIZE) - LOG_WARNING_FORMATTED(logger, "Volume {} max_data_part_size is too low ({} < {})", backQuote(name), formatReadableSizeWithBinarySuffix(max_data_part_size), formatReadableSizeWithBinarySuffix(MIN_PART_SIZE)); + LOG_WARNING(logger, "Volume {} max_data_part_size is too low ({} < {})", backQuote(name), formatReadableSizeWithBinarySuffix(max_data_part_size), formatReadableSizeWithBinarySuffix(MIN_PART_SIZE)); } DiskPtr VolumeJBOD::getNextDisk() diff --git a/src/IO/AIOContextPool.cpp b/src/IO/AIOContextPool.cpp index 43ffea00512..a06d76e86a2 100644 --- a/src/IO/AIOContextPool.cpp +++ b/src/IO/AIOContextPool.cpp @@ -98,7 +98,7 @@ void AIOContextPool::fulfillPromises(const io_event events[], const int num_even const auto it = promises.find(completed_id); if (it == std::end(promises)) { - LOG_ERROR_FORMATTED(&Poco::Logger::get("AIOcontextPool"), "Found io_event with unknown id {}", completed_id); + LOG_ERROR(&Poco::Logger::get("AIOcontextPool"), "Found io_event with unknown id {}", completed_id); continue; } diff --git a/src/IO/HTTPCommon.cpp b/src/IO/HTTPCommon.cpp index 70ba323808a..14c97ee56f4 100644 --- a/src/IO/HTTPCommon.cpp +++ b/src/IO/HTTPCommon.cpp @@ -173,7 +173,7 @@ namespace auto msg = Poco::AnyCast(session_data); if (!msg.empty()) { - LOG_TRACE_FORMATTED((&Logger::get("HTTPCommon")), "Failed communicating with {} with error '{}' will try to reconnect session", host, msg); + LOG_TRACE((&Logger::get("HTTPCommon")), "Failed communicating with {} with error '{}' will try to reconnect session", host, msg); /// Host can change IP const auto ip = DNSResolver::instance().resolveHost(host).toString(); if (ip != session->getHost()) diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp index c751716f42b..472607d0e21 100644 --- a/src/IO/ReadBufferFromS3.cpp +++ b/src/IO/ReadBufferFromS3.cpp @@ -66,7 +66,7 @@ off_t ReadBufferFromS3::getPosition() std::unique_ptr ReadBufferFromS3::initialize() { - LOG_TRACE_FORMATTED(log, "Read S3 object. Bucket: {}, Key: {}, Offset: {}", bucket, key, std::to_string(offset)); + LOG_TRACE(log, "Read S3 object. Bucket: {}, Key: {}, Offset: {}", bucket, key, std::to_string(offset)); Aws::S3::Model::GetObjectRequest req; req.SetBucket(bucket); diff --git a/src/IO/ReadWriteBufferFromHTTP.h b/src/IO/ReadWriteBufferFromHTTP.h index 51402a1f062..962c663a687 100644 --- a/src/IO/ReadWriteBufferFromHTTP.h +++ b/src/IO/ReadWriteBufferFromHTTP.h @@ -127,7 +127,7 @@ namespace detail if (!credentials.getUsername().empty()) credentials.authenticate(request); - LOG_TRACE_FORMATTED((&Logger::get("ReadWriteBufferFromHTTP")), "Sending request to {}", uri.toString()); + LOG_TRACE((&Logger::get("ReadWriteBufferFromHTTP")), "Sending request to {}", uri.toString()); auto sess = session->getSession(); diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp index 49fa1dc0c7a..5079109f1c3 100644 --- a/src/IO/S3Common.cpp +++ b/src/IO/S3Common.cpp @@ -40,13 +40,13 @@ public: void Log(Aws::Utils::Logging::LogLevel log_level, const char * tag, const char * format_str, ...) final // NOLINT { const auto & [level, prio] = convertLogLevel(log_level); - LOG_IMPL_FORMATTED(log, level, prio, "{}: {}", tag, format_str); + LOG_IMPL(log, level, prio, "{}: {}", tag, format_str); } void LogStream(Aws::Utils::Logging::LogLevel log_level, const char * tag, const Aws::OStringStream & message_stream) final { const auto & [level, prio] = convertLogLevel(log_level); - LOG_IMPL_FORMATTED(log, level, prio, "{}: {}", tag, message_stream.str()); + LOG_IMPL(log, level, prio, "{}: {}", tag, message_stream.str()); } void Flush() final {} diff --git a/src/IO/WriteBufferFromHTTP.cpp b/src/IO/WriteBufferFromHTTP.cpp index 79add0d90c0..573ac82a1d3 100644 --- a/src/IO/WriteBufferFromHTTP.cpp +++ b/src/IO/WriteBufferFromHTTP.cpp @@ -15,7 +15,7 @@ WriteBufferFromHTTP::WriteBufferFromHTTP( request.setHost(uri.getHost()); request.setChunkedTransferEncoding(true); - LOG_TRACE_FORMATTED((&Logger::get("WriteBufferToHTTP")), "Sending request to {}", uri.toString()); + LOG_TRACE((&Logger::get("WriteBufferToHTTP")), "Sending request to {}", uri.toString()); ostr = &session->sendRequest(request); } diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 593c3451d47..946871d9c63 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -100,7 +100,7 @@ void WriteBufferFromS3::initiate() if (outcome.IsSuccess()) { upload_id = outcome.GetResult().GetUploadId(); - LOG_DEBUG_FORMATTED(log, "Multipart upload initiated. Upload id: {}", upload_id); + LOG_DEBUG(log, "Multipart upload initiated. Upload id: {}", upload_id); } else throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR); @@ -115,7 +115,7 @@ void WriteBufferFromS3::writePart(const String & data) if (part_tags.size() == S3_WARN_MAX_PARTS) { // Don't throw exception here by ourselves but leave the decision to take by S3 server. - LOG_WARNING_FORMATTED(log, "Maximum part number in S3 protocol has reached (too many parts). Server may not accept this whole upload."); + LOG_WARNING(log, "Maximum part number in S3 protocol has reached (too many parts). Server may not accept this whole upload."); } Aws::S3::Model::UploadPartRequest req; @@ -129,13 +129,13 @@ void WriteBufferFromS3::writePart(const String & data) auto outcome = client_ptr->UploadPart(req); - LOG_TRACE_FORMATTED(log, "Writing part. Bucket: {}, Key: {}, Upload_id: {}, Data size: {}", bucket, key, upload_id, data.size()); + LOG_TRACE(log, "Writing part. Bucket: {}, Key: {}, Upload_id: {}, Data size: {}", bucket, key, upload_id, data.size()); if (outcome.IsSuccess()) { auto etag = outcome.GetResult().GetETag(); part_tags.push_back(etag); - LOG_DEBUG_FORMATTED(log, "Writing part finished. Total parts: {}, Upload_id: {}, Etag: {}", part_tags.size(), upload_id, etag); + LOG_DEBUG(log, "Writing part finished. Total parts: {}, Upload_id: {}, Etag: {}", part_tags.size(), upload_id, etag); } else throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR); @@ -144,7 +144,7 @@ void WriteBufferFromS3::writePart(const String & data) void WriteBufferFromS3::complete() { - LOG_DEBUG_FORMATTED(log, "Completing multipart upload. Bucket: {}, Key: {}, Upload_id: {}", bucket, key, upload_id); + LOG_DEBUG(log, "Completing multipart upload. Bucket: {}, Key: {}, Upload_id: {}", bucket, key, upload_id); Aws::S3::Model::CompleteMultipartUploadRequest req; req.SetBucket(bucket); @@ -166,7 +166,7 @@ void WriteBufferFromS3::complete() auto outcome = client_ptr->CompleteMultipartUpload(req); if (outcome.IsSuccess()) - LOG_DEBUG_FORMATTED(log, "Multipart upload completed. Upload_id: {}", upload_id); + LOG_DEBUG(log, "Multipart upload completed. Upload_id: {}", upload_id); else throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR); } diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index b96d9c0638d..98cb1018178 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -76,7 +76,7 @@ AggregatedDataVariants::~AggregatedDataVariants() void AggregatedDataVariants::convertToTwoLevel() { if (aggregator) - LOG_TRACE_FORMATTED(aggregator->log, "Converting aggregation data to two-level."); + LOG_TRACE(aggregator->log, "Converting aggregation data to two-level."); switch (type) { @@ -552,7 +552,7 @@ bool Aggregator::executeOnBlock(Columns columns, UInt64 num_rows, AggregatedData result.init(method_chosen); result.keys_size = params.keys_size; result.key_sizes = key_sizes; - LOG_TRACE_FORMATTED(log, "Aggregation method: {}", result.getMethodName()); + LOG_TRACE(log, "Aggregation method: {}", result.getMethodName()); } if (isCancelled()) @@ -723,7 +723,7 @@ void Aggregator::writeToTemporaryFile(AggregatedDataVariants & data_variants, co CompressedWriteBuffer compressed_buf(file_buf); NativeBlockOutputStream block_out(compressed_buf, ClickHouseRevision::get(), getHeader(false)); - LOG_DEBUG_FORMATTED(log, "Writing part of aggregation data into temporary file {}.", path); + LOG_DEBUG(log, "Writing part of aggregation data into temporary file {}.", path); ProfileEvents::increment(ProfileEvents::ExternalAggregationWritePart); /// Flush only two-level data and possibly overflow data. @@ -762,7 +762,7 @@ void Aggregator::writeToTemporaryFile(AggregatedDataVariants & data_variants, co ProfileEvents::increment(ProfileEvents::ExternalAggregationCompressedBytes, compressed_bytes); ProfileEvents::increment(ProfileEvents::ExternalAggregationUncompressedBytes, uncompressed_bytes); - LOG_TRACE_FORMATTED(log, + LOG_TRACE(log, "Written part in {} sec., {} rows, {} uncompressed, {} compressed," " {} uncompressed bytes per row, {} compressed bytes per row, compression rate: {}" " ({} rows/sec., {}/sec. uncompressed, {}/sec. compressed)", @@ -871,7 +871,7 @@ void Aggregator::writeToTemporaryFileImpl( /// `data_variants` will not destroy them in the destructor, they are now owned by ColumnAggregateFunction objects. data_variants.aggregator = nullptr; - LOG_TRACE_FORMATTED(log, "Max size of temporary block: {} rows, {}.", max_temporary_block_size_rows, formatReadableSizeWithBinarySuffix(max_temporary_block_size_bytes)); + LOG_TRACE(log, "Max size of temporary block: {} rows, {}.", max_temporary_block_size_rows, formatReadableSizeWithBinarySuffix(max_temporary_block_size_bytes)); } @@ -914,7 +914,7 @@ void Aggregator::execute(const BlockInputStreamPtr & stream, AggregatedDataVaria */ bool no_more_keys = false; - LOG_TRACE_FORMATTED(log, "Aggregating"); + LOG_TRACE(log, "Aggregating"); Stopwatch watch; @@ -942,7 +942,7 @@ void Aggregator::execute(const BlockInputStreamPtr & stream, AggregatedDataVaria double elapsed_seconds = watch.elapsedSeconds(); size_t rows = result.sizeWithoutOverflowRow(); - LOG_TRACE_FORMATTED(log, "Aggregated. {} to {} rows (from {}) in {} sec. ({} rows/sec., {}/sec.)", + LOG_TRACE(log, "Aggregated. {} to {} rows (from {}) in {} sec. ({} rows/sec., {}/sec.)", src_rows, rows, formatReadableSizeWithBinarySuffix(src_bytes), elapsed_seconds, src_rows / elapsed_seconds, formatReadableSizeWithBinarySuffix(src_bytes / elapsed_seconds)); @@ -1258,7 +1258,7 @@ BlocksList Aggregator::convertToBlocks(AggregatedDataVariants & data_variants, b if (isCancelled()) return BlocksList(); - LOG_TRACE_FORMATTED(log, "Converting aggregated data to blocks"); + LOG_TRACE(log, "Converting aggregated data to blocks"); Stopwatch watch; @@ -1311,7 +1311,7 @@ BlocksList Aggregator::convertToBlocks(AggregatedDataVariants & data_variants, b } double elapsed_seconds = watch.elapsedSeconds(); - LOG_TRACE_FORMATTED(log, + LOG_TRACE(log, "Converted aggregated data to blocks. {} rows, {} in {} sec. ({} rows/sec., {}/sec.)", rows, formatReadableSizeWithBinarySuffix(bytes), elapsed_seconds, rows / elapsed_seconds, @@ -1565,7 +1565,7 @@ public: ~MergingAndConvertingBlockInputStream() override { - LOG_TRACE_FORMATTED(&Logger::get(__PRETTY_FUNCTION__), "Waiting for threads to finish"); + LOG_TRACE(&Logger::get(__PRETTY_FUNCTION__), "Waiting for threads to finish"); /// We need to wait for threads to finish before destructor of 'parallel_merge_data', /// because the threads access 'parallel_merge_data'. @@ -1740,7 +1740,7 @@ ManyAggregatedDataVariants Aggregator::prepareVariantsToMerge(ManyAggregatedData if (data_variants.empty()) throw Exception("Empty data passed to Aggregator::mergeAndConvertToBlocks.", ErrorCodes::EMPTY_DATA_PASSED); - LOG_TRACE_FORMATTED(log, "Merging aggregated data"); + LOG_TRACE(log, "Merging aggregated data"); ManyAggregatedDataVariants non_empty_data; non_empty_data.reserve(data_variants.size()); @@ -1932,7 +1932,7 @@ void Aggregator::mergeStream(const BlockInputStreamPtr & stream, AggregatedDataV BucketToBlocks bucket_to_blocks; /// Read all the data. - LOG_TRACE_FORMATTED(log, "Reading blocks of partially aggregated data."); + LOG_TRACE(log, "Reading blocks of partially aggregated data."); size_t total_input_rows = 0; size_t total_input_blocks = 0; @@ -1946,7 +1946,7 @@ void Aggregator::mergeStream(const BlockInputStreamPtr & stream, AggregatedDataV bucket_to_blocks[block.info.bucket_num].emplace_back(std::move(block)); } - LOG_TRACE_FORMATTED(log, "Read {} blocks of partially aggregated data, total {} rows.", total_input_blocks, total_input_rows); + LOG_TRACE(log, "Read {} blocks of partially aggregated data, total {} rows.", total_input_blocks, total_input_rows); mergeBlocks(bucket_to_blocks, result, max_threads); } @@ -1999,7 +1999,7 @@ void Aggregator::mergeBlocks(BucketToBlocks bucket_to_blocks, AggregatedDataVari * That is, the keys in the end can be significantly larger than max_rows_to_group_by. */ - LOG_TRACE_FORMATTED(log, "Merging partially aggregated two-level data."); + LOG_TRACE(log, "Merging partially aggregated two-level data."); auto merge_bucket = [&bucket_to_blocks, &result, this](Int32 bucket, Arena * aggregates_pool, ThreadGroupStatusPtr thread_group) { @@ -2048,7 +2048,7 @@ void Aggregator::mergeBlocks(BucketToBlocks bucket_to_blocks, AggregatedDataVari if (thread_pool) thread_pool->wait(); - LOG_TRACE_FORMATTED(log, "Merged partially aggregated two-level data."); + LOG_TRACE(log, "Merged partially aggregated two-level data."); } if (isCancelled()) @@ -2059,7 +2059,7 @@ void Aggregator::mergeBlocks(BucketToBlocks bucket_to_blocks, AggregatedDataVari if (has_blocks_with_unknown_bucket) { - LOG_TRACE_FORMATTED(log, "Merging partially aggregated single-level data."); + LOG_TRACE(log, "Merging partially aggregated single-level data."); bool no_more_keys = false; @@ -2088,7 +2088,7 @@ void Aggregator::mergeBlocks(BucketToBlocks bucket_to_blocks, AggregatedDataVari throw Exception("Unknown aggregated data variant.", ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT); } - LOG_TRACE_FORMATTED(log, "Merged partially aggregated single-level data."); + LOG_TRACE(log, "Merged partially aggregated single-level data."); } } @@ -2101,7 +2101,7 @@ Block Aggregator::mergeBlocks(BlocksList & blocks, bool final) auto bucket_num = blocks.front().info.bucket_num; bool is_overflows = blocks.front().info.is_overflows; - LOG_TRACE_FORMATTED(log, "Merging partially aggregated blocks (bucket = {}).", bucket_num); + LOG_TRACE(log, "Merging partially aggregated blocks (bucket = {}).", bucket_num); Stopwatch watch; /** If possible, change 'method' to some_hash64. Otherwise, leave as is. @@ -2177,7 +2177,7 @@ Block Aggregator::mergeBlocks(BlocksList & blocks, bool final) size_t rows = block.rows(); size_t bytes = block.bytes(); double elapsed_seconds = watch.elapsedSeconds(); - LOG_TRACE_FORMATTED(log, "Merged partially aggregated blocks. {} rows, {}. in {} sec. ({} rows/sec., {}/sec.)", + LOG_TRACE(log, "Merged partially aggregated blocks. {} rows, {}. in {} sec. ({} rows/sec., {}/sec.)", rows, formatReadableSizeWithBinarySuffix(bytes), elapsed_seconds, rows / elapsed_seconds, formatReadableSizeWithBinarySuffix(bytes / elapsed_seconds)); @@ -2348,7 +2348,7 @@ void Aggregator::destroyAllAggregateStates(AggregatedDataVariants & result) if (result.empty()) return; - LOG_TRACE_FORMATTED(log, "Destroying aggregate states"); + LOG_TRACE(log, "Destroying aggregate states"); /// In what data structure is the data aggregated? if (result.type == AggregatedDataVariants::Type::without_key || params.overflow_row) diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index 7fe6bb260c5..53c61bea60e 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -200,7 +200,7 @@ void SelectStreamFactory::createForShard( ProfileEvents::increment(ProfileEvents::DistributedConnectionMissingTable); if (shard_info.hasRemoteConnections()) { - LOG_WARNING_FORMATTED(&Logger::get("ClusterProxy::SelectStreamFactory"), "There is no table {} on local replica of shard {}, will try remote replicas.", main_table.getNameForLogs(), shard_info.shard_num); + LOG_WARNING(&Logger::get("ClusterProxy::SelectStreamFactory"), "There is no table {} on local replica of shard {}, will try remote replicas.", main_table.getNameForLogs(), shard_info.shard_num); emplace_remote_stream(); } else @@ -236,7 +236,7 @@ void SelectStreamFactory::createForShard( /// If we reached this point, local replica is stale. ProfileEvents::increment(ProfileEvents::DistributedConnectionStaleReplica); - LOG_WARNING_FORMATTED(&Logger::get("ClusterProxy::SelectStreamFactory"), "Local replica of shard {} is stale (delay: {}s.)", shard_info.shard_num, local_delay); + LOG_WARNING(&Logger::get("ClusterProxy::SelectStreamFactory"), "Local replica of shard {} is stale (delay: {}s.)", shard_info.shard_num, local_delay); if (!settings.fallback_to_stale_replicas_for_distributed_queries) { @@ -284,7 +284,7 @@ void SelectStreamFactory::createForShard( catch (const Exception & ex) { if (ex.code() == ErrorCodes::ALL_CONNECTION_TRIES_FAILED) - LOG_WARNING_FORMATTED(&Logger::get("ClusterProxy::SelectStreamFactory"), "Connections to remote replicas of local shard {} failed, will use stale local replica", shard_num); + LOG_WARNING(&Logger::get("ClusterProxy::SelectStreamFactory"), "Connections to remote replicas of local shard {} failed, will use stale local replica", shard_num); else throw; } diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index aa983196d02..4605c606594 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1761,7 +1761,7 @@ void Context::updateStorageConfiguration(const Poco::Util::AbstractConfiguration } catch (Exception & e) { - LOG_ERROR_FORMATTED(shared->log, "An error has occured while reloading storage policies, storage policies were not applied: {}", e.message()); + LOG_ERROR(shared->log, "An error has occured while reloading storage policies, storage policies were not applied: {}", e.message()); } } } diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index 8836343b6be..fefeeeed988 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -238,7 +238,7 @@ DDLWorker::DDLWorker(const std::string & zk_root_dir, Context & context_, const if (context.getSettingsRef().readonly) { - LOG_WARNING_FORMATTED(log, "Distributed DDL worker is run with readonly settings, it will not be able to execute DDL queries Set appropriate system_profile or distributed_ddl.profile to fix this."); + LOG_WARNING(log, "Distributed DDL worker is run with readonly settings, it will not be able to execute DDL queries Set appropriate system_profile or distributed_ddl.profile to fix this."); } host_fqdn = getFQDNOrHostName(); @@ -334,7 +334,7 @@ bool DDLWorker::initAndCheckTask(const String & entry_name, String & out_reason, if (host_in_hostlist) { /// This check could be slow a little bit - LOG_WARNING_FORMATTED(log, "There are two the same ClickHouse instances in task {}: {} and {}. Will use the first one only.", entry_name, task->host_id.readableString(), host.readableString()); + LOG_WARNING(log, "There are two the same ClickHouse instances in task {}: {} and {}. Will use the first one only.", entry_name, task->host_id.readableString(), host.readableString()); } else { @@ -362,7 +362,7 @@ static void filterAndSortQueueNodes(Strings & all_nodes) void DDLWorker::processTasks() { - LOG_DEBUG_FORMATTED(log, "Processing tasks"); + LOG_DEBUG(log, "Processing tasks"); auto zookeeper = tryGetZooKeeper(); Strings queue_nodes = zookeeper->getChildren(queue_dir, nullptr, queue_updated_event); @@ -384,11 +384,11 @@ void DDLWorker::processTasks() { if (current_task->entry_name == entry_name) { - LOG_INFO_FORMATTED(log, "Trying to process task {} again", entry_name); + LOG_INFO(log, "Trying to process task {} again", entry_name); } else { - LOG_INFO_FORMATTED(log, "Task {} was deleted from ZooKeeper before current host committed it", current_task->entry_name); + LOG_INFO(log, "Task {} was deleted from ZooKeeper before current host committed it", current_task->entry_name); current_task = nullptr; } } @@ -398,7 +398,7 @@ void DDLWorker::processTasks() String reason; if (!initAndCheckTask(entry_name, reason, zookeeper)) { - LOG_DEBUG_FORMATTED(log, "Will not execute task {}: {}", entry_name, reason); + LOG_DEBUG(log, "Will not execute task {}: {}", entry_name, reason); last_processed_task_name = entry_name; continue; } @@ -424,7 +424,7 @@ void DDLWorker::processTasks() { if (server_startup && e.code == Coordination::ZNONODE) { - LOG_WARNING_FORMATTED(log, "ZooKeeper NONODE error during startup. Ignoring entry {} ({}) : {}", task.entry_name, task.entry.query, getCurrentExceptionMessage(true)); + LOG_WARNING(log, "ZooKeeper NONODE error during startup. Ignoring entry {} ({}) : {}", task.entry_name, task.entry.query, getCurrentExceptionMessage(true)); } else { @@ -433,13 +433,13 @@ void DDLWorker::processTasks() } catch (...) { - LOG_WARNING_FORMATTED(log, "An error occurred while processing task {} ({}) : {}", task.entry_name, task.entry.query, getCurrentExceptionMessage(true)); + LOG_WARNING(log, "An error occurred while processing task {} ({}) : {}", task.entry_name, task.entry.query, getCurrentExceptionMessage(true)); throw; } } else { - LOG_DEBUG_FORMATTED(log, "Task {} ({}) has been already processed", task.entry_name, task.entry.query); + LOG_DEBUG(log, "Task {} ({}) has been already processed", task.entry_name, task.entry.query); } last_processed_task_name = task.entry_name; @@ -506,7 +506,7 @@ void DDLWorker::parseQueryAndResolveHost(DDLTask & task) if (found_exact_match) return; - LOG_WARNING_FORMATTED(log, "Not found the exact match of host {} from task {} in cluster {} definition. Will try to find it using host name resolving.", task.host_id.readableString(), task.entry_name, task.cluster_name); + LOG_WARNING(log, "Not found the exact match of host {} from task {} in cluster {} definition. Will try to find it using host name resolving.", task.host_id.readableString(), task.entry_name, task.cluster_name); bool found_via_resolving = false; for (size_t shard_num = 0; shard_num < shards.size(); ++shard_num) @@ -542,7 +542,7 @@ void DDLWorker::parseQueryAndResolveHost(DDLTask & task) } else { - LOG_INFO_FORMATTED(log, "Resolved host {} from task {} as host {} in definition of cluster {}", task.host_id.readableString(), task.entry_name, task.address_in_cluster.readableString(), task.cluster_name); + LOG_INFO(log, "Resolved host {} from task {} as host {} in definition of cluster {}", task.host_id.readableString(), task.entry_name, task.address_in_cluster.readableString(), task.cluster_name); } } @@ -573,7 +573,7 @@ bool DDLWorker::tryExecuteQuery(const String & query, const DDLTask & task, Exec } status = ExecutionStatus(0); - LOG_DEBUG_FORMATTED(log, "Executed query: {}", query); + LOG_DEBUG(log, "Executed query: {}", query); return true; } @@ -595,7 +595,7 @@ void DDLWorker::attachToThreadGroup() void DDLWorker::processTask(DDLTask & task, const ZooKeeperPtr & zookeeper) { - LOG_DEBUG_FORMATTED(log, "Processing task {} ({})", task.entry_name, task.entry.query); + LOG_DEBUG(log, "Processing task {} ({})", task.entry_name, task.entry.query); String dummy; String active_node_path = task.entry_path + "/active/" + task.host_id_str; @@ -625,7 +625,7 @@ void DDLWorker::processTask(DDLTask & task, const ZooKeeperPtr & zookeeper) ASTPtr rewritten_ast = task.query_on_cluster->getRewrittenASTWithoutOnCluster(task.address_in_cluster.default_database); String rewritten_query = queryToString(rewritten_ast); - LOG_DEBUG_FORMATTED(log, "Executing query: {}", rewritten_query); + LOG_DEBUG(log, "Executing query: {}", rewritten_query); if (auto * query_with_table = dynamic_cast(rewritten_ast.get()); query_with_table) { @@ -695,7 +695,7 @@ void DDLWorker::checkShardConfig(const String & table, const DDLTask & task, Sto if (dynamic_cast(storage.get())) { - LOG_TRACE_FORMATTED(log, "Table {} is distributed, skip checking config.", backQuote(table)); + LOG_TRACE(log, "Table {} is distributed, skip checking config.", backQuote(table)); return; } @@ -755,7 +755,7 @@ bool DDLWorker::tryExecuteQueryOnLeaderReplica( String executed_by; if (zookeeper->tryGet(is_executed_path, executed_by)) { - LOG_DEBUG_FORMATTED(log, "Task {} has already been executed by leader replica ({}) of the same shard.", task.entry_name, executed_by); + LOG_DEBUG(log, "Task {} has already been executed by leader replica ({}) of the same shard.", task.entry_name, executed_by); return true; } @@ -816,7 +816,7 @@ bool DDLWorker::tryExecuteQueryOnLeaderReplica( void DDLWorker::cleanupQueue(Int64 current_time_seconds, const ZooKeeperPtr & zookeeper) { - LOG_DEBUG_FORMATTED(log, "Cleaning queue"); + LOG_DEBUG(log, "Cleaning queue"); Strings queue_nodes = zookeeper->getChildren(queue_dir); filterAndSortQueueNodes(queue_nodes); @@ -856,7 +856,7 @@ void DDLWorker::cleanupQueue(Int64 current_time_seconds, const ZooKeeperPtr & zo /// Skip if there are active nodes (it is weak guard) if (zookeeper->exists(node_path + "/active", &stat) && stat.numChildren > 0) { - LOG_INFO_FORMATTED(log, "Task {} should be deleted, but there are active workers. Skipping it.", node_name); + LOG_INFO(log, "Task {} should be deleted, but there are active workers. Skipping it.", node_name); continue; } @@ -865,14 +865,14 @@ void DDLWorker::cleanupQueue(Int64 current_time_seconds, const ZooKeeperPtr & zo auto lock = createSimpleZooKeeperLock(zookeeper, node_path, "lock", host_fqdn_id); if (!lock->tryLock()) { - LOG_INFO_FORMATTED(log, "Task {} should be deleted, but it is locked. Skipping it.", node_name); + LOG_INFO(log, "Task {} should be deleted, but it is locked. Skipping it.", node_name); continue; } if (node_lifetime_is_expired) - LOG_INFO_FORMATTED(log, "Lifetime of task {} is expired, deleting it", node_name); + LOG_INFO(log, "Lifetime of task {} is expired, deleting it", node_name); else if (node_is_outside_max_window) - LOG_INFO_FORMATTED(log, "Task {} is outdated, deleting it", node_name); + LOG_INFO(log, "Task {} is outdated, deleting it", node_name); /// Deleting { @@ -894,7 +894,7 @@ void DDLWorker::cleanupQueue(Int64 current_time_seconds, const ZooKeeperPtr & zo } catch (...) { - LOG_INFO_FORMATTED(log, "An error occured while checking and cleaning task {} from queue: {}", node_name, getCurrentExceptionMessage(false)); + LOG_INFO(log, "An error occured while checking and cleaning task {} from queue: {}", node_name, getCurrentExceptionMessage(false)); } } } @@ -940,7 +940,7 @@ String DDLWorker::enqueueQuery(DDLLogEntry & entry) } catch (...) { - LOG_INFO_FORMATTED(log, "An error occurred while creating auxiliary ZooKeeper directories in {} . They will be created later. Error : {}", node_path, getCurrentExceptionMessage(true)); + LOG_INFO(log, "An error occurred while creating auxiliary ZooKeeper directories in {} . They will be created later. Error : {}", node_path, getCurrentExceptionMessage(true)); } return node_path; @@ -950,7 +950,7 @@ String DDLWorker::enqueueQuery(DDLLogEntry & entry) void DDLWorker::runMainThread() { setThreadName("DDLWorker"); - LOG_DEBUG_FORMATTED(log, "Started DDLWorker thread"); + LOG_DEBUG(log, "Started DDLWorker thread"); bool initialized = false; do @@ -988,14 +988,14 @@ void DDLWorker::runMainThread() cleanup_event->set(); processTasks(); - LOG_DEBUG_FORMATTED(log, "Waiting a watch"); + LOG_DEBUG(log, "Waiting a watch"); queue_updated_event->wait(); } catch (const Coordination::Exception & e) { if (Coordination::isHardwareError(e.code)) { - LOG_DEBUG_FORMATTED(log, "Recovering ZooKeeper session after: {}", getCurrentExceptionMessage(false)); + LOG_DEBUG(log, "Recovering ZooKeeper session after: {}", getCurrentExceptionMessage(false)); while (!stop_flag) { @@ -1015,11 +1015,11 @@ void DDLWorker::runMainThread() } else if (e.code == Coordination::ZNONODE) { - LOG_ERROR_FORMATTED(log, "ZooKeeper error: {}", getCurrentExceptionMessage(true)); + LOG_ERROR(log, "ZooKeeper error: {}", getCurrentExceptionMessage(true)); } else { - LOG_ERROR_FORMATTED(log, "Unexpected ZooKeeper error: {}. Terminating.", getCurrentExceptionMessage(true)); + LOG_ERROR(log, "Unexpected ZooKeeper error: {}. Terminating.", getCurrentExceptionMessage(true)); return; } } @@ -1035,7 +1035,7 @@ void DDLWorker::runMainThread() void DDLWorker::runCleanupThread() { setThreadName("DDLWorkerClnr"); - LOG_DEBUG_FORMATTED(log, "Started DDLWorker cleanup thread"); + LOG_DEBUG(log, "Started DDLWorker cleanup thread"); Int64 last_cleanup_time_seconds = 0; while (!stop_flag) @@ -1049,7 +1049,7 @@ void DDLWorker::runCleanupThread() Int64 current_time_seconds = Poco::Timestamp().epochTime(); if (last_cleanup_time_seconds && current_time_seconds < last_cleanup_time_seconds + cleanup_delay_period) { - LOG_TRACE_FORMATTED(log, "Too early to clean queue, will do it later."); + LOG_TRACE(log, "Too early to clean queue, will do it later."); continue; } @@ -1217,7 +1217,7 @@ private: if (!ignoring_hosts.count(host)) { ignoring_hosts.emplace(host); - LOG_INFO_FORMATTED(log, "Unexpected host {} appeared in task {}", host, node_path); + LOG_INFO(log, "Unexpected host {} appeared in task {}", host, node_path); } continue; } diff --git a/src/Interpreters/DNSCacheUpdater.cpp b/src/Interpreters/DNSCacheUpdater.cpp index 98957555c24..e5a97dc76d9 100644 --- a/src/Interpreters/DNSCacheUpdater.cpp +++ b/src/Interpreters/DNSCacheUpdater.cpp @@ -21,7 +21,7 @@ void DNSCacheUpdater::run() /// Reload cluster config if IP of any host has been changed since last update. if (resolver.updateCache()) { - LOG_INFO_FORMATTED(&Poco::Logger::get("DNSCacheUpdater"), "IPs of some hosts have been changed. Will reload cluster config."); + LOG_INFO(&Poco::Logger::get("DNSCacheUpdater"), "IPs of some hosts have been changed. Will reload cluster config."); try { context.reloadClusterConfig(); diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 6bcccdff436..7606fdb255a 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -572,7 +572,7 @@ void DatabaseCatalog::enqueueDroppedTableCleanup(StorageID table_id, StoragePtr else { /// Try load table from metadata to drop it correctly (e.g. remove metadata from zk or remove data from all volumes) - LOG_INFO_FORMATTED(log, "Trying load partially dropped table {} from {}", table_id.getNameForLogs(), dropped_metadata_path); + LOG_INFO(log, "Trying load partially dropped table {} from {}", table_id.getNameForLogs(), dropped_metadata_path); ASTPtr ast = DatabaseOnDisk::parseQueryFromMetadata(log, *global_context, dropped_metadata_path, /*throw_on_error*/ false, /*remove_empty*/false); auto * create = typeid_cast(ast.get()); assert(!create || create->uuid == table_id.uuid); @@ -597,7 +597,7 @@ void DatabaseCatalog::enqueueDroppedTableCleanup(StorageID table_id, StoragePtr } else { - LOG_WARNING_FORMATTED(log, "Cannot parse metadata of partially dropped table {} from {}. Will remove metadata file and data directory. Garbage may be left in /store directory and ZooKeeper.", table_id.getNameForLogs(), dropped_metadata_path); + LOG_WARNING(log, "Cannot parse metadata of partially dropped table {} from {}. Will remove metadata file and data directory. Garbage may be left in /store directory and ZooKeeper.", table_id.getNameForLogs(), dropped_metadata_path); } drop_time = Poco::File(dropped_metadata_path).getLastModified().epochTime(); @@ -633,7 +633,7 @@ void DatabaseCatalog::dropTableDataTask() if (it != tables_marked_dropped.end()) { table = std::move(*it); - LOG_INFO_FORMATTED(log, "Will try drop {}", table.table_id.getNameForLogs()); + LOG_INFO(log, "Will try drop {}", table.table_id.getNameForLogs()); tables_marked_dropped.erase(it); } need_reschedule = !tables_marked_dropped.empty(); @@ -683,11 +683,11 @@ void DatabaseCatalog::dropTableFinally(const TableMarkedAsDropped & table) const Poco::File table_data_dir{data_path}; if (table_data_dir.exists()) { - LOG_INFO_FORMATTED(log, "Removing data directory {} of dropped table {}", data_path, table.table_id.getNameForLogs()); + LOG_INFO(log, "Removing data directory {} of dropped table {}", data_path, table.table_id.getNameForLogs()); table_data_dir.remove(true); } - LOG_INFO_FORMATTED(log, "Removing metadata {} of dropped table {}", table.metadata_path, table.table_id.getNameForLogs()); + LOG_INFO(log, "Removing metadata {} of dropped table {}", table.metadata_path, table.table_id.getNameForLogs()); Poco::File(table.metadata_path).remove(); } diff --git a/src/Interpreters/EmbeddedDictionaries.cpp b/src/Interpreters/EmbeddedDictionaries.cpp index 70d1e7144f4..9ab3cf2dcbe 100644 --- a/src/Interpreters/EmbeddedDictionaries.cpp +++ b/src/Interpreters/EmbeddedDictionaries.cpp @@ -68,7 +68,7 @@ bool EmbeddedDictionaries::reloadImpl(const bool throw_on_error, const bool forc * An attempt is made to load each directory separately. */ - LOG_INFO_FORMATTED(log, "Loading dictionaries."); + LOG_INFO(log, "Loading dictionaries."); bool was_exception = false; @@ -89,7 +89,7 @@ bool EmbeddedDictionaries::reloadImpl(const bool throw_on_error, const bool forc was_exception = true; if (!was_exception) - LOG_INFO_FORMATTED(log, "Loaded dictionaries."); + LOG_INFO(log, "Loaded dictionaries."); return !was_exception; } diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index 048b4b91f4b..fc6505e8446 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -250,7 +250,7 @@ private: { if (path.empty() || !repository.exists(path)) { - LOG_WARNING_FORMATTED(log, "Config file '{}' does not exist", path); + LOG_WARNING(log, "Config file '{}' does not exist", path); return false; } @@ -271,7 +271,7 @@ private: return false; } - LOG_TRACE_FORMATTED(log, "Loading config file '{}'.", path); + LOG_TRACE(log, "Loading config file '{}'.", path); file_info.file_contents = repository.load(path); auto & file_contents = *file_info.file_contents; @@ -286,14 +286,14 @@ private: if (!startsWith(key, settings.external_config)) { if (!startsWith(key, "comment") && !startsWith(key, "include_from")) - LOG_WARNING_FORMATTED(log, "{}: file contains unknown node '{}', expected '{}'", path, key, settings.external_config); + LOG_WARNING(log, "{}: file contains unknown node '{}', expected '{}'", path, key, settings.external_config); continue; } String object_name = file_contents.getString(key + "." + settings.external_name); if (object_name.empty()) { - LOG_WARNING_FORMATTED(log, "{}: node '{}' defines {} with an empty name. It's not allowed", path, key, type_name); + LOG_WARNING(log, "{}: node '{}' defines {} with an empty name. It's not allowed", path, key, type_name); continue; } @@ -351,10 +351,10 @@ private: if (!already_added->from_temp_repository && !repository->isTemporary()) { if (path == already_added->path && repository->getName() == already_added->repository_name) - LOG_WARNING_FORMATTED(log, "{} '{}' is found twice in the same file '{}'", + LOG_WARNING(log, "{} '{}' is found twice in the same file '{}'", type_name, object_name, path); else - LOG_WARNING_FORMATTED(log, "{} '{}' is found both in file '{}' and '{}'", + LOG_WARNING(log, "{} '{}' is found both in file '{}' and '{}'", type_name, object_name, already_added->path, path); } } @@ -444,7 +444,7 @@ public: { /// The object has been tried to load before, so it is currently in use or was in use /// and we should try to reload it with the new config. - LOG_TRACE_FORMATTED(log, "Will reload '{}' because its configuration has been changed and there were attempts to load it before", name); + LOG_TRACE(log, "Will reload '{}' because its configuration has been changed and there were attempts to load it before", name); startLoading(info, true); } } @@ -459,7 +459,7 @@ public: Info & info = infos.emplace(name, Info{name, config}).first->second; if (always_load_everything) { - LOG_TRACE_FORMATTED(log, "Will load '{}' because always_load_everything flag is set.", name); + LOG_TRACE(log, "Will load '{}' because always_load_everything flag is set.", name); startLoading(info); } } @@ -472,7 +472,7 @@ public: { const auto & info = it->second; if (info.loaded() || info.isLoading()) - LOG_TRACE_FORMATTED(log, "Unloading '{}' because its configuration has been removed or detached", name); + LOG_TRACE(log, "Unloading '{}' because its configuration has been removed or detached", name); infos.erase(it); } } @@ -669,7 +669,7 @@ public: if (!should_update_flag) { info.next_update_time = calculateNextUpdateTime(info.object, info.error_count); - LOG_TRACE_FORMATTED(log, "Object '{}' not modified, will not reload. Next update at {}", info.name, ext::to_string(info.next_update_time)); + LOG_TRACE(log, "Object '{}' not modified, will not reload. Next update at {}", info.name, ext::to_string(info.next_update_time)); continue; } @@ -681,7 +681,7 @@ public: /// Object was never loaded successfully and should be reloaded. startLoading(info); } - LOG_TRACE_FORMATTED(log, "Object '{}' is neither loaded nor failed, so it will not be reloaded as outdated.", info.name); + LOG_TRACE(log, "Object '{}' is neither loaded nor failed, so it will not be reloaded as outdated.", info.name); } } } @@ -874,7 +874,7 @@ private: { if (info.isLoading()) { - LOG_TRACE_FORMATTED(log, "The object '{}' is already being loaded, force = {}.", info.name, forced_to_reload); + LOG_TRACE(log, "The object '{}' is already being loaded, force = {}.", info.name, forced_to_reload); if (!forced_to_reload) { @@ -890,7 +890,7 @@ private: info.loading_start_time = std::chrono::system_clock::now(); info.loading_end_time = TimePoint{}; - LOG_TRACE_FORMATTED(log, "Will load the object '{}' {}, force = {}, loading_id = {}", info.name, (enable_async_loading ? std::string("in background") : "immediately"), forced_to_reload, info.loading_id); + LOG_TRACE(log, "Will load the object '{}' {}, force = {}, loading_id = {}", info.name, (enable_async_loading ? std::string("in background") : "immediately"), forced_to_reload, info.loading_id); if (enable_async_loading) { @@ -919,7 +919,7 @@ private: /// Does the loading, possibly in the separate thread. void doLoading(const String & name, size_t loading_id, bool forced_to_reload, size_t min_id_to_finish_loading_dependencies_, bool async) { - LOG_TRACE_FORMATTED(log, "Start loading object '{}'", name); + LOG_TRACE(log, "Start loading object '{}'", name); try { /// Prepare for loading. @@ -929,7 +929,7 @@ private: info = prepareToLoadSingleObject(name, loading_id, min_id_to_finish_loading_dependencies_, lock); if (!info) { - LOG_TRACE_FORMATTED(log, "Could not lock object '{}' for loading", name); + LOG_TRACE(log, "Could not lock object '{}' for loading", name); return; } } @@ -1032,17 +1032,17 @@ private: /// This is necessary because the object could be removed or load with another config while the `mutex` was unlocked. if (!info) { - LOG_TRACE_FORMATTED(log, "Next update time for '{}' will not be set because this object was not found.", name); + LOG_TRACE(log, "Next update time for '{}' will not be set because this object was not found.", name); return; } if (!info->isLoading()) { - LOG_TRACE_FORMATTED(log, "Next update time for '{}' will not be set because this object is not currently loading.", name); + LOG_TRACE(log, "Next update time for '{}' will not be set because this object is not currently loading.", name); return; } if (info->loading_id != loading_id) { - LOG_TRACE_FORMATTED(log, "Next update time for '{}' will not be set because this object's current loading_id {} is different from the specified {}.", name, info->loading_id, loading_id); + LOG_TRACE(log, "Next update time for '{}' will not be set because this object's current loading_id {} is different from the specified {}.", name, info->loading_id, loading_id); return; } @@ -1072,7 +1072,7 @@ private: info->last_successful_update_time = current_time; info->state_id = info->loading_id; info->next_update_time = next_update_time; - LOG_TRACE_FORMATTED(log, "Next update time for '{}' was set to {}", info->name, ext::to_string(next_update_time)); + LOG_TRACE(log, "Next update time for '{}' was set to {}", info->name, ext::to_string(next_update_time)); } /// Removes the references to the loading thread from the maps. @@ -1104,7 +1104,7 @@ private: { if (!loaded_object->supportUpdates()) { - LOG_TRACE_FORMATTED(log, "Supposed update time for '{}' is never (loaded, does not support updates)", loaded_object->getLoadableName()); + LOG_TRACE(log, "Supposed update time for '{}' is never (loaded, does not support updates)", loaded_object->getLoadableName()); return never; } @@ -1113,7 +1113,7 @@ private: const auto & lifetime = loaded_object->getLifetime(); if (lifetime.min_sec == 0 && lifetime.max_sec == 0) { - LOG_TRACE_FORMATTED(log, "Supposed update time for '{}' is never (loaded, lifetime 0)", loaded_object->getLoadableName()); + LOG_TRACE(log, "Supposed update time for '{}' is never (loaded, lifetime 0)", loaded_object->getLoadableName()); return never; } @@ -1121,19 +1121,19 @@ private: { std::uniform_int_distribution distribution{lifetime.min_sec, lifetime.max_sec}; auto result = std::chrono::system_clock::now() + std::chrono::seconds{distribution(rnd_engine)}; - LOG_TRACE_FORMATTED(log, "Supposed update time for '{}' is {} (loaded, lifetime [{}, {}], no errors)", + LOG_TRACE(log, "Supposed update time for '{}' is {} (loaded, lifetime [{}, {}], no errors)", loaded_object->getLoadableName(), ext::to_string(result), lifetime.min_sec, lifetime.max_sec); return result; } auto result = std::chrono::system_clock::now() + std::chrono::seconds(calculateDurationWithBackoff(rnd_engine, error_count)); - LOG_TRACE_FORMATTED(log, "Supposed update time for '{}' is {} (backoff, {} errors)", loaded_object->getLoadableName(), ext::to_string(result), error_count); + LOG_TRACE(log, "Supposed update time for '{}' is {} (backoff, {} errors)", loaded_object->getLoadableName(), ext::to_string(result), error_count); return result; } else { auto result = std::chrono::system_clock::now() + std::chrono::seconds(calculateDurationWithBackoff(rnd_engine, error_count)); - LOG_TRACE_FORMATTED(log, "Supposed update time for unspecified object is {} (backoff, {} errors.", ext::to_string(result), error_count); + LOG_TRACE(log, "Supposed update time for unspecified object is {} (backoff, {} errors.", ext::to_string(result), error_count); return result; } } diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index 776d06e645d..cc2d62a68fe 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -422,7 +422,7 @@ void HashJoin::setSampleBlock(const Block & block) /// You have to restore this lock if you call the function outside of ctor. //std::unique_lock lock(rwlock); - LOG_DEBUG_FORMATTED(log, "setSampleBlock: {}", block.dumpStructure()); + LOG_DEBUG(log, "setSampleBlock: {}", block.dumpStructure()); if (!empty()) return; diff --git a/src/Interpreters/InternalTextLogsQueue.cpp b/src/Interpreters/InternalTextLogsQueue.cpp index a28ee0c3fd7..a61b0d6dd9e 100644 --- a/src/Interpreters/InternalTextLogsQueue.cpp +++ b/src/Interpreters/InternalTextLogsQueue.cpp @@ -43,7 +43,7 @@ void InternalTextLogsQueue::pushBlock(Block && log_block) if (blocksHaveEqualStructure(sample_block, log_block)) emplace(log_block.mutateColumns()); else - LOG_WARNING_FORMATTED(&Poco::Logger::get("InternalTextLogsQueue"), "Log block have different structure"); + LOG_WARNING(&Poco::Logger::get("InternalTextLogsQueue"), "Log block have different structure"); } const char * InternalTextLogsQueue::getPriorityName(int priority) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index c4a08f0f837..c8d842dfa53 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -776,7 +776,7 @@ void InterpreterSelectQuery::executeImpl(TPipeline & pipeline, const BlockInputS /** Read the data from Storage. from_stage - to what stage the request was completed in Storage. */ executeFetchColumns(from_stage, pipeline, expressions.prewhere_info, expressions.columns_to_remove_after_prewhere, save_context_and_storage); - LOG_TRACE_FORMATTED(log, "{} -> {}", QueryProcessingStage::toString(from_stage), QueryProcessingStage::toString(options.to_stage)); + LOG_TRACE(log, "{} -> {}", QueryProcessingStage::toString(from_stage), QueryProcessingStage::toString(options.to_stage)); } if (options.to_stage > QueryProcessingStage::FetchColumns) diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 376facabca3..f3d48651148 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -398,15 +398,15 @@ void InterpreterSystemQuery::syncReplica(ASTSystemQuery &) if (auto * storage_replicated = dynamic_cast(table.get())) { - LOG_TRACE_FORMATTED(log, "Synchronizing entries in replica's queue with table's log and waiting for it to become empty"); + LOG_TRACE(log, "Synchronizing entries in replica's queue with table's log and waiting for it to become empty"); if (!storage_replicated->waitForShrinkingQueueSize(0, context.getSettingsRef().receive_timeout.totalMilliseconds())) { - LOG_ERROR_FORMATTED(log, "SYNC REPLICA {}: Timed out!", table_id.getNameForLogs()); + LOG_ERROR(log, "SYNC REPLICA {}: Timed out!", table_id.getNameForLogs()); throw Exception( "SYNC REPLICA " + table_id.getNameForLogs() + ": command timed out! " "See the 'receive_timeout' setting", ErrorCodes::TIMEOUT_EXCEEDED); } - LOG_TRACE_FORMATTED(log, "SYNC REPLICA {}: OK", table_id.getNameForLogs()); + LOG_TRACE(log, "SYNC REPLICA {}: OK", table_id.getNameForLogs()); } else throw Exception("Table " + table_id.getNameForLogs() + " is not replicated", ErrorCodes::BAD_ARGUMENTS); diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index b22bdfa9666..9120f5959df 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -84,7 +84,7 @@ ProcessList::EntryPtr ProcessList::insert(const String & query_, const IAST * as if (!is_unlimited_query && max_size && processes.size() >= max_size) { if (queue_max_wait_ms) - LOG_WARNING_FORMATTED(&Logger::get("ProcessList"), "Too many simultaneous queries, will wait {} ms.", queue_max_wait_ms); + LOG_WARNING(&Logger::get("ProcessList"), "Too many simultaneous queries, will wait {} ms.", queue_max_wait_ms); if (!queue_max_wait_ms || !have_space.wait_for(lock, std::chrono::milliseconds(queue_max_wait_ms), [&]{ return processes.size() < max_size; })) throw Exception("Too many simultaneous queries. Maximum: " + toString(max_size), ErrorCodes::TOO_MANY_SIMULTANEOUS_QUERIES); } @@ -231,7 +231,7 @@ ProcessListEntry::~ProcessListEntry() auto user_process_list_it = parent.user_to_queries.find(user); if (user_process_list_it == parent.user_to_queries.end()) { - LOG_ERROR_FORMATTED(&Logger::get("ProcessList"), "Logical error: cannot find user in ProcessList"); + LOG_ERROR(&Logger::get("ProcessList"), "Logical error: cannot find user in ProcessList"); std::terminate(); } @@ -250,7 +250,7 @@ ProcessListEntry::~ProcessListEntry() if (!found) { - LOG_ERROR_FORMATTED(&Logger::get("ProcessList"), "Logical error: cannot find query by query_id and pointer to ProcessListElement in ProcessListForUser"); + LOG_ERROR(&Logger::get("ProcessList"), "Logical error: cannot find query by query_id and pointer to ProcessListElement in ProcessListForUser"); std::terminate(); } parent.have_space.notify_all(); diff --git a/src/Interpreters/SystemLog.cpp b/src/Interpreters/SystemLog.cpp index 9ce15f12138..e7e65c4f19e 100644 --- a/src/Interpreters/SystemLog.cpp +++ b/src/Interpreters/SystemLog.cpp @@ -41,7 +41,7 @@ std::shared_ptr createSystemLog( if (database != default_database_name) { /// System tables must be loaded before other tables, but loading order is undefined for all databases except `system` - LOG_ERROR_FORMATTED(&Logger::get("SystemLog"), "Custom database name for a system table specified in config. Table `{}` will be created in `system` database instead of `{}`", table, database); + LOG_ERROR(&Logger::get("SystemLog"), "Custom database name for a system table specified in config. Table `{}` will be created in `system` database instead of `{}`", table, database); database = default_database_name; } diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h index 72d85e10f14..49b8168678f 100644 --- a/src/Interpreters/SystemLog.h +++ b/src/Interpreters/SystemLog.h @@ -242,7 +242,7 @@ void SystemLog::add(const LogElement & element) requested_flush_before = queue_end; flush_event.notify_all(); - LOG_INFO_FORMATTED(log, "Queue is half full for system log '{}'.", demangle(typeid(*this).name())); + LOG_INFO(log, "Queue is half full for system log '{}'.", demangle(typeid(*this).name())); } if (queue.size() >= DBMS_SYSTEM_LOG_QUEUE_SIZE) { // Ignore all further entries until the queue is flushed. @@ -256,7 +256,7 @@ void SystemLog::add(const LogElement & element) // TextLog sets its logger level to 0, so this log is a noop and // there is no recursive logging. - LOG_ERROR_FORMATTED(log, "Queue is full for system log '{}' at {}", demangle(typeid(*this).name()), queue_front_index); + LOG_ERROR(log, "Queue is full for system log '{}' at {}", demangle(typeid(*this).name()), queue_front_index); } return; @@ -369,7 +369,7 @@ void SystemLog::savingThreadFunction() tryLogCurrentException(__PRETTY_FUNCTION__); } } - LOG_TRACE_FORMATTED(log, "Terminating"); + LOG_TRACE(log, "Terminating"); } @@ -378,7 +378,7 @@ void SystemLog::flushImpl(const std::vector & to_flush, { try { - LOG_TRACE_FORMATTED(log, "Flushing system log, {} entries to flush", to_flush.size()); + LOG_TRACE(log, "Flushing system log, {} entries to flush", to_flush.size()); /// We check for existence of the table and create it as needed at every /// flush. This is done to allow user to drop the table at any moment @@ -417,7 +417,7 @@ void SystemLog::flushImpl(const std::vector & to_flush, flush_event.notify_all(); } - LOG_TRACE_FORMATTED(log, "Flushed system log"); + LOG_TRACE(log, "Flushed system log"); } @@ -456,7 +456,7 @@ void SystemLog::prepareTable() rename->elements.emplace_back(elem); - LOG_DEBUG_FORMATTED(log, "Existing table {} for system log has obsolete or different structure. Renaming it to {}", description, backQuoteIfNeed(to.table)); + LOG_DEBUG(log, "Existing table {} for system log has obsolete or different structure. Renaming it to {}", description, backQuoteIfNeed(to.table)); InterpreterRenameQuery(rename, context).execute(); @@ -464,13 +464,13 @@ void SystemLog::prepareTable() table = nullptr; } else if (!is_prepared) - LOG_DEBUG_FORMATTED(log, "Will use existing table {} for {}", description, LogElement::name()); + LOG_DEBUG(log, "Will use existing table {} for {}", description, LogElement::name()); } if (!table) { /// Create the table. - LOG_DEBUG_FORMATTED(log, "Creating new table {} for {}", description, LogElement::name()); + LOG_DEBUG(log, "Creating new table {} for {}", description, LogElement::name()); auto create = getCreateTableQuery(); diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index a27508303c4..bde64919bc6 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -95,7 +95,7 @@ void ThreadStatus::setupState(const ThreadGroupStatusPtr & thread_group_) Int32 new_os_thread_priority = settings.os_thread_priority; if (new_os_thread_priority && hasLinuxCapability(CAP_SYS_NICE)) { - LOG_TRACE_FORMATTED(log, "Setting nice to {}", new_os_thread_priority); + LOG_TRACE(log, "Setting nice to {}", new_os_thread_priority); if (0 != setpriority(PRIO_PROCESS, thread_id, new_os_thread_priority)) throwFromErrno("Cannot 'setpriority'", ErrorCodes::CANNOT_SET_THREAD_PRIORITY); @@ -218,10 +218,10 @@ void ThreadStatus::detachQuery(bool exit_if_already_detached, bool thread_exits) #if defined(__linux__) if (os_thread_priority) { - LOG_TRACE_FORMATTED(log, "Resetting nice"); + LOG_TRACE(log, "Resetting nice"); if (0 != setpriority(PRIO_PROCESS, thread_id, 0)) - LOG_ERROR_FORMATTED(log, "Cannot 'setpriority' back to zero: {}", errnoToString(ErrorCodes::CANNOT_SET_THREAD_PRIORITY, errno)); + LOG_ERROR(log, "Cannot 'setpriority' back to zero: {}", errnoToString(ErrorCodes::CANNOT_SET_THREAD_PRIORITY, errno)); os_thread_priority = 0; } diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index ad899aa31a9..c69631b2665 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -105,7 +105,7 @@ static void logQuery(const String & query, const Context & context, bool interna { if (internal) { - LOG_DEBUG_FORMATTED(&Logger::get("executeQuery"), "(internal) {}", joinLines(query)); + LOG_DEBUG(&Logger::get("executeQuery"), "(internal) {}", joinLines(query)); } else { @@ -113,7 +113,7 @@ static void logQuery(const String & query, const Context & context, bool interna const auto & initial_query_id = context.getClientInfo().initial_query_id; const auto & current_user = context.getClientInfo().current_user; - LOG_DEBUG_FORMATTED(&Logger::get("executeQuery"), "(from {}{}{}) {}", + LOG_DEBUG(&Logger::get("executeQuery"), "(from {}{}{}) {}", context.getClientInfo().current_address.toString(), (current_user != "default" ? ", user: " + context.getClientInfo().current_user : ""), (!initial_query_id.empty() && current_query_id != initial_query_id ? ", initial_query_id: " + initial_query_id : std::string()), @@ -145,10 +145,10 @@ static void setExceptionStackTrace(QueryLogElement & elem) static void logException(Context & context, QueryLogElement & elem) { if (elem.stack_trace.empty()) - LOG_ERROR_FORMATTED(&Logger::get("executeQuery"), "{} (from {}) (in query: {})", + LOG_ERROR(&Logger::get("executeQuery"), "{} (from {}) (in query: {})", elem.exception, context.getClientInfo().current_address.toString(), joinLines(elem.query)); else - LOG_ERROR_FORMATTED(&Logger::get("executeQuery"), "{} (from {}) (in query: {})" + LOG_ERROR(&Logger::get("executeQuery"), "{} (from {}) (in query: {})" ", Stack trace (when copying this message, always include the lines below):\n\n{}", elem.exception, context.getClientInfo().current_address.toString(), joinLines(elem.query), elem.stack_trace); } @@ -505,7 +505,7 @@ static std::tuple executeQueryImpl( if (elem.read_rows != 0) { - LOG_INFO_FORMATTED(&Logger::get("executeQuery"), "Read {} rows, {} in {} sec., {} rows/sec., {}/sec.", + LOG_INFO(&Logger::get("executeQuery"), "Read {} rows, {} in {} sec., {} rows/sec., {}/sec.", elem.read_rows, formatReadableSizeWithBinarySuffix(elem.read_bytes), elapsed_seconds, static_cast(elem.read_rows / elapsed_seconds), formatReadableSizeWithBinarySuffix(elem.read_bytes / elapsed_seconds)); @@ -585,7 +585,7 @@ static std::tuple executeQueryImpl( std::stringstream log_str; log_str << "Query pipeline:\n"; res.in->dumpTree(log_str); - LOG_DEBUG_FORMATTED(&Logger::get("executeQuery"), log_str.str()); + LOG_DEBUG(&Logger::get("executeQuery"), log_str.str()); } } } diff --git a/src/Parsers/DumpASTNode.h b/src/Parsers/DumpASTNode.h index 2fad5793304..6b945cd5c8d 100644 --- a/src/Parsers/DumpASTNode.h +++ b/src/Parsers/DumpASTNode.h @@ -102,7 +102,7 @@ public: ~DebugASTLog() { if constexpr (_enable) - LOG_DEBUG_FORMATTED(log, ss.str()); + LOG_DEBUG(log, ss.str()); } std::ostream * stream() { return (_enable ? &ss : nullptr); } diff --git a/src/Processors/Executors/PipelineExecutor.cpp b/src/Processors/Executors/PipelineExecutor.cpp index 2ae621fb3b8..38f77f90abb 100644 --- a/src/Processors/Executors/PipelineExecutor.cpp +++ b/src/Processors/Executors/PipelineExecutor.cpp @@ -486,7 +486,7 @@ void PipelineExecutor::execute(size_t num_threads) catch (...) { #ifndef NDEBUG - LOG_TRACE_FORMATTED(log, "Exception while executing query. Current state:\n{}", dumpPipeline()); + LOG_TRACE(log, "Exception while executing query. Current state:\n{}", dumpPipeline()); #endif throw; } @@ -547,7 +547,7 @@ void PipelineExecutor::executeSingleThread(size_t thread_num, size_t num_threads #ifndef NDEBUG auto & context = executor_contexts[thread_num]; - LOG_TRACE_FORMATTED(log, "Thread finished. Total time: {} sec. Execution time: {} sec. Processing time: {} sec. Wait time: {} sec.", (context->total_time_ns / 1e9), (context->execution_time_ns / 1e9), (context->processing_time_ns / 1e9), (context->wait_time_ns / 1e9)); + LOG_TRACE(log, "Thread finished. Total time: {} sec. Execution time: {} sec. Processing time: {} sec. Wait time: {} sec.", (context->total_time_ns / 1e9), (context->execution_time_ns / 1e9), (context->processing_time_ns / 1e9), (context->wait_time_ns / 1e9)); #endif } diff --git a/src/Processors/Formats/IRowInputFormat.cpp b/src/Processors/Formats/IRowInputFormat.cpp index e717ab6f717..e2676eb64c2 100644 --- a/src/Processors/Formats/IRowInputFormat.cpp +++ b/src/Processors/Formats/IRowInputFormat.cpp @@ -150,7 +150,7 @@ Chunk IRowInputFormat::generate() if (num_errors && (params.allow_errors_num > 0 || params.allow_errors_ratio > 0)) { Logger * log = &Logger::get("IRowInputFormat"); - LOG_TRACE_FORMATTED(log, "Skipped {} rows with errors while reading the input stream", num_errors); + LOG_TRACE(log, "Skipped {} rows with errors while reading the input stream", num_errors); } readSuffix(); diff --git a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp index 01d7f6c9d91..78a291de262 100644 --- a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp @@ -612,7 +612,7 @@ private: try { Poco::URI url(base_url, "/schemas/ids/" + std::to_string(id)); - LOG_TRACE_FORMATTED((&Logger::get("AvroConfluentRowInputFormat")), "Fetching schema id = {}", id); + LOG_TRACE((&Logger::get("AvroConfluentRowInputFormat")), "Fetching schema id = {}", id); /// One second for connect/send/receive. Just in case. ConnectionTimeouts timeouts({1, 0}, {1, 0}, {1, 0}); @@ -629,7 +629,7 @@ private: Poco::JSON::Parser parser; auto json_body = parser.parse(*response_body).extract(); auto schema = json_body->getValue("schema"); - LOG_TRACE_FORMATTED((&Logger::get("AvroConfluentRowInputFormat")), "Succesfully fetched schema id = {}\n{}", id, schema); + LOG_TRACE((&Logger::get("AvroConfluentRowInputFormat")), "Succesfully fetched schema id = {}\n{}", id, schema); return avro::compileJsonSchemaFromString(schema); } catch (const Exception &) diff --git a/src/Processors/Merges/Algorithms/CollapsingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/CollapsingSortedAlgorithm.cpp index b13f0f06de0..05276f87540 100644 --- a/src/Processors/Merges/Algorithms/CollapsingSortedAlgorithm.cpp +++ b/src/Processors/Merges/Algorithms/CollapsingSortedAlgorithm.cpp @@ -55,7 +55,7 @@ void CollapsingSortedAlgorithm::reportIncorrectData() * since the data is generated by external programs. * With inconsistent data, this is an unavoidable error that can not be easily corrected by admins. Therefore Warning. */ - LOG_WARNING_FORMATTED(log, + LOG_WARNING(log, "Incorrect data: number of rows with sign = 1 ({}) differs with number of rows with sign = -1 ({}) by more than one (for key: {}).", count_positive, count_negative, s.str()); } diff --git a/src/Processors/Merges/Algorithms/RowRef.h b/src/Processors/Merges/Algorithms/RowRef.h index 4ceec885a0b..efd04717e94 100644 --- a/src/Processors/Merges/Algorithms/RowRef.h +++ b/src/Processors/Merges/Algorithms/RowRef.h @@ -83,7 +83,7 @@ public: { if (free_chunks.size() != chunks.size()) { - LOG_ERROR_FORMATTED(&Logger::get("SharedChunkAllocator"), "SharedChunkAllocator was destroyed before RowRef was released. StackTrace: {}", StackTrace().toString()); + LOG_ERROR(&Logger::get("SharedChunkAllocator"), "SharedChunkAllocator was destroyed before RowRef was released. StackTrace: {}", StackTrace().toString()); return; } @@ -100,7 +100,7 @@ private: /// This may happen if allocator was removed before chunks. /// Log message and exit, because we don't want to throw exception in destructor. - LOG_ERROR_FORMATTED(&Logger::get("SharedChunkAllocator"), "SharedChunkAllocator was destroyed before RowRef was released. StackTrace: {}", StackTrace().toString()); + LOG_ERROR(&Logger::get("SharedChunkAllocator"), "SharedChunkAllocator was destroyed before RowRef was released. StackTrace: {}", StackTrace().toString()); return; } diff --git a/src/Processors/Merges/MergingSortedTransform.cpp b/src/Processors/Merges/MergingSortedTransform.cpp index 11246c30aaa..defadd0caec 100644 --- a/src/Processors/Merges/MergingSortedTransform.cpp +++ b/src/Processors/Merges/MergingSortedTransform.cpp @@ -48,9 +48,9 @@ void MergingSortedTransform::onFinish() double seconds = total_stopwatch.elapsedSeconds(); if (!seconds) - LOG_DEBUG_FORMATTED(log, "Merge sorted {} blocks, {} rows in 0 sec.", merged_data.totalChunks(), merged_data.totalMergedRows()); + LOG_DEBUG(log, "Merge sorted {} blocks, {} rows in 0 sec.", merged_data.totalChunks(), merged_data.totalMergedRows()); else - LOG_DEBUG_FORMATTED(log, "Merge sorted {} blocks, {} rows in {} sec., {} rows/sec., {}/sec", + LOG_DEBUG(log, "Merge sorted {} blocks, {} rows in {} sec., {} rows/sec., {}/sec", merged_data.totalChunks(), merged_data.totalMergedRows(), seconds, merged_data.totalMergedRows() / seconds, formatReadableSizeWithBinarySuffix(merged_data.totalAllocatedBytes() / seconds)); diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index 2a10223022e..fc8fd575c35 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -515,7 +515,7 @@ void AggregatingTransform::consume(Chunk chunk) if (!is_consume_started) { - LOG_TRACE_FORMATTED(log, "Aggregating"); + LOG_TRACE(log, "Aggregating"); is_consume_started = true; } @@ -541,7 +541,7 @@ void AggregatingTransform::initGenerate() double elapsed_seconds = watch.elapsedSeconds(); size_t rows = variants.sizeWithoutOverflowRow(); - LOG_TRACE_FORMATTED(log, "Aggregated. {} to {} rows (from {}) in {} sec. ({} rows/sec., {}/sec.)", + LOG_TRACE(log, "Aggregated. {} to {} rows (from {}) in {} sec. ({} rows/sec., {}/sec.)", src_rows, rows, formatReadableSizeWithBinarySuffix(src_bytes), elapsed_seconds, src_rows / elapsed_seconds, formatReadableSizeWithBinarySuffix(src_bytes / elapsed_seconds)); @@ -593,7 +593,7 @@ void AggregatingTransform::initGenerate() for (const auto & file : files.files) processors.emplace_back(std::make_unique(header, file->path())); - LOG_TRACE_FORMATTED(log, "Will merge {} temporary files of size {} compressed, {} uncompressed.", files.files.size(), formatReadableSizeWithBinarySuffix(files.sum_size_compressed), formatReadableSizeWithBinarySuffix(files.sum_size_uncompressed)); + LOG_TRACE(log, "Will merge {} temporary files of size {} compressed, {} uncompressed.", files.files.size(), formatReadableSizeWithBinarySuffix(files.sum_size_compressed), formatReadableSizeWithBinarySuffix(files.sum_size_uncompressed)); auto pipe = createMergingAggregatedMemoryEfficientPipe( header, params, files.files.size(), temporary_data_merge_threads); diff --git a/src/Processors/Transforms/CreatingSetsTransform.cpp b/src/Processors/Transforms/CreatingSetsTransform.cpp index 19ed3067703..af8fa4097df 100644 --- a/src/Processors/Transforms/CreatingSetsTransform.cpp +++ b/src/Processors/Transforms/CreatingSetsTransform.cpp @@ -57,11 +57,11 @@ IProcessor::Status CreatingSetsTransform::prepare() void CreatingSetsTransform::startSubquery(SubqueryForSet & subquery) { if (subquery.set) - LOG_TRACE_FORMATTED(log, "Creating set."); + LOG_TRACE(log, "Creating set."); if (subquery.join) - LOG_TRACE_FORMATTED(log, "Creating join."); + LOG_TRACE(log, "Creating join."); if (subquery.table) - LOG_TRACE_FORMATTED(log, "Filling temporary table."); + LOG_TRACE(log, "Filling temporary table."); elapsed_nanoseconds = 0; @@ -93,15 +93,15 @@ void CreatingSetsTransform::finishSubquery(SubqueryForSet & subquery) auto seconds = elapsed_nanoseconds / 1e9; if (subquery.set) - LOG_DEBUG_FORMATTED(log, "Created Set with {} entries from {} rows in {} sec.", subquery.set->getTotalRowCount(), head_rows, seconds); + LOG_DEBUG(log, "Created Set with {} entries from {} rows in {} sec.", subquery.set->getTotalRowCount(), head_rows, seconds); if (subquery.join) - LOG_DEBUG_FORMATTED(log, "Created Join with {} entries from {} rows in {} sec.", subquery.join->getTotalRowCount(), head_rows, seconds); + LOG_DEBUG(log, "Created Join with {} entries from {} rows in {} sec.", subquery.join->getTotalRowCount(), head_rows, seconds); if (subquery.table) - LOG_DEBUG_FORMATTED(log, "Created Table with {} rows in {} sec.", head_rows, seconds); + LOG_DEBUG(log, "Created Table with {} rows in {} sec.", head_rows, seconds); } else { - LOG_DEBUG_FORMATTED(log, "Subquery has empty result."); + LOG_DEBUG(log, "Subquery has empty result."); } } diff --git a/src/Processors/Transforms/MergeSortingTransform.cpp b/src/Processors/Transforms/MergeSortingTransform.cpp index 7147781b4e8..34eb833c411 100644 --- a/src/Processors/Transforms/MergeSortingTransform.cpp +++ b/src/Processors/Transforms/MergeSortingTransform.cpp @@ -36,7 +36,7 @@ public: , path(std::move(path_)), file_buf_out(path), compressed_buf_out(file_buf_out) , out_stream(std::make_shared(compressed_buf_out, 0, header)) { - LOG_INFO_FORMATTED(log, "Sorting and writing part of data into temporary file {}", path); + LOG_INFO(log, "Sorting and writing part of data into temporary file {}", path); ProfileEvents::increment(ProfileEvents::ExternalSortWritePart); out_stream->writePrefix(); } @@ -55,7 +55,7 @@ public: out_stream->writeSuffix(); compressed_buf_out.next(); file_buf_out.next(); - LOG_INFO_FORMATTED(log, "Done writing part of data into temporary file {}", path); + LOG_INFO(log, "Done writing part of data into temporary file {}", path); out_stream.reset(); @@ -229,7 +229,7 @@ void MergeSortingTransform::generate() else { ProfileEvents::increment(ProfileEvents::ExternalSortMerge); - LOG_INFO_FORMATTED(log, "There are {} temporary sorted parts to merge.", temporary_files.size()); + LOG_INFO(log, "There are {} temporary sorted parts to merge.", temporary_files.size()); if (!chunks.empty()) processors.emplace_back(std::make_shared( @@ -251,7 +251,7 @@ void MergeSortingTransform::generate() void MergeSortingTransform::remerge() { - LOG_DEBUG_FORMATTED(log, "Re-merging intermediate ORDER BY data ({} blocks with {} rows) to save memory consumption", chunks.size(), sum_rows_in_blocks); + LOG_DEBUG(log, "Re-merging intermediate ORDER BY data ({} blocks with {} rows) to save memory consumption", chunks.size(), sum_rows_in_blocks); /// NOTE Maybe concat all blocks and partial sort will be faster than merge? MergeSorter remerge_sorter(std::move(chunks), description, max_merged_block_size, limit); @@ -267,7 +267,7 @@ void MergeSortingTransform::remerge() new_chunks.emplace_back(std::move(chunk)); } - LOG_DEBUG_FORMATTED(log, "Memory usage is lowered from {} to {}", formatReadableSizeWithBinarySuffix(sum_bytes_in_blocks), formatReadableSizeWithBinarySuffix(new_sum_bytes_in_blocks)); + LOG_DEBUG(log, "Memory usage is lowered from {} to {}", formatReadableSizeWithBinarySuffix(sum_bytes_in_blocks), formatReadableSizeWithBinarySuffix(new_sum_bytes_in_blocks)); /// If the memory consumption was not lowered enough - we will not perform remerge anymore. 2 is a guess. if (new_sum_bytes_in_blocks * 2 > sum_bytes_in_blocks) diff --git a/src/Processors/Transforms/MergingAggregatedTransform.cpp b/src/Processors/Transforms/MergingAggregatedTransform.cpp index 2d787c5c3b6..f386cbe4bec 100644 --- a/src/Processors/Transforms/MergingAggregatedTransform.cpp +++ b/src/Processors/Transforms/MergingAggregatedTransform.cpp @@ -20,7 +20,7 @@ void MergingAggregatedTransform::consume(Chunk chunk) if (!consume_started) { consume_started = true; - LOG_TRACE_FORMATTED(log, "Reading blocks of partially aggregated data."); + LOG_TRACE(log, "Reading blocks of partially aggregated data."); } total_input_rows += chunk.getNumRows(); @@ -46,7 +46,7 @@ Chunk MergingAggregatedTransform::generate() if (!generate_started) { generate_started = true; - LOG_TRACE_FORMATTED(log, "Read {} blocks of partially aggregated data, total {} rows.", total_input_blocks, total_input_rows); + LOG_TRACE(log, "Read {} blocks of partially aggregated data, total {} rows.", total_input_blocks, total_input_rows); /// Exception safety. Make iterator valid in case any method below throws. next_block = blocks.begin(); diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp index c17c2090a04..fdcbe8e69f9 100644 --- a/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/src/Storages/Distributed/DirectoryMonitor.cpp @@ -153,7 +153,7 @@ void StorageDistributedDirectoryMonitor::run() } else { - LOG_DEBUG_FORMATTED(log, "Skipping send data over distributed table."); + LOG_DEBUG(log, "Skipping send data over distributed table."); } const auto now = std::chrono::system_clock::now(); @@ -261,7 +261,7 @@ bool StorageDistributedDirectoryMonitor::processFiles() void StorageDistributedDirectoryMonitor::processFile(const std::string & file_path) { - LOG_TRACE_FORMATTED(log, "Started processing `{}`", file_path); + LOG_TRACE(log, "Started processing `{}`", file_path); auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(storage.global_context->getSettingsRef()); auto connection = pool->get(timeouts); @@ -290,7 +290,7 @@ void StorageDistributedDirectoryMonitor::processFile(const std::string & file_pa Poco::File{file_path}.remove(); - LOG_TRACE_FORMATTED(log, "Finished processing `{}`", file_path); + LOG_TRACE(log, "Finished processing `{}`", file_path); } void StorageDistributedDirectoryMonitor::readHeader( @@ -317,7 +317,7 @@ void StorageDistributedDirectoryMonitor::readHeader( readVarUInt(initiator_revision, header_buf); if (ClickHouseRevision::get() < initiator_revision) { - LOG_WARNING_FORMATTED(log, "ClickHouse shard version is older than ClickHouse initiator version. It may lack support for new features."); + LOG_WARNING(log, "ClickHouse shard version is older than ClickHouse initiator version. It may lack support for new features."); } readStringBinary(insert_query, header_buf); @@ -425,7 +425,7 @@ struct StorageDistributedDirectoryMonitor::Batch String tmp_file{parent.current_batch_file_path + ".tmp"}; if (Poco::File{tmp_file}.exists()) - LOG_ERROR_FORMATTED(parent.log, "Temporary file {} exists. Unclean shutdown?", backQuote(tmp_file)); + LOG_ERROR(parent.log, "Temporary file {} exists. Unclean shutdown?", backQuote(tmp_file)); { WriteBufferFromFile out{tmp_file, O_WRONLY | O_TRUNC | O_CREAT}; @@ -451,7 +451,7 @@ struct StorageDistributedDirectoryMonitor::Batch auto file_path = file_index_to_path.find(file_idx); if (file_path == file_index_to_path.end()) { - LOG_ERROR_FORMATTED(parent.log, "Failed to send batch: file with index {} is absent", file_idx); + LOG_ERROR(parent.log, "Failed to send batch: file with index {} is absent", file_idx); batch_broken = true; break; } @@ -485,14 +485,14 @@ struct StorageDistributedDirectoryMonitor::Batch if (!batch_broken) { - LOG_TRACE_FORMATTED(parent.log, "Sent a batch of {} files.", file_indices.size()); + LOG_TRACE(parent.log, "Sent a batch of {} files.", file_indices.size()); for (UInt64 file_index : file_indices) Poco::File{file_index_to_path.at(file_index)}.remove(); } else { - LOG_ERROR_FORMATTED(parent.log, "Marking a batch of {} files as broken.", file_indices.size()); + LOG_ERROR(parent.log, "Marking a batch of {} files as broken.", file_indices.size()); for (UInt64 file_idx : file_indices) { @@ -692,7 +692,7 @@ void StorageDistributedDirectoryMonitor::markAsBroken(const std::string & file_p Poco::File{broken_path}.createDirectory(); Poco::File{file_path}.renameTo(broken_file_path); - LOG_ERROR_FORMATTED(log, "Renamed `{}` to `{}`", file_path, broken_file_path); + LOG_ERROR(log, "Renamed `{}` to `{}`", file_path, broken_file_path); } bool StorageDistributedDirectoryMonitor::maybeMarkAsBroken(const std::string & file_path, const Exception & e) const diff --git a/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/src/Storages/Distributed/DistributedBlockOutputStream.cpp index 3ce472f300d..a171e289695 100644 --- a/src/Storages/Distributed/DistributedBlockOutputStream.cpp +++ b/src/Storages/Distributed/DistributedBlockOutputStream.cpp @@ -114,7 +114,7 @@ void DistributedBlockOutputStream::write(const Block & block) if (ordinary_block.has(col.name)) { ordinary_block.erase(col.name); - LOG_DEBUG_FORMATTED(log, "{}: column {} will be removed, because it is MATERIALIZED", + LOG_DEBUG(log, "{}: column {} will be removed, because it is MATERIALIZED", storage.getStorageID().getNameForLogs(), col.name); } } @@ -228,7 +228,7 @@ void DistributedBlockOutputStream::waitForJobs() size_t num_finished_jobs = finished_jobs_count; if (num_finished_jobs < jobs_count) - LOG_WARNING_FORMATTED(log, "Expected {} writing jobs, but finished only {}", jobs_count, num_finished_jobs); + LOG_WARNING(log, "Expected {} writing jobs, but finished only {}", jobs_count, num_finished_jobs); } @@ -412,7 +412,7 @@ void DistributedBlockOutputStream::writeSuffix() auto log_performance = [this]() { double elapsed = watch.elapsedSeconds(); - LOG_DEBUG_FORMATTED(log, "It took {} sec. to insert {} blocks, {} rows per second. {}", elapsed, inserted_blocks, inserted_rows / elapsed, getCurrentStateDescription()); + LOG_DEBUG(log, "It took {} sec. to insert {} blocks, {} rows per second. {}", elapsed, inserted_blocks, inserted_rows / elapsed, getCurrentStateDescription()); }; if (insert_sync && pool) diff --git a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp index 0421c706e12..70340a93159 100644 --- a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp +++ b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp @@ -38,7 +38,7 @@ ReadBufferFromKafkaConsumer::ReadBufferFromKafkaConsumer( // called (synchroniously, during poll) when we enter the consumer group consumer->set_assignment_callback([this](const cppkafka::TopicPartitionList & topic_partitions) { - LOG_TRACE_FORMATTED(log, "Topics/partitions assigned: {}", topic_partitions); + LOG_TRACE(log, "Topics/partitions assigned: {}", topic_partitions); assignment = topic_partitions; }); @@ -47,7 +47,7 @@ ReadBufferFromKafkaConsumer::ReadBufferFromKafkaConsumer( { // Rebalance is happening now, and now we have a chance to finish the work // with topics/partitions we were working with before rebalance - LOG_TRACE_FORMATTED(log, "Rebalance initiated. Revoking partitions: {}", topic_partitions); + LOG_TRACE(log, "Rebalance initiated. Revoking partitions: {}", topic_partitions); // we can not flush data to target from that point (it is pulled, not pushed) // so the best we can now it to @@ -70,13 +70,13 @@ ReadBufferFromKafkaConsumer::ReadBufferFromKafkaConsumer( // } // catch (cppkafka::HandleException & e) // { - // LOG_WARNING_FORMATTED(log, "Commit error: {}", e.what()); + // LOG_WARNING(log, "Commit error: {}", e.what()); // } }); consumer->set_rebalance_error_callback([this](cppkafka::Error err) { - LOG_ERROR_FORMATTED(log, "Rebalance error: {}", err); + LOG_ERROR(log, "Rebalance error: {}", err); }); } @@ -104,11 +104,11 @@ void ReadBufferFromKafkaConsumer::commit() if (topic_part.get_offset() < 0) { - LOG_TRACE_FORMATTED(log, "{} {} (topic: {}, partition: {})", prefix, print_special_offset(), topic_part.get_topic(), topic_part.get_partition()); + LOG_TRACE(log, "{} {} (topic: {}, partition: {})", prefix, print_special_offset(), topic_part.get_topic(), topic_part.get_partition()); } else { - LOG_TRACE_FORMATTED(log, "{} {} (topic: {}, partition: {})", prefix, topic_part.get_offset(), topic_part.get_topic(), topic_part.get_partition()); + LOG_TRACE(log, "{} {} (topic: {}, partition: {})", prefix, topic_part.get_offset(), topic_part.get_topic(), topic_part.get_partition()); } } }; @@ -117,7 +117,7 @@ void ReadBufferFromKafkaConsumer::commit() if (hasMorePolledMessages()) { - LOG_WARNING_FORMATTED(log, "Logical error. Non all polled messages were processed."); + LOG_WARNING(log, "Logical error. Non all polled messages were processed."); } if (offsets_stored > 0) @@ -144,7 +144,7 @@ void ReadBufferFromKafkaConsumer::commit() } catch (const cppkafka::HandleException & e) { - LOG_ERROR_FORMATTED(log, "Exception during commit attempt: {}", e.what()); + LOG_ERROR(log, "Exception during commit attempt: {}", e.what()); } --max_retries; } @@ -157,7 +157,7 @@ void ReadBufferFromKafkaConsumer::commit() } else { - LOG_TRACE_FORMATTED(log, "Nothing to commit."); + LOG_TRACE(log, "Nothing to commit."); } offsets_stored = 0; @@ -166,8 +166,8 @@ void ReadBufferFromKafkaConsumer::commit() void ReadBufferFromKafkaConsumer::subscribe() { - LOG_TRACE_FORMATTED(log, "Already subscribed to topics: [{}]", boost::algorithm::join(consumer->get_subscription(), ", ")); - LOG_TRACE_FORMATTED(log, "Already assigned to: {}", assignment); + LOG_TRACE(log, "Already subscribed to topics: [{}]", boost::algorithm::join(consumer->get_subscription(), ", ")); + LOG_TRACE(log, "Already assigned to: {}", assignment); size_t max_retries = 5; @@ -195,7 +195,7 @@ void ReadBufferFromKafkaConsumer::subscribe() void ReadBufferFromKafkaConsumer::unsubscribe() { - LOG_TRACE_FORMATTED(log, "Re-joining claimed consumer after failure"); + LOG_TRACE(log, "Re-joining claimed consumer after failure"); messages.clear(); current = messages.begin(); @@ -214,7 +214,7 @@ void ReadBufferFromKafkaConsumer::unsubscribe() } catch (const cppkafka::HandleException & e) { - LOG_ERROR_FORMATTED(log, "Exception from ReadBufferFromKafkaConsumer::unsubscribe: {}", e.what()); + LOG_ERROR(log, "Exception from ReadBufferFromKafkaConsumer::unsubscribe: {}", e.what()); } } @@ -230,12 +230,12 @@ void ReadBufferFromKafkaConsumer::resetToLastCommitted(const char * msg) { if (assignment.empty()) { - LOG_TRACE_FORMATTED(log, "Not assignned. Can't reset to last committed position."); + LOG_TRACE(log, "Not assignned. Can't reset to last committed position."); return; } auto committed_offset = consumer->get_offsets_committed(consumer->get_assignment()); consumer->assign(committed_offset); - LOG_TRACE_FORMATTED(log, "{} Returned to committed position: {}", msg, committed_offset); + LOG_TRACE(log, "{} Returned to committed position: {}", msg, committed_offset); } /// Do commit messages implicitly after we processed the previous batch. @@ -303,7 +303,7 @@ bool ReadBufferFromKafkaConsumer::nextImpl() } else { - LOG_TRACE_FORMATTED(log, "Can't get assignment"); + LOG_TRACE(log, "Can't get assignment"); stalled = true; return false; } @@ -311,7 +311,7 @@ bool ReadBufferFromKafkaConsumer::nextImpl() } else { - LOG_TRACE_FORMATTED(log, "Stalled"); + LOG_TRACE(log, "Stalled"); stalled = true; return false; } @@ -320,7 +320,7 @@ bool ReadBufferFromKafkaConsumer::nextImpl() { messages = std::move(new_messages); current = messages.begin(); - LOG_TRACE_FORMATTED(log, "Polled batch of {} messages. Offset position: {}", messages.size(), consumer->get_offsets_position(consumer->get_assignment())); + LOG_TRACE(log, "Polled batch of {} messages. Offset position: {}", messages.size(), consumer->get_offsets_position(consumer->get_assignment())); break; } } @@ -331,7 +331,7 @@ bool ReadBufferFromKafkaConsumer::nextImpl() ++current; // TODO: should throw exception instead - LOG_ERROR_FORMATTED(log, "Consumer error: {}", err); + LOG_ERROR(log, "Consumer error: {}", err); return false; } diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index 4e8cb916c38..78a28361f32 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -174,7 +174,7 @@ Pipes StorageKafka::read( pipes.emplace_back(std::make_shared(std::make_shared(*this, context, column_names, 1))); } - LOG_DEBUG_FORMATTED(log, "Starting reading {} streams", pipes.size()); + LOG_DEBUG(log, "Starting reading {} streams", pipes.size()); return pipes; } @@ -212,7 +212,7 @@ void StorageKafka::shutdown() // Interrupt streaming thread stream_cancelled = true; - LOG_TRACE_FORMATTED(log, "Waiting for cleanup"); + LOG_TRACE(log, "Waiting for cleanup"); task->deactivate(); // Close all consumers @@ -325,7 +325,7 @@ void StorageKafka::updateConfiguration(cppkafka::Configuration & conf) conf.set_log_callback([this](cppkafka::KafkaHandleBase &, int level, const std::string & /* facility */, const std::string & message) { auto [poco_level, client_logs_level] = parseSyslogLevel(level); - LOG_IMPL_FORMATTED(log, client_logs_level, poco_level, message); + LOG_IMPL(log, client_logs_level, poco_level, message); }); // Configure interceptor to change thread name @@ -340,12 +340,12 @@ void StorageKafka::updateConfiguration(cppkafka::Configuration & conf) status = rd_kafka_conf_interceptor_add_on_new(conf.get_handle(), "setThreadName", rdKafkaOnNew, self); if (status != RD_KAFKA_RESP_ERR_NO_ERROR) - LOG_ERROR_FORMATTED(log, "Cannot set new interceptor"); + LOG_ERROR(log, "Cannot set new interceptor"); // cppkafka always copy the configuration status = rd_kafka_conf_interceptor_add_on_conf_dup(conf.get_handle(), "setThreadName", rdKafkaOnConfDup, self); if (status != RD_KAFKA_RESP_ERR_NO_ERROR) - LOG_ERROR_FORMATTED(log, "Cannot set dup conf interceptor"); + LOG_ERROR(log, "Cannot set dup conf interceptor"); } } @@ -391,7 +391,7 @@ void StorageKafka::threadFunc() if (!checkDependencies(table_id)) break; - LOG_DEBUG_FORMATTED(log, "Started streaming to {} attached views", dependencies_count); + LOG_DEBUG(log, "Started streaming to {} attached views", dependencies_count); // Reschedule if not limited if (!streamToViews()) diff --git a/src/Storages/MergeTree/BackgroundProcessingPool.cpp b/src/Storages/MergeTree/BackgroundProcessingPool.cpp index 5b815f461bb..ff195112929 100644 --- a/src/Storages/MergeTree/BackgroundProcessingPool.cpp +++ b/src/Storages/MergeTree/BackgroundProcessingPool.cpp @@ -52,7 +52,7 @@ BackgroundProcessingPool::BackgroundProcessingPool(int size_, , settings(pool_settings) { logger = &Logger::get(log_name); - LOG_INFO_FORMATTED(logger, "Create {} with {} threads", log_name, size); + LOG_INFO(logger, "Create {} with {} threads", log_name, size); threads.resize(size); for (auto & thread : threads) diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index 1fe26eb06da..69494ba0d85 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -82,7 +82,7 @@ void Service::processQuery(const Poco::Net::HTMLForm & params, ReadBuffer & /*bo ++data.current_table_sends; SCOPE_EXIT({--data.current_table_sends;}); - LOG_TRACE_FORMATTED(log, "Sending part {}", part_name); + LOG_TRACE(log, "Sending part {}", part_name); try { diff --git a/src/Storages/MergeTree/EphemeralLockInZooKeeper.cpp b/src/Storages/MergeTree/EphemeralLockInZooKeeper.cpp index 2f4a42848e5..758401b7f4f 100644 --- a/src/Storages/MergeTree/EphemeralLockInZooKeeper.cpp +++ b/src/Storages/MergeTree/EphemeralLockInZooKeeper.cpp @@ -107,7 +107,7 @@ EphemeralLocksInAllPartitions::EphemeralLocksInAllPartitions( int rc = zookeeper.tryMulti(lock_ops, lock_responses); if (rc == Coordination::ZBADVERSION) { - LOG_TRACE_FORMATTED(&Logger::get("EphemeralLocksInAllPartitions"), "Someone has inserted a block in a new partition while we were creating locks. Retry."); + LOG_TRACE(&Logger::get("EphemeralLocksInAllPartitions"), "Someone has inserted a block in a new partition while we were creating locks. Retry."); continue; } else if (rc != Coordination::ZOK) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 18416cebda6..de8a9bb317f 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -257,7 +257,7 @@ void IMergeTreeDataPart::removeIfNeeded() if (!startsWith(file_name, "tmp")) { - LOG_ERROR_FORMATTED(storage.log, "~DataPart() should remove part {} but its name doesn't start with tmp. Too suspicious, keeping the part.", path); + LOG_ERROR(storage.log, "~DataPart() should remove part {} but its name doesn't start with tmp. Too suspicious, keeping the part.", path); return; } } @@ -266,7 +266,7 @@ void IMergeTreeDataPart::removeIfNeeded() if (state == State::DeleteOnDestroy) { - LOG_TRACE_FORMATTED(storage.log, "Removed part from old location {}", path); + LOG_TRACE(storage.log, "Removed part from old location {}", path); } } catch (...) @@ -668,7 +668,7 @@ void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_ Names files; volume->getDisk()->listFiles(to, files); - LOG_WARNING_FORMATTED(storage.log, "Part directory {} already exists and contains {} files. Removing it.", fullPath(volume->getDisk(), to), files.size()); + LOG_WARNING(storage.log, "Part directory {} already exists and contains {} files. Removing it.", fullPath(volume->getDisk(), to), files.size()); volume->getDisk()->removeRecursive(to); } @@ -710,7 +710,7 @@ void IMergeTreeDataPart::remove() const if (volume->getDisk()->exists(to)) { - LOG_WARNING_FORMATTED(storage.log, "Directory {} (to which part must be renamed before removing) already exists. Most likely this is due to unclean restart. Removing it.", fullPath(volume->getDisk(), to)); + LOG_WARNING(storage.log, "Directory {} (to which part must be renamed before removing) already exists. Most likely this is due to unclean restart. Removing it.", fullPath(volume->getDisk(), to)); try { @@ -718,7 +718,7 @@ void IMergeTreeDataPart::remove() const } catch (...) { - LOG_ERROR_FORMATTED(storage.log, "Cannot recursively remove directory {}. Exception: {}", fullPath(volume->getDisk(), to), getCurrentExceptionMessage(false)); + LOG_ERROR(storage.log, "Cannot recursively remove directory {}. Exception: {}", fullPath(volume->getDisk(), to), getCurrentExceptionMessage(false)); throw; } } @@ -729,7 +729,7 @@ void IMergeTreeDataPart::remove() const } catch (const Poco::FileNotFoundException &) { - LOG_ERROR_FORMATTED(storage.log, "Directory {} (part to remove) doesn't exist or one of nested files has gone. Most likely this is due to manual removing. This should be discouraged. Ignoring.", fullPath(volume->getDisk(), to)); + LOG_ERROR(storage.log, "Directory {} (part to remove) doesn't exist or one of nested files has gone. Most likely this is due to manual removing. This should be discouraged. Ignoring.", fullPath(volume->getDisk(), to)); return; } @@ -765,7 +765,7 @@ void IMergeTreeDataPart::remove() const { /// Recursive directory removal does many excessive "stat" syscalls under the hood. - LOG_ERROR_FORMATTED(storage.log, "Cannot quickly remove directory {} by removing files; fallback to recursive removal. Reason: {}", fullPath(volume->getDisk(), to), getCurrentExceptionMessage(false)); + LOG_ERROR(storage.log, "Cannot quickly remove directory {} by removing files; fallback to recursive removal. Reason: {}", fullPath(volume->getDisk(), to), getCurrentExceptionMessage(false)); volume->getDisk()->removeRecursive(to + "/"); } @@ -791,7 +791,7 @@ String IMergeTreeDataPart::getRelativePathForDetachedPart(const String & prefix) if (!volume->getDisk()->exists(getFullRelativePath() + res)) return res; - LOG_WARNING_FORMATTED(storage.log, "Directory {} (to detach to) already exists. Will detach to directory with '_tryN' suffix.", res); + LOG_WARNING(storage.log, "Directory {} (to detach to) already exists. Will detach to directory with '_tryN' suffix.", res); } return res; @@ -806,7 +806,7 @@ void IMergeTreeDataPart::renameToDetached(const String & prefix) const void IMergeTreeDataPart::makeCloneInDetached(const String & prefix) const { assertOnDisk(); - LOG_INFO_FORMATTED(storage.log, "Detaching {}", relative_path); + LOG_INFO(storage.log, "Detaching {}", relative_path); String destination_path = storage.relative_data_path + getRelativePathForDetachedPart(prefix); diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index a5310b0608d..d97001bc42e 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -204,7 +204,7 @@ MergeTreeData::MergeTreeData( { if (!version_file.first.empty()) { - LOG_ERROR_FORMATTED(log, "Duplication of version file {} and {}", fullPath(version_file.second, version_file.first), current_version_file_path); + LOG_ERROR(log, "Duplication of version file {} and {}", fullPath(version_file.second, version_file.first), current_version_file_path); throw Exception("Multiple format_version.txt file", ErrorCodes::CORRUPTED_DATA); } version_file = {current_version_file_path, disk}; @@ -244,7 +244,7 @@ MergeTreeData::MergeTreeData( String reason; if (!canUsePolymorphicParts(*settings, &reason) && !reason.empty()) - LOG_WARNING_FORMATTED(log, "{} Settings 'min_bytes_for_wide_part' and 'min_bytes_for_wide_part' will be ignored.", reason); + LOG_WARNING(log, "{} Settings 'min_bytes_for_wide_part' and 'min_bytes_for_wide_part' will be ignored.", reason); } @@ -850,7 +850,7 @@ Int64 MergeTreeData::getMaxBlockNumber() const void MergeTreeData::loadDataParts(bool skip_sanity_checks) { - LOG_DEBUG_FORMATTED(log, "Loading data parts"); + LOG_DEBUG(log, "Loading data parts"); const auto settings = getSettings(); std::vector> part_names_with_disks; @@ -900,7 +900,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) if (part_names_with_disks.empty()) { - LOG_DEBUG_FORMATTED(log, "There is no data parts"); + LOG_DEBUG(log, "There is no data parts"); return; } @@ -937,7 +937,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) String marker_path = part_path + "/" + DELETE_ON_DESTROY_MARKER_PATH; if (part_disk_ptr->exists(marker_path)) { - LOG_WARNING_FORMATTED(log, "Detaching stale part {}{}, which should have been deleted after a move. That can only happen after unclean restart of ClickHouse after move of a part having an operation blocking that stale copy of part.", getFullPathOnDisk(part_disk_ptr), part_name); + LOG_WARNING(log, "Detaching stale part {}{}, which should have been deleted after a move. That can only happen after unclean restart of ClickHouse after move of a part having an operation blocking that stale copy of part.", getFullPathOnDisk(part_disk_ptr), part_name); std::lock_guard loading_lock(mutex); broken_parts_to_detach.push_back(part); ++suspicious_broken_parts; @@ -971,7 +971,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) if (part->info.level == 0) { /// It is impossible to restore level 0 parts. - LOG_ERROR_FORMATTED(log, "Considering to remove broken part {}{} because it's impossible to repair.", getFullPathOnDisk(part_disk_ptr), part_name); + LOG_ERROR(log, "Considering to remove broken part {}{} because it's impossible to repair.", getFullPathOnDisk(part_disk_ptr), part_name); std::lock_guard loading_lock(mutex); broken_parts_to_remove.push_back(part); } @@ -982,7 +982,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) /// delete it. size_t contained_parts = 0; - LOG_ERROR_FORMATTED(log, "Part {}{} is broken. Looking for parts to replace it.", getFullPathOnDisk(part_disk_ptr), part_name); + LOG_ERROR(log, "Part {}{} is broken. Looking for parts to replace it.", getFullPathOnDisk(part_disk_ptr), part_name); for (const auto & [contained_name, contained_disk_ptr] : part_names_with_disks) { @@ -995,20 +995,20 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) if (part->info.contains(contained_part_info)) { - LOG_ERROR_FORMATTED(log, "Found part {}{}", getFullPathOnDisk(contained_disk_ptr), contained_name); + LOG_ERROR(log, "Found part {}{}", getFullPathOnDisk(contained_disk_ptr), contained_name); ++contained_parts; } } if (contained_parts >= 2) { - LOG_ERROR_FORMATTED(log, "Considering to remove broken part {}{} because it covers at least 2 other parts", getFullPathOnDisk(part_disk_ptr), part_name); + LOG_ERROR(log, "Considering to remove broken part {}{} because it covers at least 2 other parts", getFullPathOnDisk(part_disk_ptr), part_name); std::lock_guard loading_lock(mutex); broken_parts_to_remove.push_back(part); } else { - LOG_ERROR_FORMATTED(log, "Detaching broken part {}{} because it covers less than 2 parts. You need to resolve this manually", getFullPathOnDisk(part_disk_ptr), part_name); + LOG_ERROR(log, "Detaching broken part {}{} because it covers less than 2 parts. You need to resolve this manually", getFullPathOnDisk(part_disk_ptr), part_name); std::lock_guard loading_lock(mutex); broken_parts_to_detach.push_back(part); ++suspicious_broken_parts; @@ -1098,7 +1098,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) calculateColumnSizesImpl(); - LOG_DEBUG_FORMATTED(log, "Loaded data parts ({} items)", data_parts_indexes.size()); + LOG_DEBUG(log, "Loaded data parts ({} items)", data_parts_indexes.size()); } @@ -1142,7 +1142,7 @@ void MergeTreeData::clearOldTemporaryDirectories(ssize_t custom_directories_life { if (disk->isDirectory(it->path()) && isOldPartDirectory(disk, it->path(), deadline)) { - LOG_WARNING_FORMATTED(log, "Removing temporary directory {}", fullPath(disk, it->path())); + LOG_WARNING(log, "Removing temporary directory {}", fullPath(disk, it->path())); disk->removeRecursive(it->path()); } } @@ -1195,7 +1195,7 @@ MergeTreeData::DataPartsVector MergeTreeData::grabOldParts(bool force) } if (!res.empty()) - LOG_TRACE_FORMATTED(log, "Found {} old parts to remove.", res.size()); + LOG_TRACE(log, "Found {} old parts to remove.", res.size()); return res; } @@ -1279,7 +1279,7 @@ void MergeTreeData::clearPartsFromFilesystem(const DataPartsVector & parts_to_re { pool.scheduleOrThrowOnError([&] { - LOG_DEBUG_FORMATTED(log, "Removing part from filesystem {}", part->name); + LOG_DEBUG(log, "Removing part from filesystem {}", part->name); part->remove(); }); } @@ -1290,7 +1290,7 @@ void MergeTreeData::clearPartsFromFilesystem(const DataPartsVector & parts_to_re { for (const DataPartPtr & part : parts_to_remove) { - LOG_DEBUG_FORMATTED(log, "Removing part from filesystem {}", part->name); + LOG_DEBUG(log, "Removing part from filesystem {}", part->name); part->remove(); } } @@ -1321,11 +1321,11 @@ void MergeTreeData::rename(const String & new_table_path, const StorageID & new_ void MergeTreeData::dropAllData() { - LOG_TRACE_FORMATTED(log, "dropAllData: waiting for locks."); + LOG_TRACE(log, "dropAllData: waiting for locks."); auto lock = lockParts(); - LOG_TRACE_FORMATTED(log, "dropAllData: removing data from memory."); + LOG_TRACE(log, "dropAllData: removing data from memory."); DataPartsVector all_parts(data_parts_by_info.begin(), data_parts_by_info.end()); @@ -1334,7 +1334,7 @@ void MergeTreeData::dropAllData() global_context.dropCaches(); - LOG_TRACE_FORMATTED(log, "dropAllData: removing data from filesystem."); + LOG_TRACE(log, "dropAllData: removing data from filesystem."); /// Removing of each data part before recursive removal of directory is to speed-up removal, because there will be less number of syscalls. clearPartsFromFilesystem(all_parts); @@ -1342,7 +1342,7 @@ void MergeTreeData::dropAllData() for (const auto & [path, disk] : getRelativeDataPathsWithDisks()) disk->removeRecursive(path); - LOG_TRACE_FORMATTED(log, "dropAllData: done."); + LOG_TRACE(log, "dropAllData: done."); } namespace @@ -1682,7 +1682,7 @@ void MergeTreeData::PartsTemporaryRename::tryRenameAll() catch (...) { old_and_new_names.resize(i); - LOG_WARNING_FORMATTED(storage.log, "Cannot rename parts to perform operation on them: {}", getCurrentExceptionMessage(false)); + LOG_WARNING(storage.log, "Cannot rename parts to perform operation on them: {}", getCurrentExceptionMessage(false)); throw; } } @@ -1818,7 +1818,7 @@ void MergeTreeData::renameTempPartAndReplace( else part_name = part->name; - LOG_TRACE_FORMATTED(log, "Renaming temporary part {} to {}.", part->relative_path, part_name); + LOG_TRACE(log, "Renaming temporary part {} to {}.", part->relative_path, part_name); auto it_duplicate = data_parts_by_info.find(part_info); if (it_duplicate != data_parts_by_info.end()) @@ -1836,7 +1836,7 @@ void MergeTreeData::renameTempPartAndReplace( if (covering_part) { - LOG_WARNING_FORMATTED(log, "Tried to add obsolete part {} covered by {}", part_name, covering_part->getNameWithState()); + LOG_WARNING(log, "Tried to add obsolete part {} covered by {}", part_name, covering_part->getNameWithState()); return; } @@ -1948,7 +1948,7 @@ MergeTreeData::DataPartsVector MergeTreeData::removePartsInRangeFromWorkingSet(c if (!skip_intersecting_parts) throw Exception(error, ErrorCodes::LOGICAL_ERROR); - LOG_WARNING_FORMATTED(log, error); + LOG_WARNING(log, error); } continue; @@ -1965,7 +1965,7 @@ MergeTreeData::DataPartsVector MergeTreeData::removePartsInRangeFromWorkingSet(c if (!skip_intersecting_parts) throw Exception(error, ErrorCodes::LOGICAL_ERROR); - LOG_WARNING_FORMATTED(log, error); + LOG_WARNING(log, error); continue; } @@ -1981,7 +1981,7 @@ MergeTreeData::DataPartsVector MergeTreeData::removePartsInRangeFromWorkingSet(c void MergeTreeData::forgetPartAndMoveToDetached(const MergeTreeData::DataPartPtr & part_to_detach, const String & prefix, bool restore_covered) { - LOG_INFO_FORMATTED(log, "Renaming {} to {}{} and forgiving it.", part_to_detach->relative_path, prefix, part_to_detach->name); + LOG_INFO(log, "Renaming {} to {}{} and forgiving it.", part_to_detach->relative_path, prefix, part_to_detach->name); auto lock = lockParts(); @@ -2002,7 +2002,7 @@ restore_covered) if (restore_covered && part->info.level == 0) { - LOG_WARNING_FORMATTED(log, "Will not recover parts covered by zero-level part {}", part->name); + LOG_WARNING(log, "Will not recover parts covered by zero-level part {}", part->name); return; } @@ -2083,12 +2083,12 @@ restore_covered) for (const String & name : restored) { - LOG_INFO_FORMATTED(log, "Activated part {}", name); + LOG_INFO(log, "Activated part {}", name); } if (error) { - LOG_ERROR_FORMATTED(log, "The set of parts restored in place of {} looks incomplete. There might or might not be a data loss.{}", part->name, (error_parts.empty() ? "" : " Suspicious parts: " + error_parts)); + LOG_ERROR(log, "The set of parts restored in place of {} looks incomplete. There might or might not be a data loss.{}", part->name, (error_parts.empty() ? "" : " Suspicious parts: " + error_parts)); } } } @@ -2100,7 +2100,7 @@ void MergeTreeData::tryRemovePartImmediately(DataPartPtr && part) { auto lock = lockParts(); - LOG_TRACE_FORMATTED(log, "Trying to immediately remove part {}", part->getNameWithState()); + LOG_TRACE(log, "Trying to immediately remove part {}", part->getNameWithState()); auto it = data_parts_by_info.find(part->info); if (it == data_parts_by_info.end() || (*it).get() != part.get()) @@ -2126,7 +2126,7 @@ void MergeTreeData::tryRemovePartImmediately(DataPartPtr && part) } removePartsFinally({part_to_delete}); - LOG_TRACE_FORMATTED(log, "Removed part {}", part_to_delete->name); + LOG_TRACE(log, "Removed part {}", part_to_delete->name); } @@ -2241,7 +2241,7 @@ void MergeTreeData::delayInsertOrThrowIfNeeded(Poco::Event * until) const CurrentMetrics::Increment metric_increment(CurrentMetrics::DelayedInserts); - LOG_INFO_FORMATTED(log, "Delaying inserting block by {} ms. because there are {} parts", delay_milliseconds, parts_count_in_partition); + LOG_INFO(log, "Delaying inserting block by {} ms. because there are {} parts", delay_milliseconds, parts_count_in_partition); if (until) until->tryWait(delay_milliseconds); @@ -2319,7 +2319,7 @@ void MergeTreeData::swapActivePart(MergeTreeData::DataPartPtr part_copy) } catch (Poco::Exception & e) { - LOG_ERROR_FORMATTED(log, "{} (while creating DeleteOnDestroy marker: {})", e.what(), backQuote(fullPath(disk, marker_path))); + LOG_ERROR(log, "{} (while creating DeleteOnDestroy marker: {})", e.what(), backQuote(fullPath(disk, marker_path))); } return; } @@ -2439,7 +2439,7 @@ void MergeTreeData::removePartContributionToColumnSizes(const DataPartPtr & part auto log_subtract = [&](size_t & from, size_t value, const char * field) { if (value > from) - LOG_ERROR_FORMATTED(log, "Possibly incorrect column size subtraction: {} - {} = {}, column: {}, field: {}", + LOG_ERROR(log, "Possibly incorrect column size subtraction: {} - {} = {}, column: {}, field: {}", from, value, from - value, column.name, field); from -= value; @@ -2471,9 +2471,9 @@ void MergeTreeData::freezePartition(const ASTPtr & partition_ast, const String & partition_id = getPartitionIDFromQuery(partition_ast, context); if (prefix) - LOG_DEBUG_FORMATTED(log, "Freezing parts with prefix {}", *prefix); + LOG_DEBUG(log, "Freezing parts with prefix {}", *prefix); else - LOG_DEBUG_FORMATTED(log, "Freezing parts with partition ID {}", partition_id); + LOG_DEBUG(log, "Freezing parts with partition ID {}", partition_id); freezePartitionsByMatcher( @@ -2765,7 +2765,7 @@ void MergeTreeData::dropDetached(const ASTPtr & partition, bool part, const Cont renamed_parts.addPart(part_info.dir_name, "deleting_" + part_info.dir_name); } - LOG_DEBUG_FORMATTED(log, "Will drop {} detached parts.", renamed_parts.old_and_new_names.size()); + LOG_DEBUG(log, "Will drop {} detached parts.", renamed_parts.old_and_new_names.size()); renamed_parts.tryRenameAll(); @@ -2773,7 +2773,7 @@ void MergeTreeData::dropDetached(const ASTPtr & partition, bool part, const Cont { const auto & [path, disk] = renamed_parts.old_part_name_to_path_and_disk[old_name]; disk->removeRecursive(path + "detached/" + new_name + "/"); - LOG_DEBUG_FORMATTED(log, "Dropped detached part {}", old_name); + LOG_DEBUG(log, "Dropped detached part {}", old_name); old_name.clear(); } } @@ -2796,7 +2796,7 @@ MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const else { String partition_id = getPartitionIDFromQuery(partition, context); - LOG_DEBUG_FORMATTED(log, "Looking for parts for partition {} in {}", partition_id, source_dir); + LOG_DEBUG(log, "Looking for parts for partition {} in {}", partition_id, source_dir); ActiveDataPartSet active_parts(format_version); const auto disks = getStoragePolicy()->getDisks(); @@ -2813,12 +2813,12 @@ MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const { continue; } - LOG_DEBUG_FORMATTED(log, "Found part {}", name); + LOG_DEBUG(log, "Found part {}", name); active_parts.add(name); name_to_disk[name] = disk; } } - LOG_DEBUG_FORMATTED(log, "{} of them are active", active_parts.size()); + LOG_DEBUG(log, "{} of them are active", active_parts.size()); /// Inactive parts rename so they can not be attached in case of repeated ATTACH. for (const auto & [name, disk] : name_to_disk) { @@ -2838,12 +2838,12 @@ MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const renamed_parts.tryRenameAll(); /// Synchronously check that added parts exist and are not broken. We will write checksums.txt if it does not exist. - LOG_DEBUG_FORMATTED(log, "Checking parts"); + LOG_DEBUG(log, "Checking parts"); MutableDataPartsVector loaded_parts; loaded_parts.reserve(renamed_parts.old_and_new_names.size()); for (const auto & part_names : renamed_parts.old_and_new_names) { - LOG_DEBUG_FORMATTED(log, "Checking part {}", part_names.second); + LOG_DEBUG(log, "Checking part {}", part_names.second); auto single_disk_volume = std::make_shared("volume_" + part_names.first, name_to_disk[part_names.first]); MutableDataPartPtr part = createPart(part_names.first, single_disk_volume, source_dir + part_names.second); loadPartAndFixMetadataImpl(part); @@ -2915,9 +2915,9 @@ ReservationPtr MergeTreeData::tryReserveSpacePreferringTTLRules(UInt64 expected_ if (!destination_ptr) { if (ttl_entry->destination_type == PartDestinationType::VOLUME) - LOG_WARNING_FORMATTED(log, "Would like to reserve space on volume '{}' by TTL rule of table '{}' but volume was not found", ttl_entry->destination_name, log_name); + LOG_WARNING(log, "Would like to reserve space on volume '{}' by TTL rule of table '{}' but volume was not found", ttl_entry->destination_name, log_name); else if (ttl_entry->destination_type == PartDestinationType::DISK) - LOG_WARNING_FORMATTED(log, "Would like to reserve space on disk '{}' by TTL rule of table '{}' but disk was not found", ttl_entry->destination_name, log_name); + LOG_WARNING(log, "Would like to reserve space on disk '{}' by TTL rule of table '{}' but disk was not found", ttl_entry->destination_name, log_name); } else { @@ -2926,9 +2926,9 @@ ReservationPtr MergeTreeData::tryReserveSpacePreferringTTLRules(UInt64 expected_ return reservation; else if (ttl_entry->destination_type == PartDestinationType::VOLUME) - LOG_WARNING_FORMATTED(log, "Would like to reserve space on volume '{}' by TTL rule of table '{}' but there is not enough space", ttl_entry->destination_name, log_name); + LOG_WARNING(log, "Would like to reserve space on volume '{}' by TTL rule of table '{}' but there is not enough space", ttl_entry->destination_name, log_name); else if (ttl_entry->destination_type == PartDestinationType::DISK) - LOG_WARNING_FORMATTED(log, "Would like to reserve space on disk '{}' by TTL rule of table '{}' but there is not enough space", ttl_entry->destination_name, log_name); + LOG_WARNING(log, "Would like to reserve space on disk '{}' by TTL rule of table '{}' but there is not enough space", ttl_entry->destination_name, log_name); } } @@ -3028,7 +3028,7 @@ void MergeTreeData::Transaction::rollback() for (const auto & part : precommitted_parts) ss << " " << part->relative_path; ss << "."; - LOG_DEBUG_FORMATTED(data.log, "Undoing transaction.{}", ss.str()); + LOG_DEBUG(data.log, "Undoing transaction.{}", ss.str()); data.removePartsFromWorkingSet( DataPartsVector(precommitted_parts.begin(), precommitted_parts.end()), @@ -3054,7 +3054,7 @@ MergeTreeData::DataPartsVector MergeTreeData::Transaction::commit(MergeTreeData: DataPartsVector covered_parts = data.getActivePartsToReplace(part->info, part->name, covering_part, *owing_parts_lock); if (covering_part) { - LOG_WARNING_FORMATTED(data.log, "Tried to commit obsolete part {} covered by {}", part->name, covering_part->getNameWithState()); + LOG_WARNING(data.log, "Tried to commit obsolete part {} covered by {}", part->name, covering_part->getNameWithState()); part->remove_time.store(0, std::memory_order_relaxed); /// The part will be removed without waiting for old_parts_lifetime seconds. data.modifyPartState(part, DataPartState::Outdated); @@ -3190,7 +3190,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::cloneAndLoadDataPartOnSameDisk( if (disk->exists(dst_part_path)) throw Exception("Part in " + fullPath(disk, dst_part_path) + " already exists", ErrorCodes::DIRECTORY_ALREADY_EXISTS); - LOG_DEBUG_FORMATTED(log, "Cloning part {} to {}", fullPath(disk, src_part_path), fullPath(disk, dst_part_path)); + LOG_DEBUG(log, "Cloning part {} to {}", fullPath(disk, src_part_path), fullPath(disk, dst_part_path)); localBackup(disk, src_part_path, dst_part_path); disk->removeIfExists(dst_part_path + "/" + DELETE_ON_DESTROY_MARKER_PATH); @@ -3275,7 +3275,7 @@ void MergeTreeData::freezePartitionsByMatcher(MatcherFn matcher, const String & : toString(increment)) + "/"; - LOG_DEBUG_FORMATTED(log, "Freezing part {} snapshot will be placed at {}", part->name, backup_path); + LOG_DEBUG(log, "Freezing part {} snapshot will be placed at {}", part->name, backup_path); String backup_part_path = backup_path + relative_data_path + part->relative_path; localBackup(part->volume->getDisk(), part->getFullRelativePath(), backup_part_path); @@ -3285,7 +3285,7 @@ void MergeTreeData::freezePartitionsByMatcher(MatcherFn matcher, const String & ++parts_processed; } - LOG_DEBUG_FORMATTED(log, "Freezed {} parts", parts_processed); + LOG_DEBUG(log, "Freezed {} parts", parts_processed); } bool MergeTreeData::canReplacePartition(const DataPartPtr & src_part) const @@ -3471,7 +3471,7 @@ MergeTreeData::CurrentlyMovingPartsTagger MergeTreeData::checkPartsForMove(const bool MergeTreeData::moveParts(CurrentlyMovingPartsTagger && moving_tagger) { - LOG_INFO_FORMATTED(log, "Got {} parts to move.", moving_tagger.parts_to_move.size()); + LOG_INFO(log, "Got {} parts to move.", moving_tagger.parts_to_move.size()); for (const auto & moving_part : moving_tagger.parts_to_move) { diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 9a570664f15..12f577eef25 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -268,7 +268,7 @@ bool MergeTreeDataMergerMutator::selectPartsToMerge( if (prev_part && part->info.partition_id == (*prev_part)->info.partition_id && part->info.min_block <= (*prev_part)->info.max_block) { - LOG_ERROR_FORMATTED(log, "Part {} intersects previous part {}", part->name, (*prev_part)->name); + LOG_ERROR(log, "Part {} intersects previous part {}", part->name, (*prev_part)->name); } prev_part = ∂ @@ -315,7 +315,7 @@ bool MergeTreeDataMergerMutator::selectPartsToMerge( parts.push_back(part); } - LOG_DEBUG_FORMATTED(log, "Selected {} parts from {} to {}", parts.size(), parts.front()->name, parts.back()->name); + LOG_DEBUG(log, "Selected {} parts from {} to {}", parts.size(), parts.front()->name, parts.back()->name); future_part.assign(std::move(parts)); return true; } @@ -366,7 +366,7 @@ bool MergeTreeDataMergerMutator::selectAllPartsToMergeWithinPartition( if (now - disk_space_warning_time > 3600) { disk_space_warning_time = now; - LOG_WARNING_FORMATTED(log, + LOG_WARNING(log, "Won't merge parts from {} to {} because not enough free space: {} free and unreserved" ", {} required now (+{}% on overhead); suppressing similar warnings for the next hour", parts.front()->name, @@ -383,7 +383,7 @@ bool MergeTreeDataMergerMutator::selectAllPartsToMergeWithinPartition( return false; } - LOG_DEBUG_FORMATTED(log, "Selected {} parts from {} to {}", parts.size(), parts.front()->name, parts.back()->name); + LOG_DEBUG(log, "Selected {} parts from {} to {}", parts.size(), parts.front()->name, parts.back()->name); future_part.assign(std::move(parts)); available_disk_space -= required_disk_space; return true; @@ -590,7 +590,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor const MergeTreeData::DataPartsVector & parts = future_part.parts; - LOG_DEBUG_FORMATTED(log, "Merging {} parts: from {} to {} into {}", parts.size(), parts.front()->name, parts.back()->name, future_part.type.toString()); + LOG_DEBUG(log, "Merging {} parts: from {} to {} into {}", parts.size(), parts.front()->name, parts.back()->name, future_part.type.toString()); auto disk = space_reservation->getDisk(); String part_path = data.relative_data_path; @@ -633,14 +633,14 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor if (need_remove_expired_values && ttl_merges_blocker.isCancelled()) { - LOG_INFO_FORMATTED(log, "Part {} has values with expired TTL, but merges with TTL are cancelled.", new_data_part->name); + LOG_INFO(log, "Part {} has values with expired TTL, but merges with TTL are cancelled.", new_data_part->name); need_remove_expired_values = false; } size_t sum_input_rows_upper_bound = merge_entry->total_rows_count; MergeAlgorithm merge_alg = chooseMergeAlgorithm(parts, sum_input_rows_upper_bound, gathering_columns, deduplicate, need_remove_expired_values); - LOG_DEBUG_FORMATTED(log, "Selected MergeAlgorithm: {}", ((merge_alg == MergeAlgorithm::Vertical) ? "Vertical" : "Horizontal")); + LOG_DEBUG(log, "Selected MergeAlgorithm: {}", ((merge_alg == MergeAlgorithm::Vertical) ? "Vertical" : "Horizontal")); /// Note: this is done before creating input streams, because otherwise data.data_parts_mutex /// (which is locked in data.getTotalActiveSizeInBytes()) @@ -694,7 +694,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor total_size += part->getBytesOnDisk(); if (total_size >= data_settings->min_merge_bytes_to_use_direct_io) { - LOG_DEBUG_FORMATTED(log, "Will merge parts reading files in O_DIRECT"); + LOG_DEBUG(log, "Will merge parts reading files in O_DIRECT"); read_with_direct_io = true; break; @@ -949,7 +949,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor /// Print overall profiling info. NOTE: it may duplicates previous messages { double elapsed_seconds = merge_entry->watch.elapsedSeconds(); - LOG_DEBUG_FORMATTED(log, "Merge sorted {} rows, containing {} columns ({} merged, {} gathered) in {} sec., {} rows/sec., {}/sec.", merge_entry->rows_read, all_column_names.size(), merging_column_names.size(), gathering_column_names.size(), elapsed_seconds, merge_entry->rows_read / elapsed_seconds, formatReadableSizeWithBinarySuffix(merge_entry->bytes_read_uncompressed / elapsed_seconds)); + LOG_DEBUG(log, "Merge sorted {} rows, containing {} columns ({} merged, {} gathered) in {} sec., {} rows/sec., {}/sec.", merge_entry->rows_read, all_column_names.size(), merging_column_names.size(), gathering_column_names.size(), elapsed_seconds, merge_entry->rows_read / elapsed_seconds, formatReadableSizeWithBinarySuffix(merge_entry->bytes_read_uncompressed / elapsed_seconds)); } if (merge_alg != MergeAlgorithm::Vertical) @@ -994,12 +994,12 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor if (!isStorageTouchedByMutations(storage_from_source_part, commands_for_part, context_for_reading)) { - LOG_TRACE_FORMATTED(log, "Part {} doesn't change up to mutation version {}", source_part->name, future_part.part_info.mutation); + LOG_TRACE(log, "Part {} doesn't change up to mutation version {}", source_part->name, future_part.part_info.mutation); return data.cloneAndLoadDataPartOnSameDisk(source_part, "tmp_clone_", future_part.part_info); } else { - LOG_TRACE_FORMATTED(log, "Mutating part {} to mutation version {}", source_part->name, future_part.part_info.mutation); + LOG_TRACE(log, "Mutating part {} to mutation version {}", source_part->name, future_part.part_info.mutation); } BlockInputStreamPtr in = nullptr; @@ -1221,7 +1221,7 @@ MergeTreeData::DataPartPtr MergeTreeDataMergerMutator::renameMergedTemporaryPart * (NOTE: Merging with part that is not in ZK is not possible, see checks in 'createLogEntryToMergeParts'.) * - and after merge, this part will be removed in addition to parts that was merged. */ - LOG_WARNING_FORMATTED(log, "Unexpected number of parts removed when adding {}: {} instead of {}", new_data_part->name, replaced_parts.size(), parts.size()); + LOG_WARNING(log, "Unexpected number of parts removed when adding {}: {} instead of {}", new_data_part->name, replaced_parts.size(), parts.size()); } else { @@ -1231,7 +1231,7 @@ MergeTreeData::DataPartPtr MergeTreeDataMergerMutator::renameMergedTemporaryPart + " instead of " + parts[i]->name, ErrorCodes::LOGICAL_ERROR); } - LOG_TRACE_FORMATTED(log, "Merged {} parts: from {} to {}", parts.size(), parts.front()->name, parts.back()->name); + LOG_TRACE(log, "Merged {} parts: from {} to {}", parts.size(), parts.front()->name, parts.back()->name); return new_data_part; } diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 005e7364b8d..cc698797366 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -107,7 +107,7 @@ size_t MergeTreeDataSelectExecutor::getApproximateTotalRowsToRead( size_t rows_count = 0; /// We will find out how many rows we would have read without sampling. - LOG_DEBUG_FORMATTED(log, "Preliminary index scan with condition: {}", key_condition.toString()); + LOG_DEBUG(log, "Preliminary index scan with condition: {}", key_condition.toString()); for (const auto & part : parts) { @@ -329,7 +329,7 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( if (relative_sample_size > 1) { relative_sample_size = convertAbsoluteSampleSizeToRelative(select_sample_size, approx_total_rows); - LOG_DEBUG_FORMATTED(log, "Selected relative sample size: {}", toString(relative_sample_size)); + LOG_DEBUG(log, "Selected relative sample size: {}", toString(relative_sample_size)); } /// SAMPLE 1 is the same as the absence of SAMPLE. @@ -342,7 +342,7 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( if (relative_sample_offset > 1) { relative_sample_offset = convertAbsoluteSampleSizeToRelative(select_sample_offset, approx_total_rows); - LOG_DEBUG_FORMATTED(log, "Selected relative sample offset: {}", toString(relative_sample_offset)); + LOG_DEBUG(log, "Selected relative sample offset: {}", toString(relative_sample_offset)); } } @@ -530,13 +530,13 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( if (no_data) { - LOG_DEBUG_FORMATTED(log, "Sampling yields no data."); + LOG_DEBUG(log, "Sampling yields no data."); return {}; } - LOG_DEBUG_FORMATTED(log, "Key condition: {}", key_condition.toString()); + LOG_DEBUG(log, "Key condition: {}", key_condition.toString()); if (minmax_idx_condition) - LOG_DEBUG_FORMATTED(log, "MinMax index condition: {}", minmax_idx_condition->toString()); + LOG_DEBUG(log, "MinMax index condition: {}", minmax_idx_condition->toString()); /// PREWHERE String prewhere_column; @@ -586,7 +586,7 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( } } - LOG_DEBUG_FORMATTED(log, "Selected {} parts by date, {} parts by key, {} marks to read from {} ranges", parts.size(), parts_with_ranges.size(), sum_marks, sum_ranges); + LOG_DEBUG(log, "Selected {} parts by date, {} parts by key, {} marks to read from {} ranges", parts.size(), parts_with_ranges.size(), sum_marks, sum_ranges); if (parts_with_ranges.empty()) return {}; @@ -785,7 +785,7 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams( column_names, MergeTreeReadPool::BackoffSettings(settings), settings.preferred_block_size_bytes, false); /// Let's estimate total number of rows for progress bar. - LOG_TRACE_FORMATTED(log, "Reading approx. {} rows with {} streams", total_rows, num_streams); + LOG_TRACE(log, "Reading approx. {} rows with {} streams", total_rows, num_streams); for (size_t i = 0; i < num_streams; ++i) { @@ -1382,7 +1382,7 @@ MarkRanges MergeTreeDataSelectExecutor::filterMarksUsingIndex( { if (!part->volume->getDisk()->exists(part->getFullRelativePath() + index->getFileName() + ".idx")) { - LOG_DEBUG_FORMATTED(log, "File for index {} does not exist. Skipping it.", backQuote(index->name)); + LOG_DEBUG(log, "File for index {} does not exist. Skipping it.", backQuote(index->name)); return ranges; } @@ -1442,7 +1442,7 @@ MarkRanges MergeTreeDataSelectExecutor::filterMarksUsingIndex( last_index_mark = index_range.end - 1; } - LOG_DEBUG_FORMATTED(log, "Index {} has dropped {} granules.", backQuote(index->name), granules_dropped); + LOG_DEBUG(log, "Index {} has dropped {} granules.", backQuote(index->name), granules_dropped); return res; } diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 98ca6613134..d873790d91c 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -252,7 +252,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa if (new_data_part->volume->getDisk()->exists(full_path)) { - LOG_WARNING_FORMATTED(log, "Removing old temporary directory {}", fullPath(new_data_part->volume->getDisk(), full_path)); + LOG_WARNING(log, "Removing old temporary directory {}", fullPath(new_data_part->volume->getDisk(), full_path)); new_data_part->volume->getDisk()->removeRecursive(full_path); } diff --git a/src/Storages/MergeTree/MergeTreePartsMover.cpp b/src/Storages/MergeTree/MergeTreePartsMover.cpp index 464c469ad7b..ae66e40e042 100644 --- a/src/Storages/MergeTree/MergeTreePartsMover.cpp +++ b/src/Storages/MergeTree/MergeTreePartsMover.cpp @@ -179,7 +179,7 @@ bool MergeTreePartsMover::selectPartsForMove( if (!parts_to_move.empty()) { - LOG_TRACE_FORMATTED(log, "Selected {} parts to move according to storage policy rules and {} parts according to TTL rules, {} total", parts_to_move_by_policy_rules, parts_to_move_by_ttl_rules, formatReadableSizeWithBinarySuffix(parts_to_move_total_size_bytes)); + LOG_TRACE(log, "Selected {} parts to move according to storage policy rules and {} parts according to TTL rules, {} total", parts_to_move_by_policy_rules, parts_to_move_by_ttl_rules, formatReadableSizeWithBinarySuffix(parts_to_move_total_size_bytes)); return true; } else @@ -191,13 +191,13 @@ MergeTreeData::DataPartPtr MergeTreePartsMover::clonePart(const MergeTreeMoveEnt if (moves_blocker.isCancelled()) throw Exception("Cancelled moving parts.", ErrorCodes::ABORTED); - LOG_TRACE_FORMATTED(log, "Cloning part {}", moving_part.part->name); + LOG_TRACE(log, "Cloning part {}", moving_part.part->name); moving_part.part->makeCloneOnDiskDetached(moving_part.reserved_space); auto single_disk_volume = std::make_shared("volume_" + moving_part.part->name, moving_part.reserved_space->getDisk()); MergeTreeData::MutableDataPartPtr cloned_part = data->createPart(moving_part.part->name, single_disk_volume, "detached/" + moving_part.part->name); - LOG_TRACE_FORMATTED(log, "Part {} was cloned to {}", moving_part.part->name, cloned_part->getFullPath()); + LOG_TRACE(log, "Part {} was cloned to {}", moving_part.part->name, cloned_part->getFullPath()); cloned_part->loadColumnsChecksumsIndexes(true, true); return cloned_part; @@ -215,7 +215,7 @@ void MergeTreePartsMover::swapClonedPart(const MergeTreeData::DataPartPtr & clon /// It's ok, because we don't block moving parts for merges or mutations if (!active_part || active_part->name != cloned_part->name) { - LOG_INFO_FORMATTED(log, "Failed to swap {}. Active part doesn't exist. Possible it was merged or mutated. Will remove copy on path '{}'.", cloned_part->name, cloned_part->getFullPath()); + LOG_INFO(log, "Failed to swap {}. Active part doesn't exist. Possible it was merged or mutated. Will remove copy on path '{}'.", cloned_part->name, cloned_part->getFullPath()); return; } @@ -225,7 +225,7 @@ void MergeTreePartsMover::swapClonedPart(const MergeTreeData::DataPartPtr & clon /// TODO what happen if server goes down here? data->swapActivePart(cloned_part); - LOG_TRACE_FORMATTED(log, "Part {} was moved to {}", cloned_part->name, cloned_part->getFullPath()); + LOG_TRACE(log, "Part {} was moved to {}", cloned_part->name, cloned_part->getFullPath()); } } diff --git a/src/Storages/MergeTree/MergeTreeReadPool.cpp b/src/Storages/MergeTree/MergeTreeReadPool.cpp index 1fcabfafcab..b266b65e19e 100644 --- a/src/Storages/MergeTree/MergeTreeReadPool.cpp +++ b/src/Storages/MergeTree/MergeTreeReadPool.cpp @@ -167,7 +167,7 @@ void MergeTreeReadPool::profileFeedback(const ReadBufferFromFileBase::ProfileInf ++backoff_state.num_events; ProfileEvents::increment(ProfileEvents::SlowRead); - LOG_DEBUG_FORMATTED(log, "Slow read, event №{}: read {} bytes in {} sec., {}/s.", + LOG_DEBUG(log, "Slow read, event №{}: read {} bytes in {} sec., {}/s.", backoff_state.num_events, info.bytes_read, info.nanoseconds / 1e9, formatReadableSizeWithBinarySuffix(throughput)); @@ -178,7 +178,7 @@ void MergeTreeReadPool::profileFeedback(const ReadBufferFromFileBase::ProfileInf --backoff_state.current_threads; ProfileEvents::increment(ProfileEvents::ReadBackoff); - LOG_DEBUG_FORMATTED(log, "Will lower number of threads to {}", backoff_state.current_threads); + LOG_DEBUG(log, "Will lower number of threads to {}", backoff_state.current_threads); } diff --git a/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp index f59543ac254..09bf784a293 100644 --- a/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp @@ -66,7 +66,7 @@ MergeTreeReverseSelectProcessor::MergeTreeReverseSelectProcessor( size_t total_rows = data_part->index_granularity.getTotalRows(); if (!quiet) - LOG_TRACE_FORMATTED(log, "Reading {} ranges in reverse order from part {}, approx. {}, up to {} rows starting from {}", + LOG_TRACE(log, "Reading {} ranges in reverse order from part {}, approx. {}, up to {} rows starting from {}", all_mark_ranges.size(), data_part->name, total_rows, data_part->index_granularity.getRowsCountInRanges(all_mark_ranges), data_part->index_granularity.getMarkStartingRow(all_mark_ranges.front().begin)); diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp index 051989108d0..df471a8b8ec 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp @@ -47,7 +47,7 @@ MergeTreeSelectProcessor::MergeTreeSelectProcessor( size_t total_rows = data_part->index_granularity.getRowsCountInRanges(all_mark_ranges); if (!quiet) - LOG_TRACE_FORMATTED(log, "Reading {} ranges from part {}, approx. {} rows starting from {}", + LOG_TRACE(log, "Reading {} ranges from part {}, approx. {} rows starting from {}", all_mark_ranges.size(), data_part->name, total_rows, data_part->index_granularity.getMarkStartingRow(all_mark_ranges.front().begin)); diff --git a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp index b27d8bcfa97..045962f44dd 100644 --- a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp +++ b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp @@ -27,10 +27,10 @@ MergeTreeSequentialSource::MergeTreeSequentialSource( { /// Print column name but don't pollute logs in case of many columns. if (columns_to_read.size() == 1) - LOG_TRACE_FORMATTED(log, "Reading {} marks from part {}, total {} rows starting from the beginning of the part, column {}", + LOG_TRACE(log, "Reading {} marks from part {}, total {} rows starting from the beginning of the part, column {}", data_part->getMarksCount(), data_part->name, data_part->rows_count, columns_to_read.front()); else - LOG_TRACE_FORMATTED(log, "Reading {} marks from part {}, total {} rows starting from the beginning of the part", + LOG_TRACE(log, "Reading {} marks from part {}, total {} rows starting from the beginning of the part", data_part->getMarksCount(), data_part->name, data_part->rows_count); } diff --git a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp index b7388f52340..69450f0ac4c 100644 --- a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp +++ b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp @@ -238,7 +238,7 @@ void MergeTreeWhereOptimizer::optimize(ASTSelectQuery & select) const select.setExpression(ASTSelectQuery::Expression::WHERE, reconstruct(where_conditions)); select.setExpression(ASTSelectQuery::Expression::PREWHERE, reconstruct(prewhere_conditions)); - LOG_DEBUG_FORMATTED(log, "MergeTreeWhereOptimizer: condition \"{}\" moved to PREWHERE", select.prewhere()); + LOG_DEBUG(log, "MergeTreeWhereOptimizer: condition \"{}\" moved to PREWHERE", select.prewhere()); } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp index 709976005f2..d5e44166536 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp @@ -147,11 +147,11 @@ void ReplicatedMergeTreeBlockOutputStream::write(const Block & block) /// That is, do not insert the same data to the same partition twice. block_id = part->info.partition_id + "_" + toString(hash_value.words[0]) + "_" + toString(hash_value.words[1]); - LOG_DEBUG_FORMATTED(log, "Wrote block with ID '{}', {} rows", block_id, current_block.block.rows()); + LOG_DEBUG(log, "Wrote block with ID '{}', {} rows", block_id, current_block.block.rows()); } else { - LOG_DEBUG_FORMATTED(log, "Wrote block with {} rows", current_block.block.rows()); + LOG_DEBUG(log, "Wrote block with {} rows", current_block.block.rows()); } try @@ -214,7 +214,7 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zoo if (!block_number_lock) { - LOG_INFO_FORMATTED(log, "Block with ID {} already exists; ignoring it.", block_id); + LOG_INFO(log, "Block with ID {} already exists; ignoring it.", block_id); part->is_duplicate = true; last_block_is_duplicate = true; ProfileEvents::increment(ProfileEvents::DuplicatedInsertedBlocks); @@ -329,7 +329,7 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zoo if (multi_code == Coordination::ZNODEEXISTS && deduplicate_block && failed_op_path == block_id_path) { /// Block with the same id have just appeared in table (or other replica), rollback thee insertion. - LOG_INFO_FORMATTED(log, "Block with ID {} already exists; ignoring it (removing part {})", block_id, part->name); + LOG_INFO(log, "Block with ID {} already exists; ignoring it (removing part {})", block_id, part->name); part->is_duplicate = true; transaction.rollback(); @@ -367,7 +367,7 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zoo if (quorum) { /// We are waiting for quorum to be satisfied. - LOG_TRACE_FORMATTED(log, "Waiting for quorum"); + LOG_TRACE(log, "Waiting for quorum"); String quorum_status_path = storage.zookeeper_path + "/quorum/status"; @@ -406,7 +406,7 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zoo ErrorCodes::UNKNOWN_STATUS_OF_INSERT); } - LOG_TRACE_FORMATTED(log, "Quorum satisfied"); + LOG_TRACE(log, "Quorum satisfied"); } } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp index 3c911fbaa0d..5fc2e2e3417 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp @@ -234,7 +234,7 @@ void ReplicatedMergeTreeCleanupThread::clearOldLogs() } } - LOG_DEBUG_FORMATTED(log, "Removed {} old log entries: {} - {}", entries.size(), entries.front(), entries.back()); + LOG_DEBUG(log, "Removed {} old log entries: {} - {}", entries.size(), entries.front(), entries.back()); } @@ -327,7 +327,7 @@ void ReplicatedMergeTreeCleanupThread::clearOldBlocks() cached_block_stats.erase(first_outdated_block->node); } else if (rc) - LOG_WARNING_FORMATTED(log, "Error while deleting ZooKeeper path `{}`: {}, ignoring.", path, zkutil::ZooKeeper::error2string(rc)); + LOG_WARNING(log, "Error while deleting ZooKeeper path `{}`: {}, ignoring.", path, zkutil::ZooKeeper::error2string(rc)); else { /// Successfully removed blocks have to be removed from cache @@ -338,7 +338,7 @@ void ReplicatedMergeTreeCleanupThread::clearOldBlocks() auto num_nodes_to_delete = timed_blocks.end() - first_outdated_block; if (num_nodes_to_delete) - LOG_TRACE_FORMATTED(log, "Cleared {} old blocks from ZooKeeper", num_nodes_to_delete); + LOG_TRACE(log, "Cleared {} old blocks from ZooKeeper", num_nodes_to_delete); } @@ -368,7 +368,7 @@ void ReplicatedMergeTreeCleanupThread::getBlocksSortedByTime(zkutil::ZooKeeper & auto not_cached_blocks = stat.numChildren - cached_block_stats.size(); if (not_cached_blocks) { - LOG_TRACE_FORMATTED(log, "Checking {} blocks ({} are not cached){}", stat.numChildren, not_cached_blocks, " to clear old ones from ZooKeeper."); + LOG_TRACE(log, "Checking {} blocks ({} are not cached){}", stat.numChildren, not_cached_blocks, " to clear old ones from ZooKeeper."); } zkutil::AsyncResponses exists_futures; @@ -454,7 +454,7 @@ void ReplicatedMergeTreeCleanupThread::clearOldMutations() /// Simultaneously with clearing the log, we check to see if replica was added since we received replicas list. ops.emplace_back(zkutil::makeCheckRequest(storage.zookeeper_path + "/replicas", replicas_stat.version)); zookeeper->multi(ops); - LOG_DEBUG_FORMATTED(log, "Removed {} old mutation entries: {} - {}", (i + 1 - batch_start_i), entries[batch_start_i], entries[i]); + LOG_DEBUG(log, "Removed {} old mutation entries: {} - {}", (i + 1 - batch_start_i), entries[batch_start_i], entries[i]); batch_start_i = i + 1; ops.clear(); } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp index f3c24d9f8fb..eeb6b9bddd7 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp @@ -82,7 +82,7 @@ void ReplicatedMergeTreePartCheckThread::searchForMissingPart(const String & par /// If the part is in ZooKeeper, remove it from there and add the task to download it to the queue. if (zookeeper->exists(part_path)) { - LOG_WARNING_FORMATTED(log, "Part {} exists in ZooKeeper but not locally. Removing from ZooKeeper and queueing a fetch.", part_name); + LOG_WARNING(log, "Part {} exists in ZooKeeper but not locally. Removing from ZooKeeper and queueing a fetch.", part_name); ProfileEvents::increment(ProfileEvents::ReplicatedPartChecksFailed); storage.removePartAndEnqueueFetch(part_name); @@ -115,7 +115,7 @@ void ReplicatedMergeTreePartCheckThread::searchForMissingPart(const String & par * and don't delete the queue entry when in doubt. */ - LOG_WARNING_FORMATTED(log, "Checking if anyone has a part covering {}.", part_name); + LOG_WARNING(log, "Checking if anyone has a part covering {}.", part_name); bool found_part_with_the_same_min_block = false; bool found_part_with_the_same_max_block = false; @@ -130,7 +130,7 @@ void ReplicatedMergeTreePartCheckThread::searchForMissingPart(const String & par if (part_on_replica_info.contains(part_info)) { - LOG_WARNING_FORMATTED(log, "Found part {} on {} that covers the missing part {}", part_on_replica, replica, part_name); + LOG_WARNING(log, "Found part {} on {} that covers the missing part {}", part_on_replica, replica, part_name); return; } @@ -143,7 +143,7 @@ void ReplicatedMergeTreePartCheckThread::searchForMissingPart(const String & par if (found_part_with_the_same_min_block && found_part_with_the_same_max_block) { - LOG_WARNING_FORMATTED(log, "Found parts with the same min block and with the same max block as the missing part {}. Hoping that it will eventually appear as a result of a merge.", part_name); + LOG_WARNING(log, "Found parts with the same min block and with the same max block as the missing part {}. Hoping that it will eventually appear as a result of a merge.", part_name); return; } } @@ -158,7 +158,7 @@ void ReplicatedMergeTreePartCheckThread::searchForMissingPart(const String & par not_found_msg = "a smaller part with the same min block."; else not_found_msg = "smaller parts with either the same min block or the same max block."; - LOG_ERROR_FORMATTED(log, "No replica has part covering {} and a merge is impossible: we didn't find {}", part_name, not_found_msg); + LOG_ERROR(log, "No replica has part covering {} and a merge is impossible: we didn't find {}", part_name, not_found_msg); ProfileEvents::increment(ProfileEvents::ReplicatedPartChecksFailed); @@ -166,21 +166,21 @@ void ReplicatedMergeTreePartCheckThread::searchForMissingPart(const String & par if (!storage.queue.remove(zookeeper, part_name)) { /// The part was not in our queue. Why did it happen? - LOG_ERROR_FORMATTED(log, "Missing part {} is not in our queue.", part_name); + LOG_ERROR(log, "Missing part {} is not in our queue.", part_name); return; } /** This situation is possible if on all the replicas where the part was, it deteriorated. * For example, a replica that has just written it has power turned off and the data has not been written from cache to disk. */ - LOG_ERROR_FORMATTED(log, "Part {} is lost forever.", part_name); + LOG_ERROR(log, "Part {} is lost forever.", part_name); ProfileEvents::increment(ProfileEvents::ReplicatedDataLoss); } CheckResult ReplicatedMergeTreePartCheckThread::checkPart(const String & part_name) { - LOG_WARNING_FORMATTED(log, "Checking part {}", part_name); + LOG_WARNING(log, "Checking part {}", part_name); ProfileEvents::increment(ProfileEvents::ReplicatedPartChecks); /// If the part is still in the PreCommitted -> Committed transition, it is not lost @@ -212,7 +212,7 @@ CheckResult ReplicatedMergeTreePartCheckThread::checkPart(const String & part_na /// If the part is in ZooKeeper, check its data with its checksums, and them with ZooKeeper. if (zookeeper->tryGet(part_path, part_znode)) { - LOG_WARNING_FORMATTED(log, "Checking data of part {}.", part_name); + LOG_WARNING(log, "Checking data of part {}.", part_name); try { @@ -239,11 +239,11 @@ CheckResult ReplicatedMergeTreePartCheckThread::checkPart(const String & part_na if (need_stop) { - LOG_INFO_FORMATTED(log, "Checking part was cancelled."); + LOG_INFO(log, "Checking part was cancelled."); return {part_name, false, "Checking part was cancelled"}; } - LOG_INFO_FORMATTED(log, "Part {} looks good.", part_name); + LOG_INFO(log, "Part {} looks good.", part_name); } catch (const Exception & e) { @@ -256,7 +256,7 @@ CheckResult ReplicatedMergeTreePartCheckThread::checkPart(const String & part_na tryLogCurrentException(log, __PRETTY_FUNCTION__); String message = "Part " + part_name + " looks broken. Removing it and queueing a fetch."; - LOG_ERROR_FORMATTED(log, message); + LOG_ERROR(log, message); ProfileEvents::increment(ProfileEvents::ReplicatedPartChecksFailed); storage.removePartAndEnqueueFetch(part_name); @@ -274,7 +274,7 @@ CheckResult ReplicatedMergeTreePartCheckThread::checkPart(const String & part_na ProfileEvents::increment(ProfileEvents::ReplicatedPartChecksFailed); String message = "Unexpected part " + part_name + " in filesystem. Removing."; - LOG_ERROR_FORMATTED(log, message); + LOG_ERROR(log, message); storage.forgetPartAndMoveToDetached(part, "unexpected"); return {part_name, false, message}; } @@ -285,14 +285,14 @@ CheckResult ReplicatedMergeTreePartCheckThread::checkPart(const String & part_na /// but remained in the filesystem and in a number of active parts. /// And then for a long time (before restarting), the data on the replicas will be different. - LOG_TRACE_FORMATTED(log, "Young part {} with age {} seconds hasn't been added to ZooKeeper yet. It's ok.", part_name, (time(nullptr) - part->modification_time)); + LOG_TRACE(log, "Young part {} with age {} seconds hasn't been added to ZooKeeper yet. It's ok.", part_name, (time(nullptr) - part->modification_time)); } } else { /// If we have a covering part, ignore all the problems with this part. /// In the worst case, errors will still appear `old_parts_lifetime` seconds in error log until the part is removed as the old one. - LOG_WARNING_FORMATTED(log, "We have part {} covering part {}", part->name, part_name); + LOG_WARNING(log, "We have part {} covering part {}", part->name, part_name); } return {part_name, true, ""}; @@ -319,7 +319,7 @@ void ReplicatedMergeTreePartCheckThread::run() { if (!parts_set.empty()) { - LOG_ERROR_FORMATTED(log, "Non-empty parts_set with empty parts_queue. This is a bug."); + LOG_ERROR(log, "Non-empty parts_set with empty parts_queue. This is a bug."); parts_set.clear(); } } @@ -353,7 +353,7 @@ void ReplicatedMergeTreePartCheckThread::run() if (parts_queue.empty()) { - LOG_ERROR_FORMATTED(log, "Someone erased cheking part from parts_queue. This is a bug."); + LOG_ERROR(log, "Someone erased cheking part from parts_queue. This is a bug."); } else { diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 5177438ffd1..c78f1f4f97a 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -49,7 +49,7 @@ bool ReplicatedMergeTreeQueue::isVirtualPart(const MergeTreeData::DataPartPtr & bool ReplicatedMergeTreeQueue::load(zkutil::ZooKeeperPtr zookeeper) { auto queue_path = replica_path + "/queue"; - LOG_DEBUG_FORMATTED(log, "Loading queue from {}", queue_path); + LOG_DEBUG(log, "Loading queue from {}", queue_path); bool updated = false; std::optional min_unprocessed_insert_time_changed; @@ -75,7 +75,7 @@ bool ReplicatedMergeTreeQueue::load(zkutil::ZooKeeperPtr zookeeper) return already_loaded_paths.count(path); }); - LOG_DEBUG_FORMATTED(log, "Having {} queue entries to load, {} entries already loaded.", (to_remove_it - children.begin()), (children.end() - to_remove_it)); + LOG_DEBUG(log, "Having {} queue entries to load, {} entries already loaded.", (to_remove_it - children.begin()), (children.end() - to_remove_it)); children.erase(to_remove_it, children.end()); std::sort(children.begin(), children.end()); @@ -104,7 +104,7 @@ bool ReplicatedMergeTreeQueue::load(zkutil::ZooKeeperPtr zookeeper) updateTimesInZooKeeper(zookeeper, min_unprocessed_insert_time_changed, {}); - LOG_TRACE_FORMATTED(log, "Loaded queue"); + LOG_TRACE(log, "Loaded queue"); return updated; } @@ -150,7 +150,7 @@ void ReplicatedMergeTreeQueue::insertUnlocked( } if (entry->type == LogEntry::ALTER_METADATA) { - LOG_TRACE_FORMATTED(log, "Adding alter metadata version {} to the queue", entry->alter_version); + LOG_TRACE(log, "Adding alter metadata version {} to the queue", entry->alter_version); alter_sequence.addMetadataAlter(entry->alter_version, entry->have_mutation, state_lock); } } @@ -232,7 +232,7 @@ void ReplicatedMergeTreeQueue::updateStateOnQueueEntryRemoval( if (entry->type == LogEntry::ALTER_METADATA) { - LOG_TRACE_FORMATTED(log, "Finishing metadata alter with version {}", entry->alter_version); + LOG_TRACE(log, "Finishing metadata alter with version {}", entry->alter_version); alter_sequence.finishMetadataAlter(entry->alter_version, state_lock); } } @@ -320,7 +320,7 @@ void ReplicatedMergeTreeQueue::updateTimesInZooKeeper( auto code = zookeeper->tryMulti(ops, responses); if (code) - LOG_ERROR_FORMATTED(log, "Couldn't set value of nodes for insert times ({}/min_unprocessed_insert_time, max_processed_insert_time): {}", replica_path, zkutil::ZooKeeper::error2string(code) + ". This shouldn't happen often."); + LOG_ERROR(log, "Couldn't set value of nodes for insert times ({}/min_unprocessed_insert_time, max_processed_insert_time): {}", replica_path, zkutil::ZooKeeper::error2string(code) + ". This shouldn't happen often."); } } @@ -365,7 +365,7 @@ void ReplicatedMergeTreeQueue::removeProcessedEntry(zkutil::ZooKeeperPtr zookeep auto code = zookeeper->tryRemove(replica_path + "/queue/" + entry->znode_name); if (code) - LOG_ERROR_FORMATTED(log, "Couldn't remove {}/queue/{}: {}. This shouldn't happen often.", replica_path, entry->znode_name, zkutil::ZooKeeper::error2string(code)); + LOG_ERROR(log, "Couldn't remove {}/queue/{}: {}. This shouldn't happen often.", replica_path, entry->znode_name, zkutil::ZooKeeper::error2string(code)); updateTimesInZooKeeper(zookeeper, min_unprocessed_insert_time_changed, max_processed_insert_time_changed); } @@ -481,7 +481,7 @@ void ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper, C UInt64 last_entry_index = parse(last_entry.substr(strlen("log-"))); - LOG_DEBUG_FORMATTED(log, "Pulling {} entries to queue: {} - {}", (end - begin), *begin, *last); + LOG_DEBUG(log, "Pulling {} entries to queue: {} - {}", (end - begin), *begin, *last); zkutil::AsyncResponses futures; futures.reserve(end - begin); @@ -555,7 +555,7 @@ void ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper, C } if (!copied_entries.empty()) - LOG_DEBUG_FORMATTED(log, "Pulled {} entries to queue.", copied_entries.size()); + LOG_DEBUG(log, "Pulled {} entries to queue.", copied_entries.size()); } if (storage.queue_task_handle) @@ -613,11 +613,11 @@ void ReplicatedMergeTreeQueue::updateMutations(zkutil::ZooKeeperPtr zookeeper, C { if (!it->second.is_done) { - LOG_DEBUG_FORMATTED(log, "Removing killed mutation {} from local state.", entry.znode_name); + LOG_DEBUG(log, "Removing killed mutation {} from local state.", entry.znode_name); some_active_mutations_were_killed = true; } else - LOG_DEBUG_FORMATTED(log, "Removing obsolete mutation {} from local state.", entry.znode_name); + LOG_DEBUG(log, "Removing obsolete mutation {} from local state.", entry.znode_name); for (const auto & partition_and_block_num : entry.block_numbers) { @@ -645,7 +645,7 @@ void ReplicatedMergeTreeQueue::updateMutations(zkutil::ZooKeeperPtr zookeeper, C if (!entries_to_load.empty()) { - LOG_INFO_FORMATTED(log, "Loading {} mutation entries: {} - {}", toString(entries_to_load.size()), entries_to_load.front(), entries_to_load.back()); + LOG_INFO(log, "Loading {} mutation entries: {} - {}", toString(entries_to_load.size()), entries_to_load.front(), entries_to_load.back()); std::vector> futures; for (const String & entry : entries_to_load) @@ -700,7 +700,7 @@ void ReplicatedMergeTreeQueue::updateMutations(zkutil::ZooKeeperPtr zookeeper, C /// otherwise it's already done if (entry->isAlterMutation() && entry->znode_name > mutation_pointer) { - LOG_TRACE_FORMATTED(log, "Adding mutation {} with alter version {} to the queue", entry->znode_name, entry->alter_version); + LOG_TRACE(log, "Adding mutation {} with alter version {} to the queue", entry->znode_name, entry->alter_version); alter_sequence.addMutationForAlter(entry->alter_version, state_lock); } } @@ -721,7 +721,7 @@ ReplicatedMergeTreeMutationEntryPtr ReplicatedMergeTreeQueue::removeMutation( auto rc = zookeeper->tryRemove(zookeeper_path + "/mutations/" + mutation_id); if (rc == Coordination::ZOK) - LOG_DEBUG_FORMATTED(log, "Removed mutation {} from ZooKeeper.", mutation_id); + LOG_DEBUG(log, "Removed mutation {} from ZooKeeper.", mutation_id); ReplicatedMergeTreeMutationEntryPtr entry; bool mutation_was_active = false; @@ -745,12 +745,12 @@ ReplicatedMergeTreeMutationEntryPtr ReplicatedMergeTreeQueue::removeMutation( if (entry->isAlterMutation()) { - LOG_DEBUG_FORMATTED(log, "Removed alter {} because mutation {} were killed.", entry->alter_version, entry->znode_name); + LOG_DEBUG(log, "Removed alter {} because mutation {} were killed.", entry->alter_version, entry->znode_name); alter_sequence.finishDataAlter(entry->alter_version, state_lock); } mutations_by_znode.erase(it); - LOG_DEBUG_FORMATTED(log, "Removed mutation {} from local state.", entry->znode_name); + LOG_DEBUG(log, "Removed mutation {} from local state.", entry->znode_name); } if (mutation_was_active) @@ -845,7 +845,7 @@ void ReplicatedMergeTreeQueue::removePartProducingOpsInRange( to_wait.push_back(*it); auto code = zookeeper->tryRemove(replica_path + "/queue/" + (*it)->znode_name); if (code) - LOG_INFO_FORMATTED(log, "Couldn't remove {}: {}", replica_path + "/queue/" + (*it)->znode_name, zkutil::ZooKeeper::error2string(code)); + LOG_INFO(log, "Couldn't remove {}: {}", replica_path + "/queue/" + (*it)->znode_name, zkutil::ZooKeeper::error2string(code)); updateStateOnQueueEntryRemoval( *it, /* is_successful = */ false, @@ -859,7 +859,7 @@ void ReplicatedMergeTreeQueue::removePartProducingOpsInRange( updateTimesInZooKeeper(zookeeper, min_unprocessed_insert_time_changed, max_processed_insert_time_changed); - LOG_DEBUG_FORMATTED(log, "Removed {} entries from queue. Waiting for {} entries that are currently executing.", removed_entries, to_wait.size()); + LOG_DEBUG(log, "Removed {} entries from queue. Waiting for {} entries that are currently executing.", removed_entries, to_wait.size()); /// Let's wait for the operations with the parts contained in the range to be deleted. for (LogEntryPtr & entry : to_wait) @@ -979,7 +979,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( if (!isNotCoveredByFuturePartsImpl(new_part_name, out_postpone_reason, state_lock)) { if (!out_postpone_reason.empty()) - LOG_DEBUG_FORMATTED(log, out_postpone_reason); + LOG_DEBUG(log, out_postpone_reason); return false; } } @@ -999,7 +999,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( { String reason = "Not merging into part " + entry.new_part_name + " because part " + name + " is not ready yet (log entry for that part is being processed)."; - LOG_TRACE_FORMATTED(log, reason); + LOG_TRACE(log, reason); out_postpone_reason = reason; return false; } @@ -1012,7 +1012,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( if (merger_mutator.merges_blocker.isCancelled()) { String reason = "Not executing log entry for part " + entry.new_part_name + " because merges and mutations are cancelled now."; - LOG_DEBUG_FORMATTED(log, reason); + LOG_DEBUG(log, reason); out_postpone_reason = reason; return false; } @@ -1033,7 +1033,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( String reason = "Not executing log entry " + entry.typeToString() + " for part " + entry.new_part_name + " because source parts size (" + formatReadableSizeWithBinarySuffix(sum_parts_size_in_bytes) + ") is greater than the current maximum (" + formatReadableSizeWithBinarySuffix(max_source_parts_size) + ")."; - LOG_DEBUG_FORMATTED(log, reason); + LOG_DEBUG(log, reason); out_postpone_reason = reason; return false; } @@ -1048,7 +1048,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( if (0 != getConflictsCountForRange(range, entry, &conflicts_description, state_lock)) { - LOG_DEBUG_FORMATTED(log, conflicts_description); + LOG_DEBUG(log, conflicts_description); return false; } } @@ -1153,13 +1153,13 @@ ReplicatedMergeTreeQueue::CurrentlyExecuting::~CurrentlyExecuting() for (const String & new_part_name : entry->getBlockingPartNames()) { if (!queue.future_parts.erase(new_part_name)) - LOG_ERROR_FORMATTED(queue.log, "Untagging already untagged future part {}. This is a bug.", new_part_name); + LOG_ERROR(queue.log, "Untagging already untagged future part {}. This is a bug.", new_part_name); } if (!entry->actual_new_part_name.empty()) { if (entry->actual_new_part_name != entry->new_part_name && !queue.future_parts.erase(entry->actual_new_part_name)) - LOG_ERROR_FORMATTED(queue.log, "Untagging already untagged future part {}. This is a bug.", entry->actual_new_part_name); + LOG_ERROR(queue.log, "Untagging already untagged future part {}. This is a bug.", entry->actual_new_part_name); entry->actual_new_part_name.clear(); } @@ -1328,7 +1328,7 @@ MutationCommands ReplicatedMergeTreeQueue::getMutationCommands( if (part->info.getDataVersion() > desired_mutation_version) { - LOG_WARNING_FORMATTED(log, "Data version of part {} is already greater than desired mutation version {}", part->name, desired_mutation_version); + LOG_WARNING(log, "Data version of part {} is already greater than desired mutation version {}", part->name, desired_mutation_version); return MutationCommands{}; } @@ -1337,7 +1337,7 @@ MutationCommands ReplicatedMergeTreeQueue::getMutationCommands( auto in_partition = mutations_by_partition.find(part->info.partition_id); if (in_partition == mutations_by_partition.end()) { - LOG_WARNING_FORMATTED(log, "There are no mutations for partition ID {} (trying to mutate part {} to {})", part->info.partition_id, part->name, toString(desired_mutation_version)); + LOG_WARNING(log, "There are no mutations for partition ID {} (trying to mutate part {} to {})", part->info.partition_id, part->name, toString(desired_mutation_version)); return MutationCommands{}; } @@ -1345,7 +1345,7 @@ MutationCommands ReplicatedMergeTreeQueue::getMutationCommands( auto end = in_partition->second.lower_bound(desired_mutation_version); if (end == in_partition->second.end() || end->first != desired_mutation_version) - LOG_WARNING_FORMATTED(log, "Mutation with version {} not found in partition ID {} (trying to mutate part {}", desired_mutation_version, part->info.partition_id, part->name + ")"); + LOG_WARNING(log, "Mutation with version {} not found in partition ID {} (trying to mutate part {}", desired_mutation_version, part->info.partition_id, part->name + ")"); else ++end; @@ -1373,18 +1373,18 @@ bool ReplicatedMergeTreeQueue::tryFinalizeMutations(zkutil::ZooKeeperPtr zookeep if (znode <= mutation_pointer) { - LOG_TRACE_FORMATTED(log, "Marking mutation {} done because it is <= mutation_pointer ({})", znode, mutation_pointer); + LOG_TRACE(log, "Marking mutation {} done because it is <= mutation_pointer ({})", znode, mutation_pointer); mutation.is_done = true; alter_sequence.finishDataAlter(mutation.entry->alter_version, lock); if (mutation.parts_to_do.size() != 0) { - LOG_INFO_FORMATTED(log, "Seems like we jumped over mutation {} when downloaded part with bigger mutation number.{}", znode, " It's OK, tasks for rest parts will be skipped, but probably a lot of mutations were executed concurrently on different replicas."); + LOG_INFO(log, "Seems like we jumped over mutation {} when downloaded part with bigger mutation number.{}", znode, " It's OK, tasks for rest parts will be skipped, but probably a lot of mutations were executed concurrently on different replicas."); mutation.parts_to_do.clear(); } } else if (mutation.parts_to_do.size() == 0) { - LOG_TRACE_FORMATTED(log, "Will check if mutation {} is done", mutation.entry->znode_name); + LOG_TRACE(log, "Will check if mutation {} is done", mutation.entry->znode_name); candidates.push_back(mutation.entry); } } @@ -1393,7 +1393,7 @@ bool ReplicatedMergeTreeQueue::tryFinalizeMutations(zkutil::ZooKeeperPtr zookeep if (candidates.empty()) return false; else - LOG_DEBUG_FORMATTED(log, "Trying to finalize {} mutations", candidates.size()); + LOG_DEBUG(log, "Trying to finalize {} mutations", candidates.size()); auto merge_pred = getMergePredicate(zookeeper); @@ -1417,11 +1417,11 @@ bool ReplicatedMergeTreeQueue::tryFinalizeMutations(zkutil::ZooKeeperPtr zookeep auto it = mutations_by_znode.find(entry->znode_name); if (it != mutations_by_znode.end()) { - LOG_TRACE_FORMATTED(log, "Mutation {} is done", entry->znode_name); + LOG_TRACE(log, "Mutation {} is done", entry->znode_name); it->second.is_done = true; if (entry->isAlterMutation()) { - LOG_TRACE_FORMATTED(log, "Finishing data alter with version {} for entry {}", entry->alter_version, entry->znode_name); + LOG_TRACE(log, "Finishing data alter with version {} for entry {}", entry->alter_version, entry->znode_name); alter_sequence.finishDataAlter(entry->alter_version, lock); } } @@ -1885,7 +1885,7 @@ bool ReplicatedMergeTreeMergePredicate::isMutationFinished(const ReplicatedMerge partition_it->second.begin(), partition_it->second.lower_bound(block_num)); if (blocks_count) { - LOG_TRACE_FORMATTED(queue.log, "Mutation {} is not done yet because in partition ID {} there are still {} uncommitted blocks.", mutation.znode_name, partition_id, blocks_count); + LOG_TRACE(queue.log, "Mutation {} is not done yet because in partition ID {} there are still {} uncommitted blocks.", mutation.znode_name, partition_id, blocks_count); return false; } } @@ -1897,7 +1897,7 @@ bool ReplicatedMergeTreeMergePredicate::isMutationFinished(const ReplicatedMerge size_t suddenly_appeared_parts = getPartNamesToMutate(mutation, queue.virtual_parts).size(); if (suddenly_appeared_parts) { - LOG_TRACE_FORMATTED(queue.log, "Mutation {} is not done yet because {} parts to mutate suddenly appeared.", mutation.znode_name, suddenly_appeared_parts); + LOG_TRACE(queue.log, "Mutation {} is not done yet because {} parts to mutate suddenly appeared.", mutation.znode_name, suddenly_appeared_parts); return false; } } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index 36805788f13..5b4f4f2a5be 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -68,11 +68,11 @@ void ReplicatedMergeTreeRestartingThread::run() if (first_time) { - LOG_DEBUG_FORMATTED(log, "Activating replica."); + LOG_DEBUG(log, "Activating replica."); } else { - LOG_WARNING_FORMATTED(log, "ZooKeeper session has expired. Switching to a new session."); + LOG_WARNING(log, "ZooKeeper session has expired. Switching to a new session."); bool old_val = false; if (storage.is_readonly.compare_exchange_strong(old_val, true)) @@ -133,7 +133,7 @@ void ReplicatedMergeTreeRestartingThread::run() storage.getReplicaDelays(absolute_delay, relative_delay); if (absolute_delay) - LOG_TRACE_FORMATTED(log, "Absolute delay: {}. Relative delay: {}.", absolute_delay, relative_delay); + LOG_TRACE(log, "Absolute delay: {}. Relative delay: {}.", absolute_delay, relative_delay); prev_time_of_check_delay = current_time; @@ -141,7 +141,7 @@ void ReplicatedMergeTreeRestartingThread::run() if (storage.is_leader && relative_delay > static_cast(storage_settings->min_relative_delay_to_yield_leadership)) { - LOG_INFO_FORMATTED(log, "Relative replica delay ({} seconds) is bigger than threshold ({}). Will yield leadership.", relative_delay, storage_settings->min_relative_delay_to_yield_leadership); + LOG_INFO(log, "Relative replica delay ({} seconds) is bigger than threshold ({}). Will yield leadership.", relative_delay, storage_settings->min_relative_delay_to_yield_leadership); ProfileEvents::increment(ProfileEvents::ReplicaYieldLeadership); @@ -188,7 +188,7 @@ bool ReplicatedMergeTreeRestartingThread::tryStartup() if (storage_settings->replicated_can_become_leader) storage.enterLeaderElection(); else - LOG_INFO_FORMATTED(log, "Will not enter leader election because replicated_can_become_leader=0"); + LOG_INFO(log, "Will not enter leader election because replicated_can_become_leader=0"); /// Anything above can throw a KeeperException if something is wrong with ZK. /// Anything below should not throw exceptions. @@ -214,7 +214,7 @@ bool ReplicatedMergeTreeRestartingThread::tryStartup() } catch (const Coordination::Exception & e) { - LOG_ERROR_FORMATTED(log, "Couldn't start replication: {}. {}", e.what(), DB::getCurrentExceptionMessage(true)); + LOG_ERROR(log, "Couldn't start replication: {}. {}", e.what(), DB::getCurrentExceptionMessage(true)); return false; } catch (const Exception & e) @@ -222,7 +222,7 @@ bool ReplicatedMergeTreeRestartingThread::tryStartup() if (e.code() != ErrorCodes::REPLICA_IS_ALREADY_ACTIVE) throw; - LOG_ERROR_FORMATTED(log, "Couldn't start replication: {}. {}", e.what(), DB::getCurrentExceptionMessage(true)); + LOG_ERROR(log, "Couldn't start replication: {}. {}", e.what(), DB::getCurrentExceptionMessage(true)); return false; } } @@ -247,7 +247,7 @@ void ReplicatedMergeTreeRestartingThread::removeFailedQuorumParts() if (part) { - LOG_DEBUG_FORMATTED(log, "Found part {} with failed quorum. Moving to detached. This shouldn't happen often.", part_name); + LOG_DEBUG(log, "Found part {} with failed quorum. Moving to detached. This shouldn't happen often.", part_name); storage.forgetPartAndMoveToDetached(part, "noquorum"); storage.queue.removeFromVirtualParts(part->info); } @@ -268,7 +268,7 @@ void ReplicatedMergeTreeRestartingThread::updateQuorumIfWeHavePart() if (!quorum_entry.replicas.count(storage.replica_name) && zookeeper->exists(storage.replica_path + "/parts/" + quorum_entry.part_name)) { - LOG_WARNING_FORMATTED(log, "We have part {} but we is not in quorum. Updating quorum. This shouldn't happen often.", quorum_entry.part_name); + LOG_WARNING(log, "We have part {} but we is not in quorum. Updating quorum. This shouldn't happen often.", quorum_entry.part_name); storage.updateQuorum(quorum_entry.part_name); } } @@ -335,7 +335,7 @@ void ReplicatedMergeTreeRestartingThread::partialShutdown() storage.partial_shutdown_event.set(); storage.replica_is_active_node = nullptr; - LOG_TRACE_FORMATTED(log, "Waiting for threads to finish"); + LOG_TRACE(log, "Waiting for threads to finish"); storage.exitLeaderElection(); @@ -346,7 +346,7 @@ void ReplicatedMergeTreeRestartingThread::partialShutdown() storage.cleanup_thread.stop(); storage.part_check_thread.stop(); - LOG_TRACE_FORMATTED(log, "Threads finished"); + LOG_TRACE(log, "Threads finished"); } @@ -355,7 +355,7 @@ void ReplicatedMergeTreeRestartingThread::shutdown() /// Stop restarting_thread before stopping other tasks - so that it won't restart them again. need_stop = true; task->deactivate(); - LOG_TRACE_FORMATTED(log, "Restarting thread finished"); + LOG_TRACE(log, "Restarting thread finished"); /// Stop other tasks. partialShutdown(); diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index 39372314e2e..95be1275d3d 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -189,7 +189,7 @@ Pipes StorageBuffer::read( { if (!dest_columns.hasPhysical(column_name)) { - LOG_WARNING_FORMATTED(log, "Destination table {} doesn't have column {}. The default values are used.", destination_id.getNameForLogs(), backQuoteIfNeed(column_name)); + LOG_WARNING(log, "Destination table {} doesn't have column {}. The default values are used.", destination_id.getNameForLogs(), backQuoteIfNeed(column_name)); boost::range::remove_erase(columns_intersection, column_name); continue; } @@ -197,14 +197,14 @@ Pipes StorageBuffer::read( const auto & col = our_columns.getPhysical(column_name); if (!dst_col.type->equals(*col.type)) { - LOG_WARNING_FORMATTED(log, "Destination table {} has different type of column {} ({} != {}). Data from destination table are converted.", destination_id.getNameForLogs(), backQuoteIfNeed(column_name), dst_col.type->getName(), col.type->getName()); + LOG_WARNING(log, "Destination table {} has different type of column {} ({} != {}). Data from destination table are converted.", destination_id.getNameForLogs(), backQuoteIfNeed(column_name), dst_col.type->getName(), col.type->getName()); header_after_adding_defaults.getByName(column_name) = ColumnWithTypeAndName(dst_col.type, column_name); } } if (columns_intersection.empty()) { - LOG_WARNING_FORMATTED(log, "Destination table {} has no common columns with block in buffer. Block of data is skipped.", destination_id.getNameForLogs()); + LOG_WARNING(log, "Destination table {} has no common columns with block in buffer. Block of data is skipped.", destination_id.getNameForLogs()); } else { @@ -346,7 +346,7 @@ public: { if (storage.destination_id) { - LOG_TRACE_FORMATTED(storage.log, "Writing block with {} rows, {} bytes directly.", rows, bytes); + LOG_TRACE(storage.log, "Writing block with {} rows, {} bytes directly.", rows, bytes); storage.writeBlockToDestination(block, destination); } return; @@ -447,7 +447,7 @@ void StorageBuffer::startup() { if (global_context.getSettingsRef().readonly) { - LOG_WARNING_FORMATTED(log, "Storage {} is run with readonly settings, it will not be able to insert data. Set appropriate system_profile to fix this.", getName()); + LOG_WARNING(log, "Storage {} is run with readonly settings, it will not be able to insert data. Set appropriate system_profile to fix this.", getName()); } @@ -589,7 +589,7 @@ void StorageBuffer::flushBuffer(Buffer & buffer, bool check_thresholds, bool loc ProfileEvents::increment(ProfileEvents::StorageBufferFlush); - LOG_TRACE_FORMATTED(log, "Flushing buffer with {} rows, {} bytes, age {} seconds {}.", rows, bytes, time_passed, (check_thresholds ? "(bg)" : "(direct)")); + LOG_TRACE(log, "Flushing buffer with {} rows, {} bytes, age {} seconds {}.", rows, bytes, time_passed, (check_thresholds ? "(bg)" : "(direct)")); if (!destination_id) return; @@ -631,7 +631,7 @@ void StorageBuffer::writeBlockToDestination(const Block & block, StoragePtr tabl if (!table) { - LOG_ERROR_FORMATTED(log, "Destination table {} doesn't exist. Block of data is discarded.", destination_id.getNameForLogs()); + LOG_ERROR(log, "Destination table {} doesn't exist. Block of data is discarded.", destination_id.getNameForLogs()); return; } @@ -653,7 +653,7 @@ void StorageBuffer::writeBlockToDestination(const Block & block, StoragePtr tabl auto column = block.getByName(dst_col.name); if (!column.type->equals(*dst_col.type)) { - LOG_WARNING_FORMATTED(log, "Destination table {} have different type of column {} ({} != {}). Block of data is converted.", destination_id.getNameForLogs(), backQuoteIfNeed(column.name), dst_col.type->getName(), column.type->getName()); + LOG_WARNING(log, "Destination table {} have different type of column {} ({} != {}). Block of data is converted.", destination_id.getNameForLogs(), backQuoteIfNeed(column.name), dst_col.type->getName(), column.type->getName()); column.column = castColumn(column, dst_col.type); column.type = dst_col.type; } @@ -664,12 +664,12 @@ void StorageBuffer::writeBlockToDestination(const Block & block, StoragePtr tabl if (block_to_write.columns() == 0) { - LOG_ERROR_FORMATTED(log, "Destination table {} have no common columns with block in buffer. Block of data is discarded.", destination_id.getNameForLogs()); + LOG_ERROR(log, "Destination table {} have no common columns with block in buffer. Block of data is discarded.", destination_id.getNameForLogs()); return; } if (block_to_write.columns() != block.columns()) - LOG_WARNING_FORMATTED(log, "Not all columns from block in buffer exist in destination table {}. Some columns are discarded.", destination_id.getNameForLogs()); + LOG_WARNING(log, "Not all columns from block in buffer exist in destination table {}. Some columns are discarded.", destination_id.getNameForLogs()); auto list_of_columns = std::make_shared(); insert->columns = list_of_columns; diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index f3e5e0a837c..62755c99642 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -439,7 +439,7 @@ bool StorageDistributed::canForceGroupByNoMerge(const Context &context, QueryPro reason = "GROUP BY " + backQuote(serializeAST(*group_by, true)); } - LOG_DEBUG_FORMATTED(log, "Force distributed_group_by_no_merge for {} (injective)", reason); + LOG_DEBUG(log, "Force distributed_group_by_no_merge for {} (injective)", reason); return true; } @@ -477,12 +477,12 @@ Pipes StorageDistributed::read( ClusterPtr optimized_cluster = getOptimizedCluster(context, query_info.query); if (optimized_cluster) { - LOG_DEBUG_FORMATTED(log, "Skipping irrelevant shards - the query will be sent to the following shards of the cluster (shard numbers): {}", makeFormattedListOfShards(optimized_cluster)); + LOG_DEBUG(log, "Skipping irrelevant shards - the query will be sent to the following shards of the cluster (shard numbers): {}", makeFormattedListOfShards(optimized_cluster)); cluster = optimized_cluster; } else { - LOG_DEBUG_FORMATTED(log, "Unable to figure out irrelevant shards from WHERE/PREWHERE clauses - the query will be sent to all shards of the cluster{}", has_sharding_key ? "" : " (no sharding key)"); + LOG_DEBUG(log, "Unable to figure out irrelevant shards from WHERE/PREWHERE clauses - the query will be sent to all shards of the cluster{}", has_sharding_key ? "" : " (no sharding key)"); } } @@ -570,7 +570,7 @@ void StorageDistributed::alter(const AlterCommands & params, const Context & con void StorageDistributed::startup() { if (remote_database.empty() && !remote_table_function_ptr) - LOG_WARNING_FORMATTED(log, "Name of remote database is empty. Default database will be used implicitly."); + LOG_WARNING(log, "Name of remote database is empty. Default database will be used implicitly."); if (!volume) return; @@ -584,7 +584,7 @@ void StorageDistributed::startup() if (inc > file_names_increment.value) file_names_increment.value.store(inc); } - LOG_DEBUG_FORMATTED(log, "Auto-increment is {}", file_names_increment.value); + LOG_DEBUG(log, "Auto-increment is {}", file_names_increment.value); } @@ -813,7 +813,7 @@ void StorageDistributed::renameOnDisk(const String & new_path_to_table_data) auto new_path = path + new_path_to_table_data; Poco::File(path + relative_data_path).renameTo(new_path); - LOG_DEBUG_FORMATTED(log, "Updating path to {}", new_path); + LOG_DEBUG(log, "Updating path to {}", new_path); std::lock_guard lock(cluster_nodes_mutex); for (auto & node : cluster_nodes_data) diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 8070432c2db..f6114d709b6 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -214,7 +214,7 @@ void StorageMergeTree::truncate(const ASTPtr &, const Context &, TableStructureW auto parts_to_remove = getDataPartsVector(); removePartsFromWorkingSet(parts_to_remove, true); - LOG_INFO_FORMATTED(log, "Removed {} parts.", parts_to_remove.size()); + LOG_INFO(log, "Removed {} parts.", parts_to_remove.size()); } clearOldMutations(true); @@ -384,18 +384,18 @@ Int64 StorageMergeTree::startMutation(const MutationCommands & commands, String auto insertion = current_mutations_by_id.emplace(mutation_file_name, std::move(entry)); current_mutations_by_version.emplace(version, insertion.first->second); - LOG_INFO_FORMATTED(log, "Added mutation: {}", mutation_file_name); + LOG_INFO(log, "Added mutation: {}", mutation_file_name); merging_mutating_task_handle->wake(); return version; } void StorageMergeTree::waitForMutation(Int64 version, const String & file_name) { - LOG_INFO_FORMATTED(log, "Waiting mutation: {}", file_name); + LOG_INFO(log, "Waiting mutation: {}", file_name); auto check = [version, this]() { return shutdown_called || isMutationDone(version); }; std::unique_lock lock(mutation_wait_mutex); mutation_wait_event.wait(lock, check); - LOG_INFO_FORMATTED(log, "Mutation {} done", file_name); + LOG_INFO(log, "Mutation {} done", file_name); } void StorageMergeTree::mutate(const MutationCommands & commands, const Context & query_context) @@ -492,7 +492,7 @@ std::vector StorageMergeTree::getMutationsStatus() cons CancellationCode StorageMergeTree::killMutation(const String & mutation_id) { - LOG_TRACE_FORMATTED(log, "Killing mutation {}", mutation_id); + LOG_TRACE(log, "Killing mutation {}", mutation_id); std::optional to_kill; { @@ -511,7 +511,7 @@ CancellationCode StorageMergeTree::killMutation(const String & mutation_id) global_context.getMergeList().cancelPartMutations({}, to_kill->block_number); to_kill->removeFile(); - LOG_TRACE_FORMATTED(log, "Cancelled part mutations and removed mutation file {}", mutation_id); + LOG_TRACE(log, "Cancelled part mutations and removed mutation file {}", mutation_id); { std::lock_guard lock(mutation_wait_mutex); mutation_wait_event.notify_all(); @@ -534,7 +534,7 @@ void StorageMergeTree::loadMutations() { MergeTreeMutationEntry entry(disk, path, it->name()); Int64 block_number = entry.block_number; - LOG_DEBUG_FORMATTED(log, "Loading mutation: {} entry, commands size: {}", it->name(), entry.commands.size()); + LOG_DEBUG(log, "Loading mutation: {} entry, commands size: {}", it->name(), entry.commands.size()); auto insertion = current_mutations_by_id.emplace(it->name(), std::move(entry)); current_mutations_by_version.emplace(block_number, insertion.first->second); } @@ -837,7 +837,7 @@ BackgroundProcessingPoolTaskResult StorageMergeTree::mergeMutateTask() { if (e.code() == ErrorCodes::ABORTED) { - LOG_INFO_FORMATTED(log, e.message()); + LOG_INFO(log, e.message()); return BackgroundProcessingPoolTaskResult::ERROR; } @@ -896,7 +896,7 @@ void StorageMergeTree::clearOldMutations(bool truncate) for (auto & mutation : mutations_to_delete) { - LOG_TRACE_FORMATTED(log, "Removing mutation: {}", mutation.file_name); + LOG_TRACE(log, "Removing mutation: {}", mutation.file_name); mutation.removeFile(); } } @@ -923,7 +923,7 @@ bool StorageMergeTree::optimize( message << ": " << disable_reason; else message << " by some reason."; - LOG_INFO_FORMATTED(log, message.str()); + LOG_INFO(log, message.str()); if (context.getSettingsRef().optimize_throw_if_noop) throw Exception(message.str(), ErrorCodes::CANNOT_ASSIGN_OPTIMIZE); @@ -945,7 +945,7 @@ bool StorageMergeTree::optimize( message << ": " << disable_reason; else message << " by some reason."; - LOG_INFO_FORMATTED(log, message.str()); + LOG_INFO(log, message.str()); if (context.getSettingsRef().optimize_throw_if_noop) throw Exception(message.str(), ErrorCodes::CANNOT_ASSIGN_OPTIMIZE); @@ -1050,15 +1050,15 @@ void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, cons /// If DETACH clone parts to detached/ directory for (const auto & part : parts_to_remove) { - LOG_INFO_FORMATTED(log, "Detaching {}", part->relative_path); + LOG_INFO(log, "Detaching {}", part->relative_path); part->makeCloneInDetached(""); } } if (detach) - LOG_INFO_FORMATTED(log, "Detached {} parts inside partition ID {}.", parts_to_remove.size(), partition_id); + LOG_INFO(log, "Detached {} parts inside partition ID {}.", parts_to_remove.size(), partition_id); else - LOG_INFO_FORMATTED(log, "Removed {} parts inside partition ID {}.", parts_to_remove.size(), partition_id); + LOG_INFO(log, "Removed {} parts inside partition ID {}.", parts_to_remove.size(), partition_id); } clearOldPartsFromFilesystem(); @@ -1074,10 +1074,10 @@ void StorageMergeTree::attachPartition(const ASTPtr & partition, bool attach_par for (size_t i = 0; i < loaded_parts.size(); ++i) { - LOG_INFO_FORMATTED(log, "Attaching part {} from {}", loaded_parts[i]->name, renamed_parts.old_and_new_names[i].second); + LOG_INFO(log, "Attaching part {} from {}", loaded_parts[i]->name, renamed_parts.old_and_new_names[i].second); renameTempPartAndAdd(loaded_parts[i], &increment); renamed_parts.old_and_new_names[i].first.clear(); - LOG_INFO_FORMATTED(log, "Finished attaching part"); + LOG_INFO(log, "Finished attaching part"); } /// New parts with other data may appear in place of deleted parts. diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 49569c96894..b90782192a4 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -214,13 +214,13 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( skip_sanity_checks = true; current_zookeeper->remove(replica_path + "/flags/force_restore_data"); - LOG_WARNING_FORMATTED(log, "Skipping the limits on severity of changes to data parts and columns (flag {}/flags/force_restore_data).", replica_path); + LOG_WARNING(log, "Skipping the limits on severity of changes to data parts and columns (flag {}/flags/force_restore_data).", replica_path); } else if (has_force_restore_data_flag) { skip_sanity_checks = true; - LOG_WARNING_FORMATTED(log, "Skipping the limits on severity of changes to data parts and columns (flag force_restore_data)."); + LOG_WARNING(log, "Skipping the limits on severity of changes to data parts and columns (flag force_restore_data)."); } loadDataParts(skip_sanity_checks); @@ -231,14 +231,14 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( throw Exception("Can't create replicated table without ZooKeeper", ErrorCodes::NO_ZOOKEEPER); /// Do not activate the replica. It will be readonly. - LOG_ERROR_FORMATTED(log, "No ZooKeeper: table will be in readonly mode."); + LOG_ERROR(log, "No ZooKeeper: table will be in readonly mode."); is_readonly = true; return; } if (attach && !current_zookeeper->exists(zookeeper_path + "/metadata")) { - LOG_WARNING_FORMATTED(log, "No metadata in ZooKeeper: table will be in readonly mode."); + LOG_WARNING(log, "No metadata in ZooKeeper: table will be in readonly mode."); is_readonly = true; return; } @@ -325,7 +325,7 @@ void StorageReplicatedMergeTree::waitMutationToFinishOnReplicas( for (const String & replica : replicas) { - LOG_DEBUG_FORMATTED(log, "Waiting for {} to apply mutation {}", replica, mutation_id); + LOG_DEBUG(log, "Waiting for {} to apply mutation {}", replica, mutation_id); while (!partial_shutdown_called) { @@ -334,7 +334,7 @@ void StorageReplicatedMergeTree::waitMutationToFinishOnReplicas( Coordination::Stat exists_stat; if (!getZooKeeper()->exists(zookeeper_path + "/mutations/" + mutation_id, &exists_stat, wait_event)) { - LOG_WARNING_FORMATTED(log, "Mutation {} was killed or manually removed. Nothing to wait.", mutation_id); + LOG_WARNING(log, "Mutation {} was killed or manually removed. Nothing to wait.", mutation_id); return; } @@ -342,7 +342,7 @@ void StorageReplicatedMergeTree::waitMutationToFinishOnReplicas( /// Replica could be inactive. if (!zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/is_active")) { - LOG_WARNING_FORMATTED(log, "Replica {} is not active during mutation. Mutation will be done asynchronously when replica becomes active.", replica); + LOG_WARNING(log, "Replica {} is not active during mutation. Mutation will be done asynchronously when replica becomes active.", replica); inactive_replicas.emplace(replica); break; @@ -354,7 +354,7 @@ void StorageReplicatedMergeTree::waitMutationToFinishOnReplicas( /// Replica could be removed if (!zookeeper->tryGet(mutation_pointer, mutation_pointer_value, &get_stat, wait_event)) { - LOG_WARNING_FORMATTED(log, "Replica {} was removed", replica); + LOG_WARNING(log, "Replica {} was removed", replica); break; } else if (mutation_pointer_value >= mutation_id) /// Maybe we already processed more fresh mutation @@ -410,7 +410,7 @@ void StorageReplicatedMergeTree::createTableIfNotExists() if (zookeeper->exists(zookeeper_path)) return; - LOG_DEBUG_FORMATTED(log, "Creating table {}", zookeeper_path); + LOG_DEBUG(log, "Creating table {}", zookeeper_path); zookeeper->createAncestors(zookeeper_path); @@ -546,7 +546,7 @@ void StorageReplicatedMergeTree::createReplica() { auto zookeeper = getZooKeeper(); - LOG_DEBUG_FORMATTED(log, "Creating replica {}", replica_path); + LOG_DEBUG(log, "Creating replica {}", replica_path); int32_t code; @@ -577,7 +577,7 @@ void StorageReplicatedMergeTree::createReplica() if (code == Coordination::Error::ZNODEEXISTS) throw Exception("Replica " + replica_path + " already exists.", ErrorCodes::REPLICA_IS_ALREADY_EXIST); else if (code == Coordination::Error::ZBADVERSION) - LOG_ERROR_FORMATTED(log, "Retrying createReplica(), because some other replicas were created at the same time"); + LOG_ERROR(log, "Retrying createReplica(), because some other replicas were created at the same time"); else zkutil::KeeperMultiException::check(code, ops, responses); } while (code == Coordination::Error::ZBADVERSION); @@ -641,7 +641,7 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks) if (MergeTreePartInfo::tryParsePartName(part_name, &part_info, format_version)) return part_info.getBlocksCount(); - LOG_ERROR_FORMATTED(log, "Unexpected part name: {}", part_name); + LOG_ERROR(log, "Unexpected part name: {}", part_name); return 0; }; @@ -683,7 +683,7 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks) } if (unexpected_parts_nonnew_rows > 0) - LOG_WARNING_FORMATTED(log, sanity_report.str()); + LOG_WARNING(log, sanity_report.str()); /// Add to the queue jobs to pick up the missing parts from other replicas and remove from ZK the information that we have them. std::vector> exists_futures; @@ -699,7 +699,7 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks) for (size_t i = 0; i < parts_to_fetch.size(); ++i) { const String & part_name = parts_to_fetch[i]; - LOG_ERROR_FORMATTED(log, "Removing locally missing part from ZooKeeper and queueing a fetch: {}", part_name); + LOG_ERROR(log, "Removing locally missing part from ZooKeeper and queueing a fetch: {}", part_name); Coordination::Requests ops; @@ -730,7 +730,7 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks) /// Remove extra local parts. for (const DataPartPtr & part : unexpected_parts) { - LOG_ERROR_FORMATTED(log, "Renaming unexpected part {} to ignored_{}", part->name, part->name); + LOG_ERROR(log, "Renaming unexpected part {} to ignored_{}", part->name, part->name); forgetPartAndMoveToDetached(part, "ignored", true); } } @@ -777,7 +777,7 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: !zookeeper->exists(current_part_path + "/columns", &columns_stat_after) || columns_stat_before.version != columns_stat_after.version) { - LOG_INFO_FORMATTED(log, "Not checking checksums of part {} with replica {} because part changed while we were reading its checksums", part_name, replica); + LOG_INFO(log, "Not checking checksums of part {} with replica {} because part changed while we were reading its checksums", part_name, replica); continue; } @@ -787,7 +787,7 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: if (replica_part_header.getColumnsHash() != local_part_header.getColumnsHash()) { - LOG_INFO_FORMATTED(log, "Not checking checksums of part {} with replica {} because columns are different", part_name, replica); + LOG_INFO(log, "Not checking checksums of part {} with replica {} because columns are different", part_name, replica); continue; } @@ -830,7 +830,7 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: } else { - LOG_WARNING_FORMATTED(log, "checkPartAndAddToZooKeeper: node {} already exists. Will not commit any nodes.", replica_path + "/parts/" + part_name); + LOG_WARNING(log, "checkPartAndAddToZooKeeper: node {} already exists. Will not commit any nodes.", replica_path + "/parts/" + part_name); } } @@ -878,7 +878,7 @@ MergeTreeData::DataPartsVector StorageReplicatedMergeTree::checkPartChecksumsAnd if (failed_op_index < num_check_ops && e.code == Coordination::ZNODEEXISTS) { - LOG_INFO_FORMATTED(log, "The part {} on a replica suddenly appeared, will recheck checksums", e.getPathForFirstFailedOp()); + LOG_INFO(log, "The part {} on a replica suddenly appeared, will recheck checksums", e.getPathForFirstFailedOp()); } else throw; @@ -923,19 +923,19 @@ bool StorageReplicatedMergeTree::executeLogEntry(LogEntry & entry) { if (!(entry.type == LogEntry::GET_PART && entry.source_replica == replica_name)) { - LOG_DEBUG_FORMATTED(log, "Skipping action for part {} because part {} already exists.", entry.new_part_name, existing_part->name); + LOG_DEBUG(log, "Skipping action for part {} because part {} already exists.", entry.new_part_name, existing_part->name); } return true; } } if (entry.type == LogEntry::GET_PART && entry.source_replica == replica_name) - LOG_WARNING_FORMATTED(log, "Part {} from own log doesn't exist.", entry.new_part_name); + LOG_WARNING(log, "Part {} from own log doesn't exist.", entry.new_part_name); /// Perhaps we don't need this part, because during write with quorum, the quorum has failed (see below about `/quorum/failed_parts`). if (entry.quorum && getZooKeeper()->exists(zookeeper_path + "/quorum/failed_parts/" + entry.new_part_name)) { - LOG_DEBUG_FORMATTED(log, "Skipping action for part {} because quorum for that part was failed.", entry.new_part_name); + LOG_DEBUG(log, "Skipping action for part {} because quorum for that part was failed.", entry.new_part_name); return true; /// NOTE Deletion from `virtual_parts` is not done, but it is only necessary for merge. } @@ -978,14 +978,14 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) for (auto i : ext::range(0, entry.source_parts.size())) source_parts_msg << (i != 0 ? ", " : "") << entry.source_parts[i]; - LOG_TRACE_FORMATTED(log, "Executing log entry to merge parts {} to {}", source_parts_msg.str(), entry.new_part_name); + LOG_TRACE(log, "Executing log entry to merge parts {} to {}", source_parts_msg.str(), entry.new_part_name); } const auto storage_settings_ptr = getSettings(); if (storage_settings_ptr->always_fetch_merged_part) { - LOG_INFO_FORMATTED(log, "Will fetch part {} because setting 'always_fetch_merged_part' is true", entry.new_part_name); + LOG_INFO(log, "Will fetch part {} because setting 'always_fetch_merged_part' is true", entry.new_part_name); return false; } @@ -1001,7 +1001,7 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) } if (part->name != name) { - LOG_WARNING_FORMATTED(log, "Part {} is covered by {} but should be merged into {}. This shouldn't happen often.", name, part->name, entry.new_part_name); + LOG_WARNING(log, "Part {} is covered by {} but should be merged into {}. This shouldn't happen often.", name, part->name, entry.new_part_name); have_all_parts = false; break; } @@ -1011,7 +1011,7 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) if (!have_all_parts) { /// If you do not have all the necessary parts, try to take some already merged part from someone. - LOG_DEBUG_FORMATTED(log, "Don't have all parts for merge {}; will try to fetch it instead", entry.new_part_name); + LOG_DEBUG(log, "Don't have all parts for merge {}; will try to fetch it instead", entry.new_part_name); return false; } else if (entry.create_time + storage_settings_ptr->prefer_fetch_merged_part_time_threshold.totalSeconds() <= time(nullptr)) @@ -1028,7 +1028,7 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) String replica = findReplicaHavingPart(entry.new_part_name, true); /// NOTE excessive ZK requests for same data later, may remove. if (!replica.empty()) { - LOG_DEBUG_FORMATTED(log, "Prefer to fetch {} from replica {}", entry.new_part_name, replica); + LOG_DEBUG(log, "Prefer to fetch {} from replica {}", entry.new_part_name, replica); return false; } } @@ -1093,7 +1093,7 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) ProfileEvents::increment(ProfileEvents::DataAfterMergeDiffersFromReplica); - LOG_ERROR_FORMATTED(log, "{}. Data after merge is not byte-identical to data on another replicas. There could be several reasons: 1. Using newer version of compression library after server update. 2. Using another compression method. 3. Non-deterministic compression algorithm (highly unlikely). 4. Non-deterministic merge algorithm due to logical error in code. 5. Data corruption in memory due to bug in code. 6. Data corruption in memory due to hardware issue. 7. Manual modification of source data after server startup. 8. Manual modification of checksums stored in ZooKeeper. 9. Part format related settings like 'enable_mixed_granularity_parts' are different on different replicas. We will download merged part from replica to force byte-identical result.", getCurrentExceptionMessage(false)); + LOG_ERROR(log, "{}. Data after merge is not byte-identical to data on another replicas. There could be several reasons: 1. Using newer version of compression library after server update. 2. Using another compression method. 3. Non-deterministic compression algorithm (highly unlikely). 4. Non-deterministic merge algorithm due to logical error in code. 5. Data corruption in memory due to bug in code. 6. Data corruption in memory due to hardware issue. 7. Manual modification of source data after server startup. 8. Manual modification of checksums stored in ZooKeeper. 9. Part format related settings like 'enable_mixed_granularity_parts' are different on different replicas. We will download merged part from replica to force byte-identical result.", getCurrentExceptionMessage(false)); write_part_log(ExecutionStatus::fromCurrentException()); @@ -1131,12 +1131,12 @@ bool StorageReplicatedMergeTree::tryExecutePartMutation(const StorageReplicatedM { const String & source_part_name = entry.source_parts.at(0); const auto storage_settings_ptr = getSettings(); - LOG_TRACE_FORMATTED(log, "Executing log entry to mutate part {} to {}", source_part_name, entry.new_part_name); + LOG_TRACE(log, "Executing log entry to mutate part {} to {}", source_part_name, entry.new_part_name); DataPartPtr source_part = getActiveContainingPart(source_part_name); if (!source_part) { - LOG_DEBUG_FORMATTED(log, "Source part {} for {} is not ready; will try to fetch it instead", source_part_name, entry.new_part_name); + LOG_DEBUG(log, "Source part {} for {} is not ready; will try to fetch it instead", source_part_name, entry.new_part_name); return false; } @@ -1158,7 +1158,7 @@ bool StorageReplicatedMergeTree::tryExecutePartMutation(const StorageReplicatedM String replica = findReplicaHavingPart(entry.new_part_name, true); /// NOTE excessive ZK requests for same data later, may remove. if (!replica.empty()) { - LOG_DEBUG_FORMATTED(log, "Prefer to fetch {} from replica {}", entry.new_part_name, replica); + LOG_DEBUG(log, "Prefer to fetch {} from replica {}", entry.new_part_name, replica); return false; } } @@ -1215,7 +1215,7 @@ bool StorageReplicatedMergeTree::tryExecutePartMutation(const StorageReplicatedM ProfileEvents::increment(ProfileEvents::DataAfterMutationDiffersFromReplica); - LOG_ERROR_FORMATTED(log, "{}. Data after mutation is not byte-identical to data on another replicas. We will download merged part from replica to force byte-identical result.", getCurrentExceptionMessage(false)); + LOG_ERROR(log, "{}. Data after mutation is not byte-identical to data on another replicas. We will download merged part from replica to force byte-identical result.", getCurrentExceptionMessage(false)); write_part_log(ExecutionStatus::fromCurrentException()); @@ -1286,7 +1286,7 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) if (entry.type != LogEntry::GET_PART) throw Exception("Logical error: log entry with quorum but type is not GET_PART", ErrorCodes::LOGICAL_ERROR); - LOG_DEBUG_FORMATTED(log, "No active replica has part {} which needs to be written with quorum. Will try to mark that quorum as failed.", entry.new_part_name); + LOG_DEBUG(log, "No active replica has part {} which needs to be written with quorum. Will try to mark that quorum as failed.", entry.new_part_name); /** Atomically: * - if replicas do not become active; @@ -1355,20 +1355,20 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) if (code == Coordination::ZOK) { - LOG_DEBUG_FORMATTED(log, "Marked quorum for part {} as failed.", entry.new_part_name); + LOG_DEBUG(log, "Marked quorum for part {} as failed.", entry.new_part_name); queue.removeFromVirtualParts(part_info); return true; } else if (code == Coordination::ZBADVERSION || code == Coordination::ZNONODE || code == Coordination::ZNODEEXISTS) { - LOG_DEBUG_FORMATTED(log, "State was changed or isn't expected when trying to mark quorum for part {} as failed. Code: {}", entry.new_part_name, zkutil::ZooKeeper::error2string(code)); + LOG_DEBUG(log, "State was changed or isn't expected when trying to mark quorum for part {} as failed. Code: {}", entry.new_part_name, zkutil::ZooKeeper::error2string(code)); } else throw Coordination::Exception(code); } else { - LOG_WARNING_FORMATTED(log, "No active replica has part {}, but that part needs quorum and /quorum/status contains entry about another part {}. It means that part was successfully written to {} replicas, but then all of them goes offline. Or it is a bug.", entry.new_part_name, quorum_entry.part_name, entry.quorum); + LOG_WARNING(log, "No active replica has part {}, but that part needs quorum and /quorum/status contains entry about another part {}. It means that part was successfully written to {} replicas, but then all of them goes offline. Or it is a bug.", entry.new_part_name, quorum_entry.part_name, entry.quorum); } } } @@ -1409,7 +1409,7 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) if (!parts_for_merge.empty() && replica.empty()) { - LOG_INFO_FORMATTED(log, "No active replica has part {}. Will fetch merged part instead.", entry.new_part_name); + LOG_INFO(log, "No active replica has part {}. Will fetch merged part instead.", entry.new_part_name); return false; } @@ -1437,9 +1437,9 @@ void StorageReplicatedMergeTree::executeDropRange(const LogEntry & entry) queue.removePartProducingOpsInRange(getZooKeeper(), drop_range_info, entry); if (entry.detach) - LOG_DEBUG_FORMATTED(log, "Detaching parts."); + LOG_DEBUG(log, "Detaching parts."); else - LOG_DEBUG_FORMATTED(log, "Removing parts."); + LOG_DEBUG(log, "Removing parts."); /// Delete the parts contained in the range to be deleted. /// It's important that no old parts remain (after the merge), because otherwise, @@ -1458,7 +1458,7 @@ void StorageReplicatedMergeTree::executeDropRange(const LogEntry & entry) /// If DETACH clone parts to detached/ directory for (const auto & part : parts_to_remove) { - LOG_INFO_FORMATTED(log, "Detaching {}", part->relative_path); + LOG_INFO(log, "Detaching {}", part->relative_path); part->makeCloneInDetached(""); } } @@ -1467,9 +1467,9 @@ void StorageReplicatedMergeTree::executeDropRange(const LogEntry & entry) tryRemovePartsFromZooKeeperWithRetries(parts_to_remove); if (entry.detach) - LOG_DEBUG_FORMATTED(log, "Detached {} parts inside {}.", parts_to_remove.size(), entry.new_part_name); + LOG_DEBUG(log, "Detached {} parts inside {}.", parts_to_remove.size(), entry.new_part_name); else - LOG_DEBUG_FORMATTED(log, "Removed {} parts inside {}.", parts_to_remove.size(), entry.new_part_name); + LOG_DEBUG(log, "Removed {} parts inside {}.", parts_to_remove.size(), entry.new_part_name); /// We want to remove dropped parts from disk as soon as possible /// To be removed a partition should have zero refcount, therefore call the cleanup thread at exit @@ -1554,14 +1554,14 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) if (parts_to_add.empty()) { - LOG_INFO_FORMATTED(log, "All parts from REPLACE PARTITION command have been already attached"); + LOG_INFO(log, "All parts from REPLACE PARTITION command have been already attached"); tryRemovePartsFromZooKeeperWithRetries(parts_to_remove); return true; } if (parts_to_add.size() < all_parts.size()) { - LOG_WARNING_FORMATTED(log, "Some (but not all) parts from REPLACE PARTITION command already exist. REPLACE PARTITION will not be atomic."); + LOG_WARNING(log, "Some (but not all) parts from REPLACE PARTITION command already exist. REPLACE PARTITION will not be atomic."); } StoragePtr source_table; @@ -1573,7 +1573,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) source_table = DatabaseCatalog::instance().tryGetTable(source_table_id); if (!source_table) { - LOG_DEBUG_FORMATTED(log, "Can't use {} as source table for REPLACE PARTITION command. It does not exist.", source_table_id.getNameForLogs()); + LOG_DEBUG(log, "Can't use {} as source table for REPLACE PARTITION command. It does not exist.", source_table_id.getNameForLogs()); return 0; } @@ -1584,7 +1584,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) } catch (Exception &) { - LOG_INFO_FORMATTED(log, "Can't use {} as source table for REPLACE PARTITION command. Will fetch all parts. Reason: {}", source_table_id.getNameForLogs(), getCurrentExceptionMessage(false)); + LOG_INFO(log, "Can't use {} as source table for REPLACE PARTITION command. Will fetch all parts. Reason: {}", source_table_id.getNameForLogs(), getCurrentExceptionMessage(false)); return 0; } @@ -1600,7 +1600,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) auto src_part = src_data->getPartIfExists(part_desc->src_part_info, valid_states); if (!src_part) { - LOG_DEBUG_FORMATTED(log, "There is no part {} in {}", part_desc->src_part_name, source_table_id.getNameForLogs()); + LOG_DEBUG(log, "There is no part {} in {}", part_desc->src_part_name, source_table_id.getNameForLogs()); continue; } @@ -1608,7 +1608,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) if (checksum_hex != part_desc->checksum_hex) { - LOG_DEBUG_FORMATTED(log, "Part {} of {} has inappropriate checksum", part_desc->src_part_name, source_table_id.getNameForLogs()); + LOG_DEBUG(log, "Part {} of {} has inappropriate checksum", part_desc->src_part_name, source_table_id.getNameForLogs()); /// TODO: check version continue; } @@ -1624,7 +1624,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) }; size_t num_clonable_parts = clone_data_parts_from_source_table(); - LOG_DEBUG_FORMATTED(log, "Found {} parts that could be cloned (of {} required parts)", num_clonable_parts, parts_to_add.size()); + LOG_DEBUG(log, "Found {} parts that could be cloned (of {} required parts)", num_clonable_parts, parts_to_add.size()); ActiveDataPartSet adding_parts_active_set(format_version); std::unordered_map part_name_to_desc; @@ -1646,7 +1646,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) if (replica.empty()) { - LOG_DEBUG_FORMATTED(log, "Part {} is not found on remote replicas", part_desc->new_part_name); + LOG_DEBUG(log, "Part {} is not found on remote replicas", part_desc->new_part_name); /// Fallback to covering part replica = findReplicaHavingCoveringPart(part_desc->new_part_name, true, found_part_name); @@ -1654,7 +1654,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) if (replica.empty()) { /// It is not fail, since adjacent parts could cover current part - LOG_DEBUG_FORMATTED(log, "Parts covering {} are not found on remote replicas", part_desc->new_part_name); + LOG_DEBUG(log, "Parts covering {} are not found on remote replicas", part_desc->new_part_name); continue; } } @@ -1801,7 +1801,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coordination::Stat source_is_lost_stat, zkutil::ZooKeeperPtr & zookeeper) { - LOG_INFO_FORMATTED(log, "Will mimic {}", source_replica); + LOG_INFO(log, "Will mimic {}", source_replica); String source_path = zookeeper_path + "/replicas/" + source_replica; @@ -1811,12 +1811,12 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo */ while (!zookeeper->exists(source_path + "/columns")) { - LOG_INFO_FORMATTED(log, "Waiting for replica {} to be fully created", source_path); + LOG_INFO(log, "Waiting for replica {} to be fully created", source_path); zkutil::EventPtr event = std::make_shared(); if (zookeeper->exists(source_path + "/columns", nullptr, event)) { - LOG_WARNING_FORMATTED(log, "Oops, a watch has leaked"); + LOG_WARNING(log, "Oops, a watch has leaked"); break; } @@ -1878,7 +1878,7 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo { /// If source replica's log_pointer changed than we probably read /// stale state of /queue and have to try one more time. - LOG_WARNING_FORMATTED(log, "Log pointer of source replica {} changed while we loading queue nodes. Will retry.", source_replica); + LOG_WARNING(log, "Log pointer of source replica {} changed while we loading queue nodes. Will retry.", source_replica); continue; } else @@ -1913,7 +1913,7 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo { queue.remove(zookeeper, part); parts_to_remove_from_zk.emplace_back(part); - LOG_WARNING_FORMATTED(log, "Source replica does not have part {}. Removing it from ZooKeeper.", part); + LOG_WARNING(log, "Source replica does not have part {}. Removing it from ZooKeeper.", part); } } tryRemovePartsFromZooKeeperWithRetries(parts_to_remove_from_zk); @@ -1925,7 +1925,7 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo if (active_parts_set.getContainingPart(part->name).empty()) { parts_to_remove_from_working_set.emplace_back(part); - LOG_WARNING_FORMATTED(log, "Source replica does not have part {}. Removing it from working set.", part->name); + LOG_WARNING(log, "Source replica does not have part {}. Removing it from working set.", part->name); } } removePartsFromWorkingSet(parts_to_remove_from_working_set, true); @@ -1941,7 +1941,7 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo zookeeper->create(replica_path + "/queue/queue-", log_entry.toString(), zkutil::CreateMode::PersistentSequential); } - LOG_DEBUG_FORMATTED(log, "Queued {} parts to be fetched", active_parts.size()); + LOG_DEBUG(log, "Queued {} parts to be fetched", active_parts.size()); /// Add content of the reference/master replica queue to the queue. for (const String & entry : source_queue) @@ -1949,7 +1949,7 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo zookeeper->create(replica_path + "/queue/queue-", entry, zkutil::CreateMode::PersistentSequential); } - LOG_DEBUG_FORMATTED(log, "Copied {} queue entries", source_queue.size()); + LOG_DEBUG(log, "Copied {} queue entries", source_queue.size()); } @@ -2103,17 +2103,17 @@ BackgroundProcessingPoolTaskResult StorageReplicatedMergeTree::queueTask() if (e.code() == ErrorCodes::NO_REPLICA_HAS_PART) { /// If no one has the right part, probably not all replicas work; We will not write to log with Error level. - LOG_INFO_FORMATTED(log, e.displayText()); + LOG_INFO(log, e.displayText()); } else if (e.code() == ErrorCodes::ABORTED) { /// Interrupted merge or downloading a part is not an error. - LOG_INFO_FORMATTED(log, e.message()); + LOG_INFO(log, e.message()); } else if (e.code() == ErrorCodes::PART_IS_TEMPORARILY_LOCKED) { /// Part cannot be added temporarily - LOG_INFO_FORMATTED(log, e.displayText()); + LOG_INFO(log, e.displayText()); cleanup_thread.wakeup(); } else @@ -2191,7 +2191,7 @@ void StorageReplicatedMergeTree::mergeSelectingTask() size_t merges_and_mutations_sum = merges_and_mutations_queued.first + merges_and_mutations_queued.second; if (merges_and_mutations_sum >= storage_settings_ptr->max_replicated_merges_in_queue) { - LOG_TRACE_FORMATTED(log, "Number of queued merges ({}) and part mutations ({}) is greater than max_replicated_merges_in_queue ({}), so won't select new parts to merge or mutate.", merges_and_mutations_queued.first, merges_and_mutations_queued.second, storage_settings_ptr->max_replicated_merges_in_queue); + LOG_TRACE(log, "Number of queued merges ({}) and part mutations ({}) is greater than max_replicated_merges_in_queue ({}), so won't select new parts to merge or mutate.", merges_and_mutations_queued.first, merges_and_mutations_queued.second, storage_settings_ptr->max_replicated_merges_in_queue); } else { @@ -2301,7 +2301,7 @@ bool StorageReplicatedMergeTree::createLogEntryToMergeParts( const auto & part = parts[i]; if (part->modification_time + MAX_AGE_OF_LOCAL_PART_THAT_WASNT_ADDED_TO_ZOOKEEPER < time(nullptr)) { - LOG_WARNING_FORMATTED(log, "Part {} (that was selected for merge) with age {} seconds exists locally but not in ZooKeeper. Won't do merge with that part and will check it.", part->name, (time(nullptr) - part->modification_time)); + LOG_WARNING(log, "Part {} (that was selected for merge) with age {} seconds exists locally but not in ZooKeeper. Won't do merge with that part and will check it.", part->name, (time(nullptr) - part->modification_time)); enqueuePartForCheck(part->name); } } @@ -2343,7 +2343,7 @@ bool StorageReplicatedMergeTree::createLogEntryToMutatePart(const IMergeTreeData { if (part.modification_time + MAX_AGE_OF_LOCAL_PART_THAT_WASNT_ADDED_TO_ZOOKEEPER < time(nullptr)) { - LOG_WARNING_FORMATTED(log, "Part {} (that was selected for mutation) with age {} seconds exists locally but not in ZooKeeper. Won't mutate that part and will check it.", part.name, (time(nullptr) - part.modification_time)); + LOG_WARNING(log, "Part {} (that was selected for mutation) with age {} seconds exists locally but not in ZooKeeper. Won't mutate that part and will check it.", part.name, (time(nullptr) - part.modification_time)); enqueuePartForCheck(part.name); } @@ -2420,7 +2420,7 @@ void StorageReplicatedMergeTree::enterLeaderElection() auto callback = [this]() { CurrentMetrics::add(CurrentMetrics::LeaderReplica); - LOG_INFO_FORMATTED(log, "Became leader"); + LOG_INFO(log, "Became leader"); is_leader = true; merge_selecting_task->activateAndSchedule(); @@ -2455,7 +2455,7 @@ void StorageReplicatedMergeTree::exitLeaderElection() if (is_leader) { CurrentMetrics::sub(CurrentMetrics::LeaderReplica); - LOG_INFO_FORMATTED(log, "Stopped being leader"); + LOG_INFO(log, "Stopped being leader"); is_leader = false; merge_selecting_task->deactivate(); @@ -2533,7 +2533,7 @@ String StorageReplicatedMergeTree::findReplicaHavingCoveringPart(LogEntry & entr String reject_reason; if (!queue.addFuturePartIfNotCoveredByThem(largest_part_found, entry, reject_reason)) { - LOG_INFO_FORMATTED(log, "Will not fetch part {} covering {}. {}", largest_part_found, entry.new_part_name, reject_reason); + LOG_INFO(log, "Will not fetch part {} covering {}. {}", largest_part_found, entry.new_part_name, reject_reason); return {}; } } @@ -2741,7 +2741,7 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Strin if (auto part = getPartIfExists(part_info, {IMergeTreeDataPart::State::Outdated, IMergeTreeDataPart::State::Deleting})) { - LOG_DEBUG_FORMATTED(log, "Part {} should be deleted after previous attempt before fetch", part->name); + LOG_DEBUG(log, "Part {} should be deleted after previous attempt before fetch", part->name); /// Force immediate parts cleanup to delete the part that was left from the previous fetch attempt. cleanup_thread.wakeup(); return false; @@ -2751,7 +2751,7 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Strin std::lock_guard lock(currently_fetching_parts_mutex); if (!currently_fetching_parts.insert(part_name).second) { - LOG_DEBUG_FORMATTED(log, "Part {} is already fetching right now", part_name); + LOG_DEBUG(log, "Part {} is already fetching right now", part_name); return false; } } @@ -2762,7 +2762,7 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Strin currently_fetching_parts.erase(part_name); }); - LOG_DEBUG_FORMATTED(log, "Fetching part {} from {}", part_name, source_replica_path); + LOG_DEBUG(log, "Fetching part {} from {}", part_name, source_replica_path); TableStructureReadLockHolder table_lock_holder; if (!to_detached) @@ -2810,7 +2810,7 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Strin if (source_part_checksums == desired_checksums) { - LOG_TRACE_FORMATTED(log, "Found local part {} with the same checksums as {}", source_part->name, part_name); + LOG_TRACE(log, "Found local part {} with the same checksums as {}", source_part->name, part_name); part_to_clone = source_part; } } @@ -2872,7 +2872,7 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Strin for (const auto & replaced_part : replaced_parts) { - LOG_DEBUG_FORMATTED(log, "Part {} is rendered obsolete by fetching part {}", replaced_part->name, part_name); + LOG_DEBUG(log, "Part {} is rendered obsolete by fetching part {}", replaced_part->name, part_name); ProfileEvents::increment(ProfileEvents::ObsoleteReplicatedParts); } @@ -2894,9 +2894,9 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Strin ProfileEvents::increment(ProfileEvents::ReplicatedPartFetches); if (part_to_clone) - LOG_DEBUG_FORMATTED(log, "Cloned part {} from {}{}", part_name, part_to_clone->name, to_detached ? " (to 'detached' directory)" : ""); + LOG_DEBUG(log, "Cloned part {} from {}{}", part_name, part_to_clone->name, to_detached ? " (to 'detached' directory)" : ""); else - LOG_DEBUG_FORMATTED(log, "Fetched part {} from {}{}", part_name, part_to_clone->name, to_detached ? " (to 'detached' directory)" : ""); + LOG_DEBUG(log, "Fetched part {} from {}{}", part_name, part_to_clone->name, to_detached ? " (to 'detached' directory)" : ""); return true; } @@ -3203,7 +3203,7 @@ bool StorageReplicatedMergeTree::optimize(const ASTPtr & query, const ASTPtr & p message << "Cannot select parts for optimization"; if (!disable_reason.empty()) message << ": " << disable_reason; - LOG_INFO_FORMATTED(log, message.str()); + LOG_INFO(log, message.str()); return handle_noop(message.str()); } @@ -3246,13 +3246,13 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer /// TODO (relax this lock) auto table_lock = lockExclusively(RWLockImpl::NO_QUERY, getSettings()->lock_acquire_timeout_for_background_operations); - LOG_INFO_FORMATTED(log, "Metadata changed in ZooKeeper. Applying changes locally."); + LOG_INFO(log, "Metadata changed in ZooKeeper. Applying changes locally."); auto metadata_diff = ReplicatedMergeTreeTableMetadata(*this).checkAndFindDiff(metadata_from_entry); setTableStructure(std::move(columns_from_entry), metadata_diff); metadata_version = entry.alter_version; - LOG_INFO_FORMATTED(log, "Applied changes to the metadata of the table. Current metadata version: {}", metadata_version); + LOG_INFO(log, "Applied changes to the metadata of the table. Current metadata version: {}", metadata_version); } /// This transaction may not happen, but it's OK, because on the next retry we will eventually create/update this node @@ -3432,12 +3432,12 @@ void StorageReplicatedMergeTree::alter( std::vector unwaited; if (query_context.getSettingsRef().replication_alter_partitions_sync == 2) { - LOG_DEBUG_FORMATTED(log, "Updated shared metadata nodes in ZooKeeper. Waiting for replicas to apply changes."); + LOG_DEBUG(log, "Updated shared metadata nodes in ZooKeeper. Waiting for replicas to apply changes."); unwaited = waitForAllReplicasToProcessLogEntry(*alter_entry, false); } else if (query_context.getSettingsRef().replication_alter_partitions_sync == 1) { - LOG_DEBUG_FORMATTED(log, "Updated shared metadata nodes in ZooKeeper. Waiting for replicas to apply changes."); + LOG_DEBUG(log, "Updated shared metadata nodes in ZooKeeper. Waiting for replicas to apply changes."); waitForReplicaToProcessLogEntry(replica_name, *alter_entry); } @@ -3446,9 +3446,9 @@ void StorageReplicatedMergeTree::alter( if (mutation_znode) { - LOG_DEBUG_FORMATTED(log, "Metadata changes applied. Will wait for data changes."); + LOG_DEBUG(log, "Metadata changes applied. Will wait for data changes."); waitMutation(*mutation_znode, query_context.getSettingsRef().replication_alter_partitions_sync); - LOG_DEBUG_FORMATTED(log, "Data changes applied."); + LOG_DEBUG(log, "Data changes applied."); } } @@ -3651,7 +3651,7 @@ void StorageReplicatedMergeTree::attachPartition(const ASTPtr & partition, bool String old_name = loaded_parts[i]->name; output.writeExistingPart(loaded_parts[i]); renamed_parts.old_and_new_names[i].first.clear(); - LOG_DEBUG_FORMATTED(log, "Attached part {} as {}", old_name, loaded_parts[i]->name); + LOG_DEBUG(log, "Attached part {} as {}", old_name, loaded_parts[i]->name); } } @@ -3691,21 +3691,21 @@ void StorageReplicatedMergeTree::drop() if (zookeeper->expired()) throw Exception("Table was not dropped because ZooKeeper session has expired.", ErrorCodes::TABLE_WAS_NOT_DROPPED); - LOG_INFO_FORMATTED(log, "Removing replica {}", replica_path); + LOG_INFO(log, "Removing replica {}", replica_path); replica_is_active_node = nullptr; /// It may left some garbage if replica_path subtree are concurently modified zookeeper->tryRemoveRecursive(replica_path); if (zookeeper->exists(replica_path)) - LOG_ERROR_FORMATTED(log, "Replica was not completely removed from ZooKeeper, {} still exists and may contain some garbage.", replica_path); + LOG_ERROR(log, "Replica was not completely removed from ZooKeeper, {} still exists and may contain some garbage.", replica_path); /// Check that `zookeeper_path` exists: it could have been deleted by another replica after execution of previous line. Strings replicas; if (zookeeper->tryGetChildren(zookeeper_path + "/replicas", replicas) == Coordination::ZOK && replicas.empty()) { - LOG_INFO_FORMATTED(log, "Removing table {} (this might take several minutes)", zookeeper_path); + LOG_INFO(log, "Removing table {} (this might take several minutes)", zookeeper_path); zookeeper->tryRemoveRecursive(zookeeper_path); if (zookeeper->exists(zookeeper_path)) - LOG_ERROR_FORMATTED(log, "Table was not completely removed from ZooKeeper, {} still exists and may contain some garbage.", zookeeper_path); + LOG_ERROR(log, "Table was not completely removed from ZooKeeper, {} still exists and may contain some garbage.", zookeeper_path); } } @@ -3800,7 +3800,7 @@ StorageReplicatedMergeTree::allocateBlockNumber( Strings StorageReplicatedMergeTree::waitForAllReplicasToProcessLogEntry(const ReplicatedMergeTreeLogEntryData & entry, bool wait_for_non_active) { - LOG_DEBUG_FORMATTED(log, "Waiting for all replicas to process {}", entry.znode_name); + LOG_DEBUG(log, "Waiting for all replicas to process {}", entry.znode_name); auto zookeeper = getZooKeeper(); Strings replicas = zookeeper->getChildren(zookeeper_path + "/replicas"); @@ -3818,7 +3818,7 @@ Strings StorageReplicatedMergeTree::waitForAllReplicasToProcessLogEntry(const Re } } - LOG_DEBUG_FORMATTED(log, "Finished waiting for all replicas to process {}", entry.znode_name); + LOG_DEBUG(log, "Finished waiting for all replicas to process {}", entry.znode_name); return unwaited; } @@ -3858,7 +3858,7 @@ bool StorageReplicatedMergeTree::waitForReplicaToProcessLogEntry(const String & UInt64 log_index = parse(entry.znode_name.substr(entry.znode_name.size() - 10)); log_node_name = entry.znode_name; - LOG_DEBUG_FORMATTED(log, "Waiting for {} to pull {} to queue", replica, log_node_name); + LOG_DEBUG(log, "Waiting for {} to pull {} to queue", replica, log_node_name); /// Let's wait until entry gets into the replica queue. while (wait_for_non_active || !check_replica_become_inactive()) @@ -3906,7 +3906,7 @@ bool StorageReplicatedMergeTree::waitForReplicaToProcessLogEntry(const String & if (found) { - LOG_DEBUG_FORMATTED(log, "Waiting for {} to pull {} to queue", replica, log_node_name); + LOG_DEBUG(log, "Waiting for {} to pull {} to queue", replica, log_node_name); /// Let's wait until the entry gets into the replica queue. while (wait_for_non_active || !check_replica_become_inactive()) @@ -3928,9 +3928,9 @@ bool StorageReplicatedMergeTree::waitForReplicaToProcessLogEntry(const String & throw Exception("Logical error: unexpected name of log node: " + entry.znode_name, ErrorCodes::LOGICAL_ERROR); if (!log_node_name.empty()) - LOG_DEBUG_FORMATTED(log, "Looking for node corresponding to {} in {} queue", log_node_name, replica); + LOG_DEBUG(log, "Looking for node corresponding to {} in {} queue", log_node_name, replica); else - LOG_DEBUG_FORMATTED(log, "Looking for corresponding node in {} queue", replica); + LOG_DEBUG(log, "Looking for corresponding node in {} queue", replica); /** Second - find the corresponding entry in the queue of the specified replica. * Its number may match neither the `log` node nor the `queue` node of the current replica (for us). @@ -3954,11 +3954,11 @@ bool StorageReplicatedMergeTree::waitForReplicaToProcessLogEntry(const String & /// While looking for the record, it has already been executed and deleted. if (queue_entry_to_wait_for.empty()) { - LOG_DEBUG_FORMATTED(log, "No corresponding node found. Assuming it has been already processed. Found {} nodes", queue_entries.size()); + LOG_DEBUG(log, "No corresponding node found. Assuming it has been already processed. Found {} nodes", queue_entries.size()); return true; } - LOG_DEBUG_FORMATTED(log, "Waiting for {} to disappear from {} queue", queue_entry_to_wait_for, replica); + LOG_DEBUG(log, "Waiting for {} to disappear from {} queue", queue_entry_to_wait_for, replica); /// Third - wait until the entry disappears from the replica queue or replica become inactive. String path_to_wait_on = zookeeper_path + "/replicas/" + replica + "/queue/" + queue_entry_to_wait_for; @@ -4238,7 +4238,7 @@ void StorageReplicatedMergeTree::fetchPartition(const ASTPtr & partition, const if (from.back() == '/') from.resize(from.size() - 1); - LOG_INFO_FORMATTED(log, "Will fetch partition {} from shard {}", partition_id, from_); + LOG_INFO(log, "Will fetch partition {} from shard {}", partition_id, from_); /** Let's check that there is no such partition in the `detached` directory (where we will write the downloaded parts). * Unreliable (there is a race condition) - such a partition may appear a little later. @@ -4309,7 +4309,7 @@ void StorageReplicatedMergeTree::fetchPartition(const ASTPtr & partition, const if (best_replica.empty()) throw Exception("Logical error: cannot choose best replica.", ErrorCodes::LOGICAL_ERROR); - LOG_INFO_FORMATTED(log, "Found {} replicas, {} of them are active. Selected {} to fetch from.", replicas.size(), active_replicas.size(), best_replica); + LOG_INFO(log, "Found {} replicas, {} of them are active. Selected {} to fetch from.", replicas.size(), active_replicas.size(), best_replica); String best_replica_path = from + "/replicas/" + best_replica; @@ -4325,7 +4325,7 @@ void StorageReplicatedMergeTree::fetchPartition(const ASTPtr & partition, const do { if (try_no) - LOG_INFO_FORMATTED(log, "Some of parts ({}) are missing. Will try to fetch covering parts.", missing_parts.size()); + LOG_INFO(log, "Some of parts ({}) are missing. Will try to fetch covering parts.", missing_parts.size()); if (try_no >= query_context.getSettings().max_fetch_partition_retries_count) throw Exception("Too many retries to fetch parts from " + best_replica_path, ErrorCodes::TOO_MANY_RETRIES_TO_FETCH_PARTS); @@ -4359,11 +4359,11 @@ void StorageReplicatedMergeTree::fetchPartition(const ASTPtr & partition, const if (!containing_part.empty()) parts_to_fetch.push_back(containing_part); else - LOG_WARNING_FORMATTED(log, "Part {} on replica {} has been vanished.", missing_part, best_replica_path); + LOG_WARNING(log, "Part {} on replica {} has been vanished.", missing_part, best_replica_path); } } - LOG_INFO_FORMATTED(log, "Parts to fetch: {}", parts_to_fetch.size()); + LOG_INFO(log, "Parts to fetch: {}", parts_to_fetch.size()); missing_parts.clear(); for (const String & part : parts_to_fetch) @@ -4378,7 +4378,7 @@ void StorageReplicatedMergeTree::fetchPartition(const ASTPtr & partition, const && e.code() != ErrorCodes::CANNOT_READ_ALL_DATA) throw; - LOG_INFO_FORMATTED(log, e.displayText()); + LOG_INFO(log, e.displayText()); missing_parts.push_back(part); } } @@ -4480,12 +4480,12 @@ void StorageReplicatedMergeTree::mutate(const MutationCommands & commands, const const String & path_created = dynamic_cast(responses[1].get())->path_created; entry.znode_name = path_created.substr(path_created.find_last_of('/') + 1); - LOG_TRACE_FORMATTED(log, "Created mutation with ID {}", entry.znode_name); + LOG_TRACE(log, "Created mutation with ID {}", entry.znode_name); break; } else if (rc == Coordination::ZBADVERSION) { - LOG_TRACE_FORMATTED(log, "Version conflict when trying to create a mutation node, retrying..."); + LOG_TRACE(log, "Version conflict when trying to create a mutation node, retrying..."); continue; } else @@ -4522,7 +4522,7 @@ CancellationCode StorageReplicatedMergeTree::killMutation(const String & mutatio zkutil::ZooKeeperPtr zookeeper = getZooKeeper(); - LOG_TRACE_FORMATTED(log, "Killing mutation {}", mutation_id); + LOG_TRACE(log, "Killing mutation {}", mutation_id); auto mutation_entry = queue.removeMutation(zookeeper, mutation_id); if (!mutation_entry) @@ -4589,7 +4589,7 @@ void StorageReplicatedMergeTree::clearOldPartsAndRemoveFromZK() remove_parts_from_filesystem(parts_to_delete_only_from_filesystem); removePartsFinally(parts_to_delete_only_from_filesystem); - LOG_DEBUG_FORMATTED(log, "Removed {} old duplicate parts", parts_to_delete_only_from_filesystem.size()); + LOG_DEBUG(log, "Removed {} old duplicate parts", parts_to_delete_only_from_filesystem.size()); } /// Delete normal parts from ZooKeeper @@ -4600,17 +4600,17 @@ void StorageReplicatedMergeTree::clearOldPartsAndRemoveFromZK() for (const auto & part : parts_to_delete_completely) part_names_to_delete_completely.emplace_back(part->name); - LOG_DEBUG_FORMATTED(log, "Removing {} old parts from ZooKeeper", parts_to_delete_completely.size()); + LOG_DEBUG(log, "Removing {} old parts from ZooKeeper", parts_to_delete_completely.size()); removePartsFromZooKeeper(zookeeper, part_names_to_delete_completely, &part_names_to_retry_deletion); } catch (...) { - LOG_ERROR_FORMATTED(log, "There is a problem with deleting parts from ZooKeeper: {}", getCurrentExceptionMessage(true)); + LOG_ERROR(log, "There is a problem with deleting parts from ZooKeeper: {}", getCurrentExceptionMessage(true)); } /// Part names that were reliably deleted from ZooKeeper should be deleted from filesystem auto num_reliably_deleted_parts = parts_to_delete_completely.size() - part_names_to_retry_deletion.size(); - LOG_DEBUG_FORMATTED(log, "Removed {} old parts from ZooKeeper. Removing them from filesystem.", num_reliably_deleted_parts); + LOG_DEBUG(log, "Removed {} old parts from ZooKeeper. Removing them from filesystem.", num_reliably_deleted_parts); /// Delete normal parts on two sets for (auto & part : parts_to_delete_completely) @@ -4625,7 +4625,7 @@ void StorageReplicatedMergeTree::clearOldPartsAndRemoveFromZK() if (!parts_to_retry_deletion.empty()) { rollbackDeletingParts(parts_to_retry_deletion); - LOG_DEBUG_FORMATTED(log, "Will retry deletion of {} parts in the next time", parts_to_retry_deletion.size()); + LOG_DEBUG(log, "Will retry deletion of {} parts in the next time", parts_to_retry_deletion.size()); } /// Remove parts from filesystem and finally from data_parts @@ -4634,7 +4634,7 @@ void StorageReplicatedMergeTree::clearOldPartsAndRemoveFromZK() remove_parts_from_filesystem(parts_to_remove_from_filesystem); removePartsFinally(parts_to_remove_from_filesystem); - LOG_DEBUG_FORMATTED(log, "Removed {} old parts", parts_to_remove_from_filesystem.size()); + LOG_DEBUG(log, "Removed {} old parts", parts_to_remove_from_filesystem.size()); } } @@ -4744,7 +4744,7 @@ void StorageReplicatedMergeTree::removePartsFromZooKeeper( } else { - LOG_DEBUG_FORMATTED(log, "There is no part {} in ZooKeeper, it was only in filesystem", part_names[i]); + LOG_DEBUG(log, "There is no part {} in ZooKeeper, it was only in filesystem", part_names[i]); // emplace invalid future so that the total number of futures is the same as part_names.size(); remove_futures.emplace_back(); } @@ -4769,7 +4769,7 @@ void StorageReplicatedMergeTree::removePartsFromZooKeeper( continue; else if (response.error == Coordination::ZNONODE) { - LOG_DEBUG_FORMATTED(log, "There is no part {} in ZooKeeper, it was only in filesystem", part_names[i]); + LOG_DEBUG(log, "There is no part {} in ZooKeeper, it was only in filesystem", part_names[i]); continue; } else if (Coordination::isHardwareError(response.error)) @@ -4779,7 +4779,7 @@ void StorageReplicatedMergeTree::removePartsFromZooKeeper( continue; } else - LOG_WARNING_FORMATTED(log, "Cannot remove part {} from ZooKeeper: {}", part_names[i], zkutil::ZooKeeper::error2string(response.error)); + LOG_WARNING(log, "Cannot remove part {} from ZooKeeper: {}", part_names[i], zkutil::ZooKeeper::error2string(response.error)); } } @@ -4828,10 +4828,10 @@ void StorageReplicatedMergeTree::clearBlocksInPartition( zookeeper.removeRecursive(path); } else if (rc) - LOG_WARNING_FORMATTED(log, "Error while deleting ZooKeeper path `{}`: {}, ignoring.", path, zkutil::ZooKeeper::error2string(rc)); + LOG_WARNING(log, "Error while deleting ZooKeeper path `{}`: {}, ignoring.", path, zkutil::ZooKeeper::error2string(rc)); } - LOG_TRACE_FORMATTED(log, "Deleted {} deduplication block IDs in partition ID {}", to_delete_futures.size(), partition_id); + LOG_TRACE(log, "Deleted {} deduplication block IDs in partition ID {}", to_delete_futures.size(), partition_id); } void StorageReplicatedMergeTree::replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, @@ -4852,7 +4852,7 @@ void StorageReplicatedMergeTree::replacePartitionFrom(const StoragePtr & source_ Strings part_checksums; std::vector ephemeral_locks; - LOG_DEBUG_FORMATTED(log, "Cloning {} parts", src_all_parts.size()); + LOG_DEBUG(log, "Cloning {} parts", src_all_parts.size()); static const String TMP_PREFIX = "tmp_replace_from_"; auto zookeeper = getZooKeeper(); @@ -4897,16 +4897,16 @@ void StorageReplicatedMergeTree::replacePartitionFrom(const StoragePtr & source_ String hash_hex = src_part->checksums.getTotalChecksumHex(); if (replace) - LOG_INFO_FORMATTED(log, "Trying to replace {} with hash_hex {}", src_part->name, hash_hex); + LOG_INFO(log, "Trying to replace {} with hash_hex {}", src_part->name, hash_hex); else - LOG_INFO_FORMATTED(log, "Trying to attach {} with hash_hex {}", src_part->name, hash_hex); + LOG_INFO(log, "Trying to attach {} with hash_hex {}", src_part->name, hash_hex); String block_id_path = replace ? "" : (zookeeper_path + "/blocks/" + partition_id + "_replace_from_" + hash_hex); auto lock = allocateBlockNumber(partition_id, zookeeper, block_id_path); if (!lock) { - LOG_INFO_FORMATTED(log, "Part {} (hash {}) has been already attached", src_part->name, hash_hex); + LOG_INFO(log, "Part {} (hash {}) has been already attached", src_part->name, hash_hex); continue; } @@ -5045,7 +5045,7 @@ void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_ta Strings part_checksums; std::vector ephemeral_locks; - LOG_DEBUG_FORMATTED(log, "Cloning {} parts", src_all_parts.size()); + LOG_DEBUG(log, "Cloning {} parts", src_all_parts.size()); static const String TMP_PREFIX = "tmp_move_from_"; auto zookeeper = getZooKeeper(); @@ -5081,7 +5081,7 @@ void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_ta auto lock = dest_table_storage->allocateBlockNumber(partition_id, zookeeper, block_id_path); if (!lock) { - LOG_INFO_FORMATTED(log, "Part {} (hash {}) has been already attached", src_part->name, hash_hex); + LOG_INFO(log, "Part {} (hash {}) has been already attached", src_part->name, hash_hex); continue; } @@ -5336,7 +5336,7 @@ bool StorageReplicatedMergeTree::dropPartsInPartition( MergeTreePartInfo drop_range_info; if (!getFakePartCoveringAllPartsInPartition(partition_id, drop_range_info)) { - LOG_INFO_FORMATTED(log, "Will not drop partition {}, it is empty.", partition_id); + LOG_INFO(log, "Will not drop partition {}, it is empty.", partition_id); return false; } @@ -5351,7 +5351,7 @@ bool StorageReplicatedMergeTree::dropPartsInPartition( queue.disableMergesInBlockRange(drop_range_fake_part_name); } - LOG_DEBUG_FORMATTED(log, "Disabled merges covered by range {}", drop_range_fake_part_name); + LOG_DEBUG(log, "Disabled merges covered by range {}", drop_range_fake_part_name); /// Finally, having achieved the necessary invariants, you can put an entry in the log. entry.type = LogEntry::DROP_RANGE; diff --git a/src/Storages/StorageSet.cpp b/src/Storages/StorageSet.cpp index 939a8367f60..4682b873dc5 100644 --- a/src/Storages/StorageSet.cpp +++ b/src/Storages/StorageSet.cpp @@ -191,7 +191,7 @@ void StorageSetOrJoinBase::restoreFromFile(const String & file_path) backup_stream.readSuffix(); /// TODO Add speed, compressed bytes, data volume in memory, compression ratio ... Generalize all statistics logging in project. - LOG_INFO_FORMATTED(&Logger::get("StorageSetOrJoinBase"), "Loaded from backup file {}. {} rows, {}. State has {} unique rows.", + LOG_INFO(&Logger::get("StorageSetOrJoinBase"), "Loaded from backup file {}. {} rows, {}. State has {} unique rows.", file_path, backup_stream.getProfileInfo().rows, formatReadableSizeWithBinarySuffix(backup_stream.getProfileInfo().bytes), getSize()); } From b99dd3661cd4475a0f2c1e0b67161a6edb47a3a7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 24 May 2020 01:25:25 +0300 Subject: [PATCH 671/738] Fix bad whitespace --- programs/copier/ZooKeeperStaff.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/copier/ZooKeeperStaff.h b/programs/copier/ZooKeeperStaff.h index f5224857259..edd0d9e43d2 100644 --- a/programs/copier/ZooKeeperStaff.h +++ b/programs/copier/ZooKeeperStaff.h @@ -187,7 +187,7 @@ public: stale->store(true); break; case Coordination::CHANGED: - LOG_DEBUG(logger, "CleanStateClock change: CHANGED, at{}", rsp.path); + LOG_DEBUG(logger, "CleanStateClock change: CHANGED, at {}", rsp.path); stale->store(true); } } From bff624e2ef481268d0feed5685abe2291ddd6a7c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 24 May 2020 01:30:43 +0300 Subject: [PATCH 672/738] Fix style --- src/Interpreters/SystemLog.h | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h index 49b8168678f..3c0d2159ef9 100644 --- a/src/Interpreters/SystemLog.h +++ b/src/Interpreters/SystemLog.h @@ -61,6 +61,7 @@ namespace ErrorCodes } #define DBMS_SYSTEM_LOG_QUEUE_SIZE 1048576 + class Context; class QueryLog; class QueryThreadLog; @@ -232,7 +233,9 @@ void SystemLog::add(const LogElement & element) if (is_shutdown) return; - if (queue.size() == DBMS_SYSTEM_LOG_QUEUE_SIZE / 2) { // The queue more than half full, time to flush. + if (queue.size() == DBMS_SYSTEM_LOG_QUEUE_SIZE / 2) + { + // The queue more than half full, time to flush. // We only check for strict equality, because messages are added one // by one, under exclusive lock, so we will see each message count. // It is enough to only wake the flushing thread once, after the message @@ -245,7 +248,9 @@ void SystemLog::add(const LogElement & element) LOG_INFO(log, "Queue is half full for system log '{}'.", demangle(typeid(*this).name())); } - if (queue.size() >= DBMS_SYSTEM_LOG_QUEUE_SIZE) { // Ignore all further entries until the queue is flushed. + if (queue.size() >= DBMS_SYSTEM_LOG_QUEUE_SIZE) + { + // Ignore all further entries until the queue is flushed. // Log a message about that. Don't spam it -- this might be especially // problematic in case of trace log. Remember what the front index of the // queue was when we last logged the message. If it changed, it means the From 538869334340670c3b28f39e8d164b14df12dc07 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 24 May 2020 01:33:29 +0300 Subject: [PATCH 673/738] Fix Arcadia build --- base/common/ya.make | 1 + 1 file changed, 1 insertion(+) diff --git a/base/common/ya.make b/base/common/ya.make index fbe63baafdf..6e45b0193c5 100644 --- a/base/common/ya.make +++ b/base/common/ya.make @@ -24,6 +24,7 @@ PEERDIR( contrib/libs/cxxsupp/libcxx-filesystem contrib/libs/poco/Net contrib/libs/poco/Util + contrib/libs/fmt contrib/restricted/boost contrib/restricted/cityhash-1.0.2 ) From a70b819ca24b0d79b3be1e299dd5e8923d7414d6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 24 May 2020 02:33:26 +0300 Subject: [PATCH 674/738] Adapted test time --- .../performance/agg_functions_min_max_any.xml | 186 +++++++++--------- 1 file changed, 93 insertions(+), 93 deletions(-) diff --git a/tests/performance/agg_functions_min_max_any.xml b/tests/performance/agg_functions_min_max_any.xml index c89cd05700a..43bb8c823ac 100644 --- a/tests/performance/agg_functions_min_max_any.xml +++ b/tests/performance/agg_functions_min_max_any.xml @@ -3,102 +3,102 @@ - hits_100m_single + test.hits -select min(Title) from hits_100m_single where Title != '' group by intHash32(UserID) % 1000000 -select max(Title) from hits_100m_single where Title != '' group by intHash32(UserID) % 1000000 -select any(Title) from hits_100m_single where Title != '' group by intHash32(UserID) % 1000000 -select anyHeavy(Title) from hits_100m_single where Title != '' group by intHash32(UserID) % 1000000 -select min(URL) from hits_100m_single where URL != '' group by intHash32(UserID) % 1000000 -select max(URL) from hits_100m_single where URL != '' group by intHash32(UserID) % 1000000 -select any(URL) from hits_100m_single where URL != '' group by intHash32(UserID) % 1000000 -select anyHeavy(URL) from hits_100m_single where URL != '' group by intHash32(UserID) % 1000000 -select min(Referer) from hits_100m_single where Referer != '' group by intHash32(UserID) % 1000000 -select max(Referer) from hits_100m_single where Referer != '' group by intHash32(UserID) % 1000000 -select any(Referer) from hits_100m_single where Referer != '' group by intHash32(UserID) % 1000000 -select anyHeavy(Referer) from hits_100m_single where Referer != '' group by intHash32(UserID) % 1000000 -select min(FlashMinor2) from hits_100m_single where FlashMinor2 != '' group by intHash32(UserID) % 1000000 -select max(FlashMinor2) from hits_100m_single where FlashMinor2 != '' group by intHash32(UserID) % 1000000 -select any(FlashMinor2) from hits_100m_single where FlashMinor2 != '' group by intHash32(UserID) % 1000000 -select anyHeavy(FlashMinor2) from hits_100m_single where FlashMinor2 != '' group by intHash32(UserID) % 1000000 -select min(MobilePhoneModel) from hits_100m_single where MobilePhoneModel != '' group by intHash32(UserID) % 1000000 -select max(MobilePhoneModel) from hits_100m_single where MobilePhoneModel != '' group by intHash32(UserID) % 1000000 -select any(MobilePhoneModel) from hits_100m_single where MobilePhoneModel != '' group by intHash32(UserID) % 1000000 -select anyHeavy(MobilePhoneModel) from hits_100m_single where MobilePhoneModel != '' group by intHash32(UserID) % 1000000 -select min(Params) from hits_100m_single where Params != '' group by intHash32(UserID) % 1000000 -select max(Params) from hits_100m_single where Params != '' group by intHash32(UserID) % 1000000 -select any(Params) from hits_100m_single where Params != '' group by intHash32(UserID) % 1000000 -select anyHeavy(Params) from hits_100m_single where Params != '' group by intHash32(UserID) % 1000000 -select min(SearchPhrase) from hits_100m_single where SearchPhrase != '' group by intHash32(UserID) % 1000000 -select max(SearchPhrase) from hits_100m_single where SearchPhrase != '' group by intHash32(UserID) % 1000000 -select any(SearchPhrase) from hits_100m_single where SearchPhrase != '' group by intHash32(UserID) % 1000000 -select anyHeavy(SearchPhrase) from hits_100m_single where SearchPhrase != '' group by intHash32(UserID) % 1000000 -select min(PageCharset) from hits_100m_single where PageCharset != '' group by intHash32(UserID) % 1000000 -select max(PageCharset) from hits_100m_single where PageCharset != '' group by intHash32(UserID) % 1000000 -select any(PageCharset) from hits_100m_single where PageCharset != '' group by intHash32(UserID) % 1000000 -select anyHeavy(PageCharset) from hits_100m_single where PageCharset != '' group by intHash32(UserID) % 1000000 -select min(OriginalURL) from hits_100m_single where OriginalURL != '' group by intHash32(UserID) % 1000000 -select max(OriginalURL) from hits_100m_single where OriginalURL != '' group by intHash32(UserID) % 1000000 -select any(OriginalURL) from hits_100m_single where OriginalURL != '' group by intHash32(UserID) % 1000000 -select anyHeavy(OriginalURL) from hits_100m_single where OriginalURL != '' group by intHash32(UserID) % 1000000 -select min(SocialNetwork) from hits_100m_single where SocialNetwork != '' group by intHash32(UserID) % 1000000 -select max(SocialNetwork) from hits_100m_single where SocialNetwork != '' group by intHash32(UserID) % 1000000 -select any(SocialNetwork) from hits_100m_single where SocialNetwork != '' group by intHash32(UserID) % 1000000 -select anyHeavy(SocialNetwork) from hits_100m_single where SocialNetwork != '' group by intHash32(UserID) % 1000000 -select min(SocialAction) from hits_100m_single where SocialAction != '' group by intHash32(UserID) % 1000000 -select max(SocialAction) from hits_100m_single where SocialAction != '' group by intHash32(UserID) % 1000000 -select any(SocialAction) from hits_100m_single where SocialAction != '' group by intHash32(UserID) % 1000000 -select anyHeavy(SocialAction) from hits_100m_single where SocialAction != '' group by intHash32(UserID) % 1000000 -select min(SocialSourcePage) from hits_100m_single where SocialSourcePage != '' group by intHash32(UserID) % 1000000 -select max(SocialSourcePage) from hits_100m_single where SocialSourcePage != '' group by intHash32(UserID) % 1000000 -select any(SocialSourcePage) from hits_100m_single where SocialSourcePage != '' group by intHash32(UserID) % 1000000 -select anyHeavy(SocialSourcePage) from hits_100m_single where SocialSourcePage != '' group by intHash32(UserID) % 1000000 -select min(ParamOrderID) from hits_100m_single where ParamOrderID != '' group by intHash32(UserID) % 1000000 -select max(ParamOrderID) from hits_100m_single where ParamOrderID != '' group by intHash32(UserID) % 1000000 -select any(ParamOrderID) from hits_100m_single where ParamOrderID != '' group by intHash32(UserID) % 1000000 -select anyHeavy(ParamOrderID) from hits_100m_single where ParamOrderID != '' group by intHash32(UserID) % 1000000 -select min(OpenstatServiceName) from hits_100m_single where OpenstatServiceName != '' group by intHash32(UserID) % 1000000 -select max(OpenstatServiceName) from hits_100m_single where OpenstatServiceName != '' group by intHash32(UserID) % 1000000 -select any(OpenstatServiceName) from hits_100m_single where OpenstatServiceName != '' group by intHash32(UserID) % 1000000 -select anyHeavy(OpenstatServiceName) from hits_100m_single where OpenstatServiceName != '' group by intHash32(UserID) % 1000000 -select min(OpenstatCampaignID) from hits_100m_single where OpenstatCampaignID != '' group by intHash32(UserID) % 1000000 -select max(OpenstatCampaignID) from hits_100m_single where OpenstatCampaignID != '' group by intHash32(UserID) % 1000000 -select any(OpenstatCampaignID) from hits_100m_single where OpenstatCampaignID != '' group by intHash32(UserID) % 1000000 -select anyHeavy(OpenstatCampaignID) from hits_100m_single where OpenstatCampaignID != '' group by intHash32(UserID) % 1000000 -select min(OpenstatAdID) from hits_100m_single where OpenstatAdID != '' group by intHash32(UserID) % 1000000 -select max(OpenstatAdID) from hits_100m_single where OpenstatAdID != '' group by intHash32(UserID) % 1000000 -select any(OpenstatAdID) from hits_100m_single where OpenstatAdID != '' group by intHash32(UserID) % 1000000 -select anyHeavy(OpenstatAdID) from hits_100m_single where OpenstatAdID != '' group by intHash32(UserID) % 1000000 -select min(OpenstatSourceID) from hits_100m_single where OpenstatSourceID != '' group by intHash32(UserID) % 1000000 -select max(OpenstatSourceID) from hits_100m_single where OpenstatSourceID != '' group by intHash32(UserID) % 1000000 -select any(OpenstatSourceID) from hits_100m_single where OpenstatSourceID != '' group by intHash32(UserID) % 1000000 -select anyHeavy(OpenstatSourceID) from hits_100m_single where OpenstatSourceID != '' group by intHash32(UserID) % 1000000 -select min(UTMSource) from hits_100m_single where UTMSource != '' group by intHash32(UserID) % 1000000 -select max(UTMSource) from hits_100m_single where UTMSource != '' group by intHash32(UserID) % 1000000 -select any(UTMSource) from hits_100m_single where UTMSource != '' group by intHash32(UserID) % 1000000 -select anyHeavy(UTMSource) from hits_100m_single where UTMSource != '' group by intHash32(UserID) % 1000000 -select min(UTMMedium) from hits_100m_single where UTMMedium != '' group by intHash32(UserID) % 1000000 -select max(UTMMedium) from hits_100m_single where UTMMedium != '' group by intHash32(UserID) % 1000000 -select any(UTMMedium) from hits_100m_single where UTMMedium != '' group by intHash32(UserID) % 1000000 -select anyHeavy(UTMMedium) from hits_100m_single where UTMMedium != '' group by intHash32(UserID) % 1000000 -select min(UTMCampaign) from hits_100m_single where UTMCampaign != '' group by intHash32(UserID) % 1000000 -select max(UTMCampaign) from hits_100m_single where UTMCampaign != '' group by intHash32(UserID) % 1000000 -select any(UTMCampaign) from hits_100m_single where UTMCampaign != '' group by intHash32(UserID) % 1000000 -select anyHeavy(UTMCampaign) from hits_100m_single where UTMCampaign != '' group by intHash32(UserID) % 1000000 -select min(UTMContent) from hits_100m_single where UTMContent != '' group by intHash32(UserID) % 1000000 -select max(UTMContent) from hits_100m_single where UTMContent != '' group by intHash32(UserID) % 1000000 -select any(UTMContent) from hits_100m_single where UTMContent != '' group by intHash32(UserID) % 1000000 -select anyHeavy(UTMContent) from hits_100m_single where UTMContent != '' group by intHash32(UserID) % 1000000 -select min(UTMTerm) from hits_100m_single where UTMTerm != '' group by intHash32(UserID) % 1000000 -select max(UTMTerm) from hits_100m_single where UTMTerm != '' group by intHash32(UserID) % 1000000 -select any(UTMTerm) from hits_100m_single where UTMTerm != '' group by intHash32(UserID) % 1000000 -select anyHeavy(UTMTerm) from hits_100m_single where UTMTerm != '' group by intHash32(UserID) % 1000000 -select min(FromTag) from hits_100m_single where FromTag != '' group by intHash32(UserID) % 1000000 -select max(FromTag) from hits_100m_single where FromTag != '' group by intHash32(UserID) % 1000000 -select any(FromTag) from hits_100m_single where FromTag != '' group by intHash32(UserID) % 1000000 -select anyHeavy(FromTag) from hits_100m_single where FromTag != '' group by intHash32(UserID) % 1000000 +select min(Title) from test.hits where Title != '' group by intHash32(UserID) % 1000000 +select max(Title) from test.hits where Title != '' group by intHash32(UserID) % 1000000 +select any(Title) from test.hits where Title != '' group by intHash32(UserID) % 1000000 +select anyHeavy(Title) from test.hits where Title != '' group by intHash32(UserID) % 1000000 +select min(URL) from test.hits where URL != '' group by intHash32(UserID) % 1000000 +select max(URL) from test.hits where URL != '' group by intHash32(UserID) % 1000000 +select any(URL) from test.hits where URL != '' group by intHash32(UserID) % 1000000 +select anyHeavy(URL) from test.hits where URL != '' group by intHash32(UserID) % 1000000 +select min(Referer) from test.hits where Referer != '' group by intHash32(UserID) % 1000000 +select max(Referer) from test.hits where Referer != '' group by intHash32(UserID) % 1000000 +select any(Referer) from test.hits where Referer != '' group by intHash32(UserID) % 1000000 +select anyHeavy(Referer) from test.hits where Referer != '' group by intHash32(UserID) % 1000000 +select min(FlashMinor2) from test.hits where FlashMinor2 != '' group by intHash32(UserID) % 1000000 +select max(FlashMinor2) from test.hits where FlashMinor2 != '' group by intHash32(UserID) % 1000000 +select any(FlashMinor2) from test.hits where FlashMinor2 != '' group by intHash32(UserID) % 1000000 +select anyHeavy(FlashMinor2) from test.hits where FlashMinor2 != '' group by intHash32(UserID) % 1000000 +select min(MobilePhoneModel) from test.hits where MobilePhoneModel != '' group by intHash32(UserID) % 1000000 +select max(MobilePhoneModel) from test.hits where MobilePhoneModel != '' group by intHash32(UserID) % 1000000 +select any(MobilePhoneModel) from test.hits where MobilePhoneModel != '' group by intHash32(UserID) % 1000000 +select anyHeavy(MobilePhoneModel) from test.hits where MobilePhoneModel != '' group by intHash32(UserID) % 1000000 +select min(Params) from test.hits where Params != '' group by intHash32(UserID) % 1000000 +select max(Params) from test.hits where Params != '' group by intHash32(UserID) % 1000000 +select any(Params) from test.hits where Params != '' group by intHash32(UserID) % 1000000 +select anyHeavy(Params) from test.hits where Params != '' group by intHash32(UserID) % 1000000 +select min(SearchPhrase) from test.hits where SearchPhrase != '' group by intHash32(UserID) % 1000000 +select max(SearchPhrase) from test.hits where SearchPhrase != '' group by intHash32(UserID) % 1000000 +select any(SearchPhrase) from test.hits where SearchPhrase != '' group by intHash32(UserID) % 1000000 +select anyHeavy(SearchPhrase) from test.hits where SearchPhrase != '' group by intHash32(UserID) % 1000000 +select min(PageCharset) from test.hits where PageCharset != '' group by intHash32(UserID) % 1000000 +select max(PageCharset) from test.hits where PageCharset != '' group by intHash32(UserID) % 1000000 +select any(PageCharset) from test.hits where PageCharset != '' group by intHash32(UserID) % 1000000 +select anyHeavy(PageCharset) from test.hits where PageCharset != '' group by intHash32(UserID) % 1000000 +select min(OriginalURL) from test.hits where OriginalURL != '' group by intHash32(UserID) % 1000000 +select max(OriginalURL) from test.hits where OriginalURL != '' group by intHash32(UserID) % 1000000 +select any(OriginalURL) from test.hits where OriginalURL != '' group by intHash32(UserID) % 1000000 +select anyHeavy(OriginalURL) from test.hits where OriginalURL != '' group by intHash32(UserID) % 1000000 +select min(SocialNetwork) from test.hits where SocialNetwork != '' group by intHash32(UserID) % 1000000 +select max(SocialNetwork) from test.hits where SocialNetwork != '' group by intHash32(UserID) % 1000000 +select any(SocialNetwork) from test.hits where SocialNetwork != '' group by intHash32(UserID) % 1000000 +select anyHeavy(SocialNetwork) from test.hits where SocialNetwork != '' group by intHash32(UserID) % 1000000 +select min(SocialAction) from test.hits where SocialAction != '' group by intHash32(UserID) % 1000000 +select max(SocialAction) from test.hits where SocialAction != '' group by intHash32(UserID) % 1000000 +select any(SocialAction) from test.hits where SocialAction != '' group by intHash32(UserID) % 1000000 +select anyHeavy(SocialAction) from test.hits where SocialAction != '' group by intHash32(UserID) % 1000000 +select min(SocialSourcePage) from test.hits where SocialSourcePage != '' group by intHash32(UserID) % 1000000 +select max(SocialSourcePage) from test.hits where SocialSourcePage != '' group by intHash32(UserID) % 1000000 +select any(SocialSourcePage) from test.hits where SocialSourcePage != '' group by intHash32(UserID) % 1000000 +select anyHeavy(SocialSourcePage) from test.hits where SocialSourcePage != '' group by intHash32(UserID) % 1000000 +select min(ParamOrderID) from test.hits where ParamOrderID != '' group by intHash32(UserID) % 1000000 +select max(ParamOrderID) from test.hits where ParamOrderID != '' group by intHash32(UserID) % 1000000 +select any(ParamOrderID) from test.hits where ParamOrderID != '' group by intHash32(UserID) % 1000000 +select anyHeavy(ParamOrderID) from test.hits where ParamOrderID != '' group by intHash32(UserID) % 1000000 +select min(OpenstatServiceName) from test.hits where OpenstatServiceName != '' group by intHash32(UserID) % 1000000 +select max(OpenstatServiceName) from test.hits where OpenstatServiceName != '' group by intHash32(UserID) % 1000000 +select any(OpenstatServiceName) from test.hits where OpenstatServiceName != '' group by intHash32(UserID) % 1000000 +select anyHeavy(OpenstatServiceName) from test.hits where OpenstatServiceName != '' group by intHash32(UserID) % 1000000 +select min(OpenstatCampaignID) from test.hits where OpenstatCampaignID != '' group by intHash32(UserID) % 1000000 +select max(OpenstatCampaignID) from test.hits where OpenstatCampaignID != '' group by intHash32(UserID) % 1000000 +select any(OpenstatCampaignID) from test.hits where OpenstatCampaignID != '' group by intHash32(UserID) % 1000000 +select anyHeavy(OpenstatCampaignID) from test.hits where OpenstatCampaignID != '' group by intHash32(UserID) % 1000000 +select min(OpenstatAdID) from test.hits where OpenstatAdID != '' group by intHash32(UserID) % 1000000 +select max(OpenstatAdID) from test.hits where OpenstatAdID != '' group by intHash32(UserID) % 1000000 +select any(OpenstatAdID) from test.hits where OpenstatAdID != '' group by intHash32(UserID) % 1000000 +select anyHeavy(OpenstatAdID) from test.hits where OpenstatAdID != '' group by intHash32(UserID) % 1000000 +select min(OpenstatSourceID) from test.hits where OpenstatSourceID != '' group by intHash32(UserID) % 1000000 +select max(OpenstatSourceID) from test.hits where OpenstatSourceID != '' group by intHash32(UserID) % 1000000 +select any(OpenstatSourceID) from test.hits where OpenstatSourceID != '' group by intHash32(UserID) % 1000000 +select anyHeavy(OpenstatSourceID) from test.hits where OpenstatSourceID != '' group by intHash32(UserID) % 1000000 +select min(UTMSource) from test.hits where UTMSource != '' group by intHash32(UserID) % 1000000 +select max(UTMSource) from test.hits where UTMSource != '' group by intHash32(UserID) % 1000000 +select any(UTMSource) from test.hits where UTMSource != '' group by intHash32(UserID) % 1000000 +select anyHeavy(UTMSource) from test.hits where UTMSource != '' group by intHash32(UserID) % 1000000 +select min(UTMMedium) from test.hits where UTMMedium != '' group by intHash32(UserID) % 1000000 +select max(UTMMedium) from test.hits where UTMMedium != '' group by intHash32(UserID) % 1000000 +select any(UTMMedium) from test.hits where UTMMedium != '' group by intHash32(UserID) % 1000000 +select anyHeavy(UTMMedium) from test.hits where UTMMedium != '' group by intHash32(UserID) % 1000000 +select min(UTMCampaign) from test.hits where UTMCampaign != '' group by intHash32(UserID) % 1000000 +select max(UTMCampaign) from test.hits where UTMCampaign != '' group by intHash32(UserID) % 1000000 +select any(UTMCampaign) from test.hits where UTMCampaign != '' group by intHash32(UserID) % 1000000 +select anyHeavy(UTMCampaign) from test.hits where UTMCampaign != '' group by intHash32(UserID) % 1000000 +select min(UTMContent) from test.hits where UTMContent != '' group by intHash32(UserID) % 1000000 +select max(UTMContent) from test.hits where UTMContent != '' group by intHash32(UserID) % 1000000 +select any(UTMContent) from test.hits where UTMContent != '' group by intHash32(UserID) % 1000000 +select anyHeavy(UTMContent) from test.hits where UTMContent != '' group by intHash32(UserID) % 1000000 +select min(UTMTerm) from test.hits where UTMTerm != '' group by intHash32(UserID) % 1000000 +select max(UTMTerm) from test.hits where UTMTerm != '' group by intHash32(UserID) % 1000000 +select any(UTMTerm) from test.hits where UTMTerm != '' group by intHash32(UserID) % 1000000 +select anyHeavy(UTMTerm) from test.hits where UTMTerm != '' group by intHash32(UserID) % 1000000 +select min(FromTag) from test.hits where FromTag != '' group by intHash32(UserID) % 1000000 +select max(FromTag) from test.hits where FromTag != '' group by intHash32(UserID) % 1000000 +select any(FromTag) from test.hits where FromTag != '' group by intHash32(UserID) % 1000000 +select anyHeavy(FromTag) from test.hits where FromTag != '' group by intHash32(UserID) % 1000000 From 2fd48696dc45dddf9ef0ab4c72bbf9df93c18d90 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 24 May 2020 12:50:30 +0300 Subject: [PATCH 675/738] Fix build --- base/daemon/BaseDaemon.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index 04f92b91874..10c7173d5b1 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -302,7 +302,7 @@ static void sanitizerDeathCallback() message << " (query_id: " << query_id << ")"; message << " Sanitizer trap."; - LOG_FATAL(log, message.rdbuf()); + LOG_FATAL(log, message.str()); } /// Just in case print our own stack trace. In case when llvm-symbolizer does not work. @@ -314,7 +314,7 @@ static void sanitizerDeathCallback() for (size_t i = stack_trace.getOffset(); i < stack_trace.getSize(); ++i) bare_stacktrace << ' ' << stack_trace.getFrames()[i]; - LOG_FATAL(log, bare_stacktrace.rdbuf()); + LOG_FATAL(log, bare_stacktrace.str()); } /// Write symbolized stack trace line by line for better grep-ability. From e3cd085230fabd38f7113e2083329abf4f4d26a1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 24 May 2020 13:42:13 +0300 Subject: [PATCH 676/738] Fix test (looks like it was incorrect) --- src/IO/S3Common.cpp | 3 ++- src/IO/tests/gtest_s3_uri.cpp | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp index a6b518afcdc..e809474bc98 100644 --- a/src/IO/S3Common.cpp +++ b/src/IO/S3Common.cpp @@ -111,7 +111,7 @@ namespace S3 if (!client_configuration.endpointOverride.empty()) { - static const RE2 region_pattern(R"(s3[.\-]([a-z0-9\-]+)\.amazonaws\.)"); + static const RE2 region_pattern(R"(^s3[.\-]([a-z0-9\-]+)\.amazonaws\.)"); Poco::URI uri(client_configuration.endpointOverride); if (uri.getScheme() == "http") client_configuration.scheme = Aws::Http::Scheme::HTTP; @@ -138,6 +138,7 @@ namespace S3 /// E.g. (https://bucket-name.s3.Region.amazonaws.com/key) /// https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html#virtual-hosted-style-access static const RE2 virtual_hosted_style_pattern(R"((.+)\.(s3[.\-][a-z0-9\-.:]+))"); + /// Case when bucket name and key represented in path of S3 URL. /// E.g. (https://s3.Region.amazonaws.com/bucket-name/key) /// https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html#path-style-access diff --git a/src/IO/tests/gtest_s3_uri.cpp b/src/IO/tests/gtest_s3_uri.cpp index accb81a6306..3609d8c25cd 100644 --- a/src/IO/tests/gtest_s3_uri.cpp +++ b/src/IO/tests/gtest_s3_uri.cpp @@ -17,7 +17,7 @@ TEST(S3UriTest, validPatterns) { { S3::URI uri(Poco::URI("https://jokserfn.s3.yandexcloud.net/data")); - ASSERT_EQ("https://jokserfn.s3.yandexcloud.net", uri.endpoint); + ASSERT_EQ("https://s3.yandexcloud.net", uri.endpoint); ASSERT_EQ("jokserfn", uri.bucket); ASSERT_EQ("data", uri.key); } From 7fca4ce4d02ead3e4d86b93dada5cbe1162f502b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 24 May 2020 13:44:03 +0300 Subject: [PATCH 677/738] Added missing test @excitoon #11036 --- src/IO/tests/gtest_s3_uri.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/IO/tests/gtest_s3_uri.cpp b/src/IO/tests/gtest_s3_uri.cpp index 3609d8c25cd..814caeaffd3 100644 --- a/src/IO/tests/gtest_s3_uri.cpp +++ b/src/IO/tests/gtest_s3_uri.cpp @@ -27,6 +27,12 @@ TEST(S3UriTest, validPatterns) ASSERT_EQ("jokserfn", uri.bucket); ASSERT_EQ("data", uri.key); } + { + S3::URI uri(Poco::URI("https://s3.us-east-2.amazonaws.com/bucketname/data")); + ASSERT_EQ("https://s3.us-east-2.amazonaws.com", uri.endpoint); + ASSERT_EQ("bucketname", uri.bucket); + ASSERT_EQ("data", uri.key); + } } TEST_P(S3UriTest, invalidPatterns) From a8e8bc4298fd36a514f66149622a8e0cd628b9be Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 24 May 2020 13:49:43 +0300 Subject: [PATCH 678/738] Added a comment --- src/Common/RadixSort.h | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/Common/RadixSort.h b/src/Common/RadixSort.h index 59a241e5533..0ad2dcafe14 100644 --- a/src/Common/RadixSort.h +++ b/src/Common/RadixSort.h @@ -17,13 +17,17 @@ /** Radix sort, has the following functionality: + * * Can sort unsigned, signed numbers, and floats. * Can sort an array of fixed length elements that contain something else besides the key. + * Can sort an array and form sorted result containing some transformation of elements. + * Can do partial sort. * Customizable radix size. * - * LSB, stable. - * NOTE For some applications it makes sense to add MSB-radix-sort, - * as well as radix-select, radix-partial-sort, radix-get-permutation algorithms based on it. + * Two flavours of radix sort are implemented: + * + * 1. LSB, stable. + * 2. MSB, unstable, with support for partial sort. */ From 32158d9aeef14ff3dfdaa15b970fa711aae0c1a4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 24 May 2020 17:42:20 +0300 Subject: [PATCH 679/738] Trying to understand and fix the code of RadixSortMSD --- src/Common/RadixSort.h | 267 ++++++++++++++++++++------------ src/Common/tests/radix_sort.cpp | 41 +++-- 2 files changed, 199 insertions(+), 109 deletions(-) diff --git a/src/Common/RadixSort.h b/src/Common/RadixSort.h index 0ad2dcafe14..aa37a79d184 100644 --- a/src/Common/RadixSort.h +++ b/src/Common/RadixSort.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -208,6 +209,10 @@ private: static constexpr size_t KEY_BITS = sizeof(Key) * 8; static constexpr size_t NUM_PASSES = (KEY_BITS + (Traits::PART_SIZE_BITS - 1)) / Traits::PART_SIZE_BITS; + + static KeyBits keyToBits(Key x) { return ext::bit_cast(x); } + static Key bitsToKey(KeyBits x) { return ext::bit_cast(x); } + static ALWAYS_INLINE KeyBits getPart(size_t N, KeyBits x) { if (Traits::Transform::transform_is_simple) @@ -216,10 +221,12 @@ private: return (x >> (N * Traits::PART_SIZE_BITS)) & PART_BITMASK; } - static KeyBits keyToBits(Key x) { return ext::bit_cast(x); } - static Key bitsToKey(KeyBits x) { return ext::bit_cast(x); } + static ALWAYS_INLINE KeyBits extractPart(size_t N, Element & elem) + { + return getPart(N, keyToBits(Traits::extractKey(elem))); + } - static void insertionSortInternal(Element *arr, size_t size) + static void insertionSortInternal(Element * arr, size_t size) { Element * end = arr + size; for (Element * i = arr + 1; i < end; ++i) @@ -236,92 +243,6 @@ private: } } - /* Main MSD radix sort subroutine - * Puts elements to buckets based on PASS-th digit, then recursively calls insertion sort or itself on the buckets - */ - template - static inline void radixSortMSDInternal(Element * arr, size_t size, size_t limit) - { - Element * last_list[HISTOGRAM_SIZE + 1]; - Element ** last = last_list + 1; - size_t count[HISTOGRAM_SIZE] = {0}; - - for (Element * i = arr; i < arr + size; ++i) - ++count[getPart(PASS, *i)]; - - last_list[0] = last_list[1] = arr; - - size_t buckets_for_recursion = HISTOGRAM_SIZE; - Element * finish = arr + size; - for (size_t i = 1; i < HISTOGRAM_SIZE; ++i) - { - last[i] = last[i - 1] + count[i - 1]; - if (last[i] >= arr + limit) - { - buckets_for_recursion = i; - finish = last[i]; - } - } - - /* At this point, we have the following variables: - * count[i] is the size of i-th bucket - * last[i] is a pointer to the beginning of i-th bucket, last[-1] == last[0] - * buckets_for_recursion is the number of buckets that should be sorted, the last of them only partially - * finish is a pointer to the end of the first buckets_for_recursion buckets - */ - - // Scatter array elements to buckets until the first buckets_for_recursion buckets are full - for (size_t i = 0; i < buckets_for_recursion; ++i) - { - Element * end = last[i - 1] + count[i]; - if (end == finish) - { - last[i] = end; - break; - } - while (last[i] != end) - { - Element swapper = *last[i]; - KeyBits tag = getPart(PASS, swapper); - if (tag != i) - { - do - { - std::swap(swapper, *last[tag]++); - } while ((tag = getPart(PASS, swapper)) != i); - *last[i] = swapper; - } - ++last[i]; - } - } - - if constexpr (PASS > 0) - { - // Recursively sort buckets, except the last one - for (size_t i = 0; i < buckets_for_recursion - 1; ++i) - { - Element * start = last[i - 1]; - size_t subsize = last[i] - last[i - 1]; - radixSortMSDInternalHelper(start, subsize, subsize); - } - - // Sort last necessary bucket with limit - Element * start = last[buckets_for_recursion - 2]; - size_t subsize = last[buckets_for_recursion - 1] - last[buckets_for_recursion - 2]; - size_t sublimit = limit - (last[buckets_for_recursion - 1] - arr); - radixSortMSDInternalHelper(start, subsize, sublimit); - } - } - - // A helper to choose sorting algorithm based on array length - template - static inline void radixSortMSDInternalHelper(Element * arr, size_t size, size_t limit) - { - if (size <= INSERTION_SORT_THRESHOLD) - insertionSortInternal(arr, size); - else - radixSortMSDInternal(arr, size, limit); - } template static NO_INLINE void radixSortLSDInternal(Element * arr, size_t size, bool reverse, Result * destination) @@ -346,7 +267,7 @@ private: Traits::extractKey(arr[i]) = bitsToKey(Traits::Transform::forward(keyToBits(Traits::extractKey(arr[i])))); for (size_t pass = 0; pass < NUM_PASSES; ++pass) - ++histograms[pass * HISTOGRAM_SIZE + getPart(pass, keyToBits(Traits::extractKey(arr[i])))]; + ++histograms[pass * HISTOGRAM_SIZE + extractPart(pass, arr[i])]; } { @@ -372,7 +293,7 @@ private: for (size_t i = 0; i < size; ++i) { - size_t pos = getPart(pass, keyToBits(Traits::extractKey(reader[i]))); + size_t pos = extractPart(pass, reader[i]); /// Place the element on the next free position. auto & dest = writer[++histograms[pass * HISTOGRAM_SIZE + pos]]; @@ -394,7 +315,7 @@ private: { for (size_t i = 0; i < size; ++i) { - size_t pos = getPart(pass, keyToBits(Traits::extractKey(reader[i]))); + size_t pos = extractPart(pass, reader[i]); writer[size - 1 - (++histograms[pass * HISTOGRAM_SIZE + pos])] = Traits::extractResult(reader[i]); } } @@ -402,7 +323,7 @@ private: { for (size_t i = 0; i < size; ++i) { - size_t pos = getPart(pass, keyToBits(Traits::extractKey(reader[i]))); + size_t pos = extractPart(pass, reader[i]); writer[++histograms[pass * HISTOGRAM_SIZE + pos]] = Traits::extractResult(reader[i]); } } @@ -413,7 +334,7 @@ private: if (NUM_PASSES % 2) memcpy(arr, swap_buffer, size * sizeof(Element)); - /// This is suboptimal, we can embed it to the last pass. + /// TODO This is suboptimal, we can embed it to the last pass. if (reverse) std::reverse(arr, arr + size); } @@ -421,6 +342,160 @@ private: allocator.deallocate(swap_buffer, size * sizeof(Element)); } + + /* Main MSD radix sort subroutine. + * Puts elements to buckets based on PASS-th digit, then recursively calls insertion sort or itself on the buckets. + * + * TODO: Provide support for 'reverse' and 'DIRECT_WRITE_TO_DESTINATION'. + * + * Invariant: higher significant parts of the elements than PASS are constant within arr or is is the first PASS. + * PASS is counted from least significant (0), so the first pass is NUM_PASSES - 1. + */ + template + static inline void radixSortMSDInternal(Element * arr, size_t size, size_t limit) + { + /// The beginning of every i-1-th bucket. 0th element will be equal to 1st. + /// Last element will point to array end. + Element * prev_buckets[HISTOGRAM_SIZE + 1]; + /// The beginning of every i-th bucket (the same array shifted by one). + Element ** buckets = &prev_buckets[1]; + + prev_buckets[0] = arr; + prev_buckets[1] = arr; + + /// The end of the range of buckets that we need with limit. + Element * finish = arr + size; + + /// Count histogram of current element parts. + + CountType count[HISTOGRAM_SIZE]{}; + for (Element * elem = arr; elem < arr + size; ++elem) + { + /// Array is already sorted by the more significant digit. + assert(PASS == NUM_PASSES - 1 + || elem == arr + || extractPart(PASS + 1, elem[0]) + == extractPart(PASS + 1, elem[-1])); + + ++count[extractPart(PASS, *elem)]; + } + + /// Fill pointers to buckets according to the histogram. + + /// How many buckets we will recurse into. + ssize_t buckets_for_recursion = HISTOGRAM_SIZE; + + for (size_t i = 1; i < HISTOGRAM_SIZE; ++i) + { + /// Positions are just a cumulative sum of counts. + buckets[i] = buckets[i - 1] + count[i - 1]; + + /// If this bucket starts after limit, we don't need it. + if (buckets[i] >= arr + limit) + { + buckets_for_recursion = i; + finish = buckets[i]; + /// We cannot break here, because we need correct pointers to all buckets, see the next loop. + } + } + + /* At this point, we have the following variables: + * count[i] is the size of i-th bucket + * buckets[i] is a pointer to the beginning of i-th bucket, buckets[-1] == buckets[0] + * buckets_for_recursion is the number of buckets that should be sorted, the last of them only partially + * finish is a pointer to the end of the first buckets_for_recursion buckets + */ + + /// Scatter array elements to buckets until the first buckets_for_recursion buckets are full + /// After the above loop, buckets are shifted towards the end and now pointing to the beginning of i+1th bucket. + + for (ssize_t i = 0; /* guarded by 'finish' */; ++i) + { + assert(i < buckets_for_recursion); + + /// We look at i-1th index, because bucket pointers are shifted right on every loop iteration, + /// and all buckets before i was completely shifted to the beginning of the next bucket. + /// So, the beginning of i-th bucket is at buckets[i - 1]. + + Element * bucket_end = buckets[i - 1] + count[i]; + if (bucket_end == finish) + { + buckets[i] = bucket_end; + break; + } + + /// Fill this bucket. + while (buckets[i] != bucket_end) + { + Element swapper = *buckets[i]; + KeyBits tag = extractPart(PASS, swapper); + + if (tag != KeyBits(i)) + { + /// Invariant: tag > i, because the elements with less tags are already at the right places. + assert(tag > KeyBits(i)); + + /// While the tag (digit) of the element is not that we need, + /// swap the element with the next element in the bucket for that tag. + + /// Interesting observation: + /// - we will definitely find the needed element, + /// because the tag's bucket will contain at least one "wrong" element, + /// because the "right" element is appeared in our bucket. + + /// After this loop we shift buckets[i] and buckets[tag] pointers to the right for all found tags. + /// And all positions that were traversed are filled with the proper values. + + do + { + std::swap(swapper, *buckets[tag]); + ++buckets[tag]; + tag = extractPart(PASS, swapper); + } while (tag != KeyBits(i)); + *buckets[i] = swapper; + } + + /// Now we have the right element at this place. + ++buckets[i]; + } + } + + /// Recursion for the relevant buckets. + + if constexpr (PASS > 0) + { + /// Recursively sort buckets, except the last one + for (ssize_t i = 0; i < buckets_for_recursion - 1; ++i) + { + Element * start = buckets[i - 1]; + ssize_t subsize = count[i]; + + radixSortMSDInternalHelper(start, subsize, subsize); + } + + /// Sort last necessary bucket with limit + { + ssize_t i = buckets_for_recursion - 1; + + Element * start = buckets[i - 1]; + ssize_t subsize = count[i]; + ssize_t sublimit = limit - (buckets[i] - arr); + + radixSortMSDInternalHelper(start, subsize, sublimit); + } + } + } + + // A helper to choose sorting algorithm based on array length + template + static inline void radixSortMSDInternalHelper(Element * arr, size_t size, size_t limit) + { + if (size <= INSERTION_SORT_THRESHOLD) + insertionSortInternal(arr, size); + else + radixSortMSDInternal(arr, size, limit); + } + public: /** Least significant digit radix sort (stable). * This function will sort inplace (modify 'arr') @@ -480,13 +555,13 @@ public: /// Use RadixSort with custom traits for complex types instead. template -void radixSortLSD(T *arr, size_t size) +void radixSortLSD(T * arr, size_t size) { RadixSort>::executeLSD(arr, size); } template -void radixSortMSD(T *arr, size_t size, size_t limit) +void radixSortMSD(T * arr, size_t size, size_t limit) { RadixSort>::executeMSD(arr, size, limit); } diff --git a/src/Common/tests/radix_sort.cpp b/src/Common/tests/radix_sort.cpp index a7313d05cec..a55d4594693 100644 --- a/src/Common/tests/radix_sort.cpp +++ b/src/Common/tests/radix_sort.cpp @@ -1,14 +1,19 @@ -#if !defined(__APPLE__) && !defined(__FreeBSD__) -#include -#endif +#include #include #include + +#if defined(NDEBUG) +#undef NDEBUG #include +#endif + #include +#include #include #include -using Key = double; + +using Key = UInt32; static void NO_INLINE sort1(Key * data, size_t size) { @@ -29,10 +34,15 @@ static void NO_INLINE sort3(Key * data, size_t size) }); } +static void NO_INLINE sort4(Key * data, size_t size) +{ + radixSortMSD(data, size, size); +} + int main(int argc, char ** argv) { - pcg64 rng; + pcg64 rng(randomSeed()); if (argc < 3) { @@ -43,9 +53,9 @@ int main(int argc, char ** argv) size_t n = DB::parse(argv[1]); size_t method = DB::parse(argv[2]); - std::vector data(n); + std::cerr << std::fixed << std::setprecision(3); -// srand(time(nullptr)); + std::vector data(n); { Stopwatch watch; @@ -70,13 +80,13 @@ int main(int argc, char ** argv) std::cerr << std::endl; } - { Stopwatch watch; - if (method == 1) sort1(data.data(), n); - if (method == 2) sort2(data.data(), n); - if (method == 3) sort3(data.data(), n); + if (method == 1) sort1(data.data(), n); + if (method == 2) sort2(data.data(), n); + if (method == 3) sort3(data.data(), n); + if (method == 4) sort4(data.data(), n); watch.stop(); double elapsed = watch.elapsedSeconds(); @@ -87,6 +97,8 @@ int main(int argc, char ** argv) << std::endl; } + bool ok = true; + { Stopwatch watch; @@ -94,7 +106,10 @@ int main(int argc, char ** argv) while (i < n) { if (!(data[i - 1] <= data[i])) + { + ok = false; break; + } ++i; } @@ -105,10 +120,10 @@ int main(int argc, char ** argv) << " (" << n / elapsed << " elem/sec., " << n * sizeof(Key) / elapsed / 1048576 << " MB/sec.)" << std::endl - << "Result: " << (i == n ? "Ok." : "Fail!") << std::endl; + << "Result: " << (ok ? "Ok." : "Fail!") << std::endl; } - if (n <= 1000) + if (!ok && n <= 100000) { std::cerr << std::endl; From 5e7ea6500ca409ffb2a154b97f2ed52e89dd01ce Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 24 May 2020 17:48:53 +0300 Subject: [PATCH 680/738] Fix error --- src/Common/RadixSort.h | 4 +++- src/Common/tests/radix_sort.cpp | 8 ++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Common/RadixSort.h b/src/Common/RadixSort.h index aa37a79d184..dcf96319508 100644 --- a/src/Common/RadixSort.h +++ b/src/Common/RadixSort.h @@ -354,6 +354,8 @@ private: template static inline void radixSortMSDInternal(Element * arr, size_t size, size_t limit) { + // std::cerr << PASS << ", " << size << ", " << limit << "\n"; + /// The beginning of every i-1-th bucket. 0th element will be equal to 1st. /// Last element will point to array end. Element * prev_buckets[HISTOGRAM_SIZE + 1]; @@ -479,7 +481,7 @@ private: Element * start = buckets[i - 1]; ssize_t subsize = count[i]; - ssize_t sublimit = limit - (buckets[i] - arr); + ssize_t sublimit = limit - (start - arr); radixSortMSDInternalHelper(start, subsize, sublimit); } diff --git a/src/Common/tests/radix_sort.cpp b/src/Common/tests/radix_sort.cpp index a55d4594693..73f8e12caeb 100644 --- a/src/Common/tests/radix_sort.cpp +++ b/src/Common/tests/radix_sort.cpp @@ -2,10 +2,10 @@ #include #include -#if defined(NDEBUG) -#undef NDEBUG +//#if defined(NDEBUG) +//#undef NDEBUG #include -#endif +//#endif #include #include @@ -13,7 +13,7 @@ #include -using Key = UInt32; +using Key = double; static void NO_INLINE sort1(Key * data, size_t size) { From 4748de9c85f2cc374ecb38d09c6949bb92b21ee6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 24 May 2020 18:26:07 +0300 Subject: [PATCH 681/738] Fix partial radix sort: it was not working at all --- src/Common/RadixSort.h | 17 +++++----- src/Common/tests/CMakeLists.txt | 1 + src/Common/tests/radix_sort.cpp | 58 ++++++++++++++++++++++++++------- 3 files changed, 56 insertions(+), 20 deletions(-) diff --git a/src/Common/RadixSort.h b/src/Common/RadixSort.h index dcf96319508..02ede67ea93 100644 --- a/src/Common/RadixSort.h +++ b/src/Common/RadixSort.h @@ -386,6 +386,7 @@ private: /// How many buckets we will recurse into. ssize_t buckets_for_recursion = HISTOGRAM_SIZE; + bool finish_early = false; for (size_t i = 1; i < HISTOGRAM_SIZE; ++i) { @@ -393,10 +394,11 @@ private: buckets[i] = buckets[i - 1] + count[i - 1]; /// If this bucket starts after limit, we don't need it. - if (buckets[i] >= arr + limit) + if (!finish_early && buckets[i] >= arr + limit) { buckets_for_recursion = i; finish = buckets[i]; + finish_early = true; /// We cannot break here, because we need correct pointers to all buckets, see the next loop. } } @@ -420,11 +422,6 @@ private: /// So, the beginning of i-th bucket is at buckets[i - 1]. Element * bucket_end = buckets[i - 1] + count[i]; - if (bucket_end == finish) - { - buckets[i] = bucket_end; - break; - } /// Fill this bucket. while (buckets[i] != bucket_end) @@ -460,6 +457,9 @@ private: /// Now we have the right element at this place. ++buckets[i]; } + + if (bucket_end == finish) + break; } /// Recursion for the relevant buckets. @@ -475,7 +475,7 @@ private: radixSortMSDInternalHelper(start, subsize, subsize); } - /// Sort last necessary bucket with limit + /// Sort the last necessary bucket with limit { ssize_t i = buckets_for_recursion - 1; @@ -519,7 +519,8 @@ public: } /* Most significant digit radix sort - * Usually slower than LSD and is not stable, but allows partial sorting + * Is not stable, but allows partial sorting. + * And it's more cache-friendly than LSD variant. * * Based on https://github.com/voutcn/kxsort, license: * The MIT License diff --git a/src/Common/tests/CMakeLists.txt b/src/Common/tests/CMakeLists.txt index 72c47d1ef49..b68e71c0b43 100644 --- a/src/Common/tests/CMakeLists.txt +++ b/src/Common/tests/CMakeLists.txt @@ -34,6 +34,7 @@ target_link_libraries (compact_array PRIVATE clickhouse_common_io) add_executable (radix_sort radix_sort.cpp) target_link_libraries (radix_sort PRIVATE clickhouse_common_io) +target_include_directories(radix_sort SYSTEM PRIVATE ${PDQSORT_INCLUDE_DIR}) if (USE_OPENCL) add_executable (bitonic_sort bitonic_sort.cpp) diff --git a/src/Common/tests/radix_sort.cpp b/src/Common/tests/radix_sort.cpp index 73f8e12caeb..0ed64d75170 100644 --- a/src/Common/tests/radix_sort.cpp +++ b/src/Common/tests/radix_sort.cpp @@ -11,9 +11,10 @@ #include #include #include +#include -using Key = double; +using Key = UInt64; static void NO_INLINE sort1(Key * data, size_t size) { @@ -29,8 +30,8 @@ static void NO_INLINE sort3(Key * data, size_t size) { std::sort(data, data + size, [](Key a, Key b) { - return RadixSortFloatTransform::forward(ext::bit_cast(a)) - < RadixSortFloatTransform::forward(ext::bit_cast(b)); + return RadixSortFloatTransform::forward(ext::bit_cast(a)) + < RadixSortFloatTransform::forward(ext::bit_cast(b)); }); } @@ -39,19 +40,48 @@ static void NO_INLINE sort4(Key * data, size_t size) radixSortMSD(data, size, size); } +static void NO_INLINE sort5(Key * data, size_t size) +{ + pdqsort(data, data + size); +} + + +static void NO_INLINE sort6(Key * data, size_t size, size_t limit) +{ + std::partial_sort(data, data + limit, data + size); +} + +static void NO_INLINE sort7(Key * data, size_t size, size_t limit) +{ + std::partial_sort(data, data + limit, data + size, [](Key a, Key b) + { + return RadixSortFloatTransform::forward(ext::bit_cast(a)) + < RadixSortFloatTransform::forward(ext::bit_cast(b)); + }); +} + +static void NO_INLINE sort8(Key * data, size_t size, size_t limit) +{ + radixSortMSD(data, size, limit); +} + int main(int argc, char ** argv) { pcg64 rng(randomSeed()); - if (argc < 3) + if (argc < 3 || argc > 4) { - std::cerr << "Usage: program n method\n"; + std::cerr << "Usage: program method n [limit]\n"; return 1; } - size_t n = DB::parse(argv[1]); - size_t method = DB::parse(argv[2]); + size_t method = DB::parse(argv[1]); + size_t n = DB::parse(argv[2]); + size_t limit = n; + + if (argc == 4) + limit = DB::parse(argv[3]); std::cerr << std::fixed << std::setprecision(3); @@ -87,6 +117,10 @@ int main(int argc, char ** argv) if (method == 2) sort2(data.data(), n); if (method == 3) sort3(data.data(), n); if (method == 4) sort4(data.data(), n); + if (method == 5) sort5(data.data(), n); + if (method == 6) sort6(data.data(), n, limit); + if (method == 7) sort7(data.data(), n, limit); + if (method == 8) sort8(data.data(), n, limit); watch.stop(); double elapsed = watch.elapsedSeconds(); @@ -103,7 +137,7 @@ int main(int argc, char ** argv) Stopwatch watch; size_t i = 1; - while (i < n) + while (i < limit) { if (!(data[i - 1] <= data[i])) { @@ -117,18 +151,18 @@ int main(int argc, char ** argv) double elapsed = watch.elapsedSeconds(); std::cerr << "Checked in " << elapsed - << " (" << n / elapsed << " elem/sec., " - << n * sizeof(Key) / elapsed / 1048576 << " MB/sec.)" + << " (" << limit / elapsed << " elem/sec., " + << limit * sizeof(Key) / elapsed / 1048576 << " MB/sec.)" << std::endl << "Result: " << (ok ? "Ok." : "Fail!") << std::endl; } - if (!ok && n <= 100000) + if (!ok && limit <= 100000) { std::cerr << std::endl; std::cerr << data[0] << ' '; - for (size_t i = 1; i < n; ++i) + for (size_t i = 1; i < limit; ++i) { if (!(data[i - 1] <= data[i])) std::cerr << "*** "; From 04022f71b4c829940c1696fd9e1d79b5d375ec27 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 24 May 2020 18:35:48 +0300 Subject: [PATCH 682/738] Added comment --- src/Common/RadixSort.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Common/RadixSort.h b/src/Common/RadixSort.h index 02ede67ea93..65a7c380ebd 100644 --- a/src/Common/RadixSort.h +++ b/src/Common/RadixSort.h @@ -522,6 +522,8 @@ public: * Is not stable, but allows partial sorting. * And it's more cache-friendly than LSD variant. * + * NOTE: It's beneficial over std::partial_sort only if limit is above ~2% of size. + * * Based on https://github.com/voutcn/kxsort, license: * The MIT License * Copyright (c) 2016 Dinghua Li From e99ed195861e906fbb42c4284c8f245972a7764f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 24 May 2020 18:48:11 +0300 Subject: [PATCH 683/738] Unrolled histogram calculation --- src/Common/RadixSort.h | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/Common/RadixSort.h b/src/Common/RadixSort.h index 65a7c380ebd..3a0490d67ea 100644 --- a/src/Common/RadixSort.h +++ b/src/Common/RadixSort.h @@ -370,17 +370,20 @@ private: /// Count histogram of current element parts. - CountType count[HISTOGRAM_SIZE]{}; - for (Element * elem = arr; elem < arr + size; ++elem) - { - /// Array is already sorted by the more significant digit. - assert(PASS == NUM_PASSES - 1 - || elem == arr - || extractPart(PASS + 1, elem[0]) - == extractPart(PASS + 1, elem[-1])); + static constexpr size_t UNROLL_COUNT = 4; + CountType count[HISTOGRAM_SIZE * UNROLL_COUNT]{}; + size_t unrolled_size = size / UNROLL_COUNT * UNROLL_COUNT; + for (Element * elem = arr; elem < arr + unrolled_size; elem += UNROLL_COUNT) + for (size_t i = 0; i < UNROLL_COUNT; ++i) + ++count[i * HISTOGRAM_SIZE + extractPart(PASS, elem[i])]; + + for (Element * elem = arr + unrolled_size; elem < arr + size; ++elem) ++count[extractPart(PASS, *elem)]; - } + + for (size_t i = 0; i < HISTOGRAM_SIZE; ++i) + for (size_t j = 1; j < UNROLL_COUNT; ++j) + count[i] += count[j * HISTOGRAM_SIZE + i]; /// Fill pointers to buckets according to the histogram. From d0c0a0aed62144b7415514ccca250cd62b882df9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 24 May 2020 18:51:23 +0300 Subject: [PATCH 684/738] Unrolled histogram calculation 8 times --- src/Common/RadixSort.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Common/RadixSort.h b/src/Common/RadixSort.h index 3a0490d67ea..f6f487c63eb 100644 --- a/src/Common/RadixSort.h +++ b/src/Common/RadixSort.h @@ -354,7 +354,7 @@ private: template static inline void radixSortMSDInternal(Element * arr, size_t size, size_t limit) { - // std::cerr << PASS << ", " << size << ", " << limit << "\n"; + //std::cerr << PASS << ", " << size << ", " << limit << "\n"; /// The beginning of every i-1-th bucket. 0th element will be equal to 1st. /// Last element will point to array end. @@ -370,7 +370,11 @@ private: /// Count histogram of current element parts. - static constexpr size_t UNROLL_COUNT = 4; + /// We use loop unrolling to minimize data dependencies and increase instruction level parallelism. + /// Unroll 8 times looks better on experiments; + /// also it corresponds with the results from https://github.com/powturbo/TurboHist + + static constexpr size_t UNROLL_COUNT = 8; CountType count[HISTOGRAM_SIZE * UNROLL_COUNT]{}; size_t unrolled_size = size / UNROLL_COUNT * UNROLL_COUNT; From 01be2dadb3f0a6963dec62d7a16eebb6f7e5284f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 24 May 2020 21:05:06 +0300 Subject: [PATCH 685/738] Various comments --- src/Common/RadixSort.h | 10 ++- src/Common/tests/radix_sort.cpp | 124 +++++++++++++++++++++++++++++--- 2 files changed, 123 insertions(+), 11 deletions(-) diff --git a/src/Common/RadixSort.h b/src/Common/RadixSort.h index f6f487c63eb..6612fa89085 100644 --- a/src/Common/RadixSort.h +++ b/src/Common/RadixSort.h @@ -354,7 +354,7 @@ private: template static inline void radixSortMSDInternal(Element * arr, size_t size, size_t limit) { - //std::cerr << PASS << ", " << size << ", " << limit << "\n"; +// std::cerr << PASS << ", " << size << ", " << limit << "\n"; /// The beginning of every i-1-th bucket. 0th element will be equal to 1st. /// Last element will point to array end. @@ -527,9 +527,13 @@ public: /* Most significant digit radix sort * Is not stable, but allows partial sorting. - * And it's more cache-friendly than LSD variant. + * And it's more cache-friendly and usually faster than LSD variant. * - * NOTE: It's beneficial over std::partial_sort only if limit is above ~2% of size. + * NOTE: It's beneficial over std::partial_sort only if limit is above ~2% of size for 8 bit radix. + * NOTE: When lowering down limit to 1%, the radix of 4..6 or 10..12 bit started to become beneficial. + * For less than 1% limit, it's not recommended to use. + * NOTE: For huge arrays without limit, the radix 11 suddenly becomes better... but not for smaller arrays. + * Maybe it because histogram will fit in half of L1d cache (2048 * 4 = 16384). * * Based on https://github.com/voutcn/kxsort, license: * The MIT License diff --git a/src/Common/tests/radix_sort.cpp b/src/Common/tests/radix_sort.cpp index 0ed64d75170..9288e32e734 100644 --- a/src/Common/tests/radix_sort.cpp +++ b/src/Common/tests/radix_sort.cpp @@ -13,6 +13,9 @@ #include #include +/// Example: +/// for i in {6,8} {11..26}; do echo $i; for j in {1..10}; do ./radix_sort $i 65536 1000; done; echo; done + using Key = UInt64; @@ -66,6 +69,93 @@ static void NO_INLINE sort8(Key * data, size_t size, size_t limit) } +template +struct RadixSortTraitsWithCustomBits : RadixSortNumTraits +{ + static constexpr size_t PART_SIZE_BITS = N; +}; + +static void NO_INLINE sort11(Key * data, size_t size, size_t limit) +{ + RadixSort>::executeMSD(data, size, limit); +} + +static void NO_INLINE sort12(Key * data, size_t size, size_t limit) +{ + RadixSort>::executeMSD(data, size, limit); +} + +static void NO_INLINE sort13(Key * data, size_t size, size_t limit) +{ + RadixSort>::executeMSD(data, size, limit); +} + +static void NO_INLINE sort14(Key * data, size_t size, size_t limit) +{ + RadixSort>::executeMSD(data, size, limit); +} + +static void NO_INLINE sort15(Key * data, size_t size, size_t limit) +{ + RadixSort>::executeMSD(data, size, limit); +} + +static void NO_INLINE sort16(Key * data, size_t size, size_t limit) +{ + RadixSort>::executeMSD(data, size, limit); +} + +static void NO_INLINE sort17(Key * data, size_t size, size_t limit) +{ + RadixSort>::executeMSD(data, size, limit); +} + +static void NO_INLINE sort18(Key * data, size_t size, size_t limit) +{ + RadixSort>::executeMSD(data, size, limit); +} + +static void NO_INLINE sort19(Key * data, size_t size, size_t limit) +{ + RadixSort>::executeMSD(data, size, limit); +} + +static void NO_INLINE sort20(Key * data, size_t size, size_t limit) +{ + RadixSort>::executeMSD(data, size, limit); +} + +static void NO_INLINE sort21(Key * data, size_t size, size_t limit) +{ + RadixSort>::executeMSD(data, size, limit); +} + +static void NO_INLINE sort22(Key * data, size_t size, size_t limit) +{ + RadixSort>::executeMSD(data, size, limit); +} + +static void NO_INLINE sort23(Key * data, size_t size, size_t limit) +{ + RadixSort>::executeMSD(data, size, limit); +} + +static void NO_INLINE sort24(Key * data, size_t size, size_t limit) +{ + RadixSort>::executeMSD(data, size, limit); +} + +static void NO_INLINE sort25(Key * data, size_t size, size_t limit) +{ + RadixSort>::executeMSD(data, size, limit); +} + +static void NO_INLINE sort26(Key * data, size_t size, size_t limit) +{ + RadixSort>::executeMSD(data, size, limit); +} + + int main(int argc, char ** argv) { pcg64 rng(randomSeed()); @@ -94,12 +184,12 @@ int main(int argc, char ** argv) elem = rng(); watch.stop(); - double elapsed = watch.elapsedSeconds(); + /* double elapsed = watch.elapsedSeconds(); std::cerr << "Filled in " << elapsed << " (" << n / elapsed << " elem/sec., " << n * sizeof(Key) / elapsed / 1048576 << " MB/sec.)" - << std::endl; + << std::endl;*/ } if (n <= 100) @@ -122,6 +212,23 @@ int main(int argc, char ** argv) if (method == 7) sort7(data.data(), n, limit); if (method == 8) sort8(data.data(), n, limit); + if (method == 11) sort11(data.data(), n, limit); + if (method == 12) sort12(data.data(), n, limit); + if (method == 13) sort13(data.data(), n, limit); + if (method == 14) sort14(data.data(), n, limit); + if (method == 15) sort15(data.data(), n, limit); + if (method == 16) sort16(data.data(), n, limit); + if (method == 17) sort17(data.data(), n, limit); + if (method == 18) sort18(data.data(), n, limit); + if (method == 19) sort19(data.data(), n, limit); + if (method == 20) sort20(data.data(), n, limit); + if (method == 21) sort21(data.data(), n, limit); + if (method == 22) sort22(data.data(), n, limit); + if (method == 23) sort23(data.data(), n, limit); + if (method == 24) sort24(data.data(), n, limit); + if (method == 25) sort25(data.data(), n, limit); + if (method == 26) sort26(data.data(), n, limit); + watch.stop(); double elapsed = watch.elapsedSeconds(); std::cerr @@ -149,12 +256,13 @@ int main(int argc, char ** argv) watch.stop(); double elapsed = watch.elapsedSeconds(); - std::cerr - << "Checked in " << elapsed - << " (" << limit / elapsed << " elem/sec., " - << limit * sizeof(Key) / elapsed / 1048576 << " MB/sec.)" - << std::endl - << "Result: " << (ok ? "Ok." : "Fail!") << std::endl; + if (!ok) + std::cerr + << "Checked in " << elapsed + << " (" << limit / elapsed << " elem/sec., " + << limit * sizeof(Key) / elapsed / 1048576 << " MB/sec.)" + << std::endl + << "Result: " << (ok ? "Ok." : "Fail!") << std::endl; } if (!ok && limit <= 100000) From 9082a70ee6bb82d17fdd8e13d353f52c2ce2750d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 24 May 2020 21:25:58 +0300 Subject: [PATCH 686/738] Fix clang-tidy --- src/Client/ConnectionPoolWithFailover.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Client/ConnectionPoolWithFailover.cpp b/src/Client/ConnectionPoolWithFailover.cpp index 0bfebd15ef7..9933cc6b555 100644 --- a/src/Client/ConnectionPoolWithFailover.cpp +++ b/src/Client/ConnectionPoolWithFailover.cpp @@ -222,7 +222,7 @@ ConnectionPoolWithFailover::tryGetEntry( auto table_status_it = status_response.table_states_by_id.find(*table_to_check); if (table_status_it == status_response.table_states_by_id.end()) { - auto message_pattern = "There is no table {}.{} on server: {}"; + const char * message_pattern = "There is no table {}.{} on server: {}"; fail_message = fmt::format(message_pattern, backQuote(table_to_check->database), backQuote(table_to_check->table), result.entry->getDescription()); LOG_WARNING(log, fail_message); ProfileEvents::increment(ProfileEvents::DistributedConnectionMissingTable); From 4ebf63fb234177ff2b34a10683210761760198ed Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 24 May 2020 22:17:15 +0300 Subject: [PATCH 687/738] Fixed error --- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index b90782192a4..618f8bb75a8 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -2896,7 +2896,7 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Strin if (part_to_clone) LOG_DEBUG(log, "Cloned part {} from {}{}", part_name, part_to_clone->name, to_detached ? " (to 'detached' directory)" : ""); else - LOG_DEBUG(log, "Fetched part {} from {}{}", part_name, part_to_clone->name, to_detached ? " (to 'detached' directory)" : ""); + LOG_DEBUG(log, "Fetched part {} from {}{}", part_name, source_replica_path, to_detached ? " (to 'detached' directory)" : ""); return true; } From 02500e4b11ddab119d683314013b557a40e1c68b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 24 May 2020 23:33:13 +0300 Subject: [PATCH 688/738] Fix long tests --- .../performance/agg_functions_min_max_any.xml | 189 +++++++++--------- 1 file changed, 92 insertions(+), 97 deletions(-) diff --git a/tests/performance/agg_functions_min_max_any.xml b/tests/performance/agg_functions_min_max_any.xml index 43bb8c823ac..4591182cdfc 100644 --- a/tests/performance/agg_functions_min_max_any.xml +++ b/tests/performance/agg_functions_min_max_any.xml @@ -1,104 +1,99 @@ - - - test.hits - -select min(Title) from test.hits where Title != '' group by intHash32(UserID) % 1000000 -select max(Title) from test.hits where Title != '' group by intHash32(UserID) % 1000000 -select any(Title) from test.hits where Title != '' group by intHash32(UserID) % 1000000 -select anyHeavy(Title) from test.hits where Title != '' group by intHash32(UserID) % 1000000 -select min(URL) from test.hits where URL != '' group by intHash32(UserID) % 1000000 -select max(URL) from test.hits where URL != '' group by intHash32(UserID) % 1000000 -select any(URL) from test.hits where URL != '' group by intHash32(UserID) % 1000000 -select anyHeavy(URL) from test.hits where URL != '' group by intHash32(UserID) % 1000000 -select min(Referer) from test.hits where Referer != '' group by intHash32(UserID) % 1000000 -select max(Referer) from test.hits where Referer != '' group by intHash32(UserID) % 1000000 -select any(Referer) from test.hits where Referer != '' group by intHash32(UserID) % 1000000 -select anyHeavy(Referer) from test.hits where Referer != '' group by intHash32(UserID) % 1000000 -select min(FlashMinor2) from test.hits where FlashMinor2 != '' group by intHash32(UserID) % 1000000 -select max(FlashMinor2) from test.hits where FlashMinor2 != '' group by intHash32(UserID) % 1000000 -select any(FlashMinor2) from test.hits where FlashMinor2 != '' group by intHash32(UserID) % 1000000 -select anyHeavy(FlashMinor2) from test.hits where FlashMinor2 != '' group by intHash32(UserID) % 1000000 -select min(MobilePhoneModel) from test.hits where MobilePhoneModel != '' group by intHash32(UserID) % 1000000 -select max(MobilePhoneModel) from test.hits where MobilePhoneModel != '' group by intHash32(UserID) % 1000000 -select any(MobilePhoneModel) from test.hits where MobilePhoneModel != '' group by intHash32(UserID) % 1000000 -select anyHeavy(MobilePhoneModel) from test.hits where MobilePhoneModel != '' group by intHash32(UserID) % 1000000 -select min(Params) from test.hits where Params != '' group by intHash32(UserID) % 1000000 -select max(Params) from test.hits where Params != '' group by intHash32(UserID) % 1000000 -select any(Params) from test.hits where Params != '' group by intHash32(UserID) % 1000000 -select anyHeavy(Params) from test.hits where Params != '' group by intHash32(UserID) % 1000000 -select min(SearchPhrase) from test.hits where SearchPhrase != '' group by intHash32(UserID) % 1000000 -select max(SearchPhrase) from test.hits where SearchPhrase != '' group by intHash32(UserID) % 1000000 -select any(SearchPhrase) from test.hits where SearchPhrase != '' group by intHash32(UserID) % 1000000 -select anyHeavy(SearchPhrase) from test.hits where SearchPhrase != '' group by intHash32(UserID) % 1000000 -select min(PageCharset) from test.hits where PageCharset != '' group by intHash32(UserID) % 1000000 -select max(PageCharset) from test.hits where PageCharset != '' group by intHash32(UserID) % 1000000 -select any(PageCharset) from test.hits where PageCharset != '' group by intHash32(UserID) % 1000000 -select anyHeavy(PageCharset) from test.hits where PageCharset != '' group by intHash32(UserID) % 1000000 -select min(OriginalURL) from test.hits where OriginalURL != '' group by intHash32(UserID) % 1000000 -select max(OriginalURL) from test.hits where OriginalURL != '' group by intHash32(UserID) % 1000000 -select any(OriginalURL) from test.hits where OriginalURL != '' group by intHash32(UserID) % 1000000 -select anyHeavy(OriginalURL) from test.hits where OriginalURL != '' group by intHash32(UserID) % 1000000 -select min(SocialNetwork) from test.hits where SocialNetwork != '' group by intHash32(UserID) % 1000000 -select max(SocialNetwork) from test.hits where SocialNetwork != '' group by intHash32(UserID) % 1000000 -select any(SocialNetwork) from test.hits where SocialNetwork != '' group by intHash32(UserID) % 1000000 -select anyHeavy(SocialNetwork) from test.hits where SocialNetwork != '' group by intHash32(UserID) % 1000000 -select min(SocialAction) from test.hits where SocialAction != '' group by intHash32(UserID) % 1000000 -select max(SocialAction) from test.hits where SocialAction != '' group by intHash32(UserID) % 1000000 -select any(SocialAction) from test.hits where SocialAction != '' group by intHash32(UserID) % 1000000 -select anyHeavy(SocialAction) from test.hits where SocialAction != '' group by intHash32(UserID) % 1000000 -select min(SocialSourcePage) from test.hits where SocialSourcePage != '' group by intHash32(UserID) % 1000000 -select max(SocialSourcePage) from test.hits where SocialSourcePage != '' group by intHash32(UserID) % 1000000 -select any(SocialSourcePage) from test.hits where SocialSourcePage != '' group by intHash32(UserID) % 1000000 -select anyHeavy(SocialSourcePage) from test.hits where SocialSourcePage != '' group by intHash32(UserID) % 1000000 -select min(ParamOrderID) from test.hits where ParamOrderID != '' group by intHash32(UserID) % 1000000 -select max(ParamOrderID) from test.hits where ParamOrderID != '' group by intHash32(UserID) % 1000000 -select any(ParamOrderID) from test.hits where ParamOrderID != '' group by intHash32(UserID) % 1000000 -select anyHeavy(ParamOrderID) from test.hits where ParamOrderID != '' group by intHash32(UserID) % 1000000 -select min(OpenstatServiceName) from test.hits where OpenstatServiceName != '' group by intHash32(UserID) % 1000000 -select max(OpenstatServiceName) from test.hits where OpenstatServiceName != '' group by intHash32(UserID) % 1000000 -select any(OpenstatServiceName) from test.hits where OpenstatServiceName != '' group by intHash32(UserID) % 1000000 -select anyHeavy(OpenstatServiceName) from test.hits where OpenstatServiceName != '' group by intHash32(UserID) % 1000000 -select min(OpenstatCampaignID) from test.hits where OpenstatCampaignID != '' group by intHash32(UserID) % 1000000 -select max(OpenstatCampaignID) from test.hits where OpenstatCampaignID != '' group by intHash32(UserID) % 1000000 -select any(OpenstatCampaignID) from test.hits where OpenstatCampaignID != '' group by intHash32(UserID) % 1000000 -select anyHeavy(OpenstatCampaignID) from test.hits where OpenstatCampaignID != '' group by intHash32(UserID) % 1000000 -select min(OpenstatAdID) from test.hits where OpenstatAdID != '' group by intHash32(UserID) % 1000000 -select max(OpenstatAdID) from test.hits where OpenstatAdID != '' group by intHash32(UserID) % 1000000 -select any(OpenstatAdID) from test.hits where OpenstatAdID != '' group by intHash32(UserID) % 1000000 -select anyHeavy(OpenstatAdID) from test.hits where OpenstatAdID != '' group by intHash32(UserID) % 1000000 -select min(OpenstatSourceID) from test.hits where OpenstatSourceID != '' group by intHash32(UserID) % 1000000 -select max(OpenstatSourceID) from test.hits where OpenstatSourceID != '' group by intHash32(UserID) % 1000000 -select any(OpenstatSourceID) from test.hits where OpenstatSourceID != '' group by intHash32(UserID) % 1000000 -select anyHeavy(OpenstatSourceID) from test.hits where OpenstatSourceID != '' group by intHash32(UserID) % 1000000 -select min(UTMSource) from test.hits where UTMSource != '' group by intHash32(UserID) % 1000000 -select max(UTMSource) from test.hits where UTMSource != '' group by intHash32(UserID) % 1000000 -select any(UTMSource) from test.hits where UTMSource != '' group by intHash32(UserID) % 1000000 -select anyHeavy(UTMSource) from test.hits where UTMSource != '' group by intHash32(UserID) % 1000000 -select min(UTMMedium) from test.hits where UTMMedium != '' group by intHash32(UserID) % 1000000 -select max(UTMMedium) from test.hits where UTMMedium != '' group by intHash32(UserID) % 1000000 -select any(UTMMedium) from test.hits where UTMMedium != '' group by intHash32(UserID) % 1000000 -select anyHeavy(UTMMedium) from test.hits where UTMMedium != '' group by intHash32(UserID) % 1000000 -select min(UTMCampaign) from test.hits where UTMCampaign != '' group by intHash32(UserID) % 1000000 -select max(UTMCampaign) from test.hits where UTMCampaign != '' group by intHash32(UserID) % 1000000 -select any(UTMCampaign) from test.hits where UTMCampaign != '' group by intHash32(UserID) % 1000000 -select anyHeavy(UTMCampaign) from test.hits where UTMCampaign != '' group by intHash32(UserID) % 1000000 -select min(UTMContent) from test.hits where UTMContent != '' group by intHash32(UserID) % 1000000 -select max(UTMContent) from test.hits where UTMContent != '' group by intHash32(UserID) % 1000000 -select any(UTMContent) from test.hits where UTMContent != '' group by intHash32(UserID) % 1000000 -select anyHeavy(UTMContent) from test.hits where UTMContent != '' group by intHash32(UserID) % 1000000 -select min(UTMTerm) from test.hits where UTMTerm != '' group by intHash32(UserID) % 1000000 -select max(UTMTerm) from test.hits where UTMTerm != '' group by intHash32(UserID) % 1000000 -select any(UTMTerm) from test.hits where UTMTerm != '' group by intHash32(UserID) % 1000000 -select anyHeavy(UTMTerm) from test.hits where UTMTerm != '' group by intHash32(UserID) % 1000000 -select min(FromTag) from test.hits where FromTag != '' group by intHash32(UserID) % 1000000 -select max(FromTag) from test.hits where FromTag != '' group by intHash32(UserID) % 1000000 -select any(FromTag) from test.hits where FromTag != '' group by intHash32(UserID) % 1000000 -select anyHeavy(FromTag) from test.hits where FromTag != '' group by intHash32(UserID) % 1000000 - +select min(Title) from test.hits where Title != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(Title) from test.hits where Title != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(Title) from test.hits where Title != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(Title) from test.hits where Title != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(URL) from test.hits where URL != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(URL) from test.hits where URL != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(URL) from test.hits where URL != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(URL) from test.hits where URL != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(Referer) from test.hits where Referer != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(Referer) from test.hits where Referer != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(Referer) from test.hits where Referer != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(Referer) from test.hits where Referer != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(FlashMinor2) from test.hits where FlashMinor2 != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(FlashMinor2) from test.hits where FlashMinor2 != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(FlashMinor2) from test.hits where FlashMinor2 != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(FlashMinor2) from test.hits where FlashMinor2 != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(MobilePhoneModel) from test.hits where MobilePhoneModel != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(MobilePhoneModel) from test.hits where MobilePhoneModel != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(MobilePhoneModel) from test.hits where MobilePhoneModel != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(MobilePhoneModel) from test.hits where MobilePhoneModel != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(Params) from test.hits where Params != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(Params) from test.hits where Params != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(Params) from test.hits where Params != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(Params) from test.hits where Params != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(SearchPhrase) from test.hits where SearchPhrase != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(SearchPhrase) from test.hits where SearchPhrase != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(SearchPhrase) from test.hits where SearchPhrase != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(SearchPhrase) from test.hits where SearchPhrase != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(PageCharset) from test.hits where PageCharset != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(PageCharset) from test.hits where PageCharset != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(PageCharset) from test.hits where PageCharset != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(PageCharset) from test.hits where PageCharset != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(OriginalURL) from test.hits where OriginalURL != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(OriginalURL) from test.hits where OriginalURL != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(OriginalURL) from test.hits where OriginalURL != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(OriginalURL) from test.hits where OriginalURL != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(SocialNetwork) from test.hits where SocialNetwork != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(SocialNetwork) from test.hits where SocialNetwork != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(SocialNetwork) from test.hits where SocialNetwork != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(SocialNetwork) from test.hits where SocialNetwork != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(SocialAction) from test.hits where SocialAction != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(SocialAction) from test.hits where SocialAction != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(SocialAction) from test.hits where SocialAction != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(SocialAction) from test.hits where SocialAction != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(SocialSourcePage) from test.hits where SocialSourcePage != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(SocialSourcePage) from test.hits where SocialSourcePage != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(SocialSourcePage) from test.hits where SocialSourcePage != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(SocialSourcePage) from test.hits where SocialSourcePage != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(ParamOrderID) from test.hits where ParamOrderID != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(ParamOrderID) from test.hits where ParamOrderID != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(ParamOrderID) from test.hits where ParamOrderID != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(ParamOrderID) from test.hits where ParamOrderID != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(OpenstatServiceName) from test.hits where OpenstatServiceName != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(OpenstatServiceName) from test.hits where OpenstatServiceName != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(OpenstatServiceName) from test.hits where OpenstatServiceName != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(OpenstatServiceName) from test.hits where OpenstatServiceName != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(OpenstatCampaignID) from test.hits where OpenstatCampaignID != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(OpenstatCampaignID) from test.hits where OpenstatCampaignID != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(OpenstatCampaignID) from test.hits where OpenstatCampaignID != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(OpenstatCampaignID) from test.hits where OpenstatCampaignID != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(OpenstatAdID) from test.hits where OpenstatAdID != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(OpenstatAdID) from test.hits where OpenstatAdID != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(OpenstatAdID) from test.hits where OpenstatAdID != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(OpenstatAdID) from test.hits where OpenstatAdID != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(OpenstatSourceID) from test.hits where OpenstatSourceID != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(OpenstatSourceID) from test.hits where OpenstatSourceID != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(OpenstatSourceID) from test.hits where OpenstatSourceID != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(OpenstatSourceID) from test.hits where OpenstatSourceID != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(UTMSource) from test.hits where UTMSource != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(UTMSource) from test.hits where UTMSource != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(UTMSource) from test.hits where UTMSource != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(UTMSource) from test.hits where UTMSource != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(UTMMedium) from test.hits where UTMMedium != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(UTMMedium) from test.hits where UTMMedium != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(UTMMedium) from test.hits where UTMMedium != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(UTMMedium) from test.hits where UTMMedium != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(UTMCampaign) from test.hits where UTMCampaign != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(UTMCampaign) from test.hits where UTMCampaign != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(UTMCampaign) from test.hits where UTMCampaign != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(UTMCampaign) from test.hits where UTMCampaign != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(UTMContent) from test.hits where UTMContent != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(UTMContent) from test.hits where UTMContent != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(UTMContent) from test.hits where UTMContent != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(UTMContent) from test.hits where UTMContent != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(UTMTerm) from test.hits where UTMTerm != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(UTMTerm) from test.hits where UTMTerm != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(UTMTerm) from test.hits where UTMTerm != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(UTMTerm) from test.hits where UTMTerm != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(FromTag) from test.hits where FromTag != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(FromTag) from test.hits where FromTag != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(FromTag) from test.hits where FromTag != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(FromTag) from test.hits where FromTag != '' group by intHash32(UserID) % 1000000 FORMAT Null From 8fa637cf73d5f7ae5a8eef184c67fd80461e2249 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 24 May 2020 23:35:49 +0300 Subject: [PATCH 689/738] Fix long tests, part 2 --- tests/performance/columns_hashing.xml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/performance/columns_hashing.xml b/tests/performance/columns_hashing.xml index 186d3a8ba62..ac3d4b1b33b 100644 --- a/tests/performance/columns_hashing.xml +++ b/tests/performance/columns_hashing.xml @@ -4,12 +4,12 @@ - hits_100m_single + test.hits - - - - - + + + + + From 75dbfd465261c279790ce7134145a3e4c38c94ee Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 25 May 2020 01:36:00 +0300 Subject: [PATCH 690/738] Fix flacky test --- ...uted_group_by_no_merge_GROUP_BY_injective_sharding_key.sql | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/queries/0_stateless/01247_distributed_group_by_no_merge_GROUP_BY_injective_sharding_key.sql b/tests/queries/0_stateless/01247_distributed_group_by_no_merge_GROUP_BY_injective_sharding_key.sql index bca3e4830d2..d4753e95f79 100644 --- a/tests/queries/0_stateless/01247_distributed_group_by_no_merge_GROUP_BY_injective_sharding_key.sql +++ b/tests/queries/0_stateless/01247_distributed_group_by_no_merge_GROUP_BY_injective_sharding_key.sql @@ -1,5 +1,9 @@ -- TODO: correct testing with real unique shards +-- Avoid "Connection failed at try №1" messages. +SET send_logs_level = 'none'; +SET connect_timeout_with_failover_ms = 5000; + set optimize_distributed_group_by_sharding_key=1; drop table if exists dist_01247; From 5a9f47e6b01965cbf2af40c53f8fe5f16f53fed0 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 25 May 2020 04:03:21 +0300 Subject: [PATCH 691/738] performance comparison --- docker/test/performance-comparison/compare.sh | 62 +++++++++---------- docker/test/performance-comparison/perf.py | 11 ++-- 2 files changed, 37 insertions(+), 36 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 22c9907e60f..0b5962065d6 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -155,9 +155,6 @@ function run_tests TIMEFORMAT=$(printf "$test_name\t%%3R\t%%3U\t%%3S\n") # the grep is to filter out set -x output and keep only time output { time "$script_dir/perf.py" --host localhost localhost --port 9001 9002 -- "$test" > "$test_name-raw.tsv" 2> "$test_name-err.log" ; } 2>&1 >/dev/null | grep -v ^+ >> "wall-clock-times.tsv" || continue - - # The test completed with zero status, so we treat stderr as warnings - mv "$test_name-err.log" "$test_name-warn.log" done unset TIMEFORMAT @@ -253,7 +250,7 @@ done unset IFS # for each query run, prepare array of metrics from query log -clickhouse-local --query " +clickhouse-local --verbose --query " create view query_runs as select * from file('analyze/query-runs.tsv', TSV, 'test text, query_index int, query_id text, version UInt8, time float'); @@ -277,9 +274,9 @@ create table query_metrics engine File(TSV, -- do not add header -- will parse w , query_runs.time ] metrics from ( - select *, 0 version from left_query_log + select query_duration_ms, memory_usage, query_id, 0 version from left_query_log union all - select *, 1 version from right_query_log + select query_duration_ms, memory_usage, query_id, 1 version from right_query_log ) query_logs right join query_runs using (query_id, version) @@ -335,7 +332,8 @@ create view query_display_names as select * from create table query_metric_stats engine File(TSVWithNamesAndTypes, 'report/query-metric-stats.tsv') as - select *, metric_name + select metric_name, left, right, diff, stat_threshold, test, query_index, + query_display_name from file ('analyze/query-reports.tsv', TSV, 'left Array(float), right Array(float), diff Array(float), stat_threshold Array(float), test text, query_index int') reports @@ -479,11 +477,12 @@ create table all_query_metrics_tsv engine File(TSV, 'report/all-query-metrics.ts order by test, query_index; " 2> >(tee -a report/errors.log 1>&2) + # Prepare source data for metrics and flamegraphs for unstable queries. for version in {right,left} - do +do rm -rf data - clickhouse-local --query " + clickhouse-local --verbose --query " create view queries_for_flamegraph as select * from file('report/queries-for-flamegraph.tsv', TSVWithNamesAndTypes, 'test text, query_index int'); @@ -544,7 +543,13 @@ create view trace_log as select * from file('$version-trace-log.tsv', TSVWithNamesAndTypes, '$(cat "$version-trace-log.tsv.columns")'); -create view addresses_src as select * +create view addresses_src as select addr, + -- Some functions change name between builds, e.g. '__clone' or 'clone' or + -- even '__GI__clone@@GLIBC_2.32'. This breaks differential flame graphs, so + -- filter them out here. + [name, 'clone.S (filtered by script)', 'pthread_cond_timedwait (filtered by script)'] + -- this line is a subscript operator of the above array + [1 + multiSearchFirstIndex(name, ['clone.S', 'pthread_cond_timedwait'])] name from file('$version-addresses.tsv', TSVWithNamesAndTypes, '$(cat "$version-addresses.tsv.columns")'); @@ -556,7 +561,8 @@ create table unstable_run_traces engine File(TSVWithNamesAndTypes, select test, query_index, query_id, count() value, - joinGet(addresses_join_$version, 'name', arrayJoin(trace)) metric + joinGet(addresses_join_$version, 'name', arrayJoin(trace)) + || '(' || toString(trace_type) || ')' metric from trace_log join unstable_query_runs using query_id group by test, query_index, query_id, metric @@ -564,7 +570,7 @@ create table unstable_run_traces engine File(TSVWithNamesAndTypes, ; create table metric_devation engine File(TSVWithNamesAndTypes, - 'metric-deviation.$version.rep') as + 'report/metric-deviation.$version.tsv') as -- first goes the key used to split the file with grep select test, query_index, query_display_name, d, q, metric @@ -583,20 +589,14 @@ create table metric_devation engine File(TSVWithNamesAndTypes, order by test, query_index, d desc ; -create table stacks engine File(TSV, 'stacks.$version.rep') as +create table stacks engine File(TSV, 'report/stacks.$version.tsv') as select -- first goes the key used to split the file with grep - test, query_index, any(query_display_name), + test, query_index, trace_type, any(query_display_name), -- next go the stacks in flamegraph format: 'func1;...;funcN count' arrayStringConcat( - arrayMap(addr -> replaceRegexpOne( - joinGet(addresses_join_$version, 'name', addr), - -- This function is at the base of the stack, and its name changes - -- surprisingly often between builds, e.g. '__clone' or 'clone' or - -- even '__GI__clone'. This breaks differential flame graphs, so - -- filter it out here. - '^clone\\.S.*', 'clone.S (name filtered by comparison script)' - ), + arrayMap( + addr -> joinGet(addresses_join_$version, 'name', addr), arrayReverse(trace) ), ';' @@ -604,8 +604,8 @@ create table stacks engine File(TSV, 'stacks.$version.rep') as count() c from trace_log join unstable_query_runs using query_id - group by test, query_index, trace - order by test, query_index, trace + group by test, query_index, trace_type, trace + order by test, query_index, trace_type, trace ; " 2> >(tee -a report/errors.log 1>&2) # do not run in parallel because they use the same data dir for StorageJoins which leads to weird errors. done @@ -615,7 +615,7 @@ wait IFS=$'\n' for version in {right,left} do - for query in $(cut -d' ' -f1,2,3 "stacks.$version.rep" | sort | uniq) + for query in $(cut -d' ' -f1-4 "report/stacks.$version.tsv" | sort | uniq) do query_file=$(echo "$query" | cut -c-120 | sed 's/[/ ]/_/g') echo "$query_file" >> report/query-files.txt @@ -623,15 +623,15 @@ do # Build separate .svg flamegraph for each query. # -F is somewhat unsafe because it might match not the beginning of the # string, but this is unlikely and escaping the query for grep is a pain. - grep -F "$query " "stacks.$version.rep" \ - | cut -f 4- \ + grep -F "$query " "report/stacks.$version.tsv" \ + | cut -f 5- \ | sed 's/\t/ /g' \ - | tee "$query_file.stacks.$version.rep" \ + | tee "report/$query_file.stacks.$version.tsv" \ | ~/fg/flamegraph.pl --hash > "$query_file.$version.svg" & # Copy metric stats into separate files as well. # Ditto the above comment about -F. - grep -F "$query " "metric-deviation.$version.rep" \ + grep -F "$query " "report/metric-deviation.$version.tsv" \ | cut -f4- > "$query_file.$version.metrics.rep" & done done @@ -642,8 +642,8 @@ unset IFS IFS=$'\n' for query_file in $(cat report/query-files.txt) do - ~/fg/difffolded.pl "$query_file.stacks.left.rep" "$query_file.stacks.right.rep" \ - | tee "$query_file.stacks.diff.rep" \ + ~/fg/difffolded.pl "report/$query_file.stacks.left.tsv" "report/$query_file.stacks.right.tsv" \ + | tee "report/$query_file.stacks.diff.tsv" \ | ~/fg/flamegraph.pl > "$query_file.diff.svg" & done unset IFS diff --git a/docker/test/performance-comparison/perf.py b/docker/test/performance-comparison/perf.py index fd16673bd89..ac6a5e03296 100755 --- a/docker/test/performance-comparison/perf.py +++ b/docker/test/performance-comparison/perf.py @@ -11,6 +11,9 @@ import string import time import traceback +def tsv_escape(s): + return s.replace('\\', '\\\\').replace('\t', '\\t').replace('\n', '\\n').replace('\r','') + stage_start_seconds = time.perf_counter() def report_stage_end(stage_name): @@ -112,8 +115,9 @@ for t in tables: try: res = c.execute("select 1 from {} limit 1".format(t)) except: - print('skipped\t' + traceback.format_exception_only(*sys.exc_info()[:2])[-1]) - traceback.print_exc() + exception_message = traceback.format_exception_only(*sys.exc_info()[:2])[-1] + skipped_message = ' '.join(exception_message.split('\n')[:2]) + print(f'skipped\t{tsv_escape(skipped_message)}') sys.exit(0) report_stage_end('preconditions') @@ -135,9 +139,6 @@ for c in connections: report_stage_end('fill') # Run test queries -def tsv_escape(s): - return s.replace('\\', '\\\\').replace('\t', '\\t').replace('\n', '\\n').replace('\r','') - test_query_templates = [q.text for q in root.findall('query')] test_queries = substitute_parameters(test_query_templates) From 3b888e4e333b220acc4fd051946f129f7c414e4e Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 25 May 2020 04:03:30 +0300 Subject: [PATCH 692/738] Add rand32 alias --- docs/en/sql-reference/functions/random-functions.md | 2 +- src/Functions/rand.cpp | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/random-functions.md b/docs/en/sql-reference/functions/random-functions.md index 5d480664ca1..788ece6371b 100644 --- a/docs/en/sql-reference/functions/random-functions.md +++ b/docs/en/sql-reference/functions/random-functions.md @@ -11,7 +11,7 @@ All the functions accept zero arguments or one argument. If an argument is passed, it can be any type, and its value is not used for anything. The only purpose of this argument is to prevent common subexpression elimination, so that two different instances of the same function return different columns with different random numbers. -## rand {#rand} +## rand, rand32 {#rand} Returns a pseudo-random UInt32 number, evenly distributed among all UInt32-type numbers. Uses a linear congruential generator. diff --git a/src/Functions/rand.cpp b/src/Functions/rand.cpp index 333396c1ecd..e704b7824b5 100644 --- a/src/Functions/rand.cpp +++ b/src/Functions/rand.cpp @@ -10,6 +10,7 @@ using FunctionRand = FunctionRandom; void registerFunctionRand(FunctionFactory & factory) { factory.registerFunction(FunctionFactory::CaseInsensitive); + factory.registerAlias("rand32", NameRand::name); } } From bf3f76be7fe67e3fac4236160dfe2c93a0dafb6e Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 25 May 2020 04:42:56 +0300 Subject: [PATCH 693/738] performance comparison --- docker/test/performance-comparison/compare.sh | 44 ++++++++++++------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 0b5962065d6..bf4599acb9a 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -231,8 +231,8 @@ done function analyze_queries { rm -v analyze-commands.txt analyze-errors.log all-queries.tsv unstable-queries.tsv ./*-report.tsv raw-queries.tsv ||: -rm -rfv analyze ||: -mkdir analyze ||: +rm -rf analyze ||: +mkdir analyze analyze/tmp ||: build_log_column_definitions @@ -250,7 +250,7 @@ done unset IFS # for each query run, prepare array of metrics from query log -clickhouse-local --verbose --query " +clickhouse-local --query " create view query_runs as select * from file('analyze/query-runs.tsv', TSV, 'test text, query_index int, query_id text, version UInt8, time float'); @@ -294,7 +294,7 @@ query_index=1 IFS=$'\n' for prefix in $(cut -f1,2 "analyze/query-run-metrics.tsv" | sort | uniq) do - file="analyze/$(echo "$prefix" | sed 's/\t/_/g').tmp" + file="analyze/tmp/$(echo "$prefix" | sed 's/\t/_/g').tsv" grep "^$prefix " "analyze/query-run-metrics.tsv" > "$file" & printf "%s\0\n" \ "clickhouse-local \ @@ -315,7 +315,7 @@ parallel --joblog analyze/parallel-log.txt --null < analyze/commands.txt 2>> ana function report { rm -r report ||: -mkdir report ||: +mkdir report report/tmp ||: rm ./*.{rep,svg} test-times.tsv test-dump.tsv unstable.tsv unstable-query-ids.tsv unstable-query-metrics.tsv changed-perf.tsv unstable-tests.tsv unstable-queries.tsv bad-tests.tsv slow-on-client.tsv all-queries.tsv ||: @@ -482,7 +482,7 @@ create table all_query_metrics_tsv engine File(TSV, 'report/all-query-metrics.ts for version in {right,left} do rm -rf data - clickhouse-local --verbose --query " + clickhouse-local --query " create view queries_for_flamegraph as select * from file('report/queries-for-flamegraph.tsv', TSVWithNamesAndTypes, 'test text, query_index int'); @@ -585,7 +585,7 @@ create table metric_devation engine File(TSVWithNamesAndTypes, group by test, query_index, metric having d > 0.5 ) metrics - left join unstable_query_runs using (test, query_index) + left join query_display_names using (test, query_index) order by test, query_index, d desc ; @@ -611,7 +611,7 @@ create table stacks engine File(TSV, 'report/stacks.$version.tsv') as done wait -# Create per-query flamegraphs and files with metrics +# Create per-query flamegraphs IFS=$'\n' for version in {right,left} do @@ -626,13 +626,8 @@ do grep -F "$query " "report/stacks.$version.tsv" \ | cut -f 5- \ | sed 's/\t/ /g' \ - | tee "report/$query_file.stacks.$version.tsv" \ + | tee "report/tmp/$query_file.stacks.$version.tsv" \ | ~/fg/flamegraph.pl --hash > "$query_file.$version.svg" & - - # Copy metric stats into separate files as well. - # Ditto the above comment about -F. - grep -F "$query " "report/metric-deviation.$version.tsv" \ - | cut -f4- > "$query_file.$version.metrics.rep" & done done wait @@ -642,13 +637,30 @@ unset IFS IFS=$'\n' for query_file in $(cat report/query-files.txt) do - ~/fg/difffolded.pl "report/$query_file.stacks.left.tsv" "report/$query_file.stacks.right.tsv" \ - | tee "report/$query_file.stacks.diff.tsv" \ + ~/fg/difffolded.pl "report/tmp/$query_file.stacks.left.tsv" \ + "report/tmp/$query_file.stacks.right.tsv" \ + | tee "report/tmp/$query_file.stacks.diff.tsv" \ | ~/fg/flamegraph.pl > "$query_file.diff.svg" & done unset IFS wait +# Create per-query files with metrics. Note that the key is different from flamegraphs. +IFS=$'\n' +for version in {right,left} +do + for query in $(cut -d' ' -f1-3 "report/metric-deviation.$version.tsv" | sort | uniq) + do + query_file=$(echo "$query" | cut -c-120 | sed 's/[/ ]/_/g') + + # Ditto the above comment about -F. + grep -F "$query " "report/metric-deviation.$version.tsv" \ + | cut -f4- > "$query_file.$version.metrics.rep" & + done +done +wait +unset IFS + # Remember that grep sets error code when nothing is found, hence the bayan # operator. grep -H -m2 -i '\(Exception\|Error\):[^:]' ./*-err.log | sed 's/:/\t/' >> run-errors.tsv ||: From 09129a6d2189f51f2dfbd7b1145ff3e49a815adb Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 25 May 2020 06:16:55 +0300 Subject: [PATCH 694/738] Function for printing array of Asts --- docker/test/performance-comparison/perf.py | 2 ++ src/Interpreters/CrossToInnerJoinVisitor.cpp | 15 +++------------ src/Interpreters/JoinedTables.cpp | 2 +- src/Parsers/IAST.h | 18 ++++++++++++++++++ 4 files changed, 24 insertions(+), 13 deletions(-) diff --git a/docker/test/performance-comparison/perf.py b/docker/test/performance-comparison/perf.py index a749b158e83..45ace027a4d 100755 --- a/docker/test/performance-comparison/perf.py +++ b/docker/test/performance-comparison/perf.py @@ -154,6 +154,8 @@ for i, q in enumerate(test_queries): for conn_index, c in enumerate(connections): res = c.execute(q, query_id = f'prewarm {0} {query_display_name}') print(f'prewarm\t{tsv_escape(query_display_name)}\t{conn_index}\t{c.last_query.elapsed}') + except KeyboardInterrupt: + raise except: # If prewarm fails for some query -- skip it, and try to test the others. # This might happen if the new test introduces some function that the diff --git a/src/Interpreters/CrossToInnerJoinVisitor.cpp b/src/Interpreters/CrossToInnerJoinVisitor.cpp index 4f3fc380341..b2f3f56be4d 100644 --- a/src/Interpreters/CrossToInnerJoinVisitor.cpp +++ b/src/Interpreters/CrossToInnerJoinVisitor.cpp @@ -281,18 +281,9 @@ bool getTables(ASTSelectQuery & select, std::vector & joined_tabl if (num_using && (num_tables - num_array_join) > 2) { - std::stringstream ss; - ss << "Multiple CROSS/COMMA JOIN do not support USING (while processing '"; - for (size_t i = 0; i < tables_with_using.size(); ++i) - { - if (i) - { - ss << ", "; - } - tables_with_using[i]->format(IAST::FormatSettings(ss, true /* one line */)); - } - ss << "')"; - throw Exception(ss.str(), ErrorCodes::NOT_IMPLEMENTED); + throw Exception("Multiple CROSS/COMMA JOIN do not support USING (while " + "processing '" + IAST::formatForErrorMessage(tables_with_using) + "')", + ErrorCodes::NOT_IMPLEMENTED); } return !(num_array_join || num_using); diff --git a/src/Interpreters/JoinedTables.cpp b/src/Interpreters/JoinedTables.cpp index 9fc867c7ae2..b348cf6d3e7 100644 --- a/src/Interpreters/JoinedTables.cpp +++ b/src/Interpreters/JoinedTables.cpp @@ -188,7 +188,7 @@ StoragePtr JoinedTables::getLeftTableStorage() bool JoinedTables::resolveTables() { tables_with_columns = getDatabaseAndTablesWithColumns(table_expressions, context); - assert(tables_with_columns.size == table_expressions.size); + assert(tables_with_columns.size() == table_expressions.size()); const auto & settings = context.getSettingsRef(); if (settings.joined_subquery_requires_alias && tables_with_columns.size() > 1) diff --git a/src/Parsers/IAST.h b/src/Parsers/IAST.h index 4eb6662b57c..88dedc54d3f 100644 --- a/src/Parsers/IAST.h +++ b/src/Parsers/IAST.h @@ -9,6 +9,7 @@ #include #include #include +#include class SipHash; @@ -217,6 +218,8 @@ public: // A simple way to add some user-readable context to an error message. std::string formatForErrorMessage() const; + template + static std::string formatForErrorMessage(const AstArray & array); void cloneChildren(); @@ -234,4 +237,19 @@ private: size_t checkDepthImpl(size_t max_depth, size_t level) const; }; +template +std::string IAST::formatForErrorMessage(const AstArray & array) +{ + std::stringstream ss; + for (size_t i = 0; i < array.size(); ++i) + { + if (i > 0) + { + ss << ", "; + } + array[i]->format(IAST::FormatSettings(ss, true /* one line */)); + } + return ss.str(); +} + } From fd48e2ce9db4c9ada7c58494c3208dd607198d0d Mon Sep 17 00:00:00 2001 From: BayoNet Date: Mon, 25 May 2020 08:04:47 +0300 Subject: [PATCH 695/738] DOCS-643: Sources of system metrics. (#11032) * CLICKHOUSEDOCS-643: Source of system metrics * Update docs/en/operations/system-tables.md Co-authored-by: Ivan Blinkov * Update docs/en/operations/system-tables.md Co-authored-by: Ivan Blinkov * Update docs/en/operations/system-tables.md Co-authored-by: Ivan Blinkov * Update docs/en/operations/system-tables.md Co-authored-by: Ivan Blinkov * CLICKHOUSEDOCS-643: Updated by comments. Co-authored-by: Sergei Shtykov Co-authored-by: Ivan Blinkov --- docs/en/operations/system-tables.md | 40 +++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/docs/en/operations/system-tables.md b/docs/en/operations/system-tables.md index a186e4f2cda..cc5456f5324 100644 --- a/docs/en/operations/system-tables.md +++ b/docs/en/operations/system-tables.md @@ -5,11 +5,41 @@ toc_title: System Tables # System Tables {#system-tables} -System tables are used for implementing part of the system’s functionality, and for providing access to information about how the system is working. -You can’t delete a system table (but you can perform DETACH). -System tables don’t have files with data on the disk or files with metadata. The server creates all the system tables when it starts. -System tables are read-only. -They are located in the ‘system’ database. +## Introduction + +System tables provide information about: + +- Server states, processes, and environment. +- Server's internal processes. + +System tables: + +- Located in the `system` database. +- Available only for reading data. +- Can't be dropped or altered, but can be detached. + +The `metric_log`, `query_log`, `query_thread_log`, `trace_log` system tables store data in a storage filesystem. Other system tables store their data in RAM. ClickHouse server creates such system tables at the start. + +### Sources of System Metrics + +For collecting system metrics ClickHouse server uses: + +- `CAP_NET_ADMIN` capability. +- [procfs](https://en.wikipedia.org/wiki/Procfs) (only in Linux). + +**procfs** + +If ClickHouse server doesn't have `CAP_NET_ADMIN` capability, it tries to fall back to `ProcfsMetricsProvider`. `ProcfsMetricsProvider` allows collecting per-query system metrics (for CPU and I/O). + +If procfs is supported and enabled on the system, ClickHouse server collects these metrics: + +- `OSCPUVirtualTimeMicroseconds` +- `OSCPUWaitMicroseconds` +- `OSIOWaitMicroseconds` +- `OSReadChars` +- `OSWriteChars` +- `OSReadBytes` +- `OSWriteBytes` ## system.asynchronous\_metrics {#system_tables-asynchronous_metrics} From aac48e0274e69ed01d45a6f2c2abb0249ad22a4d Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Fri, 22 May 2020 23:42:24 +0300 Subject: [PATCH 696/738] Update grpc to a new version. --- contrib/grpc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/grpc b/contrib/grpc index c1d176528fd..8aea4e168e7 160000 --- a/contrib/grpc +++ b/contrib/grpc @@ -1 +1 @@ -Subproject commit c1d176528fd8da9dd4066d16554bcd216d29033f +Subproject commit 8aea4e168e78f3eb9828080740fc8cb73d53bf79 From 0b0c5d3a207ba7a016ba687eb8c2a7649fbd3cf2 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Mon, 25 May 2020 09:52:39 +0300 Subject: [PATCH 697/738] [website] minor style fixes --- website/templates/common_meta.html | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/website/templates/common_meta.html b/website/templates/common_meta.html index 09bef26d575..8b65e67d780 100644 --- a/website/templates/common_meta.html +++ b/website/templates/common_meta.html @@ -1,25 +1,25 @@ {% set description = description or _('ClickHouse is a fast open-source column-oriented database management system that allows generating analytical data reports in real-time using SQL queries') %} - + {% if title %}{{ title }}{% else %}{{ _('ClickHouse - fast open-source OLAP DBMS') }}{% endif %} - + - + {% if page and not single_page %} - + {% endif %} {% include "templates/docs/ld_json.html" %} - + From 09a0c0d38bab3394e3b1890c649c780c624dbfd9 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Mon, 25 May 2020 10:03:12 +0300 Subject: [PATCH 698/738] [docs] fix title and headers @ developer-instruction.md --- docs/en/development/developer-instruction.md | 32 +++++++++----------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/docs/en/development/developer-instruction.md b/docs/en/development/developer-instruction.md index abd93e83116..3a6774037c1 100644 --- a/docs/en/development/developer-instruction.md +++ b/docs/en/development/developer-instruction.md @@ -1,19 +1,17 @@ --- toc_priority: 61 -toc_title: The Beginner ClickHouse Developer Instruction +toc_title: For Beginners --- +# The Beginner ClickHouse Developer Instruction + Building of ClickHouse is supported on Linux, FreeBSD and Mac OS X. -# If You Use Windows {#if-you-use-windows} - If you use Windows, you need to create a virtual machine with Ubuntu. To start working with a virtual machine please install VirtualBox. You can download Ubuntu from the website: https://www.ubuntu.com/#download. Please create a virtual machine from the downloaded image (you should reserve at least 4GB of RAM for it). To run a command-line terminal in Ubuntu, please locate a program containing the word “terminal” in its name (gnome-terminal, konsole etc.) or just press Ctrl+Alt+T. -# If You Use a 32-bit System {#if-you-use-a-32-bit-system} - ClickHouse cannot work or build on a 32-bit system. You should acquire access to a 64-bit system and you can continue reading. -# Creating a Repository on GitHub {#creating-a-repository-on-github} +## Creating a Repository on GitHub {#creating-a-repository-on-github} To start working with ClickHouse repository you will need a GitHub account. @@ -33,7 +31,7 @@ To do that in Ubuntu you would run in the command line terminal: A brief manual on using Git can be found here: https://services.github.com/on-demand/downloads/github-git-cheat-sheet.pdf. For a detailed manual on Git see https://git-scm.com/book/en/v2. -# Cloning a Repository to Your Development Machine {#cloning-a-repository-to-your-development-machine} +## Cloning a Repository to Your Development Machine {#cloning-a-repository-to-your-development-machine} Next, you need to download the source files onto your working machine. This is called “to clone a repository” because it creates a local copy of the repository on your working machine. @@ -77,7 +75,7 @@ You can also add original ClickHouse repo’s address to your local repository t After successfully running this command you will be able to pull updates from the main ClickHouse repo by running `git pull upstream master`. -## Working with Submodules {#working-with-submodules} +### Working with Submodules {#working-with-submodules} Working with submodules in git could be painful. Next commands will help to manage it: @@ -107,7 +105,7 @@ The next commands would help you to reset all submodules to the initial state (! git submodule foreach git submodule foreach git reset --hard git submodule foreach git submodule foreach git clean -xfd -# Build System {#build-system} +## Build System {#build-system} ClickHouse uses CMake and Ninja for building. @@ -127,11 +125,11 @@ For installing CMake and Ninja on Mac OS X first install Homebrew and then insta Next, check the version of CMake: `cmake --version`. If it is below 3.3, you should install a newer version from the website: https://cmake.org/download/. -# Optional External Libraries {#optional-external-libraries} +## Optional External Libraries {#optional-external-libraries} ClickHouse uses several external libraries for building. All of them do not need to be installed separately as they are built together with ClickHouse from the sources located in the submodules. You can check the list in `contrib`. -# C++ Compiler {#c-compiler} +## C++ Compiler {#c-compiler} Compilers GCC starting from version 9 and Clang version 8 or above are supported for building ClickHouse. @@ -145,7 +143,7 @@ Mac OS X build is supported only for Clang. Just run `brew install llvm` If you decide to use Clang, you can also install `libc++` and `lld`, if you know what it is. Using `ccache` is also recommended. -# The Building Process {#the-building-process} +## The Building Process {#the-building-process} Now that you are ready to build ClickHouse we recommend you to create a separate directory `build` inside `ClickHouse` that will contain all of the build artefacts: @@ -202,7 +200,7 @@ Upon successful build you get an executable file `ClickHouse//program ls -l programs/clickhouse -# Running the Built Executable of ClickHouse {#running-the-built-executable-of-clickhouse} +## Running the Built Executable of ClickHouse {#running-the-built-executable-of-clickhouse} To run the server under the current user you need to navigate to `ClickHouse/programs/server/` (located outside of `build`) and run: @@ -229,7 +227,7 @@ You can also run your custom-built ClickHouse binary with the config file from t sudo service clickhouse-server stop sudo -u clickhouse ClickHouse/build/programs/clickhouse server --config-file /etc/clickhouse-server/config.xml -# IDE (Integrated Development Environment) {#ide-integrated-development-environment} +## IDE (Integrated Development Environment) {#ide-integrated-development-environment} If you do not know which IDE to use, we recommend that you use CLion. CLion is commercial software, but it offers 30 days free trial period. It is also free of charge for students. CLion can be used both on Linux and on Mac OS X. @@ -239,7 +237,7 @@ As simple code editors, you can use Sublime Text or Visual Studio Code, or Kate Just in case, it is worth mentioning that CLion creates `build` path on its own, it also on its own selects `debug` for build type, for configuration it uses a version of CMake that is defined in CLion and not the one installed by you, and finally, CLion will use `make` to run build tasks instead of `ninja`. This is normal behaviour, just keep that in mind to avoid confusion. -# Writing Code {#writing-code} +## Writing Code {#writing-code} The description of ClickHouse architecture can be found here: https://clickhouse.tech/docs/en/development/architecture/ @@ -249,7 +247,7 @@ Writing tests: https://clickhouse.tech/docs/en/development/tests/ List of tasks: https://github.com/ClickHouse/ClickHouse/blob/master/testsructions/easy\_tasks\_sorted\_en.md -# Test Data {#test-data} +## Test Data {#test-data} Developing ClickHouse often requires loading realistic datasets. It is particularly important for performance testing. We have a specially prepared set of anonymized data from Yandex.Metrica. It requires additionally some 3GB of free disk space. Note that this data is not required to accomplish most of the development tasks. @@ -272,7 +270,7 @@ Developing ClickHouse often requires loading realistic datasets. It is particula clickhouse-client --max_insert_block_size 100000 --query "INSERT INTO test.hits FORMAT TSV" < hits_v1.tsv clickhouse-client --max_insert_block_size 100000 --query "INSERT INTO test.visits FORMAT TSV" < visits_v1.tsv -# Creating Pull Request {#creating-pull-request} +## Creating Pull Request {#creating-pull-request} Navigate to your fork repository in GitHub’s UI. If you have been developing in a branch, you need to select that branch. There will be a “Pull request” button located on the screen. In essence, this means “create a request for accepting my changes into the main repository”. From 6483efd0a31d5eff8b6cf358cc18aa53738f2a20 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Mon, 25 May 2020 10:04:10 +0300 Subject: [PATCH 699/738] [docs] shorter toc title --- docs/en/development/architecture.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/development/architecture.md b/docs/en/development/architecture.md index 2ff06e3a686..71a65bf8cc5 100644 --- a/docs/en/development/architecture.md +++ b/docs/en/development/architecture.md @@ -1,6 +1,6 @@ --- toc_priority: 62 -toc_title: Overview of ClickHouse Architecture +toc_title: Architecture Overview --- # Overview of ClickHouse Architecture {#overview-of-clickhouse-architecture} From 7094378d4a75230825baa3d9bcef11d066a30427 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Mon, 25 May 2020 10:05:24 +0300 Subject: [PATCH 700/738] [docs] move browse-code.md lower in toc --- docs/en/development/browse-code.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/development/browse-code.md b/docs/en/development/browse-code.md index 3e7b259bb39..b4f755ca5d2 100644 --- a/docs/en/development/browse-code.md +++ b/docs/en/development/browse-code.md @@ -1,6 +1,6 @@ --- -toc_priority: 63 -toc_title: Browse Source Code +toc_priority: 71 +toc_title: Source Code --- # Browse ClickHouse Source Code {#browse-clickhouse-source-code} From eeb4cbc43390a1197e1c70046ca738c492593a92 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Wed, 20 May 2020 16:08:43 +0300 Subject: [PATCH 701/738] Fix settings NO_PASSWORD authentication mode in users.xml. --- src/Access/UsersConfigAccessStorage.cpp | 14 ++++++++------ tests/integration/test_authentication/test.py | 4 ++++ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index ce33383548f..f5f48a2390e 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -52,18 +52,20 @@ namespace String user_config = "users." + user_name; - bool has_password = config.has(user_config + ".password"); + bool has_no_password = config.has(user_config + ".no_password"); + bool has_password_plaintext = config.has(user_config + ".password"); bool has_password_sha256_hex = config.has(user_config + ".password_sha256_hex"); bool has_password_double_sha1_hex = config.has(user_config + ".password_double_sha1_hex"); - if (has_password + has_password_sha256_hex + has_password_double_sha1_hex > 1) - throw Exception("More than one field of 'password', 'password_sha256_hex', 'password_double_sha1_hex' is used to specify password for user " + user_name + ". Must be only one of them.", + size_t num_password_fields = has_no_password + has_password_plaintext + has_password_sha256_hex + has_password_double_sha1_hex; + if (num_password_fields > 1) + throw Exception("More than one field of 'password', 'password_sha256_hex', 'password_double_sha1_hex', 'no_password' are used to specify password for user " + user_name + ". Must be only one of them.", ErrorCodes::BAD_ARGUMENTS); - if (!has_password && !has_password_sha256_hex && !has_password_double_sha1_hex) - throw Exception("Either 'password' or 'password_sha256_hex' or 'password_double_sha1_hex' must be specified for user " + user_name + ".", ErrorCodes::BAD_ARGUMENTS); + if (num_password_fields < 1) + throw Exception("Either 'password' or 'password_sha256_hex' or 'password_double_sha1_hex' or 'no_password' must be specified for user " + user_name + ".", ErrorCodes::BAD_ARGUMENTS); - if (has_password) + if (has_password_plaintext) { user->authentication = Authentication{Authentication::PLAINTEXT_PASSWORD}; user->authentication.setPassword(config.getString(user_config + ".password")); diff --git a/tests/integration/test_authentication/test.py b/tests/integration/test_authentication/test.py index 483b59813e5..dedd5410188 100644 --- a/tests/integration/test_authentication/test.py +++ b/tests/integration/test_authentication/test.py @@ -23,6 +23,10 @@ def test_authentication_pass(): assert instance.query("SELECT currentUser()", user='sasha') == 'sasha\n' assert instance.query("SELECT currentUser()", user='masha', password='qwerty') == 'masha\n' + # 'no_password' authentication type allows to login with any password. + assert instance.query("SELECT currentUser()", user='sasha', password='something') == 'sasha\n' + assert instance.query("SELECT currentUser()", user='sasha', password='something2') == 'sasha\n' + def test_authentication_fail(): # User doesn't exist. From 3d9ba9ff8acc05f230f00c7d0d5f5d7d07f9320b Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 25 May 2020 13:06:27 +0300 Subject: [PATCH 702/738] Revert #11023 --- tests/clickhouse-test | 9 --------- .../00634_performance_introspection_and_logging.sh | 6 +++--- .../00933_test_fix_extra_seek_on_compressed_cache.sh | 2 +- .../0_stateless/00956_sensitive_data_masking.sh | 6 +++--- .../0_stateless/00974_text_log_table_not_empty.sh | 2 +- .../01070_exception_code_in_query_log_table.sql | 2 +- tests/queries/0_stateless/01091_num_threads.sql | 12 ++++++------ tests/queries/0_stateless/01092_memory_profiler.sql | 2 +- tests/queries/0_stateless/01198_client_quota_key.sh | 2 +- .../0_stateless/01231_log_queries_min_type.sql | 6 +++--- 10 files changed, 20 insertions(+), 29 deletions(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 324fd13aac2..1a2735296e8 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -234,14 +234,6 @@ def run_tests_array(all_tests_with_params): clickhouse_proc = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE) clickhouse_proc.communicate("SELECT 'Running test {suite}/{case} from pid={pid}';".format(pid = os.getpid(), case = case, suite = suite)) - if not args.no_system_log_cleanup: - clickhouse_proc = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE) - clickhouse_proc.communicate("SYSTEM FLUSH LOGS") - - for table in ['query_log', 'query_thread_log', 'trace_log', 'metric_log']: - clickhouse_proc = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE) - clickhouse_proc.communicate("TRUNCATE TABLE IF EXISTS system.{}".format(table)) - reference_file = os.path.join(suite_dir, name) + '.reference' stdout_file = os.path.join(suite_tmp_dir, name) + '.stdout' stderr_file = os.path.join(suite_tmp_dir, name) + '.stderr' @@ -572,7 +564,6 @@ if __name__ == '__main__': parser.add_argument('--stop', action='store_true', default=None, dest='stop', help='Stop on network errors') parser.add_argument('--order', default='desc', choices=['asc', 'desc', 'random'], help='Run order') parser.add_argument('--testname', action='store_true', default=None, dest='testname', help='Make query with test name before test run') - parser.add_argument('--no-system-log-cleanup', action='store_true', default=None, help='Do not cleanup system.*_log tables') parser.add_argument('--hung-check', action='store_true', default=False) parser.add_argument('--force-color', action='store_true', default=False) parser.add_argument('--database', help='Database for tests (random name test_XXXXXX by default)') diff --git a/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh b/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh index efb4b3569fb..5173b5f5772 100755 --- a/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh +++ b/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh @@ -47,7 +47,7 @@ SELECT threads_realtime >= threads_time_user_system_io, any(length(thread_ids)) >= 1 FROM - (SELECT * FROM system.query_log PREWHERE query='$heavy_cpu_query' WHERE type='QueryFinish' ORDER BY event_time DESC LIMIT 1) + (SELECT * FROM system.query_log PREWHERE query='$heavy_cpu_query' WHERE event_date >= today()-1 AND type=2 ORDER BY event_time DESC LIMIT 1) ARRAY JOIN ProfileEvents.Names AS PN, ProfileEvents.Values AS PV" # Check per-thread and per-query ProfileEvents consistency @@ -58,7 +58,7 @@ SELECT PN, PVq, PVt FROM SELECT PN, sum(PV) AS PVt FROM system.query_thread_log ARRAY JOIN ProfileEvents.Names AS PN, ProfileEvents.Values AS PV - WHERE query_id='$query_id' + WHERE event_date >= today()-1 AND query_id='$query_id' GROUP BY PN ) js1 ANY INNER JOIN @@ -66,7 +66,7 @@ ANY INNER JOIN SELECT PN, PV AS PVq FROM system.query_log ARRAY JOIN ProfileEvents.Names AS PN, ProfileEvents.Values AS PV - WHERE query_id='$query_id' + WHERE event_date >= today()-1 AND query_id='$query_id' ) js2 USING PN WHERE diff --git a/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh b/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh index 9e32c30ce20..1f7571a2404 100755 --- a/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh +++ b/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh @@ -19,7 +19,7 @@ $CLICKHOUSE_CLIENT --use_uncompressed_cache=1 --query_id="test-query-uncompresse sleep 1 $CLICKHOUSE_CLIENT --query="SYSTEM FLUSH LOGS" -$CLICKHOUSE_CLIENT --query="SELECT ProfileEvents.Values[indexOf(ProfileEvents.Names, 'Seek')], ProfileEvents.Values[indexOf(ProfileEvents.Names, 'ReadCompressedBytes')], ProfileEvents.Values[indexOf(ProfileEvents.Names, 'UncompressedCacheHits')] AS hit FROM system.query_log WHERE (query_id = 'test-query-uncompressed-cache') AND (type = 'QueryFinish') ORDER BY event_time DESC LIMIT 1" +$CLICKHOUSE_CLIENT --query="SELECT ProfileEvents.Values[indexOf(ProfileEvents.Names, 'Seek')], ProfileEvents.Values[indexOf(ProfileEvents.Names, 'ReadCompressedBytes')], ProfileEvents.Values[indexOf(ProfileEvents.Names, 'UncompressedCacheHits')] AS hit FROM system.query_log WHERE (query_id = 'test-query-uncompressed-cache') AND (type = 2) AND event_date >= yesterday() ORDER BY event_time DESC LIMIT 1" $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS small_table" diff --git a/tests/queries/0_stateless/00956_sensitive_data_masking.sh b/tests/queries/0_stateless/00956_sensitive_data_masking.sh index c492fd35b89..0f76c34eaff 100755 --- a/tests/queries/0_stateless/00956_sensitive_data_masking.sh +++ b/tests/queries/0_stateless/00956_sensitive_data_masking.sh @@ -95,7 +95,7 @@ echo 7 # and finally querylog $CLICKHOUSE_CLIENT \ --server_logs_file=/dev/null \ - --query="select * from system.query_log where query like '%TOPSECRET%';" + --query="select * from system.query_log where event_time>now() - 10 and query like '%TOPSECRET%';" rm -f $tmp_file >/dev/null 2>&1 @@ -117,8 +117,8 @@ sleep 0.1; echo 9 $CLICKHOUSE_CLIENT \ --server_logs_file=/dev/null \ - --query="SELECT if( count() > 0, 'text_log non empty', 'text_log empty') FROM system.text_log WHERE message like '%find_me%'; - select * from system.text_log where message like '%TOPSECRET=TOPSECRET%';" --ignore-error --multiquery + --query="SELECT if( count() > 0, 'text_log non empty', 'text_log empty') FROM system.text_log WHERE event_time>now() - 60 and message like '%find_me%'; + select * from system.text_log where event_time>now() - 60 and message like '%TOPSECRET=TOPSECRET%';" --ignore-error --multiquery echo 'finish' rm -f $tmp_file >/dev/null 2>&1 diff --git a/tests/queries/0_stateless/00974_text_log_table_not_empty.sh b/tests/queries/0_stateless/00974_text_log_table_not_empty.sh index 149f0668bd1..c3cde4c08bb 100755 --- a/tests/queries/0_stateless/00974_text_log_table_not_empty.sh +++ b/tests/queries/0_stateless/00974_text_log_table_not_empty.sh @@ -10,7 +10,7 @@ do ${CLICKHOUSE_CLIENT} --query="SYSTEM FLUSH LOGS" sleep 0.1; -if [[ $($CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL" -d "SELECT count() > 0 FROM system.text_log WHERE position(system.text_log.message, 'SELECT 6103') > 0") == 1 ]]; then echo 1; exit; fi; +if [[ $($CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL" -d "SELECT count() > 0 FROM system.text_log WHERE position(system.text_log.message, 'SELECT 6103') > 0 AND event_date >= yesterday()") == 1 ]]; then echo 1; exit; fi; done; diff --git a/tests/queries/0_stateless/01070_exception_code_in_query_log_table.sql b/tests/queries/0_stateless/01070_exception_code_in_query_log_table.sql index e1e81614ab7..2c99ba54112 100644 --- a/tests/queries/0_stateless/01070_exception_code_in_query_log_table.sql +++ b/tests/queries/0_stateless/01070_exception_code_in_query_log_table.sql @@ -3,5 +3,5 @@ SELECT * FROM test_table_for_01070_exception_code_in_query_log_table; -- { serve CREATE TABLE test_table_for_01070_exception_code_in_query_log_table (value UInt64) ENGINE=Memory(); SELECT * FROM test_table_for_01070_exception_code_in_query_log_table; SYSTEM FLUSH LOGS; -SELECT exception_code FROM system.query_log WHERE query = 'SELECT * FROM test_table_for_01070_exception_code_in_query_log_table' ORDER BY exception_code; +SELECT exception_code FROM system.query_log WHERE query = 'SELECT * FROM test_table_for_01070_exception_code_in_query_log_table' AND event_date >= yesterday() AND event_time > now() - INTERVAL 5 MINUTE ORDER BY exception_code; DROP TABLE IF EXISTS test_table_for_01070_exception_code_in_query_log_table; diff --git a/tests/queries/0_stateless/01091_num_threads.sql b/tests/queries/0_stateless/01091_num_threads.sql index a93568fcee5..876a2d15d1a 100644 --- a/tests/queries/0_stateless/01091_num_threads.sql +++ b/tests/queries/0_stateless/01091_num_threads.sql @@ -8,13 +8,13 @@ WITH ( SELECT query_id FROM system.query_log - WHERE (query = 'SELECT 1') + WHERE (query = 'SELECT 1') AND (event_date >= (today() - 1)) ORDER BY event_time DESC LIMIT 1 ) AS id SELECT uniqExact(thread_id) FROM system.query_thread_log -WHERE (query_id = id) AND (thread_id != master_thread_id); +WHERE (event_date >= (today() - 1)) AND (query_id = id) AND (thread_id != master_thread_id); select sum(number) from numbers(1000000); SYSTEM FLUSH LOGS; @@ -23,13 +23,13 @@ WITH ( SELECT query_id FROM system.query_log - WHERE (query = 'SELECT sum(number) FROM numbers(1000000)') + WHERE (query = 'SELECT sum(number) FROM numbers(1000000)') AND (event_date >= (today() - 1)) ORDER BY event_time DESC LIMIT 1 ) AS id SELECT uniqExact(thread_id) FROM system.query_thread_log -WHERE (query_id = id) AND (thread_id != master_thread_id); +WHERE (event_date >= (today() - 1)) AND (query_id = id) AND (thread_id != master_thread_id); select sum(number) from numbers_mt(1000000); SYSTEM FLUSH LOGS; @@ -38,10 +38,10 @@ WITH ( SELECT query_id FROM system.query_log - WHERE (query = 'SELECT sum(number) FROM numbers_mt(1000000)') + WHERE (query = 'SELECT sum(number) FROM numbers_mt(1000000)') AND (event_date >= (today() - 1)) ORDER BY event_time DESC LIMIT 1 ) AS id SELECT uniqExact(thread_id) > 2 FROM system.query_thread_log -WHERE (query_id = id) AND (thread_id != master_thread_id); +WHERE (event_date >= (today() - 1)) AND (query_id = id) AND (thread_id != master_thread_id); diff --git a/tests/queries/0_stateless/01092_memory_profiler.sql b/tests/queries/0_stateless/01092_memory_profiler.sql index 980f7f73d5d..c20b5c79cdb 100644 --- a/tests/queries/0_stateless/01092_memory_profiler.sql +++ b/tests/queries/0_stateless/01092_memory_profiler.sql @@ -3,4 +3,4 @@ SET allow_introspection_functions = 1; SET memory_profiler_step = 1000000; SELECT ignore(groupArray(number), 'test memory profiler') FROM numbers(10000000); SYSTEM FLUSH LOGS; -WITH addressToSymbol(arrayJoin(trace)) AS symbol SELECT count() > 0 FROM system.trace_log t WHERE trace_type = 'Memory' AND query_id = (SELECT query_id FROM system.query_log WHERE query LIKE '%test memory profiler%' ORDER BY event_time DESC LIMIT 1); +WITH addressToSymbol(arrayJoin(trace)) AS symbol SELECT count() > 0 FROM system.trace_log t WHERE event_date >= yesterday() AND trace_type = 'Memory' AND query_id = (SELECT query_id FROM system.query_log WHERE event_date >= yesterday() AND query LIKE '%test memory profiler%' ORDER BY event_time DESC LIMIT 1); diff --git a/tests/queries/0_stateless/01198_client_quota_key.sh b/tests/queries/0_stateless/01198_client_quota_key.sh index b3bc845cd06..f4b66aea6ac 100755 --- a/tests/queries/0_stateless/01198_client_quota_key.sh +++ b/tests/queries/0_stateless/01198_client_quota_key.sh @@ -3,4 +3,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh -$CLICKHOUSE_CLIENT --quota_key Hello --query_id test_quota_key --log_queries 1 --multiquery --query "SELECT 1; SYSTEM FLUSH LOGS; SELECT DISTINCT quota_key FROM system.query_log WHERE query_id = 'test_quota_key'" +$CLICKHOUSE_CLIENT --quota_key Hello --query_id test_quota_key --log_queries 1 --multiquery --query "SELECT 1; SYSTEM FLUSH LOGS; SELECT DISTINCT quota_key FROM system.query_log WHERE event_date >= yesterday() AND event_time >= now() - 300 AND query_id = 'test_quota_key'" diff --git a/tests/queries/0_stateless/01231_log_queries_min_type.sql b/tests/queries/0_stateless/01231_log_queries_min_type.sql index 565a5880b84..f2229c94a8a 100644 --- a/tests/queries/0_stateless/01231_log_queries_min_type.sql +++ b/tests/queries/0_stateless/01231_log_queries_min_type.sql @@ -2,14 +2,14 @@ set log_queries=1; select '01231_log_queries_min_type/QUERY_START'; system flush logs; -select count() from system.query_log where query like '%01231_log_queries_min_type/%' and query not like '%system.query_log%'; +select count() from system.query_log where query like '%01231_log_queries_min_type/%' and query not like '%system.query_log%' and event_date = today() and event_time >= now() - interval 1 minute; set log_queries_min_type='EXCEPTION_BEFORE_START'; select '01231_log_queries_min_type/EXCEPTION_BEFORE_START'; system flush logs; -select count() from system.query_log where query like '%01231_log_queries_min_type/%' and query not like '%system.query_log%'; +select count() from system.query_log where query like '%01231_log_queries_min_type/%' and query not like '%system.query_log%' and event_date = today() and event_time >= now() - interval 1 minute; set log_queries_min_type='EXCEPTION_WHILE_PROCESSING'; select '01231_log_queries_min_type/', max(number) from system.numbers limit 1e6 settings max_rows_to_read='100K'; -- { serverError 158; } system flush logs; -select count() from system.query_log where query like '%01231_log_queries_min_type/%' and query not like '%system.query_log%'; +select count() from system.query_log where query like '%01231_log_queries_min_type/%' and query not like '%system.query_log%' and event_date = today() and event_time >= now() - interval 1 minute; From 42f72d526fcf6698fac10e6dc52595ad8355c6e3 Mon Sep 17 00:00:00 2001 From: BayoNet Date: Mon, 25 May 2020 14:01:28 +0300 Subject: [PATCH 703/738] DOCS-323: LowCardinality (#11060) * CLICKHOUSEDOCS-323: The first version of the description. * CLICKHOUSEDOCS-323: Update. * CLICKHOUSEDOCS-323: toLowCardinality, low_cardinality_max_dictionary_size * CLICKHOUSEDOCS-323: Added descriptions of low_cardinality_use_single_dictionary_for_part, low_cardinality_allow_in_native_format, allow_suspicious_low_cardinality_types. * Update docs/en/sql-reference/data-types/lowcardinality.md Co-authored-by: Ivan Blinkov * Update docs/en/sql-reference/data-types/lowcardinality.md Co-authored-by: Ivan Blinkov * Update docs/en/sql-reference/functions/type-conversion-functions.md Co-authored-by: Ivan Blinkov * CLICKHOUSEDOCS-323: Updated by comments. * CLICKHOUSEDOCS-323: Updated by comments. * CLICKHOUSEDOCS-323: Fixed some grammar. Co-authored-by: Sergei Shtykov Co-authored-by: Ivan Blinkov --- docs/en/operations/settings/settings.md | 59 +++++++++++++++++++ .../data-types/aggregatefunction.md | 2 +- docs/en/sql-reference/data-types/array.md | 2 +- .../data-types/lowcardinality.md | 59 +++++++++++++++++++ docs/en/sql-reference/data-types/nullable.md | 2 +- docs/en/sql-reference/data-types/tuple.md | 2 +- .../functions/type-conversion-functions.md | 41 ++++++++++++- 7 files changed, 162 insertions(+), 5 deletions(-) create mode 100644 docs/en/sql-reference/data-types/lowcardinality.md diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index e6e692315f0..459e030a4cc 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -1265,4 +1265,63 @@ Possible values: Default value: 16. +## low_cardinality_max_dictionary_size {#low_cardinality_max_dictionary_size} + +Sets a maximum size in rows of a shared global dictionary for the [LowCardinality](../../sql-reference/data-types/lowcardinality.md) data type that can be written to a storage file system. This setting prevents issues with RAM in case of unlimited dictionary growth. All the data that can't be encoded due to maximum dictionary size limitation ClickHouse writes in an ordinary method. + +Possible values: + +- Any positive integer. + +Default value: 8192. + +## low_cardinality_use_single_dictionary_for_part {#low_cardinality_use_single_dictionary_for_part} + +Turns on or turns off using of single dictionary for the data part. + +By default, ClickHouse server monitors the size of dictionaries and if a dictionary overflows then the server starts to write the next one. To prohibit creating several dictionaries set `low_cardinality_use_single_dictionary_for_part = 1`. + +Possible values: + +- 1 — Creating several dictionaries for the data part is prohibited. +- 0 — Creating several dictionaries for the data part is not prohibited. + +Default value: 0. + +## low_cardinality_allow_in_native_format {#low_cardinality_allow_in_native_format} + +Allows or restricts using the [LowCardinality](../../sql-reference/data-types/lowcardinality.md) data type with the [Native](../../interfaces/formats.md#native) format. + +If usage of `LowCardinality` is restricted, ClickHouse server converts `LowCardinality`-columns to ordinary ones for `SELECT` queries, and convert ordinary columns to `LowCardinality`-columns for `INSERT` queries. + +This setting is required mainly for third-party clients which don't support `LowCardinality` data type. + +Possible values: + +- 1 — Usage of `LowCardinality` is not restricted. +- 0 — Usage of `LowCardinality` is restricted. + +Default value: 1. + + +## allow_suspicious_low_cardinality_types {#allow_suspicious_low_cardinality_types} + +Allows or restricts using [LowCardinality](../../sql-reference/data-types/lowcardinality.md) with data types with fixed size of 8 bytes or less: numeric data types and `FixedString(8_bytes_or_less)`. + +For small fixed values using of `LowCardinality` is usually inefficient, because ClickHouse stores a numeric index for each row. As a result: + +- Disk space usage can rise. +- RAM consumption can be higher, depending on a dictionary size. +- Some functions can work slower due to extra coding/encoding operations. + +Merge times in [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md)-engine tables can grow due to all the reasons described above. + +Possible values: + +- 1 — Usage of `LowCardinality` is not restricted. +- 0 — Usage of `LowCardinality` is restricted. + +Default value: 0. + + [Original article](https://clickhouse.tech/docs/en/operations/settings/settings/) diff --git a/docs/en/sql-reference/data-types/aggregatefunction.md b/docs/en/sql-reference/data-types/aggregatefunction.md index 842b5a87578..f214db20ea7 100644 --- a/docs/en/sql-reference/data-types/aggregatefunction.md +++ b/docs/en/sql-reference/data-types/aggregatefunction.md @@ -1,5 +1,5 @@ --- -toc_priority: 52 +toc_priority: 53 toc_title: AggregateFunction --- diff --git a/docs/en/sql-reference/data-types/array.md b/docs/en/sql-reference/data-types/array.md index 70596525ca0..98c36d228e3 100644 --- a/docs/en/sql-reference/data-types/array.md +++ b/docs/en/sql-reference/data-types/array.md @@ -1,5 +1,5 @@ --- -toc_priority: 51 +toc_priority: 52 toc_title: Array(T) --- diff --git a/docs/en/sql-reference/data-types/lowcardinality.md b/docs/en/sql-reference/data-types/lowcardinality.md new file mode 100644 index 00000000000..74eac6b54cd --- /dev/null +++ b/docs/en/sql-reference/data-types/lowcardinality.md @@ -0,0 +1,59 @@ +--- +toc_priority: 51 +toc_title: LowCardinality +--- + +# LowCardinality Data Type {#lowcardinality-data-type} + +Changes the internal representation of other data types to be dictionary-encoded. + +## Syntax {#lowcardinality-syntax} + +```sql +LowCardinality(data_type) +``` + +**Parameters** + +- `data_type` — [String](string.md), [FixedString](fixedstring.md), [Date](date.md), [DateTime](datetime.md), and numbers excepting [Decimal](decimal.md). `LowCardinality` is not efficient for some data types, see the [allow_suspicious_low_cardinality_types](../../operations/settings/settings.md#allow_suspicious_low_cardinality_types) setting description. + +## Description {#lowcardinality-dscr} + +`LowCardinality` is a superstructure that changes a data storage method and rules of data processing. ClickHouse applies [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) to `LowCardinality`-columns. Operating with dictionary encoded data significantly increases performance of [SELECT](../statements/select/index.md) queries for many applications. + +The efficiency of using `LowCarditality` data type depends on data diversity. If a dictionary contains less than 10,000 distinct values, then ClickHouse mostly shows higher efficiency of data reading and storing. If a dictionary contains more than 100,000 distinct values, then ClickHouse can perform worse in comparison with using ordinary data types. + +Consider using `LowCardinality` instead of [Enum](enum.md) when working with strings. `LowCardinality` provides more flexibility in use and often reveals the same or higher efficiency. + +## Example + +Create a table with a `LowCardinality`-column: + +```sql +CREATE TABLE lc_t +( + `id` UInt16, + `strings` LowCardinality(String) +) +ENGINE = MergeTree() +ORDER BY id +``` + +## Related Settings and Functions + +Settings: + +- [low_cardinality_max_dictionary_size](../../operations/settings/settings.md#low_cardinality_max_dictionary_size) +- [low_cardinality_use_single_dictionary_for_part](../../operations/settings/settings.md#low_cardinality_use_single_dictionary_for_part) +- [low_cardinality_allow_in_native_format](../../operations/settings/settings.md#low_cardinality_allow_in_native_format) +- [allow_suspicious_low_cardinality_types](../../operations/settings/settings.md#allow_suspicious_low_cardinality_types) + +Functions: + +- [toLowCardinality](../functions/type-conversion-functions.md#tolowcardinality) + +## See Also + +- [A Magical Mystery Tour of the LowCardinality Data Type](https://www.altinity.com/blog/2019/3/27/low-cardinality). +- [Reducing Clickhouse Storage Cost with the Low Cardinality Type – Lessons from an Instana Engineer](https://www.instana.com/blog/reducing-clickhouse-storage-cost-with-the-low-cardinality-type-lessons-from-an-instana-engineer/). +- [String Optimization (video presentation in Russian)](https://youtu.be/rqf-ILRgBdY?list=PL0Z2YDlm0b3iwXCpEFiOOYmwXzVmjJfEt). [Slides in English](https://github.com/yandex/clickhouse-presentations/raw/master/meetup19/string_optimization.pdf). \ No newline at end of file diff --git a/docs/en/sql-reference/data-types/nullable.md b/docs/en/sql-reference/data-types/nullable.md index 02e25060be3..dfa2d8a3b35 100644 --- a/docs/en/sql-reference/data-types/nullable.md +++ b/docs/en/sql-reference/data-types/nullable.md @@ -1,5 +1,5 @@ --- -toc_priority: 54 +toc_priority: 55 toc_title: Nullable --- diff --git a/docs/en/sql-reference/data-types/tuple.md b/docs/en/sql-reference/data-types/tuple.md index dce267fb781..66908e2c530 100644 --- a/docs/en/sql-reference/data-types/tuple.md +++ b/docs/en/sql-reference/data-types/tuple.md @@ -1,5 +1,5 @@ --- -toc_priority: 53 +toc_priority: 54 toc_title: Tuple(T1, T2, ...) --- diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md index cf9e3251486..61d5690bec4 100644 --- a/docs/en/sql-reference/functions/type-conversion-functions.md +++ b/docs/en/sql-reference/functions/type-conversion-functions.md @@ -516,7 +516,7 @@ Result: **See Also** -- \[ISO 8601 announcement by @xkcd\](https://xkcd.com/1179/) +- [ISO 8601 announcement by @xkcd](https://xkcd.com/1179/) - [RFC 1123](https://tools.ietf.org/html/rfc1123) - [toDate](#todate) - [toDateTime](#todatetime) @@ -529,4 +529,43 @@ Same as for [parseDateTimeBestEffort](#parsedatetimebesteffort) except that it r Same as for [parseDateTimeBestEffort](#parsedatetimebesteffort) except that it returns zero date or zero date time when it encounters a date format that cannot be processed. +## toLowCardinality {#tolowcardinality} + +Converts input parameter to the [LowCardianlity](../data-types/lowcardinality.md) version of same data type. + +To convert data from the `LowCardinality` data type use the [CAST](#type_conversion_function-cast) function. For example, `CAST(x as String)`. + +**Syntax** + +```sql +toLowCardinality(expr) +``` + +**Parameters** + +- `expr` — [Expression](../syntax.md#syntax-expressions) resulting in one of the [supported data types](../data-types/index.md#data_types). + + +**Returned values** + +- Result of `expr`. + +Type: `LowCardinality(expr_result_type)` + +**Example** + +Query: + +```sql +SELECT toLowCardinality('1') +``` + +Result: + +```text +┌─toLowCardinality('1')─┐ +│ 1 │ +└───────────────────────┘ +``` + [Original article](https://clickhouse.tech/docs/en/query_language/functions/type_conversion_functions/) From 39feb4f0d43f61331c88a4cb95e24ecf3fdf527a Mon Sep 17 00:00:00 2001 From: Ivan <5627721+abyss7@users.noreply.github.com> Date: Mon, 25 May 2020 14:10:14 +0300 Subject: [PATCH 704/738] Add --branch option to backporting script --- utils/github/__init__.py | 2 -- utils/github/__main__.py | 61 ++++++++++++++++++++++++---------------- utils/github/local.py | 24 ++++++++-------- 3 files changed, 50 insertions(+), 37 deletions(-) diff --git a/utils/github/__init__.py b/utils/github/__init__.py index e0fb6c8aee5..40a96afc6ff 100644 --- a/utils/github/__init__.py +++ b/utils/github/__init__.py @@ -1,3 +1 @@ # -*- coding: utf-8 -*- - -# REMOVE ME diff --git a/utils/github/__main__.py b/utils/github/__main__.py index 40f04aeb675..1dbc9895bd6 100644 --- a/utils/github/__main__.py +++ b/utils/github/__main__.py @@ -7,15 +7,15 @@ - All pull-requests must be squash-merged or explicitly merged without rebase. - All pull-requests to master must have at least one label prefixed with `pr-`. - Labels that require pull-request to be backported must be red colored (#ff0000). - - Stable branch name must be of form `YY.NUMBER`. - - All stable branches must be forked directly from the master branch and never be merged back, + - Release branch name must be of form `YY.NUMBER`. + - All release branches must be forked directly from the master branch and never be merged back, or merged with any other branches based on the master branch (including master branch itself). Output of this script: - Commits without references from pull-requests. - Pull-requests to master without proper labels. - - Pull-requests that need to be backported, with statuses per stable branch. + - Pull-requests that need to be backported, with statuses per release branch. ''' @@ -29,7 +29,7 @@ import sys try: from termcolor import colored # `pip install termcolor` except ImportError: - sys.exit("Package 'termcolor' not found. Try run: `pip3 install termcolor`") + sys.exit("Package 'termcolor' not found. Try run: `pip3 install [--user] termcolor`") CHECK_MARK = colored('🗸', 'green') @@ -45,8 +45,6 @@ parser.add_argument('--repo', '-r', type=str, default='', metavar='PATH', help='path to the root of the ClickHouse repository') parser.add_argument('--remote', type=str, default='origin', help='remote name of the "ClickHouse/ClickHouse" upstream') -parser.add_argument('-n', type=int, default=3, dest='number', - help='number of last stable branches to consider') parser.add_argument('--token', type=str, required=True, help='token for Github access') parser.add_argument('--login', type=str, @@ -54,31 +52,46 @@ parser.add_argument('--login', type=str, parser.add_argument('--auto-label', action='store_true', dest='autolabel', default=True, help='try to automatically parse PR description and put labels') +# Either select last N release branches, or specify them manually. +group = parser.add_mutually_exclusive_group(required=True) +group.add_argument('-n', type=int, default=3, dest='number', + help='number of last release branches to consider') +group.add_argument('--branch', type=str, action='append', metavar='BRANCH', + help='specific release branch name to consider') + args = parser.parse_args() github = query.Query(args.token, 30) repo = local.Local(args.repo, args.remote, github.get_default_branch()) -stables = repo.get_stables()[-args.number:] # [(branch name, base)] -if not stables: +if not args.branch: + release_branches = repo.get_release_branches()[-args.number:] # [(branch name, base)] +else: + release_branches = [] + all_release_branches = repo.get_release_branches() + for branch in all_release_branches: + if branch[0] in args.branch: + release_branches.append(branch) + +if not release_branches: sys.exit('No release branches found!') else: print('Found release branches:') - for stable in stables: - print(f'{CHECK_MARK} {stable[0]} forked from {stable[1]}') + for branch in release_branches: + print(f'{CHECK_MARK} {branch[0]} forked from {branch[1]}') -first_commit = stables[0][1] +first_commit = release_branches[0][1] pull_requests = github.get_pull_requests(first_commit, args.login) good_commits = set(pull_request['mergeCommit']['oid'] for pull_request in pull_requests) bad_commits = [] # collect and print them in the end from_commit = repo.get_head_commit() -for i in reversed(range(len(stables))): - for commit in repo.iterate(from_commit, stables[i][1]): +for i in reversed(range(len(release_branches))): + for commit in repo.iterate(from_commit, release_branches[i][1]): if str(commit) not in good_commits and commit.author.name != 'robot-clickhouse': bad_commits.append(commit) - from_commit = stables[i][1] + from_commit = release_branches[i][1] members = set(github.get_members("ClickHouse", "ClickHouse")) def print_responsible(pull_request): @@ -146,22 +159,22 @@ if need_backporting: no_backport_labeled = set() wait = set() - for stable in stables: - if repo.comparator(stable[1]) < repo.comparator(pull_request['mergeCommit']['oid']): - targets.append(stable[0]) + for branch in release_branches: + if repo.comparator(branch[1]) < repo.comparator(pull_request['mergeCommit']['oid']): + targets.append(branch[0]) # FIXME: compatibility logic - check for a manually set label, that indicates status 'backported'. - # FIXME: O(n²) - no need to iterate all labels for every `stable` + # FIXME: O(n²) - no need to iterate all labels for every `branch` for label in github.get_labels(pull_request): if re_vlabel.match(label['name']) or re_vlabel_backported.match(label['name']): - if f'v{stable[0]}' == label['name'] or f'v{stable[0]}-backported' == label['name']: - backport_labeled.add(stable[0]) + if f'v{branch[0]}' == label['name'] or f'v{branch[0]}-backported' == label['name']: + backport_labeled.add(branch[0]) if re_vlabel_conflicts.match(label['name']): - if f'v{stable[0]}-conflicts' == label['name']: - conflict_labeled.add(stable[0]) + if f'v{branch[0]}-conflicts' == label['name']: + conflict_labeled.add(branch[0]) if re_vlabel_no_backport.match(label['name']): - if f'v{stable[0]}-no-backport' == label['name']: - no_backport_labeled.add(stable[0]) + if f'v{branch[0]}-no-backport' == label['name']: + no_backport_labeled.add(branch[0]) for event in github.get_timeline(pull_request): if(event['isCrossRepository'] or diff --git a/utils/github/local.py b/utils/github/local.py index 8dbef6b9b6a..96a1ae765bf 100644 --- a/utils/github/local.py +++ b/utils/github/local.py @@ -1,7 +1,9 @@ # -*- coding: utf-8 -*- -# `pip install …` -import git # gitpython +try: + import git # `pip3 install gitpython` +except ImportError: + sys.exit("Package 'gitpython' not found. Try run: `pip3 install [--user] gitpython`") import functools import os @@ -11,7 +13,7 @@ import re class Local: '''Implements some useful methods atop of the local repository ''' - RE_STABLE_REF = re.compile(r'^refs/remotes/.+/\d+\.\d+$') + RE_RELEASE_BRANCH_REF = re.compile(r'^refs/remotes/.+/\d+\.\d+$') def __init__(self, repo_path, remote_name, default_branch_name): self._repo = git.Repo(repo_path, search_parent_directories=(not repo_path)) @@ -42,16 +44,16 @@ class Local: * head (git.Commit)). List is sorted by commits in ascending order. ''' - def get_stables(self): - stables = [] + def get_release_branches(self): + release_branches = [] - for stable in [r for r in self._remote.refs if Local.RE_STABLE_REF.match(r.path)]: - base = self._repo.merge_base(self._default, self._repo.commit(stable)) + for branch in [r for r in self._remote.refs if Local.RE_RELEASE_BRANCH_REF.match(r.path)]: + base = self._repo.merge_base(self._default, self._repo.commit(branch)) if not base: - print(f'Branch {stable.path} is not based on branch {self._default}. Ignoring.') + print(f'Branch {branch.path} is not based on branch {self._default}. Ignoring.') elif len(base) > 1: - print(f'Branch {stable.path} has more than one base commit. Ignoring.') + print(f'Branch {branch.path} has more than one base commit. Ignoring.') else: - stables.append((os.path.basename(stable.name), base[0])) + release_branches.append((os.path.basename(branch.name), base[0])) - return sorted(stables, key=lambda x : self.comparator(x[1])) + return sorted(release_branches, key=lambda x : self.comparator(x[1])) From 04322b7b87b6a397842774181213f7bae6737258 Mon Sep 17 00:00:00 2001 From: Artem Zuikov Date: Mon, 25 May 2020 15:06:14 +0300 Subject: [PATCH 705/738] remove stale unit-test --- src/Interpreters/tests/CMakeLists.txt | 4 - .../tests/expression_analyzer.cpp | 128 ------------------ 2 files changed, 132 deletions(-) delete mode 100644 src/Interpreters/tests/expression_analyzer.cpp diff --git a/src/Interpreters/tests/CMakeLists.txt b/src/Interpreters/tests/CMakeLists.txt index 26ebf007e6c..19d302d2b30 100644 --- a/src/Interpreters/tests/CMakeLists.txt +++ b/src/Interpreters/tests/CMakeLists.txt @@ -51,10 +51,6 @@ add_executable (in_join_subqueries_preprocessor in_join_subqueries_preprocessor. target_link_libraries (in_join_subqueries_preprocessor PRIVATE dbms clickhouse_parsers) add_check(in_join_subqueries_preprocessor) -add_executable (expression_analyzer expression_analyzer.cpp) -target_link_libraries (expression_analyzer PRIVATE dbms clickhouse_storages_system clickhouse_parsers clickhouse_common_io) -add_check(expression_analyzer) - add_executable (users users.cpp) target_link_libraries (users PRIVATE dbms clickhouse_common_config) diff --git a/src/Interpreters/tests/expression_analyzer.cpp b/src/Interpreters/tests/expression_analyzer.cpp deleted file mode 100644 index 987b5c763e6..00000000000 --- a/src/Interpreters/tests/expression_analyzer.cpp +++ /dev/null @@ -1,128 +0,0 @@ -#include - -#include -#include -#include - -#include -#include - -#include -#include - -#include -#include - -#include -#include -#include - - -using namespace DB; - -namespace DB -{ - namespace ErrorCodes - { - extern const int SYNTAX_ERROR; - } -} - -struct TestEntry -{ - String query; - std::unordered_map expected_aliases; /// alias -> AST.getID() - NamesAndTypesList source_columns = {}; - - bool check(const Context & context) - { - ASTPtr ast = parse(query); - - auto res = SyntaxAnalyzer(context).analyze(ast, source_columns); - return checkAliases(*res); - } - -private: - bool checkAliases(const SyntaxAnalyzerResult & res) - { - for (const auto & alias : res.aliases) - { - const String & alias_name = alias.first; - if (expected_aliases.count(alias_name) == 0 || - expected_aliases[alias_name] != alias.second->getID()) - { - std::cout << "unexpected alias: " << alias_name << ' ' << alias.second->getID() << std::endl; - return false; - } - else - expected_aliases.erase(alias_name); - } - - if (!expected_aliases.empty()) - { - std::cout << "missing aliases: " << expected_aliases.size() << std::endl; - return false; - } - - return true; - } - - static ASTPtr parse(const std::string & query) - { - ParserSelectQuery parser; - std::string message; - const auto * text = query.data(); - if (ASTPtr ast = tryParseQuery(parser, text, text + query.size(), message, false, "", false, 0, 0)) - return ast; - throw Exception(message, ErrorCodes::SYNTAX_ERROR); - } -}; - - -int main() -{ - std::vector queries = - { - { - "SELECT number AS n FROM system.numbers LIMIT 0", - {{"n", "Identifier_number"}}, - { NameAndTypePair("number", std::make_shared()) } - }, - - { - "SELECT number AS n FROM system.numbers LIMIT 0", - {{"n", "Identifier_number"}} - } - }; - - SharedContextHolder shared_context = Context::createShared(); - Context context = Context::createGlobal(shared_context.get()); - context.makeGlobalContext(); - - auto system_database = std::make_shared("system"); - DatabaseCatalog::instance().attachDatabase("system", system_database); - //context.setCurrentDatabase("system"); - system_database->attachTable("one", StorageSystemOne::create("one"), {}); - system_database->attachTable("numbers", StorageSystemNumbers::create(StorageID("system", "numbers"), false), {}); - - size_t success = 0; - for (auto & entry : queries) - { - try - { - if (entry.check(context)) - { - ++success; - std::cout << "[OK] " << entry.query << std::endl; - } - else - std::cout << "[Failed] " << entry.query << std::endl; - } - catch (Exception & e) - { - std::cout << "[Error] " << entry.query << std::endl << e.displayText() << std::endl; - } - } - - return success != queries.size(); -} From e013edce4b4486e9d2531428a595b40da132fc0e Mon Sep 17 00:00:00 2001 From: Vladimir Chebotarev Date: Mon, 25 May 2020 12:15:11 +0300 Subject: [PATCH 706/738] Added test for multi-page S3 globbing. --- tests/integration/test_storage_s3/test.py | 28 +++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py index 720701dd616..395c7e59886 100644 --- a/tests/integration/test_storage_s3/test.py +++ b/tests/integration/test_storage_s3/test.py @@ -1,6 +1,7 @@ import json import logging import random +import threading import pytest @@ -278,3 +279,30 @@ def test_wrong_s3_syntax(cluster, s3_storage_args): query = "create table test_table_s3_syntax (id UInt32) ENGINE = S3({})".format(s3_storage_args) assert expected_err_msg in instance.query_and_get_error(query) + + +def test_s3_glob_scheherazade(cluster): + bucket = cluster.minio_bucket + instance = cluster.instances["dummy"] # type: ClickHouseInstance + table_format = "column1 UInt32, column2 UInt32, column3 UInt32" + max_path = "" + values = "(1, 1, 1)" + nights_per_job = 1001 // 30 + jobs = [] + for night in range(0, 1001, nights_per_job): + def add_tales(start, end): + for i in range(start, end): + path = "night_{}/tale.csv".format(i) + query = "insert into table function s3('http://{}:{}/{}/{}', 'CSV', '{}') values {}".format( + cluster.minio_host, cluster.minio_port, bucket, path, table_format, values) + run_query(instance, query) + + jobs.append(threading.Thread(target=add_tales, args=(night, min(night+nights_per_job, 1001)))) + jobs[-1].start() + + for job in jobs: + job.join() + + query = "select count(), sum(column1), sum(column2), sum(column3) from s3('http://{}:{}/{}/night_*/tale.csv', 'CSV', '{}')".format( + cluster.minio_redirect_host, cluster.minio_redirect_port, bucket, table_format) + assert run_query(instance, query).splitlines() == ["1001\t1001\t1001\t1001"] From 7f9a257c3138bdabd12900f422d1557557e2b742 Mon Sep 17 00:00:00 2001 From: Vladimir Chebotarev Date: Mon, 25 May 2020 12:26:50 +0300 Subject: [PATCH 707/738] Fixed S3 globbing which could fail in case of more than 1000 keys and some backends. --- src/Storages/StorageS3.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index b8710de507b..22fe4656f17 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -24,7 +24,7 @@ #include #include -#include +#include #include #include @@ -228,18 +228,18 @@ Strings listFilesWithRegexpMatching(Aws::S3::S3Client & client, const S3::URI & return {globbed_uri.key}; } - Aws::S3::Model::ListObjectsRequest request; + Aws::S3::Model::ListObjectsV2Request request; request.SetBucket(globbed_uri.bucket); request.SetPrefix(key_prefix); re2::RE2 matcher(makeRegexpPatternFromGlobs(globbed_uri.key)); Strings result; - Aws::S3::Model::ListObjectsOutcome outcome; + Aws::S3::Model::ListObjectsV2Outcome outcome; int page = 0; do { ++page; - outcome = client.ListObjects(request); + outcome = client.ListObjectsV2(request); if (!outcome.IsSuccess()) { throw Exception("Could not list objects in bucket " + quoteString(request.GetBucket()) @@ -256,7 +256,7 @@ Strings listFilesWithRegexpMatching(Aws::S3::S3Client & client, const S3::URI & result.emplace_back(std::move(key)); } - request.SetMarker(outcome.GetResult().GetNextMarker()); + request.SetContinuationToken(outcome.GetResult().GetNextContinuationToken()); } while (outcome.GetResult().GetIsTruncated()); From 6cc43b743c7115e519d0b959941ceabfa46139b0 Mon Sep 17 00:00:00 2001 From: Artem Zuikov Date: Mon, 25 May 2020 16:56:57 +0300 Subject: [PATCH 708/738] one more place to remove from --- programs/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index 65742697333..baf8270d1bf 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -207,7 +207,7 @@ if (TARGET clickhouse-server AND TARGET copy-headers) endif () if (ENABLE_TESTS AND USE_GTEST) - set (CLICKHOUSE_ALL_TESTS_TARGETS local_date_time_comparison unit_tests_libcommon unit_tests_dbms hashing_write_buffer hashing_read_buffer in_join_subqueries_preprocessor expression_analyzer) + set (CLICKHOUSE_ALL_TESTS_TARGETS local_date_time_comparison unit_tests_libcommon unit_tests_dbms hashing_write_buffer hashing_read_buffer in_join_subqueries_preprocessor) add_custom_target (clickhouse-tests ALL DEPENDS ${CLICKHOUSE_ALL_TESTS_TARGETS}) add_dependencies(clickhouse-bundle clickhouse-tests) endif() From b48deb91aca8424b5d195fddfb08b1fdb7a3bbcc Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 25 May 2020 17:33:31 +0300 Subject: [PATCH 709/738] comments and styles --- src/Columns/ColumnAggregateFunction.h | 2 +- src/Columns/ColumnArray.cpp | 2 +- src/Columns/ColumnArray.h | 2 +- src/Columns/ColumnConst.cpp | 2 +- src/Columns/ColumnConst.h | 2 +- src/Columns/ColumnDecimal.cpp | 2 +- src/Columns/ColumnFixedString.cpp | 2 +- src/Columns/ColumnFixedString.h | 2 +- src/Columns/ColumnFunction.h | 2 +- src/Columns/ColumnLowCardinality.cpp | 2 +- src/Columns/ColumnLowCardinality.h | 2 +- src/Columns/ColumnNullable.cpp | 2 +- src/Columns/ColumnNullable.h | 2 +- src/Columns/ColumnString.cpp | 29 ++++++++------------------- src/Columns/ColumnString.h | 2 +- src/Columns/ColumnTuple.cpp | 7 ++----- src/Columns/ColumnTuple.h | 2 +- src/Columns/ColumnUnique.h | 4 ++-- src/Columns/ColumnVector.cpp | 2 +- src/Columns/IColumn.h | 19 +++++++++++++++--- src/Interpreters/sortBlock.cpp | 7 +++---- 21 files changed, 47 insertions(+), 51 deletions(-) diff --git a/src/Columns/ColumnAggregateFunction.h b/src/Columns/ColumnAggregateFunction.h index 2a692d889a4..40f73665ebe 100644 --- a/src/Columns/ColumnAggregateFunction.h +++ b/src/Columns/ColumnAggregateFunction.h @@ -193,7 +193,7 @@ public: } void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override; - void updatePermutation(bool reverse, size_t limit, int, Permutation & res, EqualRanges& equal_range) const override; + void updatePermutation(bool reverse, size_t limit, int, Permutation & res, EqualRanges & equal_range) const override; /** More efficient manipulation methods */ Container & getData() diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index 49678682713..604381f0c16 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -737,7 +737,7 @@ void ColumnArray::getPermutation(bool reverse, size_t limit, int nan_direction_h } } -void ColumnArray::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res, EqualRanges& equal_range) const +void ColumnArray::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res, EqualRanges & equal_range) const { if (limit >= size() || limit >= equal_range.back().second) limit = 0; diff --git a/src/Columns/ColumnArray.h b/src/Columns/ColumnArray.h index c21628a8cf2..55935a91cde 100644 --- a/src/Columns/ColumnArray.h +++ b/src/Columns/ColumnArray.h @@ -73,7 +73,7 @@ public: template ColumnPtr indexImpl(const PaddedPODArray & indexes, size_t limit) const; int compareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const override; void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override; - void updatePermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res, EqualRanges& equal_range) const override; + void updatePermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res, EqualRanges & equal_range) const override; void reserve(size_t n) override; size_t byteSize() const override; size_t allocatedBytes() const override; diff --git a/src/Columns/ColumnConst.cpp b/src/Columns/ColumnConst.cpp index 546b909c09a..545c0b1b300 100644 --- a/src/Columns/ColumnConst.cpp +++ b/src/Columns/ColumnConst.cpp @@ -120,7 +120,7 @@ void ColumnConst::getPermutation(bool /*reverse*/, size_t /*limit*/, int /*nan_d res[i] = i; } -void ColumnConst::updatePermutation(bool, size_t, int, Permutation &, EqualRanges&) const {} +void ColumnConst::updatePermutation(bool, size_t, int, Permutation &, EqualRanges &) const {} void ColumnConst::updateWeakHash32(WeakHash32 & hash) const { diff --git a/src/Columns/ColumnConst.h b/src/Columns/ColumnConst.h index aecc04c0e11..5fc96b14be8 100644 --- a/src/Columns/ColumnConst.h +++ b/src/Columns/ColumnConst.h @@ -170,7 +170,7 @@ public: ColumnPtr permute(const Permutation & perm, size_t limit) const override; ColumnPtr index(const IColumn & indexes, size_t limit) const override; void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override; - void updatePermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res, EqualRanges& equal_range) const override; + void updatePermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res, EqualRanges & equal_range) const override; size_t byteSize() const override { diff --git a/src/Columns/ColumnDecimal.cpp b/src/Columns/ColumnDecimal.cpp index 604a2c27ab2..3e6fb833b56 100644 --- a/src/Columns/ColumnDecimal.cpp +++ b/src/Columns/ColumnDecimal.cpp @@ -109,7 +109,7 @@ void ColumnDecimal::getPermutation(bool reverse, size_t limit, int , IColumn: } template -void ColumnDecimal::updatePermutation(bool reverse, size_t limit, int, IColumn::Permutation & res, EqualRanges& equal_range) const +void ColumnDecimal::updatePermutation(bool reverse, size_t limit, int, IColumn::Permutation & res, EqualRanges & equal_range) const { if (limit >= data.size() || limit >= equal_range.back().second) limit = 0; diff --git a/src/Columns/ColumnFixedString.cpp b/src/Columns/ColumnFixedString.cpp index ffd707111d8..5a4f6f763d7 100644 --- a/src/Columns/ColumnFixedString.cpp +++ b/src/Columns/ColumnFixedString.cpp @@ -162,7 +162,7 @@ void ColumnFixedString::getPermutation(bool reverse, size_t limit, int /*nan_dir } } -void ColumnFixedString::updatePermutation(bool reverse, size_t limit, int, Permutation & res, EqualRanges& equal_range) const +void ColumnFixedString::updatePermutation(bool reverse, size_t limit, int, Permutation & res, EqualRanges & equal_range) const { if (limit >= size() || limit >= equal_range.back().second) limit = 0; diff --git a/src/Columns/ColumnFixedString.h b/src/Columns/ColumnFixedString.h index be78651f7cb..996a1f99ef1 100644 --- a/src/Columns/ColumnFixedString.h +++ b/src/Columns/ColumnFixedString.h @@ -118,7 +118,7 @@ public: void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override; - void updatePermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res, EqualRanges& equal_range) const override; + void updatePermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res, EqualRanges & equal_range) const override; void insertRangeFrom(const IColumn & src, size_t start, size_t length) override; diff --git a/src/Columns/ColumnFunction.h b/src/Columns/ColumnFunction.h index 7b3252e3379..31cb8708a6e 100644 --- a/src/Columns/ColumnFunction.h +++ b/src/Columns/ColumnFunction.h @@ -121,7 +121,7 @@ public: throw Exception("getPermutation is not implemented for " + getName(), ErrorCodes::NOT_IMPLEMENTED); } - void updatePermutation(bool, size_t, int, Permutation &, EqualRanges&) const override + void updatePermutation(bool, size_t, int, Permutation &, EqualRanges &) const override { throw Exception("updatePermutation is not implemented for " + getName(), ErrorCodes::NOT_IMPLEMENTED); } diff --git a/src/Columns/ColumnLowCardinality.cpp b/src/Columns/ColumnLowCardinality.cpp index 47fa570a7cb..9e979a507ff 100644 --- a/src/Columns/ColumnLowCardinality.cpp +++ b/src/Columns/ColumnLowCardinality.cpp @@ -314,7 +314,7 @@ void ColumnLowCardinality::getPermutation(bool reverse, size_t limit, int nan_di } } -void ColumnLowCardinality::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges& equal_range) const +void ColumnLowCardinality::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges & equal_range) const { if (limit >= size() || limit >= equal_range.back().second) limit = 0; diff --git a/src/Columns/ColumnLowCardinality.h b/src/Columns/ColumnLowCardinality.h index 4a5fe7b7517..905d15f8167 100644 --- a/src/Columns/ColumnLowCardinality.h +++ b/src/Columns/ColumnLowCardinality.h @@ -111,7 +111,7 @@ public: void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override; - void updatePermutation(bool reverse, size_t limit, int, IColumn::Permutation & res, EqualRanges& equal_range) const override; + void updatePermutation(bool reverse, size_t limit, int, IColumn::Permutation & res, EqualRanges & equal_range) const override; ColumnPtr replicate(const Offsets & offsets) const override { diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp index 42c948a45b1..a3c4e77db0d 100644 --- a/src/Columns/ColumnNullable.cpp +++ b/src/Columns/ColumnNullable.cpp @@ -321,7 +321,7 @@ void ColumnNullable::getPermutation(bool reverse, size_t limit, int null_directi } } -void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_direction_hint, IColumn::Permutation & res, EqualRanges& equal_range) const +void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_direction_hint, IColumn::Permutation & res, EqualRanges & equal_range) const { if (limit >= equal_range.back().second || limit >= size()) limit = 0; diff --git a/src/Columns/ColumnNullable.h b/src/Columns/ColumnNullable.h index 3c18fc44ffa..2cd8ff9f40f 100644 --- a/src/Columns/ColumnNullable.h +++ b/src/Columns/ColumnNullable.h @@ -78,7 +78,7 @@ public: ColumnPtr index(const IColumn & indexes, size_t limit) const override; int compareAt(size_t n, size_t m, const IColumn & rhs_, int null_direction_hint) const override; void getPermutation(bool reverse, size_t limit, int null_direction_hint, Permutation & res) const override; - void updatePermutation(bool reverse, size_t limit, int, Permutation & res, EqualRanges& equal_range) const override; + void updatePermutation(bool reverse, size_t limit, int, Permutation & res, EqualRanges & equal_range) const override; void reserve(size_t n) override; size_t byteSize() const override; size_t allocatedBytes() const override; diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp index 5a386856c93..136a30d475a 100644 --- a/src/Columns/ColumnString.cpp +++ b/src/Columns/ColumnString.cpp @@ -302,7 +302,7 @@ void ColumnString::getPermutation(bool reverse, size_t limit, int /*nan_directio } } -void ColumnString::updatePermutation(bool reverse, size_t limit, int /*nan_direction_hint*/, Permutation & res, EqualRanges &equal_range) const +void ColumnString::updatePermutation(bool reverse, size_t limit, int /*nan_direction_hint*/, Permutation & res, EqualRanges & equal_range) const { if (limit >= size() || limit > equal_range.back().second) limit = 0; @@ -329,16 +329,13 @@ void ColumnString::updatePermutation(bool reverse, size_t limit, int /*nan_direc chars.data() + offsetAt(res[new_first]), sizeAt(res[new_first]) - 1) != 0) { if (j - new_first > 1) - { new_ranges.emplace_back(new_first, j); - } + new_first = j; } } if (last - new_first > 1) - { new_ranges.emplace_back(new_first, last); - } } if (limit) @@ -356,9 +353,7 @@ void ColumnString::updatePermutation(bool reverse, size_t limit, int /*nan_direc chars.data() + offsetAt(res[new_first]), sizeAt(res[new_first]) - 1) != 0) { if (j - new_first > 1) - { new_ranges.emplace_back(new_first, j); - } new_first = j; } } @@ -374,9 +369,7 @@ void ColumnString::updatePermutation(bool reverse, size_t limit, int /*nan_direc } } if (new_last - new_first > 1) - { new_ranges.emplace_back(new_first, new_last); - } } equal_range = std::move(new_ranges); } @@ -521,14 +514,12 @@ void ColumnString::getPermutationWithCollation(const Collator & collator, bool r void ColumnString::updatePermutationWithCollation(const Collator & collator, bool reverse, size_t limit, int, Permutation &res, EqualRanges &equal_range) const { if (limit >= size() || limit >= equal_range.back().second) - { limit = 0; - } + size_t n = equal_range.size(); if (limit) - { --n; - } + EqualRanges new_ranges; for (size_t i = 0; i < n; ++i) { @@ -545,17 +536,16 @@ void ColumnString::updatePermutationWithCollation(const Collator & collator, boo reinterpret_cast(&chars[offsetAt(res[j])]), sizeAt(res[j])) != 0) { if (j - new_first > 1) - { new_ranges.emplace_back(new_first, j); - } + new_first = j; } } if (last - new_first > 1) - { new_ranges.emplace_back(new_first, last); - } + } + if (limit) { const auto& [first, last] = equal_range.back(); @@ -571,9 +561,8 @@ void ColumnString::updatePermutationWithCollation(const Collator & collator, boo reinterpret_cast(&chars[offsetAt(res[j])]), sizeAt(res[j])) != 0) { if (j - new_first > 1) - { new_ranges.emplace_back(new_first, j); - } + new_first = j; } } @@ -589,9 +578,7 @@ void ColumnString::updatePermutationWithCollation(const Collator & collator, boo } } if (new_last - new_first > 1) - { new_ranges.emplace_back(new_first, new_last); - } } equal_range = std::move(new_ranges); } diff --git a/src/Columns/ColumnString.h b/src/Columns/ColumnString.h index 74803205a20..a0b3d259b67 100644 --- a/src/Columns/ColumnString.h +++ b/src/Columns/ColumnString.h @@ -225,7 +225,7 @@ public: void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override; - void updatePermutation(bool reverse, size_t limit, int, Permutation & res, EqualRanges& equal_range) const override; + void updatePermutation(bool reverse, size_t limit, int, Permutation & res, EqualRanges & equal_range) const override; /// Sorting with respect of collation. void getPermutationWithCollation(const Collator & collator, bool reverse, size_t limit, Permutation & res) const; diff --git a/src/Columns/ColumnTuple.cpp b/src/Columns/ColumnTuple.cpp index 2e99c378e00..78117b8e310 100644 --- a/src/Columns/ColumnTuple.cpp +++ b/src/Columns/ColumnTuple.cpp @@ -329,19 +329,16 @@ void ColumnTuple::getPermutation(bool reverse, size_t limit, int nan_direction_h } } -void ColumnTuple::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges& equal_range) const +void ColumnTuple::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges & equal_range) const { for (const auto& column : columns) { column->updatePermutation(reverse, limit, nan_direction_hint, res, equal_range); while (limit && limit <= equal_range.back().first) - { equal_range.pop_back(); - } + if (equal_range.empty()) - { break; - } } } diff --git a/src/Columns/ColumnTuple.h b/src/Columns/ColumnTuple.h index cceba49c4fa..69b18e2fc0f 100644 --- a/src/Columns/ColumnTuple.h +++ b/src/Columns/ColumnTuple.h @@ -72,7 +72,7 @@ public: int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override; void getExtremes(Field & min, Field & max) const override; void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override; - void updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges& equal_range) const override; + void updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges & equal_range) const override; void reserve(size_t n) override; size_t byteSize() const override; size_t allocatedBytes() const override; diff --git a/src/Columns/ColumnUnique.h b/src/Columns/ColumnUnique.h index b5c32ade9dc..5bbac6baf4d 100644 --- a/src/Columns/ColumnUnique.h +++ b/src/Columns/ColumnUnique.h @@ -77,7 +77,7 @@ public: } int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override; - void updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges& equal_range) const override; + void updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges & equal_range) const override; void getExtremes(Field & min, Field & max) const override { column_holder->getExtremes(min, max); } bool valuesHaveFixedSize() const override { return column_holder->valuesHaveFixedSize(); } @@ -376,7 +376,7 @@ int ColumnUnique::compareAt(size_t n, size_t m, const IColumn & rhs, } template -void ColumnUnique::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges& equal_range) const +void ColumnUnique::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges & equal_range) const { bool found_null_value_index = false; for (size_t i = 0; i < equal_range.size() && !found_null_value_index; ++i) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 322ca815ddf..c4f2c6d8705 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -220,7 +220,7 @@ void ColumnVector::getPermutation(bool reverse, size_t limit, int nan_directi } template -void ColumnVector::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges &equal_range) const +void ColumnVector::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges & equal_range) const { if (limit >= data.size() || limit >= equal_range.back().second) limit = 0; diff --git a/src/Columns/IColumn.h b/src/Columns/IColumn.h index db8342a1e0e..1d92ed1c3ab 100644 --- a/src/Columns/IColumn.h +++ b/src/Columns/IColumn.h @@ -25,7 +25,12 @@ class ColumnGathererStream; class Field; class WeakHash32; -typedef std::vector> EqualRanges; + +/* + * Represents a set of equal ranges in previous column to perform sorting in current column. + * Used in sorting by tuples. + * */ +using EqualRanges = std::vector >; /// Declares interface to store columns in memory. class IColumn : public COW @@ -247,8 +252,6 @@ public: */ virtual void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const = 0; - virtual void updatePermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res, EqualRanges &equal_ranges) const = 0; - enum class SpecialSort { NONE = 0, @@ -260,6 +263,16 @@ public: getPermutation(reverse, limit, nan_direction_hint, res); } + /*in updatePermutation we pass the current permutation and the intervals at which it should be sorted + * Then for each interval separately (except for the last one, if there is a limit) + * We sort it based on data about the current column, and find all the intervals within this + * interval that had the same values in this column. we can't tell about these values in what order they + * should have been, we form a new array with intervals that need to be sorted + * If there is a limit, then for the last interval we do partial sorting and all that is described above, + * but in addition we still find all the elements equal to the largest sorted, they will also need to be sorted. + */ + virtual void updatePermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res, EqualRanges & equal_ranges) const = 0; + /** Copies each element according offsets parameter. * (i-th element should be copied offsets[i] - offsets[i - 1] times.) * It is necessary in ARRAY JOIN operation. diff --git a/src/Interpreters/sortBlock.cpp b/src/Interpreters/sortBlock.cpp index 256cc8514fb..61a741b5cf4 100644 --- a/src/Interpreters/sortBlock.cpp +++ b/src/Interpreters/sortBlock.cpp @@ -184,13 +184,12 @@ void sortBlock(Block & block, const SortDescription & description, UInt64 limit) for (const auto& column : columns_with_sort_desc) { while (!ranges.empty() && limit && limit <= ranges.back().first) - { ranges.pop_back(); - } + + if (ranges.empty()) - { break; - } + if (isCollationRequired(column.description)) { From 58a2172154e5c0c3f705b4ca152da299fe6c94ed Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Fri, 22 May 2020 23:45:13 +0300 Subject: [PATCH 710/738] Improve build scripts related to protobuf and gRPC. --- cmake/find/grpc.cmake | 77 +-- cmake/find/protobuf.cmake | 63 +-- cmake/protobuf_generate_cpp.cmake | 166 ++++++ contrib/CMakeLists.txt | 14 +- contrib/grpc-cmake/CMakeLists.txt | 480 ++---------------- contrib/protobuf-cmake/CMakeLists.txt | 13 + src/Common/config.h.in | 1 + ...StorageSystemBuildOptions.generated.cpp.in | 1 + utils/test-data-generator/CMakeLists.txt | 2 - 9 files changed, 255 insertions(+), 562 deletions(-) create mode 100644 cmake/protobuf_generate_cpp.cmake create mode 100644 contrib/protobuf-cmake/CMakeLists.txt diff --git a/cmake/find/grpc.cmake b/cmake/find/grpc.cmake index e19cac5cf5f..0019dbd5eed 100644 --- a/cmake/find/grpc.cmake +++ b/cmake/find/grpc.cmake @@ -1,57 +1,26 @@ -set(_PROTOBUF_PROTOC $) -set(_GRPC_CPP_PLUGIN_EXECUTABLE $) +option (ENABLE_GRPC "Use gRPC" ${ENABLE_LIBRARIES}) -function(PROTOBUF_GENERATE_GRPC_CPP SRCS HDRS) - if(NOT ARGN) - message(SEND_ERROR "Error: PROTOBUF_GENERATE_GRPC_CPP() called without any proto files") - return() - endif() +if (ENABLE_GRPC) + option (USE_INTERNAL_GRPC_LIBRARY "Set to FALSE to use system gRPC library instead of bundled" ${NOT_UNBUNDLED}) - if(PROTOBUF_GENERATE_CPP_APPEND_PATH) - foreach(FIL ${ARGN}) - get_filename_component(ABS_FIL ${FIL} ABSOLUTE) - get_filename_component(ABS_PATH ${ABS_FIL} PATH) - list(FIND _protobuf_include_path ${ABS_PATH} _contains_already) - if(${_contains_already} EQUAL -1) - list(APPEND _protobuf_include_path -I ${ABS_PATH}) - endif() - endforeach() - else() - set(_protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR}) - endif() + if (USE_INTERNAL_GRPC_LIBRARY) + if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/grpc/include/grpc++/grpc++.h") + message(WARNING "submodule contrib/grpc is missing. To fix try run: \n git submodule update --init --recursive") + set (USE_INTERNAL_GRPC_LIBRARY OFF) + elif (NOT USE_PROTOBUF) + message(WARNING "gRPC requires protobuf which is disabled") + set (USE_INTERNAL_GRPC_LIBRARY OFF) + else() + set (GRPC_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/grpc/include") + set (GRPC_LIBRARY "libgrpc++") + set (USE_GRPC ON) + endif() + else() + find_package(grpc) + if (GRPC_INCLUDE_DIR AND GRPC_LIBRARY) + set (USE_GRPC ON) + endif() + endif() +endif() - if(DEFINED PROTOBUF_IMPORT_DIRS) - foreach(DIR ${Protobuf_IMPORT_DIRS}) - get_filename_component(ABS_PATH ${DIR} ABSOLUTE) - list(FIND _protobuf_include_path ${ABS_PATH} _contains_already) - if(${_contains_already} EQUAL -1) - list(APPEND _protobuf_include_path -I ${ABS_PATH}) - endif() - endforeach() - endif() - - set(${SRCS}) - set(${HDRS}) - foreach(FIL ${ARGN}) - get_filename_component(ABS_FIL ${FIL} ABSOLUTE) - get_filename_component(FIL_WE ${FIL} NAME_WE) - - list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.grpc.pb.cc") - list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.grpc.pb.h") - - add_custom_command( - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.grpc.pb.cc" - "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.grpc.pb.h" - COMMAND ${_PROTOBUF_PROTOC} - ARGS --grpc_out=${CMAKE_CURRENT_BINARY_DIR} - --plugin=protoc-gen-grpc=${_GRPC_CPP_PLUGIN_EXECUTABLE} - ${_protobuf_include_path} ${ABS_FIL} - DEPENDS ${ABS_FIL} - COMMENT "Running gRPC C++ protocol buffer compiler on ${FIL}" - VERBATIM) - endforeach() - - set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE) - set(${SRCS} ${${SRCS}} PARENT_SCOPE) - set(${HDRS} ${${HDRS}} PARENT_SCOPE) -endfunction() +message(STATUS "Using gRPC=${USE_GRPC}: ${GRPC_INCLUDE_DIR} : ${GRPC_LIBRARY}") diff --git a/cmake/find/protobuf.cmake b/cmake/find/protobuf.cmake index 5f686cfd96e..51110d3b6a3 100644 --- a/cmake/find/protobuf.cmake +++ b/cmake/find/protobuf.cmake @@ -28,68 +28,6 @@ elseif(NOT MISSING_INTERNAL_PROTOBUF_LIBRARY) set(Protobuf_LITE_LIBRARY libprotobuf-lite) set(Protobuf_PROTOC_EXECUTABLE "$") - - if(NOT DEFINED PROTOBUF_GENERATE_CPP_APPEND_PATH) - set(PROTOBUF_GENERATE_CPP_APPEND_PATH TRUE) - endif() - - function(PROTOBUF_GENERATE_CPP SRCS HDRS) - if(NOT ARGN) - message(SEND_ERROR "Error: PROTOBUF_GENERATE_CPP() called without any proto files") - return() - endif() - - if(PROTOBUF_GENERATE_CPP_APPEND_PATH) - # Create an include path for each file specified - foreach(FIL ${ARGN}) - get_filename_component(ABS_FIL ${FIL} ABSOLUTE) - get_filename_component(ABS_PATH ${ABS_FIL} PATH) - list(FIND _protobuf_include_path ${ABS_PATH} _contains_already) - if(${_contains_already} EQUAL -1) - list(APPEND _protobuf_include_path -I ${ABS_PATH}) - endif() - endforeach() - else() - set(_protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR}) - endif() - - if(DEFINED PROTOBUF_IMPORT_DIRS AND NOT DEFINED Protobuf_IMPORT_DIRS) - set(Protobuf_IMPORT_DIRS "${PROTOBUF_IMPORT_DIRS}") - endif() - - if(DEFINED Protobuf_IMPORT_DIRS) - foreach(DIR ${Protobuf_IMPORT_DIRS}) - get_filename_component(ABS_PATH ${DIR} ABSOLUTE) - list(FIND _protobuf_include_path ${ABS_PATH} _contains_already) - if(${_contains_already} EQUAL -1) - list(APPEND _protobuf_include_path -I ${ABS_PATH}) - endif() - endforeach() - endif() - - set(${SRCS}) - set(${HDRS}) - foreach(FIL ${ARGN}) - get_filename_component(ABS_FIL ${FIL} ABSOLUTE) - get_filename_component(FIL_WE ${FIL} NAME_WE) - - list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc") - list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h") - - add_custom_command( - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc" - "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h" - COMMAND ${Protobuf_PROTOC_EXECUTABLE} - ARGS --cpp_out ${CMAKE_CURRENT_BINARY_DIR} ${_protobuf_include_path} ${ABS_FIL} - DEPENDS ${ABS_FIL} ${Protobuf_PROTOC_EXECUTABLE} - COMMENT "Running C++ protocol buffer compiler on ${FIL}" - VERBATIM ) - endforeach() - - set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE) - set(${SRCS} ${${SRCS}} PARENT_SCOPE) - set(${HDRS} ${${HDRS}} PARENT_SCOPE) - endfunction() endif() if(OS_FREEBSD AND SANITIZE STREQUAL "address") @@ -102,6 +40,7 @@ if(OS_FREEBSD AND SANITIZE STREQUAL "address") endif() endif() +include (${ClickHouse_SOURCE_DIR}/cmake/protobuf_generate_cpp.cmake) endif() message(STATUS "Using protobuf=${USE_PROTOBUF}: ${Protobuf_INCLUDE_DIR} : ${Protobuf_LIBRARY} : ${Protobuf_PROTOC_EXECUTABLE}") diff --git a/cmake/protobuf_generate_cpp.cmake b/cmake/protobuf_generate_cpp.cmake new file mode 100644 index 00000000000..40ec3c8e681 --- /dev/null +++ b/cmake/protobuf_generate_cpp.cmake @@ -0,0 +1,166 @@ +# This file declares functions adding custom commands for generating C++ files from *.proto files: +# function (protobuf_generate_cpp SRCS HDRS) +# function (protobuf_generate_grpc_cpp SRCS HDRS) + +if (NOT USE_PROTOBUF) + message (WARNING "Could not use protobuf_generate_cpp() without the protobuf library") + return() +endif() + +if (NOT DEFINED PROTOBUF_PROTOC_EXECUTABLE) + set (PROTOBUF_PROTOC_EXECUTABLE "$") +endif() + +if (NOT DEFINED GRPC_CPP_PLUGIN_EXECUTABLE) + set (GRPC_CPP_PLUGIN_EXECUTABLE $) +endif() + +if (NOT DEFINED PROTOBUF_GENERATE_CPP_APPEND_PATH) + set (PROTOBUF_GENERATE_CPP_APPEND_PATH TRUE) +endif() + + +function(protobuf_generate_cpp_impl SRCS HDRS MODES OUTPUT_FILE_EXTS PLUGIN) + if(NOT ARGN) + message(SEND_ERROR "Error: protobuf_generate_cpp() called without any proto files") + return() + endif() + + if(PROTOBUF_GENERATE_CPP_APPEND_PATH) + # Create an include path for each file specified + foreach(FIL ${ARGN}) + get_filename_component(ABS_FIL ${FIL} ABSOLUTE) + get_filename_component(ABS_PATH ${ABS_FIL} PATH) + list(FIND protobuf_include_path ${ABS_PATH} _contains_already) + if(${_contains_already} EQUAL -1) + list(APPEND protobuf_include_path -I ${ABS_PATH}) + endif() + endforeach() + else() + set(protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR}) + endif() + + if(DEFINED PROTOBUF_IMPORT_DIRS AND NOT DEFINED Protobuf_IMPORT_DIRS) + set(Protobuf_IMPORT_DIRS "${PROTOBUF_IMPORT_DIRS}") + endif() + + if(DEFINED Protobuf_IMPORT_DIRS) + foreach(DIR ${Protobuf_IMPORT_DIRS}) + get_filename_component(ABS_PATH ${DIR} ABSOLUTE) + list(FIND protobuf_include_path ${ABS_PATH} _contains_already) + if(${_contains_already} EQUAL -1) + list(APPEND protobuf_include_path -I ${ABS_PATH}) + endif() + endforeach() + endif() + + set (intermediate_dir ${CMAKE_CURRENT_BINARY_DIR}/intermediate) + + set (protoc_args) + foreach (mode ${MODES}) + list (APPEND protoc_args "--${mode}_out" ${intermediate_dir}) + endforeach() + if (PLUGIN) + list (APPEND protoc_args "--plugin=${PLUGIN}") + endif() + + set(srcs) + set(hdrs) + set(all_intermediate_outputs) + + foreach(input_name ${ARGN}) + get_filename_component(abs_name ${input_name} ABSOLUTE) + get_filename_component(name ${input_name} NAME_WE) + + set (intermediate_outputs) + foreach (ext ${OUTPUT_FILE_EXTS}) + set (filename "${name}${ext}") + set (output "${CMAKE_CURRENT_BINARY_DIR}/${filename}") + set (intermediate_output "${intermediate_dir}/${filename}") + list (APPEND intermediate_outputs "${intermediate_output}") + list (APPEND all_intermediate_outputs "${intermediate_output}") + + if (${ext} MATCHES ".*\\.h") + list(APPEND hdrs "${output}") + else() + list(APPEND srcs "${output}") + endif() + + add_custom_command( + OUTPUT ${output} + COMMAND ${CMAKE_COMMAND} -DPROTOBUF_GENERATE_CPP_SCRIPT_MODE=1 -DUSE_PROTOBUF=1 -DDIR=${CMAKE_CURRENT_BINARY_DIR} -DFILENAME=${filename} -DCOMPILER_ID=${CMAKE_CXX_COMPILER_ID} -P ${ClickHouse_SOURCE_DIR}/cmake/protobuf_generate_cpp.cmake + DEPENDS ${intermediate_output}) + endforeach() + + add_custom_command( + OUTPUT ${intermediate_outputs} + COMMAND ${Protobuf_PROTOC_EXECUTABLE} + ARGS ${protobuf_include_path} ${protoc_args} ${abs_name} + DEPENDS ${abs_name} ${Protobuf_PROTOC_EXECUTABLE} ${PLUGIN} + COMMENT "Running C++ protocol buffer compiler on ${name}" + VERBATIM ) + endforeach() + + set_source_files_properties(${srcs} ${hdrs} ${all_intermediate_outputs} PROPERTIES GENERATED TRUE) + set(${SRCS} ${srcs} PARENT_SCOPE) + set(${HDRS} ${hdrs} PARENT_SCOPE) +endfunction() + + +if (PROTOBUF_GENERATE_CPP_SCRIPT_MODE) + set (output "${DIR}/${FILENAME}") + set (intermediate_dir ${DIR}/intermediate) + set (intermediate_output "${intermediate_dir}/${FILENAME}") + + if (COMPILER_ID STREQUAL "GNU") + set (pragma_push "#pragma GCC diagnostic push\n") + set (pragma_pop "#pragma GCC diagnostic pop\n") + set (pragma_disable_warnings "#pragma GCC diagnostic ignored \"-Wall\"\n" + "#pragma GCC diagnostic ignored \"-Wextra\"\n" + "#pragma GCC diagnostic ignored \"-Warray-bounds\"\n") + elseif (COMPILER_ID MATCHES "Clang") + set (pragma_push "#pragma clang diagnostic push\n") + set (pragma_pop "#pragma clang diagnostic pop\n") + set (pragma_disable_warnings "#pragma clang diagnostic ignored \"-Weverything\"\n") + endif() + + if (${FILENAME} MATCHES ".*\\.h") + file(WRITE "${output}" + "#pragma once\n" + ${pragma_push} + ${pragma_disable_warnings} + "#include \"${intermediate_output}\"\n" + ${pragma_pop} + ) + else() + file(WRITE "${output}" + ${pragma_disable_warnings} + "#include \"${intermediate_output}\"\n" + ) + endif() + return() +endif() + + +function(protobuf_generate_cpp SRCS HDRS) + set (modes cpp) + set (output_file_exts ".pb.cc" ".pb.h") + set (plugin) + + protobuf_generate_cpp_impl(srcs hdrs "${modes}" "${output_file_exts}" "${plugin}" ${ARGN}) + + set(${SRCS} ${srcs} PARENT_SCOPE) + set(${HDRS} ${hdrs} PARENT_SCOPE) +endfunction() + + +function(protobuf_generate_grpc_cpp SRCS HDRS) + set (modes cpp grpc) + set (output_file_exts ".pb.cc" ".pb.h" ".grpc.pb.cc" ".grpc.pb.h") + set (plugin "protoc-gen-grpc=${GRPC_CPP_PLUGIN_EXECUTABLE}") + + protobuf_generate_cpp_impl(srcs hdrs "${modes}" "${output_file_exts}" "${plugin}" ${ARGN}) + + set(${SRCS} ${srcs} PARENT_SCOPE) + set(${HDRS} ${hdrs} PARENT_SCOPE) +endfunction() diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 99f7be2cbb7..d122188ad0b 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -21,7 +21,6 @@ add_subdirectory (consistent-hashing-sumbur) add_subdirectory (consistent-hashing) add_subdirectory (croaring) add_subdirectory (FastMemcpy) -add_subdirectory (grpc-cmake) add_subdirectory (jemalloc-cmake) add_subdirectory (libcpuid-cmake) add_subdirectory (murmurhash) @@ -260,20 +259,17 @@ if (USE_INTERNAL_BROTLI_LIBRARY) endif () if (USE_INTERNAL_PROTOBUF_LIBRARY) - if (MAKE_STATIC_LIBRARIES) - set(protobuf_BUILD_SHARED_LIBS OFF CACHE INTERNAL "" FORCE) - else () - set(protobuf_BUILD_SHARED_LIBS ON CACHE INTERNAL "" FORCE) - endif () - set(protobuf_WITH_ZLIB 0 CACHE INTERNAL "" FORCE) # actually will use zlib, but skip find - set(protobuf_BUILD_TESTS OFF CACHE INTERNAL "" FORCE) - add_subdirectory(protobuf/cmake) + add_subdirectory(protobuf-cmake) endif () if (USE_INTERNAL_HDFS3_LIBRARY) add_subdirectory(libhdfs3-cmake) endif () +if (USE_INTERNAL_GRPC_LIBRARY) + add_subdirectory(grpc-cmake) +endif () + if (USE_INTERNAL_AWS_S3_LIBRARY) set (save_CMAKE_C_FLAGS ${CMAKE_C_FLAGS}) set (save_CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES}) diff --git a/contrib/grpc-cmake/CMakeLists.txt b/contrib/grpc-cmake/CMakeLists.txt index 0180c0c1d31..a70c155da53 100644 --- a/contrib/grpc-cmake/CMakeLists.txt +++ b/contrib/grpc-cmake/CMakeLists.txt @@ -4,6 +4,8 @@ cmake_minimum_required(VERSION 3.5.1) set(GRPC_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/grpc) set(GRPC_INCLUDE_DIR ${GRPC_SOURCE_DIR}/include/) set(GRPC_BINARY_DIR ${ClickHouse_BINARY_DIR}/contrib/grpc) + + if(UNIX) if(${CMAKE_SYSTEM_NAME} MATCHES "Linux") set(_gRPC_PLATFORM_LINUX ON) @@ -14,72 +16,28 @@ if(UNIX) endif() endif() -set(CMAKE_POSITION_INDEPENDENT_CODE TRUE) - -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${_gRPC_C_CXX_FLAGS} -w") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${_gRPC_C_CXX_FLAGS} -w") - -set(_gRPC_PROTOBUF_LIBRARY_NAME "libprotobuf") - -if(gRPC_BACKWARDS_COMPATIBILITY_MODE) - add_definitions(-DGPR_BACKWARDS_COMPATIBILITY_MODE) - if (_gRPC_PLATFORM_MAC) - # some C++11 constructs not supported before OS X 10.9 - set(CMAKE_OSX_DEPLOYMENT_TARGET 10.9) - endif() +if(_gRPC_PLATFORM_MAC) + set(_gRPC_ALLTARGETS_LIBRARIES ${CMAKE_DL_LIBS} m pthread) +elseif(UNIX) + set(_gRPC_ALLTARGETS_LIBRARIES ${CMAKE_DL_LIBS} rt m pthread) endif() -if (_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC) - set(_gRPC_CORE_NOSTDCXX_FLAGS -fno-exceptions -fno-rtti) -else() - set(_gRPC_CORE_NOSTDCXX_FLAGS "") -endif() # address_sorting.cmake -include(${GRPC_SOURCE_DIR}/cmake/address_sorting.cmake) set(_gRPC_ADDRESS_SORTING_INCLUDE_DIR "${GRPC_SOURCE_DIR}/third_party/address_sorting/include") set(_gRPC_ADDRESS_SORTING_LIBRARIES address_sorting) # cares.cmake set(CARES_ROOT_DIR ${GRPC_SOURCE_DIR}/third_party/cares/cares) set(CARES_BINARY_DIR ${GRPC_BINARY_DIR}/third_party/cares/cares) -set(CARES_SHARED OFF CACHE BOOL "disable shared library") -set(CARES_STATIC ON CACHE BOOL "link cares statically") -if(gRPC_BACKWARDS_COMPATIBILITY_MODE) - # See https://github.com/grpc/grpc/issues/17255 - set(HAVE_LIBNSL OFF CACHE BOOL "avoid cares dependency on libnsl") +set(CARES_SHARED ${BUILD_SHARED_LIBS} CACHE BOOL "" FORCE) +if(BUILD_SHARED_LIBS) + set(CARES_STATIC OFF CACHE BOOL "" FORCE) +else() + set(CARES_STATIC ON CACHE BOOL "" FORCE) endif() +set(_gRPC_CARES_LIBRARIES c-ares) add_subdirectory(${CARES_ROOT_DIR} ${CARES_BINARY_DIR}) -if(TARGET c-ares) - set(_gRPC_CARES_LIBRARIES c-ares) -endif() - -# protobuf.cmake -set(PROTOBUF_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../protobuf) - -set(protobuf_BUILD_TESTS OFF CACHE BOOL "Build protobuf tests") -if(NOT protobuf_WITH_ZLIB) - set(protobuf_WITH_ZLIB OFF CACHE BOOL "Build protobuf with zlib.") -endif() -set(protobuf_MSVC_STATIC_RUNTIME OFF CACHE BOOL "Link static runtime libraries") - -set(_gRPC_PROTOBUF_LIBRARIES libprotobuf) -set(_gRPC_PROTOBUF_PROTOC_LIBRARIES libprotoc) -set(_gRPC_PROTOBUF_PROTOC protoc) -set(_gRPC_PROTOBUF_PROTOC_EXECUTABLE $) -set(_gRPC_PROTOBUF_INCLUDE_DIR "${PROTOBUF_ROOT_DIR}/src") - -if(gRPC_INSTALL) - message(WARNING "gRPC_INSTALL will be forced to FALSE because gRPC_PROTOBUF_PROVIDER is \"module\"") - set(gRPC_INSTALL FALSE) -endif() - -# ssl.cmake -set(BORINGSSL_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../ssl) -if(TARGET ssl) - set(_gRPC_SSL_LIBRARIES ssl) - set(_gRPC_SSL_INCLUDE_DIR ${BORINGSSL_ROOT_DIR}/include) -endif() # upb.cmake set(UPB_ROOT_DIR ${GRPC_SOURCE_DIR}/third_party/upb) @@ -87,23 +45,20 @@ set(_gRPC_UPB_INCLUDE_DIR "${UPB_ROOT_DIR}") set(_gRPC_UPB_GRPC_GENERATED_DIR "${GRPC_SOURCE_DIR}/src/core/ext/upb-generated") set(_gRPC_UPB_LIBRARIES upb) +# protobuf.cmake +set(_gRPC_PROTOBUF_INCLUDE_DIR ${Protobuf_INCLUDE_DIR}) +set(_gRPC_PROTOBUF_LIBRARIES ${Protobuf_LIBRARY}) +set(_gRPC_PROTOBUF_PROTOC_LIBRARIES ${Protobuf_PROTOC_LIBRARY}) +set(_gRPC_PROTOBUF_PROTOC_EXECUTABLE ${Protobuf_PROTOC_EXECUTABLE}) + +# ssl.cmake +set(_gRPC_SSL_INCLUDE_DIR ${OPENSSL_INCLUDE_DIR}) +set(_gRPC_SSL_LIBRARIES ${OPENSSL_SSL_LIBRARY}) + # zlib.cmake -set(ZLIB_ROOT_DIR ${GRPC_SOURCE_DIR}/../zlib-ng) -include_directories("${ZLIB_ROOT_DIR}") -## add_subdirectory(${ZLIB_ROOT_DIR} ${ZLIB_ROOT_DIR}) -if(TARGET zlibstatic) - set(_gRPC_ZLIB_LIBRARIES zlibstatic) - set(_gRPC_ZLIB_INCLUDE_DIR "${ZLIB_ROOT_DIR}" "${GRPC_SOURCE_DIR}/third_party/zlib") -endif() +set(_gRPC_ZLIB_INCLUDE_DIR ${ZLIB_INCLUDE_DIR}) +set(_gRPC_ZLIB_LIBRARIES ${ZLIB_LIBRARIES}) -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") - -if(_gRPC_PLATFORM_MAC) - set(_gRPC_ALLTARGETS_LIBRARIES ${CMAKE_DL_LIBS} m pthread) -elseif(UNIX) - set(_gRPC_ALLTARGETS_LIBRARIES ${CMAKE_DL_LIBS} rt m pthread) -endif() add_library(address_sorting ${GRPC_SOURCE_DIR}/third_party/address_sorting/address_sorting.c @@ -112,7 +67,6 @@ add_library(address_sorting ) target_include_directories(address_sorting - PUBLIC $ $ PRIVATE ${GRPC_SOURCE_DIR} PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} PRIVATE ${_gRPC_CARES_INCLUDE_DIR} @@ -124,11 +78,11 @@ target_include_directories(address_sorting PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR} ) target_link_libraries(address_sorting - ${_gRPC_BASELIB_LIBRARIES} ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} ) + add_library(gpr ${GRPC_SOURCE_DIR}/src/core/lib/gpr/alloc.cc ${GRPC_SOURCE_DIR}/src/core/lib/gpr/atm.cc @@ -184,6 +138,7 @@ target_include_directories(gpr PRIVATE ${_gRPC_UPB_INCLUDE_DIR} PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR} ) + target_link_libraries(gpr ${_gRPC_ALLTARGETS_LIBRARIES} ${_gRPC_PROTOBUF_LIBRARIES} @@ -569,7 +524,6 @@ add_library(grpc ${GRPC_SOURCE_DIR}/src/core/plugin_registry/grpc_plugin_registry.cc ) -target_compile_options(grpc PUBLIC -fpermissive) target_include_directories(grpc PUBLIC ${GRPC_INCLUDE_DIR} @@ -583,8 +537,8 @@ target_include_directories(grpc PRIVATE ${_gRPC_UPB_INCLUDE_DIR} PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR} ) + target_link_libraries(grpc - ${_gRPC_BASELIB_LIBRARIES} ${_gRPC_SSL_LIBRARIES} ${_gRPC_ZLIB_LIBRARIES} ${_gRPC_CARES_LIBRARIES} @@ -597,352 +551,6 @@ if (_gRPC_PLATFORM_MAC) target_link_libraries(grpc "-framework CoreFoundation") endif() -add_library(grpc_cronet - ${GRPC_SOURCE_DIR}/src/core/ext/transport/cronet/plugin_registry/grpc_cronet_plugin_registry.cc - ${GRPC_SOURCE_DIR}/src/core/lib/surface/init.cc - ${GRPC_SOURCE_DIR}/src/core/lib/avl/avl.cc - ${GRPC_SOURCE_DIR}/src/core/lib/backoff/backoff.cc - ${GRPC_SOURCE_DIR}/src/core/lib/channel/channel_args.cc - ${GRPC_SOURCE_DIR}/src/core/lib/channel/channel_stack.cc - ${GRPC_SOURCE_DIR}/src/core/lib/channel/channel_stack_builder.cc - ${GRPC_SOURCE_DIR}/src/core/lib/channel/channel_trace.cc - ${GRPC_SOURCE_DIR}/src/core/lib/channel/channelz.cc - ${GRPC_SOURCE_DIR}/src/core/lib/channel/channelz_registry.cc - ${GRPC_SOURCE_DIR}/src/core/lib/channel/connected_channel.cc - ${GRPC_SOURCE_DIR}/src/core/lib/channel/handshaker.cc - ${GRPC_SOURCE_DIR}/src/core/lib/channel/handshaker_registry.cc - ${GRPC_SOURCE_DIR}/src/core/lib/channel/status_util.cc - ${GRPC_SOURCE_DIR}/src/core/lib/compression/compression.cc - ${GRPC_SOURCE_DIR}/src/core/lib/compression/compression_args.cc - ${GRPC_SOURCE_DIR}/src/core/lib/compression/compression_internal.cc - ${GRPC_SOURCE_DIR}/src/core/lib/compression/message_compress.cc - ${GRPC_SOURCE_DIR}/src/core/lib/compression/stream_compression.cc - ${GRPC_SOURCE_DIR}/src/core/lib/compression/stream_compression_gzip.cc - ${GRPC_SOURCE_DIR}/src/core/lib/compression/stream_compression_identity.cc - ${GRPC_SOURCE_DIR}/src/core/lib/debug/stats.cc - ${GRPC_SOURCE_DIR}/src/core/lib/debug/stats_data.cc - ${GRPC_SOURCE_DIR}/src/core/lib/http/format_request.cc - ${GRPC_SOURCE_DIR}/src/core/lib/http/httpcli.cc - ${GRPC_SOURCE_DIR}/src/core/lib/http/parser.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/buffer_list.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/call_combiner.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/cfstream_handle.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/combiner.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint_cfstream.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint_pair_posix.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint_pair_uv.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint_pair_windows.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/error.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/error_cfstream.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_epoll1_linux.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_epollex_linux.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_poll_posix.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_posix.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_windows.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/exec_ctx.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/executor.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/executor/mpmcqueue.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/executor/threadpool.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/fork_posix.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/fork_windows.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/gethostname_fallback.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/gethostname_host_name_max.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/gethostname_sysconf.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/grpc_if_nametoindex_posix.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/grpc_if_nametoindex_unsupported.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/internal_errqueue.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iocp_windows.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_custom.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_internal.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_posix.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_posix_cfstream.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_uv.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_windows.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/is_epollexclusive_available.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/load_file.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/lockfree_event.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/polling_entity.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_custom.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_set.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_set_custom.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_set_windows.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_uv.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_windows.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address_custom.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address_posix.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address_windows.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/resource_quota.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/sockaddr_utils.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_factory_posix.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_mutator.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_common_posix.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_linux.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_posix.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_uv.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_windows.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_windows.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client_cfstream.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client_custom.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client_posix.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client_windows.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_custom.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_posix.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_custom.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_posix.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_utils_posix_common.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_windows.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_uv.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_windows.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/time_averaged_stats.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/timer.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_custom.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_generic.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_heap.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_manager.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_uv.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/udp_server.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/unix_sockets_posix.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/unix_sockets_posix_noop.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/wakeup_fd_eventfd.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/wakeup_fd_nospecial.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/wakeup_fd_pipe.cc - ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/wakeup_fd_posix.cc - ${GRPC_SOURCE_DIR}/src/core/lib/json/json.cc - ${GRPC_SOURCE_DIR}/src/core/lib/json/json_reader.cc - ${GRPC_SOURCE_DIR}/src/core/lib/json/json_string.cc - ${GRPC_SOURCE_DIR}/src/core/lib/json/json_writer.cc - ${GRPC_SOURCE_DIR}/src/core/lib/slice/b64.cc - ${GRPC_SOURCE_DIR}/src/core/lib/slice/percent_encoding.cc - ${GRPC_SOURCE_DIR}/src/core/lib/slice/slice.cc - ${GRPC_SOURCE_DIR}/src/core/lib/slice/slice_buffer.cc - ${GRPC_SOURCE_DIR}/src/core/lib/slice/slice_intern.cc - ${GRPC_SOURCE_DIR}/src/core/lib/slice/slice_string_helpers.cc - ${GRPC_SOURCE_DIR}/src/core/lib/surface/api_trace.cc - ${GRPC_SOURCE_DIR}/src/core/lib/surface/byte_buffer.cc - ${GRPC_SOURCE_DIR}/src/core/lib/surface/byte_buffer_reader.cc - ${GRPC_SOURCE_DIR}/src/core/lib/surface/call.cc - ${GRPC_SOURCE_DIR}/src/core/lib/surface/call_details.cc - ${GRPC_SOURCE_DIR}/src/core/lib/surface/call_log_batch.cc - ${GRPC_SOURCE_DIR}/src/core/lib/surface/channel.cc - ${GRPC_SOURCE_DIR}/src/core/lib/surface/channel_init.cc - ${GRPC_SOURCE_DIR}/src/core/lib/surface/channel_ping.cc - ${GRPC_SOURCE_DIR}/src/core/lib/surface/channel_stack_type.cc - ${GRPC_SOURCE_DIR}/src/core/lib/surface/completion_queue.cc - ${GRPC_SOURCE_DIR}/src/core/lib/surface/completion_queue_factory.cc - ${GRPC_SOURCE_DIR}/src/core/lib/surface/event_string.cc - ${GRPC_SOURCE_DIR}/src/core/lib/surface/lame_client.cc - ${GRPC_SOURCE_DIR}/src/core/lib/surface/metadata_array.cc - ${GRPC_SOURCE_DIR}/src/core/lib/surface/server.cc - ${GRPC_SOURCE_DIR}/src/core/lib/surface/validate_metadata.cc - ${GRPC_SOURCE_DIR}/src/core/lib/surface/version.cc - ${GRPC_SOURCE_DIR}/src/core/lib/transport/bdp_estimator.cc - ${GRPC_SOURCE_DIR}/src/core/lib/transport/byte_stream.cc - ${GRPC_SOURCE_DIR}/src/core/lib/transport/connectivity_state.cc - ${GRPC_SOURCE_DIR}/src/core/lib/transport/error_utils.cc - ${GRPC_SOURCE_DIR}/src/core/lib/transport/metadata.cc - ${GRPC_SOURCE_DIR}/src/core/lib/transport/metadata_batch.cc - ${GRPC_SOURCE_DIR}/src/core/lib/transport/pid_controller.cc - ${GRPC_SOURCE_DIR}/src/core/lib/transport/static_metadata.cc - ${GRPC_SOURCE_DIR}/src/core/lib/transport/status_conversion.cc - ${GRPC_SOURCE_DIR}/src/core/lib/transport/status_metadata.cc - ${GRPC_SOURCE_DIR}/src/core/lib/transport/timeout_encoding.cc - ${GRPC_SOURCE_DIR}/src/core/lib/transport/transport.cc - ${GRPC_SOURCE_DIR}/src/core/lib/transport/transport_op_string.cc - ${GRPC_SOURCE_DIR}/src/core/lib/uri/uri_parser.cc - ${GRPC_SOURCE_DIR}/src/core/lib/debug/trace.cc - ${GRPC_SOURCE_DIR}/src/core/ext/transport/cronet/client/secure/cronet_channel_create.cc - ${GRPC_SOURCE_DIR}/src/core/ext/transport/cronet/transport/cronet_api_dummy.cc - ${GRPC_SOURCE_DIR}/src/core/ext/transport/cronet/transport/cronet_transport.cc - ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/client/secure/secure_channel_create.cc - ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/bin_decoder.cc - ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/bin_encoder.cc - ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/chttp2_plugin.cc - ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/chttp2_transport.cc - ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/context_list.cc - ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/flow_control.cc - ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_data.cc - ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_goaway.cc - ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_ping.cc - ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_rst_stream.cc - ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_settings.cc - ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_window_update.cc - ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_encoder.cc - ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_parser.cc - ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_table.cc - ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/http2_settings.cc - ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/huffsyms.cc - ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/incoming_metadata.cc - ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/parsing.cc - ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/stream_lists.cc - ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/stream_map.cc - ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/varint.cc - ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/writing.cc - ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/alpn/alpn.cc - ${GRPC_SOURCE_DIR}/src/core/ext/filters/http/client/http_client_filter.cc - ${GRPC_SOURCE_DIR}/src/core/ext/filters/http/http_filters_plugin.cc - ${GRPC_SOURCE_DIR}/src/core/ext/filters/http/message_compress/message_compress_filter.cc - ${GRPC_SOURCE_DIR}/src/core/ext/filters/http/server/http_server_filter.cc - ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/backend_metric.cc - ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/backup_poller.cc - ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/channel_connectivity.cc - ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel.cc - ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel_channelz.cc - ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel_factory.cc - ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel_plugin.cc - ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/connector.cc - ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/global_subchannel_pool.cc - ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/health/health_check_client.cc - ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/http_connect_handshaker.cc - ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/http_proxy.cc - ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy.cc - ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy_registry.cc - ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/local_subchannel_pool.cc - ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/parse_address.cc - ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/proxy_mapper.cc - ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/proxy_mapper_registry.cc - ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver.cc - ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver_registry.cc - ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver_result_parsing.cc - ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolving_lb_policy.cc - ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/retry_throttle.cc - ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/server_address.cc - ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/service_config.cc - ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/subchannel.cc - ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/subchannel_pool_interface.cc - ${GRPC_SOURCE_DIR}/src/core/ext/filters/deadline/deadline_filter.cc - ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/health/v1/health.upb.c - ${GRPC_SOURCE_DIR}/third_party/upb/upb/decode.c - ${GRPC_SOURCE_DIR}/third_party/upb/upb/encode.c - ${GRPC_SOURCE_DIR}/third_party/upb/upb/msg.c - ${GRPC_SOURCE_DIR}/third_party/upb/upb/port.c - ${GRPC_SOURCE_DIR}/third_party/upb/upb/table.c - ${GRPC_SOURCE_DIR}/third_party/upb/upb/upb.c - ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/udpa/data/orca/v1/orca_load_report.upb.c - ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/gogoproto/gogo.upb.c - ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/validate/validate.upb.c - ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/api/annotations.upb.c - ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/api/http.upb.c - ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/any.upb.c - ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/descriptor.upb.c - ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/duration.upb.c - ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/empty.upb.c - ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/struct.upb.c - ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/timestamp.upb.c - ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/wrappers.upb.c - ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/rpc/status.upb.c - ${GRPC_SOURCE_DIR}/src/core/lib/http/httpcli_security_connector.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/context/security_context.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/alts_credentials.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/composite/composite_credentials.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/credentials.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/credentials_metadata.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/fake/fake_credentials.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/google_default/credentials_generic.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/google_default/google_default_credentials.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/iam/iam_credentials.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/jwt/json_token.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/jwt/jwt_credentials.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/jwt/jwt_verifier.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/local/local_credentials.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/oauth2/oauth2_credentials.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/plugin/plugin_credentials.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/ssl/ssl_credentials.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/tls/grpc_tls_credentials_options.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/tls/spiffe_credentials.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/alts/alts_security_connector.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/fake/fake_security_connector.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/load_system_roots_fallback.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/load_system_roots_linux.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/local/local_security_connector.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/security_connector.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/ssl/ssl_security_connector.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/ssl_utils.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/ssl_utils_config.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/tls/spiffe_security_connector.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/transport/client_auth_filter.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/transport/secure_endpoint.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/transport/security_handshaker.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/transport/server_auth_filter.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/transport/target_authority_table.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/transport/tsi_error.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/util/json_util.cc - ${GRPC_SOURCE_DIR}/src/core/lib/surface/init_secure.cc - ${GRPC_SOURCE_DIR}/src/core/tsi/alts/crypt/aes_gcm.cc - ${GRPC_SOURCE_DIR}/src/core/tsi/alts/crypt/gsec.cc - ${GRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/alts_counter.cc - ${GRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/alts_crypter.cc - ${GRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/alts_frame_protector.cc - ${GRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/alts_record_protocol_crypter_common.cc - ${GRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/alts_seal_privacy_integrity_crypter.cc - ${GRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/alts_unseal_privacy_integrity_crypter.cc - ${GRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/frame_handler.cc - ${GRPC_SOURCE_DIR}/src/core/tsi/alts/handshaker/alts_handshaker_client.cc - ${GRPC_SOURCE_DIR}/src/core/tsi/alts/handshaker/alts_shared_resource.cc - ${GRPC_SOURCE_DIR}/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc - ${GRPC_SOURCE_DIR}/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_integrity_only_record_protocol.cc - ${GRPC_SOURCE_DIR}/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_privacy_integrity_record_protocol.cc - ${GRPC_SOURCE_DIR}/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_record_protocol_common.cc - ${GRPC_SOURCE_DIR}/src/core/tsi/alts/zero_copy_frame_protector/alts_iovec_record_protocol.cc - ${GRPC_SOURCE_DIR}/src/core/tsi/alts/zero_copy_frame_protector/alts_zero_copy_grpc_protector.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/check_gcp_environment.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/check_gcp_environment_linux.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/check_gcp_environment_no_op.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/check_gcp_environment_windows.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/grpc_alts_credentials_client_options.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/grpc_alts_credentials_options.cc - ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/grpc_alts_credentials_server_options.cc - ${GRPC_SOURCE_DIR}/src/core/tsi/alts/handshaker/alts_tsi_utils.cc - ${GRPC_SOURCE_DIR}/src/core/tsi/alts/handshaker/transport_security_common_api.cc - ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/gcp/altscontext.upb.c - ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/gcp/handshaker.upb.c - ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/gcp/transport_security_common.upb.c - ${GRPC_SOURCE_DIR}/src/core/tsi/transport_security.cc - ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/client/insecure/channel_create.cc - ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.cc - ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/client/authority.cc - ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/client/chttp2_connector.cc - ${GRPC_SOURCE_DIR}/src/core/tsi/fake_transport_security.cc - ${GRPC_SOURCE_DIR}/src/core/tsi/local_transport_security.cc - ${GRPC_SOURCE_DIR}/src/core/tsi/ssl/session_cache/ssl_session_boringssl.cc - ${GRPC_SOURCE_DIR}/src/core/tsi/ssl/session_cache/ssl_session_cache.cc - ${GRPC_SOURCE_DIR}/src/core/tsi/ssl/session_cache/ssl_session_openssl.cc - ${GRPC_SOURCE_DIR}/src/core/tsi/ssl_transport_security.cc - ${GRPC_SOURCE_DIR}/src/core/tsi/transport_security_grpc.cc -) - -target_include_directories(grpc_cronet - PUBLIC ${GRPC_INCLUDE_DIR} - PRIVATE ${GRPC_SOURCE_DIR} - PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} - PRIVATE ${_gRPC_CARES_INCLUDE_DIR} - PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR} - PRIVATE ${_gRPC_SSL_INCLUDE_DIR} - PRIVATE ${_gRPC_UPB_GENERATED_DIR} - PRIVATE ${_gRPC_UPB_GRPC_GENERATED_DIR} - PRIVATE ${_gRPC_UPB_INCLUDE_DIR} - PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR} -) -target_link_libraries(grpc_cronet - ${_gRPC_BASELIB_LIBRARIES} - ${_gRPC_SSL_LIBRARIES} - ${_gRPC_ZLIB_LIBRARIES} - ${_gRPC_CARES_LIBRARIES} - ${_gRPC_ADDRESS_SORTING_LIBRARIES} - ${_gRPC_ALLTARGETS_LIBRARIES} - ${_gRPC_PROTOBUF_LIBRARIES} - gpr -) -if (_gRPC_PLATFORM_MAC) - target_link_libraries(grpc_cronet "-framework CoreFoundation") -endif() add_library(grpc_unsecure ${GRPC_SOURCE_DIR}/src/core/lib/surface/init.cc @@ -1249,19 +857,18 @@ add_library(grpc_unsecure ${GRPC_SOURCE_DIR}/src/core/plugin_registry/grpc_unsecure_plugin_registry.cc ) - target_include_directories(grpc_unsecure PUBLIC ${GRPC_INCLUDE_DIR} PRIVATE ${GRPC_SOURCE_DIR} PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} PRIVATE ${_gRPC_CARES_INCLUDE_DIR} PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR} - PRIVATE ${_gRPC_SSL_INCLUDE_DIR} PRIVATE ${_gRPC_UPB_GENERATED_DIR} PRIVATE ${_gRPC_UPB_GRPC_GENERATED_DIR} PRIVATE ${_gRPC_UPB_INCLUDE_DIR} PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR} ) + target_link_libraries(grpc_unsecure ${_gRPC_BASELIB_LIBRARIES} ${_gRPC_ZLIB_LIBRARIES} @@ -1271,10 +878,12 @@ target_link_libraries(grpc_unsecure ${_gRPC_PROTOBUF_LIBRARIES} gpr ) + if (_gRPC_PLATFORM_MAC) target_link_libraries(grpc_unsecure "-framework CoreFoundation") endif() + add_library(grpc++ ${GRPC_SOURCE_DIR}/src/cpp/client/insecure_credentials.cc ${GRPC_SOURCE_DIR}/src/cpp/client/secure_credentials.cc @@ -1331,8 +940,6 @@ add_library(grpc++ ${GRPC_SOURCE_DIR}/src/cpp/codegen/codegen_init.cc ) -target_compile_options(grpc++ PUBLIC -w) - target_include_directories(grpc++ PUBLIC ${GRPC_INCLUDE_DIR} PRIVATE ${GRPC_SOURCE_DIR} @@ -1344,10 +951,9 @@ target_include_directories(grpc++ PRIVATE ${_gRPC_UPB_GRPC_GENERATED_DIR} PRIVATE ${_gRPC_UPB_INCLUDE_DIR} PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR} - PRIVATE ${_gRPC_PROTO_GENS_DIR} ) + target_link_libraries(grpc++ - ${_gRPC_BASELIB_LIBRARIES} ${_gRPC_SSL_LIBRARIES} ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} @@ -1355,6 +961,7 @@ target_link_libraries(grpc++ gpr ) + add_library(grpc++_unsecure ${GRPC_SOURCE_DIR}/src/cpp/client/insecure_credentials.cc ${GRPC_SOURCE_DIR}/src/cpp/common/insecure_create_auth_context.cc @@ -1404,21 +1011,19 @@ add_library(grpc++_unsecure ${GRPC_SOURCE_DIR}/src/cpp/codegen/codegen_init.cc ) -target_compile_options(grpc++_unsecure PUBLIC -w) - target_include_directories(grpc++_unsecure PUBLIC ${GRPC_INCLUDE_DIR} PRIVATE ${GRPC_SOURCE_DIR} PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} PRIVATE ${_gRPC_CARES_INCLUDE_DIR} PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR} - PRIVATE ${_gRPC_SSL_INCLUDE_DIR} PRIVATE ${_gRPC_UPB_GENERATED_DIR} PRIVATE ${_gRPC_UPB_GRPC_GENERATED_DIR} PRIVATE ${_gRPC_UPB_INCLUDE_DIR} PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR} PRIVATE ${_gRPC_PROTO_GENS_DIR} ) + target_link_libraries(grpc++_unsecure ${_gRPC_BASELIB_LIBRARIES} ${_gRPC_PROTOBUF_LIBRARIES} @@ -1427,6 +1032,16 @@ target_link_libraries(grpc++_unsecure grpc_unsecure ) + +if (_gRPC_SSL_INCLUDE_DIR AND _gRPC_SSL_LIBRARIES) + add_library(libgrpc ALIAS grpc) + add_library(libgrpc++ ALIAS grpc++) +else() + add_library(libgrpc ALIAS grpc_unsecure) + add_library(libgrpc++ ALIAS grpc++_unsecure) +endif() + + add_library(grpc_plugin_support ${GRPC_SOURCE_DIR}/src/compiler/cpp_generator.cc ) @@ -1436,23 +1051,22 @@ target_include_directories(grpc_plugin_support PUBLIC ${GRPC_INCLUDE_DIR} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} - PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR} PRIVATE ${_gRPC_CARES_INCLUDE_DIR} - PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR} PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR} PRIVATE ${_gRPC_SSL_INCLUDE_DIR} PRIVATE ${_gRPC_UPB_GENERATED_DIR} PRIVATE ${_gRPC_UPB_GRPC_GENERATED_DIR} PRIVATE ${_gRPC_UPB_INCLUDE_DIR} PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR} - PRIVATE ${_gRPC_PROTO_GENS_DIR} ) + target_link_libraries(grpc_plugin_support ${_gRPC_PROTOBUF_PROTOC_LIBRARIES} ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} ) + add_executable(grpc_cpp_plugin ${GRPC_SOURCE_DIR}/src/compiler/cpp_plugin.cc ) @@ -1461,16 +1075,13 @@ target_include_directories(grpc_cpp_plugin PRIVATE ${GRPC_SOURCE_DIR} PUBLIC ${GRPC_INCLUDE_DIR} PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} - PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR} PRIVATE ${_gRPC_CARES_INCLUDE_DIR} - PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR} PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR} PRIVATE ${_gRPC_SSL_INCLUDE_DIR} PRIVATE ${_gRPC_UPB_GENERATED_DIR} PRIVATE ${_gRPC_UPB_GRPC_GENERATED_DIR} PRIVATE ${_gRPC_UPB_INCLUDE_DIR} PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR} - PRIVATE ${_gRPC_PROTO_GENS_DIR} ) target_link_libraries(grpc_cpp_plugin @@ -1479,4 +1090,3 @@ target_link_libraries(grpc_cpp_plugin ${_gRPC_ALLTARGETS_LIBRARIES} grpc_plugin_support ) - diff --git a/contrib/protobuf-cmake/CMakeLists.txt b/contrib/protobuf-cmake/CMakeLists.txt new file mode 100644 index 00000000000..3cb9053d647 --- /dev/null +++ b/contrib/protobuf-cmake/CMakeLists.txt @@ -0,0 +1,13 @@ +set(protobuf_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/protobuf) +set(protobuf_BINARY_DIR ${ClickHouse_BINARY_DIR}/contrib/protobuf) + +set(protobuf_WITH_ZLIB 0 CACHE INTERNAL "" FORCE) # actually will use zlib, but skip find +set(protobuf_BUILD_TESTS OFF CACHE INTERNAL "" FORCE) + +if (MAKE_STATIC_LIBRARIES) + set(protobuf_BUILD_SHARED_LIBS OFF CACHE INTERNAL "" FORCE) +else () + set(protobuf_BUILD_SHARED_LIBS ON CACHE INTERNAL "" FORCE) +endif () + +add_subdirectory(${protobuf_SOURCE_DIR}/cmake ${protobuf_BINARY_DIR}) diff --git a/src/Common/config.h.in b/src/Common/config.h.in index df2359c1c29..237ca81ff9e 100644 --- a/src/Common/config.h.in +++ b/src/Common/config.h.in @@ -9,4 +9,5 @@ #cmakedefine01 USE_BROTLI #cmakedefine01 USE_UNWIND #cmakedefine01 USE_OPENCL +#cmakedefine01 USE_GRPC #cmakedefine01 CLICKHOUSE_SPLIT_BINARY diff --git a/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in b/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in index 3691ffb508d..9f73c00d22b 100644 --- a/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in +++ b/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in @@ -53,6 +53,7 @@ const char * auto_config_build[] "USE_SSL", "@USE_SSL@", "USE_HYPERSCAN", "@USE_HYPERSCAN@", "USE_SIMDJSON", "@USE_SIMDJSON@", + "USE_GRPC", "@USE_GRPC@", nullptr, nullptr }; diff --git a/utils/test-data-generator/CMakeLists.txt b/utils/test-data-generator/CMakeLists.txt index 3a94358e86d..758c3cdc0ce 100644 --- a/utils/test-data-generator/CMakeLists.txt +++ b/utils/test-data-generator/CMakeLists.txt @@ -1,8 +1,6 @@ # Disable clang-tidy for protobuf generated files set (CMAKE_CXX_CLANG_TIDY "") -add_compile_options(-Wno-zero-as-null-pointer-constant -Wno-array-bounds) # Protobuf generated files - if (USE_PROTOBUF) protobuf_generate_cpp(ProtobufDelimitedMessagesSerializer_Srcs ProtobufDelimitedMessagesSerializer_Hdrs ${CMAKE_CURRENT_SOURCE_DIR}/../../tests/queries/0_stateless/00825_protobuf_format.proto) protobuf_generate_cpp(ProtobufDelimitedMessagesSerializer_Srcs2 ProtobufDelimitedMessagesSerializer_Hdrs2 ${CMAKE_CURRENT_SOURCE_DIR}/../../tests/queries/0_stateless/00825_protobuf_format_syntax2.proto) From a01b6496b69fcbd5e60a40a31fe829d929630bba Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 25 May 2020 22:21:42 +0300 Subject: [PATCH 711/738] Fix types for function Capture. --- src/Functions/FunctionsMiscellaneous.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Functions/FunctionsMiscellaneous.h b/src/Functions/FunctionsMiscellaneous.h index 7c884cc7040..3e2938f386b 100644 --- a/src/Functions/FunctionsMiscellaneous.h +++ b/src/Functions/FunctionsMiscellaneous.h @@ -115,6 +115,7 @@ public: String getName() const override { return "FunctionCapture"; } bool useDefaultImplementationForNulls() const override { return false; } + bool useDefaultImplementationForLowCardinalityColumns() const override { return false; } void execute(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { @@ -243,6 +244,7 @@ public: String getName() const override { return name; } bool useDefaultImplementationForNulls() const override { return false; } + bool useDefaultImplementationForLowCardinalityColumns() const override { return false; } DataTypePtr getReturnType(const ColumnsWithTypeAndName &) const override { return return_type; } size_t getNumberOfArguments() const override { return capture->captured_types.size(); } From 53a514093a94a6d4f70d4d6dda831fb2e640c7a6 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 25 May 2020 22:22:03 +0300 Subject: [PATCH 712/738] Added test. --- .../00752_low_cardinality_lambda_argument.reference | 1 + .../0_stateless/00752_low_cardinality_lambda_argument.sql | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/tests/queries/0_stateless/00752_low_cardinality_lambda_argument.reference b/tests/queries/0_stateless/00752_low_cardinality_lambda_argument.reference index 20076c05d5d..b2e236400b0 100644 --- a/tests/queries/0_stateless/00752_low_cardinality_lambda_argument.reference +++ b/tests/queries/0_stateless/00752_low_cardinality_lambda_argument.reference @@ -8,3 +8,4 @@ [0,2,4,6] [0,2,4,6] [0,2,4,6,8] +[['a']] diff --git a/tests/queries/0_stateless/00752_low_cardinality_lambda_argument.sql b/tests/queries/0_stateless/00752_low_cardinality_lambda_argument.sql index 9e2ef3f2ff6..76c3d485ccb 100644 --- a/tests/queries/0_stateless/00752_low_cardinality_lambda_argument.sql +++ b/tests/queries/0_stateless/00752_low_cardinality_lambda_argument.sql @@ -4,3 +4,8 @@ insert into lc_lambda select range(number) from system.numbers limit 10; select arrayFilter(x -> x % 2 == 0, arr) from lc_lambda; drop table if exists lc_lambda; +drop table if exists test_array; +CREATE TABLE test_array(resources_host Array(LowCardinality(String))) ENGINE = MergeTree() ORDER BY (resources_host); +insert into test_array values (['a']); +SELECT arrayMap(i -> [resources_host[i]], arrayEnumerate(resources_host)) FROM test_array; +drop table if exists test_array; From 7201018079143a2063a5d0007a17d48bf3e8e260 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 26 May 2020 00:05:15 +0300 Subject: [PATCH 713/738] Update test.py --- tests/integration/test_storage_s3/test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py index 395c7e59886..9f124507e14 100644 --- a/tests/integration/test_storage_s3/test.py +++ b/tests/integration/test_storage_s3/test.py @@ -281,6 +281,7 @@ def test_wrong_s3_syntax(cluster, s3_storage_args): assert expected_err_msg in instance.query_and_get_error(query) +# https://en.wikipedia.org/wiki/One_Thousand_and_One_Nights def test_s3_glob_scheherazade(cluster): bucket = cluster.minio_bucket instance = cluster.instances["dummy"] # type: ClickHouseInstance From 73a0ef6c0ef86acaee0c627a1e38b424555bec43 Mon Sep 17 00:00:00 2001 From: filimonov <1549571+filimonov@users.noreply.github.com> Date: Mon, 25 May 2020 23:13:10 +0200 Subject: [PATCH 714/738] Fix for the hang during deletion of engine=Kafka (one more time) (#11145) * Added flag for safer rdkafka destruction, but more testing detected another hang case, which has no straigt solutions and can be workarounded currenly only by draining the consumer queue, so destructor is back * After review fixes * After review fixes2 * After review fixes3 --- contrib/cppkafka | 2 +- .../Kafka/ReadBufferFromKafkaConsumer.cpp | 70 ++++++++++++++++++- .../Kafka/ReadBufferFromKafkaConsumer.h | 4 +- src/Storages/Kafka/StorageKafka.cpp | 1 + tests/integration/test_storage_kafka/test.py | 57 +++++++++++++++ 5 files changed, 129 insertions(+), 5 deletions(-) diff --git a/contrib/cppkafka b/contrib/cppkafka index 9b184d881c1..f555ee36aaa 160000 --- a/contrib/cppkafka +++ b/contrib/cppkafka @@ -1 +1 @@ -Subproject commit 9b184d881c15cc50784b28688c7c99d3d764db24 +Subproject commit f555ee36aaa74d17ca0dab3ce472070a610b2966 diff --git a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp index 70340a93159..2db2e59cb7e 100644 --- a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp +++ b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp @@ -15,6 +15,7 @@ namespace ErrorCodes using namespace std::chrono_literals; const auto MAX_TIME_TO_WAIT_FOR_ASSIGNMENT_MS = 15000; +const auto DRAIN_TIMEOUT_MS = 5000ms; ReadBufferFromKafkaConsumer::ReadBufferFromKafkaConsumer( @@ -80,9 +81,72 @@ ReadBufferFromKafkaConsumer::ReadBufferFromKafkaConsumer( }); } -// NOTE on removed desctuctor: There is no need to unsubscribe prior to calling rd_kafka_consumer_close(). -// check: https://github.com/edenhill/librdkafka/blob/master/INTRODUCTION.md#termination -// manual destruction was source of weird errors (hangs during droping kafka table, etc.) +ReadBufferFromKafkaConsumer::~ReadBufferFromKafkaConsumer() +{ + try + { + if (!consumer->get_subscription().empty()) + { + try + { + consumer->unsubscribe(); + } + catch (const cppkafka::HandleException & e) + { + LOG_ERROR(log, "Error during unsubscribe: " << e.what()); + } + drain(); + } + } + catch (const cppkafka::HandleException & e) + { + LOG_ERROR(log, "Error while destructing consumer: " << e.what()); + } + +} + +// Needed to drain rest of the messages / queued callback calls from the consumer +// after unsubscribe, otherwise consumer will hang on destruction +// see https://github.com/edenhill/librdkafka/issues/2077 +// https://github.com/confluentinc/confluent-kafka-go/issues/189 etc. +void ReadBufferFromKafkaConsumer::drain() +{ + auto start_time = std::chrono::steady_clock::now(); + cppkafka::Error last_error(RD_KAFKA_RESP_ERR_NO_ERROR); + + while (true) + { + auto msg = consumer->poll(100ms); + if (!msg) + break; + + auto error = msg.get_error(); + + if (error) + { + if (msg.is_eof() || error == last_error) + { + break; + } + else + { + LOG_ERROR(log, "Error during draining: " << error); + } + } + + // i don't stop draining on first error, + // only if it repeats once again sequentially + last_error = error; + + auto ts = std::chrono::steady_clock::now(); + if (std::chrono::duration_cast(ts-start_time) > DRAIN_TIMEOUT_MS) + { + LOG_ERROR(log, "Timeout during draining."); + break; + } + } +} + void ReadBufferFromKafkaConsumer::commit() { diff --git a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h index 46dace827d0..12da701a55d 100644 --- a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h +++ b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h @@ -28,7 +28,7 @@ public: const std::atomic & stopped_, const Names & _topics ); - + ~ReadBufferFromKafkaConsumer() override; void allowNext() { allowed = true; } // Allow to read next message. void commit(); // Commit all processed messages. void subscribe(); // Subscribe internal consumer to topics. @@ -75,6 +75,8 @@ private: cppkafka::TopicPartitionList assignment; const Names topics; + void drain(); + bool nextImpl() override; }; diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index 78a28361f32..d1dfbe9d875 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -293,6 +293,7 @@ ConsumerBufferPtr StorageKafka::createReadBuffer() // Create a consumer and subscribe to topics auto consumer = std::make_shared(conf); + consumer->set_destroy_flags(RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE); // Limit the number of batched messages to allow early cancellations const Settings & settings = global_context.getSettingsRef(); diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py index 9154ad67c05..8e329a1cb60 100644 --- a/tests/integration/test_storage_kafka/test.py +++ b/tests/integration/test_storage_kafka/test.py @@ -246,6 +246,50 @@ def test_kafka_consumer_hang(kafka_cluster): # 'dr'||'op' to avoid self matching assert int(instance.query("select count() from system.processes where position(lower(query),'dr'||'op')>0")) == 0 +@pytest.mark.timeout(180) +def test_kafka_consumer_hang2(kafka_cluster): + + instance.query(''' + DROP TABLE IF EXISTS test.kafka; + + CREATE TABLE test.kafka (key UInt64, value UInt64) + ENGINE = Kafka + SETTINGS kafka_broker_list = 'kafka1:19092', + kafka_topic_list = 'consumer_hang2', + kafka_group_name = 'consumer_hang2', + kafka_format = 'JSONEachRow'; + + CREATE TABLE test.kafka2 (key UInt64, value UInt64) + ENGINE = Kafka + SETTINGS kafka_broker_list = 'kafka1:19092', + kafka_topic_list = 'consumer_hang2', + kafka_group_name = 'consumer_hang2', + kafka_format = 'JSONEachRow'; + ''') + + # first consumer subscribe the topic, try to poll some data, and go to rest + instance.query('SELECT * FROM test.kafka') + + # second consumer do the same leading to rebalance in the first + # consumer, try to poll some data + instance.query('SELECT * FROM test.kafka2') + +#echo 'SELECT * FROM test.kafka; SELECT * FROM test.kafka2; DROP TABLE test.kafka;' | clickhouse client -mn & +# kafka_cluster.open_bash_shell('instance') + + # first consumer has pending rebalance callback unprocessed (no poll after select) + # one of those queries was failing because of + # https://github.com/edenhill/librdkafka/issues/2077 + # https://github.com/edenhill/librdkafka/issues/2898 + instance.query('DROP TABLE test.kafka') + instance.query('DROP TABLE test.kafka2') + + + # from a user perspective: we expect no hanging 'drop' queries + # 'dr'||'op' to avoid self matching + assert int(instance.query("select count() from system.processes where position(lower(query),'dr'||'op')>0")) == 0 + + @pytest.mark.timeout(180) def test_kafka_csv_with_delimiter(kafka_cluster): instance.query(''' @@ -1130,6 +1174,7 @@ def test_kafka_rebalance(kafka_cluster): print(instance.query('SELECT count(), uniqExact(key), max(key) + 1 FROM test.destination')) + # Some queries to debug... # SELECT * FROM test.destination where key in (SELECT key FROM test.destination group by key having count() <> 1) # select number + 1 as key from numbers(4141) left join test.destination using (key) where test.destination.key = 0; # SELECT * FROM test.destination WHERE key between 2360 and 2370 order by key; @@ -1137,6 +1182,18 @@ def test_kafka_rebalance(kafka_cluster): # select toUInt64(0) as _partition, number + 1 as _offset from numbers(400) left join test.destination using (_partition,_offset) where test.destination.key = 0 order by _offset; # SELECT * FROM test.destination WHERE _partition = 0 and _offset between 220 and 240 order by _offset; + # CREATE TABLE test.reference (key UInt64, value UInt64) ENGINE = Kafka SETTINGS kafka_broker_list = 'kafka1:19092', + # kafka_topic_list = 'topic_with_multiple_partitions', + # kafka_group_name = 'rebalance_test_group_reference', + # kafka_format = 'JSONEachRow', + # kafka_max_block_size = 100000; + # + # CREATE MATERIALIZED VIEW test.reference_mv Engine=Log AS + # SELECT key, value, _topic,_key,_offset, _partition, _timestamp, 'reference' as _consumed_by + # FROM test.reference; + # + # select * from test.reference_mv left join test.destination using (key,_topic,_offset,_partition) where test.destination._consumed_by = ''; + result = int(instance.query('SELECT count() == uniqExact(key) FROM test.destination')) for consumer_index in range(NUMBER_OF_CONSURRENT_CONSUMERS): From 4bb3381bc997f29bfbb412e75290c8036c843660 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 26 May 2020 00:53:29 +0300 Subject: [PATCH 715/738] Fix build after merge --- src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp index 2db2e59cb7e..31a9f55350d 100644 --- a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp +++ b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp @@ -93,14 +93,14 @@ ReadBufferFromKafkaConsumer::~ReadBufferFromKafkaConsumer() } catch (const cppkafka::HandleException & e) { - LOG_ERROR(log, "Error during unsubscribe: " << e.what()); + LOG_ERROR(log, "Error during unsubscribe: {}", e.what()); } drain(); } } catch (const cppkafka::HandleException & e) { - LOG_ERROR(log, "Error while destructing consumer: " << e.what()); + LOG_ERROR(log, "Error while destructing consumer: {}", e.what()); } } @@ -130,7 +130,7 @@ void ReadBufferFromKafkaConsumer::drain() } else { - LOG_ERROR(log, "Error during draining: " << error); + LOG_ERROR(log, "Error during draining: {}", error); } } From 55a02c391d0d61e296014881354a2391536c4d0c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 26 May 2020 01:06:02 +0300 Subject: [PATCH 716/738] Style --- src/Common/PODArray.cpp | 3 ++- src/Common/PODArray.h | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/Common/PODArray.cpp b/src/Common/PODArray.cpp index 95810551ac4..e0b17c8125c 100644 --- a/src/Common/PODArray.cpp +++ b/src/Common/PODArray.cpp @@ -2,7 +2,8 @@ namespace DB { + /// Used for left padding of PODArray when empty -const char EmptyPODArray[EmptyPODArraySize]{}; +const char empty_pod_array[empty_pod_array_size]{}; } diff --git a/src/Common/PODArray.h b/src/Common/PODArray.h index 8fe1f74484e..1153e700e8b 100644 --- a/src/Common/PODArray.h +++ b/src/Common/PODArray.h @@ -63,8 +63,8 @@ namespace ErrorCodes * TODO Pass alignment to Allocator. * TODO Allow greater alignment than alignof(T). Example: array of char aligned to page size. */ -static constexpr size_t EmptyPODArraySize = 1024; -extern const char EmptyPODArray[EmptyPODArraySize]; +static constexpr size_t empty_pod_array_size = 1024; +extern const char empty_pod_array[empty_pod_array_size]; /** Base class that depend only on size of element, not on element itself. * You can static_cast to this class if you want to insert some data regardless to the actual type T. @@ -81,9 +81,9 @@ protected: /// pad_left is also rounded up to 16 bytes to maintain alignment of allocated memory. static constexpr size_t pad_left = integerRoundUp(integerRoundUp(pad_left_, ELEMENT_SIZE), 16); /// Empty array will point to this static memory as padding. - static constexpr char * null = pad_left ? const_cast(EmptyPODArray) + EmptyPODArraySize : nullptr; + static constexpr char * null = pad_left ? const_cast(empty_pod_array) + empty_pod_array_size : nullptr; - static_assert(pad_left <= EmptyPODArraySize && "Left Padding exceeds EmptyPODArraySize. Is the element size too large?"); + static_assert(pad_left <= empty_pod_array_size && "Left Padding exceeds empty_pod_array_size. Is the element size too large?"); char * c_start = null; /// Does not include pad_left. char * c_end = null; From c858f4d89c61c02580ec65fe788fbf0e128eee1f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 26 May 2020 02:47:11 +0300 Subject: [PATCH 717/738] Clarifications --- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 12baef4b7be..6e048a956da 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -418,7 +418,7 @@ void IMergeTreeDataPart::loadColumnsChecksumsIndexes(bool require_columns_checks loadIndexGranularity(); calculateColumnsSizesOnDisk(); loadIndex(); /// Must be called after loadIndexGranularity as it uses the value of `index_granularity` - loadRowsCount(); /// Must be called after loadIndex() as it uses the value of `index_granularity`. + loadRowsCount(); /// Must be called after loadIndexGranularity() as it uses the value of `index_granularity`. loadPartitionAndMinMaxIndex(); loadTTLInfos(); @@ -454,16 +454,18 @@ void IMergeTreeDataPart::loadIndex() String index_path = getFullRelativePath() + "primary.idx"; auto index_file = openForReading(volume->getDisk(), index_path); - for (size_t i = 0; i < index_granularity.getMarksCount(); ++i) //-V756 + size_t marks_count = index_granularity.getMarksCount(); + + for (size_t i = 0; i < marks_count; ++i) //-V756 for (size_t j = 0; j < key_size; ++j) primary_key.data_types[j]->deserializeBinary(*loaded_index[j], *index_file); for (size_t i = 0; i < key_size; ++i) { loaded_index[i]->protect(); - if (loaded_index[i]->size() != index_granularity.getMarksCount()) + if (loaded_index[i]->size() != marks_count) throw Exception("Cannot read all data from index file " + index_path - + "(expected size: " + toString(index_granularity.getMarksCount()) + ", read: " + toString(loaded_index[i]->size()) + ")", + + "(expected size: " + toString(marks_count) + ", read: " + toString(loaded_index[i]->size()) + ")", ErrorCodes::CANNOT_READ_ALL_DATA); } From 7fac670657806de2fcc688141d55a0b22c314d09 Mon Sep 17 00:00:00 2001 From: zzsmdfj Date: Tue, 26 May 2020 13:43:19 +0800 Subject: [PATCH 718/738] update zh-docs-operations (#11192) --- docs/en/operations/troubleshooting.md | 2 +- docs/zh/operations/access-rights.md | 195 ++++++++++++++++---------- docs/zh/operations/index.md | 10 ++ docs/zh/operations/monitoring.md | 13 +- docs/zh/operations/requirements.md | 22 ++- docs/zh/operations/troubleshooting.md | 34 +++-- docs/zh/operations/update.md | 10 +- 7 files changed, 170 insertions(+), 116 deletions(-) diff --git a/docs/en/operations/troubleshooting.md b/docs/en/operations/troubleshooting.md index cb7c81fffa8..1a33aa3f689 100644 --- a/docs/en/operations/troubleshooting.md +++ b/docs/en/operations/troubleshooting.md @@ -116,7 +116,7 @@ Check: Check: - The [tcp\_port\_secure](server-configuration-parameters/settings.md#server_configuration_parameters-tcp_port_secure) setting. - - Settings for [SSL sertificates](server-configuration-parameters/settings.md#server_configuration_parameters-openssl). + - Settings for [SSL certificates](server-configuration-parameters/settings.md#server_configuration_parameters-openssl). Use proper parameters while connecting. For example, use the `port_secure` parameter with `clickhouse_client`. diff --git a/docs/zh/operations/access-rights.md b/docs/zh/operations/access-rights.md index f596e16c269..97841699d09 100644 --- a/docs/zh/operations/access-rights.md +++ b/docs/zh/operations/access-rights.md @@ -1,101 +1,146 @@ -# 访问权限 {#access-rights} +--- +toc_priority: 48 +toc_title: "访问权限和账户管理" +--- -用户和访问权限在用户配置中设置。 这通常是 `users.xml`. +# 访问权限和账户管理 {#access-rights} +ClickHouse支持基于[RBAC](https://en.wikipedia.org/wiki/Role-based_access_control)的访问控制管理。 -用户被记录在 `users` 科。 这里是一个片段 `users.xml` 文件: +ClickHouse权限实体包括： +- [用户账户](#user-account-management) +- [角色](#role-management) +- [行策略](#row-policy-management) +- [设置描述](#settings-profiles-management) +- [配额](#quotas-management) -``` xml - - - - - - +- 服务端[配置文件](configuration-files.md) `users.xml` 和 `config.xml`. - - +!!! note "警告" + 你无法同时使用两个配置的方式来管理同一个权限实体。 - - default - - default - +## 用法 {#access-control-usage} - - - - - web - default - - test - - -``` +默认ClickHouse提供了一个 `default` 账号，这个账号有所有的权限，但是不能使用SQL驱动方式的访问权限和账户管理。`default`主要用在用户名还未设置的情况，比如从客户端登录或者执行分布式查询。在分布式查询中如果服务端或者集群没有指定[用户名密码](../engines/table-engines/special/distributed.md)那默认的账户就会被使用。 -您可以看到两个用户的声明: `default`和`web`. 我们添加了 `web` 用户分开。 +如果你刚开始使用ClickHouse，考虑如下场景： -该 `default` 在用户名未通过的情况下选择用户。 该 `default` 如果服务器或群集的配置没有指定分布式查询处理，则user也用于分布式查询处理 `user` 和 `password` （见上的部分 [分布](../engines/table-engines/special/distributed.md) 发动机）。 +1. 为 `default` 用户[开启SQL驱动方式的访问权限和账户管理](#enabling-access-control) . +2. 使用 `default` 用户登录并且创建所需要的所有用户。 不要忘记创建管理员账户 (`GRANT ALL ON *.* WITH GRANT OPTION TO admin_user_account`)。 +3. [限制](settings/permissions-for-queries.md#permissions_for_queries) `default` 用户的权限并且禁用SQL驱动方式的访问权限和账户管理。 -The user that is used for exchanging information between servers combined in a cluster must not have substantial restrictions or quotas – otherwise, distributed queries will fail. +### 当前解决方案的特性 {#access-control-properties} -密码以明文（不推荐）或SHA-256形式指定。 哈希没有腌制。 在这方面，您不应将这些密码视为提供了针对潜在恶意攻击的安全性。 相反，他们是必要的保护员工。 +- 你甚至可以在数据库和表不存在的时候授予权限。 +- 如果表被删除，和这张表关联的特权不会被删除。这意味着如果你创建一张同名的表，所有的特权仍旧有效。如果想删除这张表关联的特权，你可以执行 `REVOKE ALL PRIVILEGES ON db.table FROM ALL` 查询。 +- 特权没有生命周期。 -指定允许访问的网络列表。 在此示例中，将从单独的文件加载两个用户的网络列表 (`/etc/metrika.xml`）包含 `networks` 替代。 这里是它的一个片段: +## 用户账户 {#user-account-management} -``` xml - - ... - - ::/64 - 203.0.113.0/24 - 2001:DB8::/32 - ... - - -``` +用户账户是权限实体，用来授权操作ClickHouse，用户账户包含： -您可以直接在以下内容中定义此网络列表 `users.xml`，或在文件中 `users.d` directory (for more information, see the section «[配置文件](configuration-files.md#configuration_files)»). +- 标识符信息。 +- [特权](../sql-reference/statements/grant.md#grant-privileges)用来定义用户可以执行的查询的范围。 +- 可以连接到ClickHouse的主机。 +- 指定或者默认的角色。 +- 用户登录的时候默认的限制设置。 +- 指定的设置描述。 -该配置包括解释如何从任何地方打开访问的注释。 +特权可以通过[GRANT](../sql-reference/statements/grant.md)查询授权给用户或者通过[角色](#role-management)授予。如果想撤销特权，可以使用[REVOKE](../sql-reference/statements/revoke.md)查询。查询用户所有的特权，使用[SHOW GRANTS](../sql-reference/statements/show.md#show-grants-statement)语句。 -对于在生产中使用，仅指定 `ip` 元素（IP地址及其掩码），因为使用 `host` 和 `hoost_regexp` 可能会导致额外的延迟。 +查询管理： -Next the user settings profile is specified (see the section «[设置配置文件](settings/settings-profiles.md)»). You can specify the default profile, `default'`. 配置文件可以有任何名称。 您可以为不同的用户指定相同的配置文件。 您可以在设置配置文件中编写的最重要的事情是 `readonly=1`，这确保只读访问。 -Then specify the quota to be used (see the section «[配额](quotas.md#quotas)»). You can specify the default quota: `default`. It is set in the config by default to only count resource usage, without restricting it. The quota can have any name. You can specify the same quota for different users – in this case, resource usage is calculated for each user individually. +- [CREATE USER](../sql-reference/statements/create.md#create-user-statement) +- [ALTER USER](../sql-reference/statements/alter.md#alter-user-statement) +- [DROP USER](../sql-reference/statements/misc.md#drop-user-statement) +- [SHOW CREATE USER](../sql-reference/statements/show.md#show-create-user-statement) -在可选 `` 您还可以指定用户可以访问的数据库列表。 默认情况下，所有数据库都可供用户使用。 您可以指定 `default` 数据库。 在这种情况下，默认情况下，用户将接收对数据库的访问权限。 +### 设置应用规则 {#access-control-settings-applying} -访问 `system` 始终允许数据库（因为此数据库用于处理查询）。 +对于一个用户账户来说，设置可以通过多种方式配置：通过角色扮演和设置描述。对于一个登陆的账号来说，如果一个设置对应了多个不同的权限实体，这些设置的应用规则如下（优先权从高到底）： -用户可以通过以下方式获取其中所有数据库和表的列表 `SHOW` 查询或系统表，即使不允许访问单个数据库。 +1. 用户账户设置。 +2. 用户账号默认的角色设置。如果这个设置配置了多个角色，那设置的应用是没有规定的顺序。 +3. 从设置描述分批给用户或者角色的设置。如果这个设置配置了多个角色，那设置的应用是没有规定的顺序。 +4. 对所有服务器有效的默认或者[default profile](server-configuration-parameters/settings.md#default-profile)的设置。 -数据库访问是不相关的 [只读](settings/permissions-for-queries.md#settings_readonly) 设置。 您不能授予对一个数据库的完全访问权限，并 `readonly` 进入另一个。 -[原始文章](https://clickhouse.tech/docs/en/operations/access_rights/) +## 角色 {#role-management} + +角色是权限实体的集合，可以被授予用户账号。 + +角色包括： + +- [特权](../sql-reference/statements/grant.md#grant-privileges) +- 设置和限制 +- 分配的角色列表 + +查询管理: + +- [CREATE ROLE](../sql-reference/statements/create.md#create-role-statement) +- [ALTER ROLE](../sql-reference/statements/alter.md#alter-role-statement) +- [DROP ROLE](../sql-reference/statements/misc.md#drop-role-statement) +- [SET ROLE](../sql-reference/statements/misc.md#set-role-statement) +- [SET DEFAULT ROLE](../sql-reference/statements/misc.md#set-default-role-statement) +- [SHOW CREATE ROLE](../sql-reference/statements/show.md#show-create-role-statement) + +使用[GRANT](../sql-reference/statements/grant.md) 查询可以把特权授予给角色。用[REVOKE](../sql-reference/statements/revoke.md)来撤回特权。 + +## 行策略 {#row-policy-management} + +行策略是一个过滤器，用来定义哪些行数据可以被账户或者角色访问。对一个特定的表来说，行策略包括过滤器和使用这个策略的账户和角色。 + +查询管理： + +- [CREATE ROW POLICY](../sql-reference/statements/create.md#create-row-policy-statement) +- [ALTER ROW POLICY](../sql-reference/statements/alter.md#alter-row-policy-statement) +- [DROP ROW POLICY](../sql-reference/statements/misc.md#drop-row-policy-statement) +- [SHOW CREATE ROW POLICY](../sql-reference/statements/show.md#show-create-row-policy-statement) + + +## 设置描述 {#settings-profiles-management} + +设置描述是[设置](settings/index.md)的汇总。设置汇总包括设置和限制，当然也包括这些描述的对象：角色和账户。 + +查询管理: + +- [CREATE SETTINGS PROFILE](../sql-reference/statements/create.md#create-settings-profile-statement) +- [ALTER SETTINGS PROFILE](../sql-reference/statements/alter.md#alter-settings-profile-statement) +- [DROP SETTINGS PROFILE](../sql-reference/statements/misc.md#drop-settings-profile-statement) +- [SHOW CREATE SETTINGS PROFILE](../sql-reference/statements/show.md#show-create-settings-profile-statement) + + +## 配额 {#quotas-management} + +配额用来限制资源的使用情况。参考[配额](quotas.md). + +配额包括特定时间的限制条件和使用这个配额的账户和角色。 + +Management queries: + +- [CREATE QUOTA](../sql-reference/statements/create.md#create-quota-statement) +- [ALTER QUOTA](../sql-reference/statements/alter.md#alter-quota-statement) +- [DROP QUOTA](../sql-reference/statements/misc.md#drop-quota-statement) +- [SHOW CREATE QUOTA](../sql-reference/statements/show.md#show-create-quota-statement) + + +## 开启SQL驱动方式的访问权限和账户管理 {#enabling-access-control} + +- 为配置的存储设置一个目录. + + ClickHouse把访问实体的相关配置存储在[访问控制目录](server-configuration-parameters/settings.md#access_control_path)，而这个目录可以通过服务端进行配置. + +- 为至少一个账户开启SQL驱动方式的访问权限和账户管理. + + 默认情况，SQL驱动方式的访问权限和账户管理对所有用户都是关闭的。你需要在 `users.xml` 中配置至少一个用户，并且把[权限管理](settings/settings-users.md#access_management-user-setting)的值设置为1。 + + +[Original article](https://clickhouse.tech/docs/en/operations/access_rights/) diff --git a/docs/zh/operations/index.md b/docs/zh/operations/index.md index 2544cfd1c20..f35858279f5 100644 --- a/docs/zh/operations/index.md +++ b/docs/zh/operations/index.md @@ -1,3 +1,13 @@ +--- +toc_priority: 43 +toc_title: "操作" +--- + # 操作 {#operations} +Clickhouse运维手册主要包含下面几部分： + +- 安装要求 + + [原始文章](https://clickhouse.tech/docs/en/operations/) diff --git a/docs/zh/operations/monitoring.md b/docs/zh/operations/monitoring.md index 1db80399c36..0bf8556a870 100644 --- a/docs/zh/operations/monitoring.md +++ b/docs/zh/operations/monitoring.md @@ -1,3 +1,8 @@ +--- +toc_priority: 45 +toc_title: "监控" +--- + # 监控 {#jian-kong} 可以监控到： @@ -13,7 +18,7 @@ ClickHouse 本身不会去监控硬件资源的状态。 - 处理器上的负载和温度。 - 可以使用 [dmesg](https://en.wikipedia.org/wiki/Dmesg), [turbostat](https://www.linux.org/docs/man8/turbostat.html) 或者其他工具。 + 可以使用[dmesg](https://en.wikipedia.org/wiki/Dmesg), [turbostat](https://www.linux.org/docs/man8/turbostat.html)或者其他工具。 - 磁盘存储，RAM和网络的使用率。 @@ -21,17 +26,17 @@ ClickHouse 本身不会去监控硬件资源的状态。 ClickHouse服务本身具有用于自我状态监视指标。 -要跟踪服务器事件，请观察服务器日志。 请参阅配置文件的\[logger\]（server\_settings/settings.md\#server\_settings-logger）部分。 +要跟踪服务器事件，请观察服务器日志。 请参阅配置文件的 [logger](server-configuration-parameters/settings.md#server_configuration_parameters-logger)部分。 ClickHouse 收集的指标项： - 服务用于计算的资源占用的各种指标。 - 关于查询处理的常见统计信息。 -可以在 [系统。指标](system-tables.md#system_tables-metrics) ，[系统。活动](system-tables.md#system_tables-events) 以及[系统。asynchronous\_metrics](system-tables.md#system_tables-asynchronous_metrics) 等系统表查看所有的指标项。 +可以在 [系统指标](system-tables.md#system_tables-metrics) ，[系统事件](system-tables.md#system_tables-events) 以及[系统异步指标](system-tables.md#system_tables-asynchronous_metrics) 等系统表查看所有的指标项。 可以配置ClickHouse 往 [石墨](https://github.com/graphite-project)导入指标。 参考 [石墨部分](server-configuration-parameters/settings.md#server_configuration_parameters-graphite) 配置文件。在配置指标导出之前，需要参考Graphite[官方教程](https://graphite.readthedocs.io/en/latest/install.html)搭建服务。 此外，您可以通过HTTP API监视服务器可用性。 将HTTP GET请求发送到 `/ping`。 如果服务器可用，它将以 `200 OK` 响应。 -要监视服务器集群的配置中，应设置[max\_replica\_delay\_for\_distributed\_queries](settings/settings.md#settings-max_replica_delay_for_distributed_queries)参数并使用HTTP资源`/replicas_status`。 如果副本可用，并且不延迟在其他副本之后，则对`/replicas_status`的请求将返回200 OK。 如果副本滞后，请求将返回 `503 HTTP_SERVICE_UNAVAILABLE`，包括有关待办事项大小的信息。 +要监视服务器集群的配置，应设置[max\_replica\_delay\_for\_distributed\_queries](settings/settings.md#settings-max_replica_delay_for_distributed_queries)参数并使用HTTP资源`/replicas_status`。 如果副本可用，并且不延迟在其他副本之后，则对`/replicas_status`的请求将返回200 OK。 如果副本滞后，请求将返回 `503 HTTP_SERVICE_UNAVAILABLE`，包括有关待办事项大小的信息。 diff --git a/docs/zh/operations/requirements.md b/docs/zh/operations/requirements.md index c7b61113926..28ac93de37d 100644 --- a/docs/zh/operations/requirements.md +++ b/docs/zh/operations/requirements.md @@ -1,8 +1,6 @@ --- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 44 -toc_title: "\u8981\u6C42" +toc_title: "要求" --- # 要求 {#requirements} @@ -13,20 +11,20 @@ toc_title: "\u8981\u6C42" ClickHouse实现并行数据处理并使用所有可用的硬件资源。 在选择处理器时，考虑到ClickHouse在具有大量内核但时钟速率较低的配置中的工作效率要高于具有较少内核和较高时钟速率的配置。 例如，具有2600MHz的16核心优于具有3600MHz的8核心。 -建议使用 **涡轮增压** 和 **超线程** 技术。 它显着提高了典型工作负载的性能。 +建议使用 **睿频加速** 和 **超线程** 技术。 它显着提高了典型工作负载的性能。 ## RAM {#ram} -我们建议使用至少4GB的RAM来执行非平凡的查询。 ClickHouse服务器可以使用少得多的RAM运行，但它需要处理查询的内存。 +我们建议使用至少4GB的RAM来执行重要的查询。 ClickHouse服务器可以使用少得多的RAM运行，但它需要处理查询的内存。 RAM所需的体积取决于: - 查询的复杂性。 -- 在查询中处理的数据量。 +- 查询中处理的数据量。 要计算所需的RAM体积，您应该估计临时数据的大小 [GROUP BY](../sql-reference/statements/select/group-by.md#select-group-by-clause), [DISTINCT](../sql-reference/statements/select/distinct.md#select-distinct), [JOIN](../sql-reference/statements/select/join.md#select-join) 和您使用的其他操作。 -ClickHouse可以使用外部存储器来存储临时数据。 看 [在外部存储器中分组](../sql-reference/statements/select/group-by.md#select-group-by-in-external-memory) 有关详细信息。 +ClickHouse可以使用外部存储器来存储临时数据。看 [在外部存储器中分组](../sql-reference/statements/select/group-by.md#select-group-by-in-external-memory) 有关详细信息。 ## 交换文件 {#swap-file} @@ -42,20 +40,20 @@ ClickHouse可以使用外部存储器来存储临时数据。 看 [在外部存 您可以采取数据的样本并从中获取行的平均大小。 然后将该值乘以计划存储的行数。 -- 的数据压缩系数。 +- 数据压缩系数。 - 要估计数据压缩系数，请将数据的样本加载到ClickHouse中，并将数据的实际大小与存储的表的大小进行比较。 例如，点击流数据通常被压缩6-10次。 + 要估计数据压缩系数，请将数据的样本加载到ClickHouse中，并将数据的实际大小与存储的表的大小进行比较。 例如，点击流数据通常被压缩6-10倍。 -要计算要存储的最终数据量，请将压缩系数应用于估计的数据量。 如果计划将数据存储在多个副本中，则将估计的卷乘以副本数。 +要计算要存储的最终数据量，请将压缩系数应用于估计的数据量。 如果计划将数据存储在多个副本中，则将估计的量乘以副本数。 ## 网络 {#network} 如果可能的话，使用10G或更高级别的网络。 -网络带宽对于处理具有大量中间数据的分布式查询至关重要。 此外，网络速度会影响复制过程。 +网络带宽对于处理具有大量中间结果数据的分布式查询至关重要。 此外，网络速度会影响复制过程。 ## 软件 {#software} -ClickHouse主要是为Linux系列操作系统开发的。 推荐的Linux发行版是Ubuntu。 该 `tzdata` 软件包应安装在系统中。 +ClickHouse主要是为Linux系列操作系统开发的。 推荐的Linux发行版是Ubuntu。 `tzdata` 软件包应安装在系统中。 ClickHouse也可以在其他操作系统系列中工作。 查看详细信息 [开始](../getting-started/index.md) 文档的部分。 diff --git a/docs/zh/operations/troubleshooting.md b/docs/zh/operations/troubleshooting.md index fcece8c3d6d..d9a00717d7b 100644 --- a/docs/zh/operations/troubleshooting.md +++ b/docs/zh/operations/troubleshooting.md @@ -1,23 +1,21 @@ --- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 46 -toc_title: "\u7591\u96BE\u89E3\u7B54" +toc_title: "常见问题" --- -# 疑难解答 {#troubleshooting} +# 常见问题 {#troubleshooting} -- [安装方式](#troubleshooting-installation-errors) +- [安装](#troubleshooting-installation-errors) - [连接到服务器](#troubleshooting-accepts-no-connections) - [查询处理](#troubleshooting-does-not-process-queries) - [查询处理效率](#troubleshooting-too-slow) -## 安装方式 {#troubleshooting-installation-errors} +## 安装 {#troubleshooting-installation-errors} ### 您无法使用Apt-get从ClickHouse存储库获取Deb软件包 {#you-cannot-get-deb-packages-from-clickhouse-repository-with-apt-get} - 检查防火墙设置。 -- 如果出于任何原因无法访问存储库，请按照以下文件中的描述下载软件包 [开始](../getting-started/index.md) 文章并使用手动安装它们 `sudo dpkg -i ` 指挥部 您还需要 `tzdata` 包。 +- 如果出于任何原因无法访问存储库，请按照[开始](../getting-started/index.md)中的描述下载软件包，并使用命令 `sudo dpkg -i ` 手动安装它们。除此之外你还需要 `tzdata` 包。 ## 连接到服务器 {#troubleshooting-accepts-no-connections} @@ -44,7 +42,7 @@ $ sudo service clickhouse-server start **检查日志** -主日志 `clickhouse-server` 是在 `/var/log/clickhouse-server/clickhouse-server.log` 默认情况下。 +主日志 `clickhouse-server` 默认情况是在 `/var/log/clickhouse-server/clickhouse-server.log` 下。 如果服务器成功启动，您应该看到字符串: @@ -57,13 +55,13 @@ $ sudo service clickhouse-server start 2019.01.11 15:23:25.549505 [ 45 ] {} ExternalDictionaries: Failed reloading 'event2id' external dictionary: Poco::Exception. Code: 1000, e.code() = 111, e.displayText() = Connection refused, e.what() = Connection refused ``` -如果在文件末尾没有看到错误，请从字符串开始查看整个文件: +如果在文件末尾没有看到错误，请从如下字符串开始查看整个文件: ``` text Application: starting up. ``` -如果您尝试启动第二个实例 `clickhouse-server` 在服务器上，您将看到以下日志: +如果您尝试在服务器上启动第二个实例 `clickhouse-server` ，您将看到以下日志: ``` text 2019.01.11 15:25:11.151730 [ 1 ] {} : Starting ClickHouse 19.1.0 with revision 54413 @@ -79,9 +77,9 @@ Revision: 54413 2019.01.11 15:25:11.156716 [ 2 ] {} BaseDaemon: Stop SignalListener thread ``` -**请参阅系统。d日志** +**查看系统日志** -如果你没有找到任何有用的信息 `clickhouse-server` 日志或没有任何日志，您可以查看 `system.d` 使用命令记录: +如果你在 `clickhouse-server` 没有找到任何有用的信息或根本没有任何日志，您可以使用命令查看 `system.d` : ``` bash $ sudo journalctl -u clickhouse-server @@ -99,9 +97,9 @@ $ sudo -u clickhouse /usr/bin/clickhouse-server --config-file /etc/clickhouse-se 检查: -- 码头工人设置。 +- Docker设置。 - 如果您在IPv6网络中的Docker中运行ClickHouse，请确保 `network=host` 已设置。 + 如果您在IPv6网络中的Docker中运行ClickHouse，请确保 `network=host` 被设置。 - 端点设置。 @@ -117,10 +115,10 @@ $ sudo -u clickhouse /usr/bin/clickhouse-server --config-file /etc/clickhouse-se 检查: - - 该 [tcp\_port\_secure](server-configuration-parameters/settings.md#server_configuration_parameters-tcp_port_secure) 设置。 - - 设置 [SSL序列](server-configuration-parameters/settings.md#server_configuration_parameters-openssl). + - [tcp\_port\_secure](server-configuration-parameters/settings.md#server_configuration_parameters-tcp_port_secure) 设置。 + - [SSL证书](server-configuration-parameters/settings.md#server_configuration_parameters-openssl) 设置. - 连接时使用正确的参数。 例如，使用 `port_secure` 参数 `clickhouse_client`. + 连接时使用正确的参数。 例如，使用 `clickhouse_client` 的时候使用 `port_secure` 参数 . - 用户设置。 @@ -135,7 +133,7 @@ $ curl 'http://localhost:8123/' --data-binary "SELECT a" Code: 47, e.displayText() = DB::Exception: Unknown identifier: a. Note that there are no tables (FROM clause) in your query, context: required_names: 'a' source_tables: table_aliases: private_aliases: column_aliases: public_columns: 'a' masked_columns: array_join_columns: source_columns: , e.what() = DB::Exception ``` -如果你开始 `clickhouse-client` 与 `stack-trace` 参数，ClickHouse返回包含错误描述的服务器堆栈跟踪。 +如果你使用 `clickhouse-client` 时设置了 `stack-trace` 参数，ClickHouse返回包含错误描述的服务器堆栈跟踪信息。 您可能会看到一条关于连接中断的消息。 在这种情况下，可以重复查询。 如果每次执行查询时连接中断，请检查服务器日志中是否存在错误。 diff --git a/docs/zh/operations/update.md b/docs/zh/operations/update.md index 186b6c5b315..072b3da98a9 100644 --- a/docs/zh/operations/update.md +++ b/docs/zh/operations/update.md @@ -1,11 +1,9 @@ --- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 47 -toc_title: "\u70B9\u51FB\u66F4\u65B0" +toc_title: "更新" --- -# 点击更新 {#clickhouse-update} +# 更新 {#clickhouse-update} 如果从deb包安装ClickHouse，请在服务器上执行以下命令: @@ -15,6 +13,6 @@ $ sudo apt-get install clickhouse-client clickhouse-server $ sudo service clickhouse-server restart ``` -如果您使用除推荐的deb包之外的其他内容安装ClickHouse，请使用适当的更新方法。 +如果您使用除推荐的deb包之外的其他方式安装ClickHouse，请使用适当的更新方法。 -ClickHouse不支持分布式更新。 该操作应在每个单独的服务器上连续执行。 不要同时更新群集上的所有服务器，否则群集将在一段时间内不可用。 +ClickHouse不支持分布式更新。该操作应在每个单独的服务器上连续执行。不要同时更新群集上的所有服务器，否则群集将在一段时间内不可用。 From 7142e68d7abab49f0d734e907f07ac139c6702ec Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 26 May 2020 08:54:04 +0300 Subject: [PATCH 719/738] fixup --- .../AggregateFunctionEntropy.h | 6 ++--- .../AggregateFunctionGroupUniqArray.h | 9 +++---- .../QuantileExactWeighted.h | 3 +-- src/Common/HashTable/ClearableHashMap.h | 11 ++++++++ src/Common/HashTable/ClearableHashSet.h | 12 +++++++++ src/Common/HashTable/HashMap.h | 11 ++++++++ src/Common/HashTable/HashSet.h | 17 +++++++++++++ src/Common/HashTable/HashTable.h | 21 +++------------- src/Common/SpaceSaving.h | 3 +-- src/Functions/array/arrayDistinct.cpp | 12 ++++----- src/Functions/array/arrayEnumerateExtended.h | 25 ++++++++----------- src/Functions/array/arrayEnumerateRanked.h | 5 ++-- src/Functions/array/arrayIntersect.cpp | 23 ++++++++--------- src/Functions/array/arrayUniq.cpp | 20 +++++++-------- src/Interpreters/tests/hash_map3.cpp | 6 +++-- .../functions_with_hash_tables.xml | 8 +++--- 16 files changed, 108 insertions(+), 84 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionEntropy.h b/src/AggregateFunctions/AggregateFunctionEntropy.h index 59cacd0e9ca..ff233a5ac93 100644 --- a/src/AggregateFunctions/AggregateFunctionEntropy.h +++ b/src/AggregateFunctions/AggregateFunctionEntropy.h @@ -23,12 +23,10 @@ struct EntropyData { using Weight = UInt64; - using HashingMap = HASH_TABLE_WITH_STACK_MEMORY(HashMap, Value, Weight, - HashCRC32, HashTableGrower<4>); + using HashingMap = HashMapWithStackMemory, 4>; /// For the case of pre-hashed values. - using TrivialMap = HASH_TABLE_WITH_STACK_MEMORY(HashMap, Value, Weight, - UInt128TrivialHash, HashTableGrower<4>); + using TrivialMap = HashMapWithStackMemory; using Map = std::conditional_t, TrivialMap, HashingMap>; diff --git a/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h b/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h index af97df209f6..88b1c87f526 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h +++ b/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h @@ -28,8 +28,7 @@ template struct AggregateFunctionGroupUniqArrayData { /// When creating, the hash table must be small. - using Set = HASH_TABLE_WITH_STACK_MEMORY(HashSet, T, DefaultHash, - HashTableGrower<4>); + using Set = HashSetWithStackMemory, 4>; Set value; }; @@ -122,10 +121,10 @@ public: /// Generic implementation, it uses serialized representation as object descriptor. struct AggregateFunctionGroupUniqArrayGenericData { - static constexpr size_t INITIAL_SIZE_DEGREE = 1; /// adjustable + static constexpr size_t INITIAL_SIZE_DEGREE = 3; /// adjustable - using Set = HASH_TABLE_WITH_STACK_MEMORY(HashSetWithSavedHash, - StringRef, StringRefHash, HashTableGrower); + using Set = HashSetWithSavedHashWithStackMemory; Set value; }; diff --git a/src/AggregateFunctions/QuantileExactWeighted.h b/src/AggregateFunctions/QuantileExactWeighted.h index 7731ff980fe..666dbd6b622 100644 --- a/src/AggregateFunctions/QuantileExactWeighted.h +++ b/src/AggregateFunctions/QuantileExactWeighted.h @@ -33,8 +33,7 @@ struct QuantileExactWeighted using Hasher = std::conditional_t, Int128Hash, HashCRC32>; /// When creating, the hash table must be small. - using Map = HASH_TABLE_WITH_STACK_MEMORY(HashMap, UnderlyingType, Weight, - Hasher, HashTableGrower<4>); + using Map = HashMapWithStackMemory; Map map; diff --git a/src/Common/HashTable/ClearableHashMap.h b/src/Common/HashTable/ClearableHashMap.h index 4370f6b6dc7..fda01dcf4bc 100644 --- a/src/Common/HashTable/ClearableHashMap.h +++ b/src/Common/HashTable/ClearableHashMap.h @@ -43,3 +43,14 @@ public: this->m_size = 0; } }; + +template +using ClearableHashMapWithStackMemory = ClearableHashMap< + Key, + Mapped, + Hash, + HashTableGrower, + HashTableAllocatorWithStackMemory< + (1ULL << initial_size_degree) + * sizeof(ClearableHashMapCell)>>; diff --git a/src/Common/HashTable/ClearableHashSet.h b/src/Common/HashTable/ClearableHashSet.h index 824ec9d8e5f..dc057afacd8 100644 --- a/src/Common/HashTable/ClearableHashSet.h +++ b/src/Common/HashTable/ClearableHashSet.h @@ -84,3 +84,15 @@ public: this->m_size = 0; } }; + +template +using ClearableHashSetWithStackMemory = ClearableHashSet< + Key, + Hash, + HashTableGrower, + HashTableAllocatorWithStackMemory< + (1ULL << initial_size_degree) + * sizeof( + ClearableHashTableCell< + Key, + HashTableCell>)>>; diff --git a/src/Common/HashTable/HashMap.h b/src/Common/HashTable/HashMap.h index cdc4a003af8..3ecbd9f263c 100644 --- a/src/Common/HashTable/HashMap.h +++ b/src/Common/HashTable/HashMap.h @@ -239,3 +239,14 @@ template < typename Grower = HashTableGrower<>, typename Allocator = HashTableAllocator> using HashMapWithSavedHash = HashMapTable, Hash, Grower, Allocator>; + +template +using HashMapWithStackMemory = HashMapTable< + Key, + HashMapCellWithSavedHash, + Hash, + HashTableGrower, + HashTableAllocatorWithStackMemory< + (1ULL << initial_size_degree) + * sizeof(HashMapCellWithSavedHash)>>; diff --git a/src/Common/HashTable/HashSet.h b/src/Common/HashTable/HashSet.h index c1970e898a1..c79e05073fc 100644 --- a/src/Common/HashTable/HashSet.h +++ b/src/Common/HashTable/HashSet.h @@ -93,6 +93,14 @@ template > using HashSet = HashSetTable, Hash, Grower, Allocator>; +template +using HashSetWithStackMemory = HashSet< + Key, + Hash, + HashTableGrower, + HashTableAllocatorWithStackMemory< + (1ULL << initial_size_degree) + * sizeof(HashTableCell)>>; template < @@ -102,3 +110,12 @@ template typename Allocator = HashTableAllocator > using HashSetWithSavedHash = HashSetTable, Hash, Grower, Allocator>; + +template +using HashSetWithSavedHashWithStackMemory = HashSetWithSavedHash< + Key, + Hash, + HashTableGrower, + HashTableAllocatorWithStackMemory< + (1ULL << initial_size_degree) + * sizeof(HashSetCellWithSavedHash)>>; diff --git a/src/Common/HashTable/HashTable.h b/src/Common/HashTable/HashTable.h index 2f7272ca8ec..9d66c7b7a89 100644 --- a/src/Common/HashTable/HashTable.h +++ b/src/Common/HashTable/HashTable.h @@ -208,6 +208,7 @@ struct HashTableGrower /// The state of this structure is enough to get the buffer size of the hash table. UInt8 size_degree = initial_size_degree; + static constexpr auto initial_count = 1ULL << initial_size_degree; /// The size of the hash table in the cells. size_t bufSize() const { return 1ULL << size_degree; } @@ -255,6 +256,7 @@ struct HashTableGrower template struct HashTableFixedGrower { + static constexpr auto initial_count = 1ULL << key_bits; size_t bufSize() const { return 1ULL << key_bits; } size_t place(size_t x) const { return x; } /// You could write __builtin_unreachable(), but the compiler does not optimize everything, and it turns out less efficiently. @@ -308,14 +310,6 @@ struct ZeroValueStorage const Cell * zeroValue() const { return nullptr; } }; -// These templates give the initial hash table size, so that we can check -// that it is in sync with initial allocator size. -template -constexpr size_t growerInitialCount = 0; - -template -constexpr size_t growerInitialCount> - = 1ULL << initial_size_degree; // The HashTable template @@ -337,7 +331,7 @@ public: // Export the initial buffer sizes for the ease of using allocators with // inline memory. static constexpr size_t initial_buffer_bytes - = growerInitialCount * sizeof(Cell); + = Grower::initial_count * sizeof(Cell); // If we use an allocator with inline memory, check that the initial // size of the hash table is in sync with the amount of this memory. @@ -1096,12 +1090,3 @@ public: #endif }; -// A helper macro that declares hash table with allocator with stack memory, -// and the initial size of the allocator is in sync with initial size of the -// hash table. -#define HASH_TABLE_WITH_STACK_MEMORY(HASH_TABLE_VARIANT, ...) \ - HASH_TABLE_VARIANT<__VA_ARGS__, \ - HashTableAllocatorWithStackMemory< \ - HASH_TABLE_VARIANT<__VA_ARGS__>::initial_buffer_bytes \ - > \ - > diff --git a/src/Common/SpaceSaving.h b/src/Common/SpaceSaving.h index 47b49add060..56063340240 100644 --- a/src/Common/SpaceSaving.h +++ b/src/Common/SpaceSaving.h @@ -378,8 +378,7 @@ private: counter_map[counter->key] = counter; } - using CounterMap = HASH_TABLE_WITH_STACK_MEMORY(HashMap, - TKey, Counter *, Hash, HashTableGrower<4>); + using CounterMap = HashMapWithStackMemory; CounterMap counter_map; std::vector counter_list; diff --git a/src/Functions/array/arrayDistinct.cpp b/src/Functions/array/arrayDistinct.cpp index d3da77e43d2..bb26567e8a5 100644 --- a/src/Functions/array/arrayDistinct.cpp +++ b/src/Functions/array/arrayDistinct.cpp @@ -153,8 +153,8 @@ bool FunctionArrayDistinct::executeNumber( if (nullable_col) src_null_map = &nullable_col->getNullMapData(); - using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashSet, - T, DefaultHash, HashTableGrower); + using Set = ClearableHashSetWithStackMemory, + INITIAL_SIZE_DEGREE>; Set set; @@ -199,8 +199,8 @@ bool FunctionArrayDistinct::executeString( ColumnString & res_data_column_string = typeid_cast(res_data_col); - using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashSet, - StringRef, StringRefHash, HashTableGrower); + using Set = ClearableHashSetWithStackMemory; const PaddedPODArray * src_null_map = nullptr; @@ -245,8 +245,8 @@ void FunctionArrayDistinct::executeHashed( ColumnArray::Offsets & res_offsets, const ColumnNullable * nullable_col) { - using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashSet, - UInt128, UInt128TrivialHash, HashTableGrower); + using Set = ClearableHashSetWithStackMemory; const PaddedPODArray * src_null_map = nullptr; diff --git a/src/Functions/array/arrayEnumerateExtended.h b/src/Functions/array/arrayEnumerateExtended.h index e97766c5827..e2f83ae8826 100644 --- a/src/Functions/array/arrayEnumerateExtended.h +++ b/src/Functions/array/arrayEnumerateExtended.h @@ -64,45 +64,40 @@ private: template struct MethodOneNumber { - using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap, - T, UInt32, DefaultHash, - HashTableGrower); + using Set = ClearableHashMapWithStackMemory, + INITIAL_SIZE_DEGREE>; using Method = ColumnsHashing::HashMethodOneNumber; }; struct MethodString { - using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap, - StringRef, UInt32, StringRefHash, - HashTableGrower); + using Set = ClearableHashMapWithStackMemory; using Method = ColumnsHashing::HashMethodString; }; struct MethodFixedString { - using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap, - StringRef, UInt32, StringRefHash, - HashTableGrower); + using Set = ClearableHashMapWithStackMemory; using Method = ColumnsHashing::HashMethodFixedString; }; struct MethodFixed { - using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap, - UInt128, UInt32, UInt128HashCRC32, - HashTableGrower); + using Set = ClearableHashMapWithStackMemory; using Method = ColumnsHashing::HashMethodKeysFixed; }; struct MethodHashed { - using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap, - UInt128, UInt32, UInt128TrivialHash, - HashTableGrower); + using Set = ClearableHashMapWithStackMemory; using Method = ColumnsHashing::HashMethodHashed; }; diff --git a/src/Functions/array/arrayEnumerateRanked.h b/src/Functions/array/arrayEnumerateRanked.h index 1700260dd9f..c021b5fbfd1 100644 --- a/src/Functions/array/arrayEnumerateRanked.h +++ b/src/Functions/array/arrayEnumerateRanked.h @@ -308,9 +308,8 @@ void FunctionArrayEnumerateRankedExtended::executeMethodImpl( const size_t depth_to_look = arrays_depths.max_array_depth; const auto & offsets = *offsets_by_depth[depth_to_look - 1]; - using Map = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap, - UInt128, UInt32, UInt128TrivialHash, - HashTableGrower); + using Map = ClearableHashMapWithStackMemory; Map indices; diff --git a/src/Functions/array/arrayIntersect.cpp b/src/Functions/array/arrayIntersect.cpp index acd162974c9..3525914b0d3 100644 --- a/src/Functions/array/arrayIntersect.cpp +++ b/src/Functions/array/arrayIntersect.cpp @@ -418,18 +418,15 @@ void FunctionArrayIntersect::executeImpl(Block & block, const ColumnNumbers & ar TypeListNativeNumbers::forEach(NumberExecutor(arrays, not_nullable_nested_return_type, result_column)); TypeListDecimalNumbers::forEach(DecimalExecutor(arrays, not_nullable_nested_return_type, result_column)); - using DateMap = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap, - DataTypeDate::FieldType, size_t, DefaultHash, - HashTableGrower); + using DateMap = ClearableHashMapWithStackMemory, INITIAL_SIZE_DEGREE>; - using DateTimeMap = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap, + using DateTimeMap = ClearableHashMapWithStackMemory< DataTypeDateTime::FieldType, size_t, - DefaultHash, - HashTableGrower); + DefaultHash, INITIAL_SIZE_DEGREE>; - using StringMap = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap, - StringRef, size_t, StringRefHash, - HashTableGrower); + using StringMap = ClearableHashMapWithStackMemory; if (!result_column) { @@ -457,8 +454,8 @@ void FunctionArrayIntersect::executeImpl(Block & block, const ColumnNumbers & ar template void FunctionArrayIntersect::NumberExecutor::operator()() { - using Map = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap, - T, size_t, DefaultHash, HashTableGrower); + using Map = ClearableHashMapWithStackMemory, + INITIAL_SIZE_DEGREE>; if (!result && typeid_cast *>(data_type.get())) result = execute, true>(arrays, ColumnVector::create()); @@ -467,8 +464,8 @@ void FunctionArrayIntersect::NumberExecutor::operator()() template void FunctionArrayIntersect::DecimalExecutor::operator()() { - using Map = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap, - T, size_t, DefaultHash, HashTableGrower); + using Map = ClearableHashMapWithStackMemory, + INITIAL_SIZE_DEGREE); if (!result) if (auto * decimal = typeid_cast *>(data_type.get())) diff --git a/src/Functions/array/arrayUniq.cpp b/src/Functions/array/arrayUniq.cpp index a83c535d88d..d9a44f6d8d3 100644 --- a/src/Functions/array/arrayUniq.cpp +++ b/src/Functions/array/arrayUniq.cpp @@ -66,40 +66,40 @@ private: template struct MethodOneNumber { - using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashSet, - T, DefaultHash, HashTableGrower); + using Set = ClearableHashSetWithStackMemory, + INITIAL_SIZE_DEGREE>; using Method = ColumnsHashing::HashMethodOneNumber; }; struct MethodString { - using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashSet, - StringRef, StringRefHash, HashTableGrower); + using Set = ClearableHashSetWithStackMemory; using Method = ColumnsHashing::HashMethodString; }; struct MethodFixedString { - using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashSet, - StringRef, StringRefHash, HashTableGrower); + using Set = ClearableHashSetWithStackMemory; using Method = ColumnsHashing::HashMethodFixedString; }; struct MethodFixed { - using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashSet, - UInt128, UInt128HashCRC32, HashTableGrower); + using Set = ClearableHashSetWithStackMemory; using Method = ColumnsHashing::HashMethodKeysFixed; }; struct MethodHashed { - using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashSet, - UInt128, UInt128TrivialHash, HashTableGrower); + using Set = ClearableHashSetWithStackMemory; using Method = ColumnsHashing::HashMethodHashed; }; diff --git a/src/Interpreters/tests/hash_map3.cpp b/src/Interpreters/tests/hash_map3.cpp index 804b084b2a1..1b4ce8eac77 100644 --- a/src/Interpreters/tests/hash_map3.cpp +++ b/src/Interpreters/tests/hash_map3.cpp @@ -61,11 +61,13 @@ struct Grower : public HashTableGrower<2> int main(int, char **) { - using Map = HASH_TABLE_WITH_STACK_MEMORY(HashMapWithDump, + using Map = HashMapWithDump< StringRef, UInt64, SimpleHash, - Grower); + Grower, + HashTableAllocatorWithStackMemory< + 4 * sizeof(HashMapCell)>>; Map map; diff --git a/tests/performance/functions_with_hash_tables.xml b/tests/performance/functions_with_hash_tables.xml index 3ff2633c4e7..b247f0e4f6f 100644 --- a/tests/performance/functions_with_hash_tables.xml +++ b/tests/performance/functions_with_hash_tables.xml @@ -1,8 +1,8 @@ - select arrayUniq(range(1 + (number % 10) * 10)) from numbers(1000000) format Null - select arrayDistinct(range(1 + (number % 10) * 10)) from numbers(1000000) format Null - select arrayEnumerateUniq(range(1 + (number % 10) * 10)) from numbers(1000000) format Null + select arrayUniq(range(1 + (number % 100) * 10)) from numbers(1000000) format Null + select arrayDistinct(range(1 + (number % 100) * 10)) from numbers(1000000) format Null + select arrayEnumerateUniq(range(1 + (number % 100) * 10)) from numbers(1000000) format Null select arrayIntersect(range((1 + number % 10)), range(1, (1 + number % 10) + 1)) from numbers(1000000) format Null - select groupUniqArray(number / 10) from numbers(1000 * 1000) group by number / 1000 format Null + select groupUniqArray(rand() % 100) from numbers(1000 * 1000) group by number / 1000 format Null select entropy(number / 10) from numbers(1000 * 1000) group by number / 1000 format Null From bf941755f2e776304d0c318d8d0aa6dc6dc068b3 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 26 May 2020 09:00:51 +0300 Subject: [PATCH 720/738] fixup --- src/Common/HashTable/HashTable.h | 8 ++------ src/Functions/array/arrayIntersect.cpp | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/Common/HashTable/HashTable.h b/src/Common/HashTable/HashTable.h index 9d66c7b7a89..528e719c05b 100644 --- a/src/Common/HashTable/HashTable.h +++ b/src/Common/HashTable/HashTable.h @@ -328,13 +328,10 @@ class HashTable : protected ZeroValueStorage /// empty base optimization { public: - // Export the initial buffer sizes for the ease of using allocators with - // inline memory. - static constexpr size_t initial_buffer_bytes - = Grower::initial_count * sizeof(Cell); - // If we use an allocator with inline memory, check that the initial // size of the hash table is in sync with the amount of this memory. + static constexpr size_t initial_buffer_bytes + = Grower::initial_count * sizeof(Cell); static_assert(allocatorInitialBytes == 0 || allocatorInitialBytes == initial_buffer_bytes); @@ -1089,4 +1086,3 @@ public: } #endif }; - diff --git a/src/Functions/array/arrayIntersect.cpp b/src/Functions/array/arrayIntersect.cpp index 3525914b0d3..24db3c0cd08 100644 --- a/src/Functions/array/arrayIntersect.cpp +++ b/src/Functions/array/arrayIntersect.cpp @@ -465,7 +465,7 @@ template void FunctionArrayIntersect::DecimalExecutor::operator()() { using Map = ClearableHashMapWithStackMemory, - INITIAL_SIZE_DEGREE); + INITIAL_SIZE_DEGREE>; if (!result) if (auto * decimal = typeid_cast *>(data_type.get())) From 27df8d5aca9ee12a82a6f83337fd65bfd256db4d Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 26 May 2020 09:06:46 +0300 Subject: [PATCH 721/738] fixup --- tests/performance/functions_with_hash_tables.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/performance/functions_with_hash_tables.xml b/tests/performance/functions_with_hash_tables.xml index b247f0e4f6f..72570a5925c 100644 --- a/tests/performance/functions_with_hash_tables.xml +++ b/tests/performance/functions_with_hash_tables.xml @@ -1,8 +1,8 @@ - select arrayUniq(range(1 + (number % 100) * 10)) from numbers(1000000) format Null - select arrayDistinct(range(1 + (number % 100) * 10)) from numbers(1000000) format Null - select arrayEnumerateUniq(range(1 + (number % 100) * 10)) from numbers(1000000) format Null - select arrayIntersect(range((1 + number % 10)), range(1, (1 + number % 10) + 1)) from numbers(1000000) format Null + select arrayUniq(range(1 + (number % 100) * 10)) from numbers(100000) format Null + select arrayDistinct(range(1 + (number % 100) * 10)) from numbers(10000) format Null + select arrayEnumerateUniq(range(1 + (number % 100) * 10)) from numbers(100000) format Null + select arrayIntersect(range((1 + number % 100)), range(1, (1 + number % 100) + 1)) from numbers(100000) format Null select groupUniqArray(rand() % 100) from numbers(1000 * 1000) group by number / 1000 format Null select entropy(number / 10) from numbers(1000 * 1000) group by number / 1000 format Null From 1f9e4996c79d2afdc0c15c13f874355d94e5a680 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 26 May 2020 10:51:53 +0300 Subject: [PATCH 722/738] Fix PipelineExecutor::executeStepImpl --- src/Processors/Executors/PipelineExecutor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/Executors/PipelineExecutor.cpp b/src/Processors/Executors/PipelineExecutor.cpp index 4552262040b..bd7a83058ba 100644 --- a/src/Processors/Executors/PipelineExecutor.cpp +++ b/src/Processors/Executors/PipelineExecutor.cpp @@ -570,7 +570,7 @@ void PipelineExecutor::executeStepImpl(size_t thread_num, size_t num_threads, st { /// First, find any processor to execute. /// Just travers graph and prepare any processor. - while (!finished) + while (!finished && state != nullptr) { { std::unique_lock lock(task_queue_mutex); From 5780240e884c9c3915721f159b3c5fd33b40111d Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 26 May 2020 12:46:04 +0300 Subject: [PATCH 723/738] Fix primry key check --- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 4 ++-- .../integration/test_version_update_after_mutation/test.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 6e048a956da..78548cdddb2 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -841,7 +841,7 @@ void IMergeTreeDataPart::checkConsistencyBase() const if (!checksums.empty()) { - if (storage.hasPrimaryKey() && !checksums.files.count("primary.idx")) + if (storage.isPrimaryKeyDefined() && !checksums.files.count("primary.idx")) throw Exception("No checksum for primary.idx", ErrorCodes::NO_FILE_IN_DATA_PART); if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) @@ -875,7 +875,7 @@ void IMergeTreeDataPart::checkConsistencyBase() const }; /// Check that the primary key index is not empty. - if (storage.hasPrimaryKey()) + if (storage.isPrimaryKeyDefined()) check_file_not_empty(volume->getDisk(), path + "primary.idx"); if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) diff --git a/tests/integration/test_version_update_after_mutation/test.py b/tests/integration/test_version_update_after_mutation/test.py index 3f2063bdddb..9e3471175fa 100644 --- a/tests/integration/test_version_update_after_mutation/test.py +++ b/tests/integration/test_version_update_after_mutation/test.py @@ -6,9 +6,9 @@ from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_zookeeper=True, image='yandex/clickhouse-server:20.1.6.3', with_installed_binary=True, stay_alive=True) -node2 = cluster.add_instance('node2', with_zookeeper=True, image='yandex/clickhouse-server:20.1.6.3', with_installed_binary=True, stay_alive=True) -node3 = cluster.add_instance('node3', with_zookeeper=True, image='yandex/clickhouse-server:20.1.6.3', with_installed_binary=True, stay_alive=True) +node1 = cluster.add_instance('node1', with_zookeeper=True, image='yandex/clickhouse-server:20.1.10.70', with_installed_binary=True, stay_alive=True) +node2 = cluster.add_instance('node2', with_zookeeper=True, image='yandex/clickhouse-server:20.1.10.70', with_installed_binary=True, stay_alive=True) +node3 = cluster.add_instance('node3', with_zookeeper=True, image='yandex/clickhouse-server:20.1.10.70', with_installed_binary=True, stay_alive=True) @pytest.fixture(scope="module") def start_cluster(): From 54f50745ec40c3841862640aa6711cfab0df85fb Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 26 May 2020 13:19:56 +0300 Subject: [PATCH 724/738] Fix StrictResizeProcessor. --- src/Processors/ResizeProcessor.cpp | 26 ++++++++++++++++++++++---- src/Processors/ResizeProcessor.h | 3 +++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/Processors/ResizeProcessor.cpp b/src/Processors/ResizeProcessor.cpp index 3fcf4281105..d652a342150 100644 --- a/src/Processors/ResizeProcessor.cpp +++ b/src/Processors/ResizeProcessor.cpp @@ -347,11 +347,16 @@ IProcessor::Status StrictResizeProcessor::prepare(const PortNumbers & updated_in auto & waiting_output = output_ports[input_with_data.waiting_output]; - if (waiting_output.status != OutputStatus::NeedData) - throw Exception("Invalid status for associated output.", ErrorCodes::LOGICAL_ERROR); + if (waiting_output.status == OutputStatus::NotActive) + throw Exception("Invalid status NotActive for associated output.", ErrorCodes::LOGICAL_ERROR); - waiting_output.port->pushData(input_with_data.port->pullData(/* set_not_needed = */ true)); - waiting_output.status = OutputStatus::NotActive; + if (waiting_output.status != OutputStatus::Finished) + { + waiting_output.port->pushData(input_with_data.port->pullData(/* set_not_needed = */ true)); + waiting_output.status = OutputStatus::NotActive; + } + else + abandoned_chunks.emplace_back(input_with_data.port->pullData(/* set_not_needed = */ true)); if (input_with_data.port->isFinished()) { @@ -370,6 +375,18 @@ IProcessor::Status StrictResizeProcessor::prepare(const PortNumbers & updated_in return Status::Finished; } + /// Process abandoned chunks if any. + while (!abandoned_chunks.empty() && !waiting_outputs.empty()) + { + auto & waiting_output = output_ports[waiting_outputs.front()]; + waiting_outputs.pop(); + + waiting_output.port->pushData(std::move(abandoned_chunks.back())); + abandoned_chunks.pop_back(); + + waiting_output.status = OutputStatus::NotActive; + } + /// Enable more inputs if needed. while (!disabled_input_ports.empty() && !waiting_outputs.empty()) { @@ -383,6 +400,7 @@ IProcessor::Status StrictResizeProcessor::prepare(const PortNumbers & updated_in waiting_outputs.pop(); } + /// Close all other waiting for data outputs (there is no corresponding input for them). while (!waiting_outputs.empty()) { auto & output = output_ports[waiting_outputs.front()]; diff --git a/src/Processors/ResizeProcessor.h b/src/Processors/ResizeProcessor.h index 1f364ffcf15..5c9660af113 100644 --- a/src/Processors/ResizeProcessor.h +++ b/src/Processors/ResizeProcessor.h @@ -128,6 +128,9 @@ private: std::vector input_ports; std::vector output_ports; + /// This field contained chunks which were read for output which had became finished while reading was happening. + /// They will be pushed to any next waiting output. + std::vector abandoned_chunks; }; } From 709f7058860dc674664ea1ad0202a91b9a702a62 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 26 May 2020 13:26:38 +0300 Subject: [PATCH 725/738] Fix PipelineExecutor::executeStepImpl --- src/Processors/Executors/PipelineExecutor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/Executors/PipelineExecutor.cpp b/src/Processors/Executors/PipelineExecutor.cpp index bd7a83058ba..76d06b6f128 100644 --- a/src/Processors/Executors/PipelineExecutor.cpp +++ b/src/Processors/Executors/PipelineExecutor.cpp @@ -570,7 +570,7 @@ void PipelineExecutor::executeStepImpl(size_t thread_num, size_t num_threads, st { /// First, find any processor to execute. /// Just travers graph and prepare any processor. - while (!finished && state != nullptr) + while (!finished && state == nullptr) { { std::unique_lock lock(task_queue_mutex); From b16936ba9dfd480ef9fa42538dee6f1751c11685 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 26 May 2020 13:37:10 +0300 Subject: [PATCH 726/738] Added test. --- .../queries/0_stateless/001283_strict_resize_bug.reference | 1 + tests/queries/0_stateless/001283_strict_resize_bug.sql | 7 +++++++ 2 files changed, 8 insertions(+) create mode 100644 tests/queries/0_stateless/001283_strict_resize_bug.reference create mode 100644 tests/queries/0_stateless/001283_strict_resize_bug.sql diff --git a/tests/queries/0_stateless/001283_strict_resize_bug.reference b/tests/queries/0_stateless/001283_strict_resize_bug.reference new file mode 100644 index 00000000000..e1fe7af425d --- /dev/null +++ b/tests/queries/0_stateless/001283_strict_resize_bug.reference @@ -0,0 +1 @@ +49999995000000 diff --git a/tests/queries/0_stateless/001283_strict_resize_bug.sql b/tests/queries/0_stateless/001283_strict_resize_bug.sql new file mode 100644 index 00000000000..f462f50c61f --- /dev/null +++ b/tests/queries/0_stateless/001283_strict_resize_bug.sql @@ -0,0 +1,7 @@ +drop table if exists num_10m; +create table num_10m (number UInt64) engine = MergeTree order by tuple(); +insert into num_10m select * from numbers(10000000); + +select * from (select sum(number) from num_10m union all select sum(number) from num_10m) limit 1 settings max_block_size = 1024; + +drop table if exists num_1m; From 28a2cc0a82d021277878dd305f61f955266c4faf Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Fri, 8 May 2020 17:59:58 +0300 Subject: [PATCH 727/738] Functions {to,from}UnixTimestamp64{Milli,Micro,Nano} * Family of functions to convert DateTime64 to Int64 and back, usefull for imporintg/exporting raw timestamp values of certain precision. * Docs * Tests * Fixed ya.make --- .../functions/type-conversion-functions.md | 86 ++++++++++ src/Columns/ColumnDecimal.h | 2 + src/Common/PODArray.h | 14 +- src/Common/UInt128.h | 2 +- src/Core/Types.h | 32 ++-- src/DataTypes/DataTypeDateTime64.h | 4 +- src/DataTypes/DataTypeNumberBase.h | 7 +- src/DataTypes/DataTypeString.h | 3 +- src/DataTypes/IDataType.h | 26 ++- src/Functions/FunctionUnixTimestamp64.h | 148 ++++++++++++++++++ src/Functions/fromUnixTimestamp64Micro.cpp | 20 +++ src/Functions/fromUnixTimestamp64Milli.cpp | 20 +++ src/Functions/fromUnixTimestamp64Nano.cpp | 20 +++ src/Functions/registerFunctions.cpp | 2 + .../registerFunctionsUnixTimestamp64.cpp | 25 +++ src/Functions/toUnixTimestamp64Micro.cpp | 20 +++ src/Functions/toUnixTimestamp64Milli.cpp | 20 +++ src/Functions/toUnixTimestamp64Nano.cpp | 20 +++ src/Functions/ya.make | 8 + .../01277_fromUnixTimestamp64.reference | 5 + .../0_stateless/01277_fromUnixTimestamp64.sql | 45 ++++++ .../01277_toUnixTimestamp64.reference | 8 + .../0_stateless/01277_toUnixTimestamp64.sql | 33 ++++ ...77_unixTimestamp64_compatibility.reference | 9 ++ .../01277_unixTimestamp64_compatibility.sql | 64 ++++++++ 25 files changed, 614 insertions(+), 29 deletions(-) create mode 100644 src/Functions/FunctionUnixTimestamp64.h create mode 100644 src/Functions/fromUnixTimestamp64Micro.cpp create mode 100644 src/Functions/fromUnixTimestamp64Milli.cpp create mode 100644 src/Functions/fromUnixTimestamp64Nano.cpp create mode 100644 src/Functions/registerFunctionsUnixTimestamp64.cpp create mode 100644 src/Functions/toUnixTimestamp64Micro.cpp create mode 100644 src/Functions/toUnixTimestamp64Milli.cpp create mode 100644 src/Functions/toUnixTimestamp64Nano.cpp create mode 100644 tests/queries/0_stateless/01277_fromUnixTimestamp64.reference create mode 100644 tests/queries/0_stateless/01277_fromUnixTimestamp64.sql create mode 100644 tests/queries/0_stateless/01277_toUnixTimestamp64.reference create mode 100644 tests/queries/0_stateless/01277_toUnixTimestamp64.sql create mode 100644 tests/queries/0_stateless/01277_unixTimestamp64_compatibility.reference create mode 100644 tests/queries/0_stateless/01277_unixTimestamp64_compatibility.sql diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md index 61d5690bec4..5a3bb264a84 100644 --- a/docs/en/sql-reference/functions/type-conversion-functions.md +++ b/docs/en/sql-reference/functions/type-conversion-functions.md @@ -568,4 +568,90 @@ Result: └───────────────────────┘ ``` + +## toUnixTimestamp64Milli +## toUnixTimestamp64Micro +## toUnixTimestamp64Nano + +Converts a `DateTime64` to a `Int64` value with fixed sub-second precision. Input value is scaled up or down appropriately depending on it precision. Please note that output value is a timestamp in UTC, not in timezone of `DateTime64`. + +**Syntax** + +``` sql +toUnixTimestamp64Milli(value) +``` + +**Parameters** + +- `value` — DateTime64 value with any precision. + +**Returned value** + +- `value` converted to the `Int64` data type. + +**Examples** + +Query: + +``` sql +WITH toDateTime64('2019-09-16 19:20:12.345678910', 6) AS dt64 +SELECT toUnixTimestamp64Milli(dt64) +``` + +Result: + +``` text +┌─toUnixTimestamp64Milli(dt64)─┐ +│ 1568650812345 │ +└──────────────────────────────┘ +``` + +``` sql +WITH toDateTime64('2019-09-16 19:20:12.345678910', 6) AS dt64 +SELECT toUnixTimestamp64Nano(dt64) +``` + +Result: + +``` text +┌─toUnixTimestamp64Nano(dt64)─┐ +│ 1568650812345678000 │ +└─────────────────────────────┘ +``` + +## fromUnixTimestamp64Milli +## fromUnixTimestamp64Micro +## fromUnixTimestamp64Nano + +Converts an `Int64` to a `DateTime64` value with fixed sub-second precision and optional timezone. Input value is scaled up or down appropriately depending on it's precision. Please note that input value is treated as UTC timestamp, not timestamp at given (or implicit) timezone. + +**Syntax** + +``` sql +fromUnixTimestamp64Milli(value [, ti]) +``` + +**Parameters** + +- `value` — `Int64` value with any precision. +- `timezone` — `String` (optional) timezone name of the result. + +**Returned value** + +- `value` converted to the `DateTime64` data type. + +**Examples** + +``` sql +WITH CAST(1234567891011, 'Int64') AS i64 +SELECT fromUnixTimestamp64Milli(i64, 'UTC') +``` + +``` text +┌─fromUnixTimestamp64Milli(i64, 'UTC')─┐ +│ 2009-02-13 23:31:31.011 │ +└──────────────────────────────────────┘ +``` + + [Original article](https://clickhouse.tech/docs/en/query_language/functions/type_conversion_functions/) diff --git a/src/Columns/ColumnDecimal.h b/src/Columns/ColumnDecimal.h index 9c2e232568f..86357dc8be7 100644 --- a/src/Columns/ColumnDecimal.h +++ b/src/Columns/ColumnDecimal.h @@ -153,6 +153,8 @@ public: const T & getElement(size_t n) const { return data[n]; } T & getElement(size_t n) { return data[n]; } + UInt32 getScale() const {return scale;} + protected: Container data; UInt32 scale; diff --git a/src/Common/PODArray.h b/src/Common/PODArray.h index 1153e700e8b..acbcb73dc2c 100644 --- a/src/Common/PODArray.h +++ b/src/Common/PODArray.h @@ -224,13 +224,19 @@ public: } template - void push_back_raw(const char * ptr, TAllocatorParams &&... allocator_params) + void push_back_raw(const void * ptr, TAllocatorParams &&... allocator_params) + { + push_back_raw_many(1, ptr, std::forward(allocator_params)...); + } + + template + void push_back_raw_many(size_t number_of_items, const void * ptr, TAllocatorParams &&... allocator_params) { if (unlikely(c_end == c_end_of_storage)) - reserveForNextSize(std::forward(allocator_params)...); + reserve(number_of_items, std::forward(allocator_params)...); - memcpy(c_end, ptr, ELEMENT_SIZE); - c_end += byte_size(1); + memcpy(c_end, ptr, ELEMENT_SIZE * number_of_items); + c_end += byte_size(number_of_items); } void protect() diff --git a/src/Common/UInt128.h b/src/Common/UInt128.h index b60de11552a..336edfcc56f 100644 --- a/src/Common/UInt128.h +++ b/src/Common/UInt128.h @@ -72,7 +72,7 @@ template bool inline operator<= (T a, const UInt128 b) { return UIn template bool inline operator< (T a, const UInt128 b) { return UInt128(a) < b; } template <> inline constexpr bool IsNumber = true; -template <> struct TypeName { static const char * get() { return "UInt128"; } }; +template <> struct TypeName { static constexpr const char * get() { return "UInt128"; } }; template <> struct TypeId { static constexpr const TypeIndex value = TypeIndex::UInt128; }; struct UInt128Hash diff --git a/src/Core/Types.h b/src/Core/Types.h index 208da48797e..2c2293b4658 100644 --- a/src/Core/Types.h +++ b/src/Core/Types.h @@ -85,17 +85,17 @@ template <> inline constexpr bool IsNumber = true; template struct TypeName; -template <> struct TypeName { static const char * get() { return "UInt8"; } }; -template <> struct TypeName { static const char * get() { return "UInt16"; } }; -template <> struct TypeName { static const char * get() { return "UInt32"; } }; -template <> struct TypeName { static const char * get() { return "UInt64"; } }; -template <> struct TypeName { static const char * get() { return "Int8"; } }; -template <> struct TypeName { static const char * get() { return "Int16"; } }; -template <> struct TypeName { static const char * get() { return "Int32"; } }; -template <> struct TypeName { static const char * get() { return "Int64"; } }; -template <> struct TypeName { static const char * get() { return "Float32"; } }; -template <> struct TypeName { static const char * get() { return "Float64"; } }; -template <> struct TypeName { static const char * get() { return "String"; } }; +template <> struct TypeName { static constexpr const char * get() { return "UInt8"; } }; +template <> struct TypeName { static constexpr const char * get() { return "UInt16"; } }; +template <> struct TypeName { static constexpr const char * get() { return "UInt32"; } }; +template <> struct TypeName { static constexpr const char * get() { return "UInt64"; } }; +template <> struct TypeName { static constexpr const char * get() { return "Int8"; } }; +template <> struct TypeName { static constexpr const char * get() { return "Int16"; } }; +template <> struct TypeName { static constexpr const char * get() { return "Int32"; } }; +template <> struct TypeName { static constexpr const char * get() { return "Int64"; } }; +template <> struct TypeName { static constexpr const char * get() { return "Float32"; } }; +template <> struct TypeName { static constexpr const char * get() { return "Float64"; } }; +template <> struct TypeName { static constexpr const char * get() { return "String"; } }; template struct TypeId; template <> struct TypeId { static constexpr const TypeIndex value = TypeIndex::UInt8; }; @@ -115,7 +115,7 @@ using Strings = std::vector; using Int128 = __int128; template <> inline constexpr bool IsNumber = true; -template <> struct TypeName { static const char * get() { return "Int128"; } }; +template <> struct TypeName { static constexpr const char * get() { return "Int128"; } }; template <> struct TypeId { static constexpr const TypeIndex value = TypeIndex::Int128; }; /// Own FieldType for Decimal. @@ -161,9 +161,9 @@ using Decimal128 = Decimal; using DateTime64 = Decimal64; -template <> struct TypeName { static const char * get() { return "Decimal32"; } }; -template <> struct TypeName { static const char * get() { return "Decimal64"; } }; -template <> struct TypeName { static const char * get() { return "Decimal128"; } }; +template <> struct TypeName { static constexpr const char * get() { return "Decimal32"; } }; +template <> struct TypeName { static constexpr const char * get() { return "Decimal64"; } }; +template <> struct TypeName { static constexpr const char * get() { return "Decimal128"; } }; template <> struct TypeId { static constexpr const TypeIndex value = TypeIndex::Decimal32; }; template <> struct TypeId { static constexpr const TypeIndex value = TypeIndex::Decimal64; }; @@ -183,7 +183,7 @@ template <> inline Int32 Decimal32::getScaleMultiplier(UInt32 scale) { return co template <> inline Int64 Decimal64::getScaleMultiplier(UInt32 scale) { return common::exp10_i64(scale); } template <> inline Int128 Decimal128::getScaleMultiplier(UInt32 scale) { return common::exp10_i128(scale); } -inline const char * getTypeName(TypeIndex idx) +inline constexpr const char * getTypeName(TypeIndex idx) { switch (idx) { diff --git a/src/DataTypes/DataTypeDateTime64.h b/src/DataTypes/DataTypeDateTime64.h index 85b77d88ef4..b575e9d81c1 100644 --- a/src/DataTypes/DataTypeDateTime64.h +++ b/src/DataTypes/DataTypeDateTime64.h @@ -17,7 +17,9 @@ class DataTypeDateTime64 final : public DataTypeDecimalBase, public { public: static constexpr UInt8 default_scale = 3; + static constexpr auto family_name = "DateTime64"; + static constexpr auto type_id = TypeIndex::DateTime64; explicit DataTypeDateTime64(UInt32 scale_, const std::string & time_zone_name = ""); @@ -26,7 +28,7 @@ public: const char * getFamilyName() const override { return family_name; } std::string doGetName() const override; - TypeIndex getTypeId() const override { return TypeIndex::DateTime64; } + TypeIndex getTypeId() const override { return type_id; } void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override; void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; diff --git a/src/DataTypes/DataTypeNumberBase.h b/src/DataTypes/DataTypeNumberBase.h index fb752ad5329..0390da2cb6f 100644 --- a/src/DataTypes/DataTypeNumberBase.h +++ b/src/DataTypes/DataTypeNumberBase.h @@ -19,12 +19,15 @@ class DataTypeNumberBase : public DataTypeWithSimpleSerialization public: static constexpr bool is_parametric = false; + using FieldType = T; + static constexpr auto type_id = TypeId::value; + static constexpr auto family_name = TypeName::get(); using ColumnType = ColumnVector; - const char * getFamilyName() const override { return TypeName::get(); } - TypeIndex getTypeId() const override { return TypeId::value; } + const char * getFamilyName() const override { return family_name; } + TypeIndex getTypeId() const override { return type_id; } void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override; void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override; diff --git a/src/DataTypes/DataTypeString.h b/src/DataTypes/DataTypeString.h index d34c19f2664..f6db8fe73d4 100644 --- a/src/DataTypes/DataTypeString.h +++ b/src/DataTypes/DataTypeString.h @@ -13,13 +13,14 @@ class DataTypeString final : public IDataType public: using FieldType = String; static constexpr bool is_parametric = false; + static constexpr auto type_id = TypeIndex::String; const char * getFamilyName() const override { return "String"; } - TypeIndex getTypeId() const override { return TypeIndex::String; } + TypeIndex getTypeId() const override { return type_id; } void serializeBinary(const Field & field, WriteBuffer & ostr) const override; void deserializeBinary(Field & field, ReadBuffer & istr) const override; diff --git a/src/DataTypes/IDataType.h b/src/DataTypes/IDataType.h index 28e4a4a6579..73c966c08c3 100644 --- a/src/DataTypes/IDataType.h +++ b/src/DataTypes/IDataType.h @@ -529,10 +529,15 @@ struct WhichDataType /// IDataType helpers (alternative for IDataType virtual methods with single point of truth) -inline bool isDate(const DataTypePtr & data_type) { return WhichDataType(data_type).isDate(); } -inline bool isDateOrDateTime(const DataTypePtr & data_type) { return WhichDataType(data_type).isDateOrDateTime(); } -inline bool isDateTime(const DataTypePtr & data_type) { return WhichDataType(data_type).isDateTime(); } -inline bool isDateTime64(const DataTypePtr & data_type) { return WhichDataType(data_type).isDateTime64(); } +template +inline bool isDate(const T & data_type) { return WhichDataType(data_type).isDate(); } +template +inline bool isDateOrDateTime(const T & data_type) { return WhichDataType(data_type).isDateOrDateTime(); } +template +inline bool isDateTime(const T & data_type) { return WhichDataType(data_type).isDateTime(); } +template +inline bool isDateTime64(const T & data_type) { return WhichDataType(data_type).isDateTime64(); } + inline bool isEnum(const DataTypePtr & data_type) { return WhichDataType(data_type).isEnum(); } inline bool isDecimal(const DataTypePtr & data_type) { return WhichDataType(data_type).isDecimal(); } inline bool isTuple(const DataTypePtr & data_type) { return WhichDataType(data_type).isTuple(); } @@ -636,6 +641,19 @@ inline bool isCompilableType(const DataTypePtr & data_type) return data_type->isValueRepresentedByNumber() && !isDecimal(data_type); } +template +inline bool isDataType(const DataType & data_type) +{ + WhichDataType which(data_type); + return which.idx == TYPE_IDX; +} + +template +inline bool isDataType(const DataType & data_type) +{ + return isDataType(data_type); +} + template constexpr bool IsDataTypeDecimal = false; template constexpr bool IsDataTypeNumber = false; template constexpr bool IsDataTypeDateOrDateTime = false; diff --git a/src/Functions/FunctionUnixTimestamp64.h b/src/Functions/FunctionUnixTimestamp64.h new file mode 100644 index 00000000000..2d3d12a4d0a --- /dev/null +++ b/src/Functions/FunctionUnixTimestamp64.h @@ -0,0 +1,148 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int DECIMAL_OVERFLOW; +} + +/** Casts DateTim64 to or from Int64 representation narrowed down (or scaled up) to any scale value defined in Impl. + */ +template +class FunctionUnixTimestamp64 : public IFunction +{ +public: + static constexpr auto name = Impl::name; + static constexpr auto target_scale = Impl::target_scale; + + using SourceDataType = typename Impl::SourceDataType; + using ResultDataType = typename Impl::ResultDataType; + + static constexpr bool is_result_datetime64 = std::is_same_v; + + static_assert(std::is_same_v || std::is_same_v); + + static auto create(const Context &) + { + return std::make_shared>(); + } + + String getName() const override { return name; } + size_t getNumberOfArguments() const override { return is_result_datetime64 ? 2 : 1; } + bool isVariadic() const override { return is_result_datetime64; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + if constexpr (is_result_datetime64) + { + validateFunctionArgumentTypes(*this, arguments, + FunctionArgumentDescriptors{{"value", isDataType, nullptr, std::string(SourceDataType::family_name).c_str()}}, + // optional + FunctionArgumentDescriptors{ + // {"precision", isDataType, isColumnConst, ("Precision of the result, default is " + std::to_string(target_scale)).c_str()}, + {"timezone", isStringOrFixedString, isColumnConst, "Timezone of the result"}, + }); + const auto timezone = extractTimeZoneNameFromFunctionArguments(arguments, 1, 0); + return std::make_shared(target_scale, timezone); + } + else + { + validateFunctionArgumentTypes(*this, arguments, + FunctionArgumentDescriptors{{"value", isDataType, nullptr, std::string(SourceDataType::family_name).c_str()}}); + return std::make_shared(); + } + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + { + using SourceColumnType = typename SourceDataType::ColumnType; + using ResultColumnType = typename ResultDataType::ColumnType; + + const auto & src = block.getByPosition(arguments[0]); + auto & res = block.getByPosition(result); + const auto & col = *src.column; + + const SourceColumnType * source_col_typed = checkAndGetColumn(col); + if (!source_col_typed && !(source_col_typed = checkAndGetColumnConstData(&col))) + throw Exception("Invalid column type" + col.getName() + " expected " + + std::string(SourceDataType::family_name), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + res.column = res.type->createColumn(); + + if (input_rows_count == 0) + return; + + auto & result_data = assert_cast(res.column->assumeMutableRef()).getData(); + result_data.reserve(source_col_typed->size()); + const auto & source_data = source_col_typed->getData(); + + const auto scale_diff = getScaleDiff(*checkAndGetDataType(src.type.get()), *checkAndGetDataType(res.type.get())); + if (scale_diff == 0) + { + static_assert(sizeof(typename SourceColumnType::Container::value_type) == sizeof(typename ResultColumnType::Container::value_type)); + // no conversion necessary + result_data.push_back_raw_many(source_data.size(), source_data.data()); + } + else if (scale_diff < 0) + { + const Int64 scale_multiplier = DecimalUtils::scaleMultiplier(std::abs(scale_diff)); + for (const auto & v : source_data) + { + Int64 result_value = toDestValue(v); + if (common::mulOverflow(result_value, scale_multiplier, result_value)) + throw Exception("Decimal overflow in " + getName(), ErrorCodes::DECIMAL_OVERFLOW); + + result_data.push_back(result_value); + } + } + else + { + const Int64 scale_multiplier = DecimalUtils::scaleMultiplier(scale_diff); + for (const auto & v : source_data) + result_data.push_back(static_cast(toDestValue(v) / scale_multiplier)); + } + } + +private: + static Int64 getScaleDiff(const SourceDataType & src, const ResultDataType & dst) + { + Int64 src_scale = target_scale; + if constexpr (std::is_same_v) + { + src_scale = src.getScale(); + } + + Int64 dst_scale = target_scale; + if constexpr (std::is_same_v) + { + dst_scale = dst.getScale(); + } + + return src_scale - dst_scale; + } + + static auto toDestValue(const DateTime64 & v) + { + return Int64{v.value}; + } + + template + static auto toDestValue(const T & v) + { + return Int64{v}; + } +}; + +} diff --git a/src/Functions/fromUnixTimestamp64Micro.cpp b/src/Functions/fromUnixTimestamp64Micro.cpp new file mode 100644 index 00000000000..a4bac327a3a --- /dev/null +++ b/src/Functions/fromUnixTimestamp64Micro.cpp @@ -0,0 +1,20 @@ +#include +#include + +namespace DB +{ + +struct TransformFromMicro +{ + static constexpr auto name = "fromUnixTimestamp64Micro"; + static constexpr auto target_scale = 6; + using SourceDataType = DataTypeInt64; + using ResultDataType = DataTypeDateTime64; +}; + +void registerFromUnixTimestamp64Micro(FunctionFactory & factory) +{ + factory.registerFunction>(); +} + +} diff --git a/src/Functions/fromUnixTimestamp64Milli.cpp b/src/Functions/fromUnixTimestamp64Milli.cpp new file mode 100644 index 00000000000..a4ce79ecebb --- /dev/null +++ b/src/Functions/fromUnixTimestamp64Milli.cpp @@ -0,0 +1,20 @@ +#include +#include + +namespace DB +{ + +struct TransformFromMilli +{ + static constexpr auto name = "fromUnixTimestamp64Milli"; + static constexpr auto target_scale = 3; + using SourceDataType = DataTypeInt64; + using ResultDataType = DataTypeDateTime64; +}; + +void registerFromUnixTimestamp64Milli(FunctionFactory & factory) +{ + factory.registerFunction>(); +} + +} diff --git a/src/Functions/fromUnixTimestamp64Nano.cpp b/src/Functions/fromUnixTimestamp64Nano.cpp new file mode 100644 index 00000000000..cd69aa56a2c --- /dev/null +++ b/src/Functions/fromUnixTimestamp64Nano.cpp @@ -0,0 +1,20 @@ +#include +#include + +namespace DB +{ + +struct TransformFromNano +{ + static constexpr auto name = "fromUnixTimestamp64Nano"; + static constexpr auto target_scale = 9; + using SourceDataType = DataTypeInt64; + using ResultDataType = DataTypeDateTime64; +}; + +void registerFromUnixTimestamp64Nano(FunctionFactory & factory) +{ + factory.registerFunction>(); +} + +} diff --git a/src/Functions/registerFunctions.cpp b/src/Functions/registerFunctions.cpp index 74c0c7e4112..762c551de08 100644 --- a/src/Functions/registerFunctions.cpp +++ b/src/Functions/registerFunctions.cpp @@ -37,6 +37,7 @@ void registerFunctionsIntrospection(FunctionFactory &); void registerFunctionsNull(FunctionFactory &); void registerFunctionsJSON(FunctionFactory &); void registerFunctionsConsistentHashing(FunctionFactory & factory); +void registerFunctionsUnixTimestamp64(FunctionFactory & factory); void registerFunctions() @@ -78,6 +79,7 @@ void registerFunctions() registerFunctionsJSON(factory); registerFunctionsIntrospection(factory); registerFunctionsConsistentHashing(factory); + registerFunctionsUnixTimestamp64(factory); } } diff --git a/src/Functions/registerFunctionsUnixTimestamp64.cpp b/src/Functions/registerFunctionsUnixTimestamp64.cpp new file mode 100644 index 00000000000..d7f3a4a4fc2 --- /dev/null +++ b/src/Functions/registerFunctionsUnixTimestamp64.cpp @@ -0,0 +1,25 @@ +namespace DB +{ + +class FunctionFactory; + +void registerToUnixTimestamp64Milli(FunctionFactory &); +void registerToUnixTimestamp64Micro(FunctionFactory &); +void registerToUnixTimestamp64Nano(FunctionFactory &); + +void registerFromUnixTimestamp64Milli(FunctionFactory &); +void registerFromUnixTimestamp64Micro(FunctionFactory &); +void registerFromUnixTimestamp64Nano(FunctionFactory &); + +void registerFunctionsUnixTimestamp64(FunctionFactory & factory) +{ + registerToUnixTimestamp64Milli(factory); + registerToUnixTimestamp64Micro(factory); + registerToUnixTimestamp64Nano(factory); + + registerFromUnixTimestamp64Milli(factory); + registerFromUnixTimestamp64Micro(factory); + registerFromUnixTimestamp64Nano(factory); +} + +} diff --git a/src/Functions/toUnixTimestamp64Micro.cpp b/src/Functions/toUnixTimestamp64Micro.cpp new file mode 100644 index 00000000000..845647fc0dd --- /dev/null +++ b/src/Functions/toUnixTimestamp64Micro.cpp @@ -0,0 +1,20 @@ +#include +#include + +namespace DB +{ + +struct TransformToMicro +{ + static constexpr auto name = "toUnixTimestamp64Micro"; + static constexpr auto target_scale = 6; + using SourceDataType = DataTypeDateTime64; + using ResultDataType = DataTypeInt64; +}; + +void registerToUnixTimestamp64Micro(FunctionFactory & factory) +{ + factory.registerFunction>(); +} + +} diff --git a/src/Functions/toUnixTimestamp64Milli.cpp b/src/Functions/toUnixTimestamp64Milli.cpp new file mode 100644 index 00000000000..88baf5c0b61 --- /dev/null +++ b/src/Functions/toUnixTimestamp64Milli.cpp @@ -0,0 +1,20 @@ +#include +#include + +namespace DB +{ + +struct TransformToMilli +{ + static constexpr auto name = "toUnixTimestamp64Milli"; + static constexpr auto target_scale = 3; + using SourceDataType = DataTypeDateTime64; + using ResultDataType = DataTypeInt64; +}; + +void registerToUnixTimestamp64Milli(FunctionFactory & factory) +{ + factory.registerFunction>(); +} + +} diff --git a/src/Functions/toUnixTimestamp64Nano.cpp b/src/Functions/toUnixTimestamp64Nano.cpp new file mode 100644 index 00000000000..5e90a9504cf --- /dev/null +++ b/src/Functions/toUnixTimestamp64Nano.cpp @@ -0,0 +1,20 @@ +#include +#include + +namespace DB +{ + +struct TransformToNano +{ + static constexpr auto name = "toUnixTimestamp64Nano"; + static constexpr auto target_scale = 9; + using SourceDataType = DataTypeDateTime64; + using ResultDataType = DataTypeInt64; +}; + +void registerToUnixTimestamp64Nano(FunctionFactory & factory) +{ + factory.registerFunction>(); +} + +} diff --git a/src/Functions/ya.make b/src/Functions/ya.make index 8e53ffe493d..ddb1d27b398 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -151,6 +151,9 @@ SRCS( finalizeAggregation.cpp formatDateTime.cpp formatString.cpp + fromUnixTimestamp64Micro.cpp + fromUnixTimestamp64Milli.cpp + fromUnixTimestamp64Nano.cpp FunctionFactory.cpp FunctionFQDN.cpp FunctionHelpers.cpp @@ -168,6 +171,7 @@ SRCS( FunctionsRound.cpp FunctionsStringArray.cpp FunctionsStringSimilarity.cpp + FunctionUnixTimestamp64.h GatherUtils/concat.cpp GatherUtils/createArraySink.cpp GatherUtils/createArraySource.cpp @@ -305,6 +309,7 @@ SRCS( registerFunctionsStringSearch.cpp registerFunctionsTuple.cpp registerFunctionsVisitParam.cpp + registerFunctionsUnixTimestamp64.cpp reinterpretAsFixedString.cpp reinterpretAsString.cpp reinterpretStringAs.cpp @@ -386,6 +391,9 @@ SRCS( toTimeZone.cpp toTypeName.cpp toValidUTF8.cpp + toUnixTimestamp64Micro.cpp + toUnixTimestamp64Milli.cpp + toUnixTimestamp64Nano.cpp toYear.cpp toYYYYMM.cpp toYYYYMMDD.cpp diff --git a/tests/queries/0_stateless/01277_fromUnixTimestamp64.reference b/tests/queries/0_stateless/01277_fromUnixTimestamp64.reference new file mode 100644 index 00000000000..610041de31e --- /dev/null +++ b/tests/queries/0_stateless/01277_fromUnixTimestamp64.reference @@ -0,0 +1,5 @@ +const column +UTC 1234567891011 2009-02-13 23:31:31.011 1970-01-15 06:56:07.891011 1970-01-01 00:20:34.567891011 DateTime64(9, \'UTC\') +Asia/Makassar 1234567891011 2009-02-14 07:31:31.011 1970-01-15 14:56:07.891011 1970-01-01 08:20:34.567891011 DateTime64(9, \'Asia/Makassar\') +non-const column +1234567891011 2009-02-13 23:31:31.011 1970-01-15 06:56:07.891011 1970-01-01 00:20:34.567891011 diff --git a/tests/queries/0_stateless/01277_fromUnixTimestamp64.sql b/tests/queries/0_stateless/01277_fromUnixTimestamp64.sql new file mode 100644 index 00000000000..4f1497763e1 --- /dev/null +++ b/tests/queries/0_stateless/01277_fromUnixTimestamp64.sql @@ -0,0 +1,45 @@ +-- -- Error cases +SELECT fromUnixTimestamp64Milli(); -- {serverError 42} +SELECT fromUnixTimestamp64Micro(); -- {serverError 42} +SELECT fromUnixTimestamp64Nano(); -- {serverError 42} + +SELECT fromUnixTimestamp64Milli('abc'); -- {serverError 43} +SELECT fromUnixTimestamp64Micro('abc'); -- {serverError 43} +SELECT fromUnixTimestamp64Nano('abc'); -- {serverError 43} + +SELECT fromUnixTimestamp64Milli('abc', 123); -- {serverError 43} +SELECT fromUnixTimestamp64Micro('abc', 123); -- {serverError 43} +SELECT fromUnixTimestamp64Nano('abc', 123); -- {serverError 43} + +SELECT 'const column'; +WITH + CAST(1234567891011 AS Int64) AS i64, + 'UTC' AS tz +SELECT + tz, + i64, + fromUnixTimestamp64Milli(i64, tz), + fromUnixTimestamp64Micro(i64, tz), + fromUnixTimestamp64Nano(i64, tz) as dt64, + toTypeName(dt64); + +WITH + CAST(1234567891011 AS Int64) AS i64, + 'Asia/Makassar' AS tz +SELECT + tz, + i64, + fromUnixTimestamp64Milli(i64, tz), + fromUnixTimestamp64Micro(i64, tz), + fromUnixTimestamp64Nano(i64, tz) as dt64, + toTypeName(dt64); + +SELECT 'non-const column'; +WITH + CAST(1234567891011 AS Int64) AS i64, + 'UTC' AS tz +SELECT + i64, + fromUnixTimestamp64Milli(i64, tz), + fromUnixTimestamp64Micro(i64, tz), + fromUnixTimestamp64Nano(i64, tz) as dt64; \ No newline at end of file diff --git a/tests/queries/0_stateless/01277_toUnixTimestamp64.reference b/tests/queries/0_stateless/01277_toUnixTimestamp64.reference new file mode 100644 index 00000000000..7b66586b4d2 --- /dev/null +++ b/tests/queries/0_stateless/01277_toUnixTimestamp64.reference @@ -0,0 +1,8 @@ +const column +2019-09-16 19:20:12.345 1568650812345 1568650812345000 1568650812345000000 +2019-09-16 19:20:12.345678 1568650812345 1568650812345678 1568650812345678000 +2019-09-16 19:20:12.345678910 1568650812345 1568650812345678 1568650812345678910 +non-const column +2019-09-16 19:20:12.345 1568650812345 1568650812345000 1568650812345000000 +2019-09-16 19:20:12.345678 1568650812345 1568650812345678 1568650812345678000 +2019-09-16 19:20:12.345678910 1568650812345 1568650812345678 1568650812345678910 diff --git a/tests/queries/0_stateless/01277_toUnixTimestamp64.sql b/tests/queries/0_stateless/01277_toUnixTimestamp64.sql new file mode 100644 index 00000000000..de2b132a2dc --- /dev/null +++ b/tests/queries/0_stateless/01277_toUnixTimestamp64.sql @@ -0,0 +1,33 @@ +-- Error cases +SELECT toUnixTimestamp64Milli(); -- {serverError 42} +SELECT toUnixTimestamp64Micro(); -- {serverError 42} +SELECT toUnixTimestamp64Nano(); -- {serverError 42} + +SELECT toUnixTimestamp64Milli('abc'); -- {serverError 43} +SELECT toUnixTimestamp64Micro('abc'); -- {serverError 43} +SELECT toUnixTimestamp64Nano('abc'); -- {serverError 43} + +SELECT toUnixTimestamp64Milli('abc', 123); -- {serverError 42} +SELECT toUnixTimestamp64Micro('abc', 123); -- {serverError 42} +SELECT toUnixTimestamp64Nano('abc', 123); -- {serverError 42} + +SELECT 'const column'; +WITH toDateTime64('2019-09-16 19:20:12.345678910', 3) AS dt64 +SELECT dt64, toUnixTimestamp64Milli(dt64), toUnixTimestamp64Micro(dt64), toUnixTimestamp64Nano(dt64); + +WITH toDateTime64('2019-09-16 19:20:12.345678910', 6) AS dt64 +SELECT dt64, toUnixTimestamp64Milli(dt64), toUnixTimestamp64Micro(dt64), toUnixTimestamp64Nano(dt64); + +WITH toDateTime64('2019-09-16 19:20:12.345678910', 9) AS dt64 +SELECT dt64, toUnixTimestamp64Milli(dt64), toUnixTimestamp64Micro(dt64), toUnixTimestamp64Nano(dt64); + +SELECT 'non-const column'; +WITH toDateTime64('2019-09-16 19:20:12.345678910', 3) AS x +SELECT materialize(x) as dt64, toUnixTimestamp64Milli(dt64), toUnixTimestamp64Micro(dt64), toUnixTimestamp64Nano(dt64); + +WITH toDateTime64('2019-09-16 19:20:12.345678910', 6) AS x +SELECT materialize(x) as dt64, toUnixTimestamp64Milli(dt64), toUnixTimestamp64Micro(dt64), toUnixTimestamp64Nano(dt64); + +WITH toDateTime64('2019-09-16 19:20:12.345678910', 9) AS x +SELECT materialize(x) as dt64, toUnixTimestamp64Milli(dt64), toUnixTimestamp64Micro(dt64), toUnixTimestamp64Nano(dt64); + diff --git a/tests/queries/0_stateless/01277_unixTimestamp64_compatibility.reference b/tests/queries/0_stateless/01277_unixTimestamp64_compatibility.reference new file mode 100644 index 00000000000..9a70396832c --- /dev/null +++ b/tests/queries/0_stateless/01277_unixTimestamp64_compatibility.reference @@ -0,0 +1,9 @@ +2019-09-16 19:20:12.345 2019-09-16 19:20:12.345 2019-09-16 19:20:12.345000 2019-09-16 19:20:12.345000000 +2019-09-16 19:20:12.345678 2019-09-16 19:20:12.345 2019-09-16 19:20:12.345678 2019-09-16 19:20:12.345678000 +2019-09-16 19:20:12.345678910 2019-09-16 19:20:12.345 2019-09-16 19:20:12.345678 2019-09-16 19:20:12.345678910 +with explicit timezone +2019-09-16 19:20:12.345 2019-09-16 19:20:12.345 2019-09-16 19:20:12.345000 2019-09-16 19:20:12.345000000 DateTime64(9, \'UTC\') +2019-09-16 19:20:12.345 2019-09-16 19:20:12.345 2019-09-16 19:20:12.345000 2019-09-16 19:20:12.345000000 DateTime64(9, \'Asia/Makassar\') +1234567891011 1234567891011 1234567891011 1234567891011 +with explicit timezone +1234567891011 1234567891011 1234567891011 1234567891011 Int64 diff --git a/tests/queries/0_stateless/01277_unixTimestamp64_compatibility.sql b/tests/queries/0_stateless/01277_unixTimestamp64_compatibility.sql new file mode 100644 index 00000000000..7d8d0b879b6 --- /dev/null +++ b/tests/queries/0_stateless/01277_unixTimestamp64_compatibility.sql @@ -0,0 +1,64 @@ +WITH + toDateTime64('2019-09-16 19:20:12.345678910', 3) AS dt64 +SELECT + dt64, + fromUnixTimestamp64Milli(toUnixTimestamp64Milli(dt64)), + fromUnixTimestamp64Micro(toUnixTimestamp64Micro(dt64)), + fromUnixTimestamp64Nano(toUnixTimestamp64Nano(dt64)); + +WITH + toDateTime64('2019-09-16 19:20:12.345678910', 6) AS dt64 +SELECT + dt64, + fromUnixTimestamp64Milli(toUnixTimestamp64Milli(dt64)), + fromUnixTimestamp64Micro(toUnixTimestamp64Micro(dt64)), + fromUnixTimestamp64Nano(toUnixTimestamp64Nano(dt64)); + +WITH + toDateTime64('2019-09-16 19:20:12.345678910', 9) AS dt64 +SELECT + dt64, + fromUnixTimestamp64Milli(toUnixTimestamp64Milli(dt64)), + fromUnixTimestamp64Micro(toUnixTimestamp64Micro(dt64)), + fromUnixTimestamp64Nano(toUnixTimestamp64Nano(dt64)); + +SELECT 'with explicit timezone'; +WITH + 'UTC' as timezone, + toDateTime64('2019-09-16 19:20:12.345678910', 3, timezone) AS dt64 +SELECT + dt64, + fromUnixTimestamp64Milli(toUnixTimestamp64Milli(dt64), timezone), + fromUnixTimestamp64Micro(toUnixTimestamp64Micro(dt64), timezone), + fromUnixTimestamp64Nano(toUnixTimestamp64Nano(dt64), timezone) AS v, + toTypeName(v); + +WITH + 'Asia/Makassar' as timezone, + toDateTime64('2019-09-16 19:20:12.345678910', 3, timezone) AS dt64 +SELECT + dt64, + fromUnixTimestamp64Milli(toUnixTimestamp64Milli(dt64), timezone), + fromUnixTimestamp64Micro(toUnixTimestamp64Micro(dt64), timezone), + fromUnixTimestamp64Nano(toUnixTimestamp64Nano(dt64), timezone) AS v, + toTypeName(v); + + +WITH + CAST(1234567891011 AS Int64) AS val +SELECT + val, + toUnixTimestamp64Milli(fromUnixTimestamp64Milli(val)), + toUnixTimestamp64Micro(fromUnixTimestamp64Micro(val)), + toUnixTimestamp64Nano(fromUnixTimestamp64Nano(val)); + +SELECT 'with explicit timezone'; +WITH + 'UTC' as timezone, + CAST(1234567891011 AS Int64) AS val +SELECT + val, + toUnixTimestamp64Milli(fromUnixTimestamp64Milli(val, timezone)), + toUnixTimestamp64Micro(fromUnixTimestamp64Micro(val, timezone)), + toUnixTimestamp64Nano(fromUnixTimestamp64Nano(val, timezone)) AS v, + toTypeName(v); \ No newline at end of file From e09bcb4290b374b654d2a71ddd9fbde5beeeaa1f Mon Sep 17 00:00:00 2001 From: Maxim Akhmedov Date: Tue, 26 May 2020 17:05:06 +0300 Subject: [PATCH 728/738] Make possible adding nested storages to MultipleAccessStorage in run-time. --- src/Access/MultipleAccessStorage.cpp | 8 ++++++++ src/Access/MultipleAccessStorage.h | 2 ++ 2 files changed, 10 insertions(+) diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index 0dd1f142f31..9b80c16e487 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -143,6 +143,14 @@ const IAccessStorage & MultipleAccessStorage::getStorage(const UUID & id) const return const_cast(this)->getStorage(id); } +void MultipleAccessStorage::addStorage(std::unique_ptr nested_storage) +{ + /// Note that IStorage::storage_name is not changed. It is ok as this method + /// is considered as a temporary solution allowing third-party Arcadia applications + /// using CH as a library to register their own access storages. Do not remove + /// this method without providing any alternative :) + nested_storages.emplace_back(std::move(nested_storage)); +} AccessEntityPtr MultipleAccessStorage::readImpl(const UUID & id) const { diff --git a/src/Access/MultipleAccessStorage.h b/src/Access/MultipleAccessStorage.h index ec8c8f2a101..06fb3d45c05 100644 --- a/src/Access/MultipleAccessStorage.h +++ b/src/Access/MultipleAccessStorage.h @@ -25,6 +25,8 @@ public: const Storage & getStorage(const UUID & id) const; Storage & getStorage(const UUID & id); + void addStorage(std::unique_ptr nested_storage); + Storage & getStorageByIndex(size_t i) { return *(nested_storages[i]); } const Storage & getStorageByIndex(size_t i) const { return *(nested_storages[i]); } From 9d0c28d96c1987d4f4412a679354e22ad0f6304c Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 26 May 2020 17:14:08 +0300 Subject: [PATCH 729/738] Add missing methods --- src/Storages/IStorage.cpp | 24 +++++++++++++++---- src/Storages/IStorage.h | 11 +++++++-- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 4 ++-- src/Storages/MergeTree/MergeTreeData.cpp | 6 ++--- 4 files changed, 34 insertions(+), 11 deletions(-) diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index 64e06a2534e..6564fee827e 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -444,9 +444,14 @@ void IStorage::setPartitionKey(const StorageMetadataKeyField & partition_key_) partition_key = partition_key_; } +bool IStorage::isPartitionKeyDefined() const +{ + return partition_key.definition_ast != nullptr; +} + bool IStorage::hasPartitionKey() const { - return partition_key.expression != nullptr; + return !partition_key.column_names.empty(); } Names IStorage::getColumnsRequiredForPartitionKey() const @@ -466,9 +471,14 @@ void IStorage::setSortingKey(const StorageMetadataKeyField & sorting_key_) sorting_key = sorting_key_; } +bool IStorage::isSortingKeyDefined() const +{ + return sorting_key.definition_ast != nullptr; +} + bool IStorage::hasSortingKey() const { - return sorting_key.expression != nullptr; + return !sorting_key.column_names.empty(); } Names IStorage::getColumnsRequiredForSortingKey() const @@ -502,7 +512,7 @@ bool IStorage::isPrimaryKeyDefined() const bool IStorage::hasPrimaryKey() const { - return primary_key.expression != nullptr; + return !primary_key.column_names.empty(); } Names IStorage::getColumnsRequiredForPrimaryKey() const @@ -529,9 +539,15 @@ void IStorage::setSamplingKey(const StorageMetadataKeyField & sampling_key_) sampling_key = sampling_key_; } + +bool IStorage::isSamplingKeyDefined() const +{ + return sampling_key.definition_ast != nullptr; +} + bool IStorage::hasSamplingKey() const { - return sampling_key.expression != nullptr; + return !sampling_key.column_names.empty(); } Names IStorage::getColumnsRequiredForSampling() const diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index f707b3a4b5c..655ced0d8cf 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -453,6 +453,8 @@ public: void setPartitionKey(const StorageMetadataKeyField & partition_key_); /// Returns ASTExpressionList of partition key expression for storage or nullptr if there is none. ASTPtr getPartitionKeyAST() const { return partition_key.definition_ast; } + /// Storage has user-defined (in CREATE query) partition key. + bool isPartitionKeyDefined() const; /// Storage has partition key. bool hasPartitionKey() const; /// Returns column names that need to be read to calculate partition key. @@ -466,7 +468,9 @@ public: void setSortingKey(const StorageMetadataKeyField & sorting_key_); /// Returns ASTExpressionList of sorting key expression for storage or nullptr if there is none. ASTPtr getSortingKeyAST() const { return sorting_key.definition_ast; } - /// Storage has sorting key. + /// Storage has user-defined (in CREATE query) sorting key. + bool isSortingKeyDefined() const; + /// Storage has sorting key. It means, that it contains at least one column. bool hasSortingKey() const; /// Returns column names that need to be read to calculate sorting key. Names getColumnsRequiredForSortingKey() const; @@ -483,7 +487,8 @@ public: ASTPtr getPrimaryKeyAST() const { return primary_key.definition_ast; } /// Storage has user-defined (in CREATE query) sorting key. bool isPrimaryKeyDefined() const; - /// Storage has primary key (maybe part of some other key). + /// Storage has primary key (maybe part of some other key). It means, that + /// it contains at least one column. bool hasPrimaryKey() const; /// Returns column names that need to be read to calculate primary key. Names getColumnsRequiredForPrimaryKey() const; @@ -498,6 +503,8 @@ public: void setSamplingKey(const StorageMetadataKeyField & sampling_key_); /// Returns sampling expression AST for storage or nullptr if there is none. ASTPtr getSamplingKeyAST() const { return sampling_key.definition_ast; } + /// Storage has user-defined (in CREATE query) sampling key. + bool isSamplingKeyDefined() const; /// Storage has sampling key. bool hasSamplingKey() const; /// Returns column names that need to be read to calculate sampling key. diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 78548cdddb2..6e048a956da 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -841,7 +841,7 @@ void IMergeTreeDataPart::checkConsistencyBase() const if (!checksums.empty()) { - if (storage.isPrimaryKeyDefined() && !checksums.files.count("primary.idx")) + if (storage.hasPrimaryKey() && !checksums.files.count("primary.idx")) throw Exception("No checksum for primary.idx", ErrorCodes::NO_FILE_IN_DATA_PART); if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) @@ -875,7 +875,7 @@ void IMergeTreeDataPart::checkConsistencyBase() const }; /// Check that the primary key index is not empty. - if (storage.isPrimaryKeyDefined()) + if (storage.hasPrimaryKey()) check_file_not_empty(volume->getDisk(), path + "primary.idx"); if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index d80dce167b4..48cf3934820 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -250,10 +250,10 @@ StorageInMemoryMetadata MergeTreeData::getInMemoryMetadata() const { StorageInMemoryMetadata metadata(getColumns(), getIndices(), getConstraints()); - if (hasPartitionKey()) + if (isPartitionKeyDefined()) metadata.partition_by_ast = getPartitionKeyAST()->clone(); - if (hasSortingKey()) + if (isSortingKeyDefined()) metadata.order_by_ast = getSortingKeyAST()->clone(); if (isPrimaryKeyDefined()) @@ -262,7 +262,7 @@ StorageInMemoryMetadata MergeTreeData::getInMemoryMetadata() const if (ttl_table_ast) metadata.ttl_for_table_ast = ttl_table_ast->clone(); - if (hasSamplingKey()) + if (isSamplingKeyDefined()) metadata.sample_by_ast = getSamplingKeyAST()->clone(); if (settings_ast) From 3ff28d0a7a4d53c2be2604bceae76149e3c5bf39 Mon Sep 17 00:00:00 2001 From: BayoNet Date: Tue, 26 May 2020 18:10:56 +0300 Subject: [PATCH 730/738] DOCS-112: any_join_distinct_right_table_keys (#11065) * CLICKHOUSEDOCS-112: any_join_distinct_right_table_keys * CLICKHOUSEDOCS-112: Updated link. * CLICKHOUSEDOCS-112: Updated by comments. * CLICKHOUSEDOCS-112: Updated by comments. * CLICKHOUSEDOCS-112: Fixed grammar. * CLICKHOUSEDOCS-112: Updated by comments. * CLICKHOUSEDOCS-112: Fixed typo. Co-authored-by: Sergei Shtykov --- docs/en/operations/settings/settings.md | 30 +++++++++++++++++++ .../sql-reference/statements/select/join.md | 2 ++ 2 files changed, 32 insertions(+) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 459e030a4cc..909ba7bf876 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -404,6 +404,36 @@ Possible values: Default value: 0. +## any_join_distinct_right_table_keys {#any_join_distinct_right_table_keys} + +Enables legacy ClickHouse server behavior in `ANY INNER|LEFT JOIN` operations. + +!!! note "Warning" + Use this setting only for the purpose of backward compatibility if your use cases depend on legacy `JOIN` behavior. + +When the legacy behavior enabled: + +- Results of `t1 ANY LEFT JOIN t2` and `t2 ANY RIGHT JOIN t1` operations are not equal because ClickHouse uses the logic with many-to-one left-to-right table keys mapping. +- Results of `ANY INNER JOIN` operations contain all rows from the left table like the `SEMI LEFT JOIN` operations do. + +When the legacy behavior disabled: + +- Results of `t1 ANY LEFT JOIN t2` and `t2 ANY RIGHT JOIN t1` operations are equal because ClickHouse uses the logic which provides one-to-many keys mapping in `ANY RIGHT JOIN` operations. +- Results of `ANY INNER JOIN` operations contain one row per key from both left and right tables. + +Possible values: + +- 0 — Legacy behavior is disabled. +- 1 — Legacy behavior is enabled. + + +Default value: 0. + +See also: + +- [JOIN strictness](../../sql-reference/statements/select/join.md#select-join-strictness) + + ## max\_block\_size {#setting-max_block_size} In ClickHouse, data is processed by blocks (sets of column parts). The internal processing cycles for a single block are efficient enough, but there are noticeable expenditures on each block. The `max_block_size` setting is a recommendation for what size of the block (in a count of rows) to load from tables. The block size shouldn’t be too small, so that the expenditures on each block are still noticeable, but not too large so that the query with LIMIT that is completed after the first block is processed quickly. The goal is to avoid consuming too much memory when extracting a large number of columns in multiple threads and to preserve at least some cache locality. diff --git a/docs/en/sql-reference/statements/select/join.md b/docs/en/sql-reference/statements/select/join.md index c636e1dab8b..5ac3f4a0e25 100644 --- a/docs/en/sql-reference/statements/select/join.md +++ b/docs/en/sql-reference/statements/select/join.md @@ -44,6 +44,8 @@ Modifies how matching by "join keys" is performed !!! note "Note" The default strictness value can be overriden using [join\_default\_strictness](../../../operations/settings/settings.md#settings-join_default_strictness) setting. + Also the behavior of ClickHouse server for `ANY JOIN` operations depends on the [any_join_distinct_right_table_keys](../../../operations/settings/settings.md#any_join_distinct_right_table_keys) setting. + ### ASOF JOIN Usage From 009f57fc2fc84e4cffe08eea118fa0543ca4c66c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 26 May 2020 21:26:17 +0300 Subject: [PATCH 731/738] Fix flacky test #11190 --- .../01247_dist_on_dist_group_by_sharding_key_optimization.sql | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/queries/0_stateless/01247_dist_on_dist_group_by_sharding_key_optimization.sql b/tests/queries/0_stateless/01247_dist_on_dist_group_by_sharding_key_optimization.sql index 7cf171c8c73..29e45ffdb80 100644 --- a/tests/queries/0_stateless/01247_dist_on_dist_group_by_sharding_key_optimization.sql +++ b/tests/queries/0_stateless/01247_dist_on_dist_group_by_sharding_key_optimization.sql @@ -1,5 +1,9 @@ -- TODO: correct testing with real unique shards +-- Avoid "Connection failed at try №1" messages. +SET send_logs_level = 'none'; +SET connect_timeout_with_failover_ms = 5000; + set optimize_distributed_group_by_sharding_key=1; drop table if exists dist_01247; From 90f49c0c17ff5ce8182a5d45543c5c3aab9985f4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 26 May 2020 21:37:20 +0300 Subject: [PATCH 732/738] Fix error in performance test --- tests/performance/agg_functions_min_max_any.xml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/performance/agg_functions_min_max_any.xml b/tests/performance/agg_functions_min_max_any.xml index 4591182cdfc..9c16cb88970 100644 --- a/tests/performance/agg_functions_min_max_any.xml +++ b/tests/performance/agg_functions_min_max_any.xml @@ -35,10 +35,6 @@ select max(PageCharset) from test.hits where PageCharset != '' group by intHash32(UserID) % 1000000 FORMAT Null select any(PageCharset) from test.hits where PageCharset != '' group by intHash32(UserID) % 1000000 FORMAT Null select anyHeavy(PageCharset) from test.hits where PageCharset != '' group by intHash32(UserID) % 1000000 FORMAT Null -select min(OriginalURL) from test.hits where OriginalURL != '' group by intHash32(UserID) % 1000000 FORMAT Null -select max(OriginalURL) from test.hits where OriginalURL != '' group by intHash32(UserID) % 1000000 FORMAT Null -select any(OriginalURL) from test.hits where OriginalURL != '' group by intHash32(UserID) % 1000000 FORMAT Null -select anyHeavy(OriginalURL) from test.hits where OriginalURL != '' group by intHash32(UserID) % 1000000 FORMAT Null select min(SocialNetwork) from test.hits where SocialNetwork != '' group by intHash32(UserID) % 1000000 FORMAT Null select max(SocialNetwork) from test.hits where SocialNetwork != '' group by intHash32(UserID) % 1000000 FORMAT Null select any(SocialNetwork) from test.hits where SocialNetwork != '' group by intHash32(UserID) % 1000000 FORMAT Null From e049290af7b311f60205684c9e2d5bd154e17459 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 26 May 2020 22:14:54 +0300 Subject: [PATCH 733/738] add cassandra-driver to integration tests --- docker/test/integration/runner/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/integration/runner/Dockerfile b/docker/test/integration/runner/Dockerfile index 53f41905f10..9c1fe66cf7b 100644 --- a/docker/test/integration/runner/Dockerfile +++ b/docker/test/integration/runner/Dockerfile @@ -36,7 +36,7 @@ RUN apt-get update \ ENV TZ=Europe/Moscow RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone -RUN pip install urllib3==1.23 pytest docker-compose==1.22.0 docker dicttoxml kazoo PyMySQL psycopg2==2.7.5 pymongo tzlocal kafka-python protobuf redis aerospike pytest-timeout minio rpm-confluent-schemaregistry grpcio grpcio-tools +RUN pip install urllib3==1.23 pytest docker-compose==1.22.0 docker dicttoxml kazoo PyMySQL psycopg2==2.7.5 pymongo tzlocal kafka-python protobuf redis aerospike pytest-timeout minio rpm-confluent-schemaregistry grpcio grpcio-tools cassandra-driver ENV DOCKER_CHANNEL stable ENV DOCKER_VERSION 17.09.1-ce From a5bde57acfd00acf1af44f794daa4ebb23583201 Mon Sep 17 00:00:00 2001 From: Saulius Valatka Date: Tue, 26 May 2020 23:43:20 +0300 Subject: [PATCH 734/738] add reference to stream-loader (#11206) --- docs/en/interfaces/third-party/integrations.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/en/interfaces/third-party/integrations.md b/docs/en/interfaces/third-party/integrations.md index 5e9e8f841bf..716e774871b 100644 --- a/docs/en/interfaces/third-party/integrations.md +++ b/docs/en/interfaces/third-party/integrations.md @@ -25,6 +25,7 @@ toc_title: Integrations - Message queues - [Kafka](https://kafka.apache.org) - [clickhouse\_sinker](https://github.com/housepower/clickhouse_sinker) (uses [Go client](https://github.com/ClickHouse/clickhouse-go/)) + - [stream-loader-clickhouse](https://github.com/adform/stream-loader) - Stream processing - [Flink](https://flink.apache.org) - [flink-clickhouse-sink](https://github.com/ivi-ru/flink-clickhouse-sink) From efc0b977078af8a80a2afd6a2849efeb78148201 Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Fri, 22 May 2020 18:58:08 +0200 Subject: [PATCH 735/738] Fix Kafka performance issue related to reschedules based on limits, which were always applied. --- src/Storages/Kafka/KafkaBlockInputStream.h | 1 + .../Kafka/ReadBufferFromKafkaConsumer.h | 1 + src/Storages/Kafka/StorageKafka.cpp | 31 ++++++++++---- tests/integration/test_storage_kafka/test.py | 40 ++++++++++++++++++- 4 files changed, 64 insertions(+), 9 deletions(-) diff --git a/src/Storages/Kafka/KafkaBlockInputStream.h b/src/Storages/Kafka/KafkaBlockInputStream.h index 1f94ee332d3..e3052122894 100644 --- a/src/Storages/Kafka/KafkaBlockInputStream.h +++ b/src/Storages/Kafka/KafkaBlockInputStream.h @@ -25,6 +25,7 @@ public: void readSuffixImpl() override; void commit(); + bool isStalled() const { return buffer->isStalled(); } private: StorageKafka & storage; diff --git a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h index 12da701a55d..e90e3b48881 100644 --- a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h +++ b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h @@ -38,6 +38,7 @@ public: bool hasMorePolledMessages() const; bool polledDataUnusable() const { return (was_stopped || rebalance_happened); } + bool isStalled() const { return stalled; } void storeLastReadMessageOffset(); void resetToLastCommitted(const char * msg); diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index d1dfbe9d875..7731cf3c06a 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -51,6 +51,7 @@ namespace { const auto RESCHEDULE_MS = 500; const auto CLEANUP_TIMEOUT_MS = 3000; + const auto MAX_THREAD_WORK_DURATION_MS = 60000; // once per minute leave do reschedule (we can't lock threads in pool forever) /// Configuration prefix const String CONFIG_PREFIX = "kafka"; @@ -386,6 +387,8 @@ void StorageKafka::threadFunc() size_t dependencies_count = DatabaseCatalog::instance().getDependencies(table_id).size(); if (dependencies_count) { + auto start_time = std::chrono::steady_clock::now(); + // Keep streaming as long as there are attached views and streaming is not cancelled while (!stream_cancelled && num_created_consumers > 0) { @@ -394,9 +397,21 @@ void StorageKafka::threadFunc() LOG_DEBUG(log, "Started streaming to {} attached views", dependencies_count); - // Reschedule if not limited - if (!streamToViews()) + // Exit the loop & reschedule if some stream stalled + auto some_stream_is_stalled = streamToViews(); + if (some_stream_is_stalled) + { + LOG_TRACE(log, "Stream(s) stalled. Reschedule."); break; + } + + auto ts = std::chrono::steady_clock::now(); + auto duration = std::chrono::duration_cast(ts-start_time); + if (duration.count() > MAX_THREAD_WORK_DURATION_MS) + { + LOG_TRACE(log, "Thread work duration limit exceeded. Reschedule."); + break; + } } } } @@ -459,15 +474,15 @@ bool StorageKafka::streamToViews() // It will be cancelled on underlying layer (kafka buffer) std::atomic stub = {false}; copyData(*in, *block_io.out, &stub); + + bool some_stream_is_stalled = false; for (auto & stream : streams) + { + some_stream_is_stalled = some_stream_is_stalled || stream->as()->isStalled(); stream->as()->commit(); + } - // Check whether the limits were applied during query execution - bool limits_applied = false; - const BlockStreamProfileInfo & info = in->getProfileInfo(); - limits_applied = info.hasAppliedLimit(); - - return limits_applied; + return some_stream_is_stalled; } void registerStorageKafka(StorageFactory & factory) diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py index 8e329a1cb60..d71347e83f1 100644 --- a/tests/integration/test_storage_kafka/test.py +++ b/tests/integration/test_storage_kafka/test.py @@ -1017,7 +1017,10 @@ def test_kafka_flush_by_block_size(kafka_cluster): time.sleep(1) - result = instance.query('SELECT count() FROM test.view') + # TODO: due to https://github.com/ClickHouse/ClickHouse/pull/11149 + # second flush happens earlier than expected, so we have 2 parts here instead of one + # flush by block size works correctly, so the feature checked by the test is working correctly + result = instance.query("SELECT count() FROM test.view WHERE _part='all_1_1_0'") # print(result) # kafka_cluster.open_bash_shell('instance') @@ -1390,6 +1393,41 @@ def test_commits_of_unprocessed_messages_on_drop(kafka_cluster): assert TSV(result) == TSV('{0}\t{0}\t{0}'.format(i[0]-1)), 'Missing data!' + +@pytest.mark.timeout(120) +def test_bad_reschedule(kafka_cluster): + messages = [json.dumps({'key': j+1, 'value': j+1}) for j in range(20000)] + kafka_produce('test_bad_reschedule', messages) + + instance.query(''' + CREATE TABLE test.kafka (key UInt64, value UInt64) + ENGINE = Kafka + SETTINGS kafka_broker_list = 'kafka1:19092', + kafka_topic_list = 'test_bad_reschedule', + kafka_group_name = 'test_bad_reschedule', + kafka_format = 'JSONEachRow', + kafka_max_block_size = 1000; + + CREATE MATERIALIZED VIEW test.destination Engine=Log AS + SELECT + key, + now() as consume_ts, + value, + _topic, + _key, + _offset, + _partition, + _timestamp + FROM test.kafka; + ''') + + while int(instance.query("SELECT count() FROM test.destination")) < 20000: + print("Waiting for consume") + time.sleep(1) + + assert int(instance.query("SELECT max(consume_ts) - min(consume_ts) FROM test.destination")) < 8 + + @pytest.mark.timeout(1200) def test_kafka_duplicates_when_commit_failed(kafka_cluster): messages = [json.dumps({'key': j+1, 'value': 'x' * 300}) for j in range(22)] From c73b8372e786ba6de958e8be222bf20081c97afd Mon Sep 17 00:00:00 2001 From: filimonov <1549571+filimonov@users.noreply.github.com> Date: Wed, 27 May 2020 07:44:13 +0200 Subject: [PATCH 736/738] Update test.py --- tests/integration/test_storage_kafka/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py index d71347e83f1..13577864870 100644 --- a/tests/integration/test_storage_kafka/test.py +++ b/tests/integration/test_storage_kafka/test.py @@ -1017,7 +1017,7 @@ def test_kafka_flush_by_block_size(kafka_cluster): time.sleep(1) - # TODO: due to https://github.com/ClickHouse/ClickHouse/pull/11149 + # TODO: due to https://github.com/ClickHouse/ClickHouse/issues/11216 # second flush happens earlier than expected, so we have 2 parts here instead of one # flush by block size works correctly, so the feature checked by the test is working correctly result = instance.query("SELECT count() FROM test.view WHERE _part='all_1_1_0'") From cd008f9b7fb427895ab3bcaffd163aa9746de166 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Wed, 27 May 2020 10:58:29 +0300 Subject: [PATCH 737/738] Update functions_with_hash_tables.xml --- tests/performance/functions_with_hash_tables.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/performance/functions_with_hash_tables.xml b/tests/performance/functions_with_hash_tables.xml index 72570a5925c..2e9a88107bd 100644 --- a/tests/performance/functions_with_hash_tables.xml +++ b/tests/performance/functions_with_hash_tables.xml @@ -1,6 +1,6 @@ select arrayUniq(range(1 + (number % 100) * 10)) from numbers(100000) format Null - select arrayDistinct(range(1 + (number % 100) * 10)) from numbers(10000) format Null + select arrayDistinct(range(1 + (number % 100) * 10)) from numbers(100000) format Null select arrayEnumerateUniq(range(1 + (number % 100) * 10)) from numbers(100000) format Null select arrayIntersect(range((1 + number % 100)), range(1, (1 + number % 100) + 1)) from numbers(100000) format Null select groupUniqArray(rand() % 100) from numbers(1000 * 1000) group by number / 1000 format Null From 8bfe21f10194ae05f4f1fa74409560622f5c0460 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 27 May 2020 14:09:12 +0300 Subject: [PATCH 738/738] Update 01281_group_by_limit_memory_tracking.sh --- .../queries/0_stateless/01281_group_by_limit_memory_tracking.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sh b/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sh index 79a731b9273..5922b8d74d2 100755 --- a/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sh +++ b/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sh @@ -27,6 +27,7 @@ function execute_group_by() execute_null "${opts[@]}" <<<'SELECT uniq(number) FROM numbers_mt(toUInt64(1e6)) GROUP BY number % 5e5' } +# This is needed to keep at least one running query for user for the time of test. execute_null <<<'SELECT sleep(3)' & execute_group_by # if memory accounting will be incorrect, the second query will be failed with MEMORY_LIMIT_EXCEEDED